2024-09-14T03:34:44.361 INFO:root:teuthology version: 1.2.2.dev2+g3752d38 2024-09-14T03:34:44.387 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-14T03:34:44.482 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904546 branch: main description: upgrade/reef-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - echo wait for mgr daemons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - echo wait for minority of mons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - sleep 60 - echo wait for majority of mons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - sleep 60 - echo wait for all mons to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - sleep 60 - echo wait for half of osds to upgrade - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 4 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done first-half-tasks: - workunit: branch: reef clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' first_in_suite: false job_id: '7904546' kernel: kdb: 1 sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'librbd C and C++ api tests ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi no_nested_subset: false openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: main ceph: conf: global: mon election default strategy: 1 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - mons down - mon down - MON_DOWN - out of quorum - PG_AVAILABILITY sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: main sha1: 1c941a66c1b420259b8fc5ac19e792dbbf60aded owner: scheduled_teuthology@teuthology priority: 850 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - sleep 60 - echo wait for upgrade to complete - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 8038 sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: reef clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: reef clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: reef clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 31/32 suite: upgrade suite_branch: main suite_path: /home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 1c941a66c1b420259b8fc5ac19e792dbbf60aded targets: smithi039.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBO2ArW9487sFqo+e5+3eQpfSXCY8Z+anWn+gPHQe0+YCEuSI93EbD/0eOYxyXZ70bmh3oATtKa6SocQMOpCUoWo= smithi129.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFwNIHh8g3mx9C8/NPh3iv6nnbHrc4RAgzrFyBNmZvHOxMwSKoMf8VI92MzuktJnpdY4rpA5xXbCI7flCeSYqCg= tasks: - pexec: all: - sudo dnf remove nvme-cli -y - sudo dnf install runc nvmetcli nvme-cli -y - sudo sed -i 's/^#runtime = "crun"/runtime = "runc"/g' /usr/share/containers/containers.conf - sudo sed -i 's/runtime = "crun"/#runtime = "crun"/g' /usr/share/containers/containers.conf - install: branch: reef exclude_packages: - ceph-volume - cephadm: compiled_cephadm_branch: reef conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:reef - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a timestamp: 2024-09-14_03:08:15 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.3401856 2024-09-14T03:34:44.482 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa; will attempt to use it 2024-09-14T03:34:44.483 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa/tasks 2024-09-14T03:34:44.483 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-09-14T03:34:44.485 INFO:teuthology.task.internal:Checking packages... 2024-09-14T03:34:44.506 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8' 2024-09-14T03:34:44.507 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:34:44.507 INFO:teuthology.packaging:ref: None 2024-09-14T03:34:44.507 INFO:teuthology.packaging:tag: None 2024-09-14T03:34:44.507 INFO:teuthology.packaging:branch: main 2024-09-14T03:34:44.507 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:34:44.507 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=main 2024-09-14T03:34:44.762 INFO:teuthology.task.internal:Found packages for ceph version 19.3.0-4926.g1c941a66 2024-09-14T03:34:44.763 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-09-14T03:34:44.773 INFO:teuthology.task.internal:no buildpackages task found 2024-09-14T03:34:44.773 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-09-14T03:34:44.795 INFO:teuthology.task.internal:Saving configuration 2024-09-14T03:34:44.816 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-09-14T03:34:44.825 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-09-14T03:34:44.851 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi039.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904546', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-14 03:28:51.726473', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBO2ArW9487sFqo+e5+3eQpfSXCY8Z+anWn+gPHQe0+YCEuSI93EbD/0eOYxyXZ70bmh3oATtKa6SocQMOpCUoWo='} 2024-09-14T03:34:44.872 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi129.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904546', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-09-14 03:28:51.728184', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBFwNIHh8g3mx9C8/NPh3iv6nnbHrc4RAgzrFyBNmZvHOxMwSKoMf8VI92MzuktJnpdY4rpA5xXbCI7flCeSYqCg='} 2024-09-14T03:34:44.873 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-09-14T03:34:44.880 INFO:teuthology.task.internal:roles: ubuntu@smithi039.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-09-14T03:34:44.880 INFO:teuthology.task.internal:roles: ubuntu@smithi129.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-09-14T03:34:44.880 INFO:teuthology.run_tasks:Running task console_log... 2024-09-14T03:34:44.945 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f7f5a373ac0>, signals=[15]) 2024-09-14T03:34:44.945 INFO:teuthology.run_tasks:Running task internal.connect... 2024-09-14T03:34:44.956 INFO:teuthology.task.internal:Opening connections... 2024-09-14T03:34:44.956 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi039.front.sepia.ceph.com 2024-09-14T03:34:44.958 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi039.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:34:45.032 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi129.front.sepia.ceph.com 2024-09-14T03:34:45.033 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi129.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:34:45.116 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-09-14T03:34:45.125 DEBUG:teuthology.orchestra.run.smithi039:> uname -m 2024-09-14T03:34:45.142 INFO:teuthology.orchestra.run.smithi039.stdout:x86_64 2024-09-14T03:34:45.143 DEBUG:teuthology.orchestra.run.smithi039:> cat /etc/os-release 2024-09-14T03:34:45.200 INFO:teuthology.orchestra.run.smithi039.stdout:NAME="CentOS Stream" 2024-09-14T03:34:45.200 INFO:teuthology.orchestra.run.smithi039.stdout:VERSION="9" 2024-09-14T03:34:45.200 INFO:teuthology.orchestra.run.smithi039.stdout:ID="centos" 2024-09-14T03:34:45.201 INFO:teuthology.orchestra.run.smithi039.stdout:ID_LIKE="rhel fedora" 2024-09-14T03:34:45.201 INFO:teuthology.orchestra.run.smithi039.stdout:VERSION_ID="9" 2024-09-14T03:34:45.201 INFO:teuthology.orchestra.run.smithi039.stdout:PLATFORM_ID="platform:el9" 2024-09-14T03:34:45.201 INFO:teuthology.orchestra.run.smithi039.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-14T03:34:45.201 INFO:teuthology.orchestra.run.smithi039.stdout:ANSI_COLOR="0;31" 2024-09-14T03:34:45.201 INFO:teuthology.orchestra.run.smithi039.stdout:LOGO="fedora-logo-icon" 2024-09-14T03:34:45.201 INFO:teuthology.orchestra.run.smithi039.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-14T03:34:45.201 INFO:teuthology.orchestra.run.smithi039.stdout:HOME_URL="https://centos.org/" 2024-09-14T03:34:45.201 INFO:teuthology.orchestra.run.smithi039.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-14T03:34:45.201 INFO:teuthology.orchestra.run.smithi039.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-14T03:34:45.201 INFO:teuthology.orchestra.run.smithi039.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-14T03:34:45.202 INFO:teuthology.lock.ops:Updating smithi039.front.sepia.ceph.com on lock server 2024-09-14T03:34:45.223 DEBUG:teuthology.orchestra.run.smithi129:> uname -m 2024-09-14T03:34:45.243 INFO:teuthology.orchestra.run.smithi129.stdout:x86_64 2024-09-14T03:34:45.243 DEBUG:teuthology.orchestra.run.smithi129:> cat /etc/os-release 2024-09-14T03:34:45.301 INFO:teuthology.orchestra.run.smithi129.stdout:NAME="CentOS Stream" 2024-09-14T03:34:45.301 INFO:teuthology.orchestra.run.smithi129.stdout:VERSION="9" 2024-09-14T03:34:45.301 INFO:teuthology.orchestra.run.smithi129.stdout:ID="centos" 2024-09-14T03:34:45.301 INFO:teuthology.orchestra.run.smithi129.stdout:ID_LIKE="rhel fedora" 2024-09-14T03:34:45.301 INFO:teuthology.orchestra.run.smithi129.stdout:VERSION_ID="9" 2024-09-14T03:34:45.301 INFO:teuthology.orchestra.run.smithi129.stdout:PLATFORM_ID="platform:el9" 2024-09-14T03:34:45.301 INFO:teuthology.orchestra.run.smithi129.stdout:PRETTY_NAME="CentOS Stream 9" 2024-09-14T03:34:45.301 INFO:teuthology.orchestra.run.smithi129.stdout:ANSI_COLOR="0;31" 2024-09-14T03:34:45.302 INFO:teuthology.orchestra.run.smithi129.stdout:LOGO="fedora-logo-icon" 2024-09-14T03:34:45.302 INFO:teuthology.orchestra.run.smithi129.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-09-14T03:34:45.302 INFO:teuthology.orchestra.run.smithi129.stdout:HOME_URL="https://centos.org/" 2024-09-14T03:34:45.302 INFO:teuthology.orchestra.run.smithi129.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2024-09-14T03:34:45.302 INFO:teuthology.orchestra.run.smithi129.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-09-14T03:34:45.302 INFO:teuthology.orchestra.run.smithi129.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-09-14T03:34:45.302 INFO:teuthology.lock.ops:Updating smithi129.front.sepia.ceph.com on lock server 2024-09-14T03:34:45.323 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-09-14T03:34:45.334 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-09-14T03:34:45.341 INFO:teuthology.task.internal:Checking for old test directory... 2024-09-14T03:34:45.342 DEBUG:teuthology.orchestra.run.smithi039:> test '!' -e /home/ubuntu/cephtest 2024-09-14T03:34:45.344 DEBUG:teuthology.orchestra.run.smithi129:> test '!' -e /home/ubuntu/cephtest 2024-09-14T03:34:45.360 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-09-14T03:34:45.368 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-09-14T03:34:45.368 DEBUG:teuthology.orchestra.run.smithi039:> test -z $(ls -A /var/lib/ceph) 2024-09-14T03:34:45.401 DEBUG:teuthology.orchestra.run.smithi129:> test -z $(ls -A /var/lib/ceph) 2024-09-14T03:34:45.513 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-09-14T03:34:45.569 INFO:teuthology.run_tasks:Running task kernel... 2024-09-14T03:34:45.585 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-09-14T03:34:45.585 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-09-14T03:34:45.586 DEBUG:teuthology.orchestra.run.smithi039:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-14T03:34:45.586 DEBUG:teuthology.orchestra.run.smithi129:> test -f /run/.containerenv -o -f /.dockerenv 2024-09-14T03:34:45.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T03:34:45.603 DEBUG:teuthology.orchestra.run.smithi039:> uname -r 2024-09-14T03:34:45.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T03:34:45.605 DEBUG:teuthology.orchestra.run.smithi129:> uname -r 2024-09-14T03:34:45.661 INFO:teuthology.orchestra.run.smithi039.stdout:5.14.0-503.el9.x86_64 2024-09-14T03:34:45.662 INFO:teuthology.task.kernel:Running kernel on smithi039: 5.14.0-503.el9.x86_64 2024-09-14T03:34:45.662 DEBUG:teuthology.orchestra.run.smithi039:> sudo yum install -y kernel 2024-09-14T03:34:45.663 INFO:teuthology.orchestra.run.smithi129.stdout:5.14.0-503.el9.x86_64 2024-09-14T03:34:45.663 INFO:teuthology.task.kernel:Running kernel on smithi129: 5.14.0-503.el9.x86_64 2024-09-14T03:34:45.663 DEBUG:teuthology.orchestra.run.smithi129:> sudo yum install -y kernel 2024-09-14T03:34:48.322 INFO:teuthology.orchestra.run.smithi129.stdout:CentOS Stream 9 - BaseOS 9.1 MB/s | 8.2 MB 00:00 2024-09-14T03:34:49.193 INFO:teuthology.orchestra.run.smithi039.stdout:CentOS Stream 9 - BaseOS 4.7 MB/s | 8.2 MB 00:01 2024-09-14T03:34:50.065 INFO:teuthology.orchestra.run.smithi129.stdout:CentOS Stream 9 - AppStream 33 MB/s | 20 MB 00:00 2024-09-14T03:34:51.052 INFO:teuthology.orchestra.run.smithi039.stdout:CentOS Stream 9 - AppStream 30 MB/s | 20 MB 00:00 2024-09-14T03:34:56.499 INFO:teuthology.orchestra.run.smithi129.stdout:CentOS Stream 9 - CRB 2.5 MB/s | 6.5 MB 00:02 2024-09-14T03:34:57.886 INFO:teuthology.orchestra.run.smithi039.stdout:CentOS Stream 9 - CRB 2.2 MB/s | 6.5 MB 00:02 2024-09-14T03:34:58.050 INFO:teuthology.orchestra.run.smithi129.stdout:CentOS Stream 9 - Extras packages 57 kB/s | 19 kB 00:00 2024-09-14T03:34:59.397 INFO:teuthology.orchestra.run.smithi129.stdout:Extra Packages for Enterprise Linux 18 MB/s | 23 MB 00:01 2024-09-14T03:35:00.186 INFO:teuthology.orchestra.run.smithi039.stdout:CentOS Stream 9 - Extras packages 18 kB/s | 19 kB 00:01 2024-09-14T03:35:01.140 INFO:teuthology.orchestra.run.smithi039.stdout:Extra Packages for Enterprise Linux 28 MB/s | 23 MB 00:00 2024-09-14T03:35:04.917 INFO:teuthology.orchestra.run.smithi129.stdout:lab-extras 31 kB/s | 1.7 kB 00:00 2024-09-14T03:35:05.113 INFO:teuthology.orchestra.run.smithi129.stdout:Last metadata expiration check: 0:00:01 ago on Sat 14 Sep 2024 03:35:04 AM UTC. 2024-09-14T03:35:06.763 INFO:teuthology.orchestra.run.smithi039.stdout:lab-extras 32 kB/s | 1.7 kB 00:00 2024-09-14T03:35:06.932 INFO:teuthology.orchestra.run.smithi129.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-14T03:35:06.932 INFO:teuthology.orchestra.run.smithi129.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-14T03:35:07.038 INFO:teuthology.orchestra.run.smithi129.stdout:Dependencies resolved. 2024-09-14T03:35:07.049 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:35:07.050 INFO:teuthology.orchestra.run.smithi129.stdout: Package Architecture Version Repository Size 2024-09-14T03:35:07.050 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:35:07.050 INFO:teuthology.orchestra.run.smithi129.stdout:Installing: 2024-09-14T03:35:07.050 INFO:teuthology.orchestra.run.smithi129.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-14T03:35:07.050 INFO:teuthology.orchestra.run.smithi129.stdout:Installing dependencies: 2024-09-14T03:35:07.050 INFO:teuthology.orchestra.run.smithi129.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-14T03:35:07.050 INFO:teuthology.orchestra.run.smithi129.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-14T03:35:07.050 INFO:teuthology.orchestra.run.smithi129.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-14T03:35:07.050 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:35:07.050 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction Summary 2024-09-14T03:35:07.050 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:35:07.050 INFO:teuthology.orchestra.run.smithi129.stdout:Install 4 Packages 2024-09-14T03:35:07.050 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:35:07.051 INFO:teuthology.orchestra.run.smithi129.stdout:Total download size: 79 M 2024-09-14T03:35:07.051 INFO:teuthology.orchestra.run.smithi129.stdout:Installed size: 126 M 2024-09-14T03:35:07.051 INFO:teuthology.orchestra.run.smithi129.stdout:Downloading Packages: 2024-09-14T03:35:07.394 INFO:teuthology.orchestra.run.smithi129.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 78 kB/s | 23 kB 00:00 2024-09-14T03:35:08.886 INFO:teuthology.orchestra.run.smithi039.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-09-14T03:35:08.886 INFO:teuthology.orchestra.run.smithi039.stdout:Package kernel-5.14.0-503.el9.x86_64 is already installed. 2024-09-14T03:35:08.972 INFO:teuthology.orchestra.run.smithi039.stdout:Dependencies resolved. 2024-09-14T03:35:08.976 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:35:08.976 INFO:teuthology.orchestra.run.smithi039.stdout: Package Architecture Version Repository Size 2024-09-14T03:35:08.976 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:35:08.976 INFO:teuthology.orchestra.run.smithi039.stdout:Installing: 2024-09-14T03:35:08.976 INFO:teuthology.orchestra.run.smithi039.stdout: kernel x86_64 5.14.0-505.el9 baseos 23 k 2024-09-14T03:35:08.977 INFO:teuthology.orchestra.run.smithi039.stdout:Installing dependencies: 2024-09-14T03:35:08.977 INFO:teuthology.orchestra.run.smithi039.stdout: kernel-core x86_64 5.14.0-505.el9 baseos 16 M 2024-09-14T03:35:08.977 INFO:teuthology.orchestra.run.smithi039.stdout: kernel-modules x86_64 5.14.0-505.el9 baseos 35 M 2024-09-14T03:35:08.977 INFO:teuthology.orchestra.run.smithi039.stdout: kernel-modules-core x86_64 5.14.0-505.el9 baseos 29 M 2024-09-14T03:35:08.977 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:35:08.977 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction Summary 2024-09-14T03:35:08.977 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:35:08.977 INFO:teuthology.orchestra.run.smithi039.stdout:Install 4 Packages 2024-09-14T03:35:08.977 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:35:08.978 INFO:teuthology.orchestra.run.smithi039.stdout:Total download size: 79 M 2024-09-14T03:35:08.978 INFO:teuthology.orchestra.run.smithi039.stdout:Installed size: 126 M 2024-09-14T03:35:08.978 INFO:teuthology.orchestra.run.smithi039.stdout:Downloading Packages: 2024-09-14T03:35:09.164 INFO:teuthology.orchestra.run.smithi039.stdout:(1/4): kernel-5.14.0-505.el9.x86_64.rpm 268 kB/s | 23 kB 00:00 2024-09-14T03:35:11.572 INFO:teuthology.orchestra.run.smithi039.stdout:(2/4): kernel-core-5.14.0-505.el9.x86_64.rpm 6.3 MB/s | 16 MB 00:02 2024-09-14T03:35:11.810 INFO:teuthology.orchestra.run.smithi129.stdout:(2/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 7.3 MB/s | 35 MB 00:04 2024-09-14T03:35:12.822 INFO:teuthology.orchestra.run.smithi039.stdout:(3/4): kernel-modules-core-5.14.0-505.el9.x86_6 7.8 MB/s | 29 MB 00:03 2024-09-14T03:35:13.356 INFO:teuthology.orchestra.run.smithi039.stdout:(4/4): kernel-modules-5.14.0-505.el9.x86_64.rpm 8.1 MB/s | 35 MB 00:04 2024-09-14T03:35:13.356 INFO:teuthology.orchestra.run.smithi039.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:35:13.356 INFO:teuthology.orchestra.run.smithi039.stdout:Total 18 MB/s | 79 MB 00:04 2024-09-14T03:35:14.065 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction check 2024-09-14T03:35:14.431 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction check succeeded. 2024-09-14T03:35:14.432 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction test 2024-09-14T03:35:14.993 INFO:teuthology.orchestra.run.smithi129.stdout:(3/4): kernel-core-5.14.0-505.el9.x86_64.rpm 2.0 MB/s | 16 MB 00:07 2024-09-14T03:35:15.455 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction test succeeded. 2024-09-14T03:35:15.456 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction 2024-09-14T03:35:18.003 INFO:teuthology.orchestra.run.smithi039.stdout: Preparing : 1/1 2024-09-14T03:35:18.667 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-14T03:35:18.756 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T03:35:21.557 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T03:35:21.678 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T03:35:26.814 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T03:35:26.854 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:35:32.001 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:35:41.600 INFO:teuthology.orchestra.run.smithi129.stdout:(4/4): kernel-modules-core-5.14.0-505.el9.x86_6 855 kB/s | 29 MB 00:34 2024-09-14T03:35:41.600 INFO:teuthology.orchestra.run.smithi129.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:35:41.600 INFO:teuthology.orchestra.run.smithi129.stdout:Total 2.3 MB/s | 79 MB 00:34 2024-09-14T03:35:42.283 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction check 2024-09-14T03:35:42.629 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction check succeeded. 2024-09-14T03:35:42.630 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction test 2024-09-14T03:35:43.574 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction test succeeded. 2024-09-14T03:35:43.574 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction 2024-09-14T03:35:46.042 INFO:teuthology.orchestra.run.smithi129.stdout: Preparing : 1/1 2024-09-14T03:35:46.747 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : kernel-modules-core-5.14.0-505.el9.x86_64 1/4 2024-09-14T03:35:46.824 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T03:35:49.626 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T03:35:49.750 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T03:35:54.609 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T03:35:54.736 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : kernel-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:35:59.567 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:36:11.390 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:36:11.431 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:36:11.818 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:36:11.819 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-14T03:36:11.819 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T03:36:11.819 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T03:36:12.676 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:36:12.676 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:36:12.676 INFO:teuthology.orchestra.run.smithi039.stdout:Installed: 2024-09-14T03:36:12.676 INFO:teuthology.orchestra.run.smithi039.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-14T03:36:12.676 INFO:teuthology.orchestra.run.smithi039.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-14T03:36:12.676 INFO:teuthology.orchestra.run.smithi039.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-14T03:36:12.676 INFO:teuthology.orchestra.run.smithi039.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-14T03:36:12.677 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:36:12.677 INFO:teuthology.orchestra.run.smithi039.stdout:Complete! 2024-09-14T03:36:13.033 DEBUG:teuthology.orchestra.run.smithi039:> rpm -q kernel | sort -rV | head -n 1 2024-09-14T03:36:13.131 INFO:teuthology.orchestra.run.smithi039.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-14T03:36:13.131 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-14T03:36:13.131 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-14T03:36:13.131 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-14T03:36:13.131 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-14T03:36:13.131 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-09-14T03:36:13.132 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi039.front.sepia.ceph.com, path=None, version=distro) 2024-09-14T03:36:13.132 DEBUG:teuthology.orchestra.run.smithi039:> rpm -q kernel | sort -rV | head -n 1 2024-09-14T03:36:13.191 INFO:teuthology.orchestra.run.smithi039.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-14T03:36:13.191 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-14T03:36:13.191 DEBUG:teuthology.orchestra.run.smithi039:> sudo rpm -qi grub2-tools 2024-09-14T03:36:13.229 INFO:teuthology.orchestra.run.smithi039.stdout:Name : grub2-tools 2024-09-14T03:36:13.230 INFO:teuthology.orchestra.run.smithi039.stdout:Epoch : 1 2024-09-14T03:36:13.230 INFO:teuthology.orchestra.run.smithi039.stdout:Version : 2.06 2024-09-14T03:36:13.230 INFO:teuthology.orchestra.run.smithi039.stdout:Release : 82.el9 2024-09-14T03:36:13.230 INFO:teuthology.orchestra.run.smithi039.stdout:Architecture: x86_64 2024-09-14T03:36:13.230 INFO:teuthology.orchestra.run.smithi039.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-14T03:36:13.230 INFO:teuthology.orchestra.run.smithi039.stdout:Group : Unspecified 2024-09-14T03:36:13.230 INFO:teuthology.orchestra.run.smithi039.stdout:Size : 8274670 2024-09-14T03:36:13.230 INFO:teuthology.orchestra.run.smithi039.stdout:License : GPLv3+ 2024-09-14T03:36:13.230 INFO:teuthology.orchestra.run.smithi039.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-14T03:36:13.230 INFO:teuthology.orchestra.run.smithi039.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-14T03:36:13.230 INFO:teuthology.orchestra.run.smithi039.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-14T03:36:13.230 INFO:teuthology.orchestra.run.smithi039.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-14T03:36:13.230 INFO:teuthology.orchestra.run.smithi039.stdout:Packager : builder@centos.org 2024-09-14T03:36:13.231 INFO:teuthology.orchestra.run.smithi039.stdout:Vendor : CentOS 2024-09-14T03:36:13.231 INFO:teuthology.orchestra.run.smithi039.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-14T03:36:13.231 INFO:teuthology.orchestra.run.smithi039.stdout:Summary : Support tools for GRUB. 2024-09-14T03:36:13.231 INFO:teuthology.orchestra.run.smithi039.stdout:Description : 2024-09-14T03:36:13.231 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:36:13.231 INFO:teuthology.orchestra.run.smithi039.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-14T03:36:13.231 INFO:teuthology.orchestra.run.smithi039.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-14T03:36:13.231 INFO:teuthology.orchestra.run.smithi039.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-14T03:36:13.231 INFO:teuthology.orchestra.run.smithi039.stdout:hardware devices. 2024-09-14T03:36:13.231 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:36:13.231 INFO:teuthology.orchestra.run.smithi039.stdout:This subpackage provides tools for support of all platforms. 2024-09-14T03:36:13.232 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-14T03:36:13.232 INFO:teuthology.task.kernel:Updating grub on smithi039 to boot 5.14.0-505.el9.x86_64 2024-09-14T03:36:13.232 DEBUG:teuthology.orchestra.run.smithi039:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-14T03:36:13.886 INFO:teuthology.orchestra.run.smithi039.stderr:Generating grub configuration file ... 2024-09-14T03:36:15.254 INFO:teuthology.orchestra.run.smithi039.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-14T03:36:15.301 INFO:teuthology.orchestra.run.smithi039.stderr:done 2024-09-14T03:36:15.303 DEBUG:teuthology.orchestra.run.smithi039:> mktemp 2024-09-14T03:36:15.318 INFO:teuthology.orchestra.run.smithi039.stdout:/tmp/tmp.8mpczaGB0w 2024-09-14T03:36:15.318 DEBUG:teuthology.orchestra.run.smithi039:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.8mpczaGB0w 2024-09-14T03:36:15.385 DEBUG:teuthology.orchestra.run.smithi039:> sudo chmod 0666 /tmp/tmp.8mpczaGB0w 2024-09-14T03:36:15.527 DEBUG:teuthology.orchestra.remote:smithi039:/tmp/tmp.8mpczaGB0w is 6KB 2024-09-14T03:36:15.576 DEBUG:teuthology.orchestra.run.smithi039:> rm -fr /tmp/tmp.8mpczaGB0w 2024-09-14T03:36:15.590 DEBUG:teuthology.orchestra.run.smithi039:> sudo /bin/ls /boot/loader/entries || true 2024-09-14T03:36:15.653 INFO:teuthology.orchestra.run.smithi039.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-14T03:36:15.653 INFO:teuthology.orchestra.run.smithi039.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-14T03:36:15.653 INFO:teuthology.orchestra.run.smithi039.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-14T03:36:15.653 INFO:teuthology.orchestra.run.smithi039.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-14T03:36:15.654 DEBUG:teuthology.orchestra.run.smithi039:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-14T03:36:15.808 DEBUG:teuthology.orchestra.run.smithi039:> sudo shutdown -r now 2024-09-14T03:36:37.877 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: kernel-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:36:37.917 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: kernel-modules-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:36:38.289 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: kernel-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:36:38.290 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : kernel-5.14.0-505.el9.x86_64 1/4 2024-09-14T03:36:38.290 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : kernel-core-5.14.0-505.el9.x86_64 2/4 2024-09-14T03:36:38.290 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : kernel-modules-5.14.0-505.el9.x86_64 3/4 2024-09-14T03:36:39.163 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : kernel-modules-core-5.14.0-505.el9.x86_64 4/4 2024-09-14T03:36:39.163 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:36:39.163 INFO:teuthology.orchestra.run.smithi129.stdout:Installed: 2024-09-14T03:36:39.163 INFO:teuthology.orchestra.run.smithi129.stdout: kernel-5.14.0-505.el9.x86_64 2024-09-14T03:36:39.164 INFO:teuthology.orchestra.run.smithi129.stdout: kernel-core-5.14.0-505.el9.x86_64 2024-09-14T03:36:39.164 INFO:teuthology.orchestra.run.smithi129.stdout: kernel-modules-5.14.0-505.el9.x86_64 2024-09-14T03:36:39.164 INFO:teuthology.orchestra.run.smithi129.stdout: kernel-modules-core-5.14.0-505.el9.x86_64 2024-09-14T03:36:39.164 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:36:39.164 INFO:teuthology.orchestra.run.smithi129.stdout:Complete! 2024-09-14T03:36:39.536 DEBUG:teuthology.orchestra.run.smithi129:> rpm -q kernel | sort -rV | head -n 1 2024-09-14T03:36:39.615 INFO:teuthology.orchestra.run.smithi129.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-14T03:36:39.615 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-14T03:36:39.615 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-503.el9.x86_64 Expected: 5.14.0-505.el9.x86_64 2024-09-14T03:36:39.616 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-09-14T03:36:39.616 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-09-14T03:36:39.616 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-09-14T03:36:39.616 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi129.front.sepia.ceph.com, path=None, version=distro) 2024-09-14T03:36:39.616 DEBUG:teuthology.orchestra.run.smithi129:> rpm -q kernel | sort -rV | head -n 1 2024-09-14T03:36:39.674 INFO:teuthology.orchestra.run.smithi129.stdout:kernel-5.14.0-505.el9.x86_64 2024-09-14T03:36:39.675 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-505.el9.x86_64 2024-09-14T03:36:39.675 DEBUG:teuthology.orchestra.run.smithi129:> sudo rpm -qi grub2-tools 2024-09-14T03:36:39.716 INFO:teuthology.orchestra.run.smithi129.stdout:Name : grub2-tools 2024-09-14T03:36:39.716 INFO:teuthology.orchestra.run.smithi129.stdout:Epoch : 1 2024-09-14T03:36:39.716 INFO:teuthology.orchestra.run.smithi129.stdout:Version : 2.06 2024-09-14T03:36:39.716 INFO:teuthology.orchestra.run.smithi129.stdout:Release : 82.el9 2024-09-14T03:36:39.716 INFO:teuthology.orchestra.run.smithi129.stdout:Architecture: x86_64 2024-09-14T03:36:39.716 INFO:teuthology.orchestra.run.smithi129.stdout:Install Date: Wed 28 Aug 2024 09:06:27 PM UTC 2024-09-14T03:36:39.717 INFO:teuthology.orchestra.run.smithi129.stdout:Group : Unspecified 2024-09-14T03:36:39.717 INFO:teuthology.orchestra.run.smithi129.stdout:Size : 8274670 2024-09-14T03:36:39.717 INFO:teuthology.orchestra.run.smithi129.stdout:License : GPLv3+ 2024-09-14T03:36:39.717 INFO:teuthology.orchestra.run.smithi129.stdout:Signature : RSA/SHA256, Mon 01 Jul 2024 07:46:42 AM UTC, Key ID 05b555b38483c65d 2024-09-14T03:36:39.717 INFO:teuthology.orchestra.run.smithi129.stdout:Source RPM : grub2-2.06-82.el9.src.rpm 2024-09-14T03:36:39.717 INFO:teuthology.orchestra.run.smithi129.stdout:Build Date : Fri 28 Jun 2024 09:24:44 AM UTC 2024-09-14T03:36:39.717 INFO:teuthology.orchestra.run.smithi129.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-09-14T03:36:39.717 INFO:teuthology.orchestra.run.smithi129.stdout:Packager : builder@centos.org 2024-09-14T03:36:39.717 INFO:teuthology.orchestra.run.smithi129.stdout:Vendor : CentOS 2024-09-14T03:36:39.718 INFO:teuthology.orchestra.run.smithi129.stdout:URL : http://www.gnu.org/software/grub/ 2024-09-14T03:36:39.718 INFO:teuthology.orchestra.run.smithi129.stdout:Summary : Support tools for GRUB. 2024-09-14T03:36:39.718 INFO:teuthology.orchestra.run.smithi129.stdout:Description : 2024-09-14T03:36:39.718 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:36:39.718 INFO:teuthology.orchestra.run.smithi129.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-09-14T03:36:39.718 INFO:teuthology.orchestra.run.smithi129.stdout:customizable bootloader with modular architecture. It supports a rich 2024-09-14T03:36:39.718 INFO:teuthology.orchestra.run.smithi129.stdout:variety of kernel formats, file systems, computer architectures and 2024-09-14T03:36:39.718 INFO:teuthology.orchestra.run.smithi129.stdout:hardware devices. 2024-09-14T03:36:39.718 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:36:39.718 INFO:teuthology.orchestra.run.smithi129.stdout:This subpackage provides tools for support of all platforms. 2024-09-14T03:36:39.719 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-09-14T03:36:39.720 INFO:teuthology.task.kernel:Updating grub on smithi129 to boot 5.14.0-505.el9.x86_64 2024-09-14T03:36:39.720 DEBUG:teuthology.orchestra.run.smithi129:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-09-14T03:36:40.333 INFO:teuthology.orchestra.run.smithi129.stderr:Generating grub configuration file ... 2024-09-14T03:36:41.650 INFO:teuthology.orchestra.run.smithi129.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-09-14T03:36:41.684 INFO:teuthology.orchestra.run.smithi129.stderr:done 2024-09-14T03:36:41.685 DEBUG:teuthology.orchestra.run.smithi129:> mktemp 2024-09-14T03:36:41.701 INFO:teuthology.orchestra.run.smithi129.stdout:/tmp/tmp.iKWid2g6Ro 2024-09-14T03:36:41.701 DEBUG:teuthology.orchestra.run.smithi129:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.iKWid2g6Ro 2024-09-14T03:36:41.767 DEBUG:teuthology.orchestra.run.smithi129:> sudo chmod 0666 /tmp/tmp.iKWid2g6Ro 2024-09-14T03:36:41.906 DEBUG:teuthology.orchestra.remote:smithi129:/tmp/tmp.iKWid2g6Ro is 6KB 2024-09-14T03:36:41.955 DEBUG:teuthology.orchestra.run.smithi129:> rm -fr /tmp/tmp.iKWid2g6Ro 2024-09-14T03:36:41.970 DEBUG:teuthology.orchestra.run.smithi129:> sudo /bin/ls /boot/loader/entries || true 2024-09-14T03:36:42.038 INFO:teuthology.orchestra.run.smithi129.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-0-rescue.conf 2024-09-14T03:36:42.038 INFO:teuthology.orchestra.run.smithi129.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-239.el9.x86_64.conf 2024-09-14T03:36:42.038 INFO:teuthology.orchestra.run.smithi129.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-503.el9.x86_64.conf 2024-09-14T03:36:42.038 INFO:teuthology.orchestra.run.smithi129.stdout:fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64.conf 2024-09-14T03:36:42.038 DEBUG:teuthology.orchestra.run.smithi129:> sudo grub2-set-default fbf6b6fa6cf04654a0e563f30ecbf43a-5.14.0-505.el9.x86_64 2024-09-14T03:36:42.195 DEBUG:teuthology.orchestra.run.smithi129:> sudo shutdown -r now 2024-09-14T03:36:45.813 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-09-14T03:36:45.813 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi039.front.sepia.ceph.com' 2024-09-14T03:36:45.814 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi039.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:37:04.317 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.39 2024-09-14T03:37:12.205 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-09-14T03:37:12.206 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi129.front.sepia.ceph.com' 2024-09-14T03:37:12.207 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi129.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:37:13.319 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi039.front.sepia.ceph.com' 2024-09-14T03:37:13.320 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi039.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:37:16.381 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.39 2024-09-14T03:37:28.393 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi039.front.sepia.ceph.com' 2024-09-14T03:37:28.394 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi039.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:37:30.685 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.129 2024-09-14T03:37:31.453 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.39 2024-09-14T03:37:39.693 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi129.front.sepia.ceph.com' 2024-09-14T03:37:39.694 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi129.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:37:42.749 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.129 2024-09-14T03:37:46.457 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi039.front.sepia.ceph.com' 2024-09-14T03:37:46.457 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi039.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:37:54.757 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi129.front.sepia.ceph.com' 2024-09-14T03:37:54.758 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi129.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:38:13.181 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.129 2024-09-14T03:38:28.193 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi129.front.sepia.ceph.com' 2024-09-14T03:38:28.194 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi129.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:38:28.558 DEBUG:teuthology.orchestra.run.smithi129:> true 2024-09-14T03:38:28.951 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi129.front.sepia.ceph.com' 2024-09-14T03:38:28.952 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-505.el9.x86_64"... 2024-09-14T03:38:28.952 DEBUG:teuthology.orchestra.run.smithi129:> uname -r 2024-09-14T03:38:28.966 INFO:teuthology.orchestra.run.smithi129.stdout:5.14.0-505.el9.x86_64 2024-09-14T03:38:28.966 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-14T03:38:28.967 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-14T03:38:28.967 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-14T03:38:29.967 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-09-14T03:38:29.967 DEBUG:teuthology.orchestra.run.smithi129:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-14T03:38:30.054 INFO:teuthology.orchestra.run.smithi129.stdout:ttyS1 2024-09-14T03:38:30.075 DEBUG:teuthology.parallel:result is None 2024-09-14T03:38:46.460 DEBUG:teuthology.orchestra.remote:timed out 2024-09-14T03:39:04.461 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi039.front.sepia.ceph.com' 2024-09-14T03:39:04.462 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi039.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:39:04.793 DEBUG:teuthology.orchestra.run.smithi039:> true 2024-09-14T03:39:05.169 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi039.front.sepia.ceph.com' 2024-09-14T03:39:05.169 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-505.el9.x86_64"... 2024-09-14T03:39:05.169 DEBUG:teuthology.orchestra.run.smithi039:> uname -r 2024-09-14T03:39:05.185 INFO:teuthology.orchestra.run.smithi039.stdout:5.14.0-505.el9.x86_64 2024-09-14T03:39:05.186 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-505.el9.x86_64 vs 5.14.0-505.el9.x86_64 2024-09-14T03:39:05.186 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-09-14T03:39:05.186 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-09-14T03:39:06.187 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-09-14T03:39:06.187 DEBUG:teuthology.orchestra.run.smithi039:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-09-14T03:39:06.269 INFO:teuthology.orchestra.run.smithi039.stdout:ttyS1 2024-09-14T03:39:06.289 DEBUG:teuthology.parallel:result is None 2024-09-14T03:39:06.289 INFO:teuthology.run_tasks:Running task internal.base... 2024-09-14T03:39:06.296 INFO:teuthology.task.internal:Creating test directory... 2024-09-14T03:39:06.296 DEBUG:teuthology.orchestra.run.smithi039:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-14T03:39:06.299 DEBUG:teuthology.orchestra.run.smithi129:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-09-14T03:39:06.319 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-09-14T03:39:06.325 INFO:teuthology.run_tasks:Running task internal.archive... 2024-09-14T03:39:06.331 INFO:teuthology.task.internal:Creating archive directory... 2024-09-14T03:39:06.331 DEBUG:teuthology.orchestra.run.smithi039:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-14T03:39:06.358 DEBUG:teuthology.orchestra.run.smithi129:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-09-14T03:39:06.419 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-09-14T03:39:06.426 INFO:teuthology.task.internal:Enabling coredump saving... 2024-09-14T03:39:06.426 DEBUG:teuthology.orchestra.run.smithi039:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-14T03:39:06.429 DEBUG:teuthology.orchestra.run.smithi129:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-09-14T03:39:06.470 INFO:teuthology.orchestra.run.smithi129.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-14T03:39:06.471 INFO:teuthology.orchestra.run.smithi039.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-14T03:39:06.482 INFO:teuthology.orchestra.run.smithi129.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-14T03:39:06.483 INFO:teuthology.orchestra.run.smithi039.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-09-14T03:39:06.485 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-09-14T03:39:06.493 INFO:teuthology.task.internal:Configuring sudo... 2024-09-14T03:39:06.493 DEBUG:teuthology.orchestra.run.smithi039:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-14T03:39:06.527 DEBUG:teuthology.orchestra.run.smithi129:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-09-14T03:39:06.555 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-09-14T03:39:06.564 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-09-14T03:39:06.564 DEBUG:teuthology.orchestra.run.smithi039:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-14T03:39:06.596 DEBUG:teuthology.orchestra.run.smithi129:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-09-14T03:39:06.614 DEBUG:teuthology.orchestra.run.smithi039:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-14T03:39:06.694 DEBUG:teuthology.orchestra.run.smithi039:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-14T03:39:06.743 DEBUG:teuthology.orchestra.run.smithi039:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-14T03:39:06.824 DEBUG:teuthology.orchestra.run.smithi039:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-14T03:39:06.869 DEBUG:teuthology.orchestra.run.smithi039:> set -ex 2024-09-14T03:39:06.870 DEBUG:teuthology.orchestra.run.smithi039:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-14T03:39:06.936 DEBUG:teuthology.orchestra.run.smithi129:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-14T03:39:06.978 DEBUG:teuthology.orchestra.run.smithi129:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-09-14T03:39:07.035 DEBUG:teuthology.orchestra.run.smithi129:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-14T03:39:07.114 DEBUG:teuthology.orchestra.run.smithi129:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-09-14T03:39:07.160 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T03:39:07.160 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-09-14T03:39:07.226 DEBUG:teuthology.orchestra.run.smithi039:> sudo service rsyslog restart 2024-09-14T03:39:07.229 DEBUG:teuthology.orchestra.run.smithi129:> sudo service rsyslog restart 2024-09-14T03:39:07.265 INFO:teuthology.orchestra.run.smithi039.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-14T03:39:07.318 INFO:teuthology.orchestra.run.smithi129.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-14T03:39:07.808 INFO:teuthology.run_tasks:Running task internal.timer... 2024-09-14T03:39:07.816 INFO:teuthology.task.internal:Starting timer... 2024-09-14T03:39:07.816 INFO:teuthology.run_tasks:Running task pcp... 2024-09-14T03:39:07.827 INFO:teuthology.run_tasks:Running task selinux... 2024-09-14T03:39:07.837 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-09-14T03:39:07.838 DEBUG:teuthology.orchestra.run.smithi039:> sudo service auditd rotate 2024-09-14T03:39:07.899 INFO:teuthology.orchestra.run.smithi039.stdout:Rotating logs: 2024-09-14T03:39:07.901 DEBUG:teuthology.orchestra.run.smithi129:> sudo service auditd rotate 2024-09-14T03:39:07.978 INFO:teuthology.orchestra.run.smithi129.stdout:Rotating logs: 2024-09-14T03:39:07.980 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-09-14T03:39:07.980 DEBUG:teuthology.orchestra.run.smithi039:> /usr/sbin/getenforce 2024-09-14T03:39:08.004 INFO:teuthology.orchestra.run.smithi039.stdout:Permissive 2024-09-14T03:39:08.004 DEBUG:teuthology.orchestra.run.smithi129:> /usr/sbin/getenforce 2024-09-14T03:39:08.046 INFO:teuthology.orchestra.run.smithi129.stdout:Permissive 2024-09-14T03:39:08.046 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi039.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi129.front.sepia.ceph.com': 'permissive'} 2024-09-14T03:39:08.046 DEBUG:teuthology.orchestra.run.smithi039:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-14T03:39:08.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T03:39:08.077 DEBUG:teuthology.orchestra.run.smithi129:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-14T03:39:08.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T03:39:08.113 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-09-14T03:39:08.113 DEBUG:teuthology.orchestra.run.smithi039:> sudo /usr/sbin/setenforce permissive 2024-09-14T03:39:08.151 DEBUG:teuthology.orchestra.run.smithi129:> sudo /usr/sbin/setenforce permissive 2024-09-14T03:39:08.198 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-09-14T03:39:08.207 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-09-14T03:39:08.220 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-09-14T03:39:08.222 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi039.front.sepia.ceph.com,smithi129.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-09-14T03:42:55.290 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi039.front.sepia.ceph.com'), Remote(name='ubuntu@smithi129.front.sepia.ceph.com')] 2024-09-14T03:42:55.291 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi039.front.sepia.ceph.com' 2024-09-14T03:42:55.293 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi039.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:42:55.385 DEBUG:teuthology.orchestra.run.smithi039:> true 2024-09-14T03:42:55.465 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi039.front.sepia.ceph.com' 2024-09-14T03:42:55.465 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi129.front.sepia.ceph.com' 2024-09-14T03:42:55.466 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi129.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-09-14T03:42:55.562 DEBUG:teuthology.orchestra.run.smithi129:> true 2024-09-14T03:42:55.639 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi129.front.sepia.ceph.com' 2024-09-14T03:42:55.639 INFO:teuthology.run_tasks:Running task clock... 2024-09-14T03:42:55.654 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-09-14T03:42:55.654 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-14T03:42:55.655 DEBUG:teuthology.orchestra.run.smithi039:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-14T03:42:55.658 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-09-14T03:42:55.658 DEBUG:teuthology.orchestra.run.smithi129:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-14T03:42:55.685 INFO:teuthology.orchestra.run.smithi039.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-14T03:42:55.700 INFO:teuthology.orchestra.run.smithi039.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-14T03:42:55.713 INFO:teuthology.orchestra.run.smithi129.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-09-14T03:42:55.725 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: ntpd: command not found 2024-09-14T03:42:55.730 INFO:teuthology.orchestra.run.smithi129.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-09-14T03:42:55.737 INFO:teuthology.orchestra.run.smithi039.stdout:506 Cannot talk to daemon 2024-09-14T03:42:55.752 INFO:teuthology.orchestra.run.smithi039.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-14T03:42:55.756 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: ntpd: command not found 2024-09-14T03:42:55.766 INFO:teuthology.orchestra.run.smithi039.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-14T03:42:55.768 INFO:teuthology.orchestra.run.smithi129.stdout:506 Cannot talk to daemon 2024-09-14T03:42:55.783 INFO:teuthology.orchestra.run.smithi129.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-09-14T03:42:55.797 INFO:teuthology.orchestra.run.smithi129.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-09-14T03:42:55.824 INFO:teuthology.orchestra.run.smithi039.stderr:bash: line 1: ntpq: command not found 2024-09-14T03:42:55.827 INFO:teuthology.orchestra.run.smithi039.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-14T03:42:55.827 INFO:teuthology.orchestra.run.smithi039.stdout:=============================================================================== 2024-09-14T03:42:55.827 INFO:teuthology.orchestra.run.smithi039.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:42:55.827 INFO:teuthology.orchestra.run.smithi039.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:42:55.827 INFO:teuthology.orchestra.run.smithi039.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:42:55.827 INFO:teuthology.orchestra.run.smithi039.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:42:55.855 INFO:teuthology.orchestra.run.smithi129.stderr:bash: line 1: ntpq: command not found 2024-09-14T03:42:55.858 INFO:teuthology.orchestra.run.smithi129.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-14T03:42:55.859 INFO:teuthology.orchestra.run.smithi129.stdout:=============================================================================== 2024-09-14T03:42:55.859 INFO:teuthology.orchestra.run.smithi129.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:42:55.859 INFO:teuthology.orchestra.run.smithi129.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:42:55.859 INFO:teuthology.orchestra.run.smithi129.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:42:55.859 INFO:teuthology.orchestra.run.smithi129.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T03:42:55.860 INFO:teuthology.run_tasks:Running task pexec... 2024-09-14T03:42:55.872 INFO:teuthology.task.pexec:Executing custom commands... 2024-09-14T03:42:55.872 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi039.front.sepia.ceph.com 2024-09-14T03:42:55.873 DEBUG:teuthology.orchestra.run.smithi039:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-09-14T03:42:55.873 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi129.front.sepia.ceph.com 2024-09-14T03:42:55.874 DEBUG:teuthology.orchestra.run.smithi129:> TESTDIR=/home/ubuntu/cephtest bash -s 2024-09-14T03:42:56.242 INFO:teuthology.orchestra.run.smithi039.stdout:Dependencies resolved. 2024-09-14T03:42:56.249 INFO:teuthology.orchestra.run.smithi129.stdout:Dependencies resolved. 2024-09-14T03:42:56.257 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:42:56.257 INFO:teuthology.orchestra.run.smithi039.stdout: Package Architecture Version Repository Size 2024-09-14T03:42:56.258 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:42:56.258 INFO:teuthology.orchestra.run.smithi039.stdout:Removing: 2024-09-14T03:42:56.258 INFO:teuthology.orchestra.run.smithi039.stdout: nvme-cli x86_64 2.9.1-6.el9 @baseos 5.3 M 2024-09-14T03:42:56.258 INFO:teuthology.orchestra.run.smithi039.stdout:Removing unused dependencies: 2024-09-14T03:42:56.258 INFO:teuthology.orchestra.run.smithi039.stdout: libnvme x86_64 1.9-3.el9 @baseos 273 k 2024-09-14T03:42:56.258 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:42:56.258 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction Summary 2024-09-14T03:42:56.258 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:42:56.258 INFO:teuthology.orchestra.run.smithi039.stdout:Remove 2 Packages 2024-09-14T03:42:56.258 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:42:56.258 INFO:teuthology.orchestra.run.smithi039.stdout:Freed space: 5.5 M 2024-09-14T03:42:56.258 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction check 2024-09-14T03:42:56.260 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:42:56.261 INFO:teuthology.orchestra.run.smithi129.stdout: Package Architecture Version Repository Size 2024-09-14T03:42:56.261 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:42:56.261 INFO:teuthology.orchestra.run.smithi129.stdout:Removing: 2024-09-14T03:42:56.261 INFO:teuthology.orchestra.run.smithi129.stdout: nvme-cli x86_64 2.9.1-6.el9 @baseos 5.3 M 2024-09-14T03:42:56.261 INFO:teuthology.orchestra.run.smithi129.stdout:Removing unused dependencies: 2024-09-14T03:42:56.261 INFO:teuthology.orchestra.run.smithi129.stdout: libnvme x86_64 1.9-3.el9 @baseos 273 k 2024-09-14T03:42:56.261 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:42:56.261 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction Summary 2024-09-14T03:42:56.261 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:42:56.261 INFO:teuthology.orchestra.run.smithi129.stdout:Remove 2 Packages 2024-09-14T03:42:56.261 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:42:56.261 INFO:teuthology.orchestra.run.smithi129.stdout:Freed space: 5.5 M 2024-09-14T03:42:56.261 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction check 2024-09-14T03:42:56.262 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction check succeeded. 2024-09-14T03:42:56.262 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction test 2024-09-14T03:42:56.265 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction check succeeded. 2024-09-14T03:42:56.265 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction test 2024-09-14T03:42:56.319 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction test succeeded. 2024-09-14T03:42:56.320 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction 2024-09-14T03:42:56.320 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction test succeeded. 2024-09-14T03:42:56.320 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction 2024-09-14T03:42:56.453 INFO:teuthology.orchestra.run.smithi129.stdout: Preparing : 1/1 2024-09-14T03:42:56.469 INFO:teuthology.orchestra.run.smithi039.stdout: Preparing : 1/1 2024-09-14T03:42:56.540 INFO:teuthology.orchestra.run.smithi039.stdout: Erasing : nvme-cli-2.9.1-6.el9.x86_64 1/2 2024-09-14T03:42:56.543 INFO:teuthology.orchestra.run.smithi129.stdout: Erasing : nvme-cli-2.9.1-6.el9.x86_64 1/2 2024-09-14T03:42:56.604 INFO:teuthology.orchestra.run.smithi129.stdout: Erasing : libnvme-1.9-3.el9.x86_64 2/2 2024-09-14T03:42:56.604 INFO:teuthology.orchestra.run.smithi039.stdout: Erasing : libnvme-1.9-3.el9.x86_64 2/2 2024-09-14T03:42:56.971 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: libnvme-1.9-3.el9.x86_64 2/2 2024-09-14T03:42:56.972 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/2 2024-09-14T03:42:57.109 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: libnvme-1.9-3.el9.x86_64 2/2 2024-09-14T03:42:57.109 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/2 2024-09-14T03:42:57.219 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/2 2024-09-14T03:42:57.220 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:42:57.220 INFO:teuthology.orchestra.run.smithi129.stdout:Removed: 2024-09-14T03:42:57.220 INFO:teuthology.orchestra.run.smithi129.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-14T03:42:57.220 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:42:57.220 INFO:teuthology.orchestra.run.smithi129.stdout:Complete! 2024-09-14T03:42:57.372 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/2 2024-09-14T03:42:57.372 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:42:57.372 INFO:teuthology.orchestra.run.smithi039.stdout:Removed: 2024-09-14T03:42:57.372 INFO:teuthology.orchestra.run.smithi039.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-14T03:42:57.372 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:42:57.372 INFO:teuthology.orchestra.run.smithi039.stdout:Complete! 2024-09-14T03:42:58.005 INFO:teuthology.orchestra.run.smithi129.stdout:Last metadata expiration check: 0:00:51 ago on Sat 14 Sep 2024 03:42:06 AM UTC. 2024-09-14T03:42:58.158 INFO:teuthology.orchestra.run.smithi129.stdout:Dependencies resolved. 2024-09-14T03:42:58.159 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:42:58.159 INFO:teuthology.orchestra.run.smithi129.stdout: Package Arch Version Repository Size 2024-09-14T03:42:58.159 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:42:58.159 INFO:teuthology.orchestra.run.smithi129.stdout:Installing: 2024-09-14T03:42:58.159 INFO:teuthology.orchestra.run.smithi129.stdout: nvme-cli x86_64 2.9.1-6.el9 baseos 906 k 2024-09-14T03:42:58.159 INFO:teuthology.orchestra.run.smithi129.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-09-14T03:42:58.160 INFO:teuthology.orchestra.run.smithi129.stdout: runc x86_64 4:1.1.13-4.el9 appstream 3.2 M 2024-09-14T03:42:58.160 INFO:teuthology.orchestra.run.smithi129.stdout:Installing dependencies: 2024-09-14T03:42:58.160 INFO:teuthology.orchestra.run.smithi129.stdout: libnvme x86_64 1.9-3.el9 baseos 106 k 2024-09-14T03:42:58.160 INFO:teuthology.orchestra.run.smithi129.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-09-14T03:42:58.160 INFO:teuthology.orchestra.run.smithi129.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-09-14T03:42:58.160 INFO:teuthology.orchestra.run.smithi129.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-09-14T03:42:58.160 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:42:58.160 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction Summary 2024-09-14T03:42:58.160 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:42:58.160 INFO:teuthology.orchestra.run.smithi129.stdout:Install 7 Packages 2024-09-14T03:42:58.160 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:42:58.172 INFO:teuthology.orchestra.run.smithi129.stdout:Total download size: 5.2 M 2024-09-14T03:42:58.172 INFO:teuthology.orchestra.run.smithi129.stdout:Installed size: 19 M 2024-09-14T03:42:58.173 INFO:teuthology.orchestra.run.smithi129.stdout:Downloading Packages: 2024-09-14T03:42:58.197 INFO:teuthology.orchestra.run.smithi039.stdout:Last metadata expiration check: 0:00:46 ago on Sat 14 Sep 2024 03:42:11 AM UTC. 2024-09-14T03:42:58.335 INFO:teuthology.orchestra.run.smithi039.stdout:Dependencies resolved. 2024-09-14T03:42:58.335 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:42:58.336 INFO:teuthology.orchestra.run.smithi039.stdout: Package Arch Version Repository Size 2024-09-14T03:42:58.336 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:42:58.336 INFO:teuthology.orchestra.run.smithi039.stdout:Installing: 2024-09-14T03:42:58.336 INFO:teuthology.orchestra.run.smithi039.stdout: nvme-cli x86_64 2.9.1-6.el9 baseos 906 k 2024-09-14T03:42:58.336 INFO:teuthology.orchestra.run.smithi039.stdout: nvmetcli noarch 0.7-3.el9 baseos 44 k 2024-09-14T03:42:58.336 INFO:teuthology.orchestra.run.smithi039.stdout: runc x86_64 4:1.1.13-4.el9 appstream 3.2 M 2024-09-14T03:42:58.336 INFO:teuthology.orchestra.run.smithi039.stdout:Installing dependencies: 2024-09-14T03:42:58.336 INFO:teuthology.orchestra.run.smithi039.stdout: libnvme x86_64 1.9-3.el9 baseos 106 k 2024-09-14T03:42:58.336 INFO:teuthology.orchestra.run.smithi039.stdout: python3-configshell noarch 1:1.1.30-1.el9 baseos 72 k 2024-09-14T03:42:58.336 INFO:teuthology.orchestra.run.smithi039.stdout: python3-kmod x86_64 0.9-32.el9 baseos 84 k 2024-09-14T03:42:58.336 INFO:teuthology.orchestra.run.smithi039.stdout: python3-urwid x86_64 2.1.2-4.el9 baseos 837 k 2024-09-14T03:42:58.336 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:42:58.337 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction Summary 2024-09-14T03:42:58.337 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:42:58.337 INFO:teuthology.orchestra.run.smithi039.stdout:Install 7 Packages 2024-09-14T03:42:58.337 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:42:58.348 INFO:teuthology.orchestra.run.smithi039.stdout:Total download size: 5.2 M 2024-09-14T03:42:58.348 INFO:teuthology.orchestra.run.smithi039.stdout:Installed size: 19 M 2024-09-14T03:42:58.348 INFO:teuthology.orchestra.run.smithi039.stdout:Downloading Packages: 2024-09-14T03:42:58.383 INFO:teuthology.orchestra.run.smithi129.stdout:(1/7): nvmetcli-0.7-3.el9.noarch.rpm 315 kB/s | 44 kB 00:00 2024-09-14T03:42:58.408 INFO:teuthology.orchestra.run.smithi129.stdout:(2/7): libnvme-1.9-3.el9.x86_64.rpm 638 kB/s | 106 kB 00:00 2024-09-14T03:42:58.459 INFO:teuthology.orchestra.run.smithi129.stdout:(3/7): python3-kmod-0.9-32.el9.x86_64.rpm 1.6 MB/s | 84 kB 00:00 2024-09-14T03:42:58.501 INFO:teuthology.orchestra.run.smithi129.stdout:(4/7): nvme-cli-2.9.1-6.el9.x86_64.rpm 3.4 MB/s | 906 kB 00:00 2024-09-14T03:42:58.526 INFO:teuthology.orchestra.run.smithi129.stdout:(5/7): python3-configshell-1.1.30-1.el9.noarch. 503 kB/s | 72 kB 00:00 2024-09-14T03:42:58.584 INFO:teuthology.orchestra.run.smithi129.stdout:(6/7): python3-urwid-2.1.2-4.el9.x86_64.rpm 6.5 MB/s | 837 kB 00:00 2024-09-14T03:42:58.801 INFO:teuthology.orchestra.run.smithi129.stdout:(7/7): runc-1.1.13-4.el9.x86_64.rpm 11 MB/s | 3.2 MB 00:00 2024-09-14T03:42:58.802 INFO:teuthology.orchestra.run.smithi129.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:42:58.802 INFO:teuthology.orchestra.run.smithi129.stdout:Total 8.2 MB/s | 5.2 MB 00:00 2024-09-14T03:42:58.910 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction check 2024-09-14T03:42:58.924 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction check succeeded. 2024-09-14T03:42:58.924 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction test 2024-09-14T03:42:58.991 INFO:teuthology.orchestra.run.smithi039.stdout:(1/7): nvmetcli-0.7-3.el9.noarch.rpm 438 kB/s | 44 kB 00:00 2024-09-14T03:42:59.017 INFO:teuthology.orchestra.run.smithi039.stdout:(2/7): libnvme-1.9-3.el9.x86_64.rpm 835 kB/s | 106 kB 00:00 2024-09-14T03:42:59.050 INFO:teuthology.orchestra.run.smithi039.stdout:(3/7): python3-configshell-1.1.30-1.el9.noarch. 1.2 MB/s | 72 kB 00:00 2024-09-14T03:42:59.076 INFO:teuthology.orchestra.run.smithi039.stdout:(4/7): python3-kmod-0.9-32.el9.x86_64.rpm 1.4 MB/s | 84 kB 00:00 2024-09-14T03:42:59.118 INFO:teuthology.orchestra.run.smithi039.stdout:(5/7): nvme-cli-2.9.1-6.el9.x86_64.rpm 3.9 MB/s | 906 kB 00:00 2024-09-14T03:42:59.168 INFO:teuthology.orchestra.run.smithi039.stdout:(6/7): python3-urwid-2.1.2-4.el9.x86_64.rpm 7.0 MB/s | 837 kB 00:00 2024-09-14T03:42:59.241 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction test succeeded. 2024-09-14T03:42:59.242 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction 2024-09-14T03:42:59.720 INFO:teuthology.orchestra.run.smithi129.stdout: Preparing : 1/1 2024-09-14T03:42:59.798 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 1/7 2024-09-14T03:42:59.832 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 2/7 2024-09-14T03:42:59.861 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 3/7 2024-09-14T03:43:00.043 INFO:teuthology.orchestra.run.smithi039.stdout:(7/7): runc-1.1.13-4.el9.x86_64.rpm 3.3 MB/s | 3.2 MB 00:00 2024-09-14T03:43:00.043 INFO:teuthology.orchestra.run.smithi039.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:43:00.044 INFO:teuthology.orchestra.run.smithi039.stdout:Total 3.0 MB/s | 5.2 MB 00:01 2024-09-14T03:43:00.075 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : libnvme-1.9-3.el9.x86_64 4/7 2024-09-14T03:43:00.109 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-14T03:43:00.165 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction check 2024-09-14T03:43:00.182 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction check succeeded. 2024-09-14T03:43:00.182 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction test 2024-09-14T03:43:00.520 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction test succeeded. 2024-09-14T03:43:00.520 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction 2024-09-14T03:43:00.554 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-14T03:43:00.581 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : nvmetcli-0.7-3.el9.noarch 6/7 2024-09-14T03:43:00.694 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 6/7 2024-09-14T03:43:00.757 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-14T03:43:01.022 INFO:teuthology.orchestra.run.smithi039.stdout: Preparing : 1/1 2024-09-14T03:43:01.122 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-urwid-2.1.2-4.el9.x86_64 1/7 2024-09-14T03:43:01.156 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-configshell-1:1.1.30-1.el9.noarch 2/7 2024-09-14T03:43:01.184 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-kmod-0.9-32.el9.x86_64 3/7 2024-09-14T03:43:01.404 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : libnvme-1.9-3.el9.x86_64 4/7 2024-09-14T03:43:01.455 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-14T03:43:01.761 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-14T03:43:01.761 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/7 2024-09-14T03:43:01.762 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/7 2024-09-14T03:43:01.762 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 3/7 2024-09-14T03:43:01.762 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 4/7 2024-09-14T03:43:01.762 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 5/7 2024-09-14T03:43:01.762 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 6/7 2024-09-14T03:43:01.990 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: nvme-cli-2.9.1-6.el9.x86_64 5/7 2024-09-14T03:43:02.029 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : nvmetcli-0.7-3.el9.noarch 6/7 2024-09-14T03:43:02.130 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-14T03:43:02.130 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:43:02.130 INFO:teuthology.orchestra.run.smithi129.stdout:Installed: 2024-09-14T03:43:02.130 INFO:teuthology.orchestra.run.smithi129.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-14T03:43:02.130 INFO:teuthology.orchestra.run.smithi129.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-09-14T03:43:02.131 INFO:teuthology.orchestra.run.smithi129.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-09-14T03:43:02.131 INFO:teuthology.orchestra.run.smithi129.stdout: runc-4:1.1.13-4.el9.x86_64 2024-09-14T03:43:02.131 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:43:02.131 INFO:teuthology.orchestra.run.smithi129.stdout:Complete! 2024-09-14T03:43:02.150 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: nvmetcli-0.7-3.el9.noarch 6/7 2024-09-14T03:43:02.225 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-14T03:43:02.538 DEBUG:teuthology.parallel:result is None 2024-09-14T03:43:03.164 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-14T03:43:03.165 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : libnvme-1.9-3.el9.x86_64 1/7 2024-09-14T03:43:03.165 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : nvme-cli-2.9.1-6.el9.x86_64 2/7 2024-09-14T03:43:03.165 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : nvmetcli-0.7-3.el9.noarch 3/7 2024-09-14T03:43:03.165 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-configshell-1:1.1.30-1.el9.noarch 4/7 2024-09-14T03:43:03.165 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-kmod-0.9-32.el9.x86_64 5/7 2024-09-14T03:43:03.165 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-urwid-2.1.2-4.el9.x86_64 6/7 2024-09-14T03:43:03.578 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : runc-4:1.1.13-4.el9.x86_64 7/7 2024-09-14T03:43:03.578 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:43:03.578 INFO:teuthology.orchestra.run.smithi039.stdout:Installed: 2024-09-14T03:43:03.578 INFO:teuthology.orchestra.run.smithi039.stdout: libnvme-1.9-3.el9.x86_64 nvme-cli-2.9.1-6.el9.x86_64 2024-09-14T03:43:03.578 INFO:teuthology.orchestra.run.smithi039.stdout: nvmetcli-0.7-3.el9.noarch python3-configshell-1:1.1.30-1.el9.noarch 2024-09-14T03:43:03.578 INFO:teuthology.orchestra.run.smithi039.stdout: python3-kmod-0.9-32.el9.x86_64 python3-urwid-2.1.2-4.el9.x86_64 2024-09-14T03:43:03.578 INFO:teuthology.orchestra.run.smithi039.stdout: runc-4:1.1.13-4.el9.x86_64 2024-09-14T03:43:03.578 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:43:03.578 INFO:teuthology.orchestra.run.smithi039.stdout:Complete! 2024-09-14T03:43:04.110 DEBUG:teuthology.parallel:result is None 2024-09-14T03:43:04.111 INFO:teuthology.run_tasks:Running task install... 2024-09-14T03:43:04.119 DEBUG:teuthology.task.install:project ceph 2024-09-14T03:43:04.119 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8'}} 2024-09-14T03:43:04.119 DEBUG:teuthology.task.install:config {'branch': 'reef', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8'} 2024-09-14T03:43:04.120 INFO:teuthology.task.install:Using flavor: default 2024-09-14T03:43:04.125 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-09-14T03:43:04.125 INFO:teuthology.task.install:extra packages: [] 2024-09-14T03:43:04.126 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'reef', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8', 'tag': None, 'wait_for_package': False} 2024-09-14T03:43:04.126 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:43:04.126 INFO:teuthology.packaging:ref: None 2024-09-14T03:43:04.126 INFO:teuthology.packaging:tag: None 2024-09-14T03:43:04.127 INFO:teuthology.packaging:branch: reef 2024-09-14T03:43:04.127 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:43:04.127 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-09-14T03:43:04.129 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'reef', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8', 'tag': None, 'wait_for_package': False} 2024-09-14T03:43:04.129 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:43:04.129 INFO:teuthology.packaging:ref: None 2024-09-14T03:43:04.129 INFO:teuthology.packaging:tag: None 2024-09-14T03:43:04.129 INFO:teuthology.packaging:branch: reef 2024-09-14T03:43:04.129 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:43:04.130 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-09-14T03:43:04.365 INFO:teuthology.task.install.rpm:Pulling from https://1.chacra.ceph.com/r/ceph/reef/e4e227b37858f5109138494e01e5ba1ba7b8ffe8/centos/9/flavors/default/ 2024-09-14T03:43:04.365 INFO:teuthology.task.install.rpm:Package version is 18.2.4-845.ge4e227b3 2024-09-14T03:43:04.398 INFO:teuthology.task.install.rpm:Pulling from https://1.chacra.ceph.com/r/ceph/reef/e4e227b37858f5109138494e01e5ba1ba7b8ffe8/centos/9/flavors/default/ 2024-09-14T03:43:04.398 INFO:teuthology.task.install.rpm:Package version is 18.2.4-845.ge4e227b3 2024-09-14T03:43:04.476 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://1.chacra.ceph.com/r/ceph/reef/e4e227b37858f5109138494e01e5ba1ba7b8ffe8/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://1.chacra.ceph.com/r/ceph/reef/e4e227b37858f5109138494e01e5ba1ba7b8ffe8/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://1.chacra.ceph.com/r/ceph/reef/e4e227b37858f5109138494e01e5ba1ba7b8ffe8/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-14T03:43:04.476 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T03:43:04.476 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-14T03:43:04.506 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://1.chacra.ceph.com/r/ceph/reef/e4e227b37858f5109138494e01e5ba1ba7b8ffe8/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://1.chacra.ceph.com/r/ceph/reef/e4e227b37858f5109138494e01e5ba1ba7b8ffe8/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://1.chacra.ceph.com/r/ceph/reef/e4e227b37858f5109138494e01e5ba1ba7b8ffe8/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-09-14T03:43:04.507 DEBUG:teuthology.orchestra.run.smithi039:> set -ex 2024-09-14T03:43:04.507 DEBUG:teuthology.orchestra.run.smithi039:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-09-14T03:43:04.508 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-09-14T03:43:04.508 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:43:04.508 INFO:teuthology.packaging:ref: None 2024-09-14T03:43:04.508 INFO:teuthology.packaging:tag: None 2024-09-14T03:43:04.508 INFO:teuthology.packaging:branch: reef 2024-09-14T03:43:04.508 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:43:04.508 DEBUG:teuthology.orchestra.run.smithi129:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/reef/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-14T03:43:04.540 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-09-14T03:43:04.540 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:43:04.540 INFO:teuthology.packaging:ref: None 2024-09-14T03:43:04.540 INFO:teuthology.packaging:tag: None 2024-09-14T03:43:04.540 INFO:teuthology.packaging:branch: reef 2024-09-14T03:43:04.540 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:43:04.540 DEBUG:teuthology.orchestra.run.smithi039:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/reef/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-09-14T03:43:04.579 DEBUG:teuthology.orchestra.run.smithi129:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-14T03:43:04.617 DEBUG:teuthology.orchestra.run.smithi039:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-09-14T03:43:04.659 DEBUG:teuthology.orchestra.run.smithi129:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-14T03:43:04.698 DEBUG:teuthology.orchestra.run.smithi039:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-09-14T03:43:04.726 INFO:teuthology.orchestra.run.smithi129.stdout:check_obsoletes = 1 2024-09-14T03:43:04.728 DEBUG:teuthology.orchestra.run.smithi129:> sudo yum clean all 2024-09-14T03:43:04.766 INFO:teuthology.orchestra.run.smithi039.stdout:check_obsoletes = 1 2024-09-14T03:43:04.767 DEBUG:teuthology.orchestra.run.smithi039:> sudo yum clean all 2024-09-14T03:43:04.968 INFO:teuthology.orchestra.run.smithi129.stdout:45 files removed 2024-09-14T03:43:04.995 DEBUG:teuthology.orchestra.run.smithi129:> sudo yum -y install ceph-radosgw 2024-09-14T03:43:05.009 INFO:teuthology.orchestra.run.smithi039.stdout:45 files removed 2024-09-14T03:43:05.036 DEBUG:teuthology.orchestra.run.smithi039:> sudo yum -y install ceph-radosgw 2024-09-14T03:43:05.545 INFO:teuthology.orchestra.run.smithi129.stdout:ceph packages for x86_64 290 kB/s | 77 kB 00:00 2024-09-14T03:43:05.608 INFO:teuthology.orchestra.run.smithi039.stdout:ceph packages for x86_64 284 kB/s | 77 kB 00:00 2024-09-14T03:43:05.846 INFO:teuthology.orchestra.run.smithi129.stdout:ceph noarch packages 44 kB/s | 11 kB 00:00 2024-09-14T03:43:05.984 INFO:teuthology.orchestra.run.smithi039.stdout:ceph noarch packages 35 kB/s | 11 kB 00:00 2024-09-14T03:43:06.113 INFO:teuthology.orchestra.run.smithi129.stdout:ceph source packages 8.1 kB/s | 1.9 kB 00:00 2024-09-14T03:43:06.259 INFO:teuthology.orchestra.run.smithi039.stdout:ceph source packages 7.9 kB/s | 1.9 kB 00:00 2024-09-14T03:43:06.614 INFO:teuthology.orchestra.run.smithi129.stdout:CentOS Stream 9 - BaseOS 18 MB/s | 8.2 MB 00:00 2024-09-14T03:43:06.826 INFO:teuthology.orchestra.run.smithi039.stdout:CentOS Stream 9 - BaseOS 16 MB/s | 8.2 MB 00:00 2024-09-14T03:43:09.957 INFO:teuthology.orchestra.run.smithi129.stdout:CentOS Stream 9 - AppStream 9.1 MB/s | 20 MB 00:02 2024-09-14T03:43:11.894 INFO:teuthology.orchestra.run.smithi039.stdout:CentOS Stream 9 - AppStream 5.2 MB/s | 20 MB 00:03 2024-09-14T03:43:14.366 INFO:teuthology.orchestra.run.smithi129.stdout:CentOS Stream 9 - CRB 11 MB/s | 6.5 MB 00:00 2024-09-14T03:43:15.875 INFO:teuthology.orchestra.run.smithi129.stdout:CentOS Stream 9 - Extras packages 68 kB/s | 19 kB 00:00 2024-09-14T03:43:16.445 INFO:teuthology.orchestra.run.smithi039.stdout:CentOS Stream 9 - CRB 12 MB/s | 6.5 MB 00:00 2024-09-14T03:43:16.888 INFO:teuthology.orchestra.run.smithi129.stdout:Extra Packages for Enterprise Linux 25 MB/s | 23 MB 00:00 2024-09-14T03:43:17.954 INFO:teuthology.orchestra.run.smithi039.stdout:CentOS Stream 9 - Extras packages 78 kB/s | 19 kB 00:00 2024-09-14T03:43:19.091 INFO:teuthology.orchestra.run.smithi039.stdout:Extra Packages for Enterprise Linux 22 MB/s | 23 MB 00:01 2024-09-14T03:43:22.302 INFO:teuthology.orchestra.run.smithi129.stdout:lab-extras 32 kB/s | 1.7 kB 00:00 2024-09-14T03:43:24.712 INFO:teuthology.orchestra.run.smithi129.stdout:Dependencies resolved. 2024-09-14T03:43:24.713 INFO:teuthology.orchestra.run.smithi039.stdout:lab-extras 31 kB/s | 1.7 kB 00:00 2024-09-14T03:43:24.714 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:43:24.714 INFO:teuthology.orchestra.run.smithi129.stdout: Package Arch Version Repository Size 2024-09-14T03:43:24.714 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:43:24.714 INFO:teuthology.orchestra.run.smithi129.stdout:Installing: 2024-09-14T03:43:24.714 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-radosgw x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 7.7 M 2024-09-14T03:43:24.714 INFO:teuthology.orchestra.run.smithi129.stdout:Upgrading: 2024-09-14T03:43:24.714 INFO:teuthology.orchestra.run.smithi129.stdout: librados2 x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 3.3 M 2024-09-14T03:43:24.714 INFO:teuthology.orchestra.run.smithi129.stdout: librbd1 x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 3.0 M 2024-09-14T03:43:24.714 INFO:teuthology.orchestra.run.smithi129.stdout:Installing dependencies: 2024-09-14T03:43:24.715 INFO:teuthology.orchestra.run.smithi129.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-09-14T03:43:24.715 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-base x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 5.1 M 2024-09-14T03:43:24.715 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-common x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 18 M 2024-09-14T03:43:24.715 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-selinux x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 25 k 2024-09-14T03:43:24.715 INFO:teuthology.orchestra.run.smithi129.stdout: fmt x86_64 8.1.1-5.el9 epel 111 k 2024-09-14T03:43:24.715 INFO:teuthology.orchestra.run.smithi129.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-14T03:43:24.715 INFO:teuthology.orchestra.run.smithi129.stdout: libarrow x86_64 9.0.0-11.el9 epel 4.4 M 2024-09-14T03:43:24.715 INFO:teuthology.orchestra.run.smithi129.stdout: libarrow-doc noarch 9.0.0-11.el9 epel 26 k 2024-09-14T03:43:24.715 INFO:teuthology.orchestra.run.smithi129.stdout: libcephfs2 x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 712 k 2024-09-14T03:43:24.715 INFO:teuthology.orchestra.run.smithi129.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-14T03:43:24.715 INFO:teuthology.orchestra.run.smithi129.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-14T03:43:24.715 INFO:teuthology.orchestra.run.smithi129.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-14T03:43:24.715 INFO:teuthology.orchestra.run.smithi129.stdout: libradosstriper1 x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 478 k 2024-09-14T03:43:24.716 INFO:teuthology.orchestra.run.smithi129.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-14T03:43:24.716 INFO:teuthology.orchestra.run.smithi129.stdout: librgw2 x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 4.5 M 2024-09-14T03:43:24.716 INFO:teuthology.orchestra.run.smithi129.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-14T03:43:24.716 INFO:teuthology.orchestra.run.smithi129.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-14T03:43:24.716 INFO:teuthology.orchestra.run.smithi129.stdout: parquet-libs x86_64 9.0.0-11.el9 epel 839 k 2024-09-14T03:43:24.716 INFO:teuthology.orchestra.run.smithi129.stdout: python3-ceph-argparse x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 45 k 2024-09-14T03:43:24.716 INFO:teuthology.orchestra.run.smithi129.stdout: python3-ceph-common x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 129 k 2024-09-14T03:43:24.716 INFO:teuthology.orchestra.run.smithi129.stdout: python3-cephfs x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 162 k 2024-09-14T03:43:24.716 INFO:teuthology.orchestra.run.smithi129.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-14T03:43:24.716 INFO:teuthology.orchestra.run.smithi129.stdout: python3-rados x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 321 k 2024-09-14T03:43:24.716 INFO:teuthology.orchestra.run.smithi129.stdout: python3-rbd x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 300 k 2024-09-14T03:43:24.716 INFO:teuthology.orchestra.run.smithi129.stdout: python3-rgw x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 99 k 2024-09-14T03:43:24.716 INFO:teuthology.orchestra.run.smithi129.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2024-09-14T03:43:24.716 INFO:teuthology.orchestra.run.smithi129.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-14T03:43:24.717 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:43:24.717 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction Summary 2024-09-14T03:43:24.717 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:43:24.717 INFO:teuthology.orchestra.run.smithi129.stdout:Install 28 Packages 2024-09-14T03:43:24.717 INFO:teuthology.orchestra.run.smithi129.stdout:Upgrade 2 Packages 2024-09-14T03:43:24.717 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:43:24.717 INFO:teuthology.orchestra.run.smithi129.stdout:Total download size: 53 M 2024-09-14T03:43:24.717 INFO:teuthology.orchestra.run.smithi129.stdout:Downloading Packages: 2024-09-14T03:43:25.262 INFO:teuthology.orchestra.run.smithi129.stdout:(1/30): ceph-base-18.2.4-845.ge4e227b3.el9.x86_ 12 MB/s | 5.1 MB 00:00 2024-09-14T03:43:25.337 INFO:teuthology.orchestra.run.smithi129.stdout:(2/30): ceph-selinux-18.2.4-845.ge4e227b3.el9.x 336 kB/s | 25 kB 00:00 2024-09-14T03:43:25.463 INFO:teuthology.orchestra.run.smithi129.stdout:(3/30): ceph-radosgw-18.2.4-845.ge4e227b3.el9.x 12 MB/s | 7.7 MB 00:00 2024-09-14T03:43:25.505 INFO:teuthology.orchestra.run.smithi129.stdout:(4/30): libcephfs2-18.2.4-845.ge4e227b3.el9.x86 4.2 MB/s | 712 kB 00:00 2024-09-14T03:43:25.538 INFO:teuthology.orchestra.run.smithi129.stdout:(5/30): libradosstriper1-18.2.4-845.ge4e227b3.e 6.2 MB/s | 478 kB 00:00 2024-09-14T03:43:25.630 INFO:teuthology.orchestra.run.smithi129.stdout:(6/30): python3-ceph-argparse-18.2.4-845.ge4e22 491 kB/s | 45 kB 00:00 2024-09-14T03:43:25.714 INFO:teuthology.orchestra.run.smithi129.stdout:(7/30): librgw2-18.2.4-845.ge4e227b3.el9.x86_64 21 MB/s | 4.5 MB 00:00 2024-09-14T03:43:25.739 INFO:teuthology.orchestra.run.smithi129.stdout:(8/30): python3-ceph-common-18.2.4-845.ge4e227b 1.2 MB/s | 129 kB 00:00 2024-09-14T03:43:25.765 INFO:teuthology.orchestra.run.smithi129.stdout:(9/30): python3-cephfs-18.2.4-845.ge4e227b3.el9 3.2 MB/s | 162 kB 00:00 2024-09-14T03:43:25.790 INFO:teuthology.orchestra.run.smithi129.stdout:(10/30): python3-rados-18.2.4-845.ge4e227b3.el9 6.2 MB/s | 321 kB 00:00 2024-09-14T03:43:25.815 INFO:teuthology.orchestra.run.smithi129.stdout:(11/30): python3-rbd-18.2.4-845.ge4e227b3.el9.x 5.8 MB/s | 300 kB 00:00 2024-09-14T03:43:25.841 INFO:teuthology.orchestra.run.smithi129.stdout:(12/30): python3-rgw-18.2.4-845.ge4e227b3.el9.x 1.9 MB/s | 99 kB 00:00 2024-09-14T03:43:26.033 INFO:teuthology.orchestra.run.smithi129.stdout:(13/30): ceph-common-18.2.4-845.ge4e227b3.el9.x 15 MB/s | 18 MB 00:01 2024-09-14T03:43:26.625 INFO:teuthology.orchestra.run.smithi129.stdout:(14/30): librabbitmq-0.11.0-7.el9.x86_64.rpm 76 kB/s | 45 kB 00:00 2024-09-14T03:43:26.700 INFO:teuthology.orchestra.run.smithi129.stdout:(15/30): libpmemobj-1.12.1-1.el9.x86_64.rpm 187 kB/s | 160 kB 00:00 2024-09-14T03:43:26.725 INFO:teuthology.orchestra.run.smithi129.stdout:(16/30): boost-program-options-1.75.0-8.el9.x86 117 kB/s | 107 kB 00:00 2024-09-14T03:43:27.198 INFO:teuthology.orchestra.run.smithi039.stdout:Dependencies resolved. 2024-09-14T03:43:27.200 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:43:27.200 INFO:teuthology.orchestra.run.smithi039.stdout: Package Arch Version Repository Size 2024-09-14T03:43:27.201 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:43:27.201 INFO:teuthology.orchestra.run.smithi039.stdout:Installing: 2024-09-14T03:43:27.201 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-radosgw x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 7.7 M 2024-09-14T03:43:27.201 INFO:teuthology.orchestra.run.smithi039.stdout:Upgrading: 2024-09-14T03:43:27.201 INFO:teuthology.orchestra.run.smithi039.stdout: librados2 x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 3.3 M 2024-09-14T03:43:27.201 INFO:teuthology.orchestra.run.smithi039.stdout: librbd1 x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 3.0 M 2024-09-14T03:43:27.201 INFO:teuthology.orchestra.run.smithi039.stdout:Installing dependencies: 2024-09-14T03:43:27.201 INFO:teuthology.orchestra.run.smithi039.stdout: boost-program-options x86_64 1.75.0-8.el9 appstream 107 k 2024-09-14T03:43:27.202 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-base x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 5.1 M 2024-09-14T03:43:27.202 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-common x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 18 M 2024-09-14T03:43:27.202 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-selinux x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 25 k 2024-09-14T03:43:27.202 INFO:teuthology.orchestra.run.smithi039.stdout: fmt x86_64 8.1.1-5.el9 epel 111 k 2024-09-14T03:43:27.202 INFO:teuthology.orchestra.run.smithi039.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2024-09-14T03:43:27.202 INFO:teuthology.orchestra.run.smithi039.stdout: libarrow x86_64 9.0.0-11.el9 epel 4.4 M 2024-09-14T03:43:27.202 INFO:teuthology.orchestra.run.smithi039.stdout: libarrow-doc noarch 9.0.0-11.el9 epel 26 k 2024-09-14T03:43:27.202 INFO:teuthology.orchestra.run.smithi039.stdout: libcephfs2 x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 712 k 2024-09-14T03:43:27.203 INFO:teuthology.orchestra.run.smithi039.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-09-14T03:43:27.203 INFO:teuthology.orchestra.run.smithi039.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2024-09-14T03:43:27.203 INFO:teuthology.orchestra.run.smithi039.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2024-09-14T03:43:27.203 INFO:teuthology.orchestra.run.smithi039.stdout: libradosstriper1 x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 478 k 2024-09-14T03:43:27.203 INFO:teuthology.orchestra.run.smithi039.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2024-09-14T03:43:27.203 INFO:teuthology.orchestra.run.smithi039.stdout: librgw2 x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 4.5 M 2024-09-14T03:43:27.203 INFO:teuthology.orchestra.run.smithi039.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-09-14T03:43:27.203 INFO:teuthology.orchestra.run.smithi039.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2024-09-14T03:43:27.203 INFO:teuthology.orchestra.run.smithi039.stdout: parquet-libs x86_64 9.0.0-11.el9 epel 839 k 2024-09-14T03:43:27.204 INFO:teuthology.orchestra.run.smithi039.stdout: python3-ceph-argparse x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 45 k 2024-09-14T03:43:27.204 INFO:teuthology.orchestra.run.smithi039.stdout: python3-ceph-common x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 129 k 2024-09-14T03:43:27.204 INFO:teuthology.orchestra.run.smithi039.stdout: python3-cephfs x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 162 k 2024-09-14T03:43:27.204 INFO:teuthology.orchestra.run.smithi039.stdout: python3-prettytable noarch 0.7.2-27.el9 appstream 42 k 2024-09-14T03:43:27.204 INFO:teuthology.orchestra.run.smithi039.stdout: python3-rados x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 321 k 2024-09-14T03:43:27.204 INFO:teuthology.orchestra.run.smithi039.stdout: python3-rbd x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 300 k 2024-09-14T03:43:27.204 INFO:teuthology.orchestra.run.smithi039.stdout: python3-rgw x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 99 k 2024-09-14T03:43:27.204 INFO:teuthology.orchestra.run.smithi039.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2024-09-14T03:43:27.205 INFO:teuthology.orchestra.run.smithi039.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-09-14T03:43:27.205 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:43:27.205 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction Summary 2024-09-14T03:43:27.205 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:43:27.205 INFO:teuthology.orchestra.run.smithi039.stdout:Install 28 Packages 2024-09-14T03:43:27.205 INFO:teuthology.orchestra.run.smithi039.stdout:Upgrade 2 Packages 2024-09-14T03:43:27.205 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:43:27.206 INFO:teuthology.orchestra.run.smithi039.stdout:Total download size: 53 M 2024-09-14T03:43:27.206 INFO:teuthology.orchestra.run.smithi039.stdout:Downloading Packages: 2024-09-14T03:43:27.300 INFO:teuthology.orchestra.run.smithi129.stdout:(17/30): python3-prettytable-0.7.2-27.el9.noarc 73 kB/s | 42 kB 00:00 2024-09-14T03:43:27.342 INFO:teuthology.orchestra.run.smithi129.stdout:(18/30): fmt-8.1.1-5.el9.x86_64.rpm 2.6 MB/s | 111 kB 00:00 2024-09-14T03:43:27.376 INFO:teuthology.orchestra.run.smithi129.stdout:(19/30): gperftools-libs-2.9.1-3.el9.x86_64.rpm 9.0 MB/s | 308 kB 00:00 2024-09-14T03:43:27.502 INFO:teuthology.orchestra.run.smithi129.stdout:(20/30): libarrow-9.0.0-11.el9.x86_64.rpm 35 MB/s | 4.4 MB 00:00 2024-09-14T03:43:27.527 INFO:teuthology.orchestra.run.smithi129.stdout:(21/30): libarrow-doc-9.0.0-11.el9.noarch.rpm 1.0 MB/s | 26 kB 00:00 2024-09-14T03:43:27.552 INFO:teuthology.orchestra.run.smithi129.stdout:(22/30): liboath-2.6.7-2.el9.x86_64.rpm 1.9 MB/s | 49 kB 00:00 2024-09-14T03:43:27.578 INFO:teuthology.orchestra.run.smithi129.stdout:(23/30): libunwind-1.6.2-1.el9.x86_64.rpm 2.6 MB/s | 67 kB 00:00 2024-09-14T03:43:27.611 INFO:teuthology.orchestra.run.smithi129.stdout:(24/30): lttng-ust-2.12.0-6.el9.x86_64.rpm 321 kB/s | 292 kB 00:00 2024-09-14T03:43:27.653 INFO:teuthology.orchestra.run.smithi129.stdout:(25/30): parquet-libs-9.0.0-11.el9.x86_64.rpm 11 MB/s | 839 kB 00:00 2024-09-14T03:43:27.703 INFO:teuthology.orchestra.run.smithi129.stdout:(26/30): re2-20211101-20.el9.x86_64.rpm 2.0 MB/s | 191 kB 00:00 2024-09-14T03:43:27.754 INFO:teuthology.orchestra.run.smithi129.stdout:(27/30): thrift-0.15.0-2.el9.x86_64.rpm 16 MB/s | 1.6 MB 00:00 2024-09-14T03:43:27.840 INFO:teuthology.orchestra.run.smithi039.stdout:(1/30): ceph-base-18.2.4-845.ge4e227b3.el9.x86_ 9.1 MB/s | 5.1 MB 00:00 2024-09-14T03:43:27.907 INFO:teuthology.orchestra.run.smithi039.stdout:(2/30): ceph-selinux-18.2.4-845.ge4e227b3.el9.x 379 kB/s | 25 kB 00:00 2024-09-14T03:43:28.016 INFO:teuthology.orchestra.run.smithi039.stdout:(3/30): ceph-radosgw-18.2.4-845.ge4e227b3.el9.x 11 MB/s | 7.7 MB 00:00 2024-09-14T03:43:28.060 INFO:teuthology.orchestra.run.smithi039.stdout:(4/30): libcephfs2-18.2.4-845.ge4e227b3.el9.x86 4.6 MB/s | 712 kB 00:00 2024-09-14T03:43:28.100 INFO:teuthology.orchestra.run.smithi039.stdout:(5/30): libradosstriper1-18.2.4-845.ge4e227b3.e 5.6 MB/s | 478 kB 00:00 2024-09-14T03:43:28.104 INFO:teuthology.orchestra.run.smithi129.stdout:(28/30): librdkafka-1.6.1-102.el9.x86_64.rpm 448 kB/s | 662 kB 00:01 2024-09-14T03:43:28.163 INFO:teuthology.orchestra.run.smithi129.stdout:(29/30): librados2-18.2.4-845.ge4e227b3.el9.x86 7.2 MB/s | 3.3 MB 00:00 2024-09-14T03:43:28.166 INFO:teuthology.orchestra.run.smithi039.stdout:(6/30): python3-ceph-argparse-18.2.4-845.ge4e22 675 kB/s | 45 kB 00:00 2024-09-14T03:43:28.209 INFO:teuthology.orchestra.run.smithi039.stdout:(7/30): python3-ceph-common-18.2.4-845.ge4e227b 3.0 MB/s | 129 kB 00:00 2024-09-14T03:43:28.213 INFO:teuthology.orchestra.run.smithi129.stdout:(30/30): librbd1-18.2.4-845.ge4e227b3.el9.x86_6 6.6 MB/s | 3.0 MB 00:00 2024-09-14T03:43:28.218 INFO:teuthology.orchestra.run.smithi129.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:43:28.218 INFO:teuthology.orchestra.run.smithi129.stdout:Total 15 MB/s | 53 MB 00:03 2024-09-14T03:43:28.291 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction check 2024-09-14T03:43:28.292 INFO:teuthology.orchestra.run.smithi039.stdout:(8/30): librgw2-18.2.4-845.ge4e227b3.el9.x86_64 19 MB/s | 4.5 MB 00:00 2024-09-14T03:43:28.325 INFO:teuthology.orchestra.run.smithi039.stdout:(9/30): python3-cephfs-18.2.4-845.ge4e227b3.el9 1.4 MB/s | 162 kB 00:00 2024-09-14T03:43:28.326 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction check succeeded. 2024-09-14T03:43:28.326 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction test 2024-09-14T03:43:28.351 INFO:teuthology.orchestra.run.smithi039.stdout:(10/30): python3-rados-18.2.4-845.ge4e227b3.el9 5.3 MB/s | 321 kB 00:00 2024-09-14T03:43:28.376 INFO:teuthology.orchestra.run.smithi039.stdout:(11/30): python3-rbd-18.2.4-845.ge4e227b3.el9.x 5.8 MB/s | 300 kB 00:00 2024-09-14T03:43:28.401 INFO:teuthology.orchestra.run.smithi039.stdout:(12/30): python3-rgw-18.2.4-845.ge4e227b3.el9.x 1.9 MB/s | 99 kB 00:00 2024-09-14T03:43:28.502 INFO:teuthology.orchestra.run.smithi039.stdout:(13/30): boost-program-options-1.75.0-8.el9.x86 854 kB/s | 107 kB 00:00 2024-09-14T03:43:28.527 INFO:teuthology.orchestra.run.smithi039.stdout:(14/30): libpmemobj-1.12.1-1.el9.x86_64.rpm 1.2 MB/s | 160 kB 00:00 2024-09-14T03:43:28.552 INFO:teuthology.orchestra.run.smithi039.stdout:(15/30): librabbitmq-0.11.0-7.el9.x86_64.rpm 897 kB/s | 45 kB 00:00 2024-09-14T03:43:28.611 INFO:teuthology.orchestra.run.smithi039.stdout:(16/30): librdkafka-1.6.1-102.el9.x86_64.rpm 7.8 MB/s | 662 kB 00:00 2024-09-14T03:43:28.636 INFO:teuthology.orchestra.run.smithi039.stdout:(17/30): lttng-ust-2.12.0-6.el9.x86_64.rpm 3.4 MB/s | 292 kB 00:00 2024-09-14T03:43:28.653 INFO:teuthology.orchestra.run.smithi039.stdout:(18/30): python3-prettytable-0.7.2-27.el9.noarc 1.0 MB/s | 42 kB 00:00 2024-09-14T03:43:28.783 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction test succeeded. 2024-09-14T03:43:28.783 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction 2024-09-14T03:43:28.879 INFO:teuthology.orchestra.run.smithi039.stdout:(19/30): ceph-common-18.2.4-845.ge4e227b3.el9.x 12 MB/s | 18 MB 00:01 2024-09-14T03:43:28.904 INFO:teuthology.orchestra.run.smithi039.stdout:(20/30): fmt-8.1.1-5.el9.x86_64.rpm 414 kB/s | 111 kB 00:00 2024-09-14T03:43:28.937 INFO:teuthology.orchestra.run.smithi039.stdout:(21/30): libarrow-doc-9.0.0-11.el9.noarch.rpm 1.0 MB/s | 26 kB 00:00 2024-09-14T03:43:28.963 INFO:teuthology.orchestra.run.smithi039.stdout:(22/30): liboath-2.6.7-2.el9.x86_64.rpm 2.1 MB/s | 49 kB 00:00 2024-09-14T03:43:28.988 INFO:teuthology.orchestra.run.smithi039.stdout:(23/30): gperftools-libs-2.9.1-3.el9.x86_64.rpm 920 kB/s | 308 kB 00:00 2024-09-14T03:43:29.013 INFO:teuthology.orchestra.run.smithi039.stdout:(24/30): libunwind-1.6.2-1.el9.x86_64.rpm 1.3 MB/s | 67 kB 00:00 2024-09-14T03:43:29.047 INFO:teuthology.orchestra.run.smithi039.stdout:(25/30): re2-20211101-20.el9.x86_64.rpm 5.7 MB/s | 191 kB 00:00 2024-09-14T03:43:29.089 INFO:teuthology.orchestra.run.smithi039.stdout:(26/30): parquet-libs-9.0.0-11.el9.x86_64.rpm 8.2 MB/s | 839 kB 00:00 2024-09-14T03:43:29.156 INFO:teuthology.orchestra.run.smithi039.stdout:(27/30): thrift-0.15.0-2.el9.x86_64.rpm 15 MB/s | 1.6 MB 00:00 2024-09-14T03:43:29.239 INFO:teuthology.orchestra.run.smithi039.stdout:(28/30): libarrow-9.0.0-11.el9.x86_64.rpm 12 MB/s | 4.4 MB 00:00 2024-09-14T03:43:29.365 INFO:teuthology.orchestra.run.smithi039.stdout:(29/30): librbd1-18.2.4-845.ge4e227b3.el9.x86_6 14 MB/s | 3.0 MB 00:00 2024-09-14T03:43:29.423 INFO:teuthology.orchestra.run.smithi039.stdout:(30/30): librados2-18.2.4-845.ge4e227b3.el9.x86 9.9 MB/s | 3.3 MB 00:00 2024-09-14T03:43:29.428 INFO:teuthology.orchestra.run.smithi039.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:43:29.428 INFO:teuthology.orchestra.run.smithi039.stdout:Total 24 MB/s | 53 MB 00:02 2024-09-14T03:43:29.496 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction check 2024-09-14T03:43:29.535 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction check succeeded. 2024-09-14T03:43:29.535 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction test 2024-09-14T03:43:30.022 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction test succeeded. 2024-09-14T03:43:30.022 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction 2024-09-14T03:43:30.128 INFO:teuthology.orchestra.run.smithi129.stdout: Preparing : 1/1 2024-09-14T03:43:30.215 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/32 2024-09-14T03:43:30.243 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/32 2024-09-14T03:43:30.301 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/32 2024-09-14T03:43:30.677 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : thrift-0.15.0-2.el9.x86_64 4/32 2024-09-14T03:43:30.747 INFO:teuthology.orchestra.run.smithi129.stdout: Upgrading : librados2-2:18.2.4-845.ge4e227b3.el9.x86_64 5/32 2024-09-14T03:43:30.839 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: librados2-2:18.2.4-845.ge4e227b3.el9.x86_64 5/32 2024-09-14T03:43:30.931 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-rados-2:18.2.4-845.ge4e227b3.el9.x86_64 6/32 2024-09-14T03:43:30.974 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : libcephfs2-2:18.2.4-845.ge4e227b3.el9.x86_64 7/32 2024-09-14T03:43:31.036 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: libcephfs2-2:18.2.4-845.ge4e227b3.el9.x86_64 7/32 2024-09-14T03:43:31.073 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : liboath-2.6.7-2.el9.x86_64 8/32 2024-09-14T03:43:31.116 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-ceph-argparse-2:18.2.4-845.ge4e227b3.el9.x 9/32 2024-09-14T03:43:31.183 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-cephfs-2:18.2.4-845.ge4e227b3.el9.x86_64 10/32 2024-09-14T03:43:31.209 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : libradosstriper1-2:18.2.4-845.ge4e227b3.el9.x86_64 11/32 2024-09-14T03:43:31.275 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: libradosstriper1-2:18.2.4-845.ge4e227b3.el9.x86_64 11/32 2024-09-14T03:43:31.365 INFO:teuthology.orchestra.run.smithi039.stdout: Preparing : 1/1 2024-09-14T03:43:31.381 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : re2-1:20211101-20.el9.x86_64 12/32 2024-09-14T03:43:31.417 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : libarrow-9.0.0-11.el9.x86_64 13/32 2024-09-14T03:43:31.464 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/32 2024-09-14T03:43:31.469 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/32 2024-09-14T03:43:31.499 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 15/32 2024-09-14T03:43:31.513 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/32 2024-09-14T03:43:31.526 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : libarrow-doc-9.0.0-11.el9.noarch 16/32 2024-09-14T03:43:31.562 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : fmt-8.1.1-5.el9.x86_64 17/32 2024-09-14T03:43:31.563 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/32 2024-09-14T03:43:31.595 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 18/32 2024-09-14T03:43:31.895 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 19/32 2024-09-14T03:43:31.897 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : thrift-0.15.0-2.el9.x86_64 4/32 2024-09-14T03:43:31.933 INFO:teuthology.orchestra.run.smithi129.stdout: Upgrading : librbd1-2:18.2.4-845.ge4e227b3.el9.x86_64 20/32 2024-09-14T03:43:32.003 INFO:teuthology.orchestra.run.smithi039.stdout: Upgrading : librados2-2:18.2.4-845.ge4e227b3.el9.x86_64 5/32 2024-09-14T03:43:32.022 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: librbd1-2:18.2.4-845.ge4e227b3.el9.x86_64 20/32 2024-09-14T03:43:32.054 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-rbd-2:18.2.4-845.ge4e227b3.el9.x86_64 21/32 2024-09-14T03:43:32.096 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: librados2-2:18.2.4-845.ge4e227b3.el9.x86_64 5/32 2024-09-14T03:43:32.098 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 22/32 2024-09-14T03:43:32.190 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-rados-2:18.2.4-845.ge4e227b3.el9.x86_64 6/32 2024-09-14T03:43:32.221 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : libcephfs2-2:18.2.4-845.ge4e227b3.el9.x86_64 7/32 2024-09-14T03:43:32.276 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: libcephfs2-2:18.2.4-845.ge4e227b3.el9.x86_64 7/32 2024-09-14T03:43:32.313 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : liboath-2.6.7-2.el9.x86_64 8/32 2024-09-14T03:43:32.357 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-ceph-argparse-2:18.2.4-845.ge4e227b3.el9.x 9/32 2024-09-14T03:43:32.425 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-cephfs-2:18.2.4-845.ge4e227b3.el9.x86_64 10/32 2024-09-14T03:43:32.449 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : libradosstriper1-2:18.2.4-845.ge4e227b3.el9.x86_64 11/32 2024-09-14T03:43:32.511 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : parquet-libs-9.0.0-11.el9.x86_64 23/32 2024-09-14T03:43:32.514 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: libradosstriper1-2:18.2.4-845.ge4e227b3.el9.x86_64 11/32 2024-09-14T03:43:32.544 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : librgw2-2:18.2.4-845.ge4e227b3.el9.x86_64 24/32 2024-09-14T03:43:32.607 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: librgw2-2:18.2.4-845.ge4e227b3.el9.x86_64 24/32 2024-09-14T03:43:32.625 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : re2-1:20211101-20.el9.x86_64 12/32 2024-09-14T03:43:32.657 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : libarrow-9.0.0-11.el9.x86_64 13/32 2024-09-14T03:43:32.683 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-rgw-2:18.2.4-845.ge4e227b3.el9.x86_64 25/32 2024-09-14T03:43:32.703 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/32 2024-09-14T03:43:32.713 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-ceph-common-2:18.2.4-845.ge4e227b3.el9.x86 26/32 2024-09-14T03:43:32.746 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 15/32 2024-09-14T03:43:32.773 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : libarrow-doc-9.0.0-11.el9.noarch 16/32 2024-09-14T03:43:32.810 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : fmt-8.1.1-5.el9.x86_64 17/32 2024-09-14T03:43:32.851 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 18/32 2024-09-14T03:43:33.174 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 19/32 2024-09-14T03:43:33.221 INFO:teuthology.orchestra.run.smithi039.stdout: Upgrading : librbd1-2:18.2.4-845.ge4e227b3.el9.x86_64 20/32 2024-09-14T03:43:33.305 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: librbd1-2:18.2.4-845.ge4e227b3.el9.x86_64 20/32 2024-09-14T03:43:33.334 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-rbd-2:18.2.4-845.ge4e227b3.el9.x86_64 21/32 2024-09-14T03:43:33.371 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : boost-program-options-1.75.0-8.el9.x86_64 22/32 2024-09-14T03:43:33.800 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : parquet-libs-9.0.0-11.el9.x86_64 23/32 2024-09-14T03:43:33.832 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : librgw2-2:18.2.4-845.ge4e227b3.el9.x86_64 24/32 2024-09-14T03:43:33.903 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: librgw2-2:18.2.4-845.ge4e227b3.el9.x86_64 24/32 2024-09-14T03:43:33.991 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-rgw-2:18.2.4-845.ge4e227b3.el9.x86_64 25/32 2024-09-14T03:43:34.026 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-ceph-common-2:18.2.4-845.ge4e227b3.el9.x86 26/32 2024-09-14T03:43:34.854 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: ceph-common-2:18.2.4-845.ge4e227b3.el9.x86_64 27/32 2024-09-14T03:43:34.902 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : ceph-common-2:18.2.4-845.ge4e227b3.el9.x86_64 27/32 2024-09-14T03:43:35.458 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: ceph-common-2:18.2.4-845.ge4e227b3.el9.x86_64 27/32 2024-09-14T03:43:35.505 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : ceph-base-2:18.2.4-845.ge4e227b3.el9.x86_64 28/32 2024-09-14T03:43:35.589 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: ceph-base-2:18.2.4-845.ge4e227b3.el9.x86_64 28/32 2024-09-14T03:43:35.589 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-14T03:43:35.589 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-14T03:43:35.589 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:43:35.624 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : ceph-selinux-2:18.2.4-845.ge4e227b3.el9.x86_64 29/32 2024-09-14T03:43:36.175 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: ceph-common-2:18.2.4-845.ge4e227b3.el9.x86_64 27/32 2024-09-14T03:43:36.206 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : ceph-common-2:18.2.4-845.ge4e227b3.el9.x86_64 27/32 2024-09-14T03:43:36.783 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: ceph-common-2:18.2.4-845.ge4e227b3.el9.x86_64 27/32 2024-09-14T03:43:36.951 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : ceph-base-2:18.2.4-845.ge4e227b3.el9.x86_64 28/32 2024-09-14T03:43:37.055 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: ceph-base-2:18.2.4-845.ge4e227b3.el9.x86_64 28/32 2024-09-14T03:43:37.055 INFO:teuthology.orchestra.run.smithi039.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-09-14T03:43:37.055 INFO:teuthology.orchestra.run.smithi039.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-09-14T03:43:37.056 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:43:37.094 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : ceph-selinux-2:18.2.4-845.ge4e227b3.el9.x86_64 29/32 2024-09-14T03:43:45.566 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: ceph-selinux-2:18.2.4-845.ge4e227b3.el9.x86_64 29/32 2024-09-14T03:43:45.566 INFO:teuthology.orchestra.run.smithi129.stdout:skipping the directory /sys 2024-09-14T03:43:45.566 INFO:teuthology.orchestra.run.smithi129.stdout:skipping the directory /proc 2024-09-14T03:43:45.566 INFO:teuthology.orchestra.run.smithi129.stdout:skipping the directory /mnt 2024-09-14T03:43:45.566 INFO:teuthology.orchestra.run.smithi129.stdout:skipping the directory /var/tmp 2024-09-14T03:43:45.566 INFO:teuthology.orchestra.run.smithi129.stdout:skipping the directory /home 2024-09-14T03:43:45.566 INFO:teuthology.orchestra.run.smithi129.stdout:skipping the directory /root 2024-09-14T03:43:45.567 INFO:teuthology.orchestra.run.smithi129.stdout:skipping the directory /tmp 2024-09-14T03:43:45.567 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:43:46.333 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : ceph-radosgw-2:18.2.4-845.ge4e227b3.el9.x86_64 30/32 2024-09-14T03:43:46.358 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: ceph-radosgw-2:18.2.4-845.ge4e227b3.el9.x86_64 30/32 2024-09-14T03:43:46.358 INFO:teuthology.orchestra.run.smithi129.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:43:46.358 INFO:teuthology.orchestra.run.smithi129.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-14T03:43:46.358 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-14T03:43:46.358 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-14T03:43:46.358 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:43:46.360 INFO:teuthology.orchestra.run.smithi129.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-09-14T03:43:46.436 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-09-14T03:43:46.436 INFO:teuthology.orchestra.run.smithi129.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 32/32 2024-09-14T03:43:47.558 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 32/32 2024-09-14T03:43:47.558 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : ceph-base-2:18.2.4-845.ge4e227b3.el9.x86_64 1/32 2024-09-14T03:43:47.558 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : ceph-common-2:18.2.4-845.ge4e227b3.el9.x86_64 2/32 2024-09-14T03:43:47.558 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : ceph-radosgw-2:18.2.4-845.ge4e227b3.el9.x86_64 3/32 2024-09-14T03:43:47.558 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : ceph-selinux-2:18.2.4-845.ge4e227b3.el9.x86_64 4/32 2024-09-14T03:43:47.558 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : libcephfs2-2:18.2.4-845.ge4e227b3.el9.x86_64 5/32 2024-09-14T03:43:47.558 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : libradosstriper1-2:18.2.4-845.ge4e227b3.el9.x86_64 6/32 2024-09-14T03:43:47.558 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : librgw2-2:18.2.4-845.ge4e227b3.el9.x86_64 7/32 2024-09-14T03:43:47.558 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-ceph-argparse-2:18.2.4-845.ge4e227b3.el9.x 8/32 2024-09-14T03:43:47.559 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-ceph-common-2:18.2.4-845.ge4e227b3.el9.x86 9/32 2024-09-14T03:43:47.559 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-cephfs-2:18.2.4-845.ge4e227b3.el9.x86_64 10/32 2024-09-14T03:43:47.559 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-rados-2:18.2.4-845.ge4e227b3.el9.x86_64 11/32 2024-09-14T03:43:47.559 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-rbd-2:18.2.4-845.ge4e227b3.el9.x86_64 12/32 2024-09-14T03:43:47.559 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-rgw-2:18.2.4-845.ge4e227b3.el9.x86_64 13/32 2024-09-14T03:43:47.559 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 14/32 2024-09-14T03:43:47.559 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 15/32 2024-09-14T03:43:47.559 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 16/32 2024-09-14T03:43:47.559 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 17/32 2024-09-14T03:43:47.559 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 18/32 2024-09-14T03:43:47.560 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 19/32 2024-09-14T03:43:47.560 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : fmt-8.1.1-5.el9.x86_64 20/32 2024-09-14T03:43:47.560 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 21/32 2024-09-14T03:43:47.560 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : libarrow-9.0.0-11.el9.x86_64 22/32 2024-09-14T03:43:47.560 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : libarrow-doc-9.0.0-11.el9.noarch 23/32 2024-09-14T03:43:47.560 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 24/32 2024-09-14T03:43:47.560 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 25/32 2024-09-14T03:43:47.560 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : parquet-libs-9.0.0-11.el9.x86_64 26/32 2024-09-14T03:43:47.560 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : re2-1:20211101-20.el9.x86_64 27/32 2024-09-14T03:43:47.560 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 28/32 2024-09-14T03:43:47.560 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : librados2-2:18.2.4-845.ge4e227b3.el9.x86_64 29/32 2024-09-14T03:43:47.560 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 30/32 2024-09-14T03:43:47.560 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : librbd1-2:18.2.4-845.ge4e227b3.el9.x86_64 31/32 2024-09-14T03:43:47.619 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: ceph-selinux-2:18.2.4-845.ge4e227b3.el9.x86_64 29/32 2024-09-14T03:43:47.619 INFO:teuthology.orchestra.run.smithi039.stdout:skipping the directory /sys 2024-09-14T03:43:47.619 INFO:teuthology.orchestra.run.smithi039.stdout:skipping the directory /proc 2024-09-14T03:43:47.620 INFO:teuthology.orchestra.run.smithi039.stdout:skipping the directory /mnt 2024-09-14T03:43:47.620 INFO:teuthology.orchestra.run.smithi039.stdout:skipping the directory /var/tmp 2024-09-14T03:43:47.620 INFO:teuthology.orchestra.run.smithi039.stdout:skipping the directory /home 2024-09-14T03:43:47.620 INFO:teuthology.orchestra.run.smithi039.stdout:skipping the directory /root 2024-09-14T03:43:47.620 INFO:teuthology.orchestra.run.smithi039.stdout:skipping the directory /tmp 2024-09-14T03:43:47.620 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:43:48.419 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : ceph-radosgw-2:18.2.4-845.ge4e227b3.el9.x86_64 30/32 2024-09-14T03:43:48.445 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 32/32 2024-09-14T03:43:48.445 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:43:48.445 INFO:teuthology.orchestra.run.smithi129.stdout:Upgraded: 2024-09-14T03:43:48.445 INFO:teuthology.orchestra.run.smithi129.stdout: librados2-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:48.445 INFO:teuthology.orchestra.run.smithi129.stdout: librbd1-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:48.445 INFO:teuthology.orchestra.run.smithi129.stdout:Installed: 2024-09-14T03:43:48.445 INFO:teuthology.orchestra.run.smithi129.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-09-14T03:43:48.445 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-base-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:48.445 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-common-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:48.445 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-radosgw-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:48.445 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-selinux-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:48.446 INFO:teuthology.orchestra.run.smithi129.stdout: fmt-8.1.1-5.el9.x86_64 2024-09-14T03:43:48.446 INFO:teuthology.orchestra.run.smithi129.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-14T03:43:48.446 INFO:teuthology.orchestra.run.smithi129.stdout: libarrow-9.0.0-11.el9.x86_64 2024-09-14T03:43:48.446 INFO:teuthology.orchestra.run.smithi129.stdout: libarrow-doc-9.0.0-11.el9.noarch 2024-09-14T03:43:48.446 INFO:teuthology.orchestra.run.smithi129.stdout: libcephfs2-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:48.446 INFO:teuthology.orchestra.run.smithi129.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-14T03:43:48.446 INFO:teuthology.orchestra.run.smithi129.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-14T03:43:48.446 INFO:teuthology.orchestra.run.smithi129.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-14T03:43:48.446 INFO:teuthology.orchestra.run.smithi129.stdout: libradosstriper1-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:48.446 INFO:teuthology.orchestra.run.smithi129.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-14T03:43:48.446 INFO:teuthology.orchestra.run.smithi129.stdout: librgw2-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:48.446 INFO:teuthology.orchestra.run.smithi129.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-14T03:43:48.446 INFO:teuthology.orchestra.run.smithi129.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-14T03:43:48.447 INFO:teuthology.orchestra.run.smithi129.stdout: parquet-libs-9.0.0-11.el9.x86_64 2024-09-14T03:43:48.447 INFO:teuthology.orchestra.run.smithi129.stdout: python3-ceph-argparse-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:48.447 INFO:teuthology.orchestra.run.smithi129.stdout: python3-ceph-common-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:48.447 INFO:teuthology.orchestra.run.smithi129.stdout: python3-cephfs-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:48.447 INFO:teuthology.orchestra.run.smithi129.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-14T03:43:48.447 INFO:teuthology.orchestra.run.smithi129.stdout: python3-rados-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:48.447 INFO:teuthology.orchestra.run.smithi129.stdout: python3-rbd-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:48.447 INFO:teuthology.orchestra.run.smithi129.stdout: python3-rgw-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:48.447 INFO:teuthology.orchestra.run.smithi129.stdout: re2-1:20211101-20.el9.x86_64 2024-09-14T03:43:48.447 INFO:teuthology.orchestra.run.smithi129.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-14T03:43:48.447 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:43:48.447 INFO:teuthology.orchestra.run.smithi129.stdout:Complete! 2024-09-14T03:43:48.448 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: ceph-radosgw-2:18.2.4-845.ge4e227b3.el9.x86_64 30/32 2024-09-14T03:43:48.448 INFO:teuthology.orchestra.run.smithi039.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:43:48.448 INFO:teuthology.orchestra.run.smithi039.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-09-14T03:43:48.448 INFO:teuthology.orchestra.run.smithi039.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-14T03:43:48.448 INFO:teuthology.orchestra.run.smithi039.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-09-14T03:43:48.448 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:43:48.448 INFO:teuthology.orchestra.run.smithi039.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-09-14T03:43:48.530 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 31/32 2024-09-14T03:43:48.530 INFO:teuthology.orchestra.run.smithi039.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 32/32 2024-09-14T03:43:48.813 DEBUG:teuthology.orchestra.run.smithi129:> sudo yum -y install ceph-test 2024-09-14T03:43:49.373 INFO:teuthology.orchestra.run.smithi129.stdout:Last metadata expiration check: 0:00:27 ago on Sat 14 Sep 2024 03:43:22 AM UTC. 2024-09-14T03:43:49.514 INFO:teuthology.orchestra.run.smithi129.stdout:Dependencies resolved. 2024-09-14T03:43:49.515 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:43:49.515 INFO:teuthology.orchestra.run.smithi129.stdout: Package Arch Version Repository Size 2024-09-14T03:43:49.515 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:43:49.515 INFO:teuthology.orchestra.run.smithi129.stdout:Installing: 2024-09-14T03:43:49.515 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-test x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 40 M 2024-09-14T03:43:49.515 INFO:teuthology.orchestra.run.smithi129.stdout:Installing dependencies: 2024-09-14T03:43:49.515 INFO:teuthology.orchestra.run.smithi129.stdout: libcephsqlite x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 166 k 2024-09-14T03:43:49.515 INFO:teuthology.orchestra.run.smithi129.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-14T03:43:49.515 INFO:teuthology.orchestra.run.smithi129.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-14T03:43:49.515 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:43:49.515 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction Summary 2024-09-14T03:43:49.516 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:43:49.516 INFO:teuthology.orchestra.run.smithi129.stdout:Install 4 Packages 2024-09-14T03:43:49.516 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:43:49.516 INFO:teuthology.orchestra.run.smithi129.stdout:Total download size: 40 M 2024-09-14T03:43:49.516 INFO:teuthology.orchestra.run.smithi129.stdout:Installed size: 164 M 2024-09-14T03:43:49.516 INFO:teuthology.orchestra.run.smithi129.stdout:Downloading Packages: 2024-09-14T03:43:49.662 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 32/32 2024-09-14T03:43:49.662 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : ceph-base-2:18.2.4-845.ge4e227b3.el9.x86_64 1/32 2024-09-14T03:43:49.663 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : ceph-common-2:18.2.4-845.ge4e227b3.el9.x86_64 2/32 2024-09-14T03:43:49.663 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : ceph-radosgw-2:18.2.4-845.ge4e227b3.el9.x86_64 3/32 2024-09-14T03:43:49.663 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : ceph-selinux-2:18.2.4-845.ge4e227b3.el9.x86_64 4/32 2024-09-14T03:43:49.663 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : libcephfs2-2:18.2.4-845.ge4e227b3.el9.x86_64 5/32 2024-09-14T03:43:49.663 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : libradosstriper1-2:18.2.4-845.ge4e227b3.el9.x86_64 6/32 2024-09-14T03:43:49.663 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : librgw2-2:18.2.4-845.ge4e227b3.el9.x86_64 7/32 2024-09-14T03:43:49.663 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-ceph-argparse-2:18.2.4-845.ge4e227b3.el9.x 8/32 2024-09-14T03:43:49.663 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-ceph-common-2:18.2.4-845.ge4e227b3.el9.x86 9/32 2024-09-14T03:43:49.663 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-cephfs-2:18.2.4-845.ge4e227b3.el9.x86_64 10/32 2024-09-14T03:43:49.663 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-rados-2:18.2.4-845.ge4e227b3.el9.x86_64 11/32 2024-09-14T03:43:49.664 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-rbd-2:18.2.4-845.ge4e227b3.el9.x86_64 12/32 2024-09-14T03:43:49.664 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-rgw-2:18.2.4-845.ge4e227b3.el9.x86_64 13/32 2024-09-14T03:43:49.664 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : boost-program-options-1.75.0-8.el9.x86_64 14/32 2024-09-14T03:43:49.664 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 15/32 2024-09-14T03:43:49.664 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 16/32 2024-09-14T03:43:49.664 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 17/32 2024-09-14T03:43:49.664 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 18/32 2024-09-14T03:43:49.664 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 19/32 2024-09-14T03:43:49.664 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : fmt-8.1.1-5.el9.x86_64 20/32 2024-09-14T03:43:49.664 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 21/32 2024-09-14T03:43:49.664 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : libarrow-9.0.0-11.el9.x86_64 22/32 2024-09-14T03:43:49.664 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : libarrow-doc-9.0.0-11.el9.noarch 23/32 2024-09-14T03:43:49.665 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 24/32 2024-09-14T03:43:49.665 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 25/32 2024-09-14T03:43:49.665 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : parquet-libs-9.0.0-11.el9.x86_64 26/32 2024-09-14T03:43:49.665 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : re2-1:20211101-20.el9.x86_64 27/32 2024-09-14T03:43:49.665 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 28/32 2024-09-14T03:43:49.665 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : librados2-2:18.2.4-845.ge4e227b3.el9.x86_64 29/32 2024-09-14T03:43:49.665 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 30/32 2024-09-14T03:43:49.666 INFO:teuthology.orchestra.run.smithi129.stdout:(1/4): socat-1.7.4.1-6.el9.x86_64.rpm 2.4 MB/s | 304 kB 00:00 2024-09-14T03:43:49.666 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : librbd1-2:18.2.4-845.ge4e227b3.el9.x86_64 31/32 2024-09-14T03:43:49.696 INFO:teuthology.orchestra.run.smithi129.stdout:(2/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.9 MB/s | 64 kB 00:00 2024-09-14T03:43:49.747 INFO:teuthology.orchestra.run.smithi129.stdout:(3/4): libcephsqlite-18.2.4-845.ge4e227b3.el9.x 803 kB/s | 166 kB 00:00 2024-09-14T03:43:50.753 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 32/32 2024-09-14T03:43:50.753 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:43:50.754 INFO:teuthology.orchestra.run.smithi039.stdout:Upgraded: 2024-09-14T03:43:50.754 INFO:teuthology.orchestra.run.smithi039.stdout: librados2-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:50.754 INFO:teuthology.orchestra.run.smithi039.stdout: librbd1-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:50.754 INFO:teuthology.orchestra.run.smithi039.stdout:Installed: 2024-09-14T03:43:50.754 INFO:teuthology.orchestra.run.smithi039.stdout: boost-program-options-1.75.0-8.el9.x86_64 2024-09-14T03:43:50.754 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-base-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:50.754 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-common-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:50.754 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-radosgw-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:50.754 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-selinux-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:50.754 INFO:teuthology.orchestra.run.smithi039.stdout: fmt-8.1.1-5.el9.x86_64 2024-09-14T03:43:50.754 INFO:teuthology.orchestra.run.smithi039.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2024-09-14T03:43:50.755 INFO:teuthology.orchestra.run.smithi039.stdout: libarrow-9.0.0-11.el9.x86_64 2024-09-14T03:43:50.755 INFO:teuthology.orchestra.run.smithi039.stdout: libarrow-doc-9.0.0-11.el9.noarch 2024-09-14T03:43:50.755 INFO:teuthology.orchestra.run.smithi039.stdout: libcephfs2-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:50.755 INFO:teuthology.orchestra.run.smithi039.stdout: liboath-2.6.7-2.el9.x86_64 2024-09-14T03:43:50.755 INFO:teuthology.orchestra.run.smithi039.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-09-14T03:43:50.755 INFO:teuthology.orchestra.run.smithi039.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-09-14T03:43:50.755 INFO:teuthology.orchestra.run.smithi039.stdout: libradosstriper1-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:50.755 INFO:teuthology.orchestra.run.smithi039.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-09-14T03:43:50.755 INFO:teuthology.orchestra.run.smithi039.stdout: librgw2-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:50.755 INFO:teuthology.orchestra.run.smithi039.stdout: libunwind-1.6.2-1.el9.x86_64 2024-09-14T03:43:50.755 INFO:teuthology.orchestra.run.smithi039.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-09-14T03:43:50.756 INFO:teuthology.orchestra.run.smithi039.stdout: parquet-libs-9.0.0-11.el9.x86_64 2024-09-14T03:43:50.756 INFO:teuthology.orchestra.run.smithi039.stdout: python3-ceph-argparse-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:50.756 INFO:teuthology.orchestra.run.smithi039.stdout: python3-ceph-common-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:50.756 INFO:teuthology.orchestra.run.smithi039.stdout: python3-cephfs-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:50.756 INFO:teuthology.orchestra.run.smithi039.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-09-14T03:43:50.756 INFO:teuthology.orchestra.run.smithi039.stdout: python3-rados-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:50.756 INFO:teuthology.orchestra.run.smithi039.stdout: python3-rbd-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:50.756 INFO:teuthology.orchestra.run.smithi039.stdout: python3-rgw-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:50.756 INFO:teuthology.orchestra.run.smithi039.stdout: re2-1:20211101-20.el9.x86_64 2024-09-14T03:43:50.756 INFO:teuthology.orchestra.run.smithi039.stdout: thrift-0.15.0-2.el9.x86_64 2024-09-14T03:43:50.756 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:43:50.756 INFO:teuthology.orchestra.run.smithi039.stdout:Complete! 2024-09-14T03:43:50.830 INFO:teuthology.orchestra.run.smithi129.stdout:(4/4): ceph-test-18.2.4-845.ge4e227b3.el9.x86_6 31 MB/s | 40 MB 00:01 2024-09-14T03:43:50.832 INFO:teuthology.orchestra.run.smithi129.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:43:50.833 INFO:teuthology.orchestra.run.smithi129.stdout:Total 30 MB/s | 40 MB 00:01 2024-09-14T03:43:50.859 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction check 2024-09-14T03:43:50.879 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction check succeeded. 2024-09-14T03:43:50.879 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction test 2024-09-14T03:43:51.075 DEBUG:teuthology.orchestra.run.smithi039:> sudo yum -y install ceph-test 2024-09-14T03:43:51.208 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction test succeeded. 2024-09-14T03:43:51.209 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction 2024-09-14T03:43:51.660 INFO:teuthology.orchestra.run.smithi039.stdout:Last metadata expiration check: 0:00:27 ago on Sat 14 Sep 2024 03:43:24 AM UTC. 2024-09-14T03:43:51.681 INFO:teuthology.orchestra.run.smithi129.stdout: Preparing : 1/1 2024-09-14T03:43:51.777 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-09-14T03:43:51.807 INFO:teuthology.orchestra.run.smithi039.stdout:Dependencies resolved. 2024-09-14T03:43:51.808 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:43:51.808 INFO:teuthology.orchestra.run.smithi039.stdout: Package Arch Version Repository Size 2024-09-14T03:43:51.808 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:43:51.808 INFO:teuthology.orchestra.run.smithi039.stdout:Installing: 2024-09-14T03:43:51.808 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-test x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 40 M 2024-09-14T03:43:51.808 INFO:teuthology.orchestra.run.smithi039.stdout:Installing dependencies: 2024-09-14T03:43:51.808 INFO:teuthology.orchestra.run.smithi039.stdout: libcephsqlite x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 166 k 2024-09-14T03:43:51.809 INFO:teuthology.orchestra.run.smithi039.stdout: socat x86_64 1.7.4.1-6.el9 appstream 304 k 2024-09-14T03:43:51.809 INFO:teuthology.orchestra.run.smithi039.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2024-09-14T03:43:51.809 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:43:51.809 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction Summary 2024-09-14T03:43:51.809 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:43:51.809 INFO:teuthology.orchestra.run.smithi039.stdout:Install 4 Packages 2024-09-14T03:43:51.809 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:43:51.809 INFO:teuthology.orchestra.run.smithi039.stdout:Total download size: 40 M 2024-09-14T03:43:51.810 INFO:teuthology.orchestra.run.smithi039.stdout:Installed size: 164 M 2024-09-14T03:43:51.810 INFO:teuthology.orchestra.run.smithi039.stdout:Downloading Packages: 2024-09-14T03:43:51.830 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-09-14T03:43:51.866 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : libcephsqlite-2:18.2.4-845.ge4e227b3.el9.x86_64 3/4 2024-09-14T03:43:52.295 INFO:teuthology.orchestra.run.smithi039.stdout:(1/4): socat-1.7.4.1-6.el9.x86_64.rpm 1.4 MB/s | 304 kB 00:00 2024-09-14T03:43:52.320 INFO:teuthology.orchestra.run.smithi039.stdout:(2/4): libcephsqlite-18.2.4-845.ge4e227b3.el9.x 717 kB/s | 166 kB 00:00 2024-09-14T03:43:52.346 INFO:teuthology.orchestra.run.smithi039.stdout:(3/4): xmlstarlet-1.6.1-20.el9.x86_64.rpm 1.2 MB/s | 64 kB 00:00 2024-09-14T03:43:53.404 INFO:teuthology.orchestra.run.smithi039.stdout:(4/4): ceph-test-18.2.4-845.ge4e227b3.el9.x86_6 30 MB/s | 40 MB 00:01 2024-09-14T03:43:53.406 INFO:teuthology.orchestra.run.smithi039.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:43:53.409 INFO:teuthology.orchestra.run.smithi039.stdout:Total 25 MB/s | 40 MB 00:01 2024-09-14T03:43:53.436 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction check 2024-09-14T03:43:53.458 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction check succeeded. 2024-09-14T03:43:53.458 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction test 2024-09-14T03:43:53.813 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction test succeeded. 2024-09-14T03:43:53.813 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction 2024-09-14T03:43:54.437 INFO:teuthology.orchestra.run.smithi039.stdout: Preparing : 1/1 2024-09-14T03:43:54.536 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/4 2024-09-14T03:43:54.580 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : socat-1.7.4.1-6.el9.x86_64 2/4 2024-09-14T03:43:54.615 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : libcephsqlite-2:18.2.4-845.ge4e227b3.el9.x86_64 3/4 2024-09-14T03:43:55.555 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: libcephsqlite-2:18.2.4-845.ge4e227b3.el9.x86_64 3/4 2024-09-14T03:43:55.638 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : ceph-test-2:18.2.4-845.ge4e227b3.el9.x86_64 4/4 2024-09-14T03:43:57.593 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: ceph-test-2:18.2.4-845.ge4e227b3.el9.x86_64 4/4 2024-09-14T03:43:57.593 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : ceph-test-2:18.2.4-845.ge4e227b3.el9.x86_64 1/4 2024-09-14T03:43:57.593 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : libcephsqlite-2:18.2.4-845.ge4e227b3.el9.x86_64 2/4 2024-09-14T03:43:57.594 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-09-14T03:43:57.903 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-09-14T03:43:57.903 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:43:57.903 INFO:teuthology.orchestra.run.smithi129.stdout:Installed: 2024-09-14T03:43:57.903 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-test-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:57.903 INFO:teuthology.orchestra.run.smithi129.stdout: libcephsqlite-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:43:57.904 INFO:teuthology.orchestra.run.smithi129.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-14T03:43:57.904 INFO:teuthology.orchestra.run.smithi129.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-14T03:43:57.904 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:43:57.904 INFO:teuthology.orchestra.run.smithi129.stdout:Complete! 2024-09-14T03:43:58.194 DEBUG:teuthology.orchestra.run.smithi129:> sudo yum -y install ceph 2024-09-14T03:43:58.524 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: libcephsqlite-2:18.2.4-845.ge4e227b3.el9.x86_64 3/4 2024-09-14T03:43:58.616 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : ceph-test-2:18.2.4-845.ge4e227b3.el9.x86_64 4/4 2024-09-14T03:43:58.764 INFO:teuthology.orchestra.run.smithi129.stdout:Last metadata expiration check: 0:00:36 ago on Sat 14 Sep 2024 03:43:22 AM UTC. 2024-09-14T03:43:58.908 INFO:teuthology.orchestra.run.smithi129.stdout:Dependencies resolved. 2024-09-14T03:43:58.910 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:43:58.910 INFO:teuthology.orchestra.run.smithi129.stdout: Package Arch Version Repository Size 2024-09-14T03:43:58.910 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:43:58.910 INFO:teuthology.orchestra.run.smithi129.stdout:Installing: 2024-09-14T03:43:58.910 INFO:teuthology.orchestra.run.smithi129.stdout: ceph x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 6.5 k 2024-09-14T03:43:58.911 INFO:teuthology.orchestra.run.smithi129.stdout:Installing dependencies: 2024-09-14T03:43:58.911 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-mds x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 2.1 M 2024-09-14T03:43:58.911 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-mgr x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 1.5 M 2024-09-14T03:43:58.911 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-mgr-modules-core noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 246 k 2024-09-14T03:43:58.911 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-mon x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 4.7 M 2024-09-14T03:43:58.911 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-osd x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 17 M 2024-09-14T03:43:58.911 INFO:teuthology.orchestra.run.smithi129.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-14T03:43:58.911 INFO:teuthology.orchestra.run.smithi129.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-14T03:43:58.911 INFO:teuthology.orchestra.run.smithi129.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-14T03:43:58.911 INFO:teuthology.orchestra.run.smithi129.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-14T03:43:58.911 INFO:teuthology.orchestra.run.smithi129.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-14T03:43:58.911 INFO:teuthology.orchestra.run.smithi129.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-14T03:43:58.911 INFO:teuthology.orchestra.run.smithi129.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-14T03:43:58.911 INFO:teuthology.orchestra.run.smithi129.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-09-14T03:43:58.912 INFO:teuthology.orchestra.run.smithi129.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-14T03:43:58.912 INFO:teuthology.orchestra.run.smithi129.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-14T03:43:58.912 INFO:teuthology.orchestra.run.smithi129.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-14T03:43:58.912 INFO:teuthology.orchestra.run.smithi129.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-14T03:43:58.912 INFO:teuthology.orchestra.run.smithi129.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-14T03:43:58.912 INFO:teuthology.orchestra.run.smithi129.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-14T03:43:58.912 INFO:teuthology.orchestra.run.smithi129.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-14T03:43:58.912 INFO:teuthology.orchestra.run.smithi129.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-14T03:43:58.912 INFO:teuthology.orchestra.run.smithi129.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-14T03:43:58.912 INFO:teuthology.orchestra.run.smithi129.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-14T03:43:58.912 INFO:teuthology.orchestra.run.smithi129.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-14T03:43:58.912 INFO:teuthology.orchestra.run.smithi129.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-14T03:43:58.913 INFO:teuthology.orchestra.run.smithi129.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-14T03:43:58.913 INFO:teuthology.orchestra.run.smithi129.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-14T03:43:58.913 INFO:teuthology.orchestra.run.smithi129.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-14T03:43:58.913 INFO:teuthology.orchestra.run.smithi129.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-14T03:43:58.913 INFO:teuthology.orchestra.run.smithi129.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-14T03:43:58.913 INFO:teuthology.orchestra.run.smithi129.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-14T03:43:58.913 INFO:teuthology.orchestra.run.smithi129.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-14T03:43:58.913 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:43:58.913 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction Summary 2024-09-14T03:43:58.913 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:43:58.913 INFO:teuthology.orchestra.run.smithi129.stdout:Install 33 Packages 2024-09-14T03:43:58.913 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:43:58.914 INFO:teuthology.orchestra.run.smithi129.stdout:Total download size: 29 M 2024-09-14T03:43:58.914 INFO:teuthology.orchestra.run.smithi129.stdout:Installed size: 104 M 2024-09-14T03:43:58.914 INFO:teuthology.orchestra.run.smithi129.stdout:Downloading Packages: 2024-09-14T03:43:59.689 INFO:teuthology.orchestra.run.smithi129.stdout:(1/33): ceph-18.2.4-845.ge4e227b3.el9.x86_64.rp 48 kB/s | 6.5 kB 00:00 2024-09-14T03:43:59.831 INFO:teuthology.orchestra.run.smithi129.stdout:(2/33): ceph-mgr-18.2.4-845.ge4e227b3.el9.x86_6 5.3 MB/s | 1.5 MB 00:00 2024-09-14T03:43:59.898 INFO:teuthology.orchestra.run.smithi129.stdout:(3/33): ceph-mds-18.2.4-845.ge4e227b3.el9.x86_6 6.1 MB/s | 2.1 MB 00:00 2024-09-14T03:43:59.998 INFO:teuthology.orchestra.run.smithi129.stdout:(4/33): ceph-mgr-modules-core-18.2.4-845.ge4e22 2.4 MB/s | 246 kB 00:00 2024-09-14T03:44:00.073 INFO:teuthology.orchestra.run.smithi129.stdout:(5/33): ceph-mon-18.2.4-845.ge4e227b3.el9.x86_6 12 MB/s | 4.7 MB 00:00 2024-09-14T03:44:00.265 INFO:teuthology.orchestra.run.smithi129.stdout:(6/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 949 kB/s | 253 kB 00:00 2024-09-14T03:44:00.432 INFO:teuthology.orchestra.run.smithi129.stdout:(7/33): ceph-osd-18.2.4-845.ge4e227b3.el9.x86_6 27 MB/s | 17 MB 00:00 2024-09-14T03:44:00.457 INFO:teuthology.orchestra.run.smithi129.stdout:(8/33): python3-ply-3.11-14.el9.noarch.rpm 555 kB/s | 106 kB 00:00 2024-09-14T03:44:00.525 INFO:teuthology.orchestra.run.smithi129.stdout:(9/33): python3-cryptography-36.0.1-4.el9.x86_6 2.7 MB/s | 1.2 MB 00:00 2024-09-14T03:44:00.550 INFO:teuthology.orchestra.run.smithi129.stdout:(10/33): python3-requests-2.25.1-8.el9.noarch.r 1.3 MB/s | 125 kB 00:00 2024-09-14T03:44:00.608 INFO:teuthology.orchestra.run.smithi129.stdout:(11/33): python3-urllib3-1.26.5-6.el9.noarch.rp 3.6 MB/s | 215 kB 00:00 2024-09-14T03:44:00.638 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: ceph-test-2:18.2.4-845.ge4e227b3.el9.x86_64 4/4 2024-09-14T03:44:00.639 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : ceph-test-2:18.2.4-845.ge4e227b3.el9.x86_64 1/4 2024-09-14T03:44:00.639 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : libcephsqlite-2:18.2.4-845.ge4e227b3.el9.x86_64 2/4 2024-09-14T03:44:00.639 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : socat-1.7.4.1-6.el9.x86_64 3/4 2024-09-14T03:44:00.659 INFO:teuthology.orchestra.run.smithi129.stdout:(12/33): python3-pycparser-2.20-6.el9.noarch.rp 599 kB/s | 135 kB 00:00 2024-09-14T03:44:00.944 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 4/4 2024-09-14T03:44:00.944 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:00.944 INFO:teuthology.orchestra.run.smithi039.stdout:Installed: 2024-09-14T03:44:00.944 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-test-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:44:00.944 INFO:teuthology.orchestra.run.smithi039.stdout: libcephsqlite-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:44:00.944 INFO:teuthology.orchestra.run.smithi039.stdout: socat-1.7.4.1-6.el9.x86_64 2024-09-14T03:44:00.945 INFO:teuthology.orchestra.run.smithi039.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-09-14T03:44:00.945 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:00.945 INFO:teuthology.orchestra.run.smithi039.stdout:Complete! 2024-09-14T03:44:01.230 DEBUG:teuthology.orchestra.run.smithi039:> sudo yum -y install ceph 2024-09-14T03:44:01.484 INFO:teuthology.orchestra.run.smithi129.stdout:(13/33): python3-mako-1.1.4-6.el9.noarch.rpm 184 kB/s | 172 kB 00:00 2024-09-14T03:44:01.542 INFO:teuthology.orchestra.run.smithi129.stdout:(14/33): python3-markupsafe-1.1.1-12.el9.x86_64 37 kB/s | 35 kB 00:00 2024-09-14T03:44:01.559 INFO:teuthology.orchestra.run.smithi129.stdout:(15/33): python3-pytz-2021.1-5.el9.noarch.rpm 57 kB/s | 51 kB 00:00 2024-09-14T03:44:01.593 INFO:teuthology.orchestra.run.smithi129.stdout:(16/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 883 kB/s | 43 kB 00:00 2024-09-14T03:44:01.627 INFO:teuthology.orchestra.run.smithi129.stdout:(17/33): python3-cherrypy-18.6.1-2.el9.noarch.r 11 MB/s | 358 kB 00:00 2024-09-14T03:44:01.652 INFO:teuthology.orchestra.run.smithi129.stdout:(18/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 1.8 MB/s | 172 kB 00:00 2024-09-14T03:44:01.669 INFO:teuthology.orchestra.run.smithi129.stdout:(19/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 255 kB/s | 11 kB 00:00 2024-09-14T03:44:01.686 INFO:teuthology.orchestra.run.smithi129.stdout:(20/33): python3-jaraco-classes-3.2.1-5.el9.noa 527 kB/s | 18 kB 00:00 2024-09-14T03:44:01.711 INFO:teuthology.orchestra.run.smithi129.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 550 kB/s | 23 kB 00:00 2024-09-14T03:44:01.729 INFO:teuthology.orchestra.run.smithi129.stdout:(22/33): python3-jaraco-functools-3.5.0-2.el9.n 461 kB/s | 19 kB 00:00 2024-09-14T03:44:01.745 INFO:teuthology.orchestra.run.smithi129.stdout:(23/33): python3-jaraco-text-3.2.0-6.el9.noarch 588 kB/s | 20 kB 00:00 2024-09-14T03:44:01.762 INFO:teuthology.orchestra.run.smithi129.stdout:(24/33): python3-logutils-0.3.5-21.el9.noarch.r 1.3 MB/s | 46 kB 00:00 2024-09-14T03:44:01.796 INFO:teuthology.orchestra.run.smithi129.stdout:(25/33): python3-more-itertools-8.12.0-2.el9.no 1.5 MB/s | 79 kB 00:00 2024-09-14T03:44:01.813 INFO:teuthology.orchestra.run.smithi129.stdout:(26/33): python3-portend-3.1.0-2.el9.noarch.rpm 996 kB/s | 16 kB 00:00 2024-09-14T03:44:01.819 INFO:teuthology.orchestra.run.smithi039.stdout:Last metadata expiration check: 0:00:37 ago on Sat 14 Sep 2024 03:43:24 AM UTC. 2024-09-14T03:44:01.838 INFO:teuthology.orchestra.run.smithi129.stdout:(27/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 3.5 MB/s | 90 kB 00:00 2024-09-14T03:44:01.864 INFO:teuthology.orchestra.run.smithi129.stdout:(28/33): python3-pecan-1.4.2-3.el9.noarch.rpm 2.8 MB/s | 272 kB 00:00 2024-09-14T03:44:01.889 INFO:teuthology.orchestra.run.smithi129.stdout:(29/33): python3-tempora-5.0.0-2.el9.noarch.rpm 711 kB/s | 36 kB 00:00 2024-09-14T03:44:01.914 INFO:teuthology.orchestra.run.smithi129.stdout:(30/33): python3-webob-1.8.8-2.el9.noarch.rpm 4.6 MB/s | 230 kB 00:00 2024-09-14T03:44:01.939 INFO:teuthology.orchestra.run.smithi129.stdout:(31/33): python3-zc-lockfile-2.0-10.el9.noarch. 803 kB/s | 20 kB 00:00 2024-09-14T03:44:01.968 INFO:teuthology.orchestra.run.smithi039.stdout:Dependencies resolved. 2024-09-14T03:44:01.970 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:44:01.970 INFO:teuthology.orchestra.run.smithi039.stdout: Package Arch Version Repository Size 2024-09-14T03:44:01.970 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:44:01.970 INFO:teuthology.orchestra.run.smithi039.stdout:Installing: 2024-09-14T03:44:01.970 INFO:teuthology.orchestra.run.smithi039.stdout: ceph x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 6.5 k 2024-09-14T03:44:01.970 INFO:teuthology.orchestra.run.smithi039.stdout:Installing dependencies: 2024-09-14T03:44:01.970 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-mds x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 2.1 M 2024-09-14T03:44:01.970 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-mgr x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 1.5 M 2024-09-14T03:44:01.971 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-mgr-modules-core noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 246 k 2024-09-14T03:44:01.971 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-mon x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 4.7 M 2024-09-14T03:44:01.971 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-osd x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 17 M 2024-09-14T03:44:01.971 INFO:teuthology.orchestra.run.smithi039.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-09-14T03:44:01.971 INFO:teuthology.orchestra.run.smithi039.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2024-09-14T03:44:01.971 INFO:teuthology.orchestra.run.smithi039.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-09-14T03:44:01.971 INFO:teuthology.orchestra.run.smithi039.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-09-14T03:44:01.971 INFO:teuthology.orchestra.run.smithi039.stdout: python3-cryptography x86_64 36.0.1-4.el9 baseos 1.2 M 2024-09-14T03:44:01.971 INFO:teuthology.orchestra.run.smithi039.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-09-14T03:44:01.971 INFO:teuthology.orchestra.run.smithi039.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-09-14T03:44:01.971 INFO:teuthology.orchestra.run.smithi039.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-09-14T03:44:01.971 INFO:teuthology.orchestra.run.smithi039.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-09-14T03:44:01.971 INFO:teuthology.orchestra.run.smithi039.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-09-14T03:44:01.971 INFO:teuthology.orchestra.run.smithi039.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-09-14T03:44:01.972 INFO:teuthology.orchestra.run.smithi039.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2024-09-14T03:44:01.972 INFO:teuthology.orchestra.run.smithi039.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2024-09-14T03:44:01.972 INFO:teuthology.orchestra.run.smithi039.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-09-14T03:44:01.972 INFO:teuthology.orchestra.run.smithi039.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-09-14T03:44:01.972 INFO:teuthology.orchestra.run.smithi039.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2024-09-14T03:44:01.972 INFO:teuthology.orchestra.run.smithi039.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-09-14T03:44:01.972 INFO:teuthology.orchestra.run.smithi039.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-09-14T03:44:01.972 INFO:teuthology.orchestra.run.smithi039.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2024-09-14T03:44:01.972 INFO:teuthology.orchestra.run.smithi039.stdout: python3-pytz noarch 2021.1-5.el9 appstream 51 k 2024-09-14T03:44:01.972 INFO:teuthology.orchestra.run.smithi039.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-09-14T03:44:01.972 INFO:teuthology.orchestra.run.smithi039.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-09-14T03:44:01.972 INFO:teuthology.orchestra.run.smithi039.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2024-09-14T03:44:01.972 INFO:teuthology.orchestra.run.smithi039.stdout: python3-urllib3 noarch 1.26.5-6.el9 baseos 215 k 2024-09-14T03:44:01.972 INFO:teuthology.orchestra.run.smithi039.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2024-09-14T03:44:01.973 INFO:teuthology.orchestra.run.smithi039.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-09-14T03:44:01.973 INFO:teuthology.orchestra.run.smithi039.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-09-14T03:44:01.973 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:01.973 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction Summary 2024-09-14T03:44:01.973 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:44:01.973 INFO:teuthology.orchestra.run.smithi039.stdout:Install 33 Packages 2024-09-14T03:44:01.973 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:01.974 INFO:teuthology.orchestra.run.smithi129.stdout:(32/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 5.0 MB/s | 427 kB 00:00 2024-09-14T03:44:01.974 INFO:teuthology.orchestra.run.smithi039.stdout:Total download size: 29 M 2024-09-14T03:44:01.974 INFO:teuthology.orchestra.run.smithi039.stdout:Installed size: 104 M 2024-09-14T03:44:01.974 INFO:teuthology.orchestra.run.smithi039.stdout:Downloading Packages: 2024-09-14T03:44:02.198 INFO:teuthology.orchestra.run.smithi129.stdout:(33/33): python3-toml-0.10.2-6.el9.noarch.rpm 58 kB/s | 42 kB 00:00 2024-09-14T03:44:02.203 INFO:teuthology.orchestra.run.smithi129.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:44:02.203 INFO:teuthology.orchestra.run.smithi129.stdout:Total 8.9 MB/s | 29 MB 00:03 2024-09-14T03:44:02.324 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction check 2024-09-14T03:44:02.361 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction check succeeded. 2024-09-14T03:44:02.361 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction test 2024-09-14T03:44:02.456 INFO:teuthology.orchestra.run.smithi039.stdout:(1/33): ceph-18.2.4-845.ge4e227b3.el9.x86_64.rp 55 kB/s | 6.5 kB 00:00 2024-09-14T03:44:02.673 INFO:teuthology.orchestra.run.smithi039.stdout:(2/33): ceph-mds-18.2.4-845.ge4e227b3.el9.x86_6 6.2 MB/s | 2.1 MB 00:00 2024-09-14T03:44:02.724 INFO:teuthology.orchestra.run.smithi039.stdout:(3/33): ceph-mgr-18.2.4-845.ge4e227b3.el9.x86_6 3.8 MB/s | 1.5 MB 00:00 2024-09-14T03:44:02.740 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction test succeeded. 2024-09-14T03:44:02.741 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction 2024-09-14T03:44:02.807 INFO:teuthology.orchestra.run.smithi039.stdout:(4/33): ceph-mgr-modules-core-18.2.4-845.ge4e22 2.9 MB/s | 246 kB 00:00 2024-09-14T03:44:02.883 INFO:teuthology.orchestra.run.smithi039.stdout:(5/33): ceph-mon-18.2.4-845.ge4e227b3.el9.x86_6 11 MB/s | 4.7 MB 00:00 2024-09-14T03:44:03.041 INFO:teuthology.orchestra.run.smithi039.stdout:(6/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 1.1 MB/s | 253 kB 00:00 2024-09-14T03:44:03.217 INFO:teuthology.orchestra.run.smithi039.stdout:(7/33): ceph-osd-18.2.4-845.ge4e227b3.el9.x86_6 30 MB/s | 17 MB 00:00 2024-09-14T03:44:03.242 INFO:teuthology.orchestra.run.smithi039.stdout:(8/33): python3-ply-3.11-14.el9.noarch.rpm 532 kB/s | 106 kB 00:00 2024-09-14T03:44:03.284 INFO:teuthology.orchestra.run.smithi039.stdout:(9/33): python3-cryptography-36.0.1-4.el9.x86_6 3.1 MB/s | 1.2 MB 00:00 2024-09-14T03:44:03.309 INFO:teuthology.orchestra.run.smithi039.stdout:(10/33): python3-requests-2.25.1-8.el9.noarch.r 1.8 MB/s | 125 kB 00:00 2024-09-14T03:44:03.343 INFO:teuthology.orchestra.run.smithi039.stdout:(11/33): python3-urllib3-1.26.5-6.el9.noarch.rp 3.6 MB/s | 215 kB 00:00 2024-09-14T03:44:03.368 INFO:teuthology.orchestra.run.smithi039.stdout:(12/33): python3-pycparser-2.20-6.el9.noarch.rp 896 kB/s | 135 kB 00:00 2024-09-14T03:44:03.543 INFO:teuthology.orchestra.run.smithi039.stdout:(13/33): python3-markupsafe-1.1.1-12.el9.x86_64 173 kB/s | 35 kB 00:00 2024-09-14T03:44:03.569 INFO:teuthology.orchestra.run.smithi039.stdout:(14/33): python3-pytz-2021.1-5.el9.noarch.rpm 255 kB/s | 51 kB 00:00 2024-09-14T03:44:03.611 INFO:teuthology.orchestra.run.smithi039.stdout:(15/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 1.0 MB/s | 43 kB 00:00 2024-09-14T03:44:03.636 INFO:teuthology.orchestra.run.smithi039.stdout:(16/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 6.7 MB/s | 172 kB 00:00 2024-09-14T03:44:03.661 INFO:teuthology.orchestra.run.smithi039.stdout:(17/33): python3-toml-0.10.2-6.el9.noarch.rpm 354 kB/s | 42 kB 00:00 2024-09-14T03:44:03.686 INFO:teuthology.orchestra.run.smithi039.stdout:(18/33): python3-mako-1.1.4-6.el9.noarch.rpm 457 kB/s | 172 kB 00:00 2024-09-14T03:44:03.728 INFO:teuthology.orchestra.run.smithi039.stdout:(19/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 160 kB/s | 11 kB 00:00 2024-09-14T03:44:03.753 INFO:teuthology.orchestra.run.smithi039.stdout:(20/33): python3-jaraco-classes-3.2.1-5.el9.noa 302 kB/s | 18 kB 00:00 2024-09-14T03:44:03.779 INFO:teuthology.orchestra.run.smithi039.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 480 kB/s | 23 kB 00:00 2024-09-14T03:44:03.804 INFO:teuthology.orchestra.run.smithi039.stdout:(22/33): python3-cherrypy-18.6.1-2.el9.noarch.r 2.1 MB/s | 358 kB 00:00 2024-09-14T03:44:03.829 INFO:teuthology.orchestra.run.smithi039.stdout:(23/33): python3-jaraco-functools-3.5.0-2.el9.n 387 kB/s | 19 kB 00:00 2024-09-14T03:44:03.855 INFO:teuthology.orchestra.run.smithi039.stdout:(24/33): python3-jaraco-text-3.2.0-6.el9.noarch 261 kB/s | 20 kB 00:00 2024-09-14T03:44:03.880 INFO:teuthology.orchestra.run.smithi039.stdout:(25/33): python3-logutils-0.3.5-21.el9.noarch.r 614 kB/s | 46 kB 00:00 2024-09-14T03:44:03.905 INFO:teuthology.orchestra.run.smithi039.stdout:(26/33): python3-more-itertools-8.12.0-2.el9.no 1.5 MB/s | 79 kB 00:00 2024-09-14T03:44:03.930 INFO:teuthology.orchestra.run.smithi039.stdout:(27/33): python3-portend-3.1.0-2.el9.noarch.rpm 328 kB/s | 16 kB 00:00 2024-09-14T03:44:03.952 INFO:teuthology.orchestra.run.smithi129.stdout: Preparing : 1/1 2024-09-14T03:44:03.955 INFO:teuthology.orchestra.run.smithi039.stdout:(28/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.7 MB/s | 90 kB 00:00 2024-09-14T03:44:03.981 INFO:teuthology.orchestra.run.smithi039.stdout:(29/33): python3-tempora-5.0.0-2.el9.noarch.rpm 713 kB/s | 36 kB 00:00 2024-09-14T03:44:04.006 INFO:teuthology.orchestra.run.smithi039.stdout:(30/33): python3-pecan-1.4.2-3.el9.noarch.rpm 1.8 MB/s | 272 kB 00:00 2024-09-14T03:44:04.017 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-09-14T03:44:04.031 INFO:teuthology.orchestra.run.smithi039.stdout:(31/33): python3-zc-lockfile-2.0-10.el9.noarch. 808 kB/s | 20 kB 00:00 2024-09-14T03:44:04.050 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-09-14T03:44:04.056 INFO:teuthology.orchestra.run.smithi039.stdout:(32/33): python3-webob-1.8.8-2.el9.noarch.rpm 3.0 MB/s | 230 kB 00:00 2024-09-14T03:44:04.083 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-09-14T03:44:04.090 INFO:teuthology.orchestra.run.smithi039.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 3.8 MB/s | 427 kB 00:00 2024-09-14T03:44:04.094 INFO:teuthology.orchestra.run.smithi039.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:44:04.095 INFO:teuthology.orchestra.run.smithi039.stdout:Total 14 MB/s | 29 MB 00:02 2024-09-14T03:44:04.182 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-09-14T03:44:04.222 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction check 2024-09-14T03:44:04.263 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction check succeeded. 2024-09-14T03:44:04.263 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction test 2024-09-14T03:44:04.273 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-09-14T03:44:04.311 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 6/33 2024-09-14T03:44:04.354 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-09-14T03:44:04.384 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-09-14T03:44:04.419 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-09-14T03:44:04.452 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-09-14T03:44:04.492 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-09-14T03:44:04.528 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-09-14T03:44:04.562 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-09-14T03:44:04.600 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-09-14T03:44:04.682 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-09-14T03:44:04.691 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction test succeeded. 2024-09-14T03:44:04.692 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction 2024-09-14T03:44:04.845 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-09-14T03:44:04.925 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-09-14T03:44:04.998 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 18/33 2024-09-14T03:44:05.043 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-09-14T03:44:05.099 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-09-14T03:44:05.155 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-09-14T03:44:05.347 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-09-14T03:44:05.396 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-09-14T03:44:05.474 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-09-14T03:44:05.562 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-09-14T03:44:05.596 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-09-14T03:44:05.711 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-09-14T03:44:05.870 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : ceph-mgr-modules-core-2:18.2.4-845.ge4e227b3.el9.n 28/33 2024-09-14T03:44:05.898 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : ceph-mgr-2:18.2.4-845.ge4e227b3.el9.x86_64 29/33 2024-09-14T03:44:05.907 INFO:teuthology.orchestra.run.smithi039.stdout: Preparing : 1/1 2024-09-14T03:44:05.923 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: ceph-mgr-2:18.2.4-845.ge4e227b3.el9.x86_64 29/33 2024-09-14T03:44:05.923 INFO:teuthology.orchestra.run.smithi129.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:44:05.923 INFO:teuthology.orchestra.run.smithi129.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-14T03:44:05.923 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-14T03:44:05.923 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-14T03:44:05.924 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:05.976 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-09-14T03:44:06.009 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-09-14T03:44:06.042 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-09-14T03:44:06.138 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-09-14T03:44:06.224 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-09-14T03:44:06.278 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 6/33 2024-09-14T03:44:06.330 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-09-14T03:44:06.359 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 8/33 2024-09-14T03:44:06.394 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 9/33 2024-09-14T03:44:06.427 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 10/33 2024-09-14T03:44:06.477 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 11/33 2024-09-14T03:44:06.520 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 12/33 2024-09-14T03:44:06.562 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 13/33 2024-09-14T03:44:06.600 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 14/33 2024-09-14T03:44:06.688 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 15/33 2024-09-14T03:44:06.850 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 16/33 2024-09-14T03:44:06.930 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 17/33 2024-09-14T03:44:06.990 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-urllib3-1.26.5-6.el9.noarch 18/33 2024-09-14T03:44:07.028 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 19/33 2024-09-14T03:44:07.086 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-ply-3.11-14.el9.noarch 20/33 2024-09-14T03:44:07.150 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 21/33 2024-09-14T03:44:07.353 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 22/33 2024-09-14T03:44:07.391 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : ceph-osd-2:18.2.4-845.ge4e227b3.el9.x86_64 30/33 2024-09-14T03:44:07.415 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 23/33 2024-09-14T03:44:07.420 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: ceph-osd-2:18.2.4-845.ge4e227b3.el9.x86_64 30/33 2024-09-14T03:44:07.420 INFO:teuthology.orchestra.run.smithi129.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:44:07.421 INFO:teuthology.orchestra.run.smithi129.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-14T03:44:07.421 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-14T03:44:07.421 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-14T03:44:07.421 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:07.496 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 24/33 2024-09-14T03:44:07.586 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 25/33 2024-09-14T03:44:07.622 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 26/33 2024-09-14T03:44:07.754 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 27/33 2024-09-14T03:44:07.875 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : ceph-mon-2:18.2.4-845.ge4e227b3.el9.x86_64 31/33 2024-09-14T03:44:07.900 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: ceph-mon-2:18.2.4-845.ge4e227b3.el9.x86_64 31/33 2024-09-14T03:44:07.900 INFO:teuthology.orchestra.run.smithi129.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:44:07.900 INFO:teuthology.orchestra.run.smithi129.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-14T03:44:07.900 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-14T03:44:07.900 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-14T03:44:07.900 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:07.929 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : ceph-mgr-modules-core-2:18.2.4-845.ge4e227b3.el9.n 28/33 2024-09-14T03:44:07.957 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : ceph-mgr-2:18.2.4-845.ge4e227b3.el9.x86_64 29/33 2024-09-14T03:44:07.984 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: ceph-mgr-2:18.2.4-845.ge4e227b3.el9.x86_64 29/33 2024-09-14T03:44:07.984 INFO:teuthology.orchestra.run.smithi039.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:44:07.984 INFO:teuthology.orchestra.run.smithi039.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-09-14T03:44:07.984 INFO:teuthology.orchestra.run.smithi039.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-14T03:44:07.984 INFO:teuthology.orchestra.run.smithi039.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-09-14T03:44:07.985 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:08.101 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : ceph-mds-2:18.2.4-845.ge4e227b3.el9.x86_64 32/33 2024-09-14T03:44:08.125 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: ceph-mds-2:18.2.4-845.ge4e227b3.el9.x86_64 32/33 2024-09-14T03:44:08.125 INFO:teuthology.orchestra.run.smithi129.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:44:08.125 INFO:teuthology.orchestra.run.smithi129.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-14T03:44:08.125 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-14T03:44:08.125 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-14T03:44:08.125 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:08.189 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : ceph-2:18.2.4-845.ge4e227b3.el9.x86_64 33/33 2024-09-14T03:44:09.508 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : ceph-osd-2:18.2.4-845.ge4e227b3.el9.x86_64 30/33 2024-09-14T03:44:09.542 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: ceph-osd-2:18.2.4-845.ge4e227b3.el9.x86_64 30/33 2024-09-14T03:44:09.542 INFO:teuthology.orchestra.run.smithi039.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:44:09.542 INFO:teuthology.orchestra.run.smithi039.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-09-14T03:44:09.542 INFO:teuthology.orchestra.run.smithi039.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-14T03:44:09.542 INFO:teuthology.orchestra.run.smithi039.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-09-14T03:44:09.542 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:09.860 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: ceph-2:18.2.4-845.ge4e227b3.el9.x86_64 33/33 2024-09-14T03:44:09.860 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : ceph-2:18.2.4-845.ge4e227b3.el9.x86_64 1/33 2024-09-14T03:44:09.860 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : ceph-mds-2:18.2.4-845.ge4e227b3.el9.x86_64 2/33 2024-09-14T03:44:09.860 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : ceph-mgr-2:18.2.4-845.ge4e227b3.el9.x86_64 3/33 2024-09-14T03:44:09.860 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : ceph-mon-2:18.2.4-845.ge4e227b3.el9.x86_64 4/33 2024-09-14T03:44:09.861 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : ceph-osd-2:18.2.4-845.ge4e227b3.el9.x86_64 5/33 2024-09-14T03:44:09.861 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : ceph-mgr-modules-core-2:18.2.4-845.ge4e227b3.el9.n 6/33 2024-09-14T03:44:09.861 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-09-14T03:44:09.861 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-09-14T03:44:09.861 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-09-14T03:44:09.861 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-09-14T03:44:09.861 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-09-14T03:44:09.861 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/33 2024-09-14T03:44:09.861 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-09-14T03:44:09.861 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-09-14T03:44:09.861 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-09-14T03:44:09.862 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-09-14T03:44:09.862 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-09-14T03:44:09.862 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-09-14T03:44:09.862 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-09-14T03:44:09.862 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-09-14T03:44:09.862 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-09-14T03:44:09.862 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-09-14T03:44:09.862 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-09-14T03:44:09.862 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-09-14T03:44:09.862 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-09-14T03:44:09.862 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-09-14T03:44:09.862 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-09-14T03:44:09.864 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-09-14T03:44:09.864 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-09-14T03:44:09.864 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-09-14T03:44:09.864 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 31/33 2024-09-14T03:44:09.864 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-09-14T03:44:10.009 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : ceph-mon-2:18.2.4-845.ge4e227b3.el9.x86_64 31/33 2024-09-14T03:44:10.037 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: ceph-mon-2:18.2.4-845.ge4e227b3.el9.x86_64 31/33 2024-09-14T03:44:10.037 INFO:teuthology.orchestra.run.smithi039.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:44:10.037 INFO:teuthology.orchestra.run.smithi039.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-09-14T03:44:10.038 INFO:teuthology.orchestra.run.smithi039.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-14T03:44:10.038 INFO:teuthology.orchestra.run.smithi039.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-09-14T03:44:10.038 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:10.252 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : ceph-mds-2:18.2.4-845.ge4e227b3.el9.x86_64 32/33 2024-09-14T03:44:10.277 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: ceph-mds-2:18.2.4-845.ge4e227b3.el9.x86_64 32/33 2024-09-14T03:44:10.277 INFO:teuthology.orchestra.run.smithi039.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:44:10.277 INFO:teuthology.orchestra.run.smithi039.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-09-14T03:44:10.277 INFO:teuthology.orchestra.run.smithi039.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-14T03:44:10.277 INFO:teuthology.orchestra.run.smithi039.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-09-14T03:44:10.277 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:10.354 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : ceph-2:18.2.4-845.ge4e227b3.el9.x86_64 33/33 2024-09-14T03:44:10.871 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-09-14T03:44:10.872 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:10.872 INFO:teuthology.orchestra.run.smithi129.stdout:Installed: 2024-09-14T03:44:10.872 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:44:10.872 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-mds-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:44:10.872 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-mgr-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:44:10.872 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-mgr-modules-core-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:44:10.872 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-mon-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:44:10.872 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-osd-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:44:10.872 INFO:teuthology.orchestra.run.smithi129.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-14T03:44:10.872 INFO:teuthology.orchestra.run.smithi129.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-14T03:44:10.872 INFO:teuthology.orchestra.run.smithi129.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-14T03:44:10.872 INFO:teuthology.orchestra.run.smithi129.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-14T03:44:10.872 INFO:teuthology.orchestra.run.smithi129.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-14T03:44:10.873 INFO:teuthology.orchestra.run.smithi129.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-14T03:44:10.873 INFO:teuthology.orchestra.run.smithi129.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-14T03:44:10.873 INFO:teuthology.orchestra.run.smithi129.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-14T03:44:10.873 INFO:teuthology.orchestra.run.smithi129.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-14T03:44:10.873 INFO:teuthology.orchestra.run.smithi129.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-14T03:44:10.873 INFO:teuthology.orchestra.run.smithi129.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-14T03:44:10.873 INFO:teuthology.orchestra.run.smithi129.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-14T03:44:10.873 INFO:teuthology.orchestra.run.smithi129.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-14T03:44:10.873 INFO:teuthology.orchestra.run.smithi129.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-14T03:44:10.873 INFO:teuthology.orchestra.run.smithi129.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-14T03:44:10.873 INFO:teuthology.orchestra.run.smithi129.stdout: python3-ply-3.11-14.el9.noarch 2024-09-14T03:44:10.873 INFO:teuthology.orchestra.run.smithi129.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-14T03:44:10.874 INFO:teuthology.orchestra.run.smithi129.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-14T03:44:10.874 INFO:teuthology.orchestra.run.smithi129.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-14T03:44:10.874 INFO:teuthology.orchestra.run.smithi129.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-14T03:44:10.874 INFO:teuthology.orchestra.run.smithi129.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-14T03:44:10.874 INFO:teuthology.orchestra.run.smithi129.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-14T03:44:10.874 INFO:teuthology.orchestra.run.smithi129.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-14T03:44:10.874 INFO:teuthology.orchestra.run.smithi129.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-14T03:44:10.874 INFO:teuthology.orchestra.run.smithi129.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-14T03:44:10.874 INFO:teuthology.orchestra.run.smithi129.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-14T03:44:10.874 INFO:teuthology.orchestra.run.smithi129.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-14T03:44:10.874 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:10.874 INFO:teuthology.orchestra.run.smithi129.stdout:Complete! 2024-09-14T03:44:11.268 DEBUG:teuthology.orchestra.run.smithi129:> sudo yum -y install ceph-base 2024-09-14T03:44:11.833 INFO:teuthology.orchestra.run.smithi129.stdout:Last metadata expiration check: 0:00:49 ago on Sat 14 Sep 2024 03:43:22 AM UTC. 2024-09-14T03:44:11.924 INFO:teuthology.orchestra.run.smithi129.stdout:Package ceph-base-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:44:11.951 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: ceph-2:18.2.4-845.ge4e227b3.el9.x86_64 33/33 2024-09-14T03:44:11.951 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : ceph-2:18.2.4-845.ge4e227b3.el9.x86_64 1/33 2024-09-14T03:44:11.951 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : ceph-mds-2:18.2.4-845.ge4e227b3.el9.x86_64 2/33 2024-09-14T03:44:11.952 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : ceph-mgr-2:18.2.4-845.ge4e227b3.el9.x86_64 3/33 2024-09-14T03:44:11.952 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : ceph-mon-2:18.2.4-845.ge4e227b3.el9.x86_64 4/33 2024-09-14T03:44:11.952 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : ceph-osd-2:18.2.4-845.ge4e227b3.el9.x86_64 5/33 2024-09-14T03:44:11.952 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : ceph-mgr-modules-core-2:18.2.4-845.ge4e227b3.el9.n 6/33 2024-09-14T03:44:11.952 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 7/33 2024-09-14T03:44:11.952 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 8/33 2024-09-14T03:44:11.953 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-ply-3.11-14.el9.noarch 9/33 2024-09-14T03:44:11.953 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 10/33 2024-09-14T03:44:11.953 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 11/33 2024-09-14T03:44:11.953 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-urllib3-1.26.5-6.el9.noarch 12/33 2024-09-14T03:44:11.953 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 13/33 2024-09-14T03:44:11.954 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 14/33 2024-09-14T03:44:11.954 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 15/33 2024-09-14T03:44:11.954 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 16/33 2024-09-14T03:44:11.954 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-09-14T03:44:11.954 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-09-14T03:44:11.954 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-09-14T03:44:11.955 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-09-14T03:44:11.955 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-09-14T03:44:11.955 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-09-14T03:44:11.955 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-09-14T03:44:11.955 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-09-14T03:44:11.955 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-09-14T03:44:11.956 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-09-14T03:44:11.956 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-09-14T03:44:11.956 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-09-14T03:44:11.956 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-09-14T03:44:11.956 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-09-14T03:44:11.957 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 31/33 2024-09-14T03:44:11.957 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-09-14T03:44:11.966 INFO:teuthology.orchestra.run.smithi129.stdout:Dependencies resolved. 2024-09-14T03:44:11.967 INFO:teuthology.orchestra.run.smithi129.stdout:Nothing to do. 2024-09-14T03:44:11.967 INFO:teuthology.orchestra.run.smithi129.stdout:Complete! 2024-09-14T03:44:12.030 DEBUG:teuthology.orchestra.run.smithi129:> sudo yum -y install cephadm 2024-09-14T03:44:12.591 INFO:teuthology.orchestra.run.smithi129.stdout:Last metadata expiration check: 0:00:50 ago on Sat 14 Sep 2024 03:43:22 AM UTC. 2024-09-14T03:44:12.723 INFO:teuthology.orchestra.run.smithi129.stdout:Dependencies resolved. 2024-09-14T03:44:12.724 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:12.724 INFO:teuthology.orchestra.run.smithi129.stdout: Package Arch Version Repository Size 2024-09-14T03:44:12.724 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:12.724 INFO:teuthology.orchestra.run.smithi129.stdout:Installing: 2024-09-14T03:44:12.724 INFO:teuthology.orchestra.run.smithi129.stdout: cephadm noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 226 k 2024-09-14T03:44:12.724 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:12.724 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction Summary 2024-09-14T03:44:12.724 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:12.724 INFO:teuthology.orchestra.run.smithi129.stdout:Install 1 Package 2024-09-14T03:44:12.724 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:12.724 INFO:teuthology.orchestra.run.smithi129.stdout:Total download size: 226 k 2024-09-14T03:44:12.725 INFO:teuthology.orchestra.run.smithi129.stdout:Installed size: 217 k 2024-09-14T03:44:12.725 INFO:teuthology.orchestra.run.smithi129.stdout:Downloading Packages: 2024-09-14T03:44:12.945 INFO:teuthology.orchestra.run.smithi129.stdout:cephadm-18.2.4-845.ge4e227b3.el9.noarch.rpm 1.0 MB/s | 226 kB 00:00 2024-09-14T03:44:12.945 INFO:teuthology.orchestra.run.smithi129.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:44:12.946 INFO:teuthology.orchestra.run.smithi129.stdout:Total 1.0 MB/s | 226 kB 00:00 2024-09-14T03:44:12.946 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction check 2024-09-14T03:44:12.947 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-09-14T03:44:12.947 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:12.947 INFO:teuthology.orchestra.run.smithi039.stdout:Installed: 2024-09-14T03:44:12.947 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:44:12.947 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-mds-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:44:12.947 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-mgr-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:44:12.947 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-mgr-modules-core-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:44:12.947 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-mon-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:44:12.947 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-osd-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:44:12.947 INFO:teuthology.orchestra.run.smithi039.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-09-14T03:44:12.948 INFO:teuthology.orchestra.run.smithi039.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-09-14T03:44:12.948 INFO:teuthology.orchestra.run.smithi039.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-09-14T03:44:12.948 INFO:teuthology.orchestra.run.smithi039.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-09-14T03:44:12.948 INFO:teuthology.orchestra.run.smithi039.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-09-14T03:44:12.948 INFO:teuthology.orchestra.run.smithi039.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-09-14T03:44:12.948 INFO:teuthology.orchestra.run.smithi039.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-09-14T03:44:12.948 INFO:teuthology.orchestra.run.smithi039.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-09-14T03:44:12.948 INFO:teuthology.orchestra.run.smithi039.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-09-14T03:44:12.948 INFO:teuthology.orchestra.run.smithi039.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-09-14T03:44:12.948 INFO:teuthology.orchestra.run.smithi039.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-09-14T03:44:12.948 INFO:teuthology.orchestra.run.smithi039.stdout: python3-mako-1.1.4-6.el9.noarch 2024-09-14T03:44:12.948 INFO:teuthology.orchestra.run.smithi039.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-09-14T03:44:12.949 INFO:teuthology.orchestra.run.smithi039.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-09-14T03:44:12.949 INFO:teuthology.orchestra.run.smithi039.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-09-14T03:44:12.949 INFO:teuthology.orchestra.run.smithi039.stdout: python3-ply-3.11-14.el9.noarch 2024-09-14T03:44:12.949 INFO:teuthology.orchestra.run.smithi039.stdout: python3-portend-3.1.0-2.el9.noarch 2024-09-14T03:44:12.949 INFO:teuthology.orchestra.run.smithi039.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-09-14T03:44:12.949 INFO:teuthology.orchestra.run.smithi039.stdout: python3-pycparser-2.20-6.el9.noarch 2024-09-14T03:44:12.949 INFO:teuthology.orchestra.run.smithi039.stdout: python3-pytz-2021.1-5.el9.noarch 2024-09-14T03:44:12.949 INFO:teuthology.orchestra.run.smithi039.stdout: python3-requests-2.25.1-8.el9.noarch 2024-09-14T03:44:12.949 INFO:teuthology.orchestra.run.smithi039.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-09-14T03:44:12.949 INFO:teuthology.orchestra.run.smithi039.stdout: python3-toml-0.10.2-6.el9.noarch 2024-09-14T03:44:12.949 INFO:teuthology.orchestra.run.smithi039.stdout: python3-urllib3-1.26.5-6.el9.noarch 2024-09-14T03:44:12.949 INFO:teuthology.orchestra.run.smithi039.stdout: python3-webob-1.8.8-2.el9.noarch 2024-09-14T03:44:12.949 INFO:teuthology.orchestra.run.smithi039.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-09-14T03:44:12.949 INFO:teuthology.orchestra.run.smithi039.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-09-14T03:44:12.950 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:12.950 INFO:teuthology.orchestra.run.smithi039.stdout:Complete! 2024-09-14T03:44:12.951 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction check succeeded. 2024-09-14T03:44:12.951 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction test 2024-09-14T03:44:12.954 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction test succeeded. 2024-09-14T03:44:12.954 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction 2024-09-14T03:44:13.025 INFO:teuthology.orchestra.run.smithi129.stdout: Preparing : 1/1 2024-09-14T03:44:13.369 DEBUG:teuthology.orchestra.run.smithi039:> sudo yum -y install ceph-base 2024-09-14T03:44:13.458 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 1/1 2024-09-14T03:44:13.532 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 1/1 2024-09-14T03:44:13.929 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 1/1 2024-09-14T03:44:13.970 INFO:teuthology.orchestra.run.smithi039.stdout:Last metadata expiration check: 0:00:49 ago on Sat 14 Sep 2024 03:43:24 AM UTC. 2024-09-14T03:44:14.065 INFO:teuthology.orchestra.run.smithi039.stdout:Package ceph-base-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:44:14.110 INFO:teuthology.orchestra.run.smithi039.stdout:Dependencies resolved. 2024-09-14T03:44:14.111 INFO:teuthology.orchestra.run.smithi039.stdout:Nothing to do. 2024-09-14T03:44:14.111 INFO:teuthology.orchestra.run.smithi039.stdout:Complete! 2024-09-14T03:44:14.181 DEBUG:teuthology.orchestra.run.smithi039:> sudo yum -y install cephadm 2024-09-14T03:44:14.194 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 1/1 2024-09-14T03:44:14.194 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:14.194 INFO:teuthology.orchestra.run.smithi129.stdout:Installed: 2024-09-14T03:44:14.194 INFO:teuthology.orchestra.run.smithi129.stdout: cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:44:14.194 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:14.194 INFO:teuthology.orchestra.run.smithi129.stdout:Complete! 2024-09-14T03:44:14.429 DEBUG:teuthology.orchestra.run.smithi129:> sudo yum -y install ceph-immutable-object-cache 2024-09-14T03:44:14.763 INFO:teuthology.orchestra.run.smithi039.stdout:Last metadata expiration check: 0:00:50 ago on Sat 14 Sep 2024 03:43:24 AM UTC. 2024-09-14T03:44:14.900 INFO:teuthology.orchestra.run.smithi039.stdout:Dependencies resolved. 2024-09-14T03:44:14.901 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:44:14.901 INFO:teuthology.orchestra.run.smithi039.stdout: Package Arch Version Repository Size 2024-09-14T03:44:14.901 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:44:14.901 INFO:teuthology.orchestra.run.smithi039.stdout:Installing: 2024-09-14T03:44:14.901 INFO:teuthology.orchestra.run.smithi039.stdout: cephadm noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 226 k 2024-09-14T03:44:14.902 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:14.902 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction Summary 2024-09-14T03:44:14.902 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:44:14.902 INFO:teuthology.orchestra.run.smithi039.stdout:Install 1 Package 2024-09-14T03:44:14.902 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:14.902 INFO:teuthology.orchestra.run.smithi039.stdout:Total download size: 226 k 2024-09-14T03:44:14.902 INFO:teuthology.orchestra.run.smithi039.stdout:Installed size: 217 k 2024-09-14T03:44:14.902 INFO:teuthology.orchestra.run.smithi039.stdout:Downloading Packages: 2024-09-14T03:44:15.002 INFO:teuthology.orchestra.run.smithi129.stdout:Last metadata expiration check: 0:00:53 ago on Sat 14 Sep 2024 03:43:22 AM UTC. 2024-09-14T03:44:15.134 INFO:teuthology.orchestra.run.smithi039.stdout:cephadm-18.2.4-845.ge4e227b3.el9.noarch.rpm 977 kB/s | 226 kB 00:00 2024-09-14T03:44:15.134 INFO:teuthology.orchestra.run.smithi039.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:44:15.134 INFO:teuthology.orchestra.run.smithi039.stdout:Total 971 kB/s | 226 kB 00:00 2024-09-14T03:44:15.135 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction check 2024-09-14T03:44:15.135 INFO:teuthology.orchestra.run.smithi129.stdout:Dependencies resolved. 2024-09-14T03:44:15.137 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:15.137 INFO:teuthology.orchestra.run.smithi129.stdout: Package Arch Version Repo Size 2024-09-14T03:44:15.137 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:15.138 INFO:teuthology.orchestra.run.smithi129.stdout:Installing: 2024-09-14T03:44:15.138 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-immutable-object-cache x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 143 k 2024-09-14T03:44:15.138 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:15.138 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction Summary 2024-09-14T03:44:15.138 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:15.138 INFO:teuthology.orchestra.run.smithi129.stdout:Install 1 Package 2024-09-14T03:44:15.138 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:15.138 INFO:teuthology.orchestra.run.smithi129.stdout:Total download size: 143 k 2024-09-14T03:44:15.138 INFO:teuthology.orchestra.run.smithi129.stdout:Installed size: 395 k 2024-09-14T03:44:15.138 INFO:teuthology.orchestra.run.smithi129.stdout:Downloading Packages: 2024-09-14T03:44:15.140 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction check succeeded. 2024-09-14T03:44:15.140 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction test 2024-09-14T03:44:15.143 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction test succeeded. 2024-09-14T03:44:15.143 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction 2024-09-14T03:44:15.212 INFO:teuthology.orchestra.run.smithi039.stdout: Preparing : 1/1 2024-09-14T03:44:15.335 INFO:teuthology.orchestra.run.smithi129.stdout:ceph-immutable-object-cache-18.2.4-845.ge4e227b 721 kB/s | 143 kB 00:00 2024-09-14T03:44:15.336 INFO:teuthology.orchestra.run.smithi129.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:44:15.336 INFO:teuthology.orchestra.run.smithi129.stdout:Total 715 kB/s | 143 kB 00:00 2024-09-14T03:44:15.337 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction check 2024-09-14T03:44:15.344 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction check succeeded. 2024-09-14T03:44:15.344 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction test 2024-09-14T03:44:15.402 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction test succeeded. 2024-09-14T03:44:15.402 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction 2024-09-14T03:44:15.555 INFO:teuthology.orchestra.run.smithi129.stdout: Preparing : 1/1 2024-09-14T03:44:15.637 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : ceph-immutable-object-cache-2:18.2.4-845.ge4e227b3.e 1/1 2024-09-14T03:44:15.662 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.2.4-845.ge4e227b3.e 1/1 2024-09-14T03:44:15.662 INFO:teuthology.orchestra.run.smithi129.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:44:15.662 INFO:teuthology.orchestra.run.smithi129.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-14T03:44:15.663 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:15.665 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 1/1 2024-09-14T03:44:15.750 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 1/1 2024-09-14T03:44:16.141 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 1/1 2024-09-14T03:44:16.410 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 1/1 2024-09-14T03:44:16.410 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:16.411 INFO:teuthology.orchestra.run.smithi039.stdout:Installed: 2024-09-14T03:44:16.411 INFO:teuthology.orchestra.run.smithi039.stdout: cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:44:16.411 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:16.411 INFO:teuthology.orchestra.run.smithi039.stdout:Complete! 2024-09-14T03:44:16.551 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : ceph-immutable-object-cache-2:18.2.4-845.ge4e227b3.e 1/1 2024-09-14T03:44:16.551 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:16.551 INFO:teuthology.orchestra.run.smithi129.stdout:Installed: 2024-09-14T03:44:16.551 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-immutable-object-cache-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:44:16.551 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:16.551 INFO:teuthology.orchestra.run.smithi129.stdout:Complete! 2024-09-14T03:44:16.622 DEBUG:teuthology.orchestra.run.smithi039:> sudo yum -y install ceph-immutable-object-cache 2024-09-14T03:44:16.770 DEBUG:teuthology.orchestra.run.smithi129:> sudo yum -y install ceph-mgr 2024-09-14T03:44:17.211 INFO:teuthology.orchestra.run.smithi039.stdout:Last metadata expiration check: 0:00:53 ago on Sat 14 Sep 2024 03:43:24 AM UTC. 2024-09-14T03:44:17.353 INFO:teuthology.orchestra.run.smithi039.stdout:Dependencies resolved. 2024-09-14T03:44:17.354 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:44:17.354 INFO:teuthology.orchestra.run.smithi039.stdout: Package Arch Version Repo Size 2024-09-14T03:44:17.354 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:44:17.354 INFO:teuthology.orchestra.run.smithi039.stdout:Installing: 2024-09-14T03:44:17.354 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-immutable-object-cache x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 143 k 2024-09-14T03:44:17.354 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:17.354 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction Summary 2024-09-14T03:44:17.354 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:44:17.354 INFO:teuthology.orchestra.run.smithi039.stdout:Install 1 Package 2024-09-14T03:44:17.355 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:17.355 INFO:teuthology.orchestra.run.smithi039.stdout:Total download size: 143 k 2024-09-14T03:44:17.355 INFO:teuthology.orchestra.run.smithi039.stdout:Installed size: 395 k 2024-09-14T03:44:17.355 INFO:teuthology.orchestra.run.smithi039.stdout:Downloading Packages: 2024-09-14T03:44:17.365 INFO:teuthology.orchestra.run.smithi129.stdout:Last metadata expiration check: 0:00:55 ago on Sat 14 Sep 2024 03:43:22 AM UTC. 2024-09-14T03:44:17.455 INFO:teuthology.orchestra.run.smithi129.stdout:Package ceph-mgr-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:44:17.498 INFO:teuthology.orchestra.run.smithi129.stdout:Dependencies resolved. 2024-09-14T03:44:17.499 INFO:teuthology.orchestra.run.smithi129.stdout:Nothing to do. 2024-09-14T03:44:17.499 INFO:teuthology.orchestra.run.smithi129.stdout:Complete! 2024-09-14T03:44:17.564 DEBUG:teuthology.orchestra.run.smithi129:> sudo yum -y install ceph-mgr-dashboard 2024-09-14T03:44:17.574 INFO:teuthology.orchestra.run.smithi039.stdout:ceph-immutable-object-cache-18.2.4-845.ge4e227b 653 kB/s | 143 kB 00:00 2024-09-14T03:44:17.574 INFO:teuthology.orchestra.run.smithi039.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:44:17.574 INFO:teuthology.orchestra.run.smithi039.stdout:Total 648 kB/s | 143 kB 00:00 2024-09-14T03:44:17.574 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction check 2024-09-14T03:44:17.582 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction check succeeded. 2024-09-14T03:44:17.582 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction test 2024-09-14T03:44:17.643 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction test succeeded. 2024-09-14T03:44:17.643 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction 2024-09-14T03:44:17.795 INFO:teuthology.orchestra.run.smithi039.stdout: Preparing : 1/1 2024-09-14T03:44:17.866 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : ceph-immutable-object-cache-2:18.2.4-845.ge4e227b3.e 1/1 2024-09-14T03:44:17.893 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: ceph-immutable-object-cache-2:18.2.4-845.ge4e227b3.e 1/1 2024-09-14T03:44:17.893 INFO:teuthology.orchestra.run.smithi039.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:44:17.893 INFO:teuthology.orchestra.run.smithi039.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-09-14T03:44:17.893 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:18.122 INFO:teuthology.orchestra.run.smithi129.stdout:Last metadata expiration check: 0:00:56 ago on Sat 14 Sep 2024 03:43:22 AM UTC. 2024-09-14T03:44:18.252 INFO:teuthology.orchestra.run.smithi129.stdout:Dependencies resolved. 2024-09-14T03:44:18.254 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:18.254 INFO:teuthology.orchestra.run.smithi129.stdout: Package Arch Version Repository Size 2024-09-14T03:44:18.254 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:18.254 INFO:teuthology.orchestra.run.smithi129.stdout:Installing: 2024-09-14T03:44:18.254 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-mgr-dashboard noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 3.5 M 2024-09-14T03:44:18.254 INFO:teuthology.orchestra.run.smithi129.stdout:Installing dependencies: 2024-09-14T03:44:18.254 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-grafana-dashboards noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 24 k 2024-09-14T03:44:18.254 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-prometheus-alerts noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 16 k 2024-09-14T03:44:18.254 INFO:teuthology.orchestra.run.smithi129.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-09-14T03:44:18.254 INFO:teuthology.orchestra.run.smithi129.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-09-14T03:44:18.254 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:18.254 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction Summary 2024-09-14T03:44:18.255 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:18.255 INFO:teuthology.orchestra.run.smithi129.stdout:Install 5 Packages 2024-09-14T03:44:18.255 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:18.255 INFO:teuthology.orchestra.run.smithi129.stdout:Total download size: 3.8 M 2024-09-14T03:44:18.255 INFO:teuthology.orchestra.run.smithi129.stdout:Installed size: 87 M 2024-09-14T03:44:18.255 INFO:teuthology.orchestra.run.smithi129.stdout:Downloading Packages: 2024-09-14T03:44:18.558 INFO:teuthology.orchestra.run.smithi129.stdout:(1/5): ceph-grafana-dashboards-18.2.4-845.ge4e2 164 kB/s | 24 kB 00:00 2024-09-14T03:44:18.583 INFO:teuthology.orchestra.run.smithi129.stdout:(2/5): ceph-prometheus-alerts-18.2.4-845.ge4e22 94 kB/s | 16 kB 00:00 2024-09-14T03:44:18.625 INFO:teuthology.orchestra.run.smithi129.stdout:(3/5): python3-repoze-lru-0.7-16.el9.noarch.rpm 461 kB/s | 31 kB 00:00 2024-09-14T03:44:18.650 INFO:teuthology.orchestra.run.smithi129.stdout:(4/5): python3-routes-2.5.1-5.el9.noarch.rpm 2.8 MB/s | 188 kB 00:00 2024-09-14T03:44:18.817 INFO:teuthology.orchestra.run.smithi129.stdout:(5/5): ceph-mgr-dashboard-18.2.4-845.ge4e227b3. 8.7 MB/s | 3.5 MB 00:00 2024-09-14T03:44:18.820 INFO:teuthology.orchestra.run.smithi129.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:44:18.821 INFO:teuthology.orchestra.run.smithi129.stdout:Total 6.7 MB/s | 3.8 MB 00:00 2024-09-14T03:44:18.821 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction check 2024-09-14T03:44:18.829 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction check succeeded. 2024-09-14T03:44:18.829 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction test 2024-09-14T03:44:18.861 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : ceph-immutable-object-cache-2:18.2.4-845.ge4e227b3.e 1/1 2024-09-14T03:44:18.861 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:18.861 INFO:teuthology.orchestra.run.smithi039.stdout:Installed: 2024-09-14T03:44:18.861 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-immutable-object-cache-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:44:18.861 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:18.861 INFO:teuthology.orchestra.run.smithi039.stdout:Complete! 2024-09-14T03:44:18.914 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction test succeeded. 2024-09-14T03:44:18.914 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction 2024-09-14T03:44:19.061 DEBUG:teuthology.orchestra.run.smithi039:> sudo yum -y install ceph-mgr 2024-09-14T03:44:19.157 INFO:teuthology.orchestra.run.smithi129.stdout: Preparing : 1/1 2024-09-14T03:44:19.264 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 1/5 2024-09-14T03:44:19.306 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 2/5 2024-09-14T03:44:19.334 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : ceph-prometheus-alerts-2:18.2.4-845.ge4e227b3.el9.no 3/5 2024-09-14T03:44:19.655 INFO:teuthology.orchestra.run.smithi039.stdout:Last metadata expiration check: 0:00:55 ago on Sat 14 Sep 2024 03:43:24 AM UTC. 2024-09-14T03:44:19.749 INFO:teuthology.orchestra.run.smithi039.stdout:Package ceph-mgr-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:44:19.794 INFO:teuthology.orchestra.run.smithi039.stdout:Dependencies resolved. 2024-09-14T03:44:19.795 INFO:teuthology.orchestra.run.smithi039.stdout:Nothing to do. 2024-09-14T03:44:19.795 INFO:teuthology.orchestra.run.smithi039.stdout:Complete! 2024-09-14T03:44:19.863 DEBUG:teuthology.orchestra.run.smithi039:> sudo yum -y install ceph-mgr-dashboard 2024-09-14T03:44:20.378 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : ceph-grafana-dashboards-2:18.2.4-845.ge4e227b3.el9.n 4/5 2024-09-14T03:44:20.422 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : ceph-mgr-dashboard-2:18.2.4-845.ge4e227b3.el9.noarch 5/5 2024-09-14T03:44:20.449 INFO:teuthology.orchestra.run.smithi039.stdout:Last metadata expiration check: 0:00:56 ago on Sat 14 Sep 2024 03:43:24 AM UTC. 2024-09-14T03:44:20.587 INFO:teuthology.orchestra.run.smithi039.stdout:Dependencies resolved. 2024-09-14T03:44:20.587 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:44:20.587 INFO:teuthology.orchestra.run.smithi039.stdout: Package Arch Version Repository Size 2024-09-14T03:44:20.587 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:44:20.588 INFO:teuthology.orchestra.run.smithi039.stdout:Installing: 2024-09-14T03:44:20.588 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-mgr-dashboard noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 3.5 M 2024-09-14T03:44:20.588 INFO:teuthology.orchestra.run.smithi039.stdout:Installing dependencies: 2024-09-14T03:44:20.588 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-grafana-dashboards noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 24 k 2024-09-14T03:44:20.588 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-prometheus-alerts noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 16 k 2024-09-14T03:44:20.588 INFO:teuthology.orchestra.run.smithi039.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-09-14T03:44:20.588 INFO:teuthology.orchestra.run.smithi039.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-09-14T03:44:20.588 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:20.588 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction Summary 2024-09-14T03:44:20.588 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:44:20.588 INFO:teuthology.orchestra.run.smithi039.stdout:Install 5 Packages 2024-09-14T03:44:20.588 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:20.589 INFO:teuthology.orchestra.run.smithi039.stdout:Total download size: 3.8 M 2024-09-14T03:44:20.589 INFO:teuthology.orchestra.run.smithi039.stdout:Installed size: 87 M 2024-09-14T03:44:20.589 INFO:teuthology.orchestra.run.smithi039.stdout:Downloading Packages: 2024-09-14T03:44:20.769 INFO:teuthology.orchestra.run.smithi039.stdout:(1/5): ceph-grafana-dashboards-18.2.4-845.ge4e2 163 kB/s | 24 kB 00:00 2024-09-14T03:44:20.795 INFO:teuthology.orchestra.run.smithi039.stdout:(2/5): ceph-prometheus-alerts-18.2.4-845.ge4e22 94 kB/s | 16 kB 00:00 2024-09-14T03:44:20.836 INFO:teuthology.orchestra.run.smithi039.stdout:(3/5): python3-repoze-lru-0.7-16.el9.noarch.rpm 459 kB/s | 31 kB 00:00 2024-09-14T03:44:20.870 INFO:teuthology.orchestra.run.smithi039.stdout:(4/5): python3-routes-2.5.1-5.el9.noarch.rpm 2.4 MB/s | 188 kB 00:00 2024-09-14T03:44:21.037 INFO:teuthology.orchestra.run.smithi039.stdout:(5/5): ceph-mgr-dashboard-18.2.4-845.ge4e227b3. 8.5 MB/s | 3.5 MB 00:00 2024-09-14T03:44:21.041 INFO:teuthology.orchestra.run.smithi039.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:44:21.041 INFO:teuthology.orchestra.run.smithi039.stdout:Total 8.4 MB/s | 3.8 MB 00:00 2024-09-14T03:44:21.041 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction check 2024-09-14T03:44:21.048 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction check succeeded. 2024-09-14T03:44:21.049 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction test 2024-09-14T03:44:21.136 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction test succeeded. 2024-09-14T03:44:21.137 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction 2024-09-14T03:44:21.398 INFO:teuthology.orchestra.run.smithi039.stdout: Preparing : 1/1 2024-09-14T03:44:21.486 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 1/5 2024-09-14T03:44:21.525 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 2/5 2024-09-14T03:44:21.535 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: ceph-mgr-dashboard-2:18.2.4-845.ge4e227b3.el9.noarch 5/5 2024-09-14T03:44:21.536 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : ceph-grafana-dashboards-2:18.2.4-845.ge4e227b3.el9.n 1/5 2024-09-14T03:44:21.536 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : ceph-mgr-dashboard-2:18.2.4-845.ge4e227b3.el9.noarch 2/5 2024-09-14T03:44:21.536 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : ceph-prometheus-alerts-2:18.2.4-845.ge4e227b3.el9.no 3/5 2024-09-14T03:44:21.536 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 4/5 2024-09-14T03:44:21.554 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : ceph-prometheus-alerts-2:18.2.4-845.ge4e227b3.el9.no 3/5 2024-09-14T03:44:21.800 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 5/5 2024-09-14T03:44:21.800 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:21.800 INFO:teuthology.orchestra.run.smithi129.stdout:Installed: 2024-09-14T03:44:21.800 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-grafana-dashboards-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:44:21.800 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-mgr-dashboard-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:44:21.801 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-prometheus-alerts-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:44:21.801 INFO:teuthology.orchestra.run.smithi129.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-09-14T03:44:21.801 INFO:teuthology.orchestra.run.smithi129.stdout: python3-routes-2.5.1-5.el9.noarch 2024-09-14T03:44:21.801 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:21.801 INFO:teuthology.orchestra.run.smithi129.stdout:Complete! 2024-09-14T03:44:22.020 DEBUG:teuthology.orchestra.run.smithi129:> sudo yum -y install ceph-mgr-diskprediction-local 2024-09-14T03:44:22.578 INFO:teuthology.orchestra.run.smithi129.stdout:Last metadata expiration check: 0:01:00 ago on Sat 14 Sep 2024 03:43:22 AM UTC. 2024-09-14T03:44:22.670 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : ceph-grafana-dashboards-2:18.2.4-845.ge4e227b3.el9.n 4/5 2024-09-14T03:44:22.708 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : ceph-mgr-dashboard-2:18.2.4-845.ge4e227b3.el9.noarch 5/5 2024-09-14T03:44:22.711 INFO:teuthology.orchestra.run.smithi129.stdout:Dependencies resolved. 2024-09-14T03:44:22.712 INFO:teuthology.orchestra.run.smithi129.stdout:====================================================================================== 2024-09-14T03:44:22.712 INFO:teuthology.orchestra.run.smithi129.stdout: Package Arch Version Repository Size 2024-09-14T03:44:22.712 INFO:teuthology.orchestra.run.smithi129.stdout:====================================================================================== 2024-09-14T03:44:22.712 INFO:teuthology.orchestra.run.smithi129.stdout:Installing: 2024-09-14T03:44:22.712 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-mgr-diskprediction-local noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 7.4 M 2024-09-14T03:44:22.712 INFO:teuthology.orchestra.run.smithi129.stdout:Installing dependencies: 2024-09-14T03:44:22.712 INFO:teuthology.orchestra.run.smithi129.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-09-14T03:44:22.713 INFO:teuthology.orchestra.run.smithi129.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-09-14T03:44:22.713 INFO:teuthology.orchestra.run.smithi129.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-09-14T03:44:22.713 INFO:teuthology.orchestra.run.smithi129.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-09-14T03:44:22.713 INFO:teuthology.orchestra.run.smithi129.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-09-14T03:44:22.713 INFO:teuthology.orchestra.run.smithi129.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-09-14T03:44:22.713 INFO:teuthology.orchestra.run.smithi129.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-09-14T03:44:22.713 INFO:teuthology.orchestra.run.smithi129.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-09-14T03:44:22.713 INFO:teuthology.orchestra.run.smithi129.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-09-14T03:44:22.713 INFO:teuthology.orchestra.run.smithi129.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-09-14T03:44:22.713 INFO:teuthology.orchestra.run.smithi129.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-09-14T03:44:22.713 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:22.713 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction Summary 2024-09-14T03:44:22.714 INFO:teuthology.orchestra.run.smithi129.stdout:====================================================================================== 2024-09-14T03:44:22.714 INFO:teuthology.orchestra.run.smithi129.stdout:Install 12 Packages 2024-09-14T03:44:22.714 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:22.714 INFO:teuthology.orchestra.run.smithi129.stdout:Total download size: 38 M 2024-09-14T03:44:22.714 INFO:teuthology.orchestra.run.smithi129.stdout:Installed size: 212 M 2024-09-14T03:44:22.714 INFO:teuthology.orchestra.run.smithi129.stdout:Downloading Packages: 2024-09-14T03:44:23.017 INFO:teuthology.orchestra.run.smithi129.stdout:(1/12): libquadmath-11.5.0-2.el9.x86_64.rpm 1.0 MB/s | 191 kB 00:00 2024-09-14T03:44:23.134 INFO:teuthology.orchestra.run.smithi129.stdout:(2/12): libgfortran-11.5.0-2.el9.x86_64.rpm 2.6 MB/s | 801 kB 00:00 2024-09-14T03:44:23.150 INFO:teuthology.orchestra.run.smithi129.stdout:(3/12): flexiblas-3.0.4-8.el9.x86_64.rpm 245 kB/s | 33 kB 00:00 2024-09-14T03:44:23.201 INFO:teuthology.orchestra.run.smithi129.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 362 kB/s | 18 kB 00:00 2024-09-14T03:44:23.301 INFO:teuthology.orchestra.run.smithi129.stdout:(5/12): ceph-mgr-diskprediction-local-18.2.4-84 16 MB/s | 7.4 MB 00:00 2024-09-14T03:44:23.327 INFO:teuthology.orchestra.run.smithi129.stdout:(6/12): openblas-0.3.26-2.el9.x86_64.rpm 307 kB/s | 38 kB 00:00 2024-09-14T03:44:23.435 INFO:teuthology.orchestra.run.smithi129.stdout:(7/12): python3-devel-3.9.19-8.el9.x86_64.rpm 2.2 MB/s | 245 kB 00:00 2024-09-14T03:44:23.586 INFO:teuthology.orchestra.run.smithi129.stdout:(8/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 6.6 MB/s | 3.0 MB 00:00 2024-09-14T03:44:23.653 INFO:teuthology.orchestra.run.smithi129.stdout:(9/12): python3-numpy-f2py-1.20.1-5.el9.x86_64. 3.7 MB/s | 250 kB 00:00 2024-09-14T03:44:23.820 INFO:teuthology.orchestra.run.smithi129.stdout:(10/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 13 MB/s | 5.1 MB 00:00 2024-09-14T03:44:23.847 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: ceph-mgr-dashboard-2:18.2.4-845.ge4e227b3.el9.noarch 5/5 2024-09-14T03:44:23.847 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : ceph-grafana-dashboards-2:18.2.4-845.ge4e227b3.el9.n 1/5 2024-09-14T03:44:23.847 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : ceph-mgr-dashboard-2:18.2.4-845.ge4e227b3.el9.noarch 2/5 2024-09-14T03:44:23.847 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : ceph-prometheus-alerts-2:18.2.4-845.ge4e227b3.el9.no 3/5 2024-09-14T03:44:23.847 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 4/5 2024-09-14T03:44:24.117 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 5/5 2024-09-14T03:44:24.118 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:24.118 INFO:teuthology.orchestra.run.smithi039.stdout:Installed: 2024-09-14T03:44:24.118 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-grafana-dashboards-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:44:24.118 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-mgr-dashboard-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:44:24.118 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-prometheus-alerts-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:44:24.118 INFO:teuthology.orchestra.run.smithi039.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-09-14T03:44:24.118 INFO:teuthology.orchestra.run.smithi039.stdout: python3-routes-2.5.1-5.el9.noarch 2024-09-14T03:44:24.119 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:24.119 INFO:teuthology.orchestra.run.smithi039.stdout:Complete! 2024-09-14T03:44:24.120 INFO:teuthology.orchestra.run.smithi129.stdout:(11/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 35 MB/s | 16 MB 00:00 2024-09-14T03:44:24.220 INFO:teuthology.orchestra.run.smithi129.stdout:(12/12): openblas-openmp-0.3.26-2.el9.x86_64.rp 5.4 MB/s | 4.9 MB 00:00 2024-09-14T03:44:24.221 INFO:teuthology.orchestra.run.smithi129.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:44:24.221 INFO:teuthology.orchestra.run.smithi129.stdout:Total 25 MB/s | 38 MB 00:01 2024-09-14T03:44:24.341 DEBUG:teuthology.orchestra.run.smithi039:> sudo yum -y install ceph-mgr-diskprediction-local 2024-09-14T03:44:24.575 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction check 2024-09-14T03:44:24.600 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction check succeeded. 2024-09-14T03:44:24.600 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction test 2024-09-14T03:44:24.928 INFO:teuthology.orchestra.run.smithi039.stdout:Last metadata expiration check: 0:01:00 ago on Sat 14 Sep 2024 03:43:24 AM UTC. 2024-09-14T03:44:25.068 INFO:teuthology.orchestra.run.smithi039.stdout:Dependencies resolved. 2024-09-14T03:44:25.069 INFO:teuthology.orchestra.run.smithi039.stdout:====================================================================================== 2024-09-14T03:44:25.069 INFO:teuthology.orchestra.run.smithi039.stdout: Package Arch Version Repository Size 2024-09-14T03:44:25.069 INFO:teuthology.orchestra.run.smithi039.stdout:====================================================================================== 2024-09-14T03:44:25.069 INFO:teuthology.orchestra.run.smithi039.stdout:Installing: 2024-09-14T03:44:25.069 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-mgr-diskprediction-local noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 7.4 M 2024-09-14T03:44:25.069 INFO:teuthology.orchestra.run.smithi039.stdout:Installing dependencies: 2024-09-14T03:44:25.069 INFO:teuthology.orchestra.run.smithi039.stdout: flexiblas x86_64 3.0.4-8.el9 appstream 33 k 2024-09-14T03:44:25.069 INFO:teuthology.orchestra.run.smithi039.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 appstream 3.0 M 2024-09-14T03:44:25.069 INFO:teuthology.orchestra.run.smithi039.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 appstream 18 k 2024-09-14T03:44:25.069 INFO:teuthology.orchestra.run.smithi039.stdout: libgfortran x86_64 11.5.0-2.el9 baseos 801 k 2024-09-14T03:44:25.069 INFO:teuthology.orchestra.run.smithi039.stdout: libquadmath x86_64 11.5.0-2.el9 baseos 191 k 2024-09-14T03:44:25.070 INFO:teuthology.orchestra.run.smithi039.stdout: openblas x86_64 0.3.26-2.el9 appstream 38 k 2024-09-14T03:44:25.070 INFO:teuthology.orchestra.run.smithi039.stdout: openblas-openmp x86_64 0.3.26-2.el9 appstream 4.9 M 2024-09-14T03:44:25.070 INFO:teuthology.orchestra.run.smithi039.stdout: python3-devel x86_64 3.9.19-8.el9 appstream 245 k 2024-09-14T03:44:25.070 INFO:teuthology.orchestra.run.smithi039.stdout: python3-numpy x86_64 1:1.20.1-5.el9 appstream 5.1 M 2024-09-14T03:44:25.070 INFO:teuthology.orchestra.run.smithi039.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 appstream 250 k 2024-09-14T03:44:25.070 INFO:teuthology.orchestra.run.smithi039.stdout: python3-scipy x86_64 1.6.2-8.el9 appstream 16 M 2024-09-14T03:44:25.070 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:25.070 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction Summary 2024-09-14T03:44:25.070 INFO:teuthology.orchestra.run.smithi039.stdout:====================================================================================== 2024-09-14T03:44:25.070 INFO:teuthology.orchestra.run.smithi039.stdout:Install 12 Packages 2024-09-14T03:44:25.070 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:25.071 INFO:teuthology.orchestra.run.smithi039.stdout:Total download size: 38 M 2024-09-14T03:44:25.071 INFO:teuthology.orchestra.run.smithi039.stdout:Installed size: 212 M 2024-09-14T03:44:25.071 INFO:teuthology.orchestra.run.smithi039.stdout:Downloading Packages: 2024-09-14T03:44:25.084 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction test succeeded. 2024-09-14T03:44:25.084 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction 2024-09-14T03:44:25.586 INFO:teuthology.orchestra.run.smithi039.stdout:(1/12): ceph-mgr-diskprediction-local-18.2.4-84 17 MB/s | 7.4 MB 00:00 2024-09-14T03:44:25.653 INFO:teuthology.orchestra.run.smithi039.stdout:(2/12): flexiblas-3.0.4-8.el9.x86_64.rpm 493 kB/s | 33 kB 00:00 2024-09-14T03:44:25.687 INFO:teuthology.orchestra.run.smithi039.stdout:(3/12): libquadmath-11.5.0-2.el9.x86_64.rpm 351 kB/s | 191 kB 00:00 2024-09-14T03:44:25.762 INFO:teuthology.orchestra.run.smithi039.stdout:(4/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 244 kB/s | 18 kB 00:00 2024-09-14T03:44:25.837 INFO:teuthology.orchestra.run.smithi039.stdout:(5/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 16 MB/s | 3.0 MB 00:00 2024-09-14T03:44:25.844 INFO:teuthology.orchestra.run.smithi129.stdout: Preparing : 1/1 2024-09-14T03:44:25.862 INFO:teuthology.orchestra.run.smithi039.stdout:(6/12): openblas-0.3.26-2.el9.x86_64.rpm 383 kB/s | 38 kB 00:00 2024-09-14T03:44:25.896 INFO:teuthology.orchestra.run.smithi039.stdout:(7/12): libgfortran-11.5.0-2.el9.x86_64.rpm 1.0 MB/s | 801 kB 00:00 2024-09-14T03:44:25.926 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-09-14T03:44:25.968 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 2/12 2024-09-14T03:44:25.988 INFO:teuthology.orchestra.run.smithi039.stdout:(8/12): openblas-openmp-0.3.26-2.el9.x86_64.rpm 33 MB/s | 4.9 MB 00:00 2024-09-14T03:44:26.013 INFO:teuthology.orchestra.run.smithi039.stdout:(9/12): python3-devel-3.9.19-8.el9.x86_64.rpm 1.6 MB/s | 245 kB 00:00 2024-09-14T03:44:26.047 INFO:teuthology.orchestra.run.smithi039.stdout:(10/12): python3-numpy-f2py-1.20.1-5.el9.x86_64 4.2 MB/s | 250 kB 00:00 2024-09-14T03:44:26.071 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 3/12 2024-09-14T03:44:26.111 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 4/12 2024-09-14T03:44:26.197 INFO:teuthology.orchestra.run.smithi039.stdout:(11/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 17 MB/s | 5.1 MB 00:00 2024-09-14T03:44:26.296 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : openblas-0.3.26-2.el9.x86_64 5/12 2024-09-14T03:44:26.435 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 6/12 2024-09-14T03:44:26.512 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-09-14T03:44:26.964 INFO:teuthology.orchestra.run.smithi039.stdout:(12/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 17 MB/s | 16 MB 00:00 2024-09-14T03:44:26.964 INFO:teuthology.orchestra.run.smithi039.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:44:26.965 INFO:teuthology.orchestra.run.smithi039.stdout:Total 20 MB/s | 38 MB 00:01 2024-09-14T03:44:27.178 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-09-14T03:44:27.310 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-09-14T03:44:27.331 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction check 2024-09-14T03:44:27.358 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction check succeeded. 2024-09-14T03:44:27.358 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction test 2024-09-14T03:44:27.884 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction test succeeded. 2024-09-14T03:44:27.884 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction 2024-09-14T03:44:28.560 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-09-14T03:44:28.674 INFO:teuthology.orchestra.run.smithi039.stdout: Preparing : 1/1 2024-09-14T03:44:28.745 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 1/12 2024-09-14T03:44:28.788 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : libquadmath-11.5.0-2.el9.x86_64 2/12 2024-09-14T03:44:28.891 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : libgfortran-11.5.0-2.el9.x86_64 3/12 2024-09-14T03:44:28.939 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-devel-3.9.19-8.el9.x86_64 4/12 2024-09-14T03:44:29.131 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : openblas-0.3.26-2.el9.x86_64 5/12 2024-09-14T03:44:29.162 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : openblas-openmp-0.3.26-2.el9.x86_64 6/12 2024-09-14T03:44:29.250 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-09-14T03:44:29.589 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-09-14T03:44:29.620 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : ceph-mgr-diskprediction-local-2:18.2.4-845.ge4e227 12/12 2024-09-14T03:44:29.944 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-09-14T03:44:30.071 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-09-14T03:44:31.396 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-09-14T03:44:32.102 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:18.2.4-845.ge4e227 12/12 2024-09-14T03:44:32.102 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : ceph-mgr-diskprediction-local-2:18.2.4-845.ge4e227 1/12 2024-09-14T03:44:32.102 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 2/12 2024-09-14T03:44:32.102 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 3/12 2024-09-14T03:44:32.102 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-09-14T03:44:32.102 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-09-14T03:44:32.102 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-09-14T03:44:32.102 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 7/12 2024-09-14T03:44:32.102 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 8/12 2024-09-14T03:44:32.102 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 9/12 2024-09-14T03:44:32.102 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-09-14T03:44:32.103 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-09-14T03:44:32.491 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-09-14T03:44:32.501 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-09-14T03:44:32.501 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:32.501 INFO:teuthology.orchestra.run.smithi129.stdout:Installed: 2024-09-14T03:44:32.502 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-mgr-diskprediction-local-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:44:32.502 INFO:teuthology.orchestra.run.smithi129.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-09-14T03:44:32.502 INFO:teuthology.orchestra.run.smithi129.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-09-14T03:44:32.502 INFO:teuthology.orchestra.run.smithi129.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-09-14T03:44:32.502 INFO:teuthology.orchestra.run.smithi129.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-09-14T03:44:32.502 INFO:teuthology.orchestra.run.smithi129.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-09-14T03:44:32.502 INFO:teuthology.orchestra.run.smithi129.stdout: openblas-0.3.26-2.el9.x86_64 2024-09-14T03:44:32.502 INFO:teuthology.orchestra.run.smithi129.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-09-14T03:44:32.502 INFO:teuthology.orchestra.run.smithi129.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-09-14T03:44:32.502 INFO:teuthology.orchestra.run.smithi129.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-09-14T03:44:32.502 INFO:teuthology.orchestra.run.smithi129.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-09-14T03:44:32.503 INFO:teuthology.orchestra.run.smithi129.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-09-14T03:44:32.503 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:32.503 INFO:teuthology.orchestra.run.smithi129.stdout:Complete! 2024-09-14T03:44:32.520 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : ceph-mgr-diskprediction-local-2:18.2.4-845.ge4e227 12/12 2024-09-14T03:44:32.866 DEBUG:teuthology.orchestra.run.smithi129:> sudo yum -y install ceph-mgr-rook 2024-09-14T03:44:33.441 INFO:teuthology.orchestra.run.smithi129.stdout:Last metadata expiration check: 0:01:11 ago on Sat 14 Sep 2024 03:43:22 AM UTC. 2024-09-14T03:44:33.574 INFO:teuthology.orchestra.run.smithi129.stdout:Dependencies resolved. 2024-09-14T03:44:33.575 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:33.575 INFO:teuthology.orchestra.run.smithi129.stdout: Package Arch Version Repository Size 2024-09-14T03:44:33.575 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:33.575 INFO:teuthology.orchestra.run.smithi129.stdout:Installing: 2024-09-14T03:44:33.575 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-mgr-rook noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 49 k 2024-09-14T03:44:33.575 INFO:teuthology.orchestra.run.smithi129.stdout:Installing dependencies: 2024-09-14T03:44:33.575 INFO:teuthology.orchestra.run.smithi129.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-09-14T03:44:33.575 INFO:teuthology.orchestra.run.smithi129.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-09-14T03:44:33.576 INFO:teuthology.orchestra.run.smithi129.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-09-14T03:44:33.576 INFO:teuthology.orchestra.run.smithi129.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-09-14T03:44:33.576 INFO:teuthology.orchestra.run.smithi129.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-09-14T03:44:33.576 INFO:teuthology.orchestra.run.smithi129.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-09-14T03:44:33.576 INFO:teuthology.orchestra.run.smithi129.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-09-14T03:44:33.576 INFO:teuthology.orchestra.run.smithi129.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-09-14T03:44:33.576 INFO:teuthology.orchestra.run.smithi129.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-09-14T03:44:33.576 INFO:teuthology.orchestra.run.smithi129.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-09-14T03:44:33.576 INFO:teuthology.orchestra.run.smithi129.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-09-14T03:44:33.576 INFO:teuthology.orchestra.run.smithi129.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-09-14T03:44:33.576 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:33.576 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction Summary 2024-09-14T03:44:33.577 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:33.577 INFO:teuthology.orchestra.run.smithi129.stdout:Install 13 Packages 2024-09-14T03:44:33.577 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:33.577 INFO:teuthology.orchestra.run.smithi129.stdout:Total download size: 2.2 M 2024-09-14T03:44:33.577 INFO:teuthology.orchestra.run.smithi129.stdout:Installed size: 26 M 2024-09-14T03:44:33.577 INFO:teuthology.orchestra.run.smithi129.stdout:Downloading Packages: 2024-09-14T03:44:33.797 INFO:teuthology.orchestra.run.smithi129.stdout:(1/13): python3-jsonpatch-1.21-16.el9.noarch.rp 223 kB/s | 26 kB 00:00 2024-09-14T03:44:33.814 INFO:teuthology.orchestra.run.smithi129.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 145 kB/s | 19 kB 00:00 2024-09-14T03:44:33.831 INFO:teuthology.orchestra.run.smithi129.stdout:(3/13): ceph-mgr-rook-18.2.4-845.ge4e227b3.el9. 331 kB/s | 49 kB 00:00 2024-09-14T03:44:33.964 INFO:teuthology.orchestra.run.smithi129.stdout:(4/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 1.3 MB/s | 222 kB 00:00 2024-09-14T03:44:33.990 INFO:teuthology.orchestra.run.smithi129.stdout:(5/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 904 kB/s | 159 kB 00:00 2024-09-14T03:44:34.023 INFO:teuthology.orchestra.run.smithi129.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 913 kB/s | 54 kB 00:00 2024-09-14T03:44:34.049 INFO:teuthology.orchestra.run.smithi129.stdout:(7/13): python3-pyasn1-modules-0.4.8-6.el9.noar 1.3 MB/s | 279 kB 00:00 2024-09-14T03:44:34.082 INFO:teuthology.orchestra.run.smithi129.stdout:(8/13): python3-cachetools-4.2.4-1.el9.noarch.r 349 kB/s | 32 kB 00:00 2024-09-14T03:44:34.116 INFO:teuthology.orchestra.run.smithi129.stdout:(9/13): python3-certifi-2023.05.07-4.el9.noarch 156 kB/s | 14 kB 00:00 2024-09-14T03:44:34.141 INFO:teuthology.orchestra.run.smithi129.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 2.5 MB/s | 59 kB 00:00 2024-09-14T03:44:34.175 INFO:teuthology.orchestra.run.smithi129.stdout:(11/13): python3-websocket-client-1.2.3-2.el9.n 2.6 MB/s | 90 kB 00:00 2024-09-14T03:44:34.200 INFO:teuthology.orchestra.run.smithi129.stdout:(12/13): python3-google-auth-2.34.0-1.el9.noarc 1.5 MB/s | 220 kB 00:00 2024-09-14T03:44:34.242 INFO:teuthology.orchestra.run.smithi129.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 6.8 MB/s | 1.0 MB 00:00 2024-09-14T03:44:34.244 INFO:teuthology.orchestra.run.smithi129.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:44:34.244 INFO:teuthology.orchestra.run.smithi129.stdout:Total 3.3 MB/s | 2.2 MB 00:00 2024-09-14T03:44:34.307 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction check 2024-09-14T03:44:34.322 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction check succeeded. 2024-09-14T03:44:34.322 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction test 2024-09-14T03:44:34.410 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction test succeeded. 2024-09-14T03:44:34.410 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction 2024-09-14T03:44:34.937 INFO:teuthology.orchestra.run.smithi129.stdout: Preparing : 1/1 2024-09-14T03:44:35.105 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-09-14T03:44:35.131 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:18.2.4-845.ge4e227 12/12 2024-09-14T03:44:35.131 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : ceph-mgr-diskprediction-local-2:18.2.4-845.ge4e227 1/12 2024-09-14T03:44:35.131 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : libgfortran-11.5.0-2.el9.x86_64 2/12 2024-09-14T03:44:35.131 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : libquadmath-11.5.0-2.el9.x86_64 3/12 2024-09-14T03:44:35.131 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 4/12 2024-09-14T03:44:35.131 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 5/12 2024-09-14T03:44:35.131 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 6/12 2024-09-14T03:44:35.131 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : openblas-0.3.26-2.el9.x86_64 7/12 2024-09-14T03:44:35.131 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : openblas-openmp-0.3.26-2.el9.x86_64 8/12 2024-09-14T03:44:35.132 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-devel-3.9.19-8.el9.x86_64 9/12 2024-09-14T03:44:35.132 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 10/12 2024-09-14T03:44:35.132 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 11/12 2024-09-14T03:44:35.159 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-09-14T03:44:35.224 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-09-14T03:44:35.269 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-09-14T03:44:35.310 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-09-14T03:44:35.413 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-09-14T03:44:35.515 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 7/13 2024-09-14T03:44:35.520 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 12/12 2024-09-14T03:44:35.520 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:35.521 INFO:teuthology.orchestra.run.smithi039.stdout:Installed: 2024-09-14T03:44:35.521 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-mgr-diskprediction-local-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:44:35.521 INFO:teuthology.orchestra.run.smithi039.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-09-14T03:44:35.521 INFO:teuthology.orchestra.run.smithi039.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-09-14T03:44:35.521 INFO:teuthology.orchestra.run.smithi039.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-09-14T03:44:35.521 INFO:teuthology.orchestra.run.smithi039.stdout: libgfortran-11.5.0-2.el9.x86_64 2024-09-14T03:44:35.521 INFO:teuthology.orchestra.run.smithi039.stdout: libquadmath-11.5.0-2.el9.x86_64 2024-09-14T03:44:35.521 INFO:teuthology.orchestra.run.smithi039.stdout: openblas-0.3.26-2.el9.x86_64 2024-09-14T03:44:35.521 INFO:teuthology.orchestra.run.smithi039.stdout: openblas-openmp-0.3.26-2.el9.x86_64 2024-09-14T03:44:35.521 INFO:teuthology.orchestra.run.smithi039.stdout: python3-devel-3.9.19-8.el9.x86_64 2024-09-14T03:44:35.522 INFO:teuthology.orchestra.run.smithi039.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-09-14T03:44:35.522 INFO:teuthology.orchestra.run.smithi039.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-09-14T03:44:35.522 INFO:teuthology.orchestra.run.smithi039.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-09-14T03:44:35.522 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:35.522 INFO:teuthology.orchestra.run.smithi039.stdout:Complete! 2024-09-14T03:44:35.563 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-09-14T03:44:35.935 DEBUG:teuthology.orchestra.run.smithi039:> sudo yum -y install ceph-mgr-rook 2024-09-14T03:44:36.283 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-09-14T03:44:36.327 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-09-14T03:44:36.361 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-09-14T03:44:36.409 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-09-14T03:44:36.439 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : ceph-mgr-rook-2:18.2.4-845.ge4e227b3.el9.noarch 13/13 2024-09-14T03:44:36.532 INFO:teuthology.orchestra.run.smithi039.stdout:Last metadata expiration check: 0:01:12 ago on Sat 14 Sep 2024 03:43:24 AM UTC. 2024-09-14T03:44:36.671 INFO:teuthology.orchestra.run.smithi039.stdout:Dependencies resolved. 2024-09-14T03:44:36.673 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:44:36.673 INFO:teuthology.orchestra.run.smithi039.stdout: Package Arch Version Repository Size 2024-09-14T03:44:36.673 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:44:36.673 INFO:teuthology.orchestra.run.smithi039.stdout:Installing: 2024-09-14T03:44:36.673 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-mgr-rook noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 49 k 2024-09-14T03:44:36.673 INFO:teuthology.orchestra.run.smithi039.stdout:Installing dependencies: 2024-09-14T03:44:36.673 INFO:teuthology.orchestra.run.smithi039.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-09-14T03:44:36.673 INFO:teuthology.orchestra.run.smithi039.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2024-09-14T03:44:36.673 INFO:teuthology.orchestra.run.smithi039.stdout: python3-google-auth noarch 1:2.34.0-1.el9 epel 220 k 2024-09-14T03:44:36.673 INFO:teuthology.orchestra.run.smithi039.stdout: python3-jsonpatch noarch 1.21-16.el9 appstream 26 k 2024-09-14T03:44:36.673 INFO:teuthology.orchestra.run.smithi039.stdout: python3-jsonpointer noarch 2.0-4.el9 appstream 19 k 2024-09-14T03:44:36.674 INFO:teuthology.orchestra.run.smithi039.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-09-14T03:44:36.674 INFO:teuthology.orchestra.run.smithi039.stdout: python3-oauthlib noarch 3.1.1-5.el9 appstream 222 k 2024-09-14T03:44:36.674 INFO:teuthology.orchestra.run.smithi039.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2024-09-14T03:44:36.674 INFO:teuthology.orchestra.run.smithi039.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2024-09-14T03:44:36.674 INFO:teuthology.orchestra.run.smithi039.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2024-09-14T03:44:36.674 INFO:teuthology.orchestra.run.smithi039.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-09-14T03:44:36.674 INFO:teuthology.orchestra.run.smithi039.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-09-14T03:44:36.674 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:36.674 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction Summary 2024-09-14T03:44:36.674 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:44:36.674 INFO:teuthology.orchestra.run.smithi039.stdout:Install 13 Packages 2024-09-14T03:44:36.674 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:36.675 INFO:teuthology.orchestra.run.smithi039.stdout:Total download size: 2.2 M 2024-09-14T03:44:36.675 INFO:teuthology.orchestra.run.smithi039.stdout:Installed size: 26 M 2024-09-14T03:44:36.675 INFO:teuthology.orchestra.run.smithi039.stdout:Downloading Packages: 2024-09-14T03:44:36.920 INFO:teuthology.orchestra.run.smithi039.stdout:(1/13): ceph-mgr-rook-18.2.4-845.ge4e227b3.el9. 298 kB/s | 49 kB 00:00 2024-09-14T03:44:37.193 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: ceph-mgr-rook-2:18.2.4-845.ge4e227b3.el9.noarch 13/13 2024-09-14T03:44:37.193 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : ceph-mgr-rook-2:18.2.4-845.ge4e227b3.el9.noarch 1/13 2024-09-14T03:44:37.193 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-09-14T03:44:37.193 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-09-14T03:44:37.194 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-09-14T03:44:37.194 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-09-14T03:44:37.194 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-09-14T03:44:37.194 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-09-14T03:44:37.194 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-09-14T03:44:37.194 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-09-14T03:44:37.194 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 10/13 2024-09-14T03:44:37.194 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-09-14T03:44:37.194 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-09-14T03:44:37.270 INFO:teuthology.orchestra.run.smithi039.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 37 kB/s | 19 kB 00:00 2024-09-14T03:44:37.312 INFO:teuthology.orchestra.run.smithi039.stdout:(3/13): python3-jsonpatch-1.21-16.el9.noarch.rp 46 kB/s | 26 kB 00:00 2024-09-14T03:44:37.545 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-09-14T03:44:37.545 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:37.545 INFO:teuthology.orchestra.run.smithi129.stdout:Installed: 2024-09-14T03:44:37.546 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-mgr-rook-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:44:37.546 INFO:teuthology.orchestra.run.smithi129.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-09-14T03:44:37.546 INFO:teuthology.orchestra.run.smithi129.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-09-14T03:44:37.546 INFO:teuthology.orchestra.run.smithi129.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-09-14T03:44:37.546 INFO:teuthology.orchestra.run.smithi129.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-09-14T03:44:37.546 INFO:teuthology.orchestra.run.smithi129.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-09-14T03:44:37.546 INFO:teuthology.orchestra.run.smithi129.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-09-14T03:44:37.546 INFO:teuthology.orchestra.run.smithi129.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-09-14T03:44:37.546 INFO:teuthology.orchestra.run.smithi129.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-09-14T03:44:37.547 INFO:teuthology.orchestra.run.smithi129.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-09-14T03:44:37.547 INFO:teuthology.orchestra.run.smithi129.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-09-14T03:44:37.547 INFO:teuthology.orchestra.run.smithi129.stdout: python3-rsa-4.9-2.el9.noarch 2024-09-14T03:44:37.547 INFO:teuthology.orchestra.run.smithi129.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-09-14T03:44:37.547 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:37.547 INFO:teuthology.orchestra.run.smithi129.stdout:Complete! 2024-09-14T03:44:37.662 INFO:teuthology.orchestra.run.smithi039.stdout:(4/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 299 kB/s | 222 kB 00:00 2024-09-14T03:44:37.797 DEBUG:teuthology.orchestra.run.smithi129:> sudo yum -y install ceph-mgr-cephadm 2024-09-14T03:44:38.013 INFO:teuthology.orchestra.run.smithi039.stdout:(5/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 214 kB/s | 159 kB 00:00 2024-09-14T03:44:38.054 INFO:teuthology.orchestra.run.smithi039.stdout:(6/13): python3-cachetools-4.2.4-1.el9.noarch.r 784 kB/s | 32 kB 00:00 2024-09-14T03:44:38.080 INFO:teuthology.orchestra.run.smithi039.stdout:(7/13): python3-certifi-2023.05.07-4.el9.noarch 562 kB/s | 14 kB 00:00 2024-09-14T03:44:38.113 INFO:teuthology.orchestra.run.smithi039.stdout:(8/13): python3-google-auth-2.34.0-1.el9.noarch 6.5 MB/s | 220 kB 00:00 2024-09-14T03:44:38.147 INFO:teuthology.orchestra.run.smithi039.stdout:(9/13): python3-pyasn1-modules-0.4.8-6.el9.noar 334 kB/s | 279 kB 00:00 2024-09-14T03:44:38.206 INFO:teuthology.orchestra.run.smithi039.stdout:(10/13): python3-kubernetes-26.1.0-2.el9.noarch 11 MB/s | 1.0 MB 00:00 2024-09-14T03:44:38.231 INFO:teuthology.orchestra.run.smithi039.stdout:(11/13): python3-rsa-4.9-2.el9.noarch.rpm 712 kB/s | 59 kB 00:00 2024-09-14T03:44:38.256 INFO:teuthology.orchestra.run.smithi039.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 1.8 MB/s | 90 kB 00:00 2024-09-14T03:44:38.290 INFO:teuthology.orchestra.run.smithi039.stdout:(13/13): python3-requests-oauthlib-1.3.0-12.el9 86 kB/s | 54 kB 00:00 2024-09-14T03:44:38.293 INFO:teuthology.orchestra.run.smithi039.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:44:38.293 INFO:teuthology.orchestra.run.smithi039.stdout:Total 1.4 MB/s | 2.2 MB 00:01 2024-09-14T03:44:38.354 INFO:teuthology.orchestra.run.smithi129.stdout:Last metadata expiration check: 0:01:16 ago on Sat 14 Sep 2024 03:43:22 AM UTC. 2024-09-14T03:44:38.369 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction check 2024-09-14T03:44:38.385 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction check succeeded. 2024-09-14T03:44:38.385 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction test 2024-09-14T03:44:38.485 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction test succeeded. 2024-09-14T03:44:38.487 INFO:teuthology.orchestra.run.smithi129.stdout:Dependencies resolved. 2024-09-14T03:44:38.487 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction 2024-09-14T03:44:38.488 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:38.488 INFO:teuthology.orchestra.run.smithi129.stdout: Package Arch Version Repository Size 2024-09-14T03:44:38.488 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:38.488 INFO:teuthology.orchestra.run.smithi129.stdout:Installing: 2024-09-14T03:44:38.488 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-mgr-cephadm noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 139 k 2024-09-14T03:44:38.488 INFO:teuthology.orchestra.run.smithi129.stdout:Installing dependencies: 2024-09-14T03:44:38.488 INFO:teuthology.orchestra.run.smithi129.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-09-14T03:44:38.488 INFO:teuthology.orchestra.run.smithi129.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-09-14T03:44:38.489 INFO:teuthology.orchestra.run.smithi129.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-09-14T03:44:38.489 INFO:teuthology.orchestra.run.smithi129.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-09-14T03:44:38.489 INFO:teuthology.orchestra.run.smithi129.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-09-14T03:44:38.489 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:38.489 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction Summary 2024-09-14T03:44:38.489 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:38.489 INFO:teuthology.orchestra.run.smithi129.stdout:Install 6 Packages 2024-09-14T03:44:38.489 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:38.490 INFO:teuthology.orchestra.run.smithi129.stdout:Total download size: 7.0 M 2024-09-14T03:44:38.490 INFO:teuthology.orchestra.run.smithi129.stdout:Installed size: 33 M 2024-09-14T03:44:38.490 INFO:teuthology.orchestra.run.smithi129.stdout:Downloading Packages: 2024-09-14T03:44:38.721 INFO:teuthology.orchestra.run.smithi129.stdout:(1/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 1.5 MB/s | 248 kB 00:00 2024-09-14T03:44:38.846 INFO:teuthology.orchestra.run.smithi129.stdout:(2/6): python3-babel-2.9.1-2.el9.noarch.rpm 21 MB/s | 6.0 MB 00:00 2024-09-14T03:44:38.872 INFO:teuthology.orchestra.run.smithi129.stdout:(3/6): ceph-mgr-cephadm-18.2.4-845.ge4e227b3.el 451 kB/s | 139 kB 00:00 2024-09-14T03:44:38.914 INFO:teuthology.orchestra.run.smithi129.stdout:(4/6): python3-natsort-7.1.1-5.el9.noarch.rpm 884 kB/s | 58 kB 00:00 2024-09-14T03:44:38.939 INFO:teuthology.orchestra.run.smithi129.stdout:(5/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 2.5 MB/s | 548 kB 00:00 2024-09-14T03:44:38.964 INFO:teuthology.orchestra.run.smithi129.stdout:(6/6): python3-typing-extensions-4.12.2-2.el9.n 871 kB/s | 74 kB 00:00 2024-09-14T03:44:38.967 INFO:teuthology.orchestra.run.smithi129.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:44:38.968 INFO:teuthology.orchestra.run.smithi129.stdout:Total 15 MB/s | 7.0 MB 00:00 2024-09-14T03:44:39.030 INFO:teuthology.orchestra.run.smithi039.stdout: Preparing : 1/1 2024-09-14T03:44:39.037 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction check 2024-09-14T03:44:39.048 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction check succeeded. 2024-09-14T03:44:39.048 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction test 2024-09-14T03:44:39.147 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction test succeeded. 2024-09-14T03:44:39.149 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction 2024-09-14T03:44:39.212 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-09-14T03:44:39.268 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-09-14T03:44:39.333 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-09-14T03:44:39.375 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-09-14T03:44:39.418 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 5/13 2024-09-14T03:44:39.451 INFO:teuthology.orchestra.run.smithi129.stdout: Preparing : 1/1 2024-09-14T03:44:39.525 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-09-14T03:44:39.619 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-09-14T03:44:39.643 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-google-auth-1:2.34.0-1.el9.noarch 7/13 2024-09-14T03:44:39.666 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-09-14T03:44:39.704 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-09-14T03:44:40.183 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-09-14T03:44:40.268 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-09-14T03:44:40.339 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-09-14T03:44:40.387 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : ceph-mgr-cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 6/6 2024-09-14T03:44:40.463 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-09-14T03:44:40.516 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-09-14T03:44:40.550 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-09-14T03:44:40.599 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-09-14T03:44:40.627 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : ceph-mgr-rook-2:18.2.4-845.ge4e227b3.el9.noarch 13/13 2024-09-14T03:44:41.113 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: ceph-mgr-cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 6/6 2024-09-14T03:44:41.113 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : ceph-mgr-cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 1/6 2024-09-14T03:44:41.113 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-09-14T03:44:41.113 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-09-14T03:44:41.113 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-09-14T03:44:41.113 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-09-14T03:44:41.380 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-09-14T03:44:41.380 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:41.381 INFO:teuthology.orchestra.run.smithi129.stdout:Installed: 2024-09-14T03:44:41.381 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-mgr-cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:44:41.381 INFO:teuthology.orchestra.run.smithi129.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-09-14T03:44:41.381 INFO:teuthology.orchestra.run.smithi129.stdout: python3-babel-2.9.1-2.el9.noarch 2024-09-14T03:44:41.381 INFO:teuthology.orchestra.run.smithi129.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-09-14T03:44:41.381 INFO:teuthology.orchestra.run.smithi129.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-09-14T03:44:41.381 INFO:teuthology.orchestra.run.smithi129.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-09-14T03:44:41.381 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:41.381 INFO:teuthology.orchestra.run.smithi129.stdout:Complete! 2024-09-14T03:44:41.440 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: ceph-mgr-rook-2:18.2.4-845.ge4e227b3.el9.noarch 13/13 2024-09-14T03:44:41.440 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : ceph-mgr-rook-2:18.2.4-845.ge4e227b3.el9.noarch 1/13 2024-09-14T03:44:41.440 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-09-14T03:44:41.440 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-09-14T03:44:41.440 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-09-14T03:44:41.440 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-09-14T03:44:41.441 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-09-14T03:44:41.441 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-09-14T03:44:41.441 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-09-14T03:44:41.441 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 9/13 2024-09-14T03:44:41.441 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-google-auth-1:2.34.0-1.el9.noarch 10/13 2024-09-14T03:44:41.441 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-09-14T03:44:41.441 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-09-14T03:44:41.628 DEBUG:teuthology.orchestra.run.smithi129:> sudo yum -y install ceph-fuse 2024-09-14T03:44:41.797 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-09-14T03:44:41.797 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:41.797 INFO:teuthology.orchestra.run.smithi039.stdout:Installed: 2024-09-14T03:44:41.797 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-mgr-rook-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:44:41.797 INFO:teuthology.orchestra.run.smithi039.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-09-14T03:44:41.797 INFO:teuthology.orchestra.run.smithi039.stdout: python3-certifi-2023.05.07-4.el9.noarch 2024-09-14T03:44:41.798 INFO:teuthology.orchestra.run.smithi039.stdout: python3-google-auth-1:2.34.0-1.el9.noarch 2024-09-14T03:44:41.798 INFO:teuthology.orchestra.run.smithi039.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-09-14T03:44:41.798 INFO:teuthology.orchestra.run.smithi039.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-09-14T03:44:41.798 INFO:teuthology.orchestra.run.smithi039.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-09-14T03:44:41.798 INFO:teuthology.orchestra.run.smithi039.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-09-14T03:44:41.798 INFO:teuthology.orchestra.run.smithi039.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-09-14T03:44:41.798 INFO:teuthology.orchestra.run.smithi039.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-09-14T03:44:41.798 INFO:teuthology.orchestra.run.smithi039.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-09-14T03:44:41.798 INFO:teuthology.orchestra.run.smithi039.stdout: python3-rsa-4.9-2.el9.noarch 2024-09-14T03:44:41.798 INFO:teuthology.orchestra.run.smithi039.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-09-14T03:44:41.798 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:41.798 INFO:teuthology.orchestra.run.smithi039.stdout:Complete! 2024-09-14T03:44:42.040 DEBUG:teuthology.orchestra.run.smithi039:> sudo yum -y install ceph-mgr-cephadm 2024-09-14T03:44:42.192 INFO:teuthology.orchestra.run.smithi129.stdout:Last metadata expiration check: 0:01:20 ago on Sat 14 Sep 2024 03:43:22 AM UTC. 2024-09-14T03:44:42.325 INFO:teuthology.orchestra.run.smithi129.stdout:Dependencies resolved. 2024-09-14T03:44:42.325 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:42.325 INFO:teuthology.orchestra.run.smithi129.stdout: Package Arch Version Repository Size 2024-09-14T03:44:42.325 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:42.325 INFO:teuthology.orchestra.run.smithi129.stdout:Installing: 2024-09-14T03:44:42.326 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-fuse x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 852 k 2024-09-14T03:44:42.326 INFO:teuthology.orchestra.run.smithi129.stdout:Installing dependencies: 2024-09-14T03:44:42.326 INFO:teuthology.orchestra.run.smithi129.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-14T03:44:42.326 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:42.326 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction Summary 2024-09-14T03:44:42.326 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:42.326 INFO:teuthology.orchestra.run.smithi129.stdout:Install 2 Packages 2024-09-14T03:44:42.326 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:42.327 INFO:teuthology.orchestra.run.smithi129.stdout:Total download size: 932 k 2024-09-14T03:44:42.327 INFO:teuthology.orchestra.run.smithi129.stdout:Installed size: 2.7 M 2024-09-14T03:44:42.327 INFO:teuthology.orchestra.run.smithi129.stdout:Downloading Packages: 2024-09-14T03:44:42.526 INFO:teuthology.orchestra.run.smithi129.stdout:(1/2): fuse-2.9.9-16.el9.x86_64.rpm 527 kB/s | 80 kB 00:00 2024-09-14T03:44:42.636 INFO:teuthology.orchestra.run.smithi039.stdout:Last metadata expiration check: 0:01:18 ago on Sat 14 Sep 2024 03:43:24 AM UTC. 2024-09-14T03:44:42.659 INFO:teuthology.orchestra.run.smithi129.stdout:(2/2): ceph-fuse-18.2.4-845.ge4e227b3.el9.x86_6 2.9 MB/s | 852 kB 00:00 2024-09-14T03:44:42.660 INFO:teuthology.orchestra.run.smithi129.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:44:42.660 INFO:teuthology.orchestra.run.smithi129.stdout:Total 2.7 MB/s | 932 kB 00:00 2024-09-14T03:44:42.672 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction check 2024-09-14T03:44:42.693 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction check succeeded. 2024-09-14T03:44:42.693 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction test 2024-09-14T03:44:42.774 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction test succeeded. 2024-09-14T03:44:42.774 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction 2024-09-14T03:44:42.775 INFO:teuthology.orchestra.run.smithi039.stdout:Dependencies resolved. 2024-09-14T03:44:42.776 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:44:42.776 INFO:teuthology.orchestra.run.smithi039.stdout: Package Arch Version Repository Size 2024-09-14T03:44:42.776 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:44:42.776 INFO:teuthology.orchestra.run.smithi039.stdout:Installing: 2024-09-14T03:44:42.776 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-mgr-cephadm noarch 2:18.2.4-845.ge4e227b3.el9 ceph-noarch 139 k 2024-09-14T03:44:42.776 INFO:teuthology.orchestra.run.smithi039.stdout:Installing dependencies: 2024-09-14T03:44:42.776 INFO:teuthology.orchestra.run.smithi039.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-09-14T03:44:42.776 INFO:teuthology.orchestra.run.smithi039.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2024-09-14T03:44:42.776 INFO:teuthology.orchestra.run.smithi039.stdout: python3-jinja2 noarch 2.11.3-6.el9 appstream 248 k 2024-09-14T03:44:42.776 INFO:teuthology.orchestra.run.smithi039.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-09-14T03:44:42.777 INFO:teuthology.orchestra.run.smithi039.stdout: python3-typing-extensions noarch 4.12.2-2.el9 epel 74 k 2024-09-14T03:44:42.777 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:42.777 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction Summary 2024-09-14T03:44:42.777 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:44:42.777 INFO:teuthology.orchestra.run.smithi039.stdout:Install 6 Packages 2024-09-14T03:44:42.777 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:42.777 INFO:teuthology.orchestra.run.smithi039.stdout:Total download size: 7.0 M 2024-09-14T03:44:42.778 INFO:teuthology.orchestra.run.smithi039.stdout:Installed size: 33 M 2024-09-14T03:44:42.778 INFO:teuthology.orchestra.run.smithi039.stdout:Downloading Packages: 2024-09-14T03:44:42.958 INFO:teuthology.orchestra.run.smithi129.stdout: Preparing : 1/1 2024-09-14T03:44:43.099 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-09-14T03:44:43.167 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : ceph-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 2/2 2024-09-14T03:44:43.301 INFO:teuthology.orchestra.run.smithi039.stdout:(1/6): python3-jinja2-2.11.3-6.el9.noarch.rpm 1.1 MB/s | 248 kB 00:00 2024-09-14T03:44:43.342 INFO:teuthology.orchestra.run.smithi039.stdout:(2/6): ceph-mgr-cephadm-18.2.4-845.ge4e227b3.el 547 kB/s | 139 kB 00:00 2024-09-14T03:44:43.393 INFO:teuthology.orchestra.run.smithi039.stdout:(3/6): python3-natsort-7.1.1-5.el9.noarch.rpm 1.1 MB/s | 58 kB 00:00 2024-09-14T03:44:43.418 INFO:teuthology.orchestra.run.smithi039.stdout:(4/6): python3-typing-extensions-4.12.2-2.el9.n 2.9 MB/s | 74 kB 00:00 2024-09-14T03:44:43.452 INFO:teuthology.orchestra.run.smithi039.stdout:(5/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 3.6 MB/s | 548 kB 00:00 2024-09-14T03:44:43.832 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: ceph-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 2/2 2024-09-14T03:44:43.832 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : ceph-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 1/2 2024-09-14T03:44:44.076 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-09-14T03:44:44.077 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:44.077 INFO:teuthology.orchestra.run.smithi129.stdout:Installed: 2024-09-14T03:44:44.077 INFO:teuthology.orchestra.run.smithi129.stdout: ceph-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-09-14T03:44:44.077 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:44.077 INFO:teuthology.orchestra.run.smithi129.stdout:Complete! 2024-09-14T03:44:44.358 DEBUG:teuthology.orchestra.run.smithi129:> sudo yum -y install librados-devel 2024-09-14T03:44:44.920 INFO:teuthology.orchestra.run.smithi129.stdout:Last metadata expiration check: 0:01:22 ago on Sat 14 Sep 2024 03:43:22 AM UTC. 2024-09-14T03:44:45.059 INFO:teuthology.orchestra.run.smithi129.stdout:Dependencies resolved. 2024-09-14T03:44:45.059 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:45.060 INFO:teuthology.orchestra.run.smithi129.stdout: Package Arch Version Repo Size 2024-09-14T03:44:45.060 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:45.060 INFO:teuthology.orchestra.run.smithi129.stdout:Installing: 2024-09-14T03:44:45.060 INFO:teuthology.orchestra.run.smithi129.stdout: librados-devel x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 127 k 2024-09-14T03:44:45.060 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:45.060 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction Summary 2024-09-14T03:44:45.060 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:45.060 INFO:teuthology.orchestra.run.smithi129.stdout:Install 1 Package 2024-09-14T03:44:45.060 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:45.061 INFO:teuthology.orchestra.run.smithi129.stdout:Total download size: 127 k 2024-09-14T03:44:45.061 INFO:teuthology.orchestra.run.smithi129.stdout:Installed size: 456 k 2024-09-14T03:44:45.061 INFO:teuthology.orchestra.run.smithi129.stdout:Downloading Packages: 2024-09-14T03:44:45.257 INFO:teuthology.orchestra.run.smithi129.stdout:librados-devel-18.2.4-845.ge4e227b3.el9.x86_64. 647 kB/s | 127 kB 00:00 2024-09-14T03:44:45.258 INFO:teuthology.orchestra.run.smithi129.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:44:45.258 INFO:teuthology.orchestra.run.smithi129.stdout:Total 641 kB/s | 127 kB 00:00 2024-09-14T03:44:45.258 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction check 2024-09-14T03:44:45.263 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction check succeeded. 2024-09-14T03:44:45.263 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction test 2024-09-14T03:44:45.327 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction test succeeded. 2024-09-14T03:44:45.328 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction 2024-09-14T03:44:45.460 INFO:teuthology.orchestra.run.smithi039.stdout:(6/6): python3-babel-2.9.1-2.el9.noarch.rpm 2.5 MB/s | 6.0 MB 00:02 2024-09-14T03:44:45.463 INFO:teuthology.orchestra.run.smithi039.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:44:45.464 INFO:teuthology.orchestra.run.smithi039.stdout:Total 2.6 MB/s | 7.0 MB 00:02 2024-09-14T03:44:45.474 INFO:teuthology.orchestra.run.smithi129.stdout: Preparing : 1/1 2024-09-14T03:44:45.540 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction check 2024-09-14T03:44:45.552 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction check succeeded. 2024-09-14T03:44:45.552 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction test 2024-09-14T03:44:45.579 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : librados-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:44:45.659 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction test succeeded. 2024-09-14T03:44:45.659 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction 2024-09-14T03:44:45.969 INFO:teuthology.orchestra.run.smithi039.stdout: Preparing : 1/1 2024-09-14T03:44:46.043 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: librados-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:44:46.142 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-typing-extensions-4.12.2-2.el9.noarch 1/6 2024-09-14T03:44:46.188 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-09-14T03:44:46.318 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : librados-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:44:46.318 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:46.318 INFO:teuthology.orchestra.run.smithi129.stdout:Installed: 2024-09-14T03:44:46.319 INFO:teuthology.orchestra.run.smithi129.stdout: librados-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:44:46.319 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:46.319 INFO:teuthology.orchestra.run.smithi129.stdout:Complete! 2024-09-14T03:44:46.561 DEBUG:teuthology.orchestra.run.smithi129:> sudo yum -y install libcephfs2 2024-09-14T03:44:46.742 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-09-14T03:44:46.843 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-09-14T03:44:46.913 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : python3-jinja2-2.11.3-6.el9.noarch 5/6 2024-09-14T03:44:46.957 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : ceph-mgr-cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 6/6 2024-09-14T03:44:47.136 INFO:teuthology.orchestra.run.smithi129.stdout:Last metadata expiration check: 0:01:25 ago on Sat 14 Sep 2024 03:43:22 AM UTC. 2024-09-14T03:44:47.232 INFO:teuthology.orchestra.run.smithi129.stdout:Package libcephfs2-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:44:47.277 INFO:teuthology.orchestra.run.smithi129.stdout:Dependencies resolved. 2024-09-14T03:44:47.278 INFO:teuthology.orchestra.run.smithi129.stdout:Nothing to do. 2024-09-14T03:44:47.278 INFO:teuthology.orchestra.run.smithi129.stdout:Complete! 2024-09-14T03:44:47.349 DEBUG:teuthology.orchestra.run.smithi129:> sudo yum -y install libcephfs-devel 2024-09-14T03:44:47.698 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: ceph-mgr-cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 6/6 2024-09-14T03:44:47.698 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : ceph-mgr-cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 1/6 2024-09-14T03:44:47.698 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-09-14T03:44:47.698 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-jinja2-2.11.3-6.el9.noarch 3/6 2024-09-14T03:44:47.699 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-09-14T03:44:47.699 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-09-14T03:44:47.924 INFO:teuthology.orchestra.run.smithi129.stdout:Last metadata expiration check: 0:01:25 ago on Sat 14 Sep 2024 03:43:22 AM UTC. 2024-09-14T03:44:47.983 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : python3-typing-extensions-4.12.2-2.el9.noarch 6/6 2024-09-14T03:44:47.983 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:47.984 INFO:teuthology.orchestra.run.smithi039.stdout:Installed: 2024-09-14T03:44:47.984 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-mgr-cephadm-2:18.2.4-845.ge4e227b3.el9.noarch 2024-09-14T03:44:47.984 INFO:teuthology.orchestra.run.smithi039.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-09-14T03:44:47.984 INFO:teuthology.orchestra.run.smithi039.stdout: python3-babel-2.9.1-2.el9.noarch 2024-09-14T03:44:47.984 INFO:teuthology.orchestra.run.smithi039.stdout: python3-jinja2-2.11.3-6.el9.noarch 2024-09-14T03:44:47.984 INFO:teuthology.orchestra.run.smithi039.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-09-14T03:44:47.984 INFO:teuthology.orchestra.run.smithi039.stdout: python3-typing-extensions-4.12.2-2.el9.noarch 2024-09-14T03:44:47.984 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:47.984 INFO:teuthology.orchestra.run.smithi039.stdout:Complete! 2024-09-14T03:44:48.056 INFO:teuthology.orchestra.run.smithi129.stdout:Dependencies resolved. 2024-09-14T03:44:48.057 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:48.057 INFO:teuthology.orchestra.run.smithi129.stdout: Package Arch Version Repo Size 2024-09-14T03:44:48.057 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:48.057 INFO:teuthology.orchestra.run.smithi129.stdout:Installing: 2024-09-14T03:44:48.057 INFO:teuthology.orchestra.run.smithi129.stdout: libcephfs-devel x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 31 k 2024-09-14T03:44:48.058 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:48.058 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction Summary 2024-09-14T03:44:48.058 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:48.058 INFO:teuthology.orchestra.run.smithi129.stdout:Install 1 Package 2024-09-14T03:44:48.058 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:48.058 INFO:teuthology.orchestra.run.smithi129.stdout:Total download size: 31 k 2024-09-14T03:44:48.058 INFO:teuthology.orchestra.run.smithi129.stdout:Installed size: 139 k 2024-09-14T03:44:48.058 INFO:teuthology.orchestra.run.smithi129.stdout:Downloading Packages: 2024-09-14T03:44:48.204 INFO:teuthology.orchestra.run.smithi129.stdout:libcephfs-devel-18.2.4-845.ge4e227b3.el9.x86_64 214 kB/s | 31 kB 00:00 2024-09-14T03:44:48.204 INFO:teuthology.orchestra.run.smithi129.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:44:48.204 INFO:teuthology.orchestra.run.smithi129.stdout:Total 212 kB/s | 31 kB 00:00 2024-09-14T03:44:48.205 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction check 2024-09-14T03:44:48.208 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction check succeeded. 2024-09-14T03:44:48.208 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction test 2024-09-14T03:44:48.231 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction test succeeded. 2024-09-14T03:44:48.232 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction 2024-09-14T03:44:48.245 DEBUG:teuthology.orchestra.run.smithi039:> sudo yum -y install ceph-fuse 2024-09-14T03:44:48.316 INFO:teuthology.orchestra.run.smithi129.stdout: Preparing : 1/1 2024-09-14T03:44:48.433 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : libcephfs-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:44:48.819 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: libcephfs-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:44:48.837 INFO:teuthology.orchestra.run.smithi039.stdout:Last metadata expiration check: 0:01:24 ago on Sat 14 Sep 2024 03:43:24 AM UTC. 2024-09-14T03:44:48.976 INFO:teuthology.orchestra.run.smithi039.stdout:Dependencies resolved. 2024-09-14T03:44:48.977 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:44:48.977 INFO:teuthology.orchestra.run.smithi039.stdout: Package Arch Version Repository Size 2024-09-14T03:44:48.977 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:44:48.977 INFO:teuthology.orchestra.run.smithi039.stdout:Installing: 2024-09-14T03:44:48.977 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-fuse x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 852 k 2024-09-14T03:44:48.977 INFO:teuthology.orchestra.run.smithi039.stdout:Installing dependencies: 2024-09-14T03:44:48.977 INFO:teuthology.orchestra.run.smithi039.stdout: fuse x86_64 2.9.9-16.el9 baseos 80 k 2024-09-14T03:44:48.977 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:48.978 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction Summary 2024-09-14T03:44:48.978 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:44:48.978 INFO:teuthology.orchestra.run.smithi039.stdout:Install 2 Packages 2024-09-14T03:44:48.978 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:48.980 INFO:teuthology.orchestra.run.smithi039.stdout:Total download size: 932 k 2024-09-14T03:44:48.980 INFO:teuthology.orchestra.run.smithi039.stdout:Installed size: 2.7 M 2024-09-14T03:44:48.980 INFO:teuthology.orchestra.run.smithi039.stdout:Downloading Packages: 2024-09-14T03:44:49.054 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : libcephfs-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:44:49.054 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:49.054 INFO:teuthology.orchestra.run.smithi129.stdout:Installed: 2024-09-14T03:44:49.054 INFO:teuthology.orchestra.run.smithi129.stdout: libcephfs-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:44:49.054 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:49.054 INFO:teuthology.orchestra.run.smithi129.stdout:Complete! 2024-09-14T03:44:49.178 INFO:teuthology.orchestra.run.smithi039.stdout:(1/2): fuse-2.9.9-16.el9.x86_64.rpm 522 kB/s | 80 kB 00:00 2024-09-14T03:44:49.271 DEBUG:teuthology.orchestra.run.smithi129:> sudo yum -y install librados2 2024-09-14T03:44:49.304 INFO:teuthology.orchestra.run.smithi039.stdout:(2/2): ceph-fuse-18.2.4-845.ge4e227b3.el9.x86_6 3.0 MB/s | 852 kB 00:00 2024-09-14T03:44:49.304 INFO:teuthology.orchestra.run.smithi039.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:44:49.304 INFO:teuthology.orchestra.run.smithi039.stdout:Total 2.8 MB/s | 932 kB 00:00 2024-09-14T03:44:49.317 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction check 2024-09-14T03:44:49.339 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction check succeeded. 2024-09-14T03:44:49.340 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction test 2024-09-14T03:44:49.425 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction test succeeded. 2024-09-14T03:44:49.425 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction 2024-09-14T03:44:49.626 INFO:teuthology.orchestra.run.smithi039.stdout: Preparing : 1/1 2024-09-14T03:44:49.807 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : fuse-2.9.9-16.el9.x86_64 1/2 2024-09-14T03:44:49.836 INFO:teuthology.orchestra.run.smithi129.stdout:Last metadata expiration check: 0:01:27 ago on Sat 14 Sep 2024 03:43:22 AM UTC. 2024-09-14T03:44:49.889 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : ceph-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 2/2 2024-09-14T03:44:49.928 INFO:teuthology.orchestra.run.smithi129.stdout:Package librados2-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:44:49.970 INFO:teuthology.orchestra.run.smithi129.stdout:Dependencies resolved. 2024-09-14T03:44:49.971 INFO:teuthology.orchestra.run.smithi129.stdout:Nothing to do. 2024-09-14T03:44:49.971 INFO:teuthology.orchestra.run.smithi129.stdout:Complete! 2024-09-14T03:44:50.050 DEBUG:teuthology.orchestra.run.smithi129:> sudo yum -y install librbd1 2024-09-14T03:44:50.590 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: ceph-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 2/2 2024-09-14T03:44:50.590 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : ceph-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 1/2 2024-09-14T03:44:50.613 INFO:teuthology.orchestra.run.smithi129.stdout:Last metadata expiration check: 0:01:28 ago on Sat 14 Sep 2024 03:43:22 AM UTC. 2024-09-14T03:44:50.703 INFO:teuthology.orchestra.run.smithi129.stdout:Package librbd1-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:44:50.746 INFO:teuthology.orchestra.run.smithi129.stdout:Dependencies resolved. 2024-09-14T03:44:50.746 INFO:teuthology.orchestra.run.smithi129.stdout:Nothing to do. 2024-09-14T03:44:50.747 INFO:teuthology.orchestra.run.smithi129.stdout:Complete! 2024-09-14T03:44:50.827 DEBUG:teuthology.orchestra.run.smithi129:> sudo yum -y install python3-rados 2024-09-14T03:44:50.875 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : fuse-2.9.9-16.el9.x86_64 2/2 2024-09-14T03:44:50.875 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:50.875 INFO:teuthology.orchestra.run.smithi039.stdout:Installed: 2024-09-14T03:44:50.875 INFO:teuthology.orchestra.run.smithi039.stdout: ceph-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 fuse-2.9.9-16.el9.x86_64 2024-09-14T03:44:50.875 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:50.875 INFO:teuthology.orchestra.run.smithi039.stdout:Complete! 2024-09-14T03:44:51.150 DEBUG:teuthology.orchestra.run.smithi039:> sudo yum -y install librados-devel 2024-09-14T03:44:51.394 INFO:teuthology.orchestra.run.smithi129.stdout:Last metadata expiration check: 0:01:29 ago on Sat 14 Sep 2024 03:43:22 AM UTC. 2024-09-14T03:44:51.484 INFO:teuthology.orchestra.run.smithi129.stdout:Package python3-rados-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:44:51.528 INFO:teuthology.orchestra.run.smithi129.stdout:Dependencies resolved. 2024-09-14T03:44:51.529 INFO:teuthology.orchestra.run.smithi129.stdout:Nothing to do. 2024-09-14T03:44:51.529 INFO:teuthology.orchestra.run.smithi129.stdout:Complete! 2024-09-14T03:44:51.611 DEBUG:teuthology.orchestra.run.smithi129:> sudo yum -y install python3-rgw 2024-09-14T03:44:51.743 INFO:teuthology.orchestra.run.smithi039.stdout:Last metadata expiration check: 0:01:27 ago on Sat 14 Sep 2024 03:43:24 AM UTC. 2024-09-14T03:44:51.883 INFO:teuthology.orchestra.run.smithi039.stdout:Dependencies resolved. 2024-09-14T03:44:51.884 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:44:51.884 INFO:teuthology.orchestra.run.smithi039.stdout: Package Arch Version Repo Size 2024-09-14T03:44:51.884 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:44:51.885 INFO:teuthology.orchestra.run.smithi039.stdout:Installing: 2024-09-14T03:44:51.885 INFO:teuthology.orchestra.run.smithi039.stdout: librados-devel x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 127 k 2024-09-14T03:44:51.885 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:51.885 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction Summary 2024-09-14T03:44:51.885 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:44:51.885 INFO:teuthology.orchestra.run.smithi039.stdout:Install 1 Package 2024-09-14T03:44:51.885 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:51.885 INFO:teuthology.orchestra.run.smithi039.stdout:Total download size: 127 k 2024-09-14T03:44:51.885 INFO:teuthology.orchestra.run.smithi039.stdout:Installed size: 456 k 2024-09-14T03:44:51.885 INFO:teuthology.orchestra.run.smithi039.stdout:Downloading Packages: 2024-09-14T03:44:52.100 INFO:teuthology.orchestra.run.smithi039.stdout:librados-devel-18.2.4-845.ge4e227b3.el9.x86_64. 589 kB/s | 127 kB 00:00 2024-09-14T03:44:52.101 INFO:teuthology.orchestra.run.smithi039.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:44:52.101 INFO:teuthology.orchestra.run.smithi039.stdout:Total 585 kB/s | 127 kB 00:00 2024-09-14T03:44:52.101 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction check 2024-09-14T03:44:52.107 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction check succeeded. 2024-09-14T03:44:52.107 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction test 2024-09-14T03:44:52.169 INFO:teuthology.orchestra.run.smithi129.stdout:Last metadata expiration check: 0:01:30 ago on Sat 14 Sep 2024 03:43:22 AM UTC. 2024-09-14T03:44:52.174 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction test succeeded. 2024-09-14T03:44:52.174 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction 2024-09-14T03:44:52.259 INFO:teuthology.orchestra.run.smithi129.stdout:Package python3-rgw-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:44:52.301 INFO:teuthology.orchestra.run.smithi129.stdout:Dependencies resolved. 2024-09-14T03:44:52.302 INFO:teuthology.orchestra.run.smithi129.stdout:Nothing to do. 2024-09-14T03:44:52.302 INFO:teuthology.orchestra.run.smithi129.stdout:Complete! 2024-09-14T03:44:52.333 INFO:teuthology.orchestra.run.smithi039.stdout: Preparing : 1/1 2024-09-14T03:44:52.386 DEBUG:teuthology.orchestra.run.smithi129:> sudo yum -y install python3-cephfs 2024-09-14T03:44:52.452 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : librados-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:44:52.886 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: librados-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:44:52.944 INFO:teuthology.orchestra.run.smithi129.stdout:Last metadata expiration check: 0:01:30 ago on Sat 14 Sep 2024 03:43:22 AM UTC. 2024-09-14T03:44:53.034 INFO:teuthology.orchestra.run.smithi129.stdout:Package python3-cephfs-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:44:53.076 INFO:teuthology.orchestra.run.smithi129.stdout:Dependencies resolved. 2024-09-14T03:44:53.077 INFO:teuthology.orchestra.run.smithi129.stdout:Nothing to do. 2024-09-14T03:44:53.077 INFO:teuthology.orchestra.run.smithi129.stdout:Complete! 2024-09-14T03:44:53.147 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : librados-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:44:53.147 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:53.147 INFO:teuthology.orchestra.run.smithi039.stdout:Installed: 2024-09-14T03:44:53.147 INFO:teuthology.orchestra.run.smithi039.stdout: librados-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:44:53.147 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:53.147 INFO:teuthology.orchestra.run.smithi039.stdout:Complete! 2024-09-14T03:44:53.153 DEBUG:teuthology.orchestra.run.smithi129:> sudo yum -y install python3-rbd 2024-09-14T03:44:53.376 DEBUG:teuthology.orchestra.run.smithi039:> sudo yum -y install libcephfs2 2024-09-14T03:44:53.713 INFO:teuthology.orchestra.run.smithi129.stdout:Last metadata expiration check: 0:01:31 ago on Sat 14 Sep 2024 03:43:22 AM UTC. 2024-09-14T03:44:53.804 INFO:teuthology.orchestra.run.smithi129.stdout:Package python3-rbd-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:44:53.847 INFO:teuthology.orchestra.run.smithi129.stdout:Dependencies resolved. 2024-09-14T03:44:53.848 INFO:teuthology.orchestra.run.smithi129.stdout:Nothing to do. 2024-09-14T03:44:53.848 INFO:teuthology.orchestra.run.smithi129.stdout:Complete! 2024-09-14T03:44:53.924 DEBUG:teuthology.orchestra.run.smithi129:> sudo yum -y install rbd-fuse 2024-09-14T03:44:53.965 INFO:teuthology.orchestra.run.smithi039.stdout:Last metadata expiration check: 0:01:29 ago on Sat 14 Sep 2024 03:43:24 AM UTC. 2024-09-14T03:44:54.058 INFO:teuthology.orchestra.run.smithi039.stdout:Package libcephfs2-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:44:54.104 INFO:teuthology.orchestra.run.smithi039.stdout:Dependencies resolved. 2024-09-14T03:44:54.105 INFO:teuthology.orchestra.run.smithi039.stdout:Nothing to do. 2024-09-14T03:44:54.105 INFO:teuthology.orchestra.run.smithi039.stdout:Complete! 2024-09-14T03:44:54.183 DEBUG:teuthology.orchestra.run.smithi039:> sudo yum -y install libcephfs-devel 2024-09-14T03:44:54.485 INFO:teuthology.orchestra.run.smithi129.stdout:Last metadata expiration check: 0:01:32 ago on Sat 14 Sep 2024 03:43:22 AM UTC. 2024-09-14T03:44:54.619 INFO:teuthology.orchestra.run.smithi129.stdout:Dependencies resolved. 2024-09-14T03:44:54.620 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:54.621 INFO:teuthology.orchestra.run.smithi129.stdout: Package Architecture Version Repository Size 2024-09-14T03:44:54.621 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:54.621 INFO:teuthology.orchestra.run.smithi129.stdout:Installing: 2024-09-14T03:44:54.621 INFO:teuthology.orchestra.run.smithi129.stdout: rbd-fuse x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 87 k 2024-09-14T03:44:54.621 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:54.621 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction Summary 2024-09-14T03:44:54.621 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:54.621 INFO:teuthology.orchestra.run.smithi129.stdout:Install 1 Package 2024-09-14T03:44:54.622 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:54.622 INFO:teuthology.orchestra.run.smithi129.stdout:Total download size: 87 k 2024-09-14T03:44:54.622 INFO:teuthology.orchestra.run.smithi129.stdout:Installed size: 227 k 2024-09-14T03:44:54.622 INFO:teuthology.orchestra.run.smithi129.stdout:Downloading Packages: 2024-09-14T03:44:54.769 INFO:teuthology.orchestra.run.smithi039.stdout:Last metadata expiration check: 0:01:30 ago on Sat 14 Sep 2024 03:43:24 AM UTC. 2024-09-14T03:44:54.909 INFO:teuthology.orchestra.run.smithi039.stdout:Dependencies resolved. 2024-09-14T03:44:54.910 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:44:54.910 INFO:teuthology.orchestra.run.smithi039.stdout: Package Arch Version Repo Size 2024-09-14T03:44:54.910 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:44:54.911 INFO:teuthology.orchestra.run.smithi039.stdout:Installing: 2024-09-14T03:44:54.911 INFO:teuthology.orchestra.run.smithi039.stdout: libcephfs-devel x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 31 k 2024-09-14T03:44:54.911 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:54.911 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction Summary 2024-09-14T03:44:54.911 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:44:54.911 INFO:teuthology.orchestra.run.smithi039.stdout:Install 1 Package 2024-09-14T03:44:54.911 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:54.911 INFO:teuthology.orchestra.run.smithi039.stdout:Total download size: 31 k 2024-09-14T03:44:54.911 INFO:teuthology.orchestra.run.smithi039.stdout:Installed size: 139 k 2024-09-14T03:44:54.911 INFO:teuthology.orchestra.run.smithi039.stdout:Downloading Packages: 2024-09-14T03:44:54.916 INFO:teuthology.orchestra.run.smithi129.stdout:rbd-fuse-18.2.4-845.ge4e227b3.el9.x86_64.rpm 295 kB/s | 87 kB 00:00 2024-09-14T03:44:54.917 INFO:teuthology.orchestra.run.smithi129.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:44:54.917 INFO:teuthology.orchestra.run.smithi129.stdout:Total 293 kB/s | 87 kB 00:00 2024-09-14T03:44:54.917 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction check 2024-09-14T03:44:54.924 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction check succeeded. 2024-09-14T03:44:54.924 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction test 2024-09-14T03:44:54.992 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction test succeeded. 2024-09-14T03:44:54.992 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction 2024-09-14T03:44:55.087 INFO:teuthology.orchestra.run.smithi039.stdout:libcephfs-devel-18.2.4-845.ge4e227b3.el9.x86_64 177 kB/s | 31 kB 00:00 2024-09-14T03:44:55.088 INFO:teuthology.orchestra.run.smithi039.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:44:55.088 INFO:teuthology.orchestra.run.smithi039.stdout:Total 175 kB/s | 31 kB 00:00 2024-09-14T03:44:55.088 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction check 2024-09-14T03:44:55.091 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction check succeeded. 2024-09-14T03:44:55.091 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction test 2024-09-14T03:44:55.120 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction test succeeded. 2024-09-14T03:44:55.120 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction 2024-09-14T03:44:55.153 INFO:teuthology.orchestra.run.smithi129.stdout: Preparing : 1/1 2024-09-14T03:44:55.209 INFO:teuthology.orchestra.run.smithi039.stdout: Preparing : 1/1 2024-09-14T03:44:55.254 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : rbd-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:44:55.329 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : libcephfs-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:44:55.628 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: rbd-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:44:55.696 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: libcephfs-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:44:55.905 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : rbd-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:44:55.905 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:55.905 INFO:teuthology.orchestra.run.smithi129.stdout:Installed: 2024-09-14T03:44:55.905 INFO:teuthology.orchestra.run.smithi129.stdout: rbd-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:44:55.905 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:55.905 INFO:teuthology.orchestra.run.smithi129.stdout:Complete! 2024-09-14T03:44:55.929 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : libcephfs-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:44:55.929 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:55.929 INFO:teuthology.orchestra.run.smithi039.stdout:Installed: 2024-09-14T03:44:55.929 INFO:teuthology.orchestra.run.smithi039.stdout: libcephfs-devel-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:44:55.929 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:44:55.929 INFO:teuthology.orchestra.run.smithi039.stdout:Complete! 2024-09-14T03:44:56.127 DEBUG:teuthology.orchestra.run.smithi039:> sudo yum -y install librados2 2024-09-14T03:44:56.162 DEBUG:teuthology.orchestra.run.smithi129:> sudo yum -y install rbd-mirror 2024-09-14T03:44:56.717 INFO:teuthology.orchestra.run.smithi039.stdout:Last metadata expiration check: 0:01:32 ago on Sat 14 Sep 2024 03:43:24 AM UTC. 2024-09-14T03:44:56.741 INFO:teuthology.orchestra.run.smithi129.stdout:Last metadata expiration check: 0:01:34 ago on Sat 14 Sep 2024 03:43:22 AM UTC. 2024-09-14T03:44:56.812 INFO:teuthology.orchestra.run.smithi039.stdout:Package librados2-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:44:56.858 INFO:teuthology.orchestra.run.smithi039.stdout:Dependencies resolved. 2024-09-14T03:44:56.859 INFO:teuthology.orchestra.run.smithi039.stdout:Nothing to do. 2024-09-14T03:44:56.859 INFO:teuthology.orchestra.run.smithi039.stdout:Complete! 2024-09-14T03:44:56.875 INFO:teuthology.orchestra.run.smithi129.stdout:Dependencies resolved. 2024-09-14T03:44:56.876 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:56.876 INFO:teuthology.orchestra.run.smithi129.stdout: Package Arch Version Repo Size 2024-09-14T03:44:56.876 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:56.876 INFO:teuthology.orchestra.run.smithi129.stdout:Installing: 2024-09-14T03:44:56.876 INFO:teuthology.orchestra.run.smithi129.stdout: rbd-mirror x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 3.0 M 2024-09-14T03:44:56.876 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:56.876 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction Summary 2024-09-14T03:44:56.876 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:56.876 INFO:teuthology.orchestra.run.smithi129.stdout:Install 1 Package 2024-09-14T03:44:56.876 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:56.877 INFO:teuthology.orchestra.run.smithi129.stdout:Total download size: 3.0 M 2024-09-14T03:44:56.877 INFO:teuthology.orchestra.run.smithi129.stdout:Installed size: 12 M 2024-09-14T03:44:56.877 INFO:teuthology.orchestra.run.smithi129.stdout:Downloading Packages: 2024-09-14T03:44:56.935 DEBUG:teuthology.orchestra.run.smithi039:> sudo yum -y install librbd1 2024-09-14T03:44:57.204 INFO:teuthology.orchestra.run.smithi129.stdout:rbd-mirror-18.2.4-845.ge4e227b3.el9.x86_64.rpm 9.1 MB/s | 3.0 MB 00:00 2024-09-14T03:44:57.204 INFO:teuthology.orchestra.run.smithi129.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:44:57.205 INFO:teuthology.orchestra.run.smithi129.stdout:Total 9.1 MB/s | 3.0 MB 00:00 2024-09-14T03:44:57.205 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction check 2024-09-14T03:44:57.214 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction check succeeded. 2024-09-14T03:44:57.214 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction test 2024-09-14T03:44:57.293 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction test succeeded. 2024-09-14T03:44:57.293 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction 2024-09-14T03:44:57.524 INFO:teuthology.orchestra.run.smithi039.stdout:Last metadata expiration check: 0:01:33 ago on Sat 14 Sep 2024 03:43:24 AM UTC. 2024-09-14T03:44:57.619 INFO:teuthology.orchestra.run.smithi039.stdout:Package librbd1-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:44:57.664 INFO:teuthology.orchestra.run.smithi039.stdout:Dependencies resolved. 2024-09-14T03:44:57.665 INFO:teuthology.orchestra.run.smithi039.stdout:Nothing to do. 2024-09-14T03:44:57.665 INFO:teuthology.orchestra.run.smithi039.stdout:Complete! 2024-09-14T03:44:57.709 INFO:teuthology.orchestra.run.smithi129.stdout: Preparing : 1/1 2024-09-14T03:44:57.732 DEBUG:teuthology.orchestra.run.smithi039:> sudo yum -y install python3-rados 2024-09-14T03:44:57.780 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : rbd-mirror-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:44:57.804 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: rbd-mirror-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:44:57.804 INFO:teuthology.orchestra.run.smithi129.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:44:57.805 INFO:teuthology.orchestra.run.smithi129.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-14T03:44:57.805 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-14T03:44:57.805 INFO:teuthology.orchestra.run.smithi129.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-14T03:44:57.805 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:58.317 INFO:teuthology.orchestra.run.smithi039.stdout:Last metadata expiration check: 0:01:34 ago on Sat 14 Sep 2024 03:43:24 AM UTC. 2024-09-14T03:44:58.411 INFO:teuthology.orchestra.run.smithi039.stdout:Package python3-rados-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:44:58.457 INFO:teuthology.orchestra.run.smithi039.stdout:Dependencies resolved. 2024-09-14T03:44:58.459 INFO:teuthology.orchestra.run.smithi039.stdout:Nothing to do. 2024-09-14T03:44:58.459 INFO:teuthology.orchestra.run.smithi039.stdout:Complete! 2024-09-14T03:44:58.527 DEBUG:teuthology.orchestra.run.smithi039:> sudo yum -y install python3-rgw 2024-09-14T03:44:58.888 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : rbd-mirror-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:44:58.888 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:58.888 INFO:teuthology.orchestra.run.smithi129.stdout:Installed: 2024-09-14T03:44:58.889 INFO:teuthology.orchestra.run.smithi129.stdout: rbd-mirror-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:44:58.889 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:58.889 INFO:teuthology.orchestra.run.smithi129.stdout:Complete! 2024-09-14T03:44:59.090 DEBUG:teuthology.orchestra.run.smithi129:> sudo yum -y install rbd-nbd 2024-09-14T03:44:59.116 INFO:teuthology.orchestra.run.smithi039.stdout:Last metadata expiration check: 0:01:35 ago on Sat 14 Sep 2024 03:43:24 AM UTC. 2024-09-14T03:44:59.212 INFO:teuthology.orchestra.run.smithi039.stdout:Package python3-rgw-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:44:59.257 INFO:teuthology.orchestra.run.smithi039.stdout:Dependencies resolved. 2024-09-14T03:44:59.258 INFO:teuthology.orchestra.run.smithi039.stdout:Nothing to do. 2024-09-14T03:44:59.258 INFO:teuthology.orchestra.run.smithi039.stdout:Complete! 2024-09-14T03:44:59.324 DEBUG:teuthology.orchestra.run.smithi039:> sudo yum -y install python3-cephfs 2024-09-14T03:44:59.665 INFO:teuthology.orchestra.run.smithi129.stdout:Last metadata expiration check: 0:01:37 ago on Sat 14 Sep 2024 03:43:22 AM UTC. 2024-09-14T03:44:59.805 INFO:teuthology.orchestra.run.smithi129.stdout:Dependencies resolved. 2024-09-14T03:44:59.805 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:59.805 INFO:teuthology.orchestra.run.smithi129.stdout: Package Architecture Version Repository Size 2024-09-14T03:44:59.805 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:59.805 INFO:teuthology.orchestra.run.smithi129.stdout:Installing: 2024-09-14T03:44:59.806 INFO:teuthology.orchestra.run.smithi129.stdout: rbd-nbd x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 172 k 2024-09-14T03:44:59.806 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:59.806 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction Summary 2024-09-14T03:44:59.806 INFO:teuthology.orchestra.run.smithi129.stdout:================================================================================ 2024-09-14T03:44:59.806 INFO:teuthology.orchestra.run.smithi129.stdout:Install 1 Package 2024-09-14T03:44:59.806 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:44:59.806 INFO:teuthology.orchestra.run.smithi129.stdout:Total download size: 172 k 2024-09-14T03:44:59.806 INFO:teuthology.orchestra.run.smithi129.stdout:Installed size: 494 k 2024-09-14T03:44:59.807 INFO:teuthology.orchestra.run.smithi129.stdout:Downloading Packages: 2024-09-14T03:44:59.910 INFO:teuthology.orchestra.run.smithi039.stdout:Last metadata expiration check: 0:01:35 ago on Sat 14 Sep 2024 03:43:24 AM UTC. 2024-09-14T03:44:59.992 INFO:teuthology.orchestra.run.smithi129.stdout:rbd-nbd-18.2.4-845.ge4e227b3.el9.x86_64.rpm 918 kB/s | 172 kB 00:00 2024-09-14T03:44:59.993 INFO:teuthology.orchestra.run.smithi129.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:44:59.993 INFO:teuthology.orchestra.run.smithi129.stdout:Total 912 kB/s | 172 kB 00:00 2024-09-14T03:44:59.993 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction check 2024-09-14T03:45:00.001 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction check succeeded. 2024-09-14T03:45:00.001 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction test 2024-09-14T03:45:00.004 INFO:teuthology.orchestra.run.smithi039.stdout:Package python3-cephfs-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:45:00.050 INFO:teuthology.orchestra.run.smithi039.stdout:Dependencies resolved. 2024-09-14T03:45:00.051 INFO:teuthology.orchestra.run.smithi039.stdout:Nothing to do. 2024-09-14T03:45:00.051 INFO:teuthology.orchestra.run.smithi039.stdout:Complete! 2024-09-14T03:45:00.069 INFO:teuthology.orchestra.run.smithi129.stdout:Transaction test succeeded. 2024-09-14T03:45:00.069 INFO:teuthology.orchestra.run.smithi129.stdout:Running transaction 2024-09-14T03:45:00.124 DEBUG:teuthology.orchestra.run.smithi039:> sudo yum -y install python3-rbd 2024-09-14T03:45:00.235 INFO:teuthology.orchestra.run.smithi129.stdout: Preparing : 1/1 2024-09-14T03:45:00.340 INFO:teuthology.orchestra.run.smithi129.stdout: Installing : rbd-nbd-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:45:00.714 INFO:teuthology.orchestra.run.smithi039.stdout:Last metadata expiration check: 0:01:36 ago on Sat 14 Sep 2024 03:43:24 AM UTC. 2024-09-14T03:45:00.746 INFO:teuthology.orchestra.run.smithi129.stdout: Running scriptlet: rbd-nbd-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:45:00.807 INFO:teuthology.orchestra.run.smithi039.stdout:Package python3-rbd-2:18.2.4-845.ge4e227b3.el9.x86_64 is already installed. 2024-09-14T03:45:00.853 INFO:teuthology.orchestra.run.smithi039.stdout:Dependencies resolved. 2024-09-14T03:45:00.854 INFO:teuthology.orchestra.run.smithi039.stdout:Nothing to do. 2024-09-14T03:45:00.854 INFO:teuthology.orchestra.run.smithi039.stdout:Complete! 2024-09-14T03:45:00.927 DEBUG:teuthology.orchestra.run.smithi039:> sudo yum -y install rbd-fuse 2024-09-14T03:45:01.062 INFO:teuthology.orchestra.run.smithi129.stdout: Verifying : rbd-nbd-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:45:01.062 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:45:01.062 INFO:teuthology.orchestra.run.smithi129.stdout:Installed: 2024-09-14T03:45:01.062 INFO:teuthology.orchestra.run.smithi129.stdout: rbd-nbd-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:45:01.062 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:45:01.062 INFO:teuthology.orchestra.run.smithi129.stdout:Complete! 2024-09-14T03:45:01.323 DEBUG:teuthology.parallel:result is None 2024-09-14T03:45:01.513 INFO:teuthology.orchestra.run.smithi039.stdout:Last metadata expiration check: 0:01:37 ago on Sat 14 Sep 2024 03:43:24 AM UTC. 2024-09-14T03:45:01.655 INFO:teuthology.orchestra.run.smithi039.stdout:Dependencies resolved. 2024-09-14T03:45:01.655 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:45:01.655 INFO:teuthology.orchestra.run.smithi039.stdout: Package Architecture Version Repository Size 2024-09-14T03:45:01.655 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:45:01.656 INFO:teuthology.orchestra.run.smithi039.stdout:Installing: 2024-09-14T03:45:01.656 INFO:teuthology.orchestra.run.smithi039.stdout: rbd-fuse x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 87 k 2024-09-14T03:45:01.656 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:45:01.656 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction Summary 2024-09-14T03:45:01.656 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:45:01.656 INFO:teuthology.orchestra.run.smithi039.stdout:Install 1 Package 2024-09-14T03:45:01.656 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:45:01.656 INFO:teuthology.orchestra.run.smithi039.stdout:Total download size: 87 k 2024-09-14T03:45:01.656 INFO:teuthology.orchestra.run.smithi039.stdout:Installed size: 227 k 2024-09-14T03:45:01.656 INFO:teuthology.orchestra.run.smithi039.stdout:Downloading Packages: 2024-09-14T03:45:01.857 INFO:teuthology.orchestra.run.smithi039.stdout:rbd-fuse-18.2.4-845.ge4e227b3.el9.x86_64.rpm 432 kB/s | 87 kB 00:00 2024-09-14T03:45:01.858 INFO:teuthology.orchestra.run.smithi039.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:45:01.858 INFO:teuthology.orchestra.run.smithi039.stdout:Total 430 kB/s | 87 kB 00:00 2024-09-14T03:45:01.858 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction check 2024-09-14T03:45:01.866 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction check succeeded. 2024-09-14T03:45:01.866 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction test 2024-09-14T03:45:01.939 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction test succeeded. 2024-09-14T03:45:01.939 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction 2024-09-14T03:45:02.223 INFO:teuthology.orchestra.run.smithi039.stdout: Preparing : 1/1 2024-09-14T03:45:02.332 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : rbd-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:45:02.716 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: rbd-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:45:02.979 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : rbd-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:45:02.979 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:45:02.979 INFO:teuthology.orchestra.run.smithi039.stdout:Installed: 2024-09-14T03:45:02.980 INFO:teuthology.orchestra.run.smithi039.stdout: rbd-fuse-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:45:02.980 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:45:02.980 INFO:teuthology.orchestra.run.smithi039.stdout:Complete! 2024-09-14T03:45:03.202 DEBUG:teuthology.orchestra.run.smithi039:> sudo yum -y install rbd-mirror 2024-09-14T03:45:03.794 INFO:teuthology.orchestra.run.smithi039.stdout:Last metadata expiration check: 0:01:39 ago on Sat 14 Sep 2024 03:43:24 AM UTC. 2024-09-14T03:45:03.937 INFO:teuthology.orchestra.run.smithi039.stdout:Dependencies resolved. 2024-09-14T03:45:03.937 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:45:03.937 INFO:teuthology.orchestra.run.smithi039.stdout: Package Arch Version Repo Size 2024-09-14T03:45:03.937 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:45:03.938 INFO:teuthology.orchestra.run.smithi039.stdout:Installing: 2024-09-14T03:45:03.938 INFO:teuthology.orchestra.run.smithi039.stdout: rbd-mirror x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 3.0 M 2024-09-14T03:45:03.938 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:45:03.938 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction Summary 2024-09-14T03:45:03.938 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:45:03.938 INFO:teuthology.orchestra.run.smithi039.stdout:Install 1 Package 2024-09-14T03:45:03.938 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:45:03.938 INFO:teuthology.orchestra.run.smithi039.stdout:Total download size: 3.0 M 2024-09-14T03:45:03.938 INFO:teuthology.orchestra.run.smithi039.stdout:Installed size: 12 M 2024-09-14T03:45:03.938 INFO:teuthology.orchestra.run.smithi039.stdout:Downloading Packages: 2024-09-14T03:45:04.294 INFO:teuthology.orchestra.run.smithi039.stdout:rbd-mirror-18.2.4-845.ge4e227b3.el9.x86_64.rpm 8.4 MB/s | 3.0 MB 00:00 2024-09-14T03:45:04.294 INFO:teuthology.orchestra.run.smithi039.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:45:04.295 INFO:teuthology.orchestra.run.smithi039.stdout:Total 8.4 MB/s | 3.0 MB 00:00 2024-09-14T03:45:04.295 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction check 2024-09-14T03:45:04.305 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction check succeeded. 2024-09-14T03:45:04.305 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction test 2024-09-14T03:45:04.389 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction test succeeded. 2024-09-14T03:45:04.389 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction 2024-09-14T03:45:04.831 INFO:teuthology.orchestra.run.smithi039.stdout: Preparing : 1/1 2024-09-14T03:45:04.911 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : rbd-mirror-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:45:04.937 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: rbd-mirror-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:45:04.937 INFO:teuthology.orchestra.run.smithi039.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-09-14T03:45:04.938 INFO:teuthology.orchestra.run.smithi039.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-09-14T03:45:04.938 INFO:teuthology.orchestra.run.smithi039.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-14T03:45:04.938 INFO:teuthology.orchestra.run.smithi039.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-09-14T03:45:04.938 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:45:06.105 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : rbd-mirror-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:45:06.105 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:45:06.105 INFO:teuthology.orchestra.run.smithi039.stdout:Installed: 2024-09-14T03:45:06.105 INFO:teuthology.orchestra.run.smithi039.stdout: rbd-mirror-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:45:06.105 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:45:06.105 INFO:teuthology.orchestra.run.smithi039.stdout:Complete! 2024-09-14T03:45:06.318 DEBUG:teuthology.orchestra.run.smithi039:> sudo yum -y install rbd-nbd 2024-09-14T03:45:06.910 INFO:teuthology.orchestra.run.smithi039.stdout:Last metadata expiration check: 0:01:42 ago on Sat 14 Sep 2024 03:43:24 AM UTC. 2024-09-14T03:45:07.050 INFO:teuthology.orchestra.run.smithi039.stdout:Dependencies resolved. 2024-09-14T03:45:07.051 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:45:07.051 INFO:teuthology.orchestra.run.smithi039.stdout: Package Architecture Version Repository Size 2024-09-14T03:45:07.051 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:45:07.051 INFO:teuthology.orchestra.run.smithi039.stdout:Installing: 2024-09-14T03:45:07.051 INFO:teuthology.orchestra.run.smithi039.stdout: rbd-nbd x86_64 2:18.2.4-845.ge4e227b3.el9 ceph 172 k 2024-09-14T03:45:07.051 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:45:07.051 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction Summary 2024-09-14T03:45:07.052 INFO:teuthology.orchestra.run.smithi039.stdout:================================================================================ 2024-09-14T03:45:07.052 INFO:teuthology.orchestra.run.smithi039.stdout:Install 1 Package 2024-09-14T03:45:07.052 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:45:07.052 INFO:teuthology.orchestra.run.smithi039.stdout:Total download size: 172 k 2024-09-14T03:45:07.052 INFO:teuthology.orchestra.run.smithi039.stdout:Installed size: 494 k 2024-09-14T03:45:07.052 INFO:teuthology.orchestra.run.smithi039.stdout:Downloading Packages: 2024-09-14T03:45:07.272 INFO:teuthology.orchestra.run.smithi039.stdout:rbd-nbd-18.2.4-845.ge4e227b3.el9.x86_64.rpm 784 kB/s | 172 kB 00:00 2024-09-14T03:45:07.272 INFO:teuthology.orchestra.run.smithi039.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:45:07.273 INFO:teuthology.orchestra.run.smithi039.stdout:Total 779 kB/s | 172 kB 00:00 2024-09-14T03:45:07.273 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction check 2024-09-14T03:45:07.281 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction check succeeded. 2024-09-14T03:45:07.281 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction test 2024-09-14T03:45:07.354 INFO:teuthology.orchestra.run.smithi039.stdout:Transaction test succeeded. 2024-09-14T03:45:07.354 INFO:teuthology.orchestra.run.smithi039.stdout:Running transaction 2024-09-14T03:45:07.530 INFO:teuthology.orchestra.run.smithi039.stdout: Preparing : 1/1 2024-09-14T03:45:07.647 INFO:teuthology.orchestra.run.smithi039.stdout: Installing : rbd-nbd-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:45:08.103 INFO:teuthology.orchestra.run.smithi039.stdout: Running scriptlet: rbd-nbd-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:45:08.394 INFO:teuthology.orchestra.run.smithi039.stdout: Verifying : rbd-nbd-2:18.2.4-845.ge4e227b3.el9.x86_64 1/1 2024-09-14T03:45:08.394 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:45:08.395 INFO:teuthology.orchestra.run.smithi039.stdout:Installed: 2024-09-14T03:45:08.395 INFO:teuthology.orchestra.run.smithi039.stdout: rbd-nbd-2:18.2.4-845.ge4e227b3.el9.x86_64 2024-09-14T03:45:08.395 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:45:08.395 INFO:teuthology.orchestra.run.smithi039.stdout:Complete! 2024-09-14T03:45:08.627 DEBUG:teuthology.parallel:result is None 2024-09-14T03:45:08.627 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:45:08.627 INFO:teuthology.packaging:ref: None 2024-09-14T03:45:08.627 INFO:teuthology.packaging:tag: None 2024-09-14T03:45:08.627 INFO:teuthology.packaging:branch: reef 2024-09-14T03:45:08.627 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:45:08.628 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-09-14T03:45:08.850 DEBUG:teuthology.orchestra.run.smithi039:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-14T03:45:08.872 INFO:teuthology.orchestra.run.smithi039.stdout:18.2.4-845.ge4e227b3.el9 2024-09-14T03:45:08.873 INFO:teuthology.packaging:The installed version of ceph is 18.2.4-845.ge4e227b3.el9 2024-09-14T03:45:08.873 INFO:teuthology.task.install:The correct ceph version 18.2.4-845.ge4e227b3 is installed. 2024-09-14T03:45:08.874 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-09-14T03:45:08.875 INFO:teuthology.packaging:ref: None 2024-09-14T03:45:08.875 INFO:teuthology.packaging:tag: None 2024-09-14T03:45:08.875 INFO:teuthology.packaging:branch: reef 2024-09-14T03:45:08.875 INFO:teuthology.packaging:sha1: a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:45:08.875 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=reef 2024-09-14T03:45:09.107 DEBUG:teuthology.orchestra.run.smithi129:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-09-14T03:45:09.133 INFO:teuthology.orchestra.run.smithi129.stdout:18.2.4-845.ge4e227b3.el9 2024-09-14T03:45:09.133 INFO:teuthology.packaging:The installed version of ceph is 18.2.4-845.ge4e227b3.el9 2024-09-14T03:45:09.133 INFO:teuthology.task.install:The correct ceph version 18.2.4-845.ge4e227b3 is installed. 2024-09-14T03:45:09.135 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-09-14T03:45:09.135 DEBUG:teuthology.orchestra.run.smithi039:> set -ex 2024-09-14T03:45:09.135 DEBUG:teuthology.orchestra.run.smithi039:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-14T03:45:09.163 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T03:45:09.164 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-09-14T03:45:09.202 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-09-14T03:45:09.202 DEBUG:teuthology.orchestra.run.smithi039:> set -ex 2024-09-14T03:45:09.202 DEBUG:teuthology.orchestra.run.smithi039:> sudo dd of=/usr/bin/daemon-helper 2024-09-14T03:45:09.232 DEBUG:teuthology.orchestra.run.smithi039:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-14T03:45:09.300 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T03:45:09.300 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd of=/usr/bin/daemon-helper 2024-09-14T03:45:09.330 DEBUG:teuthology.orchestra.run.smithi129:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-09-14T03:45:09.400 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-09-14T03:45:09.400 DEBUG:teuthology.orchestra.run.smithi039:> set -ex 2024-09-14T03:45:09.400 DEBUG:teuthology.orchestra.run.smithi039:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-14T03:45:09.430 DEBUG:teuthology.orchestra.run.smithi039:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-14T03:45:09.498 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T03:45:09.498 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd of=/usr/bin/adjust-ulimits 2024-09-14T03:45:09.538 DEBUG:teuthology.orchestra.run.smithi129:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-09-14T03:45:09.605 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-09-14T03:45:09.605 DEBUG:teuthology.orchestra.run.smithi039:> set -ex 2024-09-14T03:45:09.605 DEBUG:teuthology.orchestra.run.smithi039:> sudo dd of=/usr/bin/stdin-killer 2024-09-14T03:45:09.633 DEBUG:teuthology.orchestra.run.smithi039:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-14T03:45:09.700 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T03:45:09.700 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd of=/usr/bin/stdin-killer 2024-09-14T03:45:09.725 DEBUG:teuthology.orchestra.run.smithi129:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-09-14T03:45:09.790 INFO:teuthology.run_tasks:Running task cephadm... 2024-09-14T03:45:09.893 INFO:tasks.cephadm:Config: {'compiled_cephadm_branch': 'reef', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:reef', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_AVAILABILITY'], 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8'} 2024-09-14T03:45:09.893 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:reef 2024-09-14T03:45:09.894 INFO:tasks.cephadm:Cluster fsid is bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T03:45:09.894 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-09-14T03:45:09.894 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.39', 'mon.c': '[v2:172.21.15.39:3301,v1:172.21.15.39:6790]', 'mon.b': '172.21.15.129'} 2024-09-14T03:45:09.894 INFO:tasks.cephadm:First mon is mon.a on smithi039 2024-09-14T03:45:09.894 INFO:tasks.cephadm:First mgr is y 2024-09-14T03:45:09.894 INFO:tasks.cephadm:Normalizing hostnames... 2024-09-14T03:45:09.895 DEBUG:teuthology.orchestra.run.smithi039:> sudo hostname $(hostname -s) 2024-09-14T03:45:09.922 DEBUG:teuthology.orchestra.run.smithi129:> sudo hostname $(hostname -s) 2024-09-14T03:45:09.949 INFO:tasks.cephadm:Downloading "compiled" cephadm from cachra for reef 2024-09-14T03:45:09.950 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:45:10.087 INFO:tasks.cephadm:builder_project result: [{'status': 'ready', 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8', 'extra': {'build_url': 'https://jenkins.ceph.com/job/ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos9,DIST=centos9,MACHINE_SIZE=gigantic/56069/', 'root_build_cause': 'SCMTRIGGER', 'version': '19.3.0-4907-ga2782315', 'node_name': '172.21.2.8+braggi08', 'job_name': 'ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=centos9,DIST=centos9,MACHINE_SIZE=gigantic', 'package_manager_version': '19.3.0-4907.ga2782315'}, 'url': 'https://4.chacra.ceph.com/r/ceph/main/a2782315255a3e7d5f6cdc9eb35785187ff6ddf8/centos/9/flavors/default/', 'distro_codename': None, 'modified': '2024-09-12 21:34:57.306979', 'distro_version': '9', 'project': 'ceph', 'flavor': 'default', 'ref': 'main', 'chacra_url': 'https://4.chacra.ceph.com/repos/ceph/main/a2782315255a3e7d5f6cdc9eb35785187ff6ddf8/centos/9/flavors/default/', 'archs': ['x86_64', 'source'], 'distro': 'centos'}] 2024-09-14T03:45:10.308 INFO:tasks.util.chacra:got chacra host 1.chacra.ceph.com, ref reef, sha1 e4e227b37858f5109138494e01e5ba1ba7b8ffe8 from https://shaman.ceph.com/api/search/?project=ceph&distros=centos%2F9%2Fx86_64&flavor=default&ref=reef 2024-09-14T03:45:10.309 INFO:tasks.cephadm:Discovered cachra url: https://1.chacra.ceph.com/binaries/ceph/reef/e4e227b37858f5109138494e01e5ba1ba7b8ffe8/centos/9/x86_64/flavors/default/cephadm 2024-09-14T03:45:10.310 INFO:tasks.cephadm:Downloading cephadm from url: https://1.chacra.ceph.com/binaries/ceph/reef/e4e227b37858f5109138494e01e5ba1ba7b8ffe8/centos/9/x86_64/flavors/default/cephadm 2024-09-14T03:45:10.310 DEBUG:teuthology.orchestra.run.smithi039:> curl --silent -L https://1.chacra.ceph.com/binaries/ceph/reef/e4e227b37858f5109138494e01e5ba1ba7b8ffe8/centos/9/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-14T03:45:10.658 INFO:teuthology.orchestra.run.smithi039.stdout:-rw-r--r--. 1 ubuntu ubuntu 217223 Sep 14 03:45 /home/ubuntu/cephtest/cephadm 2024-09-14T03:45:10.659 DEBUG:teuthology.orchestra.run.smithi129:> curl --silent -L https://1.chacra.ceph.com/binaries/ceph/reef/e4e227b37858f5109138494e01e5ba1ba7b8ffe8/centos/9/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-09-14T03:45:10.994 INFO:teuthology.orchestra.run.smithi129.stdout:-rw-r--r--. 1 ubuntu ubuntu 217223 Sep 14 03:45 /home/ubuntu/cephtest/cephadm 2024-09-14T03:45:10.994 DEBUG:teuthology.orchestra.run.smithi039:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-14T03:45:11.014 DEBUG:teuthology.orchestra.run.smithi129:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-09-14T03:45:11.044 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:reef on all hosts... 2024-09-14T03:45:11.045 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef pull 2024-09-14T03:45:11.057 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef pull 2024-09-14T03:45:12.615 INFO:teuthology.orchestra.run.smithi039.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-09-14T03:45:12.652 INFO:teuthology.orchestra.run.smithi129.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-09-14T03:45:52.064 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T03:45:52.064 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph_version": "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)", 2024-09-14T03:45:52.064 INFO:teuthology.orchestra.run.smithi039.stdout: "image_id": "67ddd33fa72f26a51c969a9043e7a3ddf784506b3d1949ad815ae4dd791beec1", 2024-09-14T03:45:52.065 INFO:teuthology.orchestra.run.smithi039.stdout: "repo_digests": [ 2024-09-14T03:45:52.065 INFO:teuthology.orchestra.run.smithi039.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322" 2024-09-14T03:45:52.065 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T03:45:52.065 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T03:45:53.545 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T03:45:53.545 INFO:teuthology.orchestra.run.smithi129.stdout: "ceph_version": "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)", 2024-09-14T03:45:53.545 INFO:teuthology.orchestra.run.smithi129.stdout: "image_id": "67ddd33fa72f26a51c969a9043e7a3ddf784506b3d1949ad815ae4dd791beec1", 2024-09-14T03:45:53.545 INFO:teuthology.orchestra.run.smithi129.stdout: "repo_digests": [ 2024-09-14T03:45:53.545 INFO:teuthology.orchestra.run.smithi129.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322" 2024-09-14T03:45:53.545 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T03:45:53.546 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T03:45:53.563 DEBUG:teuthology.orchestra.run.smithi039:> sudo mkdir -p /etc/ceph 2024-09-14T03:45:53.599 DEBUG:teuthology.orchestra.run.smithi129:> sudo mkdir -p /etc/ceph 2024-09-14T03:45:53.630 DEBUG:teuthology.orchestra.run.smithi039:> sudo chmod 777 /etc/ceph 2024-09-14T03:45:53.670 DEBUG:teuthology.orchestra.run.smithi129:> sudo chmod 777 /etc/ceph 2024-09-14T03:45:53.695 INFO:tasks.cephadm:Writing seed config... 2024-09-14T03:45:53.697 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-09-14T03:45:53.697 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-09-14T03:45:53.697 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-09-14T03:45:53.697 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-09-14T03:45:53.698 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-09-14T03:45:53.698 INFO:tasks.cephadm: override: [global] mon election default strategy = 1 2024-09-14T03:45:53.698 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-09-14T03:45:53.698 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-09-14T03:45:53.698 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-09-14T03:45:53.698 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-09-14T03:45:53.699 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-09-14T03:45:53.700 DEBUG:teuthology.orchestra.run.smithi039:> set -ex 2024-09-14T03:45:53.700 DEBUG:teuthology.orchestra.run.smithi039:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-09-14T03:45:53.727 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = bd7c803c-724b-11ef-bceb-c7b262605968 mon election default strategy = 1 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-09-14T03:45:53.728 DEBUG:teuthology.orchestra.run.smithi039:mon.a> sudo journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mon.a.service 2024-09-14T03:45:53.771 DEBUG:teuthology.orchestra.run.smithi039:mgr.y> sudo journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mgr.y.service 2024-09-14T03:45:53.814 INFO:tasks.cephadm:Bootstrapping... 2024-09-14T03:45:53.814 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef -v bootstrap --fsid bd7c803c-724b-11ef-bceb-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.39 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-09-14T03:45:53.975 INFO:teuthology.orchestra.run.smithi039.stdout:-------------------------------------------------------------------------------- 2024-09-14T03:45:53.975 INFO:teuthology.orchestra.run.smithi039.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:reef', '-v', 'bootstrap', '--fsid', 'bd7c803c-724b-11ef-bceb-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.39', '--skip-admin-label'] 2024-09-14T03:45:54.001 INFO:teuthology.orchestra.run.smithi039.stdout:/bin/podman: stdout 5.2.2 2024-09-14T03:45:54.002 INFO:teuthology.orchestra.run.smithi039.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-09-14T03:45:54.002 INFO:teuthology.orchestra.run.smithi039.stdout:Verifying podman|docker is present... 2024-09-14T03:45:54.026 INFO:teuthology.orchestra.run.smithi039.stdout:/bin/podman: stdout 5.2.2 2024-09-14T03:45:54.026 INFO:teuthology.orchestra.run.smithi039.stdout:Verifying lvm2 is present... 2024-09-14T03:45:54.026 INFO:teuthology.orchestra.run.smithi039.stdout:Verifying time synchronization is in place... 2024-09-14T03:45:54.034 INFO:teuthology.orchestra.run.smithi039.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-14T03:45:54.034 INFO:teuthology.orchestra.run.smithi039.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-14T03:45:54.040 INFO:teuthology.orchestra.run.smithi039.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-14T03:45:54.040 INFO:teuthology.orchestra.run.smithi039.stdout:systemctl: stdout inactive 2024-09-14T03:45:54.047 INFO:teuthology.orchestra.run.smithi039.stdout:systemctl: stdout enabled 2024-09-14T03:45:54.053 INFO:teuthology.orchestra.run.smithi039.stdout:systemctl: stdout active 2024-09-14T03:45:54.053 INFO:teuthology.orchestra.run.smithi039.stdout:Unit chronyd.service is enabled and running 2024-09-14T03:45:54.053 INFO:teuthology.orchestra.run.smithi039.stdout:Repeating the final host check... 2024-09-14T03:45:54.077 INFO:teuthology.orchestra.run.smithi039.stdout:/bin/podman: stdout 5.2.2 2024-09-14T03:45:54.077 INFO:teuthology.orchestra.run.smithi039.stdout:podman (/bin/podman) version 5.2.2 is present 2024-09-14T03:45:54.077 INFO:teuthology.orchestra.run.smithi039.stdout:systemctl is present 2024-09-14T03:45:54.077 INFO:teuthology.orchestra.run.smithi039.stdout:lvcreate is present 2024-09-14T03:45:54.084 INFO:teuthology.orchestra.run.smithi039.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-09-14T03:45:54.084 INFO:teuthology.orchestra.run.smithi039.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-09-14T03:45:54.090 INFO:teuthology.orchestra.run.smithi039.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-09-14T03:45:54.090 INFO:teuthology.orchestra.run.smithi039.stdout:systemctl: stdout inactive 2024-09-14T03:45:54.097 INFO:teuthology.orchestra.run.smithi039.stdout:systemctl: stdout enabled 2024-09-14T03:45:54.104 INFO:teuthology.orchestra.run.smithi039.stdout:systemctl: stdout active 2024-09-14T03:45:54.104 INFO:teuthology.orchestra.run.smithi039.stdout:Unit chronyd.service is enabled and running 2024-09-14T03:45:54.104 INFO:teuthology.orchestra.run.smithi039.stdout:Host looks OK 2024-09-14T03:45:54.104 INFO:teuthology.orchestra.run.smithi039.stdout:Cluster fsid: bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T03:45:54.104 INFO:teuthology.orchestra.run.smithi039.stdout:Acquiring lock 140265137188432 on /run/cephadm/bd7c803c-724b-11ef-bceb-c7b262605968.lock 2024-09-14T03:45:54.104 INFO:teuthology.orchestra.run.smithi039.stdout:Lock 140265137188432 acquired on /run/cephadm/bd7c803c-724b-11ef-bceb-c7b262605968.lock 2024-09-14T03:45:54.105 INFO:teuthology.orchestra.run.smithi039.stdout:Verifying IP 172.21.15.39 port 3300 ... 2024-09-14T03:45:54.105 INFO:teuthology.orchestra.run.smithi039.stdout:Verifying IP 172.21.15.39 port 6789 ... 2024-09-14T03:45:54.105 INFO:teuthology.orchestra.run.smithi039.stdout:Base mon IP(s) is [172.21.15.39:3300, 172.21.15.39:6789], mon addrv is [v2:172.21.15.39:3300,v1:172.21.15.39:6789] 2024-09-14T03:45:54.108 INFO:teuthology.orchestra.run.smithi039.stdout:/sbin/ip: stdout default via 172.21.15.254 dev ens1f0 proto dhcp src 172.21.15.39 metric 100 2024-09-14T03:45:54.108 INFO:teuthology.orchestra.run.smithi039.stdout:/sbin/ip: stdout 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.39 metric 100 2024-09-14T03:45:54.111 INFO:teuthology.orchestra.run.smithi039.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-09-14T03:45:54.111 INFO:teuthology.orchestra.run.smithi039.stdout:/sbin/ip: stdout fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2024-09-14T03:45:54.111 INFO:teuthology.orchestra.run.smithi039.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1786sec hoplimit 64 pref medium 2024-09-14T03:45:54.113 INFO:teuthology.orchestra.run.smithi039.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-09-14T03:45:54.113 INFO:teuthology.orchestra.run.smithi039.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-09-14T03:45:54.114 INFO:teuthology.orchestra.run.smithi039.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-14T03:45:54.114 INFO:teuthology.orchestra.run.smithi039.stdout:/sbin/ip: stdout 4: ens1f0: mtu 1500 state UP qlen 1000 2024-09-14T03:45:54.114 INFO:teuthology.orchestra.run.smithi039.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:febd:1370/64 scope link 2024-09-14T03:45:54.114 INFO:teuthology.orchestra.run.smithi039.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-09-14T03:45:54.114 INFO:teuthology.orchestra.run.smithi039.stdout:Mon IP `172.21.15.39` is in CIDR network `172.21.0.0/20` 2024-09-14T03:45:54.114 INFO:teuthology.orchestra.run.smithi039.stdout:Mon IP `172.21.15.39` is in CIDR network `172.21.0.0/20` 2024-09-14T03:45:54.115 INFO:teuthology.orchestra.run.smithi039.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-09-14T03:45:54.115 INFO:teuthology.orchestra.run.smithi039.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-09-14T03:45:54.115 INFO:teuthology.orchestra.run.smithi039.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-09-14T03:45:55.437 INFO:teuthology.orchestra.run.smithi039.stdout:/bin/podman: stdout 67ddd33fa72f26a51c969a9043e7a3ddf784506b3d1949ad815ae4dd791beec1 2024-09-14T03:45:55.437 INFO:teuthology.orchestra.run.smithi039.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:reef... 2024-09-14T03:45:55.437 INFO:teuthology.orchestra.run.smithi039.stdout:/bin/podman: stderr Getting image source signatures 2024-09-14T03:45:55.437 INFO:teuthology.orchestra.run.smithi039.stdout:/bin/podman: stderr Copying blob sha256:71878612010b3dfa9e3e456d432a49e20ac26634831a8a6522f4d782181c5840 2024-09-14T03:45:55.437 INFO:teuthology.orchestra.run.smithi039.stdout:/bin/podman: stderr Copying blob sha256:e8b54c863393b7a8216a71737771b73b16a8e43ec7acf927c7faa175c255e551 2024-09-14T03:45:55.437 INFO:teuthology.orchestra.run.smithi039.stdout:/bin/podman: stderr Copying config sha256:67ddd33fa72f26a51c969a9043e7a3ddf784506b3d1949ad815ae4dd791beec1 2024-09-14T03:45:55.437 INFO:teuthology.orchestra.run.smithi039.stdout:/bin/podman: stderr Writing manifest to image destination 2024-09-14T03:46:04.721 INFO:teuthology.orchestra.run.smithi039.stdout:ceph: stdout ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable) 2024-09-14T03:46:04.721 INFO:teuthology.orchestra.run.smithi039.stdout:Ceph version: ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable) 2024-09-14T03:46:04.721 INFO:teuthology.orchestra.run.smithi039.stdout:Extracting ceph user uid/gid from container image... 2024-09-14T03:46:06.602 INFO:teuthology.orchestra.run.smithi039.stdout:stat: stdout 167 167 2024-09-14T03:46:06.602 INFO:teuthology.orchestra.run.smithi039.stdout:Creating initial keys... 2024-09-14T03:46:07.616 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph-authtool: stdout AQD+BuVmGjtJOBAAOJ8nzPndippU3HfRQAAJaA== 2024-09-14T03:46:08.605 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph-authtool: stdout AQAAB+VmuTaQARAAm74JTa/nW6aTq/X/QcspFQ== 2024-09-14T03:46:09.637 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph-authtool: stdout AQAAB+Vm1d2AORAAcU5xvY/UvViVDcmdrVzLYw== 2024-09-14T03:46:09.637 INFO:teuthology.orchestra.run.smithi039.stdout:Creating initial monmap... 2024-09-14T03:46:10.711 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-14T03:46:10.711 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = pacific 2024-09-14T03:46:10.711 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T03:46:10.711 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-14T03:46:10.711 INFO:teuthology.orchestra.run.smithi039.stdout:monmaptool for a [v2:172.21.15.39:3300,v1:172.21.15.39:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-09-14T03:46:10.711 INFO:teuthology.orchestra.run.smithi039.stdout:setting min_mon_release = pacific 2024-09-14T03:46:10.711 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/monmaptool: set fsid to bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T03:46:10.711 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-09-14T03:46:10.711 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:46:10.711 INFO:teuthology.orchestra.run.smithi039.stdout:Creating mon... 2024-09-14T03:46:12.305 INFO:teuthology.orchestra.run.smithi039.stdout:create mon.a on 2024-09-14T03:46:12.509 INFO:teuthology.orchestra.run.smithi039.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-09-14T03:46:12.652 INFO:teuthology.orchestra.run.smithi039.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-09-14T03:46:12.831 INFO:teuthology.orchestra.run.smithi039.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-bd7c803c-724b-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-bd7c803c-724b-11ef-bceb-c7b262605968.target. 2024-09-14T03:46:12.831 INFO:teuthology.orchestra.run.smithi039.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-bd7c803c-724b-11ef-bceb-c7b262605968.target → /etc/systemd/system/ceph-bd7c803c-724b-11ef-bceb-c7b262605968.target. 2024-09-14T03:46:13.066 INFO:teuthology.orchestra.run.smithi039.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mon.a 2024-09-14T03:46:13.067 INFO:teuthology.orchestra.run.smithi039.stdout:systemctl: stderr Failed to reset failed state of unit ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mon.a.service: Unit ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mon.a.service not loaded. 2024-09-14T03:46:13.233 INFO:teuthology.orchestra.run.smithi039.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-bd7c803c-724b-11ef-bceb-c7b262605968.target.wants/ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mon.a.service → /etc/systemd/system/ceph-bd7c803c-724b-11ef-bceb-c7b262605968@.service. 2024-09-14T03:46:13.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:13 smithi039 podman[29076]: 2024-09-14 03:46:13.384611068 +0000 UTC m=+0.015994027 image pull 67ddd33fa72f26a51c969a9043e7a3ddf784506b3d1949ad815ae4dd791beec1 quay.ceph.io/ceph-ci/ceph:reef 2024-09-14T03:46:13.770 INFO:teuthology.orchestra.run.smithi039.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-14T03:46:13.771 INFO:teuthology.orchestra.run.smithi039.stdout:systemctl: stdout disabled 2024-09-14T03:46:13.778 INFO:teuthology.orchestra.run.smithi039.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-14T03:46:13.779 INFO:teuthology.orchestra.run.smithi039.stdout:systemctl: stdout inactive 2024-09-14T03:46:13.779 INFO:teuthology.orchestra.run.smithi039.stdout:firewalld.service is not enabled 2024-09-14T03:46:13.779 INFO:teuthology.orchestra.run.smithi039.stdout:Not possible to enable service . firewalld.service is not available 2024-09-14T03:46:13.779 INFO:teuthology.orchestra.run.smithi039.stdout:Waiting for mon to start... 2024-09-14T03:46:13.779 INFO:teuthology.orchestra.run.smithi039.stdout:Waiting for mon... 2024-09-14T03:46:13.983 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:13 smithi039 podman[29076]: 2024-09-14 03:46:13.68117313 +0000 UTC m=+0.312556081 container init edfcf4554992eda331e28d620583d6095c86999c790de9be16d16a2ee700335c (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-a, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , ceph=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, RELEASE=reef-e4e227b) 2024-09-14T03:46:13.983 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:13 smithi039 podman[29076]: 2024-09-14 03:46:13.685733636 +0000 UTC m=+0.317116585 container start edfcf4554992eda331e28d620583d6095c86999c790de9be16d16a2ee700335c (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-a, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, RELEASE=reef-e4e227b, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-14T03:46:13.983 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:13 smithi039 ceph-mon[29113]: mkfs bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T03:46:13.983 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:13 smithi039 ceph-mon[29113]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-14T03:46:13.983 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:13 smithi039 bash[29076]: edfcf4554992eda331e28d620583d6095c86999c790de9be16d16a2ee700335c 2024-09-14T03:46:13.984 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:13 smithi039 systemd[1]: Started Ceph mon.a for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:46:14.774 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout cluster: 2024-09-14T03:46:14.776 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout id: bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T03:46:14.776 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-09-14T03:46:14.776 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout 2024-09-14T03:46:14.776 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout services: 2024-09-14T03:46:14.776 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.525587s) 2024-09-14T03:46:14.776 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-09-14T03:46:14.777 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-09-14T03:46:14.777 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout 2024-09-14T03:46:14.777 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout data: 2024-09-14T03:46:14.777 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-09-14T03:46:14.777 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-09-14T03:46:14.777 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-09-14T03:46:14.777 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout pgs: 2024-09-14T03:46:14.777 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout 2024-09-14T03:46:14.777 INFO:teuthology.orchestra.run.smithi039.stdout:mon is available 2024-09-14T03:46:14.777 INFO:teuthology.orchestra.run.smithi039.stdout:Assimilating anything we can from ceph.conf... 2024-09-14T03:46:14.897 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:14 smithi039 ceph-mon[29113]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-14T03:46:14.897 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:14 smithi039 ceph-mon[29113]: monmap e1: 1 mons at {a=[v2:172.21.15.39:3300/0,v1:172.21.15.39:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:46:14.897 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:14 smithi039 ceph-mon[29113]: fsmap 2024-09-14T03:46:14.898 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:14 smithi039 ceph-mon[29113]: osdmap e1: 0 total, 0 up, 0 in 2024-09-14T03:46:14.898 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:14 smithi039 ceph-mon[29113]: mgrmap e1: no daemons active 2024-09-14T03:46:14.898 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:14 smithi039 ceph-mon[29113]: from='client.? 172.21.15.39:0/3132188153' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-09-14T03:46:15.848 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout 2024-09-14T03:46:15.849 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout [global] 2024-09-14T03:46:15.849 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout fsid = bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T03:46:15.849 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.39:3300,v1:172.21.15.39:6789] 2024-09-14T03:46:15.849 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-14T03:46:15.849 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-14T03:46:15.849 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-14T03:46:15.849 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-14T03:46:15.849 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout 2024-09-14T03:46:15.849 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-14T03:46:15.849 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-14T03:46:15.849 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout 2024-09-14T03:46:15.849 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout [osd] 2024-09-14T03:46:15.850 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-14T03:46:15.850 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-14T03:46:15.850 INFO:teuthology.orchestra.run.smithi039.stdout:Generating new minimal ceph.conf... 2024-09-14T03:46:15.970 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:15 smithi039 ceph-mon[29113]: from='client.? 172.21.15.39:0/3995139899' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-14T03:46:15.970 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:15 smithi039 ceph-mon[29113]: from='client.? 172.21.15.39:0/3995139899' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-09-14T03:46:16.988 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:16 smithi039 ceph-mon[29113]: from='client.? 172.21.15.39:0/1798862073' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:46:16.989 INFO:teuthology.orchestra.run.smithi039.stdout:Restarting the monitor... 2024-09-14T03:46:17.266 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:17 smithi039 systemd[1]: Stopping Ceph mon.a for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:46:17.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:17 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-a[29090]: 2024-09-14T03:46:17.263+0000 7f9ccb48e640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:46:17.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:17 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-a[29090]: 2024-09-14T03:46:17.263+0000 7f9ccb48e640 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-09-14T03:46:17.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:17 smithi039 podman[29431]: 2024-09-14 03:46:17.418592364 +0000 UTC m=+0.238833595 container died edfcf4554992eda331e28d620583d6095c86999c790de9be16d16a2ee700335c (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-a, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, RELEASE=reef-e4e227b, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-14T03:46:17.922 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:17 smithi039 podman[29431]: 2024-09-14 03:46:17.736895248 +0000 UTC m=+0.557136464 container cleanup edfcf4554992eda331e28d620583d6095c86999c790de9be16d16a2ee700335c (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, RELEASE=reef-e4e227b) 2024-09-14T03:46:17.922 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:17 smithi039 bash[29431]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-a 2024-09-14T03:46:18.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:17 smithi039 podman[29448]: 2024-09-14 03:46:17.921325731 +0000 UTC m=+0.496746541 container remove edfcf4554992eda331e28d620583d6095c86999c790de9be16d16a2ee700335c (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-a, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-e4e227b, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731) 2024-09-14T03:46:18.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-14T03:46:18.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 systemd[1]: Stopped Ceph mon.a for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:46:18.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 systemd[1]: Starting Ceph mon.a for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:46:18.660 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 podman[29548]: 2024-09-14 03:46:18.363770453 +0000 UTC m=+0.098034648 container create e7b5f91fb09c59d21180020e9b0c4afbfc8e6c4fecab1b88885aabba4f957667 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-a, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-e4e227b, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True) 2024-09-14T03:46:18.661 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 podman[29548]: 2024-09-14 03:46:18.281804075 +0000 UTC m=+0.016068273 image pull 67ddd33fa72f26a51c969a9043e7a3ddf784506b3d1949ad815ae4dd791beec1 quay.ceph.io/ceph-ci/ceph:reef 2024-09-14T03:46:18.661 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 podman[29548]: 2024-09-14 03:46:18.566216142 +0000 UTC m=+0.300480332 container init e7b5f91fb09c59d21180020e9b0c4afbfc8e6c4fecab1b88885aabba4f957667 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-a, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=reef-e4e227b, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:46:18.661 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 podman[29548]: 2024-09-14 03:46:18.571221352 +0000 UTC m=+0.305485542 container start e7b5f91fb09c59d21180020e9b0c4afbfc8e6c4fecab1b88885aabba4f957667 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-a, RELEASE=reef-e4e227b, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-14T03:46:18.661 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: set uid:gid to 167:167 (ceph:ceph) 2024-09-14T03:46:18.661 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable), process ceph-mon, pid 7 2024-09-14T03:46:18.661 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: pidfile_write: ignore empty --pid-file 2024-09-14T03:46:18.661 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: load: jerasure load: lrc 2024-09-14T03:46:18.661 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: RocksDB version: 7.9.2 2024-09-14T03:46:18.661 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Git sha 0 2024-09-14T03:46:18.661 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Compile date 2024-09-13 16:45:57 2024-09-14T03:46:18.661 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: DB SUMMARY 2024-09-14T03:46:18.662 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: DB Session ID: DN2QO68ZWKQYMFEPGXBF 2024-09-14T03:46:18.662 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: CURRENT file: CURRENT 2024-09-14T03:46:18.662 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: IDENTITY file: IDENTITY 2024-09-14T03:46:18.662 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: MANIFEST file: MANIFEST-000010 size: 179 Bytes 2024-09-14T03:46:18.662 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-09-14T03:46:18.662 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000009.log size: 84546 ; 2024-09-14T03:46:18.662 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.error_if_exists: 0 2024-09-14T03:46:18.662 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.create_if_missing: 0 2024-09-14T03:46:18.662 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.paranoid_checks: 1 2024-09-14T03:46:18.662 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-14T03:46:18.662 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-14T03:46:18.662 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-14T03:46:18.663 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.env: 0x55ae704b6bc0 2024-09-14T03:46:18.663 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.fs: PosixFileSystem 2024-09-14T03:46:18.663 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.info_log: 0x55ae71230be0 2024-09-14T03:46:18.663 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.max_file_opening_threads: 16 2024-09-14T03:46:18.663 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.statistics: (nil) 2024-09-14T03:46:18.663 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.use_fsync: 0 2024-09-14T03:46:18.663 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.max_log_file_size: 0 2024-09-14T03:46:18.663 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-14T03:46:18.664 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-14T03:46:18.664 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.keep_log_file_num: 1000 2024-09-14T03:46:18.664 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.recycle_log_file_num: 0 2024-09-14T03:46:18.664 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.allow_fallocate: 1 2024-09-14T03:46:18.664 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.allow_mmap_reads: 0 2024-09-14T03:46:18.664 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.allow_mmap_writes: 0 2024-09-14T03:46:18.664 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.use_direct_reads: 0 2024-09-14T03:46:18.664 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-14T03:46:18.664 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.create_missing_column_families: 0 2024-09-14T03:46:18.664 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.db_log_dir: 2024-09-14T03:46:18.664 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.wal_dir: 2024-09-14T03:46:18.665 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-14T03:46:18.665 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-14T03:46:18.665 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-14T03:46:18.665 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-14T03:46:18.665 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-14T03:46:18.665 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-14T03:46:18.665 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.advise_random_on_open: 1 2024-09-14T03:46:18.665 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.db_write_buffer_size: 0 2024-09-14T03:46:18.665 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.write_buffer_manager: 0x55ae7123e320 2024-09-14T03:46:18.666 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-14T03:46:18.666 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-14T03:46:18.666 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-14T03:46:18.666 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.rate_limiter: (nil) 2024-09-14T03:46:18.666 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-14T03:46:18.666 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.wal_recovery_mode: 2 2024-09-14T03:46:18.666 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.enable_thread_tracking: 0 2024-09-14T03:46:18.666 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.enable_pipelined_write: 0 2024-09-14T03:46:18.666 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.unordered_write: 0 2024-09-14T03:46:18.666 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-14T03:46:18.667 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-14T03:46:18.667 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-14T03:46:18.667 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-14T03:46:18.667 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.row_cache: None 2024-09-14T03:46:18.667 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.wal_filter: None 2024-09-14T03:46:18.667 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-14T03:46:18.667 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.allow_ingest_behind: 0 2024-09-14T03:46:18.667 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.two_write_queues: 0 2024-09-14T03:46:18.667 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.manual_wal_flush: 0 2024-09-14T03:46:18.667 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.wal_compression: 0 2024-09-14T03:46:18.667 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.atomic_flush: 0 2024-09-14T03:46:18.667 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-14T03:46:18.667 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-14T03:46:18.668 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-14T03:46:18.668 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.log_readahead_size: 0 2024-09-14T03:46:18.668 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-14T03:46:18.668 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.best_efforts_recovery: 0 2024-09-14T03:46:18.668 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-14T03:46:18.668 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-14T03:46:18.668 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.allow_data_in_errors: 0 2024-09-14T03:46:18.668 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.db_host_id: __hostname__ 2024-09-14T03:46:18.668 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-14T03:46:18.668 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.max_background_jobs: 2 2024-09-14T03:46:18.669 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.max_background_compactions: -1 2024-09-14T03:46:18.669 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.max_subcompactions: 1 2024-09-14T03:46:18.669 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-14T03:46:18.669 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-14T03:46:18.669 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-14T03:46:18.669 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.max_total_wal_size: 0 2024-09-14T03:46:18.669 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-14T03:46:18.669 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-14T03:46:18.669 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-14T03:46:18.670 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-14T03:46:18.670 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.max_open_files: -1 2024-09-14T03:46:18.670 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.bytes_per_sync: 0 2024-09-14T03:46:18.670 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-14T03:46:18.670 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-14T03:46:18.670 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.compaction_readahead_size: 0 2024-09-14T03:46:18.670 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.max_background_flushes: -1 2024-09-14T03:46:18.670 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Compression algorithms supported: 2024-09-14T03:46:18.670 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: kZSTD supported: 0 2024-09-14T03:46:18.670 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: kXpressCompression supported: 0 2024-09-14T03:46:18.670 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: kBZip2Compression supported: 0 2024-09-14T03:46:18.670 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-14T03:46:18.670 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: kLZ4Compression supported: 1 2024-09-14T03:46:18.671 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: kZlibCompression supported: 1 2024-09-14T03:46:18.671 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: kLZ4HCCompression supported: 1 2024-09-14T03:46:18.671 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: kSnappyCompression supported: 1 2024-09-14T03:46:18.671 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-14T03:46:18.671 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-14T03:46:18.671 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 2024-09-14T03:46:18.671 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-14T03:46:18.671 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-14T03:46:18.671 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.merge_operator: 2024-09-14T03:46:18.671 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.compaction_filter: None 2024-09-14T03:46:18.671 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.compaction_filter_factory: None 2024-09-14T03:46:18.671 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.sst_partitioner_factory: None 2024-09-14T03:46:18.672 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-14T03:46:18.672 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-14T03:46:18.672 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ae712307e0) 2024-09-14T03:46:18.672 INFO:journalctl@ceph.mon.a.smithi039.stdout: cache_index_and_filter_blocks: 1 2024-09-14T03:46:18.672 INFO:journalctl@ceph.mon.a.smithi039.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-14T03:46:18.672 INFO:journalctl@ceph.mon.a.smithi039.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-14T03:46:18.672 INFO:journalctl@ceph.mon.a.smithi039.stdout: pin_top_level_index_and_filter: 1 2024-09-14T03:46:18.672 INFO:journalctl@ceph.mon.a.smithi039.stdout: index_type: 0 2024-09-14T03:46:18.672 INFO:journalctl@ceph.mon.a.smithi039.stdout: data_block_index_type: 0 2024-09-14T03:46:18.672 INFO:journalctl@ceph.mon.a.smithi039.stdout: index_shortening: 1 2024-09-14T03:46:18.672 INFO:journalctl@ceph.mon.a.smithi039.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-14T03:46:18.672 INFO:journalctl@ceph.mon.a.smithi039.stdout: checksum: 4 2024-09-14T03:46:18.673 INFO:journalctl@ceph.mon.a.smithi039.stdout: no_block_cache: 0 2024-09-14T03:46:18.673 INFO:journalctl@ceph.mon.a.smithi039.stdout: block_cache: 0x55ae71225350 2024-09-14T03:46:18.673 INFO:journalctl@ceph.mon.a.smithi039.stdout: block_cache_name: BinnedLRUCache 2024-09-14T03:46:18.673 INFO:journalctl@ceph.mon.a.smithi039.stdout: block_cache_options: 2024-09-14T03:46:18.673 INFO:journalctl@ceph.mon.a.smithi039.stdout: capacity : 536870912 2024-09-14T03:46:18.673 INFO:journalctl@ceph.mon.a.smithi039.stdout: num_shard_bits : 4 2024-09-14T03:46:18.673 INFO:journalctl@ceph.mon.a.smithi039.stdout: strict_capacity_limit : 0 2024-09-14T03:46:18.673 INFO:journalctl@ceph.mon.a.smithi039.stdout: high_pri_pool_ratio: 0.000 2024-09-14T03:46:18.673 INFO:journalctl@ceph.mon.a.smithi039.stdout: block_cache_compressed: (nil) 2024-09-14T03:46:18.673 INFO:journalctl@ceph.mon.a.smithi039.stdout: persistent_cache: (nil) 2024-09-14T03:46:18.673 INFO:journalctl@ceph.mon.a.smithi039.stdout: block_size: 4096 2024-09-14T03:46:18.673 INFO:journalctl@ceph.mon.a.smithi039.stdout: block_size_deviation: 10 2024-09-14T03:46:18.674 INFO:journalctl@ceph.mon.a.smithi039.stdout: block_restart_interval: 16 2024-09-14T03:46:18.674 INFO:journalctl@ceph.mon.a.smithi039.stdout: index_block_restart_interval: 1 2024-09-14T03:46:18.674 INFO:journalctl@ceph.mon.a.smithi039.stdout: metadata_block_size: 4096 2024-09-14T03:46:18.674 INFO:journalctl@ceph.mon.a.smithi039.stdout: partition_filters: 0 2024-09-14T03:46:18.674 INFO:journalctl@ceph.mon.a.smithi039.stdout: use_delta_encoding: 1 2024-09-14T03:46:18.675 INFO:journalctl@ceph.mon.a.smithi039.stdout: filter_policy: bloomfilter 2024-09-14T03:46:18.675 INFO:journalctl@ceph.mon.a.smithi039.stdout: whole_key_filtering: 1 2024-09-14T03:46:18.675 INFO:journalctl@ceph.mon.a.smithi039.stdout: verify_compression: 0 2024-09-14T03:46:18.675 INFO:journalctl@ceph.mon.a.smithi039.stdout: read_amp_bytes_per_bit: 0 2024-09-14T03:46:18.675 INFO:journalctl@ceph.mon.a.smithi039.stdout: format_version: 5 2024-09-14T03:46:18.675 INFO:journalctl@ceph.mon.a.smithi039.stdout: enable_index_compression: 1 2024-09-14T03:46:18.675 INFO:journalctl@ceph.mon.a.smithi039.stdout: block_align: 0 2024-09-14T03:46:18.675 INFO:journalctl@ceph.mon.a.smithi039.stdout: max_auto_readahead_size: 262144 2024-09-14T03:46:18.676 INFO:journalctl@ceph.mon.a.smithi039.stdout: prepopulate_block_cache: 0 2024-09-14T03:46:18.676 INFO:journalctl@ceph.mon.a.smithi039.stdout: initial_auto_readahead_size: 8192 2024-09-14T03:46:18.676 INFO:journalctl@ceph.mon.a.smithi039.stdout: num_file_reads_for_auto_readahead: 2 2024-09-14T03:46:18.676 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.write_buffer_size: 33554432 2024-09-14T03:46:18.676 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.max_write_buffer_number: 2 2024-09-14T03:46:18.676 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.compression: NoCompression 2024-09-14T03:46:18.676 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.bottommost_compression: Disabled 2024-09-14T03:46:18.676 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.prefix_extractor: nullptr 2024-09-14T03:46:18.676 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-14T03:46:18.676 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.num_levels: 7 2024-09-14T03:46:18.676 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-14T03:46:18.676 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-14T03:46:18.677 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-14T03:46:18.677 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-14T03:46:18.677 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-14T03:46:18.677 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-14T03:46:18.677 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-14T03:46:18.677 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:46:18.677 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-14T03:46:18.677 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-14T03:46:18.677 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:46:18.677 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:46:18.677 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-14T03:46:18.677 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.compression_opts.level: 32767 2024-09-14T03:46:18.677 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.compression_opts.strategy: 0 2024-09-14T03:46:18.678 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-14T03:46:18.678 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:46:18.678 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:46:18.678 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-14T03:46:18.678 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.compression_opts.enabled: false 2024-09-14T03:46:18.678 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:46:18.678 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-14T03:46:18.678 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-14T03:46:18.678 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-14T03:46:18.678 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.target_file_size_base: 67108864 2024-09-14T03:46:18.678 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-14T03:46:18.678 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-14T03:46:18.678 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-14T03:46:18.679 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-14T03:46:18.679 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-14T03:46:18.679 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-14T03:46:18.679 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-14T03:46:18.679 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-14T03:46:18.679 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-14T03:46:18.679 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-14T03:46:18.679 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-14T03:46:18.679 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-14T03:46:18.679 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-14T03:46:18.679 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-14T03:46:18.679 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.arena_block_size: 1048576 2024-09-14T03:46:18.680 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-14T03:46:18.680 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-14T03:46:18.680 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.disable_auto_compactions: 0 2024-09-14T03:46:18.680 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-14T03:46:18.680 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-14T03:46:18.680 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-14T03:46:18.680 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-14T03:46:18.680 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-14T03:46:18.680 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-14T03:46:18.680 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-14T03:46:18.680 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-14T03:46:18.681 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-14T03:46:18.681 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-14T03:46:18.681 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-14T03:46:18.681 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.inplace_update_support: 0 2024-09-14T03:46:18.681 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-14T03:46:18.681 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-14T03:46:18.681 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-14T03:46:18.681 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-14T03:46:18.681 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.bloom_locality: 0 2024-09-14T03:46:18.681 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.max_successive_merges: 0 2024-09-14T03:46:18.681 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-14T03:46:18.681 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.paranoid_file_checks: 0 2024-09-14T03:46:18.681 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.force_consistency_checks: 1 2024-09-14T03:46:18.682 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.report_bg_io_stats: 0 2024-09-14T03:46:18.682 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.ttl: 2592000 2024-09-14T03:46:18.682 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-14T03:46:18.682 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-14T03:46:18.682 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-14T03:46:18.682 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.enable_blob_files: false 2024-09-14T03:46:18.682 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.min_blob_size: 0 2024-09-14T03:46:18.682 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.blob_file_size: 268435456 2024-09-14T03:46:18.682 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-14T03:46:18.682 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-14T03:46:18.682 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-14T03:46:18.682 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-14T03:46:18.682 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-14T03:46:18.683 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.blob_file_starting_level: 0 2024-09-14T03:46:18.683 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-14T03:46:18.683 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 12, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 5 2024-09-14T03:46:18.683 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 5 2024-09-14T03:46:18.683 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 3257acd3-3704-476a-8f05-05d73ed70edd 2024-09-14T03:46:18.683 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285578597513, "job": 1, "event": "recovery_started", "wal_files": [9]} 2024-09-14T03:46:18.683 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #9 mode 2 2024-09-14T03:46:18.683 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285578598587, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 13, "file_size": 81096, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 8, "largest_seqno": 256, "table_properties": {"data_size": 79246, "index_size": 234, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10267, "raw_average_key_size": 47, "raw_value_size": 73447, "raw_average_value_size": 341, "num_data_blocks": 10, "num_entries": 215, "num_filter_entries": 215, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726285578, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "3257acd3-3704-476a-8f05-05d73ed70edd", "db_session_id": "DN2QO68ZWKQYMFEPGXBF", "orig_file_number": 13, "seqno_to_time_mapping": "N/A"}} 2024-09-14T03:46:18.683 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285578598686, "job": 1, "event": "recovery_finished"} 2024-09-14T03:46:18.683 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: [db/version_set.cc:5047] Creating manifest 15 2024-09-14T03:46:18.683 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000009.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-14T03:46:18.683 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55ae71250e00 2024-09-14T03:46:18.684 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: DB pointer 0x55ae71344000 2024-09-14T03:46:18.684 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-14T03:46:18.684 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-14T03:46:18.684 INFO:journalctl@ceph.mon.a.smithi039.stdout: ** DB Stats ** 2024-09-14T03:46:18.684 INFO:journalctl@ceph.mon.a.smithi039.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:46:18.684 INFO:journalctl@ceph.mon.a.smithi039.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-14T03:46:18.684 INFO:journalctl@ceph.mon.a.smithi039.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:46:18.684 INFO:journalctl@ceph.mon.a.smithi039.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:46:18.684 INFO:journalctl@ceph.mon.a.smithi039.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-14T03:46:18.684 INFO:journalctl@ceph.mon.a.smithi039.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:46:18.684 INFO:journalctl@ceph.mon.a.smithi039.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:46:18.684 INFO:journalctl@ceph.mon.a.smithi039.stdout: 2024-09-14T03:46:18.684 INFO:journalctl@ceph.mon.a.smithi039.stdout: ** Compaction Stats [default] ** 2024-09-14T03:46:18.685 INFO:journalctl@ceph.mon.a.smithi039.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:46:18.685 INFO:journalctl@ceph.mon.a.smithi039.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-14T03:46:18.685 INFO:journalctl@ceph.mon.a.smithi039.stdout: L0 2/0 81.06 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 104.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-14T03:46:18.685 INFO:journalctl@ceph.mon.a.smithi039.stdout: Sum 2/0 81.06 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 104.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-14T03:46:18.685 INFO:journalctl@ceph.mon.a.smithi039.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 104.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-14T03:46:18.685 INFO:journalctl@ceph.mon.a.smithi039.stdout: 2024-09-14T03:46:18.685 INFO:journalctl@ceph.mon.a.smithi039.stdout: ** Compaction Stats [default] ** 2024-09-14T03:46:18.685 INFO:journalctl@ceph.mon.a.smithi039.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:46:18.685 INFO:journalctl@ceph.mon.a.smithi039.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:46:18.685 INFO:journalctl@ceph.mon.a.smithi039.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 104.1 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-09-14T03:46:18.685 INFO:journalctl@ceph.mon.a.smithi039.stdout: 2024-09-14T03:46:18.686 INFO:teuthology.orchestra.run.smithi039.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-09-14T03:46:18.686 INFO:journalctl@ceph.mon.a.smithi039.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-14T03:46:18.686 INFO:journalctl@ceph.mon.a.smithi039.stdout: 2024-09-14T03:46:18.686 INFO:journalctl@ceph.mon.a.smithi039.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:46:18.686 INFO:journalctl@ceph.mon.a.smithi039.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-14T03:46:18.686 INFO:journalctl@ceph.mon.a.smithi039.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-14T03:46:18.686 INFO:journalctl@ceph.mon.a.smithi039.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-14T03:46:18.686 INFO:journalctl@ceph.mon.a.smithi039.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-14T03:46:18.687 INFO:journalctl@ceph.mon.a.smithi039.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-14T03:46:18.687 INFO:journalctl@ceph.mon.a.smithi039.stdout: Cumulative compaction: 0.00 GB write, 20.24 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:46:18.687 INFO:journalctl@ceph.mon.a.smithi039.stdout: Interval compaction: 0.00 GB write, 20.24 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:46:18.687 INFO:journalctl@ceph.mon.a.smithi039.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-14T03:46:18.687 INFO:journalctl@ceph.mon.a.smithi039.stdout: Block cache BinnedLRUCache@0x55ae71225350#7 capacity: 512.00 MB usage: 1.19 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2024-09-14T03:46:18.687 INFO:journalctl@ceph.mon.a.smithi039.stdout: Block cache entry stats(count,size,portion): FilterBlock(2,0.77 KB,0.000146031%) IndexBlock(2,0.42 KB,8.04663e-05%) Misc(1,0.00 KB,0%) 2024-09-14T03:46:18.687 INFO:journalctl@ceph.mon.a.smithi039.stdout: 2024-09-14T03:46:18.687 INFO:journalctl@ceph.mon.a.smithi039.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-14T03:46:18.687 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: starting mon.a rank 0 at public addrs [v2:172.21.15.39:3300/0,v1:172.21.15.39:6789/0] at bind addrs [v2:172.21.15.39:3300/0,v1:172.21.15.39:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T03:46:18.687 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: mon.a@-1(???) e1 preinit fsid bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T03:46:18.687 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: mon.a@-1(???).mds e1 new map 2024-09-14T03:46:18.687 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: mon.a@-1(???).mds e1 print_map 2024-09-14T03:46:18.687 INFO:journalctl@ceph.mon.a.smithi039.stdout: e1 2024-09-14T03:46:18.688 INFO:journalctl@ceph.mon.a.smithi039.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-14T03:46:18.688 INFO:journalctl@ceph.mon.a.smithi039.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:46:18.688 INFO:journalctl@ceph.mon.a.smithi039.stdout: legacy client fscid: -1 2024-09-14T03:46:18.688 INFO:journalctl@ceph.mon.a.smithi039.stdout: 2024-09-14T03:46:18.688 INFO:journalctl@ceph.mon.a.smithi039.stdout: No filesystems configured 2024-09-14T03:46:18.688 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-14T03:46:18.688 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-14T03:46:18.688 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-14T03:46:18.688 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-09-14T03:46:18.688 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-09-14T03:46:18.688 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: mon.a is new leader, mons a in quorum (ranks 0) 2024-09-14T03:46:18.688 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: monmap e1: 1 mons at {a=[v2:172.21.15.39:3300/0,v1:172.21.15.39:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:46:18.688 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: fsmap 2024-09-14T03:46:18.689 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: osdmap e1: 0 total, 0 up, 0 in 2024-09-14T03:46:18.689 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 ceph-mon[29586]: mgrmap e1: no daemons active 2024-09-14T03:46:18.689 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 bash[29548]: e7b5f91fb09c59d21180020e9b0c4afbfc8e6c4fecab1b88885aabba4f957667 2024-09-14T03:46:18.689 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:18 smithi039 systemd[1]: Started Ceph mon.a for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:46:19.651 INFO:teuthology.orchestra.run.smithi039.stdout:Wrote config to /etc/ceph/ceph.conf 2024-09-14T03:46:19.665 INFO:teuthology.orchestra.run.smithi039.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-09-14T03:46:19.665 INFO:teuthology.orchestra.run.smithi039.stdout:Creating mgr... 2024-09-14T03:46:19.665 INFO:teuthology.orchestra.run.smithi039.stdout:Verifying port 0.0.0.0:9283 ... 2024-09-14T03:46:19.666 INFO:teuthology.orchestra.run.smithi039.stdout:Verifying port 0.0.0.0:8765 ... 2024-09-14T03:46:19.903 INFO:teuthology.orchestra.run.smithi039.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mgr.y 2024-09-14T03:46:19.903 INFO:teuthology.orchestra.run.smithi039.stdout:systemctl: stderr Failed to reset failed state of unit ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mgr.y.service: Unit ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mgr.y.service not loaded. 2024-09-14T03:46:20.060 INFO:teuthology.orchestra.run.smithi039.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-bd7c803c-724b-11ef-bceb-c7b262605968.target.wants/ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mgr.y.service → /etc/systemd/system/ceph-bd7c803c-724b-11ef-bceb-c7b262605968@.service. 2024-09-14T03:46:20.518 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:20 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/2736260478' entity='client.admin' 2024-09-14T03:46:20.518 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:20 smithi039 podman[29820]: 2024-09-14 03:46:20.19624788 +0000 UTC m=+0.016011169 image pull 67ddd33fa72f26a51c969a9043e7a3ddf784506b3d1949ad815ae4dd791beec1 quay.ceph.io/ceph-ci/ceph:reef 2024-09-14T03:46:20.518 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:20 smithi039 podman[29820]: 2024-09-14 03:46:20.516968293 +0000 UTC m=+0.336731581 container create babf5c5c054b5d02a74e529601cb488a4f655b9750823175fbf02403fd53b812 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, ceph=True, RELEASE=reef-e4e227b, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-09-14T03:46:20.786 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:20 smithi039 podman[29820]: 2024-09-14 03:46:20.719895605 +0000 UTC m=+0.539658892 container init babf5c5c054b5d02a74e529601cb488a4f655b9750823175fbf02403fd53b812 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=reef-e4e227b, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-14T03:46:20.786 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:20 smithi039 podman[29820]: 2024-09-14 03:46:20.725360877 +0000 UTC m=+0.545124164 container start babf5c5c054b5d02a74e529601cb488a4f655b9750823175fbf02403fd53b812 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=reef-e4e227b, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:46:20.805 INFO:teuthology.orchestra.run.smithi039.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-14T03:46:20.805 INFO:teuthology.orchestra.run.smithi039.stdout:systemctl: stdout disabled 2024-09-14T03:46:20.813 INFO:teuthology.orchestra.run.smithi039.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-14T03:46:20.813 INFO:teuthology.orchestra.run.smithi039.stdout:systemctl: stdout inactive 2024-09-14T03:46:20.813 INFO:teuthology.orchestra.run.smithi039.stdout:firewalld.service is not enabled 2024-09-14T03:46:20.813 INFO:teuthology.orchestra.run.smithi039.stdout:Not possible to enable service . firewalld.service is not available 2024-09-14T03:46:20.822 INFO:teuthology.orchestra.run.smithi039.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-14T03:46:20.822 INFO:teuthology.orchestra.run.smithi039.stdout:systemctl: stdout disabled 2024-09-14T03:46:20.832 INFO:teuthology.orchestra.run.smithi039.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-14T03:46:20.832 INFO:teuthology.orchestra.run.smithi039.stdout:systemctl: stdout inactive 2024-09-14T03:46:20.832 INFO:teuthology.orchestra.run.smithi039.stdout:firewalld.service is not enabled 2024-09-14T03:46:20.832 INFO:teuthology.orchestra.run.smithi039.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2024-09-14T03:46:20.832 INFO:teuthology.orchestra.run.smithi039.stdout:Waiting for mgr to start... 2024-09-14T03:46:20.832 INFO:teuthology.orchestra.run.smithi039.stdout:Waiting for mgr... 2024-09-14T03:46:21.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:20 smithi039 bash[29820]: babf5c5c054b5d02a74e529601cb488a4f655b9750823175fbf02403fd53b812 2024-09-14T03:46:21.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:20 smithi039 systemd[1]: Started Ceph mgr.y for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:46:21.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:20 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:20.859+0000 7f906058d040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-14T03:46:21.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:20 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:20.919+0000 7f906058d040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-14T03:46:21.315 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:21 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:21.081+0000 7f906058d040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-14T03:46:21.315 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:21 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:21.156+0000 7f906058d040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-14T03:46:21.315 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:21 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:21.313+0000 7f906058d040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:46:21.692 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:21 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/3905191687' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-14T03:46:21.693 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:21 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:21.465+0000 7f906058d040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-14T03:46:22.002 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:21 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:21.796+0000 7f906058d040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-14T03:46:22.002 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:21 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:21.861+0000 7f906058d040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-14T03:46:22.007 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout 2024-09-14T03:46:22.007 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout { 2024-09-14T03:46:22.007 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "fsid": "bd7c803c-724b-11ef-bceb-c7b262605968", 2024-09-14T03:46:22.007 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "health": { 2024-09-14T03:46:22.007 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-14T03:46:22.007 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-14T03:46:22.007 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-14T03:46:22.007 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:46:22.007 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-14T03:46:22.007 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-14T03:46:22.008 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout 0 2024-09-14T03:46:22.008 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:46:22.008 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-14T03:46:22.008 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "a" 2024-09-14T03:46:22.008 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:46:22.008 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2024-09-14T03:46:22.008 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-14T03:46:22.008 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:46:22.008 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-09-14T03:46:22.008 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-14T03:46:22.009 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:46:22.009 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-14T03:46:22.009 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:46:22.009 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-14T03:46:22.009 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-14T03:46:22.009 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-14T03:46:22.012 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-14T03:46:22.012 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-14T03:46:22.012 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-14T03:46:22.012 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:46:22.013 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-14T03:46:22.013 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-14T03:46:22.013 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-14T03:46:22.013 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-14T03:46:22.013 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-14T03:46:22.013 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-14T03:46:22.013 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-14T03:46:22.013 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-14T03:46:22.013 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-14T03:46:22.013 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:46:22.013 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-14T03:46:22.013 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:46:22.013 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-14T03:46:22.013 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-14T03:46:22.014 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:46:22.014 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-14T03:46:22.014 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "available": false, 2024-09-14T03:46:22.014 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-14T03:46:22.014 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-14T03:46:22.014 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-14T03:46:22.014 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-14T03:46:22.014 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "restful" 2024-09-14T03:46:22.014 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:46:22.014 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-14T03:46:22.014 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:46:22.014 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-14T03:46:22.015 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:46:22.015 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "modified": "2024-09-14T03:46:13.726952+0000", 2024-09-14T03:46:22.015 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-14T03:46:22.015 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:46:22.015 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-14T03:46:22.015 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout } 2024-09-14T03:46:22.015 INFO:teuthology.orchestra.run.smithi039.stdout:mgr not available, waiting (1/15)... 2024-09-14T03:46:22.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:22 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:22.053+0000 7f906058d040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-14T03:46:23.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:22 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:22.996+0000 7f906058d040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-14T03:46:23.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:23 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:23.067+0000 7f906058d040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-14T03:46:23.969 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:23 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:23.846+0000 7f906058d040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-14T03:46:23.969 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:23 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:23.967+0000 7f906058d040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-14T03:46:24.244 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:24 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:24.118+0000 7f906058d040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:46:24.244 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:24 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:24.183+0000 7f906058d040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-14T03:46:24.538 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:24 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:24.341+0000 7f906058d040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-14T03:46:24.538 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:24 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:24.414+0000 7f906058d040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-14T03:46:24.538 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:24 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:24.478+0000 7f906058d040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-14T03:46:24.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:24 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/1810398778' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-14T03:46:24.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:24 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:24.646+0000 7f906058d040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-14T03:46:24.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:24 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:24.715+0000 7f906058d040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-14T03:46:24.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:24 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:24.786+0000 7f906058d040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-14T03:46:25.126 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:24 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:24.874+0000 7f906058d040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-14T03:46:25.126 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:24 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:24.936+0000 7f906058d040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-14T03:46:25.126 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:24 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:24.998+0000 7f906058d040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-14T03:46:25.128 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout 2024-09-14T03:46:25.134 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout { 2024-09-14T03:46:25.134 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "fsid": "bd7c803c-724b-11ef-bceb-c7b262605968", 2024-09-14T03:46:25.134 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "health": { 2024-09-14T03:46:25.134 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-14T03:46:25.134 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-14T03:46:25.134 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-14T03:46:25.134 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:46:25.134 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-14T03:46:25.134 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-14T03:46:25.135 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout 0 2024-09-14T03:46:25.135 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:46:25.135 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-14T03:46:25.135 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "a" 2024-09-14T03:46:25.135 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:46:25.135 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "quorum_age": 5, 2024-09-14T03:46:25.135 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-14T03:46:25.135 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:46:25.135 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-09-14T03:46:25.135 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-14T03:46:25.135 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:46:25.135 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-14T03:46:25.135 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:46:25.136 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-14T03:46:25.136 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-14T03:46:25.136 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-14T03:46:25.136 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-14T03:46:25.136 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-14T03:46:25.136 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-14T03:46:25.136 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:46:25.136 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-14T03:46:25.136 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-14T03:46:25.136 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-14T03:46:25.136 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-14T03:46:25.136 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-14T03:46:25.136 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-14T03:46:25.137 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-14T03:46:25.137 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-14T03:46:25.137 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-14T03:46:25.137 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:46:25.137 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-14T03:46:25.137 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:46:25.137 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-14T03:46:25.137 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-14T03:46:25.137 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:46:25.137 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-14T03:46:25.137 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "available": false, 2024-09-14T03:46:25.137 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-14T03:46:25.137 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-14T03:46:25.137 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-14T03:46:25.138 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-14T03:46:25.138 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "restful" 2024-09-14T03:46:25.138 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:46:25.138 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-14T03:46:25.138 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:46:25.138 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-14T03:46:25.138 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:46:25.138 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "modified": "2024-09-14T03:46:13.726952+0000", 2024-09-14T03:46:25.138 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-14T03:46:25.138 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:46:25.138 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-14T03:46:25.138 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout } 2024-09-14T03:46:25.138 INFO:teuthology.orchestra.run.smithi039.stdout:mgr not available, waiting (2/15)... 2024-09-14T03:46:26.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:25 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:25.823+0000 7f906058d040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-14T03:46:26.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:25 smithi039 ceph-mon[29586]: Activating manager daemon y 2024-09-14T03:46:26.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:25 smithi039 ceph-mon[29586]: mgrmap e2: y(active, starting, since 0.00322288s) 2024-09-14T03:46:26.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:25 smithi039 ceph-mon[29586]: from='mgr.14100 172.21.15.39:0/2651557869' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:46:26.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:25 smithi039 ceph-mon[29586]: from='mgr.14100 172.21.15.39:0/2651557869' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:46:26.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:25 smithi039 ceph-mon[29586]: from='mgr.14100 172.21.15.39:0/2651557869' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:46:26.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:25 smithi039 ceph-mon[29586]: from='mgr.14100 172.21.15.39:0/2651557869' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:46:26.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:25 smithi039 ceph-mon[29586]: from='mgr.14100 172.21.15.39:0/2651557869' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:46:26.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:25 smithi039 ceph-mon[29586]: Manager daemon y is now available 2024-09-14T03:46:26.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:25 smithi039 ceph-mon[29586]: from='mgr.14100 172.21.15.39:0/2651557869' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:46:26.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:25 smithi039 ceph-mon[29586]: from='mgr.14100 172.21.15.39:0/2651557869' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:46:26.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:25 smithi039 ceph-mon[29586]: from='mgr.14100 172.21.15.39:0/2651557869' entity='mgr.y' 2024-09-14T03:46:26.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:25 smithi039 ceph-mon[29586]: from='mgr.14100 172.21.15.39:0/2651557869' entity='mgr.y' 2024-09-14T03:46:26.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:25 smithi039 ceph-mon[29586]: from='mgr.14100 172.21.15.39:0/2651557869' entity='mgr.y' 2024-09-14T03:46:28.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:27 smithi039 ceph-mon[29586]: mgrmap e3: y(active, since 1.00981s) 2024-09-14T03:46:28.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:27 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/1909740679' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-09-14T03:46:28.411 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout 2024-09-14T03:46:28.411 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout { 2024-09-14T03:46:28.411 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "fsid": "bd7c803c-724b-11ef-bceb-c7b262605968", 2024-09-14T03:46:28.411 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "health": { 2024-09-14T03:46:28.412 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-09-14T03:46:28.412 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-09-14T03:46:28.412 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-09-14T03:46:28.412 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:46:28.412 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-09-14T03:46:28.412 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-09-14T03:46:28.412 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout 0 2024-09-14T03:46:28.412 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:46:28.412 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-09-14T03:46:28.412 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "a" 2024-09-14T03:46:28.412 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:46:28.412 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "quorum_age": 9, 2024-09-14T03:46:28.412 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "monmap": { 2024-09-14T03:46:28.412 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:46:28.413 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-09-14T03:46:28.413 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-09-14T03:46:28.413 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:46:28.413 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-09-14T03:46:28.413 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:46:28.413 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-09-14T03:46:28.413 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-09-14T03:46:28.413 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-09-14T03:46:28.413 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-09-14T03:46:28.416 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-09-14T03:46:28.416 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-09-14T03:46:28.416 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:46:28.417 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-09-14T03:46:28.417 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-09-14T03:46:28.417 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-09-14T03:46:28.417 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-09-14T03:46:28.417 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-09-14T03:46:28.417 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-09-14T03:46:28.417 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-09-14T03:46:28.417 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-09-14T03:46:28.417 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-09-14T03:46:28.417 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:46:28.417 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-09-14T03:46:28.417 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:46:28.417 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-09-14T03:46:28.418 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-09-14T03:46:28.418 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:46:28.418 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-09-14T03:46:28.418 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-14T03:46:28.418 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-09-14T03:46:28.418 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "modules": [ 2024-09-14T03:46:28.418 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "iostat", 2024-09-14T03:46:28.418 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "nfs", 2024-09-14T03:46:28.418 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "restful" 2024-09-14T03:46:28.418 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout ], 2024-09-14T03:46:28.418 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-14T03:46:28.418 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:46:28.419 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-09-14T03:46:28.419 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-09-14T03:46:28.419 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "modified": "2024-09-14T03:46:13.726952+0000", 2024-09-14T03:46:28.419 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "services": {} 2024-09-14T03:46:28.419 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout }, 2024-09-14T03:46:28.419 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-09-14T03:46:28.419 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout } 2024-09-14T03:46:28.419 INFO:teuthology.orchestra.run.smithi039.stdout:mgr is available 2024-09-14T03:46:29.028 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:28 smithi039 ceph-mon[29586]: mgrmap e4: y(active, since 2s) 2024-09-14T03:46:29.594 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout 2024-09-14T03:46:29.594 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout [global] 2024-09-14T03:46:29.594 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout fsid = bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T03:46:29.594 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-09-14T03:46:29.594 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-09-14T03:46:29.594 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-09-14T03:46:29.594 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-09-14T03:46:29.594 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout 2024-09-14T03:46:29.595 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout [mgr] 2024-09-14T03:46:29.595 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-09-14T03:46:29.595 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout 2024-09-14T03:46:29.595 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout [osd] 2024-09-14T03:46:29.595 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-09-14T03:46:29.595 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-09-14T03:46:29.595 INFO:teuthology.orchestra.run.smithi039.stdout:Enabling cephadm module... 2024-09-14T03:46:29.942 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:29 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/1552159389' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-09-14T03:46:31.202 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:30 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/2325166054' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-09-14T03:46:31.203 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:30 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ignoring --setuser ceph since I am not root 2024-09-14T03:46:31.203 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:30 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ignoring --setgroup ceph since I am not root 2024-09-14T03:46:31.203 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:30 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:30.987+0000 7f3561c3d040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-14T03:46:31.203 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:31.045+0000 7f3561c3d040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-14T03:46:31.203 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:31.200+0000 7f3561c3d040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-14T03:46:31.496 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:31.269+0000 7f3561c3d040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-14T03:46:31.496 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:31.420+0000 7f3561c3d040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:46:31.748 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:31.568+0000 7f3561c3d040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-14T03:46:32.151 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:31 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/2325166054' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-09-14T03:46:32.151 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:31 smithi039 ceph-mon[29586]: mgrmap e5: y(active, since 5s) 2024-09-14T03:46:32.151 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:31.893+0000 7f3561c3d040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-14T03:46:32.152 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:31.958+0000 7f3561c3d040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-14T03:46:32.152 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:32 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:32.149+0000 7f3561c3d040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-14T03:46:32.772 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout { 2024-09-14T03:46:32.772 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-09-14T03:46:32.772 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-14T03:46:32.772 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-14T03:46:32.772 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-14T03:46:32.772 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout } 2024-09-14T03:46:32.773 INFO:teuthology.orchestra.run.smithi039.stdout:Waiting for the mgr to restart... 2024-09-14T03:46:32.773 INFO:teuthology.orchestra.run.smithi039.stdout:Waiting for mgr epoch 5... 2024-09-14T03:46:33.029 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:32 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/3883523421' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-14T03:46:33.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:33 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:33.067+0000 7f3561c3d040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-14T03:46:33.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:33 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:33.135+0000 7f3561c3d040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-14T03:46:34.003 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:33 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:33.882+0000 7f3561c3d040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-14T03:46:34.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:34 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:34.000+0000 7f3561c3d040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-14T03:46:34.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:34 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:34.146+0000 7f3561c3d040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:46:34.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:34 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:34.209+0000 7f3561c3d040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-14T03:46:34.656 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:34 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:34.361+0000 7f3561c3d040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-14T03:46:34.656 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:34 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:34.431+0000 7f3561c3d040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-14T03:46:34.656 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:34 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:34.492+0000 7f3561c3d040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-14T03:46:34.656 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:34 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:34.654+0000 7f3561c3d040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-14T03:46:34.937 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:34 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:34.721+0000 7f3561c3d040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-14T03:46:34.937 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:34 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:34.789+0000 7f3561c3d040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-14T03:46:34.937 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:34 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:34.873+0000 7f3561c3d040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-14T03:46:34.937 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:34 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:34.935+0000 7f3561c3d040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-14T03:46:35.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:34 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:34.995+0000 7f3561c3d040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-14T03:46:36.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:35 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:35.802+0000 7f3561c3d040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-14T03:46:36.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:35 smithi039 ceph-mon[29586]: Active manager daemon y restarted 2024-09-14T03:46:36.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:35 smithi039 ceph-mon[29586]: Activating manager daemon y 2024-09-14T03:46:36.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:35 smithi039 ceph-mon[29586]: osdmap e2: 0 total, 0 up, 0 in 2024-09-14T03:46:36.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:35 smithi039 ceph-mon[29586]: mgrmap e6: y(active, starting, since 0.00815053s) 2024-09-14T03:46:36.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:35 smithi039 ceph-mon[29586]: from='mgr.14118 172.21.15.39:0/827427127' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:46:36.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:35 smithi039 ceph-mon[29586]: from='mgr.14118 172.21.15.39:0/827427127' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:46:36.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:35 smithi039 ceph-mon[29586]: from='mgr.14118 172.21.15.39:0/827427127' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:46:36.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:35 smithi039 ceph-mon[29586]: from='mgr.14118 172.21.15.39:0/827427127' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:46:36.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:35 smithi039 ceph-mon[29586]: from='mgr.14118 172.21.15.39:0/827427127' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:46:36.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:35 smithi039 ceph-mon[29586]: Manager daemon y is now available 2024-09-14T03:46:36.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:35 smithi039 ceph-mon[29586]: from='mgr.14118 172.21.15.39:0/827427127' entity='mgr.y' 2024-09-14T03:46:36.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:35 smithi039 ceph-mon[29586]: from='mgr.14118 172.21.15.39:0/827427127' entity='mgr.y' 2024-09-14T03:46:36.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:35 smithi039 ceph-mon[29586]: from='mgr.14118 172.21.15.39:0/827427127' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:46:36.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:35 smithi039 ceph-mon[29586]: from='mgr.14118 172.21.15.39:0/827427127' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:46:36.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:35 smithi039 ceph-mon[29586]: from='mgr.14118 172.21.15.39:0/827427127' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:46:36.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:35 smithi039 ceph-mon[29586]: from='mgr.14118 172.21.15.39:0/827427127' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:46:37.116 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:36 smithi039 ceph-mon[29586]: Found migration_current of "None". Setting to last migration. 2024-09-14T03:46:37.116 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:36 smithi039 ceph-mon[29586]: mgrmap e7: y(active, since 1.00544s) 2024-09-14T03:46:37.116 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:36 smithi039 ceph-mon[29586]: from='mgr.14118 172.21.15.39:0/827427127' entity='mgr.y' 2024-09-14T03:46:37.116 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:36 smithi039 ceph-mon[29586]: from='mgr.14118 172.21.15.39:0/827427127' entity='mgr.y' 2024-09-14T03:46:37.394 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout { 2024-09-14T03:46:37.394 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-09-14T03:46:37.394 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-14T03:46:37.394 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout } 2024-09-14T03:46:37.394 INFO:teuthology.orchestra.run.smithi039.stdout:mgr epoch 5 is available 2024-09-14T03:46:37.395 INFO:teuthology.orchestra.run.smithi039.stdout:Setting orchestrator backend to cephadm... 2024-09-14T03:46:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:37 smithi039 ceph-mon[29586]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-14T03:46:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:37 smithi039 ceph-mon[29586]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-14T03:46:39.016 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:38 smithi039 ceph-mon[29586]: mgrmap e8: y(active, since 2s) 2024-09-14T03:46:39.017 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:38 smithi039 ceph-mon[29586]: from='mgr.14118 172.21.15.39:0/827427127' entity='mgr.y' 2024-09-14T03:46:39.017 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:38 smithi039 ceph-mon[29586]: from='mgr.14118 172.21.15.39:0/827427127' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:46:39.285 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:39 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:39.103+0000 7f34c991d640 -1 log_channel(cephadm) log [ERR] : [14/Sep/2024:03:46:39] ENGINE Error in HTTPServer.serve 2024-09-14T03:46:39.285 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:39 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: Traceback (most recent call last): 2024-09-14T03:46:39.285 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:39 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:46:39.285 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:39 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: self._connections.run(self.expiration_interval) 2024-09-14T03:46:39.285 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:39 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:46:39.285 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:39 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: self._run(expiration_interval) 2024-09-14T03:46:39.285 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:39 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:46:39.285 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:39 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:46:39.285 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:39 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:46:39.285 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:39 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:46:39.285 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:39 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:46:39.286 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:39 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: s = self.context.wrap_socket( 2024-09-14T03:46:39.286 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:39 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:46:39.286 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:39 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: return self.sslsocket_class._create( 2024-09-14T03:46:39.286 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:39 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:46:39.286 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:39 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: self.do_handshake() 2024-09-14T03:46:39.286 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:39 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:46:39.286 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:39 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: self._sslobj.do_handshake() 2024-09-14T03:46:39.286 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:39 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:46:39.286 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:39 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:39.928 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:39 smithi039 ceph-mon[29586]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:39.929 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:39 smithi039 ceph-mon[29586]: [14/Sep/2024:03:46:38] ENGINE Bus STARTING 2024-09-14T03:46:39.929 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:39 smithi039 ceph-mon[29586]: from='mgr.14118 172.21.15.39:0/827427127' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:46:39.932 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout value unchanged 2024-09-14T03:46:39.932 INFO:teuthology.orchestra.run.smithi039.stdout:Generating ssh key... 2024-09-14T03:46:40.984 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:40 smithi039 ceph-mon[29586]: [14/Sep/2024:03:46:38] ENGINE Serving on http://172.21.15.39:8765 2024-09-14T03:46:40.984 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:40 smithi039 ceph-mon[29586]: [14/Sep/2024:03:46:39] ENGINE Serving on https://172.21.15.39:7150 2024-09-14T03:46:40.984 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:40 smithi039 ceph-mon[29586]: [14/Sep/2024:03:46:39] ENGINE Bus STARTED 2024-09-14T03:46:40.984 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:40 smithi039 ceph-mon[29586]: [14/Sep/2024:03:46:39] ENGINE Error in HTTPServer.serve 2024-09-14T03:46:40.984 INFO:journalctl@ceph.mon.a.smithi039.stdout: Traceback (most recent call last): 2024-09-14T03:46:40.985 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:46:40.985 INFO:journalctl@ceph.mon.a.smithi039.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:46:40.985 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:46:40.985 INFO:journalctl@ceph.mon.a.smithi039.stdout: self._run(expiration_interval) 2024-09-14T03:46:40.985 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:46:40.985 INFO:journalctl@ceph.mon.a.smithi039.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:46:40.985 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:46:40.985 INFO:journalctl@ceph.mon.a.smithi039.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:46:40.985 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:46:40.985 INFO:journalctl@ceph.mon.a.smithi039.stdout: s = self.context.wrap_socket( 2024-09-14T03:46:40.985 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:46:40.985 INFO:journalctl@ceph.mon.a.smithi039.stdout: return self.sslsocket_class._create( 2024-09-14T03:46:40.986 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:46:40.986 INFO:journalctl@ceph.mon.a.smithi039.stdout: self.do_handshake() 2024-09-14T03:46:40.986 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:46:40.986 INFO:journalctl@ceph.mon.a.smithi039.stdout: self._sslobj.do_handshake() 2024-09-14T03:46:40.986 INFO:journalctl@ceph.mon.a.smithi039.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:46:40.986 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:40 smithi039 ceph-mon[29586]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:40.986 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:40 smithi039 ceph-mon[29586]: from='mgr.14118 172.21.15.39:0/827427127' entity='mgr.y' 2024-09-14T03:46:40.986 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:40 smithi039 ceph-mon[29586]: from='mgr.14118 172.21.15.39:0/827427127' entity='mgr.y' 2024-09-14T03:46:40.986 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:40 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: Generating public/private rsa key pair. 2024-09-14T03:46:40.986 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:40 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: Your identification has been saved in /tmp/tmpdtf5hout/key 2024-09-14T03:46:40.987 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:40 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: Your public key has been saved in /tmp/tmpdtf5hout/key.pub 2024-09-14T03:46:40.987 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:40 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: The key fingerprint is: 2024-09-14T03:46:40.987 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:40 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: SHA256:s6T7AO1+WbxsJjFIEBjt8wqi2URiv5N+c7fadjRlszY ceph-bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T03:46:40.987 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:40 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: The key's randomart image is: 2024-09-14T03:46:40.987 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:40 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: +---[RSA 3072]----+ 2024-09-14T03:46:40.987 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:40 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: | .+.. | 2024-09-14T03:46:40.987 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:40 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: | . o | 2024-09-14T03:46:40.987 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:40 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: | . . | 2024-09-14T03:46:40.987 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:40 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: |... o.. + | 2024-09-14T03:46:40.987 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:40 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: |.o. .+..S. o o | 2024-09-14T03:46:40.987 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:40 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: |. o. ooooooo E | 2024-09-14T03:46:40.987 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:40 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: |.= .o.+ .*..o . | 2024-09-14T03:46:40.987 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:40 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: |o .+.+ +=o=. | 2024-09-14T03:46:40.987 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:40 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: | ..o =+=*o | 2024-09-14T03:46:40.988 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:40 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: +----[SHA256]-----+ 2024-09-14T03:46:41.901 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:41 smithi039 ceph-mon[29586]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:41.901 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:41 smithi039 ceph-mon[29586]: Generating ssh key... 2024-09-14T03:46:42.485 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDELs+0GJOm/3UcrrZuONYBoxrk37ZSQikNAyKZZSGNcwHgklr8kv2TFzFTyvZRBHwDpl53cGMgJi8BlNwTrBmZI8gCZ9OeH9XA/Ko/W8KPrlcCuX2aSw5IgvpwzBsCiJRI7zt9lz7X3oN3dMhdTr2FRtzsOv8hAA6ngjdSDWrAFi4TcnV5aheqFdVJJ4BSYxI2QilOae4dAyV9BoicYTct9R3Anhbq5uQAnuDygfFtQtpLrwR3ic57BEH/f5nOSUFID2p/sf73FN0CnKyKPRib9pSilRh7xAUktiUnsemAXY791MGw81ShvGKdSxM4YqNBIVltgcv0WI1eotk5GJpdD+rCvw7EeWs8OP8oUeARAzA9f9Ro1kbPPP5hvbAgJBqt+514pjEJnaWdTi/+QmOJfkbtD44dtYmzaoB50VgDVzS4jYrCJL5TpQAocI9T0y74Mp3izwYahJ59VribY6WyyZLaWjO/QzQmfKKk3l2CB/2VJ9OHypDx5kkEL7aZh1M= ceph-bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T03:46:42.486 INFO:teuthology.orchestra.run.smithi039.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-09-14T03:46:42.486 INFO:teuthology.orchestra.run.smithi039.stdout:Adding key to root@localhost authorized_keys... 2024-09-14T03:46:42.492 INFO:teuthology.orchestra.run.smithi039.stdout:Adding host smithi039... 2024-09-14T03:46:43.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:42 smithi039 ceph-mon[29586]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:44.166 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:43 smithi039 ceph-mon[29586]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi039", "addr": "172.21.15.39", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:44 smithi039 ceph-mon[29586]: Deploying cephadm binary to smithi039 2024-09-14T03:46:47.084 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout Added host 'smithi039' with addr '172.21.15.39' 2024-09-14T03:46:47.084 INFO:teuthology.orchestra.run.smithi039.stdout:Deploying unmanaged mon service... 2024-09-14T03:46:47.691 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:47 smithi039 ceph-mon[29586]: from='mgr.14118 172.21.15.39:0/827427127' entity='mgr.y' 2024-09-14T03:46:47.691 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:47 smithi039 ceph-mon[29586]: Added host smithi039 2024-09-14T03:46:47.691 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:47 smithi039 ceph-mon[29586]: from='mgr.14118 172.21.15.39:0/827427127' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:46:48.425 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-09-14T03:46:48.425 INFO:teuthology.orchestra.run.smithi039.stdout:Deploying unmanaged mgr service... 2024-09-14T03:46:48.973 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:48 smithi039 ceph-mon[29586]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:48.973 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:48 smithi039 ceph-mon[29586]: Saving service mon spec with placement count:5 2024-09-14T03:46:48.973 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:48 smithi039 ceph-mon[29586]: from='mgr.14118 172.21.15.39:0/827427127' entity='mgr.y' 2024-09-14T03:46:50.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:50 smithi039 ceph-mon[29586]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:46:50.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:50 smithi039 ceph-mon[29586]: Saving service mgr spec with placement count:2 2024-09-14T03:46:50.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:50 smithi039 ceph-mon[29586]: from='mgr.14118 172.21.15.39:0/827427127' entity='mgr.y' 2024-09-14T03:46:50.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:50 smithi039 ceph-mon[29586]: from='mgr.14118 172.21.15.39:0/827427127' entity='mgr.y' 2024-09-14T03:46:50.318 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-09-14T03:46:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:51 smithi039 ceph-mon[29586]: from='mgr.14118 172.21.15.39:0/827427127' entity='mgr.y' 2024-09-14T03:46:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:51 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/3872265018' entity='client.admin' 2024-09-14T03:46:53.552 INFO:teuthology.orchestra.run.smithi039.stdout:Enabling the dashboard module... 2024-09-14T03:46:53.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:53 smithi039 ceph-mon[29586]: from='mgr.14118 172.21.15.39:0/827427127' entity='mgr.y' 2024-09-14T03:46:53.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:53 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/3913873903' entity='client.admin' 2024-09-14T03:46:53.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:53 smithi039 ceph-mon[29586]: from='mgr.14118 172.21.15.39:0/827427127' entity='mgr.y' 2024-09-14T03:46:54.659 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:54 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/2369286854' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-09-14T03:46:54.659 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:54 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ignoring --setuser ceph since I am not root 2024-09-14T03:46:54.659 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:54 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ignoring --setgroup ceph since I am not root 2024-09-14T03:46:54.659 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:54 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:54.657+0000 7f838252c040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-14T03:46:54.941 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:54 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:54.718+0000 7f838252c040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-14T03:46:54.942 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:54 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:54.872+0000 7f838252c040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-14T03:46:54.942 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:54 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:54.940+0000 7f838252c040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-14T03:46:55.192 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:55 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:55.092+0000 7f838252c040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:46:55.476 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:55 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:55.239+0000 7f838252c040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-14T03:46:55.762 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:55 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/2369286854' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-09-14T03:46:55.762 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:55 smithi039 ceph-mon[29586]: mgrmap e9: y(active, since 18s) 2024-09-14T03:46:55.762 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:55 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:55.567+0000 7f838252c040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-14T03:46:55.762 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:55 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:55.629+0000 7f838252c040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-14T03:46:56.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:55 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:55.826+0000 7f838252c040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-14T03:46:56.827 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:56 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/3081296798' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-09-14T03:46:56.827 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:56 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:56.756+0000 7f838252c040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-14T03:46:56.827 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:56 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:56.826+0000 7f838252c040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-14T03:46:57.137 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout { 2024-09-14T03:46:57.138 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-09-14T03:46:57.138 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "available": true, 2024-09-14T03:46:57.138 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-09-14T03:46:57.138 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-09-14T03:46:57.138 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout } 2024-09-14T03:46:57.138 INFO:teuthology.orchestra.run.smithi039.stdout:Waiting for the mgr to restart... 2024-09-14T03:46:57.138 INFO:teuthology.orchestra.run.smithi039.stdout:Waiting for mgr epoch 9... 2024-09-14T03:46:57.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:57 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:57.581+0000 7f838252c040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-14T03:46:57.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:57 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:57.700+0000 7f838252c040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-14T03:46:58.135 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:57 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:57.847+0000 7f838252c040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:46:58.135 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:57 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:57.911+0000 7f838252c040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-14T03:46:58.135 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:58.063+0000 7f838252c040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-14T03:46:58.135 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:58.133+0000 7f838252c040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-14T03:46:58.426 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:58.195+0000 7f838252c040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-14T03:46:58.426 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:58.357+0000 7f838252c040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-14T03:46:58.426 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:58.425+0000 7f838252c040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-14T03:46:58.702 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:58.493+0000 7f838252c040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-14T03:46:58.702 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:58.579+0000 7f838252c040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-14T03:46:58.702 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:58.640+0000 7f838252c040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-14T03:46:58.703 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:58.701+0000 7f838252c040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-14T03:46:59.776 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:46:59 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:46:59.509+0000 7f838252c040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-14T03:46:59.776 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:59 smithi039 ceph-mon[29586]: Active manager daemon y restarted 2024-09-14T03:46:59.777 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:59 smithi039 ceph-mon[29586]: Activating manager daemon y 2024-09-14T03:46:59.777 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:59 smithi039 ceph-mon[29586]: osdmap e3: 0 total, 0 up, 0 in 2024-09-14T03:46:59.777 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:59 smithi039 ceph-mon[29586]: mgrmap e10: y(active, starting, since 0.00574342s) 2024-09-14T03:46:59.777 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:59 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:46:59.777 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:59 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:46:59.777 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:59 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:46:59.777 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:59 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:46:59.777 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:59 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:46:59.777 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:59 smithi039 ceph-mon[29586]: Manager daemon y is now available 2024-09-14T03:46:59.777 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:59 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:46:59.777 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:59 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:46:59.777 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:46:59 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:47:01.439 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout { 2024-09-14T03:47:01.439 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-09-14T03:47:01.439 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout "initialized": true 2024-09-14T03:47:01.439 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout } 2024-09-14T03:47:01.439 INFO:teuthology.orchestra.run.smithi039.stdout:mgr epoch 9 is available 2024-09-14T03:47:01.439 INFO:teuthology.orchestra.run.smithi039.stdout:Generating a dashboard self-signed certificate... 2024-09-14T03:47:01.701 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:01 smithi039 ceph-mon[29586]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-09-14T03:47:01.701 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:01 smithi039 ceph-mon[29586]: mgrmap e11: y(active, since 1.01032s) 2024-09-14T03:47:01.702 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:01 smithi039 ceph-mon[29586]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-09-14T03:47:01.702 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:01 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:01.955 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:47:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:47:01.816+0000 7f82e8976640 -1 log_channel(cephadm) log [ERR] : [14/Sep/2024:03:47:01] ENGINE Error in HTTPServer.serve 2024-09-14T03:47:01.955 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:47:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: Traceback (most recent call last): 2024-09-14T03:47:01.955 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:47:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:47:01.955 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:47:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: self._connections.run(self.expiration_interval) 2024-09-14T03:47:01.956 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:47:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:47:01.956 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:47:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: self._run(expiration_interval) 2024-09-14T03:47:01.956 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:47:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:47:01.956 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:47:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:47:01.956 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:47:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:47:01.956 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:47:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:47:01.956 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:47:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:47:01.956 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:47:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: s = self.context.wrap_socket( 2024-09-14T03:47:01.956 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:47:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:47:01.956 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:47:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: return self.sslsocket_class._create( 2024-09-14T03:47:01.956 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:47:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:47:01.956 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:47:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: self.do_handshake() 2024-09-14T03:47:01.956 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:47:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:47:01.957 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:47:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: self._sslobj.do_handshake() 2024-09-14T03:47:01.957 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:47:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:47:01.957 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:47:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:47:02.811 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-09-14T03:47:02.811 INFO:teuthology.orchestra.run.smithi039.stdout:Creating initial admin user... 2024-09-14T03:47:03.077 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:02 smithi039 ceph-mon[29586]: [14/Sep/2024:03:47:01] ENGINE Bus STARTING 2024-09-14T03:47:03.077 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:02 smithi039 ceph-mon[29586]: [14/Sep/2024:03:47:01] ENGINE Serving on http://172.21.15.39:8765 2024-09-14T03:47:03.077 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:02 smithi039 ceph-mon[29586]: [14/Sep/2024:03:47:01] ENGINE Serving on https://172.21.15.39:7150 2024-09-14T03:47:03.077 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:02 smithi039 ceph-mon[29586]: [14/Sep/2024:03:47:01] ENGINE Bus STARTED 2024-09-14T03:47:03.077 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:02 smithi039 ceph-mon[29586]: [14/Sep/2024:03:47:01] ENGINE Error in HTTPServer.serve 2024-09-14T03:47:03.077 INFO:journalctl@ceph.mon.a.smithi039.stdout: Traceback (most recent call last): 2024-09-14T03:47:03.078 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:47:03.078 INFO:journalctl@ceph.mon.a.smithi039.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:47:03.078 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:47:03.078 INFO:journalctl@ceph.mon.a.smithi039.stdout: self._run(expiration_interval) 2024-09-14T03:47:03.078 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:47:03.078 INFO:journalctl@ceph.mon.a.smithi039.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:47:03.078 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:47:03.078 INFO:journalctl@ceph.mon.a.smithi039.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:47:03.078 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:47:03.078 INFO:journalctl@ceph.mon.a.smithi039.stdout: s = self.context.wrap_socket( 2024-09-14T03:47:03.078 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:47:03.078 INFO:journalctl@ceph.mon.a.smithi039.stdout: return self.sslsocket_class._create( 2024-09-14T03:47:03.078 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:47:03.078 INFO:journalctl@ceph.mon.a.smithi039.stdout: self.do_handshake() 2024-09-14T03:47:03.079 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:47:03.079 INFO:journalctl@ceph.mon.a.smithi039.stdout: self._sslobj.do_handshake() 2024-09-14T03:47:03.079 INFO:journalctl@ceph.mon.a.smithi039.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:47:03.079 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:02 smithi039 ceph-mon[29586]: mgrmap e12: y(active, since 2s) 2024-09-14T03:47:03.079 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:02 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:03.079 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:02 smithi039 ceph-mon[29586]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:03.079 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:02 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:03.079 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:02 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:04.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:03 smithi039 ceph-mon[29586]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:04.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:03 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:05.099 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$oMgpqHyUdxllHBYky7CYdeOYuRv6wP2cg.ftm1DkafG6dqZvG/lHa", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1726285623, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-09-14T03:47:05.099 INFO:teuthology.orchestra.run.smithi039.stdout:Fetching dashboard port number... 2024-09-14T03:47:06.355 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stdout 8443 2024-09-14T03:47:06.367 INFO:teuthology.orchestra.run.smithi039.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-09-14T03:47:06.367 INFO:teuthology.orchestra.run.smithi039.stdout:systemctl: stdout disabled 2024-09-14T03:47:06.375 INFO:teuthology.orchestra.run.smithi039.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-09-14T03:47:06.375 INFO:teuthology.orchestra.run.smithi039.stdout:systemctl: stdout inactive 2024-09-14T03:47:06.376 INFO:teuthology.orchestra.run.smithi039.stdout:firewalld.service is not enabled 2024-09-14T03:47:06.376 INFO:teuthology.orchestra.run.smithi039.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-09-14T03:47:06.378 INFO:teuthology.orchestra.run.smithi039.stdout:Ceph Dashboard is now available at: 2024-09-14T03:47:06.378 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:47:06.378 INFO:teuthology.orchestra.run.smithi039.stdout: URL: https://smithi039.front.sepia.ceph.com:8443/ 2024-09-14T03:47:06.378 INFO:teuthology.orchestra.run.smithi039.stdout: User: admin 2024-09-14T03:47:06.378 INFO:teuthology.orchestra.run.smithi039.stdout: Password: r52pbyfzs0 2024-09-14T03:47:06.379 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:47:06.379 INFO:teuthology.orchestra.run.smithi039.stdout:Saving cluster configuration to /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config directory 2024-09-14T03:47:06.380 INFO:teuthology.orchestra.run.smithi039.stdout:Enabling autotune for osd_memory_target 2024-09-14T03:47:06.612 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:06 smithi039 ceph-mon[29586]: mgrmap e13: y(active, since 6s) 2024-09-14T03:47:06.612 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:06 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/3209783617' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-09-14T03:47:08.959 INFO:teuthology.orchestra.run.smithi039.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-09-14T03:47:08.959 INFO:teuthology.orchestra.run.smithi039.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-09-14T03:47:08.959 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:47:08.959 INFO:teuthology.orchestra.run.smithi039.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-09-14T03:47:08.959 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:47:08.959 INFO:teuthology.orchestra.run.smithi039.stdout:Or, if you are only running a single cluster on this host: 2024-09-14T03:47:08.959 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:47:08.959 INFO:teuthology.orchestra.run.smithi039.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-09-14T03:47:08.959 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:47:08.959 INFO:teuthology.orchestra.run.smithi039.stdout:Please consider enabling telemetry to help improve Ceph: 2024-09-14T03:47:08.959 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:47:08.959 INFO:teuthology.orchestra.run.smithi039.stdout: ceph telemetry on 2024-09-14T03:47:08.960 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:47:08.960 INFO:teuthology.orchestra.run.smithi039.stdout:For more information see: 2024-09-14T03:47:08.960 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:47:08.960 INFO:teuthology.orchestra.run.smithi039.stdout: https://docs.ceph.com/en/latest/mgr/telemetry/ 2024-09-14T03:47:08.960 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:47:08.960 INFO:teuthology.orchestra.run.smithi039.stdout:Bootstrap complete. 2024-09-14T03:47:08.999 INFO:tasks.cephadm:Fetching config... 2024-09-14T03:47:08.999 DEBUG:teuthology.orchestra.run.smithi039:> set -ex 2024-09-14T03:47:08.999 DEBUG:teuthology.orchestra.run.smithi039:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-09-14T03:47:09.018 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-09-14T03:47:09.018 DEBUG:teuthology.orchestra.run.smithi039:> set -ex 2024-09-14T03:47:09.018 DEBUG:teuthology.orchestra.run.smithi039:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-09-14T03:47:09.075 INFO:tasks.cephadm:Fetching mon keyring... 2024-09-14T03:47:09.076 DEBUG:teuthology.orchestra.run.smithi039:> set -ex 2024-09-14T03:47:09.076 DEBUG:teuthology.orchestra.run.smithi039:> sudo dd if=/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/keyring of=/dev/stdout 2024-09-14T03:47:09.142 INFO:tasks.cephadm:Fetching pub ssh key... 2024-09-14T03:47:09.142 DEBUG:teuthology.orchestra.run.smithi039:> set -ex 2024-09-14T03:47:09.142 DEBUG:teuthology.orchestra.run.smithi039:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-09-14T03:47:09.199 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-09-14T03:47:09.199 DEBUG:teuthology.orchestra.run.smithi039:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDELs+0GJOm/3UcrrZuONYBoxrk37ZSQikNAyKZZSGNcwHgklr8kv2TFzFTyvZRBHwDpl53cGMgJi8BlNwTrBmZI8gCZ9OeH9XA/Ko/W8KPrlcCuX2aSw5IgvpwzBsCiJRI7zt9lz7X3oN3dMhdTr2FRtzsOv8hAA6ngjdSDWrAFi4TcnV5aheqFdVJJ4BSYxI2QilOae4dAyV9BoicYTct9R3Anhbq5uQAnuDygfFtQtpLrwR3ic57BEH/f5nOSUFID2p/sf73FN0CnKyKPRib9pSilRh7xAUktiUnsemAXY791MGw81ShvGKdSxM4YqNBIVltgcv0WI1eotk5GJpdD+rCvw7EeWs8OP8oUeARAzA9f9Ro1kbPPP5hvbAgJBqt+514pjEJnaWdTi/+QmOJfkbtD44dtYmzaoB50VgDVzS4jYrCJL5TpQAocI9T0y74Mp3izwYahJ59VribY6WyyZLaWjO/QzQmfKKk3l2CB/2VJ9OHypDx5kkEL7aZh1M= ceph-bd7c803c-724b-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-14T03:47:09.274 INFO:teuthology.orchestra.run.smithi039.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDELs+0GJOm/3UcrrZuONYBoxrk37ZSQikNAyKZZSGNcwHgklr8kv2TFzFTyvZRBHwDpl53cGMgJi8BlNwTrBmZI8gCZ9OeH9XA/Ko/W8KPrlcCuX2aSw5IgvpwzBsCiJRI7zt9lz7X3oN3dMhdTr2FRtzsOv8hAA6ngjdSDWrAFi4TcnV5aheqFdVJJ4BSYxI2QilOae4dAyV9BoicYTct9R3Anhbq5uQAnuDygfFtQtpLrwR3ic57BEH/f5nOSUFID2p/sf73FN0CnKyKPRib9pSilRh7xAUktiUnsemAXY791MGw81ShvGKdSxM4YqNBIVltgcv0WI1eotk5GJpdD+rCvw7EeWs8OP8oUeARAzA9f9Ro1kbPPP5hvbAgJBqt+514pjEJnaWdTi/+QmOJfkbtD44dtYmzaoB50VgDVzS4jYrCJL5TpQAocI9T0y74Mp3izwYahJ59VribY6WyyZLaWjO/QzQmfKKk3l2CB/2VJ9OHypDx5kkEL7aZh1M= ceph-bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T03:47:09.286 DEBUG:teuthology.orchestra.run.smithi129:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDELs+0GJOm/3UcrrZuONYBoxrk37ZSQikNAyKZZSGNcwHgklr8kv2TFzFTyvZRBHwDpl53cGMgJi8BlNwTrBmZI8gCZ9OeH9XA/Ko/W8KPrlcCuX2aSw5IgvpwzBsCiJRI7zt9lz7X3oN3dMhdTr2FRtzsOv8hAA6ngjdSDWrAFi4TcnV5aheqFdVJJ4BSYxI2QilOae4dAyV9BoicYTct9R3Anhbq5uQAnuDygfFtQtpLrwR3ic57BEH/f5nOSUFID2p/sf73FN0CnKyKPRib9pSilRh7xAUktiUnsemAXY791MGw81ShvGKdSxM4YqNBIVltgcv0WI1eotk5GJpdD+rCvw7EeWs8OP8oUeARAzA9f9Ro1kbPPP5hvbAgJBqt+514pjEJnaWdTi/+QmOJfkbtD44dtYmzaoB50VgDVzS4jYrCJL5TpQAocI9T0y74Mp3izwYahJ59VribY6WyyZLaWjO/QzQmfKKk3l2CB/2VJ9OHypDx5kkEL7aZh1M= ceph-bd7c803c-724b-11ef-bceb-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-09-14T03:47:09.433 INFO:teuthology.orchestra.run.smithi129.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDELs+0GJOm/3UcrrZuONYBoxrk37ZSQikNAyKZZSGNcwHgklr8kv2TFzFTyvZRBHwDpl53cGMgJi8BlNwTrBmZI8gCZ9OeH9XA/Ko/W8KPrlcCuX2aSw5IgvpwzBsCiJRI7zt9lz7X3oN3dMhdTr2FRtzsOv8hAA6ngjdSDWrAFi4TcnV5aheqFdVJJ4BSYxI2QilOae4dAyV9BoicYTct9R3Anhbq5uQAnuDygfFtQtpLrwR3ic57BEH/f5nOSUFID2p/sf73FN0CnKyKPRib9pSilRh7xAUktiUnsemAXY791MGw81ShvGKdSxM4YqNBIVltgcv0WI1eotk5GJpdD+rCvw7EeWs8OP8oUeARAzA9f9Ro1kbPPP5hvbAgJBqt+514pjEJnaWdTi/+QmOJfkbtD44dtYmzaoB50VgDVzS4jYrCJL5TpQAocI9T0y74Mp3izwYahJ59VribY6WyyZLaWjO/QzQmfKKk3l2CB/2VJ9OHypDx5kkEL7aZh1M= ceph-bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T03:47:09.447 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-09-14T03:47:09.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:09 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/2245937808' entity='client.admin' 2024-09-14T03:47:09.729 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:47:12.374 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:12 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:12.375 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:12 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:12.375 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:12 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi039", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:47:12.375 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:12 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:12.375 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:12 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:12.375 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:12 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:13.000 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-09-14T03:47:13.001 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-09-14T03:47:13.565 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:13 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/664540561' entity='client.admin' 2024-09-14T03:47:13.565 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:13 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:13.566 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:13 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:13.633 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:47:16.285 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi129 2024-09-14T03:47:16.285 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T03:47:16.285 DEBUG:teuthology.orchestra.run.smithi129:> dd of=/etc/ceph/ceph.conf 2024-09-14T03:47:16.305 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T03:47:16.305 DEBUG:teuthology.orchestra.run.smithi129:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:47:16.366 INFO:tasks.cephadm:Adding host smithi129 to orchestrator... 2024-09-14T03:47:16.366 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph orch host add smithi129 2024-09-14T03:47:16.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:16 smithi039 ceph-mon[29586]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:16.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:16 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:16.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:16 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:16.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:16 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:16.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:16 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:16.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:16 smithi039 ceph-mon[29586]: Updating smithi039:/etc/ceph/ceph.conf 2024-09-14T03:47:16.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:16 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:16.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:16 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:16.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:16 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:16.565 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:47:17.652 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:17 smithi039 ceph-mon[29586]: Updating smithi039:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:47:17.652 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:17 smithi039 ceph-mon[29586]: Updating smithi039:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:47:17.652 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:17 smithi039 ceph-mon[29586]: Updating smithi039:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:47:19.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:18 smithi039 ceph-mon[29586]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi129", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:20.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:20 smithi039 ceph-mon[29586]: Deploying cephadm binary to smithi129 2024-09-14T03:47:20.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:20 smithi039 ceph-mon[29586]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:21.644 INFO:teuthology.orchestra.run.smithi039.stdout:Added host 'smithi129' with addr '172.21.15.129' 2024-09-14T03:47:22.380 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph orch host ls --format=json 2024-09-14T03:47:22.581 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:47:22.703 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:22 smithi039 ceph-mon[29586]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:22.704 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:22 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:22.704 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:22 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:22.704 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:22 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:22.704 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:22 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:23.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:23 smithi039 ceph-mon[29586]: Added host smithi129 2024-09-14T03:47:23.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:23 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:24.454 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:47:24.454 INFO:teuthology.orchestra.run.smithi039.stdout:[{"addr": "172.21.15.39", "hostname": "smithi039", "labels": [], "status": ""}, {"addr": "172.21.15.129", "hostname": "smithi129", "labels": [], "status": ""}] 2024-09-14T03:47:24.753 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:24 smithi039 ceph-mon[29586]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:25.050 INFO:tasks.cephadm:Setting crush tunables to default 2024-09-14T03:47:25.051 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd crush tunables default 2024-09-14T03:47:25.249 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:47:25.623 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:25 smithi039 ceph-mon[29586]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:47:26.725 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:26 smithi039 ceph-mon[29586]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:26.725 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:26 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:26.725 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:26 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:26.725 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:26 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:26.725 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:26 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:26.725 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:26 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi129", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:47:26.726 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:26 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:26.726 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:26 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:27.685 INFO:teuthology.orchestra.run.smithi039.stderr:adjusted tunables profile to default 2024-09-14T03:47:28.008 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:27 smithi039 ceph-mon[29586]: Updating smithi129:/etc/ceph/ceph.conf 2024-09-14T03:47:28.008 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:27 smithi039 ceph-mon[29586]: Updating smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:47:28.008 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:27 smithi039 ceph-mon[29586]: Updating smithi129:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:47:28.008 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:27 smithi039 ceph-mon[29586]: Updating smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:47:28.008 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:27 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:28.008 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:27 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:28.009 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:27 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:28.009 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:27 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/3042682452' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-14T03:47:28.300 INFO:tasks.cephadm:Adding mon.a on smithi039 2024-09-14T03:47:28.300 INFO:tasks.cephadm:Adding mon.c on smithi039 2024-09-14T03:47:28.300 INFO:tasks.cephadm:Adding mon.b on smithi129 2024-09-14T03:47:28.301 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph orch apply mon '3;smithi039:172.21.15.39=a;smithi039:[v2:172.21.15.39:3301,v1:172.21.15.39:6790]=c;smithi129:172.21.15.129=b' 2024-09-14T03:47:28.498 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:47:29.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:28 smithi039 ceph-mon[29586]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:29.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:28 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/3042682452' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-14T03:47:29.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:28 smithi039 ceph-mon[29586]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:47:30.147 INFO:teuthology.orchestra.run.smithi129.stdout:Scheduled mon update... 2024-09-14T03:47:30.751 DEBUG:teuthology.orchestra.run.smithi039:mon.c> sudo journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mon.c.service 2024-09-14T03:47:30.754 DEBUG:teuthology.orchestra.run.smithi129:mon.b> sudo journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mon.b.service 2024-09-14T03:47:30.756 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-14T03:47:30.756 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-14T03:47:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:30 smithi039 ceph-mon[29586]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:30 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:30 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:30 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:31.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:30 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:31.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:30 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:31.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:30 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:47:31.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:30 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:31.827 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:47:32.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:31 smithi039 ceph-mon[29586]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi039:172.21.15.39=a;smithi039:[v2:172.21.15.39:3301,v1:172.21.15.39:6790]=c;smithi129:172.21.15.129=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:32.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:31 smithi039 ceph-mon[29586]: Saving service mon spec with placement smithi039:172.21.15.39=a;smithi039:[v2:172.21.15.39:3301,v1:172.21.15.39:6790]=c;smithi129:172.21.15.129=b;count:3 2024-09-14T03:47:32.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:31 smithi039 ceph-mon[29586]: Deploying daemon mon.b on smithi129 2024-09-14T03:47:33.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:32 smithi039 ceph-mon[29586]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:34.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:34 smithi039 ceph-mon[29586]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:34.791 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:47:34.791 INFO:teuthology.orchestra.run.smithi129.stdout:{"epoch":1,"fsid":"bd7c803c-724b-11ef-bceb-c7b262605968","modified":"2024-09-14T03:46:10.112174Z","created":"2024-09-14T03:46:10.112174Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:3300","nonce":0},{"type":"v1","addr":"172.21.15.39:6789","nonce":0}]},"addr":"172.21.15.39:6789/0","public_addr":"172.21.15.39:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-09-14T03:47:34.791 INFO:teuthology.orchestra.run.smithi129.stderr:dumped monmap epoch 1 2024-09-14T03:47:35.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:35 smithi129 systemd[1]: Started Ceph mon.b for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:47:36.592 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-14T03:47:36.592 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-14T03:47:36.806 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:47:40.687 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:40 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:40.688 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:40 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:40.688 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:40 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:40.688 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:40 smithi039 ceph-mon[29586]: mon.a calling monitor election 2024-09-14T03:47:40.688 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:40 smithi039 ceph-mon[29586]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:40.688 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:40 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:40.688 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:40 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:40.688 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:40 smithi039 ceph-mon[29586]: mon.b calling monitor election 2024-09-14T03:47:40.688 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:40 smithi039 ceph-mon[29586]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:40.688 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:40 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:40.688 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:40 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:40.689 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:40 smithi039 ceph-mon[29586]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:40.689 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:40 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:40.689 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:40 smithi039 ceph-mon[29586]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-14T03:47:40.689 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:40 smithi039 ceph-mon[29586]: monmap e2: 2 mons at {a=[v2:172.21.15.39:3300/0,v1:172.21.15.39:6789/0],b=[v2:172.21.15.129:3300/0,v1:172.21.15.129:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:47:40.689 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:40 smithi039 ceph-mon[29586]: fsmap 2024-09-14T03:47:40.689 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:40 smithi039 ceph-mon[29586]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:47:40.689 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:40 smithi039 ceph-mon[29586]: mgrmap e13: y(active, since 40s) 2024-09-14T03:47:40.689 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:40 smithi039 ceph-mon[29586]: overall HEALTH_OK 2024-09-14T03:47:40.689 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:40 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:40.689 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:40 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:40.689 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:40 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:40.689 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:40 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:47:40.689 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:40 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:40.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:40 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:40.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:40 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:40.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:40 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:40.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:40 smithi129 ceph-mon[33173]: mon.a calling monitor election 2024-09-14T03:47:40.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:40 smithi129 ceph-mon[33173]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:40.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:40 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:40.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:40 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:40.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:40 smithi129 ceph-mon[33173]: mon.b calling monitor election 2024-09-14T03:47:40.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:40 smithi129 ceph-mon[33173]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:40.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:40 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:40.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:40 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:40.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:40 smithi129 ceph-mon[33173]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:40.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:40 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:40.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:40 smithi129 ceph-mon[33173]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-14T03:47:40.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:40 smithi129 ceph-mon[33173]: monmap e2: 2 mons at {a=[v2:172.21.15.39:3300/0,v1:172.21.15.39:6789/0],b=[v2:172.21.15.129:3300/0,v1:172.21.15.129:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:47:40.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:40 smithi129 ceph-mon[33173]: fsmap 2024-09-14T03:47:40.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:40 smithi129 ceph-mon[33173]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:47:40.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:40 smithi129 ceph-mon[33173]: mgrmap e13: y(active, since 40s) 2024-09-14T03:47:40.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:40 smithi129 ceph-mon[33173]: overall HEALTH_OK 2024-09-14T03:47:40.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:40 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:40.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:40 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:40.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:40 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:40.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:40 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:47:40.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:40 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:41.574 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:47:41.574 INFO:teuthology.orchestra.run.smithi129.stdout:{"epoch":2,"fsid":"bd7c803c-724b-11ef-bceb-c7b262605968","modified":"2024-09-14T03:47:35.270819Z","created":"2024-09-14T03:46:10.112174Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:3300","nonce":0},{"type":"v1","addr":"172.21.15.39:6789","nonce":0}]},"addr":"172.21.15.39:6789/0","public_addr":"172.21.15.39:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:3300","nonce":0},{"type":"v1","addr":"172.21.15.129:6789","nonce":0}]},"addr":"172.21.15.129:6789/0","public_addr":"172.21.15.129:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-09-14T03:47:41.574 INFO:teuthology.orchestra.run.smithi129.stderr:dumped monmap epoch 2 2024-09-14T03:47:41.704 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:41 smithi039 ceph-mon[29586]: Deploying daemon mon.c on smithi039 2024-09-14T03:47:41.704 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:41 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:41 smithi129 ceph-mon[33173]: Deploying daemon mon.c on smithi039 2024-09-14T03:47:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:41 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:42.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:42 smithi129 ceph-mon[33173]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:42.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:42 smithi129 ceph-mon[33173]: from='client.? 172.21.15.129:0/2591450155' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:47:42.744 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:42 smithi039 ceph-mon[29586]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:42.744 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:42 smithi039 ceph-mon[29586]: from='client.? 172.21.15.129:0/2591450155' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:47:43.158 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-09-14T03:47:43.159 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph mon dump -f json 2024-09-14T03:47:43.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 systemd[1]: Starting Ceph mon.c for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:47:43.362 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:47:43.646 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[29586]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:43.646 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 podman[40924]: 2024-09-14 03:47:43.297537361 +0000 UTC m=+0.016276788 image pull 67ddd33fa72f26a51c969a9043e7a3ddf784506b3d1949ad815ae4dd791beec1 quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322 2024-09-14T03:47:43.647 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 podman[40924]: 2024-09-14 03:47:43.403428409 +0000 UTC m=+0.122167894 container create 31d440e49f9378955f5058cd14dbd47dcec3e9c74c6bec6a19da4695add4077e (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-c, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=reef-e4e227b, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-14T03:47:43.647 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 podman[40924]: 2024-09-14 03:47:43.613996166 +0000 UTC m=+0.332735595 container init 31d440e49f9378955f5058cd14dbd47dcec3e9c74c6bec6a19da4695add4077e (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-c, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=reef-e4e227b, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0) 2024-09-14T03:47:43.647 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 podman[40924]: 2024-09-14 03:47:43.618960428 +0000 UTC m=+0.337699855 container start 31d440e49f9378955f5058cd14dbd47dcec3e9c74c6bec6a19da4695add4077e (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-c, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=reef-e4e227b, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-14T03:47:43.647 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: set uid:gid to 167:167 (ceph:ceph) 2024-09-14T03:47:43.647 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable), process ceph-mon, pid 7 2024-09-14T03:47:43.647 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: pidfile_write: ignore empty --pid-file 2024-09-14T03:47:43.647 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: load: jerasure load: lrc 2024-09-14T03:47:43.647 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: RocksDB version: 7.9.2 2024-09-14T03:47:43.647 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Git sha 0 2024-09-14T03:47:43.647 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Compile date 2024-09-13 16:45:57 2024-09-14T03:47:43.647 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: DB SUMMARY 2024-09-14T03:47:43.648 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: DB Session ID: UFT3MFVBK0CVAG829SGJ 2024-09-14T03:47:43.648 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: CURRENT file: CURRENT 2024-09-14T03:47:43.648 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: IDENTITY file: IDENTITY 2024-09-14T03:47:43.648 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: MANIFEST file: MANIFEST-000005 size: 59 Bytes 2024-09-14T03:47:43.648 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2024-09-14T03:47:43.648 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2024-09-14T03:47:43.648 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.error_if_exists: 0 2024-09-14T03:47:43.648 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.create_if_missing: 0 2024-09-14T03:47:43.648 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.paranoid_checks: 1 2024-09-14T03:47:43.648 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-14T03:47:43.648 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-14T03:47:43.648 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-14T03:47:43.649 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.env: 0x55eb88f90bc0 2024-09-14T03:47:43.649 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.fs: PosixFileSystem 2024-09-14T03:47:43.649 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.info_log: 0x55eb8abe0be0 2024-09-14T03:47:43.649 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.max_file_opening_threads: 16 2024-09-14T03:47:43.649 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.statistics: (nil) 2024-09-14T03:47:43.649 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.use_fsync: 0 2024-09-14T03:47:43.649 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.max_log_file_size: 0 2024-09-14T03:47:43.649 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-14T03:47:43.649 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-14T03:47:43.649 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.keep_log_file_num: 1000 2024-09-14T03:47:43.649 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.recycle_log_file_num: 0 2024-09-14T03:47:43.649 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.allow_fallocate: 1 2024-09-14T03:47:43.649 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.allow_mmap_reads: 0 2024-09-14T03:47:43.650 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.allow_mmap_writes: 0 2024-09-14T03:47:43.650 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.use_direct_reads: 0 2024-09-14T03:47:43.650 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-14T03:47:43.650 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.create_missing_column_families: 0 2024-09-14T03:47:43.650 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.db_log_dir: 2024-09-14T03:47:43.650 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.wal_dir: 2024-09-14T03:47:43.650 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-14T03:47:43.650 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-14T03:47:43.650 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-14T03:47:43.650 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-14T03:47:43.650 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-14T03:47:43.650 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-14T03:47:43.650 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.advise_random_on_open: 1 2024-09-14T03:47:43.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.db_write_buffer_size: 0 2024-09-14T03:47:43.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.write_buffer_manager: 0x55eb8abee320 2024-09-14T03:47:43.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-14T03:47:43.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-14T03:47:43.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-14T03:47:43.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.rate_limiter: (nil) 2024-09-14T03:47:43.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-14T03:47:43.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.wal_recovery_mode: 2 2024-09-14T03:47:43.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.enable_thread_tracking: 0 2024-09-14T03:47:43.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.enable_pipelined_write: 0 2024-09-14T03:47:43.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.unordered_write: 0 2024-09-14T03:47:43.652 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-14T03:47:43.652 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-14T03:47:43.652 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-14T03:47:43.652 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-14T03:47:43.652 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.row_cache: None 2024-09-14T03:47:43.652 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.wal_filter: None 2024-09-14T03:47:43.652 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-14T03:47:43.652 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.allow_ingest_behind: 0 2024-09-14T03:47:43.652 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.two_write_queues: 0 2024-09-14T03:47:43.652 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.manual_wal_flush: 0 2024-09-14T03:47:43.652 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.wal_compression: 0 2024-09-14T03:47:43.652 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.atomic_flush: 0 2024-09-14T03:47:43.653 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-14T03:47:43.653 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-14T03:47:43.656 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-14T03:47:43.656 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.log_readahead_size: 0 2024-09-14T03:47:43.656 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-14T03:47:43.656 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.best_efforts_recovery: 0 2024-09-14T03:47:43.656 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-14T03:47:43.656 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-14T03:47:43.656 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.allow_data_in_errors: 0 2024-09-14T03:47:43.656 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.db_host_id: __hostname__ 2024-09-14T03:47:43.656 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-14T03:47:43.656 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.max_background_jobs: 2 2024-09-14T03:47:43.656 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.max_background_compactions: -1 2024-09-14T03:47:43.656 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.max_subcompactions: 1 2024-09-14T03:47:43.657 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-14T03:47:43.657 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-14T03:47:43.657 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-14T03:47:43.657 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.max_total_wal_size: 0 2024-09-14T03:47:43.657 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-14T03:47:43.657 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-14T03:47:43.657 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-14T03:47:43.657 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-14T03:47:43.714 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:43 smithi129 ceph-mon[33173]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:44.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.max_open_files: -1 2024-09-14T03:47:44.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.bytes_per_sync: 0 2024-09-14T03:47:44.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-14T03:47:44.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-14T03:47:44.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.compaction_readahead_size: 0 2024-09-14T03:47:44.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.max_background_flushes: -1 2024-09-14T03:47:44.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Compression algorithms supported: 2024-09-14T03:47:44.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: kZSTD supported: 0 2024-09-14T03:47:44.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: kXpressCompression supported: 0 2024-09-14T03:47:44.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: kBZip2Compression supported: 0 2024-09-14T03:47:44.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-14T03:47:44.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: kLZ4Compression supported: 1 2024-09-14T03:47:44.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: kZlibCompression supported: 1 2024-09-14T03:47:44.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: kLZ4HCCompression supported: 1 2024-09-14T03:47:44.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: kSnappyCompression supported: 1 2024-09-14T03:47:44.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-14T03:47:44.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-14T03:47:44.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 2024-09-14T03:47:44.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-14T03:47:44.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-14T03:47:44.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.merge_operator: 2024-09-14T03:47:44.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.compaction_filter: None 2024-09-14T03:47:44.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.compaction_filter_factory: None 2024-09-14T03:47:44.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.sst_partitioner_factory: None 2024-09-14T03:47:44.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-14T03:47:44.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-14T03:47:44.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55eb8abe07e0) 2024-09-14T03:47:44.043 INFO:journalctl@ceph.mon.c.smithi039.stdout: cache_index_and_filter_blocks: 1 2024-09-14T03:47:44.043 INFO:journalctl@ceph.mon.c.smithi039.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-14T03:47:44.043 INFO:journalctl@ceph.mon.c.smithi039.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-14T03:47:44.043 INFO:journalctl@ceph.mon.c.smithi039.stdout: pin_top_level_index_and_filter: 1 2024-09-14T03:47:44.043 INFO:journalctl@ceph.mon.c.smithi039.stdout: index_type: 0 2024-09-14T03:47:44.044 INFO:journalctl@ceph.mon.c.smithi039.stdout: data_block_index_type: 0 2024-09-14T03:47:44.044 INFO:journalctl@ceph.mon.c.smithi039.stdout: index_shortening: 1 2024-09-14T03:47:44.044 INFO:journalctl@ceph.mon.c.smithi039.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-14T03:47:44.044 INFO:journalctl@ceph.mon.c.smithi039.stdout: checksum: 4 2024-09-14T03:47:44.044 INFO:journalctl@ceph.mon.c.smithi039.stdout: no_block_cache: 0 2024-09-14T03:47:44.044 INFO:journalctl@ceph.mon.c.smithi039.stdout: block_cache: 0x55eb8abd51f0 2024-09-14T03:47:44.044 INFO:journalctl@ceph.mon.c.smithi039.stdout: block_cache_name: BinnedLRUCache 2024-09-14T03:47:44.044 INFO:journalctl@ceph.mon.c.smithi039.stdout: block_cache_options: 2024-09-14T03:47:44.044 INFO:journalctl@ceph.mon.c.smithi039.stdout: capacity : 536870912 2024-09-14T03:47:44.044 INFO:journalctl@ceph.mon.c.smithi039.stdout: num_shard_bits : 4 2024-09-14T03:47:44.044 INFO:journalctl@ceph.mon.c.smithi039.stdout: strict_capacity_limit : 0 2024-09-14T03:47:44.044 INFO:journalctl@ceph.mon.c.smithi039.stdout: high_pri_pool_ratio: 0.000 2024-09-14T03:47:44.044 INFO:journalctl@ceph.mon.c.smithi039.stdout: block_cache_compressed: (nil) 2024-09-14T03:47:44.045 INFO:journalctl@ceph.mon.c.smithi039.stdout: persistent_cache: (nil) 2024-09-14T03:47:44.045 INFO:journalctl@ceph.mon.c.smithi039.stdout: block_size: 4096 2024-09-14T03:47:44.045 INFO:journalctl@ceph.mon.c.smithi039.stdout: block_size_deviation: 10 2024-09-14T03:47:44.045 INFO:journalctl@ceph.mon.c.smithi039.stdout: block_restart_interval: 16 2024-09-14T03:47:44.045 INFO:journalctl@ceph.mon.c.smithi039.stdout: index_block_restart_interval: 1 2024-09-14T03:47:44.045 INFO:journalctl@ceph.mon.c.smithi039.stdout: metadata_block_size: 4096 2024-09-14T03:47:44.045 INFO:journalctl@ceph.mon.c.smithi039.stdout: partition_filters: 0 2024-09-14T03:47:44.045 INFO:journalctl@ceph.mon.c.smithi039.stdout: use_delta_encoding: 1 2024-09-14T03:47:44.045 INFO:journalctl@ceph.mon.c.smithi039.stdout: filter_policy: bloomfilter 2024-09-14T03:47:44.045 INFO:journalctl@ceph.mon.c.smithi039.stdout: whole_key_filtering: 1 2024-09-14T03:47:44.045 INFO:journalctl@ceph.mon.c.smithi039.stdout: verify_compression: 0 2024-09-14T03:47:44.045 INFO:journalctl@ceph.mon.c.smithi039.stdout: read_amp_bytes_per_bit: 0 2024-09-14T03:47:44.045 INFO:journalctl@ceph.mon.c.smithi039.stdout: format_version: 5 2024-09-14T03:47:44.046 INFO:journalctl@ceph.mon.c.smithi039.stdout: enable_index_compression: 1 2024-09-14T03:47:44.046 INFO:journalctl@ceph.mon.c.smithi039.stdout: block_align: 0 2024-09-14T03:47:44.046 INFO:journalctl@ceph.mon.c.smithi039.stdout: max_auto_readahead_size: 262144 2024-09-14T03:47:44.046 INFO:journalctl@ceph.mon.c.smithi039.stdout: prepopulate_block_cache: 0 2024-09-14T03:47:44.046 INFO:journalctl@ceph.mon.c.smithi039.stdout: initial_auto_readahead_size: 8192 2024-09-14T03:47:44.046 INFO:journalctl@ceph.mon.c.smithi039.stdout: num_file_reads_for_auto_readahead: 2 2024-09-14T03:47:44.046 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.write_buffer_size: 33554432 2024-09-14T03:47:44.046 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.max_write_buffer_number: 2 2024-09-14T03:47:44.046 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.compression: NoCompression 2024-09-14T03:47:44.046 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.bottommost_compression: Disabled 2024-09-14T03:47:44.046 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.prefix_extractor: nullptr 2024-09-14T03:47:44.046 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-14T03:47:44.046 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.num_levels: 7 2024-09-14T03:47:44.047 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-14T03:47:44.047 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-14T03:47:44.047 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-14T03:47:44.047 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-14T03:47:44.047 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-14T03:47:44.047 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-14T03:47:44.047 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-14T03:47:44.047 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:47:44.047 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-14T03:47:44.047 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-14T03:47:44.047 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:47:44.047 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:47:44.047 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-14T03:47:44.047 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.compression_opts.level: 32767 2024-09-14T03:47:44.047 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.compression_opts.strategy: 0 2024-09-14T03:47:44.048 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-14T03:47:44.048 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:47:44.048 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:47:44.048 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-14T03:47:44.049 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.compression_opts.enabled: false 2024-09-14T03:47:44.049 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:47:44.049 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-14T03:47:44.049 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-14T03:47:44.049 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-14T03:47:44.049 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.target_file_size_base: 67108864 2024-09-14T03:47:44.049 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-14T03:47:44.049 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-14T03:47:44.049 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-14T03:47:44.050 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-14T03:47:44.050 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-14T03:47:44.050 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-14T03:47:44.050 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-14T03:47:44.050 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-14T03:47:44.050 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-14T03:47:44.050 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-14T03:47:44.050 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-14T03:47:44.050 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-14T03:47:44.050 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-14T03:47:44.050 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-14T03:47:44.050 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.arena_block_size: 1048576 2024-09-14T03:47:44.050 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-14T03:47:44.051 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-14T03:47:44.051 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.disable_auto_compactions: 0 2024-09-14T03:47:44.051 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-14T03:47:44.051 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-14T03:47:44.051 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-14T03:47:44.051 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-14T03:47:44.051 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-14T03:47:44.051 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-14T03:47:44.051 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-14T03:47:44.051 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-14T03:47:44.051 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-14T03:47:44.051 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-14T03:47:44.051 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-14T03:47:44.052 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.inplace_update_support: 0 2024-09-14T03:47:44.052 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-14T03:47:44.052 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-14T03:47:44.052 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-14T03:47:44.052 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-14T03:47:44.052 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.bloom_locality: 0 2024-09-14T03:47:44.052 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.max_successive_merges: 0 2024-09-14T03:47:44.052 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-14T03:47:44.052 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.paranoid_file_checks: 0 2024-09-14T03:47:44.052 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.force_consistency_checks: 1 2024-09-14T03:47:44.052 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.report_bg_io_stats: 0 2024-09-14T03:47:44.052 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.ttl: 2592000 2024-09-14T03:47:44.052 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-14T03:47:44.053 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-14T03:47:44.053 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-14T03:47:44.053 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.enable_blob_files: false 2024-09-14T03:47:44.053 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.min_blob_size: 0 2024-09-14T03:47:44.053 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.blob_file_size: 268435456 2024-09-14T03:47:44.053 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-14T03:47:44.053 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-14T03:47:44.053 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-14T03:47:44.053 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-14T03:47:44.053 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-14T03:47:44.053 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.blob_file_starting_level: 0 2024-09-14T03:47:44.053 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-14T03:47:44.053 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000005 succeeded,manifest_file_number is 5, next_file_number is 7, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-09-14T03:47:44.054 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 0 2024-09-14T03:47:44.054 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: d29781d4-a373-4aab-bb3e-70394ca1cc26 2024-09-14T03:47:44.054 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285663645345, "job": 1, "event": "recovery_started", "wal_files": [4]} 2024-09-14T03:47:44.054 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #4 mode 2 2024-09-14T03:47:44.054 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285663645730, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1773, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 1, "largest_seqno": 5, "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_filter_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726285663, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "d29781d4-a373-4aab-bb3e-70394ca1cc26", "db_session_id": "UFT3MFVBK0CVAG829SGJ", "orig_file_number": 8, "seqno_to_time_mapping": "N/A"}} 2024-09-14T03:47:44.054 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726285663645825, "job": 1, "event": "recovery_finished"} 2024-09-14T03:47:44.054 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: [db/version_set.cc:5047] Creating manifest 10 2024-09-14T03:47:44.054 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-14T03:47:44.054 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55eb8ac00e00 2024-09-14T03:47:44.054 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: DB pointer 0x55eb8acf4000 2024-09-14T03:47:44.054 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-14T03:47:44.054 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-14T03:47:44.054 INFO:journalctl@ceph.mon.c.smithi039.stdout: ** DB Stats ** 2024-09-14T03:47:44.055 INFO:journalctl@ceph.mon.c.smithi039.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:47:44.055 INFO:journalctl@ceph.mon.c.smithi039.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-14T03:47:44.055 INFO:journalctl@ceph.mon.c.smithi039.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:47:44.055 INFO:journalctl@ceph.mon.c.smithi039.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:47:44.055 INFO:journalctl@ceph.mon.c.smithi039.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-14T03:47:44.055 INFO:journalctl@ceph.mon.c.smithi039.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:47:44.055 INFO:journalctl@ceph.mon.c.smithi039.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:47:44.055 INFO:journalctl@ceph.mon.c.smithi039.stdout: 2024-09-14T03:47:44.055 INFO:journalctl@ceph.mon.c.smithi039.stdout: ** Compaction Stats [default] ** 2024-09-14T03:47:44.055 INFO:journalctl@ceph.mon.c.smithi039.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:47:44.055 INFO:journalctl@ceph.mon.c.smithi039.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-14T03:47:44.055 INFO:journalctl@ceph.mon.c.smithi039.stdout: L0 1/0 1.73 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.5 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-14T03:47:44.055 INFO:journalctl@ceph.mon.c.smithi039.stdout: Sum 1/0 1.73 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.5 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-14T03:47:44.055 INFO:journalctl@ceph.mon.c.smithi039.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.5 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-14T03:47:44.056 INFO:journalctl@ceph.mon.c.smithi039.stdout: 2024-09-14T03:47:44.056 INFO:journalctl@ceph.mon.c.smithi039.stdout: ** Compaction Stats [default] ** 2024-09-14T03:47:44.056 INFO:journalctl@ceph.mon.c.smithi039.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:47:44.056 INFO:journalctl@ceph.mon.c.smithi039.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:47:44.056 INFO:journalctl@ceph.mon.c.smithi039.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.5 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-09-14T03:47:44.056 INFO:journalctl@ceph.mon.c.smithi039.stdout: 2024-09-14T03:47:44.056 INFO:journalctl@ceph.mon.c.smithi039.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-14T03:47:44.056 INFO:journalctl@ceph.mon.c.smithi039.stdout: 2024-09-14T03:47:44.056 INFO:journalctl@ceph.mon.c.smithi039.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:47:44.056 INFO:journalctl@ceph.mon.c.smithi039.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-09-14T03:47:44.056 INFO:journalctl@ceph.mon.c.smithi039.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-14T03:47:44.056 INFO:journalctl@ceph.mon.c.smithi039.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-14T03:47:44.056 INFO:journalctl@ceph.mon.c.smithi039.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-14T03:47:44.057 INFO:journalctl@ceph.mon.c.smithi039.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-14T03:47:44.057 INFO:journalctl@ceph.mon.c.smithi039.stdout: Cumulative compaction: 0.00 GB write, 0.65 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:47:44.057 INFO:journalctl@ceph.mon.c.smithi039.stdout: Interval compaction: 0.00 GB write, 0.65 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:47:44.057 INFO:journalctl@ceph.mon.c.smithi039.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-14T03:47:44.057 INFO:journalctl@ceph.mon.c.smithi039.stdout: Block cache BinnedLRUCache@0x55eb8abd51f0#7 capacity: 512.00 MB usage: 0.22 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 9e-06 secs_since: 0 2024-09-14T03:47:44.057 INFO:journalctl@ceph.mon.c.smithi039.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.11 KB,2.08616e-05%) IndexBlock(1,0.11 KB,2.08616e-05%) Misc(1,0.00 KB,0%) 2024-09-14T03:47:44.057 INFO:journalctl@ceph.mon.c.smithi039.stdout: 2024-09-14T03:47:44.057 INFO:journalctl@ceph.mon.c.smithi039.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-14T03:47:44.057 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: mon.c does not exist in monmap, will attempt to join an existing cluster 2024-09-14T03:47:44.057 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: using public_addrv [v2:172.21.15.39:3301/0,v1:172.21.15.39:6790/0] 2024-09-14T03:47:44.057 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: starting mon.c rank -1 at public addrs [v2:172.21.15.39:3301/0,v1:172.21.15.39:6790/0] at bind addrs [v2:172.21.15.39:3301/0,v1:172.21.15.39:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T03:47:44.057 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: mon.c@-1(???) e0 preinit fsid bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T03:47:44.057 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: mon.c@-1(synchronizing).mds e1 new map 2024-09-14T03:47:44.057 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: mon.c@-1(synchronizing).mds e1 print_map 2024-09-14T03:47:44.058 INFO:journalctl@ceph.mon.c.smithi039.stdout: e1 2024-09-14T03:47:44.058 INFO:journalctl@ceph.mon.c.smithi039.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-14T03:47:44.058 INFO:journalctl@ceph.mon.c.smithi039.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:47:44.058 INFO:journalctl@ceph.mon.c.smithi039.stdout: legacy client fscid: -1 2024-09-14T03:47:44.058 INFO:journalctl@ceph.mon.c.smithi039.stdout: 2024-09-14T03:47:44.058 INFO:journalctl@ceph.mon.c.smithi039.stdout: No filesystems configured 2024-09-14T03:47:44.058 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2024-09-14T03:47:44.058 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2024-09-14T03:47:44.058 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2024-09-14T03:47:44.058 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2024-09-14T03:47:44.058 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2024-09-14T03:47:44.058 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2024-09-14T03:47:44.058 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2024-09-14T03:47:44.058 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-14T03:47:44.059 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-14T03:47:44.059 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2024-09-14T03:47:44.059 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: Updating smithi129:/etc/ceph/ceph.conf 2024-09-14T03:47:44.059 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: Updating smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:47:44.059 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: Updating smithi129:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:47:44.059 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: Updating smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:47:44.059 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:44.059 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:44.059 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:44.059 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/3042682452' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-09-14T03:47:44.059 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:44.059 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/3042682452' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-09-14T03:47:44.059 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:47:44.060 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:44.060 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:44.060 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:44.060 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:44.060 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:44.060 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:44.060 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:47:44.060 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:44.060 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi039:172.21.15.39=a;smithi039:[v2:172.21.15.39:3301,v1:172.21.15.39:6790]=c;smithi129:172.21.15.129=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:44.060 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: Saving service mon spec with placement smithi039:172.21.15.39=a;smithi039:[v2:172.21.15.39:3301,v1:172.21.15.39:6790]=c;smithi129:172.21.15.129=b;count:3 2024-09-14T03:47:44.060 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: Deploying daemon mon.b on smithi129 2024-09-14T03:47:44.060 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:44.060 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:44.060 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:44.061 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:44.061 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:44.061 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: mon.a calling monitor election 2024-09-14T03:47:44.061 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:44.061 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:44.061 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:44.061 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: mon.b calling monitor election 2024-09-14T03:47:44.061 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:44.061 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:44.061 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:44.061 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:44.061 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:44.062 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-09-14T03:47:44.062 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: monmap e2: 2 mons at {a=[v2:172.21.15.39:3300/0,v1:172.21.15.39:6789/0],b=[v2:172.21.15.129:3300/0,v1:172.21.15.129:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:47:44.062 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: fsmap 2024-09-14T03:47:44.062 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:47:44.062 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: mgrmap e13: y(active, since 40s) 2024-09-14T03:47:44.062 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: overall HEALTH_OK 2024-09-14T03:47:44.062 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:44.062 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:44.062 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:44.062 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:47:44.062 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:44.062 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: Deploying daemon mon.c on smithi039 2024-09-14T03:47:44.062 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:44.062 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:44.063 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: from='client.? 172.21.15.129:0/2591450155' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:47:44.063 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:44.063 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 ceph-mon[40962]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-09-14T03:47:44.063 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 bash[40924]: 31d440e49f9378955f5058cd14dbd47dcec3e9c74c6bec6a19da4695add4077e 2024-09-14T03:47:44.063 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:43 smithi039 systemd[1]: Started Ceph mon.c for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:47:49.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:48 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:49.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:48 smithi129 ceph-mon[33173]: mon.a calling monitor election 2024-09-14T03:47:49.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:48 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:49.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:48 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:48 smithi129 ceph-mon[33173]: mon.b calling monitor election 2024-09-14T03:47:49.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:48 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:48 smithi129 ceph-mon[33173]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:49.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:48 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:48 smithi129 ceph-mon[33173]: mon.c calling monitor election 2024-09-14T03:47:49.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:48 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:48 smithi129 ceph-mon[33173]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:49.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:48 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:48 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:48 smithi129 ceph-mon[33173]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:47:49.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:48 smithi129 ceph-mon[33173]: monmap e3: 3 mons at {a=[v2:172.21.15.39:3300/0,v1:172.21.15.39:6789/0],b=[v2:172.21.15.129:3300/0,v1:172.21.15.129:6789/0],c=[v2:172.21.15.39:3301/0,v1:172.21.15.39:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:47:49.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:48 smithi129 ceph-mon[33173]: fsmap 2024-09-14T03:47:49.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:48 smithi129 ceph-mon[33173]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:47:49.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:48 smithi129 ceph-mon[33173]: mgrmap e13: y(active, since 49s) 2024-09-14T03:47:49.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:48 smithi129 ceph-mon[33173]: overall HEALTH_OK 2024-09-14T03:47:49.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:48 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:49.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:48 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:49.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:48 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:49.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:48 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:49.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:49.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[29586]: mon.a calling monitor election 2024-09-14T03:47:49.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:49.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[29586]: mon.b calling monitor election 2024-09-14T03:47:49.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[29586]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:49.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[29586]: mon.c calling monitor election 2024-09-14T03:47:49.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[29586]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:49.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[29586]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:47:49.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[29586]: monmap e3: 3 mons at {a=[v2:172.21.15.39:3300/0,v1:172.21.15.39:6789/0],b=[v2:172.21.15.129:3300/0,v1:172.21.15.129:6789/0],c=[v2:172.21.15.39:3301/0,v1:172.21.15.39:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:47:49.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[29586]: fsmap 2024-09-14T03:47:49.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[29586]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:47:49.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[29586]: mgrmap e13: y(active, since 49s) 2024-09-14T03:47:49.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[29586]: overall HEALTH_OK 2024-09-14T03:47:49.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:49.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:49.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:49.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:49.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:47:49.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[40962]: mon.a calling monitor election 2024-09-14T03:47:49.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:47:49.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[40962]: mon.b calling monitor election 2024-09-14T03:47:49.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[40962]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:49.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[40962]: mon.c calling monitor election 2024-09-14T03:47:49.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[40962]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:49.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:49.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[40962]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:47:49.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[40962]: monmap e3: 3 mons at {a=[v2:172.21.15.39:3300/0,v1:172.21.15.39:6789/0],b=[v2:172.21.15.129:3300/0,v1:172.21.15.129:6789/0],c=[v2:172.21.15.39:3301/0,v1:172.21.15.39:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-09-14T03:47:49.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[40962]: fsmap 2024-09-14T03:47:49.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[40962]: osdmap e4: 0 total, 0 up, 0 in 2024-09-14T03:47:49.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[40962]: mgrmap e13: y(active, since 49s) 2024-09-14T03:47:49.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[40962]: overall HEALTH_OK 2024-09-14T03:47:49.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:49.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:49.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:49.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:48 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:49.465 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:47:49.465 INFO:teuthology.orchestra.run.smithi129.stdout:{"epoch":3,"fsid":"bd7c803c-724b-11ef-bceb-c7b262605968","modified":"2024-09-14T03:47:43.672905Z","created":"2024-09-14T03:46:10.112174Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:3300","nonce":0},{"type":"v1","addr":"172.21.15.39:6789","nonce":0}]},"addr":"172.21.15.39:6789/0","public_addr":"172.21.15.39:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:3300","nonce":0},{"type":"v1","addr":"172.21.15.129:6789","nonce":0}]},"addr":"172.21.15.129:6789/0","public_addr":"172.21.15.129:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:3301","nonce":0},{"type":"v1","addr":"172.21.15.39:6790","nonce":0}]},"addr":"172.21.15.39:6790/0","public_addr":"172.21.15.39:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-09-14T03:47:49.465 INFO:teuthology.orchestra.run.smithi129.stderr:dumped monmap epoch 3 2024-09-14T03:47:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:49 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:49 smithi129 ceph-mon[33173]: from='client.? 172.21.15.129:0/2620913108' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:47:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:49 smithi129 ceph-mon[33173]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:49 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:49 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:50.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:49 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:50.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:49 smithi039 ceph-mon[29586]: from='client.? 172.21.15.129:0/2620913108' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:47:50.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:49 smithi039 ceph-mon[29586]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:50.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:49 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:50.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:49 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:50.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:49 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:50.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:49 smithi039 ceph-mon[40962]: from='client.? 172.21.15.129:0/2620913108' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-09-14T03:47:50.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:49 smithi039 ceph-mon[40962]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:50.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:49 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:50.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:49 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:47:50.515 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-09-14T03:47:50.515 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph config generate-minimal-conf 2024-09-14T03:47:51.048 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.048 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.048 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.049 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.049 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:51.049 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:51.049 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[29586]: Updating smithi039:/etc/ceph/ceph.conf 2024-09-14T03:47:51.049 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[29586]: Updating smithi129:/etc/ceph/ceph.conf 2024-09-14T03:47:51.050 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[29586]: Updating smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:47:51.050 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[29586]: Updating smithi039:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:47:51.050 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.050 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.050 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.050 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.051 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.051 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.051 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.051 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.051 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.052 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[29586]: Reconfiguring mon.a (unknown last config time)... 2024-09-14T03:47:51.052 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:47:51.052 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:47:51.052 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:51.052 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[29586]: Reconfiguring daemon mon.a on smithi039 2024-09-14T03:47:51.053 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.053 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.053 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.053 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.053 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:51.054 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:51.054 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[40962]: Updating smithi039:/etc/ceph/ceph.conf 2024-09-14T03:47:51.054 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[40962]: Updating smithi129:/etc/ceph/ceph.conf 2024-09-14T03:47:51.054 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[40962]: Updating smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:47:51.054 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[40962]: Updating smithi039:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:47:51.054 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.055 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.055 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.055 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.055 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.055 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.056 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.056 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.056 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.056 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[40962]: Reconfiguring mon.a (unknown last config time)... 2024-09-14T03:47:51.056 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:47:51.056 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:47:51.057 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:51.057 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:50 smithi039 ceph-mon[40962]: Reconfiguring daemon mon.a on smithi039 2024-09-14T03:47:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:50 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:50 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:50 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:50 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:50 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:51.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:50 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:51.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:50 smithi129 ceph-mon[33173]: Updating smithi039:/etc/ceph/ceph.conf 2024-09-14T03:47:51.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:50 smithi129 ceph-mon[33173]: Updating smithi129:/etc/ceph/ceph.conf 2024-09-14T03:47:51.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:50 smithi129 ceph-mon[33173]: Updating smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:47:51.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:50 smithi129 ceph-mon[33173]: Updating smithi039:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:47:51.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:50 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:50 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:50 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:50 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:50 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:50 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:50 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:50 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:50 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:51.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:50 smithi129 ceph-mon[33173]: Reconfiguring mon.a (unknown last config time)... 2024-09-14T03:47:51.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:50 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:47:51.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:50 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:47:51.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:50 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:51.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:50 smithi129 ceph-mon[33173]: Reconfiguring daemon mon.a on smithi039 2024-09-14T03:47:51.388 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:47:52.085 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:51 smithi039 ceph-mon[29586]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:52.085 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:51 smithi039 ceph-mon[40962]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:51 smithi129 ceph-mon[33173]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:53.203 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:52 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:53.203 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:52 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:53.203 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:52 smithi039 ceph-mon[29586]: Reconfiguring mon.c (monmap changed)... 2024-09-14T03:47:53.203 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:52 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:47:53.203 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:52 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:47:53.203 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:52 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:53.203 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:52 smithi039 ceph-mon[29586]: Reconfiguring daemon mon.c on smithi039 2024-09-14T03:47:53.204 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:52 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:53.204 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:52 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:53.204 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:52 smithi039 ceph-mon[40962]: Reconfiguring mon.c (monmap changed)... 2024-09-14T03:47:53.204 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:52 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:47:53.204 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:52 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:47:53.204 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:52 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:53.204 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:52 smithi039 ceph-mon[40962]: Reconfiguring daemon mon.c on smithi039 2024-09-14T03:47:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:52 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:52 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:52 smithi129 ceph-mon[33173]: Reconfiguring mon.c (monmap changed)... 2024-09-14T03:47:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:52 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:47:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:52 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:47:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:52 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:53.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:52 smithi129 ceph-mon[33173]: Reconfiguring daemon mon.c on smithi039 2024-09-14T03:47:54.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:53 smithi129 ceph-mon[33173]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:54.284 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:53 smithi039 ceph-mon[29586]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:54.284 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:53 smithi039 ceph-mon[40962]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:54.739 INFO:teuthology.orchestra.run.smithi039.stdout:# minimal ceph.conf for bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T03:47:54.740 INFO:teuthology.orchestra.run.smithi039.stdout:[global] 2024-09-14T03:47:54.740 INFO:teuthology.orchestra.run.smithi039.stdout: fsid = bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T03:47:54.740 INFO:teuthology.orchestra.run.smithi039.stdout: mon_host = [v2:172.21.15.39:3300/0,v1:172.21.15.39:6789/0] [v2:172.21.15.129:3300/0,v1:172.21.15.129:6789/0] [v2:172.21.15.39:3301/0,v1:172.21.15.39:6790/0] 2024-09-14T03:47:55.347 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-09-14T03:47:55.348 DEBUG:teuthology.orchestra.run.smithi039:> set -ex 2024-09-14T03:47:55.348 DEBUG:teuthology.orchestra.run.smithi039:> sudo dd of=/etc/ceph/ceph.conf 2024-09-14T03:47:55.377 DEBUG:teuthology.orchestra.run.smithi039:> set -ex 2024-09-14T03:47:55.378 DEBUG:teuthology.orchestra.run.smithi039:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:47:55.444 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T03:47:55.444 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd of=/etc/ceph/ceph.conf 2024-09-14T03:47:55.472 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T03:47:55.473 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:47:55.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:55.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:55.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:55 smithi129 ceph-mon[33173]: Reconfiguring mon.b (monmap changed)... 2024-09-14T03:47:55.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:47:55.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:47:55.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:55.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:55 smithi129 ceph-mon[33173]: Reconfiguring daemon mon.b on smithi129 2024-09-14T03:47:55.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:55 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/3758110352' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:55.539 INFO:tasks.cephadm:Adding mgr.y on smithi039 2024-09-14T03:47:55.539 INFO:tasks.cephadm:Adding mgr.x on smithi129 2024-09-14T03:47:55.539 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph orch apply mgr '2;smithi039=y;smithi129=x' 2024-09-14T03:47:55.769 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:47:55.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:55.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:55.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:55 smithi039 ceph-mon[29586]: Reconfiguring mon.b (monmap changed)... 2024-09-14T03:47:55.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:47:55.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:47:55.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:55.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:55 smithi039 ceph-mon[29586]: Reconfiguring daemon mon.b on smithi129 2024-09-14T03:47:55.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:55 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/3758110352' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:55.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:55.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:55.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:55 smithi039 ceph-mon[40962]: Reconfiguring mon.b (monmap changed)... 2024-09-14T03:47:55.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:47:55.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:47:55.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:55.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:55 smithi039 ceph-mon[40962]: Reconfiguring daemon mon.b on smithi129 2024-09-14T03:47:55.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:55 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/3758110352' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:56.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:56 smithi129 ceph-mon[33173]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:56.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:56 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:56.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:56 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:56.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:56 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:56.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:56 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:56.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:56 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:56.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:56 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:56.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:56 smithi039 ceph-mon[29586]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:56.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:56 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:56.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:56 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:56.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:56 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:56.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:56 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:56.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:56 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:56.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:56 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:56.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:56 smithi039 ceph-mon[40962]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:56.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:56 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:56.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:56 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:56.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:56 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:56.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:56 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:56.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:56 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:56.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:56 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:57.419 INFO:teuthology.orchestra.run.smithi129.stdout:Scheduled mgr update... 2024-09-14T03:47:58.175 DEBUG:teuthology.orchestra.run.smithi129:mgr.x> sudo journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mgr.x.service 2024-09-14T03:47:58.177 DEBUG:tasks.cephadm:set 0 configs 2024-09-14T03:47:58.177 INFO:tasks.cephadm:Deploying OSDs... 2024-09-14T03:47:58.177 DEBUG:teuthology.orchestra.run.smithi039:> set -ex 2024-09-14T03:47:58.177 DEBUG:teuthology.orchestra.run.smithi039:> dd if=/scratch_devs of=/dev/stdout 2024-09-14T03:47:58.197 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-14T03:47:58.197 DEBUG:teuthology.orchestra.run.smithi039:> stat /dev/vg_nvme/lv_1 2024-09-14T03:47:58.256 INFO:teuthology.orchestra.run.smithi039.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-14T03:47:58.257 INFO:teuthology.orchestra.run.smithi039.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:47:58.257 INFO:teuthology.orchestra.run.smithi039.stdout:Device: 5h/5d Inode: 967 Links: 1 2024-09-14T03:47:58.257 INFO:teuthology.orchestra.run.smithi039.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:47:58.257 INFO:teuthology.orchestra.run.smithi039.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:47:58.257 INFO:teuthology.orchestra.run.smithi039.stdout:Access: 2024-09-14 03:47:43.122141852 +0000 2024-09-14T03:47:58.257 INFO:teuthology.orchestra.run.smithi039.stdout:Modify: 2024-09-14 03:43:02.104865387 +0000 2024-09-14T03:47:58.257 INFO:teuthology.orchestra.run.smithi039.stdout:Change: 2024-09-14 03:43:02.104865387 +0000 2024-09-14T03:47:58.257 INFO:teuthology.orchestra.run.smithi039.stdout: Birth: 2024-09-14 03:43:02.104865387 +0000 2024-09-14T03:47:58.257 DEBUG:teuthology.orchestra.run.smithi039:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-14T03:47:58.326 INFO:teuthology.orchestra.run.smithi039.stderr:1+0 records in 2024-09-14T03:47:58.326 INFO:teuthology.orchestra.run.smithi039.stderr:1+0 records out 2024-09-14T03:47:58.326 INFO:teuthology.orchestra.run.smithi039.stderr:512 bytes copied, 0.000220995 s, 2.3 MB/s 2024-09-14T03:47:58.327 DEBUG:teuthology.orchestra.run.smithi039:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-14T03:47:58.387 DEBUG:teuthology.orchestra.run.smithi039:> stat /dev/vg_nvme/lv_2 2024-09-14T03:47:58.445 INFO:teuthology.orchestra.run.smithi039.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-14T03:47:58.445 INFO:teuthology.orchestra.run.smithi039.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:47:58.445 INFO:teuthology.orchestra.run.smithi039.stdout:Device: 5h/5d Inode: 978 Links: 1 2024-09-14T03:47:58.445 INFO:teuthology.orchestra.run.smithi039.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:47:58.445 INFO:teuthology.orchestra.run.smithi039.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:47:58.445 INFO:teuthology.orchestra.run.smithi039.stdout:Access: 2024-09-14 03:47:43.122141852 +0000 2024-09-14T03:47:58.445 INFO:teuthology.orchestra.run.smithi039.stdout:Modify: 2024-09-14 03:43:02.106865346 +0000 2024-09-14T03:47:58.445 INFO:teuthology.orchestra.run.smithi039.stdout:Change: 2024-09-14 03:43:02.106865346 +0000 2024-09-14T03:47:58.446 INFO:teuthology.orchestra.run.smithi039.stdout: Birth: 2024-09-14 03:43:02.106865346 +0000 2024-09-14T03:47:58.446 DEBUG:teuthology.orchestra.run.smithi039:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-14T03:47:58.512 INFO:teuthology.orchestra.run.smithi039.stderr:1+0 records in 2024-09-14T03:47:58.512 INFO:teuthology.orchestra.run.smithi039.stderr:1+0 records out 2024-09-14T03:47:58.512 INFO:teuthology.orchestra.run.smithi039.stderr:512 bytes copied, 0.000304966 s, 1.7 MB/s 2024-09-14T03:47:58.514 DEBUG:teuthology.orchestra.run.smithi039:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-14T03:47:58.572 DEBUG:teuthology.orchestra.run.smithi039:> stat /dev/vg_nvme/lv_3 2024-09-14T03:47:58.630 INFO:teuthology.orchestra.run.smithi039.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-14T03:47:58.630 INFO:teuthology.orchestra.run.smithi039.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:47:58.630 INFO:teuthology.orchestra.run.smithi039.stdout:Device: 5h/5d Inode: 984 Links: 1 2024-09-14T03:47:58.630 INFO:teuthology.orchestra.run.smithi039.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:47:58.630 INFO:teuthology.orchestra.run.smithi039.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:47:58.631 INFO:teuthology.orchestra.run.smithi039.stdout:Access: 2024-09-14 03:47:43.123141832 +0000 2024-09-14T03:47:58.631 INFO:teuthology.orchestra.run.smithi039.stdout:Modify: 2024-09-14 03:43:02.107865325 +0000 2024-09-14T03:47:58.631 INFO:teuthology.orchestra.run.smithi039.stdout:Change: 2024-09-14 03:43:02.107865325 +0000 2024-09-14T03:47:58.631 INFO:teuthology.orchestra.run.smithi039.stdout: Birth: 2024-09-14 03:43:02.107865325 +0000 2024-09-14T03:47:58.631 DEBUG:teuthology.orchestra.run.smithi039:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-14T03:47:58.694 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:58 smithi039 ceph-mon[29586]: from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi039=y;smithi129=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:58.695 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:58 smithi039 ceph-mon[29586]: Saving service mgr spec with placement smithi039=y;smithi129=x;count:2 2024-09-14T03:47:58.695 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:58 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:58.695 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:58 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:58.695 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:58 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:58.695 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:58 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:58.695 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:58 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:58.695 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:58 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:47:58.695 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:58 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-14T03:47:58.695 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:58 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:47:58.695 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:58 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:58.695 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:58 smithi039 ceph-mon[29586]: Deploying daemon mgr.x on smithi129 2024-09-14T03:47:58.695 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:58 smithi039 ceph-mon[29586]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:58.696 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:58 smithi039 ceph-mon[40962]: from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi039=y;smithi129=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:58.696 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:58 smithi039 ceph-mon[40962]: Saving service mgr spec with placement smithi039=y;smithi129=x;count:2 2024-09-14T03:47:58.696 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:58 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:58.696 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:58 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:58.696 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:58 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:58.696 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:58 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:58.696 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:58 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:58.696 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:58 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:47:58.696 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:58 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-14T03:47:58.696 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:58 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:47:58.697 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:58 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:58.697 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:58 smithi039 ceph-mon[40962]: Deploying daemon mgr.x on smithi129 2024-09-14T03:47:58.697 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:58 smithi039 ceph-mon[40962]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:58.697 INFO:teuthology.orchestra.run.smithi039.stderr:1+0 records in 2024-09-14T03:47:58.697 INFO:teuthology.orchestra.run.smithi039.stderr:1+0 records out 2024-09-14T03:47:58.697 INFO:teuthology.orchestra.run.smithi039.stderr:512 bytes copied, 0.000290136 s, 1.8 MB/s 2024-09-14T03:47:58.698 DEBUG:teuthology.orchestra.run.smithi039:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-14T03:47:58.757 DEBUG:teuthology.orchestra.run.smithi039:> stat /dev/vg_nvme/lv_4 2024-09-14T03:47:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:58 smithi129 ceph-mon[33173]: from='client.24109 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi039=y;smithi129=x", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:47:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:58 smithi129 ceph-mon[33173]: Saving service mgr spec with placement smithi039=y;smithi129=x;count:2 2024-09-14T03:47:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:58 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:58 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:47:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:58 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:58 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:47:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:58 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:47:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:58 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:47:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:58 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-09-14T03:47:58.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:58 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:47:58.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:58 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:47:58.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:58 smithi129 ceph-mon[33173]: Deploying daemon mgr.x on smithi129 2024-09-14T03:47:58.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:58 smithi129 ceph-mon[33173]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:58.813 INFO:teuthology.orchestra.run.smithi039.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-14T03:47:58.813 INFO:teuthology.orchestra.run.smithi039.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:47:58.813 INFO:teuthology.orchestra.run.smithi039.stdout:Device: 5h/5d Inode: 972 Links: 1 2024-09-14T03:47:58.814 INFO:teuthology.orchestra.run.smithi039.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:47:58.814 INFO:teuthology.orchestra.run.smithi039.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:47:58.814 INFO:teuthology.orchestra.run.smithi039.stdout:Access: 2024-09-14 03:47:43.123141832 +0000 2024-09-14T03:47:58.814 INFO:teuthology.orchestra.run.smithi039.stdout:Modify: 2024-09-14 03:43:02.105865366 +0000 2024-09-14T03:47:58.814 INFO:teuthology.orchestra.run.smithi039.stdout:Change: 2024-09-14 03:43:02.105865366 +0000 2024-09-14T03:47:58.814 INFO:teuthology.orchestra.run.smithi039.stdout: Birth: 2024-09-14 03:43:02.105865366 +0000 2024-09-14T03:47:58.814 DEBUG:teuthology.orchestra.run.smithi039:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-14T03:47:58.879 INFO:teuthology.orchestra.run.smithi039.stderr:1+0 records in 2024-09-14T03:47:58.879 INFO:teuthology.orchestra.run.smithi039.stderr:1+0 records out 2024-09-14T03:47:58.879 INFO:teuthology.orchestra.run.smithi039.stderr:512 bytes copied, 0.000288014 s, 1.8 MB/s 2024-09-14T03:47:58.881 DEBUG:teuthology.orchestra.run.smithi039:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-14T03:47:58.938 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T03:47:58.938 DEBUG:teuthology.orchestra.run.smithi129:> dd if=/scratch_devs of=/dev/stdout 2024-09-14T03:47:58.955 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-09-14T03:47:58.956 DEBUG:teuthology.orchestra.run.smithi129:> stat /dev/vg_nvme/lv_1 2024-09-14T03:47:59.011 INFO:teuthology.orchestra.run.smithi129.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-09-14T03:47:59.012 INFO:teuthology.orchestra.run.smithi129.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:47:59.012 INFO:teuthology.orchestra.run.smithi129.stdout:Device: 5h/5d Inode: 970 Links: 1 2024-09-14T03:47:59.012 INFO:teuthology.orchestra.run.smithi129.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:47:59.012 INFO:teuthology.orchestra.run.smithi129.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:47:59.012 INFO:teuthology.orchestra.run.smithi129.stdout:Access: 2024-09-14 03:47:34.289769569 +0000 2024-09-14T03:47:59.012 INFO:teuthology.orchestra.run.smithi129.stdout:Modify: 2024-09-14 03:43:00.659582118 +0000 2024-09-14T03:47:59.012 INFO:teuthology.orchestra.run.smithi129.stdout:Change: 2024-09-14 03:43:00.659582118 +0000 2024-09-14T03:47:59.012 INFO:teuthology.orchestra.run.smithi129.stdout: Birth: 2024-09-14 03:43:00.659582118 +0000 2024-09-14T03:47:59.012 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-09-14T03:47:59.077 INFO:teuthology.orchestra.run.smithi129.stderr:1+0 records in 2024-09-14T03:47:59.078 INFO:teuthology.orchestra.run.smithi129.stderr:1+0 records out 2024-09-14T03:47:59.078 INFO:teuthology.orchestra.run.smithi129.stderr:512 bytes copied, 0.000232463 s, 2.2 MB/s 2024-09-14T03:47:59.079 DEBUG:teuthology.orchestra.run.smithi129:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-09-14T03:47:59.136 DEBUG:teuthology.orchestra.run.smithi129:> stat /dev/vg_nvme/lv_2 2024-09-14T03:47:59.193 INFO:teuthology.orchestra.run.smithi129.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-09-14T03:47:59.193 INFO:teuthology.orchestra.run.smithi129.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:47:59.193 INFO:teuthology.orchestra.run.smithi129.stdout:Device: 5h/5d Inode: 974 Links: 1 2024-09-14T03:47:59.193 INFO:teuthology.orchestra.run.smithi129.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:47:59.193 INFO:teuthology.orchestra.run.smithi129.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:47:59.193 INFO:teuthology.orchestra.run.smithi129.stdout:Access: 2024-09-14 03:47:34.289769569 +0000 2024-09-14T03:47:59.193 INFO:teuthology.orchestra.run.smithi129.stdout:Modify: 2024-09-14 03:43:00.660582158 +0000 2024-09-14T03:47:59.193 INFO:teuthology.orchestra.run.smithi129.stdout:Change: 2024-09-14 03:43:00.660582158 +0000 2024-09-14T03:47:59.193 INFO:teuthology.orchestra.run.smithi129.stdout: Birth: 2024-09-14 03:43:00.660582158 +0000 2024-09-14T03:47:59.194 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-09-14T03:47:59.332 INFO:teuthology.orchestra.run.smithi129.stderr:1+0 records in 2024-09-14T03:47:59.332 INFO:teuthology.orchestra.run.smithi129.stderr:1+0 records out 2024-09-14T03:47:59.332 INFO:teuthology.orchestra.run.smithi129.stderr:512 bytes copied, 0.000189953 s, 2.7 MB/s 2024-09-14T03:47:59.333 DEBUG:teuthology.orchestra.run.smithi129:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-09-14T03:47:59.349 DEBUG:teuthology.orchestra.run.smithi129:> stat /dev/vg_nvme/lv_3 2024-09-14T03:47:59.407 INFO:teuthology.orchestra.run.smithi129.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-09-14T03:47:59.408 INFO:teuthology.orchestra.run.smithi129.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:47:59.408 INFO:teuthology.orchestra.run.smithi129.stdout:Device: 5h/5d Inode: 977 Links: 1 2024-09-14T03:47:59.408 INFO:teuthology.orchestra.run.smithi129.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:47:59.408 INFO:teuthology.orchestra.run.smithi129.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:47:59.408 INFO:teuthology.orchestra.run.smithi129.stdout:Access: 2024-09-14 03:47:59.285236849 +0000 2024-09-14T03:47:59.408 INFO:teuthology.orchestra.run.smithi129.stdout:Modify: 2024-09-14 03:43:00.661582199 +0000 2024-09-14T03:47:59.408 INFO:teuthology.orchestra.run.smithi129.stdout:Change: 2024-09-14 03:43:00.661582199 +0000 2024-09-14T03:47:59.408 INFO:teuthology.orchestra.run.smithi129.stdout: Birth: 2024-09-14 03:43:00.661582199 +0000 2024-09-14T03:47:59.408 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-09-14T03:47:59.504 INFO:teuthology.orchestra.run.smithi129.stderr:1+0 records in 2024-09-14T03:47:59.505 INFO:teuthology.orchestra.run.smithi129.stderr:1+0 records out 2024-09-14T03:47:59.505 INFO:teuthology.orchestra.run.smithi129.stderr:512 bytes copied, 0.000232091 s, 2.2 MB/s 2024-09-14T03:47:59.506 DEBUG:teuthology.orchestra.run.smithi129:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-09-14T03:47:59.522 DEBUG:teuthology.orchestra.run.smithi129:> stat /dev/vg_nvme/lv_4 2024-09-14T03:47:59.579 INFO:teuthology.orchestra.run.smithi129.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-09-14T03:47:59.579 INFO:teuthology.orchestra.run.smithi129.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-09-14T03:47:59.579 INFO:teuthology.orchestra.run.smithi129.stdout:Device: 5h/5d Inode: 982 Links: 1 2024-09-14T03:47:59.579 INFO:teuthology.orchestra.run.smithi129.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-09-14T03:47:59.579 INFO:teuthology.orchestra.run.smithi129.stdout:Context: system_u:object_r:device_t:s0 2024-09-14T03:47:59.579 INFO:teuthology.orchestra.run.smithi129.stdout:Access: 2024-09-14 03:47:59.460233120 +0000 2024-09-14T03:47:59.579 INFO:teuthology.orchestra.run.smithi129.stdout:Modify: 2024-09-14 03:43:00.663582281 +0000 2024-09-14T03:47:59.580 INFO:teuthology.orchestra.run.smithi129.stdout:Change: 2024-09-14 03:43:00.663582281 +0000 2024-09-14T03:47:59.580 INFO:teuthology.orchestra.run.smithi129.stdout: Birth: 2024-09-14 03:43:00.663582281 +0000 2024-09-14T03:47:59.580 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-09-14T03:47:59.645 INFO:teuthology.orchestra.run.smithi129.stderr:1+0 records in 2024-09-14T03:47:59.645 INFO:teuthology.orchestra.run.smithi129.stderr:1+0 records out 2024-09-14T03:47:59.646 INFO:teuthology.orchestra.run.smithi129.stderr:512 bytes copied, 0.000191262 s, 2.7 MB/s 2024-09-14T03:47:59.647 DEBUG:teuthology.orchestra.run.smithi129:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-09-14T03:47:59.703 INFO:tasks.cephadm:Deploying osd.0 on smithi039 with /dev/vg_nvme/lv_4... 2024-09-14T03:47:59.703 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-14T03:47:59.728 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:47:59 smithi129 ceph-mon[33173]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:47:59.916 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:48:00.021 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:47:59 smithi129 podman[35209]: 2024-09-14 03:47:59.787374999 +0000 UTC m=+0.173023118 container create 813f62a05ea8d3b352c3dce63048b3a48e33f627fa1418e82d5b58549d2c86c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, RELEASE=reef-e4e227b) 2024-09-14T03:48:00.037 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:47:59 smithi039 ceph-mon[29586]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:00.038 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:47:59 smithi039 ceph-mon[40962]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:00.370 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:48:00 smithi129 podman[35209]: 2024-09-14 03:48:00.03080916 +0000 UTC m=+0.416457279 container init 813f62a05ea8d3b352c3dce63048b3a48e33f627fa1418e82d5b58549d2c86c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, RELEASE=reef-e4e227b, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-14T03:48:00.370 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:48:00 smithi129 podman[35209]: 2024-09-14 03:48:00.035853018 +0000 UTC m=+0.421501133 container start 813f62a05ea8d3b352c3dce63048b3a48e33f627fa1418e82d5b58549d2c86c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-e4e227b, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-14T03:48:00.370 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:48:00 smithi129 ceph-mgr[35285]: -- 172.21.15.129:0/2102916019 <== mon.1 v2:172.21.15.129:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ac6f0eed00 con 0x55ac6f0c3400 2024-09-14T03:48:00.370 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:48:00 smithi129 bash[35209]: 813f62a05ea8d3b352c3dce63048b3a48e33f627fa1418e82d5b58549d2c86c3 2024-09-14T03:48:00.371 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:48:00 smithi129 systemd[1]: Started Ceph mgr.x for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:48:00.371 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:48:00 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:48:00.154+0000 7fa0c5321040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-14T03:48:00.371 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:48:00 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:48:00.214+0000 7fa0c5321040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-14T03:48:00.371 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:48:00 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:48:00.368+0000 7fa0c5321040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-14T03:48:00.755 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:48:00 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:48:00.437+0000 7fa0c5321040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-14T03:48:00.755 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:48:00 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:48:00.605+0000 7fa0c5321040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:48:00.755 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:48:00 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:48:00.753+0000 7fa0c5321040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-14T03:48:01.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:01 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:01.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:01 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:01.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:01 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:01.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:01 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:01.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:01 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:01.272 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:48:01 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:48:01.082+0000 7fa0c5321040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-14T03:48:01.272 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:48:01 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:48:01.142+0000 7fa0c5321040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-14T03:48:01.365 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:01 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:01.365 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:01 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:01.365 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:01 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:01.365 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:01 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:01.365 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:01 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:01.366 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:01 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:01.366 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:01 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:01.366 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:01 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:01.366 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:01 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:01.366 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:01 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:01.771 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:48:01 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:48:01.332+0000 7fa0c5321040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-14T03:48:02.477 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:02 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:02.477 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:02 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:02.477 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:02 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:02.477 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:02 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:48:02.477 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:02 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:02.477 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:02 smithi039 ceph-mon[40962]: Reconfiguring mgr.y (unknown last config time)... 2024-09-14T03:48:02.477 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:02 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:48:02.477 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:02 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:48:02.477 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:02 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:02.477 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:02 smithi039 ceph-mon[40962]: Reconfiguring daemon mgr.y on smithi039 2024-09-14T03:48:02.477 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:02 smithi039 ceph-mon[40962]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:02.478 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:02 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:02.478 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:02 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:02.478 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:02 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:02.478 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:02 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:48:02.478 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:02 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:02.478 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:02 smithi039 ceph-mon[29586]: Reconfiguring mgr.y (unknown last config time)... 2024-09-14T03:48:02.478 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:02 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:48:02.478 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:02 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:48:02.478 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:02 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:02.478 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:02 smithi039 ceph-mon[29586]: Reconfiguring daemon mgr.y on smithi039 2024-09-14T03:48:02.478 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:02 smithi039 ceph-mon[29586]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:02.480 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:48:02.501 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi039:vg_nvme/lv_4 2024-09-14T03:48:02.521 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:48:02 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:48:02.192+0000 7fa0c5321040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-14T03:48:02.521 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:48:02 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:48:02.256+0000 7fa0c5321040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-14T03:48:02.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:02 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:02.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:02 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:02.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:02 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:02.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:02 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:48:02.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:02 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:02.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:02 smithi129 ceph-mon[33173]: Reconfiguring mgr.y (unknown last config time)... 2024-09-14T03:48:02.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:02 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:48:02.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:02 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:48:02.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:02 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:02.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:02 smithi129 ceph-mon[33173]: Reconfiguring daemon mgr.y on smithi039 2024-09-14T03:48:02.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:02 smithi129 ceph-mon[33173]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:03.272 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:48:02 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:48:02.962+0000 7fa0c5321040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-14T03:48:03.272 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:48:03 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:48:03.075+0000 7fa0c5321040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-14T03:48:03.272 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:48:03 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:48:03.212+0000 7fa0c5321040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:48:03.541 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:48:03 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:48:03.271+0000 7fa0c5321040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-14T03:48:03.541 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:48:03 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:48:03.415+0000 7fa0c5321040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-14T03:48:03.541 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:48:03 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:48:03.481+0000 7fa0c5321040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-14T03:48:03.541 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:48:03 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:48:03.539+0000 7fa0c5321040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-14T03:48:03.672 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:48:03.825 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:03 smithi129 ceph-mon[33173]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:03.825 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:48:03 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:48:03.694+0000 7fa0c5321040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-14T03:48:03.825 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:48:03 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:48:03.758+0000 7fa0c5321040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-14T03:48:03.825 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:48:03 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:48:03.823+0000 7fa0c5321040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-14T03:48:03.921 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:03 smithi039 ceph-mon[29586]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:03.921 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:03 smithi039 ceph-mon[40962]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:04.271 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:48:03 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:48:03.904+0000 7fa0c5321040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-14T03:48:04.271 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:48:03 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:48:03.961+0000 7fa0c5321040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-14T03:48:04.271 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:48:04 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:48:04.019+0000 7fa0c5321040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-14T03:48:04.888 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:04 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:04.889 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:04 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:04.889 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:04 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:04.889 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:04 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:04.889 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:04 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:48:04.889 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:04 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:04.889 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:04 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:04.889 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:04 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:04.889 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:04 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:04.889 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:04 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:04.889 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:04 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:04.890 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:04 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:48:04.890 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:04 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:04.890 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:04 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:05.021 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:48:04 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:48:04.787+0000 7fa0c5321040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-14T03:48:05.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:04 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:05.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:04 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:05.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:04 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:05.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:04 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:05.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:04 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:48:05.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:04 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:05.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:04 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:06.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:05 smithi129 ceph-mon[33173]: Standby manager daemon x started 2024-09-14T03:48:06.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:05 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.129:0/3380957397' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:48:06.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:05 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.129:0/3380957397' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:48:06.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:05 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.129:0/3380957397' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:48:06.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:05 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.129:0/3380957397' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:48:06.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:05 smithi129 ceph-mon[33173]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:06.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:05 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:48:06.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:05 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:48:06.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:05 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:06.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:05 smithi039 ceph-mon[29586]: Standby manager daemon x started 2024-09-14T03:48:06.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:05 smithi039 ceph-mon[29586]: from='mgr.? 172.21.15.129:0/3380957397' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:48:06.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:05 smithi039 ceph-mon[29586]: from='mgr.? 172.21.15.129:0/3380957397' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:48:06.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:05 smithi039 ceph-mon[29586]: from='mgr.? 172.21.15.129:0/3380957397' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:48:06.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:05 smithi039 ceph-mon[29586]: from='mgr.? 172.21.15.129:0/3380957397' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:48:06.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:05 smithi039 ceph-mon[29586]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:06.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:05 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:48:06.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:05 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:48:06.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:05 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:06.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:05 smithi039 ceph-mon[40962]: Standby manager daemon x started 2024-09-14T03:48:06.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:05 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.129:0/3380957397' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:48:06.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:05 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.129:0/3380957397' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:48:06.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:05 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.129:0/3380957397' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:48:06.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:05 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.129:0/3380957397' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:48:06.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:05 smithi039 ceph-mon[40962]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:06.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:05 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:48:06.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:05 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:48:06.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:05 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:06.981 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:06 smithi039 ceph-mon[29586]: from='client.14211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi039:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:06.981 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:06 smithi039 ceph-mon[29586]: mgrmap e14: y(active, since 66s), standbys: x 2024-09-14T03:48:06.981 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:06 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:48:06.982 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:06 smithi039 ceph-mon[40962]: from='client.14211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi039:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:06.982 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:06 smithi039 ceph-mon[40962]: mgrmap e14: y(active, since 66s), standbys: x 2024-09-14T03:48:06.982 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:06 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:48:07.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:06 smithi129 ceph-mon[33173]: from='client.14211 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi039:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:07.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:06 smithi129 ceph-mon[33173]: mgrmap e14: y(active, since 66s), standbys: x 2024-09-14T03:48:07.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:06 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:48:08.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:07 smithi039 ceph-mon[29586]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:08.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:07 smithi039 ceph-mon[40962]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:08.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:07 smithi129 ceph-mon[33173]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:09.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:08 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/4167488296' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5fdfd478-dd27-4674-a7d6-53177d9e549a"}]: dispatch 2024-09-14T03:48:09.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:08 smithi039 ceph-mon[29586]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5fdfd478-dd27-4674-a7d6-53177d9e549a"}]: dispatch 2024-09-14T03:48:09.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:08 smithi039 ceph-mon[29586]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5fdfd478-dd27-4674-a7d6-53177d9e549a"}]': finished 2024-09-14T03:48:09.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:08 smithi039 ceph-mon[29586]: osdmap e5: 1 total, 0 up, 1 in 2024-09-14T03:48:09.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:08 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:48:09.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:08 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/3903972226' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:48:09.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:08 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/4167488296' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5fdfd478-dd27-4674-a7d6-53177d9e549a"}]: dispatch 2024-09-14T03:48:09.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:08 smithi039 ceph-mon[40962]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5fdfd478-dd27-4674-a7d6-53177d9e549a"}]: dispatch 2024-09-14T03:48:09.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:08 smithi039 ceph-mon[40962]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5fdfd478-dd27-4674-a7d6-53177d9e549a"}]': finished 2024-09-14T03:48:09.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:08 smithi039 ceph-mon[40962]: osdmap e5: 1 total, 0 up, 1 in 2024-09-14T03:48:09.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:08 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:48:09.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:08 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/3903972226' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:48:09.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:08 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/4167488296' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5fdfd478-dd27-4674-a7d6-53177d9e549a"}]: dispatch 2024-09-14T03:48:09.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:08 smithi129 ceph-mon[33173]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "5fdfd478-dd27-4674-a7d6-53177d9e549a"}]: dispatch 2024-09-14T03:48:09.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:08 smithi129 ceph-mon[33173]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "5fdfd478-dd27-4674-a7d6-53177d9e549a"}]': finished 2024-09-14T03:48:09.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:08 smithi129 ceph-mon[33173]: osdmap e5: 1 total, 0 up, 1 in 2024-09-14T03:48:09.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:08 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:48:09.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:08 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/3903972226' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:48:10.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:09 smithi039 ceph-mon[29586]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:10.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:09 smithi039 ceph-mon[40962]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:10.172 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:09 smithi129 ceph-mon[33173]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:11 smithi129 ceph-mon[33173]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:11.878 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:11 smithi039 ceph-mon[29586]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:11.878 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:11 smithi039 ceph-mon[40962]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:13.775 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:13 smithi039 ceph-mon[29586]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:13.776 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:13 smithi039 ceph-mon[40962]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:14.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:13 smithi129 ceph-mon[33173]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:14.745 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:14 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-14T03:48:14.745 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:14 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:14.745 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:14 smithi039 ceph-mon[40962]: Deploying daemon osd.0 on smithi039 2024-09-14T03:48:14.746 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:14 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-14T03:48:14.746 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:14 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:14.746 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:14 smithi039 ceph-mon[29586]: Deploying daemon osd.0 on smithi039 2024-09-14T03:48:15.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:14 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-09-14T03:48:15.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:14 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:15.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:14 smithi129 ceph-mon[33173]: Deploying daemon osd.0 on smithi039 2024-09-14T03:48:15.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:15 smithi039 ceph-mon[29586]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:15.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:15 smithi039 ceph-mon[40962]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:16.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:15 smithi129 ceph-mon[33173]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:17.776 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:17 smithi039 ceph-mon[29586]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:17.777 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:17 smithi039 ceph-mon[40962]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:18.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:17 smithi129 ceph-mon[33173]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:19.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:19 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:19.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:19 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:19.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:19 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:19.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:19 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:19.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:19 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:19.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:19 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:19.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:19 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:19.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:19 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:19.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:19 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:20.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:20 smithi039 ceph-mon[29586]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:20.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:20 smithi039 ceph-mon[40962]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:20.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:20 smithi129 ceph-mon[33173]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:21.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:21 smithi129 ceph-mon[33173]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:22.030 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:21 smithi039 ceph-mon[29586]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:22.030 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:21 smithi039 ceph-mon[40962]: pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:22.680 INFO:teuthology.orchestra.run.smithi039.stdout:Created osd(s) 0 on host 'smithi039' 2024-09-14T03:48:23.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:22 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:23.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:22 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:23.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:22 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:23.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:22 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:48:23.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:22 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:23.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:22 smithi129 ceph-mon[33173]: from='osd.0 [v2:172.21.15.39:6802/1775094854,v1:172.21.15.39:6803/1775094854]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-14T03:48:23.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:22 smithi129 ceph-mon[33173]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-14T03:48:23.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:22 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:23.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:22 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:23.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:22 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:22 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:22 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:22 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:22 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:48:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:22 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:22 smithi039 ceph-mon[29586]: from='osd.0 [v2:172.21.15.39:6802/1775094854,v1:172.21.15.39:6803/1775094854]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-14T03:48:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:22 smithi039 ceph-mon[29586]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-14T03:48:23.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:22 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:23.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:22 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:23.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:22 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:23.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:22 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:23.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:22 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:23.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:22 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:23.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:22 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:48:23.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:22 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:23.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:22 smithi039 ceph-mon[40962]: from='osd.0 [v2:172.21.15.39:6802/1775094854,v1:172.21.15.39:6803/1775094854]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-14T03:48:23.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:22 smithi039 ceph-mon[40962]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-09-14T03:48:23.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:22 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:23.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:22 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:23.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:22 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:23.367 DEBUG:teuthology.orchestra.run.smithi039:osd.0> sudo journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.0.service 2024-09-14T03:48:23.369 INFO:tasks.cephadm:Deploying osd.1 on smithi039 with /dev/vg_nvme/lv_3... 2024-09-14T03:48:23.369 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-14T03:48:24.157 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:48:24.268 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:23 smithi039 ceph-mon[40962]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-14T03:48:24.268 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:23 smithi039 ceph-mon[40962]: osdmap e6: 1 total, 0 up, 1 in 2024-09-14T03:48:24.268 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:23 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:48:24.268 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:23 smithi039 ceph-mon[40962]: from='osd.0 [v2:172.21.15.39:6802/1775094854,v1:172.21.15.39:6803/1775094854]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]: dispatch 2024-09-14T03:48:24.268 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:23 smithi039 ceph-mon[40962]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]: dispatch 2024-09-14T03:48:24.268 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:23 smithi039 ceph-mon[40962]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:24.268 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 03:48:23 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0[45363]: 2024-09-14T03:48:23.964+0000 7f6a34f55640 -1 osd.0 0 waiting for initial osdmap 2024-09-14T03:48:24.269 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 03:48:23 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0[45363]: 2024-09-14T03:48:23.966+0000 7f6a30d7e640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:48:24.269 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:23 smithi039 ceph-mon[29586]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-14T03:48:24.269 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:23 smithi039 ceph-mon[29586]: osdmap e6: 1 total, 0 up, 1 in 2024-09-14T03:48:24.269 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:23 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:48:24.269 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:23 smithi039 ceph-mon[29586]: from='osd.0 [v2:172.21.15.39:6802/1775094854,v1:172.21.15.39:6803/1775094854]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]: dispatch 2024-09-14T03:48:24.269 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:23 smithi039 ceph-mon[29586]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]: dispatch 2024-09-14T03:48:24.269 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:23 smithi039 ceph-mon[29586]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:24.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:23 smithi129 ceph-mon[33173]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-14T03:48:24.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:23 smithi129 ceph-mon[33173]: osdmap e6: 1 total, 0 up, 1 in 2024-09-14T03:48:24.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:23 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:48:24.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:23 smithi129 ceph-mon[33173]: from='osd.0 [v2:172.21.15.39:6802/1775094854,v1:172.21.15.39:6803/1775094854]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]: dispatch 2024-09-14T03:48:24.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:23 smithi129 ceph-mon[33173]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]: dispatch 2024-09-14T03:48:24.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:23 smithi129 ceph-mon[33173]: pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:25.160 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:24 smithi039 ceph-mon[29586]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]': finished 2024-09-14T03:48:25.160 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:24 smithi039 ceph-mon[29586]: osdmap e7: 1 total, 0 up, 1 in 2024-09-14T03:48:25.160 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:24 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:48:25.160 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:24 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:48:25.161 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:24 smithi039 ceph-mon[40962]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]': finished 2024-09-14T03:48:25.161 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:24 smithi039 ceph-mon[40962]: osdmap e7: 1 total, 0 up, 1 in 2024-09-14T03:48:25.161 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:24 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:48:25.161 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:24 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:48:25.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:24 smithi129 ceph-mon[33173]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]': finished 2024-09-14T03:48:25.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:24 smithi129 ceph-mon[33173]: osdmap e7: 1 total, 0 up, 1 in 2024-09-14T03:48:25.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:24 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:48:25.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:24 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:48:26.246 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:25 smithi039 ceph-mon[29586]: purged_snaps scrub starts 2024-09-14T03:48:26.246 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:25 smithi039 ceph-mon[29586]: purged_snaps scrub ok 2024-09-14T03:48:26.246 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:25 smithi039 ceph-mon[29586]: osd.0 [v2:172.21.15.39:6802/1775094854,v1:172.21.15.39:6803/1775094854] boot 2024-09-14T03:48:26.246 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:25 smithi039 ceph-mon[29586]: osdmap e8: 1 total, 1 up, 1 in 2024-09-14T03:48:26.246 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:25 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:48:26.246 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:25 smithi039 ceph-mon[29586]: pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:26.246 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:25 smithi039 ceph-mon[40962]: purged_snaps scrub starts 2024-09-14T03:48:26.246 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:25 smithi039 ceph-mon[40962]: purged_snaps scrub ok 2024-09-14T03:48:26.246 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:25 smithi039 ceph-mon[40962]: osd.0 [v2:172.21.15.39:6802/1775094854,v1:172.21.15.39:6803/1775094854] boot 2024-09-14T03:48:26.247 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:25 smithi039 ceph-mon[40962]: osdmap e8: 1 total, 1 up, 1 in 2024-09-14T03:48:26.247 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:25 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:48:26.247 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:25 smithi039 ceph-mon[40962]: pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:26.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:25 smithi129 ceph-mon[33173]: purged_snaps scrub starts 2024-09-14T03:48:26.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:25 smithi129 ceph-mon[33173]: purged_snaps scrub ok 2024-09-14T03:48:26.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:25 smithi129 ceph-mon[33173]: osd.0 [v2:172.21.15.39:6802/1775094854,v1:172.21.15.39:6803/1775094854] boot 2024-09-14T03:48:26.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:25 smithi129 ceph-mon[33173]: osdmap e8: 1 total, 1 up, 1 in 2024-09-14T03:48:26.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:25 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:48:26.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:25 smithi129 ceph-mon[33173]: pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-09-14T03:48:27.184 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:26 smithi039 ceph-mon[29586]: osdmap e9: 1 total, 1 up, 1 in 2024-09-14T03:48:27.184 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:26 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:27.184 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:26 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:27.184 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:26 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:48:27.184 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:26 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:27.184 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:26 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:27.184 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:26 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:48:27.185 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:26 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:27.185 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:26 smithi039 ceph-mon[40962]: osdmap e9: 1 total, 1 up, 1 in 2024-09-14T03:48:27.185 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:26 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:27.185 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:26 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:27.185 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:26 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:48:27.185 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:26 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:27.185 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:26 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:27.185 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:26 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:48:27.185 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:26 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:27.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:26 smithi129 ceph-mon[33173]: osdmap e9: 1 total, 1 up, 1 in 2024-09-14T03:48:27.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:26 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:27.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:26 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:27.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:26 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:48:27.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:26 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:27.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:26 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:27.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:26 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:48:27.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:26 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:28.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:27 smithi129 ceph-mon[33173]: Detected new or changed devices on smithi039 2024-09-14T03:48:28.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:27 smithi129 ceph-mon[33173]: Adjusting osd_memory_target on smithi039 to 16064M 2024-09-14T03:48:28.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:27 smithi129 ceph-mon[33173]: pgmap v43: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:28.280 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:27 smithi039 ceph-mon[29586]: Detected new or changed devices on smithi039 2024-09-14T03:48:28.280 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:27 smithi039 ceph-mon[29586]: Adjusting osd_memory_target on smithi039 to 16064M 2024-09-14T03:48:28.280 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:27 smithi039 ceph-mon[29586]: pgmap v43: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:28.280 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:27 smithi039 ceph-mon[40962]: Detected new or changed devices on smithi039 2024-09-14T03:48:28.281 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:27 smithi039 ceph-mon[40962]: Adjusting osd_memory_target on smithi039 to 16064M 2024-09-14T03:48:28.281 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:27 smithi039 ceph-mon[40962]: pgmap v43: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:28.284 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:48:28.305 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi039:vg_nvme/lv_3 2024-09-14T03:48:28.513 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:48:29.670 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:29 smithi039 ceph-mon[29586]: pgmap v44: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:29.670 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:29 smithi039 ceph-mon[40962]: pgmap v44: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:30.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:29 smithi129 ceph-mon[33173]: pgmap v44: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:30.955 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:30 smithi039 ceph-mon[29586]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi039:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:30.955 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:30 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:48:30.955 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:30 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:48:30.955 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:30 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:30.956 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:30 smithi039 ceph-mon[40962]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi039:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:30.956 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:30 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:48:30.956 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:30 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:48:30.956 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:30 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:31.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:30 smithi129 ceph-mon[33173]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi039:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:31.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:30 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:48:31.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:30 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:48:31.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:30 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:31.846 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:31 smithi039 ceph-mon[29586]: pgmap v45: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:31.846 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:31 smithi039 ceph-mon[40962]: pgmap v45: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:32.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:31 smithi129 ceph-mon[33173]: pgmap v45: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:33.089 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:32 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/2087029611' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "991c236e-1679-4c1e-a1a9-3894b1b12055"}]: dispatch 2024-09-14T03:48:33.089 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:32 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/2087029611' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "991c236e-1679-4c1e-a1a9-3894b1b12055"}]': finished 2024-09-14T03:48:33.089 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:32 smithi039 ceph-mon[40962]: osdmap e10: 2 total, 1 up, 2 in 2024-09-14T03:48:33.089 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:32 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:48:33.089 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:32 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/2087029611' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "991c236e-1679-4c1e-a1a9-3894b1b12055"}]: dispatch 2024-09-14T03:48:33.089 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:32 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/2087029611' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "991c236e-1679-4c1e-a1a9-3894b1b12055"}]': finished 2024-09-14T03:48:33.089 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:32 smithi039 ceph-mon[29586]: osdmap e10: 2 total, 1 up, 2 in 2024-09-14T03:48:33.089 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:32 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:48:33.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:32 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/2087029611' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "991c236e-1679-4c1e-a1a9-3894b1b12055"}]: dispatch 2024-09-14T03:48:33.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:32 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/2087029611' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "991c236e-1679-4c1e-a1a9-3894b1b12055"}]': finished 2024-09-14T03:48:33.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:32 smithi129 ceph-mon[33173]: osdmap e10: 2 total, 1 up, 2 in 2024-09-14T03:48:33.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:32 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:48:34.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:33 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/1029016423' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:48:34.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:33 smithi129 ceph-mon[33173]: pgmap v47: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:34.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:33 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/1029016423' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:48:34.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:33 smithi039 ceph-mon[29586]: pgmap v47: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:34.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:33 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/1029016423' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:48:34.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:33 smithi039 ceph-mon[40962]: pgmap v47: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:36.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:35 smithi129 ceph-mon[33173]: pgmap v48: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:36.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:35 smithi039 ceph-mon[29586]: pgmap v48: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:36.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:35 smithi039 ceph-mon[40962]: pgmap v48: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:37.926 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:37 smithi039 ceph-mon[29586]: pgmap v49: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:37.926 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:37 smithi039 ceph-mon[40962]: pgmap v49: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:37 smithi129 ceph-mon[33173]: pgmap v49: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:39.712 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:39 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-14T03:48:39.712 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:39 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:39.712 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:39 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-14T03:48:39.712 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:39 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:39.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:39 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-09-14T03:48:39.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:39 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:40.672 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:40 smithi129 ceph-mon[33173]: Deploying daemon osd.1 on smithi039 2024-09-14T03:48:40.673 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:40 smithi129 ceph-mon[33173]: pgmap v50: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:40.705 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:40 smithi039 ceph-mon[29586]: Deploying daemon osd.1 on smithi039 2024-09-14T03:48:40.705 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:40 smithi039 ceph-mon[29586]: pgmap v50: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:40.705 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:40 smithi039 ceph-mon[40962]: Deploying daemon osd.1 on smithi039 2024-09-14T03:48:40.705 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:40 smithi039 ceph-mon[40962]: pgmap v50: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:41.778 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:41 smithi039 ceph-mon[29586]: pgmap v51: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:41.778 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:41 smithi039 ceph-mon[40962]: pgmap v51: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:42.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:41 smithi129 ceph-mon[33173]: pgmap v51: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:43.994 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:43 smithi039 ceph-mon[29586]: pgmap v52: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:43.995 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:43 smithi039 ceph-mon[40962]: pgmap v52: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:44.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:43 smithi129 ceph-mon[33173]: pgmap v52: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:44.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:44 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:44.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:44 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:44.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:44 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:44.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:44 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:45.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:44 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:45.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:44 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:46.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:45 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:46.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:45 smithi129 ceph-mon[33173]: pgmap v53: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:46.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:45 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:46.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:45 smithi039 ceph-mon[29586]: pgmap v53: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:46.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:45 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:46.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:45 smithi039 ceph-mon[40962]: pgmap v53: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:47.998 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:47 smithi039 ceph-mon[29586]: pgmap v54: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:47.998 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:47 smithi039 ceph-mon[40962]: pgmap v54: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:48.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:47 smithi129 ceph-mon[33173]: pgmap v54: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:48.033 INFO:teuthology.orchestra.run.smithi039.stdout:Created osd(s) 1 on host 'smithi039' 2024-09-14T03:48:48.659 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:48 smithi039 ceph-mon[40962]: from='osd.1 [v2:172.21.15.39:6810/2410143394,v1:172.21.15.39:6811/2410143394]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-14T03:48:48.660 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:48 smithi039 ceph-mon[40962]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-14T03:48:48.660 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:48 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:48.660 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:48 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:48.660 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:48 smithi039 ceph-mon[29586]: from='osd.1 [v2:172.21.15.39:6810/2410143394,v1:172.21.15.39:6811/2410143394]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-14T03:48:48.660 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:48 smithi039 ceph-mon[29586]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-14T03:48:48.660 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:48 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:48.660 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:48 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:48.682 DEBUG:teuthology.orchestra.run.smithi039:osd.1> sudo journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.1.service 2024-09-14T03:48:48.684 INFO:tasks.cephadm:Deploying osd.2 on smithi039 with /dev/vg_nvme/lv_2... 2024-09-14T03:48:48.685 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-14T03:48:48.930 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:48:49.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:48 smithi129 ceph-mon[33173]: from='osd.1 [v2:172.21.15.39:6810/2410143394,v1:172.21.15.39:6811/2410143394]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-14T03:48:49.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:48 smithi129 ceph-mon[33173]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-09-14T03:48:49.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:48 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:49.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:48 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:49.927 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:49 smithi039 ceph-mon[29586]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-14T03:48:49.927 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:49 smithi039 ceph-mon[29586]: osdmap e11: 2 total, 1 up, 2 in 2024-09-14T03:48:49.927 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:49 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:48:49.927 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:49 smithi039 ceph-mon[29586]: from='osd.1 [v2:172.21.15.39:6810/2410143394,v1:172.21.15.39:6811/2410143394]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]: dispatch 2024-09-14T03:48:49.927 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:49 smithi039 ceph-mon[29586]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]: dispatch 2024-09-14T03:48:49.927 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:49 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:49.927 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:49 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:49.928 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:49 smithi039 ceph-mon[29586]: pgmap v56: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:49.928 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:49 smithi039 ceph-mon[40962]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-14T03:48:49.928 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:49 smithi039 ceph-mon[40962]: osdmap e11: 2 total, 1 up, 2 in 2024-09-14T03:48:49.928 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:49 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:48:49.928 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:49 smithi039 ceph-mon[40962]: from='osd.1 [v2:172.21.15.39:6810/2410143394,v1:172.21.15.39:6811/2410143394]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]: dispatch 2024-09-14T03:48:49.928 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:49 smithi039 ceph-mon[40962]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]: dispatch 2024-09-14T03:48:49.928 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:49 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:49.928 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:49 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:49.928 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:49 smithi039 ceph-mon[40962]: pgmap v56: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:49.929 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 03:48:49 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1[51249]: 2024-09-14T03:48:49.651+0000 7f2a82c7b640 -1 osd.1 0 waiting for initial osdmap 2024-09-14T03:48:49.929 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 03:48:49 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1[51249]: 2024-09-14T03:48:49.654+0000 7f2a7eaa4640 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:48:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:49 smithi129 ceph-mon[33173]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-14T03:48:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:49 smithi129 ceph-mon[33173]: osdmap e11: 2 total, 1 up, 2 in 2024-09-14T03:48:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:49 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:48:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:49 smithi129 ceph-mon[33173]: from='osd.1 [v2:172.21.15.39:6810/2410143394,v1:172.21.15.39:6811/2410143394]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]: dispatch 2024-09-14T03:48:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:49 smithi129 ceph-mon[33173]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]: dispatch 2024-09-14T03:48:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:49 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:50.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:49 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:50.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:49 smithi129 ceph-mon[33173]: pgmap v56: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:50.917 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:50 smithi039 ceph-mon[40962]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]': finished 2024-09-14T03:48:50.917 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:50 smithi039 ceph-mon[40962]: osdmap e12: 2 total, 1 up, 2 in 2024-09-14T03:48:50.917 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:50 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:48:50.917 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:50 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:48:50.918 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:50 smithi039 ceph-mon[29586]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]': finished 2024-09-14T03:48:50.918 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:50 smithi039 ceph-mon[29586]: osdmap e12: 2 total, 1 up, 2 in 2024-09-14T03:48:50.918 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:50 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:48:50.918 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:50 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:48:51.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:50 smithi129 ceph-mon[33173]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]': finished 2024-09-14T03:48:51.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:50 smithi129 ceph-mon[33173]: osdmap e12: 2 total, 1 up, 2 in 2024-09-14T03:48:51.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:50 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:48:51.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:50 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:48:51.729 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:51 smithi039 ceph-mon[40962]: purged_snaps scrub starts 2024-09-14T03:48:51.729 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:51 smithi039 ceph-mon[40962]: purged_snaps scrub ok 2024-09-14T03:48:51.729 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:51 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:48:51.729 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:51 smithi039 ceph-mon[40962]: osd.1 [v2:172.21.15.39:6810/2410143394,v1:172.21.15.39:6811/2410143394] boot 2024-09-14T03:48:51.729 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:51 smithi039 ceph-mon[40962]: osdmap e13: 2 total, 2 up, 2 in 2024-09-14T03:48:51.729 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:51 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:48:51.730 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:51 smithi039 ceph-mon[40962]: pgmap v59: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:51.730 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:51 smithi039 ceph-mon[29586]: purged_snaps scrub starts 2024-09-14T03:48:51.730 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:51 smithi039 ceph-mon[29586]: purged_snaps scrub ok 2024-09-14T03:48:51.730 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:51 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:48:51.730 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:51 smithi039 ceph-mon[29586]: osd.1 [v2:172.21.15.39:6810/2410143394,v1:172.21.15.39:6811/2410143394] boot 2024-09-14T03:48:51.730 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:51 smithi039 ceph-mon[29586]: osdmap e13: 2 total, 2 up, 2 in 2024-09-14T03:48:51.730 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:51 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:48:51.730 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:51 smithi039 ceph-mon[29586]: pgmap v59: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:52.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:51 smithi129 ceph-mon[33173]: purged_snaps scrub starts 2024-09-14T03:48:52.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:51 smithi129 ceph-mon[33173]: purged_snaps scrub ok 2024-09-14T03:48:52.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:51 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:48:52.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:51 smithi129 ceph-mon[33173]: osd.1 [v2:172.21.15.39:6810/2410143394,v1:172.21.15.39:6811/2410143394] boot 2024-09-14T03:48:52.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:51 smithi129 ceph-mon[33173]: osdmap e13: 2 total, 2 up, 2 in 2024-09-14T03:48:52.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:51 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:48:52.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:51 smithi129 ceph-mon[33173]: pgmap v59: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-09-14T03:48:52.621 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:48:52.640 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi039:vg_nvme/lv_2 2024-09-14T03:48:52.952 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:52 smithi039 ceph-mon[29586]: osdmap e14: 2 total, 2 up, 2 in 2024-09-14T03:48:52.952 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:52 smithi039 ceph-mon[40962]: osdmap e14: 2 total, 2 up, 2 in 2024-09-14T03:48:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:52 smithi129 ceph-mon[33173]: osdmap e14: 2 total, 2 up, 2 in 2024-09-14T03:48:53.212 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:48:53.979 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:53 smithi039 ceph-mon[29586]: pgmap v61: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:48:53.979 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:53 smithi039 ceph-mon[40962]: pgmap v61: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:48:54.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:53 smithi129 ceph-mon[33173]: pgmap v61: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:48:56.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:55 smithi129 ceph-mon[33173]: Detected new or changed devices on smithi039 2024-09-14T03:48:56.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:56.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:56.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:48:56.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:48:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:55 smithi129 ceph-mon[33173]: Adjusting osd_memory_target on smithi039 to 8032M 2024-09-14T03:48:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:48:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:48:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:55 smithi129 ceph-mon[33173]: pgmap v62: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:48:56.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:55 smithi039 ceph-mon[29586]: Detected new or changed devices on smithi039 2024-09-14T03:48:56.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:56.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:56.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:48:56.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:48:56.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:55 smithi039 ceph-mon[29586]: Adjusting osd_memory_target on smithi039 to 8032M 2024-09-14T03:48:56.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:56.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:56.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:48:56.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:56.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:56.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:56.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:48:56.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:56.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:55 smithi039 ceph-mon[29586]: pgmap v62: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:48:56.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:55 smithi039 ceph-mon[40962]: Detected new or changed devices on smithi039 2024-09-14T03:48:56.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:56.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:56.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:48:56.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:48:56.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:55 smithi039 ceph-mon[40962]: Adjusting osd_memory_target on smithi039 to 8032M 2024-09-14T03:48:56.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:56.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:56.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:48:56.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:56.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:48:56.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:56.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:48:56.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:48:56.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:55 smithi039 ceph-mon[40962]: pgmap v62: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:48:56.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:56 smithi039 ceph-mon[29586]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi039:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:56.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:56 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:48:56.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:56 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:48:56.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:56 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:56.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:56 smithi039 ceph-mon[40962]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi039:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:56.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:56 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:48:56.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:56 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:48:56.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:56 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:57.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:56 smithi129 ceph-mon[33173]: from='client.24170 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi039:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:48:57.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:56 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:48:57.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:56 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:48:57.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:56 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:48:57.769 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:57 smithi039 ceph-mon[29586]: pgmap v63: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:48:57.770 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:57 smithi039 ceph-mon[40962]: pgmap v63: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:48:58.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:57 smithi129 ceph-mon[33173]: pgmap v63: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:48:59.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:58 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/4272384867' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f9611ffd-5d37-4465-8114-ecc70d4d6d14"}]: dispatch 2024-09-14T03:48:59.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:58 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/4272384867' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f9611ffd-5d37-4465-8114-ecc70d4d6d14"}]': finished 2024-09-14T03:48:59.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:58 smithi129 ceph-mon[33173]: osdmap e15: 3 total, 2 up, 3 in 2024-09-14T03:48:59.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:58 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:48:59.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:58 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/4272384867' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f9611ffd-5d37-4465-8114-ecc70d4d6d14"}]: dispatch 2024-09-14T03:48:59.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:58 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/4272384867' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f9611ffd-5d37-4465-8114-ecc70d4d6d14"}]': finished 2024-09-14T03:48:59.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:58 smithi039 ceph-mon[29586]: osdmap e15: 3 total, 2 up, 3 in 2024-09-14T03:48:59.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:58 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:48:59.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:58 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/4272384867' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f9611ffd-5d37-4465-8114-ecc70d4d6d14"}]: dispatch 2024-09-14T03:48:59.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:58 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/4272384867' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f9611ffd-5d37-4465-8114-ecc70d4d6d14"}]': finished 2024-09-14T03:48:59.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:58 smithi039 ceph-mon[40962]: osdmap e15: 3 total, 2 up, 3 in 2024-09-14T03:48:59.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:58 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:49:00.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:59 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/2677319084' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:49:00.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:48:59 smithi129 ceph-mon[33173]: pgmap v65: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:49:00.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:59 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/2677319084' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:49:00.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:48:59 smithi039 ceph-mon[29586]: pgmap v65: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:49:00.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:59 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/2677319084' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:49:00.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:48:59 smithi039 ceph-mon[40962]: pgmap v65: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:49:01.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:01 smithi039 ceph-mon[29586]: pgmap v66: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:49:01.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:01 smithi039 ceph-mon[40962]: pgmap v66: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:49:02.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:01 smithi129 ceph-mon[33173]: pgmap v66: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:49:03.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:03 smithi039 ceph-mon[40962]: pgmap v67: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:49:03.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:03 smithi039 ceph-mon[29586]: pgmap v67: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:49:04.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:03 smithi129 ceph-mon[33173]: pgmap v67: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:49:04.927 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:04 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-14T03:49:04.927 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:04 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:04.927 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:04 smithi039 ceph-mon[29586]: Deploying daemon osd.2 on smithi039 2024-09-14T03:49:04.927 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:04 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-14T03:49:04.927 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:04 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:04.927 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:04 smithi039 ceph-mon[40962]: Deploying daemon osd.2 on smithi039 2024-09-14T03:49:05.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:04 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-09-14T03:49:05.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:04 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:05.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:04 smithi129 ceph-mon[33173]: Deploying daemon osd.2 on smithi039 2024-09-14T03:49:05.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:05 smithi039 ceph-mon[29586]: pgmap v68: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:49:05.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:05 smithi039 ceph-mon[40962]: pgmap v68: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:49:06.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:05 smithi129 ceph-mon[33173]: pgmap v68: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:49:07.901 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:07 smithi039 ceph-mon[29586]: pgmap v69: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:49:07.901 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:07 smithi039 ceph-mon[40962]: pgmap v69: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:49:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:07 smithi129 ceph-mon[33173]: pgmap v69: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:49:09.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:09 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:09.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:09 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:09.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:09 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:09.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:09 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:09.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:09 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:09.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:09 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:09 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:09 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:09 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:10.672 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:10 smithi129 ceph-mon[33173]: pgmap v70: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:49:10.694 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:10 smithi039 ceph-mon[29586]: pgmap v70: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:49:10.694 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:10 smithi039 ceph-mon[40962]: pgmap v70: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:49:11.814 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:11 smithi039 ceph-mon[29586]: pgmap v71: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:49:11.814 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:11 smithi039 ceph-mon[40962]: pgmap v71: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:49:12.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:11 smithi129 ceph-mon[33173]: pgmap v71: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:49:12.869 INFO:teuthology.orchestra.run.smithi039.stdout:Created osd(s) 2 on host 'smithi039' 2024-09-14T03:49:13.514 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:13 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:13.514 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:13 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:13.514 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:13 smithi039 ceph-mon[40962]: from='osd.2 [v2:172.21.15.39:6818/3600430167,v1:172.21.15.39:6819/3600430167]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-14T03:49:13.514 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:13 smithi039 ceph-mon[40962]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-14T03:49:13.514 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:13 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:13.514 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:13 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:49:13.515 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:13 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:13.515 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:13 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:13.515 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:13 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:13.515 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:13 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:13.515 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:13 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:13.515 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:13 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:13.515 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:13 smithi039 ceph-mon[29586]: from='osd.2 [v2:172.21.15.39:6818/3600430167,v1:172.21.15.39:6819/3600430167]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-14T03:49:13.515 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:13 smithi039 ceph-mon[29586]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-14T03:49:13.515 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:13 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:13.515 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:13 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:49:13.516 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:13 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:13.516 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:13 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:13.516 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:13 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:13.516 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:13 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:13.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:13 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:13.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:13 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:13.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:13 smithi129 ceph-mon[33173]: from='osd.2 [v2:172.21.15.39:6818/3600430167,v1:172.21.15.39:6819/3600430167]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-14T03:49:13.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:13 smithi129 ceph-mon[33173]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-09-14T03:49:13.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:13 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:13.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:13 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:49:13.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:13 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:13.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:13 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:13.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:13 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:13.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:13 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:13.536 DEBUG:teuthology.orchestra.run.smithi039:osd.2> sudo journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.2.service 2024-09-14T03:49:13.539 INFO:tasks.cephadm:Deploying osd.3 on smithi039 with /dev/vg_nvme/lv_1... 2024-09-14T03:49:13.539 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-14T03:49:14.359 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:49:14.469 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:14 smithi039 ceph-mon[29586]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-14T03:49:14.469 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:14 smithi039 ceph-mon[29586]: osdmap e16: 3 total, 2 up, 3 in 2024-09-14T03:49:14.469 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:14 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:49:14.469 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:14 smithi039 ceph-mon[29586]: from='osd.2 [v2:172.21.15.39:6818/3600430167,v1:172.21.15.39:6819/3600430167]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]: dispatch 2024-09-14T03:49:14.470 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:14 smithi039 ceph-mon[29586]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]: dispatch 2024-09-14T03:49:14.470 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:14 smithi039 ceph-mon[29586]: pgmap v73: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:49:14.470 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:14 smithi039 ceph-mon[40962]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-14T03:49:14.470 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:14 smithi039 ceph-mon[40962]: osdmap e16: 3 total, 2 up, 3 in 2024-09-14T03:49:14.470 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:14 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:49:14.470 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:14 smithi039 ceph-mon[40962]: from='osd.2 [v2:172.21.15.39:6818/3600430167,v1:172.21.15.39:6819/3600430167]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]: dispatch 2024-09-14T03:49:14.470 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:14 smithi039 ceph-mon[40962]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]: dispatch 2024-09-14T03:49:14.470 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:14 smithi039 ceph-mon[40962]: pgmap v73: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:49:14.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:14 smithi129 ceph-mon[33173]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-14T03:49:14.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:14 smithi129 ceph-mon[33173]: osdmap e16: 3 total, 2 up, 3 in 2024-09-14T03:49:14.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:14 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:49:14.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:14 smithi129 ceph-mon[33173]: from='osd.2 [v2:172.21.15.39:6818/3600430167,v1:172.21.15.39:6819/3600430167]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]: dispatch 2024-09-14T03:49:14.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:14 smithi129 ceph-mon[33173]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]: dispatch 2024-09-14T03:49:14.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:14 smithi129 ceph-mon[33173]: pgmap v73: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:49:14.790 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 03:49:14 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2[57619]: 2024-09-14T03:49:14.574+0000 7febb0f31640 -1 osd.2 0 waiting for initial osdmap 2024-09-14T03:49:14.791 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 03:49:14 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2[57619]: 2024-09-14T03:49:14.581+0000 7feba853b640 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:49:15.249 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:15 smithi039 ceph-mon[29586]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]': finished 2024-09-14T03:49:15.249 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:15 smithi039 ceph-mon[29586]: osdmap e17: 3 total, 2 up, 3 in 2024-09-14T03:49:15.249 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:15 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:49:15.249 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:15 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:49:15.249 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:15 smithi039 ceph-mon[29586]: from='osd.2 ' entity='osd.2' 2024-09-14T03:49:15.249 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:15 smithi039 ceph-mon[40962]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]': finished 2024-09-14T03:49:15.249 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:15 smithi039 ceph-mon[40962]: osdmap e17: 3 total, 2 up, 3 in 2024-09-14T03:49:15.249 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:15 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:49:15.250 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:15 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:49:15.250 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:15 smithi039 ceph-mon[40962]: from='osd.2 ' entity='osd.2' 2024-09-14T03:49:15.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:15 smithi129 ceph-mon[33173]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]': finished 2024-09-14T03:49:15.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:15 smithi129 ceph-mon[33173]: osdmap e17: 3 total, 2 up, 3 in 2024-09-14T03:49:15.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:15 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:49:15.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:15 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:49:15.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:15 smithi129 ceph-mon[33173]: from='osd.2 ' entity='osd.2' 2024-09-14T03:49:16.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:16 smithi039 ceph-mon[29586]: purged_snaps scrub starts 2024-09-14T03:49:16.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:16 smithi039 ceph-mon[29586]: purged_snaps scrub ok 2024-09-14T03:49:16.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:16 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:49:16.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:16 smithi039 ceph-mon[29586]: pgmap v75: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:49:16.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:16 smithi039 ceph-mon[29586]: osd.2 [v2:172.21.15.39:6818/3600430167,v1:172.21.15.39:6819/3600430167] boot 2024-09-14T03:49:16.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:16 smithi039 ceph-mon[29586]: osdmap e18: 3 total, 3 up, 3 in 2024-09-14T03:49:16.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:16 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:49:16.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:16 smithi039 ceph-mon[40962]: purged_snaps scrub starts 2024-09-14T03:49:16.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:16 smithi039 ceph-mon[40962]: purged_snaps scrub ok 2024-09-14T03:49:16.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:16 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:49:16.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:16 smithi039 ceph-mon[40962]: pgmap v75: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:49:16.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:16 smithi039 ceph-mon[40962]: osd.2 [v2:172.21.15.39:6818/3600430167,v1:172.21.15.39:6819/3600430167] boot 2024-09-14T03:49:16.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:16 smithi039 ceph-mon[40962]: osdmap e18: 3 total, 3 up, 3 in 2024-09-14T03:49:16.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:16 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:49:16.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:16 smithi129 ceph-mon[33173]: purged_snaps scrub starts 2024-09-14T03:49:16.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:16 smithi129 ceph-mon[33173]: purged_snaps scrub ok 2024-09-14T03:49:16.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:16 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:49:16.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:16 smithi129 ceph-mon[33173]: pgmap v75: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-09-14T03:49:16.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:16 smithi129 ceph-mon[33173]: osd.2 [v2:172.21.15.39:6818/3600430167,v1:172.21.15.39:6819/3600430167] boot 2024-09-14T03:49:16.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:16 smithi129 ceph-mon[33173]: osdmap e18: 3 total, 3 up, 3 in 2024-09-14T03:49:16.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:16 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:49:18.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:18 smithi129 ceph-mon[33173]: osdmap e19: 3 total, 3 up, 3 in 2024-09-14T03:49:18.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:18 smithi129 ceph-mon[33173]: Detected new or changed devices on smithi039 2024-09-14T03:49:18.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:18 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:18.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:18 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:18.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:18 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:18.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:18 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:18.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:18 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:18.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:18 smithi129 ceph-mon[33173]: Adjusting osd_memory_target on smithi039 to 5354M 2024-09-14T03:49:18.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:18 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:18.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:18 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:18.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:18 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:49:18.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:18 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:18.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:18 smithi129 ceph-mon[33173]: pgmap v78: 0 pgs: ; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:49:18.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:18 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:49:18.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:18 smithi039 ceph-mon[29586]: osdmap e19: 3 total, 3 up, 3 in 2024-09-14T03:49:18.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:18 smithi039 ceph-mon[29586]: Detected new or changed devices on smithi039 2024-09-14T03:49:18.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:18 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:18.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:18 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:18.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:18 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:18.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:18 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:18.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:18 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:18.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:18 smithi039 ceph-mon[29586]: Adjusting osd_memory_target on smithi039 to 5354M 2024-09-14T03:49:18.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:18 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:18.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:18 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:18.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:18 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:49:18.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:18 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:18.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:18 smithi039 ceph-mon[29586]: pgmap v78: 0 pgs: ; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:49:18.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:18 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:49:18.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:18 smithi039 ceph-mon[40962]: osdmap e19: 3 total, 3 up, 3 in 2024-09-14T03:49:18.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:18 smithi039 ceph-mon[40962]: Detected new or changed devices on smithi039 2024-09-14T03:49:18.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:18 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:18.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:18 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:18.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:18 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:18.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:18 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:18.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:18 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:18.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:18 smithi039 ceph-mon[40962]: Adjusting osd_memory_target on smithi039 to 5354M 2024-09-14T03:49:18.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:18 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:18.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:18 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:18.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:18 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:49:18.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:18 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:18.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:18 smithi039 ceph-mon[40962]: pgmap v78: 0 pgs: ; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:49:18.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:18 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:49:18.673 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:49:18.694 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi039:vg_nvme/lv_1 2024-09-14T03:49:18.900 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:49:19.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:19 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-09-14T03:49:19.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:19 smithi039 ceph-mon[29586]: osdmap e20: 3 total, 3 up, 3 in 2024-09-14T03:49:19.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:19 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:49:19.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:19 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-09-14T03:49:19.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:19 smithi039 ceph-mon[40962]: osdmap e20: 3 total, 3 up, 3 in 2024-09-14T03:49:19.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:19 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:49:19.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:19 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-09-14T03:49:19.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:19 smithi129 ceph-mon[33173]: osdmap e20: 3 total, 3 up, 3 in 2024-09-14T03:49:19.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:19 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:49:20.269 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 03:49:20 smithi039 sudo[62088]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-14T03:49:20.269 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 03:49:20 smithi039 sudo[62088]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:49:20.269 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 03:49:20 smithi039 sudo[62088]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:49:20.269 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 03:49:20 smithi039 sudo[62088]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:49:20.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:20 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:49:20.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:20 smithi129 ceph-mon[33173]: osdmap e21: 3 total, 3 up, 3 in 2024-09-14T03:49:20.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:20 smithi129 ceph-mon[33173]: pgmap v81: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:49:20.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:20 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:49:20.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:20 smithi039 ceph-mon[29586]: osdmap e21: 3 total, 3 up, 3 in 2024-09-14T03:49:20.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:20 smithi039 ceph-mon[29586]: pgmap v81: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:49:20.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:20 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-09-14T03:49:20.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:20 smithi039 ceph-mon[40962]: osdmap e21: 3 total, 3 up, 3 in 2024-09-14T03:49:20.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:20 smithi039 ceph-mon[40962]: pgmap v81: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:49:21.040 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 03:49:20 smithi039 sudo[62137]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-14T03:49:21.040 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 03:49:20 smithi039 sudo[62137]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:49:21.040 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 03:49:20 smithi039 sudo[62137]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:49:21.040 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 03:49:20 smithi039 sudo[62137]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:49:21.317 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:21 smithi039 ceph-mon[29586]: osdmap e22: 3 total, 3 up, 3 in 2024-09-14T03:49:21.317 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:21 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:49:21.317 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:21 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:49:21.317 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:21 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:21.317 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:21 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:21.317 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:21 smithi039 ceph-mon[40962]: osdmap e22: 3 total, 3 up, 3 in 2024-09-14T03:49:21.317 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:21 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:49:21.317 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:21 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:49:21.318 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:21 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:21.318 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:21 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:21.318 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 03:49:21 smithi039 sudo[62263]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-14T03:49:21.318 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 03:49:21 smithi039 sudo[62263]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:49:21.318 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 03:49:21 smithi039 sudo[62263]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:49:21.318 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 03:49:21 smithi039 sudo[62263]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:49:21.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:21 smithi129 ceph-mon[33173]: osdmap e22: 3 total, 3 up, 3 in 2024-09-14T03:49:21.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:21 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:49:21.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:21 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:49:21.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:21 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:21.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:21 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:22.040 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 03:49:21 smithi039 sudo[62307]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-14T03:49:22.040 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 03:49:21 smithi039 sudo[62307]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:49:22.040 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 03:49:21 smithi039 sudo[62307]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:49:22.040 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 03:49:21 smithi039 sudo[62307]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:49:22.330 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:22 smithi039 ceph-mon[29586]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi039:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:22.330 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:22 smithi039 ceph-mon[29586]: pgmap v83: 1 pgs: 1 unknown; 0 B data, 81 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:49:22.330 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 03:49:22 smithi039 sudo[62325]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-14T03:49:22.330 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 03:49:22 smithi039 sudo[62325]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:49:22.331 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 03:49:22 smithi039 sudo[62325]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:49:22.331 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 03:49:22 smithi039 sudo[62325]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:49:22.331 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:22 smithi039 ceph-mon[40962]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi039:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:22.331 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:22 smithi039 ceph-mon[40962]: pgmap v83: 1 pgs: 1 unknown; 0 B data, 81 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:49:22.711 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 03:49:22 smithi039 sudo[62377]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-14T03:49:22.711 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 03:49:22 smithi039 sudo[62377]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:49:22.711 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 03:49:22 smithi039 sudo[62377]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:49:22.711 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 03:49:22 smithi039 sudo[62377]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:49:22.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:22 smithi129 ceph-mon[33173]: from='client.24203 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi039:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:22.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:22 smithi129 ceph-mon[33173]: pgmap v83: 1 pgs: 1 unknown; 0 B data, 81 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:49:22.961 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:22 smithi039 sudo[62419]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-14T03:49:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:22 smithi039 sudo[62419]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:49:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:22 smithi039 sudo[62419]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:49:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:22 smithi039 sudo[62419]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:49:23.678 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:23 smithi039 ceph-mon[40962]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-14T03:49:23.678 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:23 smithi039 ceph-mon[40962]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:49:23.678 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:23 smithi039 sudo[62423]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-14T03:49:23.678 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:23 smithi039 sudo[62423]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:49:23.678 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:23 smithi039 sudo[62423]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:49:23.678 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:23 smithi039 sudo[62423]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:49:23.678 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:23 smithi039 ceph-mon[29586]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-14T03:49:23.678 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:23 smithi039 ceph-mon[29586]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:49:23.692 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:23 smithi129 ceph-mon[33173]: mgrmap e15: y(active, since 2m), standbys: x 2024-09-14T03:49:23.692 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:23 smithi129 ceph-mon[33173]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:49:23.692 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:23 smithi129 sudo[35935]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-14T03:49:23.693 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:23 smithi129 sudo[35935]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:49:23.693 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:23 smithi129 sudo[35935]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:49:23.693 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:23 smithi129 sudo[35935]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:49:24.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:23 smithi129 sudo[35938]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-14T03:49:24.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:23 smithi129 sudo[35938]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:49:24.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:23 smithi129 sudo[35938]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:49:24.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:23 smithi129 sudo[35938]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:49:24.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:24 smithi039 sudo[62460]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-09-14T03:49:24.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:24 smithi039 sudo[62460]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:49:24.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:24 smithi039 sudo[62460]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:49:24.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:24 smithi039 sudo[62460]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:49:25.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:24 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/575700531' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "56b6877b-2107-47d6-8da9-015db7352288"}]: dispatch 2024-09-14T03:49:25.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:24 smithi129 ceph-mon[33173]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:49:25.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:24 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:49:25.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:24 smithi129 ceph-mon[33173]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "56b6877b-2107-47d6-8da9-015db7352288"}]: dispatch 2024-09-14T03:49:25.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:24 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:49:25.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:24 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:49:25.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:24 smithi129 ceph-mon[33173]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "56b6877b-2107-47d6-8da9-015db7352288"}]': finished 2024-09-14T03:49:25.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:24 smithi129 ceph-mon[33173]: osdmap e23: 4 total, 3 up, 4 in 2024-09-14T03:49:25.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:24 smithi129 ceph-mon[33173]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:49:25.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:24 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:49:25.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:24 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:49:25.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:24 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:49:25.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:24 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:49:25.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:24 smithi129 ceph-mon[33173]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:49:25.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:24 smithi129 ceph-mon[33173]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:49:25.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:24 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:25.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:24 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:49:25.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:24 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:49:25.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:24 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:49:25.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/575700531' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "56b6877b-2107-47d6-8da9-015db7352288"}]: dispatch 2024-09-14T03:49:25.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[29586]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:49:25.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:49:25.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[29586]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "56b6877b-2107-47d6-8da9-015db7352288"}]: dispatch 2024-09-14T03:49:25.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:49:25.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:49:25.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[29586]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "56b6877b-2107-47d6-8da9-015db7352288"}]': finished 2024-09-14T03:49:25.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[29586]: osdmap e23: 4 total, 3 up, 4 in 2024-09-14T03:49:25.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[29586]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:49:25.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:49:25.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:49:25.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:49:25.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:49:25.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[29586]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:49:25.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[29586]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:49:25.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:25.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:49:25.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:49:25.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:49:25.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:24 smithi039 sudo[62463]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-09-14T03:49:25.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:24 smithi039 sudo[62463]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-09-14T03:49:25.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:24 smithi039 sudo[62463]: pam_unix(sudo:session): session opened for user root(uid=0) by (uid=167) 2024-09-14T03:49:25.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:24 smithi039 sudo[62463]: pam_unix(sudo:session): session closed for user root 2024-09-14T03:49:25.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/575700531' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "56b6877b-2107-47d6-8da9-015db7352288"}]: dispatch 2024-09-14T03:49:25.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[40962]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:49:25.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:49:25.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[40962]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "56b6877b-2107-47d6-8da9-015db7352288"}]: dispatch 2024-09-14T03:49:25.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:49:25.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:49:25.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[40962]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "56b6877b-2107-47d6-8da9-015db7352288"}]': finished 2024-09-14T03:49:25.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[40962]: osdmap e23: 4 total, 3 up, 4 in 2024-09-14T03:49:25.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[40962]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:49:25.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:49:25.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:49:25.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:49:25.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:49:25.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[40962]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:49:25.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[40962]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:49:25.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:25.294 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:49:25.294 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:49:25.294 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:24 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:49:26.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:25 smithi129 ceph-mon[33173]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:49:26.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:25 smithi129 ceph-mon[33173]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:49:26.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:25 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/2822426222' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:49:26.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:25 smithi129 ceph-mon[33173]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:49:26.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:25 smithi039 ceph-mon[29586]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:49:26.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:25 smithi039 ceph-mon[29586]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:49:26.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:25 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/2822426222' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:49:26.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:25 smithi039 ceph-mon[29586]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:49:26.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:25 smithi039 ceph-mon[40962]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-09-14T03:49:26.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:25 smithi039 ceph-mon[40962]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-09-14T03:49:26.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:25 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/2822426222' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:49:26.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:25 smithi039 ceph-mon[40962]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:49:27.831 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:27 smithi039 ceph-mon[29586]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 4.4 KiB/s rd, 191 KiB/s wr, 13 op/s 2024-09-14T03:49:27.832 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:27 smithi039 ceph-mon[40962]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 4.4 KiB/s rd, 191 KiB/s wr, 13 op/s 2024-09-14T03:49:28.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:27 smithi129 ceph-mon[33173]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 4.4 KiB/s rd, 191 KiB/s wr, 13 op/s 2024-09-14T03:49:29.941 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:29 smithi039 ceph-mon[29586]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 2.9 KiB/s rd, 128 KiB/s wr, 9 op/s 2024-09-14T03:49:29.941 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:29 smithi039 ceph-mon[40962]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 2.9 KiB/s rd, 128 KiB/s wr, 9 op/s 2024-09-14T03:49:30.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:29 smithi129 ceph-mon[33173]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 2.9 KiB/s rd, 128 KiB/s wr, 9 op/s 2024-09-14T03:49:30.966 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:30 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-14T03:49:30.966 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:30 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:30.966 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:30 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-14T03:49:30.966 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:30 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:31.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:30 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-09-14T03:49:31.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:30 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:31.924 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:31 smithi039 ceph-mon[29586]: Deploying daemon osd.3 on smithi039 2024-09-14T03:49:31.924 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:31 smithi039 ceph-mon[29586]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 97 KiB/s wr, 6 op/s 2024-09-14T03:49:31.924 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:31 smithi039 ceph-mon[40962]: Deploying daemon osd.3 on smithi039 2024-09-14T03:49:31.924 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:31 smithi039 ceph-mon[40962]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 97 KiB/s wr, 6 op/s 2024-09-14T03:49:32.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:31 smithi129 ceph-mon[33173]: Deploying daemon osd.3 on smithi039 2024-09-14T03:49:32.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:31 smithi129 ceph-mon[33173]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 97 KiB/s wr, 6 op/s 2024-09-14T03:49:34.011 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:33 smithi039 ceph-mon[29586]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-14T03:49:34.012 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:33 smithi039 ceph-mon[40962]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-14T03:49:34.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:33 smithi129 ceph-mon[33173]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-09-14T03:49:36.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:35 smithi129 ceph-mon[33173]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-14T03:49:36.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:35 smithi039 ceph-mon[29586]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-14T03:49:36.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:35 smithi039 ceph-mon[40962]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-14T03:49:36.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:36 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:36.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:36 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:36.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:36 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:36.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:36 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:36.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:36 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:36.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:36 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:37.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:36 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:37.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:36 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:37.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:36 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:37.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:37 smithi039 ceph-mon[29586]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-14T03:49:37.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:37 smithi039 ceph-mon[40962]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-14T03:49:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:37 smithi129 ceph-mon[33173]: pgmap v92: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-09-14T03:49:39.100 INFO:teuthology.orchestra.run.smithi039.stdout:Created osd(s) 3 on host 'smithi039' 2024-09-14T03:49:39.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:39 smithi039 ceph-mon[29586]: from='osd.3 [v2:172.21.15.39:6826/1027302150,v1:172.21.15.39:6827/1027302150]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-14T03:49:39.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:39 smithi039 ceph-mon[29586]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-14T03:49:39.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:39 smithi039 ceph-mon[40962]: from='osd.3 [v2:172.21.15.39:6826/1027302150,v1:172.21.15.39:6827/1027302150]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-14T03:49:39.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:39 smithi039 ceph-mon[40962]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-14T03:49:39.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:39 smithi129 ceph-mon[33173]: from='osd.3 [v2:172.21.15.39:6826/1027302150,v1:172.21.15.39:6827/1027302150]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-14T03:49:39.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:39 smithi129 ceph-mon[33173]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-09-14T03:49:39.810 DEBUG:teuthology.orchestra.run.smithi039:osd.3> sudo journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.3.service 2024-09-14T03:49:39.812 INFO:tasks.cephadm:Deploying osd.4 on smithi129 with /dev/vg_nvme/lv_4... 2024-09-14T03:49:39.813 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_4 2024-09-14T03:49:40.014 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:49:40.134 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:40 smithi129 ceph-mon[33173]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-14T03:49:40.134 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:40 smithi129 ceph-mon[33173]: osdmap e24: 4 total, 3 up, 4 in 2024-09-14T03:49:40.134 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:40 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:49:40.134 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:40 smithi129 ceph-mon[33173]: from='osd.3 [v2:172.21.15.39:6826/1027302150,v1:172.21.15.39:6827/1027302150]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]: dispatch 2024-09-14T03:49:40.134 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:40 smithi129 ceph-mon[33173]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]: dispatch 2024-09-14T03:49:40.135 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:40 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:40.135 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:40 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:40.135 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:40 smithi129 ceph-mon[33173]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:49:40.135 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:40 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:40.135 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:40 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:40.372 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:40 smithi039 ceph-mon[29586]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-14T03:49:40.372 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:40 smithi039 ceph-mon[29586]: osdmap e24: 4 total, 3 up, 4 in 2024-09-14T03:49:40.372 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:40 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:49:40.372 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:40 smithi039 ceph-mon[29586]: from='osd.3 [v2:172.21.15.39:6826/1027302150,v1:172.21.15.39:6827/1027302150]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]: dispatch 2024-09-14T03:49:40.372 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:40 smithi039 ceph-mon[29586]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]: dispatch 2024-09-14T03:49:40.372 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:40 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:40.373 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:40 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:40.373 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:40 smithi039 ceph-mon[29586]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:49:40.373 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:40 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:40.373 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:40 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:40.373 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:40 smithi039 ceph-mon[40962]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-14T03:49:40.373 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:40 smithi039 ceph-mon[40962]: osdmap e24: 4 total, 3 up, 4 in 2024-09-14T03:49:40.373 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:40 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:49:40.373 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:40 smithi039 ceph-mon[40962]: from='osd.3 [v2:172.21.15.39:6826/1027302150,v1:172.21.15.39:6827/1027302150]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]: dispatch 2024-09-14T03:49:40.373 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:40 smithi039 ceph-mon[40962]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]: dispatch 2024-09-14T03:49:40.373 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:40 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:40.374 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:40 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:40.374 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:40 smithi039 ceph-mon[40962]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:49:40.374 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:40 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:40.374 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:40 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:40.675 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 03:49:40 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3[64221]: 2024-09-14T03:49:40.524+0000 7fd341853640 -1 osd.3 0 waiting for initial osdmap 2024-09-14T03:49:40.675 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 03:49:40 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3[64221]: 2024-09-14T03:49:40.533+0000 7fd339674640 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:49:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:41 smithi129 ceph-mon[33173]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]': finished 2024-09-14T03:49:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:41 smithi129 ceph-mon[33173]: osdmap e25: 4 total, 3 up, 4 in 2024-09-14T03:49:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:41 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:49:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:41 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:49:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:41 smithi129 ceph-mon[33173]: from='osd.3 ' entity='osd.3' 2024-09-14T03:49:41.421 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:41 smithi039 ceph-mon[29586]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]': finished 2024-09-14T03:49:41.421 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:41 smithi039 ceph-mon[29586]: osdmap e25: 4 total, 3 up, 4 in 2024-09-14T03:49:41.421 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:41 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:49:41.421 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:41 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:49:41.421 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:41 smithi039 ceph-mon[29586]: from='osd.3 ' entity='osd.3' 2024-09-14T03:49:41.421 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:41 smithi039 ceph-mon[40962]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi039", "root=default"]}]': finished 2024-09-14T03:49:41.421 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:41 smithi039 ceph-mon[40962]: osdmap e25: 4 total, 3 up, 4 in 2024-09-14T03:49:41.421 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:41 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:49:41.421 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:41 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:49:41.422 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:41 smithi039 ceph-mon[40962]: from='osd.3 ' entity='osd.3' 2024-09-14T03:49:42.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:42 smithi129 ceph-mon[33173]: purged_snaps scrub starts 2024-09-14T03:49:42.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:42 smithi129 ceph-mon[33173]: purged_snaps scrub ok 2024-09-14T03:49:42.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:42 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:49:42.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:42 smithi129 ceph-mon[33173]: osd.3 [v2:172.21.15.39:6826/1027302150,v1:172.21.15.39:6827/1027302150] boot 2024-09-14T03:49:42.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:42 smithi129 ceph-mon[33173]: osdmap e26: 4 total, 4 up, 4 in 2024-09-14T03:49:42.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:42 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:49:42.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:42 smithi129 ceph-mon[33173]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:49:42.465 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:42 smithi039 ceph-mon[29586]: purged_snaps scrub starts 2024-09-14T03:49:42.465 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:42 smithi039 ceph-mon[29586]: purged_snaps scrub ok 2024-09-14T03:49:42.465 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:42 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:49:42.465 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:42 smithi039 ceph-mon[29586]: osd.3 [v2:172.21.15.39:6826/1027302150,v1:172.21.15.39:6827/1027302150] boot 2024-09-14T03:49:42.465 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:42 smithi039 ceph-mon[29586]: osdmap e26: 4 total, 4 up, 4 in 2024-09-14T03:49:42.465 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:42 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:49:42.465 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:42 smithi039 ceph-mon[29586]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:49:42.465 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:42 smithi039 ceph-mon[40962]: purged_snaps scrub starts 2024-09-14T03:49:42.466 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:42 smithi039 ceph-mon[40962]: purged_snaps scrub ok 2024-09-14T03:49:42.466 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:42 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:49:42.466 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:42 smithi039 ceph-mon[40962]: osd.3 [v2:172.21.15.39:6826/1027302150,v1:172.21.15.39:6827/1027302150] boot 2024-09-14T03:49:42.466 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:42 smithi039 ceph-mon[40962]: osdmap e26: 4 total, 4 up, 4 in 2024-09-14T03:49:42.466 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:42 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:49:42.466 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:42 smithi039 ceph-mon[40962]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-09-14T03:49:42.553 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:49:42.572 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi129:vg_nvme/lv_4 2024-09-14T03:49:42.763 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:49:43.892 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:43 smithi129 ceph-mon[33173]: osdmap e27: 4 total, 4 up, 4 in 2024-09-14T03:49:43.892 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:43 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:43.892 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:43 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:43.892 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:43 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:43.892 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:43 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:43.893 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:43 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:43.893 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:43 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:43.893 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:43 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:43.893 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:43 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:43.893 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:43 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:49:43.893 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:43 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:43.893 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:43 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:43.893 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:43 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:43.893 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:43 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:49:43.893 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:43 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:44.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:43 smithi039 ceph-mon[29586]: osdmap e27: 4 total, 4 up, 4 in 2024-09-14T03:49:44.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:43 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:44.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:43 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:44.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:43 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:44.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:43 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:44.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:43 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:44.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:43 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:44.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:43 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:44.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:43 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:44.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:43 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:49:44.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:43 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:44.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:43 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:44.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:43 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:44.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:43 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:49:44.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:43 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:44.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:43 smithi039 ceph-mon[40962]: osdmap e27: 4 total, 4 up, 4 in 2024-09-14T03:49:44.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:44.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:44.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:44.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:44.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:44.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:49:44.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:44.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:44.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:49:44.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:44.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:44.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:44.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:49:44.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:43 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:44.862 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:44 smithi129 ceph-mon[33173]: Detected new or changed devices on smithi039 2024-09-14T03:49:44.862 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:44 smithi129 ceph-mon[33173]: Adjusting osd_memory_target on smithi039 to 4016M 2024-09-14T03:49:44.862 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:44 smithi129 ceph-mon[33173]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:49:44.863 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:44 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:49:44.863 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:44 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:49:44.863 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:44 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:44 smithi039 ceph-mon[29586]: Detected new or changed devices on smithi039 2024-09-14T03:49:45.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:44 smithi039 ceph-mon[29586]: Adjusting osd_memory_target on smithi039 to 4016M 2024-09-14T03:49:45.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:44 smithi039 ceph-mon[29586]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:49:45.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:44 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:49:45.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:44 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:49:45.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:44 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:45.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:44 smithi039 ceph-mon[40962]: Detected new or changed devices on smithi039 2024-09-14T03:49:45.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:44 smithi039 ceph-mon[40962]: Adjusting osd_memory_target on smithi039 to 4016M 2024-09-14T03:49:45.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:44 smithi039 ceph-mon[40962]: pgmap v99: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:49:45.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:44 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:49:45.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:44 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:49:45.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:44 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:45.825 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:45 smithi129 ceph-mon[33173]: from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi129:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:46.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:45 smithi039 ceph-mon[29586]: from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi129:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:46.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:45 smithi039 ceph-mon[40962]: from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi129:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:49:46.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:46 smithi129 ceph-mon[33173]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:49:47.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:46 smithi039 ceph-mon[29586]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:49:47.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:46 smithi039 ceph-mon[40962]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:49:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:47 smithi129 ceph-mon[33173]: from='client.? 172.21.15.129:0/777835752' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2d5523c4-5871-4fb3-bc0d-62c3b7d8b6a1"}]: dispatch 2024-09-14T03:49:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:47 smithi129 ceph-mon[33173]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2d5523c4-5871-4fb3-bc0d-62c3b7d8b6a1"}]: dispatch 2024-09-14T03:49:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:47 smithi129 ceph-mon[33173]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2d5523c4-5871-4fb3-bc0d-62c3b7d8b6a1"}]': finished 2024-09-14T03:49:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:47 smithi129 ceph-mon[33173]: osdmap e28: 5 total, 4 up, 5 in 2024-09-14T03:49:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:47 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:49:47.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:47 smithi129 ceph-mon[33173]: from='client.? 172.21.15.129:0/3185902074' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:49:48.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:47 smithi039 ceph-mon[29586]: from='client.? 172.21.15.129:0/777835752' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2d5523c4-5871-4fb3-bc0d-62c3b7d8b6a1"}]: dispatch 2024-09-14T03:49:48.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:47 smithi039 ceph-mon[29586]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2d5523c4-5871-4fb3-bc0d-62c3b7d8b6a1"}]: dispatch 2024-09-14T03:49:48.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:47 smithi039 ceph-mon[29586]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2d5523c4-5871-4fb3-bc0d-62c3b7d8b6a1"}]': finished 2024-09-14T03:49:48.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:47 smithi039 ceph-mon[29586]: osdmap e28: 5 total, 4 up, 5 in 2024-09-14T03:49:48.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:47 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:49:48.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:47 smithi039 ceph-mon[29586]: from='client.? 172.21.15.129:0/3185902074' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:49:48.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:47 smithi039 ceph-mon[40962]: from='client.? 172.21.15.129:0/777835752' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2d5523c4-5871-4fb3-bc0d-62c3b7d8b6a1"}]: dispatch 2024-09-14T03:49:48.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:47 smithi039 ceph-mon[40962]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2d5523c4-5871-4fb3-bc0d-62c3b7d8b6a1"}]: dispatch 2024-09-14T03:49:48.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:47 smithi039 ceph-mon[40962]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2d5523c4-5871-4fb3-bc0d-62c3b7d8b6a1"}]': finished 2024-09-14T03:49:48.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:47 smithi039 ceph-mon[40962]: osdmap e28: 5 total, 4 up, 5 in 2024-09-14T03:49:48.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:47 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:49:48.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:47 smithi039 ceph-mon[40962]: from='client.? 172.21.15.129:0/3185902074' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:49:49.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:48 smithi129 ceph-mon[33173]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:49:49.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:48 smithi039 ceph-mon[29586]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:49:49.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:48 smithi039 ceph-mon[40962]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:49:50.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:50 smithi129 ceph-mon[33173]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:49:51.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:50 smithi039 ceph-mon[29586]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:49:51.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:50 smithi039 ceph-mon[40962]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:49:52.859 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:52 smithi039 ceph-mon[29586]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:49:52.859 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:52 smithi039 ceph-mon[40962]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:49:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:52 smithi129 ceph-mon[33173]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:49:53.742 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:53 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-14T03:49:53.743 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:53 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:53.743 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:53 smithi129 ceph-mon[33173]: Deploying daemon osd.4 on smithi129 2024-09-14T03:49:53.743 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:53 smithi129 ceph-mon[33173]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:49:54.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:53 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-14T03:49:54.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:53 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:54.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:53 smithi039 ceph-mon[40962]: Deploying daemon osd.4 on smithi129 2024-09-14T03:49:54.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:53 smithi039 ceph-mon[40962]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:49:54.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:53 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-09-14T03:49:54.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:53 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:49:54.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:53 smithi039 ceph-mon[29586]: Deploying daemon osd.4 on smithi129 2024-09-14T03:49:54.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:53 smithi039 ceph-mon[29586]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:49:55.982 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:55 smithi129 ceph-mon[33173]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:49:56.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:55 smithi039 ceph-mon[40962]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:49:56.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:55 smithi039 ceph-mon[29586]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:49:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:57 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:57 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:57 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:58.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:57 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:58.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:57 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:58.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:57 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:58.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:57 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:49:58.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:57 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:58.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:57 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:49:58.907 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:49:58 smithi129 ceph-mon[33173]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:49:59.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:49:58 smithi039 ceph-mon[29586]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:49:59.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:49:58 smithi039 ceph-mon[40962]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:50:00.580 INFO:teuthology.orchestra.run.smithi129.stdout:Created osd(s) 4 on host 'smithi129' 2024-09-14T03:50:00.903 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:00 smithi129 ceph-mon[33173]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:50:00.903 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:00 smithi129 ceph-mon[33173]: overall HEALTH_OK 2024-09-14T03:50:01.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:00 smithi039 ceph-mon[29586]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:50:01.069 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:00 smithi039 ceph-mon[29586]: overall HEALTH_OK 2024-09-14T03:50:01.069 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:00 smithi039 ceph-mon[40962]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:50:01.069 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:00 smithi039 ceph-mon[40962]: overall HEALTH_OK 2024-09-14T03:50:01.200 DEBUG:teuthology.orchestra.run.smithi129:osd.4> sudo journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.4.service 2024-09-14T03:50:01.202 INFO:tasks.cephadm:Deploying osd.5 on smithi129 with /dev/vg_nvme/lv_3... 2024-09-14T03:50:01.202 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_3 2024-09-14T03:50:01.444 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:50:01.848 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:01 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:01.848 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:01 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:01.848 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:01 smithi129 ceph-mon[33173]: from='osd.4 [v2:172.21.15.129:6800/168261430,v1:172.21.15.129:6801/168261430]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-14T03:50:01.848 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:01 smithi129 ceph-mon[33173]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-14T03:50:01.849 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:01 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:01.849 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:01 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:02.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:01 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:02.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:01 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:02.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:01 smithi039 ceph-mon[29586]: from='osd.4 [v2:172.21.15.129:6800/168261430,v1:172.21.15.129:6801/168261430]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-14T03:50:02.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:01 smithi039 ceph-mon[29586]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-14T03:50:02.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:01 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:02.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:01 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:02.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:01 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:02.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:01 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:02.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:01 smithi039 ceph-mon[40962]: from='osd.4 [v2:172.21.15.129:6800/168261430,v1:172.21.15.129:6801/168261430]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-14T03:50:02.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:01 smithi039 ceph-mon[40962]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-09-14T03:50:02.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:01 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:02.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:01 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:02.750 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:02 smithi129 ceph-mon[33173]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:50:02.750 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:02 smithi129 ceph-mon[33173]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-14T03:50:02.750 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:02 smithi129 ceph-mon[33173]: osdmap e29: 5 total, 4 up, 5 in 2024-09-14T03:50:02.750 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:02 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:50:02.750 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:02 smithi129 ceph-mon[33173]: from='osd.4 [v2:172.21.15.129:6800/168261430,v1:172.21.15.129:6801/168261430]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]: dispatch 2024-09-14T03:50:02.750 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:02 smithi129 ceph-mon[33173]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]: dispatch 2024-09-14T03:50:02.750 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 03:50:02 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4[38530]: 2024-09-14T03:50:02.604+0000 7fcbdbefd640 -1 osd.4 0 waiting for initial osdmap 2024-09-14T03:50:02.751 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 03:50:02 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4[38530]: 2024-09-14T03:50:02.608+0000 7fcbd7d26640 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:50:03.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:02 smithi039 ceph-mon[29586]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:50:03.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:02 smithi039 ceph-mon[29586]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-14T03:50:03.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:02 smithi039 ceph-mon[29586]: osdmap e29: 5 total, 4 up, 5 in 2024-09-14T03:50:03.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:02 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:50:03.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:02 smithi039 ceph-mon[29586]: from='osd.4 [v2:172.21.15.129:6800/168261430,v1:172.21.15.129:6801/168261430]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]: dispatch 2024-09-14T03:50:03.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:02 smithi039 ceph-mon[29586]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]: dispatch 2024-09-14T03:50:03.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:02 smithi039 ceph-mon[40962]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:50:03.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:02 smithi039 ceph-mon[40962]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-14T03:50:03.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:02 smithi039 ceph-mon[40962]: osdmap e29: 5 total, 4 up, 5 in 2024-09-14T03:50:03.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:02 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:50:03.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:02 smithi039 ceph-mon[40962]: from='osd.4 [v2:172.21.15.129:6800/168261430,v1:172.21.15.129:6801/168261430]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]: dispatch 2024-09-14T03:50:03.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:02 smithi039 ceph-mon[40962]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]: dispatch 2024-09-14T03:50:03.987 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:03 smithi129 ceph-mon[33173]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]': finished 2024-09-14T03:50:03.988 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:03 smithi129 ceph-mon[33173]: osdmap e30: 5 total, 4 up, 5 in 2024-09-14T03:50:03.988 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:03 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:50:03.988 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:03 smithi129 ceph-mon[33173]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:50:03.988 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:03 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:50:03.988 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:50:04.006 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi129:vg_nvme/lv_3 2024-09-14T03:50:04.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:03 smithi039 ceph-mon[40962]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]': finished 2024-09-14T03:50:04.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:03 smithi039 ceph-mon[40962]: osdmap e30: 5 total, 4 up, 5 in 2024-09-14T03:50:04.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:03 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:50:04.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:03 smithi039 ceph-mon[40962]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:50:04.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:03 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:50:04.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:03 smithi039 ceph-mon[29586]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]': finished 2024-09-14T03:50:04.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:03 smithi039 ceph-mon[29586]: osdmap e30: 5 total, 4 up, 5 in 2024-09-14T03:50:04.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:03 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:50:04.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:03 smithi039 ceph-mon[29586]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-09-14T03:50:04.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:03 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:50:04.665 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:04 smithi129 ceph-mon[33173]: purged_snaps scrub starts 2024-09-14T03:50:04.666 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:04 smithi129 ceph-mon[33173]: purged_snaps scrub ok 2024-09-14T03:50:04.666 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:04 smithi129 ceph-mon[33173]: osd.4 [v2:172.21.15.129:6800/168261430,v1:172.21.15.129:6801/168261430] boot 2024-09-14T03:50:04.666 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:04 smithi129 ceph-mon[33173]: osdmap e31: 5 total, 5 up, 5 in 2024-09-14T03:50:04.666 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:04 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:50:04.711 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:50:05.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:04 smithi039 ceph-mon[29586]: purged_snaps scrub starts 2024-09-14T03:50:05.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:04 smithi039 ceph-mon[29586]: purged_snaps scrub ok 2024-09-14T03:50:05.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:04 smithi039 ceph-mon[29586]: osd.4 [v2:172.21.15.129:6800/168261430,v1:172.21.15.129:6801/168261430] boot 2024-09-14T03:50:05.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:04 smithi039 ceph-mon[29586]: osdmap e31: 5 total, 5 up, 5 in 2024-09-14T03:50:05.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:04 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:50:05.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:04 smithi039 ceph-mon[40962]: purged_snaps scrub starts 2024-09-14T03:50:05.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:04 smithi039 ceph-mon[40962]: purged_snaps scrub ok 2024-09-14T03:50:05.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:04 smithi039 ceph-mon[40962]: osd.4 [v2:172.21.15.129:6800/168261430,v1:172.21.15.129:6801/168261430] boot 2024-09-14T03:50:05.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:04 smithi039 ceph-mon[40962]: osdmap e31: 5 total, 5 up, 5 in 2024-09-14T03:50:05.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:04 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:50:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:05 smithi129 ceph-mon[33173]: osdmap e32: 5 total, 5 up, 5 in 2024-09-14T03:50:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:05 smithi129 ceph-mon[33173]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:50:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:05 smithi129 ceph-mon[33173]: osdmap e33: 5 total, 5 up, 5 in 2024-09-14T03:50:06.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:05 smithi039 ceph-mon[29586]: osdmap e32: 5 total, 5 up, 5 in 2024-09-14T03:50:06.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:05 smithi039 ceph-mon[29586]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:50:06.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:05 smithi039 ceph-mon[29586]: osdmap e33: 5 total, 5 up, 5 in 2024-09-14T03:50:06.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:05 smithi039 ceph-mon[40962]: osdmap e32: 5 total, 5 up, 5 in 2024-09-14T03:50:06.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:05 smithi039 ceph-mon[40962]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:50:06.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:05 smithi039 ceph-mon[40962]: osdmap e33: 5 total, 5 up, 5 in 2024-09-14T03:50:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:07 smithi129 ceph-mon[33173]: osdmap e34: 5 total, 5 up, 5 in 2024-09-14T03:50:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:07 smithi129 ceph-mon[33173]: from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi129:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:07 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:50:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:07 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:50:08.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:07 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:08.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:07 smithi129 ceph-mon[33173]: Detected new or changed devices on smithi129 2024-09-14T03:50:08.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:07 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:08.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:07 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:08.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:07 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:50:08.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:07 smithi129 ceph-mon[33173]: Adjusting osd_memory_target on smithi129 to 17088M 2024-09-14T03:50:08.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:07 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:08.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:07 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:08.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:07 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:08.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:07 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:08.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:07 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:08.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:07 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:08.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:07 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:08.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:07 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:08.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:07 smithi129 ceph-mon[33173]: pgmap v118: 1 pgs: 1 peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-09-14T03:50:08.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[29586]: osdmap e34: 5 total, 5 up, 5 in 2024-09-14T03:50:08.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[29586]: from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi129:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:08.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:50:08.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:50:08.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:08.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[29586]: Detected new or changed devices on smithi129 2024-09-14T03:50:08.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:08.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:08.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:50:08.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[29586]: Adjusting osd_memory_target on smithi129 to 17088M 2024-09-14T03:50:08.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:08.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:08.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:08.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:08.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:08.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:08.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:08.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:08.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[29586]: pgmap v118: 1 pgs: 1 peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-09-14T03:50:08.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[40962]: osdmap e34: 5 total, 5 up, 5 in 2024-09-14T03:50:08.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[40962]: from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi129:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:08.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:50:08.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:50:08.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:08.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[40962]: Detected new or changed devices on smithi129 2024-09-14T03:50:08.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:08.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:08.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:50:08.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[40962]: Adjusting osd_memory_target on smithi129 to 17088M 2024-09-14T03:50:08.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:08.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:08.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:08.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:08.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:08.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:08.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:08.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:08.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:07 smithi039 ceph-mon[40962]: pgmap v118: 1 pgs: 1 peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-09-14T03:50:08.756 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:08 smithi129 ceph-mon[33173]: osdmap e35: 5 total, 5 up, 5 in 2024-09-14T03:50:09.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:08 smithi039 ceph-mon[29586]: osdmap e35: 5 total, 5 up, 5 in 2024-09-14T03:50:09.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:08 smithi039 ceph-mon[40962]: osdmap e35: 5 total, 5 up, 5 in 2024-09-14T03:50:10.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:09 smithi129 ceph-mon[33173]: from='client.? 172.21.15.129:0/1261600354' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3ba55fd-f32e-4f6d-9b48-d2845a3474cd"}]: dispatch 2024-09-14T03:50:10.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:09 smithi129 ceph-mon[33173]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3ba55fd-f32e-4f6d-9b48-d2845a3474cd"}]: dispatch 2024-09-14T03:50:10.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:09 smithi129 ceph-mon[33173]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a3ba55fd-f32e-4f6d-9b48-d2845a3474cd"}]': finished 2024-09-14T03:50:10.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:09 smithi129 ceph-mon[33173]: osdmap e36: 6 total, 5 up, 6 in 2024-09-14T03:50:10.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:09 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:50:10.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:09 smithi129 ceph-mon[33173]: pgmap v121: 1 pgs: 1 peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-09-14T03:50:10.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:09 smithi129 ceph-mon[33173]: from='client.? 172.21.15.129:0/363090769' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:50:10.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:09 smithi039 ceph-mon[29586]: from='client.? 172.21.15.129:0/1261600354' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3ba55fd-f32e-4f6d-9b48-d2845a3474cd"}]: dispatch 2024-09-14T03:50:10.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:09 smithi039 ceph-mon[29586]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3ba55fd-f32e-4f6d-9b48-d2845a3474cd"}]: dispatch 2024-09-14T03:50:10.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:09 smithi039 ceph-mon[29586]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a3ba55fd-f32e-4f6d-9b48-d2845a3474cd"}]': finished 2024-09-14T03:50:10.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:09 smithi039 ceph-mon[29586]: osdmap e36: 6 total, 5 up, 6 in 2024-09-14T03:50:10.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:09 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:50:10.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:09 smithi039 ceph-mon[29586]: pgmap v121: 1 pgs: 1 peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-09-14T03:50:10.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:09 smithi039 ceph-mon[29586]: from='client.? 172.21.15.129:0/363090769' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:50:10.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:09 smithi039 ceph-mon[40962]: from='client.? 172.21.15.129:0/1261600354' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3ba55fd-f32e-4f6d-9b48-d2845a3474cd"}]: dispatch 2024-09-14T03:50:10.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:09 smithi039 ceph-mon[40962]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a3ba55fd-f32e-4f6d-9b48-d2845a3474cd"}]: dispatch 2024-09-14T03:50:10.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:09 smithi039 ceph-mon[40962]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a3ba55fd-f32e-4f6d-9b48-d2845a3474cd"}]': finished 2024-09-14T03:50:10.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:09 smithi039 ceph-mon[40962]: osdmap e36: 6 total, 5 up, 6 in 2024-09-14T03:50:10.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:09 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:50:10.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:09 smithi039 ceph-mon[40962]: pgmap v121: 1 pgs: 1 peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-09-14T03:50:10.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:09 smithi039 ceph-mon[40962]: from='client.? 172.21.15.129:0/363090769' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:50:11.910 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:11 smithi129 ceph-mon[33173]: pgmap v122: 1 pgs: 1 peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 98 KiB/s, 0 objects/s recovering 2024-09-14T03:50:12.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:11 smithi039 ceph-mon[29586]: pgmap v122: 1 pgs: 1 peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 98 KiB/s, 0 objects/s recovering 2024-09-14T03:50:12.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:11 smithi039 ceph-mon[40962]: pgmap v122: 1 pgs: 1 peering; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 98 KiB/s, 0 objects/s recovering 2024-09-14T03:50:13.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:13 smithi039 ceph-mon[29586]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-09-14T03:50:13.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:13 smithi039 ceph-mon[40962]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-09-14T03:50:13.800 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:13 smithi129 ceph-mon[33173]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-09-14T03:50:15.433 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:15 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-14T03:50:15.434 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:15 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:15.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:15 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-14T03:50:15.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:15 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:15.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:15 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-09-14T03:50:15.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:15 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:16.303 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:16 smithi129 ceph-mon[33173]: Deploying daemon osd.5 on smithi129 2024-09-14T03:50:16.304 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:16 smithi129 ceph-mon[33173]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:50:16.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:16 smithi039 ceph-mon[29586]: Deploying daemon osd.5 on smithi129 2024-09-14T03:50:16.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:16 smithi039 ceph-mon[29586]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:50:16.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:16 smithi039 ceph-mon[40962]: Deploying daemon osd.5 on smithi129 2024-09-14T03:50:16.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:16 smithi039 ceph-mon[40962]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:50:17.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:17 smithi129 ceph-mon[33173]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:50:18.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:17 smithi039 ceph-mon[29586]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:50:18.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:17 smithi039 ceph-mon[40962]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:50:19.846 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:19 smithi129 ceph-mon[33173]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:50:20.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:19 smithi039 ceph-mon[29586]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:50:20.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:19 smithi039 ceph-mon[40962]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:50:21.010 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:20 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:21.010 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:20 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:21.010 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:20 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:21.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:20 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:21.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:20 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:21.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:20 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:21.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:20 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:21.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:20 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:21.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:20 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:21.952 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:21 smithi129 ceph-mon[33173]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:50:22.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:21 smithi039 ceph-mon[29586]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:50:22.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:21 smithi039 ceph-mon[40962]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:50:23.244 INFO:teuthology.orchestra.run.smithi129.stdout:Created osd(s) 5 on host 'smithi129' 2024-09-14T03:50:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:23 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:23 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:23 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:23 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:23 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:23 smithi129 ceph-mon[33173]: from='osd.5 [v2:172.21.15.129:6808/2184993002,v1:172.21.15.129:6809/2184993002]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-14T03:50:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:23 smithi129 ceph-mon[33173]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-14T03:50:23.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:23 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:23.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:23 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:23.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:23 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:23.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:23 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:23.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:23 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:23.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:23 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:23.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:23 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:23.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:23 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:23.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:23 smithi039 ceph-mon[40962]: from='osd.5 [v2:172.21.15.129:6808/2184993002,v1:172.21.15.129:6809/2184993002]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-14T03:50:23.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:23 smithi039 ceph-mon[40962]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-14T03:50:23.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:23 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:23.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:23 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:23.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:23 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:23.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:23 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:23.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:23 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:23.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:23 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:23.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:23 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:23.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:23 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:23.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:23 smithi039 ceph-mon[29586]: from='osd.5 [v2:172.21.15.129:6808/2184993002,v1:172.21.15.129:6809/2184993002]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-14T03:50:23.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:23 smithi039 ceph-mon[29586]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-09-14T03:50:23.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:23 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:23.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:23 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:23.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:23 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:23.884 DEBUG:teuthology.orchestra.run.smithi129:osd.5> sudo journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.5.service 2024-09-14T03:50:23.887 INFO:tasks.cephadm:Deploying osd.6 on smithi129 with /dev/vg_nvme/lv_2... 2024-09-14T03:50:23.887 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_2 2024-09-14T03:50:24.619 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:50:24.740 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:24 smithi129 ceph-mon[33173]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-14T03:50:24.740 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:24 smithi129 ceph-mon[33173]: osdmap e37: 6 total, 5 up, 6 in 2024-09-14T03:50:24.740 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:24 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:50:24.740 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:24 smithi129 ceph-mon[33173]: from='osd.5 [v2:172.21.15.129:6808/2184993002,v1:172.21.15.129:6809/2184993002]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]: dispatch 2024-09-14T03:50:24.740 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:24 smithi129 ceph-mon[33173]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]: dispatch 2024-09-14T03:50:24.740 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:24 smithi129 ceph-mon[33173]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:50:24.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:24 smithi039 ceph-mon[29586]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-14T03:50:24.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:24 smithi039 ceph-mon[29586]: osdmap e37: 6 total, 5 up, 6 in 2024-09-14T03:50:24.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:24 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:50:24.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:24 smithi039 ceph-mon[29586]: from='osd.5 [v2:172.21.15.129:6808/2184993002,v1:172.21.15.129:6809/2184993002]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]: dispatch 2024-09-14T03:50:24.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:24 smithi039 ceph-mon[29586]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]: dispatch 2024-09-14T03:50:24.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:24 smithi039 ceph-mon[29586]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:50:24.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:24 smithi039 ceph-mon[40962]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-09-14T03:50:24.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:24 smithi039 ceph-mon[40962]: osdmap e37: 6 total, 5 up, 6 in 2024-09-14T03:50:24.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:24 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:50:24.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:24 smithi039 ceph-mon[40962]: from='osd.5 [v2:172.21.15.129:6808/2184993002,v1:172.21.15.129:6809/2184993002]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]: dispatch 2024-09-14T03:50:24.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:24 smithi039 ceph-mon[40962]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]: dispatch 2024-09-14T03:50:24.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:24 smithi039 ceph-mon[40962]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:50:25.021 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 03:50:24 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T03:50:24.943+0000 7fe32d782640 -1 osd.5 0 waiting for initial osdmap 2024-09-14T03:50:25.021 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 03:50:24 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T03:50:24.950+0000 7fe3255a3640 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:50:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:25 smithi129 ceph-mon[33173]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]': finished 2024-09-14T03:50:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:25 smithi129 ceph-mon[33173]: osdmap e38: 6 total, 5 up, 6 in 2024-09-14T03:50:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:25 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:50:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:25 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:50:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:25 smithi129 ceph-mon[33173]: from='osd.5 ' entity='osd.5' 2024-09-14T03:50:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:25 smithi039 ceph-mon[29586]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]': finished 2024-09-14T03:50:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:25 smithi039 ceph-mon[29586]: osdmap e38: 6 total, 5 up, 6 in 2024-09-14T03:50:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:25 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:50:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:25 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:50:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:25 smithi039 ceph-mon[29586]: from='osd.5 ' entity='osd.5' 2024-09-14T03:50:25.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:25 smithi039 ceph-mon[40962]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]': finished 2024-09-14T03:50:25.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:25 smithi039 ceph-mon[40962]: osdmap e38: 6 total, 5 up, 6 in 2024-09-14T03:50:25.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:25 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:50:25.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:25 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:50:25.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:25 smithi039 ceph-mon[40962]: from='osd.5 ' entity='osd.5' 2024-09-14T03:50:26.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:26 smithi039 ceph-mon[29586]: purged_snaps scrub starts 2024-09-14T03:50:26.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:26 smithi039 ceph-mon[29586]: purged_snaps scrub ok 2024-09-14T03:50:26.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:26 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:50:26.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:26 smithi039 ceph-mon[29586]: osd.5 [v2:172.21.15.129:6808/2184993002,v1:172.21.15.129:6809/2184993002] boot 2024-09-14T03:50:26.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:26 smithi039 ceph-mon[29586]: osdmap e39: 6 total, 6 up, 6 in 2024-09-14T03:50:26.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:26 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:50:26.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:26 smithi039 ceph-mon[29586]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:50:26.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:26 smithi039 ceph-mon[40962]: purged_snaps scrub starts 2024-09-14T03:50:26.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:26 smithi039 ceph-mon[40962]: purged_snaps scrub ok 2024-09-14T03:50:26.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:26 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:50:26.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:26 smithi039 ceph-mon[40962]: osd.5 [v2:172.21.15.129:6808/2184993002,v1:172.21.15.129:6809/2184993002] boot 2024-09-14T03:50:26.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:26 smithi039 ceph-mon[40962]: osdmap e39: 6 total, 6 up, 6 in 2024-09-14T03:50:26.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:26 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:50:26.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:26 smithi039 ceph-mon[40962]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:50:26.985 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:26 smithi129 ceph-mon[33173]: purged_snaps scrub starts 2024-09-14T03:50:26.985 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:26 smithi129 ceph-mon[33173]: purged_snaps scrub ok 2024-09-14T03:50:26.985 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:26 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:50:26.985 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:26 smithi129 ceph-mon[33173]: osd.5 [v2:172.21.15.129:6808/2184993002,v1:172.21.15.129:6809/2184993002] boot 2024-09-14T03:50:26.985 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:26 smithi129 ceph-mon[33173]: osdmap e39: 6 total, 6 up, 6 in 2024-09-14T03:50:26.985 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:26 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:50:26.985 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:26 smithi129 ceph-mon[33173]: pgmap v132: 1 pgs: 1 active+clean; 577 KiB data, 135 MiB used, 447 GiB / 447 GiB avail 2024-09-14T03:50:27.723 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:27 smithi129 ceph-mon[33173]: osdmap e40: 6 total, 6 up, 6 in 2024-09-14T03:50:27.724 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:27 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:27.724 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:27 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:27.724 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:27 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:50:27.724 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:27 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:50:27.724 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:27 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:27.724 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:27 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:27.724 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:27 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:27.724 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:27 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:28.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:27 smithi039 ceph-mon[29586]: osdmap e40: 6 total, 6 up, 6 in 2024-09-14T03:50:28.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:27 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:28.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:27 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:28.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:27 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:50:28.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:27 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:50:28.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:27 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:28.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:27 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:28.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:27 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:28.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:27 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:28.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:27 smithi039 ceph-mon[40962]: osdmap e40: 6 total, 6 up, 6 in 2024-09-14T03:50:28.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:27 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:28.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:27 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:28.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:27 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:50:28.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:27 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:50:28.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:27 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:28.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:27 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:28.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:27 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:28.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:27 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:28.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:28 smithi129 ceph-mon[33173]: Detected new or changed devices on smithi129 2024-09-14T03:50:28.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:28 smithi129 ceph-mon[33173]: Adjusting osd_memory_target on smithi129 to 8544M 2024-09-14T03:50:28.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:28 smithi129 ceph-mon[33173]: pgmap v134: 1 pgs: 1 remapped; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:50:28.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:28 smithi129 ceph-mon[33173]: osdmap e41: 6 total, 6 up, 6 in 2024-09-14T03:50:28.812 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:50:28.831 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi129:vg_nvme/lv_2 2024-09-14T03:50:29.025 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:50:29.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:28 smithi039 ceph-mon[29586]: Detected new or changed devices on smithi129 2024-09-14T03:50:29.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:28 smithi039 ceph-mon[29586]: Adjusting osd_memory_target on smithi129 to 8544M 2024-09-14T03:50:29.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:28 smithi039 ceph-mon[29586]: pgmap v134: 1 pgs: 1 remapped; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:50:29.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:28 smithi039 ceph-mon[29586]: osdmap e41: 6 total, 6 up, 6 in 2024-09-14T03:50:29.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:28 smithi039 ceph-mon[40962]: Detected new or changed devices on smithi129 2024-09-14T03:50:29.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:28 smithi039 ceph-mon[40962]: Adjusting osd_memory_target on smithi129 to 8544M 2024-09-14T03:50:29.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:28 smithi039 ceph-mon[40962]: pgmap v134: 1 pgs: 1 remapped; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:50:29.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:28 smithi039 ceph-mon[40962]: osdmap e41: 6 total, 6 up, 6 in 2024-09-14T03:50:29.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:29 smithi129 ceph-mon[33173]: osdmap e42: 6 total, 6 up, 6 in 2024-09-14T03:50:30.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:29 smithi039 ceph-mon[29586]: osdmap e42: 6 total, 6 up, 6 in 2024-09-14T03:50:30.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:29 smithi039 ceph-mon[40962]: osdmap e42: 6 total, 6 up, 6 in 2024-09-14T03:50:31.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:30 smithi129 ceph-mon[33173]: pgmap v137: 1 pgs: 1 remapped; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:50:31.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:30 smithi129 ceph-mon[33173]: osdmap e43: 6 total, 6 up, 6 in 2024-09-14T03:50:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:30 smithi039 ceph-mon[29586]: pgmap v137: 1 pgs: 1 remapped; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:50:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:30 smithi039 ceph-mon[29586]: osdmap e43: 6 total, 6 up, 6 in 2024-09-14T03:50:31.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:30 smithi039 ceph-mon[40962]: pgmap v137: 1 pgs: 1 remapped; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:50:31.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:30 smithi039 ceph-mon[40962]: osdmap e43: 6 total, 6 up, 6 in 2024-09-14T03:50:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:31 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:50:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:31 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:50:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:31 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:32.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:31 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:50:32.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:31 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:50:32.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:31 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:32.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:31 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:50:32.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:31 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:50:32.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:31 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:32.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:32 smithi129 ceph-mon[33173]: from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi129:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:32.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:32 smithi129 ceph-mon[33173]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 163 MiB used, 536 GiB / 536 GiB avail; 115 KiB/s, 0 objects/s recovering 2024-09-14T03:50:33.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:32 smithi039 ceph-mon[29586]: from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi129:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:33.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:32 smithi039 ceph-mon[29586]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 163 MiB used, 536 GiB / 536 GiB avail; 115 KiB/s, 0 objects/s recovering 2024-09-14T03:50:33.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:32 smithi039 ceph-mon[40962]: from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi129:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:33.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:32 smithi039 ceph-mon[40962]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 163 MiB used, 536 GiB / 536 GiB avail; 115 KiB/s, 0 objects/s recovering 2024-09-14T03:50:34.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:33 smithi129 ceph-mon[33173]: from='client.? 172.21.15.129:0/647946960' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "79e9c0a2-86a4-4267-a799-059d473d7b74"}]: dispatch 2024-09-14T03:50:34.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:33 smithi129 ceph-mon[33173]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "79e9c0a2-86a4-4267-a799-059d473d7b74"}]: dispatch 2024-09-14T03:50:34.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:33 smithi129 ceph-mon[33173]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "79e9c0a2-86a4-4267-a799-059d473d7b74"}]': finished 2024-09-14T03:50:34.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:33 smithi129 ceph-mon[33173]: osdmap e44: 7 total, 6 up, 7 in 2024-09-14T03:50:34.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:33 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:50:34.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:33 smithi129 ceph-mon[33173]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 163 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-14T03:50:34.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:33 smithi039 ceph-mon[29586]: from='client.? 172.21.15.129:0/647946960' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "79e9c0a2-86a4-4267-a799-059d473d7b74"}]: dispatch 2024-09-14T03:50:34.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:33 smithi039 ceph-mon[29586]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "79e9c0a2-86a4-4267-a799-059d473d7b74"}]: dispatch 2024-09-14T03:50:34.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:33 smithi039 ceph-mon[29586]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "79e9c0a2-86a4-4267-a799-059d473d7b74"}]': finished 2024-09-14T03:50:34.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:33 smithi039 ceph-mon[29586]: osdmap e44: 7 total, 6 up, 7 in 2024-09-14T03:50:34.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:33 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:50:34.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:33 smithi039 ceph-mon[29586]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 163 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-14T03:50:34.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:33 smithi039 ceph-mon[40962]: from='client.? 172.21.15.129:0/647946960' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "79e9c0a2-86a4-4267-a799-059d473d7b74"}]: dispatch 2024-09-14T03:50:34.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:33 smithi039 ceph-mon[40962]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "79e9c0a2-86a4-4267-a799-059d473d7b74"}]: dispatch 2024-09-14T03:50:34.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:33 smithi039 ceph-mon[40962]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "79e9c0a2-86a4-4267-a799-059d473d7b74"}]': finished 2024-09-14T03:50:34.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:33 smithi039 ceph-mon[40962]: osdmap e44: 7 total, 6 up, 7 in 2024-09-14T03:50:34.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:33 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:50:34.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:33 smithi039 ceph-mon[40962]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 163 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-09-14T03:50:35.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:34 smithi129 ceph-mon[33173]: from='client.? 172.21.15.129:0/3564087556' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:50:35.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:34 smithi039 ceph-mon[29586]: from='client.? 172.21.15.129:0/3564087556' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:50:35.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:34 smithi039 ceph-mon[40962]: from='client.? 172.21.15.129:0/3564087556' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:50:35.897 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:35 smithi129 ceph-mon[33173]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-09-14T03:50:36.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:35 smithi039 ceph-mon[29586]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-09-14T03:50:36.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:35 smithi039 ceph-mon[40962]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 82 KiB/s, 0 objects/s recovering 2024-09-14T03:50:37.812 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:37 smithi129 ceph-mon[33173]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-14T03:50:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:37 smithi039 ceph-mon[29586]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-14T03:50:38.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:37 smithi039 ceph-mon[40962]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-14T03:50:39.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:39 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-14T03:50:39.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:39 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:39.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:39 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-14T03:50:39.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:39 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:39.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:39 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-09-14T03:50:39.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:39 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:40.400 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:40 smithi129 ceph-mon[33173]: Deploying daemon osd.6 on smithi129 2024-09-14T03:50:40.400 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:40 smithi129 ceph-mon[33173]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:50:40.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:40 smithi039 ceph-mon[29586]: Deploying daemon osd.6 on smithi129 2024-09-14T03:50:40.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:40 smithi039 ceph-mon[29586]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:50:40.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:40 smithi039 ceph-mon[40962]: Deploying daemon osd.6 on smithi129 2024-09-14T03:50:40.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:40 smithi039 ceph-mon[40962]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:50:42.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:41 smithi129 ceph-mon[33173]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:50:42.048 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:41 smithi039 ceph-mon[29586]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:50:42.048 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:41 smithi039 ceph-mon[40962]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:50:44.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:43 smithi129 ceph-mon[33173]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:50:44.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:43 smithi039 ceph-mon[29586]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:50:44.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:43 smithi039 ceph-mon[40962]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:50:45.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:44 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:45.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:44 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:45.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:44 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:44 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:44 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:44 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:45.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:44 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:45.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:44 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:45.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:44 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:45.719 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:45 smithi129 ceph-mon[33173]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:50:46.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:45 smithi039 ceph-mon[29586]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:50:46.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:45 smithi039 ceph-mon[40962]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:50:46.973 INFO:teuthology.orchestra.run.smithi129.stdout:Created osd(s) 6 on host 'smithi129' 2024-09-14T03:50:47.594 DEBUG:teuthology.orchestra.run.smithi129:osd.6> sudo journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.6.service 2024-09-14T03:50:47.596 INFO:tasks.cephadm:Deploying osd.7 on smithi129 with /dev/vg_nvme/lv_1... 2024-09-14T03:50:47.597 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef ceph-volume -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- lvm zap /dev/vg_nvme/lv_1 2024-09-14T03:50:47.840 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:50:48.230 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:47 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:48.230 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:47 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:48.230 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:47 smithi129 ceph-mon[33173]: from='osd.6 [v2:172.21.15.129:6816/3767031353,v1:172.21.15.129:6817/3767031353]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-14T03:50:48.230 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:47 smithi129 ceph-mon[33173]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-14T03:50:48.230 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:47 smithi129 ceph-mon[33173]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:50:48.230 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:47 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:48.230 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:47 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:48.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:47 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:48.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:47 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:48.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:47 smithi039 ceph-mon[29586]: from='osd.6 [v2:172.21.15.129:6816/3767031353,v1:172.21.15.129:6817/3767031353]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-14T03:50:48.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:47 smithi039 ceph-mon[29586]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-14T03:50:48.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:47 smithi039 ceph-mon[29586]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:50:48.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:47 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:48.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:47 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:48.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:47 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:48.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:47 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:48.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:47 smithi039 ceph-mon[40962]: from='osd.6 [v2:172.21.15.129:6816/3767031353,v1:172.21.15.129:6817/3767031353]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-14T03:50:48.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:47 smithi039 ceph-mon[40962]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-09-14T03:50:48.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:47 smithi039 ceph-mon[40962]: pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:50:48.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:47 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:48.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:47 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:49.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:49 smithi129 ceph-mon[33173]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-14T03:50:49.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:49 smithi129 ceph-mon[33173]: osdmap e45: 7 total, 6 up, 7 in 2024-09-14T03:50:49.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:49 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:50:49.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:49 smithi129 ceph-mon[33173]: from='osd.6 [v2:172.21.15.129:6816/3767031353,v1:172.21.15.129:6817/3767031353]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]: dispatch 2024-09-14T03:50:49.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:49 smithi129 ceph-mon[33173]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]: dispatch 2024-09-14T03:50:49.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:48 smithi039 ceph-mon[29586]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-14T03:50:49.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:48 smithi039 ceph-mon[29586]: osdmap e45: 7 total, 6 up, 7 in 2024-09-14T03:50:49.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:48 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:50:49.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:48 smithi039 ceph-mon[29586]: from='osd.6 [v2:172.21.15.129:6816/3767031353,v1:172.21.15.129:6817/3767031353]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]: dispatch 2024-09-14T03:50:49.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:48 smithi039 ceph-mon[29586]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]: dispatch 2024-09-14T03:50:49.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:48 smithi039 ceph-mon[40962]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-09-14T03:50:49.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:48 smithi039 ceph-mon[40962]: osdmap e45: 7 total, 6 up, 7 in 2024-09-14T03:50:49.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:48 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:50:49.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:48 smithi039 ceph-mon[40962]: from='osd.6 [v2:172.21.15.129:6816/3767031353,v1:172.21.15.129:6817/3767031353]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]: dispatch 2024-09-14T03:50:49.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:48 smithi039 ceph-mon[40962]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]: dispatch 2024-09-14T03:50:49.771 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 03:50:49 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T03:50:49.415+0000 7f079892f640 -1 osd.6 0 waiting for initial osdmap 2024-09-14T03:50:49.772 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 03:50:49 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T03:50:49.421+0000 7f0790750640 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:50:50.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:50 smithi129 ceph-mon[33173]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]': finished 2024-09-14T03:50:50.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:50 smithi129 ceph-mon[33173]: osdmap e46: 7 total, 6 up, 7 in 2024-09-14T03:50:50.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:50 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:50:50.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:50 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:50:50.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:50 smithi129 ceph-mon[33173]: from='osd.6 ' entity='osd.6' 2024-09-14T03:50:50.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:50 smithi129 ceph-mon[33173]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:50:50.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:50 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:50:50.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:50 smithi039 ceph-mon[29586]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]': finished 2024-09-14T03:50:50.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:50 smithi039 ceph-mon[29586]: osdmap e46: 7 total, 6 up, 7 in 2024-09-14T03:50:50.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:50 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:50:50.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:50 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:50:50.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:50 smithi039 ceph-mon[29586]: from='osd.6 ' entity='osd.6' 2024-09-14T03:50:50.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:50 smithi039 ceph-mon[29586]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:50:50.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:50 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:50:50.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:50 smithi039 ceph-mon[40962]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]': finished 2024-09-14T03:50:50.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:50 smithi039 ceph-mon[40962]: osdmap e46: 7 total, 6 up, 7 in 2024-09-14T03:50:50.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:50 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:50:50.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:50 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:50:50.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:50 smithi039 ceph-mon[40962]: from='osd.6 ' entity='osd.6' 2024-09-14T03:50:50.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:50 smithi039 ceph-mon[40962]: pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 162 MiB used, 536 GiB / 536 GiB avail 2024-09-14T03:50:50.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:50 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:50:51.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:51 smithi039 ceph-mon[29586]: purged_snaps scrub starts 2024-09-14T03:50:51.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:51 smithi039 ceph-mon[29586]: purged_snaps scrub ok 2024-09-14T03:50:51.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:51 smithi039 ceph-mon[29586]: osd.6 [v2:172.21.15.129:6816/3767031353,v1:172.21.15.129:6817/3767031353] boot 2024-09-14T03:50:51.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:51 smithi039 ceph-mon[29586]: osdmap e47: 7 total, 7 up, 7 in 2024-09-14T03:50:51.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:51 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:50:51.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:51 smithi039 ceph-mon[40962]: purged_snaps scrub starts 2024-09-14T03:50:51.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:51 smithi039 ceph-mon[40962]: purged_snaps scrub ok 2024-09-14T03:50:51.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:51 smithi039 ceph-mon[40962]: osd.6 [v2:172.21.15.129:6816/3767031353,v1:172.21.15.129:6817/3767031353] boot 2024-09-14T03:50:51.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:51 smithi039 ceph-mon[40962]: osdmap e47: 7 total, 7 up, 7 in 2024-09-14T03:50:51.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:51 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:50:51.325 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:51 smithi129 ceph-mon[33173]: purged_snaps scrub starts 2024-09-14T03:50:51.325 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:51 smithi129 ceph-mon[33173]: purged_snaps scrub ok 2024-09-14T03:50:51.325 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:51 smithi129 ceph-mon[33173]: osd.6 [v2:172.21.15.129:6816/3767031353,v1:172.21.15.129:6817/3767031353] boot 2024-09-14T03:50:51.326 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:51 smithi129 ceph-mon[33173]: osdmap e47: 7 total, 7 up, 7 in 2024-09-14T03:50:51.326 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:51 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:50:51.328 INFO:teuthology.orchestra.run.smithi129.stdout: 2024-09-14T03:50:51.350 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph orch daemon add osd smithi129:vg_nvme/lv_1 2024-09-14T03:50:51.835 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:50:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:52 smithi129 ceph-mon[33173]: osdmap e48: 7 total, 7 up, 7 in 2024-09-14T03:50:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:52 smithi129 ceph-mon[33173]: pgmap v154: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:50:52.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:52 smithi039 ceph-mon[29586]: osdmap e48: 7 total, 7 up, 7 in 2024-09-14T03:50:52.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:52 smithi039 ceph-mon[29586]: pgmap v154: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:50:52.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:52 smithi039 ceph-mon[40962]: osdmap e48: 7 total, 7 up, 7 in 2024-09-14T03:50:52.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:52 smithi039 ceph-mon[40962]: pgmap v154: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:50:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:53 smithi129 ceph-mon[33173]: osdmap e49: 7 total, 7 up, 7 in 2024-09-14T03:50:53.441 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:53 smithi039 ceph-mon[29586]: osdmap e49: 7 total, 7 up, 7 in 2024-09-14T03:50:53.442 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:53 smithi039 ceph-mon[40962]: osdmap e49: 7 total, 7 up, 7 in 2024-09-14T03:50:54.100 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:54 smithi129 ceph-mon[33173]: osdmap e50: 7 total, 7 up, 7 in 2024-09-14T03:50:54.100 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:54 smithi129 ceph-mon[33173]: pgmap v157: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:50:54.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:54 smithi039 ceph-mon[29586]: osdmap e50: 7 total, 7 up, 7 in 2024-09-14T03:50:54.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:54 smithi039 ceph-mon[29586]: pgmap v157: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:50:54.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:54 smithi039 ceph-mon[40962]: osdmap e50: 7 total, 7 up, 7 in 2024-09-14T03:50:54.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:54 smithi039 ceph-mon[40962]: pgmap v157: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:50:55.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:55 smithi129 ceph-mon[33173]: osdmap e51: 7 total, 7 up, 7 in 2024-09-14T03:50:55.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:55 smithi129 ceph-mon[33173]: from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi129:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:55.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:50:55.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:50:55.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:55.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:55 smithi129 ceph-mon[33173]: Detected new or changed devices on smithi129 2024-09-14T03:50:55.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:55.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:55.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:50:55.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:50:55.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:50:55.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:55 smithi129 ceph-mon[33173]: Adjusting osd_memory_target on smithi129 to 5696M 2024-09-14T03:50:55.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:55.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:55.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:55.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:55.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:55.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:55.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:55.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:55 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:55.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[40962]: osdmap e51: 7 total, 7 up, 7 in 2024-09-14T03:50:55.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[40962]: from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi129:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:55.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:50:55.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:50:55.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:55.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[40962]: Detected new or changed devices on smithi129 2024-09-14T03:50:55.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:55.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:55.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:50:55.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:50:55.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:50:55.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[40962]: Adjusting osd_memory_target on smithi129 to 5696M 2024-09-14T03:50:55.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:55.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:55.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:55.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:55.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:55.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:55.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:55.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:55.542 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[29586]: osdmap e51: 7 total, 7 up, 7 in 2024-09-14T03:50:55.543 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[29586]: from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi129:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:50:55.543 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-09-14T03:50:55.543 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-09-14T03:50:55.543 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:55.543 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[29586]: Detected new or changed devices on smithi129 2024-09-14T03:50:55.543 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:55.543 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:55.543 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:50:55.544 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:50:55.544 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:50:55.544 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[29586]: Adjusting osd_memory_target on smithi129 to 5696M 2024-09-14T03:50:55.544 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:55.544 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:55.544 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:55.544 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:55.545 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:50:55.545 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:50:55.545 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:50:55.545 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:55 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:50:56.260 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:56 smithi129 ceph-mon[33173]: pgmap v159: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:50:56.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:56 smithi039 ceph-mon[29586]: pgmap v159: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:50:56.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:56 smithi039 ceph-mon[40962]: pgmap v159: 1 pgs: 1 remapped+peering; 577 KiB data, 189 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:50:57.159 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:57 smithi129 ceph-mon[33173]: from='client.? 172.21.15.129:0/2464707694' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "22f41628-4b88-4b8c-82b2-fcd2802da1bf"}]: dispatch 2024-09-14T03:50:57.160 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:57 smithi129 ceph-mon[33173]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "22f41628-4b88-4b8c-82b2-fcd2802da1bf"}]: dispatch 2024-09-14T03:50:57.160 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:57 smithi129 ceph-mon[33173]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "22f41628-4b88-4b8c-82b2-fcd2802da1bf"}]': finished 2024-09-14T03:50:57.160 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:57 smithi129 ceph-mon[33173]: osdmap e52: 8 total, 7 up, 8 in 2024-09-14T03:50:57.160 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:57 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:50:57.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:57 smithi039 ceph-mon[29586]: from='client.? 172.21.15.129:0/2464707694' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "22f41628-4b88-4b8c-82b2-fcd2802da1bf"}]: dispatch 2024-09-14T03:50:57.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:57 smithi039 ceph-mon[29586]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "22f41628-4b88-4b8c-82b2-fcd2802da1bf"}]: dispatch 2024-09-14T03:50:57.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:57 smithi039 ceph-mon[29586]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "22f41628-4b88-4b8c-82b2-fcd2802da1bf"}]': finished 2024-09-14T03:50:57.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:57 smithi039 ceph-mon[29586]: osdmap e52: 8 total, 7 up, 8 in 2024-09-14T03:50:57.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:57 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:50:57.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:57 smithi039 ceph-mon[40962]: from='client.? 172.21.15.129:0/2464707694' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "22f41628-4b88-4b8c-82b2-fcd2802da1bf"}]: dispatch 2024-09-14T03:50:57.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:57 smithi039 ceph-mon[40962]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "22f41628-4b88-4b8c-82b2-fcd2802da1bf"}]: dispatch 2024-09-14T03:50:57.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:57 smithi039 ceph-mon[40962]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "22f41628-4b88-4b8c-82b2-fcd2802da1bf"}]': finished 2024-09-14T03:50:57.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:57 smithi039 ceph-mon[40962]: osdmap e52: 8 total, 7 up, 8 in 2024-09-14T03:50:57.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:57 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:50:58.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:58 smithi129 ceph-mon[33173]: from='client.? 172.21.15.129:0/1600766106' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:50:58.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:58 smithi129 ceph-mon[33173]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 105 KiB/s, 0 objects/s recovering 2024-09-14T03:50:58.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:58 smithi039 ceph-mon[29586]: from='client.? 172.21.15.129:0/1600766106' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:50:58.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:58 smithi039 ceph-mon[29586]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 105 KiB/s, 0 objects/s recovering 2024-09-14T03:50:58.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:58 smithi039 ceph-mon[40962]: from='client.? 172.21.15.129:0/1600766106' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-09-14T03:50:58.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:58 smithi039 ceph-mon[40962]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 105 KiB/s, 0 objects/s recovering 2024-09-14T03:51:00.014 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:50:59 smithi129 ceph-mon[33173]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 89 KiB/s, 0 objects/s recovering 2024-09-14T03:51:00.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:50:59 smithi039 ceph-mon[29586]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 89 KiB/s, 0 objects/s recovering 2024-09-14T03:51:00.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:50:59 smithi039 ceph-mon[40962]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 89 KiB/s, 0 objects/s recovering 2024-09-14T03:51:01.952 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:01 smithi129 ceph-mon[33173]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-14T03:51:02.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:01 smithi039 ceph-mon[40962]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-14T03:51:02.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:01 smithi039 ceph-mon[29586]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-14T03:51:02.913 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:02 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-14T03:51:02.913 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:02 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:02.913 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:02 smithi129 ceph-mon[33173]: Deploying daemon osd.7 on smithi129 2024-09-14T03:51:03.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:02 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-14T03:51:03.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:02 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:03.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:02 smithi039 ceph-mon[29586]: Deploying daemon osd.7 on smithi129 2024-09-14T03:51:03.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:02 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-09-14T03:51:03.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:02 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:03.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:02 smithi039 ceph-mon[40962]: Deploying daemon osd.7 on smithi129 2024-09-14T03:51:03.842 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:03 smithi129 ceph-mon[33173]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-09-14T03:51:04.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:03 smithi039 ceph-mon[40962]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-09-14T03:51:04.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:03 smithi039 ceph-mon[29586]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-09-14T03:51:05.911 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:05 smithi129 ceph-mon[33173]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:51:06.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:05 smithi039 ceph-mon[29586]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:51:06.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:05 smithi039 ceph-mon[40962]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:51:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:07 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:07 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:07 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:07.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:07 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:07.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:07 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:07.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:07 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:07.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:07 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:07.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:07 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:07.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:07 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:08.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:08 smithi039 ceph-mon[40962]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:51:08.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:08 smithi039 ceph-mon[29586]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:51:08.843 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:08 smithi129 ceph-mon[33173]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:51:09.741 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:09 smithi129 ceph-mon[33173]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:51:10.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:09 smithi039 ceph-mon[29586]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:51:10.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:09 smithi039 ceph-mon[40962]: pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:51:10.991 INFO:teuthology.orchestra.run.smithi129.stdout:Created osd(s) 7 on host 'smithi129' 2024-09-14T03:51:11.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:11 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:11.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:11 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:11.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:11 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:11.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:11 smithi129 ceph-mon[33173]: from='osd.7 [v2:172.21.15.129:6824/2370414690,v1:172.21.15.129:6825/2370414690]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-14T03:51:11.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:11 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:51:11.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:11 smithi129 ceph-mon[33173]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-14T03:51:11.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:11 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:11.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:11 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:11.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:11 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:11.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:11 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:11.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:11 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:11.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:11 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:11.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:11 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:11.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:11 smithi039 ceph-mon[40962]: from='osd.7 [v2:172.21.15.129:6824/2370414690,v1:172.21.15.129:6825/2370414690]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-14T03:51:11.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:11 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:51:11.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:11 smithi039 ceph-mon[40962]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-14T03:51:11.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:11 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:11.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:11 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:11.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:11 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:11.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:11 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:11.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:11 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:11.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:11 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:11.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:11 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:11.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:11 smithi039 ceph-mon[29586]: from='osd.7 [v2:172.21.15.129:6824/2370414690,v1:172.21.15.129:6825/2370414690]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-14T03:51:11.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:11 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:51:11.542 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:11 smithi039 ceph-mon[29586]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-09-14T03:51:11.542 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:11 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:11.542 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:11 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:11.542 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:11 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:11.542 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:11 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:11.639 DEBUG:teuthology.orchestra.run.smithi129:osd.7> sudo journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.7.service 2024-09-14T03:51:11.641 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-09-14T03:51:11.642 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd stat -f json 2024-09-14T03:51:11.859 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:51:12.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:12 smithi129 ceph-mon[33173]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-14T03:51:12.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:12 smithi129 ceph-mon[33173]: osdmap e53: 8 total, 7 up, 8 in 2024-09-14T03:51:12.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:12 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:51:12.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:12 smithi129 ceph-mon[33173]: from='osd.7 [v2:172.21.15.129:6824/2370414690,v1:172.21.15.129:6825/2370414690]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]: dispatch 2024-09-14T03:51:12.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:12 smithi129 ceph-mon[33173]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]: dispatch 2024-09-14T03:51:12.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:12 smithi129 ceph-mon[33173]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:51:12.522 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 03:51:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T03:51:12.240+0000 7ff015d9e640 -1 osd.7 0 waiting for initial osdmap 2024-09-14T03:51:12.522 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 03:51:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T03:51:12.247+0000 7ff011bc7640 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T03:51:12.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:12 smithi039 ceph-mon[29586]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-14T03:51:12.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:12 smithi039 ceph-mon[29586]: osdmap e53: 8 total, 7 up, 8 in 2024-09-14T03:51:12.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:12 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:51:12.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:12 smithi039 ceph-mon[29586]: from='osd.7 [v2:172.21.15.129:6824/2370414690,v1:172.21.15.129:6825/2370414690]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]: dispatch 2024-09-14T03:51:12.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:12 smithi039 ceph-mon[29586]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]: dispatch 2024-09-14T03:51:12.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:12 smithi039 ceph-mon[29586]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:51:12.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:12 smithi039 ceph-mon[40962]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-09-14T03:51:12.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:12 smithi039 ceph-mon[40962]: osdmap e53: 8 total, 7 up, 8 in 2024-09-14T03:51:12.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:12 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:51:12.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:12 smithi039 ceph-mon[40962]: from='osd.7 [v2:172.21.15.129:6824/2370414690,v1:172.21.15.129:6825/2370414690]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]: dispatch 2024-09-14T03:51:12.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:12 smithi039 ceph-mon[40962]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]: dispatch 2024-09-14T03:51:12.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:12 smithi039 ceph-mon[40962]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 190 MiB used, 626 GiB / 626 GiB avail 2024-09-14T03:51:13.506 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:13 smithi039 ceph-mon[29586]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]': finished 2024-09-14T03:51:13.506 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:13 smithi039 ceph-mon[29586]: osdmap e54: 8 total, 7 up, 8 in 2024-09-14T03:51:13.507 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:13 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:51:13.507 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:13 smithi039 ceph-mon[29586]: osd.7 [v2:172.21.15.129:6824/2370414690,v1:172.21.15.129:6825/2370414690] boot 2024-09-14T03:51:13.507 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:13 smithi039 ceph-mon[29586]: osdmap e55: 8 total, 8 up, 8 in 2024-09-14T03:51:13.507 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:13 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:51:13.507 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:13 smithi039 ceph-mon[40962]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]': finished 2024-09-14T03:51:13.507 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:13 smithi039 ceph-mon[40962]: osdmap e54: 8 total, 7 up, 8 in 2024-09-14T03:51:13.507 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:13 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:51:13.507 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:13 smithi039 ceph-mon[40962]: osd.7 [v2:172.21.15.129:6824/2370414690,v1:172.21.15.129:6825/2370414690] boot 2024-09-14T03:51:13.507 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:13 smithi039 ceph-mon[40962]: osdmap e55: 8 total, 8 up, 8 in 2024-09-14T03:51:13.507 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:13 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:51:13.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:13 smithi129 ceph-mon[33173]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi129", "root=default"]}]': finished 2024-09-14T03:51:13.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:13 smithi129 ceph-mon[33173]: osdmap e54: 8 total, 7 up, 8 in 2024-09-14T03:51:13.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:13 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:51:13.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:13 smithi129 ceph-mon[33173]: osd.7 [v2:172.21.15.129:6824/2370414690,v1:172.21.15.129:6825/2370414690] boot 2024-09-14T03:51:13.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:13 smithi129 ceph-mon[33173]: osdmap e55: 8 total, 8 up, 8 in 2024-09-14T03:51:13.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:13 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:51:13.924 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:51:14.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:14 smithi129 ceph-mon[33173]: purged_snaps scrub starts 2024-09-14T03:51:14.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:14 smithi129 ceph-mon[33173]: purged_snaps scrub ok 2024-09-14T03:51:14.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:14 smithi129 ceph-mon[33173]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:14.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:14 smithi129 ceph-mon[33173]: osdmap e56: 8 total, 8 up, 8 in 2024-09-14T03:51:14.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:14 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/3233066874' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-14T03:51:14.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:14 smithi039 ceph-mon[40962]: purged_snaps scrub starts 2024-09-14T03:51:14.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:14 smithi039 ceph-mon[40962]: purged_snaps scrub ok 2024-09-14T03:51:14.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:14 smithi039 ceph-mon[40962]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:14.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:14 smithi039 ceph-mon[40962]: osdmap e56: 8 total, 8 up, 8 in 2024-09-14T03:51:14.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:14 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/3233066874' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-14T03:51:14.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:14 smithi039 ceph-mon[29586]: purged_snaps scrub starts 2024-09-14T03:51:14.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:14 smithi039 ceph-mon[29586]: purged_snaps scrub ok 2024-09-14T03:51:14.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:14 smithi039 ceph-mon[29586]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:14.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:14 smithi039 ceph-mon[29586]: osdmap e56: 8 total, 8 up, 8 in 2024-09-14T03:51:14.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:14 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/3233066874' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-09-14T03:51:14.616 INFO:teuthology.orchestra.run.smithi039.stdout:{"epoch":56,"num_osds":8,"num_up_osds":8,"osd_up_since":1726285873,"num_in_osds":8,"osd_in_since":1726285856,"num_remapped_pgs":1} 2024-09-14T03:51:14.617 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T03:51:14.825 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:51:15.985 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:15 smithi039 ceph-mon[29586]: Detected new or changed devices on smithi129 2024-09-14T03:51:15.985 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:15 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:15.985 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:15 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:15.985 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:15 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:15.985 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:15 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:15.985 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:15 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:15.985 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:15 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:15.985 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:15 smithi039 ceph-mon[29586]: Adjusting osd_memory_target on smithi129 to 4272M 2024-09-14T03:51:15.985 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:15 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:15.986 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:15 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:15.986 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:15 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:51:15.986 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:15 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:15.986 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:15 smithi039 ceph-mon[29586]: osdmap e57: 8 total, 8 up, 8 in 2024-09-14T03:51:15.986 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:15 smithi039 ceph-mon[40962]: Detected new or changed devices on smithi129 2024-09-14T03:51:15.986 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:15 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:15.986 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:15 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:15.986 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:15 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:15.986 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:15 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:15.986 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:15 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:15.986 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:15 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:15.987 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:15 smithi039 ceph-mon[40962]: Adjusting osd_memory_target on smithi129 to 4272M 2024-09-14T03:51:15.987 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:15 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:15.987 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:15 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:15.987 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:15 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:51:15.987 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:15 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:15.987 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:15 smithi039 ceph-mon[40962]: osdmap e57: 8 total, 8 up, 8 in 2024-09-14T03:51:16.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:15 smithi129 ceph-mon[33173]: Detected new or changed devices on smithi129 2024-09-14T03:51:16.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:15 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:16.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:15 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:16.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:15 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:16.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:15 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:16.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:15 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:16.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:15 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:16.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:15 smithi129 ceph-mon[33173]: Adjusting osd_memory_target on smithi129 to 4272M 2024-09-14T03:51:16.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:15 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:16.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:15 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:16.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:15 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:51:16.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:15 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:16.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:15 smithi129 ceph-mon[33173]: osdmap e57: 8 total, 8 up, 8 in 2024-09-14T03:51:16.699 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:51:16.699 INFO:teuthology.orchestra.run.smithi039.stdout:{"epoch":59,"fsid":"bd7c803c-724b-11ef-bceb-c7b262605968","created":"2024-09-14T03:46:13.726476+0000","modified":"2024-09-14T03:51:16.662550+0000","last_up_change":"2024-09-14T03:51:13.230622+0000","last_in_change":"2024-09-14T03:50:56.569390+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:49:18.022402+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"5fdfd478-dd27-4674-a7d6-53177d9e549a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6802","nonce":1775094854},{"type":"v1","addr":"172.21.15.39:6803","nonce":1775094854}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6804","nonce":1775094854},{"type":"v1","addr":"172.21.15.39:6805","nonce":1775094854}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6808","nonce":1775094854},{"type":"v1","addr":"172.21.15.39:6809","nonce":1775094854}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6806","nonce":1775094854},{"type":"v1","addr":"172.21.15.39:6807","nonce":1775094854}]},"public_addr":"172.21.15.39:6803/1775094854","cluster_addr":"172.21.15.39:6805/1775094854","heartbeat_back_addr":"172.21.15.39:6809/1775094854","heartbeat_front_addr":"172.21.15.39:6807/1775094854","state":["exists","up"]},{"osd":1,"uuid":"991c236e-1679-4c1e-a1a9-3894b1b12055","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6810","nonce":2410143394},{"type":"v1","addr":"172.21.15.39:6811","nonce":2410143394}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6812","nonce":2410143394},{"type":"v1","addr":"172.21.15.39:6813","nonce":2410143394}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6816","nonce":2410143394},{"type":"v1","addr":"172.21.15.39:6817","nonce":2410143394}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6814","nonce":2410143394},{"type":"v1","addr":"172.21.15.39:6815","nonce":2410143394}]},"public_addr":"172.21.15.39:6811/2410143394","cluster_addr":"172.21.15.39:6813/2410143394","heartbeat_back_addr":"172.21.15.39:6817/2410143394","heartbeat_front_addr":"172.21.15.39:6815/2410143394","state":["exists","up"]},{"osd":2,"uuid":"f9611ffd-5d37-4465-8114-ecc70d4d6d14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6818","nonce":3600430167},{"type":"v1","addr":"172.21.15.39:6819","nonce":3600430167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6820","nonce":3600430167},{"type":"v1","addr":"172.21.15.39:6821","nonce":3600430167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6824","nonce":3600430167},{"type":"v1","addr":"172.21.15.39:6825","nonce":3600430167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6822","nonce":3600430167},{"type":"v1","addr":"172.21.15.39:6823","nonce":3600430167}]},"public_addr":"172.21.15.39:6819/3600430167","cluster_addr":"172.21.15.39:6821/3600430167","heartbeat_back_addr":"172.21.15.39:6825/3600430167","heartbeat_front_addr":"172.21.15.39:6823/3600430167","state":["exists","up"]},{"osd":3,"uuid":"56b6877b-2107-47d6-8da9-015db7352288","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6826","nonce":1027302150},{"type":"v1","addr":"172.21.15.39:6827","nonce":1027302150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6828","nonce":1027302150},{"type":"v1","addr":"172.21.15.39:6829","nonce":1027302150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6832","nonce":1027302150},{"type":"v1","addr":"172.21.15.39:6833","nonce":1027302150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6830","nonce":1027302150},{"type":"v1","addr":"172.21.15.39:6831","nonce":1027302150}]},"public_addr":"172.21.15.39:6827/1027302150","cluster_addr":"172.21.15.39:6829/1027302150","heartbeat_back_addr":"172.21.15.39:6833/1027302150","heartbeat_front_addr":"172.21.15.39:6831/1027302150","state":["exists","up"]},{"osd":4,"uuid":"2d5523c4-5871-4fb3-bc0d-62c3b7d8b6a1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6800","nonce":168261430},{"type":"v1","addr":"172.21.15.129:6801","nonce":168261430}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6802","nonce":168261430},{"type":"v1","addr":"172.21.15.129:6803","nonce":168261430}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6806","nonce":168261430},{"type":"v1","addr":"172.21.15.129:6807","nonce":168261430}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6804","nonce":168261430},{"type":"v1","addr":"172.21.15.129:6805","nonce":168261430}]},"public_addr":"172.21.15.129:6801/168261430","cluster_addr":"172.21.15.129:6803/168261430","heartbeat_back_addr":"172.21.15.129:6807/168261430","heartbeat_front_addr":"172.21.15.129:6805/168261430","state":["exists","up"]},{"osd":5,"uuid":"a3ba55fd-f32e-4f6d-9b48-d2845a3474cd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6808","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6809","nonce":2184993002}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6810","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6811","nonce":2184993002}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6814","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6815","nonce":2184993002}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6812","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6813","nonce":2184993002}]},"public_addr":"172.21.15.129:6809/2184993002","cluster_addr":"172.21.15.129:6811/2184993002","heartbeat_back_addr":"172.21.15.129:6815/2184993002","heartbeat_front_addr":"172.21.15.129:6813/2184993002","state":["exists","up"]},{"osd":6,"uuid":"79e9c0a2-86a4-4267-a799-059d473d7b74","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6816","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6817","nonce":3767031353}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6818","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6819","nonce":3767031353}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6822","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6823","nonce":3767031353}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6820","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6821","nonce":3767031353}]},"public_addr":"172.21.15.129:6817/3767031353","cluster_addr":"172.21.15.129:6819/3767031353","heartbeat_back_addr":"172.21.15.129:6823/3767031353","heartbeat_front_addr":"172.21.15.129:6821/3767031353","state":["exists","up"]},{"osd":7,"uuid":"22f41628-4b88-4b8c-82b2-fcd2802da1bf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6824","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6825","nonce":2370414690}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6826","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6827","nonce":2370414690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6830","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6831","nonce":2370414690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6828","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6829","nonce":2370414690}]},"public_addr":"172.21.15.129:6825/2370414690","cluster_addr":"172.21.15.129:6827/2370414690","heartbeat_back_addr":"172.21.15.129:6831/2370414690","heartbeat_front_addr":"172.21.15.129:6829/2370414690","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:48:23.355518+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:48:48.715441+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:49:13.415323+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:49:39.935205+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:50:01.649042+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:50:24.001678+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:50:47.985011+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:51:11.584825+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.39:6800/3127470450":"2024-09-15T03:46:59.511580+0000","172.21.15.39:0/209353267":"2024-09-15T03:46:59.511580+0000","172.21.15.39:0/1530637819":"2024-09-15T03:46:59.511580+0000","172.21.15.39:0/3161547781":"2024-09-15T03:46:59.511580+0000","172.21.15.39:0/2670041591":"2024-09-15T03:46:35.805097+0000","172.21.15.39:6801/3127470450":"2024-09-15T03:46:59.511580+0000","172.21.15.39:6800/628437919":"2024-09-15T03:46:35.805097+0000","172.21.15.39:6801/628437919":"2024-09-15T03:46:35.805097+0000","172.21.15.39:0/81449629":"2024-09-15T03:46:35.805097+0000","172.21.15.39:0/2133325947":"2024-09-15T03:46:35.805097+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:51:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:16 smithi129 ceph-mon[33173]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:16 smithi129 ceph-mon[33173]: osdmap e58: 8 total, 8 up, 8 in 2024-09-14T03:51:17.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:16 smithi039 ceph-mon[29586]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:17.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:16 smithi039 ceph-mon[29586]: osdmap e58: 8 total, 8 up, 8 in 2024-09-14T03:51:17.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:16 smithi039 ceph-mon[40962]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:17.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:16 smithi039 ceph-mon[40962]: osdmap e58: 8 total, 8 up, 8 in 2024-09-14T03:51:17.441 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-09-14T03:49:18.022402+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 7.889999866485596, 'score_stable': 7.889999866485596, 'optimal_score': 0.3799999952316284, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-09-14T03:51:17.441 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-09-14T03:51:17.649 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:51:17.930 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:17 smithi039 ceph-mon[29586]: osdmap e59: 8 total, 8 up, 8 in 2024-09-14T03:51:17.930 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:17 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/626033712' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:51:17.930 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:17 smithi039 ceph-mon[29586]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:17.930 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:17 smithi039 ceph-mon[40962]: osdmap e59: 8 total, 8 up, 8 in 2024-09-14T03:51:17.931 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:17 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/626033712' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:51:17.931 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:17 smithi039 ceph-mon[40962]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:18.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:17 smithi129 ceph-mon[33173]: osdmap e59: 8 total, 8 up, 8 in 2024-09-14T03:51:18.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:17 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/626033712' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:51:18.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:17 smithi129 ceph-mon[33173]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:19.489 INFO:teuthology.orchestra.run.smithi039.stdout:pg_num: 1 2024-09-14T03:51:19.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:19 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/1438601757' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-14T03:51:19.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:19 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/1438601757' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-14T03:51:20.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:19 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/1438601757' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-09-14T03:51:20.224 INFO:tasks.cephadm:Adding prometheus.a on smithi129 2024-09-14T03:51:20.224 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph orch apply prometheus '1;smithi129=a' 2024-09-14T03:51:20.430 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:51:20.856 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:20 smithi129 ceph-mon[33173]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s, 0 objects/s recovering 2024-09-14T03:51:21.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:20 smithi039 ceph-mon[40962]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s, 0 objects/s recovering 2024-09-14T03:51:21.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:20 smithi039 ceph-mon[29586]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 98 KiB/s, 0 objects/s recovering 2024-09-14T03:51:21.945 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:21 smithi129 ceph-mon[33173]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-09-14T03:51:22.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:21 smithi039 ceph-mon[40962]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-09-14T03:51:22.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:21 smithi039 ceph-mon[29586]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-09-14T03:51:22.222 INFO:teuthology.orchestra.run.smithi129.stdout:Scheduled prometheus update... 2024-09-14T03:51:22.832 DEBUG:teuthology.orchestra.run.smithi129:prometheus.a> sudo journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@prometheus.a.service 2024-09-14T03:51:22.835 INFO:tasks.cephadm:Adding node-exporter.a on smithi039 2024-09-14T03:51:22.835 INFO:tasks.cephadm:Adding node-exporter.b on smithi129 2024-09-14T03:51:22.835 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph orch apply node-exporter '2;smithi039=a;smithi129=b' 2024-09-14T03:51:23.079 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:51:23.476 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:23 smithi129 ceph-mon[33173]: from='client.24310 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi129=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:23.477 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:23 smithi129 ceph-mon[33173]: Saving service prometheus spec with placement smithi129=a;count:1 2024-09-14T03:51:23.477 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:23 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:23.477 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:23 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:23.477 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:23 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:23.477 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:23 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:51:23.477 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:23 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:23 smithi039 ceph-mon[29586]: from='client.24310 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi129=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:23.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:23 smithi039 ceph-mon[29586]: Saving service prometheus spec with placement smithi129=a;count:1 2024-09-14T03:51:23.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:23 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:23.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:23 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:23.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:23 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:23.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:23 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:51:23.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:23 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:23.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:23 smithi039 ceph-mon[40962]: from='client.24310 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi129=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:23.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:23 smithi039 ceph-mon[40962]: Saving service prometheus spec with placement smithi129=a;count:1 2024-09-14T03:51:23.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:23 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:23.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:23 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:23.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:23 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:23.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:23 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:51:23.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:23 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:24.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:24 smithi129 ceph-mon[33173]: Deploying daemon prometheus.a on smithi129 2024-09-14T03:51:24.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:24 smithi129 ceph-mon[33173]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-14T03:51:24.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:24 smithi039 ceph-mon[29586]: Deploying daemon prometheus.a on smithi129 2024-09-14T03:51:24.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:24 smithi039 ceph-mon[29586]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-14T03:51:24.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:24 smithi039 ceph-mon[40962]: Deploying daemon prometheus.a on smithi129 2024-09-14T03:51:24.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:24 smithi039 ceph-mon[40962]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-09-14T03:51:25.381 INFO:teuthology.orchestra.run.smithi129.stdout:Scheduled node-exporter update... 2024-09-14T03:51:26.357 DEBUG:teuthology.orchestra.run.smithi039:node-exporter.a> sudo journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@node-exporter.a.service 2024-09-14T03:51:26.360 DEBUG:teuthology.orchestra.run.smithi129:node-exporter.b> sudo journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@node-exporter.b.service 2024-09-14T03:51:26.363 INFO:tasks.cephadm:Adding alertmanager.a on smithi039 2024-09-14T03:51:26.363 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph orch apply alertmanager '1;smithi039=a' 2024-09-14T03:51:26.592 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:51:26.712 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:26 smithi129 ceph-mon[33173]: from='client.24316 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi039=a;smithi129=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:26.713 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:26 smithi129 ceph-mon[33173]: Saving service node-exporter spec with placement smithi039=a;smithi129=b;count:2 2024-09-14T03:51:26.713 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:26 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:26.713 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:26 smithi129 ceph-mon[33173]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:51:26.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:26 smithi039 ceph-mon[40962]: from='client.24316 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi039=a;smithi129=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:26.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:26 smithi039 ceph-mon[40962]: Saving service node-exporter spec with placement smithi039=a;smithi129=b;count:2 2024-09-14T03:51:26.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:26 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:26.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:26 smithi039 ceph-mon[40962]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:51:26.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:26 smithi039 ceph-mon[29586]: from='client.24316 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi039=a;smithi129=b", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:26.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:26 smithi039 ceph-mon[29586]: Saving service node-exporter spec with placement smithi039=a;smithi129=b;count:2 2024-09-14T03:51:26.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:26 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:26.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:26 smithi039 ceph-mon[29586]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-09-14T03:51:28.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:27 smithi129 ceph-mon[33173]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-09-14T03:51:28.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:27 smithi039 ceph-mon[29586]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-09-14T03:51:28.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:27 smithi039 ceph-mon[40962]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-09-14T03:51:29.168 INFO:teuthology.orchestra.run.smithi129.stdout:Scheduled alertmanager update... 2024-09-14T03:51:30.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:30 smithi129 ceph-mon[33173]: from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi039=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:30.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:30 smithi129 ceph-mon[33173]: Saving service alertmanager spec with placement smithi039=a;count:1 2024-09-14T03:51:30.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:30 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:30.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:30 smithi129 ceph-mon[33173]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-14T03:51:30.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:30 smithi039 ceph-mon[29586]: from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi039=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:30.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:30 smithi039 ceph-mon[29586]: Saving service alertmanager spec with placement smithi039=a;count:1 2024-09-14T03:51:30.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:30 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:30.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:30 smithi039 ceph-mon[29586]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-14T03:51:30.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:30 smithi039 ceph-mon[40962]: from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi039=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:30.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:30 smithi039 ceph-mon[40962]: Saving service alertmanager spec with placement smithi039=a;count:1 2024-09-14T03:51:30.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:30 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:30.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:30 smithi039 ceph-mon[40962]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-09-14T03:51:30.780 DEBUG:teuthology.orchestra.run.smithi039:alertmanager.a> sudo journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@alertmanager.a.service 2024-09-14T03:51:30.784 INFO:tasks.cephadm:Adding grafana.a on smithi129 2024-09-14T03:51:30.784 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph orch apply grafana '1;smithi129=a' 2024-09-14T03:51:31.242 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:51:31.933 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:31 smithi129 ceph-mon[33173]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:32.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:31 smithi039 ceph-mon[29586]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:32.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:31 smithi039 ceph-mon[40962]: pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:33.861 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:33 smithi129 ceph-mon[33173]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:34.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:33 smithi039 ceph-mon[29586]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:34.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:33 smithi039 ceph-mon[40962]: pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:34.060 INFO:teuthology.orchestra.run.smithi129.stdout:Scheduled grafana update... 2024-09-14T03:51:35.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:35 smithi039 ceph-mon[29586]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi129=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:35.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:35 smithi039 ceph-mon[29586]: Saving service grafana spec with placement smithi129=a;count:1 2024-09-14T03:51:35.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:35 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:35.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:35 smithi039 ceph-mon[40962]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi129=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:35.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:35 smithi039 ceph-mon[40962]: Saving service grafana spec with placement smithi129=a;count:1 2024-09-14T03:51:35.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:35 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:35.493 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:35 smithi129 ceph-mon[33173]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi129=a", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:51:35.493 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:35 smithi129 ceph-mon[33173]: Saving service grafana spec with placement smithi129=a;count:1 2024-09-14T03:51:35.493 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:35 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:36.436 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:36 smithi129 ceph-mon[33173]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:36.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:36 smithi039 ceph-mon[29586]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:36.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:36 smithi039 ceph-mon[40962]: pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:37.156 DEBUG:teuthology.orchestra.run.smithi129:grafana.a> sudo journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@grafana.a.service 2024-09-14T03:51:37.159 INFO:tasks.cephadm:Setting up client nodes... 2024-09-14T03:51:37.159 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-14T03:51:37.390 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:51:37.763 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:37 smithi039 ceph-mon[29586]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:37.764 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:37 smithi039 ceph-mon[40962]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:37.929 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:37 smithi129 ceph-mon[33173]: pgmap v188: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:38.271 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:51:38 smithi129 systemd[1]: Starting Ceph prometheus.a for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:51:38.560 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:51:38 smithi129 podman[62521]: 2024-09-14 03:51:38.212742322 +0000 UTC m=+0.016388927 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-14T03:51:38.560 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:51:38 smithi129 podman[62521]: 2024-09-14 03:51:38.311320481 +0000 UTC m=+0.114967115 container create fa029a6efad8104a9fc48bc5e341c7c30bc0d2ab60ff1ad2abbf4fa9cf8ac00a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:51:38.560 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:51:38 smithi129 podman[62521]: 2024-09-14 03:51:38.555312822 +0000 UTC m=+0.358959423 container init fa029a6efad8104a9fc48bc5e341c7c30bc0d2ab60ff1ad2abbf4fa9cf8ac00a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:51:38.561 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:51:38 smithi129 podman[62521]: 2024-09-14 03:51:38.559796629 +0000 UTC m=+0.363443222 container start fa029a6efad8104a9fc48bc5e341c7c30bc0d2ab60ff1ad2abbf4fa9cf8ac00a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:51:39.021 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:51:38 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:51:38.583Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-14T03:51:39.022 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:51:38 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:51:38.583Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-14T03:51:39.022 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:51:38 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:51:38.583Z caller=main.go:570 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi129 (none))" 2024-09-14T03:51:39.022 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:51:38 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:51:38.583Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-14T03:51:39.022 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:51:38 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:51:38.583Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-14T03:51:39.022 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:51:38 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:51:38.584Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-14T03:51:39.022 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:51:38 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:51:38.584Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-14T03:51:39.022 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:51:38 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:51:38.585Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-14T03:51:39.022 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:51:38 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:51:38.585Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-14T03:51:39.022 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:51:38 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:51:38.588Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-14T03:51:39.022 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:51:38 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:51:38.588Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.706µs 2024-09-14T03:51:39.023 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:51:38 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:51:38.588Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-14T03:51:39.023 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:51:38 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:51:38.588Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-09-14T03:51:39.023 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:51:38 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:51:38.588Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=17.576µs wal_replay_duration=268.798µs wbl_replay_duration=111ns total_replay_duration=303.589µs 2024-09-14T03:51:39.023 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:51:38 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:51:38.589Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-14T03:51:39.023 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:51:38 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:51:38.589Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-14T03:51:39.023 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:51:38 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:51:38.589Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-14T03:51:39.023 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:51:38 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:51:38.609Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=20.001171ms db_storage=745ns remote_storage=1.01µs web_handler=413ns query_engine=578ns scrape=417.163µs scrape_sd=84.106µs notify=748ns notify_sd=1.4µs rules=19.16789ms tracing=5.581µs 2024-09-14T03:51:39.023 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:51:38 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:51:38.609Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-14T03:51:39.023 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:51:38 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:51:38.609Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-14T03:51:39.023 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:51:38 smithi129 bash[62521]: fa029a6efad8104a9fc48bc5e341c7c30bc0d2ab60ff1ad2abbf4fa9cf8ac00a 2024-09-14T03:51:39.023 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:51:38 smithi129 systemd[1]: Started Ceph prometheus.a for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:51:39.388 INFO:teuthology.orchestra.run.smithi039.stdout:[client.0] 2024-09-14T03:51:39.388 INFO:teuthology.orchestra.run.smithi039.stdout: key = AQBLCOVmIzevFhAAQJ+qwYP9iz07uGx0AUengQ== 2024-09-14T03:51:40.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:39 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:40.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:39 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:40.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:39 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:40.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:39 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-14T03:51:40.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:39 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/624840055' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:51:40.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:39 smithi129 ceph-mon[33173]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:51:40.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:39 smithi129 ceph-mon[33173]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-14T03:51:40.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:39 smithi129 ceph-mon[33173]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:40.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:39 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:40.022 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:39 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: ignoring --setuser ceph since I am not root 2024-09-14T03:51:40.022 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:39 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: ignoring --setgroup ceph since I am not root 2024-09-14T03:51:40.022 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:39 smithi129 ceph-mgr[35285]: -- 172.21.15.129:0/415813754 <== mon.1 v2:172.21.15.129:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d4e72c6d00 con 0x55d4e729b400 2024-09-14T03:51:40.022 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:39 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:51:39.867+0000 7f1e5415d040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-14T03:51:40.022 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:39 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:51:39.923+0000 7f1e5415d040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-14T03:51:40.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:39 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ignoring --setuser ceph since I am not root 2024-09-14T03:51:40.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:39 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ignoring --setgroup ceph since I am not root 2024-09-14T03:51:40.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:39 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:51:39.873+0000 7fbe4aa48040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-14T03:51:40.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:39 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:51:39.931+0000 7fbe4aa48040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-14T03:51:40.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:39 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:40.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:39 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:40.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:39 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:40.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:39 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-14T03:51:40.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:39 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/624840055' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:51:40.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:39 smithi039 ceph-mon[29586]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:51:40.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:39 smithi039 ceph-mon[29586]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-14T03:51:40.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:39 smithi039 ceph-mon[29586]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:40.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:39 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:40.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:39 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:40.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:39 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:40.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:39 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:40.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:39 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-09-14T03:51:40.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:39 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/624840055' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:51:40.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:39 smithi039 ceph-mon[40962]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:51:40.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:39 smithi039 ceph-mon[40962]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-14T03:51:40.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:39 smithi039 ceph-mon[40962]: pgmap v189: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:40.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:39 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' 2024-09-14T03:51:40.175 DEBUG:teuthology.orchestra.run.smithi039:> set -ex 2024-09-14T03:51:40.175 DEBUG:teuthology.orchestra.run.smithi039:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-09-14T03:51:40.175 DEBUG:teuthology.orchestra.run.smithi039:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-09-14T03:51:40.215 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-09-14T03:51:40.418 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T03:51:40.439 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:40 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:51:40.070+0000 7f1e5415d040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-14T03:51:40.439 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:40 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:51:40.134+0000 7f1e5415d040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-14T03:51:40.439 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:40 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:51:40.283+0000 7f1e5415d040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:51:40.439 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:40 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:51:40.437+0000 7f1e5415d040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-14T03:51:40.457 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:40 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:51:40.086+0000 7fbe4aa48040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-14T03:51:40.458 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:40 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:51:40.155+0000 7fbe4aa48040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-14T03:51:40.458 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:40 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:51:40.310+0000 7fbe4aa48040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:51:40.458 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:40 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:51:40.456+0000 7fbe4aa48040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-14T03:51:40.977 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:40 smithi129 ceph-mon[33173]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-14T03:51:40.977 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:40 smithi129 ceph-mon[33173]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-14T03:51:40.978 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:40 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:51:40.765+0000 7f1e5415d040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-14T03:51:40.978 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:40 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:51:40.829+0000 7f1e5415d040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-14T03:51:41.031 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:40 smithi039 ceph-mon[29586]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-14T03:51:41.032 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:40 smithi039 ceph-mon[29586]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-14T03:51:41.032 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:40 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:51:40.778+0000 7fbe4aa48040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-14T03:51:41.032 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:40 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:51:40.840+0000 7fbe4aa48040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-14T03:51:41.032 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:41 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:51:41.030+0000 7fbe4aa48040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-14T03:51:41.032 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:40 smithi039 ceph-mon[40962]: from='mgr.14150 172.21.15.39:0/3014835593' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-09-14T03:51:41.032 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:40 smithi039 ceph-mon[40962]: mgrmap e16: y(active, since 4m), standbys: x 2024-09-14T03:51:41.271 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:51:41.023+0000 7f1e5415d040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-14T03:51:42.271 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:51:41.942+0000 7f1e5415d040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-14T03:51:42.271 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:42 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:51:42.010+0000 7f1e5415d040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-14T03:51:42.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:41 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:51:41.938+0000 7fbe4aa48040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-14T03:51:42.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:42 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:51:42.006+0000 7fbe4aa48040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-14T03:51:42.438 INFO:teuthology.orchestra.run.smithi129.stdout:[client.1] 2024-09-14T03:51:42.438 INFO:teuthology.orchestra.run.smithi129.stdout: key = AQBOCOVmhcW9GRAAUXzfTs0c85YMnYoLjQezUw== 2024-09-14T03:51:42.752 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:42 smithi039 ceph-mon[29586]: from='client.? 172.21.15.129:0/3828074767' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:51:42.752 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:42 smithi039 ceph-mon[29586]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:51:42.752 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:42 smithi039 ceph-mon[29586]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-14T03:51:42.752 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:42 smithi039 ceph-mon[40962]: from='client.? 172.21.15.129:0/3828074767' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:51:42.753 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:42 smithi039 ceph-mon[40962]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:51:42.753 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:42 smithi039 ceph-mon[40962]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-14T03:51:42.753 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:42 smithi129 ceph-mon[33173]: from='client.? 172.21.15.129:0/3828074767' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:51:42.753 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:42 smithi129 ceph-mon[33173]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-09-14T03:51:42.753 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:42 smithi129 ceph-mon[33173]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-09-14T03:51:43.019 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:42 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:51:42.751+0000 7fbe4aa48040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-14T03:51:43.019 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:42 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:51:42.871+0000 7fbe4aa48040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-14T03:51:43.021 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:42 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:51:42.806+0000 7f1e5415d040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-14T03:51:43.021 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:42 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:51:42.922+0000 7f1e5415d040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-14T03:51:43.075 DEBUG:teuthology.orchestra.run.smithi129:> set -ex 2024-09-14T03:51:43.075 DEBUG:teuthology.orchestra.run.smithi129:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-09-14T03:51:43.075 DEBUG:teuthology.orchestra.run.smithi129:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-09-14T03:51:43.113 INFO:tasks.cephadm:Creating RBD pool 2024-09-14T03:51:43.114 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd pool create rbd 8 2024-09-14T03:51:43.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:43 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:51:43.017+0000 7fbe4aa48040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:51:43.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:43 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:51:43.081+0000 7fbe4aa48040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-14T03:51:43.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:43 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:51:43.236+0000 7fbe4aa48040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-14T03:51:43.319 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:51:43.346 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:43 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:51:43.064+0000 7f1e5415d040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:51:43.346 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:43 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:51:43.130+0000 7f1e5415d040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-14T03:51:43.346 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:43 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:51:43.276+0000 7f1e5415d040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-14T03:51:43.346 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:43 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:51:43.344+0000 7f1e5415d040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-14T03:51:43.579 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:43 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:51:43.313+0000 7fbe4aa48040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-14T03:51:43.579 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:43 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:51:43.381+0000 7fbe4aa48040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-14T03:51:43.579 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:43 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:51:43.547+0000 7fbe4aa48040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-14T03:51:43.624 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:43 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:51:43.404+0000 7f1e5415d040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-14T03:51:43.625 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:43 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:51:43.559+0000 7f1e5415d040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-14T03:51:43.625 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:43 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:51:43.622+0000 7f1e5415d040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-14T03:51:43.841 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:43 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:51:43.620+0000 7fbe4aa48040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-14T03:51:43.841 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:43 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:51:43.689+0000 7fbe4aa48040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-14T03:51:43.841 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:43 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:51:43.778+0000 7fbe4aa48040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-14T03:51:43.841 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:43 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:51:43.839+0000 7fbe4aa48040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-14T03:51:43.886 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:43 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:51:43.688+0000 7f1e5415d040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-14T03:51:43.886 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:43 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:51:43.769+0000 7f1e5415d040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-14T03:51:43.886 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:43 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:51:43.827+0000 7f1e5415d040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-14T03:51:43.886 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:43 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:51:43.884+0000 7f1e5415d040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-14T03:51:44.193 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:43 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:51:43.901+0000 7fbe4aa48040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-14T03:51:44.884 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:44 smithi039 ceph-mon[29586]: Standby manager daemon x restarted 2024-09-14T03:51:44.884 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:44 smithi039 ceph-mon[29586]: Standby manager daemon x started 2024-09-14T03:51:44.884 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:44 smithi039 ceph-mon[29586]: from='mgr.? 172.21.15.129:0/1688501276' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:51:44.884 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:44 smithi039 ceph-mon[29586]: from='mgr.? 172.21.15.129:0/1688501276' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:51:44.885 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:44 smithi039 ceph-mon[29586]: from='mgr.? 172.21.15.129:0/1688501276' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:51:44.885 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:44 smithi039 ceph-mon[29586]: from='mgr.? 172.21.15.129:0/1688501276' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:51:44.885 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:44 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:51:44.723+0000 7fbe4aa48040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-14T03:51:44.885 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:44 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:51:44] ENGINE Bus STARTING 2024-09-14T03:51:44.885 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:44 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: CherryPy Checker: 2024-09-14T03:51:44.885 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:44 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: The Application mounted at '' has an empty config. 2024-09-14T03:51:44.885 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:44 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:51:44.885 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:44 smithi039 ceph-mon[40962]: Standby manager daemon x restarted 2024-09-14T03:51:44.885 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:44 smithi039 ceph-mon[40962]: Standby manager daemon x started 2024-09-14T03:51:44.886 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:44 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.129:0/1688501276' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:51:44.886 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:44 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.129:0/1688501276' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:51:44.886 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:44 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.129:0/1688501276' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:51:44.886 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:44 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.129:0/1688501276' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:51:45.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:44 smithi129 ceph-mon[33173]: Standby manager daemon x restarted 2024-09-14T03:51:45.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:44 smithi129 ceph-mon[33173]: Standby manager daemon x started 2024-09-14T03:51:45.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:44 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.129:0/1688501276' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:51:45.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:44 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.129:0/1688501276' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:51:45.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:44 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.129:0/1688501276' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:51:45.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:44 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.129:0/1688501276' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:51:45.022 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:44 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:51:44.654+0000 7f1e5415d040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-14T03:51:45.022 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:44 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: [14/Sep/2024:03:51:44] ENGINE Bus STARTING 2024-09-14T03:51:45.022 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:44 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: CherryPy Checker: 2024-09-14T03:51:45.022 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:44 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: The Application mounted at '' has an empty config. 2024-09-14T03:51:45.022 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:44 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: 2024-09-14T03:51:45.022 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:44 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: [14/Sep/2024:03:51:44] ENGINE Serving on http://:::9283 2024-09-14T03:51:45.023 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:51:44 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[35262]: [14/Sep/2024:03:51:44] ENGINE Bus STARTED 2024-09-14T03:51:45.158 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:44 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:51:44] ENGINE Serving on http://:::9283 2024-09-14T03:51:45.158 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:44 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:51:44] ENGINE Bus STARTED 2024-09-14T03:51:45.794 INFO:teuthology.orchestra.run.smithi039.stderr:pool 'rbd' created 2024-09-14T03:51:46.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:45 smithi129 ceph-mon[33173]: mgrmap e17: y(active, since 4m), standbys: x 2024-09-14T03:51:46.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:45 smithi129 ceph-mon[33173]: Active manager daemon y restarted 2024-09-14T03:51:46.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:45 smithi129 ceph-mon[33173]: Activating manager daemon y 2024-09-14T03:51:46.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:45 smithi129 ceph-mon[33173]: osdmap e60: 8 total, 8 up, 8 in 2024-09-14T03:51:46.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:45 smithi129 ceph-mon[33173]: mgrmap e18: y(active, starting, since 0.00596719s), standbys: x 2024-09-14T03:51:46.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:45 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:51:46.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:45 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:51:46.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:45 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:51:46.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:45 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:51:46.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:45 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:51:46.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:45 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:51:46.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:45 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:51:46.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:45 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:51:46.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:45 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:51:46.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:45 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:51:46.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:45 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:51:46.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:45 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:51:46.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:45 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:51:46.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:45 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:51:46.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:45 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:51:46.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:45 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:51:46.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:45 smithi129 ceph-mon[33173]: Manager daemon y is now available 2024-09-14T03:51:46.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:45 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:46.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:45 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:46.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:45 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:51:46.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:45 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:51:46.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:45 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:51:46.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:45 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:51:46.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:45 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:51:46.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[29586]: mgrmap e17: y(active, since 4m), standbys: x 2024-09-14T03:51:46.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[29586]: Active manager daemon y restarted 2024-09-14T03:51:46.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[29586]: Activating manager daemon y 2024-09-14T03:51:46.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[29586]: osdmap e60: 8 total, 8 up, 8 in 2024-09-14T03:51:46.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[29586]: mgrmap e18: y(active, starting, since 0.00596719s), standbys: x 2024-09-14T03:51:46.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:51:46.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:51:46.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:51:46.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:51:46.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:51:46.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:51:46.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:51:46.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:51:46.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:51:46.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:51:46.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:51:46.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:51:46.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:51:46.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:51:46.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:51:46.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:51:46.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[29586]: Manager daemon y is now available 2024-09-14T03:51:46.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:46.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:46.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:51:46.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:51:46.044 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:51:46.044 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:51:46.044 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:51:46.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[40962]: mgrmap e17: y(active, since 4m), standbys: x 2024-09-14T03:51:46.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[40962]: Active manager daemon y restarted 2024-09-14T03:51:46.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[40962]: Activating manager daemon y 2024-09-14T03:51:46.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[40962]: osdmap e60: 8 total, 8 up, 8 in 2024-09-14T03:51:46.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[40962]: mgrmap e18: y(active, starting, since 0.00596719s), standbys: x 2024-09-14T03:51:46.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:51:46.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:51:46.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:51:46.045 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:51:46.045 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:51:46.045 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:51:46.045 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:51:46.045 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:51:46.045 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:51:46.045 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:51:46.045 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:51:46.045 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:51:46.045 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:51:46.045 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:51:46.045 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:51:46.045 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:51:46.046 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[40962]: Manager daemon y is now available 2024-09-14T03:51:46.046 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:51:46.046 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:46.046 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:51:46.046 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:51:46.046 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:51:46.046 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:51:46.046 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:45 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:51:47.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:46 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/3678949630' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-14T03:51:47.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:46 smithi039 ceph-mon[40962]: mgrmap e19: y(active, since 1.05656s), standbys: x 2024-09-14T03:51:47.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:46 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/3678949630' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-14T03:51:47.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:46 smithi039 ceph-mon[40962]: osdmap e61: 8 total, 8 up, 8 in 2024-09-14T03:51:47.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:46 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/3678949630' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-14T03:51:47.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:46 smithi039 ceph-mon[29586]: mgrmap e19: y(active, since 1.05656s), standbys: x 2024-09-14T03:51:47.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:46 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/3678949630' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-14T03:51:47.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:46 smithi039 ceph-mon[29586]: osdmap e61: 8 total, 8 up, 8 in 2024-09-14T03:51:47.245 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- rbd pool init rbd 2024-09-14T03:51:47.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:46 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/3678949630' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-09-14T03:51:47.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:46 smithi129 ceph-mon[33173]: mgrmap e19: y(active, since 1.05656s), standbys: x 2024-09-14T03:51:47.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:46 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/3678949630' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-09-14T03:51:47.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:46 smithi129 ceph-mon[33173]: osdmap e61: 8 total, 8 up, 8 in 2024-09-14T03:51:47.469 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:51:47.540 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:51:47.420+0000 7fbd9efaf640 -1 log_channel(cephadm) log [ERR] : [14/Sep/2024:03:51:47] ENGINE Error in HTTPServer.serve 2024-09-14T03:51:47.540 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: Traceback (most recent call last): 2024-09-14T03:51:47.541 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:51:47.541 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: self._connections.run(self.expiration_interval) 2024-09-14T03:51:47.541 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:51:47.541 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: self._run(expiration_interval) 2024-09-14T03:51:47.541 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:51:47.541 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:51:47.541 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:51:47.541 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:51:47.541 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:51:47.541 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: s = self.context.wrap_socket( 2024-09-14T03:51:47.541 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:51:47.541 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: return self.sslsocket_class._create( 2024-09-14T03:51:47.541 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:51:47.542 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: self.do_handshake() 2024-09-14T03:51:47.542 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:51:47.542 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: self._sslobj.do_handshake() 2024-09-14T03:51:47.542 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:51:47.542 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:51:47.886 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-mon[29586]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:47.886 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-mon[29586]: pgmap v5: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:47.886 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-mon[29586]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:51:47.886 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-mon[29586]: osdmap e62: 8 total, 8 up, 8 in 2024-09-14T03:51:47.886 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-mon[29586]: mgrmap e20: y(active, since 2s), standbys: x 2024-09-14T03:51:47.886 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:47.886 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:47.886 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:47.886 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:47.887 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:47.887 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:47.887 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:47.887 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:47.887 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:47.887 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:47.887 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-mon[40962]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:47.887 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-mon[40962]: pgmap v5: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:47.887 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-mon[40962]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:51:47.887 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-mon[40962]: osdmap e62: 8 total, 8 up, 8 in 2024-09-14T03:51:47.887 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-mon[40962]: mgrmap e20: y(active, since 2s), standbys: x 2024-09-14T03:51:47.887 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:47.888 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:47.888 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:47.888 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:47.888 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:47.888 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:47.888 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:47.888 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:47.888 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:47.888 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:47 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:48.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:47 smithi129 ceph-mon[33173]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:48.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:47 smithi129 ceph-mon[33173]: pgmap v5: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:48.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:47 smithi129 ceph-mon[33173]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:51:48.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:47 smithi129 ceph-mon[33173]: osdmap e62: 8 total, 8 up, 8 in 2024-09-14T03:51:48.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:47 smithi129 ceph-mon[33173]: mgrmap e20: y(active, since 2s), standbys: x 2024-09-14T03:51:48.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:47 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:48.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:47 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:48.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:47 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:48.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:47 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:48.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:47 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:48.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:47 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:48.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:47 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:48.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:47 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:48.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:47 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:48.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:47 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:49.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:49 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:51:49] "GET /metrics HTTP/1.1" 200 23677 "" "Prometheus/2.43.0" 2024-09-14T03:51:49.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[40962]: [14/Sep/2024:03:51:47] ENGINE Bus STARTING 2024-09-14T03:51:49.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[40962]: [14/Sep/2024:03:51:47] ENGINE Serving on http://172.21.15.39:8765 2024-09-14T03:51:49.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[40962]: [14/Sep/2024:03:51:47] ENGINE Serving on https://172.21.15.39:7150 2024-09-14T03:51:49.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[40962]: [14/Sep/2024:03:51:47] ENGINE Bus STARTED 2024-09-14T03:51:49.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[40962]: [14/Sep/2024:03:51:47] ENGINE Error in HTTPServer.serve 2024-09-14T03:51:49.041 INFO:journalctl@ceph.mon.c.smithi039.stdout: Traceback (most recent call last): 2024-09-14T03:51:49.041 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:51:49.041 INFO:journalctl@ceph.mon.c.smithi039.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:51:49.041 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:51:49.041 INFO:journalctl@ceph.mon.c.smithi039.stdout: self._run(expiration_interval) 2024-09-14T03:51:49.041 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:51:49.041 INFO:journalctl@ceph.mon.c.smithi039.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:51:49.041 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:51:49.041 INFO:journalctl@ceph.mon.c.smithi039.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:51:49.042 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:51:49.042 INFO:journalctl@ceph.mon.c.smithi039.stdout: s = self.context.wrap_socket( 2024-09-14T03:51:49.042 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:51:49.042 INFO:journalctl@ceph.mon.c.smithi039.stdout: return self.sslsocket_class._create( 2024-09-14T03:51:49.042 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:51:49.042 INFO:journalctl@ceph.mon.c.smithi039.stdout: self.do_handshake() 2024-09-14T03:51:49.042 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:51:49.042 INFO:journalctl@ceph.mon.c.smithi039.stdout: self._sslobj.do_handshake() 2024-09-14T03:51:49.042 INFO:journalctl@ceph.mon.c.smithi039.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:51:49.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:49.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:49.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:49.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:49.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[40962]: Adjusting osd_memory_target on smithi129 to 4016M 2024-09-14T03:51:49.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:49.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[40962]: osdmap e63: 8 total, 8 up, 8 in 2024-09-14T03:51:49.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:49.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:49.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:49.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:51:49.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[29586]: [14/Sep/2024:03:51:47] ENGINE Bus STARTING 2024-09-14T03:51:49.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[29586]: [14/Sep/2024:03:51:47] ENGINE Serving on http://172.21.15.39:8765 2024-09-14T03:51:49.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[29586]: [14/Sep/2024:03:51:47] ENGINE Serving on https://172.21.15.39:7150 2024-09-14T03:51:49.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[29586]: [14/Sep/2024:03:51:47] ENGINE Bus STARTED 2024-09-14T03:51:49.044 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[29586]: [14/Sep/2024:03:51:47] ENGINE Error in HTTPServer.serve 2024-09-14T03:51:49.044 INFO:journalctl@ceph.mon.a.smithi039.stdout: Traceback (most recent call last): 2024-09-14T03:51:49.044 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:51:49.044 INFO:journalctl@ceph.mon.a.smithi039.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:51:49.044 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:51:49.044 INFO:journalctl@ceph.mon.a.smithi039.stdout: self._run(expiration_interval) 2024-09-14T03:51:49.044 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:51:49.044 INFO:journalctl@ceph.mon.a.smithi039.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:51:49.044 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:51:49.044 INFO:journalctl@ceph.mon.a.smithi039.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:51:49.044 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:51:49.044 INFO:journalctl@ceph.mon.a.smithi039.stdout: s = self.context.wrap_socket( 2024-09-14T03:51:49.045 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:51:49.045 INFO:journalctl@ceph.mon.a.smithi039.stdout: return self.sslsocket_class._create( 2024-09-14T03:51:49.045 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:51:49.045 INFO:journalctl@ceph.mon.a.smithi039.stdout: self.do_handshake() 2024-09-14T03:51:49.045 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:51:49.045 INFO:journalctl@ceph.mon.a.smithi039.stdout: self._sslobj.do_handshake() 2024-09-14T03:51:49.045 INFO:journalctl@ceph.mon.a.smithi039.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:51:49.045 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:49.045 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:49.045 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:49.045 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:49.045 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[29586]: Adjusting osd_memory_target on smithi129 to 4016M 2024-09-14T03:51:49.045 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:49.045 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[29586]: osdmap e63: 8 total, 8 up, 8 in 2024-09-14T03:51:49.046 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:49.046 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:49.046 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:49.046 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:48 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:51:49.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:48 smithi129 ceph-mon[33173]: [14/Sep/2024:03:51:47] ENGINE Bus STARTING 2024-09-14T03:51:49.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:48 smithi129 ceph-mon[33173]: [14/Sep/2024:03:51:47] ENGINE Serving on http://172.21.15.39:8765 2024-09-14T03:51:49.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:48 smithi129 ceph-mon[33173]: [14/Sep/2024:03:51:47] ENGINE Serving on https://172.21.15.39:7150 2024-09-14T03:51:49.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:48 smithi129 ceph-mon[33173]: [14/Sep/2024:03:51:47] ENGINE Bus STARTED 2024-09-14T03:51:49.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:48 smithi129 ceph-mon[33173]: [14/Sep/2024:03:51:47] ENGINE Error in HTTPServer.serve 2024-09-14T03:51:49.272 INFO:journalctl@ceph.mon.b.smithi129.stdout: Traceback (most recent call last): 2024-09-14T03:51:49.272 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:51:49.272 INFO:journalctl@ceph.mon.b.smithi129.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:51:49.272 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:51:49.272 INFO:journalctl@ceph.mon.b.smithi129.stdout: self._run(expiration_interval) 2024-09-14T03:51:49.272 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:51:49.272 INFO:journalctl@ceph.mon.b.smithi129.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:51:49.272 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:51:49.272 INFO:journalctl@ceph.mon.b.smithi129.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:51:49.272 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:51:49.272 INFO:journalctl@ceph.mon.b.smithi129.stdout: s = self.context.wrap_socket( 2024-09-14T03:51:49.272 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:51:49.273 INFO:journalctl@ceph.mon.b.smithi129.stdout: return self.sslsocket_class._create( 2024-09-14T03:51:49.273 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:51:49.273 INFO:journalctl@ceph.mon.b.smithi129.stdout: self.do_handshake() 2024-09-14T03:51:49.273 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:51:49.273 INFO:journalctl@ceph.mon.b.smithi129.stdout: self._sslobj.do_handshake() 2024-09-14T03:51:49.273 INFO:journalctl@ceph.mon.b.smithi129.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:51:49.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:48 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:49.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:48 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:49.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:48 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:49.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:48 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:51:49.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:48 smithi129 ceph-mon[33173]: Adjusting osd_memory_target on smithi129 to 4016M 2024-09-14T03:51:49.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:48 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:49.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:48 smithi129 ceph-mon[33173]: osdmap e63: 8 total, 8 up, 8 in 2024-09-14T03:51:49.274 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:48 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:49.274 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:48 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:49.274 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:48 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:51:49.274 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:48 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:51:50.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:49 smithi039 ceph-mon[40962]: Updating smithi039:/etc/ceph/ceph.conf 2024-09-14T03:51:50.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:49 smithi039 ceph-mon[40962]: Updating smithi129:/etc/ceph/ceph.conf 2024-09-14T03:51:50.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:49 smithi039 ceph-mon[40962]: Updating smithi039:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:51:50.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:49 smithi039 ceph-mon[40962]: Updating smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:51:50.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:49 smithi039 ceph-mon[40962]: Updating smithi039:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:51:50.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:49 smithi039 ceph-mon[40962]: Updating smithi129:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:51:50.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:49 smithi039 ceph-mon[40962]: pgmap v8: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:50.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:49 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:50.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:49 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:50.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:49 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:50.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:49 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:50.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:49 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:50.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:49 smithi039 ceph-mon[40962]: mgrmap e21: y(active, since 4s), standbys: x 2024-09-14T03:51:50.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:49 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/2240891363' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-14T03:51:50.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:49 smithi039 ceph-mon[29586]: Updating smithi039:/etc/ceph/ceph.conf 2024-09-14T03:51:50.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:49 smithi039 ceph-mon[29586]: Updating smithi129:/etc/ceph/ceph.conf 2024-09-14T03:51:50.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:49 smithi039 ceph-mon[29586]: Updating smithi039:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:51:50.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:49 smithi039 ceph-mon[29586]: Updating smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:51:50.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:49 smithi039 ceph-mon[29586]: Updating smithi039:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:51:50.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:49 smithi039 ceph-mon[29586]: Updating smithi129:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:51:50.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:49 smithi039 ceph-mon[29586]: pgmap v8: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:50.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:49 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:50.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:49 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:50.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:49 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:50.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:49 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:50.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:49 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:50.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:49 smithi039 ceph-mon[29586]: mgrmap e21: y(active, since 4s), standbys: x 2024-09-14T03:51:50.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:49 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/2240891363' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-14T03:51:50.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:49 smithi129 ceph-mon[33173]: Updating smithi039:/etc/ceph/ceph.conf 2024-09-14T03:51:50.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:49 smithi129 ceph-mon[33173]: Updating smithi129:/etc/ceph/ceph.conf 2024-09-14T03:51:50.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:49 smithi129 ceph-mon[33173]: Updating smithi039:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:51:50.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:49 smithi129 ceph-mon[33173]: Updating smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:51:50.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:49 smithi129 ceph-mon[33173]: Updating smithi039:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:51:50.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:49 smithi129 ceph-mon[33173]: Updating smithi129:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:51:50.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:49 smithi129 ceph-mon[33173]: pgmap v8: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:50.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:49 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:50.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:49 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:50.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:49 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:50.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:49 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:50.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:49 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:51:50.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:49 smithi129 ceph-mon[33173]: mgrmap e21: y(active, since 4s), standbys: x 2024-09-14T03:51:50.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:49 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/2240891363' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]: dispatch 2024-09-14T03:51:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:50 smithi129 ceph-mon[33173]: Updating smithi039:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:51:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:50 smithi129 ceph-mon[33173]: Updating smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:51:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:50 smithi129 ceph-mon[33173]: Deploying daemon alertmanager.a on smithi039 2024-09-14T03:51:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:50 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/2240891363' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-14T03:51:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:50 smithi129 ceph-mon[33173]: osdmap e64: 8 total, 8 up, 8 in 2024-09-14T03:51:51.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:50 smithi129 ceph-mon[33173]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:51.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:50 smithi039 ceph-mon[40962]: Updating smithi039:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:51:51.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:50 smithi039 ceph-mon[40962]: Updating smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:51:51.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:50 smithi039 ceph-mon[40962]: Deploying daemon alertmanager.a on smithi039 2024-09-14T03:51:51.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:50 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/2240891363' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-14T03:51:51.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:50 smithi039 ceph-mon[40962]: osdmap e64: 8 total, 8 up, 8 in 2024-09-14T03:51:51.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:50 smithi039 ceph-mon[40962]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:51.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:50 smithi039 ceph-mon[29586]: Updating smithi039:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:51:51.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:50 smithi039 ceph-mon[29586]: Updating smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:51:51.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:50 smithi039 ceph-mon[29586]: Deploying daemon alertmanager.a on smithi039 2024-09-14T03:51:51.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:50 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/2240891363' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "rbd","app": "rbd"}]': finished 2024-09-14T03:51:51.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:50 smithi039 ceph-mon[29586]: osdmap e64: 8 total, 8 up, 8 in 2024-09-14T03:51:51.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:50 smithi039 ceph-mon[29586]: pgmap v10: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 217 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:51:52.055 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:51 smithi129 ceph-mon[33173]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:51:52.056 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:51 smithi129 ceph-mon[33173]: Cluster is now healthy 2024-09-14T03:51:52.056 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:51 smithi129 ceph-mon[33173]: osdmap e65: 8 total, 8 up, 8 in 2024-09-14T03:51:52.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:51 smithi039 ceph-mon[29586]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:51:52.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:51 smithi039 ceph-mon[29586]: Cluster is now healthy 2024-09-14T03:51:52.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:51 smithi039 ceph-mon[29586]: osdmap e65: 8 total, 8 up, 8 in 2024-09-14T03:51:52.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:51 smithi039 ceph-mon[40962]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:51:52.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:51 smithi039 ceph-mon[40962]: Cluster is now healthy 2024-09-14T03:51:52.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:51 smithi039 ceph-mon[40962]: osdmap e65: 8 total, 8 up, 8 in 2024-09-14T03:51:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:53 smithi129 ceph-mon[33173]: osdmap e66: 8 total, 8 up, 8 in 2024-09-14T03:51:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:53 smithi129 ceph-mon[33173]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 0 B/s wr, 23 op/s 2024-09-14T03:51:53.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:53 smithi039 ceph-mon[29586]: osdmap e66: 8 total, 8 up, 8 in 2024-09-14T03:51:53.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:53 smithi039 ceph-mon[29586]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 0 B/s wr, 23 op/s 2024-09-14T03:51:53.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:53 smithi039 ceph-mon[40962]: osdmap e66: 8 total, 8 up, 8 in 2024-09-14T03:51:53.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:53 smithi039 ceph-mon[40962]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 0 B/s wr, 23 op/s 2024-09-14T03:51:53.930 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph config log 1 --format=json 2024-09-14T03:51:54.810 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:51:54.933 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:54 smithi039 ceph-mon[29586]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-09-14T03:51:54.933 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:54 smithi039 ceph-mon[40962]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-09-14T03:51:55.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:54 smithi129 ceph-mon[33173]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 0 B/s wr, 19 op/s 2024-09-14T03:51:57.023 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:56 smithi039 ceph-mon[29586]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 317 B/s wr, 18 op/s 2024-09-14T03:51:57.023 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:56 smithi039 ceph-mon[40962]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 317 B/s wr, 18 op/s 2024-09-14T03:51:57.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:56 smithi129 ceph-mon[33173]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 317 B/s wr, 18 op/s 2024-09-14T03:51:58.634 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:51:58 smithi039 podman[71836]: 2024-09-14 03:51:58.463240862 +0000 UTC m=+0.097487097 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-09-14T03:51:58.725 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:51:59.031 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:51:58 smithi039 podman[71836]: 2024-09-14 03:51:58.76696141 +0000 UTC m=+0.401207629 volume create bf2280854ddec5c2baa9f8551538c71ad81cf995e4599fa1ebf33b8980d316b1 2024-09-14T03:51:59.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:51:59 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:51:59] "GET /metrics HTTP/1.1" 200 23677 "" "Prometheus/2.43.0" 2024-09-14T03:51:59.290 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:51:59 smithi039 podman[71836]: 2024-09-14 03:51:59.093729631 +0000 UTC m=+0.727975892 container create 72ce7832d5c6ac82e19597ed3764db950316972c326cd230753bf52610de9208 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:51:59.669 INFO:teuthology.orchestra.run.smithi039.stdout:[{"version":22,"timestamp":"2024-09-14T03:51:47.777692+0000","name":"","changes":[{"name":"osd/host:smithi129/osd_memory_target","previous_value":"4479544012","new_value":"4211108556"}]}] 2024-09-14T03:51:59.670 INFO:tasks.ceph_manager:config epoch is 22 2024-09-14T03:51:59.670 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-14T03:51:59.670 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-14T03:51:59.670 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-14T03:51:59.933 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:59 smithi039 ceph-mon[29586]: osdmap e67: 8 total, 8 up, 8 in 2024-09-14T03:51:59.933 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:59 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/4053579374' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-14T03:51:59.933 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:51:59 smithi039 ceph-mon[29586]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 275 B/s wr, 0 op/s 2024-09-14T03:51:59.934 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:59 smithi039 ceph-mon[40962]: osdmap e67: 8 total, 8 up, 8 in 2024-09-14T03:51:59.934 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:59 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/4053579374' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-14T03:51:59.934 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:51:59 smithi039 ceph-mon[40962]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 275 B/s wr, 0 op/s 2024-09-14T03:51:59.934 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:51:59 smithi039 podman[71836]: 2024-09-14 03:51:59.824349716 +0000 UTC m=+1.458595935 container init 72ce7832d5c6ac82e19597ed3764db950316972c326cd230753bf52610de9208 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:51:59.934 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:51:59 smithi039 podman[71836]: 2024-09-14 03:51:59.829723167 +0000 UTC m=+1.463969386 container start 72ce7832d5c6ac82e19597ed3764db950316972c326cd230753bf52610de9208 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:51:59.934 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:51:59 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[71896]: ts=2024-09-14T03:51:59.842Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-14T03:51:59.934 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:51:59 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[71896]: ts=2024-09-14T03:51:59.842Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-14T03:51:59.935 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:51:59 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[71896]: ts=2024-09-14T03:51:59.843Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.39 port=9094 2024-09-14T03:51:59.935 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:51:59 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[71896]: ts=2024-09-14T03:51:59.844Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-14T03:51:59.935 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:51:59 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[71896]: ts=2024-09-14T03:51:59.867Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-14T03:51:59.935 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:51:59 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[71896]: ts=2024-09-14T03:51:59.868Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-14T03:51:59.935 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:51:59 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[71896]: ts=2024-09-14T03:51:59.870Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-14T03:51:59.935 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:51:59 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[71896]: ts=2024-09-14T03:51:59.870Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-14T03:51:59.935 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:51:59 smithi039 bash[71836]: 72ce7832d5c6ac82e19597ed3764db950316972c326cd230753bf52610de9208 2024-09-14T03:52:00.015 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:52:00.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:59 smithi129 ceph-mon[33173]: osdmap e67: 8 total, 8 up, 8 in 2024-09-14T03:52:00.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:59 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/4053579374' entity='client.admin' cmd=[{"prefix": "config log", "num": 1, "format": "json"}]: dispatch 2024-09-14T03:52:00.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:51:59 smithi129 ceph-mon[33173]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 275 B/s wr, 0 op/s 2024-09-14T03:52:00.277 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:51:59 smithi039 systemd[1]: Started Ceph alertmanager.a for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:52:00.278 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:52:00] ENGINE Bus STOPPING 2024-09-14T03:52:00.787 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:52:00] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:52:00.787 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:52:00] ENGINE Bus STOPPED 2024-09-14T03:52:00.787 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:52:00] ENGINE Bus STARTING 2024-09-14T03:52:00.787 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:52:00] ENGINE Serving on http://:::9283 2024-09-14T03:52:00.787 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:52:00] ENGINE Bus STARTED 2024-09-14T03:52:01.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:52:01.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:01.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:01.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:01.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:01.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:01.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-mon[29586]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-14T03:52:01.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:01.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:01.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-14T03:52:01.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-mon[29586]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-14T03:52:01.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:01.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-mon[29586]: Deploying daemon grafana.a on smithi129 2024-09-14T03:52:01.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-mon[29586]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 242 B/s wr, 0 op/s 2024-09-14T03:52:01.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:52:01.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:01.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:01.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:01.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:01.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:01.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-mon[40962]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-14T03:52:01.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:01.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:01.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-14T03:52:01.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-mon[40962]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-14T03:52:01.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:01.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-mon[40962]: Deploying daemon grafana.a on smithi129 2024-09-14T03:52:01.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:00 smithi039 ceph-mon[40962]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 242 B/s wr, 0 op/s 2024-09-14T03:52:01.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:00 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:52:01.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:00 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:01.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:00 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:01.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:00 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:01.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:00 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:01.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:00 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:01.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:00 smithi129 ceph-mon[33173]: Regenerating cephadm self-signed grafana TLS certificates 2024-09-14T03:52:01.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:00 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:01.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:00 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:01.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:00 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-14T03:52:01.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:00 smithi129 ceph-mon[33173]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-09-14T03:52:01.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:00 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:01.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:00 smithi129 ceph-mon[33173]: Deploying daemon grafana.a on smithi129 2024-09-14T03:52:01.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:00 smithi129 ceph-mon[33173]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 242 B/s wr, 0 op/s 2024-09-14T03:52:02.022 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:52:02.188 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:52:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[71896]: ts=2024-09-14T03:52:01.844Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000312499s 2024-09-14T03:52:02.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:02 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/2749718596' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-14T03:52:02.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:02 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/2749718596' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-14T03:52:02.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:02 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/2749718596' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-14T03:52:02.782 INFO:teuthology.orchestra.run.smithi039.stdout:{"epoch":21,"flags":0,"active_gid":14436,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6800","nonce":3924974081},{"type":"v1","addr":"172.21.15.39:6801","nonce":3924974081}]},"active_addr":"172.21.15.39:6801/3924974081","active_change":"2024-09-14T03:51:44.726931+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":24340,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.39:8443/","prometheus":"http://172.21.15.39:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.39:0","nonce":3196173579}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.39:0","nonce":1825825785}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.39:0","nonce":1766366412}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.39:0","nonce":502495021}]}]} 2024-09-14T03:52:02.789 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-14T03:52:02.789 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-14T03:52:02.790 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T03:52:03.005 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:52:03.426 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:03 smithi039 ceph-mon[29586]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-14T03:52:03.426 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:03 smithi039 ceph-mon[40962]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-14T03:52:03.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:03 smithi129 ceph-mon[33173]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-14T03:52:04.982 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:52:04.982 INFO:teuthology.orchestra.run.smithi039.stdout:{"epoch":67,"fsid":"bd7c803c-724b-11ef-bceb-c7b262605968","created":"2024-09-14T03:46:13.726476+0000","modified":"2024-09-14T03:51:58.662322+0000","last_up_change":"2024-09-14T03:51:13.230622+0000","last_in_change":"2024-09-14T03:50:56.569390+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:49:18.022402+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T03:51:45.764151+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"5fdfd478-dd27-4674-a7d6-53177d9e549a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6802","nonce":1775094854},{"type":"v1","addr":"172.21.15.39:6803","nonce":1775094854}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6804","nonce":1775094854},{"type":"v1","addr":"172.21.15.39:6805","nonce":1775094854}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6808","nonce":1775094854},{"type":"v1","addr":"172.21.15.39:6809","nonce":1775094854}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6806","nonce":1775094854},{"type":"v1","addr":"172.21.15.39:6807","nonce":1775094854}]},"public_addr":"172.21.15.39:6803/1775094854","cluster_addr":"172.21.15.39:6805/1775094854","heartbeat_back_addr":"172.21.15.39:6809/1775094854","heartbeat_front_addr":"172.21.15.39:6807/1775094854","state":["exists","up"]},{"osd":1,"uuid":"991c236e-1679-4c1e-a1a9-3894b1b12055","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6810","nonce":2410143394},{"type":"v1","addr":"172.21.15.39:6811","nonce":2410143394}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6812","nonce":2410143394},{"type":"v1","addr":"172.21.15.39:6813","nonce":2410143394}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6816","nonce":2410143394},{"type":"v1","addr":"172.21.15.39:6817","nonce":2410143394}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6814","nonce":2410143394},{"type":"v1","addr":"172.21.15.39:6815","nonce":2410143394}]},"public_addr":"172.21.15.39:6811/2410143394","cluster_addr":"172.21.15.39:6813/2410143394","heartbeat_back_addr":"172.21.15.39:6817/2410143394","heartbeat_front_addr":"172.21.15.39:6815/2410143394","state":["exists","up"]},{"osd":2,"uuid":"f9611ffd-5d37-4465-8114-ecc70d4d6d14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6818","nonce":3600430167},{"type":"v1","addr":"172.21.15.39:6819","nonce":3600430167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6820","nonce":3600430167},{"type":"v1","addr":"172.21.15.39:6821","nonce":3600430167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6824","nonce":3600430167},{"type":"v1","addr":"172.21.15.39:6825","nonce":3600430167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6822","nonce":3600430167},{"type":"v1","addr":"172.21.15.39:6823","nonce":3600430167}]},"public_addr":"172.21.15.39:6819/3600430167","cluster_addr":"172.21.15.39:6821/3600430167","heartbeat_back_addr":"172.21.15.39:6825/3600430167","heartbeat_front_addr":"172.21.15.39:6823/3600430167","state":["exists","up"]},{"osd":3,"uuid":"56b6877b-2107-47d6-8da9-015db7352288","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6826","nonce":1027302150},{"type":"v1","addr":"172.21.15.39:6827","nonce":1027302150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6828","nonce":1027302150},{"type":"v1","addr":"172.21.15.39:6829","nonce":1027302150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6832","nonce":1027302150},{"type":"v1","addr":"172.21.15.39:6833","nonce":1027302150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6830","nonce":1027302150},{"type":"v1","addr":"172.21.15.39:6831","nonce":1027302150}]},"public_addr":"172.21.15.39:6827/1027302150","cluster_addr":"172.21.15.39:6829/1027302150","heartbeat_back_addr":"172.21.15.39:6833/1027302150","heartbeat_front_addr":"172.21.15.39:6831/1027302150","state":["exists","up"]},{"osd":4,"uuid":"2d5523c4-5871-4fb3-bc0d-62c3b7d8b6a1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6800","nonce":168261430},{"type":"v1","addr":"172.21.15.129:6801","nonce":168261430}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6802","nonce":168261430},{"type":"v1","addr":"172.21.15.129:6803","nonce":168261430}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6806","nonce":168261430},{"type":"v1","addr":"172.21.15.129:6807","nonce":168261430}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6804","nonce":168261430},{"type":"v1","addr":"172.21.15.129:6805","nonce":168261430}]},"public_addr":"172.21.15.129:6801/168261430","cluster_addr":"172.21.15.129:6803/168261430","heartbeat_back_addr":"172.21.15.129:6807/168261430","heartbeat_front_addr":"172.21.15.129:6805/168261430","state":["exists","up"]},{"osd":5,"uuid":"a3ba55fd-f32e-4f6d-9b48-d2845a3474cd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6808","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6809","nonce":2184993002}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6810","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6811","nonce":2184993002}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6814","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6815","nonce":2184993002}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6812","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6813","nonce":2184993002}]},"public_addr":"172.21.15.129:6809/2184993002","cluster_addr":"172.21.15.129:6811/2184993002","heartbeat_back_addr":"172.21.15.129:6815/2184993002","heartbeat_front_addr":"172.21.15.129:6813/2184993002","state":["exists","up"]},{"osd":6,"uuid":"79e9c0a2-86a4-4267-a799-059d473d7b74","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6816","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6817","nonce":3767031353}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6818","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6819","nonce":3767031353}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6822","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6823","nonce":3767031353}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6820","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6821","nonce":3767031353}]},"public_addr":"172.21.15.129:6817/3767031353","cluster_addr":"172.21.15.129:6819/3767031353","heartbeat_back_addr":"172.21.15.129:6823/3767031353","heartbeat_front_addr":"172.21.15.129:6821/3767031353","state":["exists","up"]},{"osd":7,"uuid":"22f41628-4b88-4b8c-82b2-fcd2802da1bf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6824","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6825","nonce":2370414690}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6826","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6827","nonce":2370414690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6830","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6831","nonce":2370414690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6828","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6829","nonce":2370414690}]},"public_addr":"172.21.15.129:6825/2370414690","cluster_addr":"172.21.15.129:6827/2370414690","heartbeat_back_addr":"172.21.15.129:6831/2370414690","heartbeat_front_addr":"172.21.15.129:6829/2370414690","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:48:23.355518+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:48:48.715441+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:49:13.415323+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:49:39.935205+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:50:01.649042+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:50:24.001678+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:50:47.985011+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:51:11.584825+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.39:0/2587896825":"2024-09-15T03:51:44.726801+0000","172.21.15.39:0/754571920":"2024-09-15T03:51:44.726801+0000","172.21.15.39:0/2133325947":"2024-09-15T03:46:35.805097+0000","172.21.15.39:0/81449629":"2024-09-15T03:46:35.805097+0000","172.21.15.39:6800/628437919":"2024-09-15T03:46:35.805097+0000","172.21.15.39:6801/3127470450":"2024-09-15T03:46:59.511580+0000","172.21.15.39:6801/748677190":"2024-09-15T03:51:44.726801+0000","172.21.15.39:6801/628437919":"2024-09-15T03:46:35.805097+0000","172.21.15.39:0/2670041591":"2024-09-15T03:46:35.805097+0000","172.21.15.39:0/3161547781":"2024-09-15T03:46:59.511580+0000","172.21.15.39:0/1530637819":"2024-09-15T03:46:59.511580+0000","172.21.15.39:0/2745905728":"2024-09-15T03:51:44.726801+0000","172.21.15.39:6800/748677190":"2024-09-15T03:51:44.726801+0000","172.21.15.39:0/209353267":"2024-09-15T03:46:59.511580+0000","172.21.15.39:6800/3127470450":"2024-09-15T03:46:59.511580+0000","172.21.15.39:0/2951004223":"2024-09-15T03:51:44.726801+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:52:05.109 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:04 smithi039 ceph-mon[29586]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-14T03:52:05.109 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:04 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:05.109 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:04 smithi039 ceph-mon[40962]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-14T03:52:05.110 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:04 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:05.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:04 smithi129 ceph-mon[33173]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail; 204 B/s wr, 0 op/s 2024-09-14T03:52:05.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:04 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:05.743 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-14T03:52:05.743 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T03:52:05.950 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:52:06.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:05 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/655358434' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:52:06.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:05 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/655358434' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:52:06.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:05 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/655358434' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:52:07.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:06 smithi129 ceph-mon[33173]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:07.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:06 smithi039 ceph-mon[29586]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:07.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:06 smithi039 ceph-mon[40962]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:07.997 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:52:07.997 INFO:teuthology.orchestra.run.smithi039.stdout:{"epoch":67,"fsid":"bd7c803c-724b-11ef-bceb-c7b262605968","created":"2024-09-14T03:46:13.726476+0000","modified":"2024-09-14T03:51:58.662322+0000","last_up_change":"2024-09-14T03:51:13.230622+0000","last_in_change":"2024-09-14T03:50:56.569390+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:49:18.022402+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T03:51:45.764151+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"5fdfd478-dd27-4674-a7d6-53177d9e549a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6802","nonce":1775094854},{"type":"v1","addr":"172.21.15.39:6803","nonce":1775094854}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6804","nonce":1775094854},{"type":"v1","addr":"172.21.15.39:6805","nonce":1775094854}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6808","nonce":1775094854},{"type":"v1","addr":"172.21.15.39:6809","nonce":1775094854}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6806","nonce":1775094854},{"type":"v1","addr":"172.21.15.39:6807","nonce":1775094854}]},"public_addr":"172.21.15.39:6803/1775094854","cluster_addr":"172.21.15.39:6805/1775094854","heartbeat_back_addr":"172.21.15.39:6809/1775094854","heartbeat_front_addr":"172.21.15.39:6807/1775094854","state":["exists","up"]},{"osd":1,"uuid":"991c236e-1679-4c1e-a1a9-3894b1b12055","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6810","nonce":2410143394},{"type":"v1","addr":"172.21.15.39:6811","nonce":2410143394}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6812","nonce":2410143394},{"type":"v1","addr":"172.21.15.39:6813","nonce":2410143394}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6816","nonce":2410143394},{"type":"v1","addr":"172.21.15.39:6817","nonce":2410143394}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6814","nonce":2410143394},{"type":"v1","addr":"172.21.15.39:6815","nonce":2410143394}]},"public_addr":"172.21.15.39:6811/2410143394","cluster_addr":"172.21.15.39:6813/2410143394","heartbeat_back_addr":"172.21.15.39:6817/2410143394","heartbeat_front_addr":"172.21.15.39:6815/2410143394","state":["exists","up"]},{"osd":2,"uuid":"f9611ffd-5d37-4465-8114-ecc70d4d6d14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6818","nonce":3600430167},{"type":"v1","addr":"172.21.15.39:6819","nonce":3600430167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6820","nonce":3600430167},{"type":"v1","addr":"172.21.15.39:6821","nonce":3600430167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6824","nonce":3600430167},{"type":"v1","addr":"172.21.15.39:6825","nonce":3600430167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6822","nonce":3600430167},{"type":"v1","addr":"172.21.15.39:6823","nonce":3600430167}]},"public_addr":"172.21.15.39:6819/3600430167","cluster_addr":"172.21.15.39:6821/3600430167","heartbeat_back_addr":"172.21.15.39:6825/3600430167","heartbeat_front_addr":"172.21.15.39:6823/3600430167","state":["exists","up"]},{"osd":3,"uuid":"56b6877b-2107-47d6-8da9-015db7352288","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6826","nonce":1027302150},{"type":"v1","addr":"172.21.15.39:6827","nonce":1027302150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6828","nonce":1027302150},{"type":"v1","addr":"172.21.15.39:6829","nonce":1027302150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6832","nonce":1027302150},{"type":"v1","addr":"172.21.15.39:6833","nonce":1027302150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6830","nonce":1027302150},{"type":"v1","addr":"172.21.15.39:6831","nonce":1027302150}]},"public_addr":"172.21.15.39:6827/1027302150","cluster_addr":"172.21.15.39:6829/1027302150","heartbeat_back_addr":"172.21.15.39:6833/1027302150","heartbeat_front_addr":"172.21.15.39:6831/1027302150","state":["exists","up"]},{"osd":4,"uuid":"2d5523c4-5871-4fb3-bc0d-62c3b7d8b6a1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6800","nonce":168261430},{"type":"v1","addr":"172.21.15.129:6801","nonce":168261430}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6802","nonce":168261430},{"type":"v1","addr":"172.21.15.129:6803","nonce":168261430}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6806","nonce":168261430},{"type":"v1","addr":"172.21.15.129:6807","nonce":168261430}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6804","nonce":168261430},{"type":"v1","addr":"172.21.15.129:6805","nonce":168261430}]},"public_addr":"172.21.15.129:6801/168261430","cluster_addr":"172.21.15.129:6803/168261430","heartbeat_back_addr":"172.21.15.129:6807/168261430","heartbeat_front_addr":"172.21.15.129:6805/168261430","state":["exists","up"]},{"osd":5,"uuid":"a3ba55fd-f32e-4f6d-9b48-d2845a3474cd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6808","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6809","nonce":2184993002}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6810","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6811","nonce":2184993002}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6814","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6815","nonce":2184993002}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6812","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6813","nonce":2184993002}]},"public_addr":"172.21.15.129:6809/2184993002","cluster_addr":"172.21.15.129:6811/2184993002","heartbeat_back_addr":"172.21.15.129:6815/2184993002","heartbeat_front_addr":"172.21.15.129:6813/2184993002","state":["exists","up"]},{"osd":6,"uuid":"79e9c0a2-86a4-4267-a799-059d473d7b74","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6816","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6817","nonce":3767031353}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6818","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6819","nonce":3767031353}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6822","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6823","nonce":3767031353}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6820","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6821","nonce":3767031353}]},"public_addr":"172.21.15.129:6817/3767031353","cluster_addr":"172.21.15.129:6819/3767031353","heartbeat_back_addr":"172.21.15.129:6823/3767031353","heartbeat_front_addr":"172.21.15.129:6821/3767031353","state":["exists","up"]},{"osd":7,"uuid":"22f41628-4b88-4b8c-82b2-fcd2802da1bf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6824","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6825","nonce":2370414690}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6826","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6827","nonce":2370414690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6830","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6831","nonce":2370414690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6828","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6829","nonce":2370414690}]},"public_addr":"172.21.15.129:6825/2370414690","cluster_addr":"172.21.15.129:6827/2370414690","heartbeat_back_addr":"172.21.15.129:6831/2370414690","heartbeat_front_addr":"172.21.15.129:6829/2370414690","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:48:23.355518+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:48:48.715441+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:49:13.415323+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:49:39.935205+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:50:01.649042+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:50:24.001678+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:50:47.985011+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:51:11.584825+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.39:0/2587896825":"2024-09-15T03:51:44.726801+0000","172.21.15.39:0/754571920":"2024-09-15T03:51:44.726801+0000","172.21.15.39:0/2133325947":"2024-09-15T03:46:35.805097+0000","172.21.15.39:0/81449629":"2024-09-15T03:46:35.805097+0000","172.21.15.39:6800/628437919":"2024-09-15T03:46:35.805097+0000","172.21.15.39:6801/3127470450":"2024-09-15T03:46:59.511580+0000","172.21.15.39:6801/748677190":"2024-09-15T03:51:44.726801+0000","172.21.15.39:6801/628437919":"2024-09-15T03:46:35.805097+0000","172.21.15.39:0/2670041591":"2024-09-15T03:46:35.805097+0000","172.21.15.39:0/3161547781":"2024-09-15T03:46:59.511580+0000","172.21.15.39:0/1530637819":"2024-09-15T03:46:59.511580+0000","172.21.15.39:0/2745905728":"2024-09-15T03:51:44.726801+0000","172.21.15.39:6800/748677190":"2024-09-15T03:51:44.726801+0000","172.21.15.39:0/209353267":"2024-09-15T03:46:59.511580+0000","172.21.15.39:6800/3127470450":"2024-09-15T03:46:59.511580+0000","172.21.15.39:0/2951004223":"2024-09-15T03:51:44.726801+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:52:08.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:08 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/2583075372' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:52:08.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:08 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/2583075372' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:52:08.363 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:08 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/2583075372' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:52:08.816 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-14T03:52:08.816 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-14T03:52:08.817 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-14T03:52:08.817 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-14T03:52:08.818 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-14T03:52:08.818 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-14T03:52:08.818 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-14T03:52:08.819 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-14T03:52:09.049 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:52:09 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:52:09] "GET /metrics HTTP/1.1" 200 28510 "" "Prometheus/2.43.0" 2024-09-14T03:52:09.177 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:09 smithi129 ceph-mon[33173]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:09.272 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:52:09.296 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:52:09.296 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:52:09.297 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:52:09.298 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:52:09.298 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:52:09.303 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:52:09.306 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:52:09.411 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:09 smithi039 ceph-mon[29586]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:09.411 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:09 smithi039 ceph-mon[40962]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:10.037 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:52:09 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[71896]: ts=2024-09-14T03:52:09.847Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002986664s 2024-09-14T03:52:10.771 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:10 smithi129 systemd[1]: Starting Ceph grafana.a for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:52:11.126 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:10 smithi129 podman[64625]: 2024-09-14 03:52:10.768828692 +0000 UTC m=+0.015063270 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-09-14T03:52:11.127 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:10 smithi129 podman[64625]: 2024-09-14 03:52:10.898418687 +0000 UTC m=+0.144662961 container create 71f29794559040dddb6fa71feb0552a20620bce62beb18a63fe6421b3951123f (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a, version=8.5, architecture=x86_64, io.openshift.tags=base rhel8, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, name=ubi8, vendor=Red Hat, Inc., release=239.1651231664, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, com.redhat.component=ubi8-container) 2024-09-14T03:52:11.127 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:10 smithi129 ceph-mon[33173]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:11.177 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:10 smithi039 ceph-mon[29586]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:11.177 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:10 smithi039 ceph-mon[40962]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:11.379 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 podman[64625]: 2024-09-14 03:52:11.125253476 +0000 UTC m=+0.371488058 container init 71f29794559040dddb6fa71feb0552a20620bce62beb18a63fe6421b3951123f (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.openshift.expose-services=, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, name=ubi8, maintainer=Paul Cuzner , io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, release=239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-09-14T03:52:11.379 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 podman[64625]: 2024-09-14 03:52:11.130435739 +0000 UTC m=+0.376670319 container start 71f29794559040dddb6fa71feb0552a20620bce62beb18a63fe6421b3951123f (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a, vendor=Red Hat, Inc., build-date=2022-04-29T11:48:11.055897, architecture=x86_64, io.openshift.tags=base rhel8, io.buildah.version=1.32.0, com.redhat.component=ubi8-container, description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, vcs-type=git, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-09-14T03:52:11.379 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 bash[64625]: 71f29794559040dddb6fa71feb0552a20620bce62beb18a63fe6421b3951123f 2024-09-14T03:52:11.379 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 systemd[1]: Started Ceph grafana.a for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:52:11.379 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=settings t=2024-09-14T03:52:11.285647577Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-09-14T03:52:11.379 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=settings t=2024-09-14T03:52:11.285830011Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-09-14T03:52:11.379 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=settings t=2024-09-14T03:52:11.285849223Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-14T03:52:11.379 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=settings t=2024-09-14T03:52:11.285856514Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-14T03:52:11.380 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=settings t=2024-09-14T03:52:11.285863181Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-14T03:52:11.380 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=settings t=2024-09-14T03:52:11.285869693Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-14T03:52:11.380 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=settings t=2024-09-14T03:52:11.285876054Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-14T03:52:11.380 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=settings t=2024-09-14T03:52:11.28588379Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-14T03:52:11.381 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=settings t=2024-09-14T03:52:11.285889963Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-14T03:52:11.381 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=settings t=2024-09-14T03:52:11.28589607Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-14T03:52:11.381 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=settings t=2024-09-14T03:52:11.285901732Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-14T03:52:11.381 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=settings t=2024-09-14T03:52:11.285912991Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-14T03:52:11.381 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=settings t=2024-09-14T03:52:11.285923506Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-14T03:52:11.381 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=settings t=2024-09-14T03:52:11.285929883Z level=info msg="App mode production" 2024-09-14T03:52:11.381 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=sqlstore t=2024-09-14T03:52:11.285976686Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-14T03:52:11.381 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=sqlstore t=2024-09-14T03:52:11.285992806Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-14T03:52:11.381 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.286895359Z level=info msg="Starting DB migrations" 2024-09-14T03:52:11.381 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.28697572Z level=info msg="Executing migration" id="create migration_log table" 2024-09-14T03:52:11.381 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.287513031Z level=info msg="Executing migration" id="create user table" 2024-09-14T03:52:11.381 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.290321809Z level=info msg="Executing migration" id="add unique index user.login" 2024-09-14T03:52:11.381 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.290820084Z level=info msg="Executing migration" id="add unique index user.email" 2024-09-14T03:52:11.381 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.291277963Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-09-14T03:52:11.382 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.2918769Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-09-14T03:52:11.382 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.292299034Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-09-14T03:52:11.382 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.293004268Z level=info msg="Executing migration" id="create user table v2" 2024-09-14T03:52:11.382 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.293433097Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-09-14T03:52:11.382 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.29382313Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-09-14T03:52:11.382 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.294289479Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-09-14T03:52:11.382 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.294655982Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-09-14T03:52:11.382 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.295014669Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-09-14T03:52:11.382 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.297802902Z level=info msg="Executing migration" id="Update user table charset" 2024-09-14T03:52:11.382 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.298136021Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-09-14T03:52:11.382 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.298563873Z level=info msg="Executing migration" id="Add missing user data" 2024-09-14T03:52:11.382 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.298933103Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-09-14T03:52:11.382 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.299397577Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-09-14T03:52:11.383 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.299814009Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-09-14T03:52:11.383 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.300288236Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-09-14T03:52:11.383 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.301448843Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-09-14T03:52:11.383 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.304456307Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-09-14T03:52:11.383 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.307145536Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-09-14T03:52:11.383 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.30761086Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-09-14T03:52:11.383 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.310446756Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-09-14T03:52:11.383 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.310868785Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-09-14T03:52:11.383 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.311171016Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-09-14T03:52:11.383 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.311521558Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-09-14T03:52:11.383 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.311867672Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-09-14T03:52:11.383 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.312266534Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-09-14T03:52:11.383 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.312624016Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-09-14T03:52:11.384 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.313308246Z level=info msg="Executing migration" id="create temp_user v2" 2024-09-14T03:52:11.384 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.313713455Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-09-14T03:52:11.384 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.314126021Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-09-14T03:52:11.384 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.314518111Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-09-14T03:52:11.384 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.314892156Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-09-14T03:52:11.384 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.315317902Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-09-14T03:52:11.384 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.315623661Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-09-14T03:52:11.384 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.31595977Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-09-14T03:52:11.384 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.316308888Z level=info msg="Executing migration" id="create star table" 2024-09-14T03:52:11.384 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.316651104Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-09-14T03:52:11.384 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.317078791Z level=info msg="Executing migration" id="create org table v1" 2024-09-14T03:52:11.384 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.317473223Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-09-14T03:52:11.384 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.317869384Z level=info msg="Executing migration" id="create org_user table v1" 2024-09-14T03:52:11.385 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.318240438Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-09-14T03:52:11.385 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.318624641Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-09-14T03:52:11.385 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.319025889Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-09-14T03:52:11.385 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.319409231Z level=info msg="Executing migration" id="Update org table charset" 2024-09-14T03:52:11.386 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.322117897Z level=info msg="Executing migration" id="Update org_user table charset" 2024-09-14T03:52:11.386 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.322414854Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-09-14T03:52:11.386 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.324695005Z level=info msg="Executing migration" id="create dashboard table" 2024-09-14T03:52:11.386 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.325098854Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-09-14T03:52:11.386 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.327371267Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-09-14T03:52:11.386 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.327803192Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-09-14T03:52:11.386 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.33047059Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-09-14T03:52:11.386 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.333045839Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-09-14T03:52:11.386 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.335771994Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-09-14T03:52:11.386 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.336690393Z level=info msg="Executing migration" id="create dashboard v2" 2024-09-14T03:52:11.386 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.337513512Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-09-14T03:52:11.386 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.33835045Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-09-14T03:52:11.387 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.339226428Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-09-14T03:52:11.387 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.339600799Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-09-14T03:52:11.387 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.340390287Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-09-14T03:52:11.387 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.340699573Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-09-14T03:52:11.387 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.341262016Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-09-14T03:52:11.387 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.34210779Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-09-14T03:52:11.387 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.342610609Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-09-14T03:52:11.387 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.343426299Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-09-14T03:52:11.387 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.343942382Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-09-14T03:52:11.387 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.344360555Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-09-14T03:52:11.387 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.345107734Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-09-14T03:52:11.387 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.345401383Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-09-14T03:52:11.387 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.346123662Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-09-14T03:52:11.387 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.346615987Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-09-14T03:52:11.388 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.347111617Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-09-14T03:52:11.388 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.34797141Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-09-14T03:52:11.388 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.348769062Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-09-14T03:52:11.388 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.349117783Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-09-14T03:52:11.388 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.349980853Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-09-14T03:52:11.388 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.350811706Z level=info msg="Executing migration" id="Update dashboard title length" 2024-09-14T03:52:11.388 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.351693297Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-09-14T03:52:11.388 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.352122699Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-09-14T03:52:11.388 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.352997052Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-09-14T03:52:11.388 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.353900421Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-09-14T03:52:11.388 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.354746075Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-09-14T03:52:11.388 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.355174944Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-09-14T03:52:11.388 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.35602981Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-09-14T03:52:11.389 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.356859774Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-09-14T03:52:11.389 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.357636741Z level=info msg="Executing migration" id="Add check_sum column" 2024-09-14T03:52:11.389 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.358175606Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-09-14T03:52:11.389 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.359023296Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-09-14T03:52:11.389 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.359864651Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-09-14T03:52:11.389 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.360717613Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-09-14T03:52:11.389 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.361167159Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-09-14T03:52:11.389 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.364056576Z level=info msg="Executing migration" id="create data_source table" 2024-09-14T03:52:11.389 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.366827242Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-09-14T03:52:11.389 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.367262811Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-09-14T03:52:11.389 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.369504875Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-09-14T03:52:11.389 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.372338625Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-09-14T03:52:11.389 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.375143592Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-09-14T03:52:11.390 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.377997267Z level=info msg="Executing migration" id="create data_source table v2" 2024-09-14T03:52:11.390 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.378431473Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-09-14T03:52:11.390 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.381050975Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-09-14T03:52:11.390 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.381511443Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-09-14T03:52:11.632 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.383822234Z level=info msg="Executing migration" id="Add column with_credentials" 2024-09-14T03:52:11.633 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.384398906Z level=info msg="Executing migration" id="Add secure json data column" 2024-09-14T03:52:11.633 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.386593774Z level=info msg="Executing migration" id="Update data_source table charset" 2024-09-14T03:52:11.633 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.386941577Z level=info msg="Executing migration" id="Update initial version to 1" 2024-09-14T03:52:11.633 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.387286644Z level=info msg="Executing migration" id="Add read_only data column" 2024-09-14T03:52:11.633 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.390049448Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-09-14T03:52:11.633 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.390405094Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-09-14T03:52:11.633 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.392712117Z level=info msg="Executing migration" id="Add uid column" 2024-09-14T03:52:11.633 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.393285554Z level=info msg="Executing migration" id="Update uid value" 2024-09-14T03:52:11.633 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.395544261Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-09-14T03:52:11.633 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.396008508Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-09-14T03:52:11.633 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.398751066Z level=info msg="Executing migration" id="create api_key table" 2024-09-14T03:52:11.633 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.401419283Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-09-14T03:52:11.634 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.404080764Z level=info msg="Executing migration" id="add index api_key.key" 2024-09-14T03:52:11.634 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.404535754Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-09-14T03:52:11.634 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.407120704Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-09-14T03:52:11.634 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.40978328Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-09-14T03:52:11.634 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.412473744Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-09-14T03:52:11.634 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.41515715Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-09-14T03:52:11.635 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.417696838Z level=info msg="Executing migration" id="create api_key table v2" 2024-09-14T03:52:11.635 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.420336016Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-09-14T03:52:11.635 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.422992226Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-09-14T03:52:11.635 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.425667627Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-09-14T03:52:11.635 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.428318565Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-09-14T03:52:11.635 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.429104917Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-09-14T03:52:11.635 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.429500525Z level=info msg="Executing migration" id="Update api_key table charset" 2024-09-14T03:52:11.635 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.430287754Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-09-14T03:52:11.635 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.430929072Z level=info msg="Executing migration" id="Add service account foreign key" 2024-09-14T03:52:11.635 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.431532262Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-09-14T03:52:11.635 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.432325431Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-09-14T03:52:11.635 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.432953961Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-09-14T03:52:11.636 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.433567291Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-09-14T03:52:11.636 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.434345891Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-09-14T03:52:11.636 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.434790186Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-09-14T03:52:11.636 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.435227514Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-09-14T03:52:11.636 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.436059322Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-09-14T03:52:11.636 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.437457348Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-09-14T03:52:11.636 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.438272216Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-09-14T03:52:11.636 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.439063185Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-09-14T03:52:11.636 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.439883942Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-09-14T03:52:11.636 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.440725406Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-09-14T03:52:11.636 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.441567359Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-09-14T03:52:11.636 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.442432648Z level=info msg="Executing migration" id="create quota table v1" 2024-09-14T03:52:11.637 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.44329261Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-09-14T03:52:11.637 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.444070425Z level=info msg="Executing migration" id="Update quota table charset" 2024-09-14T03:52:11.637 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.444948281Z level=info msg="Executing migration" id="create plugin_setting table" 2024-09-14T03:52:11.637 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.445744153Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-09-14T03:52:11.637 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.446592566Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-09-14T03:52:11.637 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.447407678Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-09-14T03:52:11.637 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.448197663Z level=info msg="Executing migration" id="create session table" 2024-09-14T03:52:11.637 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.449085587Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-09-14T03:52:11.637 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.449884401Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-09-14T03:52:11.637 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.45073735Z level=info msg="Executing migration" id="create playlist table v2" 2024-09-14T03:52:11.637 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.451604931Z level=info msg="Executing migration" id="create playlist item table v2" 2024-09-14T03:52:11.637 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.452468648Z level=info msg="Executing migration" id="Update playlist table charset" 2024-09-14T03:52:11.637 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.453305882Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-09-14T03:52:11.637 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.454094895Z level=info msg="Executing migration" id="drop preferences table v2" 2024-09-14T03:52:11.638 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.454958394Z level=info msg="Executing migration" id="drop preferences table v3" 2024-09-14T03:52:11.638 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.45582944Z level=info msg="Executing migration" id="create preferences table v3" 2024-09-14T03:52:11.638 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.456671571Z level=info msg="Executing migration" id="Update preferences table charset" 2024-09-14T03:52:11.638 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.457487573Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-09-14T03:52:11.638 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.458123985Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-09-14T03:52:11.638 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.458454223Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-09-14T03:52:11.638 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.4593125Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-09-14T03:52:11.638 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.460157988Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-09-14T03:52:11.638 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.460997353Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-09-14T03:52:11.638 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.461893519Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-09-14T03:52:11.638 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.462787408Z level=info msg="Executing migration" id="create alert table v1" 2024-09-14T03:52:11.638 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.463657511Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-09-14T03:52:11.638 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.464513955Z level=info msg="Executing migration" id="add index alert state" 2024-09-14T03:52:11.639 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.465321738Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-09-14T03:52:11.639 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.46619803Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-09-14T03:52:11.639 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.468972313Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-09-14T03:52:11.639 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.471680696Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-09-14T03:52:11.639 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.474318578Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-09-14T03:52:11.639 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.477101556Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-09-14T03:52:11.639 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.479862947Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-09-14T03:52:11.640 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.482735917Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-09-14T03:52:11.640 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.485370162Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-09-14T03:52:11.640 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.488286334Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-09-14T03:52:11.640 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.491043928Z level=info msg="Executing migration" id="Add column is_default" 2024-09-14T03:52:11.640 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.493838582Z level=info msg="Executing migration" id="Add column frequency" 2024-09-14T03:52:11.640 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.496676675Z level=info msg="Executing migration" id="Add column send_reminder" 2024-09-14T03:52:11.640 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.499354154Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-09-14T03:52:11.640 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.502133445Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-09-14T03:52:11.641 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.504960827Z level=info msg="Executing migration" id="Update alert table charset" 2024-09-14T03:52:11.641 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.507537181Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-09-14T03:52:11.641 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.510142971Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-09-14T03:52:11.641 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.512981524Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-09-14T03:52:11.641 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.515778743Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-09-14T03:52:11.641 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.518492268Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-09-14T03:52:11.641 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.52096931Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-09-14T03:52:11.641 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.523669602Z level=info msg="Executing migration" id="Add for to alert table" 2024-09-14T03:52:11.641 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.526441072Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-09-14T03:52:11.641 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.529135008Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-09-14T03:52:11.641 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.531845113Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-09-14T03:52:11.641 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.534497805Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-09-14T03:52:11.641 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.537101903Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-09-14T03:52:11.642 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.539691191Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-09-14T03:52:11.642 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.542415995Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-09-14T03:52:11.642 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.544958173Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-09-14T03:52:11.642 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.547734629Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-09-14T03:52:11.642 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.550556938Z level=info msg="Executing migration" id="create annotation table v5" 2024-09-14T03:52:11.642 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.553256485Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-09-14T03:52:11.642 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.555955041Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-09-14T03:52:11.642 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.558562232Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-09-14T03:52:11.642 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.561321523Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-09-14T03:52:11.642 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.562080455Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-09-14T03:52:11.642 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.562893761Z level=info msg="Executing migration" id="Update annotation table charset" 2024-09-14T03:52:11.642 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.563670491Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-09-14T03:52:11.642 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.564784754Z level=info msg="Executing migration" id="Drop category_id index" 2024-09-14T03:52:11.642 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.565629192Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-09-14T03:52:11.643 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.566709905Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-09-14T03:52:11.643 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.5674755Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-09-14T03:52:11.643 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.568313413Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-09-14T03:52:11.643 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.569125982Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-09-14T03:52:11.643 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.571114772Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-09-14T03:52:11.643 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.571973156Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-09-14T03:52:11.643 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.572772718Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-09-14T03:52:11.643 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.57365758Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-09-14T03:52:11.643 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.574550364Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-09-14T03:52:11.643 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.575375367Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-09-14T03:52:11.643 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.576132856Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-09-14T03:52:11.643 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.57695097Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-09-14T03:52:11.643 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.577805531Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-09-14T03:52:11.644 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.578650849Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-09-14T03:52:11.644 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.579496761Z level=info msg="Executing migration" id="Add epoch_end column" 2024-09-14T03:52:11.644 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.580352518Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-09-14T03:52:11.644 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.581178589Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-09-14T03:52:11.644 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.582025563Z level=info msg="Executing migration" id="Move region to single row" 2024-09-14T03:52:11.644 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.582903266Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-09-14T03:52:11.644 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.583757717Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-09-14T03:52:11.644 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.584611832Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-09-14T03:52:11.644 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.58547033Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-09-14T03:52:11.644 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.586312127Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-09-14T03:52:11.644 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.587145516Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-09-14T03:52:11.644 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.587988451Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-09-14T03:52:11.644 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.588848878Z level=info msg="Executing migration" id="create test_data table" 2024-09-14T03:52:11.644 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.589717364Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-09-14T03:52:11.645 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.590561221Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-09-14T03:52:11.645 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.591410912Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-09-14T03:52:11.645 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.592310904Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-09-14T03:52:11.645 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.5951543Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-09-14T03:52:11.645 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.597993946Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-09-14T03:52:11.645 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.600620013Z level=info msg="Executing migration" id="create team table" 2024-09-14T03:52:11.645 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.603431836Z level=info msg="Executing migration" id="add index team.org_id" 2024-09-14T03:52:11.645 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.606316049Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-09-14T03:52:11.645 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.609122533Z level=info msg="Executing migration" id="create team member table" 2024-09-14T03:52:11.645 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.611867022Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-09-14T03:52:11.645 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.614539533Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-09-14T03:52:11.645 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.617426816Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-09-14T03:52:11.645 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.620017044Z level=info msg="Executing migration" id="Add column email to team table" 2024-09-14T03:52:11.646 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.62279036Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-09-14T03:52:11.646 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.625446226Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-09-14T03:52:11.646 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.628413013Z level=info msg="Executing migration" id="create dashboard acl table" 2024-09-14T03:52:11.646 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.631385753Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-09-14T03:52:11.646 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.634367691Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-09-14T03:52:11.646 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.636942183Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-09-14T03:52:11.882 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.639807566Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-09-14T03:52:11.883 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.642649545Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-09-14T03:52:11.883 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.645324885Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-09-14T03:52:11.883 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.645766479Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-09-14T03:52:11.883 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.648367942Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-09-14T03:52:11.883 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.651108135Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-09-14T03:52:11.883 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.653765094Z level=info msg="Executing migration" id="create tag table" 2024-09-14T03:52:11.883 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.654176383Z level=info msg="Executing migration" id="add index tag.key_value" 2024-09-14T03:52:11.883 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.655018666Z level=info msg="Executing migration" id="create login attempt table" 2024-09-14T03:52:11.883 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.655868101Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-09-14T03:52:11.883 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.656678504Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-09-14T03:52:11.884 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.657119983Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-09-14T03:52:11.884 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.659740607Z level=info msg="Executing migration" id="create login_attempt v2" 2024-09-14T03:52:11.884 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.660527465Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-09-14T03:52:11.884 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.6613203Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-09-14T03:52:11.884 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.661675692Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-09-14T03:52:11.884 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.662524994Z level=info msg="Executing migration" id="create user auth table" 2024-09-14T03:52:11.884 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.663376492Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-09-14T03:52:11.884 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.664221435Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-09-14T03:52:11.886 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.664532553Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-09-14T03:52:11.886 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.667442347Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-09-14T03:52:11.886 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.67018827Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-09-14T03:52:11.886 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.673039437Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-09-14T03:52:11.886 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.673785434Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-09-14T03:52:11.886 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.676601031Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-09-14T03:52:11.886 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.679412539Z level=info msg="Executing migration" id="create server_lock table" 2024-09-14T03:52:11.886 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.682188136Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-09-14T03:52:11.886 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.682626333Z level=info msg="Executing migration" id="create user auth token table" 2024-09-14T03:52:11.886 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.685376734Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-09-14T03:52:11.886 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.688138452Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-09-14T03:52:11.887 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.690906848Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-09-14T03:52:11.887 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.69134628Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-09-14T03:52:11.887 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.693994028Z level=info msg="Executing migration" id="create cache_data table" 2024-09-14T03:52:11.887 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.696753089Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-09-14T03:52:11.887 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.699576543Z level=info msg="Executing migration" id="create short_url table v1" 2024-09-14T03:52:11.887 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.702203998Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-09-14T03:52:11.887 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.704832514Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-09-14T03:52:11.887 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.707544025Z level=info msg="Executing migration" id="delete alert_definition table" 2024-09-14T03:52:11.887 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.707884376Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-09-14T03:52:11.887 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.708373507Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-09-14T03:52:11.887 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.711089607Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-09-14T03:52:11.887 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.711558372Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-09-14T03:52:11.887 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.711853313Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-09-14T03:52:11.888 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.712303704Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-09-14T03:52:11.888 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.712708069Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-09-14T03:52:11.888 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.713165737Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-09-14T03:52:11.888 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.7136563Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-09-14T03:52:11.888 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.714498619Z level=info msg="Executing migration" id="drop alert_definition table" 2024-09-14T03:52:11.888 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.714937179Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-09-14T03:52:11.888 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.715279195Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-09-14T03:52:11.888 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.715690865Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-09-14T03:52:11.888 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.716171707Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-09-14T03:52:11.888 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.71661602Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-09-14T03:52:11.888 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.716929141Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-09-14T03:52:11.888 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.717388064Z level=info msg="Executing migration" id="create alert_instance table" 2024-09-14T03:52:11.888 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.717804262Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-09-14T03:52:11.889 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.718260324Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-09-14T03:52:11.889 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.718734194Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-09-14T03:52:11.889 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.719558331Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-09-14T03:52:11.889 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.720012057Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-09-14T03:52:11.889 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.72048827Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-09-14T03:52:11.889 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.724028948Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-09-14T03:52:11.889 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.727463426Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-09-14T03:52:11.889 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.727961718Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-09-14T03:52:11.889 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.728435346Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-09-14T03:52:11.889 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.729320888Z level=info msg="Executing migration" id="create alert_rule table" 2024-09-14T03:52:11.889 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.7297828Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-09-14T03:52:11.889 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.730245358Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-09-14T03:52:11.889 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.730693856Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-09-14T03:52:11.890 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.731158454Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-09-14T03:52:11.890 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.731481398Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-09-14T03:52:11.890 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.732317232Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-09-14T03:52:11.890 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.733155546Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-09-14T03:52:11.890 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.73397823Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-09-14T03:52:11.890 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.734417038Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-09-14T03:52:11.890 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.73483518Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-09-14T03:52:11.890 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.735649745Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-09-14T03:52:11.890 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.736505222Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-09-14T03:52:11.890 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.736954758Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-09-14T03:52:11.890 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.737820968Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-09-14T03:52:11.890 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.738680881Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-09-14T03:52:11.890 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.739011903Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-09-14T03:52:11.890 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.739502238Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-09-14T03:52:11.891 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.742135908Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-09-14T03:52:11.891 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.742593121Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-09-14T03:52:11.891 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.743009105Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-09-14T03:52:11.891 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.743890556Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-09-14T03:52:11.891 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.744741201Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-09-14T03:52:11.891 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.747379787Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-09-14T03:52:11.891 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.750097196Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-09-14T03:52:11.891 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.752617148Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-09-14T03:52:11.891 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.752973572Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-09-14T03:52:11.891 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.755649008Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-09-14T03:52:11.891 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.758406069Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-09-14T03:52:11.891 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.760948986Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-09-14T03:52:11.891 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.761831633Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-09-14T03:52:11.891 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.764567886Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-09-14T03:52:11.892 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.76730807Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-09-14T03:52:11.892 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.769887399Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-09-14T03:52:11.892 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.770685315Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-09-14T03:52:11.892 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.771588254Z level=info msg="Executing migration" id="create provenance_type table" 2024-09-14T03:52:11.892 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.772528585Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-09-14T03:52:11.892 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.773345642Z level=info msg="Executing migration" id="create alert_image table" 2024-09-14T03:52:11.892 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.773766158Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-09-14T03:52:11.892 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.774623475Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-09-14T03:52:11.892 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.775459816Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-09-14T03:52:11.892 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.775888088Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-09-14T03:52:11.892 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.77633213Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-09-14T03:52:11.892 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.776433058Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-09-14T03:52:11.892 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.777134465Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-09-14T03:52:11.893 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.777525947Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-09-14T03:52:11.893 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.777954632Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-09-14T03:52:11.893 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.77809744Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-09-14T03:52:11.893 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.778231627Z level=info msg="alerts found to migrate" alerts=0 2024-09-14T03:52:11.893 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.778650724Z level=info msg="Executing migration" id="create library_element table v1" 2024-09-14T03:52:11.893 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.779131245Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-09-14T03:52:11.893 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.779598405Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-09-14T03:52:11.893 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.780444348Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-09-14T03:52:11.893 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.781261542Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-09-14T03:52:11.893 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.782039699Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-09-14T03:52:11.893 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.782857524Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-09-14T03:52:11.893 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.783714315Z level=info msg="Executing migration" id="create data_keys table" 2024-09-14T03:52:11.893 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.784602694Z level=info msg="Executing migration" id="create secrets table" 2024-09-14T03:52:11.893 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.785422384Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-09-14T03:52:11.894 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.789777958Z level=info msg="Executing migration" id="add name column into data_keys" 2024-09-14T03:52:11.894 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.790707601Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-09-14T03:52:11.894 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.79158404Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-09-14T03:52:11.894 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.796015906Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-09-14T03:52:11.894 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.800434923Z level=info msg="Executing migration" id="create kv_store table v1" 2024-09-14T03:52:11.894 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.801235127Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-09-14T03:52:11.894 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.802079845Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-09-14T03:52:11.894 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.802923782Z level=info msg="Executing migration" id="create permission table" 2024-09-14T03:52:11.894 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.80335957Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-09-14T03:52:11.894 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.804195222Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-09-14T03:52:11.894 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.805066447Z level=info msg="Executing migration" id="create role table" 2024-09-14T03:52:11.894 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.805928547Z level=info msg="Executing migration" id="add column display_name" 2024-09-14T03:52:11.895 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.806895533Z level=info msg="Executing migration" id="add column group_name" 2024-09-14T03:52:11.895 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.809603951Z level=info msg="Executing migration" id="add index role.org_id" 2024-09-14T03:52:11.895 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.812416595Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-09-14T03:52:11.895 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.815249179Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-09-14T03:52:11.895 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.815761973Z level=info msg="Executing migration" id="create team role table" 2024-09-14T03:52:11.895 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.818524371Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-09-14T03:52:11.895 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.821357804Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-09-14T03:52:11.895 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.824075229Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-09-14T03:52:11.895 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.824722781Z level=info msg="Executing migration" id="create user role table" 2024-09-14T03:52:11.895 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.827217221Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-09-14T03:52:11.895 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.829903144Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-09-14T03:52:11.896 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.832614945Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-09-14T03:52:11.896 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.833111257Z level=info msg="Executing migration" id="create builtin role table" 2024-09-14T03:52:11.896 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.835827122Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-09-14T03:52:11.896 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.838806377Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-09-14T03:52:11.896 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.841275675Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-09-14T03:52:11.896 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.842297734Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-09-14T03:52:11.896 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.845039859Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-09-14T03:52:11.896 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.847706215Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-09-14T03:52:11.896 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.850239929Z level=info msg="Executing migration" id="add unique index role.uid" 2024-09-14T03:52:11.896 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.850662464Z level=info msg="Executing migration" id="create seed assignment table" 2024-09-14T03:52:11.896 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.853487767Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-09-14T03:52:11.896 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.856126723Z level=info msg="Executing migration" id="add column hidden to role table" 2024-09-14T03:52:11.896 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.85866288Z level=info msg="Executing migration" id="create query_history table v1" 2024-09-14T03:52:11.897 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.859079801Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-09-14T03:52:11.897 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.861814024Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-09-14T03:52:11.897 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.862584638Z level=info msg="Executing migration" id="teams permissions migration" 2024-09-14T03:52:11.897 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.863361663Z level=info msg="Executing migration" id="dashboard permissions" 2024-09-14T03:52:11.897 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.863817089Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-09-14T03:52:11.897 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.864653703Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-09-14T03:52:11.897 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.865496399Z level=info msg="Executing migration" id="alerting notification permissions" 2024-09-14T03:52:11.897 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.866345127Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-09-14T03:52:11.898 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.866731341Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-09-14T03:52:11.898 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.86756218Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-09-14T03:52:11.898 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.868510149Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-09-14T03:52:11.898 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.869322273Z level=info msg="Executing migration" id="create correlation table v1" 2024-09-14T03:52:11.898 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.869752273Z level=info msg="Executing migration" id="add index correlations.uid" 2024-09-14T03:52:11.898 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.87055077Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-09-14T03:52:11.898 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.871400841Z level=info msg="Executing migration" id="add correlation config column" 2024-09-14T03:52:11.898 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.872358685Z level=info msg="Executing migration" id="create entity_events table" 2024-09-14T03:52:11.898 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.872787334Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-09-14T03:52:11.898 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.873550692Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-14T03:52:11.898 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.873633006Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-09-14T03:52:11.898 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.874445111Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-14T03:52:11.898 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.874549912Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-14T03:52:11.899 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.875270406Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-09-14T03:52:11.899 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.876094029Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-09-14T03:52:11.899 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.876931379Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-09-14T03:52:11.899 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.877770283Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-09-14T03:52:11.899 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.878647288Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-09-14T03:52:11.899 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.879086016Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-14T03:52:11.899 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.879935298Z level=info msg="Executing migration" id="Drop public config table" 2024-09-14T03:52:11.899 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.880797645Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-09-14T03:52:11.899 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.881625467Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-09-14T03:52:11.899 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.882149308Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-09-14T03:52:11.899 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.882989891Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-09-14T03:52:11.899 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.883824911Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-09-14T03:52:12.271 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.886900633Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-09-14T03:52:12.271 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.88789571Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-09-14T03:52:12.271 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.888899125Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-09-14T03:52:12.272 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.88974186Z level=info msg="Executing migration" id="add share column" 2024-09-14T03:52:12.272 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.890749283Z level=info msg="Executing migration" id="create default alerting folders" 2024-09-14T03:52:12.272 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.891241528Z level=info msg="Executing migration" id="create file table" 2024-09-14T03:52:12.272 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.894011235Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-09-14T03:52:12.272 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.896811096Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-09-14T03:52:12.272 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.899559884Z level=info msg="Executing migration" id="create file_meta table" 2024-09-14T03:52:12.272 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.899966837Z level=info msg="Executing migration" id="file table idx: path key" 2024-09-14T03:52:12.272 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.902640658Z level=info msg="Executing migration" id="set path collation in file table" 2024-09-14T03:52:12.272 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.905398197Z level=info msg="Executing migration" id="managed permissions migration" 2024-09-14T03:52:12.272 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.908107432Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-09-14T03:52:12.272 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.908468264Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-09-14T03:52:12.272 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.911129002Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-09-14T03:52:12.273 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.912190938Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-09-14T03:52:12.273 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.913853527Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-09-14T03:52:12.273 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.916488899Z level=info msg="Executing migration" id="update group index for alert rules" 2024-09-14T03:52:12.273 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.916900473Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-09-14T03:52:12.273 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.9198743Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-09-14T03:52:12.273 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.922650964Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-09-14T03:52:12.273 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.925295704Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-09-14T03:52:12.273 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.926317982Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-09-14T03:52:12.274 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.929130551Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-09-14T03:52:12.274 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.93882691Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-09-14T03:52:12.274 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.94125327Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-09-14T03:52:12.274 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.941726831Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-09-14T03:52:12.274 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.945327988Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-09-14T03:52:12.274 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.947586417Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-09-14T03:52:12.274 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.950274999Z level=info msg="Executing migration" id="create folder table" 2024-09-14T03:52:12.274 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.950730154Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-09-14T03:52:12.274 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.95339008Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-09-14T03:52:12.274 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.956144097Z level=info msg="Executing migration" id="Update folder title length" 2024-09-14T03:52:12.274 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.95883786Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-09-14T03:52:12.274 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=migrator t=2024-09-14T03:52:11.959440564Z level=info msg="migrations completed" performed=485 skipped=0 duration=672.484574ms 2024-09-14T03:52:12.274 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:11 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=sqlstore t=2024-09-14T03:52:11.960100404Z level=info msg="Created default organization" 2024-09-14T03:52:12.275 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=plugin.loader t=2024-09-14T03:52:12.047140291Z level=info msg="Plugin registered" pluginID=input 2024-09-14T03:52:12.275 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=plugin.loader t=2024-09-14T03:52:12.074960049Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-09-14T03:52:12.275 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=plugin.loader t=2024-09-14T03:52:12.074983458Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-09-14T03:52:12.275 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=secrets t=2024-09-14T03:52:12.075018438Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-14T03:52:12.275 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=query_data t=2024-09-14T03:52:12.076716614Z level=info msg="Query Service initialization" 2024-09-14T03:52:12.275 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=live.push_http t=2024-09-14T03:52:12.079028548Z level=info msg="Live Push Gateway initialization" 2024-09-14T03:52:12.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:12 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:12.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:12 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:12.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:12 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:12.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:12 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:12.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:12 smithi039 ceph-mon[29586]: Deploying daemon node-exporter.a on smithi039 2024-09-14T03:52:12.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:12 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:12.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:12 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:12.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:12 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:12.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:12 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:12.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:12 smithi039 ceph-mon[40962]: Deploying daemon node-exporter.a on smithi039 2024-09-14T03:52:12.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:12 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:12.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:12 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:12.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:12 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:12.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:12 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:12.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:12 smithi129 ceph-mon[33173]: Deploying daemon node-exporter.a on smithi039 2024-09-14T03:52:13.272 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=infra.usagestats.collector t=2024-09-14T03:52:12.806174884Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-14T03:52:13.272 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=provisioning.datasources t=2024-09-14T03:52:12.8607829Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-14T03:52:13.272 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=provisioning.datasources t=2024-09-14T03:52:12.892419915Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-09-14T03:52:13.272 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=provisioning.alerting t=2024-09-14T03:52:12.913793545Z level=info msg="starting to provision alerting" 2024-09-14T03:52:13.272 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=provisioning.alerting t=2024-09-14T03:52:12.913813821Z level=info msg="finished to provision alerting" 2024-09-14T03:52:13.273 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=grafanaStorageLogger t=2024-09-14T03:52:12.914050152Z level=info msg="storage starting" 2024-09-14T03:52:13.273 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=ngalert.state.manager t=2024-09-14T03:52:12.914143856Z level=info msg="Warming state cache for startup" 2024-09-14T03:52:13.273 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=http.server t=2024-09-14T03:52:12.915663808Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-14T03:52:13.273 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=ngalert.state.manager t=2024-09-14T03:52:12.924266124Z level=info msg="State cache has been initialized" states=0 duration=10.122011ms 2024-09-14T03:52:13.273 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=ngalert.multiorg.alertmanager t=2024-09-14T03:52:12.924302259Z level=info msg="starting MultiOrg Alertmanager" 2024-09-14T03:52:13.273 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=ticker t=2024-09-14T03:52:12.924313389Z level=info msg=starting first_tick=2024-09-14T03:52:20Z 2024-09-14T03:52:13.274 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=sqlstore.transactions t=2024-09-14T03:52:12.924321924Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-14T03:52:13.274 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=sqlstore.transactions t=2024-09-14T03:52:12.969061428Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-14T03:52:13.274 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:52:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[64640]: logger=sqlstore.transactions t=2024-09-14T03:52:12.994091629Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-14T03:52:13.528 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:13 smithi039 ceph-mon[29586]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:13.529 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:13 smithi039 ceph-mon[40962]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:13.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:13 smithi129 ceph-mon[33173]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:15.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:14 smithi039 ceph-mon[29586]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:15.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:14 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:52:15.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:14 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:15.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:14 smithi039 ceph-mon[40962]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:15.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:14 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:52:15.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:14 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:15.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:14 smithi129 ceph-mon[33173]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:15.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:14 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:52:15.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:14 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:17.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:16 smithi129 ceph-mon[33173]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:17.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:16 smithi039 ceph-mon[29586]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:17.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:16 smithi039 ceph-mon[40962]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:18.947 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:18 smithi039 ceph-mon[29586]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:18.948 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:18 smithi039 ceph-mon[40962]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:19.234 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:52:19 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:52:19] "GET /metrics HTTP/1.1" 200 28507 "" "Prometheus/2.43.0" 2024-09-14T03:52:19.234 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:19 smithi039 systemd[1]: Starting Ceph node-exporter.a for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:52:19.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:18 smithi129 ceph-mon[33173]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:20.893 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:20 smithi039 ceph-mon[29586]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:20.893 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:20 smithi039 ceph-mon[40962]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:21.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:20 smithi129 ceph-mon[33173]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:22.704 INFO:teuthology.orchestra.run.smithi039.stdout:111669149730 2024-09-14T03:52:22.705 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-14T03:52:23.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:22 smithi039 ceph-mon[29586]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:23.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:22 smithi039 ceph-mon[40962]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:23.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:22 smithi129 ceph-mon[33173]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:23.672 INFO:teuthology.orchestra.run.smithi039.stdout:34359738417 2024-09-14T03:52:23.673 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-14T03:52:24.699 INFO:teuthology.orchestra.run.smithi039.stdout:55834574892 2024-09-14T03:52:24.944 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-14T03:52:24.945 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:24 smithi039 ceph-mon[29586]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:24.945 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:24 smithi039 ceph-mon[40962]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:24.946 INFO:teuthology.orchestra.run.smithi039.stdout:201863462931 2024-09-14T03:52:24.946 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-14T03:52:25.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:24 smithi129 ceph-mon[33173]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:25.669 INFO:teuthology.orchestra.run.smithi039.stdout:236223201295 2024-09-14T03:52:25.670 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-14T03:52:26.108 INFO:teuthology.orchestra.run.smithi039.stdout:77309411368 2024-09-14T03:52:26.108 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-14T03:52:27.111 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:26 smithi039 ceph-mon[29586]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:27.111 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:26 smithi039 ceph-mon[40962]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:27.133 INFO:teuthology.orchestra.run.smithi039.stdout:167503724569 2024-09-14T03:52:27.134 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-14T03:52:27.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:26 smithi129 ceph-mon[33173]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:27.701 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:27 smithi039 bash[74681]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-14T03:52:27.701 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:27 smithi039 bash[74681]: Getting image source signatures 2024-09-14T03:52:27.701 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:27 smithi039 bash[74681]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-14T03:52:27.702 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:27 smithi039 bash[74681]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-14T03:52:27.702 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:27 smithi039 bash[74681]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-14T03:52:27.823 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:52:27.858 INFO:teuthology.orchestra.run.smithi039.stdout:133143986207 2024-09-14T03:52:27.859 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-14T03:52:27.873 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:52:27.905 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:52:28.242 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:52:28.243 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:52:28.243 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:52:29.195 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:28 smithi039 ceph-mon[29586]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:29.196 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:28 smithi039 ceph-mon[40962]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:29.196 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:52:29 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:52:29] "GET /metrics HTTP/1.1" 200 28507 "" "Prometheus/2.43.0" 2024-09-14T03:52:29.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:28 smithi129 ceph-mon[33173]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:30.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:29 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:52:30.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:29 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:52:30.041 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:29 smithi039 bash[74681]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-14T03:52:30.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:29 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:52:30.449 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:30 smithi039 bash[74681]: Writing manifest to image destination 2024-09-14T03:52:30.790 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:30 smithi039 podman[74681]: 2024-09-14 03:52:30.636177235 +0000 UTC m=+3.369533241 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-14T03:52:31.179 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:30 smithi039 ceph-mon[29586]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:31.179 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:30 smithi039 ceph-mon[40962]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:31.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:30 smithi129 ceph-mon[33173]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:31.507 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 podman[74681]: 2024-09-14 03:52:31.178914575 +0000 UTC m=+3.912270569 container create 4eff582f78f5762c8bf4b28b8523961d9b934c67c20840fcd7a9870683906fc9 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:52:32.254 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 podman[74681]: 2024-09-14 03:52:31.85893817 +0000 UTC m=+4.592294167 container init 4eff582f78f5762c8bf4b28b8523961d9b934c67c20840fcd7a9870683906fc9 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:52:32.254 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 podman[74681]: 2024-09-14 03:52:31.864958464 +0000 UTC m=+4.598314459 container start 4eff582f78f5762c8bf4b28b8523961d9b934c67c20840fcd7a9870683906fc9 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:52:32.254 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.869Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-14T03:52:32.254 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.869Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-14T03:52:32.254 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.870Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-14T03:52:32.254 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.870Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-14T03:52:32.254 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.870Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-14T03:52:32.254 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.870Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-14T03:52:32.255 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.870Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-14T03:52:32.255 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=arp 2024-09-14T03:52:32.255 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-14T03:52:32.255 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-14T03:52:32.255 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-14T03:52:32.255 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-14T03:52:32.255 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-14T03:52:32.255 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-14T03:52:32.256 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-14T03:52:32.256 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-14T03:52:32.256 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=edac 2024-09-14T03:52:32.256 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-14T03:52:32.256 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-14T03:52:32.256 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-14T03:52:32.256 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-14T03:52:32.256 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-14T03:52:32.256 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-14T03:52:32.256 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-14T03:52:32.257 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-14T03:52:32.257 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-14T03:52:32.257 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-14T03:52:32.257 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-14T03:52:32.257 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-14T03:52:32.257 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-14T03:52:32.257 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-14T03:52:32.257 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-14T03:52:32.257 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-14T03:52:32.257 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=os 2024-09-14T03:52:32.257 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-14T03:52:32.257 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-14T03:52:32.257 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-14T03:52:32.258 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-14T03:52:32.258 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-14T03:52:32.258 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-14T03:52:32.258 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-14T03:52:32.258 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=stat 2024-09-14T03:52:32.258 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-14T03:52:32.258 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-14T03:52:32.258 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-14T03:52:32.258 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=time 2024-09-14T03:52:32.258 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-14T03:52:32.258 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=uname 2024-09-14T03:52:32.258 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-14T03:52:32.258 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-14T03:52:32.258 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-14T03:52:32.259 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-14T03:52:32.259 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[75140]: ts=2024-09-14T03:52:31.871Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-14T03:52:33.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:32 smithi039 ceph-mon[29586]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:33.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:32 smithi039 ceph-mon[40962]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:33.072 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:52:33.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:32 smithi129 ceph-mon[33173]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:33.540 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:33 smithi039 bash[74681]: 4eff582f78f5762c8bf4b28b8523961d9b934c67c20840fcd7a9870683906fc9 2024-09-14T03:52:33.540 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:52:33 smithi039 systemd[1]: Started Ceph node-exporter.a for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:52:34.147 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:34 smithi129 bash[64918]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-09-14T03:52:34.451 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:34 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:34.452 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:34 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:34.452 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:34 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:34.452 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:34 smithi129 ceph-mon[33173]: Deploying daemon node-exporter.b on smithi129 2024-09-14T03:52:34.452 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:34 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:34.452 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:34 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:34.452 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:34 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:34.452 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:34 smithi039 ceph-mon[29586]: Deploying daemon node-exporter.b on smithi129 2024-09-14T03:52:34.453 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:34 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:34.453 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:34 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:34.453 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:34 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:34.453 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:34 smithi039 ceph-mon[40962]: Deploying daemon node-exporter.b on smithi129 2024-09-14T03:52:34.771 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:34 smithi129 bash[64918]: Getting image source signatures 2024-09-14T03:52:34.771 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:34 smithi129 bash[64918]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-09-14T03:52:34.771 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:34 smithi129 bash[64918]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-09-14T03:52:34.771 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:34 smithi129 bash[64918]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-09-14T03:52:35.266 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:35 smithi039 ceph-mon[29586]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:35.266 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:35 smithi039 ceph-mon[40962]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:35.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:35 smithi129 ceph-mon[33173]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:35.785 INFO:teuthology.orchestra.run.smithi039.stdout:111669149733 2024-09-14T03:52:36.019 INFO:teuthology.orchestra.run.smithi039.stdout:201863462934 2024-09-14T03:52:36.021 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:35 smithi129 bash[64918]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-09-14T03:52:36.021 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:35 smithi129 bash[64918]: Writing manifest to image destination 2024-09-14T03:52:36.424 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/514442405' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-14T03:52:36.424 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/769171563' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-14T03:52:36.425 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 podman[64918]: 2024-09-14 03:52:36.073549099 +0000 UTC m=+2.111471604 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-09-14T03:52:36.425 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 podman[64918]: 2024-09-14 03:52:36.171521771 +0000 UTC m=+2.209444305 container create c7581baf44f218f222deac9bf0be7fceefc52d2f46e419a580d63c8dc7d22ebd (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:52:36.425 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 podman[64918]: 2024-09-14 03:52:36.423686692 +0000 UTC m=+2.461609184 container init c7581baf44f218f222deac9bf0be7fceefc52d2f46e419a580d63c8dc7d22ebd (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:52:36.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:36 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/514442405' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-14T03:52:36.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:36 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/769171563' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-14T03:52:36.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:36 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/514442405' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-14T03:52:36.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:36 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/769171563' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-14T03:52:36.771 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 podman[64918]: 2024-09-14 03:52:36.42910641 +0000 UTC m=+2.467028910 container start c7581baf44f218f222deac9bf0be7fceefc52d2f46e419a580d63c8dc7d22ebd (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:52:36.771 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.433Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-09-14T03:52:36.772 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.433Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-09-14T03:52:36.772 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.433Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-14T03:52:36.772 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.433Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-14T03:52:36.772 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-14T03:52:36.772 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-14T03:52:36.772 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-14T03:52:36.772 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=arp 2024-09-14T03:52:36.772 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-14T03:52:36.772 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-14T03:52:36.772 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-14T03:52:36.772 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-14T03:52:36.773 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-14T03:52:36.773 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-14T03:52:36.773 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-14T03:52:36.773 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-14T03:52:36.773 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=edac 2024-09-14T03:52:36.773 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-14T03:52:36.773 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-14T03:52:36.774 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-14T03:52:36.774 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-14T03:52:36.774 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-14T03:52:36.774 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-14T03:52:36.774 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-14T03:52:36.774 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-14T03:52:36.774 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-14T03:52:36.774 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-14T03:52:36.774 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-14T03:52:36.774 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-14T03:52:36.774 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-14T03:52:36.774 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-14T03:52:36.775 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-14T03:52:36.775 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-14T03:52:36.775 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=os 2024-09-14T03:52:36.775 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-14T03:52:36.775 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-14T03:52:36.775 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-14T03:52:36.775 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-14T03:52:36.775 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-14T03:52:36.775 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-14T03:52:36.775 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-14T03:52:36.775 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=stat 2024-09-14T03:52:36.775 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-14T03:52:36.775 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-14T03:52:36.776 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-14T03:52:36.776 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=time 2024-09-14T03:52:36.776 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-14T03:52:36.776 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=uname 2024-09-14T03:52:36.776 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-14T03:52:36.776 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-14T03:52:36.776 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-14T03:52:36.776 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-09-14T03:52:36.776 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[64994]: ts=2024-09-14T03:52:36.434Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-14T03:52:36.776 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 bash[64918]: c7581baf44f218f222deac9bf0be7fceefc52d2f46e419a580d63c8dc7d22ebd 2024-09-14T03:52:36.777 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:52:36 smithi129 systemd[1]: Started Ceph node-exporter.b for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:52:36.925 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462931 got 201863462934 for osd.6 2024-09-14T03:52:36.926 DEBUG:teuthology.parallel:result is None 2024-09-14T03:52:37.533 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149730 got 111669149733 for osd.3 2024-09-14T03:52:37.533 DEBUG:teuthology.parallel:result is None 2024-09-14T03:52:37.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:37 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:37.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:37 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:37.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:37 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:37.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:37 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:37.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:37 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:37.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:37 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:52:37.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:37 smithi039 ceph-mon[29586]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:37.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:37 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:37.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:37 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:37.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:37 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:37.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:37 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:37.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:37 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:37.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:37 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:52:37.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:37 smithi039 ceph-mon[40962]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:37 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:37 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:37 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:37 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:37 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:37 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:52:38.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:37 smithi129 ceph-mon[33173]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:38.977 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:38 smithi039 ceph-mon[29586]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:38.977 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:38 smithi039 ceph-mon[40962]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:39.244 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:52:39 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:52:39] "GET /metrics HTTP/1.1" 200 28507 "" "Prometheus/2.43.0" 2024-09-14T03:52:39.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:38 smithi129 ceph-mon[33173]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:39.410 INFO:teuthology.orchestra.run.smithi039.stdout:236223201299 2024-09-14T03:52:40.012 INFO:teuthology.orchestra.run.smithi039.stdout:34359738420 2024-09-14T03:52:40.012 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:39 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/224179259' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-14T03:52:40.012 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:39 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:40.012 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:39 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/224179259' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-14T03:52:40.012 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:39 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:40.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:39 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/224179259' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-14T03:52:40.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:39 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:41.088 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738417 got 34359738420 for osd.0 2024-09-14T03:52:41.088 DEBUG:teuthology.parallel:result is None 2024-09-14T03:52:41.183 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:40 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/3371888242' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-14T03:52:41.183 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:40 smithi039 ceph-mon[29586]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:41.183 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:40 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/3371888242' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-14T03:52:41.184 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:40 smithi039 ceph-mon[40962]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:41.251 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:40 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/3371888242' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-14T03:52:41.251 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:40 smithi129 ceph-mon[33173]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:41.612 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:52:42.012 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201295 got 236223201299 for osd.7 2024-09-14T03:52:42.013 DEBUG:teuthology.parallel:result is None 2024-09-14T03:52:42.878 INFO:teuthology.orchestra.run.smithi039.stdout:133143986210 2024-09-14T03:52:42.907 INFO:teuthology.orchestra.run.smithi039.stdout:55834574896 2024-09-14T03:52:43.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:43 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:43.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:43 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:43.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:43 smithi129 ceph-mon[33173]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:43.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:43 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/1926015676' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-14T03:52:43.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:43 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/308788636' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-14T03:52:43.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:43 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:43.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:43 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:43.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:43 smithi039 ceph-mon[29586]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:43.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:43 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/1926015676' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-14T03:52:43.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:43 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/308788636' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-14T03:52:43.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:43 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:43.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:43 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:43.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:43 smithi039 ceph-mon[40962]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:43.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:43 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/1926015676' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-14T03:52:43.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:43 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/308788636' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-14T03:52:43.564 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986207 got 133143986210 for osd.4 2024-09-14T03:52:43.564 DEBUG:teuthology.parallel:result is None 2024-09-14T03:52:44.313 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574892 got 55834574896 for osd.1 2024-09-14T03:52:44.313 DEBUG:teuthology.parallel:result is None 2024-09-14T03:52:44.711 INFO:teuthology.orchestra.run.smithi039.stdout:167503724573 2024-09-14T03:52:45.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:44 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/2651443606' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-14T03:52:45.027 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:44 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/2651443606' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-14T03:52:45.027 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:44 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/2651443606' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-14T03:52:45.713 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724569 got 167503724573 for osd.5 2024-09-14T03:52:45.714 DEBUG:teuthology.parallel:result is None 2024-09-14T03:52:46.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:45 smithi129 ceph-mon[33173]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:46.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:45 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:52:46.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:45 smithi039 ceph-mon[29586]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:46.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:45 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:52:46.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:45 smithi039 ceph-mon[40962]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:46.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:45 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:52:46.914 INFO:teuthology.orchestra.run.smithi039.stdout:77309411373 2024-09-14T03:52:47.904 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:47 smithi039 ceph-mon[29586]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:47.904 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:47 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/1983534615' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-14T03:52:47.905 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:47 smithi039 ceph-mon[40962]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:47.905 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:47 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/1983534615' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-14T03:52:47.930 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411368 got 77309411373 for osd.2 2024-09-14T03:52:47.931 DEBUG:teuthology.parallel:result is None 2024-09-14T03:52:47.931 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-14T03:52:47.931 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T03:52:48.157 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:52:48.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:47 smithi129 ceph-mon[33173]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:48.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:47 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/1983534615' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-14T03:52:49.269 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:52:49 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:52:49] "GET /metrics HTTP/1.1" 200 28506 "" "Prometheus/2.43.0" 2024-09-14T03:52:49.941 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:49 smithi039 ceph-mon[29586]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:49.941 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:49 smithi039 ceph-mon[40962]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:50.205 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:52:50.205 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T03:52:50.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:49 smithi129 ceph-mon[33173]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:51.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:50 smithi129 ceph-mon[33173]: from='client.24509 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:52:51.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:50 smithi129 ceph-mon[33173]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:51.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:50 smithi039 ceph-mon[29586]: from='client.24509 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:52:51.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:50 smithi039 ceph-mon[29586]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:51.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:50 smithi039 ceph-mon[40962]: from='client.24509 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:52:51.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:50 smithi039 ceph-mon[40962]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:51.497 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":42,"stamp":"2024-09-14T03:52:48.749568+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":181,"num_write_kb":2418,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131,"ondisk_log_size":131,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":223216,"kb_used_data":7056,"kb_used_omap":12,"kb_used_meta":216051,"kb_avail":749738000,"statfs":{"total":767960285184,"available":767731712000,"internally_reserved":0,"allocated":7225344,"data_stored":4057949,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12743,"internal_metadata":221236793},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001990"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:51:58.855420+0000","last_change":"2024-09-14T03:51:52.317818+0000","last_active":"2024-09-14T03:51:58.855420+0000","last_peered":"2024-09-14T03:51:58.855420+0000","last_clean":"2024-09-14T03:51:58.855420+0000","last_became_active":"2024-09-14T03:51:46.806805+0000","last_became_peered":"2024-09-14T03:51:46.806805+0000","last_unstale":"2024-09-14T03:51:58.855420+0000","last_undegraded":"2024-09-14T03:51:58.855420+0000","last_fullsized":"2024-09-14T03:51:58.855420+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:01:38.862572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:51:58.675667+0000","last_change":"2024-09-14T03:51:52.317867+0000","last_active":"2024-09-14T03:51:58.675667+0000","last_peered":"2024-09-14T03:51:58.675667+0000","last_clean":"2024-09-14T03:51:58.675667+0000","last_became_active":"2024-09-14T03:51:46.834009+0000","last_became_peered":"2024-09-14T03:51:46.834009+0000","last_unstale":"2024-09-14T03:51:58.675667+0000","last_undegraded":"2024-09-14T03:51:58.675667+0000","last_fullsized":"2024-09-14T03:51:58.675667+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:36:33.505359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028771999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:51:58.776691+0000","last_change":"2024-09-14T03:51:52.734322+0000","last_active":"2024-09-14T03:51:58.776691+0000","last_peered":"2024-09-14T03:51:58.776691+0000","last_clean":"2024-09-14T03:51:58.776691+0000","last_became_active":"2024-09-14T03:51:46.835000+0000","last_became_peered":"2024-09-14T03:51:46.835000+0000","last_unstale":"2024-09-14T03:51:58.776691+0000","last_undegraded":"2024-09-14T03:51:58.776691+0000","last_fullsized":"2024-09-14T03:51:58.776691+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:46:41.873402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:51:58.675661+0000","last_change":"2024-09-14T03:51:52.317915+0000","last_active":"2024-09-14T03:51:58.675661+0000","last_peered":"2024-09-14T03:51:58.675661+0000","last_clean":"2024-09-14T03:51:58.675661+0000","last_became_active":"2024-09-14T03:51:46.834577+0000","last_became_peered":"2024-09-14T03:51:46.834577+0000","last_unstale":"2024-09-14T03:51:58.675661+0000","last_undegraded":"2024-09-14T03:51:58.675661+0000","last_fullsized":"2024-09-14T03:51:58.675661+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:14:59.991639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:51:58.855553+0000","last_change":"2024-09-14T03:51:52.316945+0000","last_active":"2024-09-14T03:51:58.855553+0000","last_peered":"2024-09-14T03:51:58.855553+0000","last_clean":"2024-09-14T03:51:58.855553+0000","last_became_active":"2024-09-14T03:51:46.806040+0000","last_became_peered":"2024-09-14T03:51:46.806040+0000","last_unstale":"2024-09-14T03:51:58.855553+0000","last_undegraded":"2024-09-14T03:51:58.855553+0000","last_fullsized":"2024-09-14T03:51:58.855553+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:28:20.339010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:51:58.855078+0000","last_change":"2024-09-14T03:51:52.859768+0000","last_active":"2024-09-14T03:51:58.855078+0000","last_peered":"2024-09-14T03:51:58.855078+0000","last_clean":"2024-09-14T03:51:58.855078+0000","last_became_active":"2024-09-14T03:51:46.835278+0000","last_became_peered":"2024-09-14T03:51:46.835278+0000","last_unstale":"2024-09-14T03:51:58.855078+0000","last_undegraded":"2024-09-14T03:51:58.855078+0000","last_fullsized":"2024-09-14T03:51:58.855078+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:25.102148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:51:58.776685+0000","last_change":"2024-09-14T03:51:52.734251+0000","last_active":"2024-09-14T03:51:58.776685+0000","last_peered":"2024-09-14T03:51:58.776685+0000","last_clean":"2024-09-14T03:51:58.776685+0000","last_became_active":"2024-09-14T03:51:46.834389+0000","last_became_peered":"2024-09-14T03:51:46.834389+0000","last_unstale":"2024-09-14T03:51:58.776685+0000","last_undegraded":"2024-09-14T03:51:58.776685+0000","last_fullsized":"2024-09-14T03:51:58.776685+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:12:21.741822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:51:58.855318+0000","last_change":"2024-09-14T03:51:52.316449+0000","last_active":"2024-09-14T03:51:58.855318+0000","last_peered":"2024-09-14T03:51:58.855318+0000","last_clean":"2024-09-14T03:51:58.855318+0000","last_became_active":"2024-09-14T03:51:46.806879+0000","last_became_peered":"2024-09-14T03:51:46.806879+0000","last_unstale":"2024-09-14T03:51:58.855318+0000","last_undegraded":"2024-09-14T03:51:58.855318+0000","last_fullsized":"2024-09-14T03:51:58.855318+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:37:54.991090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"61'128","reported_seq":262,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-14T03:51:52.734021+0000","last_change":"2024-09-14T03:51:16.669763+0000","last_active":"2024-09-14T03:51:52.734021+0000","last_peered":"2024-09-14T03:51:52.734021+0000","last_clean":"2024-09-14T03:51:52.734021+0000","last_became_active":"2024-09-14T03:51:16.669377+0000","last_became_peered":"2024-09-14T03:51:16.669377+0000","last_unstale":"2024-09-14T03:51:52.734021+0000","last_undegraded":"2024-09-14T03:51:52.734021+0000","last_fullsized":"2024-09-14T03:51:52.734021+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T03:49:18.237706+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:10:34.161809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201301,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28264,"kb_used_data":1244,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716888,"statfs":{"total":95995035648,"available":95966093312,"internally_reserved":0,"allocated":1273856,"data_stored":876292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:52:17 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.71699999999999997}]},{"osd":1,"last update":"Sat Sep 14 03:52:17 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.68899999999999995}]},{"osd":2,"last update":"Sat Sep 14 03:52:17 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.63}]},{"osd":3,"last update":"Sat Sep 14 03:52:17 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.59999999999999998}]},{"osd":4,"last update":"Sat Sep 14 03:52:17 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.58699999999999997}]},{"osd":5,"last update":"Sat Sep 14 03:52:17 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.64000000000000001}]},{"osd":6,"last update":"Sat Sep 14 03:52:17 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.53000000000000003}]}]},{"osd":6,"up_from":47,"seq":201863462937,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28268,"kb_used_data":1248,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716884,"statfs":{"total":95995035648,"available":95966089216,"internally_reserved":0,"allocated":1277952,"data_stored":876311,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:51:51 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.497},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.58099999999999996}]},{"osd":1,"last update":"Sat Sep 14 03:51:51 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.45900000000000002}]},{"osd":2,"last update":"Sat Sep 14 03:51:51 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.52700000000000002}]},{"osd":3,"last update":"Sat Sep 14 03:51:51 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.51300000000000001}]},{"osd":4,"last update":"Sat Sep 14 03:51:51 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.379,"15min":0.379},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.40799999999999997}]},{"osd":5,"last update":"Sat Sep 14 03:51:51 2024","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.63200000000000001}]},{"osd":7,"last update":"Sat Sep 14 03:52:17 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.61399999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724574,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27684,"kb_used_data":664,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717468,"statfs":{"total":95995035648,"available":95966687232,"internally_reserved":0,"allocated":679936,"data_stored":285612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:52:29 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.55900000000000005,"15min":0.56299999999999994},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.66300000000000003,"5min":0.75,"15min":0.75},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.36399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.73599999999999999,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.65700000000000003}]},{"osd":1,"last update":"Sat Sep 14 03:52:29 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57499999999999996,"15min":0.57599999999999996},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.69799999999999995,"5min":0.874,"15min":0.874},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.56000000000000005,"15min":0.56299999999999994},"min":{"1min":0.41699999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.77200000000000002,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.51700000000000002}]},{"osd":2,"last update":"Sat Sep 14 03:52:29 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.55300000000000005,"15min":0.55100000000000005},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.67300000000000004,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.55000000000000004,"15min":0.55400000000000005},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.63300000000000001,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.61299999999999999}]},{"osd":3,"last update":"Sat Sep 14 03:52:29 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.55400000000000005,"15min":0.55600000000000005},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.65600000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.57199999999999995,"15min":0.57299999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.70799999999999996,"5min":0.75,"15min":0.75},"last":0.63500000000000001}]},{"osd":4,"last update":"Sat Sep 14 03:52:29 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.38,"15min":0.36899999999999999},"min":{"1min":0.315,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.497,"5min":0.38500000000000001,"15min":0.36699999999999999},"min":{"1min":0.371,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.42799999999999999}]},{"osd":6,"last update":"Sat Sep 14 03:51:51 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"last":0.40000000000000002}]},{"osd":7,"last update":"Sat Sep 14 03:52:15 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.439,"15min":0.439},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"last":0.497},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.53200000000000003}]}]},{"osd":4,"up_from":31,"seq":133143986212,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27684,"kb_used_data":664,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717468,"statfs":{"total":95995035648,"available":95966687232,"internally_reserved":0,"allocated":679936,"data_stored":285924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:52:07 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.57699999999999996,"15min":0.58599999999999997},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.70199999999999996,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.57199999999999995,"15min":0.57399999999999995},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.75700000000000001,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.61799999999999999}]},{"osd":1,"last update":"Sat Sep 14 03:52:07 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.58699999999999997,"15min":0.59299999999999997},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.747,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.54700000000000004,"15min":0.55700000000000005},"min":{"1min":0.34399999999999997,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.60999999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.55900000000000005}]},{"osd":2,"last update":"Sat Sep 14 03:52:07 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.58199999999999996,"15min":0.58899999999999997},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.67800000000000005,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.56599999999999995,"15min":0.57599999999999996},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.65700000000000003,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.50600000000000001}]},{"osd":3,"last update":"Sat Sep 14 03:52:07 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.58999999999999997,"15min":0.59999999999999998},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.70899999999999996,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.57999999999999996,"15min":0.58299999999999996},"min":{"1min":0.41799999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.72599999999999998,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.51800000000000002}]},{"osd":5,"last update":"Sat Sep 14 03:52:32 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.41299999999999998,"15min":0.40200000000000002},"min":{"1min":0.28699999999999998,"5min":0.246,"15min":0.246},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.497,"5min":0.442,"15min":0.433},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.65000000000000002,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.44900000000000001}]},{"osd":6,"last update":"Sat Sep 14 03:51:52 2024","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.624,"5min":0.624,"15min":0.624},"last":0.376}]},{"osd":7,"last update":"Sat Sep 14 03:52:16 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.42899999999999999}]}]},{"osd":3,"up_from":26,"seq":111669149736,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27684,"kb_used_data":664,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717468,"statfs":{"total":95995035648,"available":95966687232,"internally_reserved":0,"allocated":679936,"data_stored":285924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:52:44 2024","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.39300000000000002,"15min":0.40100000000000002},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.56699999999999995,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.41099999999999998,"15min":0.41599999999999998},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.70599999999999996,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.47599999999999998}]},{"osd":1,"last update":"Sat Sep 14 03:52:44 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.42599999999999999,"15min":0.434},"min":{"1min":0.28499999999999998,"5min":0.255,"15min":0.255},"max":{"1min":0.58299999999999996,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.42399999999999999,"15min":0.434},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.45100000000000001}]},{"osd":2,"last update":"Sat Sep 14 03:52:44 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.46000000000000002,"15min":0.46600000000000003},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.44500000000000001,"15min":0.45300000000000001},"min":{"1min":0.252,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.502}]},{"osd":4,"last update":"Sat Sep 14 03:52:04 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.55400000000000005,"15min":0.55000000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.86899999999999999},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.55800000000000005,"15min":0.55400000000000005},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.70999999999999996}]},{"osd":5,"last update":"Sat Sep 14 03:52:30 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.58499999999999996,"15min":0.57999999999999996},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.81699999999999995,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.56499999999999995,"15min":0.56000000000000005},"min":{"1min":0.433,"5min":0.377,"15min":0.377},"max":{"1min":0.78600000000000003,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.79300000000000004}]},{"osd":6,"last update":"Sat Sep 14 03:51:51 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.88400000000000001},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.75600000000000001}]},{"osd":7,"last update":"Sat Sep 14 03:52:18 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":1.016,"5min":1.016,"15min":1.016},"last":0.77700000000000002}]}]},{"osd":2,"up_from":18,"seq":77309411373,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27684,"kb_used_data":664,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717468,"statfs":{"total":95995035648,"available":95966687232,"internally_reserved":0,"allocated":679936,"data_stored":285924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:52:22 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.38500000000000001,"15min":0.374},"min":{"1min":0.30399999999999999,"5min":0.224,"15min":0.224},"max":{"1min":0.51500000000000001,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.372,"5min":0.39400000000000002,"15min":0.39300000000000002},"min":{"1min":0.25600000000000001,"5min":0.22,"15min":0.22},"max":{"1min":0.46800000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.32600000000000001}]},{"osd":1,"last update":"Sat Sep 14 03:52:22 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.39900000000000002,"15min":0.38700000000000001},"min":{"1min":0.25900000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.66200000000000003,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.29899999999999999},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.41799999999999998,"15min":0.40000000000000002},"min":{"1min":0.308,"5min":0.246,"15min":0.246},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.438}]},{"osd":3,"last update":"Sat Sep 14 03:51:43 2024","interfaces":[{"interface":"back","average":{"1min":0.46500000000000002,"5min":0.45700000000000002,"15min":0.45500000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.61699999999999999,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.498,"5min":0.48899999999999999,"15min":0.48699999999999999},"min":{"1min":0.33700000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.34599999999999997}]},{"osd":4,"last update":"Sat Sep 14 03:52:06 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.57699999999999996,"15min":0.58199999999999996},"min":{"1min":0.36099999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.61299999999999999,"15min":0.61199999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.46899999999999997}]},{"osd":5,"last update":"Sat Sep 14 03:52:27 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.621,"15min":0.62},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.76100000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.60999999999999999,"15min":0.60899999999999999},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.83099999999999996,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.45700000000000002}]},{"osd":6,"last update":"Sat Sep 14 03:51:52 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.58099999999999996}]},{"osd":7,"last update":"Sat Sep 14 03:52:17 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.56499999999999995}]}]},{"osd":1,"up_from":13,"seq":55834574898,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27684,"kb_used_data":664,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717468,"statfs":{"total":95995035648,"available":95966687232,"internally_reserved":0,"allocated":679936,"data_stored":285670,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:51:58 2024","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.32600000000000001,"15min":0.30199999999999999},"min":{"1min":0.245,"5min":0.187,"15min":0.187},"max":{"1min":0.501,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.34999999999999998,"15min":0.32400000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.624,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.307}]},{"osd":2,"last update":"Sat Sep 14 03:52:19 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.38900000000000001,"15min":0.372},"min":{"1min":0.28000000000000003,"5min":0.247,"15min":0.247},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.39500000000000002,"15min":0.38100000000000001},"min":{"1min":0.26200000000000001,"5min":0.249,"15min":0.249},"max":{"1min":0.66400000000000003,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.35899999999999999}]},{"osd":3,"last update":"Sat Sep 14 03:51:47 2024","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.45100000000000001,"15min":0.44900000000000001},"min":{"1min":0.33700000000000002,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.34100000000000003},{"interface":"front","average":{"1min":0.48399999999999999,"5min":0.46200000000000002,"15min":0.45800000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.42899999999999999}]},{"osd":4,"last update":"Sat Sep 14 03:52:08 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.54100000000000004,"15min":0.54700000000000004},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.72799999999999998,"5min":0.749,"15min":0.749},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.57999999999999996,"15min":0.58399999999999996},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.76500000000000001,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.47599999999999998}]},{"osd":5,"last update":"Sat Sep 14 03:52:30 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55600000000000005,"15min":0.55700000000000005},"min":{"1min":0.38500000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.60099999999999998,"15min":0.60399999999999998},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.83899999999999997,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":0.44900000000000001}]},{"osd":6,"last update":"Sat Sep 14 03:51:52 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.0169999999999999,"5min":1.0169999999999999,"15min":1.0169999999999999},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.441}]},{"osd":7,"last update":"Sat Sep 14 03:52:14 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.54700000000000004}]}]},{"osd":0,"up_from":8,"seq":34359738422,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28264,"kb_used_data":1244,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716888,"statfs":{"total":95995035648,"available":95966093312,"internally_reserved":0,"allocated":1273856,"data_stored":876292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 14 03:51:57 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.33400000000000002,"15min":0.32200000000000001},"min":{"1min":0.223,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"last":0.23200000000000001},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.35899999999999999,"15min":0.32900000000000001},"min":{"1min":0.28100000000000003,"5min":0.16800000000000001,"15min":0.16800000000000001},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.29399999999999998}]},{"osd":2,"last update":"Sat Sep 14 03:52:22 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.40899999999999997,"15min":0.40100000000000002},"min":{"1min":0.255,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.52400000000000002,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.34599999999999997,"5min":0.375,"15min":0.371},"min":{"1min":0.253,"5min":0.214,"15min":0.214},"max":{"1min":0.56599999999999995,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.27500000000000002}]},{"osd":3,"last update":"Sat Sep 14 03:51:46 2024","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.44800000000000001,"15min":0.44500000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.315,"5min":0.313,"15min":0.313},"max":{"1min":0.65900000000000003,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.33000000000000002}]},{"osd":4,"last update":"Sat Sep 14 03:52:05 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.67100000000000004,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.52700000000000002,"15min":0.52400000000000002},"min":{"1min":0.39600000000000002,"5min":0.38,"15min":0.38},"max":{"1min":0.753,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.41399999999999998}]},{"osd":5,"last update":"Sat Sep 14 03:52:31 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.60699999999999998,"15min":0.60899999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.85499999999999998,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.59999999999999998,"15min":0.60299999999999998},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.40799999999999997}]},{"osd":6,"last update":"Sat Sep 14 03:51:52 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.45400000000000001}]},{"osd":7,"last update":"Sat Sep 14 03:52:17 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.503},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.98599999999999999,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.497}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T03:52:51.499 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T03:52:52.042 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:52:53.244 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:52 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:53.244 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:52 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:53.244 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:52 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:52:53.244 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:52 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:52:53.244 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:52 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:53.244 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:52 smithi039 ceph-mon[29586]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-14T03:52:53.244 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:52 smithi039 ceph-mon[29586]: Reconfiguring daemon alertmanager.a on smithi039 2024-09-14T03:52:53.244 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:52 smithi039 ceph-mon[29586]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:53.245 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:52 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:53.245 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:52 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:53.245 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:52 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:52:53.245 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:52 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:52:53.245 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:52 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:53.245 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:52 smithi039 ceph-mon[40962]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-14T03:52:53.245 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:52 smithi039 ceph-mon[40962]: Reconfiguring daemon alertmanager.a on smithi039 2024-09-14T03:52:53.245 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:52 smithi039 ceph-mon[40962]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:52 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:52 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:52 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:52:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:52 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:52:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:52 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:52:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:52 smithi129 ceph-mon[33173]: Reconfiguring alertmanager.a (dependencies changed)... 2024-09-14T03:52:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:52 smithi129 ceph-mon[33173]: Reconfiguring daemon alertmanager.a on smithi039 2024-09-14T03:52:53.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:52 smithi129 ceph-mon[33173]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:54.777 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:52:54.778 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T03:52:55.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:54 smithi039 ceph-mon[29586]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:55.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:54 smithi039 ceph-mon[29586]: from='client.24515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:52:55.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:54 smithi039 ceph-mon[40962]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:55.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:54 smithi039 ceph-mon[40962]: from='client.24515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:52:55.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:54 smithi129 ceph-mon[33173]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:55.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:54 smithi129 ceph-mon[33173]: from='client.24515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:52:56.712 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":45,"stamp":"2024-09-14T03:52:54.750499+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":181,"num_write_kb":2418,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131,"ondisk_log_size":131,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":223216,"kb_used_data":7056,"kb_used_omap":12,"kb_used_meta":216051,"kb_avail":749738000,"statfs":{"total":767960285184,"available":767731712000,"internally_reserved":0,"allocated":7225344,"data_stored":4057949,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12743,"internal_metadata":221236793},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001832"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:51:58.855420+0000","last_change":"2024-09-14T03:51:52.317818+0000","last_active":"2024-09-14T03:51:58.855420+0000","last_peered":"2024-09-14T03:51:58.855420+0000","last_clean":"2024-09-14T03:51:58.855420+0000","last_became_active":"2024-09-14T03:51:46.806805+0000","last_became_peered":"2024-09-14T03:51:46.806805+0000","last_unstale":"2024-09-14T03:51:58.855420+0000","last_undegraded":"2024-09-14T03:51:58.855420+0000","last_fullsized":"2024-09-14T03:51:58.855420+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:01:38.862572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:51:58.675667+0000","last_change":"2024-09-14T03:51:52.317867+0000","last_active":"2024-09-14T03:51:58.675667+0000","last_peered":"2024-09-14T03:51:58.675667+0000","last_clean":"2024-09-14T03:51:58.675667+0000","last_became_active":"2024-09-14T03:51:46.834009+0000","last_became_peered":"2024-09-14T03:51:46.834009+0000","last_unstale":"2024-09-14T03:51:58.675667+0000","last_undegraded":"2024-09-14T03:51:58.675667+0000","last_fullsized":"2024-09-14T03:51:58.675667+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:36:33.505359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028771999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:51:58.776691+0000","last_change":"2024-09-14T03:51:52.734322+0000","last_active":"2024-09-14T03:51:58.776691+0000","last_peered":"2024-09-14T03:51:58.776691+0000","last_clean":"2024-09-14T03:51:58.776691+0000","last_became_active":"2024-09-14T03:51:46.835000+0000","last_became_peered":"2024-09-14T03:51:46.835000+0000","last_unstale":"2024-09-14T03:51:58.776691+0000","last_undegraded":"2024-09-14T03:51:58.776691+0000","last_fullsized":"2024-09-14T03:51:58.776691+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:46:41.873402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:51:58.675661+0000","last_change":"2024-09-14T03:51:52.317915+0000","last_active":"2024-09-14T03:51:58.675661+0000","last_peered":"2024-09-14T03:51:58.675661+0000","last_clean":"2024-09-14T03:51:58.675661+0000","last_became_active":"2024-09-14T03:51:46.834577+0000","last_became_peered":"2024-09-14T03:51:46.834577+0000","last_unstale":"2024-09-14T03:51:58.675661+0000","last_undegraded":"2024-09-14T03:51:58.675661+0000","last_fullsized":"2024-09-14T03:51:58.675661+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:14:59.991639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":21,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:51:58.855553+0000","last_change":"2024-09-14T03:51:52.316945+0000","last_active":"2024-09-14T03:51:58.855553+0000","last_peered":"2024-09-14T03:51:58.855553+0000","last_clean":"2024-09-14T03:51:58.855553+0000","last_became_active":"2024-09-14T03:51:46.806040+0000","last_became_peered":"2024-09-14T03:51:46.806040+0000","last_unstale":"2024-09-14T03:51:58.855553+0000","last_undegraded":"2024-09-14T03:51:58.855553+0000","last_fullsized":"2024-09-14T03:51:58.855553+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:28:20.339010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:51:58.855078+0000","last_change":"2024-09-14T03:51:52.859768+0000","last_active":"2024-09-14T03:51:58.855078+0000","last_peered":"2024-09-14T03:51:58.855078+0000","last_clean":"2024-09-14T03:51:58.855078+0000","last_became_active":"2024-09-14T03:51:46.835278+0000","last_became_peered":"2024-09-14T03:51:46.835278+0000","last_unstale":"2024-09-14T03:51:58.855078+0000","last_undegraded":"2024-09-14T03:51:58.855078+0000","last_fullsized":"2024-09-14T03:51:58.855078+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:25.102148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:51:58.776685+0000","last_change":"2024-09-14T03:51:52.734251+0000","last_active":"2024-09-14T03:51:58.776685+0000","last_peered":"2024-09-14T03:51:58.776685+0000","last_clean":"2024-09-14T03:51:58.776685+0000","last_became_active":"2024-09-14T03:51:46.834389+0000","last_became_peered":"2024-09-14T03:51:46.834389+0000","last_unstale":"2024-09-14T03:51:58.776685+0000","last_undegraded":"2024-09-14T03:51:58.776685+0000","last_fullsized":"2024-09-14T03:51:58.776685+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:12:21.741822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":20,"reported_epoch":67,"state":"active+clean","last_fresh":"2024-09-14T03:51:58.855318+0000","last_change":"2024-09-14T03:51:52.316449+0000","last_active":"2024-09-14T03:51:58.855318+0000","last_peered":"2024-09-14T03:51:58.855318+0000","last_clean":"2024-09-14T03:51:58.855318+0000","last_became_active":"2024-09-14T03:51:46.806879+0000","last_became_peered":"2024-09-14T03:51:46.806879+0000","last_unstale":"2024-09-14T03:51:58.855318+0000","last_undegraded":"2024-09-14T03:51:58.855318+0000","last_fullsized":"2024-09-14T03:51:58.855318+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:37:54.991090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"61'128","reported_seq":262,"reported_epoch":66,"state":"active+clean","last_fresh":"2024-09-14T03:51:52.734021+0000","last_change":"2024-09-14T03:51:16.669763+0000","last_active":"2024-09-14T03:51:52.734021+0000","last_peered":"2024-09-14T03:51:52.734021+0000","last_clean":"2024-09-14T03:51:52.734021+0000","last_became_active":"2024-09-14T03:51:16.669377+0000","last_became_peered":"2024-09-14T03:51:16.669377+0000","last_unstale":"2024-09-14T03:51:52.734021+0000","last_undegraded":"2024-09-14T03:51:52.734021+0000","last_fullsized":"2024-09-14T03:51:52.734021+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T03:49:18.237706+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:10:34.161809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201302,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28264,"kb_used_data":1244,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716888,"statfs":{"total":95995035648,"available":95966093312,"internally_reserved":0,"allocated":1273856,"data_stored":876292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:52:17 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.67600000000000005}]},{"osd":1,"last update":"Sat Sep 14 03:52:17 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.376,"5min":0.376,"15min":0.376},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.64700000000000002}]},{"osd":2,"last update":"Sat Sep 14 03:52:17 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.57199999999999995,"15min":0.57199999999999995},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.55900000000000005}]},{"osd":3,"last update":"Sat Sep 14 03:52:17 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.71399999999999997}]},{"osd":4,"last update":"Sat Sep 14 03:52:17 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.44800000000000001}]},{"osd":5,"last update":"Sat Sep 14 03:52:17 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.53900000000000003,"5min":0.53900000000000003,"15min":0.53900000000000003},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.84099999999999997,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.47299999999999998}]},{"osd":6,"last update":"Sat Sep 14 03:52:17 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.505},{"interface":"front","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.49399999999999999}]}]},{"osd":6,"up_from":47,"seq":201863462938,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28268,"kb_used_data":1248,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716884,"statfs":{"total":95995035648,"available":95966089216,"internally_reserved":0,"allocated":1277952,"data_stored":876311,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:51:51 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.86799999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.40400000000000003}]},{"osd":1,"last update":"Sat Sep 14 03:51:51 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.78500000000000003,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.5}]},{"osd":2,"last update":"Sat Sep 14 03:51:51 2024","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":0.51500000000000001,"15min":0.51500000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.39300000000000002}]},{"osd":3,"last update":"Sat Sep 14 03:51:51 2024","interfaces":[{"interface":"back","average":{"1min":0.50600000000000001,"5min":0.50600000000000001,"15min":0.50600000000000001},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.46500000000000002}]},{"osd":4,"last update":"Sat Sep 14 03:51:51 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.379,"15min":0.379},"min":{"1min":0.25,"5min":0.25,"15min":0.25},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.44600000000000001}]},{"osd":5,"last update":"Sat Sep 14 03:51:51 2024","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.57799999999999996,"5min":0.57799999999999996,"15min":0.57799999999999996},"last":0.56399999999999995}]},{"osd":7,"last update":"Sat Sep 14 03:52:17 2024","interfaces":[{"interface":"back","average":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.80800000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.5,"5min":0.5,"15min":0.5},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.54500000000000004}]}]},{"osd":5,"up_from":39,"seq":167503724575,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27684,"kb_used_data":664,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717468,"statfs":{"total":95995035648,"available":95966687232,"internally_reserved":0,"allocated":679936,"data_stored":285612,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:52:29 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.55900000000000005,"15min":0.56299999999999994},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.66300000000000003,"5min":0.75,"15min":0.75},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.54500000000000004,"15min":0.54500000000000004},"min":{"1min":0.36399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.73599999999999999,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.51500000000000001}]},{"osd":1,"last update":"Sat Sep 14 03:52:29 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57499999999999996,"15min":0.57599999999999996},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.69799999999999995,"5min":0.874,"15min":0.874},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.54400000000000004,"5min":0.56000000000000005,"15min":0.56299999999999994},"min":{"1min":0.41699999999999998,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.77200000000000002,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.48599999999999999}]},{"osd":2,"last update":"Sat Sep 14 03:52:29 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.55300000000000005,"15min":0.55100000000000005},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.67300000000000004,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.55000000000000004,"15min":0.55400000000000005},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.63300000000000001,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.60999999999999999}]},{"osd":3,"last update":"Sat Sep 14 03:52:29 2024","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.55400000000000005,"15min":0.55600000000000005},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.65600000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.57199999999999995,"15min":0.57299999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.70799999999999996,"5min":0.75,"15min":0.75},"last":0.65700000000000003}]},{"osd":4,"last update":"Sat Sep 14 03:52:29 2024","interfaces":[{"interface":"back","average":{"1min":0.44800000000000001,"5min":0.38,"15min":0.36899999999999999},"min":{"1min":0.315,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.497,"5min":0.38500000000000001,"15min":0.36699999999999999},"min":{"1min":0.371,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.39000000000000001}]},{"osd":6,"last update":"Sat Sep 14 03:51:51 2024","interfaces":[{"interface":"back","average":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"last":0.54500000000000004}]},{"osd":7,"last update":"Sat Sep 14 03:52:15 2024","interfaces":[{"interface":"back","average":{"1min":0.439,"5min":0.439,"15min":0.439},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.63700000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986213,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27684,"kb_used_data":664,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717468,"statfs":{"total":95995035648,"available":95966687232,"internally_reserved":0,"allocated":679936,"data_stored":285924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:52:07 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.57699999999999996,"15min":0.58599999999999997},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.70199999999999996,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.57199999999999995,"15min":0.57399999999999995},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.75700000000000001,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.58899999999999997}]},{"osd":1,"last update":"Sat Sep 14 03:52:07 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.58699999999999997,"15min":0.59299999999999997},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.747,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.54700000000000004,"15min":0.55700000000000005},"min":{"1min":0.34399999999999997,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.60999999999999999,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.52500000000000002}]},{"osd":2,"last update":"Sat Sep 14 03:52:07 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.58199999999999996,"15min":0.58899999999999997},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.67800000000000005,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.48799999999999999},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.56599999999999995,"15min":0.57599999999999996},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.65700000000000003,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.54700000000000004}]},{"osd":3,"last update":"Sat Sep 14 03:52:07 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.58999999999999997,"15min":0.59999999999999998},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.70899999999999996,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.57999999999999996,"15min":0.58299999999999996},"min":{"1min":0.41799999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.72599999999999998,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.58099999999999996}]},{"osd":5,"last update":"Sat Sep 14 03:52:32 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.41299999999999998,"15min":0.40200000000000002},"min":{"1min":0.28699999999999998,"5min":0.246,"15min":0.246},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.497,"5min":0.442,"15min":0.433},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.65000000000000002,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.46400000000000002}]},{"osd":6,"last update":"Sat Sep 14 03:51:52 2024","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.438},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.624,"5min":0.624,"15min":0.624},"last":0.41899999999999998}]},{"osd":7,"last update":"Sat Sep 14 03:52:16 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.501}]}]},{"osd":3,"up_from":26,"seq":111669149737,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27684,"kb_used_data":664,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717468,"statfs":{"total":95995035648,"available":95966687232,"internally_reserved":0,"allocated":679936,"data_stored":285924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:52:44 2024","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.39300000000000002,"15min":0.40100000000000002},"min":{"1min":0.191,"5min":0.191,"15min":0.191},"max":{"1min":0.56699999999999995,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.20000000000000001},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.41099999999999998,"15min":0.41599999999999998},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":0.70599999999999996,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.24099999999999999}]},{"osd":1,"last update":"Sat Sep 14 03:52:44 2024","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.42599999999999999,"15min":0.434},"min":{"1min":0.28499999999999998,"5min":0.255,"15min":0.255},"max":{"1min":0.58299999999999996,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.42399999999999999,"15min":0.434},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"last":0.26100000000000001}]},{"osd":2,"last update":"Sat Sep 14 03:52:44 2024","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.46000000000000002,"15min":0.46600000000000003},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.308},{"interface":"front","average":{"1min":0.42699999999999999,"5min":0.44500000000000001,"15min":0.45300000000000001},"min":{"1min":0.252,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.31900000000000001}]},{"osd":4,"last update":"Sat Sep 14 03:52:04 2024","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.55400000000000005,"15min":0.55000000000000004},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.499},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.55800000000000005,"15min":0.55400000000000005},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.60099999999999998}]},{"osd":5,"last update":"Sat Sep 14 03:52:30 2024","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.58499999999999996,"15min":0.57999999999999996},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.81699999999999995,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.56499999999999995,"15min":0.56000000000000005},"min":{"1min":0.433,"5min":0.377,"15min":0.377},"max":{"1min":0.78600000000000003,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.57299999999999995}]},{"osd":6,"last update":"Sat Sep 14 03:51:51 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59799999999999998,"15min":0.59799999999999998},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.90900000000000003,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.64700000000000002}]},{"osd":7,"last update":"Sat Sep 14 03:52:18 2024","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.96699999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.54000000000000004},{"interface":"front","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":1.016,"5min":1.016,"15min":1.016},"last":0.624}]}]},{"osd":2,"up_from":18,"seq":77309411374,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27684,"kb_used_data":664,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717468,"statfs":{"total":95995035648,"available":95966687232,"internally_reserved":0,"allocated":679936,"data_stored":285924,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:52:22 2024","interfaces":[{"interface":"back","average":{"1min":0.40300000000000002,"5min":0.38500000000000001,"15min":0.374},"min":{"1min":0.30399999999999999,"5min":0.224,"15min":0.224},"max":{"1min":0.51500000000000001,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.20899999999999999},{"interface":"front","average":{"1min":0.372,"5min":0.39400000000000002,"15min":0.39300000000000002},"min":{"1min":0.25600000000000001,"5min":0.22,"15min":0.22},"max":{"1min":0.46800000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.23200000000000001}]},{"osd":1,"last update":"Sat Sep 14 03:52:22 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.39900000000000002,"15min":0.38700000000000001},"min":{"1min":0.25900000000000001,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.66200000000000003,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.247},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.41799999999999998,"15min":0.40000000000000002},"min":{"1min":0.308,"5min":0.246,"15min":0.246},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.29899999999999999}]},{"osd":3,"last update":"Sat Sep 14 03:52:46 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.44700000000000001,"15min":0.45200000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.53700000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.315},{"interface":"front","average":{"1min":0.433,"5min":0.47799999999999998,"15min":0.48399999999999999},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.64000000000000001,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.26600000000000001}]},{"osd":4,"last update":"Sat Sep 14 03:52:06 2024","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.57699999999999996,"15min":0.58199999999999996},"min":{"1min":0.36099999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.61299999999999999,"15min":0.61199999999999999},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.624}]},{"osd":5,"last update":"Sat Sep 14 03:52:27 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.621,"15min":0.62},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.76100000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.60999999999999999,"15min":0.60899999999999999},"min":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"max":{"1min":0.83099999999999996,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.72199999999999998}]},{"osd":6,"last update":"Sat Sep 14 03:51:52 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.61399999999999999,"15min":0.61399999999999999},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.58499999999999996}]},{"osd":7,"last update":"Sat Sep 14 03:52:17 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"max":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.77800000000000002}]}]},{"osd":1,"up_from":13,"seq":55834574899,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27684,"kb_used_data":664,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717468,"statfs":{"total":95995035648,"available":95966687232,"internally_reserved":0,"allocated":679936,"data_stored":285670,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:51:58 2024","interfaces":[{"interface":"back","average":{"1min":0.373,"5min":0.32600000000000001,"15min":0.30199999999999999},"min":{"1min":0.245,"5min":0.187,"15min":0.187},"max":{"1min":0.501,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.34999999999999998,"15min":0.32400000000000001},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.624,"5min":0.66300000000000003,"15min":0.66300000000000003},"last":0.38900000000000001}]},{"osd":2,"last update":"Sat Sep 14 03:52:19 2024","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":0.38900000000000001,"15min":0.372},"min":{"1min":0.28000000000000003,"5min":0.247,"15min":0.247},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.41199999999999998,"5min":0.39500000000000002,"15min":0.38100000000000001},"min":{"1min":0.26200000000000001,"5min":0.249,"15min":0.249},"max":{"1min":0.66400000000000003,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.42399999999999999}]},{"osd":3,"last update":"Sat Sep 14 03:52:47 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.443,"15min":0.44600000000000001},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.70499999999999996,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.41999999999999998,"5min":0.45500000000000002,"15min":0.45600000000000002},"min":{"1min":0.33200000000000002,"5min":0.311,"15min":0.311},"max":{"1min":0.71599999999999997,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.442}]},{"osd":4,"last update":"Sat Sep 14 03:52:08 2024","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.54100000000000004,"15min":0.54700000000000004},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.72799999999999998,"5min":0.749,"15min":0.749},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.57999999999999996,"15min":0.58399999999999996},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.76500000000000001,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.63900000000000001}]},{"osd":5,"last update":"Sat Sep 14 03:52:30 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.55600000000000005,"15min":0.55700000000000005},"min":{"1min":0.38500000000000001,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.60099999999999998,"15min":0.60399999999999998},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.83899999999999997,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":0.66900000000000004}]},{"osd":6,"last update":"Sat Sep 14 03:51:52 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.0169999999999999,"5min":1.0169999999999999,"15min":1.0169999999999999},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.629}]},{"osd":7,"last update":"Sat Sep 14 03:52:14 2024","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.60799999999999998}]}]},{"osd":0,"up_from":8,"seq":34359738423,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28264,"kb_used_data":1244,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716888,"statfs":{"total":95995035648,"available":95966093312,"internally_reserved":0,"allocated":1273856,"data_stored":876292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 14 03:51:57 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.33400000000000002,"15min":0.32200000000000001},"min":{"1min":0.223,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.35899999999999999,"15min":0.32900000000000001},"min":{"1min":0.28100000000000003,"5min":0.16800000000000001,"15min":0.16800000000000001},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.35399999999999998}]},{"osd":2,"last update":"Sat Sep 14 03:52:22 2024","interfaces":[{"interface":"back","average":{"1min":0.39900000000000002,"5min":0.40899999999999997,"15min":0.40100000000000002},"min":{"1min":0.255,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.52400000000000002,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.34599999999999997,"5min":0.375,"15min":0.371},"min":{"1min":0.253,"5min":0.214,"15min":0.214},"max":{"1min":0.56599999999999995,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.51400000000000001}]},{"osd":3,"last update":"Sat Sep 14 03:52:47 2024","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.44600000000000001,"15min":0.44400000000000001},"min":{"1min":0.33400000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.622,"5min":0.622,"15min":0.622},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.44500000000000001,"15min":0.45100000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.54400000000000004,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.42899999999999999}]},{"osd":4,"last update":"Sat Sep 14 03:52:05 2024","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.67100000000000004,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.52700000000000002,"15min":0.52400000000000002},"min":{"1min":0.39600000000000002,"5min":0.38,"15min":0.38},"max":{"1min":0.753,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.495}]},{"osd":5,"last update":"Sat Sep 14 03:52:31 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.60699999999999998,"15min":0.60899999999999999},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.85499999999999998,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.59999999999999998,"15min":0.60299999999999998},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.46899999999999997}]},{"osd":6,"last update":"Sat Sep 14 03:51:52 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.48199999999999998}]},{"osd":7,"last update":"Sat Sep 14 03:52:17 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.98599999999999999,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.45000000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T03:52:56.713 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-14T03:52:56.714 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-14T03:52:56.714 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-14T03:52:56.714 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-14T03:52:57.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:56 smithi039 ceph-mon[29586]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:57.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:56 smithi039 ceph-mon[40962]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:57.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:56 smithi129 ceph-mon[33173]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:57.372 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:52:57.496 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:52:57 smithi039 systemd[1]: Stopping Ceph alertmanager.a for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:52:58.540 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:52:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[71896]: ts=2024-09-14T03:52:58.221Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-14T03:52:58.540 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:52:58 smithi039 podman[77552]: 2024-09-14 03:52:58.242143044 +0000 UTC m=+0.133813369 container died 72ce7832d5c6ac82e19597ed3764db950316972c326cd230753bf52610de9208 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:52:58.842 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:52:58 smithi039 podman[77552]: 2024-09-14 03:52:58.652716469 +0000 UTC m=+0.544386796 container cleanup 72ce7832d5c6ac82e19597ed3764db950316972c326cd230753bf52610de9208 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:52:58.843 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:52:58 smithi039 bash[77552]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a 2024-09-14T03:52:59.184 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:58 smithi039 ceph-mon[29586]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:59.184 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:52:59 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:52:59] "GET /metrics HTTP/1.1" 200 28506 "" "Prometheus/2.43.0" 2024-09-14T03:52:59.185 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:58 smithi039 ceph-mon[40962]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:52:59.185 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:52:58 smithi039 podman[77570]: 2024-09-14 03:52:58.879488362 +0000 UTC m=+0.638967399 container remove 72ce7832d5c6ac82e19597ed3764db950316972c326cd230753bf52610de9208 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:52:59.185 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:52:58 smithi039 podman[77570]: 2024-09-14 03:52:58.971960869 +0000 UTC m=+0.731439907 volume remove bf2280854ddec5c2baa9f8551538c71ad81cf995e4599fa1ebf33b8980d316b1 2024-09-14T03:52:59.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:58 smithi129 ceph-mon[33173]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:00.193 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:52:59 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:53:00.194 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:52:59 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:53:00.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:52:59 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:53:00.280 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:53:00.281 INFO:teuthology.orchestra.run.smithi039.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-14T03:53:00.541 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:53:00 smithi039 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-14T03:53:00.541 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:53:00 smithi039 systemd[1]: Stopped Ceph alertmanager.a for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:53:00.541 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:53:00 smithi039 systemd[1]: Starting Ceph alertmanager.a for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:53:01.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:00 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/1543662709' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-14T03:53:01.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:00 smithi039 ceph-mon[29586]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:01.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:00 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/1543662709' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-14T03:53:01.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:00 smithi039 ceph-mon[40962]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:01.041 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:53:00 smithi039 podman[77752]: 2024-09-14 03:53:00.553702821 +0000 UTC m=+0.018594047 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-09-14T03:53:01.104 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-14T03:53:01.104 INFO:tasks.cephadm:Setup complete, yielding 2024-09-14T03:53:01.104 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-09-14T03:53:01.117 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi039.front.sepia.ceph.com 2024-09-14T03:53:01.117 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- bash -c 'ceph fs volume create foo' 2024-09-14T03:53:01.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:00 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/1543662709' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-14T03:53:01.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:00 smithi129 ceph-mon[33173]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:01.490 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:53:01 smithi039 podman[77752]: 2024-09-14 03:53:01.194835496 +0000 UTC m=+0.659726723 volume create 1d453f423d31ff481b784f511b29881e1bdc80d69b79b1df897cf083c2fde9a1 2024-09-14T03:53:01.490 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:53:01 smithi039 podman[77752]: 2024-09-14 03:53:01.286998955 +0000 UTC m=+0.751890180 container create b5ca6b93d9ee8fb2bb28cc7735721d635ebe6e9d9db4be989ff3bffe77963602 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:53:01.745 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:53:01.868 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:53:01 smithi039 podman[77752]: 2024-09-14 03:53:01.589983073 +0000 UTC m=+1.054874299 container init b5ca6b93d9ee8fb2bb28cc7735721d635ebe6e9d9db4be989ff3bffe77963602 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:53:01.868 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:53:01 smithi039 podman[77752]: 2024-09-14 03:53:01.594661239 +0000 UTC m=+1.059552465 container start b5ca6b93d9ee8fb2bb28cc7735721d635ebe6e9d9db4be989ff3bffe77963602 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:53:01.868 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:53:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[77805]: ts=2024-09-14T03:53:01.606Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-09-14T03:53:01.868 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:53:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[77805]: ts=2024-09-14T03:53:01.606Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-09-14T03:53:01.868 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:53:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[77805]: ts=2024-09-14T03:53:01.608Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.39 port=9094 2024-09-14T03:53:01.868 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:53:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[77805]: ts=2024-09-14T03:53:01.609Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-14T03:53:01.868 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:53:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[77805]: ts=2024-09-14T03:53:01.631Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-14T03:53:01.868 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:53:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[77805]: ts=2024-09-14T03:53:01.632Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-14T03:53:01.869 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:53:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[77805]: ts=2024-09-14T03:53:01.634Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-09-14T03:53:01.869 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:53:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[77805]: ts=2024-09-14T03:53:01.634Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-14T03:53:01.869 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:53:01 smithi039 bash[77752]: b5ca6b93d9ee8fb2bb28cc7735721d635ebe6e9d9db4be989ff3bffe77963602 2024-09-14T03:53:01.869 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:53:01 smithi039 systemd[1]: Started Ceph alertmanager.a for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:53:02.880 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:02 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:02.880 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:02 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:02.881 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:02 smithi129 ceph-mon[33173]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:53:03.038 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:02 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:03.038 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:02 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:03.038 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:02 smithi039 ceph-mon[29586]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:53:03.039 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:02 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:03.039 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:02 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:03.039 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:02 smithi039 ceph-mon[40962]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:53:04.040 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:53:03 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[77805]: ts=2024-09-14T03:53:03.609Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000022574s 2024-09-14T03:53:04.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:04 smithi129 ceph-mon[33173]: Reconfiguring daemon prometheus.a on smithi129 2024-09-14T03:53:04.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:04 smithi129 ceph-mon[33173]: pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:04.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:04 smithi039 ceph-mon[40962]: Reconfiguring daemon prometheus.a on smithi129 2024-09-14T03:53:04.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:04 smithi039 ceph-mon[40962]: pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:04.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:04 smithi039 ceph-mon[29586]: Reconfiguring daemon prometheus.a on smithi129 2024-09-14T03:53:04.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:04 smithi039 ceph-mon[29586]: pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:05.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:05 smithi129 ceph-mon[33173]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:05.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:05 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-14T03:53:05.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:05 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-14T03:53:05.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:05 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-09-14T03:53:05.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:05 smithi129 ceph-mon[33173]: osdmap e68: 8 total, 8 up, 8 in 2024-09-14T03:53:05.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:05 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-14T03:53:05.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:05 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-14T03:53:05.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:05 smithi129 ceph-mon[33173]: pgmap v51: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:05.272 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:05 smithi129 systemd[1]: Stopping Ceph prometheus.a for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:53:05.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:05 smithi039 ceph-mon[40962]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:05.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:05 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-14T03:53:05.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:05 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-14T03:53:05.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:05 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-09-14T03:53:05.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:05 smithi039 ceph-mon[40962]: osdmap e68: 8 total, 8 up, 8 in 2024-09-14T03:53:05.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:05 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-14T03:53:05.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:05 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-14T03:53:05.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:05 smithi039 ceph-mon[40962]: pgmap v51: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:05.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:05 smithi039 ceph-mon[29586]: from='client.14622 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:53:05.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:05 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-14T03:53:05.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:05 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-09-14T03:53:05.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:05 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-09-14T03:53:05.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:05 smithi039 ceph-mon[29586]: osdmap e68: 8 total, 8 up, 8 in 2024-09-14T03:53:05.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:05 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-14T03:53:05.542 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:05 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-09-14T03:53:05.542 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:05 smithi039 ceph-mon[29586]: pgmap v51: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:05.542 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:05 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-a[29563]: 2024-09-14T03:53:05.235+0000 7f73108a3640 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T03:53:05.737 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:05 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:53:05.428Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-14T03:53:05.737 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:05 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:53:05.428Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-14T03:53:05.737 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:05 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:53:05.428Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-14T03:53:05.737 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:05 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:53:05.428Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-14T03:53:05.737 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:05 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:53:05.428Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-14T03:53:05.737 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:05 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:53:05.428Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-14T03:53:05.737 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:05 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:53:05.428Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-14T03:53:05.738 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:05 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:53:05.429Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-14T03:53:05.738 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:05 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:53:05.429Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-14T03:53:05.738 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:05 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:53:05.429Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-14T03:53:05.738 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:05 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:53:05.429Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-14T03:53:05.738 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:05 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[62536]: ts=2024-09-14T03:53:05.429Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-14T03:53:05.738 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:05 smithi129 podman[65876]: 2024-09-14 03:53:05.439022889 +0000 UTC m=+0.130693878 container died fa029a6efad8104a9fc48bc5e341c7c30bc0d2ab60ff1ad2abbf4fa9cf8ac00a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:53:06.021 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:05 smithi129 podman[65876]: 2024-09-14 03:53:05.843193623 +0000 UTC m=+0.534864600 container cleanup fa029a6efad8104a9fc48bc5e341c7c30bc0d2ab60ff1ad2abbf4fa9cf8ac00a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:53:06.021 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:05 smithi129 bash[65876]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a 2024-09-14T03:53:06.083 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-09-14T03:53:06.299 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:53:06.421 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:06 smithi039 ceph-mon[29586]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:53:06.421 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:06 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-09-14T03:53:06.424 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:06 smithi039 ceph-mon[29586]: osdmap e69: 8 total, 8 up, 8 in 2024-09-14T03:53:06.424 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:06 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-14T03:53:06.424 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:06 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-14T03:53:06.424 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:06 smithi039 ceph-mon[29586]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T03:53:06.424 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:06 smithi039 ceph-mon[29586]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-14T03:53:06.424 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:06 smithi039 ceph-mon[29586]: osdmap e70: 8 total, 8 up, 8 in 2024-09-14T03:53:06.424 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:06 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-09-14T03:53:06.424 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:06 smithi039 ceph-mon[29586]: fsmap foo:0 2024-09-14T03:53:06.424 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:06 smithi039 ceph-mon[29586]: Saving service mds.foo spec with placement count:2 2024-09-14T03:53:06.424 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:06 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:06.425 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:06 smithi039 ceph-mon[40962]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:53:06.425 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:06 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-09-14T03:53:06.425 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:06 smithi039 ceph-mon[40962]: osdmap e69: 8 total, 8 up, 8 in 2024-09-14T03:53:06.425 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:06 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-14T03:53:06.425 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:06 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-14T03:53:06.425 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:06 smithi039 ceph-mon[40962]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T03:53:06.425 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:06 smithi039 ceph-mon[40962]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-14T03:53:06.425 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:06 smithi039 ceph-mon[40962]: osdmap e70: 8 total, 8 up, 8 in 2024-09-14T03:53:06.425 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:06 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-09-14T03:53:06.425 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:06 smithi039 ceph-mon[40962]: fsmap foo:0 2024-09-14T03:53:06.425 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:06 smithi039 ceph-mon[40962]: Saving service mds.foo spec with placement count:2 2024-09-14T03:53:06.425 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:06 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:06.471 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 podman[65893]: 2024-09-14 03:53:06.02786116 +0000 UTC m=+0.591467522 container remove fa029a6efad8104a9fc48bc5e341c7c30bc0d2ab60ff1ad2abbf4fa9cf8ac00a (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:53:06.471 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-14T03:53:06.471 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 systemd[1]: Stopped Ceph prometheus.a for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:53:06.471 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 1.179s CPU time. 2024-09-14T03:53:06.471 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 systemd[1]: Starting Ceph prometheus.a for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:53:06.471 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-mon[33173]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:53:06.471 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-09-14T03:53:06.471 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-mon[33173]: osdmap e69: 8 total, 8 up, 8 in 2024-09-14T03:53:06.472 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-14T03:53:06.472 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-09-14T03:53:06.472 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-mon[33173]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-09-14T03:53:06.472 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-mon[33173]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-09-14T03:53:06.472 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-mon[33173]: osdmap e70: 8 total, 8 up, 8 in 2024-09-14T03:53:06.472 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-09-14T03:53:06.472 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-mon[33173]: fsmap foo:0 2024-09-14T03:53:06.472 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-mon[33173]: Saving service mds.foo spec with placement count:2 2024-09-14T03:53:06.472 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:06.723 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 podman[65990]: 2024-09-14 03:53:06.373479964 +0000 UTC m=+0.016870171 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-14T03:53:06.724 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 podman[65990]: 2024-09-14 03:53:06.497898681 +0000 UTC m=+0.141288884 container create 0275bbd079914b91fa62c7095ebe6fb834c2bb48b8d99644a98e7ffb38a76fed (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:53:06.724 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 podman[65990]: 2024-09-14 03:53:06.723075726 +0000 UTC m=+0.366465946 container init 0275bbd079914b91fa62c7095ebe6fb834c2bb48b8d99644a98e7ffb38a76fed (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:53:07.021 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 podman[65990]: 2024-09-14 03:53:06.727936857 +0000 UTC m=+0.371327066 container start 0275bbd079914b91fa62c7095ebe6fb834c2bb48b8d99644a98e7ffb38a76fed (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:53:07.022 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:53:06.751Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-14T03:53:07.022 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:53:06.751Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-14T03:53:07.022 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:53:06.751Z caller=main.go:570 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi129 (none))" 2024-09-14T03:53:07.022 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:53:06.751Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-14T03:53:07.022 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:53:06.751Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-14T03:53:07.022 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:53:06.753Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-14T03:53:07.022 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:53:06.753Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-14T03:53:07.022 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:53:06.755Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-14T03:53:07.022 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:53:06.755Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-14T03:53:07.022 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:53:06.756Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-14T03:53:07.022 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:53:06.756Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.006µs 2024-09-14T03:53:07.022 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:53:06.756Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-14T03:53:07.023 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:53:06.758Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-09-14T03:53:07.023 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:53:06.758Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-09-14T03:53:07.023 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:53:06.758Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=27.766µs wal_replay_duration=1.584018ms wbl_replay_duration=108ns total_replay_duration=1.640111ms 2024-09-14T03:53:07.023 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:53:06.759Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-14T03:53:07.023 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:53:06.759Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-14T03:53:07.023 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:53:06.759Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-14T03:53:07.023 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:53:06.779Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=19.936255ms db_storage=776ns remote_storage=1.019µs web_handler=299ns query_engine=598ns scrape=392.477µs scrape_sd=88.539µs notify=15.971µs notify_sd=18.605µs rules=19.057621ms tracing=5.547µs 2024-09-14T03:53:07.023 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:53:06.779Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-14T03:53:07.023 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:53:06.779Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-14T03:53:07.023 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 bash[65990]: 0275bbd079914b91fa62c7095ebe6fb834c2bb48b8d99644a98e7ffb38a76fed 2024-09-14T03:53:07.023 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:53:06 smithi129 systemd[1]: Started Ceph prometheus.a for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:53:07.037 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:53:06 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:53:06] ENGINE Bus STOPPING 2024-09-14T03:53:07.037 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:53:06 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:53:06] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:53:07.037 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:53:06 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:53:06] ENGINE Bus STOPPED 2024-09-14T03:53:07.037 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:53:06 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:53:06] ENGINE Bus STARTING 2024-09-14T03:53:07.038 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:53:07] ENGINE Serving on http://:::9283 2024-09-14T03:53:07.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:53:07] ENGINE Bus STARTED 2024-09-14T03:53:07.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:53:07] ENGINE Bus STOPPING 2024-09-14T03:53:07.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-mon[29586]: osdmap e71: 8 total, 8 up, 8 in 2024-09-14T03:53:07.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-mon[29586]: pgmap v55: 73 pgs: 17 creating+peering, 56 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:07.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:07.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:07.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:53:07.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi039.front.sepia.ceph.com:9093"}]: dispatch 2024-09-14T03:53:07.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:07.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:53:07.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi129.front.sepia.ceph.com:9095"}]: dispatch 2024-09-14T03:53:07.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:07.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:53:07.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi129.front.sepia.ceph.com:3000"}]: dispatch 2024-09-14T03:53:07.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:07.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:53:07.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-mon[40962]: osdmap e71: 8 total, 8 up, 8 in 2024-09-14T03:53:07.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-mon[40962]: pgmap v55: 73 pgs: 17 creating+peering, 56 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:07.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:07.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:07.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:53:07.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi039.front.sepia.ceph.com:9093"}]: dispatch 2024-09-14T03:53:07.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:07.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:53:07.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi129.front.sepia.ceph.com:9095"}]: dispatch 2024-09-14T03:53:07.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:07.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:53:07.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi129.front.sepia.ceph.com:3000"}]: dispatch 2024-09-14T03:53:07.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:07.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:53:07.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:07 smithi129 ceph-mon[33173]: osdmap e71: 8 total, 8 up, 8 in 2024-09-14T03:53:07.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:07 smithi129 ceph-mon[33173]: pgmap v55: 73 pgs: 17 creating+peering, 56 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:07.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:07 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:07.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:07 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:07.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:07 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:53:07.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:07 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi039.front.sepia.ceph.com:9093"}]: dispatch 2024-09-14T03:53:07.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:07 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:07.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:07 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:53:07.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:07 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi129.front.sepia.ceph.com:9095"}]: dispatch 2024-09-14T03:53:07.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:07 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:07.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:07 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:53:07.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:07 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi129.front.sepia.ceph.com:3000"}]: dispatch 2024-09-14T03:53:07.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:07 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:07.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:07 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:53:07.913 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:53:07] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:53:07.914 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:53:07] ENGINE Bus STOPPED 2024-09-14T03:53:07.914 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:53:07] ENGINE Bus STARTING 2024-09-14T03:53:07.914 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:53:07] ENGINE Serving on http://:::9283 2024-09-14T03:53:07.914 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:53:07] ENGINE Bus STARTED 2024-09-14T03:53:07.914 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:53:07 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:53:07] ENGINE Bus STOPPING 2024-09-14T03:53:08.250 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:53:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:53:08] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:53:08.250 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:53:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:53:08] ENGINE Bus STOPPED 2024-09-14T03:53:08.250 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:53:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:53:08] ENGINE Bus STARTING 2024-09-14T03:53:08.250 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:53:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:53:08] ENGINE Serving on http://:::9283 2024-09-14T03:53:08.250 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:53:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:53:08] ENGINE Bus STARTED 2024-09-14T03:53:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:08 smithi129 ceph-mon[33173]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:53:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:08 smithi129 ceph-mon[33173]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi039.front.sepia.ceph.com:9093"}]: dispatch 2024-09-14T03:53:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:08 smithi129 ceph-mon[33173]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:53:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:08 smithi129 ceph-mon[33173]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi129.front.sepia.ceph.com:9095"}]: dispatch 2024-09-14T03:53:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:08 smithi129 ceph-mon[33173]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:53:08.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:08 smithi129 ceph-mon[33173]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi129.front.sepia.ceph.com:3000"}]: dispatch 2024-09-14T03:53:08.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:08 smithi129 ceph-mon[33173]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:53:08.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:08 smithi129 ceph-mon[33173]: osdmap e72: 8 total, 8 up, 8 in 2024-09-14T03:53:08.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:08 smithi039 ceph-mon[29586]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:53:08.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:08 smithi039 ceph-mon[29586]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi039.front.sepia.ceph.com:9093"}]: dispatch 2024-09-14T03:53:08.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:08 smithi039 ceph-mon[29586]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:53:08.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:08 smithi039 ceph-mon[29586]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi129.front.sepia.ceph.com:9095"}]: dispatch 2024-09-14T03:53:08.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:08 smithi039 ceph-mon[29586]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:53:08.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:08 smithi039 ceph-mon[29586]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi129.front.sepia.ceph.com:3000"}]: dispatch 2024-09-14T03:53:08.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:08 smithi039 ceph-mon[29586]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:53:08.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:08 smithi039 ceph-mon[29586]: osdmap e72: 8 total, 8 up, 8 in 2024-09-14T03:53:08.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:08 smithi039 ceph-mon[40962]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:53:08.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:08 smithi039 ceph-mon[40962]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi039.front.sepia.ceph.com:9093"}]: dispatch 2024-09-14T03:53:08.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:08 smithi039 ceph-mon[40962]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:53:08.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:08 smithi039 ceph-mon[40962]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi129.front.sepia.ceph.com:9095"}]: dispatch 2024-09-14T03:53:08.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:08 smithi039 ceph-mon[40962]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:53:08.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:08 smithi039 ceph-mon[40962]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi129.front.sepia.ceph.com:3000"}]: dispatch 2024-09-14T03:53:08.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:08 smithi039 ceph-mon[40962]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:53:08.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:08 smithi039 ceph-mon[40962]: osdmap e72: 8 total, 8 up, 8 in 2024-09-14T03:53:09.377 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:09 smithi039 ceph-mon[29586]: pgmap v57: 73 pgs: 17 creating+peering, 56 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:09.377 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:09 smithi039 ceph-mon[40962]: pgmap v57: 73 pgs: 17 creating+peering, 56 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:09.405 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-09-14T03:53:09.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:09 smithi129 ceph-mon[33173]: pgmap v57: 73 pgs: 17 creating+peering, 56 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:09.629 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:53:11.740 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:11 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:11.741 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:11 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:11.741 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:11 smithi039 ceph-mon[29586]: pgmap v58: 73 pgs: 17 creating+peering, 56 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:11.741 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:11 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:11.741 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:11 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:11.741 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:11 smithi039 ceph-mon[40962]: pgmap v58: 73 pgs: 17 creating+peering, 56 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:11.741 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:53:11 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[77805]: ts=2024-09-14T03:53:11.610Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.001018918s 2024-09-14T03:53:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:11 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:11 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:11 smithi129 ceph-mon[33173]: pgmap v58: 73 pgs: 17 creating+peering, 56 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:12.894 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-09-14T03:53:12.905 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-14T03:53:12.905 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-14T03:53:12.905 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-14T03:53:13.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:12 smithi129 ceph-mon[33173]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:13.289 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:12 smithi039 ceph-mon[29586]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:13.289 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:12 smithi039 ceph-mon[40962]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:13.754 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:53:15.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:14 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:15.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:14 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:15.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:14 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:53:15.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:14 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:53:15.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:14 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:15.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:14 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi039.bbwiwf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-14T03:53:15.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:14 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi039.bbwiwf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-14T03:53:15.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:14 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi039.bbwiwf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-14T03:53:15.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:14 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:53:15.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:14 smithi129 ceph-mon[33173]: Deploying daemon mds.foo.smithi039.bbwiwf on smithi039 2024-09-14T03:53:15.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:14 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:15.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:14 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:15.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:14 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:53:15.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:14 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:53:15.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:14 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:15.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:14 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi039.bbwiwf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-14T03:53:15.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:14 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi039.bbwiwf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-14T03:53:15.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:14 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi039.bbwiwf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-14T03:53:15.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:14 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:53:15.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:14 smithi039 ceph-mon[29586]: Deploying daemon mds.foo.smithi039.bbwiwf on smithi039 2024-09-14T03:53:15.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:14 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:15.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:14 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:15.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:14 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:53:15.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:14 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:53:15.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:14 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:15.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:14 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi039.bbwiwf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-14T03:53:15.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:14 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi039.bbwiwf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-14T03:53:15.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:14 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi039.bbwiwf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-14T03:53:15.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:14 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:53:15.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:14 smithi039 ceph-mon[40962]: Deploying daemon mds.foo.smithi039.bbwiwf on smithi039 2024-09-14T03:53:16.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:15 smithi039 ceph-mon[29586]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:16.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:15 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:16.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:15 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:53:16.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:15 smithi039 ceph-mon[40962]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:16.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:15 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:16.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:15 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:53:16.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:15 smithi129 ceph-mon[33173]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:16.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:15 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:16.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:15 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:53:17.193 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:53:18.026 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:17 smithi039 ceph-mon[29586]: pgmap v61: 73 pgs: 73 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:18.026 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:17 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/3094159848' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-14T03:53:18.026 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:17 smithi039 ceph-mon[40962]: pgmap v61: 73 pgs: 73 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:18.027 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:17 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/3094159848' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-14T03:53:18.047 INFO:teuthology.orchestra.run.smithi039.stdout:{"epoch":21,"flags":0,"active_gid":14436,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6800","nonce":3924974081},{"type":"v1","addr":"172.21.15.39:6801","nonce":3924974081}]},"active_addr":"172.21.15.39:6801/3924974081","active_change":"2024-09-14T03:51:44.726931+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":24340,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.39:8443/","prometheus":"http://172.21.15.39:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.39:0","nonce":3196173579}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.39:0","nonce":1825825785}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.39:0","nonce":1766366412}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.39:0","nonce":502495021}]}]} 2024-09-14T03:53:18.052 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-14T03:53:18.052 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-14T03:53:18.052 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T03:53:18.266 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:53:18.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:17 smithi129 ceph-mon[33173]: pgmap v61: 73 pgs: 73 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:18.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:17 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/3094159848' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-09-14T03:53:18.911 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:18 smithi129 ceph-mon[33173]: mds.? [v2:172.21.15.39:6834/2717630652,v1:172.21.15.39:6835/2717630652] up:boot 2024-09-14T03:53:18.912 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:18 smithi129 ceph-mon[33173]: daemon mds.foo.smithi039.bbwiwf assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-09-14T03:53:18.912 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:18 smithi129 ceph-mon[33173]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-14T03:53:18.912 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:18 smithi129 ceph-mon[33173]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-14T03:53:18.912 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:18 smithi129 ceph-mon[33173]: Cluster is now healthy 2024-09-14T03:53:18.912 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:18 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi039.bbwiwf"}]: dispatch 2024-09-14T03:53:18.912 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:18 smithi129 ceph-mon[33173]: fsmap foo:0 1 up:standby 2024-09-14T03:53:18.912 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:18 smithi129 ceph-mon[33173]: fsmap foo:1 {0=foo.smithi039.bbwiwf=up:creating} 2024-09-14T03:53:18.912 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:18 smithi129 ceph-mon[33173]: daemon mds.foo.smithi039.bbwiwf is now active in filesystem foo as rank 0 2024-09-14T03:53:18.912 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:18 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:18.912 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:18 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:18.912 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:18 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:18.912 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:18 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi129.ybjqcw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-14T03:53:18.913 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:18 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi129.ybjqcw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-14T03:53:18.913 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:18 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi129.ybjqcw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-14T03:53:18.913 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:18 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:53:18.913 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:18 smithi129 ceph-mon[33173]: Deploying daemon mds.foo.smithi129.ybjqcw on smithi129 2024-09-14T03:53:18.913 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:18 smithi129 ceph-mon[33173]: pgmap v62: 73 pgs: 73 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:19.032 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[40962]: mds.? [v2:172.21.15.39:6834/2717630652,v1:172.21.15.39:6835/2717630652] up:boot 2024-09-14T03:53:19.032 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[40962]: daemon mds.foo.smithi039.bbwiwf assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-09-14T03:53:19.032 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[40962]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-14T03:53:19.032 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[40962]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-14T03:53:19.032 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[40962]: Cluster is now healthy 2024-09-14T03:53:19.032 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi039.bbwiwf"}]: dispatch 2024-09-14T03:53:19.033 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[40962]: fsmap foo:0 1 up:standby 2024-09-14T03:53:19.033 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[40962]: fsmap foo:1 {0=foo.smithi039.bbwiwf=up:creating} 2024-09-14T03:53:19.033 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[40962]: daemon mds.foo.smithi039.bbwiwf is now active in filesystem foo as rank 0 2024-09-14T03:53:19.033 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:19.033 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:19.033 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:19.033 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi129.ybjqcw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-14T03:53:19.033 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi129.ybjqcw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-14T03:53:19.033 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi129.ybjqcw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-14T03:53:19.033 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:53:19.033 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[40962]: Deploying daemon mds.foo.smithi129.ybjqcw on smithi129 2024-09-14T03:53:19.033 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[40962]: pgmap v62: 73 pgs: 73 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:19.034 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[29586]: mds.? [v2:172.21.15.39:6834/2717630652,v1:172.21.15.39:6835/2717630652] up:boot 2024-09-14T03:53:19.034 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[29586]: daemon mds.foo.smithi039.bbwiwf assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-09-14T03:53:19.034 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[29586]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-09-14T03:53:19.034 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[29586]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-09-14T03:53:19.034 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[29586]: Cluster is now healthy 2024-09-14T03:53:19.034 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi039.bbwiwf"}]: dispatch 2024-09-14T03:53:19.034 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[29586]: fsmap foo:0 1 up:standby 2024-09-14T03:53:19.034 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[29586]: fsmap foo:1 {0=foo.smithi039.bbwiwf=up:creating} 2024-09-14T03:53:19.034 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[29586]: daemon mds.foo.smithi039.bbwiwf is now active in filesystem foo as rank 0 2024-09-14T03:53:19.034 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:19.034 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:19.034 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:19.034 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi129.ybjqcw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-14T03:53:19.035 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi129.ybjqcw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-09-14T03:53:19.035 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi129.ybjqcw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-09-14T03:53:19.035 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:53:19.035 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[29586]: Deploying daemon mds.foo.smithi129.ybjqcw on smithi129 2024-09-14T03:53:19.035 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:18 smithi039 ceph-mon[29586]: pgmap v62: 73 pgs: 73 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:19.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:53:19 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:53:19] "GET /metrics HTTP/1.1" 200 33305 "" "Prometheus/2.43.0" 2024-09-14T03:53:19.893 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:19 smithi129 ceph-mon[33173]: mds.? [v2:172.21.15.39:6834/2717630652,v1:172.21.15.39:6835/2717630652] up:active 2024-09-14T03:53:19.893 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:19 smithi129 ceph-mon[33173]: fsmap foo:1 {0=foo.smithi039.bbwiwf=up:active} 2024-09-14T03:53:20.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:19 smithi039 ceph-mon[29586]: mds.? [v2:172.21.15.39:6834/2717630652,v1:172.21.15.39:6835/2717630652] up:active 2024-09-14T03:53:20.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:19 smithi039 ceph-mon[29586]: fsmap foo:1 {0=foo.smithi039.bbwiwf=up:active} 2024-09-14T03:53:20.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:19 smithi039 ceph-mon[40962]: mds.? [v2:172.21.15.39:6834/2717630652,v1:172.21.15.39:6835/2717630652] up:active 2024-09-14T03:53:20.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:19 smithi039 ceph-mon[40962]: fsmap foo:1 {0=foo.smithi039.bbwiwf=up:active} 2024-09-14T03:53:20.128 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:53:20.129 INFO:teuthology.orchestra.run.smithi039.stdout:{"epoch":72,"fsid":"bd7c803c-724b-11ef-bceb-c7b262605968","created":"2024-09-14T03:46:13.726476+0000","modified":"2024-09-14T03:53:07.243311+0000","last_up_change":"2024-09-14T03:51:13.230622+0000","last_in_change":"2024-09-14T03:50:56.569390+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:49:18.022402+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T03:51:45.764151+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-14T03:53:04.166504+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-14T03:53:04.665111+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"72","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"5fdfd478-dd27-4674-a7d6-53177d9e549a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6802","nonce":1775094854},{"type":"v1","addr":"172.21.15.39:6803","nonce":1775094854}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6804","nonce":1775094854},{"type":"v1","addr":"172.21.15.39:6805","nonce":1775094854}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6808","nonce":1775094854},{"type":"v1","addr":"172.21.15.39:6809","nonce":1775094854}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6806","nonce":1775094854},{"type":"v1","addr":"172.21.15.39:6807","nonce":1775094854}]},"public_addr":"172.21.15.39:6803/1775094854","cluster_addr":"172.21.15.39:6805/1775094854","heartbeat_back_addr":"172.21.15.39:6809/1775094854","heartbeat_front_addr":"172.21.15.39:6807/1775094854","state":["exists","up"]},{"osd":1,"uuid":"991c236e-1679-4c1e-a1a9-3894b1b12055","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6810","nonce":2410143394},{"type":"v1","addr":"172.21.15.39:6811","nonce":2410143394}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6812","nonce":2410143394},{"type":"v1","addr":"172.21.15.39:6813","nonce":2410143394}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6816","nonce":2410143394},{"type":"v1","addr":"172.21.15.39:6817","nonce":2410143394}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6814","nonce":2410143394},{"type":"v1","addr":"172.21.15.39:6815","nonce":2410143394}]},"public_addr":"172.21.15.39:6811/2410143394","cluster_addr":"172.21.15.39:6813/2410143394","heartbeat_back_addr":"172.21.15.39:6817/2410143394","heartbeat_front_addr":"172.21.15.39:6815/2410143394","state":["exists","up"]},{"osd":2,"uuid":"f9611ffd-5d37-4465-8114-ecc70d4d6d14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6818","nonce":3600430167},{"type":"v1","addr":"172.21.15.39:6819","nonce":3600430167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6820","nonce":3600430167},{"type":"v1","addr":"172.21.15.39:6821","nonce":3600430167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6824","nonce":3600430167},{"type":"v1","addr":"172.21.15.39:6825","nonce":3600430167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6822","nonce":3600430167},{"type":"v1","addr":"172.21.15.39:6823","nonce":3600430167}]},"public_addr":"172.21.15.39:6819/3600430167","cluster_addr":"172.21.15.39:6821/3600430167","heartbeat_back_addr":"172.21.15.39:6825/3600430167","heartbeat_front_addr":"172.21.15.39:6823/3600430167","state":["exists","up"]},{"osd":3,"uuid":"56b6877b-2107-47d6-8da9-015db7352288","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6826","nonce":1027302150},{"type":"v1","addr":"172.21.15.39:6827","nonce":1027302150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6828","nonce":1027302150},{"type":"v1","addr":"172.21.15.39:6829","nonce":1027302150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6832","nonce":1027302150},{"type":"v1","addr":"172.21.15.39:6833","nonce":1027302150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6830","nonce":1027302150},{"type":"v1","addr":"172.21.15.39:6831","nonce":1027302150}]},"public_addr":"172.21.15.39:6827/1027302150","cluster_addr":"172.21.15.39:6829/1027302150","heartbeat_back_addr":"172.21.15.39:6833/1027302150","heartbeat_front_addr":"172.21.15.39:6831/1027302150","state":["exists","up"]},{"osd":4,"uuid":"2d5523c4-5871-4fb3-bc0d-62c3b7d8b6a1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6800","nonce":168261430},{"type":"v1","addr":"172.21.15.129:6801","nonce":168261430}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6802","nonce":168261430},{"type":"v1","addr":"172.21.15.129:6803","nonce":168261430}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6806","nonce":168261430},{"type":"v1","addr":"172.21.15.129:6807","nonce":168261430}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6804","nonce":168261430},{"type":"v1","addr":"172.21.15.129:6805","nonce":168261430}]},"public_addr":"172.21.15.129:6801/168261430","cluster_addr":"172.21.15.129:6803/168261430","heartbeat_back_addr":"172.21.15.129:6807/168261430","heartbeat_front_addr":"172.21.15.129:6805/168261430","state":["exists","up"]},{"osd":5,"uuid":"a3ba55fd-f32e-4f6d-9b48-d2845a3474cd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6808","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6809","nonce":2184993002}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6810","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6811","nonce":2184993002}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6814","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6815","nonce":2184993002}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6812","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6813","nonce":2184993002}]},"public_addr":"172.21.15.129:6809/2184993002","cluster_addr":"172.21.15.129:6811/2184993002","heartbeat_back_addr":"172.21.15.129:6815/2184993002","heartbeat_front_addr":"172.21.15.129:6813/2184993002","state":["exists","up"]},{"osd":6,"uuid":"79e9c0a2-86a4-4267-a799-059d473d7b74","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6816","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6817","nonce":3767031353}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6818","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6819","nonce":3767031353}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6822","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6823","nonce":3767031353}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6820","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6821","nonce":3767031353}]},"public_addr":"172.21.15.129:6817/3767031353","cluster_addr":"172.21.15.129:6819/3767031353","heartbeat_back_addr":"172.21.15.129:6823/3767031353","heartbeat_front_addr":"172.21.15.129:6821/3767031353","state":["exists","up"]},{"osd":7,"uuid":"22f41628-4b88-4b8c-82b2-fcd2802da1bf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6824","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6825","nonce":2370414690}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6826","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6827","nonce":2370414690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6830","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6831","nonce":2370414690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6828","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6829","nonce":2370414690}]},"public_addr":"172.21.15.129:6825/2370414690","cluster_addr":"172.21.15.129:6827/2370414690","heartbeat_back_addr":"172.21.15.129:6831/2370414690","heartbeat_front_addr":"172.21.15.129:6829/2370414690","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:48:23.355518+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:48:48.715441+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:49:13.415323+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:49:39.935205+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:50:01.649042+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:50:24.001678+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:50:47.985011+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:51:11.584825+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.39:0/2587896825":"2024-09-15T03:51:44.726801+0000","172.21.15.39:0/754571920":"2024-09-15T03:51:44.726801+0000","172.21.15.39:0/2133325947":"2024-09-15T03:46:35.805097+0000","172.21.15.39:0/81449629":"2024-09-15T03:46:35.805097+0000","172.21.15.39:6800/628437919":"2024-09-15T03:46:35.805097+0000","172.21.15.39:6801/3127470450":"2024-09-15T03:46:59.511580+0000","172.21.15.39:6801/748677190":"2024-09-15T03:51:44.726801+0000","172.21.15.39:6801/628437919":"2024-09-15T03:46:35.805097+0000","172.21.15.39:0/2670041591":"2024-09-15T03:46:35.805097+0000","172.21.15.39:0/3161547781":"2024-09-15T03:46:59.511580+0000","172.21.15.39:0/1530637819":"2024-09-15T03:46:59.511580+0000","172.21.15.39:0/2745905728":"2024-09-15T03:51:44.726801+0000","172.21.15.39:6800/748677190":"2024-09-15T03:51:44.726801+0000","172.21.15.39:0/209353267":"2024-09-15T03:46:59.511580+0000","172.21.15.39:6800/3127470450":"2024-09-15T03:46:59.511580+0000","172.21.15.39:0/2951004223":"2024-09-15T03:51:44.726801+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:53:20.876 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-14T03:53:20.877 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T03:53:21.104 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:53:21.227 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:20 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/2172186309' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:53:21.227 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:20 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:21.227 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:20 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:21.227 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:20 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:21.227 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:20 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:21.227 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:20 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:21.227 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:20 smithi039 ceph-mon[29586]: pgmap v63: 73 pgs: 73 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:21.227 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:20 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:53:21.228 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:20 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/2172186309' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:53:21.228 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:20 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:21.228 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:20 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:21.228 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:20 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:21.228 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:20 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:21.228 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:20 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:21.228 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:20 smithi039 ceph-mon[40962]: pgmap v63: 73 pgs: 73 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:21.228 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:20 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:53:21.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:20 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/2172186309' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:53:21.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:20 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:21.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:20 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:21.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:20 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:21.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:20 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:21.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:20 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:21.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:20 smithi129 ceph-mon[33173]: pgmap v63: 73 pgs: 73 active+clean; 577 KiB data, 218 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:21.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:20 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:53:22.010 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:21 smithi039 ceph-mon[29586]: mds.? [v2:172.21.15.129:6832/2789452907,v1:172.21.15.129:6833/2789452907] up:boot 2024-09-14T03:53:22.010 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:21 smithi039 ceph-mon[29586]: fsmap foo:1 {0=foo.smithi039.bbwiwf=up:active} 1 up:standby 2024-09-14T03:53:22.010 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:21 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi129.ybjqcw"}]: dispatch 2024-09-14T03:53:22.010 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:21 smithi039 ceph-mon[29586]: fsmap foo:1 {0=foo.smithi039.bbwiwf=up:active} 1 up:standby 2024-09-14T03:53:22.010 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:21 smithi039 ceph-mon[40962]: mds.? [v2:172.21.15.129:6832/2789452907,v1:172.21.15.129:6833/2789452907] up:boot 2024-09-14T03:53:22.010 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:21 smithi039 ceph-mon[40962]: fsmap foo:1 {0=foo.smithi039.bbwiwf=up:active} 1 up:standby 2024-09-14T03:53:22.010 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:21 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi129.ybjqcw"}]: dispatch 2024-09-14T03:53:22.011 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:21 smithi039 ceph-mon[40962]: fsmap foo:1 {0=foo.smithi039.bbwiwf=up:active} 1 up:standby 2024-09-14T03:53:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:21 smithi129 ceph-mon[33173]: mds.? [v2:172.21.15.129:6832/2789452907,v1:172.21.15.129:6833/2789452907] up:boot 2024-09-14T03:53:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:21 smithi129 ceph-mon[33173]: fsmap foo:1 {0=foo.smithi039.bbwiwf=up:active} 1 up:standby 2024-09-14T03:53:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:21 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi129.ybjqcw"}]: dispatch 2024-09-14T03:53:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:21 smithi129 ceph-mon[33173]: fsmap foo:1 {0=foo.smithi039.bbwiwf=up:active} 1 up:standby 2024-09-14T03:53:23.836 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:53:23.837 INFO:teuthology.orchestra.run.smithi039.stdout:{"epoch":72,"fsid":"bd7c803c-724b-11ef-bceb-c7b262605968","created":"2024-09-14T03:46:13.726476+0000","modified":"2024-09-14T03:53:07.243311+0000","last_up_change":"2024-09-14T03:51:13.230622+0000","last_in_change":"2024-09-14T03:50:56.569390+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:49:18.022402+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T03:51:45.764151+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-14T03:53:04.166504+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-14T03:53:04.665111+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"72","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"5fdfd478-dd27-4674-a7d6-53177d9e549a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6802","nonce":1775094854},{"type":"v1","addr":"172.21.15.39:6803","nonce":1775094854}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6804","nonce":1775094854},{"type":"v1","addr":"172.21.15.39:6805","nonce":1775094854}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6808","nonce":1775094854},{"type":"v1","addr":"172.21.15.39:6809","nonce":1775094854}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6806","nonce":1775094854},{"type":"v1","addr":"172.21.15.39:6807","nonce":1775094854}]},"public_addr":"172.21.15.39:6803/1775094854","cluster_addr":"172.21.15.39:6805/1775094854","heartbeat_back_addr":"172.21.15.39:6809/1775094854","heartbeat_front_addr":"172.21.15.39:6807/1775094854","state":["exists","up"]},{"osd":1,"uuid":"991c236e-1679-4c1e-a1a9-3894b1b12055","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6810","nonce":2410143394},{"type":"v1","addr":"172.21.15.39:6811","nonce":2410143394}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6812","nonce":2410143394},{"type":"v1","addr":"172.21.15.39:6813","nonce":2410143394}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6816","nonce":2410143394},{"type":"v1","addr":"172.21.15.39:6817","nonce":2410143394}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6814","nonce":2410143394},{"type":"v1","addr":"172.21.15.39:6815","nonce":2410143394}]},"public_addr":"172.21.15.39:6811/2410143394","cluster_addr":"172.21.15.39:6813/2410143394","heartbeat_back_addr":"172.21.15.39:6817/2410143394","heartbeat_front_addr":"172.21.15.39:6815/2410143394","state":["exists","up"]},{"osd":2,"uuid":"f9611ffd-5d37-4465-8114-ecc70d4d6d14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6818","nonce":3600430167},{"type":"v1","addr":"172.21.15.39:6819","nonce":3600430167}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6820","nonce":3600430167},{"type":"v1","addr":"172.21.15.39:6821","nonce":3600430167}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6824","nonce":3600430167},{"type":"v1","addr":"172.21.15.39:6825","nonce":3600430167}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6822","nonce":3600430167},{"type":"v1","addr":"172.21.15.39:6823","nonce":3600430167}]},"public_addr":"172.21.15.39:6819/3600430167","cluster_addr":"172.21.15.39:6821/3600430167","heartbeat_back_addr":"172.21.15.39:6825/3600430167","heartbeat_front_addr":"172.21.15.39:6823/3600430167","state":["exists","up"]},{"osd":3,"uuid":"56b6877b-2107-47d6-8da9-015db7352288","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6826","nonce":1027302150},{"type":"v1","addr":"172.21.15.39:6827","nonce":1027302150}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6828","nonce":1027302150},{"type":"v1","addr":"172.21.15.39:6829","nonce":1027302150}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6832","nonce":1027302150},{"type":"v1","addr":"172.21.15.39:6833","nonce":1027302150}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6830","nonce":1027302150},{"type":"v1","addr":"172.21.15.39:6831","nonce":1027302150}]},"public_addr":"172.21.15.39:6827/1027302150","cluster_addr":"172.21.15.39:6829/1027302150","heartbeat_back_addr":"172.21.15.39:6833/1027302150","heartbeat_front_addr":"172.21.15.39:6831/1027302150","state":["exists","up"]},{"osd":4,"uuid":"2d5523c4-5871-4fb3-bc0d-62c3b7d8b6a1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6800","nonce":168261430},{"type":"v1","addr":"172.21.15.129:6801","nonce":168261430}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6802","nonce":168261430},{"type":"v1","addr":"172.21.15.129:6803","nonce":168261430}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6806","nonce":168261430},{"type":"v1","addr":"172.21.15.129:6807","nonce":168261430}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6804","nonce":168261430},{"type":"v1","addr":"172.21.15.129:6805","nonce":168261430}]},"public_addr":"172.21.15.129:6801/168261430","cluster_addr":"172.21.15.129:6803/168261430","heartbeat_back_addr":"172.21.15.129:6807/168261430","heartbeat_front_addr":"172.21.15.129:6805/168261430","state":["exists","up"]},{"osd":5,"uuid":"a3ba55fd-f32e-4f6d-9b48-d2845a3474cd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6808","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6809","nonce":2184993002}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6810","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6811","nonce":2184993002}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6814","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6815","nonce":2184993002}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6812","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6813","nonce":2184993002}]},"public_addr":"172.21.15.129:6809/2184993002","cluster_addr":"172.21.15.129:6811/2184993002","heartbeat_back_addr":"172.21.15.129:6815/2184993002","heartbeat_front_addr":"172.21.15.129:6813/2184993002","state":["exists","up"]},{"osd":6,"uuid":"79e9c0a2-86a4-4267-a799-059d473d7b74","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":69,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6816","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6817","nonce":3767031353}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6818","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6819","nonce":3767031353}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6822","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6823","nonce":3767031353}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6820","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6821","nonce":3767031353}]},"public_addr":"172.21.15.129:6817/3767031353","cluster_addr":"172.21.15.129:6819/3767031353","heartbeat_back_addr":"172.21.15.129:6823/3767031353","heartbeat_front_addr":"172.21.15.129:6821/3767031353","state":["exists","up"]},{"osd":7,"uuid":"22f41628-4b88-4b8c-82b2-fcd2802da1bf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":68,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6824","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6825","nonce":2370414690}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6826","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6827","nonce":2370414690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6830","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6831","nonce":2370414690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6828","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6829","nonce":2370414690}]},"public_addr":"172.21.15.129:6825/2370414690","cluster_addr":"172.21.15.129:6827/2370414690","heartbeat_back_addr":"172.21.15.129:6831/2370414690","heartbeat_front_addr":"172.21.15.129:6829/2370414690","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:48:23.355518+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:48:48.715441+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:49:13.415323+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:49:39.935205+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:50:01.649042+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:50:24.001678+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:50:47.985011+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:51:11.584825+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.39:0/2587896825":"2024-09-15T03:51:44.726801+0000","172.21.15.39:0/754571920":"2024-09-15T03:51:44.726801+0000","172.21.15.39:0/2133325947":"2024-09-15T03:46:35.805097+0000","172.21.15.39:0/81449629":"2024-09-15T03:46:35.805097+0000","172.21.15.39:6800/628437919":"2024-09-15T03:46:35.805097+0000","172.21.15.39:6801/3127470450":"2024-09-15T03:46:59.511580+0000","172.21.15.39:6801/748677190":"2024-09-15T03:51:44.726801+0000","172.21.15.39:6801/628437919":"2024-09-15T03:46:35.805097+0000","172.21.15.39:0/2670041591":"2024-09-15T03:46:35.805097+0000","172.21.15.39:0/3161547781":"2024-09-15T03:46:59.511580+0000","172.21.15.39:0/1530637819":"2024-09-15T03:46:59.511580+0000","172.21.15.39:0/2745905728":"2024-09-15T03:51:44.726801+0000","172.21.15.39:6800/748677190":"2024-09-15T03:51:44.726801+0000","172.21.15.39:0/209353267":"2024-09-15T03:46:59.511580+0000","172.21.15.39:6800/3127470450":"2024-09-15T03:46:59.511580+0000","172.21.15.39:0/2951004223":"2024-09-15T03:51:44.726801+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T03:53:24.017 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:23 smithi129 ceph-mon[33173]: mds.? [v2:172.21.15.39:6834/2717630652,v1:172.21.15.39:6835/2717630652] up:active 2024-09-14T03:53:24.017 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:23 smithi129 ceph-mon[33173]: fsmap foo:1 {0=foo.smithi039.bbwiwf=up:active} 1 up:standby 2024-09-14T03:53:24.017 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:23 smithi129 ceph-mon[33173]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:53:24.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:23 smithi039 ceph-mon[29586]: mds.? [v2:172.21.15.39:6834/2717630652,v1:172.21.15.39:6835/2717630652] up:active 2024-09-14T03:53:24.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:23 smithi039 ceph-mon[29586]: fsmap foo:1 {0=foo.smithi039.bbwiwf=up:active} 1 up:standby 2024-09-14T03:53:24.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:23 smithi039 ceph-mon[29586]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:53:24.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:23 smithi039 ceph-mon[40962]: mds.? [v2:172.21.15.39:6834/2717630652,v1:172.21.15.39:6835/2717630652] up:active 2024-09-14T03:53:24.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:23 smithi039 ceph-mon[40962]: fsmap foo:1 {0=foo.smithi039.bbwiwf=up:active} 1 up:standby 2024-09-14T03:53:24.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:23 smithi039 ceph-mon[40962]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:53:24.590 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-14T03:53:24.590 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-14T03:53:24.591 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-14T03:53:24.591 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-14T03:53:24.591 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-14T03:53:24.592 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-14T03:53:24.592 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-14T03:53:24.593 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-14T03:53:25.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:24 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/3415624767' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:53:25.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:24 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:25.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:24 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:25.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:24 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/3415624767' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:53:25.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:24 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:25.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:24 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:25.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:24 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/3415624767' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-09-14T03:53:25.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:24 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:25.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:24 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:25.665 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:53:25.668 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:53:25.670 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:53:25.674 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:53:25.674 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:53:25.676 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:53:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:25 smithi129 ceph-mon[33173]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:53:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:25 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:25 smithi129 ceph-mon[33173]: mds.? [v2:172.21.15.129:6832/2789452907,v1:172.21.15.129:6833/2789452907] up:standby 2024-09-14T03:53:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:25 smithi129 ceph-mon[33173]: fsmap foo:1 {0=foo.smithi039.bbwiwf=up:active} 1 up:standby 2024-09-14T03:53:26.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:25 smithi039 ceph-mon[29586]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:53:26.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:25 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:26.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:25 smithi039 ceph-mon[29586]: mds.? [v2:172.21.15.129:6832/2789452907,v1:172.21.15.129:6833/2789452907] up:standby 2024-09-14T03:53:26.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:25 smithi039 ceph-mon[29586]: fsmap foo:1 {0=foo.smithi039.bbwiwf=up:active} 1 up:standby 2024-09-14T03:53:26.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:25 smithi039 ceph-mon[40962]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:53:26.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:25 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:26.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:25 smithi039 ceph-mon[40962]: mds.? [v2:172.21.15.129:6832/2789452907,v1:172.21.15.129:6833/2789452907] up:standby 2024-09-14T03:53:26.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:25 smithi039 ceph-mon[40962]: fsmap foo:1 {0=foo.smithi039.bbwiwf=up:active} 1 up:standby 2024-09-14T03:53:26.968 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:26 smithi039 ceph-mon[29586]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:53:26.968 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:26 smithi039 ceph-mon[40962]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:53:27.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:26 smithi129 ceph-mon[33173]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:53:27.684 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:53:29.032 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:28 smithi039 ceph-mon[29586]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:53:29.033 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:28 smithi039 ceph-mon[40962]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:53:29.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:28 smithi129 ceph-mon[33173]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:53:29.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:53:29 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:53:29] "GET /metrics HTTP/1.1" 200 33305 "" "Prometheus/2.43.0" 2024-09-14T03:53:31.026 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:30 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:31.026 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:30 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:53:31.027 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:30 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:31.027 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:30 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:53:31.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:30 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:31.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:30 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:53:32.158 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:31 smithi039 ceph-mon[29586]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:53:32.159 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:31 smithi039 ceph-mon[40962]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:53:32.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:31 smithi129 ceph-mon[33173]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:53:33.679 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:33 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:33.679 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:33 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:33.679 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:33 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:53:33.680 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:33 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:53:33.680 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:33 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:33.680 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:33 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:53:33.680 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:33 smithi039 ceph-mon[40962]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:53:33.680 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:33 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:33.680 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:33 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:33.680 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:33 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:53:33.680 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:33 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:53:33.680 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:33 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:33.680 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:33 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:53:33.681 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:33 smithi039 ceph-mon[29586]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:53:33.700 INFO:teuthology.orchestra.run.smithi039.stdout:167503724584 2024-09-14T03:53:33.701 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-14T03:53:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:33 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:33 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:33 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:53:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:33 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:53:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:33 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:33.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:33 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:53:33.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:33 smithi129 ceph-mon[33173]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2024-09-14T03:53:35.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:34 smithi039 ceph-mon[29586]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-14T03:53:35.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:34 smithi039 ceph-mon[40962]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-14T03:53:35.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:34 smithi129 ceph-mon[33173]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-14T03:53:37.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:36 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:37.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:36 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:37.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:36 smithi129 ceph-mon[33173]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-14T03:53:37.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:36 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:37.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:36 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:37.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:36 smithi039 ceph-mon[29586]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-14T03:53:37.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:36 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:37.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:36 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:37.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:36 smithi039 ceph-mon[40962]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-09-14T03:53:37.615 INFO:teuthology.orchestra.run.smithi039.stdout:201863462948 2024-09-14T03:53:37.616 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-14T03:53:38.638 INFO:teuthology.orchestra.run.smithi039.stdout:77309411383 2024-09-14T03:53:38.639 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-14T03:53:38.936 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:38 smithi039 ceph-mon[29586]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:38.936 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:38 smithi039 ceph-mon[40962]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:39.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:38 smithi129 ceph-mon[33173]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:39.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:53:39 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:53:39] "GET /metrics HTTP/1.1" 200 33844 "" "Prometheus/2.43.0" 2024-09-14T03:53:39.655 INFO:teuthology.orchestra.run.smithi039.stdout:111669149746 2024-09-14T03:53:39.656 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-14T03:53:40.179 INFO:teuthology.orchestra.run.smithi039.stdout:34359738433 2024-09-14T03:53:40.180 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-14T03:53:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:40 smithi129 ceph-mon[33173]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:41.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:40 smithi039 ceph-mon[29586]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:41.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:40 smithi039 ceph-mon[40962]: pgmap v73: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:41.357 INFO:teuthology.orchestra.run.smithi039.stdout:133143986223 2024-09-14T03:53:41.358 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-14T03:53:41.710 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:53:41.714 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:53:42.144 INFO:teuthology.orchestra.run.smithi039.stdout:55834574909 2024-09-14T03:53:42.145 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-14T03:53:42.195 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:53:42.195 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:53:42.209 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:53:43.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:42 smithi039 ceph-mon[29586]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:43.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:42 smithi039 ceph-mon[40962]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:42 smithi129 ceph-mon[33173]: pgmap v74: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:45.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:44 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-09-14T03:53:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:44 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2024-09-14T03:53:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:44 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-09-14T03:53:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:44 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-14T03:53:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:44 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2024-09-14T03:53:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:44 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-14T03:53:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:44 smithi039 ceph-mon[29586]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:44 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:53:45.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:44 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-09-14T03:53:45.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:44 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2024-09-14T03:53:45.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:44 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-09-14T03:53:45.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:44 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-14T03:53:45.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:44 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2024-09-14T03:53:45.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:44 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-14T03:53:45.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:44 smithi039 ceph-mon[40962]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:45.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:44 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:53:45.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:44 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-09-14T03:53:45.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:44 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2024-09-14T03:53:45.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:44 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]: dispatch 2024-09-14T03:53:45.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:44 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-14T03:53:45.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:44 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]: dispatch 2024-09-14T03:53:45.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:44 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]: dispatch 2024-09-14T03:53:45.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:44 smithi129 ceph-mon[33173]: pgmap v75: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:45.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:44 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:53:45.920 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:45 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-09-14T03:53:45.920 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:45 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]': finished 2024-09-14T03:53:45.920 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:45 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-09-14T03:53:45.920 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:45 smithi039 ceph-mon[29586]: osdmap e73: 8 total, 8 up, 8 in 2024-09-14T03:53:45.920 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:45 smithi039 ceph-mon[29586]: osdmap e74: 8 total, 8 up, 8 in 2024-09-14T03:53:45.920 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:45 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-09-14T03:53:45.921 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:45 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]': finished 2024-09-14T03:53:45.921 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:45 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-09-14T03:53:45.921 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:45 smithi039 ceph-mon[40962]: osdmap e73: 8 total, 8 up, 8 in 2024-09-14T03:53:45.921 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:45 smithi039 ceph-mon[40962]: osdmap e74: 8 total, 8 up, 8 in 2024-09-14T03:53:46.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:45 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [1, 2]}]': finished 2024-09-14T03:53:46.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:45 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 5]}]': finished 2024-09-14T03:53:46.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:45 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1f", "id": [1, 2]}]': finished 2024-09-14T03:53:46.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:45 smithi129 ceph-mon[33173]: osdmap e73: 8 total, 8 up, 8 in 2024-09-14T03:53:46.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:45 smithi129 ceph-mon[33173]: osdmap e74: 8 total, 8 up, 8 in 2024-09-14T03:53:46.710 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:53:46.711 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:53:47.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:46 smithi039 ceph-mon[29586]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:47.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:46 smithi039 ceph-mon[40962]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:47.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:46 smithi129 ceph-mon[33173]: pgmap v78: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:48.660 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:53:49.032 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:48 smithi039 ceph-mon[29586]: pgmap v79: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:49.032 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:48 smithi039 ceph-mon[40962]: pgmap v79: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:49.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:48 smithi129 ceph-mon[33173]: pgmap v79: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:49.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:53:49 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:53:49] "GET /metrics HTTP/1.1" 200 33837 "" "Prometheus/2.43.0" 2024-09-14T03:53:49.470 INFO:teuthology.orchestra.run.smithi039.stdout:77309411386 2024-09-14T03:53:50.029 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:49 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/539300886' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-14T03:53:50.030 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:49 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/539300886' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-14T03:53:50.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:49 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/539300886' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-09-14T03:53:51.002 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:50 smithi039 ceph-mon[29586]: pgmap v80: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:51.002 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:50 smithi039 ceph-mon[40962]: pgmap v80: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:50 smithi129 ceph-mon[33173]: pgmap v80: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:51.812 INFO:teuthology.orchestra.run.smithi039.stdout:201863462950 2024-09-14T03:53:52.126 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:51 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/1306067729' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-14T03:53:52.127 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:51 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/1306067729' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-14T03:53:52.149 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411383 got 77309411386 for osd.2 2024-09-14T03:53:52.150 DEBUG:teuthology.parallel:result is None 2024-09-14T03:53:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:51 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/1306067729' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-09-14T03:53:52.982 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:52 smithi039 ceph-mon[29586]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:52.983 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:52 smithi039 ceph-mon[40962]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:53.152 INFO:teuthology.orchestra.run.smithi039.stdout:167503724588 2024-09-14T03:53:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:52 smithi129 ceph-mon[33173]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:53.684 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462948 got 201863462950 for osd.6 2024-09-14T03:53:53.684 DEBUG:teuthology.parallel:result is None 2024-09-14T03:53:54.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:53 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/162960629' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-14T03:53:54.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:53 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/162960629' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-14T03:53:54.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:53 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/162960629' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-09-14T03:53:54.973 INFO:teuthology.orchestra.run.smithi039.stdout:133143986226 2024-09-14T03:53:55.239 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:54 smithi039 ceph-mon[29586]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:55.240 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:54 smithi039 ceph-mon[40962]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:55.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:54 smithi129 ceph-mon[33173]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:55.501 INFO:teuthology.orchestra.run.smithi039.stdout:55834574912 2024-09-14T03:53:56.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:55 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/525822625' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-14T03:53:56.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:55 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/1775308890' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-14T03:53:56.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:55 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/525822625' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-14T03:53:56.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:55 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/1775308890' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-14T03:53:56.242 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574909 got 55834574912 for osd.1 2024-09-14T03:53:56.243 DEBUG:teuthology.parallel:result is None 2024-09-14T03:53:56.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:55 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/525822625' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-09-14T03:53:56.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:55 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/1775308890' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-09-14T03:53:57.015 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:56 smithi039 ceph-mon[29586]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:57.016 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:56 smithi039 ceph-mon[40962]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:57.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:56 smithi129 ceph-mon[33173]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:57.650 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986223 got 133143986226 for osd.4 2024-09-14T03:53:57.650 DEBUG:teuthology.parallel:result is None 2024-09-14T03:53:59.044 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724584 got 167503724588 for osd.5 2024-09-14T03:53:59.044 DEBUG:teuthology.parallel:result is None 2024-09-14T03:53:59.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:53:59 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:53:59] "GET /metrics HTTP/1.1" 200 33837 "" "Prometheus/2.43.0" 2024-09-14T03:53:59.652 INFO:teuthology.orchestra.run.smithi039.stdout:236223201316 2024-09-14T03:53:59.652 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-14T03:53:59.948 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:59 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:59.948 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:59 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:59.948 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:59 smithi039 ceph-mon[29586]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:59.948 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:59 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:53:59.948 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:59 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:53:59.948 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:53:59 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:59.949 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:59 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:59.949 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:59 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:53:59.949 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:59 smithi039 ceph-mon[40962]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:53:59.949 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:59 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:53:59.949 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:59 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:53:59.949 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:53:59 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:54:00.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:59 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:54:00.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:59 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:54:00.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:59 smithi129 ceph-mon[33173]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:00.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:59 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:54:00.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:59 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:54:00.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:53:59 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:54:00.540 INFO:teuthology.orchestra.run.smithi039.stdout:111669149751 2024-09-14T03:54:00.838 INFO:teuthology.orchestra.run.smithi039.stdout:34359738437 2024-09-14T03:54:00.838 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:00 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:54:00.839 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:00 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/1164405030' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-14T03:54:00.839 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:00 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:54:00.839 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:00 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/1164405030' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-14T03:54:01.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:00 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:54:01.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:00 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/1164405030' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-09-14T03:54:01.391 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149746 got 111669149751 for osd.3 2024-09-14T03:54:01.392 DEBUG:teuthology.parallel:result is None 2024-09-14T03:54:01.927 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:01 smithi039 ceph-mon[29586]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:01.928 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:01 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/1659505529' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-14T03:54:01.928 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:01 smithi039 ceph-mon[40962]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:01.928 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:01 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/1659505529' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-14T03:54:01.953 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738433 got 34359738437 for osd.0 2024-09-14T03:54:01.953 DEBUG:teuthology.parallel:result is None 2024-09-14T03:54:02.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:01 smithi129 ceph-mon[33173]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:02.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:01 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/1659505529' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-09-14T03:54:02.061 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:54:03.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:02 smithi129 ceph-mon[33173]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:03.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:02 smithi039 ceph-mon[29586]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:03.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:02 smithi039 ceph-mon[40962]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:03.981 INFO:teuthology.orchestra.run.smithi039.stdout:236223201317 2024-09-14T03:54:04.282 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:04 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/2316789179' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-14T03:54:04.282 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:04 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/2316789179' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-14T03:54:04.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:04 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/2316789179' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-09-14T03:54:04.672 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201316 got 236223201317 for osd.7 2024-09-14T03:54:04.672 DEBUG:teuthology.parallel:result is None 2024-09-14T03:54:04.673 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-14T03:54:04.673 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T03:54:04.880 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:54:05.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:05 smithi039 ceph-mon[29586]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:05.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:05 smithi039 ceph-mon[40962]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:05.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:05 smithi129 ceph-mon[33173]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:06.803 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:54:06.805 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T03:54:07.163 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:06 smithi039 ceph-mon[29586]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:07.163 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:06 smithi039 ceph-mon[29586]: from='client.14766 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:54:07.164 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:06 smithi039 ceph-mon[40962]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:07.164 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:06 smithi039 ceph-mon[40962]: from='client.14766 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:54:07.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:06 smithi129 ceph-mon[33173]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:07.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:06 smithi129 ceph-mon[33173]: from='client.14766 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:54:07.635 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":88,"stamp":"2024-09-14T03:54:06.765291+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592749,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":226,"num_write_kb":2432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":170,"ondisk_log_size":170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":224016,"kb_used_data":7824,"kb_used_omap":12,"kb_used_meta":216051,"kb_avail":749737200,"statfs":{"total":767960285184,"available":767730892800,"internally_reserved":0,"allocated":8011776,"data_stored":4469123,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12743,"internal_metadata":221236793},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002440"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.812800+0000","last_change":"2024-09-14T03:53:06.413742+0000","last_active":"2024-09-14T03:53:44.812800+0000","last_peered":"2024-09-14T03:53:44.812800+0000","last_clean":"2024-09-14T03:53:44.812800+0000","last_became_active":"2024-09-14T03:53:06.413085+0000","last_became_peered":"2024-09-14T03:53:06.413085+0000","last_unstale":"2024-09-14T03:53:44.812800+0000","last_undegraded":"2024-09-14T03:53:44.812800+0000","last_fullsized":"2024-09-14T03:53:44.812800+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:00:50.348234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055429+0000","last_change":"2024-09-14T03:53:05.237323+0000","last_active":"2024-09-14T03:53:45.055429+0000","last_peered":"2024-09-14T03:53:45.055429+0000","last_clean":"2024-09-14T03:53:45.055429+0000","last_became_active":"2024-09-14T03:53:05.236726+0000","last_became_peered":"2024-09-14T03:53:05.236726+0000","last_unstale":"2024-09-14T03:53:45.055429+0000","last_undegraded":"2024-09-14T03:53:45.055429+0000","last_fullsized":"2024-09-14T03:53:45.055429+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:52:00.093659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.810579+0000","last_change":"2024-09-14T03:53:05.550982+0000","last_active":"2024-09-14T03:53:44.810579+0000","last_peered":"2024-09-14T03:53:44.810579+0000","last_clean":"2024-09-14T03:53:44.810579+0000","last_became_active":"2024-09-14T03:53:05.550702+0000","last_became_peered":"2024-09-14T03:53:05.550702+0000","last_unstale":"2024-09-14T03:53:44.810579+0000","last_undegraded":"2024-09-14T03:53:44.810579+0000","last_fullsized":"2024-09-14T03:53:44.810579+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:39:18.307082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055421+0000","last_change":"2024-09-14T03:53:05.236964+0000","last_active":"2024-09-14T03:53:45.055421+0000","last_peered":"2024-09-14T03:53:45.055421+0000","last_clean":"2024-09-14T03:53:45.055421+0000","last_became_active":"2024-09-14T03:53:05.236672+0000","last_became_peered":"2024-09-14T03:53:05.236672+0000","last_unstale":"2024-09-14T03:53:45.055421+0000","last_undegraded":"2024-09-14T03:53:45.055421+0000","last_fullsized":"2024-09-14T03:53:45.055421+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:32:12.722182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055731+0000","last_change":"2024-09-14T03:53:05.551671+0000","last_active":"2024-09-14T03:53:45.055731+0000","last_peered":"2024-09-14T03:53:45.055731+0000","last_clean":"2024-09-14T03:53:45.055731+0000","last_became_active":"2024-09-14T03:53:05.550670+0000","last_became_peered":"2024-09-14T03:53:05.550670+0000","last_unstale":"2024-09-14T03:53:45.055731+0000","last_undegraded":"2024-09-14T03:53:45.055731+0000","last_fullsized":"2024-09-14T03:53:45.055731+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:01:18.148774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055309+0000","last_change":"2024-09-14T03:53:05.244003+0000","last_active":"2024-09-14T03:53:45.055309+0000","last_peered":"2024-09-14T03:53:45.055309+0000","last_clean":"2024-09-14T03:53:45.055309+0000","last_became_active":"2024-09-14T03:53:05.240743+0000","last_became_peered":"2024-09-14T03:53:05.240743+0000","last_unstale":"2024-09-14T03:53:45.055309+0000","last_undegraded":"2024-09-14T03:53:45.055309+0000","last_fullsized":"2024-09-14T03:53:45.055309+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:30.000189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.812006+0000","last_change":"2024-09-14T03:53:06.412161+0000","last_active":"2024-09-14T03:53:44.812006+0000","last_peered":"2024-09-14T03:53:44.812006+0000","last_clean":"2024-09-14T03:53:44.812006+0000","last_became_active":"2024-09-14T03:53:06.411884+0000","last_became_peered":"2024-09-14T03:53:06.411884+0000","last_unstale":"2024-09-14T03:53:44.812006+0000","last_undegraded":"2024-09-14T03:53:44.812006+0000","last_fullsized":"2024-09-14T03:53:44.812006+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:51:10.608362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.810130+0000","last_change":"2024-09-14T03:53:05.234450+0000","last_active":"2024-09-14T03:53:44.810130+0000","last_peered":"2024-09-14T03:53:44.810130+0000","last_clean":"2024-09-14T03:53:44.810130+0000","last_became_active":"2024-09-14T03:53:05.234191+0000","last_became_peered":"2024-09-14T03:53:05.234191+0000","last_unstale":"2024-09-14T03:53:44.810130+0000","last_undegraded":"2024-09-14T03:53:44.810130+0000","last_fullsized":"2024-09-14T03:53:44.810130+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:13:23.956618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055724+0000","last_change":"2024-09-14T03:53:05.239668+0000","last_active":"2024-09-14T03:53:45.055724+0000","last_peered":"2024-09-14T03:53:45.055724+0000","last_clean":"2024-09-14T03:53:45.055724+0000","last_became_active":"2024-09-14T03:53:05.239366+0000","last_became_peered":"2024-09-14T03:53:05.239366+0000","last_unstale":"2024-09-14T03:53:45.055724+0000","last_undegraded":"2024-09-14T03:53:45.055724+0000","last_fullsized":"2024-09-14T03:53:45.055724+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:40:51.865187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.810395+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T03:53:44.810395+0000","last_peered":"2024-09-14T03:53:44.810395+0000","last_clean":"2024-09-14T03:53:44.810395+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T03:53:44.810395+0000","last_undegraded":"2024-09-14T03:53:44.810395+0000","last_fullsized":"2024-09-14T03:53:44.810395+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055720+0000","last_change":"2024-09-14T03:53:05.240390+0000","last_active":"2024-09-14T03:53:45.055720+0000","last_peered":"2024-09-14T03:53:45.055720+0000","last_clean":"2024-09-14T03:53:45.055720+0000","last_became_active":"2024-09-14T03:53:05.240112+0000","last_became_peered":"2024-09-14T03:53:05.240112+0000","last_unstale":"2024-09-14T03:53:45.055720+0000","last_undegraded":"2024-09-14T03:53:45.055720+0000","last_fullsized":"2024-09-14T03:53:45.055720+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:02:47.345057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.807169+0000","last_change":"2024-09-14T03:53:06.412035+0000","last_active":"2024-09-14T03:53:44.807169+0000","last_peered":"2024-09-14T03:53:44.807169+0000","last_clean":"2024-09-14T03:53:44.807169+0000","last_became_active":"2024-09-14T03:53:06.411660+0000","last_became_peered":"2024-09-14T03:53:06.411660+0000","last_unstale":"2024-09-14T03:53:44.807169+0000","last_undegraded":"2024-09-14T03:53:44.807169+0000","last_fullsized":"2024-09-14T03:53:44.807169+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:50:06.795190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055687+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T03:53:45.055687+0000","last_peered":"2024-09-14T03:53:45.055687+0000","last_clean":"2024-09-14T03:53:45.055687+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T03:53:45.055687+0000","last_undegraded":"2024-09-14T03:53:45.055687+0000","last_fullsized":"2024-09-14T03:53:45.055687+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.809276+0000","last_change":"2024-09-14T03:53:05.550065+0000","last_active":"2024-09-14T03:53:44.809276+0000","last_peered":"2024-09-14T03:53:44.809276+0000","last_clean":"2024-09-14T03:53:44.809276+0000","last_became_active":"2024-09-14T03:53:05.549787+0000","last_became_peered":"2024-09-14T03:53:05.549787+0000","last_unstale":"2024-09-14T03:53:44.809276+0000","last_undegraded":"2024-09-14T03:53:44.809276+0000","last_fullsized":"2024-09-14T03:53:44.809276+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:34:58.380815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.810686+0000","last_change":"2024-09-14T03:53:05.240540+0000","last_active":"2024-09-14T03:53:44.810686+0000","last_peered":"2024-09-14T03:53:44.810686+0000","last_clean":"2024-09-14T03:53:44.810686+0000","last_became_active":"2024-09-14T03:53:05.240257+0000","last_became_peered":"2024-09-14T03:53:05.240257+0000","last_unstale":"2024-09-14T03:53:44.810686+0000","last_undegraded":"2024-09-14T03:53:44.810686+0000","last_fullsized":"2024-09-14T03:53:44.810686+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:41.107551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055349+0000","last_change":"2024-09-14T03:53:05.369617+0000","last_active":"2024-09-14T03:53:45.055349+0000","last_peered":"2024-09-14T03:53:45.055349+0000","last_clean":"2024-09-14T03:53:45.055349+0000","last_became_active":"2024-09-14T03:53:05.369325+0000","last_became_peered":"2024-09-14T03:53:05.369325+0000","last_unstale":"2024-09-14T03:53:45.055349+0000","last_undegraded":"2024-09-14T03:53:45.055349+0000","last_fullsized":"2024-09-14T03:53:45.055349+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:16:02.438169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.811583+0000","last_change":"2024-09-14T03:53:05.245363+0000","last_active":"2024-09-14T03:53:44.811583+0000","last_peered":"2024-09-14T03:53:44.811583+0000","last_clean":"2024-09-14T03:53:44.811583+0000","last_became_active":"2024-09-14T03:53:05.245066+0000","last_became_peered":"2024-09-14T03:53:05.245066+0000","last_unstale":"2024-09-14T03:53:44.811583+0000","last_undegraded":"2024-09-14T03:53:44.811583+0000","last_fullsized":"2024-09-14T03:53:44.811583+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:47.623005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.807329+0000","last_change":"2024-09-14T03:53:06.310362+0000","last_active":"2024-09-14T03:53:44.807329+0000","last_peered":"2024-09-14T03:53:44.807329+0000","last_clean":"2024-09-14T03:53:44.807329+0000","last_became_active":"2024-09-14T03:53:06.310039+0000","last_became_peered":"2024-09-14T03:53:06.310039+0000","last_unstale":"2024-09-14T03:53:44.807329+0000","last_undegraded":"2024-09-14T03:53:44.807329+0000","last_fullsized":"2024-09-14T03:53:44.807329+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:06:11.654831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.809671+0000","last_change":"2024-09-14T03:53:05.236501+0000","last_active":"2024-09-14T03:53:44.809671+0000","last_peered":"2024-09-14T03:53:44.809671+0000","last_clean":"2024-09-14T03:53:44.809671+0000","last_became_active":"2024-09-14T03:53:05.236225+0000","last_became_peered":"2024-09-14T03:53:05.236225+0000","last_unstale":"2024-09-14T03:53:44.809671+0000","last_undegraded":"2024-09-14T03:53:44.809671+0000","last_fullsized":"2024-09-14T03:53:44.809671+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:44.710359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.807611+0000","last_change":"2024-09-14T03:53:06.412211+0000","last_active":"2024-09-14T03:53:44.807611+0000","last_peered":"2024-09-14T03:53:44.807611+0000","last_clean":"2024-09-14T03:53:44.807611+0000","last_became_active":"2024-09-14T03:53:06.411711+0000","last_became_peered":"2024-09-14T03:53:06.411711+0000","last_unstale":"2024-09-14T03:53:44.807611+0000","last_undegraded":"2024-09-14T03:53:44.807611+0000","last_fullsized":"2024-09-14T03:53:44.807611+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:30.693735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.807285+0000","last_change":"2024-09-14T03:53:05.237902+0000","last_active":"2024-09-14T03:53:44.807285+0000","last_peered":"2024-09-14T03:53:44.807285+0000","last_clean":"2024-09-14T03:53:44.807285+0000","last_became_active":"2024-09-14T03:53:05.237615+0000","last_became_peered":"2024-09-14T03:53:05.237615+0000","last_unstale":"2024-09-14T03:53:44.807285+0000","last_undegraded":"2024-09-14T03:53:44.807285+0000","last_fullsized":"2024-09-14T03:53:44.807285+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:32:53.206648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.807784+0000","last_change":"2024-09-14T03:53:06.412288+0000","last_active":"2024-09-14T03:53:44.807784+0000","last_peered":"2024-09-14T03:53:44.807784+0000","last_clean":"2024-09-14T03:53:44.807784+0000","last_became_active":"2024-09-14T03:53:06.411577+0000","last_became_peered":"2024-09-14T03:53:06.411577+0000","last_unstale":"2024-09-14T03:53:44.807784+0000","last_undegraded":"2024-09-14T03:53:44.807784+0000","last_fullsized":"2024-09-14T03:53:44.807784+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:20:37.678658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.812677+0000","last_change":"2024-09-14T03:53:05.236651+0000","last_active":"2024-09-14T03:53:44.812677+0000","last_peered":"2024-09-14T03:53:44.812677+0000","last_clean":"2024-09-14T03:53:44.812677+0000","last_became_active":"2024-09-14T03:53:05.235902+0000","last_became_peered":"2024-09-14T03:53:05.235902+0000","last_unstale":"2024-09-14T03:53:44.812677+0000","last_undegraded":"2024-09-14T03:53:44.812677+0000","last_fullsized":"2024-09-14T03:53:44.812677+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:36:37.779052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.810557+0000","last_change":"2024-09-14T03:53:05.369591+0000","last_active":"2024-09-14T03:53:44.810557+0000","last_peered":"2024-09-14T03:53:44.810557+0000","last_clean":"2024-09-14T03:53:44.810557+0000","last_became_active":"2024-09-14T03:53:05.369232+0000","last_became_peered":"2024-09-14T03:53:05.369232+0000","last_unstale":"2024-09-14T03:53:44.810557+0000","last_undegraded":"2024-09-14T03:53:44.810557+0000","last_fullsized":"2024-09-14T03:53:44.810557+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:03:16.262268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"61'128","reported_seq":270,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055457+0000","last_change":"2024-09-14T03:51:16.669763+0000","last_active":"2024-09-14T03:53:45.055457+0000","last_peered":"2024-09-14T03:53:45.055457+0000","last_clean":"2024-09-14T03:53:45.055457+0000","last_became_active":"2024-09-14T03:51:16.669377+0000","last_became_peered":"2024-09-14T03:51:16.669377+0000","last_unstale":"2024-09-14T03:53:45.055457+0000","last_undegraded":"2024-09-14T03:53:45.055457+0000","last_fullsized":"2024-09-14T03:53:45.055457+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T03:49:18.237706+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:10:34.161809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.835463+0000","last_change":"2024-09-14T03:51:52.316449+0000","last_active":"2024-09-14T03:53:45.835463+0000","last_peered":"2024-09-14T03:53:45.835463+0000","last_clean":"2024-09-14T03:53:45.835463+0000","last_became_active":"2024-09-14T03:51:46.806879+0000","last_became_peered":"2024-09-14T03:51:46.806879+0000","last_unstale":"2024-09-14T03:53:45.835463+0000","last_undegraded":"2024-09-14T03:53:45.835463+0000","last_fullsized":"2024-09-14T03:53:45.835463+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:37:54.991090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.811513+0000","last_change":"2024-09-14T03:53:05.242652+0000","last_active":"2024-09-14T03:53:44.811513+0000","last_peered":"2024-09-14T03:53:44.811513+0000","last_clean":"2024-09-14T03:53:44.811513+0000","last_became_active":"2024-09-14T03:53:05.242362+0000","last_became_peered":"2024-09-14T03:53:05.242362+0000","last_unstale":"2024-09-14T03:53:44.811513+0000","last_undegraded":"2024-09-14T03:53:44.811513+0000","last_fullsized":"2024-09-14T03:53:44.811513+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:17:25.158508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.809479+0000","last_change":"2024-09-14T03:53:05.551268+0000","last_active":"2024-09-14T03:53:44.809479+0000","last_peered":"2024-09-14T03:53:44.809479+0000","last_clean":"2024-09-14T03:53:44.809479+0000","last_became_active":"2024-09-14T03:53:05.551015+0000","last_became_peered":"2024-09-14T03:53:05.551015+0000","last_unstale":"2024-09-14T03:53:44.809479+0000","last_undegraded":"2024-09-14T03:53:44.809479+0000","last_fullsized":"2024-09-14T03:53:44.809479+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.905475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:53:46.105252+0000","last_change":"2024-09-14T03:51:52.734251+0000","last_active":"2024-09-14T03:53:46.105252+0000","last_peered":"2024-09-14T03:53:46.105252+0000","last_clean":"2024-09-14T03:53:46.105252+0000","last_became_active":"2024-09-14T03:51:46.834389+0000","last_became_peered":"2024-09-14T03:51:46.834389+0000","last_unstale":"2024-09-14T03:53:46.105252+0000","last_undegraded":"2024-09-14T03:53:46.105252+0000","last_fullsized":"2024-09-14T03:53:46.105252+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:12:21.741822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055245+0000","last_change":"2024-09-14T03:53:05.243315+0000","last_active":"2024-09-14T03:53:45.055245+0000","last_peered":"2024-09-14T03:53:45.055245+0000","last_clean":"2024-09-14T03:53:45.055245+0000","last_became_active":"2024-09-14T03:53:05.243036+0000","last_became_peered":"2024-09-14T03:53:05.243036+0000","last_unstale":"2024-09-14T03:53:45.055245+0000","last_undegraded":"2024-09-14T03:53:45.055245+0000","last_fullsized":"2024-09-14T03:53:45.055245+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:01:10.404454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055533+0000","last_change":"2024-09-14T03:53:05.461556+0000","last_active":"2024-09-14T03:53:45.055533+0000","last_peered":"2024-09-14T03:53:45.055533+0000","last_clean":"2024-09-14T03:53:45.055533+0000","last_became_active":"2024-09-14T03:53:05.460764+0000","last_became_peered":"2024-09-14T03:53:05.460764+0000","last_unstale":"2024-09-14T03:53:45.055533+0000","last_undegraded":"2024-09-14T03:53:45.055533+0000","last_fullsized":"2024-09-14T03:53:45.055533+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:50:51.579075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:53:46.105816+0000","last_change":"2024-09-14T03:51:52.859768+0000","last_active":"2024-09-14T03:53:46.105816+0000","last_peered":"2024-09-14T03:53:46.105816+0000","last_clean":"2024-09-14T03:53:46.105816+0000","last_became_active":"2024-09-14T03:51:46.835278+0000","last_became_peered":"2024-09-14T03:51:46.835278+0000","last_unstale":"2024-09-14T03:53:46.105816+0000","last_undegraded":"2024-09-14T03:53:46.105816+0000","last_fullsized":"2024-09-14T03:53:46.105816+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:25.102148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.810580+0000","last_change":"2024-09-14T03:53:05.234040+0000","last_active":"2024-09-14T03:53:44.810580+0000","last_peered":"2024-09-14T03:53:44.810580+0000","last_clean":"2024-09-14T03:53:44.810580+0000","last_became_active":"2024-09-14T03:53:05.233722+0000","last_became_peered":"2024-09-14T03:53:05.233722+0000","last_unstale":"2024-09-14T03:53:44.810580+0000","last_undegraded":"2024-09-14T03:53:44.810580+0000","last_fullsized":"2024-09-14T03:53:44.810580+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:42:14.192664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.810638+0000","last_change":"2024-09-14T03:53:05.252316+0000","last_active":"2024-09-14T03:53:44.810638+0000","last_peered":"2024-09-14T03:53:44.810638+0000","last_clean":"2024-09-14T03:53:44.810638+0000","last_became_active":"2024-09-14T03:53:05.252009+0000","last_became_peered":"2024-09-14T03:53:05.252009+0000","last_unstale":"2024-09-14T03:53:44.810638+0000","last_undegraded":"2024-09-14T03:53:44.810638+0000","last_fullsized":"2024-09-14T03:53:44.810638+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:00:31.537605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":29,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.835425+0000","last_change":"2024-09-14T03:51:52.316945+0000","last_active":"2024-09-14T03:53:45.835425+0000","last_peered":"2024-09-14T03:53:45.835425+0000","last_clean":"2024-09-14T03:53:45.835425+0000","last_became_active":"2024-09-14T03:51:46.806040+0000","last_became_peered":"2024-09-14T03:51:46.806040+0000","last_unstale":"2024-09-14T03:53:45.835425+0000","last_undegraded":"2024-09-14T03:53:45.835425+0000","last_fullsized":"2024-09-14T03:53:45.835425+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:28:20.339010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.807517+0000","last_change":"2024-09-14T03:53:05.242366+0000","last_active":"2024-09-14T03:53:44.807517+0000","last_peered":"2024-09-14T03:53:44.807517+0000","last_clean":"2024-09-14T03:53:44.807517+0000","last_became_active":"2024-09-14T03:53:05.241689+0000","last_became_peered":"2024-09-14T03:53:05.241689+0000","last_unstale":"2024-09-14T03:53:44.807517+0000","last_undegraded":"2024-09-14T03:53:44.807517+0000","last_fullsized":"2024-09-14T03:53:44.807517+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:32:21.936986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.810822+0000","last_change":"2024-09-14T03:53:05.460904+0000","last_active":"2024-09-14T03:53:44.810822+0000","last_peered":"2024-09-14T03:53:44.810822+0000","last_clean":"2024-09-14T03:53:44.810822+0000","last_became_active":"2024-09-14T03:53:05.460645+0000","last_became_peered":"2024-09-14T03:53:05.460645+0000","last_unstale":"2024-09-14T03:53:44.810822+0000","last_undegraded":"2024-09-14T03:53:44.810822+0000","last_fullsized":"2024-09-14T03:53:44.810822+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:11:19.095675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.810960+0000","last_change":"2024-09-14T03:53:05.241606+0000","last_active":"2024-09-14T03:53:44.810960+0000","last_peered":"2024-09-14T03:53:44.810960+0000","last_clean":"2024-09-14T03:53:44.810960+0000","last_became_active":"2024-09-14T03:53:05.241310+0000","last_became_peered":"2024-09-14T03:53:05.241310+0000","last_unstale":"2024-09-14T03:53:44.810960+0000","last_undegraded":"2024-09-14T03:53:44.810960+0000","last_fullsized":"2024-09-14T03:53:44.810960+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:49:54.875110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.819679+0000","last_change":"2024-09-14T03:51:52.317915+0000","last_active":"2024-09-14T03:53:45.819679+0000","last_peered":"2024-09-14T03:53:45.819679+0000","last_clean":"2024-09-14T03:53:45.819679+0000","last_became_active":"2024-09-14T03:51:46.834577+0000","last_became_peered":"2024-09-14T03:51:46.834577+0000","last_unstale":"2024-09-14T03:53:45.819679+0000","last_undegraded":"2024-09-14T03:53:45.819679+0000","last_fullsized":"2024-09-14T03:53:45.819679+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:14:59.991639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.810150+0000","last_change":"2024-09-14T03:53:05.369605+0000","last_active":"2024-09-14T03:53:44.810150+0000","last_peered":"2024-09-14T03:53:44.810150+0000","last_clean":"2024-09-14T03:53:44.810150+0000","last_became_active":"2024-09-14T03:53:05.369242+0000","last_became_peered":"2024-09-14T03:53:05.369242+0000","last_unstale":"2024-09-14T03:53:44.810150+0000","last_undegraded":"2024-09-14T03:53:44.810150+0000","last_fullsized":"2024-09-14T03:53:44.810150+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:03:05.904256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.810979+0000","last_change":"2024-09-14T03:53:05.235449+0000","last_active":"2024-09-14T03:53:44.810979+0000","last_peered":"2024-09-14T03:53:44.810979+0000","last_clean":"2024-09-14T03:53:44.810979+0000","last_became_active":"2024-09-14T03:53:05.235108+0000","last_became_peered":"2024-09-14T03:53:05.235108+0000","last_unstale":"2024-09-14T03:53:44.810979+0000","last_undegraded":"2024-09-14T03:53:44.810979+0000","last_fullsized":"2024-09-14T03:53:44.810979+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:52:34.275278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:53:46.105682+0000","last_change":"2024-09-14T03:51:52.734322+0000","last_active":"2024-09-14T03:53:46.105682+0000","last_peered":"2024-09-14T03:53:46.105682+0000","last_clean":"2024-09-14T03:53:46.105682+0000","last_became_active":"2024-09-14T03:51:46.835000+0000","last_became_peered":"2024-09-14T03:51:46.835000+0000","last_unstale":"2024-09-14T03:53:46.105682+0000","last_undegraded":"2024-09-14T03:53:46.105682+0000","last_fullsized":"2024-09-14T03:53:46.105682+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:46:41.873402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055394+0000","last_change":"2024-09-14T03:53:05.550002+0000","last_active":"2024-09-14T03:53:45.055394+0000","last_peered":"2024-09-14T03:53:45.055394+0000","last_clean":"2024-09-14T03:53:45.055394+0000","last_became_active":"2024-09-14T03:53:05.549782+0000","last_became_peered":"2024-09-14T03:53:05.549782+0000","last_unstale":"2024-09-14T03:53:45.055394+0000","last_undegraded":"2024-09-14T03:53:45.055394+0000","last_fullsized":"2024-09-14T03:53:45.055394+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:50:50.053863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.812173+0000","last_change":"2024-09-14T03:53:05.239088+0000","last_active":"2024-09-14T03:53:44.812173+0000","last_peered":"2024-09-14T03:53:44.812173+0000","last_clean":"2024-09-14T03:53:44.812173+0000","last_became_active":"2024-09-14T03:53:05.238773+0000","last_became_peered":"2024-09-14T03:53:05.238773+0000","last_unstale":"2024-09-14T03:53:44.812173+0000","last_undegraded":"2024-09-14T03:53:44.812173+0000","last_fullsized":"2024-09-14T03:53:44.812173+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:21:45.491829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.819903+0000","last_change":"2024-09-14T03:51:52.317867+0000","last_active":"2024-09-14T03:53:45.819903+0000","last_peered":"2024-09-14T03:53:45.819903+0000","last_clean":"2024-09-14T03:53:45.819903+0000","last_became_active":"2024-09-14T03:51:46.834009+0000","last_became_peered":"2024-09-14T03:51:46.834009+0000","last_unstale":"2024-09-14T03:53:45.819903+0000","last_undegraded":"2024-09-14T03:53:45.819903+0000","last_fullsized":"2024-09-14T03:53:45.819903+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:36:33.505359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028771999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.812489+0000","last_change":"2024-09-14T03:53:06.250859+0000","last_active":"2024-09-14T03:53:44.812489+0000","last_peered":"2024-09-14T03:53:44.812489+0000","last_clean":"2024-09-14T03:53:44.812489+0000","last_became_active":"2024-09-14T03:53:06.250581+0000","last_became_peered":"2024-09-14T03:53:06.250581+0000","last_unstale":"2024-09-14T03:53:44.812489+0000","last_undegraded":"2024-09-14T03:53:44.812489+0000","last_fullsized":"2024-09-14T03:53:44.812489+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:17:15.683212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055639+0000","last_change":"2024-09-14T03:53:05.240359+0000","last_active":"2024-09-14T03:53:45.055639+0000","last_peered":"2024-09-14T03:53:45.055639+0000","last_clean":"2024-09-14T03:53:45.055639+0000","last_became_active":"2024-09-14T03:53:05.240068+0000","last_became_peered":"2024-09-14T03:53:05.240068+0000","last_unstale":"2024-09-14T03:53:45.055639+0000","last_undegraded":"2024-09-14T03:53:45.055639+0000","last_fullsized":"2024-09-14T03:53:45.055639+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:55:16.343381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.812671+0000","last_change":"2024-09-14T03:51:52.317818+0000","last_active":"2024-09-14T03:53:45.812671+0000","last_peered":"2024-09-14T03:53:45.812671+0000","last_clean":"2024-09-14T03:53:45.812671+0000","last_became_active":"2024-09-14T03:51:46.806805+0000","last_became_peered":"2024-09-14T03:51:46.806805+0000","last_unstale":"2024-09-14T03:53:45.812671+0000","last_undegraded":"2024-09-14T03:53:45.812671+0000","last_fullsized":"2024-09-14T03:53:45.812671+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:01:38.862572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.807046+0000","last_change":"2024-09-14T03:53:06.634043+0000","last_active":"2024-09-14T03:53:44.807046+0000","last_peered":"2024-09-14T03:53:44.807046+0000","last_clean":"2024-09-14T03:53:44.807046+0000","last_became_active":"2024-09-14T03:53:06.633638+0000","last_became_peered":"2024-09-14T03:53:06.633638+0000","last_unstale":"2024-09-14T03:53:44.807046+0000","last_undegraded":"2024-09-14T03:53:44.807046+0000","last_fullsized":"2024-09-14T03:53:44.807046+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:15:53.434064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.807373+0000","last_change":"2024-09-14T03:53:06.412072+0000","last_active":"2024-09-14T03:53:44.807373+0000","last_peered":"2024-09-14T03:53:44.807373+0000","last_clean":"2024-09-14T03:53:44.807373+0000","last_became_active":"2024-09-14T03:53:06.411778+0000","last_became_peered":"2024-09-14T03:53:06.411778+0000","last_unstale":"2024-09-14T03:53:44.807373+0000","last_undegraded":"2024-09-14T03:53:44.807373+0000","last_fullsized":"2024-09-14T03:53:44.807373+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:41:38.193881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"72'6","reported_seq":21,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.807605+0000","last_change":"2024-09-14T03:53:05.244055+0000","last_active":"2024-09-14T03:53:44.807605+0000","last_peered":"2024-09-14T03:53:44.807605+0000","last_clean":"2024-09-14T03:53:44.807605+0000","last_became_active":"2024-09-14T03:53:05.243752+0000","last_became_peered":"2024-09-14T03:53:05.243752+0000","last_unstale":"2024-09-14T03:53:44.807605+0000","last_undegraded":"2024-09-14T03:53:44.807605+0000","last_fullsized":"2024-09-14T03:53:44.807605+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:12:31.850064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1564,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.811158+0000","last_change":"2024-09-14T03:53:05.460859+0000","last_active":"2024-09-14T03:53:44.811158+0000","last_peered":"2024-09-14T03:53:44.811158+0000","last_clean":"2024-09-14T03:53:44.811158+0000","last_became_active":"2024-09-14T03:53:05.460555+0000","last_became_peered":"2024-09-14T03:53:05.460555+0000","last_unstale":"2024-09-14T03:53:44.811158+0000","last_undegraded":"2024-09-14T03:53:44.811158+0000","last_fullsized":"2024-09-14T03:53:44.811158+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:10:54.185175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":18,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055552+0000","last_change":"2024-09-14T03:53:05.240418+0000","last_active":"2024-09-14T03:53:45.055552+0000","last_peered":"2024-09-14T03:53:45.055552+0000","last_clean":"2024-09-14T03:53:45.055552+0000","last_became_active":"2024-09-14T03:53:05.240140+0000","last_became_peered":"2024-09-14T03:53:05.240140+0000","last_unstale":"2024-09-14T03:53:45.055552+0000","last_undegraded":"2024-09-14T03:53:45.055552+0000","last_fullsized":"2024-09-14T03:53:45.055552+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:31:00.413012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.810766+0000","last_change":"2024-09-14T03:53:05.237203+0000","last_active":"2024-09-14T03:53:44.810766+0000","last_peered":"2024-09-14T03:53:44.810766+0000","last_clean":"2024-09-14T03:53:44.810766+0000","last_became_active":"2024-09-14T03:53:05.236893+0000","last_became_peered":"2024-09-14T03:53:05.236893+0000","last_unstale":"2024-09-14T03:53:44.810766+0000","last_undegraded":"2024-09-14T03:53:44.810766+0000","last_fullsized":"2024-09-14T03:53:44.810766+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:36:55.377845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.811235+0000","last_change":"2024-09-14T03:53:06.310444+0000","last_active":"2024-09-14T03:53:44.811235+0000","last_peered":"2024-09-14T03:53:44.811235+0000","last_clean":"2024-09-14T03:53:44.811235+0000","last_became_active":"2024-09-14T03:53:06.310166+0000","last_became_peered":"2024-09-14T03:53:06.310166+0000","last_unstale":"2024-09-14T03:53:44.811235+0000","last_undegraded":"2024-09-14T03:53:44.811235+0000","last_fullsized":"2024-09-14T03:53:44.811235+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:01:08.675783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055758+0000","last_change":"2024-09-14T03:53:05.237613+0000","last_active":"2024-09-14T03:53:45.055758+0000","last_peered":"2024-09-14T03:53:45.055758+0000","last_clean":"2024-09-14T03:53:45.055758+0000","last_became_active":"2024-09-14T03:53:05.237326+0000","last_became_peered":"2024-09-14T03:53:05.237326+0000","last_unstale":"2024-09-14T03:53:45.055758+0000","last_undegraded":"2024-09-14T03:53:45.055758+0000","last_fullsized":"2024-09-14T03:53:45.055758+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:54:08.959667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":23,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:53:46.106654+0000","last_change":"2024-09-14T03:53:46.106654+0000","last_active":"2024-09-14T03:53:46.106654+0000","last_peered":"2024-09-14T03:53:46.106654+0000","last_clean":"2024-09-14T03:53:46.106654+0000","last_became_active":"2024-09-14T03:53:46.106346+0000","last_became_peered":"2024-09-14T03:53:46.106346+0000","last_unstale":"2024-09-14T03:53:46.106654+0000","last_undegraded":"2024-09-14T03:53:46.106654+0000","last_fullsized":"2024-09-14T03:53:46.106654+0000","mapping_epoch":73,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":74,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:29:16.158851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":23,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:53:46.106894+0000","last_change":"2024-09-14T03:53:46.106894+0000","last_active":"2024-09-14T03:53:46.106894+0000","last_peered":"2024-09-14T03:53:46.106894+0000","last_clean":"2024-09-14T03:53:46.106894+0000","last_became_active":"2024-09-14T03:53:46.106495+0000","last_became_peered":"2024-09-14T03:53:46.106495+0000","last_unstale":"2024-09-14T03:53:46.106894+0000","last_undegraded":"2024-09-14T03:53:46.106894+0000","last_fullsized":"2024-09-14T03:53:46.106894+0000","mapping_epoch":73,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":74,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:43:20.627135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.812590+0000","last_change":"2024-09-14T03:53:05.236831+0000","last_active":"2024-09-14T03:53:44.812590+0000","last_peered":"2024-09-14T03:53:44.812590+0000","last_clean":"2024-09-14T03:53:44.812590+0000","last_became_active":"2024-09-14T03:53:05.236360+0000","last_became_peered":"2024-09-14T03:53:05.236360+0000","last_unstale":"2024-09-14T03:53:44.812590+0000","last_undegraded":"2024-09-14T03:53:44.812590+0000","last_fullsized":"2024-09-14T03:53:44.812590+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:21:42.355910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055513+0000","last_change":"2024-09-14T03:53:05.551090+0000","last_active":"2024-09-14T03:53:45.055513+0000","last_peered":"2024-09-14T03:53:45.055513+0000","last_clean":"2024-09-14T03:53:45.055513+0000","last_became_active":"2024-09-14T03:53:05.550765+0000","last_became_peered":"2024-09-14T03:53:05.550765+0000","last_unstale":"2024-09-14T03:53:45.055513+0000","last_undegraded":"2024-09-14T03:53:45.055513+0000","last_fullsized":"2024-09-14T03:53:45.055513+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:15:06.004332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.810872+0000","last_change":"2024-09-14T03:53:05.244106+0000","last_active":"2024-09-14T03:53:44.810872+0000","last_peered":"2024-09-14T03:53:44.810872+0000","last_clean":"2024-09-14T03:53:44.810872+0000","last_became_active":"2024-09-14T03:53:05.243512+0000","last_became_peered":"2024-09-14T03:53:05.243512+0000","last_unstale":"2024-09-14T03:53:44.810872+0000","last_undegraded":"2024-09-14T03:53:44.810872+0000","last_fullsized":"2024-09-14T03:53:44.810872+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:49:19.720662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.811763+0000","last_change":"2024-09-14T03:53:06.412762+0000","last_active":"2024-09-14T03:53:44.811763+0000","last_peered":"2024-09-14T03:53:44.811763+0000","last_clean":"2024-09-14T03:53:44.811763+0000","last_became_active":"2024-09-14T03:53:06.412502+0000","last_became_peered":"2024-09-14T03:53:06.412502+0000","last_unstale":"2024-09-14T03:53:44.811763+0000","last_undegraded":"2024-09-14T03:53:44.811763+0000","last_fullsized":"2024-09-14T03:53:44.811763+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:19.745802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.807446+0000","last_change":"2024-09-14T03:53:05.239221+0000","last_active":"2024-09-14T03:53:44.807446+0000","last_peered":"2024-09-14T03:53:44.807446+0000","last_clean":"2024-09-14T03:53:44.807446+0000","last_became_active":"2024-09-14T03:53:05.238936+0000","last_became_peered":"2024-09-14T03:53:05.238936+0000","last_unstale":"2024-09-14T03:53:44.807446+0000","last_undegraded":"2024-09-14T03:53:44.807446+0000","last_fullsized":"2024-09-14T03:53:44.807446+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:14:04.731925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.812913+0000","last_change":"2024-09-14T03:53:06.412049+0000","last_active":"2024-09-14T03:53:44.812913+0000","last_peered":"2024-09-14T03:53:44.812913+0000","last_clean":"2024-09-14T03:53:44.812913+0000","last_became_active":"2024-09-14T03:53:06.411716+0000","last_became_peered":"2024-09-14T03:53:06.411716+0000","last_unstale":"2024-09-14T03:53:44.812913+0000","last_undegraded":"2024-09-14T03:53:44.812913+0000","last_fullsized":"2024-09-14T03:53:44.812913+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:09:57.721410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055479+0000","last_change":"2024-09-14T03:53:05.240514+0000","last_active":"2024-09-14T03:53:45.055479+0000","last_peered":"2024-09-14T03:53:45.055479+0000","last_clean":"2024-09-14T03:53:45.055479+0000","last_became_active":"2024-09-14T03:53:05.240213+0000","last_became_peered":"2024-09-14T03:53:05.240213+0000","last_unstale":"2024-09-14T03:53:45.055479+0000","last_undegraded":"2024-09-14T03:53:45.055479+0000","last_fullsized":"2024-09-14T03:53:45.055479+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:25:50.704382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.807264+0000","last_change":"2024-09-14T03:53:06.412040+0000","last_active":"2024-09-14T03:53:44.807264+0000","last_peered":"2024-09-14T03:53:44.807264+0000","last_clean":"2024-09-14T03:53:44.807264+0000","last_became_active":"2024-09-14T03:53:06.411769+0000","last_became_peered":"2024-09-14T03:53:06.411769+0000","last_unstale":"2024-09-14T03:53:44.807264+0000","last_undegraded":"2024-09-14T03:53:44.807264+0000","last_fullsized":"2024-09-14T03:53:44.807264+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:09:05.162369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.810579+0000","last_change":"2024-09-14T03:53:05.235171+0000","last_active":"2024-09-14T03:53:44.810579+0000","last_peered":"2024-09-14T03:53:44.810579+0000","last_clean":"2024-09-14T03:53:44.810579+0000","last_became_active":"2024-09-14T03:53:05.234895+0000","last_became_peered":"2024-09-14T03:53:05.234895+0000","last_unstale":"2024-09-14T03:53:44.810579+0000","last_undegraded":"2024-09-14T03:53:44.810579+0000","last_fullsized":"2024-09-14T03:53:44.810579+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:07:16.459146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.807259+0000","last_change":"2024-09-14T03:53:06.250614+0000","last_active":"2024-09-14T03:53:44.807259+0000","last_peered":"2024-09-14T03:53:44.807259+0000","last_clean":"2024-09-14T03:53:44.807259+0000","last_became_active":"2024-09-14T03:53:06.250371+0000","last_became_peered":"2024-09-14T03:53:06.250371+0000","last_unstale":"2024-09-14T03:53:44.807259+0000","last_undegraded":"2024-09-14T03:53:44.807259+0000","last_fullsized":"2024-09-14T03:53:44.807259+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:43:52.171802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.810858+0000","last_change":"2024-09-14T03:53:05.234444+0000","last_active":"2024-09-14T03:53:44.810858+0000","last_peered":"2024-09-14T03:53:44.810858+0000","last_clean":"2024-09-14T03:53:44.810858+0000","last_became_active":"2024-09-14T03:53:05.234115+0000","last_became_peered":"2024-09-14T03:53:05.234115+0000","last_unstale":"2024-09-14T03:53:44.810858+0000","last_undegraded":"2024-09-14T03:53:44.810858+0000","last_fullsized":"2024-09-14T03:53:44.810858+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:01:23.153937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.811306+0000","last_change":"2024-09-14T03:53:06.251023+0000","last_active":"2024-09-14T03:53:44.811306+0000","last_peered":"2024-09-14T03:53:44.811306+0000","last_clean":"2024-09-14T03:53:44.811306+0000","last_became_active":"2024-09-14T03:53:06.250566+0000","last_became_peered":"2024-09-14T03:53:06.250566+0000","last_unstale":"2024-09-14T03:53:44.811306+0000","last_undegraded":"2024-09-14T03:53:44.811306+0000","last_fullsized":"2024-09-14T03:53:44.811306+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:54:39.448363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.811913+0000","last_change":"2024-09-14T03:53:05.237437+0000","last_active":"2024-09-14T03:53:44.811913+0000","last_peered":"2024-09-14T03:53:44.811913+0000","last_clean":"2024-09-14T03:53:44.811913+0000","last_became_active":"2024-09-14T03:53:05.237151+0000","last_became_peered":"2024-09-14T03:53:05.237151+0000","last_unstale":"2024-09-14T03:53:44.811913+0000","last_undegraded":"2024-09-14T03:53:44.811913+0000","last_fullsized":"2024-09-14T03:53:44.811913+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:57:21.916179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":23,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.836449+0000","last_change":"2024-09-14T03:53:45.836449+0000","last_active":"2024-09-14T03:53:45.836449+0000","last_peered":"2024-09-14T03:53:45.836449+0000","last_clean":"2024-09-14T03:53:45.836449+0000","last_became_active":"2024-09-14T03:53:45.836108+0000","last_became_peered":"2024-09-14T03:53:45.836108+0000","last_unstale":"2024-09-14T03:53:45.836449+0000","last_undegraded":"2024-09-14T03:53:45.836449+0000","last_fullsized":"2024-09-14T03:53:45.836449+0000","mapping_epoch":73,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":74,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:52:13.095225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":19,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055693+0000","last_change":"2024-09-14T03:53:05.236307+0000","last_active":"2024-09-14T03:53:45.055693+0000","last_peered":"2024-09-14T03:53:45.055693+0000","last_clean":"2024-09-14T03:53:45.055693+0000","last_became_active":"2024-09-14T03:53:05.236014+0000","last_became_peered":"2024-09-14T03:53:05.236014+0000","last_unstale":"2024-09-14T03:53:45.055693+0000","last_undegraded":"2024-09-14T03:53:45.055693+0000","last_fullsized":"2024-09-14T03:53:45.055693+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:02.092228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2362,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201318,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28380,"kb_used_data":1356,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716772,"statfs":{"total":95995035648,"available":95965974528,"internally_reserved":0,"allocated":1388544,"data_stored":928583,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:53:21 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.54500000000000004,"15min":0.54800000000000004},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.78400000000000003,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.53300000000000003,"15min":0.52800000000000002},"min":{"1min":0.41099999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.73699999999999999,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.69899999999999995}]},{"osd":1,"last update":"Sat Sep 14 03:53:21 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53900000000000003,"15min":0.54000000000000004},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.70099999999999996,"5min":0.748,"15min":0.748},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.55800000000000005,"15min":0.55600000000000005},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.68899999999999995,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.67100000000000004}]},{"osd":2,"last update":"Sat Sep 14 03:53:21 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.54400000000000004,"15min":0.54200000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.76300000000000001,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.63},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.56799999999999995,"15min":0.56999999999999995},"min":{"1min":0.40400000000000003,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.71199999999999997,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.621}]},{"osd":3,"last update":"Sat Sep 14 03:53:21 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.52600000000000002,"15min":0.52200000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.71099999999999997,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.755,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.65300000000000002}]},{"osd":4,"last update":"Sat Sep 14 03:53:21 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.47599999999999998,"15min":0.48099999999999998},"min":{"1min":0.308,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.67300000000000004,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.47599999999999998,"15min":0.47699999999999998},"min":{"1min":0.30499999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.59899999999999998,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.43099999999999999}]},{"osd":5,"last update":"Sat Sep 14 03:53:21 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.49199999999999999,"15min":0.496},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.61599999999999999,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.498},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.53300000000000003,"15min":0.53700000000000003},"min":{"1min":0.35199999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.69699999999999995,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.60999999999999999}]},{"osd":6,"last update":"Sat Sep 14 03:53:21 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.34999999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.752,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.51900000000000002,"15min":0.51800000000000002},"min":{"1min":0.32600000000000001,"5min":0.311,"15min":0.311},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.53900000000000003}]}]},{"osd":6,"up_from":47,"seq":201863462954,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28372,"kb_used_data":1348,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716780,"statfs":{"total":95995035648,"available":95965982720,"internally_reserved":0,"allocated":1380352,"data_stored":928456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.53600000000000003,"15min":0.53900000000000003},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.75,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.51800000000000002,"15min":0.51600000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.70399999999999996,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.66400000000000003}]},{"osd":1,"last update":"Sat Sep 14 03:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53200000000000003,"15min":0.53400000000000003},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.75800000000000001,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.51100000000000001,"15min":0.51000000000000001},"min":{"1min":0.39800000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.67000000000000004,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.53100000000000003}]},{"osd":2,"last update":"Sat Sep 14 03:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.52400000000000002,"15min":0.51800000000000002},"min":{"1min":0.39700000000000002,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.745,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.52900000000000003,"15min":0.53600000000000003},"min":{"1min":0.42599999999999999,"5min":0.378,"15min":0.378},"max":{"1min":0.61399999999999999,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.51700000000000002}]},{"osd":3,"last update":"Sat Sep 14 03:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.52300000000000002,"15min":0.51100000000000001},"min":{"1min":0.378,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.64500000000000002,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.54900000000000004,"15min":0.55600000000000005},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.69399999999999995,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.70899999999999996}]},{"osd":4,"last update":"Sat Sep 14 03:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.40500000000000003,"15min":0.38700000000000001},"min":{"1min":0.30299999999999999,"5min":0.25,"15min":0.25},"max":{"1min":0.64100000000000001,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.436,"15min":0.41799999999999998},"min":{"1min":0.318,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.68200000000000005,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.61799999999999999}]},{"osd":5,"last update":"Sat Sep 14 03:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.41799999999999998,"15min":0.39800000000000002},"min":{"1min":0.29499999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.622,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.44,"15min":0.40999999999999998},"min":{"1min":0.33200000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.70899999999999996,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.69099999999999995}]},{"osd":7,"last update":"Sat Sep 14 03:53:19 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.47699999999999998,"15min":0.47499999999999998},"min":{"1min":0.34599999999999997,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.67700000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.505,"15min":0.501},"min":{"1min":0.34300000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.65200000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724591,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27780,"kb_used_data":756,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717372,"statfs":{"total":95995035648,"available":95966588928,"internally_reserved":0,"allocated":774144,"data_stored":336775,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:53:33 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.55200000000000005,"15min":0.56000000000000005},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.71499999999999997,"5min":0.75,"15min":0.75},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.54700000000000004,"15min":0.54500000000000004},"min":{"1min":0.36299999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.82299999999999995,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.65400000000000003}]},{"osd":1,"last update":"Sat Sep 14 03:53:33 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.57099999999999995,"15min":0.57499999999999996},"min":{"1min":0.40200000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.76200000000000001,"5min":0.874,"15min":0.874},"last":0.625},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.55100000000000005,"15min":0.56000000000000005},"min":{"1min":0.377,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.66200000000000003,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.59999999999999998}]},{"osd":2,"last update":"Sat Sep 14 03:53:33 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.54800000000000004,"15min":0.54900000000000004},"min":{"1min":0.439,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.54900000000000004,"15min":0.55300000000000005},"min":{"1min":0.40899999999999997,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.68700000000000006,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.56899999999999995}]},{"osd":3,"last update":"Sat Sep 14 03:53:33 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.54500000000000004,"15min":0.55300000000000005},"min":{"1min":0.378,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57199999999999995,"15min":0.57299999999999995},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.52600000000000002}]},{"osd":4,"last update":"Sat Sep 14 03:53:33 2024","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.40000000000000002,"15min":0.376},"min":{"1min":0.316,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.40799999999999997,"15min":0.374},"min":{"1min":0.34399999999999997,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.58199999999999996}]},{"osd":6,"last update":"Sat Sep 14 03:53:53 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.438,"15min":0.40999999999999998},"min":{"1min":0.33200000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.439},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.45000000000000001,"15min":0.42999999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.69899999999999995,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.55500000000000005}]},{"osd":7,"last update":"Sat Sep 14 03:53:18 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.45300000000000001,"15min":0.443},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.51300000000000001,"15min":0.50600000000000001},"min":{"1min":0.38400000000000001,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.48299999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986229,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27796,"kb_used_data":772,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717356,"statfs":{"total":95995035648,"available":95966572544,"internally_reserved":0,"allocated":790528,"data_stored":338145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:53:11 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.56299999999999994,"15min":0.58099999999999996},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.70799999999999996,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.56299999999999994,"15min":0.57099999999999995},"min":{"1min":0.40400000000000003,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.45900000000000002}]},{"osd":1,"last update":"Sat Sep 14 03:53:11 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.57399999999999995,"15min":0.58899999999999997},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.78800000000000003,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.53800000000000003,"15min":0.55400000000000005},"min":{"1min":0.34200000000000003,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.80100000000000005,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.39900000000000002}]},{"osd":2,"last update":"Sat Sep 14 03:53:11 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.56699999999999995,"15min":0.58399999999999996},"min":{"1min":0.38500000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.76300000000000001,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.55400000000000005,"15min":0.57199999999999995},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.70399999999999996,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.41299999999999998}]},{"osd":3,"last update":"Sat Sep 14 03:53:11 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.57699999999999996,"15min":0.59499999999999997},"min":{"1min":0.39300000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.78700000000000003,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.57299999999999995,"15min":0.58099999999999996},"min":{"1min":0.442,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.78900000000000003,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.505}]},{"osd":5,"last update":"Sat Sep 14 03:53:37 2024","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.42599999999999999,"15min":0.40600000000000003},"min":{"1min":0.28899999999999998,"5min":0.246,"15min":0.246},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.45900000000000002,"15min":0.439},"min":{"1min":0.35199999999999998,"5min":0.316,"15min":0.316},"max":{"1min":0.629,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.49099999999999999}]},{"osd":6,"last update":"Sat Sep 14 03:53:59 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.42899999999999999,"15min":0.42399999999999999},"min":{"1min":0.30599999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.63300000000000001,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.438},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.42599999999999999,"15min":0.42199999999999999},"min":{"1min":0.34899999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.47099999999999997}]},{"osd":7,"last update":"Sat Sep 14 03:53:16 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.503,"15min":0.504},"min":{"1min":0.317,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.67600000000000005,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.50600000000000001,"15min":0.505},"min":{"1min":0.36699999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.68500000000000005,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.63700000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149753,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27776,"kb_used_data":752,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717376,"statfs":{"total":95995035648,"available":95966593024,"internally_reserved":0,"allocated":770048,"data_stored":336525,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:53:46 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.38800000000000001,"15min":0.40000000000000002},"min":{"1min":0.20000000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.313},{"interface":"front","average":{"1min":0.33700000000000002,"5min":0.39500000000000002,"15min":0.41099999999999998},"min":{"1min":0.22900000000000001,"5min":0.221,"15min":0.221},"max":{"1min":0.51500000000000001,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.27500000000000002}]},{"osd":1,"last update":"Sat Sep 14 03:53:46 2024","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.40999999999999998,"15min":0.42799999999999999},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.49199999999999999,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.41299999999999998,"15min":0.43099999999999999},"min":{"1min":0.26100000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.41499999999999998}]},{"osd":2,"last update":"Sat Sep 14 03:53:46 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.44800000000000001,"15min":0.46200000000000002},"min":{"1min":0.29999999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.376},{"interface":"front","average":{"1min":0.378,"5min":0.42899999999999999,"15min":0.44800000000000001},"min":{"1min":0.26000000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.35599999999999998}]},{"osd":4,"last update":"Sat Sep 14 03:53:06 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.55900000000000005,"15min":0.55200000000000005},"min":{"1min":0.40100000000000002,"5min":0.378,"15min":0.378},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.626},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.55700000000000005,"15min":0.55400000000000005},"min":{"1min":0.35799999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.76500000000000001,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.51700000000000002}]},{"osd":5,"last update":"Sat Sep 14 03:53:30 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.57899999999999996,"15min":0.57799999999999996},"min":{"1min":0.372,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.70499999999999996,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.79300000000000004,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.58299999999999996}]},{"osd":6,"last update":"Sat Sep 14 03:53:56 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.58799999999999997,"15min":0.59399999999999997},"min":{"1min":0.40699999999999997,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.73899999999999999,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.60499999999999998,"15min":0.60799999999999998},"min":{"1min":0.41599999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.80500000000000005,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.48299999999999998}]},{"osd":7,"last update":"Sat Sep 14 03:53:19 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.60799999999999998,"15min":0.61799999999999999},"min":{"1min":0.39400000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.71399999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.622,"15min":0.628},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.79000000000000004,"5min":1.016,"15min":1.016},"last":0.65500000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411390,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27780,"kb_used_data":756,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717372,"statfs":{"total":95995035648,"available":95966588928,"internally_reserved":0,"allocated":774144,"data_stored":337051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:53:24 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.39200000000000002,"15min":0.376},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.38900000000000001,"15min":0.39100000000000001},"min":{"1min":0.23200000000000001,"5min":0.22,"15min":0.22},"max":{"1min":0.53100000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.34399999999999997}]},{"osd":1,"last update":"Sat Sep 14 03:53:24 2024","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.39700000000000002,"15min":0.38600000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42399999999999999,"15min":0.40200000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.309}]},{"osd":3,"last update":"Sat Sep 14 03:53:46 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.439,"15min":0.44900000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.376},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.47199999999999998,"15min":0.48199999999999998},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.39900000000000002}]},{"osd":4,"last update":"Sat Sep 14 03:53:07 2024","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.79200000000000004,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.60899999999999999,"15min":0.61099999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.81100000000000005,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.59899999999999998}]},{"osd":5,"last update":"Sat Sep 14 03:53:32 2024","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.625,"15min":0.621},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.623,"5min":0.61299999999999999,"15min":0.60999999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.82199999999999995,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.627}]},{"osd":6,"last update":"Sat Sep 14 03:53:53 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.61299999999999999,"15min":0.61099999999999999},"min":{"1min":0.376,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.75700000000000001,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.621,"15min":0.61599999999999999},"min":{"1min":0.42699999999999999,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.83999999999999997,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.54000000000000004}]},{"osd":7,"last update":"Sat Sep 14 03:53:21 2024","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.63100000000000001,"15min":0.628},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.81999999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.64900000000000002,"15min":0.64700000000000002},"min":{"1min":0.42599999999999999,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.79700000000000004,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.58199999999999996}]}]},{"osd":1,"up_from":13,"seq":55834574915,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27772,"kb_used_data":748,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717380,"statfs":{"total":95995035648,"available":95966597120,"internally_reserved":0,"allocated":765952,"data_stored":336181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:54:03 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.35799999999999998,"15min":0.312},"min":{"1min":0.17100000000000001,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":0.95899999999999996,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.38800000000000001,"15min":0.33700000000000002},"min":{"1min":0.23499999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":1.708,"5min":1.708,"15min":1.708},"last":0.27600000000000002}]},{"osd":2,"last update":"Sat Sep 14 03:53:20 2024","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.38900000000000001,"15min":0.372},"min":{"1min":0.26200000000000001,"5min":0.247,"15min":0.247},"max":{"1min":0.65200000000000002,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.379},{"interface":"front","average":{"1min":0.374,"5min":0.39500000000000002,"15min":0.38100000000000001},"min":{"1min":0.26900000000000002,"5min":0.249,"15min":0.249},"max":{"1min":0.60599999999999998,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.311}]},{"osd":3,"last update":"Sat Sep 14 03:53:48 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.45100000000000001,"15min":0.44900000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.7769999999999999,"5min":1.7769999999999999,"15min":1.7769999999999999},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.45300000000000001,"15min":0.45500000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.0329999999999999,"5min":1.0329999999999999,"15min":1.0329999999999999},"last":0.47499999999999998}]},{"osd":4,"last update":"Sat Sep 14 03:53:09 2024","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.55400000000000005,"15min":0.55200000000000005},"min":{"1min":0.374,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.45300000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.86599999999999999,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.61799999999999999}]},{"osd":5,"last update":"Sat Sep 14 03:53:33 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.56899999999999995,"15min":0.56100000000000005},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.589,"5min":1.589,"15min":1.589},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.61199999999999999,"15min":0.60799999999999998},"min":{"1min":0.41599999999999998,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.8899999999999999,"5min":1.8899999999999999,"15min":1.8899999999999999},"last":0.52200000000000002}]},{"osd":6,"last update":"Sat Sep 14 03:53:55 2024","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.59999999999999998,"15min":0.58599999999999997},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.4450000000000001,"5min":1.4450000000000001,"15min":1.4450000000000001},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.59599999999999997,"15min":0.58599999999999997},"min":{"1min":0.42399999999999999,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.645,"5min":1.645,"15min":1.645},"last":0.53700000000000003}]},{"osd":7,"last update":"Sat Sep 14 03:53:16 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.56999999999999995,"15min":0.55900000000000005},"min":{"1min":0.41299999999999998,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.84699999999999998,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.622,"5min":0.60899999999999999,"15min":0.60699999999999998},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.84699999999999998,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.68899999999999995}]}]},{"osd":0,"up_from":8,"seq":34359738439,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28360,"kb_used_data":1336,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716792,"statfs":{"total":95995035648,"available":95965995008,"internally_reserved":0,"allocated":1368064,"data_stored":927407,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 14 03:54:00 2024","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":0.34699999999999998,"15min":0.32600000000000001},"min":{"1min":0.28000000000000003,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.51000000000000001,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.377,"5min":0.379,"15min":0.33500000000000002},"min":{"1min":0.27600000000000002,"5min":0.16800000000000001,"15min":0.16800000000000001},"max":{"1min":0.50900000000000001,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.39000000000000001}]},{"osd":2,"last update":"Sat Sep 14 03:53:25 2024","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.40799999999999997,"15min":0.40000000000000002},"min":{"1min":0.26800000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.54700000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.378,"5min":0.377,"15min":0.372},"min":{"1min":0.27500000000000002,"5min":0.214,"15min":0.214},"max":{"1min":0.60299999999999998,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.35399999999999998}]},{"osd":3,"last update":"Sat Sep 14 03:53:49 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.438,"15min":0.442},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.50600000000000001,"5min":0.622,"15min":0.622},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.434,"15min":0.44700000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.51400000000000001,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.32600000000000001}]},{"osd":4,"last update":"Sat Sep 14 03:53:07 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.55900000000000005,"15min":0.55300000000000005},"min":{"1min":0.38200000000000001,"5min":0.378,"15min":0.378},"max":{"1min":0.81499999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.749},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.53100000000000003,"15min":0.52500000000000002},"min":{"1min":0.40000000000000002,"5min":0.38,"15min":0.38},"max":{"1min":0.76800000000000002,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.64200000000000002}]},{"osd":5,"last update":"Sat Sep 14 03:53:34 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.59899999999999998,"15min":0.60599999999999998},"min":{"1min":0.43099999999999999,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.752,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.59099999999999997,"15min":0.59999999999999998},"min":{"1min":0.40799999999999997,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.71399999999999997,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.69699999999999995}]},{"osd":6,"last update":"Sat Sep 14 03:53:55 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.47599999999999998,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.80600000000000005,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.624},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.621,"15min":0.627},"min":{"1min":0.53000000000000003,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.71599999999999997}]},{"osd":7,"last update":"Sat Sep 14 03:53:21 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.44600000000000001,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.77300000000000002,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.44900000000000001,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.79700000000000004,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.60299999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T03:54:07.640 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T03:54:07.851 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:54:09.032 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:08 smithi039 ceph-mon[29586]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:09.032 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:08 smithi039 ceph-mon[40962]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:09.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:08 smithi129 ceph-mon[33173]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:09.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:54:09 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:54:09] "GET /metrics HTTP/1.1" 200 33850 "" "Prometheus/2.43.0" 2024-09-14T03:54:09.921 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:54:09.922 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T03:54:10.680 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":89,"stamp":"2024-09-14T03:54:08.765940+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592749,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":226,"num_write_kb":2432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":170,"ondisk_log_size":170,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":224016,"kb_used_data":7824,"kb_used_omap":12,"kb_used_meta":216051,"kb_avail":749737200,"statfs":{"total":767960285184,"available":767730892800,"internally_reserved":0,"allocated":8011776,"data_stored":4469123,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12743,"internal_metadata":221236793},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002797"},"pg_stats":[{"pgid":"4.14","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.812800+0000","last_change":"2024-09-14T03:53:06.413742+0000","last_active":"2024-09-14T03:53:44.812800+0000","last_peered":"2024-09-14T03:53:44.812800+0000","last_clean":"2024-09-14T03:53:44.812800+0000","last_became_active":"2024-09-14T03:53:06.413085+0000","last_became_peered":"2024-09-14T03:53:06.413085+0000","last_unstale":"2024-09-14T03:53:44.812800+0000","last_undegraded":"2024-09-14T03:53:44.812800+0000","last_fullsized":"2024-09-14T03:53:44.812800+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:00:50.348234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055429+0000","last_change":"2024-09-14T03:53:05.237323+0000","last_active":"2024-09-14T03:53:45.055429+0000","last_peered":"2024-09-14T03:53:45.055429+0000","last_clean":"2024-09-14T03:53:45.055429+0000","last_became_active":"2024-09-14T03:53:05.236726+0000","last_became_peered":"2024-09-14T03:53:05.236726+0000","last_unstale":"2024-09-14T03:53:45.055429+0000","last_undegraded":"2024-09-14T03:53:45.055429+0000","last_fullsized":"2024-09-14T03:53:45.055429+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:52:00.093659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.810579+0000","last_change":"2024-09-14T03:53:05.550982+0000","last_active":"2024-09-14T03:53:44.810579+0000","last_peered":"2024-09-14T03:53:44.810579+0000","last_clean":"2024-09-14T03:53:44.810579+0000","last_became_active":"2024-09-14T03:53:05.550702+0000","last_became_peered":"2024-09-14T03:53:05.550702+0000","last_unstale":"2024-09-14T03:53:44.810579+0000","last_undegraded":"2024-09-14T03:53:44.810579+0000","last_fullsized":"2024-09-14T03:53:44.810579+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:39:18.307082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055421+0000","last_change":"2024-09-14T03:53:05.236964+0000","last_active":"2024-09-14T03:53:45.055421+0000","last_peered":"2024-09-14T03:53:45.055421+0000","last_clean":"2024-09-14T03:53:45.055421+0000","last_became_active":"2024-09-14T03:53:05.236672+0000","last_became_peered":"2024-09-14T03:53:05.236672+0000","last_unstale":"2024-09-14T03:53:45.055421+0000","last_undegraded":"2024-09-14T03:53:45.055421+0000","last_fullsized":"2024-09-14T03:53:45.055421+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:32:12.722182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055731+0000","last_change":"2024-09-14T03:53:05.551671+0000","last_active":"2024-09-14T03:53:45.055731+0000","last_peered":"2024-09-14T03:53:45.055731+0000","last_clean":"2024-09-14T03:53:45.055731+0000","last_became_active":"2024-09-14T03:53:05.550670+0000","last_became_peered":"2024-09-14T03:53:05.550670+0000","last_unstale":"2024-09-14T03:53:45.055731+0000","last_undegraded":"2024-09-14T03:53:45.055731+0000","last_fullsized":"2024-09-14T03:53:45.055731+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:01:18.148774+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055309+0000","last_change":"2024-09-14T03:53:05.244003+0000","last_active":"2024-09-14T03:53:45.055309+0000","last_peered":"2024-09-14T03:53:45.055309+0000","last_clean":"2024-09-14T03:53:45.055309+0000","last_became_active":"2024-09-14T03:53:05.240743+0000","last_became_peered":"2024-09-14T03:53:05.240743+0000","last_unstale":"2024-09-14T03:53:45.055309+0000","last_undegraded":"2024-09-14T03:53:45.055309+0000","last_fullsized":"2024-09-14T03:53:45.055309+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:30.000189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.812006+0000","last_change":"2024-09-14T03:53:06.412161+0000","last_active":"2024-09-14T03:53:44.812006+0000","last_peered":"2024-09-14T03:53:44.812006+0000","last_clean":"2024-09-14T03:53:44.812006+0000","last_became_active":"2024-09-14T03:53:06.411884+0000","last_became_peered":"2024-09-14T03:53:06.411884+0000","last_unstale":"2024-09-14T03:53:44.812006+0000","last_undegraded":"2024-09-14T03:53:44.812006+0000","last_fullsized":"2024-09-14T03:53:44.812006+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:51:10.608362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":23,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.810130+0000","last_change":"2024-09-14T03:53:05.234450+0000","last_active":"2024-09-14T03:53:44.810130+0000","last_peered":"2024-09-14T03:53:44.810130+0000","last_clean":"2024-09-14T03:53:44.810130+0000","last_became_active":"2024-09-14T03:53:05.234191+0000","last_became_peered":"2024-09-14T03:53:05.234191+0000","last_unstale":"2024-09-14T03:53:44.810130+0000","last_undegraded":"2024-09-14T03:53:44.810130+0000","last_fullsized":"2024-09-14T03:53:44.810130+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:13:23.956618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055724+0000","last_change":"2024-09-14T03:53:05.239668+0000","last_active":"2024-09-14T03:53:45.055724+0000","last_peered":"2024-09-14T03:53:45.055724+0000","last_clean":"2024-09-14T03:53:45.055724+0000","last_became_active":"2024-09-14T03:53:05.239366+0000","last_became_peered":"2024-09-14T03:53:05.239366+0000","last_unstale":"2024-09-14T03:53:45.055724+0000","last_undegraded":"2024-09-14T03:53:45.055724+0000","last_fullsized":"2024-09-14T03:53:45.055724+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:40:51.865187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.810395+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T03:53:44.810395+0000","last_peered":"2024-09-14T03:53:44.810395+0000","last_clean":"2024-09-14T03:53:44.810395+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T03:53:44.810395+0000","last_undegraded":"2024-09-14T03:53:44.810395+0000","last_fullsized":"2024-09-14T03:53:44.810395+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055720+0000","last_change":"2024-09-14T03:53:05.240390+0000","last_active":"2024-09-14T03:53:45.055720+0000","last_peered":"2024-09-14T03:53:45.055720+0000","last_clean":"2024-09-14T03:53:45.055720+0000","last_became_active":"2024-09-14T03:53:05.240112+0000","last_became_peered":"2024-09-14T03:53:05.240112+0000","last_unstale":"2024-09-14T03:53:45.055720+0000","last_undegraded":"2024-09-14T03:53:45.055720+0000","last_fullsized":"2024-09-14T03:53:45.055720+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:02:47.345057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.807169+0000","last_change":"2024-09-14T03:53:06.412035+0000","last_active":"2024-09-14T03:53:44.807169+0000","last_peered":"2024-09-14T03:53:44.807169+0000","last_clean":"2024-09-14T03:53:44.807169+0000","last_became_active":"2024-09-14T03:53:06.411660+0000","last_became_peered":"2024-09-14T03:53:06.411660+0000","last_unstale":"2024-09-14T03:53:44.807169+0000","last_undegraded":"2024-09-14T03:53:44.807169+0000","last_fullsized":"2024-09-14T03:53:44.807169+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:50:06.795190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055687+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T03:53:45.055687+0000","last_peered":"2024-09-14T03:53:45.055687+0000","last_clean":"2024-09-14T03:53:45.055687+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T03:53:45.055687+0000","last_undegraded":"2024-09-14T03:53:45.055687+0000","last_fullsized":"2024-09-14T03:53:45.055687+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.809276+0000","last_change":"2024-09-14T03:53:05.550065+0000","last_active":"2024-09-14T03:53:44.809276+0000","last_peered":"2024-09-14T03:53:44.809276+0000","last_clean":"2024-09-14T03:53:44.809276+0000","last_became_active":"2024-09-14T03:53:05.549787+0000","last_became_peered":"2024-09-14T03:53:05.549787+0000","last_unstale":"2024-09-14T03:53:44.809276+0000","last_undegraded":"2024-09-14T03:53:44.809276+0000","last_fullsized":"2024-09-14T03:53:44.809276+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:34:58.380815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.810686+0000","last_change":"2024-09-14T03:53:05.240540+0000","last_active":"2024-09-14T03:53:44.810686+0000","last_peered":"2024-09-14T03:53:44.810686+0000","last_clean":"2024-09-14T03:53:44.810686+0000","last_became_active":"2024-09-14T03:53:05.240257+0000","last_became_peered":"2024-09-14T03:53:05.240257+0000","last_unstale":"2024-09-14T03:53:44.810686+0000","last_undegraded":"2024-09-14T03:53:44.810686+0000","last_fullsized":"2024-09-14T03:53:44.810686+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:41.107551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055349+0000","last_change":"2024-09-14T03:53:05.369617+0000","last_active":"2024-09-14T03:53:45.055349+0000","last_peered":"2024-09-14T03:53:45.055349+0000","last_clean":"2024-09-14T03:53:45.055349+0000","last_became_active":"2024-09-14T03:53:05.369325+0000","last_became_peered":"2024-09-14T03:53:05.369325+0000","last_unstale":"2024-09-14T03:53:45.055349+0000","last_undegraded":"2024-09-14T03:53:45.055349+0000","last_fullsized":"2024-09-14T03:53:45.055349+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:16:02.438169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.811583+0000","last_change":"2024-09-14T03:53:05.245363+0000","last_active":"2024-09-14T03:53:44.811583+0000","last_peered":"2024-09-14T03:53:44.811583+0000","last_clean":"2024-09-14T03:53:44.811583+0000","last_became_active":"2024-09-14T03:53:05.245066+0000","last_became_peered":"2024-09-14T03:53:05.245066+0000","last_unstale":"2024-09-14T03:53:44.811583+0000","last_undegraded":"2024-09-14T03:53:44.811583+0000","last_fullsized":"2024-09-14T03:53:44.811583+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:47.623005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.807329+0000","last_change":"2024-09-14T03:53:06.310362+0000","last_active":"2024-09-14T03:53:44.807329+0000","last_peered":"2024-09-14T03:53:44.807329+0000","last_clean":"2024-09-14T03:53:44.807329+0000","last_became_active":"2024-09-14T03:53:06.310039+0000","last_became_peered":"2024-09-14T03:53:06.310039+0000","last_unstale":"2024-09-14T03:53:44.807329+0000","last_undegraded":"2024-09-14T03:53:44.807329+0000","last_fullsized":"2024-09-14T03:53:44.807329+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:06:11.654831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.809671+0000","last_change":"2024-09-14T03:53:05.236501+0000","last_active":"2024-09-14T03:53:44.809671+0000","last_peered":"2024-09-14T03:53:44.809671+0000","last_clean":"2024-09-14T03:53:44.809671+0000","last_became_active":"2024-09-14T03:53:05.236225+0000","last_became_peered":"2024-09-14T03:53:05.236225+0000","last_unstale":"2024-09-14T03:53:44.809671+0000","last_undegraded":"2024-09-14T03:53:44.809671+0000","last_fullsized":"2024-09-14T03:53:44.809671+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:44.710359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.807611+0000","last_change":"2024-09-14T03:53:06.412211+0000","last_active":"2024-09-14T03:53:44.807611+0000","last_peered":"2024-09-14T03:53:44.807611+0000","last_clean":"2024-09-14T03:53:44.807611+0000","last_became_active":"2024-09-14T03:53:06.411711+0000","last_became_peered":"2024-09-14T03:53:06.411711+0000","last_unstale":"2024-09-14T03:53:44.807611+0000","last_undegraded":"2024-09-14T03:53:44.807611+0000","last_fullsized":"2024-09-14T03:53:44.807611+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:30.693735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.807285+0000","last_change":"2024-09-14T03:53:05.237902+0000","last_active":"2024-09-14T03:53:44.807285+0000","last_peered":"2024-09-14T03:53:44.807285+0000","last_clean":"2024-09-14T03:53:44.807285+0000","last_became_active":"2024-09-14T03:53:05.237615+0000","last_became_peered":"2024-09-14T03:53:05.237615+0000","last_unstale":"2024-09-14T03:53:44.807285+0000","last_undegraded":"2024-09-14T03:53:44.807285+0000","last_fullsized":"2024-09-14T03:53:44.807285+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:32:53.206648+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.807784+0000","last_change":"2024-09-14T03:53:06.412288+0000","last_active":"2024-09-14T03:53:44.807784+0000","last_peered":"2024-09-14T03:53:44.807784+0000","last_clean":"2024-09-14T03:53:44.807784+0000","last_became_active":"2024-09-14T03:53:06.411577+0000","last_became_peered":"2024-09-14T03:53:06.411577+0000","last_unstale":"2024-09-14T03:53:44.807784+0000","last_undegraded":"2024-09-14T03:53:44.807784+0000","last_fullsized":"2024-09-14T03:53:44.807784+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:20:37.678658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.812677+0000","last_change":"2024-09-14T03:53:05.236651+0000","last_active":"2024-09-14T03:53:44.812677+0000","last_peered":"2024-09-14T03:53:44.812677+0000","last_clean":"2024-09-14T03:53:44.812677+0000","last_became_active":"2024-09-14T03:53:05.235902+0000","last_became_peered":"2024-09-14T03:53:05.235902+0000","last_unstale":"2024-09-14T03:53:44.812677+0000","last_undegraded":"2024-09-14T03:53:44.812677+0000","last_fullsized":"2024-09-14T03:53:44.812677+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:36:37.779052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.810557+0000","last_change":"2024-09-14T03:53:05.369591+0000","last_active":"2024-09-14T03:53:44.810557+0000","last_peered":"2024-09-14T03:53:44.810557+0000","last_clean":"2024-09-14T03:53:44.810557+0000","last_became_active":"2024-09-14T03:53:05.369232+0000","last_became_peered":"2024-09-14T03:53:05.369232+0000","last_unstale":"2024-09-14T03:53:44.810557+0000","last_undegraded":"2024-09-14T03:53:44.810557+0000","last_fullsized":"2024-09-14T03:53:44.810557+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:03:16.262268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"61'128","reported_seq":270,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055457+0000","last_change":"2024-09-14T03:51:16.669763+0000","last_active":"2024-09-14T03:53:45.055457+0000","last_peered":"2024-09-14T03:53:45.055457+0000","last_clean":"2024-09-14T03:53:45.055457+0000","last_became_active":"2024-09-14T03:51:16.669377+0000","last_became_peered":"2024-09-14T03:51:16.669377+0000","last_unstale":"2024-09-14T03:53:45.055457+0000","last_undegraded":"2024-09-14T03:53:45.055457+0000","last_fullsized":"2024-09-14T03:53:45.055457+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T03:49:18.237706+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:10:34.161809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.835463+0000","last_change":"2024-09-14T03:51:52.316449+0000","last_active":"2024-09-14T03:53:45.835463+0000","last_peered":"2024-09-14T03:53:45.835463+0000","last_clean":"2024-09-14T03:53:45.835463+0000","last_became_active":"2024-09-14T03:51:46.806879+0000","last_became_peered":"2024-09-14T03:51:46.806879+0000","last_unstale":"2024-09-14T03:53:45.835463+0000","last_undegraded":"2024-09-14T03:53:45.835463+0000","last_fullsized":"2024-09-14T03:53:45.835463+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:37:54.991090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.811513+0000","last_change":"2024-09-14T03:53:05.242652+0000","last_active":"2024-09-14T03:53:44.811513+0000","last_peered":"2024-09-14T03:53:44.811513+0000","last_clean":"2024-09-14T03:53:44.811513+0000","last_became_active":"2024-09-14T03:53:05.242362+0000","last_became_peered":"2024-09-14T03:53:05.242362+0000","last_unstale":"2024-09-14T03:53:44.811513+0000","last_undegraded":"2024-09-14T03:53:44.811513+0000","last_fullsized":"2024-09-14T03:53:44.811513+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:17:25.158508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.809479+0000","last_change":"2024-09-14T03:53:05.551268+0000","last_active":"2024-09-14T03:53:44.809479+0000","last_peered":"2024-09-14T03:53:44.809479+0000","last_clean":"2024-09-14T03:53:44.809479+0000","last_became_active":"2024-09-14T03:53:05.551015+0000","last_became_peered":"2024-09-14T03:53:05.551015+0000","last_unstale":"2024-09-14T03:53:44.809479+0000","last_undegraded":"2024-09-14T03:53:44.809479+0000","last_fullsized":"2024-09-14T03:53:44.809479+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.905475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:53:46.105252+0000","last_change":"2024-09-14T03:51:52.734251+0000","last_active":"2024-09-14T03:53:46.105252+0000","last_peered":"2024-09-14T03:53:46.105252+0000","last_clean":"2024-09-14T03:53:46.105252+0000","last_became_active":"2024-09-14T03:51:46.834389+0000","last_became_peered":"2024-09-14T03:51:46.834389+0000","last_unstale":"2024-09-14T03:53:46.105252+0000","last_undegraded":"2024-09-14T03:53:46.105252+0000","last_fullsized":"2024-09-14T03:53:46.105252+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:12:21.741822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055245+0000","last_change":"2024-09-14T03:53:05.243315+0000","last_active":"2024-09-14T03:53:45.055245+0000","last_peered":"2024-09-14T03:53:45.055245+0000","last_clean":"2024-09-14T03:53:45.055245+0000","last_became_active":"2024-09-14T03:53:05.243036+0000","last_became_peered":"2024-09-14T03:53:05.243036+0000","last_unstale":"2024-09-14T03:53:45.055245+0000","last_undegraded":"2024-09-14T03:53:45.055245+0000","last_fullsized":"2024-09-14T03:53:45.055245+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:01:10.404454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055533+0000","last_change":"2024-09-14T03:53:05.461556+0000","last_active":"2024-09-14T03:53:45.055533+0000","last_peered":"2024-09-14T03:53:45.055533+0000","last_clean":"2024-09-14T03:53:45.055533+0000","last_became_active":"2024-09-14T03:53:05.460764+0000","last_became_peered":"2024-09-14T03:53:05.460764+0000","last_unstale":"2024-09-14T03:53:45.055533+0000","last_undegraded":"2024-09-14T03:53:45.055533+0000","last_fullsized":"2024-09-14T03:53:45.055533+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:50:51.579075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:53:46.105816+0000","last_change":"2024-09-14T03:51:52.859768+0000","last_active":"2024-09-14T03:53:46.105816+0000","last_peered":"2024-09-14T03:53:46.105816+0000","last_clean":"2024-09-14T03:53:46.105816+0000","last_became_active":"2024-09-14T03:51:46.835278+0000","last_became_peered":"2024-09-14T03:51:46.835278+0000","last_unstale":"2024-09-14T03:53:46.105816+0000","last_undegraded":"2024-09-14T03:53:46.105816+0000","last_fullsized":"2024-09-14T03:53:46.105816+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:25.102148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.810580+0000","last_change":"2024-09-14T03:53:05.234040+0000","last_active":"2024-09-14T03:53:44.810580+0000","last_peered":"2024-09-14T03:53:44.810580+0000","last_clean":"2024-09-14T03:53:44.810580+0000","last_became_active":"2024-09-14T03:53:05.233722+0000","last_became_peered":"2024-09-14T03:53:05.233722+0000","last_unstale":"2024-09-14T03:53:44.810580+0000","last_undegraded":"2024-09-14T03:53:44.810580+0000","last_fullsized":"2024-09-14T03:53:44.810580+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:42:14.192664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.810638+0000","last_change":"2024-09-14T03:53:05.252316+0000","last_active":"2024-09-14T03:53:44.810638+0000","last_peered":"2024-09-14T03:53:44.810638+0000","last_clean":"2024-09-14T03:53:44.810638+0000","last_became_active":"2024-09-14T03:53:05.252009+0000","last_became_peered":"2024-09-14T03:53:05.252009+0000","last_unstale":"2024-09-14T03:53:44.810638+0000","last_undegraded":"2024-09-14T03:53:44.810638+0000","last_fullsized":"2024-09-14T03:53:44.810638+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:00:31.537605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":29,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.835425+0000","last_change":"2024-09-14T03:51:52.316945+0000","last_active":"2024-09-14T03:53:45.835425+0000","last_peered":"2024-09-14T03:53:45.835425+0000","last_clean":"2024-09-14T03:53:45.835425+0000","last_became_active":"2024-09-14T03:51:46.806040+0000","last_became_peered":"2024-09-14T03:51:46.806040+0000","last_unstale":"2024-09-14T03:53:45.835425+0000","last_undegraded":"2024-09-14T03:53:45.835425+0000","last_fullsized":"2024-09-14T03:53:45.835425+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:28:20.339010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.807517+0000","last_change":"2024-09-14T03:53:05.242366+0000","last_active":"2024-09-14T03:53:44.807517+0000","last_peered":"2024-09-14T03:53:44.807517+0000","last_clean":"2024-09-14T03:53:44.807517+0000","last_became_active":"2024-09-14T03:53:05.241689+0000","last_became_peered":"2024-09-14T03:53:05.241689+0000","last_unstale":"2024-09-14T03:53:44.807517+0000","last_undegraded":"2024-09-14T03:53:44.807517+0000","last_fullsized":"2024-09-14T03:53:44.807517+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:32:21.936986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.810822+0000","last_change":"2024-09-14T03:53:05.460904+0000","last_active":"2024-09-14T03:53:44.810822+0000","last_peered":"2024-09-14T03:53:44.810822+0000","last_clean":"2024-09-14T03:53:44.810822+0000","last_became_active":"2024-09-14T03:53:05.460645+0000","last_became_peered":"2024-09-14T03:53:05.460645+0000","last_unstale":"2024-09-14T03:53:44.810822+0000","last_undegraded":"2024-09-14T03:53:44.810822+0000","last_fullsized":"2024-09-14T03:53:44.810822+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:11:19.095675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.810960+0000","last_change":"2024-09-14T03:53:05.241606+0000","last_active":"2024-09-14T03:53:44.810960+0000","last_peered":"2024-09-14T03:53:44.810960+0000","last_clean":"2024-09-14T03:53:44.810960+0000","last_became_active":"2024-09-14T03:53:05.241310+0000","last_became_peered":"2024-09-14T03:53:05.241310+0000","last_unstale":"2024-09-14T03:53:44.810960+0000","last_undegraded":"2024-09-14T03:53:44.810960+0000","last_fullsized":"2024-09-14T03:53:44.810960+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:49:54.875110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.819679+0000","last_change":"2024-09-14T03:51:52.317915+0000","last_active":"2024-09-14T03:53:45.819679+0000","last_peered":"2024-09-14T03:53:45.819679+0000","last_clean":"2024-09-14T03:53:45.819679+0000","last_became_active":"2024-09-14T03:51:46.834577+0000","last_became_peered":"2024-09-14T03:51:46.834577+0000","last_unstale":"2024-09-14T03:53:45.819679+0000","last_undegraded":"2024-09-14T03:53:45.819679+0000","last_fullsized":"2024-09-14T03:53:45.819679+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:14:59.991639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.810150+0000","last_change":"2024-09-14T03:53:05.369605+0000","last_active":"2024-09-14T03:53:44.810150+0000","last_peered":"2024-09-14T03:53:44.810150+0000","last_clean":"2024-09-14T03:53:44.810150+0000","last_became_active":"2024-09-14T03:53:05.369242+0000","last_became_peered":"2024-09-14T03:53:05.369242+0000","last_unstale":"2024-09-14T03:53:44.810150+0000","last_undegraded":"2024-09-14T03:53:44.810150+0000","last_fullsized":"2024-09-14T03:53:44.810150+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:03:05.904256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.810979+0000","last_change":"2024-09-14T03:53:05.235449+0000","last_active":"2024-09-14T03:53:44.810979+0000","last_peered":"2024-09-14T03:53:44.810979+0000","last_clean":"2024-09-14T03:53:44.810979+0000","last_became_active":"2024-09-14T03:53:05.235108+0000","last_became_peered":"2024-09-14T03:53:05.235108+0000","last_unstale":"2024-09-14T03:53:44.810979+0000","last_undegraded":"2024-09-14T03:53:44.810979+0000","last_fullsized":"2024-09-14T03:53:44.810979+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:52:34.275278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:53:46.105682+0000","last_change":"2024-09-14T03:51:52.734322+0000","last_active":"2024-09-14T03:53:46.105682+0000","last_peered":"2024-09-14T03:53:46.105682+0000","last_clean":"2024-09-14T03:53:46.105682+0000","last_became_active":"2024-09-14T03:51:46.835000+0000","last_became_peered":"2024-09-14T03:51:46.835000+0000","last_unstale":"2024-09-14T03:53:46.105682+0000","last_undegraded":"2024-09-14T03:53:46.105682+0000","last_fullsized":"2024-09-14T03:53:46.105682+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:46:41.873402+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055394+0000","last_change":"2024-09-14T03:53:05.550002+0000","last_active":"2024-09-14T03:53:45.055394+0000","last_peered":"2024-09-14T03:53:45.055394+0000","last_clean":"2024-09-14T03:53:45.055394+0000","last_became_active":"2024-09-14T03:53:05.549782+0000","last_became_peered":"2024-09-14T03:53:05.549782+0000","last_unstale":"2024-09-14T03:53:45.055394+0000","last_undegraded":"2024-09-14T03:53:45.055394+0000","last_fullsized":"2024-09-14T03:53:45.055394+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:50:50.053863+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.812173+0000","last_change":"2024-09-14T03:53:05.239088+0000","last_active":"2024-09-14T03:53:44.812173+0000","last_peered":"2024-09-14T03:53:44.812173+0000","last_clean":"2024-09-14T03:53:44.812173+0000","last_became_active":"2024-09-14T03:53:05.238773+0000","last_became_peered":"2024-09-14T03:53:05.238773+0000","last_unstale":"2024-09-14T03:53:44.812173+0000","last_undegraded":"2024-09-14T03:53:44.812173+0000","last_fullsized":"2024-09-14T03:53:44.812173+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:21:45.491829+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.819903+0000","last_change":"2024-09-14T03:51:52.317867+0000","last_active":"2024-09-14T03:53:45.819903+0000","last_peered":"2024-09-14T03:53:45.819903+0000","last_clean":"2024-09-14T03:53:45.819903+0000","last_became_active":"2024-09-14T03:51:46.834009+0000","last_became_peered":"2024-09-14T03:51:46.834009+0000","last_unstale":"2024-09-14T03:53:45.819903+0000","last_undegraded":"2024-09-14T03:53:45.819903+0000","last_fullsized":"2024-09-14T03:53:45.819903+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:36:33.505359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028771999999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.812489+0000","last_change":"2024-09-14T03:53:06.250859+0000","last_active":"2024-09-14T03:53:44.812489+0000","last_peered":"2024-09-14T03:53:44.812489+0000","last_clean":"2024-09-14T03:53:44.812489+0000","last_became_active":"2024-09-14T03:53:06.250581+0000","last_became_peered":"2024-09-14T03:53:06.250581+0000","last_unstale":"2024-09-14T03:53:44.812489+0000","last_undegraded":"2024-09-14T03:53:44.812489+0000","last_fullsized":"2024-09-14T03:53:44.812489+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:17:15.683212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055639+0000","last_change":"2024-09-14T03:53:05.240359+0000","last_active":"2024-09-14T03:53:45.055639+0000","last_peered":"2024-09-14T03:53:45.055639+0000","last_clean":"2024-09-14T03:53:45.055639+0000","last_became_active":"2024-09-14T03:53:05.240068+0000","last_became_peered":"2024-09-14T03:53:05.240068+0000","last_unstale":"2024-09-14T03:53:45.055639+0000","last_undegraded":"2024-09-14T03:53:45.055639+0000","last_fullsized":"2024-09-14T03:53:45.055639+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:55:16.343381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":28,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.812671+0000","last_change":"2024-09-14T03:51:52.317818+0000","last_active":"2024-09-14T03:53:45.812671+0000","last_peered":"2024-09-14T03:53:45.812671+0000","last_clean":"2024-09-14T03:53:45.812671+0000","last_became_active":"2024-09-14T03:51:46.806805+0000","last_became_peered":"2024-09-14T03:51:46.806805+0000","last_unstale":"2024-09-14T03:53:45.812671+0000","last_undegraded":"2024-09-14T03:53:45.812671+0000","last_fullsized":"2024-09-14T03:53:45.812671+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:01:38.862572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.807046+0000","last_change":"2024-09-14T03:53:06.634043+0000","last_active":"2024-09-14T03:53:44.807046+0000","last_peered":"2024-09-14T03:53:44.807046+0000","last_clean":"2024-09-14T03:53:44.807046+0000","last_became_active":"2024-09-14T03:53:06.633638+0000","last_became_peered":"2024-09-14T03:53:06.633638+0000","last_unstale":"2024-09-14T03:53:44.807046+0000","last_undegraded":"2024-09-14T03:53:44.807046+0000","last_fullsized":"2024-09-14T03:53:44.807046+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:15:53.434064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.807373+0000","last_change":"2024-09-14T03:53:06.412072+0000","last_active":"2024-09-14T03:53:44.807373+0000","last_peered":"2024-09-14T03:53:44.807373+0000","last_clean":"2024-09-14T03:53:44.807373+0000","last_became_active":"2024-09-14T03:53:06.411778+0000","last_became_peered":"2024-09-14T03:53:06.411778+0000","last_unstale":"2024-09-14T03:53:44.807373+0000","last_undegraded":"2024-09-14T03:53:44.807373+0000","last_fullsized":"2024-09-14T03:53:44.807373+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:41:38.193881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"72'6","reported_seq":21,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.807605+0000","last_change":"2024-09-14T03:53:05.244055+0000","last_active":"2024-09-14T03:53:44.807605+0000","last_peered":"2024-09-14T03:53:44.807605+0000","last_clean":"2024-09-14T03:53:44.807605+0000","last_became_active":"2024-09-14T03:53:05.243752+0000","last_became_peered":"2024-09-14T03:53:05.243752+0000","last_unstale":"2024-09-14T03:53:44.807605+0000","last_undegraded":"2024-09-14T03:53:44.807605+0000","last_fullsized":"2024-09-14T03:53:44.807605+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:12:31.850064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1564,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.811158+0000","last_change":"2024-09-14T03:53:05.460859+0000","last_active":"2024-09-14T03:53:44.811158+0000","last_peered":"2024-09-14T03:53:44.811158+0000","last_clean":"2024-09-14T03:53:44.811158+0000","last_became_active":"2024-09-14T03:53:05.460555+0000","last_became_peered":"2024-09-14T03:53:05.460555+0000","last_unstale":"2024-09-14T03:53:44.811158+0000","last_undegraded":"2024-09-14T03:53:44.811158+0000","last_fullsized":"2024-09-14T03:53:44.811158+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:10:54.185175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":18,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055552+0000","last_change":"2024-09-14T03:53:05.240418+0000","last_active":"2024-09-14T03:53:45.055552+0000","last_peered":"2024-09-14T03:53:45.055552+0000","last_clean":"2024-09-14T03:53:45.055552+0000","last_became_active":"2024-09-14T03:53:05.240140+0000","last_became_peered":"2024-09-14T03:53:05.240140+0000","last_unstale":"2024-09-14T03:53:45.055552+0000","last_undegraded":"2024-09-14T03:53:45.055552+0000","last_fullsized":"2024-09-14T03:53:45.055552+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:31:00.413012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.810766+0000","last_change":"2024-09-14T03:53:05.237203+0000","last_active":"2024-09-14T03:53:44.810766+0000","last_peered":"2024-09-14T03:53:44.810766+0000","last_clean":"2024-09-14T03:53:44.810766+0000","last_became_active":"2024-09-14T03:53:05.236893+0000","last_became_peered":"2024-09-14T03:53:05.236893+0000","last_unstale":"2024-09-14T03:53:44.810766+0000","last_undegraded":"2024-09-14T03:53:44.810766+0000","last_fullsized":"2024-09-14T03:53:44.810766+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:36:55.377845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.811235+0000","last_change":"2024-09-14T03:53:06.310444+0000","last_active":"2024-09-14T03:53:44.811235+0000","last_peered":"2024-09-14T03:53:44.811235+0000","last_clean":"2024-09-14T03:53:44.811235+0000","last_became_active":"2024-09-14T03:53:06.310166+0000","last_became_peered":"2024-09-14T03:53:06.310166+0000","last_unstale":"2024-09-14T03:53:44.811235+0000","last_undegraded":"2024-09-14T03:53:44.811235+0000","last_fullsized":"2024-09-14T03:53:44.811235+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:01:08.675783+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055758+0000","last_change":"2024-09-14T03:53:05.237613+0000","last_active":"2024-09-14T03:53:45.055758+0000","last_peered":"2024-09-14T03:53:45.055758+0000","last_clean":"2024-09-14T03:53:45.055758+0000","last_became_active":"2024-09-14T03:53:05.237326+0000","last_became_peered":"2024-09-14T03:53:05.237326+0000","last_unstale":"2024-09-14T03:53:45.055758+0000","last_undegraded":"2024-09-14T03:53:45.055758+0000","last_fullsized":"2024-09-14T03:53:45.055758+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:54:08.959667+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":23,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:53:46.106654+0000","last_change":"2024-09-14T03:53:46.106654+0000","last_active":"2024-09-14T03:53:46.106654+0000","last_peered":"2024-09-14T03:53:46.106654+0000","last_clean":"2024-09-14T03:53:46.106654+0000","last_became_active":"2024-09-14T03:53:46.106346+0000","last_became_peered":"2024-09-14T03:53:46.106346+0000","last_unstale":"2024-09-14T03:53:46.106654+0000","last_undegraded":"2024-09-14T03:53:46.106654+0000","last_fullsized":"2024-09-14T03:53:46.106654+0000","mapping_epoch":73,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":74,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:29:16.158851+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":23,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:53:46.106894+0000","last_change":"2024-09-14T03:53:46.106894+0000","last_active":"2024-09-14T03:53:46.106894+0000","last_peered":"2024-09-14T03:53:46.106894+0000","last_clean":"2024-09-14T03:53:46.106894+0000","last_became_active":"2024-09-14T03:53:46.106495+0000","last_became_peered":"2024-09-14T03:53:46.106495+0000","last_unstale":"2024-09-14T03:53:46.106894+0000","last_undegraded":"2024-09-14T03:53:46.106894+0000","last_fullsized":"2024-09-14T03:53:46.106894+0000","mapping_epoch":73,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":74,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:43:20.627135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.812590+0000","last_change":"2024-09-14T03:53:05.236831+0000","last_active":"2024-09-14T03:53:44.812590+0000","last_peered":"2024-09-14T03:53:44.812590+0000","last_clean":"2024-09-14T03:53:44.812590+0000","last_became_active":"2024-09-14T03:53:05.236360+0000","last_became_peered":"2024-09-14T03:53:05.236360+0000","last_unstale":"2024-09-14T03:53:44.812590+0000","last_undegraded":"2024-09-14T03:53:44.812590+0000","last_fullsized":"2024-09-14T03:53:44.812590+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:21:42.355910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055513+0000","last_change":"2024-09-14T03:53:05.551090+0000","last_active":"2024-09-14T03:53:45.055513+0000","last_peered":"2024-09-14T03:53:45.055513+0000","last_clean":"2024-09-14T03:53:45.055513+0000","last_became_active":"2024-09-14T03:53:05.550765+0000","last_became_peered":"2024-09-14T03:53:05.550765+0000","last_unstale":"2024-09-14T03:53:45.055513+0000","last_undegraded":"2024-09-14T03:53:45.055513+0000","last_fullsized":"2024-09-14T03:53:45.055513+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:15:06.004332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":15,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.810872+0000","last_change":"2024-09-14T03:53:05.244106+0000","last_active":"2024-09-14T03:53:44.810872+0000","last_peered":"2024-09-14T03:53:44.810872+0000","last_clean":"2024-09-14T03:53:44.810872+0000","last_became_active":"2024-09-14T03:53:05.243512+0000","last_became_peered":"2024-09-14T03:53:05.243512+0000","last_unstale":"2024-09-14T03:53:44.810872+0000","last_undegraded":"2024-09-14T03:53:44.810872+0000","last_fullsized":"2024-09-14T03:53:44.810872+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:49:19.720662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.811763+0000","last_change":"2024-09-14T03:53:06.412762+0000","last_active":"2024-09-14T03:53:44.811763+0000","last_peered":"2024-09-14T03:53:44.811763+0000","last_clean":"2024-09-14T03:53:44.811763+0000","last_became_active":"2024-09-14T03:53:06.412502+0000","last_became_peered":"2024-09-14T03:53:06.412502+0000","last_unstale":"2024-09-14T03:53:44.811763+0000","last_undegraded":"2024-09-14T03:53:44.811763+0000","last_fullsized":"2024-09-14T03:53:44.811763+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:19.745802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.807446+0000","last_change":"2024-09-14T03:53:05.239221+0000","last_active":"2024-09-14T03:53:44.807446+0000","last_peered":"2024-09-14T03:53:44.807446+0000","last_clean":"2024-09-14T03:53:44.807446+0000","last_became_active":"2024-09-14T03:53:05.238936+0000","last_became_peered":"2024-09-14T03:53:05.238936+0000","last_unstale":"2024-09-14T03:53:44.807446+0000","last_undegraded":"2024-09-14T03:53:44.807446+0000","last_fullsized":"2024-09-14T03:53:44.807446+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:14:04.731925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.812913+0000","last_change":"2024-09-14T03:53:06.412049+0000","last_active":"2024-09-14T03:53:44.812913+0000","last_peered":"2024-09-14T03:53:44.812913+0000","last_clean":"2024-09-14T03:53:44.812913+0000","last_became_active":"2024-09-14T03:53:06.411716+0000","last_became_peered":"2024-09-14T03:53:06.411716+0000","last_unstale":"2024-09-14T03:53:44.812913+0000","last_undegraded":"2024-09-14T03:53:44.812913+0000","last_fullsized":"2024-09-14T03:53:44.812913+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:09:57.721410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055479+0000","last_change":"2024-09-14T03:53:05.240514+0000","last_active":"2024-09-14T03:53:45.055479+0000","last_peered":"2024-09-14T03:53:45.055479+0000","last_clean":"2024-09-14T03:53:45.055479+0000","last_became_active":"2024-09-14T03:53:05.240213+0000","last_became_peered":"2024-09-14T03:53:05.240213+0000","last_unstale":"2024-09-14T03:53:45.055479+0000","last_undegraded":"2024-09-14T03:53:45.055479+0000","last_fullsized":"2024-09-14T03:53:45.055479+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:25:50.704382+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.807264+0000","last_change":"2024-09-14T03:53:06.412040+0000","last_active":"2024-09-14T03:53:44.807264+0000","last_peered":"2024-09-14T03:53:44.807264+0000","last_clean":"2024-09-14T03:53:44.807264+0000","last_became_active":"2024-09-14T03:53:06.411769+0000","last_became_peered":"2024-09-14T03:53:06.411769+0000","last_unstale":"2024-09-14T03:53:44.807264+0000","last_undegraded":"2024-09-14T03:53:44.807264+0000","last_fullsized":"2024-09-14T03:53:44.807264+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:09:05.162369+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.810579+0000","last_change":"2024-09-14T03:53:05.235171+0000","last_active":"2024-09-14T03:53:44.810579+0000","last_peered":"2024-09-14T03:53:44.810579+0000","last_clean":"2024-09-14T03:53:44.810579+0000","last_became_active":"2024-09-14T03:53:05.234895+0000","last_became_peered":"2024-09-14T03:53:05.234895+0000","last_unstale":"2024-09-14T03:53:44.810579+0000","last_undegraded":"2024-09-14T03:53:44.810579+0000","last_fullsized":"2024-09-14T03:53:44.810579+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:07:16.459146+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.807259+0000","last_change":"2024-09-14T03:53:06.250614+0000","last_active":"2024-09-14T03:53:44.807259+0000","last_peered":"2024-09-14T03:53:44.807259+0000","last_clean":"2024-09-14T03:53:44.807259+0000","last_became_active":"2024-09-14T03:53:06.250371+0000","last_became_peered":"2024-09-14T03:53:06.250371+0000","last_unstale":"2024-09-14T03:53:44.807259+0000","last_undegraded":"2024-09-14T03:53:44.807259+0000","last_fullsized":"2024-09-14T03:53:44.807259+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:43:52.171802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":17,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.810858+0000","last_change":"2024-09-14T03:53:05.234444+0000","last_active":"2024-09-14T03:53:44.810858+0000","last_peered":"2024-09-14T03:53:44.810858+0000","last_clean":"2024-09-14T03:53:44.810858+0000","last_became_active":"2024-09-14T03:53:05.234115+0000","last_became_peered":"2024-09-14T03:53:05.234115+0000","last_unstale":"2024-09-14T03:53:44.810858+0000","last_undegraded":"2024-09-14T03:53:44.810858+0000","last_fullsized":"2024-09-14T03:53:44.810858+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:01:23.153937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":14,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.811306+0000","last_change":"2024-09-14T03:53:06.251023+0000","last_active":"2024-09-14T03:53:44.811306+0000","last_peered":"2024-09-14T03:53:44.811306+0000","last_clean":"2024-09-14T03:53:44.811306+0000","last_became_active":"2024-09-14T03:53:06.250566+0000","last_became_peered":"2024-09-14T03:53:06.250566+0000","last_unstale":"2024-09-14T03:53:44.811306+0000","last_undegraded":"2024-09-14T03:53:44.811306+0000","last_fullsized":"2024-09-14T03:53:44.811306+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:54:39.448363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":16,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:44.811913+0000","last_change":"2024-09-14T03:53:05.237437+0000","last_active":"2024-09-14T03:53:44.811913+0000","last_peered":"2024-09-14T03:53:44.811913+0000","last_clean":"2024-09-14T03:53:44.811913+0000","last_became_active":"2024-09-14T03:53:05.237151+0000","last_became_peered":"2024-09-14T03:53:05.237151+0000","last_unstale":"2024-09-14T03:53:44.811913+0000","last_undegraded":"2024-09-14T03:53:44.811913+0000","last_fullsized":"2024-09-14T03:53:44.811913+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:57:21.916179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":23,"reported_epoch":74,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.836449+0000","last_change":"2024-09-14T03:53:45.836449+0000","last_active":"2024-09-14T03:53:45.836449+0000","last_peered":"2024-09-14T03:53:45.836449+0000","last_clean":"2024-09-14T03:53:45.836449+0000","last_became_active":"2024-09-14T03:53:45.836108+0000","last_became_peered":"2024-09-14T03:53:45.836108+0000","last_unstale":"2024-09-14T03:53:45.836449+0000","last_undegraded":"2024-09-14T03:53:45.836449+0000","last_fullsized":"2024-09-14T03:53:45.836449+0000","mapping_epoch":73,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":74,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:52:13.095225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":19,"reported_epoch":73,"state":"active+clean","last_fresh":"2024-09-14T03:53:45.055693+0000","last_change":"2024-09-14T03:53:05.236307+0000","last_active":"2024-09-14T03:53:45.055693+0000","last_peered":"2024-09-14T03:53:45.055693+0000","last_clean":"2024-09-14T03:53:45.055693+0000","last_became_active":"2024-09-14T03:53:05.236014+0000","last_became_peered":"2024-09-14T03:53:05.236014+0000","last_unstale":"2024-09-14T03:53:45.055693+0000","last_undegraded":"2024-09-14T03:53:45.055693+0000","last_fullsized":"2024-09-14T03:53:45.055693+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:02.092228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2362,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201318,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28380,"kb_used_data":1356,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716772,"statfs":{"total":95995035648,"available":95965974528,"internally_reserved":0,"allocated":1388544,"data_stored":928583,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":27654603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:53:21 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.54500000000000004,"15min":0.54800000000000004},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.78400000000000003,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.53300000000000003,"15min":0.52800000000000002},"min":{"1min":0.41099999999999998,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.73699999999999999,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.69899999999999995}]},{"osd":1,"last update":"Sat Sep 14 03:53:21 2024","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53900000000000003,"15min":0.54000000000000004},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.70099999999999996,"5min":0.748,"15min":0.748},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.55800000000000005,"15min":0.55600000000000005},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.68899999999999995,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.67100000000000004}]},{"osd":2,"last update":"Sat Sep 14 03:53:21 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.54400000000000004,"15min":0.54200000000000004},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.76300000000000001,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.63},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.56799999999999995,"15min":0.56999999999999995},"min":{"1min":0.40400000000000003,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.71199999999999997,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.621}]},{"osd":3,"last update":"Sat Sep 14 03:53:21 2024","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.52600000000000002,"15min":0.52200000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.71099999999999997,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.59399999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.755,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.65300000000000002}]},{"osd":4,"last update":"Sat Sep 14 03:53:21 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.47599999999999998,"15min":0.48099999999999998},"min":{"1min":0.308,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.67300000000000004,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.47199999999999998},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.47599999999999998,"15min":0.47699999999999998},"min":{"1min":0.30499999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.59899999999999998,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.43099999999999999}]},{"osd":5,"last update":"Sat Sep 14 03:53:21 2024","interfaces":[{"interface":"back","average":{"1min":0.47099999999999997,"5min":0.49199999999999999,"15min":0.496},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.61599999999999999,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.498},{"interface":"front","average":{"1min":0.51300000000000001,"5min":0.53300000000000003,"15min":0.53700000000000003},"min":{"1min":0.35199999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.69699999999999995,"5min":0.84099999999999997,"15min":0.84099999999999997},"last":0.60999999999999999}]},{"osd":6,"last update":"Sat Sep 14 03:53:21 2024","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.34999999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.752,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.55500000000000005},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.51900000000000002,"15min":0.51800000000000002},"min":{"1min":0.32600000000000001,"5min":0.311,"15min":0.311},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":0.53900000000000003}]}]},{"osd":6,"up_from":47,"seq":201863462954,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28372,"kb_used_data":1348,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716780,"statfs":{"total":95995035648,"available":95965982720,"internally_reserved":0,"allocated":1380352,"data_stored":928456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.53600000000000003,"15min":0.53900000000000003},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.75,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.50700000000000001,"5min":0.51800000000000002,"15min":0.51600000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.70399999999999996,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.66400000000000003}]},{"osd":1,"last update":"Sat Sep 14 03:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.53200000000000003,"15min":0.53400000000000003},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.75800000000000001,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.51100000000000001,"15min":0.51000000000000001},"min":{"1min":0.39800000000000002,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.67000000000000004,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.53100000000000003}]},{"osd":2,"last update":"Sat Sep 14 03:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.52400000000000002,"15min":0.51800000000000002},"min":{"1min":0.39700000000000002,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.745,"5min":0.86199999999999999,"15min":0.86199999999999999},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.52900000000000003,"15min":0.53600000000000003},"min":{"1min":0.42599999999999999,"5min":0.378,"15min":0.378},"max":{"1min":0.61399999999999999,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.51700000000000002}]},{"osd":3,"last update":"Sat Sep 14 03:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.52300000000000002,"15min":0.51100000000000001},"min":{"1min":0.378,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.64500000000000002,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.54900000000000004,"15min":0.55600000000000005},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":0.69399999999999995,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.70899999999999996}]},{"osd":4,"last update":"Sat Sep 14 03:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.40500000000000003,"15min":0.38700000000000001},"min":{"1min":0.30299999999999999,"5min":0.25,"15min":0.25},"max":{"1min":0.64100000000000001,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.436,"15min":0.41799999999999998},"min":{"1min":0.318,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.68200000000000005,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.61799999999999999}]},{"osd":5,"last update":"Sat Sep 14 03:53:58 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.41799999999999998,"15min":0.39800000000000002},"min":{"1min":0.29499999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.622,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.44,"15min":0.40999999999999998},"min":{"1min":0.33200000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.70899999999999996,"5min":0.92600000000000005,"15min":0.92600000000000005},"last":0.69099999999999995}]},{"osd":7,"last update":"Sat Sep 14 03:53:19 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.47699999999999998,"15min":0.47499999999999998},"min":{"1min":0.34599999999999997,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.67700000000000005,"5min":0.80800000000000005,"15min":0.80800000000000005},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.52900000000000003,"5min":0.505,"15min":0.501},"min":{"1min":0.34300000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.65200000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724591,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27780,"kb_used_data":756,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717372,"statfs":{"total":95995035648,"available":95966588928,"internally_reserved":0,"allocated":774144,"data_stored":336775,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:53:33 2024","interfaces":[{"interface":"back","average":{"1min":0.53000000000000003,"5min":0.55200000000000005,"15min":0.56000000000000005},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.71499999999999997,"5min":0.75,"15min":0.75},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.54700000000000004,"15min":0.54500000000000004},"min":{"1min":0.36299999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.82299999999999995,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.65400000000000003}]},{"osd":1,"last update":"Sat Sep 14 03:53:33 2024","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.57099999999999995,"15min":0.57499999999999996},"min":{"1min":0.40200000000000002,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.76200000000000001,"5min":0.874,"15min":0.874},"last":0.625},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.55100000000000005,"15min":0.56000000000000005},"min":{"1min":0.377,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.66200000000000003,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.59999999999999998}]},{"osd":2,"last update":"Sat Sep 14 03:53:33 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.54800000000000004,"15min":0.54900000000000004},"min":{"1min":0.439,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.73299999999999998,"5min":0.73299999999999998,"15min":0.73299999999999998},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.54900000000000004,"15min":0.55300000000000005},"min":{"1min":0.40899999999999997,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.68700000000000006,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.56899999999999995}]},{"osd":3,"last update":"Sat Sep 14 03:53:33 2024","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.54500000000000004,"15min":0.55300000000000005},"min":{"1min":0.378,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.57099999999999995,"5min":0.57199999999999995,"15min":0.57299999999999995},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.52600000000000002}]},{"osd":4,"last update":"Sat Sep 14 03:53:33 2024","interfaces":[{"interface":"back","average":{"1min":0.46000000000000002,"5min":0.40000000000000002,"15min":0.376},"min":{"1min":0.316,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.40799999999999997,"15min":0.374},"min":{"1min":0.34399999999999997,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.66900000000000004,"5min":0.66900000000000004,"15min":0.66900000000000004},"last":0.58199999999999996}]},{"osd":6,"last update":"Sat Sep 14 03:53:53 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.438,"15min":0.40999999999999998},"min":{"1min":0.33200000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.749,"5min":0.749,"15min":0.749},"last":0.439},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.45000000000000001,"15min":0.42999999999999999},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.69899999999999995,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.55500000000000005}]},{"osd":7,"last update":"Sat Sep 14 03:53:18 2024","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.45300000000000001,"15min":0.443},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.51300000000000001,"15min":0.50600000000000001},"min":{"1min":0.38400000000000001,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.48299999999999998}]}]},{"osd":4,"up_from":31,"seq":133143986229,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27796,"kb_used_data":772,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717356,"statfs":{"total":95995035648,"available":95966572544,"internally_reserved":0,"allocated":790528,"data_stored":338145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:53:11 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.56299999999999994,"15min":0.58099999999999996},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.70799999999999996,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.56299999999999994,"15min":0.57099999999999995},"min":{"1min":0.40400000000000003,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.45900000000000002}]},{"osd":1,"last update":"Sat Sep 14 03:53:11 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.57399999999999995,"15min":0.58899999999999997},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.78800000000000003,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.53800000000000003,"15min":0.55400000000000005},"min":{"1min":0.34200000000000003,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.80100000000000005,"5min":0.86799999999999999,"15min":0.86799999999999999},"last":0.39900000000000002}]},{"osd":2,"last update":"Sat Sep 14 03:53:11 2024","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.56699999999999995,"15min":0.58399999999999996},"min":{"1min":0.38500000000000001,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.76300000000000001,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.55400000000000005,"15min":0.57199999999999995},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.70399999999999996,"5min":0.90100000000000002,"15min":0.90100000000000002},"last":0.41299999999999998}]},{"osd":3,"last update":"Sat Sep 14 03:53:11 2024","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.57699999999999996,"15min":0.59499999999999997},"min":{"1min":0.39300000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.78700000000000003,"5min":0.94799999999999995,"15min":0.94799999999999995},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.57299999999999995,"15min":0.58099999999999996},"min":{"1min":0.442,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.78900000000000003,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.505}]},{"osd":5,"last update":"Sat Sep 14 03:53:37 2024","interfaces":[{"interface":"back","average":{"1min":0.46200000000000002,"5min":0.42599999999999999,"15min":0.40600000000000003},"min":{"1min":0.28899999999999998,"5min":0.246,"15min":0.246},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.51400000000000001,"5min":0.45900000000000002,"15min":0.439},"min":{"1min":0.35199999999999998,"5min":0.316,"15min":0.316},"max":{"1min":0.629,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.49099999999999999}]},{"osd":6,"last update":"Sat Sep 14 03:53:59 2024","interfaces":[{"interface":"back","average":{"1min":0.45800000000000002,"5min":0.42899999999999999,"15min":0.42399999999999999},"min":{"1min":0.30599999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.63300000000000001,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.438},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.42599999999999999,"15min":0.42199999999999999},"min":{"1min":0.34899999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.47099999999999997}]},{"osd":7,"last update":"Sat Sep 14 03:53:16 2024","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.503,"15min":0.504},"min":{"1min":0.317,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.67600000000000005,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.58099999999999996},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.50600000000000001,"15min":0.505},"min":{"1min":0.36699999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.68500000000000005,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.63700000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149753,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27776,"kb_used_data":752,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717376,"statfs":{"total":95995035648,"available":95966593024,"internally_reserved":0,"allocated":770048,"data_stored":336525,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:53:46 2024","interfaces":[{"interface":"back","average":{"1min":0.379,"5min":0.38800000000000001,"15min":0.40000000000000002},"min":{"1min":0.20000000000000001,"5min":0.191,"15min":0.191},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.313},{"interface":"front","average":{"1min":0.33700000000000002,"5min":0.39500000000000002,"15min":0.41099999999999998},"min":{"1min":0.22900000000000001,"5min":0.221,"15min":0.221},"max":{"1min":0.51500000000000001,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.27500000000000002}]},{"osd":1,"last update":"Sat Sep 14 03:53:46 2024","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.40999999999999998,"15min":0.42799999999999999},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":0.49199999999999999,"5min":0.60099999999999998,"15min":0.60099999999999998},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.41299999999999998,"15min":0.43099999999999999},"min":{"1min":0.26100000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.69599999999999995,"5min":0.69599999999999995,"15min":0.69599999999999995},"last":0.41499999999999998}]},{"osd":2,"last update":"Sat Sep 14 03:53:46 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.44800000000000001,"15min":0.46200000000000002},"min":{"1min":0.29999999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.71499999999999997,"5min":0.71499999999999997,"15min":0.71499999999999997},"last":0.376},{"interface":"front","average":{"1min":0.378,"5min":0.42899999999999999,"15min":0.44800000000000001},"min":{"1min":0.26000000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.35599999999999998}]},{"osd":4,"last update":"Sat Sep 14 03:53:06 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.55900000000000005,"15min":0.55200000000000005},"min":{"1min":0.40100000000000002,"5min":0.378,"15min":0.378},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.626},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.55700000000000005,"15min":0.55400000000000005},"min":{"1min":0.35799999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.76500000000000001,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.51700000000000002}]},{"osd":5,"last update":"Sat Sep 14 03:53:30 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.57899999999999996,"15min":0.57799999999999996},"min":{"1min":0.372,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.70499999999999996,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.34999999999999998,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.79300000000000004,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.58299999999999996}]},{"osd":6,"last update":"Sat Sep 14 03:53:56 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.58799999999999997,"15min":0.59399999999999997},"min":{"1min":0.40699999999999997,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.73899999999999999,"5min":0.88400000000000001,"15min":0.88400000000000001},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.60499999999999998,"15min":0.60799999999999998},"min":{"1min":0.41599999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.80500000000000005,"5min":0.90900000000000003,"15min":0.90900000000000003},"last":0.48299999999999998}]},{"osd":7,"last update":"Sat Sep 14 03:53:19 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.60799999999999998,"15min":0.61799999999999999},"min":{"1min":0.39400000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.71399999999999997,"5min":0.96699999999999997,"15min":0.96699999999999997},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.622,"15min":0.628},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.79000000000000004,"5min":1.016,"15min":1.016},"last":0.65500000000000003}]}]},{"osd":2,"up_from":18,"seq":77309411390,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27780,"kb_used_data":756,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717372,"statfs":{"total":95995035648,"available":95966588928,"internally_reserved":0,"allocated":774144,"data_stored":337051,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:53:24 2024","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.39200000000000002,"15min":0.376},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.38900000000000001,"15min":0.39100000000000001},"min":{"1min":0.23200000000000001,"5min":0.22,"15min":0.22},"max":{"1min":0.53100000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"last":0.34399999999999997}]},{"osd":1,"last update":"Sat Sep 14 03:53:24 2024","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.39700000000000002,"15min":0.38600000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.32500000000000001},{"interface":"front","average":{"1min":0.42099999999999999,"5min":0.42399999999999999,"15min":0.40200000000000002},"min":{"1min":0.193,"5min":0.193,"15min":0.193},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.309}]},{"osd":3,"last update":"Sat Sep 14 03:53:46 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.439,"15min":0.44900000000000001},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.67200000000000004,"5min":0.67200000000000004,"15min":0.67200000000000004},"last":0.376},{"interface":"front","average":{"1min":0.45600000000000002,"5min":0.47199999999999998,"15min":0.48199999999999998},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.93300000000000005,"5min":0.93300000000000005,"15min":0.93300000000000005},"last":0.39900000000000002}]},{"osd":4,"last update":"Sat Sep 14 03:53:07 2024","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.79200000000000004,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.60899999999999999,"15min":0.61099999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.81100000000000005,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.59899999999999998}]},{"osd":5,"last update":"Sat Sep 14 03:53:32 2024","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.625,"15min":0.621},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.623,"5min":0.61299999999999999,"15min":0.60999999999999999},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.82199999999999995,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.627}]},{"osd":6,"last update":"Sat Sep 14 03:53:53 2024","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.61299999999999999,"15min":0.61099999999999999},"min":{"1min":0.376,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.75700000000000001,"5min":0.84299999999999997,"15min":0.84299999999999997},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.621,"15min":0.61599999999999999},"min":{"1min":0.42699999999999999,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.83999999999999997,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.54000000000000004}]},{"osd":7,"last update":"Sat Sep 14 03:53:21 2024","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.63100000000000001,"15min":0.628},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.81999999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.66100000000000003,"5min":0.64900000000000002,"15min":0.64700000000000002},"min":{"1min":0.42599999999999999,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.79700000000000004,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.58199999999999996}]}]},{"osd":1,"up_from":13,"seq":55834574915,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27772,"kb_used_data":748,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717380,"statfs":{"total":95995035648,"available":95966597120,"internally_reserved":0,"allocated":765952,"data_stored":336181,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 14 03:54:03 2024","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":0.35799999999999998,"15min":0.312},"min":{"1min":0.17100000000000001,"5min":0.17100000000000001,"15min":0.17100000000000001},"max":{"1min":0.95899999999999996,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.42499999999999999,"5min":0.38800000000000001,"15min":0.33700000000000002},"min":{"1min":0.23499999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":1.708,"5min":1.708,"15min":1.708},"last":0.27600000000000002}]},{"osd":2,"last update":"Sat Sep 14 03:53:20 2024","interfaces":[{"interface":"back","average":{"1min":0.36399999999999999,"5min":0.38900000000000001,"15min":0.372},"min":{"1min":0.26200000000000001,"5min":0.247,"15min":0.247},"max":{"1min":0.65200000000000002,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.379},{"interface":"front","average":{"1min":0.374,"5min":0.39500000000000002,"15min":0.38100000000000001},"min":{"1min":0.26900000000000002,"5min":0.249,"15min":0.249},"max":{"1min":0.60599999999999998,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.311}]},{"osd":3,"last update":"Sat Sep 14 03:53:48 2024","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.45100000000000001,"15min":0.44900000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":1.7769999999999999,"5min":1.7769999999999999,"15min":1.7769999999999999},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.44800000000000001,"5min":0.45300000000000001,"15min":0.45500000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.0329999999999999,"5min":1.0329999999999999,"15min":1.0329999999999999},"last":0.47499999999999998}]},{"osd":4,"last update":"Sat Sep 14 03:53:09 2024","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.55400000000000005,"15min":0.55200000000000005},"min":{"1min":0.374,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.89000000000000001,"5min":0.89000000000000001,"15min":0.89000000000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.45300000000000001,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.86599999999999999,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.61799999999999999}]},{"osd":5,"last update":"Sat Sep 14 03:53:33 2024","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.56899999999999995,"15min":0.56100000000000005},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":1.589,"5min":1.589,"15min":1.589},"last":0.55600000000000005},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.61199999999999999,"15min":0.60799999999999998},"min":{"1min":0.41599999999999998,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.8899999999999999,"5min":1.8899999999999999,"15min":1.8899999999999999},"last":0.52200000000000002}]},{"osd":6,"last update":"Sat Sep 14 03:53:55 2024","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":0.59999999999999998,"15min":0.58599999999999997},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":1.4450000000000001,"5min":1.4450000000000001,"15min":1.4450000000000001},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.59599999999999997,"15min":0.58599999999999997},"min":{"1min":0.42399999999999999,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":1.645,"5min":1.645,"15min":1.645},"last":0.53700000000000003}]},{"osd":7,"last update":"Sat Sep 14 03:53:16 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.56999999999999995,"15min":0.55900000000000005},"min":{"1min":0.41299999999999998,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.84699999999999998,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.622,"5min":0.60899999999999999,"15min":0.60699999999999998},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.84699999999999998,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.68899999999999995}]}]},{"osd":0,"up_from":8,"seq":34359738439,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28360,"kb_used_data":1336,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716792,"statfs":{"total":95995035648,"available":95965995008,"internally_reserved":0,"allocated":1368064,"data_stored":927407,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 14 03:54:00 2024","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":0.34699999999999998,"15min":0.32600000000000001},"min":{"1min":0.28000000000000003,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":0.51000000000000001,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.377,"5min":0.379,"15min":0.33500000000000002},"min":{"1min":0.27600000000000002,"5min":0.16800000000000001,"15min":0.16800000000000001},"max":{"1min":0.50900000000000001,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.39000000000000001}]},{"osd":2,"last update":"Sat Sep 14 03:53:25 2024","interfaces":[{"interface":"back","average":{"1min":0.38900000000000001,"5min":0.40799999999999997,"15min":0.40000000000000002},"min":{"1min":0.26800000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.54700000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.378,"5min":0.377,"15min":0.372},"min":{"1min":0.27500000000000002,"5min":0.214,"15min":0.214},"max":{"1min":0.60299999999999998,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.35399999999999998}]},{"osd":3,"last update":"Sat Sep 14 03:53:49 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.438,"15min":0.442},"min":{"1min":0.29499999999999998,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.50600000000000001,"5min":0.622,"15min":0.622},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.434,"15min":0.44700000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.51400000000000001,"5min":0.65900000000000003,"15min":0.65900000000000003},"last":0.32600000000000001}]},{"osd":4,"last update":"Sat Sep 14 03:53:07 2024","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.55900000000000005,"15min":0.55300000000000005},"min":{"1min":0.38200000000000001,"5min":0.378,"15min":0.378},"max":{"1min":0.81499999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.749},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.53100000000000003,"15min":0.52500000000000002},"min":{"1min":0.40000000000000002,"5min":0.38,"15min":0.38},"max":{"1min":0.76800000000000002,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.64200000000000002}]},{"osd":5,"last update":"Sat Sep 14 03:53:34 2024","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.59899999999999998,"15min":0.60599999999999998},"min":{"1min":0.43099999999999999,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.752,"5min":0.96899999999999997,"15min":0.96899999999999997},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.59099999999999997,"15min":0.59999999999999998},"min":{"1min":0.40799999999999997,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.71399999999999997,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.69699999999999995}]},{"osd":6,"last update":"Sat Sep 14 03:53:55 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.47599999999999998,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.80600000000000005,"5min":0.91400000000000003,"15min":0.91400000000000003},"last":0.624},{"interface":"front","average":{"1min":0.63800000000000001,"5min":0.621,"15min":0.627},"min":{"1min":0.53000000000000003,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.71599999999999997}]},{"osd":7,"last update":"Sat Sep 14 03:53:21 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.44600000000000001,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.77300000000000002,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.72599999999999998},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.44900000000000001,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.79700000000000004,"5min":0.98599999999999999,"15min":0.98599999999999999},"last":0.60299999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T03:54:10.684 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-14T03:54:10.684 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-14T03:54:10.685 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-14T03:54:10.685 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-14T03:54:10.899 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:54:11.020 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:10 smithi039 ceph-mon[29586]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:54:11.024 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:10 smithi039 ceph-mon[29586]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:11.024 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:10 smithi039 ceph-mon[40962]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:54:11.024 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:10 smithi039 ceph-mon[40962]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:11.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:10 smithi129 ceph-mon[33173]: from='client.14772 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T03:54:11.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:10 smithi129 ceph-mon[33173]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:12.778 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T03:54:12.778 INFO:teuthology.orchestra.run.smithi039.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-14T03:54:13.135 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:12 smithi039 ceph-mon[29586]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:13.135 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:12 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/3121498547' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-14T03:54:13.136 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:12 smithi039 ceph-mon[40962]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:13.136 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:12 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/3121498547' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-14T03:54:13.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:12 smithi129 ceph-mon[33173]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:13.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:12 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/3121498547' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-09-14T03:54:13.499 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-14T03:54:13.500 INFO:teuthology.run_tasks:Running task print... 2024-09-14T03:54:13.510 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-09-14T03:54:13.510 INFO:teuthology.run_tasks:Running task parallel... 2024-09-14T03:54:13.527 INFO:teuthology.task.parallel:starting parallel... 2024-09-14T03:54:13.528 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-14T03:54:13.528 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-09-14T03:54:13.534 INFO:tasks.workunit:Pulling workunits from ref reef 2024-09-14T03:54:13.534 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-14T03:54:13.534 DEBUG:teuthology.orchestra.run.smithi039:> stat -- /home/ubuntu/cephtest/mnt.0 2024-09-14T03:54:13.535 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-14T03:54:13.535 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-09-14T03:54:13.535 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi039.front.sepia.ceph.com 2024-09-14T03:54:13.535 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-09-14T03:54:13.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T03:54:13.553 INFO:teuthology.orchestra.run.smithi039.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-09-14T03:54:13.553 DEBUG:teuthology.orchestra.run.smithi039:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-09-14T03:54:13.610 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-09-14T03:54:13.610 DEBUG:teuthology.orchestra.run.smithi039:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-09-14T03:54:13.665 INFO:tasks.workunit:timeout=3h 2024-09-14T03:54:13.666 INFO:tasks.workunit:cleanup=True 2024-09-14T03:54:13.666 DEBUG:teuthology.orchestra.run.smithi039:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch reef https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-09-14T03:54:13.723 INFO:tasks.workunit.client.0.smithi039.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-09-14T03:54:13.746 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:54:15.026 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:14 smithi039 ceph-mon[29586]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:15.026 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:14 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:54:15.026 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:14 smithi039 ceph-mon[40962]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:15.026 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:14 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:54:15.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:14 smithi129 ceph-mon[33173]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:15.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:14 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:54:16.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:54:15 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:54:15] ENGINE Bus STOPPING 2024-09-14T03:54:16.485 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:54:16 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:54:16] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:54:16.485 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:54:16 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:54:16] ENGINE Bus STOPPED 2024-09-14T03:54:16.485 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:54:16 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:54:16] ENGINE Bus STARTING 2024-09-14T03:54:16.485 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:54:16 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:54:16] ENGINE Serving on http://:::9283 2024-09-14T03:54:16.485 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:54:16 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:54:16] ENGINE Bus STARTED 2024-09-14T03:54:16.506 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'ceph config set global log_to_journald false --force' 2024-09-14T03:54:16.754 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:54:16.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:16 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/852310927' entity='client.admin' 2024-09-14T03:54:16.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:16 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:54:16.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:16 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:54:16.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:16 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:54:16.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:16 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:54:16.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:16 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/852310927' entity='client.admin' 2024-09-14T03:54:16.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:16 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:54:16.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:16 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:54:16.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:16 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:54:16.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:16 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:54:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:16 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/852310927' entity='client.admin' 2024-09-14T03:54:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:16 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:54:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:16 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:54:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:16 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:54:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:16 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:54:18.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:17 smithi129 ceph-mon[33173]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:18.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:17 smithi039 ceph-mon[29586]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:18.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:17 smithi039 ceph-mon[40962]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:19.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:54:19 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:54:19] "GET /metrics HTTP/1.1" 200 33851 "" "Prometheus/2.43.0" 2024-09-14T03:54:19.693 DEBUG:teuthology.orchestra.run.smithi039:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-09-14T03:54:19.737 INFO:tasks.workunit.client.0.smithi039.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-14T03:54:19.739 INFO:tasks.workunit.client.0.smithi039.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-14T03:54:19.739 INFO:tasks.workunit.client.0.smithi039.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-14T03:54:20.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:19 smithi129 ceph-mon[33173]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:20.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:19 smithi039 ceph-mon[29586]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:20.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:19 smithi039 ceph-mon[40962]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:20.144 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'echo wait for mgr daemons to upgrade' 2024-09-14T03:54:20.349 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:54:21.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:20 smithi129 ceph-mon[33173]: pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:21.289 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:20 smithi039 ceph-mon[29586]: pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:21.289 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:20 smithi039 ceph-mon[40962]: pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:21.846 INFO:tasks.workunit.client.0.smithi039.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-14T03:54:21.918 INFO:tasks.workunit.client.0.smithi039.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-14T03:54:21.960 INFO:tasks.workunit.client.0.smithi039.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-14T03:54:21.961 INFO:tasks.workunit.client.0.smithi039.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-14T03:54:21.962 INFO:tasks.workunit.client.0.smithi039.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-14T03:54:22.001 INFO:tasks.workunit.client.0.smithi039.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-14T03:54:22.005 DEBUG:teuthology.orchestra.run.smithi039:> set -ex 2024-09-14T03:54:22.005 DEBUG:teuthology.orchestra.run.smithi039:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-09-14T03:54:22.021 INFO:tasks.workunit:Running workunits matching rbd/test_librbd.sh on client.0... 2024-09-14T03:54:22.022 INFO:tasks.workunit:Running workunit rbd/test_librbd.sh... 2024-09-14T03:54:22.023 DEBUG:teuthology.orchestra.run.smithi039:workunit test rbd/test_librbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_FEATURES=61 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/test_librbd.sh 2024-09-14T03:54:22.173 INFO:tasks.workunit.client.0.smithi039.stderr:warning: failed to enable lockdep 2024-09-14T03:54:22.173 INFO:tasks.workunit.client.0.smithi039.stdout:seed 85658 2024-09-14T03:54:22.173 INFO:tasks.workunit.client.0.smithi039.stdout:[==========] Running 490 tests from 24 test suites. 2024-09-14T03:54:22.173 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] Global test environment set-up. 2024-09-14T03:54:22.174 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 1 test from TestJournalStress 2024-09-14T03:54:22.403 INFO:teuthology.orchestra.run.smithi039.stdout:wait for mgr daemons to upgrade 2024-09-14T03:54:23.071 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mgr' 2024-09-14T03:54:23.276 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:54:23.720 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestJournalStress.DiscardWithPruneWriteOverlap 2024-09-14T03:54:24.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:23 smithi129 ceph-mon[33173]: osdmap e75: 8 total, 8 up, 8 in 2024-09-14T03:54:24.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:23 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/134524751' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:24.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:23 smithi129 ceph-mon[33173]: pgmap v97: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:24.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:23 smithi039 ceph-mon[29586]: osdmap e75: 8 total, 8 up, 8 in 2024-09-14T03:54:24.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:23 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/134524751' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:24.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:23 smithi039 ceph-mon[29586]: pgmap v97: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:24.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:23 smithi039 ceph-mon[40962]: osdmap e75: 8 total, 8 up, 8 in 2024-09-14T03:54:24.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:23 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/134524751' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:24.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:23 smithi039 ceph-mon[40962]: pgmap v97: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:24.972 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:24 smithi039 ceph-mon[29586]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:54:24.973 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:24 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/134524751' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:24.973 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:24 smithi039 ceph-mon[29586]: osdmap e76: 8 total, 8 up, 8 in 2024-09-14T03:54:24.973 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:24 smithi039 ceph-mon[40962]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:54:24.973 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:24 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/134524751' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:24.973 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:24 smithi039 ceph-mon[40962]: osdmap e76: 8 total, 8 up, 8 in 2024-09-14T03:54:25.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:24 smithi129 ceph-mon[33173]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:54:25.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:24 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/134524751' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:25.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:24 smithi129 ceph-mon[33173]: osdmap e76: 8 total, 8 up, 8 in 2024-09-14T03:54:25.753 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/journal/test_Stress.cc:37: Skipped 2024-09-14T03:54:25.753 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T03:54:25.753 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalStress.DiscardWithPruneWriteOverlap (2033 ms) 2024-09-14T03:54:26.006 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:25 smithi039 ceph-mon[29586]: osdmap e77: 8 total, 8 up, 8 in 2024-09-14T03:54:26.006 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:25 smithi039 ceph-mon[29586]: pgmap v100: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:26.006 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:25 smithi039 ceph-mon[40962]: osdmap e77: 8 total, 8 up, 8 in 2024-09-14T03:54:26.006 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:25 smithi039 ceph-mon[40962]: pgmap v100: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:25 smithi129 ceph-mon[33173]: osdmap e77: 8 total, 8 up, 8 in 2024-09-14T03:54:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:25 smithi129 ceph-mon[33173]: pgmap v100: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 219 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:26.763 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 1 test from TestJournalStress (2033 ms total) 2024-09-14T03:54:26.764 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T03:54:26.764 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 16 tests from TestJournalReplay 2024-09-14T03:54:27.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:26 smithi129 ceph-mon[33173]: from='client.24719 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:54:27.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:26 smithi129 ceph-mon[33173]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:54:27.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:26 smithi129 ceph-mon[33173]: Cluster is now healthy 2024-09-14T03:54:27.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:26 smithi129 ceph-mon[33173]: osdmap e78: 8 total, 8 up, 8 in 2024-09-14T03:54:27.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:26 smithi039 ceph-mon[29586]: from='client.24719 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:54:27.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:26 smithi039 ceph-mon[29586]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:54:27.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:26 smithi039 ceph-mon[29586]: Cluster is now healthy 2024-09-14T03:54:27.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:26 smithi039 ceph-mon[29586]: osdmap e78: 8 total, 8 up, 8 in 2024-09-14T03:54:27.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:26 smithi039 ceph-mon[40962]: from='client.24719 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "daemon_types": "mgr", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:54:27.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:26 smithi039 ceph-mon[40962]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:54:27.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:26 smithi039 ceph-mon[40962]: Cluster is now healthy 2024-09-14T03:54:27.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:26 smithi039 ceph-mon[40962]: osdmap e78: 8 total, 8 up, 8 in 2024-09-14T03:54:28.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:27 smithi039 ceph-mon[29586]: osdmap e79: 8 total, 8 up, 8 in 2024-09-14T03:54:28.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:27 smithi039 ceph-mon[29586]: pgmap v103: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:28.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:27 smithi039 ceph-mon[40962]: osdmap e79: 8 total, 8 up, 8 in 2024-09-14T03:54:28.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:27 smithi039 ceph-mon[40962]: pgmap v103: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:28.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:27 smithi129 ceph-mon[33173]: osdmap e79: 8 total, 8 up, 8 in 2024-09-14T03:54:28.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:27 smithi129 ceph-mon[33173]: pgmap v103: 73 pgs: 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:28.818 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestJournalReplay.AioDiscardEvent 2024-09-14T03:54:29.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:28 smithi129 ceph-mon[33173]: osdmap e80: 8 total, 8 up, 8 in 2024-09-14T03:54:29.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:28 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/4130902184' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:29.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:28 smithi039 ceph-mon[40962]: osdmap e80: 8 total, 8 up, 8 in 2024-09-14T03:54:29.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:28 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/4130902184' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:29.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:28 smithi039 ceph-mon[29586]: osdmap e80: 8 total, 8 up, 8 in 2024-09-14T03:54:29.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:28 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/4130902184' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-2","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:29.291 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:54:29 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:54:29] "GET /metrics HTTP/1.1" 200 33851 "" "Prometheus/2.43.0" 2024-09-14T03:54:30.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:29 smithi129 ceph-mon[33173]: pgmap v105: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:30.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:29 smithi129 ceph-mon[33173]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:54:30.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:29 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/4130902184' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:30.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:29 smithi129 ceph-mon[33173]: osdmap e81: 8 total, 8 up, 8 in 2024-09-14T03:54:30.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:29 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:54:30.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:29 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:54:30.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:29 smithi039 ceph-mon[29586]: pgmap v105: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:30.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:29 smithi039 ceph-mon[29586]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:54:30.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:29 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/4130902184' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:30.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:29 smithi039 ceph-mon[29586]: osdmap e81: 8 total, 8 up, 8 in 2024-09-14T03:54:30.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:29 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:54:30.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:29 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:54:30.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:29 smithi039 ceph-mon[40962]: pgmap v105: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:30.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:29 smithi039 ceph-mon[40962]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:54:30.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:29 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/4130902184' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-2","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:30.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:29 smithi039 ceph-mon[40962]: osdmap e81: 8 total, 8 up, 8 in 2024-09-14T03:54:30.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:29 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:54:30.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:29 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:54:30.830 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/journal/test_Replay.cc:109: Skipped 2024-09-14T03:54:30.830 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T03:54:30.830 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.AioDiscardEvent (2011 ms) 2024-09-14T03:54:30.830 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestJournalReplay.AioWriteEvent 2024-09-14T03:54:30.840 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/journal/test_Replay.cc:200: Skipped 2024-09-14T03:54:30.840 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T03:54:30.840 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.AioWriteEvent (11 ms) 2024-09-14T03:54:30.840 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestJournalReplay.AioFlushEvent 2024-09-14T03:54:30.849 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/journal/test_Replay.cc:261: Skipped 2024-09-14T03:54:30.849 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T03:54:30.849 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.AioFlushEvent (9 ms) 2024-09-14T03:54:30.849 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestJournalReplay.SnapCreate 2024-09-14T03:54:30.857 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/journal/test_Replay.cc:307: Skipped 2024-09-14T03:54:30.857 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T03:54:30.858 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.SnapCreate (8 ms) 2024-09-14T03:54:30.858 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestJournalReplay.SnapProtect 2024-09-14T03:54:30.865 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/journal/test_Replay.cc:348: Skipped 2024-09-14T03:54:30.865 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T03:54:30.865 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.SnapProtect (8 ms) 2024-09-14T03:54:30.865 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestJournalReplay.SnapUnprotect 2024-09-14T03:54:30.873 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/journal/test_Replay.cc:394: Skipped 2024-09-14T03:54:30.873 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T03:54:30.873 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.SnapUnprotect (7 ms) 2024-09-14T03:54:30.873 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestJournalReplay.SnapRename 2024-09-14T03:54:30.881 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/journal/test_Replay.cc:450: Skipped 2024-09-14T03:54:30.881 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T03:54:30.881 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.SnapRename (9 ms) 2024-09-14T03:54:30.881 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestJournalReplay.SnapRollback 2024-09-14T03:54:30.889 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/journal/test_Replay.cc:500: Skipped 2024-09-14T03:54:30.889 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T03:54:30.889 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.SnapRollback (8 ms) 2024-09-14T03:54:30.889 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestJournalReplay.SnapRemove 2024-09-14T03:54:30.896 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/journal/test_Replay.cc:541: Skipped 2024-09-14T03:54:30.896 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T03:54:30.896 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.SnapRemove (7 ms) 2024-09-14T03:54:30.896 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestJournalReplay.Rename 2024-09-14T03:54:30.904 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/journal/test_Replay.cc:590: Skipped 2024-09-14T03:54:30.904 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T03:54:30.904 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.Rename (7 ms) 2024-09-14T03:54:30.905 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestJournalReplay.Resize 2024-09-14T03:54:30.911 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/journal/test_Replay.cc:624: Skipped 2024-09-14T03:54:30.911 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T03:54:30.911 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.Resize (8 ms) 2024-09-14T03:54:30.911 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestJournalReplay.Flatten 2024-09-14T03:54:30.918 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/journal/test_Replay.cc:657: Skipped 2024-09-14T03:54:30.919 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T03:54:30.919 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.Flatten (7 ms) 2024-09-14T03:54:30.919 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestJournalReplay.UpdateFeatures 2024-09-14T03:54:30.926 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/journal/test_Replay.cc:704: Skipped 2024-09-14T03:54:30.926 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T03:54:30.926 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.UpdateFeatures (7 ms) 2024-09-14T03:54:30.926 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestJournalReplay.MetadataSet 2024-09-14T03:54:30.933 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/journal/test_Replay.cc:741: Skipped 2024-09-14T03:54:30.933 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T03:54:30.933 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.MetadataSet (8 ms) 2024-09-14T03:54:30.933 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestJournalReplay.MetadataRemove 2024-09-14T03:54:30.941 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/journal/test_Replay.cc:781: Skipped 2024-09-14T03:54:30.941 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T03:54:30.941 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.MetadataRemove (7 ms) 2024-09-14T03:54:30.941 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestJournalReplay.ObjectPosition 2024-09-14T03:54:30.948 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/journal/test_Replay.cc:824: Skipped 2024-09-14T03:54:30.949 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T03:54:30.949 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.ObjectPosition (8 ms) 2024-09-14T03:54:31.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:30 smithi129 ceph-mon[33173]: osdmap e82: 8 total, 8 up, 8 in 2024-09-14T03:54:31.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:30 smithi129 ceph-mon[33173]: pgmap v108: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:31.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:30 smithi039 ceph-mon[29586]: osdmap e82: 8 total, 8 up, 8 in 2024-09-14T03:54:31.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:30 smithi039 ceph-mon[29586]: pgmap v108: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:31.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:30 smithi039 ceph-mon[40962]: osdmap e82: 8 total, 8 up, 8 in 2024-09-14T03:54:31.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:30 smithi039 ceph-mon[40962]: pgmap v108: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 223 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:31.829 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 16 tests from TestJournalReplay (2130 ms total) 2024-09-14T03:54:31.829 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T03:54:31.830 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 4 tests from TestJournalEntries 2024-09-14T03:54:32.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:31 smithi129 ceph-mon[33173]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:54:32.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:31 smithi129 ceph-mon[33173]: Cluster is now healthy 2024-09-14T03:54:32.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:31 smithi129 ceph-mon[33173]: osdmap e83: 8 total, 8 up, 8 in 2024-09-14T03:54:32.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:31 smithi039 ceph-mon[29586]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:54:32.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:31 smithi039 ceph-mon[29586]: Cluster is now healthy 2024-09-14T03:54:32.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:31 smithi039 ceph-mon[29586]: osdmap e83: 8 total, 8 up, 8 in 2024-09-14T03:54:32.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:31 smithi039 ceph-mon[40962]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:54:32.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:31 smithi039 ceph-mon[40962]: Cluster is now healthy 2024-09-14T03:54:32.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:31 smithi039 ceph-mon[40962]: osdmap e83: 8 total, 8 up, 8 in 2024-09-14T03:54:33.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:32 smithi129 ceph-mon[33173]: osdmap e84: 8 total, 8 up, 8 in 2024-09-14T03:54:33.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:32 smithi129 ceph-mon[33173]: pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:33.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:32 smithi039 ceph-mon[29586]: osdmap e84: 8 total, 8 up, 8 in 2024-09-14T03:54:33.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:32 smithi039 ceph-mon[29586]: pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:33.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:32 smithi039 ceph-mon[40962]: osdmap e84: 8 total, 8 up, 8 in 2024-09-14T03:54:33.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:32 smithi039 ceph-mon[40962]: pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:33.857 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestJournalEntries.AioWrite 2024-09-14T03:54:34.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:33 smithi129 ceph-mon[33173]: osdmap e85: 8 total, 8 up, 8 in 2024-09-14T03:54:34.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:33 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/3157247971' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:34.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:33 smithi129 ceph-mon[33173]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:34.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:33 smithi039 ceph-mon[29586]: osdmap e85: 8 total, 8 up, 8 in 2024-09-14T03:54:34.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:33 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/3157247971' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:34.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:33 smithi039 ceph-mon[29586]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:34.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:33 smithi039 ceph-mon[40962]: osdmap e85: 8 total, 8 up, 8 in 2024-09-14T03:54:34.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:33 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/3157247971' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:34.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:33 smithi039 ceph-mon[40962]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-3","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:35.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:34 smithi129 ceph-mon[33173]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:35.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:34 smithi129 ceph-mon[33173]: osdmap e86: 8 total, 8 up, 8 in 2024-09-14T03:54:35.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:34 smithi129 ceph-mon[33173]: pgmap v114: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:35.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:34 smithi039 ceph-mon[29586]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:35.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:34 smithi039 ceph-mon[29586]: osdmap e86: 8 total, 8 up, 8 in 2024-09-14T03:54:35.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:34 smithi039 ceph-mon[29586]: pgmap v114: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:35.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:34 smithi039 ceph-mon[40962]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-3","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:35.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:34 smithi039 ceph-mon[40962]: osdmap e86: 8 total, 8 up, 8 in 2024-09-14T03:54:35.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:34 smithi039 ceph-mon[40962]: pgmap v114: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 227 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:35.888 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/journal/test_Entries.cc:118: Skipped 2024-09-14T03:54:35.888 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T03:54:35.888 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalEntries.AioWrite (2030 ms) 2024-09-14T03:54:35.888 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestJournalEntries.AioDiscard 2024-09-14T03:54:35.897 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/journal/test_Entries.cc:163: Skipped 2024-09-14T03:54:35.898 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T03:54:35.898 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalEntries.AioDiscard (10 ms) 2024-09-14T03:54:35.898 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestJournalEntries.AioDiscardWithPrune 2024-09-14T03:54:35.906 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/journal/test_Entries.cc:200: Skipped 2024-09-14T03:54:35.907 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T03:54:35.907 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalEntries.AioDiscardWithPrune (9 ms) 2024-09-14T03:54:35.907 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestJournalEntries.AioFlush 2024-09-14T03:54:35.915 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/journal/test_Entries.cc:263: Skipped 2024-09-14T03:54:35.915 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T03:54:35.915 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalEntries.AioFlush (8 ms) 2024-09-14T03:54:36.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:35 smithi129 ceph-mon[33173]: osdmap e87: 8 total, 8 up, 8 in 2024-09-14T03:54:36.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:35 smithi039 ceph-mon[29586]: osdmap e87: 8 total, 8 up, 8 in 2024-09-14T03:54:36.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:35 smithi039 ceph-mon[40962]: osdmap e87: 8 total, 8 up, 8 in 2024-09-14T03:54:36.896 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 4 tests from TestJournalEntries (2057 ms total) 2024-09-14T03:54:36.896 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T03:54:36.896 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 2 tests from TestTrash 2024-09-14T03:54:37.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:36 smithi129 ceph-mon[33173]: osdmap e88: 8 total, 8 up, 8 in 2024-09-14T03:54:37.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:36 smithi129 ceph-mon[33173]: pgmap v117: 105 pgs: 105 active+clean; 579 KiB data, 228 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:54:37.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:36 smithi039 ceph-mon[29586]: osdmap e88: 8 total, 8 up, 8 in 2024-09-14T03:54:37.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:36 smithi039 ceph-mon[29586]: pgmap v117: 105 pgs: 105 active+clean; 579 KiB data, 228 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:54:37.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:36 smithi039 ceph-mon[40962]: osdmap e88: 8 total, 8 up, 8 in 2024-09-14T03:54:37.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:36 smithi039 ceph-mon[40962]: pgmap v117: 105 pgs: 105 active+clean; 579 KiB data, 228 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:54:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:37 smithi129 ceph-mon[33173]: osdmap e89: 8 total, 8 up, 8 in 2024-09-14T03:54:38.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:37 smithi039 ceph-mon[29586]: osdmap e89: 8 total, 8 up, 8 in 2024-09-14T03:54:38.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:37 smithi039 ceph-mon[40962]: osdmap e89: 8 total, 8 up, 8 in 2024-09-14T03:54:38.935 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestTrash.UserRemovingSource 2024-09-14T03:54:39.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:38 smithi129 ceph-mon[33173]: osdmap e90: 8 total, 8 up, 8 in 2024-09-14T03:54:39.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:38 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/2700420976' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:39.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:38 smithi129 ceph-mon[33173]: pgmap v120: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 228 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:39.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:54:39 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:54:39] "GET /metrics HTTP/1.1" 200 36227 "" "Prometheus/2.43.0" 2024-09-14T03:54:39.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:38 smithi039 ceph-mon[29586]: osdmap e90: 8 total, 8 up, 8 in 2024-09-14T03:54:39.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:38 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/2700420976' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:39.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:38 smithi039 ceph-mon[29586]: pgmap v120: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 228 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:39.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:38 smithi039 ceph-mon[40962]: osdmap e90: 8 total, 8 up, 8 in 2024-09-14T03:54:39.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:38 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/2700420976' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-4","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:39.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:38 smithi039 ceph-mon[40962]: pgmap v120: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 228 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:40.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:39 smithi129 ceph-mon[33173]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:54:40.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:39 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/2700420976' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:40.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:39 smithi129 ceph-mon[33173]: osdmap e91: 8 total, 8 up, 8 in 2024-09-14T03:54:40.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:39 smithi039 ceph-mon[29586]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:54:40.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:39 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/2700420976' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:40.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:39 smithi039 ceph-mon[29586]: osdmap e91: 8 total, 8 up, 8 in 2024-09-14T03:54:40.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:39 smithi039 ceph-mon[40962]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:54:40.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:39 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/2700420976' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-4","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:40.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:39 smithi039 ceph-mon[40962]: osdmap e91: 8 total, 8 up, 8 in 2024-09-14T03:54:41.055 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestTrash.UserRemovingSource (2121 ms) 2024-09-14T03:54:41.055 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestTrash.RestoreMirroringSource 2024-09-14T03:54:41.093 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestTrash.RestoreMirroringSource (38 ms) 2024-09-14T03:54:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:40 smithi129 ceph-mon[33173]: osdmap e92: 8 total, 8 up, 8 in 2024-09-14T03:54:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:40 smithi129 ceph-mon[33173]: pgmap v123: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 228 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:41.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:40 smithi039 ceph-mon[29586]: osdmap e92: 8 total, 8 up, 8 in 2024-09-14T03:54:41.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:40 smithi039 ceph-mon[29586]: pgmap v123: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 228 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:41.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:40 smithi039 ceph-mon[40962]: osdmap e92: 8 total, 8 up, 8 in 2024-09-14T03:54:41.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:40 smithi039 ceph-mon[40962]: pgmap v123: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 228 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:41.977 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 2 tests from TestTrash (2159 ms total) 2024-09-14T03:54:41.977 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T03:54:41.978 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 1 test from TestOperations 2024-09-14T03:54:42.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:41 smithi129 ceph-mon[33173]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:54:42.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:41 smithi129 ceph-mon[33173]: Cluster is now healthy 2024-09-14T03:54:42.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:41 smithi129 ceph-mon[33173]: osdmap e93: 8 total, 8 up, 8 in 2024-09-14T03:54:42.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:41 smithi039 ceph-mon[29586]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:54:42.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:41 smithi039 ceph-mon[29586]: Cluster is now healthy 2024-09-14T03:54:42.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:41 smithi039 ceph-mon[29586]: osdmap e93: 8 total, 8 up, 8 in 2024-09-14T03:54:42.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:41 smithi039 ceph-mon[40962]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:54:42.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:41 smithi039 ceph-mon[40962]: Cluster is now healthy 2024-09-14T03:54:42.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:41 smithi039 ceph-mon[40962]: osdmap e93: 8 total, 8 up, 8 in 2024-09-14T03:54:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:42 smithi129 ceph-mon[33173]: osdmap e94: 8 total, 8 up, 8 in 2024-09-14T03:54:43.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:42 smithi129 ceph-mon[33173]: pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 236 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:43.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:42 smithi039 ceph-mon[29586]: osdmap e94: 8 total, 8 up, 8 in 2024-09-14T03:54:43.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:42 smithi039 ceph-mon[29586]: pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 236 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:43.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:42 smithi039 ceph-mon[40962]: osdmap e94: 8 total, 8 up, 8 in 2024-09-14T03:54:43.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:42 smithi039 ceph-mon[40962]: pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 236 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:44.009 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestOperations.DisableJournalingCorrupt 2024-09-14T03:54:44.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:43 smithi129 ceph-mon[33173]: osdmap e95: 8 total, 8 up, 8 in 2024-09-14T03:54:44.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:43 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/675724790' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:44.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:43 smithi129 ceph-mon[33173]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:44.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:43 smithi039 ceph-mon[29586]: osdmap e95: 8 total, 8 up, 8 in 2024-09-14T03:54:44.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:43 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/675724790' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:44.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:43 smithi039 ceph-mon[29586]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:44.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:43 smithi039 ceph-mon[40962]: osdmap e95: 8 total, 8 up, 8 in 2024-09-14T03:54:44.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:44 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/675724790' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:44.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:44 smithi039 ceph-mon[40962]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-5","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:45.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:45 smithi039 ceph-mon[29586]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:45.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:45 smithi039 ceph-mon[29586]: osdmap e96: 8 total, 8 up, 8 in 2024-09-14T03:54:45.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:45 smithi039 ceph-mon[29586]: pgmap v129: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 236 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:45.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:45 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:54:45.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:45 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:54:45.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:45 smithi039 ceph-mon[40962]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:45.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:45 smithi039 ceph-mon[40962]: osdmap e96: 8 total, 8 up, 8 in 2024-09-14T03:54:45.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:45 smithi039 ceph-mon[40962]: pgmap v129: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 236 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:45.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:45 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:54:45.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:45 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:54:45.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:45 smithi129 ceph-mon[33173]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-5","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:45.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:45 smithi129 ceph-mon[33173]: osdmap e96: 8 total, 8 up, 8 in 2024-09-14T03:54:45.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:45 smithi129 ceph-mon[33173]: pgmap v129: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 236 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:45.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:45 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:54:45.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:45 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:54:46.049 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/test_Operations.cc:18: Skipped 2024-09-14T03:54:46.049 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T03:54:46.049 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestOperations.DisableJournalingCorrupt (2040 ms) 2024-09-14T03:54:46.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:46 smithi039 ceph-mon[29586]: osdmap e97: 8 total, 8 up, 8 in 2024-09-14T03:54:46.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:46 smithi039 ceph-mon[40962]: osdmap e97: 8 total, 8 up, 8 in 2024-09-14T03:54:46.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:46 smithi129 ceph-mon[33173]: osdmap e97: 8 total, 8 up, 8 in 2024-09-14T03:54:47.073 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 1 test from TestOperations (2040 ms total) 2024-09-14T03:54:47.073 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T03:54:47.073 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 4 tests from TestObjectMap 2024-09-14T03:54:47.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:47 smithi129 ceph-mon[33173]: osdmap e98: 8 total, 8 up, 8 in 2024-09-14T03:54:47.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:47 smithi129 ceph-mon[33173]: pgmap v132: 105 pgs: 105 active+clean; 579 KiB data, 237 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:54:47.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:47 smithi039 ceph-mon[40962]: osdmap e98: 8 total, 8 up, 8 in 2024-09-14T03:54:47.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:47 smithi039 ceph-mon[40962]: pgmap v132: 105 pgs: 105 active+clean; 579 KiB data, 237 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:54:47.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:47 smithi039 ceph-mon[29586]: osdmap e98: 8 total, 8 up, 8 in 2024-09-14T03:54:47.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:47 smithi039 ceph-mon[29586]: pgmap v132: 105 pgs: 105 active+clean; 579 KiB data, 237 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:54:48.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:48 smithi129 ceph-mon[33173]: osdmap e99: 8 total, 8 up, 8 in 2024-09-14T03:54:48.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:48 smithi039 ceph-mon[29586]: osdmap e99: 8 total, 8 up, 8 in 2024-09-14T03:54:48.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:48 smithi039 ceph-mon[40962]: osdmap e99: 8 total, 8 up, 8 in 2024-09-14T03:54:49.090 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestObjectMap.RefreshInvalidatesWhenCorrupt 2024-09-14T03:54:49.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:54:49 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:54:49] "GET /metrics HTTP/1.1" 200 36232 "" "Prometheus/2.43.0" 2024-09-14T03:54:49.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:49 smithi039 ceph-mon[29586]: osdmap e100: 8 total, 8 up, 8 in 2024-09-14T03:54:49.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:49 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/3823174956' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:49.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:49 smithi039 ceph-mon[29586]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:49.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:49 smithi039 ceph-mon[29586]: pgmap v135: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:49.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:49 smithi039 ceph-mon[40962]: osdmap e100: 8 total, 8 up, 8 in 2024-09-14T03:54:49.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:49 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/3823174956' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:49.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:49 smithi039 ceph-mon[40962]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:49.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:49 smithi039 ceph-mon[40962]: pgmap v135: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:49.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:49 smithi129 ceph-mon[33173]: osdmap e100: 8 total, 8 up, 8 in 2024-09-14T03:54:49.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:49 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/3823174956' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:49.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:49 smithi129 ceph-mon[33173]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-6","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:49.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:49 smithi129 ceph-mon[33173]: pgmap v135: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:50.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:50 smithi129 ceph-mon[33173]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:54:50.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:50 smithi129 ceph-mon[33173]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:50.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:50 smithi129 ceph-mon[33173]: osdmap e101: 8 total, 8 up, 8 in 2024-09-14T03:54:50.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:50 smithi039 ceph-mon[29586]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:54:50.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:50 smithi039 ceph-mon[29586]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:50.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:50 smithi039 ceph-mon[29586]: osdmap e101: 8 total, 8 up, 8 in 2024-09-14T03:54:50.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:50 smithi039 ceph-mon[40962]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:54:50.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:50 smithi039 ceph-mon[40962]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-6","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:50.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:50 smithi039 ceph-mon[40962]: osdmap e101: 8 total, 8 up, 8 in 2024-09-14T03:54:51.168 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestObjectMap.RefreshInvalidatesWhenCorrupt (2080 ms) 2024-09-14T03:54:51.168 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestObjectMap.RefreshInvalidatesWhenTooSmall 2024-09-14T03:54:51.197 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestObjectMap.RefreshInvalidatesWhenTooSmall (29 ms) 2024-09-14T03:54:51.197 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestObjectMap.InvalidateFlagOnDisk 2024-09-14T03:54:51.238 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestObjectMap.InvalidateFlagOnDisk (40 ms) 2024-09-14T03:54:51.238 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestObjectMap.AcquireLockInvalidatesWhenTooSmall 2024-09-14T03:54:51.270 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestObjectMap.AcquireLockInvalidatesWhenTooSmall (33 ms) 2024-09-14T03:54:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:51 smithi129 ceph-mon[33173]: osdmap e102: 8 total, 8 up, 8 in 2024-09-14T03:54:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:51 smithi129 ceph-mon[33173]: pgmap v138: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:51.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:51 smithi039 ceph-mon[29586]: osdmap e102: 8 total, 8 up, 8 in 2024-09-14T03:54:51.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:51 smithi039 ceph-mon[29586]: pgmap v138: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:51.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:51 smithi039 ceph-mon[40962]: osdmap e102: 8 total, 8 up, 8 in 2024-09-14T03:54:51.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:51 smithi039 ceph-mon[40962]: pgmap v138: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:52.132 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 4 tests from TestObjectMap (2182 ms total) 2024-09-14T03:54:52.132 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T03:54:52.132 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 2 tests from TestMirroringWatcher 2024-09-14T03:54:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:52 smithi129 ceph-mon[33173]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:54:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:52 smithi129 ceph-mon[33173]: Cluster is now healthy 2024-09-14T03:54:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:52 smithi129 ceph-mon[33173]: osdmap e103: 8 total, 8 up, 8 in 2024-09-14T03:54:52.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:52 smithi039 ceph-mon[29586]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:54:52.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:52 smithi039 ceph-mon[29586]: Cluster is now healthy 2024-09-14T03:54:52.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:52 smithi039 ceph-mon[29586]: osdmap e103: 8 total, 8 up, 8 in 2024-09-14T03:54:52.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:52 smithi039 ceph-mon[40962]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:54:52.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:52 smithi039 ceph-mon[40962]: Cluster is now healthy 2024-09-14T03:54:52.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:52 smithi039 ceph-mon[40962]: osdmap e103: 8 total, 8 up, 8 in 2024-09-14T03:54:53.441 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:53 smithi039 ceph-mon[29586]: osdmap e104: 8 total, 8 up, 8 in 2024-09-14T03:54:53.442 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:53 smithi039 ceph-mon[29586]: pgmap v141: 73 pgs: 73 active+clean; 579 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:53.442 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:53 smithi039 ceph-mon[40962]: osdmap e104: 8 total, 8 up, 8 in 2024-09-14T03:54:53.442 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:53 smithi039 ceph-mon[40962]: pgmap v141: 73 pgs: 73 active+clean; 579 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:53.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:53 smithi129 ceph-mon[33173]: osdmap e104: 8 total, 8 up, 8 in 2024-09-14T03:54:53.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:53 smithi129 ceph-mon[33173]: pgmap v141: 73 pgs: 73 active+clean; 579 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:54.144 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroringWatcher.ModeUpdated 2024-09-14T03:54:54.270 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:54 smithi039 ceph-mon[29586]: osdmap e105: 8 total, 8 up, 8 in 2024-09-14T03:54:54.270 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:54 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/665899442' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:54.270 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:54 smithi039 ceph-mon[40962]: osdmap e105: 8 total, 8 up, 8 in 2024-09-14T03:54:54.270 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:54 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/665899442' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:54.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:54 smithi129 ceph-mon[33173]: osdmap e105: 8 total, 8 up, 8 in 2024-09-14T03:54:54.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:54 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/665899442' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-7","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:55.408 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:55 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/665899442' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:55.408 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:55 smithi039 ceph-mon[29586]: osdmap e106: 8 total, 8 up, 8 in 2024-09-14T03:54:55.408 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:55 smithi039 ceph-mon[29586]: pgmap v144: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:55.408 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:55 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/665899442' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:55.408 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:55 smithi039 ceph-mon[40962]: osdmap e106: 8 total, 8 up, 8 in 2024-09-14T03:54:55.408 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:55 smithi039 ceph-mon[40962]: pgmap v144: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:55.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:55 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/665899442' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-7","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:54:55.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:55 smithi129 ceph-mon[33173]: osdmap e106: 8 total, 8 up, 8 in 2024-09-14T03:54:55.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:55 smithi129 ceph-mon[33173]: pgmap v144: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 237 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:56.200 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroringWatcher.ModeUpdated (2058 ms) 2024-09-14T03:54:56.200 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroringWatcher.ImageStatusUpdated 2024-09-14T03:54:56.222 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroringWatcher.ImageStatusUpdated (22 ms) 2024-09-14T03:54:56.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:56 smithi129 ceph-mon[33173]: osdmap e107: 8 total, 8 up, 8 in 2024-09-14T03:54:56.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:56 smithi039 ceph-mon[29586]: osdmap e107: 8 total, 8 up, 8 in 2024-09-14T03:54:56.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:56 smithi039 ceph-mon[40962]: osdmap e107: 8 total, 8 up, 8 in 2024-09-14T03:54:57.184 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 2 tests from TestMirroringWatcher (2080 ms total) 2024-09-14T03:54:57.185 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T03:54:57.185 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 57 tests from TestMigration 2024-09-14T03:54:57.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:57 smithi129 ceph-mon[33173]: osdmap e108: 8 total, 8 up, 8 in 2024-09-14T03:54:57.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:57 smithi129 ceph-mon[33173]: pgmap v147: 105 pgs: 105 active+clean; 579 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:54:57.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:57 smithi039 ceph-mon[29586]: osdmap e108: 8 total, 8 up, 8 in 2024-09-14T03:54:57.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:57 smithi039 ceph-mon[29586]: pgmap v147: 105 pgs: 105 active+clean; 579 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:54:57.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:57 smithi039 ceph-mon[40962]: osdmap e108: 8 total, 8 up, 8 in 2024-09-14T03:54:57.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:57 smithi039 ceph-mon[40962]: pgmap v147: 105 pgs: 105 active+clean; 579 KiB data, 238 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T03:54:57.808 INFO:teuthology.orchestra.run.smithi039.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:54:58.435 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:58 smithi039 ceph-mon[29586]: osdmap e109: 8 total, 8 up, 8 in 2024-09-14T03:54:58.436 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:58 smithi039 ceph-mon[29586]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:54:58.436 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:58 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:54:58.436 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:58 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:54:58.436 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:58 smithi039 ceph-mon[40962]: osdmap e109: 8 total, 8 up, 8 in 2024-09-14T03:54:58.436 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:58 smithi039 ceph-mon[40962]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:54:58.436 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:58 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:54:58.436 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:58 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:54:58.457 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-14T03:54:58.488 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:58 smithi129 ceph-mon[33173]: osdmap e109: 8 total, 8 up, 8 in 2024-09-14T03:54:58.489 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:58 smithi129 ceph-mon[33173]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:54:58.489 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:58 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:54:58.489 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:58 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:54:58.664 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:54:59.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:54:59 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:54:59] "GET /metrics HTTP/1.1" 200 36232 "" "Prometheus/2.43.0" 2024-09-14T03:54:59.454 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:59 smithi129 ceph-mon[33173]: osdmap e110: 8 total, 8 up, 8 in 2024-09-14T03:54:59.454 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:59 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/1936098766' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:59.455 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:59 smithi129 ceph-mon[33173]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:59.455 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:54:59 smithi129 ceph-mon[33173]: pgmap v150: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 238 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:59.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:59 smithi039 ceph-mon[29586]: osdmap e110: 8 total, 8 up, 8 in 2024-09-14T03:54:59.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:59 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/1936098766' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:59.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:59 smithi039 ceph-mon[29586]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:59.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:54:59 smithi039 ceph-mon[29586]: pgmap v150: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 238 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:54:59.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:59 smithi039 ceph-mon[40962]: osdmap e110: 8 total, 8 up, 8 in 2024-09-14T03:54:59.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:59 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/1936098766' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:59.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:59 smithi039 ceph-mon[40962]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-8","app": "rados","yes_i_really_mean_it": true}]: dispatch 2024-09-14T03:54:59.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:54:59 smithi039 ceph-mon[40962]: pgmap v150: 105 pgs: 32 unknown, 73 active+clean; 579 KiB data, 238 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:55:00.216 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.Empty 2024-09-14T03:55:00.269 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:00 smithi039 ceph-mon[29586]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:55:00.269 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:00 smithi039 ceph-mon[29586]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:00.269 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:00 smithi039 ceph-mon[29586]: osdmap e111: 8 total, 8 up, 8 in 2024-09-14T03:55:00.269 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:00 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:00.269 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:00 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:55:00.270 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:00 smithi039 ceph-mon[40962]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:55:00.270 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:00 smithi039 ceph-mon[40962]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:00.270 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:00 smithi039 ceph-mon[40962]: osdmap e111: 8 total, 8 up, 8 in 2024-09-14T03:55:00.270 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:00 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:00.270 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:00 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:55:00.381 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:00 smithi129 ceph-mon[33173]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:55:00.381 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:00 smithi129 ceph-mon[33173]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-8","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:55:00.381 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:00 smithi129 ceph-mon[33173]: osdmap e111: 8 total, 8 up, 8 in 2024-09-14T03:55:00.381 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:00 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:00.381 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:00 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:55:00.741 INFO:teuthology.orchestra.run.smithi039.stdout:true 2024-09-14T03:55:01.307 INFO:teuthology.orchestra.run.smithi039.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:55:01.307 INFO:teuthology.orchestra.run.smithi039.stdout:alertmanager.a smithi039 *:9093,9094 running (2m) 62s ago 3m 24.4M - 0.25.0 c8568f914cd2 b5ca6b93d9ee 2024-09-14T03:55:01.307 INFO:teuthology.orchestra.run.smithi039.stdout:grafana.a smithi129 *:3000 running (2m) 85s ago 2m 82.9M - 9.4.7 954c08fa6188 71f297945590 2024-09-14T03:55:01.307 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi039.bbwiwf smithi039 running (104s) 62s ago 103s 16.4M - 18.2.4-845-ge4e227b3 67ddd33fa72f 91e6c8f85681 2024-09-14T03:55:01.308 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi129.ybjqcw smithi129 running (101s) 85s ago 100s 13.0M - 18.2.4-845-ge4e227b3 67ddd33fa72f a3fab5b37b71 2024-09-14T03:55:01.308 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.x smithi129 *:8443,8765 running (7m) 85s ago 7m 438M - 18.2.4-845-ge4e227b3 67ddd33fa72f 813f62a05ea8 2024-09-14T03:55:01.308 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.y smithi039 *:9283,8765 running (8m) 62s ago 8m 484M - 18.2.4-845-ge4e227b3 67ddd33fa72f babf5c5c054b 2024-09-14T03:55:01.308 INFO:teuthology.orchestra.run.smithi039.stdout:mon.a smithi039 running (8m) 62s ago 8m 47.0M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f e7b5f91fb09c 2024-09-14T03:55:01.308 INFO:teuthology.orchestra.run.smithi039.stdout:mon.b smithi129 running (7m) 85s ago 7m 39.3M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f eb38cc5cbe35 2024-09-14T03:55:01.308 INFO:teuthology.orchestra.run.smithi039.stdout:mon.c smithi039 running (7m) 62s ago 7m 37.8M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 31d440e49f93 2024-09-14T03:55:01.308 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.a smithi039 *:9100 running (2m) 62s ago 2m 13.4M - 1.5.0 0da6a335fe13 4eff582f78f5 2024-09-14T03:55:01.308 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.b smithi129 *:9100 running (2m) 85s ago 2m 8442k - 1.5.0 0da6a335fe13 c7581baf44f2 2024-09-14T03:55:01.308 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0 smithi039 running (6m) 62s ago 6m 43.1M 4016M 18.2.4-845-ge4e227b3 67ddd33fa72f 5048d74f27ac 2024-09-14T03:55:01.308 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1 smithi039 running (6m) 62s ago 6m 42.4M 4016M 18.2.4-845-ge4e227b3 67ddd33fa72f 91349527d88b 2024-09-14T03:55:01.308 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2 smithi039 running (5m) 62s ago 5m 50.2M 4016M 18.2.4-845-ge4e227b3 67ddd33fa72f eb903b44fc5d 2024-09-14T03:55:01.308 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3 smithi039 running (5m) 62s ago 5m 49.0M 4016M 18.2.4-845-ge4e227b3 67ddd33fa72f dff9e665bd9b 2024-09-14T03:55:01.308 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4 smithi129 running (5m) 85s ago 5m 42.6M 4016M 18.2.4-845-ge4e227b3 67ddd33fa72f 5b64727311b7 2024-09-14T03:55:01.309 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5 smithi129 running (4m) 85s ago 4m 47.4M 4016M 18.2.4-845-ge4e227b3 67ddd33fa72f f2d991a754f7 2024-09-14T03:55:01.309 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6 smithi129 running (4m) 85s ago 4m 48.2M 4016M 18.2.4-845-ge4e227b3 67ddd33fa72f 3060eaf44106 2024-09-14T03:55:01.309 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7 smithi129 running (3m) 85s ago 3m 41.1M 4016M 18.2.4-845-ge4e227b3 67ddd33fa72f 4c13b97060d1 2024-09-14T03:55:01.309 INFO:teuthology.orchestra.run.smithi039.stdout:prometheus.a smithi129 *:9095 running (114s) 85s ago 3m 33.7M - 2.43.0 a07b618ecd1d 0275bbd07991 2024-09-14T03:55:01.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:01 smithi129 ceph-mon[33173]: osdmap e112: 8 total, 8 up, 8 in 2024-09-14T03:55:01.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:01 smithi129 ceph-mon[33173]: from='client.24809 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:01.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:01 smithi129 ceph-mon[33173]: pgmap v153: 137 pgs: 64 unknown, 73 active+clean; 579 KiB data, 238 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:55:01.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:01 smithi039 ceph-mon[29586]: osdmap e112: 8 total, 8 up, 8 in 2024-09-14T03:55:01.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:01 smithi039 ceph-mon[29586]: from='client.24809 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:01.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:01 smithi039 ceph-mon[29586]: pgmap v153: 137 pgs: 64 unknown, 73 active+clean; 579 KiB data, 238 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:55:01.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:01 smithi039 ceph-mon[40962]: osdmap e112: 8 total, 8 up, 8 in 2024-09-14T03:55:01.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:01 smithi039 ceph-mon[40962]: from='client.24809 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:01.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:01 smithi039 ceph-mon[40962]: pgmap v153: 137 pgs: 64 unknown, 73 active+clean; 579 KiB data, 238 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:55:01.643 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T03:55:01.643 INFO:teuthology.orchestra.run.smithi039.stdout: "mon": { 2024-09-14T03:55:01.644 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 3 2024-09-14T03:55:01.644 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:55:01.644 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr": { 2024-09-14T03:55:01.644 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:55:01.644 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:55:01.644 INFO:teuthology.orchestra.run.smithi039.stdout: "osd": { 2024-09-14T03:55:01.644 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8 2024-09-14T03:55:01.644 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:55:01.644 INFO:teuthology.orchestra.run.smithi039.stdout: "mds": { 2024-09-14T03:55:01.644 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:55:01.644 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:55:01.644 INFO:teuthology.orchestra.run.smithi039.stdout: "overall": { 2024-09-14T03:55:01.645 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 15 2024-09-14T03:55:01.645 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T03:55:01.645 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T03:55:01.957 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T03:55:01.957 INFO:teuthology.orchestra.run.smithi039.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", 2024-09-14T03:55:01.958 INFO:teuthology.orchestra.run.smithi039.stdout: "in_progress": true, 2024-09-14T03:55:01.958 INFO:teuthology.orchestra.run.smithi039.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-14T03:55:01.958 INFO:teuthology.orchestra.run.smithi039.stdout: "services_complete": [], 2024-09-14T03:55:01.958 INFO:teuthology.orchestra.run.smithi039.stdout: "progress": "", 2024-09-14T03:55:01.958 INFO:teuthology.orchestra.run.smithi039.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 image", 2024-09-14T03:55:01.958 INFO:teuthology.orchestra.run.smithi039.stdout: "is_paused": false 2024-09-14T03:55:01.958 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T03:55:02.296 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:02.308 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:55:02.449 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:55:02.513 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:55:02.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:02 smithi129 ceph-mon[33173]: from='client.24815 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:02.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:02 smithi129 ceph-mon[33173]: osdmap e113: 8 total, 8 up, 8 in 2024-09-14T03:55:02.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:02 smithi129 ceph-mon[33173]: from='client.24821 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:02.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:02 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:02.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:02 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:02.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:02 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:55:02.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:02 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:55:02.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:02 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:02.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:02 smithi129 ceph-mon[33173]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:55:02.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:02 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/2225666436' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:55:02.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:02 smithi039 ceph-mon[29586]: from='client.24815 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:02.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:02 smithi039 ceph-mon[29586]: osdmap e113: 8 total, 8 up, 8 in 2024-09-14T03:55:02.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:02 smithi039 ceph-mon[29586]: from='client.24821 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:02.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:02 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:02.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:02 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:02.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:02 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:55:02.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:02 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:55:02.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:02 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:02.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:02 smithi039 ceph-mon[29586]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:55:02.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:02 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/2225666436' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:55:02.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:02 smithi039 ceph-mon[40962]: from='client.24815 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:02.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:02 smithi039 ceph-mon[40962]: osdmap e113: 8 total, 8 up, 8 in 2024-09-14T03:55:02.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:02 smithi039 ceph-mon[40962]: from='client.24821 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:02.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:02 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:02.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:02 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:02.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:02 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:55:02.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:02 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:55:02.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:02 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:02.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:02 smithi039 ceph-mon[40962]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:55:02.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:02 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/2225666436' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:55:02.590 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.Empty (2375 ms) 2024-09-14T03:55:02.591 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.OtherName 2024-09-14T03:55:02.623 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:02.634 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:55:02.761 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:55:02.823 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:55:02.905 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.OtherName (315 ms) 2024-09-14T03:55:02.906 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.OtherPool 2024-09-14T03:55:02.937 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:02.947 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:55:03.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:03 smithi039 ceph-mon[29586]: from='client.24833 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:03.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:03 smithi039 ceph-mon[29586]: osdmap e114: 8 total, 8 up, 8 in 2024-09-14T03:55:03.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:03 smithi039 ceph-mon[29586]: pgmap v156: 137 pgs: 32 creating+peering, 105 active+clean; 579 KiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:55:03.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:03 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:03.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:03 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:55:03.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:03 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:03.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:03 smithi039 ceph-mon[40962]: from='client.24833 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:03.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:03 smithi039 ceph-mon[40962]: osdmap e114: 8 total, 8 up, 8 in 2024-09-14T03:55:03.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:03 smithi039 ceph-mon[40962]: pgmap v156: 137 pgs: 32 creating+peering, 105 active+clean; 579 KiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:55:03.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:03 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:03.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:03 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:55:03.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:03 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:03.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:03 smithi129 ceph-mon[33173]: from='client.24833 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:03.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:03 smithi129 ceph-mon[33173]: osdmap e114: 8 total, 8 up, 8 in 2024-09-14T03:55:03.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:03 smithi129 ceph-mon[33173]: pgmap v156: 137 pgs: 32 creating+peering, 105 active+clean; 579 KiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:55:03.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:03 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:03.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:03 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:55:03.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:03 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:04.398 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:55:04.468 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:55:04.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:04 smithi039 ceph-mon[29586]: Upgrade: Target is version 19.3.0-4907-ga2782315 (unknown) 2024-09-14T03:55:04.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:04 smithi039 ceph-mon[29586]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T03:55:04.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:04 smithi039 ceph-mon[29586]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-14T03:55:04.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:04 smithi039 ceph-mon[29586]: osdmap e115: 8 total, 8 up, 8 in 2024-09-14T03:55:04.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:04 smithi039 ceph-mon[29586]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 on smithi129 2024-09-14T03:55:04.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:04 smithi039 ceph-mon[40962]: Upgrade: Target is version 19.3.0-4907-ga2782315 (unknown) 2024-09-14T03:55:04.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:04 smithi039 ceph-mon[40962]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T03:55:04.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:04 smithi039 ceph-mon[40962]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-14T03:55:04.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:04 smithi039 ceph-mon[40962]: osdmap e115: 8 total, 8 up, 8 in 2024-09-14T03:55:04.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:04 smithi039 ceph-mon[40962]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 on smithi129 2024-09-14T03:55:04.551 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.OtherPool (1646 ms) 2024-09-14T03:55:04.551 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.OtherNamespace 2024-09-14T03:55:04.580 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:04.594 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:55:04.721 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:55:04.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:04 smithi129 ceph-mon[33173]: Upgrade: Target is version 19.3.0-4907-ga2782315 (unknown) 2024-09-14T03:55:04.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:04 smithi129 ceph-mon[33173]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T03:55:04.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:04 smithi129 ceph-mon[33173]: Upgrade: Need to upgrade myself (mgr.y) 2024-09-14T03:55:04.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:04 smithi129 ceph-mon[33173]: osdmap e115: 8 total, 8 up, 8 in 2024-09-14T03:55:04.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:04 smithi129 ceph-mon[33173]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 on smithi129 2024-09-14T03:55:04.789 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:55:04.869 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.OtherNamespace (318 ms) 2024-09-14T03:55:04.869 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.DataPool 2024-09-14T03:55:04.901 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:04.912 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:55:05.052 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:55:05.123 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:55:05.203 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.DataPool (334 ms) 2024-09-14T03:55:05.203 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.AbortAfterPrepare 2024-09-14T03:55:05.231 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:05.247 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:55:05.382 INFO:tasks.workunit.client.0.smithi039.stdout:migration_abort 2024-09-14T03:55:05.474 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.AbortAfterPrepare (271 ms) 2024-09-14T03:55:05.474 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.AbortAfterFailedPrepare 2024-09-14T03:55:05.503 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:05.514 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:55:05.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:05 smithi039 ceph-mon[29586]: osdmap e116: 8 total, 8 up, 8 in 2024-09-14T03:55:05.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:05 smithi039 ceph-mon[29586]: pgmap v159: 137 pgs: 32 creating+peering, 105 active+clean; 579 KiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:55:05.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:05 smithi039 ceph-mon[40962]: osdmap e116: 8 total, 8 up, 8 in 2024-09-14T03:55:05.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:05 smithi039 ceph-mon[40962]: pgmap v159: 137 pgs: 32 creating+peering, 105 active+clean; 579 KiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:55:05.614 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.AbortAfterFailedPrepare (139 ms) 2024-09-14T03:55:05.614 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.AbortAfterExecute 2024-09-14T03:55:05.642 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:05.653 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:55:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:05 smithi129 ceph-mon[33173]: osdmap e116: 8 total, 8 up, 8 in 2024-09-14T03:55:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:05 smithi129 ceph-mon[33173]: pgmap v159: 137 pgs: 32 creating+peering, 105 active+clean; 579 KiB data, 246 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:55:05.785 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:55:05.856 INFO:tasks.workunit.client.0.smithi039.stdout:migration_abort 2024-09-14T03:55:05.951 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.AbortAfterExecute (337 ms) 2024-09-14T03:55:05.951 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.OtherPoolAbortAfterExecute 2024-09-14T03:55:05.978 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:05.990 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:55:06.113 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:55:06.176 INFO:tasks.workunit.client.0.smithi039.stdout:migration_abort 2024-09-14T03:55:06.275 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.OtherPoolAbortAfterExecute (324 ms) 2024-09-14T03:55:06.275 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.OtherNamespaceAbortAfterExecute 2024-09-14T03:55:06.303 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:06.317 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:55:06.446 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:55:06.512 INFO:tasks.workunit.client.0.smithi039.stdout:migration_abort 2024-09-14T03:55:06.625 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.OtherNamespaceAbortAfterExecute (350 ms) 2024-09-14T03:55:06.625 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.MirroringSamePool 2024-09-14T03:55:06.652 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:06.665 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/test_Migration.cc:635: Skipped 2024-09-14T03:55:06.665 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T03:55:06.671 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestMigration.MirroringSamePool (47 ms) 2024-09-14T03:55:06.671 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.MirroringAbort 2024-09-14T03:55:06.699 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:06.710 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/test_Migration.cc:653: Skipped 2024-09-14T03:55:06.710 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T03:55:06.716 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestMigration.MirroringAbort (45 ms) 2024-09-14T03:55:06.716 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.MirroringOtherPoolDisabled 2024-09-14T03:55:06.746 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:06.757 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/test_Migration.cc:676: Skipped 2024-09-14T03:55:06.757 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T03:55:06.763 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestMigration.MirroringOtherPoolDisabled (47 ms) 2024-09-14T03:55:06.763 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.MirroringOtherPoolEnabled 2024-09-14T03:55:06.792 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:06.805 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/test_Migration.cc:694: Skipped 2024-09-14T03:55:06.805 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T03:55:06.811 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestMigration.MirroringOtherPoolEnabled (48 ms) 2024-09-14T03:55:06.811 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.MirroringPool 2024-09-14T03:55:06.840 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:06.850 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/test_Migration.cc:714: Skipped 2024-09-14T03:55:06.850 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T03:55:06.856 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestMigration.MirroringPool (45 ms) 2024-09-14T03:55:06.856 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.Group 2024-09-14T03:55:06.885 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:06.903 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:55:07.035 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:55:07.099 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:55:07.188 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.Group (332 ms) 2024-09-14T03:55:07.188 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.GroupAbort 2024-09-14T03:55:07.219 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:07.239 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:55:07.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:06 smithi129 ceph-mon[33173]: pgmap v160: 137 pgs: 137 active+clean; 580 KiB data, 255 MiB used, 715 GiB / 715 GiB avail; 792 KiB/s rd, 44 KiB/s wr, 999 op/s 2024-09-14T03:55:07.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:06 smithi039 ceph-mon[29586]: pgmap v160: 137 pgs: 137 active+clean; 580 KiB data, 255 MiB used, 715 GiB / 715 GiB avail; 792 KiB/s rd, 44 KiB/s wr, 999 op/s 2024-09-14T03:55:07.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:06 smithi039 ceph-mon[40962]: pgmap v160: 137 pgs: 137 active+clean; 580 KiB data, 255 MiB used, 715 GiB / 715 GiB avail; 792 KiB/s rd, 44 KiB/s wr, 999 op/s 2024-09-14T03:55:07.383 INFO:tasks.workunit.client.0.smithi039.stdout:migration_abort 2024-09-14T03:55:07.493 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.GroupAbort (305 ms) 2024-09-14T03:55:07.494 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.NoSnaps 2024-09-14T03:55:07.559 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:07.571 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 0~8388609 2024-09-14T03:55:07.643 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 16777218~8388609 2024-09-14T03:55:07.689 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:55:07.873 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:55:07.991 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:55:08.108 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.NoSnaps (614 ms) 2024-09-14T03:55:08.108 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.NoSnapsOtherPool 2024-09-14T03:55:08.137 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:08.146 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 0~8388609 2024-09-14T03:55:08.201 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 16777218~8388609 2024-09-14T03:55:08.258 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 0~8388609 2024-09-14T03:55:08.262 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 16777218~8388609 2024-09-14T03:55:08.325 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:55:08.488 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:55:08.593 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:55:08.722 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.NoSnapsOtherPool (615 ms) 2024-09-14T03:55:08.723 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.NoSnapsDataPool 2024-09-14T03:55:08.752 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:08.762 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 0~8388609 2024-09-14T03:55:08.829 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 16777218~8388609 2024-09-14T03:55:08.886 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:55:09.070 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:55:09.198 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:55:09.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:09 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:55:09] "GET /metrics HTTP/1.1" 200 36290 "" "Prometheus/2.43.0" 2024-09-14T03:55:09.318 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.NoSnapsDataPool (596 ms) 2024-09-14T03:55:09.318 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.NoSnapsShrinkAfterPrepare 2024-09-14T03:55:09.345 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:09.355 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 0~8388609 2024-09-14T03:55:09.416 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 16777218~8388609 2024-09-14T03:55:09.467 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:55:09.645 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 41943040 2024-09-14T03:55:09.653 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:55:09.766 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:55:09.864 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.NoSnapsShrinkAfterPrepare (546 ms) 2024-09-14T03:55:09.865 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.NoSnapsShrinkToZeroBeforePrepare 2024-09-14T03:55:09.892 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:09.903 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 0~8388609 2024-09-14T03:55:09.952 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 16777218~8388609 2024-09-14T03:55:10.009 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 0 2024-09-14T03:55:10.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:09 smithi129 ceph-mon[33173]: osdmap e117: 8 total, 8 up, 8 in 2024-09-14T03:55:10.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:09 smithi129 ceph-mon[33173]: pgmap v162: 137 pgs: 137 active+clean; 580 KiB data, 255 MiB used, 715 GiB / 715 GiB avail; 733 KiB/s rd, 41 KiB/s wr, 924 op/s 2024-09-14T03:55:10.023 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:55:10.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:09 smithi039 ceph-mon[29586]: osdmap e117: 8 total, 8 up, 8 in 2024-09-14T03:55:10.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:09 smithi039 ceph-mon[29586]: pgmap v162: 137 pgs: 137 active+clean; 580 KiB data, 255 MiB used, 715 GiB / 715 GiB avail; 733 KiB/s rd, 41 KiB/s wr, 924 op/s 2024-09-14T03:55:10.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:09 smithi039 ceph-mon[40962]: osdmap e117: 8 total, 8 up, 8 in 2024-09-14T03:55:10.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:09 smithi039 ceph-mon[40962]: pgmap v162: 137 pgs: 137 active+clean; 580 KiB data, 255 MiB used, 715 GiB / 715 GiB avail; 733 KiB/s rd, 41 KiB/s wr, 924 op/s 2024-09-14T03:55:10.121 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:55:10.186 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:55:10.233 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.NoSnapsShrinkToZeroBeforePrepare (368 ms) 2024-09-14T03:55:10.233 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.NoSnapsShrinkToZeroAfterPrepare 2024-09-14T03:55:10.266 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:10.277 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 0~8388609 2024-09-14T03:55:10.337 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 16777218~8388609 2024-09-14T03:55:10.405 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:55:10.581 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 0 2024-09-14T03:55:10.594 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:55:10.654 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:55:10.704 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.NoSnapsShrinkToZeroAfterPrepare (472 ms) 2024-09-14T03:55:10.704 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.NoSnapsExpandAfterPrepare 2024-09-14T03:55:10.733 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:10.743 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 0~8388609 2024-09-14T03:55:10.801 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 16777218~8388609 2024-09-14T03:55:10.851 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:55:11.032 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 167772160 2024-09-14T03:55:11.040 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:55:11.150 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:55:11.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:10 smithi129 ceph-mon[33173]: pgmap v163: 137 pgs: 137 active+clean; 580 KiB data, 255 MiB used, 715 GiB / 715 GiB avail; 586 KiB/s rd, 33 KiB/s wr, 738 op/s 2024-09-14T03:55:11.280 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.NoSnapsExpandAfterPrepare (575 ms) 2024-09-14T03:55:11.280 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.NoSnapsSnapAfterPrepare 2024-09-14T03:55:11.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:10 smithi039 ceph-mon[29586]: pgmap v163: 137 pgs: 137 active+clean; 580 KiB data, 255 MiB used, 715 GiB / 715 GiB avail; 586 KiB/s rd, 33 KiB/s wr, 738 op/s 2024-09-14T03:55:11.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:10 smithi039 ceph-mon[40962]: pgmap v163: 137 pgs: 137 active+clean; 580 KiB data, 255 MiB used, 715 GiB / 715 GiB avail; 586 KiB/s rd, 33 KiB/s wr, 738 op/s 2024-09-14T03:55:11.309 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:11.318 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 0~8388609 2024-09-14T03:55:11.398 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 16777218~8388609 2024-09-14T03:55:11.454 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:55:11.631 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: after_prepare_snap 2024-09-14T03:55:12.889 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 41943040 2024-09-14T03:55:12.910 INFO:tasks.workunit.client.0.smithi039.stdout:write: * 0~1000 2024-09-14T03:55:12.916 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:55:13.043 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:55:13.191 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.NoSnapsSnapAfterPrepare (1911 ms) 2024-09-14T03:55:13.191 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.Snaps 2024-09-14T03:55:13.220 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:13.232 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 0~8388609 2024-09-14T03:55:13.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:12 smithi129 ceph-mon[33173]: osdmap e118: 8 total, 8 up, 8 in 2024-09-14T03:55:13.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:12 smithi129 ceph-mon[33173]: pgmap v165: 137 pgs: 137 active+clean; 145 MiB data, 720 MiB used, 715 GiB / 715 GiB avail; 46 MiB/s rd, 38 MiB/s wr, 2.06k op/s 2024-09-14T03:55:13.286 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap1 2024-09-14T03:55:13.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:12 smithi039 ceph-mon[29586]: osdmap e118: 8 total, 8 up, 8 in 2024-09-14T03:55:13.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:12 smithi039 ceph-mon[29586]: pgmap v165: 137 pgs: 137 active+clean; 145 MiB data, 720 MiB used, 715 GiB / 715 GiB avail; 46 MiB/s rd, 38 MiB/s wr, 2.06k op/s 2024-09-14T03:55:13.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:12 smithi039 ceph-mon[40962]: osdmap e118: 8 total, 8 up, 8 in 2024-09-14T03:55:13.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:12 smithi039 ceph-mon[40962]: pgmap v165: 137 pgs: 137 active+clean; 145 MiB data, 720 MiB used, 715 GiB / 715 GiB avail; 46 MiB/s rd, 38 MiB/s wr, 2.06k op/s 2024-09-14T03:55:14.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:13 smithi129 ceph-mon[33173]: osdmap e119: 8 total, 8 up, 8 in 2024-09-14T03:55:14.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:13 smithi039 ceph-mon[29586]: osdmap e119: 8 total, 8 up, 8 in 2024-09-14T03:55:14.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:13 smithi039 ceph-mon[40962]: osdmap e119: 8 total, 8 up, 8 in 2024-09-14T03:55:14.905 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 8388609~8388609 2024-09-14T03:55:14.923 INFO:tasks.workunit.client.0.smithi039.stdout:write: X 0~1000 2024-09-14T03:55:14.923 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 1010~1000 2024-09-14T03:55:14.923 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap2 2024-09-14T03:55:15.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:14 smithi129 ceph-mon[33173]: osdmap e120: 8 total, 8 up, 8 in 2024-09-14T03:55:15.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:14 smithi129 ceph-mon[33173]: pgmap v168: 137 pgs: 137 active+clean; 145 MiB data, 720 MiB used, 715 GiB / 715 GiB avail; 60 MiB/s rd, 51 MiB/s wr, 1.82k op/s 2024-09-14T03:55:15.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:14 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:55:15.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:14 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:15.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:14 smithi039 ceph-mon[40962]: osdmap e120: 8 total, 8 up, 8 in 2024-09-14T03:55:15.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:14 smithi039 ceph-mon[40962]: pgmap v168: 137 pgs: 137 active+clean; 145 MiB data, 720 MiB used, 715 GiB / 715 GiB avail; 60 MiB/s rd, 51 MiB/s wr, 1.82k op/s 2024-09-14T03:55:15.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:14 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:55:15.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:14 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:15.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:14 smithi039 ceph-mon[29586]: osdmap e120: 8 total, 8 up, 8 in 2024-09-14T03:55:15.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:14 smithi039 ceph-mon[29586]: pgmap v168: 137 pgs: 137 active+clean; 145 MiB data, 720 MiB used, 715 GiB / 715 GiB avail; 60 MiB/s rd, 51 MiB/s wr, 1.82k op/s 2024-09-14T03:55:15.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:14 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:55:15.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:14 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:16.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:15 smithi129 ceph-mon[33173]: osdmap e121: 8 total, 8 up, 8 in 2024-09-14T03:55:16.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:15 smithi039 ceph-mon[29586]: osdmap e121: 8 total, 8 up, 8 in 2024-09-14T03:55:16.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:15 smithi039 ceph-mon[40962]: osdmap e121: 8 total, 8 up, 8 in 2024-09-14T03:55:16.925 INFO:tasks.workunit.client.0.smithi039.stdout:write: X 8388609~1000 2024-09-14T03:55:16.933 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 16777228~1000 2024-09-14T03:55:16.933 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 167772160 2024-09-14T03:55:16.942 INFO:tasks.workunit.client.0.smithi039.stdout:write: 2 83886079~8388609 2024-09-14T03:55:16.959 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap3 2024-09-14T03:55:17.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:16 smithi129 ceph-mon[33173]: osdmap e122: 8 total, 8 up, 8 in 2024-09-14T03:55:17.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:16 smithi129 ceph-mon[33173]: pgmap v171: 137 pgs: 137 active+clean; 225 MiB data, 937 MiB used, 714 GiB / 715 GiB avail; 48 MiB/s rd, 28 MiB/s wr, 687 op/s 2024-09-14T03:55:17.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:16 smithi039 ceph-mon[29586]: osdmap e122: 8 total, 8 up, 8 in 2024-09-14T03:55:17.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:16 smithi039 ceph-mon[29586]: pgmap v171: 137 pgs: 137 active+clean; 225 MiB data, 937 MiB used, 714 GiB / 715 GiB avail; 48 MiB/s rd, 28 MiB/s wr, 687 op/s 2024-09-14T03:55:17.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:16 smithi039 ceph-mon[40962]: osdmap e122: 8 total, 8 up, 8 in 2024-09-14T03:55:17.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:16 smithi039 ceph-mon[40962]: pgmap v171: 137 pgs: 137 active+clean; 225 MiB data, 937 MiB used, 714 GiB / 715 GiB avail; 48 MiB/s rd, 28 MiB/s wr, 687 op/s 2024-09-14T03:55:18.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:17 smithi129 ceph-mon[33173]: osdmap e123: 8 total, 8 up, 8 in 2024-09-14T03:55:18.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:17 smithi039 ceph-mon[29586]: osdmap e123: 8 total, 8 up, 8 in 2024-09-14T03:55:18.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:17 smithi039 ceph-mon[40962]: osdmap e123: 8 total, 8 up, 8 in 2024-09-14T03:55:18.944 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:18.968 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 83886079~1 2024-09-14T03:55:18.976 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:55:19.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:18 smithi129 ceph-mon[33173]: osdmap e124: 8 total, 8 up, 8 in 2024-09-14T03:55:19.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:18 smithi129 ceph-mon[33173]: pgmap v174: 137 pgs: 137 active+clean; 225 MiB data, 937 MiB used, 714 GiB / 715 GiB avail; 48 MiB/s rd, 28 MiB/s wr, 687 op/s 2024-09-14T03:55:19.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:18 smithi039 ceph-mon[29586]: osdmap e124: 8 total, 8 up, 8 in 2024-09-14T03:55:19.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:18 smithi039 ceph-mon[29586]: pgmap v174: 137 pgs: 137 active+clean; 225 MiB data, 937 MiB used, 714 GiB / 715 GiB avail; 48 MiB/s rd, 28 MiB/s wr, 687 op/s 2024-09-14T03:55:19.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:19 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:55:19] "GET /metrics HTTP/1.1" 200 38786 "" "Prometheus/2.43.0" 2024-09-14T03:55:19.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:18 smithi039 ceph-mon[40962]: osdmap e124: 8 total, 8 up, 8 in 2024-09-14T03:55:19.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:18 smithi039 ceph-mon[40962]: pgmap v174: 137 pgs: 137 active+clean; 225 MiB data, 937 MiB used, 714 GiB / 715 GiB avail; 48 MiB/s rd, 28 MiB/s wr, 687 op/s 2024-09-14T03:55:20.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:19 smithi129 ceph-mon[33173]: osdmap e125: 8 total, 8 up, 8 in 2024-09-14T03:55:20.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:19 smithi039 ceph-mon[29586]: osdmap e125: 8 total, 8 up, 8 in 2024-09-14T03:55:20.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:19 smithi039 ceph-mon[40962]: osdmap e125: 8 total, 8 up, 8 in 2024-09-14T03:55:21.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:20 smithi129 ceph-mon[33173]: osdmap e126: 8 total, 8 up, 8 in 2024-09-14T03:55:21.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:20 smithi129 ceph-mon[33173]: pgmap v177: 137 pgs: 137 active+clean; 225 MiB data, 937 MiB used, 714 GiB / 715 GiB avail 2024-09-14T03:55:21.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:20 smithi039 ceph-mon[29586]: osdmap e126: 8 total, 8 up, 8 in 2024-09-14T03:55:21.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:20 smithi039 ceph-mon[29586]: pgmap v177: 137 pgs: 137 active+clean; 225 MiB data, 937 MiB used, 714 GiB / 715 GiB avail 2024-09-14T03:55:21.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:20 smithi039 ceph-mon[40962]: osdmap e126: 8 total, 8 up, 8 in 2024-09-14T03:55:21.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:20 smithi039 ceph-mon[40962]: pgmap v177: 137 pgs: 137 active+clean; 225 MiB data, 937 MiB used, 714 GiB / 715 GiB avail 2024-09-14T03:55:22.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:21 smithi129 ceph-mon[33173]: osdmap e127: 8 total, 8 up, 8 in 2024-09-14T03:55:22.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:21 smithi039 ceph-mon[29586]: osdmap e127: 8 total, 8 up, 8 in 2024-09-14T03:55:22.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:21 smithi039 ceph-mon[40962]: osdmap e127: 8 total, 8 up, 8 in 2024-09-14T03:55:22.365 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:55:22.496 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:55:23.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:23 smithi129 ceph-mon[33173]: osdmap e128: 8 total, 8 up, 8 in 2024-09-14T03:55:23.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:23 smithi129 ceph-mon[33173]: pgmap v180: 137 pgs: 137 active+clean; 249 MiB data, 1002 MiB used, 714 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-09-14T03:55:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:23 smithi039 ceph-mon[29586]: osdmap e128: 8 total, 8 up, 8 in 2024-09-14T03:55:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:23 smithi039 ceph-mon[29586]: pgmap v180: 137 pgs: 137 active+clean; 249 MiB data, 1002 MiB used, 714 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-09-14T03:55:23.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:23 smithi039 ceph-mon[40962]: osdmap e128: 8 total, 8 up, 8 in 2024-09-14T03:55:23.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:23 smithi039 ceph-mon[40962]: pgmap v180: 137 pgs: 137 active+clean; 249 MiB data, 1002 MiB used, 714 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-09-14T03:55:24.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:24 smithi129 ceph-mon[33173]: osdmap e129: 8 total, 8 up, 8 in 2024-09-14T03:55:24.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:24 smithi039 ceph-mon[29586]: osdmap e129: 8 total, 8 up, 8 in 2024-09-14T03:55:24.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:24 smithi039 ceph-mon[40962]: osdmap e129: 8 total, 8 up, 8 in 2024-09-14T03:55:25.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:25 smithi039 ceph-mon[29586]: osdmap e130: 8 total, 8 up, 8 in 2024-09-14T03:55:25.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:25 smithi039 ceph-mon[29586]: pgmap v183: 137 pgs: 137 active+clean; 249 MiB data, 1002 MiB used, 714 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-09-14T03:55:25.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:25 smithi039 ceph-mon[40962]: osdmap e130: 8 total, 8 up, 8 in 2024-09-14T03:55:25.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:25 smithi039 ceph-mon[40962]: pgmap v183: 137 pgs: 137 active+clean; 249 MiB data, 1002 MiB used, 714 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-09-14T03:55:25.299 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.Snaps (12108 ms) 2024-09-14T03:55:25.300 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.SnapsOtherPool 2024-09-14T03:55:25.328 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:25.339 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 0~8388609 2024-09-14T03:55:25.407 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap1 2024-09-14T03:55:25.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:25 smithi129 ceph-mon[33173]: osdmap e130: 8 total, 8 up, 8 in 2024-09-14T03:55:25.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:25 smithi129 ceph-mon[33173]: pgmap v183: 137 pgs: 137 active+clean; 249 MiB data, 1002 MiB used, 714 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-09-14T03:55:26.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:26 smithi039 ceph-mon[29586]: osdmap e131: 8 total, 8 up, 8 in 2024-09-14T03:55:26.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:26 smithi039 ceph-mon[40962]: osdmap e131: 8 total, 8 up, 8 in 2024-09-14T03:55:26.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:26 smithi129 ceph-mon[33173]: osdmap e131: 8 total, 8 up, 8 in 2024-09-14T03:55:27.084 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 8388609~8388609 2024-09-14T03:55:27.097 INFO:tasks.workunit.client.0.smithi039.stdout:write: X 0~1000 2024-09-14T03:55:27.098 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 1010~1000 2024-09-14T03:55:27.098 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap2 2024-09-14T03:55:27.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:27 smithi129 ceph-mon[33173]: osdmap e132: 8 total, 8 up, 8 in 2024-09-14T03:55:27.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:27 smithi129 ceph-mon[33173]: pgmap v186: 137 pgs: 137 active+clean; 265 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 70 MiB/s rd, 10 MiB/s wr, 493 op/s 2024-09-14T03:55:27.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:27 smithi039 ceph-mon[29586]: osdmap e132: 8 total, 8 up, 8 in 2024-09-14T03:55:27.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:27 smithi039 ceph-mon[29586]: pgmap v186: 137 pgs: 137 active+clean; 265 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 70 MiB/s rd, 10 MiB/s wr, 493 op/s 2024-09-14T03:55:27.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:27 smithi039 ceph-mon[40962]: osdmap e132: 8 total, 8 up, 8 in 2024-09-14T03:55:27.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:27 smithi039 ceph-mon[40962]: pgmap v186: 137 pgs: 137 active+clean; 265 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 70 MiB/s rd, 10 MiB/s wr, 493 op/s 2024-09-14T03:55:28.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:28 smithi129 ceph-mon[33173]: osdmap e133: 8 total, 8 up, 8 in 2024-09-14T03:55:28.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:28 smithi039 ceph-mon[29586]: osdmap e133: 8 total, 8 up, 8 in 2024-09-14T03:55:28.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:28 smithi039 ceph-mon[40962]: osdmap e133: 8 total, 8 up, 8 in 2024-09-14T03:55:28.728 INFO:tasks.workunit.client.0.smithi039.stdout:write: X 8388609~1000 2024-09-14T03:55:28.736 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 16777228~1000 2024-09-14T03:55:28.736 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 167772160 2024-09-14T03:55:28.746 INFO:tasks.workunit.client.0.smithi039.stdout:write: 2 83886079~8388609 2024-09-14T03:55:28.773 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap3 2024-09-14T03:55:29.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:29 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:55:29] "GET /metrics HTTP/1.1" 200 38786 "" "Prometheus/2.43.0" 2024-09-14T03:55:29.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:29 smithi039 ceph-mon[40962]: osdmap e134: 8 total, 8 up, 8 in 2024-09-14T03:55:29.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:29 smithi039 ceph-mon[40962]: osdmap e135: 8 total, 8 up, 8 in 2024-09-14T03:55:29.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:29 smithi039 ceph-mon[40962]: pgmap v190: 137 pgs: 137 active+clean; 265 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 75 MiB/s rd, 11 MiB/s wr, 526 op/s 2024-09-14T03:55:29.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:29 smithi039 ceph-mon[29586]: osdmap e134: 8 total, 8 up, 8 in 2024-09-14T03:55:29.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:29 smithi039 ceph-mon[29586]: osdmap e135: 8 total, 8 up, 8 in 2024-09-14T03:55:29.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:29 smithi039 ceph-mon[29586]: pgmap v190: 137 pgs: 137 active+clean; 265 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 75 MiB/s rd, 11 MiB/s wr, 526 op/s 2024-09-14T03:55:29.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:29 smithi129 ceph-mon[33173]: osdmap e134: 8 total, 8 up, 8 in 2024-09-14T03:55:29.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:29 smithi129 ceph-mon[33173]: osdmap e135: 8 total, 8 up, 8 in 2024-09-14T03:55:29.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:29 smithi129 ceph-mon[33173]: pgmap v190: 137 pgs: 137 active+clean; 265 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 75 MiB/s rd, 11 MiB/s wr, 526 op/s 2024-09-14T03:55:30.734 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:30.757 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 83886079~1 2024-09-14T03:55:30.766 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 0~8388609 2024-09-14T03:55:30.775 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 16777218~8388609 2024-09-14T03:55:30.845 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:55:31.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:30 smithi129 ceph-mon[33173]: osdmap e136: 8 total, 8 up, 8 in 2024-09-14T03:55:31.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:30 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:55:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:30 smithi039 ceph-mon[29586]: osdmap e136: 8 total, 8 up, 8 in 2024-09-14T03:55:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:30 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:55:31.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:30 smithi039 ceph-mon[40962]: osdmap e136: 8 total, 8 up, 8 in 2024-09-14T03:55:31.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:30 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:55:32.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:31 smithi129 ceph-mon[33173]: osdmap e137: 8 total, 8 up, 8 in 2024-09-14T03:55:32.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:31 smithi129 ceph-mon[33173]: pgmap v193: 137 pgs: 137 active+clean; 265 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:55:32.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:31 smithi039 ceph-mon[29586]: osdmap e137: 8 total, 8 up, 8 in 2024-09-14T03:55:32.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:31 smithi039 ceph-mon[29586]: pgmap v193: 137 pgs: 137 active+clean; 265 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:55:32.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:31 smithi039 ceph-mon[40962]: osdmap e137: 8 total, 8 up, 8 in 2024-09-14T03:55:32.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:31 smithi039 ceph-mon[40962]: pgmap v193: 137 pgs: 137 active+clean; 265 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:55:32.263 INFO:teuthology.orchestra.run.smithi039.stdout:true 2024-09-14T03:55:32.819 INFO:teuthology.orchestra.run.smithi039.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:55:32.819 INFO:teuthology.orchestra.run.smithi039.stdout:alertmanager.a smithi039 *:9093,9094 running (2m) 94s ago 3m 24.4M - 0.25.0 c8568f914cd2 b5ca6b93d9ee 2024-09-14T03:55:32.819 INFO:teuthology.orchestra.run.smithi039.stdout:grafana.a smithi129 *:3000 running (3m) 31s ago 3m 87.7M - 9.4.7 954c08fa6188 71f297945590 2024-09-14T03:55:32.819 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi039.bbwiwf smithi039 running (2m) 94s ago 2m 16.4M - 18.2.4-845-ge4e227b3 67ddd33fa72f 91e6c8f85681 2024-09-14T03:55:32.819 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi129.ybjqcw smithi129 running (2m) 31s ago 2m 13.3M - 18.2.4-845-ge4e227b3 67ddd33fa72f a3fab5b37b71 2024-09-14T03:55:32.819 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.x smithi129 *:8443,8765 running (7m) 31s ago 7m 439M - 18.2.4-845-ge4e227b3 67ddd33fa72f 813f62a05ea8 2024-09-14T03:55:32.820 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.y smithi039 *:9283,8765 running (9m) 94s ago 9m 484M - 18.2.4-845-ge4e227b3 67ddd33fa72f babf5c5c054b 2024-09-14T03:55:32.820 INFO:teuthology.orchestra.run.smithi039.stdout:mon.a smithi039 running (9m) 94s ago 9m 47.0M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f e7b5f91fb09c 2024-09-14T03:55:32.820 INFO:teuthology.orchestra.run.smithi039.stdout:mon.b smithi129 running (7m) 31s ago 7m 42.4M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f eb38cc5cbe35 2024-09-14T03:55:32.820 INFO:teuthology.orchestra.run.smithi039.stdout:mon.c smithi039 running (7m) 94s ago 7m 37.8M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 31d440e49f93 2024-09-14T03:55:32.820 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.a smithi039 *:9100 running (3m) 94s ago 2m 13.4M - 1.5.0 0da6a335fe13 4eff582f78f5 2024-09-14T03:55:32.820 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.b smithi129 *:9100 running (2m) 31s ago 2m 15.4M - 1.5.0 0da6a335fe13 c7581baf44f2 2024-09-14T03:55:32.820 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0 smithi039 running (7m) 94s ago 7m 43.1M 4016M 18.2.4-845-ge4e227b3 67ddd33fa72f 5048d74f27ac 2024-09-14T03:55:32.820 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1 smithi039 running (6m) 94s ago 6m 42.4M 4016M 18.2.4-845-ge4e227b3 67ddd33fa72f 91349527d88b 2024-09-14T03:55:32.820 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2 smithi039 running (6m) 94s ago 6m 50.2M 4016M 18.2.4-845-ge4e227b3 67ddd33fa72f eb903b44fc5d 2024-09-14T03:55:32.820 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3 smithi039 running (5m) 94s ago 5m 49.0M 4016M 18.2.4-845-ge4e227b3 67ddd33fa72f dff9e665bd9b 2024-09-14T03:55:32.820 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4 smithi129 running (5m) 31s ago 5m 48.4M 4016M 18.2.4-845-ge4e227b3 67ddd33fa72f 5b64727311b7 2024-09-14T03:55:32.820 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5 smithi129 running (5m) 31s ago 5m 55.3M 4016M 18.2.4-845-ge4e227b3 67ddd33fa72f f2d991a754f7 2024-09-14T03:55:32.820 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6 smithi129 running (4m) 31s ago 4m 55.5M 4016M 18.2.4-845-ge4e227b3 67ddd33fa72f 3060eaf44106 2024-09-14T03:55:32.820 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7 smithi129 running (4m) 31s ago 4m 48.1M 4016M 18.2.4-845-ge4e227b3 67ddd33fa72f 4c13b97060d1 2024-09-14T03:55:32.820 INFO:teuthology.orchestra.run.smithi039.stdout:prometheus.a smithi129 *:9095 running (2m) 31s ago 3m 40.8M - 2.43.0 a07b618ecd1d 0275bbd07991 2024-09-14T03:55:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:32 smithi129 ceph-mon[33173]: osdmap e138: 8 total, 8 up, 8 in 2024-09-14T03:55:33.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:32 smithi039 ceph-mon[29586]: osdmap e138: 8 total, 8 up, 8 in 2024-09-14T03:55:33.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:32 smithi039 ceph-mon[40962]: osdmap e138: 8 total, 8 up, 8 in 2024-09-14T03:55:33.156 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T03:55:33.156 INFO:teuthology.orchestra.run.smithi039.stdout: "mon": { 2024-09-14T03:55:33.156 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 3 2024-09-14T03:55:33.156 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:55:33.156 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr": { 2024-09-14T03:55:33.157 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:55:33.157 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:55:33.157 INFO:teuthology.orchestra.run.smithi039.stdout: "osd": { 2024-09-14T03:55:33.157 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8 2024-09-14T03:55:33.157 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:55:33.157 INFO:teuthology.orchestra.run.smithi039.stdout: "mds": { 2024-09-14T03:55:33.157 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:55:33.157 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:55:33.157 INFO:teuthology.orchestra.run.smithi039.stdout: "overall": { 2024-09-14T03:55:33.157 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 15 2024-09-14T03:55:33.157 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T03:55:33.157 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T03:55:33.439 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T03:55:33.439 INFO:teuthology.orchestra.run.smithi039.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:55:33.440 INFO:teuthology.orchestra.run.smithi039.stdout: "in_progress": true, 2024-09-14T03:55:33.440 INFO:teuthology.orchestra.run.smithi039.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-14T03:55:33.440 INFO:teuthology.orchestra.run.smithi039.stdout: "services_complete": [], 2024-09-14T03:55:33.440 INFO:teuthology.orchestra.run.smithi039.stdout: "progress": "0/2 daemons upgraded", 2024-09-14T03:55:33.440 INFO:teuthology.orchestra.run.smithi039.stdout: "message": "Currently upgrading mgr daemons", 2024-09-14T03:55:33.440 INFO:teuthology.orchestra.run.smithi039.stdout: "is_paused": false 2024-09-14T03:55:33.440 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T03:55:34.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:33 smithi039 ceph-mon[29586]: from='client.24839 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:34.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:33 smithi039 ceph-mon[29586]: from='client.14940 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:34.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:33 smithi039 ceph-mon[29586]: osdmap e139: 8 total, 8 up, 8 in 2024-09-14T03:55:34.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:33 smithi039 ceph-mon[29586]: pgmap v196: 137 pgs: 137 active+clean; 313 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 127 KiB/s rd, 10 MiB/s wr, 187 op/s 2024-09-14T03:55:34.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:33 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:34.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:33 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:55:34.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:33 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:55:34.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:33 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:55:34.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:33 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:55:34.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:33 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/297850279' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:55:34.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:33 smithi039 ceph-mon[40962]: from='client.24839 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:34.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:33 smithi039 ceph-mon[40962]: from='client.14940 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:34.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:33 smithi039 ceph-mon[40962]: osdmap e139: 8 total, 8 up, 8 in 2024-09-14T03:55:34.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:33 smithi039 ceph-mon[40962]: pgmap v196: 137 pgs: 137 active+clean; 313 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 127 KiB/s rd, 10 MiB/s wr, 187 op/s 2024-09-14T03:55:34.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:33 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:34.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:33 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:55:34.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:33 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:55:34.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:33 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:55:34.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:33 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:55:34.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:33 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/297850279' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:55:34.170 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:55:34.268 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:33 smithi129 ceph-mon[33173]: from='client.24839 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:34.269 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:33 smithi129 ceph-mon[33173]: from='client.14940 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:34.269 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:33 smithi129 ceph-mon[33173]: osdmap e139: 8 total, 8 up, 8 in 2024-09-14T03:55:34.269 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:33 smithi129 ceph-mon[33173]: pgmap v196: 137 pgs: 137 active+clean; 313 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 127 KiB/s rd, 10 MiB/s wr, 187 op/s 2024-09-14T03:55:34.269 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:33 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:34.269 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:33 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:55:34.269 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:33 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:55:34.269 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:33 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:55:34.269 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:33 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:55:34.269 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:33 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/297850279' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:55:34.350 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:55:34.813 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:34 smithi129 ceph-mon[33173]: from='client.14946 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:34.813 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:34 smithi129 ceph-mon[33173]: Upgrade: Updating mgr.x 2024-09-14T03:55:34.813 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:34 smithi129 ceph-mon[33173]: Deploying daemon mgr.x on smithi129 2024-09-14T03:55:34.813 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:34 smithi129 ceph-mon[33173]: from='client.14958 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:34.813 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:34 smithi129 ceph-mon[33173]: osdmap e140: 8 total, 8 up, 8 in 2024-09-14T03:55:35.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:34 smithi039 ceph-mon[29586]: from='client.14946 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:35.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:34 smithi039 ceph-mon[29586]: Upgrade: Updating mgr.x 2024-09-14T03:55:35.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:34 smithi039 ceph-mon[29586]: Deploying daemon mgr.x on smithi129 2024-09-14T03:55:35.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:34 smithi039 ceph-mon[29586]: from='client.14958 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:35.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:34 smithi039 ceph-mon[29586]: osdmap e140: 8 total, 8 up, 8 in 2024-09-14T03:55:35.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:34 smithi039 ceph-mon[40962]: from='client.14946 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:35.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:34 smithi039 ceph-mon[40962]: Upgrade: Updating mgr.x 2024-09-14T03:55:35.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:34 smithi039 ceph-mon[40962]: Deploying daemon mgr.x on smithi129 2024-09-14T03:55:35.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:34 smithi039 ceph-mon[40962]: from='client.14958 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:55:35.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:34 smithi039 ceph-mon[40962]: osdmap e140: 8 total, 8 up, 8 in 2024-09-14T03:55:35.095 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:34 smithi129 systemd[1]: Stopping Ceph mgr.x for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:55:35.382 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:35 smithi129 podman[69430]: 2024-09-14 03:55:35.094903385 +0000 UTC m=+0.138883009 container died 813f62a05ea8d3b352c3dce63048b3a48e33f627fa1418e82d5b58549d2c86c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x, RELEASE=reef-e4e227b, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-14T03:55:36.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:35 smithi129 ceph-mon[33173]: pgmap v198: 137 pgs: 137 active+clean; 313 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 125 KiB/s rd, 9.9 MiB/s wr, 184 op/s 2024-09-14T03:55:36.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:35 smithi129 ceph-mon[33173]: osdmap e141: 8 total, 8 up, 8 in 2024-09-14T03:55:36.021 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:35 smithi129 podman[69430]: 2024-09-14 03:55:35.740261795 +0000 UTC m=+0.784241415 container cleanup 813f62a05ea8d3b352c3dce63048b3a48e33f627fa1418e82d5b58549d2c86c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, RELEASE=reef-e4e227b, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS) 2024-09-14T03:55:36.021 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:35 smithi129 bash[69430]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x 2024-09-14T03:55:36.021 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:35 smithi129 podman[69449]: 2024-09-14 03:55:35.900563518 +0000 UTC m=+0.796625636 container remove 813f62a05ea8d3b352c3dce63048b3a48e33f627fa1418e82d5b58549d2c86c3 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=reef-e4e227b, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-14T03:55:36.022 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:35 smithi129 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-14T03:55:36.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:35 smithi039 ceph-mon[29586]: pgmap v198: 137 pgs: 137 active+clean; 313 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 125 KiB/s rd, 9.9 MiB/s wr, 184 op/s 2024-09-14T03:55:36.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:35 smithi039 ceph-mon[29586]: osdmap e141: 8 total, 8 up, 8 in 2024-09-14T03:55:36.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:35 smithi039 ceph-mon[40962]: pgmap v198: 137 pgs: 137 active+clean; 313 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 125 KiB/s rd, 9.9 MiB/s wr, 184 op/s 2024-09-14T03:55:36.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:35 smithi039 ceph-mon[40962]: osdmap e141: 8 total, 8 up, 8 in 2024-09-14T03:55:36.521 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:36 smithi129 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-14T03:55:36.522 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:36 smithi129 systemd[1]: Stopped Ceph mgr.x for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:55:36.522 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:36 smithi129 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mgr.x.service: Consumed 11.007s CPU time. 2024-09-14T03:55:36.522 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:36 smithi129 systemd[1]: Starting Ceph mgr.x for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:55:36.813 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:36 smithi129 podman[69580]: 2024-09-14 03:55:36.460617262 +0000 UTC m=+0.015535529 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:55:36.813 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:36 smithi129 podman[69580]: 2024-09-14 03:55:36.576561503 +0000 UTC m=+0.131479759 container create 759df831a5deb6dd936eb415a7792ddabd78086c3bffab265146384dcfa61d65 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x, io.buildah.version=1.37.2, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=main-a278231, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-14T03:55:36.813 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:36 smithi129 podman[69580]: 2024-09-14 03:55:36.812483347 +0000 UTC m=+0.367401602 container init 759df831a5deb6dd936eb415a7792ddabd78086c3bffab265146384dcfa61d65 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x, io.buildah.version=1.37.2, ceph=True, RELEASE=main-a278231, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS) 2024-09-14T03:55:37.124 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.SnapsOtherPool (11825 ms) 2024-09-14T03:55:37.124 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.SnapsDataPool 2024-09-14T03:55:37.156 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:37.167 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 0~8388609 2024-09-14T03:55:37.240 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap1 2024-09-14T03:55:37.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:36 smithi129 ceph-mon[33173]: osdmap e142: 8 total, 8 up, 8 in 2024-09-14T03:55:37.271 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:36 smithi129 podman[69580]: 2024-09-14 03:55:36.817582721 +0000 UTC m=+0.372500986 container start 759df831a5deb6dd936eb415a7792ddabd78086c3bffab265146384dcfa61d65 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-14T03:55:37.271 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:36 smithi129 ceph-mgr[69618]: -- 172.21.15.129:0/208799239 <== mon.1 v2:172.21.15.129:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x562b03ba50e0 con 0x562b03b83000 2024-09-14T03:55:37.271 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:36 smithi129 bash[69580]: 759df831a5deb6dd936eb415a7792ddabd78086c3bffab265146384dcfa61d65 2024-09-14T03:55:37.271 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:36 smithi129 systemd[1]: Started Ceph mgr.x for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:55:37.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:36 smithi039 ceph-mon[29586]: osdmap e142: 8 total, 8 up, 8 in 2024-09-14T03:55:37.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:36 smithi039 ceph-mon[40962]: osdmap e142: 8 total, 8 up, 8 in 2024-09-14T03:55:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:37 smithi129 ceph-mon[33173]: pgmap v201: 137 pgs: 1 active+clean+snaptrim, 136 active+clean; 373 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 46 MiB/s rd, 16 MiB/s wr, 421 op/s 2024-09-14T03:55:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:37 smithi129 ceph-mon[33173]: osdmap e143: 8 total, 8 up, 8 in 2024-09-14T03:55:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:37 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:37 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:38.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:37 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:55:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:37 smithi039 ceph-mon[29586]: pgmap v201: 137 pgs: 1 active+clean+snaptrim, 136 active+clean; 373 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 46 MiB/s rd, 16 MiB/s wr, 421 op/s 2024-09-14T03:55:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:37 smithi039 ceph-mon[29586]: osdmap e143: 8 total, 8 up, 8 in 2024-09-14T03:55:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:37 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:37 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:37 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:55:38.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:37 smithi039 ceph-mon[40962]: pgmap v201: 137 pgs: 1 active+clean+snaptrim, 136 active+clean; 373 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 46 MiB/s rd, 16 MiB/s wr, 421 op/s 2024-09-14T03:55:38.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:37 smithi039 ceph-mon[40962]: osdmap e143: 8 total, 8 up, 8 in 2024-09-14T03:55:38.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:37 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:38.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:37 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:38.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:37 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:55:38.716 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 8388609~8388609 2024-09-14T03:55:38.727 INFO:tasks.workunit.client.0.smithi039.stdout:write: X 0~1000 2024-09-14T03:55:38.727 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 1010~1000 2024-09-14T03:55:38.727 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap2 2024-09-14T03:55:39.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:39 smithi129 ceph-mon[33173]: osdmap e144: 8 total, 8 up, 8 in 2024-09-14T03:55:39.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:39 smithi129 ceph-mon[33173]: osdmap e145: 8 total, 8 up, 8 in 2024-09-14T03:55:39.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:39 smithi129 ceph-mon[33173]: pgmap v205: 137 pgs: 1 active+clean+snaptrim, 136 active+clean; 373 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 46 MiB/s rd, 16 MiB/s wr, 425 op/s 2024-09-14T03:55:39.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:39 smithi039 ceph-mon[29586]: osdmap e144: 8 total, 8 up, 8 in 2024-09-14T03:55:39.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:39 smithi039 ceph-mon[29586]: osdmap e145: 8 total, 8 up, 8 in 2024-09-14T03:55:39.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:39 smithi039 ceph-mon[29586]: pgmap v205: 137 pgs: 1 active+clean+snaptrim, 136 active+clean; 373 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 46 MiB/s rd, 16 MiB/s wr, 425 op/s 2024-09-14T03:55:39.291 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:39 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:55:39] "GET /metrics HTTP/1.1" 200 38783 "" "Prometheus/2.43.0" 2024-09-14T03:55:39.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:39 smithi039 ceph-mon[40962]: osdmap e144: 8 total, 8 up, 8 in 2024-09-14T03:55:39.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:39 smithi039 ceph-mon[40962]: osdmap e145: 8 total, 8 up, 8 in 2024-09-14T03:55:39.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:39 smithi039 ceph-mon[40962]: pgmap v205: 137 pgs: 1 active+clean+snaptrim, 136 active+clean; 373 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 46 MiB/s rd, 16 MiB/s wr, 425 op/s 2024-09-14T03:55:40.738 INFO:tasks.workunit.client.0.smithi039.stdout:write: X 8388609~1000 2024-09-14T03:55:40.744 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 16777228~1000 2024-09-14T03:55:40.744 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 167772160 2024-09-14T03:55:40.753 INFO:tasks.workunit.client.0.smithi039.stdout:write: 2 83886079~8388609 2024-09-14T03:55:40.768 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap3 2024-09-14T03:55:40.992 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:40 smithi129 ceph-mon[33173]: osdmap e146: 8 total, 8 up, 8 in 2024-09-14T03:55:40.992 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:40 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:40.993 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:40 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:41.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:40 smithi039 ceph-mon[40962]: osdmap e146: 8 total, 8 up, 8 in 2024-09-14T03:55:41.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:40 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:41.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:40 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:41.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:40 smithi039 ceph-mon[29586]: osdmap e146: 8 total, 8 up, 8 in 2024-09-14T03:55:41.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:40 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:41.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:40 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:42.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:41 smithi129 ceph-mon[33173]: osdmap e147: 8 total, 8 up, 8 in 2024-09-14T03:55:42.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:41 smithi129 ceph-mon[33173]: pgmap v208: 137 pgs: 1 active+clean+snaptrim, 136 active+clean; 373 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:55:42.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:41 smithi039 ceph-mon[29586]: osdmap e147: 8 total, 8 up, 8 in 2024-09-14T03:55:42.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:41 smithi039 ceph-mon[29586]: pgmap v208: 137 pgs: 1 active+clean+snaptrim, 136 active+clean; 373 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:55:42.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:41 smithi039 ceph-mon[40962]: osdmap e147: 8 total, 8 up, 8 in 2024-09-14T03:55:42.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:41 smithi039 ceph-mon[40962]: pgmap v208: 137 pgs: 1 active+clean+snaptrim, 136 active+clean; 373 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:55:42.521 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:42 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: [14/Sep/2024:03:55:42] ENGINE Bus STARTING 2024-09-14T03:55:42.521 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:42 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: CherryPy Checker: 2024-09-14T03:55:42.521 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:42 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: The Application mounted at '' has an empty config. 2024-09-14T03:55:42.521 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:42 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: 2024-09-14T03:55:42.521 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:42 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: [14/Sep/2024:03:55:42] ENGINE Serving on http://:::9283 2024-09-14T03:55:42.522 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:42 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: [14/Sep/2024:03:55:42] ENGINE Bus STARTED 2024-09-14T03:55:42.774 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:42.797 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 83886079~1 2024-09-14T03:55:42.804 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:55:43.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:42 smithi129 ceph-mon[33173]: osdmap e148: 8 total, 8 up, 8 in 2024-09-14T03:55:43.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:42 smithi129 ceph-mon[33173]: Standby manager daemon x restarted 2024-09-14T03:55:43.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:42 smithi129 ceph-mon[33173]: Standby manager daemon x started 2024-09-14T03:55:43.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:42 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:55:43.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:42 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:55:43.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:42 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:55:43.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:42 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:55:43.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:42 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:43.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:42 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:43.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:42 smithi039 ceph-mon[29586]: osdmap e148: 8 total, 8 up, 8 in 2024-09-14T03:55:43.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:42 smithi039 ceph-mon[29586]: Standby manager daemon x restarted 2024-09-14T03:55:43.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:42 smithi039 ceph-mon[29586]: Standby manager daemon x started 2024-09-14T03:55:43.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:42 smithi039 ceph-mon[29586]: from='mgr.? 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:55:43.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:42 smithi039 ceph-mon[29586]: from='mgr.? 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:55:43.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:42 smithi039 ceph-mon[29586]: from='mgr.? 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:55:43.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:42 smithi039 ceph-mon[29586]: from='mgr.? 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:55:43.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:42 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:43.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:42 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:43.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:42 smithi039 ceph-mon[40962]: osdmap e148: 8 total, 8 up, 8 in 2024-09-14T03:55:43.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:42 smithi039 ceph-mon[40962]: Standby manager daemon x restarted 2024-09-14T03:55:43.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:42 smithi039 ceph-mon[40962]: Standby manager daemon x started 2024-09-14T03:55:43.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:42 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:55:43.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:42 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:55:43.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:42 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:55:43.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:42 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:55:43.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:42 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:43.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:42 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:44.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:43 smithi129 ceph-mon[33173]: mgrmap e22: y(active, since 3m), standbys: x 2024-09-14T03:55:44.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:43 smithi129 ceph-mon[33173]: osdmap e149: 8 total, 8 up, 8 in 2024-09-14T03:55:44.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:43 smithi129 ceph-mon[33173]: pgmap v211: 137 pgs: 137 active+clean; 393 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 36 MiB/s rd, 12 MiB/s wr, 184 op/s 2024-09-14T03:55:44.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:43 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:44.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:43 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:44.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:43 smithi039 ceph-mon[29586]: mgrmap e22: y(active, since 3m), standbys: x 2024-09-14T03:55:44.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:43 smithi039 ceph-mon[29586]: osdmap e149: 8 total, 8 up, 8 in 2024-09-14T03:55:44.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:43 smithi039 ceph-mon[29586]: pgmap v211: 137 pgs: 137 active+clean; 393 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 36 MiB/s rd, 12 MiB/s wr, 184 op/s 2024-09-14T03:55:44.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:43 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:44.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:43 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:44.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:43 smithi039 ceph-mon[40962]: mgrmap e22: y(active, since 3m), standbys: x 2024-09-14T03:55:44.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:43 smithi039 ceph-mon[40962]: osdmap e149: 8 total, 8 up, 8 in 2024-09-14T03:55:44.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:43 smithi039 ceph-mon[40962]: pgmap v211: 137 pgs: 137 active+clean; 393 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 36 MiB/s rd, 12 MiB/s wr, 184 op/s 2024-09-14T03:55:44.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:43 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:44.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:43 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:44 smithi039 ceph-mon[29586]: osdmap e150: 8 total, 8 up, 8 in 2024-09-14T03:55:45.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:44 smithi039 ceph-mon[40962]: osdmap e150: 8 total, 8 up, 8 in 2024-09-14T03:55:45.231 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:44 smithi129 ceph-mon[33173]: osdmap e150: 8 total, 8 up, 8 in 2024-09-14T03:55:46.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:45 smithi039 ceph-mon[40962]: osdmap e151: 8 total, 8 up, 8 in 2024-09-14T03:55:46.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:45 smithi039 ceph-mon[40962]: pgmap v214: 137 pgs: 137 active+clean; 393 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 36 MiB/s rd, 12 MiB/s wr, 184 op/s 2024-09-14T03:55:46.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:45 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:55:46.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:45 smithi039 ceph-mon[29586]: osdmap e151: 8 total, 8 up, 8 in 2024-09-14T03:55:46.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:45 smithi039 ceph-mon[29586]: pgmap v214: 137 pgs: 137 active+clean; 393 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 36 MiB/s rd, 12 MiB/s wr, 184 op/s 2024-09-14T03:55:46.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:45 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:55:46.222 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:55:46.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:45 smithi129 ceph-mon[33173]: osdmap e151: 8 total, 8 up, 8 in 2024-09-14T03:55:46.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:45 smithi129 ceph-mon[33173]: pgmap v214: 137 pgs: 137 active+clean; 393 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 36 MiB/s rd, 12 MiB/s wr, 184 op/s 2024-09-14T03:55:46.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:45 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:55:46.359 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:55:47.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:46 smithi129 ceph-mon[33173]: osdmap e152: 8 total, 8 up, 8 in 2024-09-14T03:55:47.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:46 smithi039 ceph-mon[29586]: osdmap e152: 8 total, 8 up, 8 in 2024-09-14T03:55:47.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:46 smithi039 ceph-mon[40962]: osdmap e152: 8 total, 8 up, 8 in 2024-09-14T03:55:48.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:47 smithi129 ceph-mon[33173]: pgmap v216: 137 pgs: 137 active+clean; 393 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 88 KiB/s rd, 11 KiB/s wr, 124 op/s 2024-09-14T03:55:48.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:47 smithi129 ceph-mon[33173]: osdmap e153: 8 total, 8 up, 8 in 2024-09-14T03:55:48.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:47 smithi039 ceph-mon[29586]: pgmap v216: 137 pgs: 137 active+clean; 393 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 88 KiB/s rd, 11 KiB/s wr, 124 op/s 2024-09-14T03:55:48.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:47 smithi039 ceph-mon[29586]: osdmap e153: 8 total, 8 up, 8 in 2024-09-14T03:55:48.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:47 smithi039 ceph-mon[40962]: pgmap v216: 137 pgs: 137 active+clean; 393 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 88 KiB/s rd, 11 KiB/s wr, 124 op/s 2024-09-14T03:55:48.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:47 smithi039 ceph-mon[40962]: osdmap e153: 8 total, 8 up, 8 in 2024-09-14T03:55:49.030 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.SnapsDataPool (11905 ms) 2024-09-14T03:55:49.030 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.SnapsShrinkAfterPrepare 2024-09-14T03:55:49.065 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:49.076 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 0~8388609 2024-09-14T03:55:49.136 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap1 2024-09-14T03:55:49.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:48 smithi129 ceph-mon[33173]: osdmap e154: 8 total, 8 up, 8 in 2024-09-14T03:55:49.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:48 smithi129 ceph-mon[33173]: osdmap e155: 8 total, 8 up, 8 in 2024-09-14T03:55:49.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:48 smithi039 ceph-mon[29586]: osdmap e154: 8 total, 8 up, 8 in 2024-09-14T03:55:49.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:48 smithi039 ceph-mon[29586]: osdmap e155: 8 total, 8 up, 8 in 2024-09-14T03:55:49.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:49 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:55:49] "GET /metrics HTTP/1.1" 200 38786 "" "Prometheus/2.43.0" 2024-09-14T03:55:49.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:48 smithi039 ceph-mon[40962]: osdmap e154: 8 total, 8 up, 8 in 2024-09-14T03:55:49.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:48 smithi039 ceph-mon[40962]: osdmap e155: 8 total, 8 up, 8 in 2024-09-14T03:55:50.193 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:49 smithi129 ceph-mon[33173]: pgmap v220: 137 pgs: 137 active+clean; 393 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 88 KiB/s rd, 11 KiB/s wr, 124 op/s 2024-09-14T03:55:50.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:49 smithi039 ceph-mon[29586]: pgmap v220: 137 pgs: 137 active+clean; 393 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 88 KiB/s rd, 11 KiB/s wr, 124 op/s 2024-09-14T03:55:50.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:49 smithi039 ceph-mon[40962]: pgmap v220: 137 pgs: 137 active+clean; 393 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 88 KiB/s rd, 11 KiB/s wr, 124 op/s 2024-09-14T03:55:50.843 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 8388609~8388609 2024-09-14T03:55:50.865 INFO:tasks.workunit.client.0.smithi039.stdout:write: X 0~1000 2024-09-14T03:55:50.865 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 1010~1000 2024-09-14T03:55:50.865 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap2 2024-09-14T03:55:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:50 smithi129 ceph-mon[33173]: osdmap e156: 8 total, 8 up, 8 in 2024-09-14T03:55:51.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:50 smithi039 ceph-mon[29586]: osdmap e156: 8 total, 8 up, 8 in 2024-09-14T03:55:51.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:50 smithi039 ceph-mon[40962]: osdmap e156: 8 total, 8 up, 8 in 2024-09-14T03:55:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:51 smithi129 ceph-mon[33173]: pgmap v222: 137 pgs: 137 active+clean; 393 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:55:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:51 smithi129 ceph-mon[33173]: osdmap e157: 8 total, 8 up, 8 in 2024-09-14T03:55:52.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:51 smithi039 ceph-mon[29586]: pgmap v222: 137 pgs: 137 active+clean; 393 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:55:52.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:51 smithi039 ceph-mon[29586]: osdmap e157: 8 total, 8 up, 8 in 2024-09-14T03:55:52.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:51 smithi039 ceph-mon[40962]: pgmap v222: 137 pgs: 137 active+clean; 393 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:55:52.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:51 smithi039 ceph-mon[40962]: osdmap e157: 8 total, 8 up, 8 in 2024-09-14T03:55:52.879 INFO:tasks.workunit.client.0.smithi039.stdout:write: X 8388609~1000 2024-09-14T03:55:52.886 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 16777228~1000 2024-09-14T03:55:52.886 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 167772160 2024-09-14T03:55:52.895 INFO:tasks.workunit.client.0.smithi039.stdout:write: 2 83886079~8388609 2024-09-14T03:55:52.918 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap3 2024-09-14T03:55:53.192 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:52 smithi039 ceph-mon[29586]: osdmap e158: 8 total, 8 up, 8 in 2024-09-14T03:55:53.192 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:52 smithi039 ceph-mon[29586]: pgmap v225: 137 pgs: 137 active+clean; 409 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 69 MiB/s rd, 9.8 MiB/s wr, 466 op/s 2024-09-14T03:55:53.192 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:52 smithi039 ceph-mon[40962]: osdmap e158: 8 total, 8 up, 8 in 2024-09-14T03:55:53.192 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:52 smithi039 ceph-mon[40962]: pgmap v225: 137 pgs: 137 active+clean; 409 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 69 MiB/s rd, 9.8 MiB/s wr, 466 op/s 2024-09-14T03:55:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:52 smithi129 ceph-mon[33173]: osdmap e158: 8 total, 8 up, 8 in 2024-09-14T03:55:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:52 smithi129 ceph-mon[33173]: pgmap v225: 137 pgs: 137 active+clean; 409 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 69 MiB/s rd, 9.8 MiB/s wr, 466 op/s 2024-09-14T03:55:54.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:53 smithi129 ceph-mon[33173]: osdmap e159: 8 total, 8 up, 8 in 2024-09-14T03:55:54.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:53 smithi129 ceph-mon[33173]: osdmap e160: 8 total, 8 up, 8 in 2024-09-14T03:55:54.205 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:53 smithi039 ceph-mon[29586]: osdmap e159: 8 total, 8 up, 8 in 2024-09-14T03:55:54.205 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:53 smithi039 ceph-mon[29586]: osdmap e160: 8 total, 8 up, 8 in 2024-09-14T03:55:54.205 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:53 smithi039 ceph-mon[40962]: osdmap e159: 8 total, 8 up, 8 in 2024-09-14T03:55:54.206 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:53 smithi039 ceph-mon[40962]: osdmap e160: 8 total, 8 up, 8 in 2024-09-14T03:55:54.728 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:55:54.749 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 83886079~1 2024-09-14T03:55:54.758 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:55:55.521 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: [14/Sep/2024:03:55:55] ENGINE Bus STOPPING 2024-09-14T03:55:55.664 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:55:55.296+0000 7fbe459e8640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-14T03:55:55.665 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ignoring --setuser ceph since I am not root 2024-09-14T03:55:55.665 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ignoring --setgroup ceph since I am not root 2024-09-14T03:55:55.665 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:55:55.446+0000 7fc75ed73040 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-09-14T03:55:55.665 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:55:55.507+0000 7fc75ed73040 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-09-14T03:55:55.665 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:55:55.662+0000 7fc75ed73040 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-09-14T03:55:56.021 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: [14/Sep/2024:03:55:55] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:55:56.021 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: [14/Sep/2024:03:55:55] ENGINE Bus STOPPED 2024-09-14T03:55:56.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-mon[33173]: osdmap e161: 8 total, 8 up, 8 in 2024-09-14T03:55:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-mon[33173]: pgmap v229: 137 pgs: 137 active+clean; 409 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:55:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:55:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:55:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:55:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-mon[33173]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-14T03:55:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-14T03:55:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-mon[33173]: Activating manager daemon x 2024-09-14T03:55:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-mon[33173]: osdmap e162: 8 total, 8 up, 8 in 2024-09-14T03:55:56.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-mon[33173]: from='mgr.14436 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-14T03:55:56.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-mon[33173]: mgrmap e23: x(active, starting, since 0.00814455s) 2024-09-14T03:55:56.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:55:56.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:55:56.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:55:56.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi039.bbwiwf"}]: dispatch 2024-09-14T03:55:56.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi129.ybjqcw"}]: dispatch 2024-09-14T03:55:56.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:55:56.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:55:56.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:55:56.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:55:56.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:55:56.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:55:56.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:55:56.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:55:56.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:55:56.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:55:56.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:55:56.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:55 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:55:56.034 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:55:55.733+0000 7fc75ed73040 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-09-14T03:55:56.034 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:55:55.887+0000 7fc75ed73040 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:55:56.035 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:56 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:55:56.033+0000 7fc75ed73040 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-09-14T03:55:56.035 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[29586]: osdmap e161: 8 total, 8 up, 8 in 2024-09-14T03:55:56.035 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[29586]: pgmap v229: 137 pgs: 137 active+clean; 409 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:55:56.035 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:56.035 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:56.035 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:55:56.035 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:55:56.035 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:56.036 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:55:56.036 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[29586]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-14T03:55:56.036 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-14T03:55:56.036 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[29586]: Activating manager daemon x 2024-09-14T03:55:56.036 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[29586]: osdmap e162: 8 total, 8 up, 8 in 2024-09-14T03:55:56.036 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[29586]: from='mgr.14436 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-14T03:55:56.036 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[29586]: mgrmap e23: x(active, starting, since 0.00814455s) 2024-09-14T03:55:56.036 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:55:56.036 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:55:56.036 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:55:56.036 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi039.bbwiwf"}]: dispatch 2024-09-14T03:55:56.036 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi129.ybjqcw"}]: dispatch 2024-09-14T03:55:56.036 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:55:56.037 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:55:56.037 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:55:56.037 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:55:56.037 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:55:56.037 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:55:56.037 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:55:56.037 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:55:56.037 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:55:56.037 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:55:56.037 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:55:56.037 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:55:56.037 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[40962]: osdmap e161: 8 total, 8 up, 8 in 2024-09-14T03:55:56.038 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[40962]: pgmap v229: 137 pgs: 137 active+clean; 409 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:55:56.038 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:56.038 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:56.038 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:55:56.038 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:55:56.038 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' 2024-09-14T03:55:56.038 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:55:56.038 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[40962]: from='mgr.14436 172.21.15.39:0/1957523802' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-14T03:55:56.038 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-09-14T03:55:56.038 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[40962]: Activating manager daemon x 2024-09-14T03:55:56.038 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[40962]: osdmap e162: 8 total, 8 up, 8 in 2024-09-14T03:55:56.038 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[40962]: from='mgr.14436 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-09-14T03:55:56.038 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[40962]: mgrmap e23: x(active, starting, since 0.00814455s) 2024-09-14T03:55:56.038 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:55:56.038 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:55:56.039 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:55:56.039 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi039.bbwiwf"}]: dispatch 2024-09-14T03:55:56.039 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi129.ybjqcw"}]: dispatch 2024-09-14T03:55:56.039 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:55:56.039 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:55:56.039 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:55:56.039 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:55:56.039 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:55:56.039 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:55:56.039 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:55:56.039 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:55:56.039 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:55:56.039 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:55:56.039 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:55:56.039 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:55 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:55:56.618 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:56 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:55:56.362+0000 7fc75ed73040 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-09-14T03:55:56.618 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:56 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:55:56.425+0000 7fc75ed73040 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-09-14T03:55:56.618 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:56 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:55:56.617+0000 7fc75ed73040 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-09-14T03:55:57.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:56 smithi129 ceph-mon[33173]: Manager daemon x is now available 2024-09-14T03:55:57.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:56 smithi129 ceph-mon[33173]: osdmap e163: 8 total, 8 up, 8 in 2024-09-14T03:55:57.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:56 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:55:57.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:56 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:55:57.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:56 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:55:57.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:56 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:55:57.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:56 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:55:57.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:56 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:55:57.022 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:56 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: [14/Sep/2024:03:55:56] ENGINE Bus STARTING 2024-09-14T03:55:57.022 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:56 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: 2024-09-14T03:55:56.816+0000 7fc3c35ea640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:55:57.022 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:56 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: 2024-09-14T03:55:56.816+0000 7fc3c35ea640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:55:57.022 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:56 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: 2024-09-14T03:55:56.816+0000 7fc3c35ea640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:55:57.022 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:56 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: 2024-09-14T03:55:56.816+0000 7fc3c35ea640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:55:57.022 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:56 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: 2024-09-14T03:55:56.816+0000 7fc3c35ea640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:55:57.023 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:56 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: 2024-09-14T03:55:56.817+0000 7fc3c15e6640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:55:57.023 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:56 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: 2024-09-14T03:55:56.817+0000 7fc3c15e6640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:55:57.023 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:56 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: 2024-09-14T03:55:56.817+0000 7fc3c15e6640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:55:57.023 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:56 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: 2024-09-14T03:55:56.817+0000 7fc3c15e6640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:55:57.023 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:56 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: 2024-09-14T03:55:56.817+0000 7fc3c15e6640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:55:57.023 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:56 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: [14/Sep/2024:03:55:56] ENGINE Serving on http://:::9283 2024-09-14T03:55:57.023 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:56 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: [14/Sep/2024:03:55:56] ENGINE Bus STARTED 2024-09-14T03:55:57.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:56 smithi039 ceph-mon[29586]: Manager daemon x is now available 2024-09-14T03:55:57.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:56 smithi039 ceph-mon[29586]: osdmap e163: 8 total, 8 up, 8 in 2024-09-14T03:55:57.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:56 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:55:57.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:56 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:55:57.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:56 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:55:57.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:56 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:55:57.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:56 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:55:57.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:56 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:55:57.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:56 smithi039 ceph-mon[40962]: Manager daemon x is now available 2024-09-14T03:55:57.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:56 smithi039 ceph-mon[40962]: osdmap e163: 8 total, 8 up, 8 in 2024-09-14T03:55:57.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:56 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:55:57.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:56 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:55:57.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:56 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:55:57.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:56 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:55:57.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:56 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:55:57.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:56 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:55:57.734 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 41943040 2024-09-14T03:55:57.766 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:55:57.929 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:55:58.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:57 smithi129 ceph-mon[33173]: Checking for cert/key for grafana.a 2024-09-14T03:55:58.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:57 smithi129 ceph-mon[33173]: Migrating grafana.a cert to cert store 2024-09-14T03:55:58.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:57 smithi129 ceph-mon[33173]: Migrating grafana.a key to cert store 2024-09-14T03:55:58.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:57 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:55:58.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:57 smithi129 ceph-mon[33173]: mgrmap e24: x(active, since 1.45705s) 2024-09-14T03:55:58.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:57 smithi129 ceph-mon[33173]: pgmap v4: 137 pgs: 137 active+clean; 449 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:55:58.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:57 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:55:58.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:57 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:55:58.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:57 smithi129 ceph-mon[33173]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:55:58.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:57 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:55:58.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:57 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:55:58.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:57 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-14T03:55:58.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:57 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-14T03:55:58.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:57 smithi129 ceph-mon[33173]: osdmap e164: 8 total, 8 up, 8 in 2024-09-14T03:55:58.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:57 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:55:57.598+0000 7fc75ed73040 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-09-14T03:55:58.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:57 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:55:57.683+0000 7fc75ed73040 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-09-14T03:55:58.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:57 smithi039 ceph-mon[29586]: Checking for cert/key for grafana.a 2024-09-14T03:55:58.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:57 smithi039 ceph-mon[29586]: Migrating grafana.a cert to cert store 2024-09-14T03:55:58.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:57 smithi039 ceph-mon[29586]: Migrating grafana.a key to cert store 2024-09-14T03:55:58.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:57 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:55:58.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:57 smithi039 ceph-mon[29586]: mgrmap e24: x(active, since 1.45705s) 2024-09-14T03:55:58.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:57 smithi039 ceph-mon[29586]: pgmap v4: 137 pgs: 137 active+clean; 449 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:55:58.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:57 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:55:58.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:57 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:55:58.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:57 smithi039 ceph-mon[29586]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:55:58.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:57 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:55:58.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:57 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:55:58.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:57 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-14T03:55:58.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:57 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-14T03:55:58.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:57 smithi039 ceph-mon[29586]: osdmap e164: 8 total, 8 up, 8 in 2024-09-14T03:55:58.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:57 smithi039 ceph-mon[40962]: Checking for cert/key for grafana.a 2024-09-14T03:55:58.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:57 smithi039 ceph-mon[40962]: Migrating grafana.a cert to cert store 2024-09-14T03:55:58.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:57 smithi039 ceph-mon[40962]: Migrating grafana.a key to cert store 2024-09-14T03:55:58.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:57 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:55:58.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:57 smithi039 ceph-mon[40962]: mgrmap e24: x(active, since 1.45705s) 2024-09-14T03:55:58.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:57 smithi039 ceph-mon[40962]: pgmap v4: 137 pgs: 137 active+clean; 449 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:55:58.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:57 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:55:58.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:57 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:55:58.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:57 smithi039 ceph-mon[40962]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:55:58.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:57 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:55:58.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:57 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:55:58.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:57 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-14T03:55:58.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:57 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-09-14T03:55:58.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:57 smithi039 ceph-mon[40962]: osdmap e164: 8 total, 8 up, 8 in 2024-09-14T03:55:58.522 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:58 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: 2024-09-14T03:55:58.076+0000 7fc3a72f5640 -1 log_channel(cephadm) log [ERR] : [14/Sep/2024:03:55:58] ENGINE Error in HTTPServer.serve 2024-09-14T03:55:58.522 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:58 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: Traceback (most recent call last): 2024-09-14T03:55:58.522 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:58 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:55:58.522 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:58 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: self._connections.run(self.expiration_interval) 2024-09-14T03:55:58.522 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:58 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:55:58.522 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:58 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: self._run(expiration_interval) 2024-09-14T03:55:58.522 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:58 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:55:58.522 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:58 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:55:58.522 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:58 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:55:58.523 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:58 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:55:58.523 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:58 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:55:58.523 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:58 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: s = self.context.wrap_socket( 2024-09-14T03:55:58.523 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:58 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:55:58.523 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:58 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: return self.sslsocket_class._create( 2024-09-14T03:55:58.523 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:58 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:55:58.523 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:58 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: self.do_handshake() 2024-09-14T03:55:58.523 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:58 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:55:58.523 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:58 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: self._sslobj.do_handshake() 2024-09-14T03:55:58.523 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:58 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:55:58.523 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:55:58 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: 2024-09-14T03:55:58.764 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:55:58.494+0000 7fc75ed73040 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-09-14T03:55:58.764 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:55:58.615+0000 7fc75ed73040 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-09-14T03:55:59.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:58 smithi129 ceph-mon[33173]: Deploying cephadm binary to smithi129 2024-09-14T03:55:59.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:58 smithi129 ceph-mon[33173]: pgmap v5: 137 pgs: 137 active+clean; 449 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:55:59.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:58 smithi129 ceph-mon[33173]: [14/Sep/2024:03:55:57] ENGINE Bus STARTING 2024-09-14T03:55:59.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:58 smithi039 ceph-mon[29586]: Deploying cephadm binary to smithi129 2024-09-14T03:55:59.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:58 smithi039 ceph-mon[29586]: pgmap v5: 137 pgs: 137 active+clean; 449 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:55:59.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:58 smithi039 ceph-mon[29586]: [14/Sep/2024:03:55:57] ENGINE Bus STARTING 2024-09-14T03:55:59.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:58 smithi039 ceph-mon[40962]: Deploying cephadm binary to smithi129 2024-09-14T03:55:59.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:58 smithi039 ceph-mon[40962]: pgmap v5: 137 pgs: 137 active+clean; 449 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:55:59.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:58 smithi039 ceph-mon[40962]: [14/Sep/2024:03:55:57] ENGINE Bus STARTING 2024-09-14T03:55:59.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:55:58.775+0000 7fc75ed73040 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-09-14T03:55:59.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:55:58.842+0000 7fc75ed73040 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-09-14T03:55:59.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:59 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:55:58.999+0000 7fc75ed73040 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-09-14T03:55:59.384 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:59 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:55:59.072+0000 7fc75ed73040 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-09-14T03:55:59.384 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:59 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:55:59.141+0000 7fc75ed73040 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-09-14T03:55:59.384 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:59 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:55:59.313+0000 7fc75ed73040 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-09-14T03:55:59.384 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:59 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:55:59.382+0000 7fc75ed73040 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-09-14T03:55:59.674 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:59 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:55:59.453+0000 7fc75ed73040 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-09-14T03:55:59.675 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:59 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:55:59.543+0000 7fc75ed73040 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-09-14T03:55:59.675 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:59 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:55:59.611+0000 7fc75ed73040 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-09-14T03:56:00.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:55:59 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:55:59.673+0000 7fc75ed73040 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-09-14T03:56:00.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:59 smithi039 ceph-mon[29586]: [14/Sep/2024:03:55:57] ENGINE Serving on http://172.21.15.129:8765 2024-09-14T03:56:00.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:59 smithi039 ceph-mon[29586]: Deploying cephadm binary to smithi039 2024-09-14T03:56:00.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:59 smithi039 ceph-mon[29586]: [14/Sep/2024:03:55:58] ENGINE Serving on https://172.21.15.129:7150 2024-09-14T03:56:00.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:59 smithi039 ceph-mon[29586]: [14/Sep/2024:03:55:58] ENGINE Bus STARTED 2024-09-14T03:56:00.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:59 smithi039 ceph-mon[29586]: [14/Sep/2024:03:55:58] ENGINE Error in HTTPServer.serve 2024-09-14T03:56:00.041 INFO:journalctl@ceph.mon.a.smithi039.stdout: Traceback (most recent call last): 2024-09-14T03:56:00.041 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:56:00.041 INFO:journalctl@ceph.mon.a.smithi039.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:56:00.041 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:56:00.041 INFO:journalctl@ceph.mon.a.smithi039.stdout: self._run(expiration_interval) 2024-09-14T03:56:00.041 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:56:00.041 INFO:journalctl@ceph.mon.a.smithi039.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:56:00.041 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:56:00.042 INFO:journalctl@ceph.mon.a.smithi039.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:56:00.042 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:56:00.042 INFO:journalctl@ceph.mon.a.smithi039.stdout: s = self.context.wrap_socket( 2024-09-14T03:56:00.042 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:56:00.042 INFO:journalctl@ceph.mon.a.smithi039.stdout: return self.sslsocket_class._create( 2024-09-14T03:56:00.042 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:56:00.042 INFO:journalctl@ceph.mon.a.smithi039.stdout: self.do_handshake() 2024-09-14T03:56:00.042 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:56:00.042 INFO:journalctl@ceph.mon.a.smithi039.stdout: self._sslobj.do_handshake() 2024-09-14T03:56:00.042 INFO:journalctl@ceph.mon.a.smithi039.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:56:00.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:59 smithi039 ceph-mon[29586]: mgrmap e25: x(active, since 3s) 2024-09-14T03:56:00.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:55:59 smithi039 ceph-mon[29586]: osdmap e165: 8 total, 8 up, 8 in 2024-09-14T03:56:00.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:59 smithi039 ceph-mon[40962]: [14/Sep/2024:03:55:57] ENGINE Serving on http://172.21.15.129:8765 2024-09-14T03:56:00.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:59 smithi039 ceph-mon[40962]: Deploying cephadm binary to smithi039 2024-09-14T03:56:00.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:59 smithi039 ceph-mon[40962]: [14/Sep/2024:03:55:58] ENGINE Serving on https://172.21.15.129:7150 2024-09-14T03:56:00.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:59 smithi039 ceph-mon[40962]: [14/Sep/2024:03:55:58] ENGINE Bus STARTED 2024-09-14T03:56:00.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:59 smithi039 ceph-mon[40962]: [14/Sep/2024:03:55:58] ENGINE Error in HTTPServer.serve 2024-09-14T03:56:00.043 INFO:journalctl@ceph.mon.c.smithi039.stdout: Traceback (most recent call last): 2024-09-14T03:56:00.043 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:56:00.043 INFO:journalctl@ceph.mon.c.smithi039.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:56:00.043 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:56:00.043 INFO:journalctl@ceph.mon.c.smithi039.stdout: self._run(expiration_interval) 2024-09-14T03:56:00.043 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:56:00.043 INFO:journalctl@ceph.mon.c.smithi039.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:56:00.043 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:56:00.044 INFO:journalctl@ceph.mon.c.smithi039.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:56:00.044 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:56:00.044 INFO:journalctl@ceph.mon.c.smithi039.stdout: s = self.context.wrap_socket( 2024-09-14T03:56:00.044 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:56:00.044 INFO:journalctl@ceph.mon.c.smithi039.stdout: return self.sslsocket_class._create( 2024-09-14T03:56:00.044 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:56:00.044 INFO:journalctl@ceph.mon.c.smithi039.stdout: self.do_handshake() 2024-09-14T03:56:00.044 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:56:00.044 INFO:journalctl@ceph.mon.c.smithi039.stdout: self._sslobj.do_handshake() 2024-09-14T03:56:00.044 INFO:journalctl@ceph.mon.c.smithi039.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:56:00.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:59 smithi039 ceph-mon[40962]: mgrmap e25: x(active, since 3s) 2024-09-14T03:56:00.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:55:59 smithi039 ceph-mon[40962]: osdmap e165: 8 total, 8 up, 8 in 2024-09-14T03:56:00.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:59 smithi129 ceph-mon[33173]: [14/Sep/2024:03:55:57] ENGINE Serving on http://172.21.15.129:8765 2024-09-14T03:56:00.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:59 smithi129 ceph-mon[33173]: Deploying cephadm binary to smithi039 2024-09-14T03:56:00.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:59 smithi129 ceph-mon[33173]: [14/Sep/2024:03:55:58] ENGINE Serving on https://172.21.15.129:7150 2024-09-14T03:56:00.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:59 smithi129 ceph-mon[33173]: [14/Sep/2024:03:55:58] ENGINE Bus STARTED 2024-09-14T03:56:00.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:59 smithi129 ceph-mon[33173]: [14/Sep/2024:03:55:58] ENGINE Error in HTTPServer.serve 2024-09-14T03:56:00.271 INFO:journalctl@ceph.mon.b.smithi129.stdout: Traceback (most recent call last): 2024-09-14T03:56:00.272 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:56:00.272 INFO:journalctl@ceph.mon.b.smithi129.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:56:00.272 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:56:00.272 INFO:journalctl@ceph.mon.b.smithi129.stdout: self._run(expiration_interval) 2024-09-14T03:56:00.272 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:56:00.272 INFO:journalctl@ceph.mon.b.smithi129.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:56:00.272 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:56:00.272 INFO:journalctl@ceph.mon.b.smithi129.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:56:00.272 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:56:00.272 INFO:journalctl@ceph.mon.b.smithi129.stdout: s = self.context.wrap_socket( 2024-09-14T03:56:00.272 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:56:00.272 INFO:journalctl@ceph.mon.b.smithi129.stdout: return self.sslsocket_class._create( 2024-09-14T03:56:00.272 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:56:00.272 INFO:journalctl@ceph.mon.b.smithi129.stdout: self.do_handshake() 2024-09-14T03:56:00.272 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:56:00.273 INFO:journalctl@ceph.mon.b.smithi129.stdout: self._sslobj.do_handshake() 2024-09-14T03:56:00.273 INFO:journalctl@ceph.mon.b.smithi129.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:56:00.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:59 smithi129 ceph-mon[33173]: mgrmap e25: x(active, since 3s) 2024-09-14T03:56:00.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:55:59 smithi129 ceph-mon[33173]: osdmap e165: 8 total, 8 up, 8 in 2024-09-14T03:56:01.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:00 smithi039 ceph-mon[40962]: pgmap v8: 137 pgs: 137 active+clean; 449 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:56:01.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:00 smithi039 ceph-mon[40962]: osdmap e166: 8 total, 8 up, 8 in 2024-09-14T03:56:01.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:00 smithi039 ceph-mon[40962]: Standby manager daemon y started 2024-09-14T03:56:01.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:00 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.39:0/2566437705' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-14T03:56:01.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:00 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.39:0/2566437705' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:56:01.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:00 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.39:0/2566437705' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-14T03:56:01.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:00 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.39:0/2566437705' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:56:01.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:00 smithi039 ceph-mon[29586]: pgmap v8: 137 pgs: 137 active+clean; 449 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:56:01.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:00 smithi039 ceph-mon[29586]: osdmap e166: 8 total, 8 up, 8 in 2024-09-14T03:56:01.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:00 smithi039 ceph-mon[29586]: Standby manager daemon y started 2024-09-14T03:56:01.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:00 smithi039 ceph-mon[29586]: from='mgr.? 172.21.15.39:0/2566437705' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-14T03:56:01.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:00 smithi039 ceph-mon[29586]: from='mgr.? 172.21.15.39:0/2566437705' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:56:01.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:00 smithi039 ceph-mon[29586]: from='mgr.? 172.21.15.39:0/2566437705' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-14T03:56:01.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:00 smithi039 ceph-mon[29586]: from='mgr.? 172.21.15.39:0/2566437705' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:56:01.042 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:00 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:56:00.573+0000 7fc75ed73040 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-09-14T03:56:01.042 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:00 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:56:00] ENGINE Bus STARTING 2024-09-14T03:56:01.042 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:00 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: CherryPy Checker: 2024-09-14T03:56:01.042 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:00 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: The Application mounted at '' has an empty config. 2024-09-14T03:56:01.042 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:00 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: 2024-09-14T03:56:01.042 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:00 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:56:00] ENGINE Serving on http://:::9283 2024-09-14T03:56:01.042 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:00 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: [14/Sep/2024:03:56:00] ENGINE Bus STARTED 2024-09-14T03:56:01.113 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.SnapsShrinkAfterPrepare (12084 ms) 2024-09-14T03:56:01.113 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.SnapsShrinkToZeroBeforePrepare 2024-09-14T03:56:01.144 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:56:01.156 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 0~8388609 2024-09-14T03:56:01.164 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:00 smithi129 ceph-mon[33173]: pgmap v8: 137 pgs: 137 active+clean; 449 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:56:01.164 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:00 smithi129 ceph-mon[33173]: osdmap e166: 8 total, 8 up, 8 in 2024-09-14T03:56:01.165 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:00 smithi129 ceph-mon[33173]: Standby manager daemon y started 2024-09-14T03:56:01.165 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:00 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.39:0/2566437705' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-14T03:56:01.165 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:00 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.39:0/2566437705' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:56:01.165 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:00 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.39:0/2566437705' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-14T03:56:01.165 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:00 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.39:0/2566437705' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:56:01.231 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap1 2024-09-14T03:56:02.090 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:01 smithi129 ceph-mon[33173]: osdmap e167: 8 total, 8 up, 8 in 2024-09-14T03:56:02.090 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:01 smithi129 ceph-mon[33173]: mgrmap e26: x(active, since 5s), standbys: y 2024-09-14T03:56:02.090 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:01 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:56:02.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:01 smithi039 ceph-mon[29586]: osdmap e167: 8 total, 8 up, 8 in 2024-09-14T03:56:02.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:01 smithi039 ceph-mon[29586]: mgrmap e26: x(active, since 5s), standbys: y 2024-09-14T03:56:02.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:01 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:56:02.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:01 smithi039 ceph-mon[40962]: osdmap e167: 8 total, 8 up, 8 in 2024-09-14T03:56:02.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:01 smithi039 ceph-mon[40962]: mgrmap e26: x(active, since 5s), standbys: y 2024-09-14T03:56:02.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:01 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:56:02.832 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 8388609~8388609 2024-09-14T03:56:02.847 INFO:tasks.workunit.client.0.smithi039.stdout:write: X 0~1000 2024-09-14T03:56:02.848 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 1010~1000 2024-09-14T03:56:02.848 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap2 2024-09-14T03:56:03.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:02 smithi129 ceph-mon[33173]: pgmap v11: 137 pgs: 137 active+clean; 449 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:56:03.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:02 smithi129 ceph-mon[33173]: osdmap e168: 8 total, 8 up, 8 in 2024-09-14T03:56:03.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:02 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:03.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:02 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:03.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:02 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:03.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:02 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:03.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:02 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:03.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:02 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:03.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:02 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:03.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:02 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:03.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:02 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:03.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:02 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:03.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:02 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:03.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:02 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:03.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:02 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:03.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:02 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:03.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:02 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:03.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[29586]: pgmap v11: 137 pgs: 137 active+clean; 449 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:56:03.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[29586]: osdmap e168: 8 total, 8 up, 8 in 2024-09-14T03:56:03.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:03.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:03.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:03.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:03.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:03.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:03.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:03.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:03.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:03.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:03.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:03.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:03.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:03.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:03.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:03.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[40962]: pgmap v11: 137 pgs: 137 active+clean; 449 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:56:03.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[40962]: osdmap e168: 8 total, 8 up, 8 in 2024-09-14T03:56:03.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:03.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:03.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:03.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:03.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:03.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:03.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:03.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:03.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:03.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:03.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:03.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:03.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:03.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:03.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:02 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:03.767 INFO:teuthology.orchestra.run.smithi039.stdout:true 2024-09-14T03:56:04.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:03 smithi039 ceph-mon[29586]: Adjusting osd_memory_target on smithi039 to 2480M 2024-09-14T03:56:04.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:03 smithi039 ceph-mon[29586]: osdmap e169: 8 total, 8 up, 8 in 2024-09-14T03:56:04.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:03 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:04.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:03 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:04.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:03 smithi039 ceph-mon[40962]: Adjusting osd_memory_target on smithi039 to 2480M 2024-09-14T03:56:04.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:03 smithi039 ceph-mon[40962]: osdmap e169: 8 total, 8 up, 8 in 2024-09-14T03:56:04.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:03 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:04.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:03 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:04.256 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:03 smithi129 ceph-mon[33173]: Adjusting osd_memory_target on smithi039 to 2480M 2024-09-14T03:56:04.256 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:03 smithi129 ceph-mon[33173]: osdmap e169: 8 total, 8 up, 8 in 2024-09-14T03:56:04.256 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:03 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:04.256 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:03 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:04.355 INFO:teuthology.orchestra.run.smithi039.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:56:04.355 INFO:teuthology.orchestra.run.smithi039.stdout:alertmanager.a smithi039 *:9093,9094 running (3m) 2s ago 4m 28.6M - 0.25.0 c8568f914cd2 b5ca6b93d9ee 2024-09-14T03:56:04.355 INFO:teuthology.orchestra.run.smithi039.stdout:grafana.a smithi129 *:3000 running (3m) 1s ago 3m 89.0M - 9.4.7 954c08fa6188 71f297945590 2024-09-14T03:56:04.355 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi039.bbwiwf smithi039 running (2m) 2s ago 2m 18.2M - 18.2.4-845-ge4e227b3 67ddd33fa72f 91e6c8f85681 2024-09-14T03:56:04.355 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi129.ybjqcw smithi129 running (2m) 1s ago 2m 14.1M - 18.2.4-845-ge4e227b3 67ddd33fa72f a3fab5b37b71 2024-09-14T03:56:04.355 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.x smithi129 *:8443,9283,8765 running (27s) 1s ago 8m 497M - 19.3.0-4907-ga2782315 8801d2529e43 759df831a5de 2024-09-14T03:56:04.355 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.y smithi039 *:9283,8765 running (9m) 2s ago 9m 329M - 18.2.4-845-ge4e227b3 67ddd33fa72f babf5c5c054b 2024-09-14T03:56:04.355 INFO:teuthology.orchestra.run.smithi039.stdout:mon.a smithi039 running (9m) 2s ago 9m 57.4M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f e7b5f91fb09c 2024-09-14T03:56:04.355 INFO:teuthology.orchestra.run.smithi039.stdout:mon.b smithi129 running (8m) 1s ago 8m 45.9M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f eb38cc5cbe35 2024-09-14T03:56:04.355 INFO:teuthology.orchestra.run.smithi039.stdout:mon.c smithi039 running (8m) 2s ago 8m 46.7M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 31d440e49f93 2024-09-14T03:56:04.356 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.a smithi039 *:9100 running (3m) 2s ago 3m 13.7M - 1.5.0 0da6a335fe13 4eff582f78f5 2024-09-14T03:56:04.356 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.b smithi129 *:9100 running (3m) 1s ago 3m 16.3M - 1.5.0 0da6a335fe13 c7581baf44f2 2024-09-14T03:56:04.356 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0 smithi039 running (7m) 2s ago 7m 120M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 5048d74f27ac 2024-09-14T03:56:04.356 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1 smithi039 running (7m) 2s ago 7m 106M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 91349527d88b 2024-09-14T03:56:04.356 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2 smithi039 running (6m) 2s ago 6m 89.0M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f eb903b44fc5d 2024-09-14T03:56:04.356 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3 smithi039 running (6m) 2s ago 6m 91.1M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f dff9e665bd9b 2024-09-14T03:56:04.356 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4 smithi129 running (6m) 1s ago 6m 121M 4016M 18.2.4-845-ge4e227b3 67ddd33fa72f 5b64727311b7 2024-09-14T03:56:04.356 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5 smithi129 running (5m) 1s ago 5m 101M 4016M 18.2.4-845-ge4e227b3 67ddd33fa72f f2d991a754f7 2024-09-14T03:56:04.356 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6 smithi129 running (5m) 1s ago 5m 122M 4016M 18.2.4-845-ge4e227b3 67ddd33fa72f 3060eaf44106 2024-09-14T03:56:04.356 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7 smithi129 running (4m) 1s ago 4m 101M 4016M 18.2.4-845-ge4e227b3 67ddd33fa72f 4c13b97060d1 2024-09-14T03:56:04.356 INFO:teuthology.orchestra.run.smithi039.stdout:prometheus.a smithi129 *:9095 running (2m) 1s ago 4m 48.3M - 2.43.0 a07b618ecd1d 0275bbd07991 2024-09-14T03:56:04.704 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T03:56:04.704 INFO:teuthology.orchestra.run.smithi039.stdout: "mon": { 2024-09-14T03:56:04.704 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 3 2024-09-14T03:56:04.705 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:56:04.705 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr": { 2024-09-14T03:56:04.705 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 1, 2024-09-14T03:56:04.705 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 1 2024-09-14T03:56:04.705 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:56:04.705 INFO:teuthology.orchestra.run.smithi039.stdout: "osd": { 2024-09-14T03:56:04.705 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8 2024-09-14T03:56:04.705 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:56:04.705 INFO:teuthology.orchestra.run.smithi039.stdout: "mds": { 2024-09-14T03:56:04.705 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:56:04.705 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:56:04.705 INFO:teuthology.orchestra.run.smithi039.stdout: "overall": { 2024-09-14T03:56:04.705 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 14, 2024-09-14T03:56:04.706 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 1 2024-09-14T03:56:04.706 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T03:56:04.706 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T03:56:04.877 INFO:tasks.workunit.client.0.smithi039.stdout:write: X 8388609~1000 2024-09-14T03:56:04.885 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 16777228~1000 2024-09-14T03:56:04.885 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 167772160 2024-09-14T03:56:04.893 INFO:tasks.workunit.client.0.smithi039.stdout:write: 2 83886079~8388609 2024-09-14T03:56:04.908 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap3 2024-09-14T03:56:04.950 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:04 smithi129 ceph-mon[33173]: pgmap v14: 137 pgs: 137 active+clean; 465 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 70 MiB/s rd, 10 MiB/s wr, 540 op/s 2024-09-14T03:56:04.951 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:04 smithi129 ceph-mon[33173]: from='client.24914 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:04.951 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:04 smithi129 ceph-mon[33173]: osdmap e170: 8 total, 8 up, 8 in 2024-09-14T03:56:04.951 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:04 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/1877448560' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:04.968 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:04 smithi039 ceph-mon[29586]: pgmap v14: 137 pgs: 137 active+clean; 465 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 70 MiB/s rd, 10 MiB/s wr, 540 op/s 2024-09-14T03:56:04.968 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:04 smithi039 ceph-mon[29586]: from='client.24914 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:04.968 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:04 smithi039 ceph-mon[29586]: osdmap e170: 8 total, 8 up, 8 in 2024-09-14T03:56:04.968 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:04 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/1877448560' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:04.968 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:04 smithi039 ceph-mon[40962]: pgmap v14: 137 pgs: 137 active+clean; 465 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 70 MiB/s rd, 10 MiB/s wr, 540 op/s 2024-09-14T03:56:04.969 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:04 smithi039 ceph-mon[40962]: from='client.24914 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:04.969 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:04 smithi039 ceph-mon[40962]: osdmap e170: 8 total, 8 up, 8 in 2024-09-14T03:56:04.969 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:04 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/1877448560' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:05.016 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T03:56:05.016 INFO:teuthology.orchestra.run.smithi039.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:56:05.016 INFO:teuthology.orchestra.run.smithi039.stdout: "in_progress": true, 2024-09-14T03:56:05.016 INFO:teuthology.orchestra.run.smithi039.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-14T03:56:05.017 INFO:teuthology.orchestra.run.smithi039.stdout: "services_complete": [], 2024-09-14T03:56:05.017 INFO:teuthology.orchestra.run.smithi039.stdout: "progress": "1/2 daemons upgraded", 2024-09-14T03:56:05.017 INFO:teuthology.orchestra.run.smithi039.stdout: "message": "", 2024-09-14T03:56:05.017 INFO:teuthology.orchestra.run.smithi039.stdout: "is_paused": false 2024-09-14T03:56:05.017 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T03:56:06.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:05 smithi129 ceph-mon[33173]: from='client.15018 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:06.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:05 smithi129 ceph-mon[33173]: from='client.15024 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:06.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:05 smithi129 ceph-mon[33173]: osdmap e171: 8 total, 8 up, 8 in 2024-09-14T03:56:06.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:05 smithi039 ceph-mon[29586]: from='client.15018 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:06.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:05 smithi039 ceph-mon[29586]: from='client.15024 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:06.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:05 smithi039 ceph-mon[29586]: osdmap e171: 8 total, 8 up, 8 in 2024-09-14T03:56:06.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:05 smithi039 ceph-mon[40962]: from='client.15018 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:06.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:05 smithi039 ceph-mon[40962]: from='client.15024 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:06.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:05 smithi039 ceph-mon[40962]: osdmap e171: 8 total, 8 up, 8 in 2024-09-14T03:56:06.900 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:56:06.919 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 83886079~1 2024-09-14T03:56:06.928 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 0 2024-09-14T03:56:06.944 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:56:07.021 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:56:06.762Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.39:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.39:8765: connect: connection refused" 2024-09-14T03:56:07.021 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:56:06.762Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.39:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.39:8765: connect: connection refused" 2024-09-14T03:56:07.021 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:56:06.764Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.39:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.39:8765: connect: connection refused" 2024-09-14T03:56:07.021 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:56:06.766Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.39:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.39:8765: connect: connection refused" 2024-09-14T03:56:07.021 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:56:06.766Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.39:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.39:8765: connect: connection refused" 2024-09-14T03:56:07.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:06 smithi129 ceph-mon[33173]: from='client.15036 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:07.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:06 smithi129 ceph-mon[33173]: pgmap v17: 137 pgs: 137 active+clean; 465 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 70 MiB/s rd, 10 MiB/s wr, 540 op/s 2024-09-14T03:56:07.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:06 smithi129 ceph-mon[33173]: osdmap e172: 8 total, 8 up, 8 in 2024-09-14T03:56:07.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:06 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:07.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:06 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:07.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:06 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:07.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:06 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:07.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:06 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:07.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:06 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:07.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:06 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:07.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:06 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:07.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:06 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:07.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:06 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:07.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:06 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:07.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:06 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:56:07.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:06 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:56:07.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[40962]: from='client.15036 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:07.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[40962]: pgmap v17: 137 pgs: 137 active+clean; 465 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 70 MiB/s rd, 10 MiB/s wr, 540 op/s 2024-09-14T03:56:07.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[40962]: osdmap e172: 8 total, 8 up, 8 in 2024-09-14T03:56:07.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:07.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:07.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:07.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:07.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:07.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:07.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:07.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:07.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:07.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:07.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:07.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:56:07.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:56:07.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[29586]: from='client.15036 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:07.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[29586]: pgmap v17: 137 pgs: 137 active+clean; 465 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 70 MiB/s rd, 10 MiB/s wr, 540 op/s 2024-09-14T03:56:07.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[29586]: osdmap e172: 8 total, 8 up, 8 in 2024-09-14T03:56:07.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:07.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:07.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:07.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:07.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:07.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:07.293 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:07.293 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:07.293 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:07.293 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-09-14T03:56:07.293 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:07.293 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:56:07.293 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:06 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:56:07.982 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:07 smithi129 ceph-mon[33173]: Adjusting osd_memory_target on smithi129 to 2480M 2024-09-14T03:56:07.982 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:07 smithi129 ceph-mon[33173]: Updating smithi039:/etc/ceph/ceph.conf 2024-09-14T03:56:07.982 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:07 smithi129 ceph-mon[33173]: Updating smithi129:/etc/ceph/ceph.conf 2024-09-14T03:56:07.982 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:07 smithi129 ceph-mon[33173]: Updating smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:56:07.982 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:07 smithi129 ceph-mon[33173]: Updating smithi039:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:56:07.982 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:07 smithi129 ceph-mon[33173]: Updating smithi129:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:56:07.982 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:07 smithi129 ceph-mon[33173]: Updating smithi039:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:56:07.982 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:07 smithi129 ceph-mon[33173]: osdmap e173: 8 total, 8 up, 8 in 2024-09-14T03:56:07.982 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:07 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:07.982 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:07 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:07.982 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:07 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:07.982 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:07 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:07.983 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:07 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:08.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:07 smithi039 ceph-mon[29586]: Adjusting osd_memory_target on smithi129 to 2480M 2024-09-14T03:56:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:07 smithi039 ceph-mon[29586]: Updating smithi039:/etc/ceph/ceph.conf 2024-09-14T03:56:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:07 smithi039 ceph-mon[29586]: Updating smithi129:/etc/ceph/ceph.conf 2024-09-14T03:56:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:07 smithi039 ceph-mon[29586]: Updating smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:56:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:07 smithi039 ceph-mon[29586]: Updating smithi039:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:56:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:07 smithi039 ceph-mon[29586]: Updating smithi129:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:56:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:07 smithi039 ceph-mon[29586]: Updating smithi039:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:56:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:07 smithi039 ceph-mon[29586]: osdmap e173: 8 total, 8 up, 8 in 2024-09-14T03:56:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:07 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:07 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:07 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:07 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:07 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:08.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:07 smithi039 ceph-mon[40962]: Adjusting osd_memory_target on smithi129 to 2480M 2024-09-14T03:56:08.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:07 smithi039 ceph-mon[40962]: Updating smithi039:/etc/ceph/ceph.conf 2024-09-14T03:56:08.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:07 smithi039 ceph-mon[40962]: Updating smithi129:/etc/ceph/ceph.conf 2024-09-14T03:56:08.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:07 smithi039 ceph-mon[40962]: Updating smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:56:08.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:07 smithi039 ceph-mon[40962]: Updating smithi039:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:56:08.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:07 smithi039 ceph-mon[40962]: Updating smithi129:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:56:08.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:07 smithi039 ceph-mon[40962]: Updating smithi039:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:56:08.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:07 smithi039 ceph-mon[40962]: osdmap e173: 8 total, 8 up, 8 in 2024-09-14T03:56:08.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:07 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:08.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:07 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:08.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:07 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:08.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:07 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:08.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:07 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:09.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:08 smithi129 ceph-mon[33173]: Updating smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:56:09.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:08 smithi129 ceph-mon[33173]: Updating smithi039:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:56:09.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:08 smithi129 ceph-mon[33173]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:56:09.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:08 smithi129 ceph-mon[33173]: pgmap v20: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 70 KiB/s rd, 8.0 MiB/s wr, 103 op/s 2024-09-14T03:56:09.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:08 smithi129 ceph-mon[33173]: Reconfiguring daemon prometheus.a on smithi129 2024-09-14T03:56:09.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:08 smithi129 ceph-mon[33173]: osdmap e174: 8 total, 8 up, 8 in 2024-09-14T03:56:09.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:08 smithi129 ceph-mon[33173]: osdmap e175: 8 total, 8 up, 8 in 2024-09-14T03:56:09.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:09 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[29835]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:56:09] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-09-14T03:56:09.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:08 smithi039 ceph-mon[29586]: Updating smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:56:09.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:08 smithi039 ceph-mon[29586]: Updating smithi039:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:56:09.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:08 smithi039 ceph-mon[29586]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:56:09.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:08 smithi039 ceph-mon[29586]: pgmap v20: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 70 KiB/s rd, 8.0 MiB/s wr, 103 op/s 2024-09-14T03:56:09.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:08 smithi039 ceph-mon[29586]: Reconfiguring daemon prometheus.a on smithi129 2024-09-14T03:56:09.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:08 smithi039 ceph-mon[29586]: osdmap e174: 8 total, 8 up, 8 in 2024-09-14T03:56:09.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:08 smithi039 ceph-mon[29586]: osdmap e175: 8 total, 8 up, 8 in 2024-09-14T03:56:09.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:08 smithi039 ceph-mon[40962]: Updating smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:56:09.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:08 smithi039 ceph-mon[40962]: Updating smithi039:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:56:09.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:08 smithi039 ceph-mon[40962]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:56:09.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:08 smithi039 ceph-mon[40962]: pgmap v20: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 70 KiB/s rd, 8.0 MiB/s wr, 103 op/s 2024-09-14T03:56:09.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:08 smithi039 ceph-mon[40962]: Reconfiguring daemon prometheus.a on smithi129 2024-09-14T03:56:09.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:08 smithi039 ceph-mon[40962]: osdmap e174: 8 total, 8 up, 8 in 2024-09-14T03:56:09.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:08 smithi039 ceph-mon[40962]: osdmap e175: 8 total, 8 up, 8 in 2024-09-14T03:56:10.065 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:56:10.213 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:56:10.916 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:10 smithi129 systemd[1]: Stopping Ceph prometheus.a for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:56:10.916 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:10 smithi129 ceph-mon[33173]: pgmap v23: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 70 KiB/s rd, 8.0 MiB/s wr, 103 op/s 2024-09-14T03:56:10.917 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:10 smithi129 ceph-mon[33173]: osdmap e176: 8 total, 8 up, 8 in 2024-09-14T03:56:11.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:10 smithi039 ceph-mon[29586]: pgmap v23: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 70 KiB/s rd, 8.0 MiB/s wr, 103 op/s 2024-09-14T03:56:11.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:10 smithi039 ceph-mon[29586]: osdmap e176: 8 total, 8 up, 8 in 2024-09-14T03:56:11.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:10 smithi039 ceph-mon[40962]: pgmap v23: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 70 KiB/s rd, 8.0 MiB/s wr, 103 op/s 2024-09-14T03:56:11.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:10 smithi039 ceph-mon[40962]: osdmap e176: 8 total, 8 up, 8 in 2024-09-14T03:56:11.181 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:10 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:56:10.915Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-14T03:56:11.181 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:10 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:56:10.915Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-14T03:56:11.181 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:10 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:56:10.915Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-14T03:56:11.181 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:10 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:56:10.915Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-14T03:56:11.181 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:10 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:56:10.915Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-14T03:56:11.181 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:10 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:56:10.915Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-14T03:56:11.181 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:10 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:56:10.915Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-14T03:56:11.181 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:10 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:56:10.915Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-14T03:56:11.182 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:10 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:56:10.915Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-14T03:56:11.182 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:10 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:56:10.915Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-14T03:56:11.182 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:10 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:56:10.916Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-14T03:56:11.182 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:10 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[66005]: ts=2024-09-14T03:56:10.916Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-14T03:56:11.182 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:10 smithi129 podman[77643]: 2024-09-14 03:56:10.925641481 +0000 UTC m=+0.134558980 container died 0275bbd079914b91fa62c7095ebe6fb834c2bb48b8d99644a98e7ffb38a76fed (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:56:11.521 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:11 smithi129 podman[77643]: 2024-09-14 03:56:11.396170845 +0000 UTC m=+0.605088336 container cleanup 0275bbd079914b91fa62c7095ebe6fb834c2bb48b8d99644a98e7ffb38a76fed (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:56:11.521 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:11 smithi129 bash[77643]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a 2024-09-14T03:56:12.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:11 smithi129 ceph-mon[33173]: osdmap e177: 8 total, 8 up, 8 in 2024-09-14T03:56:12.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:11 smithi129 ceph-mon[33173]: pgmap v26: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:56:12.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:11 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:56:12.022 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:11 smithi129 podman[77660]: 2024-09-14 03:56:11.66442295 +0000 UTC m=+0.741327582 container remove 0275bbd079914b91fa62c7095ebe6fb834c2bb48b8d99644a98e7ffb38a76fed (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:56:12.022 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:11 smithi129 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-14T03:56:12.022 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:11 smithi129 systemd[1]: Stopped Ceph prometheus.a for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:56:12.022 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:11 smithi129 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 1.779s CPU time. 2024-09-14T03:56:12.022 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:11 smithi129 systemd[1]: Starting Ceph prometheus.a for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:56:12.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:11 smithi039 ceph-mon[29586]: osdmap e177: 8 total, 8 up, 8 in 2024-09-14T03:56:12.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:11 smithi039 ceph-mon[29586]: pgmap v26: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:56:12.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:11 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:56:12.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:11 smithi039 ceph-mon[40962]: osdmap e177: 8 total, 8 up, 8 in 2024-09-14T03:56:12.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:11 smithi039 ceph-mon[40962]: pgmap v26: 137 pgs: 137 active+clean; 505 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:56:12.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:11 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:56:12.302 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:12 smithi129 podman[77756]: 2024-09-14 03:56:11.942950603 +0000 UTC m=+0.016948052 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-14T03:56:12.302 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:12 smithi129 podman[77756]: 2024-09-14 03:56:12.074137792 +0000 UTC m=+0.148135234 container create 474a9ac1dffece838ca5f7d37f17342082f7d784d6457611f686c6776ad76326 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:56:12.302 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:12 smithi129 podman[77756]: 2024-09-14 03:56:12.301456963 +0000 UTC m=+0.375454405 container init 474a9ac1dffece838ca5f7d37f17342082f7d784d6457611f686c6776ad76326 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:56:12.772 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:12 smithi129 podman[77756]: 2024-09-14 03:56:12.3061476 +0000 UTC m=+0.380145043 container start 474a9ac1dffece838ca5f7d37f17342082f7d784d6457611f686c6776ad76326 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:56:12.772 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:12.330Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-14T03:56:12.772 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:12.330Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-14T03:56:12.772 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:12.330Z caller=main.go:570 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi129 (none))" 2024-09-14T03:56:12.772 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:12.330Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-14T03:56:12.772 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:12.330Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-14T03:56:12.772 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:12.331Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-14T03:56:12.772 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:12.332Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-14T03:56:12.772 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:12.333Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-14T03:56:12.773 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:12.333Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-14T03:56:12.773 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:12.335Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-14T03:56:12.773 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:12.335Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.439µs 2024-09-14T03:56:12.773 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:12.335Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-14T03:56:12.773 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:12.336Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-09-14T03:56:12.773 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:12.345Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-09-14T03:56:12.773 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:12.345Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-09-14T03:56:12.773 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:12.345Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=30.209µs wal_replay_duration=10.070797ms wbl_replay_duration=391ns total_replay_duration=10.126816ms 2024-09-14T03:56:12.773 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:12.347Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-14T03:56:12.773 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:12.347Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-14T03:56:12.773 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:12.347Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-14T03:56:12.773 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:12.367Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=20.441386ms db_storage=761ns remote_storage=1.799µs web_handler=345ns query_engine=535ns scrape=435.139µs scrape_sd=114.25µs notify=10.929µs notify_sd=12.839µs rules=19.457154ms tracing=5.783µs 2024-09-14T03:56:12.774 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:12.367Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-14T03:56:12.774 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:12 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:12.367Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-14T03:56:12.774 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:12 smithi129 bash[77756]: 474a9ac1dffece838ca5f7d37f17342082f7d784d6457611f686c6776ad76326 2024-09-14T03:56:12.774 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:12 smithi129 systemd[1]: Started Ceph prometheus.a for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:56:13.082 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:12 smithi039 ceph-mon[29586]: osdmap e178: 8 total, 8 up, 8 in 2024-09-14T03:56:13.082 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:12 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:13.083 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:12 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:13.083 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:12 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:56:13.083 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:12 smithi039 ceph-mon[29586]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:56:13.083 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:12 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:56:13.083 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:12 smithi039 ceph-mon[29586]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-14T03:56:13.083 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:12 smithi039 ceph-mon[29586]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-14T03:56:13.083 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:12 smithi039 ceph-mon[40962]: osdmap e178: 8 total, 8 up, 8 in 2024-09-14T03:56:13.083 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:12 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:13.083 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:12 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:13.084 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:12 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:56:13.084 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:12 smithi039 ceph-mon[40962]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:56:13.084 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:12 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:56:13.084 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:12 smithi039 ceph-mon[40962]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-14T03:56:13.084 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:12 smithi039 ceph-mon[40962]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-14T03:56:13.111 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.SnapsShrinkToZeroBeforePrepare (11998 ms) 2024-09-14T03:56:13.111 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.SnapsShrinkToZeroAfterPrepare 2024-09-14T03:56:13.142 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:56:13.155 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 0~8388609 2024-09-14T03:56:13.223 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap1 2024-09-14T03:56:13.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:12 smithi129 ceph-mon[33173]: osdmap e178: 8 total, 8 up, 8 in 2024-09-14T03:56:13.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:12 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:13.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:12 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:13.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:12 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:56:13.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:12 smithi129 ceph-mon[33173]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:56:13.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:12 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:56:13.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:12 smithi129 ceph-mon[33173]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-14T03:56:13.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:12 smithi129 ceph-mon[33173]: Upgrade: Need to upgrade myself (mgr.x) 2024-09-14T03:56:14.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:13 smithi129 ceph-mon[33173]: osdmap e179: 8 total, 8 up, 8 in 2024-09-14T03:56:14.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:13 smithi129 ceph-mon[33173]: pgmap v29: 137 pgs: 137 active+clean; 521 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 510 op/s 2024-09-14T03:56:14.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:13 smithi039 ceph-mon[29586]: osdmap e179: 8 total, 8 up, 8 in 2024-09-14T03:56:14.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:13 smithi039 ceph-mon[29586]: pgmap v29: 137 pgs: 137 active+clean; 521 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 510 op/s 2024-09-14T03:56:14.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:13 smithi039 ceph-mon[40962]: osdmap e179: 8 total, 8 up, 8 in 2024-09-14T03:56:14.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:13 smithi039 ceph-mon[40962]: pgmap v29: 137 pgs: 137 active+clean; 521 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 510 op/s 2024-09-14T03:56:14.929 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 8388609~8388609 2024-09-14T03:56:14.939 INFO:tasks.workunit.client.0.smithi039.stdout:write: X 0~1000 2024-09-14T03:56:14.939 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 1010~1000 2024-09-14T03:56:14.940 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap2 2024-09-14T03:56:15.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:14 smithi039 ceph-mon[29586]: osdmap e180: 8 total, 8 up, 8 in 2024-09-14T03:56:15.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:14 smithi039 ceph-mon[29586]: Upgrade: Updating mgr.y 2024-09-14T03:56:15.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:14 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:15.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:14 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:56:15.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:14 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:56:15.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:14 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:56:15.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:14 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:56:15.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:14 smithi039 ceph-mon[29586]: Deploying daemon mgr.y on smithi039 2024-09-14T03:56:15.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:14 smithi039 ceph-mon[40962]: osdmap e180: 8 total, 8 up, 8 in 2024-09-14T03:56:15.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:14 smithi039 ceph-mon[40962]: Upgrade: Updating mgr.y 2024-09-14T03:56:15.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:14 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:15.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:14 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:56:15.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:14 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:56:15.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:14 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:56:15.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:14 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:56:15.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:14 smithi039 ceph-mon[40962]: Deploying daemon mgr.y on smithi039 2024-09-14T03:56:15.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:14 smithi129 ceph-mon[33173]: osdmap e180: 8 total, 8 up, 8 in 2024-09-14T03:56:15.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:14 smithi129 ceph-mon[33173]: Upgrade: Updating mgr.y 2024-09-14T03:56:15.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:14 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:15.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:14 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:56:15.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:14 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:56:15.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:14 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:56:15.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:14 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:56:15.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:14 smithi129 ceph-mon[33173]: Deploying daemon mgr.y on smithi039 2024-09-14T03:56:16.164 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:15 smithi039 systemd[1]: Stopping Ceph mgr.y for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:56:16.164 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:15 smithi039 ceph-mon[29586]: osdmap e181: 8 total, 8 up, 8 in 2024-09-14T03:56:16.164 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:15 smithi039 ceph-mon[29586]: pgmap v32: 137 pgs: 137 active+clean; 521 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 510 op/s 2024-09-14T03:56:16.164 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:15 smithi039 ceph-mon[40962]: osdmap e181: 8 total, 8 up, 8 in 2024-09-14T03:56:16.165 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:15 smithi039 ceph-mon[40962]: pgmap v32: 137 pgs: 137 active+clean; 521 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 510 op/s 2024-09-14T03:56:16.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:15 smithi129 ceph-mon[33173]: osdmap e181: 8 total, 8 up, 8 in 2024-09-14T03:56:16.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:15 smithi129 ceph-mon[33173]: pgmap v32: 137 pgs: 137 active+clean; 521 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 30 MiB/s rd, 6.0 MiB/s wr, 510 op/s 2024-09-14T03:56:16.540 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:16 smithi039 podman[89292]: 2024-09-14 03:56:16.162650908 +0000 UTC m=+0.201026663 container died babf5c5c054b5d02a74e529601cb488a4f655b9750823175fbf02403fd53b812 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=reef-e4e227b, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-14T03:56:16.953 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:16 smithi039 podman[89292]: 2024-09-14 03:56:16.767334758 +0000 UTC m=+0.805710500 container cleanup babf5c5c054b5d02a74e529601cb488a4f655b9750823175fbf02403fd53b812 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=reef-e4e227b, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-09-14T03:56:16.953 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:16 smithi039 bash[89292]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y 2024-09-14T03:56:16.962 INFO:tasks.workunit.client.0.smithi039.stdout:write: X 8388609~1000 2024-09-14T03:56:16.968 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 16777228~1000 2024-09-14T03:56:16.968 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 167772160 2024-09-14T03:56:16.977 INFO:tasks.workunit.client.0.smithi039.stdout:write: 2 83886079~8388609 2024-09-14T03:56:16.991 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap3 2024-09-14T03:56:17.217 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:16 smithi039 ceph-mon[29586]: osdmap e182: 8 total, 8 up, 8 in 2024-09-14T03:56:17.217 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:16 smithi039 ceph-mon[40962]: osdmap e182: 8 total, 8 up, 8 in 2024-09-14T03:56:17.217 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:17 smithi039 podman[89310]: 2024-09-14 03:56:17.027678932 +0000 UTC m=+0.864007441 container remove babf5c5c054b5d02a74e529601cb488a4f655b9750823175fbf02403fd53b812 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y, ceph=True, org.label-schema.build-date=20240731, RELEASE=reef-e4e227b, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:56:17.217 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:17 smithi039 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-09-14T03:56:17.217 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:17 smithi039 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-09-14T03:56:17.218 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:17 smithi039 systemd[1]: Stopped Ceph mgr.y for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:56:17.218 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:17 smithi039 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mgr.y.service: Consumed 49.260s CPU time. 2024-09-14T03:56:17.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:16 smithi129 ceph-mon[33173]: osdmap e182: 8 total, 8 up, 8 in 2024-09-14T03:56:17.540 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:17 smithi039 systemd[1]: Starting Ceph mgr.y for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:56:17.924 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:17 smithi039 podman[89443]: 2024-09-14 03:56:17.523964231 +0000 UTC m=+0.016293325 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:56:17.924 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:17 smithi039 podman[89443]: 2024-09-14 03:56:17.662388658 +0000 UTC m=+0.154717738 container create cccc87f635a64eb07c0504f6dd1c027e68a7a6f01229a42ba18179017ca16902 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux ) 2024-09-14T03:56:18.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:17 smithi129 ceph-mon[33173]: osdmap e183: 8 total, 8 up, 8 in 2024-09-14T03:56:18.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:17 smithi129 ceph-mon[33173]: pgmap v35: 137 pgs: 137 active+clean; 537 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 8.0 MiB/s wr, 125 op/s 2024-09-14T03:56:18.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:17 smithi039 ceph-mon[29586]: osdmap e183: 8 total, 8 up, 8 in 2024-09-14T03:56:18.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:17 smithi039 ceph-mon[29586]: pgmap v35: 137 pgs: 137 active+clean; 537 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 8.0 MiB/s wr, 125 op/s 2024-09-14T03:56:18.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:17 smithi039 ceph-mon[40962]: osdmap e183: 8 total, 8 up, 8 in 2024-09-14T03:56:18.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:17 smithi039 ceph-mon[40962]: pgmap v35: 137 pgs: 137 active+clean; 537 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 8.0 MiB/s wr, 125 op/s 2024-09-14T03:56:18.291 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:17 smithi039 podman[89443]: 2024-09-14 03:56:17.923587324 +0000 UTC m=+0.415916407 container init cccc87f635a64eb07c0504f6dd1c027e68a7a6f01229a42ba18179017ca16902 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-09-14T03:56:18.291 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:17 smithi039 podman[89443]: 2024-09-14 03:56:17.928919974 +0000 UTC m=+0.421249051 container start cccc87f635a64eb07c0504f6dd1c027e68a7a6f01229a42ba18179017ca16902 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y, io.buildah.version=1.37.2, RELEASE=main-a278231, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T03:56:18.291 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:17 smithi039 ceph-mgr[89481]: -- 172.21.15.39:0/1545453063 <== mon.0 v2:172.21.15.39:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5629499730e0 con 0x562949951400 2024-09-14T03:56:18.291 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:18 smithi039 bash[89443]: cccc87f635a64eb07c0504f6dd1c027e68a7a6f01229a42ba18179017ca16902 2024-09-14T03:56:18.291 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:18 smithi039 systemd[1]: Started Ceph mgr.y for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:56:18.732 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:56:18.759 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 83886079~1 2024-09-14T03:56:18.766 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:56:19.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:18 smithi129 ceph-mon[33173]: osdmap e184: 8 total, 8 up, 8 in 2024-09-14T03:56:19.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:18 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:19.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:18 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:19.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:18 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:56:19.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:18 smithi129 ceph-mon[33173]: osdmap e185: 8 total, 8 up, 8 in 2024-09-14T03:56:19.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:18 smithi039 ceph-mon[40962]: osdmap e184: 8 total, 8 up, 8 in 2024-09-14T03:56:19.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:18 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:19.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:18 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:19.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:18 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:56:19.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:18 smithi039 ceph-mon[40962]: osdmap e185: 8 total, 8 up, 8 in 2024-09-14T03:56:19.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:18 smithi039 ceph-mon[29586]: osdmap e184: 8 total, 8 up, 8 in 2024-09-14T03:56:19.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:18 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:19.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:18 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:19.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:18 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:56:19.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:18 smithi039 ceph-mon[29586]: osdmap e185: 8 total, 8 up, 8 in 2024-09-14T03:56:20.516 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:20 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:56:20] "GET /metrics HTTP/1.1" 200 38863 "" "Prometheus/2.43.0" 2024-09-14T03:56:20.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:20 smithi129 ceph-mon[33173]: pgmap v38: 137 pgs: 137 active+clean; 537 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 8.0 MiB/s wr, 125 op/s 2024-09-14T03:56:20.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:20 smithi129 ceph-mon[33173]: osdmap e186: 8 total, 8 up, 8 in 2024-09-14T03:56:21.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:20 smithi039 ceph-mon[29586]: pgmap v38: 137 pgs: 137 active+clean; 537 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 8.0 MiB/s wr, 125 op/s 2024-09-14T03:56:21.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:20 smithi039 ceph-mon[29586]: osdmap e186: 8 total, 8 up, 8 in 2024-09-14T03:56:21.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:20 smithi039 ceph-mon[40962]: pgmap v38: 137 pgs: 137 active+clean; 537 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 24 MiB/s rd, 8.0 MiB/s wr, 125 op/s 2024-09-14T03:56:21.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:20 smithi039 ceph-mon[40962]: osdmap e186: 8 total, 8 up, 8 in 2024-09-14T03:56:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:21 smithi129 ceph-mon[33173]: osdmap e187: 8 total, 8 up, 8 in 2024-09-14T03:56:22.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:21 smithi039 ceph-mon[29586]: osdmap e187: 8 total, 8 up, 8 in 2024-09-14T03:56:22.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:21 smithi039 ceph-mon[40962]: osdmap e187: 8 total, 8 up, 8 in 2024-09-14T03:56:22.187 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 0 2024-09-14T03:56:22.280 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:56:22.412 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:56:23.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:22 smithi129 ceph-mon[33173]: pgmap v41: 137 pgs: 137 active+clean; 537 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:56:23.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:22 smithi129 ceph-mon[33173]: osdmap e188: 8 total, 8 up, 8 in 2024-09-14T03:56:23.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:22 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:23.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:22 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:23.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:22 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:23.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:22 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:23.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:22 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:23.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:22 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:23.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:22 smithi039 ceph-mon[29586]: pgmap v41: 137 pgs: 137 active+clean; 537 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:56:23.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:22 smithi039 ceph-mon[29586]: osdmap e188: 8 total, 8 up, 8 in 2024-09-14T03:56:23.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:22 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:23.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:22 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:23.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:22 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:23.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:22 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:23.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:22 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:23.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:22 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:23.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:22 smithi039 ceph-mon[40962]: pgmap v41: 137 pgs: 137 active+clean; 537 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:56:23.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:22 smithi039 ceph-mon[40962]: osdmap e188: 8 total, 8 up, 8 in 2024-09-14T03:56:23.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:22 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:23.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:22 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:23.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:22 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:23.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:22 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:23.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:22 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:23.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:22 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:23.667 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:23 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:03:56:23] ENGINE Bus STARTING 2024-09-14T03:56:23.667 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:23 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: CherryPy Checker: 2024-09-14T03:56:23.668 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:23 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: The Application mounted at '' has an empty config. 2024-09-14T03:56:23.668 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:23 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T03:56:23.668 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:23 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:03:56:23] ENGINE Serving on http://:::9283 2024-09-14T03:56:23.668 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:23 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:03:56:23] ENGINE Bus STARTED 2024-09-14T03:56:24.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:24 smithi039 ceph-mon[29586]: pgmap v43: 137 pgs: 137 active+clean; 561 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 104 KiB/s rd, 3.5 MiB/s wr, 147 op/s 2024-09-14T03:56:24.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:24 smithi039 ceph-mon[29586]: osdmap e189: 8 total, 8 up, 8 in 2024-09-14T03:56:24.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:24 smithi039 ceph-mon[29586]: Standby manager daemon y restarted 2024-09-14T03:56:24.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:24 smithi039 ceph-mon[29586]: Standby manager daemon y started 2024-09-14T03:56:24.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:24 smithi039 ceph-mon[29586]: from='mgr.? 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-14T03:56:24.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:24 smithi039 ceph-mon[29586]: from='mgr.? 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:56:24.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:24 smithi039 ceph-mon[29586]: from='mgr.? 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-14T03:56:24.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:24 smithi039 ceph-mon[29586]: from='mgr.? 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:56:24.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:24 smithi039 ceph-mon[40962]: pgmap v43: 137 pgs: 137 active+clean; 561 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 104 KiB/s rd, 3.5 MiB/s wr, 147 op/s 2024-09-14T03:56:24.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:24 smithi039 ceph-mon[40962]: osdmap e189: 8 total, 8 up, 8 in 2024-09-14T03:56:24.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:24 smithi039 ceph-mon[40962]: Standby manager daemon y restarted 2024-09-14T03:56:24.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:24 smithi039 ceph-mon[40962]: Standby manager daemon y started 2024-09-14T03:56:24.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:24 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-14T03:56:24.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:24 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:56:24.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:24 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-14T03:56:24.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:24 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:56:24.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:24 smithi129 ceph-mon[33173]: pgmap v43: 137 pgs: 137 active+clean; 561 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 104 KiB/s rd, 3.5 MiB/s wr, 147 op/s 2024-09-14T03:56:24.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:24 smithi129 ceph-mon[33173]: osdmap e189: 8 total, 8 up, 8 in 2024-09-14T03:56:24.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:24 smithi129 ceph-mon[33173]: Standby manager daemon y restarted 2024-09-14T03:56:24.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:24 smithi129 ceph-mon[33173]: Standby manager daemon y started 2024-09-14T03:56:24.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:24 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-09-14T03:56:24.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:24 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:56:24.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:24 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-09-14T03:56:24.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:24 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:56:25.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:25 smithi039 ceph-mon[29586]: osdmap e190: 8 total, 8 up, 8 in 2024-09-14T03:56:25.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:25 smithi039 ceph-mon[29586]: mgrmap e27: x(active, since 29s), standbys: y 2024-09-14T03:56:25.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:25 smithi039 ceph-mon[40962]: osdmap e190: 8 total, 8 up, 8 in 2024-09-14T03:56:25.543 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:25 smithi039 ceph-mon[40962]: mgrmap e27: x(active, since 29s), standbys: y 2024-09-14T03:56:25.686 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.SnapsShrinkToZeroAfterPrepare (12575 ms) 2024-09-14T03:56:25.687 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.SnapsExpandAfterPrepare 2024-09-14T03:56:25.718 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:56:25.730 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 0~8388609 2024-09-14T03:56:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:25 smithi129 ceph-mon[33173]: osdmap e190: 8 total, 8 up, 8 in 2024-09-14T03:56:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:25 smithi129 ceph-mon[33173]: mgrmap e27: x(active, since 29s), standbys: y 2024-09-14T03:56:25.786 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap1 2024-09-14T03:56:26.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:03:56:26] ENGINE Bus STOPPING 2024-09-14T03:56:26.439 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: 2024-09-14T03:56:26.181+0000 7fc45632e640 -1 mgr handle_mgr_map I was active but no longer am 2024-09-14T03:56:26.439 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: ignoring --setuser ceph since I am not root 2024-09-14T03:56:26.439 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: ignoring --setgroup ceph since I am not root 2024-09-14T03:56:26.439 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mgr[69618]: -- 172.21.15.129:0/2321888047 <== mon.1 v2:172.21.15.129:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55e2d48790e0 con 0x55e2d4857000 2024-09-14T03:56:26.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mon[33173]: pgmap v46: 137 pgs: 137 active+clean; 561 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 104 KiB/s rd, 3.5 MiB/s wr, 148 op/s 2024-09-14T03:56:26.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mon[33173]: osdmap e191: 8 total, 8 up, 8 in 2024-09-14T03:56:26.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:26.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:26.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:56:26.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:56:26.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:26.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:56:26.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mon[33173]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-14T03:56:26.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-14T03:56:26.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mon[33173]: Activating manager daemon y 2024-09-14T03:56:26.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mon[33173]: osdmap e192: 8 total, 8 up, 8 in 2024-09-14T03:56:26.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mon[33173]: from='mgr.24724 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-14T03:56:26.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mon[33173]: mgrmap e28: y(active, starting, since 0.00924287s) 2024-09-14T03:56:26.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:56:26.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:56:26.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:56:26.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi039.bbwiwf"}]: dispatch 2024-09-14T03:56:26.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi129.ybjqcw"}]: dispatch 2024-09-14T03:56:26.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:56:26.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:56:26.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:56:26.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:56:26.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:56:26.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:56:26.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:56:26.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:56:26.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:56:26.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:56:26.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:56:26.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:26 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:56:26.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:03:56:26] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:56:26.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:03:56:26] ENGINE Bus STOPPED 2024-09-14T03:56:26.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[40962]: pgmap v46: 137 pgs: 137 active+clean; 561 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 104 KiB/s rd, 3.5 MiB/s wr, 148 op/s 2024-09-14T03:56:26.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[40962]: osdmap e191: 8 total, 8 up, 8 in 2024-09-14T03:56:26.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:26.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:26.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:56:26.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:56:26.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:26.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:56:26.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[40962]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-14T03:56:26.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-14T03:56:26.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[40962]: Activating manager daemon y 2024-09-14T03:56:26.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[40962]: osdmap e192: 8 total, 8 up, 8 in 2024-09-14T03:56:26.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[40962]: from='mgr.24724 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-14T03:56:26.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[40962]: mgrmap e28: y(active, starting, since 0.00924287s) 2024-09-14T03:56:26.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:56:26.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:56:26.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:56:26.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi039.bbwiwf"}]: dispatch 2024-09-14T03:56:26.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi129.ybjqcw"}]: dispatch 2024-09-14T03:56:26.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:56:26.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:56:26.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:56:26.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:56:26.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:56:26.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:56:26.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:56:26.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:56:26.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:56:26.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:56:26.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:56:26.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:56:26.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[29586]: pgmap v46: 137 pgs: 137 active+clean; 561 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 104 KiB/s rd, 3.5 MiB/s wr, 148 op/s 2024-09-14T03:56:26.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[29586]: osdmap e191: 8 total, 8 up, 8 in 2024-09-14T03:56:26.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:26.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:26.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:56:26.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:56:26.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' 2024-09-14T03:56:26.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:56:26.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[29586]: from='mgr.24724 172.21.15.129:0/3131514021' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-14T03:56:26.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-09-14T03:56:26.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[29586]: Activating manager daemon y 2024-09-14T03:56:26.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[29586]: osdmap e192: 8 total, 8 up, 8 in 2024-09-14T03:56:26.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[29586]: from='mgr.24724 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-09-14T03:56:26.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[29586]: mgrmap e28: y(active, starting, since 0.00924287s) 2024-09-14T03:56:26.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:56:26.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:56:26.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:56:26.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi039.bbwiwf"}]: dispatch 2024-09-14T03:56:26.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi129.ybjqcw"}]: dispatch 2024-09-14T03:56:26.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:56:26.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:56:26.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:56:26.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:56:26.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:56:26.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:56:26.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:56:26.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:56:26.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:56:26.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:56:26.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:56:26.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:26 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:56:27.205 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 8388609~8388609 2024-09-14T03:56:27.215 INFO:tasks.workunit.client.0.smithi039.stdout:write: X 0~1000 2024-09-14T03:56:27.215 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 1010~1000 2024-09-14T03:56:27.215 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap2 2024-09-14T03:56:27.441 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:27 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T03:56:27.174+0000 7f49781e0640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:56:27.441 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:27 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T03:56:27.174+0000 7f49781e0640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:56:27.441 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:27 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T03:56:27.174+0000 7f49781e0640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:56:27.441 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:27 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T03:56:27.174+0000 7f49781e0640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:56:27.441 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:27 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T03:56:27.174+0000 7f49781e0640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:56:27.441 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:27 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T03:56:27.175+0000 7f497b1e6640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:56:27.441 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:27 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T03:56:27.175+0000 7f497b1e6640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:56:27.441 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:27 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T03:56:27.175+0000 7f497b1e6640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:56:27.441 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:27 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T03:56:27.175+0000 7f497b1e6640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:56:27.442 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:27 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T03:56:27.175+0000 7f497b1e6640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:56:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:27 smithi129 ceph-mon[33173]: Manager daemon y is now available 2024-09-14T03:56:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:27 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:56:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:27 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:56:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:27 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:56:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:27 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:56:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:27 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:56:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:27 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:56:27.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:27 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:56:27.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:27 smithi129 ceph-mon[33173]: osdmap e193: 8 total, 8 up, 8 in 2024-09-14T03:56:27.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:27 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:03:56:27] ENGINE Bus STARTING 2024-09-14T03:56:27.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:27 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:03:56:27] ENGINE Serving on http://:::9283 2024-09-14T03:56:27.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:27 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:03:56:27] ENGINE Bus STARTED 2024-09-14T03:56:27.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:27 smithi039 ceph-mon[29586]: Manager daemon y is now available 2024-09-14T03:56:27.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:27 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:56:27.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:27 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:56:27.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:27 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:56:27.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:27 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:56:27.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:27 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:56:27.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:27 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:56:27.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:27 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:56:27.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:27 smithi039 ceph-mon[29586]: osdmap e193: 8 total, 8 up, 8 in 2024-09-14T03:56:27.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:27 smithi039 ceph-mon[40962]: Manager daemon y is now available 2024-09-14T03:56:27.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:27 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:56:27.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:27 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:56:27.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:27 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:56:27.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:27 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:56:27.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:27 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:56:27.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:27 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:56:27.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:27 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:56:27.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:27 smithi039 ceph-mon[40962]: osdmap e193: 8 total, 8 up, 8 in 2024-09-14T03:56:28.742 INFO:tasks.workunit.client.0.smithi039.stdout:write: X 8388609~1000 2024-09-14T03:56:28.749 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 16777228~1000 2024-09-14T03:56:28.749 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 167772160 2024-09-14T03:56:28.757 INFO:tasks.workunit.client.0.smithi039.stdout:write: 2 83886079~8388609 2024-09-14T03:56:28.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:28 smithi129 ceph-mon[33173]: mgrmap e29: y(active, since 1.27841s) 2024-09-14T03:56:28.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:28 smithi129 ceph-mon[33173]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:56:28.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:28 smithi129 ceph-mon[33173]: pgmap v4: 137 pgs: 137 active+clean; 593 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:56:28.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:28 smithi129 ceph-mon[33173]: osdmap e194: 8 total, 8 up, 8 in 2024-09-14T03:56:28.774 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap3 2024-09-14T03:56:28.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:28 smithi039 ceph-mon[29586]: mgrmap e29: y(active, since 1.27841s) 2024-09-14T03:56:28.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:28 smithi039 ceph-mon[29586]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:56:28.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:28 smithi039 ceph-mon[29586]: pgmap v4: 137 pgs: 137 active+clean; 593 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:56:28.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:28 smithi039 ceph-mon[29586]: osdmap e194: 8 total, 8 up, 8 in 2024-09-14T03:56:28.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:28 smithi039 ceph-mon[40962]: mgrmap e29: y(active, since 1.27841s) 2024-09-14T03:56:28.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:28 smithi039 ceph-mon[40962]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:56:28.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:28 smithi039 ceph-mon[40962]: pgmap v4: 137 pgs: 137 active+clean; 593 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:56:28.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:28 smithi039 ceph-mon[40962]: osdmap e194: 8 total, 8 up, 8 in 2024-09-14T03:56:29.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:29 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T03:56:29.255+0000 7f49556de640 -1 log_channel(cephadm) log [ERR] : [14/Sep/2024:03:56:29] ENGINE Error in HTTPServer.serve 2024-09-14T03:56:29.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:29 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: Traceback (most recent call last): 2024-09-14T03:56:29.291 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:29 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:56:29.291 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:29 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: self._connections.run(self.expiration_interval) 2024-09-14T03:56:29.291 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:29 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:56:29.291 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:29 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: self._run(expiration_interval) 2024-09-14T03:56:29.291 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:29 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:56:29.291 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:29 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:56:29.291 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:29 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:56:29.291 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:29 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:56:29.291 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:29 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:56:29.291 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:29 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: s = self.context.wrap_socket( 2024-09-14T03:56:29.291 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:29 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:56:29.291 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:29 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: return self.sslsocket_class._create( 2024-09-14T03:56:29.292 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:29 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:56:29.292 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:29 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: self.do_handshake() 2024-09-14T03:56:29.292 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:29 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:56:29.292 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:29 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: self._sslobj.do_handshake() 2024-09-14T03:56:29.292 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:29 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:56:29.292 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:29 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T03:56:29.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:29 smithi129 ceph-mon[33173]: pgmap v6: 137 pgs: 137 active+clean; 593 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:56:29.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:29 smithi129 ceph-mon[33173]: mgrmap e30: y(active, since 2s) 2024-09-14T03:56:29.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:29 smithi129 ceph-mon[33173]: osdmap e195: 8 total, 8 up, 8 in 2024-09-14T03:56:29.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:29 smithi039 ceph-mon[29586]: pgmap v6: 137 pgs: 137 active+clean; 593 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:56:29.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:29 smithi039 ceph-mon[29586]: mgrmap e30: y(active, since 2s) 2024-09-14T03:56:29.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:29 smithi039 ceph-mon[29586]: osdmap e195: 8 total, 8 up, 8 in 2024-09-14T03:56:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:29 smithi039 ceph-mon[40962]: pgmap v6: 137 pgs: 137 active+clean; 593 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:56:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:29 smithi039 ceph-mon[40962]: mgrmap e30: y(active, since 2s) 2024-09-14T03:56:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:29 smithi039 ceph-mon[40962]: osdmap e195: 8 total, 8 up, 8 in 2024-09-14T03:56:30.754 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:56:30.775 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 83886079~1 2024-09-14T03:56:30.782 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:56:31.002 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:30 smithi129 ceph-mon[33173]: [14/Sep/2024:03:56:29] ENGINE Bus STARTING 2024-09-14T03:56:31.002 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:30 smithi129 ceph-mon[33173]: [14/Sep/2024:03:56:29] ENGINE Serving on http://172.21.15.39:8765 2024-09-14T03:56:31.002 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:30 smithi129 ceph-mon[33173]: [14/Sep/2024:03:56:29] ENGINE Serving on https://172.21.15.39:7150 2024-09-14T03:56:31.002 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:30 smithi129 ceph-mon[33173]: [14/Sep/2024:03:56:29] ENGINE Bus STARTED 2024-09-14T03:56:31.002 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:30 smithi129 ceph-mon[33173]: [14/Sep/2024:03:56:29] ENGINE Error in HTTPServer.serve 2024-09-14T03:56:31.002 INFO:journalctl@ceph.mon.b.smithi129.stdout: Traceback (most recent call last): 2024-09-14T03:56:31.002 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:56:31.002 INFO:journalctl@ceph.mon.b.smithi129.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:56:31.002 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:56:31.002 INFO:journalctl@ceph.mon.b.smithi129.stdout: self._run(expiration_interval) 2024-09-14T03:56:31.003 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:56:31.003 INFO:journalctl@ceph.mon.b.smithi129.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:56:31.003 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:56:31.003 INFO:journalctl@ceph.mon.b.smithi129.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:56:31.003 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:56:31.003 INFO:journalctl@ceph.mon.b.smithi129.stdout: s = self.context.wrap_socket( 2024-09-14T03:56:31.003 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:56:31.003 INFO:journalctl@ceph.mon.b.smithi129.stdout: return self.sslsocket_class._create( 2024-09-14T03:56:31.003 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:56:31.003 INFO:journalctl@ceph.mon.b.smithi129.stdout: self.do_handshake() 2024-09-14T03:56:31.003 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:56:31.003 INFO:journalctl@ceph.mon.b.smithi129.stdout: self._sslobj.do_handshake() 2024-09-14T03:56:31.003 INFO:journalctl@ceph.mon.b.smithi129.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:56:31.003 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:30 smithi129 ceph-mon[33173]: osdmap e196: 8 total, 8 up, 8 in 2024-09-14T03:56:31.004 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:30 smithi129 ceph-mon[33173]: mgrmap e31: y(active, since 4s) 2024-09-14T03:56:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:30 smithi039 ceph-mon[29586]: [14/Sep/2024:03:56:29] ENGINE Bus STARTING 2024-09-14T03:56:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:30 smithi039 ceph-mon[29586]: [14/Sep/2024:03:56:29] ENGINE Serving on http://172.21.15.39:8765 2024-09-14T03:56:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:30 smithi039 ceph-mon[29586]: [14/Sep/2024:03:56:29] ENGINE Serving on https://172.21.15.39:7150 2024-09-14T03:56:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:30 smithi039 ceph-mon[29586]: [14/Sep/2024:03:56:29] ENGINE Bus STARTED 2024-09-14T03:56:31.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:30 smithi039 ceph-mon[29586]: [14/Sep/2024:03:56:29] ENGINE Error in HTTPServer.serve 2024-09-14T03:56:31.041 INFO:journalctl@ceph.mon.a.smithi039.stdout: Traceback (most recent call last): 2024-09-14T03:56:31.041 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:56:31.041 INFO:journalctl@ceph.mon.a.smithi039.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:56:31.041 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:56:31.041 INFO:journalctl@ceph.mon.a.smithi039.stdout: self._run(expiration_interval) 2024-09-14T03:56:31.041 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:56:31.041 INFO:journalctl@ceph.mon.a.smithi039.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:56:31.041 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:56:31.041 INFO:journalctl@ceph.mon.a.smithi039.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:56:31.041 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:56:31.041 INFO:journalctl@ceph.mon.a.smithi039.stdout: s = self.context.wrap_socket( 2024-09-14T03:56:31.041 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:56:31.042 INFO:journalctl@ceph.mon.a.smithi039.stdout: return self.sslsocket_class._create( 2024-09-14T03:56:31.042 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:56:31.042 INFO:journalctl@ceph.mon.a.smithi039.stdout: self.do_handshake() 2024-09-14T03:56:31.042 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:56:31.042 INFO:journalctl@ceph.mon.a.smithi039.stdout: self._sslobj.do_handshake() 2024-09-14T03:56:31.042 INFO:journalctl@ceph.mon.a.smithi039.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:56:31.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:30 smithi039 ceph-mon[29586]: osdmap e196: 8 total, 8 up, 8 in 2024-09-14T03:56:31.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:30 smithi039 ceph-mon[29586]: mgrmap e31: y(active, since 4s) 2024-09-14T03:56:31.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:30 smithi039 ceph-mon[40962]: [14/Sep/2024:03:56:29] ENGINE Bus STARTING 2024-09-14T03:56:31.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:30 smithi039 ceph-mon[40962]: [14/Sep/2024:03:56:29] ENGINE Serving on http://172.21.15.39:8765 2024-09-14T03:56:31.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:30 smithi039 ceph-mon[40962]: [14/Sep/2024:03:56:29] ENGINE Serving on https://172.21.15.39:7150 2024-09-14T03:56:31.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:30 smithi039 ceph-mon[40962]: [14/Sep/2024:03:56:29] ENGINE Bus STARTED 2024-09-14T03:56:31.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:30 smithi039 ceph-mon[40962]: [14/Sep/2024:03:56:29] ENGINE Error in HTTPServer.serve 2024-09-14T03:56:31.043 INFO:journalctl@ceph.mon.c.smithi039.stdout: Traceback (most recent call last): 2024-09-14T03:56:31.043 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:56:31.043 INFO:journalctl@ceph.mon.c.smithi039.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:56:31.043 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:56:31.043 INFO:journalctl@ceph.mon.c.smithi039.stdout: self._run(expiration_interval) 2024-09-14T03:56:31.043 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:56:31.043 INFO:journalctl@ceph.mon.c.smithi039.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:56:31.043 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:56:31.043 INFO:journalctl@ceph.mon.c.smithi039.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:56:31.043 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:56:31.043 INFO:journalctl@ceph.mon.c.smithi039.stdout: s = self.context.wrap_socket( 2024-09-14T03:56:31.043 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:56:31.044 INFO:journalctl@ceph.mon.c.smithi039.stdout: return self.sslsocket_class._create( 2024-09-14T03:56:31.044 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:56:31.044 INFO:journalctl@ceph.mon.c.smithi039.stdout: self.do_handshake() 2024-09-14T03:56:31.044 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:56:31.044 INFO:journalctl@ceph.mon.c.smithi039.stdout: self._sslobj.do_handshake() 2024-09-14T03:56:31.044 INFO:journalctl@ceph.mon.c.smithi039.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:56:31.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:30 smithi039 ceph-mon[40962]: osdmap e196: 8 total, 8 up, 8 in 2024-09-14T03:56:31.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:30 smithi039 ceph-mon[40962]: mgrmap e31: y(active, since 4s) 2024-09-14T03:56:31.738 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:31 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: [14/Sep/2024:03:56:31] ENGINE Bus STARTING 2024-09-14T03:56:31.739 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:31 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: CherryPy Checker: 2024-09-14T03:56:31.739 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:31 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: The Application mounted at '' has an empty config. 2024-09-14T03:56:31.739 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:31 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: 2024-09-14T03:56:31.739 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:31 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: [14/Sep/2024:03:56:31] ENGINE Serving on http://:::9283 2024-09-14T03:56:31.739 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:31 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[69595]: [14/Sep/2024:03:56:31] ENGINE Bus STARTED 2024-09-14T03:56:32.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:31 smithi129 ceph-mon[33173]: pgmap v9: 137 pgs: 137 active+clean; 593 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:56:32.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:31 smithi129 ceph-mon[33173]: osdmap e197: 8 total, 8 up, 8 in 2024-09-14T03:56:32.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:31 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:32.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:31 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:32.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:31 smithi129 ceph-mon[33173]: Standby manager daemon x started 2024-09-14T03:56:32.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:31 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.129:0/688482228' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:56:32.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:31 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.129:0/688482228' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:56:32.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:31 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.129:0/688482228' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:56:32.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:31 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.129:0/688482228' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:56:32.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:31 smithi039 ceph-mon[40962]: pgmap v9: 137 pgs: 137 active+clean; 593 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:56:32.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:31 smithi039 ceph-mon[40962]: osdmap e197: 8 total, 8 up, 8 in 2024-09-14T03:56:32.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:31 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:32.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:31 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:32.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:31 smithi039 ceph-mon[40962]: Standby manager daemon x started 2024-09-14T03:56:32.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:31 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.129:0/688482228' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:56:32.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:31 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.129:0/688482228' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:56:32.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:31 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.129:0/688482228' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:56:32.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:31 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.129:0/688482228' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:56:32.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:31 smithi039 ceph-mon[29586]: pgmap v9: 137 pgs: 137 active+clean; 593 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:56:32.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:31 smithi039 ceph-mon[29586]: osdmap e197: 8 total, 8 up, 8 in 2024-09-14T03:56:32.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:31 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:32.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:31 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:32.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:31 smithi039 ceph-mon[29586]: Standby manager daemon x started 2024-09-14T03:56:32.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:31 smithi039 ceph-mon[29586]: from='mgr.? 172.21.15.129:0/688482228' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:56:32.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:31 smithi039 ceph-mon[29586]: from='mgr.? 172.21.15.129:0/688482228' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:56:32.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:31 smithi039 ceph-mon[29586]: from='mgr.? 172.21.15.129:0/688482228' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:56:32.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:31 smithi039 ceph-mon[29586]: from='mgr.? 172.21.15.129:0/688482228' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:56:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:32 smithi129 ceph-mon[33173]: osdmap e198: 8 total, 8 up, 8 in 2024-09-14T03:56:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:32 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:32 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:32 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:32 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:32 smithi129 ceph-mon[33173]: mgrmap e32: y(active, since 6s), standbys: x 2024-09-14T03:56:33.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:32 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:56:33.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:32 smithi039 ceph-mon[29586]: osdmap e198: 8 total, 8 up, 8 in 2024-09-14T03:56:33.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:32 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:33.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:32 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:33.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:32 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:33.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:32 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:33.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:32 smithi039 ceph-mon[29586]: mgrmap e32: y(active, since 6s), standbys: x 2024-09-14T03:56:33.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:32 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:56:33.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:32 smithi039 ceph-mon[40962]: osdmap e198: 8 total, 8 up, 8 in 2024-09-14T03:56:33.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:32 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:33.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:32 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:33.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:32 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:33.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:32 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:33.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:32 smithi039 ceph-mon[40962]: mgrmap e32: y(active, since 6s), standbys: x 2024-09-14T03:56:33.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:32 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:56:34.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:33 smithi129 ceph-mon[33173]: pgmap v12: 137 pgs: 137 active+clean; 593 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:56:34.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:33 smithi129 ceph-mon[33173]: osdmap e199: 8 total, 8 up, 8 in 2024-09-14T03:56:34.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:33 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:34.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:33 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:34.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:33 smithi039 ceph-mon[29586]: pgmap v12: 137 pgs: 137 active+clean; 593 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:56:34.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:33 smithi039 ceph-mon[29586]: osdmap e199: 8 total, 8 up, 8 in 2024-09-14T03:56:34.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:33 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:34.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:33 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:34.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:33 smithi039 ceph-mon[40962]: pgmap v12: 137 pgs: 137 active+clean; 593 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:56:34.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:33 smithi039 ceph-mon[40962]: osdmap e199: 8 total, 8 up, 8 in 2024-09-14T03:56:34.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:33 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:34.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:33 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:34.158 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 167772160 2024-09-14T03:56:34.168 INFO:tasks.workunit.client.0.smithi039.stdout:write: * 83886080~1000 2024-09-14T03:56:34.204 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:56:34.346 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:56:35.033 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:34 smithi039 ceph-mon[29586]: osdmap e200: 8 total, 8 up, 8 in 2024-09-14T03:56:35.033 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:34 smithi039 ceph-mon[40962]: osdmap e200: 8 total, 8 up, 8 in 2024-09-14T03:56:35.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:34 smithi129 ceph-mon[33173]: osdmap e200: 8 total, 8 up, 8 in 2024-09-14T03:56:35.331 INFO:teuthology.orchestra.run.smithi039.stdout:true 2024-09-14T03:56:35.781 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:35 smithi039 ceph-mon[40962]: pgmap v15: 137 pgs: 137 active+clean; 617 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 163 KiB/s rd, 4.0 MiB/s wr, 188 op/s 2024-09-14T03:56:35.781 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:35 smithi039 ceph-mon[40962]: osdmap e201: 8 total, 8 up, 8 in 2024-09-14T03:56:35.781 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:35 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:35.781 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:35 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:35.952 INFO:teuthology.orchestra.run.smithi039.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:56:35.952 INFO:teuthology.orchestra.run.smithi039.stdout:alertmanager.a smithi039 *:9093,9094 running (3m) 4s ago 4m 28.6M - 0.25.0 c8568f914cd2 b5ca6b93d9ee 2024-09-14T03:56:35.952 INFO:teuthology.orchestra.run.smithi039.stdout:grafana.a smithi129 *:3000 running (4m) 3s ago 4m 90.9M - 9.4.7 954c08fa6188 71f297945590 2024-09-14T03:56:35.952 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi039.bbwiwf smithi039 running (3m) 4s ago 3m 19.0M - 18.2.4-845-ge4e227b3 67ddd33fa72f 91e6c8f85681 2024-09-14T03:56:35.953 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi129.ybjqcw smithi129 running (3m) 3s ago 3m 14.4M - 18.2.4-845-ge4e227b3 67ddd33fa72f a3fab5b37b71 2024-09-14T03:56:35.953 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.x smithi129 *:8443,9283,8765 running (59s) 3s ago 8m 153M - 19.3.0-4907-ga2782315 8801d2529e43 759df831a5de 2024-09-14T03:56:35.953 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.y smithi039 *:8443,9283,8765 running (18s) 4s ago 10m 490M - 19.3.0-4907-ga2782315 8801d2529e43 cccc87f635a6 2024-09-14T03:56:35.953 INFO:teuthology.orchestra.run.smithi039.stdout:mon.a smithi039 running (10m) 4s ago 10m 57.2M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f e7b5f91fb09c 2024-09-14T03:56:35.953 INFO:teuthology.orchestra.run.smithi039.stdout:mon.b smithi129 running (9m) 3s ago 9m 49.5M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f eb38cc5cbe35 2024-09-14T03:56:35.953 INFO:teuthology.orchestra.run.smithi039.stdout:mon.c smithi039 running (8m) 4s ago 8m 47.0M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 31d440e49f93 2024-09-14T03:56:35.953 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.a smithi039 *:9100 running (4m) 4s ago 4m 14.1M - 1.5.0 0da6a335fe13 4eff582f78f5 2024-09-14T03:56:35.953 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.b smithi129 *:9100 running (3m) 3s ago 3m 15.1M - 1.5.0 0da6a335fe13 c7581baf44f2 2024-09-14T03:56:35.953 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0 smithi039 running (8m) 4s ago 8m 109M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 5048d74f27ac 2024-09-14T03:56:35.953 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1 smithi039 running (7m) 4s ago 7m 111M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 91349527d88b 2024-09-14T03:56:35.953 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2 smithi039 running (7m) 4s ago 7m 90.5M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f eb903b44fc5d 2024-09-14T03:56:35.953 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3 smithi039 running (7m) 4s ago 7m 107M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f dff9e665bd9b 2024-09-14T03:56:35.953 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4 smithi129 running (6m) 3s ago 6m 97.9M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 5b64727311b7 2024-09-14T03:56:35.954 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5 smithi129 running (6m) 3s ago 6m 86.7M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f f2d991a754f7 2024-09-14T03:56:35.954 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6 smithi129 running (5m) 3s ago 5m 118M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 3060eaf44106 2024-09-14T03:56:35.954 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7 smithi129 running (5m) 3s ago 5m 86.1M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 4c13b97060d1 2024-09-14T03:56:35.954 INFO:teuthology.orchestra.run.smithi039.stdout:prometheus.a smithi129 *:9095 running (24s) 3s ago 4m 37.7M - 2.43.0 a07b618ecd1d 474a9ac1dffe 2024-09-14T03:56:36.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:35 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:56:36.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:35 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:56:36.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:35 smithi039 ceph-mon[29586]: pgmap v15: 137 pgs: 137 active+clean; 617 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 163 KiB/s rd, 4.0 MiB/s wr, 188 op/s 2024-09-14T03:56:36.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:35 smithi039 ceph-mon[29586]: osdmap e201: 8 total, 8 up, 8 in 2024-09-14T03:56:36.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:35 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:36.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:35 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:36.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:35 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:56:36.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:35 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:56:36.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:35 smithi129 ceph-mon[33173]: pgmap v15: 137 pgs: 137 active+clean; 617 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 163 KiB/s rd, 4.0 MiB/s wr, 188 op/s 2024-09-14T03:56:36.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:35 smithi129 ceph-mon[33173]: osdmap e201: 8 total, 8 up, 8 in 2024-09-14T03:56:36.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:35 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:36.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:35 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:36.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:35 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:56:36.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:35 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:56:36.312 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T03:56:36.312 INFO:teuthology.orchestra.run.smithi039.stdout: "mon": { 2024-09-14T03:56:36.312 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 3 2024-09-14T03:56:36.312 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:56:36.312 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr": { 2024-09-14T03:56:36.312 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:56:36.313 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:56:36.313 INFO:teuthology.orchestra.run.smithi039.stdout: "osd": { 2024-09-14T03:56:36.313 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8 2024-09-14T03:56:36.313 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:56:36.313 INFO:teuthology.orchestra.run.smithi039.stdout: "mds": { 2024-09-14T03:56:36.313 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:56:36.313 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:56:36.313 INFO:teuthology.orchestra.run.smithi039.stdout: "overall": { 2024-09-14T03:56:36.313 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 13, 2024-09-14T03:56:36.313 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:56:36.313 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T03:56:36.313 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T03:56:36.609 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T03:56:36.610 INFO:teuthology.orchestra.run.smithi039.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:56:36.610 INFO:teuthology.orchestra.run.smithi039.stdout: "in_progress": true, 2024-09-14T03:56:36.610 INFO:teuthology.orchestra.run.smithi039.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-14T03:56:36.610 INFO:teuthology.orchestra.run.smithi039.stdout: "services_complete": [ 2024-09-14T03:56:36.610 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr" 2024-09-14T03:56:36.610 INFO:teuthology.orchestra.run.smithi039.stdout: ], 2024-09-14T03:56:36.610 INFO:teuthology.orchestra.run.smithi039.stdout: "progress": "2/2 daemons upgraded", 2024-09-14T03:56:36.610 INFO:teuthology.orchestra.run.smithi039.stdout: "message": "", 2024-09-14T03:56:36.610 INFO:teuthology.orchestra.run.smithi039.stdout: "is_paused": false 2024-09-14T03:56:36.610 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T03:56:36.941 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:36 smithi129 ceph-mon[33173]: from='client.15087 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:36.942 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:36 smithi129 ceph-mon[33173]: Updating smithi039:/etc/ceph/ceph.conf 2024-09-14T03:56:36.942 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:36 smithi129 ceph-mon[33173]: Updating smithi129:/etc/ceph/ceph.conf 2024-09-14T03:56:36.942 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:36 smithi129 ceph-mon[33173]: Updating smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:56:36.942 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:36 smithi129 ceph-mon[33173]: Updating smithi039:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:56:36.942 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:36 smithi129 ceph-mon[33173]: from='client.24995 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:36.942 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:36 smithi129 ceph-mon[33173]: Updating smithi129:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:56:36.942 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:36 smithi129 ceph-mon[33173]: Updating smithi039:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:56:36.942 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:36 smithi129 ceph-mon[33173]: osdmap e202: 8 total, 8 up, 8 in 2024-09-14T03:56:36.942 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:36 smithi129 ceph-mon[33173]: Updating smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:56:36.942 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:36 smithi129 ceph-mon[33173]: Updating smithi039:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:56:36.942 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:36 smithi129 ceph-mon[33173]: from='client.25001 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:36.942 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:36 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:36.943 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:36 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:36.943 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:36 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:36.943 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:36 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:36.943 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:36 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:36.943 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:36 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/580372493' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:37.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[40962]: from='client.15087 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:37.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[40962]: Updating smithi039:/etc/ceph/ceph.conf 2024-09-14T03:56:37.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[40962]: Updating smithi129:/etc/ceph/ceph.conf 2024-09-14T03:56:37.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[40962]: Updating smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:56:37.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[40962]: Updating smithi039:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:56:37.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[40962]: from='client.24995 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:37.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[40962]: Updating smithi129:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:56:37.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[40962]: Updating smithi039:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:56:37.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[40962]: osdmap e202: 8 total, 8 up, 8 in 2024-09-14T03:56:37.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[40962]: Updating smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:56:37.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[40962]: Updating smithi039:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:56:37.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[40962]: from='client.25001 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:37.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:37.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:37.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:37.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:37.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:37.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/580372493' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:37.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[29586]: from='client.15087 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:37.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[29586]: Updating smithi039:/etc/ceph/ceph.conf 2024-09-14T03:56:37.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[29586]: Updating smithi129:/etc/ceph/ceph.conf 2024-09-14T03:56:37.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[29586]: Updating smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:56:37.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[29586]: Updating smithi039:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:56:37.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[29586]: from='client.24995 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:37.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[29586]: Updating smithi129:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:56:37.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[29586]: Updating smithi039:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:56:37.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[29586]: osdmap e202: 8 total, 8 up, 8 in 2024-09-14T03:56:37.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[29586]: Updating smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:56:37.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[29586]: Updating smithi039:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:56:37.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[29586]: from='client.25001 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:37.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:37.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:37.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:37.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:37.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:37.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:36 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/580372493' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:37.136 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.SnapsExpandAfterPrepare (11449 ms) 2024-09-14T03:56:37.136 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.SnapsExpandAfterPrepare2 2024-09-14T03:56:37.167 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:56:37.179 INFO:tasks.workunit.client.0.smithi039.stdout:write: X 41943040~10 2024-09-14T03:56:37.188 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap1 2024-09-14T03:56:38.127 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:37 smithi129 ceph-mon[33173]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:56:38.127 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:37 smithi129 ceph-mon[33173]: pgmap v18: 137 pgs: 137 active+clean; 617 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 163 KiB/s rd, 4.0 MiB/s wr, 188 op/s 2024-09-14T03:56:38.127 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:37 smithi129 ceph-mon[33173]: Reconfiguring daemon prometheus.a on smithi129 2024-09-14T03:56:38.128 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:37 smithi129 ceph-mon[33173]: from='client.25013 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:38.128 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:37 smithi129 ceph-mon[33173]: osdmap e203: 8 total, 8 up, 8 in 2024-09-14T03:56:38.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:37 smithi039 ceph-mon[29586]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:56:38.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:37 smithi039 ceph-mon[29586]: pgmap v18: 137 pgs: 137 active+clean; 617 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 163 KiB/s rd, 4.0 MiB/s wr, 188 op/s 2024-09-14T03:56:38.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:37 smithi039 ceph-mon[29586]: Reconfiguring daemon prometheus.a on smithi129 2024-09-14T03:56:38.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:37 smithi039 ceph-mon[29586]: from='client.25013 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:38.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:37 smithi039 ceph-mon[29586]: osdmap e203: 8 total, 8 up, 8 in 2024-09-14T03:56:38.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:37 smithi039 ceph-mon[40962]: Reconfiguring prometheus.a (dependencies changed)... 2024-09-14T03:56:38.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:37 smithi039 ceph-mon[40962]: pgmap v18: 137 pgs: 137 active+clean; 617 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 163 KiB/s rd, 4.0 MiB/s wr, 188 op/s 2024-09-14T03:56:38.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:37 smithi039 ceph-mon[40962]: Reconfiguring daemon prometheus.a on smithi129 2024-09-14T03:56:38.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:37 smithi039 ceph-mon[40962]: from='client.25013 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:56:38.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:37 smithi039 ceph-mon[40962]: osdmap e203: 8 total, 8 up, 8 in 2024-09-14T03:56:39.053 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 41943040 2024-09-14T03:56:39.079 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:56:39.217 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:39 smithi129 ceph-mon[33173]: osdmap e204: 8 total, 8 up, 8 in 2024-09-14T03:56:39.217 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:39 smithi129 ceph-mon[33173]: pgmap v21: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 70 MiB/s rd, 6.0 MiB/s wr, 512 op/s 2024-09-14T03:56:39.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:39 smithi039 ceph-mon[29586]: osdmap e204: 8 total, 8 up, 8 in 2024-09-14T03:56:39.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:39 smithi039 ceph-mon[29586]: pgmap v21: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 70 MiB/s rd, 6.0 MiB/s wr, 512 op/s 2024-09-14T03:56:39.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:39 smithi039 ceph-mon[40962]: osdmap e204: 8 total, 8 up, 8 in 2024-09-14T03:56:39.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:39 smithi039 ceph-mon[40962]: pgmap v21: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 70 MiB/s rd, 6.0 MiB/s wr, 512 op/s 2024-09-14T03:56:39.521 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:39 smithi129 systemd[1]: Stopping Ceph prometheus.a for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:56:39.880 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:39 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:39.621Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-14T03:56:39.880 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:39 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:39.621Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-14T03:56:39.880 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:39 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:39.621Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-14T03:56:39.880 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:39 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:39.621Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-14T03:56:39.880 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:39 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:39.621Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-14T03:56:39.880 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:39 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:39.622Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-14T03:56:39.881 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:39 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:39.622Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-14T03:56:39.881 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:39 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:39.622Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-14T03:56:39.881 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:39 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:39.622Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-14T03:56:39.881 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:39 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:39.622Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-14T03:56:39.881 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:39 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:39.622Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-14T03:56:39.881 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:39 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[77771]: ts=2024-09-14T03:56:39.622Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-14T03:56:39.881 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:39 smithi129 podman[79815]: 2024-09-14 03:56:39.632383352 +0000 UTC m=+0.136124324 container died 474a9ac1dffece838ca5f7d37f17342082f7d784d6457611f686c6776ad76326 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:56:40.204 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:56:40.215 INFO:tasks.workunit.client.0.smithi039.stdout:write: Y 41943040~5 2024-09-14T03:56:40.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:40 smithi129 ceph-mon[33173]: osdmap e205: 8 total, 8 up, 8 in 2024-09-14T03:56:40.271 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:39 smithi129 podman[79815]: 2024-09-14 03:56:39.978069004 +0000 UTC m=+0.481809963 container cleanup 474a9ac1dffece838ca5f7d37f17342082f7d784d6457611f686c6776ad76326 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:56:40.271 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:39 smithi129 bash[79815]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a 2024-09-14T03:56:40.310 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:56:40.384 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:56:40.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:40 smithi039 ceph-mon[29586]: osdmap e205: 8 total, 8 up, 8 in 2024-09-14T03:56:40.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:40 smithi039 ceph-mon[40962]: osdmap e205: 8 total, 8 up, 8 in 2024-09-14T03:56:40.771 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:40 smithi129 podman[79832]: 2024-09-14 03:56:40.413668521 +0000 UTC m=+0.778530239 container remove 474a9ac1dffece838ca5f7d37f17342082f7d784d6457611f686c6776ad76326 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:56:40.771 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:40 smithi129 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-14T03:56:40.771 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:40 smithi129 systemd[1]: Stopped Ceph prometheus.a for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:56:40.771 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:40 smithi129 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 1.108s CPU time. 2024-09-14T03:56:40.771 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:40 smithi129 systemd[1]: Starting Ceph prometheus.a for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:56:41.026 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:40 smithi129 podman[79928]: 2024-09-14 03:56:40.67757917 +0000 UTC m=+0.016802736 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-09-14T03:56:41.026 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:40 smithi129 podman[79928]: 2024-09-14 03:56:40.781163427 +0000 UTC m=+0.120386979 container create e8a320b5858c9699b387a22bd4419be8a3949bc4df49a6b23526752a3aa4189f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:56:41.171 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.SnapsExpandAfterPrepare2 (4035 ms) 2024-09-14T03:56:41.171 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.SnapsSnapAfterPrepare 2024-09-14T03:56:41.216 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:56:41.226 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 0~8388609 2024-09-14T03:56:41.295 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap1 2024-09-14T03:56:41.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:41 smithi129 ceph-mon[33173]: osdmap e206: 8 total, 8 up, 8 in 2024-09-14T03:56:41.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:41 smithi129 ceph-mon[33173]: pgmap v24: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 70 MiB/s rd, 6.0 MiB/s wr, 512 op/s 2024-09-14T03:56:41.522 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:41 smithi129 podman[79928]: 2024-09-14 03:56:41.025465274 +0000 UTC m=+0.364688825 container init e8a320b5858c9699b387a22bd4419be8a3949bc4df49a6b23526752a3aa4189f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:56:41.522 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:41 smithi129 podman[79928]: 2024-09-14 03:56:41.03002149 +0000 UTC m=+0.369245043 container start e8a320b5858c9699b387a22bd4419be8a3949bc4df49a6b23526752a3aa4189f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:56:41.522 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:56:41.053Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-09-14T03:56:41.522 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:56:41.053Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-09-14T03:56:41.522 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:56:41.053Z caller=main.go:570 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi129 (none))" 2024-09-14T03:56:41.522 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:56:41.053Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-14T03:56:41.522 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:56:41.053Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-14T03:56:41.523 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:56:41.055Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-09-14T03:56:41.523 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:56:41.056Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-09-14T03:56:41.523 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:56:41.056Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-09-14T03:56:41.523 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:56:41.056Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-14T03:56:41.523 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:56:41.058Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-14T03:56:41.523 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:56:41.059Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=3.362µs 2024-09-14T03:56:41.523 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:56:41.059Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-14T03:56:41.523 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:56:41.060Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-09-14T03:56:41.523 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:56:41.070Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-09-14T03:56:41.523 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:56:41.070Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-09-14T03:56:41.523 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:56:41.071Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-09-14T03:56:41.523 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:56:41.071Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=42.637µs wal_replay_duration=12.040592ms wbl_replay_duration=135ns total_replay_duration=12.166247ms 2024-09-14T03:56:41.523 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:56:41.073Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-09-14T03:56:41.523 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:56:41.073Z caller=main.go:1029 level=info msg="TSDB started" 2024-09-14T03:56:41.524 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:56:41.073Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-14T03:56:41.524 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:56:41.096Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=23.252667ms db_storage=1.046µs remote_storage=1.786µs web_handler=442ns query_engine=786ns scrape=568.375µs scrape_sd=151.158µs notify=12.769µs notify_sd=8.6µs rules=21.861778ms tracing=6.09µs 2024-09-14T03:56:41.524 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:56:41.096Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-09-14T03:56:41.524 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:56:41.096Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-09-14T03:56:41.524 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:41 smithi129 bash[79928]: e8a320b5858c9699b387a22bd4419be8a3949bc4df49a6b23526752a3aa4189f 2024-09-14T03:56:41.524 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:56:41 smithi129 systemd[1]: Started Ceph prometheus.a for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:56:41.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:41 smithi039 ceph-mon[29586]: osdmap e206: 8 total, 8 up, 8 in 2024-09-14T03:56:41.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:41 smithi039 ceph-mon[29586]: pgmap v24: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 70 MiB/s rd, 6.0 MiB/s wr, 512 op/s 2024-09-14T03:56:41.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:41 smithi039 ceph-mon[40962]: osdmap e206: 8 total, 8 up, 8 in 2024-09-14T03:56:41.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:41 smithi039 ceph-mon[40962]: pgmap v24: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 70 MiB/s rd, 6.0 MiB/s wr, 512 op/s 2024-09-14T03:56:42.267 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:42 smithi129 ceph-mon[33173]: osdmap e207: 8 total, 8 up, 8 in 2024-09-14T03:56:42.267 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:42 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:42.267 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:42 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:42.267 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:42 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:56:42.267 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:42 smithi129 ceph-mon[33173]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:56:42.267 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:42 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:56:42.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:42 smithi039 ceph-mon[29586]: osdmap e207: 8 total, 8 up, 8 in 2024-09-14T03:56:42.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:42 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:42.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:42 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:42.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:42 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:56:42.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:42 smithi039 ceph-mon[29586]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:56:42.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:42 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:56:42.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:42 smithi039 ceph-mon[40962]: osdmap e207: 8 total, 8 up, 8 in 2024-09-14T03:56:42.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:42 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:42.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:42 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:42.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:42 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:56:42.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:42 smithi039 ceph-mon[40962]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:56:42.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:42 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:56:43.210 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 8388609~8388609 2024-09-14T03:56:43.229 INFO:tasks.workunit.client.0.smithi039.stdout:write: X 0~1000 2024-09-14T03:56:43.229 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 1010~1000 2024-09-14T03:56:43.229 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap2 2024-09-14T03:56:43.294 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:43 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:56:43.294 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:43 smithi129 ceph-mon[33173]: osdmap e208: 8 total, 8 up, 8 in 2024-09-14T03:56:43.294 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:43 smithi129 ceph-mon[33173]: pgmap v27: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:56:43.294 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:43 smithi129 ceph-mon[33173]: Upgrade: Updating mgr.x 2024-09-14T03:56:43.294 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:43 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:56:43.294 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:43 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:56:43.294 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:43 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:56:43.294 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:43 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:56:43.294 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:43 smithi129 ceph-mon[33173]: Deploying daemon mgr.x on smithi129 2024-09-14T03:56:43.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:43 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:56:43.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:43 smithi039 ceph-mon[29586]: osdmap e208: 8 total, 8 up, 8 in 2024-09-14T03:56:43.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:43 smithi039 ceph-mon[29586]: pgmap v27: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:56:43.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:43 smithi039 ceph-mon[29586]: Upgrade: Updating mgr.x 2024-09-14T03:56:43.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:43 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:56:43.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:43 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:56:43.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:43 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:56:43.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:43 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:56:43.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:43 smithi039 ceph-mon[29586]: Deploying daemon mgr.x on smithi129 2024-09-14T03:56:43.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:43 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:56:43.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:43 smithi039 ceph-mon[40962]: osdmap e208: 8 total, 8 up, 8 in 2024-09-14T03:56:43.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:43 smithi039 ceph-mon[40962]: pgmap v27: 137 pgs: 137 active+clean; 617 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:56:43.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:43 smithi039 ceph-mon[40962]: Upgrade: Updating mgr.x 2024-09-14T03:56:43.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:43 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:56:43.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:43 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-09-14T03:56:43.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:43 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-09-14T03:56:43.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:43 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:56:43.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:43 smithi039 ceph-mon[40962]: Deploying daemon mgr.x on smithi129 2024-09-14T03:56:44.521 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:44 smithi129 systemd[1]: Stopping Ceph mgr.x for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:56:44.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:44 smithi129 ceph-mon[33173]: osdmap e209: 8 total, 8 up, 8 in 2024-09-14T03:56:44.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:44 smithi129 ceph-mon[33173]: osdmap e210: 8 total, 8 up, 8 in 2024-09-14T03:56:44.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:44 smithi039 ceph-mon[29586]: osdmap e209: 8 total, 8 up, 8 in 2024-09-14T03:56:44.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:44 smithi039 ceph-mon[29586]: osdmap e210: 8 total, 8 up, 8 in 2024-09-14T03:56:44.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:44 smithi039 ceph-mon[40962]: osdmap e209: 8 total, 8 up, 8 in 2024-09-14T03:56:44.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:44 smithi039 ceph-mon[40962]: osdmap e210: 8 total, 8 up, 8 in 2024-09-14T03:56:44.740 INFO:tasks.workunit.client.0.smithi039.stdout:write: X 8388609~1000 2024-09-14T03:56:44.746 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 16777228~1000 2024-09-14T03:56:44.746 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 167772160 2024-09-14T03:56:44.754 INFO:tasks.workunit.client.0.smithi039.stdout:write: 2 83886079~8388609 2024-09-14T03:56:44.779 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap3 2024-09-14T03:56:44.807 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:44 smithi129 podman[80280]: 2024-09-14 03:56:44.552199306 +0000 UTC m=+0.145596351 container died 759df831a5deb6dd936eb415a7792ddabd78086c3bffab265146384dcfa61d65 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , ceph=True, RELEASE=main-a278231, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0) 2024-09-14T03:56:45.171 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:44 smithi129 podman[80280]: 2024-09-14 03:56:44.910837492 +0000 UTC m=+0.504234546 container cleanup 759df831a5deb6dd936eb415a7792ddabd78086c3bffab265146384dcfa61d65 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=main-a278231, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0) 2024-09-14T03:56:45.171 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:44 smithi129 bash[80280]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x 2024-09-14T03:56:45.521 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:45 smithi129 podman[80297]: 2024-09-14 03:56:45.170224218 +0000 UTC m=+0.614378768 container remove 759df831a5deb6dd936eb415a7792ddabd78086c3bffab265146384dcfa61d65 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=main-a278231, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-14T03:56:45.521 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:45 smithi129 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-09-14T03:56:45.521 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:45 smithi129 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-09-14T03:56:45.521 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:45 smithi129 systemd[1]: Stopped Ceph mgr.x for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:56:45.521 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:45 smithi129 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mgr.x.service: Consumed 17.445s CPU time. 2024-09-14T03:56:45.789 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:45 smithi129 ceph-mon[33173]: pgmap v30: 137 pgs: 137 active+clean; 633 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 305 KiB/s rd, 4.0 MiB/s wr, 395 op/s 2024-09-14T03:56:45.789 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:45 smithi129 ceph-mon[33173]: osdmap e211: 8 total, 8 up, 8 in 2024-09-14T03:56:45.789 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:45 smithi129 systemd[1]: Starting Ceph mgr.x for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:56:45.789 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:45 smithi129 podman[80427]: 2024-09-14 03:56:45.646092764 +0000 UTC m=+0.016479808 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:56:46.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:45 smithi039 ceph-mon[29586]: pgmap v30: 137 pgs: 137 active+clean; 633 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 305 KiB/s rd, 4.0 MiB/s wr, 395 op/s 2024-09-14T03:56:46.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:45 smithi039 ceph-mon[29586]: osdmap e211: 8 total, 8 up, 8 in 2024-09-14T03:56:46.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:45 smithi039 ceph-mon[40962]: pgmap v30: 137 pgs: 137 active+clean; 633 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 305 KiB/s rd, 4.0 MiB/s wr, 395 op/s 2024-09-14T03:56:46.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:45 smithi039 ceph-mon[40962]: osdmap e211: 8 total, 8 up, 8 in 2024-09-14T03:56:46.521 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:45 smithi129 podman[80427]: 2024-09-14 03:56:45.788399244 +0000 UTC m=+0.158786280 container create 448bd4e146cbc29c4c675b1e56cbb6fd5a41183ca740be695d39ce07342e7aad (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=main-a278231, maintainer=Guillaume Abrioux , ceph=True) 2024-09-14T03:56:46.521 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:46 smithi129 podman[80427]: 2024-09-14 03:56:46.040962978 +0000 UTC m=+0.411350016 container init 448bd4e146cbc29c4c675b1e56cbb6fd5a41183ca740be695d39ce07342e7aad (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x, RELEASE=main-a278231, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_BRANCH=HEAD) 2024-09-14T03:56:46.521 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:46 smithi129 podman[80427]: 2024-09-14 03:56:46.045530367 +0000 UTC m=+0.415917405 container start 448bd4e146cbc29c4c675b1e56cbb6fd5a41183ca740be695d39ce07342e7aad (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, RELEASE=main-a278231) 2024-09-14T03:56:46.521 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:46 smithi129 ceph-mgr[80465]: -- 172.21.15.129:0/1187086511 <== mon.1 v2:172.21.15.129:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56514a6390e0 con 0x56514a616800 2024-09-14T03:56:46.522 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:46 smithi129 bash[80427]: 448bd4e146cbc29c4c675b1e56cbb6fd5a41183ca740be695d39ce07342e7aad 2024-09-14T03:56:46.522 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:46 smithi129 systemd[1]: Started Ceph mgr.x for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:56:46.787 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:56:46.810 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 83886079~1 2024-09-14T03:56:46.819 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:56:47.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:46 smithi129 ceph-mon[33173]: osdmap e212: 8 total, 8 up, 8 in 2024-09-14T03:56:47.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:46 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:47.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:46 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:47.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:46 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:56:47.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:46 smithi039 ceph-mon[29586]: osdmap e212: 8 total, 8 up, 8 in 2024-09-14T03:56:47.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:46 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:47.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:46 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:47.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:46 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:56:47.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:46 smithi039 ceph-mon[40962]: osdmap e212: 8 total, 8 up, 8 in 2024-09-14T03:56:47.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:46 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:47.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:46 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:47.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:46 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:56:47.866 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:47 smithi129 ceph-mon[33173]: pgmap v33: 137 pgs: 137 active+clean; 633 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 305 KiB/s rd, 4.0 MiB/s wr, 395 op/s 2024-09-14T03:56:47.866 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:47 smithi129 ceph-mon[33173]: osdmap e213: 8 total, 8 up, 8 in 2024-09-14T03:56:48.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:47 smithi039 ceph-mon[29586]: pgmap v33: 137 pgs: 137 active+clean; 633 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 305 KiB/s rd, 4.0 MiB/s wr, 395 op/s 2024-09-14T03:56:48.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:47 smithi039 ceph-mon[29586]: osdmap e213: 8 total, 8 up, 8 in 2024-09-14T03:56:48.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:47 smithi039 ceph-mon[40962]: pgmap v33: 137 pgs: 137 active+clean; 633 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 305 KiB/s rd, 4.0 MiB/s wr, 395 op/s 2024-09-14T03:56:48.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:47 smithi039 ceph-mon[40962]: osdmap e213: 8 total, 8 up, 8 in 2024-09-14T03:56:49.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:48 smithi129 ceph-mon[33173]: osdmap e214: 8 total, 8 up, 8 in 2024-09-14T03:56:49.032 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:48 smithi039 ceph-mon[29586]: osdmap e214: 8 total, 8 up, 8 in 2024-09-14T03:56:49.032 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:48 smithi039 ceph-mon[40962]: osdmap e214: 8 total, 8 up, 8 in 2024-09-14T03:56:49.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:49 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:56:49] "GET /metrics HTTP/1.1" 200 38869 "" "Prometheus/2.43.0" 2024-09-14T03:56:50.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:49 smithi129 ceph-mon[33173]: pgmap v36: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-09-14T03:56:50.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:49 smithi129 ceph-mon[33173]: osdmap e215: 8 total, 8 up, 8 in 2024-09-14T03:56:50.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:49 smithi039 ceph-mon[29586]: pgmap v36: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-09-14T03:56:50.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:49 smithi039 ceph-mon[29586]: osdmap e215: 8 total, 8 up, 8 in 2024-09-14T03:56:50.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:49 smithi039 ceph-mon[40962]: pgmap v36: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-09-14T03:56:50.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:49 smithi039 ceph-mon[40962]: osdmap e215: 8 total, 8 up, 8 in 2024-09-14T03:56:50.308 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: after_prepare_snap 2024-09-14T03:56:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:50 smithi129 ceph-mon[33173]: osdmap e216: 8 total, 8 up, 8 in 2024-09-14T03:56:51.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:50 smithi039 ceph-mon[29586]: osdmap e216: 8 total, 8 up, 8 in 2024-09-14T03:56:51.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:50 smithi039 ceph-mon[40962]: osdmap e216: 8 total, 8 up, 8 in 2024-09-14T03:56:51.771 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:51 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[80442]: [14/Sep/2024:03:56:51] ENGINE Bus STARTING 2024-09-14T03:56:51.771 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:51 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[80442]: CherryPy Checker: 2024-09-14T03:56:51.771 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:51 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[80442]: The Application mounted at '' has an empty config. 2024-09-14T03:56:51.771 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:51 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[80442]: 2024-09-14T03:56:51.771 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:51 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[80442]: [14/Sep/2024:03:56:51] ENGINE Serving on http://:::9283 2024-09-14T03:56:51.772 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:56:51 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[80442]: [14/Sep/2024:03:56:51] ENGINE Bus STARTED 2024-09-14T03:56:51.817 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 41943040 2024-09-14T03:56:51.850 INFO:tasks.workunit.client.0.smithi039.stdout:write: * 0~1000 2024-09-14T03:56:51.856 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:56:52.012 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:56:52.100 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:51 smithi129 ceph-mon[33173]: pgmap v39: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-09-14T03:56:52.100 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:51 smithi129 ceph-mon[33173]: osdmap e217: 8 total, 8 up, 8 in 2024-09-14T03:56:52.100 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:51 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:52.100 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:51 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:52.100 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:51 smithi129 ceph-mon[33173]: Standby manager daemon x restarted 2024-09-14T03:56:52.100 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:51 smithi129 ceph-mon[33173]: Standby manager daemon x started 2024-09-14T03:56:52.101 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:51 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.129:0/2804343739' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:56:52.101 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:51 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.129:0/2804343739' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:56:52.101 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:51 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.129:0/2804343739' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:56:52.101 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:51 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.129:0/2804343739' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:56:52.101 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:51 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:52.101 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:51 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:52.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:51 smithi039 ceph-mon[29586]: pgmap v39: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-09-14T03:56:52.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:51 smithi039 ceph-mon[29586]: osdmap e217: 8 total, 8 up, 8 in 2024-09-14T03:56:52.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:51 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:52.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:51 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:52.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:51 smithi039 ceph-mon[29586]: Standby manager daemon x restarted 2024-09-14T03:56:52.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:51 smithi039 ceph-mon[29586]: Standby manager daemon x started 2024-09-14T03:56:52.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:51 smithi039 ceph-mon[29586]: from='mgr.? 172.21.15.129:0/2804343739' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:56:52.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:51 smithi039 ceph-mon[29586]: from='mgr.? 172.21.15.129:0/2804343739' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:56:52.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:51 smithi039 ceph-mon[29586]: from='mgr.? 172.21.15.129:0/2804343739' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:56:52.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:51 smithi039 ceph-mon[29586]: from='mgr.? 172.21.15.129:0/2804343739' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:56:52.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:51 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:52.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:51 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:52.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:51 smithi039 ceph-mon[40962]: pgmap v39: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 140 KiB/s rd, 8.0 MiB/s wr, 201 op/s 2024-09-14T03:56:52.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:51 smithi039 ceph-mon[40962]: osdmap e217: 8 total, 8 up, 8 in 2024-09-14T03:56:52.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:51 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:52.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:51 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:52.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:51 smithi039 ceph-mon[40962]: Standby manager daemon x restarted 2024-09-14T03:56:52.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:51 smithi039 ceph-mon[40962]: Standby manager daemon x started 2024-09-14T03:56:52.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:51 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.129:0/2804343739' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:56:52.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:51 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.129:0/2804343739' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:56:52.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:51 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.129:0/2804343739' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:56:52.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:51 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.129:0/2804343739' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:56:52.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:51 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:52.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:51 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:52.837 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:52 smithi129 ceph-mon[33173]: osdmap e218: 8 total, 8 up, 8 in 2024-09-14T03:56:52.837 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:52 smithi129 ceph-mon[33173]: mgrmap e33: y(active, since 25s), standbys: x 2024-09-14T03:56:53.191 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:52 smithi039 ceph-mon[29586]: osdmap e218: 8 total, 8 up, 8 in 2024-09-14T03:56:53.192 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:52 smithi039 ceph-mon[29586]: mgrmap e33: y(active, since 25s), standbys: x 2024-09-14T03:56:53.192 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:52 smithi039 ceph-mon[40962]: osdmap e218: 8 total, 8 up, 8 in 2024-09-14T03:56:53.192 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:52 smithi039 ceph-mon[40962]: mgrmap e33: y(active, since 25s), standbys: x 2024-09-14T03:56:54.111 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:53 smithi129 ceph-mon[33173]: pgmap v42: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:56:54.111 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:53 smithi129 ceph-mon[33173]: osdmap e219: 8 total, 8 up, 8 in 2024-09-14T03:56:54.111 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:53 smithi129 ceph-mon[33173]: osdmap e220: 8 total, 8 up, 8 in 2024-09-14T03:56:54.209 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:53 smithi039 ceph-mon[29586]: pgmap v42: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:56:54.209 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:53 smithi039 ceph-mon[29586]: osdmap e219: 8 total, 8 up, 8 in 2024-09-14T03:56:54.209 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:53 smithi039 ceph-mon[29586]: osdmap e220: 8 total, 8 up, 8 in 2024-09-14T03:56:54.209 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:53 smithi039 ceph-mon[40962]: pgmap v42: 137 pgs: 137 active+clean; 673 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:56:54.210 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:53 smithi039 ceph-mon[40962]: osdmap e219: 8 total, 8 up, 8 in 2024-09-14T03:56:54.210 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:53 smithi039 ceph-mon[40962]: osdmap e220: 8 total, 8 up, 8 in 2024-09-14T03:56:55.092 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.SnapsSnapAfterPrepare (13922 ms) 2024-09-14T03:56:55.092 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.SnapsSnapExpandAfterPrepare 2024-09-14T03:56:55.127 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:56:55.139 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 0~8388609 2024-09-14T03:56:55.207 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap1 2024-09-14T03:56:55.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: pgmap v45: 137 pgs: 137 active+clean; 701 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 50 MiB/s rd, 6.0 MiB/s wr, 531 op/s 2024-09-14T03:56:55.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:55.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:55.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:56:55.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:56:55.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:55.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:56:55.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:55.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: Upgrade: Setting container_image for all mgr 2024-09-14T03:56:55.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:55.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-14T03:56:55.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-14T03:56:55.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-14T03:56:55.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-14T03:56:55.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-14T03:56:55.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-14T03:56:55.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:55.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: Upgrade: Setting container_image for all crash 2024-09-14T03:56:55.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:55.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:55.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: Upgrade: Setting container_image for all rgw 2024-09-14T03:56:55.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:55.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:55.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: Upgrade: Setting container_image for all rbd-mirror 2024-09-14T03:56:55.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:55.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:55.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-14T03:56:55.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:55.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:55.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: Upgrade: Setting container_image for all ceph-exporter 2024-09-14T03:56:55.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:55.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:55.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: Upgrade: Setting container_image for all iscsi 2024-09-14T03:56:55.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:55.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:55.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: Upgrade: Setting container_image for all nfs 2024-09-14T03:56:55.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:55.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:55.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: Upgrade: Setting container_image for all nvmeof 2024-09-14T03:56:55.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:55.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[29586]: osdmap e221: 8 total, 8 up, 8 in 2024-09-14T03:56:55.795 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: pgmap v45: 137 pgs: 137 active+clean; 701 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 50 MiB/s rd, 6.0 MiB/s wr, 531 op/s 2024-09-14T03:56:55.795 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:55.795 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:55.795 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:56:55.795 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:56:55.795 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:55.795 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:56:55.795 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:55.795 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: Upgrade: Setting container_image for all mgr 2024-09-14T03:56:55.795 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:55.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-14T03:56:55.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-14T03:56:55.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-14T03:56:55.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-14T03:56:55.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-14T03:56:55.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-14T03:56:55.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:55.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: Upgrade: Setting container_image for all crash 2024-09-14T03:56:55.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:55.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:55.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: Upgrade: Setting container_image for all rgw 2024-09-14T03:56:55.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:55.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:55.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: Upgrade: Setting container_image for all rbd-mirror 2024-09-14T03:56:55.797 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:55.797 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:55.797 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-14T03:56:55.797 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:55.797 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:55.797 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: Upgrade: Setting container_image for all ceph-exporter 2024-09-14T03:56:55.797 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:55.797 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:55.797 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: Upgrade: Setting container_image for all iscsi 2024-09-14T03:56:55.797 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:55.797 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:55.797 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: Upgrade: Setting container_image for all nfs 2024-09-14T03:56:55.797 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:55.797 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:55.797 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: Upgrade: Setting container_image for all nvmeof 2024-09-14T03:56:55.798 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:55.798 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:55 smithi039 ceph-mon[40962]: osdmap e221: 8 total, 8 up, 8 in 2024-09-14T03:56:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: pgmap v45: 137 pgs: 137 active+clean; 701 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 50 MiB/s rd, 6.0 MiB/s wr, 531 op/s 2024-09-14T03:56:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:56:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:56:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:56:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: Upgrade: Setting container_image for all mgr 2024-09-14T03:56:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-14T03:56:56.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-09-14T03:56:56.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-09-14T03:56:56.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-14T03:56:56.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-09-14T03:56:56.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-09-14T03:56:56.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:56.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: Upgrade: Setting container_image for all crash 2024-09-14T03:56:56.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:56.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:56.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: Upgrade: Setting container_image for all rgw 2024-09-14T03:56:56.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:56.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:56.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: Upgrade: Setting container_image for all rbd-mirror 2024-09-14T03:56:56.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:56.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:56.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-14T03:56:56.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:56.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:56.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: Upgrade: Setting container_image for all ceph-exporter 2024-09-14T03:56:56.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:56.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:56.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: Upgrade: Setting container_image for all iscsi 2024-09-14T03:56:56.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:56.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:56.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: Upgrade: Setting container_image for all nfs 2024-09-14T03:56:56.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:56.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:56:56.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: Upgrade: Setting container_image for all nvmeof 2024-09-14T03:56:56.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:56:56.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:55 smithi129 ceph-mon[33173]: osdmap e221: 8 total, 8 up, 8 in 2024-09-14T03:56:56.924 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 8388609~8388609 2024-09-14T03:56:56.944 INFO:tasks.workunit.client.0.smithi039.stdout:write: X 0~1000 2024-09-14T03:56:56.945 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 1010~1000 2024-09-14T03:56:56.945 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap2 2024-09-14T03:56:57.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:56 smithi129 ceph-mon[33173]: osdmap e222: 8 total, 8 up, 8 in 2024-09-14T03:56:57.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:56 smithi039 ceph-mon[29586]: osdmap e222: 8 total, 8 up, 8 in 2024-09-14T03:56:57.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:56 smithi039 ceph-mon[40962]: osdmap e222: 8 total, 8 up, 8 in 2024-09-14T03:56:57.291 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:56:57 smithi039 systemd[1]: Stopping Ceph node-exporter.a for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:56:57.666 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:56:57 smithi039 podman[97216]: 2024-09-14 03:56:57.351808909 +0000 UTC m=+0.134234880 container died 4eff582f78f5762c8bf4b28b8523961d9b934c67c20840fcd7a9870683906fc9 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:56:58.040 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:56:57 smithi039 podman[97216]: 2024-09-14 03:56:57.889425961 +0000 UTC m=+0.671851934 container cleanup 4eff582f78f5762c8bf4b28b8523961d9b934c67c20840fcd7a9870683906fc9 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:56:58.040 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:56:57 smithi039 bash[97216]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a 2024-09-14T03:56:58.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:57 smithi039 ceph-mon[29586]: pgmap v48: 137 pgs: 137 active+clean; 701 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 50 MiB/s rd, 6.0 MiB/s wr, 531 op/s 2024-09-14T03:56:58.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:57 smithi039 ceph-mon[29586]: Upgrade: Updating node-exporter.a (1/2) 2024-09-14T03:56:58.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:57 smithi039 ceph-mon[29586]: Deploying daemon node-exporter.a on smithi039 2024-09-14T03:56:58.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:57 smithi039 ceph-mon[29586]: osdmap e223: 8 total, 8 up, 8 in 2024-09-14T03:56:58.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:57 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:56:58.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:57 smithi039 ceph-mon[40962]: pgmap v48: 137 pgs: 137 active+clean; 701 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 50 MiB/s rd, 6.0 MiB/s wr, 531 op/s 2024-09-14T03:56:58.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:57 smithi039 ceph-mon[40962]: Upgrade: Updating node-exporter.a (1/2) 2024-09-14T03:56:58.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:57 smithi039 ceph-mon[40962]: Deploying daemon node-exporter.a on smithi039 2024-09-14T03:56:58.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:57 smithi039 ceph-mon[40962]: osdmap e223: 8 total, 8 up, 8 in 2024-09-14T03:56:58.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:57 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:56:58.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:57 smithi129 ceph-mon[33173]: pgmap v48: 137 pgs: 137 active+clean; 701 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 50 MiB/s rd, 6.0 MiB/s wr, 531 op/s 2024-09-14T03:56:58.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:57 smithi129 ceph-mon[33173]: Upgrade: Updating node-exporter.a (1/2) 2024-09-14T03:56:58.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:57 smithi129 ceph-mon[33173]: Deploying daemon node-exporter.a on smithi039 2024-09-14T03:56:58.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:57 smithi129 ceph-mon[33173]: osdmap e223: 8 total, 8 up, 8 in 2024-09-14T03:56:58.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:57 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:56:58.790 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:56:58 smithi039 podman[97232]: 2024-09-14 03:56:58.433198069 +0000 UTC m=+1.084623744 container remove 4eff582f78f5762c8bf4b28b8523961d9b934c67c20840fcd7a9870683906fc9 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:56:58.790 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:56:58 smithi039 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-14T03:56:58.790 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:56:58 smithi039 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-14T03:56:58.790 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:56:58 smithi039 systemd[1]: Stopped Ceph node-exporter.a for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:56:58.790 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:56:58 smithi039 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@node-exporter.a.service: Consumed 2.472s CPU time. 2024-09-14T03:56:58.949 INFO:tasks.workunit.client.0.smithi039.stdout:write: X 8388609~1000 2024-09-14T03:56:58.956 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 16777228~1000 2024-09-14T03:56:58.956 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 167772160 2024-09-14T03:56:58.966 INFO:tasks.workunit.client.0.smithi039.stdout:write: 2 83886079~8388609 2024-09-14T03:56:58.984 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap3 2024-09-14T03:56:59.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:58 smithi129 ceph-mon[33173]: osdmap e224: 8 total, 8 up, 8 in 2024-09-14T03:56:59.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:58 smithi039 ceph-mon[29586]: osdmap e224: 8 total, 8 up, 8 in 2024-09-14T03:56:59.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:58 smithi039 ceph-mon[40962]: osdmap e224: 8 total, 8 up, 8 in 2024-09-14T03:56:59.290 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:56:58 smithi039 systemd[1]: Starting Ceph node-exporter.a for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:56:59.291 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:56:59 smithi039 bash[97367]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-09-14T03:56:59.291 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:56:59 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:56:59] "GET /metrics HTTP/1.1" 200 38865 "" "Prometheus/2.43.0" 2024-09-14T03:56:59.790 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:56:59 smithi039 bash[97367]: Getting image source signatures 2024-09-14T03:56:59.790 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:56:59 smithi039 bash[97367]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-09-14T03:56:59.790 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:56:59 smithi039 bash[97367]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-09-14T03:56:59.790 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:56:59 smithi039 bash[97367]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-09-14T03:57:00.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:59 smithi129 ceph-mon[33173]: pgmap v51: 137 pgs: 137 active+clean; 705 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 40 MiB/s rd, 8.0 MiB/s wr, 151 op/s 2024-09-14T03:57:00.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:56:59 smithi129 ceph-mon[33173]: osdmap e225: 8 total, 8 up, 8 in 2024-09-14T03:57:00.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:59 smithi039 ceph-mon[29586]: pgmap v51: 137 pgs: 137 active+clean; 705 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 40 MiB/s rd, 8.0 MiB/s wr, 151 op/s 2024-09-14T03:57:00.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:56:59 smithi039 ceph-mon[29586]: osdmap e225: 8 total, 8 up, 8 in 2024-09-14T03:57:00.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:59 smithi039 ceph-mon[40962]: pgmap v51: 137 pgs: 137 active+clean; 705 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 40 MiB/s rd, 8.0 MiB/s wr, 151 op/s 2024-09-14T03:57:00.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:56:59 smithi039 ceph-mon[40962]: osdmap e225: 8 total, 8 up, 8 in 2024-09-14T03:57:00.966 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:57:00.987 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 83886079~1 2024-09-14T03:57:00.994 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:57:01.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:00 smithi039 ceph-mon[29586]: osdmap e226: 8 total, 8 up, 8 in 2024-09-14T03:57:01.040 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:00 smithi039 bash[97367]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-09-14T03:57:01.040 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:00 smithi039 bash[97367]: Writing manifest to image destination 2024-09-14T03:57:01.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:00 smithi039 ceph-mon[40962]: osdmap e226: 8 total, 8 up, 8 in 2024-09-14T03:57:01.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:00 smithi129 ceph-mon[33173]: osdmap e226: 8 total, 8 up, 8 in 2024-09-14T03:57:01.528 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 podman[97367]: 2024-09-14 03:57:01.161923458 +0000 UTC m=+2.181057853 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-09-14T03:57:01.528 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 podman[97367]: 2024-09-14 03:57:01.251768176 +0000 UTC m=+2.270902554 container create 7dd5945adb64a31c2a4dcf8e4d7310dbb46275db071b3365f12944e05932726d (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:57:01.791 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 podman[97367]: 2024-09-14 03:57:01.527112171 +0000 UTC m=+2.546246559 container init 7dd5945adb64a31c2a4dcf8e4d7310dbb46275db071b3365f12944e05932726d (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:57:01.791 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 podman[97367]: 2024-09-14 03:57:01.532693725 +0000 UTC m=+2.551828113 container start 7dd5945adb64a31c2a4dcf8e4d7310dbb46275db071b3365f12944e05932726d (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T03:57:01.791 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.536Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-09-14T03:57:01.791 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.536Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-09-14T03:57:01.791 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-14T03:57:01.791 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-14T03:57:01.791 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-14T03:57:01.791 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-14T03:57:01.792 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-14T03:57:01.792 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=arp 2024-09-14T03:57:01.792 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-14T03:57:01.792 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-14T03:57:01.792 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-14T03:57:01.792 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-14T03:57:01.792 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-14T03:57:01.792 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-14T03:57:01.792 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-14T03:57:01.792 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-14T03:57:01.792 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=edac 2024-09-14T03:57:01.793 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-14T03:57:01.793 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-14T03:57:01.793 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-14T03:57:01.793 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-14T03:57:01.793 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-14T03:57:01.793 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-14T03:57:01.793 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-14T03:57:01.793 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-14T03:57:01.793 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-14T03:57:01.793 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-14T03:57:01.793 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-14T03:57:01.793 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-14T03:57:01.793 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-14T03:57:01.793 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-14T03:57:01.793 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-14T03:57:01.794 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-14T03:57:01.794 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=os 2024-09-14T03:57:01.794 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-14T03:57:01.794 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-14T03:57:01.794 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-14T03:57:01.794 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-14T03:57:01.794 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-14T03:57:01.794 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-14T03:57:01.794 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-14T03:57:01.794 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=stat 2024-09-14T03:57:01.795 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-14T03:57:01.795 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-14T03:57:01.795 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-14T03:57:01.795 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=time 2024-09-14T03:57:01.795 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-14T03:57:01.795 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=uname 2024-09-14T03:57:01.795 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-14T03:57:01.795 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-14T03:57:01.795 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.537Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-14T03:57:01.795 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.538Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-09-14T03:57:01.795 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a[97444]: ts=2024-09-14T03:57:01.538Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-14T03:57:01.795 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 bash[97367]: 7dd5945adb64a31c2a4dcf8e4d7310dbb46275db071b3365f12944e05932726d 2024-09-14T03:57:01.795 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 03:57:01 smithi039 systemd[1]: Started Ceph node-exporter.a for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:57:02.242 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:01 smithi129 ceph-mon[33173]: pgmap v54: 137 pgs: 137 active+clean; 705 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 40 MiB/s rd, 8.0 MiB/s wr, 151 op/s 2024-09-14T03:57:02.243 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:01 smithi129 ceph-mon[33173]: osdmap e227: 8 total, 8 up, 8 in 2024-09-14T03:57:02.243 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:01 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:02.243 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:01 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:02.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-mon[29586]: pgmap v54: 137 pgs: 137 active+clean; 705 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 40 MiB/s rd, 8.0 MiB/s wr, 151 op/s 2024-09-14T03:57:02.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-mon[29586]: osdmap e227: 8 total, 8 up, 8 in 2024-09-14T03:57:02.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:02.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:02.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-mon[40962]: pgmap v54: 137 pgs: 137 active+clean; 705 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 40 MiB/s rd, 8.0 MiB/s wr, 151 op/s 2024-09-14T03:57:02.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-mon[40962]: osdmap e227: 8 total, 8 up, 8 in 2024-09-14T03:57:02.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:02.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:01 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:03.045 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:02 smithi129 ceph-mon[33173]: osdmap e228: 8 total, 8 up, 8 in 2024-09-14T03:57:03.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:02 smithi039 ceph-mon[29586]: osdmap e228: 8 total, 8 up, 8 in 2024-09-14T03:57:03.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:02 smithi039 ceph-mon[40962]: osdmap e228: 8 total, 8 up, 8 in 2024-09-14T03:57:03.771 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:03 smithi129 systemd[1]: Stopping Ceph node-exporter.b for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:57:04.117 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:03 smithi129 podman[84052]: 2024-09-14 03:57:03.836755841 +0000 UTC m=+0.129534588 container died c7581baf44f218f222deac9bf0be7fceefc52d2f46e419a580d63c8dc7d22ebd (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:57:04.117 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:03 smithi129 ceph-mon[33173]: pgmap v57: 137 pgs: 137 active+clean; 705 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:57:04.117 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:03 smithi129 ceph-mon[33173]: osdmap e229: 8 total, 8 up, 8 in 2024-09-14T03:57:04.117 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:03 smithi129 ceph-mon[33173]: Upgrade: Updating node-exporter.b (2/2) 2024-09-14T03:57:04.118 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:03 smithi129 ceph-mon[33173]: Deploying daemon node-exporter.b on smithi129 2024-09-14T03:57:04.118 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:03 smithi129 ceph-mon[33173]: osdmap e230: 8 total, 8 up, 8 in 2024-09-14T03:57:04.165 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: after_prepare_snap 2024-09-14T03:57:04.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:03 smithi039 ceph-mon[29586]: pgmap v57: 137 pgs: 137 active+clean; 705 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:57:04.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:03 smithi039 ceph-mon[29586]: osdmap e229: 8 total, 8 up, 8 in 2024-09-14T03:57:04.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:03 smithi039 ceph-mon[29586]: Upgrade: Updating node-exporter.b (2/2) 2024-09-14T03:57:04.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:03 smithi039 ceph-mon[29586]: Deploying daemon node-exporter.b on smithi129 2024-09-14T03:57:04.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:03 smithi039 ceph-mon[29586]: osdmap e230: 8 total, 8 up, 8 in 2024-09-14T03:57:04.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:03 smithi039 ceph-mon[40962]: pgmap v57: 137 pgs: 137 active+clean; 705 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:57:04.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:03 smithi039 ceph-mon[40962]: osdmap e229: 8 total, 8 up, 8 in 2024-09-14T03:57:04.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:03 smithi039 ceph-mon[40962]: Upgrade: Updating node-exporter.b (2/2) 2024-09-14T03:57:04.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:03 smithi039 ceph-mon[40962]: Deploying daemon node-exporter.b on smithi129 2024-09-14T03:57:04.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:03 smithi039 ceph-mon[40962]: osdmap e230: 8 total, 8 up, 8 in 2024-09-14T03:57:04.409 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:04 smithi129 podman[84052]: 2024-09-14 03:57:04.216123143 +0000 UTC m=+0.508901887 container cleanup c7581baf44f218f222deac9bf0be7fceefc52d2f46e419a580d63c8dc7d22ebd (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:57:04.409 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:04 smithi129 bash[84052]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b 2024-09-14T03:57:04.771 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:04 smithi129 podman[84069]: 2024-09-14 03:57:04.408646887 +0000 UTC m=+0.574447998 container remove c7581baf44f218f222deac9bf0be7fceefc52d2f46e419a580d63c8dc7d22ebd (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:57:04.771 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:04 smithi129 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-14T03:57:04.771 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:04 smithi129 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-14T03:57:04.771 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:04 smithi129 systemd[1]: Stopped Ceph node-exporter.b for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:57:04.771 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:04 smithi129 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@node-exporter.b.service: Consumed 2.320s CPU time. 2024-09-14T03:57:05.271 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:04 smithi129 systemd[1]: Starting Ceph node-exporter.b for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:57:05.271 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:04 smithi129 bash[84202]: Trying to pull quay.io/prometheus/node-exporter:v1.7.0... 2024-09-14T03:57:05.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:04 smithi129 ceph-mon[33173]: pgmap v60: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-09-14T03:57:05.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:04 smithi039 ceph-mon[29586]: pgmap v60: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-09-14T03:57:05.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:04 smithi039 ceph-mon[40962]: pgmap v60: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-09-14T03:57:05.771 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:05 smithi129 bash[84202]: Getting image source signatures 2024-09-14T03:57:05.771 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:05 smithi129 bash[84202]: Copying blob sha256:324153f2810a9927fcce320af9e4e291e0b6e805cbdd1f338386c756b9defa24 2024-09-14T03:57:05.771 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:05 smithi129 bash[84202]: Copying blob sha256:2abcce694348cd2c949c0e98a7400ebdfd8341021bcf6b541bc72033ce982510 2024-09-14T03:57:05.771 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:05 smithi129 bash[84202]: Copying blob sha256:455fd88e5221bc1e278ef2d059cd70e4df99a24e5af050ede621534276f6cf9a 2024-09-14T03:57:06.034 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 167772160 2024-09-14T03:57:06.048 INFO:tasks.workunit.client.0.smithi039.stdout:write: * 83886080~1000 2024-09-14T03:57:06.056 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:57:06.220 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:57:06.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:05 smithi129 ceph-mon[33173]: osdmap e231: 8 total, 8 up, 8 in 2024-09-14T03:57:06.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:05 smithi039 ceph-mon[29586]: osdmap e231: 8 total, 8 up, 8 in 2024-09-14T03:57:06.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:06 smithi039 ceph-mon[40962]: osdmap e231: 8 total, 8 up, 8 in 2024-09-14T03:57:06.923 INFO:teuthology.orchestra.run.smithi039.stdout:true 2024-09-14T03:57:07.021 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:06 smithi129 bash[84202]: Copying config sha256:72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e 2024-09-14T03:57:07.021 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:06 smithi129 bash[84202]: Writing manifest to image destination 2024-09-14T03:57:07.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:07 smithi039 ceph-mon[40962]: osdmap e232: 8 total, 8 up, 8 in 2024-09-14T03:57:07.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:07 smithi039 ceph-mon[40962]: pgmap v63: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-09-14T03:57:07.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:07 smithi039 ceph-mon[40962]: from='client.25022 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:07.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:07 smithi039 ceph-mon[29586]: osdmap e232: 8 total, 8 up, 8 in 2024-09-14T03:57:07.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:07 smithi039 ceph-mon[29586]: pgmap v63: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-09-14T03:57:07.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:07 smithi039 ceph-mon[29586]: from='client.25022 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:07.430 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-mon[33173]: osdmap e232: 8 total, 8 up, 8 in 2024-09-14T03:57:07.430 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-mon[33173]: pgmap v63: 137 pgs: 137 active+clean; 729 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 4.0 MiB/s wr, 170 op/s 2024-09-14T03:57:07.430 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-mon[33173]: from='client.25022 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:07.514 INFO:teuthology.orchestra.run.smithi039.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:57:07.514 INFO:teuthology.orchestra.run.smithi039.stdout:alertmanager.a smithi039 *:9093,9094 running (4m) 36s ago 5m 28.6M - 0.25.0 c8568f914cd2 b5ca6b93d9ee 2024-09-14T03:57:07.514 INFO:teuthology.orchestra.run.smithi039.stdout:grafana.a smithi129 *:3000 running (4m) 16s ago 4m 89.6M - 9.4.7 954c08fa6188 71f297945590 2024-09-14T03:57:07.514 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi039.bbwiwf smithi039 running (3m) 36s ago 3m 19.0M - 18.2.4-845-ge4e227b3 67ddd33fa72f 91e6c8f85681 2024-09-14T03:57:07.514 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi129.ybjqcw smithi129 running (3m) 16s ago 3m 14.6M - 18.2.4-845-ge4e227b3 67ddd33fa72f a3fab5b37b71 2024-09-14T03:57:07.514 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.x smithi129 *:8443,9283,8765 running (21s) 16s ago 9m 67.1M - 19.3.0-4907-ga2782315 8801d2529e43 448bd4e146cb 2024-09-14T03:57:07.514 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.y smithi039 *:8443,9283,8765 running (49s) 36s ago 10m 490M - 19.3.0-4907-ga2782315 8801d2529e43 cccc87f635a6 2024-09-14T03:57:07.514 INFO:teuthology.orchestra.run.smithi039.stdout:mon.a smithi039 running (10m) 36s ago 10m 57.2M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f e7b5f91fb09c 2024-09-14T03:57:07.514 INFO:teuthology.orchestra.run.smithi039.stdout:mon.b smithi129 running (9m) 16s ago 9m 48.0M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f eb38cc5cbe35 2024-09-14T03:57:07.515 INFO:teuthology.orchestra.run.smithi039.stdout:mon.c smithi039 running (9m) 36s ago 9m 47.0M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 31d440e49f93 2024-09-14T03:57:07.515 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.a smithi039 *:9100 starting - - - - 2024-09-14T03:57:07.515 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.b smithi129 *:9100 running (4m) 16s ago 4m 15.4M - 1.5.0 0da6a335fe13 c7581baf44f2 2024-09-14T03:57:07.515 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0 smithi039 running (8m) 36s ago 8m 109M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 5048d74f27ac 2024-09-14T03:57:07.515 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1 smithi039 running (8m) 36s ago 8m 111M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 91349527d88b 2024-09-14T03:57:07.515 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2 smithi039 running (7m) 36s ago 7m 90.5M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f eb903b44fc5d 2024-09-14T03:57:07.515 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3 smithi039 running (7m) 36s ago 7m 107M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f dff9e665bd9b 2024-09-14T03:57:07.515 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4 smithi129 running (7m) 16s ago 7m 99.1M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 5b64727311b7 2024-09-14T03:57:07.515 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5 smithi129 running (6m) 16s ago 6m 88.5M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f f2d991a754f7 2024-09-14T03:57:07.515 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6 smithi129 running (6m) 16s ago 6m 127M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 3060eaf44106 2024-09-14T03:57:07.515 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7 smithi129 running (6m) 16s ago 6m 88.6M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 4c13b97060d1 2024-09-14T03:57:07.515 INFO:teuthology.orchestra.run.smithi039.stdout:prometheus.a smithi129 *:9095 running (26s) 16s ago 5m 35.7M - 2.43.0 a07b618ecd1d e8a320b5858c 2024-09-14T03:57:07.704 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 podman[84202]: 2024-09-14 03:57:07.345012225 +0000 UTC m=+2.442346476 image pull 72c9c208898624938c9e4183d6686ea4a5fd3f912bc29bc3f00147924c521a3e quay.io/prometheus/node-exporter:v1.7.0 2024-09-14T03:57:07.704 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 podman[84202]: 2024-09-14 03:57:07.459820338 +0000 UTC m=+2.557177480 container create 1fedb5f257e76490a74a8eca1ed1245a9a7c00a3435a716ec14dbfe46bd2a2ad (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:57:07.704 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 podman[84202]: 2024-09-14 03:57:07.703830212 +0000 UTC m=+2.801164459 container init 1fedb5f257e76490a74a8eca1ed1245a9a7c00a3435a716ec14dbfe46bd2a2ad (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:57:07.864 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T03:57:07.864 INFO:teuthology.orchestra.run.smithi039.stdout: "mon": { 2024-09-14T03:57:07.864 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 3 2024-09-14T03:57:07.864 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:57:07.864 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr": { 2024-09-14T03:57:07.864 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:57:07.864 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:57:07.864 INFO:teuthology.orchestra.run.smithi039.stdout: "osd": { 2024-09-14T03:57:07.865 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8 2024-09-14T03:57:07.865 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:57:07.865 INFO:teuthology.orchestra.run.smithi039.stdout: "mds": { 2024-09-14T03:57:07.865 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:57:07.865 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:57:07.865 INFO:teuthology.orchestra.run.smithi039.stdout: "overall": { 2024-09-14T03:57:07.865 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 13, 2024-09-14T03:57:07.865 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:57:07.865 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T03:57:07.865 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T03:57:08.022 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 podman[84202]: 2024-09-14 03:57:07.709660144 +0000 UTC m=+2.806994392 container start 1fedb5f257e76490a74a8eca1ed1245a9a7c00a3435a716ec14dbfe46bd2a2ad (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T03:57:08.022 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.713Z caller=node_exporter.go:192 level=info msg="Starting node_exporter" version="(version=1.7.0, branch=HEAD, revision=7333465abf9efba81876303bb57e6fadb946041b)" 2024-09-14T03:57:08.022 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.713Z caller=node_exporter.go:193 level=info msg="Build context" build_context="(go=go1.21.4, platform=linux/amd64, user=root@35918982f6d8, date=20231112-23:53:35, tags=netgo osusergo static_build)" 2024-09-14T03:57:08.022 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.713Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-09-14T03:57:08.022 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.714Z caller=diskstats_linux.go:265 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-09-14T03:57:08.022 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.714Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-09-14T03:57:08.022 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.714Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-09-14T03:57:08.022 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.714Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-09-14T03:57:08.022 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.714Z caller=node_exporter.go:117 level=info collector=arp 2024-09-14T03:57:08.023 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.714Z caller=node_exporter.go:117 level=info collector=bcache 2024-09-14T03:57:08.023 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.714Z caller=node_exporter.go:117 level=info collector=bonding 2024-09-14T03:57:08.023 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.714Z caller=node_exporter.go:117 level=info collector=btrfs 2024-09-14T03:57:08.023 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.714Z caller=node_exporter.go:117 level=info collector=conntrack 2024-09-14T03:57:08.023 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.714Z caller=node_exporter.go:117 level=info collector=cpu 2024-09-14T03:57:08.023 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.714Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-09-14T03:57:08.023 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.714Z caller=node_exporter.go:117 level=info collector=diskstats 2024-09-14T03:57:08.023 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.714Z caller=node_exporter.go:117 level=info collector=dmi 2024-09-14T03:57:08.024 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.714Z caller=node_exporter.go:117 level=info collector=edac 2024-09-14T03:57:08.024 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.714Z caller=node_exporter.go:117 level=info collector=entropy 2024-09-14T03:57:08.024 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.714Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-09-14T03:57:08.024 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.714Z caller=node_exporter.go:117 level=info collector=filefd 2024-09-14T03:57:08.024 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.714Z caller=node_exporter.go:117 level=info collector=filesystem 2024-09-14T03:57:08.024 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.714Z caller=node_exporter.go:117 level=info collector=hwmon 2024-09-14T03:57:08.024 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.714Z caller=node_exporter.go:117 level=info collector=infiniband 2024-09-14T03:57:08.024 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.714Z caller=node_exporter.go:117 level=info collector=ipvs 2024-09-14T03:57:08.024 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.715Z caller=node_exporter.go:117 level=info collector=loadavg 2024-09-14T03:57:08.024 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.715Z caller=node_exporter.go:117 level=info collector=mdadm 2024-09-14T03:57:08.024 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.715Z caller=node_exporter.go:117 level=info collector=meminfo 2024-09-14T03:57:08.025 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.715Z caller=node_exporter.go:117 level=info collector=netclass 2024-09-14T03:57:08.025 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.715Z caller=node_exporter.go:117 level=info collector=netdev 2024-09-14T03:57:08.025 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.715Z caller=node_exporter.go:117 level=info collector=netstat 2024-09-14T03:57:08.025 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.715Z caller=node_exporter.go:117 level=info collector=nfs 2024-09-14T03:57:08.025 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.715Z caller=node_exporter.go:117 level=info collector=nfsd 2024-09-14T03:57:08.025 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.715Z caller=node_exporter.go:117 level=info collector=nvme 2024-09-14T03:57:08.025 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.715Z caller=node_exporter.go:117 level=info collector=os 2024-09-14T03:57:08.025 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.715Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-09-14T03:57:08.025 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.715Z caller=node_exporter.go:117 level=info collector=pressure 2024-09-14T03:57:08.025 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.715Z caller=node_exporter.go:117 level=info collector=rapl 2024-09-14T03:57:08.025 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.715Z caller=node_exporter.go:117 level=info collector=schedstat 2024-09-14T03:57:08.025 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.715Z caller=node_exporter.go:117 level=info collector=selinux 2024-09-14T03:57:08.026 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.715Z caller=node_exporter.go:117 level=info collector=sockstat 2024-09-14T03:57:08.026 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.715Z caller=node_exporter.go:117 level=info collector=softnet 2024-09-14T03:57:08.026 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.715Z caller=node_exporter.go:117 level=info collector=stat 2024-09-14T03:57:08.026 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.715Z caller=node_exporter.go:117 level=info collector=tapestats 2024-09-14T03:57:08.026 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.715Z caller=node_exporter.go:117 level=info collector=textfile 2024-09-14T03:57:08.026 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.715Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-09-14T03:57:08.026 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.715Z caller=node_exporter.go:117 level=info collector=time 2024-09-14T03:57:08.026 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.715Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-09-14T03:57:08.026 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.715Z caller=node_exporter.go:117 level=info collector=uname 2024-09-14T03:57:08.026 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.715Z caller=node_exporter.go:117 level=info collector=vmstat 2024-09-14T03:57:08.026 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.715Z caller=node_exporter.go:117 level=info collector=xfs 2024-09-14T03:57:08.026 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.715Z caller=node_exporter.go:117 level=info collector=zfs 2024-09-14T03:57:08.026 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.715Z caller=tls_config.go:274 level=info msg="Listening on" address=[::]:9100 2024-09-14T03:57:08.026 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b[84275]: ts=2024-09-14T03:57:07.715Z caller=tls_config.go:277 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-09-14T03:57:08.027 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 bash[84202]: 1fedb5f257e76490a74a8eca1ed1245a9a7c00a3435a716ec14dbfe46bd2a2ad 2024-09-14T03:57:08.027 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 03:57:07 smithi129 systemd[1]: Started Ceph node-exporter.b for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:57:08.185 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T03:57:08.185 INFO:teuthology.orchestra.run.smithi039.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:57:08.185 INFO:teuthology.orchestra.run.smithi039.stdout: "in_progress": true, 2024-09-14T03:57:08.185 INFO:teuthology.orchestra.run.smithi039.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-14T03:57:08.185 INFO:teuthology.orchestra.run.smithi039.stdout: "services_complete": [ 2024-09-14T03:57:08.185 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr" 2024-09-14T03:57:08.185 INFO:teuthology.orchestra.run.smithi039.stdout: ], 2024-09-14T03:57:08.185 INFO:teuthology.orchestra.run.smithi039.stdout: "progress": "2/2 daemons upgraded", 2024-09-14T03:57:08.186 INFO:teuthology.orchestra.run.smithi039.stdout: "message": "Currently upgrading node-exporter daemons", 2024-09-14T03:57:08.186 INFO:teuthology.orchestra.run.smithi039.stdout: "is_paused": false 2024-09-14T03:57:08.186 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T03:57:08.186 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:08 smithi039 ceph-mon[29586]: osdmap e233: 8 total, 8 up, 8 in 2024-09-14T03:57:08.186 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:08 smithi039 ceph-mon[29586]: from='client.15126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:08.186 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:08 smithi039 ceph-mon[29586]: from='client.25034 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:08.186 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:08 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:08.186 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:08 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:08.186 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:08 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/4259271184' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:08.186 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:08 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:57:08.187 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:08 smithi039 ceph-mon[40962]: osdmap e233: 8 total, 8 up, 8 in 2024-09-14T03:57:08.187 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:08 smithi039 ceph-mon[40962]: from='client.15126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:08.187 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:08 smithi039 ceph-mon[40962]: from='client.25034 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:08.187 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:08 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:08.187 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:08 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:08.187 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:08 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/4259271184' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:08.187 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:08 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:57:08.448 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:08 smithi129 ceph-mon[33173]: osdmap e233: 8 total, 8 up, 8 in 2024-09-14T03:57:08.448 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:08 smithi129 ceph-mon[33173]: from='client.15126 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:08.448 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:08 smithi129 ceph-mon[33173]: from='client.25034 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:08.448 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:08 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:08.448 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:08 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:08.448 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:08 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/4259271184' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:08.449 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:08 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:57:09.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:57:09 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:57:09] "GET /metrics HTTP/1.1" 200 38865 "" "Prometheus/2.43.0" 2024-09-14T03:57:09.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:09 smithi039 ceph-mon[29586]: osdmap e234: 8 total, 8 up, 8 in 2024-09-14T03:57:09.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:09 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:09.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:09 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:09.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:09 smithi039 ceph-mon[29586]: from='client.15144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:09.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:09 smithi039 ceph-mon[29586]: pgmap v66: 137 pgs: 137 active+clean; 749 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 491 op/s 2024-09-14T03:57:09.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:09 smithi039 ceph-mon[40962]: osdmap e234: 8 total, 8 up, 8 in 2024-09-14T03:57:09.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:09 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:09.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:09 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:09.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:09 smithi039 ceph-mon[40962]: from='client.15144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:09.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:09 smithi039 ceph-mon[40962]: pgmap v66: 137 pgs: 137 active+clean; 749 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 491 op/s 2024-09-14T03:57:09.347 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:09 smithi129 ceph-mon[33173]: osdmap e234: 8 total, 8 up, 8 in 2024-09-14T03:57:09.347 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:09 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:09.347 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:09 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:09.347 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:09 smithi129 ceph-mon[33173]: from='client.15144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:09.348 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:09 smithi129 ceph-mon[33173]: pgmap v66: 137 pgs: 137 active+clean; 749 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 491 op/s 2024-09-14T03:57:09.457 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.SnapsSnapExpandAfterPrepare (14365 ms) 2024-09-14T03:57:09.458 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.Clone 2024-09-14T03:57:09.486 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:57:09.497 INFO:tasks.workunit.client.0.smithi039.stdout:write: X 0~8388609 2024-09-14T03:57:09.552 INFO:tasks.workunit.client.0.smithi039.stdout:write: X 16777218~8388609 2024-09-14T03:57:09.610 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap 2024-09-14T03:57:10.443 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:10 smithi129 ceph-mon[33173]: osdmap e235: 8 total, 8 up, 8 in 2024-09-14T03:57:10.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:10 smithi039 ceph-mon[29586]: osdmap e235: 8 total, 8 up, 8 in 2024-09-14T03:57:10.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:10 smithi039 ceph-mon[40962]: osdmap e235: 8 total, 8 up, 8 in 2024-09-14T03:57:11.194 INFO:tasks.workunit.client.0.smithi039.stdout:snap_protect: snap 2024-09-14T03:57:11.205 INFO:tasks.workunit.client.0.smithi039.stdout:clone image105 -> image108 2024-09-14T03:57:11.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:11 smithi129 ceph-mon[33173]: osdmap e236: 8 total, 8 up, 8 in 2024-09-14T03:57:11.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:11 smithi129 ceph-mon[33173]: pgmap v69: 137 pgs: 137 active+clean; 749 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 491 op/s 2024-09-14T03:57:11.315 INFO:tasks.workunit.client.0.smithi039.stdout:write: X 0~1000 2024-09-14T03:57:11.371 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 1010~1000 2024-09-14T03:57:11.373 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap 2024-09-14T03:57:11.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:11 smithi039 ceph-mon[29586]: osdmap e236: 8 total, 8 up, 8 in 2024-09-14T03:57:11.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:11 smithi039 ceph-mon[29586]: pgmap v69: 137 pgs: 137 active+clean; 749 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 491 op/s 2024-09-14T03:57:11.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:11 smithi039 ceph-mon[40962]: osdmap e236: 8 total, 8 up, 8 in 2024-09-14T03:57:11.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:11 smithi039 ceph-mon[40962]: pgmap v69: 137 pgs: 137 active+clean; 749 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 38 MiB/s rd, 6.0 MiB/s wr, 491 op/s 2024-09-14T03:57:12.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:12 smithi129 ceph-mon[33173]: osdmap e237: 8 total, 8 up, 8 in 2024-09-14T03:57:12.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:12 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:12.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:12 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:12.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:12 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:57:12.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:12 smithi039 ceph-mon[40962]: osdmap e237: 8 total, 8 up, 8 in 2024-09-14T03:57:12.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:12 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:12.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:12 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:12.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:12 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:57:12.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:12 smithi039 ceph-mon[29586]: osdmap e237: 8 total, 8 up, 8 in 2024-09-14T03:57:12.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:12 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:12.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:12 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:12.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:12 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:57:13.225 INFO:tasks.workunit.client.0.smithi039.stdout:snap_protect: snap 2024-09-14T03:57:13.236 INFO:tasks.workunit.client.0.smithi039.stdout:clone image108 -> image110 2024-09-14T03:57:13.311 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:13 smithi039 ceph-mon[29586]: osdmap e238: 8 total, 8 up, 8 in 2024-09-14T03:57:13.311 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:13 smithi039 ceph-mon[29586]: pgmap v72: 137 pgs: 137 active+clean; 749 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:57:13.311 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:13 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:13.311 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:13 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:13.311 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:13 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:13.311 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:13 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:13.312 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:13 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:13.312 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:13 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:13.312 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:13 smithi039 ceph-mon[40962]: osdmap e238: 8 total, 8 up, 8 in 2024-09-14T03:57:13.312 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:13 smithi039 ceph-mon[40962]: pgmap v72: 137 pgs: 137 active+clean; 749 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:57:13.312 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:13 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:13.312 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:13 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:13.312 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:13 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:13.312 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:13 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:13.312 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:13 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:13.313 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:13 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:13.361 INFO:tasks.workunit.client.0.smithi039.stdout:write: X 1000~1000 2024-09-14T03:57:13.421 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 2010~1000 2024-09-14T03:57:13.424 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:57:13.507 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:13 smithi129 ceph-mon[33173]: osdmap e238: 8 total, 8 up, 8 in 2024-09-14T03:57:13.507 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:13 smithi129 ceph-mon[33173]: pgmap v72: 137 pgs: 137 active+clean; 749 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:57:13.507 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:13 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:13.507 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:13 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:13.507 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:13 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:13.507 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:13 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:13.507 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:13 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:13.507 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:13 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:13.727 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:57:13.916 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:57:14.080 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.Clone (4623 ms) 2024-09-14T03:57:14.080 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.CloneParent 2024-09-14T03:57:14.110 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:57:14.120 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap 2024-09-14T03:57:14.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:14 smithi129 ceph-mon[33173]: osdmap e239: 8 total, 8 up, 8 in 2024-09-14T03:57:14.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:14 smithi129 ceph-mon[33173]: osdmap e240: 8 total, 8 up, 8 in 2024-09-14T03:57:14.487 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:14 smithi039 ceph-mon[29586]: osdmap e239: 8 total, 8 up, 8 in 2024-09-14T03:57:14.487 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:14 smithi039 ceph-mon[29586]: osdmap e240: 8 total, 8 up, 8 in 2024-09-14T03:57:14.487 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:14 smithi039 ceph-mon[40962]: osdmap e239: 8 total, 8 up, 8 in 2024-09-14T03:57:14.487 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:14 smithi039 ceph-mon[40962]: osdmap e240: 8 total, 8 up, 8 in 2024-09-14T03:57:15.747 INFO:tasks.workunit.client.0.smithi039.stdout:snap_protect: snap 2024-09-14T03:57:15.756 INFO:tasks.workunit.client.0.smithi039.stdout:clone image111 -> image114 2024-09-14T03:57:15.846 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:57:15.999 INFO:tasks.workunit.client.0.smithi039.stdout:migration_abort 2024-09-14T03:57:16.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:15 smithi129 ceph-mon[33173]: pgmap v75: 137 pgs: 137 active+clean; 769 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 42 MiB/s rd, 10 MiB/s wr, 244 op/s 2024-09-14T03:57:16.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:15 smithi129 ceph-mon[33173]: osdmap e241: 8 total, 8 up, 8 in 2024-09-14T03:57:16.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:15 smithi039 ceph-mon[29586]: pgmap v75: 137 pgs: 137 active+clean; 769 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 42 MiB/s rd, 10 MiB/s wr, 244 op/s 2024-09-14T03:57:16.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:15 smithi039 ceph-mon[29586]: osdmap e241: 8 total, 8 up, 8 in 2024-09-14T03:57:16.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:15 smithi039 ceph-mon[40962]: pgmap v75: 137 pgs: 137 active+clean; 769 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 42 MiB/s rd, 10 MiB/s wr, 244 op/s 2024-09-14T03:57:16.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:15 smithi039 ceph-mon[40962]: osdmap e241: 8 total, 8 up, 8 in 2024-09-14T03:57:16.130 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.CloneParent (2050 ms) 2024-09-14T03:57:16.130 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.CloneUpdateAfterPrepare 2024-09-14T03:57:16.167 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:57:16.179 INFO:tasks.workunit.client.0.smithi039.stdout:write: X 0~10 2024-09-14T03:57:16.188 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap 2024-09-14T03:57:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:16 smithi129 ceph-mon[33173]: osdmap e242: 8 total, 8 up, 8 in 2024-09-14T03:57:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:16 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:16 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:16 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:16 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:16 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:57:17.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:16 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:57:17.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:16 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:17.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:16 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:57:17.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:16 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:17.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:16 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:17.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:16 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:17.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:16 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:17.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:16 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:17.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:16 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:17.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:16 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:17.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:16 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:17.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:16 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:17.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:16 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:17.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[29586]: osdmap e242: 8 total, 8 up, 8 in 2024-09-14T03:57:17.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:17.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:17.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:17.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:17.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:57:17.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:57:17.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:17.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:57:17.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:17.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:17.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:17.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:17.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:17.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:17.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:17.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:17.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:17.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:17.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[40962]: osdmap e242: 8 total, 8 up, 8 in 2024-09-14T03:57:17.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:17.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:17.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:17.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:17.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:57:17.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:57:17.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:17.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:57:17.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:17.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:17.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:17.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:17.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:17.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:17.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:17.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:17.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:17.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:16 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:17.910 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:17 smithi129 ceph-mon[33173]: pgmap v78: 137 pgs: 137 active+clean; 769 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 42 MiB/s rd, 10 MiB/s wr, 244 op/s 2024-09-14T03:57:17.910 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:17 smithi129 ceph-mon[33173]: osdmap e243: 8 total, 8 up, 8 in 2024-09-14T03:57:18.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:17 smithi039 ceph-mon[29586]: pgmap v78: 137 pgs: 137 active+clean; 769 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 42 MiB/s rd, 10 MiB/s wr, 244 op/s 2024-09-14T03:57:18.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:17 smithi039 ceph-mon[29586]: osdmap e243: 8 total, 8 up, 8 in 2024-09-14T03:57:18.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:17 smithi039 ceph-mon[40962]: pgmap v78: 137 pgs: 137 active+clean; 769 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 42 MiB/s rd, 10 MiB/s wr, 244 op/s 2024-09-14T03:57:18.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:17 smithi039 ceph-mon[40962]: osdmap e243: 8 total, 8 up, 8 in 2024-09-14T03:57:18.157 INFO:tasks.workunit.client.0.smithi039.stdout:snap_protect: snap 2024-09-14T03:57:18.169 INFO:tasks.workunit.client.0.smithi039.stdout:clone image115 -> image118 2024-09-14T03:57:18.263 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:57:18.407 INFO:tasks.workunit.client.0.smithi039.stdout:write: Y 0~1 2024-09-14T03:57:18.437 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:57:18.504 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:57:18.612 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.CloneUpdateAfterPrepare (2482 ms) 2024-09-14T03:57:18.613 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.TriggerAssertSnapcSeq 2024-09-14T03:57:18.648 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:57:18.662 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 41943040~10 2024-09-14T03:57:18.671 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap1 2024-09-14T03:57:19.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:18 smithi129 ceph-mon[33173]: Upgrade: Updating prometheus.a 2024-09-14T03:57:19.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:18 smithi129 ceph-mon[33173]: osdmap e244: 8 total, 8 up, 8 in 2024-09-14T03:57:19.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:18 smithi129 ceph-mon[33173]: osdmap e245: 8 total, 8 up, 8 in 2024-09-14T03:57:19.031 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:18 smithi039 ceph-mon[29586]: Upgrade: Updating prometheus.a 2024-09-14T03:57:19.032 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:18 smithi039 ceph-mon[29586]: osdmap e244: 8 total, 8 up, 8 in 2024-09-14T03:57:19.032 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:18 smithi039 ceph-mon[29586]: osdmap e245: 8 total, 8 up, 8 in 2024-09-14T03:57:19.032 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:18 smithi039 ceph-mon[40962]: Upgrade: Updating prometheus.a 2024-09-14T03:57:19.032 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:18 smithi039 ceph-mon[40962]: osdmap e244: 8 total, 8 up, 8 in 2024-09-14T03:57:19.032 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:18 smithi039 ceph-mon[40962]: osdmap e245: 8 total, 8 up, 8 in 2024-09-14T03:57:19.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:57:19 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:57:19] "GET /metrics HTTP/1.1" 200 38862 "" "Prometheus/2.43.0" 2024-09-14T03:57:19.767 INFO:tasks.workunit.client.0.smithi039.stdout:write: B 41943041~10 2024-09-14T03:57:19.773 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:57:20.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:19 smithi129 ceph-mon[33173]: Deploying daemon prometheus.a on smithi129 2024-09-14T03:57:20.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:19 smithi129 ceph-mon[33173]: pgmap v81: 137 pgs: 137 active+clean; 777 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 3.0 MiB/s wr, 1.03k op/s 2024-09-14T03:57:20.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:19 smithi039 ceph-mon[40962]: Deploying daemon prometheus.a on smithi129 2024-09-14T03:57:20.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:19 smithi039 ceph-mon[40962]: pgmap v81: 137 pgs: 137 active+clean; 777 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 3.0 MiB/s wr, 1.03k op/s 2024-09-14T03:57:20.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:19 smithi039 ceph-mon[29586]: Deploying daemon prometheus.a on smithi129 2024-09-14T03:57:20.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:19 smithi039 ceph-mon[29586]: pgmap v81: 137 pgs: 137 active+clean; 777 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 3.0 MiB/s wr, 1.03k op/s 2024-09-14T03:57:20.924 INFO:tasks.workunit.client.0.smithi039.stdout:write: C 41943042~10 2024-09-14T03:57:20.941 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap2 2024-09-14T03:57:21.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:20 smithi129 ceph-mon[33173]: osdmap e246: 8 total, 8 up, 8 in 2024-09-14T03:57:21.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:20 smithi039 ceph-mon[29586]: osdmap e246: 8 total, 8 up, 8 in 2024-09-14T03:57:21.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:20 smithi039 ceph-mon[40962]: osdmap e246: 8 total, 8 up, 8 in 2024-09-14T03:57:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:21 smithi129 ceph-mon[33173]: pgmap v84: 137 pgs: 137 active+clean; 777 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 3.0 MiB/s wr, 1.03k op/s 2024-09-14T03:57:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:21 smithi129 ceph-mon[33173]: osdmap e247: 8 total, 8 up, 8 in 2024-09-14T03:57:22.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:21 smithi039 ceph-mon[29586]: pgmap v84: 137 pgs: 137 active+clean; 777 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 3.0 MiB/s wr, 1.03k op/s 2024-09-14T03:57:22.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:21 smithi039 ceph-mon[29586]: osdmap e247: 8 total, 8 up, 8 in 2024-09-14T03:57:22.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:21 smithi039 ceph-mon[40962]: pgmap v84: 137 pgs: 137 active+clean; 777 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 20 MiB/s rd, 3.0 MiB/s wr, 1.03k op/s 2024-09-14T03:57:22.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:21 smithi039 ceph-mon[40962]: osdmap e247: 8 total, 8 up, 8 in 2024-09-14T03:57:22.827 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 41943040 2024-09-14T03:57:22.865 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:57:22.932 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:57:23.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:22 smithi129 ceph-mon[33173]: osdmap e248: 8 total, 8 up, 8 in 2024-09-14T03:57:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:22 smithi039 ceph-mon[29586]: osdmap e248: 8 total, 8 up, 8 in 2024-09-14T03:57:23.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:22 smithi039 ceph-mon[40962]: osdmap e248: 8 total, 8 up, 8 in 2024-09-14T03:57:23.940 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.TriggerAssertSnapcSeq (5328 ms) 2024-09-14T03:57:23.941 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.SnapTrimBeforePrepare 2024-09-14T03:57:23.972 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:57:23.984 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 41943040~10 2024-09-14T03:57:23.993 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap1 2024-09-14T03:57:24.191 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:23 smithi039 ceph-mon[29586]: pgmap v87: 137 pgs: 137 active+clean; 777 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:57:24.191 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:23 smithi039 ceph-mon[29586]: osdmap e249: 8 total, 8 up, 8 in 2024-09-14T03:57:24.192 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:23 smithi039 ceph-mon[40962]: pgmap v87: 137 pgs: 137 active+clean; 777 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:57:24.192 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:23 smithi039 ceph-mon[40962]: osdmap e249: 8 total, 8 up, 8 in 2024-09-14T03:57:24.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:23 smithi129 ceph-mon[33173]: pgmap v87: 137 pgs: 137 active+clean; 777 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:57:24.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:23 smithi129 ceph-mon[33173]: osdmap e249: 8 total, 8 up, 8 in 2024-09-14T03:57:25.205 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:24 smithi039 ceph-mon[29586]: osdmap e250: 8 total, 8 up, 8 in 2024-09-14T03:57:25.205 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:24 smithi039 ceph-mon[40962]: osdmap e250: 8 total, 8 up, 8 in 2024-09-14T03:57:25.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:24 smithi129 ceph-mon[33173]: osdmap e250: 8 total, 8 up, 8 in 2024-09-14T03:57:25.854 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 41943040 2024-09-14T03:57:25.871 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:57:26.004 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:25 smithi129 ceph-mon[33173]: pgmap v90: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.0 MiB/s wr, 536 op/s 2024-09-14T03:57:26.005 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:25 smithi129 ceph-mon[33173]: osdmap e251: 8 total, 8 up, 8 in 2024-09-14T03:57:26.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:25 smithi039 ceph-mon[29586]: pgmap v90: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.0 MiB/s wr, 536 op/s 2024-09-14T03:57:26.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:25 smithi039 ceph-mon[29586]: osdmap e251: 8 total, 8 up, 8 in 2024-09-14T03:57:26.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:25 smithi039 ceph-mon[40962]: pgmap v90: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.0 MiB/s wr, 536 op/s 2024-09-14T03:57:26.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:25 smithi039 ceph-mon[40962]: osdmap e251: 8 total, 8 up, 8 in 2024-09-14T03:57:26.980 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:57:26.987 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:26 smithi129 ceph-mon[33173]: osdmap e252: 8 total, 8 up, 8 in 2024-09-14T03:57:26.990 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap3 2024-09-14T03:57:27.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:26 smithi039 ceph-mon[29586]: osdmap e252: 8 total, 8 up, 8 in 2024-09-14T03:57:27.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:26 smithi039 ceph-mon[40962]: osdmap e252: 8 total, 8 up, 8 in 2024-09-14T03:57:28.149 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:27 smithi129 ceph-mon[33173]: pgmap v93: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.0 MiB/s wr, 536 op/s 2024-09-14T03:57:28.149 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:27 smithi129 ceph-mon[33173]: osdmap e253: 8 total, 8 up, 8 in 2024-09-14T03:57:28.149 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:27 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:57:28.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:27 smithi039 ceph-mon[29586]: pgmap v93: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.0 MiB/s wr, 536 op/s 2024-09-14T03:57:28.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:27 smithi039 ceph-mon[29586]: osdmap e253: 8 total, 8 up, 8 in 2024-09-14T03:57:28.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:27 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:57:28.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:27 smithi039 ceph-mon[40962]: pgmap v93: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 1.0 MiB/s wr, 536 op/s 2024-09-14T03:57:28.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:27 smithi039 ceph-mon[40962]: osdmap e253: 8 total, 8 up, 8 in 2024-09-14T03:57:28.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:27 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:57:28.897 INFO:tasks.workunit.client.0.smithi039.stdout:write: B 41943040~10 2024-09-14T03:57:28.903 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap4 2024-09-14T03:57:29.156 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:28 smithi129 systemd[1]: Stopping Ceph prometheus.a for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:57:29.156 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:28 smithi129 ceph-mon[33173]: osdmap e254: 8 total, 8 up, 8 in 2024-09-14T03:57:29.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:28 smithi039 ceph-mon[29586]: osdmap e254: 8 total, 8 up, 8 in 2024-09-14T03:57:29.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:57:29 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:57:29] "GET /metrics HTTP/1.1" 200 38863 "" "Prometheus/2.43.0" 2024-09-14T03:57:29.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:28 smithi039 ceph-mon[40962]: osdmap e254: 8 total, 8 up, 8 in 2024-09-14T03:57:29.454 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:29 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:57:29.154Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-14T03:57:29.454 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:29 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:57:29.155Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-09-14T03:57:29.454 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:29 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:57:29.155Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-09-14T03:57:29.454 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:29 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:57:29.155Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-14T03:57:29.454 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:29 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:57:29.155Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-09-14T03:57:29.454 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:29 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:57:29.155Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-09-14T03:57:29.454 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:29 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:57:29.155Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-09-14T03:57:29.455 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:29 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:57:29.155Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-09-14T03:57:29.455 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:29 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:57:29.155Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-09-14T03:57:29.455 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:29 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:57:29.155Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-09-14T03:57:29.455 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:29 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:57:29.155Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-09-14T03:57:29.455 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:29 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[79944]: ts=2024-09-14T03:57:29.155Z caller=main.go:1147 level=info msg="See you next time!" 2024-09-14T03:57:29.455 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:29 smithi129 podman[88027]: 2024-09-14 03:57:29.165369503 +0000 UTC m=+0.120255676 container died e8a320b5858c9699b387a22bd4419be8a3949bc4df49a6b23526752a3aa4189f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:57:29.771 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:29 smithi129 podman[88027]: 2024-09-14 03:57:29.577581597 +0000 UTC m=+0.532467770 container cleanup e8a320b5858c9699b387a22bd4419be8a3949bc4df49a6b23526752a3aa4189f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:57:29.771 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:29 smithi129 bash[88027]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a 2024-09-14T03:57:30.200 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:29 smithi129 podman[88045]: 2024-09-14 03:57:29.770450875 +0000 UTC m=+0.606720385 container remove e8a320b5858c9699b387a22bd4419be8a3949bc4df49a6b23526752a3aa4189f (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:57:30.200 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:29 smithi129 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-09-14T03:57:30.200 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:29 smithi129 systemd[1]: Stopped Ceph prometheus.a for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:57:30.200 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@prometheus.a.service: Consumed 1.175s CPU time. 2024-09-14T03:57:30.200 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:29 smithi129 ceph-mon[33173]: pgmap v96: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 276 KiB/s rd, 25 KiB/s wr, 372 op/s 2024-09-14T03:57:30.200 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:29 smithi129 ceph-mon[33173]: osdmap e255: 8 total, 8 up, 8 in 2024-09-14T03:57:30.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:29 smithi039 ceph-mon[29586]: pgmap v96: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 276 KiB/s rd, 25 KiB/s wr, 372 op/s 2024-09-14T03:57:30.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:29 smithi039 ceph-mon[29586]: osdmap e255: 8 total, 8 up, 8 in 2024-09-14T03:57:30.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:29 smithi039 ceph-mon[40962]: pgmap v96: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 276 KiB/s rd, 25 KiB/s wr, 372 op/s 2024-09-14T03:57:30.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:29 smithi039 ceph-mon[40962]: osdmap e255: 8 total, 8 up, 8 in 2024-09-14T03:57:30.521 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 systemd[1]: Starting Ceph prometheus.a for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:57:30.521 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 podman[88177]: 2024-09-14 03:57:30.308607958 +0000 UTC m=+0.017000240 image pull 1d3b7f56885b6dd623f1785be963aa9c195f86bc256ea454e8d02a7980b79c53 quay.io/prometheus/prometheus:v2.51.0 2024-09-14T03:57:30.521 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 podman[88177]: 2024-09-14 03:57:30.446865501 +0000 UTC m=+0.155257778 container create cbe583f852c302c3b2474b6cacf83b5238d42bf36b2830094bdd2fe629043ae2 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:57:30.905 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:30 smithi129 ceph-mon[33173]: osdmap e256: 8 total, 8 up, 8 in 2024-09-14T03:57:30.905 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:30 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:30.905 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:30 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:30.905 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:30 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:57:30.906 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 podman[88177]: 2024-09-14 03:57:30.648857444 +0000 UTC m=+0.357249730 container init cbe583f852c302c3b2474b6cacf83b5238d42bf36b2830094bdd2fe629043ae2 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:57:30.906 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 podman[88177]: 2024-09-14 03:57:30.65370326 +0000 UTC m=+0.362095543 container start cbe583f852c302c3b2474b6cacf83b5238d42bf36b2830094bdd2fe629043ae2 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T03:57:30.906 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T03:57:30.672Z caller=main.go:617 level=info msg="Starting Prometheus Server" mode=server version="(version=2.51.0, branch=HEAD, revision=c05c15512acb675e3f6cd662a6727854e93fc024)" 2024-09-14T03:57:30.906 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T03:57:30.673Z caller=main.go:622 level=info build_context="(go=go1.22.1, platform=linux/amd64, user=root@b5723e458358, date=20240319-10:54:45, tags=netgo,builtinassets,stringlabels)" 2024-09-14T03:57:30.906 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T03:57:30.673Z caller=main.go:623 level=info host_details="(Linux 5.14.0-505.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Thu Sep 5 07:54:07 UTC 2024 x86_64 smithi129 (none))" 2024-09-14T03:57:30.906 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T03:57:30.673Z caller=main.go:624 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-09-14T03:57:30.906 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T03:57:30.673Z caller=main.go:625 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-09-14T03:57:30.906 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T03:57:30.674Z caller=web.go:568 level=info component=web msg="Start listening for connections" address=:9095 2024-09-14T03:57:30.906 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T03:57:30.675Z caller=main.go:1129 level=info msg="Starting TSDB ..." 2024-09-14T03:57:30.906 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T03:57:30.676Z caller=tls_config.go:313 level=info component=web msg="Listening on" address=[::]:9095 2024-09-14T03:57:30.906 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T03:57:30.677Z caller=tls_config.go:316 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-09-14T03:57:30.906 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T03:57:30.678Z caller=head.go:616 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-09-14T03:57:30.907 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T03:57:30.678Z caller=head.go:698 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.666µs 2024-09-14T03:57:30.907 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T03:57:30.678Z caller=head.go:706 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-09-14T03:57:30.907 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T03:57:30.680Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=4 2024-09-14T03:57:30.907 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T03:57:30.687Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=4 2024-09-14T03:57:30.907 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T03:57:30.687Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=4 2024-09-14T03:57:30.907 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T03:57:30.688Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=4 2024-09-14T03:57:30.907 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T03:57:30.688Z caller=head.go:778 level=info component=tsdb msg="WAL segment loaded" segment=4 maxSegment=4 2024-09-14T03:57:30.907 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T03:57:30.688Z caller=head.go:815 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=33.663µs wal_replay_duration=10.133365ms wbl_replay_duration=386ns total_replay_duration=10.196134ms 2024-09-14T03:57:30.907 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T03:57:30.691Z caller=main.go:1150 level=info fs_type=XFS_SUPER_MAGIC 2024-09-14T03:57:30.908 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T03:57:30.691Z caller=main.go:1153 level=info msg="TSDB started" 2024-09-14T03:57:30.908 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T03:57:30.691Z caller=main.go:1335 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-09-14T03:57:30.908 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T03:57:30.706Z caller=main.go:1372 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.242761ms db_storage=754ns remote_storage=1.375µs web_handler=310ns query_engine=555ns scrape=397.388µs scrape_sd=113.018µs notify=12.644µs notify_sd=18.648µs rules=14.360985ms tracing=5.793µs 2024-09-14T03:57:30.908 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T03:57:30.706Z caller=main.go:1114 level=info msg="Server is ready to receive web requests." 2024-09-14T03:57:30.908 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T03:57:30.706Z caller=manager.go:163 level=info component="rule manager" msg="Starting rule manager..." 2024-09-14T03:57:30.908 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 bash[88177]: cbe583f852c302c3b2474b6cacf83b5238d42bf36b2830094bdd2fe629043ae2 2024-09-14T03:57:30.908 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 03:57:30 smithi129 systemd[1]: Started Ceph prometheus.a for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:57:30.927 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 41943040 2024-09-14T03:57:30.950 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:57:31.015 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:57:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:30 smithi039 ceph-mon[29586]: osdmap e256: 8 total, 8 up, 8 in 2024-09-14T03:57:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:30 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:30 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:30 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:57:31.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:30 smithi039 ceph-mon[40962]: osdmap e256: 8 total, 8 up, 8 in 2024-09-14T03:57:31.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:30 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:31.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:30 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:31.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:30 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:57:32.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:31 smithi129 ceph-mon[33173]: pgmap v99: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 276 KiB/s rd, 25 KiB/s wr, 372 op/s 2024-09-14T03:57:32.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:31 smithi129 ceph-mon[33173]: osdmap e257: 8 total, 8 up, 8 in 2024-09-14T03:57:32.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:31 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:32.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:31 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:32.063 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.SnapTrimBeforePrepare (8123 ms) 2024-09-14T03:57:32.064 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.AbortInUseImage 2024-09-14T03:57:32.096 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:57:32.111 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:57:32.286 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.AbortInUseImage (222 ms) 2024-09-14T03:57:32.286 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.AbortWithoutSnapshots 2024-09-14T03:57:32.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:31 smithi039 ceph-mon[29586]: pgmap v99: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 276 KiB/s rd, 25 KiB/s wr, 372 op/s 2024-09-14T03:57:32.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:31 smithi039 ceph-mon[29586]: osdmap e257: 8 total, 8 up, 8 in 2024-09-14T03:57:32.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:31 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:32.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:31 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:32.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:31 smithi039 ceph-mon[40962]: pgmap v99: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 276 KiB/s rd, 25 KiB/s wr, 372 op/s 2024-09-14T03:57:32.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:31 smithi039 ceph-mon[40962]: osdmap e257: 8 total, 8 up, 8 in 2024-09-14T03:57:32.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:31 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:32.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:31 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:32.316 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:57:32.327 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 0~8388609 2024-09-14T03:57:32.381 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 16777218~8388609 2024-09-14T03:57:32.425 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:57:32.624 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 0~8388609 2024-09-14T03:57:32.660 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 16777218~8388609 2024-09-14T03:57:32.714 INFO:tasks.workunit.client.0.smithi039.stdout:migration_abort 2024-09-14T03:57:32.908 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.AbortWithoutSnapshots (623 ms) 2024-09-14T03:57:32.908 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.AbortWithSnapshots 2024-09-14T03:57:32.943 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:57:32.954 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 0~8388609 2024-09-14T03:57:33.009 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap1 2024-09-14T03:57:33.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:32 smithi129 ceph-mon[33173]: osdmap e258: 8 total, 8 up, 8 in 2024-09-14T03:57:33.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:32 smithi039 ceph-mon[29586]: osdmap e258: 8 total, 8 up, 8 in 2024-09-14T03:57:33.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:32 smithi039 ceph-mon[40962]: osdmap e258: 8 total, 8 up, 8 in 2024-09-14T03:57:34.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:33 smithi129 ceph-mon[33173]: pgmap v102: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:57:34.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:33 smithi129 ceph-mon[33173]: osdmap e259: 8 total, 8 up, 8 in 2024-09-14T03:57:34.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:33 smithi039 ceph-mon[29586]: pgmap v102: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:57:34.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:33 smithi039 ceph-mon[29586]: osdmap e259: 8 total, 8 up, 8 in 2024-09-14T03:57:34.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:33 smithi039 ceph-mon[40962]: pgmap v102: 137 pgs: 137 active+clean; 781 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:57:34.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:33 smithi039 ceph-mon[40962]: osdmap e259: 8 total, 8 up, 8 in 2024-09-14T03:57:34.748 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 8388609~8388609 2024-09-14T03:57:34.762 INFO:tasks.workunit.client.0.smithi039.stdout:write: X 0~1000 2024-09-14T03:57:34.762 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 1010~1000 2024-09-14T03:57:34.762 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap2 2024-09-14T03:57:36.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:35 smithi129 ceph-mon[33173]: pgmap v104: 137 pgs: 137 active+clean; 813 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 311 KiB/s rd, 7.4 MiB/s wr, 416 op/s 2024-09-14T03:57:36.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:35 smithi129 ceph-mon[33173]: osdmap e260: 8 total, 8 up, 8 in 2024-09-14T03:57:36.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:35 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:36.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:35 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:36.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:35 smithi039 ceph-mon[29586]: pgmap v104: 137 pgs: 137 active+clean; 813 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 311 KiB/s rd, 7.4 MiB/s wr, 416 op/s 2024-09-14T03:57:36.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:35 smithi039 ceph-mon[29586]: osdmap e260: 8 total, 8 up, 8 in 2024-09-14T03:57:36.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:35 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:36.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:35 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:36.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:35 smithi039 ceph-mon[40962]: pgmap v104: 137 pgs: 137 active+clean; 813 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 311 KiB/s rd, 7.4 MiB/s wr, 416 op/s 2024-09-14T03:57:36.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:35 smithi039 ceph-mon[40962]: osdmap e260: 8 total, 8 up, 8 in 2024-09-14T03:57:36.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:35 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:36.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:35 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:36.780 INFO:tasks.workunit.client.0.smithi039.stdout:write: X 8388609~1000 2024-09-14T03:57:36.786 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 16777228~1000 2024-09-14T03:57:36.787 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 167772160 2024-09-14T03:57:36.796 INFO:tasks.workunit.client.0.smithi039.stdout:write: 2 83886079~8388609 2024-09-14T03:57:36.814 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap3 2024-09-14T03:57:37.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:36 smithi129 ceph-mon[33173]: osdmap e261: 8 total, 8 up, 8 in 2024-09-14T03:57:37.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:36 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:37.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:36 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:37.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:36 smithi039 ceph-mon[29586]: osdmap e261: 8 total, 8 up, 8 in 2024-09-14T03:57:37.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:36 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:37.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:36 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:37.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:36 smithi039 ceph-mon[40962]: osdmap e261: 8 total, 8 up, 8 in 2024-09-14T03:57:37.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:36 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:37.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:36 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:38.007 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:37 smithi129 ceph-mon[33173]: pgmap v107: 137 pgs: 137 active+clean; 813 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 313 KiB/s rd, 7.5 MiB/s wr, 418 op/s 2024-09-14T03:57:38.007 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:37 smithi129 ceph-mon[33173]: osdmap e262: 8 total, 8 up, 8 in 2024-09-14T03:57:38.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:37 smithi039 ceph-mon[29586]: pgmap v107: 137 pgs: 137 active+clean; 813 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 313 KiB/s rd, 7.5 MiB/s wr, 418 op/s 2024-09-14T03:57:38.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:37 smithi039 ceph-mon[29586]: osdmap e262: 8 total, 8 up, 8 in 2024-09-14T03:57:38.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:37 smithi039 ceph-mon[40962]: pgmap v107: 137 pgs: 137 active+clean; 813 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 313 KiB/s rd, 7.5 MiB/s wr, 418 op/s 2024-09-14T03:57:38.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:37 smithi039 ceph-mon[40962]: osdmap e262: 8 total, 8 up, 8 in 2024-09-14T03:57:38.507 INFO:teuthology.orchestra.run.smithi039.stdout:true 2024-09-14T03:57:38.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:57:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:57:38] "GET /metrics HTTP/1.1" 200 38863 "" "Prometheus/2.51.0" 2024-09-14T03:57:38.836 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:57:38.857 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 83886079~1 2024-09-14T03:57:38.865 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:57:39.011 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:38 smithi129 ceph-mon[33173]: osdmap e263: 8 total, 8 up, 8 in 2024-09-14T03:57:39.104 INFO:teuthology.orchestra.run.smithi039.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:57:39.104 INFO:teuthology.orchestra.run.smithi039.stdout:alertmanager.a smithi039 *:9093,9094 running (4m) 27s ago 5m 27.7M - 0.25.0 c8568f914cd2 b5ca6b93d9ee 2024-09-14T03:57:39.104 INFO:teuthology.orchestra.run.smithi039.stdout:grafana.a smithi129 *:3000 running (5m) 3s ago 5m 89.4M - 9.4.7 954c08fa6188 71f297945590 2024-09-14T03:57:39.104 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi039.bbwiwf smithi039 running (4m) 27s ago 4m 19.6M - 18.2.4-845-ge4e227b3 67ddd33fa72f 91e6c8f85681 2024-09-14T03:57:39.104 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi129.ybjqcw smithi129 running (4m) 3s ago 4m 14.8M - 18.2.4-845-ge4e227b3 67ddd33fa72f a3fab5b37b71 2024-09-14T03:57:39.104 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.x smithi129 *:8443,9283,8765 running (53s) 3s ago 9m 452M - 19.3.0-4907-ga2782315 8801d2529e43 448bd4e146cb 2024-09-14T03:57:39.104 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.y smithi039 *:8443,9283,8765 running (81s) 27s ago 11m 511M - 19.3.0-4907-ga2782315 8801d2529e43 cccc87f635a6 2024-09-14T03:57:39.104 INFO:teuthology.orchestra.run.smithi039.stdout:mon.a smithi039 running (11m) 27s ago 11m 59.0M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f e7b5f91fb09c 2024-09-14T03:57:39.104 INFO:teuthology.orchestra.run.smithi039.stdout:mon.b smithi129 running (10m) 3s ago 10m 52.2M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f eb38cc5cbe35 2024-09-14T03:57:39.104 INFO:teuthology.orchestra.run.smithi039.stdout:mon.c smithi039 running (9m) 27s ago 9m 49.5M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 31d440e49f93 2024-09-14T03:57:39.104 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.a smithi039 *:9100 running (37s) 27s ago 5m 7268k - 1.7.0 72c9c2088986 7dd5945adb64 2024-09-14T03:57:39.105 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.b smithi129 *:9100 running (31s) 3s ago 5m 8128k - 1.7.0 72c9c2088986 1fedb5f257e7 2024-09-14T03:57:39.105 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0 smithi039 running (9m) 27s ago 9m 121M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 5048d74f27ac 2024-09-14T03:57:39.105 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1 smithi039 running (8m) 27s ago 8m 110M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 91349527d88b 2024-09-14T03:57:39.105 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2 smithi039 running (8m) 27s ago 8m 94.7M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f eb903b44fc5d 2024-09-14T03:57:39.105 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3 smithi039 running (8m) 27s ago 8m 110M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f dff9e665bd9b 2024-09-14T03:57:39.105 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4 smithi129 running (7m) 3s ago 7m 105M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 5b64727311b7 2024-09-14T03:57:39.105 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5 smithi129 running (7m) 3s ago 7m 89.7M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f f2d991a754f7 2024-09-14T03:57:39.105 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6 smithi129 running (6m) 3s ago 6m 132M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 3060eaf44106 2024-09-14T03:57:39.105 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7 smithi129 running (6m) 3s ago 6m 88.7M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 4c13b97060d1 2024-09-14T03:57:39.105 INFO:teuthology.orchestra.run.smithi039.stdout:prometheus.a smithi129 *:9095 running (8s) 3s ago 6m 30.8M - 2.51.0 1d3b7f56885b cbe583f852c3 2024-09-14T03:57:39.105 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:38 smithi039 ceph-mon[29586]: osdmap e263: 8 total, 8 up, 8 in 2024-09-14T03:57:39.106 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:38 smithi039 ceph-mon[40962]: osdmap e263: 8 total, 8 up, 8 in 2024-09-14T03:57:39.457 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T03:57:39.457 INFO:teuthology.orchestra.run.smithi039.stdout: "mon": { 2024-09-14T03:57:39.457 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 3 2024-09-14T03:57:39.457 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:57:39.457 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr": { 2024-09-14T03:57:39.457 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:57:39.457 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:57:39.457 INFO:teuthology.orchestra.run.smithi039.stdout: "osd": { 2024-09-14T03:57:39.457 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8 2024-09-14T03:57:39.458 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:57:39.458 INFO:teuthology.orchestra.run.smithi039.stdout: "mds": { 2024-09-14T03:57:39.458 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:57:39.458 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:57:39.458 INFO:teuthology.orchestra.run.smithi039.stdout: "overall": { 2024-09-14T03:57:39.458 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 13, 2024-09-14T03:57:39.458 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:57:39.458 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T03:57:39.458 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T03:57:39.769 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T03:57:39.769 INFO:teuthology.orchestra.run.smithi039.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:57:39.769 INFO:teuthology.orchestra.run.smithi039.stdout: "in_progress": true, 2024-09-14T03:57:39.769 INFO:teuthology.orchestra.run.smithi039.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-14T03:57:39.769 INFO:teuthology.orchestra.run.smithi039.stdout: "services_complete": [ 2024-09-14T03:57:39.769 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr" 2024-09-14T03:57:39.769 INFO:teuthology.orchestra.run.smithi039.stdout: ], 2024-09-14T03:57:39.769 INFO:teuthology.orchestra.run.smithi039.stdout: "progress": "2/2 daemons upgraded", 2024-09-14T03:57:39.769 INFO:teuthology.orchestra.run.smithi039.stdout: "message": "Currently upgrading alertmanager daemons", 2024-09-14T03:57:39.770 INFO:teuthology.orchestra.run.smithi039.stdout: "is_paused": false 2024-09-14T03:57:39.770 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T03:57:40.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[29586]: pgmap v110: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 25 MiB/s wr, 293 op/s 2024-09-14T03:57:40.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[29586]: from='client.25052 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:40.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[29586]: from='client.15156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:40.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[29586]: osdmap e264: 8 total, 8 up, 8 in 2024-09-14T03:57:40.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:40.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:40.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:57:40.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:57:40.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:40.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:57:40.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/4228691076' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:57:40.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[40962]: pgmap v110: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 25 MiB/s wr, 293 op/s 2024-09-14T03:57:40.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[40962]: from='client.25052 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:40.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[40962]: from='client.15156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:40.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[40962]: osdmap e264: 8 total, 8 up, 8 in 2024-09-14T03:57:40.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:40.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:40.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:57:40.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:57:40.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:40.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:57:40.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/4228691076' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:57:40.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:39 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:39 smithi129 ceph-mon[33173]: pgmap v110: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 25 MiB/s wr, 293 op/s 2024-09-14T03:57:40.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:39 smithi129 ceph-mon[33173]: from='client.25052 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:40.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:39 smithi129 ceph-mon[33173]: from='client.15156 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:40.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:39 smithi129 ceph-mon[33173]: osdmap e264: 8 total, 8 up, 8 in 2024-09-14T03:57:40.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:39 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:40.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:39 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:40.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:39 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:57:40.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:39 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:57:40.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:39 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:40.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:39 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:57:40.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:39 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/4228691076' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:39 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:57:40.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:39 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:39 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:39 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:39 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:39 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:39 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:39 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:39 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:39 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:39 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:40.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:39 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:57:41.092 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:40 smithi039 ceph-mon[29586]: from='client.25064 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:41.092 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:40 smithi039 ceph-mon[29586]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:57:41.093 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:40 smithi039 ceph-mon[29586]: from='client.15174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:41.093 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:40 smithi039 ceph-mon[29586]: osdmap e265: 8 total, 8 up, 8 in 2024-09-14T03:57:41.093 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:40 smithi039 ceph-mon[40962]: from='client.25064 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:41.093 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:40 smithi039 ceph-mon[40962]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:57:41.093 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:40 smithi039 ceph-mon[40962]: from='client.15174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:41.093 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:40 smithi039 ceph-mon[40962]: osdmap e265: 8 total, 8 up, 8 in 2024-09-14T03:57:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:40 smithi129 ceph-mon[33173]: from='client.25064 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:40 smithi129 ceph-mon[33173]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-09-14T03:57:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:40 smithi129 ceph-mon[33173]: from='client.15174 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:57:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:40 smithi129 ceph-mon[33173]: osdmap e265: 8 total, 8 up, 8 in 2024-09-14T03:57:42.264 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 0~8388609 2024-09-14T03:57:42.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:41 smithi129 ceph-mon[33173]: pgmap v113: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 25 MiB/s wr, 293 op/s 2024-09-14T03:57:42.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:41 smithi129 ceph-mon[33173]: osdmap e266: 8 total, 8 up, 8 in 2024-09-14T03:57:42.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:41 smithi039 ceph-mon[29586]: pgmap v113: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 25 MiB/s wr, 293 op/s 2024-09-14T03:57:42.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:41 smithi039 ceph-mon[29586]: osdmap e266: 8 total, 8 up, 8 in 2024-09-14T03:57:42.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:41 smithi039 ceph-mon[40962]: pgmap v113: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 21 MiB/s rd, 25 MiB/s wr, 293 op/s 2024-09-14T03:57:42.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:41 smithi039 ceph-mon[40962]: osdmap e266: 8 total, 8 up, 8 in 2024-09-14T03:57:42.366 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 16777218~8388609 2024-09-14T03:57:42.873 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 0~8388609 2024-09-14T03:57:42.889 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 16777218~8388609 2024-09-14T03:57:42.969 INFO:tasks.workunit.client.0.smithi039.stdout:migration_abort 2024-09-14T03:57:43.172 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:42 smithi129 ceph-mon[33173]: Upgrade: Updating alertmanager.a 2024-09-14T03:57:43.172 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:42 smithi129 ceph-mon[33173]: Deploying daemon alertmanager.a on smithi039 2024-09-14T03:57:43.172 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:42 smithi129 ceph-mon[33173]: osdmap e267: 8 total, 8 up, 8 in 2024-09-14T03:57:43.173 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:42 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:57:43.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:42 smithi039 ceph-mon[29586]: Upgrade: Updating alertmanager.a 2024-09-14T03:57:43.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:42 smithi039 ceph-mon[29586]: Deploying daemon alertmanager.a on smithi039 2024-09-14T03:57:43.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:42 smithi039 ceph-mon[29586]: osdmap e267: 8 total, 8 up, 8 in 2024-09-14T03:57:43.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:42 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:57:43.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:42 smithi039 ceph-mon[40962]: Upgrade: Updating alertmanager.a 2024-09-14T03:57:43.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:42 smithi039 ceph-mon[40962]: Deploying daemon alertmanager.a on smithi039 2024-09-14T03:57:43.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:42 smithi039 ceph-mon[40962]: osdmap e267: 8 total, 8 up, 8 in 2024-09-14T03:57:43.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:42 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:57:44.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:43 smithi129 ceph-mon[33173]: pgmap v116: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:57:44.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:43 smithi129 ceph-mon[33173]: osdmap e268: 8 total, 8 up, 8 in 2024-09-14T03:57:44.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:43 smithi129 ceph-mon[33173]: osdmap e269: 8 total, 8 up, 8 in 2024-09-14T03:57:44.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:43 smithi039 ceph-mon[29586]: pgmap v116: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:57:44.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:43 smithi039 ceph-mon[29586]: osdmap e268: 8 total, 8 up, 8 in 2024-09-14T03:57:44.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:43 smithi039 ceph-mon[29586]: osdmap e269: 8 total, 8 up, 8 in 2024-09-14T03:57:44.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:43 smithi039 ceph-mon[40962]: pgmap v116: 137 pgs: 137 active+clean; 869 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:57:44.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:43 smithi039 ceph-mon[40962]: osdmap e268: 8 total, 8 up, 8 in 2024-09-14T03:57:44.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:43 smithi039 ceph-mon[40962]: osdmap e269: 8 total, 8 up, 8 in 2024-09-14T03:57:46.008 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:45 smithi039 ceph-mon[29586]: pgmap v119: 137 pgs: 137 active+clean; 913 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 35 MiB/s rd, 11 MiB/s wr, 340 op/s 2024-09-14T03:57:46.009 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:45 smithi039 ceph-mon[29586]: osdmap e270: 8 total, 8 up, 8 in 2024-09-14T03:57:46.009 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:45 smithi039 ceph-mon[40962]: pgmap v119: 137 pgs: 137 active+clean; 913 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 35 MiB/s rd, 11 MiB/s wr, 340 op/s 2024-09-14T03:57:46.009 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:45 smithi039 ceph-mon[40962]: osdmap e270: 8 total, 8 up, 8 in 2024-09-14T03:57:46.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:45 smithi129 ceph-mon[33173]: pgmap v119: 137 pgs: 137 active+clean; 913 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 35 MiB/s rd, 11 MiB/s wr, 340 op/s 2024-09-14T03:57:46.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:45 smithi129 ceph-mon[33173]: osdmap e270: 8 total, 8 up, 8 in 2024-09-14T03:57:47.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:46 smithi129 ceph-mon[33173]: osdmap e271: 8 total, 8 up, 8 in 2024-09-14T03:57:47.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:46 smithi039 ceph-mon[29586]: osdmap e271: 8 total, 8 up, 8 in 2024-09-14T03:57:47.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:46 smithi039 ceph-mon[40962]: osdmap e271: 8 total, 8 up, 8 in 2024-09-14T03:57:47.096 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.AbortWithSnapshots (14188 ms) 2024-09-14T03:57:47.096 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.CloneV1Parent 2024-09-14T03:57:47.127 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:57:47.138 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 0~10 2024-09-14T03:57:47.146 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap1 2024-09-14T03:57:47.913 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:47 smithi039 ceph-mon[29586]: pgmap v122: 137 pgs: 137 active+clean; 913 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 35 MiB/s rd, 11 MiB/s wr, 340 op/s 2024-09-14T03:57:47.913 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:47 smithi039 ceph-mon[29586]: osdmap e272: 8 total, 8 up, 8 in 2024-09-14T03:57:47.914 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:47 smithi039 ceph-mon[40962]: pgmap v122: 137 pgs: 137 active+clean; 913 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 35 MiB/s rd, 11 MiB/s wr, 340 op/s 2024-09-14T03:57:47.914 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:47 smithi039 ceph-mon[40962]: osdmap e272: 8 total, 8 up, 8 in 2024-09-14T03:57:48.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:47 smithi129 ceph-mon[33173]: pgmap v122: 137 pgs: 137 active+clean; 913 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 35 MiB/s rd, 11 MiB/s wr, 340 op/s 2024-09-14T03:57:48.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:47 smithi129 ceph-mon[33173]: osdmap e272: 8 total, 8 up, 8 in 2024-09-14T03:57:48.803 INFO:tasks.workunit.client.0.smithi039.stdout:snap_protect: snap1 2024-09-14T03:57:48.869 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:57:48.937 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:48 smithi039 ceph-mon[29586]: osdmap e273: 8 total, 8 up, 8 in 2024-09-14T03:57:48.937 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:48 smithi039 ceph-mon[40962]: osdmap e273: 8 total, 8 up, 8 in 2024-09-14T03:57:48.937 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:57:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:57:48] "GET /metrics HTTP/1.1" 200 38864 "" "Prometheus/2.51.0" 2024-09-14T03:57:49.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:48 smithi129 ceph-mon[33173]: osdmap e273: 8 total, 8 up, 8 in 2024-09-14T03:57:49.290 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:57:49 smithi039 systemd[1]: Stopping Ceph alertmanager.a for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:57:49.751 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:57:49 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[77805]: ts=2024-09-14T03:57:49.416Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-14T03:57:49.751 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:57:49 smithi039 podman[101499]: 2024-09-14 03:57:49.457115825 +0000 UTC m=+0.175767077 container died b5ca6b93d9ee8fb2bb28cc7735721d635ebe6e9d9db4be989ff3bffe77963602 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:57:50.006 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:57:50.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:49 smithi039 ceph-mon[29586]: pgmap v125: 137 pgs: 137 active+clean; 901 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 43 MiB/s rd, 12 MiB/s wr, 255 op/s 2024-09-14T03:57:50.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:49 smithi039 ceph-mon[29586]: osdmap e274: 8 total, 8 up, 8 in 2024-09-14T03:57:50.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:49 smithi039 ceph-mon[40962]: pgmap v125: 137 pgs: 137 active+clean; 901 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 43 MiB/s rd, 12 MiB/s wr, 255 op/s 2024-09-14T03:57:50.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:49 smithi039 ceph-mon[40962]: osdmap e274: 8 total, 8 up, 8 in 2024-09-14T03:57:50.041 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:57:49 smithi039 podman[101499]: 2024-09-14 03:57:49.864649335 +0000 UTC m=+0.583300588 container cleanup b5ca6b93d9ee8fb2bb28cc7735721d635ebe6e9d9db4be989ff3bffe77963602 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:57:50.041 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:57:49 smithi039 bash[101499]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a 2024-09-14T03:57:50.087 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:57:50.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:49 smithi129 ceph-mon[33173]: pgmap v125: 137 pgs: 137 active+clean; 901 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 43 MiB/s rd, 12 MiB/s wr, 255 op/s 2024-09-14T03:57:50.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:49 smithi129 ceph-mon[33173]: osdmap e274: 8 total, 8 up, 8 in 2024-09-14T03:57:50.540 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:57:50 smithi039 podman[101515]: 2024-09-14 03:57:50.141831419 +0000 UTC m=+0.680959849 container remove b5ca6b93d9ee8fb2bb28cc7735721d635ebe6e9d9db4be989ff3bffe77963602 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:57:50.540 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:57:50 smithi039 podman[101515]: 2024-09-14 03:57:50.24263124 +0000 UTC m=+0.781759625 volume remove 1d453f423d31ff481b784f511b29881e1bdc80d69b79b1df897cf083c2fde9a1 2024-09-14T03:57:50.540 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:57:50 smithi039 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-14T03:57:50.540 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:57:50 smithi039 systemd[1]: Stopped Ceph alertmanager.a for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:57:50.541 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:57:50 smithi039 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@alertmanager.a.service: Consumed 1.361s CPU time. 2024-09-14T03:57:50.912 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:50 smithi039 ceph-mon[29586]: osdmap e275: 8 total, 8 up, 8 in 2024-09-14T03:57:50.912 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:50 smithi039 ceph-mon[40962]: osdmap e275: 8 total, 8 up, 8 in 2024-09-14T03:57:50.913 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:57:50 smithi039 systemd[1]: Starting Ceph alertmanager.a for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:57:50.913 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:57:50 smithi039 podman[101646]: 2024-09-14 03:57:50.810996809 +0000 UTC m=+0.110628084 volume create bcb5d36c46ace19d3d3d71d9584dda079e73910e851904db49e40e562e9661e6 2024-09-14T03:57:50.913 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:57:50 smithi039 podman[101646]: 2024-09-14 03:57:50.716858066 +0000 UTC m=+0.016489351 image pull 11f11916f8cdfb17c1da74b20deb2075d778357b4d60474d82453303f8d461f5 quay.io/prometheus/alertmanager:v0.27.0 2024-09-14T03:57:50.913 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:57:50 smithi039 podman[101646]: 2024-09-14 03:57:50.911780478 +0000 UTC m=+0.211411753 container create 6b564e1fc909fd6269e079207b1e58b491b75b1fc472c889595a37d94205a5fa (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:57:50.924 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.CloneV1Parent (3827 ms) 2024-09-14T03:57:50.924 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.CloneV2Parent 2024-09-14T03:57:50.954 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:57:50.965 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 0~10 2024-09-14T03:57:50.974 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap1 2024-09-14T03:57:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:50 smithi129 ceph-mon[33173]: osdmap e275: 8 total, 8 up, 8 in 2024-09-14T03:57:51.540 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:57:51 smithi039 podman[101646]: 2024-09-14 03:57:51.222968918 +0000 UTC m=+0.522600194 container init 6b564e1fc909fd6269e079207b1e58b491b75b1fc472c889595a37d94205a5fa (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:57:51.540 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:57:51 smithi039 podman[101646]: 2024-09-14 03:57:51.228032873 +0000 UTC m=+0.527664149 container start 6b564e1fc909fd6269e079207b1e58b491b75b1fc472c889595a37d94205a5fa (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T03:57:51.541 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:57:51 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[101661]: ts=2024-09-14T03:57:51.238Z caller=main.go:181 level=info msg="Starting Alertmanager" version="(version=0.27.0, branch=HEAD, revision=0aa3c2aad14cff039931923ab16b26b7481783b5)" 2024-09-14T03:57:51.541 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:57:51 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[101661]: ts=2024-09-14T03:57:51.238Z caller=main.go:182 level=info build_context="(go=go1.21.7, platform=linux/amd64, user=root@22cd11f671e9, date=20240228-11:51:20, tags=netgo)" 2024-09-14T03:57:51.541 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:57:51 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[101661]: ts=2024-09-14T03:57:51.239Z caller=cluster.go:186 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.39 port=9094 2024-09-14T03:57:51.541 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:57:51 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[101661]: ts=2024-09-14T03:57:51.241Z caller=cluster.go:683 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-09-14T03:57:51.541 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:57:51 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[101661]: ts=2024-09-14T03:57:51.263Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-14T03:57:51.541 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:57:51 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[101661]: ts=2024-09-14T03:57:51.263Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-09-14T03:57:51.541 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:57:51 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[101661]: ts=2024-09-14T03:57:51.265Z caller=tls_config.go:313 level=info msg="Listening on" address=[::]:9093 2024-09-14T03:57:51.541 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:57:51 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[101661]: ts=2024-09-14T03:57:51.265Z caller=tls_config.go:316 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-09-14T03:57:51.541 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:57:51 smithi039 bash[101646]: 6b564e1fc909fd6269e079207b1e58b491b75b1fc472c889595a37d94205a5fa 2024-09-14T03:57:51.541 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:57:51 smithi039 systemd[1]: Started Ceph alertmanager.a for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:57:52.111 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:51 smithi129 ceph-mon[33173]: pgmap v128: 137 pgs: 137 active+clean; 901 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 43 MiB/s rd, 12 MiB/s wr, 255 op/s 2024-09-14T03:57:52.111 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:51 smithi129 ceph-mon[33173]: osdmap e276: 8 total, 8 up, 8 in 2024-09-14T03:57:52.111 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:51 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:52.111 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:51 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:52.111 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:51 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:57:52.169 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:51 smithi039 ceph-mon[29586]: pgmap v128: 137 pgs: 137 active+clean; 901 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 43 MiB/s rd, 12 MiB/s wr, 255 op/s 2024-09-14T03:57:52.169 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:51 smithi039 ceph-mon[29586]: osdmap e276: 8 total, 8 up, 8 in 2024-09-14T03:57:52.169 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:51 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:52.169 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:51 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:52.170 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:51 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:57:52.170 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:51 smithi039 ceph-mon[40962]: pgmap v128: 137 pgs: 137 active+clean; 901 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 43 MiB/s rd, 12 MiB/s wr, 255 op/s 2024-09-14T03:57:52.170 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:51 smithi039 ceph-mon[40962]: osdmap e276: 8 total, 8 up, 8 in 2024-09-14T03:57:52.170 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:51 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:52.170 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:51 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:52.170 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:51 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:57:52.850 INFO:tasks.workunit.client.0.smithi039.stdout:snap_protect: snap1 2024-09-14T03:57:52.904 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:57:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:52 smithi129 ceph-mon[33173]: osdmap e277: 8 total, 8 up, 8 in 2024-09-14T03:57:53.173 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:52 smithi039 ceph-mon[29586]: osdmap e277: 8 total, 8 up, 8 in 2024-09-14T03:57:53.173 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:52 smithi039 ceph-mon[40962]: osdmap e277: 8 total, 8 up, 8 in 2024-09-14T03:57:53.540 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:57:53 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[101661]: ts=2024-09-14T03:57:53.241Z caller=cluster.go:708 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000040775s 2024-09-14T03:57:53.899 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:53 smithi129 ceph-mon[33173]: pgmap v131: 137 pgs: 137 active+clean; 901 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:57:53.900 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:53 smithi129 ceph-mon[33173]: osdmap e278: 8 total, 8 up, 8 in 2024-09-14T03:57:53.900 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:53 smithi129 ceph-mon[33173]: osdmap e279: 8 total, 8 up, 8 in 2024-09-14T03:57:53.931 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:57:54.006 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:57:54.152 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:53 smithi039 ceph-mon[29586]: pgmap v131: 137 pgs: 137 active+clean; 901 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:57:54.152 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:53 smithi039 ceph-mon[29586]: osdmap e278: 8 total, 8 up, 8 in 2024-09-14T03:57:54.152 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:53 smithi039 ceph-mon[29586]: osdmap e279: 8 total, 8 up, 8 in 2024-09-14T03:57:54.153 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:53 smithi039 ceph-mon[40962]: pgmap v131: 137 pgs: 137 active+clean; 901 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:57:54.153 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:53 smithi039 ceph-mon[40962]: osdmap e278: 8 total, 8 up, 8 in 2024-09-14T03:57:54.153 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:53 smithi039 ceph-mon[40962]: osdmap e279: 8 total, 8 up, 8 in 2024-09-14T03:57:54.973 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.CloneV2Parent (4050 ms) 2024-09-14T03:57:54.974 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.CloneV1ParentAbort 2024-09-14T03:57:55.006 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:57:55.022 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 0~10 2024-09-14T03:57:55.030 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap1 2024-09-14T03:57:56.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:55 smithi129 ceph-mon[33173]: pgmap v134: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 346 KiB/s rd, 20 KiB/s wr, 436 op/s 2024-09-14T03:57:56.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:55 smithi129 ceph-mon[33173]: osdmap e280: 8 total, 8 up, 8 in 2024-09-14T03:57:56.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:55 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:56.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:55 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:56.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:55 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:56.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:55 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:56.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:55 smithi039 ceph-mon[29586]: pgmap v134: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 346 KiB/s rd, 20 KiB/s wr, 436 op/s 2024-09-14T03:57:56.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:55 smithi039 ceph-mon[29586]: osdmap e280: 8 total, 8 up, 8 in 2024-09-14T03:57:56.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:55 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:56.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:55 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:56.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:55 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:56.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:55 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:56.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:55 smithi039 ceph-mon[40962]: pgmap v134: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 346 KiB/s rd, 20 KiB/s wr, 436 op/s 2024-09-14T03:57:56.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:55 smithi039 ceph-mon[40962]: osdmap e280: 8 total, 8 up, 8 in 2024-09-14T03:57:56.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:55 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:56.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:55 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:56.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:55 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:56.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:55 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:56.918 INFO:tasks.workunit.client.0.smithi039.stdout:snap_protect: snap1 2024-09-14T03:57:56.973 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:57:57.148 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:56 smithi039 ceph-mon[29586]: osdmap e281: 8 total, 8 up, 8 in 2024-09-14T03:57:57.148 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:56 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:57.148 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:56 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:57.148 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:56 smithi039 ceph-mon[40962]: osdmap e281: 8 total, 8 up, 8 in 2024-09-14T03:57:57.148 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:56 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:57.148 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:56 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:57.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:56 smithi129 ceph-mon[33173]: osdmap e281: 8 total, 8 up, 8 in 2024-09-14T03:57:57.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:56 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:57.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:56 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:57:58.091 INFO:tasks.workunit.client.0.smithi039.stdout:migration_abort 2024-09-14T03:57:58.268 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:57 smithi039 ceph-mon[29586]: pgmap v137: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 346 KiB/s rd, 20 KiB/s wr, 436 op/s 2024-09-14T03:57:58.268 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:57 smithi039 ceph-mon[29586]: osdmap e282: 8 total, 8 up, 8 in 2024-09-14T03:57:58.268 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:57 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:57:58.268 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:57 smithi039 ceph-mon[40962]: pgmap v137: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 346 KiB/s rd, 20 KiB/s wr, 436 op/s 2024-09-14T03:57:58.269 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:57 smithi039 ceph-mon[40962]: osdmap e282: 8 total, 8 up, 8 in 2024-09-14T03:57:58.269 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:57 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:57:58.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:57 smithi129 ceph-mon[33173]: pgmap v137: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 346 KiB/s rd, 20 KiB/s wr, 436 op/s 2024-09-14T03:57:58.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:57 smithi129 ceph-mon[33173]: osdmap e282: 8 total, 8 up, 8 in 2024-09-14T03:57:58.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:57 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:57:58.935 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:57:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:57:58] "GET /metrics HTTP/1.1" 200 38860 "" "Prometheus/2.51.0" 2024-09-14T03:57:59.068 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.CloneV1ParentAbort (4094 ms) 2024-09-14T03:57:59.068 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.CloneV2ParentAbort 2024-09-14T03:57:59.100 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:57:59.112 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 0~10 2024-09-14T03:57:59.121 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap1 2024-09-14T03:57:59.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:58 smithi129 ceph-mon[33173]: osdmap e283: 8 total, 8 up, 8 in 2024-09-14T03:57:59.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:58 smithi129 ceph-mon[33173]: mgrmap e34: y(active, since 91s), standbys: x 2024-09-14T03:57:59.281 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:58 smithi039 ceph-mon[29586]: osdmap e283: 8 total, 8 up, 8 in 2024-09-14T03:57:59.281 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:58 smithi039 ceph-mon[29586]: mgrmap e34: y(active, since 91s), standbys: x 2024-09-14T03:57:59.281 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:58 smithi039 ceph-mon[40962]: osdmap e283: 8 total, 8 up, 8 in 2024-09-14T03:57:59.281 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:58 smithi039 ceph-mon[40962]: mgrmap e34: y(active, since 91s), standbys: x 2024-09-14T03:58:00.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[29586]: pgmap v140: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 417 KiB/s rd, 27 KiB/s wr, 530 op/s 2024-09-14T03:58:00.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[29586]: osdmap e284: 8 total, 8 up, 8 in 2024-09-14T03:58:00.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:00.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:00.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:58:00.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:58:00.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:00.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:58:00.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:58:00.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[40962]: pgmap v140: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 417 KiB/s rd, 27 KiB/s wr, 530 op/s 2024-09-14T03:58:00.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[40962]: osdmap e284: 8 total, 8 up, 8 in 2024-09-14T03:58:00.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:00.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:00.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:58:00.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:58:00.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:00.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:58:00.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:58:00.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:57:59 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.244 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:59 smithi129 ceph-mon[33173]: pgmap v140: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 417 KiB/s rd, 27 KiB/s wr, 530 op/s 2024-09-14T03:58:00.244 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:59 smithi129 ceph-mon[33173]: osdmap e284: 8 total, 8 up, 8 in 2024-09-14T03:58:00.244 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:59 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:00.244 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:59 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:00.244 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:59 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:58:00.244 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:59 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:58:00.245 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:59 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:00.245 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:59 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:58:00.245 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:59 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:58:00.245 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:59 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.245 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:59 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.245 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:59 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.245 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:59 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.245 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:59 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.245 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:59 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.245 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:59 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.245 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:59 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.245 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:59 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.245 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:59 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.246 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:59 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.246 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:57:59 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:00.990 INFO:tasks.workunit.client.0.smithi039.stdout:snap_protect: snap1 2024-09-14T03:58:01.051 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:58:01.163 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:00 smithi129 ceph-mon[33173]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:58:01.163 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:00 smithi129 ceph-mon[33173]: osdmap e285: 8 total, 8 up, 8 in 2024-09-14T03:58:01.245 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:00 smithi039 ceph-mon[29586]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:58:01.245 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:00 smithi039 ceph-mon[29586]: osdmap e285: 8 total, 8 up, 8 in 2024-09-14T03:58:01.245 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:00 smithi039 ceph-mon[40962]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-09-14T03:58:01.245 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:00 smithi039 ceph-mon[40962]: osdmap e285: 8 total, 8 up, 8 in 2024-09-14T03:58:01.540 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 03:58:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[101661]: ts=2024-09-14T03:58:01.244Z caller=cluster.go:700 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.00279099s 2024-09-14T03:58:02.116 INFO:tasks.workunit.client.0.smithi039.stdout:migration_abort 2024-09-14T03:58:02.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:01 smithi129 ceph-mon[33173]: pgmap v143: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 417 KiB/s rd, 27 KiB/s wr, 530 op/s 2024-09-14T03:58:02.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:01 smithi129 ceph-mon[33173]: osdmap e286: 8 total, 8 up, 8 in 2024-09-14T03:58:02.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:01 smithi039 ceph-mon[29586]: pgmap v143: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 417 KiB/s rd, 27 KiB/s wr, 530 op/s 2024-09-14T03:58:02.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:01 smithi039 ceph-mon[29586]: osdmap e286: 8 total, 8 up, 8 in 2024-09-14T03:58:02.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:01 smithi039 ceph-mon[40962]: pgmap v143: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 417 KiB/s rd, 27 KiB/s wr, 530 op/s 2024-09-14T03:58:02.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:01 smithi039 ceph-mon[40962]: osdmap e286: 8 total, 8 up, 8 in 2024-09-14T03:58:03.146 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.CloneV2ParentAbort (4078 ms) 2024-09-14T03:58:03.146 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.CloneV1ParentAbortFixIncompleteChildReattach 2024-09-14T03:58:03.182 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:58:03.195 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 0~10 2024-09-14T03:58:03.204 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap1 2024-09-14T03:58:03.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:02 smithi129 ceph-mon[33173]: Upgrade: Updating grafana.a 2024-09-14T03:58:03.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:02 smithi129 ceph-mon[33173]: Deploying daemon grafana.a on smithi129 2024-09-14T03:58:03.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:02 smithi129 ceph-mon[33173]: osdmap e287: 8 total, 8 up, 8 in 2024-09-14T03:58:03.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:02 smithi129 ceph-mon[33173]: pgmap v146: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:58:03.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:02 smithi039 ceph-mon[29586]: Upgrade: Updating grafana.a 2024-09-14T03:58:03.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:02 smithi039 ceph-mon[29586]: Deploying daemon grafana.a on smithi129 2024-09-14T03:58:03.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:02 smithi039 ceph-mon[29586]: osdmap e287: 8 total, 8 up, 8 in 2024-09-14T03:58:03.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:02 smithi039 ceph-mon[29586]: pgmap v146: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:58:03.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:02 smithi039 ceph-mon[40962]: Upgrade: Updating grafana.a 2024-09-14T03:58:03.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:02 smithi039 ceph-mon[40962]: Deploying daemon grafana.a on smithi129 2024-09-14T03:58:03.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:02 smithi039 ceph-mon[40962]: osdmap e287: 8 total, 8 up, 8 in 2024-09-14T03:58:03.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:02 smithi039 ceph-mon[40962]: pgmap v146: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:58:04.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:03 smithi129 ceph-mon[33173]: osdmap e288: 8 total, 8 up, 8 in 2024-09-14T03:58:04.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:03 smithi129 ceph-mon[33173]: osdmap e289: 8 total, 8 up, 8 in 2024-09-14T03:58:04.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:03 smithi039 ceph-mon[29586]: osdmap e288: 8 total, 8 up, 8 in 2024-09-14T03:58:04.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:03 smithi039 ceph-mon[29586]: osdmap e289: 8 total, 8 up, 8 in 2024-09-14T03:58:04.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:03 smithi039 ceph-mon[40962]: osdmap e288: 8 total, 8 up, 8 in 2024-09-14T03:58:04.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:03 smithi039 ceph-mon[40962]: osdmap e289: 8 total, 8 up, 8 in 2024-09-14T03:58:04.759 INFO:tasks.workunit.client.0.smithi039.stdout:snap_protect: snap1 2024-09-14T03:58:04.814 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:58:05.935 INFO:tasks.workunit.client.0.smithi039.stdout:migration_abort 2024-09-14T03:58:06.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:05 smithi129 ceph-mon[33173]: pgmap v149: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 362 KiB/s rd, 25 KiB/s wr, 462 op/s 2024-09-14T03:58:06.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:05 smithi129 ceph-mon[33173]: osdmap e290: 8 total, 8 up, 8 in 2024-09-14T03:58:06.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:05 smithi039 ceph-mon[29586]: pgmap v149: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 362 KiB/s rd, 25 KiB/s wr, 462 op/s 2024-09-14T03:58:06.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:05 smithi039 ceph-mon[29586]: osdmap e290: 8 total, 8 up, 8 in 2024-09-14T03:58:06.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:05 smithi039 ceph-mon[40962]: pgmap v149: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 362 KiB/s rd, 25 KiB/s wr, 462 op/s 2024-09-14T03:58:06.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:05 smithi039 ceph-mon[40962]: osdmap e290: 8 total, 8 up, 8 in 2024-09-14T03:58:06.910 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.CloneV1ParentAbortFixIncompleteChildReattach (3764 ms) 2024-09-14T03:58:06.910 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.CloneV1ParentAbortFixParentReattach 2024-09-14T03:58:06.940 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:58:06.951 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 0~10 2024-09-14T03:58:06.960 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap1 2024-09-14T03:58:07.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:06 smithi039 ceph-mon[29586]: osdmap e291: 8 total, 8 up, 8 in 2024-09-14T03:58:07.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:06 smithi039 ceph-mon[40962]: osdmap e291: 8 total, 8 up, 8 in 2024-09-14T03:58:07.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:06 smithi129 ceph-mon[33173]: osdmap e291: 8 total, 8 up, 8 in 2024-09-14T03:58:08.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:07 smithi039 ceph-mon[29586]: pgmap v152: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 362 KiB/s rd, 25 KiB/s wr, 462 op/s 2024-09-14T03:58:08.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:07 smithi039 ceph-mon[29586]: osdmap e292: 8 total, 8 up, 8 in 2024-09-14T03:58:08.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:07 smithi039 ceph-mon[40962]: pgmap v152: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 362 KiB/s rd, 25 KiB/s wr, 462 op/s 2024-09-14T03:58:08.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:07 smithi039 ceph-mon[40962]: osdmap e292: 8 total, 8 up, 8 in 2024-09-14T03:58:08.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:07 smithi129 ceph-mon[33173]: pgmap v152: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 362 KiB/s rd, 25 KiB/s wr, 462 op/s 2024-09-14T03:58:08.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:07 smithi129 ceph-mon[33173]: osdmap e292: 8 total, 8 up, 8 in 2024-09-14T03:58:08.810 INFO:tasks.workunit.client.0.smithi039.stdout:snap_protect: snap1 2024-09-14T03:58:08.872 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:58:09.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:08 smithi039 ceph-mon[29586]: osdmap e293: 8 total, 8 up, 8 in 2024-09-14T03:58:09.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:08 smithi039 ceph-mon[40962]: osdmap e293: 8 total, 8 up, 8 in 2024-09-14T03:58:09.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:58:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:58:08] "GET /metrics HTTP/1.1" 200 38860 "" "Prometheus/2.51.0" 2024-09-14T03:58:09.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:08 smithi129 ceph-mon[33173]: osdmap e293: 8 total, 8 up, 8 in 2024-09-14T03:58:09.995 INFO:tasks.workunit.client.0.smithi039.stdout:migration_abort 2024-09-14T03:58:10.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:09 smithi129 ceph-mon[33173]: pgmap v155: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 326 KiB/s rd, 22 KiB/s wr, 424 op/s 2024-09-14T03:58:10.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:09 smithi129 ceph-mon[33173]: osdmap e294: 8 total, 8 up, 8 in 2024-09-14T03:58:10.119 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:09 smithi039 ceph-mon[29586]: pgmap v155: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 326 KiB/s rd, 22 KiB/s wr, 424 op/s 2024-09-14T03:58:10.120 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:09 smithi039 ceph-mon[29586]: osdmap e294: 8 total, 8 up, 8 in 2024-09-14T03:58:10.120 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:09 smithi039 ceph-mon[40962]: pgmap v155: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 326 KiB/s rd, 22 KiB/s wr, 424 op/s 2024-09-14T03:58:10.120 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:09 smithi039 ceph-mon[40962]: osdmap e294: 8 total, 8 up, 8 in 2024-09-14T03:58:10.120 INFO:teuthology.orchestra.run.smithi039.stdout:true 2024-09-14T03:58:10.701 INFO:teuthology.orchestra.run.smithi039.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:58:10.702 INFO:teuthology.orchestra.run.smithi039.stdout:alertmanager.a smithi039 *:9093,9094 running (19s) 14s ago 6m 16.4M - 0.27.0 11f11916f8cd 6b564e1fc909 2024-09-14T03:58:10.702 INFO:teuthology.orchestra.run.smithi039.stdout:grafana.a smithi129 *:3000 running (5m) 15s ago 5m 91.4M - 9.4.7 954c08fa6188 71f297945590 2024-09-14T03:58:10.702 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi039.bbwiwf smithi039 running (4m) 14s ago 4m 20.1M - 18.2.4-845-ge4e227b3 67ddd33fa72f 91e6c8f85681 2024-09-14T03:58:10.702 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi129.ybjqcw smithi129 running (4m) 15s ago 4m 14.9M - 18.2.4-845-ge4e227b3 67ddd33fa72f a3fab5b37b71 2024-09-14T03:58:10.702 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.x smithi129 *:8443,9283,8765 running (85s) 15s ago 10m 453M - 19.3.0-4907-ga2782315 8801d2529e43 448bd4e146cb 2024-09-14T03:58:10.702 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.y smithi039 *:8443,9283,8765 running (113s) 14s ago 11m 514M - 19.3.0-4907-ga2782315 8801d2529e43 cccc87f635a6 2024-09-14T03:58:10.702 INFO:teuthology.orchestra.run.smithi039.stdout:mon.a smithi039 running (11m) 14s ago 11m 61.6M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f e7b5f91fb09c 2024-09-14T03:58:10.702 INFO:teuthology.orchestra.run.smithi039.stdout:mon.b smithi129 running (10m) 15s ago 10m 53.8M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f eb38cc5cbe35 2024-09-14T03:58:10.702 INFO:teuthology.orchestra.run.smithi039.stdout:mon.c smithi039 running (10m) 14s ago 10m 52.2M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 31d440e49f93 2024-09-14T03:58:10.702 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.a smithi039 *:9100 running (69s) 14s ago 5m 9.90M - 1.7.0 72c9c2088986 7dd5945adb64 2024-09-14T03:58:10.702 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.b smithi129 *:9100 running (63s) 15s ago 5m 8032k - 1.7.0 72c9c2088986 1fedb5f257e7 2024-09-14T03:58:10.702 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0 smithi039 running (9m) 14s ago 9m 130M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 5048d74f27ac 2024-09-14T03:58:10.702 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1 smithi039 running (9m) 14s ago 9m 119M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 91349527d88b 2024-09-14T03:58:10.703 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2 smithi039 running (9m) 14s ago 9m 105M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f eb903b44fc5d 2024-09-14T03:58:10.703 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3 smithi039 running (8m) 14s ago 8m 119M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f dff9e665bd9b 2024-09-14T03:58:10.703 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4 smithi129 running (8m) 15s ago 8m 110M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 5b64727311b7 2024-09-14T03:58:10.703 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5 smithi129 running (7m) 15s ago 7m 94.5M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f f2d991a754f7 2024-09-14T03:58:10.703 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6 smithi129 running (7m) 15s ago 7m 136M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 3060eaf44106 2024-09-14T03:58:10.703 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7 smithi129 running (7m) 15s ago 7m 96.1M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 4c13b97060d1 2024-09-14T03:58:10.703 INFO:teuthology.orchestra.run.smithi039.stdout:prometheus.a smithi129 *:9095 running (40s) 15s ago 6m 35.4M - 2.51.0 1d3b7f56885b cbe583f852c3 2024-09-14T03:58:10.937 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:10 smithi129 ceph-mon[33173]: osdmap e295: 8 total, 8 up, 8 in 2024-09-14T03:58:10.951 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.CloneV1ParentAbortFixParentReattach (4042 ms) 2024-09-14T03:58:10.951 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.CloneV1ParentAbortRelinkNotNeeded 2024-09-14T03:58:10.989 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:58:11.001 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 0~10 2024-09-14T03:58:11.009 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap1 2024-09-14T03:58:11.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:10 smithi039 ceph-mon[29586]: osdmap e295: 8 total, 8 up, 8 in 2024-09-14T03:58:11.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:10 smithi039 ceph-mon[40962]: osdmap e295: 8 total, 8 up, 8 in 2024-09-14T03:58:11.086 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T03:58:11.087 INFO:teuthology.orchestra.run.smithi039.stdout: "mon": { 2024-09-14T03:58:11.087 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 3 2024-09-14T03:58:11.087 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:58:11.087 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr": { 2024-09-14T03:58:11.087 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:58:11.087 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:58:11.087 INFO:teuthology.orchestra.run.smithi039.stdout: "osd": { 2024-09-14T03:58:11.087 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8 2024-09-14T03:58:11.087 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:58:11.087 INFO:teuthology.orchestra.run.smithi039.stdout: "mds": { 2024-09-14T03:58:11.087 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:58:11.087 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:58:11.088 INFO:teuthology.orchestra.run.smithi039.stdout: "overall": { 2024-09-14T03:58:11.088 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 13, 2024-09-14T03:58:11.088 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:58:11.088 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T03:58:11.088 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T03:58:11.386 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T03:58:11.386 INFO:teuthology.orchestra.run.smithi039.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:58:11.387 INFO:teuthology.orchestra.run.smithi039.stdout: "in_progress": true, 2024-09-14T03:58:11.387 INFO:teuthology.orchestra.run.smithi039.stdout: "which": "Upgrading daemons of type(s) mgr", 2024-09-14T03:58:11.387 INFO:teuthology.orchestra.run.smithi039.stdout: "services_complete": [ 2024-09-14T03:58:11.387 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr" 2024-09-14T03:58:11.387 INFO:teuthology.orchestra.run.smithi039.stdout: ], 2024-09-14T03:58:11.387 INFO:teuthology.orchestra.run.smithi039.stdout: "progress": "2/2 daemons upgraded", 2024-09-14T03:58:11.387 INFO:teuthology.orchestra.run.smithi039.stdout: "message": "Currently upgrading grafana daemons", 2024-09-14T03:58:11.387 INFO:teuthology.orchestra.run.smithi039.stdout: "is_paused": false 2024-09-14T03:58:11.387 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T03:58:12.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:11 smithi129 ceph-mon[33173]: from='client.25082 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:12.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:11 smithi129 ceph-mon[33173]: pgmap v158: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 326 KiB/s rd, 22 KiB/s wr, 424 op/s 2024-09-14T03:58:12.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:11 smithi129 ceph-mon[33173]: from='client.25088 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:12.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:11 smithi129 ceph-mon[33173]: from='client.25094 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:12.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:11 smithi129 ceph-mon[33173]: osdmap e296: 8 total, 8 up, 8 in 2024-09-14T03:58:12.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:11 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/339820420' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:12.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:11 smithi039 ceph-mon[40962]: from='client.25082 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:12.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:11 smithi039 ceph-mon[40962]: pgmap v158: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 326 KiB/s rd, 22 KiB/s wr, 424 op/s 2024-09-14T03:58:12.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:11 smithi039 ceph-mon[40962]: from='client.25088 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:12.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:11 smithi039 ceph-mon[40962]: from='client.25094 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:12.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:11 smithi039 ceph-mon[40962]: osdmap e296: 8 total, 8 up, 8 in 2024-09-14T03:58:12.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:11 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/339820420' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:12.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:11 smithi039 ceph-mon[29586]: from='client.25082 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:12.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:11 smithi039 ceph-mon[29586]: pgmap v158: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 326 KiB/s rd, 22 KiB/s wr, 424 op/s 2024-09-14T03:58:12.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:11 smithi039 ceph-mon[29586]: from='client.25088 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:12.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:11 smithi039 ceph-mon[29586]: from='client.25094 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:12.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:11 smithi039 ceph-mon[29586]: osdmap e296: 8 total, 8 up, 8 in 2024-09-14T03:58:12.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:11 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/339820420' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:12.321 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:12 smithi129 systemd[1]: Stopping Ceph grafana.a for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:58:12.771 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:12 smithi129 podman[93012]: 2024-09-14 03:58:12.320100587 +0000 UTC m=+0.126373335 container died 71f29794559040dddb6fa71feb0552a20620bce62beb18a63fe6421b3951123f (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, release=239.1651231664, io.openshift.tags=base rhel8, distribution-scope=public, vcs-type=git, description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, build-date=2022-04-29T11:48:11.055897, version=8.5, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, com.redhat.component=ubi8-container, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64) 2024-09-14T03:58:12.902 INFO:tasks.workunit.client.0.smithi039.stdout:snap_protect: snap1 2024-09-14T03:58:12.965 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:58:13.150 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:12 smithi129 bash[93012]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a 2024-09-14T03:58:13.151 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:12 smithi129 podman[93012]: 2024-09-14 03:58:12.873754578 +0000 UTC m=+0.680027320 container cleanup 71f29794559040dddb6fa71feb0552a20620bce62beb18a63fe6421b3951123f (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a, maintainer=Paul Cuzner , io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, build-date=2022-04-29T11:48:11.055897, version=8.5, distribution-scope=public, description=Ceph Grafana Container, io.openshift.expose-services=) 2024-09-14T03:58:13.151 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:12 smithi129 ceph-mon[33173]: from='client.25106 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:13.151 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:12 smithi129 ceph-mon[33173]: osdmap e297: 8 total, 8 up, 8 in 2024-09-14T03:58:13.151 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:12 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:58:13.293 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:12 smithi039 ceph-mon[29586]: from='client.25106 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:13.293 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:12 smithi039 ceph-mon[29586]: osdmap e297: 8 total, 8 up, 8 in 2024-09-14T03:58:13.293 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:12 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:58:13.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:12 smithi039 ceph-mon[40962]: from='client.25106 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:13.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:12 smithi039 ceph-mon[40962]: osdmap e297: 8 total, 8 up, 8 in 2024-09-14T03:58:13.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:12 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:58:13.522 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:13 smithi129 podman[93032]: 2024-09-14 03:58:13.150000157 +0000 UTC m=+0.832321498 container remove 71f29794559040dddb6fa71feb0552a20620bce62beb18a63fe6421b3951123f (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, io.openshift.tags=base rhel8, name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, vcs-type=git, description=Ceph Grafana Container, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , vendor=Red Hat, Inc.) 2024-09-14T03:58:13.522 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:13 smithi129 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-09-14T03:58:13.522 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:13 smithi129 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-09-14T03:58:13.522 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:13 smithi129 systemd[1]: Stopped Ceph grafana.a for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:58:13.522 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:13 smithi129 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@grafana.a.service: Consumed 7.714s CPU time. 2024-09-14T03:58:13.875 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:13 smithi129 systemd[1]: Starting Ceph grafana.a for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:58:13.876 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:13 smithi129 podman[93164]: 2024-09-14 03:58:13.633523384 +0000 UTC m=+0.015992441 image pull c8b91775d855b99270fc5d22f3c6737e8cca01ef4c25c8b0362295e0746fa39b quay.io/ceph/grafana:10.4.0 2024-09-14T03:58:13.931 INFO:tasks.workunit.client.0.smithi039.stdout:migration_abort 2024-09-14T03:58:14.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:13 smithi129 ceph-mon[33173]: pgmap v161: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:58:14.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:13 smithi129 ceph-mon[33173]: osdmap e298: 8 total, 8 up, 8 in 2024-09-14T03:58:14.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:13 smithi129 ceph-mon[33173]: osdmap e299: 8 total, 8 up, 8 in 2024-09-14T03:58:14.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:13 smithi039 ceph-mon[29586]: pgmap v161: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:58:14.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:13 smithi039 ceph-mon[29586]: osdmap e298: 8 total, 8 up, 8 in 2024-09-14T03:58:14.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:13 smithi039 ceph-mon[29586]: osdmap e299: 8 total, 8 up, 8 in 2024-09-14T03:58:14.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:13 smithi039 ceph-mon[40962]: pgmap v161: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:58:14.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:13 smithi039 ceph-mon[40962]: osdmap e298: 8 total, 8 up, 8 in 2024-09-14T03:58:14.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:13 smithi039 ceph-mon[40962]: osdmap e299: 8 total, 8 up, 8 in 2024-09-14T03:58:14.771 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:14 smithi129 podman[93164]: 2024-09-14 03:58:14.57708769 +0000 UTC m=+0.959556734 container create f0492c19de92f81c5baae4551bf1c20aa26f85e6173189cea5fb8ca3ba1a1730 (image=quay.io/ceph/grafana:10.4.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-14T03:58:15.023 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.CloneV1ParentAbortRelinkNotNeeded (4072 ms) 2024-09-14T03:58:15.023 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.CloneV2ParentAbortFixIncompleteChildReattach 2024-09-14T03:58:15.057 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:58:15.071 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 0~10 2024-09-14T03:58:15.079 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap1 2024-09-14T03:58:16.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:15 smithi129 ceph-mon[33173]: pgmap v164: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 269 KiB/s rd, 17 KiB/s wr, 345 op/s 2024-09-14T03:58:16.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:15 smithi129 ceph-mon[33173]: osdmap e300: 8 total, 8 up, 8 in 2024-09-14T03:58:16.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:15 smithi039 ceph-mon[29586]: pgmap v164: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 269 KiB/s rd, 17 KiB/s wr, 345 op/s 2024-09-14T03:58:16.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:15 smithi039 ceph-mon[29586]: osdmap e300: 8 total, 8 up, 8 in 2024-09-14T03:58:16.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:15 smithi039 ceph-mon[40962]: pgmap v164: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 269 KiB/s rd, 17 KiB/s wr, 345 op/s 2024-09-14T03:58:16.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:15 smithi039 ceph-mon[40962]: osdmap e300: 8 total, 8 up, 8 in 2024-09-14T03:58:16.735 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 podman[93164]: 2024-09-14 03:58:16.483185681 +0000 UTC m=+2.865654726 container init f0492c19de92f81c5baae4551bf1c20aa26f85e6173189cea5fb8ca3ba1a1730 (image=quay.io/ceph/grafana:10.4.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-14T03:58:16.735 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 podman[93164]: 2024-09-14 03:58:16.489921341 +0000 UTC m=+2.872390383 container start f0492c19de92f81c5baae4551bf1c20aa26f85e6173189cea5fb8ca3ba1a1730 (image=quay.io/ceph/grafana:10.4.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-14T03:58:16.735 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=settings t=2024-09-14T03:58:16.613467615Z level=info msg="Starting Grafana" version=10.4.0 commit=03f502a94d17f7dc4e6c34acdf8428aedd986e4c branch=HEAD compiled=2024-09-14T03:58:16Z 2024-09-14T03:58:16.735 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=settings t=2024-09-14T03:58:16.6136959Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-09-14T03:58:16.735 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=settings t=2024-09-14T03:58:16.613703626Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-09-14T03:58:16.735 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=settings t=2024-09-14T03:58:16.613706805Z level=info msg="Config overridden from command line" arg="default.paths.data=/var/lib/grafana" 2024-09-14T03:58:16.735 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=settings t=2024-09-14T03:58:16.613709496Z level=info msg="Config overridden from command line" arg="default.paths.logs=/var/log/grafana" 2024-09-14T03:58:16.736 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=settings t=2024-09-14T03:58:16.613712144Z level=info msg="Config overridden from command line" arg="default.paths.plugins=/var/lib/grafana/plugins" 2024-09-14T03:58:16.736 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=settings t=2024-09-14T03:58:16.61371463Z level=info msg="Config overridden from command line" arg="default.paths.provisioning=/etc/grafana/provisioning" 2024-09-14T03:58:16.736 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=settings t=2024-09-14T03:58:16.613717143Z level=info msg="Config overridden from command line" arg="default.log.mode=console" 2024-09-14T03:58:16.736 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=settings t=2024-09-14T03:58:16.613719888Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-09-14T03:58:16.736 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=settings t=2024-09-14T03:58:16.613722777Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-09-14T03:58:16.736 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=settings t=2024-09-14T03:58:16.613726317Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-09-14T03:58:16.736 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=settings t=2024-09-14T03:58:16.613728767Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-09-14T03:58:16.736 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=settings t=2024-09-14T03:58:16.613731428Z level=info msg=Target target=[all] 2024-09-14T03:58:16.736 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=settings t=2024-09-14T03:58:16.613740116Z level=info msg="Path Home" path=/usr/share/grafana 2024-09-14T03:58:16.736 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=settings t=2024-09-14T03:58:16.613742546Z level=info msg="Path Data" path=/var/lib/grafana 2024-09-14T03:58:16.737 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=settings t=2024-09-14T03:58:16.613745004Z level=info msg="Path Logs" path=/var/log/grafana 2024-09-14T03:58:16.737 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=settings t=2024-09-14T03:58:16.613748056Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-09-14T03:58:16.737 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=settings t=2024-09-14T03:58:16.613750753Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-09-14T03:58:16.738 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=settings t=2024-09-14T03:58:16.613753247Z level=info msg="App mode production" 2024-09-14T03:58:16.738 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=sqlstore t=2024-09-14T03:58:16.61399148Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-09-14T03:58:16.738 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=sqlstore t=2024-09-14T03:58:16.614003898Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-09-14T03:58:16.738 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.61445508Z level=info msg="Starting DB migrations" 2024-09-14T03:58:16.738 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.625597674Z level=info msg="Executing migration" id="Add uid column to user" 2024-09-14T03:58:16.738 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.637384931Z level=info msg="Migration successfully executed" id="Add uid column to user" duration=11.786193ms 2024-09-14T03:58:16.738 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.637779645Z level=info msg="Executing migration" id="Update uid column values for users" 2024-09-14T03:58:16.738 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.637906881Z level=info msg="Migration successfully executed" id="Update uid column values for users" duration=127.445µs 2024-09-14T03:58:16.738 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.638299109Z level=info msg="Executing migration" id="Add unique index user_uid" 2024-09-14T03:58:16.738 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.639028415Z level=info msg="Migration successfully executed" id="Add unique index user_uid" duration=728.751µs 2024-09-14T03:58:16.739 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.641173594Z level=info msg="Executing migration" id="Add playlist column created_at" 2024-09-14T03:58:16.739 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.646087779Z level=info msg="Migration successfully executed" id="Add playlist column created_at" duration=4.913348ms 2024-09-14T03:58:16.739 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.646472392Z level=info msg="Executing migration" id="Add playlist column updated_at" 2024-09-14T03:58:16.739 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.651622691Z level=info msg="Migration successfully executed" id="Add playlist column updated_at" duration=5.150079ms 2024-09-14T03:58:16.739 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.652071842Z level=info msg="Executing migration" id="Add column uid in team" 2024-09-14T03:58:16.739 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.657057607Z level=info msg="Migration successfully executed" id="Add column uid in team" duration=4.984971ms 2024-09-14T03:58:16.739 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.657403125Z level=info msg="Executing migration" id="Update uid column values in team" 2024-09-14T03:58:16.739 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.657519129Z level=info msg="Migration successfully executed" id="Update uid column values in team" duration=116.169µs 2024-09-14T03:58:16.739 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.658311008Z level=info msg="Executing migration" id="Add unique index team_org_id_uid" 2024-09-14T03:58:16.739 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.659063974Z level=info msg="Migration successfully executed" id="Add unique index team_org_id_uid" duration=752.989µs 2024-09-14T03:58:16.739 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.659491182Z level=info msg="Executing migration" id="add index user_auth_token.revoked_at" 2024-09-14T03:58:16.740 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.660215492Z level=info msg="Migration successfully executed" id="add index user_auth_token.revoked_at" duration=723.782µs 2024-09-14T03:58:16.740 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.660611859Z level=info msg="Executing migration" id="add result_fingerprint column to alert_instance" 2024-09-14T03:58:16.740 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.665835965Z level=info msg="Migration successfully executed" id="add result_fingerprint column to alert_instance" duration=5.223392ms 2024-09-14T03:58:16.740 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.666264306Z level=info msg="Executing migration" id="add last_applied column to alert_configuration_history" 2024-09-14T03:58:16.740 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.671139275Z level=info msg="Migration successfully executed" id="add last_applied column to alert_configuration_history" duration=4.87429ms 2024-09-14T03:58:16.740 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.671525472Z level=info msg="Executing migration" id="alter library_element model to mediumtext" 2024-09-14T03:58:16.740 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.671562979Z level=info msg="Migration successfully executed" id="alter library_element model to mediumtext" duration=38.189µs 2024-09-14T03:58:16.740 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.672578238Z level=info msg="Executing migration" id="permission kind migration" 2024-09-14T03:58:16.740 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.677521467Z level=info msg="Migration successfully executed" id="permission kind migration" duration=4.942987ms 2024-09-14T03:58:16.740 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.677880547Z level=info msg="Executing migration" id="permission attribute migration" 2024-09-14T03:58:16.740 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.682907178Z level=info msg="Migration successfully executed" id="permission attribute migration" duration=5.026728ms 2024-09-14T03:58:16.740 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.683277816Z level=info msg="Executing migration" id="permission identifier migration" 2024-09-14T03:58:16.741 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.688628899Z level=info msg="Migration successfully executed" id="permission identifier migration" duration=5.350864ms 2024-09-14T03:58:16.741 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.688990481Z level=info msg="Executing migration" id="add permission identifier index" 2024-09-14T03:58:16.741 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.6897742Z level=info msg="Migration successfully executed" id="add permission identifier index" duration=783.639µs 2024-09-14T03:58:16.741 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.690163236Z level=info msg="Executing migration" id="add permission action scope role_id index" 2024-09-14T03:58:16.741 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.690971328Z level=info msg="Migration successfully executed" id="add permission action scope role_id index" duration=808.304µs 2024-09-14T03:58:16.741 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.691322774Z level=info msg="Executing migration" id="remove permission role_id action scope index" 2024-09-14T03:58:16.741 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.692071287Z level=info msg="Migration successfully executed" id="remove permission role_id action scope index" duration=748.33µs 2024-09-14T03:58:16.741 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.692446758Z level=info msg="Executing migration" id="rbac disabled migrator" 2024-09-14T03:58:16.741 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.69247298Z level=info msg="Migration successfully executed" id="rbac disabled migrator" duration=26.873µs 2024-09-14T03:58:16.741 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.693308249Z level=info msg="Executing migration" id="drop index IDX_correlation_uid - v1" 2024-09-14T03:58:16.741 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.694021396Z level=info msg="Migration successfully executed" id="drop index IDX_correlation_uid - v1" duration=713.065µs 2024-09-14T03:58:16.741 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.694363192Z level=info msg="Executing migration" id="drop index IDX_correlation_source_uid - v1" 2024-09-14T03:58:16.742 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.695059762Z level=info msg="Migration successfully executed" id="drop index IDX_correlation_source_uid - v1" duration=695.883µs 2024-09-14T03:58:16.742 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.69540098Z level=info msg="Executing migration" id="Rename table correlation to correlation_tmp_qwerty - v1" 2024-09-14T03:58:16.742 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.710616711Z level=info msg="Migration successfully executed" id="Rename table correlation to correlation_tmp_qwerty - v1" duration=15.214516ms 2024-09-14T03:58:16.742 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.711028359Z level=info msg="Executing migration" id="create correlation v2" 2024-09-14T03:58:16.742 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.711727896Z level=info msg="Migration successfully executed" id="create correlation v2" duration=697.393µs 2024-09-14T03:58:16.742 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.712101646Z level=info msg="Executing migration" id="create index IDX_correlation_uid - v2" 2024-09-14T03:58:16.742 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.71279983Z level=info msg="Migration successfully executed" id="create index IDX_correlation_uid - v2" duration=698.146µs 2024-09-14T03:58:16.742 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.71314935Z level=info msg="Executing migration" id="create index IDX_correlation_source_uid - v2" 2024-09-14T03:58:16.742 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.713831207Z level=info msg="Migration successfully executed" id="create index IDX_correlation_source_uid - v2" duration=681.626µs 2024-09-14T03:58:16.742 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.714205809Z level=info msg="Executing migration" id="create index IDX_correlation_org_id - v2" 2024-09-14T03:58:16.742 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.714889024Z level=info msg="Migration successfully executed" id="create index IDX_correlation_org_id - v2" duration=683.311µs 2024-09-14T03:58:16.742 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.715282621Z level=info msg="Executing migration" id="copy correlation v1 to v2" 2024-09-14T03:58:16.743 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.715468912Z level=info msg="Migration successfully executed" id="copy correlation v1 to v2" duration=186.326µs 2024-09-14T03:58:16.743 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.716116392Z level=info msg="Executing migration" id="drop correlation_tmp_qwerty" 2024-09-14T03:58:16.743 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.71674869Z level=info msg="Migration successfully executed" id="drop correlation_tmp_qwerty" duration=632.465µs 2024-09-14T03:58:16.743 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.717109948Z level=info msg="Executing migration" id="add provisioning column" 2024-09-14T03:58:16.743 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.722300215Z level=info msg="Migration successfully executed" id="add provisioning column" duration=5.189909ms 2024-09-14T03:58:16.743 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.722671957Z level=info msg="Executing migration" id="backfill empty share column fields with default of public" 2024-09-14T03:58:16.743 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.722807801Z level=info msg="Migration successfully executed" id="backfill empty share column fields with default of public" duration=135.469µs 2024-09-14T03:58:16.743 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.723531559Z level=info msg="Executing migration" id="migrate contents column to mediumblob for MySQL" 2024-09-14T03:58:16.743 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.723576699Z level=info msg="Migration successfully executed" id="migrate contents column to mediumblob for MySQL" duration=44.544µs 2024-09-14T03:58:16.743 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.724411494Z level=info msg="Executing migration" id="add origin column to seed_assignment" 2024-09-14T03:58:16.743 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.729785465Z level=info msg="Migration successfully executed" id="add origin column to seed_assignment" duration=5.374592ms 2024-09-14T03:58:16.743 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.730148141Z level=info msg="Executing migration" id="add origin to plugin seed_assignment" 2024-09-14T03:58:16.743 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.730384476Z level=info msg="Migration successfully executed" id="add origin to plugin seed_assignment" duration=236.859µs 2024-09-14T03:58:16.743 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.731003849Z level=info msg="Executing migration" id="prevent seeding OnCall access" 2024-09-14T03:58:16.744 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.731142946Z level=info msg="Migration successfully executed" id="prevent seeding OnCall access" duration=139.023µs 2024-09-14T03:58:16.744 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.731841532Z level=info msg="Executing migration" id="managed folder permissions library panel actions migration" 2024-09-14T03:58:16.744 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.732761374Z level=info msg="Migration successfully executed" id="managed folder permissions library panel actions migration" duration=918.54µs 2024-09-14T03:58:16.744 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.733123984Z level=info msg="Executing migration" id="Remove unique index for folder.title and folder.parent_uid" 2024-09-14T03:58:16.744 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.733846529Z level=info msg="Migration successfully executed" id="Remove unique index for folder.title and folder.parent_uid" duration=722.412µs 2024-09-14T03:58:16.744 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.734228509Z level=info msg="Executing migration" id="Add unique index for title, parent_uid, and org_id" 2024-09-14T03:58:16.744 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.735061194Z level=info msg="Migration successfully executed" id="Add unique index for title, parent_uid, and org_id" duration=831.803µs 2024-09-14T03:58:16.744 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.735430614Z level=info msg="Executing migration" id="Sync dashboard and folder table" 2024-09-14T03:58:16.744 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.735895895Z level=info msg="Migration successfully executed" id="Sync dashboard and folder table" duration=464.813µs 2024-09-14T03:58:16.744 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.736246881Z level=info msg="Executing migration" id="Remove ghost folders from the folder table" 2024-09-14T03:58:16.744 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.736438832Z level=info msg="Migration successfully executed" id="Remove ghost folders from the folder table" duration=190.237µs 2024-09-14T03:58:16.933 INFO:tasks.workunit.client.0.smithi039.stdout:snap_protect: snap1 2024-09-14T03:58:16.998 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:58:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-mon[33173]: osdmap e301: 8 total, 8 up, 8 in 2024-09-14T03:58:17.021 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.737061982Z level=info msg="Executing migration" id="Remove unique index UQE_folder_uid_org_id" 2024-09-14T03:58:17.022 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.737718702Z level=info msg="Migration successfully executed" id="Remove unique index UQE_folder_uid_org_id" duration=656.535µs 2024-09-14T03:58:17.022 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.738045274Z level=info msg="Executing migration" id="Add unique index UQE_folder_org_id_uid" 2024-09-14T03:58:17.022 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.738723434Z level=info msg="Migration successfully executed" id="Add unique index UQE_folder_org_id_uid" duration=677.843µs 2024-09-14T03:58:17.022 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.739058592Z level=info msg="Executing migration" id="Remove unique index UQE_folder_title_parent_uid_org_id" 2024-09-14T03:58:17.022 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.739708784Z level=info msg="Migration successfully executed" id="Remove unique index UQE_folder_title_parent_uid_org_id" duration=650.12µs 2024-09-14T03:58:17.022 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.740040092Z level=info msg="Executing migration" id="Add unique index UQE_folder_org_id_parent_uid_title" 2024-09-14T03:58:17.022 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.740721156Z level=info msg="Migration successfully executed" id="Add unique index UQE_folder_org_id_parent_uid_title" duration=680.719µs 2024-09-14T03:58:17.022 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.741050936Z level=info msg="Executing migration" id="Remove index IDX_folder_parent_uid_org_id" 2024-09-14T03:58:17.022 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.741701639Z level=info msg="Migration successfully executed" id="Remove index IDX_folder_parent_uid_org_id" duration=650.656µs 2024-09-14T03:58:17.022 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.74203386Z level=info msg="Executing migration" id="create anon_device table" 2024-09-14T03:58:17.022 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.742590717Z level=info msg="Migration successfully executed" id="create anon_device table" duration=555.499µs 2024-09-14T03:58:17.023 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.742946993Z level=info msg="Executing migration" id="add unique index anon_device.device_id" 2024-09-14T03:58:17.023 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.743721576Z level=info msg="Migration successfully executed" id="add unique index anon_device.device_id" duration=774.655µs 2024-09-14T03:58:17.023 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.744066837Z level=info msg="Executing migration" id="add index anon_device.updated_at" 2024-09-14T03:58:17.023 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.744733173Z level=info msg="Migration successfully executed" id="add index anon_device.updated_at" duration=665.834µs 2024-09-14T03:58:17.023 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.745113716Z level=info msg="Executing migration" id="create signing_key table" 2024-09-14T03:58:17.024 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.745668335Z level=info msg="Migration successfully executed" id="create signing_key table" duration=554.477µs 2024-09-14T03:58:17.024 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.746020809Z level=info msg="Executing migration" id="add unique index signing_key.key_id" 2024-09-14T03:58:17.024 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.746685073Z level=info msg="Migration successfully executed" id="add unique index signing_key.key_id" duration=664.042µs 2024-09-14T03:58:17.024 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.747022429Z level=info msg="Executing migration" id="set legacy alert migration status in kvstore" 2024-09-14T03:58:17.024 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.747664636Z level=info msg="Migration successfully executed" id="set legacy alert migration status in kvstore" duration=642.199µs 2024-09-14T03:58:17.025 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.747985747Z level=info msg="Executing migration" id="migrate record of created folders during legacy migration to kvstore" 2024-09-14T03:58:17.025 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.748410615Z level=info msg="Migration successfully executed" id="migrate record of created folders during legacy migration to kvstore" duration=406.09µs 2024-09-14T03:58:17.025 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.748802664Z level=info msg="Executing migration" id="Add folder_uid for dashboard" 2024-09-14T03:58:17.025 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.755261077Z level=info msg="Migration successfully executed" id="Add folder_uid for dashboard" duration=6.458093ms 2024-09-14T03:58:17.025 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.755613467Z level=info msg="Executing migration" id="Populate dashboard folder_uid column" 2024-09-14T03:58:17.025 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.75728008Z level=info msg="Migration successfully executed" id="Populate dashboard folder_uid column" duration=1.667198ms 2024-09-14T03:58:17.025 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.757798836Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_folder_uid_title" 2024-09-14T03:58:17.025 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.758708362Z level=info msg="Migration successfully executed" id="Add unique index for dashboard_org_id_folder_uid_title" duration=909.182µs 2024-09-14T03:58:17.025 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.759041057Z level=info msg="Executing migration" id="Delete unique index for dashboard_org_id_folder_id_title" 2024-09-14T03:58:17.025 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.759737335Z level=info msg="Migration successfully executed" id="Delete unique index for dashboard_org_id_folder_id_title" duration=696.034µs 2024-09-14T03:58:17.025 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.760074931Z level=info msg="Executing migration" id="Delete unique index for dashboard_org_id_folder_uid_title" 2024-09-14T03:58:17.026 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.760762798Z level=info msg="Migration successfully executed" id="Delete unique index for dashboard_org_id_folder_uid_title" duration=688.177µs 2024-09-14T03:58:17.026 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.761117429Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_folder_uid_title_is_folder" 2024-09-14T03:58:17.026 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.7619624Z level=info msg="Migration successfully executed" id="Add unique index for dashboard_org_id_folder_uid_title_is_folder" duration=845.031µs 2024-09-14T03:58:17.026 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.762307352Z level=info msg="Executing migration" id="Restore index for dashboard_org_id_folder_id_title" 2024-09-14T03:58:17.026 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.763089506Z level=info msg="Migration successfully executed" id="Restore index for dashboard_org_id_folder_id_title" duration=781.769µs 2024-09-14T03:58:17.026 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.763504578Z level=info msg="Executing migration" id="create sso_setting table" 2024-09-14T03:58:17.026 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.76425805Z level=info msg="Migration successfully executed" id="create sso_setting table" duration=753.802µs 2024-09-14T03:58:17.026 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.764633759Z level=info msg="Executing migration" id="copy kvstore migration status to each org" 2024-09-14T03:58:17.026 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.765287099Z level=info msg="Migration successfully executed" id="copy kvstore migration status to each org" duration=653.595µs 2024-09-14T03:58:17.026 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.765638001Z level=info msg="Executing migration" id="add back entry for orgid=0 migrated status" 2024-09-14T03:58:17.026 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.766044869Z level=info msg="Migration successfully executed" id="add back entry for orgid=0 migrated status" duration=406.82µs 2024-09-14T03:58:17.026 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.766374307Z level=info msg="Executing migration" id="alter kv_store.value to longtext" 2024-09-14T03:58:17.027 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.766412419Z level=info msg="Migration successfully executed" id="alter kv_store.value to longtext" duration=39.601µs 2024-09-14T03:58:17.027 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.766802921Z level=info msg="Executing migration" id="add notification_settings column to alert_rule table" 2024-09-14T03:58:17.027 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.772066496Z level=info msg="Migration successfully executed" id="add notification_settings column to alert_rule table" duration=5.261802ms 2024-09-14T03:58:17.027 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.772404666Z level=info msg="Executing migration" id="add notification_settings column to alert_rule_version table" 2024-09-14T03:58:17.027 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.778153594Z level=info msg="Migration successfully executed" id="add notification_settings column to alert_rule_version table" duration=5.7498ms 2024-09-14T03:58:17.027 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.778489341Z level=info msg="Executing migration" id="removing scope from alert.instances:read action migration" 2024-09-14T03:58:17.027 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.778700487Z level=info msg="Migration successfully executed" id="removing scope from alert.instances:read action migration" duration=211.259µs 2024-09-14T03:58:17.027 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=migrator t=2024-09-14T03:58:16.779065155Z level=info msg="migrations completed" performed=65 skipped=482 duration=153.511487ms 2024-09-14T03:58:17.027 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=sqlstore t=2024-09-14T03:58:16.779738685Z level=info msg="Created default organization" 2024-09-14T03:58:17.027 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=secrets t=2024-09-14T03:58:16.781745311Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-09-14T03:58:17.027 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=plugin.store t=2024-09-14T03:58:16.797209729Z level=info msg="Loading plugins..." 2024-09-14T03:58:17.028 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=local.finder t=2024-09-14T03:58:16.83969417Z level=warn msg="Skipping finding plugins as directory does not exist" path=/usr/share/grafana/plugins-bundled 2024-09-14T03:58:17.028 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=plugin.store t=2024-09-14T03:58:16.839716223Z level=info msg="Plugins loaded" count=55 duration=42.510168ms 2024-09-14T03:58:17.028 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=query_data t=2024-09-14T03:58:16.841754637Z level=info msg="Query Service initialization" 2024-09-14T03:58:17.028 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=live.push_http t=2024-09-14T03:58:16.843943707Z level=info msg="Live Push Gateway initialization" 2024-09-14T03:58:17.028 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=ngalert.migration t=2024-09-14T03:58:16.844929165Z level=info msg=Starting 2024-09-14T03:58:17.028 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=ngalert.state.manager t=2024-09-14T03:58:16.85020087Z level=info msg="Running in alternative execution of Error/NoData mode" 2024-09-14T03:58:17.028 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=infra.usagestats.collector t=2024-09-14T03:58:16.851500078Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-09-14T03:58:17.028 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=provisioning.datasources t=2024-09-14T03:58:16.853804762Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2024-09-14T03:58:17.028 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=provisioning.datasources t=2024-09-14T03:58:16.854079718Z level=info msg="inserting datasource from configuration" name=Dashboard1 uid=P43CA22E17D0F9596 2024-09-14T03:58:17.028 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 bash[93164]: f0492c19de92f81c5baae4551bf1c20aa26f85e6173189cea5fb8ca3ba1a1730 2024-09-14T03:58:17.028 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 systemd[1]: Started Ceph grafana.a for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:58:17.028 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=provisioning.alerting t=2024-09-14T03:58:16.886073523Z level=info msg="starting to provision alerting" 2024-09-14T03:58:17.029 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=provisioning.alerting t=2024-09-14T03:58:16.8860891Z level=info msg="finished to provision alerting" 2024-09-14T03:58:17.029 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=grafanaStorageLogger t=2024-09-14T03:58:16.886227304Z level=info msg="Storage starting" 2024-09-14T03:58:17.029 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=ngalert.state.manager t=2024-09-14T03:58:16.886881705Z level=info msg="Warming state cache for startup" 2024-09-14T03:58:17.029 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=http.server t=2024-09-14T03:58:16.88917223Z level=info msg="HTTP Server TLS settings" MinTLSVersion=TLS1.2 configuredciphers=TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA,TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA,TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA,TLS_RSA_WITH_AES_128_GCM_SHA256,TLS_RSA_WITH_AES_256_GCM_SHA384,TLS_RSA_WITH_AES_128_CBC_SHA,TLS_RSA_WITH_AES_256_CBC_SHA 2024-09-14T03:58:17.029 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=http.server t=2024-09-14T03:58:16.889411371Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-09-14T03:58:17.029 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=ngalert.multiorg.alertmanager t=2024-09-14T03:58:16.898435423Z level=info msg="Starting MultiOrg Alertmanager" 2024-09-14T03:58:17.029 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=ngalert.state.manager t=2024-09-14T03:58:16.92312398Z level=info msg="State cache has been initialized" states=0 duration=36.237737ms 2024-09-14T03:58:17.029 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=ngalert.scheduler t=2024-09-14T03:58:16.923236077Z level=info msg="Starting scheduler" tickInterval=10s maxAttempts=1 2024-09-14T03:58:17.029 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=ticker t=2024-09-14T03:58:16.923301769Z level=info msg=starting first_tick=2024-09-14T03:58:20Z 2024-09-14T03:58:17.029 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=provisioning.dashboard t=2024-09-14T03:58:16.932315417Z level=info msg="starting to provision dashboards" 2024-09-14T03:58:17.029 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=grafana.update.checker t=2024-09-14T03:58:16.971372114Z level=info msg="Update check succeeded" duration=85.177349ms 2024-09-14T03:58:17.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:16 smithi039 ceph-mon[29586]: osdmap e301: 8 total, 8 up, 8 in 2024-09-14T03:58:17.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:16 smithi039 ceph-mon[40962]: osdmap e301: 8 total, 8 up, 8 in 2024-09-14T03:58:17.521 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:17 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=plugins.update.checker t=2024-09-14T03:58:17.067874904Z level=info msg="Update check succeeded" duration=181.680196ms 2024-09-14T03:58:17.521 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:17 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=provisioning.dashboard t=2024-09-14T03:58:17.103321265Z level=info msg="finished to provision dashboards" 2024-09-14T03:58:17.521 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:17 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=grafana-apiserver t=2024-09-14T03:58:17.217481861Z level=info msg="Adding GroupVersion playlist.grafana.app v0alpha1 to ResourceManager" 2024-09-14T03:58:17.521 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:58:17 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=grafana-apiserver t=2024-09-14T03:58:17.217848276Z level=info msg="Adding GroupVersion featuretoggle.grafana.app v0alpha1 to ResourceManager" 2024-09-14T03:58:18.120 INFO:tasks.workunit.client.0.smithi039.stdout:migration_abort 2024-09-14T03:58:18.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:17 smithi129 ceph-mon[33173]: pgmap v167: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 269 KiB/s rd, 17 KiB/s wr, 345 op/s 2024-09-14T03:58:18.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:17 smithi129 ceph-mon[33173]: osdmap e302: 8 total, 8 up, 8 in 2024-09-14T03:58:18.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:17 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:18.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:17 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:18.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:17 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:58:18.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:17 smithi039 ceph-mon[29586]: pgmap v167: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 269 KiB/s rd, 17 KiB/s wr, 345 op/s 2024-09-14T03:58:18.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:17 smithi039 ceph-mon[29586]: osdmap e302: 8 total, 8 up, 8 in 2024-09-14T03:58:18.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:17 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:18.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:17 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:18.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:17 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:58:18.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:17 smithi039 ceph-mon[40962]: pgmap v167: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 269 KiB/s rd, 17 KiB/s wr, 345 op/s 2024-09-14T03:58:18.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:17 smithi039 ceph-mon[40962]: osdmap e302: 8 total, 8 up, 8 in 2024-09-14T03:58:18.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:17 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:18.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:17 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:18.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:17 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:58:18.969 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:58:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:58:18] "GET /metrics HTTP/1.1" 200 38861 "" "Prometheus/2.51.0" 2024-09-14T03:58:19.108 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.CloneV2ParentAbortFixIncompleteChildReattach (4084 ms) 2024-09-14T03:58:19.108 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.CloneV2ParentAbortFixParentReattach 2024-09-14T03:58:19.140 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:58:19.152 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 0~10 2024-09-14T03:58:19.161 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap1 2024-09-14T03:58:19.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:18 smithi129 ceph-mon[33173]: osdmap e303: 8 total, 8 up, 8 in 2024-09-14T03:58:19.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:18 smithi039 ceph-mon[29586]: osdmap e303: 8 total, 8 up, 8 in 2024-09-14T03:58:19.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:18 smithi039 ceph-mon[40962]: osdmap e303: 8 total, 8 up, 8 in 2024-09-14T03:58:20.266 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:19 smithi129 ceph-mon[33173]: pgmap v170: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 344 KiB/s rd, 23 KiB/s wr, 442 op/s 2024-09-14T03:58:20.267 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:19 smithi129 ceph-mon[33173]: osdmap e304: 8 total, 8 up, 8 in 2024-09-14T03:58:20.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:19 smithi039 ceph-mon[29586]: pgmap v170: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 344 KiB/s rd, 23 KiB/s wr, 442 op/s 2024-09-14T03:58:20.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:19 smithi039 ceph-mon[29586]: osdmap e304: 8 total, 8 up, 8 in 2024-09-14T03:58:20.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:19 smithi039 ceph-mon[40962]: pgmap v170: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 344 KiB/s rd, 23 KiB/s wr, 442 op/s 2024-09-14T03:58:20.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:19 smithi039 ceph-mon[40962]: osdmap e304: 8 total, 8 up, 8 in 2024-09-14T03:58:21.004 INFO:tasks.workunit.client.0.smithi039.stdout:snap_protect: snap1 2024-09-14T03:58:21.064 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:58:21.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:20 smithi129 ceph-mon[33173]: osdmap e305: 8 total, 8 up, 8 in 2024-09-14T03:58:21.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:20 smithi129 ceph-mon[33173]: pgmap v173: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 344 KiB/s rd, 23 KiB/s wr, 442 op/s 2024-09-14T03:58:21.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:20 smithi039 ceph-mon[29586]: osdmap e305: 8 total, 8 up, 8 in 2024-09-14T03:58:21.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:20 smithi039 ceph-mon[29586]: pgmap v173: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 344 KiB/s rd, 23 KiB/s wr, 442 op/s 2024-09-14T03:58:21.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:20 smithi039 ceph-mon[40962]: osdmap e305: 8 total, 8 up, 8 in 2024-09-14T03:58:21.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:20 smithi039 ceph-mon[40962]: pgmap v173: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 344 KiB/s rd, 23 KiB/s wr, 442 op/s 2024-09-14T03:58:22.162 INFO:tasks.workunit.client.0.smithi039.stdout:migration_abort 2024-09-14T03:58:22.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:21 smithi129 ceph-mon[33173]: osdmap e306: 8 total, 8 up, 8 in 2024-09-14T03:58:22.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:21 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:22.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:21 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:22.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:21 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:22.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:21 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:22.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:21 smithi039 ceph-mon[29586]: osdmap e306: 8 total, 8 up, 8 in 2024-09-14T03:58:22.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:21 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:22.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:21 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:22.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:21 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:22.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:21 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:22.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:21 smithi039 ceph-mon[40962]: osdmap e306: 8 total, 8 up, 8 in 2024-09-14T03:58:22.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:21 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:22.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:21 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:22.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:21 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:22.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:21 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:23.147 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.CloneV2ParentAbortFixParentReattach (4039 ms) 2024-09-14T03:58:23.147 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.CloneV2ParentAbortRelinkNotNeeded 2024-09-14T03:58:23.179 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:58:23.191 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 0~10 2024-09-14T03:58:23.199 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap1 2024-09-14T03:58:23.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:23 smithi129 ceph-mon[33173]: osdmap e307: 8 total, 8 up, 8 in 2024-09-14T03:58:23.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:23 smithi129 ceph-mon[33173]: pgmap v176: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:58:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:23 smithi039 ceph-mon[29586]: osdmap e307: 8 total, 8 up, 8 in 2024-09-14T03:58:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:23 smithi039 ceph-mon[29586]: pgmap v176: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:58:23.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:23 smithi039 ceph-mon[40962]: osdmap e307: 8 total, 8 up, 8 in 2024-09-14T03:58:23.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:23 smithi039 ceph-mon[40962]: pgmap v176: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T03:58:24.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:24 smithi129 ceph-mon[33173]: osdmap e308: 8 total, 8 up, 8 in 2024-09-14T03:58:24.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:24 smithi129 ceph-mon[33173]: osdmap e309: 8 total, 8 up, 8 in 2024-09-14T03:58:24.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:24 smithi039 ceph-mon[29586]: osdmap e308: 8 total, 8 up, 8 in 2024-09-14T03:58:24.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:24 smithi039 ceph-mon[29586]: osdmap e309: 8 total, 8 up, 8 in 2024-09-14T03:58:24.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:24 smithi039 ceph-mon[40962]: osdmap e308: 8 total, 8 up, 8 in 2024-09-14T03:58:24.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:24 smithi039 ceph-mon[40962]: osdmap e309: 8 total, 8 up, 8 in 2024-09-14T03:58:24.765 INFO:tasks.workunit.client.0.smithi039.stdout:snap_protect: snap1 2024-09-14T03:58:24.832 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:58:25.941 INFO:tasks.workunit.client.0.smithi039.stdout:migration_abort 2024-09-14T03:58:26.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: pgmap v179: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 365 KiB/s rd, 27 KiB/s wr, 465 op/s 2024-09-14T03:58:26.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: osdmap e310: 8 total, 8 up, 8 in 2024-09-14T03:58:26.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:26.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:26.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:58:26.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:58:26.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:26.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:58:26.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:58:26.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:26.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-14T03:58:26.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-14T03:58:26.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T03:58:26.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.025 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T03:58:26.025 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-14T03:58:26.025 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-14T03:58:26.025 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-14T03:58:26.025 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-14T03:58:26.025 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-14T03:58:26.025 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-14T03:58:26.025 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-14T03:58:26.025 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-14T03:58:26.025 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-14T03:58:26.025 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-14T03:58:26.026 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-14T03:58:26.026 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-14T03:58:26.026 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-14T03:58:26.026 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.026 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.026 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-14T03:58:26.026 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-14T03:58:26.026 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-14T03:58:26.026 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-14T03:58:26.026 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-14T03:58:26.026 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-14T03:58:26.026 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-14T03:58:26.027 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-14T03:58:26.027 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-14T03:58:26.027 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-14T03:58:26.027 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-14T03:58:26.027 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T03:58:26.027 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.027 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.027 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.028 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.028 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.028 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.028 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.028 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.028 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.028 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.028 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.028 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.028 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-14T03:58:26.028 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-14T03:58:26.028 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T03:58:26.029 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:58:26.029 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:58:26.029 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:58:26.029 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:26.029 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:58:26.029 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:58:26.029 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:58:26.029 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:25 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:26.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: pgmap v179: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 365 KiB/s rd, 27 KiB/s wr, 465 op/s 2024-09-14T03:58:26.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: osdmap e310: 8 total, 8 up, 8 in 2024-09-14T03:58:26.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:26.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:26.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:58:26.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:58:26.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:26.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:58:26.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:58:26.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:26.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-14T03:58:26.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-14T03:58:26.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T03:58:26.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: pgmap v179: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 365 KiB/s rd, 27 KiB/s wr, 465 op/s 2024-09-14T03:58:26.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: osdmap e310: 8 total, 8 up, 8 in 2024-09-14T03:58:26.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:26.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:26.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:58:26.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:58:26.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:26.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:58:26.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:58:26.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.044 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.045 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.045 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.045 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.045 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.045 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.045 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.045 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.045 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.045 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.046 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.046 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.046 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.046 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:26.046 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:26.046 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-14T03:58:26.046 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-14T03:58:26.046 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T03:58:26.046 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.047 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.047 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T03:58:26.047 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-14T03:58:26.047 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-14T03:58:26.047 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-14T03:58:26.047 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-14T03:58:26.047 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-14T03:58:26.047 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-14T03:58:26.047 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-14T03:58:26.047 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-14T03:58:26.047 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-14T03:58:26.047 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-14T03:58:26.048 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-14T03:58:26.048 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-14T03:58:26.048 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-14T03:58:26.048 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.048 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.048 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-14T03:58:26.048 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-14T03:58:26.048 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-14T03:58:26.048 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-14T03:58:26.048 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-14T03:58:26.049 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-14T03:58:26.049 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-14T03:58:26.049 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.049 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T03:58:26.049 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-14T03:58:26.049 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-14T03:58:26.049 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-14T03:58:26.049 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-14T03:58:26.050 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-14T03:58:26.050 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-14T03:58:26.050 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-14T03:58:26.050 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-14T03:58:26.050 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-14T03:58:26.050 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-14T03:58:26.050 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-14T03:58:26.050 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-14T03:58:26.050 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-14T03:58:26.050 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.050 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.050 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-14T03:58:26.051 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-14T03:58:26.051 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-14T03:58:26.051 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-14T03:58:26.051 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-14T03:58:26.051 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-14T03:58:26.051 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-14T03:58:26.051 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-14T03:58:26.051 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-14T03:58:26.052 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-14T03:58:26.052 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-14T03:58:26.052 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T03:58:26.052 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.052 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.052 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.053 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.053 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.053 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.053 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.053 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.053 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.053 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.053 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.053 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.053 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-14T03:58:26.053 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-14T03:58:26.054 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T03:58:26.054 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:58:26.054 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:58:26.054 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:58:26.054 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:26.054 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:58:26.054 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:58:26.054 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:58:26.054 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:26.055 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-14T03:58:26.055 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-14T03:58:26.055 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-14T03:58:26.055 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-14T03:58:26.055 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T03:58:26.055 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.055 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.055 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.056 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.056 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.056 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.056 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.056 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.056 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.056 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.056 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.056 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:58:26.056 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-14T03:58:26.056 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-14T03:58:26.057 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T03:58:26.057 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:58:26.057 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:58:26.057 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:58:26.057 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:26.057 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:58:26.057 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:58:26.057 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:58:26.057 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:25 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:26.931 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.CloneV2ParentAbortRelinkNotNeeded (3784 ms) 2024-09-14T03:58:26.931 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.StressNoMigrate 2024-09-14T03:58:26.964 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:58:26.975 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 51509748~7331513 2024-09-14T03:58:27.030 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 57381640~934250 2024-09-14T03:58:27.032 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 32147642~1602212 2024-09-14T03:58:27.047 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 30944394~811145 2024-09-14T03:58:27.050 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 82641915~1244165 2024-09-14T03:58:27.062 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 48055298~5495006 2024-09-14T03:58:27.065 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 53867153~7873865 2024-09-14T03:58:27.112 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 66101787~1169582 2024-09-14T03:58:27.112 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap0 2024-09-14T03:58:27.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:26 smithi129 ceph-mon[33173]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:58:27.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:26 smithi129 ceph-mon[33173]: Upgrade: Finalizing container_image settings 2024-09-14T03:58:27.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:26 smithi129 ceph-mon[33173]: Upgrade: Complete! 2024-09-14T03:58:27.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:26 smithi129 ceph-mon[33173]: osdmap e311: 8 total, 8 up, 8 in 2024-09-14T03:58:27.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:26 smithi039 ceph-mon[29586]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:58:27.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:26 smithi039 ceph-mon[29586]: Upgrade: Finalizing container_image settings 2024-09-14T03:58:27.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:26 smithi039 ceph-mon[29586]: Upgrade: Complete! 2024-09-14T03:58:27.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:26 smithi039 ceph-mon[29586]: osdmap e311: 8 total, 8 up, 8 in 2024-09-14T03:58:27.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:26 smithi039 ceph-mon[40962]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-09-14T03:58:27.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:26 smithi039 ceph-mon[40962]: Upgrade: Finalizing container_image settings 2024-09-14T03:58:27.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:26 smithi039 ceph-mon[40962]: Upgrade: Complete! 2024-09-14T03:58:27.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:26 smithi039 ceph-mon[40962]: osdmap e311: 8 total, 8 up, 8 in 2024-09-14T03:58:28.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:27 smithi129 ceph-mon[33173]: pgmap v182: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 365 KiB/s rd, 27 KiB/s wr, 465 op/s 2024-09-14T03:58:28.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:27 smithi129 ceph-mon[33173]: osdmap e312: 8 total, 8 up, 8 in 2024-09-14T03:58:28.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:27 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:58:28.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:27 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:28.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:27 smithi039 ceph-mon[40962]: pgmap v182: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 365 KiB/s rd, 27 KiB/s wr, 465 op/s 2024-09-14T03:58:28.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:27 smithi039 ceph-mon[40962]: osdmap e312: 8 total, 8 up, 8 in 2024-09-14T03:58:28.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:27 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:58:28.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:27 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:28.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:27 smithi039 ceph-mon[29586]: pgmap v182: 137 pgs: 137 active+clean; 901 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 365 KiB/s rd, 27 KiB/s wr, 465 op/s 2024-09-14T03:58:28.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:27 smithi039 ceph-mon[29586]: osdmap e312: 8 total, 8 up, 8 in 2024-09-14T03:58:28.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:27 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:58:28.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:27 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:28.846 INFO:tasks.workunit.client.0.smithi039.stdout:snap_protect: snap0 2024-09-14T03:58:28.862 INFO:tasks.workunit.client.0.smithi039.stdout:clone image159 -> image162 2024-09-14T03:58:28.976 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 157937288 2024-09-14T03:58:28.989 INFO:tasks.workunit.client.0.smithi039.stdout:write: B 4511513~5579675 2024-09-14T03:58:29.041 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 3737215~563411 2024-09-14T03:58:29.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:28 smithi039 ceph-mon[29586]: osdmap e313: 8 total, 8 up, 8 in 2024-09-14T03:58:29.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:28 smithi039 ceph-mon[40962]: osdmap e313: 8 total, 8 up, 8 in 2024-09-14T03:58:29.044 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:58:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:58:28] "GET /metrics HTTP/1.1" 200 38865 "" "Prometheus/2.51.0" 2024-09-14T03:58:29.045 INFO:tasks.workunit.client.0.smithi039.stdout:write: B 23969919~975063 2024-09-14T03:58:29.055 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 85371505~7349289 2024-09-14T03:58:29.056 INFO:tasks.workunit.client.0.smithi039.stdout:write: B 66951358~4219981 2024-09-14T03:58:29.091 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 10304954~6241902 2024-09-14T03:58:29.096 INFO:tasks.workunit.client.0.smithi039.stdout:write: B 81760186~6596055 2024-09-14T03:58:29.144 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 103606160~1404332 2024-09-14T03:58:29.144 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap1 2024-09-14T03:58:29.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:28 smithi129 ceph-mon[33173]: osdmap e313: 8 total, 8 up, 8 in 2024-09-14T03:58:30.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:29 smithi129 ceph-mon[33173]: pgmap v185: 137 pgs: 137 active+clean; 939 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 304 KiB/s rd, 8.6 MiB/s wr, 413 op/s 2024-09-14T03:58:30.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:29 smithi129 ceph-mon[33173]: osdmap e314: 8 total, 8 up, 8 in 2024-09-14T03:58:30.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:29 smithi039 ceph-mon[29586]: pgmap v185: 137 pgs: 137 active+clean; 939 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 304 KiB/s rd, 8.6 MiB/s wr, 413 op/s 2024-09-14T03:58:30.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:29 smithi039 ceph-mon[29586]: osdmap e314: 8 total, 8 up, 8 in 2024-09-14T03:58:30.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:29 smithi039 ceph-mon[40962]: pgmap v185: 137 pgs: 137 active+clean; 939 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 304 KiB/s rd, 8.6 MiB/s wr, 413 op/s 2024-09-14T03:58:30.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:29 smithi039 ceph-mon[40962]: osdmap e314: 8 total, 8 up, 8 in 2024-09-14T03:58:30.859 INFO:tasks.workunit.client.0.smithi039.stdout:snap_protect: snap1 2024-09-14T03:58:30.873 INFO:tasks.workunit.client.0.smithi039.stdout:clone image162 -> image164 2024-09-14T03:58:31.016 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 234296241 2024-09-14T03:58:31.027 INFO:tasks.workunit.client.0.smithi039.stdout:write: C 227884065~6412176 2024-09-14T03:58:31.080 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 197041341~5279117 2024-09-14T03:58:31.080 INFO:tasks.workunit.client.0.smithi039.stdout:write: C 147370710~7353967 2024-09-14T03:58:31.128 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 189362581~7054079 2024-09-14T03:58:31.128 INFO:tasks.workunit.client.0.smithi039.stdout:write: C 103441136~3757297 2024-09-14T03:58:31.149 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 69419897~4999998 2024-09-14T03:58:31.175 INFO:tasks.workunit.client.0.smithi039.stdout:write: C 199576076~6018383 2024-09-14T03:58:31.212 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 179854612~2874444 2024-09-14T03:58:31.212 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap2 2024-09-14T03:58:31.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:30 smithi129 ceph-mon[33173]: osdmap e315: 8 total, 8 up, 8 in 2024-09-14T03:58:31.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:30 smithi039 ceph-mon[29586]: osdmap e315: 8 total, 8 up, 8 in 2024-09-14T03:58:31.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:30 smithi039 ceph-mon[40962]: osdmap e315: 8 total, 8 up, 8 in 2024-09-14T03:58:32.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:31 smithi129 ceph-mon[33173]: pgmap v188: 137 pgs: 137 active+clean; 939 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 304 KiB/s rd, 8.6 MiB/s wr, 413 op/s 2024-09-14T03:58:32.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:31 smithi129 ceph-mon[33173]: osdmap e316: 8 total, 8 up, 8 in 2024-09-14T03:58:32.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:31 smithi039 ceph-mon[29586]: pgmap v188: 137 pgs: 137 active+clean; 939 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 304 KiB/s rd, 8.6 MiB/s wr, 413 op/s 2024-09-14T03:58:32.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:31 smithi039 ceph-mon[29586]: osdmap e316: 8 total, 8 up, 8 in 2024-09-14T03:58:32.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:31 smithi039 ceph-mon[40962]: pgmap v188: 137 pgs: 137 active+clean; 939 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 304 KiB/s rd, 8.6 MiB/s wr, 413 op/s 2024-09-14T03:58:32.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:31 smithi039 ceph-mon[40962]: osdmap e316: 8 total, 8 up, 8 in 2024-09-14T03:58:32.872 INFO:tasks.workunit.client.0.smithi039.stdout:snap_protect: snap2 2024-09-14T03:58:32.888 INFO:tasks.workunit.client.0.smithi039.stdout:clone image164 -> image166 2024-09-14T03:58:33.052 INFO:tasks.workunit.client.0.smithi039.stdout:write: D 208978908~1484959 2024-09-14T03:58:33.073 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 188222792~7055809 2024-09-14T03:58:33.080 INFO:tasks.workunit.client.0.smithi039.stdout:write: D 52977629~3751261 2024-09-14T03:58:33.132 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 36337087~7345557 2024-09-14T03:58:33.158 INFO:tasks.workunit.client.0.smithi039.stdout:write: D 59224301~541615 2024-09-14T03:58:33.205 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 218642966~3416261 2024-09-14T03:58:33.210 INFO:tasks.workunit.client.0.smithi039.stdout:write: D 68548693~1894366 2024-09-14T03:58:33.248 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 68817654~1780867 2024-09-14T03:58:33.250 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap3 2024-09-14T03:58:33.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:32 smithi129 ceph-mon[33173]: osdmap e317: 8 total, 8 up, 8 in 2024-09-14T03:58:33.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:32 smithi039 ceph-mon[29586]: osdmap e317: 8 total, 8 up, 8 in 2024-09-14T03:58:33.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:32 smithi039 ceph-mon[40962]: osdmap e317: 8 total, 8 up, 8 in 2024-09-14T03:58:34.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:33 smithi129 ceph-mon[33173]: pgmap v191: 137 pgs: 137 active+clean; 939 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:58:34.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:33 smithi129 ceph-mon[33173]: osdmap e318: 8 total, 8 up, 8 in 2024-09-14T03:58:34.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:33 smithi129 ceph-mon[33173]: osdmap e319: 8 total, 8 up, 8 in 2024-09-14T03:58:34.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:33 smithi039 ceph-mon[29586]: pgmap v191: 137 pgs: 137 active+clean; 939 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:58:34.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:33 smithi039 ceph-mon[29586]: osdmap e318: 8 total, 8 up, 8 in 2024-09-14T03:58:34.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:33 smithi039 ceph-mon[29586]: osdmap e319: 8 total, 8 up, 8 in 2024-09-14T03:58:34.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:33 smithi039 ceph-mon[40962]: pgmap v191: 137 pgs: 137 active+clean; 939 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:58:34.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:33 smithi039 ceph-mon[40962]: osdmap e318: 8 total, 8 up, 8 in 2024-09-14T03:58:34.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:33 smithi039 ceph-mon[40962]: osdmap e319: 8 total, 8 up, 8 in 2024-09-14T03:58:35.305 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.StressNoMigrate (8375 ms) 2024-09-14T03:58:35.305 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.Stress 2024-09-14T03:58:35.335 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:58:35.349 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 62344131~3549249 2024-09-14T03:58:35.373 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 40860978~881167 2024-09-14T03:58:35.374 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 47824205~4062584 2024-09-14T03:58:35.403 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 68801223~1228971 2024-09-14T03:58:35.403 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 62823672~2376300 2024-09-14T03:58:35.418 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 58566755~4216638 2024-09-14T03:58:35.420 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 30867421~6939244 2024-09-14T03:58:35.461 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 22383661~5744466 2024-09-14T03:58:35.462 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap0 2024-09-14T03:58:36.000 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:35 smithi039 ceph-mon[29586]: pgmap v194: 137 pgs: 137 active+clean; 1.0 GiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.8 MiB/s rd, 21 MiB/s wr, 399 op/s 2024-09-14T03:58:36.000 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:35 smithi039 ceph-mon[29586]: osdmap e320: 8 total, 8 up, 8 in 2024-09-14T03:58:36.000 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:35 smithi039 ceph-mon[40962]: pgmap v194: 137 pgs: 137 active+clean; 1.0 GiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.8 MiB/s rd, 21 MiB/s wr, 399 op/s 2024-09-14T03:58:36.000 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:35 smithi039 ceph-mon[40962]: osdmap e320: 8 total, 8 up, 8 in 2024-09-14T03:58:36.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:35 smithi129 ceph-mon[33173]: pgmap v194: 137 pgs: 137 active+clean; 1.0 GiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.8 MiB/s rd, 21 MiB/s wr, 399 op/s 2024-09-14T03:58:36.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:35 smithi129 ceph-mon[33173]: osdmap e320: 8 total, 8 up, 8 in 2024-09-14T03:58:36.889 INFO:tasks.workunit.client.0.smithi039.stdout:snap_protect: snap0 2024-09-14T03:58:36.903 INFO:tasks.workunit.client.0.smithi039.stdout:clone image167 -> image170 2024-09-14T03:58:37.006 INFO:tasks.workunit.client.0.smithi039.stdout:write: B 79142352~4743728 2024-09-14T03:58:37.057 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 5285343~3889775 2024-09-14T03:58:37.064 INFO:tasks.workunit.client.0.smithi039.stdout:write: B 24083253~1233829 2024-09-14T03:58:37.075 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 70237059~5824502 2024-09-14T03:58:37.081 INFO:tasks.workunit.client.0.smithi039.stdout:write: B 49548409~357258 2024-09-14T03:58:37.111 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 38870094~1913071 2024-09-14T03:58:37.118 INFO:tasks.workunit.client.0.smithi039.stdout:write: B 47405360~8351595 2024-09-14T03:58:37.176 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 844154~550930 2024-09-14T03:58:37.180 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap1 2024-09-14T03:58:37.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:36 smithi129 ceph-mon[33173]: osdmap e321: 8 total, 8 up, 8 in 2024-09-14T03:58:37.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:36 smithi039 ceph-mon[29586]: osdmap e321: 8 total, 8 up, 8 in 2024-09-14T03:58:37.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:36 smithi039 ceph-mon[40962]: osdmap e321: 8 total, 8 up, 8 in 2024-09-14T03:58:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:37 smithi129 ceph-mon[33173]: pgmap v197: 137 pgs: 137 active+clean; 1.0 GiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.8 MiB/s rd, 21 MiB/s wr, 399 op/s 2024-09-14T03:58:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:37 smithi129 ceph-mon[33173]: osdmap e322: 8 total, 8 up, 8 in 2024-09-14T03:58:38.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:37 smithi039 ceph-mon[29586]: pgmap v197: 137 pgs: 137 active+clean; 1.0 GiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.8 MiB/s rd, 21 MiB/s wr, 399 op/s 2024-09-14T03:58:38.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:37 smithi039 ceph-mon[29586]: osdmap e322: 8 total, 8 up, 8 in 2024-09-14T03:58:38.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:37 smithi039 ceph-mon[40962]: pgmap v197: 137 pgs: 137 active+clean; 1.0 GiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 2.8 MiB/s rd, 21 MiB/s wr, 399 op/s 2024-09-14T03:58:38.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:37 smithi039 ceph-mon[40962]: osdmap e322: 8 total, 8 up, 8 in 2024-09-14T03:58:38.916 INFO:tasks.workunit.client.0.smithi039.stdout:snap_protect: snap1 2024-09-14T03:58:38.927 INFO:tasks.workunit.client.0.smithi039.stdout:clone image170 -> image172 2024-09-14T03:58:39.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:38 smithi039 ceph-mon[29586]: osdmap e323: 8 total, 8 up, 8 in 2024-09-14T03:58:39.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:58:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:58:38] "GET /metrics HTTP/1.1" 200 38865 "" "Prometheus/2.51.0" 2024-09-14T03:58:39.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:38 smithi039 ceph-mon[40962]: osdmap e323: 8 total, 8 up, 8 in 2024-09-14T03:58:39.062 INFO:tasks.workunit.client.0.smithi039.stdout:write: C 64329486~6725319 2024-09-14T03:58:39.130 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 59433267~6305873 2024-09-14T03:58:39.138 INFO:tasks.workunit.client.0.smithi039.stdout:write: C 77631375~1324038 2024-09-14T03:58:39.158 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 66983538~559510 2024-09-14T03:58:39.161 INFO:tasks.workunit.client.0.smithi039.stdout:write: C 31469801~7835501 2024-09-14T03:58:39.216 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 22675208~5481119 2024-09-14T03:58:39.223 INFO:tasks.workunit.client.0.smithi039.stdout:write: C 43779279~7411097 2024-09-14T03:58:39.271 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 36466696~2917432 2024-09-14T03:58:39.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:38 smithi129 ceph-mon[33173]: osdmap e323: 8 total, 8 up, 8 in 2024-09-14T03:58:39.285 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap2 2024-09-14T03:58:40.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:39 smithi129 ceph-mon[33173]: pgmap v200: 137 pgs: 137 active+clean; 1.2 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 58 MiB/s rd, 25 MiB/s wr, 558 op/s 2024-09-14T03:58:40.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:39 smithi129 ceph-mon[33173]: osdmap e324: 8 total, 8 up, 8 in 2024-09-14T03:58:40.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:39 smithi039 ceph-mon[29586]: pgmap v200: 137 pgs: 137 active+clean; 1.2 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 58 MiB/s rd, 25 MiB/s wr, 558 op/s 2024-09-14T03:58:40.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:39 smithi039 ceph-mon[29586]: osdmap e324: 8 total, 8 up, 8 in 2024-09-14T03:58:40.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:39 smithi039 ceph-mon[40962]: pgmap v200: 137 pgs: 137 active+clean; 1.2 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 58 MiB/s rd, 25 MiB/s wr, 558 op/s 2024-09-14T03:58:40.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:39 smithi039 ceph-mon[40962]: osdmap e324: 8 total, 8 up, 8 in 2024-09-14T03:58:40.967 INFO:tasks.workunit.client.0.smithi039.stdout:snap_protect: snap2 2024-09-14T03:58:40.979 INFO:tasks.workunit.client.0.smithi039.stdout:clone image172 -> image174 2024-09-14T03:58:41.139 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 94104381 2024-09-14T03:58:41.151 INFO:tasks.workunit.client.0.smithi039.stdout:write: D 43733417~4557477 2024-09-14T03:58:41.220 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 45267400~2186952 2024-09-14T03:58:41.222 INFO:tasks.workunit.client.0.smithi039.stdout:write: D 62873463~803919 2024-09-14T03:58:41.248 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 3226812~3737225 2024-09-14T03:58:41.256 INFO:tasks.workunit.client.0.smithi039.stdout:write: D 42415155~8053544 2024-09-14T03:58:41.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:40 smithi129 ceph-mon[33173]: osdmap e325: 8 total, 8 up, 8 in 2024-09-14T03:58:41.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:40 smithi039 ceph-mon[29586]: osdmap e325: 8 total, 8 up, 8 in 2024-09-14T03:58:41.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:40 smithi039 ceph-mon[40962]: osdmap e325: 8 total, 8 up, 8 in 2024-09-14T03:58:41.330 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 21055666~2257891 2024-09-14T03:58:41.335 INFO:tasks.workunit.client.0.smithi039.stdout:write: D 32583067~2918090 2024-09-14T03:58:41.398 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 88587017~2195716 2024-09-14T03:58:41.398 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap3 2024-09-14T03:58:42.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:41 smithi129 ceph-mon[33173]: pgmap v203: 137 pgs: 137 active+clean; 1.2 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 58 MiB/s rd, 25 MiB/s wr, 558 op/s 2024-09-14T03:58:42.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:41 smithi129 ceph-mon[33173]: osdmap e326: 8 total, 8 up, 8 in 2024-09-14T03:58:42.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:41 smithi039 ceph-mon[29586]: pgmap v203: 137 pgs: 137 active+clean; 1.2 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 58 MiB/s rd, 25 MiB/s wr, 558 op/s 2024-09-14T03:58:42.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:41 smithi039 ceph-mon[29586]: osdmap e326: 8 total, 8 up, 8 in 2024-09-14T03:58:42.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:41 smithi039 ceph-mon[40962]: pgmap v203: 137 pgs: 137 active+clean; 1.2 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 58 MiB/s rd, 25 MiB/s wr, 558 op/s 2024-09-14T03:58:42.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:41 smithi039 ceph-mon[40962]: osdmap e326: 8 total, 8 up, 8 in 2024-09-14T03:58:42.565 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'echo wait for minority of mons to upgrade' 2024-09-14T03:58:42.775 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:58:42.978 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 116762003 2024-09-14T03:58:42.996 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:58:43.163 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:42 smithi039 ceph-mon[29586]: from='client.15210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:43.163 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:42 smithi039 ceph-mon[29586]: osdmap e327: 8 total, 8 up, 8 in 2024-09-14T03:58:43.164 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:42 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:58:43.164 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:42 smithi039 ceph-mon[40962]: from='client.15210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:43.164 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:42 smithi039 ceph-mon[40962]: osdmap e327: 8 total, 8 up, 8 in 2024-09-14T03:58:43.164 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:42 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:58:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:42 smithi129 ceph-mon[33173]: from='client.15210 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:42 smithi129 ceph-mon[33173]: osdmap e327: 8 total, 8 up, 8 in 2024-09-14T03:58:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:42 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:58:44.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:43 smithi129 ceph-mon[33173]: pgmap v206: 137 pgs: 137 active+clean; 1.2 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:58:44.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:43 smithi129 ceph-mon[33173]: osdmap e328: 8 total, 8 up, 8 in 2024-09-14T03:58:44.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:43 smithi039 ceph-mon[40962]: pgmap v206: 137 pgs: 137 active+clean; 1.2 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:58:44.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:43 smithi039 ceph-mon[40962]: osdmap e328: 8 total, 8 up, 8 in 2024-09-14T03:58:44.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:43 smithi039 ceph-mon[29586]: pgmap v206: 137 pgs: 137 active+clean; 1.2 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:58:44.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:43 smithi039 ceph-mon[29586]: osdmap e328: 8 total, 8 up, 8 in 2024-09-14T03:58:44.490 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:58:44.705 INFO:teuthology.orchestra.run.smithi039.stdout:wait for minority of mons to upgrade 2024-09-14T03:58:44.743 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:58:45.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:44 smithi129 ceph-mon[33173]: osdmap e329: 8 total, 8 up, 8 in 2024-09-14T03:58:45.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:44 smithi039 ceph-mon[40962]: osdmap e329: 8 total, 8 up, 8 in 2024-09-14T03:58:45.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:44 smithi039 ceph-mon[29586]: osdmap e329: 8 total, 8 up, 8 in 2024-09-14T03:58:45.309 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.Stress (10004 ms) 2024-09-14T03:58:45.310 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.Stress2 2024-09-14T03:58:45.340 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:58:45.351 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 70885376~677226 2024-09-14T03:58:45.361 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 22935562~2105981 2024-09-14T03:58:45.361 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 9517078~1285896 2024-09-14T03:58:45.374 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 12591936~7994585 2024-09-14T03:58:45.375 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 15116168~4040734 2024-09-14T03:58:45.402 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 5870428~8354589 2024-09-14T03:58:45.410 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 57577086~2896429 2024-09-14T03:58:45.429 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 40225099~8057380 2024-09-14T03:58:45.429 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap0 2024-09-14T03:58:45.507 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1' 2024-09-14T03:58:45.714 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:58:46.107 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:45 smithi039 ceph-mon[29586]: pgmap v209: 137 pgs: 137 active+clean; 1.3 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 28 MiB/s wr, 482 op/s 2024-09-14T03:58:46.107 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:45 smithi039 ceph-mon[29586]: osdmap e330: 8 total, 8 up, 8 in 2024-09-14T03:58:46.107 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:45 smithi039 ceph-mon[40962]: pgmap v209: 137 pgs: 137 active+clean; 1.3 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 28 MiB/s wr, 482 op/s 2024-09-14T03:58:46.107 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:45 smithi039 ceph-mon[40962]: osdmap e330: 8 total, 8 up, 8 in 2024-09-14T03:58:46.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:45 smithi129 ceph-mon[33173]: pgmap v209: 137 pgs: 137 active+clean; 1.3 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 28 MiB/s wr, 482 op/s 2024-09-14T03:58:46.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:45 smithi129 ceph-mon[33173]: osdmap e330: 8 total, 8 up, 8 in 2024-09-14T03:58:47.028 INFO:tasks.workunit.client.0.smithi039.stdout:snap_protect: snap0 2024-09-14T03:58:47.043 INFO:tasks.workunit.client.0.smithi039.stdout:clone image175 -> image178 2024-09-14T03:58:47.139 INFO:tasks.workunit.client.0.smithi039.stdout:write: B 73035853~2019284 2024-09-14T03:58:47.170 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 31766487~6836323 2024-09-14T03:58:47.178 INFO:tasks.workunit.client.0.smithi039.stdout:write: B 11114773~8184880 2024-09-14T03:58:47.231 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 24440610~1130145 2024-09-14T03:58:47.237 INFO:tasks.workunit.client.0.smithi039.stdout:write: B 13149855~4921881 2024-09-14T03:58:47.268 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 26028717~77338 2024-09-14T03:58:47.270 INFO:tasks.workunit.client.0.smithi039.stdout:write: B 42697604~563251 2024-09-14T03:58:47.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:47 smithi129 ceph-mon[33173]: osdmap e331: 8 total, 8 up, 8 in 2024-09-14T03:58:47.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:47 smithi129 ceph-mon[33173]: pgmap v212: 137 pgs: 137 active+clean; 1.3 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 28 MiB/s wr, 482 op/s 2024-09-14T03:58:47.279 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:47 smithi039 ceph-mon[29586]: osdmap e331: 8 total, 8 up, 8 in 2024-09-14T03:58:47.279 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:47 smithi039 ceph-mon[29586]: pgmap v212: 137 pgs: 137 active+clean; 1.3 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 28 MiB/s wr, 482 op/s 2024-09-14T03:58:47.279 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:47 smithi039 ceph-mon[40962]: osdmap e331: 8 total, 8 up, 8 in 2024-09-14T03:58:47.279 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:47 smithi039 ceph-mon[40962]: pgmap v212: 137 pgs: 137 active+clean; 1.3 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 15 MiB/s rd, 28 MiB/s wr, 482 op/s 2024-09-14T03:58:47.280 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 19446448~4639405 2024-09-14T03:58:47.283 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap1 2024-09-14T03:58:48.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:48 smithi129 ceph-mon[33173]: osdmap e332: 8 total, 8 up, 8 in 2024-09-14T03:58:48.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:48 smithi129 ceph-mon[33173]: from='client.25118 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:48.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:48 smithi039 ceph-mon[29586]: osdmap e332: 8 total, 8 up, 8 in 2024-09-14T03:58:48.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:48 smithi039 ceph-mon[29586]: from='client.25118 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:48.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:48 smithi039 ceph-mon[40962]: osdmap e332: 8 total, 8 up, 8 in 2024-09-14T03:58:48.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:48 smithi039 ceph-mon[40962]: from='client.25118 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:48.861 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:58:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:58:48] "GET /metrics HTTP/1.1" 200 38875 "" "Prometheus/2.51.0" 2024-09-14T03:58:49.044 INFO:tasks.workunit.client.0.smithi039.stdout:snap_protect: snap1 2024-09-14T03:58:49.055 INFO:tasks.workunit.client.0.smithi039.stdout:clone image178 -> image180 2024-09-14T03:58:49.121 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:49 smithi039 ceph-mon[29586]: osdmap e333: 8 total, 8 up, 8 in 2024-09-14T03:58:49.121 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:49 smithi039 ceph-mon[29586]: pgmap v215: 137 pgs: 137 active+clean; 1.4 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 68 MiB/s rd, 17 MiB/s wr, 1.12k op/s 2024-09-14T03:58:49.122 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:49 smithi039 ceph-mon[40962]: osdmap e333: 8 total, 8 up, 8 in 2024-09-14T03:58:49.122 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:49 smithi039 ceph-mon[40962]: pgmap v215: 137 pgs: 137 active+clean; 1.4 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 68 MiB/s rd, 17 MiB/s wr, 1.12k op/s 2024-09-14T03:58:49.182 INFO:tasks.workunit.client.0.smithi039.stdout:write: C 65239144~3797702 2024-09-14T03:58:49.237 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 58170084~1740022 2024-09-14T03:58:49.292 INFO:tasks.workunit.client.0.smithi039.stdout:write: C 35260436~4857840 2024-09-14T03:58:49.320 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 66474526~5320256 2024-09-14T03:58:49.347 INFO:tasks.workunit.client.0.smithi039.stdout:write: C 72097892~562834 2024-09-14T03:58:49.353 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 42166623~6036329 2024-09-14T03:58:49.362 INFO:tasks.workunit.client.0.smithi039.stdout:write: C 49998654~3771807 2024-09-14T03:58:49.394 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 27385346~6150573 2024-09-14T03:58:49.401 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap2 2024-09-14T03:58:49.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:49 smithi129 ceph-mon[33173]: osdmap e333: 8 total, 8 up, 8 in 2024-09-14T03:58:49.523 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:49 smithi129 ceph-mon[33173]: pgmap v215: 137 pgs: 137 active+clean; 1.4 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 68 MiB/s rd, 17 MiB/s wr, 1.12k op/s 2024-09-14T03:58:50.093 INFO:teuthology.orchestra.run.smithi039.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:58:50.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:50 smithi039 ceph-mon[29586]: osdmap e334: 8 total, 8 up, 8 in 2024-09-14T03:58:50.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:50 smithi039 ceph-mon[40962]: osdmap e334: 8 total, 8 up, 8 in 2024-09-14T03:58:50.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:50 smithi129 ceph-mon[33173]: osdmap e334: 8 total, 8 up, 8 in 2024-09-14T03:58:50.769 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-14T03:58:50.988 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T03:58:51.074 INFO:tasks.workunit.client.0.smithi039.stdout:snap_protect: snap2 2024-09-14T03:58:51.086 INFO:tasks.workunit.client.0.smithi039.stdout:clone image180 -> image182 2024-09-14T03:58:51.228 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 155940631 2024-09-14T03:58:51.240 INFO:tasks.workunit.client.0.smithi039.stdout:write: D 44596528~7397815 2024-09-14T03:58:51.313 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 59485178~1478961 2024-09-14T03:58:51.319 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:51 smithi039 ceph-mon[29586]: osdmap e335: 8 total, 8 up, 8 in 2024-09-14T03:58:51.319 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:51 smithi039 ceph-mon[29586]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:58:51.319 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:51 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:51.319 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:51 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:58:51.319 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:51 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:58:51.319 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:51 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:58:51.319 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:51 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:51.320 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:51 smithi039 ceph-mon[29586]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:58:51.320 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:51 smithi039 ceph-mon[29586]: pgmap v218: 137 pgs: 137 active+clean; 1.4 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 68 MiB/s rd, 17 MiB/s wr, 1.12k op/s 2024-09-14T03:58:51.320 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:51 smithi039 ceph-mon[40962]: osdmap e335: 8 total, 8 up, 8 in 2024-09-14T03:58:51.320 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:51 smithi039 ceph-mon[40962]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:58:51.320 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:51 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:51.320 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:51 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:58:51.320 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:51 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:58:51.320 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:51 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:58:51.320 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:51 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:51.321 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:51 smithi039 ceph-mon[40962]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:58:51.321 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:51 smithi039 ceph-mon[40962]: pgmap v218: 137 pgs: 137 active+clean; 1.4 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 68 MiB/s rd, 17 MiB/s wr, 1.12k op/s 2024-09-14T03:58:51.329 INFO:tasks.workunit.client.0.smithi039.stdout:write: D 74833794~5298969 2024-09-14T03:58:51.379 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 83928550~4333705 2024-09-14T03:58:51.380 INFO:tasks.workunit.client.0.smithi039.stdout:write: D 132204069~6981259 2024-09-14T03:58:51.431 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 124336009~4577836 2024-09-14T03:58:51.432 INFO:tasks.workunit.client.0.smithi039.stdout:write: D 150697528~2412031 2024-09-14T03:58:51.453 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 144468195~6257976 2024-09-14T03:58:51.455 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap3 2024-09-14T03:58:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:51 smithi129 ceph-mon[33173]: osdmap e335: 8 total, 8 up, 8 in 2024-09-14T03:58:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:51 smithi129 ceph-mon[33173]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:58:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:51 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:51 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:58:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:51 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:58:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:51 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:58:51.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:51 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:51.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:51 smithi129 ceph-mon[33173]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T03:58:51.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:51 smithi129 ceph-mon[33173]: pgmap v218: 137 pgs: 137 active+clean; 1.4 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 68 MiB/s rd, 17 MiB/s wr, 1.12k op/s 2024-09-14T03:58:52.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:52 smithi039 ceph-mon[29586]: osdmap e336: 8 total, 8 up, 8 in 2024-09-14T03:58:52.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:52 smithi039 ceph-mon[40962]: osdmap e336: 8 total, 8 up, 8 in 2024-09-14T03:58:52.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:52 smithi129 ceph-mon[33173]: osdmap e336: 8 total, 8 up, 8 in 2024-09-14T03:58:53.144 INFO:tasks.workunit.client.0.smithi039.stdout:snap_protect: snap3 2024-09-14T03:58:53.153 INFO:tasks.workunit.client.0.smithi039.stdout:clone image182 -> image184 2024-09-14T03:58:53.328 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:58:53.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:53 smithi129 ceph-mon[33173]: osdmap e337: 8 total, 8 up, 8 in 2024-09-14T03:58:53.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:53 smithi129 ceph-mon[33173]: pgmap v221: 137 pgs: 137 active+clean; 1.4 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:58:53.522 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:53 smithi039 ceph-mon[29586]: osdmap e337: 8 total, 8 up, 8 in 2024-09-14T03:58:53.522 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:53 smithi039 ceph-mon[29586]: pgmap v221: 137 pgs: 137 active+clean; 1.4 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:58:53.522 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:53 smithi039 ceph-mon[40962]: osdmap e337: 8 total, 8 up, 8 in 2024-09-14T03:58:53.522 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:53 smithi039 ceph-mon[40962]: pgmap v221: 137 pgs: 137 active+clean; 1.4 GiB data, 3.6 GiB used, 712 GiB / 715 GiB avail 2024-09-14T03:58:53.850 INFO:tasks.workunit.client.0.smithi039.stdout:write: a 6303631~7182116 2024-09-14T03:58:53.921 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 135432121~138206 2024-09-14T03:58:53.953 INFO:tasks.workunit.client.0.smithi039.stdout:write: a 62524966~4579340 2024-09-14T03:58:54.013 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 111795484~7554343 2024-09-14T03:58:54.039 INFO:tasks.workunit.client.0.smithi039.stdout:write: a 68931510~6512362 2024-09-14T03:58:54.042 INFO:teuthology.orchestra.run.smithi039.stdout:true 2024-09-14T03:58:54.113 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 113122504~2125408 2024-09-14T03:58:54.115 INFO:tasks.workunit.client.0.smithi039.stdout:write: a 57286728~6330945 2024-09-14T03:58:54.157 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 95138907~903623 2024-09-14T03:58:54.181 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: user0 2024-09-14T03:58:54.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:54 smithi129 ceph-mon[33173]: osdmap e338: 8 total, 8 up, 8 in 2024-09-14T03:58:54.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:54 smithi129 ceph-mon[33173]: osdmap e339: 8 total, 8 up, 8 in 2024-09-14T03:58:54.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:54 smithi039 ceph-mon[29586]: osdmap e338: 8 total, 8 up, 8 in 2024-09-14T03:58:54.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:54 smithi039 ceph-mon[29586]: osdmap e339: 8 total, 8 up, 8 in 2024-09-14T03:58:54.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:54 smithi039 ceph-mon[40962]: osdmap e338: 8 total, 8 up, 8 in 2024-09-14T03:58:54.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:54 smithi039 ceph-mon[40962]: osdmap e339: 8 total, 8 up, 8 in 2024-09-14T03:58:54.673 INFO:teuthology.orchestra.run.smithi039.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:58:54.673 INFO:teuthology.orchestra.run.smithi039.stdout:alertmanager.a smithi039 *:9093,9094 running (63s) 58s ago 6m 16.4M - 0.27.0 11f11916f8cd 6b564e1fc909 2024-09-14T03:58:54.673 INFO:teuthology.orchestra.run.smithi039.stdout:grafana.a smithi129 *:3000 running (41s) 33s ago 6m 66.7M - 10.4.0 c8b91775d855 f0492c19de92 2024-09-14T03:58:54.673 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi039.bbwiwf smithi039 running (5m) 58s ago 5m 20.1M - 18.2.4-845-ge4e227b3 67ddd33fa72f 91e6c8f85681 2024-09-14T03:58:54.673 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi129.ybjqcw smithi129 running (5m) 33s ago 5m 15.0M - 18.2.4-845-ge4e227b3 67ddd33fa72f a3fab5b37b71 2024-09-14T03:58:54.673 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.x smithi129 *:8443,9283,8765 running (2m) 33s ago 10m 453M - 19.3.0-4907-ga2782315 8801d2529e43 448bd4e146cb 2024-09-14T03:58:54.673 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.y smithi039 *:8443,9283,8765 running (2m) 58s ago 12m 514M - 19.3.0-4907-ga2782315 8801d2529e43 cccc87f635a6 2024-09-14T03:58:54.673 INFO:teuthology.orchestra.run.smithi039.stdout:mon.a smithi039 running (12m) 58s ago 12m 61.6M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f e7b5f91fb09c 2024-09-14T03:58:54.673 INFO:teuthology.orchestra.run.smithi039.stdout:mon.b smithi129 running (11m) 33s ago 11m 55.6M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f eb38cc5cbe35 2024-09-14T03:58:54.673 INFO:teuthology.orchestra.run.smithi039.stdout:mon.c smithi039 running (11m) 58s ago 11m 52.2M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f 31d440e49f93 2024-09-14T03:58:54.673 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.a smithi039 *:9100 running (113s) 58s ago 6m 9.90M - 1.7.0 72c9c2088986 7dd5945adb64 2024-09-14T03:58:54.674 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.b smithi129 *:9100 running (107s) 33s ago 6m 11.5M - 1.7.0 72c9c2088986 1fedb5f257e7 2024-09-14T03:58:54.674 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0 smithi039 running (10m) 58s ago 10m 130M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 5048d74f27ac 2024-09-14T03:58:54.674 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1 smithi039 running (10m) 58s ago 10m 119M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 91349527d88b 2024-09-14T03:58:54.674 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2 smithi039 running (9m) 58s ago 9m 105M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f eb903b44fc5d 2024-09-14T03:58:54.674 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3 smithi039 running (9m) 58s ago 9m 119M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f dff9e665bd9b 2024-09-14T03:58:54.674 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4 smithi129 running (8m) 33s ago 8m 113M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 5b64727311b7 2024-09-14T03:58:54.674 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5 smithi129 running (8m) 33s ago 8m 95.5M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f f2d991a754f7 2024-09-14T03:58:54.674 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6 smithi129 running (8m) 33s ago 8m 137M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 3060eaf44106 2024-09-14T03:58:54.674 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7 smithi129 running (7m) 33s ago 7m 98.8M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 4c13b97060d1 2024-09-14T03:58:54.674 INFO:teuthology.orchestra.run.smithi039.stdout:prometheus.a smithi129 *:9095 running (84s) 33s ago 7m 35.5M - 2.51.0 1d3b7f56885b cbe583f852c3 2024-09-14T03:58:55.030 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T03:58:55.030 INFO:teuthology.orchestra.run.smithi039.stdout: "mon": { 2024-09-14T03:58:55.030 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 3 2024-09-14T03:58:55.030 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:58:55.030 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr": { 2024-09-14T03:58:55.030 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:58:55.031 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:58:55.031 INFO:teuthology.orchestra.run.smithi039.stdout: "osd": { 2024-09-14T03:58:55.031 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8 2024-09-14T03:58:55.031 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:58:55.031 INFO:teuthology.orchestra.run.smithi039.stdout: "mds": { 2024-09-14T03:58:55.031 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:58:55.031 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:58:55.031 INFO:teuthology.orchestra.run.smithi039.stdout: "overall": { 2024-09-14T03:58:55.031 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 13, 2024-09-14T03:58:55.031 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:58:55.031 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T03:58:55.031 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T03:58:55.334 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T03:58:55.334 INFO:teuthology.orchestra.run.smithi039.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:58:55.334 INFO:teuthology.orchestra.run.smithi039.stdout: "in_progress": true, 2024-09-14T03:58:55.334 INFO:teuthology.orchestra.run.smithi039.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (1 remaining).", 2024-09-14T03:58:55.334 INFO:teuthology.orchestra.run.smithi039.stdout: "services_complete": [], 2024-09-14T03:58:55.334 INFO:teuthology.orchestra.run.smithi039.stdout: "progress": "0/3 daemons upgraded", 2024-09-14T03:58:55.335 INFO:teuthology.orchestra.run.smithi039.stdout: "message": "Currently upgrading mon daemons", 2024-09-14T03:58:55.335 INFO:teuthology.orchestra.run.smithi039.stdout: "is_paused": false 2024-09-14T03:58:55.335 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T03:58:55.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:55 smithi129 ceph-mon[33173]: from='client.15222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:55.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:55 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:55.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:55 smithi129 ceph-mon[33173]: Upgrade: Target is version 19.3.0-4907-ga2782315 (squid) 2024-09-14T03:58:55.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:55 smithi129 ceph-mon[33173]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T03:58:55.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:55 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:58:55.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:55 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:55.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:55 smithi129 ceph-mon[33173]: Upgrade: Setting container_image for all mgr 2024-09-14T03:58:55.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:55 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:55.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:55 smithi129 ceph-mon[33173]: pgmap v224: 137 pgs: 137 active+clean; 1.5 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.6 MiB/s rd, 24 MiB/s wr, 481 op/s 2024-09-14T03:58:55.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:55 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-14T03:58:55.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:55 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-14T03:58:55.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:55 smithi129 ceph-mon[33173]: Upgrade: It appears safe to stop mon.a 2024-09-14T03:58:55.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:55 smithi129 ceph-mon[33173]: from='client.15228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:55.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:55 smithi129 ceph-mon[33173]: from='client.25136 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:55.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:55 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/3893702431' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:55.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:55 smithi129 ceph-mon[33173]: osdmap e340: 8 total, 8 up, 8 in 2024-09-14T03:58:55.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[29586]: from='client.15222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:55.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:55.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[29586]: Upgrade: Target is version 19.3.0-4907-ga2782315 (squid) 2024-09-14T03:58:55.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[29586]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T03:58:55.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:58:55.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:55.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[29586]: Upgrade: Setting container_image for all mgr 2024-09-14T03:58:55.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:55.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[29586]: pgmap v224: 137 pgs: 137 active+clean; 1.5 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.6 MiB/s rd, 24 MiB/s wr, 481 op/s 2024-09-14T03:58:55.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-14T03:58:55.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-14T03:58:55.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[29586]: Upgrade: It appears safe to stop mon.a 2024-09-14T03:58:55.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[29586]: from='client.15228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:55.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[29586]: from='client.25136 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:55.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[29586]: from='client.? 172.21.15.39:0/3893702431' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:55.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[29586]: osdmap e340: 8 total, 8 up, 8 in 2024-09-14T03:58:55.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[40962]: from='client.15222 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:55.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:55.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[40962]: Upgrade: Target is version 19.3.0-4907-ga2782315 (squid) 2024-09-14T03:58:55.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[40962]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T03:58:55.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:58:55.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:55.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[40962]: Upgrade: Setting container_image for all mgr 2024-09-14T03:58:55.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:55.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[40962]: pgmap v224: 137 pgs: 137 active+clean; 1.5 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.6 MiB/s rd, 24 MiB/s wr, 481 op/s 2024-09-14T03:58:55.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-14T03:58:55.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-09-14T03:58:55.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[40962]: Upgrade: It appears safe to stop mon.a 2024-09-14T03:58:55.543 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[40962]: from='client.15228 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:55.543 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[40962]: from='client.25136 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:55.543 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[40962]: from='client.? 172.21.15.39:0/3893702431' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:58:55.543 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:55 smithi039 ceph-mon[40962]: osdmap e340: 8 total, 8 up, 8 in 2024-09-14T03:58:56.146 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 236575319 2024-09-14T03:58:56.158 INFO:tasks.workunit.client.0.smithi039.stdout:write: b 96430577~1763152 2024-09-14T03:58:56.166 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 232707800~3867519 2024-09-14T03:58:56.166 INFO:tasks.workunit.client.0.smithi039.stdout:write: b 104025766~7853103 2024-09-14T03:58:56.173 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 191425920~4873757 2024-09-14T03:58:56.173 INFO:tasks.workunit.client.0.smithi039.stdout:write: b 88240633~2764864 2024-09-14T03:58:56.177 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 203185938~3738385 2024-09-14T03:58:56.179 INFO:tasks.workunit.client.0.smithi039.stdout:write: b 47697789~2318497 2024-09-14T03:58:56.181 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 187006079~485224 2024-09-14T03:58:56.181 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: user1 2024-09-14T03:58:56.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:56 smithi129 ceph-mon[33173]: from='client.15246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:56.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:56 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:56.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:56 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:58:56.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:56 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:58:56.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:56 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:58:56.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:56 smithi129 ceph-mon[33173]: osdmap e341: 8 total, 8 up, 8 in 2024-09-14T03:58:56.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:56 smithi039 ceph-mon[29586]: from='client.15246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:56.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:56 smithi039 ceph-mon[29586]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:56.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:56 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:58:56.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:56 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:58:56.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:56 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:58:56.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:56 smithi039 ceph-mon[29586]: osdmap e341: 8 total, 8 up, 8 in 2024-09-14T03:58:56.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:56 smithi039 ceph-mon[40962]: from='client.15246 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:58:56.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:56 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:58:56.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:56 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:58:56.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:56 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:58:56.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:56 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:58:56.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:56 smithi039 ceph-mon[40962]: osdmap e341: 8 total, 8 up, 8 in 2024-09-14T03:58:57.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:57 smithi129 ceph-mon[33173]: Upgrade: Updating mon.a 2024-09-14T03:58:57.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:57 smithi129 ceph-mon[33173]: Deploying daemon mon.a on smithi039 2024-09-14T03:58:57.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:57 smithi129 ceph-mon[33173]: pgmap v227: 137 pgs: 137 active+clean; 1.5 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.6 MiB/s rd, 24 MiB/s wr, 481 op/s 2024-09-14T03:58:57.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:57 smithi129 ceph-mon[33173]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:58:57.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:57 smithi039 ceph-mon[29586]: Upgrade: Updating mon.a 2024-09-14T03:58:57.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:57 smithi039 ceph-mon[29586]: Deploying daemon mon.a on smithi039 2024-09-14T03:58:57.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:57 smithi039 ceph-mon[29586]: pgmap v227: 137 pgs: 137 active+clean; 1.5 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.6 MiB/s rd, 24 MiB/s wr, 481 op/s 2024-09-14T03:58:57.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:57 smithi039 ceph-mon[29586]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:58:57.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:57 smithi039 ceph-mon[40962]: Upgrade: Updating mon.a 2024-09-14T03:58:57.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:57 smithi039 ceph-mon[40962]: Deploying daemon mon.a on smithi039 2024-09-14T03:58:57.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:57 smithi039 ceph-mon[40962]: pgmap v227: 137 pgs: 137 active+clean; 1.5 GiB data, 3.9 GiB used, 711 GiB / 715 GiB avail; 7.6 MiB/s rd, 24 MiB/s wr, 481 op/s 2024-09-14T03:58:57.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:57 smithi039 ceph-mon[40962]: from='mgr.24944 172.21.15.39:0/57856376' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:58:58.303 INFO:tasks.workunit.client.0.smithi039.stdout:write: c 168896357~5064564 2024-09-14T03:58:58.316 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 143445772~5480165 2024-09-14T03:58:58.317 INFO:tasks.workunit.client.0.smithi039.stdout:write: c 114223032~942247 2024-09-14T03:58:58.317 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 161007809~3047711 2024-09-14T03:58:58.318 INFO:tasks.workunit.client.0.smithi039.stdout:write: c 25876693~4807760 2024-09-14T03:58:58.324 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 187001272~5916470 2024-09-14T03:58:58.324 INFO:tasks.workunit.client.0.smithi039.stdout:write: c 37202020~698787 2024-09-14T03:58:58.325 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 100517954~5992210 2024-09-14T03:58:58.325 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: user2 2024-09-14T03:58:58.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:58 smithi039 systemd[1]: Stopping Ceph mon.a for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:58:58.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:58 smithi039 ceph-mon[29586]: osdmap e342: 8 total, 8 up, 8 in 2024-09-14T03:58:58.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:58:58 smithi039 ceph-mon[40962]: osdmap e342: 8 total, 8 up, 8 in 2024-09-14T03:58:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:58:58 smithi129 ceph-mon[33173]: osdmap e342: 8 total, 8 up, 8 in 2024-09-14T03:58:58.948 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-a[29563]: 2024-09-14T03:58:58.568+0000 7f73160ae640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:58:58.948 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-a[29563]: 2024-09-14T03:58:58.568+0000 7f73160ae640 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-09-14T03:58:58.948 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:58 smithi039 podman[106281]: 2024-09-14 03:58:58.660204672 +0000 UTC m=+0.196212474 container died e7b5f91fb09c59d21180020e9b0c4afbfc8e6c4fecab1b88885aabba4f957667 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-a, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=reef-e4e227b, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-14T03:58:58.948 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:58:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:58:58] "GET /metrics HTTP/1.1" 200 38871 "" "Prometheus/2.51.0" 2024-09-14T03:58:59.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:59 smithi039 podman[106281]: 2024-09-14 03:58:59.146319449 +0000 UTC m=+0.682327273 container cleanup e7b5f91fb09c59d21180020e9b0c4afbfc8e6c4fecab1b88885aabba4f957667 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-a, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-e4e227b, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-14T03:58:59.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:59 smithi039 bash[106281]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-a 2024-09-14T03:58:59.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:59 smithi039 podman[106299]: 2024-09-14 03:58:59.414634247 +0000 UTC m=+0.756078645 container remove e7b5f91fb09c59d21180020e9b0c4afbfc8e6c4fecab1b88885aabba4f957667 (image=quay.ceph.io/ceph-ci/ceph:reef, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-a, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, RELEASE=reef-e4e227b, org.label-schema.schema-version=1.0) 2024-09-14T03:58:59.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:59 smithi039 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-14T03:58:59.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:59 smithi039 systemd[1]: Stopped Ceph mon.a for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:58:59.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:59 smithi039 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mon.a.service: Consumed 20.548s CPU time. 2024-09-14T03:59:00.182 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:58:59 smithi039 systemd[1]: Starting Ceph mon.a for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:59:00.182 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 podman[106432]: 2024-09-14 03:58:59.910600703 +0000 UTC m=+0.016545710 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:59:00.182 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 podman[106432]: 2024-09-14 03:59:00.040850362 +0000 UTC m=+0.146795419 container create 7198d4eb8e4c1000ace8fa1414a8a80d152670ad9bb4f2deeb4ab26f42e2cbf9 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-a, maintainer=Guillaume Abrioux , GIT_CLEAN=True, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, io.buildah.version=1.37.2) 2024-09-14T03:59:00.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 podman[106432]: 2024-09-14 03:59:00.285015657 +0000 UTC m=+0.390960666 container init 7198d4eb8e4c1000ace8fa1414a8a80d152670ad9bb4f2deeb4ab26f42e2cbf9 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-a, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=main-a278231, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T03:59:00.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 podman[106432]: 2024-09-14 03:59:00.289961523 +0000 UTC m=+0.395906521 container start 7198d4eb8e4c1000ace8fa1414a8a80d152670ad9bb4f2deeb4ab26f42e2cbf9 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-a, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.license=GPLv2, RELEASE=main-a278231, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True) 2024-09-14T03:59:00.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: set uid:gid to 167:167 (ceph:ceph) 2024-09-14T03:59:00.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev), process ceph-mon, pid 7 2024-09-14T03:59:00.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: pidfile_write: ignore empty --pid-file 2024-09-14T03:59:00.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: load: jerasure load: lrc 2024-09-14T03:59:00.542 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: RocksDB version: 7.9.2 2024-09-14T03:59:00.542 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Git sha 0 2024-09-14T03:59:00.542 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Compile date 2024-09-12 20:09:22 2024-09-14T03:59:00.542 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: DB SUMMARY 2024-09-14T03:59:00.542 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: DB Session ID: QBL60O8M5WI6TMZTP9WN 2024-09-14T03:59:00.542 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: CURRENT file: CURRENT 2024-09-14T03:59:00.542 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: IDENTITY file: IDENTITY 2024-09-14T03:59:00.542 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: MANIFEST file: MANIFEST-000015 size: 1520 Bytes 2024-09-14T03:59:00.542 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000032.sst 2024-09-14T03:59:00.543 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000030.log size: 6778212 ; 2024-09-14T03:59:00.543 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.error_if_exists: 0 2024-09-14T03:59:00.543 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.create_if_missing: 0 2024-09-14T03:59:00.543 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.paranoid_checks: 1 2024-09-14T03:59:00.543 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-14T03:59:00.543 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-14T03:59:00.543 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-14T03:59:00.543 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.env: 0x56274d993120 2024-09-14T03:59:00.543 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.fs: PosixFileSystem 2024-09-14T03:59:00.543 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.info_log: 0x56274e5f1700 2024-09-14T03:59:00.544 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.max_file_opening_threads: 16 2024-09-14T03:59:00.545 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.statistics: (nil) 2024-09-14T03:59:00.545 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.use_fsync: 0 2024-09-14T03:59:00.545 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.max_log_file_size: 0 2024-09-14T03:59:00.546 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-14T03:59:00.546 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-14T03:59:00.546 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.keep_log_file_num: 1000 2024-09-14T03:59:00.546 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.recycle_log_file_num: 0 2024-09-14T03:59:00.546 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.allow_fallocate: 1 2024-09-14T03:59:00.546 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.allow_mmap_reads: 0 2024-09-14T03:59:00.546 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.allow_mmap_writes: 0 2024-09-14T03:59:00.546 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.use_direct_reads: 0 2024-09-14T03:59:00.546 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-14T03:59:00.546 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.create_missing_column_families: 0 2024-09-14T03:59:00.546 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.db_log_dir: 2024-09-14T03:59:00.546 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.wal_dir: 2024-09-14T03:59:00.547 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-14T03:59:00.547 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-14T03:59:00.547 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-14T03:59:00.547 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-14T03:59:00.547 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-14T03:59:00.547 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-14T03:59:00.547 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.advise_random_on_open: 1 2024-09-14T03:59:00.547 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.db_write_buffer_size: 0 2024-09-14T03:59:00.547 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.write_buffer_manager: 0x56274e5f41e0 2024-09-14T03:59:00.547 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-14T03:59:00.547 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-14T03:59:00.547 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-14T03:59:00.548 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.rate_limiter: (nil) 2024-09-14T03:59:00.548 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-14T03:59:00.548 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.wal_recovery_mode: 2 2024-09-14T03:59:00.548 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.enable_thread_tracking: 0 2024-09-14T03:59:00.548 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.enable_pipelined_write: 0 2024-09-14T03:59:00.548 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.unordered_write: 0 2024-09-14T03:59:00.548 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-14T03:59:00.548 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-14T03:59:00.548 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-14T03:59:00.548 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-14T03:59:00.548 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.row_cache: None 2024-09-14T03:59:00.548 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.wal_filter: None 2024-09-14T03:59:00.549 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-14T03:59:00.549 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.allow_ingest_behind: 0 2024-09-14T03:59:00.549 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.two_write_queues: 0 2024-09-14T03:59:00.549 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.manual_wal_flush: 0 2024-09-14T03:59:00.549 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.wal_compression: 0 2024-09-14T03:59:00.549 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.atomic_flush: 0 2024-09-14T03:59:00.549 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-14T03:59:00.549 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-14T03:59:00.549 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-14T03:59:00.549 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.log_readahead_size: 0 2024-09-14T03:59:00.549 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-14T03:59:00.549 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.best_efforts_recovery: 0 2024-09-14T03:59:00.549 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-14T03:59:00.550 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-14T03:59:00.550 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.allow_data_in_errors: 0 2024-09-14T03:59:00.550 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.db_host_id: __hostname__ 2024-09-14T03:59:00.550 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-14T03:59:00.550 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.max_background_jobs: 2 2024-09-14T03:59:00.550 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.max_background_compactions: -1 2024-09-14T03:59:00.550 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.max_subcompactions: 1 2024-09-14T03:59:00.550 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-14T03:59:00.550 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-14T03:59:00.550 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-14T03:59:00.550 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.max_total_wal_size: 0 2024-09-14T03:59:00.550 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-14T03:59:00.550 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-14T03:59:00.551 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-14T03:59:00.551 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-14T03:59:00.551 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.max_open_files: -1 2024-09-14T03:59:00.551 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.bytes_per_sync: 0 2024-09-14T03:59:00.551 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-14T03:59:00.551 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-14T03:59:00.551 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.compaction_readahead_size: 0 2024-09-14T03:59:00.551 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.max_background_flushes: -1 2024-09-14T03:59:00.551 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Compression algorithms supported: 2024-09-14T03:59:00.551 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: kZSTD supported: 0 2024-09-14T03:59:00.551 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: kXpressCompression supported: 0 2024-09-14T03:59:00.551 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: kBZip2Compression supported: 0 2024-09-14T03:59:00.551 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-14T03:59:00.552 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: kLZ4Compression supported: 1 2024-09-14T03:59:00.552 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: kZlibCompression supported: 1 2024-09-14T03:59:00.552 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: kLZ4HCCompression supported: 1 2024-09-14T03:59:00.552 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: kSnappyCompression supported: 1 2024-09-14T03:59:00.552 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-14T03:59:00.552 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-14T03:59:00.552 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-09-14T03:59:00.552 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-14T03:59:00.552 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-14T03:59:00.552 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.merge_operator: 2024-09-14T03:59:00.552 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.compaction_filter: None 2024-09-14T03:59:00.552 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.compaction_filter_factory: None 2024-09-14T03:59:00.553 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.sst_partitioner_factory: None 2024-09-14T03:59:00.553 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-14T03:59:00.553 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-14T03:59:00.553 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56274e5f1320) 2024-09-14T03:59:00.553 INFO:journalctl@ceph.mon.a.smithi039.stdout: cache_index_and_filter_blocks: 1 2024-09-14T03:59:00.553 INFO:journalctl@ceph.mon.a.smithi039.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-14T03:59:00.553 INFO:journalctl@ceph.mon.a.smithi039.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-14T03:59:00.553 INFO:journalctl@ceph.mon.a.smithi039.stdout: pin_top_level_index_and_filter: 1 2024-09-14T03:59:00.553 INFO:journalctl@ceph.mon.a.smithi039.stdout: index_type: 0 2024-09-14T03:59:00.553 INFO:journalctl@ceph.mon.a.smithi039.stdout: data_block_index_type: 0 2024-09-14T03:59:00.553 INFO:journalctl@ceph.mon.a.smithi039.stdout: index_shortening: 1 2024-09-14T03:59:00.553 INFO:journalctl@ceph.mon.a.smithi039.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-14T03:59:00.553 INFO:journalctl@ceph.mon.a.smithi039.stdout: checksum: 4 2024-09-14T03:59:00.553 INFO:journalctl@ceph.mon.a.smithi039.stdout: no_block_cache: 0 2024-09-14T03:59:00.554 INFO:journalctl@ceph.mon.a.smithi039.stdout: block_cache: 0x56274e5e5610 2024-09-14T03:59:00.554 INFO:journalctl@ceph.mon.a.smithi039.stdout: block_cache_name: BinnedLRUCache 2024-09-14T03:59:00.554 INFO:journalctl@ceph.mon.a.smithi039.stdout: block_cache_options: 2024-09-14T03:59:00.554 INFO:journalctl@ceph.mon.a.smithi039.stdout: capacity : 536870912 2024-09-14T03:59:00.554 INFO:journalctl@ceph.mon.a.smithi039.stdout: num_shard_bits : 4 2024-09-14T03:59:00.554 INFO:journalctl@ceph.mon.a.smithi039.stdout: strict_capacity_limit : 0 2024-09-14T03:59:00.554 INFO:journalctl@ceph.mon.a.smithi039.stdout: high_pri_pool_ratio: 0.000 2024-09-14T03:59:00.554 INFO:journalctl@ceph.mon.a.smithi039.stdout: block_cache_compressed: (nil) 2024-09-14T03:59:00.554 INFO:journalctl@ceph.mon.a.smithi039.stdout: persistent_cache: (nil) 2024-09-14T03:59:00.554 INFO:journalctl@ceph.mon.a.smithi039.stdout: block_size: 4096 2024-09-14T03:59:00.554 INFO:journalctl@ceph.mon.a.smithi039.stdout: block_size_deviation: 10 2024-09-14T03:59:00.554 INFO:journalctl@ceph.mon.a.smithi039.stdout: block_restart_interval: 16 2024-09-14T03:59:00.554 INFO:journalctl@ceph.mon.a.smithi039.stdout: index_block_restart_interval: 1 2024-09-14T03:59:00.554 INFO:journalctl@ceph.mon.a.smithi039.stdout: metadata_block_size: 4096 2024-09-14T03:59:00.555 INFO:journalctl@ceph.mon.a.smithi039.stdout: partition_filters: 0 2024-09-14T03:59:00.555 INFO:journalctl@ceph.mon.a.smithi039.stdout: use_delta_encoding: 1 2024-09-14T03:59:00.555 INFO:journalctl@ceph.mon.a.smithi039.stdout: filter_policy: bloomfilter 2024-09-14T03:59:00.555 INFO:journalctl@ceph.mon.a.smithi039.stdout: whole_key_filtering: 1 2024-09-14T03:59:00.555 INFO:journalctl@ceph.mon.a.smithi039.stdout: verify_compression: 0 2024-09-14T03:59:00.555 INFO:journalctl@ceph.mon.a.smithi039.stdout: read_amp_bytes_per_bit: 0 2024-09-14T03:59:00.555 INFO:journalctl@ceph.mon.a.smithi039.stdout: format_version: 5 2024-09-14T03:59:00.555 INFO:journalctl@ceph.mon.a.smithi039.stdout: enable_index_compression: 1 2024-09-14T03:59:00.555 INFO:journalctl@ceph.mon.a.smithi039.stdout: block_align: 0 2024-09-14T03:59:00.555 INFO:journalctl@ceph.mon.a.smithi039.stdout: max_auto_readahead_size: 262144 2024-09-14T03:59:00.555 INFO:journalctl@ceph.mon.a.smithi039.stdout: prepopulate_block_cache: 0 2024-09-14T03:59:00.555 INFO:journalctl@ceph.mon.a.smithi039.stdout: initial_auto_readahead_size: 8192 2024-09-14T03:59:00.556 INFO:journalctl@ceph.mon.a.smithi039.stdout: num_file_reads_for_auto_readahead: 2 2024-09-14T03:59:00.556 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.write_buffer_size: 33554432 2024-09-14T03:59:00.556 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.max_write_buffer_number: 2 2024-09-14T03:59:00.556 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.compression: NoCompression 2024-09-14T03:59:00.556 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.bottommost_compression: Disabled 2024-09-14T03:59:00.556 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.prefix_extractor: nullptr 2024-09-14T03:59:00.556 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-14T03:59:00.556 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.num_levels: 7 2024-09-14T03:59:00.556 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-14T03:59:00.556 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-14T03:59:00.556 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-14T03:59:00.556 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-14T03:59:00.556 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-14T03:59:00.557 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-14T03:59:00.557 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-14T03:59:00.557 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:59:00.557 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-14T03:59:00.557 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-14T03:59:00.557 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:59:00.557 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:59:00.557 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-14T03:59:00.557 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.compression_opts.level: 32767 2024-09-14T03:59:00.557 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.compression_opts.strategy: 0 2024-09-14T03:59:00.557 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-14T03:59:00.557 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:59:00.557 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:59:00.557 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-14T03:59:00.558 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.compression_opts.enabled: false 2024-09-14T03:59:00.558 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:59:00.558 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-14T03:59:00.558 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-14T03:59:00.558 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-14T03:59:00.558 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.target_file_size_base: 67108864 2024-09-14T03:59:00.558 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-14T03:59:00.558 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-14T03:59:00.558 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-14T03:59:00.558 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-14T03:59:00.558 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-14T03:59:00.558 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-14T03:59:00.558 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-14T03:59:00.559 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-14T03:59:00.559 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-14T03:59:00.559 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-14T03:59:00.559 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-14T03:59:00.559 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-14T03:59:00.559 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-14T03:59:00.559 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-14T03:59:00.559 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.arena_block_size: 1048576 2024-09-14T03:59:00.559 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-14T03:59:00.559 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-14T03:59:00.559 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.disable_auto_compactions: 0 2024-09-14T03:59:00.559 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-14T03:59:00.559 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-14T03:59:00.559 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-14T03:59:00.560 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-14T03:59:00.560 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-14T03:59:00.560 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-14T03:59:00.560 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-14T03:59:00.560 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-14T03:59:00.560 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-14T03:59:00.560 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-14T03:59:00.560 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-14T03:59:00.560 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.inplace_update_support: 0 2024-09-14T03:59:00.560 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-14T03:59:00.560 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-14T03:59:00.560 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-14T03:59:00.561 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-14T03:59:00.561 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.bloom_locality: 0 2024-09-14T03:59:00.561 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.max_successive_merges: 0 2024-09-14T03:59:00.561 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-14T03:59:00.561 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.paranoid_file_checks: 0 2024-09-14T03:59:00.561 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.force_consistency_checks: 1 2024-09-14T03:59:00.561 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.report_bg_io_stats: 0 2024-09-14T03:59:00.561 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.ttl: 2592000 2024-09-14T03:59:00.561 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-14T03:59:00.561 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-14T03:59:00.561 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-14T03:59:00.561 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.enable_blob_files: false 2024-09-14T03:59:00.561 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.min_blob_size: 0 2024-09-14T03:59:00.562 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.blob_file_size: 268435456 2024-09-14T03:59:00.562 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-14T03:59:00.562 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-14T03:59:00.562 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-14T03:59:00.562 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-14T03:59:00.562 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-14T03:59:00.562 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.blob_file_starting_level: 0 2024-09-14T03:59:00.562 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-14T03:59:00.562 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-14T03:59:00.562 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 34, last_sequence is 12785, log_number is 30,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 30 2024-09-14T03:59:00.562 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 30 2024-09-14T03:59:00.562 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 3257acd3-3704-476a-8f05-05d73ed70edd 2024-09-14T03:59:00.562 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726286340317288, "job": 1, "event": "recovery_started", "wal_files": [30]} 2024-09-14T03:59:00.562 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #30 mode 2 2024-09-14T03:59:00.563 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726286340335858, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 35, "file_size": 6292338, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12786, "largest_seqno": 17111, "table_properties": {"data_size": 6271504, "index_size": 14795, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 4997, "raw_key_size": 45128, "raw_average_key_size": 22, "raw_value_size": 6228602, "raw_average_value_size": 3152, "num_data_blocks": 640, "num_entries": 1976, "num_filter_entries": 1976, "num_deletions": 270, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726286340, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "3257acd3-3704-476a-8f05-05d73ed70edd", "db_session_id": "QBL60O8M5WI6TMZTP9WN", "orig_file_number": 35, "seqno_to_time_mapping": "N/A"}} 2024-09-14T03:59:00.563 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726286340335986, "job": 1, "event": "recovery_finished"} 2024-09-14T03:59:00.563 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: [db/version_set.cc:5047] Creating manifest 37 2024-09-14T03:59:00.563 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-14T03:59:00.563 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000030.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-14T03:59:00.563 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x56274e614e00 2024-09-14T03:59:00.563 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: DB pointer 0x56274e716000 2024-09-14T03:59:00.563 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-14T03:59:00.563 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-14T03:59:00.563 INFO:journalctl@ceph.mon.a.smithi039.stdout: ** DB Stats ** 2024-09-14T03:59:00.563 INFO:journalctl@ceph.mon.a.smithi039.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:59:00.563 INFO:journalctl@ceph.mon.a.smithi039.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-14T03:59:00.564 INFO:journalctl@ceph.mon.a.smithi039.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:59:00.564 INFO:journalctl@ceph.mon.a.smithi039.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:59:00.564 INFO:journalctl@ceph.mon.a.smithi039.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-14T03:59:00.564 INFO:journalctl@ceph.mon.a.smithi039.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:59:00.564 INFO:journalctl@ceph.mon.a.smithi039.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:59:00.564 INFO:journalctl@ceph.mon.a.smithi039.stdout: 2024-09-14T03:59:00.564 INFO:journalctl@ceph.mon.a.smithi039.stdout: ** Compaction Stats [default] ** 2024-09-14T03:59:00.564 INFO:journalctl@ceph.mon.a.smithi039.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:59:00.564 INFO:journalctl@ceph.mon.a.smithi039.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-14T03:59:00.565 INFO:journalctl@ceph.mon.a.smithi039.stdout: L0 1/0 6.00 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 749.7 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-14T03:59:00.565 INFO:journalctl@ceph.mon.a.smithi039.stdout: L6 1/0 8.55 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-14T03:59:00.565 INFO:journalctl@ceph.mon.a.smithi039.stdout: Sum 2/0 14.55 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 749.7 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-14T03:59:00.565 INFO:journalctl@ceph.mon.a.smithi039.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 749.7 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-14T03:59:00.565 INFO:journalctl@ceph.mon.a.smithi039.stdout: 2024-09-14T03:59:00.565 INFO:journalctl@ceph.mon.a.smithi039.stdout: ** Compaction Stats [default] ** 2024-09-14T03:59:00.565 INFO:journalctl@ceph.mon.a.smithi039.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:59:00.565 INFO:journalctl@ceph.mon.a.smithi039.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:59:00.565 INFO:journalctl@ceph.mon.a.smithi039.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 749.7 0.01 0.00 1 0.008 0 0 0.0 0.0 2024-09-14T03:59:00.565 INFO:journalctl@ceph.mon.a.smithi039.stdout: 2024-09-14T03:59:00.565 INFO:journalctl@ceph.mon.a.smithi039.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-14T03:59:00.565 INFO:journalctl@ceph.mon.a.smithi039.stdout: 2024-09-14T03:59:00.565 INFO:journalctl@ceph.mon.a.smithi039.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:59:00.566 INFO:journalctl@ceph.mon.a.smithi039.stdout: Flush(GB): cumulative 0.006, interval 0.006 2024-09-14T03:59:00.566 INFO:journalctl@ceph.mon.a.smithi039.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-14T03:59:00.566 INFO:journalctl@ceph.mon.a.smithi039.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-14T03:59:00.566 INFO:journalctl@ceph.mon.a.smithi039.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-14T03:59:00.566 INFO:journalctl@ceph.mon.a.smithi039.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-14T03:59:00.566 INFO:journalctl@ceph.mon.a.smithi039.stdout: Cumulative compaction: 0.01 GB write, 256.91 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:59:00.566 INFO:journalctl@ceph.mon.a.smithi039.stdout: Interval compaction: 0.01 GB write, 256.91 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:59:00.566 INFO:journalctl@ceph.mon.a.smithi039.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-14T03:59:00.566 INFO:journalctl@ceph.mon.a.smithi039.stdout: Block cache BinnedLRUCache@0x56274e5e5610#7 capacity: 512.00 MB usage: 21.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-14T03:59:00.566 INFO:journalctl@ceph.mon.a.smithi039.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,5.03 KB,0.000959635%) IndexBlock(1,16.08 KB,0.00306666%) Misc(1,0.00 KB,0%) 2024-09-14T03:59:00.566 INFO:journalctl@ceph.mon.a.smithi039.stdout: 2024-09-14T03:59:00.567 INFO:journalctl@ceph.mon.a.smithi039.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-14T03:59:00.567 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: starting mon.a rank 0 at public addrs [v2:172.21.15.39:3300/0,v1:172.21.15.39:6789/0] at bind addrs [v2:172.21.15.39:3300/0,v1:172.21.15.39:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T03:59:00.567 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: mon.a@-1(???) e3 preinit fsid bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T03:59:00.567 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: mon.a@-1(???).mds e9 new map 2024-09-14T03:59:00.567 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: mon.a@-1(???).mds e9 print_map 2024-09-14T03:59:00.567 INFO:journalctl@ceph.mon.a.smithi039.stdout: e9 2024-09-14T03:59:00.567 INFO:journalctl@ceph.mon.a.smithi039.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-14T03:59:00.567 INFO:journalctl@ceph.mon.a.smithi039.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-14T03:59:00.567 INFO:journalctl@ceph.mon.a.smithi039.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:59:00.567 INFO:journalctl@ceph.mon.a.smithi039.stdout: legacy client fscid: 1 2024-09-14T03:59:00.567 INFO:journalctl@ceph.mon.a.smithi039.stdout: 2024-09-14T03:59:00.567 INFO:journalctl@ceph.mon.a.smithi039.stdout: Filesystem 'foo' (1) 2024-09-14T03:59:00.567 INFO:journalctl@ceph.mon.a.smithi039.stdout: fs_name foo 2024-09-14T03:59:00.568 INFO:journalctl@ceph.mon.a.smithi039.stdout: epoch 8 2024-09-14T03:59:00.568 INFO:journalctl@ceph.mon.a.smithi039.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-14T03:59:00.568 INFO:journalctl@ceph.mon.a.smithi039.stdout: created 2024-09-14T03:53:05.236201+0000 2024-09-14T03:59:00.568 INFO:journalctl@ceph.mon.a.smithi039.stdout: modified 2024-09-14T03:53:22.726472+0000 2024-09-14T03:59:00.568 INFO:journalctl@ceph.mon.a.smithi039.stdout: tableserver 0 2024-09-14T03:59:00.568 INFO:journalctl@ceph.mon.a.smithi039.stdout: root 0 2024-09-14T03:59:00.568 INFO:journalctl@ceph.mon.a.smithi039.stdout: session_timeout 60 2024-09-14T03:59:00.568 INFO:journalctl@ceph.mon.a.smithi039.stdout: session_autoclose 300 2024-09-14T03:59:00.568 INFO:journalctl@ceph.mon.a.smithi039.stdout: max_file_size 1099511627776 2024-09-14T03:59:00.568 INFO:journalctl@ceph.mon.a.smithi039.stdout: max_xattr_size 65536 2024-09-14T03:59:00.568 INFO:journalctl@ceph.mon.a.smithi039.stdout: required_client_features {} 2024-09-14T03:59:00.568 INFO:journalctl@ceph.mon.a.smithi039.stdout: last_failure 0 2024-09-14T03:59:00.569 INFO:journalctl@ceph.mon.a.smithi039.stdout: last_failure_osd_epoch 0 2024-09-14T03:59:00.569 INFO:journalctl@ceph.mon.a.smithi039.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:59:00.569 INFO:journalctl@ceph.mon.a.smithi039.stdout: max_mds 1 2024-09-14T03:59:00.569 INFO:journalctl@ceph.mon.a.smithi039.stdout: in 0 2024-09-14T03:59:00.569 INFO:journalctl@ceph.mon.a.smithi039.stdout: up {0=14646} 2024-09-14T03:59:00.569 INFO:journalctl@ceph.mon.a.smithi039.stdout: failed 2024-09-14T03:59:00.569 INFO:journalctl@ceph.mon.a.smithi039.stdout: damaged 2024-09-14T03:59:00.569 INFO:journalctl@ceph.mon.a.smithi039.stdout: stopped 2024-09-14T03:59:00.569 INFO:journalctl@ceph.mon.a.smithi039.stdout: data_pools [4] 2024-09-14T03:59:00.569 INFO:journalctl@ceph.mon.a.smithi039.stdout: metadata_pool 3 2024-09-14T03:59:00.570 INFO:journalctl@ceph.mon.a.smithi039.stdout: inline_data disabled 2024-09-14T03:59:00.570 INFO:journalctl@ceph.mon.a.smithi039.stdout: balancer 2024-09-14T03:59:00.570 INFO:journalctl@ceph.mon.a.smithi039.stdout: bal_rank_mask -1 2024-09-14T03:59:00.570 INFO:journalctl@ceph.mon.a.smithi039.stdout: standby_count_wanted 1 2024-09-14T03:59:00.570 INFO:journalctl@ceph.mon.a.smithi039.stdout: qdb_cluster leader: 0 members: 2024-09-14T03:59:00.570 INFO:journalctl@ceph.mon.a.smithi039.stdout: [mds.foo.smithi039.bbwiwf{0:14646} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.39:6834/2717630652,v1:172.21.15.39:6835/2717630652] compat {c=[1],r=[1],i=[7ff]}] 2024-09-14T03:59:00.570 INFO:journalctl@ceph.mon.a.smithi039.stdout: 2024-09-14T03:59:00.570 INFO:journalctl@ceph.mon.a.smithi039.stdout: 2024-09-14T03:59:00.570 INFO:journalctl@ceph.mon.a.smithi039.stdout: Standby daemons: 2024-09-14T03:59:00.570 INFO:journalctl@ceph.mon.a.smithi039.stdout: 2024-09-14T03:59:00.570 INFO:journalctl@ceph.mon.a.smithi039.stdout: [mds.foo.smithi129.ybjqcw{-1:24475} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.129:6832/2789452907,v1:172.21.15.129:6833/2789452907] compat {c=[1],r=[1],i=[7ff]}] 2024-09-14T03:59:00.570 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: mon.a@-1(???).osd e343 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-14T03:59:00.571 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: mon.a@-1(???).osd e343 crush map has features 432629239337189376, adjusting msgr requires 2024-09-14T03:59:00.571 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: mon.a@-1(???).osd e343 crush map has features 432629239337189376, adjusting msgr requires 2024-09-14T03:59:00.571 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: mon.a@-1(???).osd e343 crush map has features 432629239337189376, adjusting msgr requires 2024-09-14T03:59:00.571 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 ceph-mon[106470]: mon.a@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2024-09-14T03:59:00.571 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 bash[106432]: 7198d4eb8e4c1000ace8fa1414a8a80d152670ad9bb4f2deeb4ab26f42e2cbf9 2024-09-14T03:59:00.571 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:00 smithi039 systemd[1]: Started Ceph mon.a for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:59:01.271 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:59:01 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[80442]: ignoring --setuser ceph since I am not root 2024-09-14T03:59:01.271 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:59:01 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[80442]: ignoring --setgroup ceph since I am not root 2024-09-14T03:59:01.272 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:59:01 smithi129 ceph-mgr[80465]: -- 172.21.15.129:0/3452519608 <== mon.1 v2:172.21.15.129:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b32b95d0e0 con 0x55b32b93b000 2024-09-14T03:59:01.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:01 smithi129 ceph-mon[33173]: pgmap v229: 137 pgs: 137 active+clean; 1.6 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 28 MiB/s rd, 34 MiB/s wr, 659 op/s 2024-09-14T03:59:01.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:01 smithi129 ceph-mon[33173]: mon.a calling monitor election 2024-09-14T03:59:01.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:01 smithi129 ceph-mon[33173]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:59:01.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:01 smithi129 ceph-mon[33173]: monmap epoch 3 2024-09-14T03:59:01.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:01 smithi129 ceph-mon[33173]: fsid bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T03:59:01.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:01 smithi129 ceph-mon[33173]: last_changed 2024-09-14T03:47:43.672905+0000 2024-09-14T03:59:01.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:01 smithi129 ceph-mon[33173]: created 2024-09-14T03:46:10.112174+0000 2024-09-14T03:59:01.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:01 smithi129 ceph-mon[33173]: min_mon_release 18 (reef) 2024-09-14T03:59:01.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:01 smithi129 ceph-mon[33173]: election_strategy: 1 2024-09-14T03:59:01.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:01 smithi129 ceph-mon[33173]: 0: [v2:172.21.15.39:3300/0,v1:172.21.15.39:6789/0] mon.a 2024-09-14T03:59:01.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:01 smithi129 ceph-mon[33173]: 1: [v2:172.21.15.129:3300/0,v1:172.21.15.129:6789/0] mon.b 2024-09-14T03:59:01.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:01 smithi129 ceph-mon[33173]: 2: [v2:172.21.15.39:3301/0,v1:172.21.15.39:6790/0] mon.c 2024-09-14T03:59:01.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:01 smithi129 ceph-mon[33173]: fsmap foo:1 {0=foo.smithi039.bbwiwf=up:active} 1 up:standby 2024-09-14T03:59:01.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:01 smithi129 ceph-mon[33173]: osdmap e343: 8 total, 8 up, 8 in 2024-09-14T03:59:01.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:01 smithi129 ceph-mon[33173]: mgrmap e34: y(active, since 2m), standbys: x 2024-09-14T03:59:01.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:01 smithi129 ceph-mon[33173]: Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-09-14T03:59:01.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:01 smithi129 ceph-mon[33173]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-09-14T03:59:01.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:01 smithi129 ceph-mon[33173]: application not enabled on pool 'test-librbd-smithi039-85658-9' 2024-09-14T03:59:01.274 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:01 smithi129 ceph-mon[33173]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-09-14T03:59:01.274 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:01 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:59:01.274 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:01 smithi129 ceph-mon[33173]: mgrmap e35: y(active, since 2m), standbys: x 2024-09-14T03:59:01.274 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:01 smithi129 ceph-mon[33173]: from='mgr.24944 ' entity='' 2024-09-14T03:59:01.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ignoring --setuser ceph since I am not root 2024-09-14T03:59:01.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ignoring --setgroup ceph since I am not root 2024-09-14T03:59:01.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mgr[89481]: -- 172.21.15.39:0/2800649113 <== mon.2 v2:172.21.15.39:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55c75a44f0e0 con 0x55c75a42d000 2024-09-14T03:59:01.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[106470]: pgmap v229: 137 pgs: 137 active+clean; 1.6 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 28 MiB/s rd, 34 MiB/s wr, 659 op/s 2024-09-14T03:59:01.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[106470]: mon.a calling monitor election 2024-09-14T03:59:01.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[106470]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:59:01.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[106470]: monmap epoch 3 2024-09-14T03:59:01.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[106470]: fsid bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T03:59:01.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[106470]: last_changed 2024-09-14T03:47:43.672905+0000 2024-09-14T03:59:01.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[106470]: created 2024-09-14T03:46:10.112174+0000 2024-09-14T03:59:01.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[106470]: min_mon_release 18 (reef) 2024-09-14T03:59:01.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[106470]: election_strategy: 1 2024-09-14T03:59:01.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[106470]: 0: [v2:172.21.15.39:3300/0,v1:172.21.15.39:6789/0] mon.a 2024-09-14T03:59:01.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[106470]: 1: [v2:172.21.15.129:3300/0,v1:172.21.15.129:6789/0] mon.b 2024-09-14T03:59:01.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[106470]: 2: [v2:172.21.15.39:3301/0,v1:172.21.15.39:6790/0] mon.c 2024-09-14T03:59:01.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[106470]: fsmap foo:1 {0=foo.smithi039.bbwiwf=up:active} 1 up:standby 2024-09-14T03:59:01.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[106470]: osdmap e343: 8 total, 8 up, 8 in 2024-09-14T03:59:01.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[106470]: mgrmap e34: y(active, since 2m), standbys: x 2024-09-14T03:59:01.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[106470]: Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-09-14T03:59:01.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[106470]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-09-14T03:59:01.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[106470]: application not enabled on pool 'test-librbd-smithi039-85658-9' 2024-09-14T03:59:01.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[106470]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-09-14T03:59:01.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[106470]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:59:01.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[106470]: mgrmap e35: y(active, since 2m), standbys: x 2024-09-14T03:59:01.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[106470]: from='mgr.24944 ' entity='' 2024-09-14T03:59:01.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[40962]: pgmap v229: 137 pgs: 137 active+clean; 1.6 GiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 28 MiB/s rd, 34 MiB/s wr, 659 op/s 2024-09-14T03:59:01.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[40962]: mon.a calling monitor election 2024-09-14T03:59:01.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[40962]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:59:01.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[40962]: monmap epoch 3 2024-09-14T03:59:01.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[40962]: fsid bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T03:59:01.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[40962]: last_changed 2024-09-14T03:47:43.672905+0000 2024-09-14T03:59:01.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[40962]: created 2024-09-14T03:46:10.112174+0000 2024-09-14T03:59:01.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[40962]: min_mon_release 18 (reef) 2024-09-14T03:59:01.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[40962]: election_strategy: 1 2024-09-14T03:59:01.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[40962]: 0: [v2:172.21.15.39:3300/0,v1:172.21.15.39:6789/0] mon.a 2024-09-14T03:59:01.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[40962]: 1: [v2:172.21.15.129:3300/0,v1:172.21.15.129:6789/0] mon.b 2024-09-14T03:59:01.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[40962]: 2: [v2:172.21.15.39:3301/0,v1:172.21.15.39:6790/0] mon.c 2024-09-14T03:59:01.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[40962]: fsmap foo:1 {0=foo.smithi039.bbwiwf=up:active} 1 up:standby 2024-09-14T03:59:01.294 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[40962]: osdmap e343: 8 total, 8 up, 8 in 2024-09-14T03:59:01.294 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[40962]: mgrmap e34: y(active, since 2m), standbys: x 2024-09-14T03:59:01.294 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[40962]: Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-09-14T03:59:01.294 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[40962]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-09-14T03:59:01.294 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[40962]: application not enabled on pool 'test-librbd-smithi039-85658-9' 2024-09-14T03:59:01.294 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[40962]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-09-14T03:59:01.294 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='mgr.y' 2024-09-14T03:59:01.294 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[40962]: mgrmap e35: y(active, since 2m), standbys: x 2024-09-14T03:59:01.294 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:01 smithi039 ceph-mon[40962]: from='mgr.24944 ' entity='' 2024-09-14T03:59:02.049 INFO:tasks.workunit.client.0.smithi039.stdout:write: d 52956524~6348013 2024-09-14T03:59:02.062 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 100107868~7654722 2024-09-14T03:59:02.062 INFO:tasks.workunit.client.0.smithi039.stdout:write: d 174720289~2889974 2024-09-14T03:59:02.062 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 88820144~5576287 2024-09-14T03:59:02.063 INFO:tasks.workunit.client.0.smithi039.stdout:write: d 217527117~3973191 2024-09-14T03:59:02.114 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 38012847~771645 2024-09-14T03:59:02.118 INFO:tasks.workunit.client.0.smithi039.stdout:write: d 30474737~1962879 2024-09-14T03:59:02.143 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 100278916~1134371 2024-09-14T03:59:02.150 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: user3 2024-09-14T03:59:02.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:02 smithi039 ceph-mon[106470]: osdmap e344: 8 total, 8 up, 8 in 2024-09-14T03:59:02.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:02 smithi039 ceph-mon[40962]: osdmap e344: 8 total, 8 up, 8 in 2024-09-14T03:59:02.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:02 smithi129 ceph-mon[33173]: osdmap e344: 8 total, 8 up, 8 in 2024-09-14T03:59:03.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:03 smithi129 ceph-mon[33173]: osdmap e345: 8 total, 8 up, 8 in 2024-09-14T03:59:03.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:03 smithi039 ceph-mon[106470]: osdmap e345: 8 total, 8 up, 8 in 2024-09-14T03:59:03.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:03 smithi039 ceph-mon[40962]: osdmap e345: 8 total, 8 up, 8 in 2024-09-14T03:59:04.086 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 23657531~5914382 2024-09-14T03:59:04.089 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 25136126~2957191 2024-09-14T03:59:04.151 INFO:tasks.workunit.client.0.smithi039.stdout:write: 2 47315063~5914382 2024-09-14T03:59:04.152 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 48793658~2957191 2024-09-14T03:59:04.203 INFO:tasks.workunit.client.0.smithi039.stdout:write: 3 70972595~5914382 2024-09-14T03:59:04.205 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 72451190~2957191 2024-09-14T03:59:04.267 INFO:tasks.workunit.client.0.smithi039.stdout:write: 4 94630127~5914382 2024-09-14T03:59:04.269 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 96108722~2957191 2024-09-14T03:59:04.319 INFO:tasks.workunit.client.0.smithi039.stdout:write: 5 118287659~5914382 2024-09-14T03:59:04.321 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 119766254~2957191 2024-09-14T03:59:04.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:04 smithi129 ceph-mon[33173]: osdmap e346: 8 total, 8 up, 8 in 2024-09-14T03:59:04.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:04 smithi039 ceph-mon[106470]: osdmap e346: 8 total, 8 up, 8 in 2024-09-14T03:59:04.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:04 smithi039 ceph-mon[40962]: osdmap e346: 8 total, 8 up, 8 in 2024-09-14T03:59:05.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:05 smithi129 ceph-mon[33173]: osdmap e347: 8 total, 8 up, 8 in 2024-09-14T03:59:05.271 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 03:59:04 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=infra.usagestats t=2024-09-14T03:59:04.894806845Z level=info msg="Usage stats are ready to report" 2024-09-14T03:59:05.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:05 smithi039 ceph-mon[40962]: osdmap e347: 8 total, 8 up, 8 in 2024-09-14T03:59:05.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:05 smithi039 ceph-mon[106470]: osdmap e347: 8 total, 8 up, 8 in 2024-09-14T03:59:06.040 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:59:06.283 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:59:06.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:06 smithi129 ceph-mon[33173]: Standby manager daemon x restarted 2024-09-14T03:59:06.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:06 smithi129 ceph-mon[33173]: Standby manager daemon x started 2024-09-14T03:59:06.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:06 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.129:0/3217515832' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:59:06.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:06 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.129:0/3217515832' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:59:06.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:06 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.129:0/3217515832' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:59:06.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:06 smithi129 ceph-mon[33173]: from='mgr.? 172.21.15.129:0/3217515832' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:59:06.522 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:59:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[80442]: [14/Sep/2024:03:59:06] ENGINE Bus STARTING 2024-09-14T03:59:06.522 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:59:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[80442]: CherryPy Checker: 2024-09-14T03:59:06.522 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:59:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[80442]: The Application mounted at '' has an empty config. 2024-09-14T03:59:06.522 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:59:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[80442]: 2024-09-14T03:59:06.522 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:59:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[80442]: [14/Sep/2024:03:59:06] ENGINE Serving on http://:::9283 2024-09-14T03:59:06.522 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 03:59:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x[80442]: [14/Sep/2024:03:59:06] ENGINE Bus STARTED 2024-09-14T03:59:06.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:06 smithi039 ceph-mon[106470]: Standby manager daemon x restarted 2024-09-14T03:59:06.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:06 smithi039 ceph-mon[106470]: Standby manager daemon x started 2024-09-14T03:59:06.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:06 smithi039 ceph-mon[106470]: from='mgr.? 172.21.15.129:0/3217515832' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:59:06.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:06 smithi039 ceph-mon[106470]: from='mgr.? 172.21.15.129:0/3217515832' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:59:06.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:06 smithi039 ceph-mon[106470]: from='mgr.? 172.21.15.129:0/3217515832' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:59:06.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:06 smithi039 ceph-mon[106470]: from='mgr.? 172.21.15.129:0/3217515832' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:59:06.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:06 smithi039 ceph-mon[40962]: Standby manager daemon x restarted 2024-09-14T03:59:06.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:06 smithi039 ceph-mon[40962]: Standby manager daemon x started 2024-09-14T03:59:06.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:06 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.129:0/3217515832' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-09-14T03:59:06.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:06 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.129:0/3217515832' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-09-14T03:59:06.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:06 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.129:0/3217515832' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-09-14T03:59:06.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:06 smithi039 ceph-mon[40962]: from='mgr.? 172.21.15.129:0/3217515832' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-09-14T03:59:07.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:07 smithi129 ceph-mon[33173]: mgrmap e36: y(active, since 2m), standbys: x 2024-09-14T03:59:07.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:07 smithi129 ceph-mon[33173]: Active manager daemon y restarted 2024-09-14T03:59:07.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:07 smithi129 ceph-mon[33173]: Activating manager daemon y 2024-09-14T03:59:07.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:07 smithi129 ceph-mon[33173]: osdmap e348: 8 total, 8 up, 8 in 2024-09-14T03:59:07.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:07 smithi129 ceph-mon[33173]: mgrmap e37: y(active, starting, since 0.00896465s), standbys: x 2024-09-14T03:59:07.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:07 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:59:07.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:07 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:59:07.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:07 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:59:07.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:07 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi039.bbwiwf"}]: dispatch 2024-09-14T03:59:07.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:07 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi129.ybjqcw"}]: dispatch 2024-09-14T03:59:07.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:07 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:59:07.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:07 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:59:07.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:07 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:59:07.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:07 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:59:07.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:07 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:59:07.523 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:07 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:59:07.523 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:07 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:59:07.523 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:07 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:59:07.523 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:07 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:59:07.523 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:07 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:59:07.523 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:07 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:59:07.523 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:07 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:59:07.523 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:07 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:59:07.523 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:07 smithi129 ceph-mon[33173]: Manager daemon y is now available 2024-09-14T03:59:07.523 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:07 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:59:07.523 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:07 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:59:07.524 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:07 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:59:07.524 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:07 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:59:07.524 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:07 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-14T03:59:07.541 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T03:59:07.307+0000 7f265beae640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:59:07.541 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T03:59:07.307+0000 7f265beae640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:59:07.541 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T03:59:07.307+0000 7f265beae640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:59:07.541 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T03:59:07.307+0000 7f265beae640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:59:07.541 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T03:59:07.307+0000 7f265beae640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:59:07.541 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T03:59:07.309+0000 7f26576a5640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:59:07.542 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T03:59:07.309+0000 7f26576a5640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:59:07.542 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T03:59:07.309+0000 7f26576a5640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:59:07.542 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T03:59:07.309+0000 7f26576a5640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:59:07.542 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T03:59:07.309+0000 7f26576a5640 -1 client.0 error registering admin socket command: (17) File exists 2024-09-14T03:59:07.542 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[106470]: mgrmap e36: y(active, since 2m), standbys: x 2024-09-14T03:59:07.542 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[106470]: Active manager daemon y restarted 2024-09-14T03:59:07.543 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[106470]: Activating manager daemon y 2024-09-14T03:59:07.543 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[106470]: osdmap e348: 8 total, 8 up, 8 in 2024-09-14T03:59:07.543 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[106470]: mgrmap e37: y(active, starting, since 0.00896465s), standbys: x 2024-09-14T03:59:07.543 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:59:07.543 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:59:07.543 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:59:07.543 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi039.bbwiwf"}]: dispatch 2024-09-14T03:59:07.543 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi129.ybjqcw"}]: dispatch 2024-09-14T03:59:07.543 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:59:07.543 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:59:07.543 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:59:07.543 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:59:07.543 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:59:07.544 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:59:07.544 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:59:07.544 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:59:07.544 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:59:07.544 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:59:07.544 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:59:07.544 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:59:07.544 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:59:07.544 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[106470]: Manager daemon y is now available 2024-09-14T03:59:07.544 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:59:07.544 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:59:07.544 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:59:07.544 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:59:07.544 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-14T03:59:07.545 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[40962]: mgrmap e36: y(active, since 2m), standbys: x 2024-09-14T03:59:07.545 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[40962]: Active manager daemon y restarted 2024-09-14T03:59:07.545 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[40962]: Activating manager daemon y 2024-09-14T03:59:07.545 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[40962]: osdmap e348: 8 total, 8 up, 8 in 2024-09-14T03:59:07.545 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[40962]: mgrmap e37: y(active, starting, since 0.00896465s), standbys: x 2024-09-14T03:59:07.545 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-09-14T03:59:07.545 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-09-14T03:59:07.545 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-09-14T03:59:07.545 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi039.bbwiwf"}]: dispatch 2024-09-14T03:59:07.546 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi129.ybjqcw"}]: dispatch 2024-09-14T03:59:07.546 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-09-14T03:59:07.546 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-09-14T03:59:07.546 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-09-14T03:59:07.546 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-09-14T03:59:07.546 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-09-14T03:59:07.546 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-09-14T03:59:07.546 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-09-14T03:59:07.546 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-09-14T03:59:07.546 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-09-14T03:59:07.546 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-09-14T03:59:07.546 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-09-14T03:59:07.546 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-09-14T03:59:07.547 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-09-14T03:59:07.547 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[40962]: Manager daemon y is now available 2024-09-14T03:59:07.547 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:59:07.547 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:59:07.547 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:59:07.547 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-09-14T03:59:07.547 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-mon[40962]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"} : dispatch 2024-09-14T03:59:07.845 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T03:59:07.737+0000 7f268b1da640 -1 mgr.server handle_report got status from non-daemon mon.a 2024-09-14T03:59:07.845 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:03:59:07] ENGINE Bus STARTING 2024-09-14T03:59:07.845 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: CherryPy Checker: 2024-09-14T03:59:07.845 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: The Application mounted at '' has an empty config. 2024-09-14T03:59:07.845 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T03:59:07.846 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:03:59:07] ENGINE Serving on http://:::9283 2024-09-14T03:59:07.846 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:07 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:03:59:07] ENGINE Bus STARTED 2024-09-14T03:59:08.061 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.Stress2 (22751 ms) 2024-09-14T03:59:08.061 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMigration.StressLive 2024-09-14T03:59:08.102 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 83886080 2024-09-14T03:59:08.115 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 30889443~2056674 2024-09-14T03:59:08.137 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 5104385~2654447 2024-09-14T03:59:08.137 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 7462207~2347858 2024-09-14T03:59:08.155 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 41818760~898103 2024-09-14T03:59:08.155 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 43539930~3302657 2024-09-14T03:59:08.185 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 42849299~3606176 2024-09-14T03:59:08.187 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 19592758~3145843 2024-09-14T03:59:08.209 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 71268155~7254273 2024-09-14T03:59:08.209 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap0 2024-09-14T03:59:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:08 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:59:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:08 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-14T03:59:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:08 smithi129 ceph-mon[33173]: mgrmap e38: y(active, since 1.01676s), standbys: x 2024-09-14T03:59:08.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:08 smithi129 ceph-mon[33173]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:59:08.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:08 smithi129 ceph-mon[33173]: pgmap v3: 137 pgs: 137 active+clean; 1.8 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:59:08.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:08 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:59:08.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-14T03:59:08.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:08 smithi039 ceph-mon[106470]: mgrmap e38: y(active, since 1.01676s), standbys: x 2024-09-14T03:59:08.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:08 smithi039 ceph-mon[106470]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:59:08.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:08 smithi039 ceph-mon[106470]: pgmap v3: 137 pgs: 137 active+clean; 1.8 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:59:08.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:08 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-09-14T03:59:08.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:08 smithi039 ceph-mon[40962]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"} : dispatch 2024-09-14T03:59:08.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:08 smithi039 ceph-mon[40962]: mgrmap e38: y(active, since 1.01676s), standbys: x 2024-09-14T03:59:08.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:08 smithi039 ceph-mon[40962]: from='mon.? -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T03:59:08.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:08 smithi039 ceph-mon[40962]: pgmap v3: 137 pgs: 137 active+clean; 1.8 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:59:09.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:59:08] "GET /metrics HTTP/1.1" 200 36020 "" "Prometheus/2.51.0" 2024-09-14T03:59:09.495 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:09 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T03:59:09.162+0000 7f26394af640 -1 log_channel(cephadm) log [ERR] : [14/Sep/2024:03:59:09] ENGINE Error in HTTPServer.serve 2024-09-14T03:59:09.495 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:09 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: Traceback (most recent call last): 2024-09-14T03:59:09.495 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:09 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:59:09.495 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:09 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: self._connections.run(self.expiration_interval) 2024-09-14T03:59:09.495 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:09 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:59:09.495 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:09 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: self._run(expiration_interval) 2024-09-14T03:59:09.495 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:09 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:59:09.495 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:09 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:59:09.495 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:09 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:59:09.496 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:09 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:59:09.496 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:09 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:59:09.496 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:09 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: s = self.context.wrap_socket( 2024-09-14T03:59:09.496 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:09 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:59:09.496 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:09 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: return self.sslsocket_class._create( 2024-09-14T03:59:09.496 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:09 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:59:09.496 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:09 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: self.do_handshake() 2024-09-14T03:59:09.496 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:09 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:59:09.496 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:09 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: self._sslobj.do_handshake() 2024-09-14T03:59:09.496 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:09 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:59:09.496 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:09 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T03:59:09.751 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 90935345 2024-09-14T03:59:09.764 INFO:tasks.workunit.client.0.smithi039.stdout:write: B 17637309~4112634 2024-09-14T03:59:09.771 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 85646328~5289017 2024-09-14T03:59:09.772 INFO:tasks.workunit.client.0.smithi039.stdout:write: B 65767711~6781291 2024-09-14T03:59:09.778 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 39549060~4155632 2024-09-14T03:59:09.779 INFO:tasks.workunit.client.0.smithi039.stdout:write: B 68662532~4758129 2024-09-14T03:59:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:09 smithi039 ceph-mon[106470]: osdmap e349: 8 total, 8 up, 8 in 2024-09-14T03:59:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:09 smithi039 ceph-mon[106470]: [14/Sep/2024:03:59:08] ENGINE Bus STARTING 2024-09-14T03:59:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:09 smithi039 ceph-mon[106470]: mgrmap e39: y(active, since 2s), standbys: x 2024-09-14T03:59:09.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:09 smithi039 ceph-mon[40962]: osdmap e349: 8 total, 8 up, 8 in 2024-09-14T03:59:09.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:09 smithi039 ceph-mon[40962]: [14/Sep/2024:03:59:08] ENGINE Bus STARTING 2024-09-14T03:59:09.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:09 smithi039 ceph-mon[40962]: mgrmap e39: y(active, since 2s), standbys: x 2024-09-14T03:59:09.818 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 47495015~5136935 2024-09-14T03:59:09.818 INFO:tasks.workunit.client.0.smithi039.stdout:write: B 13539240~7346678 2024-09-14T03:59:09.823 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 74131699~3201601 2024-09-14T03:59:09.823 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap1 2024-09-14T03:59:10.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:09 smithi129 ceph-mon[33173]: osdmap e349: 8 total, 8 up, 8 in 2024-09-14T03:59:10.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:09 smithi129 ceph-mon[33173]: [14/Sep/2024:03:59:08] ENGINE Bus STARTING 2024-09-14T03:59:10.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:09 smithi129 ceph-mon[33173]: mgrmap e39: y(active, since 2s), standbys: x 2024-09-14T03:59:11.019 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:10 smithi039 ceph-mon[106470]: [14/Sep/2024:03:59:09] ENGINE Serving on http://172.21.15.39:8765 2024-09-14T03:59:11.019 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:10 smithi039 ceph-mon[106470]: [14/Sep/2024:03:59:09] ENGINE Serving on https://172.21.15.39:7150 2024-09-14T03:59:11.019 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:10 smithi039 ceph-mon[106470]: [14/Sep/2024:03:59:09] ENGINE Bus STARTED 2024-09-14T03:59:11.020 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:10 smithi039 ceph-mon[106470]: [14/Sep/2024:03:59:09] ENGINE Error in HTTPServer.serve 2024-09-14T03:59:11.020 INFO:journalctl@ceph.mon.a.smithi039.stdout: Traceback (most recent call last): 2024-09-14T03:59:11.020 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:59:11.020 INFO:journalctl@ceph.mon.a.smithi039.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:59:11.020 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:59:11.020 INFO:journalctl@ceph.mon.a.smithi039.stdout: self._run(expiration_interval) 2024-09-14T03:59:11.020 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:59:11.020 INFO:journalctl@ceph.mon.a.smithi039.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:59:11.020 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:59:11.021 INFO:journalctl@ceph.mon.a.smithi039.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:59:11.021 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:59:11.021 INFO:journalctl@ceph.mon.a.smithi039.stdout: s = self.context.wrap_socket( 2024-09-14T03:59:11.021 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:59:11.021 INFO:journalctl@ceph.mon.a.smithi039.stdout: return self.sslsocket_class._create( 2024-09-14T03:59:11.021 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:59:11.021 INFO:journalctl@ceph.mon.a.smithi039.stdout: self.do_handshake() 2024-09-14T03:59:11.021 INFO:journalctl@ceph.mon.a.smithi039.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:59:11.021 INFO:journalctl@ceph.mon.a.smithi039.stdout: self._sslobj.do_handshake() 2024-09-14T03:59:11.021 INFO:journalctl@ceph.mon.a.smithi039.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:59:11.021 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:10 smithi039 ceph-mon[106470]: osdmap e350: 8 total, 8 up, 8 in 2024-09-14T03:59:11.022 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:10 smithi039 ceph-mon[40962]: [14/Sep/2024:03:59:09] ENGINE Serving on http://172.21.15.39:8765 2024-09-14T03:59:11.022 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:10 smithi039 ceph-mon[40962]: [14/Sep/2024:03:59:09] ENGINE Serving on https://172.21.15.39:7150 2024-09-14T03:59:11.022 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:10 smithi039 ceph-mon[40962]: [14/Sep/2024:03:59:09] ENGINE Bus STARTED 2024-09-14T03:59:11.022 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:10 smithi039 ceph-mon[40962]: [14/Sep/2024:03:59:09] ENGINE Error in HTTPServer.serve 2024-09-14T03:59:11.022 INFO:journalctl@ceph.mon.c.smithi039.stdout: Traceback (most recent call last): 2024-09-14T03:59:11.022 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:59:11.022 INFO:journalctl@ceph.mon.c.smithi039.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:59:11.022 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:59:11.022 INFO:journalctl@ceph.mon.c.smithi039.stdout: self._run(expiration_interval) 2024-09-14T03:59:11.022 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:59:11.022 INFO:journalctl@ceph.mon.c.smithi039.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:59:11.022 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:59:11.022 INFO:journalctl@ceph.mon.c.smithi039.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:59:11.023 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:59:11.023 INFO:journalctl@ceph.mon.c.smithi039.stdout: s = self.context.wrap_socket( 2024-09-14T03:59:11.023 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:59:11.023 INFO:journalctl@ceph.mon.c.smithi039.stdout: return self.sslsocket_class._create( 2024-09-14T03:59:11.023 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:59:11.023 INFO:journalctl@ceph.mon.c.smithi039.stdout: self.do_handshake() 2024-09-14T03:59:11.023 INFO:journalctl@ceph.mon.c.smithi039.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:59:11.023 INFO:journalctl@ceph.mon.c.smithi039.stdout: self._sslobj.do_handshake() 2024-09-14T03:59:11.023 INFO:journalctl@ceph.mon.c.smithi039.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:59:11.023 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:10 smithi039 ceph-mon[40962]: osdmap e350: 8 total, 8 up, 8 in 2024-09-14T03:59:11.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:10 smithi129 ceph-mon[33173]: [14/Sep/2024:03:59:09] ENGINE Serving on http://172.21.15.39:8765 2024-09-14T03:59:11.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:10 smithi129 ceph-mon[33173]: [14/Sep/2024:03:59:09] ENGINE Serving on https://172.21.15.39:7150 2024-09-14T03:59:11.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:10 smithi129 ceph-mon[33173]: [14/Sep/2024:03:59:09] ENGINE Bus STARTED 2024-09-14T03:59:11.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:10 smithi129 ceph-mon[33173]: [14/Sep/2024:03:59:09] ENGINE Error in HTTPServer.serve 2024-09-14T03:59:11.024 INFO:journalctl@ceph.mon.b.smithi129.stdout: Traceback (most recent call last): 2024-09-14T03:59:11.024 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib/python3.9/site-packages/cheroot/server.py", line 1823, in serve 2024-09-14T03:59:11.024 INFO:journalctl@ceph.mon.b.smithi129.stdout: self._connections.run(self.expiration_interval) 2024-09-14T03:59:11.024 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 203, in run 2024-09-14T03:59:11.024 INFO:journalctl@ceph.mon.b.smithi129.stdout: self._run(expiration_interval) 2024-09-14T03:59:11.024 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 246, in _run 2024-09-14T03:59:11.025 INFO:journalctl@ceph.mon.b.smithi129.stdout: new_conn = self._from_server_socket(self.server.socket) 2024-09-14T03:59:11.025 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib/python3.9/site-packages/cheroot/connections.py", line 300, in _from_server_socket 2024-09-14T03:59:11.025 INFO:journalctl@ceph.mon.b.smithi129.stdout: s, ssl_env = self.server.ssl_adapter.wrap(s) 2024-09-14T03:59:11.025 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib/python3.9/site-packages/cheroot/ssl/builtin.py", line 277, in wrap 2024-09-14T03:59:11.025 INFO:journalctl@ceph.mon.b.smithi129.stdout: s = self.context.wrap_socket( 2024-09-14T03:59:11.025 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib64/python3.9/ssl.py", line 501, in wrap_socket 2024-09-14T03:59:11.025 INFO:journalctl@ceph.mon.b.smithi129.stdout: return self.sslsocket_class._create( 2024-09-14T03:59:11.025 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib64/python3.9/ssl.py", line 1074, in _create 2024-09-14T03:59:11.025 INFO:journalctl@ceph.mon.b.smithi129.stdout: self.do_handshake() 2024-09-14T03:59:11.025 INFO:journalctl@ceph.mon.b.smithi129.stdout: File "/lib64/python3.9/ssl.py", line 1343, in do_handshake 2024-09-14T03:59:11.025 INFO:journalctl@ceph.mon.b.smithi129.stdout: self._sslobj.do_handshake() 2024-09-14T03:59:11.025 INFO:journalctl@ceph.mon.b.smithi129.stdout: ssl.SSLZeroReturnError: TLS/SSL connection has been closed (EOF) (_ssl.c:1133) 2024-09-14T03:59:11.026 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:10 smithi129 ceph-mon[33173]: osdmap e350: 8 total, 8 up, 8 in 2024-09-14T03:59:11.767 INFO:tasks.workunit.client.0.smithi039.stdout:write: C 6034867~4960559 2024-09-14T03:59:11.779 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 82689165~5577858 2024-09-14T03:59:11.779 INFO:tasks.workunit.client.0.smithi039.stdout:write: C 61820587~8301700 2024-09-14T03:59:11.788 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 5689014~1314685 2024-09-14T03:59:11.804 INFO:tasks.workunit.client.0.smithi039.stdout:write: C 22532886~5349782 2024-09-14T03:59:11.805 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 1620381~4810291 2024-09-14T03:59:11.812 INFO:tasks.workunit.client.0.smithi039.stdout:write: C 1168561~4134634 2024-09-14T03:59:11.817 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 81889887~5935117 2024-09-14T03:59:11.818 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap2 2024-09-14T03:59:12.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:11 smithi129 ceph-mon[33173]: pgmap v7: 137 pgs: 137 active+clean; 1.8 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:59:12.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:11 smithi129 ceph-mon[33173]: osdmap e351: 8 total, 8 up, 8 in 2024-09-14T03:59:12.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:11 smithi129 ceph-mon[33173]: mgrmap e40: y(active, since 4s), standbys: x 2024-09-14T03:59:12.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:11 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:12.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:11 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:12.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:11 smithi039 ceph-mon[106470]: pgmap v7: 137 pgs: 137 active+clean; 1.8 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:59:12.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:11 smithi039 ceph-mon[106470]: osdmap e351: 8 total, 8 up, 8 in 2024-09-14T03:59:12.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:11 smithi039 ceph-mon[106470]: mgrmap e40: y(active, since 4s), standbys: x 2024-09-14T03:59:12.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:11 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:12.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:11 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:12.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:11 smithi039 ceph-mon[40962]: pgmap v7: 137 pgs: 137 active+clean; 1.8 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:59:12.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:11 smithi039 ceph-mon[40962]: osdmap e351: 8 total, 8 up, 8 in 2024-09-14T03:59:12.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:11 smithi039 ceph-mon[40962]: mgrmap e40: y(active, since 4s), standbys: x 2024-09-14T03:59:12.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:11 smithi039 ceph-mon[40962]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:12.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:11 smithi039 ceph-mon[40962]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:13.000 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:12 smithi039 ceph-mon[106470]: osdmap e352: 8 total, 8 up, 8 in 2024-09-14T03:59:13.000 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:12 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:13.000 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:12 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:13.000 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:12 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:13.000 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:12 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:13.001 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:12 smithi039 ceph-mon[40962]: osdmap e352: 8 total, 8 up, 8 in 2024-09-14T03:59:13.001 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:12 smithi039 ceph-mon[40962]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:13.001 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:12 smithi039 ceph-mon[40962]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:13.001 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:12 smithi039 ceph-mon[40962]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:13.001 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:12 smithi039 ceph-mon[40962]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:13.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:12 smithi129 ceph-mon[33173]: osdmap e352: 8 total, 8 up, 8 in 2024-09-14T03:59:13.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:12 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:13.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:12 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:13.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:12 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:13.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:12 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:13.797 INFO:tasks.workunit.client.0.smithi039.stdout:snap_protect: snap2 2024-09-14T03:59:13.811 INFO:tasks.workunit.client.0.smithi039.stdout:clone image185 -> image188 2024-09-14T03:59:13.915 INFO:tasks.workunit.client.0.smithi039.stdout:write: D 72835375~7095705 2024-09-14T03:59:13.979 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 61480382~4683363 2024-09-14T03:59:13.980 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:13 smithi039 ceph-mon[106470]: pgmap v10: 137 pgs: 137 active+clean; 1.8 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:59:13.980 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:13 smithi039 ceph-mon[106470]: osdmap e353: 8 total, 8 up, 8 in 2024-09-14T03:59:13.981 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:13 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:13.981 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:13 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:13.981 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:13 smithi039 ceph-mon[40962]: pgmap v10: 137 pgs: 137 active+clean; 1.8 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:59:13.981 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:13 smithi039 ceph-mon[40962]: osdmap e353: 8 total, 8 up, 8 in 2024-09-14T03:59:13.981 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:13 smithi039 ceph-mon[40962]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:13.981 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:13 smithi039 ceph-mon[40962]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:14.002 INFO:tasks.workunit.client.0.smithi039.stdout:write: D 46687659~1288564 2024-09-14T03:59:14.020 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 880877~4012380 2024-09-14T03:59:14.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:13 smithi129 ceph-mon[33173]: pgmap v10: 137 pgs: 137 active+clean; 1.8 GiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2024-09-14T03:59:14.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:13 smithi129 ceph-mon[33173]: osdmap e353: 8 total, 8 up, 8 in 2024-09-14T03:59:14.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:13 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:14.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:13 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:14.044 INFO:tasks.workunit.client.0.smithi039.stdout:write: D 74322845~3903636 2024-09-14T03:59:14.070 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 18098841~4309090 2024-09-14T03:59:14.119 INFO:tasks.workunit.client.0.smithi039.stdout:write: D 3207178~1100573 2024-09-14T03:59:14.126 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 17040625~5030357 2024-09-14T03:59:14.129 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: snap3 2024-09-14T03:59:14.945 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:14 smithi129 ceph-mon[33173]: osdmap e354: 8 total, 8 up, 8 in 2024-09-14T03:59:15.079 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:14 smithi039 ceph-mon[106470]: osdmap e354: 8 total, 8 up, 8 in 2024-09-14T03:59:15.080 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:14 smithi039 ceph-mon[40962]: osdmap e354: 8 total, 8 up, 8 in 2024-09-14T03:59:15.826 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 106776580 2024-09-14T03:59:15.841 INFO:tasks.workunit.client.0.smithi039.stdout:migration_prepare 2024-09-14T03:59:16.159 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:15 smithi129 ceph-mon[33173]: pgmap v13: 137 pgs: 137 active+clean; 1.9 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s rd, 27 MiB/s wr, 236 op/s 2024-09-14T03:59:16.160 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:15 smithi129 ceph-mon[33173]: osdmap e355: 8 total, 8 up, 8 in 2024-09-14T03:59:16.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:15 smithi039 ceph-mon[106470]: pgmap v13: 137 pgs: 137 active+clean; 1.9 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s rd, 27 MiB/s wr, 236 op/s 2024-09-14T03:59:16.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:15 smithi039 ceph-mon[106470]: osdmap e355: 8 total, 8 up, 8 in 2024-09-14T03:59:16.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:15 smithi039 ceph-mon[40962]: pgmap v13: 137 pgs: 137 active+clean; 1.9 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s rd, 27 MiB/s wr, 236 op/s 2024-09-14T03:59:16.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:15 smithi039 ceph-mon[40962]: osdmap e355: 8 total, 8 up, 8 in 2024-09-14T03:59:17.215 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:16 smithi039 ceph-mon[106470]: osdmap e356: 8 total, 8 up, 8 in 2024-09-14T03:59:17.215 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:16 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:17.215 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:16 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:17.215 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:16 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:17.215 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:16 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:17.216 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:16 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:59:17.216 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:16 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:59:17.216 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:16 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:17.216 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:16 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:17.216 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:16 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:17.216 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:16 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:17.216 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:16 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:17.216 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:16 smithi039 ceph-mon[40962]: osdmap e356: 8 total, 8 up, 8 in 2024-09-14T03:59:17.216 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:16 smithi039 ceph-mon[40962]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:17.216 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:16 smithi039 ceph-mon[40962]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:17.217 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:16 smithi039 ceph-mon[40962]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:17.217 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:16 smithi039 ceph-mon[40962]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:17.217 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:16 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:59:17.217 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:16 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:59:17.217 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:16 smithi039 ceph-mon[40962]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:17.217 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:16 smithi039 ceph-mon[40962]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:17.217 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:16 smithi039 ceph-mon[40962]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:17.217 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:16 smithi039 ceph-mon[40962]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:17.217 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:16 smithi039 ceph-mon[40962]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:17.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:16 smithi129 ceph-mon[33173]: osdmap e356: 8 total, 8 up, 8 in 2024-09-14T03:59:17.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:16 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:17.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:16 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:17.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:16 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:17.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:16 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:17.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:16 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:59:17.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:16 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:59:17.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:16 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:17.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:16 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:17.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:16 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:17.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:16 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:17.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:16 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:17.316 INFO:tasks.workunit.client.0.smithi039.stdout:migration_execute 2024-09-14T03:59:17.316 INFO:tasks.workunit.client.0.smithi039.stdout:write: a 96821500~364569 2024-09-14T03:59:17.328 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 31717058~1735115 2024-09-14T03:59:17.356 INFO:tasks.workunit.client.0.smithi039.stdout:write: a 77180705~4093768 2024-09-14T03:59:17.404 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 92927554~1584573 2024-09-14T03:59:17.410 INFO:tasks.workunit.client.0.smithi039.stdout:write: a 18546884~6330234 2024-09-14T03:59:17.472 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 36536547~4859621 2024-09-14T03:59:17.486 INFO:tasks.workunit.client.0.smithi039.stdout:write: a 51052569~1326714 2024-09-14T03:59:17.493 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 64937572~6952790 2024-09-14T03:59:17.529 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: user0 2024-09-14T03:59:18.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:17 smithi129 ceph-mon[33173]: Updating smithi039:/etc/ceph/ceph.conf 2024-09-14T03:59:18.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:17 smithi129 ceph-mon[33173]: Updating smithi129:/etc/ceph/ceph.conf 2024-09-14T03:59:18.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:17 smithi129 ceph-mon[33173]: Updating smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:59:18.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:17 smithi129 ceph-mon[33173]: Updating smithi039:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:59:18.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:17 smithi129 ceph-mon[33173]: Updating smithi129:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:59:18.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:17 smithi129 ceph-mon[33173]: Updating smithi039:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:59:18.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:17 smithi129 ceph-mon[33173]: Updating smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:59:18.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:17 smithi129 ceph-mon[33173]: Updating smithi039:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:59:18.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:17 smithi129 ceph-mon[33173]: pgmap v16: 137 pgs: 137 active+clean; 1.9 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s rd, 27 MiB/s wr, 236 op/s 2024-09-14T03:59:18.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:17 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:59:18.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:17 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:59:18.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:17 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-14T03:59:18.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:17 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-14T03:59:18.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:17 smithi129 ceph-mon[33173]: Upgrade: It appears safe to stop mon.c 2024-09-14T03:59:18.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:17 smithi129 ceph-mon[33173]: osdmap e357: 8 total, 8 up, 8 in 2024-09-14T03:59:18.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:17 smithi039 ceph-mon[106470]: Updating smithi039:/etc/ceph/ceph.conf 2024-09-14T03:59:18.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:17 smithi039 ceph-mon[106470]: Updating smithi129:/etc/ceph/ceph.conf 2024-09-14T03:59:18.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:17 smithi039 ceph-mon[106470]: Updating smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:59:18.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:17 smithi039 ceph-mon[106470]: Updating smithi039:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:59:18.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:17 smithi039 ceph-mon[106470]: Updating smithi129:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:59:18.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:17 smithi039 ceph-mon[106470]: Updating smithi039:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:59:18.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:17 smithi039 ceph-mon[106470]: Updating smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:59:18.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:17 smithi039 ceph-mon[106470]: Updating smithi039:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:59:18.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:17 smithi039 ceph-mon[106470]: pgmap v16: 137 pgs: 137 active+clean; 1.9 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s rd, 27 MiB/s wr, 236 op/s 2024-09-14T03:59:18.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:17 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:59:18.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:17 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:59:18.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:17 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-14T03:59:18.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:17 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-14T03:59:18.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:17 smithi039 ceph-mon[106470]: Upgrade: It appears safe to stop mon.c 2024-09-14T03:59:18.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:17 smithi039 ceph-mon[106470]: osdmap e357: 8 total, 8 up, 8 in 2024-09-14T03:59:18.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:17 smithi039 ceph-mon[40962]: Updating smithi039:/etc/ceph/ceph.conf 2024-09-14T03:59:18.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:17 smithi039 ceph-mon[40962]: Updating smithi129:/etc/ceph/ceph.conf 2024-09-14T03:59:18.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:17 smithi039 ceph-mon[40962]: Updating smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:59:18.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:17 smithi039 ceph-mon[40962]: Updating smithi039:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.conf 2024-09-14T03:59:18.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:17 smithi039 ceph-mon[40962]: Updating smithi129:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:59:18.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:17 smithi039 ceph-mon[40962]: Updating smithi039:/etc/ceph/ceph.client.admin.keyring 2024-09-14T03:59:18.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:17 smithi039 ceph-mon[40962]: Updating smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:59:18.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:17 smithi039 ceph-mon[40962]: Updating smithi039:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/config/ceph.client.admin.keyring 2024-09-14T03:59:18.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:17 smithi039 ceph-mon[40962]: pgmap v16: 137 pgs: 137 active+clean; 1.9 GiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 42 MiB/s rd, 27 MiB/s wr, 236 op/s 2024-09-14T03:59:18.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:17 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:59:18.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:17 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:59:18.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:17 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-14T03:59:18.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:17 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-09-14T03:59:18.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:17 smithi039 ceph-mon[40962]: Upgrade: It appears safe to stop mon.c 2024-09-14T03:59:18.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:17 smithi039 ceph-mon[40962]: osdmap e357: 8 total, 8 up, 8 in 2024-09-14T03:59:18.925 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:59:18] "GET /metrics HTTP/1.1" 200 36020 "" "Prometheus/2.51.0" 2024-09-14T03:59:18.948 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 134076341 2024-09-14T03:59:18.961 INFO:tasks.workunit.client.0.smithi039.stdout:write: b 71481662~415712 2024-09-14T03:59:18.967 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 17008010~3899509 2024-09-14T03:59:18.967 INFO:tasks.workunit.client.0.smithi039.stdout:write: b 81145684~6702470 2024-09-14T03:59:18.971 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 76247633~341659 2024-09-14T03:59:18.971 INFO:tasks.workunit.client.0.smithi039.stdout:write: b 125167537~876091 2024-09-14T03:59:18.972 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 57140897~6633396 2024-09-14T03:59:18.973 INFO:tasks.workunit.client.0.smithi039.stdout:write: b 107661262~3405423 2024-09-14T03:59:18.975 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 31437459~5103017 2024-09-14T03:59:18.975 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: user1 2024-09-14T03:59:19.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:18 smithi129 ceph-mon[33173]: osdmap e358: 8 total, 8 up, 8 in 2024-09-14T03:59:19.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:18 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:19.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:18 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:59:19.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:18 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:59:19.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:18 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:59:19.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:18 smithi039 ceph-mon[106470]: osdmap e358: 8 total, 8 up, 8 in 2024-09-14T03:59:19.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:18 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:19.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:18 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:59:19.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:18 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:59:19.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:18 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:59:19.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:18 smithi039 ceph-mon[40962]: osdmap e358: 8 total, 8 up, 8 in 2024-09-14T03:59:19.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:18 smithi039 ceph-mon[40962]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:19.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:18 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T03:59:19.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:18 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T03:59:19.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:18 smithi039 ceph-mon[40962]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:59:20.032 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:19 smithi039 ceph-mon[106470]: Upgrade: Updating mon.c 2024-09-14T03:59:20.033 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:19 smithi039 ceph-mon[106470]: Deploying daemon mon.c on smithi039 2024-09-14T03:59:20.033 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:19 smithi039 ceph-mon[106470]: pgmap v19: 137 pgs: 137 active+clean; 2.1 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 49 MiB/s rd, 24 MiB/s wr, 757 op/s 2024-09-14T03:59:20.033 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:19 smithi039 ceph-mon[106470]: osdmap e359: 8 total, 8 up, 8 in 2024-09-14T03:59:20.033 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:19 smithi039 ceph-mon[40962]: Upgrade: Updating mon.c 2024-09-14T03:59:20.033 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:19 smithi039 ceph-mon[40962]: Deploying daemon mon.c on smithi039 2024-09-14T03:59:20.033 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:19 smithi039 ceph-mon[40962]: pgmap v19: 137 pgs: 137 active+clean; 2.1 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 49 MiB/s rd, 24 MiB/s wr, 757 op/s 2024-09-14T03:59:20.033 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:19 smithi039 ceph-mon[40962]: osdmap e359: 8 total, 8 up, 8 in 2024-09-14T03:59:20.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:19 smithi129 ceph-mon[33173]: Upgrade: Updating mon.c 2024-09-14T03:59:20.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:19 smithi129 ceph-mon[33173]: Deploying daemon mon.c on smithi039 2024-09-14T03:59:20.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:19 smithi129 ceph-mon[33173]: pgmap v19: 137 pgs: 137 active+clean; 2.1 GiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 49 MiB/s rd, 24 MiB/s wr, 757 op/s 2024-09-14T03:59:20.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:19 smithi129 ceph-mon[33173]: osdmap e359: 8 total, 8 up, 8 in 2024-09-14T03:59:20.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:20 smithi039 systemd[1]: Stopping Ceph mon.c for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:59:20.655 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:20 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:03:59:20] ENGINE Bus STOPPING 2024-09-14T03:59:20.655 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:20 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-c[40939]: 2024-09-14T03:59:20.304+0000 7fc2c2203640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T03:59:20.655 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:20 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-c[40939]: 2024-09-14T03:59:20.304+0000 7fc2c2203640 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-09-14T03:59:20.655 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:20 smithi039 podman[111034]: 2024-09-14 03:59:20.407925469 +0000 UTC m=+0.227339775 container died 31d440e49f9378955f5058cd14dbd47dcec3e9c74c6bec6a19da4695add4077e (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-c, ceph=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, RELEASE=reef-e4e227b) 2024-09-14T03:59:21.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:20 smithi039 podman[111034]: 2024-09-14 03:59:20.878998421 +0000 UTC m=+0.698412728 container cleanup 31d440e49f9378955f5058cd14dbd47dcec3e9c74c6bec6a19da4695add4077e (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-c, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=reef-e4e227b, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-14T03:59:21.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:20 smithi039 bash[111034]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-c 2024-09-14T03:59:21.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:20 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:03:59:20] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T03:59:21.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:20 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:03:59:20] ENGINE Bus STOPPED 2024-09-14T03:59:21.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:20 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:03:59:20] ENGINE Bus STARTING 2024-09-14T03:59:21.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:20 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:03:59:20] ENGINE Serving on http://:::9283 2024-09-14T03:59:21.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:20 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:03:59:20] ENGINE Bus STARTED 2024-09-14T03:59:21.530 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:21 smithi039 podman[111052]: 2024-09-14 03:59:21.155653171 +0000 UTC m=+0.744065080 container remove 31d440e49f9378955f5058cd14dbd47dcec3e9c74c6bec6a19da4695add4077e (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-c, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=reef-e4e227b) 2024-09-14T03:59:21.530 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:21 smithi039 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mon.c.service: Deactivated successfully. 2024-09-14T03:59:21.530 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:21 smithi039 systemd[1]: Stopped Ceph mon.c for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:59:21.530 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:21 smithi039 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mon.c.service: Consumed 9.832s CPU time. 2024-09-14T03:59:21.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:21 smithi039 systemd[1]: Starting Ceph mon.c for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T03:59:21.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:21 smithi039 podman[111197]: 2024-09-14 03:59:21.644670077 +0000 UTC m=+0.016600381 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T03:59:21.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:21 smithi039 podman[111197]: 2024-09-14 03:59:21.765048416 +0000 UTC m=+0.136978708 container create 4b8f0f3864b1af7e8d3d02cf209fc2683f759bae1d66de523deb4db0dd1d4ba0 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-c, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231) 2024-09-14T03:59:22.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 podman[111197]: 2024-09-14 03:59:22.026235077 +0000 UTC m=+0.398165367 container init 4b8f0f3864b1af7e8d3d02cf209fc2683f759bae1d66de523deb4db0dd1d4ba0 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-c, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=main-a278231, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD) 2024-09-14T03:59:22.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 podman[111197]: 2024-09-14 03:59:22.031395391 +0000 UTC m=+0.403325683 container start 4b8f0f3864b1af7e8d3d02cf209fc2683f759bae1d66de523deb4db0dd1d4ba0 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-c, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=main-a278231, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-14T03:59:22.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: set uid:gid to 167:167 (ceph:ceph) 2024-09-14T03:59:22.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev), process ceph-mon, pid 7 2024-09-14T03:59:22.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: pidfile_write: ignore empty --pid-file 2024-09-14T03:59:22.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: load: jerasure load: lrc 2024-09-14T03:59:22.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: RocksDB version: 7.9.2 2024-09-14T03:59:22.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Git sha 0 2024-09-14T03:59:22.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Compile date 2024-09-12 20:09:22 2024-09-14T03:59:22.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: DB SUMMARY 2024-09-14T03:59:22.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: DB Session ID: WCYOSYEJO3I00KUIR3QO 2024-09-14T03:59:22.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: CURRENT file: CURRENT 2024-09-14T03:59:22.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: IDENTITY file: IDENTITY 2024-09-14T03:59:22.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: MANIFEST file: MANIFEST-000010 size: 1413 Bytes 2024-09-14T03:59:22.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2024-09-14T03:59:22.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 13826615 ; 2024-09-14T03:59:22.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.error_if_exists: 0 2024-09-14T03:59:22.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.create_if_missing: 0 2024-09-14T03:59:22.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.paranoid_checks: 1 2024-09-14T03:59:22.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-14T03:59:22.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-14T03:59:22.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-14T03:59:22.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.env: 0x556015fc5120 2024-09-14T03:59:22.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.fs: PosixFileSystem 2024-09-14T03:59:22.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.info_log: 0x556016937700 2024-09-14T03:59:22.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.max_file_opening_threads: 16 2024-09-14T03:59:22.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.statistics: (nil) 2024-09-14T03:59:22.294 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.use_fsync: 0 2024-09-14T03:59:22.294 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.max_log_file_size: 0 2024-09-14T03:59:22.294 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-14T03:59:22.294 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-14T03:59:22.294 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.keep_log_file_num: 1000 2024-09-14T03:59:22.294 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.recycle_log_file_num: 0 2024-09-14T03:59:22.294 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.allow_fallocate: 1 2024-09-14T03:59:22.294 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.allow_mmap_reads: 0 2024-09-14T03:59:22.294 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.allow_mmap_writes: 0 2024-09-14T03:59:22.294 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.use_direct_reads: 0 2024-09-14T03:59:22.294 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-14T03:59:22.294 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.create_missing_column_families: 0 2024-09-14T03:59:22.295 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.db_log_dir: 2024-09-14T03:59:22.295 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.wal_dir: 2024-09-14T03:59:22.295 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-14T03:59:22.295 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-14T03:59:22.295 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-14T03:59:22.295 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-14T03:59:22.295 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-14T03:59:22.295 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-14T03:59:22.295 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.advise_random_on_open: 1 2024-09-14T03:59:22.295 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.db_write_buffer_size: 0 2024-09-14T03:59:22.295 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.write_buffer_manager: 0x55601693a1e0 2024-09-14T03:59:22.295 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-14T03:59:22.295 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-14T03:59:22.296 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-14T03:59:22.296 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.rate_limiter: (nil) 2024-09-14T03:59:22.296 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-14T03:59:22.296 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.wal_recovery_mode: 2 2024-09-14T03:59:22.296 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.enable_thread_tracking: 0 2024-09-14T03:59:22.296 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.enable_pipelined_write: 0 2024-09-14T03:59:22.296 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.unordered_write: 0 2024-09-14T03:59:22.296 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-14T03:59:22.296 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-14T03:59:22.296 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-14T03:59:22.296 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-14T03:59:22.296 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.row_cache: None 2024-09-14T03:59:22.297 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.wal_filter: None 2024-09-14T03:59:22.297 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-14T03:59:22.297 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.allow_ingest_behind: 0 2024-09-14T03:59:22.297 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.two_write_queues: 0 2024-09-14T03:59:22.297 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.manual_wal_flush: 0 2024-09-14T03:59:22.297 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.wal_compression: 0 2024-09-14T03:59:22.297 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.atomic_flush: 0 2024-09-14T03:59:22.297 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-14T03:59:22.297 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-14T03:59:22.297 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-14T03:59:22.297 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.log_readahead_size: 0 2024-09-14T03:59:22.297 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-14T03:59:22.297 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.best_efforts_recovery: 0 2024-09-14T03:59:22.297 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-14T03:59:22.298 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-14T03:59:22.298 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.allow_data_in_errors: 0 2024-09-14T03:59:22.298 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.db_host_id: __hostname__ 2024-09-14T03:59:22.298 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-14T03:59:22.298 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.max_background_jobs: 2 2024-09-14T03:59:22.298 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.max_background_compactions: -1 2024-09-14T03:59:22.298 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.max_subcompactions: 1 2024-09-14T03:59:22.298 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-14T03:59:22.298 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-14T03:59:22.298 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-14T03:59:22.298 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.max_total_wal_size: 0 2024-09-14T03:59:22.298 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-14T03:59:22.298 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-14T03:59:22.298 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-14T03:59:22.299 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-14T03:59:22.299 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.max_open_files: -1 2024-09-14T03:59:22.299 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.bytes_per_sync: 0 2024-09-14T03:59:22.299 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-14T03:59:22.299 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-14T03:59:22.299 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.compaction_readahead_size: 0 2024-09-14T03:59:22.299 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.max_background_flushes: -1 2024-09-14T03:59:22.299 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Compression algorithms supported: 2024-09-14T03:59:22.299 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: kZSTD supported: 0 2024-09-14T03:59:22.299 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: kXpressCompression supported: 0 2024-09-14T03:59:22.299 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: kBZip2Compression supported: 0 2024-09-14T03:59:22.299 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-14T03:59:22.299 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: kLZ4Compression supported: 1 2024-09-14T03:59:22.299 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: kZlibCompression supported: 1 2024-09-14T03:59:22.300 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: kLZ4HCCompression supported: 1 2024-09-14T03:59:22.300 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: kSnappyCompression supported: 1 2024-09-14T03:59:22.300 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-14T03:59:22.300 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-14T03:59:22.300 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000010 2024-09-14T03:59:22.300 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-14T03:59:22.300 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-14T03:59:22.300 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.merge_operator: 2024-09-14T03:59:22.300 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.compaction_filter: None 2024-09-14T03:59:22.300 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.compaction_filter_factory: None 2024-09-14T03:59:22.300 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.sst_partitioner_factory: None 2024-09-14T03:59:22.300 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-14T03:59:22.300 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-14T03:59:22.300 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x556016937320) 2024-09-14T03:59:22.301 INFO:journalctl@ceph.mon.c.smithi039.stdout: cache_index_and_filter_blocks: 1 2024-09-14T03:59:22.301 INFO:journalctl@ceph.mon.c.smithi039.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-14T03:59:22.301 INFO:journalctl@ceph.mon.c.smithi039.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-14T03:59:22.301 INFO:journalctl@ceph.mon.c.smithi039.stdout: pin_top_level_index_and_filter: 1 2024-09-14T03:59:22.301 INFO:journalctl@ceph.mon.c.smithi039.stdout: index_type: 0 2024-09-14T03:59:22.301 INFO:journalctl@ceph.mon.c.smithi039.stdout: data_block_index_type: 0 2024-09-14T03:59:22.301 INFO:journalctl@ceph.mon.c.smithi039.stdout: index_shortening: 1 2024-09-14T03:59:22.301 INFO:journalctl@ceph.mon.c.smithi039.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-14T03:59:22.301 INFO:journalctl@ceph.mon.c.smithi039.stdout: checksum: 4 2024-09-14T03:59:22.301 INFO:journalctl@ceph.mon.c.smithi039.stdout: no_block_cache: 0 2024-09-14T03:59:22.301 INFO:journalctl@ceph.mon.c.smithi039.stdout: block_cache: 0x55601692b610 2024-09-14T03:59:22.301 INFO:journalctl@ceph.mon.c.smithi039.stdout: block_cache_name: BinnedLRUCache 2024-09-14T03:59:22.301 INFO:journalctl@ceph.mon.c.smithi039.stdout: block_cache_options: 2024-09-14T03:59:22.301 INFO:journalctl@ceph.mon.c.smithi039.stdout: capacity : 536870912 2024-09-14T03:59:22.302 INFO:journalctl@ceph.mon.c.smithi039.stdout: num_shard_bits : 4 2024-09-14T03:59:22.302 INFO:journalctl@ceph.mon.c.smithi039.stdout: strict_capacity_limit : 0 2024-09-14T03:59:22.302 INFO:journalctl@ceph.mon.c.smithi039.stdout: high_pri_pool_ratio: 0.000 2024-09-14T03:59:22.302 INFO:journalctl@ceph.mon.c.smithi039.stdout: block_cache_compressed: (nil) 2024-09-14T03:59:22.302 INFO:journalctl@ceph.mon.c.smithi039.stdout: persistent_cache: (nil) 2024-09-14T03:59:22.302 INFO:journalctl@ceph.mon.c.smithi039.stdout: block_size: 4096 2024-09-14T03:59:22.302 INFO:journalctl@ceph.mon.c.smithi039.stdout: block_size_deviation: 10 2024-09-14T03:59:22.302 INFO:journalctl@ceph.mon.c.smithi039.stdout: block_restart_interval: 16 2024-09-14T03:59:22.302 INFO:journalctl@ceph.mon.c.smithi039.stdout: index_block_restart_interval: 1 2024-09-14T03:59:22.302 INFO:journalctl@ceph.mon.c.smithi039.stdout: metadata_block_size: 4096 2024-09-14T03:59:22.302 INFO:journalctl@ceph.mon.c.smithi039.stdout: partition_filters: 0 2024-09-14T03:59:22.302 INFO:journalctl@ceph.mon.c.smithi039.stdout: use_delta_encoding: 1 2024-09-14T03:59:22.302 INFO:journalctl@ceph.mon.c.smithi039.stdout: filter_policy: bloomfilter 2024-09-14T03:59:22.303 INFO:journalctl@ceph.mon.c.smithi039.stdout: whole_key_filtering: 1 2024-09-14T03:59:22.303 INFO:journalctl@ceph.mon.c.smithi039.stdout: verify_compression: 0 2024-09-14T03:59:22.303 INFO:journalctl@ceph.mon.c.smithi039.stdout: read_amp_bytes_per_bit: 0 2024-09-14T03:59:22.303 INFO:journalctl@ceph.mon.c.smithi039.stdout: format_version: 5 2024-09-14T03:59:22.303 INFO:journalctl@ceph.mon.c.smithi039.stdout: enable_index_compression: 1 2024-09-14T03:59:22.303 INFO:journalctl@ceph.mon.c.smithi039.stdout: block_align: 0 2024-09-14T03:59:22.303 INFO:journalctl@ceph.mon.c.smithi039.stdout: max_auto_readahead_size: 262144 2024-09-14T03:59:22.303 INFO:journalctl@ceph.mon.c.smithi039.stdout: prepopulate_block_cache: 0 2024-09-14T03:59:22.303 INFO:journalctl@ceph.mon.c.smithi039.stdout: initial_auto_readahead_size: 8192 2024-09-14T03:59:22.303 INFO:journalctl@ceph.mon.c.smithi039.stdout: num_file_reads_for_auto_readahead: 2 2024-09-14T03:59:22.303 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.write_buffer_size: 33554432 2024-09-14T03:59:22.303 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.max_write_buffer_number: 2 2024-09-14T03:59:22.303 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.compression: NoCompression 2024-09-14T03:59:22.304 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.bottommost_compression: Disabled 2024-09-14T03:59:22.304 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.prefix_extractor: nullptr 2024-09-14T03:59:22.304 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-14T03:59:22.304 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.num_levels: 7 2024-09-14T03:59:22.304 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-14T03:59:22.304 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-14T03:59:22.304 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-14T03:59:22.304 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-14T03:59:22.304 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-14T03:59:22.304 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-14T03:59:22.304 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-14T03:59:22.304 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:59:22.304 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-14T03:59:22.304 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-14T03:59:22.305 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:59:22.305 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:59:22.305 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-14T03:59:22.305 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.compression_opts.level: 32767 2024-09-14T03:59:22.305 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.compression_opts.strategy: 0 2024-09-14T03:59:22.305 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-14T03:59:22.305 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-14T03:59:22.305 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-14T03:59:22.305 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-14T03:59:22.305 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.compression_opts.enabled: false 2024-09-14T03:59:22.305 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-14T03:59:22.305 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-14T03:59:22.305 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-14T03:59:22.305 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-14T03:59:22.306 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.target_file_size_base: 67108864 2024-09-14T03:59:22.306 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-14T03:59:22.306 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-14T03:59:22.306 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-14T03:59:22.306 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-14T03:59:22.306 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-14T03:59:22.307 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-14T03:59:22.307 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-14T03:59:22.307 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-14T03:59:22.307 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-14T03:59:22.307 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-14T03:59:22.307 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-14T03:59:22.307 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-14T03:59:22.307 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-14T03:59:22.307 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-14T03:59:22.308 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.arena_block_size: 1048576 2024-09-14T03:59:22.308 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-14T03:59:22.308 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-14T03:59:22.308 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.disable_auto_compactions: 0 2024-09-14T03:59:22.308 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-14T03:59:22.308 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-14T03:59:22.308 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-14T03:59:22.308 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-14T03:59:22.308 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-14T03:59:22.308 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-14T03:59:22.308 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-14T03:59:22.308 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-14T03:59:22.308 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-14T03:59:22.308 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-14T03:59:22.309 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-14T03:59:22.309 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.inplace_update_support: 0 2024-09-14T03:59:22.309 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-14T03:59:22.309 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-14T03:59:22.309 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-14T03:59:22.309 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-14T03:59:22.309 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.bloom_locality: 0 2024-09-14T03:59:22.309 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.max_successive_merges: 0 2024-09-14T03:59:22.309 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-14T03:59:22.309 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.paranoid_file_checks: 0 2024-09-14T03:59:22.309 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.force_consistency_checks: 1 2024-09-14T03:59:22.309 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.report_bg_io_stats: 0 2024-09-14T03:59:22.309 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.ttl: 2592000 2024-09-14T03:59:22.309 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-14T03:59:22.310 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-14T03:59:22.310 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-14T03:59:22.310 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.enable_blob_files: false 2024-09-14T03:59:22.310 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.min_blob_size: 0 2024-09-14T03:59:22.310 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.blob_file_size: 268435456 2024-09-14T03:59:22.310 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-14T03:59:22.310 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-14T03:59:22.310 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-14T03:59:22.310 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-14T03:59:22.310 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-14T03:59:22.310 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.blob_file_starting_level: 0 2024-09-14T03:59:22.310 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-14T03:59:22.310 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-14T03:59:22.310 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 29, last_sequence is 13390, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 25 2024-09-14T03:59:22.311 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-09-14T03:59:22.311 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: d29781d4-a373-4aab-bb3e-70394ca1cc26 2024-09-14T03:59:22.311 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726286362059102, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-09-14T03:59:22.311 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-09-14T03:59:22.311 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726286362089098, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 8619356, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13395, "largest_seqno": 18830, "table_properties": {"data_size": 8595240, "index_size": 17372, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 5701, "raw_key_size": 51772, "raw_average_key_size": 22, "raw_value_size": 8545821, "raw_average_value_size": 3769, "num_data_blocks": 748, "num_entries": 2267, "num_filter_entries": 2267, "num_deletions": 270, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726286362, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "d29781d4-a373-4aab-bb3e-70394ca1cc26", "db_session_id": "WCYOSYEJO3I00KUIR3QO", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-09-14T03:59:22.311 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726286362089204, "job": 1, "event": "recovery_finished"} 2024-09-14T03:59:22.311 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-09-14T03:59:22.311 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-14T03:59:22.311 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-14T03:59:22.311 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55601695ae00 2024-09-14T03:59:22.311 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: DB pointer 0x556016a5c000 2024-09-14T03:59:22.311 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-14T03:59:22.311 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-14T03:59:22.311 INFO:journalctl@ceph.mon.c.smithi039.stdout: ** DB Stats ** 2024-09-14T03:59:22.312 INFO:journalctl@ceph.mon.c.smithi039.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:59:22.312 INFO:journalctl@ceph.mon.c.smithi039.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-14T03:59:22.312 INFO:journalctl@ceph.mon.c.smithi039.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:59:22.312 INFO:journalctl@ceph.mon.c.smithi039.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:59:22.312 INFO:journalctl@ceph.mon.c.smithi039.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-14T03:59:22.312 INFO:journalctl@ceph.mon.c.smithi039.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T03:59:22.312 INFO:journalctl@ceph.mon.c.smithi039.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T03:59:22.312 INFO:journalctl@ceph.mon.c.smithi039.stdout: 2024-09-14T03:59:22.312 INFO:journalctl@ceph.mon.c.smithi039.stdout: ** Compaction Stats [default] ** 2024-09-14T03:59:22.312 INFO:journalctl@ceph.mon.c.smithi039.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:59:22.312 INFO:journalctl@ceph.mon.c.smithi039.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-14T03:59:22.312 INFO:journalctl@ceph.mon.c.smithi039.stdout: L0 1/0 8.22 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 795.7 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-09-14T03:59:22.312 INFO:journalctl@ceph.mon.c.smithi039.stdout: L6 1/0 8.55 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-14T03:59:22.313 INFO:journalctl@ceph.mon.c.smithi039.stdout: Sum 2/0 16.77 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 795.7 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-09-14T03:59:22.313 INFO:journalctl@ceph.mon.c.smithi039.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 795.7 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-09-14T03:59:22.313 INFO:journalctl@ceph.mon.c.smithi039.stdout: 2024-09-14T03:59:22.313 INFO:journalctl@ceph.mon.c.smithi039.stdout: ** Compaction Stats [default] ** 2024-09-14T03:59:22.313 INFO:journalctl@ceph.mon.c.smithi039.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T03:59:22.313 INFO:journalctl@ceph.mon.c.smithi039.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T03:59:22.313 INFO:journalctl@ceph.mon.c.smithi039.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 795.7 0.01 0.00 1 0.010 0 0 0.0 0.0 2024-09-14T03:59:22.313 INFO:journalctl@ceph.mon.c.smithi039.stdout: 2024-09-14T03:59:22.313 INFO:journalctl@ceph.mon.c.smithi039.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-14T03:59:22.313 INFO:journalctl@ceph.mon.c.smithi039.stdout: 2024-09-14T03:59:22.313 INFO:journalctl@ceph.mon.c.smithi039.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T03:59:22.313 INFO:journalctl@ceph.mon.c.smithi039.stdout: Flush(GB): cumulative 0.008, interval 0.008 2024-09-14T03:59:22.313 INFO:journalctl@ceph.mon.c.smithi039.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-14T03:59:22.313 INFO:journalctl@ceph.mon.c.smithi039.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-14T03:59:22.314 INFO:journalctl@ceph.mon.c.smithi039.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-14T03:59:22.314 INFO:journalctl@ceph.mon.c.smithi039.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-14T03:59:22.314 INFO:journalctl@ceph.mon.c.smithi039.stdout: Cumulative compaction: 0.01 GB write, 229.12 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:59:22.314 INFO:journalctl@ceph.mon.c.smithi039.stdout: Interval compaction: 0.01 GB write, 229.12 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T03:59:22.314 INFO:journalctl@ceph.mon.c.smithi039.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-14T03:59:22.314 INFO:journalctl@ceph.mon.c.smithi039.stdout: Block cache BinnedLRUCache@0x55601692b610#7 capacity: 512.00 MB usage: 26.11 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-09-14T03:59:22.314 INFO:journalctl@ceph.mon.c.smithi039.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,6.03 KB,0.00115037%) IndexBlock(1,20.08 KB,0.0038296%) Misc(1,0.00 KB,0%) 2024-09-14T03:59:22.314 INFO:journalctl@ceph.mon.c.smithi039.stdout: 2024-09-14T03:59:22.314 INFO:journalctl@ceph.mon.c.smithi039.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-14T03:59:22.314 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: starting mon.c rank 2 at public addrs [v2:172.21.15.39:3301/0,v1:172.21.15.39:6790/0] at bind addrs [v2:172.21.15.39:3301/0,v1:172.21.15.39:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T03:59:22.314 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: mon.c@-1(???) e3 preinit fsid bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T03:59:22.314 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: mon.c@-1(???).mds e9 new map 2024-09-14T03:59:22.314 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: mon.c@-1(???).mds e9 print_map 2024-09-14T03:59:22.314 INFO:journalctl@ceph.mon.c.smithi039.stdout: e9 2024-09-14T03:59:22.314 INFO:journalctl@ceph.mon.c.smithi039.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-14T03:59:22.314 INFO:journalctl@ceph.mon.c.smithi039.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-14T03:59:22.315 INFO:journalctl@ceph.mon.c.smithi039.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:59:22.315 INFO:journalctl@ceph.mon.c.smithi039.stdout: legacy client fscid: 1 2024-09-14T03:59:22.315 INFO:journalctl@ceph.mon.c.smithi039.stdout: 2024-09-14T03:59:22.315 INFO:journalctl@ceph.mon.c.smithi039.stdout: Filesystem 'foo' (1) 2024-09-14T03:59:22.315 INFO:journalctl@ceph.mon.c.smithi039.stdout: fs_name foo 2024-09-14T03:59:22.315 INFO:journalctl@ceph.mon.c.smithi039.stdout: epoch 8 2024-09-14T03:59:22.315 INFO:journalctl@ceph.mon.c.smithi039.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-14T03:59:22.315 INFO:journalctl@ceph.mon.c.smithi039.stdout: created 2024-09-14T03:53:05.236201+0000 2024-09-14T03:59:22.315 INFO:journalctl@ceph.mon.c.smithi039.stdout: modified 2024-09-14T03:53:22.726472+0000 2024-09-14T03:59:22.315 INFO:journalctl@ceph.mon.c.smithi039.stdout: tableserver 0 2024-09-14T03:59:22.315 INFO:journalctl@ceph.mon.c.smithi039.stdout: root 0 2024-09-14T03:59:22.315 INFO:journalctl@ceph.mon.c.smithi039.stdout: session_timeout 60 2024-09-14T03:59:22.315 INFO:journalctl@ceph.mon.c.smithi039.stdout: session_autoclose 300 2024-09-14T03:59:22.315 INFO:journalctl@ceph.mon.c.smithi039.stdout: max_file_size 1099511627776 2024-09-14T03:59:22.315 INFO:journalctl@ceph.mon.c.smithi039.stdout: max_xattr_size 65536 2024-09-14T03:59:22.316 INFO:journalctl@ceph.mon.c.smithi039.stdout: required_client_features {} 2024-09-14T03:59:22.316 INFO:journalctl@ceph.mon.c.smithi039.stdout: last_failure 0 2024-09-14T03:59:22.316 INFO:journalctl@ceph.mon.c.smithi039.stdout: last_failure_osd_epoch 0 2024-09-14T03:59:22.316 INFO:journalctl@ceph.mon.c.smithi039.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T03:59:22.316 INFO:journalctl@ceph.mon.c.smithi039.stdout: max_mds 1 2024-09-14T03:59:22.316 INFO:journalctl@ceph.mon.c.smithi039.stdout: in 0 2024-09-14T03:59:22.316 INFO:journalctl@ceph.mon.c.smithi039.stdout: up {0=14646} 2024-09-14T03:59:22.316 INFO:journalctl@ceph.mon.c.smithi039.stdout: failed 2024-09-14T03:59:22.316 INFO:journalctl@ceph.mon.c.smithi039.stdout: damaged 2024-09-14T03:59:22.316 INFO:journalctl@ceph.mon.c.smithi039.stdout: stopped 2024-09-14T03:59:22.316 INFO:journalctl@ceph.mon.c.smithi039.stdout: data_pools [4] 2024-09-14T03:59:22.316 INFO:journalctl@ceph.mon.c.smithi039.stdout: metadata_pool 3 2024-09-14T03:59:22.316 INFO:journalctl@ceph.mon.c.smithi039.stdout: inline_data disabled 2024-09-14T03:59:22.316 INFO:journalctl@ceph.mon.c.smithi039.stdout: balancer 2024-09-14T03:59:22.317 INFO:journalctl@ceph.mon.c.smithi039.stdout: bal_rank_mask -1 2024-09-14T03:59:22.317 INFO:journalctl@ceph.mon.c.smithi039.stdout: standby_count_wanted 1 2024-09-14T03:59:22.317 INFO:journalctl@ceph.mon.c.smithi039.stdout: qdb_cluster leader: 0 members: 2024-09-14T03:59:22.317 INFO:journalctl@ceph.mon.c.smithi039.stdout: [mds.foo.smithi039.bbwiwf{0:14646} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.39:6834/2717630652,v1:172.21.15.39:6835/2717630652] compat {c=[1],r=[1],i=[7ff]}] 2024-09-14T03:59:22.317 INFO:journalctl@ceph.mon.c.smithi039.stdout: 2024-09-14T03:59:22.317 INFO:journalctl@ceph.mon.c.smithi039.stdout: 2024-09-14T03:59:22.317 INFO:journalctl@ceph.mon.c.smithi039.stdout: Standby daemons: 2024-09-14T03:59:22.317 INFO:journalctl@ceph.mon.c.smithi039.stdout: 2024-09-14T03:59:22.317 INFO:journalctl@ceph.mon.c.smithi039.stdout: [mds.foo.smithi129.ybjqcw{-1:24475} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.129:6832/2789452907,v1:172.21.15.129:6833/2789452907] compat {c=[1],r=[1],i=[7ff]}] 2024-09-14T03:59:22.317 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: mon.c@-1(???).osd e360 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-14T03:59:22.317 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: mon.c@-1(???).osd e360 crush map has features 432629239337189376, adjusting msgr requires 2024-09-14T03:59:22.317 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: mon.c@-1(???).osd e360 crush map has features 432629239337189376, adjusting msgr requires 2024-09-14T03:59:22.317 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: mon.c@-1(???).osd e360 crush map has features 432629239337189376, adjusting msgr requires 2024-09-14T03:59:22.318 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 ceph-mon[111235]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-09-14T03:59:22.318 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 bash[111197]: 4b8f0f3864b1af7e8d3d02cf209fc2683f759bae1d66de523deb4db0dd1d4ba0 2024-09-14T03:59:22.318 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:22 smithi039 systemd[1]: Started Ceph mon.c for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T03:59:28.369 INFO:tasks.workunit.client.0.smithi039.stdout:write: c 123965877~808177 2024-09-14T03:59:28.375 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 23627349~6032477 2024-09-14T03:59:28.375 INFO:tasks.workunit.client.0.smithi039.stdout:write: c 43903170~3477283 2024-09-14T03:59:28.376 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 123469996~1907276 2024-09-14T03:59:28.390 INFO:tasks.workunit.client.0.smithi039.stdout:write: c 29486539~1590200 2024-09-14T03:59:28.392 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 56855175~4310892 2024-09-14T03:59:28.393 INFO:tasks.workunit.client.0.smithi039.stdout:write: c 123302534~4912203 2024-09-14T03:59:28.405 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 59758036~6797853 2024-09-14T03:59:28.414 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: user2 2024-09-14T03:59:28.650 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[111235]: mon.c calling monitor election 2024-09-14T03:59:28.650 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[111235]: mon.b calling monitor election 2024-09-14T03:59:28.650 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[111235]: mon.a calling monitor election 2024-09-14T03:59:28.650 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[111235]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:59:28.650 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[111235]: monmap epoch 3 2024-09-14T03:59:28.650 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[111235]: fsid bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T03:59:28.650 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[111235]: last_changed 2024-09-14T03:47:43.672905+0000 2024-09-14T03:59:28.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[111235]: created 2024-09-14T03:46:10.112174+0000 2024-09-14T03:59:28.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[111235]: min_mon_release 18 (reef) 2024-09-14T03:59:28.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[111235]: election_strategy: 1 2024-09-14T03:59:28.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[111235]: 0: [v2:172.21.15.39:3300/0,v1:172.21.15.39:6789/0] mon.a 2024-09-14T03:59:28.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[111235]: 1: [v2:172.21.15.129:3300/0,v1:172.21.15.129:6789/0] mon.b 2024-09-14T03:59:28.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[111235]: 2: [v2:172.21.15.39:3301/0,v1:172.21.15.39:6790/0] mon.c 2024-09-14T03:59:28.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[111235]: fsmap foo:1 {0=foo.smithi039.bbwiwf=up:active} 1 up:standby 2024-09-14T03:59:28.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[111235]: osdmap e360: 8 total, 8 up, 8 in 2024-09-14T03:59:28.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[111235]: mgrmap e40: y(active, since 21s), standbys: x 2024-09-14T03:59:28.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[111235]: Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-09-14T03:59:28.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[111235]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-09-14T03:59:28.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[111235]: application not enabled on pool 'test-librbd-smithi039-85658-9' 2024-09-14T03:59:28.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[111235]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-09-14T03:59:28.652 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[106470]: mon.c calling monitor election 2024-09-14T03:59:28.652 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[106470]: mon.b calling monitor election 2024-09-14T03:59:28.652 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[106470]: mon.a calling monitor election 2024-09-14T03:59:28.652 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[106470]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:59:28.652 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[106470]: monmap epoch 3 2024-09-14T03:59:28.652 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[106470]: fsid bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T03:59:28.652 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[106470]: last_changed 2024-09-14T03:47:43.672905+0000 2024-09-14T03:59:28.652 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[106470]: created 2024-09-14T03:46:10.112174+0000 2024-09-14T03:59:28.652 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[106470]: min_mon_release 18 (reef) 2024-09-14T03:59:28.652 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[106470]: election_strategy: 1 2024-09-14T03:59:28.652 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[106470]: 0: [v2:172.21.15.39:3300/0,v1:172.21.15.39:6789/0] mon.a 2024-09-14T03:59:28.653 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[106470]: 1: [v2:172.21.15.129:3300/0,v1:172.21.15.129:6789/0] mon.b 2024-09-14T03:59:28.653 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[106470]: 2: [v2:172.21.15.39:3301/0,v1:172.21.15.39:6790/0] mon.c 2024-09-14T03:59:28.653 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[106470]: fsmap foo:1 {0=foo.smithi039.bbwiwf=up:active} 1 up:standby 2024-09-14T03:59:28.653 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[106470]: osdmap e360: 8 total, 8 up, 8 in 2024-09-14T03:59:28.653 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[106470]: mgrmap e40: y(active, since 21s), standbys: x 2024-09-14T03:59:28.653 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[106470]: Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-09-14T03:59:28.653 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[106470]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-09-14T03:59:28.653 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[106470]: application not enabled on pool 'test-librbd-smithi039-85658-9' 2024-09-14T03:59:28.653 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-mon[106470]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-09-14T03:59:28.674 INFO:teuthology.orchestra.run.smithi039.stdout:true 2024-09-14T03:59:28.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:28 smithi129 ceph-mon[33173]: mon.c calling monitor election 2024-09-14T03:59:28.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:28 smithi129 ceph-mon[33173]: mon.b calling monitor election 2024-09-14T03:59:28.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:28 smithi129 ceph-mon[33173]: mon.a calling monitor election 2024-09-14T03:59:28.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:28 smithi129 ceph-mon[33173]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T03:59:28.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:28 smithi129 ceph-mon[33173]: monmap epoch 3 2024-09-14T03:59:28.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:28 smithi129 ceph-mon[33173]: fsid bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T03:59:28.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:28 smithi129 ceph-mon[33173]: last_changed 2024-09-14T03:47:43.672905+0000 2024-09-14T03:59:28.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:28 smithi129 ceph-mon[33173]: created 2024-09-14T03:46:10.112174+0000 2024-09-14T03:59:28.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:28 smithi129 ceph-mon[33173]: min_mon_release 18 (reef) 2024-09-14T03:59:28.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:28 smithi129 ceph-mon[33173]: election_strategy: 1 2024-09-14T03:59:28.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:28 smithi129 ceph-mon[33173]: 0: [v2:172.21.15.39:3300/0,v1:172.21.15.39:6789/0] mon.a 2024-09-14T03:59:28.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:28 smithi129 ceph-mon[33173]: 1: [v2:172.21.15.129:3300/0,v1:172.21.15.129:6789/0] mon.b 2024-09-14T03:59:28.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:28 smithi129 ceph-mon[33173]: 2: [v2:172.21.15.39:3301/0,v1:172.21.15.39:6790/0] mon.c 2024-09-14T03:59:28.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:28 smithi129 ceph-mon[33173]: fsmap foo:1 {0=foo.smithi039.bbwiwf=up:active} 1 up:standby 2024-09-14T03:59:28.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:28 smithi129 ceph-mon[33173]: osdmap e360: 8 total, 8 up, 8 in 2024-09-14T03:59:28.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:28 smithi129 ceph-mon[33173]: mgrmap e40: y(active, since 21s), standbys: x 2024-09-14T03:59:28.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:28 smithi129 ceph-mon[33173]: Health detail: HEALTH_WARN 1 pool(s) do not have an application enabled 2024-09-14T03:59:28.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:28 smithi129 ceph-mon[33173]: [WRN] POOL_APP_NOT_ENABLED: 1 pool(s) do not have an application enabled 2024-09-14T03:59:28.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:28 smithi129 ceph-mon[33173]: application not enabled on pool 'test-librbd-smithi039-85658-9' 2024-09-14T03:59:28.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:28 smithi129 ceph-mon[33173]: use 'ceph osd pool application enable ', where is 'cephfs', 'rbd', 'rgw', or freeform for custom applications. 2024-09-14T03:59:28.973 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:59:28] "GET /metrics HTTP/1.1" 200 38865 "" "Prometheus/2.51.0" 2024-09-14T03:59:29.268 INFO:teuthology.orchestra.run.smithi039.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T03:59:29.268 INFO:teuthology.orchestra.run.smithi039.stdout:alertmanager.a smithi039 *:9093,9094 running (98s) 17s ago 7m 18.9M - 0.27.0 11f11916f8cd 6b564e1fc909 2024-09-14T03:59:29.268 INFO:teuthology.orchestra.run.smithi039.stdout:grafana.a smithi129 *:3000 running (75s) 16s ago 7m 77.0M - 10.4.0 c8b91775d855 f0492c19de92 2024-09-14T03:59:29.268 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi039.bbwiwf smithi039 running (6m) 17s ago 6m 21.5M - 18.2.4-845-ge4e227b3 67ddd33fa72f 91e6c8f85681 2024-09-14T03:59:29.268 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi129.ybjqcw smithi129 running (6m) 16s ago 6m 15.4M - 18.2.4-845-ge4e227b3 67ddd33fa72f a3fab5b37b71 2024-09-14T03:59:29.268 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.x smithi129 *:8443,9283,8765 running (2m) 17s ago 11m 451M - 19.3.0-4907-ga2782315 8801d2529e43 448bd4e146cb 2024-09-14T03:59:29.268 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.y smithi039 *:8443,9283,8765 running (3m) 17s ago 13m 490M - 19.3.0-4907-ga2782315 8801d2529e43 cccc87f635a6 2024-09-14T03:59:29.268 INFO:teuthology.orchestra.run.smithi039.stdout:mon.a smithi039 running (29s) 17s ago 13m 36.5M 2048M 19.3.0-4907-ga2782315 8801d2529e43 7198d4eb8e4c 2024-09-14T03:59:29.268 INFO:teuthology.orchestra.run.smithi039.stdout:mon.b smithi129 running (11m) 17s ago 11m 66.0M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f eb38cc5cbe35 2024-09-14T03:59:29.268 INFO:teuthology.orchestra.run.smithi039.stdout:mon.c smithi039 starting - - - 2048M 2024-09-14T03:59:29.269 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.a smithi039 *:9100 running (2m) 17s ago 6m 11.3M - 1.7.0 72c9c2088986 7dd5945adb64 2024-09-14T03:59:29.269 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.b smithi129 *:9100 running (2m) 16s ago 6m 11.1M - 1.7.0 72c9c2088986 1fedb5f257e7 2024-09-14T03:59:29.269 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0 smithi039 running (11m) 17s ago 11m 151M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 5048d74f27ac 2024-09-14T03:59:29.269 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1 smithi039 running (10m) 17s ago 10m 147M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 91349527d88b 2024-09-14T03:59:29.269 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2 smithi039 running (10m) 17s ago 10m 127M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f eb903b44fc5d 2024-09-14T03:59:29.269 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3 smithi039 running (9m) 17s ago 9m 144M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f dff9e665bd9b 2024-09-14T03:59:29.269 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4 smithi129 running (9m) 17s ago 9m 131M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 5b64727311b7 2024-09-14T03:59:29.269 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5 smithi129 running (9m) 17s ago 9m 113M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f f2d991a754f7 2024-09-14T03:59:29.269 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6 smithi129 running (8m) 17s ago 8m 160M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 3060eaf44106 2024-09-14T03:59:29.269 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7 smithi129 running (8m) 17s ago 8m 113M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 4c13b97060d1 2024-09-14T03:59:29.269 INFO:teuthology.orchestra.run.smithi039.stdout:prometheus.a smithi129 *:9095 running (118s) 16s ago 7m 42.8M - 2.51.0 1d3b7f56885b cbe583f852c3 2024-09-14T03:59:29.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:29 smithi039 ceph-mon[106470]: osdmap e361: 8 total, 8 up, 8 in 2024-09-14T03:59:29.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:29 smithi039 ceph-mon[106470]: from='client.34143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:29.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:29 smithi039 ceph-mon[106470]: pgmap v27: 137 pgs: 137 active+clean; 2.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 31 KiB/s rd, 2.5 MiB/s wr, 51 op/s 2024-09-14T03:59:29.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:29 smithi039 ceph-mon[106470]: from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:29.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:29 smithi039 ceph-mon[111235]: osdmap e361: 8 total, 8 up, 8 in 2024-09-14T03:59:29.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:29 smithi039 ceph-mon[111235]: from='client.34143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:29.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:29 smithi039 ceph-mon[111235]: pgmap v27: 137 pgs: 137 active+clean; 2.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 31 KiB/s rd, 2.5 MiB/s wr, 51 op/s 2024-09-14T03:59:29.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:29 smithi039 ceph-mon[111235]: from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:29.658 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T03:59:29.659 INFO:teuthology.orchestra.run.smithi039.stdout: "mon": { 2024-09-14T03:59:29.659 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 1, 2024-09-14T03:59:29.659 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:59:29.659 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:59:29.659 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr": { 2024-09-14T03:59:29.659 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T03:59:29.659 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:59:29.659 INFO:teuthology.orchestra.run.smithi039.stdout: "osd": { 2024-09-14T03:59:29.659 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8 2024-09-14T03:59:29.659 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:59:29.659 INFO:teuthology.orchestra.run.smithi039.stdout: "mds": { 2024-09-14T03:59:29.659 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T03:59:29.659 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T03:59:29.660 INFO:teuthology.orchestra.run.smithi039.stdout: "overall": { 2024-09-14T03:59:29.660 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 11, 2024-09-14T03:59:29.660 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 4 2024-09-14T03:59:29.660 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T03:59:29.660 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T03:59:29.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:29 smithi129 ceph-mon[33173]: osdmap e361: 8 total, 8 up, 8 in 2024-09-14T03:59:29.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:29 smithi129 ceph-mon[33173]: from='client.34143 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:29.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:29 smithi129 ceph-mon[33173]: pgmap v27: 137 pgs: 137 active+clean; 2.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 31 KiB/s rd, 2.5 MiB/s wr, 51 op/s 2024-09-14T03:59:29.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:29 smithi129 ceph-mon[33173]: from='client.34149 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:29.985 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T03:59:29.985 INFO:teuthology.orchestra.run.smithi039.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T03:59:29.985 INFO:teuthology.orchestra.run.smithi039.stdout: "in_progress": true, 2024-09-14T03:59:29.985 INFO:teuthology.orchestra.run.smithi039.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (0 remaining).", 2024-09-14T03:59:29.985 INFO:teuthology.orchestra.run.smithi039.stdout: "services_complete": [], 2024-09-14T03:59:29.986 INFO:teuthology.orchestra.run.smithi039.stdout: "progress": "1/3 daemons upgraded", 2024-09-14T03:59:29.986 INFO:teuthology.orchestra.run.smithi039.stdout: "message": "Currently upgrading mon daemons", 2024-09-14T03:59:29.986 INFO:teuthology.orchestra.run.smithi039.stdout: "is_paused": false 2024-09-14T03:59:29.986 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T03:59:30.408 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 123324931 2024-09-14T03:59:30.436 INFO:tasks.workunit.client.0.smithi039.stdout:write: d 36606466~5619446 2024-09-14T03:59:30.447 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 24095755~6380597 2024-09-14T03:59:30.447 INFO:tasks.workunit.client.0.smithi039.stdout:write: d 57847917~4928100 2024-09-14T03:59:30.451 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 20764396~3059139 2024-09-14T03:59:30.451 INFO:tasks.workunit.client.0.smithi039.stdout:write: d 51000170~28895 2024-09-14T03:59:30.451 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 47112474~7462049 2024-09-14T03:59:30.478 INFO:tasks.workunit.client.0.smithi039.stdout:write: d 34064223~6116401 2024-09-14T03:59:30.482 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 96787066~2316483 2024-09-14T03:59:30.482 INFO:tasks.workunit.client.0.smithi039.stdout:snap_create: user3 2024-09-14T03:59:30.652 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:30 smithi039 ceph-mon[106470]: from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:30.652 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:30 smithi039 ceph-mon[106470]: osdmap e362: 8 total, 8 up, 8 in 2024-09-14T03:59:30.652 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:30 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3158344815' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:59:30.652 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:30 smithi039 ceph-mon[106470]: from='client.34167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:30.652 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:30 smithi039 ceph-mon[111235]: from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:30.653 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:30 smithi039 ceph-mon[111235]: osdmap e362: 8 total, 8 up, 8 in 2024-09-14T03:59:30.653 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:30 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3158344815' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:59:30.653 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:30 smithi039 ceph-mon[111235]: from='client.34167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:30.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:30 smithi129 ceph-mon[33173]: from='client.34155 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:30.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:30 smithi129 ceph-mon[33173]: osdmap e362: 8 total, 8 up, 8 in 2024-09-14T03:59:30.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:30 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/3158344815' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T03:59:30.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:30 smithi129 ceph-mon[33173]: from='client.34167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T03:59:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:31 smithi129 ceph-mon[33173]: osdmap e363: 8 total, 8 up, 8 in 2024-09-14T03:59:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:31 smithi129 ceph-mon[33173]: pgmap v30: 137 pgs: 137 active+clean; 2.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-14T03:59:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:31 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:31 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:31.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:31 smithi039 ceph-mon[106470]: osdmap e363: 8 total, 8 up, 8 in 2024-09-14T03:59:31.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:31 smithi039 ceph-mon[106470]: pgmap v30: 137 pgs: 137 active+clean; 2.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-14T03:59:31.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:31 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:31.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:31 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:31.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:31 smithi039 ceph-mon[111235]: osdmap e363: 8 total, 8 up, 8 in 2024-09-14T03:59:31.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:31 smithi039 ceph-mon[111235]: pgmap v30: 137 pgs: 137 active+clean; 2.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-14T03:59:31.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:31 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:31.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:31 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:32.394 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:32 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T03:59:32.102+0000 7f268b1da640 -1 mgr.server handle_report got status from non-daemon mon.c 2024-09-14T03:59:32.411 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 157769682 2024-09-14T03:59:32.429 INFO:tasks.workunit.client.0.smithi039.stdout:write: 1 15776968~3944242 2024-09-14T03:59:32.430 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 16763028~1972121 2024-09-14T03:59:32.469 INFO:tasks.workunit.client.0.smithi039.stdout:write: 2 31553936~3944242 2024-09-14T03:59:32.470 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 32539996~1972121 2024-09-14T03:59:32.491 INFO:tasks.workunit.client.0.smithi039.stdout:write: 3 47330904~3944242 2024-09-14T03:59:32.492 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 48316964~1972121 2024-09-14T03:59:32.512 INFO:tasks.workunit.client.0.smithi039.stdout:write: 4 63107872~3944242 2024-09-14T03:59:32.513 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 64093932~1972121 2024-09-14T03:59:32.540 INFO:tasks.workunit.client.0.smithi039.stdout:write: 5 78884841~3944242 2024-09-14T03:59:32.541 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 79870901~1972121 2024-09-14T03:59:32.608 INFO:tasks.workunit.client.0.smithi039.stdout:migration_commit 2024-09-14T03:59:32.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:32 smithi129 ceph-mon[33173]: osdmap e364: 8 total, 8 up, 8 in 2024-09-14T03:59:32.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:32 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:32.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:32 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:32.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:32 smithi039 ceph-mon[111235]: osdmap e364: 8 total, 8 up, 8 in 2024-09-14T03:59:32.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:32 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:32.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:32 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:32.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:32 smithi039 ceph-mon[106470]: osdmap e364: 8 total, 8 up, 8 in 2024-09-14T03:59:32.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:32 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:32.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:32 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:33.750 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:33 smithi039 ceph-mon[106470]: osdmap e365: 8 total, 8 up, 8 in 2024-09-14T03:59:33.750 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:33 smithi039 ceph-mon[106470]: pgmap v33: 137 pgs: 137 active+clean; 2.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-14T03:59:33.751 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:33 smithi039 ceph-mon[111235]: osdmap e365: 8 total, 8 up, 8 in 2024-09-14T03:59:33.751 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:33 smithi039 ceph-mon[111235]: pgmap v33: 137 pgs: 137 active+clean; 2.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-14T03:59:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:33 smithi129 ceph-mon[33173]: osdmap e365: 8 total, 8 up, 8 in 2024-09-14T03:59:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:33 smithi129 ceph-mon[33173]: pgmap v33: 137 pgs: 137 active+clean; 2.1 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail 2024-09-14T03:59:34.377 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMigration.StressLive (26316 ms) 2024-09-14T03:59:34.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:34 smithi129 ceph-mon[33173]: osdmap e366: 8 total, 8 up, 8 in 2024-09-14T03:59:34.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:34 smithi039 ceph-mon[106470]: osdmap e366: 8 total, 8 up, 8 in 2024-09-14T03:59:34.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:34 smithi039 ceph-mon[111235]: osdmap e366: 8 total, 8 up, 8 in 2024-09-14T03:59:35.455 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 57 tests from TestMigration (274161 ms total) 2024-09-14T03:59:35.455 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T03:59:35.456 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 6 tests from TestGroup 2024-09-14T03:59:35.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:35 smithi129 ceph-mon[33173]: osdmap e367: 8 total, 8 up, 8 in 2024-09-14T03:59:35.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:35 smithi129 ceph-mon[33173]: pgmap v36: 105 pgs: 105 active+clean; 2.1 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 1.4 MiB/s wr, 6 op/s 2024-09-14T03:59:35.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:35 smithi039 ceph-mon[106470]: osdmap e367: 8 total, 8 up, 8 in 2024-09-14T03:59:35.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:35 smithi039 ceph-mon[106470]: pgmap v36: 105 pgs: 105 active+clean; 2.1 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 1.4 MiB/s wr, 6 op/s 2024-09-14T03:59:35.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:35 smithi039 ceph-mon[111235]: osdmap e367: 8 total, 8 up, 8 in 2024-09-14T03:59:35.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:35 smithi039 ceph-mon[111235]: pgmap v36: 105 pgs: 105 active+clean; 2.1 GiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 1.4 MiB/s wr, 6 op/s 2024-09-14T03:59:36.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:59:36.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: Cluster is now healthy 2024-09-14T03:59:36.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: osdmap e368: 8 total, 8 up, 8 in 2024-09-14T03:59:36.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:36.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:36.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:59:36.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:59:36.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:36.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:59:36.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:59:36.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:59:36.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: Upgrade: Setting container_image for all nvmeof 2024-09-14T03:59:36.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:36.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:59:36.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: Upgrade: Setting container_image for all node-exporter 2024-09-14T03:59:36.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:36.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:59:36.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:59:36.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-14T03:59:36.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:59:36.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:59:36.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-14T03:59:36.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:59:36.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: Upgrade: Setting container_image for all prometheus 2024-09-14T03:59:36.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:59:36.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:59:36.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:59:36.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:59:36.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:59:36.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: Upgrade: Setting container_image for all alertmanager 2024-09-14T03:59:36.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:59:36.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:59:36.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:59:36.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:59:36.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:59:36.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: Upgrade: Setting container_image for all grafana 2024-09-14T03:59:36.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:59:36.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:59:36.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:59:36.775 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:59:36.775 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:59:36.775 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: Upgrade: Setting container_image for all loki 2024-09-14T03:59:36.775 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:59:36.775 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:59:36.775 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:59:36.775 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:59:36.775 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:59:36.775 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: Upgrade: Setting container_image for all promtail 2024-09-14T03:59:36.775 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:59:36.775 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:59:36.775 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:59:36.776 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:59:36.776 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: Upgrade: Finalizing container_image settings 2024-09-14T03:59:36.776 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-14T03:59:36.776 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T03:59:36.776 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T03:59:36.776 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:59:36.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:59:36.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T03:59:36.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-14T03:59:36.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T03:59:36.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-14T03:59:36.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T03:59:36.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-14T03:59:36.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T03:59:36.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-14T03:59:36.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T03:59:36.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-14T03:59:36.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T03:59:36.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:59:36.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:59:36.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-14T03:59:36.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T03:59:36.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-14T03:59:36.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T03:59:36.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-14T03:59:36.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T03:59:36.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-14T03:59:36.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T03:59:36.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T03:59:36.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:59:36.779 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:59:36.779 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:59:36.779 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:59:36.779 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:59:36.779 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:59:36.779 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:59:36.779 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:59:36.779 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:59:36.779 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:59:36.779 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:59:36.779 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:59:36.779 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: Upgrade: Complete! 2024-09-14T03:59:36.780 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-14T03:59:36.780 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T03:59:36.780 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T03:59:36.780 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:59:36.780 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:59:36.780 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:59:36.780 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:36.780 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:59:36.780 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:59:36.780 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:59:36.780 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:36 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:36.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:59:36.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T03:59:36.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: osdmap e368: 8 total, 8 up, 8 in 2024-09-14T03:59:36.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:36.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:36.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:59:36.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:59:36.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:36.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:59:36.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:59:36.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:59:36.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all nvmeof 2024-09-14T03:59:36.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:36.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:59:36.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all node-exporter 2024-09-14T03:59:36.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:36.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:59:36.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:59:36.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-14T03:59:36.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:59:36.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:59:36.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-14T03:59:36.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:59:36.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all prometheus 2024-09-14T03:59:36.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:59:36.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:59:36.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:59:36.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:59:36.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:59:36.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all alertmanager 2024-09-14T03:59:36.795 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:59:36.795 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T03:59:36.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: osdmap e368: 8 total, 8 up, 8 in 2024-09-14T03:59:36.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:36.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:36.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:59:36.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:59:36.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:36.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:59:36.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:59:36.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:59:36.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all nvmeof 2024-09-14T03:59:36.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:36.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:59:36.797 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all node-exporter 2024-09-14T03:59:36.797 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:36.797 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:59:36.797 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:59:36.797 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-09-14T03:59:36.797 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:59:36.797 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:59:36.797 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-09-14T03:59:36.797 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:59:36.797 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all prometheus 2024-09-14T03:59:36.797 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:59:36.797 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:59:36.798 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:59:36.798 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:59:36.798 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:59:36.798 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all alertmanager 2024-09-14T03:59:36.798 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:59:36.798 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:59:36.798 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:59:36.798 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:59:36.798 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:59:36.798 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all grafana 2024-09-14T03:59:36.798 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:59:36.798 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:59:36.799 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:59:36.799 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:59:36.799 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:59:36.799 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all loki 2024-09-14T03:59:36.799 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:59:36.799 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:59:36.799 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:59:36.799 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:59:36.799 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:59:36.799 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all promtail 2024-09-14T03:59:36.799 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:59:36.799 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:59:36.799 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:59:36.799 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:59:36.800 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: Upgrade: Finalizing container_image settings 2024-09-14T03:59:36.800 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-14T03:59:36.800 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T03:59:36.800 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T03:59:36.800 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:59:36.800 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:59:36.800 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:59:36.801 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:59:36.801 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:59:36.801 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:59:36.801 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all grafana 2024-09-14T03:59:36.801 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:59:36.801 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:59:36.802 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:59:36.802 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:59:36.802 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:59:36.802 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all loki 2024-09-14T03:59:36.802 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:59:36.802 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:59:36.803 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:59:36.803 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:59:36.803 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T03:59:36.803 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all promtail 2024-09-14T03:59:36.803 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-09-14T03:59:36.803 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.a"} : dispatch 2024-09-14T03:59:36.803 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-09-14T03:59:36.804 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.c"} : dispatch 2024-09-14T03:59:36.804 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: Upgrade: Finalizing container_image settings 2024-09-14T03:59:36.804 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]: dispatch 2024-09-14T03:59:36.804 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T03:59:36.804 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T03:59:36.804 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:59:36.805 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:59:36.805 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T03:59:36.805 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-14T03:59:36.805 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T03:59:36.805 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-14T03:59:36.805 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T03:59:36.805 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-14T03:59:36.805 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T03:59:36.805 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-14T03:59:36.805 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T03:59:36.805 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-14T03:59:36.805 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T03:59:36.806 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:59:36.806 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:59:36.806 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-14T03:59:36.806 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T03:59:36.806 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-14T03:59:36.806 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T03:59:36.806 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-14T03:59:36.806 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T03:59:36.806 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-14T03:59:36.806 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T03:59:36.806 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T03:59:36.807 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:59:36.807 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T03:59:36.807 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]: dispatch 2024-09-14T03:59:36.807 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T03:59:36.807 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "osd"}]: dispatch 2024-09-14T03:59:36.807 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T03:59:36.807 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mds"}]: dispatch 2024-09-14T03:59:36.807 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T03:59:36.807 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]: dispatch 2024-09-14T03:59:36.807 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T03:59:36.807 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]: dispatch 2024-09-14T03:59:36.808 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T03:59:36.808 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:59:36.808 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:59:36.808 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]: dispatch 2024-09-14T03:59:36.808 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T03:59:36.808 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]: dispatch 2024-09-14T03:59:36.808 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T03:59:36.808 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]: dispatch 2024-09-14T03:59:36.808 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T03:59:36.808 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]: dispatch 2024-09-14T03:59:36.808 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T03:59:36.809 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T03:59:36.809 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:59:36.809 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:59:36.809 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:59:36.809 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:59:36.809 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:59:36.809 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:59:36.809 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:59:36.809 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:59:36.810 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:59:36.810 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:59:36.810 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:59:36.810 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:59:36.810 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: Upgrade: Complete! 2024-09-14T03:59:36.810 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-14T03:59:36.810 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T03:59:36.810 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T03:59:36.810 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:59:36.810 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:59:36.810 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:59:36.810 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:36.810 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:59:36.811 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:59:36.811 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:59:36.811 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:36.811 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:59:36.811 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:59:36.811 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:59:36.811 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:59:36.811 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:59:36.811 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:59:36.811 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:59:36.811 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:59:36.811 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:59:36.812 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:59:36.812 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon"}]: dispatch 2024-09-14T03:59:36.812 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T03:59:36.812 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: Upgrade: Complete! 2024-09-14T03:59:36.812 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]: dispatch 2024-09-14T03:59:36.813 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T03:59:36.813 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T03:59:36.813 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:59:36.813 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:59:36.813 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:59:36.813 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:36.814 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T03:59:36.814 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T03:59:36.814 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T03:59:36.814 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:37.527 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestGroup.group_create 2024-09-14T03:59:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:37 smithi129 ceph-mon[33173]: osdmap e369: 8 total, 8 up, 8 in 2024-09-14T03:59:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:37 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/1732589770' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:59:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:37 smithi129 ceph-mon[33173]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:59:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:37 smithi129 ceph-mon[33173]: pgmap v39: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-14T03:59:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:37 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:37.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:37 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:59:37.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:37 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:37.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:37 smithi039 ceph-mon[106470]: osdmap e369: 8 total, 8 up, 8 in 2024-09-14T03:59:37.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:37 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1732589770' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:59:37.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:37 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:59:37.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:37 smithi039 ceph-mon[106470]: pgmap v39: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-14T03:59:37.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:37.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:59:37.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:37.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:37 smithi039 ceph-mon[111235]: osdmap e369: 8 total, 8 up, 8 in 2024-09-14T03:59:37.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:37 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1732589770' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:59:37.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:37 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-10","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:59:37.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:37 smithi039 ceph-mon[111235]: pgmap v39: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-14T03:59:37.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:37.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:59:37.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T03:59:38.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:38 smithi039 ceph-mon[106470]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:59:38.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:38 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:59:38.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:38 smithi039 ceph-mon[106470]: osdmap e370: 8 total, 8 up, 8 in 2024-09-14T03:59:38.791 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:59:38] "GET /metrics HTTP/1.1" 200 36354 "" "Prometheus/2.51.0" 2024-09-14T03:59:38.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:38 smithi039 ceph-mon[111235]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:59:38.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:38 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:59:38.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:38 smithi039 ceph-mon[111235]: osdmap e370: 8 total, 8 up, 8 in 2024-09-14T03:59:39.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:38 smithi129 ceph-mon[33173]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T03:59:39.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:38 smithi129 ceph-mon[33173]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-10","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:59:39.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:38 smithi129 ceph-mon[33173]: osdmap e370: 8 total, 8 up, 8 in 2024-09-14T03:59:39.578 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestGroup.group_create (2051 ms) 2024-09-14T03:59:39.578 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestGroup.group_createPP 2024-09-14T03:59:39.597 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestGroup.group_createPP (18 ms) 2024-09-14T03:59:39.597 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestGroup.add_image 2024-09-14T03:59:39.637 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestGroup.add_image (41 ms) 2024-09-14T03:59:39.637 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestGroup.add_imagePP 2024-09-14T03:59:39.675 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestGroup.add_imagePP (38 ms) 2024-09-14T03:59:39.675 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestGroup.add_snapshot 2024-09-14T03:59:40.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:39 smithi129 ceph-mon[33173]: osdmap e371: 8 total, 8 up, 8 in 2024-09-14T03:59:40.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:39 smithi129 ceph-mon[33173]: pgmap v42: 105 pgs: 105 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:59:40.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:39 smithi039 ceph-mon[106470]: osdmap e371: 8 total, 8 up, 8 in 2024-09-14T03:59:40.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:39 smithi039 ceph-mon[106470]: pgmap v42: 105 pgs: 105 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:59:40.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:39 smithi039 ceph-mon[111235]: osdmap e371: 8 total, 8 up, 8 in 2024-09-14T03:59:40.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:39 smithi039 ceph-mon[111235]: pgmap v42: 105 pgs: 105 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:59:41.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:40 smithi129 ceph-mon[33173]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:59:41.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:40 smithi129 ceph-mon[33173]: Cluster is now healthy 2024-09-14T03:59:41.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:40 smithi129 ceph-mon[33173]: osdmap e372: 8 total, 8 up, 8 in 2024-09-14T03:59:41.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:40 smithi039 ceph-mon[106470]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:59:41.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:40 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T03:59:41.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:40 smithi039 ceph-mon[106470]: osdmap e372: 8 total, 8 up, 8 in 2024-09-14T03:59:41.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:40 smithi039 ceph-mon[111235]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T03:59:41.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:40 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T03:59:41.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:40 smithi039 ceph-mon[111235]: osdmap e372: 8 total, 8 up, 8 in 2024-09-14T03:59:42.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:41 smithi129 ceph-mon[33173]: osdmap e373: 8 total, 8 up, 8 in 2024-09-14T03:59:42.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:41 smithi129 ceph-mon[33173]: pgmap v45: 105 pgs: 105 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:59:42.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:41 smithi039 ceph-mon[106470]: osdmap e373: 8 total, 8 up, 8 in 2024-09-14T03:59:42.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:41 smithi039 ceph-mon[106470]: pgmap v45: 105 pgs: 105 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:59:42.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:41 smithi039 ceph-mon[111235]: osdmap e373: 8 total, 8 up, 8 in 2024-09-14T03:59:42.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:41 smithi039 ceph-mon[111235]: pgmap v45: 105 pgs: 105 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:59:43.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:42 smithi129 ceph-mon[33173]: osdmap e374: 8 total, 8 up, 8 in 2024-09-14T03:59:43.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:42 smithi039 ceph-mon[106470]: osdmap e374: 8 total, 8 up, 8 in 2024-09-14T03:59:43.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:42 smithi039 ceph-mon[111235]: osdmap e374: 8 total, 8 up, 8 in 2024-09-14T03:59:43.922 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:43 smithi129 ceph-mon[33173]: osdmap e375: 8 total, 8 up, 8 in 2024-09-14T03:59:43.923 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:43 smithi129 ceph-mon[33173]: pgmap v48: 105 pgs: 105 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:59:44.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:43 smithi039 ceph-mon[106470]: osdmap e375: 8 total, 8 up, 8 in 2024-09-14T03:59:44.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:43 smithi039 ceph-mon[106470]: pgmap v48: 105 pgs: 105 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:59:44.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:43 smithi039 ceph-mon[111235]: osdmap e375: 8 total, 8 up, 8 in 2024-09-14T03:59:44.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:43 smithi039 ceph-mon[111235]: pgmap v48: 105 pgs: 105 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-14T03:59:44.939 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:44 smithi129 ceph-mon[33173]: osdmap e376: 8 total, 8 up, 8 in 2024-09-14T03:59:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:44 smithi039 ceph-mon[106470]: osdmap e376: 8 total, 8 up, 8 in 2024-09-14T03:59:45.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:44 smithi039 ceph-mon[111235]: osdmap e376: 8 total, 8 up, 8 in 2024-09-14T03:59:45.667 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestGroup.add_snapshot (5992 ms) 2024-09-14T03:59:45.667 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestGroup.add_snapshotPP 2024-09-14T03:59:45.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:45 smithi129 ceph-mon[33173]: osdmap e377: 8 total, 8 up, 8 in 2024-09-14T03:59:45.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:45 smithi129 ceph-mon[33173]: pgmap v51: 105 pgs: 105 active+clean; 582 KiB data, 376 MiB used, 715 GiB / 715 GiB avail; 154 KiB/s rd, 18 KiB/s wr, 208 op/s 2024-09-14T03:59:46.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:45 smithi039 ceph-mon[106470]: osdmap e377: 8 total, 8 up, 8 in 2024-09-14T03:59:46.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:45 smithi039 ceph-mon[106470]: pgmap v51: 105 pgs: 105 active+clean; 582 KiB data, 376 MiB used, 715 GiB / 715 GiB avail; 154 KiB/s rd, 18 KiB/s wr, 208 op/s 2024-09-14T03:59:46.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:45 smithi039 ceph-mon[111235]: osdmap e377: 8 total, 8 up, 8 in 2024-09-14T03:59:46.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:45 smithi039 ceph-mon[111235]: pgmap v51: 105 pgs: 105 active+clean; 582 KiB data, 376 MiB used, 715 GiB / 715 GiB avail; 154 KiB/s rd, 18 KiB/s wr, 208 op/s 2024-09-14T03:59:47.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:46 smithi129 ceph-mon[33173]: osdmap e378: 8 total, 8 up, 8 in 2024-09-14T03:59:47.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:46 smithi039 ceph-mon[106470]: osdmap e378: 8 total, 8 up, 8 in 2024-09-14T03:59:47.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:46 smithi039 ceph-mon[111235]: osdmap e378: 8 total, 8 up, 8 in 2024-09-14T03:59:48.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:47 smithi129 ceph-mon[33173]: osdmap e379: 8 total, 8 up, 8 in 2024-09-14T03:59:48.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:47 smithi129 ceph-mon[33173]: pgmap v54: 105 pgs: 105 active+clean; 582 KiB data, 376 MiB used, 715 GiB / 715 GiB avail; 154 KiB/s rd, 18 KiB/s wr, 208 op/s 2024-09-14T03:59:48.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:47 smithi039 ceph-mon[106470]: osdmap e379: 8 total, 8 up, 8 in 2024-09-14T03:59:48.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:47 smithi039 ceph-mon[106470]: pgmap v54: 105 pgs: 105 active+clean; 582 KiB data, 376 MiB used, 715 GiB / 715 GiB avail; 154 KiB/s rd, 18 KiB/s wr, 208 op/s 2024-09-14T03:59:48.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:47 smithi039 ceph-mon[111235]: osdmap e379: 8 total, 8 up, 8 in 2024-09-14T03:59:48.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:47 smithi039 ceph-mon[111235]: pgmap v54: 105 pgs: 105 active+clean; 582 KiB data, 376 MiB used, 715 GiB / 715 GiB avail; 154 KiB/s rd, 18 KiB/s wr, 208 op/s 2024-09-14T03:59:49.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:48 smithi129 ceph-mon[33173]: osdmap e380: 8 total, 8 up, 8 in 2024-09-14T03:59:49.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:48 smithi039 ceph-mon[106470]: osdmap e380: 8 total, 8 up, 8 in 2024-09-14T03:59:49.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:48 smithi039 ceph-mon[111235]: osdmap e380: 8 total, 8 up, 8 in 2024-09-14T03:59:49.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:59:48] "GET /metrics HTTP/1.1" 200 36354 "" "Prometheus/2.51.0" 2024-09-14T03:59:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:49 smithi129 ceph-mon[33173]: osdmap e381: 8 total, 8 up, 8 in 2024-09-14T03:59:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:49 smithi129 ceph-mon[33173]: pgmap v57: 105 pgs: 105 active+clean; 583 KiB data, 380 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s rd, 10 KiB/s wr, 141 op/s 2024-09-14T03:59:50.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:49 smithi039 ceph-mon[106470]: osdmap e381: 8 total, 8 up, 8 in 2024-09-14T03:59:50.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:49 smithi039 ceph-mon[106470]: pgmap v57: 105 pgs: 105 active+clean; 583 KiB data, 380 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s rd, 10 KiB/s wr, 141 op/s 2024-09-14T03:59:50.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:49 smithi039 ceph-mon[111235]: osdmap e381: 8 total, 8 up, 8 in 2024-09-14T03:59:50.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:49 smithi039 ceph-mon[111235]: pgmap v57: 105 pgs: 105 active+clean; 583 KiB data, 380 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s rd, 10 KiB/s wr, 141 op/s 2024-09-14T03:59:51.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:50 smithi129 ceph-mon[33173]: osdmap e382: 8 total, 8 up, 8 in 2024-09-14T03:59:51.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:50 smithi129 ceph-mon[33173]: osdmap e383: 8 total, 8 up, 8 in 2024-09-14T03:59:51.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:50 smithi039 ceph-mon[106470]: osdmap e382: 8 total, 8 up, 8 in 2024-09-14T03:59:51.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:50 smithi039 ceph-mon[106470]: osdmap e383: 8 total, 8 up, 8 in 2024-09-14T03:59:51.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:50 smithi039 ceph-mon[111235]: osdmap e382: 8 total, 8 up, 8 in 2024-09-14T03:59:51.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:50 smithi039 ceph-mon[111235]: osdmap e383: 8 total, 8 up, 8 in 2024-09-14T03:59:51.396 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestGroup.add_snapshotPP (5729 ms) 2024-09-14T03:59:52.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:51 smithi129 ceph-mon[33173]: pgmap v60: 105 pgs: 105 active+clean; 583 KiB data, 380 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s rd, 10 KiB/s wr, 141 op/s 2024-09-14T03:59:52.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:51 smithi129 ceph-mon[33173]: osdmap e384: 8 total, 8 up, 8 in 2024-09-14T03:59:52.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:51 smithi039 ceph-mon[106470]: pgmap v60: 105 pgs: 105 active+clean; 583 KiB data, 380 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s rd, 10 KiB/s wr, 141 op/s 2024-09-14T03:59:52.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:51 smithi039 ceph-mon[106470]: osdmap e384: 8 total, 8 up, 8 in 2024-09-14T03:59:52.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:51 smithi039 ceph-mon[111235]: pgmap v60: 105 pgs: 105 active+clean; 583 KiB data, 380 MiB used, 715 GiB / 715 GiB avail; 105 KiB/s rd, 10 KiB/s wr, 141 op/s 2024-09-14T03:59:52.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:51 smithi039 ceph-mon[111235]: osdmap e384: 8 total, 8 up, 8 in 2024-09-14T03:59:52.368 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 6 tests from TestGroup (13869 ms total) 2024-09-14T03:59:52.368 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T03:59:52.368 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 31 tests from TestDeepCopy 2024-09-14T03:59:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:52 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:59:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:52 smithi129 ceph-mon[33173]: osdmap e385: 8 total, 8 up, 8 in 2024-09-14T03:59:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:59:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:52 smithi039 ceph-mon[106470]: osdmap e385: 8 total, 8 up, 8 in 2024-09-14T03:59:53.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T03:59:53.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:52 smithi039 ceph-mon[111235]: osdmap e385: 8 total, 8 up, 8 in 2024-09-14T03:59:54.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:53 smithi129 ceph-mon[33173]: pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 380 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:59:54.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:53 smithi129 ceph-mon[33173]: osdmap e386: 8 total, 8 up, 8 in 2024-09-14T03:59:54.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:53 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/3485584571' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:59:54.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:53 smithi039 ceph-mon[106470]: pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 380 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:59:54.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:53 smithi039 ceph-mon[106470]: osdmap e386: 8 total, 8 up, 8 in 2024-09-14T03:59:54.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:53 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3485584571' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:59:54.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:53 smithi039 ceph-mon[111235]: pgmap v63: 73 pgs: 73 active+clean; 582 KiB data, 380 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:59:54.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:53 smithi039 ceph-mon[111235]: osdmap e386: 8 total, 8 up, 8 in 2024-09-14T03:59:54.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:53 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3485584571' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-11","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T03:59:54.376 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestDeepCopy.Empty 2024-09-14T03:59:55.663 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:55 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/3485584571' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:59:55.663 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:55 smithi129 ceph-mon[33173]: osdmap e387: 8 total, 8 up, 8 in 2024-09-14T03:59:55.663 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:55 smithi129 ceph-mon[33173]: pgmap v66: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 381 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:59:55.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:55 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3485584571' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:59:55.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:55 smithi039 ceph-mon[106470]: osdmap e387: 8 total, 8 up, 8 in 2024-09-14T03:59:55.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:55 smithi039 ceph-mon[106470]: pgmap v66: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 381 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:59:55.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:55 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3485584571' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-11","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T03:59:55.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:55 smithi039 ceph-mon[111235]: osdmap e387: 8 total, 8 up, 8 in 2024-09-14T03:59:55.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:55 smithi039 ceph-mon[111235]: pgmap v66: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 381 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:59:56.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:56 smithi129 ceph-mon[33173]: osdmap e388: 8 total, 8 up, 8 in 2024-09-14T03:59:56.532 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestDeepCopy.Empty (2156 ms) 2024-09-14T03:59:56.532 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestDeepCopy.NoSnaps 2024-09-14T03:59:56.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:56 smithi039 ceph-mon[106470]: osdmap e388: 8 total, 8 up, 8 in 2024-09-14T03:59:56.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:56 smithi039 ceph-mon[111235]: osdmap e388: 8 total, 8 up, 8 in 2024-09-14T03:59:56.798 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestDeepCopy.NoSnaps (266 ms) 2024-09-14T03:59:56.798 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestDeepCopy.Snaps 2024-09-14T03:59:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:57 smithi129 ceph-mon[33173]: osdmap e389: 8 total, 8 up, 8 in 2024-09-14T03:59:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:57 smithi129 ceph-mon[33173]: pgmap v69: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 381 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:59:57.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:57 smithi039 ceph-mon[106470]: osdmap e389: 8 total, 8 up, 8 in 2024-09-14T03:59:57.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:57 smithi039 ceph-mon[106470]: pgmap v69: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 381 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:59:57.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:57 smithi039 ceph-mon[111235]: osdmap e389: 8 total, 8 up, 8 in 2024-09-14T03:59:57.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:57 smithi039 ceph-mon[111235]: pgmap v69: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 381 MiB used, 715 GiB / 715 GiB avail 2024-09-14T03:59:58.692 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:58 smithi039 ceph-mon[106470]: osdmap e390: 8 total, 8 up, 8 in 2024-09-14T03:59:58.692 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 03:59:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:03:59:58] "GET /metrics HTTP/1.1" 200 36401 "" "Prometheus/2.51.0" 2024-09-14T03:59:58.693 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:58 smithi039 ceph-mon[111235]: osdmap e390: 8 total, 8 up, 8 in 2024-09-14T03:59:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:58 smithi129 ceph-mon[33173]: osdmap e390: 8 total, 8 up, 8 in 2024-09-14T03:59:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:59 smithi129 ceph-mon[33173]: osdmap e391: 8 total, 8 up, 8 in 2024-09-14T03:59:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 03:59:59 smithi129 ceph-mon[33173]: pgmap v72: 105 pgs: 105 active+clean; 49 MiB data, 510 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 146 op/s 2024-09-14T03:59:59.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:59 smithi039 ceph-mon[106470]: osdmap e391: 8 total, 8 up, 8 in 2024-09-14T03:59:59.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 03:59:59 smithi039 ceph-mon[106470]: pgmap v72: 105 pgs: 105 active+clean; 49 MiB data, 510 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 146 op/s 2024-09-14T03:59:59.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:59 smithi039 ceph-mon[111235]: osdmap e391: 8 total, 8 up, 8 in 2024-09-14T03:59:59.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 03:59:59 smithi039 ceph-mon[111235]: pgmap v72: 105 pgs: 105 active+clean; 49 MiB data, 510 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 146 op/s 2024-09-14T04:00:00.645 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestDeepCopy.Snaps (3847 ms) 2024-09-14T04:00:00.645 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestDeepCopy.SnapDiscard 2024-09-14T04:00:00.660 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:00 smithi039 ceph-mon[106470]: osdmap e392: 8 total, 8 up, 8 in 2024-09-14T04:00:00.660 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:00 smithi039 ceph-mon[106470]: overall HEALTH_OK 2024-09-14T04:00:00.660 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:00 smithi039 ceph-mon[106470]: osdmap e393: 8 total, 8 up, 8 in 2024-09-14T04:00:00.660 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:00 smithi039 ceph-mon[111235]: osdmap e392: 8 total, 8 up, 8 in 2024-09-14T04:00:00.660 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:00 smithi039 ceph-mon[111235]: overall HEALTH_OK 2024-09-14T04:00:00.660 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:00 smithi039 ceph-mon[111235]: osdmap e393: 8 total, 8 up, 8 in 2024-09-14T04:00:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:00 smithi129 ceph-mon[33173]: osdmap e392: 8 total, 8 up, 8 in 2024-09-14T04:00:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:00 smithi129 ceph-mon[33173]: overall HEALTH_OK 2024-09-14T04:00:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:00 smithi129 ceph-mon[33173]: osdmap e393: 8 total, 8 up, 8 in 2024-09-14T04:00:01.110 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'sleep 60' 2024-09-14T04:00:01.325 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:00:01.756 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:01 smithi039 ceph-mon[106470]: from='client.34197 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:00:01.756 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:01 smithi039 ceph-mon[106470]: pgmap v75: 105 pgs: 105 active+clean; 49 MiB data, 510 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 146 op/s 2024-09-14T04:00:01.756 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:01 smithi039 ceph-mon[111235]: from='client.34197 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:00:01.756 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:01 smithi039 ceph-mon[111235]: pgmap v75: 105 pgs: 105 active+clean; 49 MiB data, 510 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 146 op/s 2024-09-14T04:00:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:01 smithi129 ceph-mon[33173]: from='client.34197 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:00:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:01 smithi129 ceph-mon[33173]: pgmap v75: 105 pgs: 105 active+clean; 49 MiB data, 510 MiB used, 715 GiB / 715 GiB avail; 12 MiB/s rd, 12 MiB/s wr, 146 op/s 2024-09-14T04:00:02.609 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestDeepCopy.SnapDiscard (1963 ms) 2024-09-14T04:00:02.609 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestDeepCopy.CloneDiscard 2024-09-14T04:00:02.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:02 smithi129 ceph-mon[33173]: osdmap e394: 8 total, 8 up, 8 in 2024-09-14T04:00:02.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:02 smithi039 ceph-mon[106470]: osdmap e394: 8 total, 8 up, 8 in 2024-09-14T04:00:02.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:02 smithi039 ceph-mon[111235]: osdmap e394: 8 total, 8 up, 8 in 2024-09-14T04:00:03.695 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestDeepCopy.CloneDiscard (1087 ms) 2024-09-14T04:00:03.695 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestDeepCopy.CloneShrink 2024-09-14T04:00:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:03 smithi129 ceph-mon[33173]: osdmap e395: 8 total, 8 up, 8 in 2024-09-14T04:00:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:03 smithi129 ceph-mon[33173]: pgmap v78: 105 pgs: 105 active+clean; 49 MiB data, 534 MiB used, 715 GiB / 715 GiB avail; 6.0 MiB/s rd, 9.2 KiB/s wr, 52 op/s 2024-09-14T04:00:03.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:03 smithi039 ceph-mon[111235]: osdmap e395: 8 total, 8 up, 8 in 2024-09-14T04:00:03.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:03 smithi039 ceph-mon[111235]: pgmap v78: 105 pgs: 105 active+clean; 49 MiB data, 534 MiB used, 715 GiB / 715 GiB avail; 6.0 MiB/s rd, 9.2 KiB/s wr, 52 op/s 2024-09-14T04:00:03.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:03 smithi039 ceph-mon[106470]: osdmap e395: 8 total, 8 up, 8 in 2024-09-14T04:00:03.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:03 smithi039 ceph-mon[106470]: pgmap v78: 105 pgs: 105 active+clean; 49 MiB data, 534 MiB used, 715 GiB / 715 GiB avail; 6.0 MiB/s rd, 9.2 KiB/s wr, 52 op/s 2024-09-14T04:00:04.686 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestDeepCopy.CloneShrink (991 ms) 2024-09-14T04:00:04.687 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestDeepCopy.CloneExpand 2024-09-14T04:00:04.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:04 smithi129 ceph-mon[33173]: osdmap e396: 8 total, 8 up, 8 in 2024-09-14T04:00:04.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:04 smithi039 ceph-mon[106470]: osdmap e396: 8 total, 8 up, 8 in 2024-09-14T04:00:04.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:04 smithi039 ceph-mon[111235]: osdmap e396: 8 total, 8 up, 8 in 2024-09-14T04:00:05.753 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestDeepCopy.CloneExpand (1066 ms) 2024-09-14T04:00:05.753 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestDeepCopy.CloneHideParent 2024-09-14T04:00:05.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:05 smithi039 ceph-mon[106470]: osdmap e397: 8 total, 8 up, 8 in 2024-09-14T04:00:05.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:05 smithi039 ceph-mon[106470]: pgmap v81: 105 pgs: 105 active+clean; 65 MiB data, 574 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s rd, 4.0 MiB/s wr, 164 op/s 2024-09-14T04:00:05.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:05 smithi039 ceph-mon[111235]: osdmap e397: 8 total, 8 up, 8 in 2024-09-14T04:00:05.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:05 smithi039 ceph-mon[111235]: pgmap v81: 105 pgs: 105 active+clean; 65 MiB data, 574 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s rd, 4.0 MiB/s wr, 164 op/s 2024-09-14T04:00:06.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:05 smithi129 ceph-mon[33173]: osdmap e397: 8 total, 8 up, 8 in 2024-09-14T04:00:06.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:05 smithi129 ceph-mon[33173]: pgmap v81: 105 pgs: 105 active+clean; 65 MiB data, 574 MiB used, 715 GiB / 715 GiB avail; 24 MiB/s rd, 4.0 MiB/s wr, 164 op/s 2024-09-14T04:00:06.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:06 smithi039 ceph-mon[106470]: osdmap e398: 8 total, 8 up, 8 in 2024-09-14T04:00:06.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:06 smithi039 ceph-mon[111235]: osdmap e398: 8 total, 8 up, 8 in 2024-09-14T04:00:07.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:06 smithi129 ceph-mon[33173]: osdmap e398: 8 total, 8 up, 8 in 2024-09-14T04:00:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:07 smithi129 ceph-mon[33173]: osdmap e399: 8 total, 8 up, 8 in 2024-09-14T04:00:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:07 smithi129 ceph-mon[33173]: pgmap v84: 105 pgs: 105 active+clean; 65 MiB data, 574 MiB used, 715 GiB / 715 GiB avail; 18 MiB/s rd, 4.0 MiB/s wr, 112 op/s 2024-09-14T04:00:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:07 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T04:00:08.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:07 smithi039 ceph-mon[106470]: osdmap e399: 8 total, 8 up, 8 in 2024-09-14T04:00:08.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:07 smithi039 ceph-mon[106470]: pgmap v84: 105 pgs: 105 active+clean; 65 MiB data, 574 MiB used, 715 GiB / 715 GiB avail; 18 MiB/s rd, 4.0 MiB/s wr, 112 op/s 2024-09-14T04:00:08.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T04:00:08.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:07 smithi039 ceph-mon[111235]: osdmap e399: 8 total, 8 up, 8 in 2024-09-14T04:00:08.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:07 smithi039 ceph-mon[111235]: pgmap v84: 105 pgs: 105 active+clean; 65 MiB data, 574 MiB used, 715 GiB / 715 GiB avail; 18 MiB/s rd, 4.0 MiB/s wr, 112 op/s 2024-09-14T04:00:08.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T04:00:09.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:08 smithi129 ceph-mon[33173]: osdmap e400: 8 total, 8 up, 8 in 2024-09-14T04:00:09.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:08 smithi039 ceph-mon[106470]: osdmap e400: 8 total, 8 up, 8 in 2024-09-14T04:00:09.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:08 smithi039 ceph-mon[111235]: osdmap e400: 8 total, 8 up, 8 in 2024-09-14T04:00:09.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:00:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:00:08] "GET /metrics HTTP/1.1" 200 36411 "" "Prometheus/2.51.0" 2024-09-14T04:00:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:09 smithi039 ceph-mon[106470]: osdmap e401: 8 total, 8 up, 8 in 2024-09-14T04:00:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:09 smithi039 ceph-mon[106470]: pgmap v87: 105 pgs: 105 active+clean; 65 MiB data, 575 MiB used, 715 GiB / 715 GiB avail; 393 KiB/s rd, 30 KiB/s wr, 528 op/s 2024-09-14T04:00:09.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:09 smithi039 ceph-mon[111235]: osdmap e401: 8 total, 8 up, 8 in 2024-09-14T04:00:09.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:09 smithi039 ceph-mon[111235]: pgmap v87: 105 pgs: 105 active+clean; 65 MiB data, 575 MiB used, 715 GiB / 715 GiB avail; 393 KiB/s rd, 30 KiB/s wr, 528 op/s 2024-09-14T04:00:10.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:09 smithi129 ceph-mon[33173]: osdmap e401: 8 total, 8 up, 8 in 2024-09-14T04:00:10.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:09 smithi129 ceph-mon[33173]: pgmap v87: 105 pgs: 105 active+clean; 65 MiB data, 575 MiB used, 715 GiB / 715 GiB avail; 393 KiB/s rd, 30 KiB/s wr, 528 op/s 2024-09-14T04:00:11.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:10 smithi129 ceph-mon[33173]: osdmap e402: 8 total, 8 up, 8 in 2024-09-14T04:00:11.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:10 smithi039 ceph-mon[106470]: osdmap e402: 8 total, 8 up, 8 in 2024-09-14T04:00:11.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:10 smithi039 ceph-mon[111235]: osdmap e402: 8 total, 8 up, 8 in 2024-09-14T04:00:12.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:11 smithi129 ceph-mon[33173]: osdmap e403: 8 total, 8 up, 8 in 2024-09-14T04:00:12.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:11 smithi129 ceph-mon[33173]: pgmap v90: 105 pgs: 105 active+clean; 65 MiB data, 575 MiB used, 715 GiB / 715 GiB avail; 393 KiB/s rd, 30 KiB/s wr, 528 op/s 2024-09-14T04:00:12.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:11 smithi039 ceph-mon[106470]: osdmap e403: 8 total, 8 up, 8 in 2024-09-14T04:00:12.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:11 smithi039 ceph-mon[106470]: pgmap v90: 105 pgs: 105 active+clean; 65 MiB data, 575 MiB used, 715 GiB / 715 GiB avail; 393 KiB/s rd, 30 KiB/s wr, 528 op/s 2024-09-14T04:00:12.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:11 smithi039 ceph-mon[111235]: osdmap e403: 8 total, 8 up, 8 in 2024-09-14T04:00:12.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:11 smithi039 ceph-mon[111235]: pgmap v90: 105 pgs: 105 active+clean; 65 MiB data, 575 MiB used, 715 GiB / 715 GiB avail; 393 KiB/s rd, 30 KiB/s wr, 528 op/s 2024-09-14T04:00:12.762 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestDeepCopy.CloneHideParent (7010 ms) 2024-09-14T04:00:12.762 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestDeepCopy.Clone 2024-09-14T04:00:13.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:12 smithi129 ceph-mon[33173]: osdmap e404: 8 total, 8 up, 8 in 2024-09-14T04:00:13.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:12 smithi039 ceph-mon[106470]: osdmap e404: 8 total, 8 up, 8 in 2024-09-14T04:00:13.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:12 smithi039 ceph-mon[111235]: osdmap e404: 8 total, 8 up, 8 in 2024-09-14T04:00:13.922 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:13 smithi129 ceph-mon[33173]: osdmap e405: 8 total, 8 up, 8 in 2024-09-14T04:00:13.922 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:13 smithi129 ceph-mon[33173]: pgmap v93: 105 pgs: 105 active+clean; 65 MiB data, 580 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s rd, 5.7 KiB/s wr, 99 op/s 2024-09-14T04:00:14.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:13 smithi039 ceph-mon[106470]: osdmap e405: 8 total, 8 up, 8 in 2024-09-14T04:00:14.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:13 smithi039 ceph-mon[106470]: pgmap v93: 105 pgs: 105 active+clean; 65 MiB data, 580 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s rd, 5.7 KiB/s wr, 99 op/s 2024-09-14T04:00:14.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:13 smithi039 ceph-mon[111235]: osdmap e405: 8 total, 8 up, 8 in 2024-09-14T04:00:14.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:13 smithi039 ceph-mon[111235]: pgmap v93: 105 pgs: 105 active+clean; 65 MiB data, 580 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s rd, 5.7 KiB/s wr, 99 op/s 2024-09-14T04:00:14.941 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:14 smithi129 ceph-mon[33173]: osdmap e406: 8 total, 8 up, 8 in 2024-09-14T04:00:15.009 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestDeepCopy.Clone (2247 ms) 2024-09-14T04:00:15.009 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestDeepCopy.CloneFlatten 2024-09-14T04:00:15.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:14 smithi039 ceph-mon[106470]: osdmap e406: 8 total, 8 up, 8 in 2024-09-14T04:00:15.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:14 smithi039 ceph-mon[111235]: osdmap e406: 8 total, 8 up, 8 in 2024-09-14T04:00:16.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:15 smithi129 ceph-mon[33173]: osdmap e407: 8 total, 8 up, 8 in 2024-09-14T04:00:16.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:15 smithi129 ceph-mon[33173]: pgmap v96: 105 pgs: 105 active+clean; 65 MiB data, 580 MiB used, 715 GiB / 715 GiB avail; 99 KiB/s rd, 10 KiB/s wr, 145 op/s 2024-09-14T04:00:16.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:15 smithi039 ceph-mon[106470]: osdmap e407: 8 total, 8 up, 8 in 2024-09-14T04:00:16.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:15 smithi039 ceph-mon[106470]: pgmap v96: 105 pgs: 105 active+clean; 65 MiB data, 580 MiB used, 715 GiB / 715 GiB avail; 99 KiB/s rd, 10 KiB/s wr, 145 op/s 2024-09-14T04:00:16.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:15 smithi039 ceph-mon[111235]: osdmap e407: 8 total, 8 up, 8 in 2024-09-14T04:00:16.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:15 smithi039 ceph-mon[111235]: pgmap v96: 105 pgs: 105 active+clean; 65 MiB data, 580 MiB used, 715 GiB / 715 GiB avail; 99 KiB/s rd, 10 KiB/s wr, 145 op/s 2024-09-14T04:00:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:16 smithi129 ceph-mon[33173]: osdmap e408: 8 total, 8 up, 8 in 2024-09-14T04:00:17.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:16 smithi039 ceph-mon[106470]: osdmap e408: 8 total, 8 up, 8 in 2024-09-14T04:00:17.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:16 smithi039 ceph-mon[111235]: osdmap e408: 8 total, 8 up, 8 in 2024-09-14T04:00:17.147 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestDeepCopy.CloneFlatten (2138 ms) 2024-09-14T04:00:17.148 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestDeepCopy.Stress 2024-09-14T04:00:17.171 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 32390982~4023757 2024-09-14T04:00:17.192 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 39168389~3151868 2024-09-14T04:00:17.192 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 41017192~1602008 2024-09-14T04:00:17.197 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 14902819~3607489 2024-09-14T04:00:17.198 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 77514206~3380918 2024-09-14T04:00:17.208 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 45141625~5431187 2024-09-14T04:00:17.211 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 11050633~8055673 2024-09-14T04:00:17.237 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 14436270~7250594 2024-09-14T04:00:17.240 INFO:tasks.workunit.client.0.smithi039.stdout:snap: snap0 2024-09-14T04:00:17.714 INFO:tasks.workunit.client.0.smithi039.stdout:clone image234 -> image235 2024-09-14T04:00:17.763 INFO:tasks.workunit.client.0.smithi039.stdout:write: B 60867162~2671134 2024-09-14T04:00:17.773 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 27865854~6149118 2024-09-14T04:00:17.786 INFO:tasks.workunit.client.0.smithi039.stdout:write: B 5222560~3815403 2024-09-14T04:00:17.803 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 60263453~7869464 2024-09-14T04:00:17.806 INFO:tasks.workunit.client.0.smithi039.stdout:write: B 43740379~5104178 2024-09-14T04:00:17.823 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 27199276~5303468 2024-09-14T04:00:17.826 INFO:tasks.workunit.client.0.smithi039.stdout:write: B 9327225~7591105 2024-09-14T04:00:17.846 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 27520189~870002 2024-09-14T04:00:17.847 INFO:tasks.workunit.client.0.smithi039.stdout:snap: snap1 2024-09-14T04:00:18.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:17 smithi129 ceph-mon[33173]: osdmap e409: 8 total, 8 up, 8 in 2024-09-14T04:00:18.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:17 smithi129 ceph-mon[33173]: pgmap v99: 105 pgs: 105 active+clean; 65 MiB data, 580 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 4.2 KiB/s wr, 46 op/s 2024-09-14T04:00:18.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:17 smithi039 ceph-mon[106470]: osdmap e409: 8 total, 8 up, 8 in 2024-09-14T04:00:18.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:17 smithi039 ceph-mon[106470]: pgmap v99: 105 pgs: 105 active+clean; 65 MiB data, 580 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 4.2 KiB/s wr, 46 op/s 2024-09-14T04:00:18.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:17 smithi039 ceph-mon[111235]: osdmap e409: 8 total, 8 up, 8 in 2024-09-14T04:00:18.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:17 smithi039 ceph-mon[111235]: pgmap v99: 105 pgs: 105 active+clean; 65 MiB data, 580 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 4.2 KiB/s wr, 46 op/s 2024-09-14T04:00:18.724 INFO:tasks.workunit.client.0.smithi039.stdout:clone image235 -> image236 2024-09-14T04:00:18.789 INFO:tasks.workunit.client.0.smithi039.stdout:write: C 60491711~1428518 2024-09-14T04:00:18.801 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 13198044~597613 2024-09-14T04:00:18.828 INFO:tasks.workunit.client.0.smithi039.stdout:write: C 56360448~3788190 2024-09-14T04:00:18.837 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 45398295~8008021 2024-09-14T04:00:18.856 INFO:tasks.workunit.client.0.smithi039.stdout:write: C 48813047~302217 2024-09-14T04:00:18.859 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 41333025~1114309 2024-09-14T04:00:18.873 INFO:tasks.workunit.client.0.smithi039.stdout:write: C 37339876~2271485 2024-09-14T04:00:18.888 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 50363643~1536891 2024-09-14T04:00:18.889 INFO:tasks.workunit.client.0.smithi039.stdout:snap: snap2 2024-09-14T04:00:19.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:18 smithi129 ceph-mon[33173]: osdmap e410: 8 total, 8 up, 8 in 2024-09-14T04:00:19.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:18 smithi039 ceph-mon[106470]: osdmap e410: 8 total, 8 up, 8 in 2024-09-14T04:00:19.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:18 smithi039 ceph-mon[111235]: osdmap e410: 8 total, 8 up, 8 in 2024-09-14T04:00:19.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:00:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:00:18] "GET /metrics HTTP/1.1" 200 36411 "" "Prometheus/2.51.0" 2024-09-14T04:00:19.740 INFO:tasks.workunit.client.0.smithi039.stdout:clone image236 -> image237 2024-09-14T04:00:19.824 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 139472283 2024-09-14T04:00:19.830 INFO:tasks.workunit.client.0.smithi039.stdout:write: D 101197282~5966330 2024-09-14T04:00:19.855 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 137502157~1970126 2024-09-14T04:00:19.855 INFO:tasks.workunit.client.0.smithi039.stdout:write: D 106693976~3810352 2024-09-14T04:00:19.865 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 129471138~6966167 2024-09-14T04:00:19.866 INFO:tasks.workunit.client.0.smithi039.stdout:write: D 22776896~4873580 2024-09-14T04:00:19.890 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 18774851~2912935 2024-09-14T04:00:19.901 INFO:tasks.workunit.client.0.smithi039.stdout:write: D 25609210~2360443 2024-09-14T04:00:19.911 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 59435229~7943203 2024-09-14T04:00:19.925 INFO:tasks.workunit.client.0.smithi039.stdout:snap: snap3 2024-09-14T04:00:20.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:19 smithi129 ceph-mon[33173]: osdmap e411: 8 total, 8 up, 8 in 2024-09-14T04:00:20.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:19 smithi129 ceph-mon[33173]: pgmap v102: 105 pgs: 105 active+clean; 233 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 46 MiB/s rd, 40 MiB/s wr, 601 op/s 2024-09-14T04:00:20.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:19 smithi039 ceph-mon[106470]: osdmap e411: 8 total, 8 up, 8 in 2024-09-14T04:00:20.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:19 smithi039 ceph-mon[106470]: pgmap v102: 105 pgs: 105 active+clean; 233 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 46 MiB/s rd, 40 MiB/s wr, 601 op/s 2024-09-14T04:00:20.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:19 smithi039 ceph-mon[111235]: osdmap e411: 8 total, 8 up, 8 in 2024-09-14T04:00:20.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:19 smithi039 ceph-mon[111235]: pgmap v102: 105 pgs: 105 active+clean; 233 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 46 MiB/s rd, 40 MiB/s wr, 601 op/s 2024-09-14T04:00:21.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:20 smithi129 ceph-mon[33173]: osdmap e412: 8 total, 8 up, 8 in 2024-09-14T04:00:21.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:20 smithi039 ceph-mon[106470]: osdmap e412: 8 total, 8 up, 8 in 2024-09-14T04:00:21.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:20 smithi039 ceph-mon[111235]: osdmap e412: 8 total, 8 up, 8 in 2024-09-14T04:00:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:21 smithi129 ceph-mon[33173]: osdmap e413: 8 total, 8 up, 8 in 2024-09-14T04:00:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:21 smithi129 ceph-mon[33173]: pgmap v105: 105 pgs: 105 active+clean; 233 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 46 MiB/s rd, 40 MiB/s wr, 601 op/s 2024-09-14T04:00:22.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:21 smithi039 ceph-mon[106470]: osdmap e413: 8 total, 8 up, 8 in 2024-09-14T04:00:22.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:21 smithi039 ceph-mon[106470]: pgmap v105: 105 pgs: 105 active+clean; 233 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 46 MiB/s rd, 40 MiB/s wr, 601 op/s 2024-09-14T04:00:22.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:21 smithi039 ceph-mon[111235]: osdmap e413: 8 total, 8 up, 8 in 2024-09-14T04:00:22.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:21 smithi039 ceph-mon[111235]: pgmap v105: 105 pgs: 105 active+clean; 233 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 46 MiB/s rd, 40 MiB/s wr, 601 op/s 2024-09-14T04:00:22.163 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestDeepCopy.Stress (5016 ms) 2024-09-14T04:00:22.163 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestDeepCopy.NoSnaps_LargerDstObjSize 2024-09-14T04:00:22.424 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestDeepCopy.NoSnaps_LargerDstObjSize (260 ms) 2024-09-14T04:00:22.424 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestDeepCopy.Snaps_LargerDstObjSize 2024-09-14T04:00:23.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:22 smithi129 ceph-mon[33173]: osdmap e414: 8 total, 8 up, 8 in 2024-09-14T04:00:23.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:22 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T04:00:23.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:22 smithi039 ceph-mon[106470]: osdmap e414: 8 total, 8 up, 8 in 2024-09-14T04:00:23.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T04:00:23.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:22 smithi039 ceph-mon[111235]: osdmap e414: 8 total, 8 up, 8 in 2024-09-14T04:00:23.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T04:00:24.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:23 smithi129 ceph-mon[33173]: pgmap v107: 105 pgs: 105 active+clean; 301 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 28 MiB/s rd, 16 MiB/s wr, 403 op/s 2024-09-14T04:00:24.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:23 smithi129 ceph-mon[33173]: osdmap e415: 8 total, 8 up, 8 in 2024-09-14T04:00:24.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:23 smithi039 ceph-mon[106470]: pgmap v107: 105 pgs: 105 active+clean; 301 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 28 MiB/s rd, 16 MiB/s wr, 403 op/s 2024-09-14T04:00:24.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:23 smithi039 ceph-mon[106470]: osdmap e415: 8 total, 8 up, 8 in 2024-09-14T04:00:24.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:23 smithi039 ceph-mon[111235]: pgmap v107: 105 pgs: 105 active+clean; 301 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 28 MiB/s rd, 16 MiB/s wr, 403 op/s 2024-09-14T04:00:24.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:23 smithi039 ceph-mon[111235]: osdmap e415: 8 total, 8 up, 8 in 2024-09-14T04:00:25.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:24 smithi039 ceph-mon[106470]: osdmap e416: 8 total, 8 up, 8 in 2024-09-14T04:00:25.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:24 smithi039 ceph-mon[111235]: osdmap e416: 8 total, 8 up, 8 in 2024-09-14T04:00:25.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:24 smithi129 ceph-mon[33173]: osdmap e416: 8 total, 8 up, 8 in 2024-09-14T04:00:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:25 smithi129 ceph-mon[33173]: pgmap v110: 105 pgs: 105 active+clean; 360 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 61 MiB/s rd, 29 MiB/s wr, 562 op/s 2024-09-14T04:00:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:25 smithi129 ceph-mon[33173]: osdmap e417: 8 total, 8 up, 8 in 2024-09-14T04:00:26.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:25 smithi039 ceph-mon[106470]: pgmap v110: 105 pgs: 105 active+clean; 360 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 61 MiB/s rd, 29 MiB/s wr, 562 op/s 2024-09-14T04:00:26.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:25 smithi039 ceph-mon[106470]: osdmap e417: 8 total, 8 up, 8 in 2024-09-14T04:00:26.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:25 smithi039 ceph-mon[111235]: pgmap v110: 105 pgs: 105 active+clean; 360 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 61 MiB/s rd, 29 MiB/s wr, 562 op/s 2024-09-14T04:00:26.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:25 smithi039 ceph-mon[111235]: osdmap e417: 8 total, 8 up, 8 in 2024-09-14T04:00:26.102 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestDeepCopy.Snaps_LargerDstObjSize (3679 ms) 2024-09-14T04:00:26.102 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestDeepCopy.Clone_LargerDstObjSize 2024-09-14T04:00:27.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:26 smithi129 ceph-mon[33173]: osdmap e418: 8 total, 8 up, 8 in 2024-09-14T04:00:27.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:26 smithi039 ceph-mon[106470]: osdmap e418: 8 total, 8 up, 8 in 2024-09-14T04:00:27.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:26 smithi039 ceph-mon[111235]: osdmap e418: 8 total, 8 up, 8 in 2024-09-14T04:00:28.178 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestDeepCopy.Clone_LargerDstObjSize (2076 ms) 2024-09-14T04:00:28.178 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestDeepCopy.CloneFlatten_LargerDstObjSize 2024-09-14T04:00:28.191 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:27 smithi039 ceph-mon[106470]: pgmap v113: 105 pgs: 105 active+clean; 360 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 32 MiB/s rd, 13 MiB/s wr, 156 op/s 2024-09-14T04:00:28.191 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:27 smithi039 ceph-mon[106470]: osdmap e419: 8 total, 8 up, 8 in 2024-09-14T04:00:28.191 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:27 smithi039 ceph-mon[111235]: pgmap v113: 105 pgs: 105 active+clean; 360 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 32 MiB/s rd, 13 MiB/s wr, 156 op/s 2024-09-14T04:00:28.191 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:27 smithi039 ceph-mon[111235]: osdmap e419: 8 total, 8 up, 8 in 2024-09-14T04:00:28.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:27 smithi129 ceph-mon[33173]: pgmap v113: 105 pgs: 105 active+clean; 360 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 32 MiB/s rd, 13 MiB/s wr, 156 op/s 2024-09-14T04:00:28.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:27 smithi129 ceph-mon[33173]: osdmap e419: 8 total, 8 up, 8 in 2024-09-14T04:00:28.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:00:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:00:28] "GET /metrics HTTP/1.1" 200 36419 "" "Prometheus/2.51.0" 2024-09-14T04:00:29.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:28 smithi129 ceph-mon[33173]: osdmap e420: 8 total, 8 up, 8 in 2024-09-14T04:00:29.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:28 smithi039 ceph-mon[106470]: osdmap e420: 8 total, 8 up, 8 in 2024-09-14T04:00:29.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:28 smithi039 ceph-mon[111235]: osdmap e420: 8 total, 8 up, 8 in 2024-09-14T04:00:30.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:29 smithi129 ceph-mon[33173]: pgmap v116: 105 pgs: 105 active+clean; 404 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 MiB/s rd, 11 MiB/s wr, 192 op/s 2024-09-14T04:00:30.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:29 smithi129 ceph-mon[33173]: osdmap e421: 8 total, 8 up, 8 in 2024-09-14T04:00:30.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:29 smithi039 ceph-mon[106470]: pgmap v116: 105 pgs: 105 active+clean; 404 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 MiB/s rd, 11 MiB/s wr, 192 op/s 2024-09-14T04:00:30.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:29 smithi039 ceph-mon[106470]: osdmap e421: 8 total, 8 up, 8 in 2024-09-14T04:00:30.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:29 smithi039 ceph-mon[111235]: pgmap v116: 105 pgs: 105 active+clean; 404 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 MiB/s rd, 11 MiB/s wr, 192 op/s 2024-09-14T04:00:30.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:29 smithi039 ceph-mon[111235]: osdmap e421: 8 total, 8 up, 8 in 2024-09-14T04:00:30.406 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestDeepCopy.CloneFlatten_LargerDstObjSize (2228 ms) 2024-09-14T04:00:30.406 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestDeepCopy.Stress_LargerDstObjSize 2024-09-14T04:00:30.432 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 4653537~5270691 2024-09-14T04:00:30.450 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 15930784~5275598 2024-09-14T04:00:30.450 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 48755530~50373 2024-09-14T04:00:30.452 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 56867495~7574171 2024-09-14T04:00:30.453 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 71548983~3776177 2024-09-14T04:00:30.466 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 19060277~4113569 2024-09-14T04:00:30.467 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 62208887~6410245 2024-09-14T04:00:30.487 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 35579647~6093413 2024-09-14T04:00:30.488 INFO:tasks.workunit.client.0.smithi039.stdout:snap: snap0 2024-09-14T04:00:30.867 INFO:tasks.workunit.client.0.smithi039.stdout:clone image256 -> image257 2024-09-14T04:00:30.913 INFO:tasks.workunit.client.0.smithi039.stdout:write: B 53329764~6898795 2024-09-14T04:00:30.942 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 31021601~5366709 2024-09-14T04:00:30.946 INFO:tasks.workunit.client.0.smithi039.stdout:write: B 35216923~5358559 2024-09-14T04:00:30.960 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 71248694~5410372 2024-09-14T04:00:30.971 INFO:tasks.workunit.client.0.smithi039.stdout:write: B 52936808~1746464 2024-09-14T04:00:30.976 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 64005947~2540514 2024-09-14T04:00:30.993 INFO:tasks.workunit.client.0.smithi039.stdout:write: B 66531461~102166 2024-09-14T04:00:30.994 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 38932196~1550620 2024-09-14T04:00:30.996 INFO:tasks.workunit.client.0.smithi039.stdout:snap: snap1 2024-09-14T04:00:31.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:30 smithi129 ceph-mon[33173]: osdmap e422: 8 total, 8 up, 8 in 2024-09-14T04:00:31.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:30 smithi039 ceph-mon[111235]: osdmap e422: 8 total, 8 up, 8 in 2024-09-14T04:00:31.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:30 smithi039 ceph-mon[106470]: osdmap e422: 8 total, 8 up, 8 in 2024-09-14T04:00:31.887 INFO:tasks.workunit.client.0.smithi039.stdout:clone image257 -> image258 2024-09-14T04:00:31.952 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 121242158 2024-09-14T04:00:31.959 INFO:tasks.workunit.client.0.smithi039.stdout:write: C 4460128~6150336 2024-09-14T04:00:31.987 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 113664335~7577823 2024-09-14T04:00:31.988 INFO:tasks.workunit.client.0.smithi039.stdout:write: C 36722274~3861802 2024-09-14T04:00:32.007 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 10491697~223484 2024-09-14T04:00:32.009 INFO:tasks.workunit.client.0.smithi039.stdout:write: C 86134779~7565689 2024-09-14T04:00:32.036 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 27706929~1028016 2024-09-14T04:00:32.039 INFO:tasks.workunit.client.0.smithi039.stdout:write: C 118450314~758886 2024-09-14T04:00:32.044 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 23516813~6997564 2024-09-14T04:00:32.048 INFO:tasks.workunit.client.0.smithi039.stdout:snap: snap2 2024-09-14T04:00:32.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:31 smithi129 ceph-mon[33173]: pgmap v119: 105 pgs: 105 active+clean; 404 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 MiB/s rd, 11 MiB/s wr, 192 op/s 2024-09-14T04:00:32.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:31 smithi129 ceph-mon[33173]: osdmap e423: 8 total, 8 up, 8 in 2024-09-14T04:00:32.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:31 smithi039 ceph-mon[106470]: pgmap v119: 105 pgs: 105 active+clean; 404 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 MiB/s rd, 11 MiB/s wr, 192 op/s 2024-09-14T04:00:32.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:31 smithi039 ceph-mon[106470]: osdmap e423: 8 total, 8 up, 8 in 2024-09-14T04:00:32.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:31 smithi039 ceph-mon[111235]: pgmap v119: 105 pgs: 105 active+clean; 404 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 25 MiB/s rd, 11 MiB/s wr, 192 op/s 2024-09-14T04:00:32.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:31 smithi039 ceph-mon[111235]: osdmap e423: 8 total, 8 up, 8 in 2024-09-14T04:00:32.910 INFO:tasks.workunit.client.0.smithi039.stdout:clone image258 -> image259 2024-09-14T04:00:32.993 INFO:tasks.workunit.client.0.smithi039.stdout:write: D 19682115~3636428 2024-09-14T04:00:33.012 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 42151968~2017912 2024-09-14T04:00:33.017 INFO:tasks.workunit.client.0.smithi039.stdout:write: D 101148923~7676427 2024-09-14T04:00:33.041 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 78794850~6343754 2024-09-14T04:00:33.056 INFO:tasks.workunit.client.0.smithi039.stdout:write: D 110153801~465616 2024-09-14T04:00:33.059 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 21199773~5675189 2024-09-14T04:00:33.062 INFO:tasks.workunit.client.0.smithi039.stdout:write: D 59265778~7134557 2024-09-14T04:00:33.100 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 81946179~1814842 2024-09-14T04:00:33.101 INFO:tasks.workunit.client.0.smithi039.stdout:snap: snap3 2024-09-14T04:00:33.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:32 smithi129 ceph-mon[33173]: osdmap e424: 8 total, 8 up, 8 in 2024-09-14T04:00:33.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:32 smithi039 ceph-mon[106470]: osdmap e424: 8 total, 8 up, 8 in 2024-09-14T04:00:33.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:32 smithi039 ceph-mon[111235]: osdmap e424: 8 total, 8 up, 8 in 2024-09-14T04:00:33.915 INFO:tasks.workunit.client.0.smithi039.stdout:clone image259 -> image260 2024-09-14T04:00:34.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:33 smithi129 ceph-mon[33173]: pgmap v122: 105 pgs: 105 active+clean; 598 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 48 MiB/s rd, 43 MiB/s wr, 411 op/s 2024-09-14T04:00:34.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:33 smithi129 ceph-mon[33173]: osdmap e425: 8 total, 8 up, 8 in 2024-09-14T04:00:34.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:33 smithi039 ceph-mon[106470]: pgmap v122: 105 pgs: 105 active+clean; 598 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 48 MiB/s rd, 43 MiB/s wr, 411 op/s 2024-09-14T04:00:34.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:33 smithi039 ceph-mon[106470]: osdmap e425: 8 total, 8 up, 8 in 2024-09-14T04:00:34.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:33 smithi039 ceph-mon[111235]: pgmap v122: 105 pgs: 105 active+clean; 598 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 48 MiB/s rd, 43 MiB/s wr, 411 op/s 2024-09-14T04:00:34.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:33 smithi039 ceph-mon[111235]: osdmap e425: 8 total, 8 up, 8 in 2024-09-14T04:00:34.310 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestDeepCopy.Stress_LargerDstObjSize (3904 ms) 2024-09-14T04:00:34.310 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestDeepCopy.NoSnaps_SmallerDstObjSize 2024-09-14T04:00:34.542 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestDeepCopy.NoSnaps_SmallerDstObjSize (231 ms) 2024-09-14T04:00:34.542 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestDeepCopy.Snaps_SmallerDstObjSize 2024-09-14T04:00:35.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:34 smithi129 ceph-mon[33173]: osdmap e426: 8 total, 8 up, 8 in 2024-09-14T04:00:35.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:34 smithi039 ceph-mon[106470]: osdmap e426: 8 total, 8 up, 8 in 2024-09-14T04:00:35.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:34 smithi039 ceph-mon[111235]: osdmap e426: 8 total, 8 up, 8 in 2024-09-14T04:00:36.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:35 smithi129 ceph-mon[33173]: pgmap v125: 105 pgs: 105 active+clean; 612 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 51 MiB/s rd, 46 MiB/s wr, 632 op/s 2024-09-14T04:00:36.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:35 smithi129 ceph-mon[33173]: osdmap e427: 8 total, 8 up, 8 in 2024-09-14T04:00:36.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:35 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T04:00:36.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:35 smithi039 ceph-mon[106470]: pgmap v125: 105 pgs: 105 active+clean; 612 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 51 MiB/s rd, 46 MiB/s wr, 632 op/s 2024-09-14T04:00:36.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:35 smithi039 ceph-mon[106470]: osdmap e427: 8 total, 8 up, 8 in 2024-09-14T04:00:36.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:35 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T04:00:36.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:35 smithi039 ceph-mon[111235]: pgmap v125: 105 pgs: 105 active+clean; 612 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 51 MiB/s rd, 46 MiB/s wr, 632 op/s 2024-09-14T04:00:36.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:35 smithi039 ceph-mon[111235]: osdmap e427: 8 total, 8 up, 8 in 2024-09-14T04:00:36.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:35 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T04:00:37.213 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:36 smithi129 ceph-mon[33173]: osdmap e428: 8 total, 8 up, 8 in 2024-09-14T04:00:37.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:36 smithi039 ceph-mon[106470]: osdmap e428: 8 total, 8 up, 8 in 2024-09-14T04:00:37.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:36 smithi039 ceph-mon[111235]: osdmap e428: 8 total, 8 up, 8 in 2024-09-14T04:00:38.183 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestDeepCopy.Snaps_SmallerDstObjSize (3642 ms) 2024-09-14T04:00:38.184 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestDeepCopy.Clone_SmallerDstObjSize 2024-09-14T04:00:38.193 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:37 smithi129 ceph-mon[33173]: pgmap v128: 105 pgs: 105 active+clean; 612 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.1 MiB/s wr, 220 op/s 2024-09-14T04:00:38.193 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:37 smithi129 ceph-mon[33173]: osdmap e429: 8 total, 8 up, 8 in 2024-09-14T04:00:38.193 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:37 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T04:00:38.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:37 smithi039 ceph-mon[106470]: pgmap v128: 105 pgs: 105 active+clean; 612 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.1 MiB/s wr, 220 op/s 2024-09-14T04:00:38.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:37 smithi039 ceph-mon[106470]: osdmap e429: 8 total, 8 up, 8 in 2024-09-14T04:00:38.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T04:00:38.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:37 smithi039 ceph-mon[111235]: pgmap v128: 105 pgs: 105 active+clean; 612 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 3.1 MiB/s wr, 220 op/s 2024-09-14T04:00:38.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:37 smithi039 ceph-mon[111235]: osdmap e429: 8 total, 8 up, 8 in 2024-09-14T04:00:38.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T04:00:38.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:00:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:00:38] "GET /metrics HTTP/1.1" 200 36424 "" "Prometheus/2.51.0" 2024-09-14T04:00:39.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:38 smithi129 ceph-mon[33173]: osdmap e430: 8 total, 8 up, 8 in 2024-09-14T04:00:39.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:38 smithi039 ceph-mon[106470]: osdmap e430: 8 total, 8 up, 8 in 2024-09-14T04:00:39.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:38 smithi039 ceph-mon[111235]: osdmap e430: 8 total, 8 up, 8 in 2024-09-14T04:00:40.265 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestDeepCopy.Clone_SmallerDstObjSize (2081 ms) 2024-09-14T04:00:40.265 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestDeepCopy.CloneFlatten_SmallerDstObjSize 2024-09-14T04:00:40.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:39 smithi129 ceph-mon[33173]: pgmap v131: 105 pgs: 105 active+clean; 688 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 MiB/s rd, 17 MiB/s wr, 549 op/s 2024-09-14T04:00:40.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:39 smithi129 ceph-mon[33173]: mgrmap e41: y(active, since 92s), standbys: x 2024-09-14T04:00:40.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:39 smithi129 ceph-mon[33173]: osdmap e431: 8 total, 8 up, 8 in 2024-09-14T04:00:40.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:39 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:00:40.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:39 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:00:40.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:39 smithi039 ceph-mon[111235]: pgmap v131: 105 pgs: 105 active+clean; 688 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 MiB/s rd, 17 MiB/s wr, 549 op/s 2024-09-14T04:00:40.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:39 smithi039 ceph-mon[111235]: mgrmap e41: y(active, since 92s), standbys: x 2024-09-14T04:00:40.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:39 smithi039 ceph-mon[111235]: osdmap e431: 8 total, 8 up, 8 in 2024-09-14T04:00:40.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:39 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:00:40.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:39 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:00:40.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:39 smithi039 ceph-mon[106470]: pgmap v131: 105 pgs: 105 active+clean; 688 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 MiB/s rd, 17 MiB/s wr, 549 op/s 2024-09-14T04:00:40.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:39 smithi039 ceph-mon[106470]: mgrmap e41: y(active, since 92s), standbys: x 2024-09-14T04:00:40.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:39 smithi039 ceph-mon[106470]: osdmap e431: 8 total, 8 up, 8 in 2024-09-14T04:00:40.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:39 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:00:40.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:39 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:00:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:40 smithi129 ceph-mon[33173]: osdmap e432: 8 total, 8 up, 8 in 2024-09-14T04:00:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:40 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:00:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:40 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:00:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:40 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T04:00:41.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:40 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T04:00:41.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:40 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:00:41.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:40 smithi039 ceph-mon[106470]: osdmap e432: 8 total, 8 up, 8 in 2024-09-14T04:00:41.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:40 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:00:41.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:40 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:00:41.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:40 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T04:00:41.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:40 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T04:00:41.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:40 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:00:41.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:40 smithi039 ceph-mon[111235]: osdmap e432: 8 total, 8 up, 8 in 2024-09-14T04:00:41.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:40 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:00:41.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:40 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:00:41.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:40 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T04:00:41.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:40 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T04:00:41.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:40 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:00:42.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:41 smithi129 ceph-mon[33173]: pgmap v134: 105 pgs: 105 active+clean; 688 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 MiB/s rd, 17 MiB/s wr, 549 op/s 2024-09-14T04:00:42.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:41 smithi129 ceph-mon[33173]: osdmap e433: 8 total, 8 up, 8 in 2024-09-14T04:00:42.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:41 smithi039 ceph-mon[106470]: pgmap v134: 105 pgs: 105 active+clean; 688 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 MiB/s rd, 17 MiB/s wr, 549 op/s 2024-09-14T04:00:42.293 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:41 smithi039 ceph-mon[106470]: osdmap e433: 8 total, 8 up, 8 in 2024-09-14T04:00:42.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:41 smithi039 ceph-mon[111235]: pgmap v134: 105 pgs: 105 active+clean; 688 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 35 MiB/s rd, 17 MiB/s wr, 549 op/s 2024-09-14T04:00:42.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:41 smithi039 ceph-mon[111235]: osdmap e433: 8 total, 8 up, 8 in 2024-09-14T04:00:42.454 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestDeepCopy.CloneFlatten_SmallerDstObjSize (2190 ms) 2024-09-14T04:00:42.454 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestDeepCopy.Stress_SmallerDstObjSize 2024-09-14T04:00:42.477 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 76912949~6973131 2024-09-14T04:00:42.501 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 67324242~5666486 2024-09-14T04:00:42.502 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 18081226~481228 2024-09-14T04:00:42.505 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 32261715~7212943 2024-09-14T04:00:42.505 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 15872999~2300094 2024-09-14T04:00:42.512 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 24213223~2343711 2024-09-14T04:00:42.512 INFO:tasks.workunit.client.0.smithi039.stdout:write: A 42139732~1113695 2024-09-14T04:00:42.517 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 33583122~6723827 2024-09-14T04:00:42.517 INFO:tasks.workunit.client.0.smithi039.stdout:snap: snap0 2024-09-14T04:00:43.001 INFO:tasks.workunit.client.0.smithi039.stdout:clone image279 -> image280 2024-09-14T04:00:43.048 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 134712034 2024-09-14T04:00:43.054 INFO:tasks.workunit.client.0.smithi039.stdout:write: B 131995493~1484749 2024-09-14T04:00:43.061 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 52030671~2560916 2024-09-14T04:00:43.064 INFO:tasks.workunit.client.0.smithi039.stdout:write: B 100095994~6157743 2024-09-14T04:00:43.088 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 8830672~6554472 2024-09-14T04:00:43.095 INFO:tasks.workunit.client.0.smithi039.stdout:write: B 47410647~1663794 2024-09-14T04:00:43.101 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 86728201~881542 2024-09-14T04:00:43.102 INFO:tasks.workunit.client.0.smithi039.stdout:write: B 19343730~4546436 2024-09-14T04:00:43.121 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 88153589~3580141 2024-09-14T04:00:43.121 INFO:tasks.workunit.client.0.smithi039.stdout:snap: snap1 2024-09-14T04:00:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:42 smithi129 ceph-mon[33173]: osdmap e434: 8 total, 8 up, 8 in 2024-09-14T04:00:43.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:42 smithi039 ceph-mon[106470]: osdmap e434: 8 total, 8 up, 8 in 2024-09-14T04:00:43.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:42 smithi039 ceph-mon[111235]: osdmap e434: 8 total, 8 up, 8 in 2024-09-14T04:00:44.010 INFO:tasks.workunit.client.0.smithi039.stdout:write: C 41735435~3586150 2024-09-14T04:00:44.013 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 101368005~7184246 2024-09-14T04:00:44.014 INFO:tasks.workunit.client.0.smithi039.stdout:write: C 132575927~2136107 2024-09-14T04:00:44.014 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 91147345~3795628 2024-09-14T04:00:44.015 INFO:tasks.workunit.client.0.smithi039.stdout:write: C 50201518~4566864 2024-09-14T04:00:44.015 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 79072602~484637 2024-09-14T04:00:44.017 INFO:tasks.workunit.client.0.smithi039.stdout:write: C 65618109~4318625 2024-09-14T04:00:44.017 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 62139655~4333325 2024-09-14T04:00:44.065 INFO:tasks.workunit.client.0.smithi039.stdout:snap: snap2 2024-09-14T04:00:44.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:43 smithi129 ceph-mon[33173]: pgmap v137: 105 pgs: 105 active+clean; 863 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 70 MiB/s rd, 43 MiB/s wr, 701 op/s 2024-09-14T04:00:44.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:43 smithi129 ceph-mon[33173]: osdmap e435: 8 total, 8 up, 8 in 2024-09-14T04:00:44.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:43 smithi039 ceph-mon[106470]: pgmap v137: 105 pgs: 105 active+clean; 863 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 70 MiB/s rd, 43 MiB/s wr, 701 op/s 2024-09-14T04:00:44.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:43 smithi039 ceph-mon[106470]: osdmap e435: 8 total, 8 up, 8 in 2024-09-14T04:00:44.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:43 smithi039 ceph-mon[111235]: pgmap v137: 105 pgs: 105 active+clean; 863 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 70 MiB/s rd, 43 MiB/s wr, 701 op/s 2024-09-14T04:00:44.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:43 smithi039 ceph-mon[111235]: osdmap e435: 8 total, 8 up, 8 in 2024-09-14T04:00:45.034 INFO:tasks.workunit.client.0.smithi039.stdout:clone image280 -> image281 2024-09-14T04:00:45.104 INFO:tasks.workunit.client.0.smithi039.stdout:write: D 119274786~2866691 2024-09-14T04:00:45.118 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 110122247~5510360 2024-09-14T04:00:45.122 INFO:tasks.workunit.client.0.smithi039.stdout:write: D 19214671~6327863 2024-09-14T04:00:45.148 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 106100230~2444140 2024-09-14T04:00:45.152 INFO:tasks.workunit.client.0.smithi039.stdout:write: D 112110350~6995787 2024-09-14T04:00:45.173 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 4620757~7247507 2024-09-14T04:00:45.177 INFO:tasks.workunit.client.0.smithi039.stdout:write: D 119885561~2519826 2024-09-14T04:00:45.185 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 121707218~7873294 2024-09-14T04:00:45.187 INFO:tasks.workunit.client.0.smithi039.stdout:snap: snap3 2024-09-14T04:00:45.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:45 smithi129 ceph-mon[33173]: osdmap e436: 8 total, 8 up, 8 in 2024-09-14T04:00:45.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:45 smithi039 ceph-mon[106470]: osdmap e436: 8 total, 8 up, 8 in 2024-09-14T04:00:45.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:45 smithi039 ceph-mon[111235]: osdmap e436: 8 total, 8 up, 8 in 2024-09-14T04:00:46.037 INFO:tasks.workunit.client.0.smithi039.stdout:clone image281 -> image282 2024-09-14T04:00:46.124 INFO:tasks.workunit.client.0.smithi039.stdout:resize: 212319249 2024-09-14T04:00:46.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:46 smithi129 ceph-mon[33173]: pgmap v140: 105 pgs: 105 active+clean; 863 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 70 MiB/s rd, 43 MiB/s wr, 701 op/s 2024-09-14T04:00:46.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:46 smithi129 ceph-mon[33173]: osdmap e437: 8 total, 8 up, 8 in 2024-09-14T04:00:46.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:46 smithi039 ceph-mon[106470]: pgmap v140: 105 pgs: 105 active+clean; 863 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 70 MiB/s rd, 43 MiB/s wr, 701 op/s 2024-09-14T04:00:46.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:46 smithi039 ceph-mon[106470]: osdmap e437: 8 total, 8 up, 8 in 2024-09-14T04:00:46.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:46 smithi039 ceph-mon[111235]: pgmap v140: 105 pgs: 105 active+clean; 863 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 70 MiB/s rd, 43 MiB/s wr, 701 op/s 2024-09-14T04:00:46.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:46 smithi039 ceph-mon[111235]: osdmap e437: 8 total, 8 up, 8 in 2024-09-14T04:00:46.423 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestDeepCopy.Stress_SmallerDstObjSize (3969 ms) 2024-09-14T04:00:46.424 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestDeepCopy.NoSnaps_StrippingLargerDstObjSize 2024-09-14T04:00:46.448 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/test_DeepCopy.cc:621: Skipped 2024-09-14T04:00:46.448 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T04:00:46.521 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestDeepCopy.NoSnaps_StrippingLargerDstObjSize (98 ms) 2024-09-14T04:00:46.521 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestDeepCopy.Snaps_StrippingLargerDstObjSize 2024-09-14T04:00:46.545 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/test_DeepCopy.cc:635: Skipped 2024-09-14T04:00:46.546 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T04:00:46.625 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestDeepCopy.Snaps_StrippingLargerDstObjSize (103 ms) 2024-09-14T04:00:46.625 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestDeepCopy.Clone_StrippingLargerDstObjSize 2024-09-14T04:00:46.647 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/test_DeepCopy.cc:649: Skipped 2024-09-14T04:00:46.647 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T04:00:46.719 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestDeepCopy.Clone_StrippingLargerDstObjSize (94 ms) 2024-09-14T04:00:46.719 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestDeepCopy.CloneFlatten_StrippingLargerDstObjSize 2024-09-14T04:00:46.740 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/test_DeepCopy.cc:663: Skipped 2024-09-14T04:00:46.740 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T04:00:46.809 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestDeepCopy.CloneFlatten_StrippingLargerDstObjSize (90 ms) 2024-09-14T04:00:46.809 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestDeepCopy.Stress_StrippingLargerDstObjSize 2024-09-14T04:00:46.831 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/test_DeepCopy.cc:679: Skipped 2024-09-14T04:00:46.832 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T04:00:46.902 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestDeepCopy.Stress_StrippingLargerDstObjSize (94 ms) 2024-09-14T04:00:46.902 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestDeepCopy.NoSnaps_StrippingSmallerDstObjSize 2024-09-14T04:00:46.926 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/test_DeepCopy.cc:693: Skipped 2024-09-14T04:00:46.926 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T04:00:46.996 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestDeepCopy.NoSnaps_StrippingSmallerDstObjSize (93 ms) 2024-09-14T04:00:46.996 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestDeepCopy.Snaps_StrippingSmallerDstObjSize 2024-09-14T04:00:47.018 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/test_DeepCopy.cc:707: Skipped 2024-09-14T04:00:47.019 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T04:00:47.099 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestDeepCopy.Snaps_StrippingSmallerDstObjSize (103 ms) 2024-09-14T04:00:47.099 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestDeepCopy.Clone_StrippingSmallerDstObjSize 2024-09-14T04:00:47.121 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/test_DeepCopy.cc:721: Skipped 2024-09-14T04:00:47.121 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T04:00:47.199 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestDeepCopy.Clone_StrippingSmallerDstObjSize (101 ms) 2024-09-14T04:00:47.200 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestDeepCopy.CloneFlatten_StrippingSmallerDstObjSize 2024-09-14T04:00:47.224 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/test_DeepCopy.cc:735: Skipped 2024-09-14T04:00:47.225 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T04:00:47.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:47 smithi039 ceph-mon[106470]: osdmap e438: 8 total, 8 up, 8 in 2024-09-14T04:00:47.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:47 smithi039 ceph-mon[111235]: osdmap e438: 8 total, 8 up, 8 in 2024-09-14T04:00:47.301 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestDeepCopy.CloneFlatten_StrippingSmallerDstObjSize (102 ms) 2024-09-14T04:00:47.301 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestDeepCopy.Stress_StrippingSmallerDstObjSize 2024-09-14T04:00:47.324 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/test_DeepCopy.cc:751: Skipped 2024-09-14T04:00:47.324 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T04:00:47.395 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestDeepCopy.Stress_StrippingSmallerDstObjSize (93 ms) 2024-09-14T04:00:47.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:47 smithi129 ceph-mon[33173]: osdmap e438: 8 total, 8 up, 8 in 2024-09-14T04:00:48.039 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 31 tests from TestDeepCopy (53018 ms total) 2024-09-14T04:00:48.039 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:00:48.039 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 51 tests from TestMirroring 2024-09-14T04:00:48.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:48 smithi039 ceph-mon[106470]: pgmap v143: 105 pgs: 105 active+clean; 863 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:00:48.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:48 smithi039 ceph-mon[111235]: pgmap v143: 105 pgs: 105 active+clean; 863 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:00:48.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:48 smithi129 ceph-mon[33173]: pgmap v143: 105 pgs: 105 active+clean; 863 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:00:49.038 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:00:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:00:48] "GET /metrics HTTP/1.1" 200 36424 "" "Prometheus/2.51.0" 2024-09-14T04:00:49.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:49 smithi039 ceph-mon[111235]: osdmap e439: 8 total, 8 up, 8 in 2024-09-14T04:00:49.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:49 smithi039 ceph-mon[106470]: osdmap e439: 8 total, 8 up, 8 in 2024-09-14T04:00:49.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:49 smithi129 ceph-mon[33173]: osdmap e439: 8 total, 8 up, 8 in 2024-09-14T04:00:50.071 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeImage 2024-09-14T04:00:50.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:50 smithi129 ceph-mon[33173]: pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:00:50.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:50 smithi129 ceph-mon[33173]: osdmap e440: 8 total, 8 up, 8 in 2024-09-14T04:00:50.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:50 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:00:50.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:50 smithi039 ceph-mon[106470]: pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:00:50.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:50 smithi039 ceph-mon[106470]: osdmap e440: 8 total, 8 up, 8 in 2024-09-14T04:00:50.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:50 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:00:50.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:50 smithi039 ceph-mon[111235]: pgmap v145: 73 pgs: 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:00:50.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:50 smithi039 ceph-mon[111235]: osdmap e440: 8 total, 8 up, 8 in 2024-09-14T04:00:50.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:50 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-12","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:00:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:51 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:00:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:51 smithi129 ceph-mon[33173]: osdmap e441: 8 total, 8 up, 8 in 2024-09-14T04:00:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:51 smithi129 ceph-mon[33173]: pgmap v148: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:00:51.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:51 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:00:51.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:51 smithi039 ceph-mon[106470]: osdmap e441: 8 total, 8 up, 8 in 2024-09-14T04:00:51.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:51 smithi039 ceph-mon[106470]: pgmap v148: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:00:51.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:51 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-12","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:00:51.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:51 smithi039 ceph-mon[111235]: osdmap e441: 8 total, 8 up, 8 in 2024-09-14T04:00:51.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:51 smithi039 ceph-mon[111235]: pgmap v148: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.9 GiB used, 712 GiB / 715 GiB avail 2024-09-14T04:00:52.202 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeImage (2131 ms) 2024-09-14T04:00:52.202 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModePool 2024-09-14T04:00:52.306 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModePool (104 ms) 2024-09-14T04:00:52.307 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled 2024-09-14T04:00:52.384 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled (77 ms) 2024-09-14T04:00:52.384 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModeImage 2024-09-14T04:00:52.485 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModeImage (102 ms) 2024-09-14T04:00:52.486 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModeImage_NoObjectMap 2024-09-14T04:00:52.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:52 smithi129 ceph-mon[33173]: osdmap e442: 8 total, 8 up, 8 in 2024-09-14T04:00:52.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:52 smithi039 ceph-mon[106470]: osdmap e442: 8 total, 8 up, 8 in 2024-09-14T04:00:52.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:52 smithi039 ceph-mon[111235]: osdmap e442: 8 total, 8 up, 8 in 2024-09-14T04:00:52.578 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModeImage_NoObjectMap (92 ms) 2024-09-14T04:00:52.578 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModePool 2024-09-14T04:00:52.670 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModePool (93 ms) 2024-09-14T04:00:52.670 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.DisableImageMirror_In_MirrorModeDisabled 2024-09-14T04:00:52.776 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.DisableImageMirror_In_MirrorModeDisabled (105 ms) 2024-09-14T04:00:52.776 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.DisableImageMirrorWithPeer 2024-09-14T04:00:52.776 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/test_mirroring.cc:482: Skipped 2024-09-14T04:00:52.776 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T04:00:52.776 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestMirroring.DisableImageMirrorWithPeer (0 ms) 2024-09-14T04:00:52.776 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.DisableJournalingWithPeer 2024-09-14T04:00:52.776 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/test_mirroring.cc:520: Skipped 2024-09-14T04:00:52.776 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T04:00:52.776 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestMirroring.DisableJournalingWithPeer (0 ms) 2024-09-14T04:00:52.776 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled_WithoutJournaling 2024-09-14T04:00:52.843 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeDisabled_WithoutJournaling (68 ms) 2024-09-14T04:00:52.843 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModePool_WithoutJournaling 2024-09-14T04:00:52.917 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModePool_WithoutJournaling (74 ms) 2024-09-14T04:00:52.918 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutJournaling 2024-09-14T04:00:53.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:53 smithi129 ceph-mon[33173]: osdmap e443: 8 total, 8 up, 8 in 2024-09-14T04:00:53.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:53 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T04:00:53.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:53 smithi129 ceph-mon[33173]: pgmap v151: 105 pgs: 105 active+clean; 582 KiB data, 474 MiB used, 715 GiB / 715 GiB avail; 286 KiB/s rd, 43 KiB/s wr, 413 op/s 2024-09-14T04:00:53.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:53 smithi039 ceph-mon[106470]: osdmap e443: 8 total, 8 up, 8 in 2024-09-14T04:00:53.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:53 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T04:00:53.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:53 smithi039 ceph-mon[106470]: pgmap v151: 105 pgs: 105 active+clean; 582 KiB data, 474 MiB used, 715 GiB / 715 GiB avail; 286 KiB/s rd, 43 KiB/s wr, 413 op/s 2024-09-14T04:00:53.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:53 smithi039 ceph-mon[111235]: osdmap e443: 8 total, 8 up, 8 in 2024-09-14T04:00:53.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:53 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T04:00:53.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:53 smithi039 ceph-mon[111235]: pgmap v151: 105 pgs: 105 active+clean; 582 KiB data, 474 MiB used, 715 GiB / 715 GiB avail; 286 KiB/s rd, 43 KiB/s wr, 413 op/s 2024-09-14T04:00:54.165 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutJournaling (1248 ms) 2024-09-14T04:00:54.165 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutExclusiveLock 2024-09-14T04:00:54.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:54 smithi129 ceph-mon[33173]: osdmap e444: 8 total, 8 up, 8 in 2024-09-14T04:00:54.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:54 smithi039 ceph-mon[106470]: osdmap e444: 8 total, 8 up, 8 in 2024-09-14T04:00:54.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:54 smithi039 ceph-mon[111235]: osdmap e444: 8 total, 8 up, 8 in 2024-09-14T04:00:55.420 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.EnableImageMirror_In_MirrorModeImage_WithoutExclusiveLock (1255 ms) 2024-09-14T04:00:55.421 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModeDisabled 2024-09-14T04:00:55.511 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModeDisabled (90 ms) 2024-09-14T04:00:55.511 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModeImage 2024-09-14T04:00:55.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:55 smithi129 ceph-mon[33173]: osdmap e445: 8 total, 8 up, 8 in 2024-09-14T04:00:55.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:55 smithi129 ceph-mon[33173]: pgmap v154: 105 pgs: 105 active+clean; 582 KiB data, 474 MiB used, 715 GiB / 715 GiB avail; 286 KiB/s rd, 43 KiB/s wr, 413 op/s 2024-09-14T04:00:55.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:55 smithi039 ceph-mon[106470]: osdmap e445: 8 total, 8 up, 8 in 2024-09-14T04:00:55.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:55 smithi039 ceph-mon[106470]: pgmap v154: 105 pgs: 105 active+clean; 582 KiB data, 474 MiB used, 715 GiB / 715 GiB avail; 286 KiB/s rd, 43 KiB/s wr, 413 op/s 2024-09-14T04:00:55.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:55 smithi039 ceph-mon[111235]: osdmap e445: 8 total, 8 up, 8 in 2024-09-14T04:00:55.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:55 smithi039 ceph-mon[111235]: pgmap v154: 105 pgs: 105 active+clean; 582 KiB data, 474 MiB used, 715 GiB / 715 GiB avail; 286 KiB/s rd, 43 KiB/s wr, 413 op/s 2024-09-14T04:00:55.602 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModeImage (91 ms) 2024-09-14T04:00:55.602 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModePool 2024-09-14T04:00:55.702 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModePool (101 ms) 2024-09-14T04:00:55.703 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModePool_WithoutJournaling 2024-09-14T04:00:55.778 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModePool_WithoutJournaling (75 ms) 2024-09-14T04:00:55.778 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.CreateImage_In_MirrorModeImage_WithoutJournaling 2024-09-14T04:00:55.856 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.CreateImage_In_MirrorModeImage_WithoutJournaling (79 ms) 2024-09-14T04:00:55.856 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModeDisabled 2024-09-14T04:00:55.946 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModeDisabled (90 ms) 2024-09-14T04:00:55.947 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModeImage 2024-09-14T04:00:56.047 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModeImage (101 ms) 2024-09-14T04:00:56.048 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModeImage_SnapshotMirroringEnabled 2024-09-14T04:00:56.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:56 smithi129 ceph-mon[33173]: osdmap e446: 8 total, 8 up, 8 in 2024-09-14T04:00:56.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:56 smithi129 ceph-mon[33173]: osdmap e447: 8 total, 8 up, 8 in 2024-09-14T04:00:56.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:56 smithi039 ceph-mon[106470]: osdmap e446: 8 total, 8 up, 8 in 2024-09-14T04:00:56.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:56 smithi039 ceph-mon[106470]: osdmap e447: 8 total, 8 up, 8 in 2024-09-14T04:00:56.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:56 smithi039 ceph-mon[111235]: osdmap e446: 8 total, 8 up, 8 in 2024-09-14T04:00:56.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:56 smithi039 ceph-mon[111235]: osdmap e447: 8 total, 8 up, 8 in 2024-09-14T04:00:57.442 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModeImage_SnapshotMirroringEnabled (1395 ms) 2024-09-14T04:00:57.442 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.EnableJournaling_In_MirrorModePool 2024-09-14T04:00:57.557 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.EnableJournaling_In_MirrorModePool (114 ms) 2024-09-14T04:00:57.557 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.DisableJournaling_In_MirrorModePool 2024-09-14T04:00:57.659 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.DisableJournaling_In_MirrorModePool (102 ms) 2024-09-14T04:00:57.659 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.DisableJournaling_In_MirrorModeImage 2024-09-14T04:00:57.756 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.DisableJournaling_In_MirrorModeImage (98 ms) 2024-09-14T04:00:57.756 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.MirrorModeSet_DisabledMode_To_PoolMode 2024-09-14T04:00:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:57 smithi129 ceph-mon[33173]: osdmap e448: 8 total, 8 up, 8 in 2024-09-14T04:00:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:57 smithi129 ceph-mon[33173]: pgmap v158: 105 pgs: 105 active+clean; 582 KiB data, 474 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:00:57.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:57 smithi039 ceph-mon[106470]: osdmap e448: 8 total, 8 up, 8 in 2024-09-14T04:00:57.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:57 smithi039 ceph-mon[106470]: pgmap v158: 105 pgs: 105 active+clean; 582 KiB data, 474 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:00:57.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:57 smithi039 ceph-mon[111235]: osdmap e448: 8 total, 8 up, 8 in 2024-09-14T04:00:57.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:57 smithi039 ceph-mon[111235]: pgmap v158: 105 pgs: 105 active+clean; 582 KiB data, 474 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:00:57.904 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.MirrorModeSet_DisabledMode_To_PoolMode (148 ms) 2024-09-14T04:00:57.904 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.MirrorModeSet_PoolMode_To_DisabledMode 2024-09-14T04:00:58.059 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.MirrorModeSet_PoolMode_To_DisabledMode (154 ms) 2024-09-14T04:00:58.059 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.MirrorModeSet_DisabledMode_To_ImageMode 2024-09-14T04:00:58.194 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.MirrorModeSet_DisabledMode_To_ImageMode (135 ms) 2024-09-14T04:00:58.194 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.MirrorModeSet_PoolMode_To_ImageMode 2024-09-14T04:00:58.331 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.MirrorModeSet_PoolMode_To_ImageMode (137 ms) 2024-09-14T04:00:58.331 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.MirrorModeSet_ImageMode_To_PoolMode 2024-09-14T04:00:58.485 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.MirrorModeSet_ImageMode_To_PoolMode (155 ms) 2024-09-14T04:00:58.486 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.MirrorModeSet_ImageMode_To_DisabledMode 2024-09-14T04:00:58.647 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:58 smithi039 ceph-mon[106470]: osdmap e449: 8 total, 8 up, 8 in 2024-09-14T04:00:58.647 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:58 smithi039 ceph-mon[111235]: osdmap e449: 8 total, 8 up, 8 in 2024-09-14T04:00:58.648 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.MirrorModeSet_ImageMode_To_DisabledMode (163 ms) 2024-09-14T04:00:58.648 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.RemoveImage_With_MirrorImageEnabled 2024-09-14T04:00:58.732 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.RemoveImage_With_MirrorImageEnabled (83 ms) 2024-09-14T04:00:58.732 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.RemoveImage_With_MirrorImageDisabled 2024-09-14T04:00:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:58 smithi129 ceph-mon[33173]: osdmap e449: 8 total, 8 up, 8 in 2024-09-14T04:00:58.809 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.RemoveImage_With_MirrorImageDisabled (78 ms) 2024-09-14T04:00:58.810 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.RemoveImage_With_ImageWithoutJournal 2024-09-14T04:00:58.875 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.RemoveImage_With_ImageWithoutJournal (66 ms) 2024-09-14T04:00:58.875 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.RemoveImage_With_MirrorImageDemoted 2024-09-14T04:00:58.940 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:00:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:00:58] "GET /metrics HTTP/1.1" 200 36358 "" "Prometheus/2.51.0" 2024-09-14T04:00:58.998 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.RemoveImage_With_MirrorImageDemoted (122 ms) 2024-09-14T04:00:58.998 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.TrashMoveRestore_PoolMode 2024-09-14T04:00:59.124 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.TrashMoveRestore_PoolMode (126 ms) 2024-09-14T04:00:59.124 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.TrashMoveRestore_ImageMode_MirroringDisabled 2024-09-14T04:00:59.234 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.TrashMoveRestore_ImageMode_MirroringDisabled (111 ms) 2024-09-14T04:00:59.234 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.TrashMoveRestore_ImageMode_MirroringEnabled 2024-09-14T04:00:59.345 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.TrashMoveRestore_ImageMode_MirroringEnabled (111 ms) 2024-09-14T04:00:59.345 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.MirrorStatusList 2024-09-14T04:00:59.440 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.MirrorStatusList (95 ms) 2024-09-14T04:00:59.441 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.RemoveBootstrapped 2024-09-14T04:00:59.539 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.RemoveBootstrapped (99 ms) 2024-09-14T04:00:59.539 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.AioPromoteDemote 2024-09-14T04:00:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:00:59 smithi129 ceph-mon[33173]: pgmap v160: 105 pgs: 105 active+clean; 582 KiB data, 467 MiB used, 715 GiB / 715 GiB avail; 621 KiB/s rd, 76 KiB/s wr, 905 op/s 2024-09-14T04:00:59.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:00:59 smithi039 ceph-mon[111235]: pgmap v160: 105 pgs: 105 active+clean; 582 KiB data, 467 MiB used, 715 GiB / 715 GiB avail; 621 KiB/s rd, 76 KiB/s wr, 905 op/s 2024-09-14T04:00:59.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:00:59 smithi039 ceph-mon[106470]: pgmap v160: 105 pgs: 105 active+clean; 582 KiB data, 467 MiB used, 715 GiB / 715 GiB avail; 621 KiB/s rd, 76 KiB/s wr, 905 op/s 2024-09-14T04:00:59.905 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.AioPromoteDemote (365 ms) 2024-09-14T04:00:59.905 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.AioGetInfo 2024-09-14T04:01:00.249 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.AioGetInfo (345 ms) 2024-09-14T04:01:00.250 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.AioGetStatus 2024-09-14T04:01:00.473 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.AioGetStatus (223 ms) 2024-09-14T04:01:00.473 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.SiteName 2024-09-14T04:01:00.482 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.SiteName (10 ms) 2024-09-14T04:01:00.482 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.Bootstrap 2024-09-14T04:01:01.296 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.Bootstrap (814 ms) 2024-09-14T04:01:01.297 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.PeerDirection 2024-09-14T04:01:01.304 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.PeerDirection (8 ms) 2024-09-14T04:01:01.304 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.Snapshot 2024-09-14T04:01:01.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:01 smithi129 ceph-mon[33173]: osdmap e450: 8 total, 8 up, 8 in 2024-09-14T04:01:01.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:01 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' 2024-09-14T04:01:01.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:01 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-09-14T04:01:01.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:01 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/site_name"} : dispatch 2024-09-14T04:01:01.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:01 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-09-14T04:01:01.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:01 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-09-14T04:01:01.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:01 smithi129 ceph-mon[33173]: pgmap v162: 105 pgs: 105 active+clean; 582 KiB data, 467 MiB used, 715 GiB / 715 GiB avail; 461 KiB/s rd, 56 KiB/s wr, 672 op/s 2024-09-14T04:01:01.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:01 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"} : dispatch 2024-09-14T04:01:01.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:01 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' 2024-09-14T04:01:01.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:01 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"} : dispatch 2024-09-14T04:01:01.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:01 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-09-14T04:01:01.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:01 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-09-14T04:01:01.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:01 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/e60110c9-ddad-4dc2-8247-5156ce984fde"} : dispatch 2024-09-14T04:01:01.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:01 smithi129 ceph-mon[33173]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/e60110c9-ddad-4dc2-8247-5156ce984fde"}]': finished 2024-09-14T04:01:01.761 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:01 smithi039 ceph-mon[106470]: osdmap e450: 8 total, 8 up, 8 in 2024-09-14T04:01:01.761 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:01 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' 2024-09-14T04:01:01.761 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:01 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-09-14T04:01:01.762 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:01 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/site_name"} : dispatch 2024-09-14T04:01:01.762 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:01 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-09-14T04:01:01.762 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:01 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-09-14T04:01:01.762 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:01 smithi039 ceph-mon[106470]: pgmap v162: 105 pgs: 105 active+clean; 582 KiB data, 467 MiB used, 715 GiB / 715 GiB avail; 461 KiB/s rd, 56 KiB/s wr, 672 op/s 2024-09-14T04:01:01.762 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:01 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"} : dispatch 2024-09-14T04:01:01.762 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:01 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' 2024-09-14T04:01:01.762 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:01 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"} : dispatch 2024-09-14T04:01:01.762 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:01 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-09-14T04:01:01.762 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:01 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-09-14T04:01:01.762 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:01 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/e60110c9-ddad-4dc2-8247-5156ce984fde"} : dispatch 2024-09-14T04:01:01.762 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:01 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/e60110c9-ddad-4dc2-8247-5156ce984fde"}]': finished 2024-09-14T04:01:01.763 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:01 smithi039 ceph-mon[111235]: osdmap e450: 8 total, 8 up, 8 in 2024-09-14T04:01:01.763 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:01 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' 2024-09-14T04:01:01.763 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:01 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-09-14T04:01:01.763 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:01 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/site_name"} : dispatch 2024-09-14T04:01:01.763 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:01 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/site_name"}]': finished 2024-09-14T04:01:01.763 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:01 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-09-14T04:01:01.763 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:01 smithi039 ceph-mon[111235]: pgmap v162: 105 pgs: 105 active+clean; 582 KiB data, 467 MiB used, 715 GiB / 715 GiB avail; 461 KiB/s rd, 56 KiB/s wr, 672 op/s 2024-09-14T04:01:01.763 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:01 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer_client_id"} : dispatch 2024-09-14T04:01:01.763 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:01 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' 2024-09-14T04:01:01.763 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:01 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"} : dispatch 2024-09-14T04:01:01.763 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:01 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{ "prefix": "auth get-or-create", "entity": "client.rbd-mirror-peer", "caps": [ "mon", "profile rbd-mirror-peer", "osd", "profile rbd"], "format": "json"}]': finished 2024-09-14T04:01:01.763 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:01 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/site_name"} : dispatch 2024-09-14T04:01:01.763 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:01 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/e60110c9-ddad-4dc2-8247-5156ce984fde"} : dispatch 2024-09-14T04:01:01.764 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:01 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/e60110c9-ddad-4dc2-8247-5156ce984fde"}]': finished 2024-09-14T04:01:03.603 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:03 smithi039 ceph-mon[106470]: osdmap e451: 8 total, 8 up, 8 in 2024-09-14T04:01:03.604 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:03 smithi039 ceph-mon[106470]: pgmap v164: 105 pgs: 105 active+clean; 583 KiB data, 452 MiB used, 715 GiB / 715 GiB avail; 1.7 MiB/s rd, 243 KiB/s wr, 2.36k op/s 2024-09-14T04:01:03.604 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:03 smithi039 ceph-mon[111235]: osdmap e451: 8 total, 8 up, 8 in 2024-09-14T04:01:03.604 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:03 smithi039 ceph-mon[111235]: pgmap v164: 105 pgs: 105 active+clean; 583 KiB data, 452 MiB used, 715 GiB / 715 GiB avail; 1.7 MiB/s rd, 243 KiB/s wr, 2.36k op/s 2024-09-14T04:01:03.629 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'echo wait for majority of mons to upgrade' 2024-09-14T04:01:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:03 smithi129 ceph-mon[33173]: osdmap e451: 8 total, 8 up, 8 in 2024-09-14T04:01:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:03 smithi129 ceph-mon[33173]: pgmap v164: 105 pgs: 105 active+clean; 583 KiB data, 452 MiB used, 715 GiB / 715 GiB avail; 1.7 MiB/s rd, 243 KiB/s wr, 2.36k op/s 2024-09-14T04:01:03.842 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:01:04.672 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:04 smithi039 ceph-mon[106470]: osdmap e452: 8 total, 8 up, 8 in 2024-09-14T04:01:04.673 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:04 smithi039 ceph-mon[111235]: osdmap e452: 8 total, 8 up, 8 in 2024-09-14T04:01:04.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:04 smithi129 ceph-mon[33173]: osdmap e452: 8 total, 8 up, 8 in 2024-09-14T04:01:05.516 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:05 smithi039 ceph-mon[106470]: osdmap e453: 8 total, 8 up, 8 in 2024-09-14T04:01:05.517 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:05 smithi039 ceph-mon[106470]: pgmap v167: 105 pgs: 105 active+clean; 583 KiB data, 452 MiB used, 715 GiB / 715 GiB avail; 1.7 MiB/s rd, 263 KiB/s wr, 2.40k op/s 2024-09-14T04:01:05.517 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:05 smithi039 ceph-mon[111235]: osdmap e453: 8 total, 8 up, 8 in 2024-09-14T04:01:05.517 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:05 smithi039 ceph-mon[111235]: pgmap v167: 105 pgs: 105 active+clean; 583 KiB data, 452 MiB used, 715 GiB / 715 GiB avail; 1.7 MiB/s rd, 263 KiB/s wr, 2.40k op/s 2024-09-14T04:01:05.741 INFO:teuthology.orchestra.run.smithi039.stdout:wait for majority of mons to upgrade 2024-09-14T04:01:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:05 smithi129 ceph-mon[33173]: osdmap e453: 8 total, 8 up, 8 in 2024-09-14T04:01:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:05 smithi129 ceph-mon[33173]: pgmap v167: 105 pgs: 105 active+clean; 583 KiB data, 452 MiB used, 715 GiB / 715 GiB avail; 1.7 MiB/s rd, 263 KiB/s wr, 2.40k op/s 2024-09-14T04:01:06.439 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon --limit 1' 2024-09-14T04:01:06.651 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:01:06.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:06 smithi129 ceph-mon[33173]: osdmap e454: 8 total, 8 up, 8 in 2024-09-14T04:01:06.772 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:06 smithi039 ceph-mon[106470]: osdmap e454: 8 total, 8 up, 8 in 2024-09-14T04:01:06.772 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:06 smithi039 ceph-mon[111235]: osdmap e454: 8 total, 8 up, 8 in 2024-09-14T04:01:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:07 smithi129 ceph-mon[33173]: osdmap e455: 8 total, 8 up, 8 in 2024-09-14T04:01:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:07 smithi129 ceph-mon[33173]: pgmap v170: 105 pgs: 105 active+clean; 583 KiB data, 452 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:01:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:07 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T04:01:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:07 smithi039 ceph-mon[106470]: osdmap e455: 8 total, 8 up, 8 in 2024-09-14T04:01:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:07 smithi039 ceph-mon[106470]: pgmap v170: 105 pgs: 105 active+clean; 583 KiB data, 452 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:01:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T04:01:07.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:07 smithi039 ceph-mon[111235]: osdmap e455: 8 total, 8 up, 8 in 2024-09-14T04:01:07.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:07 smithi039 ceph-mon[111235]: pgmap v170: 105 pgs: 105 active+clean; 583 KiB data, 452 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:01:07.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-09-14T04:01:08.648 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:08 smithi039 ceph-mon[106470]: osdmap e456: 8 total, 8 up, 8 in 2024-09-14T04:01:08.649 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:08 smithi039 ceph-mon[111235]: osdmap e456: 8 total, 8 up, 8 in 2024-09-14T04:01:08.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:08 smithi129 ceph-mon[33173]: osdmap e456: 8 total, 8 up, 8 in 2024-09-14T04:01:09.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:01:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:01:08] "GET /metrics HTTP/1.1" 200 36405 "" "Prometheus/2.51.0" 2024-09-14T04:01:09.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:09 smithi039 ceph-mon[106470]: osdmap e457: 8 total, 8 up, 8 in 2024-09-14T04:01:09.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:09 smithi039 ceph-mon[106470]: from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:09.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:09 smithi039 ceph-mon[106470]: pgmap v173: 105 pgs: 105 active+clean; 583 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 6.0 KiB/s wr, 68 op/s 2024-09-14T04:01:09.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:09 smithi039 ceph-mon[111235]: osdmap e457: 8 total, 8 up, 8 in 2024-09-14T04:01:09.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:09 smithi039 ceph-mon[111235]: from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:09.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:09 smithi039 ceph-mon[111235]: pgmap v173: 105 pgs: 105 active+clean; 583 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 6.0 KiB/s wr, 68 op/s 2024-09-14T04:01:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:09 smithi129 ceph-mon[33173]: osdmap e457: 8 total, 8 up, 8 in 2024-09-14T04:01:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:09 smithi129 ceph-mon[33173]: from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "daemon_types": "mon", "limit": 1, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:09 smithi129 ceph-mon[33173]: pgmap v173: 105 pgs: 105 active+clean; 583 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 6.0 KiB/s wr, 68 op/s 2024-09-14T04:01:10.694 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:10 smithi039 ceph-mon[106470]: osdmap e458: 8 total, 8 up, 8 in 2024-09-14T04:01:10.695 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:10 smithi039 ceph-mon[106470]: osdmap e459: 8 total, 8 up, 8 in 2024-09-14T04:01:10.695 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:10 smithi039 ceph-mon[111235]: osdmap e458: 8 total, 8 up, 8 in 2024-09-14T04:01:10.695 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:10 smithi039 ceph-mon[111235]: osdmap e459: 8 total, 8 up, 8 in 2024-09-14T04:01:10.733 INFO:teuthology.orchestra.run.smithi039.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T04:01:10.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:10 smithi129 ceph-mon[33173]: osdmap e458: 8 total, 8 up, 8 in 2024-09-14T04:01:10.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:10 smithi129 ceph-mon[33173]: osdmap e459: 8 total, 8 up, 8 in 2024-09-14T04:01:11.639 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-14T04:01:11.870 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:11 smithi039 ceph-mon[106470]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T04:01:11.871 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:11 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:11.871 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:11 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T04:01:11.871 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:11 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T04:01:11.871 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:11 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T04:01:11.871 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:11 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:11.871 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:11 smithi039 ceph-mon[106470]: pgmap v176: 105 pgs: 105 active+clean; 583 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 6.0 KiB/s wr, 68 op/s 2024-09-14T04:01:11.871 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:11 smithi039 ceph-mon[106470]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T04:01:11.871 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:11 smithi039 ceph-mon[106470]: osdmap e460: 8 total, 8 up, 8 in 2024-09-14T04:01:11.871 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:11 smithi039 ceph-mon[111235]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T04:01:11.871 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:11 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:11.871 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:11 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T04:01:11.872 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:11 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T04:01:11.872 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:11 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T04:01:11.872 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:11 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:11.872 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:11 smithi039 ceph-mon[111235]: pgmap v176: 105 pgs: 105 active+clean; 583 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 6.0 KiB/s wr, 68 op/s 2024-09-14T04:01:11.872 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:11 smithi039 ceph-mon[111235]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T04:01:11.872 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:11 smithi039 ceph-mon[111235]: osdmap e460: 8 total, 8 up, 8 in 2024-09-14T04:01:12.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:11 smithi129 ceph-mon[33173]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T04:01:12.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:11 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:12.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:11 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T04:01:12.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:11 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T04:01:12.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:11 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-09-14T04:01:12.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:11 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:12.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:11 smithi129 ceph-mon[33173]: pgmap v176: 105 pgs: 105 active+clean; 583 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 6.0 KiB/s wr, 68 op/s 2024-09-14T04:01:12.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:11 smithi129 ceph-mon[33173]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T04:01:12.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:11 smithi129 ceph-mon[33173]: osdmap e460: 8 total, 8 up, 8 in 2024-09-14T04:01:12.421 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:01:13.465 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:13 smithi129 ceph-mon[33173]: osdmap e461: 8 total, 8 up, 8 in 2024-09-14T04:01:13.465 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:13 smithi129 ceph-mon[33173]: pgmap v179: 105 pgs: 105 active+clean; 583 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 6.5 KiB/s wr, 98 op/s 2024-09-14T04:01:13.465 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:13 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:13.465 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:13 smithi129 ceph-mon[33173]: Upgrade: Target is version 19.3.0-4907-ga2782315 (squid) 2024-09-14T04:01:13.466 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:13 smithi129 ceph-mon[33173]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T04:01:13.466 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:13 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T04:01:13.466 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:13 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T04:01:13.466 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:13 smithi129 ceph-mon[33173]: Upgrade: Setting container_image for all mgr 2024-09-14T04:01:13.466 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:13 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:13.466 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:13 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-14T04:01:13.466 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:13 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-14T04:01:13.466 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:13 smithi129 ceph-mon[33173]: Upgrade: It appears safe to stop mon.b 2024-09-14T04:01:13.684 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:13 smithi039 ceph-mon[106470]: osdmap e461: 8 total, 8 up, 8 in 2024-09-14T04:01:13.684 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:13 smithi039 ceph-mon[106470]: pgmap v179: 105 pgs: 105 active+clean; 583 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 6.5 KiB/s wr, 98 op/s 2024-09-14T04:01:13.685 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:13 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:13.685 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:13 smithi039 ceph-mon[106470]: Upgrade: Target is version 19.3.0-4907-ga2782315 (squid) 2024-09-14T04:01:13.685 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:13 smithi039 ceph-mon[106470]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T04:01:13.685 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:13 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T04:01:13.685 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:13 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T04:01:13.685 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:13 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all mgr 2024-09-14T04:01:13.685 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:13 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:13.685 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:13 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-14T04:01:13.685 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:13 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-14T04:01:13.685 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:13 smithi039 ceph-mon[106470]: Upgrade: It appears safe to stop mon.b 2024-09-14T04:01:13.685 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:13 smithi039 ceph-mon[111235]: osdmap e461: 8 total, 8 up, 8 in 2024-09-14T04:01:13.686 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:13 smithi039 ceph-mon[111235]: pgmap v179: 105 pgs: 105 active+clean; 583 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 6.5 KiB/s wr, 98 op/s 2024-09-14T04:01:13.686 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:13 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:13.686 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:13 smithi039 ceph-mon[111235]: Upgrade: Target is version 19.3.0-4907-ga2782315 (squid) 2024-09-14T04:01:13.686 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:13 smithi039 ceph-mon[111235]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T04:01:13.686 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:13 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-09-14T04:01:13.686 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:13 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-09-14T04:01:13.686 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:13 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all mgr 2024-09-14T04:01:13.686 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:13 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:13.686 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:13 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-09-14T04:01:13.686 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:13 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-09-14T04:01:13.686 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:13 smithi039 ceph-mon[111235]: Upgrade: It appears safe to stop mon.b 2024-09-14T04:01:14.624 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:14 smithi039 ceph-mon[111235]: osdmap e462: 8 total, 8 up, 8 in 2024-09-14T04:01:14.624 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:14 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:14.624 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:14 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T04:01:14.624 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:14 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T04:01:14.624 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:14 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T04:01:14.624 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:14 smithi039 ceph-mon[106470]: osdmap e462: 8 total, 8 up, 8 in 2024-09-14T04:01:14.624 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:14 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:14.625 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:14 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T04:01:14.625 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:14 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T04:01:14.625 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:14 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T04:01:14.667 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:14 smithi129 ceph-mon[33173]: osdmap e462: 8 total, 8 up, 8 in 2024-09-14T04:01:14.667 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:14 smithi129 ceph-mon[33173]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:14.667 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:14 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-09-14T04:01:14.667 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:14 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-09-14T04:01:14.667 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:14 smithi129 ceph-mon[33173]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-09-14T04:01:14.808 INFO:teuthology.orchestra.run.smithi039.stdout:true 2024-09-14T04:01:15.405 INFO:teuthology.orchestra.run.smithi039.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T04:01:15.405 INFO:teuthology.orchestra.run.smithi039.stdout:alertmanager.a smithi039 *:9093,9094 running (3m) 35s ago 9m 21.0M - 0.27.0 11f11916f8cd 6b564e1fc909 2024-09-14T04:01:15.405 INFO:teuthology.orchestra.run.smithi039.stdout:grafana.a smithi129 *:3000 running (3m) 35s ago 9m 81.0M - 10.4.0 c8b91775d855 f0492c19de92 2024-09-14T04:01:15.405 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi039.bbwiwf smithi039 running (7m) 35s ago 7m 22.5M - 18.2.4-845-ge4e227b3 67ddd33fa72f 91e6c8f85681 2024-09-14T04:01:15.405 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi129.ybjqcw smithi129 running (7m) 35s ago 7m 15.7M - 18.2.4-845-ge4e227b3 67ddd33fa72f a3fab5b37b71 2024-09-14T04:01:15.405 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.x smithi129 *:8443,9283,8765 running (4m) 35s ago 13m 454M - 19.3.0-4907-ga2782315 8801d2529e43 448bd4e146cb 2024-09-14T04:01:15.405 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.y smithi039 *:8443,9283,8765 running (4m) 35s ago 14m 512M - 19.3.0-4907-ga2782315 8801d2529e43 cccc87f635a6 2024-09-14T04:01:15.406 INFO:teuthology.orchestra.run.smithi039.stdout:mon.a smithi039 running (2m) 35s ago 15m 49.5M 2048M 19.3.0-4907-ga2782315 8801d2529e43 7198d4eb8e4c 2024-09-14T04:01:15.406 INFO:teuthology.orchestra.run.smithi039.stdout:mon.b smithi129 running (13m) 35s ago 13m 60.5M 2048M 18.2.4-845-ge4e227b3 67ddd33fa72f eb38cc5cbe35 2024-09-14T04:01:15.406 INFO:teuthology.orchestra.run.smithi039.stdout:mon.c smithi039 running (113s) 35s ago 13m 39.0M 2048M 19.3.0-4907-ga2782315 8801d2529e43 4b8f0f3864b1 2024-09-14T04:01:15.406 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.a smithi039 *:9100 running (4m) 35s ago 8m 11.4M - 1.7.0 72c9c2088986 7dd5945adb64 2024-09-14T04:01:15.406 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.b smithi129 *:9100 running (4m) 35s ago 8m 11.1M - 1.7.0 72c9c2088986 1fedb5f257e7 2024-09-14T04:01:15.406 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0 smithi039 running (12m) 35s ago 12m 115M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 5048d74f27ac 2024-09-14T04:01:15.406 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1 smithi039 running (12m) 35s ago 12m 130M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 91349527d88b 2024-09-14T04:01:15.406 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2 smithi039 running (12m) 35s ago 12m 164M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f eb903b44fc5d 2024-09-14T04:01:15.406 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3 smithi039 running (11m) 35s ago 11m 133M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f dff9e665bd9b 2024-09-14T04:01:15.406 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4 smithi129 running (11m) 35s ago 11m 135M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 5b64727311b7 2024-09-14T04:01:15.406 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5 smithi129 running (10m) 35s ago 10m 107M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f f2d991a754f7 2024-09-14T04:01:15.406 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6 smithi129 running (10m) 35s ago 10m 148M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 3060eaf44106 2024-09-14T04:01:15.406 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7 smithi129 running (10m) 35s ago 10m 116M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 4c13b97060d1 2024-09-14T04:01:15.407 INFO:teuthology.orchestra.run.smithi039.stdout:prometheus.a smithi129 *:9095 running (3m) 35s ago 9m 49.3M - 2.51.0 1d3b7f56885b cbe583f852c3 2024-09-14T04:01:15.701 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:15 smithi129 ceph-mon[33173]: Upgrade: Updating mon.b 2024-09-14T04:01:15.701 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:15 smithi129 ceph-mon[33173]: Deploying daemon mon.b on smithi129 2024-09-14T04:01:15.701 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:15 smithi129 ceph-mon[33173]: osdmap e463: 8 total, 8 up, 8 in 2024-09-14T04:01:15.701 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:15 smithi129 ceph-mon[33173]: pgmap v182: 105 pgs: 105 active+clean; 583 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 6.5 KiB/s wr, 98 op/s 2024-09-14T04:01:15.701 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:15 smithi129 ceph-mon[33173]: from='client.34227 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:15.701 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:15 smithi129 ceph-mon[33173]: osdmap e464: 8 total, 8 up, 8 in 2024-09-14T04:01:15.764 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:01:15.765 INFO:teuthology.orchestra.run.smithi039.stdout: "mon": { 2024-09-14T04:01:15.765 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 1, 2024-09-14T04:01:15.765 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T04:01:15.765 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:01:15.765 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr": { 2024-09-14T04:01:15.765 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T04:01:15.765 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:01:15.765 INFO:teuthology.orchestra.run.smithi039.stdout: "osd": { 2024-09-14T04:01:15.765 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8 2024-09-14T04:01:15.765 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:01:15.765 INFO:teuthology.orchestra.run.smithi039.stdout: "mds": { 2024-09-14T04:01:15.766 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T04:01:15.766 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:01:15.766 INFO:teuthology.orchestra.run.smithi039.stdout: "overall": { 2024-09-14T04:01:15.766 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 11, 2024-09-14T04:01:15.766 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 4 2024-09-14T04:01:15.766 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:01:15.766 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:01:15.766 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:15 smithi039 ceph-mon[111235]: Upgrade: Updating mon.b 2024-09-14T04:01:15.766 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:15 smithi039 ceph-mon[111235]: Deploying daemon mon.b on smithi129 2024-09-14T04:01:15.767 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:15 smithi039 ceph-mon[111235]: osdmap e463: 8 total, 8 up, 8 in 2024-09-14T04:01:15.767 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:15 smithi039 ceph-mon[111235]: pgmap v182: 105 pgs: 105 active+clean; 583 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 6.5 KiB/s wr, 98 op/s 2024-09-14T04:01:15.767 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:15 smithi039 ceph-mon[111235]: from='client.34227 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:15.767 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:15 smithi039 ceph-mon[111235]: osdmap e464: 8 total, 8 up, 8 in 2024-09-14T04:01:15.767 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:15 smithi039 ceph-mon[106470]: Upgrade: Updating mon.b 2024-09-14T04:01:15.767 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:15 smithi039 ceph-mon[106470]: Deploying daemon mon.b on smithi129 2024-09-14T04:01:15.767 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:15 smithi039 ceph-mon[106470]: osdmap e463: 8 total, 8 up, 8 in 2024-09-14T04:01:15.767 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:15 smithi039 ceph-mon[106470]: pgmap v182: 105 pgs: 105 active+clean; 583 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 6.5 KiB/s wr, 98 op/s 2024-09-14T04:01:15.767 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:15 smithi039 ceph-mon[106470]: from='client.34227 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:15.767 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:15 smithi039 ceph-mon[106470]: osdmap e464: 8 total, 8 up, 8 in 2024-09-14T04:01:16.063 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:01:16.063 INFO:teuthology.orchestra.run.smithi039.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T04:01:16.063 INFO:teuthology.orchestra.run.smithi039.stdout: "in_progress": true, 2024-09-14T04:01:16.063 INFO:teuthology.orchestra.run.smithi039.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (1 remaining).", 2024-09-14T04:01:16.064 INFO:teuthology.orchestra.run.smithi039.stdout: "services_complete": [], 2024-09-14T04:01:16.064 INFO:teuthology.orchestra.run.smithi039.stdout: "progress": "2/3 daemons upgraded", 2024-09-14T04:01:16.064 INFO:teuthology.orchestra.run.smithi039.stdout: "message": "Currently upgrading mon daemons", 2024-09-14T04:01:16.064 INFO:teuthology.orchestra.run.smithi039.stdout: "is_paused": false 2024-09-14T04:01:16.064 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:01:16.209 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:15 smithi129 systemd[1]: Stopping Ceph mon.b for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T04:01:16.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-b[33144]: 2024-09-14T04:01:16.206+0000 7fd5c4bb2640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:01:16.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-b[33144]: 2024-09-14T04:01:16.206+0000 7fd5c4bb2640 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-09-14T04:01:16.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:16 smithi129 podman[101119]: 2024-09-14 04:01:16.400310765 +0000 UTC m=+0.290360433 container died eb38cc5cbe359b7c7039cd34076d70d699b882baace6dba764e15f1167d8618d (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-b, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=reef-e4e227b, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux ) 2024-09-14T04:01:16.540 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:01:16 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:01:16] ENGINE Bus STOPPING 2024-09-14T04:01:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:16 smithi129 podman[101119]: 2024-09-14 04:01:16.779756915 +0000 UTC m=+0.669806588 container cleanup eb38cc5cbe359b7c7039cd34076d70d699b882baace6dba764e15f1167d8618d (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-b, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-e4e227b, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-14T04:01:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:16 smithi129 bash[101119]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-b 2024-09-14T04:01:17.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:01:16 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:01:16] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T04:01:17.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:01:16 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:01:16] ENGINE Bus STOPPED 2024-09-14T04:01:17.392 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 podman[101142]: 2024-09-14 04:01:17.047948342 +0000 UTC m=+0.649821388 container remove eb38cc5cbe359b7c7039cd34076d70d699b882baace6dba764e15f1167d8618d (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-b, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=reef-e4e227b) 2024-09-14T04:01:17.392 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mon.b.service: Deactivated successfully. 2024-09-14T04:01:17.392 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 systemd[1]: Stopped Ceph mon.b for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T04:01:17.392 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mon.b.service: Consumed 12.343s CPU time. 2024-09-14T04:01:17.392 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 systemd[1]: Starting Ceph mon.b for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T04:01:17.729 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 podman[101278]: 2024-09-14 04:01:17.501735684 +0000 UTC m=+0.015668419 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:01:17.729 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 podman[101278]: 2024-09-14 04:01:17.607353876 +0000 UTC m=+0.121286608 container create cff2519560f167e23398a4a0d0891211e49777809673ae8ad77db31e6701dbed (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-b, ceph=True, RELEASE=main-a278231, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T04:01:18.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 podman[101278]: 2024-09-14 04:01:17.843281484 +0000 UTC m=+0.357214225 container init cff2519560f167e23398a4a0d0891211e49777809673ae8ad77db31e6701dbed (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-b, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=main-a278231, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-14T04:01:18.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 podman[101278]: 2024-09-14 04:01:17.847797862 +0000 UTC m=+0.361730602 container start cff2519560f167e23398a4a0d0891211e49777809673ae8ad77db31e6701dbed (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-b, org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=main-a278231) 2024-09-14T04:01:18.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: set uid:gid to 167:167 (ceph:ceph) 2024-09-14T04:01:18.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev), process ceph-mon, pid 7 2024-09-14T04:01:18.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: pidfile_write: ignore empty --pid-file 2024-09-14T04:01:18.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: load: jerasure load: lrc 2024-09-14T04:01:18.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: RocksDB version: 7.9.2 2024-09-14T04:01:18.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Git sha 0 2024-09-14T04:01:18.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Compile date 2024-09-12 20:09:22 2024-09-14T04:01:18.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: DB SUMMARY 2024-09-14T04:01:18.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: DB Session ID: 5QDPM5NBSJPJQ3094IP7 2024-09-14T04:01:18.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: CURRENT file: CURRENT 2024-09-14T04:01:18.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: IDENTITY file: IDENTITY 2024-09-14T04:01:18.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: MANIFEST file: MANIFEST-000010 size: 1656 Bytes 2024-09-14T04:01:18.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000030.sst 2024-09-14T04:01:18.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000028.log size: 4849922 ; 2024-09-14T04:01:18.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.error_if_exists: 0 2024-09-14T04:01:18.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.create_if_missing: 0 2024-09-14T04:01:18.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.paranoid_checks: 1 2024-09-14T04:01:18.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.flush_verify_memtable_count: 1 2024-09-14T04:01:18.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-09-14T04:01:18.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-09-14T04:01:18.274 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.env: 0x5559982b0120 2024-09-14T04:01:18.274 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.fs: PosixFileSystem 2024-09-14T04:01:18.274 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.info_log: 0x555999953700 2024-09-14T04:01:18.274 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.max_file_opening_threads: 16 2024-09-14T04:01:18.274 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.statistics: (nil) 2024-09-14T04:01:18.274 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.use_fsync: 0 2024-09-14T04:01:18.274 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.max_log_file_size: 0 2024-09-14T04:01:18.274 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-09-14T04:01:18.274 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.log_file_time_to_roll: 0 2024-09-14T04:01:18.274 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.keep_log_file_num: 1000 2024-09-14T04:01:18.275 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.recycle_log_file_num: 0 2024-09-14T04:01:18.275 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.allow_fallocate: 1 2024-09-14T04:01:18.275 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.allow_mmap_reads: 0 2024-09-14T04:01:18.275 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.allow_mmap_writes: 0 2024-09-14T04:01:18.275 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.use_direct_reads: 0 2024-09-14T04:01:18.275 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-09-14T04:01:18.275 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.create_missing_column_families: 0 2024-09-14T04:01:18.275 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.db_log_dir: 2024-09-14T04:01:18.276 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.wal_dir: 2024-09-14T04:01:18.276 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.table_cache_numshardbits: 6 2024-09-14T04:01:18.276 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.WAL_ttl_seconds: 0 2024-09-14T04:01:18.276 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.WAL_size_limit_MB: 0 2024-09-14T04:01:18.276 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-09-14T04:01:18.276 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-09-14T04:01:18.276 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.is_fd_close_on_exec: 1 2024-09-14T04:01:18.276 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.advise_random_on_open: 1 2024-09-14T04:01:18.276 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.db_write_buffer_size: 0 2024-09-14T04:01:18.276 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.write_buffer_manager: 0x5559999561e0 2024-09-14T04:01:18.276 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-09-14T04:01:18.277 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-09-14T04:01:18.277 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.use_adaptive_mutex: 0 2024-09-14T04:01:18.277 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.rate_limiter: (nil) 2024-09-14T04:01:18.277 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-09-14T04:01:18.277 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.wal_recovery_mode: 2 2024-09-14T04:01:18.277 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.enable_thread_tracking: 0 2024-09-14T04:01:18.277 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.enable_pipelined_write: 0 2024-09-14T04:01:18.277 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.unordered_write: 0 2024-09-14T04:01:18.277 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-09-14T04:01:18.277 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-09-14T04:01:18.278 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-09-14T04:01:18.278 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-09-14T04:01:18.278 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.row_cache: None 2024-09-14T04:01:18.278 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.wal_filter: None 2024-09-14T04:01:18.282 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-09-14T04:01:18.282 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.allow_ingest_behind: 0 2024-09-14T04:01:18.282 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.two_write_queues: 0 2024-09-14T04:01:18.282 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.manual_wal_flush: 0 2024-09-14T04:01:18.282 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.wal_compression: 0 2024-09-14T04:01:18.282 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.atomic_flush: 0 2024-09-14T04:01:18.282 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-09-14T04:01:18.282 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.persist_stats_to_disk: 0 2024-09-14T04:01:18.283 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.write_dbid_to_manifest: 0 2024-09-14T04:01:18.283 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.log_readahead_size: 0 2024-09-14T04:01:18.283 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-09-14T04:01:18.283 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.best_efforts_recovery: 0 2024-09-14T04:01:18.283 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-09-14T04:01:18.283 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-09-14T04:01:18.283 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.allow_data_in_errors: 0 2024-09-14T04:01:18.283 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.db_host_id: __hostname__ 2024-09-14T04:01:18.283 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.enforce_single_del_contracts: true 2024-09-14T04:01:18.283 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.max_background_jobs: 2 2024-09-14T04:01:18.283 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.max_background_compactions: -1 2024-09-14T04:01:18.284 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.max_subcompactions: 1 2024-09-14T04:01:18.284 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-09-14T04:01:18.285 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-09-14T04:01:18.285 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.delayed_write_rate : 16777216 2024-09-14T04:01:18.285 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.max_total_wal_size: 0 2024-09-14T04:01:18.285 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-09-14T04:01:18.285 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.stats_dump_period_sec: 600 2024-09-14T04:01:18.285 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.stats_persist_period_sec: 600 2024-09-14T04:01:18.285 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-09-14T04:01:18.285 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.max_open_files: -1 2024-09-14T04:01:18.285 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.bytes_per_sync: 0 2024-09-14T04:01:18.285 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.wal_bytes_per_sync: 0 2024-09-14T04:01:18.285 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.strict_bytes_per_sync: 0 2024-09-14T04:01:18.285 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.compaction_readahead_size: 0 2024-09-14T04:01:18.285 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.max_background_flushes: -1 2024-09-14T04:01:18.286 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Compression algorithms supported: 2024-09-14T04:01:18.286 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: kZSTD supported: 0 2024-09-14T04:01:18.286 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: kXpressCompression supported: 0 2024-09-14T04:01:18.286 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: kBZip2Compression supported: 0 2024-09-14T04:01:18.286 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-09-14T04:01:18.286 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: kLZ4Compression supported: 1 2024-09-14T04:01:18.286 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: kZlibCompression supported: 1 2024-09-14T04:01:18.286 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: kLZ4HCCompression supported: 1 2024-09-14T04:01:18.286 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: kSnappyCompression supported: 1 2024-09-14T04:01:18.286 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Fast CRC32 supported: Supported on x86 2024-09-14T04:01:18.286 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: DMutex implementation: pthread_mutex_t 2024-09-14T04:01:18.286 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000010 2024-09-14T04:01:18.286 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-09-14T04:01:18.286 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-09-14T04:01:18.287 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.merge_operator: 2024-09-14T04:01:18.287 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.compaction_filter: None 2024-09-14T04:01:18.287 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.compaction_filter_factory: None 2024-09-14T04:01:18.287 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.sst_partitioner_factory: None 2024-09-14T04:01:18.287 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.memtable_factory: SkipListFactory 2024-09-14T04:01:18.287 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.table_factory: BlockBasedTable 2024-09-14T04:01:18.287 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x555999953320) 2024-09-14T04:01:18.287 INFO:journalctl@ceph.mon.b.smithi129.stdout: cache_index_and_filter_blocks: 1 2024-09-14T04:01:18.287 INFO:journalctl@ceph.mon.b.smithi129.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-09-14T04:01:18.287 INFO:journalctl@ceph.mon.b.smithi129.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-09-14T04:01:18.287 INFO:journalctl@ceph.mon.b.smithi129.stdout: pin_top_level_index_and_filter: 1 2024-09-14T04:01:18.287 INFO:journalctl@ceph.mon.b.smithi129.stdout: index_type: 0 2024-09-14T04:01:18.287 INFO:journalctl@ceph.mon.b.smithi129.stdout: data_block_index_type: 0 2024-09-14T04:01:18.287 INFO:journalctl@ceph.mon.b.smithi129.stdout: index_shortening: 1 2024-09-14T04:01:18.288 INFO:journalctl@ceph.mon.b.smithi129.stdout: data_block_hash_table_util_ratio: 0.750000 2024-09-14T04:01:18.288 INFO:journalctl@ceph.mon.b.smithi129.stdout: checksum: 4 2024-09-14T04:01:18.288 INFO:journalctl@ceph.mon.b.smithi129.stdout: no_block_cache: 0 2024-09-14T04:01:18.288 INFO:journalctl@ceph.mon.b.smithi129.stdout: block_cache: 0x555999947610 2024-09-14T04:01:18.288 INFO:journalctl@ceph.mon.b.smithi129.stdout: block_cache_name: BinnedLRUCache 2024-09-14T04:01:18.288 INFO:journalctl@ceph.mon.b.smithi129.stdout: block_cache_options: 2024-09-14T04:01:18.288 INFO:journalctl@ceph.mon.b.smithi129.stdout: capacity : 536870912 2024-09-14T04:01:18.288 INFO:journalctl@ceph.mon.b.smithi129.stdout: num_shard_bits : 4 2024-09-14T04:01:18.288 INFO:journalctl@ceph.mon.b.smithi129.stdout: strict_capacity_limit : 0 2024-09-14T04:01:18.288 INFO:journalctl@ceph.mon.b.smithi129.stdout: high_pri_pool_ratio: 0.000 2024-09-14T04:01:18.288 INFO:journalctl@ceph.mon.b.smithi129.stdout: block_cache_compressed: (nil) 2024-09-14T04:01:18.288 INFO:journalctl@ceph.mon.b.smithi129.stdout: persistent_cache: (nil) 2024-09-14T04:01:18.288 INFO:journalctl@ceph.mon.b.smithi129.stdout: block_size: 4096 2024-09-14T04:01:18.288 INFO:journalctl@ceph.mon.b.smithi129.stdout: block_size_deviation: 10 2024-09-14T04:01:18.289 INFO:journalctl@ceph.mon.b.smithi129.stdout: block_restart_interval: 16 2024-09-14T04:01:18.289 INFO:journalctl@ceph.mon.b.smithi129.stdout: index_block_restart_interval: 1 2024-09-14T04:01:18.289 INFO:journalctl@ceph.mon.b.smithi129.stdout: metadata_block_size: 4096 2024-09-14T04:01:18.289 INFO:journalctl@ceph.mon.b.smithi129.stdout: partition_filters: 0 2024-09-14T04:01:18.289 INFO:journalctl@ceph.mon.b.smithi129.stdout: use_delta_encoding: 1 2024-09-14T04:01:18.289 INFO:journalctl@ceph.mon.b.smithi129.stdout: filter_policy: bloomfilter 2024-09-14T04:01:18.289 INFO:journalctl@ceph.mon.b.smithi129.stdout: whole_key_filtering: 1 2024-09-14T04:01:18.289 INFO:journalctl@ceph.mon.b.smithi129.stdout: verify_compression: 0 2024-09-14T04:01:18.289 INFO:journalctl@ceph.mon.b.smithi129.stdout: read_amp_bytes_per_bit: 0 2024-09-14T04:01:18.289 INFO:journalctl@ceph.mon.b.smithi129.stdout: format_version: 5 2024-09-14T04:01:18.289 INFO:journalctl@ceph.mon.b.smithi129.stdout: enable_index_compression: 1 2024-09-14T04:01:18.289 INFO:journalctl@ceph.mon.b.smithi129.stdout: block_align: 0 2024-09-14T04:01:18.289 INFO:journalctl@ceph.mon.b.smithi129.stdout: max_auto_readahead_size: 262144 2024-09-14T04:01:18.289 INFO:journalctl@ceph.mon.b.smithi129.stdout: prepopulate_block_cache: 0 2024-09-14T04:01:18.290 INFO:journalctl@ceph.mon.b.smithi129.stdout: initial_auto_readahead_size: 8192 2024-09-14T04:01:18.290 INFO:journalctl@ceph.mon.b.smithi129.stdout: num_file_reads_for_auto_readahead: 2 2024-09-14T04:01:18.290 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.write_buffer_size: 33554432 2024-09-14T04:01:18.290 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.max_write_buffer_number: 2 2024-09-14T04:01:18.290 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.compression: NoCompression 2024-09-14T04:01:18.290 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.bottommost_compression: Disabled 2024-09-14T04:01:18.290 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.prefix_extractor: nullptr 2024-09-14T04:01:18.290 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-09-14T04:01:18.290 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.num_levels: 7 2024-09-14T04:01:18.290 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-09-14T04:01:18.290 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-09-14T04:01:18.290 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-09-14T04:01:18.290 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-09-14T04:01:18.291 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-09-14T04:01:18.291 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-09-14T04:01:18.291 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-09-14T04:01:18.291 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-09-14T04:01:18.291 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-09-14T04:01:18.291 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-09-14T04:01:18.291 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-09-14T04:01:18.291 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-09-14T04:01:18.291 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.compression_opts.window_bits: -14 2024-09-14T04:01:18.291 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.compression_opts.level: 32767 2024-09-14T04:01:18.291 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.compression_opts.strategy: 0 2024-09-14T04:01:18.291 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-09-14T04:01:18.292 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-09-14T04:01:18.292 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-09-14T04:01:18.292 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-09-14T04:01:18.292 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.compression_opts.enabled: false 2024-09-14T04:01:18.292 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-09-14T04:01:18.292 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-09-14T04:01:18.292 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-09-14T04:01:18.292 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-09-14T04:01:18.292 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.target_file_size_base: 67108864 2024-09-14T04:01:18.292 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.target_file_size_multiplier: 1 2024-09-14T04:01:18.292 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-09-14T04:01:18.292 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-09-14T04:01:18.292 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-09-14T04:01:18.292 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-09-14T04:01:18.293 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-09-14T04:01:18.293 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-09-14T04:01:18.293 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-09-14T04:01:18.293 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-09-14T04:01:18.293 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-09-14T04:01:18.293 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-09-14T04:01:18.293 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-09-14T04:01:18.293 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-09-14T04:01:18.293 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-09-14T04:01:18.294 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.arena_block_size: 1048576 2024-09-14T04:01:18.294 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-09-14T04:01:18.294 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-09-14T04:01:18.294 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.disable_auto_compactions: 0 2024-09-14T04:01:18.294 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-09-14T04:01:18.294 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-09-14T04:01:18.294 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-09-14T04:01:18.294 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-09-14T04:01:18.294 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-09-14T04:01:18.294 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-09-14T04:01:18.294 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-09-14T04:01:18.295 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-09-14T04:01:18.295 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-09-14T04:01:18.295 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-09-14T04:01:18.295 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-09-14T04:01:18.295 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.inplace_update_support: 0 2024-09-14T04:01:18.295 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.inplace_update_num_locks: 10000 2024-09-14T04:01:18.295 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-09-14T04:01:18.295 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-09-14T04:01:18.295 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.memtable_huge_page_size: 0 2024-09-14T04:01:18.295 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.bloom_locality: 0 2024-09-14T04:01:18.295 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.max_successive_merges: 0 2024-09-14T04:01:18.295 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.optimize_filters_for_hits: 0 2024-09-14T04:01:18.296 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.paranoid_file_checks: 0 2024-09-14T04:01:18.296 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.force_consistency_checks: 1 2024-09-14T04:01:18.296 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.report_bg_io_stats: 0 2024-09-14T04:01:18.296 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.ttl: 2592000 2024-09-14T04:01:18.296 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.periodic_compaction_seconds: 0 2024-09-14T04:01:18.296 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-09-14T04:01:18.296 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-09-14T04:01:18.296 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.enable_blob_files: false 2024-09-14T04:01:18.296 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.min_blob_size: 0 2024-09-14T04:01:18.296 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.blob_file_size: 268435456 2024-09-14T04:01:18.296 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.blob_compression_type: NoCompression 2024-09-14T04:01:18.296 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.enable_blob_garbage_collection: false 2024-09-14T04:01:18.297 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-09-14T04:01:18.297 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-09-14T04:01:18.297 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-09-14T04:01:18.297 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.blob_file_starting_level: 0 2024-09-14T04:01:18.297 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-09-14T04:01:18.297 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-14T04:01:18.297 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 32, last_sequence is 19470, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 28 2024-09-14T04:01:18.297 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-09-14T04:01:18.297 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: eb44eb2c-3ded-4633-add3-68c40e4279c8 2024-09-14T04:01:18.297 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726286477875346, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-09-14T04:01:18.297 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-09-14T04:01:18.297 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726286477889258, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 3137046, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 19476, "largest_seqno": 22016, "table_properties": {"data_size": 3125258, "index_size": 8246, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 2501, "raw_key_size": 22842, "raw_average_key_size": 23, "raw_value_size": 3102672, "raw_average_value_size": 3146, "num_data_blocks": 354, "num_entries": 986, "num_filter_entries": 986, "num_deletions": 8, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1726286477, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "eb44eb2c-3ded-4633-add3-68c40e4279c8", "db_session_id": "5QDPM5NBSJPJQ3094IP7", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-09-14T04:01:18.298 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: EVENT_LOG_v1 {"time_micros": 1726286477889400, "job": 1, "event": "recovery_finished"} 2024-09-14T04:01:18.298 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-09-14T04:01:18.298 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-09-14T04:01:18.298 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-09-14T04:01:18.298 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x555999976e00 2024-09-14T04:01:18.298 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: DB pointer 0x555999a78000 2024-09-14T04:01:18.298 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-09-14T04:01:18.298 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-09-14T04:01:18.298 INFO:journalctl@ceph.mon.b.smithi129.stdout: ** DB Stats ** 2024-09-14T04:01:18.298 INFO:journalctl@ceph.mon.b.smithi129.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T04:01:18.298 INFO:journalctl@ceph.mon.b.smithi129.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-09-14T04:01:18.298 INFO:journalctl@ceph.mon.b.smithi129.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T04:01:18.298 INFO:journalctl@ceph.mon.b.smithi129.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T04:01:18.299 INFO:journalctl@ceph.mon.b.smithi129.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-09-14T04:01:18.299 INFO:journalctl@ceph.mon.b.smithi129.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-09-14T04:01:18.299 INFO:journalctl@ceph.mon.b.smithi129.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-09-14T04:01:18.299 INFO:journalctl@ceph.mon.b.smithi129.stdout: 2024-09-14T04:01:18.299 INFO:journalctl@ceph.mon.b.smithi129.stdout: ** Compaction Stats [default] ** 2024-09-14T04:01:18.299 INFO:journalctl@ceph.mon.b.smithi129.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T04:01:18.299 INFO:journalctl@ceph.mon.b.smithi129.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-09-14T04:01:18.299 INFO:journalctl@ceph.mon.b.smithi129.stdout: L0 1/0 2.99 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 622.2 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-09-14T04:01:18.299 INFO:journalctl@ceph.mon.b.smithi129.stdout: L6 1/0 13.06 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-09-14T04:01:18.299 INFO:journalctl@ceph.mon.b.smithi129.stdout: Sum 2/0 16.05 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 622.2 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-09-14T04:01:18.299 INFO:journalctl@ceph.mon.b.smithi129.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 622.2 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-09-14T04:01:18.299 INFO:journalctl@ceph.mon.b.smithi129.stdout: 2024-09-14T04:01:18.299 INFO:journalctl@ceph.mon.b.smithi129.stdout: ** Compaction Stats [default] ** 2024-09-14T04:01:18.300 INFO:journalctl@ceph.mon.b.smithi129.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-09-14T04:01:18.300 INFO:journalctl@ceph.mon.b.smithi129.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-09-14T04:01:18.300 INFO:journalctl@ceph.mon.b.smithi129.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 622.2 0.00 0.00 1 0.005 0 0 0.0 0.0 2024-09-14T04:01:18.300 INFO:journalctl@ceph.mon.b.smithi129.stdout: 2024-09-14T04:01:18.300 INFO:journalctl@ceph.mon.b.smithi129.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-09-14T04:01:18.300 INFO:journalctl@ceph.mon.b.smithi129.stdout: 2024-09-14T04:01:18.300 INFO:journalctl@ceph.mon.b.smithi129.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-09-14T04:01:18.300 INFO:journalctl@ceph.mon.b.smithi129.stdout: Flush(GB): cumulative 0.003, interval 0.003 2024-09-14T04:01:18.300 INFO:journalctl@ceph.mon.b.smithi129.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-09-14T04:01:18.300 INFO:journalctl@ceph.mon.b.smithi129.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-09-14T04:01:18.300 INFO:journalctl@ceph.mon.b.smithi129.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-09-14T04:01:18.300 INFO:journalctl@ceph.mon.b.smithi129.stdout: AddFile(Keys): cumulative 0, interval 0 2024-09-14T04:01:18.301 INFO:journalctl@ceph.mon.b.smithi129.stdout: Cumulative compaction: 0.00 GB write, 156.84 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T04:01:18.301 INFO:journalctl@ceph.mon.b.smithi129.stdout: Interval compaction: 0.00 GB write, 156.84 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-09-14T04:01:18.301 INFO:journalctl@ceph.mon.b.smithi129.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-09-14T04:01:18.301 INFO:journalctl@ceph.mon.b.smithi129.stdout: Block cache BinnedLRUCache@0x555999947610#7 capacity: 512.00 MB usage: 11.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-09-14T04:01:18.301 INFO:journalctl@ceph.mon.b.smithi129.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,2.53 KB,0.000482798%) IndexBlock(1,9.08 KB,0.00173151%) Misc(1,0.00 KB,0%) 2024-09-14T04:01:18.301 INFO:journalctl@ceph.mon.b.smithi129.stdout: 2024-09-14T04:01:18.301 INFO:journalctl@ceph.mon.b.smithi129.stdout: ** File Read Latency Histogram By Level [default] ** 2024-09-14T04:01:18.301 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: starting mon.b rank 1 at public addrs [v2:172.21.15.129:3300/0,v1:172.21.15.129:6789/0] at bind addrs [v2:172.21.15.129:3300/0,v1:172.21.15.129:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T04:01:18.301 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: mon.b@-1(???) e3 preinit fsid bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T04:01:18.301 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: mon.b@-1(???).mds e9 new map 2024-09-14T04:01:18.301 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: mon.b@-1(???).mds e9 print_map 2024-09-14T04:01:18.301 INFO:journalctl@ceph.mon.b.smithi129.stdout: e9 2024-09-14T04:01:18.302 INFO:journalctl@ceph.mon.b.smithi129.stdout: btime 1970-01-01T00:00:00:000000+0000 2024-09-14T04:01:18.302 INFO:journalctl@ceph.mon.b.smithi129.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-09-14T04:01:18.302 INFO:journalctl@ceph.mon.b.smithi129.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T04:01:18.302 INFO:journalctl@ceph.mon.b.smithi129.stdout: legacy client fscid: 1 2024-09-14T04:01:18.302 INFO:journalctl@ceph.mon.b.smithi129.stdout: 2024-09-14T04:01:18.302 INFO:journalctl@ceph.mon.b.smithi129.stdout: Filesystem 'foo' (1) 2024-09-14T04:01:18.302 INFO:journalctl@ceph.mon.b.smithi129.stdout: fs_name foo 2024-09-14T04:01:18.302 INFO:journalctl@ceph.mon.b.smithi129.stdout: epoch 8 2024-09-14T04:01:18.302 INFO:journalctl@ceph.mon.b.smithi129.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-09-14T04:01:18.302 INFO:journalctl@ceph.mon.b.smithi129.stdout: created 2024-09-14T03:53:05.236201+0000 2024-09-14T04:01:18.302 INFO:journalctl@ceph.mon.b.smithi129.stdout: modified 2024-09-14T03:53:22.726472+0000 2024-09-14T04:01:18.302 INFO:journalctl@ceph.mon.b.smithi129.stdout: tableserver 0 2024-09-14T04:01:18.303 INFO:journalctl@ceph.mon.b.smithi129.stdout: root 0 2024-09-14T04:01:18.303 INFO:journalctl@ceph.mon.b.smithi129.stdout: session_timeout 60 2024-09-14T04:01:18.303 INFO:journalctl@ceph.mon.b.smithi129.stdout: session_autoclose 300 2024-09-14T04:01:18.303 INFO:journalctl@ceph.mon.b.smithi129.stdout: max_file_size 1099511627776 2024-09-14T04:01:18.303 INFO:journalctl@ceph.mon.b.smithi129.stdout: max_xattr_size 65536 2024-09-14T04:01:18.303 INFO:journalctl@ceph.mon.b.smithi129.stdout: required_client_features {} 2024-09-14T04:01:18.303 INFO:journalctl@ceph.mon.b.smithi129.stdout: last_failure 0 2024-09-14T04:01:18.303 INFO:journalctl@ceph.mon.b.smithi129.stdout: last_failure_osd_epoch 0 2024-09-14T04:01:18.303 INFO:journalctl@ceph.mon.b.smithi129.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-09-14T04:01:18.303 INFO:journalctl@ceph.mon.b.smithi129.stdout: max_mds 1 2024-09-14T04:01:18.303 INFO:journalctl@ceph.mon.b.smithi129.stdout: in 0 2024-09-14T04:01:18.303 INFO:journalctl@ceph.mon.b.smithi129.stdout: up {0=14646} 2024-09-14T04:01:18.304 INFO:journalctl@ceph.mon.b.smithi129.stdout: failed 2024-09-14T04:01:18.304 INFO:journalctl@ceph.mon.b.smithi129.stdout: damaged 2024-09-14T04:01:18.304 INFO:journalctl@ceph.mon.b.smithi129.stdout: stopped 2024-09-14T04:01:18.304 INFO:journalctl@ceph.mon.b.smithi129.stdout: data_pools [4] 2024-09-14T04:01:18.304 INFO:journalctl@ceph.mon.b.smithi129.stdout: metadata_pool 3 2024-09-14T04:01:18.304 INFO:journalctl@ceph.mon.b.smithi129.stdout: inline_data disabled 2024-09-14T04:01:18.304 INFO:journalctl@ceph.mon.b.smithi129.stdout: balancer 2024-09-14T04:01:18.304 INFO:journalctl@ceph.mon.b.smithi129.stdout: bal_rank_mask -1 2024-09-14T04:01:18.304 INFO:journalctl@ceph.mon.b.smithi129.stdout: standby_count_wanted 1 2024-09-14T04:01:18.304 INFO:journalctl@ceph.mon.b.smithi129.stdout: qdb_cluster leader: 0 members: 2024-09-14T04:01:18.304 INFO:journalctl@ceph.mon.b.smithi129.stdout: [mds.foo.smithi039.bbwiwf{0:14646} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.39:6834/2717630652,v1:172.21.15.39:6835/2717630652] compat {c=[1],r=[1],i=[7ff]}] 2024-09-14T04:01:18.304 INFO:journalctl@ceph.mon.b.smithi129.stdout: 2024-09-14T04:01:18.305 INFO:journalctl@ceph.mon.b.smithi129.stdout: 2024-09-14T04:01:18.305 INFO:journalctl@ceph.mon.b.smithi129.stdout: Standby daemons: 2024-09-14T04:01:18.305 INFO:journalctl@ceph.mon.b.smithi129.stdout: 2024-09-14T04:01:18.305 INFO:journalctl@ceph.mon.b.smithi129.stdout: [mds.foo.smithi129.ybjqcw{-1:24475} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.129:6832/2789452907,v1:172.21.15.129:6833/2789452907] compat {c=[1],r=[1],i=[7ff]}] 2024-09-14T04:01:18.305 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: mon.b@-1(???).osd e464 crush map has features 3314933000854323200, adjusting msgr requires 2024-09-14T04:01:18.305 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: mon.b@-1(???).osd e464 crush map has features 432629239337189376, adjusting msgr requires 2024-09-14T04:01:18.305 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: mon.b@-1(???).osd e464 crush map has features 432629239337189376, adjusting msgr requires 2024-09-14T04:01:18.305 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: mon.b@-1(???).osd e464 crush map has features 432629239337189376, adjusting msgr requires 2024-09-14T04:01:18.305 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 ceph-mon[101316]: mon.b@-1(???).paxosservice(auth 1..22) refresh upgraded, format 0 -> 3 2024-09-14T04:01:18.305 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 bash[101278]: cff2519560f167e23398a4a0d0891211e49777809673ae8ad77db31e6701dbed 2024-09-14T04:01:18.305 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:17 smithi129 systemd[1]: Started Ceph mon.b for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T04:01:19.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:01:19 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:01:19] ENGINE Bus STARTING 2024-09-14T04:01:19.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:01:19 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:01:19] ENGINE Serving on http://:::9283 2024-09-14T04:01:19.791 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:01:19 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:01:19] ENGINE Bus STARTED 2024-09-14T04:01:23.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:01:22 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T04:01:22.902+0000 7f268b1da640 -1 mgr.server handle_report got status from non-daemon mon.b 2024-09-14T04:01:24.570 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.Snapshot (23266 ms) 2024-09-14T04:01:24.571 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.SnapshotRemoveOnDisable 2024-09-14T04:01:24.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:24 smithi129 ceph-mon[101316]: pgmap v185: 105 pgs: 105 active+clean; 583 KiB data, 458 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 2.0 KiB/s wr, 29 op/s 2024-09-14T04:01:24.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:24 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T04:01:24.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:24 smithi129 ceph-mon[101316]: mon.a calling monitor election 2024-09-14T04:01:24.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:24 smithi129 ceph-mon[101316]: mon.c calling monitor election 2024-09-14T04:01:24.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:24 smithi129 ceph-mon[101316]: mon.b calling monitor election 2024-09-14T04:01:24.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:24 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T04:01:24.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:24 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T04:01:24.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:24 smithi129 ceph-mon[101316]: pgmap v187: 105 pgs: 105 active+clean; 583 KiB data, 458 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 1.9 KiB/s wr, 28 op/s 2024-09-14T04:01:24.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:24 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:24.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:24 smithi129 ceph-mon[101316]: pgmap v188: 105 pgs: 105 active+clean; 583 KiB data, 458 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 2.0 KiB/s wr, 28 op/s 2024-09-14T04:01:24.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:24 smithi129 ceph-mon[101316]: mon.a is new leader, mons a,c in quorum (ranks 0,2) 2024-09-14T04:01:24.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:24 smithi129 ceph-mon[101316]: overall HEALTH_OK 2024-09-14T04:01:24.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:24 smithi129 ceph-mon[101316]: mon.a calling monitor election 2024-09-14T04:01:24.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:24 smithi129 ceph-mon[101316]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T04:01:24.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:24 smithi129 ceph-mon[101316]: monmap epoch 4 2024-09-14T04:01:24.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:24 smithi129 ceph-mon[101316]: fsid bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T04:01:24.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:24 smithi129 ceph-mon[101316]: last_changed 2024-09-14T04:01:19.396995+0000 2024-09-14T04:01:24.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:24 smithi129 ceph-mon[101316]: created 2024-09-14T03:46:10.112174+0000 2024-09-14T04:01:24.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:24 smithi129 ceph-mon[101316]: min_mon_release 19 (squid) 2024-09-14T04:01:24.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:24 smithi129 ceph-mon[101316]: election_strategy: 1 2024-09-14T04:01:24.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:24 smithi129 ceph-mon[101316]: 0: [v2:172.21.15.39:3300/0,v1:172.21.15.39:6789/0] mon.a 2024-09-14T04:01:24.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:24 smithi129 ceph-mon[101316]: 1: [v2:172.21.15.129:3300/0,v1:172.21.15.129:6789/0] mon.b 2024-09-14T04:01:24.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:24 smithi129 ceph-mon[101316]: 2: [v2:172.21.15.39:3301/0,v1:172.21.15.39:6790/0] mon.c 2024-09-14T04:01:24.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:24 smithi129 ceph-mon[101316]: fsmap foo:1 {0=foo.smithi039.bbwiwf=up:active} 1 up:standby 2024-09-14T04:01:24.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:24 smithi129 ceph-mon[101316]: osdmap e465: 8 total, 8 up, 8 in 2024-09-14T04:01:24.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:24 smithi129 ceph-mon[101316]: mgrmap e41: y(active, since 2m), standbys: x 2024-09-14T04:01:24.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:24 smithi129 ceph-mon[101316]: overall HEALTH_OK 2024-09-14T04:01:24.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:24 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='' 2024-09-14T04:01:24.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:24 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:24.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:24 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:24.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:24 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:01:24.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[111235]: pgmap v185: 105 pgs: 105 active+clean; 583 KiB data, 458 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 2.0 KiB/s wr, 29 op/s 2024-09-14T04:01:24.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T04:01:24.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[111235]: mon.a calling monitor election 2024-09-14T04:01:24.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[111235]: mon.c calling monitor election 2024-09-14T04:01:24.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[111235]: mon.b calling monitor election 2024-09-14T04:01:24.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T04:01:24.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T04:01:24.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[111235]: pgmap v187: 105 pgs: 105 active+clean; 583 KiB data, 458 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 1.9 KiB/s wr, 28 op/s 2024-09-14T04:01:24.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:24.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[111235]: pgmap v188: 105 pgs: 105 active+clean; 583 KiB data, 458 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 2.0 KiB/s wr, 28 op/s 2024-09-14T04:01:24.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[111235]: mon.a is new leader, mons a,c in quorum (ranks 0,2) 2024-09-14T04:01:24.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[111235]: overall HEALTH_OK 2024-09-14T04:01:24.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[111235]: mon.a calling monitor election 2024-09-14T04:01:24.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[111235]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T04:01:24.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[111235]: monmap epoch 4 2024-09-14T04:01:24.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[111235]: fsid bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T04:01:24.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[111235]: last_changed 2024-09-14T04:01:19.396995+0000 2024-09-14T04:01:24.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[111235]: created 2024-09-14T03:46:10.112174+0000 2024-09-14T04:01:24.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[111235]: min_mon_release 19 (squid) 2024-09-14T04:01:24.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[111235]: election_strategy: 1 2024-09-14T04:01:24.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[111235]: 0: [v2:172.21.15.39:3300/0,v1:172.21.15.39:6789/0] mon.a 2024-09-14T04:01:24.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[111235]: 1: [v2:172.21.15.129:3300/0,v1:172.21.15.129:6789/0] mon.b 2024-09-14T04:01:24.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[111235]: 2: [v2:172.21.15.39:3301/0,v1:172.21.15.39:6790/0] mon.c 2024-09-14T04:01:24.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[111235]: fsmap foo:1 {0=foo.smithi039.bbwiwf=up:active} 1 up:standby 2024-09-14T04:01:24.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[111235]: osdmap e465: 8 total, 8 up, 8 in 2024-09-14T04:01:24.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[111235]: mgrmap e41: y(active, since 2m), standbys: x 2024-09-14T04:01:24.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[111235]: overall HEALTH_OK 2024-09-14T04:01:24.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='' 2024-09-14T04:01:24.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:24.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:24.794 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:01:24.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[106470]: pgmap v185: 105 pgs: 105 active+clean; 583 KiB data, 458 MiB used, 715 GiB / 715 GiB avail; 21 KiB/s rd, 2.0 KiB/s wr, 29 op/s 2024-09-14T04:01:24.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "a"} : dispatch 2024-09-14T04:01:24.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[106470]: mon.a calling monitor election 2024-09-14T04:01:24.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[106470]: mon.c calling monitor election 2024-09-14T04:01:24.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[106470]: mon.b calling monitor election 2024-09-14T04:01:24.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "b"} : dispatch 2024-09-14T04:01:24.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "mon metadata", "id": "c"} : dispatch 2024-09-14T04:01:24.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[106470]: pgmap v187: 105 pgs: 105 active+clean; 583 KiB data, 458 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 1.9 KiB/s wr, 28 op/s 2024-09-14T04:01:24.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:24.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[106470]: pgmap v188: 105 pgs: 105 active+clean; 583 KiB data, 458 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 2.0 KiB/s wr, 28 op/s 2024-09-14T04:01:24.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[106470]: mon.a is new leader, mons a,c in quorum (ranks 0,2) 2024-09-14T04:01:24.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[106470]: overall HEALTH_OK 2024-09-14T04:01:24.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[106470]: mon.a calling monitor election 2024-09-14T04:01:24.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[106470]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-09-14T04:01:24.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[106470]: monmap epoch 4 2024-09-14T04:01:24.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[106470]: fsid bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T04:01:24.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[106470]: last_changed 2024-09-14T04:01:19.396995+0000 2024-09-14T04:01:24.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[106470]: created 2024-09-14T03:46:10.112174+0000 2024-09-14T04:01:24.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[106470]: min_mon_release 19 (squid) 2024-09-14T04:01:24.796 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[106470]: election_strategy: 1 2024-09-14T04:01:24.796 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[106470]: 0: [v2:172.21.15.39:3300/0,v1:172.21.15.39:6789/0] mon.a 2024-09-14T04:01:24.796 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[106470]: 1: [v2:172.21.15.129:3300/0,v1:172.21.15.129:6789/0] mon.b 2024-09-14T04:01:24.796 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[106470]: 2: [v2:172.21.15.39:3301/0,v1:172.21.15.39:6790/0] mon.c 2024-09-14T04:01:24.796 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[106470]: fsmap foo:1 {0=foo.smithi039.bbwiwf=up:active} 1 up:standby 2024-09-14T04:01:24.796 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[106470]: osdmap e465: 8 total, 8 up, 8 in 2024-09-14T04:01:24.796 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[106470]: mgrmap e41: y(active, since 2m), standbys: x 2024-09-14T04:01:24.796 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[106470]: overall HEALTH_OK 2024-09-14T04:01:24.796 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='' 2024-09-14T04:01:24.796 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:24.796 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:24.796 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:24 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:01:25.741 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:25 smithi129 ceph-mon[101316]: osdmap e466: 8 total, 8 up, 8 in 2024-09-14T04:01:25.741 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:25 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/849e18ea-e8b9-4f2c-ad87-957e8c084a3a"} : dispatch 2024-09-14T04:01:25.741 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:25 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/849e18ea-e8b9-4f2c-ad87-957e8c084a3a"}]': finished 2024-09-14T04:01:25.741 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:25 smithi129 ceph-mon[101316]: pgmap v190: 105 pgs: 105 active+clean; 583 KiB data, 458 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 2.0 KiB/s wr, 28 op/s 2024-09-14T04:01:25.742 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:25 smithi129 ceph-mon[101316]: osdmap e467: 8 total, 8 up, 8 in 2024-09-14T04:01:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:25 smithi039 ceph-mon[106470]: osdmap e466: 8 total, 8 up, 8 in 2024-09-14T04:01:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:25 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/849e18ea-e8b9-4f2c-ad87-957e8c084a3a"} : dispatch 2024-09-14T04:01:25.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:25 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/849e18ea-e8b9-4f2c-ad87-957e8c084a3a"}]': finished 2024-09-14T04:01:25.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:25 smithi039 ceph-mon[106470]: pgmap v190: 105 pgs: 105 active+clean; 583 KiB data, 458 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 2.0 KiB/s wr, 28 op/s 2024-09-14T04:01:25.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:25 smithi039 ceph-mon[106470]: osdmap e467: 8 total, 8 up, 8 in 2024-09-14T04:01:25.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:25 smithi039 ceph-mon[111235]: osdmap e466: 8 total, 8 up, 8 in 2024-09-14T04:01:25.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:25 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/849e18ea-e8b9-4f2c-ad87-957e8c084a3a"} : dispatch 2024-09-14T04:01:25.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:25 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/849e18ea-e8b9-4f2c-ad87-957e8c084a3a"}]': finished 2024-09-14T04:01:25.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:25 smithi039 ceph-mon[111235]: pgmap v190: 105 pgs: 105 active+clean; 583 KiB data, 458 MiB used, 715 GiB / 715 GiB avail; 20 KiB/s rd, 2.0 KiB/s wr, 28 op/s 2024-09-14T04:01:25.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:25 smithi039 ceph-mon[111235]: osdmap e467: 8 total, 8 up, 8 in 2024-09-14T04:01:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:27 smithi129 ceph-mon[101316]: osdmap e468: 8 total, 8 up, 8 in 2024-09-14T04:01:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:27 smithi129 ceph-mon[101316]: pgmap v193: 105 pgs: 105 active+clean; 583 KiB data, 458 MiB used, 715 GiB / 715 GiB avail; 5.5 KiB/s rd, 682 B/s wr, 7 op/s 2024-09-14T04:01:27.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:27 smithi039 ceph-mon[106470]: osdmap e468: 8 total, 8 up, 8 in 2024-09-14T04:01:27.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:27 smithi039 ceph-mon[106470]: pgmap v193: 105 pgs: 105 active+clean; 583 KiB data, 458 MiB used, 715 GiB / 715 GiB avail; 5.5 KiB/s rd, 682 B/s wr, 7 op/s 2024-09-14T04:01:27.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:27 smithi039 ceph-mon[111235]: osdmap e468: 8 total, 8 up, 8 in 2024-09-14T04:01:27.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:27 smithi039 ceph-mon[111235]: pgmap v193: 105 pgs: 105 active+clean; 583 KiB data, 458 MiB used, 715 GiB / 715 GiB avail; 5.5 KiB/s rd, 682 B/s wr, 7 op/s 2024-09-14T04:01:28.494 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.SnapshotRemoveOnDisable (3924 ms) 2024-09-14T04:01:28.494 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.SnapshotUnlinkPeer 2024-09-14T04:01:28.649 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:28 smithi039 ceph-mon[111235]: osdmap e469: 8 total, 8 up, 8 in 2024-09-14T04:01:28.649 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:28 smithi039 ceph-mon[106470]: osdmap e469: 8 total, 8 up, 8 in 2024-09-14T04:01:28.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:28 smithi129 ceph-mon[101316]: osdmap e469: 8 total, 8 up, 8 in 2024-09-14T04:01:28.941 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:01:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:01:28] "GET /metrics HTTP/1.1" 200 36413 "" "Prometheus/2.51.0" 2024-09-14T04:01:29.732 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:29 smithi129 ceph-mon[101316]: osdmap e470: 8 total, 8 up, 8 in 2024-09-14T04:01:29.732 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:29 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/f1c139bc-834f-4e11-a6e5-ea8fafc90868"} : dispatch 2024-09-14T04:01:29.732 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:29 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/f1c139bc-834f-4e11-a6e5-ea8fafc90868"}]': finished 2024-09-14T04:01:29.732 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:29 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:29.732 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:29 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:29.732 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:29 smithi129 ceph-mon[101316]: pgmap v196: 105 pgs: 105 active+clean; 583 KiB data, 458 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s rd, 8.5 KiB/s wr, 98 op/s 2024-09-14T04:01:29.732 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:29 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:29.732 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:29 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:29 smithi039 ceph-mon[111235]: osdmap e470: 8 total, 8 up, 8 in 2024-09-14T04:01:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:29 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/f1c139bc-834f-4e11-a6e5-ea8fafc90868"} : dispatch 2024-09-14T04:01:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:29 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/f1c139bc-834f-4e11-a6e5-ea8fafc90868"}]': finished 2024-09-14T04:01:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:29 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:29 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:29.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:29 smithi039 ceph-mon[111235]: pgmap v196: 105 pgs: 105 active+clean; 583 KiB data, 458 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s rd, 8.5 KiB/s wr, 98 op/s 2024-09-14T04:01:29.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:29 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:29.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:29 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:29.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:29 smithi039 ceph-mon[106470]: osdmap e470: 8 total, 8 up, 8 in 2024-09-14T04:01:29.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:29 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/f1c139bc-834f-4e11-a6e5-ea8fafc90868"} : dispatch 2024-09-14T04:01:29.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:29 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/f1c139bc-834f-4e11-a6e5-ea8fafc90868"}]': finished 2024-09-14T04:01:29.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:29 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:29.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:29 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:29.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:29 smithi039 ceph-mon[106470]: pgmap v196: 105 pgs: 105 active+clean; 583 KiB data, 458 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s rd, 8.5 KiB/s wr, 98 op/s 2024-09-14T04:01:29.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:29 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:29.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:29 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:30.663 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:30 smithi129 ceph-mon[101316]: osdmap e471: 8 total, 8 up, 8 in 2024-09-14T04:01:30.663 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:30 smithi129 ceph-mon[101316]: osdmap e472: 8 total, 8 up, 8 in 2024-09-14T04:01:30.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:30 smithi039 ceph-mon[106470]: osdmap e471: 8 total, 8 up, 8 in 2024-09-14T04:01:30.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:30 smithi039 ceph-mon[106470]: osdmap e472: 8 total, 8 up, 8 in 2024-09-14T04:01:30.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:30 smithi039 ceph-mon[111235]: osdmap e471: 8 total, 8 up, 8 in 2024-09-14T04:01:30.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:30 smithi039 ceph-mon[111235]: osdmap e472: 8 total, 8 up, 8 in 2024-09-14T04:01:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:31 smithi129 ceph-mon[101316]: pgmap v199: 105 pgs: 105 active+clean; 583 KiB data, 458 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s rd, 8.5 KiB/s wr, 98 op/s 2024-09-14T04:01:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:31 smithi129 ceph-mon[101316]: osdmap e473: 8 total, 8 up, 8 in 2024-09-14T04:01:31.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:31 smithi039 ceph-mon[106470]: pgmap v199: 105 pgs: 105 active+clean; 583 KiB data, 458 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s rd, 8.5 KiB/s wr, 98 op/s 2024-09-14T04:01:31.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:31 smithi039 ceph-mon[106470]: osdmap e473: 8 total, 8 up, 8 in 2024-09-14T04:01:31.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:31 smithi039 ceph-mon[111235]: pgmap v199: 105 pgs: 105 active+clean; 583 KiB data, 458 MiB used, 715 GiB / 715 GiB avail; 70 KiB/s rd, 8.5 KiB/s wr, 98 op/s 2024-09-14T04:01:31.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:31 smithi039 ceph-mon[111235]: osdmap e473: 8 total, 8 up, 8 in 2024-09-14T04:01:33.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:33 smithi039 ceph-mon[106470]: osdmap e474: 8 total, 8 up, 8 in 2024-09-14T04:01:33.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:33 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:33.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:33 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:33.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:33 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:33.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:33 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:01:33.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:33 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:33.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:33 smithi039 ceph-mon[106470]: Reconfiguring mon.a (monmap changed)... 2024-09-14T04:01:33.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:33 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T04:01:33.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:33 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T04:01:33.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:33 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:33.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:33 smithi039 ceph-mon[106470]: Reconfiguring daemon mon.a on smithi039 2024-09-14T04:01:33.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:33 smithi039 ceph-mon[106470]: pgmap v202: 105 pgs: 105 active+clean; 583 KiB data, 459 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s rd, 11 KiB/s wr, 158 op/s 2024-09-14T04:01:33.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:33 smithi039 ceph-mon[111235]: osdmap e474: 8 total, 8 up, 8 in 2024-09-14T04:01:33.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:33 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:33.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:33 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:33.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:33 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:33.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:33 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:01:33.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:33 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:33.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:33 smithi039 ceph-mon[111235]: Reconfiguring mon.a (monmap changed)... 2024-09-14T04:01:33.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:33 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T04:01:33.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:33 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T04:01:33.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:33 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:33.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:33 smithi039 ceph-mon[111235]: Reconfiguring daemon mon.a on smithi039 2024-09-14T04:01:33.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:33 smithi039 ceph-mon[111235]: pgmap v202: 105 pgs: 105 active+clean; 583 KiB data, 459 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s rd, 11 KiB/s wr, 158 op/s 2024-09-14T04:01:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:33 smithi129 ceph-mon[101316]: osdmap e474: 8 total, 8 up, 8 in 2024-09-14T04:01:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:33 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:33 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:33 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:33 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:01:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:33 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:33 smithi129 ceph-mon[101316]: Reconfiguring mon.a (monmap changed)... 2024-09-14T04:01:33.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:33 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T04:01:33.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:33 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T04:01:33.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:33 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:33.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:33 smithi129 ceph-mon[101316]: Reconfiguring daemon mon.a on smithi039 2024-09-14T04:01:33.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:33 smithi129 ceph-mon[101316]: pgmap v202: 105 pgs: 105 active+clean; 583 KiB data, 459 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s rd, 11 KiB/s wr, 158 op/s 2024-09-14T04:01:34.710 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:34 smithi039 ceph-mon[106470]: osdmap e475: 8 total, 8 up, 8 in 2024-09-14T04:01:34.710 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:34 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/ba1fe455-8bff-44ab-b331-62e63101940e"} : dispatch 2024-09-14T04:01:34.711 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:34 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/ba1fe455-8bff-44ab-b331-62e63101940e"}]': finished 2024-09-14T04:01:34.711 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:34 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:34.711 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:34 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:34.711 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:34 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T04:01:34.711 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:34 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T04:01:34.711 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:34 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-14T04:01:34.711 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:34 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:34.711 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:34 smithi039 ceph-mon[111235]: osdmap e475: 8 total, 8 up, 8 in 2024-09-14T04:01:34.711 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:34 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/ba1fe455-8bff-44ab-b331-62e63101940e"} : dispatch 2024-09-14T04:01:34.711 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:34 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/ba1fe455-8bff-44ab-b331-62e63101940e"}]': finished 2024-09-14T04:01:34.711 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:34 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:34.712 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:34 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:34.712 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:34 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T04:01:34.712 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:34 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T04:01:34.712 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:34 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-14T04:01:34.712 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:34 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:34.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:34 smithi129 ceph-mon[101316]: osdmap e475: 8 total, 8 up, 8 in 2024-09-14T04:01:34.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:34 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/ba1fe455-8bff-44ab-b331-62e63101940e"} : dispatch 2024-09-14T04:01:34.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:34 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/ba1fe455-8bff-44ab-b331-62e63101940e"}]': finished 2024-09-14T04:01:34.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:34 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:34.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:34 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:34.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:34 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T04:01:34.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:34 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T04:01:34.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:34 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-14T04:01:34.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:34 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:35.690 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:35 smithi039 ceph-mon[106470]: Reconfiguring mgr.y (monmap changed)... 2024-09-14T04:01:35.690 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:35 smithi039 ceph-mon[106470]: Reconfiguring daemon mgr.y on smithi039 2024-09-14T04:01:35.690 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:35 smithi039 ceph-mon[106470]: osdmap e476: 8 total, 8 up, 8 in 2024-09-14T04:01:35.690 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:35 smithi039 ceph-mon[106470]: pgmap v205: 105 pgs: 105 active+clean; 583 KiB data, 459 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s rd, 11 KiB/s wr, 158 op/s 2024-09-14T04:01:35.690 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:35 smithi039 ceph-mon[106470]: osdmap e477: 8 total, 8 up, 8 in 2024-09-14T04:01:35.690 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:35 smithi039 ceph-mon[111235]: Reconfiguring mgr.y (monmap changed)... 2024-09-14T04:01:35.690 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:35 smithi039 ceph-mon[111235]: Reconfiguring daemon mgr.y on smithi039 2024-09-14T04:01:35.690 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:35 smithi039 ceph-mon[111235]: osdmap e476: 8 total, 8 up, 8 in 2024-09-14T04:01:35.691 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:35 smithi039 ceph-mon[111235]: pgmap v205: 105 pgs: 105 active+clean; 583 KiB data, 459 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s rd, 11 KiB/s wr, 158 op/s 2024-09-14T04:01:35.691 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:35 smithi039 ceph-mon[111235]: osdmap e477: 8 total, 8 up, 8 in 2024-09-14T04:01:35.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:35 smithi129 ceph-mon[101316]: Reconfiguring mgr.y (monmap changed)... 2024-09-14T04:01:35.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:35 smithi129 ceph-mon[101316]: Reconfiguring daemon mgr.y on smithi039 2024-09-14T04:01:35.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:35 smithi129 ceph-mon[101316]: osdmap e476: 8 total, 8 up, 8 in 2024-09-14T04:01:35.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:35 smithi129 ceph-mon[101316]: pgmap v205: 105 pgs: 105 active+clean; 583 KiB data, 459 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s rd, 11 KiB/s wr, 158 op/s 2024-09-14T04:01:35.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:35 smithi129 ceph-mon[101316]: osdmap e477: 8 total, 8 up, 8 in 2024-09-14T04:01:37.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:36 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:37.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:36 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:37.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:36 smithi129 ceph-mon[101316]: Reconfiguring mon.c (monmap changed)... 2024-09-14T04:01:37.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:36 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T04:01:37.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:36 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T04:01:37.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:36 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:37.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:36 smithi129 ceph-mon[101316]: Reconfiguring daemon mon.c on smithi039 2024-09-14T04:01:37.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:36 smithi129 ceph-mon[101316]: osdmap e478: 8 total, 8 up, 8 in 2024-09-14T04:01:37.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:37.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:37.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:36 smithi039 ceph-mon[106470]: Reconfiguring mon.c (monmap changed)... 2024-09-14T04:01:37.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T04:01:37.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T04:01:37.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:37.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:36 smithi039 ceph-mon[106470]: Reconfiguring daemon mon.c on smithi039 2024-09-14T04:01:37.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:36 smithi039 ceph-mon[106470]: osdmap e478: 8 total, 8 up, 8 in 2024-09-14T04:01:37.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:37.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:37.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:36 smithi039 ceph-mon[111235]: Reconfiguring mon.c (monmap changed)... 2024-09-14T04:01:37.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T04:01:37.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T04:01:37.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:37.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:36 smithi039 ceph-mon[111235]: Reconfiguring daemon mon.c on smithi039 2024-09-14T04:01:37.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:36 smithi039 ceph-mon[111235]: osdmap e478: 8 total, 8 up, 8 in 2024-09-14T04:01:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:37 smithi129 ceph-mon[101316]: pgmap v208: 105 pgs: 105 active+clean; 583 KiB data, 459 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:01:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-14T04:01:38.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:38.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:37 smithi129 ceph-mon[101316]: osdmap e479: 8 total, 8 up, 8 in 2024-09-14T04:01:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:37 smithi039 ceph-mon[106470]: pgmap v208: 105 pgs: 105 active+clean; 583 KiB data, 459 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:01:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-14T04:01:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:38.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:37 smithi039 ceph-mon[106470]: osdmap e479: 8 total, 8 up, 8 in 2024-09-14T04:01:38.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:37 smithi039 ceph-mon[111235]: pgmap v208: 105 pgs: 105 active+clean; 583 KiB data, 459 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:01:38.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:38.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:38.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:38.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-14T04:01:38.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:38.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:37 smithi039 ceph-mon[111235]: osdmap e479: 8 total, 8 up, 8 in 2024-09-14T04:01:38.459 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.SnapshotUnlinkPeer (9965 ms) 2024-09-14T04:01:38.459 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.SnapshotImageState 2024-09-14T04:01:38.907 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:38 smithi039 ceph-mon[106470]: Reconfiguring osd.0 (monmap changed)... 2024-09-14T04:01:38.907 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:38 smithi039 ceph-mon[106470]: Reconfiguring daemon osd.0 on smithi039 2024-09-14T04:01:38.907 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:38 smithi039 ceph-mon[106470]: osdmap e480: 8 total, 8 up, 8 in 2024-09-14T04:01:38.908 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:38 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/96ba93ea-88e0-4a33-93af-069e67726c31"} : dispatch 2024-09-14T04:01:38.908 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:38 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/96ba93ea-88e0-4a33-93af-069e67726c31"}]': finished 2024-09-14T04:01:38.908 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:38 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/3b1e5b5a-cf98-47fa-ac6d-df25da7e1911"} : dispatch 2024-09-14T04:01:38.908 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:38 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/3b1e5b5a-cf98-47fa-ac6d-df25da7e1911"}]': finished 2024-09-14T04:01:38.908 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:01:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:01:38] "GET /metrics HTTP/1.1" 200 36412 "" "Prometheus/2.51.0" 2024-09-14T04:01:38.908 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:38 smithi039 ceph-mon[111235]: Reconfiguring osd.0 (monmap changed)... 2024-09-14T04:01:38.908 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:38 smithi039 ceph-mon[111235]: Reconfiguring daemon osd.0 on smithi039 2024-09-14T04:01:38.909 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:38 smithi039 ceph-mon[111235]: osdmap e480: 8 total, 8 up, 8 in 2024-09-14T04:01:38.909 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:38 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/96ba93ea-88e0-4a33-93af-069e67726c31"} : dispatch 2024-09-14T04:01:38.909 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:38 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/96ba93ea-88e0-4a33-93af-069e67726c31"}]': finished 2024-09-14T04:01:38.909 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:38 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/3b1e5b5a-cf98-47fa-ac6d-df25da7e1911"} : dispatch 2024-09-14T04:01:38.909 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:38 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/3b1e5b5a-cf98-47fa-ac6d-df25da7e1911"}]': finished 2024-09-14T04:01:39.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:38 smithi129 ceph-mon[101316]: Reconfiguring osd.0 (monmap changed)... 2024-09-14T04:01:39.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:38 smithi129 ceph-mon[101316]: Reconfiguring daemon osd.0 on smithi039 2024-09-14T04:01:39.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:38 smithi129 ceph-mon[101316]: osdmap e480: 8 total, 8 up, 8 in 2024-09-14T04:01:39.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:38 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/96ba93ea-88e0-4a33-93af-069e67726c31"} : dispatch 2024-09-14T04:01:39.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:38 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/96ba93ea-88e0-4a33-93af-069e67726c31"}]': finished 2024-09-14T04:01:39.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:38 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/3b1e5b5a-cf98-47fa-ac6d-df25da7e1911"} : dispatch 2024-09-14T04:01:39.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:38 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/3b1e5b5a-cf98-47fa-ac6d-df25da7e1911"}]': finished 2024-09-14T04:01:40.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:40 smithi129 ceph-mon[101316]: pgmap v211: 105 pgs: 105 active+clean; 583 KiB data, 443 MiB used, 715 GiB / 715 GiB avail; 161 KiB/s rd, 9.0 KiB/s wr, 215 op/s 2024-09-14T04:01:40.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:40 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:40.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:40 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:40.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:40 smithi129 ceph-mon[101316]: Reconfiguring osd.1 (monmap changed)... 2024-09-14T04:01:40.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:40 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-14T04:01:40.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:40 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:40.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:40 smithi129 ceph-mon[101316]: Reconfiguring daemon osd.1 on smithi039 2024-09-14T04:01:40.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:40 smithi129 ceph-mon[101316]: osdmap e481: 8 total, 8 up, 8 in 2024-09-14T04:01:40.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:40 smithi039 ceph-mon[111235]: pgmap v211: 105 pgs: 105 active+clean; 583 KiB data, 443 MiB used, 715 GiB / 715 GiB avail; 161 KiB/s rd, 9.0 KiB/s wr, 215 op/s 2024-09-14T04:01:40.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:40 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:40.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:40 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:40.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:40 smithi039 ceph-mon[111235]: Reconfiguring osd.1 (monmap changed)... 2024-09-14T04:01:40.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:40 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-14T04:01:40.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:40 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:40.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:40 smithi039 ceph-mon[111235]: Reconfiguring daemon osd.1 on smithi039 2024-09-14T04:01:40.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:40 smithi039 ceph-mon[111235]: osdmap e481: 8 total, 8 up, 8 in 2024-09-14T04:01:40.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:40 smithi039 ceph-mon[106470]: pgmap v211: 105 pgs: 105 active+clean; 583 KiB data, 443 MiB used, 715 GiB / 715 GiB avail; 161 KiB/s rd, 9.0 KiB/s wr, 215 op/s 2024-09-14T04:01:40.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:40 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:40.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:40 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:40.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:40 smithi039 ceph-mon[106470]: Reconfiguring osd.1 (monmap changed)... 2024-09-14T04:01:40.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:40 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-14T04:01:40.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:40 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:40.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:40 smithi039 ceph-mon[106470]: Reconfiguring daemon osd.1 on smithi039 2024-09-14T04:01:40.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:40 smithi039 ceph-mon[106470]: osdmap e481: 8 total, 8 up, 8 in 2024-09-14T04:01:41.506 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:41 smithi039 ceph-mon[106470]: osdmap e482: 8 total, 8 up, 8 in 2024-09-14T04:01:41.506 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:41 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:41.507 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:41 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:41.507 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:41 smithi039 ceph-mon[106470]: Reconfiguring osd.2 (monmap changed)... 2024-09-14T04:01:41.507 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:41 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-14T04:01:41.507 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:41 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:41.507 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:41 smithi039 ceph-mon[106470]: Reconfiguring daemon osd.2 on smithi039 2024-09-14T04:01:41.507 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:41 smithi039 ceph-mon[106470]: pgmap v214: 105 pgs: 105 active+clean; 583 KiB data, 443 MiB used, 715 GiB / 715 GiB avail; 161 KiB/s rd, 9.0 KiB/s wr, 215 op/s 2024-09-14T04:01:41.507 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:41 smithi039 ceph-mon[111235]: osdmap e482: 8 total, 8 up, 8 in 2024-09-14T04:01:41.507 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:41 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:41.507 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:41 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:41.507 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:41 smithi039 ceph-mon[111235]: Reconfiguring osd.2 (monmap changed)... 2024-09-14T04:01:41.507 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:41 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-14T04:01:41.507 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:41 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:41.508 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:41 smithi039 ceph-mon[111235]: Reconfiguring daemon osd.2 on smithi039 2024-09-14T04:01:41.508 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:41 smithi039 ceph-mon[111235]: pgmap v214: 105 pgs: 105 active+clean; 583 KiB data, 443 MiB used, 715 GiB / 715 GiB avail; 161 KiB/s rd, 9.0 KiB/s wr, 215 op/s 2024-09-14T04:01:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:41 smithi129 ceph-mon[101316]: osdmap e482: 8 total, 8 up, 8 in 2024-09-14T04:01:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:41 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:41 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:41 smithi129 ceph-mon[101316]: Reconfiguring osd.2 (monmap changed)... 2024-09-14T04:01:41.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:41 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-14T04:01:41.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:41 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:41.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:41 smithi129 ceph-mon[101316]: Reconfiguring daemon osd.2 on smithi039 2024-09-14T04:01:41.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:41 smithi129 ceph-mon[101316]: pgmap v214: 105 pgs: 105 active+clean; 583 KiB data, 443 MiB used, 715 GiB / 715 GiB avail; 161 KiB/s rd, 9.0 KiB/s wr, 215 op/s 2024-09-14T04:01:42.581 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.SnapshotImageState (4122 ms) 2024-09-14T04:01:42.582 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.SnapshotPromoteDemote 2024-09-14T04:01:42.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:42 smithi129 ceph-mon[101316]: osdmap e483: 8 total, 8 up, 8 in 2024-09-14T04:01:42.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:42 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:42.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:42 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:42.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:42 smithi129 ceph-mon[101316]: Reconfiguring osd.3 (monmap changed)... 2024-09-14T04:01:42.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:42 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-14T04:01:42.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:42 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:42.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:42 smithi129 ceph-mon[101316]: Reconfiguring daemon osd.3 on smithi039 2024-09-14T04:01:42.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:42 smithi039 ceph-mon[106470]: osdmap e483: 8 total, 8 up, 8 in 2024-09-14T04:01:42.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:42 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:42.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:42 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:42.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:42 smithi039 ceph-mon[106470]: Reconfiguring osd.3 (monmap changed)... 2024-09-14T04:01:42.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:42 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-14T04:01:42.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:42 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:42.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:42 smithi039 ceph-mon[106470]: Reconfiguring daemon osd.3 on smithi039 2024-09-14T04:01:42.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:42 smithi039 ceph-mon[111235]: osdmap e483: 8 total, 8 up, 8 in 2024-09-14T04:01:42.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:42 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:42.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:42 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:42.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:42 smithi039 ceph-mon[111235]: Reconfiguring osd.3 (monmap changed)... 2024-09-14T04:01:42.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:42 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-14T04:01:42.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:42 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:42.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:42 smithi039 ceph-mon[111235]: Reconfiguring daemon osd.3 on smithi039 2024-09-14T04:01:43.817 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:43 smithi039 ceph-mon[106470]: osdmap e484: 8 total, 8 up, 8 in 2024-09-14T04:01:43.817 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:43 smithi039 ceph-mon[106470]: pgmap v217: 105 pgs: 105 active+clean; 583 KiB data, 444 MiB used, 715 GiB / 715 GiB avail; 171 KiB/s rd, 26 KiB/s wr, 250 op/s 2024-09-14T04:01:43.817 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:43 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:43.817 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:43 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:43.817 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:43 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi039.bbwiwf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T04:01:43.818 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:43 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi039.bbwiwf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T04:01:43.818 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:43 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:43.818 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:43 smithi039 ceph-mon[111235]: osdmap e484: 8 total, 8 up, 8 in 2024-09-14T04:01:43.818 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:43 smithi039 ceph-mon[111235]: pgmap v217: 105 pgs: 105 active+clean; 583 KiB data, 444 MiB used, 715 GiB / 715 GiB avail; 171 KiB/s rd, 26 KiB/s wr, 250 op/s 2024-09-14T04:01:43.818 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:43 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:43.818 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:43 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:43.818 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:43 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi039.bbwiwf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T04:01:43.818 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:43 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi039.bbwiwf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T04:01:43.818 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:43 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:44.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:43 smithi129 ceph-mon[101316]: osdmap e484: 8 total, 8 up, 8 in 2024-09-14T04:01:44.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:43 smithi129 ceph-mon[101316]: pgmap v217: 105 pgs: 105 active+clean; 583 KiB data, 444 MiB used, 715 GiB / 715 GiB avail; 171 KiB/s rd, 26 KiB/s wr, 250 op/s 2024-09-14T04:01:44.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:43 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:44.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:43 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:44.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:43 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi039.bbwiwf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T04:01:44.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:43 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi039.bbwiwf", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T04:01:44.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:43 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:44.921 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:44 smithi129 ceph-mon[101316]: Reconfiguring mds.foo.smithi039.bbwiwf (monmap changed)... 2024-09-14T04:01:44.921 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:44 smithi129 ceph-mon[101316]: Reconfiguring daemon mds.foo.smithi039.bbwiwf on smithi039 2024-09-14T04:01:44.921 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:44 smithi129 ceph-mon[101316]: osdmap e485: 8 total, 8 up, 8 in 2024-09-14T04:01:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:44 smithi039 ceph-mon[106470]: Reconfiguring mds.foo.smithi039.bbwiwf (monmap changed)... 2024-09-14T04:01:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:44 smithi039 ceph-mon[106470]: Reconfiguring daemon mds.foo.smithi039.bbwiwf on smithi039 2024-09-14T04:01:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:44 smithi039 ceph-mon[106470]: osdmap e485: 8 total, 8 up, 8 in 2024-09-14T04:01:45.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:44 smithi039 ceph-mon[111235]: Reconfiguring mds.foo.smithi039.bbwiwf (monmap changed)... 2024-09-14T04:01:45.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:44 smithi039 ceph-mon[111235]: Reconfiguring daemon mds.foo.smithi039.bbwiwf on smithi039 2024-09-14T04:01:45.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:44 smithi039 ceph-mon[111235]: osdmap e485: 8 total, 8 up, 8 in 2024-09-14T04:01:45.907 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:45 smithi129 ceph-mon[101316]: osdmap e486: 8 total, 8 up, 8 in 2024-09-14T04:01:45.907 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:45 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:45.907 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:45 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:45.907 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:45 smithi129 ceph-mon[101316]: Reconfiguring mon.b (monmap changed)... 2024-09-14T04:01:45.907 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:45 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T04:01:45.908 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:45 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T04:01:45.908 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:45 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:45.908 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:45 smithi129 ceph-mon[101316]: Reconfiguring daemon mon.b on smithi129 2024-09-14T04:01:45.908 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:45 smithi129 ceph-mon[101316]: pgmap v220: 105 pgs: 105 active+clean; 583 KiB data, 444 MiB used, 715 GiB / 715 GiB avail; 171 KiB/s rd, 26 KiB/s wr, 250 op/s 2024-09-14T04:01:45.908 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:45 smithi129 ceph-mon[101316]: osdmap e487: 8 total, 8 up, 8 in 2024-09-14T04:01:46.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:45 smithi039 ceph-mon[106470]: osdmap e486: 8 total, 8 up, 8 in 2024-09-14T04:01:46.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:45 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:46.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:45 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:46.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:45 smithi039 ceph-mon[106470]: Reconfiguring mon.b (monmap changed)... 2024-09-14T04:01:46.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:45 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T04:01:46.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:45 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T04:01:46.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:45 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:46.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:45 smithi039 ceph-mon[106470]: Reconfiguring daemon mon.b on smithi129 2024-09-14T04:01:46.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:45 smithi039 ceph-mon[106470]: pgmap v220: 105 pgs: 105 active+clean; 583 KiB data, 444 MiB used, 715 GiB / 715 GiB avail; 171 KiB/s rd, 26 KiB/s wr, 250 op/s 2024-09-14T04:01:46.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:45 smithi039 ceph-mon[106470]: osdmap e487: 8 total, 8 up, 8 in 2024-09-14T04:01:46.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:45 smithi039 ceph-mon[111235]: osdmap e486: 8 total, 8 up, 8 in 2024-09-14T04:01:46.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:45 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:46.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:45 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:46.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:45 smithi039 ceph-mon[111235]: Reconfiguring mon.b (monmap changed)... 2024-09-14T04:01:46.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:45 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "mon."} : dispatch 2024-09-14T04:01:46.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:45 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config get", "who": "mon", "key": "public_network"} : dispatch 2024-09-14T04:01:46.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:45 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:46.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:45 smithi039 ceph-mon[111235]: Reconfiguring daemon mon.b on smithi129 2024-09-14T04:01:46.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:45 smithi039 ceph-mon[111235]: pgmap v220: 105 pgs: 105 active+clean; 583 KiB data, 444 MiB used, 715 GiB / 715 GiB avail; 171 KiB/s rd, 26 KiB/s wr, 250 op/s 2024-09-14T04:01:46.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:45 smithi039 ceph-mon[111235]: osdmap e487: 8 total, 8 up, 8 in 2024-09-14T04:01:46.409 INFO:teuthology.orchestra.run.smithi039.stdout:true 2024-09-14T04:01:46.994 INFO:teuthology.orchestra.run.smithi039.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T04:01:46.994 INFO:teuthology.orchestra.run.smithi039.stdout:alertmanager.a smithi039 *:9093,9094 running (3m) 67s ago 9m 21.0M - 0.27.0 11f11916f8cd 6b564e1fc909 2024-09-14T04:01:46.994 INFO:teuthology.orchestra.run.smithi039.stdout:grafana.a smithi129 *:3000 running (3m) 18s ago 9m 81.0M - 10.4.0 c8b91775d855 f0492c19de92 2024-09-14T04:01:46.994 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi039.bbwiwf smithi039 running (8m) 67s ago 8m 22.5M - 18.2.4-845-ge4e227b3 67ddd33fa72f 91e6c8f85681 2024-09-14T04:01:46.994 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi129.ybjqcw smithi129 running (8m) 18s ago 8m 16.0M - 18.2.4-845-ge4e227b3 67ddd33fa72f a3fab5b37b71 2024-09-14T04:01:46.994 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.x smithi129 *:8443,9283,8765 running (5m) 18s ago 13m 455M - 19.3.0-4907-ga2782315 8801d2529e43 448bd4e146cb 2024-09-14T04:01:46.994 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.y smithi039 *:8443,9283,8765 running (5m) 67s ago 15m 512M - 19.3.0-4907-ga2782315 8801d2529e43 cccc87f635a6 2024-09-14T04:01:46.994 INFO:teuthology.orchestra.run.smithi039.stdout:mon.a smithi039 running (2m) 67s ago 15m 49.5M 2048M 19.3.0-4907-ga2782315 8801d2529e43 7198d4eb8e4c 2024-09-14T04:01:46.994 INFO:teuthology.orchestra.run.smithi039.stdout:mon.b smithi129 running (29s) 18s ago 14m 22.1M 2048M 19.3.0-4907-ga2782315 8801d2529e43 cff2519560f1 2024-09-14T04:01:46.994 INFO:teuthology.orchestra.run.smithi039.stdout:mon.c smithi039 running (2m) 67s ago 14m 39.0M 2048M 19.3.0-4907-ga2782315 8801d2529e43 4b8f0f3864b1 2024-09-14T04:01:46.995 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.a smithi039 *:9100 running (4m) 67s ago 9m 11.4M - 1.7.0 72c9c2088986 7dd5945adb64 2024-09-14T04:01:46.995 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.b smithi129 *:9100 running (4m) 18s ago 9m 11.2M - 1.7.0 72c9c2088986 1fedb5f257e7 2024-09-14T04:01:46.995 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0 smithi039 running (13m) 67s ago 13m 115M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 5048d74f27ac 2024-09-14T04:01:46.995 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1 smithi039 running (13m) 67s ago 13m 130M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 91349527d88b 2024-09-14T04:01:46.995 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2 smithi039 running (12m) 67s ago 12m 164M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f eb903b44fc5d 2024-09-14T04:01:46.995 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3 smithi039 running (12m) 67s ago 12m 133M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f dff9e665bd9b 2024-09-14T04:01:46.995 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4 smithi129 running (11m) 18s ago 11m 124M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 5b64727311b7 2024-09-14T04:01:46.995 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5 smithi129 running (11m) 18s ago 11m 106M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f f2d991a754f7 2024-09-14T04:01:46.995 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6 smithi129 running (11m) 18s ago 11m 134M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 3060eaf44106 2024-09-14T04:01:46.995 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7 smithi129 running (10m) 18s ago 10m 110M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 4c13b97060d1 2024-09-14T04:01:46.995 INFO:teuthology.orchestra.run.smithi039.stdout:prometheus.a smithi129 *:9095 running (4m) 18s ago 10m 49.4M - 2.51.0 1d3b7f56885b cbe583f852c3 2024-09-14T04:01:47.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:47 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:47.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:47 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:47.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:47 smithi039 ceph-mon[106470]: Reconfiguring mgr.x (monmap changed)... 2024-09-14T04:01:47.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:47 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T04:01:47.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:47 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T04:01:47.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:47 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-14T04:01:47.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:47 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:47.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:47 smithi039 ceph-mon[106470]: Reconfiguring daemon mgr.x on smithi129 2024-09-14T04:01:47.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:47 smithi039 ceph-mon[106470]: osdmap e488: 8 total, 8 up, 8 in 2024-09-14T04:01:47.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:47 smithi039 ceph-mon[106470]: from='client.34257 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:47.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:47 smithi039 ceph-mon[106470]: from='client.34263 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:47.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:47 smithi039 ceph-mon[106470]: pgmap v223: 105 pgs: 105 active+clean; 583 KiB data, 444 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:01:47.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:47 smithi039 ceph-mon[106470]: from='client.44198 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:47.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:47 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:47.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:47 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:47.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:47 smithi039 ceph-mon[111235]: Reconfiguring mgr.x (monmap changed)... 2024-09-14T04:01:47.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:47 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T04:01:47.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:47 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T04:01:47.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:47 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-14T04:01:47.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:47 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:47.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:47 smithi039 ceph-mon[111235]: Reconfiguring daemon mgr.x on smithi129 2024-09-14T04:01:47.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:47 smithi039 ceph-mon[111235]: osdmap e488: 8 total, 8 up, 8 in 2024-09-14T04:01:47.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:47 smithi039 ceph-mon[111235]: from='client.34257 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:47.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:47 smithi039 ceph-mon[111235]: from='client.34263 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:47.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:47 smithi039 ceph-mon[111235]: pgmap v223: 105 pgs: 105 active+clean; 583 KiB data, 444 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:01:47.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:47 smithi039 ceph-mon[111235]: from='client.44198 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:47.346 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:01:47.347 INFO:teuthology.orchestra.run.smithi039.stdout: "mon": { 2024-09-14T04:01:47.347 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T04:01:47.347 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:01:47.347 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr": { 2024-09-14T04:01:47.347 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T04:01:47.347 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:01:47.347 INFO:teuthology.orchestra.run.smithi039.stdout: "osd": { 2024-09-14T04:01:47.347 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8 2024-09-14T04:01:47.347 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:01:47.347 INFO:teuthology.orchestra.run.smithi039.stdout: "mds": { 2024-09-14T04:01:47.347 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T04:01:47.348 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:01:47.348 INFO:teuthology.orchestra.run.smithi039.stdout: "overall": { 2024-09-14T04:01:47.348 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 10, 2024-09-14T04:01:47.348 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 5 2024-09-14T04:01:47.348 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:01:47.348 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:01:47.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:47 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:47.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:47 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:47.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:47 smithi129 ceph-mon[101316]: Reconfiguring mgr.x (monmap changed)... 2024-09-14T04:01:47.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:47 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T04:01:47.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:47 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]} : dispatch 2024-09-14T04:01:47.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:47 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "mgr services"} : dispatch 2024-09-14T04:01:47.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:47 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:47.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:47 smithi129 ceph-mon[101316]: Reconfiguring daemon mgr.x on smithi129 2024-09-14T04:01:47.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:47 smithi129 ceph-mon[101316]: osdmap e488: 8 total, 8 up, 8 in 2024-09-14T04:01:47.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:47 smithi129 ceph-mon[101316]: from='client.34257 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:47.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:47 smithi129 ceph-mon[101316]: from='client.34263 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:47.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:47 smithi129 ceph-mon[101316]: pgmap v223: 105 pgs: 105 active+clean; 583 KiB data, 444 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:01:47.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:47 smithi129 ceph-mon[101316]: from='client.44198 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:47.658 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:01:47.658 INFO:teuthology.orchestra.run.smithi039.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T04:01:47.658 INFO:teuthology.orchestra.run.smithi039.stdout: "in_progress": true, 2024-09-14T04:01:47.659 INFO:teuthology.orchestra.run.smithi039.stdout: "which": "Upgrading daemons of type(s) mon. Upgrade limited to 1 daemons (0 remaining).", 2024-09-14T04:01:47.659 INFO:teuthology.orchestra.run.smithi039.stdout: "services_complete": [ 2024-09-14T04:01:47.659 INFO:teuthology.orchestra.run.smithi039.stdout: "mon" 2024-09-14T04:01:47.659 INFO:teuthology.orchestra.run.smithi039.stdout: ], 2024-09-14T04:01:47.659 INFO:teuthology.orchestra.run.smithi039.stdout: "progress": "3/3 daemons upgraded", 2024-09-14T04:01:47.659 INFO:teuthology.orchestra.run.smithi039.stdout: "message": "Currently upgrading mon daemons", 2024-09-14T04:01:47.659 INFO:teuthology.orchestra.run.smithi039.stdout: "is_paused": false 2024-09-14T04:01:47.659 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:01:48.231 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:48 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3818667541' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:48.231 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:48 smithi129 ceph-mon[101316]: osdmap e489: 8 total, 8 up, 8 in 2024-09-14T04:01:48.231 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:48 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:48.231 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:48 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:48.232 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:48 smithi129 ceph-mon[101316]: Reconfiguring osd.4 (monmap changed)... 2024-09-14T04:01:48.232 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:48 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-14T04:01:48.232 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:48 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:48.232 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:48 smithi129 ceph-mon[101316]: Reconfiguring daemon osd.4 on smithi129 2024-09-14T04:01:48.232 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:48 smithi129 ceph-mon[101316]: from='client.34281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:48.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:48 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3818667541' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:48.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:48 smithi039 ceph-mon[106470]: osdmap e489: 8 total, 8 up, 8 in 2024-09-14T04:01:48.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:48 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:48.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:48 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:48.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:48 smithi039 ceph-mon[106470]: Reconfiguring osd.4 (monmap changed)... 2024-09-14T04:01:48.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:48 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-14T04:01:48.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:48 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:48.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:48 smithi039 ceph-mon[106470]: Reconfiguring daemon osd.4 on smithi129 2024-09-14T04:01:48.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:48 smithi039 ceph-mon[106470]: from='client.34281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:48.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:48 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3818667541' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:48.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:48 smithi039 ceph-mon[111235]: osdmap e489: 8 total, 8 up, 8 in 2024-09-14T04:01:48.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:48 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:48.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:48 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:48.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:48 smithi039 ceph-mon[111235]: Reconfiguring osd.4 (monmap changed)... 2024-09-14T04:01:48.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:48 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.4"} : dispatch 2024-09-14T04:01:48.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:48 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:48.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:48 smithi039 ceph-mon[111235]: Reconfiguring daemon osd.4 on smithi129 2024-09-14T04:01:48.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:48 smithi039 ceph-mon[111235]: from='client.34281 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:01:49.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:01:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:01:48] "GET /metrics HTTP/1.1" 200 36412 "" "Prometheus/2.51.0" 2024-09-14T04:01:49.696 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:49 smithi129 ceph-mon[101316]: osdmap e490: 8 total, 8 up, 8 in 2024-09-14T04:01:49.697 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:49 smithi129 ceph-mon[101316]: pgmap v226: 105 pgs: 105 active+clean; 583 KiB data, 445 MiB used, 715 GiB / 715 GiB avail; 125 KiB/s rd, 9.0 KiB/s wr, 164 op/s 2024-09-14T04:01:49.697 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:49 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:49.697 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:49 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:49.697 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:49 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-14T04:01:49.697 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:49 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:49.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:49 smithi039 ceph-mon[106470]: osdmap e490: 8 total, 8 up, 8 in 2024-09-14T04:01:49.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:49 smithi039 ceph-mon[106470]: pgmap v226: 105 pgs: 105 active+clean; 583 KiB data, 445 MiB used, 715 GiB / 715 GiB avail; 125 KiB/s rd, 9.0 KiB/s wr, 164 op/s 2024-09-14T04:01:49.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:49 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:49.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:49 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:49.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:49 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-14T04:01:49.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:49 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:49.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:49 smithi039 ceph-mon[111235]: osdmap e490: 8 total, 8 up, 8 in 2024-09-14T04:01:49.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:49 smithi039 ceph-mon[111235]: pgmap v226: 105 pgs: 105 active+clean; 583 KiB data, 445 MiB used, 715 GiB / 715 GiB avail; 125 KiB/s rd, 9.0 KiB/s wr, 164 op/s 2024-09-14T04:01:49.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:49 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:49.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:49 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:49.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:49 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.5"} : dispatch 2024-09-14T04:01:49.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:49 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:50.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:50 smithi129 ceph-mon[101316]: Reconfiguring osd.5 (monmap changed)... 2024-09-14T04:01:50.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:50 smithi129 ceph-mon[101316]: Reconfiguring daemon osd.5 on smithi129 2024-09-14T04:01:50.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:50 smithi129 ceph-mon[101316]: osdmap e491: 8 total, 8 up, 8 in 2024-09-14T04:01:50.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:50 smithi039 ceph-mon[106470]: Reconfiguring osd.5 (monmap changed)... 2024-09-14T04:01:50.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:50 smithi039 ceph-mon[106470]: Reconfiguring daemon osd.5 on smithi129 2024-09-14T04:01:50.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:50 smithi039 ceph-mon[106470]: osdmap e491: 8 total, 8 up, 8 in 2024-09-14T04:01:50.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:50 smithi039 ceph-mon[111235]: Reconfiguring osd.5 (monmap changed)... 2024-09-14T04:01:50.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:50 smithi039 ceph-mon[111235]: Reconfiguring daemon osd.5 on smithi129 2024-09-14T04:01:50.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:50 smithi039 ceph-mon[111235]: osdmap e491: 8 total, 8 up, 8 in 2024-09-14T04:01:51.615 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:51 smithi129 ceph-mon[101316]: osdmap e492: 8 total, 8 up, 8 in 2024-09-14T04:01:51.616 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:51 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:51.616 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:51 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:51.616 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:51 smithi129 ceph-mon[101316]: Reconfiguring osd.6 (monmap changed)... 2024-09-14T04:01:51.616 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:51 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-14T04:01:51.616 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:51 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:51.616 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:51 smithi129 ceph-mon[101316]: Reconfiguring daemon osd.6 on smithi129 2024-09-14T04:01:51.616 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:51 smithi129 ceph-mon[101316]: pgmap v229: 105 pgs: 105 active+clean; 583 KiB data, 445 MiB used, 715 GiB / 715 GiB avail; 125 KiB/s rd, 9.0 KiB/s wr, 164 op/s 2024-09-14T04:01:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:51 smithi039 ceph-mon[106470]: osdmap e492: 8 total, 8 up, 8 in 2024-09-14T04:01:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:51 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:51 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:51.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:51 smithi039 ceph-mon[106470]: Reconfiguring osd.6 (monmap changed)... 2024-09-14T04:01:51.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:51 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-14T04:01:51.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:51 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:51.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:51 smithi039 ceph-mon[106470]: Reconfiguring daemon osd.6 on smithi129 2024-09-14T04:01:51.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:51 smithi039 ceph-mon[106470]: pgmap v229: 105 pgs: 105 active+clean; 583 KiB data, 445 MiB used, 715 GiB / 715 GiB avail; 125 KiB/s rd, 9.0 KiB/s wr, 164 op/s 2024-09-14T04:01:51.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:51 smithi039 ceph-mon[111235]: osdmap e492: 8 total, 8 up, 8 in 2024-09-14T04:01:51.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:51 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:51.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:51 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:51.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:51 smithi039 ceph-mon[111235]: Reconfiguring osd.6 (monmap changed)... 2024-09-14T04:01:51.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:51 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.6"} : dispatch 2024-09-14T04:01:51.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:51 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:51.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:51 smithi039 ceph-mon[111235]: Reconfiguring daemon osd.6 on smithi129 2024-09-14T04:01:51.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:51 smithi039 ceph-mon[111235]: pgmap v229: 105 pgs: 105 active+clean; 583 KiB data, 445 MiB used, 715 GiB / 715 GiB avail; 125 KiB/s rd, 9.0 KiB/s wr, 164 op/s 2024-09-14T04:01:52.507 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.SnapshotPromoteDemote (9926 ms) 2024-09-14T04:01:52.508 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestMirroring.AioSnapshotCreate 2024-09-14T04:01:52.629 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:52 smithi129 ceph-mon[101316]: osdmap e493: 8 total, 8 up, 8 in 2024-09-14T04:01:52.629 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:52 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:52.629 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:52 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:52.629 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:52 smithi129 ceph-mon[101316]: Reconfiguring osd.7 (monmap changed)... 2024-09-14T04:01:52.630 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-14T04:01:52.630 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:52.630 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:52 smithi129 ceph-mon[101316]: Reconfiguring daemon osd.7 on smithi129 2024-09-14T04:01:52.630 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:52.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:52 smithi039 ceph-mon[106470]: osdmap e493: 8 total, 8 up, 8 in 2024-09-14T04:01:52.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:52 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:52.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:52 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:52.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:52 smithi039 ceph-mon[106470]: Reconfiguring osd.7 (monmap changed)... 2024-09-14T04:01:52.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-14T04:01:52.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:52.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:52 smithi039 ceph-mon[106470]: Reconfiguring daemon osd.7 on smithi129 2024-09-14T04:01:52.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:52.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:52 smithi039 ceph-mon[111235]: osdmap e493: 8 total, 8 up, 8 in 2024-09-14T04:01:52.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:52 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:52.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:52 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:52.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:52 smithi039 ceph-mon[111235]: Reconfiguring osd.7 (monmap changed)... 2024-09-14T04:01:52.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.7"} : dispatch 2024-09-14T04:01:52.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:52.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:52 smithi039 ceph-mon[111235]: Reconfiguring daemon osd.7 on smithi129 2024-09-14T04:01:52.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:01:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:53 smithi129 ceph-mon[101316]: osdmap e494: 8 total, 8 up, 8 in 2024-09-14T04:01:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:53 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/b0c706db-89a5-4a6d-b067-c5211d5a5109"} : dispatch 2024-09-14T04:01:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:53 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/b0c706db-89a5-4a6d-b067-c5211d5a5109"}]': finished 2024-09-14T04:01:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:53 smithi129 ceph-mon[101316]: pgmap v232: 105 pgs: 105 active+clean; 583 KiB data, 449 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 6.7 KiB/s wr, 56 op/s 2024-09-14T04:01:53.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:53 smithi039 ceph-mon[106470]: osdmap e494: 8 total, 8 up, 8 in 2024-09-14T04:01:53.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:53 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/b0c706db-89a5-4a6d-b067-c5211d5a5109"} : dispatch 2024-09-14T04:01:53.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:53 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/b0c706db-89a5-4a6d-b067-c5211d5a5109"}]': finished 2024-09-14T04:01:53.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:53 smithi039 ceph-mon[106470]: pgmap v232: 105 pgs: 105 active+clean; 583 KiB data, 449 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 6.7 KiB/s wr, 56 op/s 2024-09-14T04:01:53.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:53 smithi039 ceph-mon[111235]: osdmap e494: 8 total, 8 up, 8 in 2024-09-14T04:01:53.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:53 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/b0c706db-89a5-4a6d-b067-c5211d5a5109"} : dispatch 2024-09-14T04:01:53.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:53 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/b0c706db-89a5-4a6d-b067-c5211d5a5109"}]': finished 2024-09-14T04:01:53.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:53 smithi039 ceph-mon[111235]: pgmap v232: 105 pgs: 105 active+clean; 583 KiB data, 449 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 6.7 KiB/s wr, 56 op/s 2024-09-14T04:01:54.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:54 smithi129 ceph-mon[101316]: osdmap e495: 8 total, 8 up, 8 in 2024-09-14T04:01:54.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:54 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:54.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:54 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:54.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:54 smithi129 ceph-mon[101316]: Reconfiguring mds.foo.smithi129.ybjqcw (monmap changed)... 2024-09-14T04:01:54.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:54 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi129.ybjqcw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T04:01:54.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:54 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi129.ybjqcw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T04:01:54.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:54 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:54.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:54 smithi129 ceph-mon[101316]: Reconfiguring daemon mds.foo.smithi129.ybjqcw on smithi129 2024-09-14T04:01:54.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:54 smithi039 ceph-mon[106470]: osdmap e495: 8 total, 8 up, 8 in 2024-09-14T04:01:54.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:54 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:54.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:54 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:54.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:54 smithi039 ceph-mon[106470]: Reconfiguring mds.foo.smithi129.ybjqcw (monmap changed)... 2024-09-14T04:01:54.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:54 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi129.ybjqcw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T04:01:54.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:54 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi129.ybjqcw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T04:01:54.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:54 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:54.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:54 smithi039 ceph-mon[106470]: Reconfiguring daemon mds.foo.smithi129.ybjqcw on smithi129 2024-09-14T04:01:54.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:54 smithi039 ceph-mon[111235]: osdmap e495: 8 total, 8 up, 8 in 2024-09-14T04:01:54.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:54 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:54.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:54 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:54.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:54 smithi039 ceph-mon[111235]: Reconfiguring mds.foo.smithi129.ybjqcw (monmap changed)... 2024-09-14T04:01:54.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:54 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi129.ybjqcw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T04:01:54.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:54 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "auth get-or-create", "entity": "mds.foo.smithi129.ybjqcw", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]} : dispatch 2024-09-14T04:01:54.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:54 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:01:54.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:54 smithi039 ceph-mon[111235]: Reconfiguring daemon mds.foo.smithi129.ybjqcw on smithi129 2024-09-14T04:01:55.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: osdmap e496: 8 total, 8 up, 8 in 2024-09-14T04:01:55.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: pgmap v235: 105 pgs: 105 active+clean; 583 KiB data, 449 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 6.7 KiB/s wr, 56 op/s 2024-09-14T04:01:55.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:55.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:55.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:01:55.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:55.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:55.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:55.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:55.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:55.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-14T04:01:55.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:55.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:55.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:55.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:55.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:55.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.775 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T04:01:55.775 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T04:01:55.775 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T04:01:55.775 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.775 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.775 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T04:01:55.776 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T04:01:55.776 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T04:01:55.776 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T04:01:55.776 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T04:01:55.776 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T04:01:55.776 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T04:01:55.776 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T04:01:55.776 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T04:01:55.776 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T04:01:55.776 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T04:01:55.776 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.776 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T04:01:55.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T04:01:55.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T04:01:55.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T04:01:55.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T04:01:55.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T04:01:55.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T04:01:55.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T04:01:55.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T04:01:55.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T04:01:55.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T04:01:55.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T04:01:55.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:01:55.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:55 smithi129 ceph-mon[101316]: osdmap e497: 8 total, 8 up, 8 in 2024-09-14T04:01:55.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: osdmap e496: 8 total, 8 up, 8 in 2024-09-14T04:01:55.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: pgmap v235: 105 pgs: 105 active+clean; 583 KiB data, 449 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 6.7 KiB/s wr, 56 op/s 2024-09-14T04:01:55.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:55.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:55.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:01:55.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:55.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:55.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:55.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:55.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:55.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-14T04:01:55.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:55.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:55.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:55.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:55.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:55.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: osdmap e496: 8 total, 8 up, 8 in 2024-09-14T04:01:55.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: pgmap v235: 105 pgs: 105 active+clean; 583 KiB data, 449 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 6.7 KiB/s wr, 56 op/s 2024-09-14T04:01:55.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:55.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:55.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:01:55.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:55.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:55.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:55.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:55.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:55.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-09-14T04:01:55.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:55.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:55.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:55.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:55.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:01:55.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon.b"} : dispatch 2024-09-14T04:01:55.795 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T04:01:55.795 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T04:01:55.795 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T04:01:55.795 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.795 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.795 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T04:01:55.795 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T04:01:55.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T04:01:55.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T04:01:55.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T04:01:55.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T04:01:55.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T04:01:55.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T04:01:55.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T04:01:55.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T04:01:55.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T04:01:55.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T04:01:55.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T04:01:55.797 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T04:01:55.797 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T04:01:55.797 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T04:01:55.797 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T04:01:55.797 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T04:01:55.797 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T04:01:55.797 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.797 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.798 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T04:01:55.798 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T04:01:55.798 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T04:01:55.798 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T04:01:55.798 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T04:01:55.798 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T04:01:55.798 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T04:01:55.798 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T04:01:55.798 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T04:01:55.798 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T04:01:55.798 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T04:01:55.798 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.798 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.799 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T04:01:55.799 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T04:01:55.799 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T04:01:55.799 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T04:01:55.799 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T04:01:55.799 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T04:01:55.799 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T04:01:55.799 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T04:01:55.799 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T04:01:55.799 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.799 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.799 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.799 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.800 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.800 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.800 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.800 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.800 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.800 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.800 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.800 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.800 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T04:01:55.800 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T04:01:55.800 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T04:01:55.800 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:01:55.800 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[106470]: osdmap e497: 8 total, 8 up, 8 in 2024-09-14T04:01:55.801 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T04:01:55.801 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T04:01:55.801 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T04:01:55.801 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T04:01:55.801 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.801 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.801 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.801 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.801 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.801 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.801 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.801 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.801 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.802 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.802 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.802 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:01:55.802 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T04:01:55.802 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T04:01:55.802 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T04:01:55.802 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:01:55.802 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:55 smithi039 ceph-mon[111235]: osdmap e497: 8 total, 8 up, 8 in 2024-09-14T04:01:56.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:56 smithi129 ceph-mon[101316]: Upgrade: Setting container_image for all nvmeof 2024-09-14T04:01:56.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:56 smithi129 ceph-mon[101316]: Upgrade: Setting container_image for all node-exporter 2024-09-14T04:01:56.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:56 smithi129 ceph-mon[101316]: Upgrade: Setting container_image for all prometheus 2024-09-14T04:01:56.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:56 smithi129 ceph-mon[101316]: Upgrade: Setting container_image for all alertmanager 2024-09-14T04:01:56.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:56 smithi129 ceph-mon[101316]: Upgrade: Setting container_image for all grafana 2024-09-14T04:01:56.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:56 smithi129 ceph-mon[101316]: Upgrade: Setting container_image for all loki 2024-09-14T04:01:56.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:56 smithi129 ceph-mon[101316]: Upgrade: Setting container_image for all promtail 2024-09-14T04:01:56.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:56 smithi129 ceph-mon[101316]: Upgrade: Finalizing container_image settings 2024-09-14T04:01:56.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:56 smithi129 ceph-mon[101316]: Upgrade: Complete! 2024-09-14T04:01:56.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:56 smithi129 ceph-mon[101316]: osdmap e498: 8 total, 8 up, 8 in 2024-09-14T04:01:56.773 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:56 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all nvmeof 2024-09-14T04:01:56.773 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:56 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all node-exporter 2024-09-14T04:01:56.773 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:56 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all prometheus 2024-09-14T04:01:56.773 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:56 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all alertmanager 2024-09-14T04:01:56.773 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:56 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all grafana 2024-09-14T04:01:56.773 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:56 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all loki 2024-09-14T04:01:56.773 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:56 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all promtail 2024-09-14T04:01:56.773 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:56 smithi039 ceph-mon[106470]: Upgrade: Finalizing container_image settings 2024-09-14T04:01:56.773 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:56 smithi039 ceph-mon[106470]: Upgrade: Complete! 2024-09-14T04:01:56.773 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:56 smithi039 ceph-mon[106470]: osdmap e498: 8 total, 8 up, 8 in 2024-09-14T04:01:56.774 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:56 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all nvmeof 2024-09-14T04:01:56.774 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:56 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all node-exporter 2024-09-14T04:01:56.774 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:56 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all prometheus 2024-09-14T04:01:56.774 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:56 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all alertmanager 2024-09-14T04:01:56.774 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:56 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all grafana 2024-09-14T04:01:56.774 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:56 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all loki 2024-09-14T04:01:56.774 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:56 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all promtail 2024-09-14T04:01:56.774 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:56 smithi039 ceph-mon[111235]: Upgrade: Finalizing container_image settings 2024-09-14T04:01:56.774 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:56 smithi039 ceph-mon[111235]: Upgrade: Complete! 2024-09-14T04:01:56.774 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:56 smithi039 ceph-mon[111235]: osdmap e498: 8 total, 8 up, 8 in 2024-09-14T04:01:57.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:57 smithi039 ceph-mon[106470]: pgmap v238: 105 pgs: 105 active+clean; 583 KiB data, 449 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:01:57.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:57 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:57.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:57 smithi039 ceph-mon[106470]: osdmap e499: 8 total, 8 up, 8 in 2024-09-14T04:01:57.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:57 smithi039 ceph-mon[111235]: pgmap v238: 105 pgs: 105 active+clean; 583 KiB data, 449 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:01:57.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:57 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:57.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:57 smithi039 ceph-mon[111235]: osdmap e499: 8 total, 8 up, 8 in 2024-09-14T04:01:58.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:57 smithi129 ceph-mon[101316]: pgmap v238: 105 pgs: 105 active+clean; 583 KiB data, 449 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:01:58.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:57 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:58.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:57 smithi129 ceph-mon[101316]: osdmap e499: 8 total, 8 up, 8 in 2024-09-14T04:01:59.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:01:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:01:58] "GET /metrics HTTP/1.1" 200 36406 "" "Prometheus/2.51.0" 2024-09-14T04:01:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:59 smithi129 ceph-mon[101316]: osdmap e500: 8 total, 8 up, 8 in 2024-09-14T04:01:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:59 smithi129 ceph-mon[101316]: pgmap v241: 105 pgs: 105 active+clean; 583 KiB data, 450 MiB used, 715 GiB / 715 GiB avail; 82 KiB/s rd, 14 KiB/s wr, 119 op/s 2024-09-14T04:01:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:01:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:59.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:59 smithi039 ceph-mon[111235]: osdmap e500: 8 total, 8 up, 8 in 2024-09-14T04:01:59.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:59 smithi039 ceph-mon[111235]: pgmap v241: 105 pgs: 105 active+clean; 583 KiB data, 450 MiB used, 715 GiB / 715 GiB avail; 82 KiB/s rd, 14 KiB/s wr, 119 op/s 2024-09-14T04:01:59.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:59.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:01:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:59.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:59 smithi039 ceph-mon[106470]: osdmap e500: 8 total, 8 up, 8 in 2024-09-14T04:01:59.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:59 smithi039 ceph-mon[106470]: pgmap v241: 105 pgs: 105 active+clean; 583 KiB data, 450 MiB used, 715 GiB / 715 GiB avail; 82 KiB/s rd, 14 KiB/s wr, 119 op/s 2024-09-14T04:01:59.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:01:59.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:01:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:02:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:00 smithi129 ceph-mon[101316]: osdmap e501: 8 total, 8 up, 8 in 2024-09-14T04:02:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:00 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:02:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:00 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:02:00.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:00 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:02:00.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:00 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:02:00.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:00 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:02:00.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:00 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:02:00.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:00 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:02:00.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:00 smithi039 ceph-mon[106470]: osdmap e501: 8 total, 8 up, 8 in 2024-09-14T04:02:00.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:00 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:02:00.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:00 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:02:00.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:00 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:02:00.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:00 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:02:00.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:00 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:02:00.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:00 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:02:00.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:00 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:02:00.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:00 smithi039 ceph-mon[111235]: osdmap e501: 8 total, 8 up, 8 in 2024-09-14T04:02:00.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:00 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:02:00.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:00 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:02:00.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:00 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:02:00.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:00 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:02:00.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:00 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:02:00.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:00 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:02:00.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:00 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:02:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:01 smithi129 ceph-mon[101316]: osdmap e502: 8 total, 8 up, 8 in 2024-09-14T04:02:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:01 smithi129 ceph-mon[101316]: pgmap v244: 105 pgs: 105 active+clean; 583 KiB data, 450 MiB used, 715 GiB / 715 GiB avail; 82 KiB/s rd, 14 KiB/s wr, 119 op/s 2024-09-14T04:02:01.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:01 smithi039 ceph-mon[106470]: osdmap e502: 8 total, 8 up, 8 in 2024-09-14T04:02:01.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:01 smithi039 ceph-mon[106470]: pgmap v244: 105 pgs: 105 active+clean; 583 KiB data, 450 MiB used, 715 GiB / 715 GiB avail; 82 KiB/s rd, 14 KiB/s wr, 119 op/s 2024-09-14T04:02:01.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:01 smithi039 ceph-mon[111235]: osdmap e502: 8 total, 8 up, 8 in 2024-09-14T04:02:01.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:01 smithi039 ceph-mon[111235]: pgmap v244: 105 pgs: 105 active+clean; 583 KiB data, 450 MiB used, 715 GiB / 715 GiB avail; 82 KiB/s rd, 14 KiB/s wr, 119 op/s 2024-09-14T04:02:02.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:02 smithi129 ceph-mon[101316]: osdmap e503: 8 total, 8 up, 8 in 2024-09-14T04:02:02.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:02 smithi039 ceph-mon[106470]: osdmap e503: 8 total, 8 up, 8 in 2024-09-14T04:02:02.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:02 smithi039 ceph-mon[111235]: osdmap e503: 8 total, 8 up, 8 in 2024-09-14T04:02:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:03 smithi129 ceph-mon[101316]: osdmap e504: 8 total, 8 up, 8 in 2024-09-14T04:02:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:03 smithi129 ceph-mon[101316]: pgmap v247: 105 pgs: 105 active+clean; 584 KiB data, 455 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 13 KiB/s wr, 112 op/s 2024-09-14T04:02:03.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:03 smithi039 ceph-mon[106470]: osdmap e504: 8 total, 8 up, 8 in 2024-09-14T04:02:03.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:03 smithi039 ceph-mon[106470]: pgmap v247: 105 pgs: 105 active+clean; 584 KiB data, 455 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 13 KiB/s wr, 112 op/s 2024-09-14T04:02:03.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:03 smithi039 ceph-mon[111235]: osdmap e504: 8 total, 8 up, 8 in 2024-09-14T04:02:03.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:03 smithi039 ceph-mon[111235]: pgmap v247: 105 pgs: 105 active+clean; 584 KiB data, 455 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 13 KiB/s wr, 112 op/s 2024-09-14T04:02:04.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:04 smithi129 ceph-mon[101316]: osdmap e505: 8 total, 8 up, 8 in 2024-09-14T04:02:04.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:04 smithi039 ceph-mon[106470]: osdmap e505: 8 total, 8 up, 8 in 2024-09-14T04:02:04.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:04 smithi039 ceph-mon[111235]: osdmap e505: 8 total, 8 up, 8 in 2024-09-14T04:02:05.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:05 smithi039 ceph-mon[106470]: osdmap e506: 8 total, 8 up, 8 in 2024-09-14T04:02:05.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:05 smithi039 ceph-mon[106470]: pgmap v250: 105 pgs: 105 active+clean; 584 KiB data, 455 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 13 KiB/s wr, 112 op/s 2024-09-14T04:02:05.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:05 smithi039 ceph-mon[111235]: osdmap e506: 8 total, 8 up, 8 in 2024-09-14T04:02:05.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:05 smithi039 ceph-mon[111235]: pgmap v250: 105 pgs: 105 active+clean; 584 KiB data, 455 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 13 KiB/s wr, 112 op/s 2024-09-14T04:02:06.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:05 smithi129 ceph-mon[101316]: osdmap e506: 8 total, 8 up, 8 in 2024-09-14T04:02:06.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:05 smithi129 ceph-mon[101316]: pgmap v250: 105 pgs: 105 active+clean; 584 KiB data, 455 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 13 KiB/s wr, 112 op/s 2024-09-14T04:02:07.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:06 smithi129 ceph-mon[101316]: osdmap e507: 8 total, 8 up, 8 in 2024-09-14T04:02:07.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:06 smithi039 ceph-mon[106470]: osdmap e507: 8 total, 8 up, 8 in 2024-09-14T04:02:07.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:06 smithi039 ceph-mon[111235]: osdmap e507: 8 total, 8 up, 8 in 2024-09-14T04:02:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:07 smithi129 ceph-mon[101316]: osdmap e508: 8 total, 8 up, 8 in 2024-09-14T04:02:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:07 smithi129 ceph-mon[101316]: pgmap v253: 105 pgs: 105 active+clean; 584 KiB data, 455 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:02:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:08.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:07 smithi039 ceph-mon[106470]: osdmap e508: 8 total, 8 up, 8 in 2024-09-14T04:02:08.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:07 smithi039 ceph-mon[106470]: pgmap v253: 105 pgs: 105 active+clean; 584 KiB data, 455 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:02:08.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:08.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:07 smithi039 ceph-mon[111235]: osdmap e508: 8 total, 8 up, 8 in 2024-09-14T04:02:08.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:07 smithi039 ceph-mon[111235]: pgmap v253: 105 pgs: 105 active+clean; 584 KiB data, 455 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:02:08.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:09.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:08 smithi129 ceph-mon[101316]: osdmap e509: 8 total, 8 up, 8 in 2024-09-14T04:02:09.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:08 smithi039 ceph-mon[106470]: osdmap e509: 8 total, 8 up, 8 in 2024-09-14T04:02:09.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:08 smithi039 ceph-mon[111235]: osdmap e509: 8 total, 8 up, 8 in 2024-09-14T04:02:09.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:02:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:02:08] "GET /metrics HTTP/1.1" 200 36417 "" "Prometheus/2.51.0" 2024-09-14T04:02:10.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:09 smithi129 ceph-mon[101316]: osdmap e510: 8 total, 8 up, 8 in 2024-09-14T04:02:10.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:09 smithi129 ceph-mon[101316]: pgmap v256: 105 pgs: 105 active+clean; 584 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 195 KiB/s rd, 20 KiB/s wr, 267 op/s 2024-09-14T04:02:10.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:09 smithi039 ceph-mon[106470]: osdmap e510: 8 total, 8 up, 8 in 2024-09-14T04:02:10.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:09 smithi039 ceph-mon[106470]: pgmap v256: 105 pgs: 105 active+clean; 584 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 195 KiB/s rd, 20 KiB/s wr, 267 op/s 2024-09-14T04:02:10.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:09 smithi039 ceph-mon[111235]: osdmap e510: 8 total, 8 up, 8 in 2024-09-14T04:02:10.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:09 smithi039 ceph-mon[111235]: pgmap v256: 105 pgs: 105 active+clean; 584 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 195 KiB/s rd, 20 KiB/s wr, 267 op/s 2024-09-14T04:02:11.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:10 smithi129 ceph-mon[101316]: osdmap e511: 8 total, 8 up, 8 in 2024-09-14T04:02:11.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:10 smithi129 ceph-mon[101316]: osdmap e512: 8 total, 8 up, 8 in 2024-09-14T04:02:11.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:10 smithi039 ceph-mon[106470]: osdmap e511: 8 total, 8 up, 8 in 2024-09-14T04:02:11.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:10 smithi039 ceph-mon[106470]: osdmap e512: 8 total, 8 up, 8 in 2024-09-14T04:02:11.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:10 smithi039 ceph-mon[111235]: osdmap e511: 8 total, 8 up, 8 in 2024-09-14T04:02:11.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:10 smithi039 ceph-mon[111235]: osdmap e512: 8 total, 8 up, 8 in 2024-09-14T04:02:12.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:11 smithi129 ceph-mon[101316]: pgmap v259: 105 pgs: 105 active+clean; 584 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 195 KiB/s rd, 20 KiB/s wr, 267 op/s 2024-09-14T04:02:12.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:11 smithi129 ceph-mon[101316]: osdmap e513: 8 total, 8 up, 8 in 2024-09-14T04:02:12.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:11 smithi039 ceph-mon[106470]: pgmap v259: 105 pgs: 105 active+clean; 584 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 195 KiB/s rd, 20 KiB/s wr, 267 op/s 2024-09-14T04:02:12.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:11 smithi039 ceph-mon[106470]: osdmap e513: 8 total, 8 up, 8 in 2024-09-14T04:02:12.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:11 smithi039 ceph-mon[111235]: pgmap v259: 105 pgs: 105 active+clean; 584 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 195 KiB/s rd, 20 KiB/s wr, 267 op/s 2024-09-14T04:02:12.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:11 smithi039 ceph-mon[111235]: osdmap e513: 8 total, 8 up, 8 in 2024-09-14T04:02:13.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:13 smithi129 ceph-mon[101316]: osdmap e514: 8 total, 8 up, 8 in 2024-09-14T04:02:13.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:13 smithi129 ceph-mon[101316]: pgmap v262: 105 pgs: 105 active+clean; 584 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s 2024-09-14T04:02:13.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:13 smithi039 ceph-mon[106470]: osdmap e514: 8 total, 8 up, 8 in 2024-09-14T04:02:13.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:13 smithi039 ceph-mon[106470]: pgmap v262: 105 pgs: 105 active+clean; 584 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s 2024-09-14T04:02:13.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:13 smithi039 ceph-mon[111235]: osdmap e514: 8 total, 8 up, 8 in 2024-09-14T04:02:13.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:13 smithi039 ceph-mon[111235]: pgmap v262: 105 pgs: 105 active+clean; 584 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s 2024-09-14T04:02:14.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:14 smithi129 ceph-mon[101316]: osdmap e515: 8 total, 8 up, 8 in 2024-09-14T04:02:14.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:14 smithi039 ceph-mon[106470]: osdmap e515: 8 total, 8 up, 8 in 2024-09-14T04:02:14.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:14 smithi039 ceph-mon[111235]: osdmap e515: 8 total, 8 up, 8 in 2024-09-14T04:02:15.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:15 smithi129 ceph-mon[101316]: osdmap e516: 8 total, 8 up, 8 in 2024-09-14T04:02:15.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:15 smithi129 ceph-mon[101316]: pgmap v265: 105 pgs: 105 active+clean; 584 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s 2024-09-14T04:02:15.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:15 smithi129 ceph-mon[101316]: osdmap e517: 8 total, 8 up, 8 in 2024-09-14T04:02:15.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:15 smithi039 ceph-mon[106470]: osdmap e516: 8 total, 8 up, 8 in 2024-09-14T04:02:15.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:15 smithi039 ceph-mon[106470]: pgmap v265: 105 pgs: 105 active+clean; 584 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s 2024-09-14T04:02:15.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:15 smithi039 ceph-mon[106470]: osdmap e517: 8 total, 8 up, 8 in 2024-09-14T04:02:15.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:15 smithi039 ceph-mon[111235]: osdmap e516: 8 total, 8 up, 8 in 2024-09-14T04:02:15.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:15 smithi039 ceph-mon[111235]: pgmap v265: 105 pgs: 105 active+clean; 584 KiB data, 440 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s 2024-09-14T04:02:15.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:15 smithi039 ceph-mon[111235]: osdmap e517: 8 total, 8 up, 8 in 2024-09-14T04:02:17.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:17 smithi129 ceph-mon[101316]: osdmap e518: 8 total, 8 up, 8 in 2024-09-14T04:02:17.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:17 smithi129 ceph-mon[101316]: pgmap v268: 105 pgs: 105 active+clean; 584 KiB data, 440 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:02:17.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:17 smithi039 ceph-mon[106470]: osdmap e518: 8 total, 8 up, 8 in 2024-09-14T04:02:17.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:17 smithi039 ceph-mon[106470]: pgmap v268: 105 pgs: 105 active+clean; 584 KiB data, 440 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:02:17.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:17 smithi039 ceph-mon[111235]: osdmap e518: 8 total, 8 up, 8 in 2024-09-14T04:02:17.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:17 smithi039 ceph-mon[111235]: pgmap v268: 105 pgs: 105 active+clean; 584 KiB data, 440 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:02:18.648 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:18 smithi039 ceph-mon[106470]: osdmap e519: 8 total, 8 up, 8 in 2024-09-14T04:02:18.648 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:18 smithi039 ceph-mon[106470]: from='client.34287 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:02:18.649 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:18 smithi039 ceph-mon[111235]: osdmap e519: 8 total, 8 up, 8 in 2024-09-14T04:02:18.649 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:18 smithi039 ceph-mon[111235]: from='client.34287 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:02:18.721 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'sleep 60' 2024-09-14T04:02:18.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:18 smithi129 ceph-mon[101316]: osdmap e519: 8 total, 8 up, 8 in 2024-09-14T04:02:18.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:18 smithi129 ceph-mon[101316]: from='client.34287 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:02:18.929 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:02:19.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:02:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:02:18] "GET /metrics HTTP/1.1" 200 36417 "" "Prometheus/2.51.0" 2024-09-14T04:02:19.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:19 smithi129 ceph-mon[101316]: osdmap e520: 8 total, 8 up, 8 in 2024-09-14T04:02:19.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:19 smithi129 ceph-mon[101316]: pgmap v271: 105 pgs: 105 active+clean; 583 KiB data, 445 MiB used, 715 GiB / 715 GiB avail; 111 KiB/s rd, 8.7 KiB/s wr, 160 op/s 2024-09-14T04:02:19.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:19 smithi039 ceph-mon[106470]: osdmap e520: 8 total, 8 up, 8 in 2024-09-14T04:02:19.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:19 smithi039 ceph-mon[106470]: pgmap v271: 105 pgs: 105 active+clean; 583 KiB data, 445 MiB used, 715 GiB / 715 GiB avail; 111 KiB/s rd, 8.7 KiB/s wr, 160 op/s 2024-09-14T04:02:19.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:19 smithi039 ceph-mon[111235]: osdmap e520: 8 total, 8 up, 8 in 2024-09-14T04:02:19.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:19 smithi039 ceph-mon[111235]: pgmap v271: 105 pgs: 105 active+clean; 583 KiB data, 445 MiB used, 715 GiB / 715 GiB avail; 111 KiB/s rd, 8.7 KiB/s wr, 160 op/s 2024-09-14T04:02:20.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:20 smithi129 ceph-mon[101316]: osdmap e521: 8 total, 8 up, 8 in 2024-09-14T04:02:20.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:20 smithi129 ceph-mon[101316]: osdmap e522: 8 total, 8 up, 8 in 2024-09-14T04:02:20.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:20 smithi039 ceph-mon[106470]: osdmap e521: 8 total, 8 up, 8 in 2024-09-14T04:02:20.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:20 smithi039 ceph-mon[106470]: osdmap e522: 8 total, 8 up, 8 in 2024-09-14T04:02:20.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:20 smithi039 ceph-mon[111235]: osdmap e521: 8 total, 8 up, 8 in 2024-09-14T04:02:20.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:20 smithi039 ceph-mon[111235]: osdmap e522: 8 total, 8 up, 8 in 2024-09-14T04:02:21.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:21 smithi129 ceph-mon[101316]: pgmap v274: 105 pgs: 105 active+clean; 583 KiB data, 445 MiB used, 715 GiB / 715 GiB avail; 111 KiB/s rd, 8.7 KiB/s wr, 160 op/s 2024-09-14T04:02:21.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:21 smithi129 ceph-mon[101316]: osdmap e523: 8 total, 8 up, 8 in 2024-09-14T04:02:21.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:21 smithi039 ceph-mon[106470]: pgmap v274: 105 pgs: 105 active+clean; 583 KiB data, 445 MiB used, 715 GiB / 715 GiB avail; 111 KiB/s rd, 8.7 KiB/s wr, 160 op/s 2024-09-14T04:02:21.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:21 smithi039 ceph-mon[106470]: osdmap e523: 8 total, 8 up, 8 in 2024-09-14T04:02:21.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:21 smithi039 ceph-mon[111235]: pgmap v274: 105 pgs: 105 active+clean; 583 KiB data, 445 MiB used, 715 GiB / 715 GiB avail; 111 KiB/s rd, 8.7 KiB/s wr, 160 op/s 2024-09-14T04:02:21.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:21 smithi039 ceph-mon[111235]: osdmap e523: 8 total, 8 up, 8 in 2024-09-14T04:02:22.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:22.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:22.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:23.561 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestMirroring.AioSnapshotCreate (31054 ms) 2024-09-14T04:02:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:23 smithi129 ceph-mon[101316]: osdmap e524: 8 total, 8 up, 8 in 2024-09-14T04:02:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:23 smithi129 ceph-mon[101316]: pgmap v277: 105 pgs: 105 active+clean; 583 KiB data, 449 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s 2024-09-14T04:02:23.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:23 smithi039 ceph-mon[106470]: osdmap e524: 8 total, 8 up, 8 in 2024-09-14T04:02:23.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:23 smithi039 ceph-mon[106470]: pgmap v277: 105 pgs: 105 active+clean; 583 KiB data, 449 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s 2024-09-14T04:02:23.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:23 smithi039 ceph-mon[111235]: osdmap e524: 8 total, 8 up, 8 in 2024-09-14T04:02:23.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:23 smithi039 ceph-mon[111235]: pgmap v277: 105 pgs: 105 active+clean; 583 KiB data, 449 MiB used, 715 GiB / 715 GiB avail; 80 KiB/s rd, 7.5 KiB/s wr, 116 op/s 2024-09-14T04:02:24.562 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 51 tests from TestMirroring (93490 ms total) 2024-09-14T04:02:24.562 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:02:24.562 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 34 tests from TestInternal 2024-09-14T04:02:24.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:24 smithi129 ceph-mon[101316]: osdmap e525: 8 total, 8 up, 8 in 2024-09-14T04:02:24.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:24 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/35abbfd3-fd02-4b97-9f2d-a1e6564d95fd"} : dispatch 2024-09-14T04:02:24.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:24 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/35abbfd3-fd02-4b97-9f2d-a1e6564d95fd"}]': finished 2024-09-14T04:02:24.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:24 smithi039 ceph-mon[111235]: osdmap e525: 8 total, 8 up, 8 in 2024-09-14T04:02:24.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:24 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/35abbfd3-fd02-4b97-9f2d-a1e6564d95fd"} : dispatch 2024-09-14T04:02:24.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:24 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/35abbfd3-fd02-4b97-9f2d-a1e6564d95fd"}]': finished 2024-09-14T04:02:24.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:24 smithi039 ceph-mon[106470]: osdmap e525: 8 total, 8 up, 8 in 2024-09-14T04:02:24.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:24 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/16/35abbfd3-fd02-4b97-9f2d-a1e6564d95fd"} : dispatch 2024-09-14T04:02:24.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:24 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1818078297' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/16/35abbfd3-fd02-4b97-9f2d-a1e6564d95fd"}]': finished 2024-09-14T04:02:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:25 smithi129 ceph-mon[101316]: osdmap e526: 8 total, 8 up, 8 in 2024-09-14T04:02:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:25 smithi129 ceph-mon[101316]: pgmap v280: 73 pgs: 73 active+clean; 582 KiB data, 449 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:02:26.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:25 smithi039 ceph-mon[106470]: osdmap e526: 8 total, 8 up, 8 in 2024-09-14T04:02:26.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:25 smithi039 ceph-mon[106470]: pgmap v280: 73 pgs: 73 active+clean; 582 KiB data, 449 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:02:26.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:25 smithi039 ceph-mon[111235]: osdmap e526: 8 total, 8 up, 8 in 2024-09-14T04:02:26.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:25 smithi039 ceph-mon[111235]: pgmap v280: 73 pgs: 73 active+clean; 582 KiB data, 449 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:02:26.588 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.OpenByID 2024-09-14T04:02:27.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:26 smithi129 ceph-mon[101316]: osdmap e527: 8 total, 8 up, 8 in 2024-09-14T04:02:27.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:26 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2626081537' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:27.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:26 smithi039 ceph-mon[106470]: osdmap e527: 8 total, 8 up, 8 in 2024-09-14T04:02:27.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:26 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2626081537' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:27.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:26 smithi039 ceph-mon[111235]: osdmap e527: 8 total, 8 up, 8 in 2024-09-14T04:02:27.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:26 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2626081537' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-13","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:02:28.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:27 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2626081537' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:28.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:27 smithi129 ceph-mon[101316]: osdmap e528: 8 total, 8 up, 8 in 2024-09-14T04:02:28.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:27 smithi129 ceph-mon[101316]: pgmap v283: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 449 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:02:28.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:27 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2626081537' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:28.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:27 smithi039 ceph-mon[106470]: osdmap e528: 8 total, 8 up, 8 in 2024-09-14T04:02:28.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:27 smithi039 ceph-mon[106470]: pgmap v283: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 449 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:02:28.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:27 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2626081537' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-13","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:02:28.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:27 smithi039 ceph-mon[111235]: osdmap e528: 8 total, 8 up, 8 in 2024-09-14T04:02:28.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:27 smithi039 ceph-mon[111235]: pgmap v283: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 449 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:02:28.658 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.OpenByID (2070 ms) 2024-09-14T04:02:28.659 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.OpenSnapDNE 2024-09-14T04:02:28.682 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.OpenSnapDNE (24 ms) 2024-09-14T04:02:28.683 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.IsExclusiveLockOwner 2024-09-14T04:02:28.707 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.IsExclusiveLockOwner (25 ms) 2024-09-14T04:02:28.707 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.ResizeLocksImage 2024-09-14T04:02:28.734 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.ResizeLocksImage (26 ms) 2024-09-14T04:02:28.734 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.ResizeFailsToLockImage 2024-09-14T04:02:28.753 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.ResizeFailsToLockImage (20 ms) 2024-09-14T04:02:28.753 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.SnapCreateLocksImage 2024-09-14T04:02:28.941 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:28 smithi039 ceph-mon[106470]: osdmap e529: 8 total, 8 up, 8 in 2024-09-14T04:02:28.942 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:02:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:02:28] "GET /metrics HTTP/1.1" 200 36409 "" "Prometheus/2.51.0" 2024-09-14T04:02:28.942 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:28 smithi039 ceph-mon[111235]: osdmap e529: 8 total, 8 up, 8 in 2024-09-14T04:02:29.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:28 smithi129 ceph-mon[101316]: osdmap e529: 8 total, 8 up, 8 in 2024-09-14T04:02:29.958 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:29 smithi039 ceph-mon[106470]: osdmap e530: 8 total, 8 up, 8 in 2024-09-14T04:02:29.958 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:29 smithi039 ceph-mon[106470]: pgmap v286: 105 pgs: 105 active+clean; 582 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T04:02:29.958 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:29 smithi039 ceph-mon[111235]: osdmap e530: 8 total, 8 up, 8 in 2024-09-14T04:02:29.959 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:29 smithi039 ceph-mon[111235]: pgmap v286: 105 pgs: 105 active+clean; 582 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T04:02:30.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:29 smithi129 ceph-mon[101316]: osdmap e530: 8 total, 8 up, 8 in 2024-09-14T04:02:30.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:29 smithi129 ceph-mon[101316]: pgmap v286: 105 pgs: 105 active+clean; 582 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T04:02:30.669 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.SnapCreateLocksImage (1916 ms) 2024-09-14T04:02:30.670 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.SnapCreateFailsToLockImage 2024-09-14T04:02:30.694 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.SnapCreateFailsToLockImage (25 ms) 2024-09-14T04:02:30.694 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.SnapRollbackLocksImage 2024-09-14T04:02:31.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:30 smithi129 ceph-mon[101316]: osdmap e531: 8 total, 8 up, 8 in 2024-09-14T04:02:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:30 smithi039 ceph-mon[106470]: osdmap e531: 8 total, 8 up, 8 in 2024-09-14T04:02:31.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:30 smithi039 ceph-mon[111235]: osdmap e531: 8 total, 8 up, 8 in 2024-09-14T04:02:32.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:31 smithi129 ceph-mon[101316]: osdmap e532: 8 total, 8 up, 8 in 2024-09-14T04:02:32.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:31 smithi129 ceph-mon[101316]: pgmap v289: 105 pgs: 105 active+clean; 582 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T04:02:32.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:31 smithi039 ceph-mon[111235]: osdmap e532: 8 total, 8 up, 8 in 2024-09-14T04:02:32.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:31 smithi039 ceph-mon[111235]: pgmap v289: 105 pgs: 105 active+clean; 582 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T04:02:32.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:31 smithi039 ceph-mon[106470]: osdmap e532: 8 total, 8 up, 8 in 2024-09-14T04:02:32.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:31 smithi039 ceph-mon[106470]: pgmap v289: 105 pgs: 105 active+clean; 582 KiB data, 453 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T04:02:32.695 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.SnapRollbackLocksImage (2001 ms) 2024-09-14T04:02:32.695 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.SnapRollbackFailsToLockImage 2024-09-14T04:02:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:32 smithi129 ceph-mon[101316]: osdmap e533: 8 total, 8 up, 8 in 2024-09-14T04:02:33.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:32 smithi039 ceph-mon[106470]: osdmap e533: 8 total, 8 up, 8 in 2024-09-14T04:02:33.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:32 smithi039 ceph-mon[111235]: osdmap e533: 8 total, 8 up, 8 in 2024-09-14T04:02:34.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:33 smithi129 ceph-mon[101316]: osdmap e534: 8 total, 8 up, 8 in 2024-09-14T04:02:34.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:33 smithi129 ceph-mon[101316]: pgmap v292: 105 pgs: 105 active+clean; 582 KiB data, 458 MiB used, 715 GiB / 715 GiB avail; 118 KiB/s rd, 13 KiB/s wr, 172 op/s 2024-09-14T04:02:34.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:33 smithi039 ceph-mon[106470]: osdmap e534: 8 total, 8 up, 8 in 2024-09-14T04:02:34.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:33 smithi039 ceph-mon[106470]: pgmap v292: 105 pgs: 105 active+clean; 582 KiB data, 458 MiB used, 715 GiB / 715 GiB avail; 118 KiB/s rd, 13 KiB/s wr, 172 op/s 2024-09-14T04:02:34.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:33 smithi039 ceph-mon[111235]: osdmap e534: 8 total, 8 up, 8 in 2024-09-14T04:02:34.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:33 smithi039 ceph-mon[111235]: pgmap v292: 105 pgs: 105 active+clean; 582 KiB data, 458 MiB used, 715 GiB / 715 GiB avail; 118 KiB/s rd, 13 KiB/s wr, 172 op/s 2024-09-14T04:02:34.712 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.SnapRollbackFailsToLockImage (2016 ms) 2024-09-14T04:02:34.712 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.SnapSetReleasesLock 2024-09-14T04:02:35.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:34 smithi129 ceph-mon[101316]: osdmap e535: 8 total, 8 up, 8 in 2024-09-14T04:02:35.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:34 smithi039 ceph-mon[106470]: osdmap e535: 8 total, 8 up, 8 in 2024-09-14T04:02:35.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:34 smithi039 ceph-mon[111235]: osdmap e535: 8 total, 8 up, 8 in 2024-09-14T04:02:36.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:35 smithi129 ceph-mon[101316]: osdmap e536: 8 total, 8 up, 8 in 2024-09-14T04:02:36.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:35 smithi129 ceph-mon[101316]: pgmap v295: 105 pgs: 105 active+clean; 582 KiB data, 458 MiB used, 715 GiB / 715 GiB avail; 118 KiB/s rd, 13 KiB/s wr, 172 op/s 2024-09-14T04:02:36.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:35 smithi129 ceph-mon[101316]: osdmap e537: 8 total, 8 up, 8 in 2024-09-14T04:02:36.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:35 smithi039 ceph-mon[106470]: osdmap e536: 8 total, 8 up, 8 in 2024-09-14T04:02:36.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:35 smithi039 ceph-mon[106470]: pgmap v295: 105 pgs: 105 active+clean; 582 KiB data, 458 MiB used, 715 GiB / 715 GiB avail; 118 KiB/s rd, 13 KiB/s wr, 172 op/s 2024-09-14T04:02:36.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:35 smithi039 ceph-mon[106470]: osdmap e537: 8 total, 8 up, 8 in 2024-09-14T04:02:36.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:35 smithi039 ceph-mon[111235]: osdmap e536: 8 total, 8 up, 8 in 2024-09-14T04:02:36.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:35 smithi039 ceph-mon[111235]: pgmap v295: 105 pgs: 105 active+clean; 582 KiB data, 458 MiB used, 715 GiB / 715 GiB avail; 118 KiB/s rd, 13 KiB/s wr, 172 op/s 2024-09-14T04:02:36.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:35 smithi039 ceph-mon[111235]: osdmap e537: 8 total, 8 up, 8 in 2024-09-14T04:02:36.422 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.SnapSetReleasesLock (1711 ms) 2024-09-14T04:02:36.422 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.FlattenLocksImage 2024-09-14T04:02:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:37 smithi129 ceph-mon[101316]: osdmap e538: 8 total, 8 up, 8 in 2024-09-14T04:02:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:37 smithi129 ceph-mon[101316]: pgmap v298: 105 pgs: 105 active+clean; 582 KiB data, 458 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:02:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:37.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:37 smithi039 ceph-mon[106470]: osdmap e538: 8 total, 8 up, 8 in 2024-09-14T04:02:37.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:37 smithi039 ceph-mon[106470]: pgmap v298: 105 pgs: 105 active+clean; 582 KiB data, 458 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:02:37.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:37.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:37 smithi039 ceph-mon[111235]: osdmap e538: 8 total, 8 up, 8 in 2024-09-14T04:02:37.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:37 smithi039 ceph-mon[111235]: pgmap v298: 105 pgs: 105 active+clean; 582 KiB data, 458 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:02:37.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:38.444 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.FlattenLocksImage (2022 ms) 2024-09-14T04:02:38.444 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.FlattenFailsToLockImage 2024-09-14T04:02:38.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:38 smithi129 ceph-mon[101316]: osdmap e539: 8 total, 8 up, 8 in 2024-09-14T04:02:38.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:38 smithi039 ceph-mon[106470]: osdmap e539: 8 total, 8 up, 8 in 2024-09-14T04:02:38.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:38 smithi039 ceph-mon[111235]: osdmap e539: 8 total, 8 up, 8 in 2024-09-14T04:02:38.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:02:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:02:38] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.51.0" 2024-09-14T04:02:39.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:39 smithi129 ceph-mon[101316]: osdmap e540: 8 total, 8 up, 8 in 2024-09-14T04:02:39.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:39 smithi129 ceph-mon[101316]: pgmap v301: 105 pgs: 105 active+clean; 583 KiB data, 459 MiB used, 715 GiB / 715 GiB avail; 127 KiB/s rd, 9.2 KiB/s wr, 173 op/s 2024-09-14T04:02:39.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:39 smithi039 ceph-mon[111235]: osdmap e540: 8 total, 8 up, 8 in 2024-09-14T04:02:39.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:39 smithi039 ceph-mon[111235]: pgmap v301: 105 pgs: 105 active+clean; 583 KiB data, 459 MiB used, 715 GiB / 715 GiB avail; 127 KiB/s rd, 9.2 KiB/s wr, 173 op/s 2024-09-14T04:02:39.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:39 smithi039 ceph-mon[106470]: osdmap e540: 8 total, 8 up, 8 in 2024-09-14T04:02:39.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:39 smithi039 ceph-mon[106470]: pgmap v301: 105 pgs: 105 active+clean; 583 KiB data, 459 MiB used, 715 GiB / 715 GiB avail; 127 KiB/s rd, 9.2 KiB/s wr, 173 op/s 2024-09-14T04:02:40.422 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.FlattenFailsToLockImage (1977 ms) 2024-09-14T04:02:40.422 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.WriteFailsToLockImageBlocklisted 2024-09-14T04:02:40.710 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:40 smithi039 ceph-mon[106470]: osdmap e541: 8 total, 8 up, 8 in 2024-09-14T04:02:40.710 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:40 smithi039 ceph-mon[106470]: osdmap e542: 8 total, 8 up, 8 in 2024-09-14T04:02:40.710 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:40 smithi039 ceph-mon[111235]: osdmap e541: 8 total, 8 up, 8 in 2024-09-14T04:02:40.710 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:40 smithi039 ceph-mon[111235]: osdmap e542: 8 total, 8 up, 8 in 2024-09-14T04:02:40.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:40 smithi129 ceph-mon[101316]: osdmap e541: 8 total, 8 up, 8 in 2024-09-14T04:02:40.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:40 smithi129 ceph-mon[101316]: osdmap e542: 8 total, 8 up, 8 in 2024-09-14T04:02:41.459 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.WriteFailsToLockImageBlocklisted (1037 ms) 2024-09-14T04:02:41.459 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.WriteFailsToLockImageBlocklistedWatch 2024-09-14T04:02:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:41 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2766717202' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.39:0/2766717202"} : dispatch 2024-09-14T04:02:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:41 smithi129 ceph-mon[101316]: pgmap v304: 105 pgs: 105 active+clean; 583 KiB data, 459 MiB used, 715 GiB / 715 GiB avail; 127 KiB/s rd, 9.2 KiB/s wr, 173 op/s 2024-09-14T04:02:41.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:41 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2766717202' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.39:0/2766717202"} : dispatch 2024-09-14T04:02:41.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:41 smithi039 ceph-mon[106470]: pgmap v304: 105 pgs: 105 active+clean; 583 KiB data, 459 MiB used, 715 GiB / 715 GiB avail; 127 KiB/s rd, 9.2 KiB/s wr, 173 op/s 2024-09-14T04:02:41.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:41 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2766717202' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.39:0/2766717202"} : dispatch 2024-09-14T04:02:41.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:41 smithi039 ceph-mon[111235]: pgmap v304: 105 pgs: 105 active+clean; 583 KiB data, 459 MiB used, 715 GiB / 715 GiB avail; 127 KiB/s rd, 9.2 KiB/s wr, 173 op/s 2024-09-14T04:02:42.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:42 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2766717202' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.39:0/2766717202"}]': finished 2024-09-14T04:02:42.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:42 smithi129 ceph-mon[101316]: osdmap e543: 8 total, 8 up, 8 in 2024-09-14T04:02:42.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:42 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1958359945' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.39:0/1958359945"} : dispatch 2024-09-14T04:02:42.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:42 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.39:0/1958359945"} : dispatch 2024-09-14T04:02:42.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:42 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2766717202' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.39:0/2766717202"}]': finished 2024-09-14T04:02:42.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:42 smithi039 ceph-mon[111235]: osdmap e543: 8 total, 8 up, 8 in 2024-09-14T04:02:42.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:42 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1958359945' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.39:0/1958359945"} : dispatch 2024-09-14T04:02:42.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:42 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.39:0/1958359945"} : dispatch 2024-09-14T04:02:42.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:42 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2766717202' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.39:0/2766717202"}]': finished 2024-09-14T04:02:42.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:42 smithi039 ceph-mon[106470]: osdmap e543: 8 total, 8 up, 8 in 2024-09-14T04:02:42.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:42 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1958359945' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.39:0/1958359945"} : dispatch 2024-09-14T04:02:42.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:42 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.39:0/1958359945"} : dispatch 2024-09-14T04:02:43.476 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.WriteFailsToLockImageBlocklistedWatch (2017 ms) 2024-09-14T04:02:43.476 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.AioWriteRequestsLock 2024-09-14T04:02:43.508 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.AioWriteRequestsLock (32 ms) 2024-09-14T04:02:43.508 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.AioDiscardRequestsLock 2024-09-14T04:02:43.538 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.AioDiscardRequestsLock (30 ms) 2024-09-14T04:02:43.538 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.CancelAsyncResize 2024-09-14T04:02:43.579 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.CancelAsyncResize (41 ms) 2024-09-14T04:02:43.579 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.MultipleResize 2024-09-14T04:02:43.636 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.MultipleResize (57 ms) 2024-09-14T04:02:43.636 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.Metadata 2024-09-14T04:02:43.679 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.Metadata (43 ms) 2024-09-14T04:02:43.679 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.MetadataConfApply 2024-09-14T04:02:43.711 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.MetadataConfApply (32 ms) 2024-09-14T04:02:43.711 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.SnapshotCopyup 2024-09-14T04:02:43.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:43 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.39:0/1958359945"}]': finished 2024-09-14T04:02:43.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:43 smithi129 ceph-mon[101316]: osdmap e544: 8 total, 8 up, 8 in 2024-09-14T04:02:43.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:43 smithi129 ceph-mon[101316]: pgmap v307: 105 pgs: 105 active+clean; 583 KiB data, 467 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s rd, 8.0 KiB/s wr, 158 op/s 2024-09-14T04:02:43.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:43 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.39:0/1958359945"}]': finished 2024-09-14T04:02:43.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:43 smithi039 ceph-mon[106470]: osdmap e544: 8 total, 8 up, 8 in 2024-09-14T04:02:43.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:43 smithi039 ceph-mon[106470]: pgmap v307: 105 pgs: 105 active+clean; 583 KiB data, 467 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s rd, 8.0 KiB/s wr, 158 op/s 2024-09-14T04:02:43.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:43 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.39:0/1958359945"}]': finished 2024-09-14T04:02:43.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:43 smithi039 ceph-mon[111235]: osdmap e544: 8 total, 8 up, 8 in 2024-09-14T04:02:43.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:43 smithi039 ceph-mon[111235]: pgmap v307: 105 pgs: 105 active+clean; 583 KiB data, 467 MiB used, 715 GiB / 715 GiB avail; 116 KiB/s rd, 8.0 KiB/s wr, 158 op/s 2024-09-14T04:02:45.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:45 smithi129 ceph-mon[101316]: osdmap e545: 8 total, 8 up, 8 in 2024-09-14T04:02:45.806 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:45 smithi129 ceph-mon[101316]: pgmap v309: 105 pgs: 105 active+clean; 583 KiB data, 467 MiB used, 715 GiB / 715 GiB avail; 106 KiB/s rd, 7.3 KiB/s wr, 144 op/s 2024-09-14T04:02:45.806 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:45 smithi129 ceph-mon[101316]: osdmap e546: 8 total, 8 up, 8 in 2024-09-14T04:02:45.807 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:45 smithi039 ceph-mon[106470]: osdmap e545: 8 total, 8 up, 8 in 2024-09-14T04:02:45.807 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:45 smithi039 ceph-mon[106470]: pgmap v309: 105 pgs: 105 active+clean; 583 KiB data, 467 MiB used, 715 GiB / 715 GiB avail; 106 KiB/s rd, 7.3 KiB/s wr, 144 op/s 2024-09-14T04:02:45.807 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:45 smithi039 ceph-mon[106470]: osdmap e546: 8 total, 8 up, 8 in 2024-09-14T04:02:45.807 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:45 smithi039 ceph-mon[111235]: osdmap e545: 8 total, 8 up, 8 in 2024-09-14T04:02:45.807 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:45 smithi039 ceph-mon[111235]: pgmap v309: 105 pgs: 105 active+clean; 583 KiB data, 467 MiB used, 715 GiB / 715 GiB avail; 106 KiB/s rd, 7.3 KiB/s wr, 144 op/s 2024-09-14T04:02:45.807 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:45 smithi039 ceph-mon[111235]: osdmap e546: 8 total, 8 up, 8 in 2024-09-14T04:02:46.511 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.SnapshotCopyup (2801 ms) 2024-09-14T04:02:46.512 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.SnapshotCopyupZeros 2024-09-14T04:02:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:47 smithi129 ceph-mon[101316]: osdmap e547: 8 total, 8 up, 8 in 2024-09-14T04:02:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:47 smithi129 ceph-mon[101316]: pgmap v312: 105 pgs: 105 active+clean; 583 KiB data, 467 MiB used, 715 GiB / 715 GiB avail; 107 KiB/s rd, 7.4 KiB/s wr, 146 op/s 2024-09-14T04:02:47.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:47 smithi039 ceph-mon[106470]: osdmap e547: 8 total, 8 up, 8 in 2024-09-14T04:02:47.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:47 smithi039 ceph-mon[106470]: pgmap v312: 105 pgs: 105 active+clean; 583 KiB data, 467 MiB used, 715 GiB / 715 GiB avail; 107 KiB/s rd, 7.4 KiB/s wr, 146 op/s 2024-09-14T04:02:47.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:47 smithi039 ceph-mon[111235]: osdmap e547: 8 total, 8 up, 8 in 2024-09-14T04:02:47.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:47 smithi039 ceph-mon[111235]: pgmap v312: 105 pgs: 105 active+clean; 583 KiB data, 467 MiB used, 715 GiB / 715 GiB avail; 107 KiB/s rd, 7.4 KiB/s wr, 146 op/s 2024-09-14T04:02:48.468 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.SnapshotCopyupZeros (1957 ms) 2024-09-14T04:02:48.468 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.SnapshotCopyupZerosMigration 2024-09-14T04:02:48.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:48 smithi129 ceph-mon[101316]: osdmap e548: 8 total, 8 up, 8 in 2024-09-14T04:02:48.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:48 smithi039 ceph-mon[106470]: osdmap e548: 8 total, 8 up, 8 in 2024-09-14T04:02:48.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:02:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:02:48] "GET /metrics HTTP/1.1" 200 36408 "" "Prometheus/2.51.0" 2024-09-14T04:02:48.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:48 smithi039 ceph-mon[111235]: osdmap e548: 8 total, 8 up, 8 in 2024-09-14T04:02:49.480 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.SnapshotCopyupZerosMigration (1011 ms) 2024-09-14T04:02:49.480 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.ResizeCopyup 2024-09-14T04:02:49.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:49 smithi129 ceph-mon[101316]: osdmap e549: 8 total, 8 up, 8 in 2024-09-14T04:02:49.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:49 smithi129 ceph-mon[101316]: pgmap v315: 105 pgs: 105 active+clean; 586 KiB data, 468 MiB used, 715 GiB / 715 GiB avail; 299 KiB/s rd, 32 KiB/s wr, 416 op/s 2024-09-14T04:02:49.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:49 smithi039 ceph-mon[106470]: osdmap e549: 8 total, 8 up, 8 in 2024-09-14T04:02:49.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:49 smithi039 ceph-mon[106470]: pgmap v315: 105 pgs: 105 active+clean; 586 KiB data, 468 MiB used, 715 GiB / 715 GiB avail; 299 KiB/s rd, 32 KiB/s wr, 416 op/s 2024-09-14T04:02:49.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:49 smithi039 ceph-mon[111235]: osdmap e549: 8 total, 8 up, 8 in 2024-09-14T04:02:49.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:49 smithi039 ceph-mon[111235]: pgmap v315: 105 pgs: 105 active+clean; 586 KiB data, 468 MiB used, 715 GiB / 715 GiB avail; 299 KiB/s rd, 32 KiB/s wr, 416 op/s 2024-09-14T04:02:50.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:50 smithi129 ceph-mon[101316]: osdmap e550: 8 total, 8 up, 8 in 2024-09-14T04:02:50.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:50 smithi039 ceph-mon[106470]: osdmap e550: 8 total, 8 up, 8 in 2024-09-14T04:02:50.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:50 smithi039 ceph-mon[111235]: osdmap e550: 8 total, 8 up, 8 in 2024-09-14T04:02:51.533 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.ResizeCopyup (2053 ms) 2024-09-14T04:02:51.533 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.DiscardCopyup 2024-09-14T04:02:51.541 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/test_internal.cc:1056: Skipped 2024-09-14T04:02:51.541 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T04:02:51.541 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestInternal.DiscardCopyup (9 ms) 2024-09-14T04:02:51.541 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.ImageOptions 2024-09-14T04:02:51.549 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.ImageOptions (7 ms) 2024-09-14T04:02:51.549 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.WriteFullCopyup 2024-09-14T04:02:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:51 smithi129 ceph-mon[101316]: osdmap e551: 8 total, 8 up, 8 in 2024-09-14T04:02:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:51 smithi129 ceph-mon[101316]: pgmap v318: 105 pgs: 105 active+clean; 586 KiB data, 468 MiB used, 715 GiB / 715 GiB avail; 299 KiB/s rd, 32 KiB/s wr, 416 op/s 2024-09-14T04:02:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:51 smithi039 ceph-mon[106470]: osdmap e551: 8 total, 8 up, 8 in 2024-09-14T04:02:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:51 smithi039 ceph-mon[106470]: pgmap v318: 105 pgs: 105 active+clean; 586 KiB data, 468 MiB used, 715 GiB / 715 GiB avail; 299 KiB/s rd, 32 KiB/s wr, 416 op/s 2024-09-14T04:02:51.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:51 smithi039 ceph-mon[111235]: osdmap e551: 8 total, 8 up, 8 in 2024-09-14T04:02:51.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:51 smithi039 ceph-mon[111235]: pgmap v318: 105 pgs: 105 active+clean; 586 KiB data, 468 MiB used, 715 GiB / 715 GiB avail; 299 KiB/s rd, 32 KiB/s wr, 416 op/s 2024-09-14T04:02:52.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:52 smithi129 ceph-mon[101316]: osdmap e552: 8 total, 8 up, 8 in 2024-09-14T04:02:52.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:52.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:52 smithi039 ceph-mon[106470]: osdmap e552: 8 total, 8 up, 8 in 2024-09-14T04:02:52.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:52.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:52 smithi039 ceph-mon[111235]: osdmap e552: 8 total, 8 up, 8 in 2024-09-14T04:02:52.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:02:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:53 smithi129 ceph-mon[101316]: osdmap e553: 8 total, 8 up, 8 in 2024-09-14T04:02:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:53 smithi129 ceph-mon[101316]: pgmap v321: 105 pgs: 105 active+clean; 4.6 MiB data, 469 MiB used, 715 GiB / 715 GiB avail; 226 KiB/s rd, 1.0 MiB/s wr, 314 op/s 2024-09-14T04:02:53.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:53 smithi039 ceph-mon[106470]: osdmap e553: 8 total, 8 up, 8 in 2024-09-14T04:02:53.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:53 smithi039 ceph-mon[106470]: pgmap v321: 105 pgs: 105 active+clean; 4.6 MiB data, 469 MiB used, 715 GiB / 715 GiB avail; 226 KiB/s rd, 1.0 MiB/s wr, 314 op/s 2024-09-14T04:02:53.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:53 smithi039 ceph-mon[111235]: osdmap e553: 8 total, 8 up, 8 in 2024-09-14T04:02:53.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:53 smithi039 ceph-mon[111235]: pgmap v321: 105 pgs: 105 active+clean; 4.6 MiB data, 469 MiB used, 715 GiB / 715 GiB avail; 226 KiB/s rd, 1.0 MiB/s wr, 314 op/s 2024-09-14T04:02:54.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:54 smithi129 ceph-mon[101316]: osdmap e554: 8 total, 8 up, 8 in 2024-09-14T04:02:54.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:54 smithi039 ceph-mon[106470]: osdmap e554: 8 total, 8 up, 8 in 2024-09-14T04:02:54.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:54 smithi039 ceph-mon[111235]: osdmap e554: 8 total, 8 up, 8 in 2024-09-14T04:02:55.543 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.WriteFullCopyup (3994 ms) 2024-09-14T04:02:55.544 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.DiffIterateCloneOverwrite 2024-09-14T04:02:55.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:55 smithi039 ceph-mon[106470]: osdmap e555: 8 total, 8 up, 8 in 2024-09-14T04:02:55.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:55 smithi039 ceph-mon[106470]: pgmap v324: 105 pgs: 105 active+clean; 4.6 MiB data, 469 MiB used, 715 GiB / 715 GiB avail; 226 KiB/s rd, 1.0 MiB/s wr, 314 op/s 2024-09-14T04:02:55.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:55 smithi039 ceph-mon[111235]: osdmap e555: 8 total, 8 up, 8 in 2024-09-14T04:02:55.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:55 smithi039 ceph-mon[111235]: pgmap v324: 105 pgs: 105 active+clean; 4.6 MiB data, 469 MiB used, 715 GiB / 715 GiB avail; 226 KiB/s rd, 1.0 MiB/s wr, 314 op/s 2024-09-14T04:02:56.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:55 smithi129 ceph-mon[101316]: osdmap e555: 8 total, 8 up, 8 in 2024-09-14T04:02:56.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:55 smithi129 ceph-mon[101316]: pgmap v324: 105 pgs: 105 active+clean; 4.6 MiB data, 469 MiB used, 715 GiB / 715 GiB avail; 226 KiB/s rd, 1.0 MiB/s wr, 314 op/s 2024-09-14T04:02:56.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:56 smithi039 ceph-mon[111235]: osdmap e556: 8 total, 8 up, 8 in 2024-09-14T04:02:56.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:56 smithi039 ceph-mon[106470]: osdmap e556: 8 total, 8 up, 8 in 2024-09-14T04:02:57.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:56 smithi129 ceph-mon[101316]: osdmap e556: 8 total, 8 up, 8 in 2024-09-14T04:02:57.569 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.DiffIterateCloneOverwrite (2025 ms) 2024-09-14T04:02:57.569 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.TestCoR 2024-09-14T04:02:57.581 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/test_internal.cc:1313: Skipped 2024-09-14T04:02:57.581 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to disabled copy-on-read 2024-09-14T04:02:57.581 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestInternal.TestCoR (13 ms) 2024-09-14T04:02:57.581 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.FlattenNoEmptyObjects 2024-09-14T04:02:57.803 INFO:tasks.workunit.client.0.smithi039.stdout:made parent image "image393": 4194304K (1024 * 4K) 2024-09-14T04:02:57.803 INFO:tasks.workunit.client.0.smithi039.stdout:generated random write map: 2024-09-14T04:02:57.803 INFO:tasks.workunit.client.0.smithi039.stdout: [7 , 31739 ] 2024-09-14T04:02:57.803 INFO:tasks.workunit.client.0.smithi039.stdout: [14 , 58484 ] 2024-09-14T04:02:57.803 INFO:tasks.workunit.client.0.smithi039.stdout: [17 , 70543 ] 2024-09-14T04:02:57.803 INFO:tasks.workunit.client.0.smithi039.stdout: [26 , 107264 ] 2024-09-14T04:02:57.803 INFO:tasks.workunit.client.0.smithi039.stdout: [38 , 157301 ] 2024-09-14T04:02:57.803 INFO:tasks.workunit.client.0.smithi039.stdout: [65 , 266981 ] 2024-09-14T04:02:57.804 INFO:tasks.workunit.client.0.smithi039.stdout: [79 , 324770 ] 2024-09-14T04:02:57.804 INFO:tasks.workunit.client.0.smithi039.stdout: [113 , 464061 ] 2024-09-14T04:02:57.804 INFO:tasks.workunit.client.0.smithi039.stdout: [118 , 486685 ] 2024-09-14T04:02:57.804 INFO:tasks.workunit.client.0.smithi039.stdout: [136 , 557904 ] 2024-09-14T04:02:57.804 INFO:tasks.workunit.client.0.smithi039.stdout: [146 , 600477 ] 2024-09-14T04:02:57.804 INFO:tasks.workunit.client.0.smithi039.stdout: [155 , 635058 ] 2024-09-14T04:02:57.804 INFO:tasks.workunit.client.0.smithi039.stdout: [157 , 643833 ] 2024-09-14T04:02:57.804 INFO:tasks.workunit.client.0.smithi039.stdout: [164 , 674856 ] 2024-09-14T04:02:57.804 INFO:tasks.workunit.client.0.smithi039.stdout: [165 , 676659 ] 2024-09-14T04:02:57.804 INFO:tasks.workunit.client.0.smithi039.stdout: [179 , 736203 ] 2024-09-14T04:02:57.804 INFO:tasks.workunit.client.0.smithi039.stdout: [180 , 739678 ] 2024-09-14T04:02:57.804 INFO:tasks.workunit.client.0.smithi039.stdout: [201 , 823624 ] 2024-09-14T04:02:57.805 INFO:tasks.workunit.client.0.smithi039.stdout: [205 , 841720 ] 2024-09-14T04:02:57.805 INFO:tasks.workunit.client.0.smithi039.stdout: [218 , 893680 ] 2024-09-14T04:02:57.805 INFO:tasks.workunit.client.0.smithi039.stdout: [235 , 964842 ] 2024-09-14T04:02:57.805 INFO:tasks.workunit.client.0.smithi039.stdout: [259 , 1061806 ] 2024-09-14T04:02:57.805 INFO:tasks.workunit.client.0.smithi039.stdout: [271 , 1111623 ] 2024-09-14T04:02:57.805 INFO:tasks.workunit.client.0.smithi039.stdout: [276 , 1132850 ] 2024-09-14T04:02:57.805 INFO:tasks.workunit.client.0.smithi039.stdout: [281 , 1153607 ] 2024-09-14T04:02:57.805 INFO:tasks.workunit.client.0.smithi039.stdout: [285 , 1169090 ] 2024-09-14T04:02:57.805 INFO:tasks.workunit.client.0.smithi039.stdout: [301 , 1235813 ] 2024-09-14T04:02:57.805 INFO:tasks.workunit.client.0.smithi039.stdout: [306 , 1255041 ] 2024-09-14T04:02:57.805 INFO:tasks.workunit.client.0.smithi039.stdout: [308 , 1262265 ] 2024-09-14T04:02:57.805 INFO:tasks.workunit.client.0.smithi039.stdout: [320 , 1312762 ] 2024-09-14T04:02:57.806 INFO:tasks.workunit.client.0.smithi039.stdout: [323 , 1325234 ] 2024-09-14T04:02:57.806 INFO:tasks.workunit.client.0.smithi039.stdout: [334 , 1370707 ] 2024-09-14T04:02:57.806 INFO:tasks.workunit.client.0.smithi039.stdout: [348 , 1428897 ] 2024-09-14T04:02:57.806 INFO:tasks.workunit.client.0.smithi039.stdout: [350 , 1435151 ] 2024-09-14T04:02:57.806 INFO:tasks.workunit.client.0.smithi039.stdout: [402 , 1648826 ] 2024-09-14T04:02:57.806 INFO:tasks.workunit.client.0.smithi039.stdout: [414 , 1698313 ] 2024-09-14T04:02:57.806 INFO:tasks.workunit.client.0.smithi039.stdout: [419 , 1717585 ] 2024-09-14T04:02:57.806 INFO:tasks.workunit.client.0.smithi039.stdout: [431 , 1766607 ] 2024-09-14T04:02:57.806 INFO:tasks.workunit.client.0.smithi039.stdout: [433 , 1774488 ] 2024-09-14T04:02:57.806 INFO:tasks.workunit.client.0.smithi039.stdout: [434 , 1778635 ] 2024-09-14T04:02:57.806 INFO:tasks.workunit.client.0.smithi039.stdout: [436 , 1788018 ] 2024-09-14T04:02:57.807 INFO:tasks.workunit.client.0.smithi039.stdout: [467 , 1915440 ] 2024-09-14T04:02:57.807 INFO:tasks.workunit.client.0.smithi039.stdout: [475 , 1945940 ] 2024-09-14T04:02:57.807 INFO:tasks.workunit.client.0.smithi039.stdout: [476 , 1949837 ] 2024-09-14T04:02:57.807 INFO:tasks.workunit.client.0.smithi039.stdout: [484 , 1984186 ] 2024-09-14T04:02:57.807 INFO:tasks.workunit.client.0.smithi039.stdout: [495 , 2027776 ] 2024-09-14T04:02:57.807 INFO:tasks.workunit.client.0.smithi039.stdout: [514 , 2107088 ] 2024-09-14T04:02:57.807 INFO:tasks.workunit.client.0.smithi039.stdout: [519 , 2126267 ] 2024-09-14T04:02:57.807 INFO:tasks.workunit.client.0.smithi039.stdout: [525 , 2150712 ] 2024-09-14T04:02:57.807 INFO:tasks.workunit.client.0.smithi039.stdout: [536 , 2197667 ] 2024-09-14T04:02:57.807 INFO:tasks.workunit.client.0.smithi039.stdout: [547 , 2242699 ] 2024-09-14T04:02:57.808 INFO:tasks.workunit.client.0.smithi039.stdout: [548 , 2245788 ] 2024-09-14T04:02:57.808 INFO:tasks.workunit.client.0.smithi039.stdout: [559 , 2293175 ] 2024-09-14T04:02:57.808 INFO:tasks.workunit.client.0.smithi039.stdout: [568 , 2328682 ] 2024-09-14T04:02:57.808 INFO:tasks.workunit.client.0.smithi039.stdout: [585 , 2399559 ] 2024-09-14T04:02:57.808 INFO:tasks.workunit.client.0.smithi039.stdout: [591 , 2423366 ] 2024-09-14T04:02:57.808 INFO:tasks.workunit.client.0.smithi039.stdout: [600 , 2460130 ] 2024-09-14T04:02:57.808 INFO:tasks.workunit.client.0.smithi039.stdout: [637 , 2609950 ] 2024-09-14T04:02:57.808 INFO:tasks.workunit.client.0.smithi039.stdout: [639 , 2619691 ] 2024-09-14T04:02:57.808 INFO:tasks.workunit.client.0.smithi039.stdout: [646 , 2647382 ] 2024-09-14T04:02:57.808 INFO:tasks.workunit.client.0.smithi039.stdout: [649 , 2660712 ] 2024-09-14T04:02:57.809 INFO:tasks.workunit.client.0.smithi039.stdout: [650 , 2663697 ] 2024-09-14T04:02:57.809 INFO:tasks.workunit.client.0.smithi039.stdout: [659 , 2701260 ] 2024-09-14T04:02:57.809 INFO:tasks.workunit.client.0.smithi039.stdout: [665 , 2725981 ] 2024-09-14T04:02:57.809 INFO:tasks.workunit.client.0.smithi039.stdout: [672 , 2752989 ] 2024-09-14T04:02:57.809 INFO:tasks.workunit.client.0.smithi039.stdout: [678 , 2779502 ] 2024-09-14T04:02:57.809 INFO:tasks.workunit.client.0.smithi039.stdout: [690 , 2828445 ] 2024-09-14T04:02:57.809 INFO:tasks.workunit.client.0.smithi039.stdout: [712 , 2918149 ] 2024-09-14T04:02:57.809 INFO:tasks.workunit.client.0.smithi039.stdout: [727 , 2978985 ] 2024-09-14T04:02:57.809 INFO:tasks.workunit.client.0.smithi039.stdout: [729 , 2987490 ] 2024-09-14T04:02:57.809 INFO:tasks.workunit.client.0.smithi039.stdout: [741 , 3035571 ] 2024-09-14T04:02:57.809 INFO:tasks.workunit.client.0.smithi039.stdout: [749 , 3069882 ] 2024-09-14T04:02:57.809 INFO:tasks.workunit.client.0.smithi039.stdout: [759 , 3111003 ] 2024-09-14T04:02:57.810 INFO:tasks.workunit.client.0.smithi039.stdout: [768 , 3149165 ] 2024-09-14T04:02:57.810 INFO:tasks.workunit.client.0.smithi039.stdout: [779 , 3191657 ] 2024-09-14T04:02:57.810 INFO:tasks.workunit.client.0.smithi039.stdout: [811 , 3322740 ] 2024-09-14T04:02:57.810 INFO:tasks.workunit.client.0.smithi039.stdout: [814 , 3335189 ] 2024-09-14T04:02:57.810 INFO:tasks.workunit.client.0.smithi039.stdout: [830 , 3401367 ] 2024-09-14T04:02:57.810 INFO:tasks.workunit.client.0.smithi039.stdout: [835 , 3421895 ] 2024-09-14T04:02:57.810 INFO:tasks.workunit.client.0.smithi039.stdout: [839 , 3436859 ] 2024-09-14T04:02:57.810 INFO:tasks.workunit.client.0.smithi039.stdout: [847 , 3470086 ] 2024-09-14T04:02:57.810 INFO:tasks.workunit.client.0.smithi039.stdout: [879 , 3601369 ] 2024-09-14T04:02:57.810 INFO:tasks.workunit.client.0.smithi039.stdout: [884 , 3623457 ] 2024-09-14T04:02:57.810 INFO:tasks.workunit.client.0.smithi039.stdout: [925 , 3790946 ] 2024-09-14T04:02:57.810 INFO:tasks.workunit.client.0.smithi039.stdout: [930 , 3812825 ] 2024-09-14T04:02:57.810 INFO:tasks.workunit.client.0.smithi039.stdout: [943 , 3864031 ] 2024-09-14T04:02:57.811 INFO:tasks.workunit.client.0.smithi039.stdout: [948 , 3884116 ] 2024-09-14T04:02:57.811 INFO:tasks.workunit.client.0.smithi039.stdout: [950 , 3893292 ] 2024-09-14T04:02:57.811 INFO:tasks.workunit.client.0.smithi039.stdout: [959 , 3931181 ] 2024-09-14T04:02:57.811 INFO:tasks.workunit.client.0.smithi039.stdout: [976 , 3999073 ] 2024-09-14T04:02:57.811 INFO:tasks.workunit.client.0.smithi039.stdout: [995 , 4079008 ] 2024-09-14T04:02:57.811 INFO:tasks.workunit.client.0.smithi039.stdout: [1002 , 4104277 ] 2024-09-14T04:02:57.811 INFO:tasks.workunit.client.0.smithi039.stdout: [1005 , 4118396 ] 2024-09-14T04:02:57.811 INFO:tasks.workunit.client.0.smithi039.stdout: [1006 , 4120956 ] 2024-09-14T04:02:57.811 INFO:tasks.workunit.client.0.smithi039.stdout: [1008 , 4130630 ] 2024-09-14T04:02:57.811 INFO:tasks.workunit.client.0.smithi039.stdout: [1013 , 4152380 ] 2024-09-14T04:02:57.811 INFO:tasks.workunit.client.0.smithi039.stdout:write data based on random map 2024-09-14T04:02:57.811 INFO:tasks.workunit.client.0.smithi039.stdout: write object-7 2024-09-14T04:02:57.812 INFO:tasks.workunit.client.0.smithi039.stdout: write object-14 2024-09-14T04:02:57.812 INFO:tasks.workunit.client.0.smithi039.stdout: write object-17 2024-09-14T04:02:57.812 INFO:tasks.workunit.client.0.smithi039.stdout: write object-26 2024-09-14T04:02:57.812 INFO:tasks.workunit.client.0.smithi039.stdout: write object-38 2024-09-14T04:02:57.812 INFO:tasks.workunit.client.0.smithi039.stdout: write object-65 2024-09-14T04:02:57.812 INFO:tasks.workunit.client.0.smithi039.stdout: write object-79 2024-09-14T04:02:57.812 INFO:tasks.workunit.client.0.smithi039.stdout: write object-113 2024-09-14T04:02:57.812 INFO:tasks.workunit.client.0.smithi039.stdout: write object-118 2024-09-14T04:02:57.812 INFO:tasks.workunit.client.0.smithi039.stdout: write object-136 2024-09-14T04:02:57.812 INFO:tasks.workunit.client.0.smithi039.stdout: write object-146 2024-09-14T04:02:57.812 INFO:tasks.workunit.client.0.smithi039.stdout: write object-155 2024-09-14T04:02:57.812 INFO:tasks.workunit.client.0.smithi039.stdout: write object-157 2024-09-14T04:02:57.812 INFO:tasks.workunit.client.0.smithi039.stdout: write object-164 2024-09-14T04:02:57.812 INFO:tasks.workunit.client.0.smithi039.stdout: write object-165 2024-09-14T04:02:57.812 INFO:tasks.workunit.client.0.smithi039.stdout: write object-179 2024-09-14T04:02:57.813 INFO:tasks.workunit.client.0.smithi039.stdout: write object-180 2024-09-14T04:02:57.813 INFO:tasks.workunit.client.0.smithi039.stdout: write object-201 2024-09-14T04:02:57.813 INFO:tasks.workunit.client.0.smithi039.stdout: write object-205 2024-09-14T04:02:57.813 INFO:tasks.workunit.client.0.smithi039.stdout: write object-218 2024-09-14T04:02:57.813 INFO:tasks.workunit.client.0.smithi039.stdout: write object-235 2024-09-14T04:02:57.813 INFO:tasks.workunit.client.0.smithi039.stdout: write object-259 2024-09-14T04:02:57.813 INFO:tasks.workunit.client.0.smithi039.stdout: write object-271 2024-09-14T04:02:57.813 INFO:tasks.workunit.client.0.smithi039.stdout: write object-276 2024-09-14T04:02:57.813 INFO:tasks.workunit.client.0.smithi039.stdout: write object-281 2024-09-14T04:02:57.813 INFO:tasks.workunit.client.0.smithi039.stdout: write object-285 2024-09-14T04:02:57.813 INFO:tasks.workunit.client.0.smithi039.stdout: write object-301 2024-09-14T04:02:57.814 INFO:tasks.workunit.client.0.smithi039.stdout: write object-306 2024-09-14T04:02:57.814 INFO:tasks.workunit.client.0.smithi039.stdout: write object-308 2024-09-14T04:02:57.814 INFO:tasks.workunit.client.0.smithi039.stdout: write object-320 2024-09-14T04:02:57.814 INFO:tasks.workunit.client.0.smithi039.stdout: write object-323 2024-09-14T04:02:57.814 INFO:tasks.workunit.client.0.smithi039.stdout: write object-334 2024-09-14T04:02:57.814 INFO:tasks.workunit.client.0.smithi039.stdout: write object-348 2024-09-14T04:02:57.814 INFO:tasks.workunit.client.0.smithi039.stdout: write object-350 2024-09-14T04:02:57.814 INFO:tasks.workunit.client.0.smithi039.stdout: write object-402 2024-09-14T04:02:57.814 INFO:tasks.workunit.client.0.smithi039.stdout: write object-414 2024-09-14T04:02:57.814 INFO:tasks.workunit.client.0.smithi039.stdout: write object-419 2024-09-14T04:02:57.814 INFO:tasks.workunit.client.0.smithi039.stdout: write object-431 2024-09-14T04:02:57.814 INFO:tasks.workunit.client.0.smithi039.stdout: write object-433 2024-09-14T04:02:57.814 INFO:tasks.workunit.client.0.smithi039.stdout: write object-434 2024-09-14T04:02:57.814 INFO:tasks.workunit.client.0.smithi039.stdout: write object-436 2024-09-14T04:02:57.815 INFO:tasks.workunit.client.0.smithi039.stdout: write object-467 2024-09-14T04:02:57.815 INFO:tasks.workunit.client.0.smithi039.stdout: write object-475 2024-09-14T04:02:57.815 INFO:tasks.workunit.client.0.smithi039.stdout: write object-476 2024-09-14T04:02:57.815 INFO:tasks.workunit.client.0.smithi039.stdout: write object-484 2024-09-14T04:02:57.815 INFO:tasks.workunit.client.0.smithi039.stdout: write object-495 2024-09-14T04:02:57.815 INFO:tasks.workunit.client.0.smithi039.stdout: write object-514 2024-09-14T04:02:57.815 INFO:tasks.workunit.client.0.smithi039.stdout: write object-519 2024-09-14T04:02:57.815 INFO:tasks.workunit.client.0.smithi039.stdout: write object-525 2024-09-14T04:02:57.815 INFO:tasks.workunit.client.0.smithi039.stdout: write object-536 2024-09-14T04:02:57.815 INFO:tasks.workunit.client.0.smithi039.stdout: write object-547 2024-09-14T04:02:57.815 INFO:tasks.workunit.client.0.smithi039.stdout: write object-548 2024-09-14T04:02:57.815 INFO:tasks.workunit.client.0.smithi039.stdout: write object-559 2024-09-14T04:02:57.815 INFO:tasks.workunit.client.0.smithi039.stdout: write object-568 2024-09-14T04:02:57.815 INFO:tasks.workunit.client.0.smithi039.stdout: write object-585 2024-09-14T04:02:57.815 INFO:tasks.workunit.client.0.smithi039.stdout: write object-591 2024-09-14T04:02:57.816 INFO:tasks.workunit.client.0.smithi039.stdout: write object-600 2024-09-14T04:02:57.816 INFO:tasks.workunit.client.0.smithi039.stdout: write object-637 2024-09-14T04:02:57.816 INFO:tasks.workunit.client.0.smithi039.stdout: write object-639 2024-09-14T04:02:57.816 INFO:tasks.workunit.client.0.smithi039.stdout: write object-646 2024-09-14T04:02:57.816 INFO:tasks.workunit.client.0.smithi039.stdout: write object-649 2024-09-14T04:02:57.816 INFO:tasks.workunit.client.0.smithi039.stdout: write object-650 2024-09-14T04:02:57.816 INFO:tasks.workunit.client.0.smithi039.stdout: write object-659 2024-09-14T04:02:57.816 INFO:tasks.workunit.client.0.smithi039.stdout: write object-665 2024-09-14T04:02:57.816 INFO:tasks.workunit.client.0.smithi039.stdout: write object-672 2024-09-14T04:02:57.816 INFO:tasks.workunit.client.0.smithi039.stdout: write object-678 2024-09-14T04:02:57.816 INFO:tasks.workunit.client.0.smithi039.stdout: write object-690 2024-09-14T04:02:57.817 INFO:tasks.workunit.client.0.smithi039.stdout: write object-712 2024-09-14T04:02:57.817 INFO:tasks.workunit.client.0.smithi039.stdout: write object-727 2024-09-14T04:02:57.817 INFO:tasks.workunit.client.0.smithi039.stdout: write object-729 2024-09-14T04:02:57.817 INFO:tasks.workunit.client.0.smithi039.stdout: write object-741 2024-09-14T04:02:57.817 INFO:tasks.workunit.client.0.smithi039.stdout: write object-749 2024-09-14T04:02:57.817 INFO:tasks.workunit.client.0.smithi039.stdout: write object-759 2024-09-14T04:02:57.817 INFO:tasks.workunit.client.0.smithi039.stdout: write object-768 2024-09-14T04:02:57.817 INFO:tasks.workunit.client.0.smithi039.stdout: write object-779 2024-09-14T04:02:57.817 INFO:tasks.workunit.client.0.smithi039.stdout: write object-811 2024-09-14T04:02:57.817 INFO:tasks.workunit.client.0.smithi039.stdout: write object-814 2024-09-14T04:02:57.817 INFO:tasks.workunit.client.0.smithi039.stdout: write object-830 2024-09-14T04:02:57.818 INFO:tasks.workunit.client.0.smithi039.stdout: write object-835 2024-09-14T04:02:57.818 INFO:tasks.workunit.client.0.smithi039.stdout: write object-839 2024-09-14T04:02:57.818 INFO:tasks.workunit.client.0.smithi039.stdout: write object-847 2024-09-14T04:02:57.818 INFO:tasks.workunit.client.0.smithi039.stdout: write object-879 2024-09-14T04:02:57.818 INFO:tasks.workunit.client.0.smithi039.stdout: write object-884 2024-09-14T04:02:57.818 INFO:tasks.workunit.client.0.smithi039.stdout: write object-925 2024-09-14T04:02:57.818 INFO:tasks.workunit.client.0.smithi039.stdout: write object-930 2024-09-14T04:02:57.818 INFO:tasks.workunit.client.0.smithi039.stdout: write object-943 2024-09-14T04:02:57.818 INFO:tasks.workunit.client.0.smithi039.stdout: write object-948 2024-09-14T04:02:57.818 INFO:tasks.workunit.client.0.smithi039.stdout: write object-950 2024-09-14T04:02:58.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:57 smithi129 ceph-mon[101316]: osdmap e557: 8 total, 8 up, 8 in 2024-09-14T04:02:58.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:57 smithi129 ceph-mon[101316]: pgmap v327: 105 pgs: 105 active+clean; 4.6 MiB data, 469 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:02:58.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:57 smithi039 ceph-mon[106470]: osdmap e557: 8 total, 8 up, 8 in 2024-09-14T04:02:58.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:57 smithi039 ceph-mon[106470]: pgmap v327: 105 pgs: 105 active+clean; 4.6 MiB data, 469 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:02:58.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:57 smithi039 ceph-mon[111235]: osdmap e557: 8 total, 8 up, 8 in 2024-09-14T04:02:58.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:57 smithi039 ceph-mon[111235]: pgmap v327: 105 pgs: 105 active+clean; 4.6 MiB data, 469 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:02:58.752 INFO:tasks.workunit.client.0.smithi039.stdout: write object-959 2024-09-14T04:02:58.752 INFO:tasks.workunit.client.0.smithi039.stdout: write object-976 2024-09-14T04:02:58.752 INFO:tasks.workunit.client.0.smithi039.stdout: write object-995 2024-09-14T04:02:58.752 INFO:tasks.workunit.client.0.smithi039.stdout: write object-1002 2024-09-14T04:02:58.752 INFO:tasks.workunit.client.0.smithi039.stdout: write object-1005 2024-09-14T04:02:58.752 INFO:tasks.workunit.client.0.smithi039.stdout: write object-1006 2024-09-14T04:02:58.752 INFO:tasks.workunit.client.0.smithi039.stdout: write object-1008 2024-09-14T04:02:58.753 INFO:tasks.workunit.client.0.smithi039.stdout: write object-1013 2024-09-14T04:02:58.753 INFO:tasks.workunit.client.0.smithi039.stdout:verify written data by reading 2024-09-14T04:02:58.753 INFO:tasks.workunit.client.0.smithi039.stdout: read object-7 2024-09-14T04:02:58.753 INFO:tasks.workunit.client.0.smithi039.stdout:made snapshot "image393@parent_snap" and protect it 2024-09-14T04:02:58.753 INFO:tasks.workunit.client.0.smithi039.stdout:made and opened clone "image394" 2024-09-14T04:02:58.753 INFO:tasks.workunit.client.0.smithi039.stdout:flattening clone: "image394" 2024-09-14T04:02:58.753 INFO:tasks.workunit.client.0.smithi039.stdout:check whether child image has the same set of objects as parent 2024-09-14T04:02:58.753 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.FlattenNoEmptyObjects (1170 ms) 2024-09-14T04:02:58.753 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.PoolMetadataConfApply 2024-09-14T04:02:58.828 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.PoolMetadataConfApply (76 ms) 2024-09-14T04:02:58.828 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.Sparsify 2024-09-14T04:02:58.852 INFO:tasks.workunit.client.0.smithi039.stdout:sparsify_supported=1 2024-09-14T04:02:58.852 INFO:tasks.workunit.client.0.smithi039.stdout:sparse_read_supported=1 2024-09-14T04:02:58.879 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.Sparsify (51 ms) 2024-09-14T04:02:58.879 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.SparsifyClone 2024-09-14T04:02:58.895 INFO:tasks.workunit.client.0.smithi039.stdout:sparsify_supported=1 2024-09-14T04:02:58.941 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:58 smithi039 ceph-mon[106470]: osdmap e558: 8 total, 8 up, 8 in 2024-09-14T04:02:58.942 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:02:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:02:58] "GET /metrics HTTP/1.1" 200 36410 "" "Prometheus/2.51.0" 2024-09-14T04:02:58.942 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:02:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:02:58] ENGINE Bus STOPPING 2024-09-14T04:02:58.942 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:58 smithi039 ceph-mon[111235]: osdmap e558: 8 total, 8 up, 8 in 2024-09-14T04:02:59.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:58 smithi129 ceph-mon[101316]: osdmap e558: 8 total, 8 up, 8 in 2024-09-14T04:02:59.252 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:02:59 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:02:59] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T04:02:59.252 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:02:59 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:02:59] ENGINE Bus STOPPED 2024-09-14T04:02:59.252 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:02:59 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:02:59] ENGINE Bus STARTING 2024-09-14T04:02:59.252 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:02:59 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:02:59] ENGINE Serving on http://:::9283 2024-09-14T04:02:59.252 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:02:59 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:02:59] ENGINE Bus STARTED 2024-09-14T04:02:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:59 smithi129 ceph-mon[101316]: osdmap e559: 8 total, 8 up, 8 in 2024-09-14T04:02:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:59 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2626081537' entity='client.admin' 2024-09-14T04:02:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:59 smithi129 ceph-mon[101316]: pgmap v330: 105 pgs: 105 active+clean; 4.8 MiB data, 474 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.1 MiB/s wr, 348 op/s 2024-09-14T04:02:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:02:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:02:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-14T04:02:59.960 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:59 smithi039 ceph-mon[106470]: osdmap e559: 8 total, 8 up, 8 in 2024-09-14T04:02:59.960 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:59 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2626081537' entity='client.admin' 2024-09-14T04:02:59.960 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:59 smithi039 ceph-mon[106470]: pgmap v330: 105 pgs: 105 active+clean; 4.8 MiB data, 474 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.1 MiB/s wr, 348 op/s 2024-09-14T04:02:59.961 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:02:59.961 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:02:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-14T04:02:59.961 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:59 smithi039 ceph-mon[111235]: osdmap e559: 8 total, 8 up, 8 in 2024-09-14T04:02:59.962 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:59 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2626081537' entity='client.admin' 2024-09-14T04:02:59.962 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:59 smithi039 ceph-mon[111235]: pgmap v330: 105 pgs: 105 active+clean; 4.8 MiB data, 474 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.1 MiB/s wr, 348 op/s 2024-09-14T04:02:59.962 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:02:59.963 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:02:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-14T04:03:00.423 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.SparsifyClone (1545 ms) 2024-09-14T04:03:00.423 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestInternal.MissingDataPool 2024-09-14T04:03:01.159 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:00 smithi129 ceph-mon[101316]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T04:03:01.160 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:00 smithi129 ceph-mon[101316]: osdmap e560: 8 total, 8 up, 8 in 2024-09-14T04:03:01.160 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:00 smithi129 ceph-mon[101316]: osdmap e561: 8 total, 8 up, 8 in 2024-09-14T04:03:01.230 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:00 smithi039 ceph-mon[106470]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T04:03:01.230 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:00 smithi039 ceph-mon[106470]: osdmap e560: 8 total, 8 up, 8 in 2024-09-14T04:03:01.231 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:00 smithi039 ceph-mon[106470]: osdmap e561: 8 total, 8 up, 8 in 2024-09-14T04:03:01.231 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:00 smithi039 ceph-mon[111235]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T04:03:01.231 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:00 smithi039 ceph-mon[111235]: osdmap e560: 8 total, 8 up, 8 in 2024-09-14T04:03:01.231 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:00 smithi039 ceph-mon[111235]: osdmap e561: 8 total, 8 up, 8 in 2024-09-14T04:03:01.527 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestInternal.MissingDataPool (1103 ms) 2024-09-14T04:03:02.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:01 smithi129 ceph-mon[101316]: pgmap v333: 105 pgs: 105 active+clean; 4.8 MiB data, 474 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.1 MiB/s wr, 348 op/s 2024-09-14T04:03:02.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:01 smithi129 ceph-mon[101316]: osdmap e562: 8 total, 8 up, 8 in 2024-09-14T04:03:02.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:01 smithi039 ceph-mon[106470]: pgmap v333: 105 pgs: 105 active+clean; 4.8 MiB data, 474 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.1 MiB/s wr, 348 op/s 2024-09-14T04:03:02.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:01 smithi039 ceph-mon[106470]: osdmap e562: 8 total, 8 up, 8 in 2024-09-14T04:03:02.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:01 smithi039 ceph-mon[111235]: pgmap v333: 105 pgs: 105 active+clean; 4.8 MiB data, 474 MiB used, 715 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.1 MiB/s wr, 348 op/s 2024-09-14T04:03:02.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:01 smithi039 ceph-mon[111235]: osdmap e562: 8 total, 8 up, 8 in 2024-09-14T04:03:02.423 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 34 tests from TestInternal (34939 ms total) 2024-09-14T04:03:02.423 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:03:02.423 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 17 tests from TestImageWatcher 2024-09-14T04:03:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:03 smithi129 ceph-mon[101316]: osdmap e563: 8 total, 8 up, 8 in 2024-09-14T04:03:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:03 smithi129 ceph-mon[101316]: pgmap v336: 73 pgs: 73 active+clean; 582 KiB data, 477 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:03:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:03 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:03 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:03 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:03:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:03 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:03:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:03 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:03.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:03 smithi039 ceph-mon[106470]: osdmap e563: 8 total, 8 up, 8 in 2024-09-14T04:03:03.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:03 smithi039 ceph-mon[106470]: pgmap v336: 73 pgs: 73 active+clean; 582 KiB data, 477 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:03:03.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:03 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:03.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:03 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:03.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:03 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:03:03.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:03 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:03:03.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:03 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:03.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:03 smithi039 ceph-mon[111235]: osdmap e563: 8 total, 8 up, 8 in 2024-09-14T04:03:03.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:03 smithi039 ceph-mon[111235]: pgmap v336: 73 pgs: 73 active+clean; 582 KiB data, 477 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:03:03.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:03 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:03.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:03 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:03.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:03 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:03:03.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:03 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:03:03.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:03 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:04.445 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestImageWatcher.NotifyHeaderUpdate 2024-09-14T04:03:04.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:04 smithi129 ceph-mon[101316]: osdmap e564: 8 total, 8 up, 8 in 2024-09-14T04:03:04.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:04 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/491963914' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:04.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:04 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:04.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:04 smithi039 ceph-mon[106470]: osdmap e564: 8 total, 8 up, 8 in 2024-09-14T04:03:04.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:04 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/491963914' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:04.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:04 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:04.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:04 smithi039 ceph-mon[111235]: osdmap e564: 8 total, 8 up, 8 in 2024-09-14T04:03:04.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:04 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/491963914' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:04.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:04 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-14","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:05 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:05 smithi129 ceph-mon[101316]: osdmap e565: 8 total, 8 up, 8 in 2024-09-14T04:03:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:05 smithi129 ceph-mon[101316]: pgmap v339: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 477 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:03:05.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:05 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:05.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:05 smithi039 ceph-mon[106470]: osdmap e565: 8 total, 8 up, 8 in 2024-09-14T04:03:05.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:05 smithi039 ceph-mon[106470]: pgmap v339: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 477 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:03:05.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:05 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-14","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:05.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:05 smithi039 ceph-mon[111235]: osdmap e565: 8 total, 8 up, 8 in 2024-09-14T04:03:05.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:05 smithi039 ceph-mon[111235]: pgmap v339: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 477 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:03:06.486 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestImageWatcher.NotifyHeaderUpdate (2040 ms) 2024-09-14T04:03:06.486 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestImageWatcher.NotifyFlatten 2024-09-14T04:03:06.509 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestImageWatcher.NotifyFlatten (24 ms) 2024-09-14T04:03:06.509 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestImageWatcher.NotifyResize 2024-09-14T04:03:06.536 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestImageWatcher.NotifyResize (26 ms) 2024-09-14T04:03:06.536 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestImageWatcher.NotifyRebuildObjectMap 2024-09-14T04:03:06.559 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestImageWatcher.NotifyRebuildObjectMap (23 ms) 2024-09-14T04:03:06.559 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestImageWatcher.NotifyUpdateFeatures 2024-09-14T04:03:06.580 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestImageWatcher.NotifyUpdateFeatures (22 ms) 2024-09-14T04:03:06.580 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestImageWatcher.NotifySnapCreate 2024-09-14T04:03:06.602 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestImageWatcher.NotifySnapCreate (22 ms) 2024-09-14T04:03:06.603 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestImageWatcher.NotifySnapCreateError 2024-09-14T04:03:06.625 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestImageWatcher.NotifySnapCreateError (23 ms) 2024-09-14T04:03:06.626 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestImageWatcher.NotifySnapRename 2024-09-14T04:03:06.644 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestImageWatcher.NotifySnapRename (19 ms) 2024-09-14T04:03:06.644 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestImageWatcher.NotifySnapRenameError 2024-09-14T04:03:06.663 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestImageWatcher.NotifySnapRenameError (19 ms) 2024-09-14T04:03:06.663 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestImageWatcher.NotifySnapRemove 2024-09-14T04:03:06.687 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestImageWatcher.NotifySnapRemove (24 ms) 2024-09-14T04:03:06.687 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestImageWatcher.NotifySnapProtect 2024-09-14T04:03:06.707 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestImageWatcher.NotifySnapProtect (20 ms) 2024-09-14T04:03:06.708 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestImageWatcher.NotifySnapUnprotect 2024-09-14T04:03:06.726 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestImageWatcher.NotifySnapUnprotect (19 ms) 2024-09-14T04:03:06.727 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestImageWatcher.NotifyRename 2024-09-14T04:03:06.745 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestImageWatcher.NotifyRename (19 ms) 2024-09-14T04:03:06.746 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestImageWatcher.NotifyAsyncTimedOut 2024-09-14T04:03:06.764 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestImageWatcher.NotifyAsyncTimedOut (18 ms) 2024-09-14T04:03:06.764 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestImageWatcher.NotifyAsyncError 2024-09-14T04:03:06.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:06 smithi129 ceph-mon[101316]: osdmap e566: 8 total, 8 up, 8 in 2024-09-14T04:03:06.784 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestImageWatcher.NotifyAsyncError (20 ms) 2024-09-14T04:03:06.784 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestImageWatcher.NotifyAsyncCompleteError 2024-09-14T04:03:06.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:06 smithi039 ceph-mon[106470]: osdmap e566: 8 total, 8 up, 8 in 2024-09-14T04:03:06.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:06 smithi039 ceph-mon[111235]: osdmap e566: 8 total, 8 up, 8 in 2024-09-14T04:03:06.804 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestImageWatcher.NotifyAsyncCompleteError (21 ms) 2024-09-14T04:03:06.805 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestImageWatcher.NotifyAsyncRequestTimedOut 2024-09-14T04:03:06.822 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestImageWatcher.NotifyAsyncRequestTimedOut (18 ms) 2024-09-14T04:03:07.470 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 17 tests from TestImageWatcher (2377 ms total) 2024-09-14T04:03:07.470 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:03:07.470 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 170 tests from TestLibRBD 2024-09-14T04:03:07.500 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.CreateAndStat 2024-09-14T04:03:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:07 smithi129 ceph-mon[101316]: osdmap e567: 8 total, 8 up, 8 in 2024-09-14T04:03:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:07 smithi129 ceph-mon[101316]: pgmap v342: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 477 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:03:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:07 smithi039 ceph-mon[106470]: osdmap e567: 8 total, 8 up, 8 in 2024-09-14T04:03:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:07 smithi039 ceph-mon[106470]: pgmap v342: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 477 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:03:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:07.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:07 smithi039 ceph-mon[111235]: osdmap e567: 8 total, 8 up, 8 in 2024-09-14T04:03:07.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:07 smithi039 ceph-mon[111235]: pgmap v342: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 477 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:03:07.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:08.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:08 smithi129 ceph-mon[101316]: osdmap e568: 8 total, 8 up, 8 in 2024-09-14T04:03:08.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:08 smithi039 ceph-mon[106470]: osdmap e568: 8 total, 8 up, 8 in 2024-09-14T04:03:08.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:03:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:03:08] "GET /metrics HTTP/1.1" 200 36360 "" "Prometheus/2.51.0" 2024-09-14T04:03:08.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:08 smithi039 ceph-mon[111235]: osdmap e568: 8 total, 8 up, 8 in 2024-09-14T04:03:09.501 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:03:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:09 smithi129 ceph-mon[101316]: osdmap e569: 8 total, 8 up, 8 in 2024-09-14T04:03:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:09 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3741249510' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:09 smithi129 ceph-mon[101316]: pgmap v345: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:03:09.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:09 smithi039 ceph-mon[111235]: osdmap e569: 8 total, 8 up, 8 in 2024-09-14T04:03:09.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:09 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3741249510' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:09.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:09 smithi039 ceph-mon[111235]: pgmap v345: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:03:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:09 smithi039 ceph-mon[106470]: osdmap e569: 8 total, 8 up, 8 in 2024-09-14T04:03:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:09 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3741249510' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-15","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:09.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:09 smithi039 ceph-mon[106470]: pgmap v345: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:03:10.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:10 smithi129 ceph-mon[101316]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:10.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:10 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3741249510' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:10.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:10 smithi129 ceph-mon[101316]: osdmap e570: 8 total, 8 up, 8 in 2024-09-14T04:03:10.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:10 smithi039 ceph-mon[111235]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:10.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:10 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3741249510' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:10.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:10 smithi039 ceph-mon[111235]: osdmap e570: 8 total, 8 up, 8 in 2024-09-14T04:03:10.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:10 smithi039 ceph-mon[106470]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:10.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:10 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3741249510' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-15","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:10.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:10 smithi039 ceph-mon[106470]: osdmap e570: 8 total, 8 up, 8 in 2024-09-14T04:03:11.574 INFO:tasks.workunit.client.0.smithi039.stdout:image has size 2097152 and order 22 2024-09-14T04:03:11.575 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.CreateAndStat (4075 ms) 2024-09-14T04:03:11.575 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.CreateWithSameDataPool 2024-09-14T04:03:11.575 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:03:11.592 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.CreateWithSameDataPool (17 ms) 2024-09-14T04:03:11.592 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.CreateAndStatPP 2024-09-14T04:03:11.592 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:03:11.616 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.CreateAndStatPP (25 ms) 2024-09-14T04:03:11.616 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.GetId 2024-09-14T04:03:11.616 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:03:11.639 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.GetId (22 ms) 2024-09-14T04:03:11.639 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.GetIdPP 2024-09-14T04:03:11.639 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:03:11.665 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.GetIdPP (26 ms) 2024-09-14T04:03:11.665 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.GetBlockNamePrefix 2024-09-14T04:03:11.665 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:03:11.682 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.GetBlockNamePrefix (18 ms) 2024-09-14T04:03:11.682 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.GetBlockNamePrefixPP 2024-09-14T04:03:11.682 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:03:11.698 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.GetBlockNamePrefixPP (16 ms) 2024-09-14T04:03:11.699 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestGetCreateTimestamp 2024-09-14T04:03:11.699 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:03:11.713 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestGetCreateTimestamp (15 ms) 2024-09-14T04:03:11.714 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.GetCreateTimestampPP 2024-09-14T04:03:11.714 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:03:11.729 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.GetCreateTimestampPP (15 ms) 2024-09-14T04:03:11.729 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.OpenAio 2024-09-14T04:03:11.729 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:03:11.743 INFO:tasks.workunit.client.0.smithi039.stdout:image has size 2097152 and order 22 2024-09-14T04:03:11.743 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.OpenAio (15 ms) 2024-09-14T04:03:11.744 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.OpenAioFail 2024-09-14T04:03:11.746 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.OpenAioFail (2 ms) 2024-09-14T04:03:11.746 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.OpenAioPP 2024-09-14T04:03:11.746 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:03:11.767 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.OpenAioPP (22 ms) 2024-09-14T04:03:11.767 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.OpenAioFailPP 2024-09-14T04:03:11.770 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.OpenAioFailPP (3 ms) 2024-09-14T04:03:11.771 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.ResizeAndStat 2024-09-14T04:03:11.771 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:03:11.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:11 smithi039 ceph-mon[106470]: osdmap e571: 8 total, 8 up, 8 in 2024-09-14T04:03:11.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:11 smithi039 ceph-mon[106470]: pgmap v348: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:03:11.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:11 smithi039 ceph-mon[111235]: osdmap e571: 8 total, 8 up, 8 in 2024-09-14T04:03:11.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:11 smithi039 ceph-mon[111235]: pgmap v348: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:03:11.805 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.ResizeAndStat (35 ms) 2024-09-14T04:03:11.805 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.ResizeAndStatPP 2024-09-14T04:03:11.805 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:03:11.833 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.ResizeAndStatPP (28 ms) 2024-09-14T04:03:11.833 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.UpdateWatchAndResize 2024-09-14T04:03:11.833 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:03:11.863 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.UpdateWatchAndResize (30 ms) 2024-09-14T04:03:11.864 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.UpdateWatchAndResizePP 2024-09-14T04:03:11.864 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:03:11.899 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.UpdateWatchAndResizePP (35 ms) 2024-09-14T04:03:11.899 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestCreateLsDelete 2024-09-14T04:03:12.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:11 smithi129 ceph-mon[101316]: osdmap e571: 8 total, 8 up, 8 in 2024-09-14T04:03:12.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:11 smithi129 ceph-mon[101316]: pgmap v348: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 462 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:03:13.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:12 smithi129 ceph-mon[101316]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:13.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:12 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T04:03:13.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:12 smithi129 ceph-mon[101316]: osdmap e572: 8 total, 8 up, 8 in 2024-09-14T04:03:13.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:12 smithi039 ceph-mon[106470]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:13.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:12 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T04:03:13.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:12 smithi039 ceph-mon[106470]: osdmap e572: 8 total, 8 up, 8 in 2024-09-14T04:03:13.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:12 smithi039 ceph-mon[111235]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:13.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:12 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T04:03:13.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:12 smithi039 ceph-mon[111235]: osdmap e572: 8 total, 8 up, 8 in 2024-09-14T04:03:13.606 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:03:13.922 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:13 smithi129 ceph-mon[101316]: osdmap e573: 8 total, 8 up, 8 in 2024-09-14T04:03:13.923 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:13 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/795373940' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:13.923 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:13 smithi129 ceph-mon[101316]: pgmap v351: 137 pgs: 32 creating+peering, 105 active+clean; 583 KiB data, 467 MiB used, 715 GiB / 715 GiB avail; 181 KiB/s rd, 20 KiB/s wr, 250 op/s 2024-09-14T04:03:14.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:13 smithi039 ceph-mon[111235]: osdmap e573: 8 total, 8 up, 8 in 2024-09-14T04:03:14.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:13 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/795373940' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:14.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:13 smithi039 ceph-mon[111235]: pgmap v351: 137 pgs: 32 creating+peering, 105 active+clean; 583 KiB data, 467 MiB used, 715 GiB / 715 GiB avail; 181 KiB/s rd, 20 KiB/s wr, 250 op/s 2024-09-14T04:03:14.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:13 smithi039 ceph-mon[106470]: osdmap e573: 8 total, 8 up, 8 in 2024-09-14T04:03:14.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:13 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/795373940' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-16","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:14.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:13 smithi039 ceph-mon[106470]: pgmap v351: 137 pgs: 32 creating+peering, 105 active+clean; 583 KiB data, 467 MiB used, 715 GiB / 715 GiB avail; 181 KiB/s rd, 20 KiB/s wr, 250 op/s 2024-09-14T04:03:15.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:14 smithi129 ceph-mon[101316]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:15.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:14 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/795373940' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:15.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:14 smithi129 ceph-mon[101316]: osdmap e574: 8 total, 8 up, 8 in 2024-09-14T04:03:15.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:14 smithi039 ceph-mon[106470]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:15.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:14 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/795373940' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:15.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:14 smithi039 ceph-mon[106470]: osdmap e574: 8 total, 8 up, 8 in 2024-09-14T04:03:15.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:14 smithi039 ceph-mon[111235]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:15.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:14 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/795373940' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-16","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:15.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:14 smithi039 ceph-mon[111235]: osdmap e574: 8 total, 8 up, 8 in 2024-09-14T04:03:15.435 INFO:tasks.workunit.client.0.smithi039.stdout:image: image18 2024-09-14T04:03:15.435 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image18 2024-09-14T04:03:15.436 INFO:tasks.workunit.client.0.smithi039.stdout:found image18 2024-09-14T04:03:15.436 INFO:tasks.workunit.client.0.smithi039.stdout:erased image18 2024-09-14T04:03:15.436 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:03:15.518 INFO:tasks.workunit.client.0.smithi039.stdout:image: image18 2024-09-14T04:03:15.518 INFO:tasks.workunit.client.0.smithi039.stdout:image: image19 2024-09-14T04:03:15.518 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image18 2024-09-14T04:03:15.518 INFO:tasks.workunit.client.0.smithi039.stdout:found image18 2024-09-14T04:03:15.518 INFO:tasks.workunit.client.0.smithi039.stdout:erased image18 2024-09-14T04:03:15.518 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image19 2024-09-14T04:03:15.518 INFO:tasks.workunit.client.0.smithi039.stdout:found image19 2024-09-14T04:03:15.518 INFO:tasks.workunit.client.0.smithi039.stdout:erased image19 2024-09-14T04:03:15.518 INFO:tasks.workunit.client.0.smithi039.stdout:image: image19 2024-09-14T04:03:15.518 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image19 2024-09-14T04:03:15.519 INFO:tasks.workunit.client.0.smithi039.stdout:found image19 2024-09-14T04:03:15.519 INFO:tasks.workunit.client.0.smithi039.stdout:erased image19 2024-09-14T04:03:15.519 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestCreateLsDelete (3619 ms) 2024-09-14T04:03:15.519 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestCreateLsDeletePP 2024-09-14T04:03:16.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:15 smithi129 ceph-mon[101316]: osdmap e575: 8 total, 8 up, 8 in 2024-09-14T04:03:16.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:15 smithi129 ceph-mon[101316]: pgmap v354: 137 pgs: 32 creating+peering, 105 active+clean; 583 KiB data, 467 MiB used, 715 GiB / 715 GiB avail; 181 KiB/s rd, 20 KiB/s wr, 250 op/s 2024-09-14T04:03:16.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:15 smithi129 ceph-mon[101316]: osdmap e576: 8 total, 8 up, 8 in 2024-09-14T04:03:16.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:15 smithi039 ceph-mon[106470]: osdmap e575: 8 total, 8 up, 8 in 2024-09-14T04:03:16.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:15 smithi039 ceph-mon[106470]: pgmap v354: 137 pgs: 32 creating+peering, 105 active+clean; 583 KiB data, 467 MiB used, 715 GiB / 715 GiB avail; 181 KiB/s rd, 20 KiB/s wr, 250 op/s 2024-09-14T04:03:16.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:15 smithi039 ceph-mon[106470]: osdmap e576: 8 total, 8 up, 8 in 2024-09-14T04:03:16.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:15 smithi039 ceph-mon[111235]: osdmap e575: 8 total, 8 up, 8 in 2024-09-14T04:03:16.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:15 smithi039 ceph-mon[111235]: pgmap v354: 137 pgs: 32 creating+peering, 105 active+clean; 583 KiB data, 467 MiB used, 715 GiB / 715 GiB avail; 181 KiB/s rd, 20 KiB/s wr, 250 op/s 2024-09-14T04:03:16.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:15 smithi039 ceph-mon[111235]: osdmap e576: 8 total, 8 up, 8 in 2024-09-14T04:03:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:16 smithi129 ceph-mon[101316]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:16 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T04:03:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:16 smithi129 ceph-mon[101316]: osdmap e577: 8 total, 8 up, 8 in 2024-09-14T04:03:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:16 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/482775526' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:17.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:16 smithi039 ceph-mon[106470]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:17.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:16 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T04:03:17.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:16 smithi039 ceph-mon[106470]: osdmap e577: 8 total, 8 up, 8 in 2024-09-14T04:03:17.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:16 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/482775526' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:17.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:16 smithi039 ceph-mon[111235]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:17.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:16 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T04:03:17.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:16 smithi039 ceph-mon[111235]: osdmap e577: 8 total, 8 up, 8 in 2024-09-14T04:03:17.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:16 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/482775526' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-17","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:17.427 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:03:18.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:17 smithi129 ceph-mon[101316]: pgmap v357: 169 pgs: 32 unknown, 32 creating+peering, 105 active+clean; 583 KiB data, 467 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:03:18.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:17 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/482775526' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:18.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:17 smithi129 ceph-mon[101316]: osdmap e578: 8 total, 8 up, 8 in 2024-09-14T04:03:18.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:17 smithi039 ceph-mon[106470]: pgmap v357: 169 pgs: 32 unknown, 32 creating+peering, 105 active+clean; 583 KiB data, 467 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:03:18.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:17 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/482775526' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:18.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:17 smithi039 ceph-mon[106470]: osdmap e578: 8 total, 8 up, 8 in 2024-09-14T04:03:18.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:17 smithi039 ceph-mon[111235]: pgmap v357: 169 pgs: 32 unknown, 32 creating+peering, 105 active+clean; 583 KiB data, 467 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:03:18.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:17 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/482775526' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-17","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:18.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:17 smithi039 ceph-mon[111235]: osdmap e578: 8 total, 8 up, 8 in 2024-09-14T04:03:19.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:18 smithi129 ceph-mon[101316]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:19.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:18 smithi129 ceph-mon[101316]: osdmap e579: 8 total, 8 up, 8 in 2024-09-14T04:03:19.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:18 smithi039 ceph-mon[106470]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:19.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:18 smithi039 ceph-mon[106470]: osdmap e579: 8 total, 8 up, 8 in 2024-09-14T04:03:19.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:03:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:03:18] "GET /metrics HTTP/1.1" 200 36360 "" "Prometheus/2.51.0" 2024-09-14T04:03:19.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:18 smithi039 ceph-mon[111235]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:19.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:18 smithi039 ceph-mon[111235]: osdmap e579: 8 total, 8 up, 8 in 2024-09-14T04:03:19.455 INFO:tasks.workunit.client.0.smithi039.stdout:num images is: 1 2024-09-14T04:03:19.455 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 1 2024-09-14T04:03:19.455 INFO:tasks.workunit.client.0.smithi039.stdout:image: image20 2024-09-14T04:03:19.455 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image20 2024-09-14T04:03:19.455 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:03:19.465 INFO:tasks.workunit.client.0.smithi039.stdout:num images is: 2 2024-09-14T04:03:19.465 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 2 2024-09-14T04:03:19.466 INFO:tasks.workunit.client.0.smithi039.stdout:image: image20 2024-09-14T04:03:19.466 INFO:tasks.workunit.client.0.smithi039.stdout:image: image21 2024-09-14T04:03:19.466 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image20 2024-09-14T04:03:19.466 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image21 2024-09-14T04:03:19.523 INFO:tasks.workunit.client.0.smithi039.stdout:num images is: 1 2024-09-14T04:03:19.523 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 1 2024-09-14T04:03:19.523 INFO:tasks.workunit.client.0.smithi039.stdout:image: image21 2024-09-14T04:03:19.523 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image21 2024-09-14T04:03:19.523 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestCreateLsDeletePP (4006 ms) 2024-09-14T04:03:19.523 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestCopy 2024-09-14T04:03:20.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:19 smithi129 ceph-mon[101316]: pgmap v360: 169 pgs: 169 active+clean; 583 KiB data, 476 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 3.2 KiB/s wr, 41 op/s 2024-09-14T04:03:20.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:19 smithi129 ceph-mon[101316]: osdmap e580: 8 total, 8 up, 8 in 2024-09-14T04:03:20.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:19 smithi039 ceph-mon[106470]: pgmap v360: 169 pgs: 169 active+clean; 583 KiB data, 476 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 3.2 KiB/s wr, 41 op/s 2024-09-14T04:03:20.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:19 smithi039 ceph-mon[106470]: osdmap e580: 8 total, 8 up, 8 in 2024-09-14T04:03:20.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:19 smithi039 ceph-mon[111235]: pgmap v360: 169 pgs: 169 active+clean; 583 KiB data, 476 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 3.2 KiB/s wr, 41 op/s 2024-09-14T04:03:20.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:19 smithi039 ceph-mon[111235]: osdmap e580: 8 total, 8 up, 8 in 2024-09-14T04:03:20.880 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:20 smithi039 ceph-mon[106470]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:20.881 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:20 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T04:03:20.881 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:20 smithi039 ceph-mon[106470]: osdmap e581: 8 total, 8 up, 8 in 2024-09-14T04:03:20.881 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:20 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1492926914' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:20.881 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:20 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:20.881 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:20 smithi039 ceph-mon[111235]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:20.881 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:20 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T04:03:20.881 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:20 smithi039 ceph-mon[111235]: osdmap e581: 8 total, 8 up, 8 in 2024-09-14T04:03:20.881 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:20 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1492926914' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:20.881 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:20 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:21.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:20 smithi129 ceph-mon[101316]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:21.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:20 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T04:03:21.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:20 smithi129 ceph-mon[101316]: osdmap e581: 8 total, 8 up, 8 in 2024-09-14T04:03:21.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:20 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1492926914' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:21.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:20 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-18","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:21.410 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'echo wait for all mons to upgrade' 2024-09-14T04:03:21.432 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:03:21.640 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:03:21.942 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:21 smithi039 ceph-mon[106470]: pgmap v363: 201 pgs: 32 unknown, 169 active+clean; 583 KiB data, 476 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 3.2 KiB/s wr, 41 op/s 2024-09-14T04:03:21.942 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:21 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:21.942 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:21 smithi039 ceph-mon[106470]: osdmap e582: 8 total, 8 up, 8 in 2024-09-14T04:03:21.942 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:21 smithi039 ceph-mon[111235]: pgmap v363: 201 pgs: 32 unknown, 169 active+clean; 583 KiB data, 476 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 3.2 KiB/s wr, 41 op/s 2024-09-14T04:03:21.942 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:21 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:21.943 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:21 smithi039 ceph-mon[111235]: osdmap e582: 8 total, 8 up, 8 in 2024-09-14T04:03:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:21 smithi129 ceph-mon[101316]: pgmap v363: 201 pgs: 32 unknown, 169 active+clean; 583 KiB data, 476 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 3.2 KiB/s wr, 41 op/s 2024-09-14T04:03:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:21 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-18","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:21 smithi129 ceph-mon[101316]: osdmap e582: 8 total, 8 up, 8 in 2024-09-14T04:03:22.923 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:22 smithi039 ceph-mon[106470]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:22.923 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:22 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:22.923 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:22.924 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:22 smithi039 ceph-mon[106470]: osdmap e583: 8 total, 8 up, 8 in 2024-09-14T04:03:22.924 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:22 smithi039 ceph-mon[111235]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:22.924 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:22 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:22.924 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:22.924 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:22 smithi039 ceph-mon[111235]: osdmap e583: 8 total, 8 up, 8 in 2024-09-14T04:03:23.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:22 smithi129 ceph-mon[101316]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:23.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:22 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:23.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:23.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:22 smithi129 ceph-mon[101316]: osdmap e583: 8 total, 8 up, 8 in 2024-09-14T04:03:23.427 INFO:teuthology.orchestra.run.smithi039.stdout:wait for all mons to upgrade 2024-09-14T04:03:23.823 INFO:tasks.workunit.client.0.smithi039.stdout:image: image22 2024-09-14T04:03:23.823 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image22 2024-09-14T04:03:23.823 INFO:tasks.workunit.client.0.smithi039.stdout:found image22 2024-09-14T04:03:23.823 INFO:tasks.workunit.client.0.smithi039.stdout:erased image22 2024-09-14T04:03:23.823 INFO:tasks.workunit.client.0.smithi039.stdout:image: image22 2024-09-14T04:03:23.823 INFO:tasks.workunit.client.0.smithi039.stdout:image: image23 2024-09-14T04:03:23.824 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image22 2024-09-14T04:03:23.824 INFO:tasks.workunit.client.0.smithi039.stdout:found image22 2024-09-14T04:03:23.824 INFO:tasks.workunit.client.0.smithi039.stdout:erased image22 2024-09-14T04:03:23.824 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image23 2024-09-14T04:03:23.824 INFO:tasks.workunit.client.0.smithi039.stdout:found image23 2024-09-14T04:03:23.824 INFO:tasks.workunit.client.0.smithi039.stdout:erased image23 2024-09-14T04:03:23.824 INFO:tasks.workunit.client.0.smithi039.stdout:100.00% done 2024-09-14T04:03:23.824 INFO:tasks.workunit.client.0.smithi039.stdout:image: image22 2024-09-14T04:03:23.824 INFO:tasks.workunit.client.0.smithi039.stdout:image: image23 2024-09-14T04:03:23.824 INFO:tasks.workunit.client.0.smithi039.stdout:image: image24 2024-09-14T04:03:23.824 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image22 2024-09-14T04:03:23.824 INFO:tasks.workunit.client.0.smithi039.stdout:found image22 2024-09-14T04:03:23.824 INFO:tasks.workunit.client.0.smithi039.stdout:erased image22 2024-09-14T04:03:23.824 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image23 2024-09-14T04:03:23.825 INFO:tasks.workunit.client.0.smithi039.stdout:found image23 2024-09-14T04:03:23.825 INFO:tasks.workunit.client.0.smithi039.stdout:erased image23 2024-09-14T04:03:23.825 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image24 2024-09-14T04:03:23.825 INFO:tasks.workunit.client.0.smithi039.stdout:found image24 2024-09-14T04:03:23.825 INFO:tasks.workunit.client.0.smithi039.stdout:erased image24 2024-09-14T04:03:23.825 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestCopy (4299 ms) 2024-09-14T04:03:23.825 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestCopyPP 2024-09-14T04:03:24.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:23 smithi129 ceph-mon[101316]: pgmap v366: 201 pgs: 201 active+clean; 583 KiB data, 476 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 3.5 KiB/s wr, 41 op/s 2024-09-14T04:03:24.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:23 smithi129 ceph-mon[101316]: osdmap e584: 8 total, 8 up, 8 in 2024-09-14T04:03:24.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:23 smithi039 ceph-mon[106470]: pgmap v366: 201 pgs: 201 active+clean; 583 KiB data, 476 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 3.5 KiB/s wr, 41 op/s 2024-09-14T04:03:24.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:23 smithi039 ceph-mon[106470]: osdmap e584: 8 total, 8 up, 8 in 2024-09-14T04:03:24.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:23 smithi039 ceph-mon[111235]: pgmap v366: 201 pgs: 201 active+clean; 583 KiB data, 476 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 3.5 KiB/s wr, 41 op/s 2024-09-14T04:03:24.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:23 smithi039 ceph-mon[111235]: osdmap e584: 8 total, 8 up, 8 in 2024-09-14T04:03:24.163 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types mon' 2024-09-14T04:03:24.410 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:03:24.786 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:24 smithi039 ceph-mon[106470]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:24.786 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:24 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T04:03:24.787 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:24 smithi039 ceph-mon[111235]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:24.787 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:24 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T04:03:25.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:24 smithi129 ceph-mon[101316]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:25.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:24 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T04:03:25.436 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:03:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:25 smithi039 ceph-mon[106470]: osdmap e585: 8 total, 8 up, 8 in 2024-09-14T04:03:25.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:25 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1460008327' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:25.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:25 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:25.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:25 smithi039 ceph-mon[106470]: pgmap v369: 233 pgs: 32 unknown, 201 active+clean; 583 KiB data, 476 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 3.5 KiB/s wr, 41 op/s 2024-09-14T04:03:25.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:25 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:25.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:25 smithi039 ceph-mon[106470]: osdmap e586: 8 total, 8 up, 8 in 2024-09-14T04:03:25.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:25 smithi039 ceph-mon[111235]: osdmap e585: 8 total, 8 up, 8 in 2024-09-14T04:03:25.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:25 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1460008327' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:25.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:25 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:25.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:25 smithi039 ceph-mon[111235]: pgmap v369: 233 pgs: 32 unknown, 201 active+clean; 583 KiB data, 476 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 3.5 KiB/s wr, 41 op/s 2024-09-14T04:03:25.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:25 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:25.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:25 smithi039 ceph-mon[111235]: osdmap e586: 8 total, 8 up, 8 in 2024-09-14T04:03:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:25 smithi129 ceph-mon[101316]: osdmap e585: 8 total, 8 up, 8 in 2024-09-14T04:03:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:25 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1460008327' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:25 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-19","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:25 smithi129 ceph-mon[101316]: pgmap v369: 233 pgs: 32 unknown, 201 active+clean; 583 KiB data, 476 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 3.5 KiB/s wr, 41 op/s 2024-09-14T04:03:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:25 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-19","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:26.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:25 smithi129 ceph-mon[101316]: osdmap e586: 8 total, 8 up, 8 in 2024-09-14T04:03:27.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:26 smithi129 ceph-mon[101316]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:27.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:26 smithi129 ceph-mon[101316]: osdmap e587: 8 total, 8 up, 8 in 2024-09-14T04:03:27.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:26 smithi039 ceph-mon[106470]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:27.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:26 smithi039 ceph-mon[106470]: osdmap e587: 8 total, 8 up, 8 in 2024-09-14T04:03:27.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:26 smithi039 ceph-mon[111235]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:27.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:26 smithi039 ceph-mon[111235]: osdmap e587: 8 total, 8 up, 8 in 2024-09-14T04:03:27.696 INFO:tasks.workunit.client.0.smithi039.stdout:num images is: 1 2024-09-14T04:03:27.696 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 1 2024-09-14T04:03:27.696 INFO:tasks.workunit.client.0.smithi039.stdout:image: image25 2024-09-14T04:03:27.696 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image25 2024-09-14T04:03:27.714 INFO:tasks.workunit.client.0.smithi039.stdout:num images is: 2 2024-09-14T04:03:27.714 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 2 2024-09-14T04:03:27.714 INFO:tasks.workunit.client.0.smithi039.stdout:image: image25 2024-09-14T04:03:27.714 INFO:tasks.workunit.client.0.smithi039.stdout:image: image26 2024-09-14T04:03:27.714 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image25 2024-09-14T04:03:27.714 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image26 2024-09-14T04:03:27.759 INFO:tasks.workunit.client.0.smithi039.stdout:100.00% done 2024-09-14T04:03:27.759 INFO:tasks.workunit.client.0.smithi039.stdout:num images is: 3 2024-09-14T04:03:27.760 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 3 2024-09-14T04:03:27.760 INFO:tasks.workunit.client.0.smithi039.stdout:image: image25 2024-09-14T04:03:27.760 INFO:tasks.workunit.client.0.smithi039.stdout:image: image26 2024-09-14T04:03:27.760 INFO:tasks.workunit.client.0.smithi039.stdout:image: image27 2024-09-14T04:03:27.760 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image25 2024-09-14T04:03:27.760 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image26 2024-09-14T04:03:27.760 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image27 2024-09-14T04:03:27.787 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestCopyPP (3965 ms) 2024-09-14T04:03:27.787 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestDeepCopy 2024-09-14T04:03:28.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:27 smithi129 ceph-mon[101316]: from='client.34371 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "daemon_types": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:03:28.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:27 smithi129 ceph-mon[101316]: pgmap v372: 233 pgs: 32 unknown, 201 active+clean; 583 KiB data, 476 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:03:28.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:27 smithi129 ceph-mon[101316]: osdmap e588: 8 total, 8 up, 8 in 2024-09-14T04:03:28.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:27 smithi039 ceph-mon[106470]: from='client.34371 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "daemon_types": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:03:28.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:27 smithi039 ceph-mon[106470]: pgmap v372: 233 pgs: 32 unknown, 201 active+clean; 583 KiB data, 476 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:03:28.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:27 smithi039 ceph-mon[106470]: osdmap e588: 8 total, 8 up, 8 in 2024-09-14T04:03:28.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:27 smithi039 ceph-mon[111235]: from='client.34371 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "daemon_types": "mon", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:03:28.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:27 smithi039 ceph-mon[111235]: pgmap v372: 233 pgs: 32 unknown, 201 active+clean; 583 KiB data, 476 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:03:28.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:27 smithi039 ceph-mon[111235]: osdmap e588: 8 total, 8 up, 8 in 2024-09-14T04:03:28.366 INFO:teuthology.orchestra.run.smithi039.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T04:03:28.945 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:03:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:03:28] "GET /metrics HTTP/1.1" 200 43527 "" "Prometheus/2.51.0" 2024-09-14T04:03:28.945 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:28 smithi039 ceph-mon[106470]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:28.945 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:28 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T04:03:28.946 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:28 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:28.946 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:28 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:03:28.946 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:28 smithi039 ceph-mon[111235]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:28.946 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:28 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T04:03:28.946 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:28 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:28.946 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:28 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:03:28.967 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-14T04:03:29.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:28 smithi129 ceph-mon[101316]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:29.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:28 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T04:03:29.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:28 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:29.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:28 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:03:29.563 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:03:29.758 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:03:30.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:29 smithi039 ceph-mon[106470]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T04:03:30.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:29 smithi039 ceph-mon[106470]: osdmap e589: 8 total, 8 up, 8 in 2024-09-14T04:03:30.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:29 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4024895651' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:30.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:29 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:30.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:29 smithi039 ceph-mon[106470]: pgmap v375: 265 pgs: 32 unknown, 233 active+clean; 583 KiB data, 481 MiB used, 715 GiB / 715 GiB avail; 664 KiB/s rd, 44 KiB/s wr, 777 op/s 2024-09-14T04:03:30.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:29 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:30.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:29 smithi039 ceph-mon[106470]: osdmap e590: 8 total, 8 up, 8 in 2024-09-14T04:03:30.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:29 smithi039 ceph-mon[111235]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T04:03:30.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:29 smithi039 ceph-mon[111235]: osdmap e589: 8 total, 8 up, 8 in 2024-09-14T04:03:30.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:29 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4024895651' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:30.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:29 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:30.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:29 smithi039 ceph-mon[111235]: pgmap v375: 265 pgs: 32 unknown, 233 active+clean; 583 KiB data, 481 MiB used, 715 GiB / 715 GiB avail; 664 KiB/s rd, 44 KiB/s wr, 777 op/s 2024-09-14T04:03:30.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:29 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:30.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:29 smithi039 ceph-mon[111235]: osdmap e590: 8 total, 8 up, 8 in 2024-09-14T04:03:30.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:29 smithi129 ceph-mon[101316]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T04:03:30.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:29 smithi129 ceph-mon[101316]: osdmap e589: 8 total, 8 up, 8 in 2024-09-14T04:03:30.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:29 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4024895651' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:30.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:29 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-20","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:30.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:29 smithi129 ceph-mon[101316]: pgmap v375: 265 pgs: 32 unknown, 233 active+clean; 583 KiB data, 481 MiB used, 715 GiB / 715 GiB avail; 664 KiB/s rd, 44 KiB/s wr, 777 op/s 2024-09-14T04:03:30.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:29 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-20","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:30.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:29 smithi129 ceph-mon[101316]: osdmap e590: 8 total, 8 up, 8 in 2024-09-14T04:03:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:30 smithi039 ceph-mon[106470]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:30 smithi039 ceph-mon[106470]: osdmap e591: 8 total, 8 up, 8 in 2024-09-14T04:03:31.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:30 smithi039 ceph-mon[111235]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:31.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:30 smithi039 ceph-mon[111235]: osdmap e591: 8 total, 8 up, 8 in 2024-09-14T04:03:31.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:30 smithi129 ceph-mon[101316]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:31.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:30 smithi129 ceph-mon[101316]: osdmap e591: 8 total, 8 up, 8 in 2024-09-14T04:03:31.972 INFO:teuthology.orchestra.run.smithi039.stdout:true 2024-09-14T04:03:31.973 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:31 smithi039 ceph-mon[106470]: pgmap v378: 265 pgs: 32 unknown, 233 active+clean; 583 KiB data, 481 MiB used, 715 GiB / 715 GiB avail; 664 KiB/s rd, 44 KiB/s wr, 777 op/s 2024-09-14T04:03:31.973 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:31 smithi039 ceph-mon[106470]: osdmap e592: 8 total, 8 up, 8 in 2024-09-14T04:03:31.973 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:31 smithi039 ceph-mon[111235]: pgmap v378: 265 pgs: 32 unknown, 233 active+clean; 583 KiB data, 481 MiB used, 715 GiB / 715 GiB avail; 664 KiB/s rd, 44 KiB/s wr, 777 op/s 2024-09-14T04:03:31.973 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:31 smithi039 ceph-mon[111235]: osdmap e592: 8 total, 8 up, 8 in 2024-09-14T04:03:32.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:31 smithi129 ceph-mon[101316]: pgmap v378: 265 pgs: 32 unknown, 233 active+clean; 583 KiB data, 481 MiB used, 715 GiB / 715 GiB avail; 664 KiB/s rd, 44 KiB/s wr, 777 op/s 2024-09-14T04:03:32.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:31 smithi129 ceph-mon[101316]: osdmap e592: 8 total, 8 up, 8 in 2024-09-14T04:03:32.583 INFO:teuthology.orchestra.run.smithi039.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T04:03:32.583 INFO:teuthology.orchestra.run.smithi039.stdout:alertmanager.a smithi039 *:9093,9094 running (5m) 93s ago 11m 20.6M - 0.27.0 11f11916f8cd 6b564e1fc909 2024-09-14T04:03:32.584 INFO:teuthology.orchestra.run.smithi039.stdout:grafana.a smithi129 *:3000 running (5m) 29s ago 11m 81.3M - 10.4.0 c8b91775d855 f0492c19de92 2024-09-14T04:03:32.584 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi039.bbwiwf smithi039 running (10m) 93s ago 10m 23.2M - 18.2.4-845-ge4e227b3 67ddd33fa72f 91e6c8f85681 2024-09-14T04:03:32.584 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi129.ybjqcw smithi129 running (10m) 29s ago 10m 16.4M - 18.2.4-845-ge4e227b3 67ddd33fa72f a3fab5b37b71 2024-09-14T04:03:32.584 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.x smithi129 *:8443,9283,8765 running (6m) 29s ago 15m 456M - 19.3.0-4907-ga2782315 8801d2529e43 448bd4e146cb 2024-09-14T04:03:32.584 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.y smithi039 *:8443,9283,8765 running (7m) 93s ago 17m 515M - 19.3.0-4907-ga2782315 8801d2529e43 cccc87f635a6 2024-09-14T04:03:32.584 INFO:teuthology.orchestra.run.smithi039.stdout:mon.a smithi039 running (4m) 93s ago 17m 60.0M 2048M 19.3.0-4907-ga2782315 8801d2529e43 7198d4eb8e4c 2024-09-14T04:03:32.584 INFO:teuthology.orchestra.run.smithi039.stdout:mon.b smithi129 running (2m) 29s ago 15m 40.1M 2048M 19.3.0-4907-ga2782315 8801d2529e43 cff2519560f1 2024-09-14T04:03:32.584 INFO:teuthology.orchestra.run.smithi039.stdout:mon.c smithi039 running (4m) 93s ago 15m 50.3M 2048M 19.3.0-4907-ga2782315 8801d2529e43 4b8f0f3864b1 2024-09-14T04:03:32.584 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.a smithi039 *:9100 running (6m) 93s ago 10m 11.5M - 1.7.0 72c9c2088986 7dd5945adb64 2024-09-14T04:03:32.584 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.b smithi129 *:9100 running (6m) 29s ago 10m 10.8M - 1.7.0 72c9c2088986 1fedb5f257e7 2024-09-14T04:03:32.584 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0 smithi039 running (15m) 93s ago 15m 112M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 5048d74f27ac 2024-09-14T04:03:32.584 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1 smithi039 running (14m) 93s ago 14m 119M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 91349527d88b 2024-09-14T04:03:32.584 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2 smithi039 running (14m) 93s ago 14m 136M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f eb903b44fc5d 2024-09-14T04:03:32.585 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3 smithi039 running (13m) 93s ago 13m 128M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f dff9e665bd9b 2024-09-14T04:03:32.585 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4 smithi129 running (13m) 29s ago 13m 127M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 5b64727311b7 2024-09-14T04:03:32.585 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5 smithi129 running (13m) 29s ago 13m 113M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f f2d991a754f7 2024-09-14T04:03:32.585 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6 smithi129 running (12m) 29s ago 12m 133M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 3060eaf44106 2024-09-14T04:03:32.585 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7 smithi129 running (12m) 29s ago 12m 116M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 4c13b97060d1 2024-09-14T04:03:32.585 INFO:teuthology.orchestra.run.smithi039.stdout:prometheus.a smithi129 *:9095 running (6m) 29s ago 11m 55.9M - 2.51.0 1d3b7f56885b cbe583f852c3 2024-09-14T04:03:32.971 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:03:32.971 INFO:teuthology.orchestra.run.smithi039.stdout: "mon": { 2024-09-14T04:03:32.971 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T04:03:32.972 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:03:32.972 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr": { 2024-09-14T04:03:32.972 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T04:03:32.972 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:03:32.972 INFO:teuthology.orchestra.run.smithi039.stdout: "osd": { 2024-09-14T04:03:32.972 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8 2024-09-14T04:03:32.972 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:03:32.972 INFO:teuthology.orchestra.run.smithi039.stdout: "mds": { 2024-09-14T04:03:32.972 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T04:03:32.972 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:03:32.972 INFO:teuthology.orchestra.run.smithi039.stdout: "overall": { 2024-09-14T04:03:32.972 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 10, 2024-09-14T04:03:32.972 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 5 2024-09-14T04:03:32.973 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:03:32.973 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:03:32.973 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:32 smithi039 ceph-mon[106470]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:32.973 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:32 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T04:03:32.973 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:32 smithi039 ceph-mon[106470]: from='client.34383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:03:32.973 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:32 smithi039 ceph-mon[111235]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:32.973 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:32 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T04:03:32.973 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:32 smithi039 ceph-mon[111235]: from='client.34383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:03:33.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:32 smithi129 ceph-mon[101316]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:33.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:32 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T04:03:33.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:32 smithi129 ceph-mon[101316]: from='client.34383 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:03:33.274 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:03:33.275 INFO:teuthology.orchestra.run.smithi039.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", 2024-09-14T04:03:33.275 INFO:teuthology.orchestra.run.smithi039.stdout: "in_progress": true, 2024-09-14T04:03:33.275 INFO:teuthology.orchestra.run.smithi039.stdout: "which": "Upgrading daemons of type(s) mon", 2024-09-14T04:03:33.275 INFO:teuthology.orchestra.run.smithi039.stdout: "services_complete": [], 2024-09-14T04:03:33.275 INFO:teuthology.orchestra.run.smithi039.stdout: "progress": "", 2024-09-14T04:03:33.275 INFO:teuthology.orchestra.run.smithi039.stdout: "message": "Currently upgrading mon daemons", 2024-09-14T04:03:33.275 INFO:teuthology.orchestra.run.smithi039.stdout: "is_paused": false 2024-09-14T04:03:33.275 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:03:34.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:33 smithi129 ceph-mon[101316]: from='client.44318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:03:34.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:33 smithi129 ceph-mon[101316]: from='client.34395 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:03:34.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:33 smithi129 ceph-mon[101316]: pgmap v380: 265 pgs: 265 active+clean; 583 KiB data, 490 MiB used, 715 GiB / 715 GiB avail; 364 KiB/s rd, 22 KiB/s wr, 424 op/s 2024-09-14T04:03:34.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:33 smithi129 ceph-mon[101316]: osdmap e593: 8 total, 8 up, 8 in 2024-09-14T04:03:34.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:33 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1914154417' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:34.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:33 smithi039 ceph-mon[106470]: from='client.44318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:03:34.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:33 smithi039 ceph-mon[106470]: from='client.34395 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:03:34.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:33 smithi039 ceph-mon[106470]: pgmap v380: 265 pgs: 265 active+clean; 583 KiB data, 490 MiB used, 715 GiB / 715 GiB avail; 364 KiB/s rd, 22 KiB/s wr, 424 op/s 2024-09-14T04:03:34.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:33 smithi039 ceph-mon[106470]: osdmap e593: 8 total, 8 up, 8 in 2024-09-14T04:03:34.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:33 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1914154417' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:34.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:33 smithi039 ceph-mon[111235]: from='client.44318 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:03:34.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:33 smithi039 ceph-mon[111235]: from='client.34395 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:03:34.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:33 smithi039 ceph-mon[111235]: pgmap v380: 265 pgs: 265 active+clean; 583 KiB data, 490 MiB used, 715 GiB / 715 GiB avail; 364 KiB/s rd, 22 KiB/s wr, 424 op/s 2024-09-14T04:03:34.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:33 smithi039 ceph-mon[111235]: osdmap e593: 8 total, 8 up, 8 in 2024-09-14T04:03:34.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:33 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1914154417' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:35.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:34 smithi129 ceph-mon[101316]: from='client.34407 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:03:35.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:34 smithi129 ceph-mon[101316]: osdmap e594: 8 total, 8 up, 8 in 2024-09-14T04:03:35.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:34 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:35.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:34 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:35.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:34 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:03:35.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:34 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:03:35.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:34 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:35.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:34 smithi039 ceph-mon[106470]: from='client.34407 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:03:35.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:34 smithi039 ceph-mon[106470]: osdmap e594: 8 total, 8 up, 8 in 2024-09-14T04:03:35.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:34 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:35.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:34 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:35.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:34 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:03:35.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:34 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:03:35.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:34 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:35.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:34 smithi039 ceph-mon[111235]: from='client.34407 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:03:35.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:34 smithi039 ceph-mon[111235]: osdmap e594: 8 total, 8 up, 8 in 2024-09-14T04:03:35.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:34 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:35.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:34 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:35.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:34 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:03:35.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:34 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:03:35.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:34 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:35.498 INFO:tasks.workunit.client.0.smithi039.stdout:image: image28 2024-09-14T04:03:35.498 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image28 2024-09-14T04:03:35.498 INFO:tasks.workunit.client.0.smithi039.stdout:found image28 2024-09-14T04:03:35.498 INFO:tasks.workunit.client.0.smithi039.stdout:erased image28 2024-09-14T04:03:35.498 INFO:tasks.workunit.client.0.smithi039.stdout:image: image28 2024-09-14T04:03:35.498 INFO:tasks.workunit.client.0.smithi039.stdout:image: image29 2024-09-14T04:03:35.499 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image28 2024-09-14T04:03:35.499 INFO:tasks.workunit.client.0.smithi039.stdout:found image28 2024-09-14T04:03:35.499 INFO:tasks.workunit.client.0.smithi039.stdout:erased image28 2024-09-14T04:03:35.499 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image29 2024-09-14T04:03:35.499 INFO:tasks.workunit.client.0.smithi039.stdout:found image29 2024-09-14T04:03:35.499 INFO:tasks.workunit.client.0.smithi039.stdout:erased image29 2024-09-14T04:03:35.499 INFO:tasks.workunit.client.0.smithi039.stdout:100.00% done 2024-09-14T04:03:35.499 INFO:tasks.workunit.client.0.smithi039.stdout:image: image28 2024-09-14T04:03:35.499 INFO:tasks.workunit.client.0.smithi039.stdout:image: image29 2024-09-14T04:03:35.499 INFO:tasks.workunit.client.0.smithi039.stdout:image: image30 2024-09-14T04:03:35.499 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image28 2024-09-14T04:03:35.499 INFO:tasks.workunit.client.0.smithi039.stdout:found image28 2024-09-14T04:03:35.499 INFO:tasks.workunit.client.0.smithi039.stdout:erased image28 2024-09-14T04:03:35.500 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image29 2024-09-14T04:03:35.500 INFO:tasks.workunit.client.0.smithi039.stdout:found image29 2024-09-14T04:03:35.500 INFO:tasks.workunit.client.0.smithi039.stdout:erased image29 2024-09-14T04:03:35.500 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image30 2024-09-14T04:03:35.500 INFO:tasks.workunit.client.0.smithi039.stdout:found image30 2024-09-14T04:03:35.500 INFO:tasks.workunit.client.0.smithi039.stdout:erased image30 2024-09-14T04:03:35.500 INFO:tasks.workunit.client.0.smithi039.stdout:image: image28 2024-09-14T04:03:35.500 INFO:tasks.workunit.client.0.smithi039.stdout:image: image29 2024-09-14T04:03:35.500 INFO:tasks.workunit.client.0.smithi039.stdout:image: image30 2024-09-14T04:03:35.500 INFO:tasks.workunit.client.0.smithi039.stdout:image: image31 2024-09-14T04:03:35.500 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image28 2024-09-14T04:03:35.501 INFO:tasks.workunit.client.0.smithi039.stdout:found image28 2024-09-14T04:03:35.501 INFO:tasks.workunit.client.0.smithi039.stdout:erased image28 2024-09-14T04:03:35.501 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image29 2024-09-14T04:03:35.501 INFO:tasks.workunit.client.0.smithi039.stdout:found image29 2024-09-14T04:03:35.501 INFO:tasks.workunit.client.0.smithi039.stdout:erased image29 2024-09-14T04:03:35.501 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image30 2024-09-14T04:03:35.501 INFO:tasks.workunit.client.0.smithi039.stdout:found image30 2024-09-14T04:03:35.501 INFO:tasks.workunit.client.0.smithi039.stdout:erased image30 2024-09-14T04:03:35.501 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image31 2024-09-14T04:03:35.501 INFO:tasks.workunit.client.0.smithi039.stdout:found image31 2024-09-14T04:03:35.501 INFO:tasks.workunit.client.0.smithi039.stdout:erased image31 2024-09-14T04:03:35.501 INFO:tasks.workunit.client.0.smithi039.stdout:image: image28 2024-09-14T04:03:35.502 INFO:tasks.workunit.client.0.smithi039.stdout:image: image29 2024-09-14T04:03:35.502 INFO:tasks.workunit.client.0.smithi039.stdout:image: image30 2024-09-14T04:03:35.502 INFO:tasks.workunit.client.0.smithi039.stdout:image: image31 2024-09-14T04:03:35.502 INFO:tasks.workunit.client.0.smithi039.stdout:image: image32 2024-09-14T04:03:35.502 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image28 2024-09-14T04:03:35.502 INFO:tasks.workunit.client.0.smithi039.stdout:found image28 2024-09-14T04:03:35.502 INFO:tasks.workunit.client.0.smithi039.stdout:erased image28 2024-09-14T04:03:35.502 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image29 2024-09-14T04:03:35.502 INFO:tasks.workunit.client.0.smithi039.stdout:found image29 2024-09-14T04:03:35.502 INFO:tasks.workunit.client.0.smithi039.stdout:erased image29 2024-09-14T04:03:35.503 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image30 2024-09-14T04:03:35.503 INFO:tasks.workunit.client.0.smithi039.stdout:found image30 2024-09-14T04:03:35.503 INFO:tasks.workunit.client.0.smithi039.stdout:erased image30 2024-09-14T04:03:35.503 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image31 2024-09-14T04:03:35.503 INFO:tasks.workunit.client.0.smithi039.stdout:found image31 2024-09-14T04:03:35.503 INFO:tasks.workunit.client.0.smithi039.stdout:erased image31 2024-09-14T04:03:35.503 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image32 2024-09-14T04:03:35.503 INFO:tasks.workunit.client.0.smithi039.stdout:found image32 2024-09-14T04:03:35.503 INFO:tasks.workunit.client.0.smithi039.stdout:erased image32 2024-09-14T04:03:35.503 INFO:tasks.workunit.client.0.smithi039.stdout:100.00% done 2024-09-14T04:03:35.503 INFO:tasks.workunit.client.0.smithi039.stdout:image: image28 2024-09-14T04:03:35.503 INFO:tasks.workunit.client.0.smithi039.stdout:image: image29 2024-09-14T04:03:35.504 INFO:tasks.workunit.client.0.smithi039.stdout:image: image30 2024-09-14T04:03:35.504 INFO:tasks.workunit.client.0.smithi039.stdout:image: image31 2024-09-14T04:03:35.504 INFO:tasks.workunit.client.0.smithi039.stdout:image: image32 2024-09-14T04:03:35.504 INFO:tasks.workunit.client.0.smithi039.stdout:image: image33 2024-09-14T04:03:35.504 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image28 2024-09-14T04:03:35.504 INFO:tasks.workunit.client.0.smithi039.stdout:found image28 2024-09-14T04:03:35.504 INFO:tasks.workunit.client.0.smithi039.stdout:erased image28 2024-09-14T04:03:35.505 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image29 2024-09-14T04:03:35.505 INFO:tasks.workunit.client.0.smithi039.stdout:found image29 2024-09-14T04:03:35.505 INFO:tasks.workunit.client.0.smithi039.stdout:erased image29 2024-09-14T04:03:35.505 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image30 2024-09-14T04:03:35.505 INFO:tasks.workunit.client.0.smithi039.stdout:found image30 2024-09-14T04:03:35.505 INFO:tasks.workunit.client.0.smithi039.stdout:erased image30 2024-09-14T04:03:35.505 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image31 2024-09-14T04:03:35.505 INFO:tasks.workunit.client.0.smithi039.stdout:found image31 2024-09-14T04:03:35.506 INFO:tasks.workunit.client.0.smithi039.stdout:erased image31 2024-09-14T04:03:35.506 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image32 2024-09-14T04:03:35.506 INFO:tasks.workunit.client.0.smithi039.stdout:found image32 2024-09-14T04:03:35.506 INFO:tasks.workunit.client.0.smithi039.stdout:erased image32 2024-09-14T04:03:35.506 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image33 2024-09-14T04:03:35.506 INFO:tasks.workunit.client.0.smithi039.stdout:found image33 2024-09-14T04:03:35.506 INFO:tasks.workunit.client.0.smithi039.stdout:erased image33 2024-09-14T04:03:35.506 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestDeepCopy (7711 ms) 2024-09-14T04:03:35.506 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestDeepCopyPP 2024-09-14T04:03:36.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:35 smithi039 ceph-mon[106470]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T04:03:36.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:35 smithi039 ceph-mon[106470]: pgmap v383: 265 pgs: 265 active+clean; 583 KiB data, 490 MiB used, 715 GiB / 715 GiB avail; 336 KiB/s rd, 20 KiB/s wr, 392 op/s 2024-09-14T04:03:36.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:35 smithi039 ceph-mon[106470]: osdmap e595: 8 total, 8 up, 8 in 2024-09-14T04:03:36.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:35 smithi039 ceph-mon[106470]: osdmap e596: 8 total, 8 up, 8 in 2024-09-14T04:03:36.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:35 smithi039 ceph-mon[111235]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T04:03:36.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:35 smithi039 ceph-mon[111235]: pgmap v383: 265 pgs: 265 active+clean; 583 KiB data, 490 MiB used, 715 GiB / 715 GiB avail; 336 KiB/s rd, 20 KiB/s wr, 392 op/s 2024-09-14T04:03:36.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:35 smithi039 ceph-mon[111235]: osdmap e595: 8 total, 8 up, 8 in 2024-09-14T04:03:36.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:35 smithi039 ceph-mon[111235]: osdmap e596: 8 total, 8 up, 8 in 2024-09-14T04:03:36.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:35 smithi129 ceph-mon[101316]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T04:03:36.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:35 smithi129 ceph-mon[101316]: pgmap v383: 265 pgs: 265 active+clean; 583 KiB data, 490 MiB used, 715 GiB / 715 GiB avail; 336 KiB/s rd, 20 KiB/s wr, 392 op/s 2024-09-14T04:03:36.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:35 smithi129 ceph-mon[101316]: osdmap e595: 8 total, 8 up, 8 in 2024-09-14T04:03:36.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:35 smithi129 ceph-mon[101316]: osdmap e596: 8 total, 8 up, 8 in 2024-09-14T04:03:37.451 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:03:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: osdmap e597: 8 total, 8 up, 8 in 2024-09-14T04:03:37.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/774191769' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:37.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:37.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: Upgrade: Target is version 19.3.0-4907-ga2782315 (squid) 2024-09-14T04:03:37.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T04:03:37.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:03:37.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: Upgrade: Setting container_image for all mgr 2024-09-14T04:03:37.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: Upgrade: Setting container_image for all mon 2024-09-14T04:03:37.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: Upgrade: Setting container_image for all crash 2024-09-14T04:03:37.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: Upgrade: Setting container_image for all rgw 2024-09-14T04:03:37.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: Upgrade: Setting container_image for all rbd-mirror 2024-09-14T04:03:37.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-14T04:03:37.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: Upgrade: Setting container_image for all ceph-exporter 2024-09-14T04:03:37.775 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.775 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.775 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: Upgrade: Setting container_image for all iscsi 2024-09-14T04:03:37.775 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.775 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.775 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: Upgrade: Setting container_image for all nfs 2024-09-14T04:03:37.775 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.775 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.775 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: Upgrade: Setting container_image for all nvmeof 2024-09-14T04:03:37.775 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.775 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.776 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: Upgrade: Setting container_image for all node-exporter 2024-09-14T04:03:37.776 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.776 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: Upgrade: Setting container_image for all prometheus 2024-09-14T04:03:37.776 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.776 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: Upgrade: Setting container_image for all alertmanager 2024-09-14T04:03:37.776 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.776 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: Upgrade: Setting container_image for all grafana 2024-09-14T04:03:37.776 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.776 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: Upgrade: Setting container_image for all loki 2024-09-14T04:03:37.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: Upgrade: Setting container_image for all promtail 2024-09-14T04:03:37.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: Upgrade: Finalizing container_image settings 2024-09-14T04:03:37.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T04:03:37.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T04:03:37.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T04:03:37.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T04:03:37.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T04:03:37.777 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T04:03:37.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-14T04:03:37.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T04:03:37.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T04:03:37.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T04:03:37.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T04:03:37.778 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T04:03:37.779 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T04:03:37.779 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-14T04:03:37.779 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T04:03:37.779 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T04:03:37.779 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-14T04:03:37.779 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.779 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.779 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T04:03:37.779 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T04:03:37.780 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-14T04:03:37.780 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T04:03:37.780 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T04:03:37.780 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-14T04:03:37.780 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T04:03:37.780 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T04:03:37.780 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-14T04:03:37.780 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T04:03:37.780 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T04:03:37.781 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T04:03:37.781 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.781 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.781 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.781 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.781 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.781 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.781 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.781 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.782 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.782 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.782 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.782 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.782 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: Upgrade: Complete! 2024-09-14T04:03:37.782 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T04:03:37.782 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T04:03:37.782 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T04:03:37.782 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:03:37.782 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:03:37.782 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:03:37.782 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.783 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: pgmap v387: 297 pgs: 32 unknown, 265 active+clean; 583 KiB data, 490 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:03:37.783 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.783 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:37.783 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: osdmap e597: 8 total, 8 up, 8 in 2024-09-14T04:03:37.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/774191769' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:37.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:37.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: Upgrade: Target is version 19.3.0-4907-ga2782315 (squid) 2024-09-14T04:03:37.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T04:03:37.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:03:37.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all mgr 2024-09-14T04:03:37.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all mon 2024-09-14T04:03:37.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all crash 2024-09-14T04:03:37.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all rgw 2024-09-14T04:03:37.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all rbd-mirror 2024-09-14T04:03:37.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-14T04:03:37.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all ceph-exporter 2024-09-14T04:03:37.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all iscsi 2024-09-14T04:03:37.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all nfs 2024-09-14T04:03:37.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: osdmap e597: 8 total, 8 up, 8 in 2024-09-14T04:03:37.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/774191769' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:37.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-21","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:03:37.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: Upgrade: Target is version 19.3.0-4907-ga2782315 (squid) 2024-09-14T04:03:37.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T04:03:37.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:03:37.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all mgr 2024-09-14T04:03:37.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all mon 2024-09-14T04:03:37.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all crash 2024-09-14T04:03:37.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.796 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all rgw 2024-09-14T04:03:37.796 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.796 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.796 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all rbd-mirror 2024-09-14T04:03:37.796 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.796 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.796 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all cephfs-mirror 2024-09-14T04:03:37.796 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.796 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all ceph-exporter 2024-09-14T04:03:37.796 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.796 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.796 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all iscsi 2024-09-14T04:03:37.797 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.797 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.797 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all nfs 2024-09-14T04:03:37.797 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.797 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.797 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all nvmeof 2024-09-14T04:03:37.797 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.797 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.797 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all node-exporter 2024-09-14T04:03:37.797 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.797 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all prometheus 2024-09-14T04:03:37.797 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.798 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all alertmanager 2024-09-14T04:03:37.798 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.798 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all grafana 2024-09-14T04:03:37.798 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.798 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all loki 2024-09-14T04:03:37.798 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.798 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all promtail 2024-09-14T04:03:37.798 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: Upgrade: Finalizing container_image settings 2024-09-14T04:03:37.798 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T04:03:37.798 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T04:03:37.798 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T04:03:37.799 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.799 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.799 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T04:03:37.799 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T04:03:37.799 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T04:03:37.799 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-14T04:03:37.799 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T04:03:37.799 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T04:03:37.799 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T04:03:37.799 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T04:03:37.799 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T04:03:37.800 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all nvmeof 2024-09-14T04:03:37.800 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.800 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.800 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all node-exporter 2024-09-14T04:03:37.800 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.800 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all prometheus 2024-09-14T04:03:37.800 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.801 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all alertmanager 2024-09-14T04:03:37.801 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.801 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all grafana 2024-09-14T04:03:37.801 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.801 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all loki 2024-09-14T04:03:37.801 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:03:37.801 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all promtail 2024-09-14T04:03:37.801 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: Upgrade: Finalizing container_image settings 2024-09-14T04:03:37.801 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T04:03:37.801 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T04:03:37.801 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T04:03:37.801 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.802 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.802 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T04:03:37.802 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T04:03:37.802 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T04:03:37.802 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-14T04:03:37.802 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T04:03:37.802 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T04:03:37.802 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T04:03:37.802 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T04:03:37.802 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T04:03:37.803 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T04:03:37.803 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-14T04:03:37.803 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T04:03:37.803 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T04:03:37.803 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-14T04:03:37.803 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.803 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.803 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T04:03:37.803 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T04:03:37.804 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-14T04:03:37.804 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T04:03:37.804 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T04:03:37.804 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-14T04:03:37.804 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T04:03:37.804 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T04:03:37.804 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-14T04:03:37.804 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T04:03:37.804 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T04:03:37.804 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T04:03:37.804 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.805 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.805 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.805 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.805 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.805 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T04:03:37.805 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rgw"}]': finished 2024-09-14T04:03:37.805 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T04:03:37.806 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T04:03:37.806 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"}]': finished 2024-09-14T04:03:37.806 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.806 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.806 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T04:03:37.806 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T04:03:37.806 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"}]': finished 2024-09-14T04:03:37.806 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T04:03:37.806 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T04:03:37.807 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.iscsi"}]': finished 2024-09-14T04:03:37.807 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T04:03:37.807 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T04:03:37.807 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nfs"}]': finished 2024-09-14T04:03:37.807 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T04:03:37.807 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T04:03:37.807 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T04:03:37.807 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.807 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.807 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.807 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.807 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.808 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.808 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.808 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.808 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.808 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.808 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.808 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.808 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: Upgrade: Complete! 2024-09-14T04:03:37.809 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T04:03:37.809 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T04:03:37.809 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T04:03:37.809 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:03:37.809 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:03:37.809 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:03:37.809 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.809 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: pgmap v387: 297 pgs: 32 unknown, 265 active+clean; 583 KiB data, 490 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:03:37.809 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.809 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:37.809 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.810 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.810 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.810 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.810 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.810 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.810 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.810 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:03:37.810 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: Upgrade: Complete! 2024-09-14T04:03:37.810 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T04:03:37.810 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T04:03:37.810 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T04:03:37.811 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:03:37.811 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:03:37.811 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:03:37.811 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.811 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: pgmap v387: 297 pgs: 32 unknown, 265 active+clean; 583 KiB data, 490 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:03:37.811 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:37.811 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:37.811 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:38.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:38 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:38.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:38 smithi129 ceph-mon[101316]: osdmap e598: 8 total, 8 up, 8 in 2024-09-14T04:03:38.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:38 smithi129 ceph-mon[101316]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:38.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:03:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:03:38] "GET /metrics HTTP/1.1" 200 50792 "" "Prometheus/2.51.0" 2024-09-14T04:03:38.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:38 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:38.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:38 smithi039 ceph-mon[106470]: osdmap e598: 8 total, 8 up, 8 in 2024-09-14T04:03:38.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:38 smithi039 ceph-mon[106470]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:38.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:38 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-21","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:03:38.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:38 smithi039 ceph-mon[111235]: osdmap e598: 8 total, 8 up, 8 in 2024-09-14T04:03:38.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:38 smithi039 ceph-mon[111235]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:03:39.761 INFO:tasks.workunit.client.0.smithi039.stdout:num images is: 1 2024-09-14T04:03:39.761 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 1 2024-09-14T04:03:39.761 INFO:tasks.workunit.client.0.smithi039.stdout:image: image34 2024-09-14T04:03:39.761 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image34 2024-09-14T04:03:39.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:39 smithi129 ceph-mon[101316]: osdmap e599: 8 total, 8 up, 8 in 2024-09-14T04:03:39.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:39 smithi129 ceph-mon[101316]: pgmap v390: 297 pgs: 297 active+clean; 584 KiB data, 491 MiB used, 715 GiB / 715 GiB avail; 205 KiB/s rd, 10 KiB/s wr, 247 op/s 2024-09-14T04:03:39.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:39 smithi039 ceph-mon[106470]: osdmap e599: 8 total, 8 up, 8 in 2024-09-14T04:03:39.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:39 smithi039 ceph-mon[106470]: pgmap v390: 297 pgs: 297 active+clean; 584 KiB data, 491 MiB used, 715 GiB / 715 GiB avail; 205 KiB/s rd, 10 KiB/s wr, 247 op/s 2024-09-14T04:03:39.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:39 smithi039 ceph-mon[111235]: osdmap e599: 8 total, 8 up, 8 in 2024-09-14T04:03:39.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:39 smithi039 ceph-mon[111235]: pgmap v390: 297 pgs: 297 active+clean; 584 KiB data, 491 MiB used, 715 GiB / 715 GiB avail; 205 KiB/s rd, 10 KiB/s wr, 247 op/s 2024-09-14T04:03:39.793 INFO:tasks.workunit.client.0.smithi039.stdout:num images is: 2 2024-09-14T04:03:39.793 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 2 2024-09-14T04:03:39.793 INFO:tasks.workunit.client.0.smithi039.stdout:image: image34 2024-09-14T04:03:39.793 INFO:tasks.workunit.client.0.smithi039.stdout:image: image35 2024-09-14T04:03:39.793 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image34 2024-09-14T04:03:39.793 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image35 2024-09-14T04:03:39.846 INFO:tasks.workunit.client.0.smithi039.stdout:100.00% done 2024-09-14T04:03:39.846 INFO:tasks.workunit.client.0.smithi039.stdout:num images is: 3 2024-09-14T04:03:39.847 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 3 2024-09-14T04:03:39.847 INFO:tasks.workunit.client.0.smithi039.stdout:image: image34 2024-09-14T04:03:39.847 INFO:tasks.workunit.client.0.smithi039.stdout:image: image35 2024-09-14T04:03:39.847 INFO:tasks.workunit.client.0.smithi039.stdout:image: image36 2024-09-14T04:03:39.847 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image34 2024-09-14T04:03:39.847 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image35 2024-09-14T04:03:39.847 INFO:tasks.workunit.client.0.smithi039.stdout:expected = image36 2024-09-14T04:03:39.873 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestDeepCopyPP (4375 ms) 2024-09-14T04:03:39.873 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestCreateLsDeleteSnap 2024-09-14T04:03:39.873 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:03:40.758 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:40 smithi039 ceph-mon[106470]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:40.758 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:40 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T04:03:40.758 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:40 smithi039 ceph-mon[106470]: osdmap e600: 8 total, 8 up, 8 in 2024-09-14T04:03:40.758 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:40 smithi039 ceph-mon[111235]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:40.758 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:40 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T04:03:40.758 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:40 smithi039 ceph-mon[111235]: osdmap e600: 8 total, 8 up, 8 in 2024-09-14T04:03:40.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:40 smithi129 ceph-mon[101316]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:03:40.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:40 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T04:03:40.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:40 smithi129 ceph-mon[101316]: osdmap e600: 8 total, 8 up, 8 in 2024-09-14T04:03:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:41 smithi129 ceph-mon[101316]: osdmap e601: 8 total, 8 up, 8 in 2024-09-14T04:03:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:41 smithi129 ceph-mon[101316]: pgmap v393: 297 pgs: 297 active+clean; 584 KiB data, 491 MiB used, 715 GiB / 715 GiB avail; 200 KiB/s rd, 10 KiB/s wr, 241 op/s 2024-09-14T04:03:41.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:41 smithi039 ceph-mon[106470]: osdmap e601: 8 total, 8 up, 8 in 2024-09-14T04:03:41.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:41 smithi039 ceph-mon[106470]: pgmap v393: 297 pgs: 297 active+clean; 584 KiB data, 491 MiB used, 715 GiB / 715 GiB avail; 200 KiB/s rd, 10 KiB/s wr, 241 op/s 2024-09-14T04:03:41.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:41 smithi039 ceph-mon[111235]: osdmap e601: 8 total, 8 up, 8 in 2024-09-14T04:03:41.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:41 smithi039 ceph-mon[111235]: pgmap v393: 297 pgs: 297 active+clean; 584 KiB data, 491 MiB used, 715 GiB / 715 GiB avail; 200 KiB/s rd, 10 KiB/s wr, 241 op/s 2024-09-14T04:03:42.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:42 smithi129 ceph-mon[101316]: osdmap e602: 8 total, 8 up, 8 in 2024-09-14T04:03:42.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:42 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:42.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:42 smithi039 ceph-mon[106470]: osdmap e602: 8 total, 8 up, 8 in 2024-09-14T04:03:42.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:42 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:42.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:42 smithi039 ceph-mon[111235]: osdmap e602: 8 total, 8 up, 8 in 2024-09-14T04:03:42.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:42 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:03:43.577 INFO:tasks.workunit.client.0.smithi039.stdout:num snaps is: 1 2024-09-14T04:03:43.578 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 1 2024-09-14T04:03:43.578 INFO:tasks.workunit.client.0.smithi039.stdout:snap: snap1 2024-09-14T04:03:43.578 INFO:tasks.workunit.client.0.smithi039.stdout:found snap1 with size 2097152 2024-09-14T04:03:43.578 INFO:tasks.workunit.client.0.smithi039.stdout:num snaps is: 2 2024-09-14T04:03:43.578 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 2 2024-09-14T04:03:43.578 INFO:tasks.workunit.client.0.smithi039.stdout:snap: snap1 2024-09-14T04:03:43.578 INFO:tasks.workunit.client.0.smithi039.stdout:snap: snap2 2024-09-14T04:03:43.578 INFO:tasks.workunit.client.0.smithi039.stdout:found snap1 with size 2097152 2024-09-14T04:03:43.578 INFO:tasks.workunit.client.0.smithi039.stdout:found snap2 with size 4194304 2024-09-14T04:03:43.578 INFO:tasks.workunit.client.0.smithi039.stdout:num snaps is: 1 2024-09-14T04:03:43.578 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 1 2024-09-14T04:03:43.578 INFO:tasks.workunit.client.0.smithi039.stdout:snap: snap2 2024-09-14T04:03:43.578 INFO:tasks.workunit.client.0.smithi039.stdout:found snap2 with size 4194304 2024-09-14T04:03:43.579 INFO:tasks.workunit.client.0.smithi039.stdout:num snaps is: 0 2024-09-14T04:03:43.579 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 0 2024-09-14T04:03:43.579 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestCreateLsDeleteSnap (3704 ms) 2024-09-14T04:03:43.579 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestGetSnapShotTimeStamp 2024-09-14T04:03:43.579 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:03:44.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:43 smithi129 ceph-mon[101316]: osdmap e603: 8 total, 8 up, 8 in 2024-09-14T04:03:44.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:43 smithi129 ceph-mon[101316]: pgmap v396: 297 pgs: 297 active+clean; 584 KiB data, 509 MiB used, 715 GiB / 715 GiB avail; 382 KiB/s rd, 27 KiB/s wr, 459 op/s 2024-09-14T04:03:44.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:43 smithi039 ceph-mon[106470]: osdmap e603: 8 total, 8 up, 8 in 2024-09-14T04:03:44.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:43 smithi039 ceph-mon[106470]: pgmap v396: 297 pgs: 297 active+clean; 584 KiB data, 509 MiB used, 715 GiB / 715 GiB avail; 382 KiB/s rd, 27 KiB/s wr, 459 op/s 2024-09-14T04:03:44.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:43 smithi039 ceph-mon[111235]: osdmap e603: 8 total, 8 up, 8 in 2024-09-14T04:03:44.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:43 smithi039 ceph-mon[111235]: pgmap v396: 297 pgs: 297 active+clean; 584 KiB data, 509 MiB used, 715 GiB / 715 GiB avail; 382 KiB/s rd, 27 KiB/s wr, 459 op/s 2024-09-14T04:03:45.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:44 smithi129 ceph-mon[101316]: osdmap e604: 8 total, 8 up, 8 in 2024-09-14T04:03:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:44 smithi039 ceph-mon[106470]: osdmap e604: 8 total, 8 up, 8 in 2024-09-14T04:03:45.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:44 smithi039 ceph-mon[111235]: osdmap e604: 8 total, 8 up, 8 in 2024-09-14T04:03:45.464 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestGetSnapShotTimeStamp (1887 ms) 2024-09-14T04:03:45.464 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestCreateLsDeleteSnapPP 2024-09-14T04:03:45.464 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:03:46.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:45 smithi129 ceph-mon[101316]: osdmap e605: 8 total, 8 up, 8 in 2024-09-14T04:03:46.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:45 smithi129 ceph-mon[101316]: pgmap v399: 297 pgs: 297 active+clean; 584 KiB data, 509 MiB used, 715 GiB / 715 GiB avail; 382 KiB/s rd, 27 KiB/s wr, 459 op/s 2024-09-14T04:03:46.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:45 smithi129 ceph-mon[101316]: osdmap e606: 8 total, 8 up, 8 in 2024-09-14T04:03:46.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:45 smithi039 ceph-mon[106470]: osdmap e605: 8 total, 8 up, 8 in 2024-09-14T04:03:46.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:45 smithi039 ceph-mon[106470]: pgmap v399: 297 pgs: 297 active+clean; 584 KiB data, 509 MiB used, 715 GiB / 715 GiB avail; 382 KiB/s rd, 27 KiB/s wr, 459 op/s 2024-09-14T04:03:46.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:45 smithi039 ceph-mon[106470]: osdmap e606: 8 total, 8 up, 8 in 2024-09-14T04:03:46.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:45 smithi039 ceph-mon[111235]: osdmap e605: 8 total, 8 up, 8 in 2024-09-14T04:03:46.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:45 smithi039 ceph-mon[111235]: pgmap v399: 297 pgs: 297 active+clean; 584 KiB data, 509 MiB used, 715 GiB / 715 GiB avail; 382 KiB/s rd, 27 KiB/s wr, 459 op/s 2024-09-14T04:03:46.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:45 smithi039 ceph-mon[111235]: osdmap e606: 8 total, 8 up, 8 in 2024-09-14T04:03:46.448 INFO:tasks.workunit.client.0.smithi039.stdout:num snaps is: 1 2024-09-14T04:03:46.448 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 1 2024-09-14T04:03:46.448 INFO:tasks.workunit.client.0.smithi039.stdout:snap: snap1 2024-09-14T04:03:46.448 INFO:tasks.workunit.client.0.smithi039.stdout:found snap1 with size 2097152 2024-09-14T04:03:47.461 INFO:tasks.workunit.client.0.smithi039.stdout:num snaps is: 2 2024-09-14T04:03:47.462 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 2 2024-09-14T04:03:47.462 INFO:tasks.workunit.client.0.smithi039.stdout:snap: snap1 2024-09-14T04:03:47.462 INFO:tasks.workunit.client.0.smithi039.stdout:snap: snap2 2024-09-14T04:03:47.462 INFO:tasks.workunit.client.0.smithi039.stdout:found snap1 with size 2097152 2024-09-14T04:03:47.462 INFO:tasks.workunit.client.0.smithi039.stdout:found snap2 with size 4194304 2024-09-14T04:03:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:47 smithi129 ceph-mon[101316]: osdmap e607: 8 total, 8 up, 8 in 2024-09-14T04:03:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:47 smithi129 ceph-mon[101316]: pgmap v402: 297 pgs: 297 active+clean; 584 KiB data, 509 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:03:47.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:47 smithi039 ceph-mon[106470]: osdmap e607: 8 total, 8 up, 8 in 2024-09-14T04:03:47.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:47 smithi039 ceph-mon[106470]: pgmap v402: 297 pgs: 297 active+clean; 584 KiB data, 509 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:03:47.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:47 smithi039 ceph-mon[111235]: osdmap e607: 8 total, 8 up, 8 in 2024-09-14T04:03:47.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:47 smithi039 ceph-mon[111235]: pgmap v402: 297 pgs: 297 active+clean; 584 KiB data, 509 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:03:48.469 INFO:tasks.workunit.client.0.smithi039.stdout:num snaps is: 1 2024-09-14T04:03:48.469 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 1 2024-09-14T04:03:48.469 INFO:tasks.workunit.client.0.smithi039.stdout:snap: snap2 2024-09-14T04:03:48.470 INFO:tasks.workunit.client.0.smithi039.stdout:found snap2 with size 4194304 2024-09-14T04:03:48.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:48 smithi129 ceph-mon[101316]: osdmap e608: 8 total, 8 up, 8 in 2024-09-14T04:03:48.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:48 smithi039 ceph-mon[106470]: osdmap e608: 8 total, 8 up, 8 in 2024-09-14T04:03:48.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:48 smithi039 ceph-mon[111235]: osdmap e608: 8 total, 8 up, 8 in 2024-09-14T04:03:48.791 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:03:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:03:48] "GET /metrics HTTP/1.1" 200 50792 "" "Prometheus/2.51.0" 2024-09-14T04:03:49.487 INFO:tasks.workunit.client.0.smithi039.stdout:num snaps is: 0 2024-09-14T04:03:49.487 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 0 2024-09-14T04:03:49.496 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestCreateLsDeleteSnapPP (4032 ms) 2024-09-14T04:03:49.496 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestGetNameIdSnapPP 2024-09-14T04:03:49.496 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:03:49.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:49 smithi129 ceph-mon[101316]: osdmap e609: 8 total, 8 up, 8 in 2024-09-14T04:03:49.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:49 smithi129 ceph-mon[101316]: pgmap v405: 297 pgs: 297 active+clean; 584 KiB data, 514 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 6.5 KiB/s wr, 82 op/s 2024-09-14T04:03:49.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:49 smithi039 ceph-mon[106470]: osdmap e609: 8 total, 8 up, 8 in 2024-09-14T04:03:49.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:49 smithi039 ceph-mon[106470]: pgmap v405: 297 pgs: 297 active+clean; 584 KiB data, 514 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 6.5 KiB/s wr, 82 op/s 2024-09-14T04:03:49.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:49 smithi039 ceph-mon[111235]: osdmap e609: 8 total, 8 up, 8 in 2024-09-14T04:03:49.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:49 smithi039 ceph-mon[111235]: pgmap v405: 297 pgs: 297 active+clean; 584 KiB data, 514 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 6.5 KiB/s wr, 82 op/s 2024-09-14T04:03:50.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:50 smithi129 ceph-mon[101316]: osdmap e610: 8 total, 8 up, 8 in 2024-09-14T04:03:50.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:50 smithi129 ceph-mon[101316]: osdmap e611: 8 total, 8 up, 8 in 2024-09-14T04:03:50.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:50 smithi039 ceph-mon[106470]: osdmap e610: 8 total, 8 up, 8 in 2024-09-14T04:03:50.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:50 smithi039 ceph-mon[106470]: osdmap e611: 8 total, 8 up, 8 in 2024-09-14T04:03:50.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:50 smithi039 ceph-mon[111235]: osdmap e610: 8 total, 8 up, 8 in 2024-09-14T04:03:50.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:50 smithi039 ceph-mon[111235]: osdmap e611: 8 total, 8 up, 8 in 2024-09-14T04:03:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:51 smithi129 ceph-mon[101316]: pgmap v408: 297 pgs: 297 active+clean; 584 KiB data, 514 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 6.5 KiB/s wr, 82 op/s 2024-09-14T04:03:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:51 smithi129 ceph-mon[101316]: osdmap e612: 8 total, 8 up, 8 in 2024-09-14T04:03:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:51 smithi039 ceph-mon[106470]: pgmap v408: 297 pgs: 297 active+clean; 584 KiB data, 514 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 6.5 KiB/s wr, 82 op/s 2024-09-14T04:03:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:51 smithi039 ceph-mon[106470]: osdmap e612: 8 total, 8 up, 8 in 2024-09-14T04:03:51.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:51 smithi039 ceph-mon[111235]: pgmap v408: 297 pgs: 297 active+clean; 584 KiB data, 514 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 6.5 KiB/s wr, 82 op/s 2024-09-14T04:03:51.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:51 smithi039 ceph-mon[111235]: osdmap e612: 8 total, 8 up, 8 in 2024-09-14T04:03:52.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:52.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:52 smithi129 ceph-mon[101316]: osdmap e613: 8 total, 8 up, 8 in 2024-09-14T04:03:52.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:52.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:52 smithi039 ceph-mon[106470]: osdmap e613: 8 total, 8 up, 8 in 2024-09-14T04:03:52.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:03:52.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:52 smithi039 ceph-mon[111235]: osdmap e613: 8 total, 8 up, 8 in 2024-09-14T04:03:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:53 smithi129 ceph-mon[101316]: pgmap v411: 297 pgs: 297 active+clean; 584 KiB data, 514 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 5.2 KiB/s wr, 66 op/s 2024-09-14T04:03:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:53 smithi129 ceph-mon[101316]: osdmap e614: 8 total, 8 up, 8 in 2024-09-14T04:03:53.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:53 smithi039 ceph-mon[106470]: pgmap v411: 297 pgs: 297 active+clean; 584 KiB data, 514 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 5.2 KiB/s wr, 66 op/s 2024-09-14T04:03:53.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:53 smithi039 ceph-mon[106470]: osdmap e614: 8 total, 8 up, 8 in 2024-09-14T04:03:53.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:53 smithi039 ceph-mon[111235]: pgmap v411: 297 pgs: 297 active+clean; 584 KiB data, 514 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 5.2 KiB/s wr, 66 op/s 2024-09-14T04:03:53.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:53 smithi039 ceph-mon[111235]: osdmap e614: 8 total, 8 up, 8 in 2024-09-14T04:03:55.443 INFO:tasks.workunit.client.0.smithi039.stdout:num snaps is: 0 2024-09-14T04:03:55.443 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 0 2024-09-14T04:03:55.451 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestGetNameIdSnapPP (5954 ms) 2024-09-14T04:03:55.451 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestCreateLsRenameSnapPP 2024-09-14T04:03:55.451 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:03:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:55 smithi129 ceph-mon[101316]: osdmap e615: 8 total, 8 up, 8 in 2024-09-14T04:03:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:55 smithi129 ceph-mon[101316]: pgmap v414: 297 pgs: 297 active+clean; 584 KiB data, 514 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 5.2 KiB/s wr, 66 op/s 2024-09-14T04:03:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:55 smithi129 ceph-mon[101316]: osdmap e616: 8 total, 8 up, 8 in 2024-09-14T04:03:55.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:55 smithi039 ceph-mon[106470]: osdmap e615: 8 total, 8 up, 8 in 2024-09-14T04:03:55.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:55 smithi039 ceph-mon[106470]: pgmap v414: 297 pgs: 297 active+clean; 584 KiB data, 514 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 5.2 KiB/s wr, 66 op/s 2024-09-14T04:03:55.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:55 smithi039 ceph-mon[106470]: osdmap e616: 8 total, 8 up, 8 in 2024-09-14T04:03:55.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:55 smithi039 ceph-mon[111235]: osdmap e615: 8 total, 8 up, 8 in 2024-09-14T04:03:55.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:55 smithi039 ceph-mon[111235]: pgmap v414: 297 pgs: 297 active+clean; 584 KiB data, 514 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 5.2 KiB/s wr, 66 op/s 2024-09-14T04:03:55.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:55 smithi039 ceph-mon[111235]: osdmap e616: 8 total, 8 up, 8 in 2024-09-14T04:03:56.484 INFO:tasks.workunit.client.0.smithi039.stdout:num snaps is: 1 2024-09-14T04:03:56.484 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 1 2024-09-14T04:03:56.485 INFO:tasks.workunit.client.0.smithi039.stdout:snap: snap1 2024-09-14T04:03:56.485 INFO:tasks.workunit.client.0.smithi039.stdout:found snap1 with size 2097152 2024-09-14T04:03:57.482 INFO:tasks.workunit.client.0.smithi039.stdout:num snaps is: 2 2024-09-14T04:03:57.483 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 2 2024-09-14T04:03:57.483 INFO:tasks.workunit.client.0.smithi039.stdout:snap: snap1 2024-09-14T04:03:57.483 INFO:tasks.workunit.client.0.smithi039.stdout:snap: snap2 2024-09-14T04:03:57.483 INFO:tasks.workunit.client.0.smithi039.stdout:found snap1 with size 2097152 2024-09-14T04:03:57.483 INFO:tasks.workunit.client.0.smithi039.stdout:found snap2 with size 4194304 2024-09-14T04:03:57.486 INFO:tasks.workunit.client.0.smithi039.stdout:num snaps is: 2 2024-09-14T04:03:57.486 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 2 2024-09-14T04:03:57.486 INFO:tasks.workunit.client.0.smithi039.stdout:snap: snap1-rename 2024-09-14T04:03:57.487 INFO:tasks.workunit.client.0.smithi039.stdout:snap: snap2 2024-09-14T04:03:57.487 INFO:tasks.workunit.client.0.smithi039.stdout:found snap1-rename with size 2097152 2024-09-14T04:03:57.487 INFO:tasks.workunit.client.0.smithi039.stdout:found snap2 with size 4194304 2024-09-14T04:03:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:57 smithi129 ceph-mon[101316]: osdmap e617: 8 total, 8 up, 8 in 2024-09-14T04:03:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:57 smithi129 ceph-mon[101316]: pgmap v417: 297 pgs: 297 active+clean; 584 KiB data, 514 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:03:57.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:57 smithi039 ceph-mon[106470]: osdmap e617: 8 total, 8 up, 8 in 2024-09-14T04:03:57.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:57 smithi039 ceph-mon[106470]: pgmap v417: 297 pgs: 297 active+clean; 584 KiB data, 514 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:03:57.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:57 smithi039 ceph-mon[111235]: osdmap e617: 8 total, 8 up, 8 in 2024-09-14T04:03:57.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:57 smithi039 ceph-mon[111235]: pgmap v417: 297 pgs: 297 active+clean; 584 KiB data, 514 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:03:58.501 INFO:tasks.workunit.client.0.smithi039.stdout:num snaps is: 1 2024-09-14T04:03:58.501 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 1 2024-09-14T04:03:58.501 INFO:tasks.workunit.client.0.smithi039.stdout:snap: snap2-rename 2024-09-14T04:03:58.501 INFO:tasks.workunit.client.0.smithi039.stdout:found snap2-rename with size 4194304 2024-09-14T04:03:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:58 smithi129 ceph-mon[101316]: osdmap e618: 8 total, 8 up, 8 in 2024-09-14T04:03:58.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:58 smithi039 ceph-mon[106470]: osdmap e618: 8 total, 8 up, 8 in 2024-09-14T04:03:58.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:58 smithi039 ceph-mon[111235]: osdmap e618: 8 total, 8 up, 8 in 2024-09-14T04:03:58.791 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:03:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:03:58] "GET /metrics HTTP/1.1" 200 50836 "" "Prometheus/2.51.0" 2024-09-14T04:03:59.518 INFO:tasks.workunit.client.0.smithi039.stdout:num snaps is: 0 2024-09-14T04:03:59.519 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 0 2024-09-14T04:03:59.527 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestCreateLsRenameSnapPP (4077 ms) 2024-09-14T04:03:59.527 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.ConcurrentCreatesUnvalidatedPool 2024-09-14T04:03:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:59 smithi129 ceph-mon[101316]: osdmap e619: 8 total, 8 up, 8 in 2024-09-14T04:03:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:03:59 smithi129 ceph-mon[101316]: pgmap v420: 297 pgs: 297 active+clean; 584 KiB data, 515 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 6.2 KiB/s wr, 82 op/s 2024-09-14T04:03:59.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:59 smithi039 ceph-mon[106470]: osdmap e619: 8 total, 8 up, 8 in 2024-09-14T04:03:59.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:03:59 smithi039 ceph-mon[106470]: pgmap v420: 297 pgs: 297 active+clean; 584 KiB data, 515 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 6.2 KiB/s wr, 82 op/s 2024-09-14T04:03:59.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:59 smithi039 ceph-mon[111235]: osdmap e619: 8 total, 8 up, 8 in 2024-09-14T04:03:59.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:03:59 smithi039 ceph-mon[111235]: pgmap v420: 297 pgs: 297 active+clean; 584 KiB data, 515 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 6.2 KiB/s wr, 82 op/s 2024-09-14T04:04:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:00 smithi129 ceph-mon[101316]: osdmap e620: 8 total, 8 up, 8 in 2024-09-14T04:04:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:00 smithi129 ceph-mon[101316]: osdmap e621: 8 total, 8 up, 8 in 2024-09-14T04:04:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:00 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/400374920' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:00.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:00 smithi039 ceph-mon[106470]: osdmap e620: 8 total, 8 up, 8 in 2024-09-14T04:04:00.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:00 smithi039 ceph-mon[106470]: osdmap e621: 8 total, 8 up, 8 in 2024-09-14T04:04:00.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:00 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/400374920' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:00.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:00 smithi039 ceph-mon[111235]: osdmap e620: 8 total, 8 up, 8 in 2024-09-14T04:04:00.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:00 smithi039 ceph-mon[111235]: osdmap e621: 8 total, 8 up, 8 in 2024-09-14T04:04:00.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:00 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/400374920' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-22","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:01.451 INFO:tasks.workunit.client.0.smithi039.stdout:using new format!using new format! 2024-09-14T04:04:01.451 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:04:01.451 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:04:01.451 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:04:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:01 smithi129 ceph-mon[101316]: pgmap v423: 329 pgs: 32 unknown, 297 active+clean; 584 KiB data, 515 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 6.2 KiB/s wr, 82 op/s 2024-09-14T04:04:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:01 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/400374920' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:01 smithi129 ceph-mon[101316]: osdmap e622: 8 total, 8 up, 8 in 2024-09-14T04:04:01.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:01 smithi039 ceph-mon[106470]: pgmap v423: 329 pgs: 32 unknown, 297 active+clean; 584 KiB data, 515 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 6.2 KiB/s wr, 82 op/s 2024-09-14T04:04:01.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:01 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/400374920' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:01.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:01 smithi039 ceph-mon[106470]: osdmap e622: 8 total, 8 up, 8 in 2024-09-14T04:04:01.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:01 smithi039 ceph-mon[111235]: pgmap v423: 329 pgs: 32 unknown, 297 active+clean; 584 KiB data, 515 MiB used, 715 GiB / 715 GiB avail; 59 KiB/s rd, 6.2 KiB/s wr, 82 op/s 2024-09-14T04:04:01.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:01 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/400374920' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-22","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:01.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:01 smithi039 ceph-mon[111235]: osdmap e622: 8 total, 8 up, 8 in 2024-09-14T04:04:02.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:02 smithi129 ceph-mon[101316]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:02.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:02 smithi129 ceph-mon[101316]: osdmap e623: 8 total, 8 up, 8 in 2024-09-14T04:04:02.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:02 smithi039 ceph-mon[106470]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:02.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:02 smithi039 ceph-mon[106470]: osdmap e623: 8 total, 8 up, 8 in 2024-09-14T04:04:02.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:02 smithi039 ceph-mon[111235]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:02.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:02 smithi039 ceph-mon[111235]: osdmap e623: 8 total, 8 up, 8 in 2024-09-14T04:04:03.706 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.ConcurrentCreatesUnvalidatedPool (4179 ms) 2024-09-14T04:04:03.707 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.RemoveFullTry 2024-09-14T04:04:03.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:03 smithi039 ceph-mon[106470]: pgmap v426: 329 pgs: 329 active+clean; 584 KiB data, 515 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 1.7 KiB/s wr, 25 op/s 2024-09-14T04:04:03.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:03 smithi039 ceph-mon[106470]: osdmap e624: 8 total, 8 up, 8 in 2024-09-14T04:04:03.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:03 smithi039 ceph-mon[111235]: pgmap v426: 329 pgs: 329 active+clean; 584 KiB data, 515 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 1.7 KiB/s wr, 25 op/s 2024-09-14T04:04:03.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:03 smithi039 ceph-mon[111235]: osdmap e624: 8 total, 8 up, 8 in 2024-09-14T04:04:04.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:03 smithi129 ceph-mon[101316]: pgmap v426: 329 pgs: 329 active+clean; 584 KiB data, 515 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 1.7 KiB/s wr, 25 op/s 2024-09-14T04:04:04.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:03 smithi129 ceph-mon[101316]: osdmap e624: 8 total, 8 up, 8 in 2024-09-14T04:04:04.354 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'sleep 60' 2024-09-14T04:04:04.577 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:04:04.699 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:04 smithi039 ceph-mon[106470]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:04.700 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:04 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T04:04:04.700 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:04 smithi039 ceph-mon[106470]: from='client.44354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:04:04.700 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:04 smithi039 ceph-mon[111235]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:04.700 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:04 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T04:04:04.700 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:04 smithi039 ceph-mon[111235]: from='client.44354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:04:05.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:04 smithi129 ceph-mon[101316]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:05.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:04 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T04:04:05.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:04 smithi129 ceph-mon[101316]: from='client.44354 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:04:05.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:05 smithi039 ceph-mon[106470]: osdmap e625: 8 total, 8 up, 8 in 2024-09-14T04:04:05.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:05 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2661728608' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-23","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:05.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:05 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-23","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:05.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:05 smithi039 ceph-mon[106470]: pgmap v429: 361 pgs: 32 unknown, 329 active+clean; 584 KiB data, 515 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 1.7 KiB/s wr, 25 op/s 2024-09-14T04:04:05.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:05 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:05.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:05 smithi039 ceph-mon[106470]: osdmap e626: 8 total, 8 up, 8 in 2024-09-14T04:04:05.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:05 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-23","app": "rbd","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:05.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:05 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-23","app": "rbd","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:05.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:05 smithi039 ceph-mon[111235]: osdmap e625: 8 total, 8 up, 8 in 2024-09-14T04:04:05.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:05 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2661728608' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-23","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:05.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:05 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-23","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:05.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:05 smithi039 ceph-mon[111235]: pgmap v429: 361 pgs: 32 unknown, 329 active+clean; 584 KiB data, 515 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 1.7 KiB/s wr, 25 op/s 2024-09-14T04:04:05.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:05 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:05.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:05 smithi039 ceph-mon[111235]: osdmap e626: 8 total, 8 up, 8 in 2024-09-14T04:04:05.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:05 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-23","app": "rbd","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:05.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:05 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-23","app": "rbd","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:06.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:05 smithi129 ceph-mon[101316]: osdmap e625: 8 total, 8 up, 8 in 2024-09-14T04:04:06.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:05 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2661728608' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-23","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:06.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:05 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-23","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:06.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:05 smithi129 ceph-mon[101316]: pgmap v429: 361 pgs: 32 unknown, 329 active+clean; 584 KiB data, 515 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 1.7 KiB/s wr, 25 op/s 2024-09-14T04:04:06.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:05 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-23","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:06.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:05 smithi129 ceph-mon[101316]: osdmap e626: 8 total, 8 up, 8 in 2024-09-14T04:04:06.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:05 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-23","app": "rbd","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:06.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:05 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-23","app": "rbd","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:07.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:06 smithi129 ceph-mon[101316]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:07.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:06 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-23","app": "rbd","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:07.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:06 smithi129 ceph-mon[101316]: osdmap e627: 8 total, 8 up, 8 in 2024-09-14T04:04:07.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:06 smithi039 ceph-mon[106470]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:07.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:06 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-23","app": "rbd","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:07.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:06 smithi039 ceph-mon[106470]: osdmap e627: 8 total, 8 up, 8 in 2024-09-14T04:04:07.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:06 smithi039 ceph-mon[111235]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:07.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:06 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-23","app": "rbd","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:07.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:06 smithi039 ceph-mon[111235]: osdmap e627: 8 total, 8 up, 8 in 2024-09-14T04:04:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:07 smithi129 ceph-mon[101316]: pgmap v432: 361 pgs: 32 unknown, 329 active+clean; 584 KiB data, 515 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:04:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:04:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:07 smithi129 ceph-mon[101316]: osdmap e628: 8 total, 8 up, 8 in 2024-09-14T04:04:08.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:07 smithi039 ceph-mon[106470]: pgmap v432: 361 pgs: 32 unknown, 329 active+clean; 584 KiB data, 515 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:04:08.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:04:08.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:08.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:07 smithi039 ceph-mon[106470]: osdmap e628: 8 total, 8 up, 8 in 2024-09-14T04:04:08.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:07 smithi039 ceph-mon[111235]: pgmap v432: 361 pgs: 32 unknown, 329 active+clean; 584 KiB data, 515 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:04:08.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:04:08.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:08.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:07 smithi039 ceph-mon[111235]: osdmap e628: 8 total, 8 up, 8 in 2024-09-14T04:04:08.472 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:04:08.755 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:08 smithi039 ceph-mon[106470]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:08.755 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:08 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T04:04:08.755 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:08 smithi039 ceph-mon[106470]: osdmap e629: 8 total, 8 up, 8 in 2024-09-14T04:04:08.755 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:08 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' 2024-09-14T04:04:08.755 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:08 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:04:08.755 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:08 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-23", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-14T04:04:08.756 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:08 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-23", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-14T04:04:08.756 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:08 smithi039 ceph-mon[111235]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:08.756 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:08 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T04:04:08.756 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:08 smithi039 ceph-mon[111235]: osdmap e629: 8 total, 8 up, 8 in 2024-09-14T04:04:08.756 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:08 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' 2024-09-14T04:04:08.756 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:08 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:04:08.756 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:08 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-23", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-14T04:04:08.756 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:08 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-23", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-14T04:04:08.757 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:04:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:04:08] ENGINE Bus STOPPING 2024-09-14T04:04:08.757 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:04:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:04:08] "GET /metrics HTTP/1.1" 200 55599 "" "Prometheus/2.51.0" 2024-09-14T04:04:08.757 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:04:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:04:08] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T04:04:08.757 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:04:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:04:08] ENGINE Bus STOPPED 2024-09-14T04:04:08.757 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:04:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:04:08] ENGINE Bus STARTING 2024-09-14T04:04:08.757 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:04:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:04:08] ENGINE Serving on http://:::9283 2024-09-14T04:04:08.757 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:04:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:04:08] ENGINE Bus STARTED 2024-09-14T04:04:09.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:08 smithi129 ceph-mon[101316]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:09.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:08 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T04:04:09.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:08 smithi129 ceph-mon[101316]: osdmap e629: 8 total, 8 up, 8 in 2024-09-14T04:04:09.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:08 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' 2024-09-14T04:04:09.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:08 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:04:09.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:08 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-23", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-14T04:04:09.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:08 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-23", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-14T04:04:09.921 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:09 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-14T04:04:09.921 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:09 smithi129 ceph-mon[101316]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T04:04:09.922 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:09 smithi129 ceph-mon[101316]: pgmap v435: 361 pgs: 361 active+clean; 584 KiB data, 520 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 8.2 KiB/s wr, 144 op/s 2024-09-14T04:04:09.922 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:09 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-23", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-14T04:04:09.922 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:09 smithi129 ceph-mon[101316]: osdmap e630: 8 total, 8 up, 8 in 2024-09-14T04:04:10.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:09 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-14T04:04:10.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:09 smithi039 ceph-mon[106470]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T04:04:10.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:09 smithi039 ceph-mon[106470]: pgmap v435: 361 pgs: 361 active+clean; 584 KiB data, 520 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 8.2 KiB/s wr, 144 op/s 2024-09-14T04:04:10.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:09 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-23", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-14T04:04:10.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:09 smithi039 ceph-mon[106470]: osdmap e630: 8 total, 8 up, 8 in 2024-09-14T04:04:10.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:09 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-14T04:04:10.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:09 smithi039 ceph-mon[111235]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T04:04:10.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:09 smithi039 ceph-mon[111235]: pgmap v435: 361 pgs: 361 active+clean; 584 KiB data, 520 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 8.2 KiB/s wr, 144 op/s 2024-09-14T04:04:10.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:09 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-23", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-14T04:04:10.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:09 smithi039 ceph-mon[111235]: osdmap e630: 8 total, 8 up, 8 in 2024-09-14T04:04:11.664 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:11 smithi129 ceph-mon[101316]: osdmap e631: 8 total, 8 up, 8 in 2024-09-14T04:04:11.665 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:11 smithi129 ceph-mon[101316]: pgmap v438: 361 pgs: 361 active+clean; 584 KiB data, 520 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 8.2 KiB/s wr, 144 op/s 2024-09-14T04:04:11.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:11 smithi039 ceph-mon[106470]: osdmap e631: 8 total, 8 up, 8 in 2024-09-14T04:04:11.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:11 smithi039 ceph-mon[106470]: pgmap v438: 361 pgs: 361 active+clean; 584 KiB data, 520 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 8.2 KiB/s wr, 144 op/s 2024-09-14T04:04:11.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:11 smithi039 ceph-mon[111235]: osdmap e631: 8 total, 8 up, 8 in 2024-09-14T04:04:11.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:11 smithi039 ceph-mon[111235]: pgmap v438: 361 pgs: 361 active+clean; 584 KiB data, 520 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 8.2 KiB/s wr, 144 op/s 2024-09-14T04:04:14.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:13 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:04:14.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:13 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:04:14.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:13 smithi129 ceph-mon[101316]: pgmap v439: 361 pgs: 361 active+clean; 4.6 MiB data, 530 MiB used, 715 GiB / 715 GiB avail; 81 KiB/s rd, 771 KiB/s wr, 118 op/s 2024-09-14T04:04:14.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:13 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:04:14.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:13 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:04:14.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:13 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:04:14.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:13 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:04:14.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:13 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:04:14.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:13 smithi039 ceph-mon[111235]: pgmap v439: 361 pgs: 361 active+clean; 4.6 MiB data, 530 MiB used, 715 GiB / 715 GiB avail; 81 KiB/s rd, 771 KiB/s wr, 118 op/s 2024-09-14T04:04:14.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:13 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:04:14.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:13 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:04:14.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:13 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:04:14.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:13 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:04:14.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:13 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:04:14.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:13 smithi039 ceph-mon[106470]: pgmap v439: 361 pgs: 361 active+clean; 4.6 MiB data, 530 MiB used, 715 GiB / 715 GiB avail; 81 KiB/s rd, 771 KiB/s wr, 118 op/s 2024-09-14T04:04:14.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:13 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:04:14.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:13 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:04:14.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:13 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:04:16.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:15 smithi129 ceph-mon[101316]: pgmap v440: 361 pgs: 361 active+clean; 4.6 MiB data, 530 MiB used, 715 GiB / 715 GiB avail; 5.3 KiB/s rd, 645 KiB/s wr, 9 op/s 2024-09-14T04:04:16.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:15 smithi129 ceph-mon[101316]: osdmap e632: 8 total, 8 up, 8 in 2024-09-14T04:04:16.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:15 smithi039 ceph-mon[106470]: pgmap v440: 361 pgs: 361 active+clean; 4.6 MiB data, 530 MiB used, 715 GiB / 715 GiB avail; 5.3 KiB/s rd, 645 KiB/s wr, 9 op/s 2024-09-14T04:04:16.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:15 smithi039 ceph-mon[106470]: osdmap e632: 8 total, 8 up, 8 in 2024-09-14T04:04:16.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:15 smithi039 ceph-mon[111235]: pgmap v440: 361 pgs: 361 active+clean; 4.6 MiB data, 530 MiB used, 715 GiB / 715 GiB avail; 5.3 KiB/s rd, 645 KiB/s wr, 9 op/s 2024-09-14T04:04:16.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:15 smithi039 ceph-mon[111235]: osdmap e632: 8 total, 8 up, 8 in 2024-09-14T04:04:18.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:17 smithi129 ceph-mon[101316]: pgmap v442: 361 pgs: 361 active+clean; 4.6 MiB data, 530 MiB used, 715 GiB / 715 GiB avail; 4.6 KiB/s rd, 560 KiB/s wr, 8 op/s 2024-09-14T04:04:18.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:17 smithi039 ceph-mon[106470]: pgmap v442: 361 pgs: 361 active+clean; 4.6 MiB data, 530 MiB used, 715 GiB / 715 GiB avail; 4.6 KiB/s rd, 560 KiB/s wr, 8 op/s 2024-09-14T04:04:18.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:17 smithi039 ceph-mon[111235]: pgmap v442: 361 pgs: 361 active+clean; 4.6 MiB data, 530 MiB used, 715 GiB / 715 GiB avail; 4.6 KiB/s rd, 560 KiB/s wr, 8 op/s 2024-09-14T04:04:19.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:04:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:04:18] "GET /metrics HTTP/1.1" 200 55599 "" "Prometheus/2.51.0" 2024-09-14T04:04:20.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:19 smithi129 ceph-mon[101316]: pgmap v443: 361 pgs: 361 active+clean; 9.6 MiB data, 549 MiB used, 715 GiB / 715 GiB avail; 5.3 KiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-09-14T04:04:20.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:19 smithi039 ceph-mon[111235]: pgmap v443: 361 pgs: 361 active+clean; 9.6 MiB data, 549 MiB used, 715 GiB / 715 GiB avail; 5.3 KiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-09-14T04:04:20.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:19 smithi039 ceph-mon[106470]: pgmap v443: 361 pgs: 361 active+clean; 9.6 MiB data, 549 MiB used, 715 GiB / 715 GiB avail; 5.3 KiB/s rd, 1.1 MiB/s wr, 9 op/s 2024-09-14T04:04:22.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:21 smithi129 ceph-mon[101316]: pgmap v444: 361 pgs: 361 active+clean; 9.6 MiB data, 549 MiB used, 715 GiB / 715 GiB avail; 4.4 KiB/s rd, 922 KiB/s wr, 8 op/s 2024-09-14T04:04:22.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:21 smithi039 ceph-mon[106470]: pgmap v444: 361 pgs: 361 active+clean; 9.6 MiB data, 549 MiB used, 715 GiB / 715 GiB avail; 4.4 KiB/s rd, 922 KiB/s wr, 8 op/s 2024-09-14T04:04:22.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:21 smithi039 ceph-mon[111235]: pgmap v444: 361 pgs: 361 active+clean; 9.6 MiB data, 549 MiB used, 715 GiB / 715 GiB avail; 4.4 KiB/s rd, 922 KiB/s wr, 8 op/s 2024-09-14T04:04:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:23 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:04:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:23 smithi129 ceph-mon[101316]: pgmap v445: 361 pgs: 361 active+clean; 15 MiB data, 564 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.0 MiB/s wr, 3 op/s 2024-09-14T04:04:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:23 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:04:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:23 smithi039 ceph-mon[106470]: pgmap v445: 361 pgs: 361 active+clean; 15 MiB data, 564 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.0 MiB/s wr, 3 op/s 2024-09-14T04:04:23.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:23 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:04:23.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:23.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:23 smithi039 ceph-mon[111235]: pgmap v445: 361 pgs: 361 active+clean; 15 MiB data, 564 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.0 MiB/s wr, 3 op/s 2024-09-14T04:04:25.678 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.RemoveFullTry (21971 ms) 2024-09-14T04:04:25.678 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.RemoveFullTryDataPool 2024-09-14T04:04:26.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:25 smithi129 ceph-mon[101316]: pgmap v446: 361 pgs: 361 active+clean; 15 MiB data, 564 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.0 MiB/s wr, 3 op/s 2024-09-14T04:04:26.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:25 smithi129 ceph-mon[101316]: pool 'test-librbd-smithi039-85658-23' is full (reached quota's max_bytes: 10 MiB) 2024-09-14T04:04:26.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:25 smithi129 ceph-mon[101316]: Health check failed: 1 pool(s) full (POOL_FULL) 2024-09-14T04:04:26.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:25 smithi129 ceph-mon[101316]: osdmap e633: 8 total, 8 up, 8 in 2024-09-14T04:04:26.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:25 smithi039 ceph-mon[106470]: pgmap v446: 361 pgs: 361 active+clean; 15 MiB data, 564 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.0 MiB/s wr, 3 op/s 2024-09-14T04:04:26.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:25 smithi039 ceph-mon[106470]: pool 'test-librbd-smithi039-85658-23' is full (reached quota's max_bytes: 10 MiB) 2024-09-14T04:04:26.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:25 smithi039 ceph-mon[106470]: Health check failed: 1 pool(s) full (POOL_FULL) 2024-09-14T04:04:26.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:25 smithi039 ceph-mon[106470]: osdmap e633: 8 total, 8 up, 8 in 2024-09-14T04:04:26.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:25 smithi039 ceph-mon[111235]: pgmap v446: 361 pgs: 361 active+clean; 15 MiB data, 564 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.0 MiB/s wr, 3 op/s 2024-09-14T04:04:26.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:25 smithi039 ceph-mon[111235]: pool 'test-librbd-smithi039-85658-23' is full (reached quota's max_bytes: 10 MiB) 2024-09-14T04:04:26.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:25 smithi039 ceph-mon[111235]: Health check failed: 1 pool(s) full (POOL_FULL) 2024-09-14T04:04:26.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:25 smithi039 ceph-mon[111235]: osdmap e633: 8 total, 8 up, 8 in 2024-09-14T04:04:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:27 smithi129 ceph-mon[101316]: osdmap e634: 8 total, 8 up, 8 in 2024-09-14T04:04:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:27 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1511502327' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-24","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:27.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:27 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-24","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:27.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:27 smithi129 ceph-mon[101316]: pgmap v449: 393 pgs: 32 unknown, 361 active+clean; 15 MiB data, 564 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 640 KiB/s wr, 1 op/s 2024-09-14T04:04:27.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:27 smithi039 ceph-mon[106470]: osdmap e634: 8 total, 8 up, 8 in 2024-09-14T04:04:27.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:27 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1511502327' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-24","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:27.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:27 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-24","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:27.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:27 smithi039 ceph-mon[106470]: pgmap v449: 393 pgs: 32 unknown, 361 active+clean; 15 MiB data, 564 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 640 KiB/s wr, 1 op/s 2024-09-14T04:04:27.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:27 smithi039 ceph-mon[111235]: osdmap e634: 8 total, 8 up, 8 in 2024-09-14T04:04:27.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:27 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1511502327' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-24","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:27.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:27 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-24","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:27.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:27 smithi039 ceph-mon[111235]: pgmap v449: 393 pgs: 32 unknown, 361 active+clean; 15 MiB data, 564 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 640 KiB/s wr, 1 op/s 2024-09-14T04:04:28.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:28 smithi129 ceph-mon[101316]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:28.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:28 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:28.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:28 smithi129 ceph-mon[101316]: osdmap e635: 8 total, 8 up, 8 in 2024-09-14T04:04:28.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:28 smithi039 ceph-mon[106470]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:28.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:28 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:28.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:28 smithi039 ceph-mon[106470]: osdmap e635: 8 total, 8 up, 8 in 2024-09-14T04:04:28.791 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:04:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:04:28] "GET /metrics HTTP/1.1" 200 55662 "" "Prometheus/2.51.0" 2024-09-14T04:04:28.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:28 smithi039 ceph-mon[111235]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:28.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:28 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-24","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:28.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:28 smithi039 ceph-mon[111235]: osdmap e635: 8 total, 8 up, 8 in 2024-09-14T04:04:29.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:29 smithi129 ceph-mon[101316]: osdmap e636: 8 total, 8 up, 8 in 2024-09-14T04:04:29.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:29 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2609228669' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-25","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:29.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:29 smithi129 ceph-mon[101316]: pgmap v452: 425 pgs: 32 unknown, 393 active+clean; 584 KiB data, 525 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 514 KiB/s wr, 41 op/s 2024-09-14T04:04:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:29 smithi039 ceph-mon[111235]: osdmap e636: 8 total, 8 up, 8 in 2024-09-14T04:04:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:29 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2609228669' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-25","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:29 smithi039 ceph-mon[111235]: pgmap v452: 425 pgs: 32 unknown, 393 active+clean; 584 KiB data, 525 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 514 KiB/s wr, 41 op/s 2024-09-14T04:04:29.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:29 smithi039 ceph-mon[106470]: osdmap e636: 8 total, 8 up, 8 in 2024-09-14T04:04:29.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:29 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2609228669' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-25","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:29.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:29 smithi039 ceph-mon[106470]: pgmap v452: 425 pgs: 32 unknown, 393 active+clean; 584 KiB data, 525 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 514 KiB/s wr, 41 op/s 2024-09-14T04:04:30.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:30 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2609228669' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:30.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:30 smithi129 ceph-mon[101316]: osdmap e637: 8 total, 8 up, 8 in 2024-09-14T04:04:30.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:30 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-24","app": "rbd","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:30.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:30 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-24","app": "rbd","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:30.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:30 smithi129 ceph-mon[101316]: pool 'test-librbd-smithi039-85658-23' no longer out of quota; removing NO_QUOTA flag 2024-09-14T04:04:30.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:30 smithi129 ceph-mon[101316]: Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-09-14T04:04:30.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:30 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-24","app": "rbd","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:30.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:30 smithi129 ceph-mon[101316]: osdmap e638: 8 total, 8 up, 8 in 2024-09-14T04:04:30.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:30 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2609228669' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:30.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:30 smithi039 ceph-mon[106470]: osdmap e637: 8 total, 8 up, 8 in 2024-09-14T04:04:30.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:30 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-24","app": "rbd","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:30.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:30 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-24","app": "rbd","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:30.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:30 smithi039 ceph-mon[106470]: pool 'test-librbd-smithi039-85658-23' no longer out of quota; removing NO_QUOTA flag 2024-09-14T04:04:30.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:30 smithi039 ceph-mon[106470]: Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-09-14T04:04:30.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:30 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-24","app": "rbd","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:30.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:30 smithi039 ceph-mon[106470]: osdmap e638: 8 total, 8 up, 8 in 2024-09-14T04:04:30.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:30 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2609228669' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-25","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:30.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:30 smithi039 ceph-mon[111235]: osdmap e637: 8 total, 8 up, 8 in 2024-09-14T04:04:30.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:30 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-24","app": "rbd","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:30.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:30 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-24","app": "rbd","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:30.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:30 smithi039 ceph-mon[111235]: pool 'test-librbd-smithi039-85658-23' no longer out of quota; removing NO_QUOTA flag 2024-09-14T04:04:30.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:30 smithi039 ceph-mon[111235]: Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-09-14T04:04:30.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:30 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-24","app": "rbd","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:30.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:30 smithi039 ceph-mon[111235]: osdmap e638: 8 total, 8 up, 8 in 2024-09-14T04:04:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:31 smithi129 ceph-mon[101316]: pgmap v455: 425 pgs: 32 unknown, 393 active+clean; 584 KiB data, 525 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 514 KiB/s wr, 41 op/s 2024-09-14T04:04:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:31 smithi129 ceph-mon[101316]: osdmap e639: 8 total, 8 up, 8 in 2024-09-14T04:04:31.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:31 smithi039 ceph-mon[111235]: pgmap v455: 425 pgs: 32 unknown, 393 active+clean; 584 KiB data, 525 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 514 KiB/s wr, 41 op/s 2024-09-14T04:04:31.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:31 smithi039 ceph-mon[111235]: osdmap e639: 8 total, 8 up, 8 in 2024-09-14T04:04:31.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:31 smithi039 ceph-mon[106470]: pgmap v455: 425 pgs: 32 unknown, 393 active+clean; 584 KiB data, 525 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 514 KiB/s wr, 41 op/s 2024-09-14T04:04:31.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:31 smithi039 ceph-mon[106470]: osdmap e639: 8 total, 8 up, 8 in 2024-09-14T04:04:32.480 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:04:32.759 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:32 smithi039 ceph-mon[106470]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:32.759 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:32 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T04:04:32.760 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:32 smithi039 ceph-mon[106470]: osdmap e640: 8 total, 8 up, 8 in 2024-09-14T04:04:32.760 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:32 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' 2024-09-14T04:04:32.760 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:32 smithi039 ceph-mon[111235]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:32.760 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:32 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T04:04:32.760 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:32 smithi039 ceph-mon[111235]: osdmap e640: 8 total, 8 up, 8 in 2024-09-14T04:04:32.760 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:32 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' 2024-09-14T04:04:32.760 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:04:32 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:04:32] ENGINE Bus STOPPING 2024-09-14T04:04:32.760 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:04:32 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:04:32] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T04:04:32.761 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:04:32 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:04:32] ENGINE Bus STOPPED 2024-09-14T04:04:32.761 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:04:32 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:04:32] ENGINE Bus STARTING 2024-09-14T04:04:32.761 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:04:32 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:04:32] ENGINE Serving on http://:::9283 2024-09-14T04:04:32.761 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:04:32 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:04:32] ENGINE Bus STARTED 2024-09-14T04:04:32.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:32 smithi129 ceph-mon[101316]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:04:32.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:32 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T04:04:32.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:32 smithi129 ceph-mon[101316]: osdmap e640: 8 total, 8 up, 8 in 2024-09-14T04:04:32.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:32 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' 2024-09-14T04:04:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:33 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:04:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:33 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:04:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:33 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:04:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:33 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:04:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:33 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-14T04:04:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:33 smithi129 ceph-mon[101316]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T04:04:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:33 smithi129 ceph-mon[101316]: pgmap v458: 425 pgs: 425 active+clean; 584 KiB data, 530 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 767 B/s wr, 0 op/s 2024-09-14T04:04:33.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:33 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:04:33.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:33 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:04:33.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:33 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:04:33.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:33 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:04:33.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:33 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-14T04:04:33.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:33 smithi039 ceph-mon[111235]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T04:04:33.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:33 smithi039 ceph-mon[111235]: pgmap v458: 425 pgs: 425 active+clean; 584 KiB data, 530 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 767 B/s wr, 0 op/s 2024-09-14T04:04:33.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:33 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:04:33.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:33 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:04:33.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:33 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:04:33.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:33 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:04:33.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:33 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-14T04:04:33.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:33 smithi039 ceph-mon[106470]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T04:04:33.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:33 smithi039 ceph-mon[106470]: pgmap v458: 425 pgs: 425 active+clean; 584 KiB data, 530 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 767 B/s wr, 0 op/s 2024-09-14T04:04:34.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:34 smithi129 ceph-mon[101316]: osdmap e641: 8 total, 8 up, 8 in 2024-09-14T04:04:34.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:34 smithi039 ceph-mon[106470]: osdmap e641: 8 total, 8 up, 8 in 2024-09-14T04:04:34.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:34 smithi039 ceph-mon[111235]: osdmap e641: 8 total, 8 up, 8 in 2024-09-14T04:04:35.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:35 smithi039 ceph-mon[106470]: osdmap e642: 8 total, 8 up, 8 in 2024-09-14T04:04:35.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:35 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-25", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-14T04:04:35.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:35 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-25", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-14T04:04:35.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:35 smithi039 ceph-mon[106470]: pgmap v461: 425 pgs: 425 active+clean; 584 KiB data, 530 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 767 B/s wr, 0 op/s 2024-09-14T04:04:35.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:35 smithi039 ceph-mon[111235]: osdmap e642: 8 total, 8 up, 8 in 2024-09-14T04:04:35.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:35 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-25", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-14T04:04:35.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:35 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-25", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-14T04:04:35.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:35 smithi039 ceph-mon[111235]: pgmap v461: 425 pgs: 425 active+clean; 584 KiB data, 530 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 767 B/s wr, 0 op/s 2024-09-14T04:04:36.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:35 smithi129 ceph-mon[101316]: osdmap e642: 8 total, 8 up, 8 in 2024-09-14T04:04:36.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:35 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-25", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-14T04:04:36.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:35 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-25", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-14T04:04:36.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:35 smithi129 ceph-mon[101316]: pgmap v461: 425 pgs: 425 active+clean; 584 KiB data, 530 MiB used, 715 GiB / 715 GiB avail; 255 B/s rd, 767 B/s wr, 0 op/s 2024-09-14T04:04:36.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:36 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-25", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-14T04:04:36.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:36 smithi039 ceph-mon[106470]: osdmap e643: 8 total, 8 up, 8 in 2024-09-14T04:04:36.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:36 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-25", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-14T04:04:36.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:36 smithi039 ceph-mon[111235]: osdmap e643: 8 total, 8 up, 8 in 2024-09-14T04:04:37.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:36 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-25", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-14T04:04:37.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:36 smithi129 ceph-mon[101316]: osdmap e643: 8 total, 8 up, 8 in 2024-09-14T04:04:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:37 smithi129 ceph-mon[101316]: pgmap v463: 425 pgs: 425 active+clean; 584 KiB data, 530 MiB used, 715 GiB / 715 GiB avail; 234 B/s rd, 704 B/s wr, 0 op/s 2024-09-14T04:04:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:37 smithi039 ceph-mon[106470]: pgmap v463: 425 pgs: 425 active+clean; 584 KiB data, 530 MiB used, 715 GiB / 715 GiB avail; 234 B/s rd, 704 B/s wr, 0 op/s 2024-09-14T04:04:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:38.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:37 smithi039 ceph-mon[111235]: pgmap v463: 425 pgs: 425 active+clean; 584 KiB data, 530 MiB used, 715 GiB / 715 GiB avail; 234 B/s rd, 704 B/s wr, 0 op/s 2024-09-14T04:04:38.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:39.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:04:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:04:38] "GET /metrics HTTP/1.1" 200 60422 "" "Prometheus/2.51.0" 2024-09-14T04:04:40.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:39 smithi129 ceph-mon[101316]: pgmap v464: 425 pgs: 425 active+clean; 3.6 MiB data, 536 MiB used, 715 GiB / 715 GiB avail; 5.3 KiB/s rd, 513 KiB/s wr, 9 op/s 2024-09-14T04:04:40.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:39 smithi039 ceph-mon[106470]: pgmap v464: 425 pgs: 425 active+clean; 3.6 MiB data, 536 MiB used, 715 GiB / 715 GiB avail; 5.3 KiB/s rd, 513 KiB/s wr, 9 op/s 2024-09-14T04:04:40.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:39 smithi039 ceph-mon[111235]: pgmap v464: 425 pgs: 425 active+clean; 3.6 MiB data, 536 MiB used, 715 GiB / 715 GiB avail; 5.3 KiB/s rd, 513 KiB/s wr, 9 op/s 2024-09-14T04:04:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:41 smithi129 ceph-mon[101316]: osdmap e644: 8 total, 8 up, 8 in 2024-09-14T04:04:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:41 smithi129 ceph-mon[101316]: pgmap v466: 425 pgs: 425 active+clean; 3.6 MiB data, 536 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 490 KiB/s wr, 9 op/s 2024-09-14T04:04:41.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:41 smithi039 ceph-mon[106470]: osdmap e644: 8 total, 8 up, 8 in 2024-09-14T04:04:41.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:41 smithi039 ceph-mon[106470]: pgmap v466: 425 pgs: 425 active+clean; 3.6 MiB data, 536 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 490 KiB/s wr, 9 op/s 2024-09-14T04:04:41.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:41 smithi039 ceph-mon[111235]: osdmap e644: 8 total, 8 up, 8 in 2024-09-14T04:04:41.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:41 smithi039 ceph-mon[111235]: pgmap v466: 425 pgs: 425 active+clean; 3.6 MiB data, 536 MiB used, 715 GiB / 715 GiB avail; 5.1 KiB/s rd, 490 KiB/s wr, 9 op/s 2024-09-14T04:04:44.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:43 smithi129 ceph-mon[101316]: pgmap v467: 425 pgs: 425 active+clean; 8.6 MiB data, 551 MiB used, 715 GiB / 715 GiB avail; 4.6 KiB/s rd, 1.0 MiB/s wr, 8 op/s 2024-09-14T04:04:44.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:43 smithi039 ceph-mon[106470]: pgmap v467: 425 pgs: 425 active+clean; 8.6 MiB data, 551 MiB used, 715 GiB / 715 GiB avail; 4.6 KiB/s rd, 1.0 MiB/s wr, 8 op/s 2024-09-14T04:04:44.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:43 smithi039 ceph-mon[111235]: pgmap v467: 425 pgs: 425 active+clean; 8.6 MiB data, 551 MiB used, 715 GiB / 715 GiB avail; 4.6 KiB/s rd, 1.0 MiB/s wr, 8 op/s 2024-09-14T04:04:46.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:45 smithi129 ceph-mon[101316]: pgmap v468: 425 pgs: 425 active+clean; 8.6 MiB data, 551 MiB used, 715 GiB / 715 GiB avail; 4.0 KiB/s rd, 884 KiB/s wr, 7 op/s 2024-09-14T04:04:46.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:45 smithi039 ceph-mon[106470]: pgmap v468: 425 pgs: 425 active+clean; 8.6 MiB data, 551 MiB used, 715 GiB / 715 GiB avail; 4.0 KiB/s rd, 884 KiB/s wr, 7 op/s 2024-09-14T04:04:46.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:45 smithi039 ceph-mon[111235]: pgmap v468: 425 pgs: 425 active+clean; 8.6 MiB data, 551 MiB used, 715 GiB / 715 GiB avail; 4.0 KiB/s rd, 884 KiB/s wr, 7 op/s 2024-09-14T04:04:48.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:47 smithi129 ceph-mon[101316]: pgmap v469: 425 pgs: 425 active+clean; 8.6 MiB data, 551 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 820 KiB/s wr, 7 op/s 2024-09-14T04:04:48.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:47 smithi039 ceph-mon[106470]: pgmap v469: 425 pgs: 425 active+clean; 8.6 MiB data, 551 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 820 KiB/s wr, 7 op/s 2024-09-14T04:04:48.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:47 smithi039 ceph-mon[111235]: pgmap v469: 425 pgs: 425 active+clean; 8.6 MiB data, 551 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 820 KiB/s wr, 7 op/s 2024-09-14T04:04:49.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:04:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:04:48] "GET /metrics HTTP/1.1" 200 60422 "" "Prometheus/2.51.0" 2024-09-14T04:04:50.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:49 smithi129 ceph-mon[101316]: pgmap v470: 425 pgs: 425 active+clean; 14 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.0 MiB/s wr, 3 op/s 2024-09-14T04:04:50.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:49 smithi039 ceph-mon[106470]: pgmap v470: 425 pgs: 425 active+clean; 14 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.0 MiB/s wr, 3 op/s 2024-09-14T04:04:50.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:49 smithi039 ceph-mon[111235]: pgmap v470: 425 pgs: 425 active+clean; 14 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.0 MiB/s wr, 3 op/s 2024-09-14T04:04:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:50 smithi129 ceph-mon[101316]: pool 'test-librbd-smithi039-85658-25' is full (reached quota's max_bytes: 10 MiB) 2024-09-14T04:04:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:50 smithi129 ceph-mon[101316]: Health check failed: 1 pool(s) full (POOL_FULL) 2024-09-14T04:04:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:50 smithi129 ceph-mon[101316]: osdmap e645: 8 total, 8 up, 8 in 2024-09-14T04:04:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:50 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-24", "field": "max_objects", "val": "6"} : dispatch 2024-09-14T04:04:51.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:50 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-24", "field": "max_objects", "val": "6"} : dispatch 2024-09-14T04:04:51.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:50 smithi039 ceph-mon[106470]: pool 'test-librbd-smithi039-85658-25' is full (reached quota's max_bytes: 10 MiB) 2024-09-14T04:04:51.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:50 smithi039 ceph-mon[106470]: Health check failed: 1 pool(s) full (POOL_FULL) 2024-09-14T04:04:51.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:50 smithi039 ceph-mon[106470]: osdmap e645: 8 total, 8 up, 8 in 2024-09-14T04:04:51.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:50 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-24", "field": "max_objects", "val": "6"} : dispatch 2024-09-14T04:04:51.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:50 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-24", "field": "max_objects", "val": "6"} : dispatch 2024-09-14T04:04:51.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:50 smithi039 ceph-mon[111235]: pool 'test-librbd-smithi039-85658-25' is full (reached quota's max_bytes: 10 MiB) 2024-09-14T04:04:51.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:50 smithi039 ceph-mon[111235]: Health check failed: 1 pool(s) full (POOL_FULL) 2024-09-14T04:04:51.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:50 smithi039 ceph-mon[111235]: osdmap e645: 8 total, 8 up, 8 in 2024-09-14T04:04:51.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:50 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-24", "field": "max_objects", "val": "6"} : dispatch 2024-09-14T04:04:51.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:50 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-24", "field": "max_objects", "val": "6"} : dispatch 2024-09-14T04:04:51.466 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:04:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:51 smithi129 ceph-mon[101316]: pgmap v472: 425 pgs: 425 active+clean; 14 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.0 MiB/s wr, 3 op/s 2024-09-14T04:04:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:51 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-24", "field": "max_objects", "val": "6"}]': finished 2024-09-14T04:04:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:51 smithi129 ceph-mon[101316]: osdmap e646: 8 total, 8 up, 8 in 2024-09-14T04:04:52.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:51 smithi039 ceph-mon[106470]: pgmap v472: 425 pgs: 425 active+clean; 14 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.0 MiB/s wr, 3 op/s 2024-09-14T04:04:52.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:51 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-24", "field": "max_objects", "val": "6"}]': finished 2024-09-14T04:04:52.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:51 smithi039 ceph-mon[106470]: osdmap e646: 8 total, 8 up, 8 in 2024-09-14T04:04:52.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:51 smithi039 ceph-mon[111235]: pgmap v472: 425 pgs: 425 active+clean; 14 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 1.0 MiB/s wr, 3 op/s 2024-09-14T04:04:52.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:51 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-24", "field": "max_objects", "val": "6"}]': finished 2024-09-14T04:04:52.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:51 smithi039 ceph-mon[111235]: osdmap e646: 8 total, 8 up, 8 in 2024-09-14T04:04:52.481 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:04:53.493 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:04:53.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:53 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:04:53.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:53 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:53.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:53 smithi129 ceph-mon[101316]: pgmap v474: 425 pgs: 425 active+clean; 16 MiB data, 576 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 897 KiB/s wr, 6 op/s 2024-09-14T04:04:53.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:53 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:04:53.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:53 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:53.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:53 smithi039 ceph-mon[106470]: pgmap v474: 425 pgs: 425 active+clean; 16 MiB data, 576 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 897 KiB/s wr, 6 op/s 2024-09-14T04:04:53.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:53 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:04:53.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:53 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:04:53.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:53 smithi039 ceph-mon[111235]: pgmap v474: 425 pgs: 425 active+clean; 16 MiB data, 576 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 897 KiB/s wr, 6 op/s 2024-09-14T04:04:54.504 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:04:55.516 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:04:55.588 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.RemoveFullTryDataPool (29911 ms) 2024-09-14T04:04:55.589 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.RemoveFullTryNamespace 2024-09-14T04:04:56.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:55 smithi129 ceph-mon[101316]: pgmap v475: 425 pgs: 425 active+clean; 16 MiB data, 576 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 897 KiB/s wr, 6 op/s 2024-09-14T04:04:56.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:55 smithi129 ceph-mon[101316]: pool 'test-librbd-smithi039-85658-24' is full (reached quota's max_objects: 6) 2024-09-14T04:04:56.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:55 smithi129 ceph-mon[101316]: Health check update: 2 pool(s) full (POOL_FULL) 2024-09-14T04:04:56.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:55 smithi129 ceph-mon[101316]: osdmap e647: 8 total, 8 up, 8 in 2024-09-14T04:04:56.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:55 smithi039 ceph-mon[106470]: pgmap v475: 425 pgs: 425 active+clean; 16 MiB data, 576 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 897 KiB/s wr, 6 op/s 2024-09-14T04:04:56.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:55 smithi039 ceph-mon[106470]: pool 'test-librbd-smithi039-85658-24' is full (reached quota's max_objects: 6) 2024-09-14T04:04:56.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:55 smithi039 ceph-mon[106470]: Health check update: 2 pool(s) full (POOL_FULL) 2024-09-14T04:04:56.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:55 smithi039 ceph-mon[106470]: osdmap e647: 8 total, 8 up, 8 in 2024-09-14T04:04:56.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:55 smithi039 ceph-mon[111235]: pgmap v475: 425 pgs: 425 active+clean; 16 MiB data, 576 MiB used, 715 GiB / 715 GiB avail; 2.7 KiB/s rd, 897 KiB/s wr, 6 op/s 2024-09-14T04:04:56.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:55 smithi039 ceph-mon[111235]: pool 'test-librbd-smithi039-85658-24' is full (reached quota's max_objects: 6) 2024-09-14T04:04:56.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:55 smithi039 ceph-mon[111235]: Health check update: 2 pool(s) full (POOL_FULL) 2024-09-14T04:04:56.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:55 smithi039 ceph-mon[111235]: osdmap e647: 8 total, 8 up, 8 in 2024-09-14T04:04:57.505 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:04:57.765 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:57 smithi039 ceph-mon[106470]: osdmap e648: 8 total, 8 up, 8 in 2024-09-14T04:04:57.765 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:57 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2198547336' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-26","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:57.765 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:57 smithi039 ceph-mon[106470]: pgmap v478: 457 pgs: 32 unknown, 425 active+clean; 16 MiB data, 576 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 343 KiB/s wr, 4 op/s 2024-09-14T04:04:57.765 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:57 smithi039 ceph-mon[111235]: osdmap e648: 8 total, 8 up, 8 in 2024-09-14T04:04:57.765 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:57 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2198547336' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-26","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:57.765 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:57 smithi039 ceph-mon[111235]: pgmap v478: 457 pgs: 32 unknown, 425 active+clean; 16 MiB data, 576 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 343 KiB/s wr, 4 op/s 2024-09-14T04:04:57.765 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:04:57 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:04:57] ENGINE Bus STOPPING 2024-09-14T04:04:57.766 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:04:57 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:04:57] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T04:04:57.766 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:04:57 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:04:57] ENGINE Bus STOPPED 2024-09-14T04:04:57.766 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:04:57 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:04:57] ENGINE Bus STARTING 2024-09-14T04:04:57.766 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:04:57 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:04:57] ENGINE Serving on http://:::9283 2024-09-14T04:04:57.766 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:04:57 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:04:57] ENGINE Bus STARTED 2024-09-14T04:04:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:57 smithi129 ceph-mon[101316]: osdmap e648: 8 total, 8 up, 8 in 2024-09-14T04:04:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:57 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2198547336' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-26","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:04:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:57 smithi129 ceph-mon[101316]: pgmap v478: 457 pgs: 32 unknown, 425 active+clean; 16 MiB data, 576 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 343 KiB/s wr, 4 op/s 2024-09-14T04:04:58.641 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:58 smithi039 ceph-mon[106470]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:58.641 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:58 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2198547336' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:58.642 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:58 smithi039 ceph-mon[106470]: osdmap e649: 8 total, 8 up, 8 in 2024-09-14T04:04:58.642 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:58 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' 2024-09-14T04:04:58.642 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:58 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:04:58.642 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:58 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-14T04:04:58.642 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:58 smithi039 ceph-mon[106470]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T04:04:58.642 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:58 smithi039 ceph-mon[111235]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:58.642 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:58 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2198547336' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:58.642 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:58 smithi039 ceph-mon[111235]: osdmap e649: 8 total, 8 up, 8 in 2024-09-14T04:04:58.642 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:58 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' 2024-09-14T04:04:58.643 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:58 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:04:58.643 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:58 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-14T04:04:58.643 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:58 smithi039 ceph-mon[111235]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T04:04:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:58 smithi129 ceph-mon[101316]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:04:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:58 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2198547336' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-26","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:04:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:58 smithi129 ceph-mon[101316]: osdmap e649: 8 total, 8 up, 8 in 2024-09-14T04:04:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:58 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' 2024-09-14T04:04:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:58 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:04:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:58 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-14T04:04:58.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:58 smithi129 ceph-mon[101316]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T04:04:59.011 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:04:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:04:58] "GET /metrics HTTP/1.1" 200 60537 "" "Prometheus/2.51.0" 2024-09-14T04:04:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:59 smithi129 ceph-mon[101316]: osdmap e650: 8 total, 8 up, 8 in 2024-09-14T04:04:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:04:59 smithi129 ceph-mon[101316]: pgmap v481: 457 pgs: 457 active+clean; 584 KiB data, 531 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 4.7 KiB/s wr, 48 op/s 2024-09-14T04:04:59.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:59 smithi039 ceph-mon[106470]: osdmap e650: 8 total, 8 up, 8 in 2024-09-14T04:04:59.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:04:59 smithi039 ceph-mon[106470]: pgmap v481: 457 pgs: 457 active+clean; 584 KiB data, 531 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 4.7 KiB/s wr, 48 op/s 2024-09-14T04:04:59.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:59 smithi039 ceph-mon[111235]: osdmap e650: 8 total, 8 up, 8 in 2024-09-14T04:04:59.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:04:59 smithi039 ceph-mon[111235]: pgmap v481: 457 pgs: 457 active+clean; 584 KiB data, 531 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 4.7 KiB/s wr, 48 op/s 2024-09-14T04:05:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:00 smithi129 ceph-mon[101316]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:05:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:00 smithi129 ceph-mon[101316]: osdmap e651: 8 total, 8 up, 8 in 2024-09-14T04:05:00.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:00 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-26", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-14T04:05:00.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:00 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-26", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-14T04:05:00.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:00 smithi129 ceph-mon[101316]: pool 'test-librbd-smithi039-85658-25' no longer out of quota; removing NO_QUOTA flag 2024-09-14T04:05:00.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:00 smithi129 ceph-mon[101316]: Health check update: 1 pool(s) full (POOL_FULL) 2024-09-14T04:05:00.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:00 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-26", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-14T04:05:00.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:00 smithi129 ceph-mon[101316]: osdmap e652: 8 total, 8 up, 8 in 2024-09-14T04:05:00.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:00 smithi039 ceph-mon[106470]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:05:00.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:00 smithi039 ceph-mon[106470]: osdmap e651: 8 total, 8 up, 8 in 2024-09-14T04:05:00.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:00 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-26", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-14T04:05:00.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:00 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-26", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-14T04:05:00.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:00 smithi039 ceph-mon[106470]: pool 'test-librbd-smithi039-85658-25' no longer out of quota; removing NO_QUOTA flag 2024-09-14T04:05:00.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:00 smithi039 ceph-mon[106470]: Health check update: 1 pool(s) full (POOL_FULL) 2024-09-14T04:05:00.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:00 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-26", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-14T04:05:00.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:00 smithi039 ceph-mon[106470]: osdmap e652: 8 total, 8 up, 8 in 2024-09-14T04:05:00.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:00 smithi039 ceph-mon[111235]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:05:00.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:00 smithi039 ceph-mon[111235]: osdmap e651: 8 total, 8 up, 8 in 2024-09-14T04:05:00.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:00 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-26", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-14T04:05:00.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:00 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-26", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-14T04:05:00.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:00 smithi039 ceph-mon[111235]: pool 'test-librbd-smithi039-85658-25' no longer out of quota; removing NO_QUOTA flag 2024-09-14T04:05:00.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:00 smithi039 ceph-mon[111235]: Health check update: 1 pool(s) full (POOL_FULL) 2024-09-14T04:05:00.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:00 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-26", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-14T04:05:00.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:00 smithi039 ceph-mon[111235]: osdmap e652: 8 total, 8 up, 8 in 2024-09-14T04:05:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:01 smithi129 ceph-mon[101316]: pgmap v484: 457 pgs: 457 active+clean; 584 KiB data, 531 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 4.7 KiB/s wr, 48 op/s 2024-09-14T04:05:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:01 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:01 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:01.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:01 smithi039 ceph-mon[106470]: pgmap v484: 457 pgs: 457 active+clean; 584 KiB data, 531 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 4.7 KiB/s wr, 48 op/s 2024-09-14T04:05:01.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:01 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:01.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:01 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:01.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:01 smithi039 ceph-mon[111235]: pgmap v484: 457 pgs: 457 active+clean; 584 KiB data, 531 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 4.7 KiB/s wr, 48 op/s 2024-09-14T04:05:01.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:01 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:01.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:01 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:02.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:02 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:05:02.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:02 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:05:02.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:02 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:02.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:02 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:05:02.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:02 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:05:02.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:02 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:02.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:02 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:05:02.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:02 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:05:02.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:02 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:03 smithi129 ceph-mon[101316]: pgmap v485: 457 pgs: 457 active+clean; 3.6 MiB data, 550 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 578 KiB/s wr, 46 op/s 2024-09-14T04:05:03.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:03 smithi039 ceph-mon[106470]: pgmap v485: 457 pgs: 457 active+clean; 3.6 MiB data, 550 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 578 KiB/s wr, 46 op/s 2024-09-14T04:05:03.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:03 smithi039 ceph-mon[111235]: pgmap v485: 457 pgs: 457 active+clean; 3.6 MiB data, 550 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 578 KiB/s wr, 46 op/s 2024-09-14T04:05:06.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:05 smithi129 ceph-mon[101316]: pgmap v486: 457 pgs: 457 active+clean; 3.6 MiB data, 550 MiB used, 715 GiB / 715 GiB avail; 5.2 KiB/s rd, 485 KiB/s wr, 9 op/s 2024-09-14T04:05:06.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:05 smithi129 ceph-mon[101316]: osdmap e653: 8 total, 8 up, 8 in 2024-09-14T04:05:06.289 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:05 smithi039 ceph-mon[106470]: pgmap v486: 457 pgs: 457 active+clean; 3.6 MiB data, 550 MiB used, 715 GiB / 715 GiB avail; 5.2 KiB/s rd, 485 KiB/s wr, 9 op/s 2024-09-14T04:05:06.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:05 smithi039 ceph-mon[106470]: osdmap e653: 8 total, 8 up, 8 in 2024-09-14T04:05:06.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:05 smithi039 ceph-mon[111235]: pgmap v486: 457 pgs: 457 active+clean; 3.6 MiB data, 550 MiB used, 715 GiB / 715 GiB avail; 5.2 KiB/s rd, 485 KiB/s wr, 9 op/s 2024-09-14T04:05:06.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:05 smithi039 ceph-mon[111235]: osdmap e653: 8 total, 8 up, 8 in 2024-09-14T04:05:07.060 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'echo wait for half of osds to upgrade' 2024-09-14T04:05:07.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:06 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.6", "id": [7, 5]} : dispatch 2024-09-14T04:05:07.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:06 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.15", "id": [7, 4]} : dispatch 2024-09-14T04:05:07.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:06 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.6", "id": [7, 5]} : dispatch 2024-09-14T04:05:07.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:06 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.17", "id": [7, 0]} : dispatch 2024-09-14T04:05:07.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:06 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.15", "id": [7, 4]} : dispatch 2024-09-14T04:05:07.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:06 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.c", "id": [7, 1]} : dispatch 2024-09-14T04:05:07.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:06 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.17", "id": [7, 0]} : dispatch 2024-09-14T04:05:07.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:06 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.e", "id": [7, 0]} : dispatch 2024-09-14T04:05:07.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:06 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.11", "id": [7, 0]} : dispatch 2024-09-14T04:05:07.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:06 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.c", "id": [7, 1]} : dispatch 2024-09-14T04:05:07.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:06 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.7", "id": [0, 2]} : dispatch 2024-09-14T04:05:07.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:06 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.e", "id": [7, 0]} : dispatch 2024-09-14T04:05:07.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:06 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.11", "id": [0, 2]} : dispatch 2024-09-14T04:05:07.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:06 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.a", "id": [6, 4]} : dispatch 2024-09-14T04:05:07.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:06 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.11", "id": [7, 0]} : dispatch 2024-09-14T04:05:07.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:06 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.3", "id": [6, 5]} : dispatch 2024-09-14T04:05:07.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:06 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.7", "id": [0, 2]} : dispatch 2024-09-14T04:05:07.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:06 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.11", "id": [0, 2]} : dispatch 2024-09-14T04:05:07.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:06 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.a", "id": [6, 4]} : dispatch 2024-09-14T04:05:07.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:06 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.3", "id": [6, 5]} : dispatch 2024-09-14T04:05:07.273 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:05:07.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.6", "id": [7, 5]} : dispatch 2024-09-14T04:05:07.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.15", "id": [7, 4]} : dispatch 2024-09-14T04:05:07.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.6", "id": [7, 5]} : dispatch 2024-09-14T04:05:07.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.17", "id": [7, 0]} : dispatch 2024-09-14T04:05:07.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.15", "id": [7, 4]} : dispatch 2024-09-14T04:05:07.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.c", "id": [7, 1]} : dispatch 2024-09-14T04:05:07.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.17", "id": [7, 0]} : dispatch 2024-09-14T04:05:07.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.e", "id": [7, 0]} : dispatch 2024-09-14T04:05:07.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.11", "id": [7, 0]} : dispatch 2024-09-14T04:05:07.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.c", "id": [7, 1]} : dispatch 2024-09-14T04:05:07.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.7", "id": [0, 2]} : dispatch 2024-09-14T04:05:07.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.e", "id": [7, 0]} : dispatch 2024-09-14T04:05:07.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.11", "id": [0, 2]} : dispatch 2024-09-14T04:05:07.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.a", "id": [6, 4]} : dispatch 2024-09-14T04:05:07.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.11", "id": [7, 0]} : dispatch 2024-09-14T04:05:07.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.3", "id": [6, 5]} : dispatch 2024-09-14T04:05:07.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.7", "id": [0, 2]} : dispatch 2024-09-14T04:05:07.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.11", "id": [0, 2]} : dispatch 2024-09-14T04:05:07.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.a", "id": [6, 4]} : dispatch 2024-09-14T04:05:07.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.3", "id": [6, 5]} : dispatch 2024-09-14T04:05:07.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.6", "id": [7, 5]} : dispatch 2024-09-14T04:05:07.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.15", "id": [7, 4]} : dispatch 2024-09-14T04:05:07.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.6", "id": [7, 5]} : dispatch 2024-09-14T04:05:07.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.17", "id": [7, 0]} : dispatch 2024-09-14T04:05:07.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.15", "id": [7, 4]} : dispatch 2024-09-14T04:05:07.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.c", "id": [7, 1]} : dispatch 2024-09-14T04:05:07.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.17", "id": [7, 0]} : dispatch 2024-09-14T04:05:07.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.e", "id": [7, 0]} : dispatch 2024-09-14T04:05:07.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.11", "id": [7, 0]} : dispatch 2024-09-14T04:05:07.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.c", "id": [7, 1]} : dispatch 2024-09-14T04:05:07.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.7", "id": [0, 2]} : dispatch 2024-09-14T04:05:07.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.e", "id": [7, 0]} : dispatch 2024-09-14T04:05:07.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.11", "id": [0, 2]} : dispatch 2024-09-14T04:05:07.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.a", "id": [6, 4]} : dispatch 2024-09-14T04:05:07.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.11", "id": [7, 0]} : dispatch 2024-09-14T04:05:07.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.3", "id": [6, 5]} : dispatch 2024-09-14T04:05:07.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.7", "id": [0, 2]} : dispatch 2024-09-14T04:05:07.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.11", "id": [0, 2]} : dispatch 2024-09-14T04:05:07.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.a", "id": [6, 4]} : dispatch 2024-09-14T04:05:07.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:06 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.3", "id": [6, 5]} : dispatch 2024-09-14T04:05:08.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:07 smithi129 ceph-mon[101316]: pgmap v488: 457 pgs: 457 active+clean; 3.6 MiB data, 550 MiB used, 715 GiB / 715 GiB avail; 4.5 KiB/s rd, 420 KiB/s wr, 7 op/s 2024-09-14T04:05:08.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.6", "id": [7, 5]}]': finished 2024-09-14T04:05:08.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.15", "id": [7, 4]}]': finished 2024-09-14T04:05:08.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.17", "id": [7, 0]}]': finished 2024-09-14T04:05:08.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.c", "id": [7, 1]}]': finished 2024-09-14T04:05:08.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.e", "id": [7, 0]}]': finished 2024-09-14T04:05:08.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.11", "id": [7, 0]}]': finished 2024-09-14T04:05:08.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.7", "id": [0, 2]}]': finished 2024-09-14T04:05:08.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.11", "id": [0, 2]}]': finished 2024-09-14T04:05:08.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.a", "id": [6, 4]}]': finished 2024-09-14T04:05:08.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.3", "id": [6, 5]}]': finished 2024-09-14T04:05:08.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:07 smithi129 ceph-mon[101316]: osdmap e654: 8 total, 8 up, 8 in 2024-09-14T04:05:08.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:08.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:07 smithi039 ceph-mon[106470]: pgmap v488: 457 pgs: 457 active+clean; 3.6 MiB data, 550 MiB used, 715 GiB / 715 GiB avail; 4.5 KiB/s rd, 420 KiB/s wr, 7 op/s 2024-09-14T04:05:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.6", "id": [7, 5]}]': finished 2024-09-14T04:05:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.15", "id": [7, 4]}]': finished 2024-09-14T04:05:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.17", "id": [7, 0]}]': finished 2024-09-14T04:05:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.c", "id": [7, 1]}]': finished 2024-09-14T04:05:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.e", "id": [7, 0]}]': finished 2024-09-14T04:05:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.11", "id": [7, 0]}]': finished 2024-09-14T04:05:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.7", "id": [0, 2]}]': finished 2024-09-14T04:05:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.11", "id": [0, 2]}]': finished 2024-09-14T04:05:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.a", "id": [6, 4]}]': finished 2024-09-14T04:05:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.3", "id": [6, 5]}]': finished 2024-09-14T04:05:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:07 smithi039 ceph-mon[106470]: osdmap e654: 8 total, 8 up, 8 in 2024-09-14T04:05:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:08.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:07 smithi039 ceph-mon[111235]: pgmap v488: 457 pgs: 457 active+clean; 3.6 MiB data, 550 MiB used, 715 GiB / 715 GiB avail; 4.5 KiB/s rd, 420 KiB/s wr, 7 op/s 2024-09-14T04:05:08.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.6", "id": [7, 5]}]': finished 2024-09-14T04:05:08.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.15", "id": [7, 4]}]': finished 2024-09-14T04:05:08.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.17", "id": [7, 0]}]': finished 2024-09-14T04:05:08.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.c", "id": [7, 1]}]': finished 2024-09-14T04:05:08.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.e", "id": [7, 0]}]': finished 2024-09-14T04:05:08.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "22.11", "id": [7, 0]}]': finished 2024-09-14T04:05:08.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.7", "id": [0, 2]}]': finished 2024-09-14T04:05:08.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "25.11", "id": [0, 2]}]': finished 2024-09-14T04:05:08.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "28.a", "id": [6, 4]}]': finished 2024-09-14T04:05:08.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "29.3", "id": [6, 5]}]': finished 2024-09-14T04:05:08.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:07 smithi039 ceph-mon[111235]: osdmap e654: 8 total, 8 up, 8 in 2024-09-14T04:05:08.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:08.895 INFO:teuthology.orchestra.run.smithi039.stdout:wait for half of osds to upgrade 2024-09-14T04:05:08.932 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:05:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:05:08] "GET /metrics HTTP/1.1" 200 62951 "" "Prometheus/2.51.0" 2024-09-14T04:05:09.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:08 smithi129 ceph-mon[101316]: osdmap e655: 8 total, 8 up, 8 in 2024-09-14T04:05:09.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:08 smithi039 ceph-mon[111235]: osdmap e655: 8 total, 8 up, 8 in 2024-09-14T04:05:09.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:08 smithi039 ceph-mon[106470]: osdmap e655: 8 total, 8 up, 8 in 2024-09-14T04:05:09.459 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 --daemon-types osd --limit 4' 2024-09-14T04:05:09.671 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:05:10.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:09 smithi039 ceph-mon[106470]: pgmap v491: 457 pgs: 10 peering, 447 active+clean; 8.6 MiB data, 565 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 853 KiB/s wr, 2 op/s 2024-09-14T04:05:10.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:09 smithi039 ceph-mon[106470]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:05:10.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:09 smithi039 ceph-mon[111235]: pgmap v491: 457 pgs: 10 peering, 447 active+clean; 8.6 MiB data, 565 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 853 KiB/s wr, 2 op/s 2024-09-14T04:05:10.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:09 smithi039 ceph-mon[111235]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:05:10.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:09 smithi129 ceph-mon[101316]: pgmap v491: 457 pgs: 10 peering, 447 active+clean; 8.6 MiB data, 565 MiB used, 715 GiB / 715 GiB avail; 852 B/s rd, 853 KiB/s wr, 2 op/s 2024-09-14T04:05:10.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:09 smithi129 ceph-mon[101316]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:05:12.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:11 smithi129 ceph-mon[101316]: pgmap v492: 457 pgs: 10 peering, 447 active+clean; 8.6 MiB data, 565 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 853 KiB/s wr, 2 op/s 2024-09-14T04:05:12.280 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:11 smithi039 ceph-mon[106470]: pgmap v492: 457 pgs: 10 peering, 447 active+clean; 8.6 MiB data, 565 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 853 KiB/s wr, 2 op/s 2024-09-14T04:05:12.281 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:11 smithi039 ceph-mon[111235]: pgmap v492: 457 pgs: 10 peering, 447 active+clean; 8.6 MiB data, 565 MiB used, 715 GiB / 715 GiB avail; 853 B/s rd, 853 KiB/s wr, 2 op/s 2024-09-14T04:05:13.253 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:12 smithi039 ceph-mon[106470]: from='client.44384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "daemon_types": "osd", "limit": 4, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:05:13.253 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:12 smithi039 ceph-mon[111235]: from='client.44384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "daemon_types": "osd", "limit": 4, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:05:13.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:12 smithi129 ceph-mon[101316]: from='client.44384 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", "daemon_types": "osd", "limit": 4, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:05:13.578 INFO:teuthology.orchestra.run.smithi039.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T04:05:14.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:13 smithi129 ceph-mon[101316]: pgmap v493: 457 pgs: 457 active+clean; 14 MiB data, 580 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 1.4 MiB/s wr, 4 op/s; 4 B/s, 0 objects/s recovering 2024-09-14T04:05:14.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:13 smithi129 ceph-mon[101316]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-14T04:05:14.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:13 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:14.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:13 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:05:14.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:13 smithi039 ceph-mon[106470]: pgmap v493: 457 pgs: 457 active+clean; 14 MiB data, 580 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 1.4 MiB/s wr, 4 op/s; 4 B/s, 0 objects/s recovering 2024-09-14T04:05:14.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:13 smithi039 ceph-mon[106470]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-14T04:05:14.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:13 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:14.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:13 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:05:14.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:13 smithi039 ceph-mon[111235]: pgmap v493: 457 pgs: 457 active+clean; 14 MiB data, 580 MiB used, 715 GiB / 715 GiB avail; 2.0 KiB/s rd, 1.4 MiB/s wr, 4 op/s; 4 B/s, 0 objects/s recovering 2024-09-14T04:05:14.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:13 smithi039 ceph-mon[111235]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-14T04:05:14.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:13 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:14.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:13 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:05:14.384 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -e sha1=a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 -- bash -c 'while ceph orch upgrade status | jq '"'"'.in_progress'"'"' | grep true && ! ceph orch upgrade status | jq '"'"'.message'"'"' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; sleep 30 ; done' 2024-09-14T04:05:14.598 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:05:14.973 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:14 smithi039 ceph-mon[106470]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T04:05:15.245 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:14 smithi129 ceph-mon[101316]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T04:05:15.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:14 smithi039 ceph-mon[111235]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T04:05:15.611 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.RemoveFullTryNamespace (20022 ms) 2024-09-14T04:05:15.611 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.RemoveFullTryNamespaceDataPool 2024-09-14T04:05:16.162 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:15 smithi039 ceph-mon[106470]: pgmap v494: 457 pgs: 457 active+clean; 14 MiB data, 580 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1.3 MiB/s wr, 4 op/s; 4 B/s, 0 objects/s recovering 2024-09-14T04:05:16.162 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:15 smithi039 ceph-mon[106470]: pool 'test-librbd-smithi039-85658-26' is full (reached quota's max_bytes: 10 MiB) 2024-09-14T04:05:16.162 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:15 smithi039 ceph-mon[106470]: Health check update: 2 pool(s) full (POOL_FULL) 2024-09-14T04:05:16.162 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:15 smithi039 ceph-mon[106470]: osdmap e656: 8 total, 8 up, 8 in 2024-09-14T04:05:16.163 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:15 smithi039 ceph-mon[111235]: pgmap v494: 457 pgs: 457 active+clean; 14 MiB data, 580 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1.3 MiB/s wr, 4 op/s; 4 B/s, 0 objects/s recovering 2024-09-14T04:05:16.163 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:15 smithi039 ceph-mon[111235]: pool 'test-librbd-smithi039-85658-26' is full (reached quota's max_bytes: 10 MiB) 2024-09-14T04:05:16.163 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:15 smithi039 ceph-mon[111235]: Health check update: 2 pool(s) full (POOL_FULL) 2024-09-14T04:05:16.163 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:15 smithi039 ceph-mon[111235]: osdmap e656: 8 total, 8 up, 8 in 2024-09-14T04:05:16.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:15 smithi129 ceph-mon[101316]: pgmap v494: 457 pgs: 457 active+clean; 14 MiB data, 580 MiB used, 715 GiB / 715 GiB avail; 1.9 KiB/s rd, 1.3 MiB/s wr, 4 op/s; 4 B/s, 0 objects/s recovering 2024-09-14T04:05:16.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:15 smithi129 ceph-mon[101316]: pool 'test-librbd-smithi039-85658-26' is full (reached quota's max_bytes: 10 MiB) 2024-09-14T04:05:16.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:15 smithi129 ceph-mon[101316]: Health check update: 2 pool(s) full (POOL_FULL) 2024-09-14T04:05:16.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:15 smithi129 ceph-mon[101316]: osdmap e656: 8 total, 8 up, 8 in 2024-09-14T04:05:16.609 INFO:teuthology.orchestra.run.smithi039.stdout:true 2024-09-14T04:05:17.197 INFO:teuthology.orchestra.run.smithi039.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T04:05:17.197 INFO:teuthology.orchestra.run.smithi039.stdout:alertmanager.a smithi039 *:9093,9094 running (7m) 16s ago 13m 19.9M - 0.27.0 11f11916f8cd 6b564e1fc909 2024-09-14T04:05:17.197 INFO:teuthology.orchestra.run.smithi039.stdout:grafana.a smithi129 *:3000 running (7m) 64s ago 13m 79.9M - 10.4.0 c8b91775d855 f0492c19de92 2024-09-14T04:05:17.197 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi039.bbwiwf smithi039 running (12m) 16s ago 11m 25.0M - 18.2.4-845-ge4e227b3 67ddd33fa72f 91e6c8f85681 2024-09-14T04:05:17.197 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi129.ybjqcw smithi129 running (11m) 64s ago 11m 16.7M - 18.2.4-845-ge4e227b3 67ddd33fa72f a3fab5b37b71 2024-09-14T04:05:17.198 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.x smithi129 *:8443,9283,8765 running (8m) 64s ago 17m 456M - 19.3.0-4907-ga2782315 8801d2529e43 448bd4e146cb 2024-09-14T04:05:17.198 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.y smithi039 *:8443,9283,8765 running (8m) 16s ago 18m 528M - 19.3.0-4907-ga2782315 8801d2529e43 cccc87f635a6 2024-09-14T04:05:17.198 INFO:teuthology.orchestra.run.smithi039.stdout:mon.a smithi039 running (6m) 16s ago 19m 72.1M 2048M 19.3.0-4907-ga2782315 8801d2529e43 7198d4eb8e4c 2024-09-14T04:05:17.198 INFO:teuthology.orchestra.run.smithi039.stdout:mon.b smithi129 running (3m) 64s ago 17m 48.0M 2048M 19.3.0-4907-ga2782315 8801d2529e43 cff2519560f1 2024-09-14T04:05:17.198 INFO:teuthology.orchestra.run.smithi039.stdout:mon.c smithi039 running (5m) 16s ago 17m 57.1M 2048M 19.3.0-4907-ga2782315 8801d2529e43 4b8f0f3864b1 2024-09-14T04:05:17.198 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.a smithi039 *:9100 running (8m) 16s ago 12m 11.6M - 1.7.0 72c9c2088986 7dd5945adb64 2024-09-14T04:05:17.198 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.b smithi129 *:9100 running (8m) 64s ago 12m 10.7M - 1.7.0 72c9c2088986 1fedb5f257e7 2024-09-14T04:05:17.198 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0 smithi039 running (16m) 16s ago 16m 118M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 5048d74f27ac 2024-09-14T04:05:17.198 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1 smithi039 running (16m) 16s ago 16m 124M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 91349527d88b 2024-09-14T04:05:17.198 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2 smithi039 running (16m) 16s ago 16m 143M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f eb903b44fc5d 2024-09-14T04:05:17.198 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3 smithi039 running (15m) 16s ago 15m 140M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f dff9e665bd9b 2024-09-14T04:05:17.199 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4 smithi129 running (15m) 64s ago 15m 132M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 5b64727311b7 2024-09-14T04:05:17.199 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5 smithi129 running (14m) 64s ago 14m 117M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f f2d991a754f7 2024-09-14T04:05:17.199 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6 smithi129 running (14m) 64s ago 14m 138M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 3060eaf44106 2024-09-14T04:05:17.199 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7 smithi129 running (14m) 64s ago 14m 119M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 4c13b97060d1 2024-09-14T04:05:17.199 INFO:teuthology.orchestra.run.smithi039.stdout:prometheus.a smithi129 *:9095 running (7m) 64s ago 13m 58.0M - 2.51.0 1d3b7f56885b cbe583f852c3 2024-09-14T04:05:17.578 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:05:17.578 INFO:teuthology.orchestra.run.smithi039.stdout: "mon": { 2024-09-14T04:05:17.578 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T04:05:17.578 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:05:17.579 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr": { 2024-09-14T04:05:17.579 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T04:05:17.579 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:05:17.579 INFO:teuthology.orchestra.run.smithi039.stdout: "osd": { 2024-09-14T04:05:17.579 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8 2024-09-14T04:05:17.579 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:05:17.579 INFO:teuthology.orchestra.run.smithi039.stdout: "mds": { 2024-09-14T04:05:17.579 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T04:05:17.579 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:05:17.579 INFO:teuthology.orchestra.run.smithi039.stdout: "overall": { 2024-09-14T04:05:17.579 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 10, 2024-09-14T04:05:17.579 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 5 2024-09-14T04:05:17.579 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:05:17.579 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:05:17.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:17 smithi129 ceph-mon[101316]: osdmap e657: 8 total, 8 up, 8 in 2024-09-14T04:05:17.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:17 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/896590454' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-27","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:17.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:17 smithi129 ceph-mon[101316]: from='client.44396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:05:17.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:17 smithi129 ceph-mon[101316]: pgmap v497: 489 pgs: 32 unknown, 457 active+clean; 14 MiB data, 580 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 640 KiB/s wr, 2 op/s; 4 B/s, 0 objects/s recovering 2024-09-14T04:05:17.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:17 smithi129 ceph-mon[101316]: from='client.34473 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:05:17.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:17 smithi039 ceph-mon[106470]: osdmap e657: 8 total, 8 up, 8 in 2024-09-14T04:05:17.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:17 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/896590454' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-27","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:17.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:17 smithi039 ceph-mon[106470]: from='client.44396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:05:17.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:17 smithi039 ceph-mon[106470]: pgmap v497: 489 pgs: 32 unknown, 457 active+clean; 14 MiB data, 580 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 640 KiB/s wr, 2 op/s; 4 B/s, 0 objects/s recovering 2024-09-14T04:05:17.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:17 smithi039 ceph-mon[106470]: from='client.34473 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:05:17.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:17 smithi039 ceph-mon[111235]: osdmap e657: 8 total, 8 up, 8 in 2024-09-14T04:05:17.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:17 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/896590454' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-27","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:17.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:17 smithi039 ceph-mon[111235]: from='client.44396 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:05:17.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:17 smithi039 ceph-mon[111235]: pgmap v497: 489 pgs: 32 unknown, 457 active+clean; 14 MiB data, 580 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 640 KiB/s wr, 2 op/s; 4 B/s, 0 objects/s recovering 2024-09-14T04:05:17.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:17 smithi039 ceph-mon[111235]: from='client.34473 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:05:17.879 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:05:17.880 INFO:teuthology.orchestra.run.smithi039.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8", 2024-09-14T04:05:17.880 INFO:teuthology.orchestra.run.smithi039.stdout: "in_progress": true, 2024-09-14T04:05:17.880 INFO:teuthology.orchestra.run.smithi039.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (4 remaining).", 2024-09-14T04:05:17.880 INFO:teuthology.orchestra.run.smithi039.stdout: "services_complete": [], 2024-09-14T04:05:17.880 INFO:teuthology.orchestra.run.smithi039.stdout: "progress": "", 2024-09-14T04:05:17.880 INFO:teuthology.orchestra.run.smithi039.stdout: "message": "Doing first pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 image", 2024-09-14T04:05:17.880 INFO:teuthology.orchestra.run.smithi039.stdout: "is_paused": false 2024-09-14T04:05:17.880 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:05:18.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:18 smithi129 ceph-mon[101316]: from='client.34479 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:05:18.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:18 smithi129 ceph-mon[101316]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:05:18.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:18 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/896590454' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:18.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:18 smithi129 ceph-mon[101316]: osdmap e658: 8 total, 8 up, 8 in 2024-09-14T04:05:18.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:18 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3141664073' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:05:18.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:18 smithi129 ceph-mon[101316]: from='client.34497 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:05:18.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:18 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:18.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:18 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:18.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:18 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:05:18.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:18 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:05:18.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:18 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:18.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:05:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:05:18] "GET /metrics HTTP/1.1" 200 62951 "" "Prometheus/2.51.0" 2024-09-14T04:05:18.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:18 smithi039 ceph-mon[106470]: from='client.34479 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:05:18.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:18 smithi039 ceph-mon[106470]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:05:18.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:18 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/896590454' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:18.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:18 smithi039 ceph-mon[106470]: osdmap e658: 8 total, 8 up, 8 in 2024-09-14T04:05:18.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:18 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3141664073' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:05:18.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:18 smithi039 ceph-mon[106470]: from='client.34497 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:05:18.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:18 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:18.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:18 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:18.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:18 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:05:18.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:18 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:05:18.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:18 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:18.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:18 smithi039 ceph-mon[111235]: from='client.34479 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:05:18.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:18 smithi039 ceph-mon[111235]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:05:18.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:18 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/896590454' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-27","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:18.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:18 smithi039 ceph-mon[111235]: osdmap e658: 8 total, 8 up, 8 in 2024-09-14T04:05:18.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:18 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3141664073' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:05:18.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:18 smithi039 ceph-mon[111235]: from='client.34497 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:05:18.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:18 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:18.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:18 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:18.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:18 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:05:18.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:18 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:05:18.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:18 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:19.523 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:05:19.626 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:05:19 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:05:19] ENGINE Bus STOPPING 2024-09-14T04:05:19.626 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:19 smithi039 ceph-mon[111235]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T04:05:19.626 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:19 smithi039 ceph-mon[111235]: osdmap e659: 8 total, 8 up, 8 in 2024-09-14T04:05:19.626 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:19 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3072842303' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-28","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:19.626 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:19 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-28","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:19.626 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:19 smithi039 ceph-mon[111235]: pgmap v500: 521 pgs: 32 unknown, 489 active+clean; 584 KiB data, 544 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 514 KiB/s wr, 39 op/s 2024-09-14T04:05:19.627 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:19 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:19.627 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:19 smithi039 ceph-mon[111235]: osdmap e660: 8 total, 8 up, 8 in 2024-09-14T04:05:19.627 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:19 smithi039 ceph-mon[106470]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T04:05:19.627 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:19 smithi039 ceph-mon[106470]: osdmap e659: 8 total, 8 up, 8 in 2024-09-14T04:05:19.627 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:19 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3072842303' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-28","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:19.627 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:19 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-28","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:19.627 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:19 smithi039 ceph-mon[106470]: pgmap v500: 521 pgs: 32 unknown, 489 active+clean; 584 KiB data, 544 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 514 KiB/s wr, 39 op/s 2024-09-14T04:05:19.627 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:19 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:19.627 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:19 smithi039 ceph-mon[106470]: osdmap e660: 8 total, 8 up, 8 in 2024-09-14T04:05:19.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:19 smithi129 ceph-mon[101316]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:a2782315255a3e7d5f6cdc9eb35785187ff6ddf8 2024-09-14T04:05:19.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:19 smithi129 ceph-mon[101316]: osdmap e659: 8 total, 8 up, 8 in 2024-09-14T04:05:19.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:19 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3072842303' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-28","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:19.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:19 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-28","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:05:19.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:19 smithi129 ceph-mon[101316]: pgmap v500: 521 pgs: 32 unknown, 489 active+clean; 584 KiB data, 544 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 514 KiB/s wr, 39 op/s 2024-09-14T04:05:19.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:19 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-28","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:05:19.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:19 smithi129 ceph-mon[101316]: osdmap e660: 8 total, 8 up, 8 in 2024-09-14T04:05:20.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:05:19 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:05:19] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T04:05:20.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:05:19 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:05:19] ENGINE Bus STOPPED 2024-09-14T04:05:20.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:05:19 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:05:19] ENGINE Bus STARTING 2024-09-14T04:05:20.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:05:19 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:05:19] ENGINE Serving on http://:::9283 2024-09-14T04:05:20.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:05:19 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:05:19] ENGINE Bus STARTED 2024-09-14T04:05:20.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:20 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' 2024-09-14T04:05:20.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:20 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-14T04:05:20.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:20 smithi129 ceph-mon[101316]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T04:05:20.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:20 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:20.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:20 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:05:20.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:20 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:05:20.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:20 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:20.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:20 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:05:20.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:20 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:20.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:20 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:05:20.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:20 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:20.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:20 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-14T04:05:20.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:20 smithi129 ceph-mon[101316]: pool 'test-librbd-smithi039-85658-26' no longer out of quota; removing NO_QUOTA flag 2024-09-14T04:05:20.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:20 smithi129 ceph-mon[101316]: Health check update: 1 pool(s) full (POOL_FULL) 2024-09-14T04:05:20.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:20 smithi129 ceph-mon[101316]: osdmap e661: 8 total, 8 up, 8 in 2024-09-14T04:05:20.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T04:05:20.432+0000 7f268b1da640 -1 mgr.server reply reply (11) Resource temporarily unavailable 32 pgs have unknown state; cannot draw any conclusions 2024-09-14T04:05:20.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T04:05:20.432+0000 7f268b1da640 -1 mgr.server reply reply (16) Device or resource busy 32 pgs have unknown state; cannot draw any conclusionsunsafe to stop osd(s) at this time (0 PGs are or would become offline) 2024-09-14T04:05:20.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' 2024-09-14T04:05:20.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-14T04:05:20.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-mon[111235]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T04:05:20.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:20.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:05:20.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:05:20.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:20.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:05:20.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:20.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:05:20.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:20.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-14T04:05:20.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-mon[111235]: pool 'test-librbd-smithi039-85658-26' no longer out of quota; removing NO_QUOTA flag 2024-09-14T04:05:20.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-mon[111235]: Health check update: 1 pool(s) full (POOL_FULL) 2024-09-14T04:05:20.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-mon[111235]: osdmap e661: 8 total, 8 up, 8 in 2024-09-14T04:05:20.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' 2024-09-14T04:05:20.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-14T04:05:20.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-mon[106470]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T04:05:20.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:20.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:05:20.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:05:20.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:20.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:05:20.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:20.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:05:20.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:20.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-14T04:05:20.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-mon[106470]: pool 'test-librbd-smithi039-85658-26' no longer out of quota; removing NO_QUOTA flag 2024-09-14T04:05:20.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-mon[106470]: Health check update: 1 pool(s) full (POOL_FULL) 2024-09-14T04:05:20.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:20 smithi039 ceph-mon[106470]: osdmap e661: 8 total, 8 up, 8 in 2024-09-14T04:05:21.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:21 smithi129 ceph-mon[101316]: Upgrade: Target is version 19.3.0-4907-ga2782315 (squid) 2024-09-14T04:05:21.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:21 smithi129 ceph-mon[101316]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T04:05:21.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:21 smithi129 ceph-mon[101316]: Upgrade: Setting container_image for all mgr 2024-09-14T04:05:21.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:21 smithi129 ceph-mon[101316]: Upgrade: Setting container_image for all mon 2024-09-14T04:05:21.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:21 smithi129 ceph-mon[101316]: Upgrade: Setting container_image for all crash 2024-09-14T04:05:21.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:21 smithi129 ceph-mon[101316]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-14T04:05:21.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:21 smithi129 ceph-mon[101316]: Upgrade: 32 pgs have unknown state; cannot draw any conclusions 2024-09-14T04:05:21.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:21 smithi129 ceph-mon[101316]: pgmap v503: 521 pgs: 32 unknown, 489 active+clean; 584 KiB data, 544 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 514 KiB/s wr, 39 op/s 2024-09-14T04:05:21.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:21 smithi129 ceph-mon[101316]: osdmap e662: 8 total, 8 up, 8 in 2024-09-14T04:05:21.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:21 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-28", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-14T04:05:21.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:21 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-28", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-14T04:05:21.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:21 smithi039 ceph-mon[111235]: Upgrade: Target is version 19.3.0-4907-ga2782315 (squid) 2024-09-14T04:05:21.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:21 smithi039 ceph-mon[111235]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T04:05:21.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:21 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all mgr 2024-09-14T04:05:21.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:21 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all mon 2024-09-14T04:05:21.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:21 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all crash 2024-09-14T04:05:21.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:21 smithi039 ceph-mon[111235]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-14T04:05:21.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:21 smithi039 ceph-mon[111235]: Upgrade: 32 pgs have unknown state; cannot draw any conclusions 2024-09-14T04:05:21.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:21 smithi039 ceph-mon[111235]: pgmap v503: 521 pgs: 32 unknown, 489 active+clean; 584 KiB data, 544 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 514 KiB/s wr, 39 op/s 2024-09-14T04:05:21.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:21 smithi039 ceph-mon[111235]: osdmap e662: 8 total, 8 up, 8 in 2024-09-14T04:05:21.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:21 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-28", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-14T04:05:21.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:21 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-28", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-14T04:05:21.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:21 smithi039 ceph-mon[106470]: Upgrade: Target is version 19.3.0-4907-ga2782315 (squid) 2024-09-14T04:05:21.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:21 smithi039 ceph-mon[106470]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, digests ['quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802'] 2024-09-14T04:05:21.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:21 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all mgr 2024-09-14T04:05:21.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:21 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all mon 2024-09-14T04:05:21.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:21 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all crash 2024-09-14T04:05:21.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:21 smithi039 ceph-mon[106470]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-14T04:05:21.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:21 smithi039 ceph-mon[106470]: Upgrade: 32 pgs have unknown state; cannot draw any conclusions 2024-09-14T04:05:21.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:21 smithi039 ceph-mon[106470]: pgmap v503: 521 pgs: 32 unknown, 489 active+clean; 584 KiB data, 544 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 514 KiB/s wr, 39 op/s 2024-09-14T04:05:21.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:21 smithi039 ceph-mon[106470]: osdmap e662: 8 total, 8 up, 8 in 2024-09-14T04:05:21.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:21 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-28", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-14T04:05:21.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:21 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-28", "field": "max_bytes", "val": "10485760"} : dispatch 2024-09-14T04:05:22.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:22 smithi129 ceph-mon[101316]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:05:22.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:22.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:22 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-28", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-14T04:05:22.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:22 smithi129 ceph-mon[101316]: osdmap e663: 8 total, 8 up, 8 in 2024-09-14T04:05:22.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:22 smithi039 ceph-mon[106470]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:05:22.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:22.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:22 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-28", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-14T04:05:22.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:22 smithi039 ceph-mon[106470]: osdmap e663: 8 total, 8 up, 8 in 2024-09-14T04:05:22.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:22 smithi039 ceph-mon[111235]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:05:22.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:22.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:22 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-28", "field": "max_bytes", "val": "10485760"}]': finished 2024-09-14T04:05:22.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:22 smithi039 ceph-mon[111235]: osdmap e663: 8 total, 8 up, 8 in 2024-09-14T04:05:23.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:23 smithi039 ceph-mon[106470]: pgmap v506: 521 pgs: 521 active+clean; 1.6 MiB data, 546 MiB used, 715 GiB / 715 GiB avail; 8.7 KiB/s rd, 259 KiB/s wr, 15 op/s 2024-09-14T04:05:23.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:23 smithi039 ceph-mon[111235]: pgmap v506: 521 pgs: 521 active+clean; 1.6 MiB data, 546 MiB used, 715 GiB / 715 GiB avail; 8.7 KiB/s rd, 259 KiB/s wr, 15 op/s 2024-09-14T04:05:24.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:23 smithi129 ceph-mon[101316]: pgmap v506: 521 pgs: 521 active+clean; 1.6 MiB data, 546 MiB used, 715 GiB / 715 GiB avail; 8.7 KiB/s rd, 259 KiB/s wr, 15 op/s 2024-09-14T04:05:26.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:25 smithi129 ceph-mon[101316]: pgmap v507: 521 pgs: 521 active+clean; 1.6 MiB data, 546 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 193 KiB/s wr, 11 op/s 2024-09-14T04:05:26.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:25 smithi129 ceph-mon[101316]: osdmap e664: 8 total, 8 up, 8 in 2024-09-14T04:05:26.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:25 smithi039 ceph-mon[106470]: pgmap v507: 521 pgs: 521 active+clean; 1.6 MiB data, 546 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 193 KiB/s wr, 11 op/s 2024-09-14T04:05:26.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:25 smithi039 ceph-mon[106470]: osdmap e664: 8 total, 8 up, 8 in 2024-09-14T04:05:26.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:25 smithi039 ceph-mon[111235]: pgmap v507: 521 pgs: 521 active+clean; 1.6 MiB data, 546 MiB used, 715 GiB / 715 GiB avail; 6.5 KiB/s rd, 193 KiB/s wr, 11 op/s 2024-09-14T04:05:26.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:25 smithi039 ceph-mon[111235]: osdmap e664: 8 total, 8 up, 8 in 2024-09-14T04:05:28.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:27 smithi129 ceph-mon[101316]: pgmap v509: 521 pgs: 521 active+clean; 1.6 MiB data, 546 MiB used, 715 GiB / 715 GiB avail; 5.8 KiB/s rd, 172 KiB/s wr, 10 op/s 2024-09-14T04:05:28.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:27 smithi039 ceph-mon[106470]: pgmap v509: 521 pgs: 521 active+clean; 1.6 MiB data, 546 MiB used, 715 GiB / 715 GiB avail; 5.8 KiB/s rd, 172 KiB/s wr, 10 op/s 2024-09-14T04:05:28.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:27 smithi039 ceph-mon[111235]: pgmap v509: 521 pgs: 521 active+clean; 1.6 MiB data, 546 MiB used, 715 GiB / 715 GiB avail; 5.8 KiB/s rd, 172 KiB/s wr, 10 op/s 2024-09-14T04:05:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:05:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:05:28] "GET /metrics HTTP/1.1" 200 67670 "" "Prometheus/2.51.0" 2024-09-14T04:05:30.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:29 smithi129 ceph-mon[101316]: pgmap v510: 521 pgs: 521 active+clean; 6.6 MiB data, 561 MiB used, 715 GiB / 715 GiB avail; 5.4 KiB/s rd, 836 KiB/s wr, 10 op/s 2024-09-14T04:05:30.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:29 smithi039 ceph-mon[106470]: pgmap v510: 521 pgs: 521 active+clean; 6.6 MiB data, 561 MiB used, 715 GiB / 715 GiB avail; 5.4 KiB/s rd, 836 KiB/s wr, 10 op/s 2024-09-14T04:05:30.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:29 smithi039 ceph-mon[111235]: pgmap v510: 521 pgs: 521 active+clean; 6.6 MiB data, 561 MiB used, 715 GiB / 715 GiB avail; 5.4 KiB/s rd, 836 KiB/s wr, 10 op/s 2024-09-14T04:05:32.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:31 smithi129 ceph-mon[101316]: pgmap v511: 521 pgs: 521 active+clean; 6.6 MiB data, 561 MiB used, 715 GiB / 715 GiB avail; 4.8 KiB/s rd, 736 KiB/s wr, 9 op/s 2024-09-14T04:05:32.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:31 smithi039 ceph-mon[106470]: pgmap v511: 521 pgs: 521 active+clean; 6.6 MiB data, 561 MiB used, 715 GiB / 715 GiB avail; 4.8 KiB/s rd, 736 KiB/s wr, 9 op/s 2024-09-14T04:05:32.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:31 smithi039 ceph-mon[111235]: pgmap v511: 521 pgs: 521 active+clean; 6.6 MiB data, 561 MiB used, 715 GiB / 715 GiB avail; 4.8 KiB/s rd, 736 KiB/s wr, 9 op/s 2024-09-14T04:05:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:33 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:33 smithi129 ceph-mon[101316]: pgmap v512: 521 pgs: 521 active+clean; 12 MiB data, 576 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1024 KiB/s wr, 2 op/s 2024-09-14T04:05:33.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:33 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:33.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:33 smithi039 ceph-mon[106470]: pgmap v512: 521 pgs: 521 active+clean; 12 MiB data, 576 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1024 KiB/s wr, 2 op/s 2024-09-14T04:05:33.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:33 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:33.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:33 smithi039 ceph-mon[111235]: pgmap v512: 521 pgs: 521 active+clean; 12 MiB data, 576 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1024 KiB/s wr, 2 op/s 2024-09-14T04:05:36.250 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:35 smithi039 ceph-mon[106470]: pgmap v513: 521 pgs: 521 active+clean; 12 MiB data, 576 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1024 KiB/s wr, 2 op/s 2024-09-14T04:05:36.250 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:35 smithi039 ceph-mon[106470]: pool 'test-librbd-smithi039-85658-28' is full (reached quota's max_bytes: 10 MiB) 2024-09-14T04:05:36.250 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:35 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-14T04:05:36.250 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:35 smithi039 ceph-mon[106470]: Health check update: 2 pool(s) full (POOL_FULL) 2024-09-14T04:05:36.250 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:35 smithi039 ceph-mon[106470]: osdmap e665: 8 total, 8 up, 8 in 2024-09-14T04:05:36.250 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:35 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-27", "field": "max_objects", "val": "7"} : dispatch 2024-09-14T04:05:36.250 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:35 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-27", "field": "max_objects", "val": "7"} : dispatch 2024-09-14T04:05:36.251 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:35 smithi039 ceph-mon[111235]: pgmap v513: 521 pgs: 521 active+clean; 12 MiB data, 576 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1024 KiB/s wr, 2 op/s 2024-09-14T04:05:36.251 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:35 smithi039 ceph-mon[111235]: pool 'test-librbd-smithi039-85658-28' is full (reached quota's max_bytes: 10 MiB) 2024-09-14T04:05:36.251 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:35 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-14T04:05:36.251 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:35 smithi039 ceph-mon[111235]: Health check update: 2 pool(s) full (POOL_FULL) 2024-09-14T04:05:36.251 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:35 smithi039 ceph-mon[111235]: osdmap e665: 8 total, 8 up, 8 in 2024-09-14T04:05:36.251 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:35 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-27", "field": "max_objects", "val": "7"} : dispatch 2024-09-14T04:05:36.251 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:35 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-27", "field": "max_objects", "val": "7"} : dispatch 2024-09-14T04:05:36.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:35 smithi129 ceph-mon[101316]: pgmap v513: 521 pgs: 521 active+clean; 12 MiB data, 576 MiB used, 715 GiB / 715 GiB avail; 1023 B/s rd, 1024 KiB/s wr, 2 op/s 2024-09-14T04:05:36.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:35 smithi129 ceph-mon[101316]: pool 'test-librbd-smithi039-85658-28' is full (reached quota's max_bytes: 10 MiB) 2024-09-14T04:05:36.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:35 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16} : dispatch 2024-09-14T04:05:36.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:35 smithi129 ceph-mon[101316]: Health check update: 2 pool(s) full (POOL_FULL) 2024-09-14T04:05:36.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:35 smithi129 ceph-mon[101316]: osdmap e665: 8 total, 8 up, 8 in 2024-09-14T04:05:36.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:35 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-27", "field": "max_objects", "val": "7"} : dispatch 2024-09-14T04:05:36.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:35 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-27", "field": "max_objects", "val": "7"} : dispatch 2024-09-14T04:05:36.478 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:05:36.979 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:36 smithi039 ceph-mon[106470]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-14T04:05:36.979 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:36 smithi039 ceph-mon[106470]: Upgrade: osd.0 is safe to restart 2024-09-14T04:05:36.979 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:36 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-27", "field": "max_objects", "val": "7"}]': finished 2024-09-14T04:05:36.979 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:36 smithi039 ceph-mon[106470]: osdmap e666: 8 total, 8 up, 8 in 2024-09-14T04:05:36.979 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:36 smithi039 ceph-mon[111235]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-14T04:05:36.980 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:36 smithi039 ceph-mon[111235]: Upgrade: osd.0 is safe to restart 2024-09-14T04:05:36.980 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:36 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-27", "field": "max_objects", "val": "7"}]': finished 2024-09-14T04:05:36.980 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:36 smithi039 ceph-mon[111235]: osdmap e666: 8 total, 8 up, 8 in 2024-09-14T04:05:37.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:36 smithi129 ceph-mon[101316]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-09-14T04:05:37.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:36 smithi129 ceph-mon[101316]: Upgrade: osd.0 is safe to restart 2024-09-14T04:05:37.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:36 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set-quota", "pool": "test-librbd-smithi039-85658-27", "field": "max_objects", "val": "7"}]': finished 2024-09-14T04:05:37.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:36 smithi129 ceph-mon[101316]: osdmap e666: 8 total, 8 up, 8 in 2024-09-14T04:05:37.488 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:05:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:38 smithi129 ceph-mon[101316]: pgmap v516: 521 pgs: 521 active+clean; 12 MiB data, 576 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 640 KiB/s wr, 1 op/s 2024-09-14T04:05:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:38 smithi129 ceph-mon[101316]: Upgrade: Updating osd.0 2024-09-14T04:05:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:38 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:38 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-14T04:05:38.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:38 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:05:38.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:38 smithi129 ceph-mon[101316]: Deploying daemon osd.0 on smithi039 2024-09-14T04:05:38.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:38 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:38.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:38 smithi039 ceph-mon[106470]: pgmap v516: 521 pgs: 521 active+clean; 12 MiB data, 576 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 640 KiB/s wr, 1 op/s 2024-09-14T04:05:38.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:38 smithi039 ceph-mon[106470]: Upgrade: Updating osd.0 2024-09-14T04:05:38.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:38 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:38.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:38 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-14T04:05:38.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:38 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:05:38.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:38 smithi039 ceph-mon[106470]: Deploying daemon osd.0 on smithi039 2024-09-14T04:05:38.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:38 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:38.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:38 smithi039 ceph-mon[111235]: pgmap v516: 521 pgs: 521 active+clean; 12 MiB data, 576 MiB used, 715 GiB / 715 GiB avail; 639 B/s rd, 640 KiB/s wr, 1 op/s 2024-09-14T04:05:38.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:38 smithi039 ceph-mon[111235]: Upgrade: Updating osd.0 2024-09-14T04:05:38.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:38 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:38.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:38 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.0"} : dispatch 2024-09-14T04:05:38.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:38 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:05:38.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:38 smithi039 ceph-mon[111235]: Deploying daemon osd.0 on smithi039 2024-09-14T04:05:38.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:38 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:38.498 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:05:38.970 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:05:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:05:38] "GET /metrics HTTP/1.1" 200 67756 "" "Prometheus/2.51.0" 2024-09-14T04:05:39.507 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:05:40.276 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:40 smithi039 ceph-mon[106470]: pgmap v517: 521 pgs: 521 active+clean; 14 MiB data, 585 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 897 KiB/s wr, 6 op/s 2024-09-14T04:05:40.276 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:40 smithi039 ceph-mon[111235]: pgmap v517: 521 pgs: 521 active+clean; 14 MiB data, 585 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 897 KiB/s wr, 6 op/s 2024-09-14T04:05:40.277 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:40 smithi039 systemd[1]: Stopping Ceph osd.0 for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T04:05:40.517 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:05:40.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:40 smithi129 ceph-mon[101316]: pgmap v517: 521 pgs: 521 active+clean; 14 MiB data, 585 MiB used, 715 GiB / 715 GiB avail; 3.0 KiB/s rd, 897 KiB/s wr, 6 op/s 2024-09-14T04:05:40.540 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:40 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0[45363]: 2024-09-14T04:05:40.274+0000 7f6a35f6d640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:05:40.540 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:40 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0[45363]: 2024-09-14T04:05:40.274+0000 7f6a35f6d640 -1 osd.0 666 *** Got signal Terminated *** 2024-09-14T04:05:40.540 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:40 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0[45363]: 2024-09-14T04:05:40.274+0000 7f6a35f6d640 -1 osd.0 666 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T04:05:41.290 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:41 smithi039 podman[123575]: 2024-09-14 04:05:41.011529184 +0000 UTC m=+0.852024094 container died 5048d74f27acab67c717c4d6250daca9f8cbe8b14c9fdac95c26bf3d10c9c3c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0, RELEASE=reef-e4e227b, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-14T04:05:41.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:41 smithi039 ceph-mon[106470]: osd.0 marked itself down and dead 2024-09-14T04:05:41.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:41 smithi039 ceph-mon[106470]: pool 'test-librbd-smithi039-85658-27' is full (reached quota's max_objects: 7) 2024-09-14T04:05:41.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:41 smithi039 ceph-mon[106470]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T04:05:41.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:41 smithi039 ceph-mon[106470]: Health check update: 3 pool(s) full (POOL_FULL) 2024-09-14T04:05:41.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:41 smithi039 ceph-mon[106470]: osdmap e667: 8 total, 7 up, 8 in 2024-09-14T04:05:41.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:41 smithi039 ceph-mon[111235]: osd.0 marked itself down and dead 2024-09-14T04:05:41.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:41 smithi039 ceph-mon[111235]: pool 'test-librbd-smithi039-85658-27' is full (reached quota's max_objects: 7) 2024-09-14T04:05:41.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:41 smithi039 ceph-mon[111235]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T04:05:41.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:41 smithi039 ceph-mon[111235]: Health check update: 3 pool(s) full (POOL_FULL) 2024-09-14T04:05:41.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:41 smithi039 ceph-mon[111235]: osdmap e667: 8 total, 7 up, 8 in 2024-09-14T04:05:41.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:41 smithi129 ceph-mon[101316]: osd.0 marked itself down and dead 2024-09-14T04:05:41.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:41 smithi129 ceph-mon[101316]: pool 'test-librbd-smithi039-85658-27' is full (reached quota's max_objects: 7) 2024-09-14T04:05:41.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:41 smithi129 ceph-mon[101316]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T04:05:41.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:41 smithi129 ceph-mon[101316]: Health check update: 3 pool(s) full (POOL_FULL) 2024-09-14T04:05:41.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:41 smithi129 ceph-mon[101316]: osdmap e667: 8 total, 7 up, 8 in 2024-09-14T04:05:41.584 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.RemoveFullTryNamespaceDataPool (25973 ms) 2024-09-14T04:05:41.584 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestIO 2024-09-14T04:05:41.584 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:05:41.790 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:41 smithi039 podman[123575]: 2024-09-14 04:05:41.547675256 +0000 UTC m=+1.388170165 container cleanup 5048d74f27acab67c717c4d6250daca9f8cbe8b14c9fdac95c26bf3d10c9c3c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0, RELEASE=reef-e4e227b, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T04:05:41.790 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:41 smithi039 bash[123575]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0 2024-09-14T04:05:42.268 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:42 smithi039 ceph-mon[106470]: pgmap v519: 521 pgs: 61 stale+active+clean, 460 active+clean; 14 MiB data, 585 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 343 KiB/s wr, 6 op/s 2024-09-14T04:05:42.268 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:42 smithi039 ceph-mon[106470]: osdmap e668: 8 total, 7 up, 8 in 2024-09-14T04:05:42.268 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:42 smithi039 ceph-mon[111235]: pgmap v519: 521 pgs: 61 stale+active+clean, 460 active+clean; 14 MiB data, 585 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 343 KiB/s wr, 6 op/s 2024-09-14T04:05:42.268 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:42 smithi039 ceph-mon[111235]: osdmap e668: 8 total, 7 up, 8 in 2024-09-14T04:05:42.268 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:41 smithi039 podman[123596]: 2024-09-14 04:05:41.791087935 +0000 UTC m=+0.777357554 container remove 5048d74f27acab67c717c4d6250daca9f8cbe8b14c9fdac95c26bf3d10c9c3c8 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=reef-e4e227b, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD) 2024-09-14T04:05:42.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:42 smithi129 ceph-mon[101316]: pgmap v519: 521 pgs: 61 stale+active+clean, 460 active+clean; 14 MiB data, 585 MiB used, 715 GiB / 715 GiB avail; 3.2 KiB/s rd, 343 KiB/s wr, 6 op/s 2024-09-14T04:05:42.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:42 smithi129 ceph-mon[101316]: osdmap e668: 8 total, 7 up, 8 in 2024-09-14T04:05:42.540 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:42 smithi039 podman[123690]: 2024-09-14 04:05:42.170300147 +0000 UTC m=+0.018152266 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:05:42.540 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:42 smithi039 podman[123690]: 2024-09-14 04:05:42.309148841 +0000 UTC m=+0.157000957 container create d7c6d6b7fce8657982b0c115d29edf82e17a18fc0821cd8ecf2b4a847c9e4b78 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T04:05:43.040 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:42 smithi039 podman[123690]: 2024-09-14 04:05:42.604192398 +0000 UTC m=+0.452044515 container init d7c6d6b7fce8657982b0c115d29edf82e17a18fc0821cd8ecf2b4a847c9e4b78 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=main-a278231, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T04:05:43.040 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:42 smithi039 podman[123690]: 2024-09-14 04:05:42.609418235 +0000 UTC m=+0.457270356 container start d7c6d6b7fce8657982b0c115d29edf82e17a18fc0821cd8ecf2b4a847c9e4b78 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.license=GPLv2, RELEASE=main-a278231, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T04:05:43.041 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:42 smithi039 podman[123690]: 2024-09-14 04:05:42.712554279 +0000 UTC m=+0.560406410 container attach d7c6d6b7fce8657982b0c115d29edf82e17a18fc0821cd8ecf2b4a847c9e4b78 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=main-a278231, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-14T04:05:43.041 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:42 smithi039 podman[123690]: 2024-09-14 04:05:42.827110227 +0000 UTC m=+0.674962357 container died d7c6d6b7fce8657982b0c115d29edf82e17a18fc0821cd8ecf2b4a847c9e4b78 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0-deactivate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=main-a278231, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-09-14T04:05:43.230 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:05:43.231 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:05:43.231 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:05:43.231 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:05:43.231 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:05:43.231 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:43.231 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:05:43.231 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:43.231 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:43.231 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:05:43.231 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:43.231 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:05:43.231 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:43.231 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:43.231 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:05:43.232 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:43.232 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:05:43.232 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:43.232 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:43.232 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:05:43.232 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:43.232 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:05:43.232 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:43.232 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:43.232 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:05:43.232 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:43.232 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:05:43.232 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:43.233 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:43.233 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:05:43.233 INFO:tasks.workunit.client.0.smithi039.stdout:start compare and write 2024-09-14T04:05:43.233 INFO:tasks.workunit.client.0.smithi039.stdout:compare and wrote: 512 2024-09-14T04:05:43.233 INFO:tasks.workunit.client.0.smithi039.stdout:start compare and write 2024-09-14T04:05:43.233 INFO:tasks.workunit.client.0.smithi039.stdout:compare and wrote: 512 2024-09-14T04:05:43.233 INFO:tasks.workunit.client.0.smithi039.stdout:start compare and write 2024-09-14T04:05:43.233 INFO:tasks.workunit.client.0.smithi039.stdout:compare and wrote: 512 2024-09-14T04:05:43.233 INFO:tasks.workunit.client.0.smithi039.stdout:start compare and write 2024-09-14T04:05:43.233 INFO:tasks.workunit.client.0.smithi039.stdout:compare and wrote: 512 2024-09-14T04:05:43.233 INFO:tasks.workunit.client.0.smithi039.stdout:start compare and write 2024-09-14T04:05:43.233 INFO:tasks.workunit.client.0.smithi039.stdout:compare and wrote: 512 2024-09-14T04:05:43.233 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:43.233 INFO:tasks.workunit.client.0.smithi039.stdout:started aio compare and write 2024-09-14T04:05:43.234 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:43.234 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:43.234 INFO:tasks.workunit.client.0.smithi039.stdout:finished aio compare and write 2024-09-14T04:05:43.234 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:43.234 INFO:tasks.workunit.client.0.smithi039.stdout:started aio compare and write 2024-09-14T04:05:43.234 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:43.234 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:43.234 INFO:tasks.workunit.client.0.smithi039.stdout:finished aio compare and write 2024-09-14T04:05:43.234 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:43.234 INFO:tasks.workunit.client.0.smithi039.stdout:started aio compare and write 2024-09-14T04:05:43.235 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:43.235 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:43.235 INFO:tasks.workunit.client.0.smithi039.stdout:finished aio compare and write 2024-09-14T04:05:43.235 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:43.235 INFO:tasks.workunit.client.0.smithi039.stdout:started aio compare and write 2024-09-14T04:05:43.235 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:43.235 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:43.235 INFO:tasks.workunit.client.0.smithi039.stdout:finished aio compare and write 2024-09-14T04:05:43.235 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:43.235 INFO:tasks.workunit.client.0.smithi039.stdout:started aio compare and write 2024-09-14T04:05:43.235 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:43.235 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:43.235 INFO:tasks.workunit.client.0.smithi039.stdout:finished aio compare and write 2024-09-14T04:05:43.235 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:05:43.236 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:05:43.236 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:05:43.236 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:05:43.236 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:05:43.236 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:43.236 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:05:43.236 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:05:43.236 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:05:43.236 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:43.236 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:05:43.236 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:05:43.236 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:05:43.236 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:43.237 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:05:43.237 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:05:43.237 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:05:43.237 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:43.237 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:05:43.237 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:05:43.237 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:05:43.237 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:43.237 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:05:43.237 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:05:43.237 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:05:43.237 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 512~512 = 512 2024-09-14T04:05:43.237 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:43.237 INFO:tasks.workunit.client.0.smithi039.stdout:aio discard: 1536~512 = 0 2024-09-14T04:05:43.237 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:05:43.238 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:05:43.238 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:05:43.238 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:05:43.238 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:05:43.238 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 0 2024-09-14T04:05:43.238 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:43.238 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:43.238 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 0 2024-09-14T04:05:43.238 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:43.238 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:43.238 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 16384 2024-09-14T04:05:43.238 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:43.238 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:43.238 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 16384 2024-09-14T04:05:43.239 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:43.239 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:43.239 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:43.239 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:43.239 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 49152 2024-09-14T04:05:43.239 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:43.239 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:43.239 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 49152 2024-09-14T04:05:43.239 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:43.239 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:43.239 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 65536 2024-09-14T04:05:43.239 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:43.239 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:43.239 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 65536 2024-09-14T04:05:43.239 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:43.240 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:43.240 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:43.240 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:43.240 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 98304 2024-09-14T04:05:43.240 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:43.240 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:43.240 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 98304 2024-09-14T04:05:43.240 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:43.240 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:43.240 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 114688 2024-09-14T04:05:43.240 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:43.240 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:43.240 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 114688 2024-09-14T04:05:43.240 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:43.240 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:43.241 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:43.241 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:43.241 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 147456 2024-09-14T04:05:43.241 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:43.241 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:43.241 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 147456 2024-09-14T04:05:43.241 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:43.241 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:43.241 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 163840 2024-09-14T04:05:43.241 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:43.241 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:43.241 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 163840 2024-09-14T04:05:43.241 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:43.241 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:43.242 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:43.242 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:43.242 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 196608 2024-09-14T04:05:43.242 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:43.242 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:43.242 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 196608 2024-09-14T04:05:43.242 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:43.242 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:43.242 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 212992 2024-09-14T04:05:43.242 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:43.242 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:43.242 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 212992 2024-09-14T04:05:43.242 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:43.242 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:43.243 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:43.243 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:43.243 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:43.243 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:43.243 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:43.243 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:43.243 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:43.243 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:43.243 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:43.243 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:43.244 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:43.244 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:43.244 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:43.244 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:43.244 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:43.244 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:43.244 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:43.244 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:43.244 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:43.244 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:43.244 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:43.244 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:43.244 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:43.245 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:43.245 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:43.245 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:43.245 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:43.245 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:43.245 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:43.245 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:43.246 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:43.246 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:43.246 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:43.246 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:43.246 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:43.246 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:43.246 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:43.246 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:43.246 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:43.246 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:43.246 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:43.246 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:43.246 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:43.246 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:43.246 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:43.247 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:43.247 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:43.247 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:43.247 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:43.247 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:43.247 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:43.485 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:43 smithi039 ceph-mon[106470]: Health check failed: Degraded data redundancy: 78/621 objects degraded (12.560%), 57 pgs degraded (PG_DEGRADED) 2024-09-14T04:05:43.485 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:43 smithi039 ceph-mon[111235]: Health check failed: Degraded data redundancy: 78/621 objects degraded (12.560%), 57 pgs degraded (PG_DEGRADED) 2024-09-14T04:05:43.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:43 smithi129 ceph-mon[101316]: Health check failed: Degraded data redundancy: 78/621 objects degraded (12.560%), 57 pgs degraded (PG_DEGRADED) 2024-09-14T04:05:43.735 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:43 smithi039 podman[123732]: 2024-09-14 04:05:43.484085706 +0000 UTC m=+0.650098417 container remove d7c6d6b7fce8657982b0c115d29edf82e17a18fc0821cd8ecf2b4a847c9e4b78 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0-deactivate, maintainer=Guillaume Abrioux , RELEASE=main-a278231, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T04:05:43.735 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:43 smithi039 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.0.service: Deactivated successfully. 2024-09-14T04:05:43.735 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:43 smithi039 systemd[1]: Stopped Ceph osd.0 for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T04:05:43.736 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:43 smithi039 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.0.service: Consumed 33.943s CPU time. 2024-09-14T04:05:44.002 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:43 smithi039 systemd[1]: Starting Ceph osd.0 for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T04:05:44.002 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:43 smithi039 podman[123833]: 2024-09-14 04:05:43.864853571 +0000 UTC m=+0.018336879 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:05:44.263 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:44 smithi039 ceph-mon[106470]: pgmap v521: 521 pgs: 126 active+undersized, 57 active+undersized+degraded, 338 active+clean; 591 KiB data, 522 MiB used, 715 GiB / 715 GiB avail; 242 KiB/s rd, 460 KiB/s wr, 499 op/s; 78/621 objects degraded (12.560%) 2024-09-14T04:05:44.263 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:44 smithi039 podman[123833]: 2024-09-14 04:05:44.001938061 +0000 UTC m=+0.155421365 container create e64615fbe93de3c51d1fa3997a5062104c4b3364f0d1af831f21d527b62271ff (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0-activate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=main-a278231, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-09-14T04:05:44.264 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:44 smithi039 ceph-mon[111235]: pgmap v521: 521 pgs: 126 active+undersized, 57 active+undersized+degraded, 338 active+clean; 591 KiB data, 522 MiB used, 715 GiB / 715 GiB avail; 242 KiB/s rd, 460 KiB/s wr, 499 op/s; 78/621 objects degraded (12.560%) 2024-09-14T04:05:44.422 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:44 smithi129 ceph-mon[101316]: pgmap v521: 521 pgs: 126 active+undersized, 57 active+undersized+degraded, 338 active+clean; 591 KiB data, 522 MiB used, 715 GiB / 715 GiB avail; 242 KiB/s rd, 460 KiB/s wr, 499 op/s; 78/621 objects degraded (12.560%) 2024-09-14T04:05:44.540 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:44 smithi039 podman[123833]: 2024-09-14 04:05:44.262915362 +0000 UTC m=+0.416398669 container init e64615fbe93de3c51d1fa3997a5062104c4b3364f0d1af831f21d527b62271ff (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True) 2024-09-14T04:05:44.540 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:44 smithi039 podman[123833]: 2024-09-14 04:05:44.26862332 +0000 UTC m=+0.422106620 container start e64615fbe93de3c51d1fa3997a5062104c4b3364f0d1af831f21d527b62271ff (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0-activate, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-14T04:05:44.540 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:44 smithi039 podman[123833]: 2024-09-14 04:05:44.432794804 +0000 UTC m=+0.586278116 container attach e64615fbe93de3c51d1fa3997a5062104c4b3364f0d1af831f21d527b62271ff (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0-activate, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=main-a278231, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T04:05:44.540 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:44 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0-activate[123848]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:05:44.541 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:44 smithi039 bash[123833]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:05:44.541 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:44 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0-activate[123848]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:05:44.541 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:44 smithi039 bash[123833]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:05:44.662 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:44.662 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:44.662 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:44.662 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:44.662 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:44.663 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:44.663 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:44.663 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:44.663 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:44.663 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:44.663 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:44.663 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:44.663 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:44.663 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:44.663 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:44.663 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:44.664 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:44.664 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:44.664 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:44.664 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:44.664 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:44.664 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:44.664 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:44.664 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:44.664 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:44.664 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:44.664 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:44.664 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:44.665 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:44.665 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:44.665 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:44.665 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:44.665 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:44.665 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:44.665 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:44.665 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:44.665 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:44.665 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:44.665 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:44.665 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:44.665 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:44.665 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:44.666 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:44.666 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:44.666 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:44.666 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:44.666 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:44.666 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:44.666 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:44.666 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:44.666 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:44.666 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:44.666 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:44.666 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:44.666 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:44.666 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:44.667 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:44.667 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:44.667 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:44.667 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:44.667 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:44.667 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:44.667 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:44.667 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:44.667 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:44.667 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:44.667 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:44.667 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:44.667 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:44.668 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:44.668 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:44.668 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:44.668 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:44.668 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:44.668 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:44.668 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:44.668 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:44.668 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:44.668 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:44.668 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:44.668 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:44.668 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:44.668 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:44.668 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:44.669 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:44.669 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:44.669 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:44.669 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:44.669 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:44.669 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:44.669 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:44.669 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:44.669 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:44.669 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:44.669 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:44.669 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:44.669 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:44.670 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:44.670 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:44.670 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:44.670 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:44.670 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:44.670 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:44.670 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:44.670 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:44.670 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:44.670 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:44.670 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:44.670 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:44.670 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:44.671 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:44.671 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:44.671 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:44.671 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:44.671 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:44.671 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:44.671 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:44.671 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:44.671 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:44.671 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:44.671 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:44.671 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:05:44.671 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:05:44.671 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:05:44.672 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:05:44.672 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestIO (3079 ms) 2024-09-14T04:05:44.672 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestEncryptionLUKS1 2024-09-14T04:05:44.672 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:05:45.213 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:44 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0-activate[123848]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-14T04:05:45.214 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:44 smithi039 bash[123833]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-14T04:05:45.214 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:44 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0-activate[123848]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-14T04:05:45.214 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:44 smithi039 bash[123833]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-14T04:05:45.214 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:45 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0-activate[123848]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-14T04:05:45.214 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:45 smithi039 bash[123833]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-14T04:05:45.540 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:45 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0-activate[123848]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-14T04:05:45.540 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:45 smithi039 bash[123833]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-14T04:05:45.540 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:45 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0-activate[123848]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-14T04:05:45.540 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:45 smithi039 bash[123833]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-09-14T04:05:45.540 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:45 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0-activate[123848]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-14T04:05:45.540 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:45 smithi039 bash[123833]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-09-14T04:05:45.541 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:45 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0-activate[123848]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-14T04:05:45.541 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:45 smithi039 bash[123833]: --> ceph-volume raw activate successful for osd ID: 0 2024-09-14T04:05:45.541 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:45 smithi039 podman[123833]: 2024-09-14 04:05:45.240741812 +0000 UTC m=+1.394225121 container died e64615fbe93de3c51d1fa3997a5062104c4b3364f0d1af831f21d527b62271ff (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=main-a278231, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-14T04:05:46.228 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:46 smithi039 ceph-mon[106470]: pgmap v522: 521 pgs: 126 active+undersized, 57 active+undersized+degraded, 338 active+clean; 591 KiB data, 522 MiB used, 715 GiB / 715 GiB avail; 193 KiB/s rd, 366 KiB/s wr, 397 op/s; 78/621 objects degraded (12.560%) 2024-09-14T04:05:46.228 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:46 smithi039 ceph-mon[106470]: pool 'test-librbd-smithi039-85658-28' no longer out of quota; removing NO_QUOTA flag 2024-09-14T04:05:46.229 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:46 smithi039 ceph-mon[106470]: Health check update: 2 pool(s) full (POOL_FULL) 2024-09-14T04:05:46.229 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:46 smithi039 ceph-mon[106470]: osdmap e669: 8 total, 7 up, 8 in 2024-09-14T04:05:46.229 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:46 smithi039 ceph-mon[111235]: pgmap v522: 521 pgs: 126 active+undersized, 57 active+undersized+degraded, 338 active+clean; 591 KiB data, 522 MiB used, 715 GiB / 715 GiB avail; 193 KiB/s rd, 366 KiB/s wr, 397 op/s; 78/621 objects degraded (12.560%) 2024-09-14T04:05:46.229 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:46 smithi039 ceph-mon[111235]: pool 'test-librbd-smithi039-85658-28' no longer out of quota; removing NO_QUOTA flag 2024-09-14T04:05:46.229 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:46 smithi039 ceph-mon[111235]: Health check update: 2 pool(s) full (POOL_FULL) 2024-09-14T04:05:46.229 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:46 smithi039 ceph-mon[111235]: osdmap e669: 8 total, 7 up, 8 in 2024-09-14T04:05:46.229 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:45 smithi039 podman[123995]: 2024-09-14 04:05:45.967937464 +0000 UTC m=+0.720772828 container remove e64615fbe93de3c51d1fa3997a5062104c4b3364f0d1af831f21d527b62271ff (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0-activate, RELEASE=main-a278231, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0) 2024-09-14T04:05:46.230 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:46 smithi039 podman[124061]: 2024-09-14 04:05:46.089975319 +0000 UTC m=+0.021749544 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:05:46.230 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:46 smithi039 podman[124061]: 2024-09-14 04:05:46.227437779 +0000 UTC m=+0.159211980 container create 36686aa8a5e4fdfd2b22b9b843052d7d423eba1d57764b5532f236b95221911b (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=main-a278231) 2024-09-14T04:05:46.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:46 smithi129 ceph-mon[101316]: pgmap v522: 521 pgs: 126 active+undersized, 57 active+undersized+degraded, 338 active+clean; 591 KiB data, 522 MiB used, 715 GiB / 715 GiB avail; 193 KiB/s rd, 366 KiB/s wr, 397 op/s; 78/621 objects degraded (12.560%) 2024-09-14T04:05:46.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:46 smithi129 ceph-mon[101316]: pool 'test-librbd-smithi039-85658-28' no longer out of quota; removing NO_QUOTA flag 2024-09-14T04:05:46.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:46 smithi129 ceph-mon[101316]: Health check update: 2 pool(s) full (POOL_FULL) 2024-09-14T04:05:46.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:46 smithi129 ceph-mon[101316]: osdmap e669: 8 total, 7 up, 8 in 2024-09-14T04:05:46.790 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:46 smithi039 podman[124061]: 2024-09-14 04:05:46.504782698 +0000 UTC m=+0.436556902 container init 36686aa8a5e4fdfd2b22b9b843052d7d423eba1d57764b5532f236b95221911b (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, ceph=True, RELEASE=main-a278231, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-14T04:05:46.790 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:46 smithi039 podman[124061]: 2024-09-14 04:05:46.510085299 +0000 UTC m=+0.441859501 container start 36686aa8a5e4fdfd2b22b9b843052d7d423eba1d57764b5532f236b95221911b (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T04:05:46.790 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:46 smithi039 ceph-osd[124098]: -- 172.21.15.39:0/3545931026 <== mon.0 v2:172.21.15.39:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55a6a139fa40 con 0x55a6a216e400 2024-09-14T04:05:46.791 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:46 smithi039 bash[124061]: 36686aa8a5e4fdfd2b22b9b843052d7d423eba1d57764b5532f236b95221911b 2024-09-14T04:05:46.791 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:46 smithi039 systemd[1]: Started Ceph osd.0 for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T04:05:48.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:47 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:48.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:47 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:48.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:47 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:48.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:47 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:05:48.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:47 smithi129 ceph-mon[101316]: pgmap v524: 521 pgs: 126 active+undersized, 57 active+undersized+degraded, 338 active+clean; 591 KiB data, 522 MiB used, 715 GiB / 715 GiB avail; 239 KiB/s rd, 137 KiB/s wr, 493 op/s; 78/621 objects degraded (12.560%) 2024-09-14T04:05:48.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:47 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:48.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:47 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:48.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:47 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:48.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:47 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:05:48.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:47 smithi039 ceph-mon[106470]: pgmap v524: 521 pgs: 126 active+undersized, 57 active+undersized+degraded, 338 active+clean; 591 KiB data, 522 MiB used, 715 GiB / 715 GiB avail; 239 KiB/s rd, 137 KiB/s wr, 493 op/s; 78/621 objects degraded (12.560%) 2024-09-14T04:05:48.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:47 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:48.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:47 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:48.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:47 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:48.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:47 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:05:48.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:47 smithi039 ceph-mon[111235]: pgmap v524: 521 pgs: 126 active+undersized, 57 active+undersized+degraded, 338 active+clean; 591 KiB data, 522 MiB used, 715 GiB / 715 GiB avail; 239 KiB/s rd, 137 KiB/s wr, 493 op/s; 78/621 objects degraded (12.560%) 2024-09-14T04:05:48.206 INFO:teuthology.orchestra.run.smithi039.stdout:true 2024-09-14T04:05:48.540 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0[124075]: 2024-09-14T04:05:48.155+0000 7fb5560c2640 -1 Falling back to public interface 2024-09-14T04:05:48.880 INFO:teuthology.orchestra.run.smithi039.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T04:05:48.881 INFO:teuthology.orchestra.run.smithi039.stdout:alertmanager.a smithi039 *:9093,9094 running (7m) 47s ago 13m 19.9M - 0.27.0 11f11916f8cd 6b564e1fc909 2024-09-14T04:05:48.881 INFO:teuthology.orchestra.run.smithi039.stdout:grafana.a smithi129 *:3000 running (7m) 30s ago 13m 80.2M - 10.4.0 c8b91775d855 f0492c19de92 2024-09-14T04:05:48.881 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi039.bbwiwf smithi039 running (12m) 47s ago 12m 25.0M - 18.2.4-845-ge4e227b3 67ddd33fa72f 91e6c8f85681 2024-09-14T04:05:48.881 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi129.ybjqcw smithi129 running (12m) 30s ago 12m 16.9M - 18.2.4-845-ge4e227b3 67ddd33fa72f a3fab5b37b71 2024-09-14T04:05:48.881 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.x smithi129 *:8443,9283,8765 running (9m) 30s ago 17m 457M - 19.3.0-4907-ga2782315 8801d2529e43 448bd4e146cb 2024-09-14T04:05:48.881 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.y smithi039 *:8443,9283,8765 running (9m) 47s ago 19m 528M - 19.3.0-4907-ga2782315 8801d2529e43 cccc87f635a6 2024-09-14T04:05:48.881 INFO:teuthology.orchestra.run.smithi039.stdout:mon.a smithi039 running (6m) 47s ago 19m 72.1M 2048M 19.3.0-4907-ga2782315 8801d2529e43 7198d4eb8e4c 2024-09-14T04:05:48.881 INFO:teuthology.orchestra.run.smithi039.stdout:mon.b smithi129 running (4m) 30s ago 18m 48.8M 2048M 19.3.0-4907-ga2782315 8801d2529e43 cff2519560f1 2024-09-14T04:05:48.881 INFO:teuthology.orchestra.run.smithi039.stdout:mon.c smithi039 running (6m) 47s ago 18m 57.1M 2048M 19.3.0-4907-ga2782315 8801d2529e43 4b8f0f3864b1 2024-09-14T04:05:48.881 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.a smithi039 *:9100 running (8m) 47s ago 13m 11.6M - 1.7.0 72c9c2088986 7dd5945adb64 2024-09-14T04:05:48.881 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.b smithi129 *:9100 running (8m) 30s ago 13m 12.0M - 1.7.0 72c9c2088986 1fedb5f257e7 2024-09-14T04:05:48.882 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0 smithi039 starting - - - 2480M 2024-09-14T04:05:48.882 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1 smithi039 running (17m) 47s ago 17m 124M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 91349527d88b 2024-09-14T04:05:48.882 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2 smithi039 running (16m) 47s ago 16m 143M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f eb903b44fc5d 2024-09-14T04:05:48.882 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3 smithi039 running (16m) 47s ago 16m 140M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f dff9e665bd9b 2024-09-14T04:05:48.882 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4 smithi129 running (15m) 30s ago 15m 126M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 5b64727311b7 2024-09-14T04:05:48.882 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5 smithi129 running (15m) 30s ago 15m 116M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f f2d991a754f7 2024-09-14T04:05:48.882 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6 smithi129 running (15m) 30s ago 15m 136M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 3060eaf44106 2024-09-14T04:05:48.882 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7 smithi129 running (14m) 30s ago 14m 119M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 4c13b97060d1 2024-09-14T04:05:48.882 INFO:teuthology.orchestra.run.smithi039.stdout:prometheus.a smithi129 *:9095 running (8m) 30s ago 14m 58.1M - 2.51.0 1d3b7f56885b cbe583f852c3 2024-09-14T04:05:48.947 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:05:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:05:48] "GET /metrics HTTP/1.1" 200 67756 "" "Prometheus/2.51.0" 2024-09-14T04:05:49.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:48 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 78/630 objects degraded (12.381%), 57 pgs degraded (PG_DEGRADED) 2024-09-14T04:05:49.276 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:05:49.276 INFO:teuthology.orchestra.run.smithi039.stdout: "mon": { 2024-09-14T04:05:49.277 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T04:05:49.277 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:05:49.277 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr": { 2024-09-14T04:05:49.277 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T04:05:49.277 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:05:49.277 INFO:teuthology.orchestra.run.smithi039.stdout: "osd": { 2024-09-14T04:05:49.277 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 7 2024-09-14T04:05:49.277 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:05:49.278 INFO:teuthology.orchestra.run.smithi039.stdout: "mds": { 2024-09-14T04:05:49.278 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T04:05:49.278 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:05:49.278 INFO:teuthology.orchestra.run.smithi039.stdout: "overall": { 2024-09-14T04:05:49.278 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 9, 2024-09-14T04:05:49.278 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 5 2024-09-14T04:05:49.278 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:05:49.278 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:05:49.279 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:48 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 78/630 objects degraded (12.381%), 57 pgs degraded (PG_DEGRADED) 2024-09-14T04:05:49.279 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:48 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 78/630 objects degraded (12.381%), 57 pgs degraded (PG_DEGRADED) 2024-09-14T04:05:49.631 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:05:49.632 INFO:teuthology.orchestra.run.smithi039.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T04:05:49.632 INFO:teuthology.orchestra.run.smithi039.stdout: "in_progress": true, 2024-09-14T04:05:49.632 INFO:teuthology.orchestra.run.smithi039.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (3 remaining).", 2024-09-14T04:05:49.632 INFO:teuthology.orchestra.run.smithi039.stdout: "services_complete": [], 2024-09-14T04:05:49.632 INFO:teuthology.orchestra.run.smithi039.stdout: "progress": "0/8 daemons upgraded", 2024-09-14T04:05:49.632 INFO:teuthology.orchestra.run.smithi039.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T04:05:49.632 INFO:teuthology.orchestra.run.smithi039.stdout: "is_paused": false 2024-09-14T04:05:49.632 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:05:50.205 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:05:50.205 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:05:50.205 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:05:50.205 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:05:50.205 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:05:50.205 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:50.205 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:05:50.205 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:50.206 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:50.206 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:05:50.206 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:50.206 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:05:50.206 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:50.206 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:50.206 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:05:50.206 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:50.206 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:05:50.206 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:50.206 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:50.206 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:05:50.206 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:50.206 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:05:50.206 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:50.206 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:50.207 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:05:50.207 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:50.207 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:05:50.207 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:50.207 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:50.207 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:05:50.207 INFO:tasks.workunit.client.0.smithi039.stdout:start compare and write 2024-09-14T04:05:50.207 INFO:tasks.workunit.client.0.smithi039.stdout:compare and wrote: 512 2024-09-14T04:05:50.207 INFO:tasks.workunit.client.0.smithi039.stdout:start compare and write 2024-09-14T04:05:50.207 INFO:tasks.workunit.client.0.smithi039.stdout:compare and wrote: 512 2024-09-14T04:05:50.207 INFO:tasks.workunit.client.0.smithi039.stdout:start compare and write 2024-09-14T04:05:50.207 INFO:tasks.workunit.client.0.smithi039.stdout:compare and wrote: 512 2024-09-14T04:05:50.207 INFO:tasks.workunit.client.0.smithi039.stdout:start compare and write 2024-09-14T04:05:50.207 INFO:tasks.workunit.client.0.smithi039.stdout:compare and wrote: 512 2024-09-14T04:05:50.208 INFO:tasks.workunit.client.0.smithi039.stdout:start compare and write 2024-09-14T04:05:50.208 INFO:tasks.workunit.client.0.smithi039.stdout:compare and wrote: 512 2024-09-14T04:05:50.208 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:50.208 INFO:tasks.workunit.client.0.smithi039.stdout:started aio compare and write 2024-09-14T04:05:50.208 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:50.208 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:50.208 INFO:tasks.workunit.client.0.smithi039.stdout:finished aio compare and write 2024-09-14T04:05:50.208 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:50.208 INFO:tasks.workunit.client.0.smithi039.stdout:started aio compare and write 2024-09-14T04:05:50.208 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:50.208 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:50.208 INFO:tasks.workunit.client.0.smithi039.stdout:finished aio compare and write 2024-09-14T04:05:50.208 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:50.208 INFO:tasks.workunit.client.0.smithi039.stdout:started aio compare and write 2024-09-14T04:05:50.208 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:50.209 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:50.209 INFO:tasks.workunit.client.0.smithi039.stdout:finished aio compare and write 2024-09-14T04:05:50.209 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:50.209 INFO:tasks.workunit.client.0.smithi039.stdout:started aio compare and write 2024-09-14T04:05:50.209 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:50.209 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:50.209 INFO:tasks.workunit.client.0.smithi039.stdout:finished aio compare and write 2024-09-14T04:05:50.209 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:50.209 INFO:tasks.workunit.client.0.smithi039.stdout:started aio compare and write 2024-09-14T04:05:50.209 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:50.209 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:50.209 INFO:tasks.workunit.client.0.smithi039.stdout:finished aio compare and write 2024-09-14T04:05:50.209 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:05:50.210 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:05:50.210 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:05:50.210 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:05:50.210 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:05:50.210 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:50.210 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:05:50.210 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:05:50.210 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:05:50.210 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:50.210 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:05:50.210 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:05:50.210 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:05:50.210 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:50.211 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:05:50.211 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:05:50.211 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:05:50.211 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:50.211 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:05:50.211 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:05:50.211 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:05:50.211 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:50.211 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:05:50.211 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:05:50.211 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:05:50.211 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 512~512 = 512 2024-09-14T04:05:50.211 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:50.211 INFO:tasks.workunit.client.0.smithi039.stdout:aio discard: 1536~512 = 0 2024-09-14T04:05:50.211 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:05:50.212 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:05:50.212 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:05:50.212 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:05:50.212 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:05:50.212 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 0 2024-09-14T04:05:50.212 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:50.212 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:50.212 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 0 2024-09-14T04:05:50.212 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:50.212 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:50.212 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 16384 2024-09-14T04:05:50.212 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:50.213 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:50.213 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 16384 2024-09-14T04:05:50.213 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:50.213 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:50.213 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:50.213 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:50.213 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 49152 2024-09-14T04:05:50.213 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:50.213 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:50.213 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 49152 2024-09-14T04:05:50.213 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:50.213 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:50.213 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 65536 2024-09-14T04:05:50.214 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:50.214 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:50.214 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 65536 2024-09-14T04:05:50.214 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:50.214 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:50.214 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:50.214 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:50.214 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 98304 2024-09-14T04:05:50.214 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:50.214 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:50.214 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 98304 2024-09-14T04:05:50.214 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:50.214 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:50.214 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 114688 2024-09-14T04:05:50.214 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:50.214 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:50.215 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 114688 2024-09-14T04:05:50.215 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:50.215 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:50.215 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:50.215 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:50.215 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 147456 2024-09-14T04:05:50.215 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:50.215 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:50.215 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 147456 2024-09-14T04:05:50.215 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:50.215 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:50.215 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 163840 2024-09-14T04:05:50.215 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:50.215 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:50.215 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 163840 2024-09-14T04:05:50.216 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:50.216 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:50.216 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:50.216 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:50.216 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 196608 2024-09-14T04:05:50.216 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:50.216 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:50.216 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 196608 2024-09-14T04:05:50.216 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:50.216 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:50.216 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 212992 2024-09-14T04:05:50.216 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:50.216 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:50.216 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 212992 2024-09-14T04:05:50.217 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:50.217 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:50.217 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:50.217 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:50.217 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:50.217 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:50.217 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:50.217 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:50.217 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:50.217 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:50.217 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:50.217 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:50.217 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:50.218 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:50.218 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:50.218 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:50.218 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:50.218 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:50.218 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:50.218 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:50.218 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:50.218 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:50.218 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:50.218 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:50.218 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:50.218 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:50.219 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:50.219 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:50.219 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:50.219 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:50.219 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:50.219 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:50.219 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:50.219 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:50.219 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:50.219 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:50.219 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:50.219 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:50.220 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:50.220 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:50.220 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:50.220 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:50.220 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:50.220 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:50.220 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:50.220 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:50.220 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:50.220 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:50.220 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:50.220 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:50.220 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:50.220 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:50.221 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:50.229 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:49 smithi039 ceph-mon[111235]: from='client.44435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:05:50.229 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:49 smithi039 ceph-mon[111235]: from='client.44441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:05:50.229 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:49 smithi039 ceph-mon[111235]: pgmap v525: 521 pgs: 126 active+undersized, 57 active+undersized+degraded, 338 active+clean; 2.6 MiB data, 522 MiB used, 715 GiB / 715 GiB avail; 506 KiB/s rd, 266 KiB/s wr, 1.04k op/s; 78/630 objects degraded (12.381%) 2024-09-14T04:05:50.229 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:49 smithi039 ceph-mon[111235]: from='client.44447 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:05:50.229 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:49 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1300672107' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:05:50.229 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:49 smithi039 ceph-mon[106470]: from='client.44435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:05:50.229 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:49 smithi039 ceph-mon[106470]: from='client.44441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:05:50.229 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:49 smithi039 ceph-mon[106470]: pgmap v525: 521 pgs: 126 active+undersized, 57 active+undersized+degraded, 338 active+clean; 2.6 MiB data, 522 MiB used, 715 GiB / 715 GiB avail; 506 KiB/s rd, 266 KiB/s wr, 1.04k op/s; 78/630 objects degraded (12.381%) 2024-09-14T04:05:50.229 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:49 smithi039 ceph-mon[106470]: from='client.44447 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:05:50.230 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:49 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1300672107' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:05:50.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:49 smithi129 ceph-mon[101316]: from='client.44435 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:05:50.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:49 smithi129 ceph-mon[101316]: from='client.44441 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:05:50.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:49 smithi129 ceph-mon[101316]: pgmap v525: 521 pgs: 126 active+undersized, 57 active+undersized+degraded, 338 active+clean; 2.6 MiB data, 522 MiB used, 715 GiB / 715 GiB avail; 506 KiB/s rd, 266 KiB/s wr, 1.04k op/s; 78/630 objects degraded (12.381%) 2024-09-14T04:05:50.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:49 smithi129 ceph-mon[101316]: from='client.44447 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:05:50.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:49 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1300672107' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:05:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:50 smithi129 ceph-mon[101316]: from='client.34530 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:05:51.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:50 smithi039 ceph-mon[106470]: from='client.34530 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:05:51.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:50 smithi039 ceph-mon[111235]: from='client.34530 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:05:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:51 smithi129 ceph-mon[101316]: pgmap v526: 521 pgs: 126 active+undersized, 57 active+undersized+degraded, 338 active+clean; 2.6 MiB data, 522 MiB used, 715 GiB / 715 GiB avail; 432 KiB/s rd, 227 KiB/s wr, 891 op/s; 78/630 objects degraded (12.381%) 2024-09-14T04:05:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:51 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:51 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:51 smithi129 ceph-mon[101316]: from='osd.0 [v2:172.21.15.39:6802/3832421293,v1:172.21.15.39:6803/3832421293]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-14T04:05:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:51 smithi129 ceph-mon[101316]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-14T04:05:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:51 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:52.290 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:51 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0[124075]: 2024-09-14T04:05:51.840+0000 7fb5560c2640 -1 osd.0 667 log_to_monitors true 2024-09-14T04:05:52.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:51 smithi039 ceph-mon[106470]: pgmap v526: 521 pgs: 126 active+undersized, 57 active+undersized+degraded, 338 active+clean; 2.6 MiB data, 522 MiB used, 715 GiB / 715 GiB avail; 432 KiB/s rd, 227 KiB/s wr, 891 op/s; 78/630 objects degraded (12.381%) 2024-09-14T04:05:52.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:51 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:52.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:51 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:52.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:51 smithi039 ceph-mon[106470]: from='osd.0 [v2:172.21.15.39:6802/3832421293,v1:172.21.15.39:6803/3832421293]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-14T04:05:52.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:51 smithi039 ceph-mon[106470]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-14T04:05:52.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:51 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:52.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:51 smithi039 ceph-mon[111235]: pgmap v526: 521 pgs: 126 active+undersized, 57 active+undersized+degraded, 338 active+clean; 2.6 MiB data, 522 MiB used, 715 GiB / 715 GiB avail; 432 KiB/s rd, 227 KiB/s wr, 891 op/s; 78/630 objects degraded (12.381%) 2024-09-14T04:05:52.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:51 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:52.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:51 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:52.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:51 smithi039 ceph-mon[111235]: from='osd.0 [v2:172.21.15.39:6802/3832421293,v1:172.21.15.39:6803/3832421293]' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-14T04:05:52.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:51 smithi039 ceph-mon[111235]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]} : dispatch 2024-09-14T04:05:52.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:51 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:52.764 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 04:05:52 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0[124075]: 2024-09-14T04:05:52.403+0000 7fb54de6c640 -1 osd.0 667 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T04:05:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:53 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:53 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:53 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:53 smithi039 ceph-mon[106470]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-14T04:05:53.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:53 smithi039 ceph-mon[106470]: osdmap e670: 8 total, 7 up, 8 in 2024-09-14T04:05:53.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:53 smithi039 ceph-mon[106470]: from='osd.0 [v2:172.21.15.39:6802/3832421293,v1:172.21.15.39:6803/3832421293]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi039", "root=default"]} : dispatch 2024-09-14T04:05:53.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:53 smithi039 ceph-mon[106470]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi039", "root=default"]} : dispatch 2024-09-14T04:05:53.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:53 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:53.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:53 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:53.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:53 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:53.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:53 smithi039 ceph-mon[111235]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-14T04:05:53.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:53 smithi039 ceph-mon[111235]: osdmap e670: 8 total, 7 up, 8 in 2024-09-14T04:05:53.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:53 smithi039 ceph-mon[111235]: from='osd.0 [v2:172.21.15.39:6802/3832421293,v1:172.21.15.39:6803/3832421293]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi039", "root=default"]} : dispatch 2024-09-14T04:05:53.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:53 smithi039 ceph-mon[111235]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi039", "root=default"]} : dispatch 2024-09-14T04:05:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:53 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:53 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:53 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:05:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:53 smithi129 ceph-mon[101316]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-09-14T04:05:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:53 smithi129 ceph-mon[101316]: osdmap e670: 8 total, 7 up, 8 in 2024-09-14T04:05:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:53 smithi129 ceph-mon[101316]: from='osd.0 [v2:172.21.15.39:6802/3832421293,v1:172.21.15.39:6803/3832421293]' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi039", "root=default"]} : dispatch 2024-09-14T04:05:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:53 smithi129 ceph-mon[101316]: from='osd.0 ' entity='osd.0' cmd={"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi039", "root=default"]} : dispatch 2024-09-14T04:05:54.222 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:54 smithi039 ceph-mon[106470]: pgmap v528: 521 pgs: 126 active+undersized, 57 active+undersized+degraded, 338 active+clean; 11 MiB data, 508 MiB used, 715 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.2 MiB/s wr, 1.71k op/s; 78/639 objects degraded (12.207%) 2024-09-14T04:05:54.222 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:54 smithi039 ceph-mon[106470]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T04:05:54.222 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:54 smithi039 ceph-mon[106470]: osd.0 [v2:172.21.15.39:6802/3832421293,v1:172.21.15.39:6803/3832421293] boot 2024-09-14T04:05:54.222 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:54 smithi039 ceph-mon[106470]: osdmap e671: 8 total, 8 up, 8 in 2024-09-14T04:05:54.222 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:54 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-14T04:05:54.222 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:54 smithi039 ceph-mon[111235]: pgmap v528: 521 pgs: 126 active+undersized, 57 active+undersized+degraded, 338 active+clean; 11 MiB data, 508 MiB used, 715 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.2 MiB/s wr, 1.71k op/s; 78/639 objects degraded (12.207%) 2024-09-14T04:05:54.222 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:54 smithi039 ceph-mon[111235]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T04:05:54.223 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:54 smithi039 ceph-mon[111235]: osd.0 [v2:172.21.15.39:6802/3832421293,v1:172.21.15.39:6803/3832421293] boot 2024-09-14T04:05:54.223 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:54 smithi039 ceph-mon[111235]: osdmap e671: 8 total, 8 up, 8 in 2024-09-14T04:05:54.223 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:54 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-14T04:05:54.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:54 smithi129 ceph-mon[101316]: pgmap v528: 521 pgs: 126 active+undersized, 57 active+undersized+degraded, 338 active+clean; 11 MiB data, 508 MiB used, 715 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.2 MiB/s wr, 1.71k op/s; 78/639 objects degraded (12.207%) 2024-09-14T04:05:54.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:54 smithi129 ceph-mon[101316]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T04:05:54.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:54 smithi129 ceph-mon[101316]: osd.0 [v2:172.21.15.39:6802/3832421293,v1:172.21.15.39:6803/3832421293] boot 2024-09-14T04:05:54.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:54 smithi129 ceph-mon[101316]: osdmap e671: 8 total, 8 up, 8 in 2024-09-14T04:05:54.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:54 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 0} : dispatch 2024-09-14T04:05:55.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:55 smithi039 ceph-mon[106470]: osdmap e672: 8 total, 8 up, 8 in 2024-09-14T04:05:55.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:55 smithi039 ceph-mon[106470]: pgmap v531: 521 pgs: 126 active+undersized, 57 active+undersized+degraded, 338 active+clean; 11 MiB data, 508 MiB used, 715 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.4 MiB/s wr, 1.42k op/s; 78/639 objects degraded (12.207%) 2024-09-14T04:05:55.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:55 smithi039 ceph-mon[111235]: osdmap e672: 8 total, 8 up, 8 in 2024-09-14T04:05:55.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:55 smithi039 ceph-mon[111235]: pgmap v531: 521 pgs: 126 active+undersized, 57 active+undersized+degraded, 338 active+clean; 11 MiB data, 508 MiB used, 715 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.4 MiB/s wr, 1.42k op/s; 78/639 objects degraded (12.207%) 2024-09-14T04:05:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:55 smithi129 ceph-mon[101316]: osdmap e672: 8 total, 8 up, 8 in 2024-09-14T04:05:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:55 smithi129 ceph-mon[101316]: pgmap v531: 521 pgs: 126 active+undersized, 57 active+undersized+degraded, 338 active+clean; 11 MiB data, 508 MiB used, 715 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.4 MiB/s wr, 1.42k op/s; 78/639 objects degraded (12.207%) 2024-09-14T04:05:56.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:05:55 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T04:05:55.735+0000 7f268b1da640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (49 PGs are or would become offline) 2024-09-14T04:05:56.160 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:56.161 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:56.161 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:56.161 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:56.161 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:56.161 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:56.161 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:56.161 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:56.161 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:56.161 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:56.161 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:56.161 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:56.161 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:56.161 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:56.161 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:56.162 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:56.162 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:56.162 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:56.162 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:56.162 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:56.162 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:56.162 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:56.162 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:56.162 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:56.162 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:56.162 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:56.162 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:56.162 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:56.163 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:56.163 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:56.163 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:56.163 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:56.163 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:56.163 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:56.163 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:56.163 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:56.163 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:56.163 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:56.163 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:56.163 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:56.163 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:56.163 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:56.164 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:56.164 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:56.164 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:56.164 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:56.164 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:56.164 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:56.164 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:56.164 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:56.164 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:56.164 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:56.164 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:56.164 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:56.164 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:56.165 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:56.165 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:56.165 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:56.165 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:56.165 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:56.165 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:56.165 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:56.165 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:56.165 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:56.165 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:56.165 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:56.165 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:56.165 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:56.165 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:56.165 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:56.166 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:56.166 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:56.166 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:56.166 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:56.166 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:56.166 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:56.166 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:56.166 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:56.166 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:56.166 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:56.166 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:56.166 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:56.166 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:56.166 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:56.167 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:56.167 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:56.167 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:56.167 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:56.167 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:56.167 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:56.167 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:56.167 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:56.167 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:56.167 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:56.167 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:56.167 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:56.167 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:56.168 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:56.168 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:56.168 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:56.168 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:56.168 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:56.168 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:56.168 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:56.168 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:56.168 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:56.168 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:56.168 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:56.168 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:56.168 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:56.169 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:05:56.169 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:05:56.169 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:05:56.169 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:05:56.169 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:05:56.169 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:56.169 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:56.169 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:56.169 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:05:56.169 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:05:56.169 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:05:56.169 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:05:56.169 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:05:56.169 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:05:56.169 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:05:56.170 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 4 2024-09-14T04:05:56.170 INFO:tasks.workunit.client.0.smithi039.stdout:read: 4 2024-09-14T04:05:56.170 INFO:tasks.workunit.client.0.smithi039.stdout:read: 4 2024-09-14T04:05:56.170 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestEncryptionLUKS1 (11498 ms) 2024-09-14T04:05:56.170 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestEncryptionLUKS2 2024-09-14T04:05:56.170 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:05:56.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:56 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 78/639 objects degraded (12.207%), 57 pgs degraded (PG_DEGRADED) 2024-09-14T04:05:56.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:56 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:56.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:56 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:56.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:56 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:05:56.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:56 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:05:56.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:56 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:56.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:56 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:05:56.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:56 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:05:56.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:56 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:05:56.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:56 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:05:56.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:56 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-14T04:05:56.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:56 smithi129 ceph-mon[101316]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-14T04:05:56.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:56 smithi129 ceph-mon[101316]: Upgrade: unsafe to stop osd(s) at this time (49 PGs are or would become offline) 2024-09-14T04:05:56.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:56 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 78/639 objects degraded (12.207%), 57 pgs degraded (PG_DEGRADED) 2024-09-14T04:05:56.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:56 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:56.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:56 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:56.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:56 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:05:56.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:56 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:05:56.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:56 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:56.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:56 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:05:56.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:56 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:05:56.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:56 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:05:56.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:56 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:05:56.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:56 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-14T04:05:56.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:56 smithi039 ceph-mon[111235]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-14T04:05:56.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:56 smithi039 ceph-mon[111235]: Upgrade: unsafe to stop osd(s) at this time (49 PGs are or would become offline) 2024-09-14T04:05:56.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:56 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 78/639 objects degraded (12.207%), 57 pgs degraded (PG_DEGRADED) 2024-09-14T04:05:56.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:56 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:56.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:56 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:56.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:56 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:05:56.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:56 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:05:56.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:56 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:05:56.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:56 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:05:56.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:56 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:05:56.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:56 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:05:56.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:56 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:05:56.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:56 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-14T04:05:56.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:56 smithi039 ceph-mon[106470]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-14T04:05:56.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:56 smithi039 ceph-mon[106470]: Upgrade: unsafe to stop osd(s) at this time (49 PGs are or would become offline) 2024-09-14T04:05:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:57 smithi129 ceph-mon[101316]: pgmap v532: 521 pgs: 126 active+undersized, 57 active+undersized+degraded, 338 active+clean; 11 MiB data, 508 MiB used, 715 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.4 MiB/s wr, 1.42k op/s; 78/639 objects degraded (12.207%) 2024-09-14T04:05:57.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:57 smithi039 ceph-mon[106470]: pgmap v532: 521 pgs: 126 active+undersized, 57 active+undersized+degraded, 338 active+clean; 11 MiB data, 508 MiB used, 715 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.4 MiB/s wr, 1.42k op/s; 78/639 objects degraded (12.207%) 2024-09-14T04:05:57.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:57 smithi039 ceph-mon[111235]: pgmap v532: 521 pgs: 126 active+undersized, 57 active+undersized+degraded, 338 active+clean; 11 MiB data, 508 MiB used, 715 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.4 MiB/s wr, 1.42k op/s; 78/639 objects degraded (12.207%) 2024-09-14T04:05:58.955 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:58 smithi039 ceph-mon[111235]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 78/639 objects degraded (12.207%), 57 pgs degraded) 2024-09-14T04:05:58.955 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:05:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:05:58] "GET /metrics HTTP/1.1" 200 67903 "" "Prometheus/2.51.0" 2024-09-14T04:05:59.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:58 smithi129 ceph-mon[101316]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 78/639 objects degraded (12.207%), 57 pgs degraded) 2024-09-14T04:05:59.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:58 smithi039 ceph-mon[106470]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 78/639 objects degraded (12.207%), 57 pgs degraded) 2024-09-14T04:06:00.241 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:05:59 smithi039 ceph-mon[106470]: pgmap v533: 521 pgs: 521 active+clean; 11 MiB data, 504 MiB used, 715 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.2 MiB/s wr, 1.32k op/s; 80 B/s, 0 objects/s recovering 2024-09-14T04:06:00.242 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:05:59 smithi039 ceph-mon[111235]: pgmap v533: 521 pgs: 521 active+clean; 11 MiB data, 504 MiB used, 715 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.2 MiB/s wr, 1.32k op/s; 80 B/s, 0 objects/s recovering 2024-09-14T04:06:00.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:05:59 smithi129 ceph-mon[101316]: pgmap v533: 521 pgs: 521 active+clean; 11 MiB data, 504 MiB used, 715 GiB / 715 GiB avail; 2.6 MiB/s rd, 1.2 MiB/s wr, 1.32k op/s; 80 B/s, 0 objects/s recovering 2024-09-14T04:06:02.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:01 smithi129 ceph-mon[101316]: pgmap v534: 521 pgs: 521 active+clean; 11 MiB data, 504 MiB used, 715 GiB / 715 GiB avail; 554 KiB/s rd, 511 B/s wr, 9 op/s; 65 B/s, 0 objects/s recovering 2024-09-14T04:06:02.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:01 smithi039 ceph-mon[106470]: pgmap v534: 521 pgs: 521 active+clean; 11 MiB data, 504 MiB used, 715 GiB / 715 GiB avail; 554 KiB/s rd, 511 B/s wr, 9 op/s; 65 B/s, 0 objects/s recovering 2024-09-14T04:06:02.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:01 smithi039 ceph-mon[111235]: pgmap v534: 521 pgs: 521 active+clean; 11 MiB data, 504 MiB used, 715 GiB / 715 GiB avail; 554 KiB/s rd, 511 B/s wr, 9 op/s; 65 B/s, 0 objects/s recovering 2024-09-14T04:06:04.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:03 smithi129 ceph-mon[101316]: pgmap v535: 521 pgs: 521 active+clean; 11 MiB data, 504 MiB used, 715 GiB / 715 GiB avail; 466 KiB/s rd, 430 B/s wr, 7 op/s; 54 B/s, 0 objects/s recovering 2024-09-14T04:06:04.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:03 smithi039 ceph-mon[106470]: pgmap v535: 521 pgs: 521 active+clean; 11 MiB data, 504 MiB used, 715 GiB / 715 GiB avail; 466 KiB/s rd, 430 B/s wr, 7 op/s; 54 B/s, 0 objects/s recovering 2024-09-14T04:06:04.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:03 smithi039 ceph-mon[111235]: pgmap v535: 521 pgs: 521 active+clean; 11 MiB data, 504 MiB used, 715 GiB / 715 GiB avail; 466 KiB/s rd, 430 B/s wr, 7 op/s; 54 B/s, 0 objects/s recovering 2024-09-14T04:06:06.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:05 smithi129 ceph-mon[101316]: pgmap v536: 521 pgs: 521 active+clean; 11 MiB data, 504 MiB used, 715 GiB / 715 GiB avail; 422 KiB/s rd, 389 B/s wr, 6 op/s; 49 B/s, 0 objects/s recovering 2024-09-14T04:06:06.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:05 smithi039 ceph-mon[106470]: pgmap v536: 521 pgs: 521 active+clean; 11 MiB data, 504 MiB used, 715 GiB / 715 GiB avail; 422 KiB/s rd, 389 B/s wr, 6 op/s; 49 B/s, 0 objects/s recovering 2024-09-14T04:06:06.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:05 smithi039 ceph-mon[111235]: pgmap v536: 521 pgs: 521 active+clean; 11 MiB data, 504 MiB used, 715 GiB / 715 GiB avail; 422 KiB/s rd, 389 B/s wr, 6 op/s; 49 B/s, 0 objects/s recovering 2024-09-14T04:06:07.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 4]} : dispatch 2024-09-14T04:06:07.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 4]} : dispatch 2024-09-14T04:06:07.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 4]} : dispatch 2024-09-14T04:06:07.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 4]} : dispatch 2024-09-14T04:06:07.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 4]} : dispatch 2024-09-14T04:06:07.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 4]} : dispatch 2024-09-14T04:06:08.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:08 smithi129 ceph-mon[101316]: pgmap v537: 521 pgs: 521 active+clean; 11 MiB data, 504 MiB used, 715 GiB / 715 GiB avail; 370 KiB/s rd, 341 B/s wr, 6 op/s; 43 B/s, 0 objects/s recovering 2024-09-14T04:06:08.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:08 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 4]}]': finished 2024-09-14T04:06:08.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:08 smithi129 ceph-mon[101316]: osdmap e673: 8 total, 8 up, 8 in 2024-09-14T04:06:08.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:08 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:08.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:08 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:08.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:08 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:08.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:08 smithi039 ceph-mon[106470]: pgmap v537: 521 pgs: 521 active+clean; 11 MiB data, 504 MiB used, 715 GiB / 715 GiB avail; 370 KiB/s rd, 341 B/s wr, 6 op/s; 43 B/s, 0 objects/s recovering 2024-09-14T04:06:08.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 4]}]': finished 2024-09-14T04:06:08.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:08 smithi039 ceph-mon[106470]: osdmap e673: 8 total, 8 up, 8 in 2024-09-14T04:06:08.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:08.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:08 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:08.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:08.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:08 smithi039 ceph-mon[111235]: pgmap v537: 521 pgs: 521 active+clean; 11 MiB data, 504 MiB used, 715 GiB / 715 GiB avail; 370 KiB/s rd, 341 B/s wr, 6 op/s; 43 B/s, 0 objects/s recovering 2024-09-14T04:06:08.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:08 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "20.d", "id": [7, 4]}]': finished 2024-09-14T04:06:08.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:08 smithi039 ceph-mon[111235]: osdmap e673: 8 total, 8 up, 8 in 2024-09-14T04:06:08.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:08 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:08.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:08 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:08.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:08 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:09.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:06:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:06:08] "GET /metrics HTTP/1.1" 200 67913 "" "Prometheus/2.51.0" 2024-09-14T04:06:09.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:09 smithi129 ceph-mon[101316]: osdmap e674: 8 total, 8 up, 8 in 2024-09-14T04:06:09.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:09 smithi039 ceph-mon[106470]: osdmap e674: 8 total, 8 up, 8 in 2024-09-14T04:06:09.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:09 smithi039 ceph-mon[111235]: osdmap e674: 8 total, 8 up, 8 in 2024-09-14T04:06:10.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:10 smithi129 ceph-mon[101316]: pgmap v540: 521 pgs: 1 peering, 520 active+clean; 11 MiB data, 504 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:06:10.544 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:10 smithi039 ceph-mon[106470]: pgmap v540: 521 pgs: 1 peering, 520 active+clean; 11 MiB data, 504 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:06:10.545 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:10 smithi039 ceph-mon[111235]: pgmap v540: 521 pgs: 1 peering, 520 active+clean; 11 MiB data, 504 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:06:11.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:06:10 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T04:06:10.751+0000 7f268b1da640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-14T04:06:11.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:11 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-14T04:06:11.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:11 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-14T04:06:11.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:11 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-14T04:06:11.664 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:06:11.664 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:06:11.664 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:06:11.664 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:06:11.664 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:06:11.665 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:11.665 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:06:11.665 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:06:11.665 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:06:11.665 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:06:11.665 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:11.665 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:06:11.665 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:06:11.665 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:06:11.665 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:06:11.666 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:11.666 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:06:11.666 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:06:11.666 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:06:11.666 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:06:11.666 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:11.666 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:06:11.666 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:06:11.666 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:06:11.666 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:06:11.666 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:11.666 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:06:11.666 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:06:11.667 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:06:11.667 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:06:11.667 INFO:tasks.workunit.client.0.smithi039.stdout:start compare and write 2024-09-14T04:06:11.667 INFO:tasks.workunit.client.0.smithi039.stdout:compare and wrote: 512 2024-09-14T04:06:11.667 INFO:tasks.workunit.client.0.smithi039.stdout:start compare and write 2024-09-14T04:06:11.667 INFO:tasks.workunit.client.0.smithi039.stdout:compare and wrote: 512 2024-09-14T04:06:11.667 INFO:tasks.workunit.client.0.smithi039.stdout:start compare and write 2024-09-14T04:06:11.667 INFO:tasks.workunit.client.0.smithi039.stdout:compare and wrote: 512 2024-09-14T04:06:11.667 INFO:tasks.workunit.client.0.smithi039.stdout:start compare and write 2024-09-14T04:06:11.667 INFO:tasks.workunit.client.0.smithi039.stdout:compare and wrote: 512 2024-09-14T04:06:11.667 INFO:tasks.workunit.client.0.smithi039.stdout:start compare and write 2024-09-14T04:06:11.667 INFO:tasks.workunit.client.0.smithi039.stdout:compare and wrote: 512 2024-09-14T04:06:11.667 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:11.668 INFO:tasks.workunit.client.0.smithi039.stdout:started aio compare and write 2024-09-14T04:06:11.668 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:06:11.668 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:06:11.668 INFO:tasks.workunit.client.0.smithi039.stdout:finished aio compare and write 2024-09-14T04:06:11.668 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:11.668 INFO:tasks.workunit.client.0.smithi039.stdout:started aio compare and write 2024-09-14T04:06:11.668 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:06:11.668 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:06:11.668 INFO:tasks.workunit.client.0.smithi039.stdout:finished aio compare and write 2024-09-14T04:06:11.669 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:11.669 INFO:tasks.workunit.client.0.smithi039.stdout:started aio compare and write 2024-09-14T04:06:11.669 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:06:11.669 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:06:11.669 INFO:tasks.workunit.client.0.smithi039.stdout:finished aio compare and write 2024-09-14T04:06:11.669 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:11.669 INFO:tasks.workunit.client.0.smithi039.stdout:started aio compare and write 2024-09-14T04:06:11.669 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:06:11.669 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:06:11.669 INFO:tasks.workunit.client.0.smithi039.stdout:finished aio compare and write 2024-09-14T04:06:11.669 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:11.669 INFO:tasks.workunit.client.0.smithi039.stdout:started aio compare and write 2024-09-14T04:06:11.670 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:06:11.670 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:06:11.670 INFO:tasks.workunit.client.0.smithi039.stdout:finished aio compare and write 2024-09-14T04:06:11.670 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:06:11.670 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:06:11.670 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:06:11.670 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:06:11.670 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:06:11.670 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:11.670 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:06:11.671 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:06:11.671 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:06:11.671 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:11.671 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:06:11.671 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:06:11.671 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:06:11.671 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:11.671 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:06:11.671 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:06:11.671 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:06:11.671 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:11.671 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:06:11.671 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:06:11.672 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:06:11.672 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:11.672 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:06:11.672 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:06:11.672 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:06:11.672 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 512~512 = 512 2024-09-14T04:06:11.672 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:06:11.672 INFO:tasks.workunit.client.0.smithi039.stdout:aio discard: 1536~512 = 0 2024-09-14T04:06:11.672 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:06:11.672 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:06:11.672 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:06:11.672 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:06:11.672 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:06:11.673 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 0 2024-09-14T04:06:11.673 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:11.673 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:11.673 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 0 2024-09-14T04:06:11.673 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:11.673 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:11.673 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 16384 2024-09-14T04:06:11.673 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:11.673 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:11.673 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 16384 2024-09-14T04:06:11.674 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:11.674 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:11.674 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:06:11.674 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:06:11.674 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 49152 2024-09-14T04:06:11.674 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:11.675 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:11.675 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 49152 2024-09-14T04:06:11.675 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:11.675 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:11.675 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 65536 2024-09-14T04:06:11.675 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:11.676 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:11.676 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 65536 2024-09-14T04:06:11.676 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:11.676 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:11.676 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:06:11.676 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:06:11.677 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 98304 2024-09-14T04:06:11.677 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:11.677 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:11.677 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 98304 2024-09-14T04:06:11.677 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:11.677 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:11.678 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 114688 2024-09-14T04:06:11.678 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:11.678 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:11.678 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 114688 2024-09-14T04:06:11.679 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:11.679 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:11.679 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:06:11.679 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:06:11.679 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 147456 2024-09-14T04:06:11.680 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:11.680 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:11.680 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 147456 2024-09-14T04:06:11.680 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:11.680 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:11.680 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 163840 2024-09-14T04:06:11.680 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:11.681 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:11.681 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 163840 2024-09-14T04:06:11.681 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:11.681 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:11.681 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:06:11.681 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:06:11.681 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 196608 2024-09-14T04:06:11.681 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:11.682 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:11.682 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 196608 2024-09-14T04:06:11.682 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:11.682 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:11.682 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 212992 2024-09-14T04:06:11.682 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:11.682 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:11.682 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 212992 2024-09-14T04:06:11.682 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:11.682 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:11.682 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:06:11.683 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:06:11.683 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:11.683 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:06:11.683 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:06:11.683 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:06:11.683 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:06:11.683 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:11.683 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:11.683 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:11.683 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:06:11.683 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:06:11.683 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:06:11.684 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:06:11.684 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:11.685 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:11.685 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:11.685 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:06:11.685 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:06:11.685 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:06:11.685 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:06:11.685 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:11.685 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:11.685 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:11.685 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:06:11.685 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:06:11.686 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:06:11.686 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:06:11.686 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:11.686 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:11.686 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:11.686 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:06:11.686 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:06:11.686 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:11.686 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:06:11.686 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:06:11.686 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:11.686 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:06:11.686 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:06:11.687 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:06:11.687 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:06:11.687 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:11.687 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:11.687 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:11.687 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:06:11.687 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:06:11.687 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:06:11.687 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:06:11.687 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:11.687 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:11.687 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:12.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:12 smithi129 ceph-mon[101316]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-14T04:06:12.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:12 smithi129 ceph-mon[101316]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-14T04:06:12.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:12 smithi129 ceph-mon[101316]: pgmap v541: 521 pgs: 1 peering, 520 active+clean; 11 MiB data, 504 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:06:12.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:12 smithi039 ceph-mon[106470]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-14T04:06:12.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:12 smithi039 ceph-mon[106470]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-14T04:06:12.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:12 smithi039 ceph-mon[106470]: pgmap v541: 521 pgs: 1 peering, 520 active+clean; 11 MiB data, 504 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:06:12.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:12 smithi039 ceph-mon[111235]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-14T04:06:12.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:12 smithi039 ceph-mon[111235]: Upgrade: unsafe to stop osd(s) at this time (1 PGs are or would become offline) 2024-09-14T04:06:12.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:12 smithi039 ceph-mon[111235]: pgmap v541: 521 pgs: 1 peering, 520 active+clean; 11 MiB data, 504 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:06:14.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:14 smithi129 ceph-mon[101316]: pgmap v542: 521 pgs: 521 active+clean; 27 MiB data, 553 MiB used, 715 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.5 MiB/s wr, 877 op/s 2024-09-14T04:06:14.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:14 smithi039 ceph-mon[106470]: pgmap v542: 521 pgs: 521 active+clean; 27 MiB data, 553 MiB used, 715 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.5 MiB/s wr, 877 op/s 2024-09-14T04:06:14.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:14 smithi039 ceph-mon[111235]: pgmap v542: 521 pgs: 521 active+clean; 27 MiB data, 553 MiB used, 715 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.5 MiB/s wr, 877 op/s 2024-09-14T04:06:16.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:16 smithi129 ceph-mon[101316]: pgmap v543: 521 pgs: 521 active+clean; 27 MiB data, 553 MiB used, 715 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.5 MiB/s wr, 877 op/s 2024-09-14T04:06:16.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:16 smithi039 ceph-mon[106470]: pgmap v543: 521 pgs: 521 active+clean; 27 MiB data, 553 MiB used, 715 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.5 MiB/s wr, 877 op/s 2024-09-14T04:06:16.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:16 smithi039 ceph-mon[111235]: pgmap v543: 521 pgs: 521 active+clean; 27 MiB data, 553 MiB used, 715 GiB / 715 GiB avail; 3.6 MiB/s rd, 2.5 MiB/s wr, 877 op/s 2024-09-14T04:06:17.656 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:06:17.656 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:06:17.656 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:06:17.656 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:06:17.656 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:17.656 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:17.656 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:17.656 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:06:17.657 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:06:17.657 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:06:17.657 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:06:17.657 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:17.657 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:17.657 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:17.657 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:06:17.657 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:06:17.657 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:17.657 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:06:17.657 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:06:17.657 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:17.657 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:06:17.658 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:06:17.658 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:06:17.658 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:06:17.658 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:17.658 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:17.658 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:17.658 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:06:17.658 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:06:17.658 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:06:17.658 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:06:17.658 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:17.658 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:17.658 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:17.659 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:06:17.659 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:06:17.659 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:06:17.659 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:06:17.659 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:17.659 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:17.659 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:17.659 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:06:17.659 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:06:17.659 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:06:17.659 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:06:17.659 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:17.659 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:17.660 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:17.660 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:06:17.660 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:06:17.660 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:17.660 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:06:17.660 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:06:17.660 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:17.660 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:06:17.660 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:06:17.660 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:06:17.660 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:06:17.660 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:17.661 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:17.661 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:17.661 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:06:17.661 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:06:17.661 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:06:17.661 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:06:17.661 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:17.661 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:17.661 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:17.662 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:06:17.662 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:06:17.662 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:06:17.662 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:06:17.662 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:17.662 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:17.662 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:17.662 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:06:17.662 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:06:17.662 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:06:17.662 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:06:17.662 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:17.662 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:17.662 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:17.663 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:06:17.663 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:06:17.663 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:17.663 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:06:17.663 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:06:17.663 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:17.663 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:06:17.663 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:06:17.663 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:06:17.663 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:06:17.663 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:17.663 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:17.663 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:17.663 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:06:17.663 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:06:17.664 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:06:17.664 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:06:17.664 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:17.664 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:17.664 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:17.664 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:06:17.664 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:06:17.664 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:06:17.664 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:06:17.664 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:17.664 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:17.664 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:17.664 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:06:17.664 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:06:17.665 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:06:17.665 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:06:17.665 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:06:17.665 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:06:17.665 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:17.665 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:06:17.665 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:06:17.665 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:06:17.665 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:06:17.666 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:06:17.666 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:06:17.666 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:06:17.666 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:06:17.666 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:06:17.666 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 4 2024-09-14T04:06:17.666 INFO:tasks.workunit.client.0.smithi039.stdout:read: 4 2024-09-14T04:06:17.666 INFO:tasks.workunit.client.0.smithi039.stdout:read: 4 2024-09-14T04:06:17.666 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestEncryptionLUKS2 (21495 ms) 2024-09-14T04:06:17.666 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestCloneEncryption 2024-09-14T04:06:17.666 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:06:18.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:18 smithi129 ceph-mon[101316]: pgmap v544: 521 pgs: 521 active+clean; 27 MiB data, 553 MiB used, 715 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.0 MiB/s wr, 713 op/s 2024-09-14T04:06:18.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:18 smithi039 ceph-mon[106470]: pgmap v544: 521 pgs: 521 active+clean; 27 MiB data, 553 MiB used, 715 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.0 MiB/s wr, 713 op/s 2024-09-14T04:06:18.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:18 smithi039 ceph-mon[111235]: pgmap v544: 521 pgs: 521 active+clean; 27 MiB data, 553 MiB used, 715 GiB / 715 GiB avail; 2.9 MiB/s rd, 2.0 MiB/s wr, 713 op/s 2024-09-14T04:06:19.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:06:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:06:18] "GET /metrics HTTP/1.1" 200 67913 "" "Prometheus/2.51.0" 2024-09-14T04:06:19.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:19 smithi129 ceph-mon[101316]: osdmap e675: 8 total, 8 up, 8 in 2024-09-14T04:06:19.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:19 smithi039 ceph-mon[106470]: osdmap e675: 8 total, 8 up, 8 in 2024-09-14T04:06:19.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:19 smithi039 ceph-mon[111235]: osdmap e675: 8 total, 8 up, 8 in 2024-09-14T04:06:19.950 INFO:teuthology.orchestra.run.smithi039.stdout:true 2024-09-14T04:06:20.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:20 smithi039 ceph-mon[106470]: pgmap v546: 521 pgs: 521 active+clean; 31 MiB data, 553 MiB used, 715 GiB / 715 GiB avail; 4.4 MiB/s rd, 2.0 MiB/s wr, 875 op/s 2024-09-14T04:06:20.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:20 smithi039 ceph-mon[106470]: from='client.34539 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:06:20.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:20 smithi039 ceph-mon[111235]: pgmap v546: 521 pgs: 521 active+clean; 31 MiB data, 553 MiB used, 715 GiB / 715 GiB avail; 4.4 MiB/s rd, 2.0 MiB/s wr, 875 op/s 2024-09-14T04:06:20.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:20 smithi039 ceph-mon[111235]: from='client.34539 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:06:20.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:20 smithi129 ceph-mon[101316]: pgmap v546: 521 pgs: 521 active+clean; 31 MiB data, 553 MiB used, 715 GiB / 715 GiB avail; 4.4 MiB/s rd, 2.0 MiB/s wr, 875 op/s 2024-09-14T04:06:20.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:20 smithi129 ceph-mon[101316]: from='client.34539 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:06:20.544 INFO:teuthology.orchestra.run.smithi039.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T04:06:20.544 INFO:teuthology.orchestra.run.smithi039.stdout:alertmanager.a smithi039 *:9093,9094 running (8m) 29s ago 14m 19.7M - 0.27.0 11f11916f8cd 6b564e1fc909 2024-09-14T04:06:20.544 INFO:teuthology.orchestra.run.smithi039.stdout:grafana.a smithi129 *:3000 running (8m) 62s ago 14m 80.2M - 10.4.0 c8b91775d855 f0492c19de92 2024-09-14T04:06:20.545 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi039.bbwiwf smithi039 running (13m) 29s ago 13m 25.4M - 18.2.4-845-ge4e227b3 67ddd33fa72f 91e6c8f85681 2024-09-14T04:06:20.545 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi129.ybjqcw smithi129 running (13m) 62s ago 12m 16.9M - 18.2.4-845-ge4e227b3 67ddd33fa72f a3fab5b37b71 2024-09-14T04:06:20.545 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.x smithi129 *:8443,9283,8765 running (9m) 62s ago 18m 457M - 19.3.0-4907-ga2782315 8801d2529e43 448bd4e146cb 2024-09-14T04:06:20.545 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.y smithi039 *:8443,9283,8765 running (10m) 29s ago 19m 533M - 19.3.0-4907-ga2782315 8801d2529e43 cccc87f635a6 2024-09-14T04:06:20.545 INFO:teuthology.orchestra.run.smithi039.stdout:mon.a smithi039 running (7m) 29s ago 20m 76.2M 2048M 19.3.0-4907-ga2782315 8801d2529e43 7198d4eb8e4c 2024-09-14T04:06:20.545 INFO:teuthology.orchestra.run.smithi039.stdout:mon.b smithi129 running (5m) 62s ago 18m 48.8M 2048M 19.3.0-4907-ga2782315 8801d2529e43 cff2519560f1 2024-09-14T04:06:20.545 INFO:teuthology.orchestra.run.smithi039.stdout:mon.c smithi039 running (6m) 29s ago 18m 61.1M 2048M 19.3.0-4907-ga2782315 8801d2529e43 4b8f0f3864b1 2024-09-14T04:06:20.545 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.a smithi039 *:9100 running (9m) 29s ago 13m 11.1M - 1.7.0 72c9c2088986 7dd5945adb64 2024-09-14T04:06:20.545 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.b smithi129 *:9100 running (9m) 62s ago 13m 12.0M - 1.7.0 72c9c2088986 1fedb5f257e7 2024-09-14T04:06:20.545 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0 smithi039 running (34s) 29s ago 18m 12.2M 2480M 19.3.0-4907-ga2782315 8801d2529e43 36686aa8a5e4 2024-09-14T04:06:20.546 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1 smithi039 running (17m) 29s ago 17m 125M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 91349527d88b 2024-09-14T04:06:20.546 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2 smithi039 running (17m) 29s ago 17m 143M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f eb903b44fc5d 2024-09-14T04:06:20.546 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3 smithi039 running (16m) 29s ago 16m 142M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f dff9e665bd9b 2024-09-14T04:06:20.546 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4 smithi129 running (16m) 62s ago 16m 126M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 5b64727311b7 2024-09-14T04:06:20.546 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5 smithi129 running (16m) 62s ago 16m 116M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f f2d991a754f7 2024-09-14T04:06:20.546 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6 smithi129 running (15m) 62s ago 15m 136M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 3060eaf44106 2024-09-14T04:06:20.546 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7 smithi129 running (15m) 62s ago 15m 119M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 4c13b97060d1 2024-09-14T04:06:20.546 INFO:teuthology.orchestra.run.smithi039.stdout:prometheus.a smithi129 *:9095 running (8m) 62s ago 14m 58.1M - 2.51.0 1d3b7f56885b cbe583f852c3 2024-09-14T04:06:20.903 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:06:20.904 INFO:teuthology.orchestra.run.smithi039.stdout: "mon": { 2024-09-14T04:06:20.904 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T04:06:20.904 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:06:20.904 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr": { 2024-09-14T04:06:20.904 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T04:06:20.904 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:06:20.904 INFO:teuthology.orchestra.run.smithi039.stdout: "osd": { 2024-09-14T04:06:20.904 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 7, 2024-09-14T04:06:20.904 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 1 2024-09-14T04:06:20.904 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:06:20.905 INFO:teuthology.orchestra.run.smithi039.stdout: "mds": { 2024-09-14T04:06:20.905 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T04:06:20.905 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:06:20.905 INFO:teuthology.orchestra.run.smithi039.stdout: "overall": { 2024-09-14T04:06:20.905 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 9, 2024-09-14T04:06:20.905 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 6 2024-09-14T04:06:20.905 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:06:20.905 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:06:21.201 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:06:21.201 INFO:teuthology.orchestra.run.smithi039.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T04:06:21.202 INFO:teuthology.orchestra.run.smithi039.stdout: "in_progress": true, 2024-09-14T04:06:21.202 INFO:teuthology.orchestra.run.smithi039.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (3 remaining).", 2024-09-14T04:06:21.202 INFO:teuthology.orchestra.run.smithi039.stdout: "services_complete": [], 2024-09-14T04:06:21.202 INFO:teuthology.orchestra.run.smithi039.stdout: "progress": "1/8 daemons upgraded", 2024-09-14T04:06:21.202 INFO:teuthology.orchestra.run.smithi039.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T04:06:21.202 INFO:teuthology.orchestra.run.smithi039.stdout: "is_paused": false 2024-09-14T04:06:21.202 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:06:21.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:21 smithi129 ceph-mon[101316]: from='client.34545 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:06:21.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:21 smithi129 ceph-mon[101316]: from='client.34551 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:06:21.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:21 smithi129 ceph-mon[101316]: pgmap v547: 521 pgs: 521 active+clean; 31 MiB data, 553 MiB used, 715 GiB / 715 GiB avail; 4.4 MiB/s rd, 2.0 MiB/s wr, 875 op/s 2024-09-14T04:06:21.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:21 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/751226762' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:06:21.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:21 smithi039 ceph-mon[111235]: from='client.34545 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:06:21.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:21 smithi039 ceph-mon[111235]: from='client.34551 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:06:21.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:21 smithi039 ceph-mon[111235]: pgmap v547: 521 pgs: 521 active+clean; 31 MiB data, 553 MiB used, 715 GiB / 715 GiB avail; 4.4 MiB/s rd, 2.0 MiB/s wr, 875 op/s 2024-09-14T04:06:21.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:21 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/751226762' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:06:21.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:21 smithi039 ceph-mon[106470]: from='client.34545 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:06:21.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:21 smithi039 ceph-mon[106470]: from='client.34551 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:06:21.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:21 smithi039 ceph-mon[106470]: pgmap v547: 521 pgs: 521 active+clean; 31 MiB data, 553 MiB used, 715 GiB / 715 GiB avail; 4.4 MiB/s rd, 2.0 MiB/s wr, 875 op/s 2024-09-14T04:06:21.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:21 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/751226762' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:06:22.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:22 smithi129 ceph-mon[101316]: from='client.34563 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:06:22.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:22.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:22 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:22.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:22 smithi039 ceph-mon[111235]: from='client.34563 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:06:22.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:22.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:22 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:22.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:22 smithi039 ceph-mon[106470]: from='client.34563 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:06:22.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:22.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:22 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:23 smithi129 ceph-mon[101316]: pgmap v548: 521 pgs: 521 active+clean; 35 MiB data, 565 MiB used, 715 GiB / 715 GiB avail; 1.6 MiB/s rd, 475 KiB/s wr, 196 op/s 2024-09-14T04:06:23.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:23 smithi039 ceph-mon[106470]: pgmap v548: 521 pgs: 521 active+clean; 35 MiB data, 565 MiB used, 715 GiB / 715 GiB avail; 1.6 MiB/s rd, 475 KiB/s wr, 196 op/s 2024-09-14T04:06:23.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:23 smithi039 ceph-mon[111235]: pgmap v548: 521 pgs: 521 active+clean; 35 MiB data, 565 MiB used, 715 GiB / 715 GiB avail; 1.6 MiB/s rd, 475 KiB/s wr, 196 op/s 2024-09-14T04:06:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:25 smithi129 ceph-mon[101316]: osdmap e676: 8 total, 8 up, 8 in 2024-09-14T04:06:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:25 smithi129 ceph-mon[101316]: pgmap v550: 521 pgs: 521 active+clean; 35 MiB data, 565 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 593 KiB/s wr, 245 op/s 2024-09-14T04:06:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:25 smithi039 ceph-mon[106470]: osdmap e676: 8 total, 8 up, 8 in 2024-09-14T04:06:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:25 smithi039 ceph-mon[106470]: pgmap v550: 521 pgs: 521 active+clean; 35 MiB data, 565 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 593 KiB/s wr, 245 op/s 2024-09-14T04:06:25.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:25 smithi039 ceph-mon[111235]: osdmap e676: 8 total, 8 up, 8 in 2024-09-14T04:06:25.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:25 smithi039 ceph-mon[111235]: pgmap v550: 521 pgs: 521 active+clean; 35 MiB data, 565 MiB used, 715 GiB / 715 GiB avail; 2.0 MiB/s rd, 593 KiB/s wr, 245 op/s 2024-09-14T04:06:26.714 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:26 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-14T04:06:26.714 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:26 smithi039 ceph-mon[106470]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-14T04:06:26.714 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:26 smithi039 ceph-mon[106470]: Upgrade: osd.1 is safe to restart 2024-09-14T04:06:26.715 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:26 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-14T04:06:26.715 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:26 smithi039 ceph-mon[111235]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-14T04:06:26.715 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:26 smithi039 ceph-mon[111235]: Upgrade: osd.1 is safe to restart 2024-09-14T04:06:26.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:26 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16} : dispatch 2024-09-14T04:06:26.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:26 smithi129 ceph-mon[101316]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-09-14T04:06:26.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:26 smithi129 ceph-mon[101316]: Upgrade: osd.1 is safe to restart 2024-09-14T04:06:27.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:27 smithi129 ceph-mon[101316]: pgmap v551: 521 pgs: 521 active+clean; 35 MiB data, 565 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 469 KiB/s wr, 26 op/s 2024-09-14T04:06:27.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:27 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:27.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:27 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-14T04:06:27.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:27 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:06:27.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:27 smithi039 ceph-mon[106470]: pgmap v551: 521 pgs: 521 active+clean; 35 MiB data, 565 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 469 KiB/s wr, 26 op/s 2024-09-14T04:06:27.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:27 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:27.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:27 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-14T04:06:27.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:27 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:06:27.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:27 smithi039 ceph-mon[111235]: pgmap v551: 521 pgs: 521 active+clean; 35 MiB data, 565 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 469 KiB/s wr, 26 op/s 2024-09-14T04:06:27.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:27 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:27.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:27 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.1"} : dispatch 2024-09-14T04:06:27.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:27 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:06:28.522 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:28 smithi039 ceph-mon[106470]: Upgrade: Updating osd.1 2024-09-14T04:06:28.522 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:28 smithi039 ceph-mon[106470]: Deploying daemon osd.1 on smithi039 2024-09-14T04:06:28.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:06:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:06:28] "GET /metrics HTTP/1.1" 200 67906 "" "Prometheus/2.51.0" 2024-09-14T04:06:28.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:28 smithi039 ceph-mon[111235]: Upgrade: Updating osd.1 2024-09-14T04:06:28.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:28 smithi039 ceph-mon[111235]: Deploying daemon osd.1 on smithi039 2024-09-14T04:06:29.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:28 smithi129 ceph-mon[101316]: Upgrade: Updating osd.1 2024-09-14T04:06:29.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:28 smithi129 ceph-mon[101316]: Deploying daemon osd.1 on smithi039 2024-09-14T04:06:29.582 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:29 smithi039 ceph-mon[106470]: pgmap v552: 521 pgs: 521 active+clean; 35 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s rd, 412 KiB/s wr, 52 op/s 2024-09-14T04:06:29.582 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:29 smithi039 ceph-mon[111235]: pgmap v552: 521 pgs: 521 active+clean; 35 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s rd, 412 KiB/s wr, 52 op/s 2024-09-14T04:06:30.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:29 smithi129 ceph-mon[101316]: pgmap v552: 521 pgs: 521 active+clean; 35 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s rd, 412 KiB/s wr, 52 op/s 2024-09-14T04:06:31.121 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:30 smithi039 systemd[1]: Stopping Ceph osd.1 for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T04:06:31.375 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1[51249]: 2024-09-14T04:06:31.192+0000 7f2a83c93640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:06:31.375 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1[51249]: 2024-09-14T04:06:31.192+0000 7f2a83c93640 -1 osd.1 676 *** Got signal Terminated *** 2024-09-14T04:06:31.375 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:31 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1[51249]: 2024-09-14T04:06:31.192+0000 7f2a83c93640 -1 osd.1 676 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T04:06:32.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:31 smithi129 ceph-mon[101316]: pgmap v553: 521 pgs: 521 active+clean; 35 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s rd, 412 KiB/s wr, 52 op/s 2024-09-14T04:06:32.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:31 smithi129 ceph-mon[101316]: osd.1 marked itself down and dead 2024-09-14T04:06:32.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:31 smithi039 ceph-mon[106470]: pgmap v553: 521 pgs: 521 active+clean; 35 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s rd, 412 KiB/s wr, 52 op/s 2024-09-14T04:06:32.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:31 smithi039 ceph-mon[106470]: osd.1 marked itself down and dead 2024-09-14T04:06:32.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:31 smithi039 ceph-mon[111235]: pgmap v553: 521 pgs: 521 active+clean; 35 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 69 KiB/s rd, 412 KiB/s wr, 52 op/s 2024-09-14T04:06:32.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:31 smithi039 ceph-mon[111235]: osd.1 marked itself down and dead 2024-09-14T04:06:32.593 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:32 smithi039 podman[130355]: 2024-09-14 04:06:32.591562561 +0000 UTC m=+1.572110916 container died 91349527d88bfe29642058751bc55482fe8247b9e4a9932622b498bed7b570b9 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=reef-e4e227b, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-09-14T04:06:33.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:32 smithi129 ceph-mon[101316]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T04:06:33.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:32 smithi129 ceph-mon[101316]: osdmap e677: 8 total, 7 up, 8 in 2024-09-14T04:06:33.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:32 smithi129 ceph-mon[101316]: osd.1 marked itself dead as of e677 2024-09-14T04:06:33.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:32 smithi039 ceph-mon[106470]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T04:06:33.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:32 smithi039 ceph-mon[106470]: osdmap e677: 8 total, 7 up, 8 in 2024-09-14T04:06:33.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:32 smithi039 ceph-mon[106470]: osd.1 marked itself dead as of e677 2024-09-14T04:06:33.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:32 smithi039 ceph-mon[111235]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T04:06:33.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:32 smithi039 ceph-mon[111235]: osdmap e677: 8 total, 7 up, 8 in 2024-09-14T04:06:33.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:32 smithi039 ceph-mon[111235]: osd.1 marked itself dead as of e677 2024-09-14T04:06:33.291 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:33 smithi039 podman[130355]: 2024-09-14 04:06:33.1145486 +0000 UTC m=+2.095096951 container cleanup 91349527d88bfe29642058751bc55482fe8247b9e4a9932622b498bed7b570b9 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=reef-e4e227b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-14T04:06:33.291 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:33 smithi039 bash[130355]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1 2024-09-14T04:06:33.751 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:33 smithi039 podman[130453]: 2024-09-14 04:06:33.31520936 +0000 UTC m=+0.720993390 container remove 91349527d88bfe29642058751bc55482fe8247b9e4a9932622b498bed7b570b9 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1, RELEASE=reef-e4e227b, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T04:06:34.020 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:33 smithi039 ceph-mon[106470]: pgmap v555: 521 pgs: 189 peering, 332 active+clean; 51 MiB data, 602 MiB used, 715 GiB / 715 GiB avail; 7.9 KiB/s rd, 1.9 MiB/s wr, 14 op/s 2024-09-14T04:06:34.020 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:33 smithi039 ceph-mon[106470]: Health check failed: Reduced data availability: 31 pgs inactive, 56 pgs peering (PG_AVAILABILITY) 2024-09-14T04:06:34.020 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:33 smithi039 ceph-mon[106470]: osdmap e678: 8 total, 7 up, 8 in 2024-09-14T04:06:34.021 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:33 smithi039 ceph-mon[111235]: pgmap v555: 521 pgs: 189 peering, 332 active+clean; 51 MiB data, 602 MiB used, 715 GiB / 715 GiB avail; 7.9 KiB/s rd, 1.9 MiB/s wr, 14 op/s 2024-09-14T04:06:34.021 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:33 smithi039 ceph-mon[111235]: Health check failed: Reduced data availability: 31 pgs inactive, 56 pgs peering (PG_AVAILABILITY) 2024-09-14T04:06:34.021 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:33 smithi039 ceph-mon[111235]: osdmap e678: 8 total, 7 up, 8 in 2024-09-14T04:06:34.021 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:33 smithi039 podman[130573]: 2024-09-14 04:06:33.750261018 +0000 UTC m=+0.119989014 container create 5d671bab7be28956c3a97904752e945007cb5a56560294dbe18daf9bfb8cad74 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731) 2024-09-14T04:06:34.021 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:33 smithi039 podman[130573]: 2024-09-14 04:06:33.653301282 +0000 UTC m=+0.023029280 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:06:34.021 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:34 smithi039 podman[130573]: 2024-09-14 04:06:34.019785183 +0000 UTC m=+0.389513189 container init 5d671bab7be28956c3a97904752e945007cb5a56560294dbe18daf9bfb8cad74 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=main-a278231, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-14T04:06:34.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:33 smithi129 ceph-mon[101316]: pgmap v555: 521 pgs: 189 peering, 332 active+clean; 51 MiB data, 602 MiB used, 715 GiB / 715 GiB avail; 7.9 KiB/s rd, 1.9 MiB/s wr, 14 op/s 2024-09-14T04:06:34.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:33 smithi129 ceph-mon[101316]: Health check failed: Reduced data availability: 31 pgs inactive, 56 pgs peering (PG_AVAILABILITY) 2024-09-14T04:06:34.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:33 smithi129 ceph-mon[101316]: osdmap e678: 8 total, 7 up, 8 in 2024-09-14T04:06:34.290 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:34 smithi039 podman[130573]: 2024-09-14 04:06:34.02491687 +0000 UTC m=+0.394644871 container start 5d671bab7be28956c3a97904752e945007cb5a56560294dbe18daf9bfb8cad74 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_BRANCH=HEAD, RELEASE=main-a278231, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T04:06:34.290 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:34 smithi039 podman[130573]: 2024-09-14 04:06:34.119706973 +0000 UTC m=+0.489434990 container attach 5d671bab7be28956c3a97904752e945007cb5a56560294dbe18daf9bfb8cad74 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, CEPH_POINT_RELEASE=) 2024-09-14T04:06:34.290 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:34 smithi039 podman[130573]: 2024-09-14 04:06:34.222358965 +0000 UTC m=+0.592086962 container died 5d671bab7be28956c3a97904752e945007cb5a56560294dbe18daf9bfb8cad74 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=main-a278231) 2024-09-14T04:06:35.290 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:34 smithi039 podman[130616]: 2024-09-14 04:06:34.865748452 +0000 UTC m=+0.634228646 container remove 5d671bab7be28956c3a97904752e945007cb5a56560294dbe18daf9bfb8cad74 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731) 2024-09-14T04:06:35.290 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:34 smithi039 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.1.service: Deactivated successfully. 2024-09-14T04:06:35.290 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:34 smithi039 systemd[1]: Stopped Ceph osd.1 for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T04:06:35.290 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:34 smithi039 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.1.service: Consumed 34.641s CPU time. 2024-09-14T04:06:35.291 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:35 smithi039 systemd[1]: Starting Ceph osd.1 for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T04:06:35.595 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:35 smithi039 podman[130721]: 2024-09-14 04:06:35.211405168 +0000 UTC m=+0.016212867 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:06:35.595 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:35 smithi039 podman[130721]: 2024-09-14 04:06:35.316872331 +0000 UTC m=+0.121680029 container create dbdcd2d4f9fd09e8f9284fa3b684964a8c39ed4651093f1b6bc4c366eee0796b (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, RELEASE=main-a278231, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-09-14T04:06:35.957 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:35 smithi039 podman[130721]: 2024-09-14 04:06:35.594836794 +0000 UTC m=+0.399644492 container init dbdcd2d4f9fd09e8f9284fa3b684964a8c39ed4651093f1b6bc4c366eee0796b (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1-activate, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux ) 2024-09-14T04:06:35.957 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:35 smithi039 podman[130721]: 2024-09-14 04:06:35.599685271 +0000 UTC m=+0.404492972 container start dbdcd2d4f9fd09e8f9284fa3b684964a8c39ed4651093f1b6bc4c366eee0796b (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1-activate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, RELEASE=main-a278231, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T04:06:35.957 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:35 smithi039 podman[130721]: 2024-09-14 04:06:35.686361591 +0000 UTC m=+0.491169306 container attach dbdcd2d4f9fd09e8f9284fa3b684964a8c39ed4651093f1b6bc4c366eee0796b (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1-activate, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-14T04:06:35.957 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:35 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1-activate[130735]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:06:35.958 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:35 smithi039 bash[130721]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:06:35.958 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:35 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1-activate[130735]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:06:35.958 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:35 smithi039 bash[130721]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:06:36.210 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:35 smithi039 ceph-mon[106470]: pgmap v557: 521 pgs: 189 peering, 332 active+clean; 51 MiB data, 602 MiB used, 715 GiB / 715 GiB avail; 8.2 KiB/s rd, 2.0 MiB/s wr, 14 op/s 2024-09-14T04:06:36.211 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:35 smithi039 ceph-mon[111235]: pgmap v557: 521 pgs: 189 peering, 332 active+clean; 51 MiB data, 602 MiB used, 715 GiB / 715 GiB avail; 8.2 KiB/s rd, 2.0 MiB/s wr, 14 op/s 2024-09-14T04:06:36.211 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:36 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1-activate[130735]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-14T04:06:36.211 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:36 smithi039 bash[130721]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-14T04:06:36.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:35 smithi129 ceph-mon[101316]: pgmap v557: 521 pgs: 189 peering, 332 active+clean; 51 MiB data, 602 MiB used, 715 GiB / 715 GiB avail; 8.2 KiB/s rd, 2.0 MiB/s wr, 14 op/s 2024-09-14T04:06:36.511 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:36 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1-activate[130735]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-14T04:06:36.511 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:36 smithi039 bash[130721]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-09-14T04:06:36.511 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:36 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1-activate[130735]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-14T04:06:36.511 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:36 smithi039 bash[130721]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-09-14T04:06:36.790 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:36 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1-activate[130735]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-14T04:06:36.790 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:36 smithi039 bash[130721]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-09-14T04:06:36.791 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:36 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1-activate[130735]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-14T04:06:36.791 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:36 smithi039 bash[130721]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-09-14T04:06:36.791 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:36 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1-activate[130735]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-14T04:06:36.791 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:36 smithi039 bash[130721]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-09-14T04:06:36.791 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:36 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1-activate[130735]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-14T04:06:36.791 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:36 smithi039 bash[130721]: --> ceph-volume raw activate successful for osd ID: 1 2024-09-14T04:06:36.791 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:36 smithi039 podman[130721]: 2024-09-14 04:06:36.5386934 +0000 UTC m=+1.343501107 container died dbdcd2d4f9fd09e8f9284fa3b684964a8c39ed4651093f1b6bc4c366eee0796b (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1-activate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, ceph=True, io.buildah.version=1.37.2) 2024-09-14T04:06:37.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:36 smithi129 ceph-mon[101316]: osdmap e679: 8 total, 7 up, 8 in 2024-09-14T04:06:37.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:36 smithi039 ceph-mon[106470]: osdmap e679: 8 total, 7 up, 8 in 2024-09-14T04:06:37.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:36 smithi039 ceph-mon[111235]: osdmap e679: 8 total, 7 up, 8 in 2024-09-14T04:06:37.735 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:37 smithi039 podman[130890]: 2024-09-14 04:06:37.441411422 +0000 UTC m=+0.896080821 container remove dbdcd2d4f9fd09e8f9284fa3b684964a8c39ed4651093f1b6bc4c366eee0796b (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1-activate, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0) 2024-09-14T04:06:37.735 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:37 smithi039 podman[131160]: 2024-09-14 04:06:37.585171182 +0000 UTC m=+0.019916643 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:06:37.735 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:37 smithi039 podman[131160]: 2024-09-14 04:06:37.734276622 +0000 UTC m=+0.169022082 container create 6ff655878a341c2da1c4e7814b7b7b756a17e375213c9f2782fe3ee4d5e7aca5 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, ceph=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2) 2024-09-14T04:06:38.002 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:37 smithi039 ceph-mon[106470]: pgmap v559: 521 pgs: 189 peering, 332 active+clean; 51 MiB data, 602 MiB used, 715 GiB / 715 GiB avail; 2.7 MiB/s wr, 2 op/s 2024-09-14T04:06:38.002 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:38.002 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:38.003 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:37 smithi039 ceph-mon[111235]: pgmap v559: 521 pgs: 189 peering, 332 active+clean; 51 MiB data, 602 MiB used, 715 GiB / 715 GiB avail; 2.7 MiB/s wr, 2 op/s 2024-09-14T04:06:38.003 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:38.003 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:38.003 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:37 smithi039 podman[131160]: 2024-09-14 04:06:37.970171577 +0000 UTC m=+0.404917045 container init 6ff655878a341c2da1c4e7814b7b7b756a17e375213c9f2782fe3ee4d5e7aca5 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1, org.label-schema.schema-version=1.0, RELEASE=main-a278231, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T04:06:38.003 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:37 smithi039 podman[131160]: 2024-09-14 04:06:37.977413896 +0000 UTC m=+0.412159365 container start 6ff655878a341c2da1c4e7814b7b7b756a17e375213c9f2782fe3ee4d5e7aca5 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=main-a278231, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-14T04:06:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:37 smithi129 ceph-mon[101316]: pgmap v559: 521 pgs: 189 peering, 332 active+clean; 51 MiB data, 602 MiB used, 715 GiB / 715 GiB avail; 2.7 MiB/s wr, 2 op/s 2024-09-14T04:06:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:38.290 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:38 smithi039 ceph-osd[131306]: -- 172.21.15.39:0/1880865454 <== mon.2 v2:172.21.15.39:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x559e065c3a40 con 0x559e07392000 2024-09-14T04:06:38.290 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:38 smithi039 bash[131160]: 6ff655878a341c2da1c4e7814b7b7b756a17e375213c9f2782fe3ee4d5e7aca5 2024-09-14T04:06:38.290 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:38 smithi039 systemd[1]: Started Ceph osd.1 for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T04:06:39.022 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:06:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:06:38] "GET /metrics HTTP/1.1" 200 67993 "" "Prometheus/2.51.0" 2024-09-14T04:06:39.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:39 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:39.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:39 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:39.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:39 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:39.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:39 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:06:39.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:39 smithi039 ceph-mon[106470]: pgmap v560: 521 pgs: 132 active+undersized, 57 active+undersized+degraded, 332 active+clean; 55 MiB data, 618 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 2.9 MiB/s wr, 27 op/s; 89/738 objects degraded (12.060%) 2024-09-14T04:06:39.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:39 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:39.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:39 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:39.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:39 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:39.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:39 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:06:39.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:39 smithi039 ceph-mon[111235]: pgmap v560: 521 pgs: 132 active+undersized, 57 active+undersized+degraded, 332 active+clean; 55 MiB data, 618 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 2.9 MiB/s wr, 27 op/s; 89/738 objects degraded (12.060%) 2024-09-14T04:06:39.498 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:39 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:39.499 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:39 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:39.499 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:39 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:39.499 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:39 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:06:39.499 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:39 smithi129 ceph-mon[101316]: pgmap v560: 521 pgs: 132 active+undersized, 57 active+undersized+degraded, 332 active+clean; 55 MiB data, 618 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 2.9 MiB/s wr, 27 op/s; 89/738 objects degraded (12.060%) 2024-09-14T04:06:40.137 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:39 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1[131251]: 2024-09-14T04:06:39.863+0000 7f23bcab0640 -1 Falling back to public interface 2024-09-14T04:06:40.137 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:40 smithi039 ceph-mon[106470]: Health check failed: Degraded data redundancy: 89/738 objects degraded (12.060%), 57 pgs degraded (PG_DEGRADED) 2024-09-14T04:06:40.137 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:40 smithi039 ceph-mon[106470]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 31 pgs inactive, 58 pgs peering) 2024-09-14T04:06:40.512 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:40 smithi129 ceph-mon[101316]: Health check failed: Degraded data redundancy: 89/738 objects degraded (12.060%), 57 pgs degraded (PG_DEGRADED) 2024-09-14T04:06:40.512 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:40 smithi129 ceph-mon[101316]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 31 pgs inactive, 58 pgs peering) 2024-09-14T04:06:40.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:40 smithi039 ceph-mon[111235]: Health check failed: Degraded data redundancy: 89/738 objects degraded (12.060%), 57 pgs degraded (PG_DEGRADED) 2024-09-14T04:06:40.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:40 smithi039 ceph-mon[111235]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 31 pgs inactive, 58 pgs peering) 2024-09-14T04:06:41.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:41 smithi129 ceph-mon[101316]: pgmap v561: 521 pgs: 132 active+undersized, 57 active+undersized+degraded, 332 active+clean; 55 MiB data, 618 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 514 KiB/s wr, 45 op/s; 89/738 objects degraded (12.060%) 2024-09-14T04:06:41.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:41 smithi039 ceph-mon[106470]: pgmap v561: 521 pgs: 132 active+undersized, 57 active+undersized+degraded, 332 active+clean; 55 MiB data, 618 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 514 KiB/s wr, 45 op/s; 89/738 objects degraded (12.060%) 2024-09-14T04:06:41.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:41 smithi039 ceph-mon[111235]: pgmap v561: 521 pgs: 132 active+undersized, 57 active+undersized+degraded, 332 active+clean; 55 MiB data, 618 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 514 KiB/s wr, 45 op/s; 89/738 objects degraded (12.060%) 2024-09-14T04:06:43.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:43 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:43.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:43 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:43.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:43 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:43.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:43 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:43.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:43 smithi129 ceph-mon[101316]: pgmap v562: 521 pgs: 132 active+undersized, 57 active+undersized+degraded, 332 active+clean; 71 MiB data, 654 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s rd, 2.0 MiB/s wr, 41 op/s; 92/750 objects degraded (12.267%) 2024-09-14T04:06:43.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:43 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:43.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:43 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:43.540 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:43 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1[131251]: 2024-09-14T04:06:43.329+0000 7f23bcab0640 -1 osd.1 677 log_to_monitors true 2024-09-14T04:06:43.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:43 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:43.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:43 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:43.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:43 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:43.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:43 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:43.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:43 smithi039 ceph-mon[106470]: pgmap v562: 521 pgs: 132 active+undersized, 57 active+undersized+degraded, 332 active+clean; 71 MiB data, 654 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s rd, 2.0 MiB/s wr, 41 op/s; 92/750 objects degraded (12.267%) 2024-09-14T04:06:43.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:43 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:43.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:43 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:43.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:43 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:43.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:43 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:43.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:43 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:43.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:43 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:43.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:43 smithi039 ceph-mon[111235]: pgmap v562: 521 pgs: 132 active+undersized, 57 active+undersized+degraded, 332 active+clean; 71 MiB data, 654 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s rd, 2.0 MiB/s wr, 41 op/s; 92/750 objects degraded (12.267%) 2024-09-14T04:06:43.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:43 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:43.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:43 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:44.221 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:44 smithi039 ceph-mon[106470]: from='osd.1 [v2:172.21.15.39:6810/1186072955,v1:172.21.15.39:6811/1186072955]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-14T04:06:44.222 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:44 smithi039 ceph-mon[111235]: from='osd.1 [v2:172.21.15.39:6810/1186072955,v1:172.21.15.39:6811/1186072955]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-14T04:06:44.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:44 smithi129 ceph-mon[101316]: from='osd.1 [v2:172.21.15.39:6810/1186072955,v1:172.21.15.39:6811/1186072955]' entity='osd.1' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]} : dispatch 2024-09-14T04:06:45.040 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 04:06:44 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1[131251]: 2024-09-14T04:06:44.777+0000 7f23b0852640 -1 osd.1 677 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T04:06:45.390 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:45 smithi039 ceph-mon[106470]: from='osd.1 [v2:172.21.15.39:6810/1186072955,v1:172.21.15.39:6811/1186072955]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-14T04:06:45.390 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:45 smithi039 ceph-mon[106470]: osdmap e680: 8 total, 7 up, 8 in 2024-09-14T04:06:45.390 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:45 smithi039 ceph-mon[106470]: from='osd.1 [v2:172.21.15.39:6810/1186072955,v1:172.21.15.39:6811/1186072955]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi039", "root=default"]} : dispatch 2024-09-14T04:06:45.391 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:45 smithi039 ceph-mon[106470]: from='osd.1 [v2:172.21.15.39:6810/1186072955,v1:172.21.15.39:6811/1186072955]' entity='osd.1' 2024-09-14T04:06:45.391 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:45 smithi039 ceph-mon[106470]: pgmap v564: 521 pgs: 132 active+undersized, 57 active+undersized+degraded, 332 active+clean; 71 MiB data, 637 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 2.2 MiB/s wr, 45 op/s; 92/750 objects degraded (12.267%) 2024-09-14T04:06:45.391 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:45 smithi039 ceph-mon[111235]: from='osd.1 [v2:172.21.15.39:6810/1186072955,v1:172.21.15.39:6811/1186072955]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-14T04:06:45.391 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:45 smithi039 ceph-mon[111235]: osdmap e680: 8 total, 7 up, 8 in 2024-09-14T04:06:45.391 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:45 smithi039 ceph-mon[111235]: from='osd.1 [v2:172.21.15.39:6810/1186072955,v1:172.21.15.39:6811/1186072955]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi039", "root=default"]} : dispatch 2024-09-14T04:06:45.391 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:45 smithi039 ceph-mon[111235]: from='osd.1 [v2:172.21.15.39:6810/1186072955,v1:172.21.15.39:6811/1186072955]' entity='osd.1' 2024-09-14T04:06:45.391 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:45 smithi039 ceph-mon[111235]: pgmap v564: 521 pgs: 132 active+undersized, 57 active+undersized+degraded, 332 active+clean; 71 MiB data, 637 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 2.2 MiB/s wr, 45 op/s; 92/750 objects degraded (12.267%) 2024-09-14T04:06:45.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:45 smithi129 ceph-mon[101316]: from='osd.1 [v2:172.21.15.39:6810/1186072955,v1:172.21.15.39:6811/1186072955]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-09-14T04:06:45.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:45 smithi129 ceph-mon[101316]: osdmap e680: 8 total, 7 up, 8 in 2024-09-14T04:06:45.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:45 smithi129 ceph-mon[101316]: from='osd.1 [v2:172.21.15.39:6810/1186072955,v1:172.21.15.39:6811/1186072955]' entity='osd.1' cmd={"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi039", "root=default"]} : dispatch 2024-09-14T04:06:45.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:45 smithi129 ceph-mon[101316]: from='osd.1 [v2:172.21.15.39:6810/1186072955,v1:172.21.15.39:6811/1186072955]' entity='osd.1' 2024-09-14T04:06:45.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:45 smithi129 ceph-mon[101316]: pgmap v564: 521 pgs: 132 active+undersized, 57 active+undersized+degraded, 332 active+clean; 71 MiB data, 637 MiB used, 715 GiB / 715 GiB avail; 100 KiB/s rd, 2.2 MiB/s wr, 45 op/s; 92/750 objects degraded (12.267%) 2024-09-14T04:06:46.398 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:46 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 92/750 objects degraded (12.267%), 57 pgs degraded (PG_DEGRADED) 2024-09-14T04:06:46.398 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:46 smithi039 ceph-mon[106470]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T04:06:46.398 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:46 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-14T04:06:46.398 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:46 smithi039 ceph-mon[106470]: osd.1 [v2:172.21.15.39:6810/1186072955,v1:172.21.15.39:6811/1186072955] boot 2024-09-14T04:06:46.398 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:46 smithi039 ceph-mon[106470]: osdmap e681: 8 total, 8 up, 8 in 2024-09-14T04:06:46.398 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:46 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 92/750 objects degraded (12.267%), 57 pgs degraded (PG_DEGRADED) 2024-09-14T04:06:46.398 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:46 smithi039 ceph-mon[111235]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T04:06:46.399 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:46 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-14T04:06:46.399 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:46 smithi039 ceph-mon[111235]: osd.1 [v2:172.21.15.39:6810/1186072955,v1:172.21.15.39:6811/1186072955] boot 2024-09-14T04:06:46.399 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:46 smithi039 ceph-mon[111235]: osdmap e681: 8 total, 8 up, 8 in 2024-09-14T04:06:46.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:46 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 92/750 objects degraded (12.267%), 57 pgs degraded (PG_DEGRADED) 2024-09-14T04:06:46.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:46 smithi129 ceph-mon[101316]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T04:06:46.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:46 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 1} : dispatch 2024-09-14T04:06:46.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:46 smithi129 ceph-mon[101316]: osd.1 [v2:172.21.15.39:6810/1186072955,v1:172.21.15.39:6811/1186072955] boot 2024-09-14T04:06:46.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:46 smithi129 ceph-mon[101316]: osdmap e681: 8 total, 8 up, 8 in 2024-09-14T04:06:47.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:06:47 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T04:06:47.002+0000 7f268b1da640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (64 PGs are or would become offline) 2024-09-14T04:06:48.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:47 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:48.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:47 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:48.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:47 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:06:48.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:47 smithi039 ceph-mon[106470]: osdmap e682: 8 total, 8 up, 8 in 2024-09-14T04:06:48.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:47 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:06:48.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:47 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:48.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:47 smithi039 ceph-mon[106470]: pgmap v567: 521 pgs: 132 active+undersized, 57 active+undersized+degraded, 332 active+clean; 71 MiB data, 637 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 2.7 MiB/s wr, 7 op/s; 92/750 objects degraded (12.267%) 2024-09-14T04:06:48.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:47 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:06:48.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:47 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:06:48.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:47 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:06:48.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:47 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:06:48.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:47 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T04:06:48.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:47 smithi039 ceph-mon[106470]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T04:06:48.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:47 smithi039 ceph-mon[106470]: Upgrade: unsafe to stop osd(s) at this time (64 PGs are or would become offline) 2024-09-14T04:06:48.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:47 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:48.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:47 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:48.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:47 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:06:48.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:47 smithi039 ceph-mon[111235]: osdmap e682: 8 total, 8 up, 8 in 2024-09-14T04:06:48.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:47 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:06:48.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:47 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:48.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:47 smithi039 ceph-mon[111235]: pgmap v567: 521 pgs: 132 active+undersized, 57 active+undersized+degraded, 332 active+clean; 71 MiB data, 637 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 2.7 MiB/s wr, 7 op/s; 92/750 objects degraded (12.267%) 2024-09-14T04:06:48.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:47 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:06:48.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:47 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:06:48.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:47 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:06:48.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:47 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:06:48.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:47 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T04:06:48.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:47 smithi039 ceph-mon[111235]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T04:06:48.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:47 smithi039 ceph-mon[111235]: Upgrade: unsafe to stop osd(s) at this time (64 PGs are or would become offline) 2024-09-14T04:06:48.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:47 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:48.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:47 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:48.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:47 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:06:48.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:47 smithi129 ceph-mon[101316]: osdmap e682: 8 total, 8 up, 8 in 2024-09-14T04:06:48.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:47 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:06:48.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:47 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:48.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:47 smithi129 ceph-mon[101316]: pgmap v567: 521 pgs: 132 active+undersized, 57 active+undersized+degraded, 332 active+clean; 71 MiB data, 637 MiB used, 715 GiB / 715 GiB avail; 51 KiB/s rd, 2.7 MiB/s wr, 7 op/s; 92/750 objects degraded (12.267%) 2024-09-14T04:06:48.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:47 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:06:48.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:47 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:06:48.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:47 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:06:48.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:47 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:06:48.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:47 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T04:06:48.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:47 smithi129 ceph-mon[101316]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T04:06:48.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:47 smithi129 ceph-mon[101316]: Upgrade: unsafe to stop osd(s) at this time (64 PGs are or would become offline) 2024-09-14T04:06:49.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:06:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:06:48] "GET /metrics HTTP/1.1" 200 67993 "" "Prometheus/2.51.0" 2024-09-14T04:06:50.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:49 smithi129 ceph-mon[101316]: pgmap v568: 521 pgs: 50 active+undersized, 27 active+undersized+degraded, 444 active+clean; 71 MiB data, 642 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 0 op/s; 47/750 objects degraded (6.267%); 1.3 MiB/s, 0 objects/s recovering 2024-09-14T04:06:50.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:49 smithi039 ceph-mon[106470]: pgmap v568: 521 pgs: 50 active+undersized, 27 active+undersized+degraded, 444 active+clean; 71 MiB data, 642 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 0 op/s; 47/750 objects degraded (6.267%); 1.3 MiB/s, 0 objects/s recovering 2024-09-14T04:06:50.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:49 smithi039 ceph-mon[111235]: pgmap v568: 521 pgs: 50 active+undersized, 27 active+undersized+degraded, 444 active+clean; 71 MiB data, 642 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 0 op/s; 47/750 objects degraded (6.267%); 1.3 MiB/s, 0 objects/s recovering 2024-09-14T04:06:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:50 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 47/750 objects degraded (6.267%), 27 pgs degraded (PG_DEGRADED) 2024-09-14T04:06:51.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:50 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 47/750 objects degraded (6.267%), 27 pgs degraded (PG_DEGRADED) 2024-09-14T04:06:51.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:50 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 47/750 objects degraded (6.267%), 27 pgs degraded (PG_DEGRADED) 2024-09-14T04:06:51.515 INFO:teuthology.orchestra.run.smithi039.stdout:true 2024-09-14T04:06:52.102 INFO:teuthology.orchestra.run.smithi039.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T04:06:52.102 INFO:teuthology.orchestra.run.smithi039.stdout:alertmanager.a smithi039 *:9093,9094 running (9m) 9s ago 14m 19.7M - 0.27.0 11f11916f8cd 6b564e1fc909 2024-09-14T04:06:52.102 INFO:teuthology.orchestra.run.smithi039.stdout:grafana.a smithi129 *:3000 running (8m) 9s ago 14m 80.5M - 10.4.0 c8b91775d855 f0492c19de92 2024-09-14T04:06:52.102 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi039.bbwiwf smithi039 running (13m) 9s ago 13m 25.7M - 18.2.4-845-ge4e227b3 67ddd33fa72f 91e6c8f85681 2024-09-14T04:06:52.102 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi129.ybjqcw smithi129 running (13m) 9s ago 13m 17.3M - 18.2.4-845-ge4e227b3 67ddd33fa72f a3fab5b37b71 2024-09-14T04:06:52.103 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.x smithi129 *:8443,9283,8765 running (10m) 9s ago 18m 458M - 19.3.0-4907-ga2782315 8801d2529e43 448bd4e146cb 2024-09-14T04:06:52.103 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.y smithi039 *:8443,9283,8765 running (10m) 9s ago 20m 536M - 19.3.0-4907-ga2782315 8801d2529e43 cccc87f635a6 2024-09-14T04:06:52.103 INFO:teuthology.orchestra.run.smithi039.stdout:mon.a smithi039 running (7m) 9s ago 20m 80.0M 2048M 19.3.0-4907-ga2782315 8801d2529e43 7198d4eb8e4c 2024-09-14T04:06:52.103 INFO:teuthology.orchestra.run.smithi039.stdout:mon.b smithi129 running (5m) 9s ago 19m 54.6M 2048M 19.3.0-4907-ga2782315 8801d2529e43 cff2519560f1 2024-09-14T04:06:52.103 INFO:teuthology.orchestra.run.smithi039.stdout:mon.c smithi039 running (7m) 9s ago 19m 64.7M 2048M 19.3.0-4907-ga2782315 8801d2529e43 4b8f0f3864b1 2024-09-14T04:06:52.103 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.a smithi039 *:9100 running (9m) 9s ago 14m 11.1M - 1.7.0 72c9c2088986 7dd5945adb64 2024-09-14T04:06:52.103 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.b smithi129 *:9100 running (9m) 9s ago 14m 11.5M - 1.7.0 72c9c2088986 1fedb5f257e7 2024-09-14T04:06:52.103 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0 smithi039 running (66s) 9s ago 18m 58.3M 2480M 19.3.0-4907-ga2782315 8801d2529e43 36686aa8a5e4 2024-09-14T04:06:52.103 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1 smithi039 running (14s) 9s ago 18m 12.2M 2480M 19.3.0-4907-ga2782315 8801d2529e43 6ff655878a34 2024-09-14T04:06:52.103 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2 smithi039 running (17m) 9s ago 17m 143M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f eb903b44fc5d 2024-09-14T04:06:52.103 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3 smithi039 running (17m) 9s ago 17m 143M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f dff9e665bd9b 2024-09-14T04:06:52.103 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4 smithi129 running (16m) 9s ago 16m 134M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 5b64727311b7 2024-09-14T04:06:52.103 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5 smithi129 running (16m) 9s ago 16m 119M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f f2d991a754f7 2024-09-14T04:06:52.104 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6 smithi129 running (16m) 9s ago 16m 136M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 3060eaf44106 2024-09-14T04:06:52.104 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7 smithi129 running (15m) 9s ago 15m 129M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 4c13b97060d1 2024-09-14T04:06:52.104 INFO:teuthology.orchestra.run.smithi039.stdout:prometheus.a smithi129 *:9095 running (9m) 9s ago 15m 60.3M - 2.51.0 1d3b7f56885b cbe583f852c3 2024-09-14T04:06:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:51 smithi129 ceph-mon[101316]: pgmap v569: 521 pgs: 521 active+clean; 71 MiB data, 658 MiB used, 715 GiB / 715 GiB avail; 1.8 MiB/s, 0 objects/s recovering 2024-09-14T04:06:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:51 smithi129 ceph-mon[101316]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 47/750 objects degraded (6.267%), 27 pgs degraded) 2024-09-14T04:06:52.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:51 smithi039 ceph-mon[106470]: pgmap v569: 521 pgs: 521 active+clean; 71 MiB data, 658 MiB used, 715 GiB / 715 GiB avail; 1.8 MiB/s, 0 objects/s recovering 2024-09-14T04:06:52.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:51 smithi039 ceph-mon[106470]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 47/750 objects degraded (6.267%), 27 pgs degraded) 2024-09-14T04:06:52.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:51 smithi039 ceph-mon[111235]: pgmap v569: 521 pgs: 521 active+clean; 71 MiB data, 658 MiB used, 715 GiB / 715 GiB avail; 1.8 MiB/s, 0 objects/s recovering 2024-09-14T04:06:52.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:51 smithi039 ceph-mon[111235]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 47/750 objects degraded (6.267%), 27 pgs degraded) 2024-09-14T04:06:52.463 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:06:52.464 INFO:teuthology.orchestra.run.smithi039.stdout: "mon": { 2024-09-14T04:06:52.464 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T04:06:52.464 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:06:52.464 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr": { 2024-09-14T04:06:52.464 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T04:06:52.464 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:06:52.464 INFO:teuthology.orchestra.run.smithi039.stdout: "osd": { 2024-09-14T04:06:52.464 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 6, 2024-09-14T04:06:52.464 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T04:06:52.464 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:06:52.464 INFO:teuthology.orchestra.run.smithi039.stdout: "mds": { 2024-09-14T04:06:52.465 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T04:06:52.465 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:06:52.465 INFO:teuthology.orchestra.run.smithi039.stdout: "overall": { 2024-09-14T04:06:52.465 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 8, 2024-09-14T04:06:52.465 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 7 2024-09-14T04:06:52.465 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:06:52.465 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:06:52.768 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:06:52.768 INFO:teuthology.orchestra.run.smithi039.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T04:06:52.768 INFO:teuthology.orchestra.run.smithi039.stdout: "in_progress": true, 2024-09-14T04:06:52.768 INFO:teuthology.orchestra.run.smithi039.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (2 remaining).", 2024-09-14T04:06:52.768 INFO:teuthology.orchestra.run.smithi039.stdout: "services_complete": [], 2024-09-14T04:06:52.768 INFO:teuthology.orchestra.run.smithi039.stdout: "progress": "2/8 daemons upgraded", 2024-09-14T04:06:52.768 INFO:teuthology.orchestra.run.smithi039.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T04:06:52.768 INFO:teuthology.orchestra.run.smithi039.stdout: "is_paused": false 2024-09-14T04:06:52.769 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:06:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:52 smithi039 ceph-mon[106470]: from='client.34575 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:06:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:52 smithi039 ceph-mon[106470]: from='client.34581 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:06:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:52 smithi039 ceph-mon[106470]: from='client.34587 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:06:53.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:52 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:53.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:53.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:52 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1115778234' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:06:53.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:52 smithi039 ceph-mon[111235]: from='client.34575 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:06:53.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:52 smithi039 ceph-mon[111235]: from='client.34581 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:06:53.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:52 smithi039 ceph-mon[111235]: from='client.34587 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:06:53.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:52 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:53.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:53.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:52 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1115778234' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:06:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:52 smithi129 ceph-mon[101316]: from='client.34575 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:06:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:52 smithi129 ceph-mon[101316]: from='client.34581 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:06:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:52 smithi129 ceph-mon[101316]: from='client.34587 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:06:53.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:52 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:53.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:06:53.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:52 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1115778234' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:06:54.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:53 smithi129 ceph-mon[101316]: from='client.34599 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:06:54.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:53 smithi129 ceph-mon[101316]: pgmap v570: 521 pgs: 521 active+clean; 71 MiB data, 658 MiB used, 715 GiB / 715 GiB avail; 1.5 MiB/s, 0 objects/s recovering 2024-09-14T04:06:54.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:53 smithi039 ceph-mon[106470]: from='client.34599 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:06:54.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:53 smithi039 ceph-mon[106470]: pgmap v570: 521 pgs: 521 active+clean; 71 MiB data, 658 MiB used, 715 GiB / 715 GiB avail; 1.5 MiB/s, 0 objects/s recovering 2024-09-14T04:06:54.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:53 smithi039 ceph-mon[111235]: from='client.34599 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:06:54.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:53 smithi039 ceph-mon[111235]: pgmap v570: 521 pgs: 521 active+clean; 71 MiB data, 658 MiB used, 715 GiB / 715 GiB avail; 1.5 MiB/s, 0 objects/s recovering 2024-09-14T04:06:55.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:55 smithi129 ceph-mon[101316]: osdmap e683: 8 total, 8 up, 8 in 2024-09-14T04:06:55.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:55 smithi039 ceph-mon[106470]: osdmap e683: 8 total, 8 up, 8 in 2024-09-14T04:06:55.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:55 smithi039 ceph-mon[111235]: osdmap e683: 8 total, 8 up, 8 in 2024-09-14T04:06:56.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:56 smithi129 ceph-mon[101316]: pgmap v572: 521 pgs: 521 active+clean; 71 MiB data, 658 MiB used, 715 GiB / 715 GiB avail; 107 KiB/s rd, 126 B/s wr, 1 op/s; 1.5 MiB/s, 0 objects/s recovering 2024-09-14T04:06:56.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:56 smithi039 ceph-mon[106470]: pgmap v572: 521 pgs: 521 active+clean; 71 MiB data, 658 MiB used, 715 GiB / 715 GiB avail; 107 KiB/s rd, 126 B/s wr, 1 op/s; 1.5 MiB/s, 0 objects/s recovering 2024-09-14T04:06:56.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:56 smithi039 ceph-mon[111235]: pgmap v572: 521 pgs: 521 active+clean; 71 MiB data, 658 MiB used, 715 GiB / 715 GiB avail; 107 KiB/s rd, 126 B/s wr, 1 op/s; 1.5 MiB/s, 0 objects/s recovering 2024-09-14T04:06:58.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:58 smithi129 ceph-mon[101316]: pgmap v573: 521 pgs: 521 active+clean; 71 MiB data, 658 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 102 B/s wr, 1 op/s; 1.2 MiB/s, 0 objects/s recovering 2024-09-14T04:06:58.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:06:58 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:58.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:58 smithi039 ceph-mon[106470]: pgmap v573: 521 pgs: 521 active+clean; 71 MiB data, 658 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 102 B/s wr, 1 op/s; 1.2 MiB/s, 0 objects/s recovering 2024-09-14T04:06:58.293 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:06:58 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:58.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:58 smithi039 ceph-mon[111235]: pgmap v573: 521 pgs: 521 active+clean; 71 MiB data, 658 MiB used, 715 GiB / 715 GiB avail; 87 KiB/s rd, 102 B/s wr, 1 op/s; 1.2 MiB/s, 0 objects/s recovering 2024-09-14T04:06:58.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:06:58 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:06:59.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:06:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:06:58] "GET /metrics HTTP/1.1" 200 67994 "" "Prometheus/2.51.0" 2024-09-14T04:07:00.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:00 smithi129 ceph-mon[101316]: pgmap v574: 521 pgs: 521 active+clean; 71 MiB data, 658 MiB used, 715 GiB / 715 GiB avail; 145 KiB/s rd, 1.1 KiB/s wr, 36 op/s; 409 KiB/s, 0 objects/s recovering 2024-09-14T04:07:00.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:00 smithi039 ceph-mon[106470]: pgmap v574: 521 pgs: 521 active+clean; 71 MiB data, 658 MiB used, 715 GiB / 715 GiB avail; 145 KiB/s rd, 1.1 KiB/s wr, 36 op/s; 409 KiB/s, 0 objects/s recovering 2024-09-14T04:07:00.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:00 smithi039 ceph-mon[111235]: pgmap v574: 521 pgs: 521 active+clean; 71 MiB data, 658 MiB used, 715 GiB / 715 GiB avail; 145 KiB/s rd, 1.1 KiB/s wr, 36 op/s; 409 KiB/s, 0 objects/s recovering 2024-09-14T04:07:02.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:02 smithi039 ceph-mon[106470]: pgmap v575: 521 pgs: 521 active+clean; 71 MiB data, 658 MiB used, 715 GiB / 715 GiB avail; 150 KiB/s rd, 1.1 KiB/s wr, 42 op/s 2024-09-14T04:07:02.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:02 smithi039 ceph-mon[111235]: pgmap v575: 521 pgs: 521 active+clean; 71 MiB data, 658 MiB used, 715 GiB / 715 GiB avail; 150 KiB/s rd, 1.1 KiB/s wr, 42 op/s 2024-09-14T04:07:02.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:02 smithi129 ceph-mon[101316]: pgmap v575: 521 pgs: 521 active+clean; 71 MiB data, 658 MiB used, 715 GiB / 715 GiB avail; 150 KiB/s rd, 1.1 KiB/s wr, 42 op/s 2024-09-14T04:07:03.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:03 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T04:07:03.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:03 smithi039 ceph-mon[106470]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T04:07:03.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:03 smithi039 ceph-mon[106470]: Upgrade: osd.2 is safe to restart 2024-09-14T04:07:03.039 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:03 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T04:07:03.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:03 smithi039 ceph-mon[111235]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T04:07:03.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:03 smithi039 ceph-mon[111235]: Upgrade: osd.2 is safe to restart 2024-09-14T04:07:03.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:03 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16} : dispatch 2024-09-14T04:07:03.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:03 smithi129 ceph-mon[101316]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-09-14T04:07:03.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:03 smithi129 ceph-mon[101316]: Upgrade: osd.2 is safe to restart 2024-09-14T04:07:04.458 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:04 smithi039 ceph-mon[106470]: pgmap v576: 521 pgs: 521 active+clean; 283 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 214 KiB/s rd, 21 MiB/s wr, 113 op/s 2024-09-14T04:07:04.458 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:04 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:04.458 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:04 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-14T04:07:04.458 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:04 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:07:04.458 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:04 smithi039 ceph-mon[111235]: pgmap v576: 521 pgs: 521 active+clean; 283 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 214 KiB/s rd, 21 MiB/s wr, 113 op/s 2024-09-14T04:07:04.458 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:04 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:04.458 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:04 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-14T04:07:04.459 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:04 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:07:04.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:04 smithi129 ceph-mon[101316]: pgmap v576: 521 pgs: 521 active+clean; 283 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 214 KiB/s rd, 21 MiB/s wr, 113 op/s 2024-09-14T04:07:04.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:04 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:04.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:04 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.2"} : dispatch 2024-09-14T04:07:04.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:04 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:07:05.237 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:05 smithi039 ceph-mon[106470]: Upgrade: Updating osd.2 2024-09-14T04:07:05.238 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:05 smithi039 ceph-mon[106470]: Deploying daemon osd.2 on smithi039 2024-09-14T04:07:05.238 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:05 smithi039 ceph-mon[106470]: osdmap e684: 8 total, 8 up, 8 in 2024-09-14T04:07:05.238 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:05 smithi039 ceph-mon[111235]: Upgrade: Updating osd.2 2024-09-14T04:07:05.238 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:05 smithi039 ceph-mon[111235]: Deploying daemon osd.2 on smithi039 2024-09-14T04:07:05.238 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:05 smithi039 ceph-mon[111235]: osdmap e684: 8 total, 8 up, 8 in 2024-09-14T04:07:05.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:05 smithi129 ceph-mon[101316]: Upgrade: Updating osd.2 2024-09-14T04:07:05.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:05 smithi129 ceph-mon[101316]: Deploying daemon osd.2 on smithi039 2024-09-14T04:07:05.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:05 smithi129 ceph-mon[101316]: osdmap e684: 8 total, 8 up, 8 in 2024-09-14T04:07:06.229 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:06 smithi039 ceph-mon[106470]: pgmap v578: 521 pgs: 521 active+clean; 279 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 618 KiB/s rd, 26 MiB/s wr, 111 op/s 2024-09-14T04:07:06.229 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:06 smithi039 ceph-mon[106470]: osdmap e685: 8 total, 8 up, 8 in 2024-09-14T04:07:06.229 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:06 smithi039 ceph-mon[111235]: pgmap v578: 521 pgs: 521 active+clean; 279 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 618 KiB/s rd, 26 MiB/s wr, 111 op/s 2024-09-14T04:07:06.230 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:06 smithi039 ceph-mon[111235]: osdmap e685: 8 total, 8 up, 8 in 2024-09-14T04:07:06.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:06 smithi129 ceph-mon[101316]: pgmap v578: 521 pgs: 521 active+clean; 279 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 618 KiB/s rd, 26 MiB/s wr, 111 op/s 2024-09-14T04:07:06.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:06 smithi129 ceph-mon[101316]: osdmap e685: 8 total, 8 up, 8 in 2024-09-14T04:07:07.178 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 4 2024-09-14T04:07:07.179 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 4 2024-09-14T04:07:07.179 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 4 2024-09-14T04:07:07.179 INFO:tasks.workunit.client.0.smithi039.stdout:read: 4 2024-09-14T04:07:07.179 INFO:tasks.workunit.client.0.smithi039.stdout:read: 4 2024-09-14T04:07:07.179 INFO:tasks.workunit.client.0.smithi039.stdout:read: 4 2024-09-14T04:07:07.179 INFO:tasks.workunit.client.0.smithi039.stdout:read: 4 2024-09-14T04:07:07.179 INFO:tasks.workunit.client.0.smithi039.stdout:read: 4 2024-09-14T04:07:07.179 INFO:tasks.workunit.client.0.smithi039.stdout:read: 4 2024-09-14T04:07:07.179 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestCloneEncryption (49523 ms) 2024-09-14T04:07:07.179 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.LUKS1UnderLUKS2WithoutResize 2024-09-14T04:07:07.179 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:07:07.278 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:07 smithi039 ceph-mon[106470]: osdmap e686: 8 total, 8 up, 8 in 2024-09-14T04:07:07.278 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:07 smithi039 ceph-mon[111235]: osdmap e686: 8 total, 8 up, 8 in 2024-09-14T04:07:07.279 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:06 smithi039 systemd[1]: Stopping Ceph osd.2 for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T04:07:07.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:07 smithi129 ceph-mon[101316]: osdmap e686: 8 total, 8 up, 8 in 2024-09-14T04:07:07.540 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:07 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2[57619]: 2024-09-14T04:07:07.276+0000 7febb1732640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:07:07.540 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:07 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2[57619]: 2024-09-14T04:07:07.276+0000 7febb1732640 -1 osd.2 687 *** Got signal Terminated *** 2024-09-14T04:07:07.540 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:07 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2[57619]: 2024-09-14T04:07:07.276+0000 7febb1732640 -1 osd.2 687 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T04:07:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:08 smithi129 ceph-mon[101316]: pgmap v581: 521 pgs: 521 active+clean; 279 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 837 KiB/s rd, 43 MiB/s wr, 126 op/s 2024-09-14T04:07:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:08 smithi129 ceph-mon[101316]: osdmap e687: 8 total, 8 up, 8 in 2024-09-14T04:07:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:08 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:07:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:08 smithi129 ceph-mon[101316]: osd.2 marked itself down and dead 2024-09-14T04:07:08.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:08 smithi039 ceph-mon[106470]: pgmap v581: 521 pgs: 521 active+clean; 279 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 837 KiB/s rd, 43 MiB/s wr, 126 op/s 2024-09-14T04:07:08.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:08 smithi039 ceph-mon[106470]: osdmap e687: 8 total, 8 up, 8 in 2024-09-14T04:07:08.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:08 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:07:08.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:08 smithi039 ceph-mon[106470]: osd.2 marked itself down and dead 2024-09-14T04:07:08.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:08 smithi039 ceph-mon[111235]: pgmap v581: 521 pgs: 521 active+clean; 279 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 837 KiB/s rd, 43 MiB/s wr, 126 op/s 2024-09-14T04:07:08.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:08 smithi039 ceph-mon[111235]: osdmap e687: 8 total, 8 up, 8 in 2024-09-14T04:07:08.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:08 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:07:08.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:08 smithi039 ceph-mon[111235]: osd.2 marked itself down and dead 2024-09-14T04:07:09.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:07:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:07:08] "GET /metrics HTTP/1.1" 200 67997 "" "Prometheus/2.51.0" 2024-09-14T04:07:09.040 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:08 smithi039 podman[136286]: 2024-09-14 04:07:08.652452743 +0000 UTC m=+1.503163505 container died eb903b44fc5d80663e7e7d63564153e95c17da01b27d3a132ef779d9804484e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=reef-e4e227b, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-14T04:07:09.377 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:09 smithi039 ceph-mon[106470]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T04:07:09.377 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:09 smithi039 ceph-mon[106470]: osdmap e688: 8 total, 7 up, 8 in 2024-09-14T04:07:09.378 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:09 smithi039 ceph-mon[111235]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T04:07:09.378 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:09 smithi039 ceph-mon[111235]: osdmap e688: 8 total, 7 up, 8 in 2024-09-14T04:07:09.378 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:09 smithi039 podman[136286]: 2024-09-14 04:07:09.17467803 +0000 UTC m=+2.025388789 container cleanup eb903b44fc5d80663e7e7d63564153e95c17da01b27d3a132ef779d9804484e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=reef-e4e227b, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-14T04:07:09.378 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:09 smithi039 bash[136286]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2 2024-09-14T04:07:09.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:09 smithi129 ceph-mon[101316]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T04:07:09.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:09 smithi129 ceph-mon[101316]: osdmap e688: 8 total, 7 up, 8 in 2024-09-14T04:07:09.790 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:09 smithi039 podman[136307]: 2024-09-14 04:07:09.375992673 +0000 UTC m=+0.718514160 container remove eb903b44fc5d80663e7e7d63564153e95c17da01b27d3a132ef779d9804484e5 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=reef-e4e227b, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-14T04:07:10.080 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:09 smithi039 podman[136401]: 2024-09-14 04:07:09.702627965 +0000 UTC m=+0.016608362 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:07:10.081 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:09 smithi039 podman[136401]: 2024-09-14 04:07:09.810342613 +0000 UTC m=+0.124323006 container create e21ff5d01291b0be31daf969d0aa3949986d38aafa121a2f102a46eb4dc14fb3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS) 2024-09-14T04:07:10.417 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:10 smithi039 ceph-mon[106470]: pgmap v584: 521 pgs: 54 stale+active+clean, 467 active+clean; 39 MiB data, 579 MiB used, 715 GiB / 715 GiB avail; 170 KiB/s rd, 15 KiB/s wr, 250 op/s 2024-09-14T04:07:10.417 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:10 smithi039 ceph-mon[106470]: osdmap e689: 8 total, 7 up, 8 in 2024-09-14T04:07:10.418 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:10 smithi039 ceph-mon[111235]: pgmap v584: 521 pgs: 54 stale+active+clean, 467 active+clean; 39 MiB data, 579 MiB used, 715 GiB / 715 GiB avail; 170 KiB/s rd, 15 KiB/s wr, 250 op/s 2024-09-14T04:07:10.418 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:10 smithi039 ceph-mon[111235]: osdmap e689: 8 total, 7 up, 8 in 2024-09-14T04:07:10.418 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:10 smithi039 podman[136401]: 2024-09-14 04:07:10.07998317 +0000 UTC m=+0.393963572 container init e21ff5d01291b0be31daf969d0aa3949986d38aafa121a2f102a46eb4dc14fb3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-14T04:07:10.418 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:10 smithi039 podman[136401]: 2024-09-14 04:07:10.085328394 +0000 UTC m=+0.399308796 container start e21ff5d01291b0be31daf969d0aa3949986d38aafa121a2f102a46eb4dc14fb3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, RELEASE=main-a278231, CEPH_POINT_RELEASE=) 2024-09-14T04:07:10.418 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:10 smithi039 podman[136401]: 2024-09-14 04:07:10.188313907 +0000 UTC m=+0.502294303 container attach e21ff5d01291b0be31daf969d0aa3949986d38aafa121a2f102a46eb4dc14fb3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, RELEASE=main-a278231, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2) 2024-09-14T04:07:10.418 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:10 smithi039 podman[136401]: 2024-09-14 04:07:10.283604452 +0000 UTC m=+0.597584868 container died e21ff5d01291b0be31daf969d0aa3949986d38aafa121a2f102a46eb4dc14fb3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2-deactivate, RELEASE=main-a278231, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True) 2024-09-14T04:07:10.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:10 smithi129 ceph-mon[101316]: pgmap v584: 521 pgs: 54 stale+active+clean, 467 active+clean; 39 MiB data, 579 MiB used, 715 GiB / 715 GiB avail; 170 KiB/s rd, 15 KiB/s wr, 250 op/s 2024-09-14T04:07:10.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:10 smithi129 ceph-mon[101316]: osdmap e689: 8 total, 7 up, 8 in 2024-09-14T04:07:11.290 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:10 smithi039 podman[136444]: 2024-09-14 04:07:10.959426441 +0000 UTC m=+0.669292800 container remove e21ff5d01291b0be31daf969d0aa3949986d38aafa121a2f102a46eb4dc14fb3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2-deactivate, RELEASE=main-a278231, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-14T04:07:11.291 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:10 smithi039 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.2.service: Deactivated successfully. 2024-09-14T04:07:11.291 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:10 smithi039 systemd[1]: Stopped Ceph osd.2 for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T04:07:11.291 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:10 smithi039 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.2.service: Consumed 33.828s CPU time. 2024-09-14T04:07:11.291 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:11 smithi039 systemd[1]: Starting Ceph osd.2 for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T04:07:11.680 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:11 smithi039 ceph-mon[106470]: osdmap e690: 8 total, 7 up, 8 in 2024-09-14T04:07:11.680 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:11 smithi039 ceph-mon[106470]: pgmap v587: 521 pgs: 22 active+undersized, 47 stale+active+clean, 11 active+undersized+degraded, 441 active+clean; 31 MiB data, 547 MiB used, 715 GiB / 715 GiB avail; 202 KiB/s rd, 17 KiB/s wr, 290 op/s; 15/681 objects degraded (2.203%) 2024-09-14T04:07:11.681 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:11 smithi039 ceph-mon[111235]: osdmap e690: 8 total, 7 up, 8 in 2024-09-14T04:07:11.681 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:11 smithi039 ceph-mon[111235]: pgmap v587: 521 pgs: 22 active+undersized, 47 stale+active+clean, 11 active+undersized+degraded, 441 active+clean; 31 MiB data, 547 MiB used, 715 GiB / 715 GiB avail; 202 KiB/s rd, 17 KiB/s wr, 290 op/s; 15/681 objects degraded (2.203%) 2024-09-14T04:07:11.681 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:11 smithi039 podman[136548]: 2024-09-14 04:07:11.314754201 +0000 UTC m=+0.016461465 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:07:11.681 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:11 smithi039 podman[136548]: 2024-09-14 04:07:11.418955484 +0000 UTC m=+0.120662797 container create e48086299239b44c2d00d202e7f458391204ead79eaf943dc340c837adfe3247 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, RELEASE=main-a278231, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True) 2024-09-14T04:07:11.681 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:11 smithi039 podman[136548]: 2024-09-14 04:07:11.679961134 +0000 UTC m=+0.381668383 container init e48086299239b44c2d00d202e7f458391204ead79eaf943dc340c837adfe3247 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2-activate, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, RELEASE=main-a278231, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-14T04:07:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:11 smithi129 ceph-mon[101316]: osdmap e690: 8 total, 7 up, 8 in 2024-09-14T04:07:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:11 smithi129 ceph-mon[101316]: pgmap v587: 521 pgs: 22 active+undersized, 47 stale+active+clean, 11 active+undersized+degraded, 441 active+clean; 31 MiB data, 547 MiB used, 715 GiB / 715 GiB avail; 202 KiB/s rd, 17 KiB/s wr, 290 op/s; 15/681 objects degraded (2.203%) 2024-09-14T04:07:12.040 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:11 smithi039 podman[136548]: 2024-09-14 04:07:11.685017486 +0000 UTC m=+0.386724735 container start e48086299239b44c2d00d202e7f458391204ead79eaf943dc340c837adfe3247 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2-activate, RELEASE=main-a278231, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2) 2024-09-14T04:07:12.040 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:11 smithi039 podman[136548]: 2024-09-14 04:07:11.771816648 +0000 UTC m=+0.473523906 container attach e48086299239b44c2d00d202e7f458391204ead79eaf943dc340c837adfe3247 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2-activate, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2) 2024-09-14T04:07:12.041 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:11 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2-activate[136563]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:07:12.041 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:11 smithi039 bash[136548]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:07:12.041 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:11 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2-activate[136563]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:07:12.041 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:11 smithi039 bash[136548]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:07:12.587 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:12 smithi039 ceph-mon[106470]: Health check failed: Degraded data redundancy: 15/681 objects degraded (2.203%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:07:12.587 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:12 smithi039 ceph-mon[106470]: osdmap e691: 8 total, 7 up, 8 in 2024-09-14T04:07:12.588 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:12 smithi039 ceph-mon[111235]: Health check failed: Degraded data redundancy: 15/681 objects degraded (2.203%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:07:12.588 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:12 smithi039 ceph-mon[111235]: osdmap e691: 8 total, 7 up, 8 in 2024-09-14T04:07:12.588 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:12 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2-activate[136563]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-14T04:07:12.588 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:12 smithi039 bash[136548]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-14T04:07:12.589 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:12 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2-activate[136563]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-14T04:07:12.589 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:12 smithi039 bash[136548]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-09-14T04:07:12.589 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:12 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2-activate[136563]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-14T04:07:12.589 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:12 smithi039 bash[136548]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-09-14T04:07:12.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:12 smithi129 ceph-mon[101316]: Health check failed: Degraded data redundancy: 15/681 objects degraded (2.203%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:07:12.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:12 smithi129 ceph-mon[101316]: osdmap e691: 8 total, 7 up, 8 in 2024-09-14T04:07:12.953 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:12 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2-activate[136563]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-14T04:07:12.953 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:12 smithi039 bash[136548]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-09-14T04:07:12.953 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:12 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2-activate[136563]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-14T04:07:12.953 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:12 smithi039 bash[136548]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-09-14T04:07:12.954 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:12 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2-activate[136563]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-14T04:07:12.954 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:12 smithi039 bash[136548]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-09-14T04:07:12.954 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:12 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2-activate[136563]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-14T04:07:12.954 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:12 smithi039 bash[136548]: --> ceph-volume raw activate successful for osd ID: 2 2024-09-14T04:07:12.954 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:12 smithi039 podman[136548]: 2024-09-14 04:07:12.634845179 +0000 UTC m=+1.336552428 container died e48086299239b44c2d00d202e7f458391204ead79eaf943dc340c837adfe3247 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2-activate, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0) 2024-09-14T04:07:13.570 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:13 smithi039 ceph-mon[106470]: pgmap v589: 521 pgs: 150 active+undersized, 50 active+undersized+degraded, 321 active+clean; 56 MiB data, 606 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 5.2 MiB/s wr, 96 op/s; 95/714 objects degraded (13.305%) 2024-09-14T04:07:13.570 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:13 smithi039 podman[136808]: 2024-09-14 04:07:13.292716555 +0000 UTC m=+0.647000327 container remove e48086299239b44c2d00d202e7f458391204ead79eaf943dc340c837adfe3247 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2-activate, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, RELEASE=main-a278231, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, ceph=True) 2024-09-14T04:07:13.571 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:13 smithi039 podman[137051]: 2024-09-14 04:07:13.438229962 +0000 UTC m=+0.020829709 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:07:13.571 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:13 smithi039 ceph-mon[111235]: pgmap v589: 521 pgs: 150 active+undersized, 50 active+undersized+degraded, 321 active+clean; 56 MiB data, 606 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 5.2 MiB/s wr, 96 op/s; 95/714 objects degraded (13.305%) 2024-09-14T04:07:13.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:13 smithi129 ceph-mon[101316]: pgmap v589: 521 pgs: 150 active+undersized, 50 active+undersized+degraded, 321 active+clean; 56 MiB data, 606 MiB used, 715 GiB / 715 GiB avail; 73 KiB/s rd, 5.2 MiB/s wr, 96 op/s; 95/714 objects degraded (13.305%) 2024-09-14T04:07:13.822 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:13 smithi039 podman[137051]: 2024-09-14 04:07:13.569014211 +0000 UTC m=+0.151613950 container create 662e3d705b70827d44f715bdb57db06d2548bc5f6befb08f6f07fe1ad08712d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2, maintainer=Guillaume Abrioux , RELEASE=main-a278231, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731) 2024-09-14T04:07:14.290 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:13 smithi039 podman[137051]: 2024-09-14 04:07:13.821799029 +0000 UTC m=+0.404398783 container init 662e3d705b70827d44f715bdb57db06d2548bc5f6befb08f6f07fe1ad08712d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, maintainer=Guillaume Abrioux , RELEASE=main-a278231, CEPH_POINT_RELEASE=) 2024-09-14T04:07:14.290 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:13 smithi039 podman[137051]: 2024-09-14 04:07:13.829549054 +0000 UTC m=+0.412148793 container start 662e3d705b70827d44f715bdb57db06d2548bc5f6befb08f6f07fe1ad08712d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=main-a278231, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD) 2024-09-14T04:07:14.290 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:13 smithi039 ceph-osd[137186]: -- 172.21.15.39:0/1025850734 <== mon.0 v2:172.21.15.39:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55fe3940ba40 con 0x55fe3a1da400 2024-09-14T04:07:14.291 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:13 smithi039 bash[137051]: 662e3d705b70827d44f715bdb57db06d2548bc5f6befb08f6f07fe1ad08712d6 2024-09-14T04:07:14.291 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:13 smithi039 systemd[1]: Started Ceph osd.2 for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T04:07:15.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:15 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:15.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:15 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:15.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:15 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:15.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:15 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:07:15.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:15 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:15.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:15 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:15.290 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:15 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2[137127]: 2024-09-14T04:07:15.216+0000 7f94c5d35640 -1 Falling back to public interface 2024-09-14T04:07:15.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:15 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:15.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:15 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:15.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:15 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:15.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:15 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:07:15.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:15 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:15.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:15 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:15.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:15 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:15.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:15 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:15.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:15 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:15.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:15 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:07:15.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:15 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:15.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:15 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:16.271 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:16 smithi039 ceph-mon[106470]: pgmap v590: 521 pgs: 150 active+undersized, 50 active+undersized+degraded, 321 active+clean; 60 MiB data, 610 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s rd, 4.8 MiB/s wr, 77 op/s; 95/717 objects degraded (13.250%) 2024-09-14T04:07:16.271 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:16 smithi039 ceph-mon[106470]: osdmap e692: 8 total, 7 up, 8 in 2024-09-14T04:07:16.271 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:16 smithi039 ceph-mon[111235]: pgmap v590: 521 pgs: 150 active+undersized, 50 active+undersized+degraded, 321 active+clean; 60 MiB data, 610 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s rd, 4.8 MiB/s wr, 77 op/s; 95/717 objects degraded (13.250%) 2024-09-14T04:07:16.271 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:16 smithi039 ceph-mon[111235]: osdmap e692: 8 total, 7 up, 8 in 2024-09-14T04:07:16.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:16 smithi129 ceph-mon[101316]: pgmap v590: 521 pgs: 150 active+undersized, 50 active+undersized+degraded, 321 active+clean; 60 MiB data, 610 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s rd, 4.8 MiB/s wr, 77 op/s; 95/717 objects degraded (13.250%) 2024-09-14T04:07:16.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:16 smithi129 ceph-mon[101316]: osdmap e692: 8 total, 7 up, 8 in 2024-09-14T04:07:18.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:18 smithi039 ceph-mon[106470]: pgmap v592: 521 pgs: 150 active+undersized, 50 active+undersized+degraded, 321 active+clean; 60 MiB data, 610 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 4.5 MiB/s wr, 47 op/s; 95/717 objects degraded (13.250%) 2024-09-14T04:07:18.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:18 smithi039 ceph-mon[111235]: pgmap v592: 521 pgs: 150 active+undersized, 50 active+undersized+degraded, 321 active+clean; 60 MiB data, 610 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 4.5 MiB/s wr, 47 op/s; 95/717 objects degraded (13.250%) 2024-09-14T04:07:18.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:18 smithi129 ceph-mon[101316]: pgmap v592: 521 pgs: 150 active+undersized, 50 active+undersized+degraded, 321 active+clean; 60 MiB data, 610 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 4.5 MiB/s wr, 47 op/s; 95/717 objects degraded (13.250%) 2024-09-14T04:07:19.040 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2[137127]: 2024-09-14T04:07:18.764+0000 7f94c5d35640 -1 osd.2 688 log_to_monitors true 2024-09-14T04:07:19.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:07:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:07:18] "GET /metrics HTTP/1.1" 200 67997 "" "Prometheus/2.51.0" 2024-09-14T04:07:19.862 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 04:07:19 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2[137127]: 2024-09-14T04:07:19.723+0000 7f94bdadf640 -1 osd.2 688 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T04:07:19.862 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:19 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:19.862 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:19 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:19.862 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:19 smithi039 ceph-mon[106470]: from='osd.2 [v2:172.21.15.39:6818/2359749690,v1:172.21.15.39:6819/2359749690]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-14T04:07:19.862 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:19 smithi039 ceph-mon[106470]: pgmap v593: 521 pgs: 150 active+undersized, 50 active+undersized+degraded, 321 active+clean; 60 MiB data, 610 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 3.6 MiB/s wr, 38 op/s; 95/717 objects degraded (13.250%) 2024-09-14T04:07:19.862 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:19 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:19.862 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:19 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:19.863 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:19 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:19.863 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:19 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:19.863 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:19 smithi039 ceph-mon[111235]: from='osd.2 [v2:172.21.15.39:6818/2359749690,v1:172.21.15.39:6819/2359749690]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-14T04:07:19.863 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:19 smithi039 ceph-mon[111235]: pgmap v593: 521 pgs: 150 active+undersized, 50 active+undersized+degraded, 321 active+clean; 60 MiB data, 610 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 3.6 MiB/s wr, 38 op/s; 95/717 objects degraded (13.250%) 2024-09-14T04:07:19.863 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:19 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:19.863 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:19 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:20.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:19 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:20.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:19 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:20.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:19 smithi129 ceph-mon[101316]: from='osd.2 [v2:172.21.15.39:6818/2359749690,v1:172.21.15.39:6819/2359749690]' entity='osd.2' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]} : dispatch 2024-09-14T04:07:20.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:19 smithi129 ceph-mon[101316]: pgmap v593: 521 pgs: 150 active+undersized, 50 active+undersized+degraded, 321 active+clean; 60 MiB data, 610 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 3.6 MiB/s wr, 38 op/s; 95/717 objects degraded (13.250%) 2024-09-14T04:07:20.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:19 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:20.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:19 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:21.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:20 smithi129 ceph-mon[101316]: from='osd.2 [v2:172.21.15.39:6818/2359749690,v1:172.21.15.39:6819/2359749690]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-14T04:07:21.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:20 smithi129 ceph-mon[101316]: osdmap e693: 8 total, 7 up, 8 in 2024-09-14T04:07:21.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:20 smithi129 ceph-mon[101316]: from='osd.2 [v2:172.21.15.39:6818/2359749690,v1:172.21.15.39:6819/2359749690]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi039", "root=default"]} : dispatch 2024-09-14T04:07:21.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:20 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 95/717 objects degraded (13.250%), 50 pgs degraded (PG_DEGRADED) 2024-09-14T04:07:21.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:20 smithi039 ceph-mon[106470]: from='osd.2 [v2:172.21.15.39:6818/2359749690,v1:172.21.15.39:6819/2359749690]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-14T04:07:21.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:20 smithi039 ceph-mon[106470]: osdmap e693: 8 total, 7 up, 8 in 2024-09-14T04:07:21.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:20 smithi039 ceph-mon[106470]: from='osd.2 [v2:172.21.15.39:6818/2359749690,v1:172.21.15.39:6819/2359749690]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi039", "root=default"]} : dispatch 2024-09-14T04:07:21.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:20 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 95/717 objects degraded (13.250%), 50 pgs degraded (PG_DEGRADED) 2024-09-14T04:07:21.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:20 smithi039 ceph-mon[111235]: from='osd.2 [v2:172.21.15.39:6818/2359749690,v1:172.21.15.39:6819/2359749690]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-09-14T04:07:21.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:20 smithi039 ceph-mon[111235]: osdmap e693: 8 total, 7 up, 8 in 2024-09-14T04:07:21.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:20 smithi039 ceph-mon[111235]: from='osd.2 [v2:172.21.15.39:6818/2359749690,v1:172.21.15.39:6819/2359749690]' entity='osd.2' cmd={"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi039", "root=default"]} : dispatch 2024-09-14T04:07:21.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:20 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 95/717 objects degraded (13.250%), 50 pgs degraded (PG_DEGRADED) 2024-09-14T04:07:21.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:21 smithi039 ceph-mon[106470]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T04:07:21.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:21 smithi039 ceph-mon[106470]: osd.2 [v2:172.21.15.39:6818/2359749690,v1:172.21.15.39:6819/2359749690] boot 2024-09-14T04:07:21.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:21 smithi039 ceph-mon[106470]: osdmap e694: 8 total, 8 up, 8 in 2024-09-14T04:07:21.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:21 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-14T04:07:21.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:21 smithi039 ceph-mon[106470]: pgmap v596: 521 pgs: 150 active+undersized, 50 active+undersized+degraded, 321 active+clean; 60 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 95/717 objects degraded (13.250%) 2024-09-14T04:07:21.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:21 smithi039 ceph-mon[106470]: osdmap e695: 8 total, 8 up, 8 in 2024-09-14T04:07:21.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:21 smithi039 ceph-mon[111235]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T04:07:21.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:21 smithi039 ceph-mon[111235]: osd.2 [v2:172.21.15.39:6818/2359749690,v1:172.21.15.39:6819/2359749690] boot 2024-09-14T04:07:21.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:21 smithi039 ceph-mon[111235]: osdmap e694: 8 total, 8 up, 8 in 2024-09-14T04:07:21.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:21 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-14T04:07:21.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:21 smithi039 ceph-mon[111235]: pgmap v596: 521 pgs: 150 active+undersized, 50 active+undersized+degraded, 321 active+clean; 60 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 95/717 objects degraded (13.250%) 2024-09-14T04:07:21.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:21 smithi039 ceph-mon[111235]: osdmap e695: 8 total, 8 up, 8 in 2024-09-14T04:07:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:21 smithi129 ceph-mon[101316]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T04:07:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:21 smithi129 ceph-mon[101316]: osd.2 [v2:172.21.15.39:6818/2359749690,v1:172.21.15.39:6819/2359749690] boot 2024-09-14T04:07:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:21 smithi129 ceph-mon[101316]: osdmap e694: 8 total, 8 up, 8 in 2024-09-14T04:07:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:21 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 2} : dispatch 2024-09-14T04:07:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:21 smithi129 ceph-mon[101316]: pgmap v596: 521 pgs: 150 active+undersized, 50 active+undersized+degraded, 321 active+clean; 60 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 95/717 objects degraded (13.250%) 2024-09-14T04:07:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:21 smithi129 ceph-mon[101316]: osdmap e695: 8 total, 8 up, 8 in 2024-09-14T04:07:22.649 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.LUKS1UnderLUKS2WithoutResize (15471 ms) 2024-09-14T04:07:22.649 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.LUKS2UnderLUKS1WithoutResize 2024-09-14T04:07:22.649 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:07:23.113 INFO:teuthology.orchestra.run.smithi039.stdout:true 2024-09-14T04:07:23.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:07:23 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T04:07:23.166+0000 7f268b1da640 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (22 PGs are or would become offline) 2024-09-14T04:07:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:23 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:23.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:07:23.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:23 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:23.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:23 smithi039 ceph-mon[106470]: pgmap v598: 521 pgs: 61 active+undersized, 19 active+undersized+degraded, 441 active+clean; 72 MiB data, 617 MiB used, 715 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.0 MiB/s wr, 14 op/s; 33/735 objects degraded (4.490%); 2.7 MiB/s, 2 keys/s, 1 objects/s recovering 2024-09-14T04:07:23.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:23 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:23.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:23 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:23.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:07:23.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:07:23.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:23 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:23.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:23 smithi039 ceph-mon[106470]: from='client.44540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:07:23.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:07:23.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:23.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:23.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:23.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-14T04:07:23.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:23 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:23.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:07:23.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:23 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:23.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:23 smithi039 ceph-mon[111235]: pgmap v598: 521 pgs: 61 active+undersized, 19 active+undersized+degraded, 441 active+clean; 72 MiB data, 617 MiB used, 715 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.0 MiB/s wr, 14 op/s; 33/735 objects degraded (4.490%); 2.7 MiB/s, 2 keys/s, 1 objects/s recovering 2024-09-14T04:07:23.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:23 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:23.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:23 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:23.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:07:23.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:07:23.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:23 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:23.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:23 smithi039 ceph-mon[111235]: from='client.44540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:07:23.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:07:23.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:23.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:23.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:23.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-14T04:07:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:23 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:07:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:23 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:23 smithi129 ceph-mon[101316]: pgmap v598: 521 pgs: 61 active+undersized, 19 active+undersized+degraded, 441 active+clean; 72 MiB data, 617 MiB used, 715 GiB / 715 GiB avail; 1.6 MiB/s rd, 2.0 MiB/s wr, 14 op/s; 33/735 objects degraded (4.490%); 2.7 MiB/s, 2 keys/s, 1 objects/s recovering 2024-09-14T04:07:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:23 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:23 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:23.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:07:23.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:07:23.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:23 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:23.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:23 smithi129 ceph-mon[101316]: from='client.44540 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:07:23.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:07:23.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:23.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:23.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:23.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-14T04:07:23.718 INFO:teuthology.orchestra.run.smithi039.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T04:07:23.719 INFO:teuthology.orchestra.run.smithi039.stdout:alertmanager.a smithi039 *:9093,9094 running (9m) 5s ago 15m 21.1M - 0.27.0 11f11916f8cd 6b564e1fc909 2024-09-14T04:07:23.719 INFO:teuthology.orchestra.run.smithi039.stdout:grafana.a smithi129 *:3000 running (9m) 41s ago 15m 80.5M - 10.4.0 c8b91775d855 f0492c19de92 2024-09-14T04:07:23.719 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi039.bbwiwf smithi039 running (14m) 5s ago 14m 26.0M - 18.2.4-845-ge4e227b3 67ddd33fa72f 91e6c8f85681 2024-09-14T04:07:23.719 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi129.ybjqcw smithi129 running (14m) 41s ago 14m 17.3M - 18.2.4-845-ge4e227b3 67ddd33fa72f a3fab5b37b71 2024-09-14T04:07:23.719 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.x smithi129 *:8443,9283,8765 running (10m) 41s ago 19m 458M - 19.3.0-4907-ga2782315 8801d2529e43 448bd4e146cb 2024-09-14T04:07:23.719 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.y smithi039 *:8443,9283,8765 running (11m) 5s ago 21m 538M - 19.3.0-4907-ga2782315 8801d2529e43 cccc87f635a6 2024-09-14T04:07:23.719 INFO:teuthology.orchestra.run.smithi039.stdout:mon.a smithi039 running (8m) 5s ago 21m 80.7M 2048M 19.3.0-4907-ga2782315 8801d2529e43 7198d4eb8e4c 2024-09-14T04:07:23.719 INFO:teuthology.orchestra.run.smithi039.stdout:mon.b smithi129 running (6m) 41s ago 19m 54.6M 2048M 19.3.0-4907-ga2782315 8801d2529e43 cff2519560f1 2024-09-14T04:07:23.719 INFO:teuthology.orchestra.run.smithi039.stdout:mon.c smithi039 running (8m) 5s ago 19m 63.2M 2048M 19.3.0-4907-ga2782315 8801d2529e43 4b8f0f3864b1 2024-09-14T04:07:23.719 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.a smithi039 *:9100 running (10m) 5s ago 14m 10.6M - 1.7.0 72c9c2088986 7dd5945adb64 2024-09-14T04:07:23.719 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.b smithi129 *:9100 running (10m) 41s ago 14m 11.5M - 1.7.0 72c9c2088986 1fedb5f257e7 2024-09-14T04:07:23.719 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0 smithi039 running (97s) 5s ago 19m 62.1M 2480M 19.3.0-4907-ga2782315 8801d2529e43 36686aa8a5e4 2024-09-14T04:07:23.720 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1 smithi039 running (46s) 5s ago 18m 69.7M 2480M 19.3.0-4907-ga2782315 8801d2529e43 6ff655878a34 2024-09-14T04:07:23.720 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2 smithi039 running (10s) 5s ago 18m 14.6M 2480M 19.3.0-4907-ga2782315 8801d2529e43 662e3d705b70 2024-09-14T04:07:23.720 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3 smithi039 running (17m) 5s ago 17m 144M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f dff9e665bd9b 2024-09-14T04:07:23.720 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4 smithi129 running (17m) 41s ago 17m 134M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 5b64727311b7 2024-09-14T04:07:23.720 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5 smithi129 running (17m) 41s ago 17m 119M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f f2d991a754f7 2024-09-14T04:07:23.720 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6 smithi129 running (16m) 41s ago 16m 136M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 3060eaf44106 2024-09-14T04:07:23.720 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7 smithi129 running (16m) 41s ago 16m 129M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 4c13b97060d1 2024-09-14T04:07:23.720 INFO:teuthology.orchestra.run.smithi039.stdout:prometheus.a smithi129 *:9095 running (9m) 41s ago 15m 60.3M - 2.51.0 1d3b7f56885b cbe583f852c3 2024-09-14T04:07:24.097 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:07:24.097 INFO:teuthology.orchestra.run.smithi039.stdout: "mon": { 2024-09-14T04:07:24.097 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T04:07:24.097 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:07:24.097 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr": { 2024-09-14T04:07:24.097 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T04:07:24.097 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:07:24.097 INFO:teuthology.orchestra.run.smithi039.stdout: "osd": { 2024-09-14T04:07:24.097 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 5, 2024-09-14T04:07:24.097 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T04:07:24.098 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:07:24.098 INFO:teuthology.orchestra.run.smithi039.stdout: "mds": { 2024-09-14T04:07:24.098 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T04:07:24.098 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:07:24.098 INFO:teuthology.orchestra.run.smithi039.stdout: "overall": { 2024-09-14T04:07:24.098 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 7, 2024-09-14T04:07:24.098 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 8 2024-09-14T04:07:24.098 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:07:24.098 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:07:24.428 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:07:24.429 INFO:teuthology.orchestra.run.smithi039.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T04:07:24.429 INFO:teuthology.orchestra.run.smithi039.stdout: "in_progress": true, 2024-09-14T04:07:24.429 INFO:teuthology.orchestra.run.smithi039.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (1 remaining).", 2024-09-14T04:07:24.429 INFO:teuthology.orchestra.run.smithi039.stdout: "services_complete": [], 2024-09-14T04:07:24.429 INFO:teuthology.orchestra.run.smithi039.stdout: "progress": "3/8 daemons upgraded", 2024-09-14T04:07:24.429 INFO:teuthology.orchestra.run.smithi039.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T04:07:24.429 INFO:teuthology.orchestra.run.smithi039.stdout: "is_paused": false 2024-09-14T04:07:24.429 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:07:24.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:24 smithi129 ceph-mon[101316]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-14T04:07:24.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:24 smithi129 ceph-mon[101316]: Upgrade: unsafe to stop osd(s) at this time (22 PGs are or would become offline) 2024-09-14T04:07:24.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:24 smithi129 ceph-mon[101316]: from='client.34617 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:07:24.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:24 smithi129 ceph-mon[101316]: from='client.44552 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:07:24.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:24 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/674555107' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:24.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:24 smithi039 ceph-mon[106470]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-14T04:07:24.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:24 smithi039 ceph-mon[106470]: Upgrade: unsafe to stop osd(s) at this time (22 PGs are or would become offline) 2024-09-14T04:07:24.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:24 smithi039 ceph-mon[106470]: from='client.34617 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:07:24.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:24 smithi039 ceph-mon[106470]: from='client.44552 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:07:24.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:24 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/674555107' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:24.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:24 smithi039 ceph-mon[111235]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-14T04:07:24.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:24 smithi039 ceph-mon[111235]: Upgrade: unsafe to stop osd(s) at this time (22 PGs are or would become offline) 2024-09-14T04:07:24.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:24 smithi039 ceph-mon[111235]: from='client.34617 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:07:24.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:24 smithi039 ceph-mon[111235]: from='client.44552 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:07:24.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:24 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/674555107' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:25 smithi129 ceph-mon[101316]: from='client.44564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:07:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:25 smithi129 ceph-mon[101316]: pgmap v599: 521 pgs: 521 active+clean; 76 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.7 MiB/s wr, 15 op/s; 4.0 MiB/s, 3 keys/s, 1 objects/s recovering 2024-09-14T04:07:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:25 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 33/735 objects degraded (4.490%), 19 pgs degraded (PG_DEGRADED) 2024-09-14T04:07:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:25 smithi039 ceph-mon[106470]: from='client.44564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:07:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:25 smithi039 ceph-mon[106470]: pgmap v599: 521 pgs: 521 active+clean; 76 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.7 MiB/s wr, 15 op/s; 4.0 MiB/s, 3 keys/s, 1 objects/s recovering 2024-09-14T04:07:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:25 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 33/735 objects degraded (4.490%), 19 pgs degraded (PG_DEGRADED) 2024-09-14T04:07:25.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:25 smithi039 ceph-mon[111235]: from='client.44564 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:07:25.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:25 smithi039 ceph-mon[111235]: pgmap v599: 521 pgs: 521 active+clean; 76 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.7 MiB/s wr, 15 op/s; 4.0 MiB/s, 3 keys/s, 1 objects/s recovering 2024-09-14T04:07:25.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:25 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 33/735 objects degraded (4.490%), 19 pgs degraded (PG_DEGRADED) 2024-09-14T04:07:26.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:26 smithi129 ceph-mon[101316]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 33/735 objects degraded (4.490%), 19 pgs degraded) 2024-09-14T04:07:26.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:26 smithi039 ceph-mon[106470]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 33/735 objects degraded (4.490%), 19 pgs degraded) 2024-09-14T04:07:26.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:26 smithi039 ceph-mon[111235]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 33/735 objects degraded (4.490%), 19 pgs degraded) 2024-09-14T04:07:27.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:27 smithi039 ceph-mon[106470]: pgmap v600: 521 pgs: 521 active+clean; 76 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.2 MiB/s wr, 12 op/s; 3.3 MiB/s, 2 keys/s, 1 objects/s recovering 2024-09-14T04:07:27.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:27 smithi039 ceph-mon[111235]: pgmap v600: 521 pgs: 521 active+clean; 76 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.2 MiB/s wr, 12 op/s; 3.3 MiB/s, 2 keys/s, 1 objects/s recovering 2024-09-14T04:07:28.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:27 smithi129 ceph-mon[101316]: pgmap v600: 521 pgs: 521 active+clean; 76 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 1.8 MiB/s rd, 2.2 MiB/s wr, 12 op/s; 3.3 MiB/s, 2 keys/s, 1 objects/s recovering 2024-09-14T04:07:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:07:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:07:28] "GET /metrics HTTP/1.1" 200 67841 "" "Prometheus/2.51.0" 2024-09-14T04:07:30.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:29 smithi129 ceph-mon[101316]: pgmap v601: 521 pgs: 521 active+clean; 76 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.9 MiB/s wr, 11 op/s; 2.9 MiB/s, 2 keys/s, 1 objects/s recovering 2024-09-14T04:07:30.275 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:29 smithi039 ceph-mon[106470]: pgmap v601: 521 pgs: 521 active+clean; 76 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.9 MiB/s wr, 11 op/s; 2.9 MiB/s, 2 keys/s, 1 objects/s recovering 2024-09-14T04:07:30.275 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:29 smithi039 ceph-mon[111235]: pgmap v601: 521 pgs: 521 active+clean; 76 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.9 MiB/s wr, 11 op/s; 2.9 MiB/s, 2 keys/s, 1 objects/s recovering 2024-09-14T04:07:32.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:31 smithi129 ceph-mon[101316]: pgmap v602: 521 pgs: 521 active+clean; 76 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.6 MiB/s wr, 9 op/s; 2.4 MiB/s, 1 keys/s, 1 objects/s recovering 2024-09-14T04:07:32.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:31 smithi039 ceph-mon[106470]: pgmap v602: 521 pgs: 521 active+clean; 76 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.6 MiB/s wr, 9 op/s; 2.4 MiB/s, 1 keys/s, 1 objects/s recovering 2024-09-14T04:07:32.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:31 smithi039 ceph-mon[111235]: pgmap v602: 521 pgs: 521 active+clean; 76 MiB data, 641 MiB used, 715 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.6 MiB/s wr, 9 op/s; 2.4 MiB/s, 1 keys/s, 1 objects/s recovering 2024-09-14T04:07:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:33 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:33 smithi129 ceph-mon[101316]: pgmap v603: 521 pgs: 521 active+clean; 101 MiB data, 719 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.6 MiB/s wr, 17 op/s; 2.1 MiB/s, 1 keys/s, 1 objects/s recovering 2024-09-14T04:07:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:33 smithi129 ceph-mon[101316]: osdmap e696: 8 total, 8 up, 8 in 2024-09-14T04:07:33.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:33 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:33.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:33 smithi039 ceph-mon[106470]: pgmap v603: 521 pgs: 521 active+clean; 101 MiB data, 719 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.6 MiB/s wr, 17 op/s; 2.1 MiB/s, 1 keys/s, 1 objects/s recovering 2024-09-14T04:07:33.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:33 smithi039 ceph-mon[106470]: osdmap e696: 8 total, 8 up, 8 in 2024-09-14T04:07:33.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:33 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:33.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:33 smithi039 ceph-mon[111235]: pgmap v603: 521 pgs: 521 active+clean; 101 MiB data, 719 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.6 MiB/s wr, 17 op/s; 2.1 MiB/s, 1 keys/s, 1 objects/s recovering 2024-09-14T04:07:33.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:33 smithi039 ceph-mon[111235]: osdmap e696: 8 total, 8 up, 8 in 2024-09-14T04:07:36.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:35 smithi129 ceph-mon[101316]: pgmap v605: 521 pgs: 521 active+clean; 117 MiB data, 755 MiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 10 op/s 2024-09-14T04:07:36.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:35 smithi039 ceph-mon[106470]: pgmap v605: 521 pgs: 521 active+clean; 117 MiB data, 755 MiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 10 op/s 2024-09-14T04:07:36.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:35 smithi039 ceph-mon[111235]: pgmap v605: 521 pgs: 521 active+clean; 117 MiB data, 755 MiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 10 op/s 2024-09-14T04:07:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:37 smithi129 ceph-mon[101316]: pgmap v606: 521 pgs: 521 active+clean; 117 MiB data, 755 MiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 10 op/s 2024-09-14T04:07:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:38.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:07:38.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:37 smithi039 ceph-mon[106470]: pgmap v606: 521 pgs: 521 active+clean; 117 MiB data, 755 MiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 10 op/s 2024-09-14T04:07:38.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:38.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:07:38.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:37 smithi039 ceph-mon[111235]: pgmap v606: 521 pgs: 521 active+clean; 117 MiB data, 755 MiB used, 714 GiB / 715 GiB avail; 5.7 KiB/s rd, 4.1 MiB/s wr, 10 op/s 2024-09-14T04:07:38.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:38.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:07:38.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:07:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:07:38] "GET /metrics HTTP/1.1" 200 67991 "" "Prometheus/2.51.0" 2024-09-14T04:07:39.061 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:38 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-14T04:07:39.061 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:38 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-14T04:07:39.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:38 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16} : dispatch 2024-09-14T04:07:40.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:39 smithi129 ceph-mon[101316]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-14T04:07:40.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:39 smithi129 ceph-mon[101316]: Upgrade: osd.3 is safe to restart 2024-09-14T04:07:40.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:39 smithi129 ceph-mon[101316]: pgmap v607: 521 pgs: 521 active+clean; 121 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 53 KiB/s rd, 4.5 MiB/s wr, 35 op/s 2024-09-14T04:07:40.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:39 smithi039 ceph-mon[106470]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-14T04:07:40.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:39 smithi039 ceph-mon[106470]: Upgrade: osd.3 is safe to restart 2024-09-14T04:07:40.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:39 smithi039 ceph-mon[106470]: pgmap v607: 521 pgs: 521 active+clean; 121 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 53 KiB/s rd, 4.5 MiB/s wr, 35 op/s 2024-09-14T04:07:40.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:39 smithi039 ceph-mon[111235]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-09-14T04:07:40.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:39 smithi039 ceph-mon[111235]: Upgrade: osd.3 is safe to restart 2024-09-14T04:07:40.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:39 smithi039 ceph-mon[111235]: pgmap v607: 521 pgs: 521 active+clean; 121 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 53 KiB/s rd, 4.5 MiB/s wr, 35 op/s 2024-09-14T04:07:40.934 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.LUKS2UnderLUKS1WithoutResize (18284 ms) 2024-09-14T04:07:40.934 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.EncryptionFormatNoData 2024-09-14T04:07:40.934 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:07:41.503 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:41 smithi039 ceph-mon[106470]: Upgrade: Updating osd.3 2024-09-14T04:07:41.504 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:41 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:41.504 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:41 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-14T04:07:41.504 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:41 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:07:41.504 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:41 smithi039 ceph-mon[106470]: Deploying daemon osd.3 on smithi039 2024-09-14T04:07:41.504 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:41 smithi039 ceph-mon[111235]: Upgrade: Updating osd.3 2024-09-14T04:07:41.504 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:41 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:41.504 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:41 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-14T04:07:41.504 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:41 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:07:41.504 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:41 smithi039 ceph-mon[111235]: Deploying daemon osd.3 on smithi039 2024-09-14T04:07:41.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:41 smithi129 ceph-mon[101316]: Upgrade: Updating osd.3 2024-09-14T04:07:41.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:41 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:41.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:41 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "osd.3"} : dispatch 2024-09-14T04:07:41.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:41 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:07:41.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:41 smithi129 ceph-mon[101316]: Deploying daemon osd.3 on smithi039 2024-09-14T04:07:42.132 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:42 smithi039 ceph-mon[106470]: pgmap v608: 521 pgs: 521 active+clean; 121 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 83 KiB/s rd, 4.5 MiB/s wr, 36 op/s 2024-09-14T04:07:42.132 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:42 smithi039 ceph-mon[111235]: pgmap v608: 521 pgs: 521 active+clean; 121 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 83 KiB/s rd, 4.5 MiB/s wr, 36 op/s 2024-09-14T04:07:42.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:42 smithi129 ceph-mon[101316]: pgmap v608: 521 pgs: 521 active+clean; 121 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 83 KiB/s rd, 4.5 MiB/s wr, 36 op/s 2024-09-14T04:07:43.290 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:43 smithi039 systemd[1]: Stopping Ceph osd.3 for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T04:07:43.790 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:43 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3[64221]: 2024-09-14T04:07:43.390+0000 7fd34286b640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:07:43.790 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:43 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3[64221]: 2024-09-14T04:07:43.390+0000 7fd34286b640 -1 osd.3 696 *** Got signal Terminated *** 2024-09-14T04:07:43.790 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:43 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3[64221]: 2024-09-14T04:07:43.390+0000 7fd34286b640 -1 osd.3 696 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T04:07:44.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:44 smithi129 ceph-mon[101316]: pgmap v609: 521 pgs: 521 active+clean; 121 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 2.0 MiB/s wr, 33 op/s 2024-09-14T04:07:44.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:44 smithi129 ceph-mon[101316]: osd.3 marked itself down and dead 2024-09-14T04:07:44.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:44 smithi039 ceph-mon[111235]: pgmap v609: 521 pgs: 521 active+clean; 121 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 2.0 MiB/s wr, 33 op/s 2024-09-14T04:07:44.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:44 smithi039 ceph-mon[111235]: osd.3 marked itself down and dead 2024-09-14T04:07:44.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:44 smithi039 ceph-mon[106470]: pgmap v609: 521 pgs: 521 active+clean; 121 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 2.0 MiB/s wr, 33 op/s 2024-09-14T04:07:44.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:44 smithi039 ceph-mon[106470]: osd.3 marked itself down and dead 2024-09-14T04:07:45.040 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:44 smithi039 podman[142970]: 2024-09-14 04:07:44.637505138 +0000 UTC m=+1.359083439 container died dff9e665bd9bab0404da9f2043b344d063184c080fef9aee56f72048c98e7a52 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-e4e227b, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-14T04:07:45.247 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.EncryptionFormatNoData (4313 ms) 2024-09-14T04:07:45.247 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.EncryptionLoadBadSize 2024-09-14T04:07:45.247 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:07:45.422 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:45 smithi129 ceph-mon[101316]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T04:07:45.423 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:45 smithi129 ceph-mon[101316]: osdmap e697: 8 total, 7 up, 8 in 2024-09-14T04:07:45.423 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:45 smithi129 ceph-mon[101316]: osd.3 marked itself dead as of e697 2024-09-14T04:07:45.425 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:45 smithi039 ceph-mon[106470]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T04:07:45.426 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:45 smithi039 ceph-mon[106470]: osdmap e697: 8 total, 7 up, 8 in 2024-09-14T04:07:45.426 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:45 smithi039 ceph-mon[106470]: osd.3 marked itself dead as of e697 2024-09-14T04:07:45.426 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:45 smithi039 ceph-mon[111235]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T04:07:45.426 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:45 smithi039 ceph-mon[111235]: osdmap e697: 8 total, 7 up, 8 in 2024-09-14T04:07:45.426 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:45 smithi039 ceph-mon[111235]: osd.3 marked itself dead as of e697 2024-09-14T04:07:45.426 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:45 smithi039 podman[142970]: 2024-09-14 04:07:45.181875071 +0000 UTC m=+1.903453373 container cleanup dff9e665bd9bab0404da9f2043b344d063184c080fef9aee56f72048c98e7a52 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=reef-e4e227b, maintainer=Guillaume Abrioux ) 2024-09-14T04:07:45.426 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:45 smithi039 bash[142970]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3 2024-09-14T04:07:45.790 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:45 smithi039 podman[142989]: 2024-09-14 04:07:45.425282644 +0000 UTC m=+0.783065974 container remove dff9e665bd9bab0404da9f2043b344d063184c080fef9aee56f72048c98e7a52 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , RELEASE=reef-e4e227b, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-14T04:07:46.155 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:46 smithi039 ceph-mon[106470]: pgmap v611: 521 pgs: 71 peering, 54 stale+active+clean, 396 active+clean; 121 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 411 KiB/s wr, 16 op/s 2024-09-14T04:07:46.156 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:46 smithi039 ceph-mon[106470]: Health check failed: Reduced data availability: 14 pgs peering (PG_AVAILABILITY) 2024-09-14T04:07:46.156 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:46 smithi039 ceph-mon[106470]: osdmap e698: 8 total, 7 up, 8 in 2024-09-14T04:07:46.156 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:45 smithi039 podman[143078]: 2024-09-14 04:07:45.901300328 +0000 UTC m=+0.114851882 container create 7611af25d6fa21acfda1d2b53c6e4a296f8251cef27bf28689a73d5df7412f62 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.build-date=20240731, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-09-14T04:07:46.156 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:45 smithi039 podman[143078]: 2024-09-14 04:07:45.804281051 +0000 UTC m=+0.017832611 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:07:46.156 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:46 smithi039 ceph-mon[111235]: pgmap v611: 521 pgs: 71 peering, 54 stale+active+clean, 396 active+clean; 121 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 411 KiB/s wr, 16 op/s 2024-09-14T04:07:46.156 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:46 smithi039 ceph-mon[111235]: Health check failed: Reduced data availability: 14 pgs peering (PG_AVAILABILITY) 2024-09-14T04:07:46.156 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:46 smithi039 ceph-mon[111235]: osdmap e698: 8 total, 7 up, 8 in 2024-09-14T04:07:46.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:46 smithi129 ceph-mon[101316]: pgmap v611: 521 pgs: 71 peering, 54 stale+active+clean, 396 active+clean; 121 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 411 KiB/s wr, 16 op/s 2024-09-14T04:07:46.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:46 smithi129 ceph-mon[101316]: Health check failed: Reduced data availability: 14 pgs peering (PG_AVAILABILITY) 2024-09-14T04:07:46.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:46 smithi129 ceph-mon[101316]: osdmap e698: 8 total, 7 up, 8 in 2024-09-14T04:07:46.540 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:46 smithi039 podman[143078]: 2024-09-14 04:07:46.154183435 +0000 UTC m=+0.367734990 container init 7611af25d6fa21acfda1d2b53c6e4a296f8251cef27bf28689a73d5df7412f62 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T04:07:46.540 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:46 smithi039 podman[143078]: 2024-09-14 04:07:46.159809563 +0000 UTC m=+0.373361110 container start 7611af25d6fa21acfda1d2b53c6e4a296f8251cef27bf28689a73d5df7412f62 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-09-14T04:07:46.540 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:46 smithi039 podman[143078]: 2024-09-14 04:07:46.262259122 +0000 UTC m=+0.475810687 container attach 7611af25d6fa21acfda1d2b53c6e4a296f8251cef27bf28689a73d5df7412f62 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=main-a278231, org.label-schema.vendor=CentOS) 2024-09-14T04:07:46.540 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:46 smithi039 podman[143078]: 2024-09-14 04:07:46.356569049 +0000 UTC m=+0.570120605 container died 7611af25d6fa21acfda1d2b53c6e4a296f8251cef27bf28689a73d5df7412f62 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=) 2024-09-14T04:07:47.290 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:46 smithi039 podman[143122]: 2024-09-14 04:07:46.982567436 +0000 UTC m=+0.619514258 container remove 7611af25d6fa21acfda1d2b53c6e4a296f8251cef27bf28689a73d5df7412f62 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3-deactivate, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-14T04:07:47.291 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:46 smithi039 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.3.service: Deactivated successfully. 2024-09-14T04:07:47.291 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:47 smithi039 systemd[1]: Stopped Ceph osd.3 for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T04:07:47.291 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:47 smithi039 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.3.service: Consumed 33.545s CPU time. 2024-09-14T04:07:47.291 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:47 smithi039 systemd[1]: Starting Ceph osd.3 for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T04:07:47.704 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:47 smithi039 podman[143228]: 2024-09-14 04:07:47.343976659 +0000 UTC m=+0.018613202 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:07:47.704 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:47 smithi039 podman[143228]: 2024-09-14 04:07:47.450553726 +0000 UTC m=+0.125190250 container create fae12c39b76873addd037368439315ea98e6461b2f0059cd3fb3ad56202c4a22 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3-activate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, RELEASE=main-a278231, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-14T04:07:48.040 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:47 smithi039 podman[143228]: 2024-09-14 04:07:47.703193602 +0000 UTC m=+0.377830128 container init fae12c39b76873addd037368439315ea98e6461b2f0059cd3fb3ad56202c4a22 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3-activate, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=main-a278231, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T04:07:48.041 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:47 smithi039 podman[143228]: 2024-09-14 04:07:47.708416612 +0000 UTC m=+0.383053136 container start fae12c39b76873addd037368439315ea98e6461b2f0059cd3fb3ad56202c4a22 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3-activate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, org.label-schema.vendor=CentOS, ceph=True) 2024-09-14T04:07:48.041 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:47 smithi039 podman[143228]: 2024-09-14 04:07:47.794853861 +0000 UTC m=+0.469490390 container attach fae12c39b76873addd037368439315ea98e6461b2f0059cd3fb3ad56202c4a22 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3-activate, org.label-schema.vendor=CentOS, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, RELEASE=main-a278231, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux ) 2024-09-14T04:07:48.041 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:47 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3-activate[143244]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:07:48.041 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:47 smithi039 bash[143228]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:07:48.041 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:47 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3-activate[143244]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:07:48.041 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:47 smithi039 bash[143228]: Running command: /usr/bin/ceph-authtool --gen-print-key 2024-09-14T04:07:48.326 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:48 smithi039 ceph-mon[106470]: pgmap v613: 521 pgs: 71 peering, 54 stale+active+clean, 396 active+clean; 121 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 1023 B/s wr, 1 op/s 2024-09-14T04:07:48.326 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:48 smithi039 ceph-mon[111235]: pgmap v613: 521 pgs: 71 peering, 54 stale+active+clean, 396 active+clean; 121 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 1023 B/s wr, 1 op/s 2024-09-14T04:07:48.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:48 smithi129 ceph-mon[101316]: pgmap v613: 521 pgs: 71 peering, 54 stale+active+clean, 396 active+clean; 121 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 1023 B/s wr, 1 op/s 2024-09-14T04:07:48.611 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3-activate[143244]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-14T04:07:48.612 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:48 smithi039 bash[143228]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-14T04:07:48.612 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3-activate[143244]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-14T04:07:48.612 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:48 smithi039 bash[143228]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-09-14T04:07:48.612 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3-activate[143244]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-14T04:07:48.612 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:48 smithi039 bash[143228]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-09-14T04:07:48.957 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:07:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:07:48] "GET /metrics HTTP/1.1" 200 67991 "" "Prometheus/2.51.0" 2024-09-14T04:07:48.958 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3-activate[143244]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-14T04:07:48.958 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:48 smithi039 bash[143228]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-09-14T04:07:48.958 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3-activate[143244]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-14T04:07:48.958 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:48 smithi039 bash[143228]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-09-14T04:07:48.958 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3-activate[143244]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-14T04:07:48.958 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:48 smithi039 bash[143228]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-09-14T04:07:48.958 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3-activate[143244]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-14T04:07:48.958 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:48 smithi039 bash[143228]: --> ceph-volume raw activate successful for osd ID: 3 2024-09-14T04:07:48.958 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:48 smithi039 podman[143228]: 2024-09-14 04:07:48.639733654 +0000 UTC m=+1.314370188 container died fae12c39b76873addd037368439315ea98e6461b2f0059cd3fb3ad56202c4a22 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=main-a278231, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-14T04:07:49.283 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:49 smithi039 ceph-mon[106470]: Health check failed: Degraded data redundancy: 76/801 objects degraded (9.488%), 40 pgs degraded (PG_DEGRADED) 2024-09-14T04:07:49.283 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:49 smithi039 ceph-mon[111235]: Health check failed: Degraded data redundancy: 76/801 objects degraded (9.488%), 40 pgs degraded (PG_DEGRADED) 2024-09-14T04:07:49.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:49 smithi129 ceph-mon[101316]: Health check failed: Degraded data redundancy: 76/801 objects degraded (9.488%), 40 pgs degraded (PG_DEGRADED) 2024-09-14T04:07:49.540 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:49 smithi039 podman[143405]: 2024-09-14 04:07:49.282272456 +0000 UTC m=+0.636305233 container remove fae12c39b76873addd037368439315ea98e6461b2f0059cd3fb3ad56202c4a22 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3-activate, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, RELEASE=main-a278231, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-14T04:07:49.540 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:49 smithi039 podman[143469]: 2024-09-14 04:07:49.401542212 +0000 UTC m=+0.022741288 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T04:07:49.540 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:49 smithi039 podman[143469]: 2024-09-14 04:07:49.525224669 +0000 UTC m=+0.146423727 container create 645ba846152cdedf7f4cbcbfbcb3354428da4b043a353cd003969652470c1657 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, RELEASE=main-a278231, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T04:07:50.040 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:49 smithi039 podman[143469]: 2024-09-14 04:07:49.735840469 +0000 UTC m=+0.357039528 container init 645ba846152cdedf7f4cbcbfbcb3354428da4b043a353cd003969652470c1657 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731) 2024-09-14T04:07:50.040 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:49 smithi039 podman[143469]: 2024-09-14 04:07:49.74077102 +0000 UTC m=+0.361970079 container start 645ba846152cdedf7f4cbcbfbcb3354428da4b043a353cd003969652470c1657 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=main-a278231) 2024-09-14T04:07:50.041 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:49 smithi039 ceph-osd[143507]: -- 172.21.15.39:0/422701661 <== mon.2 v2:172.21.15.39:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5588e7261a40 con 0x5588e8030400 2024-09-14T04:07:50.041 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:49 smithi039 bash[143469]: 645ba846152cdedf7f4cbcbfbcb3354428da4b043a353cd003969652470c1657 2024-09-14T04:07:50.041 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:49 smithi039 systemd[1]: Started Ceph osd.3 for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T04:07:50.514 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:50 smithi039 ceph-mon[106470]: pgmap v614: 521 pgs: 109 active+undersized, 71 peering, 40 active+undersized+degraded, 301 active+clean; 125 MiB data, 784 MiB used, 714 GiB / 715 GiB avail; 513 KiB/s wr, 1 op/s; 76/801 objects degraded (9.488%) 2024-09-14T04:07:50.514 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:50 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:50.515 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:50 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:50.515 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:50 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:50.515 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:50 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:07:50.515 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:50 smithi039 ceph-mon[111235]: pgmap v614: 521 pgs: 109 active+undersized, 71 peering, 40 active+undersized+degraded, 301 active+clean; 125 MiB data, 784 MiB used, 714 GiB / 715 GiB avail; 513 KiB/s wr, 1 op/s; 76/801 objects degraded (9.488%) 2024-09-14T04:07:50.515 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:50 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:50.515 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:50 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:50.515 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:50 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:50.515 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:50 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:07:50.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:50 smithi129 ceph-mon[101316]: pgmap v614: 521 pgs: 109 active+undersized, 71 peering, 40 active+undersized+degraded, 301 active+clean; 125 MiB data, 784 MiB used, 714 GiB / 715 GiB avail; 513 KiB/s wr, 1 op/s; 76/801 objects degraded (9.488%) 2024-09-14T04:07:50.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:50 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:50.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:50 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:50.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:50 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:50.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:50 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:07:51.061 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.EncryptionLoadBadSize (5814 ms) 2024-09-14T04:07:51.061 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.EncryptionLoadBadStripePattern 2024-09-14T04:07:51.061 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/test_librbd.cc:2905: Skipped 2024-09-14T04:07:51.061 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T04:07:51.061 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestLibRBD.EncryptionLoadBadStripePattern (0 ms) 2024-09-14T04:07:51.061 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.EncryptionLoadFormatMismatch 2024-09-14T04:07:51.061 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:07:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:51 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:51 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:51 smithi129 ceph-mon[101316]: Health check update: Reduced data availability: 8 pgs inactive, 14 pgs peering (PG_AVAILABILITY) 2024-09-14T04:07:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:51 smithi129 ceph-mon[101316]: pgmap v615: 521 pgs: 161 active+undersized, 59 active+undersized+degraded, 301 active+clean; 125 MiB data, 784 MiB used, 714 GiB / 715 GiB avail; 513 KiB/s wr, 2 op/s; 118/807 objects degraded (14.622%) 2024-09-14T04:07:51.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:51 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:51.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:51 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:51.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:51 smithi039 ceph-mon[106470]: Health check update: Reduced data availability: 8 pgs inactive, 14 pgs peering (PG_AVAILABILITY) 2024-09-14T04:07:51.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:51 smithi039 ceph-mon[106470]: pgmap v615: 521 pgs: 161 active+undersized, 59 active+undersized+degraded, 301 active+clean; 125 MiB data, 784 MiB used, 714 GiB / 715 GiB avail; 513 KiB/s wr, 2 op/s; 118/807 objects degraded (14.622%) 2024-09-14T04:07:51.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:51 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:51.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:51 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:51.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:51 smithi039 ceph-mon[111235]: Health check update: Reduced data availability: 8 pgs inactive, 14 pgs peering (PG_AVAILABILITY) 2024-09-14T04:07:51.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:51 smithi039 ceph-mon[111235]: pgmap v615: 521 pgs: 161 active+undersized, 59 active+undersized+degraded, 301 active+clean; 125 MiB data, 784 MiB used, 714 GiB / 715 GiB avail; 513 KiB/s wr, 2 op/s; 118/807 objects degraded (14.622%) 2024-09-14T04:07:51.895 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:51 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3[143484]: 2024-09-14T04:07:51.622+0000 7f003f0ad640 -1 Falling back to public interface 2024-09-14T04:07:52.497 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:52 smithi039 ceph-mon[106470]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 14 pgs peering) 2024-09-14T04:07:52.498 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:52 smithi039 ceph-mon[111235]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 14 pgs peering) 2024-09-14T04:07:52.512 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:52 smithi129 ceph-mon[101316]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 8 pgs inactive, 14 pgs peering) 2024-09-14T04:07:53.502 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:53 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:53.502 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:53 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:07:53.503 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:53 smithi039 ceph-mon[106470]: pgmap v616: 521 pgs: 161 active+undersized, 59 active+undersized+degraded, 301 active+clean; 129 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 926 KiB/s wr, 2 op/s; 118/813 objects degraded (14.514%) 2024-09-14T04:07:53.503 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:53 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:53.503 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:53 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:07:53.503 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:53 smithi039 ceph-mon[111235]: pgmap v616: 521 pgs: 161 active+undersized, 59 active+undersized+degraded, 301 active+clean; 129 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 926 KiB/s wr, 2 op/s; 118/813 objects degraded (14.514%) 2024-09-14T04:07:53.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:53 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:53.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:53 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:07:53.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:53 smithi129 ceph-mon[101316]: pgmap v616: 521 pgs: 161 active+undersized, 59 active+undersized+degraded, 301 active+clean; 129 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 926 KiB/s wr, 2 op/s; 118/813 objects degraded (14.514%) 2024-09-14T04:07:54.787 INFO:teuthology.orchestra.run.smithi039.stdout:true 2024-09-14T04:07:55.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:54 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:55.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:54 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:55.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:54 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:55.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:54 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:55.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:54 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:55.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:54 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:55.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:54 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:55.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:54 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:55.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:54 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:55.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:54 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:55.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:54 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:55.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:54 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:55.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:54 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:55.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:54 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:55.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:54 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:55.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:54 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:55.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:54 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:55.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:54 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:55.435 INFO:teuthology.orchestra.run.smithi039.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-09-14T04:07:55.435 INFO:teuthology.orchestra.run.smithi039.stdout:alertmanager.a smithi039 *:9093,9094 running (10m) 1s ago 15m 21.1M - 0.27.0 11f11916f8cd 6b564e1fc909 2024-09-14T04:07:55.435 INFO:teuthology.orchestra.run.smithi039.stdout:grafana.a smithi129 *:3000 running (9m) 1s ago 15m 82.3M - 10.4.0 c8b91775d855 f0492c19de92 2024-09-14T04:07:55.435 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi039.bbwiwf smithi039 running (14m) 1s ago 14m 26.3M - 18.2.4-845-ge4e227b3 67ddd33fa72f 91e6c8f85681 2024-09-14T04:07:55.435 INFO:teuthology.orchestra.run.smithi039.stdout:mds.foo.smithi129.ybjqcw smithi129 running (14m) 1s ago 14m 17.6M - 18.2.4-845-ge4e227b3 67ddd33fa72f a3fab5b37b71 2024-09-14T04:07:55.436 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.x smithi129 *:8443,9283,8765 running (11m) 1s ago 19m 458M - 19.3.0-4907-ga2782315 8801d2529e43 448bd4e146cb 2024-09-14T04:07:55.436 INFO:teuthology.orchestra.run.smithi039.stdout:mgr.y smithi039 *:8443,9283,8765 running (11m) 1s ago 21m 540M - 19.3.0-4907-ga2782315 8801d2529e43 cccc87f635a6 2024-09-14T04:07:55.436 INFO:teuthology.orchestra.run.smithi039.stdout:mon.a smithi039 running (8m) 1s ago 21m 83.2M 2048M 19.3.0-4907-ga2782315 8801d2529e43 7198d4eb8e4c 2024-09-14T04:07:55.436 INFO:teuthology.orchestra.run.smithi039.stdout:mon.b smithi129 running (6m) 1s ago 20m 54.4M 2048M 19.3.0-4907-ga2782315 8801d2529e43 cff2519560f1 2024-09-14T04:07:55.436 INFO:teuthology.orchestra.run.smithi039.stdout:mon.c smithi039 running (8m) 1s ago 20m 66.0M 2048M 19.3.0-4907-ga2782315 8801d2529e43 4b8f0f3864b1 2024-09-14T04:07:55.436 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.a smithi039 *:9100 running (10m) 1s ago 15m 10.9M - 1.7.0 72c9c2088986 7dd5945adb64 2024-09-14T04:07:55.436 INFO:teuthology.orchestra.run.smithi039.stdout:node-exporter.b smithi129 *:9100 running (10m) 1s ago 15m 11.2M - 1.7.0 72c9c2088986 1fedb5f257e7 2024-09-14T04:07:55.436 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0 smithi039 running (2m) 1s ago 19m 64.4M 2480M 19.3.0-4907-ga2782315 8801d2529e43 36686aa8a5e4 2024-09-14T04:07:55.436 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1 smithi039 running (77s) 1s ago 19m 82.0M 2480M 19.3.0-4907-ga2782315 8801d2529e43 6ff655878a34 2024-09-14T04:07:55.436 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2 smithi039 running (41s) 1s ago 18m 72.4M 2480M 19.3.0-4907-ga2782315 8801d2529e43 662e3d705b70 2024-09-14T04:07:55.436 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3 smithi039 running (6s) 1s ago 18m 11.9M 2480M 19.3.0-4907-ga2782315 8801d2529e43 645ba846152c 2024-09-14T04:07:55.436 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4 smithi129 running (17m) 1s ago 17m 134M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 5b64727311b7 2024-09-14T04:07:55.436 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5 smithi129 running (17m) 1s ago 17m 121M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f f2d991a754f7 2024-09-14T04:07:55.437 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6 smithi129 running (17m) 1s ago 17m 145M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 3060eaf44106 2024-09-14T04:07:55.437 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7 smithi129 running (16m) 1s ago 16m 136M 2480M 18.2.4-845-ge4e227b3 67ddd33fa72f 4c13b97060d1 2024-09-14T04:07:55.437 INFO:teuthology.orchestra.run.smithi039.stdout:prometheus.a smithi129 *:9095 running (10m) 1s ago 16m 60.3M - 2.51.0 1d3b7f56885b cbe583f852c3 2024-09-14T04:07:55.540 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:55 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3[143484]: 2024-09-14T04:07:55.400+0000 7f003f0ad640 -1 osd.3 697 log_to_monitors true 2024-09-14T04:07:55.794 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:07:55.794 INFO:teuthology.orchestra.run.smithi039.stdout: "mon": { 2024-09-14T04:07:55.794 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T04:07:55.795 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:07:55.795 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr": { 2024-09-14T04:07:55.795 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 2 2024-09-14T04:07:55.795 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:07:55.795 INFO:teuthology.orchestra.run.smithi039.stdout: "osd": { 2024-09-14T04:07:55.795 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 4, 2024-09-14T04:07:55.795 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 3 2024-09-14T04:07:55.795 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:07:55.795 INFO:teuthology.orchestra.run.smithi039.stdout: "mds": { 2024-09-14T04:07:55.795 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 2 2024-09-14T04:07:55.795 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:07:55.795 INFO:teuthology.orchestra.run.smithi039.stdout: "overall": { 2024-09-14T04:07:55.795 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 18.2.4-845-ge4e227b3 (e4e227b37858f5109138494e01e5ba1ba7b8ffe8) reef (stable)": 6, 2024-09-14T04:07:55.795 INFO:teuthology.orchestra.run.smithi039.stdout: "ceph version 19.3.0-4907-ga2782315 (a2782315255a3e7d5f6cdc9eb35785187ff6ddf8) squid (dev)": 8 2024-09-14T04:07:55.795 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:07:55.796 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:07:56.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:55 smithi129 ceph-mon[101316]: from='client.44573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:07:56.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:55 smithi129 ceph-mon[101316]: pgmap v617: 521 pgs: 161 active+undersized, 59 active+undersized+degraded, 301 active+clean; 129 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 0 B/s rd, 820 KiB/s wr, 19 op/s; 120/819 objects degraded (14.652%) 2024-09-14T04:07:56.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:55 smithi129 ceph-mon[101316]: from='client.44579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:07:56.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:55 smithi129 ceph-mon[101316]: osdmap e699: 8 total, 7 up, 8 in 2024-09-14T04:07:56.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:55 smithi129 ceph-mon[101316]: from='osd.3 [v2:172.21.15.39:6826/2092194762,v1:172.21.15.39:6827/2092194762]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-14T04:07:56.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:55 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 118/813 objects degraded (14.514%), 59 pgs degraded (PG_DEGRADED) 2024-09-14T04:07:56.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:55 smithi039 ceph-mon[111235]: from='client.44573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:07:56.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:55 smithi039 ceph-mon[111235]: pgmap v617: 521 pgs: 161 active+undersized, 59 active+undersized+degraded, 301 active+clean; 129 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 0 B/s rd, 820 KiB/s wr, 19 op/s; 120/819 objects degraded (14.652%) 2024-09-14T04:07:56.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:55 smithi039 ceph-mon[111235]: from='client.44579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:07:56.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:55 smithi039 ceph-mon[111235]: osdmap e699: 8 total, 7 up, 8 in 2024-09-14T04:07:56.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:55 smithi039 ceph-mon[111235]: from='osd.3 [v2:172.21.15.39:6826/2092194762,v1:172.21.15.39:6827/2092194762]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-14T04:07:56.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:55 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 118/813 objects degraded (14.514%), 59 pgs degraded (PG_DEGRADED) 2024-09-14T04:07:56.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:55 smithi039 ceph-mon[106470]: from='client.44573 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:07:56.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:55 smithi039 ceph-mon[106470]: pgmap v617: 521 pgs: 161 active+undersized, 59 active+undersized+degraded, 301 active+clean; 129 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 0 B/s rd, 820 KiB/s wr, 19 op/s; 120/819 objects degraded (14.652%) 2024-09-14T04:07:56.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:55 smithi039 ceph-mon[106470]: from='client.44579 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:07:56.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:55 smithi039 ceph-mon[106470]: osdmap e699: 8 total, 7 up, 8 in 2024-09-14T04:07:56.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:55 smithi039 ceph-mon[106470]: from='osd.3 [v2:172.21.15.39:6826/2092194762,v1:172.21.15.39:6827/2092194762]' entity='osd.3' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]} : dispatch 2024-09-14T04:07:56.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:55 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 118/813 objects degraded (14.514%), 59 pgs degraded (PG_DEGRADED) 2024-09-14T04:07:56.093 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:07:56.094 INFO:teuthology.orchestra.run.smithi039.stdout: "target_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T04:07:56.094 INFO:teuthology.orchestra.run.smithi039.stdout: "in_progress": true, 2024-09-14T04:07:56.094 INFO:teuthology.orchestra.run.smithi039.stdout: "which": "Upgrading daemons of type(s) osd. Upgrade limited to 4 daemons (0 remaining).", 2024-09-14T04:07:56.094 INFO:teuthology.orchestra.run.smithi039.stdout: "services_complete": [], 2024-09-14T04:07:56.094 INFO:teuthology.orchestra.run.smithi039.stdout: "progress": "4/8 daemons upgraded", 2024-09-14T04:07:56.094 INFO:teuthology.orchestra.run.smithi039.stdout: "message": "Currently upgrading osd daemons", 2024-09-14T04:07:56.094 INFO:teuthology.orchestra.run.smithi039.stdout: "is_paused": false 2024-09-14T04:07:56.094 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:07:56.754 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 04:07:56 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3[143484]: 2024-09-14T04:07:56.459+0000 7f0036e57640 -1 osd.3 697 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T04:07:57.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:56 smithi129 ceph-mon[101316]: from='client.44585 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:07:57.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:56 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2111340789' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:57.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:56 smithi129 ceph-mon[101316]: from='client.44600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:07:57.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:56 smithi129 ceph-mon[101316]: from='osd.3 [v2:172.21.15.39:6826/2092194762,v1:172.21.15.39:6827/2092194762]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-14T04:07:57.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:56 smithi129 ceph-mon[101316]: osdmap e700: 8 total, 7 up, 8 in 2024-09-14T04:07:57.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:56 smithi129 ceph-mon[101316]: from='osd.3 [v2:172.21.15.39:6826/2092194762,v1:172.21.15.39:6827/2092194762]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi039", "root=default"]} : dispatch 2024-09-14T04:07:57.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:56 smithi039 ceph-mon[106470]: from='client.44585 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:07:57.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:56 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2111340789' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:57.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:56 smithi039 ceph-mon[106470]: from='client.44600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:07:57.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:56 smithi039 ceph-mon[106470]: from='osd.3 [v2:172.21.15.39:6826/2092194762,v1:172.21.15.39:6827/2092194762]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-14T04:07:57.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:56 smithi039 ceph-mon[106470]: osdmap e700: 8 total, 7 up, 8 in 2024-09-14T04:07:57.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:56 smithi039 ceph-mon[106470]: from='osd.3 [v2:172.21.15.39:6826/2092194762,v1:172.21.15.39:6827/2092194762]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi039", "root=default"]} : dispatch 2024-09-14T04:07:57.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:56 smithi039 ceph-mon[111235]: from='client.44585 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:07:57.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:56 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2111340789' entity='client.admin' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:57.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:56 smithi039 ceph-mon[111235]: from='client.44600 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:07:57.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:56 smithi039 ceph-mon[111235]: from='osd.3 [v2:172.21.15.39:6826/2092194762,v1:172.21.15.39:6827/2092194762]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-09-14T04:07:57.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:56 smithi039 ceph-mon[111235]: osdmap e700: 8 total, 7 up, 8 in 2024-09-14T04:07:57.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:56 smithi039 ceph-mon[111235]: from='osd.3 [v2:172.21.15.39:6826/2092194762,v1:172.21.15.39:6827/2092194762]' entity='osd.3' cmd={"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi039", "root=default"]} : dispatch 2024-09-14T04:07:58.015 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:57 smithi039 ceph-mon[106470]: pgmap v620: 521 pgs: 161 active+undersized, 59 active+undersized+degraded, 301 active+clean; 129 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 89 KiB/s rd, 513 KiB/s wr, 24 op/s; 120/819 objects degraded (14.652%) 2024-09-14T04:07:58.015 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:57 smithi039 ceph-mon[106470]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T04:07:58.066 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:57 smithi039 ceph-mon[106470]: osd.3 [v2:172.21.15.39:6826/2092194762,v1:172.21.15.39:6827/2092194762] boot 2024-09-14T04:07:58.066 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:57 smithi039 ceph-mon[106470]: osdmap e701: 8 total, 8 up, 8 in 2024-09-14T04:07:58.066 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:57 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-14T04:07:58.067 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:57 smithi039 ceph-mon[111235]: pgmap v620: 521 pgs: 161 active+undersized, 59 active+undersized+degraded, 301 active+clean; 129 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 89 KiB/s rd, 513 KiB/s wr, 24 op/s; 120/819 objects degraded (14.652%) 2024-09-14T04:07:58.067 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:57 smithi039 ceph-mon[111235]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T04:07:58.067 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:57 smithi039 ceph-mon[111235]: osd.3 [v2:172.21.15.39:6826/2092194762,v1:172.21.15.39:6827/2092194762] boot 2024-09-14T04:07:58.067 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:57 smithi039 ceph-mon[111235]: osdmap e701: 8 total, 8 up, 8 in 2024-09-14T04:07:58.067 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:57 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-14T04:07:58.067 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:57 smithi129 ceph-mon[101316]: pgmap v620: 521 pgs: 161 active+undersized, 59 active+undersized+degraded, 301 active+clean; 129 MiB data, 776 MiB used, 714 GiB / 715 GiB avail; 89 KiB/s rd, 513 KiB/s wr, 24 op/s; 120/819 objects degraded (14.652%) 2024-09-14T04:07:58.067 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:57 smithi129 ceph-mon[101316]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T04:07:58.068 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:57 smithi129 ceph-mon[101316]: osd.3 [v2:172.21.15.39:6826/2092194762,v1:172.21.15.39:6827/2092194762] boot 2024-09-14T04:07:58.068 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:57 smithi129 ceph-mon[101316]: osdmap e701: 8 total, 8 up, 8 in 2024-09-14T04:07:58.068 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:57 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 3} : dispatch 2024-09-14T04:07:59.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:07:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:07:58] "GET /metrics HTTP/1.1" 200 67992 "" "Prometheus/2.51.0" 2024-09-14T04:07:59.482 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:59.482 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:59.482 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:07:59.483 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:07:59.483 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:59.483 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:07:59.483 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:59.483 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:59.483 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:59.483 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:59.483 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:59.483 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:59.483 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:59.483 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:59.483 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:59.484 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T04:07:59.484 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T04:07:59.484 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T04:07:59.484 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.484 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.484 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T04:07:59.484 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T04:07:59.484 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T04:07:59.484 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-14T04:07:59.484 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T04:07:59.484 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T04:07:59.484 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T04:07:59.485 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:59.485 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:59.485 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:07:59.485 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:07:59.485 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:59.485 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:07:59.485 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:59.485 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:59.485 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:59.485 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:59.485 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:59.485 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:59.486 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:59.486 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:59.486 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:59.486 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T04:07:59.486 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T04:07:59.486 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T04:07:59.486 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.486 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.486 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T04:07:59.486 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T04:07:59.486 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T04:07:59.486 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-14T04:07:59.486 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T04:07:59.486 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T04:07:59.487 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T04:07:59.487 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T04:07:59.487 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T04:07:59.487 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T04:07:59.487 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T04:07:59.487 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T04:07:59.487 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.488 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.488 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T04:07:59.488 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T04:07:59.488 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T04:07:59.488 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T04:07:59.488 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T04:07:59.488 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T04:07:59.488 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T04:07:59.488 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T04:07:59.488 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T04:07:59.488 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.488 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.488 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.489 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.489 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.489 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.489 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.489 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.489 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T04:07:59.489 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T04:07:59.489 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T04:07:59.489 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T04:07:59.489 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T04:07:59.489 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.490 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.490 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T04:07:59.490 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T04:07:59.490 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T04:07:59.490 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T04:07:59.490 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T04:07:59.490 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T04:07:59.490 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T04:07:59.490 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T04:07:59.490 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T04:07:59.490 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.490 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.490 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.490 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.491 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.491 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.491 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.491 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.491 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.491 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.491 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.491 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.491 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T04:07:59.492 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T04:07:59.492 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T04:07:59.492 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:07:59.492 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:07:59.492 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:07:59.492 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:59.492 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:07:59.492 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:07:59.492 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:07:59.492 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:59.492 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[106470]: osdmap e702: 8 total, 8 up, 8 in 2024-09-14T04:07:59.492 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.493 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.493 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.493 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.493 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T04:07:59.493 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T04:07:59.493 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T04:07:59.493 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:07:59.493 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:07:59.493 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:07:59.493 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:59.493 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:07:59.493 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:07:59.493 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:07:59.494 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:59.494 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:07:59 smithi039 ceph-mon[111235]: osdmap e702: 8 total, 8 up, 8 in 2024-09-14T04:07:59.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:59.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:59.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:07:59.523 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:07:59.523 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:59.523 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:07:59.523 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:59.523 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:59.523 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:59.523 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:59.523 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:59.523 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:59.523 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:59.524 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:59.524 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "versions"} : dispatch 2024-09-14T04:07:59.524 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T04:07:59.524 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mgr"} : dispatch 2024-09-14T04:07:59.524 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr"}]': finished 2024-09-14T04:07:59.524 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.524 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.524 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon"}]': finished 2024-09-14T04:07:59.524 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T04:07:59.524 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.crash"} : dispatch 2024-09-14T04:07:59.524 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.crash"}]': finished 2024-09-14T04:07:59.524 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T04:07:59.525 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "osd"} : dispatch 2024-09-14T04:07:59.525 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T04:07:59.525 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mds"} : dispatch 2024-09-14T04:07:59.525 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T04:07:59.525 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rgw"} : dispatch 2024-09-14T04:07:59.525 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T04:07:59.525 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.rbd-mirror"} : dispatch 2024-09-14T04:07:59.525 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.525 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.525 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T04:07:59.526 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.ceph-exporter"} : dispatch 2024-09-14T04:07:59.526 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T04:07:59.526 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.iscsi"} : dispatch 2024-09-14T04:07:59.526 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T04:07:59.526 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nfs"} : dispatch 2024-09-14T04:07:59.526 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T04:07:59.526 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"} : dispatch 2024-09-14T04:07:59.526 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "client.nvmeof"}]': finished 2024-09-14T04:07:59.526 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.526 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.527 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.527 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.527 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.527 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.527 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.527 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.527 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.527 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.527 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.527 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "config rm", "name": "container_image", "who": "mon"} : dispatch 2024-09-14T04:07:59.527 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T04:07:59.527 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"} : dispatch 2024-09-14T04:07:59.528 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix":"config-key del","key":"mgr/cephadm/upgrade_state"}]': finished 2024-09-14T04:07:59.528 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:07:59.528 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:07:59.528 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:07:59.528 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:59.528 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:07:59.528 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:07:59.528 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:07:59.528 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:07:59.528 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:07:59 smithi129 ceph-mon[101316]: osdmap e702: 8 total, 8 up, 8 in 2024-09-14T04:08:00.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:00 smithi129 ceph-mon[101316]: Upgrade: Setting container_image for all nvmeof 2024-09-14T04:08:00.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:00 smithi129 ceph-mon[101316]: Upgrade: Finalizing container_image settings 2024-09-14T04:08:00.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:00 smithi129 ceph-mon[101316]: Upgrade: Complete! 2024-09-14T04:08:00.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:00 smithi129 ceph-mon[101316]: pgmap v623: 521 pgs: 77 peering, 105 active+undersized, 38 active+undersized+degraded, 301 active+clean; 133 MiB data, 784 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 1.0 MiB/s wr, 28 op/s; 82/840 objects degraded (9.762%) 2024-09-14T04:08:00.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:00 smithi039 ceph-mon[106470]: Upgrade: Setting container_image for all nvmeof 2024-09-14T04:08:00.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:00 smithi039 ceph-mon[106470]: Upgrade: Finalizing container_image settings 2024-09-14T04:08:00.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:00 smithi039 ceph-mon[106470]: Upgrade: Complete! 2024-09-14T04:08:00.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:00 smithi039 ceph-mon[106470]: pgmap v623: 521 pgs: 77 peering, 105 active+undersized, 38 active+undersized+degraded, 301 active+clean; 133 MiB data, 784 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 1.0 MiB/s wr, 28 op/s; 82/840 objects degraded (9.762%) 2024-09-14T04:08:00.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:00 smithi039 ceph-mon[111235]: Upgrade: Setting container_image for all nvmeof 2024-09-14T04:08:00.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:00 smithi039 ceph-mon[111235]: Upgrade: Finalizing container_image settings 2024-09-14T04:08:00.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:00 smithi039 ceph-mon[111235]: Upgrade: Complete! 2024-09-14T04:08:00.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:00 smithi039 ceph-mon[111235]: pgmap v623: 521 pgs: 77 peering, 105 active+undersized, 38 active+undersized+degraded, 301 active+clean; 133 MiB data, 784 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 1.0 MiB/s wr, 28 op/s; 82/840 objects degraded (9.762%) 2024-09-14T04:08:01.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:01 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 82/840 objects degraded (9.762%), 38 pgs degraded (PG_DEGRADED) 2024-09-14T04:08:01.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:01 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 82/840 objects degraded (9.762%), 38 pgs degraded (PG_DEGRADED) 2024-09-14T04:08:01.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:01 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 82/840 objects degraded (9.762%), 38 pgs degraded (PG_DEGRADED) 2024-09-14T04:08:02.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:02 smithi129 ceph-mon[101316]: pgmap v624: 521 pgs: 77 peering, 75 active+undersized, 23 active+undersized+degraded, 346 active+clean; 133 MiB data, 788 MiB used, 714 GiB / 715 GiB avail; 54 KiB/s rd, 748 KiB/s wr, 72 op/s; 47/846 objects degraded (5.556%); 3 B/s, 4 keys/s, 0 objects/s recovering 2024-09-14T04:08:02.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:02 smithi039 ceph-mon[106470]: pgmap v624: 521 pgs: 77 peering, 75 active+undersized, 23 active+undersized+degraded, 346 active+clean; 133 MiB data, 788 MiB used, 714 GiB / 715 GiB avail; 54 KiB/s rd, 748 KiB/s wr, 72 op/s; 47/846 objects degraded (5.556%); 3 B/s, 4 keys/s, 0 objects/s recovering 2024-09-14T04:08:02.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:02 smithi039 ceph-mon[111235]: pgmap v624: 521 pgs: 77 peering, 75 active+undersized, 23 active+undersized+degraded, 346 active+clean; 133 MiB data, 788 MiB used, 714 GiB / 715 GiB avail; 54 KiB/s rd, 748 KiB/s wr, 72 op/s; 47/846 objects degraded (5.556%); 3 B/s, 4 keys/s, 0 objects/s recovering 2024-09-14T04:08:04.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:03 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:08:04.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:03 smithi129 ceph-mon[101316]: pgmap v625: 521 pgs: 521 active+clean; 133 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 47 KiB/s rd, 632 KiB/s wr, 60 op/s; 629 KiB/s, 13 keys/s, 1 objects/s recovering 2024-09-14T04:08:04.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:03 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:08:04.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:03 smithi039 ceph-mon[106470]: pgmap v625: 521 pgs: 521 active+clean; 133 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 47 KiB/s rd, 632 KiB/s wr, 60 op/s; 629 KiB/s, 13 keys/s, 1 objects/s recovering 2024-09-14T04:08:04.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:03 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:08:04.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:03 smithi039 ceph-mon[111235]: pgmap v625: 521 pgs: 521 active+clean; 133 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 47 KiB/s rd, 632 KiB/s wr, 60 op/s; 629 KiB/s, 13 keys/s, 1 objects/s recovering 2024-09-14T04:08:05.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:04 smithi129 ceph-mon[101316]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 47/846 objects degraded (5.556%), 23 pgs degraded) 2024-09-14T04:08:05.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:04 smithi039 ceph-mon[106470]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 47/846 objects degraded (5.556%), 23 pgs degraded) 2024-09-14T04:08:05.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:04 smithi039 ceph-mon[111235]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 47/846 objects degraded (5.556%), 23 pgs degraded) 2024-09-14T04:08:06.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:05 smithi129 ceph-mon[101316]: pgmap v626: 521 pgs: 521 active+clean; 133 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 38 KiB/s rd, 515 KiB/s wr, 49 op/s; 512 KiB/s, 10 keys/s, 1 objects/s recovering 2024-09-14T04:08:06.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:05 smithi039 ceph-mon[106470]: pgmap v626: 521 pgs: 521 active+clean; 133 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 38 KiB/s rd, 515 KiB/s wr, 49 op/s; 512 KiB/s, 10 keys/s, 1 objects/s recovering 2024-09-14T04:08:06.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:05 smithi039 ceph-mon[111235]: pgmap v626: 521 pgs: 521 active+clean; 133 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 38 KiB/s rd, 515 KiB/s wr, 49 op/s; 512 KiB/s, 10 keys/s, 1 objects/s recovering 2024-09-14T04:08:08.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:07 smithi129 ceph-mon[101316]: pgmap v627: 521 pgs: 521 active+clean; 133 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 433 KiB/s wr, 41 op/s; 431 KiB/s, 9 keys/s, 1 objects/s recovering 2024-09-14T04:08:08.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:08:08.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:08:08.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:07 smithi039 ceph-mon[106470]: pgmap v627: 521 pgs: 521 active+clean; 133 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 433 KiB/s wr, 41 op/s; 431 KiB/s, 9 keys/s, 1 objects/s recovering 2024-09-14T04:08:08.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:08:08.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:08:08.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:07 smithi039 ceph-mon[111235]: pgmap v627: 521 pgs: 521 active+clean; 133 MiB data, 792 MiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 433 KiB/s wr, 41 op/s; 431 KiB/s, 9 keys/s, 1 objects/s recovering 2024-09-14T04:08:08.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:08:08.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:08:09.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:08:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:08:08] "GET /metrics HTTP/1.1" 200 67993 "" "Prometheus/2.51.0" 2024-09-14T04:08:10.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:09 smithi129 ceph-mon[101316]: pgmap v628: 521 pgs: 521 active+clean; 141 MiB data, 796 MiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 783 KiB/s wr, 34 op/s; 391 KiB/s, 8 keys/s, 1 objects/s recovering 2024-09-14T04:08:10.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:09 smithi039 ceph-mon[106470]: pgmap v628: 521 pgs: 521 active+clean; 141 MiB data, 796 MiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 783 KiB/s wr, 34 op/s; 391 KiB/s, 8 keys/s, 1 objects/s recovering 2024-09-14T04:08:10.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:09 smithi039 ceph-mon[111235]: pgmap v628: 521 pgs: 521 active+clean; 141 MiB data, 796 MiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 783 KiB/s wr, 34 op/s; 391 KiB/s, 8 keys/s, 1 objects/s recovering 2024-09-14T04:08:12.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:11 smithi129 ceph-mon[101316]: pgmap v629: 521 pgs: 521 active+clean; 145 MiB data, 821 MiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 1.0 MiB/s wr, 35 op/s; 341 KiB/s, 7 keys/s, 0 objects/s recovering 2024-09-14T04:08:12.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:11 smithi039 ceph-mon[106470]: pgmap v629: 521 pgs: 521 active+clean; 145 MiB data, 821 MiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 1.0 MiB/s wr, 35 op/s; 341 KiB/s, 7 keys/s, 0 objects/s recovering 2024-09-14T04:08:12.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:12 smithi039 ceph-mon[111235]: pgmap v629: 521 pgs: 521 active+clean; 145 MiB data, 821 MiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 1.0 MiB/s wr, 35 op/s; 341 KiB/s, 7 keys/s, 0 objects/s recovering 2024-09-14T04:08:14.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:13 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:08:14.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:13 smithi129 ceph-mon[101316]: pgmap v630: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.3 MiB/s wr, 13 op/s; 341 KiB/s, 5 keys/s, 0 objects/s recovering 2024-09-14T04:08:14.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:13 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:08:14.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:13 smithi039 ceph-mon[106470]: pgmap v630: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.3 MiB/s wr, 13 op/s; 341 KiB/s, 5 keys/s, 0 objects/s recovering 2024-09-14T04:08:14.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:13 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:08:14.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:13 smithi039 ceph-mon[111235]: pgmap v630: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.3 MiB/s wr, 13 op/s; 341 KiB/s, 5 keys/s, 0 objects/s recovering 2024-09-14T04:08:16.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:15 smithi129 ceph-mon[101316]: pgmap v631: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-09-14T04:08:16.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:15 smithi039 ceph-mon[106470]: pgmap v631: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-09-14T04:08:16.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:15 smithi039 ceph-mon[111235]: pgmap v631: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.3 MiB/s wr, 13 op/s 2024-09-14T04:08:17.271 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 04:08:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=cleanup t=2024-09-14T04:08:16.896287402Z level=info msg="Completed cleanup jobs" duration=9.540985ms 2024-09-14T04:08:17.271 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 04:08:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=sqlstore.transactions t=2024-09-14T04:08:16.897302235Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-14T04:08:17.271 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 04:08:17 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=grafana.update.checker t=2024-09-14T04:08:17.067874887Z level=info msg="Update check succeeded" duration=95.574378ms 2024-09-14T04:08:17.271 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 04:08:17 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=plugins.update.checker t=2024-09-14T04:08:17.129229882Z level=info msg="Update check succeeded" duration=61.295837ms 2024-09-14T04:08:18.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:18 smithi129 ceph-mon[101316]: pgmap v632: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-09-14T04:08:18.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:18 smithi039 ceph-mon[106470]: pgmap v632: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-09-14T04:08:18.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:18 smithi039 ceph-mon[111235]: pgmap v632: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 5.1 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-09-14T04:08:19.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:08:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:08:18] "GET /metrics HTTP/1.1" 200 67993 "" "Prometheus/2.51.0" 2024-09-14T04:08:20.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:20 smithi129 ceph-mon[101316]: pgmap v633: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-09-14T04:08:20.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:20 smithi039 ceph-mon[106470]: pgmap v633: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-09-14T04:08:20.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:20 smithi039 ceph-mon[111235]: pgmap v633: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 1.3 MiB/s wr, 15 op/s 2024-09-14T04:08:22.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:22 smithi039 ceph-mon[106470]: pgmap v634: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 682 KiB/s wr, 9 op/s 2024-09-14T04:08:22.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:22 smithi039 ceph-mon[111235]: pgmap v634: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 682 KiB/s wr, 9 op/s 2024-09-14T04:08:22.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:22 smithi129 ceph-mon[101316]: pgmap v634: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 5.5 MiB/s rd, 682 KiB/s wr, 9 op/s 2024-09-14T04:08:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:08:23.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:08:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:08:24.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:24 smithi039 ceph-mon[106470]: pgmap v635: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 341 KiB/s wr, 4 op/s 2024-09-14T04:08:24.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:24 smithi039 ceph-mon[111235]: pgmap v635: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 341 KiB/s wr, 4 op/s 2024-09-14T04:08:24.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:24 smithi129 ceph-mon[101316]: pgmap v635: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 341 KiB/s wr, 4 op/s 2024-09-14T04:08:26.409 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:26 smithi039 ceph-mon[106470]: pgmap v636: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 2 op/s 2024-09-14T04:08:26.409 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:26 smithi039 ceph-mon[111235]: pgmap v636: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 2 op/s 2024-09-14T04:08:26.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:26 smithi129 ceph-mon[101316]: pgmap v636: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 2 op/s 2024-09-14T04:08:27.279 DEBUG:teuthology.parallel:result is None 2024-09-14T04:08:28.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:28 smithi129 ceph-mon[101316]: from='client.34677 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:08:28.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:28 smithi129 ceph-mon[101316]: pgmap v637: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 2 op/s 2024-09-14T04:08:28.441 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:28 smithi039 ceph-mon[106470]: from='client.34677 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:08:28.441 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:28 smithi039 ceph-mon[106470]: pgmap v637: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 2 op/s 2024-09-14T04:08:28.442 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:28 smithi039 ceph-mon[111235]: from='client.34677 -' entity='client.admin' cmd=[{"prefix": "orch upgrade status", "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:08:28.442 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:28 smithi039 ceph-mon[111235]: pgmap v637: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 4.5 MiB/s rd, 2 op/s 2024-09-14T04:08:28.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:08:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:08:28] "GET /metrics HTTP/1.1" 200 67987 "" "Prometheus/2.51.0" 2024-09-14T04:08:30.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:30 smithi129 ceph-mon[101316]: pgmap v638: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1 op/s 2024-09-14T04:08:30.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:30 smithi039 ceph-mon[106470]: pgmap v638: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1 op/s 2024-09-14T04:08:30.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:30 smithi039 ceph-mon[111235]: pgmap v638: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1 op/s 2024-09-14T04:08:32.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:32 smithi129 ceph-mon[101316]: pgmap v639: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1 op/s 2024-09-14T04:08:32.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:32 smithi039 ceph-mon[106470]: pgmap v639: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1 op/s 2024-09-14T04:08:32.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:32 smithi039 ceph-mon[111235]: pgmap v639: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1 op/s 2024-09-14T04:08:34.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:34 smithi129 ceph-mon[101316]: pgmap v640: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1 op/s 2024-09-14T04:08:34.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:34 smithi039 ceph-mon[106470]: pgmap v640: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1 op/s 2024-09-14T04:08:34.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:34 smithi039 ceph-mon[111235]: pgmap v640: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1 op/s 2024-09-14T04:08:36.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:36 smithi129 ceph-mon[101316]: pgmap v641: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 0 op/s 2024-09-14T04:08:36.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:36 smithi039 ceph-mon[106470]: pgmap v641: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 0 op/s 2024-09-14T04:08:36.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:36 smithi039 ceph-mon[111235]: pgmap v641: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 0 op/s 2024-09-14T04:08:38.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:38 smithi129 ceph-mon[101316]: pgmap v642: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 1 op/s 2024-09-14T04:08:38.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:38 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:08:38.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:38 smithi039 ceph-mon[106470]: pgmap v642: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 1 op/s 2024-09-14T04:08:38.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:38 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:08:38.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:38 smithi039 ceph-mon[111235]: pgmap v642: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 1 op/s 2024-09-14T04:08:38.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:38 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:08:39.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:08:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:08:38] "GET /metrics HTTP/1.1" 200 67985 "" "Prometheus/2.51.0" 2024-09-14T04:08:40.524 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:40 smithi129 ceph-mon[101316]: pgmap v643: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1 op/s 2024-09-14T04:08:40.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:40 smithi039 ceph-mon[106470]: pgmap v643: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1 op/s 2024-09-14T04:08:40.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:40 smithi039 ceph-mon[111235]: pgmap v643: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 1 op/s 2024-09-14T04:08:42.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:42 smithi129 ceph-mon[101316]: pgmap v644: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1 op/s 2024-09-14T04:08:42.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:42 smithi039 ceph-mon[106470]: pgmap v644: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1 op/s 2024-09-14T04:08:42.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:42 smithi039 ceph-mon[111235]: pgmap v644: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1 op/s 2024-09-14T04:08:44.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:44 smithi129 ceph-mon[101316]: pgmap v645: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1 op/s 2024-09-14T04:08:44.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:44 smithi039 ceph-mon[106470]: pgmap v645: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1 op/s 2024-09-14T04:08:44.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:44 smithi039 ceph-mon[111235]: pgmap v645: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 1 op/s 2024-09-14T04:08:46.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:46 smithi129 ceph-mon[101316]: pgmap v646: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1 op/s 2024-09-14T04:08:46.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:46 smithi039 ceph-mon[106470]: pgmap v646: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1 op/s 2024-09-14T04:08:46.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:46 smithi039 ceph-mon[111235]: pgmap v646: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1 op/s 2024-09-14T04:08:48.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:48 smithi129 ceph-mon[101316]: pgmap v647: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 1 op/s 2024-09-14T04:08:48.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:48 smithi039 ceph-mon[106470]: pgmap v647: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 1 op/s 2024-09-14T04:08:48.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:48 smithi039 ceph-mon[111235]: pgmap v647: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s rd, 1 op/s 2024-09-14T04:08:49.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:08:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:08:48] "GET /metrics HTTP/1.1" 200 67985 "" "Prometheus/2.51.0" 2024-09-14T04:08:50.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:50 smithi129 ceph-mon[101316]: pgmap v648: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1 op/s 2024-09-14T04:08:50.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:50 smithi039 ceph-mon[106470]: pgmap v648: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1 op/s 2024-09-14T04:08:50.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:50 smithi039 ceph-mon[111235]: pgmap v648: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 1 op/s 2024-09-14T04:08:52.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:52 smithi129 ceph-mon[101316]: pgmap v649: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 807 KiB/s rd, 5 op/s 2024-09-14T04:08:52.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:52 smithi039 ceph-mon[106470]: pgmap v649: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 807 KiB/s rd, 5 op/s 2024-09-14T04:08:52.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:52 smithi039 ceph-mon[111235]: pgmap v649: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 807 KiB/s rd, 5 op/s 2024-09-14T04:08:53.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:53 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:08:53.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:53 smithi129 ceph-mon[101316]: pgmap v650: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 881 KiB/s rd, 8 op/s 2024-09-14T04:08:53.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:53 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:08:53.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:53 smithi039 ceph-mon[106470]: pgmap v650: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 881 KiB/s rd, 8 op/s 2024-09-14T04:08:53.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:53 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:08:53.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:53 smithi039 ceph-mon[111235]: pgmap v650: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 881 KiB/s rd, 8 op/s 2024-09-14T04:08:56.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:56 smithi039 ceph-mon[106470]: pgmap v651: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 153 KiB/s rd, 85 B/s wr, 11 op/s 2024-09-14T04:08:56.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:56 smithi039 ceph-mon[111235]: pgmap v651: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 153 KiB/s rd, 85 B/s wr, 11 op/s 2024-09-14T04:08:56.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:56 smithi129 ceph-mon[101316]: pgmap v651: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 153 KiB/s rd, 85 B/s wr, 11 op/s 2024-09-14T04:08:58.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:58 smithi039 ceph-mon[106470]: pgmap v652: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 85 B/s wr, 11 op/s 2024-09-14T04:08:58.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:58 smithi039 ceph-mon[111235]: pgmap v652: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 85 B/s wr, 11 op/s 2024-09-14T04:08:58.524 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:58 smithi129 ceph-mon[101316]: pgmap v652: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 177 KiB/s rd, 85 B/s wr, 11 op/s 2024-09-14T04:08:59.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:08:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:08:58] "GET /metrics HTTP/1.1" 200 67986 "" "Prometheus/2.51.0" 2024-09-14T04:08:59.334 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:08:59 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:08:59.335 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:08:59 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:08:59.372 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:08:59 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:09:00.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:00 smithi129 ceph-mon[101316]: pgmap v653: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 179 KiB/s rd, 85 B/s wr, 14 op/s 2024-09-14T04:09:00.380 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:00 smithi039 ceph-mon[106470]: pgmap v653: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 179 KiB/s rd, 85 B/s wr, 14 op/s 2024-09-14T04:09:00.380 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:00 smithi039 ceph-mon[111235]: pgmap v653: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 179 KiB/s rd, 85 B/s wr, 14 op/s 2024-09-14T04:09:02.392 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:02 smithi039 ceph-mon[106470]: pgmap v654: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 170 B/s wr, 16 op/s 2024-09-14T04:09:02.392 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:02 smithi039 ceph-mon[111235]: pgmap v654: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 170 B/s wr, 16 op/s 2024-09-14T04:09:02.420 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:02 smithi129 ceph-mon[101316]: pgmap v654: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 170 B/s wr, 16 op/s 2024-09-14T04:09:04.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:03 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:09:04.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:03 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:09:04.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:03 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:09:04.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:03 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:09:04.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:03 smithi129 ceph-mon[101316]: pgmap v655: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 179 KiB/s rd, 255 B/s wr, 14 op/s 2024-09-14T04:09:04.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:03 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:09:04.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:03 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:09:04.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:03 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:09:04.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:03 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:09:04.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:03 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:09:04.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:03 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:09:04.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:03 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:09:04.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:03 smithi039 ceph-mon[106470]: pgmap v655: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 179 KiB/s rd, 255 B/s wr, 14 op/s 2024-09-14T04:09:04.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:03 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:09:04.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:03 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:09:04.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:03 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:09:04.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:03 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:09:04.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:03 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:09:04.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:03 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:09:04.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:03 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:09:04.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:03 smithi039 ceph-mon[111235]: pgmap v655: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 179 KiB/s rd, 255 B/s wr, 14 op/s 2024-09-14T04:09:04.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:03 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:09:04.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:03 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:09:04.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:03 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:09:06.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:06 smithi129 ceph-mon[101316]: pgmap v656: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 110 KiB/s rd, 255 B/s wr, 16 op/s 2024-09-14T04:09:06.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:06 smithi039 ceph-mon[106470]: pgmap v656: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 110 KiB/s rd, 255 B/s wr, 16 op/s 2024-09-14T04:09:06.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:06 smithi039 ceph-mon[111235]: pgmap v656: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 110 KiB/s rd, 255 B/s wr, 16 op/s 2024-09-14T04:09:08.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:08 smithi039 ceph-mon[106470]: pgmap v657: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 107 KiB/s rd, 170 B/s wr, 14 op/s 2024-09-14T04:09:08.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:08 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:09:08.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:08 smithi039 ceph-mon[111235]: pgmap v657: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 107 KiB/s rd, 170 B/s wr, 14 op/s 2024-09-14T04:09:08.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:08 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:09:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:08 smithi129 ceph-mon[101316]: pgmap v657: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 107 KiB/s rd, 170 B/s wr, 14 op/s 2024-09-14T04:09:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:08 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:09:09.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:09:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:09:08] "GET /metrics HTTP/1.1" 200 67982 "" "Prometheus/2.51.0" 2024-09-14T04:09:10.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:10 smithi039 ceph-mon[106470]: pgmap v658: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 133 KiB/s rd, 170 B/s wr, 16 op/s 2024-09-14T04:09:10.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:10 smithi039 ceph-mon[111235]: pgmap v658: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 133 KiB/s rd, 170 B/s wr, 16 op/s 2024-09-14T04:09:10.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:10 smithi129 ceph-mon[101316]: pgmap v658: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 133 KiB/s rd, 170 B/s wr, 16 op/s 2024-09-14T04:09:12.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:12 smithi039 ceph-mon[106470]: pgmap v659: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 112 KiB/s rd, 170 B/s wr, 19 op/s 2024-09-14T04:09:12.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:12 smithi039 ceph-mon[111235]: pgmap v659: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 112 KiB/s rd, 170 B/s wr, 19 op/s 2024-09-14T04:09:12.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:12 smithi129 ceph-mon[101316]: pgmap v659: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 112 KiB/s rd, 170 B/s wr, 19 op/s 2024-09-14T04:09:14.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:14 smithi039 ceph-mon[106470]: pgmap v660: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 182 KiB/s rd, 85 B/s wr, 16 op/s 2024-09-14T04:09:14.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:14 smithi039 ceph-mon[111235]: pgmap v660: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 182 KiB/s rd, 85 B/s wr, 16 op/s 2024-09-14T04:09:14.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:14 smithi129 ceph-mon[101316]: pgmap v660: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 182 KiB/s rd, 85 B/s wr, 16 op/s 2024-09-14T04:09:16.323 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:16 smithi129 ceph-mon[101316]: pgmap v661: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 136 KiB/s rd, 18 op/s 2024-09-14T04:09:16.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:16 smithi039 ceph-mon[106470]: pgmap v661: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 136 KiB/s rd, 18 op/s 2024-09-14T04:09:16.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:16 smithi039 ceph-mon[111235]: pgmap v661: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 136 KiB/s rd, 18 op/s 2024-09-14T04:09:18.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:18 smithi129 ceph-mon[101316]: pgmap v662: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 155 KiB/s rd, 13 op/s 2024-09-14T04:09:18.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:18 smithi039 ceph-mon[106470]: pgmap v662: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 155 KiB/s rd, 13 op/s 2024-09-14T04:09:18.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:18 smithi039 ceph-mon[111235]: pgmap v662: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 155 KiB/s rd, 13 op/s 2024-09-14T04:09:19.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:09:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:09:18] "GET /metrics HTTP/1.1" 200 67982 "" "Prometheus/2.51.0" 2024-09-14T04:09:20.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:20 smithi129 ceph-mon[101316]: pgmap v663: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 206 KiB/s rd, 16 op/s 2024-09-14T04:09:20.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:20 smithi039 ceph-mon[106470]: pgmap v663: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 206 KiB/s rd, 16 op/s 2024-09-14T04:09:20.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:20 smithi039 ceph-mon[111235]: pgmap v663: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 206 KiB/s rd, 16 op/s 2024-09-14T04:09:22.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:22 smithi129 ceph-mon[101316]: pgmap v664: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 158 KiB/s rd, 16 op/s 2024-09-14T04:09:22.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:22 smithi039 ceph-mon[106470]: pgmap v664: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 158 KiB/s rd, 16 op/s 2024-09-14T04:09:22.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:22 smithi039 ceph-mon[111235]: pgmap v664: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 158 KiB/s rd, 16 op/s 2024-09-14T04:09:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:09:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:09:23.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:09:24.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:24 smithi129 ceph-mon[101316]: pgmap v665: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 204 KiB/s rd, 14 op/s 2024-09-14T04:09:24.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:24 smithi039 ceph-mon[106470]: pgmap v665: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 204 KiB/s rd, 14 op/s 2024-09-14T04:09:24.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:24 smithi039 ceph-mon[111235]: pgmap v665: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 204 KiB/s rd, 14 op/s 2024-09-14T04:09:26.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:26 smithi129 ceph-mon[101316]: pgmap v666: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-09-14T04:09:26.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:26 smithi039 ceph-mon[106470]: pgmap v666: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-09-14T04:09:26.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:26 smithi039 ceph-mon[111235]: pgmap v666: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-09-14T04:09:28.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:28 smithi129 ceph-mon[101316]: pgmap v667: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 153 KiB/s rd, 11 op/s 2024-09-14T04:09:28.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:28 smithi039 ceph-mon[106470]: pgmap v667: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 153 KiB/s rd, 11 op/s 2024-09-14T04:09:28.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:28 smithi039 ceph-mon[111235]: pgmap v667: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 153 KiB/s rd, 11 op/s 2024-09-14T04:09:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:09:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:09:28] "GET /metrics HTTP/1.1" 200 67986 "" "Prometheus/2.51.0" 2024-09-14T04:09:30.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:30 smithi129 ceph-mon[101316]: pgmap v668: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 179 KiB/s rd, 13 op/s 2024-09-14T04:09:30.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:30 smithi039 ceph-mon[106470]: pgmap v668: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 179 KiB/s rd, 13 op/s 2024-09-14T04:09:30.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:30 smithi039 ceph-mon[111235]: pgmap v668: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 179 KiB/s rd, 13 op/s 2024-09-14T04:09:32.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:32 smithi129 ceph-mon[101316]: pgmap v669: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-09-14T04:09:32.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:32 smithi039 ceph-mon[106470]: pgmap v669: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-09-14T04:09:32.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:32 smithi039 ceph-mon[111235]: pgmap v669: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-09-14T04:09:33.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:33 smithi129 ceph-mon[101316]: pgmap v670: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 157 KiB/s rd, 16 op/s 2024-09-14T04:09:33.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:33 smithi039 ceph-mon[106470]: pgmap v670: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 157 KiB/s rd, 16 op/s 2024-09-14T04:09:33.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:33 smithi039 ceph-mon[111235]: pgmap v670: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 157 KiB/s rd, 16 op/s 2024-09-14T04:09:36.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:36 smithi129 ceph-mon[101316]: pgmap v671: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 112 KiB/s rd, 18 op/s 2024-09-14T04:09:36.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:36 smithi039 ceph-mon[106470]: pgmap v671: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 112 KiB/s rd, 18 op/s 2024-09-14T04:09:36.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:36 smithi039 ceph-mon[111235]: pgmap v671: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 112 KiB/s rd, 18 op/s 2024-09-14T04:09:38.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:38 smithi129 ceph-mon[101316]: pgmap v672: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-09-14T04:09:38.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:38 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:09:38.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:38 smithi039 ceph-mon[106470]: pgmap v672: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-09-14T04:09:38.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:38 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:09:38.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:38 smithi039 ceph-mon[111235]: pgmap v672: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 16 op/s 2024-09-14T04:09:38.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:38 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:09:39.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:09:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:09:38] "GET /metrics HTTP/1.1" 200 67986 "" "Prometheus/2.51.0" 2024-09-14T04:09:40.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:40 smithi129 ceph-mon[101316]: pgmap v673: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 159 KiB/s rd, 18 op/s 2024-09-14T04:09:40.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:40 smithi039 ceph-mon[106470]: pgmap v673: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 159 KiB/s rd, 18 op/s 2024-09-14T04:09:40.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:40 smithi039 ceph-mon[111235]: pgmap v673: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 159 KiB/s rd, 18 op/s 2024-09-14T04:09:42.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:42 smithi129 ceph-mon[101316]: pgmap v674: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 114 KiB/s rd, 21 op/s 2024-09-14T04:09:42.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:42 smithi039 ceph-mon[106470]: pgmap v674: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 114 KiB/s rd, 21 op/s 2024-09-14T04:09:42.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:42 smithi039 ceph-mon[111235]: pgmap v674: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 114 KiB/s rd, 21 op/s 2024-09-14T04:09:44.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:44 smithi129 ceph-mon[101316]: pgmap v675: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 182 KiB/s rd, 16 op/s 2024-09-14T04:09:44.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:44 smithi039 ceph-mon[106470]: pgmap v675: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 182 KiB/s rd, 16 op/s 2024-09-14T04:09:44.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:44 smithi039 ceph-mon[111235]: pgmap v675: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 182 KiB/s rd, 16 op/s 2024-09-14T04:09:46.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:46 smithi129 ceph-mon[101316]: pgmap v676: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 160 KiB/s rd, 19 op/s 2024-09-14T04:09:46.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:46 smithi039 ceph-mon[106470]: pgmap v676: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 160 KiB/s rd, 19 op/s 2024-09-14T04:09:46.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:46 smithi039 ceph-mon[111235]: pgmap v676: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 160 KiB/s rd, 19 op/s 2024-09-14T04:09:48.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:48 smithi129 ceph-mon[101316]: pgmap v677: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 180 KiB/s rd, 14 op/s 2024-09-14T04:09:48.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:48 smithi039 ceph-mon[106470]: pgmap v677: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 180 KiB/s rd, 14 op/s 2024-09-14T04:09:48.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:48 smithi039 ceph-mon[111235]: pgmap v677: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 180 KiB/s rd, 14 op/s 2024-09-14T04:09:49.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:09:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:09:48] "GET /metrics HTTP/1.1" 200 67986 "" "Prometheus/2.51.0" 2024-09-14T04:09:50.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:50 smithi129 ceph-mon[101316]: pgmap v678: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 206 KiB/s rd, 16 op/s 2024-09-14T04:09:50.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:50 smithi039 ceph-mon[111235]: pgmap v678: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 206 KiB/s rd, 16 op/s 2024-09-14T04:09:50.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:50 smithi039 ceph-mon[106470]: pgmap v678: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 206 KiB/s rd, 16 op/s 2024-09-14T04:09:52.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:52 smithi129 ceph-mon[101316]: pgmap v679: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 155 KiB/s rd, 13 op/s 2024-09-14T04:09:52.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:52 smithi039 ceph-mon[106470]: pgmap v679: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 155 KiB/s rd, 13 op/s 2024-09-14T04:09:52.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:52 smithi039 ceph-mon[111235]: pgmap v679: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 155 KiB/s rd, 13 op/s 2024-09-14T04:09:53.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:53 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:09:53.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:53 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:09:53.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:53 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:09:54.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:54 smithi129 ceph-mon[101316]: pgmap v680: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 201 KiB/s rd, 11 op/s 2024-09-14T04:09:54.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:54 smithi039 ceph-mon[106470]: pgmap v680: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 201 KiB/s rd, 11 op/s 2024-09-14T04:09:54.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:54 smithi039 ceph-mon[111235]: pgmap v680: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 201 KiB/s rd, 11 op/s 2024-09-14T04:09:56.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:56 smithi129 ceph-mon[101316]: pgmap v681: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 85 B/s wr, 16 op/s 2024-09-14T04:09:56.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:56 smithi039 ceph-mon[106470]: pgmap v681: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 85 B/s wr, 16 op/s 2024-09-14T04:09:56.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:56 smithi039 ceph-mon[111235]: pgmap v681: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 85 B/s wr, 16 op/s 2024-09-14T04:09:57.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:09:57 smithi129 ceph-mon[101316]: pgmap v682: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 153 KiB/s rd, 85 B/s wr, 11 op/s 2024-09-14T04:09:57.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:09:57 smithi039 ceph-mon[106470]: pgmap v682: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 153 KiB/s rd, 85 B/s wr, 11 op/s 2024-09-14T04:09:57.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:09:57 smithi039 ceph-mon[111235]: pgmap v682: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 153 KiB/s rd, 85 B/s wr, 11 op/s 2024-09-14T04:09:59.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:09:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:09:58] "GET /metrics HTTP/1.1" 200 67986 "" "Prometheus/2.51.0" 2024-09-14T04:10:00.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:00 smithi129 ceph-mon[101316]: pgmap v683: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 155 KiB/s rd, 85 B/s wr, 13 op/s 2024-09-14T04:10:00.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:00 smithi129 ceph-mon[101316]: Health detail: HEALTH_WARN 2 pool(s) full 2024-09-14T04:10:00.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:00 smithi129 ceph-mon[101316]: [WRN] POOL_FULL: 2 pool(s) full 2024-09-14T04:10:00.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:00 smithi129 ceph-mon[101316]: pool 'test-librbd-smithi039-85658-24' is full (running out of quota) 2024-09-14T04:10:00.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:00 smithi129 ceph-mon[101316]: pool 'test-librbd-smithi039-85658-27' is full (running out of quota) 2024-09-14T04:10:00.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:00 smithi039 ceph-mon[106470]: pgmap v683: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 155 KiB/s rd, 85 B/s wr, 13 op/s 2024-09-14T04:10:00.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:00 smithi039 ceph-mon[106470]: Health detail: HEALTH_WARN 2 pool(s) full 2024-09-14T04:10:00.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:00 smithi039 ceph-mon[106470]: [WRN] POOL_FULL: 2 pool(s) full 2024-09-14T04:10:00.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:00 smithi039 ceph-mon[106470]: pool 'test-librbd-smithi039-85658-24' is full (running out of quota) 2024-09-14T04:10:00.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:00 smithi039 ceph-mon[106470]: pool 'test-librbd-smithi039-85658-27' is full (running out of quota) 2024-09-14T04:10:00.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:00 smithi039 ceph-mon[111235]: pgmap v683: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 155 KiB/s rd, 85 B/s wr, 13 op/s 2024-09-14T04:10:00.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:00 smithi039 ceph-mon[111235]: Health detail: HEALTH_WARN 2 pool(s) full 2024-09-14T04:10:00.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:00 smithi039 ceph-mon[111235]: [WRN] POOL_FULL: 2 pool(s) full 2024-09-14T04:10:00.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:00 smithi039 ceph-mon[111235]: pool 'test-librbd-smithi039-85658-24' is full (running out of quota) 2024-09-14T04:10:00.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:00 smithi039 ceph-mon[111235]: pool 'test-librbd-smithi039-85658-27' is full (running out of quota) 2024-09-14T04:10:02.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:02 smithi129 ceph-mon[101316]: pgmap v684: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 110 KiB/s rd, 170 B/s wr, 16 op/s 2024-09-14T04:10:02.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:02 smithi039 ceph-mon[106470]: pgmap v684: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 110 KiB/s rd, 170 B/s wr, 16 op/s 2024-09-14T04:10:02.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:02 smithi039 ceph-mon[111235]: pgmap v684: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 110 KiB/s rd, 170 B/s wr, 16 op/s 2024-09-14T04:10:04.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:04 smithi129 ceph-mon[101316]: pgmap v685: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 160 KiB/s rd, 255 B/s wr, 19 op/s 2024-09-14T04:10:04.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:04 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:10:04.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:04 smithi039 ceph-mon[106470]: pgmap v685: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 160 KiB/s rd, 255 B/s wr, 19 op/s 2024-09-14T04:10:04.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:04 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:10:04.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:04 smithi039 ceph-mon[111235]: pgmap v685: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 160 KiB/s rd, 255 B/s wr, 19 op/s 2024-09-14T04:10:04.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:04 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:10:06.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:06 smithi129 ceph-mon[101316]: pgmap v686: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 114 KiB/s rd, 255 B/s wr, 21 op/s 2024-09-14T04:10:06.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:06 smithi039 ceph-mon[106470]: pgmap v686: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 114 KiB/s rd, 255 B/s wr, 21 op/s 2024-09-14T04:10:06.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:06 smithi039 ceph-mon[111235]: pgmap v686: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 114 KiB/s rd, 255 B/s wr, 21 op/s 2024-09-14T04:10:08.495 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.EncryptionLoadFormatMismatch (137435 ms) 2024-09-14T04:10:08.495 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.EncryptedResize 2024-09-14T04:10:08.495 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:08 smithi129 ceph-mon[101316]: pgmap v687: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 170 B/s wr, 16 op/s 2024-09-14T04:10:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:08 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:10:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:08 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:10:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:08 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:10:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:08 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:10:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:08 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:10:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:08 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:10:08.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:08 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:10:08.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:08 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:10:08.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:08 smithi039 ceph-mon[111235]: pgmap v687: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 170 B/s wr, 16 op/s 2024-09-14T04:10:08.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:08 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:10:08.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:08 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:10:08.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:08 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:10:08.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:08 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:10:08.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:08 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:10:08.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:08 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:10:08.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:08 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:10:08.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:08 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:10:08.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:08 smithi039 ceph-mon[106470]: pgmap v687: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 170 B/s wr, 16 op/s 2024-09-14T04:10:08.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:10:08.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:10:08.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:08 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:10:08.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:10:08.542 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:10:08.542 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:08 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:10:08.542 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:08 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:10:08.542 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:10:09.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:10:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:10:08] "GET /metrics HTTP/1.1" 200 67987 "" "Prometheus/2.51.0" 2024-09-14T04:10:10.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:10 smithi129 ceph-mon[101316]: pgmap v688: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 157 KiB/s rd, 170 B/s wr, 16 op/s 2024-09-14T04:10:10.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:10 smithi039 ceph-mon[106470]: pgmap v688: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 157 KiB/s rd, 170 B/s wr, 16 op/s 2024-09-14T04:10:10.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:10 smithi039 ceph-mon[111235]: pgmap v688: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 157 KiB/s rd, 170 B/s wr, 16 op/s 2024-09-14T04:10:12.523 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:12 smithi129 ceph-mon[101316]: pgmap v689: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 131 KiB/s rd, 255 B/s wr, 14 op/s 2024-09-14T04:10:12.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:12 smithi039 ceph-mon[106470]: pgmap v689: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 131 KiB/s rd, 255 B/s wr, 14 op/s 2024-09-14T04:10:12.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:12 smithi039 ceph-mon[111235]: pgmap v689: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 131 KiB/s rd, 255 B/s wr, 14 op/s 2024-09-14T04:10:14.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:14 smithi129 ceph-mon[101316]: pgmap v690: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 129 KiB/s rd, 426 B/s wr, 11 op/s 2024-09-14T04:10:14.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:14 smithi039 ceph-mon[106470]: pgmap v690: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 129 KiB/s rd, 426 B/s wr, 11 op/s 2024-09-14T04:10:14.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:14 smithi039 ceph-mon[111235]: pgmap v690: 521 pgs: 521 active+clean; 149 MiB data, 845 MiB used, 714 GiB / 715 GiB avail; 129 KiB/s rd, 426 B/s wr, 11 op/s 2024-09-14T04:10:16.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:16 smithi129 ceph-mon[101316]: pgmap v691: 521 pgs: 521 active+clean; 153 MiB data, 857 MiB used, 714 GiB / 715 GiB avail; 79 KiB/s rd, 341 KiB/s wr, 9 op/s 2024-09-14T04:10:16.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:16 smithi039 ceph-mon[106470]: pgmap v691: 521 pgs: 521 active+clean; 153 MiB data, 857 MiB used, 714 GiB / 715 GiB avail; 79 KiB/s rd, 341 KiB/s wr, 9 op/s 2024-09-14T04:10:16.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:16 smithi039 ceph-mon[111235]: pgmap v691: 521 pgs: 521 active+clean; 153 MiB data, 857 MiB used, 714 GiB / 715 GiB avail; 79 KiB/s rd, 341 KiB/s wr, 9 op/s 2024-09-14T04:10:18.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:18 smithi129 ceph-mon[101316]: pgmap v692: 521 pgs: 521 active+clean; 153 MiB data, 865 MiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 341 KiB/s wr, 4 op/s 2024-09-14T04:10:18.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:18 smithi039 ceph-mon[106470]: pgmap v692: 521 pgs: 521 active+clean; 153 MiB data, 865 MiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 341 KiB/s wr, 4 op/s 2024-09-14T04:10:18.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:18 smithi039 ceph-mon[111235]: pgmap v692: 521 pgs: 521 active+clean; 153 MiB data, 865 MiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 341 KiB/s wr, 4 op/s 2024-09-14T04:10:19.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:10:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:10:18] "GET /metrics HTTP/1.1" 200 67987 "" "Prometheus/2.51.0" 2024-09-14T04:10:19.122 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.EncryptedResize (10626 ms) 2024-09-14T04:10:19.122 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.EncryptedFlattenSmallData 2024-09-14T04:10:19.122 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:20.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:20 smithi129 ceph-mon[101316]: pgmap v693: 521 pgs: 521 active+clean; 165 MiB data, 892 MiB used, 714 GiB / 715 GiB avail; 137 KiB/s rd, 1.6 MiB/s wr, 23 op/s 2024-09-14T04:10:20.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:20 smithi039 ceph-mon[106470]: pgmap v693: 521 pgs: 521 active+clean; 165 MiB data, 892 MiB used, 714 GiB / 715 GiB avail; 137 KiB/s rd, 1.6 MiB/s wr, 23 op/s 2024-09-14T04:10:20.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:20 smithi039 ceph-mon[111235]: pgmap v693: 521 pgs: 521 active+clean; 165 MiB data, 892 MiB used, 714 GiB / 715 GiB avail; 137 KiB/s rd, 1.6 MiB/s wr, 23 op/s 2024-09-14T04:10:21.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:21 smithi039 ceph-mon[106470]: pgmap v694: 521 pgs: 521 active+clean; 165 MiB data, 896 MiB used, 714 GiB / 715 GiB avail; 89 KiB/s rd, 1.6 MiB/s wr, 24 op/s 2024-09-14T04:10:21.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:21 smithi039 ceph-mon[111235]: pgmap v694: 521 pgs: 521 active+clean; 165 MiB data, 896 MiB used, 714 GiB / 715 GiB avail; 89 KiB/s rd, 1.6 MiB/s wr, 24 op/s 2024-09-14T04:10:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:21 smithi129 ceph-mon[101316]: pgmap v694: 521 pgs: 521 active+clean; 165 MiB data, 896 MiB used, 714 GiB / 715 GiB avail; 89 KiB/s rd, 1.6 MiB/s wr, 24 op/s 2024-09-14T04:10:22.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:10:22.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:10:23.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:10:24.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:23 smithi129 ceph-mon[101316]: pgmap v695: 521 pgs: 521 active+clean; 165 MiB data, 893 MiB used, 714 GiB / 715 GiB avail; 95 KiB/s rd, 1.6 MiB/s wr, 32 op/s 2024-09-14T04:10:24.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:23 smithi039 ceph-mon[106470]: pgmap v695: 521 pgs: 521 active+clean; 165 MiB data, 893 MiB used, 714 GiB / 715 GiB avail; 95 KiB/s rd, 1.6 MiB/s wr, 32 op/s 2024-09-14T04:10:24.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:23 smithi039 ceph-mon[111235]: pgmap v695: 521 pgs: 521 active+clean; 165 MiB data, 893 MiB used, 714 GiB / 715 GiB avail; 95 KiB/s rd, 1.6 MiB/s wr, 32 op/s 2024-09-14T04:10:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:25 smithi129 ceph-mon[101316]: osdmap e703: 8 total, 8 up, 8 in 2024-09-14T04:10:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:25 smithi129 ceph-mon[101316]: pgmap v697: 521 pgs: 521 active+clean; 177 MiB data, 909 MiB used, 714 GiB / 715 GiB avail; 114 KiB/s rd, 2.7 MiB/s wr, 39 op/s 2024-09-14T04:10:26.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:25 smithi039 ceph-mon[106470]: osdmap e703: 8 total, 8 up, 8 in 2024-09-14T04:10:26.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:25 smithi039 ceph-mon[106470]: pgmap v697: 521 pgs: 521 active+clean; 177 MiB data, 909 MiB used, 714 GiB / 715 GiB avail; 114 KiB/s rd, 2.7 MiB/s wr, 39 op/s 2024-09-14T04:10:26.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:25 smithi039 ceph-mon[111235]: osdmap e703: 8 total, 8 up, 8 in 2024-09-14T04:10:26.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:25 smithi039 ceph-mon[111235]: pgmap v697: 521 pgs: 521 active+clean; 177 MiB data, 909 MiB used, 714 GiB / 715 GiB avail; 114 KiB/s rd, 2.7 MiB/s wr, 39 op/s 2024-09-14T04:10:28.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:28 smithi129 ceph-mon[101316]: pgmap v698: 521 pgs: 521 active+clean; 181 MiB data, 921 MiB used, 714 GiB / 715 GiB avail; 172 KiB/s rd, 3.1 MiB/s wr, 40 op/s 2024-09-14T04:10:28.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:28 smithi039 ceph-mon[106470]: pgmap v698: 521 pgs: 521 active+clean; 181 MiB data, 921 MiB used, 714 GiB / 715 GiB avail; 172 KiB/s rd, 3.1 MiB/s wr, 40 op/s 2024-09-14T04:10:28.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:28 smithi039 ceph-mon[111235]: pgmap v698: 521 pgs: 521 active+clean; 181 MiB data, 921 MiB used, 714 GiB / 715 GiB avail; 172 KiB/s rd, 3.1 MiB/s wr, 40 op/s 2024-09-14T04:10:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:10:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:10:28] "GET /metrics HTTP/1.1" 200 67986 "" "Prometheus/2.51.0" 2024-09-14T04:10:30.016 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.EncryptedFlattenSmallData (10894 ms) 2024-09-14T04:10:30.016 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestIOWithIOHint 2024-09-14T04:10:30.016 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:30.326 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:30 smithi039 ceph-mon[106470]: pgmap v699: 521 pgs: 521 active+clean; 181 MiB data, 941 MiB used, 714 GiB / 715 GiB avail; 87 KiB/s rd, 1.6 MiB/s wr, 39 op/s 2024-09-14T04:10:30.327 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:30 smithi039 ceph-mon[111235]: pgmap v699: 521 pgs: 521 active+clean; 181 MiB data, 941 MiB used, 714 GiB / 715 GiB avail; 87 KiB/s rd, 1.6 MiB/s wr, 39 op/s 2024-09-14T04:10:30.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:30 smithi129 ceph-mon[101316]: pgmap v699: 521 pgs: 521 active+clean; 181 MiB data, 941 MiB used, 714 GiB / 715 GiB avail; 87 KiB/s rd, 1.6 MiB/s wr, 39 op/s 2024-09-14T04:10:31.997 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:10:31.997 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:10:31.997 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:10:31.997 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:10:31.997 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:10:31.997 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:31.997 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:10:31.997 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:31.998 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:31.998 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:10:31.998 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:31.998 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:10:31.998 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:31.998 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:31.998 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:10:31.998 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:31.998 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:10:31.998 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:31.998 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:31.998 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:10:31.998 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:31.998 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:10:31.999 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:31.999 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:31.999 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:10:31.999 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:31.999 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:10:31.999 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:31.999 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:31.999 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:10:31.999 INFO:tasks.workunit.client.0.smithi039.stdout:start compare and write 2024-09-14T04:10:31.999 INFO:tasks.workunit.client.0.smithi039.stdout:compare and wrote: 512 2024-09-14T04:10:31.999 INFO:tasks.workunit.client.0.smithi039.stdout:start compare and write 2024-09-14T04:10:31.999 INFO:tasks.workunit.client.0.smithi039.stdout:compare and wrote: 512 2024-09-14T04:10:31.999 INFO:tasks.workunit.client.0.smithi039.stdout:start compare and write 2024-09-14T04:10:32.000 INFO:tasks.workunit.client.0.smithi039.stdout:compare and wrote: 512 2024-09-14T04:10:32.000 INFO:tasks.workunit.client.0.smithi039.stdout:start compare and write 2024-09-14T04:10:32.000 INFO:tasks.workunit.client.0.smithi039.stdout:compare and wrote: 512 2024-09-14T04:10:32.000 INFO:tasks.workunit.client.0.smithi039.stdout:start compare and write 2024-09-14T04:10:32.000 INFO:tasks.workunit.client.0.smithi039.stdout:compare and wrote: 512 2024-09-14T04:10:32.000 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:32.000 INFO:tasks.workunit.client.0.smithi039.stdout:started aio compare and write 2024-09-14T04:10:32.000 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:32.000 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:32.000 INFO:tasks.workunit.client.0.smithi039.stdout:finished aio compare and write 2024-09-14T04:10:32.000 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:32.000 INFO:tasks.workunit.client.0.smithi039.stdout:started aio compare and write 2024-09-14T04:10:32.001 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:32.001 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:32.001 INFO:tasks.workunit.client.0.smithi039.stdout:finished aio compare and write 2024-09-14T04:10:32.001 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:32.001 INFO:tasks.workunit.client.0.smithi039.stdout:started aio compare and write 2024-09-14T04:10:32.001 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:32.001 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:32.001 INFO:tasks.workunit.client.0.smithi039.stdout:finished aio compare and write 2024-09-14T04:10:32.001 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:32.001 INFO:tasks.workunit.client.0.smithi039.stdout:started aio compare and write 2024-09-14T04:10:32.001 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:32.001 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:32.001 INFO:tasks.workunit.client.0.smithi039.stdout:finished aio compare and write 2024-09-14T04:10:32.001 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:32.002 INFO:tasks.workunit.client.0.smithi039.stdout:started aio compare and write 2024-09-14T04:10:32.002 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:32.002 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:32.002 INFO:tasks.workunit.client.0.smithi039.stdout:finished aio compare and write 2024-09-14T04:10:32.002 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:32.002 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:32.002 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:32.002 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:32.002 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:32.002 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:32.002 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:10:32.002 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:10:32.002 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:10:32.002 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:32.003 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:10:32.003 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:10:32.003 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:10:32.003 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:32.003 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:10:32.003 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:10:32.003 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:10:32.003 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:32.003 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:10:32.003 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:10:32.003 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:10:32.003 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:32.003 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:10:32.003 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:10:32.004 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:10:32.004 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 512~512 = 512 2024-09-14T04:10:32.004 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:32.004 INFO:tasks.workunit.client.0.smithi039.stdout:aio discard: 1536~512 = 0 2024-09-14T04:10:32.004 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:32.004 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:32.004 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:32.004 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:32.004 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:32.004 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 0 2024-09-14T04:10:32.004 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:32.004 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:32.005 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 0 2024-09-14T04:10:32.005 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:32.005 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:32.005 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 16384 2024-09-14T04:10:32.005 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:32.005 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:32.006 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 16384 2024-09-14T04:10:32.006 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:32.006 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:32.006 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:32.006 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:32.006 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 49152 2024-09-14T04:10:32.006 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:32.006 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:32.006 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 49152 2024-09-14T04:10:32.007 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:32.007 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:32.007 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 65536 2024-09-14T04:10:32.007 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:32.007 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:32.007 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 65536 2024-09-14T04:10:32.007 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:32.007 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:32.007 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:32.007 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:32.007 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 98304 2024-09-14T04:10:32.007 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:32.007 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:32.007 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 98304 2024-09-14T04:10:32.008 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:32.008 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:32.008 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 114688 2024-09-14T04:10:32.008 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:32.008 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:32.008 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 114688 2024-09-14T04:10:32.008 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:32.008 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:32.008 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:32.008 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:32.008 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 147456 2024-09-14T04:10:32.008 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:32.008 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:32.008 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 147456 2024-09-14T04:10:32.009 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:32.009 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:32.009 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 163840 2024-09-14T04:10:32.009 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:32.009 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:32.009 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 163840 2024-09-14T04:10:32.009 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:32.009 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:32.010 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:32.010 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:32.010 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 196608 2024-09-14T04:10:32.010 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:32.010 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:32.010 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 196608 2024-09-14T04:10:32.010 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:32.010 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:32.010 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 212992 2024-09-14T04:10:32.010 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:32.010 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:32.010 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 212992 2024-09-14T04:10:32.010 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:32.010 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:32.011 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:32.011 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:32.011 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:32.011 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:32.011 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:32.011 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:32.011 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:32.011 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:32.011 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:32.011 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:32.011 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:32.011 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:32.011 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:32.011 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:32.012 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:32.012 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:32.012 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:32.012 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:32.012 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:32.012 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:32.012 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:32.012 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:32.012 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:32.012 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:32.012 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:32.012 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:32.012 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:32.013 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:32.013 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:32.013 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:32.013 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:32.013 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:32.013 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:32.013 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:32.013 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:32.013 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:32.013 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:32.013 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:32.013 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:32.014 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:32.014 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:32.014 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:32.014 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:32.014 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:32.014 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:32.014 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:32.014 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:32.014 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:32.014 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:32.014 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:32.014 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:32.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:32 smithi129 ceph-mon[101316]: pgmap v700: 521 pgs: 521 active+clean; 185 MiB data, 957 MiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 2.0 MiB/s wr, 40 op/s 2024-09-14T04:10:32.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:32 smithi039 ceph-mon[106470]: pgmap v700: 521 pgs: 521 active+clean; 185 MiB data, 957 MiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 2.0 MiB/s wr, 40 op/s 2024-09-14T04:10:32.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:32 smithi039 ceph-mon[111235]: pgmap v700: 521 pgs: 521 active+clean; 185 MiB data, 957 MiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 2.0 MiB/s wr, 40 op/s 2024-09-14T04:10:33.787 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:33.787 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:33.787 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:33.787 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:33.787 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:33.787 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:33.787 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:33.787 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:33.788 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:33.788 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:33.788 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:33.788 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:33.788 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:33.788 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:33.788 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:33.788 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:33.788 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:33.788 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:33.788 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:33.788 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:33.788 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:33.788 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:33.789 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:33.789 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:33.789 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:33.789 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:33.789 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:33.789 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:33.789 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:33.789 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:33.789 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:33.789 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:33.789 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:33.789 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:33.789 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:33.790 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:33.790 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:33.790 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:33.790 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:33.790 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:33.790 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:33.790 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:33.790 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:33.790 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:33.790 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:33.790 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:33.790 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:33.790 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:33.790 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:33.791 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:33.791 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:33.791 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:33.791 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:33.791 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:33.791 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:33.791 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:33.791 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:33.791 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:33.791 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:33.791 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:33.791 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:33.791 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:33.791 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:33.791 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:33.792 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:33.792 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:33.792 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:33.792 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:33.792 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:33.792 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:33.792 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:33.792 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:33.792 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:33.792 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:33.792 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:33.793 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:33.793 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:33.793 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:33.793 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:33.793 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:33.793 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:33.793 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:33.793 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:33.793 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:33.793 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:33.793 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:33.793 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:33.794 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:33.794 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:33.794 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:33.794 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:33.794 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:33.794 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:33.794 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:33.794 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:33.794 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:33.794 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:33.794 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:33.794 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:33.794 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:33.794 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:33.795 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:33.795 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:33.795 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:33.795 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:33.795 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:33.795 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:33.795 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:33.795 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:33.795 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:33.795 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:33.795 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:33.795 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:33.795 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:33.795 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:33.796 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:33.796 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:33.796 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:33.796 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:33.796 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:33.796 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:33.796 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:10:33.796 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:10:33.796 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:10:33.796 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestIOWithIOHint (3771 ms) 2024-09-14T04:10:33.796 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestDataPoolIO 2024-09-14T04:10:34.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:34 smithi129 ceph-mon[101316]: pgmap v701: 521 pgs: 521 active+clean; 197 MiB data, 991 MiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.4 MiB/s wr, 677 op/s 2024-09-14T04:10:34.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:34 smithi039 ceph-mon[106470]: pgmap v701: 521 pgs: 521 active+clean; 197 MiB data, 991 MiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.4 MiB/s wr, 677 op/s 2024-09-14T04:10:34.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:34 smithi039 ceph-mon[111235]: pgmap v701: 521 pgs: 521 active+clean; 197 MiB data, 991 MiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 3.4 MiB/s wr, 677 op/s 2024-09-14T04:10:35.125 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:35.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:35 smithi129 ceph-mon[101316]: osdmap e704: 8 total, 8 up, 8 in 2024-09-14T04:10:35.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:35 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2351553588' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-29","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:10:35.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:35 smithi039 ceph-mon[106470]: osdmap e704: 8 total, 8 up, 8 in 2024-09-14T04:10:35.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:35 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2351553588' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-29","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:10:35.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:35 smithi039 ceph-mon[111235]: osdmap e704: 8 total, 8 up, 8 in 2024-09-14T04:10:35.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:35 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2351553588' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-29","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:10:36.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:36 smithi129 ceph-mon[101316]: pgmap v703: 553 pgs: 13 creating+peering, 19 unknown, 521 active+clean; 197 MiB data, 992 MiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.2 MiB/s wr, 674 op/s 2024-09-14T04:10:36.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:36 smithi129 ceph-mon[101316]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:10:36.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:36 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2351553588' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:10:36.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:36 smithi129 ceph-mon[101316]: osdmap e705: 8 total, 8 up, 8 in 2024-09-14T04:10:36.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:36 smithi039 ceph-mon[106470]: pgmap v703: 553 pgs: 13 creating+peering, 19 unknown, 521 active+clean; 197 MiB data, 992 MiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.2 MiB/s wr, 674 op/s 2024-09-14T04:10:36.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:36 smithi039 ceph-mon[106470]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:10:36.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:36 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2351553588' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:10:36.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:36 smithi039 ceph-mon[106470]: osdmap e705: 8 total, 8 up, 8 in 2024-09-14T04:10:36.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:36 smithi039 ceph-mon[111235]: pgmap v703: 553 pgs: 13 creating+peering, 19 unknown, 521 active+clean; 197 MiB data, 992 MiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 2.2 MiB/s wr, 674 op/s 2024-09-14T04:10:36.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:36 smithi039 ceph-mon[111235]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:10:36.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:36 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2351553588' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-29","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:10:36.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:36 smithi039 ceph-mon[111235]: osdmap e705: 8 total, 8 up, 8 in 2024-09-14T04:10:37.219 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:10:37.220 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:10:37.220 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:10:37.220 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:10:37.220 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:10:37.220 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:37.220 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:10:37.220 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:37.220 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:37.220 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:10:37.220 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:37.221 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:10:37.221 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:37.221 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:37.221 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:10:37.221 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:37.221 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:10:37.221 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:37.221 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:37.221 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:10:37.221 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:37.221 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:10:37.221 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:37.221 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:37.222 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:10:37.222 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:37.222 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:10:37.222 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:37.222 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:37.222 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:10:37.222 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:37.222 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:37.222 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:37.222 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:37.222 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:37.223 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:37.223 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:10:37.223 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:10:37.223 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:10:37.223 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:37.223 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:10:37.223 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:10:37.223 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:10:37.223 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:37.223 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:10:37.223 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:10:37.223 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:10:37.223 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:37.223 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:10:37.224 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:10:37.224 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:10:37.224 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:37.224 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:10:37.224 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:10:37.224 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:10:37.224 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 512~512 = 512 2024-09-14T04:10:37.224 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:37.224 INFO:tasks.workunit.client.0.smithi039.stdout:aio discard: 1536~512 = 0 2024-09-14T04:10:37.224 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:37.224 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:37.224 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:37.225 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:37.225 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:37.225 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:10:37.225 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:10:37.225 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestDataPoolIO (3433 ms) 2024-09-14T04:10:37.225 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteMismatch 2024-09-14T04:10:37.225 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:37.248 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestCompareAndWriteMismatch (28 ms) 2024-09-14T04:10:37.248 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteMismatch 2024-09-14T04:10:37.248 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:37.277 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteMismatch (30 ms) 2024-09-14T04:10:37.277 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteSuccess 2024-09-14T04:10:37.278 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:37.309 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestCompareAndWriteSuccess (31 ms) 2024-09-14T04:10:37.309 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteSuccess 2024-09-14T04:10:37.309 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:37.334 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteSuccess (26 ms) 2024-09-14T04:10:37.335 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitUnaligned 2024-09-14T04:10:37.335 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:37.409 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitUnaligned (74 ms) 2024-09-14T04:10:37.409 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnaligned 2024-09-14T04:10:37.409 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:37.479 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnaligned (71 ms) 2024-09-14T04:10:37.480 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteTooLarge 2024-09-14T04:10:37.480 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:37.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:37 smithi129 ceph-mon[101316]: osdmap e706: 8 total, 8 up, 8 in 2024-09-14T04:10:37.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:37 smithi039 ceph-mon[106470]: osdmap e706: 8 total, 8 up, 8 in 2024-09-14T04:10:37.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:37 smithi039 ceph-mon[111235]: osdmap e706: 8 total, 8 up, 8 in 2024-09-14T04:10:37.545 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestCompareAndWriteTooLarge (66 ms) 2024-09-14T04:10:37.546 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteTooLarge 2024-09-14T04:10:37.546 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:37.611 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteTooLarge (66 ms) 2024-09-14T04:10:37.612 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitSuccess 2024-09-14T04:10:37.612 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:37.942 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitSuccess (330 ms) 2024-09-14T04:10:37.942 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccess 2024-09-14T04:10:37.942 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:38.242 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccess (300 ms) 2024-09-14T04:10:38.242 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteVIovecLenDiffers 2024-09-14T04:10:38.242 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:38.279 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteVIovecLenDiffers (37 ms) 2024-09-14T04:10:38.279 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteVMismatch 2024-09-14T04:10:38.279 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:38.311 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteVMismatch (32 ms) 2024-09-14T04:10:38.311 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteVSuccess 2024-09-14T04:10:38.311 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:38.338 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteVSuccess (28 ms) 2024-09-14T04:10:38.339 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestScatterGatherIO 2024-09-14T04:10:38.339 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:38.364 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestScatterGatherIO (26 ms) 2024-09-14T04:10:38.364 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestEmptyDiscard 2024-09-14T04:10:38.364 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:38.385 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:38.385 INFO:tasks.workunit.client.0.smithi039.stdout:aio discard: 0~1048576 = 0 2024-09-14T04:10:38.386 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:38.386 INFO:tasks.workunit.client.0.smithi039.stdout:aio discard: 0~4194304 = 0 2024-09-14T04:10:38.386 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:38.386 INFO:tasks.workunit.client.0.smithi039.stdout:aio discard: 3145728~1048576 = 0 2024-09-14T04:10:38.386 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestEmptyDiscard (21 ms) 2024-09-14T04:10:38.386 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestFUA 2024-09-14T04:10:38.386 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:38.496 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:10:38.496 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:10:38.497 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:10:38.497 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:10:38.497 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:10:38.497 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:38.497 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:38.497 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:38.498 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:38.498 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:38.498 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:38.498 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:10:38.498 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:38.499 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:38.499 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:10:38.499 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:38.499 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:10:38.499 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:38.499 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:38.499 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:10:38.499 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:38.499 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:10:38.499 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:38.499 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:38.499 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:10:38.500 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:38.500 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:10:38.500 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:38.500 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:38.500 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:10:38.500 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:38.500 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:10:38.500 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:38.500 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:38.500 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:10:38.500 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:38.500 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:10:38.500 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:10:38.501 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:10:38.501 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:38.501 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:10:38.501 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:10:38.501 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:10:38.501 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:38.501 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:10:38.501 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:10:38.501 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:10:38.501 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:38.501 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:10:38.501 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:10:38.501 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:10:38.501 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:38.502 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:10:38.502 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:10:38.502 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:10:38.502 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestFUA (111 ms) 2024-09-14T04:10:38.502 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestIOPP 2024-09-14T04:10:38.502 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:38.521 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:10:38.521 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:10:38.521 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:10:38.522 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:10:38.522 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:10:38.522 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:38.522 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:10:38.522 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:38.522 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:38.522 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:10:38.522 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:38.522 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:10:38.522 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:38.522 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:38.522 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:10:38.522 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:38.522 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:38.523 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:10:38.523 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:38.523 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:10:38.523 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:38.523 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:38.523 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:10:38.523 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:38.523 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:10:38.523 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:38.523 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:38.524 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:38 smithi129 ceph-mon[101316]: pgmap v706: 553 pgs: 13 creating+peering, 12 unknown, 528 active+clean; 197 MiB data, 992 MiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 2.3 MiB/s wr, 1.08k op/s 2024-09-14T04:10:38.524 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:38 smithi129 ceph-mon[101316]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:10:38.524 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:38 smithi129 ceph-mon[101316]: osdmap e707: 8 total, 8 up, 8 in 2024-09-14T04:10:38.524 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:38 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:10:38.537 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:10:38.537 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:38.537 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:10:38.537 INFO:tasks.workunit.client.0.smithi039.stdout:start compare and write 2024-09-14T04:10:38.537 INFO:tasks.workunit.client.0.smithi039.stdout:compare and wrote: 512 2024-09-14T04:10:38.538 INFO:tasks.workunit.client.0.smithi039.stdout:start compare and write 2024-09-14T04:10:38.538 INFO:tasks.workunit.client.0.smithi039.stdout:compare and wrote: 512 2024-09-14T04:10:38.538 INFO:tasks.workunit.client.0.smithi039.stdout:start compare and write 2024-09-14T04:10:38.538 INFO:tasks.workunit.client.0.smithi039.stdout:compare and wrote: 512 2024-09-14T04:10:38.538 INFO:tasks.workunit.client.0.smithi039.stdout:start compare and write 2024-09-14T04:10:38.538 INFO:tasks.workunit.client.0.smithi039.stdout:compare and wrote: 512 2024-09-14T04:10:38.538 INFO:tasks.workunit.client.0.smithi039.stdout:start compare and write 2024-09-14T04:10:38.538 INFO:tasks.workunit.client.0.smithi039.stdout:compare and wrote: 512 2024-09-14T04:10:38.538 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:38.538 INFO:tasks.workunit.client.0.smithi039.stdout:started aio compare and write 2024-09-14T04:10:38.538 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:38.539 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:38.539 INFO:tasks.workunit.client.0.smithi039.stdout:finished aio compare and write 2024-09-14T04:10:38.539 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:38.539 INFO:tasks.workunit.client.0.smithi039.stdout:started aio compare and write 2024-09-14T04:10:38.539 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:38.539 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:38.539 INFO:tasks.workunit.client.0.smithi039.stdout:finished aio compare and write 2024-09-14T04:10:38.539 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:38.540 INFO:tasks.workunit.client.0.smithi039.stdout:started aio compare and write 2024-09-14T04:10:38.540 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:38.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:38 smithi039 ceph-mon[106470]: pgmap v706: 553 pgs: 13 creating+peering, 12 unknown, 528 active+clean; 197 MiB data, 992 MiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 2.3 MiB/s wr, 1.08k op/s 2024-09-14T04:10:38.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:38 smithi039 ceph-mon[106470]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:10:38.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:38 smithi039 ceph-mon[106470]: osdmap e707: 8 total, 8 up, 8 in 2024-09-14T04:10:38.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:38 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:10:38.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:38 smithi039 ceph-mon[111235]: pgmap v706: 553 pgs: 13 creating+peering, 12 unknown, 528 active+clean; 197 MiB data, 992 MiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 2.3 MiB/s wr, 1.08k op/s 2024-09-14T04:10:38.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:38 smithi039 ceph-mon[111235]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:10:38.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:38 smithi039 ceph-mon[111235]: osdmap e707: 8 total, 8 up, 8 in 2024-09-14T04:10:38.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:38 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:10:38.541 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:38.541 INFO:tasks.workunit.client.0.smithi039.stdout:finished aio compare and write 2024-09-14T04:10:38.541 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:38.541 INFO:tasks.workunit.client.0.smithi039.stdout:started aio compare and write 2024-09-14T04:10:38.541 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:38.542 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:38.542 INFO:tasks.workunit.client.0.smithi039.stdout:finished aio compare and write 2024-09-14T04:10:38.542 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:38.542 INFO:tasks.workunit.client.0.smithi039.stdout:started aio compare and write 2024-09-14T04:10:38.542 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:38.543 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:38.543 INFO:tasks.workunit.client.0.smithi039.stdout:finished aio compare and write 2024-09-14T04:10:38.543 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:38.543 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:38.543 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:38.543 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:38.543 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:38.543 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:38.543 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:10:38.543 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:10:38.544 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:10:38.544 INFO:tasks.workunit.client.0.smithi039.stdout:finished read 2024-09-14T04:10:38.544 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:38.545 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:10:38.545 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:10:38.545 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:10:38.545 INFO:tasks.workunit.client.0.smithi039.stdout:finished read 2024-09-14T04:10:38.545 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:38.545 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:10:38.545 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:10:38.545 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:10:38.545 INFO:tasks.workunit.client.0.smithi039.stdout:finished read 2024-09-14T04:10:38.545 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:38.545 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:10:38.545 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:10:38.545 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:10:38.546 INFO:tasks.workunit.client.0.smithi039.stdout:finished read 2024-09-14T04:10:38.546 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:38.546 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:10:38.546 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:10:38.546 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:10:38.546 INFO:tasks.workunit.client.0.smithi039.stdout:finished read 2024-09-14T04:10:38.546 INFO:tasks.workunit.client.0.smithi039.stdout:discard: 512~512 2024-09-14T04:10:38.546 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:39.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:10:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:10:38] "GET /metrics HTTP/1.1" 200 70376 "" "Prometheus/2.51.0" 2024-09-14T04:10:39.531 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:39.531 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:39.531 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:39.531 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:39.531 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:39.531 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 0 2024-09-14T04:10:39.531 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.531 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.531 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 0 2024-09-14T04:10:39.532 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.532 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.532 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 16384 2024-09-14T04:10:39.532 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.532 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.532 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 16384 2024-09-14T04:10:39.532 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.532 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.532 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:39.532 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:39.532 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 49152 2024-09-14T04:10:39.532 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.533 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.533 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 49152 2024-09-14T04:10:39.533 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.533 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.533 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 65536 2024-09-14T04:10:39.533 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.533 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.533 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 65536 2024-09-14T04:10:39.533 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.533 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.533 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:39.533 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:39.534 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 98304 2024-09-14T04:10:39.534 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.534 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.534 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 98304 2024-09-14T04:10:39.534 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.534 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.534 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 114688 2024-09-14T04:10:39.534 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.534 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.534 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 114688 2024-09-14T04:10:39.534 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.534 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.535 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:39.535 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:39.535 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 147456 2024-09-14T04:10:39.535 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.535 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.535 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 147456 2024-09-14T04:10:39.535 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.535 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.535 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 163840 2024-09-14T04:10:39.536 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.536 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.536 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 163840 2024-09-14T04:10:39.536 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.536 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.536 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:39.536 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:39.536 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 196608 2024-09-14T04:10:39.536 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.536 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.536 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 196608 2024-09-14T04:10:39.537 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.537 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.537 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 212992 2024-09-14T04:10:39.537 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.537 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.537 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 212992 2024-09-14T04:10:39.537 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.537 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.537 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:39.537 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:39.537 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:39.538 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:39.538 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:39.538 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:39.538 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:39.538 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.538 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.538 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:39.538 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:39.538 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:39.538 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:39.538 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:39.538 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.539 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.539 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:39.539 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:39.539 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:39.539 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:39.539 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:39.539 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.539 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.540 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:39.540 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:39.540 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:39.545 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:39.545 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:39.545 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.545 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.545 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:39.546 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:39.546 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:39.546 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:39.546 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:39.546 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:39.546 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:39.546 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:39.546 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:39.570 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:39.571 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:39.571 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.571 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.572 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:39.572 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:39.572 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:39.594 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:39.595 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:39.595 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.595 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.595 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:39.595 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:39.595 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:39.624 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:39.624 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:39.624 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.624 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.624 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:39.624 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:39.624 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:39.654 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:39.654 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:39.654 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.654 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.654 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:39.654 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:39.654 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:39.654 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:39.654 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:39.654 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:39.655 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:39.655 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:39.655 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:39.698 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:39.699 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:39.699 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.699 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.699 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:39.699 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:39.699 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:39.743 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:39.743 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:39.743 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.743 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.743 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:39.743 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:39.744 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:39.798 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:39.799 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:39.799 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.799 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.799 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:39.799 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:39.799 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:39.845 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:39.845 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:39.845 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.845 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.845 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:39.846 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:39.846 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:39.846 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:39.846 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:39.846 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:39.846 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:39.846 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:39.846 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:39.928 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:39.929 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:39.929 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:39.929 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:39.929 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:39.929 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:39.929 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:39.999 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:40.000 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:40.000 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:40.000 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:40.000 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:40.000 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:40.000 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:40.080 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:40.080 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:40.080 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:40.080 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:40.080 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:40.080 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:40.080 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:40.151 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:40.151 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:40.151 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:40.151 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:40.151 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:40.151 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:40.151 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:40.151 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:40.151 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:40.151 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:40.152 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:40.152 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:40.152 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:40.260 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:40.260 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:40.260 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:40.261 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:40.261 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:40.261 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:40.261 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:40.359 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:40.359 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:40.360 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:40.360 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:40.360 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:40.360 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:40.360 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:40.481 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:40.481 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:40.481 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:40.481 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:40.481 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:40.481 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:40.481 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:40.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:40 smithi129 ceph-mon[101316]: pgmap v708: 553 pgs: 13 creating+peering, 540 active+clean; 217 MiB data, 998 MiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 3.3 MiB/s wr, 503 op/s 2024-09-14T04:10:40.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:40 smithi039 ceph-mon[106470]: pgmap v708: 553 pgs: 13 creating+peering, 540 active+clean; 217 MiB data, 998 MiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 3.3 MiB/s wr, 503 op/s 2024-09-14T04:10:40.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:40 smithi039 ceph-mon[111235]: pgmap v708: 553 pgs: 13 creating+peering, 540 active+clean; 217 MiB data, 998 MiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 3.3 MiB/s wr, 503 op/s 2024-09-14T04:10:40.572 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:40.572 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:40.572 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:40.572 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:40.572 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:40.572 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:40.572 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:40.572 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:40.572 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:40.572 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:40.572 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestIOPP (2075 ms) 2024-09-14T04:10:40.573 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteCompareTooSmallPP 2024-09-14T04:10:40.573 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:40.595 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestCompareAndWriteCompareTooSmallPP (24 ms) 2024-09-14T04:10:40.596 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteCompareTooSmallPP 2024-09-14T04:10:40.596 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:40.621 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteCompareTooSmallPP (25 ms) 2024-09-14T04:10:40.621 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteWriteTooSmallPP 2024-09-14T04:10:40.621 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:40.646 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestCompareAndWriteWriteTooSmallPP (25 ms) 2024-09-14T04:10:40.646 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteWriteTooSmallPP 2024-09-14T04:10:40.646 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:40.671 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteWriteTooSmallPP (25 ms) 2024-09-14T04:10:40.671 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteMismatchPP 2024-09-14T04:10:40.671 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:40.694 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestCompareAndWriteMismatchPP (24 ms) 2024-09-14T04:10:40.694 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteMismatchPP 2024-09-14T04:10:40.694 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:40.717 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:40.720 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteMismatchPP (26 ms) 2024-09-14T04:10:40.721 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteMismatchBufferlistGreaterLenPP 2024-09-14T04:10:40.721 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:40.747 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestCompareAndWriteMismatchBufferlistGreaterLenPP (26 ms) 2024-09-14T04:10:40.747 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteMismatchBufferlistGreaterLenPP 2024-09-14T04:10:40.747 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:40.771 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:40.774 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteMismatchBufferlistGreaterLenPP (28 ms) 2024-09-14T04:10:40.775 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteSuccessPP 2024-09-14T04:10:40.775 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:40.799 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestCompareAndWriteSuccessPP (23 ms) 2024-09-14T04:10:40.799 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteSuccessPP 2024-09-14T04:10:40.799 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:40.818 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:40.822 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteSuccessPP (24 ms) 2024-09-14T04:10:40.822 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteSuccessBufferlistGreaterLenPP 2024-09-14T04:10:40.822 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:40.854 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestCompareAndWriteSuccessBufferlistGreaterLenPP (33 ms) 2024-09-14T04:10:40.855 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteSuccessBufferlistGreaterLenPP 2024-09-14T04:10:40.855 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:40.883 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:40.887 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteSuccessBufferlistGreaterLenPP (33 ms) 2024-09-14T04:10:40.887 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitUnalignedPP 2024-09-14T04:10:40.887 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:40.953 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitUnalignedPP (65 ms) 2024-09-14T04:10:40.953 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnalignedPP 2024-09-14T04:10:40.953 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:40.992 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:41.010 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitUnalignedPP (57 ms) 2024-09-14T04:10:41.010 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteTooLargePP 2024-09-14T04:10:41.010 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:41.071 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestCompareAndWriteTooLargePP (62 ms) 2024-09-14T04:10:41.072 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteTooLargePP 2024-09-14T04:10:41.072 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:41.114 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:41.136 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteTooLargePP (65 ms) 2024-09-14T04:10:41.136 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestCompareAndWriteStripeUnitSuccessPP 2024-09-14T04:10:41.136 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:41.405 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestCompareAndWriteStripeUnitSuccessPP (268 ms) 2024-09-14T04:10:41.405 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccessPP 2024-09-14T04:10:41.405 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:41.478 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:41 smithi039 ceph-mon[106470]: pgmap v709: 553 pgs: 553 active+clean; 233 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 6.7 MiB/s wr, 459 op/s 2024-09-14T04:10:41.478 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:41 smithi039 ceph-mon[111235]: pgmap v709: 553 pgs: 553 active+clean; 233 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 6.7 MiB/s wr, 459 op/s 2024-09-14T04:10:41.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:41 smithi129 ceph-mon[101316]: pgmap v709: 553 pgs: 553 active+clean; 233 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 6.7 MiB/s wr, 459 op/s 2024-09-14T04:10:41.686 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:41.702 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestAioCompareAndWriteStripeUnitSuccessPP (297 ms) 2024-09-14T04:10:41.702 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestIOPPWithIOHint 2024-09-14T04:10:41.702 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:41.731 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:10:41.731 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:10:41.731 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:10:41.731 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:10:41.731 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:10:41.731 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:41.731 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:10:41.731 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:41.732 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:41.732 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:10:41.732 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:41.732 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:10:41.732 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:41.732 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:41.733 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:10:41.733 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:41.733 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:10:41.733 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:41.733 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:41.733 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:10:41.734 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:41.734 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:10:41.734 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:41.736 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:41.737 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:10:41.737 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:41.737 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:10:41.737 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:41.737 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:41.738 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:10:41.738 INFO:tasks.workunit.client.0.smithi039.stdout:read: 512 2024-09-14T04:10:41.738 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:41.738 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:10:41.738 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:10:41.738 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:10:41.738 INFO:tasks.workunit.client.0.smithi039.stdout:finished read 2024-09-14T04:10:41.738 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:41.738 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:10:41.738 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:10:41.738 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:10:41.738 INFO:tasks.workunit.client.0.smithi039.stdout:finished read 2024-09-14T04:10:41.738 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:41.738 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:10:41.739 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:10:41.739 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:10:41.739 INFO:tasks.workunit.client.0.smithi039.stdout:finished read 2024-09-14T04:10:41.739 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:41.739 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:10:41.739 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:10:41.739 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:10:41.739 INFO:tasks.workunit.client.0.smithi039.stdout:finished read 2024-09-14T04:10:41.739 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:41.740 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:10:41.740 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:10:42.703 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:10:42.704 INFO:tasks.workunit.client.0.smithi039.stdout:finished read 2024-09-14T04:10:42.704 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 0 2024-09-14T04:10:42.704 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:42.704 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:42.704 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 0 2024-09-14T04:10:42.704 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:42.704 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:42.704 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 16384 2024-09-14T04:10:42.704 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:42.704 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:42.704 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 16384 2024-09-14T04:10:42.705 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:42.705 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:42.705 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:42.705 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:42.705 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 49152 2024-09-14T04:10:42.705 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:42.705 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:42.705 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 49152 2024-09-14T04:10:42.705 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:42.705 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:42.705 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 65536 2024-09-14T04:10:42.705 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:42.705 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:42.705 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 65536 2024-09-14T04:10:42.706 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:42.706 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:42.706 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:42.706 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:42.706 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 98304 2024-09-14T04:10:42.706 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:42.706 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:42.706 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 98304 2024-09-14T04:10:42.706 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:42.706 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:42.706 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 114688 2024-09-14T04:10:42.706 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:42.706 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:42.706 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 114688 2024-09-14T04:10:42.707 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:42.707 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:42.707 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:42.707 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:42.707 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 147456 2024-09-14T04:10:42.707 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:42.707 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:42.707 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 147456 2024-09-14T04:10:42.707 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:42.707 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:42.707 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 163840 2024-09-14T04:10:42.707 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:42.707 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:42.707 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 163840 2024-09-14T04:10:42.708 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:42.708 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:42.708 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:42.708 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:42.708 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 196608 2024-09-14T04:10:42.708 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:42.708 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:42.708 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 196608 2024-09-14T04:10:42.708 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:42.708 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:42.708 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 212992 2024-09-14T04:10:42.708 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:42.708 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:42.708 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 212992 2024-09-14T04:10:42.709 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:42.709 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:42.709 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:42.709 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:42.709 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:42.709 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:42.709 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:42.709 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:42.709 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:42.709 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:42.709 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:42.709 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:42.709 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:42.709 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:42.710 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:42.710 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:42.710 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:42.710 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:42.710 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:42.710 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:42.710 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:42.712 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:42.713 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:42.713 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:42.713 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:42.713 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:42.713 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:42.713 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:42.722 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:42.722 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:42.722 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:42.722 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:42.723 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:42.723 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:42.723 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:42.723 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:42.723 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:42.723 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:42.723 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:42.723 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:42.723 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:42.744 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:42.744 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:42.744 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:42.744 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:42.744 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:42.744 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:42.744 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:42.765 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:42.765 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:42.765 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:42.765 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:42.765 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:42.765 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:42.766 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:42.795 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:42.795 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:42.795 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:42.795 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:42.795 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:42.795 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:42.796 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:42.823 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:42.823 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:42.823 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:42.823 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:42.823 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:42.824 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:42.824 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:42.824 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:42.824 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:42.824 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:42.824 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:42.824 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:42.824 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:42.864 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:42.865 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:42.865 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:42.865 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:42.865 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:42.865 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:42.865 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:42.908 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:42.908 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:42.908 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:42.908 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:42.908 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:42.908 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:42.908 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:42.960 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:42.960 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:42.960 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:42.960 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:42.960 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:42.960 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:42.960 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:43.010 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:43.010 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:43.011 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:43.011 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:43.011 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:43.011 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:43.011 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:43.011 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:43.011 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:43.011 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:43.011 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:43.011 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:43.011 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:43.092 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:43.093 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:43.093 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:43.093 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:43.093 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:43.093 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:43.093 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:43.159 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:43.160 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:43.160 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:43.160 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:43.160 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:43.160 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:43.160 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:43.227 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:43.227 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:43.228 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:43.228 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:43.228 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:43.228 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:43.228 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:43.298 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:43.298 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:43.299 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:43.299 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:43.299 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:43.299 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:43.299 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:43.299 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:43.299 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:43.299 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:43.299 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:43.299 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:43.299 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:43.416 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:43.416 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:43.417 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:43.417 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:43.417 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:43.417 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:43.417 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:43.500 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:43.500 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:43.500 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:43.501 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:43.501 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:43.501 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:43.501 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:43.605 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:43.605 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:43.605 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:43.605 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:43.605 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:43.605 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:43.606 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:10:43.704 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:10:43.704 INFO:tasks.workunit.client.0.smithi039.stdout:finished writesame 2024-09-14T04:10:43.704 INFO:tasks.workunit.client.0.smithi039.stdout:to verify the data 2024-09-14T04:10:43.705 INFO:tasks.workunit.client.0.smithi039.stdout:verified 2024-09-14T04:10:43.705 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:43.705 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:43.705 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:43.705 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:10:43.705 INFO:tasks.workunit.client.0.smithi039.stdout:started writesame 2024-09-14T04:10:43.705 INFO:tasks.workunit.client.0.smithi039.stdout:expected fail, finished writesame 2024-09-14T04:10:43.705 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestIOPPWithIOHint (2002 ms) 2024-09-14T04:10:43.705 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestIOToSnapshot 2024-09-14T04:10:43.705 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:44.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:44 smithi129 ceph-mon[101316]: pgmap v710: 553 pgs: 553 active+clean; 269 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 10 MiB/s wr, 1.61k op/s 2024-09-14T04:10:44.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:44 smithi039 ceph-mon[106470]: pgmap v710: 553 pgs: 553 active+clean; 269 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 10 MiB/s wr, 1.61k op/s 2024-09-14T04:10:44.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:44 smithi039 ceph-mon[111235]: pgmap v710: 553 pgs: 553 active+clean; 269 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 10 MiB/s wr, 1.61k op/s 2024-09-14T04:10:45.131 INFO:tasks.workunit.client.0.smithi039.stdout:num snaps is: 0 2024-09-14T04:10:45.131 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 0 2024-09-14T04:10:45.132 INFO:tasks.workunit.client.0.smithi039.stdout:num snaps is: 1 2024-09-14T04:10:45.132 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 1 2024-09-14T04:10:45.132 INFO:tasks.workunit.client.0.smithi039.stdout:snap: orig 2024-09-14T04:10:45.132 INFO:tasks.workunit.client.0.smithi039.stdout:found orig with size 2097152 2024-09-14T04:10:45.132 INFO:tasks.workunit.client.0.smithi039.stdout:read: 80 2024-09-14T04:10:45.132 INFO:tasks.workunit.client.0.smithi039.stdout:write test data! 2024-09-14T04:10:45.132 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 80 2024-09-14T04:10:45.132 INFO:tasks.workunit.client.0.smithi039.stdout:num snaps is: 2 2024-09-14T04:10:45.132 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 2 2024-09-14T04:10:45.132 INFO:tasks.workunit.client.0.smithi039.stdout:snap: orig 2024-09-14T04:10:45.133 INFO:tasks.workunit.client.0.smithi039.stdout:snap: written 2024-09-14T04:10:45.133 INFO:tasks.workunit.client.0.smithi039.stdout:found orig with size 2097152 2024-09-14T04:10:45.133 INFO:tasks.workunit.client.0.smithi039.stdout:found written with size 2097152 2024-09-14T04:10:45.133 INFO:tasks.workunit.client.0.smithi039.stdout:read: 80 2024-09-14T04:10:45.133 INFO:tasks.workunit.client.0.smithi039.stdout:read: 80 2024-09-14T04:10:45.133 INFO:tasks.workunit.client.0.smithi039.stdout:read: 80 2024-09-14T04:10:45.133 INFO:tasks.workunit.client.0.smithi039.stdout:write to snapshot returned -30 2024-09-14T04:10:45.133 INFO:tasks.workunit.client.0.smithi039.stdout:Read-only file system 2024-09-14T04:10:45.156 INFO:tasks.workunit.client.0.smithi039.stdout:read: 80 2024-09-14T04:10:45.157 INFO:tasks.workunit.client.0.smithi039.stdout:read: 80 2024-09-14T04:10:45.157 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 80 2024-09-14T04:10:45.157 INFO:tasks.workunit.client.0.smithi039.stdout:opening testimg@orig 2024-09-14T04:10:45.157 INFO:tasks.workunit.client.0.smithi039.stdout:read: 80 2024-09-14T04:10:45.157 INFO:tasks.workunit.client.0.smithi039.stdout:write to snapshot returned -30 2024-09-14T04:10:45.157 INFO:tasks.workunit.client.0.smithi039.stdout:Read-only file system 2024-09-14T04:10:45.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:45 smithi129 ceph-mon[101316]: osdmap e708: 8 total, 8 up, 8 in 2024-09-14T04:10:45.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:45 smithi039 ceph-mon[106470]: osdmap e708: 8 total, 8 up, 8 in 2024-09-14T04:10:45.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:45 smithi039 ceph-mon[111235]: osdmap e708: 8 total, 8 up, 8 in 2024-09-14T04:10:46.422 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:46 smithi129 ceph-mon[101316]: pgmap v712: 553 pgs: 553 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 15 MiB/s wr, 2.68k op/s 2024-09-14T04:10:46.422 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:46 smithi129 ceph-mon[101316]: osdmap e709: 8 total, 8 up, 8 in 2024-09-14T04:10:46.422 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:46 smithi129 ceph-mon[101316]: osdmap e710: 8 total, 8 up, 8 in 2024-09-14T04:10:46.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:46 smithi039 ceph-mon[106470]: pgmap v712: 553 pgs: 553 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 15 MiB/s wr, 2.68k op/s 2024-09-14T04:10:46.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:46 smithi039 ceph-mon[106470]: osdmap e709: 8 total, 8 up, 8 in 2024-09-14T04:10:46.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:46 smithi039 ceph-mon[106470]: osdmap e710: 8 total, 8 up, 8 in 2024-09-14T04:10:46.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:46 smithi039 ceph-mon[111235]: pgmap v712: 553 pgs: 553 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 15 MiB/s wr, 2.68k op/s 2024-09-14T04:10:46.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:46 smithi039 ceph-mon[111235]: osdmap e709: 8 total, 8 up, 8 in 2024-09-14T04:10:46.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:46 smithi039 ceph-mon[111235]: osdmap e710: 8 total, 8 up, 8 in 2024-09-14T04:10:46.550 INFO:tasks.workunit.client.0.smithi039.stdout:num snaps is: 2 2024-09-14T04:10:46.550 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 2 2024-09-14T04:10:46.550 INFO:tasks.workunit.client.0.smithi039.stdout:snap: orig 2024-09-14T04:10:46.550 INFO:tasks.workunit.client.0.smithi039.stdout:snap: written 2024-09-14T04:10:46.550 INFO:tasks.workunit.client.0.smithi039.stdout:found orig with size 2097152 2024-09-14T04:10:46.550 INFO:tasks.workunit.client.0.smithi039.stdout:found written with size 2097152 2024-09-14T04:10:46.550 INFO:tasks.workunit.client.0.smithi039.stdout:num snaps is: 1 2024-09-14T04:10:46.550 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 1 2024-09-14T04:10:46.550 INFO:tasks.workunit.client.0.smithi039.stdout:snap: orig 2024-09-14T04:10:46.551 INFO:tasks.workunit.client.0.smithi039.stdout:found orig with size 2097152 2024-09-14T04:10:46.551 INFO:tasks.workunit.client.0.smithi039.stdout:num snaps is: 0 2024-09-14T04:10:46.551 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 0 2024-09-14T04:10:46.551 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestIOToSnapshot (2846 ms) 2024-09-14T04:10:46.551 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestSnapshotDeletedIo 2024-09-14T04:10:46.551 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:47.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:47 smithi039 ceph-mon[106470]: osdmap e711: 8 total, 8 up, 8 in 2024-09-14T04:10:47.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:47 smithi039 ceph-mon[106470]: pgmap v716: 553 pgs: 553 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 8.5 MiB/s wr, 2.19k op/s 2024-09-14T04:10:47.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:47 smithi039 ceph-mon[111235]: osdmap e711: 8 total, 8 up, 8 in 2024-09-14T04:10:47.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:47 smithi039 ceph-mon[111235]: pgmap v716: 553 pgs: 553 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 8.5 MiB/s wr, 2.19k op/s 2024-09-14T04:10:48.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:47 smithi129 ceph-mon[101316]: osdmap e711: 8 total, 8 up, 8 in 2024-09-14T04:10:48.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:47 smithi129 ceph-mon[101316]: pgmap v716: 553 pgs: 553 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 8.5 MiB/s wr, 2.19k op/s 2024-09-14T04:10:48.566 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestSnapshotDeletedIo (2016 ms) 2024-09-14T04:10:48.567 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestClone 2024-09-14T04:10:48.567 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:48.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:48 smithi039 ceph-mon[106470]: osdmap e712: 8 total, 8 up, 8 in 2024-09-14T04:10:48.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:10:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:10:48] "GET /metrics HTTP/1.1" 200 70376 "" "Prometheus/2.51.0" 2024-09-14T04:10:48.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:48 smithi039 ceph-mon[111235]: osdmap e712: 8 total, 8 up, 8 in 2024-09-14T04:10:49.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:48 smithi129 ceph-mon[101316]: osdmap e712: 8 total, 8 up, 8 in 2024-09-14T04:10:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:49 smithi129 ceph-mon[101316]: osdmap e713: 8 total, 8 up, 8 in 2024-09-14T04:10:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:49 smithi129 ceph-mon[101316]: pgmap v719: 553 pgs: 553 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 64 KiB/s rd, 7.4 KiB/s wr, 93 op/s 2024-09-14T04:10:50.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:49 smithi039 ceph-mon[106470]: osdmap e713: 8 total, 8 up, 8 in 2024-09-14T04:10:50.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:49 smithi039 ceph-mon[106470]: pgmap v719: 553 pgs: 553 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 64 KiB/s rd, 7.4 KiB/s wr, 93 op/s 2024-09-14T04:10:50.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:49 smithi039 ceph-mon[111235]: osdmap e713: 8 total, 8 up, 8 in 2024-09-14T04:10:50.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:49 smithi039 ceph-mon[111235]: pgmap v719: 553 pgs: 553 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 64 KiB/s rd, 7.4 KiB/s wr, 93 op/s 2024-09-14T04:10:50.598 INFO:tasks.workunit.client.0.smithi039.stdout:made parent image "parent" 2024-09-14T04:10:50.598 INFO:tasks.workunit.client.0.smithi039.stdout:parent has no parent info 2024-09-14T04:10:50.598 INFO:tasks.workunit.client.0.smithi039.stdout:made snapshot "parent@parent_snap" 2024-09-14T04:10:50.598 INFO:tasks.workunit.client.0.smithi039.stdout:can't unprotect an unprotected snap 2024-09-14T04:10:50.598 INFO:tasks.workunit.client.0.smithi039.stdout:can't protect a protected snap 2024-09-14T04:10:50.598 INFO:tasks.workunit.client.0.smithi039.stdout:made and opened clone "child" 2024-09-14T04:10:50.598 INFO:tasks.workunit.client.0.smithi039.stdout:read: 8 2024-09-14T04:10:50.598 INFO:tasks.workunit.client.0.smithi039.stdout:read: 8 2024-09-14T04:10:50.598 INFO:tasks.workunit.client.0.smithi039.stdout:read: 8 2024-09-14T04:10:50.598 INFO:tasks.workunit.client.0.smithi039.stdout:sizes and overlaps are good between parent and child 2024-09-14T04:10:50.598 INFO:tasks.workunit.client.0.smithi039.stdout:child image successfully cloned all image-meta pairs 2024-09-14T04:10:50.598 INFO:tasks.workunit.client.0.smithi039.stdout:sized down clone, changed overlap 2024-09-14T04:10:50.599 INFO:tasks.workunit.client.0.smithi039.stdout:parent info: size 4194304 obj_size 4194304 parent_pool 18446744073709551615 2024-09-14T04:10:50.599 INFO:tasks.workunit.client.0.smithi039.stdout:sized up clone, changed size but not overlap or parent's size 2024-09-14T04:10:50.599 INFO:tasks.workunit.client.0.smithi039.stdout:can't remove parent while child still exists 2024-09-14T04:10:50.599 INFO:tasks.workunit.client.0.smithi039.stdout:can't remove parent while still protected 2024-09-14T04:10:50.599 INFO:tasks.workunit.client.0.smithi039.stdout:removed parent snap after unprotecting 2024-09-14T04:10:50.599 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestClone (2031 ms) 2024-09-14T04:10:50.599 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestClone2 2024-09-14T04:10:50.599 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:51.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:50 smithi129 ceph-mon[101316]: osdmap e714: 8 total, 8 up, 8 in 2024-09-14T04:10:51.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:50 smithi039 ceph-mon[106470]: osdmap e714: 8 total, 8 up, 8 in 2024-09-14T04:10:51.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:50 smithi039 ceph-mon[111235]: osdmap e714: 8 total, 8 up, 8 in 2024-09-14T04:10:51.696 INFO:tasks.workunit.client.0.smithi039.stdout:made parent image "parent" 2024-09-14T04:10:51.696 INFO:tasks.workunit.client.0.smithi039.stdout:parent has no parent info 2024-09-14T04:10:51.696 INFO:tasks.workunit.client.0.smithi039.stdout:made snapshot "parent@parent_snap" 2024-09-14T04:10:51.697 INFO:tasks.workunit.client.0.smithi039.stdout:made and opened clone "child" 2024-09-14T04:10:51.697 INFO:tasks.workunit.client.0.smithi039.stdout:child image successfully cloned all image-meta pairs 2024-09-14T04:10:51.697 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestClone2 (1099 ms) 2024-09-14T04:10:51.697 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.ListChildren 2024-09-14T04:10:52.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:51 smithi129 ceph-mon[101316]: osdmap e715: 8 total, 8 up, 8 in 2024-09-14T04:10:52.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:51 smithi129 ceph-mon[101316]: pgmap v722: 553 pgs: 553 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 79 KiB/s rd, 9.5 KiB/s wr, 115 op/s 2024-09-14T04:10:52.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:51 smithi039 ceph-mon[111235]: osdmap e715: 8 total, 8 up, 8 in 2024-09-14T04:10:52.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:51 smithi039 ceph-mon[111235]: pgmap v722: 553 pgs: 553 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 79 KiB/s rd, 9.5 KiB/s wr, 115 op/s 2024-09-14T04:10:52.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:51 smithi039 ceph-mon[106470]: osdmap e715: 8 total, 8 up, 8 in 2024-09-14T04:10:52.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:51 smithi039 ceph-mon[106470]: pgmap v722: 553 pgs: 553 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 79 KiB/s rd, 9.5 KiB/s wr, 115 op/s 2024-09-14T04:10:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:52 smithi129 ceph-mon[101316]: osdmap e716: 8 total, 8 up, 8 in 2024-09-14T04:10:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:10:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:52 smithi039 ceph-mon[106470]: osdmap e716: 8 total, 8 up, 8 in 2024-09-14T04:10:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:10:53.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:52 smithi039 ceph-mon[111235]: osdmap e716: 8 total, 8 up, 8 in 2024-09-14T04:10:53.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:10:54.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:53 smithi129 ceph-mon[101316]: osdmap e717: 8 total, 8 up, 8 in 2024-09-14T04:10:54.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:53 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2427756749' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-30","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:10:54.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:53 smithi129 ceph-mon[101316]: pgmap v725: 585 pgs: 14 creating+peering, 18 unknown, 553 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 461 KiB/s rd, 34 KiB/s wr, 557 op/s 2024-09-14T04:10:54.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:53 smithi039 ceph-mon[106470]: osdmap e717: 8 total, 8 up, 8 in 2024-09-14T04:10:54.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:53 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2427756749' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-30","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:10:54.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:53 smithi039 ceph-mon[106470]: pgmap v725: 585 pgs: 14 creating+peering, 18 unknown, 553 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 461 KiB/s rd, 34 KiB/s wr, 557 op/s 2024-09-14T04:10:54.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:53 smithi039 ceph-mon[111235]: osdmap e717: 8 total, 8 up, 8 in 2024-09-14T04:10:54.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:53 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2427756749' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-30","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:10:54.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:53 smithi039 ceph-mon[111235]: pgmap v725: 585 pgs: 14 creating+peering, 18 unknown, 553 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 461 KiB/s rd, 34 KiB/s wr, 557 op/s 2024-09-14T04:10:55.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:54 smithi129 ceph-mon[101316]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:10:55.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:54 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2427756749' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:10:55.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:54 smithi129 ceph-mon[101316]: osdmap e718: 8 total, 8 up, 8 in 2024-09-14T04:10:55.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:54 smithi039 ceph-mon[106470]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:10:55.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:54 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2427756749' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:10:55.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:54 smithi039 ceph-mon[106470]: osdmap e718: 8 total, 8 up, 8 in 2024-09-14T04:10:55.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:54 smithi039 ceph-mon[111235]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:10:55.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:54 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2427756749' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-30","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:10:55.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:54 smithi039 ceph-mon[111235]: osdmap e718: 8 total, 8 up, 8 in 2024-09-14T04:10:55.531 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:10:56.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:55 smithi129 ceph-mon[101316]: osdmap e719: 8 total, 8 up, 8 in 2024-09-14T04:10:56.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:55 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2787737148' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-31","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:10:56.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:55 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-31","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:10:56.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:55 smithi129 ceph-mon[101316]: pgmap v728: 617 pgs: 17 creating+peering, 41 unknown, 559 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 770 KiB/s rd, 54 KiB/s wr, 916 op/s 2024-09-14T04:10:56.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:55 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:10:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:55 smithi129 ceph-mon[101316]: osdmap e720: 8 total, 8 up, 8 in 2024-09-14T04:10:56.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:55 smithi039 ceph-mon[106470]: osdmap e719: 8 total, 8 up, 8 in 2024-09-14T04:10:56.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:55 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2787737148' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-31","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:10:56.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:55 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-31","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:10:56.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:55 smithi039 ceph-mon[106470]: pgmap v728: 617 pgs: 17 creating+peering, 41 unknown, 559 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 770 KiB/s rd, 54 KiB/s wr, 916 op/s 2024-09-14T04:10:56.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:55 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:10:56.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:55 smithi039 ceph-mon[106470]: osdmap e720: 8 total, 8 up, 8 in 2024-09-14T04:10:56.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:55 smithi039 ceph-mon[111235]: osdmap e719: 8 total, 8 up, 8 in 2024-09-14T04:10:56.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:55 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2787737148' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-31","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:10:56.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:55 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-31","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:10:56.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:55 smithi039 ceph-mon[111235]: pgmap v728: 617 pgs: 17 creating+peering, 41 unknown, 559 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 770 KiB/s rd, 54 KiB/s wr, 916 op/s 2024-09-14T04:10:56.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:55 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-31","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:10:56.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:55 smithi039 ceph-mon[111235]: osdmap e720: 8 total, 8 up, 8 in 2024-09-14T04:10:57.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:57 smithi039 ceph-mon[106470]: osdmap e721: 8 total, 8 up, 8 in 2024-09-14T04:10:57.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:57 smithi039 ceph-mon[106470]: pgmap v731: 617 pgs: 17 creating+peering, 31 unknown, 569 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 330 KiB/s rd, 22 KiB/s wr, 390 op/s 2024-09-14T04:10:57.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:57 smithi039 ceph-mon[111235]: osdmap e721: 8 total, 8 up, 8 in 2024-09-14T04:10:57.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:57 smithi039 ceph-mon[111235]: pgmap v731: 617 pgs: 17 creating+peering, 31 unknown, 569 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 330 KiB/s rd, 22 KiB/s wr, 390 op/s 2024-09-14T04:10:58.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:57 smithi129 ceph-mon[101316]: osdmap e721: 8 total, 8 up, 8 in 2024-09-14T04:10:58.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:57 smithi129 ceph-mon[101316]: pgmap v731: 617 pgs: 17 creating+peering, 31 unknown, 569 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 330 KiB/s rd, 22 KiB/s wr, 390 op/s 2024-09-14T04:10:58.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:58 smithi039 ceph-mon[106470]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:10:58.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:58 smithi039 ceph-mon[106470]: osdmap e722: 8 total, 8 up, 8 in 2024-09-14T04:10:58.791 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:10:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:10:58] "GET /metrics HTTP/1.1" 200 70390 "" "Prometheus/2.51.0" 2024-09-14T04:10:58.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:58 smithi039 ceph-mon[111235]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:10:58.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:58 smithi039 ceph-mon[111235]: osdmap e722: 8 total, 8 up, 8 in 2024-09-14T04:10:59.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:58 smithi129 ceph-mon[101316]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:10:59.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:58 smithi129 ceph-mon[101316]: osdmap e722: 8 total, 8 up, 8 in 2024-09-14T04:11:00.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:59 smithi129 ceph-mon[101316]: osdmap e723: 8 total, 8 up, 8 in 2024-09-14T04:11:00.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:10:59 smithi129 ceph-mon[101316]: pgmap v734: 617 pgs: 3 creating+peering, 2 unknown, 612 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s rd, 1023 B/s wr, 5 op/s 2024-09-14T04:11:00.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:59 smithi039 ceph-mon[106470]: osdmap e723: 8 total, 8 up, 8 in 2024-09-14T04:11:00.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:10:59 smithi039 ceph-mon[106470]: pgmap v734: 617 pgs: 3 creating+peering, 2 unknown, 612 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s rd, 1023 B/s wr, 5 op/s 2024-09-14T04:11:00.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:59 smithi039 ceph-mon[111235]: osdmap e723: 8 total, 8 up, 8 in 2024-09-14T04:11:00.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:10:59 smithi039 ceph-mon[111235]: pgmap v734: 617 pgs: 3 creating+peering, 2 unknown, 612 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.2 KiB/s rd, 1023 B/s wr, 5 op/s 2024-09-14T04:11:01.005 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.005 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-31/image125 2024-09-14T04:11:01.005 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-31/image125 2024-09-14T04:11:01.005 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-31/image125 2024-09-14T04:11:01.005 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.005 INFO:tasks.workunit.client.0.smithi039.stdout:num children is: 1 2024-09-14T04:11:01.005 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 1 2024-09-14T04:11:01.006 INFO:tasks.workunit.client.0.smithi039.stdout:child: image125 2024-09-14T04:11:01.006 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-31/image125/ace2f5230d4e 2024-09-14T04:11:01.006 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.006 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.006 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-31/image125 2024-09-14T04:11:01.006 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.006 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-31/image125 2024-09-14T04:11:01.006 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-31/image125 2024-09-14T04:11:01.006 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.006 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.006 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.006 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.007 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.007 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.007 INFO:tasks.workunit.client.0.smithi039.stdout:num children is: 2 2024-09-14T04:11:01.007 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 2 2024-09-14T04:11:01.007 INFO:tasks.workunit.client.0.smithi039.stdout:child: image126 2024-09-14T04:11:01.007 INFO:tasks.workunit.client.0.smithi039.stdout:child: image125 2024-09-14T04:11:01.007 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-31/image125/ace2f5230d4e 2024-09-14T04:11:01.007 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.007 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-30/image126/ace22fa25831 2024-09-14T04:11:01.007 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.007 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.007 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-31/image125 2024-09-14T04:11:01.007 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.008 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-31/image125 2024-09-14T04:11:01.008 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-31/image125 2024-09-14T04:11:01.008 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.008 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.008 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.008 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.008 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.008 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.008 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.008 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-31/image127 2024-09-14T04:11:01.008 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.009 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-31/image125 2024-09-14T04:11:01.009 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-31/image127 2024-09-14T04:11:01.009 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-31/image127 2024-09-14T04:11:01.009 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.009 INFO:tasks.workunit.client.0.smithi039.stdout:num children is: 3 2024-09-14T04:11:01.009 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 3 2024-09-14T04:11:01.009 INFO:tasks.workunit.client.0.smithi039.stdout:child: image126 2024-09-14T04:11:01.009 INFO:tasks.workunit.client.0.smithi039.stdout:child: image125 2024-09-14T04:11:01.009 INFO:tasks.workunit.client.0.smithi039.stdout:child: image127 2024-09-14T04:11:01.009 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-31/image125/ace2f5230d4e 2024-09-14T04:11:01.010 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.010 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-30/image126/ace22fa25831 2024-09-14T04:11:01.010 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.010 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-31/image127/ace23a4c94c9 2024-09-14T04:11:01.010 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.010 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.010 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-31/image125 2024-09-14T04:11:01.010 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.010 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-31/image125 2024-09-14T04:11:01.010 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-31/image125 2024-09-14T04:11:01.010 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.010 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.011 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.011 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.011 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.011 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.011 INFO:tasks.workunit.client.0.smithi039.stdout:num children is: 3 2024-09-14T04:11:01.011 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 3 2024-09-14T04:11:01.011 INFO:tasks.workunit.client.0.smithi039.stdout:child: image126 2024-09-14T04:11:01.011 INFO:tasks.workunit.client.0.smithi039.stdout:child: image125 2024-09-14T04:11:01.012 INFO:tasks.workunit.client.0.smithi039.stdout:child: image127 2024-09-14T04:11:01.012 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-31/image125/ace2f5230d4e 2024-09-14T04:11:01.012 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.012 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-30/image126/ace22fa25831 2024-09-14T04:11:01.012 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.012 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-31/image127/ace23a4c94c9 2024-09-14T04:11:01.012 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.012 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.012 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-31/image125 2024-09-14T04:11:01.012 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.012 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-31/image125 2024-09-14T04:11:01.012 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-31/image125 2024-09-14T04:11:01.012 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.013 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.013 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.013 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.013 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.013 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.013 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.013 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-31/image128 2024-09-14T04:11:01.013 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.013 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-31/image125 2024-09-14T04:11:01.013 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-31/image128 2024-09-14T04:11:01.014 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-31/image128 2024-09-14T04:11:01.014 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.014 INFO:tasks.workunit.client.0.smithi039.stdout:num children is: 4 2024-09-14T04:11:01.014 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 4 2024-09-14T04:11:01.014 INFO:tasks.workunit.client.0.smithi039.stdout:child: image126 2024-09-14T04:11:01.014 INFO:tasks.workunit.client.0.smithi039.stdout:child: image125 2024-09-14T04:11:01.014 INFO:tasks.workunit.client.0.smithi039.stdout:child: image127 2024-09-14T04:11:01.014 INFO:tasks.workunit.client.0.smithi039.stdout:child: image128 2024-09-14T04:11:01.014 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-31/image125/ace2f5230d4e 2024-09-14T04:11:01.014 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.014 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-30/image126/ace22fa25831 2024-09-14T04:11:01.014 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.014 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-31/image127/ace23a4c94c9 2024-09-14T04:11:01.014 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.014 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-31/image128/ace232bb685a 2024-09-14T04:11:01.015 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.015 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.015 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-31/image125 2024-09-14T04:11:01.015 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.015 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-31/image125 2024-09-14T04:11:01.015 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-31/image125 2024-09-14T04:11:01.015 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.015 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.015 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.016 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.016 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.016 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.016 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.016 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-31/image127 2024-09-14T04:11:01.016 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.016 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-31/image125 2024-09-14T04:11:01.016 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-31/image127 2024-09-14T04:11:01.016 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-31/image127 2024-09-14T04:11:01.016 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.016 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.016 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-31/image128 2024-09-14T04:11:01.017 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:00 smithi129 ceph-mon[101316]: osdmap e724: 8 total, 8 up, 8 in 2024-09-14T04:11:01.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:00 smithi129 ceph-mon[101316]: osdmap e725: 8 total, 8 up, 8 in 2024-09-14T04:11:01.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:00 smithi039 ceph-mon[106470]: osdmap e724: 8 total, 8 up, 8 in 2024-09-14T04:11:01.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:00 smithi039 ceph-mon[106470]: osdmap e725: 8 total, 8 up, 8 in 2024-09-14T04:11:01.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:00 smithi039 ceph-mon[111235]: osdmap e724: 8 total, 8 up, 8 in 2024-09-14T04:11:01.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:00 smithi039 ceph-mon[111235]: osdmap e725: 8 total, 8 up, 8 in 2024-09-14T04:11:01.661 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-31/image125 2024-09-14T04:11:01.661 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-31/image127 2024-09-14T04:11:01.661 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-31/image128 2024-09-14T04:11:01.661 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-31/image128 2024-09-14T04:11:01.661 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.661 INFO:tasks.workunit.client.0.smithi039.stdout:num children is: 4 2024-09-14T04:11:01.662 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 4 2024-09-14T04:11:01.662 INFO:tasks.workunit.client.0.smithi039.stdout:child: image126 2024-09-14T04:11:01.662 INFO:tasks.workunit.client.0.smithi039.stdout:child: image125 2024-09-14T04:11:01.662 INFO:tasks.workunit.client.0.smithi039.stdout:child: image127 2024-09-14T04:11:01.662 INFO:tasks.workunit.client.0.smithi039.stdout:child: image128 2024-09-14T04:11:01.662 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-31/image125/ace2f5230d4e 2024-09-14T04:11:01.662 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.662 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-30/image126/ace22fa25831 2024-09-14T04:11:01.662 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.662 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-31/image127/ace23a4c94c9 2024-09-14T04:11:01.662 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.662 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-31/image128/ace232bb685a 2024-09-14T04:11:01.662 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.663 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.663 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.663 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.663 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.663 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.663 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.663 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-31/image127 2024-09-14T04:11:01.663 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.663 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-31/image127 2024-09-14T04:11:01.663 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-31/image127 2024-09-14T04:11:01.663 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.663 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.663 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-31/image128 2024-09-14T04:11:01.663 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.664 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-31/image127 2024-09-14T04:11:01.664 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-31/image128 2024-09-14T04:11:01.664 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-31/image128 2024-09-14T04:11:01.664 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.664 INFO:tasks.workunit.client.0.smithi039.stdout:num children is: 3 2024-09-14T04:11:01.664 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 3 2024-09-14T04:11:01.664 INFO:tasks.workunit.client.0.smithi039.stdout:child: image126 2024-09-14T04:11:01.664 INFO:tasks.workunit.client.0.smithi039.stdout:child: image127 2024-09-14T04:11:01.664 INFO:tasks.workunit.client.0.smithi039.stdout:child: image128 2024-09-14T04:11:01.664 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-30/image126/ace22fa25831 2024-09-14T04:11:01.664 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.664 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-31/image127/ace23a4c94c9 2024-09-14T04:11:01.664 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.664 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-31/image128/ace232bb685a 2024-09-14T04:11:01.665 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.665 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.665 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.665 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.665 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.665 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.665 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.665 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-31/image128 2024-09-14T04:11:01.665 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.665 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-31/image128 2024-09-14T04:11:01.665 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-31/image128 2024-09-14T04:11:01.665 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.665 INFO:tasks.workunit.client.0.smithi039.stdout:num children is: 2 2024-09-14T04:11:01.666 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 2 2024-09-14T04:11:01.666 INFO:tasks.workunit.client.0.smithi039.stdout:child: image126 2024-09-14T04:11:01.666 INFO:tasks.workunit.client.0.smithi039.stdout:child: image128 2024-09-14T04:11:01.666 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-30/image126/ace22fa25831 2024-09-14T04:11:01.666 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.666 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-31/image128/ace232bb685a 2024-09-14T04:11:01.666 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.666 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.666 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.666 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.666 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-30/image126 2024-09-14T04:11:01.666 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.666 INFO:tasks.workunit.client.0.smithi039.stdout:num children is: 1 2024-09-14T04:11:01.666 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 1 2024-09-14T04:11:01.667 INFO:tasks.workunit.client.0.smithi039.stdout:child: image126 2024-09-14T04:11:01.667 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-30/image126/ace22fa25831 2024-09-14T04:11:01.667 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:01.667 INFO:tasks.workunit.client.0.smithi039.stdout:num children is: 0 2024-09-14T04:11:01.667 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 0 2024-09-14T04:11:01.667 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.ListChildren (9964 ms) 2024-09-14T04:11:01.667 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.ListChildrenTiered 2024-09-14T04:11:02.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:01 smithi129 ceph-mon[101316]: pgmap v737: 617 pgs: 617 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.0 KiB/s wr, 34 op/s 2024-09-14T04:11:02.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:01 smithi039 ceph-mon[106470]: pgmap v737: 617 pgs: 617 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.0 KiB/s wr, 34 op/s 2024-09-14T04:11:02.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:01 smithi039 ceph-mon[111235]: pgmap v737: 617 pgs: 617 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.0 KiB/s wr, 34 op/s 2024-09-14T04:11:03.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:02 smithi129 ceph-mon[101316]: osdmap e726: 8 total, 8 up, 8 in 2024-09-14T04:11:03.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:02 smithi039 ceph-mon[106470]: osdmap e726: 8 total, 8 up, 8 in 2024-09-14T04:11:03.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:02 smithi039 ceph-mon[111235]: osdmap e726: 8 total, 8 up, 8 in 2024-09-14T04:11:04.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:03 smithi129 ceph-mon[101316]: osdmap e727: 8 total, 8 up, 8 in 2024-09-14T04:11:04.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:03 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2525889445' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-32","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:11:04.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:03 smithi129 ceph-mon[101316]: pgmap v740: 649 pgs: 12 creating+peering, 20 unknown, 617 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 386 KiB/s rd, 11 KiB/s wr, 481 op/s 2024-09-14T04:11:04.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:03 smithi039 ceph-mon[111235]: osdmap e727: 8 total, 8 up, 8 in 2024-09-14T04:11:04.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:03 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2525889445' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-32","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:11:04.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:03 smithi039 ceph-mon[111235]: pgmap v740: 649 pgs: 12 creating+peering, 20 unknown, 617 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 386 KiB/s rd, 11 KiB/s wr, 481 op/s 2024-09-14T04:11:04.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:03 smithi039 ceph-mon[106470]: osdmap e727: 8 total, 8 up, 8 in 2024-09-14T04:11:04.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:03 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2525889445' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-32","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:11:04.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:03 smithi039 ceph-mon[106470]: pgmap v740: 649 pgs: 12 creating+peering, 20 unknown, 617 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 386 KiB/s rd, 11 KiB/s wr, 481 op/s 2024-09-14T04:11:05.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:04 smithi129 ceph-mon[101316]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:11:05.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:04 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2525889445' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:11:05.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:04 smithi129 ceph-mon[101316]: osdmap e728: 8 total, 8 up, 8 in 2024-09-14T04:11:05.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:04 smithi039 ceph-mon[111235]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:11:05.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:04 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2525889445' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:11:05.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:04 smithi039 ceph-mon[111235]: osdmap e728: 8 total, 8 up, 8 in 2024-09-14T04:11:05.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:04 smithi039 ceph-mon[106470]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:11:05.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:04 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2525889445' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-32","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:11:05.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:04 smithi039 ceph-mon[106470]: osdmap e728: 8 total, 8 up, 8 in 2024-09-14T04:11:06.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:05 smithi129 ceph-mon[101316]: osdmap e729: 8 total, 8 up, 8 in 2024-09-14T04:11:06.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:05 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1396167229' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-33","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:11:06.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:05 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-33","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:11:06.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:05 smithi129 ceph-mon[101316]: pgmap v743: 681 pgs: 15 creating+peering, 35 unknown, 631 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 568 KiB/s rd, 14 KiB/s wr, 694 op/s 2024-09-14T04:11:06.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:05 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:11:06.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:05 smithi129 ceph-mon[101316]: osdmap e730: 8 total, 8 up, 8 in 2024-09-14T04:11:06.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:05 smithi039 ceph-mon[111235]: osdmap e729: 8 total, 8 up, 8 in 2024-09-14T04:11:06.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:05 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1396167229' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-33","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:11:06.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:05 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-33","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:11:06.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:05 smithi039 ceph-mon[111235]: pgmap v743: 681 pgs: 15 creating+peering, 35 unknown, 631 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 568 KiB/s rd, 14 KiB/s wr, 694 op/s 2024-09-14T04:11:06.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:05 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:11:06.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:05 smithi039 ceph-mon[111235]: osdmap e730: 8 total, 8 up, 8 in 2024-09-14T04:11:06.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:05 smithi039 ceph-mon[106470]: osdmap e729: 8 total, 8 up, 8 in 2024-09-14T04:11:06.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:05 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1396167229' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-33","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:11:06.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:05 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-33","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:11:06.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:05 smithi039 ceph-mon[106470]: pgmap v743: 681 pgs: 15 creating+peering, 35 unknown, 631 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 568 KiB/s rd, 14 KiB/s wr, 694 op/s 2024-09-14T04:11:06.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:05 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-33","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:11:06.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:05 smithi039 ceph-mon[106470]: osdmap e730: 8 total, 8 up, 8 in 2024-09-14T04:11:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:07 smithi129 ceph-mon[101316]: osdmap e731: 8 total, 8 up, 8 in 2024-09-14T04:11:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:07 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/880653998' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-34","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:11:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:07 smithi129 ceph-mon[101316]: pgmap v746: 713 pgs: 15 creating+peering, 64 unknown, 634 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 208 KiB/s rd, 4.0 KiB/s wr, 245 op/s 2024-09-14T04:11:08.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:11:08.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:11:08.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:07 smithi039 ceph-mon[106470]: osdmap e731: 8 total, 8 up, 8 in 2024-09-14T04:11:08.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:07 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/880653998' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-34","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:11:08.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:07 smithi039 ceph-mon[106470]: pgmap v746: 713 pgs: 15 creating+peering, 64 unknown, 634 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 208 KiB/s rd, 4.0 KiB/s wr, 245 op/s 2024-09-14T04:11:08.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:11:08.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:11:08.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:07 smithi039 ceph-mon[111235]: osdmap e731: 8 total, 8 up, 8 in 2024-09-14T04:11:08.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:07 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/880653998' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-34","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:11:08.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:07 smithi039 ceph-mon[111235]: pgmap v746: 713 pgs: 15 creating+peering, 64 unknown, 634 active+clean; 297 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 208 KiB/s rd, 4.0 KiB/s wr, 245 op/s 2024-09-14T04:11:08.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:11:08.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:11:08.834 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:08 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/880653998' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:11:08.834 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:08 smithi129 ceph-mon[101316]: osdmap e732: 8 total, 8 up, 8 in 2024-09-14T04:11:08.834 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:08 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd tier add", "pool": "test-librbd-smithi039-85658-32", "tierpool":"test-librbd-smithi039-85658-34", "force_nonempty":""} : dispatch 2024-09-14T04:11:08.834 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:08 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd tier add", "pool": "test-librbd-smithi039-85658-32", "tierpool":"test-librbd-smithi039-85658-34", "force_nonempty":""} : dispatch 2024-09-14T04:11:08.834 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:08 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:11:08.906 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:11:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:11:08] "GET /metrics HTTP/1.1" 200 82286 "" "Prometheus/2.51.0" 2024-09-14T04:11:08.907 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:08 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/880653998' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:11:08.907 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:08 smithi039 ceph-mon[106470]: osdmap e732: 8 total, 8 up, 8 in 2024-09-14T04:11:08.907 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:08 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd tier add", "pool": "test-librbd-smithi039-85658-32", "tierpool":"test-librbd-smithi039-85658-34", "force_nonempty":""} : dispatch 2024-09-14T04:11:08.907 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:08 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd tier add", "pool": "test-librbd-smithi039-85658-32", "tierpool":"test-librbd-smithi039-85658-34", "force_nonempty":""} : dispatch 2024-09-14T04:11:08.907 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:08 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:11:08.907 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:08 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/880653998' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-34","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:11:08.907 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:08 smithi039 ceph-mon[111235]: osdmap e732: 8 total, 8 up, 8 in 2024-09-14T04:11:08.908 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:08 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd tier add", "pool": "test-librbd-smithi039-85658-32", "tierpool":"test-librbd-smithi039-85658-34", "force_nonempty":""} : dispatch 2024-09-14T04:11:08.908 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:08 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd tier add", "pool": "test-librbd-smithi039-85658-32", "tierpool":"test-librbd-smithi039-85658-34", "force_nonempty":""} : dispatch 2024-09-14T04:11:08.908 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:08 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:11:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:09 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi039-85658-32", "tierpool":"test-librbd-smithi039-85658-34", "force_nonempty":""}]': finished 2024-09-14T04:11:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:09 smithi129 ceph-mon[101316]: osdmap e733: 8 total, 8 up, 8 in 2024-09-14T04:11:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:09 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi039-85658-34", "mode":"writeback"} : dispatch 2024-09-14T04:11:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:09 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi039-85658-34", "mode":"writeback"} : dispatch 2024-09-14T04:11:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:09 smithi129 ceph-mon[101316]: pgmap v749: 713 pgs: 9 creating+peering, 14 unknown, 690 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:11:09.909 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:09 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi039-85658-32", "tierpool":"test-librbd-smithi039-85658-34", "force_nonempty":""}]': finished 2024-09-14T04:11:09.909 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:09 smithi039 ceph-mon[106470]: osdmap e733: 8 total, 8 up, 8 in 2024-09-14T04:11:09.909 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:09 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi039-85658-34", "mode":"writeback"} : dispatch 2024-09-14T04:11:09.909 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:09 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi039-85658-34", "mode":"writeback"} : dispatch 2024-09-14T04:11:09.909 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:09 smithi039 ceph-mon[106470]: pgmap v749: 713 pgs: 9 creating+peering, 14 unknown, 690 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:11:09.910 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:09 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier add", "pool": "test-librbd-smithi039-85658-32", "tierpool":"test-librbd-smithi039-85658-34", "force_nonempty":""}]': finished 2024-09-14T04:11:09.910 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:09 smithi039 ceph-mon[111235]: osdmap e733: 8 total, 8 up, 8 in 2024-09-14T04:11:09.910 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:09 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi039-85658-34", "mode":"writeback"} : dispatch 2024-09-14T04:11:09.910 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:09 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi039-85658-34", "mode":"writeback"} : dispatch 2024-09-14T04:11:09.910 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:09 smithi039 ceph-mon[111235]: pgmap v749: 713 pgs: 9 creating+peering, 14 unknown, 690 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:11:10.540 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:10.639 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:10 smithi129 ceph-mon[101316]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:11:10.639 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:10 smithi129 ceph-mon[101316]: Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-09-14T04:11:10.639 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:10 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi039-85658-34", "mode":"writeback"}]': finished 2024-09-14T04:11:10.639 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:10 smithi129 ceph-mon[101316]: osdmap e734: 8 total, 8 up, 8 in 2024-09-14T04:11:10.639 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:10 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi039-85658-32", "overlaypool":"test-librbd-smithi039-85658-34"} : dispatch 2024-09-14T04:11:10.639 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:10 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi039-85658-32", "overlaypool":"test-librbd-smithi039-85658-34"} : dispatch 2024-09-14T04:11:10.639 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:10 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi039-85658-32", "overlaypool":"test-librbd-smithi039-85658-34"}]': finished 2024-09-14T04:11:10.639 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:10 smithi129 ceph-mon[101316]: osdmap e735: 8 total, 8 up, 8 in 2024-09-14T04:11:10.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:10 smithi039 ceph-mon[106470]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:11:10.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:10 smithi039 ceph-mon[106470]: Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-09-14T04:11:10.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:10 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi039-85658-34", "mode":"writeback"}]': finished 2024-09-14T04:11:10.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:10 smithi039 ceph-mon[106470]: osdmap e734: 8 total, 8 up, 8 in 2024-09-14T04:11:10.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:10 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi039-85658-32", "overlaypool":"test-librbd-smithi039-85658-34"} : dispatch 2024-09-14T04:11:10.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:10 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi039-85658-32", "overlaypool":"test-librbd-smithi039-85658-34"} : dispatch 2024-09-14T04:11:10.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:10 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi039-85658-32", "overlaypool":"test-librbd-smithi039-85658-34"}]': finished 2024-09-14T04:11:10.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:10 smithi039 ceph-mon[106470]: osdmap e735: 8 total, 8 up, 8 in 2024-09-14T04:11:10.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:10 smithi039 ceph-mon[111235]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:11:10.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:10 smithi039 ceph-mon[111235]: Health check failed: 1 cache pools are missing hit_sets (CACHE_POOL_NO_HIT_SET) 2024-09-14T04:11:10.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:10 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier cache-mode", "pool": "test-librbd-smithi039-85658-34", "mode":"writeback"}]': finished 2024-09-14T04:11:10.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:10 smithi039 ceph-mon[111235]: osdmap e734: 8 total, 8 up, 8 in 2024-09-14T04:11:10.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:10 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi039-85658-32", "overlaypool":"test-librbd-smithi039-85658-34"} : dispatch 2024-09-14T04:11:10.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:10 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi039-85658-32", "overlaypool":"test-librbd-smithi039-85658-34"} : dispatch 2024-09-14T04:11:10.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:10 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier set-overlay", "pool": "test-librbd-smithi039-85658-32", "overlaypool":"test-librbd-smithi039-85658-34"}]': finished 2024-09-14T04:11:10.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:10 smithi039 ceph-mon[111235]: osdmap e735: 8 total, 8 up, 8 in 2024-09-14T04:11:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:11 smithi129 ceph-mon[101316]: pgmap v752: 713 pgs: 6 creating+peering, 6 unknown, 701 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:11:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:11 smithi129 ceph-mon[101316]: osdmap e736: 8 total, 8 up, 8 in 2024-09-14T04:11:12.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:11 smithi039 ceph-mon[106470]: pgmap v752: 713 pgs: 6 creating+peering, 6 unknown, 701 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:11:12.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:11 smithi039 ceph-mon[106470]: osdmap e736: 8 total, 8 up, 8 in 2024-09-14T04:11:12.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:11 smithi039 ceph-mon[111235]: pgmap v752: 713 pgs: 6 creating+peering, 6 unknown, 701 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:11:12.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:11 smithi039 ceph-mon[111235]: osdmap e736: 8 total, 8 up, 8 in 2024-09-14T04:11:13.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:12 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:11:13.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:12 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:11:13.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:12 smithi129 ceph-mon[101316]: osdmap e737: 8 total, 8 up, 8 in 2024-09-14T04:11:13.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:12 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:11:13.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:12 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:11:13.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:12 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:11:13.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:12 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:11:13.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:12 smithi039 ceph-mon[106470]: osdmap e737: 8 total, 8 up, 8 in 2024-09-14T04:11:13.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:12 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:11:13.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:12 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:11:13.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:12 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:11:13.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:12 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:11:13.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:12 smithi039 ceph-mon[111235]: osdmap e737: 8 total, 8 up, 8 in 2024-09-14T04:11:13.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:12 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:11:13.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:12 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:11:14.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:13 smithi129 ceph-mon[101316]: pgmap v755: 713 pgs: 713 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1023 B/s wr, 2 op/s 2024-09-14T04:11:14.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:13 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:11:14.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:13 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:11:14.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:13 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:11:14.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:13 smithi129 ceph-mon[101316]: osdmap e738: 8 total, 8 up, 8 in 2024-09-14T04:11:14.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:13 smithi039 ceph-mon[106470]: pgmap v755: 713 pgs: 713 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1023 B/s wr, 2 op/s 2024-09-14T04:11:14.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:13 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:11:14.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:13 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:11:14.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:13 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:11:14.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:13 smithi039 ceph-mon[106470]: osdmap e738: 8 total, 8 up, 8 in 2024-09-14T04:11:14.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:13 smithi039 ceph-mon[111235]: pgmap v755: 713 pgs: 713 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1023 B/s rd, 1023 B/s wr, 2 op/s 2024-09-14T04:11:14.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:13 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:11:14.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:13 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:11:14.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:13 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:11:14.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:13 smithi039 ceph-mon[111235]: osdmap e738: 8 total, 8 up, 8 in 2024-09-14T04:11:16.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:15 smithi129 ceph-mon[101316]: osdmap e739: 8 total, 8 up, 8 in 2024-09-14T04:11:16.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:15 smithi129 ceph-mon[101316]: pgmap v758: 713 pgs: 713 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.0 KiB/s wr, 34 op/s 2024-09-14T04:11:16.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:15 smithi039 ceph-mon[106470]: osdmap e739: 8 total, 8 up, 8 in 2024-09-14T04:11:16.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:15 smithi039 ceph-mon[106470]: pgmap v758: 713 pgs: 713 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.0 KiB/s wr, 34 op/s 2024-09-14T04:11:16.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:15 smithi039 ceph-mon[111235]: osdmap e739: 8 total, 8 up, 8 in 2024-09-14T04:11:16.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:15 smithi039 ceph-mon[111235]: pgmap v758: 713 pgs: 713 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.0 KiB/s wr, 34 op/s 2024-09-14T04:11:16.060 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.060 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-33/image130 2024-09-14T04:11:16.060 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-33/image130 2024-09-14T04:11:16.060 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-33/image130 2024-09-14T04:11:16.060 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.060 INFO:tasks.workunit.client.0.smithi039.stdout:num children is: 1 2024-09-14T04:11:16.060 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 1 2024-09-14T04:11:16.060 INFO:tasks.workunit.client.0.smithi039.stdout:child: image130 2024-09-14T04:11:16.060 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-33/image130/ace25b1cc0ab 2024-09-14T04:11:16.061 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.061 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.061 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-33/image130 2024-09-14T04:11:16.061 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:16.061 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-33/image130 2024-09-14T04:11:16.061 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-33/image130 2024-09-14T04:11:16.061 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.061 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.061 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:16.061 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:16.061 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:16.061 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.062 INFO:tasks.workunit.client.0.smithi039.stdout:num children is: 2 2024-09-14T04:11:16.062 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 2 2024-09-14T04:11:16.062 INFO:tasks.workunit.client.0.smithi039.stdout:child: image131 2024-09-14T04:11:16.062 INFO:tasks.workunit.client.0.smithi039.stdout:child: image130 2024-09-14T04:11:16.062 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-33/image130/ace25b1cc0ab 2024-09-14T04:11:16.062 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.062 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-32/image131/ace246da8075 2024-09-14T04:11:16.062 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.062 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.062 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-33/image130 2024-09-14T04:11:16.062 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:16.062 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-33/image130 2024-09-14T04:11:16.062 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-33/image130 2024-09-14T04:11:16.062 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.063 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.063 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:16.063 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:16.063 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:16.063 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.063 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.063 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-33/image132 2024-09-14T04:11:16.063 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:16.063 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-33/image130 2024-09-14T04:11:16.063 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-33/image132 2024-09-14T04:11:16.063 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-33/image132 2024-09-14T04:11:16.063 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.063 INFO:tasks.workunit.client.0.smithi039.stdout:num children is: 3 2024-09-14T04:11:16.064 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 3 2024-09-14T04:11:16.064 INFO:tasks.workunit.client.0.smithi039.stdout:child: image131 2024-09-14T04:11:16.064 INFO:tasks.workunit.client.0.smithi039.stdout:child: image130 2024-09-14T04:11:16.064 INFO:tasks.workunit.client.0.smithi039.stdout:child: image132 2024-09-14T04:11:16.064 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-33/image130/ace25b1cc0ab 2024-09-14T04:11:16.064 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.064 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-32/image131/ace246da8075 2024-09-14T04:11:16.064 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.064 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-33/image132/ace271752138 2024-09-14T04:11:16.064 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.064 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.064 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-33/image130 2024-09-14T04:11:16.065 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:16.065 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-33/image130 2024-09-14T04:11:16.065 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-33/image130 2024-09-14T04:11:16.065 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.065 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.065 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:16.065 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:16.065 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:16.065 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.065 INFO:tasks.workunit.client.0.smithi039.stdout:num children is: 3 2024-09-14T04:11:16.065 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 3 2024-09-14T04:11:16.065 INFO:tasks.workunit.client.0.smithi039.stdout:child: image131 2024-09-14T04:11:16.065 INFO:tasks.workunit.client.0.smithi039.stdout:child: image130 2024-09-14T04:11:16.065 INFO:tasks.workunit.client.0.smithi039.stdout:child: image132 2024-09-14T04:11:16.066 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-33/image130/ace25b1cc0ab 2024-09-14T04:11:16.066 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.066 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-32/image131/ace246da8075 2024-09-14T04:11:16.066 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.066 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-33/image132/ace271752138 2024-09-14T04:11:16.066 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.066 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.066 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-33/image130 2024-09-14T04:11:16.066 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:16.066 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-33/image130 2024-09-14T04:11:16.066 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-33/image130 2024-09-14T04:11:16.066 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.066 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.067 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:16.067 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:16.067 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:16.067 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.067 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.067 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-33/image133 2024-09-14T04:11:16.067 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:16.067 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-33/image130 2024-09-14T04:11:16.067 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-33/image133 2024-09-14T04:11:16.067 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-33/image133 2024-09-14T04:11:16.067 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.067 INFO:tasks.workunit.client.0.smithi039.stdout:num children is: 4 2024-09-14T04:11:16.067 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 4 2024-09-14T04:11:16.068 INFO:tasks.workunit.client.0.smithi039.stdout:child: image131 2024-09-14T04:11:16.068 INFO:tasks.workunit.client.0.smithi039.stdout:child: image130 2024-09-14T04:11:16.068 INFO:tasks.workunit.client.0.smithi039.stdout:child: image132 2024-09-14T04:11:16.068 INFO:tasks.workunit.client.0.smithi039.stdout:child: image133 2024-09-14T04:11:16.068 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-33/image130/ace25b1cc0ab 2024-09-14T04:11:16.068 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.068 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-32/image131/ace246da8075 2024-09-14T04:11:16.068 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.068 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-33/image132/ace271752138 2024-09-14T04:11:16.068 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.068 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-33/image133/ace2ec7f9046 2024-09-14T04:11:16.068 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.068 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.069 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-33/image130 2024-09-14T04:11:16.069 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:16.069 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-33/image130 2024-09-14T04:11:16.069 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-33/image130 2024-09-14T04:11:16.069 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.069 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.069 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:16.069 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:16.069 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:16.069 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.069 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.069 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-33/image132 2024-09-14T04:11:16.069 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:16.069 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-33/image130 2024-09-14T04:11:16.069 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-33/image132 2024-09-14T04:11:16.070 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-33/image132 2024-09-14T04:11:16.070 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.070 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:16.070 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-33/image133 2024-09-14T04:11:16.070 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:16 smithi129 ceph-mon[101316]: osdmap e740: 8 total, 8 up, 8 in 2024-09-14T04:11:17.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:16 smithi039 ceph-mon[106470]: osdmap e740: 8 total, 8 up, 8 in 2024-09-14T04:11:17.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:16 smithi039 ceph-mon[111235]: osdmap e740: 8 total, 8 up, 8 in 2024-09-14T04:11:18.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:17 smithi129 ceph-mon[101316]: osdmap e741: 8 total, 8 up, 8 in 2024-09-14T04:11:18.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:17 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi039-85658-32"} : dispatch 2024-09-14T04:11:18.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:17 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi039-85658-32"} : dispatch 2024-09-14T04:11:18.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:17 smithi129 ceph-mon[101316]: pgmap v761: 713 pgs: 713 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 1.5 KiB/s wr, 46 op/s 2024-09-14T04:11:18.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:17 smithi039 ceph-mon[106470]: osdmap e741: 8 total, 8 up, 8 in 2024-09-14T04:11:18.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:17 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi039-85658-32"} : dispatch 2024-09-14T04:11:18.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:17 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi039-85658-32"} : dispatch 2024-09-14T04:11:18.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:17 smithi039 ceph-mon[106470]: pgmap v761: 713 pgs: 713 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 1.5 KiB/s wr, 46 op/s 2024-09-14T04:11:18.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:17 smithi039 ceph-mon[111235]: osdmap e741: 8 total, 8 up, 8 in 2024-09-14T04:11:18.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:17 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi039-85658-32"} : dispatch 2024-09-14T04:11:18.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:17 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi039-85658-32"} : dispatch 2024-09-14T04:11:18.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:17 smithi039 ceph-mon[111235]: pgmap v761: 713 pgs: 713 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 1.5 KiB/s wr, 46 op/s 2024-09-14T04:11:18.602 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-33/image130 2024-09-14T04:11:18.602 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-33/image132 2024-09-14T04:11:18.602 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-33/image133 2024-09-14T04:11:18.602 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-33/image133 2024-09-14T04:11:18.602 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:18.602 INFO:tasks.workunit.client.0.smithi039.stdout:num children is: 4 2024-09-14T04:11:18.602 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 4 2024-09-14T04:11:18.602 INFO:tasks.workunit.client.0.smithi039.stdout:child: image131 2024-09-14T04:11:18.602 INFO:tasks.workunit.client.0.smithi039.stdout:child: image130 2024-09-14T04:11:18.602 INFO:tasks.workunit.client.0.smithi039.stdout:child: image132 2024-09-14T04:11:18.602 INFO:tasks.workunit.client.0.smithi039.stdout:child: image133 2024-09-14T04:11:18.603 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-33/image130/ace25b1cc0ab 2024-09-14T04:11:18.603 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:18.603 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-32/image131/ace246da8075 2024-09-14T04:11:18.603 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:18.603 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-33/image132/ace271752138 2024-09-14T04:11:18.603 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:18.603 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-33/image133/ace2ec7f9046 2024-09-14T04:11:18.603 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:18.603 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:18.603 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:18.603 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:18.603 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:18.603 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:18.603 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:18.604 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-33/image132 2024-09-14T04:11:18.604 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:18.604 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-33/image132 2024-09-14T04:11:18.604 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-33/image132 2024-09-14T04:11:18.604 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:18.604 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:18.604 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-33/image133 2024-09-14T04:11:18.604 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:18.604 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-33/image132 2024-09-14T04:11:18.604 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-33/image133 2024-09-14T04:11:18.604 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-33/image133 2024-09-14T04:11:18.604 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:18.604 INFO:tasks.workunit.client.0.smithi039.stdout:num children is: 3 2024-09-14T04:11:18.604 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 3 2024-09-14T04:11:18.605 INFO:tasks.workunit.client.0.smithi039.stdout:child: image131 2024-09-14T04:11:18.605 INFO:tasks.workunit.client.0.smithi039.stdout:child: image132 2024-09-14T04:11:18.605 INFO:tasks.workunit.client.0.smithi039.stdout:child: image133 2024-09-14T04:11:18.605 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-32/image131/ace246da8075 2024-09-14T04:11:18.605 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:18.605 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-33/image132/ace271752138 2024-09-14T04:11:18.605 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:18.605 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-33/image133/ace2ec7f9046 2024-09-14T04:11:18.605 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:18.605 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:18.605 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:18.605 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:18.605 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:18.605 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:18.606 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:18.606 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-33/image133 2024-09-14T04:11:18.606 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:18.606 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-33/image133 2024-09-14T04:11:18.606 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-33/image133 2024-09-14T04:11:18.606 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:18.606 INFO:tasks.workunit.client.0.smithi039.stdout:num children is: 2 2024-09-14T04:11:18.606 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 2 2024-09-14T04:11:18.606 INFO:tasks.workunit.client.0.smithi039.stdout:child: image131 2024-09-14T04:11:18.606 INFO:tasks.workunit.client.0.smithi039.stdout:child: image133 2024-09-14T04:11:18.606 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-32/image131/ace246da8075 2024-09-14T04:11:18.606 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:18.606 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-33/image133/ace2ec7f9046 2024-09-14T04:11:18.606 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:18.607 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:18.607 INFO:tasks.workunit.client.0.smithi039.stdout:trying to find test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:18.607 INFO:tasks.workunit.client.0.smithi039.stdout:checking test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:18.607 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-32/image131 2024-09-14T04:11:18.607 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:18.607 INFO:tasks.workunit.client.0.smithi039.stdout:num children is: 1 2024-09-14T04:11:18.607 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 1 2024-09-14T04:11:18.607 INFO:tasks.workunit.client.0.smithi039.stdout:child: image131 2024-09-14T04:11:18.607 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-32/image131/ace246da8075 2024-09-14T04:11:18.607 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:11:18.607 INFO:tasks.workunit.client.0.smithi039.stdout:num children is: 0 2024-09-14T04:11:18.607 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 0 2024-09-14T04:11:18.607 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.ListChildrenTiered (16941 ms) 2024-09-14T04:11:18.607 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.LockingPP 2024-09-14T04:11:18.608 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:18.666 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.LockingPP (64 ms) 2024-09-14T04:11:18.666 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.FlushAio 2024-09-14T04:11:18.667 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:18.906 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.FlushAio (240 ms) 2024-09-14T04:11:18.906 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.FlushAioPP 2024-09-14T04:11:18.906 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:19.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:18 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi039-85658-32"}]': finished 2024-09-14T04:11:19.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:18 smithi129 ceph-mon[101316]: osdmap e742: 8 total, 8 up, 8 in 2024-09-14T04:11:19.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:18 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd tier remove", "pool": "test-librbd-smithi039-85658-32", "tierpool":"test-librbd-smithi039-85658-34"} : dispatch 2024-09-14T04:11:19.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:18 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd tier remove", "pool": "test-librbd-smithi039-85658-32", "tierpool":"test-librbd-smithi039-85658-34"} : dispatch 2024-09-14T04:11:19.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:11:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:11:18] "GET /metrics HTTP/1.1" 200 82286 "" "Prometheus/2.51.0" 2024-09-14T04:11:19.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:18 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi039-85658-32"}]': finished 2024-09-14T04:11:19.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:18 smithi039 ceph-mon[106470]: osdmap e742: 8 total, 8 up, 8 in 2024-09-14T04:11:19.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:18 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd tier remove", "pool": "test-librbd-smithi039-85658-32", "tierpool":"test-librbd-smithi039-85658-34"} : dispatch 2024-09-14T04:11:19.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:18 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd tier remove", "pool": "test-librbd-smithi039-85658-32", "tierpool":"test-librbd-smithi039-85658-34"} : dispatch 2024-09-14T04:11:19.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:18 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier remove-overlay", "pool": "test-librbd-smithi039-85658-32"}]': finished 2024-09-14T04:11:19.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:18 smithi039 ceph-mon[111235]: osdmap e742: 8 total, 8 up, 8 in 2024-09-14T04:11:19.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:18 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd tier remove", "pool": "test-librbd-smithi039-85658-32", "tierpool":"test-librbd-smithi039-85658-34"} : dispatch 2024-09-14T04:11:19.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:18 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd tier remove", "pool": "test-librbd-smithi039-85658-32", "tierpool":"test-librbd-smithi039-85658-34"} : dispatch 2024-09-14T04:11:19.091 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.FlushAioPP (185 ms) 2024-09-14T04:11:19.091 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.ZeroLengthWrite 2024-09-14T04:11:19.091 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:19.113 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.ZeroLengthWrite (23 ms) 2024-09-14T04:11:19.114 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.ZeroLengthDiscard 2024-09-14T04:11:19.114 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:19.137 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.ZeroLengthDiscard (24 ms) 2024-09-14T04:11:19.138 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.ZeroLengthRead 2024-09-14T04:11:19.138 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:19.154 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.ZeroLengthRead (16 ms) 2024-09-14T04:11:19.154 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.LargeCacheRead 2024-09-14T04:11:19.154 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:19.198 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.LargeCacheRead (44 ms) 2024-09-14T04:11:19.198 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestPendingAio 2024-09-14T04:11:19.198 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:19.409 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestPendingAio (212 ms) 2024-09-14T04:11:19.410 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.CompareAndWriteCopyup 2024-09-14T04:11:19.410 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:19.699 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:20.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:19 smithi129 ceph-mon[101316]: Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-09-14T04:11:20.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:19 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi039-85658-32", "tierpool":"test-librbd-smithi039-85658-34"}]': finished 2024-09-14T04:11:20.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:19 smithi129 ceph-mon[101316]: osdmap e743: 8 total, 8 up, 8 in 2024-09-14T04:11:20.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:19 smithi129 ceph-mon[101316]: pgmap v764: 713 pgs: 713 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 382 KiB/s rd, 8.5 KiB/s wr, 474 op/s 2024-09-14T04:11:20.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:19 smithi039 ceph-mon[106470]: Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-09-14T04:11:20.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:19 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi039-85658-32", "tierpool":"test-librbd-smithi039-85658-34"}]': finished 2024-09-14T04:11:20.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:19 smithi039 ceph-mon[106470]: osdmap e743: 8 total, 8 up, 8 in 2024-09-14T04:11:20.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:19 smithi039 ceph-mon[106470]: pgmap v764: 713 pgs: 713 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 382 KiB/s rd, 8.5 KiB/s wr, 474 op/s 2024-09-14T04:11:20.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:19 smithi039 ceph-mon[111235]: Health check cleared: CACHE_POOL_NO_HIT_SET (was: 1 cache pools are missing hit_sets) 2024-09-14T04:11:20.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:19 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd tier remove", "pool": "test-librbd-smithi039-85658-32", "tierpool":"test-librbd-smithi039-85658-34"}]': finished 2024-09-14T04:11:20.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:19 smithi039 ceph-mon[111235]: osdmap e743: 8 total, 8 up, 8 in 2024-09-14T04:11:20.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:19 smithi039 ceph-mon[111235]: pgmap v764: 713 pgs: 713 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 382 KiB/s rd, 8.5 KiB/s wr, 474 op/s 2024-09-14T04:11:21.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:20 smithi129 ceph-mon[101316]: osdmap e744: 8 total, 8 up, 8 in 2024-09-14T04:11:21.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:20 smithi129 ceph-mon[101316]: osdmap e745: 8 total, 8 up, 8 in 2024-09-14T04:11:21.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:20 smithi039 ceph-mon[106470]: osdmap e744: 8 total, 8 up, 8 in 2024-09-14T04:11:21.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:20 smithi039 ceph-mon[106470]: osdmap e745: 8 total, 8 up, 8 in 2024-09-14T04:11:21.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:20 smithi039 ceph-mon[111235]: osdmap e744: 8 total, 8 up, 8 in 2024-09-14T04:11:21.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:20 smithi039 ceph-mon[111235]: osdmap e745: 8 total, 8 up, 8 in 2024-09-14T04:11:21.574 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.CompareAndWriteCopyup (2165 ms) 2024-09-14T04:11:21.574 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.CompareAndWriteCopyupMismatch 2024-09-14T04:11:21.575 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:21 smithi129 ceph-mon[101316]: pgmap v767: 713 pgs: 713 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 639 KiB/s rd, 19 KiB/s wr, 783 op/s 2024-09-14T04:11:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:21 smithi129 ceph-mon[101316]: osdmap e746: 8 total, 8 up, 8 in 2024-09-14T04:11:22.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:21 smithi039 ceph-mon[106470]: pgmap v767: 713 pgs: 713 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 639 KiB/s rd, 19 KiB/s wr, 783 op/s 2024-09-14T04:11:22.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:21 smithi039 ceph-mon[106470]: osdmap e746: 8 total, 8 up, 8 in 2024-09-14T04:11:22.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:21 smithi039 ceph-mon[111235]: pgmap v767: 713 pgs: 713 active+clean; 297 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 639 KiB/s rd, 19 KiB/s wr, 783 op/s 2024-09-14T04:11:22.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:21 smithi039 ceph-mon[111235]: osdmap e746: 8 total, 8 up, 8 in 2024-09-14T04:11:22.633 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:23.523 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:23 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:11:23.523 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:11:23.523 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:23 smithi129 ceph-mon[101316]: osdmap e747: 8 total, 8 up, 8 in 2024-09-14T04:11:23.523 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:23 smithi129 ceph-mon[101316]: pgmap v770: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 998 KiB/s rd, 740 KiB/s wr, 810 op/s 2024-09-14T04:11:23.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:23 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:11:23.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:11:23.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:23 smithi039 ceph-mon[106470]: osdmap e747: 8 total, 8 up, 8 in 2024-09-14T04:11:23.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:23 smithi039 ceph-mon[106470]: pgmap v770: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 998 KiB/s rd, 740 KiB/s wr, 810 op/s 2024-09-14T04:11:23.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:23 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:11:23.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:11:23.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:23 smithi039 ceph-mon[111235]: osdmap e747: 8 total, 8 up, 8 in 2024-09-14T04:11:23.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:23 smithi039 ceph-mon[111235]: pgmap v770: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 998 KiB/s rd, 740 KiB/s wr, 810 op/s 2024-09-14T04:11:24.612 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.CompareAndWriteCopyupMismatch (3038 ms) 2024-09-14T04:11:24.613 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.Flatten 2024-09-14T04:11:24.613 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:25.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:24 smithi129 ceph-mon[101316]: osdmap e748: 8 total, 8 up, 8 in 2024-09-14T04:11:25.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:24 smithi039 ceph-mon[106470]: osdmap e748: 8 total, 8 up, 8 in 2024-09-14T04:11:25.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:24 smithi039 ceph-mon[111235]: osdmap e748: 8 total, 8 up, 8 in 2024-09-14T04:11:25.639 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.Flatten (1027 ms) 2024-09-14T04:11:25.639 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.Sparsify 2024-09-14T04:11:25.640 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:25.676 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 32768 2024-09-14T04:11:25.676 INFO:tasks.workunit.client.0.smithi039.stdout:read: 32768 2024-09-14T04:11:25.676 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.Sparsify (37 ms) 2024-09-14T04:11:25.677 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.SparsifyPP 2024-09-14T04:11:25.677 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:25.710 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.SparsifyPP (34 ms) 2024-09-14T04:11:25.711 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.SnapshotLimit 2024-09-14T04:11:25.711 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:25 smithi129 ceph-mon[101316]: osdmap e749: 8 total, 8 up, 8 in 2024-09-14T04:11:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:25 smithi129 ceph-mon[101316]: pgmap v773: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 821 KiB/s rd, 736 KiB/s wr, 603 op/s 2024-09-14T04:11:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:25 smithi129 ceph-mon[101316]: osdmap e750: 8 total, 8 up, 8 in 2024-09-14T04:11:26.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:25 smithi039 ceph-mon[106470]: osdmap e749: 8 total, 8 up, 8 in 2024-09-14T04:11:26.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:25 smithi039 ceph-mon[106470]: pgmap v773: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 821 KiB/s rd, 736 KiB/s wr, 603 op/s 2024-09-14T04:11:26.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:25 smithi039 ceph-mon[106470]: osdmap e750: 8 total, 8 up, 8 in 2024-09-14T04:11:26.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:25 smithi039 ceph-mon[111235]: osdmap e749: 8 total, 8 up, 8 in 2024-09-14T04:11:26.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:25 smithi039 ceph-mon[111235]: pgmap v773: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 821 KiB/s rd, 736 KiB/s wr, 603 op/s 2024-09-14T04:11:26.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:25 smithi039 ceph-mon[111235]: osdmap e750: 8 total, 8 up, 8 in 2024-09-14T04:11:28.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:27 smithi129 ceph-mon[101316]: osdmap e751: 8 total, 8 up, 8 in 2024-09-14T04:11:28.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:27 smithi129 ceph-mon[101316]: pgmap v776: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 173 KiB/s rd, 11 KiB/s wr, 226 op/s 2024-09-14T04:11:28.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:27 smithi039 ceph-mon[106470]: osdmap e751: 8 total, 8 up, 8 in 2024-09-14T04:11:28.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:27 smithi039 ceph-mon[106470]: pgmap v776: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 173 KiB/s rd, 11 KiB/s wr, 226 op/s 2024-09-14T04:11:28.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:27 smithi039 ceph-mon[111235]: osdmap e751: 8 total, 8 up, 8 in 2024-09-14T04:11:28.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:27 smithi039 ceph-mon[111235]: pgmap v776: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 173 KiB/s rd, 11 KiB/s wr, 226 op/s 2024-09-14T04:11:29.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:28 smithi129 ceph-mon[101316]: osdmap e752: 8 total, 8 up, 8 in 2024-09-14T04:11:29.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:28 smithi039 ceph-mon[106470]: osdmap e752: 8 total, 8 up, 8 in 2024-09-14T04:11:29.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:28 smithi039 ceph-mon[111235]: osdmap e752: 8 total, 8 up, 8 in 2024-09-14T04:11:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:11:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:11:28] "GET /metrics HTTP/1.1" 200 82368 "" "Prometheus/2.51.0" 2024-09-14T04:11:29.941 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:29 smithi039 ceph-mon[106470]: osdmap e753: 8 total, 8 up, 8 in 2024-09-14T04:11:29.942 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:29 smithi039 ceph-mon[106470]: pgmap v779: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 133 KiB/s rd, 25 KiB/s wr, 158 op/s 2024-09-14T04:11:29.942 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:29 smithi039 ceph-mon[111235]: osdmap e753: 8 total, 8 up, 8 in 2024-09-14T04:11:29.942 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:29 smithi039 ceph-mon[111235]: pgmap v779: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 133 KiB/s rd, 25 KiB/s wr, 158 op/s 2024-09-14T04:11:30.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:29 smithi129 ceph-mon[101316]: osdmap e753: 8 total, 8 up, 8 in 2024-09-14T04:11:30.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:29 smithi129 ceph-mon[101316]: pgmap v779: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 133 KiB/s rd, 25 KiB/s wr, 158 op/s 2024-09-14T04:11:30.681 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.SnapshotLimit (4970 ms) 2024-09-14T04:11:30.681 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.SnapshotLimitPP 2024-09-14T04:11:30.681 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:31.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:30 smithi129 ceph-mon[101316]: osdmap e754: 8 total, 8 up, 8 in 2024-09-14T04:11:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:30 smithi039 ceph-mon[106470]: osdmap e754: 8 total, 8 up, 8 in 2024-09-14T04:11:31.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:30 smithi039 ceph-mon[111235]: osdmap e754: 8 total, 8 up, 8 in 2024-09-14T04:11:32.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:31 smithi129 ceph-mon[101316]: osdmap e755: 8 total, 8 up, 8 in 2024-09-14T04:11:32.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:31 smithi129 ceph-mon[101316]: pgmap v782: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 110 KiB/s rd, 31 KiB/s wr, 133 op/s 2024-09-14T04:11:32.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:31 smithi039 ceph-mon[106470]: osdmap e755: 8 total, 8 up, 8 in 2024-09-14T04:11:32.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:31 smithi039 ceph-mon[106470]: pgmap v782: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 110 KiB/s rd, 31 KiB/s wr, 133 op/s 2024-09-14T04:11:32.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:31 smithi039 ceph-mon[111235]: osdmap e755: 8 total, 8 up, 8 in 2024-09-14T04:11:32.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:31 smithi039 ceph-mon[111235]: pgmap v782: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 110 KiB/s rd, 31 KiB/s wr, 133 op/s 2024-09-14T04:11:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:32 smithi129 ceph-mon[101316]: osdmap e756: 8 total, 8 up, 8 in 2024-09-14T04:11:33.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:32 smithi039 ceph-mon[106470]: osdmap e756: 8 total, 8 up, 8 in 2024-09-14T04:11:33.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:32 smithi039 ceph-mon[111235]: osdmap e756: 8 total, 8 up, 8 in 2024-09-14T04:11:34.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:33 smithi129 ceph-mon[101316]: osdmap e757: 8 total, 8 up, 8 in 2024-09-14T04:11:34.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:33 smithi129 ceph-mon[101316]: pgmap v785: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 90 KiB/s rd, 12 KiB/s wr, 117 op/s 2024-09-14T04:11:34.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:33 smithi039 ceph-mon[106470]: osdmap e757: 8 total, 8 up, 8 in 2024-09-14T04:11:34.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:33 smithi039 ceph-mon[106470]: pgmap v785: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 90 KiB/s rd, 12 KiB/s wr, 117 op/s 2024-09-14T04:11:34.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:33 smithi039 ceph-mon[111235]: osdmap e757: 8 total, 8 up, 8 in 2024-09-14T04:11:34.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:33 smithi039 ceph-mon[111235]: pgmap v785: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 90 KiB/s rd, 12 KiB/s wr, 117 op/s 2024-09-14T04:11:35.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:34 smithi129 ceph-mon[101316]: osdmap e758: 8 total, 8 up, 8 in 2024-09-14T04:11:35.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:34 smithi039 ceph-mon[106470]: osdmap e758: 8 total, 8 up, 8 in 2024-09-14T04:11:35.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:34 smithi039 ceph-mon[111235]: osdmap e758: 8 total, 8 up, 8 in 2024-09-14T04:11:35.741 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.SnapshotLimitPP (5060 ms) 2024-09-14T04:11:35.741 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.RebuildObjectMapViaLockOwner 2024-09-14T04:11:35.741 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:35.795 INFO:tasks.workunit.client.0.smithi039.stdout:0.00% done 2024-09-14T04:11:35.796 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.RebuildObjectMapViaLockOwner (55 ms) 2024-09-14T04:11:35.796 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.RenameViaLockOwner 2024-09-14T04:11:35.796 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:35.856 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.RenameViaLockOwner (60 ms) 2024-09-14T04:11:35.856 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.SnapCreateViaLockOwner 2024-09-14T04:11:35.856 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:36.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:35 smithi129 ceph-mon[101316]: osdmap e759: 8 total, 8 up, 8 in 2024-09-14T04:11:36.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:35 smithi129 ceph-mon[101316]: pgmap v788: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 3.2 KiB/s wr, 37 op/s 2024-09-14T04:11:36.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:35 smithi039 ceph-mon[106470]: osdmap e759: 8 total, 8 up, 8 in 2024-09-14T04:11:36.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:35 smithi039 ceph-mon[106470]: pgmap v788: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 3.2 KiB/s wr, 37 op/s 2024-09-14T04:11:36.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:35 smithi039 ceph-mon[111235]: osdmap e759: 8 total, 8 up, 8 in 2024-09-14T04:11:36.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:35 smithi039 ceph-mon[111235]: pgmap v788: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 3.2 KiB/s wr, 37 op/s 2024-09-14T04:11:36.758 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.SnapCreateViaLockOwner (903 ms) 2024-09-14T04:11:36.758 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.SnapRemoveViaLockOwner 2024-09-14T04:11:36.759 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:37.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:36 smithi129 ceph-mon[101316]: osdmap e760: 8 total, 8 up, 8 in 2024-09-14T04:11:37.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:36 smithi039 ceph-mon[106470]: osdmap e760: 8 total, 8 up, 8 in 2024-09-14T04:11:37.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:36 smithi039 ceph-mon[111235]: osdmap e760: 8 total, 8 up, 8 in 2024-09-14T04:11:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:37 smithi129 ceph-mon[101316]: osdmap e761: 8 total, 8 up, 8 in 2024-09-14T04:11:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:37 smithi129 ceph-mon[101316]: pgmap v791: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 42 KiB/s rd, 3.0 KiB/s wr, 52 op/s 2024-09-14T04:11:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:11:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:37 smithi039 ceph-mon[106470]: osdmap e761: 8 total, 8 up, 8 in 2024-09-14T04:11:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:37 smithi039 ceph-mon[106470]: pgmap v791: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 42 KiB/s rd, 3.0 KiB/s wr, 52 op/s 2024-09-14T04:11:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:11:38.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:37 smithi039 ceph-mon[111235]: osdmap e761: 8 total, 8 up, 8 in 2024-09-14T04:11:38.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:37 smithi039 ceph-mon[111235]: pgmap v791: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 42 KiB/s rd, 3.0 KiB/s wr, 52 op/s 2024-09-14T04:11:38.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:11:38.782 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.SnapRemoveViaLockOwner (2024 ms) 2024-09-14T04:11:38.782 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.UpdateFeaturesViaLockOwner 2024-09-14T04:11:38.783 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:38.836 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.UpdateFeaturesViaLockOwner (53 ms) 2024-09-14T04:11:38.836 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.EnableJournalingViaLockOwner 2024-09-14T04:11:38.836 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/test_librbd.cc:9736: Skipped 2024-09-14T04:11:38.836 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T04:11:38.836 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestLibRBD.EnableJournalingViaLockOwner (0 ms) 2024-09-14T04:11:38.836 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.SnapRemove2 2024-09-14T04:11:38.837 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:39.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:38 smithi129 ceph-mon[101316]: osdmap e762: 8 total, 8 up, 8 in 2024-09-14T04:11:39.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:38 smithi039 ceph-mon[106470]: osdmap e762: 8 total, 8 up, 8 in 2024-09-14T04:11:39.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:11:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:11:38] "GET /metrics HTTP/1.1" 200 82363 "" "Prometheus/2.51.0" 2024-09-14T04:11:39.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:38 smithi039 ceph-mon[111235]: osdmap e762: 8 total, 8 up, 8 in 2024-09-14T04:11:40.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:39 smithi129 ceph-mon[101316]: osdmap e763: 8 total, 8 up, 8 in 2024-09-14T04:11:40.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:39 smithi129 ceph-mon[101316]: pgmap v794: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 96 KiB/s rd, 8.5 KiB/s wr, 128 op/s 2024-09-14T04:11:40.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:39 smithi039 ceph-mon[106470]: osdmap e763: 8 total, 8 up, 8 in 2024-09-14T04:11:40.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:39 smithi039 ceph-mon[106470]: pgmap v794: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 96 KiB/s rd, 8.5 KiB/s wr, 128 op/s 2024-09-14T04:11:40.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:39 smithi039 ceph-mon[111235]: osdmap e763: 8 total, 8 up, 8 in 2024-09-14T04:11:40.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:39 smithi039 ceph-mon[111235]: pgmap v794: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 96 KiB/s rd, 8.5 KiB/s wr, 128 op/s 2024-09-14T04:11:40.573 INFO:tasks.workunit.client.0.smithi039.stdout:100.00% done 2024-09-14T04:11:40.573 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.SnapRemove2 (1737 ms) 2024-09-14T04:11:40.573 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.SnapRenameViaLockOwner 2024-09-14T04:11:40.573 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:41.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:40 smithi129 ceph-mon[101316]: osdmap e764: 8 total, 8 up, 8 in 2024-09-14T04:11:41.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:40 smithi129 ceph-mon[101316]: osdmap e765: 8 total, 8 up, 8 in 2024-09-14T04:11:41.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:40 smithi039 ceph-mon[106470]: osdmap e764: 8 total, 8 up, 8 in 2024-09-14T04:11:41.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:40 smithi039 ceph-mon[106470]: osdmap e765: 8 total, 8 up, 8 in 2024-09-14T04:11:41.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:40 smithi039 ceph-mon[111235]: osdmap e764: 8 total, 8 up, 8 in 2024-09-14T04:11:41.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:40 smithi039 ceph-mon[111235]: osdmap e765: 8 total, 8 up, 8 in 2024-09-14T04:11:41.589 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.SnapRenameViaLockOwner (1016 ms) 2024-09-14T04:11:41.589 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.SnapProtectViaLockOwner 2024-09-14T04:11:41.589 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:42.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:41 smithi039 ceph-mon[106470]: pgmap v797: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 81 KiB/s rd, 10 KiB/s wr, 120 op/s 2024-09-14T04:11:42.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:41 smithi039 ceph-mon[106470]: osdmap e766: 8 total, 8 up, 8 in 2024-09-14T04:11:42.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:41 smithi039 ceph-mon[111235]: pgmap v797: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 81 KiB/s rd, 10 KiB/s wr, 120 op/s 2024-09-14T04:11:42.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:41 smithi039 ceph-mon[111235]: osdmap e766: 8 total, 8 up, 8 in 2024-09-14T04:11:42.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:41 smithi129 ceph-mon[101316]: pgmap v797: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 81 KiB/s rd, 10 KiB/s wr, 120 op/s 2024-09-14T04:11:42.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:41 smithi129 ceph-mon[101316]: osdmap e766: 8 total, 8 up, 8 in 2024-09-14T04:11:42.593 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.SnapProtectViaLockOwner (1004 ms) 2024-09-14T04:11:42.593 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.SnapUnprotectViaLockOwner 2024-09-14T04:11:42.593 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:43.612 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.SnapUnprotectViaLockOwner (1019 ms) 2024-09-14T04:11:43.612 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.FlattenViaLockOwner 2024-09-14T04:11:43.612 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:44.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:43 smithi129 ceph-mon[101316]: osdmap e767: 8 total, 8 up, 8 in 2024-09-14T04:11:44.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:43 smithi129 ceph-mon[101316]: pgmap v800: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 115 KiB/s rd, 11 KiB/s wr, 160 op/s 2024-09-14T04:11:44.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:43 smithi039 ceph-mon[106470]: osdmap e767: 8 total, 8 up, 8 in 2024-09-14T04:11:44.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:43 smithi039 ceph-mon[106470]: pgmap v800: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 115 KiB/s rd, 11 KiB/s wr, 160 op/s 2024-09-14T04:11:44.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:43 smithi039 ceph-mon[111235]: osdmap e767: 8 total, 8 up, 8 in 2024-09-14T04:11:44.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:43 smithi039 ceph-mon[111235]: pgmap v800: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 115 KiB/s rd, 11 KiB/s wr, 160 op/s 2024-09-14T04:11:44.684 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.FlattenViaLockOwner (1072 ms) 2024-09-14T04:11:44.684 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.ResizeViaLockOwner 2024-09-14T04:11:44.684 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:44.725 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.ResizeViaLockOwner (41 ms) 2024-09-14T04:11:44.725 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.SparsifyViaLockOwner 2024-09-14T04:11:44.725 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:44.769 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.SparsifyViaLockOwner (45 ms) 2024-09-14T04:11:44.769 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.ObjectMapConsistentSnap 2024-09-14T04:11:44.769 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:45.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:44 smithi129 ceph-mon[101316]: osdmap e768: 8 total, 8 up, 8 in 2024-09-14T04:11:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:44 smithi039 ceph-mon[106470]: osdmap e768: 8 total, 8 up, 8 in 2024-09-14T04:11:45.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:44 smithi039 ceph-mon[111235]: osdmap e768: 8 total, 8 up, 8 in 2024-09-14T04:11:46.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:45 smithi129 ceph-mon[101316]: osdmap e769: 8 total, 8 up, 8 in 2024-09-14T04:11:46.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:45 smithi129 ceph-mon[101316]: pgmap v803: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 181 KiB/s rd, 13 KiB/s wr, 244 op/s 2024-09-14T04:11:46.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:45 smithi129 ceph-mon[101316]: osdmap e770: 8 total, 8 up, 8 in 2024-09-14T04:11:46.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:45 smithi039 ceph-mon[106470]: osdmap e769: 8 total, 8 up, 8 in 2024-09-14T04:11:46.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:45 smithi039 ceph-mon[106470]: pgmap v803: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 181 KiB/s rd, 13 KiB/s wr, 244 op/s 2024-09-14T04:11:46.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:45 smithi039 ceph-mon[106470]: osdmap e770: 8 total, 8 up, 8 in 2024-09-14T04:11:46.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:45 smithi039 ceph-mon[111235]: osdmap e769: 8 total, 8 up, 8 in 2024-09-14T04:11:46.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:45 smithi039 ceph-mon[111235]: pgmap v803: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 181 KiB/s rd, 13 KiB/s wr, 244 op/s 2024-09-14T04:11:46.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:45 smithi039 ceph-mon[111235]: osdmap e770: 8 total, 8 up, 8 in 2024-09-14T04:11:47.823 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:47 smithi129 ceph-mon[101316]: osdmap e771: 8 total, 8 up, 8 in 2024-09-14T04:11:47.823 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:47 smithi129 ceph-mon[101316]: pgmap v806: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 104 KiB/s rd, 6.2 KiB/s wr, 142 op/s 2024-09-14T04:11:48.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:47 smithi039 ceph-mon[106470]: osdmap e771: 8 total, 8 up, 8 in 2024-09-14T04:11:48.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:47 smithi039 ceph-mon[106470]: pgmap v806: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 104 KiB/s rd, 6.2 KiB/s wr, 142 op/s 2024-09-14T04:11:48.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:47 smithi039 ceph-mon[111235]: osdmap e771: 8 total, 8 up, 8 in 2024-09-14T04:11:48.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:47 smithi039 ceph-mon[111235]: pgmap v806: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 104 KiB/s rd, 6.2 KiB/s wr, 142 op/s 2024-09-14T04:11:49.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:48 smithi129 ceph-mon[101316]: osdmap e772: 8 total, 8 up, 8 in 2024-09-14T04:11:49.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:48 smithi039 ceph-mon[106470]: osdmap e772: 8 total, 8 up, 8 in 2024-09-14T04:11:49.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:11:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:11:48] "GET /metrics HTTP/1.1" 200 82363 "" "Prometheus/2.51.0" 2024-09-14T04:11:49.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:48 smithi039 ceph-mon[111235]: osdmap e772: 8 total, 8 up, 8 in 2024-09-14T04:11:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:49 smithi129 ceph-mon[101316]: osdmap e773: 8 total, 8 up, 8 in 2024-09-14T04:11:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:49 smithi129 ceph-mon[101316]: pgmap v809: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 159 KiB/s rd, 8.5 KiB/s wr, 203 op/s 2024-09-14T04:11:50.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:49 smithi039 ceph-mon[106470]: osdmap e773: 8 total, 8 up, 8 in 2024-09-14T04:11:50.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:49 smithi039 ceph-mon[106470]: pgmap v809: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 159 KiB/s rd, 8.5 KiB/s wr, 203 op/s 2024-09-14T04:11:50.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:49 smithi039 ceph-mon[111235]: osdmap e773: 8 total, 8 up, 8 in 2024-09-14T04:11:50.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:49 smithi039 ceph-mon[111235]: pgmap v809: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 159 KiB/s rd, 8.5 KiB/s wr, 203 op/s 2024-09-14T04:11:51.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:50 smithi129 ceph-mon[101316]: osdmap e774: 8 total, 8 up, 8 in 2024-09-14T04:11:51.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:50 smithi129 ceph-mon[101316]: osdmap e775: 8 total, 8 up, 8 in 2024-09-14T04:11:51.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:50 smithi039 ceph-mon[106470]: osdmap e774: 8 total, 8 up, 8 in 2024-09-14T04:11:51.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:50 smithi039 ceph-mon[106470]: osdmap e775: 8 total, 8 up, 8 in 2024-09-14T04:11:51.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:50 smithi039 ceph-mon[111235]: osdmap e774: 8 total, 8 up, 8 in 2024-09-14T04:11:51.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:50 smithi039 ceph-mon[111235]: osdmap e775: 8 total, 8 up, 8 in 2024-09-14T04:11:52.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:51 smithi129 ceph-mon[101316]: pgmap v812: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 148 KiB/s rd, 9.2 KiB/s wr, 194 op/s 2024-09-14T04:11:52.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:51 smithi129 ceph-mon[101316]: osdmap e776: 8 total, 8 up, 8 in 2024-09-14T04:11:52.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:51 smithi039 ceph-mon[106470]: pgmap v812: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 148 KiB/s rd, 9.2 KiB/s wr, 194 op/s 2024-09-14T04:11:52.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:51 smithi039 ceph-mon[106470]: osdmap e776: 8 total, 8 up, 8 in 2024-09-14T04:11:52.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:51 smithi039 ceph-mon[111235]: pgmap v812: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 148 KiB/s rd, 9.2 KiB/s wr, 194 op/s 2024-09-14T04:11:52.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:51 smithi039 ceph-mon[111235]: osdmap e776: 8 total, 8 up, 8 in 2024-09-14T04:11:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:11:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:52 smithi129 ceph-mon[101316]: osdmap e777: 8 total, 8 up, 8 in 2024-09-14T04:11:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:11:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:52 smithi039 ceph-mon[106470]: osdmap e777: 8 total, 8 up, 8 in 2024-09-14T04:11:53.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:11:53.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:52 smithi039 ceph-mon[111235]: osdmap e777: 8 total, 8 up, 8 in 2024-09-14T04:11:54.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:53 smithi129 ceph-mon[101316]: pgmap v815: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 70 KiB/s rd, 4.5 KiB/s wr, 97 op/s 2024-09-14T04:11:54.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:53 smithi129 ceph-mon[101316]: osdmap e778: 8 total, 8 up, 8 in 2024-09-14T04:11:54.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:53 smithi039 ceph-mon[106470]: pgmap v815: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 70 KiB/s rd, 4.5 KiB/s wr, 97 op/s 2024-09-14T04:11:54.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:53 smithi039 ceph-mon[106470]: osdmap e778: 8 total, 8 up, 8 in 2024-09-14T04:11:54.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:53 smithi039 ceph-mon[111235]: pgmap v815: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 70 KiB/s rd, 4.5 KiB/s wr, 97 op/s 2024-09-14T04:11:54.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:53 smithi039 ceph-mon[111235]: osdmap e778: 8 total, 8 up, 8 in 2024-09-14T04:11:55.185 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.ObjectMapConsistentSnap (10415 ms) 2024-09-14T04:11:55.185 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.Metadata 2024-09-14T04:11:55.185 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:55.664 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.Metadata (480 ms) 2024-09-14T04:11:55.664 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.MetadataPP 2024-09-14T04:11:55.664 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:56.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:55 smithi129 ceph-mon[101316]: osdmap e779: 8 total, 8 up, 8 in 2024-09-14T04:11:56.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:55 smithi129 ceph-mon[101316]: pgmap v818: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 67 KiB/s rd, 4.7 KiB/s wr, 90 op/s 2024-09-14T04:11:56.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:55 smithi039 ceph-mon[106470]: osdmap e779: 8 total, 8 up, 8 in 2024-09-14T04:11:56.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:55 smithi039 ceph-mon[106470]: pgmap v818: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 67 KiB/s rd, 4.7 KiB/s wr, 90 op/s 2024-09-14T04:11:56.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:55 smithi039 ceph-mon[111235]: osdmap e779: 8 total, 8 up, 8 in 2024-09-14T04:11:56.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:55 smithi039 ceph-mon[111235]: pgmap v818: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 67 KiB/s rd, 4.7 KiB/s wr, 90 op/s 2024-09-14T04:11:56.682 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.MetadataPP (1017 ms) 2024-09-14T04:11:56.682 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.UpdateFeatures 2024-09-14T04:11:56.682 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:56.800 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.UpdateFeatures (119 ms) 2024-09-14T04:11:56.800 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.FeaturesBitmaskString 2024-09-14T04:11:56.801 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.FeaturesBitmaskString (0 ms) 2024-09-14T04:11:56.801 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.RebuildObjectMap 2024-09-14T04:11:56.801 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:57.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:56 smithi129 ceph-mon[101316]: osdmap e780: 8 total, 8 up, 8 in 2024-09-14T04:11:57.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:56 smithi039 ceph-mon[106470]: osdmap e780: 8 total, 8 up, 8 in 2024-09-14T04:11:57.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:56 smithi039 ceph-mon[111235]: osdmap e780: 8 total, 8 up, 8 in 2024-09-14T04:11:57.646 INFO:tasks.workunit.client.0.smithi039.stdout:0.00% done 2024-09-14T04:11:57.647 INFO:tasks.workunit.client.0.smithi039.stdout:25.00% done 2024-09-14T04:11:57.647 INFO:tasks.workunit.client.0.smithi039.stdout:50.00% done 2024-09-14T04:11:57.647 INFO:tasks.workunit.client.0.smithi039.stdout:75.00% done 2024-09-14T04:11:57.647 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.RebuildObjectMap (846 ms) 2024-09-14T04:11:57.647 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.RebuildNewObjectMap 2024-09-14T04:11:57.678 INFO:tasks.workunit.client.0.smithi039.stdout:0.00% done 2024-09-14T04:11:57.678 INFO:tasks.workunit.client.0.smithi039.stdout:25.00% done 2024-09-14T04:11:57.678 INFO:tasks.workunit.client.0.smithi039.stdout:50.00% done 2024-09-14T04:11:57.679 INFO:tasks.workunit.client.0.smithi039.stdout:75.00% done 2024-09-14T04:11:57.679 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.RebuildNewObjectMap (32 ms) 2024-09-14T04:11:57.679 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.CheckObjectMap 2024-09-14T04:11:57.679 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:58.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:57 smithi129 ceph-mon[101316]: osdmap e781: 8 total, 8 up, 8 in 2024-09-14T04:11:58.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:57 smithi129 ceph-mon[101316]: pgmap v821: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 9.5 KiB/s wr, 17 op/s 2024-09-14T04:11:58.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:57 smithi039 ceph-mon[106470]: osdmap e781: 8 total, 8 up, 8 in 2024-09-14T04:11:58.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:57 smithi039 ceph-mon[106470]: pgmap v821: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 9.5 KiB/s wr, 17 op/s 2024-09-14T04:11:58.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:57 smithi039 ceph-mon[111235]: osdmap e781: 8 total, 8 up, 8 in 2024-09-14T04:11:58.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:57 smithi039 ceph-mon[111235]: pgmap v821: 713 pgs: 713 active+clean; 305 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 8.2 KiB/s rd, 9.5 KiB/s wr, 17 op/s 2024-09-14T04:11:58.653 INFO:tasks.workunit.client.0.smithi039.stdout:0.00% done 2024-09-14T04:11:58.653 INFO:tasks.workunit.client.0.smithi039.stdout:25.00% done 2024-09-14T04:11:58.653 INFO:tasks.workunit.client.0.smithi039.stdout:50.00% done 2024-09-14T04:11:58.653 INFO:tasks.workunit.client.0.smithi039.stdout:75.00% done 2024-09-14T04:11:58.654 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.CheckObjectMap (974 ms) 2024-09-14T04:11:58.654 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.BlockingAIO 2024-09-14T04:11:58.654 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:58.685 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.BlockingAIO (32 ms) 2024-09-14T04:11:58.685 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.ExclusiveLockTransition 2024-09-14T04:11:58.685 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:58.811 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.ExclusiveLockTransition (127 ms) 2024-09-14T04:11:58.812 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.ExclusiveLockReadTransition 2024-09-14T04:11:58.812 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/test_librbd.cc:10793: Skipped 2024-09-14T04:11:58.812 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T04:11:58.812 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestLibRBD.ExclusiveLockReadTransition (0 ms) 2024-09-14T04:11:58.812 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.CacheMayCopyOnWrite 2024-09-14T04:11:58.812 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:59.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:58 smithi129 ceph-mon[101316]: osdmap e782: 8 total, 8 up, 8 in 2024-09-14T04:11:59.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:58 smithi039 ceph-mon[106470]: osdmap e782: 8 total, 8 up, 8 in 2024-09-14T04:11:59.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:58 smithi039 ceph-mon[111235]: osdmap e782: 8 total, 8 up, 8 in 2024-09-14T04:11:59.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:11:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:11:58] "GET /metrics HTTP/1.1" 200 82360 "" "Prometheus/2.51.0" 2024-09-14T04:11:59.716 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.CacheMayCopyOnWrite (904 ms) 2024-09-14T04:11:59.716 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.FlushEmptyOpsOnExternalSnapshot 2024-09-14T04:11:59.716 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:11:59.941 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:59 smithi039 ceph-mon[106470]: osdmap e783: 8 total, 8 up, 8 in 2024-09-14T04:11:59.941 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:11:59 smithi039 ceph-mon[106470]: pgmap v824: 713 pgs: 713 active+clean; 305 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 524 KiB/s rd, 195 KiB/s wr, 845 op/s 2024-09-14T04:11:59.942 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:59 smithi039 ceph-mon[111235]: osdmap e783: 8 total, 8 up, 8 in 2024-09-14T04:11:59.942 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:11:59 smithi039 ceph-mon[111235]: pgmap v824: 713 pgs: 713 active+clean; 305 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 524 KiB/s rd, 195 KiB/s wr, 845 op/s 2024-09-14T04:12:00.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:59 smithi129 ceph-mon[101316]: osdmap e783: 8 total, 8 up, 8 in 2024-09-14T04:12:00.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:11:59 smithi129 ceph-mon[101316]: pgmap v824: 713 pgs: 713 active+clean; 305 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 524 KiB/s rd, 195 KiB/s wr, 845 op/s 2024-09-14T04:12:00.671 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.FlushEmptyOpsOnExternalSnapshot (955 ms) 2024-09-14T04:12:00.671 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestImageOptions 2024-09-14T04:12:01.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:00 smithi129 ceph-mon[101316]: osdmap e784: 8 total, 8 up, 8 in 2024-09-14T04:12:01.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:00 smithi039 ceph-mon[106470]: osdmap e784: 8 total, 8 up, 8 in 2024-09-14T04:12:01.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:00 smithi039 ceph-mon[111235]: osdmap e784: 8 total, 8 up, 8 in 2024-09-14T04:12:01.771 INFO:tasks.workunit.client.0.smithi039.stdout:0.00% done 2024-09-14T04:12:01.771 INFO:tasks.workunit.client.0.smithi039.stdout:100.00% done 2024-09-14T04:12:01.771 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestImageOptions (1100 ms) 2024-09-14T04:12:01.771 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestImageOptionsPP 2024-09-14T04:12:02.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:01 smithi129 ceph-mon[101316]: osdmap e785: 8 total, 8 up, 8 in 2024-09-14T04:12:02.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:01 smithi129 ceph-mon[101316]: pgmap v827: 713 pgs: 713 active+clean; 305 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 733 KiB/s rd, 229 KiB/s wr, 1.11k op/s 2024-09-14T04:12:02.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:01 smithi039 ceph-mon[106470]: osdmap e785: 8 total, 8 up, 8 in 2024-09-14T04:12:02.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:01 smithi039 ceph-mon[106470]: pgmap v827: 713 pgs: 713 active+clean; 305 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 733 KiB/s rd, 229 KiB/s wr, 1.11k op/s 2024-09-14T04:12:02.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:01 smithi039 ceph-mon[111235]: osdmap e785: 8 total, 8 up, 8 in 2024-09-14T04:12:02.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:01 smithi039 ceph-mon[111235]: pgmap v827: 713 pgs: 713 active+clean; 305 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 733 KiB/s rd, 229 KiB/s wr, 1.11k op/s 2024-09-14T04:12:02.765 INFO:tasks.workunit.client.0.smithi039.stdout:0.00% done 2024-09-14T04:12:02.766 INFO:tasks.workunit.client.0.smithi039.stdout:100.00% done 2024-09-14T04:12:02.766 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestImageOptionsPP (995 ms) 2024-09-14T04:12:02.766 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.EventSocketPipe 2024-09-14T04:12:02.766 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.EventSocketPipe (0 ms) 2024-09-14T04:12:02.766 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.EventSocketEventfd 2024-09-14T04:12:02.766 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.EventSocketEventfd (0 ms) 2024-09-14T04:12:02.766 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.ImagePollIO 2024-09-14T04:12:02.766 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:02.812 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:12:02.812 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:12:02.812 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:12:02.813 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:12:02.813 INFO:tasks.workunit.client.0.smithi039.stdout:wrote: 512 2024-09-14T04:12:02.813 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:12:02.813 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:12:02.813 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:12:02.813 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:12:02.813 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:12:02.813 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:12:02.813 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:12:02.813 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:12:02.813 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:12:02.813 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:12:02.813 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:12:02.814 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:12:02.814 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:12:02.814 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:12:02.814 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:12:02.814 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:12:02.814 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:12:02.814 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:12:02.814 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:12:02.814 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:12:02.814 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:12:02.814 INFO:tasks.workunit.client.0.smithi039.stdout:started write 2024-09-14T04:12:02.814 INFO:tasks.workunit.client.0.smithi039.stdout:write completion cb called! 2024-09-14T04:12:02.814 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 0 2024-09-14T04:12:02.815 INFO:tasks.workunit.client.0.smithi039.stdout:finished write 2024-09-14T04:12:02.815 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:12:02.815 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:12:02.815 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:12:02.815 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:12:02.815 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:12:02.815 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:12:02.815 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:12:02.815 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:12:02.815 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:12:02.815 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:12:02.815 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:12:02.815 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:12:02.816 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:12:02.816 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:12:02.816 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:12:02.816 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:12:02.816 INFO:tasks.workunit.client.0.smithi039.stdout:created completion 2024-09-14T04:12:02.816 INFO:tasks.workunit.client.0.smithi039.stdout:started read 2024-09-14T04:12:02.816 INFO:tasks.workunit.client.0.smithi039.stdout:read completion cb called! 2024-09-14T04:12:02.816 INFO:tasks.workunit.client.0.smithi039.stdout:return value is: 512 2024-09-14T04:12:02.816 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.ImagePollIO (47 ms) 2024-09-14T04:12:02.816 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.Mirror 2024-09-14T04:12:02.817 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:02.825 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:03.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:02 smithi129 ceph-mon[101316]: osdmap e786: 8 total, 8 up, 8 in 2024-09-14T04:12:03.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:02 smithi039 ceph-mon[106470]: osdmap e786: 8 total, 8 up, 8 in 2024-09-14T04:12:03.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:02 smithi039 ceph-mon[111235]: osdmap e786: 8 total, 8 up, 8 in 2024-09-14T04:12:03.849 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.Mirror (1037 ms) 2024-09-14T04:12:03.849 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.MirrorPeerAttributes 2024-09-14T04:12:03.866 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.MirrorPeerAttributes (16 ms) 2024-09-14T04:12:03.866 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.CreateWithMirrorEnabled 2024-09-14T04:12:04.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:03 smithi129 ceph-mon[101316]: osdmap e787: 8 total, 8 up, 8 in 2024-09-14T04:12:04.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:03 smithi129 ceph-mon[101316]: pgmap v830: 713 pgs: 713 active+clean; 310 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 476 KiB/s rd, 1.3 MiB/s wr, 625 op/s 2024-09-14T04:12:04.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:03 smithi039 ceph-mon[106470]: osdmap e787: 8 total, 8 up, 8 in 2024-09-14T04:12:04.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:03 smithi039 ceph-mon[106470]: pgmap v830: 713 pgs: 713 active+clean; 310 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 476 KiB/s rd, 1.3 MiB/s wr, 625 op/s 2024-09-14T04:12:04.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:03 smithi039 ceph-mon[111235]: osdmap e787: 8 total, 8 up, 8 in 2024-09-14T04:12:04.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:03 smithi039 ceph-mon[111235]: pgmap v830: 713 pgs: 713 active+clean; 310 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 476 KiB/s rd, 1.3 MiB/s wr, 625 op/s 2024-09-14T04:12:05.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:04 smithi129 ceph-mon[101316]: osdmap e788: 8 total, 8 up, 8 in 2024-09-14T04:12:05.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:04 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"} : dispatch 2024-09-14T04:12:05.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:04 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]': finished 2024-09-14T04:12:05.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:04 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/19/e824712f-1401-40ed-9e06-9b583af5b68a"} : dispatch 2024-09-14T04:12:05.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:04 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/e824712f-1401-40ed-9e06-9b583af5b68a"}]': finished 2024-09-14T04:12:05.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:04 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4780ba17-d092-460a-beb1-845141d27416"} : dispatch 2024-09-14T04:12:05.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:04 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4780ba17-d092-460a-beb1-845141d27416"}]': finished 2024-09-14T04:12:05.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:04 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/19/aee26f1a-39f1-4a7a-b2c0-4853ff14f6cc"} : dispatch 2024-09-14T04:12:05.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:04 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/aee26f1a-39f1-4a7a-b2c0-4853ff14f6cc"}]': finished 2024-09-14T04:12:05.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:04 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer/19/77264779-cf44-4869-9ec0-5f9bda2943c2"} : dispatch 2024-09-14T04:12:05.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:04 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' 2024-09-14T04:12:05.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:04 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer/19/77264779-cf44-4869-9ec0-5f9bda2943c2"} : dispatch 2024-09-14T04:12:05.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:04 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/19/77264779-cf44-4869-9ec0-5f9bda2943c2"} : dispatch 2024-09-14T04:12:05.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:04 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/77264779-cf44-4869-9ec0-5f9bda2943c2"}]': finished 2024-09-14T04:12:05.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:04 smithi039 ceph-mon[106470]: osdmap e788: 8 total, 8 up, 8 in 2024-09-14T04:12:05.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:04 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"} : dispatch 2024-09-14T04:12:05.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:04 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]': finished 2024-09-14T04:12:05.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:04 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/19/e824712f-1401-40ed-9e06-9b583af5b68a"} : dispatch 2024-09-14T04:12:05.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:04 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/e824712f-1401-40ed-9e06-9b583af5b68a"}]': finished 2024-09-14T04:12:05.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:04 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4780ba17-d092-460a-beb1-845141d27416"} : dispatch 2024-09-14T04:12:05.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:04 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4780ba17-d092-460a-beb1-845141d27416"}]': finished 2024-09-14T04:12:05.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:04 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/19/aee26f1a-39f1-4a7a-b2c0-4853ff14f6cc"} : dispatch 2024-09-14T04:12:05.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:04 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/aee26f1a-39f1-4a7a-b2c0-4853ff14f6cc"}]': finished 2024-09-14T04:12:05.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:04 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer/19/77264779-cf44-4869-9ec0-5f9bda2943c2"} : dispatch 2024-09-14T04:12:05.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:04 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' 2024-09-14T04:12:05.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:04 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer/19/77264779-cf44-4869-9ec0-5f9bda2943c2"} : dispatch 2024-09-14T04:12:05.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:04 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/19/77264779-cf44-4869-9ec0-5f9bda2943c2"} : dispatch 2024-09-14T04:12:05.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:04 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/77264779-cf44-4869-9ec0-5f9bda2943c2"}]': finished 2024-09-14T04:12:05.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:04 smithi039 ceph-mon[111235]: osdmap e788: 8 total, 8 up, 8 in 2024-09-14T04:12:05.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:04 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"} : dispatch 2024-09-14T04:12:05.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:04 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/uuid4"}]': finished 2024-09-14T04:12:05.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:04 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/19/e824712f-1401-40ed-9e06-9b583af5b68a"} : dispatch 2024-09-14T04:12:05.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:04 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/e824712f-1401-40ed-9e06-9b583af5b68a"}]': finished 2024-09-14T04:12:05.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:04 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4780ba17-d092-460a-beb1-845141d27416"} : dispatch 2024-09-14T04:12:05.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:04 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/4780ba17-d092-460a-beb1-845141d27416"}]': finished 2024-09-14T04:12:05.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:04 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/19/aee26f1a-39f1-4a7a-b2c0-4853ff14f6cc"} : dispatch 2024-09-14T04:12:05.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:04 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/aee26f1a-39f1-4a7a-b2c0-4853ff14f6cc"}]': finished 2024-09-14T04:12:05.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:04 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer/19/77264779-cf44-4869-9ec0-5f9bda2943c2"} : dispatch 2024-09-14T04:12:05.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:04 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' 2024-09-14T04:12:05.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:04 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd={"prefix": "config-key get", "key": "rbd/mirror/peer/19/77264779-cf44-4869-9ec0-5f9bda2943c2"} : dispatch 2024-09-14T04:12:05.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:04 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd={"prefix": "config-key rm", "key": "rbd/mirror/peer/19/77264779-cf44-4869-9ec0-5f9bda2943c2"} : dispatch 2024-09-14T04:12:05.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:04 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/438231588' entity='client.admin' cmd='[{"prefix": "config-key rm", "key": "rbd/mirror/peer/19/77264779-cf44-4869-9ec0-5f9bda2943c2"}]': finished 2024-09-14T04:12:06.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:05 smithi129 ceph-mon[101316]: osdmap e789: 8 total, 8 up, 8 in 2024-09-14T04:12:06.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:05 smithi129 ceph-mon[101316]: pgmap v833: 713 pgs: 713 active+clean; 312 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 336 KiB/s rd, 1.9 MiB/s wr, 450 op/s 2024-09-14T04:12:06.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:05 smithi039 ceph-mon[106470]: osdmap e789: 8 total, 8 up, 8 in 2024-09-14T04:12:06.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:05 smithi039 ceph-mon[106470]: pgmap v833: 713 pgs: 713 active+clean; 312 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 336 KiB/s rd, 1.9 MiB/s wr, 450 op/s 2024-09-14T04:12:06.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:05 smithi039 ceph-mon[111235]: osdmap e789: 8 total, 8 up, 8 in 2024-09-14T04:12:06.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:05 smithi039 ceph-mon[111235]: pgmap v833: 713 pgs: 713 active+clean; 312 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 336 KiB/s rd, 1.9 MiB/s wr, 450 op/s 2024-09-14T04:12:07.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:06 smithi129 ceph-mon[101316]: osdmap e790: 8 total, 8 up, 8 in 2024-09-14T04:12:07.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:06 smithi039 ceph-mon[106470]: osdmap e790: 8 total, 8 up, 8 in 2024-09-14T04:12:07.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:06 smithi039 ceph-mon[111235]: osdmap e790: 8 total, 8 up, 8 in 2024-09-14T04:12:08.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:07 smithi129 ceph-mon[101316]: osdmap e791: 8 total, 8 up, 8 in 2024-09-14T04:12:08.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:07 smithi129 ceph-mon[101316]: pgmap v836: 713 pgs: 713 active+clean; 313 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 146 KiB/s rd, 846 KiB/s wr, 196 op/s 2024-09-14T04:12:08.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:12:08.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.c", "id": [2, 4]} : dispatch 2024-09-14T04:12:08.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.11", "id": [3, 4]} : dispatch 2024-09-14T04:12:08.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.c", "id": [2, 4]} : dispatch 2024-09-14T04:12:08.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.1b", "id": [2, 5]} : dispatch 2024-09-14T04:12:08.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.a", "id": [4, 1]} : dispatch 2024-09-14T04:12:08.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.11", "id": [3, 4]} : dispatch 2024-09-14T04:12:08.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.2", "id": [6, 3]} : dispatch 2024-09-14T04:12:08.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.0", "id": [0, 6]} : dispatch 2024-09-14T04:12:08.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.1b", "id": [2, 5]} : dispatch 2024-09-14T04:12:08.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.3", "id": [0, 3]} : dispatch 2024-09-14T04:12:08.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.1f", "id": [0, 3]} : dispatch 2024-09-14T04:12:08.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.a", "id": [4, 1]} : dispatch 2024-09-14T04:12:08.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.e", "id": [0, 1]} : dispatch 2024-09-14T04:12:08.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.2", "id": [6, 3]} : dispatch 2024-09-14T04:12:08.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.18", "id": [7, 1]} : dispatch 2024-09-14T04:12:08.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.0", "id": [0, 6]} : dispatch 2024-09-14T04:12:08.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.3", "id": [0, 3]} : dispatch 2024-09-14T04:12:08.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.1f", "id": [0, 3]} : dispatch 2024-09-14T04:12:08.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.e", "id": [0, 1]} : dispatch 2024-09-14T04:12:08.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.18", "id": [7, 1]} : dispatch 2024-09-14T04:12:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[106470]: osdmap e791: 8 total, 8 up, 8 in 2024-09-14T04:12:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[106470]: pgmap v836: 713 pgs: 713 active+clean; 313 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 146 KiB/s rd, 846 KiB/s wr, 196 op/s 2024-09-14T04:12:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:12:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.c", "id": [2, 4]} : dispatch 2024-09-14T04:12:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.11", "id": [3, 4]} : dispatch 2024-09-14T04:12:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.c", "id": [2, 4]} : dispatch 2024-09-14T04:12:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.1b", "id": [2, 5]} : dispatch 2024-09-14T04:12:08.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.a", "id": [4, 1]} : dispatch 2024-09-14T04:12:08.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.11", "id": [3, 4]} : dispatch 2024-09-14T04:12:08.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.2", "id": [6, 3]} : dispatch 2024-09-14T04:12:08.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.0", "id": [0, 6]} : dispatch 2024-09-14T04:12:08.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.1b", "id": [2, 5]} : dispatch 2024-09-14T04:12:08.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.3", "id": [0, 3]} : dispatch 2024-09-14T04:12:08.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.1f", "id": [0, 3]} : dispatch 2024-09-14T04:12:08.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.a", "id": [4, 1]} : dispatch 2024-09-14T04:12:08.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.e", "id": [0, 1]} : dispatch 2024-09-14T04:12:08.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.2", "id": [6, 3]} : dispatch 2024-09-14T04:12:08.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.18", "id": [7, 1]} : dispatch 2024-09-14T04:12:08.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.0", "id": [0, 6]} : dispatch 2024-09-14T04:12:08.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.3", "id": [0, 3]} : dispatch 2024-09-14T04:12:08.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.1f", "id": [0, 3]} : dispatch 2024-09-14T04:12:08.293 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.e", "id": [0, 1]} : dispatch 2024-09-14T04:12:08.293 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.18", "id": [7, 1]} : dispatch 2024-09-14T04:12:08.293 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[111235]: osdmap e791: 8 total, 8 up, 8 in 2024-09-14T04:12:08.294 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[111235]: pgmap v836: 713 pgs: 713 active+clean; 313 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 146 KiB/s rd, 846 KiB/s wr, 196 op/s 2024-09-14T04:12:08.294 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:12:08.295 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.c", "id": [2, 4]} : dispatch 2024-09-14T04:12:08.295 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.11", "id": [3, 4]} : dispatch 2024-09-14T04:12:08.295 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.c", "id": [2, 4]} : dispatch 2024-09-14T04:12:08.295 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.1b", "id": [2, 5]} : dispatch 2024-09-14T04:12:08.295 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.a", "id": [4, 1]} : dispatch 2024-09-14T04:12:08.296 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.11", "id": [3, 4]} : dispatch 2024-09-14T04:12:08.296 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.2", "id": [6, 3]} : dispatch 2024-09-14T04:12:08.296 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.0", "id": [0, 6]} : dispatch 2024-09-14T04:12:08.296 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.1b", "id": [2, 5]} : dispatch 2024-09-14T04:12:08.296 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.3", "id": [0, 3]} : dispatch 2024-09-14T04:12:08.296 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.1f", "id": [0, 3]} : dispatch 2024-09-14T04:12:08.296 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.a", "id": [4, 1]} : dispatch 2024-09-14T04:12:08.296 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.e", "id": [0, 1]} : dispatch 2024-09-14T04:12:08.297 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.2", "id": [6, 3]} : dispatch 2024-09-14T04:12:08.297 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.18", "id": [7, 1]} : dispatch 2024-09-14T04:12:08.297 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.0", "id": [0, 6]} : dispatch 2024-09-14T04:12:08.297 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.3", "id": [0, 3]} : dispatch 2024-09-14T04:12:08.297 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.1f", "id": [0, 3]} : dispatch 2024-09-14T04:12:08.297 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.e", "id": [0, 1]} : dispatch 2024-09-14T04:12:08.297 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.18", "id": [7, 1]} : dispatch 2024-09-14T04:12:08.965 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:12:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:12:08] "GET /metrics HTTP/1.1" 200 82357 "" "Prometheus/2.51.0" 2024-09-14T04:12:09.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:08 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.c", "id": [2, 4]}]': finished 2024-09-14T04:12:09.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:08 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.11", "id": [3, 4]}]': finished 2024-09-14T04:12:09.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:08 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.1b", "id": [2, 5]}]': finished 2024-09-14T04:12:09.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:08 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.a", "id": [4, 1]}]': finished 2024-09-14T04:12:09.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:08 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.2", "id": [6, 3]}]': finished 2024-09-14T04:12:09.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:08 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.0", "id": [0, 6]}]': finished 2024-09-14T04:12:09.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:08 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.3", "id": [0, 3]}]': finished 2024-09-14T04:12:09.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:08 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.1f", "id": [0, 3]}]': finished 2024-09-14T04:12:09.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:08 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.e", "id": [0, 1]}]': finished 2024-09-14T04:12:09.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:08 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.18", "id": [7, 1]}]': finished 2024-09-14T04:12:09.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:08 smithi129 ceph-mon[101316]: osdmap e792: 8 total, 8 up, 8 in 2024-09-14T04:12:09.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.c", "id": [2, 4]}]': finished 2024-09-14T04:12:09.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.11", "id": [3, 4]}]': finished 2024-09-14T04:12:09.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.1b", "id": [2, 5]}]': finished 2024-09-14T04:12:09.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.a", "id": [4, 1]}]': finished 2024-09-14T04:12:09.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.2", "id": [6, 3]}]': finished 2024-09-14T04:12:09.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.0", "id": [0, 6]}]': finished 2024-09-14T04:12:09.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.3", "id": [0, 3]}]': finished 2024-09-14T04:12:09.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.1f", "id": [0, 3]}]': finished 2024-09-14T04:12:09.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.e", "id": [0, 1]}]': finished 2024-09-14T04:12:09.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.18", "id": [7, 1]}]': finished 2024-09-14T04:12:09.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:08 smithi039 ceph-mon[106470]: osdmap e792: 8 total, 8 up, 8 in 2024-09-14T04:12:09.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:08 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.c", "id": [2, 4]}]': finished 2024-09-14T04:12:09.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:08 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.11", "id": [3, 4]}]': finished 2024-09-14T04:12:09.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:08 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "33.1b", "id": [2, 5]}]': finished 2024-09-14T04:12:09.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:08 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "35.a", "id": [4, 1]}]': finished 2024-09-14T04:12:09.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:08 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "36.2", "id": [6, 3]}]': finished 2024-09-14T04:12:09.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:08 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.0", "id": [0, 6]}]': finished 2024-09-14T04:12:09.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:08 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.3", "id": [0, 3]}]': finished 2024-09-14T04:12:09.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:08 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "37.1f", "id": [0, 3]}]': finished 2024-09-14T04:12:09.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:08 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.e", "id": [0, 1]}]': finished 2024-09-14T04:12:09.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:08 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "38.18", "id": [7, 1]}]': finished 2024-09-14T04:12:09.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:08 smithi039 ceph-mon[111235]: osdmap e792: 8 total, 8 up, 8 in 2024-09-14T04:12:10.017 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.CreateWithMirrorEnabled (6152 ms) 2024-09-14T04:12:10.018 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.FlushCacheWithCopyupOnExternalSnapshot 2024-09-14T04:12:10.018 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:10.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:09 smithi129 ceph-mon[101316]: osdmap e793: 8 total, 8 up, 8 in 2024-09-14T04:12:10.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:09 smithi129 ceph-mon[101316]: pgmap v839: 713 pgs: 713 active+clean; 313 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 167 KiB/s rd, 271 KiB/s wr, 214 op/s 2024-09-14T04:12:10.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:09 smithi039 ceph-mon[106470]: osdmap e793: 8 total, 8 up, 8 in 2024-09-14T04:12:10.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:09 smithi039 ceph-mon[106470]: pgmap v839: 713 pgs: 713 active+clean; 313 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 167 KiB/s rd, 271 KiB/s wr, 214 op/s 2024-09-14T04:12:10.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:09 smithi039 ceph-mon[111235]: osdmap e793: 8 total, 8 up, 8 in 2024-09-14T04:12:10.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:09 smithi039 ceph-mon[111235]: pgmap v839: 713 pgs: 713 active+clean; 313 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 167 KiB/s rd, 271 KiB/s wr, 214 op/s 2024-09-14T04:12:11.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:10 smithi129 ceph-mon[101316]: osdmap e794: 8 total, 8 up, 8 in 2024-09-14T04:12:11.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:10 smithi039 ceph-mon[106470]: osdmap e794: 8 total, 8 up, 8 in 2024-09-14T04:12:11.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:10 smithi039 ceph-mon[111235]: osdmap e794: 8 total, 8 up, 8 in 2024-09-14T04:12:12.027 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.FlushCacheWithCopyupOnExternalSnapshot (2009 ms) 2024-09-14T04:12:12.027 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.ExclusiveLock 2024-09-14T04:12:12.027 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:12.080 INFO:tasks.workunit.client.0.smithi039.stdout:1: acquiring exclusive lock 2024-09-14T04:12:12.080 INFO:tasks.workunit.client.0.smithi039.stdout:2: acquiring exclusive lock 2024-09-14T04:12:12.083 INFO:tasks.workunit.client.0.smithi039.stdout:2: exclusive lock acquired 2024-09-14T04:12:12.131 INFO:tasks.workunit.client.0.smithi039.stdout:2: releasing exclusive lock 2024-09-14T04:12:12.134 INFO:tasks.workunit.client.0.smithi039.stdout:2: exclusive lock released 2024-09-14T04:12:12.134 INFO:tasks.workunit.client.0.smithi039.stdout:2: acquiring exclusive lock 2024-09-14T04:12:12.138 INFO:tasks.workunit.client.0.smithi039.stdout:1: exclusive lock acquired 2024-09-14T04:12:12.164 INFO:tasks.workunit.client.0.smithi039.stdout:1: releasing exclusive lock 2024-09-14T04:12:12.167 INFO:tasks.workunit.client.0.smithi039.stdout:1: exclusive lock released 2024-09-14T04:12:12.167 INFO:tasks.workunit.client.0.smithi039.stdout:1: acquiring exclusive lock 2024-09-14T04:12:12.171 INFO:tasks.workunit.client.0.smithi039.stdout:2: exclusive lock acquired 2024-09-14T04:12:12.178 INFO:tasks.workunit.client.0.smithi039.stdout:2: releasing exclusive lock 2024-09-14T04:12:12.182 INFO:tasks.workunit.client.0.smithi039.stdout:2: exclusive lock released 2024-09-14T04:12:12.182 INFO:tasks.workunit.client.0.smithi039.stdout:2: acquiring exclusive lock 2024-09-14T04:12:12.185 INFO:tasks.workunit.client.0.smithi039.stdout:2: exclusive lock acquired 2024-09-14T04:12:12.225 INFO:tasks.workunit.client.0.smithi039.stdout:2: releasing exclusive lock 2024-09-14T04:12:12.227 INFO:tasks.workunit.client.0.smithi039.stdout:2: exclusive lock released 2024-09-14T04:12:12.227 INFO:tasks.workunit.client.0.smithi039.stdout:2: acquiring exclusive lock 2024-09-14T04:12:12.231 INFO:tasks.workunit.client.0.smithi039.stdout:1: exclusive lock acquired 2024-09-14T04:12:12.237 INFO:tasks.workunit.client.0.smithi039.stdout:1: releasing exclusive lock 2024-09-14T04:12:12.240 INFO:tasks.workunit.client.0.smithi039.stdout:1: exclusive lock released 2024-09-14T04:12:12.240 INFO:tasks.workunit.client.0.smithi039.stdout:1: acquiring exclusive lock 2024-09-14T04:12:12.243 INFO:tasks.workunit.client.0.smithi039.stdout:2: exclusive lock acquired 2024-09-14T04:12:12.256 INFO:tasks.workunit.client.0.smithi039.stdout:2: releasing exclusive lock 2024-09-14T04:12:12.258 INFO:tasks.workunit.client.0.smithi039.stdout:2: exclusive lock released 2024-09-14T04:12:12.258 INFO:tasks.workunit.client.0.smithi039.stdout:2: acquiring exclusive lock 2024-09-14T04:12:12.262 INFO:tasks.workunit.client.0.smithi039.stdout:1: exclusive lock acquired 2024-09-14T04:12:12.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:11 smithi129 ceph-mon[101316]: osdmap e795: 8 total, 8 up, 8 in 2024-09-14T04:12:12.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:11 smithi129 ceph-mon[101316]: pgmap v842: 713 pgs: 713 active+clean; 313 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 12 KiB/s wr, 175 op/s; 0 B/s, 0 objects/s recovering 2024-09-14T04:12:12.282 INFO:tasks.workunit.client.0.smithi039.stdout:1: releasing exclusive lock 2024-09-14T04:12:12.285 INFO:tasks.workunit.client.0.smithi039.stdout:1: exclusive lock released 2024-09-14T04:12:12.285 INFO:tasks.workunit.client.0.smithi039.stdout:1: acquiring exclusive lock 2024-09-14T04:12:12.289 INFO:tasks.workunit.client.0.smithi039.stdout:2: exclusive lock acquired 2024-09-14T04:12:12.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:11 smithi039 ceph-mon[106470]: osdmap e795: 8 total, 8 up, 8 in 2024-09-14T04:12:12.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:11 smithi039 ceph-mon[106470]: pgmap v842: 713 pgs: 713 active+clean; 313 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 12 KiB/s wr, 175 op/s; 0 B/s, 0 objects/s recovering 2024-09-14T04:12:12.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:11 smithi039 ceph-mon[111235]: osdmap e795: 8 total, 8 up, 8 in 2024-09-14T04:12:12.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:11 smithi039 ceph-mon[111235]: pgmap v842: 713 pgs: 713 active+clean; 313 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 134 KiB/s rd, 12 KiB/s wr, 175 op/s; 0 B/s, 0 objects/s recovering 2024-09-14T04:12:12.305 INFO:tasks.workunit.client.0.smithi039.stdout:2: releasing exclusive lock 2024-09-14T04:12:12.308 INFO:tasks.workunit.client.0.smithi039.stdout:2: exclusive lock released 2024-09-14T04:12:12.311 INFO:tasks.workunit.client.0.smithi039.stdout:1: exclusive lock acquired 2024-09-14T04:12:12.348 INFO:tasks.workunit.client.0.smithi039.stdout:1: releasing exclusive lock 2024-09-14T04:12:12.352 INFO:tasks.workunit.client.0.smithi039.stdout:1: exclusive lock released 2024-09-14T04:12:12.352 INFO:tasks.workunit.client.0.smithi039.stdout:1: acquiring exclusive lock 2024-09-14T04:12:12.355 INFO:tasks.workunit.client.0.smithi039.stdout:1: exclusive lock acquired 2024-09-14T04:12:12.375 INFO:tasks.workunit.client.0.smithi039.stdout:1: releasing exclusive lock 2024-09-14T04:12:12.378 INFO:tasks.workunit.client.0.smithi039.stdout:1: exclusive lock released 2024-09-14T04:12:12.392 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.ExclusiveLock (366 ms) 2024-09-14T04:12:12.392 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.BreakLock 2024-09-14T04:12:12.409 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:13.035 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.BreakLock (642 ms) 2024-09-14T04:12:13.035 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.DiscardAfterWrite 2024-09-14T04:12:13.035 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:13.058 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.DiscardAfterWrite (24 ms) 2024-09-14T04:12:13.058 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.DefaultFeatures 2024-09-14T04:12:13.058 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.DefaultFeatures (0 ms) 2024-09-14T04:12:13.059 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestTrashMoveAndPurge 2024-09-14T04:12:13.059 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:13.119 INFO:tasks.workunit.client.0.smithi039.stdout:0.00% done 2024-09-14T04:12:13.119 INFO:tasks.workunit.client.0.smithi039.stdout:1.56% done 2024-09-14T04:12:13.119 INFO:tasks.workunit.client.0.smithi039.stdout:3.12% done 2024-09-14T04:12:13.119 INFO:tasks.workunit.client.0.smithi039.stdout:4.69% done 2024-09-14T04:12:13.119 INFO:tasks.workunit.client.0.smithi039.stdout:6.25% done 2024-09-14T04:12:13.119 INFO:tasks.workunit.client.0.smithi039.stdout:7.81% done 2024-09-14T04:12:13.120 INFO:tasks.workunit.client.0.smithi039.stdout:9.38% done 2024-09-14T04:12:13.120 INFO:tasks.workunit.client.0.smithi039.stdout:10.94% done 2024-09-14T04:12:13.120 INFO:tasks.workunit.client.0.smithi039.stdout:12.50% done 2024-09-14T04:12:13.120 INFO:tasks.workunit.client.0.smithi039.stdout:14.06% done 2024-09-14T04:12:13.120 INFO:tasks.workunit.client.0.smithi039.stdout:15.62% done 2024-09-14T04:12:13.120 INFO:tasks.workunit.client.0.smithi039.stdout:17.19% done 2024-09-14T04:12:13.120 INFO:tasks.workunit.client.0.smithi039.stdout:18.75% done 2024-09-14T04:12:13.120 INFO:tasks.workunit.client.0.smithi039.stdout:20.31% done 2024-09-14T04:12:13.120 INFO:tasks.workunit.client.0.smithi039.stdout:21.88% done 2024-09-14T04:12:13.120 INFO:tasks.workunit.client.0.smithi039.stdout:23.44% done 2024-09-14T04:12:13.121 INFO:tasks.workunit.client.0.smithi039.stdout:25.00% done 2024-09-14T04:12:13.121 INFO:tasks.workunit.client.0.smithi039.stdout:26.56% done 2024-09-14T04:12:13.121 INFO:tasks.workunit.client.0.smithi039.stdout:28.12% done 2024-09-14T04:12:13.121 INFO:tasks.workunit.client.0.smithi039.stdout:29.69% done 2024-09-14T04:12:13.121 INFO:tasks.workunit.client.0.smithi039.stdout:31.25% done 2024-09-14T04:12:13.121 INFO:tasks.workunit.client.0.smithi039.stdout:32.81% done 2024-09-14T04:12:13.121 INFO:tasks.workunit.client.0.smithi039.stdout:34.38% done 2024-09-14T04:12:13.121 INFO:tasks.workunit.client.0.smithi039.stdout:35.94% done 2024-09-14T04:12:13.121 INFO:tasks.workunit.client.0.smithi039.stdout:37.50% done 2024-09-14T04:12:13.121 INFO:tasks.workunit.client.0.smithi039.stdout:39.06% done 2024-09-14T04:12:13.121 INFO:tasks.workunit.client.0.smithi039.stdout:40.62% done 2024-09-14T04:12:13.122 INFO:tasks.workunit.client.0.smithi039.stdout:42.19% done 2024-09-14T04:12:13.122 INFO:tasks.workunit.client.0.smithi039.stdout:43.75% done 2024-09-14T04:12:13.122 INFO:tasks.workunit.client.0.smithi039.stdout:45.31% done 2024-09-14T04:12:13.122 INFO:tasks.workunit.client.0.smithi039.stdout:46.88% done 2024-09-14T04:12:13.122 INFO:tasks.workunit.client.0.smithi039.stdout:48.44% done 2024-09-14T04:12:13.122 INFO:tasks.workunit.client.0.smithi039.stdout:50.00% done 2024-09-14T04:12:13.122 INFO:tasks.workunit.client.0.smithi039.stdout:51.56% done 2024-09-14T04:12:13.122 INFO:tasks.workunit.client.0.smithi039.stdout:53.12% done 2024-09-14T04:12:13.122 INFO:tasks.workunit.client.0.smithi039.stdout:54.69% done 2024-09-14T04:12:13.122 INFO:tasks.workunit.client.0.smithi039.stdout:56.25% done 2024-09-14T04:12:13.122 INFO:tasks.workunit.client.0.smithi039.stdout:57.81% done 2024-09-14T04:12:13.122 INFO:tasks.workunit.client.0.smithi039.stdout:59.38% done 2024-09-14T04:12:13.122 INFO:tasks.workunit.client.0.smithi039.stdout:60.94% done 2024-09-14T04:12:13.122 INFO:tasks.workunit.client.0.smithi039.stdout:62.50% done 2024-09-14T04:12:13.123 INFO:tasks.workunit.client.0.smithi039.stdout:64.06% done 2024-09-14T04:12:13.123 INFO:tasks.workunit.client.0.smithi039.stdout:65.62% done 2024-09-14T04:12:13.123 INFO:tasks.workunit.client.0.smithi039.stdout:67.19% done 2024-09-14T04:12:13.123 INFO:tasks.workunit.client.0.smithi039.stdout:68.75% done 2024-09-14T04:12:13.123 INFO:tasks.workunit.client.0.smithi039.stdout:70.31% done 2024-09-14T04:12:13.123 INFO:tasks.workunit.client.0.smithi039.stdout:71.88% done 2024-09-14T04:12:13.123 INFO:tasks.workunit.client.0.smithi039.stdout:73.44% done 2024-09-14T04:12:13.123 INFO:tasks.workunit.client.0.smithi039.stdout:75.00% done 2024-09-14T04:12:13.123 INFO:tasks.workunit.client.0.smithi039.stdout:76.56% done 2024-09-14T04:12:13.123 INFO:tasks.workunit.client.0.smithi039.stdout:78.12% done 2024-09-14T04:12:13.123 INFO:tasks.workunit.client.0.smithi039.stdout:79.69% done 2024-09-14T04:12:13.123 INFO:tasks.workunit.client.0.smithi039.stdout:81.25% done 2024-09-14T04:12:13.123 INFO:tasks.workunit.client.0.smithi039.stdout:82.81% done 2024-09-14T04:12:13.123 INFO:tasks.workunit.client.0.smithi039.stdout:84.38% done 2024-09-14T04:12:13.124 INFO:tasks.workunit.client.0.smithi039.stdout:85.94% done 2024-09-14T04:12:13.124 INFO:tasks.workunit.client.0.smithi039.stdout:87.50% done 2024-09-14T04:12:13.124 INFO:tasks.workunit.client.0.smithi039.stdout:89.06% done 2024-09-14T04:12:13.124 INFO:tasks.workunit.client.0.smithi039.stdout:90.62% done 2024-09-14T04:12:13.124 INFO:tasks.workunit.client.0.smithi039.stdout:92.19% done 2024-09-14T04:12:13.124 INFO:tasks.workunit.client.0.smithi039.stdout:93.75% done 2024-09-14T04:12:13.124 INFO:tasks.workunit.client.0.smithi039.stdout:95.31% done 2024-09-14T04:12:13.124 INFO:tasks.workunit.client.0.smithi039.stdout:96.88% done 2024-09-14T04:12:13.124 INFO:tasks.workunit.client.0.smithi039.stdout:98.44% done 2024-09-14T04:12:13.124 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestTrashMoveAndPurge (60 ms) 2024-09-14T04:12:13.124 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestTrashMoveAndPurgeNonExpiredDelay 2024-09-14T04:12:13.124 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:13.181 INFO:tasks.workunit.client.0.smithi039.stdout:0.00% done 2024-09-14T04:12:13.182 INFO:tasks.workunit.client.0.smithi039.stdout:1.56% done 2024-09-14T04:12:13.182 INFO:tasks.workunit.client.0.smithi039.stdout:3.12% done 2024-09-14T04:12:13.182 INFO:tasks.workunit.client.0.smithi039.stdout:4.69% done 2024-09-14T04:12:13.182 INFO:tasks.workunit.client.0.smithi039.stdout:6.25% done 2024-09-14T04:12:13.182 INFO:tasks.workunit.client.0.smithi039.stdout:7.81% done 2024-09-14T04:12:13.182 INFO:tasks.workunit.client.0.smithi039.stdout:9.38% done 2024-09-14T04:12:13.182 INFO:tasks.workunit.client.0.smithi039.stdout:10.94% done 2024-09-14T04:12:13.182 INFO:tasks.workunit.client.0.smithi039.stdout:12.50% done 2024-09-14T04:12:13.183 INFO:tasks.workunit.client.0.smithi039.stdout:14.06% done 2024-09-14T04:12:13.183 INFO:tasks.workunit.client.0.smithi039.stdout:15.62% done 2024-09-14T04:12:13.183 INFO:tasks.workunit.client.0.smithi039.stdout:17.19% done 2024-09-14T04:12:13.183 INFO:tasks.workunit.client.0.smithi039.stdout:18.75% done 2024-09-14T04:12:13.183 INFO:tasks.workunit.client.0.smithi039.stdout:20.31% done 2024-09-14T04:12:13.183 INFO:tasks.workunit.client.0.smithi039.stdout:21.88% done 2024-09-14T04:12:13.184 INFO:tasks.workunit.client.0.smithi039.stdout:23.44% done 2024-09-14T04:12:13.184 INFO:tasks.workunit.client.0.smithi039.stdout:25.00% done 2024-09-14T04:12:13.184 INFO:tasks.workunit.client.0.smithi039.stdout:26.56% done 2024-09-14T04:12:13.184 INFO:tasks.workunit.client.0.smithi039.stdout:28.12% done 2024-09-14T04:12:13.184 INFO:tasks.workunit.client.0.smithi039.stdout:29.69% done 2024-09-14T04:12:13.184 INFO:tasks.workunit.client.0.smithi039.stdout:31.25% done 2024-09-14T04:12:13.185 INFO:tasks.workunit.client.0.smithi039.stdout:32.81% done 2024-09-14T04:12:13.185 INFO:tasks.workunit.client.0.smithi039.stdout:34.38% done 2024-09-14T04:12:13.185 INFO:tasks.workunit.client.0.smithi039.stdout:35.94% done 2024-09-14T04:12:13.185 INFO:tasks.workunit.client.0.smithi039.stdout:37.50% done 2024-09-14T04:12:13.185 INFO:tasks.workunit.client.0.smithi039.stdout:39.06% done 2024-09-14T04:12:13.185 INFO:tasks.workunit.client.0.smithi039.stdout:40.62% done 2024-09-14T04:12:13.185 INFO:tasks.workunit.client.0.smithi039.stdout:42.19% done 2024-09-14T04:12:13.186 INFO:tasks.workunit.client.0.smithi039.stdout:43.75% done 2024-09-14T04:12:13.186 INFO:tasks.workunit.client.0.smithi039.stdout:45.31% done 2024-09-14T04:12:13.186 INFO:tasks.workunit.client.0.smithi039.stdout:46.88% done 2024-09-14T04:12:13.186 INFO:tasks.workunit.client.0.smithi039.stdout:48.44% done 2024-09-14T04:12:13.186 INFO:tasks.workunit.client.0.smithi039.stdout:50.00% done 2024-09-14T04:12:13.186 INFO:tasks.workunit.client.0.smithi039.stdout:51.56% done 2024-09-14T04:12:13.186 INFO:tasks.workunit.client.0.smithi039.stdout:53.12% done 2024-09-14T04:12:13.186 INFO:tasks.workunit.client.0.smithi039.stdout:54.69% done 2024-09-14T04:12:13.187 INFO:tasks.workunit.client.0.smithi039.stdout:56.25% done 2024-09-14T04:12:13.187 INFO:tasks.workunit.client.0.smithi039.stdout:57.81% done 2024-09-14T04:12:13.187 INFO:tasks.workunit.client.0.smithi039.stdout:59.38% done 2024-09-14T04:12:13.187 INFO:tasks.workunit.client.0.smithi039.stdout:60.94% done 2024-09-14T04:12:13.187 INFO:tasks.workunit.client.0.smithi039.stdout:62.50% done 2024-09-14T04:12:13.187 INFO:tasks.workunit.client.0.smithi039.stdout:64.06% done 2024-09-14T04:12:13.187 INFO:tasks.workunit.client.0.smithi039.stdout:65.62% done 2024-09-14T04:12:13.187 INFO:tasks.workunit.client.0.smithi039.stdout:67.19% done 2024-09-14T04:12:13.188 INFO:tasks.workunit.client.0.smithi039.stdout:68.75% done 2024-09-14T04:12:13.188 INFO:tasks.workunit.client.0.smithi039.stdout:70.31% done 2024-09-14T04:12:13.188 INFO:tasks.workunit.client.0.smithi039.stdout:71.88% done 2024-09-14T04:12:13.188 INFO:tasks.workunit.client.0.smithi039.stdout:73.44% done 2024-09-14T04:12:13.188 INFO:tasks.workunit.client.0.smithi039.stdout:75.00% done 2024-09-14T04:12:13.188 INFO:tasks.workunit.client.0.smithi039.stdout:76.56% done 2024-09-14T04:12:13.188 INFO:tasks.workunit.client.0.smithi039.stdout:78.12% done 2024-09-14T04:12:13.189 INFO:tasks.workunit.client.0.smithi039.stdout:79.69% done 2024-09-14T04:12:13.189 INFO:tasks.workunit.client.0.smithi039.stdout:81.25% done 2024-09-14T04:12:13.189 INFO:tasks.workunit.client.0.smithi039.stdout:82.81% done 2024-09-14T04:12:13.189 INFO:tasks.workunit.client.0.smithi039.stdout:84.38% done 2024-09-14T04:12:13.189 INFO:tasks.workunit.client.0.smithi039.stdout:85.94% done 2024-09-14T04:12:13.189 INFO:tasks.workunit.client.0.smithi039.stdout:87.50% done 2024-09-14T04:12:13.190 INFO:tasks.workunit.client.0.smithi039.stdout:89.06% done 2024-09-14T04:12:13.190 INFO:tasks.workunit.client.0.smithi039.stdout:90.62% done 2024-09-14T04:12:13.190 INFO:tasks.workunit.client.0.smithi039.stdout:92.19% done 2024-09-14T04:12:13.190 INFO:tasks.workunit.client.0.smithi039.stdout:93.75% done 2024-09-14T04:12:13.190 INFO:tasks.workunit.client.0.smithi039.stdout:95.31% done 2024-09-14T04:12:13.190 INFO:tasks.workunit.client.0.smithi039.stdout:96.88% done 2024-09-14T04:12:13.190 INFO:tasks.workunit.client.0.smithi039.stdout:98.44% done 2024-09-14T04:12:13.190 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestTrashMoveAndPurgeNonExpiredDelay (63 ms) 2024-09-14T04:12:13.191 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestTrashPurge 2024-09-14T04:12:13.191 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:13.192 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:13.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:13 smithi129 ceph-mon[101316]: osdmap e796: 8 total, 8 up, 8 in 2024-09-14T04:12:13.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:13 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.39:0/1864073859"} : dispatch 2024-09-14T04:12:13.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:13 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.39:0/1864073859"} : dispatch 2024-09-14T04:12:13.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:13 smithi039 ceph-mon[111235]: osdmap e796: 8 total, 8 up, 8 in 2024-09-14T04:12:13.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:13 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.39:0/1864073859"} : dispatch 2024-09-14T04:12:13.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:13 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.39:0/1864073859"} : dispatch 2024-09-14T04:12:13.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:13 smithi039 ceph-mon[106470]: osdmap e796: 8 total, 8 up, 8 in 2024-09-14T04:12:13.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:13 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/263347546' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.39:0/1864073859"} : dispatch 2024-09-14T04:12:13.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:13 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.39:0/1864073859"} : dispatch 2024-09-14T04:12:13.314 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestTrashPurge (132 ms) 2024-09-14T04:12:13.314 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestTrashMoveAndRestore 2024-09-14T04:12:13.314 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:13.365 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestTrashMoveAndRestore (52 ms) 2024-09-14T04:12:13.366 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestListWatchers 2024-09-14T04:12:13.366 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:13.400 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestListWatchers (34 ms) 2024-09-14T04:12:13.400 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestSetSnapById 2024-09-14T04:12:13.400 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:14.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:14 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.39:0/1864073859"}]': finished 2024-09-14T04:12:14.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:14 smithi039 ceph-mon[106470]: osdmap e797: 8 total, 8 up, 8 in 2024-09-14T04:12:14.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:14 smithi039 ceph-mon[106470]: pgmap v845: 713 pgs: 713 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 243 KiB/s rd, 74 KiB/s wr, 259 op/s; 0 B/s, 0 objects/s recovering 2024-09-14T04:12:14.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:14 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:12:14.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:14 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.39:0/1864073859"}]': finished 2024-09-14T04:12:14.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:14 smithi039 ceph-mon[111235]: osdmap e797: 8 total, 8 up, 8 in 2024-09-14T04:12:14.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:14 smithi039 ceph-mon[111235]: pgmap v845: 713 pgs: 713 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 243 KiB/s rd, 74 KiB/s wr, 259 op/s; 0 B/s, 0 objects/s recovering 2024-09-14T04:12:14.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:14 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:12:14.067 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestSetSnapById (667 ms) 2024-09-14T04:12:14.067 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.Namespaces 2024-09-14T04:12:14.093 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.Namespaces (27 ms) 2024-09-14T04:12:14.093 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.NamespacesPP 2024-09-14T04:12:14.125 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:14.130 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:14.206 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.NamespacesPP (113 ms) 2024-09-14T04:12:14.206 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.Migration 2024-09-14T04:12:14.206 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:14.207 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:14.524 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:14 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd blocklist", "blocklistop": "add", "addr": "172.21.15.39:0/1864073859"}]': finished 2024-09-14T04:12:14.524 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:14 smithi129 ceph-mon[101316]: osdmap e797: 8 total, 8 up, 8 in 2024-09-14T04:12:14.524 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:14 smithi129 ceph-mon[101316]: pgmap v845: 713 pgs: 713 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 243 KiB/s rd, 74 KiB/s wr, 259 op/s; 0 B/s, 0 objects/s recovering 2024-09-14T04:12:14.524 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:14 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:12:14.661 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.Migration (455 ms) 2024-09-14T04:12:14.662 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.MigrationPP 2024-09-14T04:12:14.662 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:14.662 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:15.077 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.MigrationPP (415 ms) 2024-09-14T04:12:15.077 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestGetAccessTimestamp 2024-09-14T04:12:15.077 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:15.095 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestGetAccessTimestamp (18 ms) 2024-09-14T04:12:15.095 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.TestGetModifyTimestamp 2024-09-14T04:12:15.095 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:15.123 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.TestGetModifyTimestamp (28 ms) 2024-09-14T04:12:15.123 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.ZeroOverlapFlatten 2024-09-14T04:12:15.123 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:15.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:15 smithi129 ceph-mon[101316]: osdmap e798: 8 total, 8 up, 8 in 2024-09-14T04:12:15.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:15 smithi039 ceph-mon[106470]: osdmap e798: 8 total, 8 up, 8 in 2024-09-14T04:12:15.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:15 smithi039 ceph-mon[111235]: osdmap e798: 8 total, 8 up, 8 in 2024-09-14T04:12:15.632 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.ZeroOverlapFlatten (509 ms) 2024-09-14T04:12:15.632 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.PoolMetadata 2024-09-14T04:12:15.665 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.PoolMetadata (34 ms) 2024-09-14T04:12:15.665 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.PoolMetadataPP 2024-09-14T04:12:15.683 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.PoolMetadataPP (17 ms) 2024-09-14T04:12:15.683 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.Config 2024-09-14T04:12:15.689 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:15.727 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.Config (44 ms) 2024-09-14T04:12:15.727 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.ConfigPP 2024-09-14T04:12:15.731 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:15.772 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.ConfigPP (45 ms) 2024-09-14T04:12:15.772 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.PoolStatsPP 2024-09-14T04:12:16.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:12:15 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:12:15] ENGINE Bus STOPPING 2024-09-14T04:12:16.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:12:15 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:12:15] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T04:12:16.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:12:15 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:12:15] ENGINE Bus STOPPED 2024-09-14T04:12:16.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:12:15 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:12:15] ENGINE Bus STARTING 2024-09-14T04:12:16.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:12:15 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:12:15] ENGINE Serving on http://:::9283 2024-09-14T04:12:16.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:12:15 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:04:12:15] ENGINE Bus STARTED 2024-09-14T04:12:16.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:16 smithi039 ceph-mon[106470]: pgmap v847: 713 pgs: 713 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 316 KiB/s rd, 141 KiB/s wr, 372 op/s 2024-09-14T04:12:16.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:16 smithi039 ceph-mon[106470]: osdmap e799: 8 total, 8 up, 8 in 2024-09-14T04:12:16.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:16 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' 2024-09-14T04:12:16.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:16 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-14T04:12:16.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:16 smithi039 ceph-mon[111235]: pgmap v847: 713 pgs: 713 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 316 KiB/s rd, 141 KiB/s wr, 372 op/s 2024-09-14T04:12:16.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:16 smithi039 ceph-mon[111235]: osdmap e799: 8 total, 8 up, 8 in 2024-09-14T04:12:16.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:16 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' 2024-09-14T04:12:16.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:16 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-14T04:12:16.321 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:16 smithi129 ceph-mon[101316]: pgmap v847: 713 pgs: 713 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 316 KiB/s rd, 141 KiB/s wr, 372 op/s 2024-09-14T04:12:16.321 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:16 smithi129 ceph-mon[101316]: osdmap e799: 8 total, 8 up, 8 in 2024-09-14T04:12:16.322 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:16 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' 2024-09-14T04:12:16.322 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:16 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "orch prometheus get-credentials"} : dispatch 2024-09-14T04:12:17.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:17 smithi039 ceph-mon[106470]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T04:12:17.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:17 smithi039 ceph-mon[106470]: osdmap e800: 8 total, 8 up, 8 in 2024-09-14T04:12:17.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:17 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/29103494' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-35","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:12:17.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:17 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-35","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:12:17.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:17 smithi039 ceph-mon[111235]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T04:12:17.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:17 smithi039 ceph-mon[111235]: osdmap e800: 8 total, 8 up, 8 in 2024-09-14T04:12:17.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:17 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/29103494' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-35","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:12:17.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:17 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-35","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:12:17.368 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:17 smithi129 ceph-mon[101316]: from='mon.2 -' entity='mon.' cmd=[{"prefix": "orch prometheus get-credentials"}]: dispatch 2024-09-14T04:12:17.368 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:17 smithi129 ceph-mon[101316]: osdmap e800: 8 total, 8 up, 8 in 2024-09-14T04:12:17.368 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:17 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/29103494' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-35","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:12:17.368 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:17 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-35","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:12:17.672 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:18.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:18 smithi129 ceph-mon[101316]: pgmap v850: 745 pgs: 32 unknown, 713 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 313 KiB/s rd, 83 KiB/s wr, 412 op/s 2024-09-14T04:12:18.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:18 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:12:18.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:18 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:12:18.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:18 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:12:18.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:18 smithi129 ceph-mon[101316]: osdmap e801: 8 total, 8 up, 8 in 2024-09-14T04:12:18.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:18 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:12:18.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:18 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:12:18.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:18 smithi039 ceph-mon[106470]: pgmap v850: 745 pgs: 32 unknown, 713 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 313 KiB/s rd, 83 KiB/s wr, 412 op/s 2024-09-14T04:12:18.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:18 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:12:18.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:18 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:12:18.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:18 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:12:18.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:18 smithi039 ceph-mon[106470]: osdmap e801: 8 total, 8 up, 8 in 2024-09-14T04:12:18.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:18 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:12:18.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:18 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:12:18.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:18 smithi039 ceph-mon[111235]: pgmap v850: 745 pgs: 32 unknown, 713 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 313 KiB/s rd, 83 KiB/s wr, 412 op/s 2024-09-14T04:12:18.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:18 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:12:18.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:18 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:12:18.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:18 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-35","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:12:18.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:18 smithi039 ceph-mon[111235]: osdmap e801: 8 total, 8 up, 8 in 2024-09-14T04:12:18.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:18 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:12:18.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:18 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:12:19.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:12:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:12:18] "GET /metrics HTTP/1.1" 200 82357 "" "Prometheus/2.51.0" 2024-09-14T04:12:19.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:19 smithi129 ceph-mon[101316]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:12:19.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:19 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:12:19.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:19 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:12:19.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:19 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:12:19.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:19 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:12:19.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:19 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:12:19.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:19 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:12:19.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:19 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:12:19.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:19 smithi129 ceph-mon[101316]: osdmap e802: 8 total, 8 up, 8 in 2024-09-14T04:12:19.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:19 smithi039 ceph-mon[106470]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:12:19.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:19 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:12:19.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:19 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:12:19.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:19 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:12:19.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:19 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:12:19.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:19 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:12:19.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:19 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:12:19.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:19 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:12:19.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:19 smithi039 ceph-mon[106470]: osdmap e802: 8 total, 8 up, 8 in 2024-09-14T04:12:19.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:19 smithi039 ceph-mon[111235]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:12:19.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:19 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:12:19.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:19 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:12:19.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:19 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:12:19.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:19 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:12:19.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:19 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:12:19.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:19 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:12:19.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:19 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:12:19.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:19 smithi039 ceph-mon[111235]: osdmap e802: 8 total, 8 up, 8 in 2024-09-14T04:12:19.687 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:19.696 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:19.704 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:20.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:20 smithi129 ceph-mon[101316]: pgmap v853: 745 pgs: 4 creating+peering, 14 unknown, 727 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 640 KiB/s rd, 44 KiB/s wr, 823 op/s 2024-09-14T04:12:20.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:20 smithi129 ceph-mon[101316]: osdmap e803: 8 total, 8 up, 8 in 2024-09-14T04:12:20.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:20 smithi039 ceph-mon[106470]: pgmap v853: 745 pgs: 4 creating+peering, 14 unknown, 727 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 640 KiB/s rd, 44 KiB/s wr, 823 op/s 2024-09-14T04:12:20.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:20 smithi039 ceph-mon[106470]: osdmap e803: 8 total, 8 up, 8 in 2024-09-14T04:12:20.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:20 smithi039 ceph-mon[111235]: pgmap v853: 745 pgs: 4 creating+peering, 14 unknown, 727 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 640 KiB/s rd, 44 KiB/s wr, 823 op/s 2024-09-14T04:12:20.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:20 smithi039 ceph-mon[111235]: osdmap e803: 8 total, 8 up, 8 in 2024-09-14T04:12:20.616 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.PoolStatsPP (4845 ms) 2024-09-14T04:12:20.616 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.ImageSpec 2024-09-14T04:12:21.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:21 smithi129 ceph-mon[101316]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:12:21.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:21 smithi129 ceph-mon[101316]: osdmap e804: 8 total, 8 up, 8 in 2024-09-14T04:12:21.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:21 smithi039 ceph-mon[106470]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:12:21.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:21 smithi039 ceph-mon[106470]: osdmap e804: 8 total, 8 up, 8 in 2024-09-14T04:12:21.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:21 smithi039 ceph-mon[111235]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:12:21.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:21 smithi039 ceph-mon[111235]: osdmap e804: 8 total, 8 up, 8 in 2024-09-14T04:12:22.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:22 smithi129 ceph-mon[101316]: pgmap v856: 745 pgs: 4 creating+peering, 8 unknown, 733 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 521 KiB/s rd, 38 KiB/s wr, 691 op/s 2024-09-14T04:12:22.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:22 smithi129 ceph-mon[101316]: osdmap e805: 8 total, 8 up, 8 in 2024-09-14T04:12:22.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:22 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/848066605' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-36","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:12:22.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:22 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-36","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:12:22.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:22 smithi039 ceph-mon[106470]: pgmap v856: 745 pgs: 4 creating+peering, 8 unknown, 733 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 521 KiB/s rd, 38 KiB/s wr, 691 op/s 2024-09-14T04:12:22.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:22 smithi039 ceph-mon[106470]: osdmap e805: 8 total, 8 up, 8 in 2024-09-14T04:12:22.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:22 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/848066605' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-36","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:12:22.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:22 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-36","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:12:22.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:22 smithi039 ceph-mon[111235]: pgmap v856: 745 pgs: 4 creating+peering, 8 unknown, 733 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 521 KiB/s rd, 38 KiB/s wr, 691 op/s 2024-09-14T04:12:22.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:22 smithi039 ceph-mon[111235]: osdmap e805: 8 total, 8 up, 8 in 2024-09-14T04:12:22.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:22 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/848066605' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-36","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:12:22.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:22 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-36","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:12:22.572 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:12:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:23 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:12:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:23 smithi129 ceph-mon[101316]: osdmap e806: 8 total, 8 up, 8 in 2024-09-14T04:12:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:12:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:23 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:12:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:23 smithi039 ceph-mon[106470]: osdmap e806: 8 total, 8 up, 8 in 2024-09-14T04:12:23.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:12:23.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:23 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-36","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:12:23.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:23 smithi039 ceph-mon[111235]: osdmap e806: 8 total, 8 up, 8 in 2024-09-14T04:12:24.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:24 smithi129 ceph-mon[101316]: pgmap v859: 777 pgs: 8 creating+peering, 9 unknown, 760 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 124 KiB/s rd, 9.7 KiB/s wr, 170 op/s 2024-09-14T04:12:24.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:24 smithi129 ceph-mon[101316]: osdmap e807: 8 total, 8 up, 8 in 2024-09-14T04:12:24.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:24 smithi039 ceph-mon[106470]: pgmap v859: 777 pgs: 8 creating+peering, 9 unknown, 760 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 124 KiB/s rd, 9.7 KiB/s wr, 170 op/s 2024-09-14T04:12:24.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:24 smithi039 ceph-mon[106470]: osdmap e807: 8 total, 8 up, 8 in 2024-09-14T04:12:24.545 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:24 smithi039 ceph-mon[111235]: pgmap v859: 777 pgs: 8 creating+peering, 9 unknown, 760 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 124 KiB/s rd, 9.7 KiB/s wr, 170 op/s 2024-09-14T04:12:24.545 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:24 smithi039 ceph-mon[111235]: osdmap e807: 8 total, 8 up, 8 in 2024-09-14T04:12:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:25 smithi129 ceph-mon[101316]: osdmap e808: 8 total, 8 up, 8 in 2024-09-14T04:12:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:25 smithi129 ceph-mon[101316]: pgmap v862: 777 pgs: 8 creating+peering, 4 unknown, 765 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 6.2 KiB/s wr, 60 op/s 2024-09-14T04:12:26.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:25 smithi039 ceph-mon[106470]: osdmap e808: 8 total, 8 up, 8 in 2024-09-14T04:12:26.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:25 smithi039 ceph-mon[106470]: pgmap v862: 777 pgs: 8 creating+peering, 4 unknown, 765 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 6.2 KiB/s wr, 60 op/s 2024-09-14T04:12:26.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:25 smithi039 ceph-mon[111235]: osdmap e808: 8 total, 8 up, 8 in 2024-09-14T04:12:26.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:25 smithi039 ceph-mon[111235]: pgmap v862: 777 pgs: 8 creating+peering, 4 unknown, 765 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 44 KiB/s rd, 6.2 KiB/s wr, 60 op/s 2024-09-14T04:12:26.740 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.ImageSpec (6124 ms) 2024-09-14T04:12:26.741 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.SnapRemoveWithChildMissing 2024-09-14T04:12:27.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:26 smithi129 ceph-mon[101316]: osdmap e809: 8 total, 8 up, 8 in 2024-09-14T04:12:27.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:26 smithi039 ceph-mon[106470]: osdmap e809: 8 total, 8 up, 8 in 2024-09-14T04:12:27.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:26 smithi039 ceph-mon[111235]: osdmap e809: 8 total, 8 up, 8 in 2024-09-14T04:12:28.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:27 smithi129 ceph-mon[101316]: osdmap e810: 8 total, 8 up, 8 in 2024-09-14T04:12:28.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:27 smithi129 ceph-mon[101316]: pgmap v865: 777 pgs: 8 creating+peering, 769 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.5 KiB/s rd, 1.7 KiB/s wr, 7 op/s 2024-09-14T04:12:28.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:27 smithi039 ceph-mon[106470]: osdmap e810: 8 total, 8 up, 8 in 2024-09-14T04:12:28.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:27 smithi039 ceph-mon[106470]: pgmap v865: 777 pgs: 8 creating+peering, 769 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.5 KiB/s rd, 1.7 KiB/s wr, 7 op/s 2024-09-14T04:12:28.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:27 smithi039 ceph-mon[111235]: osdmap e810: 8 total, 8 up, 8 in 2024-09-14T04:12:28.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:27 smithi039 ceph-mon[111235]: pgmap v865: 777 pgs: 8 creating+peering, 769 active+clean; 314 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 4.5 KiB/s rd, 1.7 KiB/s wr, 7 op/s 2024-09-14T04:12:28.617 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:28.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:28 smithi129 ceph-mon[101316]: osdmap e811: 8 total, 8 up, 8 in 2024-09-14T04:12:28.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:28 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3421444264' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-37","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:12:29.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:28 smithi039 ceph-mon[106470]: osdmap e811: 8 total, 8 up, 8 in 2024-09-14T04:12:29.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:28 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3421444264' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-37","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:12:29.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:12:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:12:28] "GET /metrics HTTP/1.1" 200 87128 "" "Prometheus/2.51.0" 2024-09-14T04:12:29.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:28 smithi039 ceph-mon[111235]: osdmap e811: 8 total, 8 up, 8 in 2024-09-14T04:12:29.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:28 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3421444264' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-37","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:12:29.942 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:29 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3421444264' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-37","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:12:29.942 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:29 smithi039 ceph-mon[106470]: osdmap e812: 8 total, 8 up, 8 in 2024-09-14T04:12:29.942 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:29 smithi039 ceph-mon[106470]: pgmap v868: 809 pgs: 19 creating+peering, 13 unknown, 777 active+clean; 314 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 136 KiB/s rd, 5.7 KiB/s wr, 169 op/s 2024-09-14T04:12:29.942 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:29 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3421444264' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-37","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:12:29.943 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:29 smithi039 ceph-mon[111235]: osdmap e812: 8 total, 8 up, 8 in 2024-09-14T04:12:29.943 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:29 smithi039 ceph-mon[111235]: pgmap v868: 809 pgs: 19 creating+peering, 13 unknown, 777 active+clean; 314 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 136 KiB/s rd, 5.7 KiB/s wr, 169 op/s 2024-09-14T04:12:30.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:29 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3421444264' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-37","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:12:30.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:29 smithi129 ceph-mon[101316]: osdmap e812: 8 total, 8 up, 8 in 2024-09-14T04:12:30.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:29 smithi129 ceph-mon[101316]: pgmap v868: 809 pgs: 19 creating+peering, 13 unknown, 777 active+clean; 314 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 136 KiB/s rd, 5.7 KiB/s wr, 169 op/s 2024-09-14T04:12:31.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:30 smithi129 ceph-mon[101316]: osdmap e813: 8 total, 8 up, 8 in 2024-09-14T04:12:31.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:30 smithi129 ceph-mon[101316]: osdmap e814: 8 total, 8 up, 8 in 2024-09-14T04:12:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:30 smithi039 ceph-mon[106470]: osdmap e813: 8 total, 8 up, 8 in 2024-09-14T04:12:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:30 smithi039 ceph-mon[106470]: osdmap e814: 8 total, 8 up, 8 in 2024-09-14T04:12:31.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:30 smithi039 ceph-mon[111235]: osdmap e813: 8 total, 8 up, 8 in 2024-09-14T04:12:31.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:30 smithi039 ceph-mon[111235]: osdmap e814: 8 total, 8 up, 8 in 2024-09-14T04:12:32.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:31 smithi129 ceph-mon[101316]: pgmap v871: 809 pgs: 19 creating+peering, 8 unknown, 782 active+clean; 314 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 138 KiB/s rd, 6.0 KiB/s wr, 174 op/s 2024-09-14T04:12:32.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:31 smithi129 ceph-mon[101316]: osdmap e815: 8 total, 8 up, 8 in 2024-09-14T04:12:32.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:31 smithi039 ceph-mon[106470]: pgmap v871: 809 pgs: 19 creating+peering, 8 unknown, 782 active+clean; 314 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 138 KiB/s rd, 6.0 KiB/s wr, 174 op/s 2024-09-14T04:12:32.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:31 smithi039 ceph-mon[106470]: osdmap e815: 8 total, 8 up, 8 in 2024-09-14T04:12:32.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:31 smithi039 ceph-mon[111235]: pgmap v871: 809 pgs: 19 creating+peering, 8 unknown, 782 active+clean; 314 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 138 KiB/s rd, 6.0 KiB/s wr, 174 op/s 2024-09-14T04:12:32.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:31 smithi039 ceph-mon[111235]: osdmap e815: 8 total, 8 up, 8 in 2024-09-14T04:12:34.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:33 smithi129 ceph-mon[101316]: osdmap e816: 8 total, 8 up, 8 in 2024-09-14T04:12:34.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:33 smithi129 ceph-mon[101316]: pgmap v874: 809 pgs: 809 active+clean; 314 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 47 KiB/s rd, 5.5 KiB/s wr, 60 op/s 2024-09-14T04:12:34.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:33 smithi039 ceph-mon[106470]: osdmap e816: 8 total, 8 up, 8 in 2024-09-14T04:12:34.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:33 smithi039 ceph-mon[106470]: pgmap v874: 809 pgs: 809 active+clean; 314 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 47 KiB/s rd, 5.5 KiB/s wr, 60 op/s 2024-09-14T04:12:34.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:33 smithi039 ceph-mon[111235]: osdmap e816: 8 total, 8 up, 8 in 2024-09-14T04:12:34.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:33 smithi039 ceph-mon[111235]: pgmap v874: 809 pgs: 809 active+clean; 314 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 47 KiB/s rd, 5.5 KiB/s wr, 60 op/s 2024-09-14T04:12:35.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:34 smithi129 ceph-mon[101316]: osdmap e817: 8 total, 8 up, 8 in 2024-09-14T04:12:35.046 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:34 smithi039 ceph-mon[106470]: osdmap e817: 8 total, 8 up, 8 in 2024-09-14T04:12:35.046 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:34 smithi039 ceph-mon[111235]: osdmap e817: 8 total, 8 up, 8 in 2024-09-14T04:12:35.592 INFO:tasks.workunit.client.0.smithi039.stdout:num children is: 3 2024-09-14T04:12:35.592 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 3 2024-09-14T04:12:35.592 INFO:tasks.workunit.client.0.smithi039.stdout:child: image230 2024-09-14T04:12:35.592 INFO:tasks.workunit.client.0.smithi039.stdout:child: image232 2024-09-14T04:12:35.592 INFO:tasks.workunit.client.0.smithi039.stdout:child: image231 2024-09-14T04:12:35.592 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-15/image230/ace236e6f5fb 2024-09-14T04:12:35.592 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:12:35.592 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-37/image231/ace2f66da68e 2024-09-14T04:12:35.592 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:12:35.592 INFO:tasks.workunit.client.0.smithi039.stdout:found child test-librbd-smithi039-85658-15/image232/ace21baf9e5a 2024-09-14T04:12:35.592 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:12:35.593 INFO:tasks.workunit.client.0.smithi039.stdout:num children is: 0 2024-09-14T04:12:35.593 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 0 2024-09-14T04:12:35.593 INFO:tasks.workunit.client.0.smithi039.stdout:num snaps is: 0 2024-09-14T04:12:35.593 INFO:tasks.workunit.client.0.smithi039.stdout:expected: 0 2024-09-14T04:12:35.593 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.SnapRemoveWithChildMissing (8851 ms) 2024-09-14T04:12:35.593 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.QuiesceWatch 2024-09-14T04:12:36.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:35 smithi129 ceph-mon[101316]: osdmap e818: 8 total, 8 up, 8 in 2024-09-14T04:12:36.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:35 smithi129 ceph-mon[101316]: pgmap v877: 777 pgs: 777 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:12:36.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:35 smithi129 ceph-mon[101316]: osdmap e819: 8 total, 8 up, 8 in 2024-09-14T04:12:36.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:35 smithi039 ceph-mon[106470]: osdmap e818: 8 total, 8 up, 8 in 2024-09-14T04:12:36.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:35 smithi039 ceph-mon[106470]: pgmap v877: 777 pgs: 777 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:12:36.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:35 smithi039 ceph-mon[106470]: osdmap e819: 8 total, 8 up, 8 in 2024-09-14T04:12:36.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:35 smithi039 ceph-mon[111235]: osdmap e818: 8 total, 8 up, 8 in 2024-09-14T04:12:36.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:35 smithi039 ceph-mon[111235]: pgmap v877: 777 pgs: 777 active+clean; 2.6 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:12:36.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:35 smithi039 ceph-mon[111235]: osdmap e819: 8 total, 8 up, 8 in 2024-09-14T04:12:37.614 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:37 smithi129 ceph-mon[101316]: osdmap e820: 8 total, 8 up, 8 in 2024-09-14T04:12:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:37 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2248250751' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-38","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:12:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:37 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-38","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:12:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:37 smithi129 ceph-mon[101316]: pgmap v880: 809 pgs: 32 unknown, 777 active+clean; 2.6 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:12:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:12:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:37 smithi039 ceph-mon[106470]: osdmap e820: 8 total, 8 up, 8 in 2024-09-14T04:12:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:37 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2248250751' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-38","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:12:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:37 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-38","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:12:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:37 smithi039 ceph-mon[106470]: pgmap v880: 809 pgs: 32 unknown, 777 active+clean; 2.6 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:12:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:12:38.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:37 smithi039 ceph-mon[111235]: osdmap e820: 8 total, 8 up, 8 in 2024-09-14T04:12:38.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:37 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2248250751' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-38","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:12:38.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:37 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-38","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:12:38.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:37 smithi039 ceph-mon[111235]: pgmap v880: 809 pgs: 32 unknown, 777 active+clean; 2.6 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:12:38.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:12:39.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:38 smithi129 ceph-mon[101316]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:12:39.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:38 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-38","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:12:39.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:38 smithi129 ceph-mon[101316]: osdmap e821: 8 total, 8 up, 8 in 2024-09-14T04:12:39.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:12:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:12:38] "GET /metrics HTTP/1.1" 200 89514 "" "Prometheus/2.51.0" 2024-09-14T04:12:39.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:38 smithi039 ceph-mon[106470]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:12:39.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:38 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-38","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:12:39.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:38 smithi039 ceph-mon[106470]: osdmap e821: 8 total, 8 up, 8 in 2024-09-14T04:12:39.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:38 smithi039 ceph-mon[111235]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:12:39.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:38 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-38","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:12:39.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:38 smithi039 ceph-mon[111235]: osdmap e821: 8 total, 8 up, 8 in 2024-09-14T04:12:40.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:39 smithi129 ceph-mon[101316]: osdmap e822: 8 total, 8 up, 8 in 2024-09-14T04:12:40.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:39 smithi129 ceph-mon[101316]: pgmap v883: 809 pgs: 1 creating+peering, 12 unknown, 796 active+clean; 2.6 MiB data, 606 MiB used, 715 GiB / 715 GiB avail; 91 KiB/s rd, 6.5 KiB/s wr, 118 op/s 2024-09-14T04:12:40.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:39 smithi039 ceph-mon[106470]: osdmap e822: 8 total, 8 up, 8 in 2024-09-14T04:12:40.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:39 smithi039 ceph-mon[106470]: pgmap v883: 809 pgs: 1 creating+peering, 12 unknown, 796 active+clean; 2.6 MiB data, 606 MiB used, 715 GiB / 715 GiB avail; 91 KiB/s rd, 6.5 KiB/s wr, 118 op/s 2024-09-14T04:12:40.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:39 smithi039 ceph-mon[111235]: osdmap e822: 8 total, 8 up, 8 in 2024-09-14T04:12:40.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:39 smithi039 ceph-mon[111235]: pgmap v883: 809 pgs: 1 creating+peering, 12 unknown, 796 active+clean; 2.6 MiB data, 606 MiB used, 715 GiB / 715 GiB avail; 91 KiB/s rd, 6.5 KiB/s wr, 118 op/s 2024-09-14T04:12:41.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:40 smithi129 ceph-mon[101316]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:12:41.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:40 smithi129 ceph-mon[101316]: osdmap e823: 8 total, 8 up, 8 in 2024-09-14T04:12:41.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:40 smithi129 ceph-mon[101316]: osdmap e824: 8 total, 8 up, 8 in 2024-09-14T04:12:41.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:40 smithi039 ceph-mon[106470]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:12:41.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:40 smithi039 ceph-mon[106470]: osdmap e823: 8 total, 8 up, 8 in 2024-09-14T04:12:41.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:40 smithi039 ceph-mon[106470]: osdmap e824: 8 total, 8 up, 8 in 2024-09-14T04:12:41.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:40 smithi039 ceph-mon[111235]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:12:41.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:40 smithi039 ceph-mon[111235]: osdmap e823: 8 total, 8 up, 8 in 2024-09-14T04:12:41.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:40 smithi039 ceph-mon[111235]: osdmap e824: 8 total, 8 up, 8 in 2024-09-14T04:12:42.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:41 smithi129 ceph-mon[101316]: pgmap v886: 809 pgs: 1 creating+peering, 2 unknown, 806 active+clean; 2.6 MiB data, 574 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s rd, 6.2 KiB/s wr, 117 op/s 2024-09-14T04:12:42.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:41 smithi129 ceph-mon[101316]: osdmap e825: 8 total, 8 up, 8 in 2024-09-14T04:12:42.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:41 smithi039 ceph-mon[106470]: pgmap v886: 809 pgs: 1 creating+peering, 2 unknown, 806 active+clean; 2.6 MiB data, 574 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s rd, 6.2 KiB/s wr, 117 op/s 2024-09-14T04:12:42.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:41 smithi039 ceph-mon[106470]: osdmap e825: 8 total, 8 up, 8 in 2024-09-14T04:12:42.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:41 smithi039 ceph-mon[111235]: pgmap v886: 809 pgs: 1 creating+peering, 2 unknown, 806 active+clean; 2.6 MiB data, 574 MiB used, 715 GiB / 715 GiB avail; 90 KiB/s rd, 6.2 KiB/s wr, 117 op/s 2024-09-14T04:12:42.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:41 smithi039 ceph-mon[111235]: osdmap e825: 8 total, 8 up, 8 in 2024-09-14T04:12:44.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:43 smithi129 ceph-mon[101316]: osdmap e826: 8 total, 8 up, 8 in 2024-09-14T04:12:44.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:43 smithi129 ceph-mon[101316]: pgmap v889: 809 pgs: 809 active+clean; 2.6 MiB data, 575 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 4.2 KiB/s wr, 59 op/s 2024-09-14T04:12:44.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:43 smithi039 ceph-mon[106470]: osdmap e826: 8 total, 8 up, 8 in 2024-09-14T04:12:44.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:43 smithi039 ceph-mon[106470]: pgmap v889: 809 pgs: 809 active+clean; 2.6 MiB data, 575 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 4.2 KiB/s wr, 59 op/s 2024-09-14T04:12:44.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:43 smithi039 ceph-mon[111235]: osdmap e826: 8 total, 8 up, 8 in 2024-09-14T04:12:44.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:43 smithi039 ceph-mon[111235]: pgmap v889: 809 pgs: 809 active+clean; 2.6 MiB data, 575 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 4.2 KiB/s wr, 59 op/s 2024-09-14T04:12:45.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:44 smithi129 ceph-mon[101316]: osdmap e827: 8 total, 8 up, 8 in 2024-09-14T04:12:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:44 smithi039 ceph-mon[106470]: osdmap e827: 8 total, 8 up, 8 in 2024-09-14T04:12:45.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:44 smithi039 ceph-mon[111235]: osdmap e827: 8 total, 8 up, 8 in 2024-09-14T04:12:45.652 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.QuiesceWatch (10061 ms) 2024-09-14T04:12:45.652 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.QuiesceWatchPP 2024-09-14T04:12:45.653 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:46.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:45 smithi129 ceph-mon[101316]: osdmap e828: 8 total, 8 up, 8 in 2024-09-14T04:12:46.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:45 smithi129 ceph-mon[101316]: pgmap v892: 809 pgs: 809 active+clean; 2.6 MiB data, 575 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s rd, 4.0 KiB/s wr, 60 op/s 2024-09-14T04:12:46.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:45 smithi129 ceph-mon[101316]: osdmap e829: 8 total, 8 up, 8 in 2024-09-14T04:12:46.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:45 smithi039 ceph-mon[106470]: osdmap e828: 8 total, 8 up, 8 in 2024-09-14T04:12:46.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:45 smithi039 ceph-mon[106470]: pgmap v892: 809 pgs: 809 active+clean; 2.6 MiB data, 575 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s rd, 4.0 KiB/s wr, 60 op/s 2024-09-14T04:12:46.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:45 smithi039 ceph-mon[106470]: osdmap e829: 8 total, 8 up, 8 in 2024-09-14T04:12:46.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:45 smithi039 ceph-mon[111235]: osdmap e828: 8 total, 8 up, 8 in 2024-09-14T04:12:46.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:45 smithi039 ceph-mon[111235]: pgmap v892: 809 pgs: 809 active+clean; 2.6 MiB data, 575 MiB used, 715 GiB / 715 GiB avail; 45 KiB/s rd, 4.0 KiB/s wr, 60 op/s 2024-09-14T04:12:46.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:45 smithi039 ceph-mon[111235]: osdmap e829: 8 total, 8 up, 8 in 2024-09-14T04:12:48.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:47 smithi129 ceph-mon[101316]: osdmap e830: 8 total, 8 up, 8 in 2024-09-14T04:12:48.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:47 smithi129 ceph-mon[101316]: pgmap v895: 809 pgs: 809 active+clean; 2.6 MiB data, 575 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 511 B/s wr, 4 op/s 2024-09-14T04:12:48.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:47 smithi039 ceph-mon[106470]: osdmap e830: 8 total, 8 up, 8 in 2024-09-14T04:12:48.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:47 smithi039 ceph-mon[106470]: pgmap v895: 809 pgs: 809 active+clean; 2.6 MiB data, 575 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 511 B/s wr, 4 op/s 2024-09-14T04:12:48.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:47 smithi039 ceph-mon[111235]: osdmap e830: 8 total, 8 up, 8 in 2024-09-14T04:12:48.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:47 smithi039 ceph-mon[111235]: pgmap v895: 809 pgs: 809 active+clean; 2.6 MiB data, 575 MiB used, 715 GiB / 715 GiB avail; 2.5 KiB/s rd, 511 B/s wr, 4 op/s 2024-09-14T04:12:49.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:48 smithi129 ceph-mon[101316]: osdmap e831: 8 total, 8 up, 8 in 2024-09-14T04:12:49.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:48 smithi039 ceph-mon[106470]: osdmap e831: 8 total, 8 up, 8 in 2024-09-14T04:12:49.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:48 smithi039 ceph-mon[111235]: osdmap e831: 8 total, 8 up, 8 in 2024-09-14T04:12:49.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:12:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:12:48] "GET /metrics HTTP/1.1" 200 89514 "" "Prometheus/2.51.0" 2024-09-14T04:12:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:49 smithi129 ceph-mon[101316]: osdmap e832: 8 total, 8 up, 8 in 2024-09-14T04:12:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:49 smithi129 ceph-mon[101316]: pgmap v898: 809 pgs: 809 active+clean; 2.6 MiB data, 576 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 3.2 KiB/s wr, 59 op/s 2024-09-14T04:12:50.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:49 smithi039 ceph-mon[106470]: osdmap e832: 8 total, 8 up, 8 in 2024-09-14T04:12:50.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:49 smithi039 ceph-mon[106470]: pgmap v898: 809 pgs: 809 active+clean; 2.6 MiB data, 576 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 3.2 KiB/s wr, 59 op/s 2024-09-14T04:12:50.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:49 smithi039 ceph-mon[111235]: osdmap e832: 8 total, 8 up, 8 in 2024-09-14T04:12:50.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:49 smithi039 ceph-mon[111235]: pgmap v898: 809 pgs: 809 active+clean; 2.6 MiB data, 576 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 3.2 KiB/s wr, 59 op/s 2024-09-14T04:12:51.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:50 smithi129 ceph-mon[101316]: osdmap e833: 8 total, 8 up, 8 in 2024-09-14T04:12:51.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:50 smithi129 ceph-mon[101316]: osdmap e834: 8 total, 8 up, 8 in 2024-09-14T04:12:51.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:50 smithi039 ceph-mon[106470]: osdmap e833: 8 total, 8 up, 8 in 2024-09-14T04:12:51.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:50 smithi039 ceph-mon[106470]: osdmap e834: 8 total, 8 up, 8 in 2024-09-14T04:12:51.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:50 smithi039 ceph-mon[111235]: osdmap e833: 8 total, 8 up, 8 in 2024-09-14T04:12:51.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:50 smithi039 ceph-mon[111235]: osdmap e834: 8 total, 8 up, 8 in 2024-09-14T04:12:51.657 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.QuiesceWatchPP (6004 ms) 2024-09-14T04:12:51.657 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.QuiesceWatchError 2024-09-14T04:12:51.657 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:52.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:51 smithi129 ceph-mon[101316]: pgmap v901: 809 pgs: 809 active+clean; 2.6 MiB data, 576 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 6.5 KiB/s wr, 119 op/s 2024-09-14T04:12:52.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:51 smithi129 ceph-mon[101316]: osdmap e835: 8 total, 8 up, 8 in 2024-09-14T04:12:52.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:51 smithi039 ceph-mon[111235]: pgmap v901: 809 pgs: 809 active+clean; 2.6 MiB data, 576 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 6.5 KiB/s wr, 119 op/s 2024-09-14T04:12:52.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:51 smithi039 ceph-mon[111235]: osdmap e835: 8 total, 8 up, 8 in 2024-09-14T04:12:52.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:51 smithi039 ceph-mon[106470]: pgmap v901: 809 pgs: 809 active+clean; 2.6 MiB data, 576 MiB used, 715 GiB / 715 GiB avail; 89 KiB/s rd, 6.5 KiB/s wr, 119 op/s 2024-09-14T04:12:52.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:51 smithi039 ceph-mon[106470]: osdmap e835: 8 total, 8 up, 8 in 2024-09-14T04:12:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:12:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:12:53.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:12:54.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:53 smithi129 ceph-mon[101316]: osdmap e836: 8 total, 8 up, 8 in 2024-09-14T04:12:54.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:53 smithi129 ceph-mon[101316]: pgmap v904: 809 pgs: 809 active+clean; 2.6 MiB data, 565 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 5.7 KiB/s wr, 92 op/s 2024-09-14T04:12:54.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:53 smithi039 ceph-mon[106470]: osdmap e836: 8 total, 8 up, 8 in 2024-09-14T04:12:54.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:53 smithi039 ceph-mon[106470]: pgmap v904: 809 pgs: 809 active+clean; 2.6 MiB data, 565 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 5.7 KiB/s wr, 92 op/s 2024-09-14T04:12:54.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:53 smithi039 ceph-mon[111235]: osdmap e836: 8 total, 8 up, 8 in 2024-09-14T04:12:54.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:53 smithi039 ceph-mon[111235]: pgmap v904: 809 pgs: 809 active+clean; 2.6 MiB data, 565 MiB used, 715 GiB / 715 GiB avail; 68 KiB/s rd, 5.7 KiB/s wr, 92 op/s 2024-09-14T04:12:55.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:54 smithi129 ceph-mon[101316]: osdmap e837: 8 total, 8 up, 8 in 2024-09-14T04:12:55.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:54 smithi039 ceph-mon[106470]: osdmap e837: 8 total, 8 up, 8 in 2024-09-14T04:12:55.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:54 smithi039 ceph-mon[111235]: osdmap e837: 8 total, 8 up, 8 in 2024-09-14T04:12:56.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:55 smithi129 ceph-mon[101316]: osdmap e838: 8 total, 8 up, 8 in 2024-09-14T04:12:56.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:55 smithi129 ceph-mon[101316]: pgmap v907: 809 pgs: 809 active+clean; 2.6 MiB data, 565 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s rd, 5.2 KiB/s wr, 88 op/s 2024-09-14T04:12:56.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:55 smithi129 ceph-mon[101316]: osdmap e839: 8 total, 8 up, 8 in 2024-09-14T04:12:56.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:55 smithi039 ceph-mon[106470]: osdmap e838: 8 total, 8 up, 8 in 2024-09-14T04:12:56.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:55 smithi039 ceph-mon[106470]: pgmap v907: 809 pgs: 809 active+clean; 2.6 MiB data, 565 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s rd, 5.2 KiB/s wr, 88 op/s 2024-09-14T04:12:56.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:55 smithi039 ceph-mon[106470]: osdmap e839: 8 total, 8 up, 8 in 2024-09-14T04:12:56.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:55 smithi039 ceph-mon[111235]: osdmap e838: 8 total, 8 up, 8 in 2024-09-14T04:12:56.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:55 smithi039 ceph-mon[111235]: pgmap v907: 809 pgs: 809 active+clean; 2.6 MiB data, 565 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s rd, 5.2 KiB/s wr, 88 op/s 2024-09-14T04:12:56.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:55 smithi039 ceph-mon[111235]: osdmap e839: 8 total, 8 up, 8 in 2024-09-14T04:12:57.683 INFO:tasks.workunit.client.0.smithi039.stdout:10.00% done 2024-09-14T04:12:57.683 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.QuiesceWatchError (6026 ms) 2024-09-14T04:12:57.683 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.QuiesceWatchTimeout 2024-09-14T04:12:57.684 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:12:57.700 INFO:tasks.workunit.client.0.smithi039.stderr:test quiesce is not long enough to time out 2024-09-14T04:12:58.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:57 smithi129 ceph-mon[101316]: osdmap e840: 8 total, 8 up, 8 in 2024-09-14T04:12:58.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:57 smithi129 ceph-mon[101316]: pgmap v910: 809 pgs: 809 active+clean; 2.6 MiB data, 565 MiB used, 715 GiB / 715 GiB avail; 44 KiB/s rd, 3.0 KiB/s wr, 58 op/s 2024-09-14T04:12:58.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:57 smithi039 ceph-mon[106470]: osdmap e840: 8 total, 8 up, 8 in 2024-09-14T04:12:58.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:57 smithi039 ceph-mon[106470]: pgmap v910: 809 pgs: 809 active+clean; 2.6 MiB data, 565 MiB used, 715 GiB / 715 GiB avail; 44 KiB/s rd, 3.0 KiB/s wr, 58 op/s 2024-09-14T04:12:58.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:57 smithi039 ceph-mon[111235]: osdmap e840: 8 total, 8 up, 8 in 2024-09-14T04:12:58.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:57 smithi039 ceph-mon[111235]: pgmap v910: 809 pgs: 809 active+clean; 2.6 MiB data, 565 MiB used, 715 GiB / 715 GiB avail; 44 KiB/s rd, 3.0 KiB/s wr, 58 op/s 2024-09-14T04:12:59.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:58 smithi129 ceph-mon[101316]: osdmap e841: 8 total, 8 up, 8 in 2024-09-14T04:12:59.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:58 smithi039 ceph-mon[106470]: osdmap e841: 8 total, 8 up, 8 in 2024-09-14T04:12:59.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:58 smithi039 ceph-mon[111235]: osdmap e841: 8 total, 8 up, 8 in 2024-09-14T04:12:59.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:12:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:12:58] "GET /metrics HTTP/1.1" 200 89542 "" "Prometheus/2.51.0" 2024-09-14T04:12:59.942 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:12:59 smithi039 ceph-mon[106470]: pgmap v912: 809 pgs: 809 active+clean; 2.6 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 66 KiB/s rd, 5.2 KiB/s wr, 92 op/s 2024-09-14T04:12:59.942 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:12:59 smithi039 ceph-mon[111235]: pgmap v912: 809 pgs: 809 active+clean; 2.6 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 66 KiB/s rd, 5.2 KiB/s wr, 92 op/s 2024-09-14T04:13:00.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:12:59 smithi129 ceph-mon[101316]: pgmap v912: 809 pgs: 809 active+clean; 2.6 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 66 KiB/s rd, 5.2 KiB/s wr, 92 op/s 2024-09-14T04:13:02.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:01 smithi129 ceph-mon[101316]: osdmap e842: 8 total, 8 up, 8 in 2024-09-14T04:13:02.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:01 smithi129 ceph-mon[101316]: pgmap v914: 809 pgs: 809 active+clean; 2.6 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 4.2 KiB/s wr, 71 op/s 2024-09-14T04:13:02.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:01 smithi039 ceph-mon[106470]: osdmap e842: 8 total, 8 up, 8 in 2024-09-14T04:13:02.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:01 smithi039 ceph-mon[106470]: pgmap v914: 809 pgs: 809 active+clean; 2.6 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 4.2 KiB/s wr, 71 op/s 2024-09-14T04:13:02.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:01 smithi039 ceph-mon[111235]: osdmap e842: 8 total, 8 up, 8 in 2024-09-14T04:13:02.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:01 smithi039 ceph-mon[111235]: pgmap v914: 809 pgs: 809 active+clean; 2.6 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 4.2 KiB/s wr, 71 op/s 2024-09-14T04:13:04.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:04 smithi129 ceph-mon[101316]: pgmap v915: 809 pgs: 809 active+clean; 2.6 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s rd, 3.7 KiB/s wr, 65 op/s 2024-09-14T04:13:04.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:04 smithi039 ceph-mon[106470]: pgmap v915: 809 pgs: 809 active+clean; 2.6 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s rd, 3.7 KiB/s wr, 65 op/s 2024-09-14T04:13:04.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:04 smithi039 ceph-mon[111235]: pgmap v915: 809 pgs: 809 active+clean; 2.6 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 47 KiB/s rd, 3.7 KiB/s wr, 65 op/s 2024-09-14T04:13:06.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:06 smithi129 ceph-mon[101316]: pgmap v916: 809 pgs: 809 active+clean; 2.6 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 38 KiB/s rd, 3.0 KiB/s wr, 52 op/s 2024-09-14T04:13:06.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:06 smithi129 ceph-mon[101316]: osdmap e843: 8 total, 8 up, 8 in 2024-09-14T04:13:06.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:06 smithi039 ceph-mon[106470]: pgmap v916: 809 pgs: 809 active+clean; 2.6 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 38 KiB/s rd, 3.0 KiB/s wr, 52 op/s 2024-09-14T04:13:06.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:06 smithi039 ceph-mon[106470]: osdmap e843: 8 total, 8 up, 8 in 2024-09-14T04:13:06.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:06 smithi039 ceph-mon[111235]: pgmap v916: 809 pgs: 809 active+clean; 2.6 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 38 KiB/s rd, 3.0 KiB/s wr, 52 op/s 2024-09-14T04:13:06.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:06 smithi039 ceph-mon[111235]: osdmap e843: 8 total, 8 up, 8 in 2024-09-14T04:13:06.597 INFO:tasks.workunit.client.0.smithi039.stderr:test quiesce is timed out 2024-09-14T04:13:06.599 INFO:tasks.workunit.client.0.smithi039.stderr:waiting for timed out ... 0 2024-09-14T04:13:07.600 INFO:tasks.workunit.client.0.smithi039.stderr:waiting for timed out ... 1 2024-09-14T04:13:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:07 smithi129 ceph-mon[101316]: osdmap e844: 8 total, 8 up, 8 in 2024-09-14T04:13:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:07 smithi129 ceph-mon[101316]: pgmap v919: 809 pgs: 809 active+clean; 2.6 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 156 B/s wr, 4 op/s 2024-09-14T04:13:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:13:08.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:07 smithi039 ceph-mon[106470]: osdmap e844: 8 total, 8 up, 8 in 2024-09-14T04:13:08.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:07 smithi039 ceph-mon[106470]: pgmap v919: 809 pgs: 809 active+clean; 2.6 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 156 B/s wr, 4 op/s 2024-09-14T04:13:08.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:13:08.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:07 smithi039 ceph-mon[111235]: osdmap e844: 8 total, 8 up, 8 in 2024-09-14T04:13:08.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:07 smithi039 ceph-mon[111235]: pgmap v919: 809 pgs: 809 active+clean; 2.6 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 3.4 KiB/s rd, 156 B/s wr, 4 op/s 2024-09-14T04:13:08.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:13:08.600 INFO:tasks.workunit.client.0.smithi039.stderr:waiting for timed out ... 2 2024-09-14T04:13:09.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:08 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.7", "id": [3, 7]} : dispatch 2024-09-14T04:13:09.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:08 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.8", "id": [6, 1]} : dispatch 2024-09-14T04:13:09.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:08 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.9", "id": [3, 1]} : dispatch 2024-09-14T04:13:09.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:08 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.7", "id": [3, 7]} : dispatch 2024-09-14T04:13:09.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:08 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.e", "id": [3, 1]} : dispatch 2024-09-14T04:13:09.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:08 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.8", "id": [6, 1]} : dispatch 2024-09-14T04:13:09.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:08 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.9", "id": [3, 1]} : dispatch 2024-09-14T04:13:09.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:08 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.e", "id": [3, 1]} : dispatch 2024-09-14T04:13:09.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:13:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:13:08] "GET /metrics HTTP/1.1" 200 89550 "" "Prometheus/2.51.0" 2024-09-14T04:13:09.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:08 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.7", "id": [3, 7]} : dispatch 2024-09-14T04:13:09.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:08 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.8", "id": [6, 1]} : dispatch 2024-09-14T04:13:09.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:08 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.9", "id": [3, 1]} : dispatch 2024-09-14T04:13:09.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.7", "id": [3, 7]} : dispatch 2024-09-14T04:13:09.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:08 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.e", "id": [3, 1]} : dispatch 2024-09-14T04:13:09.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.8", "id": [6, 1]} : dispatch 2024-09-14T04:13:09.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.9", "id": [3, 1]} : dispatch 2024-09-14T04:13:09.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.e", "id": [3, 1]} : dispatch 2024-09-14T04:13:09.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:08 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.7", "id": [3, 7]} : dispatch 2024-09-14T04:13:09.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:08 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.8", "id": [6, 1]} : dispatch 2024-09-14T04:13:09.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:08 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.9", "id": [3, 1]} : dispatch 2024-09-14T04:13:09.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:08 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.7", "id": [3, 7]} : dispatch 2024-09-14T04:13:09.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:08 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.e", "id": [3, 1]} : dispatch 2024-09-14T04:13:09.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:08 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.8", "id": [6, 1]} : dispatch 2024-09-14T04:13:09.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:08 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.9", "id": [3, 1]} : dispatch 2024-09-14T04:13:09.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:08 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.e", "id": [3, 1]} : dispatch 2024-09-14T04:13:09.600 INFO:tasks.workunit.client.0.smithi039.stderr:waiting for timed out ... 3 2024-09-14T04:13:10.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:09 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.7", "id": [3, 7]}]': finished 2024-09-14T04:13:10.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:09 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.8", "id": [6, 1]}]': finished 2024-09-14T04:13:10.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:09 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.9", "id": [3, 1]}]': finished 2024-09-14T04:13:10.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:09 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.e", "id": [3, 1]}]': finished 2024-09-14T04:13:10.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:09 smithi129 ceph-mon[101316]: osdmap e845: 8 total, 8 up, 8 in 2024-09-14T04:13:10.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:09 smithi129 ceph-mon[101316]: pgmap v921: 809 pgs: 809 active+clean; 2.6 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 170 B/s wr, 0 op/s 2024-09-14T04:13:10.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:09 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.7", "id": [3, 7]}]': finished 2024-09-14T04:13:10.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:09 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.8", "id": [6, 1]}]': finished 2024-09-14T04:13:10.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:09 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.9", "id": [3, 1]}]': finished 2024-09-14T04:13:10.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:09 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.e", "id": [3, 1]}]': finished 2024-09-14T04:13:10.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:09 smithi039 ceph-mon[106470]: osdmap e845: 8 total, 8 up, 8 in 2024-09-14T04:13:10.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:09 smithi039 ceph-mon[106470]: pgmap v921: 809 pgs: 809 active+clean; 2.6 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 170 B/s wr, 0 op/s 2024-09-14T04:13:10.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:09 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.7", "id": [3, 7]}]': finished 2024-09-14T04:13:10.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:09 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.8", "id": [6, 1]}]': finished 2024-09-14T04:13:10.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:09 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.9", "id": [3, 1]}]': finished 2024-09-14T04:13:10.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:09 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "39.e", "id": [3, 1]}]': finished 2024-09-14T04:13:10.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:09 smithi039 ceph-mon[111235]: osdmap e845: 8 total, 8 up, 8 in 2024-09-14T04:13:10.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:09 smithi039 ceph-mon[111235]: pgmap v921: 809 pgs: 809 active+clean; 2.6 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 170 B/s wr, 0 op/s 2024-09-14T04:13:10.600 INFO:tasks.workunit.client.0.smithi039.stderr:waiting for timed out ... 4 2024-09-14T04:13:11.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:10 smithi129 ceph-mon[101316]: osdmap e846: 8 total, 8 up, 8 in 2024-09-14T04:13:11.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:10 smithi039 ceph-mon[106470]: osdmap e846: 8 total, 8 up, 8 in 2024-09-14T04:13:11.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:10 smithi039 ceph-mon[111235]: osdmap e846: 8 total, 8 up, 8 in 2024-09-14T04:13:11.601 INFO:tasks.workunit.client.0.smithi039.stderr:waiting for timed out ... 5 2024-09-14T04:13:12.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:11 smithi129 ceph-mon[101316]: pgmap v923: 809 pgs: 3 peering, 806 active+clean; 2.6 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 741 B/s rd, 185 B/s wr, 1 op/s 2024-09-14T04:13:12.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:11 smithi039 ceph-mon[106470]: pgmap v923: 809 pgs: 3 peering, 806 active+clean; 2.6 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 741 B/s rd, 185 B/s wr, 1 op/s 2024-09-14T04:13:12.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:11 smithi039 ceph-mon[111235]: pgmap v923: 809 pgs: 3 peering, 806 active+clean; 2.6 MiB data, 566 MiB used, 715 GiB / 715 GiB avail; 741 B/s rd, 185 B/s wr, 1 op/s 2024-09-14T04:13:12.601 INFO:tasks.workunit.client.0.smithi039.stderr:waiting for timed out ... 6 2024-09-14T04:13:13.601 INFO:tasks.workunit.client.0.smithi039.stderr:waiting for timed out ... 7 2024-09-14T04:13:14.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:14 smithi129 ceph-mon[101316]: pgmap v924: 809 pgs: 3 peering, 806 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 4.4 KiB/s rd, 471 B/s wr, 5 op/s; 0 B/s, 1 keys/s, 0 objects/s recovering 2024-09-14T04:13:14.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:14 smithi039 ceph-mon[106470]: pgmap v924: 809 pgs: 3 peering, 806 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 4.4 KiB/s rd, 471 B/s wr, 5 op/s; 0 B/s, 1 keys/s, 0 objects/s recovering 2024-09-14T04:13:14.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:14 smithi039 ceph-mon[111235]: pgmap v924: 809 pgs: 3 peering, 806 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 4.4 KiB/s rd, 471 B/s wr, 5 op/s; 0 B/s, 1 keys/s, 0 objects/s recovering 2024-09-14T04:13:14.601 INFO:tasks.workunit.client.0.smithi039.stderr:waiting for timed out ... 8 2024-09-14T04:13:15.601 INFO:tasks.workunit.client.0.smithi039.stderr:waiting for timed out ... 9 2024-09-14T04:13:16.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:16 smithi129 ceph-mon[101316]: pgmap v925: 809 pgs: 809 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s; 8 B/s, 0 keys/s, 0 objects/s recovering 2024-09-14T04:13:16.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:16 smithi039 ceph-mon[106470]: pgmap v925: 809 pgs: 809 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s; 8 B/s, 0 keys/s, 0 objects/s recovering 2024-09-14T04:13:16.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:16 smithi039 ceph-mon[111235]: pgmap v925: 809 pgs: 809 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 3.6 KiB/s rd, 383 B/s wr, 4 op/s; 8 B/s, 0 keys/s, 0 objects/s recovering 2024-09-14T04:13:16.601 INFO:tasks.workunit.client.0.smithi039.stderr:waiting for timed out ... 10 2024-09-14T04:13:17.271 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 04:13:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=sqlstore.transactions t=2024-09-14T04:13:16.858159609Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-09-14T04:13:18.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:18 smithi129 ceph-mon[101316]: pgmap v926: 809 pgs: 809 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 240 B/s wr, 3 op/s; 8 B/s, 0 keys/s, 0 objects/s recovering 2024-09-14T04:13:18.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:18 smithi039 ceph-mon[106470]: pgmap v926: 809 pgs: 809 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 240 B/s wr, 3 op/s; 8 B/s, 0 keys/s, 0 objects/s recovering 2024-09-14T04:13:18.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:18 smithi039 ceph-mon[111235]: pgmap v926: 809 pgs: 809 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 3.1 KiB/s rd, 240 B/s wr, 3 op/s; 8 B/s, 0 keys/s, 0 objects/s recovering 2024-09-14T04:13:19.017 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:13:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:13:18] "GET /metrics HTTP/1.1" 200 89550 "" "Prometheus/2.51.0" 2024-09-14T04:13:19.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:19 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:13:19.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:19 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:13:19.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:19 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:13:20.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:20 smithi129 ceph-mon[101316]: pgmap v927: 809 pgs: 809 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 204 B/s wr, 3 op/s; 6 B/s, 0 keys/s, 0 objects/s recovering 2024-09-14T04:13:20.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:20 smithi039 ceph-mon[106470]: pgmap v927: 809 pgs: 809 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 204 B/s wr, 3 op/s; 6 B/s, 0 keys/s, 0 objects/s recovering 2024-09-14T04:13:20.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:20 smithi039 ceph-mon[111235]: pgmap v927: 809 pgs: 809 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 2.6 KiB/s rd, 204 B/s wr, 3 op/s; 6 B/s, 0 keys/s, 0 objects/s recovering 2024-09-14T04:13:21.478 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:21 smithi039 ceph-mon[106470]: pgmap v928: 809 pgs: 809 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 178 B/s wr, 2 op/s; 6 B/s, 0 keys/s, 0 objects/s recovering 2024-09-14T04:13:21.478 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:21 smithi039 ceph-mon[111235]: pgmap v928: 809 pgs: 809 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 178 B/s wr, 2 op/s; 6 B/s, 0 keys/s, 0 objects/s recovering 2024-09-14T04:13:21.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:21 smithi129 ceph-mon[101316]: pgmap v928: 809 pgs: 809 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 2.3 KiB/s rd, 178 B/s wr, 2 op/s; 6 B/s, 0 keys/s, 0 objects/s recovering 2024-09-14T04:13:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:23 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:13:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:23 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:13:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:13:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:23 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:13:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:23 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:13:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:13:23.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:13:23.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:23 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:13:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:23 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:13:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:23 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:13:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:13:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:23 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:13:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:23 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:13:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:13:23.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:13:23.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:23 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:13:23.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:23 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:13:23.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:23 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:13:23.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:13:23.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:23 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:13:23.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:23 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:13:23.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:13:23.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:13:23.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:23 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:13:24.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:24 smithi129 ceph-mon[101316]: pgmap v929: 809 pgs: 809 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 170 B/s wr, 2 op/s; 5 B/s, 0 keys/s, 0 objects/s recovering 2024-09-14T04:13:24.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:24 smithi039 ceph-mon[106470]: pgmap v929: 809 pgs: 809 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 170 B/s wr, 2 op/s; 5 B/s, 0 keys/s, 0 objects/s recovering 2024-09-14T04:13:24.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:24 smithi039 ceph-mon[111235]: pgmap v929: 809 pgs: 809 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s rd, 170 B/s wr, 2 op/s; 5 B/s, 0 keys/s, 0 objects/s recovering 2024-09-14T04:13:26.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:26 smithi129 ceph-mon[101316]: pgmap v930: 809 pgs: 809 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 5 B/s, 0 objects/s recovering 2024-09-14T04:13:26.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:26 smithi039 ceph-mon[106470]: pgmap v930: 809 pgs: 809 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 5 B/s, 0 objects/s recovering 2024-09-14T04:13:26.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:26 smithi039 ceph-mon[111235]: pgmap v930: 809 pgs: 809 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 5 B/s, 0 objects/s recovering 2024-09-14T04:13:27.602 INFO:tasks.workunit.client.0.smithi039.stderr:test retry succeeds 2024-09-14T04:13:28.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:28 smithi129 ceph-mon[101316]: pgmap v931: 809 pgs: 809 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:13:28.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:28 smithi039 ceph-mon[106470]: pgmap v931: 809 pgs: 809 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:13:28.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:28 smithi039 ceph-mon[111235]: pgmap v931: 809 pgs: 809 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:13:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:13:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:13:28] "GET /metrics HTTP/1.1" 200 89550 "" "Prometheus/2.51.0" 2024-09-14T04:13:29.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:29 smithi129 ceph-mon[101316]: osdmap e847: 8 total, 8 up, 8 in 2024-09-14T04:13:29.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:29 smithi039 ceph-mon[106470]: osdmap e847: 8 total, 8 up, 8 in 2024-09-14T04:13:29.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:29 smithi039 ceph-mon[111235]: osdmap e847: 8 total, 8 up, 8 in 2024-09-14T04:13:30.303 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.QuiesceWatchTimeout (32618 ms) 2024-09-14T04:13:30.304 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.WriteZeroes 2024-09-14T04:13:30.304 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:13:30.339 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:30 smithi039 ceph-mon[106470]: pgmap v933: 809 pgs: 809 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:13:30.339 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:30 smithi039 ceph-mon[106470]: osdmap e848: 8 total, 8 up, 8 in 2024-09-14T04:13:30.339 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:30 smithi039 ceph-mon[111235]: pgmap v933: 809 pgs: 809 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:13:30.339 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:30 smithi039 ceph-mon[111235]: osdmap e848: 8 total, 8 up, 8 in 2024-09-14T04:13:30.369 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.WriteZeroes (68 ms) 2024-09-14T04:13:30.369 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.WriteZeroesThickProvision 2024-09-14T04:13:30.369 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:13:30.443 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.WriteZeroesThickProvision (75 ms) 2024-09-14T04:13:30.443 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestLibRBD.ConcurentOperations 2024-09-14T04:13:30.444 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:13:30.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:30 smithi129 ceph-mon[101316]: pgmap v933: 809 pgs: 809 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:13:30.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:30 smithi129 ceph-mon[101316]: osdmap e848: 8 total, 8 up, 8 in 2024-09-14T04:13:31.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:31 smithi129 ceph-mon[101316]: osdmap e849: 8 total, 8 up, 8 in 2024-09-14T04:13:31.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:31 smithi129 ceph-mon[101316]: pgmap v936: 809 pgs: 809 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-14T04:13:31.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:31 smithi039 ceph-mon[106470]: osdmap e849: 8 total, 8 up, 8 in 2024-09-14T04:13:31.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:31 smithi039 ceph-mon[106470]: pgmap v936: 809 pgs: 809 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-14T04:13:31.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:31 smithi039 ceph-mon[111235]: osdmap e849: 8 total, 8 up, 8 in 2024-09-14T04:13:31.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:31 smithi039 ceph-mon[111235]: pgmap v936: 809 pgs: 809 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 341 B/s rd, 0 op/s 2024-09-14T04:13:32.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:32 smithi129 ceph-mon[101316]: osdmap e850: 8 total, 8 up, 8 in 2024-09-14T04:13:32.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:32 smithi039 ceph-mon[106470]: osdmap e850: 8 total, 8 up, 8 in 2024-09-14T04:13:32.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:32 smithi039 ceph-mon[111235]: osdmap e850: 8 total, 8 up, 8 in 2024-09-14T04:13:33.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:33 smithi129 ceph-mon[101316]: osdmap e851: 8 total, 8 up, 8 in 2024-09-14T04:13:33.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:33 smithi129 ceph-mon[101316]: pgmap v939: 809 pgs: 809 active+clean; 2.6 MiB data, 551 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 7.7 KiB/s wr, 89 op/s 2024-09-14T04:13:33.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:33 smithi039 ceph-mon[106470]: osdmap e851: 8 total, 8 up, 8 in 2024-09-14T04:13:33.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:33 smithi039 ceph-mon[106470]: pgmap v939: 809 pgs: 809 active+clean; 2.6 MiB data, 551 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 7.7 KiB/s wr, 89 op/s 2024-09-14T04:13:33.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:33 smithi039 ceph-mon[111235]: osdmap e851: 8 total, 8 up, 8 in 2024-09-14T04:13:33.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:33 smithi039 ceph-mon[111235]: pgmap v939: 809 pgs: 809 active+clean; 2.6 MiB data, 551 MiB used, 715 GiB / 715 GiB avail; 60 KiB/s rd, 7.7 KiB/s wr, 89 op/s 2024-09-14T04:13:34.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:34 smithi129 ceph-mon[101316]: osdmap e852: 8 total, 8 up, 8 in 2024-09-14T04:13:34.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:34 smithi039 ceph-mon[106470]: osdmap e852: 8 total, 8 up, 8 in 2024-09-14T04:13:34.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:34 smithi039 ceph-mon[111235]: osdmap e852: 8 total, 8 up, 8 in 2024-09-14T04:13:35.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:35 smithi129 ceph-mon[101316]: osdmap e853: 8 total, 8 up, 8 in 2024-09-14T04:13:35.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:35 smithi129 ceph-mon[101316]: pgmap v942: 809 pgs: 809 active+clean; 2.6 MiB data, 552 MiB used, 715 GiB / 715 GiB avail; 132 KiB/s rd, 12 KiB/s wr, 185 op/s 2024-09-14T04:13:35.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:35 smithi039 ceph-mon[106470]: osdmap e853: 8 total, 8 up, 8 in 2024-09-14T04:13:35.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:35 smithi039 ceph-mon[106470]: pgmap v942: 809 pgs: 809 active+clean; 2.6 MiB data, 552 MiB used, 715 GiB / 715 GiB avail; 132 KiB/s rd, 12 KiB/s wr, 185 op/s 2024-09-14T04:13:35.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:35 smithi039 ceph-mon[111235]: osdmap e853: 8 total, 8 up, 8 in 2024-09-14T04:13:35.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:35 smithi039 ceph-mon[111235]: pgmap v942: 809 pgs: 809 active+clean; 2.6 MiB data, 552 MiB used, 715 GiB / 715 GiB avail; 132 KiB/s rd, 12 KiB/s wr, 185 op/s 2024-09-14T04:13:36.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:36 smithi129 ceph-mon[101316]: osdmap e854: 8 total, 8 up, 8 in 2024-09-14T04:13:36.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:36 smithi129 ceph-mon[101316]: osdmap e855: 8 total, 8 up, 8 in 2024-09-14T04:13:36.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:36 smithi039 ceph-mon[106470]: osdmap e854: 8 total, 8 up, 8 in 2024-09-14T04:13:36.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:36 smithi039 ceph-mon[106470]: osdmap e855: 8 total, 8 up, 8 in 2024-09-14T04:13:36.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:36 smithi039 ceph-mon[111235]: osdmap e854: 8 total, 8 up, 8 in 2024-09-14T04:13:36.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:36 smithi039 ceph-mon[111235]: osdmap e855: 8 total, 8 up, 8 in 2024-09-14T04:13:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:37 smithi129 ceph-mon[101316]: osdmap e856: 8 total, 8 up, 8 in 2024-09-14T04:13:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:37 smithi129 ceph-mon[101316]: pgmap v946: 809 pgs: 809 active+clean; 2.6 MiB data, 552 MiB used, 715 GiB / 715 GiB avail; 79 KiB/s rd, 4.6 KiB/s wr, 104 op/s 2024-09-14T04:13:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:13:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:37 smithi039 ceph-mon[106470]: osdmap e856: 8 total, 8 up, 8 in 2024-09-14T04:13:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:37 smithi039 ceph-mon[106470]: pgmap v946: 809 pgs: 809 active+clean; 2.6 MiB data, 552 MiB used, 715 GiB / 715 GiB avail; 79 KiB/s rd, 4.6 KiB/s wr, 104 op/s 2024-09-14T04:13:38.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:13:38.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:37 smithi039 ceph-mon[111235]: osdmap e856: 8 total, 8 up, 8 in 2024-09-14T04:13:38.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:37 smithi039 ceph-mon[111235]: pgmap v946: 809 pgs: 809 active+clean; 2.6 MiB data, 552 MiB used, 715 GiB / 715 GiB avail; 79 KiB/s rd, 4.6 KiB/s wr, 104 op/s 2024-09-14T04:13:38.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:13:39.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:38 smithi129 ceph-mon[101316]: osdmap e857: 8 total, 8 up, 8 in 2024-09-14T04:13:39.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:38 smithi039 ceph-mon[106470]: osdmap e857: 8 total, 8 up, 8 in 2024-09-14T04:13:39.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:13:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:13:38] "GET /metrics HTTP/1.1" 200 89543 "" "Prometheus/2.51.0" 2024-09-14T04:13:39.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:38 smithi039 ceph-mon[111235]: osdmap e857: 8 total, 8 up, 8 in 2024-09-14T04:13:40.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:39 smithi129 ceph-mon[101316]: osdmap e858: 8 total, 8 up, 8 in 2024-09-14T04:13:40.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:39 smithi129 ceph-mon[101316]: pgmap v949: 809 pgs: 809 active+clean; 2.6 MiB data, 552 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 800 B/s wr, 5 op/s 2024-09-14T04:13:40.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:39 smithi039 ceph-mon[106470]: osdmap e858: 8 total, 8 up, 8 in 2024-09-14T04:13:40.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:39 smithi039 ceph-mon[106470]: pgmap v949: 809 pgs: 809 active+clean; 2.6 MiB data, 552 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 800 B/s wr, 5 op/s 2024-09-14T04:13:40.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:39 smithi039 ceph-mon[111235]: osdmap e858: 8 total, 8 up, 8 in 2024-09-14T04:13:40.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:39 smithi039 ceph-mon[111235]: pgmap v949: 809 pgs: 809 active+clean; 2.6 MiB data, 552 MiB used, 715 GiB / 715 GiB avail; 3.9 KiB/s rd, 800 B/s wr, 5 op/s 2024-09-14T04:13:41.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:40 smithi129 ceph-mon[101316]: osdmap e859: 8 total, 8 up, 8 in 2024-09-14T04:13:41.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:40 smithi039 ceph-mon[106470]: osdmap e859: 8 total, 8 up, 8 in 2024-09-14T04:13:41.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:40 smithi039 ceph-mon[111235]: osdmap e859: 8 total, 8 up, 8 in 2024-09-14T04:13:41.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:41 smithi039 ceph-mon[106470]: osdmap e860: 8 total, 8 up, 8 in 2024-09-14T04:13:41.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:41 smithi039 ceph-mon[106470]: pgmap v952: 809 pgs: 809 active+clean; 2.6 MiB data, 553 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 3.5 KiB/s wr, 13 op/s 2024-09-14T04:13:41.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:41 smithi039 ceph-mon[111235]: osdmap e860: 8 total, 8 up, 8 in 2024-09-14T04:13:41.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:41 smithi039 ceph-mon[111235]: pgmap v952: 809 pgs: 809 active+clean; 2.6 MiB data, 553 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 3.5 KiB/s wr, 13 op/s 2024-09-14T04:13:42.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:41 smithi129 ceph-mon[101316]: osdmap e860: 8 total, 8 up, 8 in 2024-09-14T04:13:42.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:41 smithi129 ceph-mon[101316]: pgmap v952: 809 pgs: 809 active+clean; 2.6 MiB data, 553 MiB used, 715 GiB / 715 GiB avail; 10 KiB/s rd, 3.5 KiB/s wr, 13 op/s 2024-09-14T04:13:43.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:42 smithi129 ceph-mon[101316]: osdmap e861: 8 total, 8 up, 8 in 2024-09-14T04:13:43.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:42 smithi039 ceph-mon[106470]: osdmap e861: 8 total, 8 up, 8 in 2024-09-14T04:13:43.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:42 smithi039 ceph-mon[111235]: osdmap e861: 8 total, 8 up, 8 in 2024-09-14T04:13:44.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:43 smithi129 ceph-mon[101316]: osdmap e862: 8 total, 8 up, 8 in 2024-09-14T04:13:44.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:43 smithi129 ceph-mon[101316]: pgmap v955: 809 pgs: 809 active+clean; 2.6 MiB data, 553 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 3.0 KiB/s wr, 19 op/s 2024-09-14T04:13:44.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:43 smithi039 ceph-mon[106470]: osdmap e862: 8 total, 8 up, 8 in 2024-09-14T04:13:44.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:43 smithi039 ceph-mon[106470]: pgmap v955: 809 pgs: 809 active+clean; 2.6 MiB data, 553 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 3.0 KiB/s wr, 19 op/s 2024-09-14T04:13:44.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:43 smithi039 ceph-mon[111235]: osdmap e862: 8 total, 8 up, 8 in 2024-09-14T04:13:44.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:43 smithi039 ceph-mon[111235]: pgmap v955: 809 pgs: 809 active+clean; 2.6 MiB data, 553 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 3.0 KiB/s wr, 19 op/s 2024-09-14T04:13:45.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:44 smithi129 ceph-mon[101316]: osdmap e863: 8 total, 8 up, 8 in 2024-09-14T04:13:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:44 smithi039 ceph-mon[106470]: osdmap e863: 8 total, 8 up, 8 in 2024-09-14T04:13:45.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:44 smithi039 ceph-mon[111235]: osdmap e863: 8 total, 8 up, 8 in 2024-09-14T04:13:46.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:45 smithi129 ceph-mon[101316]: osdmap e864: 8 total, 8 up, 8 in 2024-09-14T04:13:46.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:45 smithi129 ceph-mon[101316]: pgmap v958: 809 pgs: 809 active+clean; 2.6 MiB data, 554 MiB used, 715 GiB / 715 GiB avail; 179 KiB/s rd, 5.7 KiB/s wr, 249 op/s 2024-09-14T04:13:46.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:45 smithi039 ceph-mon[106470]: osdmap e864: 8 total, 8 up, 8 in 2024-09-14T04:13:46.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:45 smithi039 ceph-mon[106470]: pgmap v958: 809 pgs: 809 active+clean; 2.6 MiB data, 554 MiB used, 715 GiB / 715 GiB avail; 179 KiB/s rd, 5.7 KiB/s wr, 249 op/s 2024-09-14T04:13:46.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:45 smithi039 ceph-mon[111235]: osdmap e864: 8 total, 8 up, 8 in 2024-09-14T04:13:46.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:45 smithi039 ceph-mon[111235]: pgmap v958: 809 pgs: 809 active+clean; 2.6 MiB data, 554 MiB used, 715 GiB / 715 GiB avail; 179 KiB/s rd, 5.7 KiB/s wr, 249 op/s 2024-09-14T04:13:47.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:46 smithi129 ceph-mon[101316]: osdmap e865: 8 total, 8 up, 8 in 2024-09-14T04:13:47.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:46 smithi039 ceph-mon[106470]: osdmap e865: 8 total, 8 up, 8 in 2024-09-14T04:13:47.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:46 smithi039 ceph-mon[111235]: osdmap e865: 8 total, 8 up, 8 in 2024-09-14T04:13:48.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:47 smithi129 ceph-mon[101316]: osdmap e866: 8 total, 8 up, 8 in 2024-09-14T04:13:48.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:47 smithi129 ceph-mon[101316]: pgmap v961: 809 pgs: 809 active+clean; 2.6 MiB data, 554 MiB used, 715 GiB / 715 GiB avail; 176 KiB/s rd, 5.5 KiB/s wr, 243 op/s 2024-09-14T04:13:48.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:47 smithi039 ceph-mon[106470]: osdmap e866: 8 total, 8 up, 8 in 2024-09-14T04:13:48.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:47 smithi039 ceph-mon[106470]: pgmap v961: 809 pgs: 809 active+clean; 2.6 MiB data, 554 MiB used, 715 GiB / 715 GiB avail; 176 KiB/s rd, 5.5 KiB/s wr, 243 op/s 2024-09-14T04:13:48.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:47 smithi039 ceph-mon[111235]: osdmap e866: 8 total, 8 up, 8 in 2024-09-14T04:13:48.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:47 smithi039 ceph-mon[111235]: pgmap v961: 809 pgs: 809 active+clean; 2.6 MiB data, 554 MiB used, 715 GiB / 715 GiB avail; 176 KiB/s rd, 5.5 KiB/s wr, 243 op/s 2024-09-14T04:13:49.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:48 smithi129 ceph-mon[101316]: osdmap e867: 8 total, 8 up, 8 in 2024-09-14T04:13:49.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:13:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:13:48] "GET /metrics HTTP/1.1" 200 89543 "" "Prometheus/2.51.0" 2024-09-14T04:13:49.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:48 smithi039 ceph-mon[106470]: osdmap e867: 8 total, 8 up, 8 in 2024-09-14T04:13:49.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:48 smithi039 ceph-mon[111235]: osdmap e867: 8 total, 8 up, 8 in 2024-09-14T04:13:50.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:49 smithi129 ceph-mon[101316]: osdmap e868: 8 total, 8 up, 8 in 2024-09-14T04:13:50.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:49 smithi129 ceph-mon[101316]: pgmap v964: 809 pgs: 809 active+clean; 2.6 MiB data, 558 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-14T04:13:50.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:49 smithi039 ceph-mon[106470]: osdmap e868: 8 total, 8 up, 8 in 2024-09-14T04:13:50.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:49 smithi039 ceph-mon[106470]: pgmap v964: 809 pgs: 809 active+clean; 2.6 MiB data, 558 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-14T04:13:50.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:49 smithi039 ceph-mon[111235]: osdmap e868: 8 total, 8 up, 8 in 2024-09-14T04:13:50.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:49 smithi039 ceph-mon[111235]: pgmap v964: 809 pgs: 809 active+clean; 2.6 MiB data, 558 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s rd, 0 B/s wr, 3 op/s 2024-09-14T04:13:51.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:50 smithi039 ceph-mon[106470]: osdmap e869: 8 total, 8 up, 8 in 2024-09-14T04:13:51.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:50 smithi039 ceph-mon[106470]: osdmap e870: 8 total, 8 up, 8 in 2024-09-14T04:13:51.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:50 smithi039 ceph-mon[111235]: osdmap e869: 8 total, 8 up, 8 in 2024-09-14T04:13:51.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:50 smithi039 ceph-mon[111235]: osdmap e870: 8 total, 8 up, 8 in 2024-09-14T04:13:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:50 smithi129 ceph-mon[101316]: osdmap e869: 8 total, 8 up, 8 in 2024-09-14T04:13:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:50 smithi129 ceph-mon[101316]: osdmap e870: 8 total, 8 up, 8 in 2024-09-14T04:13:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:51 smithi129 ceph-mon[101316]: pgmap v967: 809 pgs: 809 active+clean; 2.6 MiB data, 558 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 5.0 KiB/s wr, 22 op/s 2024-09-14T04:13:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:51 smithi129 ceph-mon[101316]: osdmap e871: 8 total, 8 up, 8 in 2024-09-14T04:13:52.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:51 smithi039 ceph-mon[106470]: pgmap v967: 809 pgs: 809 active+clean; 2.6 MiB data, 558 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 5.0 KiB/s wr, 22 op/s 2024-09-14T04:13:52.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:51 smithi039 ceph-mon[106470]: osdmap e871: 8 total, 8 up, 8 in 2024-09-14T04:13:52.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:51 smithi039 ceph-mon[111235]: pgmap v967: 809 pgs: 809 active+clean; 2.6 MiB data, 558 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 5.0 KiB/s wr, 22 op/s 2024-09-14T04:13:52.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:51 smithi039 ceph-mon[111235]: osdmap e871: 8 total, 8 up, 8 in 2024-09-14T04:13:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:13:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:52 smithi129 ceph-mon[101316]: osdmap e872: 8 total, 8 up, 8 in 2024-09-14T04:13:53.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:13:53.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:52 smithi039 ceph-mon[106470]: osdmap e872: 8 total, 8 up, 8 in 2024-09-14T04:13:53.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:13:53.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:52 smithi039 ceph-mon[111235]: osdmap e872: 8 total, 8 up, 8 in 2024-09-14T04:13:54.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:53 smithi129 ceph-mon[101316]: pgmap v970: 809 pgs: 809 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 5.2 KiB/s wr, 25 op/s 2024-09-14T04:13:54.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:53 smithi129 ceph-mon[101316]: osdmap e873: 8 total, 8 up, 8 in 2024-09-14T04:13:54.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:53 smithi039 ceph-mon[106470]: pgmap v970: 809 pgs: 809 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 5.2 KiB/s wr, 25 op/s 2024-09-14T04:13:54.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:53 smithi039 ceph-mon[106470]: osdmap e873: 8 total, 8 up, 8 in 2024-09-14T04:13:54.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:53 smithi039 ceph-mon[111235]: pgmap v970: 809 pgs: 809 active+clean; 2.6 MiB data, 567 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 5.2 KiB/s wr, 25 op/s 2024-09-14T04:13:54.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:53 smithi039 ceph-mon[111235]: osdmap e873: 8 total, 8 up, 8 in 2024-09-14T04:13:55.673 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestLibRBD.ConcurentOperations (25230 ms) 2024-09-14T04:13:56.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:55 smithi129 ceph-mon[101316]: osdmap e874: 8 total, 8 up, 8 in 2024-09-14T04:13:56.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:55 smithi129 ceph-mon[101316]: pgmap v973: 809 pgs: 809 active+clean; 2.6 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 66 KiB/s rd, 3.7 KiB/s wr, 86 op/s 2024-09-14T04:13:56.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:55 smithi129 ceph-mon[101316]: osdmap e875: 8 total, 8 up, 8 in 2024-09-14T04:13:56.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:55 smithi039 ceph-mon[106470]: osdmap e874: 8 total, 8 up, 8 in 2024-09-14T04:13:56.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:55 smithi039 ceph-mon[106470]: pgmap v973: 809 pgs: 809 active+clean; 2.6 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 66 KiB/s rd, 3.7 KiB/s wr, 86 op/s 2024-09-14T04:13:56.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:55 smithi039 ceph-mon[106470]: osdmap e875: 8 total, 8 up, 8 in 2024-09-14T04:13:56.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:55 smithi039 ceph-mon[111235]: osdmap e874: 8 total, 8 up, 8 in 2024-09-14T04:13:56.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:55 smithi039 ceph-mon[111235]: pgmap v973: 809 pgs: 809 active+clean; 2.6 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 66 KiB/s rd, 3.7 KiB/s wr, 86 op/s 2024-09-14T04:13:56.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:55 smithi039 ceph-mon[111235]: osdmap e875: 8 total, 8 up, 8 in 2024-09-14T04:13:58.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:57 smithi129 ceph-mon[101316]: osdmap e876: 8 total, 8 up, 8 in 2024-09-14T04:13:58.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:57 smithi129 ceph-mon[101316]: pgmap v976: 777 pgs: 777 active+clean; 2.6 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 767 B/s wr, 53 op/s 2024-09-14T04:13:58.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:57 smithi039 ceph-mon[106470]: osdmap e876: 8 total, 8 up, 8 in 2024-09-14T04:13:58.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:57 smithi039 ceph-mon[106470]: pgmap v976: 777 pgs: 777 active+clean; 2.6 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 767 B/s wr, 53 op/s 2024-09-14T04:13:58.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:57 smithi039 ceph-mon[111235]: osdmap e876: 8 total, 8 up, 8 in 2024-09-14T04:13:58.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:57 smithi039 ceph-mon[111235]: pgmap v976: 777 pgs: 777 active+clean; 2.6 MiB data, 572 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 767 B/s wr, 53 op/s 2024-09-14T04:13:59.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:58 smithi129 ceph-mon[101316]: osdmap e877: 8 total, 8 up, 8 in 2024-09-14T04:13:59.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:58 smithi039 ceph-mon[106470]: osdmap e877: 8 total, 8 up, 8 in 2024-09-14T04:13:59.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:13:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:13:58] "GET /metrics HTTP/1.1" 200 89560 "" "Prometheus/2.51.0" 2024-09-14T04:13:59.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:58 smithi039 ceph-mon[111235]: osdmap e877: 8 total, 8 up, 8 in 2024-09-14T04:13:59.941 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:59 smithi039 ceph-mon[106470]: osdmap e878: 8 total, 8 up, 8 in 2024-09-14T04:13:59.942 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:13:59 smithi039 ceph-mon[106470]: pgmap v979: 713 pgs: 713 active+clean; 2.6 MiB data, 598 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:13:59.942 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:59 smithi039 ceph-mon[111235]: osdmap e878: 8 total, 8 up, 8 in 2024-09-14T04:13:59.942 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:13:59 smithi039 ceph-mon[111235]: pgmap v979: 713 pgs: 713 active+clean; 2.6 MiB data, 598 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:00.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:59 smithi129 ceph-mon[101316]: osdmap e878: 8 total, 8 up, 8 in 2024-09-14T04:14:00.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:13:59 smithi129 ceph-mon[101316]: pgmap v979: 713 pgs: 713 active+clean; 2.6 MiB data, 598 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:01.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:00 smithi129 ceph-mon[101316]: osdmap e879: 8 total, 8 up, 8 in 2024-09-14T04:14:01.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:00 smithi129 ceph-mon[101316]: osdmap e880: 8 total, 8 up, 8 in 2024-09-14T04:14:01.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:00 smithi039 ceph-mon[106470]: osdmap e879: 8 total, 8 up, 8 in 2024-09-14T04:14:01.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:00 smithi039 ceph-mon[106470]: osdmap e880: 8 total, 8 up, 8 in 2024-09-14T04:14:01.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:00 smithi039 ceph-mon[111235]: osdmap e879: 8 total, 8 up, 8 in 2024-09-14T04:14:01.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:00 smithi039 ceph-mon[111235]: osdmap e880: 8 total, 8 up, 8 in 2024-09-14T04:14:02.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:01 smithi129 ceph-mon[101316]: pgmap v982: 649 pgs: 649 active+clean; 2.6 MiB data, 598 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:02.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:01 smithi129 ceph-mon[101316]: osdmap e881: 8 total, 8 up, 8 in 2024-09-14T04:14:02.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:01 smithi039 ceph-mon[106470]: pgmap v982: 649 pgs: 649 active+clean; 2.6 MiB data, 598 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:02.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:01 smithi039 ceph-mon[106470]: osdmap e881: 8 total, 8 up, 8 in 2024-09-14T04:14:02.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:01 smithi039 ceph-mon[111235]: pgmap v982: 649 pgs: 649 active+clean; 2.6 MiB data, 598 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:02.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:01 smithi039 ceph-mon[111235]: osdmap e881: 8 total, 8 up, 8 in 2024-09-14T04:14:04.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:03 smithi129 ceph-mon[101316]: osdmap e882: 8 total, 8 up, 8 in 2024-09-14T04:14:04.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:03 smithi129 ceph-mon[101316]: pgmap v985: 585 pgs: 585 active+clean; 2.6 MiB data, 582 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:04.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:03 smithi039 ceph-mon[106470]: osdmap e882: 8 total, 8 up, 8 in 2024-09-14T04:14:04.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:03 smithi039 ceph-mon[106470]: pgmap v985: 585 pgs: 585 active+clean; 2.6 MiB data, 582 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:04.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:03 smithi039 ceph-mon[111235]: osdmap e882: 8 total, 8 up, 8 in 2024-09-14T04:14:04.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:03 smithi039 ceph-mon[111235]: pgmap v985: 585 pgs: 585 active+clean; 2.6 MiB data, 582 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:05.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:04 smithi129 ceph-mon[101316]: osdmap e883: 8 total, 8 up, 8 in 2024-09-14T04:14:05.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:04 smithi039 ceph-mon[106470]: osdmap e883: 8 total, 8 up, 8 in 2024-09-14T04:14:05.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:04 smithi039 ceph-mon[111235]: osdmap e883: 8 total, 8 up, 8 in 2024-09-14T04:14:06.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:05 smithi129 ceph-mon[101316]: Health check update: 1 pool(s) full (POOL_FULL) 2024-09-14T04:14:06.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:05 smithi129 ceph-mon[101316]: osdmap e884: 8 total, 8 up, 8 in 2024-09-14T04:14:06.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:05 smithi129 ceph-mon[101316]: pgmap v988: 521 pgs: 521 active+clean; 2.6 MiB data, 586 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:06.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:05 smithi129 ceph-mon[101316]: osdmap e885: 8 total, 8 up, 8 in 2024-09-14T04:14:06.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:05 smithi039 ceph-mon[106470]: Health check update: 1 pool(s) full (POOL_FULL) 2024-09-14T04:14:06.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:05 smithi039 ceph-mon[106470]: osdmap e884: 8 total, 8 up, 8 in 2024-09-14T04:14:06.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:05 smithi039 ceph-mon[106470]: pgmap v988: 521 pgs: 521 active+clean; 2.6 MiB data, 586 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:06.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:05 smithi039 ceph-mon[106470]: osdmap e885: 8 total, 8 up, 8 in 2024-09-14T04:14:06.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:05 smithi039 ceph-mon[111235]: Health check update: 1 pool(s) full (POOL_FULL) 2024-09-14T04:14:06.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:05 smithi039 ceph-mon[111235]: osdmap e884: 8 total, 8 up, 8 in 2024-09-14T04:14:06.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:05 smithi039 ceph-mon[111235]: pgmap v988: 521 pgs: 521 active+clean; 2.6 MiB data, 586 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:06.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:05 smithi039 ceph-mon[111235]: osdmap e885: 8 total, 8 up, 8 in 2024-09-14T04:14:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:07 smithi129 ceph-mon[101316]: osdmap e886: 8 total, 8 up, 8 in 2024-09-14T04:14:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:07 smithi129 ceph-mon[101316]: pgmap v991: 457 pgs: 457 active+clean; 2.6 MiB data, 586 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:14:08.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:07 smithi039 ceph-mon[106470]: osdmap e886: 8 total, 8 up, 8 in 2024-09-14T04:14:08.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:07 smithi039 ceph-mon[106470]: pgmap v991: 457 pgs: 457 active+clean; 2.6 MiB data, 586 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:08.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:14:08.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:07 smithi039 ceph-mon[111235]: osdmap e886: 8 total, 8 up, 8 in 2024-09-14T04:14:08.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:07 smithi039 ceph-mon[111235]: pgmap v991: 457 pgs: 457 active+clean; 2.6 MiB data, 586 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:08.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:14:09.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:08 smithi129 ceph-mon[101316]: Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-09-14T04:14:09.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:08 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T04:14:09.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:08 smithi129 ceph-mon[101316]: osdmap e887: 8 total, 8 up, 8 in 2024-09-14T04:14:09.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:08 smithi129 ceph-mon[101316]: osdmap e888: 8 total, 8 up, 8 in 2024-09-14T04:14:09.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:14:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:14:08] "GET /metrics HTTP/1.1" 200 63075 "" "Prometheus/2.51.0" 2024-09-14T04:14:09.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:08 smithi039 ceph-mon[106470]: Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-09-14T04:14:09.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:08 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T04:14:09.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:08 smithi039 ceph-mon[106470]: osdmap e887: 8 total, 8 up, 8 in 2024-09-14T04:14:09.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:08 smithi039 ceph-mon[106470]: osdmap e888: 8 total, 8 up, 8 in 2024-09-14T04:14:09.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:08 smithi039 ceph-mon[111235]: Health check cleared: POOL_FULL (was: 1 pool(s) full) 2024-09-14T04:14:09.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:08 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T04:14:09.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:08 smithi039 ceph-mon[111235]: osdmap e887: 8 total, 8 up, 8 in 2024-09-14T04:14:09.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:08 smithi039 ceph-mon[111235]: osdmap e888: 8 total, 8 up, 8 in 2024-09-14T04:14:10.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:09 smithi129 ceph-mon[101316]: pgmap v994: 393 pgs: 393 active+clean; 2.6 MiB data, 571 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:10.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:09 smithi129 ceph-mon[101316]: osdmap e889: 8 total, 8 up, 8 in 2024-09-14T04:14:10.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:09 smithi039 ceph-mon[106470]: pgmap v994: 393 pgs: 393 active+clean; 2.6 MiB data, 571 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:10.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:09 smithi039 ceph-mon[106470]: osdmap e889: 8 total, 8 up, 8 in 2024-09-14T04:14:10.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:09 smithi039 ceph-mon[111235]: pgmap v994: 393 pgs: 393 active+clean; 2.6 MiB data, 571 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:10.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:09 smithi039 ceph-mon[111235]: osdmap e889: 8 total, 8 up, 8 in 2024-09-14T04:14:12.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:11 smithi129 ceph-mon[101316]: osdmap e890: 8 total, 8 up, 8 in 2024-09-14T04:14:12.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:11 smithi129 ceph-mon[101316]: pgmap v997: 329 pgs: 329 active+clean; 587 KiB data, 571 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:12.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:11 smithi039 ceph-mon[106470]: osdmap e890: 8 total, 8 up, 8 in 2024-09-14T04:14:12.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:11 smithi039 ceph-mon[106470]: pgmap v997: 329 pgs: 329 active+clean; 587 KiB data, 571 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:12.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:11 smithi039 ceph-mon[111235]: osdmap e890: 8 total, 8 up, 8 in 2024-09-14T04:14:12.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:11 smithi039 ceph-mon[111235]: pgmap v997: 329 pgs: 329 active+clean; 587 KiB data, 571 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:13.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:12 smithi129 ceph-mon[101316]: osdmap e891: 8 total, 8 up, 8 in 2024-09-14T04:14:13.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:12 smithi039 ceph-mon[106470]: osdmap e891: 8 total, 8 up, 8 in 2024-09-14T04:14:13.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:12 smithi039 ceph-mon[111235]: osdmap e891: 8 total, 8 up, 8 in 2024-09-14T04:14:14.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:13 smithi129 ceph-mon[101316]: osdmap e892: 8 total, 8 up, 8 in 2024-09-14T04:14:14.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:13 smithi129 ceph-mon[101316]: pgmap v1000: 265 pgs: 265 active+clean; 587 KiB data, 576 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:14.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:13 smithi129 ceph-mon[101316]: osdmap e893: 8 total, 8 up, 8 in 2024-09-14T04:14:14.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:13 smithi039 ceph-mon[111235]: osdmap e892: 8 total, 8 up, 8 in 2024-09-14T04:14:14.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:13 smithi039 ceph-mon[111235]: pgmap v1000: 265 pgs: 265 active+clean; 587 KiB data, 576 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:14.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:13 smithi039 ceph-mon[111235]: osdmap e893: 8 total, 8 up, 8 in 2024-09-14T04:14:14.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:13 smithi039 ceph-mon[106470]: osdmap e892: 8 total, 8 up, 8 in 2024-09-14T04:14:14.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:13 smithi039 ceph-mon[106470]: pgmap v1000: 265 pgs: 265 active+clean; 587 KiB data, 576 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:14.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:13 smithi039 ceph-mon[106470]: osdmap e893: 8 total, 8 up, 8 in 2024-09-14T04:14:16.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:15 smithi129 ceph-mon[101316]: osdmap e894: 8 total, 8 up, 8 in 2024-09-14T04:14:16.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:15 smithi129 ceph-mon[101316]: pgmap v1003: 201 pgs: 201 active+clean; 587 KiB data, 576 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:16.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:15 smithi039 ceph-mon[106470]: osdmap e894: 8 total, 8 up, 8 in 2024-09-14T04:14:16.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:15 smithi039 ceph-mon[106470]: pgmap v1003: 201 pgs: 201 active+clean; 587 KiB data, 576 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:16.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:15 smithi039 ceph-mon[111235]: osdmap e894: 8 total, 8 up, 8 in 2024-09-14T04:14:16.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:15 smithi039 ceph-mon[111235]: pgmap v1003: 201 pgs: 201 active+clean; 587 KiB data, 576 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:16 smithi129 ceph-mon[101316]: osdmap e895: 8 total, 8 up, 8 in 2024-09-14T04:14:17.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:16 smithi039 ceph-mon[106470]: osdmap e895: 8 total, 8 up, 8 in 2024-09-14T04:14:17.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:16 smithi039 ceph-mon[111235]: osdmap e895: 8 total, 8 up, 8 in 2024-09-14T04:14:17.923 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:17 smithi129 ceph-mon[101316]: osdmap e896: 8 total, 8 up, 8 in 2024-09-14T04:14:17.923 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:17 smithi129 ceph-mon[101316]: pgmap v1006: 137 pgs: 137 active+clean; 586 KiB data, 576 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:18.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:17 smithi039 ceph-mon[106470]: osdmap e896: 8 total, 8 up, 8 in 2024-09-14T04:14:18.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:17 smithi039 ceph-mon[106470]: pgmap v1006: 137 pgs: 137 active+clean; 586 KiB data, 576 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:18.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:17 smithi039 ceph-mon[111235]: osdmap e896: 8 total, 8 up, 8 in 2024-09-14T04:14:18.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:17 smithi039 ceph-mon[111235]: pgmap v1006: 137 pgs: 137 active+clean; 586 KiB data, 576 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:18.659 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 170 tests from TestLibRBD (648174 ms total) 2024-09-14T04:14:18.659 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:14:18.660 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 7 tests from EncryptedFlattenTest/0, where TypeParam = PlaintextUnderLUKS1 2024-09-14T04:14:18.691 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/0.Simple 2024-09-14T04:14:19.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:18 smithi129 ceph-mon[101316]: osdmap e897: 8 total, 8 up, 8 in 2024-09-14T04:14:19.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:18 smithi039 ceph-mon[106470]: osdmap e897: 8 total, 8 up, 8 in 2024-09-14T04:14:19.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:18 smithi039 ceph-mon[111235]: osdmap e897: 8 total, 8 up, 8 in 2024-09-14T04:14:19.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:14:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:14:18] "GET /metrics HTTP/1.1" 200 63075 "" "Prometheus/2.51.0" 2024-09-14T04:14:20.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:19 smithi129 ceph-mon[101316]: osdmap e898: 8 total, 8 up, 8 in 2024-09-14T04:14:20.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:19 smithi129 ceph-mon[101316]: pgmap v1009: 73 pgs: 73 active+clean; 582 KiB data, 576 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:20.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:19 smithi039 ceph-mon[106470]: osdmap e898: 8 total, 8 up, 8 in 2024-09-14T04:14:20.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:19 smithi039 ceph-mon[106470]: pgmap v1009: 73 pgs: 73 active+clean; 582 KiB data, 576 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:20.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:19 smithi039 ceph-mon[111235]: osdmap e898: 8 total, 8 up, 8 in 2024-09-14T04:14:20.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:19 smithi039 ceph-mon[111235]: pgmap v1009: 73 pgs: 73 active+clean; 582 KiB data, 576 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:20.695 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:14:21.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:20 smithi129 ceph-mon[101316]: osdmap e899: 8 total, 8 up, 8 in 2024-09-14T04:14:21.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:20 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/497636167' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-39","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:14:21.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:20 smithi039 ceph-mon[106470]: osdmap e899: 8 total, 8 up, 8 in 2024-09-14T04:14:21.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:20 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/497636167' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-39","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:14:21.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:20 smithi039 ceph-mon[111235]: osdmap e899: 8 total, 8 up, 8 in 2024-09-14T04:14:21.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:20 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/497636167' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-39","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:14:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:21 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/497636167' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-39","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:14:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:21 smithi129 ceph-mon[101316]: osdmap e900: 8 total, 8 up, 8 in 2024-09-14T04:14:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:21 smithi129 ceph-mon[101316]: pgmap v1012: 105 pgs: 4 creating+peering, 28 unknown, 73 active+clean; 582 KiB data, 576 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:22.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:21 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/497636167' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-39","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:14:22.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:21 smithi039 ceph-mon[106470]: osdmap e900: 8 total, 8 up, 8 in 2024-09-14T04:14:22.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:21 smithi039 ceph-mon[106470]: pgmap v1012: 105 pgs: 4 creating+peering, 28 unknown, 73 active+clean; 582 KiB data, 576 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:22.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:21 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/497636167' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-39","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:14:22.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:21 smithi039 ceph-mon[111235]: osdmap e900: 8 total, 8 up, 8 in 2024-09-14T04:14:22.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:21 smithi039 ceph-mon[111235]: pgmap v1012: 105 pgs: 4 creating+peering, 28 unknown, 73 active+clean; 582 KiB data, 576 MiB used, 715 GiB / 715 GiB avail 2024-09-14T04:14:23.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:22 smithi129 ceph-mon[101316]: osdmap e901: 8 total, 8 up, 8 in 2024-09-14T04:14:23.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:22 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:14:23.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:14:23.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:22 smithi039 ceph-mon[106470]: osdmap e901: 8 total, 8 up, 8 in 2024-09-14T04:14:23.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:22 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:14:23.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:14:23.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:22 smithi039 ceph-mon[111235]: osdmap e901: 8 total, 8 up, 8 in 2024-09-14T04:14:23.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:22 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:14:23.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:14:24.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:23 smithi129 ceph-mon[101316]: osdmap e902: 8 total, 8 up, 8 in 2024-09-14T04:14:24.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:14:24.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:23 smithi129 ceph-mon[101316]: pgmap v1015: 105 pgs: 1 creating+activating, 4 creating+peering, 4 unknown, 96 active+clean; 17 MiB data, 577 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 4.0 MiB/s wr, 20 op/s 2024-09-14T04:14:24.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:23 smithi039 ceph-mon[106470]: osdmap e902: 8 total, 8 up, 8 in 2024-09-14T04:14:24.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:14:24.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:23 smithi039 ceph-mon[106470]: pgmap v1015: 105 pgs: 1 creating+activating, 4 creating+peering, 4 unknown, 96 active+clean; 17 MiB data, 577 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 4.0 MiB/s wr, 20 op/s 2024-09-14T04:14:24.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:23 smithi039 ceph-mon[111235]: osdmap e902: 8 total, 8 up, 8 in 2024-09-14T04:14:24.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:14:24.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:23 smithi039 ceph-mon[111235]: pgmap v1015: 105 pgs: 1 creating+activating, 4 creating+peering, 4 unknown, 96 active+clean; 17 MiB data, 577 MiB used, 715 GiB / 715 GiB avail; 11 KiB/s rd, 4.0 MiB/s wr, 20 op/s 2024-09-14T04:14:24.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:24 smithi039 ceph-mon[106470]: osdmap e903: 8 total, 8 up, 8 in 2024-09-14T04:14:24.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:24 smithi039 ceph-mon[111235]: osdmap e903: 8 total, 8 up, 8 in 2024-09-14T04:14:25.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:24 smithi129 ceph-mon[101316]: osdmap e903: 8 total, 8 up, 8 in 2024-09-14T04:14:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:25 smithi129 ceph-mon[101316]: pgmap v1017: 105 pgs: 1 creating+activating, 104 active+clean; 22 MiB data, 594 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 4.7 MiB/s wr, 63 op/s 2024-09-14T04:14:26.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:25 smithi039 ceph-mon[106470]: pgmap v1017: 105 pgs: 1 creating+activating, 104 active+clean; 22 MiB data, 594 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 4.7 MiB/s wr, 63 op/s 2024-09-14T04:14:26.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:25 smithi039 ceph-mon[111235]: pgmap v1017: 105 pgs: 1 creating+activating, 104 active+clean; 22 MiB data, 594 MiB used, 715 GiB / 715 GiB avail; 46 KiB/s rd, 4.7 MiB/s wr, 63 op/s 2024-09-14T04:14:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:27 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:14:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:27 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:14:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:27 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:14:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:27 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:14:27.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:27 smithi129 ceph-mon[101316]: pgmap v1018: 105 pgs: 105 active+clean; 26 MiB data, 598 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 4.2 MiB/s wr, 47 op/s 2024-09-14T04:14:27.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:27 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:14:27.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:27 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:14:27.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:27 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:14:27.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:27 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:14:27.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:27 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:14:27.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:27 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:14:27.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:27 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:14:27.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:27 smithi039 ceph-mon[111235]: pgmap v1018: 105 pgs: 105 active+clean; 26 MiB data, 598 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 4.2 MiB/s wr, 47 op/s 2024-09-14T04:14:27.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:27 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:14:27.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:27 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:14:27.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:27 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:14:27.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:27 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:14:27.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:27 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:14:27.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:27 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:14:27.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:27 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:14:27.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:27 smithi039 ceph-mon[106470]: pgmap v1018: 105 pgs: 105 active+clean; 26 MiB data, 598 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 4.2 MiB/s wr, 47 op/s 2024-09-14T04:14:27.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:27 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:14:27.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:27 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:14:27.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:27 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:14:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:14:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:14:28] "GET /metrics HTTP/1.1" 200 36613 "" "Prometheus/2.51.0" 2024-09-14T04:14:30.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:30 smithi129 ceph-mon[101316]: pgmap v1019: 105 pgs: 105 active+clean; 30 MiB data, 660 MiB used, 715 GiB / 715 GiB avail; 79 KiB/s rd, 3.9 MiB/s wr, 55 op/s 2024-09-14T04:14:30.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:30 smithi039 ceph-mon[106470]: pgmap v1019: 105 pgs: 105 active+clean; 30 MiB data, 660 MiB used, 715 GiB / 715 GiB avail; 79 KiB/s rd, 3.9 MiB/s wr, 55 op/s 2024-09-14T04:14:30.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:30 smithi039 ceph-mon[111235]: pgmap v1019: 105 pgs: 105 active+clean; 30 MiB data, 660 MiB used, 715 GiB / 715 GiB avail; 79 KiB/s rd, 3.9 MiB/s wr, 55 op/s 2024-09-14T04:14:31.646 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/0.Simple (12956 ms) 2024-09-14T04:14:31.647 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/0.Grow 2024-09-14T04:14:31.647 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:14:31.843 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:31 smithi039 ceph-mon[106470]: osdmap e904: 8 total, 8 up, 8 in 2024-09-14T04:14:31.843 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:31 smithi039 ceph-mon[106470]: pgmap v1021: 105 pgs: 105 active+clean; 39 MiB data, 681 MiB used, 715 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.8 MiB/s wr, 47 op/s 2024-09-14T04:14:31.844 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:31 smithi039 ceph-mon[111235]: osdmap e904: 8 total, 8 up, 8 in 2024-09-14T04:14:31.844 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:31 smithi039 ceph-mon[111235]: pgmap v1021: 105 pgs: 105 active+clean; 39 MiB data, 681 MiB used, 715 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.8 MiB/s wr, 47 op/s 2024-09-14T04:14:32.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:31 smithi129 ceph-mon[101316]: osdmap e904: 8 total, 8 up, 8 in 2024-09-14T04:14:32.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:31 smithi129 ceph-mon[101316]: pgmap v1021: 105 pgs: 105 active+clean; 39 MiB data, 681 MiB used, 715 GiB / 715 GiB avail; 2.4 MiB/s rd, 2.8 MiB/s wr, 47 op/s 2024-09-14T04:14:34.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:33 smithi129 ceph-mon[101316]: osdmap e905: 8 total, 8 up, 8 in 2024-09-14T04:14:34.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:33 smithi129 ceph-mon[101316]: pgmap v1023: 105 pgs: 105 active+clean; 72 MiB data, 781 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.8 MiB/s wr, 71 op/s 2024-09-14T04:14:34.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:33 smithi039 ceph-mon[111235]: osdmap e905: 8 total, 8 up, 8 in 2024-09-14T04:14:34.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:33 smithi039 ceph-mon[111235]: pgmap v1023: 105 pgs: 105 active+clean; 72 MiB data, 781 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.8 MiB/s wr, 71 op/s 2024-09-14T04:14:34.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:33 smithi039 ceph-mon[106470]: osdmap e905: 8 total, 8 up, 8 in 2024-09-14T04:14:34.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:33 smithi039 ceph-mon[106470]: pgmap v1023: 105 pgs: 105 active+clean; 72 MiB data, 781 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.8 MiB/s wr, 71 op/s 2024-09-14T04:14:36.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:36 smithi129 ceph-mon[101316]: pgmap v1024: 105 pgs: 105 active+clean; 76 MiB data, 801 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 6.8 MiB/s wr, 72 op/s 2024-09-14T04:14:36.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:36 smithi039 ceph-mon[106470]: pgmap v1024: 105 pgs: 105 active+clean; 76 MiB data, 801 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 6.8 MiB/s wr, 72 op/s 2024-09-14T04:14:36.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:36 smithi039 ceph-mon[111235]: pgmap v1024: 105 pgs: 105 active+clean; 76 MiB data, 801 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 6.8 MiB/s wr, 72 op/s 2024-09-14T04:14:37.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:37 smithi129 ceph-mon[101316]: pgmap v1025: 105 pgs: 105 active+clean; 80 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 6.8 MiB/s wr, 78 op/s 2024-09-14T04:14:37.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:37 smithi039 ceph-mon[106470]: pgmap v1025: 105 pgs: 105 active+clean; 80 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 6.8 MiB/s wr, 78 op/s 2024-09-14T04:14:37.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:37 smithi039 ceph-mon[111235]: pgmap v1025: 105 pgs: 105 active+clean; 80 MiB data, 814 MiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 6.8 MiB/s wr, 78 op/s 2024-09-14T04:14:38.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:38 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:14:38.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:38 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:14:38.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:38 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:14:39.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:14:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:14:38] "GET /metrics HTTP/1.1" 200 36677 "" "Prometheus/2.51.0" 2024-09-14T04:14:39.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:39 smithi129 ceph-mon[101316]: pgmap v1026: 105 pgs: 105 active+clean; 80 MiB data, 822 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.3 MiB/s wr, 73 op/s 2024-09-14T04:14:39.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:39 smithi039 ceph-mon[111235]: pgmap v1026: 105 pgs: 105 active+clean; 80 MiB data, 822 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.3 MiB/s wr, 73 op/s 2024-09-14T04:14:39.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:39 smithi039 ceph-mon[106470]: pgmap v1026: 105 pgs: 105 active+clean; 80 MiB data, 822 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.3 MiB/s wr, 73 op/s 2024-09-14T04:14:40.660 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/0.Grow (9014 ms) 2024-09-14T04:14:40.660 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/0.Shrink 2024-09-14T04:14:40.660 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:14:42.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:41 smithi129 ceph-mon[101316]: osdmap e906: 8 total, 8 up, 8 in 2024-09-14T04:14:42.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:41 smithi129 ceph-mon[101316]: pgmap v1028: 105 pgs: 105 active+clean; 93 MiB data, 855 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 6.8 MiB/s wr, 65 op/s 2024-09-14T04:14:42.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:41 smithi039 ceph-mon[106470]: osdmap e906: 8 total, 8 up, 8 in 2024-09-14T04:14:42.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:41 smithi039 ceph-mon[106470]: pgmap v1028: 105 pgs: 105 active+clean; 93 MiB data, 855 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 6.8 MiB/s wr, 65 op/s 2024-09-14T04:14:42.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:41 smithi039 ceph-mon[111235]: osdmap e906: 8 total, 8 up, 8 in 2024-09-14T04:14:42.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:41 smithi039 ceph-mon[111235]: pgmap v1028: 105 pgs: 105 active+clean; 93 MiB data, 855 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 6.8 MiB/s wr, 65 op/s 2024-09-14T04:14:44.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:44 smithi129 ceph-mon[101316]: pgmap v1029: 105 pgs: 105 active+clean; 134 MiB data, 968 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.6 MiB/s wr, 79 op/s 2024-09-14T04:14:44.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:44 smithi039 ceph-mon[106470]: pgmap v1029: 105 pgs: 105 active+clean; 134 MiB data, 968 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.6 MiB/s wr, 79 op/s 2024-09-14T04:14:44.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:44 smithi039 ceph-mon[111235]: pgmap v1029: 105 pgs: 105 active+clean; 134 MiB data, 968 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.6 MiB/s wr, 79 op/s 2024-09-14T04:14:45.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:45 smithi129 ceph-mon[101316]: pgmap v1030: 105 pgs: 105 active+clean; 134 MiB data, 977 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.2 MiB/s wr, 80 op/s 2024-09-14T04:14:45.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:45 smithi039 ceph-mon[106470]: pgmap v1030: 105 pgs: 105 active+clean; 134 MiB data, 977 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.2 MiB/s wr, 80 op/s 2024-09-14T04:14:45.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:45 smithi039 ceph-mon[111235]: pgmap v1030: 105 pgs: 105 active+clean; 134 MiB data, 977 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.2 MiB/s wr, 80 op/s 2024-09-14T04:14:48.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:48 smithi129 ceph-mon[101316]: pgmap v1031: 105 pgs: 105 active+clean; 134 MiB data, 977 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.8 MiB/s wr, 64 op/s 2024-09-14T04:14:48.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:48 smithi039 ceph-mon[111235]: pgmap v1031: 105 pgs: 105 active+clean; 134 MiB data, 977 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.8 MiB/s wr, 64 op/s 2024-09-14T04:14:48.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:48 smithi039 ceph-mon[106470]: pgmap v1031: 105 pgs: 105 active+clean; 134 MiB data, 977 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.8 MiB/s wr, 64 op/s 2024-09-14T04:14:48.717 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/0.Shrink (8056 ms) 2024-09-14T04:14:48.717 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/0.ShrinkToOne 2024-09-14T04:14:48.717 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:14:49.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:14:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:14:48] "GET /metrics HTTP/1.1" 200 36677 "" "Prometheus/2.51.0" 2024-09-14T04:14:49.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:49 smithi129 ceph-mon[101316]: pgmap v1032: 105 pgs: 105 active+clean; 155 MiB data, 1020 MiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 8.3 MiB/s wr, 77 op/s 2024-09-14T04:14:49.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:49 smithi039 ceph-mon[111235]: pgmap v1032: 105 pgs: 105 active+clean; 155 MiB data, 1020 MiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 8.3 MiB/s wr, 77 op/s 2024-09-14T04:14:49.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:49 smithi039 ceph-mon[106470]: pgmap v1032: 105 pgs: 105 active+clean; 155 MiB data, 1020 MiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 8.3 MiB/s wr, 77 op/s 2024-09-14T04:14:50.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:50 smithi129 ceph-mon[101316]: osdmap e907: 8 total, 8 up, 8 in 2024-09-14T04:14:50.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:50 smithi039 ceph-mon[106470]: osdmap e907: 8 total, 8 up, 8 in 2024-09-14T04:14:50.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:50 smithi039 ceph-mon[111235]: osdmap e907: 8 total, 8 up, 8 in 2024-09-14T04:14:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:51 smithi129 ceph-mon[101316]: pgmap v1034: 105 pgs: 105 active+clean; 171 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 8.2 MiB/s wr, 109 op/s 2024-09-14T04:14:51.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:51 smithi039 ceph-mon[106470]: pgmap v1034: 105 pgs: 105 active+clean; 171 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 8.2 MiB/s wr, 109 op/s 2024-09-14T04:14:51.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:51 smithi039 ceph-mon[111235]: pgmap v1034: 105 pgs: 105 active+clean; 171 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 8.2 MiB/s wr, 109 op/s 2024-09-14T04:14:52.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:14:52.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:14:52.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:14:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:53 smithi129 ceph-mon[101316]: pgmap v1035: 105 pgs: 105 active+clean; 192 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.2 MiB/s wr, 71 op/s 2024-09-14T04:14:53.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:53 smithi039 ceph-mon[106470]: pgmap v1035: 105 pgs: 105 active+clean; 192 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.2 MiB/s wr, 71 op/s 2024-09-14T04:14:53.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:53 smithi039 ceph-mon[111235]: pgmap v1035: 105 pgs: 105 active+clean; 192 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.2 MiB/s wr, 71 op/s 2024-09-14T04:14:56.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:56 smithi129 ceph-mon[101316]: pgmap v1036: 105 pgs: 105 active+clean; 192 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.2 MiB/s wr, 72 op/s 2024-09-14T04:14:56.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:56 smithi039 ceph-mon[106470]: pgmap v1036: 105 pgs: 105 active+clean; 192 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.2 MiB/s wr, 72 op/s 2024-09-14T04:14:56.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:56 smithi039 ceph-mon[111235]: pgmap v1036: 105 pgs: 105 active+clean; 192 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.2 MiB/s wr, 72 op/s 2024-09-14T04:14:57.043 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/0.ShrinkToOne (8326 ms) 2024-09-14T04:14:57.043 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/0.ShrinkToOneAfterSnapshot 2024-09-14T04:14:57.043 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:14:57.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:57 smithi129 ceph-mon[101316]: pgmap v1037: 105 pgs: 105 active+clean; 196 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 7.0 MiB/s wr, 73 op/s 2024-09-14T04:14:57.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:57 smithi039 ceph-mon[106470]: pgmap v1037: 105 pgs: 105 active+clean; 196 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 7.0 MiB/s wr, 73 op/s 2024-09-14T04:14:57.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:57 smithi039 ceph-mon[111235]: pgmap v1037: 105 pgs: 105 active+clean; 196 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 7.0 MiB/s wr, 73 op/s 2024-09-14T04:14:58.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:58 smithi129 ceph-mon[101316]: osdmap e908: 8 total, 8 up, 8 in 2024-09-14T04:14:58.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:58 smithi039 ceph-mon[106470]: osdmap e908: 8 total, 8 up, 8 in 2024-09-14T04:14:58.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:58 smithi039 ceph-mon[111235]: osdmap e908: 8 total, 8 up, 8 in 2024-09-14T04:14:59.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:14:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:14:58] "GET /metrics HTTP/1.1" 200 36682 "" "Prometheus/2.51.0" 2024-09-14T04:14:59.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:14:59 smithi129 ceph-mon[101316]: pgmap v1039: 105 pgs: 105 active+clean; 217 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 7.1 MiB/s wr, 81 op/s 2024-09-14T04:14:59.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:14:59 smithi039 ceph-mon[106470]: pgmap v1039: 105 pgs: 105 active+clean; 217 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 7.1 MiB/s wr, 81 op/s 2024-09-14T04:14:59.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:14:59 smithi039 ceph-mon[111235]: pgmap v1039: 105 pgs: 105 active+clean; 217 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 7.1 MiB/s wr, 81 op/s 2024-09-14T04:15:02.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:02 smithi129 ceph-mon[101316]: pgmap v1040: 105 pgs: 105 active+clean; 221 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 5.8 MiB/s wr, 47 op/s 2024-09-14T04:15:02.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:02 smithi039 ceph-mon[106470]: pgmap v1040: 105 pgs: 105 active+clean; 221 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 5.8 MiB/s wr, 47 op/s 2024-09-14T04:15:02.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:02 smithi039 ceph-mon[111235]: pgmap v1040: 105 pgs: 105 active+clean; 221 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 5.8 MiB/s wr, 47 op/s 2024-09-14T04:15:03.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:03 smithi129 ceph-mon[101316]: pgmap v1041: 105 pgs: 105 active+clean; 221 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 950 KiB/s rd, 3.7 MiB/s wr, 60 op/s 2024-09-14T04:15:03.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:03 smithi039 ceph-mon[106470]: pgmap v1041: 105 pgs: 105 active+clean; 221 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 950 KiB/s rd, 3.7 MiB/s wr, 60 op/s 2024-09-14T04:15:03.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:03 smithi039 ceph-mon[111235]: pgmap v1041: 105 pgs: 105 active+clean; 221 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 950 KiB/s rd, 3.7 MiB/s wr, 60 op/s 2024-09-14T04:15:04.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:04 smithi129 ceph-mon[101316]: osdmap e909: 8 total, 8 up, 8 in 2024-09-14T04:15:04.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:04 smithi039 ceph-mon[106470]: osdmap e909: 8 total, 8 up, 8 in 2024-09-14T04:15:04.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:04 smithi039 ceph-mon[111235]: osdmap e909: 8 total, 8 up, 8 in 2024-09-14T04:15:05.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:05 smithi129 ceph-mon[101316]: pgmap v1043: 105 pgs: 105 active+clean; 237 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 5.6 MiB/s wr, 71 op/s 2024-09-14T04:15:05.535 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/0.ShrinkToOneAfterSnapshot (8492 ms) 2024-09-14T04:15:05.535 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/0.MinOverlap 2024-09-14T04:15:05.535 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:15:05.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:05 smithi039 ceph-mon[106470]: pgmap v1043: 105 pgs: 105 active+clean; 237 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 5.6 MiB/s wr, 71 op/s 2024-09-14T04:15:05.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:05 smithi039 ceph-mon[111235]: pgmap v1043: 105 pgs: 105 active+clean; 237 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 5.6 MiB/s wr, 71 op/s 2024-09-14T04:15:07.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:07 smithi129 ceph-mon[101316]: osdmap e910: 8 total, 8 up, 8 in 2024-09-14T04:15:07.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:07 smithi129 ceph-mon[101316]: pgmap v1045: 105 pgs: 105 active+clean; 245 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 3.5 MiB/s wr, 57 op/s 2024-09-14T04:15:07.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:07 smithi039 ceph-mon[106470]: osdmap e910: 8 total, 8 up, 8 in 2024-09-14T04:15:07.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:07 smithi039 ceph-mon[106470]: pgmap v1045: 105 pgs: 105 active+clean; 245 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 3.5 MiB/s wr, 57 op/s 2024-09-14T04:15:07.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:07 smithi039 ceph-mon[111235]: osdmap e910: 8 total, 8 up, 8 in 2024-09-14T04:15:07.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:07 smithi039 ceph-mon[111235]: pgmap v1045: 105 pgs: 105 active+clean; 245 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 3.5 MiB/s wr, 57 op/s 2024-09-14T04:15:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:08 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:15:08.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:08 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:15:08.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:08 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:15:09.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:15:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:15:08] "GET /metrics HTTP/1.1" 200 36686 "" "Prometheus/2.51.0" 2024-09-14T04:15:09.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:09 smithi129 ceph-mon[101316]: pgmap v1046: 105 pgs: 105 active+clean; 303 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 10 MiB/s wr, 95 op/s 2024-09-14T04:15:09.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:09 smithi039 ceph-mon[106470]: pgmap v1046: 105 pgs: 105 active+clean; 303 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 10 MiB/s wr, 95 op/s 2024-09-14T04:15:09.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:09 smithi039 ceph-mon[111235]: pgmap v1046: 105 pgs: 105 active+clean; 303 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 10 MiB/s wr, 95 op/s 2024-09-14T04:15:12.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:12 smithi129 ceph-mon[101316]: pgmap v1047: 105 pgs: 105 active+clean; 307 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 11 MiB/s wr, 83 op/s 2024-09-14T04:15:12.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:12 smithi039 ceph-mon[106470]: pgmap v1047: 105 pgs: 105 active+clean; 307 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 11 MiB/s wr, 83 op/s 2024-09-14T04:15:12.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:12 smithi039 ceph-mon[111235]: pgmap v1047: 105 pgs: 105 active+clean; 307 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 11 MiB/s wr, 83 op/s 2024-09-14T04:15:13.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:13 smithi129 ceph-mon[101316]: pgmap v1048: 105 pgs: 105 active+clean; 315 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 9.9 MiB/s wr, 85 op/s 2024-09-14T04:15:13.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:13 smithi039 ceph-mon[106470]: pgmap v1048: 105 pgs: 105 active+clean; 315 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 9.9 MiB/s wr, 85 op/s 2024-09-14T04:15:13.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:13 smithi039 ceph-mon[111235]: pgmap v1048: 105 pgs: 105 active+clean; 315 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 9.9 MiB/s wr, 85 op/s 2024-09-14T04:15:14.100 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/0.MinOverlap (8564 ms) 2024-09-14T04:15:14.100 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/0.ZeroOverlap 2024-09-14T04:15:14.100 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:15:15.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:15 smithi129 ceph-mon[101316]: osdmap e911: 8 total, 8 up, 8 in 2024-09-14T04:15:15.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:15 smithi129 ceph-mon[101316]: pgmap v1050: 105 pgs: 105 active+clean; 316 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 9.3 MiB/s wr, 106 op/s 2024-09-14T04:15:15.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:15 smithi039 ceph-mon[106470]: osdmap e911: 8 total, 8 up, 8 in 2024-09-14T04:15:15.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:15 smithi039 ceph-mon[106470]: pgmap v1050: 105 pgs: 105 active+clean; 316 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 9.3 MiB/s wr, 106 op/s 2024-09-14T04:15:15.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:15 smithi039 ceph-mon[111235]: osdmap e911: 8 total, 8 up, 8 in 2024-09-14T04:15:15.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:15 smithi039 ceph-mon[111235]: pgmap v1050: 105 pgs: 105 active+clean; 316 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 9.3 MiB/s wr, 106 op/s 2024-09-14T04:15:18.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:18 smithi129 ceph-mon[101316]: pgmap v1051: 105 pgs: 105 active+clean; 324 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 8.7 MiB/s wr, 82 op/s 2024-09-14T04:15:18.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:18 smithi039 ceph-mon[106470]: pgmap v1051: 105 pgs: 105 active+clean; 324 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 8.7 MiB/s wr, 82 op/s 2024-09-14T04:15:18.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:18 smithi039 ceph-mon[111235]: pgmap v1051: 105 pgs: 105 active+clean; 324 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 8.7 MiB/s wr, 82 op/s 2024-09-14T04:15:19.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:15:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:15:18] "GET /metrics HTTP/1.1" 200 36686 "" "Prometheus/2.51.0" 2024-09-14T04:15:19.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:19 smithi129 ceph-mon[101316]: pgmap v1052: 105 pgs: 105 active+clean; 340 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.5 MiB/s wr, 84 op/s 2024-09-14T04:15:19.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:19 smithi039 ceph-mon[106470]: pgmap v1052: 105 pgs: 105 active+clean; 340 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.5 MiB/s wr, 84 op/s 2024-09-14T04:15:19.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:19 smithi039 ceph-mon[111235]: pgmap v1052: 105 pgs: 105 active+clean; 340 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.5 MiB/s wr, 84 op/s 2024-09-14T04:15:22.010 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/0.ZeroOverlap (7911 ms) 2024-09-14T04:15:22.226 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 7 tests from EncryptedFlattenTest/0 (63320 ms total) 2024-09-14T04:15:22.226 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:15:22.227 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 7 tests from EncryptedFlattenTest/1, where TypeParam = PlaintextUnderLUKS2 2024-09-14T04:15:22.256 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/1.Simple 2024-09-14T04:15:22.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:22 smithi129 ceph-mon[101316]: pgmap v1053: 105 pgs: 105 active+clean; 344 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 4.5 MiB/s wr, 67 op/s 2024-09-14T04:15:22.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:22 smithi039 ceph-mon[106470]: pgmap v1053: 105 pgs: 105 active+clean; 344 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 4.5 MiB/s wr, 67 op/s 2024-09-14T04:15:22.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:22 smithi039 ceph-mon[111235]: pgmap v1053: 105 pgs: 105 active+clean; 344 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 4.5 MiB/s wr, 67 op/s 2024-09-14T04:15:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:23 smithi129 ceph-mon[101316]: osdmap e912: 8 total, 8 up, 8 in 2024-09-14T04:15:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:15:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:23 smithi129 ceph-mon[101316]: pgmap v1055: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:15:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:23 smithi039 ceph-mon[106470]: osdmap e912: 8 total, 8 up, 8 in 2024-09-14T04:15:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:15:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:23 smithi039 ceph-mon[106470]: pgmap v1055: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:15:23.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:23 smithi039 ceph-mon[111235]: osdmap e912: 8 total, 8 up, 8 in 2024-09-14T04:15:23.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:15:23.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:23 smithi039 ceph-mon[111235]: pgmap v1055: 73 pgs: 73 active+clean; 582 KiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:15:24.250 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:15:24.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:24 smithi129 ceph-mon[101316]: osdmap e913: 8 total, 8 up, 8 in 2024-09-14T04:15:24.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:24 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1738783131' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-40","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:15:24.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:24 smithi039 ceph-mon[106470]: osdmap e913: 8 total, 8 up, 8 in 2024-09-14T04:15:24.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:24 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1738783131' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-40","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:15:24.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:24 smithi039 ceph-mon[111235]: osdmap e913: 8 total, 8 up, 8 in 2024-09-14T04:15:24.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:24 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1738783131' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-40","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:15:25.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:25 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1738783131' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-40","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:15:25.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:25 smithi129 ceph-mon[101316]: osdmap e914: 8 total, 8 up, 8 in 2024-09-14T04:15:25.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:25 smithi129 ceph-mon[101316]: pgmap v1058: 105 pgs: 25 unknown, 80 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:15:25.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:25 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1738783131' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-40","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:15:25.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:25 smithi039 ceph-mon[106470]: osdmap e914: 8 total, 8 up, 8 in 2024-09-14T04:15:25.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:25 smithi039 ceph-mon[106470]: pgmap v1058: 105 pgs: 25 unknown, 80 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:15:25.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:25 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1738783131' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-40","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:15:25.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:25 smithi039 ceph-mon[111235]: osdmap e914: 8 total, 8 up, 8 in 2024-09-14T04:15:25.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:25 smithi039 ceph-mon[111235]: pgmap v1058: 105 pgs: 25 unknown, 80 active+clean; 582 KiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:15:26.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:26 smithi129 ceph-mon[101316]: osdmap e915: 8 total, 8 up, 8 in 2024-09-14T04:15:26.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:26 smithi039 ceph-mon[106470]: osdmap e915: 8 total, 8 up, 8 in 2024-09-14T04:15:26.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:26 smithi039 ceph-mon[111235]: osdmap e915: 8 total, 8 up, 8 in 2024-09-14T04:15:27.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:27 smithi039 ceph-mon[106470]: osdmap e916: 8 total, 8 up, 8 in 2024-09-14T04:15:27.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:27 smithi039 ceph-mon[106470]: pgmap v1061: 105 pgs: 20 unknown, 85 active+clean; 582 KiB data, 947 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 255 B/s wr, 0 op/s 2024-09-14T04:15:27.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:27 smithi039 ceph-mon[111235]: osdmap e916: 8 total, 8 up, 8 in 2024-09-14T04:15:27.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:27 smithi039 ceph-mon[111235]: pgmap v1061: 105 pgs: 20 unknown, 85 active+clean; 582 KiB data, 947 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 255 B/s wr, 0 op/s 2024-09-14T04:15:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:27 smithi129 ceph-mon[101316]: osdmap e916: 8 total, 8 up, 8 in 2024-09-14T04:15:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:27 smithi129 ceph-mon[101316]: pgmap v1061: 105 pgs: 20 unknown, 85 active+clean; 582 KiB data, 947 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 255 B/s wr, 0 op/s 2024-09-14T04:15:28.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:28 smithi129 ceph-mon[101316]: osdmap e917: 8 total, 8 up, 8 in 2024-09-14T04:15:28.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:28 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:15:28.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:28 smithi039 ceph-mon[106470]: osdmap e917: 8 total, 8 up, 8 in 2024-09-14T04:15:28.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:28 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:15:28.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:28 smithi039 ceph-mon[111235]: osdmap e917: 8 total, 8 up, 8 in 2024-09-14T04:15:28.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:28 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:15:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:15:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:15:28] "GET /metrics HTTP/1.1" 200 34255 "" "Prometheus/2.51.0" 2024-09-14T04:15:29.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:29 smithi039 ceph-mon[106470]: pgmap v1063: 105 pgs: 105 active+clean; 22 MiB data, 643 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 4.3 MiB/s wr, 71 op/s 2024-09-14T04:15:29.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:29 smithi039 ceph-mon[111235]: pgmap v1063: 105 pgs: 105 active+clean; 22 MiB data, 643 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 4.3 MiB/s wr, 71 op/s 2024-09-14T04:15:29.682 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:29 smithi129 ceph-mon[101316]: pgmap v1063: 105 pgs: 105 active+clean; 22 MiB data, 643 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 4.3 MiB/s wr, 71 op/s 2024-09-14T04:15:32.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:32 smithi129 ceph-mon[101316]: pgmap v1064: 105 pgs: 105 active+clean; 26 MiB data, 659 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 4.2 MiB/s wr, 59 op/s 2024-09-14T04:15:32.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:32 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:15:32.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:32 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:15:32.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:32 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:15:32.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:32 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:15:32.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:32 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:15:32.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:32 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:15:32.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:32 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:15:32.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:32 smithi039 ceph-mon[106470]: pgmap v1064: 105 pgs: 105 active+clean; 26 MiB data, 659 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 4.2 MiB/s wr, 59 op/s 2024-09-14T04:15:32.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:32 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:15:32.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:32 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:15:32.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:32 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:15:32.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:32 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:15:32.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:32 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:15:32.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:32 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:15:32.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:32 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:15:32.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:32 smithi039 ceph-mon[111235]: pgmap v1064: 105 pgs: 105 active+clean; 26 MiB data, 659 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 4.2 MiB/s wr, 59 op/s 2024-09-14T04:15:32.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:32 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:15:32.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:32 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:15:32.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:32 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:15:32.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:32 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:15:32.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:32 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:15:32.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:32 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:15:32.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:32 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:15:33.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:33 smithi129 ceph-mon[101316]: pgmap v1065: 105 pgs: 105 active+clean; 26 MiB data, 676 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 3.2 MiB/s wr, 47 op/s 2024-09-14T04:15:33.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:33 smithi039 ceph-mon[106470]: pgmap v1065: 105 pgs: 105 active+clean; 26 MiB data, 676 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 3.2 MiB/s wr, 47 op/s 2024-09-14T04:15:33.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:33 smithi039 ceph-mon[111235]: pgmap v1065: 105 pgs: 105 active+clean; 26 MiB data, 676 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 3.2 MiB/s wr, 47 op/s 2024-09-14T04:15:35.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:35 smithi129 ceph-mon[101316]: pgmap v1066: 105 pgs: 105 active+clean; 26 MiB data, 676 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 2.8 MiB/s wr, 41 op/s 2024-09-14T04:15:35.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:35 smithi039 ceph-mon[106470]: pgmap v1066: 105 pgs: 105 active+clean; 26 MiB data, 676 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 2.8 MiB/s wr, 41 op/s 2024-09-14T04:15:35.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:35 smithi039 ceph-mon[111235]: pgmap v1066: 105 pgs: 105 active+clean; 26 MiB data, 676 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 2.8 MiB/s wr, 41 op/s 2024-09-14T04:15:37.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:36 smithi129 ceph-mon[101316]: osdmap e918: 8 total, 8 up, 8 in 2024-09-14T04:15:37.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:36 smithi039 ceph-mon[106470]: osdmap e918: 8 total, 8 up, 8 in 2024-09-14T04:15:37.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:36 smithi039 ceph-mon[111235]: osdmap e918: 8 total, 8 up, 8 in 2024-09-14T04:15:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:37 smithi129 ceph-mon[101316]: pgmap v1068: 105 pgs: 105 active+clean; 30 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 2.9 MiB/s wr, 37 op/s 2024-09-14T04:15:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:15:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:37 smithi039 ceph-mon[106470]: pgmap v1068: 105 pgs: 105 active+clean; 30 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 2.9 MiB/s wr, 37 op/s 2024-09-14T04:15:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:15:38.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:37 smithi039 ceph-mon[111235]: pgmap v1068: 105 pgs: 105 active+clean; 30 MiB data, 680 MiB used, 715 GiB / 715 GiB avail; 27 KiB/s rd, 2.9 MiB/s wr, 37 op/s 2024-09-14T04:15:38.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:15:38.737 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/1.Simple (16481 ms) 2024-09-14T04:15:38.737 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/1.Grow 2024-09-14T04:15:38.737 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:15:39.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:15:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:15:38] "GET /metrics HTTP/1.1" 200 36669 "" "Prometheus/2.51.0" 2024-09-14T04:15:40.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:39 smithi129 ceph-mon[101316]: osdmap e919: 8 total, 8 up, 8 in 2024-09-14T04:15:40.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:39 smithi129 ceph-mon[101316]: pgmap v1070: 105 pgs: 105 active+clean; 55 MiB data, 774 MiB used, 714 GiB / 715 GiB avail; 8.5 MiB/s rd, 4.6 MiB/s wr, 35 op/s 2024-09-14T04:15:40.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:39 smithi039 ceph-mon[106470]: osdmap e919: 8 total, 8 up, 8 in 2024-09-14T04:15:40.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:39 smithi039 ceph-mon[106470]: pgmap v1070: 105 pgs: 105 active+clean; 55 MiB data, 774 MiB used, 714 GiB / 715 GiB avail; 8.5 MiB/s rd, 4.6 MiB/s wr, 35 op/s 2024-09-14T04:15:40.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:39 smithi039 ceph-mon[111235]: osdmap e919: 8 total, 8 up, 8 in 2024-09-14T04:15:40.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:39 smithi039 ceph-mon[111235]: pgmap v1070: 105 pgs: 105 active+clean; 55 MiB data, 774 MiB used, 714 GiB / 715 GiB avail; 8.5 MiB/s rd, 4.6 MiB/s wr, 35 op/s 2024-09-14T04:15:41.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:41 smithi129 ceph-mon[101316]: pgmap v1071: 105 pgs: 105 active+clean; 71 MiB data, 824 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 7.1 MiB/s wr, 43 op/s 2024-09-14T04:15:41.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:41 smithi039 ceph-mon[106470]: pgmap v1071: 105 pgs: 105 active+clean; 71 MiB data, 824 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 7.1 MiB/s wr, 43 op/s 2024-09-14T04:15:41.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:41 smithi039 ceph-mon[111235]: pgmap v1071: 105 pgs: 105 active+clean; 71 MiB data, 824 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 7.1 MiB/s wr, 43 op/s 2024-09-14T04:15:43.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:43 smithi129 ceph-mon[101316]: pgmap v1072: 105 pgs: 105 active+clean; 92 MiB data, 878 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 10 MiB/s wr, 83 op/s 2024-09-14T04:15:43.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:43 smithi039 ceph-mon[106470]: pgmap v1072: 105 pgs: 105 active+clean; 92 MiB data, 878 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 10 MiB/s wr, 83 op/s 2024-09-14T04:15:43.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:43 smithi039 ceph-mon[111235]: pgmap v1072: 105 pgs: 105 active+clean; 92 MiB data, 878 MiB used, 714 GiB / 715 GiB avail; 15 MiB/s rd, 10 MiB/s wr, 83 op/s 2024-09-14T04:15:45.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:45 smithi129 ceph-mon[101316]: pgmap v1073: 105 pgs: 105 active+clean; 92 MiB data, 878 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.6 MiB/s wr, 69 op/s 2024-09-14T04:15:45.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:45 smithi039 ceph-mon[106470]: pgmap v1073: 105 pgs: 105 active+clean; 92 MiB data, 878 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.6 MiB/s wr, 69 op/s 2024-09-14T04:15:45.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:45 smithi039 ceph-mon[111235]: pgmap v1073: 105 pgs: 105 active+clean; 92 MiB data, 878 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.6 MiB/s wr, 69 op/s 2024-09-14T04:15:47.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:47 smithi129 ceph-mon[101316]: pgmap v1074: 105 pgs: 105 active+clean; 92 MiB data, 878 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.8 MiB/s wr, 66 op/s 2024-09-14T04:15:47.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:47 smithi039 ceph-mon[106470]: pgmap v1074: 105 pgs: 105 active+clean; 92 MiB data, 878 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.8 MiB/s wr, 66 op/s 2024-09-14T04:15:47.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:47 smithi039 ceph-mon[111235]: pgmap v1074: 105 pgs: 105 active+clean; 92 MiB data, 878 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.8 MiB/s wr, 66 op/s 2024-09-14T04:15:49.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:15:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:15:48] "GET /metrics HTTP/1.1" 200 36669 "" "Prometheus/2.51.0" 2024-09-14T04:15:49.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:49 smithi129 ceph-mon[101316]: pgmap v1075: 105 pgs: 105 active+clean; 100 MiB data, 902 MiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 5.2 MiB/s wr, 49 op/s 2024-09-14T04:15:49.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:49 smithi039 ceph-mon[106470]: pgmap v1075: 105 pgs: 105 active+clean; 100 MiB data, 902 MiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 5.2 MiB/s wr, 49 op/s 2024-09-14T04:15:49.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:49 smithi039 ceph-mon[111235]: pgmap v1075: 105 pgs: 105 active+clean; 100 MiB data, 902 MiB used, 714 GiB / 715 GiB avail; 4.8 MiB/s rd, 5.2 MiB/s wr, 49 op/s 2024-09-14T04:15:50.548 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/1.Grow (11812 ms) 2024-09-14T04:15:50.548 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/1.Shrink 2024-09-14T04:15:50.548 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:15:52.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:51 smithi129 ceph-mon[101316]: osdmap e920: 8 total, 8 up, 8 in 2024-09-14T04:15:52.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:51 smithi129 ceph-mon[101316]: pgmap v1077: 105 pgs: 105 active+clean; 116 MiB data, 964 MiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 5.7 MiB/s wr, 44 op/s 2024-09-14T04:15:52.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:51 smithi039 ceph-mon[106470]: osdmap e920: 8 total, 8 up, 8 in 2024-09-14T04:15:52.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:51 smithi039 ceph-mon[106470]: pgmap v1077: 105 pgs: 105 active+clean; 116 MiB data, 964 MiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 5.7 MiB/s wr, 44 op/s 2024-09-14T04:15:52.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:51 smithi039 ceph-mon[111235]: osdmap e920: 8 total, 8 up, 8 in 2024-09-14T04:15:52.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:51 smithi039 ceph-mon[111235]: pgmap v1077: 105 pgs: 105 active+clean; 116 MiB data, 964 MiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 5.7 MiB/s wr, 44 op/s 2024-09-14T04:15:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:15:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:15:53.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:15:54.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:53 smithi129 ceph-mon[101316]: pgmap v1078: 105 pgs: 105 active+clean; 146 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 7.0 MiB/s wr, 61 op/s 2024-09-14T04:15:54.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:53 smithi039 ceph-mon[106470]: pgmap v1078: 105 pgs: 105 active+clean; 146 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 7.0 MiB/s wr, 61 op/s 2024-09-14T04:15:54.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:53 smithi039 ceph-mon[111235]: pgmap v1078: 105 pgs: 105 active+clean; 146 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 7.0 MiB/s wr, 61 op/s 2024-09-14T04:15:55.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:55 smithi129 ceph-mon[101316]: pgmap v1079: 105 pgs: 105 active+clean; 158 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 68 op/s 2024-09-14T04:15:55.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:55 smithi039 ceph-mon[106470]: pgmap v1079: 105 pgs: 105 active+clean; 158 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 68 op/s 2024-09-14T04:15:55.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:55 smithi039 ceph-mon[111235]: pgmap v1079: 105 pgs: 105 active+clean; 158 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 68 op/s 2024-09-14T04:15:57.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:57 smithi129 ceph-mon[101316]: pgmap v1080: 105 pgs: 105 active+clean; 162 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.6 MiB/s wr, 68 op/s 2024-09-14T04:15:57.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:57 smithi039 ceph-mon[106470]: pgmap v1080: 105 pgs: 105 active+clean; 162 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.6 MiB/s wr, 68 op/s 2024-09-14T04:15:57.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:57 smithi039 ceph-mon[111235]: pgmap v1080: 105 pgs: 105 active+clean; 162 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.6 MiB/s wr, 68 op/s 2024-09-14T04:15:59.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:15:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:15:58] "GET /metrics HTTP/1.1" 200 36683 "" "Prometheus/2.51.0" 2024-09-14T04:15:59.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:15:59 smithi129 ceph-mon[101316]: pgmap v1081: 105 pgs: 105 active+clean; 166 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 62 op/s 2024-09-14T04:15:59.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:15:59 smithi039 ceph-mon[106470]: pgmap v1081: 105 pgs: 105 active+clean; 166 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 62 op/s 2024-09-14T04:15:59.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:15:59 smithi039 ceph-mon[111235]: pgmap v1081: 105 pgs: 105 active+clean; 166 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 62 op/s 2024-09-14T04:16:00.248 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/1.Shrink (9700 ms) 2024-09-14T04:16:00.248 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/1.ShrinkToOne 2024-09-14T04:16:00.248 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:16:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:01 smithi129 ceph-mon[101316]: osdmap e921: 8 total, 8 up, 8 in 2024-09-14T04:16:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:01 smithi129 ceph-mon[101316]: pgmap v1083: 105 pgs: 105 active+clean; 182 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 8.2 MiB/s wr, 82 op/s 2024-09-14T04:16:01.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:01 smithi039 ceph-mon[106470]: osdmap e921: 8 total, 8 up, 8 in 2024-09-14T04:16:01.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:01 smithi039 ceph-mon[106470]: pgmap v1083: 105 pgs: 105 active+clean; 182 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 8.2 MiB/s wr, 82 op/s 2024-09-14T04:16:01.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:01 smithi039 ceph-mon[111235]: osdmap e921: 8 total, 8 up, 8 in 2024-09-14T04:16:01.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:01 smithi039 ceph-mon[111235]: pgmap v1083: 105 pgs: 105 active+clean; 182 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 8.2 MiB/s wr, 82 op/s 2024-09-14T04:16:03.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:03 smithi129 ceph-mon[101316]: pgmap v1084: 105 pgs: 105 active+clean; 216 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.6 MiB/s wr, 74 op/s 2024-09-14T04:16:03.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:03 smithi039 ceph-mon[106470]: pgmap v1084: 105 pgs: 105 active+clean; 216 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.6 MiB/s wr, 74 op/s 2024-09-14T04:16:03.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:03 smithi039 ceph-mon[111235]: pgmap v1084: 105 pgs: 105 active+clean; 216 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.6 MiB/s wr, 74 op/s 2024-09-14T04:16:05.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:05 smithi129 ceph-mon[101316]: pgmap v1085: 105 pgs: 105 active+clean; 224 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 69 op/s 2024-09-14T04:16:05.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:05 smithi039 ceph-mon[106470]: pgmap v1085: 105 pgs: 105 active+clean; 224 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 69 op/s 2024-09-14T04:16:05.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:05 smithi039 ceph-mon[111235]: pgmap v1085: 105 pgs: 105 active+clean; 224 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 69 op/s 2024-09-14T04:16:07.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:07 smithi129 ceph-mon[101316]: pgmap v1086: 105 pgs: 105 active+clean; 224 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.8 MiB/s wr, 68 op/s 2024-09-14T04:16:07.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:16:07.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:07 smithi039 ceph-mon[106470]: pgmap v1086: 105 pgs: 105 active+clean; 224 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.8 MiB/s wr, 68 op/s 2024-09-14T04:16:07.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:16:07.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:07 smithi039 ceph-mon[111235]: pgmap v1086: 105 pgs: 105 active+clean; 224 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.8 MiB/s wr, 68 op/s 2024-09-14T04:16:07.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:16:09.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:16:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:16:08] "GET /metrics HTTP/1.1" 200 36684 "" "Prometheus/2.51.0" 2024-09-14T04:16:09.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:09 smithi129 ceph-mon[101316]: pgmap v1087: 105 pgs: 105 active+clean; 232 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 75 op/s 2024-09-14T04:16:09.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:09 smithi039 ceph-mon[106470]: pgmap v1087: 105 pgs: 105 active+clean; 232 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 75 op/s 2024-09-14T04:16:09.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:09 smithi039 ceph-mon[111235]: pgmap v1087: 105 pgs: 105 active+clean; 232 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.2 MiB/s wr, 75 op/s 2024-09-14T04:16:11.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:11 smithi129 ceph-mon[101316]: pgmap v1088: 105 pgs: 105 active+clean; 232 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 5.4 MiB/s wr, 49 op/s 2024-09-14T04:16:11.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:11 smithi039 ceph-mon[106470]: pgmap v1088: 105 pgs: 105 active+clean; 232 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 5.4 MiB/s wr, 49 op/s 2024-09-14T04:16:11.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:11 smithi039 ceph-mon[111235]: pgmap v1088: 105 pgs: 105 active+clean; 232 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 5.4 MiB/s wr, 49 op/s 2024-09-14T04:16:11.997 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/1.ShrinkToOne (11748 ms) 2024-09-14T04:16:11.997 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/1.ShrinkToOneAfterSnapshot 2024-09-14T04:16:11.997 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:16:13.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:13 smithi129 ceph-mon[101316]: osdmap e922: 8 total, 8 up, 8 in 2024-09-14T04:16:13.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:13 smithi129 ceph-mon[101316]: pgmap v1090: 105 pgs: 105 active+clean; 253 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 5.3 MiB/s wr, 51 op/s 2024-09-14T04:16:13.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:13 smithi039 ceph-mon[106470]: osdmap e922: 8 total, 8 up, 8 in 2024-09-14T04:16:13.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:13 smithi039 ceph-mon[106470]: pgmap v1090: 105 pgs: 105 active+clean; 253 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 5.3 MiB/s wr, 51 op/s 2024-09-14T04:16:13.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:13 smithi039 ceph-mon[111235]: osdmap e922: 8 total, 8 up, 8 in 2024-09-14T04:16:13.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:13 smithi039 ceph-mon[111235]: pgmap v1090: 105 pgs: 105 active+clean; 253 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.9 MiB/s rd, 5.3 MiB/s wr, 51 op/s 2024-09-14T04:16:15.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:15 smithi129 ceph-mon[101316]: pgmap v1091: 105 pgs: 105 active+clean; 261 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 5.7 MiB/s wr, 58 op/s 2024-09-14T04:16:15.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:15 smithi039 ceph-mon[106470]: pgmap v1091: 105 pgs: 105 active+clean; 261 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 5.7 MiB/s wr, 58 op/s 2024-09-14T04:16:15.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:15 smithi039 ceph-mon[111235]: pgmap v1091: 105 pgs: 105 active+clean; 261 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 5.7 MiB/s wr, 58 op/s 2024-09-14T04:16:17.524 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:17 smithi129 ceph-mon[101316]: pgmap v1092: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.5 MiB/s wr, 60 op/s 2024-09-14T04:16:17.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:17 smithi039 ceph-mon[106470]: pgmap v1092: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.5 MiB/s wr, 60 op/s 2024-09-14T04:16:17.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:17 smithi039 ceph-mon[111235]: pgmap v1092: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.5 MiB/s wr, 60 op/s 2024-09-14T04:16:19.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:16:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:16:18] "GET /metrics HTTP/1.1" 200 36684 "" "Prometheus/2.51.0" 2024-09-14T04:16:19.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:19 smithi129 ceph-mon[101316]: pgmap v1093: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 5.7 MiB/s wr, 49 op/s 2024-09-14T04:16:19.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:19 smithi039 ceph-mon[106470]: pgmap v1093: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 5.7 MiB/s wr, 49 op/s 2024-09-14T04:16:19.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:19 smithi039 ceph-mon[111235]: pgmap v1093: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 5.7 MiB/s wr, 49 op/s 2024-09-14T04:16:21.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:21 smithi129 ceph-mon[101316]: pgmap v1094: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 5.7 MiB/s wr, 49 op/s 2024-09-14T04:16:21.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:21 smithi039 ceph-mon[106470]: pgmap v1094: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 5.7 MiB/s wr, 49 op/s 2024-09-14T04:16:21.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:21 smithi039 ceph-mon[111235]: pgmap v1094: 105 pgs: 105 active+clean; 269 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 5.7 MiB/s wr, 49 op/s 2024-09-14T04:16:22.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:16:22.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:16:22.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:16:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:23 smithi039 ceph-mon[106470]: pgmap v1095: 105 pgs: 105 active+clean; 277 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 819 KiB/s rd, 4.5 MiB/s wr, 54 op/s 2024-09-14T04:16:23.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:23 smithi039 ceph-mon[111235]: pgmap v1095: 105 pgs: 105 active+clean; 277 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 819 KiB/s rd, 4.5 MiB/s wr, 54 op/s 2024-09-14T04:16:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:23 smithi129 ceph-mon[101316]: pgmap v1095: 105 pgs: 105 active+clean; 277 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 819 KiB/s rd, 4.5 MiB/s wr, 54 op/s 2024-09-14T04:16:24.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:24 smithi039 ceph-mon[106470]: osdmap e923: 8 total, 8 up, 8 in 2024-09-14T04:16:24.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:24 smithi039 ceph-mon[111235]: osdmap e923: 8 total, 8 up, 8 in 2024-09-14T04:16:24.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:24 smithi129 ceph-mon[101316]: osdmap e923: 8 total, 8 up, 8 in 2024-09-14T04:16:25.596 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/1.ShrinkToOneAfterSnapshot (13599 ms) 2024-09-14T04:16:25.596 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/1.MinOverlap 2024-09-14T04:16:25.596 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:16:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:25 smithi129 ceph-mon[101316]: pgmap v1097: 105 pgs: 105 active+clean; 301 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.0 MiB/s wr, 14 op/s 2024-09-14T04:16:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:25 smithi039 ceph-mon[106470]: pgmap v1097: 105 pgs: 105 active+clean; 301 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.0 MiB/s wr, 14 op/s 2024-09-14T04:16:25.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:25 smithi039 ceph-mon[111235]: pgmap v1097: 105 pgs: 105 active+clean; 301 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.0 MiB/s wr, 14 op/s 2024-09-14T04:16:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:27 smithi129 ceph-mon[101316]: osdmap e924: 8 total, 8 up, 8 in 2024-09-14T04:16:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:27 smithi129 ceph-mon[101316]: pgmap v1099: 105 pgs: 105 active+clean; 318 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 6.1 MiB/s wr, 19 op/s 2024-09-14T04:16:27.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:27 smithi039 ceph-mon[106470]: osdmap e924: 8 total, 8 up, 8 in 2024-09-14T04:16:27.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:27 smithi039 ceph-mon[106470]: pgmap v1099: 105 pgs: 105 active+clean; 318 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 6.1 MiB/s wr, 19 op/s 2024-09-14T04:16:27.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:27 smithi039 ceph-mon[111235]: osdmap e924: 8 total, 8 up, 8 in 2024-09-14T04:16:27.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:27 smithi039 ceph-mon[111235]: pgmap v1099: 105 pgs: 105 active+clean; 318 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 6.1 MiB/s wr, 19 op/s 2024-09-14T04:16:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:16:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:16:28] "GET /metrics HTTP/1.1" 200 36684 "" "Prometheus/2.51.0" 2024-09-14T04:16:29.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:29 smithi129 ceph-mon[101316]: pgmap v1100: 105 pgs: 105 active+clean; 379 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 14 MiB/s wr, 102 op/s 2024-09-14T04:16:29.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:29 smithi039 ceph-mon[106470]: pgmap v1100: 105 pgs: 105 active+clean; 379 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 14 MiB/s wr, 102 op/s 2024-09-14T04:16:29.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:29 smithi039 ceph-mon[111235]: pgmap v1100: 105 pgs: 105 active+clean; 379 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 14 MiB/s wr, 102 op/s 2024-09-14T04:16:31.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:31 smithi129 ceph-mon[101316]: pgmap v1101: 105 pgs: 105 active+clean; 379 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 5.3 MiB/s rd, 13 MiB/s wr, 88 op/s 2024-09-14T04:16:31.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:31 smithi039 ceph-mon[106470]: pgmap v1101: 105 pgs: 105 active+clean; 379 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 5.3 MiB/s rd, 13 MiB/s wr, 88 op/s 2024-09-14T04:16:31.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:31 smithi039 ceph-mon[111235]: pgmap v1101: 105 pgs: 105 active+clean; 379 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 5.3 MiB/s rd, 13 MiB/s wr, 88 op/s 2024-09-14T04:16:32.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:32 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:16:32.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:32 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:16:32.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:32 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:16:33.264 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:33 smithi039 ceph-mon[106470]: pgmap v1102: 105 pgs: 105 active+clean; 379 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 10 MiB/s wr, 71 op/s 2024-09-14T04:16:33.265 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:33 smithi039 ceph-mon[111235]: pgmap v1102: 105 pgs: 105 active+clean; 379 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 10 MiB/s wr, 71 op/s 2024-09-14T04:16:33.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:33 smithi129 ceph-mon[101316]: pgmap v1102: 105 pgs: 105 active+clean; 379 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.3 MiB/s rd, 10 MiB/s wr, 71 op/s 2024-09-14T04:16:35.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:35 smithi129 ceph-mon[101316]: pgmap v1103: 105 pgs: 105 active+clean; 383 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 8.2 MiB/s wr, 68 op/s 2024-09-14T04:16:35.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:35 smithi039 ceph-mon[106470]: pgmap v1103: 105 pgs: 105 active+clean; 383 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 8.2 MiB/s wr, 68 op/s 2024-09-14T04:16:35.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:35 smithi039 ceph-mon[111235]: pgmap v1103: 105 pgs: 105 active+clean; 383 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 2.9 MiB/s rd, 8.2 MiB/s wr, 68 op/s 2024-09-14T04:16:37.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:16:37.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:16:37.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:16:37.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:16:37.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:16:37.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:16:37.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:16:37.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:16:37.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:16:37.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:16:37.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:16:37.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:16:37.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:16:37.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:16:37.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:16:37.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:16:37.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:16:37.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:16:37.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:16:37.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:16:37.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:16:38.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:38 smithi129 ceph-mon[101316]: pgmap v1104: 105 pgs: 105 active+clean; 391 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 8.7 MiB/s wr, 64 op/s 2024-09-14T04:16:38.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:38 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:16:38.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:38 smithi039 ceph-mon[106470]: pgmap v1104: 105 pgs: 105 active+clean; 391 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 8.7 MiB/s wr, 64 op/s 2024-09-14T04:16:38.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:38 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:16:38.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:38 smithi039 ceph-mon[111235]: pgmap v1104: 105 pgs: 105 active+clean; 391 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 8.7 MiB/s wr, 64 op/s 2024-09-14T04:16:38.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:38 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:16:38.661 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/1.MinOverlap (13066 ms) 2024-09-14T04:16:38.661 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/1.ZeroOverlap 2024-09-14T04:16:38.661 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:16:39.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:16:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:16:38] "GET /metrics HTTP/1.1" 200 36687 "" "Prometheus/2.51.0" 2024-09-14T04:16:40.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:40 smithi129 ceph-mon[101316]: pgmap v1105: 105 pgs: 105 active+clean; 399 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 7.8 MiB/s wr, 75 op/s 2024-09-14T04:16:40.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:40 smithi129 ceph-mon[101316]: osdmap e925: 8 total, 8 up, 8 in 2024-09-14T04:16:40.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:40 smithi039 ceph-mon[106470]: pgmap v1105: 105 pgs: 105 active+clean; 399 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 7.8 MiB/s wr, 75 op/s 2024-09-14T04:16:40.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:40 smithi039 ceph-mon[106470]: osdmap e925: 8 total, 8 up, 8 in 2024-09-14T04:16:40.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:40 smithi039 ceph-mon[111235]: pgmap v1105: 105 pgs: 105 active+clean; 399 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 7.8 MiB/s wr, 75 op/s 2024-09-14T04:16:40.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:40 smithi039 ceph-mon[111235]: osdmap e925: 8 total, 8 up, 8 in 2024-09-14T04:16:41.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:41 smithi129 ceph-mon[101316]: pgmap v1107: 105 pgs: 105 active+clean; 411 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 4.8 MiB/s wr, 25 op/s 2024-09-14T04:16:41.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:41 smithi039 ceph-mon[106470]: pgmap v1107: 105 pgs: 105 active+clean; 411 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 4.8 MiB/s wr, 25 op/s 2024-09-14T04:16:41.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:41 smithi039 ceph-mon[111235]: pgmap v1107: 105 pgs: 105 active+clean; 411 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 4.8 MiB/s wr, 25 op/s 2024-09-14T04:16:43.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:43 smithi129 ceph-mon[101316]: pgmap v1108: 105 pgs: 105 active+clean; 424 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 6.5 MiB/s wr, 62 op/s 2024-09-14T04:16:43.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:43 smithi039 ceph-mon[106470]: pgmap v1108: 105 pgs: 105 active+clean; 424 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 6.5 MiB/s wr, 62 op/s 2024-09-14T04:16:43.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:43 smithi039 ceph-mon[111235]: pgmap v1108: 105 pgs: 105 active+clean; 424 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 6.5 MiB/s wr, 62 op/s 2024-09-14T04:16:45.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:45 smithi129 ceph-mon[101316]: pgmap v1109: 105 pgs: 105 active+clean; 424 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 6.1 MiB/s wr, 62 op/s 2024-09-14T04:16:45.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:45 smithi039 ceph-mon[106470]: pgmap v1109: 105 pgs: 105 active+clean; 424 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 6.1 MiB/s wr, 62 op/s 2024-09-14T04:16:45.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:45 smithi039 ceph-mon[111235]: pgmap v1109: 105 pgs: 105 active+clean; 424 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 6.1 MiB/s wr, 62 op/s 2024-09-14T04:16:47.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:47 smithi129 ceph-mon[101316]: pgmap v1110: 105 pgs: 105 active+clean; 424 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.9 MiB/s wr, 61 op/s 2024-09-14T04:16:47.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:47 smithi039 ceph-mon[106470]: pgmap v1110: 105 pgs: 105 active+clean; 424 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.9 MiB/s wr, 61 op/s 2024-09-14T04:16:47.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:47 smithi039 ceph-mon[111235]: pgmap v1110: 105 pgs: 105 active+clean; 424 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.9 MiB/s wr, 61 op/s 2024-09-14T04:16:49.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:16:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:16:48] "GET /metrics HTTP/1.1" 200 36687 "" "Prometheus/2.51.0" 2024-09-14T04:16:49.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:49 smithi129 ceph-mon[101316]: pgmap v1111: 105 pgs: 105 active+clean; 432 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 4.1 MiB/s wr, 45 op/s 2024-09-14T04:16:49.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:49 smithi039 ceph-mon[106470]: pgmap v1111: 105 pgs: 105 active+clean; 432 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 4.1 MiB/s wr, 45 op/s 2024-09-14T04:16:49.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:49 smithi039 ceph-mon[111235]: pgmap v1111: 105 pgs: 105 active+clean; 432 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 4.1 MiB/s wr, 45 op/s 2024-09-14T04:16:50.754 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/1.ZeroOverlap (12092 ms) 2024-09-14T04:16:50.817 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 7 tests from EncryptedFlattenTest/1 (88498 ms total) 2024-09-14T04:16:50.817 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:16:50.817 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 7 tests from EncryptedFlattenTest/2, where TypeParam = UnformattedLUKS1 2024-09-14T04:16:50.845 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/2.Simple 2024-09-14T04:16:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:51 smithi129 ceph-mon[101316]: osdmap e926: 8 total, 8 up, 8 in 2024-09-14T04:16:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:51 smithi129 ceph-mon[101316]: pgmap v1113: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:16:52.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:51 smithi039 ceph-mon[111235]: osdmap e926: 8 total, 8 up, 8 in 2024-09-14T04:16:52.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:51 smithi039 ceph-mon[111235]: pgmap v1113: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:16:52.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:51 smithi039 ceph-mon[106470]: osdmap e926: 8 total, 8 up, 8 in 2024-09-14T04:16:52.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:51 smithi039 ceph-mon[106470]: pgmap v1113: 73 pgs: 73 active+clean; 582 KiB data, 1.9 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:16:52.848 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:16:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:52 smithi129 ceph-mon[101316]: osdmap e927: 8 total, 8 up, 8 in 2024-09-14T04:16:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:52 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/214271972' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-41","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:16:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:16:53.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:52 smithi039 ceph-mon[111235]: osdmap e927: 8 total, 8 up, 8 in 2024-09-14T04:16:53.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:52 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/214271972' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-41","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:16:53.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:16:53.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:52 smithi039 ceph-mon[106470]: osdmap e927: 8 total, 8 up, 8 in 2024-09-14T04:16:53.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:52 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/214271972' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-41","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:16:53.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:16:54.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:53 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/214271972' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-41","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:16:54.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:53 smithi129 ceph-mon[101316]: osdmap e928: 8 total, 8 up, 8 in 2024-09-14T04:16:54.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:53 smithi129 ceph-mon[101316]: pgmap v1116: 105 pgs: 3 creating+peering, 15 unknown, 87 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:16:54.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:53 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/214271972' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-41","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:16:54.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:53 smithi039 ceph-mon[106470]: osdmap e928: 8 total, 8 up, 8 in 2024-09-14T04:16:54.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:53 smithi039 ceph-mon[106470]: pgmap v1116: 105 pgs: 3 creating+peering, 15 unknown, 87 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:16:54.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:53 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/214271972' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-41","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:16:54.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:53 smithi039 ceph-mon[111235]: osdmap e928: 8 total, 8 up, 8 in 2024-09-14T04:16:54.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:53 smithi039 ceph-mon[111235]: pgmap v1116: 105 pgs: 3 creating+peering, 15 unknown, 87 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:16:55.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:54 smithi129 ceph-mon[101316]: osdmap e929: 8 total, 8 up, 8 in 2024-09-14T04:16:55.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:54 smithi039 ceph-mon[106470]: osdmap e929: 8 total, 8 up, 8 in 2024-09-14T04:16:55.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:54 smithi039 ceph-mon[111235]: osdmap e929: 8 total, 8 up, 8 in 2024-09-14T04:16:56.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:55 smithi129 ceph-mon[101316]: osdmap e930: 8 total, 8 up, 8 in 2024-09-14T04:16:56.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:55 smithi129 ceph-mon[101316]: pgmap v1119: 105 pgs: 3 creating+peering, 6 unknown, 96 active+clean; 582 KiB data, 755 MiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T04:16:56.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:55 smithi039 ceph-mon[106470]: osdmap e930: 8 total, 8 up, 8 in 2024-09-14T04:16:56.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:55 smithi039 ceph-mon[106470]: pgmap v1119: 105 pgs: 3 creating+peering, 6 unknown, 96 active+clean; 582 KiB data, 755 MiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T04:16:56.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:55 smithi039 ceph-mon[111235]: osdmap e930: 8 total, 8 up, 8 in 2024-09-14T04:16:56.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:55 smithi039 ceph-mon[111235]: pgmap v1119: 105 pgs: 3 creating+peering, 6 unknown, 96 active+clean; 582 KiB data, 755 MiB used, 714 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-09-14T04:16:57.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:57 smithi129 ceph-mon[101316]: pgmap v1120: 105 pgs: 3 creating+peering, 102 active+clean; 582 KiB data, 624 MiB used, 715 GiB / 715 GiB avail; 574 B/s rd, 574 B/s wr, 2 op/s 2024-09-14T04:16:57.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:57 smithi039 ceph-mon[111235]: pgmap v1120: 105 pgs: 3 creating+peering, 102 active+clean; 582 KiB data, 624 MiB used, 715 GiB / 715 GiB avail; 574 B/s rd, 574 B/s wr, 2 op/s 2024-09-14T04:16:57.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:57 smithi039 ceph-mon[106470]: pgmap v1120: 105 pgs: 3 creating+peering, 102 active+clean; 582 KiB data, 624 MiB used, 715 GiB / 715 GiB avail; 574 B/s rd, 574 B/s wr, 2 op/s 2024-09-14T04:16:59.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:16:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:16:58] "GET /metrics HTTP/1.1" 200 36613 "" "Prometheus/2.51.0" 2024-09-14T04:16:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:59 smithi129 ceph-mon[101316]: osdmap e931: 8 total, 8 up, 8 in 2024-09-14T04:16:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:16:59 smithi129 ceph-mon[101316]: pgmap v1122: 105 pgs: 105 active+clean; 582 KiB data, 624 MiB used, 715 GiB / 715 GiB avail; 4.5 KiB/s rd, 1023 B/s wr, 7 op/s 2024-09-14T04:16:59.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:59 smithi039 ceph-mon[106470]: osdmap e931: 8 total, 8 up, 8 in 2024-09-14T04:16:59.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:16:59 smithi039 ceph-mon[106470]: pgmap v1122: 105 pgs: 105 active+clean; 582 KiB data, 624 MiB used, 715 GiB / 715 GiB avail; 4.5 KiB/s rd, 1023 B/s wr, 7 op/s 2024-09-14T04:16:59.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:59 smithi039 ceph-mon[111235]: osdmap e931: 8 total, 8 up, 8 in 2024-09-14T04:16:59.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:16:59 smithi039 ceph-mon[111235]: pgmap v1122: 105 pgs: 105 active+clean; 582 KiB data, 624 MiB used, 715 GiB / 715 GiB avail; 4.5 KiB/s rd, 1023 B/s wr, 7 op/s 2024-09-14T04:17:01.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:01 smithi129 ceph-mon[101316]: pgmap v1123: 105 pgs: 105 active+clean; 582 KiB data, 645 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 1.4 KiB/s wr, 11 op/s 2024-09-14T04:17:01.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:01 smithi039 ceph-mon[106470]: pgmap v1123: 105 pgs: 105 active+clean; 582 KiB data, 645 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 1.4 KiB/s wr, 11 op/s 2024-09-14T04:17:01.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:01 smithi039 ceph-mon[111235]: pgmap v1123: 105 pgs: 105 active+clean; 582 KiB data, 645 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 1.4 KiB/s wr, 11 op/s 2024-09-14T04:17:03.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:03 smithi129 ceph-mon[101316]: pgmap v1124: 105 pgs: 105 active+clean; 39 MiB data, 716 MiB used, 715 GiB / 715 GiB avail; 5.3 MiB/s rd, 4.6 MiB/s wr, 71 op/s 2024-09-14T04:17:03.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:03 smithi039 ceph-mon[106470]: pgmap v1124: 105 pgs: 105 active+clean; 39 MiB data, 716 MiB used, 715 GiB / 715 GiB avail; 5.3 MiB/s rd, 4.6 MiB/s wr, 71 op/s 2024-09-14T04:17:03.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:03 smithi039 ceph-mon[111235]: pgmap v1124: 105 pgs: 105 active+clean; 39 MiB data, 716 MiB used, 715 GiB / 715 GiB avail; 5.3 MiB/s rd, 4.6 MiB/s wr, 71 op/s 2024-09-14T04:17:04.991 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/2.Simple (14145 ms) 2024-09-14T04:17:04.991 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/2.Grow 2024-09-14T04:17:04.991 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:17:05.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:05 smithi129 ceph-mon[101316]: pgmap v1125: 105 pgs: 105 active+clean; 51 MiB data, 740 MiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 5.0 MiB/s wr, 60 op/s 2024-09-14T04:17:05.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:05 smithi039 ceph-mon[106470]: pgmap v1125: 105 pgs: 105 active+clean; 51 MiB data, 740 MiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 5.0 MiB/s wr, 60 op/s 2024-09-14T04:17:05.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:05 smithi039 ceph-mon[111235]: pgmap v1125: 105 pgs: 105 active+clean; 51 MiB data, 740 MiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 5.0 MiB/s wr, 60 op/s 2024-09-14T04:17:07.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:06 smithi129 ceph-mon[101316]: osdmap e932: 8 total, 8 up, 8 in 2024-09-14T04:17:07.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:06 smithi039 ceph-mon[106470]: osdmap e932: 8 total, 8 up, 8 in 2024-09-14T04:17:07.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:06 smithi039 ceph-mon[111235]: osdmap e932: 8 total, 8 up, 8 in 2024-09-14T04:17:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:07 smithi129 ceph-mon[101316]: pgmap v1127: 105 pgs: 105 active+clean; 59 MiB data, 756 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 6.7 MiB/s wr, 66 op/s 2024-09-14T04:17:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:17:08.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:07 smithi039 ceph-mon[106470]: pgmap v1127: 105 pgs: 105 active+clean; 59 MiB data, 756 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 6.7 MiB/s wr, 66 op/s 2024-09-14T04:17:08.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:17:08.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:07 smithi039 ceph-mon[111235]: pgmap v1127: 105 pgs: 105 active+clean; 59 MiB data, 756 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 6.7 MiB/s wr, 66 op/s 2024-09-14T04:17:08.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:17:09.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:17:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:17:08] "GET /metrics HTTP/1.1" 200 36674 "" "Prometheus/2.51.0" 2024-09-14T04:17:10.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:09 smithi129 ceph-mon[101316]: osdmap e933: 8 total, 8 up, 8 in 2024-09-14T04:17:10.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:09 smithi129 ceph-mon[101316]: pgmap v1129: 105 pgs: 105 active+clean; 59 MiB data, 798 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.3 MiB/s wr, 72 op/s 2024-09-14T04:17:10.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:09 smithi039 ceph-mon[106470]: osdmap e933: 8 total, 8 up, 8 in 2024-09-14T04:17:10.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:09 smithi039 ceph-mon[106470]: pgmap v1129: 105 pgs: 105 active+clean; 59 MiB data, 798 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.3 MiB/s wr, 72 op/s 2024-09-14T04:17:10.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:09 smithi039 ceph-mon[111235]: osdmap e933: 8 total, 8 up, 8 in 2024-09-14T04:17:10.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:09 smithi039 ceph-mon[111235]: pgmap v1129: 105 pgs: 105 active+clean; 59 MiB data, 798 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.3 MiB/s wr, 72 op/s 2024-09-14T04:17:11.524 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:11 smithi129 ceph-mon[101316]: pgmap v1130: 105 pgs: 105 active+clean; 64 MiB data, 820 MiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 3.1 MiB/s wr, 16 op/s 2024-09-14T04:17:11.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:11 smithi039 ceph-mon[106470]: pgmap v1130: 105 pgs: 105 active+clean; 64 MiB data, 820 MiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 3.1 MiB/s wr, 16 op/s 2024-09-14T04:17:11.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:11 smithi039 ceph-mon[111235]: pgmap v1130: 105 pgs: 105 active+clean; 64 MiB data, 820 MiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 3.1 MiB/s wr, 16 op/s 2024-09-14T04:17:13.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:13 smithi129 ceph-mon[101316]: pgmap v1131: 105 pgs: 105 active+clean; 88 MiB data, 885 MiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 4.6 MiB/s wr, 58 op/s 2024-09-14T04:17:13.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:13 smithi039 ceph-mon[106470]: pgmap v1131: 105 pgs: 105 active+clean; 88 MiB data, 885 MiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 4.6 MiB/s wr, 58 op/s 2024-09-14T04:17:13.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:13 smithi039 ceph-mon[111235]: pgmap v1131: 105 pgs: 105 active+clean; 88 MiB data, 885 MiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 4.6 MiB/s wr, 58 op/s 2024-09-14T04:17:15.127 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/2.Grow (10136 ms) 2024-09-14T04:17:15.127 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/2.Shrink 2024-09-14T04:17:15.127 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:17:15.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:15 smithi129 ceph-mon[101316]: pgmap v1132: 105 pgs: 105 active+clean; 89 MiB data, 902 MiB used, 714 GiB / 715 GiB avail; 8.5 MiB/s rd, 4.0 MiB/s wr, 58 op/s 2024-09-14T04:17:15.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:15 smithi039 ceph-mon[106470]: pgmap v1132: 105 pgs: 105 active+clean; 89 MiB data, 902 MiB used, 714 GiB / 715 GiB avail; 8.5 MiB/s rd, 4.0 MiB/s wr, 58 op/s 2024-09-14T04:17:15.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:15 smithi039 ceph-mon[111235]: pgmap v1132: 105 pgs: 105 active+clean; 89 MiB data, 902 MiB used, 714 GiB / 715 GiB avail; 8.5 MiB/s rd, 4.0 MiB/s wr, 58 op/s 2024-09-14T04:17:17.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:17 smithi129 ceph-mon[101316]: pgmap v1133: 105 pgs: 105 active+clean; 97 MiB data, 910 MiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 3.8 MiB/s wr, 55 op/s 2024-09-14T04:17:17.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:17 smithi039 ceph-mon[106470]: pgmap v1133: 105 pgs: 105 active+clean; 97 MiB data, 910 MiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 3.8 MiB/s wr, 55 op/s 2024-09-14T04:17:17.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:17 smithi039 ceph-mon[111235]: pgmap v1133: 105 pgs: 105 active+clean; 97 MiB data, 910 MiB used, 714 GiB / 715 GiB avail; 6.5 MiB/s rd, 3.8 MiB/s wr, 55 op/s 2024-09-14T04:17:19.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:17:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:17:18] "GET /metrics HTTP/1.1" 200 36674 "" "Prometheus/2.51.0" 2024-09-14T04:17:20.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:19 smithi129 ceph-mon[101316]: osdmap e934: 8 total, 8 up, 8 in 2024-09-14T04:17:20.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:19 smithi129 ceph-mon[101316]: pgmap v1135: 105 pgs: 105 active+clean; 117 MiB data, 972 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 5.8 MiB/s wr, 65 op/s 2024-09-14T04:17:20.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:19 smithi039 ceph-mon[106470]: osdmap e934: 8 total, 8 up, 8 in 2024-09-14T04:17:20.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:19 smithi039 ceph-mon[106470]: pgmap v1135: 105 pgs: 105 active+clean; 117 MiB data, 972 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 5.8 MiB/s wr, 65 op/s 2024-09-14T04:17:20.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:19 smithi039 ceph-mon[111235]: osdmap e934: 8 total, 8 up, 8 in 2024-09-14T04:17:20.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:19 smithi039 ceph-mon[111235]: pgmap v1135: 105 pgs: 105 active+clean; 117 MiB data, 972 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 5.8 MiB/s wr, 65 op/s 2024-09-14T04:17:21.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:21 smithi129 ceph-mon[101316]: pgmap v1136: 105 pgs: 105 active+clean; 125 MiB data, 1001 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.1 MiB/s wr, 71 op/s 2024-09-14T04:17:21.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:21 smithi039 ceph-mon[106470]: pgmap v1136: 105 pgs: 105 active+clean; 125 MiB data, 1001 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.1 MiB/s wr, 71 op/s 2024-09-14T04:17:21.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:21 smithi039 ceph-mon[111235]: pgmap v1136: 105 pgs: 105 active+clean; 125 MiB data, 1001 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.1 MiB/s wr, 71 op/s 2024-09-14T04:17:22.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:17:22.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:17:22.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:17:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:23 smithi039 ceph-mon[106470]: pgmap v1137: 105 pgs: 105 active+clean; 146 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 5.8 MiB/s wr, 64 op/s 2024-09-14T04:17:23.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:23 smithi039 ceph-mon[111235]: pgmap v1137: 105 pgs: 105 active+clean; 146 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 5.8 MiB/s wr, 64 op/s 2024-09-14T04:17:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:23 smithi129 ceph-mon[101316]: pgmap v1137: 105 pgs: 105 active+clean; 146 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s rd, 5.8 MiB/s wr, 64 op/s 2024-09-14T04:17:25.260 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/2.Shrink (10133 ms) 2024-09-14T04:17:25.260 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/2.ShrinkToOne 2024-09-14T04:17:25.260 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:17:25.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:25 smithi129 ceph-mon[101316]: pgmap v1138: 105 pgs: 105 active+clean; 158 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 6.9 MiB/s wr, 65 op/s 2024-09-14T04:17:25.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:25 smithi039 ceph-mon[106470]: pgmap v1138: 105 pgs: 105 active+clean; 158 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 6.9 MiB/s wr, 65 op/s 2024-09-14T04:17:25.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:25 smithi039 ceph-mon[111235]: pgmap v1138: 105 pgs: 105 active+clean; 158 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 6.9 MiB/s wr, 65 op/s 2024-09-14T04:17:27.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:27 smithi129 ceph-mon[101316]: pgmap v1139: 105 pgs: 105 active+clean; 162 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.5 MiB/s wr, 72 op/s 2024-09-14T04:17:27.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:27 smithi039 ceph-mon[106470]: pgmap v1139: 105 pgs: 105 active+clean; 162 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.5 MiB/s wr, 72 op/s 2024-09-14T04:17:27.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:27 smithi039 ceph-mon[111235]: pgmap v1139: 105 pgs: 105 active+clean; 162 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.5 MiB/s wr, 72 op/s 2024-09-14T04:17:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:17:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:17:28] "GET /metrics HTTP/1.1" 200 36689 "" "Prometheus/2.51.0" 2024-09-14T04:17:30.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:29 smithi129 ceph-mon[101316]: osdmap e935: 8 total, 8 up, 8 in 2024-09-14T04:17:30.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:29 smithi129 ceph-mon[101316]: pgmap v1141: 105 pgs: 105 active+clean; 175 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 5.8 MiB/s wr, 65 op/s 2024-09-14T04:17:30.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:29 smithi039 ceph-mon[106470]: osdmap e935: 8 total, 8 up, 8 in 2024-09-14T04:17:30.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:29 smithi039 ceph-mon[106470]: pgmap v1141: 105 pgs: 105 active+clean; 175 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 5.8 MiB/s wr, 65 op/s 2024-09-14T04:17:30.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:29 smithi039 ceph-mon[111235]: osdmap e935: 8 total, 8 up, 8 in 2024-09-14T04:17:30.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:29 smithi039 ceph-mon[111235]: pgmap v1141: 105 pgs: 105 active+clean; 175 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 5.8 MiB/s wr, 65 op/s 2024-09-14T04:17:31.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:31 smithi129 ceph-mon[101316]: pgmap v1142: 105 pgs: 105 active+clean; 183 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.8 MiB/s wr, 57 op/s 2024-09-14T04:17:31.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:31 smithi039 ceph-mon[106470]: pgmap v1142: 105 pgs: 105 active+clean; 183 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.8 MiB/s wr, 57 op/s 2024-09-14T04:17:31.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:31 smithi039 ceph-mon[111235]: pgmap v1142: 105 pgs: 105 active+clean; 183 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.8 MiB/s wr, 57 op/s 2024-09-14T04:17:33.524 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:33 smithi129 ceph-mon[101316]: pgmap v1143: 105 pgs: 105 active+clean; 204 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 5.8 MiB/s wr, 65 op/s 2024-09-14T04:17:33.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:33 smithi039 ceph-mon[111235]: pgmap v1143: 105 pgs: 105 active+clean; 204 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 5.8 MiB/s wr, 65 op/s 2024-09-14T04:17:33.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:33 smithi039 ceph-mon[106470]: pgmap v1143: 105 pgs: 105 active+clean; 204 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 5.8 MiB/s wr, 65 op/s 2024-09-14T04:17:35.249 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/2.ShrinkToOne (9989 ms) 2024-09-14T04:17:35.249 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/2.ShrinkToOneAfterSnapshot 2024-09-14T04:17:35.249 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:17:35.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:35 smithi129 ceph-mon[101316]: pgmap v1144: 105 pgs: 105 active+clean; 204 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 4.6 MiB/s wr, 56 op/s 2024-09-14T04:17:35.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:35 smithi039 ceph-mon[106470]: pgmap v1144: 105 pgs: 105 active+clean; 204 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 4.6 MiB/s wr, 56 op/s 2024-09-14T04:17:35.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:35 smithi039 ceph-mon[111235]: pgmap v1144: 105 pgs: 105 active+clean; 204 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.1 MiB/s rd, 4.6 MiB/s wr, 56 op/s 2024-09-14T04:17:37.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:36 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:17:37.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:36 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:17:37.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:36 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:17:38.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:38 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:17:38.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:38 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:17:38.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:38 smithi039 ceph-mon[106470]: pgmap v1145: 105 pgs: 105 active+clean; 204 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 4.2 MiB/s wr, 51 op/s 2024-09-14T04:17:38.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:38 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:17:38.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:38 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:17:38.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:38 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:17:38.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:38 smithi039 ceph-mon[111235]: pgmap v1145: 105 pgs: 105 active+clean; 204 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 4.2 MiB/s wr, 51 op/s 2024-09-14T04:17:38.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:38 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:17:38.330 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:38 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:17:38.330 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:38 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:17:38.330 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:38 smithi129 ceph-mon[101316]: pgmap v1145: 105 pgs: 105 active+clean; 204 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 4.9 MiB/s rd, 4.2 MiB/s wr, 51 op/s 2024-09-14T04:17:38.330 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:38 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:17:38.937 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:17:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:17:38] "GET /metrics HTTP/1.1" 200 36677 "" "Prometheus/2.51.0" 2024-09-14T04:17:40.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:40 smithi039 ceph-mon[106470]: osdmap e936: 8 total, 8 up, 8 in 2024-09-14T04:17:40.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:40 smithi039 ceph-mon[106470]: pgmap v1147: 105 pgs: 105 active+clean; 212 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.1 MiB/s wr, 55 op/s 2024-09-14T04:17:40.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:40 smithi039 ceph-mon[111235]: osdmap e936: 8 total, 8 up, 8 in 2024-09-14T04:17:40.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:40 smithi039 ceph-mon[111235]: pgmap v1147: 105 pgs: 105 active+clean; 212 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.1 MiB/s wr, 55 op/s 2024-09-14T04:17:40.441 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:40 smithi129 ceph-mon[101316]: osdmap e936: 8 total, 8 up, 8 in 2024-09-14T04:17:40.441 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:40 smithi129 ceph-mon[101316]: pgmap v1147: 105 pgs: 105 active+clean; 212 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.1 MiB/s wr, 55 op/s 2024-09-14T04:17:42.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:41 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:17:42.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:41 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:17:42.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:41 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:17:42.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:41 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:17:42.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:41 smithi129 ceph-mon[101316]: pgmap v1148: 105 pgs: 105 active+clean; 220 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 979 KiB/s rd, 4.1 MiB/s wr, 62 op/s 2024-09-14T04:17:42.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:41 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:17:42.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:41 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:17:42.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:41 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:17:42.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:41 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:17:42.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:41 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:17:42.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:41 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:17:42.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:41 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:17:42.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:41 smithi039 ceph-mon[106470]: pgmap v1148: 105 pgs: 105 active+clean; 220 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 979 KiB/s rd, 4.1 MiB/s wr, 62 op/s 2024-09-14T04:17:42.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:41 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:17:42.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:41 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:17:42.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:41 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:17:42.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:41 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:17:42.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:41 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:17:42.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:41 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:17:42.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:41 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:17:42.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:41 smithi039 ceph-mon[111235]: pgmap v1148: 105 pgs: 105 active+clean; 220 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 979 KiB/s rd, 4.1 MiB/s wr, 62 op/s 2024-09-14T04:17:42.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:41 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:17:42.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:41 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:17:42.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:41 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:17:43.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:43 smithi129 ceph-mon[101316]: pgmap v1149: 105 pgs: 105 active+clean; 241 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 918 KiB/s rd, 4.1 MiB/s wr, 56 op/s 2024-09-14T04:17:43.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:43 smithi039 ceph-mon[106470]: pgmap v1149: 105 pgs: 105 active+clean; 241 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 918 KiB/s rd, 4.1 MiB/s wr, 56 op/s 2024-09-14T04:17:43.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:43 smithi039 ceph-mon[111235]: pgmap v1149: 105 pgs: 105 active+clean; 241 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 918 KiB/s rd, 4.1 MiB/s wr, 56 op/s 2024-09-14T04:17:45.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:45 smithi039 ceph-mon[106470]: osdmap e937: 8 total, 8 up, 8 in 2024-09-14T04:17:45.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:45 smithi039 ceph-mon[106470]: pgmap v1151: 105 pgs: 105 active+clean; 286 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 11 MiB/s wr, 77 op/s 2024-09-14T04:17:45.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:45 smithi039 ceph-mon[111235]: osdmap e937: 8 total, 8 up, 8 in 2024-09-14T04:17:45.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:45 smithi039 ceph-mon[111235]: pgmap v1151: 105 pgs: 105 active+clean; 286 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 11 MiB/s wr, 77 op/s 2024-09-14T04:17:45.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:45 smithi129 ceph-mon[101316]: osdmap e937: 8 total, 8 up, 8 in 2024-09-14T04:17:45.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:45 smithi129 ceph-mon[101316]: pgmap v1151: 105 pgs: 105 active+clean; 286 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2.7 MiB/s rd, 11 MiB/s wr, 77 op/s 2024-09-14T04:17:46.593 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/2.ShrinkToOneAfterSnapshot (11344 ms) 2024-09-14T04:17:46.593 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/2.MinOverlap 2024-09-14T04:17:46.593 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:17:47.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:47 smithi129 ceph-mon[101316]: pgmap v1152: 105 pgs: 105 active+clean; 286 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 9.2 MiB/s wr, 61 op/s 2024-09-14T04:17:47.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:47 smithi039 ceph-mon[106470]: pgmap v1152: 105 pgs: 105 active+clean; 286 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 9.2 MiB/s wr, 61 op/s 2024-09-14T04:17:47.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:47 smithi039 ceph-mon[111235]: pgmap v1152: 105 pgs: 105 active+clean; 286 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.1 MiB/s rd, 9.2 MiB/s wr, 61 op/s 2024-09-14T04:17:49.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:17:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:17:48] "GET /metrics HTTP/1.1" 200 36677 "" "Prometheus/2.51.0" 2024-09-14T04:17:49.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:49 smithi129 ceph-mon[101316]: pgmap v1153: 105 pgs: 105 active+clean; 302 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 9.0 MiB/s wr, 70 op/s 2024-09-14T04:17:49.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:49 smithi039 ceph-mon[106470]: pgmap v1153: 105 pgs: 105 active+clean; 302 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 9.0 MiB/s wr, 70 op/s 2024-09-14T04:17:49.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:49 smithi039 ceph-mon[111235]: pgmap v1153: 105 pgs: 105 active+clean; 302 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 3.4 MiB/s rd, 9.0 MiB/s wr, 70 op/s 2024-09-14T04:17:51.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:51 smithi039 ceph-mon[106470]: osdmap e938: 8 total, 8 up, 8 in 2024-09-14T04:17:51.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:51 smithi039 ceph-mon[106470]: pgmap v1155: 105 pgs: 105 active+clean; 302 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 7.6 MiB/s wr, 41 op/s 2024-09-14T04:17:51.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:51 smithi039 ceph-mon[111235]: osdmap e938: 8 total, 8 up, 8 in 2024-09-14T04:17:51.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:51 smithi039 ceph-mon[111235]: pgmap v1155: 105 pgs: 105 active+clean; 302 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 7.6 MiB/s wr, 41 op/s 2024-09-14T04:17:51.586 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:51 smithi129 ceph-mon[101316]: osdmap e938: 8 total, 8 up, 8 in 2024-09-14T04:17:51.586 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:51 smithi129 ceph-mon[101316]: pgmap v1155: 105 pgs: 105 active+clean; 302 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 7.6 MiB/s wr, 41 op/s 2024-09-14T04:17:52.527 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:17:52.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:17:52.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:17:53.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:53 smithi039 ceph-mon[106470]: pgmap v1156: 105 pgs: 105 active+clean; 327 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 9.7 MiB/s wr, 73 op/s 2024-09-14T04:17:53.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:53 smithi039 ceph-mon[111235]: pgmap v1156: 105 pgs: 105 active+clean; 327 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 9.7 MiB/s wr, 73 op/s 2024-09-14T04:17:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:53 smithi129 ceph-mon[101316]: pgmap v1156: 105 pgs: 105 active+clean; 327 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 9.7 MiB/s wr, 73 op/s 2024-09-14T04:17:55.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:55 smithi129 ceph-mon[101316]: pgmap v1157: 105 pgs: 105 active+clean; 335 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 4.9 MiB/s wr, 67 op/s 2024-09-14T04:17:55.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:55 smithi039 ceph-mon[106470]: pgmap v1157: 105 pgs: 105 active+clean; 335 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 4.9 MiB/s wr, 67 op/s 2024-09-14T04:17:55.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:55 smithi039 ceph-mon[111235]: pgmap v1157: 105 pgs: 105 active+clean; 335 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.6 MiB/s rd, 4.9 MiB/s wr, 67 op/s 2024-09-14T04:17:56.722 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/2.MinOverlap (10130 ms) 2024-09-14T04:17:56.722 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/2.ZeroOverlap 2024-09-14T04:17:56.722 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:17:57.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:57 smithi129 ceph-mon[101316]: pgmap v1158: 105 pgs: 105 active+clean; 335 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 4.9 MiB/s wr, 67 op/s 2024-09-14T04:17:57.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:57 smithi039 ceph-mon[106470]: pgmap v1158: 105 pgs: 105 active+clean; 335 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 4.9 MiB/s wr, 67 op/s 2024-09-14T04:17:57.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:57 smithi039 ceph-mon[111235]: pgmap v1158: 105 pgs: 105 active+clean; 335 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 4.9 MiB/s wr, 67 op/s 2024-09-14T04:17:59.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:17:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:17:58] "GET /metrics HTTP/1.1" 200 36680 "" "Prometheus/2.51.0" 2024-09-14T04:17:59.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:17:59 smithi129 ceph-mon[101316]: pgmap v1159: 105 pgs: 105 active+clean; 339 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.1 MiB/s wr, 62 op/s 2024-09-14T04:17:59.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:17:59 smithi039 ceph-mon[106470]: pgmap v1159: 105 pgs: 105 active+clean; 339 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.1 MiB/s wr, 62 op/s 2024-09-14T04:17:59.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:17:59 smithi039 ceph-mon[111235]: pgmap v1159: 105 pgs: 105 active+clean; 339 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.1 MiB/s wr, 62 op/s 2024-09-14T04:18:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:01 smithi129 ceph-mon[101316]: osdmap e939: 8 total, 8 up, 8 in 2024-09-14T04:18:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:01 smithi129 ceph-mon[101316]: pgmap v1161: 105 pgs: 105 active+clean; 339 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.1 MiB/s wr, 60 op/s 2024-09-14T04:18:01.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:01 smithi039 ceph-mon[106470]: osdmap e939: 8 total, 8 up, 8 in 2024-09-14T04:18:01.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:01 smithi039 ceph-mon[106470]: pgmap v1161: 105 pgs: 105 active+clean; 339 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.1 MiB/s wr, 60 op/s 2024-09-14T04:18:01.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:01 smithi039 ceph-mon[111235]: osdmap e939: 8 total, 8 up, 8 in 2024-09-14T04:18:01.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:01 smithi039 ceph-mon[111235]: pgmap v1161: 105 pgs: 105 active+clean; 339 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.1 MiB/s wr, 60 op/s 2024-09-14T04:18:03.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:03 smithi129 ceph-mon[101316]: pgmap v1162: 105 pgs: 105 active+clean; 359 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.6 MiB/s wr, 58 op/s 2024-09-14T04:18:03.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:03 smithi039 ceph-mon[106470]: pgmap v1162: 105 pgs: 105 active+clean; 359 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.6 MiB/s wr, 58 op/s 2024-09-14T04:18:03.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:03 smithi039 ceph-mon[111235]: pgmap v1162: 105 pgs: 105 active+clean; 359 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 3.6 MiB/s wr, 58 op/s 2024-09-14T04:18:05.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:05 smithi129 ceph-mon[101316]: pgmap v1163: 105 pgs: 105 active+clean; 368 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.7 MiB/s wr, 58 op/s 2024-09-14T04:18:05.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:05 smithi039 ceph-mon[106470]: pgmap v1163: 105 pgs: 105 active+clean; 368 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.7 MiB/s wr, 58 op/s 2024-09-14T04:18:05.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:05 smithi039 ceph-mon[111235]: pgmap v1163: 105 pgs: 105 active+clean; 368 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.7 MiB/s wr, 58 op/s 2024-09-14T04:18:06.773 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/2.ZeroOverlap (10051 ms) 2024-09-14T04:18:06.841 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 7 tests from EncryptedFlattenTest/2 (75928 ms total) 2024-09-14T04:18:06.841 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:18:06.842 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 7 tests from EncryptedFlattenTest/3, where TypeParam = LUKS1UnderLUKS1 2024-09-14T04:18:06.869 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/3.Simple 2024-09-14T04:18:08.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:07 smithi129 ceph-mon[101316]: osdmap e940: 8 total, 8 up, 8 in 2024-09-14T04:18:08.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:07 smithi129 ceph-mon[101316]: pgmap v1165: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:18:08.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:18:08.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:07 smithi039 ceph-mon[106470]: osdmap e940: 8 total, 8 up, 8 in 2024-09-14T04:18:08.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:07 smithi039 ceph-mon[106470]: pgmap v1165: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:18:08.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:18:08.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:07 smithi039 ceph-mon[111235]: osdmap e940: 8 total, 8 up, 8 in 2024-09-14T04:18:08.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:07 smithi039 ceph-mon[111235]: pgmap v1165: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:18:08.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:18:08.865 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:18:09.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:08 smithi039 ceph-mon[106470]: osdmap e941: 8 total, 8 up, 8 in 2024-09-14T04:18:09.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:08 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1061064488' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-42","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:18:09.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:08 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-42","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:18:09.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:18:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:18:08] "GET /metrics HTTP/1.1" 200 34255 "" "Prometheus/2.51.0" 2024-09-14T04:18:09.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:08 smithi039 ceph-mon[111235]: osdmap e941: 8 total, 8 up, 8 in 2024-09-14T04:18:09.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:08 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1061064488' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-42","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:18:09.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:08 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-42","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:18:09.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:08 smithi129 ceph-mon[101316]: osdmap e941: 8 total, 8 up, 8 in 2024-09-14T04:18:09.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:08 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1061064488' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-42","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:18:09.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:08 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-42","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:18:10.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:09 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-42","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:18:10.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:09 smithi129 ceph-mon[101316]: osdmap e942: 8 total, 8 up, 8 in 2024-09-14T04:18:10.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:09 smithi129 ceph-mon[101316]: pgmap v1168: 105 pgs: 13 creating+peering, 19 unknown, 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:18:10.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:09 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-42","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:18:10.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:09 smithi039 ceph-mon[106470]: osdmap e942: 8 total, 8 up, 8 in 2024-09-14T04:18:10.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:09 smithi039 ceph-mon[106470]: pgmap v1168: 105 pgs: 13 creating+peering, 19 unknown, 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:18:10.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:09 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-42","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:18:10.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:09 smithi039 ceph-mon[111235]: osdmap e942: 8 total, 8 up, 8 in 2024-09-14T04:18:10.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:09 smithi039 ceph-mon[111235]: pgmap v1168: 105 pgs: 13 creating+peering, 19 unknown, 73 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:18:11.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:10 smithi129 ceph-mon[101316]: osdmap e943: 8 total, 8 up, 8 in 2024-09-14T04:18:11.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:10 smithi039 ceph-mon[106470]: osdmap e943: 8 total, 8 up, 8 in 2024-09-14T04:18:11.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:10 smithi039 ceph-mon[111235]: osdmap e943: 8 total, 8 up, 8 in 2024-09-14T04:18:12.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:11 smithi129 ceph-mon[101316]: osdmap e944: 8 total, 8 up, 8 in 2024-09-14T04:18:12.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:11 smithi129 ceph-mon[101316]: pgmap v1171: 105 pgs: 13 creating+peering, 8 unknown, 84 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:18:12.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:11 smithi039 ceph-mon[111235]: osdmap e944: 8 total, 8 up, 8 in 2024-09-14T04:18:12.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:11 smithi039 ceph-mon[111235]: pgmap v1171: 105 pgs: 13 creating+peering, 8 unknown, 84 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:18:12.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:11 smithi039 ceph-mon[106470]: osdmap e944: 8 total, 8 up, 8 in 2024-09-14T04:18:12.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:11 smithi039 ceph-mon[106470]: pgmap v1171: 105 pgs: 13 creating+peering, 8 unknown, 84 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:18:13.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:13 smithi129 ceph-mon[101316]: pgmap v1172: 105 pgs: 105 active+clean; 582 KiB data, 634 MiB used, 715 GiB / 715 GiB avail; 4.9 KiB/s rd, 769 B/s wr, 7 op/s 2024-09-14T04:18:13.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:13 smithi039 ceph-mon[106470]: pgmap v1172: 105 pgs: 105 active+clean; 582 KiB data, 634 MiB used, 715 GiB / 715 GiB avail; 4.9 KiB/s rd, 769 B/s wr, 7 op/s 2024-09-14T04:18:13.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:13 smithi039 ceph-mon[111235]: pgmap v1172: 105 pgs: 105 active+clean; 582 KiB data, 634 MiB used, 715 GiB / 715 GiB avail; 4.9 KiB/s rd, 769 B/s wr, 7 op/s 2024-09-14T04:18:16.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:15 smithi129 ceph-mon[101316]: osdmap e945: 8 total, 8 up, 8 in 2024-09-14T04:18:16.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:15 smithi129 ceph-mon[101316]: pgmap v1174: 105 pgs: 105 active+clean; 13 MiB data, 634 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 2.0 MiB/s wr, 22 op/s 2024-09-14T04:18:16.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:15 smithi039 ceph-mon[106470]: osdmap e945: 8 total, 8 up, 8 in 2024-09-14T04:18:16.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:15 smithi039 ceph-mon[106470]: pgmap v1174: 105 pgs: 105 active+clean; 13 MiB data, 634 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 2.0 MiB/s wr, 22 op/s 2024-09-14T04:18:16.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:15 smithi039 ceph-mon[111235]: osdmap e945: 8 total, 8 up, 8 in 2024-09-14T04:18:16.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:15 smithi039 ceph-mon[111235]: pgmap v1174: 105 pgs: 105 active+clean; 13 MiB data, 634 MiB used, 715 GiB / 715 GiB avail; 14 KiB/s rd, 2.0 MiB/s wr, 22 op/s 2024-09-14T04:18:16.902 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:16 smithi129 ceph-mon[101316]: osdmap e946: 8 total, 8 up, 8 in 2024-09-14T04:18:17.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:16 smithi039 ceph-mon[106470]: osdmap e946: 8 total, 8 up, 8 in 2024-09-14T04:18:17.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:16 smithi039 ceph-mon[111235]: osdmap e946: 8 total, 8 up, 8 in 2024-09-14T04:18:17.271 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 04:18:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=cleanup t=2024-09-14T04:18:16.900635978Z level=info msg="Completed cleanup jobs" duration=13.817179ms 2024-09-14T04:18:17.271 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 04:18:17 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=grafana.update.checker t=2024-09-14T04:18:17.037008809Z level=info msg="Update check succeeded" duration=65.030058ms 2024-09-14T04:18:17.271 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 04:18:17 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=plugins.update.checker t=2024-09-14T04:18:17.139446289Z level=info msg="Update check succeeded" duration=70.461794ms 2024-09-14T04:18:18.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:17 smithi129 ceph-mon[101316]: pgmap v1176: 105 pgs: 105 active+clean; 13 MiB data, 638 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 1.9 MiB/s wr, 21 op/s 2024-09-14T04:18:18.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:17 smithi039 ceph-mon[111235]: pgmap v1176: 105 pgs: 105 active+clean; 13 MiB data, 638 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 1.9 MiB/s wr, 21 op/s 2024-09-14T04:18:18.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:17 smithi039 ceph-mon[106470]: pgmap v1176: 105 pgs: 105 active+clean; 13 MiB data, 638 MiB used, 715 GiB / 715 GiB avail; 13 KiB/s rd, 1.9 MiB/s wr, 21 op/s 2024-09-14T04:18:19.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:18:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:18:18] "GET /metrics HTTP/1.1" 200 34255 "" "Prometheus/2.51.0" 2024-09-14T04:18:19.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:19 smithi129 ceph-mon[101316]: pgmap v1177: 105 pgs: 105 active+clean; 26 MiB data, 693 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 3.1 MiB/s wr, 46 op/s 2024-09-14T04:18:19.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:19 smithi039 ceph-mon[106470]: pgmap v1177: 105 pgs: 105 active+clean; 26 MiB data, 693 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 3.1 MiB/s wr, 46 op/s 2024-09-14T04:18:19.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:19 smithi039 ceph-mon[111235]: pgmap v1177: 105 pgs: 105 active+clean; 26 MiB data, 693 MiB used, 715 GiB / 715 GiB avail; 34 KiB/s rd, 3.1 MiB/s wr, 46 op/s 2024-09-14T04:18:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:21 smithi129 ceph-mon[101316]: pgmap v1178: 105 pgs: 105 active+clean; 30 MiB data, 725 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 3.6 MiB/s wr, 44 op/s 2024-09-14T04:18:22.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:21 smithi039 ceph-mon[106470]: pgmap v1178: 105 pgs: 105 active+clean; 30 MiB data, 725 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 3.6 MiB/s wr, 44 op/s 2024-09-14T04:18:22.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:21 smithi039 ceph-mon[111235]: pgmap v1178: 105 pgs: 105 active+clean; 30 MiB data, 725 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 3.6 MiB/s wr, 44 op/s 2024-09-14T04:18:23.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:18:23.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:18:23.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:18:24.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:23 smithi129 ceph-mon[101316]: pgmap v1179: 105 pgs: 105 active+clean; 42 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 5.2 MiB/s wr, 58 op/s 2024-09-14T04:18:24.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:23 smithi039 ceph-mon[106470]: pgmap v1179: 105 pgs: 105 active+clean; 42 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 5.2 MiB/s wr, 58 op/s 2024-09-14T04:18:24.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:23 smithi039 ceph-mon[111235]: pgmap v1179: 105 pgs: 105 active+clean; 42 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 3.6 MiB/s rd, 5.2 MiB/s wr, 58 op/s 2024-09-14T04:18:24.435 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/3.Simple (17566 ms) 2024-09-14T04:18:24.435 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/3.Grow 2024-09-14T04:18:24.435 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:18:25.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:25 smithi039 ceph-mon[106470]: pgmap v1180: 105 pgs: 105 active+clean; 54 MiB data, 753 MiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 4.5 MiB/s wr, 50 op/s 2024-09-14T04:18:25.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:25 smithi039 ceph-mon[111235]: pgmap v1180: 105 pgs: 105 active+clean; 54 MiB data, 753 MiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 4.5 MiB/s wr, 50 op/s 2024-09-14T04:18:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:25 smithi129 ceph-mon[101316]: pgmap v1180: 105 pgs: 105 active+clean; 54 MiB data, 753 MiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 4.5 MiB/s wr, 50 op/s 2024-09-14T04:18:27.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:27 smithi129 ceph-mon[101316]: pgmap v1181: 105 pgs: 105 active+clean; 59 MiB data, 766 MiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 4.3 MiB/s wr, 47 op/s 2024-09-14T04:18:27.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:27 smithi039 ceph-mon[106470]: pgmap v1181: 105 pgs: 105 active+clean; 59 MiB data, 766 MiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 4.3 MiB/s wr, 47 op/s 2024-09-14T04:18:27.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:27 smithi039 ceph-mon[111235]: pgmap v1181: 105 pgs: 105 active+clean; 59 MiB data, 766 MiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 4.3 MiB/s wr, 47 op/s 2024-09-14T04:18:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:18:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:18:28] "GET /metrics HTTP/1.1" 200 36667 "" "Prometheus/2.51.0" 2024-09-14T04:18:29.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:29 smithi129 ceph-mon[101316]: osdmap e947: 8 total, 8 up, 8 in 2024-09-14T04:18:29.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:29 smithi129 ceph-mon[101316]: pgmap v1183: 105 pgs: 105 active+clean; 59 MiB data, 808 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.7 MiB/s wr, 32 op/s 2024-09-14T04:18:29.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:29 smithi039 ceph-mon[106470]: osdmap e947: 8 total, 8 up, 8 in 2024-09-14T04:18:29.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:29 smithi039 ceph-mon[106470]: pgmap v1183: 105 pgs: 105 active+clean; 59 MiB data, 808 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.7 MiB/s wr, 32 op/s 2024-09-14T04:18:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:29 smithi039 ceph-mon[111235]: osdmap e947: 8 total, 8 up, 8 in 2024-09-14T04:18:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:29 smithi039 ceph-mon[111235]: pgmap v1183: 105 pgs: 105 active+clean; 59 MiB data, 808 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.7 MiB/s wr, 32 op/s 2024-09-14T04:18:31.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:31 smithi039 ceph-mon[106470]: pgmap v1184: 105 pgs: 105 active+clean; 63 MiB data, 829 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.7 MiB/s wr, 44 op/s 2024-09-14T04:18:31.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:31 smithi039 ceph-mon[111235]: pgmap v1184: 105 pgs: 105 active+clean; 63 MiB data, 829 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.7 MiB/s wr, 44 op/s 2024-09-14T04:18:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:31 smithi129 ceph-mon[101316]: pgmap v1184: 105 pgs: 105 active+clean; 63 MiB data, 829 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 3.7 MiB/s wr, 44 op/s 2024-09-14T04:18:33.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:33 smithi129 ceph-mon[101316]: pgmap v1185: 105 pgs: 105 active+clean; 92 MiB data, 899 MiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 5.0 MiB/s wr, 49 op/s 2024-09-14T04:18:33.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:33 smithi039 ceph-mon[106470]: pgmap v1185: 105 pgs: 105 active+clean; 92 MiB data, 899 MiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 5.0 MiB/s wr, 49 op/s 2024-09-14T04:18:33.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:33 smithi039 ceph-mon[111235]: pgmap v1185: 105 pgs: 105 active+clean; 92 MiB data, 899 MiB used, 714 GiB / 715 GiB avail; 7.5 MiB/s rd, 5.0 MiB/s wr, 49 op/s 2024-09-14T04:18:35.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:35 smithi039 ceph-mon[106470]: pgmap v1186: 105 pgs: 105 active+clean; 92 MiB data, 903 MiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 3.8 MiB/s wr, 43 op/s 2024-09-14T04:18:35.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:35 smithi039 ceph-mon[111235]: pgmap v1186: 105 pgs: 105 active+clean; 92 MiB data, 903 MiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 3.8 MiB/s wr, 43 op/s 2024-09-14T04:18:35.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:35 smithi129 ceph-mon[101316]: pgmap v1186: 105 pgs: 105 active+clean; 92 MiB data, 903 MiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 3.8 MiB/s wr, 43 op/s 2024-09-14T04:18:37.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:37 smithi129 ceph-mon[101316]: pgmap v1187: 105 pgs: 105 active+clean; 92 MiB data, 920 MiB used, 714 GiB / 715 GiB avail; 879 KiB/s rd, 3.3 MiB/s wr, 42 op/s 2024-09-14T04:18:37.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:18:37.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:37 smithi039 ceph-mon[106470]: pgmap v1187: 105 pgs: 105 active+clean; 92 MiB data, 920 MiB used, 714 GiB / 715 GiB avail; 879 KiB/s rd, 3.3 MiB/s wr, 42 op/s 2024-09-14T04:18:37.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:18:37.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:37 smithi039 ceph-mon[111235]: pgmap v1187: 105 pgs: 105 active+clean; 92 MiB data, 920 MiB used, 714 GiB / 715 GiB avail; 879 KiB/s rd, 3.3 MiB/s wr, 42 op/s 2024-09-14T04:18:37.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:18:38.247 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/3.Grow (13811 ms) 2024-09-14T04:18:38.247 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/3.Shrink 2024-09-14T04:18:38.247 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:18:39.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:18:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:18:38] "GET /metrics HTTP/1.1" 200 36673 "" "Prometheus/2.51.0" 2024-09-14T04:18:39.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:39 smithi039 ceph-mon[106470]: pgmap v1188: 105 pgs: 105 active+clean; 109 MiB data, 973 MiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 5.0 MiB/s wr, 47 op/s 2024-09-14T04:18:39.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:39 smithi039 ceph-mon[111235]: pgmap v1188: 105 pgs: 105 active+clean; 109 MiB data, 973 MiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 5.0 MiB/s wr, 47 op/s 2024-09-14T04:18:39.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:39 smithi129 ceph-mon[101316]: pgmap v1188: 105 pgs: 105 active+clean; 109 MiB data, 973 MiB used, 714 GiB / 715 GiB avail; 5.9 MiB/s rd, 5.0 MiB/s wr, 47 op/s 2024-09-14T04:18:41.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:41 smithi129 ceph-mon[101316]: pgmap v1189: 105 pgs: 105 active+clean; 117 MiB data, 982 MiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 5.2 MiB/s wr, 51 op/s 2024-09-14T04:18:41.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:41 smithi039 ceph-mon[106470]: pgmap v1189: 105 pgs: 105 active+clean; 117 MiB data, 982 MiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 5.2 MiB/s wr, 51 op/s 2024-09-14T04:18:41.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:41 smithi039 ceph-mon[111235]: pgmap v1189: 105 pgs: 105 active+clean; 117 MiB data, 982 MiB used, 714 GiB / 715 GiB avail; 7.4 MiB/s rd, 5.2 MiB/s wr, 51 op/s 2024-09-14T04:18:42.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:42 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:18:42.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:42 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:18:42.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:42 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:18:42.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:42 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:18:42.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:42 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:18:42.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:42 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:18:42.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:42 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:18:42.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:42 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:18:42.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:42 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:18:43.910 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:43 smithi129 ceph-mon[101316]: osdmap e948: 8 total, 8 up, 8 in 2024-09-14T04:18:43.910 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:43 smithi129 ceph-mon[101316]: pgmap v1191: 105 pgs: 105 active+clean; 121 MiB data, 1019 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.3 MiB/s wr, 54 op/s 2024-09-14T04:18:43.913 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:43 smithi039 ceph-mon[106470]: osdmap e948: 8 total, 8 up, 8 in 2024-09-14T04:18:43.913 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:43 smithi039 ceph-mon[106470]: pgmap v1191: 105 pgs: 105 active+clean; 121 MiB data, 1019 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.3 MiB/s wr, 54 op/s 2024-09-14T04:18:43.913 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:43 smithi039 ceph-mon[111235]: osdmap e948: 8 total, 8 up, 8 in 2024-09-14T04:18:43.913 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:43 smithi039 ceph-mon[111235]: pgmap v1191: 105 pgs: 105 active+clean; 121 MiB data, 1019 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.3 MiB/s wr, 54 op/s 2024-09-14T04:18:46.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:46 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:18:46.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:46 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:18:46.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:46 smithi039 ceph-mon[106470]: pgmap v1192: 105 pgs: 105 active+clean; 134 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 4.6 MiB/s wr, 60 op/s 2024-09-14T04:18:46.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:46 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:18:46.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:46 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:18:46.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:46 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:18:46.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:46 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:18:46.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:46 smithi039 ceph-mon[111235]: pgmap v1192: 105 pgs: 105 active+clean; 134 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 4.6 MiB/s wr, 60 op/s 2024-09-14T04:18:46.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:46 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:18:46.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:46 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:18:46.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:46 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:18:46.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:46 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:18:46.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:46 smithi129 ceph-mon[101316]: pgmap v1192: 105 pgs: 105 active+clean; 134 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 4.6 MiB/s wr, 60 op/s 2024-09-14T04:18:46.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:46 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:18:46.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:46 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:18:47.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:47 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:18:47.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:47 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:18:47.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:47 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:18:47.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:47 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:18:47.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:47 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:18:47.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:47 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:18:47.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:47 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:18:47.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:47 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:18:47.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:47 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:18:48.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:48 smithi129 ceph-mon[101316]: pgmap v1193: 105 pgs: 105 active+clean; 150 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.2 MiB/s wr, 57 op/s 2024-09-14T04:18:48.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:48 smithi039 ceph-mon[106470]: pgmap v1193: 105 pgs: 105 active+clean; 150 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.2 MiB/s wr, 57 op/s 2024-09-14T04:18:48.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:48 smithi039 ceph-mon[111235]: pgmap v1193: 105 pgs: 105 active+clean; 150 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 6.2 MiB/s wr, 57 op/s 2024-09-14T04:18:49.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:18:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:18:48] "GET /metrics HTTP/1.1" 200 36673 "" "Prometheus/2.51.0" 2024-09-14T04:18:49.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:49 smithi039 ceph-mon[106470]: pgmap v1194: 105 pgs: 105 active+clean; 150 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 4.1 MiB/s wr, 53 op/s 2024-09-14T04:18:49.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:49 smithi039 ceph-mon[111235]: pgmap v1194: 105 pgs: 105 active+clean; 150 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 4.1 MiB/s wr, 53 op/s 2024-09-14T04:18:49.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:49 smithi129 ceph-mon[101316]: pgmap v1194: 105 pgs: 105 active+clean; 150 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 4.1 MiB/s wr, 53 op/s 2024-09-14T04:18:51.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:51 smithi039 ceph-mon[106470]: pgmap v1195: 105 pgs: 105 active+clean; 150 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.3 MiB/s wr, 45 op/s 2024-09-14T04:18:51.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:51 smithi039 ceph-mon[111235]: pgmap v1195: 105 pgs: 105 active+clean; 150 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.3 MiB/s wr, 45 op/s 2024-09-14T04:18:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:51 smithi129 ceph-mon[101316]: pgmap v1195: 105 pgs: 105 active+clean; 150 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.3 MiB/s wr, 45 op/s 2024-09-14T04:18:52.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:18:52.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:18:52.672 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:18:52.682 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/3.Shrink (14436 ms) 2024-09-14T04:18:52.682 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/3.ShrinkToOne 2024-09-14T04:18:52.682 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:18:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:53 smithi129 ceph-mon[101316]: pgmap v1196: 105 pgs: 105 active+clean; 171 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 5.5 MiB/s wr, 59 op/s 2024-09-14T04:18:53.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:53 smithi039 ceph-mon[106470]: pgmap v1196: 105 pgs: 105 active+clean; 171 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 5.5 MiB/s wr, 59 op/s 2024-09-14T04:18:53.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:53 smithi039 ceph-mon[111235]: pgmap v1196: 105 pgs: 105 active+clean; 171 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 7.7 MiB/s rd, 5.5 MiB/s wr, 59 op/s 2024-09-14T04:18:55.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:55 smithi039 ceph-mon[106470]: pgmap v1197: 105 pgs: 105 active+clean; 175 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 4.8 MiB/s wr, 36 op/s 2024-09-14T04:18:55.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:55 smithi039 ceph-mon[111235]: pgmap v1197: 105 pgs: 105 active+clean; 175 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 4.8 MiB/s wr, 36 op/s 2024-09-14T04:18:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:55 smithi129 ceph-mon[101316]: pgmap v1197: 105 pgs: 105 active+clean; 175 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.4 MiB/s rd, 4.8 MiB/s wr, 36 op/s 2024-09-14T04:18:57.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:57 smithi039 ceph-mon[106470]: osdmap e949: 8 total, 8 up, 8 in 2024-09-14T04:18:57.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:57 smithi039 ceph-mon[106470]: pgmap v1199: 105 pgs: 105 active+clean; 179 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.3 MiB/s wr, 33 op/s 2024-09-14T04:18:57.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:57 smithi039 ceph-mon[111235]: osdmap e949: 8 total, 8 up, 8 in 2024-09-14T04:18:57.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:57 smithi039 ceph-mon[111235]: pgmap v1199: 105 pgs: 105 active+clean; 179 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.3 MiB/s wr, 33 op/s 2024-09-14T04:18:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:57 smithi129 ceph-mon[101316]: osdmap e949: 8 total, 8 up, 8 in 2024-09-14T04:18:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:57 smithi129 ceph-mon[101316]: pgmap v1199: 105 pgs: 105 active+clean; 179 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.3 MiB/s wr, 33 op/s 2024-09-14T04:18:59.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:18:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:18:58] "GET /metrics HTTP/1.1" 200 36680 "" "Prometheus/2.51.0" 2024-09-14T04:18:59.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:18:59 smithi039 ceph-mon[106470]: pgmap v1200: 105 pgs: 105 active+clean; 192 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 4.6 MiB/s wr, 59 op/s 2024-09-14T04:18:59.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:18:59 smithi039 ceph-mon[111235]: pgmap v1200: 105 pgs: 105 active+clean; 192 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 4.6 MiB/s wr, 59 op/s 2024-09-14T04:18:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:18:59 smithi129 ceph-mon[101316]: pgmap v1200: 105 pgs: 105 active+clean; 192 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 4.6 MiB/s wr, 59 op/s 2024-09-14T04:19:01.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:01 smithi039 ceph-mon[106470]: pgmap v1201: 105 pgs: 105 active+clean; 204 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.8 MiB/s wr, 58 op/s 2024-09-14T04:19:01.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:01 smithi039 ceph-mon[111235]: pgmap v1201: 105 pgs: 105 active+clean; 204 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.8 MiB/s wr, 58 op/s 2024-09-14T04:19:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:01 smithi129 ceph-mon[101316]: pgmap v1201: 105 pgs: 105 active+clean; 204 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.8 MiB/s wr, 58 op/s 2024-09-14T04:19:03.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:03 smithi039 ceph-mon[106470]: pgmap v1202: 105 pgs: 105 active+clean; 208 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 3.7 MiB/s wr, 50 op/s 2024-09-14T04:19:03.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:03 smithi039 ceph-mon[111235]: pgmap v1202: 105 pgs: 105 active+clean; 208 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 3.7 MiB/s wr, 50 op/s 2024-09-14T04:19:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:03 smithi129 ceph-mon[101316]: pgmap v1202: 105 pgs: 105 active+clean; 208 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 3.8 MiB/s rd, 3.7 MiB/s wr, 50 op/s 2024-09-14T04:19:05.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:05 smithi039 ceph-mon[106470]: pgmap v1203: 105 pgs: 105 active+clean; 208 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 908 KiB/s rd, 3.3 MiB/s wr, 41 op/s 2024-09-14T04:19:05.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:05 smithi039 ceph-mon[111235]: pgmap v1203: 105 pgs: 105 active+clean; 208 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 908 KiB/s rd, 3.3 MiB/s wr, 41 op/s 2024-09-14T04:19:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:05 smithi129 ceph-mon[101316]: pgmap v1203: 105 pgs: 105 active+clean; 208 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 908 KiB/s rd, 3.3 MiB/s wr, 41 op/s 2024-09-14T04:19:06.161 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/3.ShrinkToOne (13478 ms) 2024-09-14T04:19:06.161 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/3.ShrinkToOneAfterSnapshot 2024-09-14T04:19:06.161 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:19:07.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:07 smithi039 ceph-mon[106470]: pgmap v1204: 105 pgs: 105 active+clean; 208 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.4 MiB/s wr, 39 op/s 2024-09-14T04:19:07.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:19:07.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:07 smithi039 ceph-mon[111235]: pgmap v1204: 105 pgs: 105 active+clean; 208 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.4 MiB/s wr, 39 op/s 2024-09-14T04:19:07.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:19:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:07 smithi129 ceph-mon[101316]: pgmap v1204: 105 pgs: 105 active+clean; 208 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.4 MiB/s wr, 39 op/s 2024-09-14T04:19:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:19:08.946 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:19:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:19:08] "GET /metrics HTTP/1.1" 200 36681 "" "Prometheus/2.51.0" 2024-09-14T04:19:09.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:08 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.16", "id": [0, 4]} : dispatch 2024-09-14T04:19:09.283 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:08 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.18", "id": [0, 4]} : dispatch 2024-09-14T04:19:09.283 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:08 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.16", "id": [0, 4]} : dispatch 2024-09-14T04:19:09.283 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:08 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.18", "id": [0, 4]} : dispatch 2024-09-14T04:19:09.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:08 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.16", "id": [0, 4]} : dispatch 2024-09-14T04:19:09.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:08 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.18", "id": [0, 4]} : dispatch 2024-09-14T04:19:09.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.16", "id": [0, 4]} : dispatch 2024-09-14T04:19:09.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.18", "id": [0, 4]} : dispatch 2024-09-14T04:19:09.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:08 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.16", "id": [0, 4]} : dispatch 2024-09-14T04:19:09.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:08 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.18", "id": [0, 4]} : dispatch 2024-09-14T04:19:09.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:08 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.16", "id": [0, 4]} : dispatch 2024-09-14T04:19:09.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:08 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.18", "id": [0, 4]} : dispatch 2024-09-14T04:19:10.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:09 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.16", "id": [0, 4]}]': finished 2024-09-14T04:19:10.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:09 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.18", "id": [0, 4]}]': finished 2024-09-14T04:19:10.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:09 smithi129 ceph-mon[101316]: osdmap e950: 8 total, 8 up, 8 in 2024-09-14T04:19:10.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:09 smithi129 ceph-mon[101316]: pgmap v1206: 105 pgs: 105 active+clean; 212 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 923 KiB/s rd, 2.8 MiB/s wr, 25 op/s 2024-09-14T04:19:10.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:09 smithi129 ceph-mon[101316]: osdmap e951: 8 total, 8 up, 8 in 2024-09-14T04:19:10.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:09 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.16", "id": [0, 4]}]': finished 2024-09-14T04:19:10.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:09 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.18", "id": [0, 4]}]': finished 2024-09-14T04:19:10.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:09 smithi039 ceph-mon[106470]: osdmap e950: 8 total, 8 up, 8 in 2024-09-14T04:19:10.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:09 smithi039 ceph-mon[106470]: pgmap v1206: 105 pgs: 105 active+clean; 212 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 923 KiB/s rd, 2.8 MiB/s wr, 25 op/s 2024-09-14T04:19:10.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:09 smithi039 ceph-mon[106470]: osdmap e951: 8 total, 8 up, 8 in 2024-09-14T04:19:10.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:09 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.16", "id": [0, 4]}]': finished 2024-09-14T04:19:10.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:09 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "46.18", "id": [0, 4]}]': finished 2024-09-14T04:19:10.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:09 smithi039 ceph-mon[111235]: osdmap e950: 8 total, 8 up, 8 in 2024-09-14T04:19:10.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:09 smithi039 ceph-mon[111235]: pgmap v1206: 105 pgs: 105 active+clean; 212 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 923 KiB/s rd, 2.8 MiB/s wr, 25 op/s 2024-09-14T04:19:10.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:09 smithi039 ceph-mon[111235]: osdmap e951: 8 total, 8 up, 8 in 2024-09-14T04:19:11.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:11 smithi039 ceph-mon[106470]: pgmap v1208: 105 pgs: 1 peering, 104 active+clean; 216 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.0 MiB/s rd, 2.0 MiB/s wr, 20 op/s 2024-09-14T04:19:11.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:11 smithi039 ceph-mon[111235]: pgmap v1208: 105 pgs: 1 peering, 104 active+clean; 216 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.0 MiB/s rd, 2.0 MiB/s wr, 20 op/s 2024-09-14T04:19:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:11 smithi129 ceph-mon[101316]: pgmap v1208: 105 pgs: 1 peering, 104 active+clean; 216 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 1.0 MiB/s rd, 2.0 MiB/s wr, 20 op/s 2024-09-14T04:19:13.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:13 smithi039 ceph-mon[106470]: pgmap v1209: 105 pgs: 1 peering, 104 active+clean; 240 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 591 KiB/s rd, 5.0 MiB/s wr, 59 op/s; 1.5 MiB/s, 0 objects/s recovering 2024-09-14T04:19:13.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:13 smithi039 ceph-mon[111235]: pgmap v1209: 105 pgs: 1 peering, 104 active+clean; 240 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 591 KiB/s rd, 5.0 MiB/s wr, 59 op/s; 1.5 MiB/s, 0 objects/s recovering 2024-09-14T04:19:13.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:13 smithi129 ceph-mon[101316]: pgmap v1209: 105 pgs: 1 peering, 104 active+clean; 240 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 591 KiB/s rd, 5.0 MiB/s wr, 59 op/s; 1.5 MiB/s, 0 objects/s recovering 2024-09-14T04:19:15.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:15 smithi039 ceph-mon[106470]: pgmap v1210: 105 pgs: 105 active+clean; 245 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 80 KiB/s rd, 5.1 MiB/s wr, 59 op/s; 2.5 MiB/s, 0 objects/s recovering 2024-09-14T04:19:15.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:15 smithi039 ceph-mon[111235]: pgmap v1210: 105 pgs: 105 active+clean; 245 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 80 KiB/s rd, 5.1 MiB/s wr, 59 op/s; 2.5 MiB/s, 0 objects/s recovering 2024-09-14T04:19:15.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:15 smithi129 ceph-mon[101316]: pgmap v1210: 105 pgs: 105 active+clean; 245 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 80 KiB/s rd, 5.1 MiB/s wr, 59 op/s; 2.5 MiB/s, 0 objects/s recovering 2024-09-14T04:19:17.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:17 smithi039 ceph-mon[106470]: pgmap v1211: 105 pgs: 105 active+clean; 245 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 65 KiB/s rd, 4.0 MiB/s wr, 40 op/s; 2.4 MiB/s, 0 objects/s recovering 2024-09-14T04:19:17.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:17 smithi039 ceph-mon[111235]: pgmap v1211: 105 pgs: 105 active+clean; 245 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 65 KiB/s rd, 4.0 MiB/s wr, 40 op/s; 2.4 MiB/s, 0 objects/s recovering 2024-09-14T04:19:17.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:17 smithi129 ceph-mon[101316]: pgmap v1211: 105 pgs: 105 active+clean; 245 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 65 KiB/s rd, 4.0 MiB/s wr, 40 op/s; 2.4 MiB/s, 0 objects/s recovering 2024-09-14T04:19:19.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:19:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:19:18] "GET /metrics HTTP/1.1" 200 36681 "" "Prometheus/2.51.0" 2024-09-14T04:19:19.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:19 smithi039 ceph-mon[106470]: osdmap e952: 8 total, 8 up, 8 in 2024-09-14T04:19:19.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:19 smithi039 ceph-mon[106470]: pgmap v1213: 105 pgs: 105 active+clean; 245 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 95 KiB/s rd, 3.1 MiB/s wr, 45 op/s; 2.2 MiB/s, 0 objects/s recovering 2024-09-14T04:19:19.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:19 smithi039 ceph-mon[111235]: osdmap e952: 8 total, 8 up, 8 in 2024-09-14T04:19:19.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:19 smithi039 ceph-mon[111235]: pgmap v1213: 105 pgs: 105 active+clean; 245 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 95 KiB/s rd, 3.1 MiB/s wr, 45 op/s; 2.2 MiB/s, 0 objects/s recovering 2024-09-14T04:19:19.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:19 smithi129 ceph-mon[101316]: osdmap e952: 8 total, 8 up, 8 in 2024-09-14T04:19:19.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:19 smithi129 ceph-mon[101316]: pgmap v1213: 105 pgs: 105 active+clean; 245 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 95 KiB/s rd, 3.1 MiB/s wr, 45 op/s; 2.2 MiB/s, 0 objects/s recovering 2024-09-14T04:19:20.576 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/3.ShrinkToOneAfterSnapshot (14416 ms) 2024-09-14T04:19:20.576 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/3.MinOverlap 2024-09-14T04:19:20.576 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:19:21.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:21 smithi039 ceph-mon[106470]: pgmap v1214: 105 pgs: 105 active+clean; 257 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.0 MiB/s rd, 4.1 MiB/s wr, 42 op/s; 2.0 MiB/s, 0 objects/s recovering 2024-09-14T04:19:21.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:21 smithi039 ceph-mon[111235]: pgmap v1214: 105 pgs: 105 active+clean; 257 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.0 MiB/s rd, 4.1 MiB/s wr, 42 op/s; 2.0 MiB/s, 0 objects/s recovering 2024-09-14T04:19:21.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:21 smithi129 ceph-mon[101316]: pgmap v1214: 105 pgs: 105 active+clean; 257 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.0 MiB/s rd, 4.1 MiB/s wr, 42 op/s; 2.0 MiB/s, 0 objects/s recovering 2024-09-14T04:19:22.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:19:22.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:19:22.671 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:19:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:23 smithi039 ceph-mon[106470]: pgmap v1215: 105 pgs: 105 active+clean; 302 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 6.2 MiB/s wr, 44 op/s; 819 KiB/s, 0 objects/s recovering 2024-09-14T04:19:23.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:23 smithi039 ceph-mon[111235]: pgmap v1215: 105 pgs: 105 active+clean; 302 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 6.2 MiB/s wr, 44 op/s; 819 KiB/s, 0 objects/s recovering 2024-09-14T04:19:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:23 smithi129 ceph-mon[101316]: pgmap v1215: 105 pgs: 105 active+clean; 302 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 6.2 MiB/s wr, 44 op/s; 819 KiB/s, 0 objects/s recovering 2024-09-14T04:19:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:25 smithi129 ceph-mon[101316]: osdmap e953: 8 total, 8 up, 8 in 2024-09-14T04:19:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:25 smithi129 ceph-mon[101316]: pgmap v1217: 105 pgs: 105 active+clean; 307 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 7.8 MiB/s wr, 58 op/s 2024-09-14T04:19:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:25 smithi039 ceph-mon[106470]: osdmap e953: 8 total, 8 up, 8 in 2024-09-14T04:19:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:25 smithi039 ceph-mon[106470]: pgmap v1217: 105 pgs: 105 active+clean; 307 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 7.8 MiB/s wr, 58 op/s 2024-09-14T04:19:25.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:25 smithi039 ceph-mon[111235]: osdmap e953: 8 total, 8 up, 8 in 2024-09-14T04:19:25.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:25 smithi039 ceph-mon[111235]: pgmap v1217: 105 pgs: 105 active+clean; 307 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.7 MiB/s rd, 7.8 MiB/s wr, 58 op/s 2024-09-14T04:19:27.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:27 smithi039 ceph-mon[106470]: pgmap v1218: 105 pgs: 105 active+clean; 307 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 7.0 MiB/s wr, 42 op/s 2024-09-14T04:19:27.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:27 smithi039 ceph-mon[111235]: pgmap v1218: 105 pgs: 105 active+clean; 307 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 7.0 MiB/s wr, 42 op/s 2024-09-14T04:19:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:27 smithi129 ceph-mon[101316]: pgmap v1218: 105 pgs: 105 active+clean; 307 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3.3 MiB/s rd, 7.0 MiB/s wr, 42 op/s 2024-09-14T04:19:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:19:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:19:28] "GET /metrics HTTP/1.1" 200 36700 "" "Prometheus/2.51.0" 2024-09-14T04:19:29.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:29 smithi039 ceph-mon[106470]: pgmap v1219: 105 pgs: 105 active+clean; 335 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 9.0 MiB/s wr, 72 op/s 2024-09-14T04:19:29.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:29 smithi039 ceph-mon[111235]: pgmap v1219: 105 pgs: 105 active+clean; 335 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 9.0 MiB/s wr, 72 op/s 2024-09-14T04:19:29.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:29 smithi129 ceph-mon[101316]: pgmap v1219: 105 pgs: 105 active+clean; 335 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 9.0 MiB/s wr, 72 op/s 2024-09-14T04:19:31.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:31 smithi039 ceph-mon[106470]: pgmap v1220: 105 pgs: 105 active+clean; 335 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 7.8 MiB/s wr, 71 op/s 2024-09-14T04:19:31.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:31 smithi039 ceph-mon[111235]: pgmap v1220: 105 pgs: 105 active+clean; 335 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 7.8 MiB/s wr, 71 op/s 2024-09-14T04:19:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:31 smithi129 ceph-mon[101316]: pgmap v1220: 105 pgs: 105 active+clean; 335 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 7.8 MiB/s wr, 71 op/s 2024-09-14T04:19:33.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:33 smithi039 ceph-mon[106470]: pgmap v1221: 105 pgs: 105 active+clean; 335 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 945 KiB/s rd, 3.7 MiB/s wr, 53 op/s 2024-09-14T04:19:33.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:33 smithi039 ceph-mon[111235]: pgmap v1221: 105 pgs: 105 active+clean; 335 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 945 KiB/s rd, 3.7 MiB/s wr, 53 op/s 2024-09-14T04:19:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:33 smithi129 ceph-mon[101316]: pgmap v1221: 105 pgs: 105 active+clean; 335 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 945 KiB/s rd, 3.7 MiB/s wr, 53 op/s 2024-09-14T04:19:34.178 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/3.MinOverlap (13601 ms) 2024-09-14T04:19:34.178 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/3.ZeroOverlap 2024-09-14T04:19:34.179 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:19:35.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:35 smithi039 ceph-mon[106470]: pgmap v1222: 105 pgs: 105 active+clean; 335 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 868 KiB/s rd, 3.4 MiB/s wr, 49 op/s 2024-09-14T04:19:35.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:35 smithi039 ceph-mon[111235]: pgmap v1222: 105 pgs: 105 active+clean; 335 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 868 KiB/s rd, 3.4 MiB/s wr, 49 op/s 2024-09-14T04:19:35.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:35 smithi129 ceph-mon[101316]: pgmap v1222: 105 pgs: 105 active+clean; 335 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 868 KiB/s rd, 3.4 MiB/s wr, 49 op/s 2024-09-14T04:19:37.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:37 smithi039 ceph-mon[106470]: pgmap v1223: 105 pgs: 105 active+clean; 339 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.3 MiB/s wr, 43 op/s 2024-09-14T04:19:37.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:19:37.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:37 smithi039 ceph-mon[111235]: pgmap v1223: 105 pgs: 105 active+clean; 339 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.3 MiB/s wr, 43 op/s 2024-09-14T04:19:37.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:19:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:37 smithi129 ceph-mon[101316]: pgmap v1223: 105 pgs: 105 active+clean; 339 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.3 MiB/s wr, 43 op/s 2024-09-14T04:19:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:19:39.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:19:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:19:38] "GET /metrics HTTP/1.1" 200 36678 "" "Prometheus/2.51.0" 2024-09-14T04:19:39.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:39 smithi039 ceph-mon[106470]: osdmap e954: 8 total, 8 up, 8 in 2024-09-14T04:19:39.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:39 smithi039 ceph-mon[106470]: pgmap v1225: 105 pgs: 105 active+clean; 347 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.0 MiB/s wr, 22 op/s 2024-09-14T04:19:39.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:39 smithi039 ceph-mon[111235]: osdmap e954: 8 total, 8 up, 8 in 2024-09-14T04:19:39.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:39 smithi039 ceph-mon[111235]: pgmap v1225: 105 pgs: 105 active+clean; 347 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.0 MiB/s wr, 22 op/s 2024-09-14T04:19:39.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:39 smithi129 ceph-mon[101316]: osdmap e954: 8 total, 8 up, 8 in 2024-09-14T04:19:39.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:39 smithi129 ceph-mon[101316]: pgmap v1225: 105 pgs: 105 active+clean; 347 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 2.0 MiB/s wr, 22 op/s 2024-09-14T04:19:41.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:41 smithi039 ceph-mon[106470]: pgmap v1226: 105 pgs: 105 active+clean; 359 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.2 MiB/s wr, 23 op/s 2024-09-14T04:19:41.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:41 smithi039 ceph-mon[111235]: pgmap v1226: 105 pgs: 105 active+clean; 359 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.2 MiB/s wr, 23 op/s 2024-09-14T04:19:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:41 smithi129 ceph-mon[101316]: pgmap v1226: 105 pgs: 105 active+clean; 359 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 3.2 MiB/s wr, 23 op/s 2024-09-14T04:19:43.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:43 smithi039 ceph-mon[111235]: pgmap v1227: 105 pgs: 105 active+clean; 368 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.7 MiB/s wr, 46 op/s 2024-09-14T04:19:43.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:43 smithi039 ceph-mon[106470]: pgmap v1227: 105 pgs: 105 active+clean; 368 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.7 MiB/s wr, 46 op/s 2024-09-14T04:19:43.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:43 smithi129 ceph-mon[101316]: pgmap v1227: 105 pgs: 105 active+clean; 368 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.2 MiB/s rd, 3.7 MiB/s wr, 46 op/s 2024-09-14T04:19:45.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:45 smithi039 ceph-mon[106470]: pgmap v1228: 105 pgs: 105 active+clean; 372 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.1 MiB/s wr, 47 op/s 2024-09-14T04:19:45.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:45 smithi039 ceph-mon[111235]: pgmap v1228: 105 pgs: 105 active+clean; 372 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.1 MiB/s wr, 47 op/s 2024-09-14T04:19:45.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:45 smithi129 ceph-mon[101316]: pgmap v1228: 105 pgs: 105 active+clean; 372 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.1 MiB/s wr, 47 op/s 2024-09-14T04:19:46.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:46 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:19:46.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:46 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:19:46.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:46 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:19:47.631 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:47 smithi129 ceph-mon[101316]: pgmap v1229: 105 pgs: 105 active+clean; 372 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 91 KiB/s rd, 3.3 MiB/s wr, 46 op/s 2024-09-14T04:19:47.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:47 smithi039 ceph-mon[106470]: pgmap v1229: 105 pgs: 105 active+clean; 372 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 91 KiB/s rd, 3.3 MiB/s wr, 46 op/s 2024-09-14T04:19:47.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:47 smithi039 ceph-mon[111235]: pgmap v1229: 105 pgs: 105 active+clean; 372 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 91 KiB/s rd, 3.3 MiB/s wr, 46 op/s 2024-09-14T04:19:48.204 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/3.ZeroOverlap (14026 ms) 2024-09-14T04:19:48.393 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 7 tests from EncryptedFlattenTest/3 (101334 ms total) 2024-09-14T04:19:48.393 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:19:48.394 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 7 tests from EncryptedFlattenTest/4, where TypeParam = LUKS1UnderLUKS2 2024-09-14T04:19:48.421 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/4.Simple 2024-09-14T04:19:49.024 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:19:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:19:48] "GET /metrics HTTP/1.1" 200 36678 "" "Prometheus/2.51.0" 2024-09-14T04:19:49.540 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:49 smithi129 ceph-mon[101316]: osdmap e955: 8 total, 8 up, 8 in 2024-09-14T04:19:49.541 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:49 smithi129 ceph-mon[101316]: pgmap v1231: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:19:49.698 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:49 smithi039 ceph-mon[106470]: osdmap e955: 8 total, 8 up, 8 in 2024-09-14T04:19:49.698 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:49 smithi039 ceph-mon[106470]: pgmap v1231: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:19:49.698 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:49 smithi039 ceph-mon[111235]: osdmap e955: 8 total, 8 up, 8 in 2024-09-14T04:19:49.698 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:49 smithi039 ceph-mon[111235]: pgmap v1231: 73 pgs: 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:19:50.424 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:19:50.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:50 smithi129 ceph-mon[101316]: osdmap e956: 8 total, 8 up, 8 in 2024-09-14T04:19:50.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:50 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/498659855' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-43","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:19:50.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:50 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-43","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:19:50.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:50 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:19:50.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:50 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:19:50.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:50 smithi039 ceph-mon[106470]: osdmap e956: 8 total, 8 up, 8 in 2024-09-14T04:19:50.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:50 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/498659855' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-43","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:19:50.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:50 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-43","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:19:50.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:50 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:19:50.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:50 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:19:50.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:50 smithi039 ceph-mon[111235]: osdmap e956: 8 total, 8 up, 8 in 2024-09-14T04:19:50.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:50 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/498659855' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-43","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:19:50.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:50 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-43","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:19:50.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:50 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:19:50.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:50 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:19:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:51 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-43","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:19:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:51 smithi129 ceph-mon[101316]: osdmap e957: 8 total, 8 up, 8 in 2024-09-14T04:19:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:51 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:19:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:51 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:19:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:51 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:19:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:51 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:19:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:51 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:19:51.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:51 smithi129 ceph-mon[101316]: pgmap v1234: 105 pgs: 8 creating+peering, 24 unknown, 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:19:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:51 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-43","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:19:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:51 smithi039 ceph-mon[106470]: osdmap e957: 8 total, 8 up, 8 in 2024-09-14T04:19:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:51 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:19:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:51 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:19:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:51 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:19:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:51 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:19:51.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:51 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:19:51.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:51 smithi039 ceph-mon[106470]: pgmap v1234: 105 pgs: 8 creating+peering, 24 unknown, 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:19:51.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:51 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-43","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:19:51.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:51 smithi039 ceph-mon[111235]: osdmap e957: 8 total, 8 up, 8 in 2024-09-14T04:19:51.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:51 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:19:51.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:51 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:19:51.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:51 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:19:51.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:51 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:19:51.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:51 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:19:51.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:51 smithi039 ceph-mon[111235]: pgmap v1234: 105 pgs: 8 creating+peering, 24 unknown, 73 active+clean; 582 KiB data, 1.7 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:19:52.687 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:52 smithi129 ceph-mon[101316]: osdmap e958: 8 total, 8 up, 8 in 2024-09-14T04:19:52.687 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:19:52.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:52 smithi039 ceph-mon[106470]: osdmap e958: 8 total, 8 up, 8 in 2024-09-14T04:19:52.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:19:52.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:52 smithi039 ceph-mon[111235]: osdmap e958: 8 total, 8 up, 8 in 2024-09-14T04:19:52.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:19:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:53 smithi129 ceph-mon[101316]: osdmap e959: 8 total, 8 up, 8 in 2024-09-14T04:19:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:53 smithi129 ceph-mon[101316]: pgmap v1237: 105 pgs: 8 creating+peering, 97 active+clean; 582 KiB data, 888 MiB used, 714 GiB / 715 GiB avail; 7.0 KiB/s rd, 1.7 KiB/s wr, 12 op/s 2024-09-14T04:19:53.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:53 smithi039 ceph-mon[106470]: osdmap e959: 8 total, 8 up, 8 in 2024-09-14T04:19:53.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:53 smithi039 ceph-mon[106470]: pgmap v1237: 105 pgs: 8 creating+peering, 97 active+clean; 582 KiB data, 888 MiB used, 714 GiB / 715 GiB avail; 7.0 KiB/s rd, 1.7 KiB/s wr, 12 op/s 2024-09-14T04:19:53.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:53 smithi039 ceph-mon[111235]: osdmap e959: 8 total, 8 up, 8 in 2024-09-14T04:19:53.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:53 smithi039 ceph-mon[111235]: pgmap v1237: 105 pgs: 8 creating+peering, 97 active+clean; 582 KiB data, 888 MiB used, 714 GiB / 715 GiB avail; 7.0 KiB/s rd, 1.7 KiB/s wr, 12 op/s 2024-09-14T04:19:55.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:55 smithi039 ceph-mon[106470]: pgmap v1238: 105 pgs: 105 active+clean; 582 KiB data, 636 MiB used, 715 GiB / 715 GiB avail; 5.2 KiB/s rd, 1.4 KiB/s wr, 9 op/s 2024-09-14T04:19:55.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:55 smithi039 ceph-mon[111235]: pgmap v1238: 105 pgs: 105 active+clean; 582 KiB data, 636 MiB used, 715 GiB / 715 GiB avail; 5.2 KiB/s rd, 1.4 KiB/s wr, 9 op/s 2024-09-14T04:19:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:55 smithi129 ceph-mon[101316]: pgmap v1238: 105 pgs: 105 active+clean; 582 KiB data, 636 MiB used, 715 GiB / 715 GiB avail; 5.2 KiB/s rd, 1.4 KiB/s wr, 9 op/s 2024-09-14T04:19:57.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:57 smithi039 ceph-mon[106470]: osdmap e960: 8 total, 8 up, 8 in 2024-09-14T04:19:57.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:57 smithi039 ceph-mon[106470]: pgmap v1240: 105 pgs: 105 active+clean; 4.6 MiB data, 636 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 684 KiB/s wr, 8 op/s 2024-09-14T04:19:57.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:57 smithi039 ceph-mon[111235]: osdmap e960: 8 total, 8 up, 8 in 2024-09-14T04:19:57.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:57 smithi039 ceph-mon[111235]: pgmap v1240: 105 pgs: 105 active+clean; 4.6 MiB data, 636 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 684 KiB/s wr, 8 op/s 2024-09-14T04:19:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:57 smithi129 ceph-mon[101316]: osdmap e960: 8 total, 8 up, 8 in 2024-09-14T04:19:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:57 smithi129 ceph-mon[101316]: pgmap v1240: 105 pgs: 105 active+clean; 4.6 MiB data, 636 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 684 KiB/s wr, 8 op/s 2024-09-14T04:19:59.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:19:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:19:58] "GET /metrics HTTP/1.1" 200 36615 "" "Prometheus/2.51.0" 2024-09-14T04:19:59.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:19:59 smithi039 ceph-mon[106470]: pgmap v1241: 105 pgs: 105 active+clean; 26 MiB data, 694 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 3.2 MiB/s wr, 49 op/s 2024-09-14T04:19:59.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:19:59 smithi039 ceph-mon[111235]: pgmap v1241: 105 pgs: 105 active+clean; 26 MiB data, 694 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 3.2 MiB/s wr, 49 op/s 2024-09-14T04:19:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:19:59 smithi129 ceph-mon[101316]: pgmap v1241: 105 pgs: 105 active+clean; 26 MiB data, 694 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 3.2 MiB/s wr, 49 op/s 2024-09-14T04:20:00.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:00 smithi039 ceph-mon[106470]: overall HEALTH_OK 2024-09-14T04:20:00.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:00 smithi039 ceph-mon[111235]: overall HEALTH_OK 2024-09-14T04:20:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:00 smithi129 ceph-mon[101316]: overall HEALTH_OK 2024-09-14T04:20:02.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:01 smithi129 ceph-mon[101316]: osdmap e961: 8 total, 8 up, 8 in 2024-09-14T04:20:02.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:01 smithi129 ceph-mon[101316]: pgmap v1243: 105 pgs: 105 active+clean; 30 MiB data, 715 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 3.6 MiB/s wr, 44 op/s 2024-09-14T04:20:02.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:01 smithi039 ceph-mon[106470]: osdmap e961: 8 total, 8 up, 8 in 2024-09-14T04:20:02.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:01 smithi039 ceph-mon[106470]: pgmap v1243: 105 pgs: 105 active+clean; 30 MiB data, 715 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 3.6 MiB/s wr, 44 op/s 2024-09-14T04:20:02.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:01 smithi039 ceph-mon[111235]: osdmap e961: 8 total, 8 up, 8 in 2024-09-14T04:20:02.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:01 smithi039 ceph-mon[111235]: pgmap v1243: 105 pgs: 105 active+clean; 30 MiB data, 715 MiB used, 715 GiB / 715 GiB avail; 33 KiB/s rd, 3.6 MiB/s wr, 44 op/s 2024-09-14T04:20:03.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:03 smithi039 ceph-mon[106470]: pgmap v1244: 105 pgs: 105 active+clean; 42 MiB data, 755 MiB used, 714 GiB / 715 GiB avail; 78 KiB/s rd, 5.1 MiB/s wr, 57 op/s 2024-09-14T04:20:03.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:03 smithi039 ceph-mon[111235]: pgmap v1244: 105 pgs: 105 active+clean; 42 MiB data, 755 MiB used, 714 GiB / 715 GiB avail; 78 KiB/s rd, 5.1 MiB/s wr, 57 op/s 2024-09-14T04:20:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:03 smithi129 ceph-mon[101316]: pgmap v1244: 105 pgs: 105 active+clean; 42 MiB data, 755 MiB used, 714 GiB / 715 GiB avail; 78 KiB/s rd, 5.1 MiB/s wr, 57 op/s 2024-09-14T04:20:05.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:05 smithi039 ceph-mon[106470]: pgmap v1245: 105 pgs: 105 active+clean; 46 MiB data, 767 MiB used, 714 GiB / 715 GiB avail; 70 KiB/s rd, 5.0 MiB/s wr, 52 op/s 2024-09-14T04:20:05.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:05 smithi039 ceph-mon[111235]: pgmap v1245: 105 pgs: 105 active+clean; 46 MiB data, 767 MiB used, 714 GiB / 715 GiB avail; 70 KiB/s rd, 5.0 MiB/s wr, 52 op/s 2024-09-14T04:20:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:05 smithi129 ceph-mon[101316]: pgmap v1245: 105 pgs: 105 active+clean; 46 MiB data, 767 MiB used, 714 GiB / 715 GiB avail; 70 KiB/s rd, 5.0 MiB/s wr, 52 op/s 2024-09-14T04:20:07.348 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/4.Simple (18927 ms) 2024-09-14T04:20:07.349 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/4.Grow 2024-09-14T04:20:07.349 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:20:07.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:07 smithi039 ceph-mon[106470]: pgmap v1246: 105 pgs: 105 active+clean; 46 MiB data, 775 MiB used, 714 GiB / 715 GiB avail; 882 KiB/s rd, 4.1 MiB/s wr, 46 op/s 2024-09-14T04:20:07.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:20:07.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:07 smithi039 ceph-mon[111235]: pgmap v1246: 105 pgs: 105 active+clean; 46 MiB data, 775 MiB used, 714 GiB / 715 GiB avail; 882 KiB/s rd, 4.1 MiB/s wr, 46 op/s 2024-09-14T04:20:07.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:20:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:07 smithi129 ceph-mon[101316]: pgmap v1246: 105 pgs: 105 active+clean; 46 MiB data, 775 MiB used, 714 GiB / 715 GiB avail; 882 KiB/s rd, 4.1 MiB/s wr, 46 op/s 2024-09-14T04:20:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:20:09.031 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:20:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:20:08] "GET /metrics HTTP/1.1" 200 36669 "" "Prometheus/2.51.0" 2024-09-14T04:20:09.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:09 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "47.c", "id": [1, 2]} : dispatch 2024-09-14T04:20:09.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:09 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "47.c", "id": [1, 2]} : dispatch 2024-09-14T04:20:09.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:09 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "47.c", "id": [1, 2]} : dispatch 2024-09-14T04:20:09.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:09 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "47.c", "id": [1, 2]} : dispatch 2024-09-14T04:20:09.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:09 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "47.c", "id": [1, 2]} : dispatch 2024-09-14T04:20:09.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:09 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "47.c", "id": [1, 2]} : dispatch 2024-09-14T04:20:10.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:10 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "47.c", "id": [1, 2]}]': finished 2024-09-14T04:20:10.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:10 smithi129 ceph-mon[101316]: osdmap e962: 8 total, 8 up, 8 in 2024-09-14T04:20:10.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:10 smithi129 ceph-mon[101316]: pgmap v1248: 105 pgs: 105 active+clean; 63 MiB data, 817 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.3 MiB/s wr, 36 op/s 2024-09-14T04:20:10.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:10 smithi129 ceph-mon[101316]: osdmap e963: 8 total, 8 up, 8 in 2024-09-14T04:20:10.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:10 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "47.c", "id": [1, 2]}]': finished 2024-09-14T04:20:10.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:10 smithi039 ceph-mon[106470]: osdmap e962: 8 total, 8 up, 8 in 2024-09-14T04:20:10.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:10 smithi039 ceph-mon[106470]: pgmap v1248: 105 pgs: 105 active+clean; 63 MiB data, 817 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.3 MiB/s wr, 36 op/s 2024-09-14T04:20:10.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:10 smithi039 ceph-mon[106470]: osdmap e963: 8 total, 8 up, 8 in 2024-09-14T04:20:10.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:10 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "47.c", "id": [1, 2]}]': finished 2024-09-14T04:20:10.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:10 smithi039 ceph-mon[111235]: osdmap e962: 8 total, 8 up, 8 in 2024-09-14T04:20:10.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:10 smithi039 ceph-mon[111235]: pgmap v1248: 105 pgs: 105 active+clean; 63 MiB data, 817 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 5.3 MiB/s wr, 36 op/s 2024-09-14T04:20:10.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:10 smithi039 ceph-mon[111235]: osdmap e963: 8 total, 8 up, 8 in 2024-09-14T04:20:12.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:12 smithi129 ceph-mon[101316]: osdmap e964: 8 total, 8 up, 8 in 2024-09-14T04:20:12.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:12 smithi129 ceph-mon[101316]: pgmap v1251: 105 pgs: 1 peering, 104 active+clean; 71 MiB data, 846 MiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 6.8 MiB/s wr, 37 op/s 2024-09-14T04:20:12.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:12 smithi039 ceph-mon[106470]: osdmap e964: 8 total, 8 up, 8 in 2024-09-14T04:20:12.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:12 smithi039 ceph-mon[106470]: pgmap v1251: 105 pgs: 1 peering, 104 active+clean; 71 MiB data, 846 MiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 6.8 MiB/s wr, 37 op/s 2024-09-14T04:20:12.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:12 smithi039 ceph-mon[111235]: osdmap e964: 8 total, 8 up, 8 in 2024-09-14T04:20:12.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:12 smithi039 ceph-mon[111235]: pgmap v1251: 105 pgs: 1 peering, 104 active+clean; 71 MiB data, 846 MiB used, 714 GiB / 715 GiB avail; 19 MiB/s rd, 6.8 MiB/s wr, 37 op/s 2024-09-14T04:20:13.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:13 smithi039 ceph-mon[106470]: pgmap v1252: 105 pgs: 1 peering, 104 active+clean; 87 MiB data, 899 MiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 9.5 MiB/s wr, 82 op/s 2024-09-14T04:20:13.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:13 smithi039 ceph-mon[111235]: pgmap v1252: 105 pgs: 1 peering, 104 active+clean; 87 MiB data, 899 MiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 9.5 MiB/s wr, 82 op/s 2024-09-14T04:20:13.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:13 smithi129 ceph-mon[101316]: pgmap v1252: 105 pgs: 1 peering, 104 active+clean; 87 MiB data, 899 MiB used, 714 GiB / 715 GiB avail; 18 MiB/s rd, 9.5 MiB/s wr, 82 op/s 2024-09-14T04:20:15.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:15 smithi039 ceph-mon[106470]: pgmap v1253: 105 pgs: 105 active+clean; 100 MiB data, 929 MiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 6.6 MiB/s wr, 62 op/s 2024-09-14T04:20:15.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:15 smithi039 ceph-mon[111235]: pgmap v1253: 105 pgs: 105 active+clean; 100 MiB data, 929 MiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 6.6 MiB/s wr, 62 op/s 2024-09-14T04:20:15.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:15 smithi129 ceph-mon[101316]: pgmap v1253: 105 pgs: 105 active+clean; 100 MiB data, 929 MiB used, 714 GiB / 715 GiB avail; 4.6 MiB/s rd, 6.6 MiB/s wr, 62 op/s 2024-09-14T04:20:17.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:17 smithi039 ceph-mon[106470]: pgmap v1254: 105 pgs: 105 active+clean; 104 MiB data, 933 MiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 5.6 MiB/s wr, 48 op/s 2024-09-14T04:20:17.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:17 smithi039 ceph-mon[111235]: pgmap v1254: 105 pgs: 105 active+clean; 104 MiB data, 933 MiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 5.6 MiB/s wr, 48 op/s 2024-09-14T04:20:17.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:17 smithi129 ceph-mon[101316]: pgmap v1254: 105 pgs: 105 active+clean; 104 MiB data, 933 MiB used, 714 GiB / 715 GiB avail; 3.5 MiB/s rd, 5.6 MiB/s wr, 48 op/s 2024-09-14T04:20:19.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:20:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:20:18] "GET /metrics HTTP/1.1" 200 36669 "" "Prometheus/2.51.0" 2024-09-14T04:20:19.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:19 smithi039 ceph-mon[106470]: pgmap v1255: 105 pgs: 105 active+clean; 116 MiB data, 965 MiB used, 714 GiB / 715 GiB avail; 66 KiB/s rd, 4.9 MiB/s wr, 46 op/s 2024-09-14T04:20:19.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:19 smithi039 ceph-mon[111235]: pgmap v1255: 105 pgs: 105 active+clean; 116 MiB data, 965 MiB used, 714 GiB / 715 GiB avail; 66 KiB/s rd, 4.9 MiB/s wr, 46 op/s 2024-09-14T04:20:19.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:19 smithi129 ceph-mon[101316]: pgmap v1255: 105 pgs: 105 active+clean; 116 MiB data, 965 MiB used, 714 GiB / 715 GiB avail; 66 KiB/s rd, 4.9 MiB/s wr, 46 op/s 2024-09-14T04:20:21.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:21 smithi039 ceph-mon[106470]: pgmap v1256: 105 pgs: 105 active+clean; 116 MiB data, 969 MiB used, 714 GiB / 715 GiB avail; 62 KiB/s rd, 4.4 MiB/s wr, 46 op/s 2024-09-14T04:20:21.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:21 smithi039 ceph-mon[111235]: pgmap v1256: 105 pgs: 105 active+clean; 116 MiB data, 969 MiB used, 714 GiB / 715 GiB avail; 62 KiB/s rd, 4.4 MiB/s wr, 46 op/s 2024-09-14T04:20:21.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:21 smithi129 ceph-mon[101316]: pgmap v1256: 105 pgs: 105 active+clean; 116 MiB data, 969 MiB used, 714 GiB / 715 GiB avail; 62 KiB/s rd, 4.4 MiB/s wr, 46 op/s 2024-09-14T04:20:22.222 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/4.Grow (14874 ms) 2024-09-14T04:20:22.223 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/4.Shrink 2024-09-14T04:20:22.223 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:20:22.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:20:22.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:20:22.672 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:20:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:23 smithi129 ceph-mon[101316]: pgmap v1257: 105 pgs: 105 active+clean; 137 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 6.8 MiB/s wr, 53 op/s 2024-09-14T04:20:23.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:23 smithi039 ceph-mon[106470]: pgmap v1257: 105 pgs: 105 active+clean; 137 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 6.8 MiB/s wr, 53 op/s 2024-09-14T04:20:23.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:23 smithi039 ceph-mon[111235]: pgmap v1257: 105 pgs: 105 active+clean; 137 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 6.2 MiB/s rd, 6.8 MiB/s wr, 53 op/s 2024-09-14T04:20:25.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:25 smithi039 ceph-mon[106470]: pgmap v1258: 105 pgs: 105 active+clean; 141 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s rd, 5.8 MiB/s wr, 37 op/s 2024-09-14T04:20:25.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:25 smithi039 ceph-mon[111235]: pgmap v1258: 105 pgs: 105 active+clean; 141 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s rd, 5.8 MiB/s wr, 37 op/s 2024-09-14T04:20:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:25 smithi129 ceph-mon[101316]: pgmap v1258: 105 pgs: 105 active+clean; 141 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s rd, 5.8 MiB/s wr, 37 op/s 2024-09-14T04:20:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:27 smithi129 ceph-mon[101316]: osdmap e965: 8 total, 8 up, 8 in 2024-09-14T04:20:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:27 smithi129 ceph-mon[101316]: pgmap v1260: 105 pgs: 105 active+clean; 141 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.3 MiB/s wr, 36 op/s 2024-09-14T04:20:27.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:27 smithi039 ceph-mon[106470]: osdmap e965: 8 total, 8 up, 8 in 2024-09-14T04:20:27.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:27 smithi039 ceph-mon[106470]: pgmap v1260: 105 pgs: 105 active+clean; 141 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.3 MiB/s wr, 36 op/s 2024-09-14T04:20:27.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:27 smithi039 ceph-mon[111235]: osdmap e965: 8 total, 8 up, 8 in 2024-09-14T04:20:27.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:27 smithi039 ceph-mon[111235]: pgmap v1260: 105 pgs: 105 active+clean; 141 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.3 MiB/s wr, 36 op/s 2024-09-14T04:20:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:20:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:20:28] "GET /metrics HTTP/1.1" 200 36684 "" "Prometheus/2.51.0" 2024-09-14T04:20:29.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:29 smithi039 ceph-mon[106470]: pgmap v1261: 105 pgs: 105 active+clean; 162 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.2 MiB/s wr, 57 op/s 2024-09-14T04:20:29.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:29 smithi039 ceph-mon[111235]: pgmap v1261: 105 pgs: 105 active+clean; 162 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.2 MiB/s wr, 57 op/s 2024-09-14T04:20:29.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:29 smithi129 ceph-mon[101316]: pgmap v1261: 105 pgs: 105 active+clean; 162 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.2 MiB/s wr, 57 op/s 2024-09-14T04:20:31.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:31 smithi039 ceph-mon[106470]: pgmap v1262: 105 pgs: 105 active+clean; 170 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.0 MiB/s wr, 60 op/s 2024-09-14T04:20:31.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:31 smithi039 ceph-mon[111235]: pgmap v1262: 105 pgs: 105 active+clean; 170 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.0 MiB/s wr, 60 op/s 2024-09-14T04:20:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:31 smithi129 ceph-mon[101316]: pgmap v1262: 105 pgs: 105 active+clean; 170 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.0 MiB/s wr, 60 op/s 2024-09-14T04:20:33.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:33 smithi039 ceph-mon[106470]: pgmap v1263: 105 pgs: 105 active+clean; 170 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 3.3 MiB/s wr, 43 op/s 2024-09-14T04:20:33.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:33 smithi039 ceph-mon[111235]: pgmap v1263: 105 pgs: 105 active+clean; 170 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 3.3 MiB/s wr, 43 op/s 2024-09-14T04:20:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:33 smithi129 ceph-mon[101316]: pgmap v1263: 105 pgs: 105 active+clean; 170 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 4.3 MiB/s rd, 3.3 MiB/s wr, 43 op/s 2024-09-14T04:20:35.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:35 smithi039 ceph-mon[106470]: pgmap v1264: 105 pgs: 105 active+clean; 170 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 952 KiB/s rd, 2.9 MiB/s wr, 41 op/s 2024-09-14T04:20:35.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:35 smithi039 ceph-mon[111235]: pgmap v1264: 105 pgs: 105 active+clean; 170 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 952 KiB/s rd, 2.9 MiB/s wr, 41 op/s 2024-09-14T04:20:35.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:35 smithi129 ceph-mon[101316]: pgmap v1264: 105 pgs: 105 active+clean; 170 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 952 KiB/s rd, 2.9 MiB/s wr, 41 op/s 2024-09-14T04:20:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:37 smithi129 ceph-mon[101316]: pgmap v1265: 105 pgs: 105 active+clean; 170 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 872 KiB/s rd, 2.7 MiB/s wr, 38 op/s 2024-09-14T04:20:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:20:37.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:37 smithi039 ceph-mon[106470]: pgmap v1265: 105 pgs: 105 active+clean; 170 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 872 KiB/s rd, 2.7 MiB/s wr, 38 op/s 2024-09-14T04:20:37.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:20:37.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:37 smithi039 ceph-mon[111235]: pgmap v1265: 105 pgs: 105 active+clean; 170 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 872 KiB/s rd, 2.7 MiB/s wr, 38 op/s 2024-09-14T04:20:37.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:20:39.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:20:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:20:38] "GET /metrics HTTP/1.1" 200 36682 "" "Prometheus/2.51.0" 2024-09-14T04:20:39.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:39 smithi039 ceph-mon[106470]: pgmap v1266: 105 pgs: 105 active+clean; 186 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 76 KiB/s rd, 3.8 MiB/s wr, 38 op/s 2024-09-14T04:20:39.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:39 smithi039 ceph-mon[111235]: pgmap v1266: 105 pgs: 105 active+clean; 186 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 76 KiB/s rd, 3.8 MiB/s wr, 38 op/s 2024-09-14T04:20:39.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:39 smithi129 ceph-mon[101316]: pgmap v1266: 105 pgs: 105 active+clean; 186 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 76 KiB/s rd, 3.8 MiB/s wr, 38 op/s 2024-09-14T04:20:40.741 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/4.Shrink (18519 ms) 2024-09-14T04:20:40.741 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/4.ShrinkToOne 2024-09-14T04:20:40.741 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:20:41.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:41 smithi039 ceph-mon[106470]: pgmap v1267: 105 pgs: 105 active+clean; 195 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.8 MiB/s wr, 28 op/s 2024-09-14T04:20:41.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:41 smithi039 ceph-mon[111235]: pgmap v1267: 105 pgs: 105 active+clean; 195 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.8 MiB/s wr, 28 op/s 2024-09-14T04:20:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:41 smithi129 ceph-mon[101316]: pgmap v1267: 105 pgs: 105 active+clean; 195 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 2.2 MiB/s rd, 2.8 MiB/s wr, 28 op/s 2024-09-14T04:20:43.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:43 smithi039 ceph-mon[106470]: pgmap v1268: 105 pgs: 105 active+clean; 211 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s rd, 4.8 MiB/s wr, 27 op/s 2024-09-14T04:20:43.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:43 smithi039 ceph-mon[111235]: pgmap v1268: 105 pgs: 105 active+clean; 211 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s rd, 4.8 MiB/s wr, 27 op/s 2024-09-14T04:20:43.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:43 smithi129 ceph-mon[101316]: pgmap v1268: 105 pgs: 105 active+clean; 211 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.0 MiB/s rd, 4.8 MiB/s wr, 27 op/s 2024-09-14T04:20:45.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:45 smithi129 ceph-mon[101316]: osdmap e966: 8 total, 8 up, 8 in 2024-09-14T04:20:45.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:45 smithi129 ceph-mon[101316]: pgmap v1270: 105 pgs: 105 active+clean; 215 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.1 MiB/s wr, 50 op/s 2024-09-14T04:20:45.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:45 smithi039 ceph-mon[106470]: osdmap e966: 8 total, 8 up, 8 in 2024-09-14T04:20:45.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:45 smithi039 ceph-mon[106470]: pgmap v1270: 105 pgs: 105 active+clean; 215 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.1 MiB/s wr, 50 op/s 2024-09-14T04:20:45.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:45 smithi039 ceph-mon[111235]: osdmap e966: 8 total, 8 up, 8 in 2024-09-14T04:20:45.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:45 smithi039 ceph-mon[111235]: pgmap v1270: 105 pgs: 105 active+clean; 215 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.1 MiB/s wr, 50 op/s 2024-09-14T04:20:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:47 smithi129 ceph-mon[101316]: pgmap v1271: 105 pgs: 105 active+clean; 219 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.5 MiB/s wr, 51 op/s 2024-09-14T04:20:47.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:47 smithi039 ceph-mon[106470]: pgmap v1271: 105 pgs: 105 active+clean; 219 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.5 MiB/s wr, 51 op/s 2024-09-14T04:20:47.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:47 smithi039 ceph-mon[111235]: pgmap v1271: 105 pgs: 105 active+clean; 219 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 6.5 MiB/s wr, 51 op/s 2024-09-14T04:20:49.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:20:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:20:48] "GET /metrics HTTP/1.1" 200 36682 "" "Prometheus/2.51.0" 2024-09-14T04:20:49.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:49 smithi039 ceph-mon[106470]: pgmap v1272: 105 pgs: 105 active+clean; 240 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.0 MiB/s wr, 60 op/s 2024-09-14T04:20:49.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:49 smithi039 ceph-mon[111235]: pgmap v1272: 105 pgs: 105 active+clean; 240 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.0 MiB/s wr, 60 op/s 2024-09-14T04:20:49.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:49 smithi129 ceph-mon[101316]: pgmap v1272: 105 pgs: 105 active+clean; 240 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 7.0 MiB/s wr, 60 op/s 2024-09-14T04:20:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:51 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:20:51.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:51 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:20:51.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:51 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:20:52.349 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:52 smithi129 ceph-mon[101316]: pgmap v1273: 105 pgs: 105 active+clean; 240 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 6.1 MiB/s wr, 44 op/s 2024-09-14T04:20:52.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:52 smithi039 ceph-mon[106470]: pgmap v1273: 105 pgs: 105 active+clean; 240 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 6.1 MiB/s wr, 44 op/s 2024-09-14T04:20:52.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:52 smithi039 ceph-mon[111235]: pgmap v1273: 105 pgs: 105 active+clean; 240 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 6.1 MiB/s wr, 44 op/s 2024-09-14T04:20:53.183 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:53 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:20:53.184 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:53 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:20:53.365 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:53 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:20:54.196 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:54 smithi129 ceph-mon[101316]: pgmap v1274: 105 pgs: 105 active+clean; 248 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 984 KiB/s rd, 3.7 MiB/s wr, 46 op/s 2024-09-14T04:20:54.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:54 smithi039 ceph-mon[106470]: pgmap v1274: 105 pgs: 105 active+clean; 248 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 984 KiB/s rd, 3.7 MiB/s wr, 46 op/s 2024-09-14T04:20:54.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:54 smithi039 ceph-mon[111235]: pgmap v1274: 105 pgs: 105 active+clean; 248 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 984 KiB/s rd, 3.7 MiB/s wr, 46 op/s 2024-09-14T04:20:56.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:56 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:20:56.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:56 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:20:56.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:56 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:20:56.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:56 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:20:56.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:56 smithi129 ceph-mon[101316]: pgmap v1275: 105 pgs: 105 active+clean; 252 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 910 KiB/s rd, 3.8 MiB/s wr, 46 op/s 2024-09-14T04:20:56.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:56 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:20:56.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:56 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:20:56.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:56 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:20:56.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:56 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:20:56.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:56 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:20:56.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:56 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:20:56.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:56 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:20:56.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:56 smithi039 ceph-mon[106470]: pgmap v1275: 105 pgs: 105 active+clean; 252 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 910 KiB/s rd, 3.8 MiB/s wr, 46 op/s 2024-09-14T04:20:56.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:56 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:20:56.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:56 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:20:56.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:56 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:20:56.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:56 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:20:56.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:56 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:20:56.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:56 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:20:56.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:56 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:20:56.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:56 smithi039 ceph-mon[111235]: pgmap v1275: 105 pgs: 105 active+clean; 252 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 910 KiB/s rd, 3.8 MiB/s wr, 46 op/s 2024-09-14T04:20:56.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:56 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:20:56.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:56 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:20:56.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:56 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:20:57.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:57 smithi039 ceph-mon[106470]: pgmap v1276: 105 pgs: 105 active+clean; 256 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 39 KiB/s rd, 3.4 MiB/s wr, 22 op/s 2024-09-14T04:20:57.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:57 smithi039 ceph-mon[111235]: pgmap v1276: 105 pgs: 105 active+clean; 256 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 39 KiB/s rd, 3.4 MiB/s wr, 22 op/s 2024-09-14T04:20:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:57 smithi129 ceph-mon[101316]: pgmap v1276: 105 pgs: 105 active+clean; 256 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 39 KiB/s rd, 3.4 MiB/s wr, 22 op/s 2024-09-14T04:20:59.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:20:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:20:58] "GET /metrics HTTP/1.1" 200 36681 "" "Prometheus/2.51.0" 2024-09-14T04:20:59.370 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/4.ShrinkToOne (18629 ms) 2024-09-14T04:20:59.370 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/4.ShrinkToOneAfterSnapshot 2024-09-14T04:20:59.370 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:20:59.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:20:59 smithi039 ceph-mon[106470]: pgmap v1277: 105 pgs: 105 active+clean; 256 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 3.8 MiB/s wr, 29 op/s 2024-09-14T04:20:59.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:20:59 smithi039 ceph-mon[111235]: pgmap v1277: 105 pgs: 105 active+clean; 256 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 3.8 MiB/s wr, 29 op/s 2024-09-14T04:20:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:20:59 smithi129 ceph-mon[101316]: pgmap v1277: 105 pgs: 105 active+clean; 256 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.1 MiB/s rd, 3.8 MiB/s wr, 29 op/s 2024-09-14T04:21:01.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:01 smithi039 ceph-mon[106470]: pgmap v1278: 105 pgs: 105 active+clean; 260 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 3.0 MiB/s wr, 18 op/s 2024-09-14T04:21:01.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:01 smithi039 ceph-mon[111235]: pgmap v1278: 105 pgs: 105 active+clean; 260 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 3.0 MiB/s wr, 18 op/s 2024-09-14T04:21:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:01 smithi129 ceph-mon[101316]: pgmap v1278: 105 pgs: 105 active+clean; 260 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 1.4 MiB/s rd, 3.0 MiB/s wr, 18 op/s 2024-09-14T04:21:04.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:04 smithi039 ceph-mon[106470]: osdmap e967: 8 total, 8 up, 8 in 2024-09-14T04:21:04.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:04 smithi039 ceph-mon[106470]: pgmap v1280: 105 pgs: 105 active+clean; 277 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.9 MiB/s wr, 31 op/s 2024-09-14T04:21:04.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:04 smithi039 ceph-mon[111235]: osdmap e967: 8 total, 8 up, 8 in 2024-09-14T04:21:04.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:04 smithi039 ceph-mon[111235]: pgmap v1280: 105 pgs: 105 active+clean; 277 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.9 MiB/s wr, 31 op/s 2024-09-14T04:21:04.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:04 smithi129 ceph-mon[101316]: osdmap e967: 8 total, 8 up, 8 in 2024-09-14T04:21:04.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:04 smithi129 ceph-mon[101316]: pgmap v1280: 105 pgs: 105 active+clean; 277 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.9 MiB/s wr, 31 op/s 2024-09-14T04:21:05.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:05 smithi039 ceph-mon[106470]: pgmap v1281: 105 pgs: 105 active+clean; 277 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.5 MiB/s wr, 29 op/s 2024-09-14T04:21:05.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:05 smithi039 ceph-mon[111235]: pgmap v1281: 105 pgs: 105 active+clean; 277 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.5 MiB/s wr, 29 op/s 2024-09-14T04:21:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:05 smithi129 ceph-mon[101316]: pgmap v1281: 105 pgs: 105 active+clean; 277 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.5 MiB/s wr, 29 op/s 2024-09-14T04:21:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:07 smithi129 ceph-mon[101316]: pgmap v1282: 105 pgs: 105 active+clean; 285 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.9 MiB/s wr, 30 op/s 2024-09-14T04:21:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:21:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:07 smithi039 ceph-mon[106470]: pgmap v1282: 105 pgs: 105 active+clean; 285 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.9 MiB/s wr, 30 op/s 2024-09-14T04:21:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:21:07.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:07 smithi039 ceph-mon[111235]: pgmap v1282: 105 pgs: 105 active+clean; 285 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.9 MiB/s wr, 30 op/s 2024-09-14T04:21:07.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:21:09.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:21:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:21:08] "GET /metrics HTTP/1.1" 200 36684 "" "Prometheus/2.51.0" 2024-09-14T04:21:09.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:09 smithi039 ceph-mon[106470]: pgmap v1283: 105 pgs: 105 active+clean; 301 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 886 KiB/s rd, 5.7 MiB/s wr, 52 op/s 2024-09-14T04:21:09.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:09 smithi039 ceph-mon[111235]: pgmap v1283: 105 pgs: 105 active+clean; 301 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 886 KiB/s rd, 5.7 MiB/s wr, 52 op/s 2024-09-14T04:21:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:09 smithi129 ceph-mon[101316]: pgmap v1283: 105 pgs: 105 active+clean; 301 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 886 KiB/s rd, 5.7 MiB/s wr, 52 op/s 2024-09-14T04:21:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:11 smithi129 ceph-mon[101316]: osdmap e968: 8 total, 8 up, 8 in 2024-09-14T04:21:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:11 smithi129 ceph-mon[101316]: pgmap v1285: 105 pgs: 105 active+clean; 301 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 67 KiB/s rd, 2.9 MiB/s wr, 42 op/s 2024-09-14T04:21:11.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:11 smithi039 ceph-mon[106470]: osdmap e968: 8 total, 8 up, 8 in 2024-09-14T04:21:11.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:11 smithi039 ceph-mon[106470]: pgmap v1285: 105 pgs: 105 active+clean; 301 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 67 KiB/s rd, 2.9 MiB/s wr, 42 op/s 2024-09-14T04:21:11.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:11 smithi039 ceph-mon[111235]: osdmap e968: 8 total, 8 up, 8 in 2024-09-14T04:21:11.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:11 smithi039 ceph-mon[111235]: pgmap v1285: 105 pgs: 105 active+clean; 301 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 67 KiB/s rd, 2.9 MiB/s wr, 42 op/s 2024-09-14T04:21:11.832 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/4.ShrinkToOneAfterSnapshot (12461 ms) 2024-09-14T04:21:11.832 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/4.MinOverlap 2024-09-14T04:21:11.832 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:21:13.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:13 smithi039 ceph-mon[106470]: pgmap v1286: 105 pgs: 105 active+clean; 350 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 7.3 MiB/s wr, 71 op/s 2024-09-14T04:21:13.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:13 smithi039 ceph-mon[111235]: pgmap v1286: 105 pgs: 105 active+clean; 350 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 7.3 MiB/s wr, 71 op/s 2024-09-14T04:21:13.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:13 smithi129 ceph-mon[101316]: pgmap v1286: 105 pgs: 105 active+clean; 350 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 4.2 MiB/s rd, 7.3 MiB/s wr, 71 op/s 2024-09-14T04:21:15.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:15 smithi039 ceph-mon[111235]: pgmap v1287: 105 pgs: 105 active+clean; 374 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 9.7 MiB/s wr, 72 op/s 2024-09-14T04:21:15.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:15 smithi039 ceph-mon[106470]: pgmap v1287: 105 pgs: 105 active+clean; 374 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 9.7 MiB/s wr, 72 op/s 2024-09-14T04:21:15.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:15 smithi129 ceph-mon[101316]: pgmap v1287: 105 pgs: 105 active+clean; 374 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 9.7 MiB/s wr, 72 op/s 2024-09-14T04:21:17.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:17 smithi129 ceph-mon[101316]: osdmap e969: 8 total, 8 up, 8 in 2024-09-14T04:21:17.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:17 smithi129 ceph-mon[101316]: pgmap v1289: 105 pgs: 105 active+clean; 378 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 9.6 MiB/s wr, 56 op/s 2024-09-14T04:21:17.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:17 smithi039 ceph-mon[106470]: osdmap e969: 8 total, 8 up, 8 in 2024-09-14T04:21:17.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:17 smithi039 ceph-mon[106470]: pgmap v1289: 105 pgs: 105 active+clean; 378 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 9.6 MiB/s wr, 56 op/s 2024-09-14T04:21:17.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:17 smithi039 ceph-mon[111235]: osdmap e969: 8 total, 8 up, 8 in 2024-09-14T04:21:17.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:17 smithi039 ceph-mon[111235]: pgmap v1289: 105 pgs: 105 active+clean; 378 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 5.2 MiB/s rd, 9.6 MiB/s wr, 56 op/s 2024-09-14T04:21:19.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:21:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:21:18] "GET /metrics HTTP/1.1" 200 36684 "" "Prometheus/2.51.0" 2024-09-14T04:21:19.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:19 smithi129 ceph-mon[101316]: pgmap v1290: 105 pgs: 105 active+clean; 399 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 11 MiB/s wr, 82 op/s 2024-09-14T04:21:19.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:19 smithi039 ceph-mon[106470]: pgmap v1290: 105 pgs: 105 active+clean; 399 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 11 MiB/s wr, 82 op/s 2024-09-14T04:21:19.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:19 smithi039 ceph-mon[111235]: pgmap v1290: 105 pgs: 105 active+clean; 399 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.7 MiB/s rd, 11 MiB/s wr, 82 op/s 2024-09-14T04:21:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:21 smithi129 ceph-mon[101316]: pgmap v1291: 105 pgs: 105 active+clean; 403 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 10 MiB/s wr, 74 op/s 2024-09-14T04:21:22.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:21 smithi039 ceph-mon[106470]: pgmap v1291: 105 pgs: 105 active+clean; 403 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 10 MiB/s wr, 74 op/s 2024-09-14T04:21:22.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:21 smithi039 ceph-mon[111235]: pgmap v1291: 105 pgs: 105 active+clean; 403 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 10 MiB/s wr, 74 op/s 2024-09-14T04:21:23.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:21:23.357 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:21:23.358 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:21:24.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:23 smithi129 ceph-mon[101316]: pgmap v1292: 105 pgs: 105 active+clean; 403 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 5.3 MiB/s wr, 37 op/s 2024-09-14T04:21:24.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:23 smithi039 ceph-mon[106470]: pgmap v1292: 105 pgs: 105 active+clean; 403 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 5.3 MiB/s wr, 37 op/s 2024-09-14T04:21:24.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:23 smithi039 ceph-mon[111235]: pgmap v1292: 105 pgs: 105 active+clean; 403 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 28 KiB/s rd, 5.3 MiB/s wr, 37 op/s 2024-09-14T04:21:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:25 smithi129 ceph-mon[101316]: pgmap v1293: 105 pgs: 105 active+clean; 403 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 2.9 MiB/s wr, 35 op/s 2024-09-14T04:21:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:25 smithi039 ceph-mon[106470]: pgmap v1293: 105 pgs: 105 active+clean; 403 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 2.9 MiB/s wr, 35 op/s 2024-09-14T04:21:25.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:25 smithi039 ceph-mon[111235]: pgmap v1293: 105 pgs: 105 active+clean; 403 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 2.9 MiB/s wr, 35 op/s 2024-09-14T04:21:27.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:27 smithi039 ceph-mon[106470]: pgmap v1294: 105 pgs: 105 active+clean; 403 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 2.7 MiB/s wr, 34 op/s 2024-09-14T04:21:27.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:27 smithi039 ceph-mon[111235]: pgmap v1294: 105 pgs: 105 active+clean; 403 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 2.7 MiB/s wr, 34 op/s 2024-09-14T04:21:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:27 smithi129 ceph-mon[101316]: pgmap v1294: 105 pgs: 105 active+clean; 403 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 2.7 MiB/s wr, 34 op/s 2024-09-14T04:21:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:21:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:21:28] "GET /metrics HTTP/1.1" 200 36685 "" "Prometheus/2.51.0" 2024-09-14T04:21:29.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:29 smithi129 ceph-mon[101316]: pgmap v1295: 105 pgs: 105 active+clean; 419 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 756 KiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-09-14T04:21:29.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:29 smithi039 ceph-mon[106470]: pgmap v1295: 105 pgs: 105 active+clean; 419 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 756 KiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-09-14T04:21:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:29 smithi039 ceph-mon[111235]: pgmap v1295: 105 pgs: 105 active+clean; 419 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 756 KiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-09-14T04:21:29.990 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/4.MinOverlap (18159 ms) 2024-09-14T04:21:29.990 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/4.ZeroOverlap 2024-09-14T04:21:29.990 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:21:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:31 smithi129 ceph-mon[101316]: pgmap v1296: 105 pgs: 105 active+clean; 423 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 3.0 MiB/s wr, 13 op/s 2024-09-14T04:21:31.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:31 smithi039 ceph-mon[106470]: pgmap v1296: 105 pgs: 105 active+clean; 423 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 3.0 MiB/s wr, 13 op/s 2024-09-14T04:21:31.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:31 smithi039 ceph-mon[111235]: pgmap v1296: 105 pgs: 105 active+clean; 423 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 1.4 MiB/s rd, 3.0 MiB/s wr, 13 op/s 2024-09-14T04:21:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:33 smithi129 ceph-mon[101316]: pgmap v1297: 105 pgs: 105 active+clean; 423 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 3.3 MiB/s wr, 21 op/s 2024-09-14T04:21:33.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:33 smithi039 ceph-mon[106470]: pgmap v1297: 105 pgs: 105 active+clean; 423 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 3.3 MiB/s wr, 21 op/s 2024-09-14T04:21:33.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:33 smithi039 ceph-mon[111235]: pgmap v1297: 105 pgs: 105 active+clean; 423 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 3.3 MiB/s wr, 21 op/s 2024-09-14T04:21:35.592 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:35 smithi039 ceph-mon[106470]: osdmap e970: 8 total, 8 up, 8 in 2024-09-14T04:21:35.592 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:35 smithi039 ceph-mon[106470]: pgmap v1299: 105 pgs: 105 active+clean; 427 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 4.4 MiB/s wr, 50 op/s 2024-09-14T04:21:35.593 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:35 smithi039 ceph-mon[111235]: osdmap e970: 8 total, 8 up, 8 in 2024-09-14T04:21:35.593 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:35 smithi039 ceph-mon[111235]: pgmap v1299: 105 pgs: 105 active+clean; 427 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 4.4 MiB/s wr, 50 op/s 2024-09-14T04:21:35.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:35 smithi129 ceph-mon[101316]: osdmap e970: 8 total, 8 up, 8 in 2024-09-14T04:21:35.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:35 smithi129 ceph-mon[101316]: pgmap v1299: 105 pgs: 105 active+clean; 427 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 4.4 MiB/s wr, 50 op/s 2024-09-14T04:21:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:37 smithi129 ceph-mon[101316]: pgmap v1300: 105 pgs: 105 active+clean; 431 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 4.8 MiB/s wr, 55 op/s 2024-09-14T04:21:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:21:37.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:37 smithi039 ceph-mon[106470]: pgmap v1300: 105 pgs: 105 active+clean; 431 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 4.8 MiB/s wr, 55 op/s 2024-09-14T04:21:37.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:21:37.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:37 smithi039 ceph-mon[111235]: pgmap v1300: 105 pgs: 105 active+clean; 431 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 4.8 MiB/s wr, 55 op/s 2024-09-14T04:21:37.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:21:39.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:21:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:21:38] "GET /metrics HTTP/1.1" 200 36685 "" "Prometheus/2.51.0" 2024-09-14T04:21:39.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:39 smithi129 ceph-mon[101316]: pgmap v1301: 105 pgs: 105 active+clean; 452 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 4.5 MiB/s wr, 50 op/s 2024-09-14T04:21:39.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:39 smithi039 ceph-mon[106470]: pgmap v1301: 105 pgs: 105 active+clean; 452 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 4.5 MiB/s wr, 50 op/s 2024-09-14T04:21:39.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:39 smithi039 ceph-mon[111235]: pgmap v1301: 105 pgs: 105 active+clean; 452 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 4.5 MiB/s wr, 50 op/s 2024-09-14T04:21:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:41 smithi129 ceph-mon[101316]: pgmap v1302: 105 pgs: 105 active+clean; 452 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.7 MiB/s wr, 49 op/s 2024-09-14T04:21:41.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:41 smithi039 ceph-mon[106470]: pgmap v1302: 105 pgs: 105 active+clean; 452 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.7 MiB/s wr, 49 op/s 2024-09-14T04:21:41.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:41 smithi039 ceph-mon[111235]: pgmap v1302: 105 pgs: 105 active+clean; 452 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.7 MiB/s wr, 49 op/s 2024-09-14T04:21:43.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:43 smithi129 ceph-mon[101316]: pgmap v1303: 105 pgs: 105 active+clean; 464 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 472 KiB/s rd, 4.1 MiB/s wr, 46 op/s 2024-09-14T04:21:43.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:43 smithi039 ceph-mon[106470]: pgmap v1303: 105 pgs: 105 active+clean; 464 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 472 KiB/s rd, 4.1 MiB/s wr, 46 op/s 2024-09-14T04:21:43.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:43 smithi039 ceph-mon[111235]: pgmap v1303: 105 pgs: 105 active+clean; 464 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 472 KiB/s rd, 4.1 MiB/s wr, 46 op/s 2024-09-14T04:21:45.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:45 smithi129 ceph-mon[101316]: pgmap v1304: 105 pgs: 105 active+clean; 468 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 862 KiB/s rd, 4.5 MiB/s wr, 47 op/s 2024-09-14T04:21:45.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:45 smithi039 ceph-mon[106470]: pgmap v1304: 105 pgs: 105 active+clean; 468 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 862 KiB/s rd, 4.5 MiB/s wr, 47 op/s 2024-09-14T04:21:45.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:45 smithi039 ceph-mon[111235]: pgmap v1304: 105 pgs: 105 active+clean; 468 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 862 KiB/s rd, 4.5 MiB/s wr, 47 op/s 2024-09-14T04:21:45.984 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/4.ZeroOverlap (15993 ms) 2024-09-14T04:21:46.313 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 7 tests from EncryptedFlattenTest/4 (117562 ms total) 2024-09-14T04:21:46.314 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:21:46.314 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 7 tests from EncryptedFlattenTest/5, where TypeParam = UnformattedLUKS2 2024-09-14T04:21:46.347 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/5.Simple 2024-09-14T04:21:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:47 smithi129 ceph-mon[101316]: osdmap e971: 8 total, 8 up, 8 in 2024-09-14T04:21:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:47 smithi129 ceph-mon[101316]: pgmap v1306: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:21:47.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:47 smithi039 ceph-mon[106470]: osdmap e971: 8 total, 8 up, 8 in 2024-09-14T04:21:47.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:47 smithi039 ceph-mon[106470]: pgmap v1306: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:21:47.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:47 smithi039 ceph-mon[111235]: osdmap e971: 8 total, 8 up, 8 in 2024-09-14T04:21:47.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:47 smithi039 ceph-mon[111235]: pgmap v1306: 73 pgs: 73 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:21:48.348 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:21:48.649 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:48 smithi039 ceph-mon[106470]: osdmap e972: 8 total, 8 up, 8 in 2024-09-14T04:21:48.649 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:48 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3326073543' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-44","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:21:48.649 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:48 smithi039 ceph-mon[111235]: osdmap e972: 8 total, 8 up, 8 in 2024-09-14T04:21:48.650 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:48 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3326073543' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-44","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:21:48.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:48 smithi129 ceph-mon[101316]: osdmap e972: 8 total, 8 up, 8 in 2024-09-14T04:21:48.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:48 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3326073543' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-44","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:21:49.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:21:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:21:48] "GET /metrics HTTP/1.1" 200 36685 "" "Prometheus/2.51.0" 2024-09-14T04:21:49.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:49 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3326073543' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-44","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:21:49.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:49 smithi129 ceph-mon[101316]: osdmap e973: 8 total, 8 up, 8 in 2024-09-14T04:21:49.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:49 smithi129 ceph-mon[101316]: pgmap v1309: 105 pgs: 19 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:21:49.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:49 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3326073543' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-44","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:21:49.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:49 smithi039 ceph-mon[106470]: osdmap e973: 8 total, 8 up, 8 in 2024-09-14T04:21:49.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:49 smithi039 ceph-mon[106470]: pgmap v1309: 105 pgs: 19 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:21:49.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:49 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3326073543' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-44","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:21:49.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:49 smithi039 ceph-mon[111235]: osdmap e973: 8 total, 8 up, 8 in 2024-09-14T04:21:49.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:49 smithi039 ceph-mon[111235]: pgmap v1309: 105 pgs: 19 creating+peering, 13 unknown, 73 active+clean; 582 KiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:21:50.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:50 smithi129 ceph-mon[101316]: osdmap e974: 8 total, 8 up, 8 in 2024-09-14T04:21:50.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:50 smithi039 ceph-mon[106470]: osdmap e974: 8 total, 8 up, 8 in 2024-09-14T04:21:50.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:50 smithi039 ceph-mon[111235]: osdmap e974: 8 total, 8 up, 8 in 2024-09-14T04:21:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:51 smithi129 ceph-mon[101316]: osdmap e975: 8 total, 8 up, 8 in 2024-09-14T04:21:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:51 smithi129 ceph-mon[101316]: pgmap v1312: 105 pgs: 19 creating+peering, 5 unknown, 81 active+clean; 582 KiB data, 825 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:21:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:51 smithi039 ceph-mon[106470]: osdmap e975: 8 total, 8 up, 8 in 2024-09-14T04:21:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:51 smithi039 ceph-mon[106470]: pgmap v1312: 105 pgs: 19 creating+peering, 5 unknown, 81 active+clean; 582 KiB data, 825 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:21:51.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:51 smithi039 ceph-mon[111235]: osdmap e975: 8 total, 8 up, 8 in 2024-09-14T04:21:51.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:51 smithi039 ceph-mon[111235]: pgmap v1312: 105 pgs: 19 creating+peering, 5 unknown, 81 active+clean; 582 KiB data, 825 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:21:52.672 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:21:52.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:21:52.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:21:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:53 smithi129 ceph-mon[101316]: pgmap v1313: 105 pgs: 105 active+clean; 582 KiB data, 667 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.0 KiB/s wr, 3 op/s 2024-09-14T04:21:53.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:53 smithi039 ceph-mon[106470]: pgmap v1313: 105 pgs: 105 active+clean; 582 KiB data, 667 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.0 KiB/s wr, 3 op/s 2024-09-14T04:21:53.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:53 smithi039 ceph-mon[111235]: pgmap v1313: 105 pgs: 105 active+clean; 582 KiB data, 667 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s rd, 1.0 KiB/s wr, 3 op/s 2024-09-14T04:21:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:55 smithi129 ceph-mon[101316]: pgmap v1314: 105 pgs: 105 active+clean; 582 KiB data, 667 MiB used, 715 GiB / 715 GiB avail; 4.4 KiB/s rd, 1.2 KiB/s wr, 7 op/s 2024-09-14T04:21:55.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:55 smithi039 ceph-mon[106470]: pgmap v1314: 105 pgs: 105 active+clean; 582 KiB data, 667 MiB used, 715 GiB / 715 GiB avail; 4.4 KiB/s rd, 1.2 KiB/s wr, 7 op/s 2024-09-14T04:21:55.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:55 smithi039 ceph-mon[111235]: pgmap v1314: 105 pgs: 105 active+clean; 582 KiB data, 667 MiB used, 715 GiB / 715 GiB avail; 4.4 KiB/s rd, 1.2 KiB/s wr, 7 op/s 2024-09-14T04:21:56.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:56 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:21:56.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:56 smithi039 ceph-mon[106470]: osdmap e976: 8 total, 8 up, 8 in 2024-09-14T04:21:56.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:56 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:21:56.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:56 smithi039 ceph-mon[111235]: osdmap e976: 8 total, 8 up, 8 in 2024-09-14T04:21:56.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:56 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:21:56.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:56 smithi129 ceph-mon[101316]: osdmap e976: 8 total, 8 up, 8 in 2024-09-14T04:21:57.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:57 smithi129 ceph-mon[101316]: pgmap v1316: 105 pgs: 105 active+clean; 582 KiB data, 667 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 1.0 KiB/s wr, 6 op/s 2024-09-14T04:21:57.646 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:57 smithi039 ceph-mon[106470]: pgmap v1316: 105 pgs: 105 active+clean; 582 KiB data, 667 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 1.0 KiB/s wr, 6 op/s 2024-09-14T04:21:57.646 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:57 smithi039 ceph-mon[111235]: pgmap v1316: 105 pgs: 105 active+clean; 582 KiB data, 667 MiB used, 715 GiB / 715 GiB avail; 3.8 KiB/s rd, 1.0 KiB/s wr, 6 op/s 2024-09-14T04:21:58.826 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:21:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:21:58] "GET /metrics HTTP/1.1" 200 36631 "" "Prometheus/2.51.0" 2024-09-14T04:22:00.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:59 smithi129 ceph-mon[101316]: osdmap e977: 8 total, 8 up, 8 in 2024-09-14T04:22:00.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:59 smithi129 ceph-mon[101316]: pgmap v1318: 105 pgs: 105 active+clean; 582 KiB data, 667 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 1023 B/s wr, 6 op/s 2024-09-14T04:22:00.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:21:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:22:00.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:59 smithi039 ceph-mon[106470]: osdmap e977: 8 total, 8 up, 8 in 2024-09-14T04:22:00.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:59 smithi039 ceph-mon[106470]: pgmap v1318: 105 pgs: 105 active+clean; 582 KiB data, 667 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 1023 B/s wr, 6 op/s 2024-09-14T04:22:00.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:21:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:22:00.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:59 smithi039 ceph-mon[111235]: osdmap e977: 8 total, 8 up, 8 in 2024-09-14T04:22:00.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:59 smithi039 ceph-mon[111235]: pgmap v1318: 105 pgs: 105 active+clean; 582 KiB data, 667 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 1023 B/s wr, 6 op/s 2024-09-14T04:22:00.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:21:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:22:01.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:00 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:22:01.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:00 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:22:01.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:00 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:22:01.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:00 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:22:01.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:00 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:22:01.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:00 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:22:01.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:00 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:22:01.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:00 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:22:01.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:00 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:22:01.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:00 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:22:01.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:00 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:22:01.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:00 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:22:01.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:00 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:22:01.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:00 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:22:01.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:00 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:22:01.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:00 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:22:01.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:00 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:22:01.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:00 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:22:02.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:01 smithi129 ceph-mon[101316]: pgmap v1319: 105 pgs: 105 active+clean; 17 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 2.0 MiB/s wr, 38 op/s 2024-09-14T04:22:02.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:01 smithi039 ceph-mon[106470]: pgmap v1319: 105 pgs: 105 active+clean; 17 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 2.0 MiB/s wr, 38 op/s 2024-09-14T04:22:02.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:01 smithi039 ceph-mon[111235]: pgmap v1319: 105 pgs: 105 active+clean; 17 MiB data, 679 MiB used, 715 GiB / 715 GiB avail; 67 KiB/s rd, 2.0 MiB/s wr, 38 op/s 2024-09-14T04:22:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:03 smithi129 ceph-mon[101316]: pgmap v1320: 105 pgs: 105 active+clean; 42 MiB data, 774 MiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 5.1 MiB/s wr, 52 op/s 2024-09-14T04:22:03.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:03 smithi039 ceph-mon[106470]: pgmap v1320: 105 pgs: 105 active+clean; 42 MiB data, 774 MiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 5.1 MiB/s wr, 52 op/s 2024-09-14T04:22:03.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:03 smithi039 ceph-mon[111235]: pgmap v1320: 105 pgs: 105 active+clean; 42 MiB data, 774 MiB used, 714 GiB / 715 GiB avail; 75 KiB/s rd, 5.1 MiB/s wr, 52 op/s 2024-09-14T04:22:05.504 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/5.Simple (19158 ms) 2024-09-14T04:22:05.505 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/5.Grow 2024-09-14T04:22:05.505 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:22:05.592 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:05 smithi039 ceph-mon[111235]: pgmap v1321: 105 pgs: 105 active+clean; 50 MiB data, 823 MiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 5.1 MiB/s wr, 50 op/s 2024-09-14T04:22:05.592 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:05 smithi039 ceph-mon[106470]: pgmap v1321: 105 pgs: 105 active+clean; 50 MiB data, 823 MiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 5.1 MiB/s wr, 50 op/s 2024-09-14T04:22:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:05 smithi129 ceph-mon[101316]: pgmap v1321: 105 pgs: 105 active+clean; 50 MiB data, 823 MiB used, 714 GiB / 715 GiB avail; 3.0 MiB/s rd, 5.1 MiB/s wr, 50 op/s 2024-09-14T04:22:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:07 smithi129 ceph-mon[101316]: pgmap v1322: 105 pgs: 105 active+clean; 62 MiB data, 851 MiB used, 714 GiB / 715 GiB avail; 5.7 MiB/s rd, 6.1 MiB/s wr, 50 op/s 2024-09-14T04:22:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:22:07.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:07 smithi039 ceph-mon[106470]: pgmap v1322: 105 pgs: 105 active+clean; 62 MiB data, 851 MiB used, 714 GiB / 715 GiB avail; 5.7 MiB/s rd, 6.1 MiB/s wr, 50 op/s 2024-09-14T04:22:07.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:22:07.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:07 smithi039 ceph-mon[111235]: pgmap v1322: 105 pgs: 105 active+clean; 62 MiB data, 851 MiB used, 714 GiB / 715 GiB avail; 5.7 MiB/s rd, 6.1 MiB/s wr, 50 op/s 2024-09-14T04:22:07.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:22:09.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:22:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:22:08] "GET /metrics HTTP/1.1" 200 36669 "" "Prometheus/2.51.0" 2024-09-14T04:22:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:09 smithi129 ceph-mon[101316]: pgmap v1323: 105 pgs: 105 active+clean; 83 MiB data, 939 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 7.9 MiB/s wr, 65 op/s 2024-09-14T04:22:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:09 smithi039 ceph-mon[106470]: pgmap v1323: 105 pgs: 105 active+clean; 83 MiB data, 939 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 7.9 MiB/s wr, 65 op/s 2024-09-14T04:22:09.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:09 smithi039 ceph-mon[111235]: pgmap v1323: 105 pgs: 105 active+clean; 83 MiB data, 939 MiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 7.9 MiB/s wr, 65 op/s 2024-09-14T04:22:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:11 smithi129 ceph-mon[101316]: pgmap v1324: 105 pgs: 105 active+clean; 83 MiB data, 939 MiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 6.8 MiB/s wr, 57 op/s 2024-09-14T04:22:11.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:11 smithi039 ceph-mon[106470]: pgmap v1324: 105 pgs: 105 active+clean; 83 MiB data, 939 MiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 6.8 MiB/s wr, 57 op/s 2024-09-14T04:22:11.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:11 smithi039 ceph-mon[111235]: pgmap v1324: 105 pgs: 105 active+clean; 83 MiB data, 939 MiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 6.8 MiB/s wr, 57 op/s 2024-09-14T04:22:13.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:13 smithi129 ceph-mon[101316]: pgmap v1325: 105 pgs: 105 active+clean; 83 MiB data, 939 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 5.5 MiB/s wr, 34 op/s 2024-09-14T04:22:13.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:13 smithi039 ceph-mon[106470]: pgmap v1325: 105 pgs: 105 active+clean; 83 MiB data, 939 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 5.5 MiB/s wr, 34 op/s 2024-09-14T04:22:13.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:13 smithi039 ceph-mon[111235]: pgmap v1325: 105 pgs: 105 active+clean; 83 MiB data, 939 MiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 5.5 MiB/s wr, 34 op/s 2024-09-14T04:22:15.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:15 smithi129 ceph-mon[101316]: osdmap e978: 8 total, 8 up, 8 in 2024-09-14T04:22:15.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:15 smithi129 ceph-mon[101316]: pgmap v1327: 105 pgs: 105 active+clean; 91 MiB data, 972 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 4.1 MiB/s wr, 20 op/s 2024-09-14T04:22:15.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:15 smithi039 ceph-mon[106470]: osdmap e978: 8 total, 8 up, 8 in 2024-09-14T04:22:15.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:15 smithi039 ceph-mon[106470]: pgmap v1327: 105 pgs: 105 active+clean; 91 MiB data, 972 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 4.1 MiB/s wr, 20 op/s 2024-09-14T04:22:15.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:15 smithi039 ceph-mon[111235]: osdmap e978: 8 total, 8 up, 8 in 2024-09-14T04:22:15.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:15 smithi039 ceph-mon[111235]: pgmap v1327: 105 pgs: 105 active+clean; 91 MiB data, 972 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 4.1 MiB/s wr, 20 op/s 2024-09-14T04:22:17.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:17 smithi129 ceph-mon[101316]: pgmap v1328: 105 pgs: 105 active+clean; 99 MiB data, 989 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 3.7 MiB/s wr, 19 op/s 2024-09-14T04:22:17.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:17 smithi039 ceph-mon[106470]: pgmap v1328: 105 pgs: 105 active+clean; 99 MiB data, 989 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 3.7 MiB/s wr, 19 op/s 2024-09-14T04:22:17.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:17 smithi039 ceph-mon[111235]: pgmap v1328: 105 pgs: 105 active+clean; 99 MiB data, 989 MiB used, 714 GiB / 715 GiB avail; 6.0 MiB/s rd, 3.7 MiB/s wr, 19 op/s 2024-09-14T04:22:19.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:22:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:22:18] "GET /metrics HTTP/1.1" 200 36669 "" "Prometheus/2.51.0" 2024-09-14T04:22:19.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:19 smithi129 ceph-mon[101316]: pgmap v1329: 105 pgs: 105 active+clean; 124 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 498 KiB/s rd, 4.1 MiB/s wr, 41 op/s 2024-09-14T04:22:19.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:19 smithi039 ceph-mon[106470]: pgmap v1329: 105 pgs: 105 active+clean; 124 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 498 KiB/s rd, 4.1 MiB/s wr, 41 op/s 2024-09-14T04:22:19.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:19 smithi039 ceph-mon[111235]: pgmap v1329: 105 pgs: 105 active+clean; 124 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 498 KiB/s rd, 4.1 MiB/s wr, 41 op/s 2024-09-14T04:22:20.726 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/5.Grow (15222 ms) 2024-09-14T04:22:20.726 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/5.Shrink 2024-09-14T04:22:20.726 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:22:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:21 smithi129 ceph-mon[101316]: pgmap v1330: 105 pgs: 105 active+clean; 132 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.9 MiB/s wr, 43 op/s 2024-09-14T04:22:22.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:21 smithi039 ceph-mon[106470]: pgmap v1330: 105 pgs: 105 active+clean; 132 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.9 MiB/s wr, 43 op/s 2024-09-14T04:22:22.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:21 smithi039 ceph-mon[111235]: pgmap v1330: 105 pgs: 105 active+clean; 132 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 4.9 MiB/s wr, 43 op/s 2024-09-14T04:22:23.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:22:23.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:22:23.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:22:24.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:23 smithi129 ceph-mon[101316]: pgmap v1331: 105 pgs: 105 active+clean; 165 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 8.2 MiB/s wr, 66 op/s 2024-09-14T04:22:24.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:23 smithi039 ceph-mon[106470]: pgmap v1331: 105 pgs: 105 active+clean; 165 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 8.2 MiB/s wr, 66 op/s 2024-09-14T04:22:24.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:23 smithi039 ceph-mon[111235]: pgmap v1331: 105 pgs: 105 active+clean; 165 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 8.2 MiB/s wr, 66 op/s 2024-09-14T04:22:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:25 smithi129 ceph-mon[101316]: pgmap v1332: 105 pgs: 105 active+clean; 165 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 7.1 MiB/s wr, 64 op/s 2024-09-14T04:22:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:25 smithi039 ceph-mon[106470]: pgmap v1332: 105 pgs: 105 active+clean; 165 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 7.1 MiB/s wr, 64 op/s 2024-09-14T04:22:25.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:25 smithi039 ceph-mon[111235]: pgmap v1332: 105 pgs: 105 active+clean; 165 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 7.1 MiB/s wr, 64 op/s 2024-09-14T04:22:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:27 smithi129 ceph-mon[101316]: pgmap v1333: 105 pgs: 105 active+clean; 165 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 6.2 MiB/s wr, 58 op/s 2024-09-14T04:22:27.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:27 smithi039 ceph-mon[106470]: pgmap v1333: 105 pgs: 105 active+clean; 165 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 6.2 MiB/s wr, 58 op/s 2024-09-14T04:22:27.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:27 smithi039 ceph-mon[111235]: pgmap v1333: 105 pgs: 105 active+clean; 165 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 6.2 MiB/s wr, 58 op/s 2024-09-14T04:22:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:22:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:22:28] "GET /metrics HTTP/1.1" 200 36680 "" "Prometheus/2.51.0" 2024-09-14T04:22:29.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:29 smithi129 ceph-mon[101316]: osdmap e979: 8 total, 8 up, 8 in 2024-09-14T04:22:29.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:29 smithi129 ceph-mon[101316]: pgmap v1335: 105 pgs: 105 active+clean; 165 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 4.1 MiB/s wr, 29 op/s 2024-09-14T04:22:29.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:29 smithi039 ceph-mon[106470]: osdmap e979: 8 total, 8 up, 8 in 2024-09-14T04:22:29.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:29 smithi039 ceph-mon[106470]: pgmap v1335: 105 pgs: 105 active+clean; 165 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 4.1 MiB/s wr, 29 op/s 2024-09-14T04:22:29.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:29 smithi039 ceph-mon[111235]: osdmap e979: 8 total, 8 up, 8 in 2024-09-14T04:22:29.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:29 smithi039 ceph-mon[111235]: pgmap v1335: 105 pgs: 105 active+clean; 165 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 4.1 MiB/s wr, 29 op/s 2024-09-14T04:22:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:31 smithi129 ceph-mon[101316]: pgmap v1336: 105 pgs: 105 active+clean; 169 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 3.7 MiB/s wr, 56 op/s 2024-09-14T04:22:31.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:31 smithi039 ceph-mon[106470]: pgmap v1336: 105 pgs: 105 active+clean; 169 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 3.7 MiB/s wr, 56 op/s 2024-09-14T04:22:31.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:31 smithi039 ceph-mon[111235]: pgmap v1336: 105 pgs: 105 active+clean; 169 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 3.7 MiB/s wr, 56 op/s 2024-09-14T04:22:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:33 smithi129 ceph-mon[101316]: pgmap v1337: 105 pgs: 105 active+clean; 235 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 7.0 MiB/s wr, 57 op/s 2024-09-14T04:22:33.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:33 smithi039 ceph-mon[106470]: pgmap v1337: 105 pgs: 105 active+clean; 235 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 7.0 MiB/s wr, 57 op/s 2024-09-14T04:22:33.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:33 smithi039 ceph-mon[111235]: pgmap v1337: 105 pgs: 105 active+clean; 235 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4.4 MiB/s rd, 7.0 MiB/s wr, 57 op/s 2024-09-14T04:22:34.786 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/5.Shrink (14060 ms) 2024-09-14T04:22:34.787 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/5.ShrinkToOne 2024-09-14T04:22:34.787 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:22:35.592 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:35 smithi039 ceph-mon[106470]: pgmap v1338: 105 pgs: 105 active+clean; 243 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 7.8 MiB/s wr, 63 op/s 2024-09-14T04:22:35.593 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:35 smithi039 ceph-mon[111235]: pgmap v1338: 105 pgs: 105 active+clean; 243 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 7.8 MiB/s wr, 63 op/s 2024-09-14T04:22:35.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:35 smithi129 ceph-mon[101316]: pgmap v1338: 105 pgs: 105 active+clean; 243 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.2 MiB/s rd, 7.8 MiB/s wr, 63 op/s 2024-09-14T04:22:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:37 smithi129 ceph-mon[101316]: pgmap v1339: 105 pgs: 105 active+clean; 243 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 7.8 MiB/s wr, 64 op/s 2024-09-14T04:22:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:22:37.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:37 smithi039 ceph-mon[106470]: pgmap v1339: 105 pgs: 105 active+clean; 243 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 7.8 MiB/s wr, 64 op/s 2024-09-14T04:22:37.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:22:37.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:37 smithi039 ceph-mon[111235]: pgmap v1339: 105 pgs: 105 active+clean; 243 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 7.2 MiB/s rd, 7.8 MiB/s wr, 64 op/s 2024-09-14T04:22:37.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:22:39.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:22:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:22:38] "GET /metrics HTTP/1.1" 200 36681 "" "Prometheus/2.51.0" 2024-09-14T04:22:39.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:39 smithi129 ceph-mon[101316]: pgmap v1340: 105 pgs: 105 active+clean; 247 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 7.6 MiB/s wr, 64 op/s 2024-09-14T04:22:39.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:39 smithi039 ceph-mon[106470]: pgmap v1340: 105 pgs: 105 active+clean; 247 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 7.6 MiB/s wr, 64 op/s 2024-09-14T04:22:39.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:39 smithi039 ceph-mon[111235]: pgmap v1340: 105 pgs: 105 active+clean; 247 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 7.6 MiB/s wr, 64 op/s 2024-09-14T04:22:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:41 smithi129 ceph-mon[101316]: pgmap v1341: 105 pgs: 105 active+clean; 247 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 6.8 MiB/s wr, 59 op/s 2024-09-14T04:22:41.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:41 smithi039 ceph-mon[106470]: pgmap v1341: 105 pgs: 105 active+clean; 247 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 6.8 MiB/s wr, 59 op/s 2024-09-14T04:22:41.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:41 smithi039 ceph-mon[111235]: pgmap v1341: 105 pgs: 105 active+clean; 247 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 6.8 MiB/s wr, 59 op/s 2024-09-14T04:22:43.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:43 smithi129 ceph-mon[101316]: osdmap e980: 8 total, 8 up, 8 in 2024-09-14T04:22:43.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:43 smithi129 ceph-mon[101316]: pgmap v1343: 105 pgs: 105 active+clean; 275 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 4.0 MiB/s wr, 39 op/s 2024-09-14T04:22:43.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:43 smithi039 ceph-mon[106470]: osdmap e980: 8 total, 8 up, 8 in 2024-09-14T04:22:43.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:43 smithi039 ceph-mon[106470]: pgmap v1343: 105 pgs: 105 active+clean; 275 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 4.0 MiB/s wr, 39 op/s 2024-09-14T04:22:43.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:43 smithi039 ceph-mon[111235]: osdmap e980: 8 total, 8 up, 8 in 2024-09-14T04:22:43.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:43 smithi039 ceph-mon[111235]: pgmap v1343: 105 pgs: 105 active+clean; 275 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 7.8 MiB/s rd, 4.0 MiB/s wr, 39 op/s 2024-09-14T04:22:46.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:46 smithi129 ceph-mon[101316]: pgmap v1344: 105 pgs: 105 active+clean; 283 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 4.0 MiB/s wr, 49 op/s 2024-09-14T04:22:46.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:46 smithi039 ceph-mon[106470]: pgmap v1344: 105 pgs: 105 active+clean; 283 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 4.0 MiB/s wr, 49 op/s 2024-09-14T04:22:46.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:46 smithi039 ceph-mon[111235]: pgmap v1344: 105 pgs: 105 active+clean; 283 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 6.6 MiB/s rd, 4.0 MiB/s wr, 49 op/s 2024-09-14T04:22:48.533 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/5.ShrinkToOne (13747 ms) 2024-09-14T04:22:48.533 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/5.ShrinkToOneAfterSnapshot 2024-09-14T04:22:48.533 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:22:48.648 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:48 smithi039 ceph-mon[106470]: pgmap v1345: 105 pgs: 105 active+clean; 296 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.0 MiB/s rd, 5.3 MiB/s wr, 49 op/s 2024-09-14T04:22:48.648 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:48 smithi039 ceph-mon[111235]: pgmap v1345: 105 pgs: 105 active+clean; 296 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.0 MiB/s rd, 5.3 MiB/s wr, 49 op/s 2024-09-14T04:22:48.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:48 smithi129 ceph-mon[101316]: pgmap v1345: 105 pgs: 105 active+clean; 296 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.0 MiB/s rd, 5.3 MiB/s wr, 49 op/s 2024-09-14T04:22:49.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:22:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:22:48] "GET /metrics HTTP/1.1" 200 36681 "" "Prometheus/2.51.0" 2024-09-14T04:22:50.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:50 smithi129 ceph-mon[101316]: pgmap v1346: 105 pgs: 105 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 6.5 MiB/s wr, 45 op/s 2024-09-14T04:22:50.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:50 smithi039 ceph-mon[106470]: pgmap v1346: 105 pgs: 105 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 6.5 MiB/s wr, 45 op/s 2024-09-14T04:22:50.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:50 smithi039 ceph-mon[111235]: pgmap v1346: 105 pgs: 105 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 6.5 MiB/s wr, 45 op/s 2024-09-14T04:22:52.573 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:52 smithi129 ceph-mon[101316]: pgmap v1347: 105 pgs: 105 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.5 MiB/s wr, 56 op/s 2024-09-14T04:22:52.573 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:22:52.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:52 smithi039 ceph-mon[111235]: pgmap v1347: 105 pgs: 105 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.5 MiB/s wr, 56 op/s 2024-09-14T04:22:52.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:22:52.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:52 smithi039 ceph-mon[106470]: pgmap v1347: 105 pgs: 105 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.1 MiB/s rd, 6.5 MiB/s wr, 56 op/s 2024-09-14T04:22:52.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:22:54.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:54 smithi129 ceph-mon[101316]: pgmap v1348: 105 pgs: 105 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 6.0 MiB/s wr, 56 op/s 2024-09-14T04:22:54.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:54 smithi039 ceph-mon[106470]: pgmap v1348: 105 pgs: 105 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 6.0 MiB/s wr, 56 op/s 2024-09-14T04:22:54.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:54 smithi039 ceph-mon[111235]: pgmap v1348: 105 pgs: 105 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 6.0 MiB/s wr, 56 op/s 2024-09-14T04:22:56.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:56 smithi129 ceph-mon[101316]: pgmap v1349: 105 pgs: 105 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.1 MiB/s wr, 33 op/s 2024-09-14T04:22:56.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:56 smithi129 ceph-mon[101316]: osdmap e981: 8 total, 8 up, 8 in 2024-09-14T04:22:56.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:56 smithi039 ceph-mon[106470]: pgmap v1349: 105 pgs: 105 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.1 MiB/s wr, 33 op/s 2024-09-14T04:22:56.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:56 smithi039 ceph-mon[106470]: osdmap e981: 8 total, 8 up, 8 in 2024-09-14T04:22:56.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:56 smithi039 ceph-mon[111235]: pgmap v1349: 105 pgs: 105 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.1 MiB/s wr, 33 op/s 2024-09-14T04:22:56.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:56 smithi039 ceph-mon[111235]: osdmap e981: 8 total, 8 up, 8 in 2024-09-14T04:22:58.649 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:22:58 smithi039 ceph-mon[106470]: pgmap v1351: 105 pgs: 105 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.6 MiB/s wr, 20 op/s 2024-09-14T04:22:58.649 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:22:58 smithi039 ceph-mon[111235]: pgmap v1351: 105 pgs: 105 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.6 MiB/s wr, 20 op/s 2024-09-14T04:22:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:22:58 smithi129 ceph-mon[101316]: pgmap v1351: 105 pgs: 105 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.6 MiB/s wr, 20 op/s 2024-09-14T04:22:59.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:22:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:22:58] "GET /metrics HTTP/1.1" 200 36679 "" "Prometheus/2.51.0" 2024-09-14T04:23:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:00 smithi129 ceph-mon[101316]: pgmap v1352: 105 pgs: 105 active+clean; 349 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 101 KiB/s rd, 4.1 MiB/s wr, 59 op/s 2024-09-14T04:23:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:00 smithi129 ceph-mon[101316]: osdmap e982: 8 total, 8 up, 8 in 2024-09-14T04:23:00.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:00 smithi039 ceph-mon[106470]: pgmap v1352: 105 pgs: 105 active+clean; 349 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 101 KiB/s rd, 4.1 MiB/s wr, 59 op/s 2024-09-14T04:23:00.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:00 smithi039 ceph-mon[106470]: osdmap e982: 8 total, 8 up, 8 in 2024-09-14T04:23:00.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:00 smithi039 ceph-mon[111235]: pgmap v1352: 105 pgs: 105 active+clean; 349 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 101 KiB/s rd, 4.1 MiB/s wr, 59 op/s 2024-09-14T04:23:00.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:00 smithi039 ceph-mon[111235]: osdmap e982: 8 total, 8 up, 8 in 2024-09-14T04:23:01.702 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/5.ShrinkToOneAfterSnapshot (13169 ms) 2024-09-14T04:23:01.703 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/5.MinOverlap 2024-09-14T04:23:01.703 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:23:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:01 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:23:01.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:01 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:23:01.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:01 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:23:02.419 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:02 smithi039 ceph-mon[106470]: pgmap v1354: 105 pgs: 105 active+clean; 362 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 111 KiB/s rd, 6.8 MiB/s wr, 53 op/s 2024-09-14T04:23:02.419 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:02 smithi039 ceph-mon[111235]: pgmap v1354: 105 pgs: 105 active+clean; 362 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 111 KiB/s rd, 6.8 MiB/s wr, 53 op/s 2024-09-14T04:23:02.737 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:02 smithi129 ceph-mon[101316]: pgmap v1354: 105 pgs: 105 active+clean; 362 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 111 KiB/s rd, 6.8 MiB/s wr, 53 op/s 2024-09-14T04:23:04.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:04 smithi129 ceph-mon[101316]: pgmap v1355: 105 pgs: 105 active+clean; 418 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.3 MiB/s rd, 14 MiB/s wr, 86 op/s 2024-09-14T04:23:04.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:04 smithi039 ceph-mon[106470]: pgmap v1355: 105 pgs: 105 active+clean; 418 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.3 MiB/s rd, 14 MiB/s wr, 86 op/s 2024-09-14T04:23:04.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:04 smithi039 ceph-mon[111235]: pgmap v1355: 105 pgs: 105 active+clean; 418 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 7.3 MiB/s rd, 14 MiB/s wr, 86 op/s 2024-09-14T04:23:05.692 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:05 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:23:05.692 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:05 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:23:05.692 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:05 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:23:05.692 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:05 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:23:05.692 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:05 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:23:05.693 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:05 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:23:05.693 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:05 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:23:05.693 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:05 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:23:05.693 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:05 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:23:05.693 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:05 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:23:05.693 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:05 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:23:05.693 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:05 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:23:05.693 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:05 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:23:05.694 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:05 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:23:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:05 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:23:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:05 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:23:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:05 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:23:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:05 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:23:05.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:05 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:23:05.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:05 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:23:05.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:05 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:23:07.276 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:07 smithi039 ceph-mon[106470]: pgmap v1356: 105 pgs: 105 active+clean; 434 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 13 MiB/s wr, 81 op/s 2024-09-14T04:23:07.277 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:07 smithi039 ceph-mon[111235]: pgmap v1356: 105 pgs: 105 active+clean; 434 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 13 MiB/s wr, 81 op/s 2024-09-14T04:23:07.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:07 smithi129 ceph-mon[101316]: pgmap v1356: 105 pgs: 105 active+clean; 434 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.9 MiB/s rd, 13 MiB/s wr, 81 op/s 2024-09-14T04:23:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:08 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:23:08.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:08 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:23:08.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:08 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:23:09.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:23:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:23:08] "GET /metrics HTTP/1.1" 200 36682 "" "Prometheus/2.51.0" 2024-09-14T04:23:09.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:09 smithi129 ceph-mon[101316]: pgmap v1357: 105 pgs: 105 active+clean; 434 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 13 MiB/s wr, 83 op/s 2024-09-14T04:23:09.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:09 smithi039 ceph-mon[106470]: pgmap v1357: 105 pgs: 105 active+clean; 434 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 13 MiB/s wr, 83 op/s 2024-09-14T04:23:09.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:09 smithi039 ceph-mon[111235]: pgmap v1357: 105 pgs: 105 active+clean; 434 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 13 MiB/s wr, 83 op/s 2024-09-14T04:23:10.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:10 smithi129 ceph-mon[101316]: osdmap e983: 8 total, 8 up, 8 in 2024-09-14T04:23:10.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:10 smithi039 ceph-mon[106470]: osdmap e983: 8 total, 8 up, 8 in 2024-09-14T04:23:10.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:10 smithi039 ceph-mon[111235]: osdmap e983: 8 total, 8 up, 8 in 2024-09-14T04:23:11.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:11 smithi129 ceph-mon[101316]: pgmap v1359: 105 pgs: 105 active+clean; 434 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 8.6 MiB/s wr, 42 op/s 2024-09-14T04:23:11.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:11 smithi039 ceph-mon[106470]: pgmap v1359: 105 pgs: 105 active+clean; 434 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 8.6 MiB/s wr, 42 op/s 2024-09-14T04:23:11.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:11 smithi039 ceph-mon[111235]: pgmap v1359: 105 pgs: 105 active+clean; 434 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 8.6 MiB/s wr, 42 op/s 2024-09-14T04:23:13.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:13 smithi129 ceph-mon[101316]: pgmap v1360: 105 pgs: 105 active+clean; 442 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 8.0 MiB/s wr, 50 op/s 2024-09-14T04:23:13.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:13 smithi039 ceph-mon[106470]: pgmap v1360: 105 pgs: 105 active+clean; 442 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 8.0 MiB/s wr, 50 op/s 2024-09-14T04:23:13.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:13 smithi039 ceph-mon[111235]: pgmap v1360: 105 pgs: 105 active+clean; 442 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 5.8 MiB/s rd, 8.0 MiB/s wr, 50 op/s 2024-09-14T04:23:14.739 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/5.MinOverlap (13037 ms) 2024-09-14T04:23:14.739 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/5.ZeroOverlap 2024-09-14T04:23:14.740 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:23:15.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:15 smithi129 ceph-mon[101316]: pgmap v1361: 105 pgs: 105 active+clean; 487 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 859 KiB/s rd, 6.9 MiB/s wr, 58 op/s 2024-09-14T04:23:15.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:15 smithi039 ceph-mon[106470]: pgmap v1361: 105 pgs: 105 active+clean; 487 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 859 KiB/s rd, 6.9 MiB/s wr, 58 op/s 2024-09-14T04:23:15.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:15 smithi039 ceph-mon[111235]: pgmap v1361: 105 pgs: 105 active+clean; 487 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 859 KiB/s rd, 6.9 MiB/s wr, 58 op/s 2024-09-14T04:23:17.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:17 smithi129 ceph-mon[101316]: pgmap v1362: 105 pgs: 105 active+clean; 487 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 5.3 MiB/s wr, 47 op/s 2024-09-14T04:23:17.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:17 smithi039 ceph-mon[111235]: pgmap v1362: 105 pgs: 105 active+clean; 487 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 5.3 MiB/s wr, 47 op/s 2024-09-14T04:23:17.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:17 smithi039 ceph-mon[106470]: pgmap v1362: 105 pgs: 105 active+clean; 487 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 5.3 MiB/s wr, 47 op/s 2024-09-14T04:23:19.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:23:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:23:18] "GET /metrics HTTP/1.1" 200 36682 "" "Prometheus/2.51.0" 2024-09-14T04:23:19.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:19 smithi129 ceph-mon[101316]: pgmap v1363: 105 pgs: 105 active+clean; 495 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 6.5 MiB/s wr, 62 op/s 2024-09-14T04:23:19.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:19 smithi039 ceph-mon[106470]: pgmap v1363: 105 pgs: 105 active+clean; 495 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 6.5 MiB/s wr, 62 op/s 2024-09-14T04:23:19.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:19 smithi039 ceph-mon[111235]: pgmap v1363: 105 pgs: 105 active+clean; 495 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 6.5 MiB/s wr, 62 op/s 2024-09-14T04:23:21.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:21 smithi129 ceph-mon[101316]: pgmap v1364: 105 pgs: 105 active+clean; 495 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 6.5 MiB/s wr, 63 op/s 2024-09-14T04:23:21.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:21 smithi039 ceph-mon[106470]: pgmap v1364: 105 pgs: 105 active+clean; 495 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 6.5 MiB/s wr, 63 op/s 2024-09-14T04:23:21.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:21 smithi039 ceph-mon[111235]: pgmap v1364: 105 pgs: 105 active+clean; 495 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 6.5 MiB/s wr, 63 op/s 2024-09-14T04:23:22.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:22 smithi129 ceph-mon[101316]: pgmap v1365: 105 pgs: 105 active+clean; 495 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 5.4 MiB/s wr, 53 op/s 2024-09-14T04:23:22.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:22 smithi129 ceph-mon[101316]: osdmap e984: 8 total, 8 up, 8 in 2024-09-14T04:23:22.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:22 smithi039 ceph-mon[106470]: pgmap v1365: 105 pgs: 105 active+clean; 495 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 5.4 MiB/s wr, 53 op/s 2024-09-14T04:23:22.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:22 smithi039 ceph-mon[106470]: osdmap e984: 8 total, 8 up, 8 in 2024-09-14T04:23:22.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:22 smithi039 ceph-mon[111235]: pgmap v1365: 105 pgs: 105 active+clean; 495 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 5.4 MiB/s wr, 53 op/s 2024-09-14T04:23:22.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:22 smithi039 ceph-mon[111235]: osdmap e984: 8 total, 8 up, 8 in 2024-09-14T04:23:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:23:23.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:23:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:23:24.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:24 smithi039 ceph-mon[106470]: pgmap v1367: 105 pgs: 105 active+clean; 528 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 4.5 MiB/s wr, 62 op/s 2024-09-14T04:23:24.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:24 smithi039 ceph-mon[111235]: pgmap v1367: 105 pgs: 105 active+clean; 528 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 4.5 MiB/s wr, 62 op/s 2024-09-14T04:23:24.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:24 smithi129 ceph-mon[101316]: pgmap v1367: 105 pgs: 105 active+clean; 528 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.6 MiB/s rd, 4.5 MiB/s wr, 62 op/s 2024-09-14T04:23:26.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:26 smithi129 ceph-mon[101316]: pgmap v1368: 105 pgs: 105 active+clean; 536 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 5.3 MiB/s wr, 62 op/s 2024-09-14T04:23:26.784 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/5.ZeroOverlap (12044 ms) 2024-09-14T04:23:26.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:26 smithi039 ceph-mon[106470]: pgmap v1368: 105 pgs: 105 active+clean; 536 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 5.3 MiB/s wr, 62 op/s 2024-09-14T04:23:26.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:26 smithi039 ceph-mon[111235]: pgmap v1368: 105 pgs: 105 active+clean; 536 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 5.3 MiB/s wr, 62 op/s 2024-09-14T04:23:27.315 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 7 tests from EncryptedFlattenTest/5 (100437 ms total) 2024-09-14T04:23:27.315 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:23:27.315 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 7 tests from EncryptedFlattenTest/6, where TypeParam = LUKS2UnderLUKS2 2024-09-14T04:23:27.345 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/6.Simple 2024-09-14T04:23:28.648 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:28 smithi039 ceph-mon[106470]: pgmap v1369: 105 pgs: 105 active+clean; 536 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 63 KiB/s rd, 4.1 MiB/s wr, 48 op/s 2024-09-14T04:23:28.648 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:28 smithi039 ceph-mon[106470]: osdmap e985: 8 total, 8 up, 8 in 2024-09-14T04:23:28.648 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:28 smithi039 ceph-mon[111235]: pgmap v1369: 105 pgs: 105 active+clean; 536 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 63 KiB/s rd, 4.1 MiB/s wr, 48 op/s 2024-09-14T04:23:28.648 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:28 smithi039 ceph-mon[111235]: osdmap e985: 8 total, 8 up, 8 in 2024-09-14T04:23:28.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:28 smithi129 ceph-mon[101316]: pgmap v1369: 105 pgs: 105 active+clean; 536 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 63 KiB/s rd, 4.1 MiB/s wr, 48 op/s 2024-09-14T04:23:28.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:28 smithi129 ceph-mon[101316]: osdmap e985: 8 total, 8 up, 8 in 2024-09-14T04:23:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:23:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:23:28] "GET /metrics HTTP/1.1" 200 36681 "" "Prometheus/2.51.0" 2024-09-14T04:23:29.338 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:23:29.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:29 smithi129 ceph-mon[101316]: osdmap e986: 8 total, 8 up, 8 in 2024-09-14T04:23:29.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:29 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/327618468' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-45","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:23:29.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:29 smithi039 ceph-mon[106470]: osdmap e986: 8 total, 8 up, 8 in 2024-09-14T04:23:29.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:29 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/327618468' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-45","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:23:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:29 smithi039 ceph-mon[111235]: osdmap e986: 8 total, 8 up, 8 in 2024-09-14T04:23:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:29 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/327618468' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-45","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:23:30.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:30 smithi129 ceph-mon[101316]: pgmap v1372: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:23:30.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:30 smithi129 ceph-mon[101316]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:23:30.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:30 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/327618468' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-45","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:23:30.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:30 smithi129 ceph-mon[101316]: osdmap e987: 8 total, 8 up, 8 in 2024-09-14T04:23:30.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:30 smithi039 ceph-mon[106470]: pgmap v1372: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:23:30.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:30 smithi039 ceph-mon[106470]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:23:30.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:30 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/327618468' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-45","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:23:30.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:30 smithi039 ceph-mon[106470]: osdmap e987: 8 total, 8 up, 8 in 2024-09-14T04:23:30.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:30 smithi039 ceph-mon[111235]: pgmap v1372: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:23:30.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:30 smithi039 ceph-mon[111235]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:23:30.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:30 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/327618468' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-45","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:23:30.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:30 smithi039 ceph-mon[111235]: osdmap e987: 8 total, 8 up, 8 in 2024-09-14T04:23:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:31 smithi129 ceph-mon[101316]: osdmap e988: 8 total, 8 up, 8 in 2024-09-14T04:23:31.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:31 smithi039 ceph-mon[106470]: osdmap e988: 8 total, 8 up, 8 in 2024-09-14T04:23:31.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:31 smithi039 ceph-mon[111235]: osdmap e988: 8 total, 8 up, 8 in 2024-09-14T04:23:32.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:32 smithi129 ceph-mon[101316]: pgmap v1375: 105 pgs: 5 creating+peering, 22 unknown, 78 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:23:32.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:32 smithi129 ceph-mon[101316]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:23:32.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:32 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T04:23:32.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:32 smithi129 ceph-mon[101316]: osdmap e989: 8 total, 8 up, 8 in 2024-09-14T04:23:32.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:32 smithi039 ceph-mon[106470]: pgmap v1375: 105 pgs: 5 creating+peering, 22 unknown, 78 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:23:32.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:32 smithi039 ceph-mon[106470]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:23:32.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:32 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T04:23:32.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:32 smithi039 ceph-mon[106470]: osdmap e989: 8 total, 8 up, 8 in 2024-09-14T04:23:32.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:32 smithi039 ceph-mon[111235]: pgmap v1375: 105 pgs: 5 creating+peering, 22 unknown, 78 active+clean; 582 KiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:23:32.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:32 smithi039 ceph-mon[111235]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:23:32.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:32 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T04:23:32.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:32 smithi039 ceph-mon[111235]: osdmap e989: 8 total, 8 up, 8 in 2024-09-14T04:23:34.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:34 smithi129 ceph-mon[101316]: pgmap v1377: 105 pgs: 5 creating+peering, 100 active+clean; 582 KiB data, 699 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.0 KiB/s wr, 3 op/s 2024-09-14T04:23:34.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:34 smithi039 ceph-mon[106470]: pgmap v1377: 105 pgs: 5 creating+peering, 100 active+clean; 582 KiB data, 699 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.0 KiB/s wr, 3 op/s 2024-09-14T04:23:34.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:34 smithi039 ceph-mon[111235]: pgmap v1377: 105 pgs: 5 creating+peering, 100 active+clean; 582 KiB data, 699 MiB used, 715 GiB / 715 GiB avail; 1.0 KiB/s rd, 1.0 KiB/s wr, 3 op/s 2024-09-14T04:23:37.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:36 smithi129 ceph-mon[101316]: pgmap v1378: 105 pgs: 105 active+clean; 582 KiB data, 699 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.3 KiB/s wr, 8 op/s 2024-09-14T04:23:37.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:36 smithi129 ceph-mon[101316]: osdmap e990: 8 total, 8 up, 8 in 2024-09-14T04:23:37.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:36 smithi039 ceph-mon[106470]: pgmap v1378: 105 pgs: 105 active+clean; 582 KiB data, 699 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.3 KiB/s wr, 8 op/s 2024-09-14T04:23:37.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:36 smithi039 ceph-mon[106470]: osdmap e990: 8 total, 8 up, 8 in 2024-09-14T04:23:37.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:36 smithi039 ceph-mon[111235]: pgmap v1378: 105 pgs: 105 active+clean; 582 KiB data, 699 MiB used, 715 GiB / 715 GiB avail; 5.0 KiB/s rd, 1.3 KiB/s wr, 8 op/s 2024-09-14T04:23:37.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:36 smithi039 ceph-mon[111235]: osdmap e990: 8 total, 8 up, 8 in 2024-09-14T04:23:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:23:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:23:38.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:23:39.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:38 smithi129 ceph-mon[101316]: pgmap v1380: 105 pgs: 105 active+clean; 582 KiB data, 699 MiB used, 715 GiB / 715 GiB avail; 4.3 KiB/s rd, 1.2 KiB/s wr, 7 op/s 2024-09-14T04:23:39.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:38 smithi039 ceph-mon[106470]: pgmap v1380: 105 pgs: 105 active+clean; 582 KiB data, 699 MiB used, 715 GiB / 715 GiB avail; 4.3 KiB/s rd, 1.2 KiB/s wr, 7 op/s 2024-09-14T04:23:39.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:23:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:23:38] "GET /metrics HTTP/1.1" 200 36646 "" "Prometheus/2.51.0" 2024-09-14T04:23:39.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:38 smithi039 ceph-mon[111235]: pgmap v1380: 105 pgs: 105 active+clean; 582 KiB data, 699 MiB used, 715 GiB / 715 GiB avail; 4.3 KiB/s rd, 1.2 KiB/s wr, 7 op/s 2024-09-14T04:23:41.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:40 smithi129 ceph-mon[101316]: pgmap v1381: 105 pgs: 105 active+clean; 582 KiB data, 703 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 1023 B/s wr, 6 op/s 2024-09-14T04:23:41.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:40 smithi129 ceph-mon[101316]: osdmap e991: 8 total, 8 up, 8 in 2024-09-14T04:23:41.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:40 smithi039 ceph-mon[106470]: pgmap v1381: 105 pgs: 105 active+clean; 582 KiB data, 703 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 1023 B/s wr, 6 op/s 2024-09-14T04:23:41.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:40 smithi039 ceph-mon[106470]: osdmap e991: 8 total, 8 up, 8 in 2024-09-14T04:23:41.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:40 smithi039 ceph-mon[111235]: pgmap v1381: 105 pgs: 105 active+clean; 582 KiB data, 703 MiB used, 715 GiB / 715 GiB avail; 3.7 KiB/s rd, 1023 B/s wr, 6 op/s 2024-09-14T04:23:41.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:40 smithi039 ceph-mon[111235]: osdmap e991: 8 total, 8 up, 8 in 2024-09-14T04:23:43.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:42 smithi129 ceph-mon[101316]: pgmap v1383: 105 pgs: 105 active+clean; 13 MiB data, 711 MiB used, 715 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.5 MiB/s wr, 6 op/s 2024-09-14T04:23:43.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:42 smithi039 ceph-mon[106470]: pgmap v1383: 105 pgs: 105 active+clean; 13 MiB data, 711 MiB used, 715 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.5 MiB/s wr, 6 op/s 2024-09-14T04:23:43.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:42 smithi039 ceph-mon[111235]: pgmap v1383: 105 pgs: 105 active+clean; 13 MiB data, 711 MiB used, 715 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.5 MiB/s wr, 6 op/s 2024-09-14T04:23:45.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:44 smithi129 ceph-mon[101316]: pgmap v1384: 105 pgs: 105 active+clean; 42 MiB data, 802 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 5.1 MiB/s wr, 24 op/s 2024-09-14T04:23:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:44 smithi039 ceph-mon[106470]: pgmap v1384: 105 pgs: 105 active+clean; 42 MiB data, 802 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 5.1 MiB/s wr, 24 op/s 2024-09-14T04:23:45.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:44 smithi039 ceph-mon[111235]: pgmap v1384: 105 pgs: 105 active+clean; 42 MiB data, 802 MiB used, 714 GiB / 715 GiB avail; 16 KiB/s rd, 5.1 MiB/s wr, 24 op/s 2024-09-14T04:23:47.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:46 smithi129 ceph-mon[101316]: pgmap v1385: 105 pgs: 105 active+clean; 42 MiB data, 826 MiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 4.3 MiB/s wr, 36 op/s 2024-09-14T04:23:47.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:46 smithi039 ceph-mon[106470]: pgmap v1385: 105 pgs: 105 active+clean; 42 MiB data, 826 MiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 4.3 MiB/s wr, 36 op/s 2024-09-14T04:23:47.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:46 smithi039 ceph-mon[111235]: pgmap v1385: 105 pgs: 105 active+clean; 42 MiB data, 826 MiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 4.3 MiB/s wr, 36 op/s 2024-09-14T04:23:49.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:48 smithi129 ceph-mon[101316]: pgmap v1386: 105 pgs: 105 active+clean; 42 MiB data, 826 MiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-09-14T04:23:49.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:48 smithi039 ceph-mon[106470]: pgmap v1386: 105 pgs: 105 active+clean; 42 MiB data, 826 MiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-09-14T04:23:49.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:23:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:23:48] "GET /metrics HTTP/1.1" 200 36646 "" "Prometheus/2.51.0" 2024-09-14T04:23:49.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:48 smithi039 ceph-mon[111235]: pgmap v1386: 105 pgs: 105 active+clean; 42 MiB data, 826 MiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-09-14T04:23:51.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:50 smithi129 ceph-mon[101316]: pgmap v1387: 105 pgs: 105 active+clean; 54 MiB data, 858 MiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 5.3 MiB/s wr, 42 op/s 2024-09-14T04:23:51.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:50 smithi039 ceph-mon[106470]: pgmap v1387: 105 pgs: 105 active+clean; 54 MiB data, 858 MiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 5.3 MiB/s wr, 42 op/s 2024-09-14T04:23:51.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:50 smithi039 ceph-mon[111235]: pgmap v1387: 105 pgs: 105 active+clean; 54 MiB data, 858 MiB used, 714 GiB / 715 GiB avail; 59 KiB/s rd, 5.3 MiB/s wr, 42 op/s 2024-09-14T04:23:51.229 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/6.Simple (23884 ms) 2024-09-14T04:23:51.230 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/6.Grow 2024-09-14T04:23:51.230 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:23:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:52 smithi129 ceph-mon[101316]: pgmap v1388: 105 pgs: 105 active+clean; 62 MiB data, 890 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 4.2 MiB/s wr, 39 op/s 2024-09-14T04:23:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:23:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:52 smithi039 ceph-mon[106470]: pgmap v1388: 105 pgs: 105 active+clean; 62 MiB data, 890 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 4.2 MiB/s wr, 39 op/s 2024-09-14T04:23:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:23:53.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:52 smithi039 ceph-mon[111235]: pgmap v1388: 105 pgs: 105 active+clean; 62 MiB data, 890 MiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 4.2 MiB/s wr, 39 op/s 2024-09-14T04:23:53.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:23:55.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:54 smithi129 ceph-mon[101316]: pgmap v1389: 105 pgs: 105 active+clean; 83 MiB data, 962 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 6.8 MiB/s wr, 54 op/s 2024-09-14T04:23:55.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:54 smithi039 ceph-mon[106470]: pgmap v1389: 105 pgs: 105 active+clean; 83 MiB data, 962 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 6.8 MiB/s wr, 54 op/s 2024-09-14T04:23:55.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:54 smithi039 ceph-mon[111235]: pgmap v1389: 105 pgs: 105 active+clean; 83 MiB data, 962 MiB used, 714 GiB / 715 GiB avail; 9.1 MiB/s rd, 6.8 MiB/s wr, 54 op/s 2024-09-14T04:23:57.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:56 smithi039 ceph-mon[106470]: pgmap v1390: 105 pgs: 105 active+clean; 83 MiB data, 975 MiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.8 MiB/s wr, 42 op/s 2024-09-14T04:23:57.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:56 smithi039 ceph-mon[111235]: pgmap v1390: 105 pgs: 105 active+clean; 83 MiB data, 975 MiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.8 MiB/s wr, 42 op/s 2024-09-14T04:23:57.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:56 smithi129 ceph-mon[101316]: pgmap v1390: 105 pgs: 105 active+clean; 83 MiB data, 975 MiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.8 MiB/s wr, 42 op/s 2024-09-14T04:23:58.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:57 smithi039 ceph-mon[106470]: osdmap e992: 8 total, 8 up, 8 in 2024-09-14T04:23:58.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:57 smithi039 ceph-mon[111235]: osdmap e992: 8 total, 8 up, 8 in 2024-09-14T04:23:58.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:57 smithi129 ceph-mon[101316]: osdmap e992: 8 total, 8 up, 8 in 2024-09-14T04:23:59.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:23:58 smithi039 ceph-mon[106470]: pgmap v1392: 105 pgs: 105 active+clean; 83 MiB data, 975 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.7 MiB/s wr, 34 op/s 2024-09-14T04:23:59.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:23:58 smithi039 ceph-mon[111235]: pgmap v1392: 105 pgs: 105 active+clean; 83 MiB data, 975 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.7 MiB/s wr, 34 op/s 2024-09-14T04:23:59.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:23:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:23:58] "GET /metrics HTTP/1.1" 200 36689 "" "Prometheus/2.51.0" 2024-09-14T04:23:59.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:23:58 smithi129 ceph-mon[101316]: pgmap v1392: 105 pgs: 105 active+clean; 83 MiB data, 975 MiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 5.7 MiB/s wr, 34 op/s 2024-09-14T04:24:01.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:00 smithi129 ceph-mon[101316]: pgmap v1393: 105 pgs: 105 active+clean; 119 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.1 MiB/s wr, 61 op/s 2024-09-14T04:24:01.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:00 smithi039 ceph-mon[106470]: pgmap v1393: 105 pgs: 105 active+clean; 119 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.1 MiB/s wr, 61 op/s 2024-09-14T04:24:01.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:00 smithi039 ceph-mon[111235]: pgmap v1393: 105 pgs: 105 active+clean; 119 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 8.1 MiB/s wr, 61 op/s 2024-09-14T04:24:03.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:02 smithi129 ceph-mon[101316]: pgmap v1394: 105 pgs: 105 active+clean; 124 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 7.8 MiB/s wr, 57 op/s 2024-09-14T04:24:03.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:02 smithi039 ceph-mon[106470]: pgmap v1394: 105 pgs: 105 active+clean; 124 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 7.8 MiB/s wr, 57 op/s 2024-09-14T04:24:03.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:02 smithi039 ceph-mon[111235]: pgmap v1394: 105 pgs: 105 active+clean; 124 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 7.8 MiB/s wr, 57 op/s 2024-09-14T04:24:05.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:04 smithi129 ceph-mon[101316]: pgmap v1395: 105 pgs: 105 active+clean; 124 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 845 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-09-14T04:24:05.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:04 smithi039 ceph-mon[106470]: pgmap v1395: 105 pgs: 105 active+clean; 124 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 845 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-09-14T04:24:05.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:04 smithi039 ceph-mon[111235]: pgmap v1395: 105 pgs: 105 active+clean; 124 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 845 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-09-14T04:24:06.099 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:05 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:24:06.107 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:05 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:24:06.107 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:05 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:24:07.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:06 smithi129 ceph-mon[101316]: pgmap v1396: 105 pgs: 105 active+clean; 128 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.5 MiB/s wr, 36 op/s 2024-09-14T04:24:07.192 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:06 smithi039 ceph-mon[106470]: pgmap v1396: 105 pgs: 105 active+clean; 128 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.5 MiB/s wr, 36 op/s 2024-09-14T04:24:07.192 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:06 smithi039 ceph-mon[111235]: pgmap v1396: 105 pgs: 105 active+clean; 128 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.5 MiB/s wr, 36 op/s 2024-09-14T04:24:08.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:24:08.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:24:08.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:24:09.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:08 smithi039 ceph-mon[106470]: pgmap v1397: 105 pgs: 105 active+clean; 132 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 4.7 MiB/s wr, 34 op/s 2024-09-14T04:24:09.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:08 smithi039 ceph-mon[111235]: pgmap v1397: 105 pgs: 105 active+clean; 132 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 4.7 MiB/s wr, 34 op/s 2024-09-14T04:24:09.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:24:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:24:08] "GET /metrics HTTP/1.1" 200 36694 "" "Prometheus/2.51.0" 2024-09-14T04:24:09.214 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:08 smithi129 ceph-mon[101316]: pgmap v1397: 105 pgs: 105 active+clean; 132 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 4.7 MiB/s wr, 34 op/s 2024-09-14T04:24:11.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:10 smithi129 ceph-mon[101316]: pgmap v1398: 105 pgs: 105 active+clean; 140 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 52 KiB/s rd, 4.8 MiB/s wr, 38 op/s 2024-09-14T04:24:11.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:10 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:24:11.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:10 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:24:11.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:10 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:24:11.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:10 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:24:11.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:10 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:24:11.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:10 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:24:11.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:10 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:24:11.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:10 smithi039 ceph-mon[106470]: pgmap v1398: 105 pgs: 105 active+clean; 140 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 52 KiB/s rd, 4.8 MiB/s wr, 38 op/s 2024-09-14T04:24:11.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:10 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:24:11.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:10 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:24:11.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:10 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:24:11.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:10 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:24:11.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:10 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:24:11.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:10 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:24:11.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:10 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:24:11.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:10 smithi039 ceph-mon[111235]: pgmap v1398: 105 pgs: 105 active+clean; 140 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 52 KiB/s rd, 4.8 MiB/s wr, 38 op/s 2024-09-14T04:24:11.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:10 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:24:11.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:10 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:24:11.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:10 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:24:11.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:10 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:24:11.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:10 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:24:11.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:10 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:24:11.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:10 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:24:11.266 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/6.Grow (20037 ms) 2024-09-14T04:24:11.266 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/6.Shrink 2024-09-14T04:24:11.266 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:24:13.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:12 smithi129 ceph-mon[101316]: pgmap v1399: 105 pgs: 105 active+clean; 144 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 908 KiB/s rd, 2.4 MiB/s wr, 11 op/s 2024-09-14T04:24:13.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:12 smithi039 ceph-mon[106470]: pgmap v1399: 105 pgs: 105 active+clean; 144 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 908 KiB/s rd, 2.4 MiB/s wr, 11 op/s 2024-09-14T04:24:13.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:12 smithi039 ceph-mon[111235]: pgmap v1399: 105 pgs: 105 active+clean; 144 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 908 KiB/s rd, 2.4 MiB/s wr, 11 op/s 2024-09-14T04:24:15.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:14 smithi129 ceph-mon[101316]: pgmap v1400: 105 pgs: 105 active+clean; 165 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 4.8 MiB/s wr, 29 op/s 2024-09-14T04:24:15.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:14 smithi039 ceph-mon[106470]: pgmap v1400: 105 pgs: 105 active+clean; 165 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 4.8 MiB/s wr, 29 op/s 2024-09-14T04:24:15.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:14 smithi039 ceph-mon[111235]: pgmap v1400: 105 pgs: 105 active+clean; 165 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.4 MiB/s rd, 4.8 MiB/s wr, 29 op/s 2024-09-14T04:24:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:16 smithi129 ceph-mon[101316]: pgmap v1401: 105 pgs: 105 active+clean; 165 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.8 MiB/s wr, 30 op/s 2024-09-14T04:24:17.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:16 smithi039 ceph-mon[106470]: pgmap v1401: 105 pgs: 105 active+clean; 165 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.8 MiB/s wr, 30 op/s 2024-09-14T04:24:17.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:16 smithi039 ceph-mon[111235]: pgmap v1401: 105 pgs: 105 active+clean; 165 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.8 MiB/s wr, 30 op/s 2024-09-14T04:24:19.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:18 smithi129 ceph-mon[101316]: pgmap v1402: 105 pgs: 105 active+clean; 165 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.4 MiB/s wr, 29 op/s 2024-09-14T04:24:19.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:18 smithi039 ceph-mon[106470]: pgmap v1402: 105 pgs: 105 active+clean; 165 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.4 MiB/s wr, 29 op/s 2024-09-14T04:24:19.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:18 smithi039 ceph-mon[111235]: pgmap v1402: 105 pgs: 105 active+clean; 165 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.4 MiB/s wr, 29 op/s 2024-09-14T04:24:19.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:24:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:24:18] "GET /metrics HTTP/1.1" 200 36694 "" "Prometheus/2.51.0" 2024-09-14T04:24:21.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:20 smithi129 ceph-mon[101316]: pgmap v1403: 105 pgs: 105 active+clean; 177 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 5.1 MiB/s wr, 30 op/s 2024-09-14T04:24:21.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:20 smithi129 ceph-mon[101316]: osdmap e993: 8 total, 8 up, 8 in 2024-09-14T04:24:21.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:20 smithi039 ceph-mon[106470]: pgmap v1403: 105 pgs: 105 active+clean; 177 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 5.1 MiB/s wr, 30 op/s 2024-09-14T04:24:21.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:20 smithi039 ceph-mon[106470]: osdmap e993: 8 total, 8 up, 8 in 2024-09-14T04:24:21.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:20 smithi039 ceph-mon[111235]: pgmap v1403: 105 pgs: 105 active+clean; 177 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 5.1 MiB/s wr, 30 op/s 2024-09-14T04:24:21.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:20 smithi039 ceph-mon[111235]: osdmap e993: 8 total, 8 up, 8 in 2024-09-14T04:24:23.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:22 smithi129 ceph-mon[101316]: pgmap v1405: 105 pgs: 105 active+clean; 177 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 4.5 MiB/s wr, 24 op/s 2024-09-14T04:24:23.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:24:23.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:22 smithi039 ceph-mon[106470]: pgmap v1405: 105 pgs: 105 active+clean; 177 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 4.5 MiB/s wr, 24 op/s 2024-09-14T04:24:23.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:24:23.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:22 smithi039 ceph-mon[111235]: pgmap v1405: 105 pgs: 105 active+clean; 177 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 11 MiB/s rd, 4.5 MiB/s wr, 24 op/s 2024-09-14T04:24:23.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:24:25.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:24 smithi129 ceph-mon[101316]: pgmap v1406: 105 pgs: 105 active+clean; 206 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 435 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-09-14T04:24:25.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:24 smithi039 ceph-mon[106470]: pgmap v1406: 105 pgs: 105 active+clean; 206 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 435 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-09-14T04:24:25.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:24 smithi039 ceph-mon[111235]: pgmap v1406: 105 pgs: 105 active+clean; 206 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 435 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-09-14T04:24:27.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:26 smithi129 ceph-mon[101316]: pgmap v1407: 105 pgs: 105 active+clean; 206 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-09-14T04:24:27.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:26 smithi039 ceph-mon[106470]: pgmap v1407: 105 pgs: 105 active+clean; 206 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-09-14T04:24:27.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:26 smithi039 ceph-mon[111235]: pgmap v1407: 105 pgs: 105 active+clean; 206 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-09-14T04:24:29.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:28 smithi129 ceph-mon[101316]: pgmap v1408: 105 pgs: 105 active+clean; 206 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-09-14T04:24:29.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:28 smithi039 ceph-mon[106470]: pgmap v1408: 105 pgs: 105 active+clean; 206 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-09-14T04:24:29.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:28 smithi039 ceph-mon[111235]: pgmap v1408: 105 pgs: 105 active+clean; 206 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-09-14T04:24:29.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:24:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:24:28] "GET /metrics HTTP/1.1" 200 36687 "" "Prometheus/2.51.0" 2024-09-14T04:24:31.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:30 smithi129 ceph-mon[101316]: pgmap v1409: 105 pgs: 105 active+clean; 218 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 62 KiB/s rd, 4.1 MiB/s wr, 46 op/s 2024-09-14T04:24:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:30 smithi039 ceph-mon[106470]: pgmap v1409: 105 pgs: 105 active+clean; 218 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 62 KiB/s rd, 4.1 MiB/s wr, 46 op/s 2024-09-14T04:24:31.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:30 smithi039 ceph-mon[111235]: pgmap v1409: 105 pgs: 105 active+clean; 218 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 62 KiB/s rd, 4.1 MiB/s wr, 46 op/s 2024-09-14T04:24:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:32 smithi129 ceph-mon[101316]: pgmap v1410: 105 pgs: 105 active+clean; 218 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53 KiB/s rd, 3.5 MiB/s wr, 39 op/s 2024-09-14T04:24:33.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:32 smithi039 ceph-mon[106470]: pgmap v1410: 105 pgs: 105 active+clean; 218 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53 KiB/s rd, 3.5 MiB/s wr, 39 op/s 2024-09-14T04:24:33.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:32 smithi039 ceph-mon[111235]: pgmap v1410: 105 pgs: 105 active+clean; 218 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 53 KiB/s rd, 3.5 MiB/s wr, 39 op/s 2024-09-14T04:24:33.329 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/6.Shrink (22063 ms) 2024-09-14T04:24:33.329 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/6.ShrinkToOne 2024-09-14T04:24:33.329 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:24:35.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:34 smithi129 ceph-mon[101316]: pgmap v1411: 105 pgs: 105 active+clean; 239 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 6.2 MiB/s wr, 54 op/s 2024-09-14T04:24:35.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:34 smithi039 ceph-mon[106470]: pgmap v1411: 105 pgs: 105 active+clean; 239 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 6.2 MiB/s wr, 54 op/s 2024-09-14T04:24:35.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:34 smithi039 ceph-mon[111235]: pgmap v1411: 105 pgs: 105 active+clean; 239 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 6.2 MiB/s wr, 54 op/s 2024-09-14T04:24:37.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:36 smithi129 ceph-mon[101316]: pgmap v1412: 105 pgs: 105 active+clean; 247 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 4.8 MiB/s wr, 27 op/s 2024-09-14T04:24:37.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:36 smithi039 ceph-mon[106470]: pgmap v1412: 105 pgs: 105 active+clean; 247 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 4.8 MiB/s wr, 27 op/s 2024-09-14T04:24:37.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:36 smithi039 ceph-mon[111235]: pgmap v1412: 105 pgs: 105 active+clean; 247 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 4.8 MiB/s wr, 27 op/s 2024-09-14T04:24:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:24:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:24:38.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:24:39.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:38 smithi129 ceph-mon[101316]: pgmap v1413: 105 pgs: 105 active+clean; 247 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 4.8 MiB/s wr, 27 op/s 2024-09-14T04:24:39.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:38 smithi039 ceph-mon[106470]: pgmap v1413: 105 pgs: 105 active+clean; 247 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 4.8 MiB/s wr, 27 op/s 2024-09-14T04:24:39.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:24:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:24:38] "GET /metrics HTTP/1.1" 200 36693 "" "Prometheus/2.51.0" 2024-09-14T04:24:39.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:38 smithi039 ceph-mon[111235]: pgmap v1413: 105 pgs: 105 active+clean; 247 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 4.8 MiB/s wr, 27 op/s 2024-09-14T04:24:41.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:40 smithi129 ceph-mon[101316]: pgmap v1414: 105 pgs: 105 active+clean; 247 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.8 MiB/s wr, 31 op/s 2024-09-14T04:24:41.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:40 smithi039 ceph-mon[106470]: pgmap v1414: 105 pgs: 105 active+clean; 247 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.8 MiB/s wr, 31 op/s 2024-09-14T04:24:41.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:40 smithi039 ceph-mon[111235]: pgmap v1414: 105 pgs: 105 active+clean; 247 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.8 MiB/s rd, 4.8 MiB/s wr, 31 op/s 2024-09-14T04:24:43.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:42 smithi129 ceph-mon[101316]: pgmap v1415: 105 pgs: 105 active+clean; 247 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 3.8 MiB/s wr, 21 op/s 2024-09-14T04:24:43.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:42 smithi129 ceph-mon[101316]: osdmap e994: 8 total, 8 up, 8 in 2024-09-14T04:24:43.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:42 smithi039 ceph-mon[106470]: pgmap v1415: 105 pgs: 105 active+clean; 247 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 3.8 MiB/s wr, 21 op/s 2024-09-14T04:24:43.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:42 smithi039 ceph-mon[106470]: osdmap e994: 8 total, 8 up, 8 in 2024-09-14T04:24:43.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:42 smithi039 ceph-mon[111235]: pgmap v1415: 105 pgs: 105 active+clean; 247 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 3.8 MiB/s wr, 21 op/s 2024-09-14T04:24:43.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:42 smithi039 ceph-mon[111235]: osdmap e994: 8 total, 8 up, 8 in 2024-09-14T04:24:45.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:44 smithi129 ceph-mon[101316]: pgmap v1417: 105 pgs: 105 active+clean; 267 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 3.2 MiB/s wr, 25 op/s 2024-09-14T04:24:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:44 smithi039 ceph-mon[106470]: pgmap v1417: 105 pgs: 105 active+clean; 267 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 3.2 MiB/s wr, 25 op/s 2024-09-14T04:24:45.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:44 smithi039 ceph-mon[111235]: pgmap v1417: 105 pgs: 105 active+clean; 267 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5.3 MiB/s rd, 3.2 MiB/s wr, 25 op/s 2024-09-14T04:24:47.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:46 smithi129 ceph-mon[101316]: pgmap v1418: 105 pgs: 105 active+clean; 288 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 4.1 MiB/s wr, 39 op/s 2024-09-14T04:24:47.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:46 smithi039 ceph-mon[106470]: pgmap v1418: 105 pgs: 105 active+clean; 288 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 4.1 MiB/s wr, 39 op/s 2024-09-14T04:24:47.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:46 smithi039 ceph-mon[111235]: pgmap v1418: 105 pgs: 105 active+clean; 288 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 4.1 MiB/s wr, 39 op/s 2024-09-14T04:24:49.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:48 smithi039 ceph-mon[106470]: pgmap v1419: 105 pgs: 105 active+clean; 288 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.1 MiB/s wr, 39 op/s 2024-09-14T04:24:49.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:24:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:24:48] "GET /metrics HTTP/1.1" 200 36693 "" "Prometheus/2.51.0" 2024-09-14T04:24:49.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:48 smithi039 ceph-mon[111235]: pgmap v1419: 105 pgs: 105 active+clean; 288 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.1 MiB/s wr, 39 op/s 2024-09-14T04:24:49.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:48 smithi129 ceph-mon[101316]: pgmap v1419: 105 pgs: 105 active+clean; 288 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s rd, 4.1 MiB/s wr, 39 op/s 2024-09-14T04:24:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:50 smithi129 ceph-mon[101316]: pgmap v1420: 105 pgs: 105 active+clean; 288 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 4.1 MiB/s wr, 42 op/s 2024-09-14T04:24:51.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:50 smithi039 ceph-mon[106470]: pgmap v1420: 105 pgs: 105 active+clean; 288 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 4.1 MiB/s wr, 42 op/s 2024-09-14T04:24:51.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:50 smithi039 ceph-mon[111235]: pgmap v1420: 105 pgs: 105 active+clean; 288 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 4.1 MiB/s wr, 42 op/s 2024-09-14T04:24:53.074 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:52 smithi129 ceph-mon[101316]: pgmap v1421: 105 pgs: 105 active+clean; 288 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 4.1 MiB/s wr, 42 op/s 2024-09-14T04:24:53.074 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:24:53.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:52 smithi039 ceph-mon[106470]: pgmap v1421: 105 pgs: 105 active+clean; 288 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 4.1 MiB/s wr, 42 op/s 2024-09-14T04:24:53.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:24:53.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:52 smithi039 ceph-mon[111235]: pgmap v1421: 105 pgs: 105 active+clean; 288 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 4.1 MiB/s wr, 42 op/s 2024-09-14T04:24:53.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:24:55.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:54 smithi129 ceph-mon[101316]: pgmap v1422: 105 pgs: 105 active+clean; 304 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 54 KiB/s rd, 4.6 MiB/s wr, 40 op/s 2024-09-14T04:24:55.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:54 smithi039 ceph-mon[106470]: pgmap v1422: 105 pgs: 105 active+clean; 304 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 54 KiB/s rd, 4.6 MiB/s wr, 40 op/s 2024-09-14T04:24:55.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:54 smithi039 ceph-mon[111235]: pgmap v1422: 105 pgs: 105 active+clean; 304 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 54 KiB/s rd, 4.6 MiB/s wr, 40 op/s 2024-09-14T04:24:55.368 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/6.ShrinkToOne (22039 ms) 2024-09-14T04:24:55.369 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/6.ShrinkToOneAfterSnapshot 2024-09-14T04:24:55.369 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:24:57.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:56 smithi129 ceph-mon[101316]: pgmap v1423: 105 pgs: 105 active+clean; 304 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 69 KiB/s rd, 3.1 MiB/s wr, 31 op/s 2024-09-14T04:24:57.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:56 smithi039 ceph-mon[106470]: pgmap v1423: 105 pgs: 105 active+clean; 304 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 69 KiB/s rd, 3.1 MiB/s wr, 31 op/s 2024-09-14T04:24:57.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:56 smithi039 ceph-mon[111235]: pgmap v1423: 105 pgs: 105 active+clean; 304 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 69 KiB/s rd, 3.1 MiB/s wr, 31 op/s 2024-09-14T04:24:59.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:24:58 smithi039 ceph-mon[106470]: pgmap v1424: 105 pgs: 105 active+clean; 304 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 426 KiB/s rd, 1.7 MiB/s wr, 17 op/s 2024-09-14T04:24:59.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:24:58 smithi039 ceph-mon[111235]: pgmap v1424: 105 pgs: 105 active+clean; 304 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 426 KiB/s rd, 1.7 MiB/s wr, 17 op/s 2024-09-14T04:24:59.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:24:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:24:58] "GET /metrics HTTP/1.1" 200 36692 "" "Prometheus/2.51.0" 2024-09-14T04:24:59.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:24:58 smithi129 ceph-mon[101316]: pgmap v1424: 105 pgs: 105 active+clean; 304 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 426 KiB/s rd, 1.7 MiB/s wr, 17 op/s 2024-09-14T04:25:01.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:00 smithi129 ceph-mon[101316]: pgmap v1425: 105 pgs: 105 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.3 MiB/s wr, 23 op/s 2024-09-14T04:25:01.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:00 smithi039 ceph-mon[106470]: pgmap v1425: 105 pgs: 105 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.3 MiB/s wr, 23 op/s 2024-09-14T04:25:01.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:00 smithi039 ceph-mon[111235]: pgmap v1425: 105 pgs: 105 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 3.3 MiB/s wr, 23 op/s 2024-09-14T04:25:03.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:02 smithi129 ceph-mon[101316]: pgmap v1426: 105 pgs: 105 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.3 MiB/s wr, 17 op/s 2024-09-14T04:25:03.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:02 smithi039 ceph-mon[106470]: pgmap v1426: 105 pgs: 105 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.3 MiB/s wr, 17 op/s 2024-09-14T04:25:03.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:02 smithi039 ceph-mon[111235]: pgmap v1426: 105 pgs: 105 active+clean; 308 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 3.3 MiB/s wr, 17 op/s 2024-09-14T04:25:05.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:04 smithi129 ceph-mon[101316]: pgmap v1427: 105 pgs: 105 active+clean; 329 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 5.1 MiB/s wr, 26 op/s 2024-09-14T04:25:05.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:04 smithi129 ceph-mon[101316]: osdmap e995: 8 total, 8 up, 8 in 2024-09-14T04:25:05.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:04 smithi039 ceph-mon[106470]: pgmap v1427: 105 pgs: 105 active+clean; 329 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 5.1 MiB/s wr, 26 op/s 2024-09-14T04:25:05.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:04 smithi039 ceph-mon[106470]: osdmap e995: 8 total, 8 up, 8 in 2024-09-14T04:25:05.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:04 smithi039 ceph-mon[111235]: pgmap v1427: 105 pgs: 105 active+clean; 329 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 1.8 MiB/s rd, 5.1 MiB/s wr, 26 op/s 2024-09-14T04:25:05.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:04 smithi039 ceph-mon[111235]: osdmap e995: 8 total, 8 up, 8 in 2024-09-14T04:25:07.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:06 smithi129 ceph-mon[101316]: pgmap v1429: 105 pgs: 105 active+clean; 341 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 5.7 MiB/s wr, 21 op/s 2024-09-14T04:25:07.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:06 smithi039 ceph-mon[106470]: pgmap v1429: 105 pgs: 105 active+clean; 341 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 5.7 MiB/s wr, 21 op/s 2024-09-14T04:25:07.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:06 smithi039 ceph-mon[111235]: pgmap v1429: 105 pgs: 105 active+clean; 341 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 2.0 MiB/s rd, 5.7 MiB/s wr, 21 op/s 2024-09-14T04:25:08.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:25:08.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:25:08.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:25:09.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:08 smithi039 ceph-mon[106470]: pgmap v1430: 105 pgs: 105 active+clean; 345 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 5.7 MiB/s wr, 20 op/s 2024-09-14T04:25:09.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:08 smithi039 ceph-mon[111235]: pgmap v1430: 105 pgs: 105 active+clean; 345 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 5.7 MiB/s wr, 20 op/s 2024-09-14T04:25:09.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:25:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:25:08] "GET /metrics HTTP/1.1" 200 36689 "" "Prometheus/2.51.0" 2024-09-14T04:25:09.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:08 smithi129 ceph-mon[101316]: pgmap v1430: 105 pgs: 105 active+clean; 345 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 1.6 MiB/s rd, 5.7 MiB/s wr, 20 op/s 2024-09-14T04:25:11.211 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:10 smithi039 ceph-mon[106470]: pgmap v1431: 105 pgs: 105 active+clean; 349 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-09-14T04:25:11.211 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:10 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:25:11.211 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:10 smithi039 ceph-mon[111235]: pgmap v1431: 105 pgs: 105 active+clean; 349 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-09-14T04:25:11.211 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:10 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:25:11.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:10 smithi129 ceph-mon[101316]: pgmap v1431: 105 pgs: 105 active+clean; 349 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-09-14T04:25:11.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:10 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:25:13.195 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:12 smithi129 ceph-mon[101316]: pgmap v1432: 105 pgs: 105 active+clean; 349 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-09-14T04:25:13.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:12 smithi039 ceph-mon[106470]: pgmap v1432: 105 pgs: 105 active+clean; 349 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-09-14T04:25:13.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:12 smithi039 ceph-mon[111235]: pgmap v1432: 105 pgs: 105 active+clean; 349 MiB data, 1.7 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-09-14T04:25:15.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:14 smithi129 ceph-mon[101316]: pgmap v1433: 105 pgs: 105 active+clean; 357 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 2.8 MiB/s wr, 34 op/s 2024-09-14T04:25:15.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:14 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:25:15.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:14 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:25:15.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:14 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:25:15.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:14 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:25:15.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:14 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:25:15.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:14 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:25:15.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:14 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:25:15.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:14 smithi039 ceph-mon[106470]: pgmap v1433: 105 pgs: 105 active+clean; 357 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 2.8 MiB/s wr, 34 op/s 2024-09-14T04:25:15.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:14 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:25:15.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:14 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:25:15.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:14 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:25:15.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:14 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:25:15.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:14 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:25:15.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:14 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:25:15.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:14 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:25:15.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:14 smithi039 ceph-mon[111235]: pgmap v1433: 105 pgs: 105 active+clean; 357 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 2.8 MiB/s wr, 34 op/s 2024-09-14T04:25:15.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:14 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:25:15.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:14 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:25:15.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:14 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:25:15.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:14 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:25:15.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:14 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:25:15.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:14 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:25:15.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:14 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:25:17.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:16 smithi129 ceph-mon[101316]: pgmap v1434: 105 pgs: 105 active+clean; 365 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 74 KiB/s rd, 3.1 MiB/s wr, 31 op/s 2024-09-14T04:25:17.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:16 smithi039 ceph-mon[106470]: pgmap v1434: 105 pgs: 105 active+clean; 365 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 74 KiB/s rd, 3.1 MiB/s wr, 31 op/s 2024-09-14T04:25:17.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:16 smithi039 ceph-mon[111235]: pgmap v1434: 105 pgs: 105 active+clean; 365 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 74 KiB/s rd, 3.1 MiB/s wr, 31 op/s 2024-09-14T04:25:18.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:17 smithi129 ceph-mon[101316]: osdmap e996: 8 total, 8 up, 8 in 2024-09-14T04:25:18.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:17 smithi039 ceph-mon[106470]: osdmap e996: 8 total, 8 up, 8 in 2024-09-14T04:25:18.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:17 smithi039 ceph-mon[111235]: osdmap e996: 8 total, 8 up, 8 in 2024-09-14T04:25:18.937 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:25:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:25:18] "GET /metrics HTTP/1.1" 200 36689 "" "Prometheus/2.51.0" 2024-09-14T04:25:18.937 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:18 smithi039 ceph-mon[106470]: pgmap v1436: 105 pgs: 105 active+clean; 365 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 2.0 MiB/s wr, 33 op/s 2024-09-14T04:25:19.201 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/6.ShrinkToOneAfterSnapshot (23833 ms) 2024-09-14T04:25:19.201 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/6.MinOverlap 2024-09-14T04:25:19.201 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:25:19.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:18 smithi129 ceph-mon[101316]: pgmap v1436: 105 pgs: 105 active+clean; 365 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 2.0 MiB/s wr, 33 op/s 2024-09-14T04:25:19.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:18 smithi039 ceph-mon[111235]: pgmap v1436: 105 pgs: 105 active+clean; 365 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 2.0 MiB/s wr, 33 op/s 2024-09-14T04:25:21.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:20 smithi129 ceph-mon[101316]: pgmap v1437: 105 pgs: 105 active+clean; 406 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 5.7 MiB/s wr, 41 op/s 2024-09-14T04:25:21.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:20 smithi039 ceph-mon[106470]: pgmap v1437: 105 pgs: 105 active+clean; 406 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 5.7 MiB/s wr, 41 op/s 2024-09-14T04:25:21.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:20 smithi039 ceph-mon[111235]: pgmap v1437: 105 pgs: 105 active+clean; 406 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 5.7 MiB/s wr, 41 op/s 2024-09-14T04:25:23.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:22 smithi129 ceph-mon[101316]: pgmap v1438: 105 pgs: 105 active+clean; 426 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 7.7 MiB/s wr, 43 op/s 2024-09-14T04:25:23.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:25:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:22 smithi039 ceph-mon[106470]: pgmap v1438: 105 pgs: 105 active+clean; 426 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 7.7 MiB/s wr, 43 op/s 2024-09-14T04:25:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:25:23.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:22 smithi039 ceph-mon[111235]: pgmap v1438: 105 pgs: 105 active+clean; 426 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 7.7 MiB/s wr, 43 op/s 2024-09-14T04:25:23.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:25:25.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:24 smithi129 ceph-mon[101316]: pgmap v1439: 105 pgs: 105 active+clean; 434 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 7.7 MiB/s wr, 39 op/s 2024-09-14T04:25:25.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:24 smithi039 ceph-mon[106470]: pgmap v1439: 105 pgs: 105 active+clean; 434 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 7.7 MiB/s wr, 39 op/s 2024-09-14T04:25:25.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:24 smithi039 ceph-mon[111235]: pgmap v1439: 105 pgs: 105 active+clean; 434 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 7.7 MiB/s wr, 39 op/s 2024-09-14T04:25:27.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:26 smithi129 ceph-mon[101316]: pgmap v1440: 105 pgs: 105 active+clean; 434 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 6.9 MiB/s wr, 38 op/s 2024-09-14T04:25:27.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:26 smithi039 ceph-mon[106470]: pgmap v1440: 105 pgs: 105 active+clean; 434 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 6.9 MiB/s wr, 38 op/s 2024-09-14T04:25:27.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:26 smithi039 ceph-mon[111235]: pgmap v1440: 105 pgs: 105 active+clean; 434 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 6.9 MiB/s wr, 38 op/s 2024-09-14T04:25:28.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:28 smithi129 ceph-mon[101316]: osdmap e997: 8 total, 8 up, 8 in 2024-09-14T04:25:28.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:28 smithi039 ceph-mon[106470]: osdmap e997: 8 total, 8 up, 8 in 2024-09-14T04:25:28.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:28 smithi039 ceph-mon[111235]: osdmap e997: 8 total, 8 up, 8 in 2024-09-14T04:25:29.008 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:25:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:25:28] "GET /metrics HTTP/1.1" 200 36696 "" "Prometheus/2.51.0" 2024-09-14T04:25:29.009 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:29 smithi039 ceph-mon[106470]: pgmap v1442: 105 pgs: 105 active+clean; 442 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 7.7 MiB/s wr, 38 op/s 2024-09-14T04:25:29.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:29 smithi129 ceph-mon[101316]: pgmap v1442: 105 pgs: 105 active+clean; 442 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 7.7 MiB/s wr, 38 op/s 2024-09-14T04:25:29.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:29 smithi039 ceph-mon[111235]: pgmap v1442: 105 pgs: 105 active+clean; 442 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4.2 MiB/s rd, 7.7 MiB/s wr, 38 op/s 2024-09-14T04:25:31.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:31 smithi129 ceph-mon[101316]: pgmap v1443: 105 pgs: 105 active+clean; 470 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 6.4 MiB/s wr, 29 op/s 2024-09-14T04:25:31.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:31 smithi039 ceph-mon[106470]: pgmap v1443: 105 pgs: 105 active+clean; 470 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 6.4 MiB/s wr, 29 op/s 2024-09-14T04:25:31.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:31 smithi039 ceph-mon[111235]: pgmap v1443: 105 pgs: 105 active+clean; 470 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 1.2 MiB/s rd, 6.4 MiB/s wr, 29 op/s 2024-09-14T04:25:33.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:33 smithi039 ceph-mon[106470]: pgmap v1444: 105 pgs: 105 active+clean; 475 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 434 KiB/s rd, 4.9 MiB/s wr, 36 op/s 2024-09-14T04:25:33.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:33 smithi039 ceph-mon[111235]: pgmap v1444: 105 pgs: 105 active+clean; 475 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 434 KiB/s rd, 4.9 MiB/s wr, 36 op/s 2024-09-14T04:25:33.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:33 smithi129 ceph-mon[101316]: pgmap v1444: 105 pgs: 105 active+clean; 475 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 434 KiB/s rd, 4.9 MiB/s wr, 36 op/s 2024-09-14T04:25:35.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:35 smithi039 ceph-mon[106470]: pgmap v1445: 105 pgs: 105 active+clean; 475 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-09-14T04:25:35.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:35 smithi039 ceph-mon[111235]: pgmap v1445: 105 pgs: 105 active+clean; 475 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-09-14T04:25:35.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:35 smithi129 ceph-mon[101316]: pgmap v1445: 105 pgs: 105 active+clean; 475 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-09-14T04:25:37.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:37 smithi039 ceph-mon[106470]: pgmap v1446: 105 pgs: 105 active+clean; 479 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.5 MiB/s wr, 35 op/s 2024-09-14T04:25:37.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:37 smithi039 ceph-mon[111235]: pgmap v1446: 105 pgs: 105 active+clean; 479 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.5 MiB/s wr, 35 op/s 2024-09-14T04:25:37.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:37 smithi129 ceph-mon[101316]: pgmap v1446: 105 pgs: 105 active+clean; 479 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.5 MiB/s wr, 35 op/s 2024-09-14T04:25:38.395 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:38 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:25:38.396 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:38 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:25:38.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:38 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:25:39.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:25:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:25:38] "GET /metrics HTTP/1.1" 200 36699 "" "Prometheus/2.51.0" 2024-09-14T04:25:39.511 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:39 smithi039 ceph-mon[106470]: pgmap v1447: 105 pgs: 105 active+clean; 479 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 3.6 MiB/s wr, 37 op/s 2024-09-14T04:25:39.512 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:39 smithi039 ceph-mon[111235]: pgmap v1447: 105 pgs: 105 active+clean; 479 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 3.6 MiB/s wr, 37 op/s 2024-09-14T04:25:39.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:39 smithi129 ceph-mon[101316]: pgmap v1447: 105 pgs: 105 active+clean; 479 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 3.6 MiB/s wr, 37 op/s 2024-09-14T04:25:40.959 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/6.MinOverlap (21757 ms) 2024-09-14T04:25:40.959 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/6.ZeroOverlap 2024-09-14T04:25:40.959 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:25:41.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:41 smithi129 ceph-mon[101316]: pgmap v1448: 105 pgs: 105 active+clean; 491 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s rd, 4.1 MiB/s wr, 40 op/s 2024-09-14T04:25:41.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:41 smithi039 ceph-mon[106470]: pgmap v1448: 105 pgs: 105 active+clean; 491 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s rd, 4.1 MiB/s wr, 40 op/s 2024-09-14T04:25:41.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:41 smithi039 ceph-mon[111235]: pgmap v1448: 105 pgs: 105 active+clean; 491 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 77 KiB/s rd, 4.1 MiB/s wr, 40 op/s 2024-09-14T04:25:43.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:43 smithi129 ceph-mon[101316]: pgmap v1449: 105 pgs: 105 active+clean; 491 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 405 KiB/s rd, 2.1 MiB/s wr, 23 op/s 2024-09-14T04:25:43.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:43 smithi039 ceph-mon[106470]: pgmap v1449: 105 pgs: 105 active+clean; 491 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 405 KiB/s rd, 2.1 MiB/s wr, 23 op/s 2024-09-14T04:25:43.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:43 smithi039 ceph-mon[111235]: pgmap v1449: 105 pgs: 105 active+clean; 491 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 405 KiB/s rd, 2.1 MiB/s wr, 23 op/s 2024-09-14T04:25:45.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:45 smithi129 ceph-mon[101316]: pgmap v1450: 105 pgs: 105 active+clean; 495 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.3 MiB/s wr, 29 op/s 2024-09-14T04:25:45.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:45 smithi039 ceph-mon[106470]: pgmap v1450: 105 pgs: 105 active+clean; 495 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.3 MiB/s wr, 29 op/s 2024-09-14T04:25:45.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:45 smithi039 ceph-mon[111235]: pgmap v1450: 105 pgs: 105 active+clean; 495 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.3 MiB/s wr, 29 op/s 2024-09-14T04:25:47.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:47 smithi129 ceph-mon[101316]: pgmap v1451: 105 pgs: 105 active+clean; 495 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.3 MiB/s wr, 25 op/s 2024-09-14T04:25:47.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:47 smithi039 ceph-mon[106470]: pgmap v1451: 105 pgs: 105 active+clean; 495 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.3 MiB/s wr, 25 op/s 2024-09-14T04:25:47.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:47 smithi039 ceph-mon[111235]: pgmap v1451: 105 pgs: 105 active+clean; 495 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.3 MiB/s wr, 25 op/s 2024-09-14T04:25:48.929 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:25:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:25:48] "GET /metrics HTTP/1.1" 200 36699 "" "Prometheus/2.51.0" 2024-09-14T04:25:49.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:49 smithi039 ceph-mon[111235]: pgmap v1452: 105 pgs: 105 active+clean; 495 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.0 MiB/s wr, 25 op/s 2024-09-14T04:25:49.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:49 smithi039 ceph-mon[106470]: pgmap v1452: 105 pgs: 105 active+clean; 495 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.0 MiB/s wr, 25 op/s 2024-09-14T04:25:49.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:49 smithi129 ceph-mon[101316]: pgmap v1452: 105 pgs: 105 active+clean; 495 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 3.0 MiB/s wr, 25 op/s 2024-09-14T04:25:50.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:50 smithi129 ceph-mon[101316]: osdmap e998: 8 total, 8 up, 8 in 2024-09-14T04:25:50.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:50 smithi039 ceph-mon[106470]: osdmap e998: 8 total, 8 up, 8 in 2024-09-14T04:25:50.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:50 smithi039 ceph-mon[111235]: osdmap e998: 8 total, 8 up, 8 in 2024-09-14T04:25:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:51 smithi129 ceph-mon[101316]: pgmap v1454: 105 pgs: 105 active+clean; 495 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-09-14T04:25:51.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:51 smithi039 ceph-mon[106470]: pgmap v1454: 105 pgs: 105 active+clean; 495 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-09-14T04:25:51.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:51 smithi039 ceph-mon[111235]: pgmap v1454: 105 pgs: 105 active+clean; 495 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 3.2 MiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-09-14T04:25:53.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:53 smithi129 ceph-mon[101316]: pgmap v1455: 105 pgs: 105 active+clean; 499 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-09-14T04:25:53.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:53 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:25:53.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:53 smithi039 ceph-mon[106470]: pgmap v1455: 105 pgs: 105 active+clean; 499 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-09-14T04:25:53.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:53 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:25:53.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:53 smithi039 ceph-mon[111235]: pgmap v1455: 105 pgs: 105 active+clean; 499 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 2.4 MiB/s wr, 17 op/s 2024-09-14T04:25:53.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:53 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:25:55.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:55 smithi129 ceph-mon[101316]: pgmap v1456: 105 pgs: 105 active+clean; 536 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-09-14T04:25:55.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:55 smithi039 ceph-mon[106470]: pgmap v1456: 105 pgs: 105 active+clean; 536 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-09-14T04:25:55.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:55 smithi039 ceph-mon[111235]: pgmap v1456: 105 pgs: 105 active+clean; 536 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-09-14T04:25:57.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:57 smithi129 ceph-mon[101316]: pgmap v1457: 105 pgs: 105 active+clean; 536 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-09-14T04:25:57.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:57 smithi039 ceph-mon[106470]: pgmap v1457: 105 pgs: 105 active+clean; 536 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-09-14T04:25:57.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:57 smithi039 ceph-mon[111235]: pgmap v1457: 105 pgs: 105 active+clean; 536 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-09-14T04:25:59.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:25:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:25:58] "GET /metrics HTTP/1.1" 200 36692 "" "Prometheus/2.51.0" 2024-09-14T04:25:59.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:25:59 smithi129 ceph-mon[101316]: pgmap v1458: 105 pgs: 105 active+clean; 536 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-09-14T04:25:59.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:25:59 smithi039 ceph-mon[106470]: pgmap v1458: 105 pgs: 105 active+clean; 536 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-09-14T04:25:59.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:25:59 smithi039 ceph-mon[111235]: pgmap v1458: 105 pgs: 105 active+clean; 536 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 35 op/s 2024-09-14T04:26:01.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:01 smithi129 ceph-mon[101316]: pgmap v1459: 105 pgs: 105 active+clean; 552 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 61 KiB/s rd, 5.6 MiB/s wr, 45 op/s 2024-09-14T04:26:01.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:01 smithi039 ceph-mon[106470]: pgmap v1459: 105 pgs: 105 active+clean; 552 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 61 KiB/s rd, 5.6 MiB/s wr, 45 op/s 2024-09-14T04:26:01.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:01 smithi039 ceph-mon[111235]: pgmap v1459: 105 pgs: 105 active+clean; 552 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 61 KiB/s rd, 5.6 MiB/s wr, 45 op/s 2024-09-14T04:26:02.744 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/6.ZeroOverlap (21786 ms) 2024-09-14T04:26:03.206 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 7 tests from EncryptedFlattenTest/6 (155399 ms total) 2024-09-14T04:26:03.206 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:26:03.206 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 7 tests from EncryptedFlattenTest/7, where TypeParam = LUKS2UnderLUKS1 2024-09-14T04:26:03.240 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/7.Simple 2024-09-14T04:26:03.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:03 smithi129 ceph-mon[101316]: pgmap v1460: 105 pgs: 105 active+clean; 552 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s rd, 4.8 MiB/s wr, 38 op/s 2024-09-14T04:26:03.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:03 smithi039 ceph-mon[106470]: pgmap v1460: 105 pgs: 105 active+clean; 552 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s rd, 4.8 MiB/s wr, 38 op/s 2024-09-14T04:26:03.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:03 smithi039 ceph-mon[111235]: pgmap v1460: 105 pgs: 105 active+clean; 552 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 52 KiB/s rd, 4.8 MiB/s wr, 38 op/s 2024-09-14T04:26:04.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:04 smithi129 ceph-mon[101316]: osdmap e999: 8 total, 8 up, 8 in 2024-09-14T04:26:04.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:04 smithi039 ceph-mon[106470]: osdmap e999: 8 total, 8 up, 8 in 2024-09-14T04:26:04.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:04 smithi039 ceph-mon[111235]: osdmap e999: 8 total, 8 up, 8 in 2024-09-14T04:26:05.255 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:26:05.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:05 smithi129 ceph-mon[101316]: pgmap v1462: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:26:05.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:05 smithi129 ceph-mon[101316]: osdmap e1000: 8 total, 8 up, 8 in 2024-09-14T04:26:05.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:05 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3122198169' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-46","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:26:05.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:05 smithi039 ceph-mon[106470]: pgmap v1462: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:26:05.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:05 smithi039 ceph-mon[106470]: osdmap e1000: 8 total, 8 up, 8 in 2024-09-14T04:26:05.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:05 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3122198169' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-46","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:26:05.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:05 smithi039 ceph-mon[111235]: pgmap v1462: 73 pgs: 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:26:05.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:05 smithi039 ceph-mon[111235]: osdmap e1000: 8 total, 8 up, 8 in 2024-09-14T04:26:05.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:05 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3122198169' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-46","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:26:06.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:06 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3122198169' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-46","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:26:06.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:06 smithi129 ceph-mon[101316]: osdmap e1001: 8 total, 8 up, 8 in 2024-09-14T04:26:06.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:06 smithi129 ceph-mon[101316]: osdmap e1002: 8 total, 8 up, 8 in 2024-09-14T04:26:06.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:06 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3122198169' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-46","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:26:06.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:06 smithi039 ceph-mon[106470]: osdmap e1001: 8 total, 8 up, 8 in 2024-09-14T04:26:06.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:06 smithi039 ceph-mon[106470]: osdmap e1002: 8 total, 8 up, 8 in 2024-09-14T04:26:06.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:06 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3122198169' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-46","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:26:06.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:06 smithi039 ceph-mon[111235]: osdmap e1001: 8 total, 8 up, 8 in 2024-09-14T04:26:06.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:06 smithi039 ceph-mon[111235]: osdmap e1002: 8 total, 8 up, 8 in 2024-09-14T04:26:07.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:07 smithi129 ceph-mon[101316]: pgmap v1465: 105 pgs: 6 creating+peering, 26 unknown, 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:26:07.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:07 smithi039 ceph-mon[106470]: pgmap v1465: 105 pgs: 6 creating+peering, 26 unknown, 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:26:07.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:07 smithi039 ceph-mon[111235]: pgmap v1465: 105 pgs: 6 creating+peering, 26 unknown, 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:26:08.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:08 smithi039 ceph-mon[106470]: osdmap e1003: 8 total, 8 up, 8 in 2024-09-14T04:26:08.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:08 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:26:08.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:08 smithi039 ceph-mon[111235]: osdmap e1003: 8 total, 8 up, 8 in 2024-09-14T04:26:08.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:08 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:26:08.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:08 smithi129 ceph-mon[101316]: osdmap e1003: 8 total, 8 up, 8 in 2024-09-14T04:26:08.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:08 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:26:09.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:26:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:26:08] "GET /metrics HTTP/1.1" 200 36630 "" "Prometheus/2.51.0" 2024-09-14T04:26:09.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:09 smithi039 ceph-mon[106470]: pgmap v1468: 105 pgs: 6 creating+peering, 23 unknown, 76 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:26:09.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:09 smithi039 ceph-mon[111235]: pgmap v1468: 105 pgs: 6 creating+peering, 23 unknown, 76 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:26:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:09 smithi129 ceph-mon[101316]: pgmap v1468: 105 pgs: 6 creating+peering, 23 unknown, 76 active+clean; 582 KiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:26:10.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:10 smithi129 ceph-mon[101316]: pgmap v1469: 105 pgs: 5 creating+peering, 100 active+clean; 582 KiB data, 744 MiB used, 714 GiB / 715 GiB avail; 5.9 KiB/s rd, 1.6 KiB/s wr, 10 op/s 2024-09-14T04:26:10.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:10 smithi039 ceph-mon[106470]: pgmap v1469: 105 pgs: 5 creating+peering, 100 active+clean; 582 KiB data, 744 MiB used, 714 GiB / 715 GiB avail; 5.9 KiB/s rd, 1.6 KiB/s wr, 10 op/s 2024-09-14T04:26:10.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:10 smithi039 ceph-mon[111235]: pgmap v1469: 105 pgs: 5 creating+peering, 100 active+clean; 582 KiB data, 744 MiB used, 714 GiB / 715 GiB avail; 5.9 KiB/s rd, 1.6 KiB/s wr, 10 op/s 2024-09-14T04:26:12.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:12 smithi129 ceph-mon[101316]: pgmap v1470: 105 pgs: 105 active+clean; 582 KiB data, 744 MiB used, 714 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.3 KiB/s wr, 8 op/s 2024-09-14T04:26:12.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:12 smithi039 ceph-mon[106470]: pgmap v1470: 105 pgs: 105 active+clean; 582 KiB data, 744 MiB used, 714 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.3 KiB/s wr, 8 op/s 2024-09-14T04:26:12.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:12 smithi039 ceph-mon[111235]: pgmap v1470: 105 pgs: 105 active+clean; 582 KiB data, 744 MiB used, 714 GiB / 715 GiB avail; 4.9 KiB/s rd, 1.3 KiB/s wr, 8 op/s 2024-09-14T04:26:14.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:14 smithi129 ceph-mon[101316]: pgmap v1471: 105 pgs: 105 active+clean; 582 KiB data, 744 MiB used, 714 GiB / 715 GiB avail; 3.7 KiB/s rd, 1023 B/s wr, 6 op/s 2024-09-14T04:26:14.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:14 smithi039 ceph-mon[106470]: pgmap v1471: 105 pgs: 105 active+clean; 582 KiB data, 744 MiB used, 714 GiB / 715 GiB avail; 3.7 KiB/s rd, 1023 B/s wr, 6 op/s 2024-09-14T04:26:14.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:14 smithi039 ceph-mon[111235]: pgmap v1471: 105 pgs: 105 active+clean; 582 KiB data, 744 MiB used, 714 GiB / 715 GiB avail; 3.7 KiB/s rd, 1023 B/s wr, 6 op/s 2024-09-14T04:26:15.662 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:15 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:26:15.662 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:15 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:26:15.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:15 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:26:16.481 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:16 smithi129 ceph-mon[101316]: pgmap v1472: 105 pgs: 105 active+clean; 8.6 MiB data, 772 MiB used, 714 GiB / 715 GiB avail; 3.3 KiB/s rd, 904 KiB/s wr, 6 op/s 2024-09-14T04:26:16.481 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:16 smithi129 ceph-mon[101316]: osdmap e1004: 8 total, 8 up, 8 in 2024-09-14T04:26:16.481 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:16 smithi129 ceph-mon[101316]: osdmap e1005: 8 total, 8 up, 8 in 2024-09-14T04:26:16.745 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:16 smithi039 ceph-mon[106470]: pgmap v1472: 105 pgs: 105 active+clean; 8.6 MiB data, 772 MiB used, 714 GiB / 715 GiB avail; 3.3 KiB/s rd, 904 KiB/s wr, 6 op/s 2024-09-14T04:26:16.745 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:16 smithi039 ceph-mon[106470]: osdmap e1004: 8 total, 8 up, 8 in 2024-09-14T04:26:16.745 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:16 smithi039 ceph-mon[106470]: osdmap e1005: 8 total, 8 up, 8 in 2024-09-14T04:26:16.745 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:16 smithi039 ceph-mon[111235]: pgmap v1472: 105 pgs: 105 active+clean; 8.6 MiB data, 772 MiB used, 714 GiB / 715 GiB avail; 3.3 KiB/s rd, 904 KiB/s wr, 6 op/s 2024-09-14T04:26:16.746 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:16 smithi039 ceph-mon[111235]: osdmap e1004: 8 total, 8 up, 8 in 2024-09-14T04:26:16.746 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:16 smithi039 ceph-mon[111235]: osdmap e1005: 8 total, 8 up, 8 in 2024-09-14T04:26:18.505 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:18 smithi039 ceph-mon[106470]: pgmap v1475: 105 pgs: 105 active+clean; 13 MiB data, 785 MiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s wr, 0 op/s 2024-09-14T04:26:18.505 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:18 smithi039 ceph-mon[111235]: pgmap v1475: 105 pgs: 105 active+clean; 13 MiB data, 785 MiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s wr, 0 op/s 2024-09-14T04:26:18.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:18 smithi129 ceph-mon[101316]: pgmap v1475: 105 pgs: 105 active+clean; 13 MiB data, 785 MiB used, 714 GiB / 715 GiB avail; 1.5 MiB/s wr, 0 op/s 2024-09-14T04:26:18.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:26:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:26:18] "GET /metrics HTTP/1.1" 200 36630 "" "Prometheus/2.51.0" 2024-09-14T04:26:20.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:19 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:26:20.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:19 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:26:20.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:19 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:26:20.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:19 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:26:20.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:19 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:26:20.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:19 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:26:20.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:19 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:26:20.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:19 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:26:20.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:19 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:26:20.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:19 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:26:20.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:19 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:26:20.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:19 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:26:20.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:19 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:26:20.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:19 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:26:20.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:19 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:26:20.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:19 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:26:20.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:19 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:26:20.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:19 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:26:20.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:19 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:26:20.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:19 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:26:20.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:19 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:26:21.525 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:21 smithi129 ceph-mon[101316]: pgmap v1476: 105 pgs: 105 active+clean; 38 MiB data, 863 MiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 4.6 MiB/s wr, 44 op/s 2024-09-14T04:26:21.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:21 smithi039 ceph-mon[106470]: pgmap v1476: 105 pgs: 105 active+clean; 38 MiB data, 863 MiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 4.6 MiB/s wr, 44 op/s 2024-09-14T04:26:21.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:21 smithi039 ceph-mon[111235]: pgmap v1476: 105 pgs: 105 active+clean; 38 MiB data, 863 MiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 4.6 MiB/s wr, 44 op/s 2024-09-14T04:26:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:23 smithi129 ceph-mon[101316]: pgmap v1477: 105 pgs: 105 active+clean; 42 MiB data, 867 MiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 5.1 MiB/s wr, 44 op/s 2024-09-14T04:26:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:26:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:23 smithi039 ceph-mon[106470]: pgmap v1477: 105 pgs: 105 active+clean; 42 MiB data, 867 MiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 5.1 MiB/s wr, 44 op/s 2024-09-14T04:26:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:26:23.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:23 smithi039 ceph-mon[111235]: pgmap v1477: 105 pgs: 105 active+clean; 42 MiB data, 867 MiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 5.1 MiB/s wr, 44 op/s 2024-09-14T04:26:23.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:26:25.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:25 smithi129 ceph-mon[101316]: pgmap v1478: 105 pgs: 105 active+clean; 46 MiB data, 879 MiB used, 714 GiB / 715 GiB avail; 105 KiB/s rd, 4.6 MiB/s wr, 46 op/s 2024-09-14T04:26:25.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:25 smithi039 ceph-mon[106470]: pgmap v1478: 105 pgs: 105 active+clean; 46 MiB data, 879 MiB used, 714 GiB / 715 GiB avail; 105 KiB/s rd, 4.6 MiB/s wr, 46 op/s 2024-09-14T04:26:25.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:25 smithi039 ceph-mon[111235]: pgmap v1478: 105 pgs: 105 active+clean; 46 MiB data, 879 MiB used, 714 GiB / 715 GiB avail; 105 KiB/s rd, 4.6 MiB/s wr, 46 op/s 2024-09-14T04:26:27.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:27 smithi129 ceph-mon[101316]: pgmap v1479: 105 pgs: 105 active+clean; 46 MiB data, 879 MiB used, 714 GiB / 715 GiB avail; 88 KiB/s rd, 3.7 MiB/s wr, 40 op/s 2024-09-14T04:26:27.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:27 smithi039 ceph-mon[106470]: pgmap v1479: 105 pgs: 105 active+clean; 46 MiB data, 879 MiB used, 714 GiB / 715 GiB avail; 88 KiB/s rd, 3.7 MiB/s wr, 40 op/s 2024-09-14T04:26:27.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:27 smithi039 ceph-mon[111235]: pgmap v1479: 105 pgs: 105 active+clean; 46 MiB data, 879 MiB used, 714 GiB / 715 GiB avail; 88 KiB/s rd, 3.7 MiB/s wr, 40 op/s 2024-09-14T04:26:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:26:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:26:28] "GET /metrics HTTP/1.1" 200 36678 "" "Prometheus/2.51.0" 2024-09-14T04:26:29.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:29 smithi129 ceph-mon[101316]: pgmap v1480: 105 pgs: 105 active+clean; 46 MiB data, 879 MiB used, 714 GiB / 715 GiB avail; 100 KiB/s rd, 3.2 MiB/s wr, 34 op/s 2024-09-14T04:26:29.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:29 smithi039 ceph-mon[106470]: pgmap v1480: 105 pgs: 105 active+clean; 46 MiB data, 879 MiB used, 714 GiB / 715 GiB avail; 100 KiB/s rd, 3.2 MiB/s wr, 34 op/s 2024-09-14T04:26:29.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:29 smithi039 ceph-mon[111235]: pgmap v1480: 105 pgs: 105 active+clean; 46 MiB data, 879 MiB used, 714 GiB / 715 GiB avail; 100 KiB/s rd, 3.2 MiB/s wr, 34 op/s 2024-09-14T04:26:29.545 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/7.Simple (26305 ms) 2024-09-14T04:26:29.545 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/7.Grow 2024-09-14T04:26:29.545 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:26:31.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:31 smithi129 ceph-mon[101316]: pgmap v1481: 105 pgs: 105 active+clean; 66 MiB data, 907 MiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 4.8 MiB/s wr, 50 op/s 2024-09-14T04:26:31.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:31 smithi039 ceph-mon[106470]: pgmap v1481: 105 pgs: 105 active+clean; 66 MiB data, 907 MiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 4.8 MiB/s wr, 50 op/s 2024-09-14T04:26:31.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:31 smithi039 ceph-mon[111235]: pgmap v1481: 105 pgs: 105 active+clean; 66 MiB data, 907 MiB used, 714 GiB / 715 GiB avail; 5.4 MiB/s rd, 4.8 MiB/s wr, 50 op/s 2024-09-14T04:26:33.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:33 smithi129 ceph-mon[101316]: pgmap v1482: 105 pgs: 105 active+clean; 70 MiB data, 923 MiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 3.0 MiB/s wr, 21 op/s 2024-09-14T04:26:33.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:33 smithi039 ceph-mon[106470]: pgmap v1482: 105 pgs: 105 active+clean; 70 MiB data, 923 MiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 3.0 MiB/s wr, 21 op/s 2024-09-14T04:26:33.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:33 smithi039 ceph-mon[111235]: pgmap v1482: 105 pgs: 105 active+clean; 70 MiB data, 923 MiB used, 714 GiB / 715 GiB avail; 6.7 MiB/s rd, 3.0 MiB/s wr, 21 op/s 2024-09-14T04:26:35.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:35 smithi129 ceph-mon[101316]: pgmap v1483: 105 pgs: 105 active+clean; 74 MiB data, 963 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 3.0 MiB/s wr, 26 op/s 2024-09-14T04:26:35.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:35 smithi039 ceph-mon[106470]: pgmap v1483: 105 pgs: 105 active+clean; 74 MiB data, 963 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 3.0 MiB/s wr, 26 op/s 2024-09-14T04:26:35.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:35 smithi039 ceph-mon[111235]: pgmap v1483: 105 pgs: 105 active+clean; 74 MiB data, 963 MiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 3.0 MiB/s wr, 26 op/s 2024-09-14T04:26:37.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:37 smithi129 ceph-mon[101316]: pgmap v1484: 105 pgs: 105 active+clean; 74 MiB data, 963 MiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.7 MiB/s wr, 25 op/s 2024-09-14T04:26:37.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:37 smithi039 ceph-mon[106470]: pgmap v1484: 105 pgs: 105 active+clean; 74 MiB data, 963 MiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.7 MiB/s wr, 25 op/s 2024-09-14T04:26:37.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:37 smithi039 ceph-mon[111235]: pgmap v1484: 105 pgs: 105 active+clean; 74 MiB data, 963 MiB used, 714 GiB / 715 GiB avail; 8.7 MiB/s rd, 2.7 MiB/s wr, 25 op/s 2024-09-14T04:26:38.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:38 smithi129 ceph-mon[101316]: osdmap e1006: 8 total, 8 up, 8 in 2024-09-14T04:26:38.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:38 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:26:38.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:38 smithi039 ceph-mon[106470]: osdmap e1006: 8 total, 8 up, 8 in 2024-09-14T04:26:38.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:38 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:26:38.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:38 smithi039 ceph-mon[111235]: osdmap e1006: 8 total, 8 up, 8 in 2024-09-14T04:26:38.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:38 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:26:39.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:26:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:26:38] "GET /metrics HTTP/1.1" 200 36683 "" "Prometheus/2.51.0" 2024-09-14T04:26:39.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:39 smithi129 ceph-mon[101316]: pgmap v1486: 105 pgs: 105 active+clean; 74 MiB data, 963 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.2 MiB/s wr, 27 op/s 2024-09-14T04:26:39.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:39 smithi039 ceph-mon[106470]: pgmap v1486: 105 pgs: 105 active+clean; 74 MiB data, 963 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.2 MiB/s wr, 27 op/s 2024-09-14T04:26:39.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:39 smithi039 ceph-mon[111235]: pgmap v1486: 105 pgs: 105 active+clean; 74 MiB data, 963 MiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 3.2 MiB/s wr, 27 op/s 2024-09-14T04:26:41.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:41 smithi129 ceph-mon[101316]: pgmap v1487: 105 pgs: 105 active+clean; 103 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 4.1 MiB/s rd, 3.7 MiB/s wr, 40 op/s 2024-09-14T04:26:41.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:41 smithi039 ceph-mon[106470]: pgmap v1487: 105 pgs: 105 active+clean; 103 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 4.1 MiB/s rd, 3.7 MiB/s wr, 40 op/s 2024-09-14T04:26:41.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:41 smithi039 ceph-mon[111235]: pgmap v1487: 105 pgs: 105 active+clean; 103 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 4.1 MiB/s rd, 3.7 MiB/s wr, 40 op/s 2024-09-14T04:26:43.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:43 smithi039 ceph-mon[106470]: pgmap v1488: 105 pgs: 105 active+clean; 115 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 4.5 MiB/s wr, 41 op/s 2024-09-14T04:26:43.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:43 smithi039 ceph-mon[111235]: pgmap v1488: 105 pgs: 105 active+clean; 115 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 4.5 MiB/s wr, 41 op/s 2024-09-14T04:26:43.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:43 smithi129 ceph-mon[101316]: pgmap v1488: 105 pgs: 105 active+clean; 115 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.5 MiB/s rd, 4.5 MiB/s wr, 41 op/s 2024-09-14T04:26:45.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:45 smithi039 ceph-mon[106470]: pgmap v1489: 105 pgs: 105 active+clean; 119 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 4.5 MiB/s wr, 35 op/s 2024-09-14T04:26:45.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:45 smithi039 ceph-mon[111235]: pgmap v1489: 105 pgs: 105 active+clean; 119 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 4.5 MiB/s wr, 35 op/s 2024-09-14T04:26:45.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:45 smithi129 ceph-mon[101316]: pgmap v1489: 105 pgs: 105 active+clean; 119 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 4.5 MiB/s wr, 35 op/s 2024-09-14T04:26:47.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:47 smithi039 ceph-mon[106470]: pgmap v1490: 105 pgs: 105 active+clean; 119 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 84 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-09-14T04:26:47.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:47 smithi039 ceph-mon[111235]: pgmap v1490: 105 pgs: 105 active+clean; 119 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 84 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-09-14T04:26:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:47 smithi129 ceph-mon[101316]: pgmap v1490: 105 pgs: 105 active+clean; 119 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 84 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-09-14T04:26:48.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:48 smithi039 ceph-mon[106470]: pgmap v1491: 105 pgs: 105 active+clean; 119 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 83 KiB/s rd, 4.5 MiB/s wr, 36 op/s 2024-09-14T04:26:48.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:48 smithi039 ceph-mon[111235]: pgmap v1491: 105 pgs: 105 active+clean; 119 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 83 KiB/s rd, 4.5 MiB/s wr, 36 op/s 2024-09-14T04:26:48.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:48 smithi129 ceph-mon[101316]: pgmap v1491: 105 pgs: 105 active+clean; 119 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 83 KiB/s rd, 4.5 MiB/s wr, 36 op/s 2024-09-14T04:26:49.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:26:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:26:48] "GET /metrics HTTP/1.1" 200 36683 "" "Prometheus/2.51.0" 2024-09-14T04:26:50.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:50 smithi129 ceph-mon[101316]: pgmap v1492: 105 pgs: 105 active+clean; 123 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 184 KiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-09-14T04:26:50.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:50 smithi039 ceph-mon[106470]: pgmap v1492: 105 pgs: 105 active+clean; 123 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 184 KiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-09-14T04:26:50.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:50 smithi039 ceph-mon[111235]: pgmap v1492: 105 pgs: 105 active+clean; 123 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 184 KiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-09-14T04:26:51.428 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/7.Grow (21884 ms) 2024-09-14T04:26:51.429 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/7.Shrink 2024-09-14T04:26:51.429 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:26:52.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:52 smithi129 ceph-mon[101316]: pgmap v1493: 105 pgs: 105 active+clean; 127 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2024-09-14T04:26:52.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:26:52.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:52 smithi039 ceph-mon[106470]: pgmap v1493: 105 pgs: 105 active+clean; 127 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2024-09-14T04:26:52.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:26:52.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:52 smithi039 ceph-mon[111235]: pgmap v1493: 105 pgs: 105 active+clean; 127 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 1.9 MiB/s rd, 2.0 MiB/s wr, 16 op/s 2024-09-14T04:26:52.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:26:54.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:54 smithi129 ceph-mon[101316]: pgmap v1494: 105 pgs: 105 active+clean; 147 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.5 MiB/s rd, 3.0 MiB/s wr, 28 op/s 2024-09-14T04:26:54.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:54 smithi039 ceph-mon[106470]: pgmap v1494: 105 pgs: 105 active+clean; 147 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.5 MiB/s rd, 3.0 MiB/s wr, 28 op/s 2024-09-14T04:26:54.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:54 smithi039 ceph-mon[111235]: pgmap v1494: 105 pgs: 105 active+clean; 147 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 8.5 MiB/s rd, 3.0 MiB/s wr, 28 op/s 2024-09-14T04:26:56.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:56 smithi129 ceph-mon[101316]: pgmap v1495: 105 pgs: 105 active+clean; 147 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 2.7 MiB/s wr, 28 op/s 2024-09-14T04:26:56.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:56 smithi039 ceph-mon[106470]: pgmap v1495: 105 pgs: 105 active+clean; 147 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 2.7 MiB/s wr, 28 op/s 2024-09-14T04:26:56.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:56 smithi039 ceph-mon[111235]: pgmap v1495: 105 pgs: 105 active+clean; 147 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 2.7 MiB/s wr, 28 op/s 2024-09-14T04:26:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:58 smithi129 ceph-mon[101316]: pgmap v1496: 105 pgs: 105 active+clean; 147 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 2.7 MiB/s wr, 28 op/s 2024-09-14T04:26:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:26:58 smithi129 ceph-mon[101316]: osdmap e1007: 8 total, 8 up, 8 in 2024-09-14T04:26:58.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:26:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:26:58] "GET /metrics HTTP/1.1" 200 36691 "" "Prometheus/2.51.0" 2024-09-14T04:26:58.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:58 smithi039 ceph-mon[106470]: pgmap v1496: 105 pgs: 105 active+clean; 147 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 2.7 MiB/s wr, 28 op/s 2024-09-14T04:26:58.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:26:58 smithi039 ceph-mon[106470]: osdmap e1007: 8 total, 8 up, 8 in 2024-09-14T04:26:58.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:58 smithi039 ceph-mon[111235]: pgmap v1496: 105 pgs: 105 active+clean; 147 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 8.8 MiB/s rd, 2.7 MiB/s wr, 28 op/s 2024-09-14T04:26:58.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:26:58 smithi039 ceph-mon[111235]: osdmap e1007: 8 total, 8 up, 8 in 2024-09-14T04:27:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:00 smithi129 ceph-mon[101316]: pgmap v1498: 105 pgs: 105 active+clean; 184 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 6.5 MiB/s wr, 45 op/s 2024-09-14T04:27:00.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:00 smithi039 ceph-mon[106470]: pgmap v1498: 105 pgs: 105 active+clean; 184 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 6.5 MiB/s wr, 45 op/s 2024-09-14T04:27:00.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:00 smithi039 ceph-mon[111235]: pgmap v1498: 105 pgs: 105 active+clean; 184 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 10 MiB/s rd, 6.5 MiB/s wr, 45 op/s 2024-09-14T04:27:02.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:02 smithi129 ceph-mon[101316]: pgmap v1499: 105 pgs: 105 active+clean; 188 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 6.5 MiB/s wr, 37 op/s 2024-09-14T04:27:02.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:02 smithi039 ceph-mon[106470]: pgmap v1499: 105 pgs: 105 active+clean; 188 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 6.5 MiB/s wr, 37 op/s 2024-09-14T04:27:02.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:02 smithi039 ceph-mon[111235]: pgmap v1499: 105 pgs: 105 active+clean; 188 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 6.5 MiB/s wr, 37 op/s 2024-09-14T04:27:04.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:04 smithi129 ceph-mon[101316]: pgmap v1500: 105 pgs: 105 active+clean; 192 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 464 KiB/s rd, 4.5 MiB/s wr, 35 op/s 2024-09-14T04:27:04.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:04 smithi039 ceph-mon[106470]: pgmap v1500: 105 pgs: 105 active+clean; 192 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 464 KiB/s rd, 4.5 MiB/s wr, 35 op/s 2024-09-14T04:27:04.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:04 smithi039 ceph-mon[111235]: pgmap v1500: 105 pgs: 105 active+clean; 192 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 464 KiB/s rd, 4.5 MiB/s wr, 35 op/s 2024-09-14T04:27:06.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:06 smithi129 ceph-mon[101316]: pgmap v1501: 105 pgs: 105 active+clean; 192 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-09-14T04:27:06.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:06 smithi039 ceph-mon[111235]: pgmap v1501: 105 pgs: 105 active+clean; 192 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-09-14T04:27:06.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:06 smithi039 ceph-mon[106470]: pgmap v1501: 105 pgs: 105 active+clean; 192 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 55 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-09-14T04:27:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:27:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:27:07.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:27:08.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:08 smithi129 ceph-mon[101316]: pgmap v1502: 105 pgs: 105 active+clean; 192 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 4.5 MiB/s wr, 38 op/s 2024-09-14T04:27:08.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:08 smithi039 ceph-mon[106470]: pgmap v1502: 105 pgs: 105 active+clean; 192 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 4.5 MiB/s wr, 38 op/s 2024-09-14T04:27:08.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:27:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:27:08] "GET /metrics HTTP/1.1" 200 36687 "" "Prometheus/2.51.0" 2024-09-14T04:27:08.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:08 smithi039 ceph-mon[111235]: pgmap v1502: 105 pgs: 105 active+clean; 192 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 57 KiB/s rd, 4.5 MiB/s wr, 38 op/s 2024-09-14T04:27:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:09 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "50.11", "id": [6, 1]} : dispatch 2024-09-14T04:27:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:09 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "50.16", "id": [5, 3]} : dispatch 2024-09-14T04:27:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:09 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "50.11", "id": [6, 1]} : dispatch 2024-09-14T04:27:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:09 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "50.16", "id": [5, 3]} : dispatch 2024-09-14T04:27:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:09 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "50.11", "id": [6, 1]} : dispatch 2024-09-14T04:27:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:09 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "50.16", "id": [5, 3]} : dispatch 2024-09-14T04:27:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:09 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "50.11", "id": [6, 1]} : dispatch 2024-09-14T04:27:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:09 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "50.16", "id": [5, 3]} : dispatch 2024-09-14T04:27:09.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:09 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "50.11", "id": [6, 1]} : dispatch 2024-09-14T04:27:09.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:09 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "50.16", "id": [5, 3]} : dispatch 2024-09-14T04:27:09.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:09 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "50.11", "id": [6, 1]} : dispatch 2024-09-14T04:27:09.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:09 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "50.16", "id": [5, 3]} : dispatch 2024-09-14T04:27:10.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:10 smithi129 ceph-mon[101316]: pgmap v1503: 105 pgs: 105 active+clean; 196 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 212 KiB/s rd, 4.1 MiB/s wr, 42 op/s 2024-09-14T04:27:10.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:10 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "50.11", "id": [6, 1]}]': finished 2024-09-14T04:27:10.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:10 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "50.16", "id": [5, 3]}]': finished 2024-09-14T04:27:10.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:10 smithi129 ceph-mon[101316]: osdmap e1008: 8 total, 8 up, 8 in 2024-09-14T04:27:10.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:10 smithi039 ceph-mon[106470]: pgmap v1503: 105 pgs: 105 active+clean; 196 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 212 KiB/s rd, 4.1 MiB/s wr, 42 op/s 2024-09-14T04:27:10.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:10 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "50.11", "id": [6, 1]}]': finished 2024-09-14T04:27:10.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:10 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "50.16", "id": [5, 3]}]': finished 2024-09-14T04:27:10.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:10 smithi039 ceph-mon[106470]: osdmap e1008: 8 total, 8 up, 8 in 2024-09-14T04:27:10.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:10 smithi039 ceph-mon[111235]: pgmap v1503: 105 pgs: 105 active+clean; 196 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 212 KiB/s rd, 4.1 MiB/s wr, 42 op/s 2024-09-14T04:27:10.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:10 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "50.11", "id": [6, 1]}]': finished 2024-09-14T04:27:10.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:10 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "50.16", "id": [5, 3]}]': finished 2024-09-14T04:27:10.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:10 smithi039 ceph-mon[111235]: osdmap e1008: 8 total, 8 up, 8 in 2024-09-14T04:27:11.092 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/7.Shrink (19664 ms) 2024-09-14T04:27:11.092 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/7.ShrinkToOne 2024-09-14T04:27:11.093 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:27:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:11 smithi129 ceph-mon[101316]: osdmap e1009: 8 total, 8 up, 8 in 2024-09-14T04:27:11.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:11 smithi039 ceph-mon[106470]: osdmap e1009: 8 total, 8 up, 8 in 2024-09-14T04:27:11.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:11 smithi039 ceph-mon[111235]: osdmap e1009: 8 total, 8 up, 8 in 2024-09-14T04:27:12.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:12 smithi129 ceph-mon[101316]: pgmap v1506: 105 pgs: 1 peering, 104 active+clean; 200 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.0 MiB/s wr, 21 op/s 2024-09-14T04:27:12.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:12 smithi039 ceph-mon[106470]: pgmap v1506: 105 pgs: 1 peering, 104 active+clean; 200 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.0 MiB/s wr, 21 op/s 2024-09-14T04:27:12.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:12 smithi039 ceph-mon[111235]: pgmap v1506: 105 pgs: 1 peering, 104 active+clean; 200 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.0 MiB/s wr, 21 op/s 2024-09-14T04:27:14.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:14 smithi039 ceph-mon[106470]: pgmap v1507: 105 pgs: 1 peering, 104 active+clean; 220 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 4.0 MiB/s wr, 37 op/s 2024-09-14T04:27:14.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:14 smithi039 ceph-mon[111235]: pgmap v1507: 105 pgs: 1 peering, 104 active+clean; 220 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 4.0 MiB/s wr, 37 op/s 2024-09-14T04:27:15.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:14 smithi129 ceph-mon[101316]: pgmap v1507: 105 pgs: 1 peering, 104 active+clean; 220 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 13 MiB/s rd, 4.0 MiB/s wr, 37 op/s 2024-09-14T04:27:16.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:16 smithi039 ceph-mon[106470]: pgmap v1508: 105 pgs: 105 active+clean; 220 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 4.0 MiB/s wr, 37 op/s; 1024 KiB/s, 0 objects/s recovering 2024-09-14T04:27:16.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:16 smithi039 ceph-mon[111235]: pgmap v1508: 105 pgs: 105 active+clean; 220 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 4.0 MiB/s wr, 37 op/s; 1024 KiB/s, 0 objects/s recovering 2024-09-14T04:27:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:16 smithi129 ceph-mon[101316]: pgmap v1508: 105 pgs: 105 active+clean; 220 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 4.0 MiB/s wr, 37 op/s; 1024 KiB/s, 0 objects/s recovering 2024-09-14T04:27:18.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:18 smithi039 ceph-mon[106470]: pgmap v1509: 105 pgs: 105 active+clean; 220 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.5 MiB/s wr, 22 op/s; 1024 KiB/s, 0 objects/s recovering 2024-09-14T04:27:18.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:18 smithi039 ceph-mon[111235]: pgmap v1509: 105 pgs: 105 active+clean; 220 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.5 MiB/s wr, 22 op/s; 1024 KiB/s, 0 objects/s recovering 2024-09-14T04:27:18.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:27:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:27:18] "GET /metrics HTTP/1.1" 200 36687 "" "Prometheus/2.51.0" 2024-09-14T04:27:19.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:18 smithi129 ceph-mon[101316]: pgmap v1509: 105 pgs: 105 active+clean; 220 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 12 MiB/s rd, 3.5 MiB/s wr, 22 op/s; 1024 KiB/s, 0 objects/s recovering 2024-09-14T04:27:20.930 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:20 smithi129 ceph-mon[101316]: pgmap v1510: 105 pgs: 105 active+clean; 220 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 2.8 MiB/s wr, 20 op/s; 833 KiB/s, 0 objects/s recovering 2024-09-14T04:27:20.930 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:20 smithi129 ceph-mon[101316]: osdmap e1010: 8 total, 8 up, 8 in 2024-09-14T04:27:20.930 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:20 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:27:21.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:20 smithi039 ceph-mon[106470]: pgmap v1510: 105 pgs: 105 active+clean; 220 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 2.8 MiB/s wr, 20 op/s; 833 KiB/s, 0 objects/s recovering 2024-09-14T04:27:21.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:20 smithi039 ceph-mon[106470]: osdmap e1010: 8 total, 8 up, 8 in 2024-09-14T04:27:21.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:20 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:27:21.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:20 smithi039 ceph-mon[111235]: pgmap v1510: 105 pgs: 105 active+clean; 220 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 9.7 MiB/s rd, 2.8 MiB/s wr, 20 op/s; 833 KiB/s, 0 objects/s recovering 2024-09-14T04:27:21.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:20 smithi039 ceph-mon[111235]: osdmap e1010: 8 total, 8 up, 8 in 2024-09-14T04:27:21.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:20 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:27:22.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:22 smithi129 ceph-mon[101316]: pgmap v1512: 105 pgs: 105 active+clean; 228 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 3.2 MiB/s wr, 19 op/s; 819 KiB/s, 0 objects/s recovering 2024-09-14T04:27:22.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:27:22.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:22 smithi039 ceph-mon[106470]: pgmap v1512: 105 pgs: 105 active+clean; 228 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 3.2 MiB/s wr, 19 op/s; 819 KiB/s, 0 objects/s recovering 2024-09-14T04:27:22.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:27:22.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:22 smithi039 ceph-mon[111235]: pgmap v1512: 105 pgs: 105 active+clean; 228 MiB data, 1.4 GiB used, 714 GiB / 715 GiB avail; 8.3 MiB/s rd, 3.2 MiB/s wr, 19 op/s; 819 KiB/s, 0 objects/s recovering 2024-09-14T04:27:22.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:27:25.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:24 smithi129 ceph-mon[101316]: pgmap v1513: 105 pgs: 105 active+clean; 261 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s rd, 4.1 MiB/s wr, 23 op/s; 819 KiB/s, 0 objects/s recovering 2024-09-14T04:27:25.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:24 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:27:25.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:24 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:27:25.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:24 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:27:25.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:24 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:27:25.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:24 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:27:25.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:24 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:27:25.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:24 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:27:25.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:24 smithi039 ceph-mon[111235]: pgmap v1513: 105 pgs: 105 active+clean; 261 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s rd, 4.1 MiB/s wr, 23 op/s; 819 KiB/s, 0 objects/s recovering 2024-09-14T04:27:25.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:24 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:27:25.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:24 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:27:25.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:24 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:27:25.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:24 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:27:25.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:24 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:27:25.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:24 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:27:25.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:24 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:27:25.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:24 smithi039 ceph-mon[106470]: pgmap v1513: 105 pgs: 105 active+clean; 261 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 0 B/s rd, 4.1 MiB/s wr, 23 op/s; 819 KiB/s, 0 objects/s recovering 2024-09-14T04:27:25.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:24 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:27:25.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:24 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:27:25.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:24 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:27:25.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:24 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:27:25.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:24 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:27:25.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:24 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:27:25.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:24 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:27:27.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:26 smithi129 ceph-mon[101316]: pgmap v1514: 105 pgs: 105 active+clean; 261 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-09-14T04:27:27.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:26 smithi039 ceph-mon[106470]: pgmap v1514: 105 pgs: 105 active+clean; 261 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-09-14T04:27:27.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:26 smithi039 ceph-mon[111235]: pgmap v1514: 105 pgs: 105 active+clean; 261 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-09-14T04:27:29.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:28 smithi129 ceph-mon[101316]: pgmap v1515: 105 pgs: 105 active+clean; 261 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-09-14T04:27:29.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:28 smithi039 ceph-mon[106470]: pgmap v1515: 105 pgs: 105 active+clean; 261 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-09-14T04:27:29.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:28 smithi039 ceph-mon[111235]: pgmap v1515: 105 pgs: 105 active+clean; 261 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 4.1 MiB/s wr, 36 op/s 2024-09-14T04:27:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:27:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:27:28] "GET /metrics HTTP/1.1" 200 36701 "" "Prometheus/2.51.0" 2024-09-14T04:27:31.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:30 smithi129 ceph-mon[101316]: pgmap v1516: 105 pgs: 105 active+clean; 269 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 191 KiB/s rd, 4.9 MiB/s wr, 43 op/s 2024-09-14T04:27:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:30 smithi039 ceph-mon[106470]: pgmap v1516: 105 pgs: 105 active+clean; 269 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 191 KiB/s rd, 4.9 MiB/s wr, 43 op/s 2024-09-14T04:27:31.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:30 smithi039 ceph-mon[111235]: pgmap v1516: 105 pgs: 105 active+clean; 269 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 191 KiB/s rd, 4.9 MiB/s wr, 43 op/s 2024-09-14T04:27:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:32 smithi129 ceph-mon[101316]: pgmap v1517: 105 pgs: 105 active+clean; 269 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 212 KiB/s rd, 4.2 MiB/s wr, 38 op/s 2024-09-14T04:27:33.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:32 smithi039 ceph-mon[106470]: pgmap v1517: 105 pgs: 105 active+clean; 269 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 212 KiB/s rd, 4.2 MiB/s wr, 38 op/s 2024-09-14T04:27:33.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:32 smithi039 ceph-mon[111235]: pgmap v1517: 105 pgs: 105 active+clean; 269 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 212 KiB/s rd, 4.2 MiB/s wr, 38 op/s 2024-09-14T04:27:33.296 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/7.ShrinkToOne (22204 ms) 2024-09-14T04:27:33.296 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/7.ShrinkToOneAfterSnapshot 2024-09-14T04:27:33.296 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:27:35.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:34 smithi129 ceph-mon[101316]: pgmap v1518: 105 pgs: 105 active+clean; 269 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 920 KiB/s rd, 4.4 MiB/s wr, 44 op/s 2024-09-14T04:27:35.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:34 smithi039 ceph-mon[106470]: pgmap v1518: 105 pgs: 105 active+clean; 269 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 920 KiB/s rd, 4.4 MiB/s wr, 44 op/s 2024-09-14T04:27:35.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:34 smithi039 ceph-mon[111235]: pgmap v1518: 105 pgs: 105 active+clean; 269 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 920 KiB/s rd, 4.4 MiB/s wr, 44 op/s 2024-09-14T04:27:37.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:36 smithi129 ceph-mon[101316]: pgmap v1519: 105 pgs: 105 active+clean; 269 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 911 KiB/s rd, 1.7 MiB/s wr, 31 op/s 2024-09-14T04:27:37.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:36 smithi039 ceph-mon[106470]: pgmap v1519: 105 pgs: 105 active+clean; 269 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 911 KiB/s rd, 1.7 MiB/s wr, 31 op/s 2024-09-14T04:27:37.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:36 smithi039 ceph-mon[111235]: pgmap v1519: 105 pgs: 105 active+clean; 269 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 911 KiB/s rd, 1.7 MiB/s wr, 31 op/s 2024-09-14T04:27:37.941 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:27:37.942 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:27:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:27:38.958 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:38 smithi039 ceph-mon[106470]: pgmap v1520: 105 pgs: 105 active+clean; 269 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 901 KiB/s rd, 1.7 MiB/s wr, 18 op/s 2024-09-14T04:27:38.958 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:27:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:27:38] "GET /metrics HTTP/1.1" 200 36690 "" "Prometheus/2.51.0" 2024-09-14T04:27:38.959 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:38 smithi039 ceph-mon[111235]: pgmap v1520: 105 pgs: 105 active+clean; 269 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 901 KiB/s rd, 1.7 MiB/s wr, 18 op/s 2024-09-14T04:27:39.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:38 smithi129 ceph-mon[101316]: pgmap v1520: 105 pgs: 105 active+clean; 269 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 901 KiB/s rd, 1.7 MiB/s wr, 18 op/s 2024-09-14T04:27:40.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:40 smithi039 ceph-mon[106470]: pgmap v1521: 105 pgs: 105 active+clean; 269 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 903 KiB/s rd, 1.7 MiB/s wr, 23 op/s 2024-09-14T04:27:40.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:40 smithi039 ceph-mon[111235]: pgmap v1521: 105 pgs: 105 active+clean; 269 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 903 KiB/s rd, 1.7 MiB/s wr, 23 op/s 2024-09-14T04:27:41.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:40 smithi129 ceph-mon[101316]: pgmap v1521: 105 pgs: 105 active+clean; 269 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 903 KiB/s rd, 1.7 MiB/s wr, 23 op/s 2024-09-14T04:27:43.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:42 smithi129 ceph-mon[101316]: pgmap v1522: 105 pgs: 105 active+clean; 269 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 765 KiB/s rd, 1.0 MiB/s wr, 16 op/s 2024-09-14T04:27:43.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:42 smithi129 ceph-mon[101316]: osdmap e1011: 8 total, 8 up, 8 in 2024-09-14T04:27:43.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:42 smithi039 ceph-mon[106470]: pgmap v1522: 105 pgs: 105 active+clean; 269 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 765 KiB/s rd, 1.0 MiB/s wr, 16 op/s 2024-09-14T04:27:43.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:42 smithi039 ceph-mon[106470]: osdmap e1011: 8 total, 8 up, 8 in 2024-09-14T04:27:43.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:42 smithi039 ceph-mon[111235]: pgmap v1522: 105 pgs: 105 active+clean; 269 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 765 KiB/s rd, 1.0 MiB/s wr, 16 op/s 2024-09-14T04:27:43.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:42 smithi039 ceph-mon[111235]: osdmap e1011: 8 total, 8 up, 8 in 2024-09-14T04:27:45.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:44 smithi129 ceph-mon[101316]: pgmap v1524: 105 pgs: 105 active+clean; 302 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 3.3 MiB/s wr, 42 op/s 2024-09-14T04:27:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:44 smithi039 ceph-mon[106470]: pgmap v1524: 105 pgs: 105 active+clean; 302 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 3.3 MiB/s wr, 42 op/s 2024-09-14T04:27:45.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:44 smithi039 ceph-mon[111235]: pgmap v1524: 105 pgs: 105 active+clean; 302 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 3.3 MiB/s wr, 42 op/s 2024-09-14T04:27:47.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:46 smithi129 ceph-mon[101316]: pgmap v1525: 105 pgs: 105 active+clean; 310 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 4.1 MiB/s wr, 39 op/s 2024-09-14T04:27:47.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:46 smithi039 ceph-mon[106470]: pgmap v1525: 105 pgs: 105 active+clean; 310 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 4.1 MiB/s wr, 39 op/s 2024-09-14T04:27:47.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:46 smithi039 ceph-mon[111235]: pgmap v1525: 105 pgs: 105 active+clean; 310 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 4.1 MiB/s wr, 39 op/s 2024-09-14T04:27:49.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:48 smithi129 ceph-mon[101316]: pgmap v1526: 105 pgs: 105 active+clean; 310 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 4.1 MiB/s wr, 41 op/s 2024-09-14T04:27:49.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:48 smithi039 ceph-mon[106470]: pgmap v1526: 105 pgs: 105 active+clean; 310 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 4.1 MiB/s wr, 41 op/s 2024-09-14T04:27:49.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:27:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:27:48] "GET /metrics HTTP/1.1" 200 36690 "" "Prometheus/2.51.0" 2024-09-14T04:27:49.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:48 smithi039 ceph-mon[111235]: pgmap v1526: 105 pgs: 105 active+clean; 310 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 4.1 MiB/s wr, 41 op/s 2024-09-14T04:27:50.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:50 smithi039 ceph-mon[106470]: pgmap v1527: 105 pgs: 105 active+clean; 314 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 84 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-09-14T04:27:50.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:50 smithi039 ceph-mon[111235]: pgmap v1527: 105 pgs: 105 active+clean; 314 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 84 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-09-14T04:27:51.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:50 smithi129 ceph-mon[101316]: pgmap v1527: 105 pgs: 105 active+clean; 314 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 84 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-09-14T04:27:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:52 smithi129 ceph-mon[101316]: pgmap v1528: 105 pgs: 105 active+clean; 314 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 84 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-09-14T04:27:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:27:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:52 smithi039 ceph-mon[106470]: pgmap v1528: 105 pgs: 105 active+clean; 314 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 84 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-09-14T04:27:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:27:53.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:52 smithi039 ceph-mon[111235]: pgmap v1528: 105 pgs: 105 active+clean; 314 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 84 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-09-14T04:27:53.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:27:55.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:54 smithi129 ceph-mon[101316]: pgmap v1529: 105 pgs: 105 active+clean; 318 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 214 KiB/s rd, 4.2 MiB/s wr, 39 op/s 2024-09-14T04:27:55.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:54 smithi129 ceph-mon[101316]: osdmap e1012: 8 total, 8 up, 8 in 2024-09-14T04:27:55.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:54 smithi039 ceph-mon[106470]: pgmap v1529: 105 pgs: 105 active+clean; 318 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 214 KiB/s rd, 4.2 MiB/s wr, 39 op/s 2024-09-14T04:27:55.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:54 smithi039 ceph-mon[106470]: osdmap e1012: 8 total, 8 up, 8 in 2024-09-14T04:27:55.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:54 smithi039 ceph-mon[111235]: pgmap v1529: 105 pgs: 105 active+clean; 318 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 214 KiB/s rd, 4.2 MiB/s wr, 39 op/s 2024-09-14T04:27:55.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:54 smithi039 ceph-mon[111235]: osdmap e1012: 8 total, 8 up, 8 in 2024-09-14T04:27:55.956 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/7.ShrinkToOneAfterSnapshot (22659 ms) 2024-09-14T04:27:55.956 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/7.MinOverlap 2024-09-14T04:27:55.956 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:27:57.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:56 smithi129 ceph-mon[101316]: pgmap v1531: 105 pgs: 105 active+clean; 326 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 634 KiB/s rd, 1.6 MiB/s wr, 12 op/s 2024-09-14T04:27:57.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:56 smithi039 ceph-mon[106470]: pgmap v1531: 105 pgs: 105 active+clean; 326 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 634 KiB/s rd, 1.6 MiB/s wr, 12 op/s 2024-09-14T04:27:57.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:56 smithi039 ceph-mon[111235]: pgmap v1531: 105 pgs: 105 active+clean; 326 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 634 KiB/s rd, 1.6 MiB/s wr, 12 op/s 2024-09-14T04:27:59.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:27:58 smithi129 ceph-mon[101316]: pgmap v1532: 105 pgs: 105 active+clean; 326 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 639 KiB/s rd, 1.6 MiB/s wr, 18 op/s 2024-09-14T04:27:59.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:27:58 smithi039 ceph-mon[106470]: pgmap v1532: 105 pgs: 105 active+clean; 326 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 639 KiB/s rd, 1.6 MiB/s wr, 18 op/s 2024-09-14T04:27:59.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:27:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:27:58] "GET /metrics HTTP/1.1" 200 36689 "" "Prometheus/2.51.0" 2024-09-14T04:27:59.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:27:58 smithi039 ceph-mon[111235]: pgmap v1532: 105 pgs: 105 active+clean; 326 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 639 KiB/s rd, 1.6 MiB/s wr, 18 op/s 2024-09-14T04:28:01.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:00 smithi129 ceph-mon[101316]: pgmap v1533: 105 pgs: 105 active+clean; 374 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 6.0 MiB/s wr, 43 op/s 2024-09-14T04:28:01.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:00 smithi039 ceph-mon[106470]: pgmap v1533: 105 pgs: 105 active+clean; 374 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 6.0 MiB/s wr, 43 op/s 2024-09-14T04:28:01.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:00 smithi039 ceph-mon[111235]: pgmap v1533: 105 pgs: 105 active+clean; 374 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 6.0 MiB/s wr, 43 op/s 2024-09-14T04:28:03.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:02 smithi129 ceph-mon[101316]: pgmap v1534: 105 pgs: 105 active+clean; 374 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 6.0 MiB/s wr, 44 op/s 2024-09-14T04:28:03.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:02 smithi039 ceph-mon[106470]: pgmap v1534: 105 pgs: 105 active+clean; 374 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 6.0 MiB/s wr, 44 op/s 2024-09-14T04:28:03.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:02 smithi039 ceph-mon[111235]: pgmap v1534: 105 pgs: 105 active+clean; 374 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3.0 MiB/s rd, 6.0 MiB/s wr, 44 op/s 2024-09-14T04:28:05.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:04 smithi129 ceph-mon[101316]: pgmap v1535: 105 pgs: 105 active+clean; 374 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 5.6 MiB/s wr, 35 op/s 2024-09-14T04:28:05.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:04 smithi129 ceph-mon[101316]: osdmap e1013: 8 total, 8 up, 8 in 2024-09-14T04:28:05.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:04 smithi039 ceph-mon[106470]: pgmap v1535: 105 pgs: 105 active+clean; 374 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 5.6 MiB/s wr, 35 op/s 2024-09-14T04:28:05.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:04 smithi039 ceph-mon[106470]: osdmap e1013: 8 total, 8 up, 8 in 2024-09-14T04:28:05.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:04 smithi039 ceph-mon[111235]: pgmap v1535: 105 pgs: 105 active+clean; 374 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.8 MiB/s rd, 5.6 MiB/s wr, 35 op/s 2024-09-14T04:28:05.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:04 smithi039 ceph-mon[111235]: osdmap e1013: 8 total, 8 up, 8 in 2024-09-14T04:28:07.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:06 smithi129 ceph-mon[101316]: pgmap v1537: 105 pgs: 105 active+clean; 390 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 6.4 MiB/s wr, 45 op/s 2024-09-14T04:28:07.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:06 smithi039 ceph-mon[106470]: pgmap v1537: 105 pgs: 105 active+clean; 390 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 6.4 MiB/s wr, 45 op/s 2024-09-14T04:28:07.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:06 smithi039 ceph-mon[111235]: pgmap v1537: 105 pgs: 105 active+clean; 390 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2.5 MiB/s rd, 6.4 MiB/s wr, 45 op/s 2024-09-14T04:28:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:28:08.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:28:08.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:28:09.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:08 smithi129 ceph-mon[101316]: pgmap v1538: 105 pgs: 105 active+clean; 394 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 6.8 MiB/s wr, 37 op/s 2024-09-14T04:28:09.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:08 smithi039 ceph-mon[106470]: pgmap v1538: 105 pgs: 105 active+clean; 394 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 6.8 MiB/s wr, 37 op/s 2024-09-14T04:28:09.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:28:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:28:08] "GET /metrics HTTP/1.1" 200 36694 "" "Prometheus/2.51.0" 2024-09-14T04:28:09.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:08 smithi039 ceph-mon[111235]: pgmap v1538: 105 pgs: 105 active+clean; 394 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 2.4 MiB/s rd, 6.8 MiB/s wr, 37 op/s 2024-09-14T04:28:11.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:10 smithi129 ceph-mon[101316]: pgmap v1539: 105 pgs: 105 active+clean; 419 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 4.5 MiB/s wr, 36 op/s 2024-09-14T04:28:11.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:10 smithi039 ceph-mon[106470]: pgmap v1539: 105 pgs: 105 active+clean; 419 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 4.5 MiB/s wr, 36 op/s 2024-09-14T04:28:11.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:10 smithi039 ceph-mon[111235]: pgmap v1539: 105 pgs: 105 active+clean; 419 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 4.5 MiB/s wr, 36 op/s 2024-09-14T04:28:13.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:12 smithi129 ceph-mon[101316]: pgmap v1540: 105 pgs: 105 active+clean; 419 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 4.5 MiB/s wr, 35 op/s 2024-09-14T04:28:13.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:12 smithi039 ceph-mon[106470]: pgmap v1540: 105 pgs: 105 active+clean; 419 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 4.5 MiB/s wr, 35 op/s 2024-09-14T04:28:13.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:12 smithi039 ceph-mon[111235]: pgmap v1540: 105 pgs: 105 active+clean; 419 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 55 KiB/s rd, 4.5 MiB/s wr, 35 op/s 2024-09-14T04:28:15.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:14 smithi039 ceph-mon[106470]: pgmap v1541: 105 pgs: 105 active+clean; 423 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 193 KiB/s rd, 4.9 MiB/s wr, 45 op/s 2024-09-14T04:28:15.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:14 smithi039 ceph-mon[111235]: pgmap v1541: 105 pgs: 105 active+clean; 423 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 193 KiB/s rd, 4.9 MiB/s wr, 45 op/s 2024-09-14T04:28:15.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:14 smithi129 ceph-mon[101316]: pgmap v1541: 105 pgs: 105 active+clean; 423 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 193 KiB/s rd, 4.9 MiB/s wr, 45 op/s 2024-09-14T04:28:17.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:16 smithi039 ceph-mon[111235]: pgmap v1542: 105 pgs: 105 active+clean; 423 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 191 KiB/s rd, 4.2 MiB/s wr, 39 op/s 2024-09-14T04:28:17.040 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:16 smithi129 ceph-mon[101316]: pgmap v1542: 105 pgs: 105 active+clean; 423 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 191 KiB/s rd, 4.2 MiB/s wr, 39 op/s 2024-09-14T04:28:17.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:16 smithi039 ceph-mon[106470]: pgmap v1542: 105 pgs: 105 active+clean; 423 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 191 KiB/s rd, 4.2 MiB/s wr, 39 op/s 2024-09-14T04:28:17.041 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 04:28:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=cleanup t=2024-09-14T04:28:16.891386305Z level=info msg="Completed cleanup jobs" duration=4.490921ms 2024-09-14T04:28:17.041 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 04:28:17 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=grafana.update.checker t=2024-09-14T04:28:17.039289705Z level=info msg="Update check succeeded" duration=67.283857ms 2024-09-14T04:28:17.521 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 04:28:17 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=plugins.update.checker t=2024-09-14T04:28:17.136720441Z level=info msg="Update check succeeded" duration=68.487872ms 2024-09-14T04:28:17.716 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/7.MinOverlap (21761 ms) 2024-09-14T04:28:17.716 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] EncryptedFlattenTest/7.ZeroOverlap 2024-09-14T04:28:17.716 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:28:19.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:18 smithi039 ceph-mon[106470]: pgmap v1543: 105 pgs: 105 active+clean; 419 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 206 KiB/s rd, 2.8 MiB/s wr, 33 op/s 2024-09-14T04:28:19.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:28:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:28:18] "GET /metrics HTTP/1.1" 200 36694 "" "Prometheus/2.51.0" 2024-09-14T04:28:19.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:18 smithi039 ceph-mon[111235]: pgmap v1543: 105 pgs: 105 active+clean; 419 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 206 KiB/s rd, 2.8 MiB/s wr, 33 op/s 2024-09-14T04:28:19.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:18 smithi129 ceph-mon[101316]: pgmap v1543: 105 pgs: 105 active+clean; 419 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 206 KiB/s rd, 2.8 MiB/s wr, 33 op/s 2024-09-14T04:28:21.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:20 smithi129 ceph-mon[101316]: pgmap v1544: 105 pgs: 105 active+clean; 423 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 3.4 MiB/s wr, 42 op/s 2024-09-14T04:28:21.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:20 smithi039 ceph-mon[106470]: pgmap v1544: 105 pgs: 105 active+clean; 423 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 3.4 MiB/s wr, 42 op/s 2024-09-14T04:28:21.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:20 smithi039 ceph-mon[111235]: pgmap v1544: 105 pgs: 105 active+clean; 423 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 3.4 MiB/s wr, 42 op/s 2024-09-14T04:28:23.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:22 smithi129 ceph-mon[101316]: pgmap v1545: 105 pgs: 105 active+clean; 423 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.3 MiB/s wr, 21 op/s 2024-09-14T04:28:23.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:28:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:22 smithi039 ceph-mon[106470]: pgmap v1545: 105 pgs: 105 active+clean; 423 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.3 MiB/s wr, 21 op/s 2024-09-14T04:28:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:28:23.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:22 smithi039 ceph-mon[111235]: pgmap v1545: 105 pgs: 105 active+clean; 423 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.3 MiB/s wr, 21 op/s 2024-09-14T04:28:23.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:28:25.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:24 smithi129 ceph-mon[101316]: pgmap v1546: 105 pgs: 105 active+clean; 423 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.3 MiB/s wr, 25 op/s 2024-09-14T04:28:25.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:24 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:28:25.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:24 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:28:25.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:24 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:28:25.120 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:24 smithi039 ceph-mon[106470]: pgmap v1546: 105 pgs: 105 active+clean; 423 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.3 MiB/s wr, 25 op/s 2024-09-14T04:28:25.120 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:24 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:28:25.120 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:24 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:28:25.121 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:24 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:28:25.121 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:24 smithi039 ceph-mon[111235]: pgmap v1546: 105 pgs: 105 active+clean; 423 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.9 MiB/s rd, 1.3 MiB/s wr, 25 op/s 2024-09-14T04:28:25.121 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:24 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:28:25.121 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:24 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:28:25.121 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:24 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:28:27.136 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:26 smithi039 ceph-mon[106470]: pgmap v1547: 105 pgs: 105 active+clean; 423 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.0 MiB/s wr, 17 op/s 2024-09-14T04:28:27.136 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:26 smithi039 ceph-mon[106470]: osdmap e1014: 8 total, 8 up, 8 in 2024-09-14T04:28:27.136 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:26 smithi039 ceph-mon[111235]: pgmap v1547: 105 pgs: 105 active+clean; 423 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.0 MiB/s wr, 17 op/s 2024-09-14T04:28:27.137 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:26 smithi039 ceph-mon[111235]: osdmap e1014: 8 total, 8 up, 8 in 2024-09-14T04:28:27.222 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:26 smithi129 ceph-mon[101316]: pgmap v1547: 105 pgs: 105 active+clean; 423 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 1.7 MiB/s rd, 1.0 MiB/s wr, 17 op/s 2024-09-14T04:28:27.223 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:26 smithi129 ceph-mon[101316]: osdmap e1014: 8 total, 8 up, 8 in 2024-09-14T04:28:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:28:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:28:28] "GET /metrics HTTP/1.1" 200 36696 "" "Prometheus/2.51.0" 2024-09-14T04:28:29.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:28 smithi039 ceph-mon[106470]: pgmap v1549: 105 pgs: 105 active+clean; 423 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.2 MiB/s wr, 42 op/s 2024-09-14T04:28:29.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:28 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:28:29.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:28 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:28:29.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:28 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:28:29.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:28 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:28:29.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:28 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:28:29.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:28 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:28:29.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:28 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:28:29.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:28 smithi039 ceph-mon[111235]: pgmap v1549: 105 pgs: 105 active+clean; 423 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.2 MiB/s wr, 42 op/s 2024-09-14T04:28:29.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:28 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:28:29.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:28 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:28:29.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:28 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:28:29.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:28 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:28:29.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:28 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:28:29.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:28 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:28:29.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:28 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:28:29.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:28 smithi129 ceph-mon[101316]: pgmap v1549: 105 pgs: 105 active+clean; 423 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 2.1 MiB/s rd, 1.2 MiB/s wr, 42 op/s 2024-09-14T04:28:29.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:28 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:28:29.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:28 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:28:29.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:28 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:28:29.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:28 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:28:29.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:28 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:28:29.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:28 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:28:29.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:28 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:28:31.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:30 smithi129 ceph-mon[101316]: pgmap v1550: 105 pgs: 105 active+clean; 464 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 4.1 MiB/s wr, 34 op/s 2024-09-14T04:28:31.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:30 smithi039 ceph-mon[106470]: pgmap v1550: 105 pgs: 105 active+clean; 464 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 4.1 MiB/s wr, 34 op/s 2024-09-14T04:28:31.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:30 smithi039 ceph-mon[111235]: pgmap v1550: 105 pgs: 105 active+clean; 464 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 24 KiB/s rd, 4.1 MiB/s wr, 34 op/s 2024-09-14T04:28:33.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:32 smithi129 ceph-mon[101316]: pgmap v1551: 105 pgs: 105 active+clean; 464 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 4.1 MiB/s wr, 40 op/s 2024-09-14T04:28:33.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:32 smithi039 ceph-mon[106470]: pgmap v1551: 105 pgs: 105 active+clean; 464 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 4.1 MiB/s wr, 40 op/s 2024-09-14T04:28:33.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:32 smithi039 ceph-mon[111235]: pgmap v1551: 105 pgs: 105 active+clean; 464 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 4.1 MiB/s wr, 40 op/s 2024-09-14T04:28:35.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:34 smithi129 ceph-mon[101316]: pgmap v1552: 105 pgs: 105 active+clean; 468 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 84 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-09-14T04:28:35.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:34 smithi039 ceph-mon[106470]: pgmap v1552: 105 pgs: 105 active+clean; 468 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 84 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-09-14T04:28:35.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:34 smithi039 ceph-mon[111235]: pgmap v1552: 105 pgs: 105 active+clean; 468 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 84 KiB/s rd, 4.5 MiB/s wr, 37 op/s 2024-09-14T04:28:37.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:36 smithi129 ceph-mon[101316]: pgmap v1553: 105 pgs: 105 active+clean; 468 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 86 KiB/s rd, 4.5 MiB/s wr, 40 op/s 2024-09-14T04:28:37.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:36 smithi039 ceph-mon[106470]: pgmap v1553: 105 pgs: 105 active+clean; 468 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 86 KiB/s rd, 4.5 MiB/s wr, 40 op/s 2024-09-14T04:28:37.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:36 smithi039 ceph-mon[111235]: pgmap v1553: 105 pgs: 105 active+clean; 468 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 86 KiB/s rd, 4.5 MiB/s wr, 40 op/s 2024-09-14T04:28:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:28:38.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:28:38.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:28:39.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:38 smithi039 ceph-mon[106470]: pgmap v1554: 105 pgs: 105 active+clean; 468 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 3.9 MiB/s wr, 38 op/s 2024-09-14T04:28:39.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:28:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:28:38] "GET /metrics HTTP/1.1" 200 36695 "" "Prometheus/2.51.0" 2024-09-14T04:28:39.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:38 smithi039 ceph-mon[111235]: pgmap v1554: 105 pgs: 105 active+clean; 468 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 3.9 MiB/s wr, 38 op/s 2024-09-14T04:28:39.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:38 smithi129 ceph-mon[101316]: pgmap v1554: 105 pgs: 105 active+clean; 468 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 78 KiB/s rd, 3.9 MiB/s wr, 38 op/s 2024-09-14T04:28:39.554 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] EncryptedFlattenTest/7.ZeroOverlap (21838 ms) 2024-09-14T04:28:39.907 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 7 tests from EncryptedFlattenTest/7 (156315 ms total) 2024-09-14T04:28:39.907 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:28:39.907 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 19 tests from DiffIterateTest/0, where TypeParam = DiffIterateParams 2024-09-14T04:28:39.936 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/0.DiffIterate 2024-09-14T04:28:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:40 smithi129 ceph-mon[101316]: pgmap v1555: 105 pgs: 105 active+clean; 468 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 560 KiB/s rd, 4.4 MiB/s wr, 20 op/s 2024-09-14T04:28:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:40 smithi129 ceph-mon[101316]: osdmap e1015: 8 total, 8 up, 8 in 2024-09-14T04:28:41.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:40 smithi039 ceph-mon[106470]: pgmap v1555: 105 pgs: 105 active+clean; 468 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 560 KiB/s rd, 4.4 MiB/s wr, 20 op/s 2024-09-14T04:28:41.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:40 smithi039 ceph-mon[106470]: osdmap e1015: 8 total, 8 up, 8 in 2024-09-14T04:28:41.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:40 smithi039 ceph-mon[111235]: pgmap v1555: 105 pgs: 105 active+clean; 468 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 560 KiB/s rd, 4.4 MiB/s wr, 20 op/s 2024-09-14T04:28:41.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:40 smithi039 ceph-mon[111235]: osdmap e1015: 8 total, 8 up, 8 in 2024-09-14T04:28:41.936 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:28:42.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:41 smithi129 ceph-mon[101316]: osdmap e1016: 8 total, 8 up, 8 in 2024-09-14T04:28:42.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:41 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1266330879' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-47","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:28:42.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:41 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-47","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:28:42.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:41 smithi039 ceph-mon[106470]: osdmap e1016: 8 total, 8 up, 8 in 2024-09-14T04:28:42.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:41 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1266330879' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-47","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:28:42.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:41 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-47","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:28:42.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:41 smithi039 ceph-mon[111235]: osdmap e1016: 8 total, 8 up, 8 in 2024-09-14T04:28:42.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:41 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1266330879' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-47","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:28:42.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:41 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-47","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:28:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:42 smithi129 ceph-mon[101316]: pgmap v1558: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:28:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:42 smithi129 ceph-mon[101316]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:28:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:42 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-47","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:28:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:42 smithi129 ceph-mon[101316]: osdmap e1017: 8 total, 8 up, 8 in 2024-09-14T04:28:43.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:42 smithi039 ceph-mon[106470]: pgmap v1558: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:28:43.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:42 smithi039 ceph-mon[106470]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:28:43.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:42 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-47","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:28:43.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:42 smithi039 ceph-mon[106470]: osdmap e1017: 8 total, 8 up, 8 in 2024-09-14T04:28:43.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:42 smithi039 ceph-mon[111235]: pgmap v1558: 105 pgs: 32 unknown, 73 active+clean; 582 KiB data, 2.1 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:28:43.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:42 smithi039 ceph-mon[111235]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:28:43.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:42 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-47","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:28:43.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:42 smithi039 ceph-mon[111235]: osdmap e1017: 8 total, 8 up, 8 in 2024-09-14T04:28:43.989 INFO:tasks.workunit.client.0.smithi039.stdout: wrote [1237053~39551,3419561~35025,3545656~20885,4115647~17099,6039082~98458,8037421~81988,14604525~21052,15618262~117,15827735~89790] 2024-09-14T04:28:44.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:43 smithi129 ceph-mon[101316]: osdmap e1018: 8 total, 8 up, 8 in 2024-09-14T04:28:44.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:43 smithi039 ceph-mon[106470]: osdmap e1018: 8 total, 8 up, 8 in 2024-09-14T04:28:44.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:43 smithi039 ceph-mon[111235]: osdmap e1018: 8 total, 8 up, 8 in 2024-09-14T04:28:44.972 INFO:tasks.workunit.client.0.smithi039.stdout: wrote [1279339~76697,3801620~22600,5693366~5530,7332775~50914,8180869~73805,9530782~90455,16945793~48112,17236481~97855,18301513~20564,18453913~8334] 2024-09-14T04:28:44.980 INFO:tasks.workunit.client.0.smithi039.stdout: diff was [1279339~76697,3801620~22600,5693366~5530,7332775~50914,8119409~135265,8388608~1232629,16777216~1685031] 2024-09-14T04:28:44.984 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/0.DiffIterate (5048 ms) 2024-09-14T04:28:44.985 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/0.DiffIterateDeterministic 2024-09-14T04:28:44.985 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:28:45.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:44 smithi129 ceph-mon[101316]: pgmap v1561: 105 pgs: 7 creating+peering, 4 unknown, 94 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:28:45.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:44 smithi129 ceph-mon[101316]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:28:45.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:44 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T04:28:45.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:44 smithi129 ceph-mon[101316]: osdmap e1019: 8 total, 8 up, 8 in 2024-09-14T04:28:45.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:44 smithi039 ceph-mon[106470]: pgmap v1561: 105 pgs: 7 creating+peering, 4 unknown, 94 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:28:45.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:44 smithi039 ceph-mon[106470]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:28:45.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:44 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T04:28:45.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:44 smithi039 ceph-mon[106470]: osdmap e1019: 8 total, 8 up, 8 in 2024-09-14T04:28:45.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:44 smithi039 ceph-mon[111235]: pgmap v1561: 105 pgs: 7 creating+peering, 4 unknown, 94 active+clean; 582 KiB data, 1.0 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:28:45.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:44 smithi039 ceph-mon[111235]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:28:45.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:44 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T04:28:45.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:44 smithi039 ceph-mon[111235]: osdmap e1019: 8 total, 8 up, 8 in 2024-09-14T04:28:46.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:45 smithi129 ceph-mon[101316]: osdmap e1020: 8 total, 8 up, 8 in 2024-09-14T04:28:46.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:45 smithi039 ceph-mon[106470]: osdmap e1020: 8 total, 8 up, 8 in 2024-09-14T04:28:46.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:45 smithi039 ceph-mon[111235]: osdmap e1020: 8 total, 8 up, 8 in 2024-09-14T04:28:47.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:46 smithi129 ceph-mon[101316]: pgmap v1564: 105 pgs: 7 creating+peering, 98 active+clean; 582 KiB data, 770 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:28:47.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:46 smithi129 ceph-mon[101316]: osdmap e1021: 8 total, 8 up, 8 in 2024-09-14T04:28:47.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:46 smithi039 ceph-mon[106470]: pgmap v1564: 105 pgs: 7 creating+peering, 98 active+clean; 582 KiB data, 770 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:28:47.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:46 smithi039 ceph-mon[106470]: osdmap e1021: 8 total, 8 up, 8 in 2024-09-14T04:28:47.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:46 smithi039 ceph-mon[111235]: pgmap v1564: 105 pgs: 7 creating+peering, 98 active+clean; 582 KiB data, 770 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:28:47.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:46 smithi039 ceph-mon[111235]: osdmap e1021: 8 total, 8 up, 8 in 2024-09-14T04:28:48.019 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:28:48.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:47 smithi129 ceph-mon[101316]: osdmap e1022: 8 total, 8 up, 8 in 2024-09-14T04:28:48.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:47 smithi039 ceph-mon[106470]: osdmap e1022: 8 total, 8 up, 8 in 2024-09-14T04:28:48.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:47 smithi039 ceph-mon[111235]: osdmap e1022: 8 total, 8 up, 8 in 2024-09-14T04:28:48.989 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:28:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:28:48] "GET /metrics HTTP/1.1" 200 36695 "" "Prometheus/2.51.0" 2024-09-14T04:28:49.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:48 smithi129 ceph-mon[101316]: pgmap v1567: 105 pgs: 105 active+clean; 7.8 MiB data, 770 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 84 KiB/s wr, 21 op/s 2024-09-14T04:28:49.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:48 smithi129 ceph-mon[101316]: osdmap e1023: 8 total, 8 up, 8 in 2024-09-14T04:28:49.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:48 smithi039 ceph-mon[106470]: pgmap v1567: 105 pgs: 105 active+clean; 7.8 MiB data, 770 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 84 KiB/s wr, 21 op/s 2024-09-14T04:28:49.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:48 smithi039 ceph-mon[106470]: osdmap e1023: 8 total, 8 up, 8 in 2024-09-14T04:28:49.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:48 smithi039 ceph-mon[111235]: pgmap v1567: 105 pgs: 105 active+clean; 7.8 MiB data, 770 MiB used, 714 GiB / 715 GiB avail; 10 KiB/s rd, 84 KiB/s wr, 21 op/s 2024-09-14T04:28:49.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:48 smithi039 ceph-mon[111235]: osdmap e1023: 8 total, 8 up, 8 in 2024-09-14T04:28:50.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:50 smithi039 ceph-mon[106470]: osdmap e1024: 8 total, 8 up, 8 in 2024-09-14T04:28:50.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:50 smithi039 ceph-mon[111235]: osdmap e1024: 8 total, 8 up, 8 in 2024-09-14T04:28:50.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:50 smithi129 ceph-mon[101316]: osdmap e1024: 8 total, 8 up, 8 in 2024-09-14T04:28:50.783 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:28:51.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:51 smithi039 ceph-mon[106470]: pgmap v1570: 105 pgs: 105 active+clean; 14 MiB data, 766 MiB used, 714 GiB / 715 GiB avail; 70 KiB/s rd, 240 KiB/s wr, 112 op/s 2024-09-14T04:28:51.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:51 smithi039 ceph-mon[106470]: osdmap e1025: 8 total, 8 up, 8 in 2024-09-14T04:28:51.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:51 smithi039 ceph-mon[106470]: osdmap e1026: 8 total, 8 up, 8 in 2024-09-14T04:28:51.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:51 smithi039 ceph-mon[111235]: pgmap v1570: 105 pgs: 105 active+clean; 14 MiB data, 766 MiB used, 714 GiB / 715 GiB avail; 70 KiB/s rd, 240 KiB/s wr, 112 op/s 2024-09-14T04:28:51.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:51 smithi039 ceph-mon[111235]: osdmap e1025: 8 total, 8 up, 8 in 2024-09-14T04:28:51.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:51 smithi039 ceph-mon[111235]: osdmap e1026: 8 total, 8 up, 8 in 2024-09-14T04:28:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:51 smithi129 ceph-mon[101316]: pgmap v1570: 105 pgs: 105 active+clean; 14 MiB data, 766 MiB used, 714 GiB / 715 GiB avail; 70 KiB/s rd, 240 KiB/s wr, 112 op/s 2024-09-14T04:28:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:51 smithi129 ceph-mon[101316]: osdmap e1025: 8 total, 8 up, 8 in 2024-09-14T04:28:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:51 smithi129 ceph-mon[101316]: osdmap e1026: 8 total, 8 up, 8 in 2024-09-14T04:28:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:52 smithi129 ceph-mon[101316]: pgmap v1573: 105 pgs: 105 active+clean; 14 MiB data, 765 MiB used, 714 GiB / 715 GiB avail; 60 KiB/s rd, 156 KiB/s wr, 91 op/s 2024-09-14T04:28:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:52 smithi129 ceph-mon[101316]: osdmap e1027: 8 total, 8 up, 8 in 2024-09-14T04:28:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:28:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:52 smithi039 ceph-mon[106470]: pgmap v1573: 105 pgs: 105 active+clean; 14 MiB data, 765 MiB used, 714 GiB / 715 GiB avail; 60 KiB/s rd, 156 KiB/s wr, 91 op/s 2024-09-14T04:28:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:52 smithi039 ceph-mon[106470]: osdmap e1027: 8 total, 8 up, 8 in 2024-09-14T04:28:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:28:53.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:52 smithi039 ceph-mon[111235]: pgmap v1573: 105 pgs: 105 active+clean; 14 MiB data, 765 MiB used, 714 GiB / 715 GiB avail; 60 KiB/s rd, 156 KiB/s wr, 91 op/s 2024-09-14T04:28:53.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:52 smithi039 ceph-mon[111235]: osdmap e1027: 8 total, 8 up, 8 in 2024-09-14T04:28:53.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:28:53.810 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:28:54.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:53 smithi039 ceph-mon[106470]: osdmap e1028: 8 total, 8 up, 8 in 2024-09-14T04:28:54.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:53 smithi039 ceph-mon[111235]: osdmap e1028: 8 total, 8 up, 8 in 2024-09-14T04:28:54.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:53 smithi129 ceph-mon[101316]: osdmap e1028: 8 total, 8 up, 8 in 2024-09-14T04:28:55.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:54 smithi039 ceph-mon[106470]: pgmap v1576: 105 pgs: 105 active+clean; 16 MiB data, 761 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 4.2 KiB/s wr, 27 op/s 2024-09-14T04:28:55.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:54 smithi039 ceph-mon[106470]: osdmap e1029: 8 total, 8 up, 8 in 2024-09-14T04:28:55.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:54 smithi039 ceph-mon[111235]: pgmap v1576: 105 pgs: 105 active+clean; 16 MiB data, 761 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 4.2 KiB/s wr, 27 op/s 2024-09-14T04:28:55.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:54 smithi039 ceph-mon[111235]: osdmap e1029: 8 total, 8 up, 8 in 2024-09-14T04:28:55.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:54 smithi129 ceph-mon[101316]: pgmap v1576: 105 pgs: 105 active+clean; 16 MiB data, 761 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 4.2 KiB/s wr, 27 op/s 2024-09-14T04:28:55.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:54 smithi129 ceph-mon[101316]: osdmap e1029: 8 total, 8 up, 8 in 2024-09-14T04:28:56.171 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:55 smithi129 ceph-mon[101316]: osdmap e1030: 8 total, 8 up, 8 in 2024-09-14T04:28:56.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:55 smithi039 ceph-mon[106470]: osdmap e1030: 8 total, 8 up, 8 in 2024-09-14T04:28:56.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:55 smithi039 ceph-mon[111235]: osdmap e1030: 8 total, 8 up, 8 in 2024-09-14T04:28:56.872 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:28:57.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:56 smithi129 ceph-mon[101316]: pgmap v1579: 105 pgs: 105 active+clean; 16 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 61 KiB/s rd, 9.0 KiB/s wr, 95 op/s 2024-09-14T04:28:57.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:56 smithi129 ceph-mon[101316]: osdmap e1031: 8 total, 8 up, 8 in 2024-09-14T04:28:57.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:56 smithi039 ceph-mon[106470]: pgmap v1579: 105 pgs: 105 active+clean; 16 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 61 KiB/s rd, 9.0 KiB/s wr, 95 op/s 2024-09-14T04:28:57.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:56 smithi039 ceph-mon[106470]: osdmap e1031: 8 total, 8 up, 8 in 2024-09-14T04:28:57.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:56 smithi039 ceph-mon[111235]: pgmap v1579: 105 pgs: 105 active+clean; 16 MiB data, 759 MiB used, 714 GiB / 715 GiB avail; 61 KiB/s rd, 9.0 KiB/s wr, 95 op/s 2024-09-14T04:28:57.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:56 smithi039 ceph-mon[111235]: osdmap e1031: 8 total, 8 up, 8 in 2024-09-14T04:28:58.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:57 smithi129 ceph-mon[101316]: osdmap e1032: 8 total, 8 up, 8 in 2024-09-14T04:28:58.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:57 smithi039 ceph-mon[106470]: osdmap e1032: 8 total, 8 up, 8 in 2024-09-14T04:28:58.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:57 smithi039 ceph-mon[111235]: osdmap e1032: 8 total, 8 up, 8 in 2024-09-14T04:28:59.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:58 smithi039 ceph-mon[106470]: pgmap v1582: 105 pgs: 105 active+clean; 18 MiB data, 758 MiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 6.0 KiB/s wr, 74 op/s 2024-09-14T04:28:59.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:58 smithi039 ceph-mon[106470]: osdmap e1033: 8 total, 8 up, 8 in 2024-09-14T04:28:59.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:28:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:28:58] "GET /metrics HTTP/1.1" 200 36672 "" "Prometheus/2.51.0" 2024-09-14T04:28:59.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:58 smithi039 ceph-mon[111235]: pgmap v1582: 105 pgs: 105 active+clean; 18 MiB data, 758 MiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 6.0 KiB/s wr, 74 op/s 2024-09-14T04:28:59.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:58 smithi039 ceph-mon[111235]: osdmap e1033: 8 total, 8 up, 8 in 2024-09-14T04:28:59.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:58 smithi129 ceph-mon[101316]: pgmap v1582: 105 pgs: 105 active+clean; 18 MiB data, 758 MiB used, 714 GiB / 715 GiB avail; 51 KiB/s rd, 6.0 KiB/s wr, 74 op/s 2024-09-14T04:28:59.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:58 smithi129 ceph-mon[101316]: osdmap e1033: 8 total, 8 up, 8 in 2024-09-14T04:28:59.910 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/0.DiffIterateDeterministic (14926 ms) 2024-09-14T04:28:59.910 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/0.DiffIterateDeterministicPP 2024-09-14T04:28:59.910 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:29:00.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:28:59 smithi129 ceph-mon[101316]: osdmap e1034: 8 total, 8 up, 8 in 2024-09-14T04:29:00.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:28:59 smithi039 ceph-mon[106470]: osdmap e1034: 8 total, 8 up, 8 in 2024-09-14T04:29:00.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:28:59 smithi039 ceph-mon[111235]: osdmap e1034: 8 total, 8 up, 8 in 2024-09-14T04:29:01.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:00 smithi129 ceph-mon[101316]: pgmap v1585: 105 pgs: 105 active+clean; 20 MiB data, 746 MiB used, 714 GiB / 715 GiB avail; 53 KiB/s rd, 8.5 KiB/s wr, 85 op/s 2024-09-14T04:29:01.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:00 smithi129 ceph-mon[101316]: osdmap e1035: 8 total, 8 up, 8 in 2024-09-14T04:29:01.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:00 smithi039 ceph-mon[106470]: pgmap v1585: 105 pgs: 105 active+clean; 20 MiB data, 746 MiB used, 714 GiB / 715 GiB avail; 53 KiB/s rd, 8.5 KiB/s wr, 85 op/s 2024-09-14T04:29:01.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:00 smithi039 ceph-mon[106470]: osdmap e1035: 8 total, 8 up, 8 in 2024-09-14T04:29:01.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:00 smithi039 ceph-mon[111235]: pgmap v1585: 105 pgs: 105 active+clean; 20 MiB data, 746 MiB used, 714 GiB / 715 GiB avail; 53 KiB/s rd, 8.5 KiB/s wr, 85 op/s 2024-09-14T04:29:01.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:00 smithi039 ceph-mon[111235]: osdmap e1035: 8 total, 8 up, 8 in 2024-09-14T04:29:02.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:01 smithi129 ceph-mon[101316]: osdmap e1036: 8 total, 8 up, 8 in 2024-09-14T04:29:02.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:01 smithi039 ceph-mon[106470]: osdmap e1036: 8 total, 8 up, 8 in 2024-09-14T04:29:02.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:01 smithi039 ceph-mon[111235]: osdmap e1036: 8 total, 8 up, 8 in 2024-09-14T04:29:02.953 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:29:03.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:02 smithi129 ceph-mon[101316]: pgmap v1588: 105 pgs: 105 active+clean; 20 MiB data, 745 MiB used, 714 GiB / 715 GiB avail; 52 KiB/s rd, 7.2 KiB/s wr, 79 op/s 2024-09-14T04:29:03.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:02 smithi129 ceph-mon[101316]: osdmap e1037: 8 total, 8 up, 8 in 2024-09-14T04:29:03.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:02 smithi039 ceph-mon[106470]: pgmap v1588: 105 pgs: 105 active+clean; 20 MiB data, 745 MiB used, 714 GiB / 715 GiB avail; 52 KiB/s rd, 7.2 KiB/s wr, 79 op/s 2024-09-14T04:29:03.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:02 smithi039 ceph-mon[106470]: osdmap e1037: 8 total, 8 up, 8 in 2024-09-14T04:29:03.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:02 smithi039 ceph-mon[111235]: pgmap v1588: 105 pgs: 105 active+clean; 20 MiB data, 745 MiB used, 714 GiB / 715 GiB avail; 52 KiB/s rd, 7.2 KiB/s wr, 79 op/s 2024-09-14T04:29:03.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:02 smithi039 ceph-mon[111235]: osdmap e1037: 8 total, 8 up, 8 in 2024-09-14T04:29:04.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:03 smithi129 ceph-mon[101316]: osdmap e1038: 8 total, 8 up, 8 in 2024-09-14T04:29:04.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:03 smithi039 ceph-mon[106470]: osdmap e1038: 8 total, 8 up, 8 in 2024-09-14T04:29:04.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:03 smithi039 ceph-mon[111235]: osdmap e1038: 8 total, 8 up, 8 in 2024-09-14T04:29:05.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:04 smithi129 ceph-mon[101316]: pgmap v1591: 105 pgs: 105 active+clean; 28 MiB data, 740 MiB used, 714 GiB / 715 GiB avail; 60 KiB/s rd, 7.5 KiB/s wr, 97 op/s 2024-09-14T04:29:05.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:04 smithi129 ceph-mon[101316]: osdmap e1039: 8 total, 8 up, 8 in 2024-09-14T04:29:05.271 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 04:29:04 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=infra.usagestats t=2024-09-14T04:29:04.902514284Z level=info msg="Usage stats are ready to report" 2024-09-14T04:29:05.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:04 smithi039 ceph-mon[106470]: pgmap v1591: 105 pgs: 105 active+clean; 28 MiB data, 740 MiB used, 714 GiB / 715 GiB avail; 60 KiB/s rd, 7.5 KiB/s wr, 97 op/s 2024-09-14T04:29:05.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:04 smithi039 ceph-mon[106470]: osdmap e1039: 8 total, 8 up, 8 in 2024-09-14T04:29:05.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:04 smithi039 ceph-mon[111235]: pgmap v1591: 105 pgs: 105 active+clean; 28 MiB data, 740 MiB used, 714 GiB / 715 GiB avail; 60 KiB/s rd, 7.5 KiB/s wr, 97 op/s 2024-09-14T04:29:05.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:04 smithi039 ceph-mon[111235]: osdmap e1039: 8 total, 8 up, 8 in 2024-09-14T04:29:06.009 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:29:06.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:05 smithi129 ceph-mon[101316]: osdmap e1040: 8 total, 8 up, 8 in 2024-09-14T04:29:06.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:05 smithi039 ceph-mon[106470]: osdmap e1040: 8 total, 8 up, 8 in 2024-09-14T04:29:06.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:05 smithi039 ceph-mon[111235]: osdmap e1040: 8 total, 8 up, 8 in 2024-09-14T04:29:07.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:06 smithi129 ceph-mon[101316]: pgmap v1594: 105 pgs: 105 active+clean; 31 MiB data, 738 MiB used, 714 GiB / 715 GiB avail; 62 KiB/s rd, 8.5 KiB/s wr, 102 op/s 2024-09-14T04:29:07.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:06 smithi129 ceph-mon[101316]: osdmap e1041: 8 total, 8 up, 8 in 2024-09-14T04:29:07.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:06 smithi039 ceph-mon[111235]: pgmap v1594: 105 pgs: 105 active+clean; 31 MiB data, 738 MiB used, 714 GiB / 715 GiB avail; 62 KiB/s rd, 8.5 KiB/s wr, 102 op/s 2024-09-14T04:29:07.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:06 smithi039 ceph-mon[111235]: osdmap e1041: 8 total, 8 up, 8 in 2024-09-14T04:29:07.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:06 smithi039 ceph-mon[106470]: pgmap v1594: 105 pgs: 105 active+clean; 31 MiB data, 738 MiB used, 714 GiB / 715 GiB avail; 62 KiB/s rd, 8.5 KiB/s wr, 102 op/s 2024-09-14T04:29:07.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:06 smithi039 ceph-mon[106470]: osdmap e1041: 8 total, 8 up, 8 in 2024-09-14T04:29:08.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:08 smithi129 ceph-mon[101316]: osdmap e1042: 8 total, 8 up, 8 in 2024-09-14T04:29:08.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:08 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:29:08.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:08 smithi039 ceph-mon[106470]: osdmap e1042: 8 total, 8 up, 8 in 2024-09-14T04:29:08.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:08 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:29:08.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:08 smithi039 ceph-mon[111235]: osdmap e1042: 8 total, 8 up, 8 in 2024-09-14T04:29:08.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:08 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:29:09.016 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:29:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:29:08] "GET /metrics HTTP/1.1" 200 36675 "" "Prometheus/2.51.0" 2024-09-14T04:29:09.076 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:29:09.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:09 smithi129 ceph-mon[101316]: pgmap v1597: 105 pgs: 105 active+clean; 31 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 3.0 KiB/s wr, 20 op/s 2024-09-14T04:29:09.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:09 smithi129 ceph-mon[101316]: osdmap e1043: 8 total, 8 up, 8 in 2024-09-14T04:29:09.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:09 smithi039 ceph-mon[106470]: pgmap v1597: 105 pgs: 105 active+clean; 31 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 3.0 KiB/s wr, 20 op/s 2024-09-14T04:29:09.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:09 smithi039 ceph-mon[106470]: osdmap e1043: 8 total, 8 up, 8 in 2024-09-14T04:29:09.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:09 smithi039 ceph-mon[111235]: pgmap v1597: 105 pgs: 105 active+clean; 31 MiB data, 737 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 3.0 KiB/s wr, 20 op/s 2024-09-14T04:29:09.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:09 smithi039 ceph-mon[111235]: osdmap e1043: 8 total, 8 up, 8 in 2024-09-14T04:29:10.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:10 smithi129 ceph-mon[101316]: osdmap e1044: 8 total, 8 up, 8 in 2024-09-14T04:29:10.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:10 smithi039 ceph-mon[106470]: osdmap e1044: 8 total, 8 up, 8 in 2024-09-14T04:29:10.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:10 smithi039 ceph-mon[111235]: osdmap e1044: 8 total, 8 up, 8 in 2024-09-14T04:29:11.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:11 smithi129 ceph-mon[101316]: pgmap v1600: 105 pgs: 105 active+clean; 37 MiB data, 735 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 7.0 KiB/s wr, 67 op/s 2024-09-14T04:29:11.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:11 smithi129 ceph-mon[101316]: osdmap e1045: 8 total, 8 up, 8 in 2024-09-14T04:29:11.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:11 smithi039 ceph-mon[106470]: pgmap v1600: 105 pgs: 105 active+clean; 37 MiB data, 735 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 7.0 KiB/s wr, 67 op/s 2024-09-14T04:29:11.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:11 smithi039 ceph-mon[106470]: osdmap e1045: 8 total, 8 up, 8 in 2024-09-14T04:29:11.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:11 smithi039 ceph-mon[111235]: pgmap v1600: 105 pgs: 105 active+clean; 37 MiB data, 735 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 7.0 KiB/s wr, 67 op/s 2024-09-14T04:29:11.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:11 smithi039 ceph-mon[111235]: osdmap e1045: 8 total, 8 up, 8 in 2024-09-14T04:29:12.114 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:29:12.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:12 smithi129 ceph-mon[101316]: osdmap e1046: 8 total, 8 up, 8 in 2024-09-14T04:29:12.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:12 smithi039 ceph-mon[106470]: osdmap e1046: 8 total, 8 up, 8 in 2024-09-14T04:29:12.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:12 smithi039 ceph-mon[111235]: osdmap e1046: 8 total, 8 up, 8 in 2024-09-14T04:29:13.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:13 smithi129 ceph-mon[101316]: pgmap v1603: 105 pgs: 105 active+clean; 37 MiB data, 735 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 5.0 KiB/s wr, 52 op/s 2024-09-14T04:29:13.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:13 smithi129 ceph-mon[101316]: osdmap e1047: 8 total, 8 up, 8 in 2024-09-14T04:29:13.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:13 smithi039 ceph-mon[106470]: pgmap v1603: 105 pgs: 105 active+clean; 37 MiB data, 735 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 5.0 KiB/s wr, 52 op/s 2024-09-14T04:29:13.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:13 smithi039 ceph-mon[106470]: osdmap e1047: 8 total, 8 up, 8 in 2024-09-14T04:29:13.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:13 smithi039 ceph-mon[111235]: pgmap v1603: 105 pgs: 105 active+clean; 37 MiB data, 735 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 5.0 KiB/s wr, 52 op/s 2024-09-14T04:29:13.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:13 smithi039 ceph-mon[111235]: osdmap e1047: 8 total, 8 up, 8 in 2024-09-14T04:29:14.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:14 smithi129 ceph-mon[101316]: osdmap e1048: 8 total, 8 up, 8 in 2024-09-14T04:29:14.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:14 smithi039 ceph-mon[106470]: osdmap e1048: 8 total, 8 up, 8 in 2024-09-14T04:29:14.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:14 smithi039 ceph-mon[111235]: osdmap e1048: 8 total, 8 up, 8 in 2024-09-14T04:29:15.142 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/0.DiffIterateDeterministicPP (15232 ms) 2024-09-14T04:29:15.142 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/0.DiffIterateDeterministicLUKS1 2024-09-14T04:29:15.142 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:29:15.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:15 smithi129 ceph-mon[101316]: pgmap v1606: 105 pgs: 105 active+clean; 46 MiB data, 733 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s rd, 9.2 KiB/s wr, 103 op/s 2024-09-14T04:29:15.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:15 smithi129 ceph-mon[101316]: osdmap e1049: 8 total, 8 up, 8 in 2024-09-14T04:29:15.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:15 smithi039 ceph-mon[106470]: pgmap v1606: 105 pgs: 105 active+clean; 46 MiB data, 733 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s rd, 9.2 KiB/s wr, 103 op/s 2024-09-14T04:29:15.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:15 smithi039 ceph-mon[106470]: osdmap e1049: 8 total, 8 up, 8 in 2024-09-14T04:29:15.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:15 smithi039 ceph-mon[111235]: pgmap v1606: 105 pgs: 105 active+clean; 46 MiB data, 733 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s rd, 9.2 KiB/s wr, 103 op/s 2024-09-14T04:29:15.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:15 smithi039 ceph-mon[111235]: osdmap e1049: 8 total, 8 up, 8 in 2024-09-14T04:29:16.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:16 smithi129 ceph-mon[101316]: osdmap e1050: 8 total, 8 up, 8 in 2024-09-14T04:29:16.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:16 smithi039 ceph-mon[106470]: osdmap e1050: 8 total, 8 up, 8 in 2024-09-14T04:29:16.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:16 smithi039 ceph-mon[111235]: osdmap e1050: 8 total, 8 up, 8 in 2024-09-14T04:29:17.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:17 smithi129 ceph-mon[101316]: pgmap v1609: 105 pgs: 105 active+clean; 46 MiB data, 731 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s rd, 9.2 KiB/s wr, 103 op/s 2024-09-14T04:29:17.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:17 smithi039 ceph-mon[106470]: pgmap v1609: 105 pgs: 105 active+clean; 46 MiB data, 731 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s rd, 9.2 KiB/s wr, 103 op/s 2024-09-14T04:29:17.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:17 smithi039 ceph-mon[111235]: pgmap v1609: 105 pgs: 105 active+clean; 46 MiB data, 731 MiB used, 715 GiB / 715 GiB avail; 65 KiB/s rd, 9.2 KiB/s wr, 103 op/s 2024-09-14T04:29:19.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:29:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:29:18] "GET /metrics HTTP/1.1" 200 36675 "" "Prometheus/2.51.0" 2024-09-14T04:29:19.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:19 smithi129 ceph-mon[101316]: pgmap v1610: 105 pgs: 105 active+clean; 46 MiB data, 730 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 6.8 KiB/s wr, 83 op/s 2024-09-14T04:29:19.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:19 smithi039 ceph-mon[106470]: pgmap v1610: 105 pgs: 105 active+clean; 46 MiB data, 730 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 6.8 KiB/s wr, 83 op/s 2024-09-14T04:29:19.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:19 smithi039 ceph-mon[111235]: pgmap v1610: 105 pgs: 105 active+clean; 46 MiB data, 730 MiB used, 715 GiB / 715 GiB avail; 53 KiB/s rd, 6.8 KiB/s wr, 83 op/s 2024-09-14T04:29:20.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:20 smithi129 ceph-mon[101316]: osdmap e1051: 8 total, 8 up, 8 in 2024-09-14T04:29:20.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:20 smithi039 ceph-mon[106470]: osdmap e1051: 8 total, 8 up, 8 in 2024-09-14T04:29:20.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:20 smithi039 ceph-mon[111235]: osdmap e1051: 8 total, 8 up, 8 in 2024-09-14T04:29:21.197 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:29:21.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:21 smithi129 ceph-mon[101316]: pgmap v1612: 105 pgs: 105 active+clean; 54 MiB data, 729 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 4.5 KiB/s wr, 46 op/s 2024-09-14T04:29:21.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:21 smithi129 ceph-mon[101316]: osdmap e1052: 8 total, 8 up, 8 in 2024-09-14T04:29:21.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:21 smithi039 ceph-mon[106470]: pgmap v1612: 105 pgs: 105 active+clean; 54 MiB data, 729 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 4.5 KiB/s wr, 46 op/s 2024-09-14T04:29:21.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:21 smithi039 ceph-mon[106470]: osdmap e1052: 8 total, 8 up, 8 in 2024-09-14T04:29:21.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:21 smithi039 ceph-mon[111235]: pgmap v1612: 105 pgs: 105 active+clean; 54 MiB data, 729 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 4.5 KiB/s wr, 46 op/s 2024-09-14T04:29:21.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:21 smithi039 ceph-mon[111235]: osdmap e1052: 8 total, 8 up, 8 in 2024-09-14T04:29:22.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:22 smithi129 ceph-mon[101316]: osdmap e1053: 8 total, 8 up, 8 in 2024-09-14T04:29:22.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:22 smithi039 ceph-mon[106470]: osdmap e1053: 8 total, 8 up, 8 in 2024-09-14T04:29:22.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:22 smithi039 ceph-mon[111235]: osdmap e1053: 8 total, 8 up, 8 in 2024-09-14T04:29:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:23 smithi129 ceph-mon[101316]: pgmap v1615: 105 pgs: 105 active+clean; 54 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 4.5 KiB/s wr, 46 op/s 2024-09-14T04:29:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:29:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:23 smithi039 ceph-mon[106470]: pgmap v1615: 105 pgs: 105 active+clean; 54 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 4.5 KiB/s wr, 46 op/s 2024-09-14T04:29:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:29:23.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:23 smithi039 ceph-mon[111235]: pgmap v1615: 105 pgs: 105 active+clean; 54 MiB data, 728 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 4.5 KiB/s wr, 46 op/s 2024-09-14T04:29:23.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:29:25.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:25 smithi129 ceph-mon[101316]: pgmap v1616: 105 pgs: 105 active+clean; 58 MiB data, 724 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 688 KiB/s wr, 54 op/s 2024-09-14T04:29:25.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:25 smithi039 ceph-mon[106470]: pgmap v1616: 105 pgs: 105 active+clean; 58 MiB data, 724 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 688 KiB/s wr, 54 op/s 2024-09-14T04:29:25.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:25 smithi039 ceph-mon[111235]: pgmap v1616: 105 pgs: 105 active+clean; 58 MiB data, 724 MiB used, 715 GiB / 715 GiB avail; 32 KiB/s rd, 688 KiB/s wr, 54 op/s 2024-09-14T04:29:26.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:26 smithi129 ceph-mon[101316]: osdmap e1054: 8 total, 8 up, 8 in 2024-09-14T04:29:26.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:26 smithi039 ceph-mon[106470]: osdmap e1054: 8 total, 8 up, 8 in 2024-09-14T04:29:26.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:26 smithi039 ceph-mon[111235]: osdmap e1054: 8 total, 8 up, 8 in 2024-09-14T04:29:27.237 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:29:27.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:27 smithi129 ceph-mon[101316]: pgmap v1618: 105 pgs: 105 active+clean; 58 MiB data, 724 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 686 KiB/s wr, 36 op/s 2024-09-14T04:29:27.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:27 smithi129 ceph-mon[101316]: osdmap e1055: 8 total, 8 up, 8 in 2024-09-14T04:29:27.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:27 smithi039 ceph-mon[106470]: pgmap v1618: 105 pgs: 105 active+clean; 58 MiB data, 724 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 686 KiB/s wr, 36 op/s 2024-09-14T04:29:27.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:27 smithi039 ceph-mon[106470]: osdmap e1055: 8 total, 8 up, 8 in 2024-09-14T04:29:27.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:27 smithi039 ceph-mon[111235]: pgmap v1618: 105 pgs: 105 active+clean; 58 MiB data, 724 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 686 KiB/s wr, 36 op/s 2024-09-14T04:29:27.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:27 smithi039 ceph-mon[111235]: osdmap e1055: 8 total, 8 up, 8 in 2024-09-14T04:29:28.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:28 smithi129 ceph-mon[101316]: osdmap e1056: 8 total, 8 up, 8 in 2024-09-14T04:29:28.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:28 smithi039 ceph-mon[106470]: osdmap e1056: 8 total, 8 up, 8 in 2024-09-14T04:29:28.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:28 smithi039 ceph-mon[111235]: osdmap e1056: 8 total, 8 up, 8 in 2024-09-14T04:29:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:29:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:29:28] "GET /metrics HTTP/1.1" 200 36673 "" "Prometheus/2.51.0" 2024-09-14T04:29:29.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:29 smithi129 ceph-mon[101316]: pgmap v1621: 105 pgs: 105 active+clean; 58 MiB data, 724 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 686 KiB/s wr, 36 op/s 2024-09-14T04:29:29.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:29 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:29:29.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:29 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:29:29.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:29 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:29:29.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:29 smithi039 ceph-mon[106470]: pgmap v1621: 105 pgs: 105 active+clean; 58 MiB data, 724 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 686 KiB/s wr, 36 op/s 2024-09-14T04:29:29.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:29 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:29:29.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:29 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:29:29.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:29 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:29:29.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:29 smithi039 ceph-mon[111235]: pgmap v1621: 105 pgs: 105 active+clean; 58 MiB data, 724 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 686 KiB/s wr, 36 op/s 2024-09-14T04:29:29.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:29 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:29:29.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:29 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:29:29.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:29 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:29:31.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:31 smithi129 ceph-mon[101316]: pgmap v1622: 105 pgs: 105 active+clean; 64 MiB data, 741 MiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 687 KiB/s wr, 39 op/s 2024-09-14T04:29:31.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:31 smithi039 ceph-mon[106470]: pgmap v1622: 105 pgs: 105 active+clean; 64 MiB data, 741 MiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 687 KiB/s wr, 39 op/s 2024-09-14T04:29:31.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:31 smithi039 ceph-mon[111235]: pgmap v1622: 105 pgs: 105 active+clean; 64 MiB data, 741 MiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 687 KiB/s wr, 39 op/s 2024-09-14T04:29:32.490 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:32 smithi039 ceph-mon[106470]: osdmap e1057: 8 total, 8 up, 8 in 2024-09-14T04:29:32.490 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:32 smithi039 ceph-mon[111235]: osdmap e1057: 8 total, 8 up, 8 in 2024-09-14T04:29:32.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:32 smithi129 ceph-mon[101316]: osdmap e1057: 8 total, 8 up, 8 in 2024-09-14T04:29:33.285 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:29:33.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:33 smithi129 ceph-mon[101316]: pgmap v1624: 105 pgs: 105 active+clean; 64 MiB data, 741 MiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 686 KiB/s wr, 35 op/s 2024-09-14T04:29:33.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:33 smithi129 ceph-mon[101316]: osdmap e1058: 8 total, 8 up, 8 in 2024-09-14T04:29:33.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:33 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:29:33.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:33 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:29:33.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:33 smithi039 ceph-mon[111235]: pgmap v1624: 105 pgs: 105 active+clean; 64 MiB data, 741 MiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 686 KiB/s wr, 35 op/s 2024-09-14T04:29:33.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:33 smithi039 ceph-mon[111235]: osdmap e1058: 8 total, 8 up, 8 in 2024-09-14T04:29:33.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:33 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:29:33.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:33 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:29:33.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:33 smithi039 ceph-mon[106470]: pgmap v1624: 105 pgs: 105 active+clean; 64 MiB data, 741 MiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 686 KiB/s wr, 35 op/s 2024-09-14T04:29:33.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:33 smithi039 ceph-mon[106470]: osdmap e1058: 8 total, 8 up, 8 in 2024-09-14T04:29:33.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:33 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:29:33.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:33 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:29:34.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:34 smithi129 ceph-mon[101316]: osdmap e1059: 8 total, 8 up, 8 in 2024-09-14T04:29:34.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:34 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:29:34.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:34 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:29:34.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:34 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:29:34.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:34 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:29:34.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:34 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:29:34.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:34 smithi039 ceph-mon[106470]: osdmap e1059: 8 total, 8 up, 8 in 2024-09-14T04:29:34.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:34 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:29:34.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:34 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:29:34.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:34 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:29:34.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:34 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:29:34.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:34 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:29:34.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:34 smithi039 ceph-mon[111235]: osdmap e1059: 8 total, 8 up, 8 in 2024-09-14T04:29:34.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:34 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:29:34.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:34 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:29:34.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:34 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:29:34.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:34 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:29:34.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:34 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:29:35.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:35 smithi039 ceph-mon[106470]: pgmap v1627: 105 pgs: 105 active+clean; 70 MiB data, 749 MiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 1.3 MiB/s wr, 53 op/s 2024-09-14T04:29:35.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:35 smithi039 ceph-mon[111235]: pgmap v1627: 105 pgs: 105 active+clean; 70 MiB data, 749 MiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 1.3 MiB/s wr, 53 op/s 2024-09-14T04:29:35.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:35 smithi129 ceph-mon[101316]: pgmap v1627: 105 pgs: 105 active+clean; 70 MiB data, 749 MiB used, 714 GiB / 715 GiB avail; 33 KiB/s rd, 1.3 MiB/s wr, 53 op/s 2024-09-14T04:29:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:37 smithi129 ceph-mon[101316]: pgmap v1628: 105 pgs: 105 active+clean; 70 MiB data, 753 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 685 KiB/s wr, 17 op/s 2024-09-14T04:29:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:29:37.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:37 smithi039 ceph-mon[111235]: pgmap v1628: 105 pgs: 105 active+clean; 70 MiB data, 753 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 685 KiB/s wr, 17 op/s 2024-09-14T04:29:37.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:29:37.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:37 smithi039 ceph-mon[106470]: pgmap v1628: 105 pgs: 105 active+clean; 70 MiB data, 753 MiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 685 KiB/s wr, 17 op/s 2024-09-14T04:29:37.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:29:38.617 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:38 smithi039 ceph-mon[106470]: osdmap e1060: 8 total, 8 up, 8 in 2024-09-14T04:29:38.618 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:38 smithi039 ceph-mon[111235]: osdmap e1060: 8 total, 8 up, 8 in 2024-09-14T04:29:38.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:38 smithi129 ceph-mon[101316]: osdmap e1060: 8 total, 8 up, 8 in 2024-09-14T04:29:39.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:29:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:29:38] "GET /metrics HTTP/1.1" 200 36680 "" "Prometheus/2.51.0" 2024-09-14T04:29:39.365 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:29:39.624 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:39 smithi039 ceph-mon[106470]: pgmap v1630: 105 pgs: 105 active+clean; 70 MiB data, 753 MiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 686 KiB/s wr, 32 op/s 2024-09-14T04:29:39.624 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:39 smithi039 ceph-mon[106470]: osdmap e1061: 8 total, 8 up, 8 in 2024-09-14T04:29:39.625 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:39 smithi039 ceph-mon[111235]: pgmap v1630: 105 pgs: 105 active+clean; 70 MiB data, 753 MiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 686 KiB/s wr, 32 op/s 2024-09-14T04:29:39.625 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:39 smithi039 ceph-mon[111235]: osdmap e1061: 8 total, 8 up, 8 in 2024-09-14T04:29:39.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:39 smithi129 ceph-mon[101316]: pgmap v1630: 105 pgs: 105 active+clean; 70 MiB data, 753 MiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 686 KiB/s wr, 32 op/s 2024-09-14T04:29:39.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:39 smithi129 ceph-mon[101316]: osdmap e1061: 8 total, 8 up, 8 in 2024-09-14T04:29:40.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:40 smithi129 ceph-mon[101316]: osdmap e1062: 8 total, 8 up, 8 in 2024-09-14T04:29:40.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:40 smithi129 ceph-mon[101316]: pgmap v1633: 105 pgs: 105 active+clean; 76 MiB data, 761 MiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 686 KiB/s wr, 37 op/s 2024-09-14T04:29:40.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:40 smithi039 ceph-mon[106470]: osdmap e1062: 8 total, 8 up, 8 in 2024-09-14T04:29:40.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:40 smithi039 ceph-mon[106470]: pgmap v1633: 105 pgs: 105 active+clean; 76 MiB data, 761 MiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 686 KiB/s wr, 37 op/s 2024-09-14T04:29:40.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:40 smithi039 ceph-mon[111235]: osdmap e1062: 8 total, 8 up, 8 in 2024-09-14T04:29:40.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:40 smithi039 ceph-mon[111235]: pgmap v1633: 105 pgs: 105 active+clean; 76 MiB data, 761 MiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 686 KiB/s wr, 37 op/s 2024-09-14T04:29:43.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:42 smithi129 ceph-mon[101316]: pgmap v1634: 105 pgs: 105 active+clean; 76 MiB data, 761 MiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 686 KiB/s wr, 38 op/s 2024-09-14T04:29:43.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:42 smithi039 ceph-mon[111235]: pgmap v1634: 105 pgs: 105 active+clean; 76 MiB data, 761 MiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 686 KiB/s wr, 38 op/s 2024-09-14T04:29:43.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:42 smithi039 ceph-mon[106470]: pgmap v1634: 105 pgs: 105 active+clean; 76 MiB data, 761 MiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 686 KiB/s wr, 38 op/s 2024-09-14T04:29:45.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:44 smithi039 ceph-mon[111235]: pgmap v1635: 105 pgs: 105 active+clean; 76 MiB data, 766 MiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 682 KiB/s wr, 48 op/s 2024-09-14T04:29:45.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:44 smithi039 ceph-mon[111235]: osdmap e1063: 8 total, 8 up, 8 in 2024-09-14T04:29:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:44 smithi039 ceph-mon[106470]: pgmap v1635: 105 pgs: 105 active+clean; 76 MiB data, 766 MiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 682 KiB/s wr, 48 op/s 2024-09-14T04:29:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:44 smithi039 ceph-mon[106470]: osdmap e1063: 8 total, 8 up, 8 in 2024-09-14T04:29:45.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:44 smithi129 ceph-mon[101316]: pgmap v1635: 105 pgs: 105 active+clean; 76 MiB data, 766 MiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 682 KiB/s wr, 48 op/s 2024-09-14T04:29:45.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:44 smithi129 ceph-mon[101316]: osdmap e1063: 8 total, 8 up, 8 in 2024-09-14T04:29:45.823 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/0.DiffIterateDeterministicLUKS1 (30680 ms) 2024-09-14T04:29:45.823 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/0.DiffIterateDeterministicLUKS1PP 2024-09-14T04:29:45.823 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:29:46.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:45 smithi129 ceph-mon[101316]: osdmap e1064: 8 total, 8 up, 8 in 2024-09-14T04:29:46.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:45 smithi039 ceph-mon[106470]: osdmap e1064: 8 total, 8 up, 8 in 2024-09-14T04:29:46.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:45 smithi039 ceph-mon[111235]: osdmap e1064: 8 total, 8 up, 8 in 2024-09-14T04:29:47.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:46 smithi129 ceph-mon[101316]: pgmap v1638: 105 pgs: 105 active+clean; 76 MiB data, 774 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 2.5 KiB/s wr, 29 op/s 2024-09-14T04:29:47.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:46 smithi129 ceph-mon[101316]: osdmap e1065: 8 total, 8 up, 8 in 2024-09-14T04:29:47.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:46 smithi039 ceph-mon[106470]: pgmap v1638: 105 pgs: 105 active+clean; 76 MiB data, 774 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 2.5 KiB/s wr, 29 op/s 2024-09-14T04:29:47.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:46 smithi039 ceph-mon[106470]: osdmap e1065: 8 total, 8 up, 8 in 2024-09-14T04:29:47.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:46 smithi039 ceph-mon[111235]: pgmap v1638: 105 pgs: 105 active+clean; 76 MiB data, 774 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 2.5 KiB/s wr, 29 op/s 2024-09-14T04:29:47.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:46 smithi039 ceph-mon[111235]: osdmap e1065: 8 total, 8 up, 8 in 2024-09-14T04:29:49.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:48 smithi039 ceph-mon[106470]: pgmap v1640: 105 pgs: 105 active+clean; 76 MiB data, 774 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 2.3 KiB/s wr, 28 op/s 2024-09-14T04:29:49.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:29:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:29:48] "GET /metrics HTTP/1.1" 200 36680 "" "Prometheus/2.51.0" 2024-09-14T04:29:49.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:48 smithi039 ceph-mon[111235]: pgmap v1640: 105 pgs: 105 active+clean; 76 MiB data, 774 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 2.3 KiB/s wr, 28 op/s 2024-09-14T04:29:49.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:48 smithi129 ceph-mon[101316]: pgmap v1640: 105 pgs: 105 active+clean; 76 MiB data, 774 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 2.3 KiB/s wr, 28 op/s 2024-09-14T04:29:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:50 smithi129 ceph-mon[101316]: pgmap v1641: 105 pgs: 105 active+clean; 88 MiB data, 782 MiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 686 KiB/s wr, 43 op/s 2024-09-14T04:29:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:50 smithi129 ceph-mon[101316]: osdmap e1066: 8 total, 8 up, 8 in 2024-09-14T04:29:51.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:50 smithi039 ceph-mon[106470]: pgmap v1641: 105 pgs: 105 active+clean; 88 MiB data, 782 MiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 686 KiB/s wr, 43 op/s 2024-09-14T04:29:51.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:50 smithi039 ceph-mon[106470]: osdmap e1066: 8 total, 8 up, 8 in 2024-09-14T04:29:51.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:50 smithi039 ceph-mon[111235]: pgmap v1641: 105 pgs: 105 active+clean; 88 MiB data, 782 MiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 686 KiB/s wr, 43 op/s 2024-09-14T04:29:51.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:50 smithi039 ceph-mon[111235]: osdmap e1066: 8 total, 8 up, 8 in 2024-09-14T04:29:51.904 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:29:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:51 smithi129 ceph-mon[101316]: osdmap e1067: 8 total, 8 up, 8 in 2024-09-14T04:29:52.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:51 smithi039 ceph-mon[106470]: osdmap e1067: 8 total, 8 up, 8 in 2024-09-14T04:29:52.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:51 smithi039 ceph-mon[111235]: osdmap e1067: 8 total, 8 up, 8 in 2024-09-14T04:29:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:52 smithi129 ceph-mon[101316]: pgmap v1644: 105 pgs: 105 active+clean; 88 MiB data, 782 MiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 686 KiB/s wr, 40 op/s 2024-09-14T04:29:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:52 smithi129 ceph-mon[101316]: osdmap e1068: 8 total, 8 up, 8 in 2024-09-14T04:29:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:29:53.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:52 smithi039 ceph-mon[106470]: pgmap v1644: 105 pgs: 105 active+clean; 88 MiB data, 782 MiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 686 KiB/s wr, 40 op/s 2024-09-14T04:29:53.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:52 smithi039 ceph-mon[106470]: osdmap e1068: 8 total, 8 up, 8 in 2024-09-14T04:29:53.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:29:53.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:52 smithi039 ceph-mon[111235]: pgmap v1644: 105 pgs: 105 active+clean; 88 MiB data, 782 MiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 686 KiB/s wr, 40 op/s 2024-09-14T04:29:53.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:52 smithi039 ceph-mon[111235]: osdmap e1068: 8 total, 8 up, 8 in 2024-09-14T04:29:53.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:29:55.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:54 smithi129 ceph-mon[101316]: pgmap v1646: 105 pgs: 105 active+clean; 93 MiB data, 799 MiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 1.3 MiB/s wr, 54 op/s 2024-09-14T04:29:55.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:54 smithi039 ceph-mon[106470]: pgmap v1646: 105 pgs: 105 active+clean; 93 MiB data, 799 MiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 1.3 MiB/s wr, 54 op/s 2024-09-14T04:29:55.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:54 smithi039 ceph-mon[111235]: pgmap v1646: 105 pgs: 105 active+clean; 93 MiB data, 799 MiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 1.3 MiB/s wr, 54 op/s 2024-09-14T04:29:57.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:56 smithi129 ceph-mon[101316]: pgmap v1647: 105 pgs: 105 active+clean; 93 MiB data, 799 MiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 686 KiB/s wr, 38 op/s 2024-09-14T04:29:57.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:56 smithi129 ceph-mon[101316]: osdmap e1069: 8 total, 8 up, 8 in 2024-09-14T04:29:57.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:56 smithi039 ceph-mon[106470]: pgmap v1647: 105 pgs: 105 active+clean; 93 MiB data, 799 MiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 686 KiB/s wr, 38 op/s 2024-09-14T04:29:57.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:56 smithi039 ceph-mon[106470]: osdmap e1069: 8 total, 8 up, 8 in 2024-09-14T04:29:57.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:56 smithi039 ceph-mon[111235]: pgmap v1647: 105 pgs: 105 active+clean; 93 MiB data, 799 MiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 686 KiB/s wr, 38 op/s 2024-09-14T04:29:57.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:56 smithi039 ceph-mon[111235]: osdmap e1069: 8 total, 8 up, 8 in 2024-09-14T04:29:57.940 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:29:58.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:57 smithi129 ceph-mon[101316]: osdmap e1070: 8 total, 8 up, 8 in 2024-09-14T04:29:58.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:57 smithi039 ceph-mon[106470]: osdmap e1070: 8 total, 8 up, 8 in 2024-09-14T04:29:58.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:57 smithi039 ceph-mon[111235]: osdmap e1070: 8 total, 8 up, 8 in 2024-09-14T04:29:58.908 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:29:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:29:58] "GET /metrics HTTP/1.1" 200 36679 "" "Prometheus/2.51.0" 2024-09-14T04:29:59.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:58 smithi129 ceph-mon[101316]: pgmap v1650: 105 pgs: 105 active+clean; 93 MiB data, 799 MiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 686 KiB/s wr, 40 op/s 2024-09-14T04:29:59.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:29:58 smithi129 ceph-mon[101316]: osdmap e1071: 8 total, 8 up, 8 in 2024-09-14T04:29:59.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:58 smithi039 ceph-mon[106470]: pgmap v1650: 105 pgs: 105 active+clean; 93 MiB data, 799 MiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 686 KiB/s wr, 40 op/s 2024-09-14T04:29:59.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:29:58 smithi039 ceph-mon[106470]: osdmap e1071: 8 total, 8 up, 8 in 2024-09-14T04:29:59.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:58 smithi039 ceph-mon[111235]: pgmap v1650: 105 pgs: 105 active+clean; 93 MiB data, 799 MiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 686 KiB/s wr, 40 op/s 2024-09-14T04:29:59.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:29:58 smithi039 ceph-mon[111235]: osdmap e1071: 8 total, 8 up, 8 in 2024-09-14T04:30:01.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:00 smithi129 ceph-mon[101316]: pgmap v1652: 105 pgs: 105 active+clean; 100 MiB data, 811 MiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 686 KiB/s wr, 37 op/s 2024-09-14T04:30:01.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:00 smithi129 ceph-mon[101316]: overall HEALTH_OK 2024-09-14T04:30:01.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:00 smithi039 ceph-mon[106470]: pgmap v1652: 105 pgs: 105 active+clean; 100 MiB data, 811 MiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 686 KiB/s wr, 37 op/s 2024-09-14T04:30:01.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:00 smithi039 ceph-mon[106470]: overall HEALTH_OK 2024-09-14T04:30:01.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:00 smithi039 ceph-mon[111235]: pgmap v1652: 105 pgs: 105 active+clean; 100 MiB data, 811 MiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 686 KiB/s wr, 37 op/s 2024-09-14T04:30:01.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:00 smithi039 ceph-mon[111235]: overall HEALTH_OK 2024-09-14T04:30:03.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:02 smithi129 ceph-mon[101316]: pgmap v1653: 105 pgs: 105 active+clean; 103 MiB data, 811 MiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 686 KiB/s wr, 35 op/s 2024-09-14T04:30:03.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:02 smithi129 ceph-mon[101316]: osdmap e1072: 8 total, 8 up, 8 in 2024-09-14T04:30:03.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:02 smithi039 ceph-mon[106470]: pgmap v1653: 105 pgs: 105 active+clean; 103 MiB data, 811 MiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 686 KiB/s wr, 35 op/s 2024-09-14T04:30:03.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:02 smithi039 ceph-mon[106470]: osdmap e1072: 8 total, 8 up, 8 in 2024-09-14T04:30:03.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:02 smithi039 ceph-mon[111235]: pgmap v1653: 105 pgs: 105 active+clean; 103 MiB data, 811 MiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 686 KiB/s wr, 35 op/s 2024-09-14T04:30:03.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:02 smithi039 ceph-mon[111235]: osdmap e1072: 8 total, 8 up, 8 in 2024-09-14T04:30:03.989 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:30:04.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:03 smithi129 ceph-mon[101316]: osdmap e1073: 8 total, 8 up, 8 in 2024-09-14T04:30:04.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:03 smithi039 ceph-mon[106470]: osdmap e1073: 8 total, 8 up, 8 in 2024-09-14T04:30:04.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:03 smithi039 ceph-mon[111235]: osdmap e1073: 8 total, 8 up, 8 in 2024-09-14T04:30:05.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:04 smithi129 ceph-mon[101316]: pgmap v1656: 105 pgs: 105 active+clean; 110 MiB data, 819 MiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 1.3 MiB/s wr, 45 op/s 2024-09-14T04:30:05.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:04 smithi129 ceph-mon[101316]: osdmap e1074: 8 total, 8 up, 8 in 2024-09-14T04:30:05.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:04 smithi039 ceph-mon[106470]: pgmap v1656: 105 pgs: 105 active+clean; 110 MiB data, 819 MiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 1.3 MiB/s wr, 45 op/s 2024-09-14T04:30:05.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:04 smithi039 ceph-mon[106470]: osdmap e1074: 8 total, 8 up, 8 in 2024-09-14T04:30:05.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:04 smithi039 ceph-mon[111235]: pgmap v1656: 105 pgs: 105 active+clean; 110 MiB data, 819 MiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 1.3 MiB/s wr, 45 op/s 2024-09-14T04:30:05.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:04 smithi039 ceph-mon[111235]: osdmap e1074: 8 total, 8 up, 8 in 2024-09-14T04:30:07.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:06 smithi129 ceph-mon[101316]: pgmap v1658: 105 pgs: 105 active+clean; 110 MiB data, 823 MiB used, 714 GiB / 715 GiB avail; 35 KiB/s rd, 687 KiB/s wr, 53 op/s 2024-09-14T04:30:07.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:06 smithi039 ceph-mon[111235]: pgmap v1658: 105 pgs: 105 active+clean; 110 MiB data, 823 MiB used, 714 GiB / 715 GiB avail; 35 KiB/s rd, 687 KiB/s wr, 53 op/s 2024-09-14T04:30:07.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:06 smithi039 ceph-mon[106470]: pgmap v1658: 105 pgs: 105 active+clean; 110 MiB data, 823 MiB used, 714 GiB / 715 GiB avail; 35 KiB/s rd, 687 KiB/s wr, 53 op/s 2024-09-14T04:30:08.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:30:08.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:30:08.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:30:09.003 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:30:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:30:08] "GET /metrics HTTP/1.1" 200 36682 "" "Prometheus/2.51.0" 2024-09-14T04:30:09.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:09 smithi129 ceph-mon[101316]: pgmap v1659: 105 pgs: 105 active+clean; 110 MiB data, 827 MiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 686 KiB/s wr, 32 op/s 2024-09-14T04:30:09.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:09 smithi129 ceph-mon[101316]: osdmap e1075: 8 total, 8 up, 8 in 2024-09-14T04:30:09.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:09 smithi039 ceph-mon[106470]: pgmap v1659: 105 pgs: 105 active+clean; 110 MiB data, 827 MiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 686 KiB/s wr, 32 op/s 2024-09-14T04:30:09.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:09 smithi039 ceph-mon[106470]: osdmap e1075: 8 total, 8 up, 8 in 2024-09-14T04:30:09.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:09 smithi039 ceph-mon[111235]: pgmap v1659: 105 pgs: 105 active+clean; 110 MiB data, 827 MiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 686 KiB/s wr, 32 op/s 2024-09-14T04:30:09.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:09 smithi039 ceph-mon[111235]: osdmap e1075: 8 total, 8 up, 8 in 2024-09-14T04:30:10.061 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:30:10.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:10 smithi039 ceph-mon[106470]: osdmap e1076: 8 total, 8 up, 8 in 2024-09-14T04:30:10.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:10 smithi039 ceph-mon[111235]: osdmap e1076: 8 total, 8 up, 8 in 2024-09-14T04:30:10.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:10 smithi129 ceph-mon[101316]: osdmap e1076: 8 total, 8 up, 8 in 2024-09-14T04:30:11.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:11 smithi039 ceph-mon[106470]: pgmap v1662: 105 pgs: 105 active+clean; 113 MiB data, 832 MiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 3.3 KiB/s wr, 37 op/s 2024-09-14T04:30:11.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:11 smithi039 ceph-mon[106470]: osdmap e1077: 8 total, 8 up, 8 in 2024-09-14T04:30:11.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:11 smithi039 ceph-mon[111235]: pgmap v1662: 105 pgs: 105 active+clean; 113 MiB data, 832 MiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 3.3 KiB/s wr, 37 op/s 2024-09-14T04:30:11.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:11 smithi039 ceph-mon[111235]: osdmap e1077: 8 total, 8 up, 8 in 2024-09-14T04:30:11.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:11 smithi129 ceph-mon[101316]: pgmap v1662: 105 pgs: 105 active+clean; 113 MiB data, 832 MiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 3.3 KiB/s wr, 37 op/s 2024-09-14T04:30:11.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:11 smithi129 ceph-mon[101316]: osdmap e1077: 8 total, 8 up, 8 in 2024-09-14T04:30:13.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:13 smithi039 ceph-mon[106470]: pgmap v1664: 105 pgs: 105 active+clean; 113 MiB data, 832 MiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s rd, 2.3 KiB/s wr, 18 op/s 2024-09-14T04:30:13.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:13 smithi039 ceph-mon[111235]: pgmap v1664: 105 pgs: 105 active+clean; 113 MiB data, 832 MiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s rd, 2.3 KiB/s wr, 18 op/s 2024-09-14T04:30:13.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:13 smithi129 ceph-mon[101316]: pgmap v1664: 105 pgs: 105 active+clean; 113 MiB data, 832 MiB used, 714 GiB / 715 GiB avail; 8.5 KiB/s rd, 2.3 KiB/s wr, 18 op/s 2024-09-14T04:30:15.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:15 smithi129 ceph-mon[101316]: pgmap v1665: 105 pgs: 105 active+clean; 123 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 688 KiB/s wr, 54 op/s 2024-09-14T04:30:15.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:15 smithi129 ceph-mon[101316]: osdmap e1078: 8 total, 8 up, 8 in 2024-09-14T04:30:15.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:15 smithi039 ceph-mon[106470]: pgmap v1665: 105 pgs: 105 active+clean; 123 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 688 KiB/s wr, 54 op/s 2024-09-14T04:30:15.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:15 smithi039 ceph-mon[106470]: osdmap e1078: 8 total, 8 up, 8 in 2024-09-14T04:30:15.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:15 smithi039 ceph-mon[111235]: pgmap v1665: 105 pgs: 105 active+clean; 123 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 32 KiB/s rd, 688 KiB/s wr, 54 op/s 2024-09-14T04:30:15.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:15 smithi039 ceph-mon[111235]: osdmap e1078: 8 total, 8 up, 8 in 2024-09-14T04:30:16.108 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/0.DiffIterateDeterministicLUKS1PP (30286 ms) 2024-09-14T04:30:16.108 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/0.DiffIterateDeterministicLUKS2 2024-09-14T04:30:16.108 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:30:16.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:16 smithi129 ceph-mon[101316]: osdmap e1079: 8 total, 8 up, 8 in 2024-09-14T04:30:16.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:16 smithi039 ceph-mon[106470]: osdmap e1079: 8 total, 8 up, 8 in 2024-09-14T04:30:16.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:16 smithi039 ceph-mon[111235]: osdmap e1079: 8 total, 8 up, 8 in 2024-09-14T04:30:17.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:17 smithi129 ceph-mon[101316]: pgmap v1668: 105 pgs: 105 active+clean; 123 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 686 KiB/s wr, 37 op/s 2024-09-14T04:30:17.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:17 smithi129 ceph-mon[101316]: osdmap e1080: 8 total, 8 up, 8 in 2024-09-14T04:30:17.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:17 smithi039 ceph-mon[106470]: pgmap v1668: 105 pgs: 105 active+clean; 123 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 686 KiB/s wr, 37 op/s 2024-09-14T04:30:17.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:17 smithi039 ceph-mon[106470]: osdmap e1080: 8 total, 8 up, 8 in 2024-09-14T04:30:17.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:17 smithi039 ceph-mon[111235]: pgmap v1668: 105 pgs: 105 active+clean; 123 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 686 KiB/s wr, 37 op/s 2024-09-14T04:30:17.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:17 smithi039 ceph-mon[111235]: osdmap e1080: 8 total, 8 up, 8 in 2024-09-14T04:30:19.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:30:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:30:18] "GET /metrics HTTP/1.1" 200 36682 "" "Prometheus/2.51.0" 2024-09-14T04:30:19.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:19 smithi129 ceph-mon[101316]: pgmap v1670: 105 pgs: 105 active+clean; 123 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 686 KiB/s wr, 38 op/s 2024-09-14T04:30:19.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:19 smithi039 ceph-mon[106470]: pgmap v1670: 105 pgs: 105 active+clean; 123 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 686 KiB/s wr, 38 op/s 2024-09-14T04:30:19.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:19 smithi039 ceph-mon[111235]: pgmap v1670: 105 pgs: 105 active+clean; 123 MiB data, 840 MiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 686 KiB/s wr, 38 op/s 2024-09-14T04:30:21.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:21 smithi129 ceph-mon[101316]: pgmap v1671: 105 pgs: 105 active+clean; 135 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 686 KiB/s wr, 40 op/s 2024-09-14T04:30:21.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:21 smithi039 ceph-mon[106470]: pgmap v1671: 105 pgs: 105 active+clean; 135 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 686 KiB/s wr, 40 op/s 2024-09-14T04:30:21.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:21 smithi039 ceph-mon[111235]: pgmap v1671: 105 pgs: 105 active+clean; 135 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 686 KiB/s wr, 40 op/s 2024-09-14T04:30:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:23 smithi129 ceph-mon[101316]: pgmap v1672: 105 pgs: 105 active+clean; 135 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 563 KiB/s wr, 33 op/s 2024-09-14T04:30:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:30:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:23 smithi039 ceph-mon[106470]: pgmap v1672: 105 pgs: 105 active+clean; 135 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 563 KiB/s wr, 33 op/s 2024-09-14T04:30:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:30:23.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:23 smithi039 ceph-mon[111235]: pgmap v1672: 105 pgs: 105 active+clean; 135 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 563 KiB/s wr, 33 op/s 2024-09-14T04:30:23.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:30:25.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:25 smithi129 ceph-mon[101316]: pgmap v1673: 105 pgs: 105 active+clean; 135 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 495 KiB/s wr, 29 op/s 2024-09-14T04:30:25.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:25 smithi129 ceph-mon[101316]: osdmap e1081: 8 total, 8 up, 8 in 2024-09-14T04:30:25.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:25 smithi039 ceph-mon[106470]: pgmap v1673: 105 pgs: 105 active+clean; 135 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 495 KiB/s wr, 29 op/s 2024-09-14T04:30:25.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:25 smithi039 ceph-mon[106470]: osdmap e1081: 8 total, 8 up, 8 in 2024-09-14T04:30:25.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:25 smithi039 ceph-mon[111235]: pgmap v1673: 105 pgs: 105 active+clean; 135 MiB data, 853 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 495 KiB/s wr, 29 op/s 2024-09-14T04:30:25.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:25 smithi039 ceph-mon[111235]: osdmap e1081: 8 total, 8 up, 8 in 2024-09-14T04:30:26.226 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:30:26.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:26 smithi129 ceph-mon[101316]: osdmap e1082: 8 total, 8 up, 8 in 2024-09-14T04:30:26.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:26 smithi039 ceph-mon[106470]: osdmap e1082: 8 total, 8 up, 8 in 2024-09-14T04:30:26.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:26 smithi039 ceph-mon[111235]: osdmap e1082: 8 total, 8 up, 8 in 2024-09-14T04:30:27.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:27 smithi129 ceph-mon[101316]: pgmap v1676: 105 pgs: 105 active+clean; 135 MiB data, 857 MiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 516 KiB/s wr, 36 op/s 2024-09-14T04:30:27.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:27 smithi129 ceph-mon[101316]: osdmap e1083: 8 total, 8 up, 8 in 2024-09-14T04:30:27.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:27 smithi039 ceph-mon[106470]: pgmap v1676: 105 pgs: 105 active+clean; 135 MiB data, 857 MiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 516 KiB/s wr, 36 op/s 2024-09-14T04:30:27.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:27 smithi039 ceph-mon[106470]: osdmap e1083: 8 total, 8 up, 8 in 2024-09-14T04:30:27.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:27 smithi039 ceph-mon[111235]: pgmap v1676: 105 pgs: 105 active+clean; 135 MiB data, 857 MiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 516 KiB/s wr, 36 op/s 2024-09-14T04:30:27.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:27 smithi039 ceph-mon[111235]: osdmap e1083: 8 total, 8 up, 8 in 2024-09-14T04:30:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:30:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:30:28] "GET /metrics HTTP/1.1" 200 36686 "" "Prometheus/2.51.0" 2024-09-14T04:30:29.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:29 smithi129 ceph-mon[101316]: pgmap v1678: 105 pgs: 105 active+clean; 135 MiB data, 857 MiB used, 714 GiB / 715 GiB avail; 4.8 KiB/s rd, 2.2 KiB/s wr, 8 op/s 2024-09-14T04:30:29.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:29 smithi039 ceph-mon[106470]: pgmap v1678: 105 pgs: 105 active+clean; 135 MiB data, 857 MiB used, 714 GiB / 715 GiB avail; 4.8 KiB/s rd, 2.2 KiB/s wr, 8 op/s 2024-09-14T04:30:29.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:29 smithi039 ceph-mon[111235]: pgmap v1678: 105 pgs: 105 active+clean; 135 MiB data, 857 MiB used, 714 GiB / 715 GiB avail; 4.8 KiB/s rd, 2.2 KiB/s wr, 8 op/s 2024-09-14T04:30:31.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:31 smithi129 ceph-mon[101316]: pgmap v1679: 105 pgs: 105 active+clean; 151 MiB data, 901 MiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 2.7 MiB/s wr, 44 op/s 2024-09-14T04:30:31.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:31 smithi039 ceph-mon[106470]: pgmap v1679: 105 pgs: 105 active+clean; 151 MiB data, 901 MiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 2.7 MiB/s wr, 44 op/s 2024-09-14T04:30:31.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:31 smithi039 ceph-mon[111235]: pgmap v1679: 105 pgs: 105 active+clean; 151 MiB data, 901 MiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 2.7 MiB/s wr, 44 op/s 2024-09-14T04:30:33.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:33 smithi129 ceph-mon[101316]: pgmap v1680: 105 pgs: 105 active+clean; 151 MiB data, 901 MiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 2.2 MiB/s wr, 37 op/s 2024-09-14T04:30:33.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:33 smithi039 ceph-mon[106470]: pgmap v1680: 105 pgs: 105 active+clean; 151 MiB data, 901 MiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 2.2 MiB/s wr, 37 op/s 2024-09-14T04:30:33.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:33 smithi039 ceph-mon[111235]: pgmap v1680: 105 pgs: 105 active+clean; 151 MiB data, 901 MiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 2.2 MiB/s wr, 37 op/s 2024-09-14T04:30:34.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:34 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:30:34.533 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:34 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:30:34.533 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:34 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:30:35.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:35 smithi129 ceph-mon[101316]: pgmap v1681: 105 pgs: 105 active+clean; 151 MiB data, 901 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 2.0 MiB/s wr, 26 op/s 2024-09-14T04:30:35.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:35 smithi039 ceph-mon[106470]: pgmap v1681: 105 pgs: 105 active+clean; 151 MiB data, 901 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 2.0 MiB/s wr, 26 op/s 2024-09-14T04:30:35.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:35 smithi039 ceph-mon[111235]: pgmap v1681: 105 pgs: 105 active+clean; 151 MiB data, 901 MiB used, 714 GiB / 715 GiB avail; 17 KiB/s rd, 2.0 MiB/s wr, 26 op/s 2024-09-14T04:30:36.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:36 smithi129 ceph-mon[101316]: osdmap e1084: 8 total, 8 up, 8 in 2024-09-14T04:30:36.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:36 smithi039 ceph-mon[106470]: osdmap e1084: 8 total, 8 up, 8 in 2024-09-14T04:30:36.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:36 smithi039 ceph-mon[111235]: osdmap e1084: 8 total, 8 up, 8 in 2024-09-14T04:30:37.324 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:37 smithi039 ceph-mon[111235]: pgmap v1683: 105 pgs: 105 active+clean; 151 MiB data, 913 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 1.7 MiB/s wr, 23 op/s 2024-09-14T04:30:37.324 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:37 smithi039 ceph-mon[111235]: osdmap e1085: 8 total, 8 up, 8 in 2024-09-14T04:30:37.324 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:30:37.325 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:37 smithi039 ceph-mon[106470]: pgmap v1683: 105 pgs: 105 active+clean; 151 MiB data, 913 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 1.7 MiB/s wr, 23 op/s 2024-09-14T04:30:37.325 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:37 smithi039 ceph-mon[106470]: osdmap e1085: 8 total, 8 up, 8 in 2024-09-14T04:30:37.325 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:30:37.336 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:30:37.597 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:37 smithi129 ceph-mon[101316]: pgmap v1683: 105 pgs: 105 active+clean; 151 MiB data, 913 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 1.7 MiB/s wr, 23 op/s 2024-09-14T04:30:37.597 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:37 smithi129 ceph-mon[101316]: osdmap e1085: 8 total, 8 up, 8 in 2024-09-14T04:30:37.598 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:30:38.649 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:38 smithi039 ceph-mon[106470]: osdmap e1086: 8 total, 8 up, 8 in 2024-09-14T04:30:38.650 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:38 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:30:38.650 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:38 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:30:38.650 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:38 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:30:38.650 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:38 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:30:38.650 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:38 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:30:38.650 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:38 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:30:38.650 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:38 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:30:38.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:38 smithi039 ceph-mon[111235]: osdmap e1086: 8 total, 8 up, 8 in 2024-09-14T04:30:38.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:38 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:30:38.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:38 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:30:38.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:38 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:30:38.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:38 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:30:38.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:38 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:30:38.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:38 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:30:38.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:38 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:30:38.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:38 smithi129 ceph-mon[101316]: osdmap e1086: 8 total, 8 up, 8 in 2024-09-14T04:30:38.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:38 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:30:38.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:38 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:30:38.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:38 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:30:38.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:38 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:30:38.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:38 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:30:38.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:38 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:30:38.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:38 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:30:39.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:30:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:30:38] "GET /metrics HTTP/1.1" 200 36682 "" "Prometheus/2.51.0" 2024-09-14T04:30:39.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:39 smithi129 ceph-mon[101316]: pgmap v1686: 105 pgs: 105 active+clean; 152 MiB data, 913 MiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.0 KiB/s wr, 10 op/s 2024-09-14T04:30:39.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:39 smithi039 ceph-mon[106470]: pgmap v1686: 105 pgs: 105 active+clean; 152 MiB data, 913 MiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.0 KiB/s wr, 10 op/s 2024-09-14T04:30:39.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:39 smithi039 ceph-mon[111235]: pgmap v1686: 105 pgs: 105 active+clean; 152 MiB data, 913 MiB used, 714 GiB / 715 GiB avail; 6.2 KiB/s rd, 3.0 KiB/s wr, 10 op/s 2024-09-14T04:30:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:41 smithi129 ceph-mon[101316]: pgmap v1687: 105 pgs: 105 active+clean; 169 MiB data, 949 MiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 2.7 MiB/s wr, 37 op/s 2024-09-14T04:30:41.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:41 smithi039 ceph-mon[106470]: pgmap v1687: 105 pgs: 105 active+clean; 169 MiB data, 949 MiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 2.7 MiB/s wr, 37 op/s 2024-09-14T04:30:41.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:41 smithi039 ceph-mon[111235]: pgmap v1687: 105 pgs: 105 active+clean; 169 MiB data, 949 MiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 2.7 MiB/s wr, 37 op/s 2024-09-14T04:30:42.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:42 smithi129 ceph-mon[101316]: pgmap v1688: 105 pgs: 105 active+clean; 169 MiB data, 949 MiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 2.6 MiB/s wr, 36 op/s 2024-09-14T04:30:42.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:42 smithi039 ceph-mon[106470]: pgmap v1688: 105 pgs: 105 active+clean; 169 MiB data, 949 MiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 2.6 MiB/s wr, 36 op/s 2024-09-14T04:30:42.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:42 smithi039 ceph-mon[111235]: pgmap v1688: 105 pgs: 105 active+clean; 169 MiB data, 949 MiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 2.6 MiB/s wr, 36 op/s 2024-09-14T04:30:44.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:44 smithi129 ceph-mon[101316]: pgmap v1689: 105 pgs: 105 active+clean; 169 MiB data, 949 MiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 2.0 MiB/s wr, 33 op/s 2024-09-14T04:30:44.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:44 smithi039 ceph-mon[106470]: pgmap v1689: 105 pgs: 105 active+clean; 169 MiB data, 949 MiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 2.0 MiB/s wr, 33 op/s 2024-09-14T04:30:44.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:44 smithi039 ceph-mon[111235]: pgmap v1689: 105 pgs: 105 active+clean; 169 MiB data, 949 MiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 2.0 MiB/s wr, 33 op/s 2024-09-14T04:30:46.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:46 smithi129 ceph-mon[101316]: pgmap v1690: 105 pgs: 105 active+clean; 169 MiB data, 949 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 1.8 MiB/s wr, 29 op/s 2024-09-14T04:30:46.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:46 smithi039 ceph-mon[106470]: pgmap v1690: 105 pgs: 105 active+clean; 169 MiB data, 949 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 1.8 MiB/s wr, 29 op/s 2024-09-14T04:30:46.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:46 smithi039 ceph-mon[111235]: pgmap v1690: 105 pgs: 105 active+clean; 169 MiB data, 949 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 1.8 MiB/s wr, 29 op/s 2024-09-14T04:30:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:47 smithi129 ceph-mon[101316]: osdmap e1087: 8 total, 8 up, 8 in 2024-09-14T04:30:47.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:47 smithi039 ceph-mon[106470]: osdmap e1087: 8 total, 8 up, 8 in 2024-09-14T04:30:47.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:47 smithi039 ceph-mon[111235]: osdmap e1087: 8 total, 8 up, 8 in 2024-09-14T04:30:48.494 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:30:48.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:48 smithi129 ceph-mon[101316]: pgmap v1692: 105 pgs: 105 active+clean; 177 MiB data, 957 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 2.4 MiB/s wr, 23 op/s 2024-09-14T04:30:48.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:48 smithi129 ceph-mon[101316]: osdmap e1088: 8 total, 8 up, 8 in 2024-09-14T04:30:48.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:48 smithi039 ceph-mon[106470]: pgmap v1692: 105 pgs: 105 active+clean; 177 MiB data, 957 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 2.4 MiB/s wr, 23 op/s 2024-09-14T04:30:48.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:48 smithi039 ceph-mon[106470]: osdmap e1088: 8 total, 8 up, 8 in 2024-09-14T04:30:48.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:30:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:30:48] "GET /metrics HTTP/1.1" 200 36682 "" "Prometheus/2.51.0" 2024-09-14T04:30:48.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:48 smithi039 ceph-mon[111235]: pgmap v1692: 105 pgs: 105 active+clean; 177 MiB data, 957 MiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 2.4 MiB/s wr, 23 op/s 2024-09-14T04:30:48.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:48 smithi039 ceph-mon[111235]: osdmap e1088: 8 total, 8 up, 8 in 2024-09-14T04:30:49.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:49 smithi129 ceph-mon[101316]: osdmap e1089: 8 total, 8 up, 8 in 2024-09-14T04:30:49.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:49 smithi039 ceph-mon[106470]: osdmap e1089: 8 total, 8 up, 8 in 2024-09-14T04:30:49.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:49 smithi039 ceph-mon[111235]: osdmap e1089: 8 total, 8 up, 8 in 2024-09-14T04:30:50.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:50 smithi129 ceph-mon[101316]: pgmap v1695: 105 pgs: 105 active+clean; 186 MiB data, 990 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 2.7 MiB/s wr, 17 op/s 2024-09-14T04:30:50.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:50 smithi039 ceph-mon[106470]: pgmap v1695: 105 pgs: 105 active+clean; 186 MiB data, 990 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 2.7 MiB/s wr, 17 op/s 2024-09-14T04:30:50.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:50 smithi039 ceph-mon[111235]: pgmap v1695: 105 pgs: 105 active+clean; 186 MiB data, 990 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 2.7 MiB/s wr, 17 op/s 2024-09-14T04:30:52.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:52 smithi129 ceph-mon[101316]: pgmap v1696: 105 pgs: 105 active+clean; 186 MiB data, 998 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 2.7 MiB/s wr, 17 op/s 2024-09-14T04:30:52.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:30:52.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:52 smithi039 ceph-mon[106470]: pgmap v1696: 105 pgs: 105 active+clean; 186 MiB data, 998 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 2.7 MiB/s wr, 17 op/s 2024-09-14T04:30:52.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:30:52.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:52 smithi039 ceph-mon[111235]: pgmap v1696: 105 pgs: 105 active+clean; 186 MiB data, 998 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 2.7 MiB/s wr, 17 op/s 2024-09-14T04:30:52.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:30:54.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:54 smithi039 ceph-mon[106470]: pgmap v1697: 105 pgs: 105 active+clean; 187 MiB data, 998 MiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 2.3 MiB/s wr, 38 op/s 2024-09-14T04:30:54.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:54 smithi039 ceph-mon[111235]: pgmap v1697: 105 pgs: 105 active+clean; 187 MiB data, 998 MiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 2.3 MiB/s wr, 38 op/s 2024-09-14T04:30:55.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:54 smithi129 ceph-mon[101316]: pgmap v1697: 105 pgs: 105 active+clean; 187 MiB data, 998 MiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 2.3 MiB/s wr, 38 op/s 2024-09-14T04:30:56.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:56 smithi039 ceph-mon[106470]: pgmap v1698: 105 pgs: 105 active+clean; 187 MiB data, 998 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 1.0 MiB/s wr, 29 op/s 2024-09-14T04:30:56.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:56 smithi039 ceph-mon[111235]: pgmap v1698: 105 pgs: 105 active+clean; 187 MiB data, 998 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 1.0 MiB/s wr, 29 op/s 2024-09-14T04:30:57.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:56 smithi129 ceph-mon[101316]: pgmap v1698: 105 pgs: 105 active+clean; 187 MiB data, 998 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 1.0 MiB/s wr, 29 op/s 2024-09-14T04:30:57.921 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:57 smithi129 ceph-mon[101316]: osdmap e1090: 8 total, 8 up, 8 in 2024-09-14T04:30:58.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:57 smithi039 ceph-mon[106470]: osdmap e1090: 8 total, 8 up, 8 in 2024-09-14T04:30:58.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:57 smithi039 ceph-mon[111235]: osdmap e1090: 8 total, 8 up, 8 in 2024-09-14T04:30:58.636 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:30:59.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:58 smithi129 ceph-mon[101316]: pgmap v1700: 105 pgs: 105 active+clean; 195 MiB data, 1006 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 922 KiB/s wr, 18 op/s 2024-09-14T04:30:59.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:58 smithi129 ceph-mon[101316]: osdmap e1091: 8 total, 8 up, 8 in 2024-09-14T04:30:59.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:58 smithi039 ceph-mon[111235]: pgmap v1700: 105 pgs: 105 active+clean; 195 MiB data, 1006 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 922 KiB/s wr, 18 op/s 2024-09-14T04:30:59.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:58 smithi039 ceph-mon[111235]: osdmap e1091: 8 total, 8 up, 8 in 2024-09-14T04:30:59.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:58 smithi039 ceph-mon[106470]: pgmap v1700: 105 pgs: 105 active+clean; 195 MiB data, 1006 MiB used, 714 GiB / 715 GiB avail; 11 KiB/s rd, 922 KiB/s wr, 18 op/s 2024-09-14T04:30:59.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:58 smithi039 ceph-mon[106470]: osdmap e1091: 8 total, 8 up, 8 in 2024-09-14T04:30:59.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:30:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:30:58] "GET /metrics HTTP/1.1" 200 36689 "" "Prometheus/2.51.0" 2024-09-14T04:31:00.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:30:59 smithi129 ceph-mon[101316]: osdmap e1092: 8 total, 8 up, 8 in 2024-09-14T04:31:00.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:30:59 smithi039 ceph-mon[106470]: osdmap e1092: 8 total, 8 up, 8 in 2024-09-14T04:31:00.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:30:59 smithi039 ceph-mon[111235]: osdmap e1092: 8 total, 8 up, 8 in 2024-09-14T04:31:01.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:00 smithi129 ceph-mon[101316]: pgmap v1703: 105 pgs: 105 active+clean; 205 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 2.7 MiB/s wr, 21 op/s 2024-09-14T04:31:01.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:00 smithi039 ceph-mon[106470]: pgmap v1703: 105 pgs: 105 active+clean; 205 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 2.7 MiB/s wr, 21 op/s 2024-09-14T04:31:01.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:00 smithi039 ceph-mon[111235]: pgmap v1703: 105 pgs: 105 active+clean; 205 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 2.7 MiB/s wr, 21 op/s 2024-09-14T04:31:03.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:02 smithi129 ceph-mon[101316]: pgmap v1704: 105 pgs: 105 active+clean; 205 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 2.7 MiB/s wr, 21 op/s 2024-09-14T04:31:03.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:02 smithi039 ceph-mon[106470]: pgmap v1704: 105 pgs: 105 active+clean; 205 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 2.7 MiB/s wr, 21 op/s 2024-09-14T04:31:03.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:02 smithi039 ceph-mon[111235]: pgmap v1704: 105 pgs: 105 active+clean; 205 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 2.7 MiB/s wr, 21 op/s 2024-09-14T04:31:05.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:04 smithi129 ceph-mon[101316]: pgmap v1705: 105 pgs: 105 active+clean; 205 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 2.4 MiB/s wr, 39 op/s 2024-09-14T04:31:05.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:04 smithi039 ceph-mon[106470]: pgmap v1705: 105 pgs: 105 active+clean; 205 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 2.4 MiB/s wr, 39 op/s 2024-09-14T04:31:05.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:04 smithi039 ceph-mon[111235]: pgmap v1705: 105 pgs: 105 active+clean; 205 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 2.4 MiB/s wr, 39 op/s 2024-09-14T04:31:07.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:06 smithi129 ceph-mon[101316]: pgmap v1706: 105 pgs: 105 active+clean; 205 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 1.0 MiB/s wr, 33 op/s 2024-09-14T04:31:07.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:06 smithi039 ceph-mon[106470]: pgmap v1706: 105 pgs: 105 active+clean; 205 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 1.0 MiB/s wr, 33 op/s 2024-09-14T04:31:07.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:06 smithi039 ceph-mon[111235]: pgmap v1706: 105 pgs: 105 active+clean; 205 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 1.0 MiB/s wr, 33 op/s 2024-09-14T04:31:07.941 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:07 smithi039 ceph-mon[106470]: osdmap e1093: 8 total, 8 up, 8 in 2024-09-14T04:31:07.942 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:31:07.942 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:07 smithi039 ceph-mon[111235]: osdmap e1093: 8 total, 8 up, 8 in 2024-09-14T04:31:07.942 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:31:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:07 smithi129 ceph-mon[101316]: osdmap e1093: 8 total, 8 up, 8 in 2024-09-14T04:31:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:31:08.720 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/0.DiffIterateDeterministicLUKS2 (52612 ms) 2024-09-14T04:31:08.720 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/0.DiffIterateDeterministicLUKS2PP 2024-09-14T04:31:08.720 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:31:09.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:08 smithi129 ceph-mon[101316]: pgmap v1708: 105 pgs: 105 active+clean; 205 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 1.3 KiB/s wr, 18 op/s 2024-09-14T04:31:09.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:08 smithi129 ceph-mon[101316]: osdmap e1094: 8 total, 8 up, 8 in 2024-09-14T04:31:09.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:08 smithi039 ceph-mon[106470]: pgmap v1708: 105 pgs: 105 active+clean; 205 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 1.3 KiB/s wr, 18 op/s 2024-09-14T04:31:09.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:08 smithi039 ceph-mon[106470]: osdmap e1094: 8 total, 8 up, 8 in 2024-09-14T04:31:09.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:31:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:31:08] "GET /metrics HTTP/1.1" 200 36689 "" "Prometheus/2.51.0" 2024-09-14T04:31:09.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:08 smithi039 ceph-mon[111235]: pgmap v1708: 105 pgs: 105 active+clean; 205 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 12 KiB/s rd, 1.3 KiB/s wr, 18 op/s 2024-09-14T04:31:09.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:08 smithi039 ceph-mon[111235]: osdmap e1094: 8 total, 8 up, 8 in 2024-09-14T04:31:10.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:09 smithi129 ceph-mon[101316]: osdmap e1095: 8 total, 8 up, 8 in 2024-09-14T04:31:10.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:09 smithi039 ceph-mon[106470]: osdmap e1095: 8 total, 8 up, 8 in 2024-09-14T04:31:10.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:09 smithi039 ceph-mon[111235]: osdmap e1095: 8 total, 8 up, 8 in 2024-09-14T04:31:11.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:10 smithi129 ceph-mon[101316]: pgmap v1711: 105 pgs: 105 active+clean; 225 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 2.7 MiB/s wr, 33 op/s 2024-09-14T04:31:11.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:10 smithi039 ceph-mon[106470]: pgmap v1711: 105 pgs: 105 active+clean; 225 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 2.7 MiB/s wr, 33 op/s 2024-09-14T04:31:11.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:10 smithi039 ceph-mon[111235]: pgmap v1711: 105 pgs: 105 active+clean; 225 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 2.7 MiB/s wr, 33 op/s 2024-09-14T04:31:13.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:12 smithi129 ceph-mon[101316]: pgmap v1712: 105 pgs: 105 active+clean; 225 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 2.7 MiB/s wr, 34 op/s 2024-09-14T04:31:13.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:12 smithi039 ceph-mon[106470]: pgmap v1712: 105 pgs: 105 active+clean; 225 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 2.7 MiB/s wr, 34 op/s 2024-09-14T04:31:13.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:12 smithi039 ceph-mon[111235]: pgmap v1712: 105 pgs: 105 active+clean; 225 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 2.7 MiB/s wr, 34 op/s 2024-09-14T04:31:15.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:14 smithi129 ceph-mon[101316]: pgmap v1713: 105 pgs: 105 active+clean; 229 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 2.4 MiB/s wr, 39 op/s 2024-09-14T04:31:15.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:14 smithi039 ceph-mon[106470]: pgmap v1713: 105 pgs: 105 active+clean; 229 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 2.4 MiB/s wr, 39 op/s 2024-09-14T04:31:15.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:14 smithi039 ceph-mon[111235]: pgmap v1713: 105 pgs: 105 active+clean; 229 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 2.4 MiB/s wr, 39 op/s 2024-09-14T04:31:17.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:16 smithi039 ceph-mon[106470]: pgmap v1714: 105 pgs: 105 active+clean; 229 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 2.0 MiB/s wr, 33 op/s 2024-09-14T04:31:17.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:16 smithi039 ceph-mon[111235]: pgmap v1714: 105 pgs: 105 active+clean; 229 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 2.0 MiB/s wr, 33 op/s 2024-09-14T04:31:17.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:16 smithi129 ceph-mon[101316]: pgmap v1714: 105 pgs: 105 active+clean; 229 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 2.0 MiB/s wr, 33 op/s 2024-09-14T04:31:18.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:17 smithi039 ceph-mon[106470]: osdmap e1096: 8 total, 8 up, 8 in 2024-09-14T04:31:18.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:17 smithi039 ceph-mon[111235]: osdmap e1096: 8 total, 8 up, 8 in 2024-09-14T04:31:18.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:17 smithi129 ceph-mon[101316]: osdmap e1096: 8 total, 8 up, 8 in 2024-09-14T04:31:18.829 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:31:19.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:18 smithi039 ceph-mon[106470]: pgmap v1716: 105 pgs: 105 active+clean; 233 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 945 KiB/s wr, 23 op/s 2024-09-14T04:31:19.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:18 smithi039 ceph-mon[106470]: osdmap e1097: 8 total, 8 up, 8 in 2024-09-14T04:31:19.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:31:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:31:18] "GET /metrics HTTP/1.1" 200 36689 "" "Prometheus/2.51.0" 2024-09-14T04:31:19.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:18 smithi039 ceph-mon[111235]: pgmap v1716: 105 pgs: 105 active+clean; 233 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 945 KiB/s wr, 23 op/s 2024-09-14T04:31:19.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:18 smithi039 ceph-mon[111235]: osdmap e1097: 8 total, 8 up, 8 in 2024-09-14T04:31:19.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:18 smithi129 ceph-mon[101316]: pgmap v1716: 105 pgs: 105 active+clean; 233 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 945 KiB/s wr, 23 op/s 2024-09-14T04:31:19.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:18 smithi129 ceph-mon[101316]: osdmap e1097: 8 total, 8 up, 8 in 2024-09-14T04:31:20.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:19 smithi129 ceph-mon[101316]: osdmap e1098: 8 total, 8 up, 8 in 2024-09-14T04:31:20.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:19 smithi039 ceph-mon[106470]: osdmap e1098: 8 total, 8 up, 8 in 2024-09-14T04:31:20.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:19 smithi039 ceph-mon[111235]: osdmap e1098: 8 total, 8 up, 8 in 2024-09-14T04:31:21.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:20 smithi129 ceph-mon[101316]: pgmap v1719: 105 pgs: 105 active+clean; 245 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 2.7 MiB/s wr, 14 op/s 2024-09-14T04:31:21.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:20 smithi039 ceph-mon[106470]: pgmap v1719: 105 pgs: 105 active+clean; 245 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 2.7 MiB/s wr, 14 op/s 2024-09-14T04:31:21.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:20 smithi039 ceph-mon[111235]: pgmap v1719: 105 pgs: 105 active+clean; 245 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 2.7 MiB/s wr, 14 op/s 2024-09-14T04:31:23.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:22 smithi129 ceph-mon[101316]: pgmap v1720: 105 pgs: 105 active+clean; 245 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 2.7 MiB/s wr, 14 op/s 2024-09-14T04:31:23.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:31:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:22 smithi039 ceph-mon[106470]: pgmap v1720: 105 pgs: 105 active+clean; 245 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 2.7 MiB/s wr, 14 op/s 2024-09-14T04:31:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:31:23.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:22 smithi039 ceph-mon[111235]: pgmap v1720: 105 pgs: 105 active+clean; 245 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 2.7 MiB/s wr, 14 op/s 2024-09-14T04:31:23.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:31:25.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:24 smithi129 ceph-mon[101316]: pgmap v1721: 105 pgs: 105 active+clean; 245 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 1.8 MiB/s wr, 40 op/s 2024-09-14T04:31:25.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:24 smithi039 ceph-mon[106470]: pgmap v1721: 105 pgs: 105 active+clean; 245 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 1.8 MiB/s wr, 40 op/s 2024-09-14T04:31:25.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:24 smithi039 ceph-mon[111235]: pgmap v1721: 105 pgs: 105 active+clean; 245 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 1.8 MiB/s wr, 40 op/s 2024-09-14T04:31:27.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:26 smithi129 ceph-mon[101316]: pgmap v1722: 105 pgs: 105 active+clean; 245 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 1.5 MiB/s wr, 33 op/s 2024-09-14T04:31:27.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:26 smithi039 ceph-mon[106470]: pgmap v1722: 105 pgs: 105 active+clean; 245 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 1.5 MiB/s wr, 33 op/s 2024-09-14T04:31:27.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:26 smithi039 ceph-mon[111235]: pgmap v1722: 105 pgs: 105 active+clean; 245 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 1.5 MiB/s wr, 33 op/s 2024-09-14T04:31:28.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:27 smithi129 ceph-mon[101316]: osdmap e1099: 8 total, 8 up, 8 in 2024-09-14T04:31:28.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:27 smithi039 ceph-mon[106470]: osdmap e1099: 8 total, 8 up, 8 in 2024-09-14T04:31:28.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:27 smithi039 ceph-mon[111235]: osdmap e1099: 8 total, 8 up, 8 in 2024-09-14T04:31:28.936 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:31:29.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:28 smithi039 ceph-mon[106470]: pgmap v1724: 105 pgs: 105 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 480 KiB/s wr, 21 op/s 2024-09-14T04:31:29.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:28 smithi039 ceph-mon[106470]: osdmap e1100: 8 total, 8 up, 8 in 2024-09-14T04:31:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:31:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:31:28] "GET /metrics HTTP/1.1" 200 36690 "" "Prometheus/2.51.0" 2024-09-14T04:31:29.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:28 smithi039 ceph-mon[111235]: pgmap v1724: 105 pgs: 105 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 480 KiB/s wr, 21 op/s 2024-09-14T04:31:29.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:28 smithi039 ceph-mon[111235]: osdmap e1100: 8 total, 8 up, 8 in 2024-09-14T04:31:29.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:28 smithi129 ceph-mon[101316]: pgmap v1724: 105 pgs: 105 active+clean; 249 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 480 KiB/s wr, 21 op/s 2024-09-14T04:31:29.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:28 smithi129 ceph-mon[101316]: osdmap e1100: 8 total, 8 up, 8 in 2024-09-14T04:31:30.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:29 smithi129 ceph-mon[101316]: osdmap e1101: 8 total, 8 up, 8 in 2024-09-14T04:31:30.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:29 smithi039 ceph-mon[106470]: osdmap e1101: 8 total, 8 up, 8 in 2024-09-14T04:31:30.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:29 smithi039 ceph-mon[111235]: osdmap e1101: 8 total, 8 up, 8 in 2024-09-14T04:31:31.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:30 smithi129 ceph-mon[101316]: pgmap v1727: 105 pgs: 105 active+clean; 261 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.3 KiB/s rd, 2.7 MiB/s wr, 15 op/s 2024-09-14T04:31:31.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:30 smithi039 ceph-mon[106470]: pgmap v1727: 105 pgs: 105 active+clean; 261 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.3 KiB/s rd, 2.7 MiB/s wr, 15 op/s 2024-09-14T04:31:31.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:30 smithi039 ceph-mon[111235]: pgmap v1727: 105 pgs: 105 active+clean; 261 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.3 KiB/s rd, 2.7 MiB/s wr, 15 op/s 2024-09-14T04:31:33.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:32 smithi129 ceph-mon[101316]: pgmap v1728: 105 pgs: 105 active+clean; 261 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.3 KiB/s rd, 2.7 MiB/s wr, 15 op/s 2024-09-14T04:31:33.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:32 smithi039 ceph-mon[106470]: pgmap v1728: 105 pgs: 105 active+clean; 261 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.3 KiB/s rd, 2.7 MiB/s wr, 15 op/s 2024-09-14T04:31:33.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:32 smithi039 ceph-mon[111235]: pgmap v1728: 105 pgs: 105 active+clean; 261 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 9.3 KiB/s rd, 2.7 MiB/s wr, 15 op/s 2024-09-14T04:31:35.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:34 smithi129 ceph-mon[101316]: pgmap v1729: 105 pgs: 105 active+clean; 267 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 1.8 MiB/s wr, 41 op/s 2024-09-14T04:31:35.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:34 smithi039 ceph-mon[106470]: pgmap v1729: 105 pgs: 105 active+clean; 267 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 1.8 MiB/s wr, 41 op/s 2024-09-14T04:31:35.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:34 smithi039 ceph-mon[111235]: pgmap v1729: 105 pgs: 105 active+clean; 267 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 1.8 MiB/s wr, 41 op/s 2024-09-14T04:31:37.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:36 smithi129 ceph-mon[101316]: pgmap v1730: 105 pgs: 105 active+clean; 267 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 1.5 MiB/s wr, 33 op/s 2024-09-14T04:31:37.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:36 smithi039 ceph-mon[106470]: pgmap v1730: 105 pgs: 105 active+clean; 267 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 1.5 MiB/s wr, 33 op/s 2024-09-14T04:31:37.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:36 smithi039 ceph-mon[111235]: pgmap v1730: 105 pgs: 105 active+clean; 267 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 1.5 MiB/s wr, 33 op/s 2024-09-14T04:31:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:37 smithi129 ceph-mon[101316]: osdmap e1102: 8 total, 8 up, 8 in 2024-09-14T04:31:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:31:38.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:37 smithi039 ceph-mon[106470]: osdmap e1102: 8 total, 8 up, 8 in 2024-09-14T04:31:38.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:31:38.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:37 smithi039 ceph-mon[111235]: osdmap e1102: 8 total, 8 up, 8 in 2024-09-14T04:31:38.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:31:38.976 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:31:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:31:38] "GET /metrics HTTP/1.1" 200 36688 "" "Prometheus/2.51.0" 2024-09-14T04:31:38.997 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:31:39.199 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:38 smithi129 ceph-mon[101316]: pgmap v1732: 105 pgs: 105 active+clean; 267 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 2.6 KiB/s wr, 21 op/s 2024-09-14T04:31:39.199 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:38 smithi129 ceph-mon[101316]: osdmap e1103: 8 total, 8 up, 8 in 2024-09-14T04:31:39.199 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:38 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:31:39.257 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:38 smithi039 ceph-mon[106470]: pgmap v1732: 105 pgs: 105 active+clean; 267 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 2.6 KiB/s wr, 21 op/s 2024-09-14T04:31:39.257 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:38 smithi039 ceph-mon[106470]: osdmap e1103: 8 total, 8 up, 8 in 2024-09-14T04:31:39.257 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:38 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:31:39.257 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:38 smithi039 ceph-mon[111235]: pgmap v1732: 105 pgs: 105 active+clean; 267 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 2.6 KiB/s wr, 21 op/s 2024-09-14T04:31:39.257 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:38 smithi039 ceph-mon[111235]: osdmap e1103: 8 total, 8 up, 8 in 2024-09-14T04:31:39.257 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:38 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:31:40.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:39 smithi129 ceph-mon[101316]: osdmap e1104: 8 total, 8 up, 8 in 2024-09-14T04:31:40.276 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:39 smithi039 ceph-mon[106470]: osdmap e1104: 8 total, 8 up, 8 in 2024-09-14T04:31:40.277 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:39 smithi039 ceph-mon[111235]: osdmap e1104: 8 total, 8 up, 8 in 2024-09-14T04:31:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:40 smithi129 ceph-mon[101316]: pgmap v1735: 105 pgs: 105 active+clean; 279 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 2.0 MiB/s wr, 14 op/s 2024-09-14T04:31:41.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:40 smithi039 ceph-mon[106470]: pgmap v1735: 105 pgs: 105 active+clean; 279 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 2.0 MiB/s wr, 14 op/s 2024-09-14T04:31:41.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:40 smithi039 ceph-mon[111235]: pgmap v1735: 105 pgs: 105 active+clean; 279 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 2.0 MiB/s wr, 14 op/s 2024-09-14T04:31:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:43 smithi129 ceph-mon[101316]: pgmap v1736: 105 pgs: 105 active+clean; 279 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 2.0 MiB/s wr, 14 op/s 2024-09-14T04:31:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:43 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:31:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:43 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:31:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:43 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:31:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:43 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:31:43.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:42 smithi039 ceph-mon[106470]: pgmap v1736: 105 pgs: 105 active+clean; 279 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 2.0 MiB/s wr, 14 op/s 2024-09-14T04:31:43.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:43 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:31:43.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:43 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:31:43.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:43 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:31:43.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:43 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:31:43.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:43 smithi039 ceph-mon[111235]: pgmap v1736: 105 pgs: 105 active+clean; 279 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 9.2 KiB/s rd, 2.0 MiB/s wr, 14 op/s 2024-09-14T04:31:43.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:43 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:31:43.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:43 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:31:43.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:43 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:31:43.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:43 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:31:44.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:44 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:31:44.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:44 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:31:44.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:44 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:31:44.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:44 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:31:44.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:44 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:31:44.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:44 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:31:45.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:45 smithi129 ceph-mon[101316]: pgmap v1737: 105 pgs: 105 active+clean; 289 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 2.5 MiB/s wr, 41 op/s 2024-09-14T04:31:45.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:45 smithi039 ceph-mon[106470]: pgmap v1737: 105 pgs: 105 active+clean; 289 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 2.5 MiB/s wr, 41 op/s 2024-09-14T04:31:45.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:45 smithi039 ceph-mon[111235]: pgmap v1737: 105 pgs: 105 active+clean; 289 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 2.5 MiB/s wr, 41 op/s 2024-09-14T04:31:46.390 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:46 smithi129 ceph-mon[101316]: osdmap e1105: 8 total, 8 up, 8 in 2024-09-14T04:31:46.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:46 smithi039 ceph-mon[106470]: osdmap e1105: 8 total, 8 up, 8 in 2024-09-14T04:31:46.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:46 smithi039 ceph-mon[111235]: osdmap e1105: 8 total, 8 up, 8 in 2024-09-14T04:31:47.328 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:31:47.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:47 smithi039 ceph-mon[111235]: pgmap v1739: 105 pgs: 105 active+clean; 289 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 556 KiB/s wr, 24 op/s 2024-09-14T04:31:47.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:47 smithi039 ceph-mon[111235]: osdmap e1106: 8 total, 8 up, 8 in 2024-09-14T04:31:47.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:47 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:31:47.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:47 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:31:47.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:47 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:31:47.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:47 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:31:47.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:47 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:31:47.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:47 smithi039 ceph-mon[106470]: pgmap v1739: 105 pgs: 105 active+clean; 289 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 556 KiB/s wr, 24 op/s 2024-09-14T04:31:47.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:47 smithi039 ceph-mon[106470]: osdmap e1106: 8 total, 8 up, 8 in 2024-09-14T04:31:47.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:47 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:31:47.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:47 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:31:47.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:47 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:31:47.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:47 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:31:47.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:47 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:31:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:47 smithi129 ceph-mon[101316]: pgmap v1739: 105 pgs: 105 active+clean; 289 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 556 KiB/s wr, 24 op/s 2024-09-14T04:31:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:47 smithi129 ceph-mon[101316]: osdmap e1106: 8 total, 8 up, 8 in 2024-09-14T04:31:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:47 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:31:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:47 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:31:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:47 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:31:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:47 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:31:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:47 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:31:48.648 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:48 smithi039 ceph-mon[106470]: osdmap e1107: 8 total, 8 up, 8 in 2024-09-14T04:31:48.649 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:48 smithi039 ceph-mon[111235]: osdmap e1107: 8 total, 8 up, 8 in 2024-09-14T04:31:48.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:48 smithi129 ceph-mon[101316]: osdmap e1107: 8 total, 8 up, 8 in 2024-09-14T04:31:49.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:31:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:31:48] "GET /metrics HTTP/1.1" 200 36688 "" "Prometheus/2.51.0" 2024-09-14T04:31:49.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:49 smithi129 ceph-mon[101316]: pgmap v1742: 105 pgs: 105 active+clean; 289 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 688 KiB/s wr, 30 op/s 2024-09-14T04:31:49.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:49 smithi039 ceph-mon[106470]: pgmap v1742: 105 pgs: 105 active+clean; 289 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 688 KiB/s wr, 30 op/s 2024-09-14T04:31:49.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:49 smithi039 ceph-mon[111235]: pgmap v1742: 105 pgs: 105 active+clean; 289 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 688 KiB/s wr, 30 op/s 2024-09-14T04:31:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:51 smithi129 ceph-mon[101316]: pgmap v1743: 105 pgs: 105 active+clean; 311 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.7 MiB/s wr, 44 op/s 2024-09-14T04:31:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:51 smithi039 ceph-mon[106470]: pgmap v1743: 105 pgs: 105 active+clean; 311 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.7 MiB/s wr, 44 op/s 2024-09-14T04:31:51.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:51 smithi039 ceph-mon[111235]: pgmap v1743: 105 pgs: 105 active+clean; 311 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.7 MiB/s wr, 44 op/s 2024-09-14T04:31:52.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:52 smithi129 ceph-mon[101316]: pgmap v1744: 105 pgs: 105 active+clean; 311 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.6 MiB/s wr, 43 op/s 2024-09-14T04:31:52.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:31:52.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:52 smithi039 ceph-mon[106470]: pgmap v1744: 105 pgs: 105 active+clean; 311 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.6 MiB/s wr, 43 op/s 2024-09-14T04:31:52.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:31:52.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:52 smithi039 ceph-mon[111235]: pgmap v1744: 105 pgs: 105 active+clean; 311 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.6 MiB/s wr, 43 op/s 2024-09-14T04:31:52.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:31:54.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:54 smithi129 ceph-mon[101316]: osdmap e1108: 8 total, 8 up, 8 in 2024-09-14T04:31:54.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:54 smithi129 ceph-mon[101316]: pgmap v1746: 105 pgs: 105 active+clean; 323 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 3.9 MiB/s wr, 43 op/s 2024-09-14T04:31:54.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:54 smithi039 ceph-mon[106470]: osdmap e1108: 8 total, 8 up, 8 in 2024-09-14T04:31:54.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:54 smithi039 ceph-mon[106470]: pgmap v1746: 105 pgs: 105 active+clean; 323 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 3.9 MiB/s wr, 43 op/s 2024-09-14T04:31:54.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:54 smithi039 ceph-mon[111235]: osdmap e1108: 8 total, 8 up, 8 in 2024-09-14T04:31:54.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:54 smithi039 ceph-mon[111235]: pgmap v1746: 105 pgs: 105 active+clean; 323 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 3.9 MiB/s wr, 43 op/s 2024-09-14T04:31:55.401 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/0.DiffIterateDeterministicLUKS2PP (46681 ms) 2024-09-14T04:31:55.401 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/0.DiffIterateDiscard 2024-09-14T04:31:55.401 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:31:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:55 smithi129 ceph-mon[101316]: osdmap e1109: 8 total, 8 up, 8 in 2024-09-14T04:31:55.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:55 smithi039 ceph-mon[106470]: osdmap e1109: 8 total, 8 up, 8 in 2024-09-14T04:31:55.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:55 smithi039 ceph-mon[111235]: osdmap e1109: 8 total, 8 up, 8 in 2024-09-14T04:31:56.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:56 smithi129 ceph-mon[101316]: osdmap e1110: 8 total, 8 up, 8 in 2024-09-14T04:31:56.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:56 smithi129 ceph-mon[101316]: pgmap v1749: 105 pgs: 105 active+clean; 327 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.7 MiB/s wr, 7 op/s 2024-09-14T04:31:56.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:56 smithi039 ceph-mon[106470]: osdmap e1110: 8 total, 8 up, 8 in 2024-09-14T04:31:56.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:56 smithi039 ceph-mon[106470]: pgmap v1749: 105 pgs: 105 active+clean; 327 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.7 MiB/s wr, 7 op/s 2024-09-14T04:31:56.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:56 smithi039 ceph-mon[111235]: osdmap e1110: 8 total, 8 up, 8 in 2024-09-14T04:31:56.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:56 smithi039 ceph-mon[111235]: pgmap v1749: 105 pgs: 105 active+clean; 327 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 3.7 KiB/s rd, 2.7 MiB/s wr, 7 op/s 2024-09-14T04:31:57.672 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:57 smithi129 ceph-mon[101316]: osdmap e1111: 8 total, 8 up, 8 in 2024-09-14T04:31:57.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:57 smithi039 ceph-mon[106470]: osdmap e1111: 8 total, 8 up, 8 in 2024-09-14T04:31:57.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:57 smithi039 ceph-mon[111235]: osdmap e1111: 8 total, 8 up, 8 in 2024-09-14T04:31:58.439 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/0.DiffIterateDiscard (3038 ms) 2024-09-14T04:31:58.439 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/0.DiffIterateTruncate 2024-09-14T04:31:58.440 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:31:58.689 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:58 smithi129 ceph-mon[101316]: pgmap v1751: 105 pgs: 105 active+clean; 331 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 507 B/s rd, 1018 KiB/s wr, 6 op/s 2024-09-14T04:31:58.690 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:58 smithi129 ceph-mon[101316]: osdmap e1112: 8 total, 8 up, 8 in 2024-09-14T04:31:58.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:58 smithi039 ceph-mon[106470]: pgmap v1751: 105 pgs: 105 active+clean; 331 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 507 B/s rd, 1018 KiB/s wr, 6 op/s 2024-09-14T04:31:58.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:58 smithi039 ceph-mon[106470]: osdmap e1112: 8 total, 8 up, 8 in 2024-09-14T04:31:58.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:58 smithi039 ceph-mon[111235]: pgmap v1751: 105 pgs: 105 active+clean; 331 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 507 B/s rd, 1018 KiB/s wr, 6 op/s 2024-09-14T04:31:58.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:58 smithi039 ceph-mon[111235]: osdmap e1112: 8 total, 8 up, 8 in 2024-09-14T04:31:58.791 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:31:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:31:58] "GET /metrics HTTP/1.1" 200 36688 "" "Prometheus/2.51.0" 2024-09-14T04:31:59.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:31:59 smithi129 ceph-mon[101316]: osdmap e1113: 8 total, 8 up, 8 in 2024-09-14T04:31:59.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:31:59 smithi039 ceph-mon[106470]: osdmap e1113: 8 total, 8 up, 8 in 2024-09-14T04:31:59.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:31:59 smithi039 ceph-mon[111235]: osdmap e1113: 8 total, 8 up, 8 in 2024-09-14T04:32:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:00 smithi129 ceph-mon[101316]: pgmap v1754: 105 pgs: 105 active+clean; 335 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 64 KiB/s rd, 14 KiB/s wr, 100 op/s 2024-09-14T04:32:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:00 smithi129 ceph-mon[101316]: osdmap e1114: 8 total, 8 up, 8 in 2024-09-14T04:32:00.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:00 smithi039 ceph-mon[106470]: pgmap v1754: 105 pgs: 105 active+clean; 335 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 64 KiB/s rd, 14 KiB/s wr, 100 op/s 2024-09-14T04:32:00.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:00 smithi039 ceph-mon[106470]: osdmap e1114: 8 total, 8 up, 8 in 2024-09-14T04:32:00.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:00 smithi039 ceph-mon[111235]: pgmap v1754: 105 pgs: 105 active+clean; 335 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 64 KiB/s rd, 14 KiB/s wr, 100 op/s 2024-09-14T04:32:00.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:00 smithi039 ceph-mon[111235]: osdmap e1114: 8 total, 8 up, 8 in 2024-09-14T04:32:01.527 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/0.DiffIterateTruncate (3087 ms) 2024-09-14T04:32:01.527 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/0.DiffIterateWriteAndTruncate 2024-09-14T04:32:01.527 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:32:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:01 smithi129 ceph-mon[101316]: osdmap e1115: 8 total, 8 up, 8 in 2024-09-14T04:32:01.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:01 smithi039 ceph-mon[106470]: osdmap e1115: 8 total, 8 up, 8 in 2024-09-14T04:32:01.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:01 smithi039 ceph-mon[111235]: osdmap e1115: 8 total, 8 up, 8 in 2024-09-14T04:32:02.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:02 smithi129 ceph-mon[101316]: pgmap v1757: 105 pgs: 105 active+clean; 335 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 64 KiB/s rd, 12 KiB/s wr, 95 op/s 2024-09-14T04:32:02.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:02 smithi129 ceph-mon[101316]: osdmap e1116: 8 total, 8 up, 8 in 2024-09-14T04:32:02.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:02 smithi039 ceph-mon[106470]: pgmap v1757: 105 pgs: 105 active+clean; 335 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 64 KiB/s rd, 12 KiB/s wr, 95 op/s 2024-09-14T04:32:02.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:02 smithi039 ceph-mon[106470]: osdmap e1116: 8 total, 8 up, 8 in 2024-09-14T04:32:02.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:02 smithi039 ceph-mon[111235]: pgmap v1757: 105 pgs: 105 active+clean; 335 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 64 KiB/s rd, 12 KiB/s wr, 95 op/s 2024-09-14T04:32:02.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:02 smithi039 ceph-mon[111235]: osdmap e1116: 8 total, 8 up, 8 in 2024-09-14T04:32:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:03 smithi129 ceph-mon[101316]: osdmap e1117: 8 total, 8 up, 8 in 2024-09-14T04:32:03.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:03 smithi039 ceph-mon[106470]: osdmap e1117: 8 total, 8 up, 8 in 2024-09-14T04:32:03.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:03 smithi039 ceph-mon[111235]: osdmap e1117: 8 total, 8 up, 8 in 2024-09-14T04:32:04.550 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/0.DiffIterateWriteAndTruncate (3023 ms) 2024-09-14T04:32:04.550 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/0.DiffIterateStress 2024-09-14T04:32:04.550 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:32:04.586 INFO:tasks.workunit.client.0.smithi039.stdout: i=0 exists [37646563~400097,88204619~2080603,91239181~3784160,111630227~2186380,131651550~8027031,141457300~1259719,227720383~1883594,284005579~2154065,396114200~451861] wrote [37646563~400097,88204619~2080603,91239181~3784160,111630227~2186380,131651550~8027031,141457300~1259719,227720383~1883594,284005579~2154065,396114200~451861] 2024-09-14T04:32:04.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:04 smithi129 ceph-mon[101316]: pgmap v1760: 105 pgs: 105 active+clean; 335 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50 KiB/s rd, 7.2 KiB/s wr, 78 op/s 2024-09-14T04:32:04.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:04 smithi129 ceph-mon[101316]: osdmap e1118: 8 total, 8 up, 8 in 2024-09-14T04:32:04.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:04 smithi039 ceph-mon[106470]: pgmap v1760: 105 pgs: 105 active+clean; 335 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50 KiB/s rd, 7.2 KiB/s wr, 78 op/s 2024-09-14T04:32:04.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:04 smithi039 ceph-mon[106470]: osdmap e1118: 8 total, 8 up, 8 in 2024-09-14T04:32:04.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:04 smithi039 ceph-mon[111235]: pgmap v1760: 105 pgs: 105 active+clean; 335 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 50 KiB/s rd, 7.2 KiB/s wr, 78 op/s 2024-09-14T04:32:04.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:04 smithi039 ceph-mon[111235]: osdmap e1118: 8 total, 8 up, 8 in 2024-09-14T04:32:05.619 INFO:tasks.workunit.client.0.smithi039.stdout: i=1 exists [24532549~4433745,31686399~8868921,49625492~2822234,88204619~6818722,111630227~2186380,131651550~8027031,141457300~1259719,227720383~1883594,260868422~6932272,284005579~2154065,317734509~4630977,361205710~1955910,396114200~451861] wrote [24532549~4433745,31686399~8868921,49625492~2822234,89359629~3636978,260868422~6932272,317734509~4630977,361205710~1955910] 2024-09-14T04:32:05.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:05 smithi039 ceph-mon[106470]: osdmap e1119: 8 total, 8 up, 8 in 2024-09-14T04:32:05.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:05 smithi039 ceph-mon[111235]: osdmap e1119: 8 total, 8 up, 8 in 2024-09-14T04:32:06.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:05 smithi129 ceph-mon[101316]: osdmap e1119: 8 total, 8 up, 8 in 2024-09-14T04:32:06.628 INFO:tasks.workunit.client.0.smithi039.stdout: i=2 exists [24532549~4433745,31686399~8868921,49625492~2822234,88204619~6818722,111630227~2186380,114085331~2130967,131651550~8027031,141457300~1259719,227720383~1883594,234445135~3605828,260868422~6932272,282924374~6084694,292332323~11963618,306936351~4352606,317734509~4630977,360584803~2576817,396114200~451861,403429066~7830275] wrote [114085331~2130967,234445135~3605828,282924374~6084694,292332323~11963618,306936351~4352606,360584803~2356626,403429066~7830275] 2024-09-14T04:32:06.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:06 smithi039 ceph-mon[106470]: pgmap v1763: 105 pgs: 105 active+clean; 336 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 69 KiB/s rd, 264 KiB/s wr, 109 op/s 2024-09-14T04:32:06.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:06 smithi039 ceph-mon[106470]: osdmap e1120: 8 total, 8 up, 8 in 2024-09-14T04:32:06.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:06 smithi039 ceph-mon[111235]: pgmap v1763: 105 pgs: 105 active+clean; 336 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 69 KiB/s rd, 264 KiB/s wr, 109 op/s 2024-09-14T04:32:06.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:06 smithi039 ceph-mon[111235]: osdmap e1120: 8 total, 8 up, 8 in 2024-09-14T04:32:07.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:06 smithi129 ceph-mon[101316]: pgmap v1763: 105 pgs: 105 active+clean; 336 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 69 KiB/s rd, 264 KiB/s wr, 109 op/s 2024-09-14T04:32:07.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:06 smithi129 ceph-mon[101316]: osdmap e1120: 8 total, 8 up, 8 in 2024-09-14T04:32:07.648 INFO:tasks.workunit.client.0.smithi039.stdout: i=3 exists [24532549~4433745,31686399~8868921,49625492~2822234,81217002~1396733,88204619~6818722,111630227~2186380,114085331~2130967,131651550~8027031,141457300~1259719,145214205~2726028,167034219~6779119,204206046~5919797,227720383~1883594,234445135~3605828,260868422~8920602,282924374~6084694,292332323~11963618,306936351~4352606,311792749~10572737,330947088~803453,342666453~5582380,360584803~2576817,376743309~7871673,396114200~451861,403429066~7830275] wrote [81217002~1396733,145214205~2726028,167034219~6779119,204206046~5919797,262905843~6883181,311792749~7730750,330947088~803453,342666453~5582380,376743309~7871673,406171321~4595128] 2024-09-14T04:32:07.951 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:07 smithi039 ceph-mon[106470]: osdmap e1121: 8 total, 8 up, 8 in 2024-09-14T04:32:07.951 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:32:07.951 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:07 smithi039 ceph-mon[111235]: osdmap e1121: 8 total, 8 up, 8 in 2024-09-14T04:32:07.951 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:32:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:07 smithi129 ceph-mon[101316]: osdmap e1121: 8 total, 8 up, 8 in 2024-09-14T04:32:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:32:08.648 INFO:tasks.workunit.client.0.smithi039.stdout: i=4 exists [24532549~4433745,31686399~8868921,42887289~3072128,49625492~10462063,81217002~1396733,88204619~6818722,111630227~2186380,114085331~2130967,131651550~8027031,141457300~1259719,145214205~2726028,167034219~6779119,204206046~5919797,210193421~7415433,227720383~1883594,234445135~3605828,251644403~5938599,258312268~1647698,260868422~8920602,282924374~7610022,292332323~11963618,306936351~4352606,311792749~10572737,328959633~4415758,342666453~5582380,358912822~7718906,376743309~7871673,389122080~4380744,396114200~451861,403429066~7830275] wrote [42887289~3072128,51969188~8118367,89587254~5233203,210193421~7415433,251644403~5938599,258312268~1647698,285482181~5052215,328959633~4415758,358912822~7718906,389122080~4380744] 2024-09-14T04:32:08.982 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:08 smithi039 ceph-mon[106470]: pgmap v1766: 105 pgs: 105 active+clean; 350 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 2.9 MiB/s wr, 48 op/s 2024-09-14T04:32:08.982 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:08 smithi039 ceph-mon[106470]: osdmap e1122: 8 total, 8 up, 8 in 2024-09-14T04:32:08.983 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:32:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:32:08] "GET /metrics HTTP/1.1" 200 36689 "" "Prometheus/2.51.0" 2024-09-14T04:32:08.983 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:08 smithi039 ceph-mon[111235]: pgmap v1766: 105 pgs: 105 active+clean; 350 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 2.9 MiB/s wr, 48 op/s 2024-09-14T04:32:08.983 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:08 smithi039 ceph-mon[111235]: osdmap e1122: 8 total, 8 up, 8 in 2024-09-14T04:32:09.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:08 smithi129 ceph-mon[101316]: pgmap v1766: 105 pgs: 105 active+clean; 350 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 2.9 MiB/s wr, 48 op/s 2024-09-14T04:32:09.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:08 smithi129 ceph-mon[101316]: osdmap e1122: 8 total, 8 up, 8 in 2024-09-14T04:32:09.661 INFO:tasks.workunit.client.0.smithi039.stdout: i=5 exists [2471184~3081596,24366452~16188868,42887289~3072128,49625492~10462063,81217002~1396733,88204619~6818722,110037204~9726671,131651550~8027031,141457300~1259719,144649336~3290897,167034219~6779119,178985346~3361728,183215660~3502696,204206046~5919797,210193421~7415433,227720383~1883594,234445135~3605828,251644403~5938599,258312268~1647698,260868422~8920602,282924374~7610022,292332323~11963618,306936351~4352606,311792749~10572737,328959633~4415758,342666453~5582380,358912822~7718906,376743309~7871673,389122080~4380744,396114200~451861,403429066~7830275] wrote [2471184~3081596,24366452~9505140,110037204~9726671,144649336~641416,167109705~3086191,178985346~3361728,183215660~3502696,206380204~2324485] 2024-09-14T04:32:09.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:09 smithi129 ceph-mon[101316]: osdmap e1123: 8 total, 8 up, 8 in 2024-09-14T04:32:10.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:09 smithi039 ceph-mon[106470]: osdmap e1123: 8 total, 8 up, 8 in 2024-09-14T04:32:10.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:09 smithi039 ceph-mon[111235]: osdmap e1123: 8 total, 8 up, 8 in 2024-09-14T04:32:10.663 INFO:tasks.workunit.client.0.smithi039.stdout: i=6 exists [2471184~3081596,18815204~3919713,24366452~16188868,42887289~3072128,49625492~10462063,81217002~1396733,88204619~6818722,110037204~9726671,129593184~10085397,141457300~1259719,144649336~3290897,167034219~6779119,177156787~5190287,183215660~3502696,204206046~5919797,210193421~7415433,227720383~1883594,229976328~4454459,234445135~3605828,240560472~1907077,251644403~5938599,258312268~1647698,260868422~8920602,282924374~7610022,292332323~11963618,306936351~4352606,311792749~10572737,328959633~4415758,342666453~5582380,352926650~13705078,376743309~7871673,389122080~4380744,396114200~451861,403429066~7830275] wrote [18815204~3919713,129593184~3289710,177156787~1880366,207366448~1572094,229976328~4454459,240560472~1907077,254016000~1071233,261607553~1789794,352926650~10870250] 2024-09-14T04:32:11.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:10 smithi129 ceph-mon[101316]: pgmap v1769: 105 pgs: 105 active+clean; 505 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 127 KiB/s rd, 32 MiB/s wr, 219 op/s 2024-09-14T04:32:11.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:10 smithi129 ceph-mon[101316]: osdmap e1124: 8 total, 8 up, 8 in 2024-09-14T04:32:11.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:10 smithi039 ceph-mon[106470]: pgmap v1769: 105 pgs: 105 active+clean; 505 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 127 KiB/s rd, 32 MiB/s wr, 219 op/s 2024-09-14T04:32:11.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:10 smithi039 ceph-mon[106470]: osdmap e1124: 8 total, 8 up, 8 in 2024-09-14T04:32:11.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:10 smithi039 ceph-mon[111235]: pgmap v1769: 105 pgs: 105 active+clean; 505 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 127 KiB/s rd, 32 MiB/s wr, 219 op/s 2024-09-14T04:32:11.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:10 smithi039 ceph-mon[111235]: osdmap e1124: 8 total, 8 up, 8 in 2024-09-14T04:32:11.660 INFO:tasks.workunit.client.0.smithi039.stdout: i=7 exists [1560185~3992595,18815204~3919713,24366452~16188868,42887289~3072128,49625492~10462063,81217002~1396733,88204619~6818722,108930368~10833507,129593184~10085397,141457300~1259719,144649336~3290897,167034219~6779119,177156787~5190287,183215660~3502696,204206046~5919797,210193421~7415433,227037563~7393224,234445135~3605828,240560472~1907077,251644403~5938599,258312268~1647698,260868422~8920602,282924374~7610022,292332323~11963618,304604600~522118,306936351~4352606,311792749~10572737,328959633~4415758,339416815~2817899,342666453~5582380,352926650~13705078,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396114200~451861,403429066~7830275] wrote [1560185~1377854,28814760~445192,108930368~6044747,118756847~532421,227037563~3194028,300028752~3863222,304604600~522118,339416815~2817899,367856693~3247417,371654262~71789] 2024-09-14T04:32:12.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:11 smithi129 ceph-mon[101316]: osdmap e1125: 8 total, 8 up, 8 in 2024-09-14T04:32:12.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:11 smithi039 ceph-mon[106470]: osdmap e1125: 8 total, 8 up, 8 in 2024-09-14T04:32:12.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:11 smithi039 ceph-mon[111235]: osdmap e1125: 8 total, 8 up, 8 in 2024-09-14T04:32:12.742 INFO:tasks.workunit.client.0.smithi039.stdout: i=8 exists [1560185~3992595,18815204~3919713,24366452~16188868,42821576~5427684,49625492~10874967,81217002~1396733,88204619~6818722,108930368~10833507,125750260~3113143,129593184~10085397,141457300~1259719,144649336~3290897,167034219~6779119,177156787~5190287,183215660~3502696,204206046~5919797,210193421~7415433,227037563~7393224,234445135~3605828,240560472~1907077,251644403~5938599,258312268~1647698,260868422~8920602,271903192~485755,282924374~7610022,292332323~11963618,304604600~522118,306936351~18977152,328959633~4415758,339416815~2817899,342666453~9554642,352231748~211803,352926650~13705078,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396114200~451861,396800802~6077541,403429066~9121060] wrote [42821576~5427684,56498898~4001561,125750260~3113143,271903192~485755,311272896~6368811,319011790~6901713,348104878~4116217,352231748~211803,396800802~6077541,410335706~2214420] 2024-09-14T04:32:13.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:12 smithi129 ceph-mon[101316]: pgmap v1772: 105 pgs: 105 active+clean; 510 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 121 KiB/s rd, 31 MiB/s wr, 202 op/s 2024-09-14T04:32:13.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:12 smithi129 ceph-mon[101316]: osdmap e1126: 8 total, 8 up, 8 in 2024-09-14T04:32:13.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:12 smithi039 ceph-mon[106470]: pgmap v1772: 105 pgs: 105 active+clean; 510 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 121 KiB/s rd, 31 MiB/s wr, 202 op/s 2024-09-14T04:32:13.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:12 smithi039 ceph-mon[106470]: osdmap e1126: 8 total, 8 up, 8 in 2024-09-14T04:32:13.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:12 smithi039 ceph-mon[111235]: pgmap v1772: 105 pgs: 105 active+clean; 510 MiB data, 1.6 GiB used, 714 GiB / 715 GiB avail; 121 KiB/s rd, 31 MiB/s wr, 202 op/s 2024-09-14T04:32:13.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:12 smithi039 ceph-mon[111235]: osdmap e1126: 8 total, 8 up, 8 in 2024-09-14T04:32:13.719 INFO:tasks.workunit.client.0.smithi039.stdout: i=9 exists [1560185~3992595,18815204~3919713,24366452~16188868,42821576~5427684,49625492~10874967,77103770~678283,81217002~1396733,87561278~7462063,102891979~16871896,125750260~3113143,129593184~10085397,141457300~1259719,144649336~3290897,167034219~6779119,177156787~5609834,183215660~3502696,204206046~5919797,210193421~7415433,227037563~7393224,234445135~3605828,240560472~1907077,251644403~5938599,258312268~1647698,260868422~8920602,271903192~485755,282924374~7610022,292332323~33581180,328959633~4415758,339416815~2817899,342666453~9554642,352231748~211803,352926650~13705078,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396114200~451861,396800802~6077541,403429066~9121060] wrote [33861843~922299,77103770~678283,87561278~3303988,102891979~6289219,113214585~112979,178708911~4057710,303295189~4263431,318925599~4902066,343616361~175529,377219330~1842760] 2024-09-14T04:32:14.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:13 smithi129 ceph-mon[101316]: osdmap e1127: 8 total, 8 up, 8 in 2024-09-14T04:32:14.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:13 smithi039 ceph-mon[106470]: osdmap e1127: 8 total, 8 up, 8 in 2024-09-14T04:32:14.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:13 smithi039 ceph-mon[111235]: osdmap e1127: 8 total, 8 up, 8 in 2024-09-14T04:32:14.769 INFO:tasks.workunit.client.0.smithi039.stdout: i=10 exists [1560185~3992595,18815204~3919713,24366452~23882808,49625492~10874967,77103770~678283,79792402~6496993,87561278~7462063,102891979~20007158,125750260~3113143,129593184~10085397,141457300~1259719,144649336~7944386,167034219~6779119,177156787~5609834,183215660~3502696,204206046~5919797,210193421~7415433,227037563~7393224,234445135~3605828,240560472~1907077,244100700~3181961,251644403~18144621,271903192~485755,277143881~13390515,292332323~33581180,328959633~4415758,339416815~2817899,342666453~9554642,352231748~211803,352926650~13705078,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396114200~451861,396800802~6077541,403429066~9121060] wrote [35645530~2311397,39500721~8119064,79792402~6496993,88623968~370312,117127695~5771442,145121240~7472482,244100700~3181961,256962846~4050912,277143881~6968355,311587502~4444723] 2024-09-14T04:32:15.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:14 smithi129 ceph-mon[101316]: pgmap v1775: 105 pgs: 105 active+clean; 704 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 162 KiB/s rd, 42 MiB/s wr, 264 op/s 2024-09-14T04:32:15.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:14 smithi129 ceph-mon[101316]: osdmap e1128: 8 total, 8 up, 8 in 2024-09-14T04:32:15.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:14 smithi039 ceph-mon[106470]: pgmap v1775: 105 pgs: 105 active+clean; 704 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 162 KiB/s rd, 42 MiB/s wr, 264 op/s 2024-09-14T04:32:15.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:14 smithi039 ceph-mon[106470]: osdmap e1128: 8 total, 8 up, 8 in 2024-09-14T04:32:15.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:14 smithi039 ceph-mon[111235]: pgmap v1775: 105 pgs: 105 active+clean; 704 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 162 KiB/s rd, 42 MiB/s wr, 264 op/s 2024-09-14T04:32:15.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:14 smithi039 ceph-mon[111235]: osdmap e1128: 8 total, 8 up, 8 in 2024-09-14T04:32:15.752 INFO:tasks.workunit.client.0.smithi039.stdout: i=11 exists [1532502~4624507,18815204~3919713,24366452~23882808,49625492~10874967,69873797~1502064,77103770~678283,79792402~6496993,87561278~7462063,99264373~856643,102891979~20007158,125750260~3113143,129593184~10085397,141457300~1259719,144649336~7944386,167034219~6779119,177156787~5609834,183215660~3502696,204206046~13402808,227037563~7393224,234445135~3605828,240560472~1907077,244100700~3181961,251644403~18144621,271903192~485755,277143881~13390515,292332323~33581180,328959633~4415758,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396114200~451861,396800802~6077541,403429066~9121060] wrote [1532502~4624507,40281884~3184041,69873797~1502064,99264373~856643,209917978~1359027,297545811~6956556,344830868~904290,350270295~5576933,378366520~4061648] 2024-09-14T04:32:16.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:15 smithi129 ceph-mon[101316]: osdmap e1129: 8 total, 8 up, 8 in 2024-09-14T04:32:16.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:15 smithi039 ceph-mon[106470]: osdmap e1129: 8 total, 8 up, 8 in 2024-09-14T04:32:16.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:15 smithi039 ceph-mon[111235]: osdmap e1129: 8 total, 8 up, 8 in 2024-09-14T04:32:16.723 INFO:tasks.workunit.client.0.smithi039.stdout: i=12 exists [1532502~4624507,18815204~3919713,24366452~23951097,49625492~10874967,65187704~6913593,77103770~678283,79792402~6496993,87561278~7462063,99264373~856643,102891979~20007158,125750260~3113143,129593184~10085397,141457300~1259719,144649336~7944386,164144631~755755,167034219~7005812,177156787~5609834,183215660~3502696,204206046~13402808,227037563~7393224,234445135~3605828,240560472~1907077,244100700~3181961,251644403~18144621,271903192~485755,277143881~13390515,291130899~619944,292332323~33581180,328615132~4760259,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396114200~451861,396800802~6077541,403429066~9121060] wrote [46171736~2145813,65187704~6913593,91960020~945991,134282812~4188311,141461050~1072987,164144631~755755,168867811~5172220,291130899~619944,315889099~2662551,328615132~3469460] 2024-09-14T04:32:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:16 smithi129 ceph-mon[101316]: pgmap v1778: 105 pgs: 105 active+clean; 725 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 162 KiB/s rd, 44 MiB/s wr, 266 op/s 2024-09-14T04:32:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:16 smithi129 ceph-mon[101316]: osdmap e1130: 8 total, 8 up, 8 in 2024-09-14T04:32:17.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:16 smithi039 ceph-mon[106470]: pgmap v1778: 105 pgs: 105 active+clean; 725 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 162 KiB/s rd, 44 MiB/s wr, 266 op/s 2024-09-14T04:32:17.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:16 smithi039 ceph-mon[106470]: osdmap e1130: 8 total, 8 up, 8 in 2024-09-14T04:32:17.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:16 smithi039 ceph-mon[111235]: pgmap v1778: 105 pgs: 105 active+clean; 725 MiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 162 KiB/s rd, 44 MiB/s wr, 266 op/s 2024-09-14T04:32:17.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:16 smithi039 ceph-mon[111235]: osdmap e1130: 8 total, 8 up, 8 in 2024-09-14T04:32:17.774 INFO:tasks.workunit.client.0.smithi039.stdout: i=13 exists [1532502~4624507,18815204~3919713,24366452~24816062,49625492~10874967,65187704~6913593,77103770~678283,79792402~6496993,87561278~7462063,99264373~856643,102891979~20007158,125750260~3113143,129593184~10085397,141457300~1259719,144649336~10643842,155839505~3780252,164144631~755755,167034219~7005812,177156787~5609834,183215660~3502696,201412400~16196454,218384295~132186,225312546~9118241,234445135~3605828,240560472~7894340,249153849~20635175,271903192~485755,277143881~13390515,291130899~619944,292332323~33581180,328615132~4760259,338917256~193834,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396114200~451861,396800802~6077541,403429066~9121060] wrote [41735170~7447344,148411414~6881764,155839505~3780252,201412400~4186671,218384295~132186,225312546~5051871,242085844~6368968,249153849~4403906,338917256~193834,378397449~5799119] 2024-09-14T04:32:18.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:17 smithi129 ceph-mon[101316]: osdmap e1131: 8 total, 8 up, 8 in 2024-09-14T04:32:18.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:17 smithi039 ceph-mon[106470]: osdmap e1131: 8 total, 8 up, 8 in 2024-09-14T04:32:18.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:17 smithi039 ceph-mon[111235]: osdmap e1131: 8 total, 8 up, 8 in 2024-09-14T04:32:18.702 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:32:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:32:18] "GET /metrics HTTP/1.1" 200 36689 "" "Prometheus/2.51.0" 2024-09-14T04:32:18.802 INFO:tasks.workunit.client.0.smithi039.stdout: i=14 exists [1532502~4624507,18815204~3919713,24366452~24816062,49625492~10874967,65187704~6913593,77103770~678283,79792402~6496993,87561278~7462063,99264373~856643,102891979~20007158,125750260~3113143,129593184~10085397,141457300~1259719,144649336~10643842,155839505~3780252,164144631~755755,167034219~7005812,177156787~5609834,183215660~7587630,201412400~16196454,218384295~132186,225312546~9118241,234445135~14009677,249153849~20635175,271903192~485755,277143881~48769622,328615132~4760259,338917256~193834,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396114200~451861,396800802~6077541,403429066~9121060] wrote [37722924~6209571,89021763~2406992,183524730~7278560,237089596~5877557,253979221~3341796,262851191~6527342,284870704~8098864,307097205~6187809] 2024-09-14T04:32:19.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:18 smithi129 ceph-mon[101316]: pgmap v1781: 105 pgs: 105 active+clean; 766 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s wr, 7 op/s 2024-09-14T04:32:19.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:18 smithi129 ceph-mon[101316]: osdmap e1132: 8 total, 8 up, 8 in 2024-09-14T04:32:19.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:18 smithi039 ceph-mon[106470]: pgmap v1781: 105 pgs: 105 active+clean; 766 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s wr, 7 op/s 2024-09-14T04:32:19.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:18 smithi039 ceph-mon[106470]: osdmap e1132: 8 total, 8 up, 8 in 2024-09-14T04:32:19.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:18 smithi039 ceph-mon[111235]: pgmap v1781: 105 pgs: 105 active+clean; 766 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 13 MiB/s wr, 7 op/s 2024-09-14T04:32:19.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:18 smithi039 ceph-mon[111235]: osdmap e1132: 8 total, 8 up, 8 in 2024-09-14T04:32:19.812 INFO:tasks.workunit.client.0.smithi039.stdout: i=15 exists [1532502~4624507,18815204~3919713,24366452~24816062,49625492~13014907,65187704~6913593,77103770~678283,79792402~6496993,87561278~7462063,99264373~856643,102891979~20007158,125750260~3113143,129593184~25699994,155839505~3780252,164144631~755755,167034219~7005812,177156787~5609834,183215660~7587630,201412400~16196454,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~20635175,271903192~485755,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396114200~451861,396800802~6077541,403429066~9121060] wrote [55349631~7290768,112894852~9167908,139378274~7402656,207369398~3608367,222441104~495517,276825837~452599,307864787~4327691,347886358~1792403] 2024-09-14T04:32:20.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:19 smithi129 ceph-mon[101316]: osdmap e1133: 8 total, 8 up, 8 in 2024-09-14T04:32:20.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:19 smithi039 ceph-mon[106470]: osdmap e1133: 8 total, 8 up, 8 in 2024-09-14T04:32:20.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:19 smithi039 ceph-mon[111235]: osdmap e1133: 8 total, 8 up, 8 in 2024-09-14T04:32:20.819 INFO:tasks.workunit.client.0.smithi039.stdout: i=16 exists [1532502~4624507,18815204~3919713,24366452~24816062,49625492~13014907,65187704~6913593,77103770~678283,79792402~6496993,87561278~7462063,99264373~856643,102891979~20007158,125750260~3113143,129593184~25699994,155839505~11131128,167034219~7005812,177156787~5609834,183215660~7587630,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~20635175,271903192~485755,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~28437657,371654262~71789,376743309~7871673,389122080~4380744,396114200~451861,396800802~6077541,403429066~9121060] wrote [53772129~974098,110614496~2472756,132996120~2690483,155908264~11062369,195446529~6684577,257815293~3393375,286210107~4006077,332042044~606164,360827666~8007252] 2024-09-14T04:32:21.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:20 smithi129 ceph-mon[101316]: pgmap v1784: 105 pgs: 105 active+clean; 899 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 192 KiB/s rd, 39 MiB/s wr, 306 op/s 2024-09-14T04:32:21.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:20 smithi129 ceph-mon[101316]: osdmap e1134: 8 total, 8 up, 8 in 2024-09-14T04:32:21.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:20 smithi039 ceph-mon[106470]: pgmap v1784: 105 pgs: 105 active+clean; 899 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 192 KiB/s rd, 39 MiB/s wr, 306 op/s 2024-09-14T04:32:21.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:20 smithi039 ceph-mon[106470]: osdmap e1134: 8 total, 8 up, 8 in 2024-09-14T04:32:21.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:20 smithi039 ceph-mon[111235]: pgmap v1784: 105 pgs: 105 active+clean; 899 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 192 KiB/s rd, 39 MiB/s wr, 306 op/s 2024-09-14T04:32:21.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:20 smithi039 ceph-mon[111235]: osdmap e1134: 8 total, 8 up, 8 in 2024-09-14T04:32:21.856 INFO:tasks.workunit.client.0.smithi039.stdout: i=17 exists [1532502~4624507,18815204~3919713,24366452~24816062,49625492~15162845,65187704~6913593,77103770~678283,79792402~6496993,87561278~7462063,99264373~856643,102891979~20007158,125750260~3113143,129593184~37377449,167034219~7005812,177156787~5609834,183215660~7587630,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271903192~485755,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~7871673,389122080~4380744,396114200~451861,396800802~6077541,403429066~9121060] wrote [50026379~3466398,55475851~9312486,111778834~3517091,150858815~5451956,156634296~3077940,262779491~7688249,317400597~2551594,370420162~6009885] 2024-09-14T04:32:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:21 smithi129 ceph-mon[101316]: osdmap e1135: 8 total, 8 up, 8 in 2024-09-14T04:32:22.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:21 smithi039 ceph-mon[106470]: osdmap e1135: 8 total, 8 up, 8 in 2024-09-14T04:32:22.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:21 smithi039 ceph-mon[111235]: osdmap e1135: 8 total, 8 up, 8 in 2024-09-14T04:32:22.846 INFO:tasks.workunit.client.0.smithi039.stdout: i=18 exists [1532502~4624507,18815204~3919713,24366452~24816062,49625492~15162845,65187704~13622813,79792402~6496993,87561278~7462063,99264373~856643,102891979~20007158,125750260~3113143,129593184~44446847,177156787~5609834,183215660~7587630,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~7871673,389122080~4380744,396114200~451861,396800802~6077541,403429066~9121060] wrote [41504883~1986423,71362387~7448130,109021991~3523985,152118700~6330839,164724774~1239558,166279969~7210228,202395118~3947699,255161782~6019745,271309101~3402633,390011506~750434] 2024-09-14T04:32:23.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:22 smithi129 ceph-mon[101316]: pgmap v1787: 105 pgs: 105 active+clean; 899 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 192 KiB/s rd, 30 MiB/s wr, 301 op/s 2024-09-14T04:32:23.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:22 smithi129 ceph-mon[101316]: osdmap e1136: 8 total, 8 up, 8 in 2024-09-14T04:32:23.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:32:23.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:22 smithi039 ceph-mon[106470]: pgmap v1787: 105 pgs: 105 active+clean; 899 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 192 KiB/s rd, 30 MiB/s wr, 301 op/s 2024-09-14T04:32:23.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:22 smithi039 ceph-mon[106470]: osdmap e1136: 8 total, 8 up, 8 in 2024-09-14T04:32:23.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:32:23.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:22 smithi039 ceph-mon[111235]: pgmap v1787: 105 pgs: 105 active+clean; 899 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 192 KiB/s rd, 30 MiB/s wr, 301 op/s 2024-09-14T04:32:23.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:22 smithi039 ceph-mon[111235]: osdmap e1136: 8 total, 8 up, 8 in 2024-09-14T04:32:23.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:32:23.842 INFO:tasks.workunit.client.0.smithi039.stdout: i=19 exists [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7462063,99264373~856643,102891979~20007158,125750260~3113143,129593184~46565631,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396114200~451861,396800802~6077541,403429066~13497595] wrote [15956065~5397876,77848113~2942085,169499290~6659525,189388097~2921176,192662147~2659618,231412373~2251966,258794170~5803514,324417134~321003,384246345~6869073,409194704~7731957] 2024-09-14T04:32:24.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:23 smithi039 ceph-mon[106470]: osdmap e1137: 8 total, 8 up, 8 in 2024-09-14T04:32:24.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:23 smithi039 ceph-mon[111235]: osdmap e1137: 8 total, 8 up, 8 in 2024-09-14T04:32:24.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:23 smithi129 ceph-mon[101316]: osdmap e1137: 8 total, 8 up, 8 in 2024-09-14T04:32:24.804 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 1 diff [24532549~4433745,31686399~8868921,49625492~2822234,89359629~3636978,260868422~6932272,317734509~4630977,361205710~1955910] 2024-09-14T04:32:24.804 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [24532549~4433745,31686399~8868921,49625492~2822234,89359629~3636978,260868422~6932272,317734509~4630977,361205710~1955910] 2024-09-14T04:32:24.824 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [20971520~7994774,29360128~11195192,46137344~6310382,89359629~3636978,260046848~7753846,314572800~7792686,360710144~2451476] 2024-09-14T04:32:24.824 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 2 diff [24532549~4433745,31686399~8868921,49625492~2822234,89359629~3636978,114085331~2130967,234445135~3605828,260868422~6932272,282924374~6084694,292332323~11963618,306936351~4352606,317734509~4630977,360584803~2576817,403429066~7830275] 2024-09-14T04:32:24.824 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [24532549~4433745,31686399~8868921,49625492~2822234,89359629~3636978,114085331~2130967,234445135~3605828,260868422~6932272,282924374~6084694,292332323~11963618,306936351~4352606,317734509~4630977,360584803~2576817,403429066~7830275] 2024-09-14T04:32:24.830 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [20971520~7994774,29360128~11195192,46137344~6310382,89359629~3636978,113816607~2399691,230686720~7364243,260046848~7753846,282924374~6084694,289406976~14888965,306184192~5104765,314572800~7792686,356515840~6645780,402653184~8606157] 2024-09-14T04:32:24.831 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 3 diff [24532549~4433745,31686399~8868921,49625492~2822234,81217002~1396733,89359629~3636978,114085331~2130967,145214205~2726028,167034219~6779119,204206046~5919797,234445135~3605828,260868422~8920602,282924374~6084694,292332323~11963618,306936351~4352606,311792749~10572737,330947088~803453,342666453~5582380,360584803~2576817,376743309~7871673,403429066~7830275] 2024-09-14T04:32:24.831 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [24532549~4433745,31686399~8868921,49625492~2822234,81217002~1396733,89359629~3636978,114085331~2130967,145214205~2726028,167034219~6779119,204206046~5919797,234445135~3605828,260868422~8920602,282924374~6084694,292332323~11963618,306936351~4352606,311792749~10572737,330947088~803453,342666453~5582380,360584803~2576817,376743309~7871673,403429066~7830275] 2024-09-14T04:32:24.837 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [20971520~7994774,29360128~11195192,46137344~6310382,79691776~2921959,89359629~3636978,113816607~2399691,142717019~5223214,163577856~10235482,201326592~8799251,230686720~7364243,260046848~9742176,282924374~6084694,289406976~14888965,306184192~16181294,327155712~4594829,339738624~8510209,356515840~6645780,373293056~11321926,402653184~8606157] 2024-09-14T04:32:24.837 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 4 diff [24532549~4433745,31686399~8868921,42887289~3072128,49625492~10462063,81217002~1396733,89359629~5460828,114085331~2130967,145214205~2726028,167034219~6779119,204206046~5919797,210193421~7415433,234445135~3605828,251644403~5938599,258312268~1647698,260868422~8920602,282924374~7610022,292332323~11963618,306936351~4352606,311792749~10572737,328959633~4415758,342666453~5582380,358912822~7718906,376743309~7871673,389122080~4380744,403429066~7830275] 2024-09-14T04:32:24.837 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [24532549~4433745,31686399~8868921,42887289~3072128,49625492~10462063,81217002~1396733,89359629~5460828,114085331~2130967,145214205~2726028,167034219~6779119,204206046~5919797,210193421~7415433,234445135~3605828,251644403~5938599,258312268~1647698,260868422~8920602,282924374~7610022,292332323~11963618,306936351~4352606,311792749~10572737,328959633~4415758,342666453~5582380,358912822~7718906,376743309~7871673,389122080~4380744,403429066~7830275] 2024-09-14T04:32:24.844 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [20971520~7994774,29360128~11195192,41943040~4016377,46137344~13950211,79691776~2921959,89359629~5460828,113816607~2399691,142717019~5223214,163577856~10235482,201326592~16282262,230686720~7364243,247463936~12496030,260046848~9742176,282924374~21371567,306184192~16181294,327155712~6219679,339738624~8510209,356515840~10115888,373293056~11321926,385875968~7626856,402653184~8606157] 2024-09-14T04:32:24.844 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 5 diff [2471184~3081596,24366452~16188868,42887289~3072128,49625492~10462063,81217002~1396733,89359629~5460828,110037204~9726671,144649336~3290897,167034219~6779119,178985346~3361728,183215660~3502696,204206046~5919797,210193421~7415433,234445135~3605828,251644403~5938599,258312268~1647698,260868422~8920602,282924374~7610022,292332323~11963618,306936351~4352606,311792749~10572737,328959633~4415758,342666453~5582380,358912822~7718906,376743309~7871673,389122080~4380744,403429066~7830275] 2024-09-14T04:32:24.844 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [2471184~3081596,24366452~16188868,42887289~3072128,49625492~10462063,81217002~1396733,89359629~5460828,110037204~9726671,144649336~3290897,167034219~6779119,178985346~3361728,183215660~3502696,204206046~5919797,210193421~7415433,234445135~3605828,251644403~5938599,258312268~1647698,260868422~8920602,282924374~7610022,292332323~11963618,306936351~4352606,311792749~10572737,328959633~4415758,342666453~5582380,358912822~7718906,376743309~7871673,389122080~4380744,403429066~7830275] 2024-09-14T04:32:24.850 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~5552780,20971520~19583800,41943040~4016377,46137344~13950211,79691776~2921959,89359629~5460828,110037204~9726671,142717019~5223214,163577856~10235482,176160768~10557588,201326592~16282262,230686720~7364243,247463936~12496030,260046848~9742176,282924374~21371567,306184192~16181294,327155712~6219679,339738624~8510209,356515840~10115888,373293056~11321926,385875968~7626856,402653184~8606157] 2024-09-14T04:32:24.850 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 6 diff [2471184~3081596,18815204~3919713,24366452~16188868,42887289~3072128,49625492~10462063,81217002~1396733,89359629~5460828,110037204~9726671,129593184~3289710,144649336~3290897,167034219~6779119,177156787~5190287,183215660~3502696,204206046~5919797,210193421~7415433,229976328~4454459,234445135~3605828,240560472~1907077,251644403~5938599,258312268~1647698,260868422~8920602,282924374~7610022,292332323~11963618,306936351~4352606,311792749~10572737,328959633~4415758,342666453~5582380,352926650~13705078,376743309~7871673,389122080~4380744,403429066~7830275] 2024-09-14T04:32:24.850 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [2471184~3081596,18815204~3919713,24366452~16188868,42887289~3072128,49625492~10462063,81217002~1396733,89359629~5460828,110037204~9726671,129593184~3289710,144649336~3290897,167034219~6779119,177156787~5190287,183215660~3502696,204206046~5919797,210193421~7415433,229976328~4454459,234445135~3605828,240560472~1907077,251644403~5938599,258312268~1647698,260868422~8920602,282924374~7610022,292332323~11963618,306936351~4352606,311792749~10572737,328959633~4415758,342666453~5582380,352926650~13705078,376743309~7871673,389122080~4380744,403429066~7830275] 2024-09-14T04:32:24.857 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~5552780,16777216~23778104,41943040~4016377,46137344~13950211,79691776~2921959,89359629~5460828,110037204~9726671,125829120~7053774,142717019~5223214,163577856~10235482,176160768~10557588,201326592~16282262,229603977~8446986,239075328~3392221,247463936~12496030,260046848~9742176,282924374~21371567,306184192~16181294,327155712~6219679,339738624~8510209,352321536~14310192,373293056~11321926,385875968~7626856,402653184~8606157] 2024-09-14T04:32:24.857 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 7 diff [1560185~3992595,18815204~3919713,24366452~16188868,42887289~3072128,49625492~10462063,81217002~1396733,89359629~5460828,108930368~10833507,129593184~3289710,144649336~3290897,167034219~6779119,177156787~5190287,183215660~3502696,204206046~5919797,210193421~7415433,227037563~7393224,234445135~3605828,240560472~1907077,251644403~5938599,258312268~1647698,260868422~8920602,282924374~7610022,292332323~11963618,304604600~522118,306936351~4352606,311792749~10572737,328959633~4415758,339416815~2817899,342666453~5582380,352926650~13705078,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,403429066~7830275] 2024-09-14T04:32:24.857 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1560185~3992595,18815204~3919713,24366452~16188868,42887289~3072128,49625492~10462063,81217002~1396733,89359629~5460828,108930368~10833507,129593184~3289710,144649336~3290897,167034219~6779119,177156787~5190287,183215660~3502696,204206046~5919797,210193421~7415433,227037563~7393224,234445135~3605828,240560472~1907077,251644403~5938599,258312268~1647698,260868422~8920602,282924374~7610022,292332323~11963618,304604600~522118,306936351~4352606,311792749~10572737,328959633~4415758,339416815~2817899,342666453~5582380,352926650~13705078,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,403429066~7830275] 2024-09-14T04:32:24.863 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~5552780,16777216~23778104,41943040~4016377,46137344~13950211,79691776~2921959,89359629~5460828,104857600~14906275,125829120~7053774,142717019~5223214,163577856~10235482,176160768~10557588,201326592~16282262,227037563~11013400,239075328~3392221,247463936~12496030,260046848~9742176,282924374~22202344,306184192~16181294,327155712~6219679,335544320~12704513,352321536~19404515,373293056~11321926,385875968~7626856,402653184~8606157] 2024-09-14T04:32:24.863 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 8 diff [1560185~3992595,18815204~3919713,24366452~16188868,42821576~5427684,49625492~10874967,81217002~1396733,89359629~5460828,108930368~10833507,125750260~3113143,129593184~3289710,144649336~3290897,167034219~6779119,177156787~5190287,183215660~3502696,204206046~5919797,210193421~7415433,227037563~7393224,234445135~3605828,240560472~1907077,251644403~5938599,258312268~1647698,260868422~8920602,271903192~485755,282924374~7610022,292332323~11963618,304604600~522118,306936351~18977152,328959633~4415758,339416815~2817899,342666453~9554642,352231748~211803,352926650~13705078,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:24.863 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1560185~3992595,18815204~3919713,24366452~16188868,42821576~5427684,49625492~10874967,81217002~1396733,89359629~5460828,108930368~10833507,125750260~3113143,129593184~3289710,144649336~3290897,167034219~6779119,177156787~5190287,183215660~3502696,204206046~5919797,210193421~7415433,227037563~7393224,234445135~3605828,240560472~1907077,251644403~5938599,258312268~1647698,260868422~8920602,271903192~485755,282924374~7610022,292332323~11963618,304604600~522118,306936351~18977152,328959633~4415758,339416815~2817899,342666453~9554642,352231748~211803,352926650~13705078,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:24.870 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~5552780,16777216~23778104,41943040~18557419,79691776~2921959,89359629~5460828,104857600~14906275,121634816~11248078,142717019~5223214,163577856~10235482,176160768~10557588,201326592~16282262,227037563~11013400,239075328~3392221,247463936~12496030,260046848~12342099,282924374~22202344,306184192~19729311,327155712~6219679,335544320~36181731,373293056~11321926,385875968~7626856,396566061~15984065] 2024-09-14T04:32:24.870 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 9 diff [1560185~3992595,18815204~3919713,24366452~16188868,42821576~5427684,49625492~10874967,77103770~678283,81217002~1396733,87561278~7259179,102891979~16871896,125750260~3113143,129593184~3289710,144649336~3290897,167034219~6779119,177156787~5609834,183215660~3502696,204206046~5919797,210193421~7415433,227037563~7393224,234445135~3605828,240560472~1907077,251644403~5938599,258312268~1647698,260868422~8920602,271903192~485755,282924374~7610022,292332323~33581180,328959633~4415758,339416815~2817899,342666453~9554642,352231748~211803,352926650~13705078,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:24.870 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1560185~3992595,18815204~3919713,24366452~16188868,42821576~5427684,49625492~10874967,77103770~678283,81217002~1396733,87561278~7259179,102891979~16871896,125750260~3113143,129593184~3289710,144649336~3290897,167034219~6779119,177156787~5609834,183215660~3502696,204206046~5919797,210193421~7415433,227037563~7393224,234445135~3605828,240560472~1907077,251644403~5938599,258312268~1647698,260868422~8920602,271903192~485755,282924374~7610022,292332323~33581180,328959633~4415758,339416815~2817899,342666453~9554642,352231748~211803,352926650~13705078,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:24.876 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~5552780,16777216~23778104,41943040~18557419,75497472~2284581,79691776~2921959,83886080~10934377,100663296~19100579,121634816~11248078,142717019~5223214,163577856~10235482,176160768~10557588,201326592~16282262,227037563~11013400,239075328~3392221,247463936~12496030,260046848~12342099,282924374~42989129,327155712~6219679,335544320~36181731,373293056~11321926,385875968~7626856,396566061~15984065] 2024-09-14T04:32:24.876 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 10 diff [1560185~3992595,18815204~3919713,24366452~23882808,49625492~10874967,77103770~678283,79792402~6496993,87561278~7259179,102891979~20007158,125750260~3113143,129593184~3289710,144649336~7944386,167034219~6779119,177156787~5609834,183215660~3502696,204206046~5919797,210193421~7415433,227037563~7393224,234445135~3605828,240560472~1907077,244100700~3181961,251644403~18144621,271903192~485755,277143881~13390515,292332323~33581180,328959633~4415758,339416815~2817899,342666453~9554642,352231748~211803,352926650~13705078,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:24.877 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1560185~3992595,18815204~3919713,24366452~23882808,49625492~10874967,77103770~678283,79792402~6496993,87561278~7259179,102891979~20007158,125750260~3113143,129593184~3289710,144649336~7944386,167034219~6779119,177156787~5609834,183215660~3502696,204206046~5919797,210193421~7415433,227037563~7393224,234445135~3605828,240560472~1907077,244100700~3181961,251644403~18144621,271903192~485755,277143881~13390515,292332323~33581180,328959633~4415758,339416815~2817899,342666453~9554642,352231748~211803,352926650~13705078,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:24.883 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~5552780,16777216~43723243,75497472~2284581,79691776~15128681,100663296~32219598,142717019~9876703,163577856~10235482,176160768~10557588,201326592~16282262,227037563~11013400,239075328~3392221,243269632~4013029,247463936~24925011,276824064~49089439,327155712~6219679,335544320~36181731,373293056~11321926,385875968~7626856,396566061~15984065] 2024-09-14T04:32:24.883 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 11 diff [1532502~4624507,18815204~3919713,24366452~23882808,49625492~10874967,69873797~1502064,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,144649336~7944386,167034219~6779119,177156787~5609834,183215660~3502696,204206046~13402808,227037563~7393224,234445135~3605828,240560472~1907077,244100700~3181961,251644403~18144621,271903192~485755,277143881~13390515,292332323~33581180,328959633~4415758,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:24.883 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~23882808,49625492~10874967,69873797~1502064,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,144649336~7944386,167034219~6779119,177156787~5609834,183215660~3502696,204206046~13402808,227037563~7393224,234445135~3605828,240560472~1907077,244100700~3181961,251644403~18144621,271903192~485755,277143881~13390515,292332323~33581180,328959633~4415758,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:24.893 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~43723243,67108864~4266997,75497472~2284581,79691776~15128681,96468992~3652024,100663296~32219598,142717019~9876703,163577856~10235482,176160768~10557588,201326592~16282262,227037563~11013400,239075328~3392221,243269632~4013029,247463936~24925011,276824064~49089439,327155712~6219679,335544320~36181731,373293056~11321926,385875968~7626856,396566061~15984065] 2024-09-14T04:32:24.893 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 12 diff [1532502~4624507,18815204~3919713,24366452~23951097,49625492~10874967,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,144649336~7944386,164144631~755755,167034219~7005812,177156787~5609834,183215660~3502696,204206046~13402808,227037563~7393224,234445135~3605828,240560472~1907077,244100700~3181961,251644403~18144621,271903192~485755,277143881~13390515,291130899~619944,292332323~33581180,328615132~4760259,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:24.893 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~23951097,49625492~10874967,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,144649336~7944386,164144631~755755,167034219~7005812,177156787~5609834,183215660~3502696,204206046~13402808,227037563~7393224,234445135~3605828,240560472~1907077,244100700~3181961,251644403~18144621,271903192~485755,277143881~13390515,291130899~619944,292332323~33581180,328615132~4760259,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:24.895 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~43723243,62914560~9186737,75497472~2284581,79691776~15128681,96468992~3652024,100663296~32219598,134282812~4188311,141461050~1072987,142717019~9876703,163577856~10462175,176160768~10557588,201326592~16282262,227037563~11013400,239075328~3392221,243269632~4013029,247463936~24925011,276824064~49089439,327155712~6219679,335544320~36181731,373293056~11321926,385875968~7626856,396566061~15984065] 2024-09-14T04:32:24.895 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 13 diff [1532502~4624507,18815204~3919713,24366452~24816062,49625492~10874967,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,144649336~10643842,155839505~3780252,164144631~755755,167034219~7005812,177156787~5609834,183215660~3502696,201412400~16196454,218384295~132186,225312546~9118241,234445135~3605828,240560472~7894340,249153849~20635175,271903192~485755,277143881~13390515,291130899~619944,292332323~33581180,328615132~4760259,338917256~193834,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:24.895 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~24816062,49625492~10874967,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,144649336~10643842,155839505~3780252,164144631~755755,167034219~7005812,177156787~5609834,183215660~3502696,201412400~16196454,218384295~132186,225312546~9118241,234445135~3605828,240560472~7894340,249153849~20635175,271903192~485755,277143881~13390515,291130899~619944,292332323~33581180,328615132~4760259,338917256~193834,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:24.901 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~43723243,62914560~9186737,75497472~2284581,79691776~15128681,96468992~3652024,100663296~32219598,134282812~4188311,141461050~1072987,142717019~16902738,163577856~10462175,176160768~10557588,201326592~16282262,218103808~412673,222298112~15752851,239075328~33313619,276824064~49089439,327155712~6219679,335544320~36181731,373293056~11321926,385875968~7626856,396566061~15984065] 2024-09-14T04:32:24.901 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 14 diff [1532502~4624507,18815204~3919713,24366452~24816062,49625492~10874967,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,144649336~10643842,155839505~3780252,164144631~755755,167034219~7005812,177156787~5609834,183215660~7587630,201412400~16196454,218384295~132186,225312546~9118241,234445135~14009677,249153849~20635175,271903192~485755,277143881~48769622,328615132~4760259,338917256~193834,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:24.901 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~24816062,49625492~10874967,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,144649336~10643842,155839505~3780252,164144631~755755,167034219~7005812,177156787~5609834,183215660~7587630,201412400~16196454,218384295~132186,225312546~9118241,234445135~14009677,249153849~20635175,271903192~485755,277143881~48769622,328615132~4760259,338917256~193834,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:24.907 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~43723243,62914560~9186737,75497472~2284581,79691776~15128681,96468992~3652024,100663296~32219598,134282812~4188311,141461050~1072987,142717019~16902738,163577856~10462175,176160768~14642522,201326592~16282262,218103808~412673,222298112~50090835,276824064~49089439,327155712~6219679,335544320~36181731,373293056~11321926,385875968~7626856,396566061~15984065] 2024-09-14T04:32:24.907 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 15 diff [1532502~4624507,18815204~3919713,24366452~24816062,49625492~13014907,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,139378274~15914904,155839505~3780252,164144631~755755,167034219~7005812,177156787~5609834,183215660~7587630,201412400~16196454,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~20635175,271903192~485755,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:24.907 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~24816062,49625492~13014907,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,139378274~15914904,155839505~3780252,164144631~755755,167034219~7005812,177156787~5609834,183215660~7587630,201412400~16196454,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~20635175,271903192~485755,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:24.913 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~45863183,62914560~9186737,75497472~2284581,79691776~15128681,96468992~3652024,100663296~32219598,134282812~4188311,139378274~20241483,163577856~10462175,176160768~14642522,201326592~16282262,218103808~412673,222298112~50090835,276824064~49089439,327155712~6219679,335544320~36181731,373293056~11321926,385875968~7626856,396566061~15984065] 2024-09-14T04:32:24.913 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 16 diff [1532502~4624507,18815204~3919713,24366452~24816062,49625492~13014907,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~15914904,155839505~11131128,167034219~7005812,177156787~5609834,183215660~7587630,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~20635175,271903192~485755,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~28437657,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:24.913 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~24816062,49625492~13014907,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~15914904,155839505~11131128,167034219~7005812,177156787~5609834,183215660~7587630,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~20635175,271903192~485755,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~28437657,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:24.919 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~45863183,62914560~9186737,75497472~2284581,79691776~15128681,96468992~3652024,100663296~32219598,132996120~5475003,139378274~34661757,176160768~14642522,192937984~24670870,218103808~412673,222298112~50090835,276824064~49089439,327155712~6219679,335544320~36181731,373293056~11321926,385875968~7626856,396566061~15984065] 2024-09-14T04:32:24.919 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 17 diff [1532502~4624507,18815204~3919713,24366452~24816062,49625492~15162845,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~27592359,167034219~7005812,177156787~5609834,183215660~7587630,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271903192~485755,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:24.919 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~24816062,49625492~15162845,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~27592359,167034219~7005812,177156787~5609834,183215660~7587630,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271903192~485755,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:24.925 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~55324081,75497472~2284581,79691776~15128681,96468992~3652024,100663296~32219598,132996120~5475003,139378274~34661757,176160768~14642522,192937984~24670870,218103808~412673,222298112~50090835,276824064~49089439,327155712~6219679,335544320~49070662,385875968~7626856,396566061~15984065] 2024-09-14T04:32:24.925 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 18 diff [1532502~4624507,18815204~3919713,24366452~24816062,49625492~15162845,65187704~13622813,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~34661757,177156787~5609834,183215660~7587630,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:24.925 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~24816062,49625492~15162845,65187704~13622813,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~34661757,177156787~5609834,183215660~7587630,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:24.932 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~62033301,79691776~15128681,96468992~3652024,100663296~32219598,132996120~5475003,139378274~34661757,176160768~14642522,192937984~24670870,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~49070662,385875968~7626856,396566061~15984065] 2024-09-14T04:32:24.932 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 19 diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:24.932 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:24.937 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~82237545,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:24.937 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 2 diff [114085331~2130967,234445135~3605828,282924374~6084694,292332323~11963618,306936351~4352606,360584803~2356626,403429066~7830275] 2024-09-14T04:32:24.937 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [114085331~2130967,234445135~3605828,282924374~6084694,292332323~11963618,306936351~4352606,360584803~2356626,403429066~7830275] 2024-09-14T04:32:24.943 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [113816607~2399691,230686720~7364243,282924374~6084694,289406976~14888965,306184192~5104765,356515840~6425589,402653184~8606157] 2024-09-14T04:32:24.943 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 3 diff [81217002~1396733,114085331~2130967,145214205~2726028,167034219~6779119,204206046~5919797,234445135~3605828,262905843~6883181,282924374~6084694,292332323~11963618,306936351~4352606,311792749~7730750,330947088~803453,342666453~5582380,360584803~2356626,376743309~7871673,403429066~7830275] 2024-09-14T04:32:24.943 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [81217002~1396733,114085331~2130967,145214205~2726028,167034219~6779119,204206046~5919797,234445135~3605828,262905843~6883181,282924374~6084694,292332323~11963618,306936351~4352606,311792749~7730750,330947088~803453,342666453~5582380,360584803~2356626,376743309~7871673,403429066~7830275] 2024-09-14T04:32:24.949 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [79691776~2921959,113816607~2399691,142717019~5223214,163577856~10235482,201326592~8799251,230686720~7364243,262905843~6883181,282924374~6084694,289406976~14888965,306184192~13339307,327155712~4594829,339738624~8510209,356515840~6425589,373293056~11321926,402653184~8606157] 2024-09-14T04:32:24.949 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 4 diff [42887289~3072128,51969188~8118367,81217002~1396733,89587254~5233203,114085331~2130967,145214205~2726028,167034219~6779119,204206046~5919797,210193421~7415433,234445135~3605828,251644403~5938599,258312268~1647698,262905843~6883181,282924374~7610022,292332323~11963618,306936351~4352606,311792749~7730750,328959633~4415758,342666453~5582380,358912822~7718906,376743309~7871673,389122080~4380744,403429066~7830275] 2024-09-14T04:32:24.949 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [42887289~3072128,51969188~8118367,81217002~1396733,89587254~5233203,114085331~2130967,145214205~2726028,167034219~6779119,204206046~5919797,210193421~7415433,234445135~3605828,251644403~5938599,258312268~1647698,262905843~6883181,282924374~7610022,292332323~11963618,306936351~4352606,311792749~7730750,328959633~4415758,342666453~5582380,358912822~7718906,376743309~7871673,389122080~4380744,403429066~7830275] 2024-09-14T04:32:24.955 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [41943040~4016377,51969188~8118367,79691776~2921959,89587254~5233203,113816607~2399691,142717019~5223214,163577856~10235482,201326592~16282262,230686720~7364243,247463936~12496030,262905843~6883181,282924374~21371567,306184192~13339307,327155712~6219679,339738624~8510209,356515840~10115888,373293056~11321926,385875968~7626856,402653184~8606157] 2024-09-14T04:32:24.955 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 5 diff [2471184~3081596,24366452~9505140,42887289~3072128,51969188~8118367,81217002~1396733,89587254~5233203,110037204~9726671,144649336~3290897,167034219~6779119,178985346~3361728,183215660~3502696,204206046~5919797,210193421~7415433,234445135~3605828,251644403~5938599,258312268~1647698,262905843~6883181,282924374~7610022,292332323~11963618,306936351~4352606,311792749~7730750,328959633~4415758,342666453~5582380,358912822~7718906,376743309~7871673,389122080~4380744,403429066~7830275] 2024-09-14T04:32:24.955 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [2471184~3081596,24366452~9505140,42887289~3072128,51969188~8118367,81217002~1396733,89587254~5233203,110037204~9726671,144649336~3290897,167034219~6779119,178985346~3361728,183215660~3502696,204206046~5919797,210193421~7415433,234445135~3605828,251644403~5938599,258312268~1647698,262905843~6883181,282924374~7610022,292332323~11963618,306936351~4352606,311792749~7730750,328959633~4415758,342666453~5582380,358912822~7718906,376743309~7871673,389122080~4380744,403429066~7830275] 2024-09-14T04:32:24.960 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~5552780,24366452~9505140,41943040~4016377,51969188~8118367,79691776~2921959,89587254~5233203,110037204~9726671,142717019~5223214,163577856~10235482,176160768~10557588,201326592~16282262,230686720~7364243,247463936~12496030,262905843~6883181,282924374~21371567,306184192~13339307,327155712~6219679,339738624~8510209,356515840~10115888,373293056~11321926,385875968~7626856,402653184~8606157] 2024-09-14T04:32:24.960 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 6 diff [2471184~3081596,18815204~3919713,24366452~9505140,42887289~3072128,51969188~8118367,81217002~1396733,89587254~5233203,110037204~9726671,129593184~3289710,144649336~3290897,167034219~6779119,177156787~5190287,183215660~3502696,204206046~5919797,210193421~7415433,229976328~4454459,234445135~3605828,240560472~1907077,251644403~5938599,258312268~1647698,261607553~8181471,282924374~7610022,292332323~11963618,306936351~4352606,311792749~7730750,328959633~4415758,342666453~5582380,352926650~13705078,376743309~7871673,389122080~4380744,403429066~7830275] 2024-09-14T04:32:24.960 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [2471184~3081596,18815204~3919713,24366452~9505140,42887289~3072128,51969188~8118367,81217002~1396733,89587254~5233203,110037204~9726671,129593184~3289710,144649336~3290897,167034219~6779119,177156787~5190287,183215660~3502696,204206046~5919797,210193421~7415433,229976328~4454459,234445135~3605828,240560472~1907077,251644403~5938599,258312268~1647698,261607553~8181471,282924374~7610022,292332323~11963618,306936351~4352606,311792749~7730750,328959633~4415758,342666453~5582380,352926650~13705078,376743309~7871673,389122080~4380744,403429066~7830275] 2024-09-14T04:32:24.966 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~5552780,16777216~5957701,24366452~9505140,41943040~4016377,51969188~8118367,79691776~2921959,89587254~5233203,110037204~9726671,125829120~7053774,142717019~5223214,163577856~10235482,176160768~10557588,201326592~16282262,229603977~8446986,239075328~3392221,247463936~12496030,261607553~8181471,282924374~21371567,306184192~13339307,327155712~6219679,339738624~8510209,352321536~14310192,373293056~11321926,385875968~7626856,402653184~8606157] 2024-09-14T04:32:24.966 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 7 diff [1560185~3992595,18815204~3919713,24366452~9505140,42887289~3072128,51969188~8118367,81217002~1396733,89587254~5233203,108930368~10833507,129593184~3289710,144649336~3290897,167034219~6779119,177156787~5190287,183215660~3502696,204206046~5919797,210193421~7415433,227037563~7393224,234445135~3605828,240560472~1907077,251644403~5938599,258312268~1647698,261607553~8181471,282924374~7610022,292332323~11963618,304604600~522118,306936351~4352606,311792749~7730750,328959633~4415758,339416815~2817899,342666453~5582380,352926650~13705078,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,403429066~7830275] 2024-09-14T04:32:24.966 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1560185~3992595,18815204~3919713,24366452~9505140,42887289~3072128,51969188~8118367,81217002~1396733,89587254~5233203,108930368~10833507,129593184~3289710,144649336~3290897,167034219~6779119,177156787~5190287,183215660~3502696,204206046~5919797,210193421~7415433,227037563~7393224,234445135~3605828,240560472~1907077,251644403~5938599,258312268~1647698,261607553~8181471,282924374~7610022,292332323~11963618,304604600~522118,306936351~4352606,311792749~7730750,328959633~4415758,339416815~2817899,342666453~5582380,352926650~13705078,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,403429066~7830275] 2024-09-14T04:32:24.971 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~5552780,16777216~5957701,24366452~9505140,41943040~4016377,51969188~8118367,79691776~2921959,89587254~5233203,104857600~14906275,125829120~7053774,142717019~5223214,163577856~10235482,176160768~10557588,201326592~16282262,227037563~11013400,239075328~3392221,247463936~12496030,261607553~8181471,282924374~22202344,306184192~13339307,327155712~6219679,335544320~12704513,352321536~19404515,373293056~11321926,385875968~7626856,402653184~8606157] 2024-09-14T04:32:24.971 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 8 diff [1560185~3992595,18815204~3919713,24366452~9505140,42821576~5427684,51969188~8531271,81217002~1396733,89587254~5233203,108930368~10833507,125750260~3113143,129593184~3289710,144649336~3290897,167034219~6779119,177156787~5190287,183215660~3502696,204206046~5919797,210193421~7415433,227037563~7393224,234445135~3605828,240560472~1907077,251644403~5938599,258312268~1647698,261607553~8181471,271903192~485755,282924374~7610022,292332323~11963618,304604600~522118,306936351~18977152,328959633~4415758,339416815~2817899,342666453~9554642,352231748~211803,352926650~13705078,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:24.971 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1560185~3992595,18815204~3919713,24366452~9505140,42821576~5427684,51969188~8531271,81217002~1396733,89587254~5233203,108930368~10833507,125750260~3113143,129593184~3289710,144649336~3290897,167034219~6779119,177156787~5190287,183215660~3502696,204206046~5919797,210193421~7415433,227037563~7393224,234445135~3605828,240560472~1907077,251644403~5938599,258312268~1647698,261607553~8181471,271903192~485755,282924374~7610022,292332323~11963618,304604600~522118,306936351~18977152,328959633~4415758,339416815~2817899,342666453~9554642,352231748~211803,352926650~13705078,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:24.977 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~5552780,16777216~5957701,24366452~9505140,41943040~6306220,51969188~8531271,79691776~2921959,89587254~5233203,104857600~14906275,121634816~11248078,142717019~5223214,163577856~10235482,176160768~10557588,201326592~16282262,227037563~11013400,239075328~3392221,247463936~12496030,261607553~10781394,282924374~22202344,306184192~19729311,327155712~6219679,335544320~36181731,373293056~11321926,385875968~7626856,396566061~15984065] 2024-09-14T04:32:24.977 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 9 diff [1560185~3992595,18815204~3919713,24366452~10417690,42821576~5427684,51969188~8531271,77103770~678283,81217002~1396733,87561278~7259179,102891979~16871896,125750260~3113143,129593184~3289710,144649336~3290897,167034219~6779119,177156787~5609834,183215660~3502696,204206046~5919797,210193421~7415433,227037563~7393224,234445135~3605828,240560472~1907077,251644403~5938599,258312268~1647698,261607553~8181471,271903192~485755,282924374~7610022,292332323~33581180,328959633~4415758,339416815~2817899,342666453~9554642,352231748~211803,352926650~13705078,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:24.977 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1560185~3992595,18815204~3919713,24366452~10417690,42821576~5427684,51969188~8531271,77103770~678283,81217002~1396733,87561278~7259179,102891979~16871896,125750260~3113143,129593184~3289710,144649336~3290897,167034219~6779119,177156787~5609834,183215660~3502696,204206046~5919797,210193421~7415433,227037563~7393224,234445135~3605828,240560472~1907077,251644403~5938599,258312268~1647698,261607553~8181471,271903192~485755,282924374~7610022,292332323~33581180,328959633~4415758,339416815~2817899,342666453~9554642,352231748~211803,352926650~13705078,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:24.983 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~5552780,16777216~5957701,24366452~10417690,41943040~6306220,51969188~8531271,75497472~2284581,79691776~2921959,83886080~10934377,100663296~19100579,121634816~11248078,142717019~5223214,163577856~10235482,176160768~10557588,201326592~16282262,227037563~11013400,239075328~3392221,247463936~12496030,261607553~10781394,282924374~42989129,327155712~6219679,335544320~36181731,373293056~11321926,385875968~7626856,396566061~15984065] 2024-09-14T04:32:24.983 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 10 diff [1560185~3992595,18815204~3919713,24366452~10417690,35645530~2311397,39500721~8748539,51969188~8531271,77103770~678283,79792402~6496993,87561278~7259179,102891979~20007158,125750260~3113143,129593184~3289710,144649336~7944386,167034219~6779119,177156787~5609834,183215660~3502696,204206046~5919797,210193421~7415433,227037563~7393224,234445135~3605828,240560472~1907077,244100700~3181961,251644403~9369355,261607553~8181471,271903192~485755,277143881~13390515,292332323~33581180,328959633~4415758,339416815~2817899,342666453~9554642,352231748~211803,352926650~13705078,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:24.984 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1560185~3992595,18815204~3919713,24366452~10417690,35645530~2311397,39500721~8748539,51969188~8531271,77103770~678283,79792402~6496993,87561278~7259179,102891979~20007158,125750260~3113143,129593184~3289710,144649336~7944386,167034219~6779119,177156787~5609834,183215660~3502696,204206046~5919797,210193421~7415433,227037563~7393224,234445135~3605828,240560472~1907077,244100700~3181961,251644403~9369355,261607553~8181471,271903192~485755,277143881~13390515,292332323~33581180,328959633~4415758,339416815~2817899,342666453~9554642,352231748~211803,352926650~13705078,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:24.989 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~5552780,16777216~5957701,24366452~10417690,35645530~2311397,39500721~8748539,51969188~8531271,75497472~2284581,79691776~15128681,100663296~32219598,142717019~9876703,163577856~10235482,176160768~10557588,201326592~16282262,227037563~11013400,239075328~3392221,243269632~4013029,247463936~13549822,261607553~10781394,276824064~49089439,327155712~6219679,335544320~36181731,373293056~11321926,385875968~7626856,396566061~15984065] 2024-09-14T04:32:24.989 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 11 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~2311397,39500721~8748539,51969188~8531271,69873797~1502064,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,144649336~7944386,167034219~6779119,177156787~5609834,183215660~3502696,204206046~13402808,227037563~7393224,234445135~3605828,240560472~1907077,244100700~3181961,251644403~9369355,261607553~8181471,271903192~485755,277143881~13390515,292332323~33581180,328959633~4415758,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:24.989 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~2311397,39500721~8748539,51969188~8531271,69873797~1502064,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,144649336~7944386,167034219~6779119,177156787~5609834,183215660~3502696,204206046~13402808,227037563~7393224,234445135~3605828,240560472~1907077,244100700~3181961,251644403~9369355,261607553~8181471,271903192~485755,277143881~13390515,292332323~33581180,328959633~4415758,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:24.994 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~2311397,39500721~8748539,51969188~8531271,67108864~4266997,75497472~2284581,79691776~15128681,96468992~3652024,100663296~32219598,142717019~9876703,163577856~10235482,176160768~10557588,201326592~16282262,227037563~11013400,239075328~3392221,243269632~4013029,247463936~13549822,261607553~10781394,276824064~49089439,327155712~6219679,335544320~36181731,373293056~11321926,385875968~7626856,396566061~15984065] 2024-09-14T04:32:24.995 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 12 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~2311397,39500721~8816828,51969188~8531271,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,144649336~7944386,164144631~755755,167034219~7005812,177156787~5609834,183215660~3502696,204206046~13402808,227037563~7393224,234445135~3605828,240560472~1907077,244100700~3181961,251644403~9369355,261607553~8181471,271903192~485755,277143881~13390515,291130899~619944,292332323~33581180,328615132~4760259,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:24.995 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~2311397,39500721~8816828,51969188~8531271,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,144649336~7944386,164144631~755755,167034219~7005812,177156787~5609834,183215660~3502696,204206046~13402808,227037563~7393224,234445135~3605828,240560472~1907077,244100700~3181961,251644403~9369355,261607553~8181471,271903192~485755,277143881~13390515,291130899~619944,292332323~33581180,328615132~4760259,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:25.000 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~2311397,39500721~8816828,51969188~8531271,62914560~9186737,75497472~2284581,79691776~15128681,96468992~3652024,100663296~32219598,134282812~4188311,141461050~1072987,142717019~9876703,163577856~10462175,176160768~10557588,201326592~16282262,227037563~11013400,239075328~3392221,243269632~4013029,247463936~13549822,261607553~10781394,276824064~49089439,327155712~6219679,335544320~36181731,373293056~11321926,385875968~7626856,396566061~15984065] 2024-09-14T04:32:25.000 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 13 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~2311397,39500721~9681793,51969188~8531271,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,144649336~10643842,155839505~3780252,164144631~755755,167034219~7005812,177156787~5609834,183215660~3502696,201412400~16196454,218384295~132186,225312546~9118241,234445135~3605828,240560472~7894340,249153849~11859909,261607553~8181471,271903192~485755,277143881~13390515,291130899~619944,292332323~33581180,328615132~4760259,338917256~193834,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:25.000 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~2311397,39500721~9681793,51969188~8531271,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,144649336~10643842,155839505~3780252,164144631~755755,167034219~7005812,177156787~5609834,183215660~3502696,201412400~16196454,218384295~132186,225312546~9118241,234445135~3605828,240560472~7894340,249153849~11859909,261607553~8181471,271903192~485755,277143881~13390515,291130899~619944,292332323~33581180,328615132~4760259,338917256~193834,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:25.005 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~2311397,39500721~9681793,51969188~8531271,62914560~9186737,75497472~2284581,79691776~15128681,96468992~3652024,100663296~32219598,134282812~4188311,141461050~1072987,142717019~16902738,163577856~10462175,176160768~10557588,201326592~16282262,218103808~412673,222298112~15752851,239075328~21938430,261607553~10781394,276824064~49089439,327155712~6219679,335544320~36181731,373293056~11321926,385875968~7626856,396566061~15984065] 2024-09-14T04:32:25.006 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 14 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,51969188~8531271,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,144649336~10643842,155839505~3780252,164144631~755755,167034219~7005812,177156787~5609834,183215660~7587630,201412400~16196454,218384295~132186,225312546~9118241,234445135~14009677,249153849~11859909,261607553~8181471,271903192~485755,277143881~48769622,328615132~4760259,338917256~193834,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:25.006 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,51969188~8531271,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,144649336~10643842,155839505~3780252,164144631~755755,167034219~7005812,177156787~5609834,183215660~7587630,201412400~16196454,218384295~132186,225312546~9118241,234445135~14009677,249153849~11859909,261607553~8181471,271903192~485755,277143881~48769622,328615132~4760259,338917256~193834,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:25.011 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~13536984,51969188~8531271,62914560~9186737,75497472~2284581,79691776~15128681,96468992~3652024,100663296~32219598,134282812~4188311,141461050~1072987,142717019~16902738,163577856~10462175,176160768~14642522,201326592~16282262,218103808~412673,222298112~38715646,261607553~10781394,276824064~49089439,327155712~6219679,335544320~36181731,373293056~11321926,385875968~7626856,396566061~15984065] 2024-09-14T04:32:25.011 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 15 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,51969188~10671211,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,139378274~15914904,155839505~3780252,164144631~755755,167034219~7005812,177156787~5609834,183215660~7587630,201412400~16196454,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~11859909,261607553~8181471,271903192~485755,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:25.011 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,51969188~10671211,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,139378274~15914904,155839505~3780252,164144631~755755,167034219~7005812,177156787~5609834,183215660~7587630,201412400~16196454,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~11859909,261607553~8181471,271903192~485755,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:25.018 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~13536984,51969188~10671211,62914560~9186737,75497472~2284581,79691776~15128681,96468992~3652024,100663296~32219598,134282812~4188311,139378274~20241483,163577856~10462175,176160768~14642522,201326592~16282262,218103808~412673,222298112~38715646,261607553~10781394,276824064~49089439,327155712~6219679,335544320~36181731,373293056~11321926,385875968~7626856,396566061~15984065] 2024-09-14T04:32:25.018 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 16 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,51969188~10671211,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~15914904,155839505~11131128,167034219~7005812,177156787~5609834,183215660~7587630,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~12054819,261607553~8181471,271903192~485755,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~28437657,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:25.018 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,51969188~10671211,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~15914904,155839505~11131128,167034219~7005812,177156787~5609834,183215660~7587630,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~12054819,261607553~8181471,271903192~485755,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~28437657,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:25.023 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~13536984,51969188~10671211,62914560~9186737,75497472~2284581,79691776~15128681,96468992~3652024,100663296~32219598,132996120~5475003,139378274~34661757,176160768~14642522,192937984~24670870,218103808~412673,222298112~38910556,261607553~10781394,276824064~49089439,327155712~6219679,335544320~36181731,373293056~11321926,385875968~7626856,396566061~15984065] 2024-09-14T04:32:25.023 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 17 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,50026379~14761958,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~27592359,167034219~7005812,177156787~5609834,183215660~7587630,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~12054819,261607553~8860187,271903192~485755,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:25.023 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,50026379~14761958,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~27592359,167034219~7005812,177156787~5609834,183215660~7587630,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~12054819,261607553~8860187,271903192~485755,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:25.029 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~13536984,50026379~22074918,75497472~2284581,79691776~15128681,96468992~3652024,100663296~32219598,132996120~5475003,139378274~34661757,176160768~14642522,192937984~24670870,218103808~412673,222298112~38910556,261607553~10781394,276824064~49089439,327155712~6219679,335544320~49070662,385875968~7626856,396566061~15984065] 2024-09-14T04:32:25.029 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 18 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,50026379~14761958,65187704~13622813,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~34661757,177156787~5609834,183215660~7587630,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~12054819,261607553~8860187,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:25.030 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,50026379~14761958,65187704~13622813,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~34661757,177156787~5609834,183215660~7587630,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~12054819,261607553~8860187,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~7871673,389122080~4380744,396800802~6077541,403429066~9121060] 2024-09-14T04:32:25.034 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~13536984,50026379~28784138,79691776~15128681,96468992~3652024,100663296~32219598,132996120~5475003,139378274~34661757,176160768~14642522,192937984~24670870,218103808~412673,222298112~38910556,261607553~13104181,276824064~49089439,327155712~6219679,335544320~49070662,385875968~7626856,396566061~15984065] 2024-09-14T04:32:25.035 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 19 diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:25.035 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:25.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:24 smithi039 ceph-mon[111235]: pgmap v1790: 105 pgs: 105 active+clean; 1.1 GiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 224 KiB/s rd, 46 MiB/s wr, 348 op/s 2024-09-14T04:32:25.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:24 smithi039 ceph-mon[111235]: osdmap e1138: 8 total, 8 up, 8 in 2024-09-14T04:32:25.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:24 smithi039 ceph-mon[106470]: pgmap v1790: 105 pgs: 105 active+clean; 1.1 GiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 224 KiB/s rd, 46 MiB/s wr, 348 op/s 2024-09-14T04:32:25.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:24 smithi039 ceph-mon[106470]: osdmap e1138: 8 total, 8 up, 8 in 2024-09-14T04:32:25.041 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:25.041 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 3 diff [81217002~1396733,145214205~2726028,167034219~6779119,204206046~5919797,262905843~6883181,311792749~7730750,330947088~803453,342666453~5582380,376743309~7871673,406171321~4595128] 2024-09-14T04:32:25.041 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [81217002~1396733,145214205~2726028,167034219~6779119,204206046~5919797,262905843~6883181,311792749~7730750,330947088~803453,342666453~5582380,376743309~7871673,406171321~4595128] 2024-09-14T04:32:25.046 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [79691776~2921959,142717019~5223214,163577856~10235482,201326592~8799251,262905843~6883181,311288957~8234542,327155712~4594829,339738624~8510209,373293056~11321926,406171321~4595128] 2024-09-14T04:32:25.046 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 4 diff [42887289~3072128,51969188~8118367,81217002~1396733,89587254~5233203,145214205~2726028,167034219~6779119,204206046~5919797,210193421~7415433,251644403~5938599,258312268~1647698,262905843~6883181,285482181~5052215,311792749~7730750,328959633~4415758,342666453~5582380,358912822~7718906,376743309~7871673,389122080~4380744,406171321~4595128] 2024-09-14T04:32:25.046 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [42887289~3072128,51969188~8118367,81217002~1396733,89587254~5233203,145214205~2726028,167034219~6779119,204206046~5919797,210193421~7415433,251644403~5938599,258312268~1647698,262905843~6883181,285482181~5052215,311792749~7730750,328959633~4415758,342666453~5582380,358912822~7718906,376743309~7871673,389122080~4380744,406171321~4595128] 2024-09-14T04:32:25.052 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [41943040~4016377,51969188~8118367,79691776~2921959,89587254~5233203,142717019~5223214,163577856~10235482,201326592~16282262,247463936~12496030,262905843~6883181,285482181~5052215,311288957~8234542,327155712~6219679,339738624~8510209,358912822~7718906,373293056~11321926,385875968~7626856,406171321~4595128] 2024-09-14T04:32:25.052 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 5 diff [2471184~3081596,24366452~9505140,42887289~3072128,51969188~8118367,81217002~1396733,89587254~5233203,110037204~9726671,144649336~3290897,167034219~6779119,178985346~3361728,183215660~3502696,204206046~5919797,210193421~7415433,251644403~5938599,258312268~1647698,262905843~6883181,285482181~5052215,311792749~7730750,328959633~4415758,342666453~5582380,358912822~7718906,376743309~7871673,389122080~4380744,406171321~4595128] 2024-09-14T04:32:25.052 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [2471184~3081596,24366452~9505140,42887289~3072128,51969188~8118367,81217002~1396733,89587254~5233203,110037204~9726671,144649336~3290897,167034219~6779119,178985346~3361728,183215660~3502696,204206046~5919797,210193421~7415433,251644403~5938599,258312268~1647698,262905843~6883181,285482181~5052215,311792749~7730750,328959633~4415758,342666453~5582380,358912822~7718906,376743309~7871673,389122080~4380744,406171321~4595128] 2024-09-14T04:32:25.057 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~5552780,24366452~9505140,41943040~4016377,51969188~8118367,79691776~2921959,89587254~5233203,110037204~9726671,142717019~5223214,163577856~10235482,176160768~10557588,201326592~16282262,247463936~12496030,262905843~6883181,285482181~5052215,311288957~8234542,327155712~6219679,339738624~8510209,358912822~7718906,373293056~11321926,385875968~7626856,406171321~4595128] 2024-09-14T04:32:25.057 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 6 diff [2471184~3081596,18815204~3919713,24366452~9505140,42887289~3072128,51969188~8118367,81217002~1396733,89587254~5233203,110037204~9726671,129593184~3289710,144649336~3290897,167034219~6779119,177156787~5190287,183215660~3502696,204206046~5919797,210193421~7415433,229976328~4454459,240560472~1907077,251644403~5938599,258312268~1647698,261607553~8181471,285482181~5052215,311792749~7730750,328959633~4415758,342666453~5582380,352926650~13705078,376743309~7871673,389122080~4380744,406171321~4595128] 2024-09-14T04:32:25.057 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [2471184~3081596,18815204~3919713,24366452~9505140,42887289~3072128,51969188~8118367,81217002~1396733,89587254~5233203,110037204~9726671,129593184~3289710,144649336~3290897,167034219~6779119,177156787~5190287,183215660~3502696,204206046~5919797,210193421~7415433,229976328~4454459,240560472~1907077,251644403~5938599,258312268~1647698,261607553~8181471,285482181~5052215,311792749~7730750,328959633~4415758,342666453~5582380,352926650~13705078,376743309~7871673,389122080~4380744,406171321~4595128] 2024-09-14T04:32:25.062 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~5552780,16777216~5957701,24366452~9505140,41943040~4016377,51969188~8118367,79691776~2921959,89587254~5233203,110037204~9726671,125829120~7053774,142717019~5223214,163577856~10235482,176160768~10557588,201326592~16282262,229603977~4826810,239075328~3392221,247463936~12496030,261607553~8181471,285482181~5052215,311288957~8234542,327155712~6219679,339738624~8510209,352321536~14310192,373293056~11321926,385875968~7626856,406171321~4595128] 2024-09-14T04:32:25.063 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 7 diff [1560185~3992595,18815204~3919713,24366452~9505140,42887289~3072128,51969188~8118367,81217002~1396733,89587254~5233203,108930368~10833507,129593184~3289710,144649336~3290897,167034219~6779119,177156787~5190287,183215660~3502696,204206046~5919797,210193421~7415433,227037563~7393224,240560472~1907077,251644403~5938599,258312268~1647698,261607553~8181471,285482181~5052215,300028752~3863222,304604600~522118,311792749~7730750,328959633~4415758,339416815~2817899,342666453~5582380,352926650~13705078,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,406171321~4595128] 2024-09-14T04:32:25.063 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1560185~3992595,18815204~3919713,24366452~9505140,42887289~3072128,51969188~8118367,81217002~1396733,89587254~5233203,108930368~10833507,129593184~3289710,144649336~3290897,167034219~6779119,177156787~5190287,183215660~3502696,204206046~5919797,210193421~7415433,227037563~7393224,240560472~1907077,251644403~5938599,258312268~1647698,261607553~8181471,285482181~5052215,300028752~3863222,304604600~522118,311792749~7730750,328959633~4415758,339416815~2817899,342666453~5582380,352926650~13705078,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,406171321~4595128] 2024-09-14T04:32:25.068 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~5552780,16777216~5957701,24366452~9505140,41943040~4016377,51969188~8118367,79691776~2921959,89587254~5233203,104857600~14906275,125829120~7053774,142717019~5223214,163577856~10235482,176160768~10557588,201326592~16282262,227037563~7393224,239075328~3392221,247463936~12496030,261607553~8181471,285482181~5052215,300028752~3863222,304295941~830777,311288957~8234542,327155712~6219679,335544320~12704513,352321536~19404515,373293056~11321926,385875968~7626856,406171321~4595128] 2024-09-14T04:32:25.069 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 8 diff [1560185~3992595,18815204~3919713,24366452~9505140,42821576~5427684,51969188~8531271,81217002~1396733,89587254~5233203,108930368~10833507,125750260~3113143,129593184~3289710,144649336~3290897,167034219~6779119,177156787~5190287,183215660~3502696,204206046~5919797,210193421~7415433,227037563~7393224,240560472~1907077,251644403~5938599,258312268~1647698,261607553~8181471,271903192~485755,285482181~5052215,300028752~3863222,304604600~522118,311272896~14640607,328959633~4415758,339416815~2817899,342666453~9554642,352231748~211803,352926650~13705078,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,406171321~6378805] 2024-09-14T04:32:25.069 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1560185~3992595,18815204~3919713,24366452~9505140,42821576~5427684,51969188~8531271,81217002~1396733,89587254~5233203,108930368~10833507,125750260~3113143,129593184~3289710,144649336~3290897,167034219~6779119,177156787~5190287,183215660~3502696,204206046~5919797,210193421~7415433,227037563~7393224,240560472~1907077,251644403~5938599,258312268~1647698,261607553~8181471,271903192~485755,285482181~5052215,300028752~3863222,304604600~522118,311272896~14640607,328959633~4415758,339416815~2817899,342666453~9554642,352231748~211803,352926650~13705078,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,406171321~6378805] 2024-09-14T04:32:25.074 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~5552780,16777216~5957701,24366452~9505140,41943040~6306220,51969188~8531271,79691776~2921959,89587254~5233203,104857600~14906275,121634816~11248078,142717019~5223214,163577856~10235482,176160768~10557588,201326592~16282262,227037563~7393224,239075328~3392221,247463936~12496030,261607553~10781394,285482181~5052215,300028752~3863222,304295941~830777,311272896~14640607,327155712~6219679,335544320~36181731,373293056~11321926,385875968~7626856,396566061~6312282,406171321~6378805] 2024-09-14T04:32:25.074 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 9 diff [1560185~3992595,18815204~3919713,24366452~10417690,42821576~5427684,51969188~8531271,77103770~678283,81217002~1396733,87561278~7259179,102891979~16871896,125750260~3113143,129593184~3289710,144649336~3290897,167034219~6779119,177156787~5609834,183215660~3502696,204206046~5919797,210193421~7415433,227037563~7393224,240560472~1907077,251644403~5938599,258312268~1647698,261607553~8181471,271903192~485755,285482181~5052215,300028752~7529868,311272896~14640607,328959633~4415758,339416815~2817899,342666453~9554642,352231748~211803,352926650~13705078,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,406171321~6378805] 2024-09-14T04:32:25.074 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1560185~3992595,18815204~3919713,24366452~10417690,42821576~5427684,51969188~8531271,77103770~678283,81217002~1396733,87561278~7259179,102891979~16871896,125750260~3113143,129593184~3289710,144649336~3290897,167034219~6779119,177156787~5609834,183215660~3502696,204206046~5919797,210193421~7415433,227037563~7393224,240560472~1907077,251644403~5938599,258312268~1647698,261607553~8181471,271903192~485755,285482181~5052215,300028752~7529868,311272896~14640607,328959633~4415758,339416815~2817899,342666453~9554642,352231748~211803,352926650~13705078,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,406171321~6378805] 2024-09-14T04:32:25.080 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~5552780,16777216~5957701,24366452~10417690,41943040~6306220,51969188~8531271,75497472~2284581,79691776~2921959,83886080~10934377,100663296~19100579,121634816~11248078,142717019~5223214,163577856~10235482,176160768~10557588,201326592~16282262,227037563~7393224,239075328~3392221,247463936~12496030,261607553~10781394,285482181~5052215,300028752~7529868,311272896~14640607,327155712~6219679,335544320~36181731,373293056~11321926,385875968~7626856,396566061~6312282,406171321~6378805] 2024-09-14T04:32:25.080 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 10 diff [1560185~3992595,18815204~3919713,24366452~10417690,35645530~2311397,39500721~8748539,51969188~8531271,77103770~678283,79792402~6496993,87561278~7259179,102891979~20007158,125750260~3113143,129593184~3289710,144649336~7944386,167034219~6779119,177156787~5609834,183215660~3502696,204206046~5919797,210193421~7415433,227037563~7393224,240560472~1907077,244100700~3181961,251644403~9369355,261607553~8181471,271903192~485755,277143881~6968355,285482181~5052215,300028752~7529868,311272896~14640607,328959633~4415758,339416815~2817899,342666453~9554642,352231748~211803,352926650~13705078,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,406171321~6378805] 2024-09-14T04:32:25.080 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1560185~3992595,18815204~3919713,24366452~10417690,35645530~2311397,39500721~8748539,51969188~8531271,77103770~678283,79792402~6496993,87561278~7259179,102891979~20007158,125750260~3113143,129593184~3289710,144649336~7944386,167034219~6779119,177156787~5609834,183215660~3502696,204206046~5919797,210193421~7415433,227037563~7393224,240560472~1907077,244100700~3181961,251644403~9369355,261607553~8181471,271903192~485755,277143881~6968355,285482181~5052215,300028752~7529868,311272896~14640607,328959633~4415758,339416815~2817899,342666453~9554642,352231748~211803,352926650~13705078,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,406171321~6378805] 2024-09-14T04:32:25.085 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~5552780,16777216~5957701,24366452~10417690,35645530~2311397,39500721~8748539,51969188~8531271,75497472~2284581,79691776~15128681,100663296~32219598,142717019~9876703,163577856~10235482,176160768~10557588,201326592~16282262,227037563~7393224,239075328~3392221,243269632~4013029,247463936~13549822,261607553~10781394,276824064~7288172,285482181~5052215,300028752~7529868,311272896~14640607,327155712~6219679,335544320~36181731,373293056~11321926,385875968~7626856,396566061~6312282,406171321~6378805] 2024-09-14T04:32:25.085 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 11 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~2311397,39500721~8748539,51969188~8531271,69873797~1502064,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,144649336~7944386,167034219~6779119,177156787~5609834,183215660~3502696,204206046~13402808,227037563~7393224,240560472~1907077,244100700~3181961,251644403~9369355,261607553~8181471,271903192~485755,277143881~6968355,285482181~5052215,297545811~10012809,311272896~14640607,328959633~4415758,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,406171321~6378805] 2024-09-14T04:32:25.085 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~2311397,39500721~8748539,51969188~8531271,69873797~1502064,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,144649336~7944386,167034219~6779119,177156787~5609834,183215660~3502696,204206046~13402808,227037563~7393224,240560472~1907077,244100700~3181961,251644403~9369355,261607553~8181471,271903192~485755,277143881~6968355,285482181~5052215,297545811~10012809,311272896~14640607,328959633~4415758,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,406171321~6378805] 2024-09-14T04:32:25.093 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~2311397,39500721~8748539,51969188~8531271,67108864~4266997,75497472~2284581,79691776~15128681,96468992~3652024,100663296~32219598,142717019~9876703,163577856~10235482,176160768~10557588,201326592~16282262,227037563~7393224,239075328~3392221,243269632~4013029,247463936~13549822,261607553~10781394,276824064~7288172,285482181~5052215,297545811~10012809,311272896~14640607,327155712~6219679,335544320~36181731,373293056~11321926,385875968~7626856,396566061~6312282,406171321~6378805] 2024-09-14T04:32:25.094 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 12 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~2311397,39500721~8816828,51969188~8531271,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,144649336~7944386,164144631~755755,167034219~7005812,177156787~5609834,183215660~3502696,204206046~13402808,227037563~7393224,240560472~1907077,244100700~3181961,251644403~9369355,261607553~8181471,271903192~485755,277143881~6968355,285482181~5052215,291130899~619944,297545811~10012809,311272896~14640607,328615132~4760259,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,406171321~6378805] 2024-09-14T04:32:25.094 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~2311397,39500721~8816828,51969188~8531271,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,144649336~7944386,164144631~755755,167034219~7005812,177156787~5609834,183215660~3502696,204206046~13402808,227037563~7393224,240560472~1907077,244100700~3181961,251644403~9369355,261607553~8181471,271903192~485755,277143881~6968355,285482181~5052215,291130899~619944,297545811~10012809,311272896~14640607,328615132~4760259,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,406171321~6378805] 2024-09-14T04:32:25.096 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~2311397,39500721~8816828,51969188~8531271,62914560~9186737,75497472~2284581,79691776~15128681,96468992~3652024,100663296~32219598,134282812~4188311,141461050~1072987,142717019~9876703,163577856~10462175,176160768~10557588,201326592~16282262,227037563~7393224,239075328~3392221,243269632~4013029,247463936~13549822,261607553~10781394,276824064~7288172,285482181~5052215,291130899~619944,297545811~10012809,311272896~14640607,327155712~6219679,335544320~36181731,373293056~11321926,385875968~7626856,396566061~6312282,406171321~6378805] 2024-09-14T04:32:25.096 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 13 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~2311397,39500721~9681793,51969188~8531271,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,144649336~10643842,155839505~3780252,164144631~755755,167034219~7005812,177156787~5609834,183215660~3502696,201412400~16196454,218384295~132186,225312546~9118241,240560472~7894340,249153849~11859909,261607553~8181471,271903192~485755,277143881~6968355,285482181~5052215,291130899~619944,297545811~10012809,311272896~14640607,328615132~4760259,338917256~193834,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,406171321~6378805] 2024-09-14T04:32:25.096 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~2311397,39500721~9681793,51969188~8531271,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,144649336~10643842,155839505~3780252,164144631~755755,167034219~7005812,177156787~5609834,183215660~3502696,201412400~16196454,218384295~132186,225312546~9118241,240560472~7894340,249153849~11859909,261607553~8181471,271903192~485755,277143881~6968355,285482181~5052215,291130899~619944,297545811~10012809,311272896~14640607,328615132~4760259,338917256~193834,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,406171321~6378805] 2024-09-14T04:32:25.102 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~2311397,39500721~9681793,51969188~8531271,62914560~9186737,75497472~2284581,79691776~15128681,96468992~3652024,100663296~32219598,134282812~4188311,141461050~1072987,142717019~16902738,163577856~10462175,176160768~10557588,201326592~16282262,218103808~412673,222298112~12132675,239075328~21938430,261607553~10781394,276824064~7288172,285482181~5052215,291130899~619944,297545811~10012809,311272896~14640607,327155712~6219679,335544320~36181731,373293056~11321926,385875968~7626856,396566061~6312282,406171321~6378805] 2024-09-14T04:32:25.102 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 14 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,51969188~8531271,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,144649336~10643842,155839505~3780252,164144631~755755,167034219~7005812,177156787~5609834,183215660~7587630,201412400~16196454,218384295~132186,225312546~9118241,237089596~11365216,249153849~11859909,261607553~8181471,271903192~485755,277143881~6968355,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,406171321~6378805] 2024-09-14T04:32:25.102 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,51969188~8531271,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,144649336~10643842,155839505~3780252,164144631~755755,167034219~7005812,177156787~5609834,183215660~7587630,201412400~16196454,218384295~132186,225312546~9118241,237089596~11365216,249153849~11859909,261607553~8181471,271903192~485755,277143881~6968355,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,406171321~6378805] 2024-09-14T04:32:25.108 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~13536984,51969188~8531271,62914560~9186737,75497472~2284581,79691776~15128681,96468992~3652024,100663296~32219598,134282812~4188311,141461050~1072987,142717019~16902738,163577856~10462175,176160768~14642522,201326592~16282262,218103808~412673,222298112~12132675,237089596~23924162,261607553~10781394,276824064~7288172,284870704~8098864,297545811~28367692,327155712~6219679,335544320~36181731,373293056~11321926,385875968~7626856,396566061~6312282,406171321~6378805] 2024-09-14T04:32:25.108 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 15 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,51969188~10671211,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,139378274~15914904,155839505~3780252,164144631~755755,167034219~7005812,177156787~5609834,183215660~7587630,201412400~16196454,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~11859909,261607553~8181471,271903192~485755,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,406171321~6378805] 2024-09-14T04:32:25.108 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,51969188~10671211,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,139378274~15914904,155839505~3780252,164144631~755755,167034219~7005812,177156787~5609834,183215660~7587630,201412400~16196454,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~11859909,261607553~8181471,271903192~485755,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~23965275,367856693~3247417,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,406171321~6378805] 2024-09-14T04:32:25.114 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~13536984,51969188~10671211,62914560~9186737,75497472~2284581,79691776~15128681,96468992~3652024,100663296~32219598,134282812~4188311,139378274~20241483,163577856~10462175,176160768~14642522,201326592~16282262,218103808~412673,222298112~12132675,237089596~23924162,261607553~10781394,276824064~7288172,284870704~8098864,297545811~28367692,327155712~6219679,335544320~36181731,373293056~11321926,385875968~7626856,396566061~6312282,406171321~6378805] 2024-09-14T04:32:25.114 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 16 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,51969188~10671211,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~15914904,155839505~11131128,167034219~7005812,177156787~5609834,183215660~7587630,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~12054819,261607553~8181471,271903192~485755,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~28437657,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,406171321~6378805] 2024-09-14T04:32:25.114 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,51969188~10671211,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~15914904,155839505~11131128,167034219~7005812,177156787~5609834,183215660~7587630,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~12054819,261607553~8181471,271903192~485755,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~28437657,371654262~71789,376743309~7871673,389122080~4380744,396800802~6077541,406171321~6378805] 2024-09-14T04:32:25.119 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~13536984,51969188~10671211,62914560~9186737,75497472~2284581,79691776~15128681,96468992~3652024,100663296~32219598,132996120~5475003,139378274~34661757,176160768~14642522,192937984~24670870,218103808~412673,222298112~12132675,237089596~24119072,261607553~10781394,276824064~7288172,284870704~8098864,297545811~28367692,327155712~6219679,335544320~36181731,373293056~11321926,385875968~7626856,396566061~6312282,406171321~6378805] 2024-09-14T04:32:25.119 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 17 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,50026379~14761958,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~27592359,167034219~7005812,177156787~5609834,183215660~7587630,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~12054819,261607553~8860187,271903192~485755,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~7871673,389122080~4380744,396800802~6077541,406171321~6378805] 2024-09-14T04:32:25.119 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,50026379~14761958,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~27592359,167034219~7005812,177156787~5609834,183215660~7587630,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~12054819,261607553~8860187,271903192~485755,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~7871673,389122080~4380744,396800802~6077541,406171321~6378805] 2024-09-14T04:32:25.125 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~13536984,50026379~22074918,75497472~2284581,79691776~15128681,96468992~3652024,100663296~32219598,132996120~5475003,139378274~34661757,176160768~14642522,192937984~24670870,218103808~412673,222298112~12132675,237089596~24119072,261607553~10781394,276824064~7288172,284870704~8098864,297545811~28367692,327155712~6219679,335544320~49070662,385875968~7626856,396566061~6312282,406171321~6378805] 2024-09-14T04:32:25.125 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 18 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,50026379~14761958,65187704~13622813,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~34661757,177156787~5609834,183215660~7587630,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~12054819,261607553~8860187,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~7871673,389122080~4380744,396800802~6077541,406171321~6378805] 2024-09-14T04:32:25.125 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,50026379~14761958,65187704~13622813,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~34661757,177156787~5609834,183215660~7587630,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~12054819,261607553~8860187,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~7871673,389122080~4380744,396800802~6077541,406171321~6378805] 2024-09-14T04:32:25.130 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~13536984,50026379~28784138,79691776~15128681,96468992~3652024,100663296~32219598,132996120~5475003,139378274~34661757,176160768~14642522,192937984~24670870,218103808~412673,222298112~12132675,237089596~24119072,261607553~13104181,276824064~7288172,284870704~8098864,297545811~28367692,327155712~6219679,335544320~49070662,385875968~7626856,396566061~6312282,406171321~6378805] 2024-09-14T04:32:25.131 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 19 diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:25.131 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:25.137 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,327155712~6219679,335544320~57958504,396566061~6312282,406171321~10755340] 2024-09-14T04:32:25.137 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 4 diff [42887289~3072128,51969188~8118367,89587254~5233203,210193421~7415433,251644403~5938599,258312268~1647698,285482181~5052215,328959633~4415758,358912822~7718906,389122080~4380744] 2024-09-14T04:32:25.137 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [42887289~3072128,51969188~8118367,89587254~5233203,210193421~7415433,251644403~5938599,258312268~1647698,285482181~5052215,328959633~4415758,358912822~7718906,389122080~4380744] 2024-09-14T04:32:25.142 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [41943040~4016377,51969188~8118367,89587254~5233203,210125843~7483011,247463936~12496030,285482181~5052215,328959633~4415758,358912822~7718906,385875968~7626856] 2024-09-14T04:32:25.142 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 5 diff [2471184~3081596,24366452~9505140,42887289~3072128,51969188~8118367,89587254~5233203,110037204~9726671,144649336~641416,167109705~3086191,178985346~3361728,183215660~3502696,206380204~2324485,210193421~7415433,251644403~5938599,258312268~1647698,285482181~5052215,328959633~4415758,358912822~7718906,389122080~4380744] 2024-09-14T04:32:25.142 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [2471184~3081596,24366452~9505140,42887289~3072128,51969188~8118367,89587254~5233203,110037204~9726671,144649336~641416,167109705~3086191,178985346~3361728,183215660~3502696,206380204~2324485,210193421~7415433,251644403~5938599,258312268~1647698,285482181~5052215,328959633~4415758,358912822~7718906,389122080~4380744] 2024-09-14T04:32:25.147 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~5552780,24366452~9505140,41943040~4016377,51969188~8118367,89587254~5233203,110037204~9726671,144649336~641416,167109705~3086191,176160768~10557588,206380204~2324485,210125843~7483011,247463936~12496030,285482181~5052215,328959633~4415758,358912822~7718906,385875968~7626856] 2024-09-14T04:32:25.148 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 6 diff [2471184~3081596,18815204~3919713,24366452~9505140,42887289~3072128,51969188~8118367,89587254~5233203,110037204~9726671,129593184~3289710,144649336~641416,167109705~3086191,177156787~5190287,183215660~3502696,206380204~2558338,210193421~7415433,229976328~4454459,240560472~1907077,251644403~5938599,258312268~1647698,261607553~1789794,285482181~5052215,328959633~4415758,352926650~13705078,389122080~4380744] 2024-09-14T04:32:25.148 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [2471184~3081596,18815204~3919713,24366452~9505140,42887289~3072128,51969188~8118367,89587254~5233203,110037204~9726671,129593184~3289710,144649336~641416,167109705~3086191,177156787~5190287,183215660~3502696,206380204~2558338,210193421~7415433,229976328~4454459,240560472~1907077,251644403~5938599,258312268~1647698,261607553~1789794,285482181~5052215,328959633~4415758,352926650~13705078,389122080~4380744] 2024-09-14T04:32:25.153 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~5552780,16777216~5957701,24366452~9505140,41943040~4016377,51969188~8118367,89587254~5233203,110037204~9726671,125829120~7053774,144649336~641416,167109705~3086191,176160768~10557588,206380204~2558338,210125843~7483011,229603977~4826810,239075328~3392221,247463936~12496030,261607553~1789794,285482181~5052215,328959633~4415758,352321536~14310192,385875968~7626856] 2024-09-14T04:32:25.153 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 7 diff [1560185~3992595,18815204~3919713,24366452~9505140,42887289~3072128,51969188~8118367,89587254~5233203,108930368~10833507,129593184~3289710,144649336~641416,167109705~3086191,177156787~5190287,183215660~3502696,206380204~2558338,210193421~7415433,227037563~7393224,240560472~1907077,251644403~5938599,258312268~1647698,261607553~1789794,285482181~5052215,300028752~3863222,304604600~522118,328959633~4415758,339416815~2817899,352926650~13705078,367856693~3247417,371654262~71789,389122080~4380744] 2024-09-14T04:32:25.153 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1560185~3992595,18815204~3919713,24366452~9505140,42887289~3072128,51969188~8118367,89587254~5233203,108930368~10833507,129593184~3289710,144649336~641416,167109705~3086191,177156787~5190287,183215660~3502696,206380204~2558338,210193421~7415433,227037563~7393224,240560472~1907077,251644403~5938599,258312268~1647698,261607553~1789794,285482181~5052215,300028752~3863222,304604600~522118,328959633~4415758,339416815~2817899,352926650~13705078,367856693~3247417,371654262~71789,389122080~4380744] 2024-09-14T04:32:25.159 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~5552780,16777216~5957701,24366452~9505140,41943040~4016377,51969188~8118367,89587254~5233203,104857600~14906275,125829120~7053774,144649336~641416,167109705~3086191,176160768~10557588,206380204~2558338,210125843~7483011,227037563~7393224,239075328~3392221,247463936~12496030,261607553~1789794,285482181~5052215,300028752~3863222,304295941~830777,328959633~4415758,335544320~6690394,352321536~19404515,385875968~7626856] 2024-09-14T04:32:25.159 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 8 diff [1560185~3992595,18815204~3919713,24366452~9505140,42821576~5427684,51969188~8531271,89587254~5233203,108930368~10833507,125750260~3113143,129593184~3289710,144649336~641416,167109705~3086191,177156787~5190287,183215660~3502696,206380204~2558338,210193421~7415433,227037563~7393224,240560472~1907077,251644403~5938599,258312268~1647698,261607553~1789794,271903192~485755,285482181~5052215,300028752~3863222,304604600~522118,311272896~6368811,319011790~6901713,328959633~4415758,339416815~2817899,348104878~4116217,352231748~211803,352926650~13705078,367856693~3247417,371654262~71789,389122080~4380744,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.159 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1560185~3992595,18815204~3919713,24366452~9505140,42821576~5427684,51969188~8531271,89587254~5233203,108930368~10833507,125750260~3113143,129593184~3289710,144649336~641416,167109705~3086191,177156787~5190287,183215660~3502696,206380204~2558338,210193421~7415433,227037563~7393224,240560472~1907077,251644403~5938599,258312268~1647698,261607553~1789794,271903192~485755,285482181~5052215,300028752~3863222,304604600~522118,311272896~6368811,319011790~6901713,328959633~4415758,339416815~2817899,348104878~4116217,352231748~211803,352926650~13705078,367856693~3247417,371654262~71789,389122080~4380744,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.164 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~5552780,16777216~5957701,24366452~9505140,41943040~6306220,51969188~8531271,89587254~5233203,104857600~14906275,121634816~11248078,144649336~641416,167109705~3086191,176160768~10557588,206380204~2558338,210125843~7483011,227037563~7393224,239075328~3392221,247463936~12496030,261607553~1789794,269789024~2599923,285482181~5052215,300028752~3863222,304295941~830777,311272896~6368811,319011790~6901713,328959633~4415758,335544320~6690394,348104878~23621173,385875968~7626856,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.164 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 9 diff [1560185~3992595,18815204~3919713,24366452~10417690,42821576~5427684,51969188~8531271,77103770~678283,87561278~7259179,102891979~16871896,125750260~3113143,129593184~3289710,144649336~641416,167109705~3086191,177156787~5609834,183215660~3502696,206380204~2558338,210193421~7415433,227037563~7393224,240560472~1907077,251644403~5938599,258312268~1647698,261607553~1789794,271903192~485755,285482181~5052215,300028752~7529868,311272896~6368811,318925599~6987904,328959633~4415758,339416815~2817899,343616361~175529,348104878~4116217,352231748~211803,352926650~13705078,367856693~3247417,371654262~71789,377219330~1842760,389122080~4380744,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.165 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1560185~3992595,18815204~3919713,24366452~10417690,42821576~5427684,51969188~8531271,77103770~678283,87561278~7259179,102891979~16871896,125750260~3113143,129593184~3289710,144649336~641416,167109705~3086191,177156787~5609834,183215660~3502696,206380204~2558338,210193421~7415433,227037563~7393224,240560472~1907077,251644403~5938599,258312268~1647698,261607553~1789794,271903192~485755,285482181~5052215,300028752~7529868,311272896~6368811,318925599~6987904,328959633~4415758,339416815~2817899,343616361~175529,348104878~4116217,352231748~211803,352926650~13705078,367856693~3247417,371654262~71789,377219330~1842760,389122080~4380744,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.170 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~5552780,16777216~5957701,24366452~10417690,41943040~6306220,51969188~8531271,75497472~2284581,83886080~10934377,100663296~19100579,121634816~11248078,144649336~641416,167109705~3086191,176160768~10557588,206380204~2558338,210125843~7483011,227037563~7393224,239075328~3392221,247463936~12496030,261607553~1789794,269789024~2599923,285482181~5052215,300028752~7529868,311272896~6368811,318925599~6987904,328959633~4415758,335544320~6690394,343616361~175529,348104878~23621173,377219330~1842760,385875968~7626856,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.170 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 10 diff [1560185~3992595,18815204~3919713,24366452~10417690,35645530~2311397,39500721~8748539,51969188~8531271,77103770~678283,79792402~6496993,87561278~7259179,102891979~20007158,125750260~3113143,129593184~3289710,144649336~7944386,167109705~3086191,177156787~5609834,183215660~3502696,206380204~2558338,210193421~7415433,227037563~7393224,240560472~1907077,244100700~3181961,251644403~9369355,261607553~1789794,271903192~485755,277143881~6968355,285482181~5052215,300028752~7529868,311272896~6368811,318925599~6987904,328959633~4415758,339416815~2817899,343616361~175529,348104878~4116217,352231748~211803,352926650~13705078,367856693~3247417,371654262~71789,377219330~1842760,389122080~4380744,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.170 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1560185~3992595,18815204~3919713,24366452~10417690,35645530~2311397,39500721~8748539,51969188~8531271,77103770~678283,79792402~6496993,87561278~7259179,102891979~20007158,125750260~3113143,129593184~3289710,144649336~7944386,167109705~3086191,177156787~5609834,183215660~3502696,206380204~2558338,210193421~7415433,227037563~7393224,240560472~1907077,244100700~3181961,251644403~9369355,261607553~1789794,271903192~485755,277143881~6968355,285482181~5052215,300028752~7529868,311272896~6368811,318925599~6987904,328959633~4415758,339416815~2817899,343616361~175529,348104878~4116217,352231748~211803,352926650~13705078,367856693~3247417,371654262~71789,377219330~1842760,389122080~4380744,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.175 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~5552780,16777216~5957701,24366452~10417690,35645530~2311397,39500721~8748539,51969188~8531271,75497472~2284581,79792402~15028055,100663296~32219598,144649336~7944386,167109705~3086191,176160768~10557588,206380204~2558338,210125843~7483011,227037563~7393224,239075328~3392221,243269632~4013029,247463936~13549822,261607553~1789794,269789024~2599923,276824064~7288172,285482181~5052215,300028752~7529868,311272896~6368811,318925599~6987904,328959633~4415758,335544320~6690394,343616361~175529,348104878~23621173,377219330~1842760,385875968~7626856,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.175 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 11 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~2311397,39500721~8748539,51969188~8531271,69873797~1502064,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,144649336~7944386,167109705~3086191,177156787~5609834,183215660~3502696,206380204~2558338,209917978~7690876,227037563~7393224,240560472~1907077,244100700~3181961,251644403~9369355,261607553~1789794,271903192~485755,277143881~6968355,285482181~5052215,297545811~10012809,311272896~6368811,318925599~6987904,328959633~4415758,339416815~2817899,343616361~175529,344830868~904290,348104878~18526850,367856693~3247417,371654262~71789,377219330~5208838,389122080~4380744,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.175 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~2311397,39500721~8748539,51969188~8531271,69873797~1502064,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,144649336~7944386,167109705~3086191,177156787~5609834,183215660~3502696,206380204~2558338,209917978~7690876,227037563~7393224,240560472~1907077,244100700~3181961,251644403~9369355,261607553~1789794,271903192~485755,277143881~6968355,285482181~5052215,297545811~10012809,311272896~6368811,318925599~6987904,328959633~4415758,339416815~2817899,343616361~175529,344830868~904290,348104878~18526850,367856693~3247417,371654262~71789,377219330~5208838,389122080~4380744,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.181 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~2311397,39500721~8748539,51969188~8531271,67108864~4266997,75497472~2284581,79792402~15028055,96468992~3652024,100663296~32219598,144649336~7944386,167109705~3086191,176160768~10557588,206380204~2558338,209917978~7690876,227037563~7393224,239075328~3392221,243269632~4013029,247463936~13549822,261607553~1789794,269789024~2599923,276824064~7288172,285482181~5052215,297545811~10012809,311272896~6368811,318925599~6987904,328959633~4415758,335544320~6690394,343616361~175529,344830868~904290,348104878~23621173,377219330~5208838,385875968~7626856,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.181 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 12 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~2311397,39500721~8816828,51969188~8531271,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,144649336~7944386,164144631~755755,167109705~6930326,177156787~5609834,183215660~3502696,206380204~2558338,209917978~7690876,227037563~7393224,240560472~1907077,244100700~3181961,251644403~9369355,261607553~1789794,271903192~485755,277143881~6968355,285482181~5052215,291130899~619944,297545811~10012809,311272896~7278754,318925599~6987904,328615132~4760259,339416815~2817899,343616361~175529,344830868~904290,348104878~18526850,367856693~3247417,371654262~71789,377219330~5208838,389122080~4380744,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.181 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~2311397,39500721~8816828,51969188~8531271,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,144649336~7944386,164144631~755755,167109705~6930326,177156787~5609834,183215660~3502696,206380204~2558338,209917978~7690876,227037563~7393224,240560472~1907077,244100700~3181961,251644403~9369355,261607553~1789794,271903192~485755,277143881~6968355,285482181~5052215,291130899~619944,297545811~10012809,311272896~7278754,318925599~6987904,328615132~4760259,339416815~2817899,343616361~175529,344830868~904290,348104878~18526850,367856693~3247417,371654262~71789,377219330~5208838,389122080~4380744,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.187 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~2311397,39500721~8816828,51969188~8531271,62914560~9186737,75497472~2284581,79792402~15028055,96468992~3652024,100663296~32219598,134282812~4188311,141461050~1072987,144649336~7944386,164144631~755755,167109705~6930326,176160768~10557588,206380204~2558338,209917978~7690876,227037563~7393224,239075328~3392221,243269632~4013029,247463936~13549822,261607553~1789794,269789024~2599923,276824064~7288172,285482181~5052215,291130899~619944,297545811~10012809,311272896~7278754,318925599~6987904,328615132~4760259,335544320~6690394,343616361~175529,344830868~904290,348104878~23621173,377219330~5208838,385875968~7626856,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.187 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 13 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~2311397,39500721~9681793,51969188~8531271,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,144649336~10643842,155839505~3780252,164144631~755755,167109705~6930326,177156787~5609834,183215660~3502696,201412400~4186671,206380204~2558338,209917978~7690876,218384295~132186,225312546~9118241,240560472~7894340,249153849~11859909,261607553~1789794,271903192~485755,277143881~6968355,285482181~5052215,291130899~619944,297545811~10012809,311272896~7278754,318925599~6987904,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,348104878~18526850,367856693~3247417,371654262~71789,377219330~6977238,389122080~4380744,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.187 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~2311397,39500721~9681793,51969188~8531271,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,144649336~10643842,155839505~3780252,164144631~755755,167109705~6930326,177156787~5609834,183215660~3502696,201412400~4186671,206380204~2558338,209917978~7690876,218384295~132186,225312546~9118241,240560472~7894340,249153849~11859909,261607553~1789794,271903192~485755,277143881~6968355,285482181~5052215,291130899~619944,297545811~10012809,311272896~7278754,318925599~6987904,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,348104878~18526850,367856693~3247417,371654262~71789,377219330~6977238,389122080~4380744,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.192 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~2311397,39500721~9681793,51969188~8531271,62914560~9186737,75497472~2284581,79792402~15028055,96468992~3652024,100663296~32219598,134282812~4188311,141461050~1072987,144649336~14970421,164144631~755755,167109705~6930326,176160768~10557588,201412400~4186671,206380204~2558338,209917978~7690876,218103808~412673,222298112~12132675,239075328~21938430,261607553~1789794,269789024~2599923,276824064~7288172,285482181~5052215,291130899~619944,297545811~10012809,311272896~7278754,318925599~6987904,328615132~4760259,335544320~6690394,343616361~175529,344830868~904290,348104878~23621173,377219330~6977238,385875968~7626856,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.192 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 14 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,51969188~8531271,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,144649336~10643842,155839505~3780252,164144631~755755,167109705~6930326,177156787~5609834,183215660~7587630,201412400~4186671,206380204~2558338,209917978~7690876,218384295~132186,225312546~9118241,237089596~11365216,249153849~11859909,261607553~7770980,271903192~485755,277143881~6968355,284870704~8098864,297545811~21005839,318925599~6987904,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,348104878~18526850,367856693~3247417,371654262~71789,377219330~6977238,389122080~4380744,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.192 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,51969188~8531271,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,144649336~10643842,155839505~3780252,164144631~755755,167109705~6930326,177156787~5609834,183215660~7587630,201412400~4186671,206380204~2558338,209917978~7690876,218384295~132186,225312546~9118241,237089596~11365216,249153849~11859909,261607553~7770980,271903192~485755,277143881~6968355,284870704~8098864,297545811~21005839,318925599~6987904,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,348104878~18526850,367856693~3247417,371654262~71789,377219330~6977238,389122080~4380744,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.198 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~13536984,51969188~8531271,62914560~9186737,75497472~2284581,79792402~15028055,96468992~3652024,100663296~32219598,134282812~4188311,141461050~1072987,144649336~14970421,164144631~755755,167109705~6930326,176160768~14642522,201412400~4186671,206380204~2558338,209917978~7690876,218103808~412673,222298112~12132675,237089596~23924162,261607553~7770980,269789024~2599923,276824064~7288172,284870704~8098864,297545811~21005839,318925599~6987904,328615132~4760259,335544320~6690394,343616361~175529,344830868~904290,348104878~23621173,377219330~6977238,385875968~7626856,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.198 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 15 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,51969188~10671211,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,139378274~15914904,155839505~3780252,164144631~755755,167109705~6930326,177156787~5609834,183215660~7587630,201412400~4186671,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~11859909,261607553~7770980,271903192~485755,276825837~7286399,284870704~8098864,297545811~21005839,318925599~6987904,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~18745370,367856693~3247417,371654262~71789,377219330~6977238,389122080~4380744,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.198 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,51969188~10671211,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,139378274~15914904,155839505~3780252,164144631~755755,167109705~6930326,177156787~5609834,183215660~7587630,201412400~4186671,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~11859909,261607553~7770980,271903192~485755,276825837~7286399,284870704~8098864,297545811~21005839,318925599~6987904,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~18745370,367856693~3247417,371654262~71789,377219330~6977238,389122080~4380744,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.203 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~13536984,51969188~10671211,62914560~9186737,75497472~2284581,79792402~15028055,96468992~3652024,100663296~32219598,134282812~4188311,139378274~20241483,164144631~755755,167109705~6930326,176160768~14642522,201412400~4186671,206380204~11228650,218103808~412673,222298112~12132675,237089596~23924162,261607553~7770980,269789024~2599923,276824064~7288172,284870704~8098864,297545811~21005839,318925599~6987904,328615132~4760259,335544320~6690394,343616361~175529,344830868~904290,347886358~23839693,377219330~6977238,385875968~7626856,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.203 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 16 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,51969188~10671211,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~15914904,155839505~11131128,167109705~6930326,177156787~5609834,183215660~7587630,195446529~10152542,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~12054819,261607553~7770980,271903192~485755,276825837~7286399,284870704~8098864,297545811~21005839,318925599~6987904,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~23217752,371654262~71789,377219330~6977238,389122080~4380744,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.204 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,51969188~10671211,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~15914904,155839505~11131128,167109705~6930326,177156787~5609834,183215660~7587630,195446529~10152542,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~12054819,261607553~7770980,271903192~485755,276825837~7286399,284870704~8098864,297545811~21005839,318925599~6987904,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~23217752,371654262~71789,377219330~6977238,389122080~4380744,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.209 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~13536984,51969188~10671211,62914560~9186737,75497472~2284581,79792402~15028055,96468992~3652024,100663296~32219598,132996120~5475003,139378274~27592359,167109705~6930326,176160768~14642522,192937984~12661087,206380204~11228650,218103808~412673,222298112~12132675,237089596~24119072,261607553~7770980,269789024~2599923,276824064~7288172,284870704~8098864,297545811~21005839,318925599~6987904,328615132~4760259,335544320~6690394,343616361~175529,344830868~904290,347886358~23839693,377219330~6977238,385875968~7626856,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.209 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 17 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,50026379~14761958,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~27592359,167109705~6930326,177156787~5609834,183215660~7587630,195446529~10152542,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~12054819,261607553~8860187,271903192~485755,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,389122080~4380744,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.209 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,50026379~14761958,65187704~6913593,77103770~678283,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~27592359,167109705~6930326,177156787~5609834,183215660~7587630,195446529~10152542,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~12054819,261607553~8860187,271903192~485755,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,389122080~4380744,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.215 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~13536984,50026379~22074918,75497472~2284581,79792402~15028055,96468992~3652024,100663296~32219598,132996120~5475003,139378274~27592359,167109705~6930326,176160768~14642522,192937984~12661087,206380204~11228650,218103808~412673,222298112~12132675,237089596~24119072,261607553~10781394,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4760259,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,385875968~7626856,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.215 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 18 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,50026379~14761958,65187704~13622813,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~34661757,177156787~5609834,183215660~7587630,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~12054819,261607553~8860187,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,389122080~4380744,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.215 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,50026379~14761958,65187704~13622813,79792402~6496993,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~34661757,177156787~5609834,183215660~7587630,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~12054819,261607553~8860187,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,389122080~4380744,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.220 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~13536984,50026379~28784138,79792402~15028055,96468992~3652024,100663296~32219598,132996120~5475003,139378274~34661757,176160768~14642522,192937984~13404833,206380204~11228650,218103808~412673,222298112~12132675,237089596~24119072,261607553~13104181,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4760259,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,385875968~7626856,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.220 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 19 diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:25.220 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:25.226 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~30182049,206380204~11228650,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4760259,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396566061~6312282,409194704~7731957] 2024-09-14T04:32:25.226 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to 5 diff [2471184~3081596,24366452~9505140,110037204~9726671,144649336~641416,167109705~3086191,178985346~3361728,183215660~3502696,206380204~2324485] 2024-09-14T04:32:25.226 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [2471184~3081596,24366452~9505140,110037204~9726671,144649336~641416,167109705~3086191,178985346~3361728,183215660~3502696,206380204~2324485] 2024-09-14T04:32:25.232 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~5552780,24366452~9505140,110037204~9726671,144649336~641416,167109705~3086191,176160768~10557588,206380204~2324485] 2024-09-14T04:32:25.232 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to 6 diff [2471184~3081596,18815204~3919713,24366452~9505140,110037204~9726671,129593184~3289710,144649336~641416,167109705~3086191,177156787~5190287,183215660~3502696,206380204~2558338,229976328~4454459,240560472~1907077,254016000~1071233,261607553~1789794,352926650~10870250] 2024-09-14T04:32:25.232 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [2471184~3081596,18815204~3919713,24366452~9505140,110037204~9726671,129593184~3289710,144649336~641416,167109705~3086191,177156787~5190287,183215660~3502696,206380204~2558338,229976328~4454459,240560472~1907077,254016000~1071233,261607553~1789794,352926650~10870250] 2024-09-14T04:32:25.237 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~5552780,16777216~5957701,24366452~9505140,110037204~9726671,125829120~7053774,144649336~641416,167109705~3086191,176160768~10557588,206380204~2558338,229603977~4826810,239075328~3392221,254016000~1071233,261607553~1789794,352321536~11475364] 2024-09-14T04:32:25.237 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to 7 diff [1560185~3992595,18815204~3919713,24366452~9505140,108930368~10833507,129593184~3289710,144649336~641416,167109705~3086191,177156787~5190287,183215660~3502696,206380204~2558338,227037563~7393224,240560472~1907077,254016000~1071233,261607553~1789794,300028752~3863222,304604600~522118,339416815~2817899,352926650~10870250,367856693~3247417,371654262~71789] 2024-09-14T04:32:25.238 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1560185~3992595,18815204~3919713,24366452~9505140,108930368~10833507,129593184~3289710,144649336~641416,167109705~3086191,177156787~5190287,183215660~3502696,206380204~2558338,227037563~7393224,240560472~1907077,254016000~1071233,261607553~1789794,300028752~3863222,304604600~522118,339416815~2817899,352926650~10870250,367856693~3247417,371654262~71789] 2024-09-14T04:32:25.243 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~5552780,16777216~5957701,24366452~9505140,104857600~14906275,125829120~7053774,144649336~641416,167109705~3086191,176160768~10557588,206380204~2558338,227037563~7393224,239075328~3392221,254016000~1071233,261607553~1789794,300028752~3863222,304295941~830777,335544320~6690394,352321536~11475364,366631728~5094323] 2024-09-14T04:32:25.243 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to 8 diff [1560185~3992595,18815204~3919713,24366452~9505140,42821576~5427684,56498898~4001561,108930368~10833507,125750260~3113143,129593184~3289710,144649336~641416,167109705~3086191,177156787~5190287,183215660~3502696,206380204~2558338,227037563~7393224,240560472~1907077,254016000~1071233,261607553~1789794,271903192~485755,300028752~3863222,304604600~522118,311272896~6368811,319011790~6901713,339416815~2817899,348104878~4116217,352231748~211803,352926650~10870250,367856693~3247417,371654262~71789,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.243 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1560185~3992595,18815204~3919713,24366452~9505140,42821576~5427684,56498898~4001561,108930368~10833507,125750260~3113143,129593184~3289710,144649336~641416,167109705~3086191,177156787~5190287,183215660~3502696,206380204~2558338,227037563~7393224,240560472~1907077,254016000~1071233,261607553~1789794,271903192~485755,300028752~3863222,304604600~522118,311272896~6368811,319011790~6901713,339416815~2817899,348104878~4116217,352231748~211803,352926650~10870250,367856693~3247417,371654262~71789,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.249 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~5552780,16777216~5957701,24366452~9505140,42821576~5427684,56498898~4001561,104857600~14906275,121634816~11248078,144649336~641416,167109705~3086191,176160768~10557588,206380204~2558338,227037563~7393224,239075328~3392221,254016000~1071233,261607553~1789794,269789024~2599923,300028752~3863222,304295941~830777,311272896~6368811,319011790~6901713,335544320~6690394,348104878~15692022,366631728~5094323,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.249 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to 9 diff [1560185~3992595,18815204~3919713,24366452~10417690,42821576~5427684,56498898~4001561,77103770~678283,87561278~3303988,102891979~16871896,125750260~3113143,129593184~3289710,144649336~641416,167109705~3086191,177156787~5609834,183215660~3502696,206380204~2558338,227037563~7393224,240560472~1907077,254016000~1071233,261607553~1789794,271903192~485755,300028752~7529868,311272896~6368811,318925599~6987904,339416815~2817899,343616361~175529,348104878~4116217,352231748~211803,352926650~10870250,367856693~3247417,371654262~71789,377219330~1842760,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.249 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1560185~3992595,18815204~3919713,24366452~10417690,42821576~5427684,56498898~4001561,77103770~678283,87561278~3303988,102891979~16871896,125750260~3113143,129593184~3289710,144649336~641416,167109705~3086191,177156787~5609834,183215660~3502696,206380204~2558338,227037563~7393224,240560472~1907077,254016000~1071233,261607553~1789794,271903192~485755,300028752~7529868,311272896~6368811,318925599~6987904,339416815~2817899,343616361~175529,348104878~4116217,352231748~211803,352926650~10870250,367856693~3247417,371654262~71789,377219330~1842760,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.254 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~5552780,16777216~5957701,24366452~10417690,42821576~5427684,56498898~4001561,75497472~2284581,83886080~6979186,100663296~19100579,121634816~11248078,144649336~641416,167109705~3086191,176160768~10557588,206380204~2558338,227037563~7393224,239075328~3392221,254016000~1071233,261607553~1789794,269789024~2599923,300028752~7529868,311272896~6368811,318925599~6987904,335544320~6690394,343616361~175529,348104878~15692022,366631728~5094323,377219330~1842760,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.254 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to 10 diff [1560185~3992595,18815204~3919713,24366452~10417690,35645530~2311397,39500721~8748539,56498898~4001561,77103770~678283,79792402~6496993,87561278~3303988,102891979~20007158,125750260~3113143,129593184~3289710,144649336~7944386,167109705~3086191,177156787~5609834,183215660~3502696,206380204~2558338,227037563~7393224,240560472~1907077,244100700~3181961,254016000~1071233,256962846~4050912,261607553~1789794,271903192~485755,277143881~6968355,300028752~7529868,311272896~6368811,318925599~6987904,339416815~2817899,343616361~175529,348104878~4116217,352231748~211803,352926650~10870250,367856693~3247417,371654262~71789,377219330~1842760,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.254 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1560185~3992595,18815204~3919713,24366452~10417690,35645530~2311397,39500721~8748539,56498898~4001561,77103770~678283,79792402~6496993,87561278~3303988,102891979~20007158,125750260~3113143,129593184~3289710,144649336~7944386,167109705~3086191,177156787~5609834,183215660~3502696,206380204~2558338,227037563~7393224,240560472~1907077,244100700~3181961,254016000~1071233,256962846~4050912,261607553~1789794,271903192~485755,277143881~6968355,300028752~7529868,311272896~6368811,318925599~6987904,339416815~2817899,343616361~175529,348104878~4116217,352231748~211803,352926650~10870250,367856693~3247417,371654262~71789,377219330~1842760,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.259 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~5552780,16777216~5957701,24366452~10417690,35645530~2311397,39500721~8748539,56498898~4001561,75497472~2284581,79792402~11072864,100663296~32219598,144649336~7944386,167109705~3086191,176160768~10557588,206380204~2558338,227037563~7393224,239075328~3392221,243269632~4013029,254016000~1071233,256962846~4050912,261607553~1789794,269789024~2599923,276824064~7288172,300028752~7529868,311272896~6368811,318925599~6987904,335544320~6690394,343616361~175529,348104878~15692022,366631728~5094323,377219330~1842760,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.260 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to 11 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~2311397,39500721~8748539,56498898~4001561,69873797~1502064,77103770~678283,79792402~6496993,87561278~3303988,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,144649336~7944386,167109705~3086191,177156787~5609834,183215660~3502696,206380204~2558338,209917978~1359027,227037563~7393224,240560472~1907077,244100700~3181961,254016000~1071233,256962846~4050912,261607553~1789794,271903192~485755,277143881~6968355,297545811~10012809,311272896~6368811,318925599~6987904,339416815~2817899,343616361~175529,344830868~904290,348104878~15692022,367856693~3247417,371654262~71789,377219330~5208838,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.260 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~2311397,39500721~8748539,56498898~4001561,69873797~1502064,77103770~678283,79792402~6496993,87561278~3303988,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,144649336~7944386,167109705~3086191,177156787~5609834,183215660~3502696,206380204~2558338,209917978~1359027,227037563~7393224,240560472~1907077,244100700~3181961,254016000~1071233,256962846~4050912,261607553~1789794,271903192~485755,277143881~6968355,297545811~10012809,311272896~6368811,318925599~6987904,339416815~2817899,343616361~175529,344830868~904290,348104878~15692022,367856693~3247417,371654262~71789,377219330~5208838,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.265 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~2311397,39500721~8748539,56498898~4001561,67108864~4266997,75497472~2284581,79792402~11072864,96468992~3652024,100663296~32219598,144649336~7944386,167109705~3086191,176160768~10557588,206380204~2558338,209917978~1359027,227037563~7393224,239075328~3392221,243269632~4013029,254016000~1071233,256962846~4050912,261607553~1789794,269789024~2599923,276824064~7288172,297545811~10012809,311272896~6368811,318925599~6987904,335544320~6690394,343616361~175529,344830868~904290,348104878~15692022,366631728~5094323,377219330~5208838,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.265 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to 12 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~2311397,39500721~8816828,56498898~4001561,65187704~6913593,77103770~678283,79792402~6496993,87561278~3303988,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,144649336~7944386,164144631~755755,167109705~6930326,177156787~5609834,183215660~3502696,206380204~2558338,209917978~1359027,227037563~7393224,240560472~1907077,244100700~3181961,254016000~1071233,256962846~4050912,261607553~1789794,271903192~485755,277143881~6968355,291130899~619944,297545811~10012809,311272896~7278754,318925599~6987904,328615132~3469460,339416815~2817899,343616361~175529,344830868~904290,348104878~15692022,367856693~3247417,371654262~71789,377219330~5208838,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.265 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~2311397,39500721~8816828,56498898~4001561,65187704~6913593,77103770~678283,79792402~6496993,87561278~3303988,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,144649336~7944386,164144631~755755,167109705~6930326,177156787~5609834,183215660~3502696,206380204~2558338,209917978~1359027,227037563~7393224,240560472~1907077,244100700~3181961,254016000~1071233,256962846~4050912,261607553~1789794,271903192~485755,277143881~6968355,291130899~619944,297545811~10012809,311272896~7278754,318925599~6987904,328615132~3469460,339416815~2817899,343616361~175529,344830868~904290,348104878~15692022,367856693~3247417,371654262~71789,377219330~5208838,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:24 smithi129 ceph-mon[101316]: pgmap v1790: 105 pgs: 105 active+clean; 1.1 GiB data, 3.1 GiB used, 712 GiB / 715 GiB avail; 224 KiB/s rd, 46 MiB/s wr, 348 op/s 2024-09-14T04:32:25.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:24 smithi129 ceph-mon[101316]: osdmap e1138: 8 total, 8 up, 8 in 2024-09-14T04:32:25.271 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~2311397,39500721~8816828,56498898~4001561,62914560~9186737,75497472~2284581,79792402~11072864,91960020~945991,96468992~3652024,100663296~32219598,134282812~4188311,141461050~1072987,144649336~7944386,164144631~755755,167109705~6930326,176160768~10557588,206380204~2558338,209917978~1359027,227037563~7393224,239075328~3392221,243269632~4013029,254016000~1071233,256962846~4050912,261607553~1789794,269789024~2599923,276824064~7288172,291130899~619944,297545811~10012809,311272896~7278754,318925599~6987904,328615132~3469460,335544320~6690394,343616361~175529,344830868~904290,348104878~15692022,366631728~5094323,377219330~5208838,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.271 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to 13 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~2311397,39500721~9681793,56498898~4001561,65187704~6913593,77103770~678283,79792402~6496993,87561278~3303988,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,144649336~10643842,155839505~3780252,164144631~755755,167109705~6930326,177156787~5609834,183215660~3502696,201412400~4186671,206380204~2558338,209917978~1359027,218384295~132186,225312546~9118241,240560472~7894340,249153849~4403906,254016000~1071233,256962846~4050912,261607553~1789794,271903192~485755,277143881~6968355,291130899~619944,297545811~10012809,311272896~7278754,318925599~6987904,328615132~3469460,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,348104878~15692022,367856693~3247417,371654262~71789,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.271 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~2311397,39500721~9681793,56498898~4001561,65187704~6913593,77103770~678283,79792402~6496993,87561278~3303988,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,144649336~10643842,155839505~3780252,164144631~755755,167109705~6930326,177156787~5609834,183215660~3502696,201412400~4186671,206380204~2558338,209917978~1359027,218384295~132186,225312546~9118241,240560472~7894340,249153849~4403906,254016000~1071233,256962846~4050912,261607553~1789794,271903192~485755,277143881~6968355,291130899~619944,297545811~10012809,311272896~7278754,318925599~6987904,328615132~3469460,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,348104878~15692022,367856693~3247417,371654262~71789,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.277 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~2311397,39500721~9681793,56498898~4001561,62914560~9186737,75497472~2284581,79792402~11072864,91960020~945991,96468992~3652024,100663296~32219598,134282812~4188311,141461050~1072987,144649336~14970421,164144631~755755,167109705~6930326,176160768~10557588,201412400~4186671,206380204~2558338,209917978~1359027,218103808~412673,222298112~12132675,239075328~9379484,249153849~4403906,254016000~1071233,256962846~4050912,261607553~1789794,269789024~2599923,276824064~7288172,291130899~619944,297545811~10012809,311272896~7278754,318925599~6987904,328615132~3469460,335544320~6690394,343616361~175529,344830868~904290,348104878~15692022,366631728~5094323,377219330~6977238,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.277 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to 14 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,56498898~4001561,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,144649336~10643842,155839505~3780252,164144631~755755,167109705~6930326,177156787~5609834,183215660~7587630,201412400~4186671,206380204~2558338,209917978~1359027,218384295~132186,225312546~9118241,237089596~11365216,249153849~4403906,253979221~7034537,261607553~7770980,271903192~485755,277143881~6968355,284870704~8098864,297545811~21005839,318925599~6987904,328615132~3469460,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,348104878~15692022,367856693~3247417,371654262~71789,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.277 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,56498898~4001561,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,144649336~10643842,155839505~3780252,164144631~755755,167109705~6930326,177156787~5609834,183215660~7587630,201412400~4186671,206380204~2558338,209917978~1359027,218384295~132186,225312546~9118241,237089596~11365216,249153849~4403906,253979221~7034537,261607553~7770980,271903192~485755,277143881~6968355,284870704~8098864,297545811~21005839,318925599~6987904,328615132~3469460,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,348104878~15692022,367856693~3247417,371654262~71789,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.282 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~13536984,56498898~4001561,62914560~9186737,75497472~2284581,79792402~11636353,91960020~945991,96468992~3652024,100663296~32219598,134282812~4188311,141461050~1072987,144649336~14970421,164144631~755755,167109705~6930326,176160768~14642522,201412400~4186671,206380204~2558338,209917978~1359027,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~7034537,261607553~7770980,269789024~2599923,276824064~7288172,284870704~8098864,297545811~21005839,318925599~6987904,328615132~3469460,335544320~6690394,343616361~175529,344830868~904290,348104878~15692022,366631728~5094323,377219330~6977238,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.283 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to 15 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,55349631~7290768,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,139378274~15914904,155839505~3780252,164144631~755755,167109705~6930326,177156787~5609834,183215660~7587630,201412400~4186671,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~7034537,261607553~7770980,271903192~485755,276825837~7286399,284870704~8098864,297545811~21005839,318925599~6987904,328615132~3469460,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~15910542,367856693~3247417,371654262~71789,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.283 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,55349631~7290768,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,139378274~15914904,155839505~3780252,164144631~755755,167109705~6930326,177156787~5609834,183215660~7587630,201412400~4186671,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~7034537,261607553~7770980,271903192~485755,276825837~7286399,284870704~8098864,297545811~21005839,318925599~6987904,328615132~3469460,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~15910542,367856693~3247417,371654262~71789,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.288 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~13536984,55349631~7290768,62914560~9186737,75497472~2284581,79792402~11636353,91960020~945991,96468992~3652024,100663296~32219598,134282812~4188311,139378274~20241483,164144631~755755,167109705~6930326,176160768~14642522,201412400~4186671,206380204~4896801,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~7034537,261607553~7770980,269789024~2599923,276824064~7288172,284870704~8098864,297545811~21005839,318925599~6987904,328615132~3469460,335544320~6690394,343616361~175529,344830868~904290,347886358~15910542,366631728~5094323,377219330~6977238,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.288 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to 16 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,53772129~974098,55349631~7290768,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~15914904,155839505~11131128,167109705~6930326,177156787~5609834,183215660~7587630,195446529~10152542,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~7229447,261607553~7770980,271903192~485755,276825837~7286399,284870704~8098864,297545811~21005839,318925599~6987904,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~23217752,371654262~71789,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.288 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,53772129~974098,55349631~7290768,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~15914904,155839505~11131128,167109705~6930326,177156787~5609834,183215660~7587630,195446529~10152542,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~7229447,261607553~7770980,271903192~485755,276825837~7286399,284870704~8098864,297545811~21005839,318925599~6987904,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~23217752,371654262~71789,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.294 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~13536984,53772129~974098,55349631~7290768,62914560~9186737,75497472~2284581,79792402~11636353,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~27592359,167109705~6930326,176160768~14642522,192937984~12661087,206380204~4896801,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~7229447,261607553~7770980,269789024~2599923,276824064~7288172,284870704~8098864,297545811~21005839,318925599~6987904,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~23839693,377219330~6977238,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.294 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to 17 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~27592359,167109705~6930326,177156787~5609834,183215660~7587630,195446529~10152542,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~7229447,261607553~8860187,271903192~485755,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.294 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~27592359,167109705~6930326,177156787~5609834,183215660~7587630,195446529~10152542,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~7229447,261607553~8860187,271903192~485755,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.300 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~16751666,75497472~2284581,79792402~11636353,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~27592359,167109705~6930326,176160768~14642522,192937984~12661087,206380204~4896801,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~7229447,261607553~10781394,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.300 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to 18 diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~13622813,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~34661757,177156787~5609834,183215660~7587630,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~7229447,261607553~8860187,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,390011506~750434,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.300 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~13622813,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~34661757,177156787~5609834,183215660~7587630,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~7229447,261607553~8860187,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,390011506~750434,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.306 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,16777216~5957701,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~23460886,79792402~11636353,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~34661757,176160768~14642522,192937984~13404833,206380204~4896801,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~7229447,261607553~13104181,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,390011506~750434,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.306 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to 19 diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:25.306 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:25.311 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~30182049,206380204~4896801,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:25.311 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to 6 diff [18815204~3919713,129593184~3289710,177156787~1880366,207366448~1572094,229976328~4454459,240560472~1907077,254016000~1071233,261607553~1789794,352926650~10870250] 2024-09-14T04:32:25.311 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [18815204~3919713,129593184~3289710,177156787~1880366,207366448~1572094,229976328~4454459,240560472~1907077,254016000~1071233,261607553~1789794,352926650~10870250] 2024-09-14T04:32:25.317 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [16777216~5957701,125829120~7053774,177156787~1880366,207366448~1572094,229603977~4826810,239075328~3392221,254016000~1071233,261607553~1789794,352321536~11475364] 2024-09-14T04:32:25.317 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to 7 diff [1560185~1377854,18815204~3919713,28814760~445192,108930368~6044747,118756847~532421,129593184~3289710,177156787~1880366,207366448~1572094,227037563~7393224,240560472~1907077,254016000~1071233,261607553~1789794,300028752~3863222,304604600~522118,339416815~2817899,352926650~10870250,367856693~3247417,371654262~71789] 2024-09-14T04:32:25.317 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1560185~1377854,18815204~3919713,28814760~445192,108930368~6044747,118756847~532421,129593184~3289710,177156787~1880366,207366448~1572094,227037563~7393224,240560472~1907077,254016000~1071233,261607553~1789794,300028752~3863222,304604600~522118,339416815~2817899,352926650~10870250,367856693~3247417,371654262~71789] 2024-09-14T04:32:25.322 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1560185~1377854,16777216~5957701,28814760~445192,104857600~10117515,118756847~532421,125829120~7053774,177156787~1880366,207366448~1572094,227037563~7393224,239075328~3392221,254016000~1071233,261607553~1789794,300028752~3863222,304295941~830777,335544320~6690394,352321536~11475364,366631728~5094323] 2024-09-14T04:32:25.323 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to 8 diff [1560185~1377854,18815204~3919713,28814760~445192,42821576~5427684,56498898~4001561,108930368~6044747,118756847~532421,125750260~3113143,129593184~3289710,177156787~1880366,207366448~1572094,227037563~7393224,240560472~1907077,254016000~1071233,261607553~1789794,271903192~485755,300028752~3863222,304604600~522118,311272896~6368811,319011790~6901713,339416815~2817899,348104878~4116217,352231748~211803,352926650~10870250,367856693~3247417,371654262~71789,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.323 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1560185~1377854,18815204~3919713,28814760~445192,42821576~5427684,56498898~4001561,108930368~6044747,118756847~532421,125750260~3113143,129593184~3289710,177156787~1880366,207366448~1572094,227037563~7393224,240560472~1907077,254016000~1071233,261607553~1789794,271903192~485755,300028752~3863222,304604600~522118,311272896~6368811,319011790~6901713,339416815~2817899,348104878~4116217,352231748~211803,352926650~10870250,367856693~3247417,371654262~71789,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.328 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1560185~1377854,16777216~5957701,28814760~445192,42821576~5427684,56498898~4001561,104857600~10117515,118756847~532421,121634816~11248078,177156787~1880366,207366448~1572094,227037563~7393224,239075328~3392221,254016000~1071233,261607553~1789794,269789024~2599923,300028752~3863222,304295941~830777,311272896~6368811,319011790~6901713,335544320~6690394,348104878~15692022,366631728~5094323,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.328 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to 9 diff [1560185~1377854,18815204~3919713,28814760~445192,33861843~922299,42821576~5427684,56498898~4001561,77103770~678283,87561278~3303988,102891979~12083136,118756847~532421,125750260~3113143,129593184~3289710,177156787~5609834,207366448~1572094,227037563~7393224,240560472~1907077,254016000~1071233,261607553~1789794,271903192~485755,300028752~7529868,311272896~6368811,318925599~6987904,339416815~2817899,343616361~175529,348104878~4116217,352231748~211803,352926650~10870250,367856693~3247417,371654262~71789,377219330~1842760,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.328 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1560185~1377854,18815204~3919713,28814760~445192,33861843~922299,42821576~5427684,56498898~4001561,77103770~678283,87561278~3303988,102891979~12083136,118756847~532421,125750260~3113143,129593184~3289710,177156787~5609834,207366448~1572094,227037563~7393224,240560472~1907077,254016000~1071233,261607553~1789794,271903192~485755,300028752~7529868,311272896~6368811,318925599~6987904,339416815~2817899,343616361~175529,348104878~4116217,352231748~211803,352926650~10870250,367856693~3247417,371654262~71789,377219330~1842760,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.334 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1560185~1377854,16777216~5957701,28814760~445192,33861843~922299,42821576~5427684,56498898~4001561,75497472~2284581,83886080~6979186,100663296~14311819,118756847~532421,121634816~11248078,177156787~5609834,207366448~1572094,227037563~7393224,239075328~3392221,254016000~1071233,261607553~1789794,269789024~2599923,300028752~7529868,311272896~6368811,318925599~6987904,335544320~6690394,343616361~175529,348104878~15692022,366631728~5094323,377219330~1842760,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.334 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to 10 diff [1560185~1377854,18815204~3919713,28814760~445192,33861843~922299,35645530~2311397,39500721~8748539,56498898~4001561,77103770~678283,79792402~6496993,87561278~3303988,102891979~12083136,117127695~5771442,125750260~3113143,129593184~3289710,145121240~7472482,177156787~5609834,207366448~1572094,227037563~7393224,240560472~1907077,244100700~3181961,254016000~1071233,256962846~4050912,261607553~1789794,271903192~485755,277143881~6968355,300028752~7529868,311272896~6368811,318925599~6987904,339416815~2817899,343616361~175529,348104878~4116217,352231748~211803,352926650~10870250,367856693~3247417,371654262~71789,377219330~1842760,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.335 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1560185~1377854,18815204~3919713,28814760~445192,33861843~922299,35645530~2311397,39500721~8748539,56498898~4001561,77103770~678283,79792402~6496993,87561278~3303988,102891979~12083136,117127695~5771442,125750260~3113143,129593184~3289710,145121240~7472482,177156787~5609834,207366448~1572094,227037563~7393224,240560472~1907077,244100700~3181961,254016000~1071233,256962846~4050912,261607553~1789794,271903192~485755,277143881~6968355,300028752~7529868,311272896~6368811,318925599~6987904,339416815~2817899,343616361~175529,348104878~4116217,352231748~211803,352926650~10870250,367856693~3247417,371654262~71789,377219330~1842760,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.340 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1560185~1377854,16777216~5957701,28814760~445192,33861843~922299,35645530~2311397,39500721~8748539,56498898~4001561,75497472~2284581,79792402~11072864,100663296~14311819,117127695~15755199,145121240~7472482,177156787~5609834,207366448~1572094,227037563~7393224,239075328~3392221,243269632~4013029,254016000~1071233,256962846~4050912,261607553~1789794,269789024~2599923,276824064~7288172,300028752~7529868,311272896~6368811,318925599~6987904,335544320~6690394,343616361~175529,348104878~15692022,366631728~5094323,377219330~1842760,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.340 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to 11 diff [1532502~4624507,18815204~3919713,28814760~445192,33861843~922299,35645530~2311397,39500721~8748539,56498898~4001561,69873797~1502064,77103770~678283,79792402~6496993,87561278~3303988,99264373~856643,102891979~12083136,117127695~5771442,125750260~3113143,129593184~3289710,145121240~7472482,177156787~5609834,207366448~1572094,209917978~1359027,227037563~7393224,240560472~1907077,244100700~3181961,254016000~1071233,256962846~4050912,261607553~1789794,271903192~485755,277143881~6968355,297545811~10012809,311272896~6368811,318925599~6987904,339416815~2817899,343616361~175529,344830868~904290,348104878~15692022,367856693~3247417,371654262~71789,377219330~5208838,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.340 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,28814760~445192,33861843~922299,35645530~2311397,39500721~8748539,56498898~4001561,69873797~1502064,77103770~678283,79792402~6496993,87561278~3303988,99264373~856643,102891979~12083136,117127695~5771442,125750260~3113143,129593184~3289710,145121240~7472482,177156787~5609834,207366448~1572094,209917978~1359027,227037563~7393224,240560472~1907077,244100700~3181961,254016000~1071233,256962846~4050912,261607553~1789794,271903192~485755,277143881~6968355,297545811~10012809,311272896~6368811,318925599~6987904,339416815~2817899,343616361~175529,344830868~904290,348104878~15692022,367856693~3247417,371654262~71789,377219330~5208838,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.345 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,16777216~5957701,28814760~445192,33861843~922299,35645530~2311397,39500721~8748539,56498898~4001561,67108864~4266997,75497472~2284581,79792402~11072864,96468992~3652024,100663296~14311819,117127695~15755199,145121240~7472482,177156787~5609834,207366448~1572094,209917978~1359027,227037563~7393224,239075328~3392221,243269632~4013029,254016000~1071233,256962846~4050912,261607553~1789794,269789024~2599923,276824064~7288172,297545811~10012809,311272896~6368811,318925599~6987904,335544320~6690394,343616361~175529,344830868~904290,348104878~15692022,366631728~5094323,377219330~5208838,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.345 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to 12 diff [1532502~4624507,18815204~3919713,28814760~445192,33861843~922299,35645530~2311397,39500721~8816828,56498898~4001561,65187704~6913593,77103770~678283,79792402~6496993,87561278~3303988,91960020~945991,99264373~856643,102891979~12083136,117127695~5771442,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,145121240~7472482,164144631~755755,168867811~5172220,177156787~5609834,207366448~1572094,209917978~1359027,227037563~7393224,240560472~1907077,244100700~3181961,254016000~1071233,256962846~4050912,261607553~1789794,271903192~485755,277143881~6968355,291130899~619944,297545811~10012809,311272896~7278754,318925599~6987904,328615132~3469460,339416815~2817899,343616361~175529,344830868~904290,348104878~15692022,367856693~3247417,371654262~71789,377219330~5208838,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.345 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,28814760~445192,33861843~922299,35645530~2311397,39500721~8816828,56498898~4001561,65187704~6913593,77103770~678283,79792402~6496993,87561278~3303988,91960020~945991,99264373~856643,102891979~12083136,117127695~5771442,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,145121240~7472482,164144631~755755,168867811~5172220,177156787~5609834,207366448~1572094,209917978~1359027,227037563~7393224,240560472~1907077,244100700~3181961,254016000~1071233,256962846~4050912,261607553~1789794,271903192~485755,277143881~6968355,291130899~619944,297545811~10012809,311272896~7278754,318925599~6987904,328615132~3469460,339416815~2817899,343616361~175529,344830868~904290,348104878~15692022,367856693~3247417,371654262~71789,377219330~5208838,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.351 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,16777216~5957701,28814760~445192,33861843~922299,35645530~2311397,39500721~8816828,56498898~4001561,62914560~9186737,75497472~2284581,79792402~11072864,91960020~945991,96468992~3652024,100663296~14311819,117127695~15755199,134282812~4188311,141461050~1072987,145121240~7472482,164144631~755755,168867811~5172220,177156787~5609834,207366448~1572094,209917978~1359027,227037563~7393224,239075328~3392221,243269632~4013029,254016000~1071233,256962846~4050912,261607553~1789794,269789024~2599923,276824064~7288172,291130899~619944,297545811~10012809,311272896~7278754,318925599~6987904,328615132~3469460,335544320~6690394,343616361~175529,344830868~904290,348104878~15692022,366631728~5094323,377219330~5208838,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.351 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to 13 diff [1532502~4624507,18815204~3919713,28814760~445192,33861843~922299,35645530~2311397,39500721~9681793,56498898~4001561,65187704~6913593,77103770~678283,79792402~6496993,87561278~3303988,91960020~945991,99264373~856643,102891979~12083136,117127695~5771442,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,145121240~10171938,155839505~3780252,164144631~755755,168867811~5172220,177156787~5609834,201412400~4186671,207366448~1572094,209917978~1359027,218384295~132186,225312546~9118241,240560472~7894340,249153849~4403906,254016000~1071233,256962846~4050912,261607553~1789794,271903192~485755,277143881~6968355,291130899~619944,297545811~10012809,311272896~7278754,318925599~6987904,328615132~3469460,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,348104878~15692022,367856693~3247417,371654262~71789,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.351 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,28814760~445192,33861843~922299,35645530~2311397,39500721~9681793,56498898~4001561,65187704~6913593,77103770~678283,79792402~6496993,87561278~3303988,91960020~945991,99264373~856643,102891979~12083136,117127695~5771442,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,145121240~10171938,155839505~3780252,164144631~755755,168867811~5172220,177156787~5609834,201412400~4186671,207366448~1572094,209917978~1359027,218384295~132186,225312546~9118241,240560472~7894340,249153849~4403906,254016000~1071233,256962846~4050912,261607553~1789794,271903192~485755,277143881~6968355,291130899~619944,297545811~10012809,311272896~7278754,318925599~6987904,328615132~3469460,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,348104878~15692022,367856693~3247417,371654262~71789,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.357 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,16777216~5957701,28814760~445192,33861843~922299,35645530~2311397,39500721~9681793,56498898~4001561,62914560~9186737,75497472~2284581,79792402~11072864,91960020~945991,96468992~3652024,100663296~14311819,117127695~15755199,134282812~4188311,141461050~1072987,145121240~14498517,164144631~755755,168867811~5172220,177156787~5609834,201412400~4186671,207366448~1572094,209917978~1359027,218103808~412673,222298112~12132675,239075328~9379484,249153849~4403906,254016000~1071233,256962846~4050912,261607553~1789794,269789024~2599923,276824064~7288172,291130899~619944,297545811~10012809,311272896~7278754,318925599~6987904,328615132~3469460,335544320~6690394,343616361~175529,344830868~904290,348104878~15692022,366631728~5094323,377219330~6977238,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.357 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to 14 diff [1532502~4624507,18815204~3919713,28814760~445192,33861843~922299,35645530~13536984,56498898~4001561,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~12083136,117127695~5771442,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,145121240~10171938,155839505~3780252,164144631~755755,168867811~5172220,177156787~5609834,183524730~7278560,201412400~4186671,207366448~1572094,209917978~1359027,218384295~132186,225312546~9118241,237089596~11365216,249153849~4403906,253979221~7034537,261607553~7770980,271903192~485755,277143881~6968355,284870704~8098864,297545811~21005839,318925599~6987904,328615132~3469460,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,348104878~15692022,367856693~3247417,371654262~71789,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.357 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,28814760~445192,33861843~922299,35645530~13536984,56498898~4001561,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~12083136,117127695~5771442,125750260~3113143,129593184~3289710,134282812~4188311,141461050~1072987,145121240~10171938,155839505~3780252,164144631~755755,168867811~5172220,177156787~5609834,183524730~7278560,201412400~4186671,207366448~1572094,209917978~1359027,218384295~132186,225312546~9118241,237089596~11365216,249153849~4403906,253979221~7034537,261607553~7770980,271903192~485755,277143881~6968355,284870704~8098864,297545811~21005839,318925599~6987904,328615132~3469460,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,348104878~15692022,367856693~3247417,371654262~71789,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.362 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,16777216~5957701,28814760~445192,33861843~922299,35645530~13536984,56498898~4001561,62914560~9186737,75497472~2284581,79792402~11636353,91960020~945991,96468992~3652024,100663296~14311819,117127695~15755199,134282812~4188311,141461050~1072987,145121240~14498517,164144631~755755,168867811~5172220,177156787~5609834,183524730~7278560,201412400~4186671,207366448~1572094,209917978~1359027,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~7034537,261607553~7770980,269789024~2599923,276824064~7288172,284870704~8098864,297545811~21005839,318925599~6987904,328615132~3469460,335544320~6690394,343616361~175529,344830868~904290,348104878~15692022,366631728~5094323,377219330~6977238,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.362 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to 15 diff [1532502~4624507,18815204~3919713,28814760~445192,33861843~922299,35645530~13536984,55349631~7290768,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,139378274~15914904,155839505~3780252,164144631~755755,168867811~5172220,177156787~5609834,183524730~7278560,201412400~4186671,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~7034537,261607553~7770980,271903192~485755,276825837~7286399,284870704~8098864,297545811~21005839,318925599~6987904,328615132~3469460,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~15910542,367856693~3247417,371654262~71789,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.362 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,28814760~445192,33861843~922299,35645530~13536984,55349631~7290768,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,134282812~4188311,139378274~15914904,155839505~3780252,164144631~755755,168867811~5172220,177156787~5609834,183524730~7278560,201412400~4186671,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~7034537,261607553~7770980,271903192~485755,276825837~7286399,284870704~8098864,297545811~21005839,318925599~6987904,328615132~3469460,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~15910542,367856693~3247417,371654262~71789,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.368 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,16777216~5957701,28814760~445192,33861843~922299,35645530~13536984,55349631~7290768,62914560~9186737,75497472~2284581,79792402~11636353,91960020~945991,96468992~3652024,100663296~32219598,134282812~4188311,139378274~20241483,164144631~755755,168867811~5172220,177156787~5609834,183524730~7278560,201412400~4186671,207366448~3910557,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~7034537,261607553~7770980,269789024~2599923,276824064~7288172,284870704~8098864,297545811~21005839,318925599~6987904,328615132~3469460,335544320~6690394,343616361~175529,344830868~904290,347886358~15910542,366631728~5094323,377219330~6977238,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.368 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to 16 diff [1532502~4624507,18815204~3919713,28814760~445192,33861843~922299,35645530~13536984,53772129~974098,55349631~7290768,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~15914904,155839505~11131128,168867811~5172220,177156787~5609834,183524730~7278560,195446529~10152542,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~7229447,261607553~7770980,271903192~485755,276825837~7286399,284870704~8098864,297545811~21005839,318925599~6987904,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~23217752,371654262~71789,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.368 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,28814760~445192,33861843~922299,35645530~13536984,53772129~974098,55349631~7290768,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~15914904,155839505~11131128,168867811~5172220,177156787~5609834,183524730~7278560,195446529~10152542,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~7229447,261607553~7770980,271903192~485755,276825837~7286399,284870704~8098864,297545811~21005839,318925599~6987904,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~23217752,371654262~71789,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.374 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,16777216~5957701,28814760~445192,33861843~922299,35645530~13536984,53772129~974098,55349631~7290768,62914560~9186737,75497472~2284581,79792402~11636353,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~27592359,168867811~5172220,177156787~5609834,183524730~7278560,192937984~12661087,207366448~3910557,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~7229447,261607553~7770980,269789024~2599923,276824064~7288172,284870704~8098864,297545811~21005839,318925599~6987904,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~23839693,377219330~6977238,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.374 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to 17 diff [1532502~4624507,18815204~3919713,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~27592359,168867811~5172220,177156787~5609834,183524730~7278560,195446529~10152542,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~7229447,261607553~8860187,271903192~485755,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.374 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~27592359,168867811~5172220,177156787~5609834,183524730~7278560,195446529~10152542,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~7229447,261607553~8860187,271903192~485755,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.380 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,16777216~5957701,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~16751666,75497472~2284581,79792402~11636353,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~27592359,168867811~5172220,177156787~5609834,183524730~7278560,192937984~12661087,207366448~3910557,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~7229447,261607553~10781394,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.380 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to 18 diff [1532502~4624507,18815204~3919713,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~13622813,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~34661757,177156787~5609834,183524730~7278560,195446529~10896288,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~7229447,261607553~8860187,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,390011506~750434,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.380 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,18815204~3919713,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~13622813,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~34661757,177156787~5609834,183524730~7278560,195446529~10896288,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~7229447,261607553~8860187,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,390011506~750434,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.386 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,16777216~5957701,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~23460886,79792402~11636353,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~34661757,177156787~5609834,183524730~7278560,192937984~13404833,207366448~3910557,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~7229447,261607553~13104181,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,390011506~750434,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.386 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to 19 diff [1532502~4624507,15956065~6778852,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183524730~8784543,192662147~2659618,195446529~10896288,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:25.386 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183524730~8784543,192662147~2659618,195446529~10896288,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:25.392 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~10152005,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,177156787~5609834,183524730~22818087,207366448~3910557,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:25.392 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to 7 diff [1560185~1377854,28814760~445192,108930368~6044747,118756847~532421,227037563~3194028,300028752~3863222,304604600~522118,339416815~2817899,367856693~3247417,371654262~71789] 2024-09-14T04:32:25.392 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1560185~1377854,28814760~445192,108930368~6044747,118756847~532421,227037563~3194028,300028752~3863222,304604600~522118,339416815~2817899,367856693~3247417,371654262~71789] 2024-09-14T04:32:25.398 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1560185~1377854,28814760~445192,104857600~10117515,118756847~532421,227037563~3194028,300028752~3863222,304295941~830777,335544320~6690394,366631728~5094323] 2024-09-14T04:32:25.398 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to 8 diff [1560185~1377854,28814760~445192,42821576~5427684,56498898~4001561,108930368~6044747,118756847~532421,125750260~3113143,227037563~3194028,271903192~485755,300028752~3863222,304604600~522118,311272896~6368811,319011790~6901713,339416815~2817899,348104878~4116217,352231748~211803,367856693~3247417,371654262~71789,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.398 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1560185~1377854,28814760~445192,42821576~5427684,56498898~4001561,108930368~6044747,118756847~532421,125750260~3113143,227037563~3194028,271903192~485755,300028752~3863222,304604600~522118,311272896~6368811,319011790~6901713,339416815~2817899,348104878~4116217,352231748~211803,367856693~3247417,371654262~71789,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.403 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1560185~1377854,28814760~445192,42821576~5427684,56498898~4001561,104857600~10117515,118756847~532421,121634816~7228587,227037563~3194028,269789024~2599923,300028752~3863222,304295941~830777,311272896~6368811,319011790~6901713,335544320~6690394,348104878~4338673,366631728~5094323,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.403 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to 9 diff [1560185~1377854,28814760~445192,33861843~922299,42821576~5427684,56498898~4001561,77103770~678283,87561278~3303988,102891979~12083136,118756847~532421,125750260~3113143,178708911~4057710,227037563~3194028,271903192~485755,300028752~7529868,311272896~6368811,318925599~6987904,339416815~2817899,343616361~175529,348104878~4116217,352231748~211803,367856693~3247417,371654262~71789,377219330~1842760,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.404 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1560185~1377854,28814760~445192,33861843~922299,42821576~5427684,56498898~4001561,77103770~678283,87561278~3303988,102891979~12083136,118756847~532421,125750260~3113143,178708911~4057710,227037563~3194028,271903192~485755,300028752~7529868,311272896~6368811,318925599~6987904,339416815~2817899,343616361~175529,348104878~4116217,352231748~211803,367856693~3247417,371654262~71789,377219330~1842760,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.409 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1560185~1377854,28814760~445192,33861843~922299,42821576~5427684,56498898~4001561,75497472~2284581,83886080~6979186,100663296~14311819,118756847~532421,121634816~7228587,178708911~4057710,227037563~3194028,269789024~2599923,300028752~7529868,311272896~6368811,318925599~6987904,335544320~6690394,343616361~175529,348104878~4338673,366631728~5094323,377219330~1842760,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.409 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to 10 diff [1560185~1377854,28814760~445192,33861843~922299,35645530~2311397,39500721~8748539,56498898~4001561,77103770~678283,79792402~6496993,87561278~3303988,102891979~12083136,117127695~5771442,125750260~3113143,145121240~7472482,178708911~4057710,227037563~3194028,244100700~3181961,256962846~4050912,271903192~485755,277143881~6968355,300028752~7529868,311272896~6368811,318925599~6987904,339416815~2817899,343616361~175529,348104878~4116217,352231748~211803,367856693~3247417,371654262~71789,377219330~1842760,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.409 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1560185~1377854,28814760~445192,33861843~922299,35645530~2311397,39500721~8748539,56498898~4001561,77103770~678283,79792402~6496993,87561278~3303988,102891979~12083136,117127695~5771442,125750260~3113143,145121240~7472482,178708911~4057710,227037563~3194028,244100700~3181961,256962846~4050912,271903192~485755,277143881~6968355,300028752~7529868,311272896~6368811,318925599~6987904,339416815~2817899,343616361~175529,348104878~4116217,352231748~211803,367856693~3247417,371654262~71789,377219330~1842760,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.415 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1560185~1377854,28814760~445192,33861843~922299,35645530~2311397,39500721~8748539,56498898~4001561,75497472~2284581,79792402~11072864,100663296~14311819,117127695~11735708,145121240~7472482,178708911~4057710,227037563~3194028,243269632~4013029,256962846~4050912,269789024~2599923,276824064~7288172,300028752~7529868,311272896~6368811,318925599~6987904,335544320~6690394,343616361~175529,348104878~4338673,366631728~5094323,377219330~1842760,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.415 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to 11 diff [1532502~4624507,28814760~445192,33861843~922299,35645530~2311397,39500721~8748539,56498898~4001561,69873797~1502064,77103770~678283,79792402~6496993,87561278~3303988,99264373~856643,102891979~12083136,117127695~5771442,125750260~3113143,145121240~7472482,178708911~4057710,209917978~1359027,227037563~3194028,244100700~3181961,256962846~4050912,271903192~485755,277143881~6968355,297545811~10012809,311272896~6368811,318925599~6987904,339416815~2817899,343616361~175529,344830868~904290,348104878~7742350,367856693~3247417,371654262~71789,377219330~5208838,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.415 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,28814760~445192,33861843~922299,35645530~2311397,39500721~8748539,56498898~4001561,69873797~1502064,77103770~678283,79792402~6496993,87561278~3303988,99264373~856643,102891979~12083136,117127695~5771442,125750260~3113143,145121240~7472482,178708911~4057710,209917978~1359027,227037563~3194028,244100700~3181961,256962846~4050912,271903192~485755,277143881~6968355,297545811~10012809,311272896~6368811,318925599~6987904,339416815~2817899,343616361~175529,344830868~904290,348104878~7742350,367856693~3247417,371654262~71789,377219330~5208838,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.421 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,28814760~445192,33861843~922299,35645530~2311397,39500721~8748539,56498898~4001561,67108864~4266997,75497472~2284581,79792402~11072864,96468992~3652024,100663296~14311819,117127695~11735708,145121240~7472482,178708911~4057710,209917978~1359027,227037563~3194028,243269632~4013029,256962846~4050912,269789024~2599923,276824064~7288172,297545811~10012809,311272896~6368811,318925599~6987904,335544320~6690394,343616361~175529,344830868~904290,348104878~7742350,366631728~5094323,377219330~5208838,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.421 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to 12 diff [1532502~4624507,28814760~445192,33861843~922299,35645530~2311397,39500721~8816828,56498898~4001561,65187704~6913593,77103770~678283,79792402~6496993,87561278~3303988,91960020~945991,99264373~856643,102891979~12083136,117127695~5771442,125750260~3113143,134282812~4188311,141461050~1072987,145121240~7472482,164144631~755755,168867811~5172220,178708911~4057710,209917978~1359027,227037563~3194028,244100700~3181961,256962846~4050912,271903192~485755,277143881~6968355,291130899~619944,297545811~10012809,311272896~7278754,318925599~6987904,328615132~3469460,339416815~2817899,343616361~175529,344830868~904290,348104878~7742350,367856693~3247417,371654262~71789,377219330~5208838,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.421 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,28814760~445192,33861843~922299,35645530~2311397,39500721~8816828,56498898~4001561,65187704~6913593,77103770~678283,79792402~6496993,87561278~3303988,91960020~945991,99264373~856643,102891979~12083136,117127695~5771442,125750260~3113143,134282812~4188311,141461050~1072987,145121240~7472482,164144631~755755,168867811~5172220,178708911~4057710,209917978~1359027,227037563~3194028,244100700~3181961,256962846~4050912,271903192~485755,277143881~6968355,291130899~619944,297545811~10012809,311272896~7278754,318925599~6987904,328615132~3469460,339416815~2817899,343616361~175529,344830868~904290,348104878~7742350,367856693~3247417,371654262~71789,377219330~5208838,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.426 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,28814760~445192,33861843~922299,35645530~2311397,39500721~8816828,56498898~4001561,62914560~9186737,75497472~2284581,79792402~11072864,91960020~945991,96468992~3652024,100663296~14311819,117127695~11735708,134282812~4188311,141461050~1072987,145121240~7472482,164144631~755755,168867811~5172220,178708911~4057710,209917978~1359027,227037563~3194028,243269632~4013029,256962846~4050912,269789024~2599923,276824064~7288172,291130899~619944,297545811~10012809,311272896~7278754,318925599~6987904,328615132~3469460,335544320~6690394,343616361~175529,344830868~904290,348104878~7742350,366631728~5094323,377219330~5208838,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.427 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to 13 diff [1532502~4624507,28814760~445192,33861843~922299,35645530~2311397,39500721~9681793,56498898~4001561,65187704~6913593,77103770~678283,79792402~6496993,87561278~3303988,91960020~945991,99264373~856643,102891979~12083136,117127695~5771442,125750260~3113143,134282812~4188311,141461050~1072987,145121240~10171938,155839505~3780252,164144631~755755,168867811~5172220,178708911~4057710,201412400~4186671,209917978~1359027,218384295~132186,225312546~5051871,242085844~6368968,249153849~4403906,256962846~4050912,271903192~485755,277143881~6968355,291130899~619944,297545811~10012809,311272896~7278754,318925599~6987904,328615132~3469460,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,348104878~7742350,367856693~3247417,371654262~71789,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.427 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,28814760~445192,33861843~922299,35645530~2311397,39500721~9681793,56498898~4001561,65187704~6913593,77103770~678283,79792402~6496993,87561278~3303988,91960020~945991,99264373~856643,102891979~12083136,117127695~5771442,125750260~3113143,134282812~4188311,141461050~1072987,145121240~10171938,155839505~3780252,164144631~755755,168867811~5172220,178708911~4057710,201412400~4186671,209917978~1359027,218384295~132186,225312546~5051871,242085844~6368968,249153849~4403906,256962846~4050912,271903192~485755,277143881~6968355,291130899~619944,297545811~10012809,311272896~7278754,318925599~6987904,328615132~3469460,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,348104878~7742350,367856693~3247417,371654262~71789,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.432 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,28814760~445192,33861843~922299,35645530~2311397,39500721~9681793,56498898~4001561,62914560~9186737,75497472~2284581,79792402~11072864,91960020~945991,96468992~3652024,100663296~14311819,117127695~11735708,134282812~4188311,141461050~1072987,145121240~14498517,164144631~755755,168867811~5172220,178708911~4057710,201412400~4186671,209917978~1359027,218103808~412673,222298112~8066305,242085844~6368968,249153849~4403906,256962846~4050912,269789024~2599923,276824064~7288172,291130899~619944,297545811~10012809,311272896~7278754,318925599~6987904,328615132~3469460,335544320~6690394,343616361~175529,344830868~904290,348104878~7742350,366631728~5094323,377219330~6977238,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.432 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to 14 diff [1532502~4624507,28814760~445192,33861843~922299,35645530~13536984,56498898~4001561,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~12083136,117127695~5771442,125750260~3113143,134282812~4188311,141461050~1072987,145121240~10171938,155839505~3780252,164144631~755755,168867811~5172220,178708911~4057710,183524730~7278560,201412400~4186671,209917978~1359027,218384295~132186,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7034537,262851191~6527342,271903192~485755,277143881~6968355,284870704~8098864,297545811~21005839,318925599~6987904,328615132~3469460,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,348104878~7742350,367856693~3247417,371654262~71789,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.433 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,28814760~445192,33861843~922299,35645530~13536984,56498898~4001561,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~12083136,117127695~5771442,125750260~3113143,134282812~4188311,141461050~1072987,145121240~10171938,155839505~3780252,164144631~755755,168867811~5172220,178708911~4057710,183524730~7278560,201412400~4186671,209917978~1359027,218384295~132186,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7034537,262851191~6527342,271903192~485755,277143881~6968355,284870704~8098864,297545811~21005839,318925599~6987904,328615132~3469460,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,348104878~7742350,367856693~3247417,371654262~71789,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.438 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,28814760~445192,33861843~922299,35645530~13536984,56498898~4001561,62914560~9186737,75497472~2284581,79792402~11636353,91960020~945991,96468992~3652024,100663296~14311819,117127695~11735708,134282812~4188311,141461050~1072987,145121240~14498517,164144631~755755,168867811~5172220,178708911~4057710,183524730~7278560,201412400~4186671,209917978~1359027,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~7034537,262851191~6527342,269789024~2599923,276824064~7288172,284870704~8098864,297545811~21005839,318925599~6987904,328615132~3469460,335544320~6690394,343616361~175529,344830868~904290,348104878~7742350,366631728~5094323,377219330~6977238,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.438 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to 15 diff [1532502~4624507,28814760~445192,33861843~922299,35645530~13536984,55349631~7290768,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,134282812~4188311,139378274~15914904,155839505~3780252,164144631~755755,168867811~5172220,178708911~4057710,183524730~7278560,201412400~4186671,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7034537,262851191~6527342,271903192~485755,276825837~7286399,284870704~8098864,297545811~21005839,318925599~6987904,328615132~3469460,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,367856693~3247417,371654262~71789,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.438 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,28814760~445192,33861843~922299,35645530~13536984,55349631~7290768,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,134282812~4188311,139378274~15914904,155839505~3780252,164144631~755755,168867811~5172220,178708911~4057710,183524730~7278560,201412400~4186671,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7034537,262851191~6527342,271903192~485755,276825837~7286399,284870704~8098864,297545811~21005839,318925599~6987904,328615132~3469460,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,367856693~3247417,371654262~71789,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.444 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,28814760~445192,33861843~922299,35645530~13536984,55349631~7290768,62914560~9186737,75497472~2284581,79792402~11636353,91960020~945991,96468992~3652024,100663296~28200107,134282812~4188311,139378274~20241483,164144631~755755,168867811~5172220,178708911~4057710,183524730~7278560,201412400~4186671,207369398~3907607,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~7034537,262851191~6527342,269789024~2599923,276824064~7288172,284870704~8098864,297545811~21005839,318925599~6987904,328615132~3469460,335544320~6690394,343616361~175529,344830868~904290,347886358~7960870,366631728~5094323,377219330~6977238,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.444 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to 16 diff [1532502~4624507,28814760~445192,33861843~922299,35645530~13536984,53772129~974098,55349631~7290768,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~15914904,155839505~11131128,168867811~5172220,178708911~4057710,183524730~7278560,195446529~10152542,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7229447,262851191~6527342,271903192~485755,276825837~7286399,284870704~8098864,297545811~21005839,318925599~6987904,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~10276444,371654262~71789,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.444 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,28814760~445192,33861843~922299,35645530~13536984,53772129~974098,55349631~7290768,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~15914904,155839505~11131128,168867811~5172220,178708911~4057710,183524730~7278560,195446529~10152542,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7229447,262851191~6527342,271903192~485755,276825837~7286399,284870704~8098864,297545811~21005839,318925599~6987904,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~10276444,371654262~71789,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.449 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,28814760~445192,33861843~922299,35645530~13536984,53772129~974098,55349631~7290768,62914560~9186737,75497472~2284581,79792402~11636353,91960020~945991,96468992~3652024,100663296~28200107,132996120~5475003,139378274~27592359,168867811~5172220,178708911~4057710,183524730~7278560,192937984~12661087,207369398~3907607,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~7229447,262851191~6527342,269789024~2599923,276824064~7288172,284870704~8098864,297545811~21005839,318925599~6987904,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~7960870,360827666~10898385,377219330~6977238,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.449 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to 17 diff [1532502~4624507,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~27592359,168867811~5172220,178708911~4057710,183524730~7278560,195446529~10152542,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7229447,262779491~7688249,271903192~485755,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.449 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~27592359,168867811~5172220,178708911~4057710,183524730~7278560,195446529~10152542,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7229447,262779491~7688249,271903192~485755,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.455 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~16751666,75497472~2284581,79792402~11636353,91960020~945991,96468992~3652024,100663296~28200107,132996120~5475003,139378274~27592359,168867811~5172220,178708911~4057710,183524730~7278560,192937984~12661087,207369398~3907607,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~7229447,262779491~9609456,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.455 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to 18 diff [1532502~4624507,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~13622813,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~34661757,178708911~4057710,183524730~7278560,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7229447,262779491~7688249,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,390011506~750434,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.455 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~13622813,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~34661757,178708911~4057710,183524730~7278560,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7229447,262779491~7688249,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,390011506~750434,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.461 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~23460886,79792402~11636353,91960020~945991,96468992~3652024,100663296~28200107,132996120~5475003,139378274~34661757,178708911~4057710,183524730~7278560,192937984~13404833,207369398~3907607,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~7229447,262779491~11932243,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,390011506~750434,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.461 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to 19 diff [1532502~4624507,15956065~5397876,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:25.461 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:25.467 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~28200107,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:25.468 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to 8 diff [42821576~5427684,56498898~4001561,125750260~3113143,271903192~485755,311272896~6368811,319011790~6901713,348104878~4116217,352231748~211803,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.468 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [42821576~5427684,56498898~4001561,125750260~3113143,271903192~485755,311272896~6368811,319011790~6901713,348104878~4116217,352231748~211803,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.473 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [42821576~5427684,56498898~4001561,121634816~7228587,269789024~2599923,311272896~6368811,319011790~6901713,348104878~4338673,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.473 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to 9 diff [33861843~922299,42821576~5427684,56498898~4001561,77103770~678283,87561278~3303988,102891979~6289219,113214585~112979,125750260~3113143,178708911~4057710,271903192~485755,303295189~4263431,311272896~6368811,318925599~6987904,343616361~175529,348104878~4116217,352231748~211803,377219330~1842760,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.473 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [33861843~922299,42821576~5427684,56498898~4001561,77103770~678283,87561278~3303988,102891979~6289219,113214585~112979,125750260~3113143,178708911~4057710,271903192~485755,303295189~4263431,311272896~6368811,318925599~6987904,343616361~175529,348104878~4116217,352231748~211803,377219330~1842760,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.478 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [33861843~922299,42821576~5427684,56498898~4001561,75497472~2284581,83886080~6979186,100663296~8517902,113214585~112979,121634816~7228587,178708911~4057710,269789024~2599923,303295189~4263431,311272896~6368811,318925599~6987904,343616361~175529,348104878~4338673,377219330~1842760,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.479 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to 10 diff [33861843~922299,35645530~2311397,39500721~8748539,56498898~4001561,77103770~678283,79792402~6496993,87561278~3303988,102891979~6289219,113214585~112979,117127695~5771442,125750260~3113143,145121240~7472482,178708911~4057710,244100700~3181961,256962846~4050912,271903192~485755,277143881~6968355,303295189~4263431,311272896~6368811,318925599~6987904,343616361~175529,348104878~4116217,352231748~211803,377219330~1842760,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.479 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [33861843~922299,35645530~2311397,39500721~8748539,56498898~4001561,77103770~678283,79792402~6496993,87561278~3303988,102891979~6289219,113214585~112979,117127695~5771442,125750260~3113143,145121240~7472482,178708911~4057710,244100700~3181961,256962846~4050912,271903192~485755,277143881~6968355,303295189~4263431,311272896~6368811,318925599~6987904,343616361~175529,348104878~4116217,352231748~211803,377219330~1842760,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.484 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [33861843~922299,35645530~2311397,39500721~8748539,56498898~4001561,75497472~2284581,79792402~11072864,100663296~8517902,113214585~112979,117127695~11735708,145121240~7472482,178708911~4057710,243269632~4013029,256962846~4050912,269789024~2599923,276824064~7288172,303295189~4263431,311272896~6368811,318925599~6987904,343616361~175529,348104878~4338673,377219330~1842760,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.484 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to 11 diff [1532502~4624507,33861843~922299,35645530~2311397,39500721~8748539,56498898~4001561,69873797~1502064,77103770~678283,79792402~6496993,87561278~3303988,99264373~856643,102891979~6289219,113214585~112979,117127695~5771442,125750260~3113143,145121240~7472482,178708911~4057710,209917978~1359027,244100700~3181961,256962846~4050912,271903192~485755,277143881~6968355,297545811~10012809,311272896~6368811,318925599~6987904,343616361~175529,344830868~904290,348104878~7742350,377219330~5208838,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.484 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,33861843~922299,35645530~2311397,39500721~8748539,56498898~4001561,69873797~1502064,77103770~678283,79792402~6496993,87561278~3303988,99264373~856643,102891979~6289219,113214585~112979,117127695~5771442,125750260~3113143,145121240~7472482,178708911~4057710,209917978~1359027,244100700~3181961,256962846~4050912,271903192~485755,277143881~6968355,297545811~10012809,311272896~6368811,318925599~6987904,343616361~175529,344830868~904290,348104878~7742350,377219330~5208838,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.489 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,33861843~922299,35645530~2311397,39500721~8748539,56498898~4001561,67108864~4266997,75497472~2284581,79792402~11072864,96468992~3652024,100663296~8517902,113214585~112979,117127695~11735708,145121240~7472482,178708911~4057710,209917978~1359027,243269632~4013029,256962846~4050912,269789024~2599923,276824064~7288172,297545811~10012809,311272896~6368811,318925599~6987904,343616361~175529,344830868~904290,348104878~7742350,377219330~5208838,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.490 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to 12 diff [1532502~4624507,33861843~922299,35645530~2311397,39500721~8816828,56498898~4001561,65187704~6913593,77103770~678283,79792402~6496993,87561278~3303988,91960020~945991,99264373~856643,102891979~6289219,113214585~112979,117127695~5771442,125750260~3113143,134282812~4188311,141461050~1072987,145121240~7472482,164144631~755755,168867811~5172220,178708911~4057710,209917978~1359027,244100700~3181961,256962846~4050912,271903192~485755,277143881~6968355,291130899~619944,297545811~10012809,311272896~7278754,318925599~6987904,328615132~3469460,343616361~175529,344830868~904290,348104878~7742350,377219330~5208838,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.490 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,33861843~922299,35645530~2311397,39500721~8816828,56498898~4001561,65187704~6913593,77103770~678283,79792402~6496993,87561278~3303988,91960020~945991,99264373~856643,102891979~6289219,113214585~112979,117127695~5771442,125750260~3113143,134282812~4188311,141461050~1072987,145121240~7472482,164144631~755755,168867811~5172220,178708911~4057710,209917978~1359027,244100700~3181961,256962846~4050912,271903192~485755,277143881~6968355,291130899~619944,297545811~10012809,311272896~7278754,318925599~6987904,328615132~3469460,343616361~175529,344830868~904290,348104878~7742350,377219330~5208838,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.495 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,33861843~922299,35645530~2311397,39500721~8816828,56498898~4001561,62914560~9186737,75497472~2284581,79792402~11072864,91960020~945991,96468992~3652024,100663296~8517902,113214585~112979,117127695~11735708,134282812~4188311,141461050~1072987,145121240~7472482,164144631~755755,168867811~5172220,178708911~4057710,209917978~1359027,243269632~4013029,256962846~4050912,269789024~2599923,276824064~7288172,291130899~619944,297545811~10012809,311272896~7278754,318925599~6987904,328615132~3469460,343616361~175529,344830868~904290,348104878~7742350,377219330~5208838,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.495 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to 13 diff [1532502~4624507,33861843~922299,35645530~2311397,39500721~9681793,56498898~4001561,65187704~6913593,77103770~678283,79792402~6496993,87561278~3303988,91960020~945991,99264373~856643,102891979~6289219,113214585~112979,117127695~5771442,125750260~3113143,134282812~4188311,141461050~1072987,145121240~10171938,155839505~3780252,164144631~755755,168867811~5172220,178708911~4057710,201412400~4186671,209917978~1359027,218384295~132186,225312546~5051871,242085844~6368968,249153849~4403906,256962846~4050912,271903192~485755,277143881~6968355,291130899~619944,297545811~10012809,311272896~7278754,318925599~6987904,328615132~3469460,338917256~193834,343616361~175529,344830868~904290,348104878~7742350,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.495 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,33861843~922299,35645530~2311397,39500721~9681793,56498898~4001561,65187704~6913593,77103770~678283,79792402~6496993,87561278~3303988,91960020~945991,99264373~856643,102891979~6289219,113214585~112979,117127695~5771442,125750260~3113143,134282812~4188311,141461050~1072987,145121240~10171938,155839505~3780252,164144631~755755,168867811~5172220,178708911~4057710,201412400~4186671,209917978~1359027,218384295~132186,225312546~5051871,242085844~6368968,249153849~4403906,256962846~4050912,271903192~485755,277143881~6968355,291130899~619944,297545811~10012809,311272896~7278754,318925599~6987904,328615132~3469460,338917256~193834,343616361~175529,344830868~904290,348104878~7742350,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.501 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,33861843~922299,35645530~2311397,39500721~9681793,56498898~4001561,62914560~9186737,75497472~2284581,79792402~11072864,91960020~945991,96468992~3652024,100663296~8517902,113214585~112979,117127695~11735708,134282812~4188311,141461050~1072987,145121240~14498517,164144631~755755,168867811~5172220,178708911~4057710,201412400~4186671,209917978~1359027,218103808~412673,222298112~8066305,242085844~6368968,249153849~4403906,256962846~4050912,269789024~2599923,276824064~7288172,291130899~619944,297545811~10012809,311272896~7278754,318925599~6987904,328615132~3469460,338917256~193834,343616361~175529,344830868~904290,348104878~7742350,377219330~6977238,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.501 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to 14 diff [1532502~4624507,33861843~922299,35645530~13536984,56498898~4001561,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~6289219,113214585~112979,117127695~5771442,125750260~3113143,134282812~4188311,141461050~1072987,145121240~10171938,155839505~3780252,164144631~755755,168867811~5172220,178708911~4057710,183524730~7278560,201412400~4186671,209917978~1359027,218384295~132186,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7034537,262851191~6527342,271903192~485755,277143881~6968355,284870704~8098864,297545811~21005839,318925599~6987904,328615132~3469460,338917256~193834,343616361~175529,344830868~904290,348104878~7742350,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.501 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,33861843~922299,35645530~13536984,56498898~4001561,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~6289219,113214585~112979,117127695~5771442,125750260~3113143,134282812~4188311,141461050~1072987,145121240~10171938,155839505~3780252,164144631~755755,168867811~5172220,178708911~4057710,183524730~7278560,201412400~4186671,209917978~1359027,218384295~132186,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7034537,262851191~6527342,271903192~485755,277143881~6968355,284870704~8098864,297545811~21005839,318925599~6987904,328615132~3469460,338917256~193834,343616361~175529,344830868~904290,348104878~7742350,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.509 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,33861843~922299,35645530~13536984,56498898~4001561,62914560~9186737,75497472~2284581,79792402~11636353,91960020~945991,96468992~3652024,100663296~8517902,113214585~112979,117127695~11735708,134282812~4188311,141461050~1072987,145121240~14498517,164144631~755755,168867811~5172220,178708911~4057710,183524730~7278560,201412400~4186671,209917978~1359027,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~7034537,262851191~6527342,269789024~2599923,276824064~7288172,284870704~8098864,297545811~21005839,318925599~6987904,328615132~3469460,338917256~193834,343616361~175529,344830868~904290,348104878~7742350,377219330~6977238,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.509 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to 15 diff [1532502~4624507,33861843~922299,35645530~13536984,55349631~7290768,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~6289219,112894852~10004285,125750260~3113143,134282812~4188311,139378274~15914904,155839505~3780252,164144631~755755,168867811~5172220,178708911~4057710,183524730~7278560,201412400~4186671,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7034537,262851191~6527342,271903192~485755,276825837~7286399,284870704~8098864,297545811~21005839,318925599~6987904,328615132~3469460,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.509 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,33861843~922299,35645530~13536984,55349631~7290768,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~6289219,112894852~10004285,125750260~3113143,134282812~4188311,139378274~15914904,155839505~3780252,164144631~755755,168867811~5172220,178708911~4057710,183524730~7278560,201412400~4186671,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7034537,262851191~6527342,271903192~485755,276825837~7286399,284870704~8098864,297545811~21005839,318925599~6987904,328615132~3469460,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.511 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,33861843~922299,35645530~13536984,55349631~7290768,62914560~9186737,75497472~2284581,79792402~11636353,91960020~945991,96468992~3652024,100663296~8517902,112894852~15968551,134282812~4188311,139378274~20241483,164144631~755755,168867811~5172220,178708911~4057710,183524730~7278560,201412400~4186671,207369398~3907607,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~7034537,262851191~6527342,269789024~2599923,276824064~7288172,284870704~8098864,297545811~21005839,318925599~6987904,328615132~3469460,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,377219330~6977238,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.511 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to 16 diff [1532502~4624507,33861843~922299,35645530~13536984,53772129~974098,55349631~7290768,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~6289219,110614496~12284641,125750260~3113143,132996120~5475003,139378274~15914904,155839505~11131128,168867811~5172220,178708911~4057710,183524730~7278560,195446529~10152542,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7229447,262851191~6527342,271903192~485755,276825837~7286399,284870704~8098864,297545811~21005839,318925599~6987904,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.512 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,33861843~922299,35645530~13536984,53772129~974098,55349631~7290768,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~6289219,110614496~12284641,125750260~3113143,132996120~5475003,139378274~15914904,155839505~11131128,168867811~5172220,178708911~4057710,183524730~7278560,195446529~10152542,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7229447,262851191~6527342,271903192~485755,276825837~7286399,284870704~8098864,297545811~21005839,318925599~6987904,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.518 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,33861843~922299,35645530~13536984,53772129~974098,55349631~7290768,62914560~9186737,75497472~2284581,79792402~11636353,91960020~945991,96468992~3652024,100663296~8517902,110614496~18248907,132996120~5475003,139378274~27592359,168867811~5172220,178708911~4057710,183524730~7278560,192937984~12661087,207369398~3907607,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~7229447,262851191~6527342,269789024~2599923,276824064~7288172,284870704~8098864,297545811~21005839,318925599~6987904,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,377219330~6977238,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.518 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to 17 diff [1532502~4624507,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~6289219,110614496~12284641,125750260~3113143,132996120~5475003,139378274~27592359,168867811~5172220,178708911~4057710,183524730~7278560,195446529~10152542,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7229447,262779491~7688249,271903192~485755,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.518 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~6289219,110614496~12284641,125750260~3113143,132996120~5475003,139378274~27592359,168867811~5172220,178708911~4057710,183524730~7278560,195446529~10152542,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7229447,262779491~7688249,271903192~485755,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.523 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~16751666,75497472~2284581,79792402~11636353,91960020~945991,96468992~3652024,100663296~8517902,110614496~18248907,132996120~5475003,139378274~27592359,168867811~5172220,178708911~4057710,183524730~7278560,192937984~12661087,207369398~3907607,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~7229447,262779491~9609456,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.523 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to 18 diff [1532502~4624507,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~13622813,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~34661757,178708911~4057710,183524730~7278560,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7229447,262779491~7688249,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,390011506~750434,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.524 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~13622813,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~34661757,178708911~4057710,183524730~7278560,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7229447,262779491~7688249,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,390011506~750434,396800802~6077541,410335706~2214420] 2024-09-14T04:32:25.529 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~23460886,79792402~11636353,91960020~945991,96468992~3652024,100663296~28200107,132996120~5475003,139378274~34661757,178708911~4057710,183524730~7278560,192937984~13404833,207369398~3907607,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~7229447,262779491~11932243,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,390011506~750434,396566061~6312282,410335706~2214420] 2024-09-14T04:32:25.529 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to 19 diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:25.529 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:25.535 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~28200107,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:25.535 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to 9 diff [33861843~922299,77103770~678283,87561278~3303988,102891979~6289219,113214585~112979,178708911~4057710,303295189~4263431,318925599~4902066,343616361~175529,377219330~1842760] 2024-09-14T04:32:25.535 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [33861843~922299,77103770~678283,87561278~3303988,102891979~6289219,113214585~112979,178708911~4057710,303295189~4263431,318925599~4902066,343616361~175529,377219330~1842760] 2024-09-14T04:32:25.540 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [33861843~922299,75497472~2284581,83886080~6979186,100663296~8517902,113214585~112979,178708911~4057710,303295189~4263431,318925599~4902066,343616361~175529,377219330~1842760] 2024-09-14T04:32:25.541 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to 10 diff [33861843~922299,35645530~2311397,39500721~8119064,77103770~678283,79792402~6496993,87561278~3303988,102891979~6289219,113214585~112979,117127695~5771442,145121240~7472482,178708911~4057710,244100700~3181961,256962846~4050912,277143881~6968355,303295189~4263431,311587502~4444723,318925599~4902066,343616361~175529,377219330~1842760] 2024-09-14T04:32:25.541 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [33861843~922299,35645530~2311397,39500721~8119064,77103770~678283,79792402~6496993,87561278~3303988,102891979~6289219,113214585~112979,117127695~5771442,145121240~7472482,178708911~4057710,244100700~3181961,256962846~4050912,277143881~6968355,303295189~4263431,311587502~4444723,318925599~4902066,343616361~175529,377219330~1842760] 2024-09-14T04:32:25.546 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [33861843~922299,35645530~2311397,39500721~8119064,75497472~2284581,79792402~11072864,100663296~8517902,113214585~112979,117127695~5771442,145121240~7472482,178708911~4057710,243269632~4013029,256962846~4050912,276824064~7288172,303295189~4263431,311587502~4444723,318925599~4902066,343616361~175529,377219330~1842760] 2024-09-14T04:32:25.547 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to 11 diff [1532502~4624507,33861843~922299,35645530~2311397,39500721~8119064,69873797~1502064,77103770~678283,79792402~6496993,87561278~3303988,99264373~856643,102891979~6289219,113214585~112979,117127695~5771442,145121240~7472482,178708911~4057710,209917978~1359027,244100700~3181961,256962846~4050912,277143881~6968355,297545811~10012809,311587502~4444723,318925599~4902066,343616361~175529,344830868~904290,350270295~5576933,377219330~5208838] 2024-09-14T04:32:25.547 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,33861843~922299,35645530~2311397,39500721~8119064,69873797~1502064,77103770~678283,79792402~6496993,87561278~3303988,99264373~856643,102891979~6289219,113214585~112979,117127695~5771442,145121240~7472482,178708911~4057710,209917978~1359027,244100700~3181961,256962846~4050912,277143881~6968355,297545811~10012809,311587502~4444723,318925599~4902066,343616361~175529,344830868~904290,350270295~5576933,377219330~5208838] 2024-09-14T04:32:25.552 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,33861843~922299,35645530~2311397,39500721~8119064,67108864~4266997,75497472~2284581,79792402~11072864,96468992~3652024,100663296~8517902,113214585~112979,117127695~5771442,145121240~7472482,178708911~4057710,209917978~1359027,243269632~4013029,256962846~4050912,276824064~7288172,297545811~10012809,311587502~4444723,318925599~4902066,343616361~175529,344830868~904290,350270295~5576933,377219330~5208838] 2024-09-14T04:32:25.552 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to 12 diff [1532502~4624507,33861843~922299,35645530~2311397,39500721~8816828,65187704~6913593,77103770~678283,79792402~6496993,87561278~3303988,91960020~945991,99264373~856643,102891979~6289219,113214585~112979,117127695~5771442,134282812~4188311,141461050~1072987,145121240~7472482,164144631~755755,168867811~5172220,178708911~4057710,209917978~1359027,244100700~3181961,256962846~4050912,277143881~6968355,291130899~619944,297545811~10012809,311587502~6964148,318925599~4902066,328615132~3469460,343616361~175529,344830868~904290,350270295~5576933,377219330~5208838] 2024-09-14T04:32:25.553 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,33861843~922299,35645530~2311397,39500721~8816828,65187704~6913593,77103770~678283,79792402~6496993,87561278~3303988,91960020~945991,99264373~856643,102891979~6289219,113214585~112979,117127695~5771442,134282812~4188311,141461050~1072987,145121240~7472482,164144631~755755,168867811~5172220,178708911~4057710,209917978~1359027,244100700~3181961,256962846~4050912,277143881~6968355,291130899~619944,297545811~10012809,311587502~6964148,318925599~4902066,328615132~3469460,343616361~175529,344830868~904290,350270295~5576933,377219330~5208838] 2024-09-14T04:32:25.558 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,33861843~922299,35645530~2311397,39500721~8816828,62914560~9186737,75497472~2284581,79792402~11072864,91960020~945991,96468992~3652024,100663296~8517902,113214585~112979,117127695~5771442,134282812~4188311,141461050~1072987,145121240~7472482,164144631~755755,168867811~5172220,178708911~4057710,209917978~1359027,243269632~4013029,256962846~4050912,276824064~7288172,291130899~619944,297545811~10012809,311587502~6964148,318925599~4902066,328615132~3469460,343616361~175529,344830868~904290,350270295~5576933,377219330~5208838] 2024-09-14T04:32:25.558 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to 13 diff [1532502~4624507,33861843~922299,35645530~2311397,39500721~9681793,65187704~6913593,77103770~678283,79792402~6496993,87561278~3303988,91960020~945991,99264373~856643,102891979~6289219,113214585~112979,117127695~5771442,134282812~4188311,141461050~1072987,145121240~10171938,155839505~3780252,164144631~755755,168867811~5172220,178708911~4057710,201412400~4186671,209917978~1359027,218384295~132186,225312546~5051871,242085844~6368968,249153849~4403906,256962846~4050912,277143881~6968355,291130899~619944,297545811~10012809,311587502~6964148,318925599~4902066,328615132~3469460,338917256~193834,343616361~175529,344830868~904290,350270295~5576933,377219330~6977238] 2024-09-14T04:32:25.558 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,33861843~922299,35645530~2311397,39500721~9681793,65187704~6913593,77103770~678283,79792402~6496993,87561278~3303988,91960020~945991,99264373~856643,102891979~6289219,113214585~112979,117127695~5771442,134282812~4188311,141461050~1072987,145121240~10171938,155839505~3780252,164144631~755755,168867811~5172220,178708911~4057710,201412400~4186671,209917978~1359027,218384295~132186,225312546~5051871,242085844~6368968,249153849~4403906,256962846~4050912,277143881~6968355,291130899~619944,297545811~10012809,311587502~6964148,318925599~4902066,328615132~3469460,338917256~193834,343616361~175529,344830868~904290,350270295~5576933,377219330~6977238] 2024-09-14T04:32:25.563 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,33861843~922299,35645530~2311397,39500721~9681793,62914560~9186737,75497472~2284581,79792402~11072864,91960020~945991,96468992~3652024,100663296~8517902,113214585~112979,117127695~5771442,134282812~4188311,141461050~1072987,145121240~14498517,164144631~755755,168867811~5172220,178708911~4057710,201412400~4186671,209917978~1359027,218103808~412673,222298112~8066305,242085844~6368968,249153849~4403906,256962846~4050912,276824064~7288172,291130899~619944,297545811~10012809,311587502~6964148,318925599~4902066,328615132~3469460,338917256~193834,343616361~175529,344830868~904290,350270295~5576933,377219330~6977238] 2024-09-14T04:32:25.564 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to 14 diff [1532502~4624507,33861843~922299,35645530~13536984,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~6289219,113214585~112979,117127695~5771442,134282812~4188311,141461050~1072987,145121240~10171938,155839505~3780252,164144631~755755,168867811~5172220,178708911~4057710,183524730~7278560,201412400~4186671,209917978~1359027,218384295~132186,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7034537,262851191~6527342,277143881~6968355,284870704~8098864,297545811~21005839,318925599~4902066,328615132~3469460,338917256~193834,343616361~175529,344830868~904290,350270295~5576933,377219330~6977238] 2024-09-14T04:32:25.564 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,33861843~922299,35645530~13536984,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~6289219,113214585~112979,117127695~5771442,134282812~4188311,141461050~1072987,145121240~10171938,155839505~3780252,164144631~755755,168867811~5172220,178708911~4057710,183524730~7278560,201412400~4186671,209917978~1359027,218384295~132186,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7034537,262851191~6527342,277143881~6968355,284870704~8098864,297545811~21005839,318925599~4902066,328615132~3469460,338917256~193834,343616361~175529,344830868~904290,350270295~5576933,377219330~6977238] 2024-09-14T04:32:25.569 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,33861843~922299,35645530~13536984,62914560~9186737,75497472~2284581,79792402~11636353,91960020~945991,96468992~3652024,100663296~8517902,113214585~112979,117127695~5771442,134282812~4188311,141461050~1072987,145121240~14498517,164144631~755755,168867811~5172220,178708911~4057710,183524730~7278560,201412400~4186671,209917978~1359027,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~7034537,262851191~6527342,276824064~7288172,284870704~8098864,297545811~21005839,318925599~4902066,328615132~3469460,338917256~193834,343616361~175529,344830868~904290,350270295~5576933,377219330~6977238] 2024-09-14T04:32:25.569 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to 15 diff [1532502~4624507,33861843~922299,35645530~13536984,55349631~7290768,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~6289219,112894852~10004285,134282812~4188311,139378274~15914904,155839505~3780252,164144631~755755,168867811~5172220,178708911~4057710,183524730~7278560,201412400~4186671,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7034537,262851191~6527342,276825837~7286399,284870704~8098864,297545811~21005839,318925599~4902066,328615132~3469460,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,377219330~6977238] 2024-09-14T04:32:25.569 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,33861843~922299,35645530~13536984,55349631~7290768,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~6289219,112894852~10004285,134282812~4188311,139378274~15914904,155839505~3780252,164144631~755755,168867811~5172220,178708911~4057710,183524730~7278560,201412400~4186671,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7034537,262851191~6527342,276825837~7286399,284870704~8098864,297545811~21005839,318925599~4902066,328615132~3469460,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,377219330~6977238] 2024-09-14T04:32:25.574 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,33861843~922299,35645530~13536984,55349631~7290768,62914560~9186737,75497472~2284581,79792402~11636353,91960020~945991,96468992~3652024,100663296~8517902,112894852~10004285,134282812~4188311,139378274~20241483,164144631~755755,168867811~5172220,178708911~4057710,183524730~7278560,201412400~4186671,207369398~3907607,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~7034537,262851191~6527342,276824064~7288172,284870704~8098864,297545811~21005839,318925599~4902066,328615132~3469460,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,377219330~6977238] 2024-09-14T04:32:25.575 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to 16 diff [1532502~4624507,33861843~922299,35645530~13536984,53772129~974098,55349631~7290768,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~6289219,110614496~12284641,132996120~5475003,139378274~15914904,155839505~11131128,168867811~5172220,178708911~4057710,183524730~7278560,195446529~10152542,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7229447,262851191~6527342,276825837~7286399,284870704~8098864,297545811~21005839,318925599~4902066,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,377219330~6977238] 2024-09-14T04:32:25.575 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,33861843~922299,35645530~13536984,53772129~974098,55349631~7290768,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~6289219,110614496~12284641,132996120~5475003,139378274~15914904,155839505~11131128,168867811~5172220,178708911~4057710,183524730~7278560,195446529~10152542,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7229447,262851191~6527342,276825837~7286399,284870704~8098864,297545811~21005839,318925599~4902066,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,377219330~6977238] 2024-09-14T04:32:25.580 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,33861843~922299,35645530~13536984,53772129~974098,55349631~7290768,62914560~9186737,75497472~2284581,79792402~11636353,91960020~945991,96468992~3652024,100663296~8517902,110614496~12284641,132996120~5475003,139378274~27592359,168867811~5172220,178708911~4057710,183524730~7278560,192937984~12661087,207369398~3907607,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~7229447,262851191~6527342,276824064~7288172,284870704~8098864,297545811~21005839,318925599~4902066,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,377219330~6977238] 2024-09-14T04:32:25.580 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to 17 diff [1532502~4624507,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~6289219,110614496~12284641,132996120~5475003,139378274~27592359,168867811~5172220,178708911~4057710,183524730~7278560,195446529~10152542,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7229447,262779491~7688249,276825837~7286399,284870704~8098864,297545811~26281854,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238] 2024-09-14T04:32:25.580 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~6913593,77103770~678283,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~6289219,110614496~12284641,132996120~5475003,139378274~27592359,168867811~5172220,178708911~4057710,183524730~7278560,195446529~10152542,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7229447,262779491~7688249,276825837~7286399,284870704~8098864,297545811~26281854,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238] 2024-09-14T04:32:25.586 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~16751666,75497472~2284581,79792402~11636353,91960020~945991,96468992~3652024,100663296~8517902,110614496~12284641,132996120~5475003,139378274~27592359,168867811~5172220,178708911~4057710,183524730~7278560,192937984~12661087,207369398~3907607,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~7229447,262779491~7688249,276824064~7288172,284870704~8098864,297545811~26281854,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238] 2024-09-14T04:32:25.586 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to 18 diff [1532502~4624507,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~13622813,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,132996120~5475003,139378274~34661757,178708911~4057710,183524730~7278560,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7229447,262779491~7688249,271309101~3402633,276825837~7286399,284870704~8098864,297545811~26281854,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,390011506~750434] 2024-09-14T04:32:25.586 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~13622813,79792402~6496993,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,132996120~5475003,139378274~34661757,178708911~4057710,183524730~7278560,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7229447,262779491~7688249,271309101~3402633,276825837~7286399,284870704~8098864,297545811~26281854,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,390011506~750434] 2024-09-14T04:32:25.592 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~23460886,79792402~11636353,91960020~945991,96468992~3652024,100663296~22235841,132996120~5475003,139378274~34661757,178708911~4057710,183524730~7278560,192937984~13404833,207369398~3907607,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~7229447,262779491~7688249,271309101~3402633,276824064~7288172,284870704~8098864,297545811~26281854,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,390011506~750434] 2024-09-14T04:32:25.592 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to 19 diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.592 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.598 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~22235841,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276824064~7288172,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.598 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to 10 diff [35645530~2311397,39500721~8119064,79792402~6496993,88623968~370312,117127695~5771442,145121240~7472482,244100700~3181961,256962846~4050912,277143881~6968355,311587502~4444723] 2024-09-14T04:32:25.598 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [35645530~2311397,39500721~8119064,79792402~6496993,88623968~370312,117127695~5771442,145121240~7472482,244100700~3181961,256962846~4050912,277143881~6968355,311587502~4444723] 2024-09-14T04:32:25.603 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [35645530~2311397,39500721~8119064,79792402~6496993,88623968~370312,117127695~5771442,145121240~7472482,243269632~4013029,256962846~4050912,276824064~7288172,311587502~4444723] 2024-09-14T04:32:25.603 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to 11 diff [1532502~4624507,35645530~2311397,39500721~8119064,69873797~1502064,79792402~6496993,88623968~370312,99264373~856643,117127695~5771442,145121240~7472482,209917978~1359027,244100700~3181961,256962846~4050912,277143881~6968355,297545811~6956556,311587502~4444723,344830868~904290,350270295~5576933,378366520~4061648] 2024-09-14T04:32:25.603 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,35645530~2311397,39500721~8119064,69873797~1502064,79792402~6496993,88623968~370312,99264373~856643,117127695~5771442,145121240~7472482,209917978~1359027,244100700~3181961,256962846~4050912,277143881~6968355,297545811~6956556,311587502~4444723,344830868~904290,350270295~5576933,378366520~4061648] 2024-09-14T04:32:25.609 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,35645530~2311397,39500721~8119064,67108864~4266997,79792402~6496993,88623968~370312,96468992~3652024,117127695~5771442,145121240~7472482,209917978~1359027,243269632~4013029,256962846~4050912,276824064~7288172,297545811~6956556,311587502~4444723,344830868~904290,350270295~5576933,378366520~4061648] 2024-09-14T04:32:25.609 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to 12 diff [1532502~4624507,35645530~2311397,39500721~8816828,65187704~6913593,79792402~6496993,88623968~370312,91960020~945991,99264373~856643,117127695~5771442,134282812~4188311,141461050~1072987,145121240~7472482,164144631~755755,168867811~5172220,209917978~1359027,244100700~3181961,256962846~4050912,277143881~6968355,291130899~619944,297545811~6956556,311587502~6964148,328615132~3469460,344830868~904290,350270295~5576933,378366520~4061648] 2024-09-14T04:32:25.609 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,35645530~2311397,39500721~8816828,65187704~6913593,79792402~6496993,88623968~370312,91960020~945991,99264373~856643,117127695~5771442,134282812~4188311,141461050~1072987,145121240~7472482,164144631~755755,168867811~5172220,209917978~1359027,244100700~3181961,256962846~4050912,277143881~6968355,291130899~619944,297545811~6956556,311587502~6964148,328615132~3469460,344830868~904290,350270295~5576933,378366520~4061648] 2024-09-14T04:32:25.614 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,35645530~2311397,39500721~8816828,62914560~9186737,79792402~6496993,88623968~370312,91960020~945991,96468992~3652024,117127695~5771442,134282812~4188311,141461050~1072987,145121240~7472482,164144631~755755,168867811~5172220,209917978~1359027,243269632~4013029,256962846~4050912,276824064~7288172,291130899~619944,297545811~6956556,311587502~6964148,328615132~3469460,344830868~904290,350270295~5576933,378366520~4061648] 2024-09-14T04:32:25.614 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to 13 diff [1532502~4624507,35645530~2311397,39500721~9681793,65187704~6913593,79792402~6496993,88623968~370312,91960020~945991,99264373~856643,117127695~5771442,134282812~4188311,141461050~1072987,145121240~10171938,155839505~3780252,164144631~755755,168867811~5172220,201412400~4186671,209917978~1359027,218384295~132186,225312546~5051871,242085844~6368968,249153849~4403906,256962846~4050912,277143881~6968355,291130899~619944,297545811~6956556,311587502~6964148,328615132~3469460,338917256~193834,344830868~904290,350270295~5576933,378366520~5830048] 2024-09-14T04:32:25.614 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,35645530~2311397,39500721~9681793,65187704~6913593,79792402~6496993,88623968~370312,91960020~945991,99264373~856643,117127695~5771442,134282812~4188311,141461050~1072987,145121240~10171938,155839505~3780252,164144631~755755,168867811~5172220,201412400~4186671,209917978~1359027,218384295~132186,225312546~5051871,242085844~6368968,249153849~4403906,256962846~4050912,277143881~6968355,291130899~619944,297545811~6956556,311587502~6964148,328615132~3469460,338917256~193834,344830868~904290,350270295~5576933,378366520~5830048] 2024-09-14T04:32:25.620 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,35645530~2311397,39500721~9681793,62914560~9186737,79792402~6496993,88623968~370312,91960020~945991,96468992~3652024,117127695~5771442,134282812~4188311,141461050~1072987,145121240~14498517,164144631~755755,168867811~5172220,201412400~4186671,209917978~1359027,218103808~412673,222298112~8066305,242085844~6368968,249153849~4403906,256962846~4050912,276824064~7288172,291130899~619944,297545811~6956556,311587502~6964148,328615132~3469460,338917256~193834,344830868~904290,350270295~5576933,378366520~5830048] 2024-09-14T04:32:25.620 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to 14 diff [1532502~4624507,35645530~13536984,65187704~6913593,79792402~6496993,88623968~370312,89021763~2406992,91960020~945991,99264373~856643,117127695~5771442,134282812~4188311,141461050~1072987,145121240~10171938,155839505~3780252,164144631~755755,168867811~5172220,183524730~7278560,201412400~4186671,209917978~1359027,218384295~132186,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7034537,262851191~6527342,277143881~6968355,284870704~8098864,297545811~6956556,307097205~11454445,328615132~3469460,338917256~193834,344830868~904290,350270295~5576933,378366520~5830048] 2024-09-14T04:32:25.620 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,35645530~13536984,65187704~6913593,79792402~6496993,88623968~370312,89021763~2406992,91960020~945991,99264373~856643,117127695~5771442,134282812~4188311,141461050~1072987,145121240~10171938,155839505~3780252,164144631~755755,168867811~5172220,183524730~7278560,201412400~4186671,209917978~1359027,218384295~132186,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7034537,262851191~6527342,277143881~6968355,284870704~8098864,297545811~6956556,307097205~11454445,328615132~3469460,338917256~193834,344830868~904290,350270295~5576933,378366520~5830048] 2024-09-14T04:32:25.626 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,35645530~13536984,62914560~9186737,79792402~6496993,88623968~370312,89021763~2406992,91960020~945991,96468992~3652024,117127695~5771442,134282812~4188311,141461050~1072987,145121240~14498517,164144631~755755,168867811~5172220,183524730~7278560,201412400~4186671,209917978~1359027,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~7034537,262851191~6527342,276824064~7288172,284870704~8098864,297545811~6956556,307097205~11454445,328615132~3469460,338917256~193834,344830868~904290,350270295~5576933,378366520~5830048] 2024-09-14T04:32:25.626 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to 15 diff [1532502~4624507,35645530~13536984,55349631~7290768,65187704~6913593,79792402~6496993,88623968~370312,89021763~2406992,91960020~945991,99264373~856643,112894852~10004285,134282812~4188311,139378274~15914904,155839505~3780252,164144631~755755,168867811~5172220,183524730~7278560,201412400~4186671,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7034537,262851191~6527342,276825837~7286399,284870704~8098864,297545811~6956556,307097205~11454445,328615132~3469460,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,378366520~5830048] 2024-09-14T04:32:25.626 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,35645530~13536984,55349631~7290768,65187704~6913593,79792402~6496993,88623968~370312,89021763~2406992,91960020~945991,99264373~856643,112894852~10004285,134282812~4188311,139378274~15914904,155839505~3780252,164144631~755755,168867811~5172220,183524730~7278560,201412400~4186671,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7034537,262851191~6527342,276825837~7286399,284870704~8098864,297545811~6956556,307097205~11454445,328615132~3469460,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,378366520~5830048] 2024-09-14T04:32:25.631 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,35645530~13536984,55349631~7290768,62914560~9186737,79792402~6496993,88623968~370312,89021763~2406992,91960020~945991,96468992~3652024,112894852~10004285,134282812~4188311,139378274~20241483,164144631~755755,168867811~5172220,183524730~7278560,201412400~4186671,207369398~3907607,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~7034537,262851191~6527342,276824064~7288172,284870704~8098864,297545811~6956556,307097205~11454445,328615132~3469460,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,378366520~5830048] 2024-09-14T04:32:25.632 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to 16 diff [1532502~4624507,35645530~13536984,53772129~974098,55349631~7290768,65187704~6913593,79792402~6496993,88623968~370312,89021763~2406992,91960020~945991,99264373~856643,110614496~12284641,132996120~5475003,139378274~15914904,155839505~11131128,168867811~5172220,183524730~7278560,195446529~10152542,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7229447,262851191~6527342,276825837~7286399,284870704~8098864,297545811~6956556,307097205~11454445,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,378366520~5830048] 2024-09-14T04:32:25.632 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,35645530~13536984,53772129~974098,55349631~7290768,65187704~6913593,79792402~6496993,88623968~370312,89021763~2406992,91960020~945991,99264373~856643,110614496~12284641,132996120~5475003,139378274~15914904,155839505~11131128,168867811~5172220,183524730~7278560,195446529~10152542,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7229447,262851191~6527342,276825837~7286399,284870704~8098864,297545811~6956556,307097205~11454445,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,378366520~5830048] 2024-09-14T04:32:25.637 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,35645530~13536984,53772129~974098,55349631~7290768,62914560~9186737,79792402~6496993,88623968~370312,89021763~2406992,91960020~945991,96468992~3652024,110614496~12284641,132996120~5475003,139378274~27592359,168867811~5172220,183524730~7278560,192937984~12661087,207369398~3907607,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~7229447,262851191~6527342,276824064~7288172,284870704~8098864,297545811~6956556,307097205~11454445,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,378366520~5830048] 2024-09-14T04:32:25.638 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to 17 diff [1532502~4624507,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~6913593,79792402~6496993,88623968~370312,89021763~2406992,91960020~945991,99264373~856643,110614496~12284641,132996120~5475003,139378274~27592359,168867811~5172220,183524730~7278560,195446529~10152542,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7229447,262779491~7688249,276825837~7286399,284870704~8098864,297545811~6956556,307097205~12854986,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048] 2024-09-14T04:32:25.638 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~6913593,79792402~6496993,88623968~370312,89021763~2406992,91960020~945991,99264373~856643,110614496~12284641,132996120~5475003,139378274~27592359,168867811~5172220,183524730~7278560,195446529~10152542,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7229447,262779491~7688249,276825837~7286399,284870704~8098864,297545811~6956556,307097205~12854986,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048] 2024-09-14T04:32:25.643 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,35645530~13536984,50026379~3466398,53772129~974098,55349631~16751666,79792402~6496993,88623968~370312,89021763~2406992,91960020~945991,96468992~3652024,110614496~12284641,132996120~5475003,139378274~27592359,168867811~5172220,183524730~7278560,192937984~12661087,207369398~3907607,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~7229447,262779491~7688249,276824064~7288172,284870704~8098864,297545811~6956556,307097205~12854986,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048] 2024-09-14T04:32:25.643 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to 18 diff [1532502~4624507,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~13622813,79792402~6496993,88623968~370312,89021763~2406992,91960020~945991,99264373~856643,109021991~13877146,132996120~5475003,139378274~34661757,183524730~7278560,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7229447,262779491~7688249,271309101~3402633,276825837~7286399,284870704~8098864,297545811~6956556,307097205~12854986,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,390011506~750434] 2024-09-14T04:32:25.643 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~13622813,79792402~6496993,88623968~370312,89021763~2406992,91960020~945991,99264373~856643,109021991~13877146,132996120~5475003,139378274~34661757,183524730~7278560,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7229447,262779491~7688249,271309101~3402633,276825837~7286399,284870704~8098864,297545811~6956556,307097205~12854986,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,390011506~750434] 2024-09-14T04:32:25.648 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,35645530~13536984,50026379~3466398,53772129~974098,55349631~23460886,79792402~6496993,88623968~370312,89021763~2406992,91960020~945991,96468992~3652024,109021991~13877146,132996120~5475003,139378274~34661757,183524730~7278560,192937984~13404833,207369398~3907607,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~7229447,262779491~7688249,271309101~3402633,276824064~7288172,284870704~8098864,297545811~6956556,307097205~12854986,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,390011506~750434] 2024-09-14T04:32:25.648 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to 19 diff [1532502~4624507,15956065~5397876,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,88623968~370312,89021763~2406992,91960020~945991,99264373~856643,109021991~13877146,132996120~5475003,139378274~36780541,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~6956556,307097205~12854986,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.649 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,88623968~370312,89021763~2406992,91960020~945991,99264373~856643,109021991~13877146,132996120~5475003,139378274~36780541,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~6956556,307097205~12854986,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.654 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,35645530~13536984,50026379~3466398,53772129~974098,55349631~30939764,88623968~370312,89021763~2406992,91960020~945991,96468992~3652024,109021991~13877146,132996120~5475003,139378274~36780541,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276824064~7288172,284870704~8098864,297545811~6956556,307097205~12854986,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.654 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to 11 diff [1532502~4624507,40281884~3184041,69873797~1502064,99264373~856643,209917978~1359027,297545811~6956556,344830868~904290,350270295~5576933,378366520~4061648] 2024-09-14T04:32:25.654 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,40281884~3184041,69873797~1502064,99264373~856643,209917978~1359027,297545811~6956556,344830868~904290,350270295~5576933,378366520~4061648] 2024-09-14T04:32:25.659 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,40281884~3184041,67108864~4266997,96468992~3652024,209917978~1359027,297545811~6956556,344830868~904290,350270295~5576933,378366520~4061648] 2024-09-14T04:32:25.659 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to 12 diff [1532502~4624507,40281884~3184041,46171736~2145813,65187704~6913593,91960020~945991,99264373~856643,134282812~4188311,141461050~1072987,164144631~755755,168867811~5172220,209917978~1359027,291130899~619944,297545811~6956556,315889099~2662551,328615132~3469460,344830868~904290,350270295~5576933,378366520~4061648] 2024-09-14T04:32:25.659 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,40281884~3184041,46171736~2145813,65187704~6913593,91960020~945991,99264373~856643,134282812~4188311,141461050~1072987,164144631~755755,168867811~5172220,209917978~1359027,291130899~619944,297545811~6956556,315889099~2662551,328615132~3469460,344830868~904290,350270295~5576933,378366520~4061648] 2024-09-14T04:32:25.665 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,40281884~3184041,46171736~2145813,62914560~9186737,91960020~945991,96468992~3652024,134282812~4188311,141461050~1072987,164144631~755755,168867811~5172220,209917978~1359027,291130899~619944,297545811~6956556,315889099~2662551,328615132~3469460,344830868~904290,350270295~5576933,378366520~4061648] 2024-09-14T04:32:25.665 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to 13 diff [1532502~4624507,40281884~8900630,65187704~6913593,91960020~945991,99264373~856643,134282812~4188311,141461050~1072987,148411414~6881764,155839505~3780252,164144631~755755,168867811~5172220,201412400~4186671,209917978~1359027,218384295~132186,225312546~5051871,242085844~6368968,249153849~4403906,291130899~619944,297545811~6956556,315889099~2662551,328615132~3469460,338917256~193834,344830868~904290,350270295~5576933,378366520~5830048] 2024-09-14T04:32:25.665 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,40281884~8900630,65187704~6913593,91960020~945991,99264373~856643,134282812~4188311,141461050~1072987,148411414~6881764,155839505~3780252,164144631~755755,168867811~5172220,201412400~4186671,209917978~1359027,218384295~132186,225312546~5051871,242085844~6368968,249153849~4403906,291130899~619944,297545811~6956556,315889099~2662551,328615132~3469460,338917256~193834,344830868~904290,350270295~5576933,378366520~5830048] 2024-09-14T04:32:25.671 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,40281884~8900630,62914560~9186737,91960020~945991,96468992~3652024,134282812~4188311,141461050~1072987,148411414~11208343,164144631~755755,168867811~5172220,201412400~4186671,209917978~1359027,218103808~412673,222298112~8066305,242085844~6368968,249153849~4403906,291130899~619944,297545811~6956556,315889099~2662551,328615132~3469460,338917256~193834,344830868~904290,350270295~5576933,378366520~5830048] 2024-09-14T04:32:25.671 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to 14 diff [1532502~4624507,37722924~11459590,65187704~6913593,89021763~2406992,91960020~945991,99264373~856643,134282812~4188311,141461050~1072987,148411414~6881764,155839505~3780252,164144631~755755,168867811~5172220,183524730~7278560,201412400~4186671,209917978~1359027,218384295~132186,225312546~5051871,237089596~11365216,249153849~4403906,253979221~3341796,262851191~6527342,284870704~8098864,297545811~6956556,307097205~6187809,315889099~2662551,328615132~3469460,338917256~193834,344830868~904290,350270295~5576933,378366520~5830048] 2024-09-14T04:32:25.671 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,37722924~11459590,65187704~6913593,89021763~2406992,91960020~945991,99264373~856643,134282812~4188311,141461050~1072987,148411414~6881764,155839505~3780252,164144631~755755,168867811~5172220,183524730~7278560,201412400~4186671,209917978~1359027,218384295~132186,225312546~5051871,237089596~11365216,249153849~4403906,253979221~3341796,262851191~6527342,284870704~8098864,297545811~6956556,307097205~6187809,315889099~2662551,328615132~3469460,338917256~193834,344830868~904290,350270295~5576933,378366520~5830048] 2024-09-14T04:32:25.677 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,37722924~11459590,62914560~9186737,89021763~2406992,91960020~945991,96468992~3652024,134282812~4188311,141461050~1072987,148411414~11208343,164144631~755755,168867811~5172220,183524730~7278560,201412400~4186671,209917978~1359027,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~3341796,262851191~6527342,284870704~8098864,297545811~6956556,307097205~6187809,315889099~2662551,328615132~3469460,338917256~193834,344830868~904290,350270295~5576933,378366520~5830048] 2024-09-14T04:32:25.677 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to 15 diff [1532502~4624507,37722924~11459590,55349631~7290768,65187704~6913593,89021763~2406992,91960020~945991,99264373~856643,112894852~9167908,134282812~4188311,139378274~7402656,148411414~6881764,155839505~3780252,164144631~755755,168867811~5172220,183524730~7278560,201412400~4186671,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~3341796,262851191~6527342,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~2662551,328615132~3469460,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,378366520~5830048] 2024-09-14T04:32:25.677 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,37722924~11459590,55349631~7290768,65187704~6913593,89021763~2406992,91960020~945991,99264373~856643,112894852~9167908,134282812~4188311,139378274~7402656,148411414~6881764,155839505~3780252,164144631~755755,168867811~5172220,183524730~7278560,201412400~4186671,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~3341796,262851191~6527342,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~2662551,328615132~3469460,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,378366520~5830048] 2024-09-14T04:32:25.682 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,37722924~11459590,55349631~7290768,62914560~9186737,89021763~2406992,91960020~945991,96468992~3652024,112894852~9167908,134282812~4188311,139378274~7402656,148411414~11208343,164144631~755755,168867811~5172220,183524730~7278560,201412400~4186671,207369398~3907607,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~3341796,262851191~6527342,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~2662551,328615132~3469460,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,378366520~5830048] 2024-09-14T04:32:25.682 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to 16 diff [1532502~4624507,37722924~11459590,53772129~974098,55349631~7290768,65187704~6913593,89021763~2406992,91960020~945991,99264373~856643,110614496~11448264,132996120~5475003,139378274~7402656,148411414~6881764,155839505~11131128,168867811~5172220,183524730~7278560,195446529~10152542,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~3341796,257815293~3393375,262851191~6527342,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~2662551,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,378366520~5830048] 2024-09-14T04:32:25.682 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,37722924~11459590,53772129~974098,55349631~7290768,65187704~6913593,89021763~2406992,91960020~945991,99264373~856643,110614496~11448264,132996120~5475003,139378274~7402656,148411414~6881764,155839505~11131128,168867811~5172220,183524730~7278560,195446529~10152542,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~3341796,257815293~3393375,262851191~6527342,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~2662551,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,378366520~5830048] 2024-09-14T04:32:25.687 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,37722924~11459590,53772129~974098,55349631~7290768,62914560~9186737,89021763~2406992,91960020~945991,96468992~3652024,110614496~11448264,132996120~5475003,139378274~7402656,148411414~18559219,168867811~5172220,183524730~7278560,192937984~12661087,207369398~3907607,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~3341796,257815293~3393375,262851191~6527342,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~2662551,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,378366520~5830048] 2024-09-14T04:32:25.688 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to 17 diff [1532502~4624507,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~6913593,89021763~2406992,91960020~945991,99264373~856643,110614496~11448264,132996120~5475003,139378274~7402656,148411414~18559219,168867811~5172220,183524730~7278560,195446529~10152542,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~3341796,257815293~3393375,262779491~7688249,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048] 2024-09-14T04:32:25.688 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~6913593,89021763~2406992,91960020~945991,99264373~856643,110614496~11448264,132996120~5475003,139378274~7402656,148411414~18559219,168867811~5172220,183524730~7278560,195446529~10152542,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~3341796,257815293~3393375,262779491~7688249,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048] 2024-09-14T04:32:25.693 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,37722924~11459590,50026379~3466398,53772129~974098,55349631~16751666,89021763~2406992,91960020~945991,96468992~3652024,110614496~11448264,132996120~5475003,139378274~7402656,148411414~18559219,168867811~5172220,183524730~7278560,192937984~12661087,207369398~3907607,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~3341796,257815293~3393375,262779491~7688249,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048] 2024-09-14T04:32:25.693 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to 18 diff [1532502~4624507,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~13622813,89021763~2406992,91960020~945991,99264373~856643,109021991~13040769,132996120~5475003,139378274~7402656,148411414~25628617,183524730~7278560,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7229447,262779491~7688249,271309101~3402633,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,390011506~750434] 2024-09-14T04:32:25.693 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~13622813,89021763~2406992,91960020~945991,99264373~856643,109021991~13040769,132996120~5475003,139378274~7402656,148411414~25628617,183524730~7278560,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7229447,262779491~7688249,271309101~3402633,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,390011506~750434] 2024-09-14T04:32:25.699 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,37722924~11459590,50026379~3466398,53772129~974098,55349631~23460886,89021763~2406992,91960020~945991,96468992~3652024,109021991~13040769,132996120~5475003,139378274~7402656,148411414~25628617,183524730~7278560,192937984~13404833,207369398~3907607,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~7229447,262779491~7688249,271309101~3402633,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,390011506~750434] 2024-09-14T04:32:25.699 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to 19 diff [1532502~4624507,15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,99264373~856643,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.699 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,99264373~856643,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.706 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,37722924~11459590,50026379~3466398,53772129~974098,55349631~25440567,89021763~2406992,91960020~945991,96468992~3652024,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.706 INFO:tasks.workunit.client.0.smithi039.stdout:from 11 to 12 diff [46171736~2145813,65187704~6913593,91960020~945991,134282812~4188311,141461050~1072987,164144631~755755,168867811~5172220,291130899~619944,315889099~2662551,328615132~3469460] 2024-09-14T04:32:25.706 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [46171736~2145813,65187704~6913593,91960020~945991,134282812~4188311,141461050~1072987,164144631~755755,168867811~5172220,291130899~619944,315889099~2662551,328615132~3469460] 2024-09-14T04:32:25.710 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [46171736~2145813,62914560~9186737,91960020~945991,134282812~4188311,141461050~1072987,164144631~755755,168867811~5172220,291130899~619944,315889099~2662551,328615132~3469460] 2024-09-14T04:32:25.710 INFO:tasks.workunit.client.0.smithi039.stdout:from 11 to 13 diff [41735170~7447344,65187704~6913593,91960020~945991,134282812~4188311,141461050~1072987,148411414~6881764,155839505~3780252,164144631~755755,168867811~5172220,201412400~4186671,218384295~132186,225312546~5051871,242085844~6368968,249153849~4403906,291130899~619944,315889099~2662551,328615132~3469460,338917256~193834,378397449~5799119] 2024-09-14T04:32:25.710 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [41735170~7447344,65187704~6913593,91960020~945991,134282812~4188311,141461050~1072987,148411414~6881764,155839505~3780252,164144631~755755,168867811~5172220,201412400~4186671,218384295~132186,225312546~5051871,242085844~6368968,249153849~4403906,291130899~619944,315889099~2662551,328615132~3469460,338917256~193834,378397449~5799119] 2024-09-14T04:32:25.716 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [41735170~7447344,62914560~9186737,91960020~945991,134282812~4188311,141461050~1072987,148411414~11208343,164144631~755755,168867811~5172220,201412400~4186671,218103808~412673,222298112~8066305,242085844~6368968,249153849~4403906,291130899~619944,315889099~2662551,328615132~3469460,338917256~193834,378397449~5799119] 2024-09-14T04:32:25.716 INFO:tasks.workunit.client.0.smithi039.stdout:from 11 to 14 diff [37722924~11459590,65187704~6913593,89021763~2406992,91960020~945991,134282812~4188311,141461050~1072987,148411414~6881764,155839505~3780252,164144631~755755,168867811~5172220,183524730~7278560,201412400~4186671,218384295~132186,225312546~5051871,237089596~11365216,249153849~4403906,253979221~3341796,262851191~6527342,284870704~8098864,307097205~6187809,315889099~2662551,328615132~3469460,338917256~193834,378397449~5799119] 2024-09-14T04:32:25.716 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [37722924~11459590,65187704~6913593,89021763~2406992,91960020~945991,134282812~4188311,141461050~1072987,148411414~6881764,155839505~3780252,164144631~755755,168867811~5172220,183524730~7278560,201412400~4186671,218384295~132186,225312546~5051871,237089596~11365216,249153849~4403906,253979221~3341796,262851191~6527342,284870704~8098864,307097205~6187809,315889099~2662551,328615132~3469460,338917256~193834,378397449~5799119] 2024-09-14T04:32:25.721 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [37722924~11459590,62914560~9186737,89021763~2406992,91960020~945991,134282812~4188311,141461050~1072987,148411414~11208343,164144631~755755,168867811~5172220,183524730~7278560,201412400~4186671,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~3341796,262851191~6527342,284870704~8098864,307097205~6187809,315889099~2662551,328615132~3469460,338917256~193834,378397449~5799119] 2024-09-14T04:32:25.721 INFO:tasks.workunit.client.0.smithi039.stdout:from 11 to 15 diff [37722924~11459590,55349631~7290768,65187704~6913593,89021763~2406992,91960020~945991,112894852~9167908,134282812~4188311,139378274~7402656,148411414~6881764,155839505~3780252,164144631~755755,168867811~5172220,183524730~7278560,201412400~4186671,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~3341796,262851191~6527342,276825837~452599,284870704~8098864,307097205~6187809,315889099~2662551,328615132~3469460,338917256~193834,347886358~1792403,378397449~5799119] 2024-09-14T04:32:25.721 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [37722924~11459590,55349631~7290768,65187704~6913593,89021763~2406992,91960020~945991,112894852~9167908,134282812~4188311,139378274~7402656,148411414~6881764,155839505~3780252,164144631~755755,168867811~5172220,183524730~7278560,201412400~4186671,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~3341796,262851191~6527342,276825837~452599,284870704~8098864,307097205~6187809,315889099~2662551,328615132~3469460,338917256~193834,347886358~1792403,378397449~5799119] 2024-09-14T04:32:25.727 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [37722924~11459590,55349631~7290768,62914560~9186737,89021763~2406992,91960020~945991,112894852~9167908,134282812~4188311,139378274~7402656,148411414~11208343,164144631~755755,168867811~5172220,183524730~7278560,201412400~4186671,207369398~3608367,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~3341796,262851191~6527342,276825837~452599,284870704~8098864,307097205~6187809,315889099~2662551,328615132~3469460,338917256~193834,347886358~1792403,378397449~5799119] 2024-09-14T04:32:25.727 INFO:tasks.workunit.client.0.smithi039.stdout:from 11 to 16 diff [37722924~11459590,53772129~974098,55349631~7290768,65187704~6913593,89021763~2406992,91960020~945991,110614496~11448264,132996120~5475003,139378274~7402656,148411414~6881764,155839505~11131128,168867811~5172220,183524730~7278560,195446529~10152542,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~3341796,257815293~3393375,262851191~6527342,276825837~452599,284870704~8098864,307097205~6187809,315889099~2662551,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,378397449~5799119] 2024-09-14T04:32:25.727 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [37722924~11459590,53772129~974098,55349631~7290768,65187704~6913593,89021763~2406992,91960020~945991,110614496~11448264,132996120~5475003,139378274~7402656,148411414~6881764,155839505~11131128,168867811~5172220,183524730~7278560,195446529~10152542,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~3341796,257815293~3393375,262851191~6527342,276825837~452599,284870704~8098864,307097205~6187809,315889099~2662551,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,378397449~5799119] 2024-09-14T04:32:25.733 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [37722924~11459590,53772129~974098,55349631~7290768,62914560~9186737,89021763~2406992,91960020~945991,110614496~11448264,132996120~5475003,139378274~7402656,148411414~18559219,168867811~5172220,183524730~7278560,192937984~12661087,207369398~3608367,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~3341796,257815293~3393375,262851191~6527342,276825837~452599,284870704~8098864,307097205~6187809,315889099~2662551,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,378397449~5799119] 2024-09-14T04:32:25.733 INFO:tasks.workunit.client.0.smithi039.stdout:from 11 to 17 diff [37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~6913593,89021763~2406992,91960020~945991,110614496~11448264,132996120~5475003,139378274~7402656,148411414~18559219,168867811~5172220,183524730~7278560,195446529~10152542,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~3341796,257815293~3393375,262779491~7688249,276825837~452599,284870704~8098864,307097205~6187809,315889099~4063092,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119] 2024-09-14T04:32:25.733 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~6913593,89021763~2406992,91960020~945991,110614496~11448264,132996120~5475003,139378274~7402656,148411414~18559219,168867811~5172220,183524730~7278560,195446529~10152542,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~3341796,257815293~3393375,262779491~7688249,276825837~452599,284870704~8098864,307097205~6187809,315889099~4063092,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119] 2024-09-14T04:32:25.739 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [37722924~11459590,50026379~3466398,53772129~974098,55349631~16751666,89021763~2406992,91960020~945991,110614496~11448264,132996120~5475003,139378274~7402656,148411414~18559219,168867811~5172220,183524730~7278560,192937984~12661087,207369398~3608367,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~3341796,257815293~3393375,262779491~7688249,276825837~452599,284870704~8098864,307097205~6187809,315889099~4063092,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119] 2024-09-14T04:32:25.739 INFO:tasks.workunit.client.0.smithi039.stdout:from 11 to 18 diff [37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~13622813,89021763~2406992,91960020~945991,109021991~13040769,132996120~5475003,139378274~7402656,148411414~25628617,183524730~7278560,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7229447,262779491~7688249,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,315889099~4063092,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,390011506~750434] 2024-09-14T04:32:25.739 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~13622813,89021763~2406992,91960020~945991,109021991~13040769,132996120~5475003,139378274~7402656,148411414~25628617,183524730~7278560,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7229447,262779491~7688249,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,315889099~4063092,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,390011506~750434] 2024-09-14T04:32:25.744 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [37722924~11459590,50026379~3466398,53772129~974098,55349631~23460886,89021763~2406992,91960020~945991,109021991~13040769,132996120~5475003,139378274~7402656,148411414~25628617,183524730~7278560,192937984~13404833,207369398~3608367,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~7229447,262779491~7688249,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,315889099~4063092,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,390011506~750434] 2024-09-14T04:32:25.744 INFO:tasks.workunit.client.0.smithi039.stdout:from 11 to 19 diff [15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.744 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.750 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,37722924~11459590,50026379~3466398,53772129~974098,55349631~25440567,89021763~2406992,91960020~945991,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~22818087,207369398~3608367,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.750 INFO:tasks.workunit.client.0.smithi039.stdout:from 12 to 13 diff [41735170~7447344,148411414~6881764,155839505~3780252,201412400~4186671,218384295~132186,225312546~5051871,242085844~6368968,249153849~4403906,338917256~193834,378397449~5799119] 2024-09-14T04:32:25.750 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [41735170~7447344,148411414~6881764,155839505~3780252,201412400~4186671,218384295~132186,225312546~5051871,242085844~6368968,249153849~4403906,338917256~193834,378397449~5799119] 2024-09-14T04:32:25.755 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [41735170~7447344,148411414~11208343,201412400~4186671,218103808~412673,222298112~8066305,242085844~6368968,249153849~4403906,338917256~193834,378397449~5799119] 2024-09-14T04:32:25.755 INFO:tasks.workunit.client.0.smithi039.stdout:from 12 to 14 diff [37722924~11459590,89021763~2406992,148411414~6881764,155839505~3780252,183524730~7278560,201412400~4186671,218384295~132186,225312546~5051871,237089596~11365216,249153849~4403906,253979221~3341796,262851191~6527342,284870704~8098864,307097205~6187809,338917256~193834,378397449~5799119] 2024-09-14T04:32:25.755 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [37722924~11459590,89021763~2406992,148411414~6881764,155839505~3780252,183524730~7278560,201412400~4186671,218384295~132186,225312546~5051871,237089596~11365216,249153849~4403906,253979221~3341796,262851191~6527342,284870704~8098864,307097205~6187809,338917256~193834,378397449~5799119] 2024-09-14T04:32:25.761 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [37722924~11459590,89021763~2406992,148411414~11208343,183524730~7278560,201412400~4186671,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~3341796,262851191~6527342,284870704~8098864,307097205~6187809,338917256~193834,378397449~5799119] 2024-09-14T04:32:25.761 INFO:tasks.workunit.client.0.smithi039.stdout:from 12 to 15 diff [37722924~11459590,55349631~7290768,89021763~2406992,112894852~9167908,139378274~7402656,148411414~6881764,155839505~3780252,183524730~7278560,201412400~4186671,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~3341796,262851191~6527342,276825837~452599,284870704~8098864,307097205~6187809,338917256~193834,347886358~1792403,378397449~5799119] 2024-09-14T04:32:25.761 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [37722924~11459590,55349631~7290768,89021763~2406992,112894852~9167908,139378274~7402656,148411414~6881764,155839505~3780252,183524730~7278560,201412400~4186671,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~3341796,262851191~6527342,276825837~452599,284870704~8098864,307097205~6187809,338917256~193834,347886358~1792403,378397449~5799119] 2024-09-14T04:32:25.767 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [37722924~11459590,55349631~7290768,89021763~2406992,112894852~9167908,139378274~7402656,148411414~11208343,183524730~7278560,201412400~4186671,207369398~3608367,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~3341796,262851191~6527342,276825837~452599,284870704~8098864,307097205~6187809,338917256~193834,347886358~1792403,378397449~5799119] 2024-09-14T04:32:25.767 INFO:tasks.workunit.client.0.smithi039.stdout:from 12 to 16 diff [37722924~11459590,53772129~974098,55349631~7290768,89021763~2406992,110614496~11448264,132996120~2690483,139378274~7402656,148411414~6881764,155839505~11131128,183524730~7278560,195446529~10152542,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~3341796,257815293~3393375,262851191~6527342,276825837~452599,284870704~8098864,307097205~6187809,332042044~606164,338917256~193834,347886358~1792403,360827666~8007252,378397449~5799119] 2024-09-14T04:32:25.767 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [37722924~11459590,53772129~974098,55349631~7290768,89021763~2406992,110614496~11448264,132996120~2690483,139378274~7402656,148411414~6881764,155839505~11131128,183524730~7278560,195446529~10152542,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~3341796,257815293~3393375,262851191~6527342,276825837~452599,284870704~8098864,307097205~6187809,332042044~606164,338917256~193834,347886358~1792403,360827666~8007252,378397449~5799119] 2024-09-14T04:32:25.772 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [37722924~11459590,53772129~974098,55349631~7290768,89021763~2406992,110614496~11448264,132996120~2690483,139378274~7402656,148411414~18559219,183524730~7278560,192937984~12661087,207369398~3608367,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~3341796,257815293~3393375,262851191~6527342,276825837~452599,284870704~8098864,307097205~6187809,332042044~606164,338917256~193834,347886358~1792403,360827666~8007252,378397449~5799119] 2024-09-14T04:32:25.772 INFO:tasks.workunit.client.0.smithi039.stdout:from 12 to 17 diff [37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,89021763~2406992,110614496~11448264,132996120~2690483,139378274~7402656,148411414~18559219,183524730~7278560,195446529~10152542,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~3341796,257815293~3393375,262779491~7688249,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,332042044~606164,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119] 2024-09-14T04:32:25.772 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,89021763~2406992,110614496~11448264,132996120~2690483,139378274~7402656,148411414~18559219,183524730~7278560,195446529~10152542,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~3341796,257815293~3393375,262779491~7688249,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,332042044~606164,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119] 2024-09-14T04:32:25.778 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,89021763~2406992,110614496~11448264,132996120~2690483,139378274~7402656,148411414~18559219,183524730~7278560,192937984~12661087,207369398~3608367,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~3341796,257815293~3393375,262779491~7688249,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,332042044~606164,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119] 2024-09-14T04:32:25.778 INFO:tasks.workunit.client.0.smithi039.stdout:from 12 to 18 diff [37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,71362387~7448130,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,148411414~25078783,183524730~7278560,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7229447,262779491~7688249,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,332042044~606164,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,390011506~750434] 2024-09-14T04:32:25.778 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,71362387~7448130,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,148411414~25078783,183524730~7278560,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,237089596~11365216,249153849~4403906,253979221~7229447,262779491~7688249,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,332042044~606164,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,390011506~750434] 2024-09-14T04:32:25.783 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,71362387~7448130,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,148411414~25078783,183524730~7278560,192937984~13404833,207369398~3608367,218103808~412673,222298112~8066305,237089596~11365216,249153849~4403906,253979221~7229447,262779491~7688249,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,332042044~606164,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,390011506~750434] 2024-09-14T04:32:25.783 INFO:tasks.workunit.client.0.smithi039.stdout:from 12 to 19 diff [15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.783 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.789 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,148411414~27747401,183524730~22818087,207369398~3608367,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.789 INFO:tasks.workunit.client.0.smithi039.stdout:from 13 to 14 diff [37722924~6209571,89021763~2406992,183524730~7278560,237089596~5877557,253979221~3341796,262851191~6527342,284870704~8098864,307097205~6187809] 2024-09-14T04:32:25.789 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [37722924~6209571,89021763~2406992,183524730~7278560,237089596~5877557,253979221~3341796,262851191~6527342,284870704~8098864,307097205~6187809] 2024-09-14T04:32:25.795 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [37722924~6209571,89021763~2406992,183524730~7278560,237089596~5877557,253979221~3341796,262851191~6527342,284870704~8098864,307097205~6187809] 2024-09-14T04:32:25.795 INFO:tasks.workunit.client.0.smithi039.stdout:from 13 to 15 diff [37722924~6209571,55349631~7290768,89021763~2406992,112894852~9167908,139378274~7402656,183524730~7278560,207369398~3608367,222441104~495517,237089596~5877557,253979221~3341796,262851191~6527342,276825837~452599,284870704~8098864,307097205~6187809,347886358~1792403] 2024-09-14T04:32:25.795 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [37722924~6209571,55349631~7290768,89021763~2406992,112894852~9167908,139378274~7402656,183524730~7278560,207369398~3608367,222441104~495517,237089596~5877557,253979221~3341796,262851191~6527342,276825837~452599,284870704~8098864,307097205~6187809,347886358~1792403] 2024-09-14T04:32:25.801 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [37722924~6209571,55349631~7290768,89021763~2406992,112894852~9167908,139378274~7402656,183524730~7278560,207369398~3608367,222441104~495517,237089596~5877557,253979221~3341796,262851191~6527342,276825837~452599,284870704~8098864,307097205~6187809,347886358~1792403] 2024-09-14T04:32:25.802 INFO:tasks.workunit.client.0.smithi039.stdout:from 13 to 16 diff [37722924~6209571,53772129~974098,55349631~7290768,89021763~2406992,110614496~11448264,132996120~2690483,139378274~7402656,155908264~11062369,183524730~7278560,195446529~6684577,207369398~3608367,222441104~495517,237089596~5877557,253979221~3341796,257815293~3393375,262851191~6527342,276825837~452599,284870704~8098864,307097205~6187809,332042044~606164,347886358~1792403,360827666~8007252] 2024-09-14T04:32:25.802 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [37722924~6209571,53772129~974098,55349631~7290768,89021763~2406992,110614496~11448264,132996120~2690483,139378274~7402656,155908264~11062369,183524730~7278560,195446529~6684577,207369398~3608367,222441104~495517,237089596~5877557,253979221~3341796,257815293~3393375,262851191~6527342,276825837~452599,284870704~8098864,307097205~6187809,332042044~606164,347886358~1792403,360827666~8007252] 2024-09-14T04:32:25.808 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [37722924~6209571,53772129~974098,55349631~7290768,89021763~2406992,110614496~11448264,132996120~2690483,139378274~7402656,155908264~11062369,183524730~7278560,192937984~9193122,207369398~3608367,222441104~495517,237089596~5877557,253979221~3341796,257815293~3393375,262851191~6527342,276825837~452599,284870704~8098864,307097205~6187809,332042044~606164,347886358~1792403,360827666~8007252] 2024-09-14T04:32:25.808 INFO:tasks.workunit.client.0.smithi039.stdout:from 13 to 17 diff [37722924~6209571,50026379~3466398,53772129~974098,55349631~9438706,89021763~2406992,110614496~11448264,132996120~2690483,139378274~7402656,150858815~16111818,183524730~7278560,195446529~6684577,207369398~3608367,222441104~495517,237089596~5877557,253979221~3341796,257815293~3393375,262779491~7688249,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885] 2024-09-14T04:32:25.808 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [37722924~6209571,50026379~3466398,53772129~974098,55349631~9438706,89021763~2406992,110614496~11448264,132996120~2690483,139378274~7402656,150858815~16111818,183524730~7278560,195446529~6684577,207369398~3608367,222441104~495517,237089596~5877557,253979221~3341796,257815293~3393375,262779491~7688249,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885] 2024-09-14T04:32:25.814 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [37722924~6209571,50026379~3466398,53772129~974098,55349631~9438706,89021763~2406992,110614496~11448264,132996120~2690483,139378274~7402656,150858815~16111818,183524730~7278560,192937984~9193122,207369398~3608367,222441104~495517,237089596~5877557,253979221~3341796,257815293~3393375,262779491~7688249,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885] 2024-09-14T04:32:25.814 INFO:tasks.workunit.client.0.smithi039.stdout:from 13 to 18 diff [37722924~6209571,50026379~3466398,53772129~974098,55349631~9438706,71362387~7448130,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,150858815~22631382,183524730~7278560,195446529~6684577,202395118~3947699,207369398~3608367,222441104~495517,237089596~5877557,253979221~7229447,262779491~7688249,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,390011506~750434] 2024-09-14T04:32:25.814 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [37722924~6209571,50026379~3466398,53772129~974098,55349631~9438706,71362387~7448130,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,150858815~22631382,183524730~7278560,195446529~6684577,202395118~3947699,207369398~3608367,222441104~495517,237089596~5877557,253979221~7229447,262779491~7688249,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,390011506~750434] 2024-09-14T04:32:25.821 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [37722924~6209571,50026379~3466398,53772129~974098,55349631~9438706,71362387~7448130,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,150858815~22631382,183524730~7278560,192937984~9193122,202395118~3947699,207369398~3608367,222441104~495517,237089596~5877557,253979221~7229447,262779491~7688249,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,390011506~750434] 2024-09-14T04:32:25.821 INFO:tasks.workunit.client.0.smithi039.stdout:from 13 to 19 diff [15956065~5397876,37722924~6209571,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,183524730~8784543,192662147~2659618,195446529~6684577,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,237089596~5877557,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.821 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,37722924~6209571,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,183524730~8784543,192662147~2659618,195446529~6684577,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,237089596~5877557,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.825 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,37722924~6209571,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,183524730~18606376,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,237089596~5877557,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.825 INFO:tasks.workunit.client.0.smithi039.stdout:from 14 to 15 diff [55349631~7290768,112894852~9167908,139378274~7402656,207369398~3608367,222441104~495517,276825837~452599,307864787~4327691,347886358~1792403] 2024-09-14T04:32:25.825 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [55349631~7290768,112894852~9167908,139378274~7402656,207369398~3608367,222441104~495517,276825837~452599,307864787~4327691,347886358~1792403] 2024-09-14T04:32:25.831 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [55349631~7290768,112894852~9167908,139378274~7402656,207369398~3608367,222441104~495517,276825837~452599,307864787~4327691,347886358~1792403] 2024-09-14T04:32:25.831 INFO:tasks.workunit.client.0.smithi039.stdout:from 14 to 16 diff [53772129~974098,55349631~7290768,110614496~11448264,132996120~2690483,139378274~7402656,155908264~11062369,195446529~6684577,207369398~3608367,222441104~495517,257815293~3393375,276825837~452599,286210107~4006077,307864787~4327691,332042044~606164,347886358~1792403,360827666~8007252] 2024-09-14T04:32:25.831 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [53772129~974098,55349631~7290768,110614496~11448264,132996120~2690483,139378274~7402656,155908264~11062369,195446529~6684577,207369398~3608367,222441104~495517,257815293~3393375,276825837~452599,286210107~4006077,307864787~4327691,332042044~606164,347886358~1792403,360827666~8007252] 2024-09-14T04:32:25.836 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [53772129~974098,55349631~7290768,110614496~11448264,132996120~2690483,139378274~7402656,155908264~11062369,192937984~9193122,207369398~3608367,222441104~495517,257815293~3393375,276825837~452599,286210107~4006077,307864787~4327691,332042044~606164,347886358~1792403,360827666~8007252] 2024-09-14T04:32:25.836 INFO:tasks.workunit.client.0.smithi039.stdout:from 14 to 17 diff [50026379~3466398,53772129~974098,55349631~9438706,110614496~11448264,132996120~2690483,139378274~7402656,150858815~16111818,195446529~6684577,207369398~3608367,222441104~495517,257815293~3393375,262779491~7688249,276825837~452599,286210107~4006077,307864787~4327691,317400597~2551594,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885] 2024-09-14T04:32:25.836 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [50026379~3466398,53772129~974098,55349631~9438706,110614496~11448264,132996120~2690483,139378274~7402656,150858815~16111818,195446529~6684577,207369398~3608367,222441104~495517,257815293~3393375,262779491~7688249,276825837~452599,286210107~4006077,307864787~4327691,317400597~2551594,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885] 2024-09-14T04:32:25.842 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [50026379~3466398,53772129~974098,55349631~9438706,110614496~11448264,132996120~2690483,139378274~7402656,150858815~16111818,192937984~9193122,207369398~3608367,222441104~495517,257815293~3393375,262779491~7688249,276825837~452599,286210107~4006077,307864787~4327691,317400597~2551594,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885] 2024-09-14T04:32:25.842 INFO:tasks.workunit.client.0.smithi039.stdout:from 14 to 18 diff [41504883~1986423,50026379~3466398,53772129~974098,55349631~9438706,71362387~7448130,109021991~13040769,132996120~2690483,139378274~7402656,150858815~22631382,195446529~6684577,202395118~3947699,207369398~3608367,222441104~495517,255161782~6046886,262779491~7688249,271309101~3402633,276825837~452599,286210107~4006077,307864787~4327691,317400597~2551594,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,390011506~750434] 2024-09-14T04:32:25.842 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [41504883~1986423,50026379~3466398,53772129~974098,55349631~9438706,71362387~7448130,109021991~13040769,132996120~2690483,139378274~7402656,150858815~22631382,195446529~6684577,202395118~3947699,207369398~3608367,222441104~495517,255161782~6046886,262779491~7688249,271309101~3402633,276825837~452599,286210107~4006077,307864787~4327691,317400597~2551594,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,390011506~750434] 2024-09-14T04:32:25.847 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [41504883~1986423,50026379~3466398,53772129~974098,55349631~9438706,71362387~7448130,109021991~13040769,132996120~2690483,139378274~7402656,150858815~22631382,192937984~9193122,202395118~3947699,207369398~3608367,222441104~495517,255161782~6046886,262779491~7688249,271309101~3402633,276825837~452599,286210107~4006077,307864787~4327691,317400597~2551594,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,390011506~750434] 2024-09-14T04:32:25.848 INFO:tasks.workunit.client.0.smithi039.stdout:from 14 to 19 diff [15956065~5397876,41504883~1986423,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,189388097~2921176,192662147~2659618,195446529~6684577,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,255161782~15305958,271309101~3402633,276825837~452599,286210107~4006077,307864787~4327691,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.848 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,41504883~1986423,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,189388097~2921176,192662147~2659618,195446529~6684577,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,255161782~15305958,271309101~3402633,276825837~452599,286210107~4006077,307864787~4327691,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.853 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,41504883~1986423,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,189388097~12743009,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,255161782~15305958,271309101~3402633,276825837~452599,286210107~4006077,307864787~4327691,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.853 INFO:tasks.workunit.client.0.smithi039.stdout:from 15 to 16 diff [53772129~974098,110614496~2472756,132996120~2690483,155908264~11062369,195446529~6684577,257815293~3393375,286210107~4006077,332042044~606164,360827666~8007252] 2024-09-14T04:32:25.853 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [53772129~974098,110614496~2472756,132996120~2690483,155908264~11062369,195446529~6684577,257815293~3393375,286210107~4006077,332042044~606164,360827666~8007252] 2024-09-14T04:32:25.859 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [53772129~974098,110614496~2472756,132996120~2690483,155908264~11062369,192937984~9193122,257815293~3393375,286210107~4006077,332042044~606164,360827666~8007252] 2024-09-14T04:32:25.859 INFO:tasks.workunit.client.0.smithi039.stdout:from 15 to 17 diff [50026379~3466398,53772129~974098,55475851~9312486,110614496~4681429,132996120~2690483,150858815~16111818,195446529~6684577,257815293~3393375,262779491~7688249,286210107~4006077,317400597~2551594,332042044~606164,360827666~8007252,370420162~6009885] 2024-09-14T04:32:25.859 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [50026379~3466398,53772129~974098,55475851~9312486,110614496~4681429,132996120~2690483,150858815~16111818,195446529~6684577,257815293~3393375,262779491~7688249,286210107~4006077,317400597~2551594,332042044~606164,360827666~8007252,370420162~6009885] 2024-09-14T04:32:25.864 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [50026379~3466398,53772129~974098,55475851~9312486,110614496~4681429,132996120~2690483,150858815~16111818,192937984~9193122,257815293~3393375,262779491~7688249,286210107~4006077,317400597~2551594,332042044~606164,360827666~8007252,370420162~6009885] 2024-09-14T04:32:25.864 INFO:tasks.workunit.client.0.smithi039.stdout:from 15 to 18 diff [41504883~1986423,50026379~3466398,53772129~974098,55475851~9312486,71362387~7448130,109021991~6273934,132996120~2690483,150858815~22631382,195446529~6684577,202395118~3947699,255161782~6046886,262779491~7688249,271309101~3402633,286210107~4006077,317400597~2551594,332042044~606164,360827666~8007252,370420162~6009885,390011506~750434] 2024-09-14T04:32:25.864 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [41504883~1986423,50026379~3466398,53772129~974098,55475851~9312486,71362387~7448130,109021991~6273934,132996120~2690483,150858815~22631382,195446529~6684577,202395118~3947699,255161782~6046886,262779491~7688249,271309101~3402633,286210107~4006077,317400597~2551594,332042044~606164,360827666~8007252,370420162~6009885,390011506~750434] 2024-09-14T04:32:25.870 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [41504883~1986423,50026379~3466398,53772129~974098,55475851~9312486,71362387~7448130,109021991~6273934,132996120~2690483,150858815~22631382,192937984~9193122,202395118~3947699,255161782~6046886,262779491~7688249,271309101~3402633,286210107~4006077,317400597~2551594,332042044~606164,360827666~8007252,370420162~6009885,390011506~750434] 2024-09-14T04:32:25.870 INFO:tasks.workunit.client.0.smithi039.stdout:from 15 to 19 diff [15956065~5397876,41504883~1986423,50026379~3466398,53772129~974098,55475851~9312486,71362387~9427811,109021991~6273934,132996120~2690483,150858815~25300000,189388097~2921176,192662147~2659618,195446529~6684577,202395118~3947699,231412373~2251966,255161782~15305958,271309101~3402633,286210107~4006077,317400597~2551594,324417134~321003,332042044~606164,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.870 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,41504883~1986423,50026379~3466398,53772129~974098,55475851~9312486,71362387~9427811,109021991~6273934,132996120~2690483,150858815~25300000,189388097~2921176,192662147~2659618,195446529~6684577,202395118~3947699,231412373~2251966,255161782~15305958,271309101~3402633,286210107~4006077,317400597~2551594,324417134~321003,332042044~606164,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.876 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,41504883~1986423,50026379~3466398,53772129~974098,55475851~9312486,71362387~9427811,109021991~6273934,132996120~2690483,150858815~25300000,189388097~12743009,202395118~3947699,231412373~2251966,255161782~15305958,271309101~3402633,286210107~4006077,317400597~2551594,324417134~321003,332042044~606164,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.877 INFO:tasks.workunit.client.0.smithi039.stdout:from 16 to 17 diff [50026379~3466398,55475851~9312486,111778834~3517091,150858815~5451956,156634296~3077940,262779491~7688249,317400597~2551594,370420162~6009885] 2024-09-14T04:32:25.877 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [50026379~3466398,55475851~9312486,111778834~3517091,150858815~5451956,156634296~3077940,262779491~7688249,317400597~2551594,370420162~6009885] 2024-09-14T04:32:25.881 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [50026379~3466398,55475851~9312486,111778834~3517091,150858815~5451956,156634296~3077940,262779491~7688249,317400597~2551594,370420162~6009885] 2024-09-14T04:32:25.881 INFO:tasks.workunit.client.0.smithi039.stdout:from 16 to 18 diff [41504883~1986423,50026379~3466398,55475851~9312486,71362387~7448130,109021991~6273934,150858815~8853421,164724774~1239558,166279969~7210228,202395118~3947699,255161782~6019745,262779491~7688249,271309101~3402633,317400597~2551594,370420162~6009885,390011506~750434] 2024-09-14T04:32:25.881 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [41504883~1986423,50026379~3466398,55475851~9312486,71362387~7448130,109021991~6273934,150858815~8853421,164724774~1239558,166279969~7210228,202395118~3947699,255161782~6019745,262779491~7688249,271309101~3402633,317400597~2551594,370420162~6009885,390011506~750434] 2024-09-14T04:32:25.887 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [41504883~1986423,50026379~3466398,55475851~9312486,71362387~7448130,109021991~6273934,150858815~8853421,164724774~1239558,166279969~7210228,202395118~3947699,255161782~6019745,262779491~7688249,271309101~3402633,317400597~2551594,370420162~6009885,390011506~750434] 2024-09-14T04:32:25.887 INFO:tasks.workunit.client.0.smithi039.stdout:from 16 to 19 diff [15956065~5397876,41504883~1986423,50026379~3466398,55475851~9312486,71362387~9427811,109021991~6273934,150858815~8853421,164724774~1239558,166279969~9878846,189388097~2921176,192662147~2659618,202395118~3947699,231412373~2251966,255161782~15305958,271309101~3402633,317400597~2551594,324417134~321003,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.887 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,41504883~1986423,50026379~3466398,55475851~9312486,71362387~9427811,109021991~6273934,150858815~8853421,164724774~1239558,166279969~9878846,189388097~2921176,192662147~2659618,202395118~3947699,231412373~2251966,255161782~15305958,271309101~3402633,317400597~2551594,324417134~321003,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.893 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,41504883~1986423,50026379~3466398,55475851~9312486,71362387~9427811,109021991~6273934,150858815~8853421,164724774~1239558,166279969~9878846,189388097~5933668,202395118~3947699,231412373~2251966,255161782~15305958,271309101~3402633,317400597~2551594,324417134~321003,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.893 INFO:tasks.workunit.client.0.smithi039.stdout:from 17 to 18 diff [41504883~1986423,71362387~7448130,109021991~3523985,152118700~6330839,164724774~1239558,166279969~7210228,202395118~3947699,255161782~6019745,271309101~3402633,390011506~750434] 2024-09-14T04:32:25.893 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [41504883~1986423,71362387~7448130,109021991~3523985,152118700~6330839,164724774~1239558,166279969~7210228,202395118~3947699,255161782~6019745,271309101~3402633,390011506~750434] 2024-09-14T04:32:25.898 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [41504883~1986423,71362387~7448130,109021991~3523985,152118700~6330839,164724774~1239558,166279969~7210228,202395118~3947699,255161782~6019745,271309101~3402633,390011506~750434] 2024-09-14T04:32:25.898 INFO:tasks.workunit.client.0.smithi039.stdout:from 17 to 19 diff [15956065~5397876,41504883~1986423,71362387~9427811,109021991~3523985,152118700~6330839,164724774~1239558,166279969~9878846,189388097~2921176,192662147~2659618,202395118~3947699,231412373~2251966,255161782~9435902,271309101~3402633,324417134~321003,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.898 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,41504883~1986423,71362387~9427811,109021991~3523985,152118700~6330839,164724774~1239558,166279969~9878846,189388097~2921176,192662147~2659618,202395118~3947699,231412373~2251966,255161782~9435902,271309101~3402633,324417134~321003,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.903 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,41504883~1986423,71362387~9427811,109021991~3523985,152118700~6330839,164724774~1239558,166279969~9878846,189388097~5933668,202395118~3947699,231412373~2251966,255161782~9435902,271309101~3402633,324417134~321003,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.903 INFO:tasks.workunit.client.0.smithi039.stdout:from 18 to 19 diff [15956065~5397876,77848113~2942085,169499290~6659525,189388097~2921176,192662147~2659618,231412373~2251966,258794170~5803514,324417134~321003,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.904 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,77848113~2942085,169499290~6659525,189388097~2921176,192662147~2659618,231412373~2251966,258794170~5803514,324417134~321003,384246345~6869073,409194704~7731957] 2024-09-14T04:32:25.909 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,77848113~2942085,169499290~6659525,189388097~5933668,231412373~2251966,258794170~5803514,324417134~321003,384246345~6869073,409194704~7731957] 2024-09-14T04:32:26.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:25 smithi129 ceph-mon[101316]: osdmap e1139: 8 total, 8 up, 8 in 2024-09-14T04:32:26.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:25 smithi039 ceph-mon[106470]: osdmap e1139: 8 total, 8 up, 8 in 2024-09-14T04:32:26.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:25 smithi039 ceph-mon[111235]: osdmap e1139: 8 total, 8 up, 8 in 2024-09-14T04:32:26.824 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:26.824 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:26.845 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~82237545,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:26.846 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:26.846 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:26.852 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:26.852 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:26.852 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:26.858 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,327155712~6219679,335544320~57958504,396566061~6312282,406171321~10755340] 2024-09-14T04:32:26.858 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:26.858 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:26.864 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~30182049,206380204~11228650,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4760259,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396566061~6312282,409194704~7731957] 2024-09-14T04:32:26.864 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:26.864 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:26.871 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~30182049,206380204~4896801,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:26.871 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to HEAD diff [1532502~4624507,15956065~6778852,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183524730~8784543,192662147~2659618,195446529~10896288,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:26.871 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183524730~8784543,192662147~2659618,195446529~10896288,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:26.877 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~10152005,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,177156787~5609834,183524730~22818087,207366448~3910557,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:26.877 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to HEAD diff [1532502~4624507,15956065~5397876,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:26.877 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:26.883 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~28200107,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:26.883 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to HEAD diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:26.883 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:26.889 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~28200107,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:26.889 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to HEAD diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:26.889 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:26.895 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~22235841,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276824064~7288172,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:26.895 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to HEAD diff [1532502~4624507,15956065~5397876,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,88623968~370312,89021763~2406992,91960020~945991,99264373~856643,109021991~13877146,132996120~5475003,139378274~36780541,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~6956556,307097205~12854986,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:26.895 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,88623968~370312,89021763~2406992,91960020~945991,99264373~856643,109021991~13877146,132996120~5475003,139378274~36780541,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~6956556,307097205~12854986,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:26.900 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,35645530~13536984,50026379~3466398,53772129~974098,55349631~30939764,88623968~370312,89021763~2406992,91960020~945991,96468992~3652024,109021991~13877146,132996120~5475003,139378274~36780541,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276824064~7288172,284870704~8098864,297545811~6956556,307097205~12854986,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:26.901 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to HEAD diff [1532502~4624507,15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,99264373~856643,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:26.901 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,99264373~856643,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:26.906 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,37722924~11459590,50026379~3466398,53772129~974098,55349631~25440567,89021763~2406992,91960020~945991,96468992~3652024,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:26.907 INFO:tasks.workunit.client.0.smithi039.stdout:from 11 to HEAD diff [15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:26.907 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:26.912 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,37722924~11459590,50026379~3466398,53772129~974098,55349631~25440567,89021763~2406992,91960020~945991,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~22818087,207369398~3608367,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:26.912 INFO:tasks.workunit.client.0.smithi039.stdout:from 12 to HEAD diff [15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:26.912 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:26.918 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,148411414~27747401,183524730~22818087,207369398~3608367,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:26.918 INFO:tasks.workunit.client.0.smithi039.stdout:from 13 to HEAD diff [15956065~5397876,37722924~6209571,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,183524730~8784543,192662147~2659618,195446529~6684577,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,237089596~5877557,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:26.918 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,37722924~6209571,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,183524730~8784543,192662147~2659618,195446529~6684577,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,237089596~5877557,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:26.923 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,37722924~6209571,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,183524730~18606376,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,237089596~5877557,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:26.924 INFO:tasks.workunit.client.0.smithi039.stdout:from 14 to HEAD diff [15956065~5397876,41504883~1986423,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,189388097~2921176,192662147~2659618,195446529~6684577,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,255161782~15305958,271309101~3402633,276825837~452599,286210107~4006077,307864787~4327691,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:26.924 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,41504883~1986423,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,189388097~2921176,192662147~2659618,195446529~6684577,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,255161782~15305958,271309101~3402633,276825837~452599,286210107~4006077,307864787~4327691,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:26.929 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,41504883~1986423,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,189388097~12743009,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,255161782~15305958,271309101~3402633,276825837~452599,286210107~4006077,307864787~4327691,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:26.929 INFO:tasks.workunit.client.0.smithi039.stdout:from 15 to HEAD diff [15956065~5397876,41504883~1986423,50026379~3466398,53772129~974098,55475851~9312486,71362387~9427811,109021991~6273934,132996120~2690483,150858815~25300000,189388097~2921176,192662147~2659618,195446529~6684577,202395118~3947699,231412373~2251966,255161782~15305958,271309101~3402633,286210107~4006077,317400597~2551594,324417134~321003,332042044~606164,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:26.929 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,41504883~1986423,50026379~3466398,53772129~974098,55475851~9312486,71362387~9427811,109021991~6273934,132996120~2690483,150858815~25300000,189388097~2921176,192662147~2659618,195446529~6684577,202395118~3947699,231412373~2251966,255161782~15305958,271309101~3402633,286210107~4006077,317400597~2551594,324417134~321003,332042044~606164,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:26.935 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,41504883~1986423,50026379~3466398,53772129~974098,55475851~9312486,71362387~9427811,109021991~6273934,132996120~2690483,150858815~25300000,189388097~12743009,202395118~3947699,231412373~2251966,255161782~15305958,271309101~3402633,286210107~4006077,317400597~2551594,324417134~321003,332042044~606164,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:26.935 INFO:tasks.workunit.client.0.smithi039.stdout:from 16 to HEAD diff [15956065~5397876,41504883~1986423,50026379~3466398,55475851~9312486,71362387~9427811,109021991~6273934,150858815~8853421,164724774~1239558,166279969~9878846,189388097~2921176,192662147~2659618,202395118~3947699,231412373~2251966,255161782~15305958,271309101~3402633,317400597~2551594,324417134~321003,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:26.935 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,41504883~1986423,50026379~3466398,55475851~9312486,71362387~9427811,109021991~6273934,150858815~8853421,164724774~1239558,166279969~9878846,189388097~2921176,192662147~2659618,202395118~3947699,231412373~2251966,255161782~15305958,271309101~3402633,317400597~2551594,324417134~321003,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:26.941 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,41504883~1986423,50026379~3466398,55475851~9312486,71362387~9427811,109021991~6273934,150858815~8853421,164724774~1239558,166279969~9878846,189388097~5933668,202395118~3947699,231412373~2251966,255161782~15305958,271309101~3402633,317400597~2551594,324417134~321003,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:26.941 INFO:tasks.workunit.client.0.smithi039.stdout:from 17 to HEAD diff [15956065~5397876,41504883~1986423,71362387~9427811,109021991~3523985,152118700~6330839,164724774~1239558,166279969~9878846,189388097~2921176,192662147~2659618,202395118~3947699,231412373~2251966,255161782~9435902,271309101~3402633,324417134~321003,384246345~6869073,409194704~7731957] 2024-09-14T04:32:26.941 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,41504883~1986423,71362387~9427811,109021991~3523985,152118700~6330839,164724774~1239558,166279969~9878846,189388097~2921176,192662147~2659618,202395118~3947699,231412373~2251966,255161782~9435902,271309101~3402633,324417134~321003,384246345~6869073,409194704~7731957] 2024-09-14T04:32:26.946 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,41504883~1986423,71362387~9427811,109021991~3523985,152118700~6330839,164724774~1239558,166279969~9878846,189388097~5933668,202395118~3947699,231412373~2251966,255161782~9435902,271309101~3402633,324417134~321003,384246345~6869073,409194704~7731957] 2024-09-14T04:32:27.172 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:26 smithi129 ceph-mon[101316]: pgmap v1793: 105 pgs: 105 active+clean; 1.1 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 224 KiB/s rd, 51 MiB/s wr, 352 op/s 2024-09-14T04:32:27.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:26 smithi039 ceph-mon[106470]: pgmap v1793: 105 pgs: 105 active+clean; 1.1 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 224 KiB/s rd, 51 MiB/s wr, 352 op/s 2024-09-14T04:32:27.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:26 smithi039 ceph-mon[111235]: pgmap v1793: 105 pgs: 105 active+clean; 1.1 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 224 KiB/s rd, 51 MiB/s wr, 352 op/s 2024-09-14T04:32:27.840 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:27.840 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:27.861 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~82237545,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:27.861 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:27.861 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:27.867 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:27.868 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:27.868 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:27.874 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,327155712~6219679,335544320~57958504,396566061~6312282,406171321~10755340] 2024-09-14T04:32:27.874 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:27.874 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:27.880 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~30182049,206380204~11228650,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4760259,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396566061~6312282,409194704~7731957] 2024-09-14T04:32:27.881 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:27.881 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:27.887 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~30182049,206380204~4896801,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:27.887 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to HEAD diff [1532502~4624507,15956065~6778852,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183524730~8784543,192662147~2659618,195446529~10896288,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:27.887 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183524730~8784543,192662147~2659618,195446529~10896288,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:27.893 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~10152005,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,177156787~5609834,183524730~22818087,207366448~3910557,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:27.893 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to HEAD diff [1532502~4624507,15956065~5397876,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:27.893 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:27.899 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~28200107,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:27.899 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to HEAD diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:27.899 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:27.905 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~28200107,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:27.905 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to HEAD diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:27.906 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:27.911 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~22235841,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276824064~7288172,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:27.911 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to HEAD diff [1532502~4624507,15956065~5397876,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,88623968~370312,89021763~2406992,91960020~945991,99264373~856643,109021991~13877146,132996120~5475003,139378274~36780541,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~6956556,307097205~12854986,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:27.911 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,88623968~370312,89021763~2406992,91960020~945991,99264373~856643,109021991~13877146,132996120~5475003,139378274~36780541,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~6956556,307097205~12854986,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:27.917 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,35645530~13536984,50026379~3466398,53772129~974098,55349631~30939764,88623968~370312,89021763~2406992,91960020~945991,96468992~3652024,109021991~13877146,132996120~5475003,139378274~36780541,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276824064~7288172,284870704~8098864,297545811~6956556,307097205~12854986,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:27.917 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to HEAD diff [1532502~4624507,15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,99264373~856643,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:27.917 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,99264373~856643,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:27.923 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,37722924~11459590,50026379~3466398,53772129~974098,55349631~25440567,89021763~2406992,91960020~945991,96468992~3652024,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:27.923 INFO:tasks.workunit.client.0.smithi039.stdout:from 11 to HEAD diff [15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:27.923 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:27.928 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,37722924~11459590,50026379~3466398,53772129~974098,55349631~25440567,89021763~2406992,91960020~945991,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~22818087,207369398~3608367,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:27.929 INFO:tasks.workunit.client.0.smithi039.stdout:from 12 to HEAD diff [15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:27.929 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:27.934 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,148411414~27747401,183524730~22818087,207369398~3608367,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:27.934 INFO:tasks.workunit.client.0.smithi039.stdout:from 13 to HEAD diff [15956065~5397876,37722924~6209571,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,183524730~8784543,192662147~2659618,195446529~6684577,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,237089596~5877557,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:27.934 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,37722924~6209571,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,183524730~8784543,192662147~2659618,195446529~6684577,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,237089596~5877557,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:27.940 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,37722924~6209571,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,183524730~18606376,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,237089596~5877557,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:27.940 INFO:tasks.workunit.client.0.smithi039.stdout:from 14 to HEAD diff [15956065~5397876,41504883~1986423,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,189388097~2921176,192662147~2659618,195446529~6684577,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,255161782~15305958,271309101~3402633,276825837~452599,286210107~4006077,307864787~4327691,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:27.940 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,41504883~1986423,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,189388097~2921176,192662147~2659618,195446529~6684577,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,255161782~15305958,271309101~3402633,276825837~452599,286210107~4006077,307864787~4327691,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:27.946 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,41504883~1986423,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,189388097~12743009,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,255161782~15305958,271309101~3402633,276825837~452599,286210107~4006077,307864787~4327691,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:27.946 INFO:tasks.workunit.client.0.smithi039.stdout:from 15 to HEAD diff [15956065~5397876,41504883~1986423,50026379~3466398,53772129~974098,55475851~9312486,71362387~9427811,109021991~6273934,132996120~2690483,150858815~25300000,189388097~2921176,192662147~2659618,195446529~6684577,202395118~3947699,231412373~2251966,255161782~15305958,271309101~3402633,286210107~4006077,317400597~2551594,324417134~321003,332042044~606164,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:27.946 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,41504883~1986423,50026379~3466398,53772129~974098,55475851~9312486,71362387~9427811,109021991~6273934,132996120~2690483,150858815~25300000,189388097~2921176,192662147~2659618,195446529~6684577,202395118~3947699,231412373~2251966,255161782~15305958,271309101~3402633,286210107~4006077,317400597~2551594,324417134~321003,332042044~606164,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:27.952 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,41504883~1986423,50026379~3466398,53772129~974098,55475851~9312486,71362387~9427811,109021991~6273934,132996120~2690483,150858815~25300000,189388097~12743009,202395118~3947699,231412373~2251966,255161782~15305958,271309101~3402633,286210107~4006077,317400597~2551594,324417134~321003,332042044~606164,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:27.952 INFO:tasks.workunit.client.0.smithi039.stdout:from 16 to HEAD diff [15956065~5397876,41504883~1986423,50026379~3466398,55475851~9312486,71362387~9427811,109021991~6273934,150858815~8853421,164724774~1239558,166279969~9878846,189388097~2921176,192662147~2659618,202395118~3947699,231412373~2251966,255161782~15305958,271309101~3402633,317400597~2551594,324417134~321003,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:27.952 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,41504883~1986423,50026379~3466398,55475851~9312486,71362387~9427811,109021991~6273934,150858815~8853421,164724774~1239558,166279969~9878846,189388097~2921176,192662147~2659618,202395118~3947699,231412373~2251966,255161782~15305958,271309101~3402633,317400597~2551594,324417134~321003,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:27.957 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,41504883~1986423,50026379~3466398,55475851~9312486,71362387~9427811,109021991~6273934,150858815~8853421,164724774~1239558,166279969~9878846,189388097~5933668,202395118~3947699,231412373~2251966,255161782~15305958,271309101~3402633,317400597~2551594,324417134~321003,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:28.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:27 smithi129 ceph-mon[101316]: osdmap e1140: 8 total, 8 up, 8 in 2024-09-14T04:32:28.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:27 smithi039 ceph-mon[106470]: osdmap e1140: 8 total, 8 up, 8 in 2024-09-14T04:32:28.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:27 smithi039 ceph-mon[111235]: osdmap e1140: 8 total, 8 up, 8 in 2024-09-14T04:32:28.863 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:28.863 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:28.880 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~82237545,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:28.880 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:28.880 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:28.885 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:28.885 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:28.885 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:28.891 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,327155712~6219679,335544320~57958504,396566061~6312282,406171321~10755340] 2024-09-14T04:32:28.891 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:28.892 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:28.897 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~30182049,206380204~11228650,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4760259,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396566061~6312282,409194704~7731957] 2024-09-14T04:32:28.898 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:28.898 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:28.903 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~30182049,206380204~4896801,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:28.903 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to HEAD diff [1532502~4624507,15956065~6778852,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183524730~8784543,192662147~2659618,195446529~10896288,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:28.903 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183524730~8784543,192662147~2659618,195446529~10896288,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:28.909 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~10152005,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,177156787~5609834,183524730~22818087,207366448~3910557,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:28.909 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to HEAD diff [1532502~4624507,15956065~5397876,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:28.909 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:28.915 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~28200107,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:28.915 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to HEAD diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:28.915 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:28.922 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~28200107,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:28.922 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to HEAD diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:28.922 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:28.927 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~22235841,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276824064~7288172,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:28.927 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to HEAD diff [1532502~4624507,15956065~5397876,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,88623968~370312,89021763~2406992,91960020~945991,99264373~856643,109021991~13877146,132996120~5475003,139378274~36780541,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~6956556,307097205~12854986,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:28.927 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,88623968~370312,89021763~2406992,91960020~945991,99264373~856643,109021991~13877146,132996120~5475003,139378274~36780541,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~6956556,307097205~12854986,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:28.933 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,35645530~13536984,50026379~3466398,53772129~974098,55349631~30939764,88623968~370312,89021763~2406992,91960020~945991,96468992~3652024,109021991~13877146,132996120~5475003,139378274~36780541,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276824064~7288172,284870704~8098864,297545811~6956556,307097205~12854986,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:28.933 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to HEAD diff [1532502~4624507,15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,99264373~856643,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:28.933 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,99264373~856643,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:28.939 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,37722924~11459590,50026379~3466398,53772129~974098,55349631~25440567,89021763~2406992,91960020~945991,96468992~3652024,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:28.939 INFO:tasks.workunit.client.0.smithi039.stdout:from 11 to HEAD diff [15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:28.939 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:28.944 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,37722924~11459590,50026379~3466398,53772129~974098,55349631~25440567,89021763~2406992,91960020~945991,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~22818087,207369398~3608367,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:28.944 INFO:tasks.workunit.client.0.smithi039.stdout:from 12 to HEAD diff [15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:28.945 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:28.950 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,148411414~27747401,183524730~22818087,207369398~3608367,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:28.950 INFO:tasks.workunit.client.0.smithi039.stdout:from 13 to HEAD diff [15956065~5397876,37722924~6209571,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,183524730~8784543,192662147~2659618,195446529~6684577,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,237089596~5877557,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:28.950 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,37722924~6209571,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,183524730~8784543,192662147~2659618,195446529~6684577,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,237089596~5877557,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:28.956 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,37722924~6209571,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,183524730~18606376,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,237089596~5877557,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:28.956 INFO:tasks.workunit.client.0.smithi039.stdout:from 14 to HEAD diff [15956065~5397876,41504883~1986423,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,189388097~2921176,192662147~2659618,195446529~6684577,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,255161782~15305958,271309101~3402633,276825837~452599,286210107~4006077,307864787~4327691,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:28.956 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,41504883~1986423,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,189388097~2921176,192662147~2659618,195446529~6684577,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,255161782~15305958,271309101~3402633,276825837~452599,286210107~4006077,307864787~4327691,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:28.961 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,41504883~1986423,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,189388097~12743009,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,255161782~15305958,271309101~3402633,276825837~452599,286210107~4006077,307864787~4327691,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:28.961 INFO:tasks.workunit.client.0.smithi039.stdout:from 15 to HEAD diff [15956065~5397876,41504883~1986423,50026379~3466398,53772129~974098,55475851~9312486,71362387~9427811,109021991~6273934,132996120~2690483,150858815~25300000,189388097~2921176,192662147~2659618,195446529~6684577,202395118~3947699,231412373~2251966,255161782~15305958,271309101~3402633,286210107~4006077,317400597~2551594,324417134~321003,332042044~606164,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:28.961 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,41504883~1986423,50026379~3466398,53772129~974098,55475851~9312486,71362387~9427811,109021991~6273934,132996120~2690483,150858815~25300000,189388097~2921176,192662147~2659618,195446529~6684577,202395118~3947699,231412373~2251966,255161782~15305958,271309101~3402633,286210107~4006077,317400597~2551594,324417134~321003,332042044~606164,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:28.966 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,41504883~1986423,50026379~3466398,53772129~974098,55475851~9312486,71362387~9427811,109021991~6273934,132996120~2690483,150858815~25300000,189388097~12743009,202395118~3947699,231412373~2251966,255161782~15305958,271309101~3402633,286210107~4006077,317400597~2551594,324417134~321003,332042044~606164,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:29.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:28 smithi039 ceph-mon[106470]: pgmap v1795: 105 pgs: 105 active+clean; 1.1 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 8.7 MiB/s wr, 716 op/s 2024-09-14T04:32:29.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:28 smithi039 ceph-mon[106470]: osdmap e1141: 8 total, 8 up, 8 in 2024-09-14T04:32:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:32:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:32:28] "GET /metrics HTTP/1.1" 200 36690 "" "Prometheus/2.51.0" 2024-09-14T04:32:29.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:28 smithi039 ceph-mon[111235]: pgmap v1795: 105 pgs: 105 active+clean; 1.1 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 8.7 MiB/s wr, 716 op/s 2024-09-14T04:32:29.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:28 smithi039 ceph-mon[111235]: osdmap e1141: 8 total, 8 up, 8 in 2024-09-14T04:32:29.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:28 smithi129 ceph-mon[101316]: pgmap v1795: 105 pgs: 105 active+clean; 1.1 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 7.1 KiB/s rd, 8.7 MiB/s wr, 716 op/s 2024-09-14T04:32:29.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:28 smithi129 ceph-mon[101316]: osdmap e1141: 8 total, 8 up, 8 in 2024-09-14T04:32:29.875 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:29.875 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:29.890 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~82237545,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:29.890 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:29.890 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:29.895 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:29.895 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:29.895 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:29.901 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,327155712~6219679,335544320~57958504,396566061~6312282,406171321~10755340] 2024-09-14T04:32:29.902 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:29.902 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:29.908 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~30182049,206380204~11228650,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4760259,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396566061~6312282,409194704~7731957] 2024-09-14T04:32:29.908 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:29.908 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:29.914 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~30182049,206380204~4896801,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:29.914 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to HEAD diff [1532502~4624507,15956065~6778852,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183524730~8784543,192662147~2659618,195446529~10896288,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:29.914 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183524730~8784543,192662147~2659618,195446529~10896288,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:29.920 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~10152005,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,177156787~5609834,183524730~22818087,207366448~3910557,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:29.920 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to HEAD diff [1532502~4624507,15956065~5397876,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:29.920 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:29.925 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~28200107,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:29.926 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to HEAD diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:29.926 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:29.931 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~28200107,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:29.932 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to HEAD diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:29.932 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:29.937 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~22235841,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276824064~7288172,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:29.937 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to HEAD diff [1532502~4624507,15956065~5397876,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,88623968~370312,89021763~2406992,91960020~945991,99264373~856643,109021991~13877146,132996120~5475003,139378274~36780541,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~6956556,307097205~12854986,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:29.938 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,88623968~370312,89021763~2406992,91960020~945991,99264373~856643,109021991~13877146,132996120~5475003,139378274~36780541,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~6956556,307097205~12854986,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:29.944 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,35645530~13536984,50026379~3466398,53772129~974098,55349631~30939764,88623968~370312,89021763~2406992,91960020~945991,96468992~3652024,109021991~13877146,132996120~5475003,139378274~36780541,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276824064~7288172,284870704~8098864,297545811~6956556,307097205~12854986,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:29.944 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to HEAD diff [1532502~4624507,15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,99264373~856643,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:29.944 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,99264373~856643,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:29.949 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,37722924~11459590,50026379~3466398,53772129~974098,55349631~25440567,89021763~2406992,91960020~945991,96468992~3652024,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:29.949 INFO:tasks.workunit.client.0.smithi039.stdout:from 11 to HEAD diff [15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:29.949 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:29.955 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,37722924~11459590,50026379~3466398,53772129~974098,55349631~25440567,89021763~2406992,91960020~945991,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~22818087,207369398~3608367,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:29.955 INFO:tasks.workunit.client.0.smithi039.stdout:from 12 to HEAD diff [15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:29.955 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:29.961 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,148411414~27747401,183524730~22818087,207369398~3608367,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:29.961 INFO:tasks.workunit.client.0.smithi039.stdout:from 13 to HEAD diff [15956065~5397876,37722924~6209571,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,183524730~8784543,192662147~2659618,195446529~6684577,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,237089596~5877557,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:29.961 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,37722924~6209571,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,183524730~8784543,192662147~2659618,195446529~6684577,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,237089596~5877557,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:29.966 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,37722924~6209571,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,183524730~18606376,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,237089596~5877557,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:29.966 INFO:tasks.workunit.client.0.smithi039.stdout:from 14 to HEAD diff [15956065~5397876,41504883~1986423,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,189388097~2921176,192662147~2659618,195446529~6684577,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,255161782~15305958,271309101~3402633,276825837~452599,286210107~4006077,307864787~4327691,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:29.966 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,41504883~1986423,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,189388097~2921176,192662147~2659618,195446529~6684577,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,255161782~15305958,271309101~3402633,276825837~452599,286210107~4006077,307864787~4327691,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:29.971 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,41504883~1986423,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,189388097~12743009,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,255161782~15305958,271309101~3402633,276825837~452599,286210107~4006077,307864787~4327691,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:30.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:29 smithi129 ceph-mon[101316]: osdmap e1142: 8 total, 8 up, 8 in 2024-09-14T04:32:30.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:29 smithi039 ceph-mon[106470]: osdmap e1142: 8 total, 8 up, 8 in 2024-09-14T04:32:30.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:29 smithi039 ceph-mon[111235]: osdmap e1142: 8 total, 8 up, 8 in 2024-09-14T04:32:30.871 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:30.871 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:30.891 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~82237545,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:30.891 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:30.891 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:30.896 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:30.896 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:30.896 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:30.902 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,327155712~6219679,335544320~57958504,396566061~6312282,406171321~10755340] 2024-09-14T04:32:30.902 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:30.902 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:30.908 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~30182049,206380204~11228650,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4760259,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396566061~6312282,409194704~7731957] 2024-09-14T04:32:30.908 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:30.908 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:30.914 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~30182049,206380204~4896801,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:30.914 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to HEAD diff [1532502~4624507,15956065~6778852,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183524730~8784543,192662147~2659618,195446529~10896288,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:30.914 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183524730~8784543,192662147~2659618,195446529~10896288,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:30.921 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~10152005,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,177156787~5609834,183524730~22818087,207366448~3910557,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:30.921 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to HEAD diff [1532502~4624507,15956065~5397876,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:30.921 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:30.926 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~28200107,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:30.926 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to HEAD diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:30.926 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:30.931 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~28200107,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:30.932 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to HEAD diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:30.932 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:30.937 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~22235841,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276824064~7288172,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:30.937 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to HEAD diff [1532502~4624507,15956065~5397876,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,88623968~370312,89021763~2406992,91960020~945991,99264373~856643,109021991~13877146,132996120~5475003,139378274~36780541,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~6956556,307097205~12854986,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:30.938 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,88623968~370312,89021763~2406992,91960020~945991,99264373~856643,109021991~13877146,132996120~5475003,139378274~36780541,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~6956556,307097205~12854986,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:30.943 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,35645530~13536984,50026379~3466398,53772129~974098,55349631~30939764,88623968~370312,89021763~2406992,91960020~945991,96468992~3652024,109021991~13877146,132996120~5475003,139378274~36780541,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276824064~7288172,284870704~8098864,297545811~6956556,307097205~12854986,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:30.943 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to HEAD diff [1532502~4624507,15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,99264373~856643,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:30.943 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,99264373~856643,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:30.949 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,37722924~11459590,50026379~3466398,53772129~974098,55349631~25440567,89021763~2406992,91960020~945991,96468992~3652024,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:30.949 INFO:tasks.workunit.client.0.smithi039.stdout:from 11 to HEAD diff [15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:30.949 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:30.954 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,37722924~11459590,50026379~3466398,53772129~974098,55349631~25440567,89021763~2406992,91960020~945991,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~22818087,207369398~3608367,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:30.954 INFO:tasks.workunit.client.0.smithi039.stdout:from 12 to HEAD diff [15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:30.954 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:30.960 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,148411414~27747401,183524730~22818087,207369398~3608367,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:30.960 INFO:tasks.workunit.client.0.smithi039.stdout:from 13 to HEAD diff [15956065~5397876,37722924~6209571,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,183524730~8784543,192662147~2659618,195446529~6684577,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,237089596~5877557,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:30.960 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,37722924~6209571,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,183524730~8784543,192662147~2659618,195446529~6684577,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,237089596~5877557,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:30.965 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,37722924~6209571,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,150858815~25300000,183524730~18606376,202395118~3947699,207369398~3608367,222441104~495517,231412373~2251966,237089596~5877557,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,347886358~1792403,360827666~8007252,370420162~6009885,384246345~6869073,409194704~7731957] 2024-09-14T04:32:31.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:30 smithi129 ceph-mon[101316]: pgmap v1798: 105 pgs: 105 active+clean; 1.1 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 192 KiB/s rd, 17 MiB/s wr, 4.68k op/s 2024-09-14T04:32:31.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:30 smithi129 ceph-mon[101316]: osdmap e1143: 8 total, 8 up, 8 in 2024-09-14T04:32:31.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:30 smithi039 ceph-mon[106470]: pgmap v1798: 105 pgs: 105 active+clean; 1.1 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 192 KiB/s rd, 17 MiB/s wr, 4.68k op/s 2024-09-14T04:32:31.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:30 smithi039 ceph-mon[106470]: osdmap e1143: 8 total, 8 up, 8 in 2024-09-14T04:32:31.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:30 smithi039 ceph-mon[111235]: pgmap v1798: 105 pgs: 105 active+clean; 1.1 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 192 KiB/s rd, 17 MiB/s wr, 4.68k op/s 2024-09-14T04:32:31.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:30 smithi039 ceph-mon[111235]: osdmap e1143: 8 total, 8 up, 8 in 2024-09-14T04:32:31.889 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:31.889 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:31.907 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~82237545,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:31.907 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:31.907 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:31.913 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:31.913 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:31.913 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:31.918 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,327155712~6219679,335544320~57958504,396566061~6312282,406171321~10755340] 2024-09-14T04:32:31.918 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:31.918 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:31.924 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~30182049,206380204~11228650,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4760259,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396566061~6312282,409194704~7731957] 2024-09-14T04:32:31.924 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:31.924 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:31.930 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~30182049,206380204~4896801,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:31.930 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to HEAD diff [1532502~4624507,15956065~6778852,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183524730~8784543,192662147~2659618,195446529~10896288,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:31.930 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183524730~8784543,192662147~2659618,195446529~10896288,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:31.936 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~10152005,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,177156787~5609834,183524730~22818087,207366448~3910557,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:31.936 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to HEAD diff [1532502~4624507,15956065~5397876,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:31.936 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:31.942 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~28200107,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:31.942 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to HEAD diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:31.942 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:31.947 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~28200107,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:31.947 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to HEAD diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:31.948 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:31.953 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~22235841,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276824064~7288172,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:31.953 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to HEAD diff [1532502~4624507,15956065~5397876,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,88623968~370312,89021763~2406992,91960020~945991,99264373~856643,109021991~13877146,132996120~5475003,139378274~36780541,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~6956556,307097205~12854986,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:31.953 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,88623968~370312,89021763~2406992,91960020~945991,99264373~856643,109021991~13877146,132996120~5475003,139378274~36780541,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~6956556,307097205~12854986,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:31.959 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,35645530~13536984,50026379~3466398,53772129~974098,55349631~30939764,88623968~370312,89021763~2406992,91960020~945991,96468992~3652024,109021991~13877146,132996120~5475003,139378274~36780541,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276824064~7288172,284870704~8098864,297545811~6956556,307097205~12854986,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:31.959 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to HEAD diff [1532502~4624507,15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,99264373~856643,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:31.959 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,99264373~856643,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:31.964 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,37722924~11459590,50026379~3466398,53772129~974098,55349631~25440567,89021763~2406992,91960020~945991,96468992~3652024,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:31.965 INFO:tasks.workunit.client.0.smithi039.stdout:from 11 to HEAD diff [15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:31.965 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:31.970 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,37722924~11459590,50026379~3466398,53772129~974098,55349631~25440567,89021763~2406992,91960020~945991,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~22818087,207369398~3608367,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:31.970 INFO:tasks.workunit.client.0.smithi039.stdout:from 12 to HEAD diff [15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:31.970 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:31.975 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,71362387~9427811,89021763~2406992,109021991~13040769,132996120~2690483,139378274~7402656,148411414~27747401,183524730~22818087,207369398~3608367,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,317400597~2551594,324417134~321003,332042044~606164,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:32.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:31 smithi129 ceph-mon[101316]: osdmap e1144: 8 total, 8 up, 8 in 2024-09-14T04:32:32.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:31 smithi039 ceph-mon[106470]: osdmap e1144: 8 total, 8 up, 8 in 2024-09-14T04:32:32.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:31 smithi039 ceph-mon[111235]: osdmap e1144: 8 total, 8 up, 8 in 2024-09-14T04:32:32.904 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:32.905 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:32.923 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~82237545,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:32.923 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:32.923 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:32.929 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:32.929 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:32.930 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:32.935 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,327155712~6219679,335544320~57958504,396566061~6312282,406171321~10755340] 2024-09-14T04:32:32.935 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:32.936 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:32.941 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~30182049,206380204~11228650,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4760259,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396566061~6312282,409194704~7731957] 2024-09-14T04:32:32.941 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:32.941 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:32.947 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~30182049,206380204~4896801,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:32.947 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to HEAD diff [1532502~4624507,15956065~6778852,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183524730~8784543,192662147~2659618,195446529~10896288,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:32.947 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183524730~8784543,192662147~2659618,195446529~10896288,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:32.954 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~10152005,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,177156787~5609834,183524730~22818087,207366448~3910557,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:32.954 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to HEAD diff [1532502~4624507,15956065~5397876,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:32.954 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:32.960 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~28200107,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:32.960 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to HEAD diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:32.960 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:32.966 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~28200107,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:32.966 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to HEAD diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:32.966 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:32.971 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~22235841,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276824064~7288172,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:32.971 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to HEAD diff [1532502~4624507,15956065~5397876,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,88623968~370312,89021763~2406992,91960020~945991,99264373~856643,109021991~13877146,132996120~5475003,139378274~36780541,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~6956556,307097205~12854986,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:32.972 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,88623968~370312,89021763~2406992,91960020~945991,99264373~856643,109021991~13877146,132996120~5475003,139378274~36780541,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~6956556,307097205~12854986,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:32.977 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,35645530~13536984,50026379~3466398,53772129~974098,55349631~30939764,88623968~370312,89021763~2406992,91960020~945991,96468992~3652024,109021991~13877146,132996120~5475003,139378274~36780541,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276824064~7288172,284870704~8098864,297545811~6956556,307097205~12854986,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:32.977 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to HEAD diff [1532502~4624507,15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,99264373~856643,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:32.977 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,99264373~856643,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:32.983 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,37722924~11459590,50026379~3466398,53772129~974098,55349631~25440567,89021763~2406992,91960020~945991,96468992~3652024,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:32.983 INFO:tasks.workunit.client.0.smithi039.stdout:from 11 to HEAD diff [15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:32.983 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3608367,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:32.988 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8771029,37722924~11459590,50026379~3466398,53772129~974098,55349631~25440567,89021763~2406992,91960020~945991,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~22818087,207369398~3608367,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,347886358~1792403,360827666~8007252,370420162~6009885,378397449~5799119,384246345~6869073,409194704~7731957] 2024-09-14T04:32:33.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:32 smithi129 ceph-mon[101316]: pgmap v1801: 105 pgs: 105 active+clean; 1.1 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 214 KiB/s rd, 14 MiB/s wr, 4.69k op/s 2024-09-14T04:32:33.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:32 smithi129 ceph-mon[101316]: osdmap e1145: 8 total, 8 up, 8 in 2024-09-14T04:32:33.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:32 smithi039 ceph-mon[106470]: pgmap v1801: 105 pgs: 105 active+clean; 1.1 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 214 KiB/s rd, 14 MiB/s wr, 4.69k op/s 2024-09-14T04:32:33.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:32 smithi039 ceph-mon[106470]: osdmap e1145: 8 total, 8 up, 8 in 2024-09-14T04:32:33.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:32 smithi039 ceph-mon[111235]: pgmap v1801: 105 pgs: 105 active+clean; 1.1 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 214 KiB/s rd, 14 MiB/s wr, 4.69k op/s 2024-09-14T04:32:33.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:32 smithi039 ceph-mon[111235]: osdmap e1145: 8 total, 8 up, 8 in 2024-09-14T04:32:33.914 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:33.914 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:33.931 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~82237545,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:33.931 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:33.931 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:33.937 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:33.937 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:33.937 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:33.943 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,327155712~6219679,335544320~57958504,396566061~6312282,406171321~10755340] 2024-09-14T04:32:33.943 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:33.943 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:33.951 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~30182049,206380204~11228650,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4760259,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396566061~6312282,409194704~7731957] 2024-09-14T04:32:33.951 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:33.951 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:33.957 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~30182049,206380204~4896801,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:33.957 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to HEAD diff [1532502~4624507,15956065~6778852,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183524730~8784543,192662147~2659618,195446529~10896288,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:33.957 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183524730~8784543,192662147~2659618,195446529~10896288,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:33.963 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~10152005,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,177156787~5609834,183524730~22818087,207366448~3910557,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:33.963 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to HEAD diff [1532502~4624507,15956065~5397876,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:33.963 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:33.969 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~28200107,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:33.969 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to HEAD diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:33.969 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:33.975 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~28200107,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:33.975 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to HEAD diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:33.975 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:33.981 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~22235841,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276824064~7288172,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:33.981 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to HEAD diff [1532502~4624507,15956065~5397876,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,88623968~370312,89021763~2406992,91960020~945991,99264373~856643,109021991~13877146,132996120~5475003,139378274~36780541,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~6956556,307097205~12854986,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:33.981 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,88623968~370312,89021763~2406992,91960020~945991,99264373~856643,109021991~13877146,132996120~5475003,139378274~36780541,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~6956556,307097205~12854986,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:33.986 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,35645530~13536984,50026379~3466398,53772129~974098,55349631~30939764,88623968~370312,89021763~2406992,91960020~945991,96468992~3652024,109021991~13877146,132996120~5475003,139378274~36780541,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276824064~7288172,284870704~8098864,297545811~6956556,307097205~12854986,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:33.986 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to HEAD diff [1532502~4624507,15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,99264373~856643,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:33.986 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,37722924~11459590,50026379~3466398,53772129~974098,55349631~9438706,65187704~15602494,89021763~2406992,91960020~945991,99264373~856643,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:33.992 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,37722924~11459590,50026379~3466398,53772129~974098,55349631~25440567,89021763~2406992,91960020~945991,96468992~3652024,109021991~13040769,132996120~5475003,139378274~7402656,148411414~27747401,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~452599,284870704~8098864,297545811~6956556,307097205~6187809,315889099~4063092,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:34.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:33 smithi129 ceph-mon[101316]: osdmap e1146: 8 total, 8 up, 8 in 2024-09-14T04:32:34.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:33 smithi039 ceph-mon[106470]: osdmap e1146: 8 total, 8 up, 8 in 2024-09-14T04:32:34.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:33 smithi039 ceph-mon[111235]: osdmap e1146: 8 total, 8 up, 8 in 2024-09-14T04:32:34.913 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:34.913 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:34.940 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~82237545,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:34.940 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:34.940 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:34.946 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:34.946 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:34.947 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:34.952 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,327155712~6219679,335544320~57958504,396566061~6312282,406171321~10755340] 2024-09-14T04:32:34.952 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:34.952 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:34.958 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~30182049,206380204~11228650,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4760259,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396566061~6312282,409194704~7731957] 2024-09-14T04:32:34.958 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:34.958 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:34.964 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~30182049,206380204~4896801,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:34.964 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to HEAD diff [1532502~4624507,15956065~6778852,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183524730~8784543,192662147~2659618,195446529~10896288,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:34.964 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183524730~8784543,192662147~2659618,195446529~10896288,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:34.970 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~10152005,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,177156787~5609834,183524730~22818087,207366448~3910557,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:34.970 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to HEAD diff [1532502~4624507,15956065~5397876,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:34.970 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:34.975 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~28200107,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:34.975 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to HEAD diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:34.975 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:34.981 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~28200107,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:34.982 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to HEAD diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:34.982 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:34.987 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~22235841,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276824064~7288172,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:34.988 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to HEAD diff [1532502~4624507,15956065~5397876,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,88623968~370312,89021763~2406992,91960020~945991,99264373~856643,109021991~13877146,132996120~5475003,139378274~36780541,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~6956556,307097205~12854986,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:34.988 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,88623968~370312,89021763~2406992,91960020~945991,99264373~856643,109021991~13877146,132996120~5475003,139378274~36780541,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~6956556,307097205~12854986,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:34.992 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,35645530~13536984,50026379~3466398,53772129~974098,55349631~30939764,88623968~370312,89021763~2406992,91960020~945991,96468992~3652024,109021991~13877146,132996120~5475003,139378274~36780541,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276824064~7288172,284870704~8098864,297545811~6956556,307097205~12854986,324417134~321003,328615132~4033076,338917256~193834,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,378366520~5830048,384246345~6869073,409194704~7731957] 2024-09-14T04:32:35.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:34 smithi129 ceph-mon[101316]: pgmap v1804: 105 pgs: 105 active+clean; 1.1 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 137 KiB/s rd, 6.2 KiB/s wr, 1.98k op/s 2024-09-14T04:32:35.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:34 smithi129 ceph-mon[101316]: osdmap e1147: 8 total, 8 up, 8 in 2024-09-14T04:32:35.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:34 smithi039 ceph-mon[106470]: pgmap v1804: 105 pgs: 105 active+clean; 1.1 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 137 KiB/s rd, 6.2 KiB/s wr, 1.98k op/s 2024-09-14T04:32:35.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:34 smithi039 ceph-mon[106470]: osdmap e1147: 8 total, 8 up, 8 in 2024-09-14T04:32:35.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:34 smithi039 ceph-mon[111235]: pgmap v1804: 105 pgs: 105 active+clean; 1.1 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 137 KiB/s rd, 6.2 KiB/s wr, 1.98k op/s 2024-09-14T04:32:35.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:34 smithi039 ceph-mon[111235]: osdmap e1147: 8 total, 8 up, 8 in 2024-09-14T04:32:35.815 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:35.815 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:35.836 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~82237545,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:35.836 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:35.836 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:35.842 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:35.842 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:35.842 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:35.848 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,327155712~6219679,335544320~57958504,396566061~6312282,406171321~10755340] 2024-09-14T04:32:35.848 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:35.848 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:35.854 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~30182049,206380204~11228650,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4760259,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396566061~6312282,409194704~7731957] 2024-09-14T04:32:35.854 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:35.854 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:35.860 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~30182049,206380204~4896801,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:35.860 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to HEAD diff [1532502~4624507,15956065~6778852,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183524730~8784543,192662147~2659618,195446529~10896288,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:35.860 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183524730~8784543,192662147~2659618,195446529~10896288,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:35.866 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~10152005,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,177156787~5609834,183524730~22818087,207366448~3910557,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:35.866 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to HEAD diff [1532502~4624507,15956065~5397876,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:35.866 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:35.871 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~28200107,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:35.871 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to HEAD diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:35.871 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:35.877 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~28200107,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:35.877 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to HEAD diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:35.877 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:35.882 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~22235841,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276824064~7288172,284870704~8098864,297545811~26281854,324417134~321003,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~1792403,350270295~5576933,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,409194704~7731957] 2024-09-14T04:32:36.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:35 smithi129 ceph-mon[101316]: osdmap e1148: 8 total, 8 up, 8 in 2024-09-14T04:32:36.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:35 smithi129 ceph-mon[101316]: osdmap e1149: 8 total, 8 up, 8 in 2024-09-14T04:32:36.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:35 smithi039 ceph-mon[106470]: osdmap e1148: 8 total, 8 up, 8 in 2024-09-14T04:32:36.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:35 smithi039 ceph-mon[106470]: osdmap e1149: 8 total, 8 up, 8 in 2024-09-14T04:32:36.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:35 smithi039 ceph-mon[111235]: osdmap e1148: 8 total, 8 up, 8 in 2024-09-14T04:32:36.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:35 smithi039 ceph-mon[111235]: osdmap e1149: 8 total, 8 up, 8 in 2024-09-14T04:32:36.812 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:36.812 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:36.834 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~82237545,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:36.834 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:36.834 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:36.839 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:36.840 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:36.840 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:36.845 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,327155712~6219679,335544320~57958504,396566061~6312282,406171321~10755340] 2024-09-14T04:32:36.846 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:36.846 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:36.851 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~30182049,206380204~11228650,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4760259,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396566061~6312282,409194704~7731957] 2024-09-14T04:32:36.851 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:36.852 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:36.857 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~30182049,206380204~4896801,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:36.857 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to HEAD diff [1532502~4624507,15956065~6778852,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183524730~8784543,192662147~2659618,195446529~10896288,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:36.857 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183524730~8784543,192662147~2659618,195446529~10896288,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:36.863 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~10152005,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,177156787~5609834,183524730~22818087,207366448~3910557,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:36.863 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to HEAD diff [1532502~4624507,15956065~5397876,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:36.863 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:36.869 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~28200107,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:36.869 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to HEAD diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:36.869 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:36.874 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~28200107,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,343616361~175529,344830868~904290,347886358~7960870,360827666~8007252,370420162~6009885,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:37.191 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:36 smithi039 ceph-mon[106470]: pgmap v1807: 105 pgs: 105 active+clean; 1.1 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 136 KiB/s rd, 6.2 KiB/s wr, 2.04k op/s 2024-09-14T04:32:37.191 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:36 smithi039 ceph-mon[106470]: osdmap e1150: 8 total, 8 up, 8 in 2024-09-14T04:32:37.192 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:36 smithi039 ceph-mon[111235]: pgmap v1807: 105 pgs: 105 active+clean; 1.1 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 136 KiB/s rd, 6.2 KiB/s wr, 2.04k op/s 2024-09-14T04:32:37.192 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:36 smithi039 ceph-mon[111235]: osdmap e1150: 8 total, 8 up, 8 in 2024-09-14T04:32:37.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:36 smithi129 ceph-mon[101316]: pgmap v1807: 105 pgs: 105 active+clean; 1.1 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 136 KiB/s rd, 6.2 KiB/s wr, 2.04k op/s 2024-09-14T04:32:37.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:36 smithi129 ceph-mon[101316]: osdmap e1150: 8 total, 8 up, 8 in 2024-09-14T04:32:37.812 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:37.813 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:37.837 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~82237545,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:37.837 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:37.837 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:37.843 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:37.843 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:37.843 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:37.849 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,327155712~6219679,335544320~57958504,396566061~6312282,406171321~10755340] 2024-09-14T04:32:37.849 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:37.849 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:37.855 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~30182049,206380204~11228650,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4760259,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396566061~6312282,409194704~7731957] 2024-09-14T04:32:37.855 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:37.855 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:37.861 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~30182049,206380204~4896801,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:37.861 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to HEAD diff [1532502~4624507,15956065~6778852,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183524730~8784543,192662147~2659618,195446529~10896288,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:37.861 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183524730~8784543,192662147~2659618,195446529~10896288,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:37.868 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~10152005,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,177156787~5609834,183524730~22818087,207366448~3910557,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:37.868 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to HEAD diff [1532502~4624507,15956065~5397876,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:37.869 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~5397876,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,132996120~5475003,139378274~36780541,178708911~4057710,183524730~8784543,192662147~2659618,195446529~10896288,207369398~3907607,218384295~132186,222441104~495517,225312546~5051871,231412373~2251966,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:37.872 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~8771029,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~28200107,132996120~5475003,139378274~36780541,178708911~4057710,183524730~22818087,207369398~3907607,218103808~412673,222298112~8066305,231412373~2251966,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~7960870,360827666~15602381,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:32:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:37 smithi129 ceph-mon[101316]: osdmap e1151: 8 total, 8 up, 8 in 2024-09-14T04:32:38.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:32:38.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:37 smithi039 ceph-mon[106470]: osdmap e1151: 8 total, 8 up, 8 in 2024-09-14T04:32:38.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:32:38.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:37 smithi039 ceph-mon[111235]: osdmap e1151: 8 total, 8 up, 8 in 2024-09-14T04:32:38.823 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:38.823 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:38.847 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~82237545,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:38.847 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:38.847 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:38.853 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:38.853 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:38.854 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:38.860 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,327155712~6219679,335544320~57958504,396566061~6312282,406171321~10755340] 2024-09-14T04:32:38.860 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:38.860 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:38.866 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~30182049,206380204~11228650,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4760259,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396566061~6312282,409194704~7731957] 2024-09-14T04:32:38.866 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:38.866 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:38.872 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~30182049,206380204~4896801,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:38.872 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to HEAD diff [1532502~4624507,15956065~6778852,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183524730~8784543,192662147~2659618,195446529~10896288,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:38.872 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183524730~8784543,192662147~2659618,195446529~10896288,207366448~3910557,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:38.878 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [1532502~4624507,12582912~10152005,28814760~445192,33861843~922299,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,177156787~5609834,183524730~22818087,207366448~3910557,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:38.925 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:32:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:32:38] "GET /metrics HTTP/1.1" 200 36692 "" "Prometheus/2.51.0" 2024-09-14T04:32:39.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:38 smithi129 ceph-mon[101316]: pgmap v1810: 105 pgs: 105 active+clean; 1.1 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 412 op/s 2024-09-14T04:32:39.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:38 smithi129 ceph-mon[101316]: osdmap e1152: 8 total, 8 up, 8 in 2024-09-14T04:32:39.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:38 smithi039 ceph-mon[111235]: pgmap v1810: 105 pgs: 105 active+clean; 1.1 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 412 op/s 2024-09-14T04:32:39.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:38 smithi039 ceph-mon[111235]: osdmap e1152: 8 total, 8 up, 8 in 2024-09-14T04:32:39.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:38 smithi039 ceph-mon[106470]: pgmap v1810: 105 pgs: 105 active+clean; 1.1 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 0 B/s rd, 412 op/s 2024-09-14T04:32:39.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:38 smithi039 ceph-mon[106470]: osdmap e1152: 8 total, 8 up, 8 in 2024-09-14T04:32:39.825 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:39.825 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:39.848 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~82237545,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:39.848 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:39.848 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:39.854 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:39.854 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:39.854 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:39.860 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,327155712~6219679,335544320~57958504,396566061~6312282,406171321~10755340] 2024-09-14T04:32:39.860 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:39.860 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:39.866 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~30182049,206380204~11228650,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4760259,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396566061~6312282,409194704~7731957] 2024-09-14T04:32:39.866 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:39.866 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~9438706,65187704~21101691,87561278~3867477,91960020~945991,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~4896801,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~4403906,253979221~16488519,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4033076,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396800802~6077541,409194704~7731957] 2024-09-14T04:32:39.872 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~3466398,53772129~974098,55349631~36079124,91960020~945991,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~30182049,206380204~4896801,218103808~412673,222298112~12132675,237089596~11365216,249153849~4403906,253979221~20732513,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4033076,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~6869073,396566061~6312282,409194704~7731957] 2024-09-14T04:32:40.820 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:40.820 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:40.844 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~82237545,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:40.844 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:40.844 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:40.849 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:40.849 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:40.849 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:40.855 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,327155712~6219679,335544320~57958504,396566061~6312282,406171321~10755340] 2024-09-14T04:32:40.856 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:40.856 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~10896288,206380204~11228650,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396800802~6077541,409194704~7731957] 2024-09-14T04:32:40.861 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~30182049,206380204~11228650,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,328615132~4760259,335544320~6690394,343616361~175529,344830868~904290,347886358~28543689,377219330~6977238,384246345~9256479,396566061~6312282,409194704~7731957] 2024-09-14T04:32:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:40 smithi129 ceph-mon[101316]: pgmap v1813: 105 pgs: 105 active+clean; 1.0 GiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 104 KiB/s rd, 6.2 KiB/s wr, 1.36k op/s 2024-09-14T04:32:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:40 smithi129 ceph-mon[101316]: osdmap e1153: 8 total, 8 up, 8 in 2024-09-14T04:32:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:40 smithi129 ceph-mon[101316]: osdmap e1154: 8 total, 8 up, 8 in 2024-09-14T04:32:41.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:40 smithi039 ceph-mon[106470]: pgmap v1813: 105 pgs: 105 active+clean; 1.0 GiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 104 KiB/s rd, 6.2 KiB/s wr, 1.36k op/s 2024-09-14T04:32:41.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:40 smithi039 ceph-mon[106470]: osdmap e1153: 8 total, 8 up, 8 in 2024-09-14T04:32:41.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:40 smithi039 ceph-mon[106470]: osdmap e1154: 8 total, 8 up, 8 in 2024-09-14T04:32:41.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:40 smithi039 ceph-mon[111235]: pgmap v1813: 105 pgs: 105 active+clean; 1.0 GiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 104 KiB/s rd, 6.2 KiB/s wr, 1.36k op/s 2024-09-14T04:32:41.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:40 smithi039 ceph-mon[111235]: osdmap e1153: 8 total, 8 up, 8 in 2024-09-14T04:32:41.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:40 smithi039 ceph-mon[111235]: osdmap e1154: 8 total, 8 up, 8 in 2024-09-14T04:32:41.835 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:41.835 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:41.860 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~82237545,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:41.861 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:41.861 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:41.866 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:41.866 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:41.866 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,237089596~11365216,249153849~21313891,271309101~3402633,276825837~7286399,284870704~8098864,297545811~28367692,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,406171321~10755340] 2024-09-14T04:32:41.872 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~12132675,237089596~37622138,276824064~7288172,284870704~8098864,297545811~28367692,327155712~6219679,335544320~57958504,396566061~6312282,406171321~10755340] 2024-09-14T04:32:42.855 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:42.855 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:42.872 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~82237545,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:42.872 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:42.872 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~10417690,35645530~13536984,50026379~14761958,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:42.878 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~10152005,24366452~10417690,35645530~13536984,50026379~44794078,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:42 smithi129 ceph-mon[101316]: pgmap v1816: 105 pgs: 105 active+clean; 1.0 GiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 104 KiB/s rd, 6.2 KiB/s wr, 1.14k op/s 2024-09-14T04:32:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:42 smithi129 ceph-mon[101316]: osdmap e1155: 8 total, 8 up, 8 in 2024-09-14T04:32:43.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:42 smithi039 ceph-mon[106470]: pgmap v1816: 105 pgs: 105 active+clean; 1.0 GiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 104 KiB/s rd, 6.2 KiB/s wr, 1.14k op/s 2024-09-14T04:32:43.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:42 smithi039 ceph-mon[106470]: osdmap e1155: 8 total, 8 up, 8 in 2024-09-14T04:32:43.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:42 smithi039 ceph-mon[111235]: pgmap v1816: 105 pgs: 105 active+clean; 1.0 GiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 104 KiB/s rd, 6.2 KiB/s wr, 1.14k op/s 2024-09-14T04:32:43.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:42 smithi039 ceph-mon[111235]: osdmap e1155: 8 total, 8 up, 8 in 2024-09-14T04:32:43.870 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:43.870 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [1532502~4624507,15956065~6778852,24366452~24816062,49625492~15162845,65187704~21101691,87561278~7259179,99264373~856643,102891979~20007158,125750260~3113143,129593184~3289710,132996120~5475003,139378274~36780541,177156787~5609834,183215660~9093613,192662147~2659618,195446529~22162325,218384295~132186,222441104~495517,225312546~9118241,234445135~14009677,249153849~21313891,271309101~3402633,276825837~49087666,328615132~4760259,338917256~193834,339416815~2817899,342666453~33763594,376743309~16759515,396800802~6077541,403429066~13497595] 2024-09-14T04:32:43.885 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~6157009,12582912~82237545,96468992~3652024,100663296~32219598,132996120~5475003,139378274~36780541,176160768~41448086,218103808~412673,222298112~52413622,276824064~49089439,327155712~6219679,335544320~57958504,396566061~20360600] 2024-09-14T04:32:44.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:43 smithi129 ceph-mon[101316]: osdmap e1156: 8 total, 8 up, 8 in 2024-09-14T04:32:44.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:43 smithi039 ceph-mon[106470]: osdmap e1156: 8 total, 8 up, 8 in 2024-09-14T04:32:44.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:43 smithi039 ceph-mon[111235]: osdmap e1156: 8 total, 8 up, 8 in 2024-09-14T04:32:44.894 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/0.DiffIterateStress (40344 ms) 2024-09-14T04:32:44.894 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/0.DiffIterateParent 2024-09-14T04:32:44.894 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:32:45.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:44 smithi129 ceph-mon[101316]: pgmap v1819: 105 pgs: 105 active+clean; 909 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 71 KiB/s rd, 6.0 KiB/s wr, 690 op/s 2024-09-14T04:32:45.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:44 smithi129 ceph-mon[101316]: osdmap e1157: 8 total, 8 up, 8 in 2024-09-14T04:32:45.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:44 smithi039 ceph-mon[106470]: pgmap v1819: 105 pgs: 105 active+clean; 909 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 71 KiB/s rd, 6.0 KiB/s wr, 690 op/s 2024-09-14T04:32:45.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:44 smithi039 ceph-mon[106470]: osdmap e1157: 8 total, 8 up, 8 in 2024-09-14T04:32:45.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:44 smithi039 ceph-mon[111235]: pgmap v1819: 105 pgs: 105 active+clean; 909 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 71 KiB/s rd, 6.0 KiB/s wr, 690 op/s 2024-09-14T04:32:45.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:44 smithi039 ceph-mon[111235]: osdmap e1157: 8 total, 8 up, 8 in 2024-09-14T04:32:45.915 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/0.DiffIterateParent (1021 ms) 2024-09-14T04:32:45.915 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/0.DiffIterateIgnoreParent 2024-09-14T04:32:45.916 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:32:46.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:45 smithi129 ceph-mon[101316]: osdmap e1158: 8 total, 8 up, 8 in 2024-09-14T04:32:46.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:45 smithi129 ceph-mon[101316]: osdmap e1159: 8 total, 8 up, 8 in 2024-09-14T04:32:46.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:45 smithi039 ceph-mon[106470]: osdmap e1158: 8 total, 8 up, 8 in 2024-09-14T04:32:46.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:45 smithi039 ceph-mon[106470]: osdmap e1159: 8 total, 8 up, 8 in 2024-09-14T04:32:46.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:45 smithi039 ceph-mon[111235]: osdmap e1158: 8 total, 8 up, 8 in 2024-09-14T04:32:46.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:45 smithi039 ceph-mon[111235]: osdmap e1159: 8 total, 8 up, 8 in 2024-09-14T04:32:47.004 INFO:tasks.workunit.client.0.smithi039.stdout: wrote [292073~48710,4780695~15182,5835342~84641,7251057~30031,10007405~82835,10981539~93653,12361912~38724,12757108~29551,15318516~51861,15752291~37192] to clone 2024-09-14T04:32:47.004 INFO:tasks.workunit.client.0.smithi039.stdout: diff was [0~340783,4194304~3086784,8388608~4012028,12582912~3206571] 2024-09-14T04:32:47.008 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/0.DiffIterateIgnoreParent (1093 ms) 2024-09-14T04:32:47.008 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/0.DiffIterateCallbackError 2024-09-14T04:32:47.008 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:32:47.033 INFO:tasks.workunit.client.0.smithi039.stdout: wrote [1399057~17018,2772941~97636,6706198~90305,8142220~82172,11410242~12291,14290858~760,14605669~49526,15208909~38999,17949386~53314] 2024-09-14T04:32:47.042 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/0.DiffIterateCallbackError (34 ms) 2024-09-14T04:32:47.042 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/0.DiffIterateParentDiscard 2024-09-14T04:32:47.042 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:32:47.157 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:46 smithi129 ceph-mon[101316]: pgmap v1822: 105 pgs: 105 active+clean; 890 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s rd, 6.0 KiB/s wr, 712 op/s 2024-09-14T04:32:47.158 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:46 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:32:47.189 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:46 smithi039 ceph-mon[106470]: pgmap v1822: 105 pgs: 105 active+clean; 890 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s rd, 6.0 KiB/s wr, 712 op/s 2024-09-14T04:32:47.189 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:46 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:32:47.189 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:46 smithi039 ceph-mon[111235]: pgmap v1822: 105 pgs: 105 active+clean; 890 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 71 KiB/s rd, 6.0 KiB/s wr, 712 op/s 2024-09-14T04:32:47.189 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:46 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:32:48.154 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:47 smithi129 ceph-mon[101316]: osdmap e1160: 8 total, 8 up, 8 in 2024-09-14T04:32:48.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:47 smithi039 ceph-mon[106470]: osdmap e1160: 8 total, 8 up, 8 in 2024-09-14T04:32:48.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:47 smithi039 ceph-mon[111235]: osdmap e1160: 8 total, 8 up, 8 in 2024-09-14T04:32:48.991 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/0.DiffIterateParentDiscard (1948 ms) 2024-09-14T04:32:48.991 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/0.DiffIterateUnalignedSmall 2024-09-14T04:32:48.991 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:32:49.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:48 smithi039 ceph-mon[106470]: pgmap v1825: 105 pgs: 105 active+clean; 861 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.0 MiB/s wr, 82 op/s 2024-09-14T04:32:49.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:48 smithi039 ceph-mon[106470]: osdmap e1161: 8 total, 8 up, 8 in 2024-09-14T04:32:49.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:32:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:32:48] "GET /metrics HTTP/1.1" 200 36692 "" "Prometheus/2.51.0" 2024-09-14T04:32:49.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:48 smithi039 ceph-mon[111235]: pgmap v1825: 105 pgs: 105 active+clean; 861 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.0 MiB/s wr, 82 op/s 2024-09-14T04:32:49.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:48 smithi039 ceph-mon[111235]: osdmap e1161: 8 total, 8 up, 8 in 2024-09-14T04:32:49.045 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/0.DiffIterateUnalignedSmall (54 ms) 2024-09-14T04:32:49.045 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/0.DiffIterateUnaligned 2024-09-14T04:32:49.045 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:32:49.126 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/0.DiffIterateUnaligned (81 ms) 2024-09-14T04:32:49.126 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/0.DiffIterateTryAcquireLock 2024-09-14T04:32:49.126 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:32:49.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:48 smithi129 ceph-mon[101316]: pgmap v1825: 105 pgs: 105 active+clean; 861 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 7.7 KiB/s rd, 4.0 MiB/s wr, 82 op/s 2024-09-14T04:32:49.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:48 smithi129 ceph-mon[101316]: osdmap e1161: 8 total, 8 up, 8 in 2024-09-14T04:32:50.185 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:49 smithi039 ceph-mon[106470]: osdmap e1162: 8 total, 8 up, 8 in 2024-09-14T04:32:50.185 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:49 smithi039 ceph-mon[111235]: osdmap e1162: 8 total, 8 up, 8 in 2024-09-14T04:32:50.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:49 smithi129 ceph-mon[101316]: osdmap e1162: 8 total, 8 up, 8 in 2024-09-14T04:32:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:50 smithi129 ceph-mon[101316]: pgmap v1828: 105 pgs: 105 active+clean; 822 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 11 MiB/s wr, 289 op/s 2024-09-14T04:32:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:50 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:32:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:50 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:32:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:50 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:32:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:50 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:32:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:50 smithi129 ceph-mon[101316]: osdmap e1163: 8 total, 8 up, 8 in 2024-09-14T04:32:51.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:50 smithi039 ceph-mon[106470]: pgmap v1828: 105 pgs: 105 active+clean; 822 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 11 MiB/s wr, 289 op/s 2024-09-14T04:32:51.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:50 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:32:51.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:50 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:32:51.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:50 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:32:51.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:50 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:32:51.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:50 smithi039 ceph-mon[106470]: osdmap e1163: 8 total, 8 up, 8 in 2024-09-14T04:32:51.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:50 smithi039 ceph-mon[111235]: pgmap v1828: 105 pgs: 105 active+clean; 822 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 5.1 MiB/s rd, 11 MiB/s wr, 289 op/s 2024-09-14T04:32:51.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:50 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:32:51.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:50 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:32:51.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:50 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:32:51.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:50 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:32:51.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:50 smithi039 ceph-mon[111235]: osdmap e1163: 8 total, 8 up, 8 in 2024-09-14T04:32:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:51 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:32:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:51 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:32:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:51 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:32:52.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:51 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:32:52.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:51 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:32:52.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:51 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:32:52.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:51 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:32:52.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:51 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:32:52.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:51 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:32:53.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:53 smithi129 ceph-mon[101316]: pgmap v1830: 105 pgs: 105 active+clean; 822 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 6.0 MiB/s wr, 258 op/s 2024-09-14T04:32:53.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:53 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:32:53.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:53 smithi039 ceph-mon[106470]: pgmap v1830: 105 pgs: 105 active+clean; 822 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 6.0 MiB/s wr, 258 op/s 2024-09-14T04:32:53.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:53 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:32:53.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:53 smithi039 ceph-mon[111235]: pgmap v1830: 105 pgs: 105 active+clean; 822 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 4.6 MiB/s rd, 6.0 MiB/s wr, 258 op/s 2024-09-14T04:32:53.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:53 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:32:54.162 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/0.DiffIterateTryAcquireLock (5037 ms) 2024-09-14T04:32:54.162 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/0.DiffIterateStriping 2024-09-14T04:32:54.163 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/test_librbd.cc:8958: Skipped 2024-09-14T04:32:54.163 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T04:32:54.163 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] DiffIterateTest/0.DiffIterateStriping (0 ms) 2024-09-14T04:32:55.130 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 19 tests from DiffIterateTest/0 (254226 ms total) 2024-09-14T04:32:55.130 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:32:55.130 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 19 tests from DiffIterateTest/1, where TypeParam = DiffIterateParams 2024-09-14T04:32:55.162 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/1.DiffIterate 2024-09-14T04:32:55.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:55 smithi129 ceph-mon[101316]: pgmap v1831: 105 pgs: 105 active+clean; 873 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s rd, 9.3 MiB/s wr, 322 op/s 2024-09-14T04:32:55.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:55 smithi039 ceph-mon[106470]: pgmap v1831: 105 pgs: 105 active+clean; 873 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s rd, 9.3 MiB/s wr, 322 op/s 2024-09-14T04:32:55.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:55 smithi039 ceph-mon[111235]: pgmap v1831: 105 pgs: 105 active+clean; 873 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 3.6 MiB/s rd, 9.3 MiB/s wr, 322 op/s 2024-09-14T04:32:56.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:56 smithi129 ceph-mon[101316]: osdmap e1164: 8 total, 8 up, 8 in 2024-09-14T04:32:56.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:56 smithi039 ceph-mon[106470]: osdmap e1164: 8 total, 8 up, 8 in 2024-09-14T04:32:56.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:56 smithi039 ceph-mon[111235]: osdmap e1164: 8 total, 8 up, 8 in 2024-09-14T04:32:57.162 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:32:57.422 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:57 smithi129 ceph-mon[101316]: pgmap v1833: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:32:57.423 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:57 smithi129 ceph-mon[101316]: osdmap e1165: 8 total, 8 up, 8 in 2024-09-14T04:32:57.423 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:57 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4088314566' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-48","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:32:57.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:57 smithi039 ceph-mon[106470]: pgmap v1833: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:32:57.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:57 smithi039 ceph-mon[106470]: osdmap e1165: 8 total, 8 up, 8 in 2024-09-14T04:32:57.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:57 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4088314566' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-48","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:32:57.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:57 smithi039 ceph-mon[111235]: pgmap v1833: 73 pgs: 73 active+clean; 582 KiB data, 2.6 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:32:57.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:57 smithi039 ceph-mon[111235]: osdmap e1165: 8 total, 8 up, 8 in 2024-09-14T04:32:57.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:57 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4088314566' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-48","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T04:32:58.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:58 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4088314566' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-48","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:32:58.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:58 smithi129 ceph-mon[101316]: osdmap e1166: 8 total, 8 up, 8 in 2024-09-14T04:32:58.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:58 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4088314566' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-48","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:32:58.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:58 smithi039 ceph-mon[106470]: osdmap e1166: 8 total, 8 up, 8 in 2024-09-14T04:32:58.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:58 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4088314566' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-librbd-smithi039-85658-48","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T04:32:58.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:58 smithi039 ceph-mon[111235]: osdmap e1166: 8 total, 8 up, 8 in 2024-09-14T04:32:59.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:32:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:32:58] "GET /metrics HTTP/1.1" 200 36700 "" "Prometheus/2.51.0" 2024-09-14T04:32:59.226 INFO:tasks.workunit.client.0.smithi039.stdout: wrote [1671333~97421,6756915~89462,7687435~85348,8489417~91196,12219607~95997,13314587~40146,16436512~94590,17419073~43549] 2024-09-14T04:32:59.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:59 smithi129 ceph-mon[101316]: pgmap v1836: 105 pgs: 7 creating+peering, 25 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:32:59.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:32:59 smithi129 ceph-mon[101316]: osdmap e1167: 8 total, 8 up, 8 in 2024-09-14T04:32:59.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:59 smithi039 ceph-mon[106470]: pgmap v1836: 105 pgs: 7 creating+peering, 25 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:32:59.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:32:59 smithi039 ceph-mon[106470]: osdmap e1167: 8 total, 8 up, 8 in 2024-09-14T04:32:59.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:59 smithi039 ceph-mon[111235]: pgmap v1836: 105 pgs: 7 creating+peering, 25 unknown, 73 active+clean; 582 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:32:59.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:32:59 smithi039 ceph-mon[111235]: osdmap e1167: 8 total, 8 up, 8 in 2024-09-14T04:33:00.238 INFO:tasks.workunit.client.0.smithi039.stdout: wrote [0~4194304,8388608~8388608] 2024-09-14T04:33:00.246 INFO:tasks.workunit.client.0.smithi039.stdout: diff was [0~4194304,8388608~8388608] 2024-09-14T04:33:00.250 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/1.DiffIterate (5088 ms) 2024-09-14T04:33:00.250 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/1.DiffIterateDeterministic 2024-09-14T04:33:00.251 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:33:00.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:00 smithi129 ceph-mon[101316]: osdmap e1168: 8 total, 8 up, 8 in 2024-09-14T04:33:00.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:00 smithi039 ceph-mon[106470]: osdmap e1168: 8 total, 8 up, 8 in 2024-09-14T04:33:00.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:00 smithi039 ceph-mon[111235]: osdmap e1168: 8 total, 8 up, 8 in 2024-09-14T04:33:01.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:01 smithi129 ceph-mon[101316]: pgmap v1839: 105 pgs: 7 creating+peering, 2 unknown, 96 active+clean; 582 KiB data, 895 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 255 B/s wr, 0 op/s 2024-09-14T04:33:01.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:01 smithi129 ceph-mon[101316]: osdmap e1169: 8 total, 8 up, 8 in 2024-09-14T04:33:01.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:01 smithi039 ceph-mon[106470]: pgmap v1839: 105 pgs: 7 creating+peering, 2 unknown, 96 active+clean; 582 KiB data, 895 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 255 B/s wr, 0 op/s 2024-09-14T04:33:01.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:01 smithi039 ceph-mon[106470]: osdmap e1169: 8 total, 8 up, 8 in 2024-09-14T04:33:01.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:01 smithi039 ceph-mon[111235]: pgmap v1839: 105 pgs: 7 creating+peering, 2 unknown, 96 active+clean; 582 KiB data, 895 MiB used, 714 GiB / 715 GiB avail; 511 B/s rd, 255 B/s wr, 0 op/s 2024-09-14T04:33:01.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:01 smithi039 ceph-mon[111235]: osdmap e1169: 8 total, 8 up, 8 in 2024-09-14T04:33:02.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:02 smithi129 ceph-mon[101316]: osdmap e1170: 8 total, 8 up, 8 in 2024-09-14T04:33:02.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:02 smithi039 ceph-mon[106470]: osdmap e1170: 8 total, 8 up, 8 in 2024-09-14T04:33:02.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:02 smithi039 ceph-mon[111235]: osdmap e1170: 8 total, 8 up, 8 in 2024-09-14T04:33:03.294 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:33:03.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:03 smithi039 ceph-mon[106470]: pgmap v1842: 105 pgs: 7 creating+peering, 98 active+clean; 2.3 MiB data, 696 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 24 KiB/s wr, 0 op/s 2024-09-14T04:33:03.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:03 smithi039 ceph-mon[106470]: osdmap e1171: 8 total, 8 up, 8 in 2024-09-14T04:33:03.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:03 smithi039 ceph-mon[111235]: pgmap v1842: 105 pgs: 7 creating+peering, 98 active+clean; 2.3 MiB data, 696 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 24 KiB/s wr, 0 op/s 2024-09-14T04:33:03.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:03 smithi039 ceph-mon[111235]: osdmap e1171: 8 total, 8 up, 8 in 2024-09-14T04:33:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:03 smithi129 ceph-mon[101316]: pgmap v1842: 105 pgs: 7 creating+peering, 98 active+clean; 2.3 MiB data, 696 MiB used, 715 GiB / 715 GiB avail; 511 B/s rd, 24 KiB/s wr, 0 op/s 2024-09-14T04:33:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:03 smithi129 ceph-mon[101316]: osdmap e1171: 8 total, 8 up, 8 in 2024-09-14T04:33:04.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:04 smithi039 ceph-mon[106470]: osdmap e1172: 8 total, 8 up, 8 in 2024-09-14T04:33:04.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:04 smithi039 ceph-mon[111235]: osdmap e1172: 8 total, 8 up, 8 in 2024-09-14T04:33:04.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:04 smithi129 ceph-mon[101316]: osdmap e1172: 8 total, 8 up, 8 in 2024-09-14T04:33:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:05 smithi129 ceph-mon[101316]: pgmap v1845: 105 pgs: 105 active+clean; 14 MiB data, 699 MiB used, 715 GiB / 715 GiB avail; 41 KiB/s rd, 268 KiB/s wr, 67 op/s 2024-09-14T04:33:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:05 smithi129 ceph-mon[101316]: osdmap e1173: 8 total, 8 up, 8 in 2024-09-14T04:33:05.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:05 smithi039 ceph-mon[106470]: pgmap v1845: 105 pgs: 105 active+clean; 14 MiB data, 699 MiB used, 715 GiB / 715 GiB avail; 41 KiB/s rd, 268 KiB/s wr, 67 op/s 2024-09-14T04:33:05.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:05 smithi039 ceph-mon[106470]: osdmap e1173: 8 total, 8 up, 8 in 2024-09-14T04:33:05.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:05 smithi039 ceph-mon[111235]: pgmap v1845: 105 pgs: 105 active+clean; 14 MiB data, 699 MiB used, 715 GiB / 715 GiB avail; 41 KiB/s rd, 268 KiB/s wr, 67 op/s 2024-09-14T04:33:05.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:05 smithi039 ceph-mon[111235]: osdmap e1173: 8 total, 8 up, 8 in 2024-09-14T04:33:05.834 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:33:06.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:06 smithi129 ceph-mon[101316]: osdmap e1174: 8 total, 8 up, 8 in 2024-09-14T04:33:06.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:06 smithi129 ceph-mon[101316]: osdmap e1175: 8 total, 8 up, 8 in 2024-09-14T04:33:06.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:06 smithi039 ceph-mon[111235]: osdmap e1174: 8 total, 8 up, 8 in 2024-09-14T04:33:06.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:06 smithi039 ceph-mon[111235]: osdmap e1175: 8 total, 8 up, 8 in 2024-09-14T04:33:06.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:06 smithi039 ceph-mon[106470]: osdmap e1174: 8 total, 8 up, 8 in 2024-09-14T04:33:06.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:06 smithi039 ceph-mon[106470]: osdmap e1175: 8 total, 8 up, 8 in 2024-09-14T04:33:07.691 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:07 smithi039 ceph-mon[106470]: pgmap v1848: 105 pgs: 105 active+clean; 15 MiB data, 700 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 282 KiB/s wr, 85 op/s 2024-09-14T04:33:07.691 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:07 smithi039 ceph-mon[106470]: osdmap e1176: 8 total, 8 up, 8 in 2024-09-14T04:33:07.692 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:33:07.692 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:07 smithi039 ceph-mon[111235]: pgmap v1848: 105 pgs: 105 active+clean; 15 MiB data, 700 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 282 KiB/s wr, 85 op/s 2024-09-14T04:33:07.692 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:07 smithi039 ceph-mon[111235]: osdmap e1176: 8 total, 8 up, 8 in 2024-09-14T04:33:07.692 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:33:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:07 smithi129 ceph-mon[101316]: pgmap v1848: 105 pgs: 105 active+clean; 15 MiB data, 700 MiB used, 715 GiB / 715 GiB avail; 54 KiB/s rd, 282 KiB/s wr, 85 op/s 2024-09-14T04:33:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:07 smithi129 ceph-mon[101316]: osdmap e1176: 8 total, 8 up, 8 in 2024-09-14T04:33:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:33:08.865 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:33:08.990 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:08 smithi039 ceph-mon[106470]: pgmap v1851: 105 pgs: 105 active+clean; 16 MiB data, 700 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 39 KiB/s wr, 50 op/s 2024-09-14T04:33:08.990 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:08 smithi039 ceph-mon[106470]: osdmap e1177: 8 total, 8 up, 8 in 2024-09-14T04:33:08.990 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:33:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:33:08] "GET /metrics HTTP/1.1" 200 36675 "" "Prometheus/2.51.0" 2024-09-14T04:33:08.991 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:08 smithi039 ceph-mon[111235]: pgmap v1851: 105 pgs: 105 active+clean; 16 MiB data, 700 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 39 KiB/s wr, 50 op/s 2024-09-14T04:33:08.991 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:08 smithi039 ceph-mon[111235]: osdmap e1177: 8 total, 8 up, 8 in 2024-09-14T04:33:09.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:08 smithi129 ceph-mon[101316]: pgmap v1851: 105 pgs: 105 active+clean; 16 MiB data, 700 MiB used, 715 GiB / 715 GiB avail; 36 KiB/s rd, 39 KiB/s wr, 50 op/s 2024-09-14T04:33:09.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:08 smithi129 ceph-mon[101316]: osdmap e1177: 8 total, 8 up, 8 in 2024-09-14T04:33:10.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:09 smithi129 ceph-mon[101316]: osdmap e1178: 8 total, 8 up, 8 in 2024-09-14T04:33:10.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:09 smithi039 ceph-mon[106470]: osdmap e1178: 8 total, 8 up, 8 in 2024-09-14T04:33:10.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:09 smithi039 ceph-mon[111235]: osdmap e1178: 8 total, 8 up, 8 in 2024-09-14T04:33:11.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:10 smithi129 ceph-mon[101316]: pgmap v1854: 105 pgs: 105 active+clean; 19 MiB data, 701 MiB used, 715 GiB / 715 GiB avail; 56 KiB/s rd, 7.5 KiB/s wr, 87 op/s 2024-09-14T04:33:11.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:10 smithi129 ceph-mon[101316]: osdmap e1179: 8 total, 8 up, 8 in 2024-09-14T04:33:11.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:10 smithi039 ceph-mon[106470]: pgmap v1854: 105 pgs: 105 active+clean; 19 MiB data, 701 MiB used, 715 GiB / 715 GiB avail; 56 KiB/s rd, 7.5 KiB/s wr, 87 op/s 2024-09-14T04:33:11.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:10 smithi039 ceph-mon[106470]: osdmap e1179: 8 total, 8 up, 8 in 2024-09-14T04:33:11.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:10 smithi039 ceph-mon[111235]: pgmap v1854: 105 pgs: 105 active+clean; 19 MiB data, 701 MiB used, 715 GiB / 715 GiB avail; 56 KiB/s rd, 7.5 KiB/s wr, 87 op/s 2024-09-14T04:33:11.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:10 smithi039 ceph-mon[111235]: osdmap e1179: 8 total, 8 up, 8 in 2024-09-14T04:33:11.906 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:33:12.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:11 smithi129 ceph-mon[101316]: osdmap e1180: 8 total, 8 up, 8 in 2024-09-14T04:33:12.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:11 smithi039 ceph-mon[106470]: osdmap e1180: 8 total, 8 up, 8 in 2024-09-14T04:33:12.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:11 smithi039 ceph-mon[111235]: osdmap e1180: 8 total, 8 up, 8 in 2024-09-14T04:33:13.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:12 smithi129 ceph-mon[101316]: pgmap v1857: 105 pgs: 105 active+clean; 19 MiB data, 701 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 7.5 KiB/s wr, 81 op/s 2024-09-14T04:33:13.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:12 smithi129 ceph-mon[101316]: osdmap e1181: 8 total, 8 up, 8 in 2024-09-14T04:33:13.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:12 smithi039 ceph-mon[106470]: pgmap v1857: 105 pgs: 105 active+clean; 19 MiB data, 701 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 7.5 KiB/s wr, 81 op/s 2024-09-14T04:33:13.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:12 smithi039 ceph-mon[106470]: osdmap e1181: 8 total, 8 up, 8 in 2024-09-14T04:33:13.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:12 smithi039 ceph-mon[111235]: pgmap v1857: 105 pgs: 105 active+clean; 19 MiB data, 701 MiB used, 715 GiB / 715 GiB avail; 52 KiB/s rd, 7.5 KiB/s wr, 81 op/s 2024-09-14T04:33:13.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:12 smithi039 ceph-mon[111235]: osdmap e1181: 8 total, 8 up, 8 in 2024-09-14T04:33:14.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:13 smithi129 ceph-mon[101316]: osdmap e1182: 8 total, 8 up, 8 in 2024-09-14T04:33:14.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:13 smithi039 ceph-mon[106470]: osdmap e1182: 8 total, 8 up, 8 in 2024-09-14T04:33:14.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:13 smithi039 ceph-mon[111235]: osdmap e1182: 8 total, 8 up, 8 in 2024-09-14T04:33:14.944 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/1.DiffIterateDeterministic (14694 ms) 2024-09-14T04:33:14.944 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/1.DiffIterateDeterministicPP 2024-09-14T04:33:14.944 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:33:15.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:14 smithi129 ceph-mon[101316]: pgmap v1860: 105 pgs: 105 active+clean; 22 MiB data, 701 MiB used, 715 GiB / 715 GiB avail; 56 KiB/s rd, 7.5 KiB/s wr, 85 op/s 2024-09-14T04:33:15.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:14 smithi129 ceph-mon[101316]: osdmap e1183: 8 total, 8 up, 8 in 2024-09-14T04:33:15.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:14 smithi039 ceph-mon[106470]: pgmap v1860: 105 pgs: 105 active+clean; 22 MiB data, 701 MiB used, 715 GiB / 715 GiB avail; 56 KiB/s rd, 7.5 KiB/s wr, 85 op/s 2024-09-14T04:33:15.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:14 smithi039 ceph-mon[106470]: osdmap e1183: 8 total, 8 up, 8 in 2024-09-14T04:33:15.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:14 smithi039 ceph-mon[111235]: pgmap v1860: 105 pgs: 105 active+clean; 22 MiB data, 701 MiB used, 715 GiB / 715 GiB avail; 56 KiB/s rd, 7.5 KiB/s wr, 85 op/s 2024-09-14T04:33:15.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:14 smithi039 ceph-mon[111235]: osdmap e1183: 8 total, 8 up, 8 in 2024-09-14T04:33:16.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:15 smithi129 ceph-mon[101316]: osdmap e1184: 8 total, 8 up, 8 in 2024-09-14T04:33:16.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:15 smithi039 ceph-mon[106470]: osdmap e1184: 8 total, 8 up, 8 in 2024-09-14T04:33:16.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:15 smithi039 ceph-mon[111235]: osdmap e1184: 8 total, 8 up, 8 in 2024-09-14T04:33:17.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:16 smithi129 ceph-mon[101316]: pgmap v1863: 105 pgs: 105 active+clean; 30 MiB data, 702 MiB used, 715 GiB / 715 GiB avail; 62 KiB/s rd, 8.5 KiB/s wr, 91 op/s 2024-09-14T04:33:17.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:16 smithi129 ceph-mon[101316]: osdmap e1185: 8 total, 8 up, 8 in 2024-09-14T04:33:17.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:16 smithi039 ceph-mon[111235]: pgmap v1863: 105 pgs: 105 active+clean; 30 MiB data, 702 MiB used, 715 GiB / 715 GiB avail; 62 KiB/s rd, 8.5 KiB/s wr, 91 op/s 2024-09-14T04:33:17.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:16 smithi039 ceph-mon[111235]: osdmap e1185: 8 total, 8 up, 8 in 2024-09-14T04:33:17.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:16 smithi039 ceph-mon[106470]: pgmap v1863: 105 pgs: 105 active+clean; 30 MiB data, 702 MiB used, 715 GiB / 715 GiB avail; 62 KiB/s rd, 8.5 KiB/s wr, 91 op/s 2024-09-14T04:33:17.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:16 smithi039 ceph-mon[106470]: osdmap e1185: 8 total, 8 up, 8 in 2024-09-14T04:33:17.984 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:33:18.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:17 smithi129 ceph-mon[101316]: osdmap e1186: 8 total, 8 up, 8 in 2024-09-14T04:33:18.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:17 smithi039 ceph-mon[106470]: osdmap e1186: 8 total, 8 up, 8 in 2024-09-14T04:33:18.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:17 smithi039 ceph-mon[111235]: osdmap e1186: 8 total, 8 up, 8 in 2024-09-14T04:33:18.950 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:33:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:33:18] "GET /metrics HTTP/1.1" 200 36675 "" "Prometheus/2.51.0" 2024-09-14T04:33:19.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:18 smithi129 ceph-mon[101316]: pgmap v1866: 105 pgs: 105 active+clean; 30 MiB data, 702 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 3.2 KiB/s wr, 45 op/s 2024-09-14T04:33:19.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:18 smithi129 ceph-mon[101316]: osdmap e1187: 8 total, 8 up, 8 in 2024-09-14T04:33:19.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:18 smithi039 ceph-mon[111235]: pgmap v1866: 105 pgs: 105 active+clean; 30 MiB data, 702 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 3.2 KiB/s wr, 45 op/s 2024-09-14T04:33:19.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:18 smithi039 ceph-mon[111235]: osdmap e1187: 8 total, 8 up, 8 in 2024-09-14T04:33:19.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:18 smithi039 ceph-mon[106470]: pgmap v1866: 105 pgs: 105 active+clean; 30 MiB data, 702 MiB used, 715 GiB / 715 GiB avail; 31 KiB/s rd, 3.2 KiB/s wr, 45 op/s 2024-09-14T04:33:19.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:18 smithi039 ceph-mon[106470]: osdmap e1187: 8 total, 8 up, 8 in 2024-09-14T04:33:20.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:19 smithi129 ceph-mon[101316]: osdmap e1188: 8 total, 8 up, 8 in 2024-09-14T04:33:20.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:19 smithi039 ceph-mon[106470]: osdmap e1188: 8 total, 8 up, 8 in 2024-09-14T04:33:20.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:19 smithi039 ceph-mon[111235]: osdmap e1188: 8 total, 8 up, 8 in 2024-09-14T04:33:21.020 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:33:21.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:20 smithi129 ceph-mon[101316]: pgmap v1869: 105 pgs: 105 active+clean; 30 MiB data, 702 MiB used, 715 GiB / 715 GiB avail; 61 KiB/s rd, 8.5 KiB/s wr, 89 op/s 2024-09-14T04:33:21.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:20 smithi129 ceph-mon[101316]: osdmap e1189: 8 total, 8 up, 8 in 2024-09-14T04:33:21.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:20 smithi039 ceph-mon[106470]: pgmap v1869: 105 pgs: 105 active+clean; 30 MiB data, 702 MiB used, 715 GiB / 715 GiB avail; 61 KiB/s rd, 8.5 KiB/s wr, 89 op/s 2024-09-14T04:33:21.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:20 smithi039 ceph-mon[106470]: osdmap e1189: 8 total, 8 up, 8 in 2024-09-14T04:33:21.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:20 smithi039 ceph-mon[111235]: pgmap v1869: 105 pgs: 105 active+clean; 30 MiB data, 702 MiB used, 715 GiB / 715 GiB avail; 61 KiB/s rd, 8.5 KiB/s wr, 89 op/s 2024-09-14T04:33:21.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:20 smithi039 ceph-mon[111235]: osdmap e1189: 8 total, 8 up, 8 in 2024-09-14T04:33:22.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:22 smithi129 ceph-mon[101316]: osdmap e1190: 8 total, 8 up, 8 in 2024-09-14T04:33:22.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:22 smithi039 ceph-mon[106470]: osdmap e1190: 8 total, 8 up, 8 in 2024-09-14T04:33:22.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:22 smithi039 ceph-mon[111235]: osdmap e1190: 8 total, 8 up, 8 in 2024-09-14T04:33:23.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:23 smithi129 ceph-mon[101316]: pgmap v1872: 105 pgs: 105 active+clean; 30 MiB data, 702 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 7.7 KiB/s wr, 77 op/s 2024-09-14T04:33:23.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:23 smithi129 ceph-mon[101316]: osdmap e1191: 8 total, 8 up, 8 in 2024-09-14T04:33:23.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:33:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:23 smithi039 ceph-mon[106470]: pgmap v1872: 105 pgs: 105 active+clean; 30 MiB data, 702 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 7.7 KiB/s wr, 77 op/s 2024-09-14T04:33:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:23 smithi039 ceph-mon[106470]: osdmap e1191: 8 total, 8 up, 8 in 2024-09-14T04:33:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:33:23.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:23 smithi039 ceph-mon[111235]: pgmap v1872: 105 pgs: 105 active+clean; 30 MiB data, 702 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s rd, 7.7 KiB/s wr, 77 op/s 2024-09-14T04:33:23.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:23 smithi039 ceph-mon[111235]: osdmap e1191: 8 total, 8 up, 8 in 2024-09-14T04:33:23.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:33:24.080 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:33:24.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:24 smithi039 ceph-mon[106470]: osdmap e1192: 8 total, 8 up, 8 in 2024-09-14T04:33:24.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:24 smithi039 ceph-mon[111235]: osdmap e1192: 8 total, 8 up, 8 in 2024-09-14T04:33:24.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:24 smithi129 ceph-mon[101316]: osdmap e1192: 8 total, 8 up, 8 in 2024-09-14T04:33:25.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:25 smithi129 ceph-mon[101316]: pgmap v1875: 105 pgs: 105 active+clean; 39 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 56 KiB/s rd, 6.7 KiB/s wr, 85 op/s 2024-09-14T04:33:25.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:25 smithi129 ceph-mon[101316]: osdmap e1193: 8 total, 8 up, 8 in 2024-09-14T04:33:25.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:25 smithi039 ceph-mon[106470]: pgmap v1875: 105 pgs: 105 active+clean; 39 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 56 KiB/s rd, 6.7 KiB/s wr, 85 op/s 2024-09-14T04:33:25.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:25 smithi039 ceph-mon[106470]: osdmap e1193: 8 total, 8 up, 8 in 2024-09-14T04:33:25.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:25 smithi039 ceph-mon[111235]: pgmap v1875: 105 pgs: 105 active+clean; 39 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 56 KiB/s rd, 6.7 KiB/s wr, 85 op/s 2024-09-14T04:33:25.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:25 smithi039 ceph-mon[111235]: osdmap e1193: 8 total, 8 up, 8 in 2024-09-14T04:33:26.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:26 smithi129 ceph-mon[101316]: osdmap e1194: 8 total, 8 up, 8 in 2024-09-14T04:33:26.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:26 smithi039 ceph-mon[106470]: osdmap e1194: 8 total, 8 up, 8 in 2024-09-14T04:33:26.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:26 smithi039 ceph-mon[111235]: osdmap e1194: 8 total, 8 up, 8 in 2024-09-14T04:33:27.098 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:33:27.422 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:27 smithi129 ceph-mon[101316]: pgmap v1878: 105 pgs: 105 active+clean; 39 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s rd, 11 KiB/s wr, 136 op/s 2024-09-14T04:33:27.422 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:27 smithi129 ceph-mon[101316]: osdmap e1195: 8 total, 8 up, 8 in 2024-09-14T04:33:27.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:27 smithi039 ceph-mon[106470]: pgmap v1878: 105 pgs: 105 active+clean; 39 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s rd, 11 KiB/s wr, 136 op/s 2024-09-14T04:33:27.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:27 smithi039 ceph-mon[106470]: osdmap e1195: 8 total, 8 up, 8 in 2024-09-14T04:33:27.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:27 smithi039 ceph-mon[111235]: pgmap v1878: 105 pgs: 105 active+clean; 39 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 92 KiB/s rd, 11 KiB/s wr, 136 op/s 2024-09-14T04:33:27.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:27 smithi039 ceph-mon[111235]: osdmap e1195: 8 total, 8 up, 8 in 2024-09-14T04:33:28.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:28 smithi129 ceph-mon[101316]: osdmap e1196: 8 total, 8 up, 8 in 2024-09-14T04:33:28.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:28 smithi039 ceph-mon[106470]: osdmap e1196: 8 total, 8 up, 8 in 2024-09-14T04:33:28.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:28 smithi039 ceph-mon[111235]: osdmap e1196: 8 total, 8 up, 8 in 2024-09-14T04:33:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:33:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:33:28] "GET /metrics HTTP/1.1" 200 36675 "" "Prometheus/2.51.0" 2024-09-14T04:33:29.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:29 smithi129 ceph-mon[101316]: pgmap v1881: 105 pgs: 105 active+clean; 45 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 39 KiB/s rd, 5.5 KiB/s wr, 57 op/s 2024-09-14T04:33:29.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:29 smithi129 ceph-mon[101316]: osdmap e1197: 8 total, 8 up, 8 in 2024-09-14T04:33:29.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:29 smithi039 ceph-mon[111235]: pgmap v1881: 105 pgs: 105 active+clean; 45 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 39 KiB/s rd, 5.5 KiB/s wr, 57 op/s 2024-09-14T04:33:29.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:29 smithi039 ceph-mon[111235]: osdmap e1197: 8 total, 8 up, 8 in 2024-09-14T04:33:29.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:29 smithi039 ceph-mon[106470]: pgmap v1881: 105 pgs: 105 active+clean; 45 MiB data, 703 MiB used, 715 GiB / 715 GiB avail; 39 KiB/s rd, 5.5 KiB/s wr, 57 op/s 2024-09-14T04:33:29.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:29 smithi039 ceph-mon[106470]: osdmap e1197: 8 total, 8 up, 8 in 2024-09-14T04:33:30.134 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/1.DiffIterateDeterministicPP (15189 ms) 2024-09-14T04:33:30.134 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/1.DiffIterateDeterministicLUKS1 2024-09-14T04:33:30.134 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:33:30.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:30 smithi129 ceph-mon[101316]: osdmap e1198: 8 total, 8 up, 8 in 2024-09-14T04:33:30.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:30 smithi039 ceph-mon[106470]: osdmap e1198: 8 total, 8 up, 8 in 2024-09-14T04:33:30.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:30 smithi039 ceph-mon[111235]: osdmap e1198: 8 total, 8 up, 8 in 2024-09-14T04:33:31.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:31 smithi129 ceph-mon[101316]: pgmap v1884: 105 pgs: 105 active+clean; 52 MiB data, 704 MiB used, 715 GiB / 715 GiB avail; 49 KiB/s rd, 6.5 KiB/s wr, 73 op/s 2024-09-14T04:33:31.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:31 smithi129 ceph-mon[101316]: osdmap e1199: 8 total, 8 up, 8 in 2024-09-14T04:33:31.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:31 smithi039 ceph-mon[106470]: pgmap v1884: 105 pgs: 105 active+clean; 52 MiB data, 704 MiB used, 715 GiB / 715 GiB avail; 49 KiB/s rd, 6.5 KiB/s wr, 73 op/s 2024-09-14T04:33:31.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:31 smithi039 ceph-mon[106470]: osdmap e1199: 8 total, 8 up, 8 in 2024-09-14T04:33:31.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:31 smithi039 ceph-mon[111235]: pgmap v1884: 105 pgs: 105 active+clean; 52 MiB data, 704 MiB used, 715 GiB / 715 GiB avail; 49 KiB/s rd, 6.5 KiB/s wr, 73 op/s 2024-09-14T04:33:31.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:31 smithi039 ceph-mon[111235]: osdmap e1199: 8 total, 8 up, 8 in 2024-09-14T04:33:33.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:33 smithi129 ceph-mon[101316]: pgmap v1886: 105 pgs: 105 active+clean; 52 MiB data, 704 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 4.8 KiB/s wr, 61 op/s 2024-09-14T04:33:33.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:33 smithi039 ceph-mon[106470]: pgmap v1886: 105 pgs: 105 active+clean; 52 MiB data, 704 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 4.8 KiB/s wr, 61 op/s 2024-09-14T04:33:33.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:33 smithi039 ceph-mon[111235]: pgmap v1886: 105 pgs: 105 active+clean; 52 MiB data, 704 MiB used, 715 GiB / 715 GiB avail; 42 KiB/s rd, 4.8 KiB/s wr, 61 op/s 2024-09-14T04:33:35.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:35 smithi129 ceph-mon[101316]: pgmap v1887: 105 pgs: 105 active+clean; 56 MiB data, 704 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s rd, 6.8 KiB/s wr, 86 op/s 2024-09-14T04:33:35.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:35 smithi129 ceph-mon[101316]: osdmap e1200: 8 total, 8 up, 8 in 2024-09-14T04:33:35.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:35 smithi039 ceph-mon[106470]: pgmap v1887: 105 pgs: 105 active+clean; 56 MiB data, 704 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s rd, 6.8 KiB/s wr, 86 op/s 2024-09-14T04:33:35.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:35 smithi039 ceph-mon[106470]: osdmap e1200: 8 total, 8 up, 8 in 2024-09-14T04:33:35.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:35 smithi039 ceph-mon[111235]: pgmap v1887: 105 pgs: 105 active+clean; 56 MiB data, 704 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s rd, 6.8 KiB/s wr, 86 op/s 2024-09-14T04:33:35.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:35 smithi039 ceph-mon[111235]: osdmap e1200: 8 total, 8 up, 8 in 2024-09-14T04:33:36.196 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:33:36.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:36 smithi129 ceph-mon[101316]: osdmap e1201: 8 total, 8 up, 8 in 2024-09-14T04:33:36.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:36 smithi039 ceph-mon[106470]: osdmap e1201: 8 total, 8 up, 8 in 2024-09-14T04:33:36.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:36 smithi039 ceph-mon[111235]: osdmap e1201: 8 total, 8 up, 8 in 2024-09-14T04:33:37.441 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:37 smithi039 ceph-mon[106470]: pgmap v1890: 105 pgs: 105 active+clean; 56 MiB data, 704 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 3.5 KiB/s wr, 42 op/s 2024-09-14T04:33:37.441 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:37 smithi039 ceph-mon[106470]: osdmap e1202: 8 total, 8 up, 8 in 2024-09-14T04:33:37.441 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:37 smithi039 ceph-mon[111235]: pgmap v1890: 105 pgs: 105 active+clean; 56 MiB data, 704 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 3.5 KiB/s wr, 42 op/s 2024-09-14T04:33:37.441 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:37 smithi039 ceph-mon[111235]: osdmap e1202: 8 total, 8 up, 8 in 2024-09-14T04:33:37.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:37 smithi129 ceph-mon[101316]: pgmap v1890: 105 pgs: 105 active+clean; 56 MiB data, 704 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 3.5 KiB/s wr, 42 op/s 2024-09-14T04:33:37.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:37 smithi129 ceph-mon[101316]: osdmap e1202: 8 total, 8 up, 8 in 2024-09-14T04:33:38.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:38 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:33:38.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:38 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:33:38.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:38 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:33:39.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:33:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:33:38] "GET /metrics HTTP/1.1" 200 36671 "" "Prometheus/2.51.0" 2024-09-14T04:33:39.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:39 smithi129 ceph-mon[101316]: pgmap v1892: 105 pgs: 105 active+clean; 56 MiB data, 708 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 5.2 KiB/s wr, 54 op/s 2024-09-14T04:33:39.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:39 smithi039 ceph-mon[106470]: pgmap v1892: 105 pgs: 105 active+clean; 56 MiB data, 708 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 5.2 KiB/s wr, 54 op/s 2024-09-14T04:33:39.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:39 smithi039 ceph-mon[111235]: pgmap v1892: 105 pgs: 105 active+clean; 56 MiB data, 708 MiB used, 715 GiB / 715 GiB avail; 35 KiB/s rd, 5.2 KiB/s wr, 54 op/s 2024-09-14T04:33:41.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:41 smithi129 ceph-mon[101316]: pgmap v1893: 105 pgs: 105 active+clean; 60 MiB data, 717 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 687 KiB/s wr, 42 op/s 2024-09-14T04:33:41.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:41 smithi129 ceph-mon[101316]: osdmap e1203: 8 total, 8 up, 8 in 2024-09-14T04:33:41.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:41 smithi039 ceph-mon[106470]: pgmap v1893: 105 pgs: 105 active+clean; 60 MiB data, 717 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 687 KiB/s wr, 42 op/s 2024-09-14T04:33:41.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:41 smithi039 ceph-mon[106470]: osdmap e1203: 8 total, 8 up, 8 in 2024-09-14T04:33:41.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:41 smithi039 ceph-mon[111235]: pgmap v1893: 105 pgs: 105 active+clean; 60 MiB data, 717 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 687 KiB/s wr, 42 op/s 2024-09-14T04:33:41.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:41 smithi039 ceph-mon[111235]: osdmap e1203: 8 total, 8 up, 8 in 2024-09-14T04:33:42.245 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:33:42.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:42 smithi129 ceph-mon[101316]: osdmap e1204: 8 total, 8 up, 8 in 2024-09-14T04:33:42.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:42 smithi039 ceph-mon[106470]: osdmap e1204: 8 total, 8 up, 8 in 2024-09-14T04:33:42.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:42 smithi039 ceph-mon[111235]: osdmap e1204: 8 total, 8 up, 8 in 2024-09-14T04:33:43.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:43 smithi129 ceph-mon[101316]: pgmap v1896: 105 pgs: 105 active+clean; 60 MiB data, 717 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 687 KiB/s wr, 41 op/s 2024-09-14T04:33:43.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:43 smithi129 ceph-mon[101316]: osdmap e1205: 8 total, 8 up, 8 in 2024-09-14T04:33:43.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:43 smithi039 ceph-mon[106470]: pgmap v1896: 105 pgs: 105 active+clean; 60 MiB data, 717 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 687 KiB/s wr, 41 op/s 2024-09-14T04:33:43.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:43 smithi039 ceph-mon[106470]: osdmap e1205: 8 total, 8 up, 8 in 2024-09-14T04:33:43.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:43 smithi039 ceph-mon[111235]: pgmap v1896: 105 pgs: 105 active+clean; 60 MiB data, 717 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 687 KiB/s wr, 41 op/s 2024-09-14T04:33:43.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:43 smithi039 ceph-mon[111235]: osdmap e1205: 8 total, 8 up, 8 in 2024-09-14T04:33:45.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:45 smithi129 ceph-mon[101316]: pgmap v1898: 105 pgs: 105 active+clean; 62 MiB data, 721 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 688 KiB/s wr, 64 op/s 2024-09-14T04:33:45.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:45 smithi039 ceph-mon[106470]: pgmap v1898: 105 pgs: 105 active+clean; 62 MiB data, 721 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 688 KiB/s wr, 64 op/s 2024-09-14T04:33:45.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:45 smithi039 ceph-mon[111235]: pgmap v1898: 105 pgs: 105 active+clean; 62 MiB data, 721 MiB used, 715 GiB / 715 GiB avail; 43 KiB/s rd, 688 KiB/s wr, 64 op/s 2024-09-14T04:33:47.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:47 smithi129 ceph-mon[101316]: pgmap v1899: 105 pgs: 105 active+clean; 66 MiB data, 729 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 686 KiB/s wr, 38 op/s 2024-09-14T04:33:47.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:47 smithi129 ceph-mon[101316]: osdmap e1206: 8 total, 8 up, 8 in 2024-09-14T04:33:47.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:47 smithi039 ceph-mon[106470]: pgmap v1899: 105 pgs: 105 active+clean; 66 MiB data, 729 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 686 KiB/s wr, 38 op/s 2024-09-14T04:33:47.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:47 smithi039 ceph-mon[106470]: osdmap e1206: 8 total, 8 up, 8 in 2024-09-14T04:33:47.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:47 smithi039 ceph-mon[111235]: pgmap v1899: 105 pgs: 105 active+clean; 66 MiB data, 729 MiB used, 715 GiB / 715 GiB avail; 26 KiB/s rd, 686 KiB/s wr, 38 op/s 2024-09-14T04:33:47.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:47 smithi039 ceph-mon[111235]: osdmap e1206: 8 total, 8 up, 8 in 2024-09-14T04:33:48.273 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:33:48.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:48 smithi129 ceph-mon[101316]: osdmap e1207: 8 total, 8 up, 8 in 2024-09-14T04:33:48.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:48 smithi039 ceph-mon[106470]: osdmap e1207: 8 total, 8 up, 8 in 2024-09-14T04:33:48.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:48 smithi039 ceph-mon[111235]: osdmap e1207: 8 total, 8 up, 8 in 2024-09-14T04:33:49.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:33:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:33:48] "GET /metrics HTTP/1.1" 200 36671 "" "Prometheus/2.51.0" 2024-09-14T04:33:49.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:49 smithi129 ceph-mon[101316]: pgmap v1902: 105 pgs: 105 active+clean; 70 MiB data, 733 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 1.3 MiB/s wr, 42 op/s 2024-09-14T04:33:49.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:49 smithi129 ceph-mon[101316]: osdmap e1208: 8 total, 8 up, 8 in 2024-09-14T04:33:49.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:49 smithi039 ceph-mon[106470]: pgmap v1902: 105 pgs: 105 active+clean; 70 MiB data, 733 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 1.3 MiB/s wr, 42 op/s 2024-09-14T04:33:49.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:49 smithi039 ceph-mon[106470]: osdmap e1208: 8 total, 8 up, 8 in 2024-09-14T04:33:49.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:49 smithi039 ceph-mon[111235]: pgmap v1902: 105 pgs: 105 active+clean; 70 MiB data, 733 MiB used, 715 GiB / 715 GiB avail; 28 KiB/s rd, 1.3 MiB/s wr, 42 op/s 2024-09-14T04:33:49.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:49 smithi039 ceph-mon[111235]: osdmap e1208: 8 total, 8 up, 8 in 2024-09-14T04:33:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:51 smithi129 ceph-mon[101316]: pgmap v1904: 105 pgs: 105 active+clean; 72 MiB data, 741 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 1.3 MiB/s wr, 21 op/s 2024-09-14T04:33:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:51 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:33:51.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:51 smithi039 ceph-mon[106470]: pgmap v1904: 105 pgs: 105 active+clean; 72 MiB data, 741 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 1.3 MiB/s wr, 21 op/s 2024-09-14T04:33:51.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:51 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:33:51.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:51 smithi039 ceph-mon[111235]: pgmap v1904: 105 pgs: 105 active+clean; 72 MiB data, 741 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 1.3 MiB/s wr, 21 op/s 2024-09-14T04:33:51.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:51 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:33:53.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:53 smithi039 ceph-mon[106470]: pgmap v1905: 105 pgs: 105 active+clean; 72 MiB data, 741 MiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 686 KiB/s wr, 36 op/s 2024-09-14T04:33:53.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:53 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:33:53.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:53 smithi039 ceph-mon[106470]: osdmap e1209: 8 total, 8 up, 8 in 2024-09-14T04:33:53.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:53 smithi039 ceph-mon[111235]: pgmap v1905: 105 pgs: 105 active+clean; 72 MiB data, 741 MiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 686 KiB/s wr, 36 op/s 2024-09-14T04:33:53.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:53 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:33:53.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:53 smithi039 ceph-mon[111235]: osdmap e1209: 8 total, 8 up, 8 in 2024-09-14T04:33:53.560 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:53 smithi129 ceph-mon[101316]: pgmap v1905: 105 pgs: 105 active+clean; 72 MiB data, 741 MiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 686 KiB/s wr, 36 op/s 2024-09-14T04:33:53.560 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:53 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:33:53.561 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:53 smithi129 ceph-mon[101316]: osdmap e1209: 8 total, 8 up, 8 in 2024-09-14T04:33:54.331 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:33:54.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:54 smithi129 ceph-mon[101316]: osdmap e1210: 8 total, 8 up, 8 in 2024-09-14T04:33:54.601 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:54 smithi039 ceph-mon[106470]: osdmap e1210: 8 total, 8 up, 8 in 2024-09-14T04:33:54.602 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:54 smithi039 ceph-mon[111235]: osdmap e1210: 8 total, 8 up, 8 in 2024-09-14T04:33:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:55 smithi129 ceph-mon[101316]: pgmap v1908: 105 pgs: 105 active+clean; 73 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 5.0 KiB/s wr, 45 op/s 2024-09-14T04:33:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:55 smithi129 ceph-mon[101316]: osdmap e1211: 8 total, 8 up, 8 in 2024-09-14T04:33:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:33:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:33:55.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:55 smithi039 ceph-mon[111235]: pgmap v1908: 105 pgs: 105 active+clean; 73 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 5.0 KiB/s wr, 45 op/s 2024-09-14T04:33:55.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:55 smithi039 ceph-mon[111235]: osdmap e1211: 8 total, 8 up, 8 in 2024-09-14T04:33:55.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:33:55.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:33:55.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:55 smithi039 ceph-mon[106470]: pgmap v1908: 105 pgs: 105 active+clean; 73 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 5.0 KiB/s wr, 45 op/s 2024-09-14T04:33:55.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:55 smithi039 ceph-mon[106470]: osdmap e1211: 8 total, 8 up, 8 in 2024-09-14T04:33:55.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:33:55.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:33:56.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:56 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:33:56.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:56 smithi129 ceph-mon[101316]: pgmap v1910: 105 pgs: 105 active+clean; 73 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 42 KiB/s rd, 4.2 KiB/s wr, 60 op/s 2024-09-14T04:33:56.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:56 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:33:56.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:56 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:33:56.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:56 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:33:56.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:56 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:33:56.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:56 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:33:56.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:56 smithi039 ceph-mon[106470]: pgmap v1910: 105 pgs: 105 active+clean; 73 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 42 KiB/s rd, 4.2 KiB/s wr, 60 op/s 2024-09-14T04:33:56.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:56 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:33:56.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:56 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:33:56.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:56 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:33:56.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:56 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:33:56.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:56 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:33:56.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:56 smithi039 ceph-mon[111235]: pgmap v1910: 105 pgs: 105 active+clean; 73 MiB data, 750 MiB used, 714 GiB / 715 GiB avail; 42 KiB/s rd, 4.2 KiB/s wr, 60 op/s 2024-09-14T04:33:56.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:56 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:33:56.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:56 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:33:56.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:56 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:33:56.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:56 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:33:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:58 smithi129 ceph-mon[101316]: pgmap v1911: 105 pgs: 105 active+clean; 78 MiB data, 754 MiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 686 KiB/s wr, 41 op/s 2024-09-14T04:33:58.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:58 smithi039 ceph-mon[111235]: pgmap v1911: 105 pgs: 105 active+clean; 78 MiB data, 754 MiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 686 KiB/s wr, 41 op/s 2024-09-14T04:33:58.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:58 smithi039 ceph-mon[106470]: pgmap v1911: 105 pgs: 105 active+clean; 78 MiB data, 754 MiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 686 KiB/s wr, 41 op/s 2024-09-14T04:33:58.791 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:33:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:33:58] "GET /metrics HTTP/1.1" 200 36683 "" "Prometheus/2.51.0" 2024-09-14T04:33:59.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:33:59 smithi129 ceph-mon[101316]: osdmap e1212: 8 total, 8 up, 8 in 2024-09-14T04:33:59.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:33:59 smithi039 ceph-mon[106470]: osdmap e1212: 8 total, 8 up, 8 in 2024-09-14T04:33:59.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:33:59 smithi039 ceph-mon[111235]: osdmap e1212: 8 total, 8 up, 8 in 2024-09-14T04:34:00.508 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/1.DiffIterateDeterministicLUKS1 (30375 ms) 2024-09-14T04:34:00.508 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/1.DiffIterateDeterministicLUKS1PP 2024-09-14T04:34:00.508 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:34:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:00 smithi129 ceph-mon[101316]: pgmap v1913: 105 pgs: 105 active+clean; 78 MiB data, 754 MiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 671 KiB/s wr, 36 op/s 2024-09-14T04:34:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:00 smithi129 ceph-mon[101316]: osdmap e1213: 8 total, 8 up, 8 in 2024-09-14T04:34:00.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:00 smithi039 ceph-mon[106470]: pgmap v1913: 105 pgs: 105 active+clean; 78 MiB data, 754 MiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 671 KiB/s wr, 36 op/s 2024-09-14T04:34:00.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:00 smithi039 ceph-mon[106470]: osdmap e1213: 8 total, 8 up, 8 in 2024-09-14T04:34:00.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:00 smithi039 ceph-mon[111235]: pgmap v1913: 105 pgs: 105 active+clean; 78 MiB data, 754 MiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 671 KiB/s wr, 36 op/s 2024-09-14T04:34:00.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:00 smithi039 ceph-mon[111235]: osdmap e1213: 8 total, 8 up, 8 in 2024-09-14T04:34:01.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:01 smithi129 ceph-mon[101316]: osdmap e1214: 8 total, 8 up, 8 in 2024-09-14T04:34:01.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:01 smithi039 ceph-mon[106470]: osdmap e1214: 8 total, 8 up, 8 in 2024-09-14T04:34:01.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:01 smithi039 ceph-mon[111235]: osdmap e1214: 8 total, 8 up, 8 in 2024-09-14T04:34:02.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:02 smithi129 ceph-mon[101316]: pgmap v1916: 105 pgs: 105 active+clean; 86 MiB data, 758 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 1.3 MiB/s wr, 21 op/s 2024-09-14T04:34:02.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:02 smithi039 ceph-mon[106470]: pgmap v1916: 105 pgs: 105 active+clean; 86 MiB data, 758 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 1.3 MiB/s wr, 21 op/s 2024-09-14T04:34:02.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:02 smithi039 ceph-mon[111235]: pgmap v1916: 105 pgs: 105 active+clean; 86 MiB data, 758 MiB used, 714 GiB / 715 GiB avail; 14 KiB/s rd, 1.3 MiB/s wr, 21 op/s 2024-09-14T04:34:04.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:04 smithi129 ceph-mon[101316]: pgmap v1917: 105 pgs: 105 active+clean; 90 MiB data, 766 MiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 687 KiB/s wr, 38 op/s 2024-09-14T04:34:04.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:04 smithi039 ceph-mon[106470]: pgmap v1917: 105 pgs: 105 active+clean; 90 MiB data, 766 MiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 687 KiB/s wr, 38 op/s 2024-09-14T04:34:04.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:04 smithi039 ceph-mon[111235]: pgmap v1917: 105 pgs: 105 active+clean; 90 MiB data, 766 MiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 687 KiB/s wr, 38 op/s 2024-09-14T04:34:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:05 smithi129 ceph-mon[101316]: osdmap e1215: 8 total, 8 up, 8 in 2024-09-14T04:34:05.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:05 smithi039 ceph-mon[106470]: osdmap e1215: 8 total, 8 up, 8 in 2024-09-14T04:34:05.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:05 smithi039 ceph-mon[111235]: osdmap e1215: 8 total, 8 up, 8 in 2024-09-14T04:34:06.559 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:34:06.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:06 smithi129 ceph-mon[101316]: pgmap v1919: 105 pgs: 105 active+clean; 90 MiB data, 767 MiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 687 KiB/s wr, 43 op/s 2024-09-14T04:34:06.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:06 smithi129 ceph-mon[101316]: osdmap e1216: 8 total, 8 up, 8 in 2024-09-14T04:34:06.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:06 smithi039 ceph-mon[106470]: pgmap v1919: 105 pgs: 105 active+clean; 90 MiB data, 767 MiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 687 KiB/s wr, 43 op/s 2024-09-14T04:34:06.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:06 smithi039 ceph-mon[106470]: osdmap e1216: 8 total, 8 up, 8 in 2024-09-14T04:34:06.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:06 smithi039 ceph-mon[111235]: pgmap v1919: 105 pgs: 105 active+clean; 90 MiB data, 767 MiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 687 KiB/s wr, 43 op/s 2024-09-14T04:34:06.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:06 smithi039 ceph-mon[111235]: osdmap e1216: 8 total, 8 up, 8 in 2024-09-14T04:34:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:07 smithi039 ceph-mon[106470]: osdmap e1217: 8 total, 8 up, 8 in 2024-09-14T04:34:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:34:07.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:07 smithi039 ceph-mon[111235]: osdmap e1217: 8 total, 8 up, 8 in 2024-09-14T04:34:07.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:34:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:07 smithi129 ceph-mon[101316]: osdmap e1217: 8 total, 8 up, 8 in 2024-09-14T04:34:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:34:08.953 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:08 smithi039 ceph-mon[106470]: pgmap v1922: 105 pgs: 105 active+clean; 94 MiB data, 771 MiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 686 KiB/s wr, 36 op/s 2024-09-14T04:34:08.953 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:34:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:34:08] "GET /metrics HTTP/1.1" 200 36681 "" "Prometheus/2.51.0" 2024-09-14T04:34:08.953 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:08 smithi039 ceph-mon[111235]: pgmap v1922: 105 pgs: 105 active+clean; 94 MiB data, 771 MiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 686 KiB/s wr, 36 op/s 2024-09-14T04:34:09.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:08 smithi129 ceph-mon[101316]: pgmap v1922: 105 pgs: 105 active+clean; 94 MiB data, 771 MiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 686 KiB/s wr, 36 op/s 2024-09-14T04:34:11.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:10 smithi129 ceph-mon[101316]: pgmap v1923: 105 pgs: 105 active+clean; 94 MiB data, 779 MiB used, 714 GiB / 715 GiB avail; 35 KiB/s rd, 688 KiB/s wr, 53 op/s 2024-09-14T04:34:11.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:10 smithi039 ceph-mon[106470]: pgmap v1923: 105 pgs: 105 active+clean; 94 MiB data, 779 MiB used, 714 GiB / 715 GiB avail; 35 KiB/s rd, 688 KiB/s wr, 53 op/s 2024-09-14T04:34:11.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:10 smithi039 ceph-mon[111235]: pgmap v1923: 105 pgs: 105 active+clean; 94 MiB data, 779 MiB used, 714 GiB / 715 GiB avail; 35 KiB/s rd, 688 KiB/s wr, 53 op/s 2024-09-14T04:34:12.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:11 smithi129 ceph-mon[101316]: osdmap e1218: 8 total, 8 up, 8 in 2024-09-14T04:34:12.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:11 smithi039 ceph-mon[106470]: osdmap e1218: 8 total, 8 up, 8 in 2024-09-14T04:34:12.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:11 smithi039 ceph-mon[111235]: osdmap e1218: 8 total, 8 up, 8 in 2024-09-14T04:34:12.618 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:34:13.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:12 smithi129 ceph-mon[101316]: pgmap v1925: 105 pgs: 105 active+clean; 94 MiB data, 779 MiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 686 KiB/s wr, 39 op/s 2024-09-14T04:34:13.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:12 smithi129 ceph-mon[101316]: osdmap e1219: 8 total, 8 up, 8 in 2024-09-14T04:34:13.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:12 smithi039 ceph-mon[106470]: pgmap v1925: 105 pgs: 105 active+clean; 94 MiB data, 779 MiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 686 KiB/s wr, 39 op/s 2024-09-14T04:34:13.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:12 smithi039 ceph-mon[106470]: osdmap e1219: 8 total, 8 up, 8 in 2024-09-14T04:34:13.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:12 smithi039 ceph-mon[111235]: pgmap v1925: 105 pgs: 105 active+clean; 94 MiB data, 779 MiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 686 KiB/s wr, 39 op/s 2024-09-14T04:34:13.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:12 smithi039 ceph-mon[111235]: osdmap e1219: 8 total, 8 up, 8 in 2024-09-14T04:34:14.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:13 smithi129 ceph-mon[101316]: osdmap e1220: 8 total, 8 up, 8 in 2024-09-14T04:34:14.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:13 smithi039 ceph-mon[106470]: osdmap e1220: 8 total, 8 up, 8 in 2024-09-14T04:34:14.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:13 smithi039 ceph-mon[111235]: osdmap e1220: 8 total, 8 up, 8 in 2024-09-14T04:34:15.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:14 smithi129 ceph-mon[101316]: pgmap v1928: 105 pgs: 105 active+clean; 101 MiB data, 787 MiB used, 714 GiB / 715 GiB avail; 37 KiB/s rd, 689 KiB/s wr, 56 op/s 2024-09-14T04:34:15.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:14 smithi039 ceph-mon[106470]: pgmap v1928: 105 pgs: 105 active+clean; 101 MiB data, 787 MiB used, 714 GiB / 715 GiB avail; 37 KiB/s rd, 689 KiB/s wr, 56 op/s 2024-09-14T04:34:15.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:14 smithi039 ceph-mon[111235]: pgmap v1928: 105 pgs: 105 active+clean; 101 MiB data, 787 MiB used, 714 GiB / 715 GiB avail; 37 KiB/s rd, 689 KiB/s wr, 56 op/s 2024-09-14T04:34:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:16 smithi129 ceph-mon[101316]: pgmap v1929: 105 pgs: 105 active+clean; 104 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 686 KiB/s wr, 27 op/s 2024-09-14T04:34:17.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:16 smithi039 ceph-mon[106470]: pgmap v1929: 105 pgs: 105 active+clean; 104 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 686 KiB/s wr, 27 op/s 2024-09-14T04:34:17.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:16 smithi039 ceph-mon[111235]: pgmap v1929: 105 pgs: 105 active+clean; 104 MiB data, 791 MiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 686 KiB/s wr, 27 op/s 2024-09-14T04:34:18.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:17 smithi129 ceph-mon[101316]: osdmap e1221: 8 total, 8 up, 8 in 2024-09-14T04:34:18.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:17 smithi039 ceph-mon[106470]: osdmap e1221: 8 total, 8 up, 8 in 2024-09-14T04:34:18.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:17 smithi039 ceph-mon[111235]: osdmap e1221: 8 total, 8 up, 8 in 2024-09-14T04:34:18.678 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:34:18.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:18 smithi039 ceph-mon[106470]: pgmap v1931: 105 pgs: 105 active+clean; 108 MiB data, 795 MiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 1.3 MiB/s wr, 43 op/s 2024-09-14T04:34:18.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:18 smithi039 ceph-mon[106470]: osdmap e1222: 8 total, 8 up, 8 in 2024-09-14T04:34:18.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:18 smithi039 ceph-mon[111235]: pgmap v1931: 105 pgs: 105 active+clean; 108 MiB data, 795 MiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 1.3 MiB/s wr, 43 op/s 2024-09-14T04:34:18.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:18 smithi039 ceph-mon[111235]: osdmap e1222: 8 total, 8 up, 8 in 2024-09-14T04:34:18.791 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:34:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:34:18] "GET /metrics HTTP/1.1" 200 36681 "" "Prometheus/2.51.0" 2024-09-14T04:34:19.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:18 smithi129 ceph-mon[101316]: pgmap v1931: 105 pgs: 105 active+clean; 108 MiB data, 795 MiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 1.3 MiB/s wr, 43 op/s 2024-09-14T04:34:19.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:18 smithi129 ceph-mon[101316]: osdmap e1222: 8 total, 8 up, 8 in 2024-09-14T04:34:20.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:19 smithi129 ceph-mon[101316]: osdmap e1223: 8 total, 8 up, 8 in 2024-09-14T04:34:20.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:19 smithi039 ceph-mon[106470]: osdmap e1223: 8 total, 8 up, 8 in 2024-09-14T04:34:20.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:19 smithi039 ceph-mon[111235]: osdmap e1223: 8 total, 8 up, 8 in 2024-09-14T04:34:21.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:20 smithi129 ceph-mon[101316]: pgmap v1934: 105 pgs: 105 active+clean; 114 MiB data, 808 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 686 KiB/s wr, 29 op/s 2024-09-14T04:34:21.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:20 smithi039 ceph-mon[106470]: pgmap v1934: 105 pgs: 105 active+clean; 114 MiB data, 808 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 686 KiB/s wr, 29 op/s 2024-09-14T04:34:21.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:20 smithi039 ceph-mon[111235]: pgmap v1934: 105 pgs: 105 active+clean; 114 MiB data, 808 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 686 KiB/s wr, 29 op/s 2024-09-14T04:34:23.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:22 smithi129 ceph-mon[101316]: pgmap v1935: 105 pgs: 105 active+clean; 114 MiB data, 808 MiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 686 KiB/s wr, 45 op/s 2024-09-14T04:34:23.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:34:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:22 smithi039 ceph-mon[106470]: pgmap v1935: 105 pgs: 105 active+clean; 114 MiB data, 808 MiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 686 KiB/s wr, 45 op/s 2024-09-14T04:34:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:34:23.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:22 smithi039 ceph-mon[111235]: pgmap v1935: 105 pgs: 105 active+clean; 114 MiB data, 808 MiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 686 KiB/s wr, 45 op/s 2024-09-14T04:34:23.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:34:24.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:23 smithi129 ceph-mon[101316]: osdmap e1224: 8 total, 8 up, 8 in 2024-09-14T04:34:24.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:23 smithi039 ceph-mon[106470]: osdmap e1224: 8 total, 8 up, 8 in 2024-09-14T04:34:24.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:23 smithi039 ceph-mon[111235]: osdmap e1224: 8 total, 8 up, 8 in 2024-09-14T04:34:24.882 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:34:25.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:24 smithi129 ceph-mon[101316]: pgmap v1937: 105 pgs: 105 active+clean; 118 MiB data, 808 MiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 686 KiB/s wr, 40 op/s 2024-09-14T04:34:25.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:24 smithi129 ceph-mon[101316]: osdmap e1225: 8 total, 8 up, 8 in 2024-09-14T04:34:25.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:24 smithi039 ceph-mon[106470]: pgmap v1937: 105 pgs: 105 active+clean; 118 MiB data, 808 MiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 686 KiB/s wr, 40 op/s 2024-09-14T04:34:25.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:24 smithi039 ceph-mon[106470]: osdmap e1225: 8 total, 8 up, 8 in 2024-09-14T04:34:25.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:24 smithi039 ceph-mon[111235]: pgmap v1937: 105 pgs: 105 active+clean; 118 MiB data, 808 MiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 686 KiB/s wr, 40 op/s 2024-09-14T04:34:25.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:24 smithi039 ceph-mon[111235]: osdmap e1225: 8 total, 8 up, 8 in 2024-09-14T04:34:26.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:25 smithi129 ceph-mon[101316]: osdmap e1226: 8 total, 8 up, 8 in 2024-09-14T04:34:26.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:25 smithi039 ceph-mon[106470]: osdmap e1226: 8 total, 8 up, 8 in 2024-09-14T04:34:26.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:25 smithi039 ceph-mon[111235]: osdmap e1226: 8 total, 8 up, 8 in 2024-09-14T04:34:27.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:26 smithi129 ceph-mon[101316]: pgmap v1940: 105 pgs: 105 active+clean; 121 MiB data, 812 MiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 685 KiB/s wr, 32 op/s 2024-09-14T04:34:27.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:26 smithi039 ceph-mon[106470]: pgmap v1940: 105 pgs: 105 active+clean; 121 MiB data, 812 MiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 685 KiB/s wr, 32 op/s 2024-09-14T04:34:27.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:26 smithi039 ceph-mon[111235]: pgmap v1940: 105 pgs: 105 active+clean; 121 MiB data, 812 MiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 685 KiB/s wr, 32 op/s 2024-09-14T04:34:29.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:28 smithi039 ceph-mon[106470]: pgmap v1941: 105 pgs: 105 active+clean; 121 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 685 KiB/s wr, 21 op/s 2024-09-14T04:34:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:34:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:34:28] "GET /metrics HTTP/1.1" 200 36685 "" "Prometheus/2.51.0" 2024-09-14T04:34:29.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:28 smithi039 ceph-mon[111235]: pgmap v1941: 105 pgs: 105 active+clean; 121 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 685 KiB/s wr, 21 op/s 2024-09-14T04:34:29.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:28 smithi129 ceph-mon[101316]: pgmap v1941: 105 pgs: 105 active+clean; 121 MiB data, 816 MiB used, 714 GiB / 715 GiB avail; 13 KiB/s rd, 685 KiB/s wr, 21 op/s 2024-09-14T04:34:30.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:29 smithi129 ceph-mon[101316]: osdmap e1227: 8 total, 8 up, 8 in 2024-09-14T04:34:30.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:29 smithi039 ceph-mon[106470]: osdmap e1227: 8 total, 8 up, 8 in 2024-09-14T04:34:30.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:29 smithi039 ceph-mon[111235]: osdmap e1227: 8 total, 8 up, 8 in 2024-09-14T04:34:30.925 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/1.DiffIterateDeterministicLUKS1PP (30416 ms) 2024-09-14T04:34:30.925 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/1.DiffIterateDeterministicLUKS2 2024-09-14T04:34:30.925 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:34:31.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:30 smithi129 ceph-mon[101316]: pgmap v1943: 105 pgs: 105 active+clean; 128 MiB data, 820 MiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 4.2 KiB/s wr, 41 op/s 2024-09-14T04:34:31.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:30 smithi129 ceph-mon[101316]: osdmap e1228: 8 total, 8 up, 8 in 2024-09-14T04:34:31.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:30 smithi039 ceph-mon[106470]: pgmap v1943: 105 pgs: 105 active+clean; 128 MiB data, 820 MiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 4.2 KiB/s wr, 41 op/s 2024-09-14T04:34:31.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:30 smithi039 ceph-mon[106470]: osdmap e1228: 8 total, 8 up, 8 in 2024-09-14T04:34:31.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:30 smithi039 ceph-mon[111235]: pgmap v1943: 105 pgs: 105 active+clean; 128 MiB data, 820 MiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 4.2 KiB/s wr, 41 op/s 2024-09-14T04:34:31.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:30 smithi039 ceph-mon[111235]: osdmap e1228: 8 total, 8 up, 8 in 2024-09-14T04:34:32.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:31 smithi129 ceph-mon[101316]: osdmap e1229: 8 total, 8 up, 8 in 2024-09-14T04:34:32.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:31 smithi039 ceph-mon[106470]: osdmap e1229: 8 total, 8 up, 8 in 2024-09-14T04:34:32.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:31 smithi039 ceph-mon[111235]: osdmap e1229: 8 total, 8 up, 8 in 2024-09-14T04:34:33.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:32 smithi129 ceph-mon[101316]: pgmap v1946: 105 pgs: 105 active+clean; 128 MiB data, 820 MiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 4.0 KiB/s wr, 41 op/s 2024-09-14T04:34:33.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:32 smithi039 ceph-mon[106470]: pgmap v1946: 105 pgs: 105 active+clean; 128 MiB data, 820 MiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 4.0 KiB/s wr, 41 op/s 2024-09-14T04:34:33.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:32 smithi039 ceph-mon[111235]: pgmap v1946: 105 pgs: 105 active+clean; 128 MiB data, 820 MiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 4.0 KiB/s wr, 41 op/s 2024-09-14T04:34:35.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:34 smithi129 ceph-mon[101316]: pgmap v1947: 105 pgs: 105 active+clean; 136 MiB data, 833 MiB used, 714 GiB / 715 GiB avail; 48 KiB/s rd, 688 KiB/s wr, 69 op/s 2024-09-14T04:34:35.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:34 smithi039 ceph-mon[106470]: pgmap v1947: 105 pgs: 105 active+clean; 136 MiB data, 833 MiB used, 714 GiB / 715 GiB avail; 48 KiB/s rd, 688 KiB/s wr, 69 op/s 2024-09-14T04:34:35.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:34 smithi039 ceph-mon[111235]: pgmap v1947: 105 pgs: 105 active+clean; 136 MiB data, 833 MiB used, 714 GiB / 715 GiB avail; 48 KiB/s rd, 688 KiB/s wr, 69 op/s 2024-09-14T04:34:37.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:36 smithi129 ceph-mon[101316]: pgmap v1948: 105 pgs: 105 active+clean; 136 MiB data, 833 MiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 629 KiB/s wr, 39 op/s 2024-09-14T04:34:37.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:36 smithi039 ceph-mon[106470]: pgmap v1948: 105 pgs: 105 active+clean; 136 MiB data, 833 MiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 629 KiB/s wr, 39 op/s 2024-09-14T04:34:37.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:36 smithi039 ceph-mon[111235]: pgmap v1948: 105 pgs: 105 active+clean; 136 MiB data, 833 MiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 629 KiB/s wr, 39 op/s 2024-09-14T04:34:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:34:38.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:34:38.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:34:38.937 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:34:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:34:38] "GET /metrics HTTP/1.1" 200 36683 "" "Prometheus/2.51.0" 2024-09-14T04:34:39.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:38 smithi129 ceph-mon[101316]: pgmap v1949: 105 pgs: 105 active+clean; 136 MiB data, 833 MiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 515 KiB/s wr, 31 op/s 2024-09-14T04:34:39.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:38 smithi039 ceph-mon[111235]: pgmap v1949: 105 pgs: 105 active+clean; 136 MiB data, 833 MiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 515 KiB/s wr, 31 op/s 2024-09-14T04:34:39.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:38 smithi039 ceph-mon[106470]: pgmap v1949: 105 pgs: 105 active+clean; 136 MiB data, 833 MiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 515 KiB/s wr, 31 op/s 2024-09-14T04:34:40.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:39 smithi129 ceph-mon[101316]: osdmap e1230: 8 total, 8 up, 8 in 2024-09-14T04:34:40.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:39 smithi039 ceph-mon[106470]: osdmap e1230: 8 total, 8 up, 8 in 2024-09-14T04:34:40.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:39 smithi039 ceph-mon[111235]: osdmap e1230: 8 total, 8 up, 8 in 2024-09-14T04:34:40.993 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:34:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:40 smithi129 ceph-mon[101316]: pgmap v1951: 105 pgs: 105 active+clean; 140 MiB data, 841 MiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 963 KiB/s wr, 29 op/s 2024-09-14T04:34:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:40 smithi129 ceph-mon[101316]: osdmap e1231: 8 total, 8 up, 8 in 2024-09-14T04:34:41.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:40 smithi039 ceph-mon[106470]: pgmap v1951: 105 pgs: 105 active+clean; 140 MiB data, 841 MiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 963 KiB/s wr, 29 op/s 2024-09-14T04:34:41.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:40 smithi039 ceph-mon[106470]: osdmap e1231: 8 total, 8 up, 8 in 2024-09-14T04:34:41.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:40 smithi039 ceph-mon[111235]: pgmap v1951: 105 pgs: 105 active+clean; 140 MiB data, 841 MiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 963 KiB/s wr, 29 op/s 2024-09-14T04:34:41.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:40 smithi039 ceph-mon[111235]: osdmap e1231: 8 total, 8 up, 8 in 2024-09-14T04:34:42.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:41 smithi129 ceph-mon[101316]: osdmap e1232: 8 total, 8 up, 8 in 2024-09-14T04:34:42.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:41 smithi039 ceph-mon[106470]: osdmap e1232: 8 total, 8 up, 8 in 2024-09-14T04:34:42.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:41 smithi039 ceph-mon[111235]: osdmap e1232: 8 total, 8 up, 8 in 2024-09-14T04:34:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:42 smithi129 ceph-mon[101316]: pgmap v1954: 105 pgs: 105 active+clean; 140 MiB data, 841 MiB used, 714 GiB / 715 GiB avail; 682 KiB/s wr, 0 op/s 2024-09-14T04:34:43.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:42 smithi039 ceph-mon[106470]: pgmap v1954: 105 pgs: 105 active+clean; 140 MiB data, 841 MiB used, 714 GiB / 715 GiB avail; 682 KiB/s wr, 0 op/s 2024-09-14T04:34:43.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:42 smithi039 ceph-mon[111235]: pgmap v1954: 105 pgs: 105 active+clean; 140 MiB data, 841 MiB used, 714 GiB / 715 GiB avail; 682 KiB/s wr, 0 op/s 2024-09-14T04:34:45.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:44 smithi129 ceph-mon[101316]: pgmap v1955: 105 pgs: 105 active+clean; 152 MiB data, 881 MiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 45 op/s 2024-09-14T04:34:45.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:44 smithi039 ceph-mon[106470]: pgmap v1955: 105 pgs: 105 active+clean; 152 MiB data, 881 MiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 45 op/s 2024-09-14T04:34:45.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:44 smithi039 ceph-mon[111235]: pgmap v1955: 105 pgs: 105 active+clean; 152 MiB data, 881 MiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 45 op/s 2024-09-14T04:34:47.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:46 smithi129 ceph-mon[101316]: pgmap v1956: 105 pgs: 105 active+clean; 152 MiB data, 881 MiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 2.5 MiB/s wr, 42 op/s 2024-09-14T04:34:47.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:46 smithi039 ceph-mon[106470]: pgmap v1956: 105 pgs: 105 active+clean; 152 MiB data, 881 MiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 2.5 MiB/s wr, 42 op/s 2024-09-14T04:34:47.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:46 smithi039 ceph-mon[111235]: pgmap v1956: 105 pgs: 105 active+clean; 152 MiB data, 881 MiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 2.5 MiB/s wr, 42 op/s 2024-09-14T04:34:48.981 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:34:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:34:48] "GET /metrics HTTP/1.1" 200 36683 "" "Prometheus/2.51.0" 2024-09-14T04:34:49.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:48 smithi129 ceph-mon[101316]: pgmap v1957: 105 pgs: 105 active+clean; 152 MiB data, 881 MiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 1.5 MiB/s wr, 34 op/s 2024-09-14T04:34:49.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:48 smithi039 ceph-mon[106470]: pgmap v1957: 105 pgs: 105 active+clean; 152 MiB data, 881 MiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 1.5 MiB/s wr, 34 op/s 2024-09-14T04:34:49.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:48 smithi039 ceph-mon[111235]: pgmap v1957: 105 pgs: 105 active+clean; 152 MiB data, 881 MiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 1.5 MiB/s wr, 34 op/s 2024-09-14T04:34:50.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:50 smithi129 ceph-mon[101316]: osdmap e1233: 8 total, 8 up, 8 in 2024-09-14T04:34:50.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:50 smithi039 ceph-mon[106470]: osdmap e1233: 8 total, 8 up, 8 in 2024-09-14T04:34:50.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:50 smithi039 ceph-mon[111235]: osdmap e1233: 8 total, 8 up, 8 in 2024-09-14T04:34:51.057 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:34:51.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:51 smithi039 ceph-mon[106470]: pgmap v1959: 105 pgs: 105 active+clean; 152 MiB data, 885 MiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 1.4 MiB/s wr, 38 op/s 2024-09-14T04:34:51.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:51 smithi039 ceph-mon[106470]: osdmap e1234: 8 total, 8 up, 8 in 2024-09-14T04:34:51.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:51 smithi039 ceph-mon[111235]: pgmap v1959: 105 pgs: 105 active+clean; 152 MiB data, 885 MiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 1.4 MiB/s wr, 38 op/s 2024-09-14T04:34:51.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:51 smithi039 ceph-mon[111235]: osdmap e1234: 8 total, 8 up, 8 in 2024-09-14T04:34:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:51 smithi129 ceph-mon[101316]: pgmap v1959: 105 pgs: 105 active+clean; 152 MiB data, 885 MiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 1.4 MiB/s wr, 38 op/s 2024-09-14T04:34:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:51 smithi129 ceph-mon[101316]: osdmap e1234: 8 total, 8 up, 8 in 2024-09-14T04:34:52.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:52 smithi039 ceph-mon[106470]: osdmap e1235: 8 total, 8 up, 8 in 2024-09-14T04:34:52.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:52 smithi039 ceph-mon[111235]: osdmap e1235: 8 total, 8 up, 8 in 2024-09-14T04:34:52.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:52 smithi129 ceph-mon[101316]: osdmap e1235: 8 total, 8 up, 8 in 2024-09-14T04:34:53.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:53 smithi039 ceph-mon[106470]: pgmap v1962: 105 pgs: 105 active+clean; 152 MiB data, 885 MiB used, 714 GiB / 715 GiB avail; 4.8 KiB/s rd, 1.8 KiB/s wr, 8 op/s 2024-09-14T04:34:53.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:53 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:34:53.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:53 smithi039 ceph-mon[111235]: pgmap v1962: 105 pgs: 105 active+clean; 152 MiB data, 885 MiB used, 714 GiB / 715 GiB avail; 4.8 KiB/s rd, 1.8 KiB/s wr, 8 op/s 2024-09-14T04:34:53.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:53 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:34:53.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:53 smithi129 ceph-mon[101316]: pgmap v1962: 105 pgs: 105 active+clean; 152 MiB data, 885 MiB used, 714 GiB / 715 GiB avail; 4.8 KiB/s rd, 1.8 KiB/s wr, 8 op/s 2024-09-14T04:34:53.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:53 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:34:55.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:55 smithi129 ceph-mon[101316]: pgmap v1963: 105 pgs: 105 active+clean; 169 MiB data, 930 MiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.7 MiB/s wr, 41 op/s 2024-09-14T04:34:55.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:55 smithi039 ceph-mon[106470]: pgmap v1963: 105 pgs: 105 active+clean; 169 MiB data, 930 MiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.7 MiB/s wr, 41 op/s 2024-09-14T04:34:55.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:55 smithi039 ceph-mon[111235]: pgmap v1963: 105 pgs: 105 active+clean; 169 MiB data, 930 MiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.7 MiB/s wr, 41 op/s 2024-09-14T04:34:56.436 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:56 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:34:56.538 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:56 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:34:56.538 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:56 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:34:57.395 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:57 smithi129 ceph-mon[101316]: pgmap v1964: 105 pgs: 105 active+clean; 170 MiB data, 930 MiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 2.5 MiB/s wr, 43 op/s 2024-09-14T04:34:57.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:57 smithi039 ceph-mon[106470]: pgmap v1964: 105 pgs: 105 active+clean; 170 MiB data, 930 MiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 2.5 MiB/s wr, 43 op/s 2024-09-14T04:34:57.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:57 smithi039 ceph-mon[111235]: pgmap v1964: 105 pgs: 105 active+clean; 170 MiB data, 930 MiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 2.5 MiB/s wr, 43 op/s 2024-09-14T04:34:59.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:34:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:34:58] "GET /metrics HTTP/1.1" 200 36681 "" "Prometheus/2.51.0" 2024-09-14T04:34:59.374 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:34:59 smithi039 ceph-mon[106470]: pgmap v1965: 105 pgs: 105 active+clean; 170 MiB data, 930 MiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 2.0 MiB/s wr, 28 op/s 2024-09-14T04:34:59.374 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:34:59 smithi039 ceph-mon[111235]: pgmap v1965: 105 pgs: 105 active+clean; 170 MiB data, 930 MiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 2.0 MiB/s wr, 28 op/s 2024-09-14T04:34:59.502 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:34:59 smithi129 ceph-mon[101316]: pgmap v1965: 105 pgs: 105 active+clean; 170 MiB data, 930 MiB used, 714 GiB / 715 GiB avail; 19 KiB/s rd, 2.0 MiB/s wr, 28 op/s 2024-09-14T04:35:00.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:00 smithi129 ceph-mon[101316]: osdmap e1236: 8 total, 8 up, 8 in 2024-09-14T04:35:00.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:00 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:35:00.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:00 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:35:00.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:00 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:35:00.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:00 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:35:00.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:00 smithi039 ceph-mon[111235]: osdmap e1236: 8 total, 8 up, 8 in 2024-09-14T04:35:00.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:00 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:35:00.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:00 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:35:00.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:00 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:35:00.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:00 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:35:00.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:00 smithi039 ceph-mon[106470]: osdmap e1236: 8 total, 8 up, 8 in 2024-09-14T04:35:00.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:00 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:35:00.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:00 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:35:00.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:00 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:35:00.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:00 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:35:01.144 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:35:01.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:01 smithi129 ceph-mon[101316]: pgmap v1967: 105 pgs: 105 active+clean; 174 MiB data, 938 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 2.4 MiB/s wr, 27 op/s 2024-09-14T04:35:01.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:01 smithi129 ceph-mon[101316]: osdmap e1237: 8 total, 8 up, 8 in 2024-09-14T04:35:01.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:01 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:35:01.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:01 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:35:01.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:01 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:35:01.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:01 smithi039 ceph-mon[106470]: pgmap v1967: 105 pgs: 105 active+clean; 174 MiB data, 938 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 2.4 MiB/s wr, 27 op/s 2024-09-14T04:35:01.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:01 smithi039 ceph-mon[106470]: osdmap e1237: 8 total, 8 up, 8 in 2024-09-14T04:35:01.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:01 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:35:01.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:01 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:35:01.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:01 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:35:01.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:01 smithi039 ceph-mon[111235]: pgmap v1967: 105 pgs: 105 active+clean; 174 MiB data, 938 MiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 2.4 MiB/s wr, 27 op/s 2024-09-14T04:35:01.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:01 smithi039 ceph-mon[111235]: osdmap e1237: 8 total, 8 up, 8 in 2024-09-14T04:35:01.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:01 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:35:01.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:01 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:35:01.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:01 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:35:02.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:02 smithi129 ceph-mon[101316]: osdmap e1238: 8 total, 8 up, 8 in 2024-09-14T04:35:02.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:02 smithi039 ceph-mon[106470]: osdmap e1238: 8 total, 8 up, 8 in 2024-09-14T04:35:02.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:02 smithi039 ceph-mon[111235]: osdmap e1238: 8 total, 8 up, 8 in 2024-09-14T04:35:03.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:03 smithi129 ceph-mon[101316]: pgmap v1970: 105 pgs: 105 active+clean; 178 MiB data, 942 MiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s wr, 0 op/s 2024-09-14T04:35:03.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:03 smithi039 ceph-mon[106470]: pgmap v1970: 105 pgs: 105 active+clean; 178 MiB data, 942 MiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s wr, 0 op/s 2024-09-14T04:35:03.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:03 smithi039 ceph-mon[111235]: pgmap v1970: 105 pgs: 105 active+clean; 178 MiB data, 942 MiB used, 714 GiB / 715 GiB avail; 1.3 MiB/s wr, 0 op/s 2024-09-14T04:35:05.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:05 smithi129 ceph-mon[101316]: pgmap v1971: 105 pgs: 105 active+clean; 187 MiB data, 978 MiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 44 op/s 2024-09-14T04:35:05.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:05 smithi039 ceph-mon[106470]: pgmap v1971: 105 pgs: 105 active+clean; 187 MiB data, 978 MiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 44 op/s 2024-09-14T04:35:05.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:05 smithi039 ceph-mon[111235]: pgmap v1971: 105 pgs: 105 active+clean; 187 MiB data, 978 MiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 44 op/s 2024-09-14T04:35:07.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:07 smithi129 ceph-mon[101316]: pgmap v1972: 105 pgs: 105 active+clean; 188 MiB data, 978 MiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 2.5 MiB/s wr, 43 op/s 2024-09-14T04:35:07.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:07 smithi039 ceph-mon[106470]: pgmap v1972: 105 pgs: 105 active+clean; 188 MiB data, 978 MiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 2.5 MiB/s wr, 43 op/s 2024-09-14T04:35:07.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:07 smithi039 ceph-mon[111235]: pgmap v1972: 105 pgs: 105 active+clean; 188 MiB data, 978 MiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 2.5 MiB/s wr, 43 op/s 2024-09-14T04:35:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:08 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:35:08.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:08 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:35:08.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:08 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:35:09.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:35:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:35:08] "GET /metrics HTTP/1.1" 200 36688 "" "Prometheus/2.51.0" 2024-09-14T04:35:09.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:09 smithi129 ceph-mon[101316]: pgmap v1973: 105 pgs: 105 active+clean; 188 MiB data, 978 MiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 1.5 MiB/s wr, 34 op/s 2024-09-14T04:35:09.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:09 smithi039 ceph-mon[106470]: pgmap v1973: 105 pgs: 105 active+clean; 188 MiB data, 978 MiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 1.5 MiB/s wr, 34 op/s 2024-09-14T04:35:09.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:09 smithi039 ceph-mon[111235]: pgmap v1973: 105 pgs: 105 active+clean; 188 MiB data, 978 MiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 1.5 MiB/s wr, 34 op/s 2024-09-14T04:35:10.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:10 smithi129 ceph-mon[101316]: osdmap e1239: 8 total, 8 up, 8 in 2024-09-14T04:35:10.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:10 smithi039 ceph-mon[106470]: osdmap e1239: 8 total, 8 up, 8 in 2024-09-14T04:35:10.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:10 smithi039 ceph-mon[111235]: osdmap e1239: 8 total, 8 up, 8 in 2024-09-14T04:35:11.225 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:35:11.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:11 smithi129 ceph-mon[101316]: pgmap v1975: 105 pgs: 105 active+clean; 192 MiB data, 982 MiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 1.4 MiB/s wr, 33 op/s 2024-09-14T04:35:11.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:11 smithi129 ceph-mon[101316]: osdmap e1240: 8 total, 8 up, 8 in 2024-09-14T04:35:11.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:11 smithi039 ceph-mon[106470]: pgmap v1975: 105 pgs: 105 active+clean; 192 MiB data, 982 MiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 1.4 MiB/s wr, 33 op/s 2024-09-14T04:35:11.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:11 smithi039 ceph-mon[106470]: osdmap e1240: 8 total, 8 up, 8 in 2024-09-14T04:35:11.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:11 smithi039 ceph-mon[111235]: pgmap v1975: 105 pgs: 105 active+clean; 192 MiB data, 982 MiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 1.4 MiB/s wr, 33 op/s 2024-09-14T04:35:11.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:11 smithi039 ceph-mon[111235]: osdmap e1240: 8 total, 8 up, 8 in 2024-09-14T04:35:12.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:12 smithi129 ceph-mon[101316]: osdmap e1241: 8 total, 8 up, 8 in 2024-09-14T04:35:12.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:12 smithi039 ceph-mon[106470]: osdmap e1241: 8 total, 8 up, 8 in 2024-09-14T04:35:12.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:12 smithi039 ceph-mon[111235]: osdmap e1241: 8 total, 8 up, 8 in 2024-09-14T04:35:13.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:13 smithi129 ceph-mon[101316]: pgmap v1978: 105 pgs: 105 active+clean; 192 MiB data, 982 MiB used, 714 GiB / 715 GiB avail; 683 KiB/s wr, 0 op/s 2024-09-14T04:35:13.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:13 smithi039 ceph-mon[106470]: pgmap v1978: 105 pgs: 105 active+clean; 192 MiB data, 982 MiB used, 714 GiB / 715 GiB avail; 683 KiB/s wr, 0 op/s 2024-09-14T04:35:13.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:13 smithi039 ceph-mon[111235]: pgmap v1978: 105 pgs: 105 active+clean; 192 MiB data, 982 MiB used, 714 GiB / 715 GiB avail; 683 KiB/s wr, 0 op/s 2024-09-14T04:35:15.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:15 smithi129 ceph-mon[101316]: pgmap v1979: 105 pgs: 105 active+clean; 206 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 45 op/s 2024-09-14T04:35:15.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:15 smithi039 ceph-mon[106470]: pgmap v1979: 105 pgs: 105 active+clean; 206 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 45 op/s 2024-09-14T04:35:15.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:15 smithi039 ceph-mon[111235]: pgmap v1979: 105 pgs: 105 active+clean; 206 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 45 op/s 2024-09-14T04:35:17.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:17 smithi129 ceph-mon[101316]: pgmap v1980: 105 pgs: 105 active+clean; 206 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 2.6 MiB/s wr, 44 op/s 2024-09-14T04:35:17.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:17 smithi039 ceph-mon[106470]: pgmap v1980: 105 pgs: 105 active+clean; 206 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 2.6 MiB/s wr, 44 op/s 2024-09-14T04:35:17.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:17 smithi039 ceph-mon[111235]: pgmap v1980: 105 pgs: 105 active+clean; 206 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 2.6 MiB/s wr, 44 op/s 2024-09-14T04:35:19.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:35:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:35:18] "GET /metrics HTTP/1.1" 200 36688 "" "Prometheus/2.51.0" 2024-09-14T04:35:19.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:19 smithi129 ceph-mon[101316]: pgmap v1981: 105 pgs: 105 active+clean; 206 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 1.5 MiB/s wr, 34 op/s 2024-09-14T04:35:19.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:19 smithi039 ceph-mon[106470]: pgmap v1981: 105 pgs: 105 active+clean; 206 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 1.5 MiB/s wr, 34 op/s 2024-09-14T04:35:19.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:19 smithi039 ceph-mon[111235]: pgmap v1981: 105 pgs: 105 active+clean; 206 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 1.5 MiB/s wr, 34 op/s 2024-09-14T04:35:21.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:21 smithi129 ceph-mon[101316]: pgmap v1982: 105 pgs: 105 active+clean; 206 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 1.3 MiB/s wr, 29 op/s 2024-09-14T04:35:21.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:21 smithi129 ceph-mon[101316]: osdmap e1242: 8 total, 8 up, 8 in 2024-09-14T04:35:21.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:21 smithi039 ceph-mon[106470]: pgmap v1982: 105 pgs: 105 active+clean; 206 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 1.3 MiB/s wr, 29 op/s 2024-09-14T04:35:21.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:21 smithi039 ceph-mon[106470]: osdmap e1242: 8 total, 8 up, 8 in 2024-09-14T04:35:21.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:21 smithi039 ceph-mon[111235]: pgmap v1982: 105 pgs: 105 active+clean; 206 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 1.3 MiB/s wr, 29 op/s 2024-09-14T04:35:21.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:21 smithi039 ceph-mon[111235]: osdmap e1242: 8 total, 8 up, 8 in 2024-09-14T04:35:22.326 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/1.DiffIterateDeterministicLUKS2 (51402 ms) 2024-09-14T04:35:22.326 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/1.DiffIterateDeterministicLUKS2PP 2024-09-14T04:35:22.326 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:35:22.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:22 smithi039 ceph-mon[106470]: osdmap e1243: 8 total, 8 up, 8 in 2024-09-14T04:35:22.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:35:22.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:22 smithi039 ceph-mon[111235]: osdmap e1243: 8 total, 8 up, 8 in 2024-09-14T04:35:22.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:35:22.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:22 smithi129 ceph-mon[101316]: osdmap e1243: 8 total, 8 up, 8 in 2024-09-14T04:35:22.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:35:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:23 smithi039 ceph-mon[106470]: pgmap v1985: 105 pgs: 105 active+clean; 206 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 127 B/s wr, 0 op/s 2024-09-14T04:35:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:23 smithi039 ceph-mon[106470]: osdmap e1244: 8 total, 8 up, 8 in 2024-09-14T04:35:23.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:23 smithi039 ceph-mon[111235]: pgmap v1985: 105 pgs: 105 active+clean; 206 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 127 B/s wr, 0 op/s 2024-09-14T04:35:23.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:23 smithi039 ceph-mon[111235]: osdmap e1244: 8 total, 8 up, 8 in 2024-09-14T04:35:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:23 smithi129 ceph-mon[101316]: pgmap v1985: 105 pgs: 105 active+clean; 206 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 255 B/s rd, 127 B/s wr, 0 op/s 2024-09-14T04:35:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:23 smithi129 ceph-mon[101316]: osdmap e1244: 8 total, 8 up, 8 in 2024-09-14T04:35:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:25 smithi129 ceph-mon[101316]: pgmap v1987: 105 pgs: 105 active+clean; 222 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.0 MiB/s wr, 40 op/s 2024-09-14T04:35:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:25 smithi039 ceph-mon[106470]: pgmap v1987: 105 pgs: 105 active+clean; 222 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.0 MiB/s wr, 40 op/s 2024-09-14T04:35:25.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:25 smithi039 ceph-mon[111235]: pgmap v1987: 105 pgs: 105 active+clean; 222 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.0 MiB/s wr, 40 op/s 2024-09-14T04:35:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:27 smithi129 ceph-mon[101316]: pgmap v1988: 105 pgs: 105 active+clean; 230 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 2.7 MiB/s wr, 43 op/s 2024-09-14T04:35:27.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:27 smithi039 ceph-mon[106470]: pgmap v1988: 105 pgs: 105 active+clean; 230 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 2.7 MiB/s wr, 43 op/s 2024-09-14T04:35:27.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:27 smithi039 ceph-mon[111235]: pgmap v1988: 105 pgs: 105 active+clean; 230 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 28 KiB/s rd, 2.7 MiB/s wr, 43 op/s 2024-09-14T04:35:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:35:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:35:28] "GET /metrics HTTP/1.1" 200 36691 "" "Prometheus/2.51.0" 2024-09-14T04:35:29.573 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:29 smithi129 ceph-mon[101316]: pgmap v1989: 105 pgs: 105 active+clean; 230 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 2.2 MiB/s wr, 38 op/s 2024-09-14T04:35:29.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:29 smithi039 ceph-mon[106470]: pgmap v1989: 105 pgs: 105 active+clean; 230 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 2.2 MiB/s wr, 38 op/s 2024-09-14T04:35:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:29 smithi039 ceph-mon[111235]: pgmap v1989: 105 pgs: 105 active+clean; 230 MiB data, 1.0 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 2.2 MiB/s wr, 38 op/s 2024-09-14T04:35:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:31 smithi129 ceph-mon[101316]: pgmap v1990: 105 pgs: 105 active+clean; 230 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 2.0 MiB/s wr, 34 op/s 2024-09-14T04:35:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:31 smithi129 ceph-mon[101316]: osdmap e1245: 8 total, 8 up, 8 in 2024-09-14T04:35:31.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:31 smithi039 ceph-mon[106470]: pgmap v1990: 105 pgs: 105 active+clean; 230 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 2.0 MiB/s wr, 34 op/s 2024-09-14T04:35:31.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:31 smithi039 ceph-mon[106470]: osdmap e1245: 8 total, 8 up, 8 in 2024-09-14T04:35:31.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:31 smithi039 ceph-mon[111235]: pgmap v1990: 105 pgs: 105 active+clean; 230 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 2.0 MiB/s wr, 34 op/s 2024-09-14T04:35:31.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:31 smithi039 ceph-mon[111235]: osdmap e1245: 8 total, 8 up, 8 in 2024-09-14T04:35:32.429 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:35:32.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:32 smithi129 ceph-mon[101316]: osdmap e1246: 8 total, 8 up, 8 in 2024-09-14T04:35:32.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:32 smithi129 ceph-mon[101316]: pgmap v1993: 105 pgs: 105 active+clean; 230 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.1 KiB/s rd, 513 KiB/s wr, 4 op/s 2024-09-14T04:35:32.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:32 smithi039 ceph-mon[106470]: osdmap e1246: 8 total, 8 up, 8 in 2024-09-14T04:35:32.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:32 smithi039 ceph-mon[106470]: pgmap v1993: 105 pgs: 105 active+clean; 230 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.1 KiB/s rd, 513 KiB/s wr, 4 op/s 2024-09-14T04:35:32.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:32 smithi039 ceph-mon[111235]: osdmap e1246: 8 total, 8 up, 8 in 2024-09-14T04:35:32.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:32 smithi039 ceph-mon[111235]: pgmap v1993: 105 pgs: 105 active+clean; 230 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 2.1 KiB/s rd, 513 KiB/s wr, 4 op/s 2024-09-14T04:35:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:33 smithi129 ceph-mon[101316]: osdmap e1247: 8 total, 8 up, 8 in 2024-09-14T04:35:33.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:33 smithi039 ceph-mon[106470]: osdmap e1247: 8 total, 8 up, 8 in 2024-09-14T04:35:33.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:33 smithi039 ceph-mon[111235]: osdmap e1247: 8 total, 8 up, 8 in 2024-09-14T04:35:34.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:34 smithi129 ceph-mon[101316]: pgmap v1995: 105 pgs: 105 active+clean; 242 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.0 MiB/s wr, 39 op/s 2024-09-14T04:35:34.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:34 smithi039 ceph-mon[106470]: pgmap v1995: 105 pgs: 105 active+clean; 242 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.0 MiB/s wr, 39 op/s 2024-09-14T04:35:34.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:34 smithi039 ceph-mon[111235]: pgmap v1995: 105 pgs: 105 active+clean; 242 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.0 MiB/s wr, 39 op/s 2024-09-14T04:35:36.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:36 smithi129 ceph-mon[101316]: pgmap v1996: 105 pgs: 105 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.7 MiB/s wr, 41 op/s 2024-09-14T04:35:36.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:36 smithi039 ceph-mon[106470]: pgmap v1996: 105 pgs: 105 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.7 MiB/s wr, 41 op/s 2024-09-14T04:35:36.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:36 smithi039 ceph-mon[111235]: pgmap v1996: 105 pgs: 105 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.7 MiB/s wr, 41 op/s 2024-09-14T04:35:37.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:35:37.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:35:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:35:38.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:38 smithi039 ceph-mon[106470]: pgmap v1997: 105 pgs: 105 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 2.3 MiB/s wr, 39 op/s 2024-09-14T04:35:38.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:35:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:35:38] "GET /metrics HTTP/1.1" 200 36687 "" "Prometheus/2.51.0" 2024-09-14T04:35:38.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:38 smithi039 ceph-mon[111235]: pgmap v1997: 105 pgs: 105 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 2.3 MiB/s wr, 39 op/s 2024-09-14T04:35:39.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:38 smithi129 ceph-mon[101316]: pgmap v1997: 105 pgs: 105 active+clean; 246 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 26 KiB/s rd, 2.3 MiB/s wr, 39 op/s 2024-09-14T04:35:40.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:39 smithi129 ceph-mon[101316]: osdmap e1248: 8 total, 8 up, 8 in 2024-09-14T04:35:40.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:39 smithi039 ceph-mon[106470]: osdmap e1248: 8 total, 8 up, 8 in 2024-09-14T04:35:40.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:39 smithi039 ceph-mon[111235]: osdmap e1248: 8 total, 8 up, 8 in 2024-09-14T04:35:40.585 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:35:41.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:40 smithi129 ceph-mon[101316]: pgmap v1999: 105 pgs: 105 active+clean; 258 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 3.5 MiB/s wr, 39 op/s 2024-09-14T04:35:41.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:40 smithi129 ceph-mon[101316]: osdmap e1249: 8 total, 8 up, 8 in 2024-09-14T04:35:41.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:40 smithi039 ceph-mon[106470]: pgmap v1999: 105 pgs: 105 active+clean; 258 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 3.5 MiB/s wr, 39 op/s 2024-09-14T04:35:41.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:40 smithi039 ceph-mon[106470]: osdmap e1249: 8 total, 8 up, 8 in 2024-09-14T04:35:41.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:40 smithi039 ceph-mon[111235]: pgmap v1999: 105 pgs: 105 active+clean; 258 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 25 KiB/s rd, 3.5 MiB/s wr, 39 op/s 2024-09-14T04:35:41.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:40 smithi039 ceph-mon[111235]: osdmap e1249: 8 total, 8 up, 8 in 2024-09-14T04:35:42.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:41 smithi129 ceph-mon[101316]: osdmap e1250: 8 total, 8 up, 8 in 2024-09-14T04:35:42.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:41 smithi039 ceph-mon[106470]: osdmap e1250: 8 total, 8 up, 8 in 2024-09-14T04:35:42.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:41 smithi039 ceph-mon[111235]: osdmap e1250: 8 total, 8 up, 8 in 2024-09-14T04:35:43.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:42 smithi129 ceph-mon[101316]: pgmap v2002: 105 pgs: 105 active+clean; 258 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.3 KiB/s rd, 2.0 MiB/s wr, 11 op/s 2024-09-14T04:35:43.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:42 smithi039 ceph-mon[106470]: pgmap v2002: 105 pgs: 105 active+clean; 258 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.3 KiB/s rd, 2.0 MiB/s wr, 11 op/s 2024-09-14T04:35:43.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:42 smithi039 ceph-mon[111235]: pgmap v2002: 105 pgs: 105 active+clean; 258 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 6.3 KiB/s rd, 2.0 MiB/s wr, 11 op/s 2024-09-14T04:35:45.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:44 smithi129 ceph-mon[101316]: pgmap v2003: 105 pgs: 105 active+clean; 268 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 45 op/s 2024-09-14T04:35:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:44 smithi039 ceph-mon[106470]: pgmap v2003: 105 pgs: 105 active+clean; 268 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 45 op/s 2024-09-14T04:35:45.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:44 smithi039 ceph-mon[111235]: pgmap v2003: 105 pgs: 105 active+clean; 268 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 45 op/s 2024-09-14T04:35:47.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:46 smithi129 ceph-mon[101316]: pgmap v2004: 105 pgs: 105 active+clean; 268 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 598 KiB/s wr, 34 op/s 2024-09-14T04:35:47.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:46 smithi039 ceph-mon[106470]: pgmap v2004: 105 pgs: 105 active+clean; 268 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 598 KiB/s wr, 34 op/s 2024-09-14T04:35:47.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:46 smithi039 ceph-mon[111235]: pgmap v2004: 105 pgs: 105 active+clean; 268 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 598 KiB/s wr, 34 op/s 2024-09-14T04:35:49.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:48 smithi129 ceph-mon[101316]: pgmap v2005: 105 pgs: 105 active+clean; 268 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 515 KiB/s wr, 29 op/s 2024-09-14T04:35:49.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:48 smithi039 ceph-mon[106470]: pgmap v2005: 105 pgs: 105 active+clean; 268 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 515 KiB/s wr, 29 op/s 2024-09-14T04:35:49.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:35:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:35:48] "GET /metrics HTTP/1.1" 200 36687 "" "Prometheus/2.51.0" 2024-09-14T04:35:49.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:48 smithi039 ceph-mon[111235]: pgmap v2005: 105 pgs: 105 active+clean; 268 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 20 KiB/s rd, 515 KiB/s wr, 29 op/s 2024-09-14T04:35:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:49 smithi129 ceph-mon[101316]: osdmap e1251: 8 total, 8 up, 8 in 2024-09-14T04:35:50.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:49 smithi039 ceph-mon[106470]: osdmap e1251: 8 total, 8 up, 8 in 2024-09-14T04:35:50.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:49 smithi039 ceph-mon[111235]: osdmap e1251: 8 total, 8 up, 8 in 2024-09-14T04:35:50.656 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:35:51.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:50 smithi129 ceph-mon[101316]: pgmap v2007: 105 pgs: 105 active+clean; 268 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 464 KiB/s wr, 26 op/s 2024-09-14T04:35:51.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:50 smithi129 ceph-mon[101316]: osdmap e1252: 8 total, 8 up, 8 in 2024-09-14T04:35:51.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:50 smithi039 ceph-mon[106470]: pgmap v2007: 105 pgs: 105 active+clean; 268 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 464 KiB/s wr, 26 op/s 2024-09-14T04:35:51.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:50 smithi039 ceph-mon[106470]: osdmap e1252: 8 total, 8 up, 8 in 2024-09-14T04:35:51.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:50 smithi039 ceph-mon[111235]: pgmap v2007: 105 pgs: 105 active+clean; 268 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 18 KiB/s rd, 464 KiB/s wr, 26 op/s 2024-09-14T04:35:51.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:50 smithi039 ceph-mon[111235]: osdmap e1252: 8 total, 8 up, 8 in 2024-09-14T04:35:52.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:51 smithi129 ceph-mon[101316]: osdmap e1253: 8 total, 8 up, 8 in 2024-09-14T04:35:52.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:51 smithi039 ceph-mon[106470]: osdmap e1253: 8 total, 8 up, 8 in 2024-09-14T04:35:52.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:51 smithi039 ceph-mon[111235]: osdmap e1253: 8 total, 8 up, 8 in 2024-09-14T04:35:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:52 smithi129 ceph-mon[101316]: pgmap v2010: 105 pgs: 105 active+clean; 268 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:35:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:35:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:52 smithi039 ceph-mon[106470]: pgmap v2010: 105 pgs: 105 active+clean; 268 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:35:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:35:53.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:52 smithi039 ceph-mon[111235]: pgmap v2010: 105 pgs: 105 active+clean; 268 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:35:53.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:35:55.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:54 smithi129 ceph-mon[101316]: pgmap v2011: 105 pgs: 105 active+clean; 290 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 2.7 MiB/s wr, 43 op/s 2024-09-14T04:35:55.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:54 smithi039 ceph-mon[106470]: pgmap v2011: 105 pgs: 105 active+clean; 290 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 2.7 MiB/s wr, 43 op/s 2024-09-14T04:35:55.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:54 smithi039 ceph-mon[111235]: pgmap v2011: 105 pgs: 105 active+clean; 290 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 29 KiB/s rd, 2.7 MiB/s wr, 43 op/s 2024-09-14T04:35:57.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:56 smithi129 ceph-mon[101316]: pgmap v2012: 105 pgs: 105 active+clean; 290 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.3 MiB/s wr, 40 op/s 2024-09-14T04:35:57.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:56 smithi039 ceph-mon[106470]: pgmap v2012: 105 pgs: 105 active+clean; 290 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.3 MiB/s wr, 40 op/s 2024-09-14T04:35:57.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:56 smithi039 ceph-mon[111235]: pgmap v2012: 105 pgs: 105 active+clean; 290 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.3 MiB/s wr, 40 op/s 2024-09-14T04:35:59.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:58 smithi129 ceph-mon[101316]: pgmap v2013: 105 pgs: 105 active+clean; 290 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 2.0 MiB/s wr, 34 op/s 2024-09-14T04:35:59.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:58 smithi039 ceph-mon[111235]: pgmap v2013: 105 pgs: 105 active+clean; 290 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 2.0 MiB/s wr, 34 op/s 2024-09-14T04:35:59.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:58 smithi039 ceph-mon[106470]: pgmap v2013: 105 pgs: 105 active+clean; 290 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 23 KiB/s rd, 2.0 MiB/s wr, 34 op/s 2024-09-14T04:35:59.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:35:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:35:58] "GET /metrics HTTP/1.1" 200 36690 "" "Prometheus/2.51.0" 2024-09-14T04:36:00.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:35:59 smithi129 ceph-mon[101316]: osdmap e1254: 8 total, 8 up, 8 in 2024-09-14T04:36:00.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:35:59 smithi039 ceph-mon[106470]: osdmap e1254: 8 total, 8 up, 8 in 2024-09-14T04:36:00.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:35:59 smithi039 ceph-mon[111235]: osdmap e1254: 8 total, 8 up, 8 in 2024-09-14T04:36:00.779 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:36:01.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:00 smithi129 ceph-mon[101316]: pgmap v2015: 105 pgs: 105 active+clean; 290 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 1.8 MiB/s wr, 31 op/s 2024-09-14T04:36:01.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:00 smithi129 ceph-mon[101316]: osdmap e1255: 8 total, 8 up, 8 in 2024-09-14T04:36:01.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:00 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:36:01.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:00 smithi039 ceph-mon[106470]: pgmap v2015: 105 pgs: 105 active+clean; 290 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 1.8 MiB/s wr, 31 op/s 2024-09-14T04:36:01.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:00 smithi039 ceph-mon[106470]: osdmap e1255: 8 total, 8 up, 8 in 2024-09-14T04:36:01.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:00 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:36:01.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:00 smithi039 ceph-mon[111235]: pgmap v2015: 105 pgs: 105 active+clean; 290 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 21 KiB/s rd, 1.8 MiB/s wr, 31 op/s 2024-09-14T04:36:01.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:00 smithi039 ceph-mon[111235]: osdmap e1255: 8 total, 8 up, 8 in 2024-09-14T04:36:01.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:00 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:36:02.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:01 smithi039 ceph-mon[106470]: osdmap e1256: 8 total, 8 up, 8 in 2024-09-14T04:36:02.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:01 smithi039 ceph-mon[111235]: osdmap e1256: 8 total, 8 up, 8 in 2024-09-14T04:36:02.043 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:01 smithi129 ceph-mon[101316]: osdmap e1256: 8 total, 8 up, 8 in 2024-09-14T04:36:03.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:02 smithi129 ceph-mon[101316]: pgmap v2018: 105 pgs: 105 active+clean; 290 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:36:03.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:02 smithi039 ceph-mon[106470]: pgmap v2018: 105 pgs: 105 active+clean; 290 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:36:03.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:02 smithi039 ceph-mon[111235]: pgmap v2018: 105 pgs: 105 active+clean; 290 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail 2024-09-14T04:36:05.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:04 smithi039 ceph-mon[106470]: pgmap v2019: 105 pgs: 105 active+clean; 312 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 45 op/s 2024-09-14T04:36:05.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:04 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:36:05.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:04 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:36:05.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:04 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:36:05.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:04 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:36:05.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:04 smithi039 ceph-mon[111235]: pgmap v2019: 105 pgs: 105 active+clean; 312 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 45 op/s 2024-09-14T04:36:05.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:04 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:36:05.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:04 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:36:05.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:04 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:36:05.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:04 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:36:05.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:04 smithi129 ceph-mon[101316]: pgmap v2019: 105 pgs: 105 active+clean; 312 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 30 KiB/s rd, 2.7 MiB/s wr, 45 op/s 2024-09-14T04:36:05.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:04 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:36:05.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:04 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:36:05.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:04 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:36:05.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:04 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:36:06.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:05 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:36:06.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:05 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:36:06.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:05 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:36:06.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:05 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:36:06.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:05 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:36:06.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:05 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:36:06.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:05 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:36:06.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:05 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:36:06.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:05 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:36:07.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:06 smithi129 ceph-mon[101316]: pgmap v2020: 105 pgs: 105 active+clean; 312 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.4 MiB/s wr, 41 op/s 2024-09-14T04:36:07.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:06 smithi039 ceph-mon[106470]: pgmap v2020: 105 pgs: 105 active+clean; 312 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.4 MiB/s wr, 41 op/s 2024-09-14T04:36:07.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:07 smithi039 ceph-mon[111235]: pgmap v2020: 105 pgs: 105 active+clean; 312 MiB data, 1.2 GiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 2.4 MiB/s wr, 41 op/s 2024-09-14T04:36:08.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:08 smithi039 ceph-mon[106470]: osdmap e1257: 8 total, 8 up, 8 in 2024-09-14T04:36:08.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:08 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:36:08.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:08 smithi039 ceph-mon[111235]: osdmap e1257: 8 total, 8 up, 8 in 2024-09-14T04:36:08.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:08 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:36:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:08 smithi129 ceph-mon[101316]: osdmap e1257: 8 total, 8 up, 8 in 2024-09-14T04:36:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:08 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:36:08.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:36:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:36:08] "GET /metrics HTTP/1.1" 200 36689 "" "Prometheus/2.51.0" 2024-09-14T04:36:09.061 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/1.DiffIterateDeterministicLUKS2PP (46735 ms) 2024-09-14T04:36:09.061 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/1.DiffIterateDiscard 2024-09-14T04:36:09.061 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:36:09.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:09 smithi039 ceph-mon[111235]: pgmap v2022: 105 pgs: 105 active+clean; 312 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 2.1 MiB/s wr, 36 op/s 2024-09-14T04:36:09.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:09 smithi039 ceph-mon[111235]: osdmap e1258: 8 total, 8 up, 8 in 2024-09-14T04:36:09.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:09 smithi039 ceph-mon[106470]: pgmap v2022: 105 pgs: 105 active+clean; 312 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 2.1 MiB/s wr, 36 op/s 2024-09-14T04:36:09.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:09 smithi039 ceph-mon[106470]: osdmap e1258: 8 total, 8 up, 8 in 2024-09-14T04:36:09.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:09 smithi129 ceph-mon[101316]: pgmap v2022: 105 pgs: 105 active+clean; 312 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 24 KiB/s rd, 2.1 MiB/s wr, 36 op/s 2024-09-14T04:36:09.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:09 smithi129 ceph-mon[101316]: osdmap e1258: 8 total, 8 up, 8 in 2024-09-14T04:36:10.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:10 smithi129 ceph-mon[101316]: osdmap e1259: 8 total, 8 up, 8 in 2024-09-14T04:36:10.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:10 smithi039 ceph-mon[106470]: osdmap e1259: 8 total, 8 up, 8 in 2024-09-14T04:36:10.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:10 smithi039 ceph-mon[111235]: osdmap e1259: 8 total, 8 up, 8 in 2024-09-14T04:36:11.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:11 smithi129 ceph-mon[101316]: pgmap v2025: 105 pgs: 105 active+clean; 324 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 2.0 MiB/s wr, 15 op/s 2024-09-14T04:36:11.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:11 smithi129 ceph-mon[101316]: osdmap e1260: 8 total, 8 up, 8 in 2024-09-14T04:36:11.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:11 smithi039 ceph-mon[106470]: pgmap v2025: 105 pgs: 105 active+clean; 324 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 2.0 MiB/s wr, 15 op/s 2024-09-14T04:36:11.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:11 smithi039 ceph-mon[106470]: osdmap e1260: 8 total, 8 up, 8 in 2024-09-14T04:36:11.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:11 smithi039 ceph-mon[111235]: pgmap v2025: 105 pgs: 105 active+clean; 324 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 9.7 KiB/s rd, 2.0 MiB/s wr, 15 op/s 2024-09-14T04:36:11.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:11 smithi039 ceph-mon[111235]: osdmap e1260: 8 total, 8 up, 8 in 2024-09-14T04:36:12.095 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/1.DiffIterateDiscard (3033 ms) 2024-09-14T04:36:12.095 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/1.DiffIterateTruncate 2024-09-14T04:36:12.095 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:36:12.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:12 smithi129 ceph-mon[101316]: osdmap e1261: 8 total, 8 up, 8 in 2024-09-14T04:36:12.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:12 smithi039 ceph-mon[106470]: osdmap e1261: 8 total, 8 up, 8 in 2024-09-14T04:36:12.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:12 smithi039 ceph-mon[111235]: osdmap e1261: 8 total, 8 up, 8 in 2024-09-14T04:36:13.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:13 smithi129 ceph-mon[101316]: pgmap v2028: 105 pgs: 105 active+clean; 324 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 3.0 MiB/s wr, 24 op/s 2024-09-14T04:36:13.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:13 smithi129 ceph-mon[101316]: osdmap e1262: 8 total, 8 up, 8 in 2024-09-14T04:36:13.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:13 smithi039 ceph-mon[106470]: pgmap v2028: 105 pgs: 105 active+clean; 324 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 3.0 MiB/s wr, 24 op/s 2024-09-14T04:36:13.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:13 smithi039 ceph-mon[106470]: osdmap e1262: 8 total, 8 up, 8 in 2024-09-14T04:36:13.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:13 smithi039 ceph-mon[111235]: pgmap v2028: 105 pgs: 105 active+clean; 324 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 15 KiB/s rd, 3.0 MiB/s wr, 24 op/s 2024-09-14T04:36:13.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:13 smithi039 ceph-mon[111235]: osdmap e1262: 8 total, 8 up, 8 in 2024-09-14T04:36:14.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:14 smithi129 ceph-mon[101316]: osdmap e1263: 8 total, 8 up, 8 in 2024-09-14T04:36:14.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:14 smithi039 ceph-mon[106470]: osdmap e1263: 8 total, 8 up, 8 in 2024-09-14T04:36:14.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:14 smithi039 ceph-mon[111235]: osdmap e1263: 8 total, 8 up, 8 in 2024-09-14T04:36:15.156 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/1.DiffIterateTruncate (3062 ms) 2024-09-14T04:36:15.156 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/1.DiffIterateWriteAndTruncate 2024-09-14T04:36:15.157 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:36:15.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:15 smithi129 ceph-mon[101316]: pgmap v2031: 105 pgs: 105 active+clean; 336 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 73 KiB/s rd, 1.0 MiB/s wr, 112 op/s 2024-09-14T04:36:15.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:15 smithi129 ceph-mon[101316]: osdmap e1264: 8 total, 8 up, 8 in 2024-09-14T04:36:15.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:15 smithi039 ceph-mon[106470]: pgmap v2031: 105 pgs: 105 active+clean; 336 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 73 KiB/s rd, 1.0 MiB/s wr, 112 op/s 2024-09-14T04:36:15.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:15 smithi039 ceph-mon[106470]: osdmap e1264: 8 total, 8 up, 8 in 2024-09-14T04:36:15.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:15 smithi039 ceph-mon[111235]: pgmap v2031: 105 pgs: 105 active+clean; 336 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 73 KiB/s rd, 1.0 MiB/s wr, 112 op/s 2024-09-14T04:36:15.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:15 smithi039 ceph-mon[111235]: osdmap e1264: 8 total, 8 up, 8 in 2024-09-14T04:36:16.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:16 smithi129 ceph-mon[101316]: osdmap e1265: 8 total, 8 up, 8 in 2024-09-14T04:36:16.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:16 smithi039 ceph-mon[106470]: osdmap e1265: 8 total, 8 up, 8 in 2024-09-14T04:36:16.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:16 smithi039 ceph-mon[111235]: osdmap e1265: 8 total, 8 up, 8 in 2024-09-14T04:36:17.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:17 smithi129 ceph-mon[101316]: pgmap v2034: 105 pgs: 105 active+clean; 336 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 73 KiB/s rd, 1.0 MiB/s wr, 111 op/s 2024-09-14T04:36:17.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:17 smithi129 ceph-mon[101316]: osdmap e1266: 8 total, 8 up, 8 in 2024-09-14T04:36:17.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:17 smithi039 ceph-mon[106470]: pgmap v2034: 105 pgs: 105 active+clean; 336 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 73 KiB/s rd, 1.0 MiB/s wr, 111 op/s 2024-09-14T04:36:17.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:17 smithi039 ceph-mon[106470]: osdmap e1266: 8 total, 8 up, 8 in 2024-09-14T04:36:17.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:17 smithi039 ceph-mon[111235]: pgmap v2034: 105 pgs: 105 active+clean; 336 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 73 KiB/s rd, 1.0 MiB/s wr, 111 op/s 2024-09-14T04:36:17.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:17 smithi039 ceph-mon[111235]: osdmap e1266: 8 total, 8 up, 8 in 2024-09-14T04:36:18.236 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/1.DiffIterateWriteAndTruncate (3080 ms) 2024-09-14T04:36:18.236 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/1.DiffIterateStress 2024-09-14T04:36:18.236 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:36:18.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:18 smithi039 ceph-mon[106470]: osdmap e1267: 8 total, 8 up, 8 in 2024-09-14T04:36:18.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:18 smithi039 ceph-mon[111235]: osdmap e1267: 8 total, 8 up, 8 in 2024-09-14T04:36:18.313 INFO:tasks.workunit.client.0.smithi039.stdout: i=0 exists [72925159~637170,79353072~7745075,102282066~7744112,167056113~2678542,181451266~2588970,277629004~1190340,313094120~3281439,371132931~7012252,398834797~3151765,402108524~3794397] wrote [72925159~637170,79353072~7745075,102282066~7744112,167056113~2678542,181451266~2588970,277629004~1190340,313094120~3281439,371132931~7012252,398834797~3151765,402108524~3794397] 2024-09-14T04:36:18.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:18 smithi129 ceph-mon[101316]: osdmap e1267: 8 total, 8 up, 8 in 2024-09-14T04:36:18.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:36:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:36:18] "GET /metrics HTTP/1.1" 200 36689 "" "Prometheus/2.51.0" 2024-09-14T04:36:19.287 INFO:tasks.workunit.client.0.smithi039.stdout: i=1 exists [11887288~5662787,69638263~7887969,79353072~7745075,91242197~3834346,102282066~7744112,115398259~5671915,122936187~2471923,155719375~7486318,167056113~2678542,181451266~2588970,277629004~1190340,291620468~299131,313094120~3281439,351120982~3177352,354830695~2325758,371132931~7012252,398834797~3151765,402108524~3794397] wrote [11887288~5662787,69638263~7887969,91242197~3834346,115398259~5671915,122936187~2471923,155719375~7486318,291620468~299131,351120982~3177352,354830695~2325758] 2024-09-14T04:36:19.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:19 smithi129 ceph-mon[101316]: pgmap v2037: 105 pgs: 105 active+clean; 336 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 37 KiB/s rd, 3.0 KiB/s wr, 52 op/s 2024-09-14T04:36:19.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:19 smithi129 ceph-mon[101316]: osdmap e1268: 8 total, 8 up, 8 in 2024-09-14T04:36:19.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:19 smithi039 ceph-mon[106470]: pgmap v2037: 105 pgs: 105 active+clean; 336 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 37 KiB/s rd, 3.0 KiB/s wr, 52 op/s 2024-09-14T04:36:19.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:19 smithi039 ceph-mon[106470]: osdmap e1268: 8 total, 8 up, 8 in 2024-09-14T04:36:19.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:19 smithi039 ceph-mon[111235]: pgmap v2037: 105 pgs: 105 active+clean; 336 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 37 KiB/s rd, 3.0 KiB/s wr, 52 op/s 2024-09-14T04:36:19.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:19 smithi039 ceph-mon[111235]: osdmap e1268: 8 total, 8 up, 8 in 2024-09-14T04:36:20.282 INFO:tasks.workunit.client.0.smithi039.stdout: i=2 exists [11887288~5662787,56141750~5694441,69638263~7887969,79353072~7745075,91242197~3834346,102282066~7744112,115398259~5671915,122936187~2471923,134881109~5724314,155719375~7486318,167056113~2678542,181451266~2588970,224392668~6553249,258573857~2428766,277629004~1190340,291620468~299131,313094120~3281439,329714488~4075149,351031190~3267144,354830695~2325758,371132931~7012252,398834797~3151765,402108524~3794397] wrote [56141750~5694441,103288566~4835243,134881109~5724314,224392668~6553249,258573857~2428766,329714488~4075149,351031190~1224605,376921095~385229] 2024-09-14T04:36:20.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:20 smithi129 ceph-mon[101316]: osdmap e1269: 8 total, 8 up, 8 in 2024-09-14T04:36:20.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:20 smithi039 ceph-mon[106470]: osdmap e1269: 8 total, 8 up, 8 in 2024-09-14T04:36:20.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:20 smithi039 ceph-mon[111235]: osdmap e1269: 8 total, 8 up, 8 in 2024-09-14T04:36:21.303 INFO:tasks.workunit.client.0.smithi039.stdout: i=3 exists [11887288~10521620,32687506~7710394,41114705~3744162,56141750~5694441,64240513~1576117,67559229~9967003,79353072~7745075,91242197~3834346,102282066~7744112,115398259~5671915,122936187~2471923,134881109~5724314,155719375~7486318,167056113~2678542,181451266~2588970,224392668~6553249,258573857~7960829,277629004~1190340,282563342~1638386,291620468~299131,294990075~2103191,313094120~3281439,329714488~4075149,351031190~3267144,354830695~2325758,371132931~7012252,398834797~3151765,402108524~3794397] wrote [14355543~8053365,32687506~7710394,41114705~3744162,64240513~1576117,67559229~6232661,260748931~5785755,282563342~1638386,294990075~2103191] 2024-09-14T04:36:21.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:21 smithi129 ceph-mon[101316]: pgmap v2040: 105 pgs: 105 active+clean; 347 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 72 KiB/s rd, 2.4 MiB/s wr, 110 op/s 2024-09-14T04:36:21.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:21 smithi129 ceph-mon[101316]: osdmap e1270: 8 total, 8 up, 8 in 2024-09-14T04:36:21.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:21 smithi039 ceph-mon[106470]: pgmap v2040: 105 pgs: 105 active+clean; 347 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 72 KiB/s rd, 2.4 MiB/s wr, 110 op/s 2024-09-14T04:36:21.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:21 smithi039 ceph-mon[106470]: osdmap e1270: 8 total, 8 up, 8 in 2024-09-14T04:36:21.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:21 smithi039 ceph-mon[111235]: pgmap v2040: 105 pgs: 105 active+clean; 347 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 72 KiB/s rd, 2.4 MiB/s wr, 110 op/s 2024-09-14T04:36:21.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:21 smithi039 ceph-mon[111235]: osdmap e1270: 8 total, 8 up, 8 in 2024-09-14T04:36:22.299 INFO:tasks.workunit.client.0.smithi039.stdout: i=4 exists [4110412~5651722,11887288~10521620,25701809~5373875,32687506~7710394,41114705~3744162,48457049~2187385,56141750~5694441,64240513~1576117,67559229~9967003,78390235~8707912,91242197~3834346,100435587~9590591,115398259~5671915,122936187~2471923,134881109~5724314,155719375~7486318,167056113~2678542,181451266~2588970,194065808~412513,196743392~7209431,214872430~5811099,224392668~6553249,258573857~7960829,277629004~1190340,282563342~1638386,291620468~299131,294990075~2103191,313094120~3281439,329714488~4075149,351031190~3267144,354830695~2325758,371132931~7012252,382942458~1112875,398834797~3151765,402108524~3794397] wrote [4110412~5651722,25701809~5373875,48457049~2187385,78390235~1955101,100435587~5958906,194065808~412513,196743392~7209431,214872430~5811099,382942458~1112875] 2024-09-14T04:36:22.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:22 smithi129 ceph-mon[101316]: osdmap e1271: 8 total, 8 up, 8 in 2024-09-14T04:36:22.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:22 smithi039 ceph-mon[106470]: osdmap e1271: 8 total, 8 up, 8 in 2024-09-14T04:36:22.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:22 smithi039 ceph-mon[111235]: osdmap e1271: 8 total, 8 up, 8 in 2024-09-14T04:36:23.305 INFO:tasks.workunit.client.0.smithi039.stdout: i=5 exists [4110412~5651722,11887288~10521620,25701809~5373875,32687506~7710394,41114705~3744162,48457049~2187385,50695614~11140577,64240513~1576117,67559229~9967003,78390235~8707912,91242197~3834346,100435587~9590591,115398259~5671915,122936187~2471923,129604318~11001105,141877568~2164147,155719375~7486318,167056113~2678542,181451266~2588970,194065808~412513,196743392~7209431,214872430~5811099,224392668~6553249,251065550~15469136,277629004~1190340,282563342~1638386,291620468~299131,294990075~2103191,308611242~11387100,329714488~4075149,351031190~3267144,354830695~2325758,362486986~885579,365000431~13144752,382942458~1112875,397823802~4162760,402108524~3794397] wrote [50695614~7567906,94142152~280222,129604318~7007834,141877568~2164147,251065550~7830772,308611242~11387100,362486986~885579,365000431~6265164,397823802~2154897] 2024-09-14T04:36:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:23 smithi129 ceph-mon[101316]: pgmap v2043: 105 pgs: 105 active+clean; 352 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 41 KiB/s rd, 3.5 MiB/s wr, 72 op/s 2024-09-14T04:36:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:23 smithi129 ceph-mon[101316]: osdmap e1272: 8 total, 8 up, 8 in 2024-09-14T04:36:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:36:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:23 smithi039 ceph-mon[106470]: pgmap v2043: 105 pgs: 105 active+clean; 352 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 41 KiB/s rd, 3.5 MiB/s wr, 72 op/s 2024-09-14T04:36:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:23 smithi039 ceph-mon[106470]: osdmap e1272: 8 total, 8 up, 8 in 2024-09-14T04:36:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:36:23.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:23 smithi039 ceph-mon[111235]: pgmap v2043: 105 pgs: 105 active+clean; 352 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 41 KiB/s rd, 3.5 MiB/s wr, 72 op/s 2024-09-14T04:36:23.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:23 smithi039 ceph-mon[111235]: osdmap e1272: 8 total, 8 up, 8 in 2024-09-14T04:36:23.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:36:24.312 INFO:tasks.workunit.client.0.smithi039.stdout: i=6 exists [4110412~5651722,11887288~10521620,25427962~14969938,40905142~3953725,48457049~2187385,50695614~11140577,64240513~1576117,67559229~9967003,78390235~8707912,91242197~3834346,100435587~9590591,115398259~5671915,122936187~2471923,129604318~11001105,141250032~2791683,155719375~7486318,167056113~2678542,181451266~2588970,185135672~6941871,194065808~412513,196743392~7209431,214872430~5811099,224392668~6553249,247445634~1215233,251065550~15469136,277629004~1190340,282563342~1638386,291620468~299131,294990075~2103191,308611242~11387100,329714488~4075149,351031190~3267144,354830695~2325758,362486986~885579,365000431~13144752,382942458~1112875,384628196~1140403,397823802~4162760,402108524~7542603] wrote [25427962~7301821,40905142~3691208,72247829~1590853,81815080~4730940,141250032~879831,185135672~6941871,215216156~4160446,247445634~1215233,384628196~1140403,405544240~4106887] 2024-09-14T04:36:24.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:24 smithi129 ceph-mon[101316]: osdmap e1273: 8 total, 8 up, 8 in 2024-09-14T04:36:24.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:24 smithi039 ceph-mon[106470]: osdmap e1273: 8 total, 8 up, 8 in 2024-09-14T04:36:24.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:24 smithi039 ceph-mon[111235]: osdmap e1273: 8 total, 8 up, 8 in 2024-09-14T04:36:25.337 INFO:tasks.workunit.client.0.smithi039.stdout: i=7 exists [4110412~5651722,11887288~10521620,25427962~14969938,40905142~3953725,48457049~2187385,50695614~11140577,64240513~1576117,67559229~9967003,78390235~8707912,91242197~3834346,100435587~9590591,115398259~5671915,122936187~2471923,129604318~11001105,141250032~2791683,155719375~7486318,167056113~2678542,175690063~4797849,181451266~2588970,185135672~6941871,194065808~412513,196695121~9735800,214872430~5811099,224392668~6553249,247445634~1215233,251065550~15469136,277629004~1190340,282563342~1638386,291620468~299131,294990075~2103191,308611242~11387100,329714488~4075149,335935454~742496,346324021~153743,351031190~3267144,354830695~2325758,359400736~4485830,365000431~13144752,382942458~1112875,384628196~1140403,396357712~13293415] wrote [29749860~6049820,76316753~618095,175690063~4797849,196695121~4189363,202523965~3906956,309467376~945910,335935454~742496,346324021~153743,359400736~4485830,396357712~8064732] 2024-09-14T04:36:25.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:25 smithi129 ceph-mon[101316]: pgmap v2046: 105 pgs: 105 active+clean; 547 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 159 KiB/s rd, 39 MiB/s wr, 271 op/s 2024-09-14T04:36:25.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:25 smithi129 ceph-mon[101316]: osdmap e1274: 8 total, 8 up, 8 in 2024-09-14T04:36:25.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:25 smithi039 ceph-mon[106470]: pgmap v2046: 105 pgs: 105 active+clean; 547 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 159 KiB/s rd, 39 MiB/s wr, 271 op/s 2024-09-14T04:36:25.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:25 smithi039 ceph-mon[106470]: osdmap e1274: 8 total, 8 up, 8 in 2024-09-14T04:36:25.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:25 smithi039 ceph-mon[111235]: pgmap v2046: 105 pgs: 105 active+clean; 547 MiB data, 1.7 GiB used, 714 GiB / 715 GiB avail; 159 KiB/s rd, 39 MiB/s wr, 271 op/s 2024-09-14T04:36:25.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:25 smithi039 ceph-mon[111235]: osdmap e1274: 8 total, 8 up, 8 in 2024-09-14T04:36:26.327 INFO:tasks.workunit.client.0.smithi039.stdout: i=8 exists [4110412~5651722,11887288~10521620,25427962~14969938,40905142~3953725,48457049~2187385,50695614~11140577,64240513~1576117,67559229~9967003,78390235~8749027,88509529~729509,91242197~3834346,100435587~11422819,115398259~5671915,122936187~2471923,127576318~13029105,141250032~2791683,146257963~2391747,155719375~7486318,164746663~1850949,167056113~2678542,175690063~4797849,181451266~2588970,185135672~6941871,194065808~412513,196695121~10516891,214872430~5811099,224392668~6553249,247445634~1215233,251065550~15469136,277629004~1190340,282563342~1638386,291620468~299131,294990075~2103191,308611242~11387100,329714488~4075149,335935454~742496,345526626~5202311,351031190~3267144,354830695~2325758,359400736~4485830,365000431~13144752,382942458~1112875,384628196~1140403,396357712~13419002] wrote [27425710~120230,79341514~7797748,88509529~729509,105205841~6652565,127576318~7171026,146257963~2391747,164746663~1850949,204811839~2400173,345526626~5202311,404879055~4897659] 2024-09-14T04:36:26.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:26 smithi129 ceph-mon[101316]: osdmap e1275: 8 total, 8 up, 8 in 2024-09-14T04:36:26.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:26 smithi039 ceph-mon[106470]: osdmap e1275: 8 total, 8 up, 8 in 2024-09-14T04:36:26.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:26 smithi039 ceph-mon[111235]: osdmap e1275: 8 total, 8 up, 8 in 2024-09-14T04:36:27.379 INFO:tasks.workunit.client.0.smithi039.stdout: i=9 exists [4110412~5651722,11887288~10521620,25427962~14969938,40905142~3953725,48198075~13638116,64240513~1576117,67559229~9967003,78390235~10848803,91242197~3834346,100435587~11422819,115398259~5671915,122936187~2471923,127576318~17965159,146257963~2391747,155719375~7486318,164746663~1850949,167056113~2678542,175690063~4797849,181451266~2588970,185135672~6941871,194065808~412513,196260384~10951628,214872430~5811099,224392668~6553249,247445634~1215233,251065550~17567676,277629004~1190340,282563342~1638386,291620468~299131,294990075~2103191,308611242~11387100,329714488~4075149,335935454~742496,345526626~5202311,351031190~3267144,354830695~2325758,359400736~4485830,365000431~13144752,382942458~1112875,384628196~1140403,396357712~13419002] wrote [25638602~7235385,48198075~5977188,81955714~6782964,108776093~942379,110273232~269454,138532686~7008791,196260384~2983987,261406263~7226963,332500894~768966] 2024-09-14T04:36:27.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:27 smithi129 ceph-mon[101316]: pgmap v2049: 105 pgs: 105 active+clean; 589 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 153 KiB/s rd, 46 MiB/s wr, 261 op/s 2024-09-14T04:36:27.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:27 smithi129 ceph-mon[101316]: osdmap e1276: 8 total, 8 up, 8 in 2024-09-14T04:36:27.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:27 smithi039 ceph-mon[106470]: pgmap v2049: 105 pgs: 105 active+clean; 589 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 153 KiB/s rd, 46 MiB/s wr, 261 op/s 2024-09-14T04:36:27.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:27 smithi039 ceph-mon[106470]: osdmap e1276: 8 total, 8 up, 8 in 2024-09-14T04:36:27.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:27 smithi039 ceph-mon[111235]: pgmap v2049: 105 pgs: 105 active+clean; 589 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 153 KiB/s rd, 46 MiB/s wr, 261 op/s 2024-09-14T04:36:27.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:27 smithi039 ceph-mon[111235]: osdmap e1276: 8 total, 8 up, 8 in 2024-09-14T04:36:28.362 INFO:tasks.workunit.client.0.smithi039.stdout: i=10 exists [4110412~5651722,11887288~10521620,23978254~16419646,40905142~3953725,48198075~13638116,64240513~1576117,67559229~9967003,78390235~10848803,91242197~3834346,100435587~11422819,115398259~5671915,122936187~2471923,127576318~23287192,155719375~7486318,164746663~1850949,167056113~2678542,175690063~4797849,181451266~2588970,185135672~6941871,194065808~412513,195117996~12094016,214872430~5811099,224392668~6553249,243930844~6073518,251065550~17567676,277629004~1190340,282563342~1638386,291620468~299131,294990075~2103191,298906752~105619,308611242~11387100,329714488~4075149,335935454~742496,345526626~5202311,351031190~3267144,354830695~2325758,359400736~18744447,382942458~1112875,384628196~1140403,396357712~13419002] wrote [23978254~5042854,51462050~4168374,142768820~8094690,195117996~1667209,243930844~6073518,253562930~8390998,277748562~952481,298906752~105619,360359142~6713836] 2024-09-14T04:36:28.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:28 smithi039 ceph-mon[106470]: osdmap e1277: 8 total, 8 up, 8 in 2024-09-14T04:36:28.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:28 smithi039 ceph-mon[111235]: osdmap e1277: 8 total, 8 up, 8 in 2024-09-14T04:36:28.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:28 smithi129 ceph-mon[101316]: osdmap e1277: 8 total, 8 up, 8 in 2024-09-14T04:36:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:36:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:36:28] "GET /metrics HTTP/1.1" 200 36689 "" "Prometheus/2.51.0" 2024-09-14T04:36:29.381 INFO:tasks.workunit.client.0.smithi039.stdout: i=11 exists [4110412~5651722,11887288~10521620,22565018~17832882,40905142~3953725,47007192~14828999,64240513~1576117,67559229~9967003,78390235~10848803,91242197~3834346,100435587~11422819,115398259~5671915,122936187~2471923,127576318~23287192,155719375~7486318,164746663~1850949,167056113~2678542,172227548~8260364,181451266~2588970,185135672~6941871,194065808~412513,195117996~12094016,214872430~5811099,224392668~7058532,243930844~6073518,251065550~17567676,277629004~1190340,282563342~1638386,291620468~299131,294990075~2103191,298906752~105619,308611242~11387100,329714488~4075149,335935454~742496,345526626~8771708,354830695~2325758,359400736~18744447,382942458~1112875,384628196~1140403,386448093~7108729,396357712~13419002] wrote [22565018~4993711,35449085~4852098,47007192~7824634,67727480~1112374,172227548~4999299,199068232~133704,226143154~5308046,346927034~4195007,386448093~7108729] 2024-09-14T04:36:29.672 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:29 smithi129 ceph-mon[101316]: pgmap v2052: 105 pgs: 105 active+clean; 658 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 22 MiB/s wr, 96 op/s 2024-09-14T04:36:29.672 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:29 smithi129 ceph-mon[101316]: osdmap e1278: 8 total, 8 up, 8 in 2024-09-14T04:36:29.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:29 smithi039 ceph-mon[106470]: pgmap v2052: 105 pgs: 105 active+clean; 658 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 22 MiB/s wr, 96 op/s 2024-09-14T04:36:29.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:29 smithi039 ceph-mon[106470]: osdmap e1278: 8 total, 8 up, 8 in 2024-09-14T04:36:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:29 smithi039 ceph-mon[111235]: pgmap v2052: 105 pgs: 105 active+clean; 658 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 64 KiB/s rd, 22 MiB/s wr, 96 op/s 2024-09-14T04:36:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:29 smithi039 ceph-mon[111235]: osdmap e1278: 8 total, 8 up, 8 in 2024-09-14T04:36:30.407 INFO:tasks.workunit.client.0.smithi039.stdout: i=12 exists [2784308~6977826,11887288~10521620,22565018~17832882,40905142~3953725,47007192~14828999,64240513~1576117,67559229~9967003,78390235~10848803,91242197~3834346,100435587~11422819,115398259~5671915,122936187~2471923,127576318~23287192,155719375~7486318,164746663~1850949,167056113~2678542,170474801~10013111,181451266~2588970,185135672~6941871,194065808~412513,195117996~12094016,214872430~5811099,224392668~8211582,243930844~6073518,251065550~17567676,277629004~1190340,282563342~1638386,291620468~299131,294990075~2103191,298906752~105619,308611242~11387100,329714488~4075149,335935454~742496,345526626~8771708,354830695~2325758,359400736~18744447,382942458~1112875,384628196~1140403,386448093~7108729,396357712~13419002] wrote [2784308~4582638,13462117~6600254,56260242~2339572,137470913~1400849,144184284~4667160,170474801~3543328,226512296~6091954,400483671~7205559] 2024-09-14T04:36:30.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:30 smithi129 ceph-mon[101316]: osdmap e1279: 8 total, 8 up, 8 in 2024-09-14T04:36:30.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:30 smithi039 ceph-mon[106470]: osdmap e1279: 8 total, 8 up, 8 in 2024-09-14T04:36:30.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:30 smithi039 ceph-mon[111235]: osdmap e1279: 8 total, 8 up, 8 in 2024-09-14T04:36:31.408 INFO:tasks.workunit.client.0.smithi039.stdout: i=13 exists [2784308~6977826,11887288~10521620,22565018~17832882,40905142~20931049,64240513~1576117,67559229~9967003,78390235~10848803,91242197~3834346,100435587~11422819,115398259~5671915,122936187~2471923,127576318~23287192,155719375~7486318,164746663~1850949,167056113~2678542,170474801~10013111,181451266~2588970,185135672~6941871,194065808~412513,195117996~14803691,212034915~8648614,224392668~14504425,243930844~6073518,251065550~17567676,277629004~1190340,282563342~1638386,291620468~299131,294990075~2103191,298906752~105619,308611242~11387100,329714488~4075149,335935454~742496,345526626~8801346,354830695~2325758,359400736~18744447,382942458~1112875,384628196~1140403,386448093~7108729,396357712~13419002] wrote [43061289~5613086,128847729~3617344,137099307~5342730,198423233~1580221,203656826~6264861,212034915~6602642,230876465~8020628,353912731~415241,376976803~544359,400100114~2874723] 2024-09-14T04:36:31.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:31 smithi129 ceph-mon[101316]: pgmap v2055: 105 pgs: 105 active+clean; 792 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 182 KiB/s rd, 44 MiB/s wr, 297 op/s 2024-09-14T04:36:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:31 smithi129 ceph-mon[101316]: osdmap e1280: 8 total, 8 up, 8 in 2024-09-14T04:36:31.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:31 smithi039 ceph-mon[106470]: pgmap v2055: 105 pgs: 105 active+clean; 792 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 182 KiB/s rd, 44 MiB/s wr, 297 op/s 2024-09-14T04:36:31.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:31 smithi039 ceph-mon[106470]: osdmap e1280: 8 total, 8 up, 8 in 2024-09-14T04:36:31.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:31 smithi039 ceph-mon[111235]: pgmap v2055: 105 pgs: 105 active+clean; 792 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 182 KiB/s rd, 44 MiB/s wr, 297 op/s 2024-09-14T04:36:31.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:31 smithi039 ceph-mon[111235]: osdmap e1280: 8 total, 8 up, 8 in 2024-09-14T04:36:32.428 INFO:tasks.workunit.client.0.smithi039.stdout: i=14 exists [2784308~6977826,11887288~10521620,22565018~17832882,40905142~24911488,67559229~9967003,78390235~10848803,91242197~3834346,100435587~11422819,115398259~5671915,122936187~2471923,127576318~23287192,151793584~2811596,155719375~7486318,164746663~1850949,167056113~2678542,170474801~10013111,181451266~2588970,185135672~6941871,194065808~412513,195117996~14803691,212034915~8648614,224392668~14504425,243930844~6073518,251065550~17567676,271800130~8125645,282563342~1638386,291620468~299131,294990075~2103191,298906752~105619,308611242~11387100,322843953~10945684,335935454~742496,345526626~32618557,382942458~1112875,384628196~1140403,386448093~7108729,396357712~14708164] wrote [18955335~1913564,29691367~850074,41673045~6120789,59706226~5012685,151793584~2811596,271800130~8125645,322843953~8188296,353598562~6460537,403457931~7607945] 2024-09-14T04:36:32.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:32 smithi129 ceph-mon[101316]: osdmap e1281: 8 total, 8 up, 8 in 2024-09-14T04:36:32.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:32 smithi039 ceph-mon[106470]: osdmap e1281: 8 total, 8 up, 8 in 2024-09-14T04:36:32.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:32 smithi039 ceph-mon[111235]: osdmap e1281: 8 total, 8 up, 8 in 2024-09-14T04:36:33.437 INFO:tasks.workunit.client.0.smithi039.stdout: i=15 exists [2784308~19624600,22565018~17832882,40905142~24911488,67559229~9967003,78390235~10848803,91242197~3834346,100435587~11422819,115398259~5671915,122936187~2471923,127576318~23287192,151793584~11412109,164746663~1850949,167056113~2678542,170474801~10013111,181451266~2588970,185135672~6941871,192403686~4032,194065808~412513,195117996~14803691,212034915~8648614,224392668~14504425,243930844~6073518,251065550~17567676,271800130~8125645,282563342~1638386,291620468~299131,294990075~2103191,298906752~105619,308611242~11387100,322843953~10945684,335935454~742496,345526626~32618557,382153463~3622002,386448093~24617783] wrote [7537104~5484383,87074807~1202767,154163758~2685522,192403686~4032,195727980~6243000,255406913~3926190,382153463~3622002,386779328~1769504,392097068~5965065] 2024-09-14T04:36:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:33 smithi129 ceph-mon[101316]: pgmap v2058: 105 pgs: 105 active+clean; 794 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 31 MiB/s wr, 205 op/s 2024-09-14T04:36:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:33 smithi129 ceph-mon[101316]: osdmap e1282: 8 total, 8 up, 8 in 2024-09-14T04:36:33.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:33 smithi039 ceph-mon[106470]: pgmap v2058: 105 pgs: 105 active+clean; 794 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 31 MiB/s wr, 205 op/s 2024-09-14T04:36:33.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:33 smithi039 ceph-mon[106470]: osdmap e1282: 8 total, 8 up, 8 in 2024-09-14T04:36:33.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:33 smithi039 ceph-mon[111235]: pgmap v2058: 105 pgs: 105 active+clean; 794 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 119 KiB/s rd, 31 MiB/s wr, 205 op/s 2024-09-14T04:36:33.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:33 smithi039 ceph-mon[111235]: osdmap e1282: 8 total, 8 up, 8 in 2024-09-14T04:36:34.449 INFO:tasks.workunit.client.0.smithi039.stdout: i=16 exists [2784308~19624600,22565018~17832882,40905142~24911488,67559229~9967003,78390235~10848803,91242197~3834346,100435587~11422819,115398259~5671915,122936187~2471923,127576318~23287192,151793584~11412109,164746663~1850949,167056113~2678542,170474801~10013111,181451266~2588970,185135672~6941871,192403686~4032,194065808~412513,195117996~14803691,212034915~8648614,224392668~14504425,243930844~6073518,251065550~17567676,271800130~8125645,282563342~1638386,291620468~299131,292291687~6935580,308611242~25178395,335935454~742496,345526626~32618557,382153463~3622002,386448093~24617783] wrote [7984367~2445535,28520836~1300392,79557394~4803576,89011694~55479,292291687~6935580,313671960~13025259,356400377~7370315,373382900~2706758] 2024-09-14T04:36:34.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:34 smithi129 ceph-mon[101316]: osdmap e1283: 8 total, 8 up, 8 in 2024-09-14T04:36:34.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:34 smithi039 ceph-mon[106470]: osdmap e1283: 8 total, 8 up, 8 in 2024-09-14T04:36:34.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:34 smithi039 ceph-mon[111235]: osdmap e1283: 8 total, 8 up, 8 in 2024-09-14T04:36:35.466 INFO:tasks.workunit.client.0.smithi039.stdout: i=17 exists [2784308~19624600,22565018~17832882,40905142~24911488,67559229~9967003,78390235~10848803,91242197~3834346,100435587~11422819,115398259~5671915,122936187~2471923,127576318~23287192,151793584~17941071,170474801~10013111,181451266~2588970,185135672~6941871,192403686~4032,194065808~412513,195117996~14803691,212034915~8648614,224392668~14504425,243930844~6073518,251065550~17567676,271800130~8125645,282563342~1638386,291620468~299131,292291687~12395085,308611242~25178395,335935454~742496,345526626~32618557,382153463~3622002,386448093~24617783] wrote [14188420~4848926,27611390~6180989,49583341~1221776,55161587~5179711,76282034~323569,140456790~1142231,160150603~7985999,297968314~6718458,325138524~7369376,389007392~714198] 2024-09-14T04:36:35.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:35 smithi129 ceph-mon[101316]: pgmap v2061: 105 pgs: 105 active+clean; 957 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 211 KiB/s rd, 42 MiB/s wr, 331 op/s 2024-09-14T04:36:35.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:35 smithi129 ceph-mon[101316]: osdmap e1284: 8 total, 8 up, 8 in 2024-09-14T04:36:35.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:35 smithi039 ceph-mon[106470]: pgmap v2061: 105 pgs: 105 active+clean; 957 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 211 KiB/s rd, 42 MiB/s wr, 331 op/s 2024-09-14T04:36:35.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:35 smithi039 ceph-mon[106470]: osdmap e1284: 8 total, 8 up, 8 in 2024-09-14T04:36:35.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:35 smithi039 ceph-mon[111235]: pgmap v2061: 105 pgs: 105 active+clean; 957 MiB data, 2.7 GiB used, 712 GiB / 715 GiB avail; 211 KiB/s rd, 42 MiB/s wr, 331 op/s 2024-09-14T04:36:35.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:35 smithi039 ceph-mon[111235]: osdmap e1284: 8 total, 8 up, 8 in 2024-09-14T04:36:36.507 INFO:tasks.workunit.client.0.smithi039.stdout: i=18 exists [2784308~19624600,22565018~17832882,40905142~24911488,67559229~9967003,78390235~10848803,91242197~3834346,100435587~11422819,115398259~5671915,122936187~2471923,127576318~42158337,170474801~10013111,181451266~2588970,185135672~6941871,192403686~4032,194065808~412513,195117996~14803691,212034915~8648614,224392668~14504425,243930844~6073518,251065550~17567676,271588263~8337512,282563342~1638386,291620468~299131,292291687~12395085,308611242~25178395,335935454~742496,336742064~6933347,345526626~32618557,382153463~3622002,386448093~24617783] wrote [13508252~6068175,131177525~6799001,144883682~13246595,232121214~1265411,271588263~4174028,322251149~4978135,336742064~6933347] 2024-09-14T04:36:36.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:36 smithi129 ceph-mon[101316]: osdmap e1285: 8 total, 8 up, 8 in 2024-09-14T04:36:36.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:36 smithi129 ceph-mon[101316]: pgmap v2064: 105 pgs: 105 active+clean; 991 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 211 KiB/s rd, 50 MiB/s wr, 336 op/s 2024-09-14T04:36:36.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:36 smithi039 ceph-mon[106470]: osdmap e1285: 8 total, 8 up, 8 in 2024-09-14T04:36:36.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:36 smithi039 ceph-mon[106470]: pgmap v2064: 105 pgs: 105 active+clean; 991 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 211 KiB/s rd, 50 MiB/s wr, 336 op/s 2024-09-14T04:36:36.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:36 smithi039 ceph-mon[111235]: osdmap e1285: 8 total, 8 up, 8 in 2024-09-14T04:36:36.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:36 smithi039 ceph-mon[111235]: pgmap v2064: 105 pgs: 105 active+clean; 991 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 211 KiB/s rd, 50 MiB/s wr, 336 op/s 2024-09-14T04:36:37.506 INFO:tasks.workunit.client.0.smithi039.stdout: i=19 exists [2784308~19624600,22565018~17832882,40905142~24911488,67559229~9967003,78390235~10848803,91242197~3834346,100435587~11422819,115398259~10009851,127576318~42158337,170474801~21602742,192403686~4032,194065808~412513,195117996~14803691,212034915~8648614,224392668~14504425,243930844~6073518,251065550~17567676,271588263~8337512,282563342~1638386,291620468~299131,292291687~12395085,308611242~25178395,335935454~742496,336742064~41403119,382153463~3622002,386448093~24617783] wrote [8843266~6059610,101240114~6647693,119839974~5349904,130389988~13138224,178504952~7200410,340544484~6705872,398359971~2153798] 2024-09-14T04:36:37.691 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:37 smithi039 ceph-mon[106470]: osdmap e1286: 8 total, 8 up, 8 in 2024-09-14T04:36:37.691 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:36:37.691 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:37 smithi039 ceph-mon[111235]: osdmap e1286: 8 total, 8 up, 8 in 2024-09-14T04:36:37.692 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:36:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:37 smithi129 ceph-mon[101316]: osdmap e1286: 8 total, 8 up, 8 in 2024-09-14T04:36:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:36:38.442 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 1 diff [8388608~8388608,67108864~8388608,88080384~4194304,113246208~12582912,155189248~8388608,289406976~4194304,348127232~8388608] 2024-09-14T04:36:38.442 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [8388608~8388608,67108864~8388608,88080384~4194304,113246208~12582912,155189248~8388608,289406976~4194304,348127232~8388608] 2024-09-14T04:36:38.457 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [8388608~12582912,67108864~12582912,88080384~8388608,113246208~12582912,155189248~8388608,289406976~4194304,348127232~12582912] 2024-09-14T04:36:38.457 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 2 diff [8388608~8388608,54525952~8388608,67108864~8388608,88080384~4194304,100663296~8388608,113246208~12582912,134217728~8388608,155189248~8388608,222298112~8388608,255852544~4194304,289406976~4194304,327155712~4194304,348127232~8388608,373293056~4194304] 2024-09-14T04:36:38.457 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [8388608~8388608,54525952~8388608,67108864~8388608,88080384~4194304,100663296~8388608,113246208~12582912,134217728~8388608,155189248~8388608,222298112~8388608,255852544~4194304,289406976~4194304,327155712~4194304,348127232~8388608,373293056~4194304] 2024-09-14T04:36:38.459 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [8388608~12582912,54525952~8388608,67108864~12582912,88080384~8388608,100663296~8388608,113246208~12582912,134217728~8388608,155189248~8388608,222298112~12582912,255852544~8388608,289406976~4194304,327155712~8388608,348127232~12582912,373293056~4194304] 2024-09-14T04:36:38.459 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 3 diff [8388608~12582912,29360128~12582912,54525952~25165824,88080384~4194304,100663296~8388608,113246208~12582912,134217728~8388608,155189248~8388608,222298112~8388608,255852544~8388608,281018368~4194304,289406976~8388608,327155712~4194304,348127232~8388608,373293056~4194304] 2024-09-14T04:36:38.459 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [8388608~12582912,29360128~12582912,54525952~25165824,88080384~4194304,100663296~8388608,113246208~12582912,134217728~8388608,155189248~8388608,222298112~8388608,255852544~8388608,281018368~4194304,289406976~8388608,327155712~4194304,348127232~8388608,373293056~4194304] 2024-09-14T04:36:38.462 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [8388608~16777216,29360128~16777216,54525952~25165824,88080384~8388608,100663296~8388608,113246208~12582912,134217728~8388608,155189248~8388608,222298112~12582912,255852544~12582912,281018368~4194304,289406976~8388608,327155712~8388608,348127232~12582912,373293056~4194304] 2024-09-14T04:36:38.462 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 4 diff [0~20971520,25165824~16777216,46137344~4194304,54525952~25165824,88080384~4194304,96468992~8388608,113246208~12582912,134217728~8388608,155189248~8388608,192937984~8388608,213909504~16777216,255852544~8388608,281018368~4194304,289406976~8388608,327155712~4194304,348127232~8388608,373293056~4194304,381681664~4194304] 2024-09-14T04:36:38.463 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~20971520,25165824~16777216,46137344~4194304,54525952~25165824,88080384~4194304,96468992~8388608,113246208~12582912,134217728~8388608,155189248~8388608,192937984~8388608,213909504~16777216,255852544~8388608,281018368~4194304,289406976~8388608,327155712~4194304,348127232~8388608,373293056~4194304,381681664~4194304] 2024-09-14T04:36:38.466 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~83886080,88080384~20971520,113246208~12582912,134217728~8388608,155189248~8388608,192937984~12582912,213909504~20971520,255852544~12582912,281018368~4194304,289406976~8388608,327155712~8388608,348127232~12582912,373293056~4194304,381681664~4194304] 2024-09-14T04:36:38.466 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 5 diff [0~20971520,25165824~16777216,46137344~33554432,88080384~4194304,96468992~8388608,113246208~29360128,155189248~8388608,192937984~8388608,213909504~16777216,247463936~16777216,281018368~4194304,289406976~8388608,306184192~12582912,327155712~4194304,348127232~8388608,360710144~16777216,381681664~4194304,394264576~4194304] 2024-09-14T04:36:38.466 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~20971520,25165824~16777216,46137344~33554432,88080384~4194304,96468992~8388608,113246208~29360128,155189248~8388608,192937984~8388608,213909504~16777216,247463936~16777216,281018368~4194304,289406976~8388608,306184192~12582912,327155712~4194304,348127232~8388608,360710144~16777216,381681664~4194304,394264576~4194304] 2024-09-14T04:36:38.469 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~83886080,88080384~20971520,113246208~33554432,155189248~8388608,192937984~12582912,213909504~20971520,247463936~20971520,281018368~4194304,289406976~8388608,306184192~16777216,327155712~8388608,348127232~29360128,381681664~4194304,394264576~8388608] 2024-09-14T04:36:38.469 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 6 diff [0~20971520,25165824~16777216,46137344~46137344,96468992~8388608,113246208~29360128,155189248~8388608,184549376~16777216,213909504~16777216,243269632~20971520,281018368~4194304,289406976~8388608,306184192~12582912,327155712~4194304,348127232~8388608,360710144~16777216,381681664~4194304,394264576~4194304,402653184~4194304] 2024-09-14T04:36:38.469 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~20971520,25165824~16777216,46137344~46137344,96468992~8388608,113246208~29360128,155189248~8388608,184549376~16777216,213909504~16777216,243269632~20971520,281018368~4194304,289406976~8388608,306184192~12582912,327155712~4194304,348127232~8388608,360710144~16777216,381681664~4194304,394264576~4194304,402653184~4194304] 2024-09-14T04:36:38.474 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~109051904,113246208~33554432,155189248~8388608,184549376~20971520,213909504~20971520,243269632~25165824,281018368~4194304,289406976~8388608,306184192~16777216,327155712~8388608,348127232~29360128,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.474 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 7 diff [0~20971520,25165824~16777216,46137344~46137344,96468992~8388608,113246208~29360128,155189248~8388608,171966464~8388608,184549376~20971520,213909504~16777216,243269632~20971520,281018368~4194304,289406976~8388608,306184192~12582912,327155712~4194304,335544320~4194304,343932928~33554432,381681664~4194304,394264576~12582912] 2024-09-14T04:36:38.474 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~20971520,25165824~16777216,46137344~46137344,96468992~8388608,113246208~29360128,155189248~8388608,171966464~8388608,184549376~20971520,213909504~16777216,243269632~20971520,281018368~4194304,289406976~8388608,306184192~12582912,327155712~4194304,335544320~4194304,343932928~33554432,381681664~4194304,394264576~12582912] 2024-09-14T04:36:38.478 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~109051904,113246208~33554432,155189248~8388608,171966464~37748736,213909504~20971520,243269632~25165824,281018368~4194304,289406976~8388608,306184192~16777216,327155712~12582912,343932928~33554432,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.479 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 8 diff [0~20971520,25165824~16777216,46137344~46137344,96468992~12582912,113246208~33554432,155189248~12582912,171966464~8388608,184549376~20971520,213909504~16777216,243269632~20971520,281018368~4194304,289406976~8388608,306184192~12582912,327155712~4194304,335544320~4194304,343932928~33554432,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.479 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~20971520,25165824~16777216,46137344~46137344,96468992~12582912,113246208~33554432,155189248~12582912,171966464~8388608,184549376~20971520,213909504~16777216,243269632~20971520,281018368~4194304,289406976~8388608,306184192~12582912,327155712~4194304,335544320~4194304,343932928~33554432,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.483 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~150994944,155189248~12582912,171966464~37748736,213909504~20971520,243269632~25165824,281018368~4194304,289406976~8388608,306184192~16777216,327155712~12582912,343932928~33554432,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.483 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 9 diff [0~20971520,25165824~16777216,46137344~46137344,96468992~12582912,113246208~33554432,155189248~12582912,171966464~8388608,184549376~20971520,213909504~16777216,243269632~25165824,281018368~4194304,289406976~8388608,306184192~12582912,327155712~4194304,335544320~4194304,343932928~33554432,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.483 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~20971520,25165824~16777216,46137344~46137344,96468992~12582912,113246208~33554432,155189248~12582912,171966464~8388608,184549376~20971520,213909504~16777216,243269632~25165824,281018368~4194304,289406976~8388608,306184192~12582912,327155712~4194304,335544320~4194304,343932928~33554432,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.488 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~150994944,155189248~12582912,171966464~37748736,213909504~20971520,243269632~29360128,281018368~4194304,289406976~8388608,306184192~16777216,327155712~12582912,343932928~33554432,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.489 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 10 diff [0~41943040,46137344~46137344,96468992~12582912,113246208~37748736,155189248~12582912,171966464~8388608,184549376~20971520,213909504~16777216,243269632~25165824,276824064~8388608,289406976~12582912,306184192~12582912,327155712~4194304,335544320~4194304,343932928~25165824,373293056~4194304,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.489 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~41943040,46137344~46137344,96468992~12582912,113246208~37748736,155189248~12582912,171966464~8388608,184549376~20971520,213909504~16777216,243269632~25165824,276824064~8388608,289406976~12582912,306184192~12582912,327155712~4194304,335544320~4194304,343932928~25165824,373293056~4194304,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.494 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~150994944,155189248~12582912,171966464~37748736,213909504~20971520,243269632~29360128,276824064~8388608,289406976~12582912,306184192~16777216,327155712~12582912,343932928~33554432,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.494 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 11 diff [0~41943040,46137344~46137344,96468992~12582912,113246208~37748736,155189248~12582912,171966464~8388608,184549376~20971520,213909504~16777216,243269632~25165824,276824064~8388608,289406976~12582912,306184192~12582912,327155712~4194304,335544320~4194304,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.494 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~41943040,46137344~46137344,96468992~12582912,113246208~37748736,155189248~12582912,171966464~8388608,184549376~20971520,213909504~16777216,243269632~25165824,276824064~8388608,289406976~12582912,306184192~12582912,327155712~4194304,335544320~4194304,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.500 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~150994944,155189248~12582912,171966464~37748736,213909504~20971520,243269632~29360128,276824064~8388608,289406976~12582912,306184192~16777216,327155712~12582912,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.500 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 12 diff [0~41943040,46137344~46137344,96468992~12582912,113246208~37748736,155189248~25165824,184549376~20971520,213909504~16777216,243269632~25165824,276824064~8388608,289406976~12582912,306184192~12582912,327155712~4194304,335544320~4194304,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.500 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~41943040,46137344~46137344,96468992~12582912,113246208~37748736,155189248~25165824,184549376~20971520,213909504~16777216,243269632~25165824,276824064~8388608,289406976~12582912,306184192~12582912,327155712~4194304,335544320~4194304,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.507 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~150994944,155189248~54525952,213909504~20971520,243269632~29360128,276824064~8388608,289406976~12582912,306184192~16777216,327155712~12582912,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.507 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 13 diff [0~58720256,62914560~29360128,96468992~12582912,113246208~37748736,155189248~25165824,184549376~54525952,243269632~25165824,276824064~8388608,289406976~12582912,306184192~12582912,327155712~4194304,335544320~4194304,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.508 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~58720256,62914560~29360128,96468992~12582912,113246208~37748736,155189248~25165824,184549376~54525952,243269632~25165824,276824064~8388608,289406976~12582912,306184192~12582912,327155712~4194304,335544320~4194304,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.514 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~150994944,155189248~83886080,243269632~29360128,276824064~8388608,289406976~12582912,306184192~16777216,327155712~12582912,343932928~67108864] 2024-09-14T04:36:38.514 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 14 diff [0~62914560,67108864~25165824,96468992~12582912,113246208~67108864,184549376~54525952,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~4194304,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.514 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~25165824,96468992~12582912,113246208~67108864,184549376~54525952,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~4194304,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.522 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~239075328,243269632~41943040,289406976~12582912,306184192~33554432,343932928~71303168] 2024-09-14T04:36:38.523 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 15 diff [0~62914560,67108864~25165824,96468992~12582912,113246208~67108864,184549376~54525952,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~4194304,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.523 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~25165824,96468992~12582912,113246208~67108864,184549376~54525952,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~4194304,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.529 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~239075328,243269632~41943040,289406976~12582912,306184192~33554432,343932928~71303168] 2024-09-14T04:36:38.529 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 16 diff [0~62914560,67108864~25165824,96468992~12582912,113246208~67108864,184549376~54525952,243269632~33554432,281018368~4194304,289406976~8388608,306184192~33554432,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.530 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~25165824,96468992~12582912,113246208~67108864,184549376~54525952,243269632~33554432,281018368~4194304,289406976~8388608,306184192~33554432,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.537 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~239075328,243269632~41943040,289406976~12582912,306184192~33554432,343932928~71303168] 2024-09-14T04:36:38.538 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 17 diff [0~62914560,67108864~25165824,96468992~12582912,113246208~67108864,184549376~54525952,243269632~33554432,281018368~4194304,289406976~12582912,306184192~33554432,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.538 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~25165824,96468992~12582912,113246208~67108864,184549376~54525952,243269632~33554432,281018368~4194304,289406976~12582912,306184192~33554432,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.546 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~239075328,243269632~41943040,289406976~50331648,343932928~71303168] 2024-09-14T04:36:38.546 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 18 diff [0~62914560,67108864~25165824,96468992~12582912,113246208~67108864,184549376~54525952,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:38.546 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~25165824,96468992~12582912,113246208~67108864,184549376~54525952,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:38.554 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~239075328,243269632~41943040,289406976~125829120] 2024-09-14T04:36:38.554 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to 19 diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:38.554 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:38.564 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~239075328,243269632~41943040,289406976~125829120] 2024-09-14T04:36:38.564 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 2 diff [54525952~8388608,100663296~8388608,134217728~8388608,222298112~8388608,255852544~4194304,327155712~4194304,348127232~4194304,373293056~4194304] 2024-09-14T04:36:38.564 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [54525952~8388608,100663296~8388608,134217728~8388608,222298112~8388608,255852544~4194304,327155712~4194304,348127232~4194304,373293056~4194304] 2024-09-14T04:36:38.565 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [54525952~8388608,100663296~8388608,134217728~8388608,222298112~12582912,255852544~8388608,327155712~8388608,348127232~4194304,373293056~4194304] 2024-09-14T04:36:38.565 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 3 diff [12582912~8388608,29360128~12582912,54525952~20971520,100663296~8388608,134217728~8388608,222298112~8388608,255852544~8388608,281018368~4194304,293601280~4194304,327155712~4194304,348127232~4194304,373293056~4194304] 2024-09-14T04:36:38.565 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [12582912~8388608,29360128~12582912,54525952~20971520,100663296~8388608,134217728~8388608,222298112~8388608,255852544~8388608,281018368~4194304,293601280~4194304,327155712~4194304,348127232~4194304,373293056~4194304] 2024-09-14T04:36:38.566 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~12582912,29360128~16777216,54525952~20971520,100663296~8388608,134217728~8388608,222298112~12582912,255852544~12582912,281018368~4194304,293601280~4194304,327155712~8388608,348127232~4194304,373293056~4194304] 2024-09-14T04:36:38.566 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 4 diff [0~8388608,12582912~8388608,25165824~16777216,46137344~4194304,54525952~25165824,96468992~8388608,134217728~8388608,192937984~8388608,213909504~16777216,255852544~8388608,281018368~4194304,293601280~4194304,327155712~4194304,348127232~4194304,373293056~4194304,381681664~4194304] 2024-09-14T04:36:38.566 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~8388608,25165824~16777216,46137344~4194304,54525952~25165824,96468992~8388608,134217728~8388608,192937984~8388608,213909504~16777216,255852544~8388608,281018368~4194304,293601280~4194304,327155712~4194304,348127232~4194304,373293056~4194304,381681664~4194304] 2024-09-14T04:36:38.568 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~83886080,96468992~12582912,134217728~8388608,192937984~12582912,213909504~20971520,255852544~12582912,281018368~4194304,293601280~4194304,327155712~8388608,348127232~4194304,373293056~4194304,381681664~4194304] 2024-09-14T04:36:38.568 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 5 diff [0~8388608,12582912~8388608,25165824~16777216,46137344~33554432,92274688~12582912,125829120~16777216,192937984~8388608,213909504~16777216,247463936~16777216,281018368~4194304,293601280~4194304,306184192~12582912,327155712~4194304,348127232~4194304,360710144~16777216,381681664~4194304,394264576~4194304] 2024-09-14T04:36:38.569 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~8388608,25165824~16777216,46137344~33554432,92274688~12582912,125829120~16777216,192937984~8388608,213909504~16777216,247463936~16777216,281018368~4194304,293601280~4194304,306184192~12582912,327155712~4194304,348127232~4194304,360710144~16777216,381681664~4194304,394264576~4194304] 2024-09-14T04:36:38.571 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~83886080,92274688~16777216,125829120~20971520,192937984~12582912,213909504~20971520,247463936~20971520,281018368~4194304,293601280~4194304,306184192~16777216,327155712~8388608,348127232~4194304,360710144~16777216,381681664~4194304,394264576~8388608] 2024-09-14T04:36:38.571 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 6 diff [0~8388608,12582912~8388608,25165824~16777216,46137344~41943040,92274688~12582912,125829120~16777216,184549376~16777216,213909504~16777216,243269632~20971520,281018368~4194304,293601280~4194304,306184192~12582912,327155712~4194304,348127232~4194304,360710144~16777216,381681664~4194304,394264576~4194304,402653184~4194304] 2024-09-14T04:36:38.571 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~8388608,25165824~16777216,46137344~41943040,92274688~12582912,125829120~16777216,184549376~16777216,213909504~16777216,243269632~20971520,281018368~4194304,293601280~4194304,306184192~12582912,327155712~4194304,348127232~4194304,360710144~16777216,381681664~4194304,394264576~4194304,402653184~4194304] 2024-09-14T04:36:38.573 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~88080384,92274688~16777216,125829120~20971520,184549376~20971520,213909504~20971520,243269632~25165824,281018368~4194304,293601280~4194304,306184192~16777216,327155712~8388608,348127232~4194304,360710144~16777216,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.574 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 7 diff [0~8388608,12582912~8388608,25165824~16777216,46137344~41943040,92274688~12582912,125829120~16777216,171966464~8388608,184549376~20971520,213909504~16777216,243269632~20971520,281018368~4194304,293601280~4194304,306184192~12582912,327155712~4194304,335544320~4194304,343932928~8388608,356515840~20971520,381681664~4194304,394264576~12582912] 2024-09-14T04:36:38.574 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~8388608,25165824~16777216,46137344~41943040,92274688~12582912,125829120~16777216,171966464~8388608,184549376~20971520,213909504~16777216,243269632~20971520,281018368~4194304,293601280~4194304,306184192~12582912,327155712~4194304,335544320~4194304,343932928~8388608,356515840~20971520,381681664~4194304,394264576~12582912] 2024-09-14T04:36:38.576 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~88080384,92274688~16777216,125829120~20971520,171966464~37748736,213909504~20971520,243269632~25165824,281018368~4194304,293601280~4194304,306184192~16777216,327155712~12582912,343932928~8388608,356515840~20971520,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.576 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 8 diff [0~8388608,12582912~8388608,25165824~16777216,46137344~62914560,125829120~20971520,163577856~4194304,171966464~8388608,184549376~20971520,213909504~16777216,243269632~20971520,281018368~4194304,293601280~4194304,306184192~12582912,327155712~4194304,335544320~4194304,343932928~8388608,356515840~20971520,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.577 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~8388608,25165824~16777216,46137344~62914560,125829120~20971520,163577856~4194304,171966464~8388608,184549376~20971520,213909504~16777216,243269632~20971520,281018368~4194304,293601280~4194304,306184192~12582912,327155712~4194304,335544320~4194304,343932928~8388608,356515840~20971520,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.581 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,125829120~25165824,163577856~4194304,171966464~37748736,213909504~20971520,243269632~25165824,281018368~4194304,293601280~4194304,306184192~16777216,327155712~12582912,343932928~8388608,356515840~20971520,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.581 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 9 diff [0~8388608,12582912~8388608,25165824~16777216,46137344~41943040,92274688~16777216,125829120~20971520,163577856~4194304,171966464~8388608,184549376~20971520,213909504~16777216,243269632~25165824,281018368~4194304,293601280~4194304,306184192~12582912,327155712~4194304,335544320~4194304,343932928~8388608,356515840~20971520,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.581 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~8388608,25165824~16777216,46137344~41943040,92274688~16777216,125829120~20971520,163577856~4194304,171966464~8388608,184549376~20971520,213909504~16777216,243269632~25165824,281018368~4194304,293601280~4194304,306184192~12582912,327155712~4194304,335544320~4194304,343932928~8388608,356515840~20971520,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.585 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,125829120~25165824,163577856~4194304,171966464~37748736,213909504~20971520,243269632~29360128,281018368~4194304,293601280~4194304,306184192~16777216,327155712~12582912,343932928~8388608,356515840~20971520,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.586 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 10 diff [0~8388608,12582912~29360128,46137344~41943040,92274688~16777216,125829120~25165824,163577856~4194304,171966464~8388608,184549376~20971520,213909504~16777216,243269632~25165824,276824064~8388608,293601280~8388608,306184192~12582912,327155712~4194304,335544320~4194304,343932928~8388608,356515840~12582912,373293056~4194304,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.586 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~29360128,46137344~41943040,92274688~16777216,125829120~25165824,163577856~4194304,171966464~8388608,184549376~20971520,213909504~16777216,243269632~25165824,276824064~8388608,293601280~8388608,306184192~12582912,327155712~4194304,335544320~4194304,343932928~8388608,356515840~12582912,373293056~4194304,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.590 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,125829120~25165824,163577856~4194304,171966464~37748736,213909504~20971520,243269632~29360128,276824064~8388608,293601280~8388608,306184192~16777216,327155712~12582912,343932928~8388608,356515840~20971520,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.590 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 11 diff [0~8388608,12582912~29360128,46137344~41943040,92274688~16777216,125829120~25165824,163577856~4194304,171966464~8388608,184549376~20971520,213909504~16777216,243269632~25165824,276824064~8388608,293601280~8388608,306184192~12582912,327155712~4194304,335544320~4194304,343932928~8388608,356515840~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.590 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~29360128,46137344~41943040,92274688~16777216,125829120~25165824,163577856~4194304,171966464~8388608,184549376~20971520,213909504~16777216,243269632~25165824,276824064~8388608,293601280~8388608,306184192~12582912,327155712~4194304,335544320~4194304,343932928~8388608,356515840~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.594 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,125829120~25165824,163577856~4194304,171966464~37748736,213909504~20971520,243269632~29360128,276824064~8388608,293601280~8388608,306184192~16777216,327155712~12582912,343932928~8388608,356515840~20971520,381681664~29360128] 2024-09-14T04:36:38.594 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 12 diff [0~8388608,12582912~29360128,46137344~41943040,92274688~16777216,125829120~25165824,163577856~16777216,184549376~20971520,213909504~16777216,243269632~25165824,276824064~8388608,293601280~8388608,306184192~12582912,327155712~4194304,335544320~4194304,343932928~8388608,356515840~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.594 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~29360128,46137344~41943040,92274688~16777216,125829120~25165824,163577856~16777216,184549376~20971520,213909504~16777216,243269632~25165824,276824064~8388608,293601280~8388608,306184192~12582912,327155712~4194304,335544320~4194304,343932928~8388608,356515840~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.600 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,125829120~25165824,163577856~46137344,213909504~20971520,243269632~29360128,276824064~8388608,293601280~8388608,306184192~16777216,327155712~12582912,343932928~8388608,356515840~20971520,381681664~29360128] 2024-09-14T04:36:38.600 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 13 diff [0~8388608,12582912~46137344,62914560~25165824,92274688~16777216,125829120~25165824,163577856~16777216,184549376~54525952,243269632~25165824,276824064~8388608,293601280~8388608,306184192~12582912,327155712~4194304,335544320~4194304,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.600 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~46137344,62914560~25165824,92274688~16777216,125829120~25165824,163577856~16777216,184549376~54525952,243269632~25165824,276824064~8388608,293601280~8388608,306184192~12582912,327155712~4194304,335544320~4194304,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.606 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,125829120~25165824,163577856~75497472,243269632~29360128,276824064~8388608,293601280~8388608,306184192~16777216,327155712~12582912,343932928~67108864] 2024-09-14T04:36:38.606 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 14 diff [0~8388608,12582912~50331648,67108864~20971520,92274688~16777216,125829120~29360128,163577856~16777216,184549376~54525952,243269632~33554432,281018368~4194304,293601280~8388608,306184192~25165824,335544320~4194304,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.606 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~50331648,67108864~20971520,92274688~16777216,125829120~29360128,163577856~16777216,184549376~54525952,243269632~33554432,281018368~4194304,293601280~8388608,306184192~25165824,335544320~4194304,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.612 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,125829120~29360128,163577856~75497472,243269632~41943040,293601280~8388608,306184192~33554432,343932928~71303168] 2024-09-14T04:36:38.612 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 15 diff [0~62914560,67108864~20971520,92274688~16777216,125829120~33554432,163577856~16777216,184549376~54525952,243269632~33554432,281018368~4194304,293601280~8388608,306184192~25165824,335544320~4194304,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.612 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,125829120~33554432,163577856~16777216,184549376~54525952,243269632~33554432,281018368~4194304,293601280~8388608,306184192~25165824,335544320~4194304,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.618 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,125829120~33554432,163577856~75497472,243269632~41943040,293601280~8388608,306184192~33554432,343932928~71303168] 2024-09-14T04:36:38.618 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 16 diff [0~62914560,67108864~20971520,92274688~16777216,125829120~33554432,163577856~16777216,184549376~54525952,243269632~33554432,281018368~4194304,289406976~8388608,306184192~33554432,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.618 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,125829120~33554432,163577856~16777216,184549376~54525952,243269632~33554432,281018368~4194304,289406976~8388608,306184192~33554432,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.625 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,125829120~33554432,163577856~75497472,243269632~41943040,289406976~12582912,306184192~33554432,343932928~71303168] 2024-09-14T04:36:38.625 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 17 diff [0~62914560,67108864~20971520,92274688~16777216,125829120~54525952,184549376~54525952,243269632~33554432,281018368~4194304,289406976~12582912,306184192~33554432,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.625 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,125829120~54525952,184549376~54525952,243269632~33554432,281018368~4194304,289406976~12582912,306184192~33554432,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.631 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,125829120~113246208,243269632~41943040,289406976~50331648,343932928~71303168] 2024-09-14T04:36:38.631 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 18 diff [0~62914560,67108864~20971520,92274688~16777216,125829120~54525952,184549376~54525952,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:38.631 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,125829120~54525952,184549376~54525952,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:38.638 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,125829120~113246208,243269632~41943040,289406976~125829120] 2024-09-14T04:36:38.638 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to 19 diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:38.638 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:38.646 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:38.646 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 3 diff [12582912~8388608,29360128~12582912,62914560~12582912,260046848~8388608,281018368~4194304,293601280~4194304] 2024-09-14T04:36:38.646 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [12582912~8388608,29360128~12582912,62914560~12582912,260046848~8388608,281018368~4194304,293601280~4194304] 2024-09-14T04:36:38.648 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~12582912,29360128~16777216,62914560~12582912,260046848~8388608,281018368~4194304,293601280~4194304] 2024-09-14T04:36:38.648 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 4 diff [0~8388608,12582912~8388608,25165824~16777216,46137344~4194304,62914560~16777216,96468992~8388608,192937984~8388608,213909504~8388608,260046848~8388608,281018368~4194304,293601280~4194304,381681664~4194304] 2024-09-14T04:36:38.648 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~8388608,25165824~16777216,46137344~4194304,62914560~16777216,96468992~8388608,192937984~8388608,213909504~8388608,260046848~8388608,281018368~4194304,293601280~4194304,381681664~4194304] 2024-09-14T04:36:38.649 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~54525952,62914560~20971520,96468992~12582912,192937984~12582912,213909504~8388608,260046848~8388608,281018368~4194304,293601280~4194304,381681664~4194304] 2024-09-14T04:36:38.650 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 5 diff [0~8388608,12582912~8388608,25165824~16777216,46137344~12582912,62914560~16777216,92274688~12582912,125829120~8388608,138412032~4194304,192937984~8388608,213909504~8388608,247463936~8388608,260046848~8388608,281018368~4194304,293601280~4194304,306184192~12582912,360710144~12582912,381681664~4194304,394264576~4194304] 2024-09-14T04:36:38.650 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~8388608,25165824~16777216,46137344~12582912,62914560~16777216,92274688~12582912,125829120~8388608,138412032~4194304,192937984~8388608,213909504~8388608,247463936~8388608,260046848~8388608,281018368~4194304,293601280~4194304,306184192~12582912,360710144~12582912,381681664~4194304,394264576~4194304] 2024-09-14T04:36:38.651 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~58720256,62914560~20971520,92274688~16777216,125829120~20971520,192937984~12582912,213909504~8388608,247463936~20971520,281018368~4194304,293601280~4194304,306184192~16777216,360710144~12582912,381681664~4194304,394264576~8388608] 2024-09-14T04:36:38.651 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 6 diff [0~8388608,12582912~8388608,25165824~16777216,46137344~12582912,62914560~25165824,92274688~12582912,125829120~8388608,138412032~4194304,184549376~16777216,213909504~8388608,243269632~12582912,260046848~8388608,281018368~4194304,293601280~4194304,306184192~12582912,360710144~12582912,381681664~4194304,394264576~4194304,402653184~4194304] 2024-09-14T04:36:38.652 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~8388608,25165824~16777216,46137344~12582912,62914560~25165824,92274688~12582912,125829120~8388608,138412032~4194304,184549376~16777216,213909504~8388608,243269632~12582912,260046848~8388608,281018368~4194304,293601280~4194304,306184192~12582912,360710144~12582912,381681664~4194304,394264576~4194304,402653184~4194304] 2024-09-14T04:36:38.654 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~58720256,62914560~25165824,92274688~16777216,125829120~20971520,184549376~20971520,213909504~8388608,243269632~25165824,281018368~4194304,293601280~4194304,306184192~16777216,360710144~12582912,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.654 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 7 diff [0~8388608,12582912~8388608,25165824~16777216,46137344~12582912,62914560~25165824,92274688~12582912,125829120~8388608,138412032~4194304,171966464~8388608,184549376~20971520,213909504~8388608,243269632~12582912,260046848~8388608,281018368~4194304,293601280~4194304,306184192~12582912,335544320~4194304,343932928~4194304,356515840~16777216,381681664~4194304,394264576~12582912] 2024-09-14T04:36:38.654 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~8388608,25165824~16777216,46137344~12582912,62914560~25165824,92274688~12582912,125829120~8388608,138412032~4194304,171966464~8388608,184549376~20971520,213909504~8388608,243269632~12582912,260046848~8388608,281018368~4194304,293601280~4194304,306184192~12582912,335544320~4194304,343932928~4194304,356515840~16777216,381681664~4194304,394264576~12582912] 2024-09-14T04:36:38.657 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~58720256,62914560~25165824,92274688~16777216,125829120~20971520,171966464~37748736,213909504~8388608,243269632~25165824,281018368~4194304,293601280~4194304,306184192~16777216,335544320~4194304,343932928~4194304,356515840~16777216,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.657 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 8 diff [0~8388608,12582912~8388608,25165824~16777216,46137344~12582912,62914560~46137344,125829120~20971520,163577856~4194304,171966464~8388608,184549376~20971520,213909504~8388608,243269632~12582912,260046848~8388608,281018368~4194304,293601280~4194304,306184192~12582912,335544320~4194304,343932928~8388608,356515840~16777216,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.657 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~8388608,25165824~16777216,46137344~12582912,62914560~46137344,125829120~20971520,163577856~4194304,171966464~8388608,184549376~20971520,213909504~8388608,243269632~12582912,260046848~8388608,281018368~4194304,293601280~4194304,306184192~12582912,335544320~4194304,343932928~8388608,356515840~16777216,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.660 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~58720256,62914560~50331648,125829120~25165824,163577856~4194304,171966464~37748736,213909504~8388608,243269632~25165824,281018368~4194304,293601280~4194304,306184192~16777216,335544320~4194304,343932928~8388608,356515840~16777216,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.661 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 9 diff [0~8388608,12582912~8388608,25165824~16777216,46137344~12582912,62914560~25165824,92274688~16777216,125829120~20971520,163577856~4194304,171966464~8388608,184549376~20971520,213909504~8388608,243269632~12582912,260046848~8388608,281018368~4194304,293601280~4194304,306184192~12582912,331350016~8388608,343932928~8388608,356515840~16777216,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.661 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~8388608,25165824~16777216,46137344~12582912,62914560~25165824,92274688~16777216,125829120~20971520,163577856~4194304,171966464~8388608,184549376~20971520,213909504~8388608,243269632~12582912,260046848~8388608,281018368~4194304,293601280~4194304,306184192~12582912,331350016~8388608,343932928~8388608,356515840~16777216,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.664 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~58720256,62914560~50331648,125829120~25165824,163577856~4194304,171966464~37748736,213909504~8388608,243269632~29360128,281018368~4194304,293601280~4194304,306184192~16777216,331350016~8388608,343932928~8388608,356515840~16777216,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.664 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 10 diff [0~8388608,12582912~29360128,46137344~12582912,62914560~25165824,92274688~16777216,125829120~25165824,163577856~4194304,171966464~8388608,184549376~20971520,213909504~8388608,243269632~25165824,276824064~8388608,293601280~8388608,306184192~12582912,331350016~8388608,343932928~8388608,356515840~12582912,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.664 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~29360128,46137344~12582912,62914560~25165824,92274688~16777216,125829120~25165824,163577856~4194304,171966464~8388608,184549376~20971520,213909504~8388608,243269632~25165824,276824064~8388608,293601280~8388608,306184192~12582912,331350016~8388608,343932928~8388608,356515840~12582912,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.667 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~58720256,62914560~50331648,125829120~25165824,163577856~4194304,171966464~37748736,213909504~8388608,243269632~29360128,276824064~8388608,293601280~8388608,306184192~16777216,331350016~8388608,343932928~8388608,356515840~16777216,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.668 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 11 diff [0~8388608,12582912~29360128,46137344~12582912,62914560~25165824,92274688~16777216,125829120~25165824,163577856~4194304,171966464~8388608,184549376~20971520,213909504~16777216,243269632~25165824,276824064~8388608,293601280~8388608,306184192~12582912,331350016~8388608,343932928~8388608,356515840~12582912,381681664~29360128] 2024-09-14T04:36:38.668 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~29360128,46137344~12582912,62914560~25165824,92274688~16777216,125829120~25165824,163577856~4194304,171966464~8388608,184549376~20971520,213909504~16777216,243269632~25165824,276824064~8388608,293601280~8388608,306184192~12582912,331350016~8388608,343932928~8388608,356515840~12582912,381681664~29360128] 2024-09-14T04:36:38.671 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~58720256,62914560~50331648,125829120~25165824,163577856~4194304,171966464~37748736,213909504~20971520,243269632~29360128,276824064~8388608,293601280~8388608,306184192~16777216,331350016~8388608,343932928~8388608,356515840~16777216,381681664~29360128] 2024-09-14T04:36:38.672 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 12 diff [0~8388608,12582912~29360128,46137344~12582912,62914560~25165824,92274688~16777216,125829120~25165824,163577856~16777216,184549376~20971520,213909504~16777216,243269632~25165824,276824064~8388608,293601280~8388608,306184192~12582912,331350016~8388608,343932928~8388608,356515840~12582912,381681664~29360128] 2024-09-14T04:36:38.672 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~29360128,46137344~12582912,62914560~25165824,92274688~16777216,125829120~25165824,163577856~16777216,184549376~20971520,213909504~16777216,243269632~25165824,276824064~8388608,293601280~8388608,306184192~12582912,331350016~8388608,343932928~8388608,356515840~12582912,381681664~29360128] 2024-09-14T04:36:38.676 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~58720256,62914560~50331648,125829120~25165824,163577856~46137344,213909504~20971520,243269632~29360128,276824064~8388608,293601280~8388608,306184192~16777216,331350016~8388608,343932928~8388608,356515840~16777216,381681664~29360128] 2024-09-14T04:36:38.677 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 13 diff [0~8388608,12582912~46137344,62914560~25165824,92274688~16777216,125829120~25165824,163577856~16777216,184549376~54525952,243269632~25165824,276824064~8388608,293601280~8388608,306184192~12582912,331350016~8388608,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.677 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~46137344,62914560~25165824,92274688~16777216,125829120~25165824,163577856~16777216,184549376~54525952,243269632~25165824,276824064~8388608,293601280~8388608,306184192~12582912,331350016~8388608,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.681 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~58720256,62914560~50331648,125829120~25165824,163577856~75497472,243269632~29360128,276824064~8388608,293601280~8388608,306184192~16777216,331350016~8388608,343932928~67108864] 2024-09-14T04:36:38.681 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 14 diff [0~8388608,12582912~75497472,92274688~16777216,125829120~29360128,163577856~16777216,184549376~54525952,243269632~33554432,281018368~4194304,293601280~8388608,306184192~20971520,331350016~8388608,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.681 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~75497472,92274688~16777216,125829120~29360128,163577856~16777216,184549376~54525952,243269632~33554432,281018368~4194304,293601280~8388608,306184192~20971520,331350016~8388608,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.687 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,125829120~29360128,163577856~75497472,243269632~41943040,293601280~8388608,306184192~33554432,343932928~71303168] 2024-09-14T04:36:38.687 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 15 diff [0~88080384,92274688~16777216,125829120~33554432,163577856~16777216,184549376~54525952,243269632~33554432,281018368~4194304,293601280~8388608,306184192~20971520,331350016~8388608,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.687 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~88080384,92274688~16777216,125829120~33554432,163577856~16777216,184549376~54525952,243269632~33554432,281018368~4194304,293601280~8388608,306184192~20971520,331350016~8388608,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.693 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,125829120~33554432,163577856~75497472,243269632~41943040,293601280~8388608,306184192~33554432,343932928~71303168] 2024-09-14T04:36:38.693 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 16 diff [0~88080384,92274688~16777216,125829120~33554432,163577856~16777216,184549376~54525952,243269632~33554432,281018368~4194304,289406976~8388608,306184192~33554432,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.693 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~88080384,92274688~16777216,125829120~33554432,163577856~16777216,184549376~54525952,243269632~33554432,281018368~4194304,289406976~8388608,306184192~33554432,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.699 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,125829120~33554432,163577856~75497472,243269632~41943040,289406976~12582912,306184192~33554432,343932928~71303168] 2024-09-14T04:36:38.699 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 17 diff [0~62914560,67108864~20971520,92274688~16777216,125829120~54525952,184549376~54525952,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~4194304,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.700 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,125829120~54525952,184549376~54525952,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~4194304,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.706 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,125829120~113246208,243269632~41943040,289406976~50331648,343932928~71303168] 2024-09-14T04:36:38.706 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 18 diff [0~62914560,67108864~20971520,92274688~16777216,125829120~54525952,184549376~54525952,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:38.706 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,125829120~54525952,184549376~54525952,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:38.714 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,125829120~113246208,243269632~41943040,289406976~125829120] 2024-09-14T04:36:38.714 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to 19 diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:38.714 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:38.720 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:38.720 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 4 diff [0~8388608,25165824~8388608,46137344~4194304,75497472~4194304,96468992~8388608,192937984~8388608,213909504~8388608,381681664~4194304] 2024-09-14T04:36:38.720 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,25165824~8388608,46137344~4194304,75497472~4194304,96468992~8388608,192937984~8388608,213909504~8388608,381681664~4194304] 2024-09-14T04:36:38.721 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~12582912,25165824~8388608,46137344~8388608,75497472~8388608,96468992~12582912,192937984~12582912,213909504~8388608,381681664~4194304] 2024-09-14T04:36:38.721 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 5 diff [0~8388608,25165824~8388608,46137344~12582912,75497472~4194304,92274688~12582912,125829120~8388608,138412032~4194304,192937984~8388608,213909504~8388608,247463936~8388608,306184192~12582912,360710144~12582912,381681664~4194304,394264576~4194304] 2024-09-14T04:36:38.721 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,25165824~8388608,46137344~12582912,75497472~4194304,92274688~12582912,125829120~8388608,138412032~4194304,192937984~8388608,213909504~8388608,247463936~8388608,306184192~12582912,360710144~12582912,381681664~4194304,394264576~4194304] 2024-09-14T04:36:38.723 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~12582912,25165824~8388608,46137344~12582912,75497472~8388608,92274688~16777216,125829120~20971520,192937984~12582912,213909504~8388608,247463936~12582912,306184192~16777216,360710144~12582912,381681664~4194304,394264576~8388608] 2024-09-14T04:36:38.723 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 6 diff [0~8388608,25165824~8388608,37748736~4194304,46137344~12582912,71303168~16777216,92274688~12582912,125829120~8388608,138412032~4194304,184549376~16777216,213909504~8388608,243269632~12582912,306184192~12582912,360710144~12582912,381681664~4194304,394264576~4194304,402653184~4194304] 2024-09-14T04:36:38.723 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,25165824~8388608,37748736~4194304,46137344~12582912,71303168~16777216,92274688~12582912,125829120~8388608,138412032~4194304,184549376~16777216,213909504~8388608,243269632~12582912,306184192~12582912,360710144~12582912,381681664~4194304,394264576~4194304,402653184~4194304] 2024-09-14T04:36:38.724 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~12582912,25165824~8388608,37748736~20971520,71303168~16777216,92274688~16777216,125829120~20971520,184549376~20971520,213909504~8388608,243269632~16777216,306184192~16777216,360710144~12582912,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.725 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 7 diff [0~8388608,25165824~16777216,46137344~12582912,71303168~16777216,92274688~12582912,125829120~8388608,138412032~4194304,171966464~8388608,184549376~20971520,213909504~8388608,243269632~12582912,306184192~12582912,335544320~4194304,343932928~4194304,356515840~16777216,381681664~4194304,394264576~12582912] 2024-09-14T04:36:38.725 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,25165824~16777216,46137344~12582912,71303168~16777216,92274688~12582912,125829120~8388608,138412032~4194304,171966464~8388608,184549376~20971520,213909504~8388608,243269632~12582912,306184192~12582912,335544320~4194304,343932928~4194304,356515840~16777216,381681664~4194304,394264576~12582912] 2024-09-14T04:36:38.727 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~12582912,25165824~33554432,71303168~16777216,92274688~16777216,125829120~20971520,171966464~37748736,213909504~8388608,243269632~16777216,306184192~16777216,335544320~4194304,343932928~4194304,356515840~16777216,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.727 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 8 diff [0~8388608,25165824~16777216,46137344~12582912,71303168~37748736,125829120~20971520,163577856~4194304,171966464~8388608,184549376~20971520,213909504~8388608,243269632~12582912,306184192~12582912,335544320~4194304,343932928~8388608,356515840~16777216,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.727 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,25165824~16777216,46137344~12582912,71303168~37748736,125829120~20971520,163577856~4194304,171966464~8388608,184549376~20971520,213909504~8388608,243269632~12582912,306184192~12582912,335544320~4194304,343932928~8388608,356515840~16777216,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.730 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~12582912,25165824~33554432,71303168~41943040,125829120~25165824,163577856~4194304,171966464~37748736,213909504~8388608,243269632~16777216,306184192~16777216,335544320~4194304,343932928~8388608,356515840~16777216,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.730 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 9 diff [0~8388608,25165824~16777216,46137344~12582912,71303168~16777216,92274688~16777216,125829120~20971520,163577856~4194304,171966464~8388608,184549376~20971520,213909504~8388608,243269632~12582912,260046848~8388608,306184192~12582912,331350016~8388608,343932928~8388608,356515840~16777216,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.730 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,25165824~16777216,46137344~12582912,71303168~16777216,92274688~16777216,125829120~20971520,163577856~4194304,171966464~8388608,184549376~20971520,213909504~8388608,243269632~12582912,260046848~8388608,306184192~12582912,331350016~8388608,343932928~8388608,356515840~16777216,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.733 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~12582912,25165824~33554432,71303168~41943040,125829120~25165824,163577856~4194304,171966464~37748736,213909504~8388608,243269632~29360128,306184192~16777216,331350016~8388608,343932928~8388608,356515840~16777216,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.733 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 10 diff [0~8388608,20971520~12582912,37748736~4194304,46137344~12582912,71303168~16777216,92274688~16777216,125829120~25165824,163577856~4194304,171966464~8388608,184549376~20971520,213909504~8388608,243269632~25165824,276824064~4194304,297795584~4194304,306184192~12582912,331350016~8388608,343932928~8388608,356515840~12582912,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.734 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,20971520~12582912,37748736~4194304,46137344~12582912,71303168~16777216,92274688~16777216,125829120~25165824,163577856~4194304,171966464~8388608,184549376~20971520,213909504~8388608,243269632~25165824,276824064~4194304,297795584~4194304,306184192~12582912,331350016~8388608,343932928~8388608,356515840~12582912,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.737 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~12582912,20971520~37748736,71303168~41943040,125829120~25165824,163577856~4194304,171966464~37748736,213909504~8388608,243269632~29360128,276824064~4194304,297795584~4194304,306184192~16777216,331350016~8388608,343932928~8388608,356515840~16777216,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.737 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 11 diff [0~8388608,20971520~20971520,46137344~12582912,67108864~20971520,92274688~16777216,125829120~25165824,163577856~4194304,171966464~8388608,184549376~20971520,213909504~16777216,243269632~25165824,276824064~4194304,297795584~4194304,306184192~12582912,331350016~8388608,343932928~8388608,356515840~12582912,381681664~29360128] 2024-09-14T04:36:38.737 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,20971520~20971520,46137344~12582912,67108864~20971520,92274688~16777216,125829120~25165824,163577856~4194304,171966464~8388608,184549376~20971520,213909504~16777216,243269632~25165824,276824064~4194304,297795584~4194304,306184192~12582912,331350016~8388608,343932928~8388608,356515840~12582912,381681664~29360128] 2024-09-14T04:36:38.741 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~12582912,20971520~37748736,67108864~46137344,125829120~25165824,163577856~4194304,171966464~37748736,213909504~20971520,243269632~29360128,276824064~4194304,297795584~4194304,306184192~16777216,331350016~8388608,343932928~8388608,356515840~16777216,381681664~29360128] 2024-09-14T04:36:38.741 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 12 diff [0~8388608,12582912~29360128,46137344~12582912,67108864~20971520,92274688~16777216,125829120~25165824,163577856~16777216,184549376~20971520,213909504~16777216,243269632~25165824,276824064~4194304,297795584~4194304,306184192~12582912,331350016~8388608,343932928~8388608,356515840~12582912,381681664~29360128] 2024-09-14T04:36:38.741 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~29360128,46137344~12582912,67108864~20971520,92274688~16777216,125829120~25165824,163577856~16777216,184549376~20971520,213909504~16777216,243269632~25165824,276824064~4194304,297795584~4194304,306184192~12582912,331350016~8388608,343932928~8388608,356515840~12582912,381681664~29360128] 2024-09-14T04:36:38.745 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~58720256,67108864~46137344,125829120~25165824,163577856~46137344,213909504~20971520,243269632~29360128,276824064~4194304,297795584~4194304,306184192~16777216,331350016~8388608,343932928~8388608,356515840~16777216,381681664~29360128] 2024-09-14T04:36:38.745 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 13 diff [0~8388608,12582912~46137344,67108864~20971520,92274688~16777216,125829120~25165824,163577856~16777216,184549376~54525952,243269632~25165824,276824064~4194304,297795584~4194304,306184192~12582912,331350016~8388608,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.745 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~46137344,67108864~20971520,92274688~16777216,125829120~25165824,163577856~16777216,184549376~54525952,243269632~25165824,276824064~4194304,297795584~4194304,306184192~12582912,331350016~8388608,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.750 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~58720256,67108864~46137344,125829120~25165824,163577856~75497472,243269632~29360128,276824064~4194304,297795584~4194304,306184192~16777216,331350016~8388608,343932928~67108864] 2024-09-14T04:36:38.750 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 14 diff [0~8388608,12582912~75497472,92274688~16777216,125829120~29360128,163577856~16777216,184549376~54525952,243269632~33554432,297795584~4194304,306184192~20971520,331350016~8388608,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.750 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~75497472,92274688~16777216,125829120~29360128,163577856~16777216,184549376~54525952,243269632~33554432,297795584~4194304,306184192~20971520,331350016~8388608,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.755 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,125829120~29360128,163577856~75497472,243269632~37748736,297795584~4194304,306184192~33554432,343932928~71303168] 2024-09-14T04:36:38.755 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 15 diff [0~88080384,92274688~16777216,125829120~33554432,163577856~16777216,184549376~54525952,243269632~33554432,297795584~4194304,306184192~20971520,331350016~8388608,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.756 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~88080384,92274688~16777216,125829120~33554432,163577856~16777216,184549376~54525952,243269632~33554432,297795584~4194304,306184192~20971520,331350016~8388608,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.761 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,125829120~33554432,163577856~75497472,243269632~37748736,297795584~4194304,306184192~33554432,343932928~71303168] 2024-09-14T04:36:38.761 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 16 diff [0~88080384,92274688~16777216,125829120~33554432,163577856~16777216,184549376~54525952,243269632~33554432,289406976~8388608,306184192~33554432,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.761 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~88080384,92274688~16777216,125829120~33554432,163577856~16777216,184549376~54525952,243269632~33554432,289406976~8388608,306184192~33554432,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.767 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,125829120~33554432,163577856~75497472,243269632~37748736,289406976~12582912,306184192~33554432,343932928~71303168] 2024-09-14T04:36:38.767 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 17 diff [0~62914560,67108864~20971520,92274688~16777216,125829120~54525952,184549376~54525952,243269632~33554432,289406976~12582912,306184192~25165824,335544320~4194304,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.767 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,125829120~54525952,184549376~54525952,243269632~33554432,289406976~12582912,306184192~25165824,335544320~4194304,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:38 smithi129 ceph-mon[101316]: osdmap e1287: 8 total, 8 up, 8 in 2024-09-14T04:36:38.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:38 smithi129 ceph-mon[101316]: pgmap v2067: 105 pgs: 105 active+clean; 1.0 GiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 129 KiB/s rd, 27 MiB/s wr, 186 op/s 2024-09-14T04:36:38.773 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,125829120~113246208,243269632~37748736,289406976~50331648,343932928~71303168] 2024-09-14T04:36:38.773 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 18 diff [0~62914560,67108864~20971520,92274688~16777216,125829120~54525952,184549376~54525952,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:38.773 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,125829120~54525952,184549376~54525952,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:38.779 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,125829120~113246208,243269632~37748736,289406976~125829120] 2024-09-14T04:36:38.779 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to 19 diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:38.779 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:38.786 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~37748736,289406976~125829120] 2024-09-14T04:36:38.786 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to 5 diff [50331648~8388608,92274688~4194304,125829120~8388608,138412032~4194304,247463936~8388608,306184192~12582912,360710144~12582912,394264576~4194304] 2024-09-14T04:36:38.788 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [50331648~8388608,92274688~4194304,125829120~8388608,138412032~4194304,247463936~8388608,306184192~12582912,360710144~12582912,394264576~4194304] 2024-09-14T04:36:38.789 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [50331648~8388608,92274688~4194304,125829120~20971520,247463936~12582912,306184192~16777216,360710144~12582912,394264576~8388608] 2024-09-14T04:36:38.789 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to 6 diff [25165824~8388608,37748736~4194304,50331648~8388608,71303168~4194304,79691776~8388608,92274688~4194304,125829120~8388608,138412032~4194304,184549376~8388608,213909504~4194304,243269632~12582912,306184192~12582912,360710144~12582912,381681664~4194304,394264576~4194304,402653184~4194304] 2024-09-14T04:36:38.789 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [25165824~8388608,37748736~4194304,50331648~8388608,71303168~4194304,79691776~8388608,92274688~4194304,125829120~8388608,138412032~4194304,184549376~8388608,213909504~4194304,243269632~12582912,306184192~12582912,360710144~12582912,381681664~4194304,394264576~4194304,402653184~4194304] 2024-09-14T04:36:38.790 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [25165824~8388608,37748736~8388608,50331648~8388608,71303168~4194304,79691776~8388608,92274688~4194304,125829120~20971520,184549376~8388608,213909504~8388608,243269632~16777216,306184192~16777216,360710144~12582912,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.790 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to 7 diff [25165824~16777216,50331648~8388608,71303168~16777216,92274688~4194304,125829120~8388608,138412032~4194304,171966464~8388608,184549376~12582912,201326592~4194304,213909504~4194304,243269632~12582912,306184192~12582912,335544320~4194304,343932928~4194304,356515840~16777216,381681664~4194304,394264576~12582912] 2024-09-14T04:36:38.790 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [25165824~16777216,50331648~8388608,71303168~16777216,92274688~4194304,125829120~8388608,138412032~4194304,171966464~8388608,184549376~12582912,201326592~4194304,213909504~4194304,243269632~12582912,306184192~12582912,335544320~4194304,343932928~4194304,356515840~16777216,381681664~4194304,394264576~12582912] 2024-09-14T04:36:38.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:38 smithi039 ceph-mon[106470]: osdmap e1287: 8 total, 8 up, 8 in 2024-09-14T04:36:38.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:38 smithi039 ceph-mon[106470]: pgmap v2067: 105 pgs: 105 active+clean; 1.0 GiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 129 KiB/s rd, 27 MiB/s wr, 186 op/s 2024-09-14T04:36:38.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:36:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:36:38] "GET /metrics HTTP/1.1" 200 36692 "" "Prometheus/2.51.0" 2024-09-14T04:36:38.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:38 smithi039 ceph-mon[111235]: osdmap e1287: 8 total, 8 up, 8 in 2024-09-14T04:36:38.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:38 smithi039 ceph-mon[111235]: pgmap v2067: 105 pgs: 105 active+clean; 1.0 GiB data, 3.0 GiB used, 712 GiB / 715 GiB avail; 129 KiB/s rd, 27 MiB/s wr, 186 op/s 2024-09-14T04:36:38.791 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [25165824~20971520,50331648~8388608,71303168~16777216,92274688~4194304,125829120~20971520,171966464~37748736,213909504~8388608,243269632~16777216,306184192~16777216,335544320~4194304,343932928~4194304,356515840~16777216,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.791 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to 8 diff [25165824~16777216,50331648~8388608,71303168~12582912,88080384~8388608,104857600~8388608,125829120~20971520,163577856~4194304,171966464~8388608,184549376~12582912,201326592~8388608,213909504~4194304,243269632~12582912,306184192~12582912,335544320~4194304,343932928~8388608,356515840~16777216,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.792 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [25165824~16777216,50331648~8388608,71303168~12582912,88080384~8388608,104857600~8388608,125829120~20971520,163577856~4194304,171966464~8388608,184549376~12582912,201326592~8388608,213909504~4194304,243269632~12582912,306184192~12582912,335544320~4194304,343932928~8388608,356515840~16777216,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.794 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [25165824~20971520,50331648~8388608,71303168~25165824,104857600~8388608,125829120~25165824,163577856~4194304,171966464~37748736,213909504~8388608,243269632~16777216,306184192~16777216,335544320~4194304,343932928~8388608,356515840~16777216,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.794 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to 9 diff [25165824~16777216,46137344~12582912,71303168~16777216,92274688~4194304,104857600~8388608,125829120~20971520,163577856~4194304,171966464~8388608,184549376~25165824,213909504~4194304,243269632~12582912,260046848~8388608,306184192~12582912,331350016~8388608,343932928~8388608,356515840~16777216,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.794 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [25165824~16777216,46137344~12582912,71303168~16777216,92274688~4194304,104857600~8388608,125829120~20971520,163577856~4194304,171966464~8388608,184549376~25165824,213909504~4194304,243269632~12582912,260046848~8388608,306184192~12582912,331350016~8388608,343932928~8388608,356515840~16777216,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.796 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [25165824~33554432,71303168~25165824,104857600~8388608,125829120~25165824,163577856~4194304,171966464~37748736,213909504~8388608,243269632~29360128,306184192~16777216,331350016~8388608,343932928~8388608,356515840~16777216,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.796 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to 10 diff [20971520~12582912,37748736~4194304,46137344~12582912,71303168~16777216,92274688~4194304,104857600~8388608,125829120~25165824,163577856~4194304,171966464~8388608,184549376~25165824,213909504~4194304,243269632~25165824,276824064~4194304,297795584~4194304,306184192~12582912,331350016~8388608,343932928~8388608,356515840~12582912,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.796 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [20971520~12582912,37748736~4194304,46137344~12582912,71303168~16777216,92274688~4194304,104857600~8388608,125829120~25165824,163577856~4194304,171966464~8388608,184549376~25165824,213909504~4194304,243269632~25165824,276824064~4194304,297795584~4194304,306184192~12582912,331350016~8388608,343932928~8388608,356515840~12582912,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.799 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [20971520~37748736,71303168~25165824,104857600~8388608,125829120~25165824,163577856~4194304,171966464~37748736,213909504~8388608,243269632~29360128,276824064~4194304,297795584~4194304,306184192~16777216,331350016~8388608,343932928~8388608,356515840~16777216,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.799 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to 11 diff [20971520~20971520,46137344~12582912,67108864~20971520,92274688~4194304,104857600~8388608,125829120~25165824,163577856~4194304,171966464~8388608,184549376~25165824,213909504~4194304,222298112~8388608,243269632~25165824,276824064~4194304,297795584~4194304,306184192~12582912,331350016~8388608,343932928~8388608,356515840~12582912,381681664~29360128] 2024-09-14T04:36:38.799 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [20971520~20971520,46137344~12582912,67108864~20971520,92274688~4194304,104857600~8388608,125829120~25165824,163577856~4194304,171966464~8388608,184549376~25165824,213909504~4194304,222298112~8388608,243269632~25165824,276824064~4194304,297795584~4194304,306184192~12582912,331350016~8388608,343932928~8388608,356515840~12582912,381681664~29360128] 2024-09-14T04:36:38.803 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [20971520~37748736,67108864~29360128,104857600~8388608,125829120~25165824,163577856~4194304,171966464~37748736,213909504~20971520,243269632~29360128,276824064~4194304,297795584~4194304,306184192~16777216,331350016~8388608,343932928~8388608,356515840~16777216,381681664~29360128] 2024-09-14T04:36:38.803 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to 12 diff [0~8388608,12582912~29360128,46137344~12582912,67108864~20971520,92274688~4194304,104857600~8388608,125829120~25165824,163577856~16777216,184549376~25165824,213909504~4194304,222298112~8388608,243269632~25165824,276824064~4194304,297795584~4194304,306184192~12582912,331350016~8388608,343932928~8388608,356515840~12582912,381681664~29360128] 2024-09-14T04:36:38.803 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~29360128,46137344~12582912,67108864~20971520,92274688~4194304,104857600~8388608,125829120~25165824,163577856~16777216,184549376~25165824,213909504~4194304,222298112~8388608,243269632~25165824,276824064~4194304,297795584~4194304,306184192~12582912,331350016~8388608,343932928~8388608,356515840~12582912,381681664~29360128] 2024-09-14T04:36:38.806 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~8388608,12582912~46137344,67108864~29360128,104857600~8388608,125829120~25165824,163577856~46137344,213909504~20971520,243269632~29360128,276824064~4194304,297795584~4194304,306184192~16777216,331350016~8388608,343932928~8388608,356515840~16777216,381681664~29360128] 2024-09-14T04:36:38.806 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to 13 diff [0~8388608,12582912~46137344,67108864~20971520,92274688~4194304,104857600~8388608,125829120~25165824,163577856~16777216,184549376~33554432,222298112~16777216,243269632~25165824,276824064~4194304,297795584~4194304,306184192~12582912,331350016~8388608,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.806 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~46137344,67108864~20971520,92274688~4194304,104857600~8388608,125829120~25165824,163577856~16777216,184549376~33554432,222298112~16777216,243269632~25165824,276824064~4194304,297795584~4194304,306184192~12582912,331350016~8388608,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.810 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~8388608,12582912~46137344,67108864~29360128,104857600~8388608,125829120~25165824,163577856~75497472,243269632~29360128,276824064~4194304,297795584~4194304,306184192~16777216,331350016~8388608,343932928~67108864] 2024-09-14T04:36:38.810 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to 14 diff [0~8388608,12582912~75497472,92274688~4194304,104857600~8388608,125829120~29360128,163577856~16777216,184549376~33554432,222298112~16777216,243269632~33554432,297795584~4194304,306184192~20971520,331350016~8388608,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.810 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~75497472,92274688~4194304,104857600~8388608,125829120~29360128,163577856~16777216,184549376~33554432,222298112~16777216,243269632~33554432,297795584~4194304,306184192~20971520,331350016~8388608,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.814 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~8388608,12582912~83886080,104857600~8388608,125829120~29360128,163577856~75497472,243269632~37748736,297795584~4194304,306184192~33554432,343932928~71303168] 2024-09-14T04:36:38.815 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to 15 diff [0~88080384,92274688~4194304,104857600~8388608,125829120~33554432,163577856~16777216,184549376~33554432,222298112~16777216,243269632~33554432,297795584~4194304,306184192~20971520,331350016~8388608,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.815 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~88080384,92274688~4194304,104857600~8388608,125829120~33554432,163577856~16777216,184549376~33554432,222298112~16777216,243269632~33554432,297795584~4194304,306184192~20971520,331350016~8388608,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.819 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~96468992,104857600~8388608,125829120~33554432,163577856~75497472,243269632~37748736,297795584~4194304,306184192~33554432,343932928~71303168] 2024-09-14T04:36:38.819 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to 16 diff [0~88080384,92274688~4194304,104857600~8388608,125829120~33554432,163577856~16777216,184549376~33554432,222298112~16777216,243269632~33554432,289406976~8388608,306184192~33554432,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.819 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~88080384,92274688~4194304,104857600~8388608,125829120~33554432,163577856~16777216,184549376~33554432,222298112~16777216,243269632~33554432,289406976~8388608,306184192~33554432,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.825 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~96468992,104857600~8388608,125829120~33554432,163577856~75497472,243269632~37748736,289406976~12582912,306184192~33554432,343932928~71303168] 2024-09-14T04:36:38.825 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to 17 diff [0~62914560,67108864~20971520,92274688~4194304,104857600~8388608,125829120~54525952,184549376~33554432,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~4194304,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.825 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~4194304,104857600~8388608,125829120~54525952,184549376~33554432,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~4194304,343932928~33554432,381681664~29360128] 2024-09-14T04:36:38.831 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~96468992,104857600~8388608,125829120~113246208,243269632~37748736,289406976~50331648,343932928~71303168] 2024-09-14T04:36:38.831 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to 18 diff [0~62914560,67108864~20971520,92274688~4194304,104857600~8388608,125829120~54525952,184549376~33554432,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:38.831 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~4194304,104857600~8388608,125829120~54525952,184549376~33554432,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:38.836 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~96468992,104857600~8388608,125829120~113246208,243269632~37748736,289406976~125829120] 2024-09-14T04:36:38.836 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to 19 diff [0~62914560,67108864~20971520,92274688~4194304,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:38.836 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~4194304,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:38.844 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~96468992,100663296~12582912,117440512~121634816,243269632~37748736,289406976~125829120] 2024-09-14T04:36:38.844 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to 6 diff [25165824~8388608,37748736~4194304,71303168~4194304,79691776~8388608,138412032~4194304,184549376~8388608,213909504~4194304,243269632~4194304,381681664~4194304,402653184~4194304] 2024-09-14T04:36:38.844 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [25165824~8388608,37748736~4194304,71303168~4194304,79691776~8388608,138412032~4194304,184549376~8388608,213909504~4194304,243269632~4194304,381681664~4194304,402653184~4194304] 2024-09-14T04:36:38.845 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [25165824~8388608,37748736~8388608,71303168~4194304,79691776~8388608,138412032~4194304,184549376~8388608,213909504~8388608,243269632~8388608,381681664~4194304,402653184~8388608] 2024-09-14T04:36:38.845 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to 7 diff [25165824~16777216,71303168~16777216,138412032~4194304,171966464~8388608,184549376~12582912,201326592~4194304,213909504~4194304,243269632~4194304,306184192~4194304,335544320~4194304,343932928~4194304,356515840~8388608,381681664~4194304,394264576~12582912] 2024-09-14T04:36:38.845 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [25165824~16777216,71303168~16777216,138412032~4194304,171966464~8388608,184549376~12582912,201326592~4194304,213909504~4194304,243269632~4194304,306184192~4194304,335544320~4194304,343932928~4194304,356515840~8388608,381681664~4194304,394264576~12582912] 2024-09-14T04:36:38.846 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [25165824~20971520,71303168~16777216,138412032~4194304,171966464~37748736,213909504~8388608,243269632~8388608,306184192~8388608,335544320~4194304,343932928~4194304,356515840~8388608,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.846 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to 8 diff [25165824~16777216,71303168~12582912,88080384~4194304,104857600~8388608,125829120~8388608,138412032~8388608,163577856~4194304,171966464~8388608,184549376~12582912,201326592~8388608,213909504~4194304,243269632~4194304,306184192~4194304,335544320~4194304,343932928~8388608,356515840~8388608,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.846 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [25165824~16777216,71303168~12582912,88080384~4194304,104857600~8388608,125829120~8388608,138412032~8388608,163577856~4194304,171966464~8388608,184549376~12582912,201326592~8388608,213909504~4194304,243269632~4194304,306184192~4194304,335544320~4194304,343932928~8388608,356515840~8388608,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.848 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [25165824~20971520,71303168~20971520,104857600~8388608,125829120~25165824,163577856~4194304,171966464~37748736,213909504~8388608,243269632~8388608,306184192~8388608,335544320~4194304,343932928~8388608,356515840~8388608,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.848 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to 9 diff [25165824~16777216,46137344~8388608,71303168~16777216,104857600~8388608,125829120~8388608,138412032~8388608,163577856~4194304,171966464~8388608,184549376~25165824,213909504~4194304,243269632~4194304,260046848~8388608,306184192~4194304,331350016~8388608,343932928~8388608,356515840~8388608,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.848 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [25165824~16777216,46137344~8388608,71303168~16777216,104857600~8388608,125829120~8388608,138412032~8388608,163577856~4194304,171966464~8388608,184549376~25165824,213909504~4194304,243269632~4194304,260046848~8388608,306184192~4194304,331350016~8388608,343932928~8388608,356515840~8388608,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.850 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [25165824~29360128,71303168~20971520,104857600~8388608,125829120~25165824,163577856~4194304,171966464~37748736,213909504~8388608,243269632~8388608,260046848~12582912,306184192~8388608,331350016~8388608,343932928~8388608,356515840~8388608,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.850 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to 10 diff [20971520~12582912,37748736~4194304,46137344~8388608,71303168~16777216,104857600~8388608,125829120~8388608,138412032~12582912,163577856~4194304,171966464~8388608,184549376~25165824,213909504~4194304,243269632~25165824,276824064~4194304,297795584~4194304,306184192~4194304,331350016~8388608,343932928~8388608,356515840~8388608,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.850 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [20971520~12582912,37748736~4194304,46137344~8388608,71303168~16777216,104857600~8388608,125829120~8388608,138412032~12582912,163577856~4194304,171966464~8388608,184549376~25165824,213909504~4194304,243269632~25165824,276824064~4194304,297795584~4194304,306184192~4194304,331350016~8388608,343932928~8388608,356515840~8388608,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.853 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [20971520~37748736,71303168~20971520,104857600~8388608,125829120~25165824,163577856~4194304,171966464~37748736,213909504~8388608,243269632~29360128,276824064~4194304,297795584~4194304,306184192~8388608,331350016~8388608,343932928~8388608,356515840~12582912,381681664~4194304,394264576~16777216] 2024-09-14T04:36:38.853 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to 11 diff [20971520~20971520,46137344~12582912,67108864~20971520,104857600~8388608,125829120~8388608,138412032~12582912,163577856~4194304,171966464~8388608,184549376~25165824,213909504~4194304,222298112~8388608,243269632~25165824,276824064~4194304,297795584~4194304,306184192~4194304,331350016~8388608,343932928~8388608,356515840~8388608,381681664~29360128] 2024-09-14T04:36:38.853 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [20971520~20971520,46137344~12582912,67108864~20971520,104857600~8388608,125829120~8388608,138412032~12582912,163577856~4194304,171966464~8388608,184549376~25165824,213909504~4194304,222298112~8388608,243269632~25165824,276824064~4194304,297795584~4194304,306184192~4194304,331350016~8388608,343932928~8388608,356515840~8388608,381681664~29360128] 2024-09-14T04:36:38.856 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [20971520~37748736,67108864~25165824,104857600~8388608,125829120~25165824,163577856~4194304,171966464~37748736,213909504~20971520,243269632~29360128,276824064~4194304,297795584~4194304,306184192~8388608,331350016~8388608,343932928~8388608,356515840~12582912,381681664~29360128] 2024-09-14T04:36:38.856 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to 12 diff [0~8388608,12582912~29360128,46137344~12582912,67108864~20971520,104857600~8388608,125829120~25165824,163577856~16777216,184549376~25165824,213909504~4194304,222298112~8388608,243269632~25165824,276824064~4194304,297795584~4194304,306184192~4194304,331350016~8388608,343932928~8388608,356515840~8388608,381681664~29360128] 2024-09-14T04:36:38.856 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~29360128,46137344~12582912,67108864~20971520,104857600~8388608,125829120~25165824,163577856~16777216,184549376~25165824,213909504~4194304,222298112~8388608,243269632~25165824,276824064~4194304,297795584~4194304,306184192~4194304,331350016~8388608,343932928~8388608,356515840~8388608,381681664~29360128] 2024-09-14T04:36:38.860 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~8388608,12582912~46137344,67108864~25165824,104857600~8388608,125829120~25165824,163577856~46137344,213909504~20971520,243269632~29360128,276824064~4194304,297795584~4194304,306184192~8388608,331350016~8388608,343932928~8388608,356515840~12582912,381681664~29360128] 2024-09-14T04:36:38.861 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to 13 diff [0~8388608,12582912~46137344,67108864~20971520,104857600~8388608,125829120~25165824,163577856~16777216,184549376~33554432,222298112~16777216,243269632~25165824,276824064~4194304,297795584~4194304,306184192~4194304,331350016~8388608,343932928~20971520,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.861 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~46137344,67108864~20971520,104857600~8388608,125829120~25165824,163577856~16777216,184549376~33554432,222298112~16777216,243269632~25165824,276824064~4194304,297795584~4194304,306184192~4194304,331350016~8388608,343932928~20971520,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.864 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~8388608,12582912~46137344,67108864~25165824,104857600~8388608,125829120~25165824,163577856~75497472,243269632~29360128,276824064~4194304,297795584~4194304,306184192~8388608,331350016~8388608,343932928~25165824,373293056~37748736] 2024-09-14T04:36:38.864 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to 14 diff [0~8388608,12582912~75497472,104857600~8388608,125829120~29360128,163577856~16777216,184549376~33554432,222298112~16777216,243269632~33554432,297795584~4194304,306184192~4194304,318767104~8388608,331350016~8388608,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.864 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~75497472,104857600~8388608,125829120~29360128,163577856~16777216,184549376~33554432,222298112~16777216,243269632~33554432,297795584~4194304,306184192~4194304,318767104~8388608,331350016~8388608,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.867 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~8388608,12582912~79691776,104857600~8388608,125829120~29360128,163577856~75497472,243269632~37748736,297795584~4194304,306184192~8388608,318767104~20971520,343932928~25165824,373293056~41943040] 2024-09-14T04:36:38.868 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to 15 diff [0~88080384,104857600~8388608,125829120~33554432,163577856~16777216,184549376~33554432,222298112~16777216,243269632~33554432,297795584~4194304,306184192~4194304,318767104~8388608,331350016~8388608,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.868 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~88080384,104857600~8388608,125829120~33554432,163577856~16777216,184549376~33554432,222298112~16777216,243269632~33554432,297795584~4194304,306184192~4194304,318767104~8388608,331350016~8388608,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.872 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~92274688,104857600~8388608,125829120~33554432,163577856~75497472,243269632~37748736,297795584~4194304,306184192~8388608,318767104~20971520,343932928~25165824,373293056~41943040] 2024-09-14T04:36:38.872 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to 16 diff [0~88080384,104857600~8388608,125829120~33554432,163577856~16777216,184549376~33554432,222298112~16777216,243269632~33554432,289406976~8388608,306184192~33554432,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.872 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~88080384,104857600~8388608,125829120~33554432,163577856~16777216,184549376~33554432,222298112~16777216,243269632~33554432,289406976~8388608,306184192~33554432,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.877 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~92274688,104857600~8388608,125829120~33554432,163577856~75497472,243269632~37748736,289406976~12582912,306184192~33554432,343932928~25165824,373293056~41943040] 2024-09-14T04:36:38.878 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to 17 diff [0~62914560,67108864~20971520,104857600~8388608,125829120~54525952,184549376~33554432,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~4194304,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.878 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,104857600~8388608,125829120~54525952,184549376~33554432,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~4194304,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.883 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~92274688,104857600~8388608,125829120~113246208,243269632~37748736,289406976~50331648,343932928~25165824,373293056~41943040] 2024-09-14T04:36:38.883 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to 18 diff [0~62914560,67108864~20971520,104857600~8388608,125829120~54525952,184549376~33554432,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.883 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,104857600~8388608,125829120~54525952,184549376~33554432,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.889 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~92274688,104857600~8388608,125829120~113246208,243269632~37748736,289406976~79691776,373293056~41943040] 2024-09-14T04:36:38.889 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to 19 diff [0~62914560,67108864~20971520,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.889 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.895 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~92274688,100663296~12582912,117440512~121634816,243269632~37748736,289406976~79691776,373293056~41943040] 2024-09-14T04:36:38.895 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to 7 diff [29360128~8388608,75497472~4194304,171966464~8388608,192937984~4194304,201326592~4194304,306184192~4194304,335544320~4194304,343932928~4194304,356515840~8388608,394264576~8388608] 2024-09-14T04:36:38.895 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [29360128~8388608,75497472~4194304,171966464~8388608,192937984~4194304,201326592~4194304,306184192~4194304,335544320~4194304,343932928~4194304,356515840~8388608,394264576~8388608] 2024-09-14T04:36:38.896 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [29360128~8388608,75497472~4194304,171966464~12582912,192937984~16777216,306184192~8388608,335544320~4194304,343932928~4194304,356515840~8388608,394264576~12582912] 2024-09-14T04:36:38.896 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to 8 diff [25165824~12582912,75497472~8388608,88080384~4194304,104857600~8388608,125829120~8388608,142606336~4194304,163577856~4194304,171966464~8388608,192937984~4194304,201326592~8388608,306184192~4194304,335544320~4194304,343932928~8388608,356515840~8388608,394264576~16777216] 2024-09-14T04:36:38.896 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [25165824~12582912,75497472~8388608,88080384~4194304,104857600~8388608,125829120~8388608,142606336~4194304,163577856~4194304,171966464~8388608,192937984~4194304,201326592~8388608,306184192~4194304,335544320~4194304,343932928~8388608,356515840~8388608,394264576~16777216] 2024-09-14T04:36:38.898 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [25165824~12582912,75497472~16777216,104857600~8388608,125829120~12582912,142606336~8388608,163577856~4194304,171966464~12582912,192937984~16777216,306184192~8388608,335544320~4194304,343932928~8388608,356515840~8388608,394264576~16777216] 2024-09-14T04:36:38.898 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to 9 diff [25165824~12582912,46137344~8388608,75497472~12582912,104857600~8388608,125829120~8388608,138412032~8388608,163577856~4194304,171966464~8388608,192937984~16777216,260046848~8388608,306184192~4194304,331350016~8388608,343932928~8388608,356515840~8388608,394264576~16777216] 2024-09-14T04:36:38.898 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [25165824~12582912,46137344~8388608,75497472~12582912,104857600~8388608,125829120~8388608,138412032~8388608,163577856~4194304,171966464~8388608,192937984~16777216,260046848~8388608,306184192~4194304,331350016~8388608,343932928~8388608,356515840~8388608,394264576~16777216] 2024-09-14T04:36:38.900 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [25165824~12582912,46137344~8388608,75497472~16777216,104857600~8388608,125829120~25165824,163577856~4194304,171966464~12582912,192937984~16777216,260046848~12582912,306184192~8388608,331350016~8388608,343932928~8388608,356515840~8388608,394264576~16777216] 2024-09-14T04:36:38.900 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to 10 diff [20971520~12582912,46137344~8388608,75497472~12582912,104857600~8388608,125829120~8388608,138412032~12582912,163577856~4194304,171966464~8388608,192937984~16777216,243269632~25165824,276824064~4194304,297795584~4194304,306184192~4194304,331350016~8388608,343932928~8388608,356515840~8388608,394264576~16777216] 2024-09-14T04:36:38.900 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [20971520~12582912,46137344~8388608,75497472~12582912,104857600~8388608,125829120~8388608,138412032~12582912,163577856~4194304,171966464~8388608,192937984~16777216,243269632~25165824,276824064~4194304,297795584~4194304,306184192~4194304,331350016~8388608,343932928~8388608,356515840~8388608,394264576~16777216] 2024-09-14T04:36:38.903 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [20971520~16777216,46137344~12582912,75497472~16777216,104857600~8388608,125829120~25165824,163577856~4194304,171966464~12582912,192937984~16777216,243269632~29360128,276824064~4194304,297795584~4194304,306184192~8388608,331350016~8388608,343932928~8388608,356515840~12582912,394264576~16777216] 2024-09-14T04:36:38.903 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to 11 diff [20971520~20971520,46137344~12582912,67108864~4194304,75497472~12582912,104857600~8388608,125829120~8388608,138412032~12582912,163577856~4194304,171966464~8388608,192937984~16777216,222298112~8388608,243269632~25165824,276824064~4194304,297795584~4194304,306184192~4194304,331350016~8388608,343932928~8388608,356515840~8388608,385875968~25165824] 2024-09-14T04:36:38.903 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [20971520~20971520,46137344~12582912,67108864~4194304,75497472~12582912,104857600~8388608,125829120~8388608,138412032~12582912,163577856~4194304,171966464~8388608,192937984~16777216,222298112~8388608,243269632~25165824,276824064~4194304,297795584~4194304,306184192~4194304,331350016~8388608,343932928~8388608,356515840~8388608,385875968~25165824] 2024-09-14T04:36:38.905 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [20971520~20971520,46137344~12582912,67108864~4194304,75497472~16777216,104857600~8388608,125829120~25165824,163577856~4194304,171966464~12582912,192937984~16777216,222298112~12582912,243269632~29360128,276824064~4194304,297795584~4194304,306184192~8388608,331350016~8388608,343932928~8388608,356515840~12582912,385875968~25165824] 2024-09-14T04:36:38.905 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to 12 diff [0~8388608,12582912~29360128,46137344~12582912,67108864~4194304,75497472~12582912,104857600~8388608,125829120~25165824,163577856~16777216,192937984~16777216,222298112~8388608,243269632~25165824,276824064~4194304,297795584~4194304,306184192~4194304,331350016~8388608,343932928~8388608,356515840~8388608,385875968~25165824] 2024-09-14T04:36:38.906 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~29360128,46137344~12582912,67108864~4194304,75497472~12582912,104857600~8388608,125829120~25165824,163577856~16777216,192937984~16777216,222298112~8388608,243269632~25165824,276824064~4194304,297795584~4194304,306184192~4194304,331350016~8388608,343932928~8388608,356515840~8388608,385875968~25165824] 2024-09-14T04:36:38.909 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~8388608,12582912~29360128,46137344~12582912,67108864~4194304,75497472~16777216,104857600~8388608,125829120~25165824,163577856~20971520,192937984~16777216,222298112~12582912,243269632~29360128,276824064~4194304,297795584~4194304,306184192~8388608,331350016~8388608,343932928~8388608,356515840~12582912,385875968~25165824] 2024-09-14T04:36:38.909 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to 13 diff [0~8388608,12582912~46137344,67108864~4194304,75497472~12582912,104857600~8388608,125829120~25165824,163577856~16777216,192937984~25165824,222298112~16777216,243269632~25165824,276824064~4194304,297795584~4194304,306184192~4194304,331350016~8388608,343932928~20971520,373293056~4194304,385875968~25165824] 2024-09-14T04:36:38.909 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~46137344,67108864~4194304,75497472~12582912,104857600~8388608,125829120~25165824,163577856~16777216,192937984~25165824,222298112~16777216,243269632~25165824,276824064~4194304,297795584~4194304,306184192~4194304,331350016~8388608,343932928~20971520,373293056~4194304,385875968~25165824] 2024-09-14T04:36:38.912 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~8388608,12582912~46137344,67108864~4194304,75497472~16777216,104857600~8388608,125829120~25165824,163577856~20971520,192937984~46137344,243269632~29360128,276824064~4194304,297795584~4194304,306184192~8388608,331350016~8388608,343932928~25165824,373293056~8388608,385875968~25165824] 2024-09-14T04:36:38.912 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to 14 diff [0~8388608,12582912~58720256,75497472~12582912,104857600~8388608,125829120~29360128,163577856~16777216,192937984~25165824,222298112~16777216,243269632~33554432,297795584~4194304,306184192~4194304,318767104~8388608,331350016~8388608,343932928~25165824,373293056~4194304,385875968~25165824] 2024-09-14T04:36:38.912 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~58720256,75497472~12582912,104857600~8388608,125829120~29360128,163577856~16777216,192937984~25165824,222298112~16777216,243269632~33554432,297795584~4194304,306184192~4194304,318767104~8388608,331350016~8388608,343932928~25165824,373293056~4194304,385875968~25165824] 2024-09-14T04:36:38.916 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~8388608,12582912~58720256,75497472~16777216,104857600~8388608,125829120~29360128,163577856~20971520,192937984~46137344,243269632~37748736,297795584~4194304,306184192~8388608,318767104~20971520,343932928~25165824,373293056~8388608,385875968~29360128] 2024-09-14T04:36:38.916 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to 15 diff [0~71303168,75497472~12582912,104857600~8388608,125829120~33554432,163577856~16777216,188743680~29360128,222298112~16777216,243269632~33554432,297795584~4194304,306184192~4194304,318767104~8388608,331350016~8388608,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.916 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~71303168,75497472~12582912,104857600~8388608,125829120~33554432,163577856~16777216,188743680~29360128,222298112~16777216,243269632~33554432,297795584~4194304,306184192~4194304,318767104~8388608,331350016~8388608,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.919 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,104857600~8388608,125829120~33554432,163577856~20971520,188743680~50331648,243269632~37748736,297795584~4194304,306184192~8388608,318767104~20971520,343932928~25165824,373293056~41943040] 2024-09-14T04:36:38.919 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to 16 diff [0~71303168,75497472~12582912,104857600~8388608,125829120~33554432,163577856~16777216,188743680~29360128,222298112~16777216,243269632~33554432,289406976~8388608,306184192~33554432,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.920 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~71303168,75497472~12582912,104857600~8388608,125829120~33554432,163577856~16777216,188743680~29360128,222298112~16777216,243269632~33554432,289406976~8388608,306184192~33554432,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.924 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,104857600~8388608,125829120~33554432,163577856~20971520,188743680~50331648,243269632~37748736,289406976~12582912,306184192~33554432,343932928~25165824,373293056~41943040] 2024-09-14T04:36:38.924 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to 17 diff [0~62914560,67108864~4194304,75497472~12582912,104857600~8388608,125829120~54525952,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~4194304,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.924 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~12582912,104857600~8388608,125829120~54525952,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~4194304,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.928 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,104857600~8388608,125829120~58720256,188743680~50331648,243269632~37748736,289406976~50331648,343932928~25165824,373293056~41943040] 2024-09-14T04:36:38.928 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to 18 diff [0~62914560,67108864~4194304,75497472~12582912,104857600~8388608,125829120~54525952,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.928 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~12582912,104857600~8388608,125829120~54525952,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.934 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,104857600~8388608,125829120~58720256,188743680~50331648,243269632~37748736,289406976~79691776,373293056~41943040] 2024-09-14T04:36:38.934 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to 19 diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.934 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.939 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~79691776,373293056~41943040] 2024-09-14T04:36:38.939 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to 8 diff [25165824~4194304,75497472~8388608,88080384~4194304,104857600~8388608,125829120~8388608,142606336~4194304,163577856~4194304,201326592~4194304,343932928~8388608,402653184~8388608] 2024-09-14T04:36:38.939 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [25165824~4194304,75497472~8388608,88080384~4194304,104857600~8388608,125829120~8388608,142606336~4194304,163577856~4194304,201326592~4194304,343932928~8388608,402653184~8388608] 2024-09-14T04:36:38.940 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [25165824~4194304,75497472~16777216,104857600~8388608,125829120~12582912,142606336~8388608,163577856~4194304,201326592~8388608,343932928~8388608,402653184~8388608] 2024-09-14T04:36:38.940 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to 9 diff [25165824~8388608,46137344~8388608,75497472~12582912,104857600~8388608,125829120~8388608,138412032~8388608,163577856~4194304,192937984~4194304,201326592~4194304,260046848~8388608,331350016~4194304,343932928~8388608,402653184~8388608] 2024-09-14T04:36:38.940 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [25165824~8388608,46137344~8388608,75497472~12582912,104857600~8388608,125829120~8388608,138412032~8388608,163577856~4194304,192937984~4194304,201326592~4194304,260046848~8388608,331350016~4194304,343932928~8388608,402653184~8388608] 2024-09-14T04:36:38.942 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [25165824~8388608,46137344~8388608,75497472~16777216,104857600~8388608,125829120~25165824,163577856~4194304,192937984~16777216,260046848~12582912,331350016~4194304,343932928~8388608,402653184~8388608] 2024-09-14T04:36:38.942 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to 10 diff [20971520~12582912,46137344~8388608,75497472~12582912,104857600~8388608,125829120~8388608,138412032~12582912,163577856~4194304,192937984~4194304,201326592~4194304,243269632~25165824,276824064~4194304,297795584~4194304,331350016~4194304,343932928~8388608,356515840~8388608,402653184~8388608] 2024-09-14T04:36:38.942 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [20971520~12582912,46137344~8388608,75497472~12582912,104857600~8388608,125829120~8388608,138412032~12582912,163577856~4194304,192937984~4194304,201326592~4194304,243269632~25165824,276824064~4194304,297795584~4194304,331350016~4194304,343932928~8388608,356515840~8388608,402653184~8388608] 2024-09-14T04:36:38.944 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [20971520~12582912,46137344~12582912,75497472~16777216,104857600~8388608,125829120~25165824,163577856~4194304,192937984~16777216,243269632~29360128,276824064~4194304,297795584~4194304,331350016~4194304,343932928~8388608,356515840~12582912,402653184~8388608] 2024-09-14T04:36:38.944 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to 11 diff [20971520~20971520,46137344~12582912,67108864~4194304,75497472~12582912,104857600~8388608,125829120~8388608,138412032~12582912,163577856~4194304,171966464~8388608,192937984~4194304,201326592~4194304,222298112~8388608,243269632~25165824,276824064~4194304,297795584~4194304,331350016~4194304,343932928~8388608,356515840~8388608,385875968~8388608,402653184~8388608] 2024-09-14T04:36:38.944 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [20971520~20971520,46137344~12582912,67108864~4194304,75497472~12582912,104857600~8388608,125829120~8388608,138412032~12582912,163577856~4194304,171966464~8388608,192937984~4194304,201326592~4194304,222298112~8388608,243269632~25165824,276824064~4194304,297795584~4194304,331350016~4194304,343932928~8388608,356515840~8388608,385875968~8388608,402653184~8388608] 2024-09-14T04:36:38.946 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [20971520~20971520,46137344~12582912,67108864~4194304,75497472~16777216,104857600~8388608,125829120~25165824,163577856~4194304,171966464~8388608,192937984~16777216,222298112~12582912,243269632~29360128,276824064~4194304,297795584~4194304,331350016~4194304,343932928~8388608,356515840~12582912,385875968~8388608,402653184~8388608] 2024-09-14T04:36:38.946 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to 12 diff [0~8388608,12582912~29360128,46137344~12582912,67108864~4194304,75497472~12582912,104857600~8388608,125829120~25165824,163577856~12582912,192937984~4194304,201326592~4194304,222298112~8388608,243269632~25165824,276824064~4194304,297795584~4194304,331350016~4194304,343932928~8388608,356515840~8388608,385875968~8388608,398458880~12582912] 2024-09-14T04:36:38.946 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~29360128,46137344~12582912,67108864~4194304,75497472~12582912,104857600~8388608,125829120~25165824,163577856~12582912,192937984~4194304,201326592~4194304,222298112~8388608,243269632~25165824,276824064~4194304,297795584~4194304,331350016~4194304,343932928~8388608,356515840~8388608,385875968~8388608,398458880~12582912] 2024-09-14T04:36:38.949 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~8388608,12582912~29360128,46137344~12582912,67108864~4194304,75497472~16777216,104857600~8388608,125829120~25165824,163577856~16777216,192937984~16777216,222298112~12582912,243269632~29360128,276824064~4194304,297795584~4194304,331350016~4194304,343932928~8388608,356515840~12582912,385875968~8388608,398458880~12582912] 2024-09-14T04:36:38.949 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to 13 diff [0~8388608,12582912~46137344,67108864~4194304,75497472~12582912,104857600~8388608,125829120~25165824,163577856~12582912,192937984~25165824,222298112~16777216,243269632~25165824,276824064~4194304,297795584~4194304,331350016~4194304,343932928~20971520,373293056~4194304,385875968~8388608,398458880~12582912] 2024-09-14T04:36:38.949 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~46137344,67108864~4194304,75497472~12582912,104857600~8388608,125829120~25165824,163577856~12582912,192937984~25165824,222298112~16777216,243269632~25165824,276824064~4194304,297795584~4194304,331350016~4194304,343932928~20971520,373293056~4194304,385875968~8388608,398458880~12582912] 2024-09-14T04:36:38.952 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~8388608,12582912~46137344,67108864~4194304,75497472~16777216,104857600~8388608,125829120~25165824,163577856~16777216,192937984~46137344,243269632~29360128,276824064~4194304,297795584~4194304,331350016~4194304,343932928~25165824,373293056~8388608,385875968~8388608,398458880~12582912] 2024-09-14T04:36:38.952 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to 14 diff [0~8388608,12582912~58720256,75497472~12582912,104857600~8388608,125829120~29360128,163577856~12582912,192937984~25165824,222298112~16777216,243269632~33554432,297795584~4194304,318767104~8388608,331350016~4194304,343932928~20971520,373293056~4194304,385875968~8388608,398458880~12582912] 2024-09-14T04:36:38.952 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~58720256,75497472~12582912,104857600~8388608,125829120~29360128,163577856~12582912,192937984~25165824,222298112~16777216,243269632~33554432,297795584~4194304,318767104~8388608,331350016~4194304,343932928~20971520,373293056~4194304,385875968~8388608,398458880~12582912] 2024-09-14T04:36:38.955 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~8388608,12582912~58720256,75497472~16777216,104857600~8388608,125829120~29360128,163577856~16777216,192937984~46137344,243269632~37748736,297795584~4194304,318767104~16777216,343932928~25165824,373293056~8388608,385875968~8388608,398458880~16777216] 2024-09-14T04:36:38.955 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to 15 diff [0~71303168,75497472~12582912,104857600~8388608,125829120~33554432,163577856~12582912,188743680~29360128,222298112~16777216,243269632~33554432,297795584~4194304,318767104~8388608,331350016~4194304,343932928~20971520,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.956 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~71303168,75497472~12582912,104857600~8388608,125829120~33554432,163577856~12582912,188743680~29360128,222298112~16777216,243269632~33554432,297795584~4194304,318767104~8388608,331350016~4194304,343932928~20971520,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.959 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,104857600~8388608,125829120~33554432,163577856~16777216,188743680~50331648,243269632~37748736,297795584~4194304,318767104~16777216,343932928~25165824,373293056~41943040] 2024-09-14T04:36:38.959 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to 16 diff [0~71303168,75497472~12582912,104857600~8388608,125829120~33554432,163577856~12582912,188743680~29360128,222298112~16777216,243269632~33554432,289406976~8388608,310378496~25165824,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.959 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~71303168,75497472~12582912,104857600~8388608,125829120~33554432,163577856~12582912,188743680~29360128,222298112~16777216,243269632~33554432,289406976~8388608,310378496~25165824,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.962 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,104857600~8388608,125829120~33554432,163577856~16777216,188743680~50331648,243269632~37748736,289406976~12582912,310378496~25165824,343932928~25165824,373293056~41943040] 2024-09-14T04:36:38.963 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to 17 diff [0~62914560,67108864~4194304,75497472~12582912,104857600~8388608,125829120~50331648,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.963 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~12582912,104857600~8388608,125829120~50331648,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.967 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,104857600~8388608,125829120~54525952,188743680~50331648,243269632~37748736,289406976~16777216,310378496~25165824,343932928~25165824,373293056~41943040] 2024-09-14T04:36:38.967 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to 18 diff [0~62914560,67108864~4194304,75497472~12582912,104857600~8388608,125829120~50331648,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.967 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~12582912,104857600~8388608,125829120~50331648,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.971 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,104857600~8388608,125829120~54525952,188743680~50331648,243269632~37748736,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:38.971 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to 19 diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.972 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.976 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:38.977 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to 9 diff [25165824~8388608,46137344~8388608,79691776~8388608,104857600~8388608,138412032~8388608,192937984~4194304,260046848~8388608,331350016~4194304] 2024-09-14T04:36:38.977 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [25165824~8388608,46137344~8388608,79691776~8388608,104857600~8388608,138412032~8388608,192937984~4194304,260046848~8388608,331350016~4194304] 2024-09-14T04:36:38.977 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [25165824~8388608,46137344~8388608,79691776~12582912,104857600~8388608,138412032~8388608,192937984~8388608,260046848~12582912,331350016~4194304] 2024-09-14T04:36:38.978 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to 10 diff [20971520~12582912,46137344~8388608,79691776~8388608,104857600~8388608,138412032~12582912,192937984~4194304,243269632~25165824,276824064~4194304,297795584~4194304,331350016~4194304,356515840~8388608] 2024-09-14T04:36:38.978 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [20971520~12582912,46137344~8388608,79691776~8388608,104857600~8388608,138412032~12582912,192937984~4194304,243269632~25165824,276824064~4194304,297795584~4194304,331350016~4194304,356515840~8388608] 2024-09-14T04:36:38.979 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [20971520~12582912,46137344~12582912,79691776~12582912,104857600~8388608,138412032~12582912,192937984~8388608,243269632~29360128,276824064~4194304,297795584~4194304,331350016~4194304,356515840~12582912] 2024-09-14T04:36:38.979 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to 11 diff [20971520~20971520,46137344~12582912,67108864~4194304,79691776~8388608,104857600~8388608,138412032~12582912,171966464~8388608,192937984~4194304,222298112~8388608,243269632~25165824,276824064~4194304,297795584~4194304,331350016~4194304,343932928~8388608,356515840~8388608,385875968~8388608] 2024-09-14T04:36:38.979 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [20971520~20971520,46137344~12582912,67108864~4194304,79691776~8388608,104857600~8388608,138412032~12582912,171966464~8388608,192937984~4194304,222298112~8388608,243269632~25165824,276824064~4194304,297795584~4194304,331350016~4194304,343932928~8388608,356515840~8388608,385875968~8388608] 2024-09-14T04:36:38.980 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [20971520~20971520,46137344~12582912,67108864~4194304,79691776~12582912,104857600~8388608,138412032~12582912,171966464~8388608,192937984~8388608,222298112~12582912,243269632~29360128,276824064~4194304,297795584~4194304,331350016~4194304,343932928~8388608,356515840~12582912,385875968~8388608] 2024-09-14T04:36:38.980 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to 12 diff [0~8388608,12582912~29360128,46137344~12582912,67108864~4194304,79691776~8388608,104857600~8388608,134217728~16777216,167772160~8388608,192937984~4194304,222298112~8388608,243269632~25165824,276824064~4194304,297795584~4194304,331350016~4194304,343932928~8388608,356515840~8388608,385875968~8388608,398458880~8388608] 2024-09-14T04:36:38.981 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~29360128,46137344~12582912,67108864~4194304,79691776~8388608,104857600~8388608,134217728~16777216,167772160~8388608,192937984~4194304,222298112~8388608,243269632~25165824,276824064~4194304,297795584~4194304,331350016~4194304,343932928~8388608,356515840~8388608,385875968~8388608,398458880~8388608] 2024-09-14T04:36:38.983 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~8388608,12582912~29360128,46137344~12582912,67108864~4194304,79691776~12582912,104857600~8388608,134217728~16777216,167772160~12582912,192937984~8388608,222298112~12582912,243269632~29360128,276824064~4194304,297795584~4194304,331350016~4194304,343932928~8388608,356515840~12582912,385875968~8388608,398458880~12582912] 2024-09-14T04:36:38.983 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to 13 diff [0~8388608,12582912~46137344,67108864~4194304,79691776~8388608,104857600~8388608,125829120~4194304,134217728~16777216,167772160~8388608,192937984~25165824,222298112~16777216,243269632~25165824,276824064~4194304,297795584~4194304,331350016~4194304,343932928~20971520,373293056~4194304,385875968~8388608,398458880~8388608] 2024-09-14T04:36:38.983 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~46137344,67108864~4194304,79691776~8388608,104857600~8388608,125829120~4194304,134217728~16777216,167772160~8388608,192937984~25165824,222298112~16777216,243269632~25165824,276824064~4194304,297795584~4194304,331350016~4194304,343932928~20971520,373293056~4194304,385875968~8388608,398458880~8388608] 2024-09-14T04:36:38.985 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~8388608,12582912~46137344,67108864~4194304,79691776~12582912,104857600~8388608,125829120~25165824,167772160~12582912,192937984~46137344,243269632~29360128,276824064~4194304,297795584~4194304,331350016~4194304,343932928~25165824,373293056~8388608,385875968~8388608,398458880~12582912] 2024-09-14T04:36:38.985 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to 14 diff [0~8388608,12582912~58720256,79691776~8388608,104857600~8388608,125829120~4194304,134217728~20971520,167772160~8388608,192937984~25165824,222298112~16777216,243269632~33554432,297795584~4194304,318767104~8388608,331350016~4194304,343932928~20971520,373293056~4194304,385875968~8388608,398458880~12582912] 2024-09-14T04:36:38.986 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~58720256,79691776~8388608,104857600~8388608,125829120~4194304,134217728~20971520,167772160~8388608,192937984~25165824,222298112~16777216,243269632~33554432,297795584~4194304,318767104~8388608,331350016~4194304,343932928~20971520,373293056~4194304,385875968~8388608,398458880~12582912] 2024-09-14T04:36:38.989 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~8388608,12582912~58720256,79691776~12582912,104857600~8388608,125829120~29360128,167772160~12582912,192937984~46137344,243269632~37748736,297795584~4194304,318767104~16777216,343932928~25165824,373293056~8388608,385875968~8388608,398458880~16777216] 2024-09-14T04:36:38.989 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to 15 diff [0~71303168,79691776~8388608,104857600~8388608,125829120~4194304,134217728~25165824,167772160~8388608,188743680~29360128,222298112~16777216,243269632~33554432,297795584~4194304,318767104~8388608,331350016~4194304,343932928~20971520,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.989 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~71303168,79691776~8388608,104857600~8388608,125829120~4194304,134217728~25165824,167772160~8388608,188743680~29360128,222298112~16777216,243269632~33554432,297795584~4194304,318767104~8388608,331350016~4194304,343932928~20971520,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.991 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,79691776~12582912,104857600~8388608,125829120~33554432,167772160~12582912,188743680~50331648,243269632~37748736,297795584~4194304,318767104~16777216,343932928~25165824,373293056~41943040] 2024-09-14T04:36:38.991 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to 16 diff [0~71303168,75497472~16777216,104857600~8388608,125829120~4194304,134217728~25165824,167772160~8388608,188743680~29360128,222298112~16777216,243269632~33554432,289406976~8388608,310378496~25165824,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.991 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~71303168,75497472~16777216,104857600~8388608,125829120~4194304,134217728~25165824,167772160~8388608,188743680~29360128,222298112~16777216,243269632~33554432,289406976~8388608,310378496~25165824,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.996 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,104857600~8388608,125829120~33554432,167772160~12582912,188743680~50331648,243269632~37748736,289406976~12582912,310378496~25165824,343932928~25165824,373293056~41943040] 2024-09-14T04:36:38.996 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to 17 diff [0~62914560,67108864~4194304,75497472~16777216,104857600~8388608,125829120~4194304,134217728~41943040,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.996 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~16777216,104857600~8388608,125829120~4194304,134217728~41943040,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:38.999 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,104857600~8388608,125829120~54525952,188743680~50331648,243269632~37748736,289406976~16777216,310378496~25165824,343932928~25165824,373293056~41943040] 2024-09-14T04:36:38.999 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to 18 diff [0~62914560,67108864~4194304,75497472~16777216,104857600~8388608,125829120~29360128,159383552~16777216,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.000 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~16777216,104857600~8388608,125829120~29360128,159383552~16777216,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.003 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,104857600~8388608,125829120~54525952,188743680~50331648,243269632~37748736,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:39.004 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to 19 diff [0~62914560,67108864~4194304,75497472~16777216,100663296~12582912,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.004 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~16777216,100663296~12582912,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.009 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:39.009 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to 10 diff [20971520~8388608,50331648~4194304,142606336~8388608,192937984~4194304,243269632~20971520,276824064~4194304,297795584~4194304,356515840~8388608] 2024-09-14T04:36:39.009 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [20971520~8388608,50331648~4194304,142606336~8388608,192937984~4194304,243269632~20971520,276824064~4194304,297795584~4194304,356515840~8388608] 2024-09-14T04:36:39.009 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [20971520~8388608,50331648~8388608,142606336~8388608,192937984~4194304,243269632~20971520,276824064~4194304,297795584~4194304,356515840~12582912] 2024-09-14T04:36:39.010 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to 11 diff [20971520~8388608,33554432~8388608,46137344~12582912,67108864~4194304,142606336~8388608,171966464~8388608,192937984~8388608,222298112~8388608,243269632~20971520,276824064~4194304,297795584~4194304,343932928~8388608,356515840~8388608,385875968~8388608] 2024-09-14T04:36:39.010 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [20971520~8388608,33554432~8388608,46137344~12582912,67108864~4194304,142606336~8388608,171966464~8388608,192937984~8388608,222298112~8388608,243269632~20971520,276824064~4194304,297795584~4194304,343932928~8388608,356515840~8388608,385875968~8388608] 2024-09-14T04:36:39.011 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [20971520~8388608,33554432~8388608,46137344~12582912,67108864~4194304,142606336~8388608,171966464~8388608,192937984~8388608,222298112~12582912,243269632~20971520,276824064~4194304,297795584~4194304,343932928~8388608,356515840~12582912,385875968~8388608] 2024-09-14T04:36:39.011 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to 12 diff [0~8388608,12582912~16777216,33554432~8388608,46137344~12582912,67108864~4194304,134217728~4194304,142606336~8388608,167772160~8388608,192937984~8388608,222298112~8388608,243269632~20971520,276824064~4194304,297795584~4194304,343932928~8388608,356515840~8388608,385875968~8388608,398458880~8388608] 2024-09-14T04:36:39.011 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~16777216,33554432~8388608,46137344~12582912,67108864~4194304,134217728~4194304,142606336~8388608,167772160~8388608,192937984~8388608,222298112~8388608,243269632~20971520,276824064~4194304,297795584~4194304,343932928~8388608,356515840~8388608,385875968~8388608,398458880~8388608] 2024-09-14T04:36:39.013 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~8388608,12582912~16777216,33554432~8388608,46137344~12582912,67108864~4194304,134217728~16777216,167772160~12582912,192937984~8388608,222298112~12582912,243269632~20971520,276824064~4194304,297795584~4194304,343932928~8388608,356515840~12582912,385875968~8388608,398458880~12582912] 2024-09-14T04:36:39.013 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to 13 diff [0~8388608,12582912~16777216,33554432~25165824,67108864~4194304,125829120~4194304,134217728~16777216,167772160~8388608,192937984~25165824,222298112~16777216,243269632~20971520,276824064~4194304,297795584~4194304,343932928~20971520,373293056~4194304,385875968~8388608,398458880~8388608] 2024-09-14T04:36:39.013 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~16777216,33554432~25165824,67108864~4194304,125829120~4194304,134217728~16777216,167772160~8388608,192937984~25165824,222298112~16777216,243269632~20971520,276824064~4194304,297795584~4194304,343932928~20971520,373293056~4194304,385875968~8388608,398458880~8388608] 2024-09-14T04:36:39.015 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~8388608,12582912~16777216,33554432~25165824,67108864~4194304,125829120~25165824,167772160~12582912,192937984~46137344,243269632~20971520,276824064~4194304,297795584~4194304,343932928~25165824,373293056~8388608,385875968~8388608,398458880~12582912] 2024-09-14T04:36:39.015 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to 14 diff [0~8388608,12582912~58720256,125829120~4194304,134217728~20971520,167772160~8388608,192937984~25165824,222298112~16777216,243269632~20971520,268435456~8388608,297795584~4194304,318767104~8388608,343932928~20971520,373293056~4194304,385875968~8388608,398458880~12582912] 2024-09-14T04:36:39.015 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~58720256,125829120~4194304,134217728~20971520,167772160~8388608,192937984~25165824,222298112~16777216,243269632~20971520,268435456~8388608,297795584~4194304,318767104~8388608,343932928~20971520,373293056~4194304,385875968~8388608,398458880~12582912] 2024-09-14T04:36:39.018 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~8388608,12582912~58720256,125829120~29360128,167772160~12582912,192937984~46137344,243269632~20971520,268435456~12582912,297795584~4194304,318767104~12582912,343932928~25165824,373293056~8388608,385875968~8388608,398458880~16777216] 2024-09-14T04:36:39.018 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to 15 diff [0~71303168,83886080~4194304,125829120~4194304,134217728~25165824,167772160~8388608,188743680~29360128,222298112~16777216,243269632~20971520,268435456~8388608,297795584~4194304,318767104~8388608,343932928~20971520,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.018 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~71303168,83886080~4194304,125829120~4194304,134217728~25165824,167772160~8388608,188743680~29360128,222298112~16777216,243269632~20971520,268435456~8388608,297795584~4194304,318767104~8388608,343932928~20971520,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.021 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,83886080~8388608,125829120~33554432,167772160~12582912,188743680~50331648,243269632~20971520,268435456~12582912,297795584~4194304,318767104~12582912,343932928~25165824,373293056~41943040] 2024-09-14T04:36:39.021 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to 16 diff [0~29360128,33554432~37748736,75497472~16777216,125829120~4194304,134217728~25165824,167772160~8388608,188743680~29360128,222298112~16777216,243269632~20971520,268435456~8388608,289406976~8388608,310378496~20971520,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.021 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~29360128,33554432~37748736,75497472~16777216,125829120~4194304,134217728~25165824,167772160~8388608,188743680~29360128,222298112~16777216,243269632~20971520,268435456~8388608,289406976~8388608,310378496~20971520,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.024 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,125829120~33554432,167772160~12582912,188743680~50331648,243269632~20971520,268435456~12582912,289406976~12582912,310378496~20971520,343932928~25165824,373293056~41943040] 2024-09-14T04:36:39.024 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to 17 diff [0~62914560,67108864~4194304,75497472~16777216,125829120~4194304,134217728~41943040,188743680~29360128,222298112~16777216,243269632~20971520,268435456~8388608,289406976~12582912,310378496~20971520,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.024 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~16777216,125829120~4194304,134217728~41943040,188743680~29360128,222298112~16777216,243269632~20971520,268435456~8388608,289406976~12582912,310378496~20971520,343932928~25165824,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.028 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,125829120~54525952,188743680~50331648,243269632~20971520,268435456~12582912,289406976~16777216,310378496~25165824,343932928~25165824,373293056~41943040] 2024-09-14T04:36:39.028 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to 18 diff [0~62914560,67108864~4194304,75497472~16777216,125829120~50331648,188743680~29360128,222298112~16777216,243269632~20971520,268435456~8388608,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.028 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~16777216,125829120~50331648,188743680~29360128,222298112~16777216,243269632~20971520,268435456~8388608,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.032 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,125829120~54525952,188743680~50331648,243269632~20971520,268435456~12582912,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:39.032 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to 19 diff [0~62914560,67108864~4194304,75497472~16777216,100663296~8388608,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~20971520,268435456~8388608,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.032 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~16777216,100663296~8388608,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~20971520,268435456~8388608,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.036 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~8388608,117440512~121634816,243269632~20971520,268435456~12582912,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:39.036 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to 11 diff [20971520~8388608,33554432~8388608,46137344~8388608,67108864~4194304,171966464~8388608,197132288~4194304,222298112~8388608,343932928~8388608,385875968~8388608] 2024-09-14T04:36:39.037 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [20971520~8388608,33554432~8388608,46137344~8388608,67108864~4194304,171966464~8388608,197132288~4194304,222298112~8388608,343932928~8388608,385875968~8388608] 2024-09-14T04:36:39.037 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [20971520~8388608,33554432~8388608,46137344~12582912,67108864~4194304,171966464~8388608,197132288~4194304,222298112~12582912,343932928~8388608,385875968~8388608] 2024-09-14T04:36:39.037 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to 12 diff [0~8388608,12582912~16777216,33554432~8388608,46137344~12582912,67108864~4194304,134217728~4194304,142606336~8388608,167772160~8388608,197132288~4194304,222298112~8388608,343932928~8388608,385875968~8388608,398458880~8388608] 2024-09-14T04:36:39.038 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~16777216,33554432~8388608,46137344~12582912,67108864~4194304,134217728~4194304,142606336~8388608,167772160~8388608,197132288~4194304,222298112~8388608,343932928~8388608,385875968~8388608,398458880~8388608] 2024-09-14T04:36:39.039 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~8388608,12582912~16777216,33554432~8388608,46137344~12582912,67108864~4194304,134217728~16777216,167772160~12582912,197132288~4194304,222298112~12582912,343932928~8388608,385875968~8388608,398458880~12582912] 2024-09-14T04:36:39.040 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to 13 diff [0~8388608,12582912~16777216,33554432~25165824,67108864~4194304,125829120~4194304,134217728~16777216,167772160~8388608,197132288~20971520,222298112~16777216,343932928~12582912,373293056~4194304,385875968~8388608,398458880~8388608] 2024-09-14T04:36:39.040 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~16777216,33554432~25165824,67108864~4194304,125829120~4194304,134217728~16777216,167772160~8388608,197132288~20971520,222298112~16777216,343932928~12582912,373293056~4194304,385875968~8388608,398458880~8388608] 2024-09-14T04:36:39.040 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~8388608,12582912~16777216,33554432~25165824,67108864~4194304,125829120~25165824,167772160~12582912,197132288~41943040,343932928~12582912,373293056~8388608,385875968~8388608,398458880~12582912] 2024-09-14T04:36:39.041 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to 14 diff [0~8388608,12582912~58720256,125829120~4194304,134217728~20971520,167772160~8388608,197132288~20971520,222298112~16777216,268435456~8388608,318767104~8388608,343932928~16777216,373293056~4194304,385875968~8388608,398458880~12582912] 2024-09-14T04:36:39.041 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~58720256,125829120~4194304,134217728~20971520,167772160~8388608,197132288~20971520,222298112~16777216,268435456~8388608,318767104~8388608,343932928~16777216,373293056~4194304,385875968~8388608,398458880~12582912] 2024-09-14T04:36:39.043 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~8388608,12582912~58720256,125829120~29360128,167772160~12582912,197132288~41943040,268435456~12582912,318767104~12582912,343932928~16777216,373293056~8388608,385875968~8388608,398458880~16777216] 2024-09-14T04:36:39.044 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to 15 diff [0~71303168,83886080~4194304,125829120~4194304,134217728~25165824,167772160~8388608,188743680~29360128,222298112~16777216,251658240~4194304,268435456~8388608,318767104~8388608,343932928~16777216,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.044 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~71303168,83886080~4194304,125829120~4194304,134217728~25165824,167772160~8388608,188743680~29360128,222298112~16777216,251658240~4194304,268435456~8388608,318767104~8388608,343932928~16777216,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.047 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,83886080~8388608,125829120~33554432,167772160~12582912,188743680~50331648,251658240~8388608,268435456~12582912,318767104~12582912,343932928~16777216,373293056~41943040] 2024-09-14T04:36:39.047 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to 16 diff [0~29360128,33554432~37748736,75497472~16777216,125829120~4194304,134217728~25165824,167772160~8388608,188743680~29360128,222298112~16777216,251658240~4194304,268435456~8388608,289406976~8388608,310378496~20971520,343932928~20971520,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.047 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~29360128,33554432~37748736,75497472~16777216,125829120~4194304,134217728~25165824,167772160~8388608,188743680~29360128,222298112~16777216,251658240~4194304,268435456~8388608,289406976~8388608,310378496~20971520,343932928~20971520,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.050 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,125829120~33554432,167772160~12582912,188743680~50331648,251658240~8388608,268435456~12582912,289406976~12582912,310378496~20971520,343932928~20971520,373293056~41943040] 2024-09-14T04:36:39.050 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to 17 diff [0~71303168,75497472~16777216,125829120~4194304,134217728~41943040,188743680~29360128,222298112~16777216,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,343932928~20971520,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.050 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~71303168,75497472~16777216,125829120~4194304,134217728~41943040,188743680~29360128,222298112~16777216,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,343932928~20971520,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.054 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,125829120~54525952,188743680~50331648,251658240~8388608,268435456~12582912,289406976~16777216,310378496~25165824,343932928~20971520,373293056~41943040] 2024-09-14T04:36:39.054 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to 18 diff [0~71303168,75497472~16777216,125829120~50331648,188743680~29360128,222298112~16777216,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~29360128,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.054 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~71303168,75497472~16777216,125829120~50331648,188743680~29360128,222298112~16777216,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~29360128,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.058 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,125829120~54525952,188743680~50331648,251658240~8388608,268435456~12582912,289406976~16777216,310378496~54525952,373293056~41943040] 2024-09-14T04:36:39.058 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to 19 diff [0~71303168,75497472~16777216,100663296~8388608,117440512~67108864,188743680~29360128,222298112~16777216,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~29360128,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.058 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~71303168,75497472~16777216,100663296~8388608,117440512~67108864,188743680~29360128,222298112~16777216,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~29360128,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.062 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~8388608,117440512~121634816,251658240~8388608,268435456~12582912,289406976~16777216,310378496~54525952,373293056~41943040] 2024-09-14T04:36:39.062 INFO:tasks.workunit.client.0.smithi039.stdout:from 11 to 12 diff [0~8388608,12582912~8388608,54525952~4194304,134217728~4194304,142606336~8388608,167772160~4194304,226492416~8388608,398458880~8388608] 2024-09-14T04:36:39.062 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~8388608,54525952~4194304,134217728~4194304,142606336~8388608,167772160~4194304,226492416~8388608,398458880~8388608] 2024-09-14T04:36:39.063 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~8388608,12582912~8388608,54525952~4194304,134217728~16777216,167772160~8388608,226492416~8388608,398458880~12582912] 2024-09-14T04:36:39.063 INFO:tasks.workunit.client.0.smithi039.stdout:from 11 to 13 diff [0~8388608,12582912~8388608,41943040~8388608,54525952~4194304,125829120~4194304,134217728~16777216,167772160~4194304,197132288~20971520,226492416~12582912,352321536~4194304,373293056~4194304,398458880~8388608] 2024-09-14T04:36:39.063 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~8388608,41943040~8388608,54525952~4194304,125829120~4194304,134217728~16777216,167772160~4194304,197132288~20971520,226492416~12582912,352321536~4194304,373293056~4194304,398458880~8388608] 2024-09-14T04:36:39.064 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~8388608,12582912~8388608,41943040~8388608,54525952~4194304,125829120~25165824,167772160~8388608,197132288~25165824,226492416~12582912,352321536~4194304,373293056~8388608,398458880~12582912] 2024-09-14T04:36:39.064 INFO:tasks.workunit.client.0.smithi039.stdout:from 11 to 14 diff [0~8388608,12582912~8388608,29360128~4194304,37748736~8388608,54525952~12582912,125829120~4194304,134217728~20971520,167772160~4194304,197132288~20971520,226492416~12582912,268435456~8388608,318767104~8388608,352321536~8388608,373293056~4194304,398458880~12582912] 2024-09-14T04:36:39.064 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~8388608,12582912~8388608,29360128~4194304,37748736~8388608,54525952~12582912,125829120~4194304,134217728~20971520,167772160~4194304,197132288~20971520,226492416~12582912,268435456~8388608,318767104~8388608,352321536~8388608,373293056~4194304,398458880~12582912] 2024-09-14T04:36:39.066 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~8388608,12582912~8388608,29360128~4194304,37748736~12582912,54525952~12582912,125829120~29360128,167772160~8388608,197132288~25165824,226492416~12582912,268435456~12582912,318767104~12582912,352321536~8388608,373293056~8388608,398458880~16777216] 2024-09-14T04:36:39.066 INFO:tasks.workunit.client.0.smithi039.stdout:from 11 to 15 diff [0~20971520,29360128~4194304,37748736~8388608,54525952~12582912,83886080~4194304,125829120~4194304,134217728~25165824,167772160~4194304,188743680~29360128,226492416~12582912,251658240~4194304,268435456~8388608,318767104~8388608,352321536~8388608,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.066 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~20971520,29360128~4194304,37748736~8388608,54525952~12582912,83886080~4194304,125829120~4194304,134217728~25165824,167772160~4194304,188743680~29360128,226492416~12582912,251658240~4194304,268435456~8388608,318767104~8388608,352321536~8388608,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.068 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~20971520,29360128~4194304,37748736~12582912,54525952~12582912,83886080~8388608,125829120~33554432,167772160~8388608,188743680~33554432,226492416~12582912,251658240~8388608,268435456~12582912,318767104~12582912,352321536~8388608,373293056~41943040] 2024-09-14T04:36:39.068 INFO:tasks.workunit.client.0.smithi039.stdout:from 11 to 16 diff [0~20971520,25165824~4194304,37748736~8388608,54525952~12582912,75497472~16777216,125829120~4194304,134217728~25165824,167772160~4194304,188743680~29360128,226492416~12582912,251658240~4194304,268435456~8388608,289406976~8388608,310378496~20971520,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.068 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~20971520,25165824~4194304,37748736~8388608,54525952~12582912,75497472~16777216,125829120~4194304,134217728~25165824,167772160~4194304,188743680~29360128,226492416~12582912,251658240~4194304,268435456~8388608,289406976~8388608,310378496~20971520,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.071 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~20971520,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,125829120~33554432,167772160~8388608,188743680~33554432,226492416~12582912,251658240~8388608,268435456~12582912,289406976~12582912,310378496~20971520,352321536~12582912,373293056~41943040] 2024-09-14T04:36:39.071 INFO:tasks.workunit.client.0.smithi039.stdout:from 11 to 17 diff [0~20971520,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,125829120~4194304,134217728~37748736,188743680~29360128,226492416~12582912,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.071 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~20971520,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,125829120~4194304,134217728~37748736,188743680~29360128,226492416~12582912,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.074 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~20971520,25165824~41943040,75497472~16777216,125829120~50331648,188743680~33554432,226492416~12582912,251658240~8388608,268435456~12582912,289406976~16777216,310378496~25165824,352321536~12582912,373293056~41943040] 2024-09-14T04:36:39.074 INFO:tasks.workunit.client.0.smithi039.stdout:from 11 to 18 diff [0~20971520,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,125829120~46137344,188743680~29360128,226492416~12582912,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~8388608,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.074 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~20971520,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,125829120~46137344,188743680~29360128,226492416~12582912,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~8388608,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.077 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~20971520,25165824~41943040,75497472~16777216,125829120~50331648,188743680~33554432,226492416~12582912,251658240~8388608,268435456~12582912,289406976~16777216,310378496~33554432,352321536~12582912,373293056~41943040] 2024-09-14T04:36:39.078 INFO:tasks.workunit.client.0.smithi039.stdout:from 11 to 19 diff [0~20971520,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~54525952,176160768~8388608,188743680~29360128,226492416~12582912,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.078 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~20971520,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~54525952,176160768~8388608,188743680~29360128,226492416~12582912,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.081 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~20971520,25165824~41943040,75497472~16777216,100663296~8388608,117440512~104857600,226492416~12582912,251658240~8388608,268435456~12582912,289406976~16777216,310378496~37748736,352321536~12582912,373293056~41943040] 2024-09-14T04:36:39.081 INFO:tasks.workunit.client.0.smithi039.stdout:from 12 to 13 diff [41943040~8388608,125829120~4194304,134217728~8388608,197132288~20971520,230686720~8388608,352321536~4194304,373293056~4194304,398458880~4194304] 2024-09-14T04:36:39.081 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [41943040~8388608,125829120~4194304,134217728~8388608,197132288~20971520,230686720~8388608,352321536~4194304,373293056~4194304,398458880~4194304] 2024-09-14T04:36:39.082 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [41943040~8388608,125829120~16777216,197132288~25165824,230686720~8388608,352321536~4194304,373293056~8388608,398458880~8388608] 2024-09-14T04:36:39.082 INFO:tasks.workunit.client.0.smithi039.stdout:from 12 to 14 diff [16777216~4194304,29360128~4194304,37748736~8388608,58720256~8388608,125829120~4194304,134217728~8388608,150994944~4194304,197132288~20971520,230686720~8388608,268435456~8388608,318767104~8388608,352321536~8388608,373293056~4194304,398458880~12582912] 2024-09-14T04:36:39.082 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [16777216~4194304,29360128~4194304,37748736~8388608,58720256~8388608,125829120~4194304,134217728~8388608,150994944~4194304,197132288~20971520,230686720~8388608,268435456~8388608,318767104~8388608,352321536~8388608,373293056~4194304,398458880~12582912] 2024-09-14T04:36:39.084 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [16777216~4194304,29360128~4194304,37748736~12582912,58720256~8388608,125829120~16777216,150994944~4194304,197132288~25165824,230686720~8388608,268435456~12582912,318767104~12582912,352321536~8388608,373293056~8388608,398458880~16777216] 2024-09-14T04:36:39.084 INFO:tasks.workunit.client.0.smithi039.stdout:from 12 to 15 diff [4194304~8388608,16777216~4194304,29360128~4194304,37748736~8388608,58720256~8388608,83886080~4194304,125829120~4194304,134217728~8388608,150994944~8388608,188743680~29360128,230686720~8388608,251658240~4194304,268435456~8388608,318767104~8388608,352321536~8388608,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.084 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~8388608,16777216~4194304,29360128~4194304,37748736~8388608,58720256~8388608,83886080~4194304,125829120~4194304,134217728~8388608,150994944~8388608,188743680~29360128,230686720~8388608,251658240~4194304,268435456~8388608,318767104~8388608,352321536~8388608,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.085 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,29360128~4194304,37748736~12582912,58720256~8388608,83886080~8388608,125829120~16777216,150994944~8388608,188743680~33554432,230686720~8388608,251658240~8388608,268435456~12582912,318767104~12582912,352321536~8388608,373293056~41943040] 2024-09-14T04:36:39.086 INFO:tasks.workunit.client.0.smithi039.stdout:from 12 to 16 diff [4194304~8388608,16777216~4194304,25165824~4194304,37748736~8388608,58720256~8388608,75497472~16777216,125829120~4194304,134217728~8388608,150994944~8388608,188743680~29360128,230686720~8388608,251658240~4194304,268435456~8388608,289406976~8388608,310378496~20971520,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.086 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~8388608,16777216~4194304,25165824~4194304,37748736~8388608,58720256~8388608,75497472~16777216,125829120~4194304,134217728~8388608,150994944~8388608,188743680~29360128,230686720~8388608,251658240~4194304,268435456~8388608,289406976~8388608,310378496~20971520,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.088 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~8388608,37748736~12582912,58720256~8388608,75497472~16777216,125829120~16777216,150994944~8388608,188743680~33554432,230686720~8388608,251658240~8388608,268435456~12582912,289406976~12582912,310378496~20971520,352321536~12582912,373293056~41943040] 2024-09-14T04:36:39.089 INFO:tasks.workunit.client.0.smithi039.stdout:from 12 to 17 diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,125829120~4194304,134217728~8388608,150994944~16777216,188743680~29360128,230686720~8388608,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.089 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,125829120~4194304,134217728~8388608,150994944~16777216,188743680~29360128,230686720~8388608,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.091 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~41943040,75497472~16777216,125829120~16777216,150994944~20971520,188743680~33554432,230686720~8388608,251658240~8388608,268435456~12582912,289406976~16777216,310378496~25165824,352321536~12582912,373293056~41943040] 2024-09-14T04:36:39.091 INFO:tasks.workunit.client.0.smithi039.stdout:from 12 to 18 diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,125829120~41943040,188743680~29360128,230686720~8388608,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~8388608,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.091 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,125829120~41943040,188743680~29360128,230686720~8388608,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~8388608,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.094 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~41943040,75497472~16777216,125829120~46137344,188743680~33554432,230686720~8388608,251658240~8388608,268435456~12582912,289406976~16777216,310378496~33554432,352321536~12582912,373293056~41943040] 2024-09-14T04:36:39.094 INFO:tasks.workunit.client.0.smithi039.stdout:from 12 to 19 diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~50331648,176160768~8388608,188743680~29360128,230686720~8388608,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.094 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~50331648,176160768~8388608,188743680~29360128,230686720~8388608,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.097 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~41943040,75497472~16777216,100663296~8388608,117440512~54525952,176160768~46137344,230686720~8388608,251658240~8388608,268435456~12582912,289406976~16777216,310378496~37748736,352321536~12582912,373293056~41943040] 2024-09-14T04:36:39.098 INFO:tasks.workunit.client.0.smithi039.stdout:from 13 to 14 diff [16777216~4194304,29360128~4194304,37748736~8388608,58720256~8388608,150994944~4194304,268435456~8388608,318767104~8388608,352321536~8388608,402653184~8388608] 2024-09-14T04:36:39.098 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [16777216~4194304,29360128~4194304,37748736~8388608,58720256~8388608,150994944~4194304,268435456~8388608,318767104~8388608,352321536~8388608,402653184~8388608] 2024-09-14T04:36:39.099 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [16777216~4194304,29360128~4194304,37748736~12582912,58720256~8388608,150994944~4194304,268435456~12582912,318767104~12582912,352321536~8388608,402653184~12582912] 2024-09-14T04:36:39.099 INFO:tasks.workunit.client.0.smithi039.stdout:from 13 to 15 diff [4194304~8388608,16777216~4194304,29360128~4194304,37748736~8388608,58720256~8388608,83886080~4194304,150994944~8388608,188743680~12582912,251658240~4194304,268435456~8388608,318767104~8388608,352321536~8388608,381681664~16777216,402653184~8388608] 2024-09-14T04:36:39.099 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~8388608,16777216~4194304,29360128~4194304,37748736~8388608,58720256~8388608,83886080~4194304,150994944~8388608,188743680~12582912,251658240~4194304,268435456~8388608,318767104~8388608,352321536~8388608,381681664~16777216,402653184~8388608] 2024-09-14T04:36:39.100 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,29360128~4194304,37748736~12582912,58720256~8388608,83886080~8388608,150994944~8388608,188743680~16777216,251658240~8388608,268435456~12582912,318767104~12582912,352321536~8388608,381681664~16777216,402653184~12582912] 2024-09-14T04:36:39.100 INFO:tasks.workunit.client.0.smithi039.stdout:from 13 to 16 diff [4194304~8388608,16777216~4194304,25165824~4194304,37748736~8388608,58720256~8388608,75497472~16777216,150994944~8388608,188743680~12582912,251658240~4194304,268435456~8388608,289406976~8388608,310378496~20971520,352321536~12582912,373293056~4194304,381681664~16777216,402653184~8388608] 2024-09-14T04:36:39.100 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~8388608,16777216~4194304,25165824~4194304,37748736~8388608,58720256~8388608,75497472~16777216,150994944~8388608,188743680~12582912,251658240~4194304,268435456~8388608,289406976~8388608,310378496~20971520,352321536~12582912,373293056~4194304,381681664~16777216,402653184~8388608] 2024-09-14T04:36:39.102 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~8388608,37748736~12582912,58720256~8388608,75497472~16777216,150994944~8388608,188743680~16777216,251658240~8388608,268435456~12582912,289406976~12582912,310378496~20971520,352321536~12582912,373293056~4194304,381681664~16777216,402653184~12582912] 2024-09-14T04:36:39.102 INFO:tasks.workunit.client.0.smithi039.stdout:from 13 to 17 diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,138412032~4194304,150994944~16777216,188743680~12582912,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,352321536~12582912,373293056~4194304,381681664~16777216,402653184~8388608] 2024-09-14T04:36:39.102 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,138412032~4194304,150994944~16777216,188743680~12582912,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,352321536~12582912,373293056~4194304,381681664~16777216,402653184~8388608] 2024-09-14T04:36:39.105 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~41943040,75497472~16777216,138412032~4194304,150994944~20971520,188743680~16777216,251658240~8388608,268435456~12582912,289406976~16777216,310378496~25165824,352321536~12582912,373293056~4194304,381681664~16777216,402653184~12582912] 2024-09-14T04:36:39.105 INFO:tasks.workunit.client.0.smithi039.stdout:from 13 to 18 diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,130023424~37748736,188743680~12582912,230686720~4194304,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~8388608,352321536~12582912,373293056~4194304,381681664~16777216,402653184~8388608] 2024-09-14T04:36:39.105 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,130023424~37748736,188743680~12582912,230686720~4194304,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~8388608,352321536~12582912,373293056~4194304,381681664~16777216,402653184~8388608] 2024-09-14T04:36:39.107 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~41943040,75497472~16777216,130023424~41943040,188743680~16777216,230686720~4194304,251658240~8388608,268435456~12582912,289406976~16777216,310378496~33554432,352321536~12582912,373293056~4194304,381681664~16777216,402653184~12582912] 2024-09-14T04:36:39.108 INFO:tasks.workunit.client.0.smithi039.stdout:from 13 to 19 diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,188743680~12582912,230686720~4194304,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~16777216,402653184~8388608] 2024-09-14T04:36:39.108 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,188743680~12582912,230686720~4194304,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~16777216,402653184~8388608] 2024-09-14T04:36:39.110 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~41943040,75497472~16777216,100663296~8388608,117440512~8388608,130023424~41943040,176160768~29360128,230686720~4194304,251658240~8388608,268435456~12582912,289406976~16777216,310378496~37748736,352321536~12582912,373293056~4194304,381681664~33554432] 2024-09-14T04:36:39.110 INFO:tasks.workunit.client.0.smithi039.stdout:from 14 to 15 diff [4194304~8388608,83886080~4194304,150994944~4194304,188743680~12582912,251658240~4194304,381681664~16777216] 2024-09-14T04:36:39.110 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~8388608,83886080~4194304,150994944~4194304,188743680~12582912,251658240~4194304,381681664~16777216] 2024-09-14T04:36:39.111 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~12582912,83886080~8388608,150994944~8388608,188743680~16777216,251658240~8388608,381681664~16777216] 2024-09-14T04:36:39.112 INFO:tasks.workunit.client.0.smithi039.stdout:from 14 to 16 diff [4194304~8388608,25165824~4194304,75497472~16777216,150994944~4194304,188743680~12582912,251658240~4194304,289406976~8388608,310378496~16777216,352321536~8388608,373293056~4194304,381681664~16777216] 2024-09-14T04:36:39.112 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~8388608,25165824~4194304,75497472~16777216,150994944~4194304,188743680~12582912,251658240~4194304,289406976~8388608,310378496~16777216,352321536~8388608,373293056~4194304,381681664~16777216] 2024-09-14T04:36:39.113 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~12582912,25165824~8388608,75497472~16777216,150994944~8388608,188743680~16777216,251658240~8388608,289406976~12582912,310378496~16777216,352321536~12582912,373293056~4194304,381681664~16777216] 2024-09-14T04:36:39.113 INFO:tasks.workunit.client.0.smithi039.stdout:from 14 to 17 diff [4194304~16777216,25165824~8388608,46137344~4194304,54525952~8388608,75497472~16777216,138412032~4194304,150994944~4194304,159383552~8388608,188743680~12582912,251658240~4194304,289406976~12582912,310378496~20971520,352321536~8388608,373293056~4194304,381681664~16777216] 2024-09-14T04:36:39.113 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~16777216,25165824~8388608,46137344~4194304,54525952~8388608,75497472~16777216,138412032~4194304,150994944~4194304,159383552~8388608,188743680~12582912,251658240~4194304,289406976~12582912,310378496~20971520,352321536~8388608,373293056~4194304,381681664~16777216] 2024-09-14T04:36:39.115 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~12582912,46137344~16777216,75497472~16777216,138412032~4194304,150994944~20971520,188743680~16777216,251658240~8388608,289406976~16777216,310378496~25165824,352321536~12582912,373293056~4194304,381681664~16777216] 2024-09-14T04:36:39.115 INFO:tasks.workunit.client.0.smithi039.stdout:from 14 to 18 diff [4194304~16777216,25165824~8388608,46137344~4194304,54525952~8388608,75497472~16777216,130023424~37748736,188743680~12582912,230686720~4194304,251658240~4194304,268435456~4194304,289406976~12582912,310378496~20971520,335544320~8388608,352321536~8388608,373293056~4194304,381681664~16777216] 2024-09-14T04:36:39.115 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~16777216,25165824~8388608,46137344~4194304,54525952~8388608,75497472~16777216,130023424~37748736,188743680~12582912,230686720~4194304,251658240~4194304,268435456~4194304,289406976~12582912,310378496~20971520,335544320~8388608,352321536~8388608,373293056~4194304,381681664~16777216] 2024-09-14T04:36:39.118 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~12582912,46137344~16777216,75497472~16777216,130023424~41943040,188743680~16777216,230686720~4194304,251658240~8388608,268435456~8388608,289406976~16777216,310378496~33554432,352321536~12582912,373293056~4194304,381681664~16777216] 2024-09-14T04:36:39.118 INFO:tasks.workunit.client.0.smithi039.stdout:from 14 to 19 diff [4194304~12582912,25165824~8388608,46137344~4194304,54525952~8388608,75497472~16777216,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,188743680~12582912,230686720~4194304,251658240~4194304,268435456~4194304,289406976~12582912,310378496~20971520,335544320~12582912,352321536~8388608,373293056~4194304,381681664~16777216] 2024-09-14T04:36:39.118 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~12582912,25165824~8388608,46137344~4194304,54525952~8388608,75497472~16777216,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,188743680~12582912,230686720~4194304,251658240~4194304,268435456~4194304,289406976~12582912,310378496~20971520,335544320~12582912,352321536~8388608,373293056~4194304,381681664~16777216] 2024-09-14T04:36:39.121 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~12582912,46137344~16777216,75497472~16777216,100663296~8388608,117440512~8388608,130023424~41943040,176160768~29360128,230686720~4194304,251658240~8388608,268435456~8388608,289406976~16777216,310378496~37748736,352321536~12582912,373293056~4194304,381681664~20971520] 2024-09-14T04:36:39.121 INFO:tasks.workunit.client.0.smithi039.stdout:from 15 to 16 diff [4194304~4194304,25165824~4194304,75497472~8388608,88080384~4194304,289406976~8388608,310378496~16777216,352321536~8388608,373293056~4194304] 2024-09-14T04:36:39.121 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~4194304,25165824~4194304,75497472~8388608,88080384~4194304,289406976~8388608,310378496~16777216,352321536~8388608,373293056~4194304] 2024-09-14T04:36:39.122 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~8388608,25165824~8388608,75497472~16777216,289406976~12582912,310378496~16777216,352321536~12582912,373293056~4194304] 2024-09-14T04:36:39.122 INFO:tasks.workunit.client.0.smithi039.stdout:from 15 to 17 diff [4194304~4194304,12582912~8388608,25165824~8388608,46137344~4194304,54525952~8388608,75497472~8388608,88080384~4194304,138412032~4194304,159383552~8388608,289406976~12582912,310378496~20971520,352321536~8388608,373293056~4194304,385875968~4194304] 2024-09-14T04:36:39.122 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~4194304,12582912~8388608,25165824~8388608,46137344~4194304,54525952~8388608,75497472~8388608,88080384~4194304,138412032~4194304,159383552~8388608,289406976~12582912,310378496~20971520,352321536~8388608,373293056~4194304,385875968~4194304] 2024-09-14T04:36:39.123 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~12582912,46137344~16777216,75497472~16777216,138412032~4194304,159383552~12582912,289406976~16777216,310378496~25165824,352321536~12582912,373293056~4194304,385875968~4194304] 2024-09-14T04:36:39.124 INFO:tasks.workunit.client.0.smithi039.stdout:from 15 to 18 diff [4194304~4194304,12582912~8388608,25165824~8388608,46137344~4194304,54525952~8388608,75497472~8388608,88080384~4194304,130023424~37748736,230686720~4194304,268435456~4194304,289406976~12582912,310378496~20971520,335544320~8388608,352321536~8388608,373293056~4194304,385875968~4194304] 2024-09-14T04:36:39.124 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~4194304,12582912~8388608,25165824~8388608,46137344~4194304,54525952~8388608,75497472~8388608,88080384~4194304,130023424~37748736,230686720~4194304,268435456~4194304,289406976~12582912,310378496~20971520,335544320~8388608,352321536~8388608,373293056~4194304,385875968~4194304] 2024-09-14T04:36:39.126 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~12582912,46137344~16777216,75497472~16777216,130023424~41943040,230686720~4194304,268435456~8388608,289406976~16777216,310378496~33554432,352321536~12582912,373293056~4194304,385875968~4194304] 2024-09-14T04:36:39.126 INFO:tasks.workunit.client.0.smithi039.stdout:from 15 to 19 diff [4194304~12582912,25165824~8388608,46137344~4194304,54525952~8388608,75497472~8388608,88080384~4194304,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,230686720~4194304,268435456~4194304,289406976~12582912,310378496~20971520,335544320~12582912,352321536~8388608,373293056~4194304,385875968~4194304,394264576~4194304] 2024-09-14T04:36:39.126 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~12582912,25165824~8388608,46137344~4194304,54525952~8388608,75497472~8388608,88080384~4194304,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,230686720~4194304,268435456~4194304,289406976~12582912,310378496~20971520,335544320~12582912,352321536~8388608,373293056~4194304,385875968~4194304,394264576~4194304] 2024-09-14T04:36:39.128 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~12582912,46137344~16777216,75497472~16777216,100663296~8388608,117440512~8388608,130023424~41943040,176160768~12582912,230686720~4194304,268435456~8388608,289406976~16777216,310378496~37748736,352321536~12582912,373293056~4194304,385875968~4194304,394264576~8388608] 2024-09-14T04:36:39.128 INFO:tasks.workunit.client.0.smithi039.stdout:from 16 to 17 diff [12582912~8388608,25165824~8388608,46137344~4194304,54525952~8388608,75497472~4194304,138412032~4194304,159383552~8388608,297795584~8388608,322961408~8388608,385875968~4194304] 2024-09-14T04:36:39.128 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [12582912~8388608,25165824~8388608,46137344~4194304,54525952~8388608,75497472~4194304,138412032~4194304,159383552~8388608,297795584~8388608,322961408~8388608,385875968~4194304] 2024-09-14T04:36:39.129 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8388608,25165824~12582912,46137344~16777216,75497472~4194304,138412032~4194304,159383552~12582912,297795584~8388608,322961408~12582912,385875968~4194304] 2024-09-14T04:36:39.129 INFO:tasks.workunit.client.0.smithi039.stdout:from 16 to 18 diff [12582912~8388608,25165824~8388608,46137344~4194304,54525952~8388608,75497472~4194304,130023424~37748736,230686720~4194304,268435456~4194304,297795584~8388608,318767104~12582912,335544320~8388608,385875968~4194304] 2024-09-14T04:36:39.129 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [12582912~8388608,25165824~8388608,46137344~4194304,54525952~8388608,75497472~4194304,130023424~37748736,230686720~4194304,268435456~4194304,297795584~8388608,318767104~12582912,335544320~8388608,385875968~4194304] 2024-09-14T04:36:39.131 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8388608,25165824~12582912,46137344~16777216,75497472~4194304,130023424~41943040,230686720~4194304,268435456~8388608,297795584~8388608,318767104~25165824,385875968~4194304] 2024-09-14T04:36:39.131 INFO:tasks.workunit.client.0.smithi039.stdout:from 16 to 19 diff [8388608~12582912,25165824~8388608,46137344~4194304,54525952~8388608,75497472~4194304,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,230686720~4194304,268435456~4194304,297795584~8388608,318767104~12582912,335544320~12582912,385875968~4194304,394264576~4194304] 2024-09-14T04:36:39.131 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [8388608~12582912,25165824~8388608,46137344~4194304,54525952~8388608,75497472~4194304,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,230686720~4194304,268435456~4194304,297795584~8388608,318767104~12582912,335544320~12582912,385875968~4194304,394264576~4194304] 2024-09-14T04:36:39.133 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [8388608~12582912,25165824~12582912,46137344~16777216,75497472~4194304,100663296~8388608,117440512~8388608,130023424~41943040,176160768~12582912,230686720~4194304,268435456~8388608,297795584~8388608,318767104~29360128,385875968~4194304,394264576~8388608] 2024-09-14T04:36:39.133 INFO:tasks.workunit.client.0.smithi039.stdout:from 17 to 18 diff [12582912~8388608,130023424~8388608,142606336~16777216,230686720~4194304,268435456~4194304,318767104~8388608,335544320~8388608] 2024-09-14T04:36:39.133 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [12582912~8388608,130023424~8388608,142606336~16777216,230686720~4194304,268435456~4194304,318767104~8388608,335544320~8388608] 2024-09-14T04:36:39.134 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [12582912~8388608,130023424~8388608,142606336~16777216,230686720~4194304,268435456~8388608,318767104~12582912,335544320~8388608] 2024-09-14T04:36:39.134 INFO:tasks.workunit.client.0.smithi039.stdout:from 17 to 19 diff [8388608~12582912,100663296~8388608,117440512~8388608,130023424~29360128,176160768~8388608,230686720~4194304,268435456~4194304,318767104~8388608,335544320~12582912,394264576~4194304] 2024-09-14T04:36:39.134 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [8388608~12582912,100663296~8388608,117440512~8388608,130023424~29360128,176160768~8388608,230686720~4194304,268435456~4194304,318767104~8388608,335544320~12582912,394264576~4194304] 2024-09-14T04:36:39.135 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [8388608~12582912,100663296~8388608,117440512~8388608,130023424~29360128,176160768~12582912,230686720~4194304,268435456~8388608,318767104~12582912,335544320~12582912,394264576~8388608] 2024-09-14T04:36:39.135 INFO:tasks.workunit.client.0.smithi039.stdout:from 18 to 19 diff [8388608~8388608,100663296~8388608,117440512~8388608,130023424~16777216,176160768~8388608,339738624~8388608,394264576~4194304] 2024-09-14T04:36:39.135 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [8388608~8388608,100663296~8388608,117440512~8388608,130023424~16777216,176160768~8388608,339738624~8388608,394264576~4194304] 2024-09-14T04:36:39.136 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [8388608~8388608,100663296~8388608,117440512~8388608,130023424~16777216,176160768~12582912,339738624~8388608,394264576~8388608] 2024-09-14T04:36:39.432 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:39.432 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:39.452 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~239075328,243269632~41943040,289406976~125829120] 2024-09-14T04:36:39.452 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:39.452 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:39.459 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:39.459 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:39.459 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:39.466 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:39.466 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:39.467 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:39.472 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~37748736,289406976~125829120] 2024-09-14T04:36:39.473 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to HEAD diff [0~62914560,67108864~20971520,92274688~4194304,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:39.473 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~4194304,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:39.479 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~96468992,100663296~12582912,117440512~121634816,243269632~37748736,289406976~125829120] 2024-09-14T04:36:39.479 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to HEAD diff [0~62914560,67108864~20971520,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.479 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.485 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~92274688,100663296~12582912,117440512~121634816,243269632~37748736,289406976~79691776,373293056~41943040] 2024-09-14T04:36:39.485 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to HEAD diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.485 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.491 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~79691776,373293056~41943040] 2024-09-14T04:36:39.491 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to HEAD diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.491 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.495 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:39.496 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to HEAD diff [0~62914560,67108864~4194304,75497472~16777216,100663296~12582912,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.496 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~16777216,100663296~12582912,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.500 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:39.500 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to HEAD diff [0~62914560,67108864~4194304,75497472~16777216,100663296~8388608,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~20971520,268435456~8388608,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.500 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~16777216,100663296~8388608,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~20971520,268435456~8388608,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.504 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~8388608,117440512~121634816,243269632~20971520,268435456~12582912,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:39.504 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to HEAD diff [0~71303168,75497472~16777216,100663296~8388608,117440512~67108864,188743680~29360128,222298112~16777216,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~29360128,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.504 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~71303168,75497472~16777216,100663296~8388608,117440512~67108864,188743680~29360128,222298112~16777216,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~29360128,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.508 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~8388608,117440512~121634816,251658240~8388608,268435456~12582912,289406976~16777216,310378496~54525952,373293056~41943040] 2024-09-14T04:36:39.508 INFO:tasks.workunit.client.0.smithi039.stdout:from 11 to HEAD diff [0~20971520,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~54525952,176160768~8388608,188743680~29360128,226492416~12582912,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.508 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~20971520,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~54525952,176160768~8388608,188743680~29360128,226492416~12582912,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.511 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~20971520,25165824~41943040,75497472~16777216,100663296~8388608,117440512~104857600,226492416~12582912,251658240~8388608,268435456~12582912,289406976~16777216,310378496~37748736,352321536~12582912,373293056~41943040] 2024-09-14T04:36:39.511 INFO:tasks.workunit.client.0.smithi039.stdout:from 12 to HEAD diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~50331648,176160768~8388608,188743680~29360128,230686720~8388608,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.512 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~50331648,176160768~8388608,188743680~29360128,230686720~8388608,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:39.515 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~41943040,75497472~16777216,100663296~8388608,117440512~54525952,176160768~46137344,230686720~8388608,251658240~8388608,268435456~12582912,289406976~16777216,310378496~37748736,352321536~12582912,373293056~41943040] 2024-09-14T04:36:39.515 INFO:tasks.workunit.client.0.smithi039.stdout:from 13 to HEAD diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,188743680~12582912,230686720~4194304,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~16777216,402653184~8388608] 2024-09-14T04:36:39.515 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,188743680~12582912,230686720~4194304,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~16777216,402653184~8388608] 2024-09-14T04:36:39.518 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~41943040,75497472~16777216,100663296~8388608,117440512~8388608,130023424~41943040,176160768~29360128,230686720~4194304,251658240~8388608,268435456~12582912,289406976~16777216,310378496~37748736,352321536~12582912,373293056~4194304,381681664~33554432] 2024-09-14T04:36:39.518 INFO:tasks.workunit.client.0.smithi039.stdout:from 14 to HEAD diff [4194304~12582912,25165824~8388608,46137344~4194304,54525952~8388608,75497472~16777216,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,188743680~12582912,230686720~4194304,251658240~4194304,268435456~4194304,289406976~12582912,310378496~20971520,335544320~12582912,352321536~8388608,373293056~4194304,381681664~16777216] 2024-09-14T04:36:39.518 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~12582912,25165824~8388608,46137344~4194304,54525952~8388608,75497472~16777216,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,188743680~12582912,230686720~4194304,251658240~4194304,268435456~4194304,289406976~12582912,310378496~20971520,335544320~12582912,352321536~8388608,373293056~4194304,381681664~16777216] 2024-09-14T04:36:39.521 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~12582912,46137344~16777216,75497472~16777216,100663296~8388608,117440512~8388608,130023424~41943040,176160768~29360128,230686720~4194304,251658240~8388608,268435456~8388608,289406976~16777216,310378496~37748736,352321536~12582912,373293056~4194304,381681664~20971520] 2024-09-14T04:36:39.521 INFO:tasks.workunit.client.0.smithi039.stdout:from 15 to HEAD diff [4194304~12582912,25165824~8388608,46137344~4194304,54525952~8388608,75497472~8388608,88080384~4194304,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,230686720~4194304,268435456~4194304,289406976~12582912,310378496~20971520,335544320~12582912,352321536~8388608,373293056~4194304,385875968~4194304,394264576~4194304] 2024-09-14T04:36:39.521 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~12582912,25165824~8388608,46137344~4194304,54525952~8388608,75497472~8388608,88080384~4194304,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,230686720~4194304,268435456~4194304,289406976~12582912,310378496~20971520,335544320~12582912,352321536~8388608,373293056~4194304,385875968~4194304,394264576~4194304] 2024-09-14T04:36:39.523 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~12582912,46137344~16777216,75497472~16777216,100663296~8388608,117440512~8388608,130023424~41943040,176160768~12582912,230686720~4194304,268435456~8388608,289406976~16777216,310378496~37748736,352321536~12582912,373293056~4194304,385875968~4194304,394264576~8388608] 2024-09-14T04:36:39.523 INFO:tasks.workunit.client.0.smithi039.stdout:from 16 to HEAD diff [8388608~12582912,25165824~8388608,46137344~4194304,54525952~8388608,75497472~4194304,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,230686720~4194304,268435456~4194304,297795584~8388608,318767104~12582912,335544320~12582912,385875968~4194304,394264576~4194304] 2024-09-14T04:36:39.523 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [8388608~12582912,25165824~8388608,46137344~4194304,54525952~8388608,75497472~4194304,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,230686720~4194304,268435456~4194304,297795584~8388608,318767104~12582912,335544320~12582912,385875968~4194304,394264576~4194304] 2024-09-14T04:36:39.524 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [8388608~12582912,25165824~12582912,46137344~16777216,75497472~4194304,100663296~8388608,117440512~8388608,130023424~41943040,176160768~12582912,230686720~4194304,268435456~8388608,297795584~8388608,318767104~29360128,385875968~4194304,394264576~8388608] 2024-09-14T04:36:39.524 INFO:tasks.workunit.client.0.smithi039.stdout:from 17 to HEAD diff [8388608~12582912,100663296~8388608,117440512~8388608,130023424~29360128,176160768~8388608,230686720~4194304,268435456~4194304,318767104~8388608,335544320~12582912,394264576~4194304] 2024-09-14T04:36:39.524 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [8388608~12582912,100663296~8388608,117440512~8388608,130023424~29360128,176160768~8388608,230686720~4194304,268435456~4194304,318767104~8388608,335544320~12582912,394264576~4194304] 2024-09-14T04:36:39.525 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [8388608~12582912,100663296~8388608,117440512~8388608,130023424~29360128,176160768~12582912,230686720~4194304,268435456~8388608,318767104~12582912,335544320~12582912,394264576~8388608] 2024-09-14T04:36:39.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:39 smithi129 ceph-mon[101316]: osdmap e1288: 8 total, 8 up, 8 in 2024-09-14T04:36:39.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:39 smithi039 ceph-mon[106470]: osdmap e1288: 8 total, 8 up, 8 in 2024-09-14T04:36:39.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:39 smithi039 ceph-mon[111235]: osdmap e1288: 8 total, 8 up, 8 in 2024-09-14T04:36:40.465 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:40.465 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:40.481 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~239075328,243269632~41943040,289406976~125829120] 2024-09-14T04:36:40.481 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:40.481 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:40.489 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:40.489 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:40.489 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:40.496 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:40.497 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:40.497 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:40.503 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~37748736,289406976~125829120] 2024-09-14T04:36:40.503 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to HEAD diff [0~62914560,67108864~20971520,92274688~4194304,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:40.503 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~4194304,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:40.509 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~96468992,100663296~12582912,117440512~121634816,243269632~37748736,289406976~125829120] 2024-09-14T04:36:40.509 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to HEAD diff [0~62914560,67108864~20971520,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:40.510 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:40.515 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~92274688,100663296~12582912,117440512~121634816,243269632~37748736,289406976~79691776,373293056~41943040] 2024-09-14T04:36:40.516 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to HEAD diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:40.516 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:40.521 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~79691776,373293056~41943040] 2024-09-14T04:36:40.521 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to HEAD diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:40.521 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:40.527 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:40.527 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to HEAD diff [0~62914560,67108864~4194304,75497472~16777216,100663296~12582912,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:40.528 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~16777216,100663296~12582912,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:40.531 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:40.531 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to HEAD diff [0~62914560,67108864~4194304,75497472~16777216,100663296~8388608,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~20971520,268435456~8388608,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:40.531 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~16777216,100663296~8388608,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~20971520,268435456~8388608,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:40.536 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~8388608,117440512~121634816,243269632~20971520,268435456~12582912,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:40.536 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to HEAD diff [0~71303168,75497472~16777216,100663296~8388608,117440512~67108864,188743680~29360128,222298112~16777216,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~29360128,373293056~4194304,381681664~29360128] 2024-09-14T04:36:40.536 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~71303168,75497472~16777216,100663296~8388608,117440512~67108864,188743680~29360128,222298112~16777216,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~29360128,373293056~4194304,381681664~29360128] 2024-09-14T04:36:40.541 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~8388608,117440512~121634816,251658240~8388608,268435456~12582912,289406976~16777216,310378496~54525952,373293056~41943040] 2024-09-14T04:36:40.541 INFO:tasks.workunit.client.0.smithi039.stdout:from 11 to HEAD diff [0~20971520,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~54525952,176160768~8388608,188743680~29360128,226492416~12582912,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:40.541 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~20971520,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~54525952,176160768~8388608,188743680~29360128,226492416~12582912,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:40.545 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~20971520,25165824~41943040,75497472~16777216,100663296~8388608,117440512~104857600,226492416~12582912,251658240~8388608,268435456~12582912,289406976~16777216,310378496~37748736,352321536~12582912,373293056~41943040] 2024-09-14T04:36:40.545 INFO:tasks.workunit.client.0.smithi039.stdout:from 12 to HEAD diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~50331648,176160768~8388608,188743680~29360128,230686720~8388608,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:40.545 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~50331648,176160768~8388608,188743680~29360128,230686720~8388608,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:40.549 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~41943040,75497472~16777216,100663296~8388608,117440512~54525952,176160768~46137344,230686720~8388608,251658240~8388608,268435456~12582912,289406976~16777216,310378496~37748736,352321536~12582912,373293056~41943040] 2024-09-14T04:36:40.549 INFO:tasks.workunit.client.0.smithi039.stdout:from 13 to HEAD diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,188743680~12582912,230686720~4194304,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~16777216,402653184~8388608] 2024-09-14T04:36:40.549 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,188743680~12582912,230686720~4194304,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~16777216,402653184~8388608] 2024-09-14T04:36:40.552 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~41943040,75497472~16777216,100663296~8388608,117440512~8388608,130023424~41943040,176160768~29360128,230686720~4194304,251658240~8388608,268435456~12582912,289406976~16777216,310378496~37748736,352321536~12582912,373293056~4194304,381681664~33554432] 2024-09-14T04:36:40.552 INFO:tasks.workunit.client.0.smithi039.stdout:from 14 to HEAD diff [4194304~12582912,25165824~8388608,46137344~4194304,54525952~8388608,75497472~16777216,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,188743680~12582912,230686720~4194304,251658240~4194304,268435456~4194304,289406976~12582912,310378496~20971520,335544320~12582912,352321536~8388608,373293056~4194304,381681664~16777216] 2024-09-14T04:36:40.552 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~12582912,25165824~8388608,46137344~4194304,54525952~8388608,75497472~16777216,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,188743680~12582912,230686720~4194304,251658240~4194304,268435456~4194304,289406976~12582912,310378496~20971520,335544320~12582912,352321536~8388608,373293056~4194304,381681664~16777216] 2024-09-14T04:36:40.555 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~12582912,46137344~16777216,75497472~16777216,100663296~8388608,117440512~8388608,130023424~41943040,176160768~29360128,230686720~4194304,251658240~8388608,268435456~8388608,289406976~16777216,310378496~37748736,352321536~12582912,373293056~4194304,381681664~20971520] 2024-09-14T04:36:40.555 INFO:tasks.workunit.client.0.smithi039.stdout:from 15 to HEAD diff [4194304~12582912,25165824~8388608,46137344~4194304,54525952~8388608,75497472~8388608,88080384~4194304,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,230686720~4194304,268435456~4194304,289406976~12582912,310378496~20971520,335544320~12582912,352321536~8388608,373293056~4194304,385875968~4194304,394264576~4194304] 2024-09-14T04:36:40.555 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~12582912,25165824~8388608,46137344~4194304,54525952~8388608,75497472~8388608,88080384~4194304,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,230686720~4194304,268435456~4194304,289406976~12582912,310378496~20971520,335544320~12582912,352321536~8388608,373293056~4194304,385875968~4194304,394264576~4194304] 2024-09-14T04:36:40.557 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~12582912,46137344~16777216,75497472~16777216,100663296~8388608,117440512~8388608,130023424~41943040,176160768~12582912,230686720~4194304,268435456~8388608,289406976~16777216,310378496~37748736,352321536~12582912,373293056~4194304,385875968~4194304,394264576~8388608] 2024-09-14T04:36:40.557 INFO:tasks.workunit.client.0.smithi039.stdout:from 16 to HEAD diff [8388608~12582912,25165824~8388608,46137344~4194304,54525952~8388608,75497472~4194304,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,230686720~4194304,268435456~4194304,297795584~8388608,318767104~12582912,335544320~12582912,385875968~4194304,394264576~4194304] 2024-09-14T04:36:40.557 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [8388608~12582912,25165824~8388608,46137344~4194304,54525952~8388608,75497472~4194304,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,230686720~4194304,268435456~4194304,297795584~8388608,318767104~12582912,335544320~12582912,385875968~4194304,394264576~4194304] 2024-09-14T04:36:40.559 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [8388608~12582912,25165824~12582912,46137344~16777216,75497472~4194304,100663296~8388608,117440512~8388608,130023424~41943040,176160768~12582912,230686720~4194304,268435456~8388608,297795584~8388608,318767104~29360128,385875968~4194304,394264576~8388608] 2024-09-14T04:36:40.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:40 smithi129 ceph-mon[101316]: osdmap e1289: 8 total, 8 up, 8 in 2024-09-14T04:36:40.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:40 smithi129 ceph-mon[101316]: pgmap v2070: 105 pgs: 105 active+clean; 1.2 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 283 KiB/s rd, 50 MiB/s wr, 460 op/s 2024-09-14T04:36:40.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:40 smithi039 ceph-mon[106470]: osdmap e1289: 8 total, 8 up, 8 in 2024-09-14T04:36:40.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:40 smithi039 ceph-mon[106470]: pgmap v2070: 105 pgs: 105 active+clean; 1.2 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 283 KiB/s rd, 50 MiB/s wr, 460 op/s 2024-09-14T04:36:40.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:40 smithi039 ceph-mon[111235]: osdmap e1289: 8 total, 8 up, 8 in 2024-09-14T04:36:40.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:40 smithi039 ceph-mon[111235]: pgmap v2070: 105 pgs: 105 active+clean; 1.2 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 283 KiB/s rd, 50 MiB/s wr, 460 op/s 2024-09-14T04:36:41.480 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:41.481 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:41.502 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~239075328,243269632~41943040,289406976~125829120] 2024-09-14T04:36:41.502 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:41.502 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:41.510 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:41.510 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:41.510 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:41.516 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:41.516 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:41.516 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:41.523 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~37748736,289406976~125829120] 2024-09-14T04:36:41.523 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to HEAD diff [0~62914560,67108864~20971520,92274688~4194304,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:41.523 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~4194304,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:41.529 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~96468992,100663296~12582912,117440512~121634816,243269632~37748736,289406976~125829120] 2024-09-14T04:36:41.529 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to HEAD diff [0~62914560,67108864~20971520,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:41.529 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:41.534 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~92274688,100663296~12582912,117440512~121634816,243269632~37748736,289406976~79691776,373293056~41943040] 2024-09-14T04:36:41.534 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to HEAD diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:41.534 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:41.539 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~79691776,373293056~41943040] 2024-09-14T04:36:41.539 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to HEAD diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:41.539 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:41.543 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:41.544 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to HEAD diff [0~62914560,67108864~4194304,75497472~16777216,100663296~12582912,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:41.544 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~16777216,100663296~12582912,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:41.548 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:41.549 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to HEAD diff [0~62914560,67108864~4194304,75497472~16777216,100663296~8388608,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~20971520,268435456~8388608,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:41.549 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~16777216,100663296~8388608,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~20971520,268435456~8388608,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:41.552 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~8388608,117440512~121634816,243269632~20971520,268435456~12582912,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:41.552 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to HEAD diff [0~71303168,75497472~16777216,100663296~8388608,117440512~67108864,188743680~29360128,222298112~16777216,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~29360128,373293056~4194304,381681664~29360128] 2024-09-14T04:36:41.552 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~71303168,75497472~16777216,100663296~8388608,117440512~67108864,188743680~29360128,222298112~16777216,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~29360128,373293056~4194304,381681664~29360128] 2024-09-14T04:36:41.555 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~8388608,117440512~121634816,251658240~8388608,268435456~12582912,289406976~16777216,310378496~54525952,373293056~41943040] 2024-09-14T04:36:41.555 INFO:tasks.workunit.client.0.smithi039.stdout:from 11 to HEAD diff [0~20971520,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~54525952,176160768~8388608,188743680~29360128,226492416~12582912,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:41.555 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~20971520,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~54525952,176160768~8388608,188743680~29360128,226492416~12582912,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:41.558 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~20971520,25165824~41943040,75497472~16777216,100663296~8388608,117440512~104857600,226492416~12582912,251658240~8388608,268435456~12582912,289406976~16777216,310378496~37748736,352321536~12582912,373293056~41943040] 2024-09-14T04:36:41.558 INFO:tasks.workunit.client.0.smithi039.stdout:from 12 to HEAD diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~50331648,176160768~8388608,188743680~29360128,230686720~8388608,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:41.558 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~50331648,176160768~8388608,188743680~29360128,230686720~8388608,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:41.561 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~41943040,75497472~16777216,100663296~8388608,117440512~54525952,176160768~46137344,230686720~8388608,251658240~8388608,268435456~12582912,289406976~16777216,310378496~37748736,352321536~12582912,373293056~41943040] 2024-09-14T04:36:41.561 INFO:tasks.workunit.client.0.smithi039.stdout:from 13 to HEAD diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,188743680~12582912,230686720~4194304,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~16777216,402653184~8388608] 2024-09-14T04:36:41.561 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,188743680~12582912,230686720~4194304,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~16777216,402653184~8388608] 2024-09-14T04:36:41.563 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~41943040,75497472~16777216,100663296~8388608,117440512~8388608,130023424~41943040,176160768~29360128,230686720~4194304,251658240~8388608,268435456~12582912,289406976~16777216,310378496~37748736,352321536~12582912,373293056~4194304,381681664~33554432] 2024-09-14T04:36:41.563 INFO:tasks.workunit.client.0.smithi039.stdout:from 14 to HEAD diff [4194304~12582912,25165824~8388608,46137344~4194304,54525952~8388608,75497472~16777216,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,188743680~12582912,230686720~4194304,251658240~4194304,268435456~4194304,289406976~12582912,310378496~20971520,335544320~12582912,352321536~8388608,373293056~4194304,381681664~16777216] 2024-09-14T04:36:41.563 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~12582912,25165824~8388608,46137344~4194304,54525952~8388608,75497472~16777216,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,188743680~12582912,230686720~4194304,251658240~4194304,268435456~4194304,289406976~12582912,310378496~20971520,335544320~12582912,352321536~8388608,373293056~4194304,381681664~16777216] 2024-09-14T04:36:41.565 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~12582912,46137344~16777216,75497472~16777216,100663296~8388608,117440512~8388608,130023424~41943040,176160768~29360128,230686720~4194304,251658240~8388608,268435456~8388608,289406976~16777216,310378496~37748736,352321536~12582912,373293056~4194304,381681664~20971520] 2024-09-14T04:36:41.565 INFO:tasks.workunit.client.0.smithi039.stdout:from 15 to HEAD diff [4194304~12582912,25165824~8388608,46137344~4194304,54525952~8388608,75497472~8388608,88080384~4194304,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,230686720~4194304,268435456~4194304,289406976~12582912,310378496~20971520,335544320~12582912,352321536~8388608,373293056~4194304,385875968~4194304,394264576~4194304] 2024-09-14T04:36:41.565 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~12582912,25165824~8388608,46137344~4194304,54525952~8388608,75497472~8388608,88080384~4194304,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,230686720~4194304,268435456~4194304,289406976~12582912,310378496~20971520,335544320~12582912,352321536~8388608,373293056~4194304,385875968~4194304,394264576~4194304] 2024-09-14T04:36:41.566 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~12582912,46137344~16777216,75497472~16777216,100663296~8388608,117440512~8388608,130023424~41943040,176160768~12582912,230686720~4194304,268435456~8388608,289406976~16777216,310378496~37748736,352321536~12582912,373293056~4194304,385875968~4194304,394264576~8388608] 2024-09-14T04:36:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:41 smithi129 ceph-mon[101316]: osdmap e1290: 8 total, 8 up, 8 in 2024-09-14T04:36:41.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:41 smithi039 ceph-mon[111235]: osdmap e1290: 8 total, 8 up, 8 in 2024-09-14T04:36:41.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:41 smithi039 ceph-mon[106470]: osdmap e1290: 8 total, 8 up, 8 in 2024-09-14T04:36:42.499 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:42.499 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:42.515 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~239075328,243269632~41943040,289406976~125829120] 2024-09-14T04:36:42.515 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:42.515 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:42.521 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:42.522 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:42.522 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:42.527 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:42.528 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:42.528 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:42.533 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~37748736,289406976~125829120] 2024-09-14T04:36:42.533 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to HEAD diff [0~62914560,67108864~20971520,92274688~4194304,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:42.533 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~4194304,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:42.539 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~96468992,100663296~12582912,117440512~121634816,243269632~37748736,289406976~125829120] 2024-09-14T04:36:42.539 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to HEAD diff [0~62914560,67108864~20971520,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:42.539 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:42.544 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~92274688,100663296~12582912,117440512~121634816,243269632~37748736,289406976~79691776,373293056~41943040] 2024-09-14T04:36:42.544 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to HEAD diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:42.544 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:42.548 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~79691776,373293056~41943040] 2024-09-14T04:36:42.548 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to HEAD diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:42.548 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:42.551 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:42.552 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to HEAD diff [0~62914560,67108864~4194304,75497472~16777216,100663296~12582912,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:42.552 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~16777216,100663296~12582912,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:42.555 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:42.555 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to HEAD diff [0~62914560,67108864~4194304,75497472~16777216,100663296~8388608,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~20971520,268435456~8388608,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:42.555 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~16777216,100663296~8388608,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~20971520,268435456~8388608,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:42.558 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~8388608,117440512~121634816,243269632~20971520,268435456~12582912,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:42.559 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to HEAD diff [0~71303168,75497472~16777216,100663296~8388608,117440512~67108864,188743680~29360128,222298112~16777216,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~29360128,373293056~4194304,381681664~29360128] 2024-09-14T04:36:42.559 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~71303168,75497472~16777216,100663296~8388608,117440512~67108864,188743680~29360128,222298112~16777216,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~29360128,373293056~4194304,381681664~29360128] 2024-09-14T04:36:42.562 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~8388608,117440512~121634816,251658240~8388608,268435456~12582912,289406976~16777216,310378496~54525952,373293056~41943040] 2024-09-14T04:36:42.562 INFO:tasks.workunit.client.0.smithi039.stdout:from 11 to HEAD diff [0~20971520,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~54525952,176160768~8388608,188743680~29360128,226492416~12582912,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:42.562 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~20971520,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~54525952,176160768~8388608,188743680~29360128,226492416~12582912,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:42.565 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~20971520,25165824~41943040,75497472~16777216,100663296~8388608,117440512~104857600,226492416~12582912,251658240~8388608,268435456~12582912,289406976~16777216,310378496~37748736,352321536~12582912,373293056~41943040] 2024-09-14T04:36:42.565 INFO:tasks.workunit.client.0.smithi039.stdout:from 12 to HEAD diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~50331648,176160768~8388608,188743680~29360128,230686720~8388608,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:42.565 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~50331648,176160768~8388608,188743680~29360128,230686720~8388608,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:42.567 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~41943040,75497472~16777216,100663296~8388608,117440512~54525952,176160768~46137344,230686720~8388608,251658240~8388608,268435456~12582912,289406976~16777216,310378496~37748736,352321536~12582912,373293056~41943040] 2024-09-14T04:36:42.567 INFO:tasks.workunit.client.0.smithi039.stdout:from 13 to HEAD diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,188743680~12582912,230686720~4194304,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~16777216,402653184~8388608] 2024-09-14T04:36:42.567 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,188743680~12582912,230686720~4194304,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~16777216,402653184~8388608] 2024-09-14T04:36:42.569 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~41943040,75497472~16777216,100663296~8388608,117440512~8388608,130023424~41943040,176160768~29360128,230686720~4194304,251658240~8388608,268435456~12582912,289406976~16777216,310378496~37748736,352321536~12582912,373293056~4194304,381681664~33554432] 2024-09-14T04:36:42.569 INFO:tasks.workunit.client.0.smithi039.stdout:from 14 to HEAD diff [4194304~12582912,25165824~8388608,46137344~4194304,54525952~8388608,75497472~16777216,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,188743680~12582912,230686720~4194304,251658240~4194304,268435456~4194304,289406976~12582912,310378496~20971520,335544320~12582912,352321536~8388608,373293056~4194304,381681664~16777216] 2024-09-14T04:36:42.569 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~12582912,25165824~8388608,46137344~4194304,54525952~8388608,75497472~16777216,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,188743680~12582912,230686720~4194304,251658240~4194304,268435456~4194304,289406976~12582912,310378496~20971520,335544320~12582912,352321536~8388608,373293056~4194304,381681664~16777216] 2024-09-14T04:36:42.570 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~12582912,46137344~16777216,75497472~16777216,100663296~8388608,117440512~8388608,130023424~41943040,176160768~29360128,230686720~4194304,251658240~8388608,268435456~8388608,289406976~16777216,310378496~37748736,352321536~12582912,373293056~4194304,381681664~20971520] 2024-09-14T04:36:42.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:42 smithi129 ceph-mon[101316]: pgmap v2072: 105 pgs: 105 active+clean; 1.2 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 204 KiB/s rd, 31 MiB/s wr, 379 op/s 2024-09-14T04:36:42.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:42 smithi129 ceph-mon[101316]: osdmap e1291: 8 total, 8 up, 8 in 2024-09-14T04:36:42.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:42 smithi039 ceph-mon[106470]: pgmap v2072: 105 pgs: 105 active+clean; 1.2 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 204 KiB/s rd, 31 MiB/s wr, 379 op/s 2024-09-14T04:36:42.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:42 smithi039 ceph-mon[106470]: osdmap e1291: 8 total, 8 up, 8 in 2024-09-14T04:36:42.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:42 smithi039 ceph-mon[111235]: pgmap v2072: 105 pgs: 105 active+clean; 1.2 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 204 KiB/s rd, 31 MiB/s wr, 379 op/s 2024-09-14T04:36:42.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:42 smithi039 ceph-mon[111235]: osdmap e1291: 8 total, 8 up, 8 in 2024-09-14T04:36:43.512 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:43.512 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:43.531 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~239075328,243269632~41943040,289406976~125829120] 2024-09-14T04:36:43.531 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:43.531 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:43.538 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:43.538 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:43.538 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:43.544 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:43.544 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:43.544 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:43.549 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~37748736,289406976~125829120] 2024-09-14T04:36:43.549 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to HEAD diff [0~62914560,67108864~20971520,92274688~4194304,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:43.550 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~4194304,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:43.555 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~96468992,100663296~12582912,117440512~121634816,243269632~37748736,289406976~125829120] 2024-09-14T04:36:43.555 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to HEAD diff [0~62914560,67108864~20971520,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:43.556 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:43.560 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~92274688,100663296~12582912,117440512~121634816,243269632~37748736,289406976~79691776,373293056~41943040] 2024-09-14T04:36:43.560 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to HEAD diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:43.560 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:43.564 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~79691776,373293056~41943040] 2024-09-14T04:36:43.564 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to HEAD diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:43.564 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:43.567 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:43.567 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to HEAD diff [0~62914560,67108864~4194304,75497472~16777216,100663296~12582912,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:43.567 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~16777216,100663296~12582912,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:43.571 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:43.571 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to HEAD diff [0~62914560,67108864~4194304,75497472~16777216,100663296~8388608,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~20971520,268435456~8388608,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:43.571 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~16777216,100663296~8388608,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~20971520,268435456~8388608,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:43.573 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~8388608,117440512~121634816,243269632~20971520,268435456~12582912,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:43.573 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to HEAD diff [0~71303168,75497472~16777216,100663296~8388608,117440512~67108864,188743680~29360128,222298112~16777216,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~29360128,373293056~4194304,381681664~29360128] 2024-09-14T04:36:43.574 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~71303168,75497472~16777216,100663296~8388608,117440512~67108864,188743680~29360128,222298112~16777216,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~29360128,373293056~4194304,381681664~29360128] 2024-09-14T04:36:43.576 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~8388608,117440512~121634816,251658240~8388608,268435456~12582912,289406976~16777216,310378496~54525952,373293056~41943040] 2024-09-14T04:36:43.576 INFO:tasks.workunit.client.0.smithi039.stdout:from 11 to HEAD diff [0~20971520,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~54525952,176160768~8388608,188743680~29360128,226492416~12582912,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:43.577 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~20971520,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~54525952,176160768~8388608,188743680~29360128,226492416~12582912,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:43.579 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~20971520,25165824~41943040,75497472~16777216,100663296~8388608,117440512~104857600,226492416~12582912,251658240~8388608,268435456~12582912,289406976~16777216,310378496~37748736,352321536~12582912,373293056~41943040] 2024-09-14T04:36:43.579 INFO:tasks.workunit.client.0.smithi039.stdout:from 12 to HEAD diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~50331648,176160768~8388608,188743680~29360128,230686720~8388608,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:43.579 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~50331648,176160768~8388608,188743680~29360128,230686720~8388608,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:43.581 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~41943040,75497472~16777216,100663296~8388608,117440512~54525952,176160768~46137344,230686720~8388608,251658240~8388608,268435456~12582912,289406976~16777216,310378496~37748736,352321536~12582912,373293056~41943040] 2024-09-14T04:36:43.581 INFO:tasks.workunit.client.0.smithi039.stdout:from 13 to HEAD diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,188743680~12582912,230686720~4194304,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~16777216,402653184~8388608] 2024-09-14T04:36:43.581 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~8388608,130023424~37748736,176160768~8388608,188743680~12582912,230686720~4194304,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~16777216,402653184~8388608] 2024-09-14T04:36:43.582 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~41943040,75497472~16777216,100663296~8388608,117440512~8388608,130023424~41943040,176160768~29360128,230686720~4194304,251658240~8388608,268435456~12582912,289406976~16777216,310378496~37748736,352321536~12582912,373293056~4194304,381681664~33554432] 2024-09-14T04:36:43.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:43 smithi129 ceph-mon[101316]: osdmap e1292: 8 total, 8 up, 8 in 2024-09-14T04:36:43.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:43 smithi039 ceph-mon[106470]: osdmap e1292: 8 total, 8 up, 8 in 2024-09-14T04:36:43.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:43 smithi039 ceph-mon[111235]: osdmap e1292: 8 total, 8 up, 8 in 2024-09-14T04:36:44.517 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:44.517 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:44.548 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~239075328,243269632~41943040,289406976~125829120] 2024-09-14T04:36:44.548 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:44.548 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:44.553 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:44.553 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:44.553 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:44.559 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:44.559 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:44.559 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:44.565 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~37748736,289406976~125829120] 2024-09-14T04:36:44.565 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to HEAD diff [0~62914560,67108864~20971520,92274688~4194304,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:44.565 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~4194304,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:44.570 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~96468992,100663296~12582912,117440512~121634816,243269632~37748736,289406976~125829120] 2024-09-14T04:36:44.570 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to HEAD diff [0~62914560,67108864~20971520,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:44.570 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:44.574 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~92274688,100663296~12582912,117440512~121634816,243269632~37748736,289406976~79691776,373293056~41943040] 2024-09-14T04:36:44.574 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to HEAD diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:44.574 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:44.579 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~79691776,373293056~41943040] 2024-09-14T04:36:44.579 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to HEAD diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:44.579 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:44.582 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:44.583 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to HEAD diff [0~62914560,67108864~4194304,75497472~16777216,100663296~12582912,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:44.583 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~16777216,100663296~12582912,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:44.586 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:44.586 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to HEAD diff [0~62914560,67108864~4194304,75497472~16777216,100663296~8388608,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~20971520,268435456~8388608,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:44.586 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~16777216,100663296~8388608,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~20971520,268435456~8388608,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:44.588 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~8388608,117440512~121634816,243269632~20971520,268435456~12582912,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:44.589 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to HEAD diff [0~71303168,75497472~16777216,100663296~8388608,117440512~67108864,188743680~29360128,222298112~16777216,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~29360128,373293056~4194304,381681664~29360128] 2024-09-14T04:36:44.589 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~71303168,75497472~16777216,100663296~8388608,117440512~67108864,188743680~29360128,222298112~16777216,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~29360128,373293056~4194304,381681664~29360128] 2024-09-14T04:36:44.591 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~8388608,117440512~121634816,251658240~8388608,268435456~12582912,289406976~16777216,310378496~54525952,373293056~41943040] 2024-09-14T04:36:44.591 INFO:tasks.workunit.client.0.smithi039.stdout:from 11 to HEAD diff [0~20971520,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~54525952,176160768~8388608,188743680~29360128,226492416~12582912,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:44.591 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~20971520,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~54525952,176160768~8388608,188743680~29360128,226492416~12582912,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:44.593 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~20971520,25165824~41943040,75497472~16777216,100663296~8388608,117440512~104857600,226492416~12582912,251658240~8388608,268435456~12582912,289406976~16777216,310378496~37748736,352321536~12582912,373293056~41943040] 2024-09-14T04:36:44.593 INFO:tasks.workunit.client.0.smithi039.stdout:from 12 to HEAD diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~50331648,176160768~8388608,188743680~29360128,230686720~8388608,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:44.593 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [4194304~16777216,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~50331648,176160768~8388608,188743680~29360128,230686720~8388608,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:44.594 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [4194304~16777216,25165824~41943040,75497472~16777216,100663296~8388608,117440512~54525952,176160768~46137344,230686720~8388608,251658240~8388608,268435456~12582912,289406976~16777216,310378496~37748736,352321536~12582912,373293056~41943040] 2024-09-14T04:36:44.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:44 smithi129 ceph-mon[101316]: pgmap v2075: 105 pgs: 105 active+clean; 1.2 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 686 KiB/s rd, 3.6 MiB/s wr, 1.28k op/s 2024-09-14T04:36:44.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:44 smithi129 ceph-mon[101316]: osdmap e1293: 8 total, 8 up, 8 in 2024-09-14T04:36:44.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:44 smithi039 ceph-mon[111235]: pgmap v2075: 105 pgs: 105 active+clean; 1.2 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 686 KiB/s rd, 3.6 MiB/s wr, 1.28k op/s 2024-09-14T04:36:44.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:44 smithi039 ceph-mon[111235]: osdmap e1293: 8 total, 8 up, 8 in 2024-09-14T04:36:44.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:44 smithi039 ceph-mon[106470]: pgmap v2075: 105 pgs: 105 active+clean; 1.2 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 686 KiB/s rd, 3.6 MiB/s wr, 1.28k op/s 2024-09-14T04:36:44.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:44 smithi039 ceph-mon[106470]: osdmap e1293: 8 total, 8 up, 8 in 2024-09-14T04:36:45.553 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:45.553 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:45.576 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~239075328,243269632~41943040,289406976~125829120] 2024-09-14T04:36:45.577 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:45.577 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:45.581 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:45.581 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:45.581 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:45.586 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:45.586 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:45.586 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:45.590 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~37748736,289406976~125829120] 2024-09-14T04:36:45.590 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to HEAD diff [0~62914560,67108864~20971520,92274688~4194304,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:45.590 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~4194304,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:45.594 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~96468992,100663296~12582912,117440512~121634816,243269632~37748736,289406976~125829120] 2024-09-14T04:36:45.594 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to HEAD diff [0~62914560,67108864~20971520,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:45.594 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:45.598 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~92274688,100663296~12582912,117440512~121634816,243269632~37748736,289406976~79691776,373293056~41943040] 2024-09-14T04:36:45.598 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to HEAD diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:45.598 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:45.600 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~79691776,373293056~41943040] 2024-09-14T04:36:45.601 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to HEAD diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:45.601 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:45.603 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:45.603 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to HEAD diff [0~62914560,67108864~4194304,75497472~16777216,100663296~12582912,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:45.603 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~16777216,100663296~12582912,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:45.605 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:45.605 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to HEAD diff [0~62914560,67108864~4194304,75497472~16777216,100663296~8388608,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~20971520,268435456~8388608,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:45.606 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~16777216,100663296~8388608,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~20971520,268435456~8388608,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:45.607 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~8388608,117440512~121634816,243269632~20971520,268435456~12582912,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:45.607 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to HEAD diff [0~71303168,75497472~16777216,100663296~8388608,117440512~67108864,188743680~29360128,222298112~16777216,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~29360128,373293056~4194304,381681664~29360128] 2024-09-14T04:36:45.607 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~71303168,75497472~16777216,100663296~8388608,117440512~67108864,188743680~29360128,222298112~16777216,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~29360128,373293056~4194304,381681664~29360128] 2024-09-14T04:36:45.609 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~8388608,117440512~121634816,251658240~8388608,268435456~12582912,289406976~16777216,310378496~54525952,373293056~41943040] 2024-09-14T04:36:45.609 INFO:tasks.workunit.client.0.smithi039.stdout:from 11 to HEAD diff [0~20971520,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~54525952,176160768~8388608,188743680~29360128,226492416~12582912,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:45.609 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~20971520,25165824~8388608,37748736~12582912,54525952~12582912,75497472~16777216,100663296~8388608,117440512~54525952,176160768~8388608,188743680~29360128,226492416~12582912,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~12582912,352321536~12582912,373293056~4194304,381681664~29360128] 2024-09-14T04:36:45.610 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~20971520,25165824~41943040,75497472~16777216,100663296~8388608,117440512~104857600,226492416~12582912,251658240~8388608,268435456~12582912,289406976~16777216,310378496~37748736,352321536~12582912,373293056~41943040] 2024-09-14T04:36:45.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:45 smithi039 ceph-mon[106470]: osdmap e1294: 8 total, 8 up, 8 in 2024-09-14T04:36:45.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:45 smithi039 ceph-mon[111235]: osdmap e1294: 8 total, 8 up, 8 in 2024-09-14T04:36:45.841 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:45.841 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:45.864 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~239075328,243269632~41943040,289406976~125829120] 2024-09-14T04:36:45.864 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:45.864 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:45.872 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:45.872 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:45.872 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:45.877 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:45.877 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:45.877 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:45.881 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~37748736,289406976~125829120] 2024-09-14T04:36:45.881 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to HEAD diff [0~62914560,67108864~20971520,92274688~4194304,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:45.881 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~4194304,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:45.885 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~96468992,100663296~12582912,117440512~121634816,243269632~37748736,289406976~125829120] 2024-09-14T04:36:45.885 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to HEAD diff [0~62914560,67108864~20971520,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:45.885 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:45.888 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~92274688,100663296~12582912,117440512~121634816,243269632~37748736,289406976~79691776,373293056~41943040] 2024-09-14T04:36:45.888 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to HEAD diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:45.888 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:45.891 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~79691776,373293056~41943040] 2024-09-14T04:36:45.891 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to HEAD diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:45.891 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:45.893 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:45.894 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to HEAD diff [0~62914560,67108864~4194304,75497472~16777216,100663296~12582912,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:45.894 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~16777216,100663296~12582912,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:45.895 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:45.895 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to HEAD diff [0~62914560,67108864~4194304,75497472~16777216,100663296~8388608,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~20971520,268435456~8388608,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:45.896 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~16777216,100663296~8388608,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~20971520,268435456~8388608,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:45.897 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~8388608,117440512~121634816,243269632~20971520,268435456~12582912,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:45.898 INFO:tasks.workunit.client.0.smithi039.stdout:from 10 to HEAD diff [0~71303168,75497472~16777216,100663296~8388608,117440512~67108864,188743680~29360128,222298112~16777216,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~29360128,373293056~4194304,381681664~29360128] 2024-09-14T04:36:45.898 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~71303168,75497472~16777216,100663296~8388608,117440512~67108864,188743680~29360128,222298112~16777216,251658240~4194304,268435456~8388608,289406976~12582912,310378496~20971520,335544320~29360128,373293056~4194304,381681664~29360128] 2024-09-14T04:36:45.898 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~8388608,117440512~121634816,251658240~8388608,268435456~12582912,289406976~16777216,310378496~54525952,373293056~41943040] 2024-09-14T04:36:46.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:45 smithi129 ceph-mon[101316]: osdmap e1294: 8 total, 8 up, 8 in 2024-09-14T04:36:46.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:46 smithi039 ceph-mon[106470]: pgmap v2078: 105 pgs: 105 active+clean; 1.1 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 673 KiB/s rd, 3.6 MiB/s wr, 1.25k op/s 2024-09-14T04:36:46.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:46 smithi039 ceph-mon[106470]: osdmap e1295: 8 total, 8 up, 8 in 2024-09-14T04:36:46.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:46 smithi039 ceph-mon[106470]: osdmap e1296: 8 total, 8 up, 8 in 2024-09-14T04:36:46.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:46 smithi039 ceph-mon[111235]: pgmap v2078: 105 pgs: 105 active+clean; 1.1 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 673 KiB/s rd, 3.6 MiB/s wr, 1.25k op/s 2024-09-14T04:36:46.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:46 smithi039 ceph-mon[111235]: osdmap e1295: 8 total, 8 up, 8 in 2024-09-14T04:36:46.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:46 smithi039 ceph-mon[111235]: osdmap e1296: 8 total, 8 up, 8 in 2024-09-14T04:36:46.862 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:46.862 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:46.918 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~239075328,243269632~41943040,289406976~125829120] 2024-09-14T04:36:46.918 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:46.919 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:46.924 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:46.924 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:46.924 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:46.928 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:46.928 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:46.928 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:46.932 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~37748736,289406976~125829120] 2024-09-14T04:36:46.932 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to HEAD diff [0~62914560,67108864~20971520,92274688~4194304,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:46.932 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~4194304,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:46.936 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~96468992,100663296~12582912,117440512~121634816,243269632~37748736,289406976~125829120] 2024-09-14T04:36:46.936 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to HEAD diff [0~62914560,67108864~20971520,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:46.936 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:46.939 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~92274688,100663296~12582912,117440512~121634816,243269632~37748736,289406976~79691776,373293056~41943040] 2024-09-14T04:36:46.939 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to HEAD diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:46.939 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:46.942 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~79691776,373293056~41943040] 2024-09-14T04:36:46.942 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to HEAD diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:46.942 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:46.945 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:46.945 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to HEAD diff [0~62914560,67108864~4194304,75497472~16777216,100663296~12582912,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:46.945 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~16777216,100663296~12582912,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:46.948 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:46.948 INFO:tasks.workunit.client.0.smithi039.stdout:from 9 to HEAD diff [0~62914560,67108864~4194304,75497472~16777216,100663296~8388608,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~20971520,268435456~8388608,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:46.948 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~16777216,100663296~8388608,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~20971520,268435456~8388608,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:46.949 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~8388608,117440512~121634816,243269632~20971520,268435456~12582912,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:47.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:46 smithi129 ceph-mon[101316]: pgmap v2078: 105 pgs: 105 active+clean; 1.1 GiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 673 KiB/s rd, 3.6 MiB/s wr, 1.25k op/s 2024-09-14T04:36:47.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:46 smithi129 ceph-mon[101316]: osdmap e1295: 8 total, 8 up, 8 in 2024-09-14T04:36:47.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:46 smithi129 ceph-mon[101316]: osdmap e1296: 8 total, 8 up, 8 in 2024-09-14T04:36:47.865 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:47.866 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:47.886 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~239075328,243269632~41943040,289406976~125829120] 2024-09-14T04:36:47.886 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:47.886 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:47.893 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:47.893 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:47.893 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:47.896 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:47.896 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:47.897 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:47.900 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~37748736,289406976~125829120] 2024-09-14T04:36:47.900 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to HEAD diff [0~62914560,67108864~20971520,92274688~4194304,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:47.900 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~4194304,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:47.904 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~96468992,100663296~12582912,117440512~121634816,243269632~37748736,289406976~125829120] 2024-09-14T04:36:47.904 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to HEAD diff [0~62914560,67108864~20971520,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:47.904 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:47.907 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~92274688,100663296~12582912,117440512~121634816,243269632~37748736,289406976~79691776,373293056~41943040] 2024-09-14T04:36:47.907 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to HEAD diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:47.907 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:47.909 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~79691776,373293056~41943040] 2024-09-14T04:36:47.909 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to HEAD diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:47.909 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:47.911 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:47.911 INFO:tasks.workunit.client.0.smithi039.stdout:from 8 to HEAD diff [0~62914560,67108864~4194304,75497472~16777216,100663296~12582912,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:47.911 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~16777216,100663296~12582912,117440512~37748736,159383552~25165824,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:47.912 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:48.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:47 smithi129 ceph-mon[101316]: osdmap e1297: 8 total, 8 up, 8 in 2024-09-14T04:36:48.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:47 smithi039 ceph-mon[106470]: osdmap e1297: 8 total, 8 up, 8 in 2024-09-14T04:36:48.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:47 smithi039 ceph-mon[111235]: osdmap e1297: 8 total, 8 up, 8 in 2024-09-14T04:36:48.876 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:48.877 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:48.891 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~239075328,243269632~41943040,289406976~125829120] 2024-09-14T04:36:48.891 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:48.891 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:48.896 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:48.896 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:48.896 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:48.899 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:48.899 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:48.899 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:48.902 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~37748736,289406976~125829120] 2024-09-14T04:36:48.902 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to HEAD diff [0~62914560,67108864~20971520,92274688~4194304,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:48.902 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~4194304,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:48.904 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~96468992,100663296~12582912,117440512~121634816,243269632~37748736,289406976~125829120] 2024-09-14T04:36:48.905 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to HEAD diff [0~62914560,67108864~20971520,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:48.905 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:48.907 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~92274688,100663296~12582912,117440512~121634816,243269632~37748736,289406976~79691776,373293056~41943040] 2024-09-14T04:36:48.907 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to HEAD diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:48.907 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:48.909 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~79691776,373293056~41943040] 2024-09-14T04:36:48.909 INFO:tasks.workunit.client.0.smithi039.stdout:from 7 to HEAD diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:48.909 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,310378496~20971520,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:48.910 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~16777216,310378496~58720256,373293056~41943040] 2024-09-14T04:36:49.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:36:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:36:48] "GET /metrics HTTP/1.1" 200 36692 "" "Prometheus/2.51.0" 2024-09-14T04:36:49.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:48 smithi039 ceph-mon[106470]: pgmap v2082: 105 pgs: 105 active+clean; 1.1 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 176 KiB/s rd, 5.0 KiB/s wr, 274 op/s 2024-09-14T04:36:49.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:48 smithi039 ceph-mon[106470]: osdmap e1298: 8 total, 8 up, 8 in 2024-09-14T04:36:49.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:48 smithi039 ceph-mon[111235]: pgmap v2082: 105 pgs: 105 active+clean; 1.1 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 176 KiB/s rd, 5.0 KiB/s wr, 274 op/s 2024-09-14T04:36:49.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:48 smithi039 ceph-mon[111235]: osdmap e1298: 8 total, 8 up, 8 in 2024-09-14T04:36:49.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:48 smithi129 ceph-mon[101316]: pgmap v2082: 105 pgs: 105 active+clean; 1.1 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 176 KiB/s rd, 5.0 KiB/s wr, 274 op/s 2024-09-14T04:36:49.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:48 smithi129 ceph-mon[101316]: osdmap e1298: 8 total, 8 up, 8 in 2024-09-14T04:36:49.907 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:50.033 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:50.036 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~239075328,243269632~41943040,289406976~125829120] 2024-09-14T04:36:50.036 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:50.036 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:50.036 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:50.036 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:50.036 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:50.036 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:50.036 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:50.036 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:50.037 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~37748736,289406976~125829120] 2024-09-14T04:36:50.037 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to HEAD diff [0~62914560,67108864~20971520,92274688~4194304,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:50.037 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~4194304,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:50.037 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~96468992,100663296~12582912,117440512~121634816,243269632~37748736,289406976~125829120] 2024-09-14T04:36:50.037 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to HEAD diff [0~62914560,67108864~20971520,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:50.037 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:50.037 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~92274688,100663296~12582912,117440512~121634816,243269632~37748736,289406976~79691776,373293056~41943040] 2024-09-14T04:36:50.037 INFO:tasks.workunit.client.0.smithi039.stdout:from 6 to HEAD diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:50.037 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~4194304,75497472~12582912,100663296~12582912,117440512~67108864,188743680~29360128,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:50.037 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~71303168,75497472~16777216,100663296~12582912,117440512~121634816,243269632~37748736,289406976~79691776,373293056~41943040] 2024-09-14T04:36:50.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:49 smithi129 ceph-mon[101316]: osdmap e1299: 8 total, 8 up, 8 in 2024-09-14T04:36:50.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:49 smithi039 ceph-mon[106470]: osdmap e1299: 8 total, 8 up, 8 in 2024-09-14T04:36:50.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:49 smithi039 ceph-mon[111235]: osdmap e1299: 8 total, 8 up, 8 in 2024-09-14T04:36:50.858 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:50.858 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:50.875 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~239075328,243269632~41943040,289406976~125829120] 2024-09-14T04:36:50.876 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:50.876 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:50.881 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:50.881 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:50.881 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:50.884 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:50.885 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:50.885 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:50.886 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~37748736,289406976~125829120] 2024-09-14T04:36:50.886 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to HEAD diff [0~62914560,67108864~20971520,92274688~4194304,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:50.886 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~4194304,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:50.888 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~96468992,100663296~12582912,117440512~121634816,243269632~37748736,289406976~125829120] 2024-09-14T04:36:50.888 INFO:tasks.workunit.client.0.smithi039.stdout:from 5 to HEAD diff [0~62914560,67108864~20971520,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:50.888 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~33554432,373293056~4194304,381681664~29360128] 2024-09-14T04:36:50.890 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~92274688,100663296~12582912,117440512~121634816,243269632~37748736,289406976~79691776,373293056~41943040] 2024-09-14T04:36:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:50 smithi129 ceph-mon[101316]: pgmap v2085: 105 pgs: 105 active+clean; 1.1 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 243 KiB/s rd, 6.4 KiB/s wr, 408 op/s 2024-09-14T04:36:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:50 smithi129 ceph-mon[101316]: osdmap e1300: 8 total, 8 up, 8 in 2024-09-14T04:36:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:50 smithi129 ceph-mon[101316]: osdmap e1301: 8 total, 8 up, 8 in 2024-09-14T04:36:51.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:50 smithi039 ceph-mon[106470]: pgmap v2085: 105 pgs: 105 active+clean; 1.1 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 243 KiB/s rd, 6.4 KiB/s wr, 408 op/s 2024-09-14T04:36:51.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:50 smithi039 ceph-mon[106470]: osdmap e1300: 8 total, 8 up, 8 in 2024-09-14T04:36:51.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:50 smithi039 ceph-mon[106470]: osdmap e1301: 8 total, 8 up, 8 in 2024-09-14T04:36:51.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:50 smithi039 ceph-mon[111235]: pgmap v2085: 105 pgs: 105 active+clean; 1.1 GiB data, 3.3 GiB used, 712 GiB / 715 GiB avail; 243 KiB/s rd, 6.4 KiB/s wr, 408 op/s 2024-09-14T04:36:51.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:50 smithi039 ceph-mon[111235]: osdmap e1300: 8 total, 8 up, 8 in 2024-09-14T04:36:51.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:50 smithi039 ceph-mon[111235]: osdmap e1301: 8 total, 8 up, 8 in 2024-09-14T04:36:51.922 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:51.923 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:51.940 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~239075328,243269632~41943040,289406976~125829120] 2024-09-14T04:36:51.940 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:51.940 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:51.943 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:51.943 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:51.943 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:51.945 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:51.945 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:51.945 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:51.947 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~37748736,289406976~125829120] 2024-09-14T04:36:51.947 INFO:tasks.workunit.client.0.smithi039.stdout:from 4 to HEAD diff [0~62914560,67108864~20971520,92274688~4194304,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:51.947 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~4194304,100663296~12582912,117440512~100663296,222298112~16777216,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:51.948 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~96468992,100663296~12582912,117440512~121634816,243269632~37748736,289406976~125829120] 2024-09-14T04:36:52.955 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:52.955 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:52.968 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~239075328,243269632~41943040,289406976~125829120] 2024-09-14T04:36:52.968 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:52.968 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:52.970 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:52.970 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:52.970 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:52.972 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:52.972 INFO:tasks.workunit.client.0.smithi039.stdout:from 3 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:52.972 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:52.973 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~37748736,289406976~125829120] 2024-09-14T04:36:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:52 smithi129 ceph-mon[101316]: pgmap v2088: 105 pgs: 105 active+clean; 1.1 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 111 KiB/s rd, 1.2 KiB/s wr, 223 op/s 2024-09-14T04:36:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:52 smithi129 ceph-mon[101316]: osdmap e1302: 8 total, 8 up, 8 in 2024-09-14T04:36:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:36:53.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:52 smithi039 ceph-mon[106470]: pgmap v2088: 105 pgs: 105 active+clean; 1.1 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 111 KiB/s rd, 1.2 KiB/s wr, 223 op/s 2024-09-14T04:36:53.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:52 smithi039 ceph-mon[106470]: osdmap e1302: 8 total, 8 up, 8 in 2024-09-14T04:36:53.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:36:53.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:52 smithi039 ceph-mon[111235]: pgmap v2088: 105 pgs: 105 active+clean; 1.1 GiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 111 KiB/s rd, 1.2 KiB/s wr, 223 op/s 2024-09-14T04:36:53.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:52 smithi039 ceph-mon[111235]: osdmap e1302: 8 total, 8 up, 8 in 2024-09-14T04:36:53.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:36:53.948 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:53.948 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:53.962 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~239075328,243269632~41943040,289406976~125829120] 2024-09-14T04:36:53.963 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:53.963 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:53.965 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:53.965 INFO:tasks.workunit.client.0.smithi039.stdout:from 2 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:53.965 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~25165824,335544320~41943040,381681664~29360128] 2024-09-14T04:36:53.967 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:54.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:53 smithi129 ceph-mon[101316]: osdmap e1303: 8 total, 8 up, 8 in 2024-09-14T04:36:54.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:53 smithi039 ceph-mon[106470]: osdmap e1303: 8 total, 8 up, 8 in 2024-09-14T04:36:54.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:53 smithi039 ceph-mon[111235]: osdmap e1303: 8 total, 8 up, 8 in 2024-09-14T04:36:54.964 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:54.964 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:54.972 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~239075328,243269632~41943040,289406976~125829120] 2024-09-14T04:36:54.972 INFO:tasks.workunit.client.0.smithi039.stdout:from 1 to HEAD diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:54.972 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~20971520,92274688~16777216,117440512~121634816,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:54.973 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~113246208,117440512~121634816,243269632~41943040,289406976~125829120] 2024-09-14T04:36:55.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:54 smithi129 ceph-mon[101316]: pgmap v2091: 105 pgs: 105 active+clean; 957 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 148 KiB/s rd, 6.0 KiB/s wr, 242 op/s 2024-09-14T04:36:55.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:54 smithi129 ceph-mon[101316]: osdmap e1304: 8 total, 8 up, 8 in 2024-09-14T04:36:55.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:54 smithi039 ceph-mon[106470]: pgmap v2091: 105 pgs: 105 active+clean; 957 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 148 KiB/s rd, 6.0 KiB/s wr, 242 op/s 2024-09-14T04:36:55.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:54 smithi039 ceph-mon[106470]: osdmap e1304: 8 total, 8 up, 8 in 2024-09-14T04:36:55.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:54 smithi039 ceph-mon[111235]: pgmap v2091: 105 pgs: 105 active+clean; 957 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 148 KiB/s rd, 6.0 KiB/s wr, 242 op/s 2024-09-14T04:36:55.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:54 smithi039 ceph-mon[111235]: osdmap e1304: 8 total, 8 up, 8 in 2024-09-14T04:36:55.861 INFO:tasks.workunit.client.0.smithi039.stdout:from 0 to HEAD diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:55.861 INFO:tasks.workunit.client.0.smithi039.stdout: limited diff [0~62914560,67108864~25165824,96468992~12582912,113246208~125829120,243269632~33554432,281018368~4194304,289406976~12582912,306184192~71303168,381681664~29360128] 2024-09-14T04:36:55.874 INFO:tasks.workunit.client.0.smithi039.stdout: actual was [0~239075328,243269632~41943040,289406976~125829120] 2024-09-14T04:36:56.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:55 smithi129 ceph-mon[101316]: osdmap e1305: 8 total, 8 up, 8 in 2024-09-14T04:36:56.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:55 smithi129 ceph-mon[101316]: osdmap e1306: 8 total, 8 up, 8 in 2024-09-14T04:36:56.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:55 smithi039 ceph-mon[106470]: osdmap e1305: 8 total, 8 up, 8 in 2024-09-14T04:36:56.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:55 smithi039 ceph-mon[106470]: osdmap e1306: 8 total, 8 up, 8 in 2024-09-14T04:36:56.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:55 smithi039 ceph-mon[111235]: osdmap e1305: 8 total, 8 up, 8 in 2024-09-14T04:36:56.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:55 smithi039 ceph-mon[111235]: osdmap e1306: 8 total, 8 up, 8 in 2024-09-14T04:36:56.868 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/1.DiffIterateStress (38632 ms) 2024-09-14T04:36:56.868 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/1.DiffIterateParent 2024-09-14T04:36:56.868 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:36:57.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:56 smithi129 ceph-mon[101316]: pgmap v2094: 105 pgs: 105 active+clean; 922 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 139 KiB/s rd, 6.0 KiB/s wr, 224 op/s 2024-09-14T04:36:57.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:56 smithi129 ceph-mon[101316]: osdmap e1307: 8 total, 8 up, 8 in 2024-09-14T04:36:57.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:56 smithi039 ceph-mon[106470]: pgmap v2094: 105 pgs: 105 active+clean; 922 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 139 KiB/s rd, 6.0 KiB/s wr, 224 op/s 2024-09-14T04:36:57.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:56 smithi039 ceph-mon[106470]: osdmap e1307: 8 total, 8 up, 8 in 2024-09-14T04:36:57.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:56 smithi039 ceph-mon[111235]: pgmap v2094: 105 pgs: 105 active+clean; 922 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 139 KiB/s rd, 6.0 KiB/s wr, 224 op/s 2024-09-14T04:36:57.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:56 smithi039 ceph-mon[111235]: osdmap e1307: 8 total, 8 up, 8 in 2024-09-14T04:36:57.959 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/1.DiffIterateParent (1091 ms) 2024-09-14T04:36:57.960 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/1.DiffIterateIgnoreParent 2024-09-14T04:36:57.960 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:36:59.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:58 smithi039 ceph-mon[106470]: pgmap v2097: 105 pgs: 105 active+clean; 865 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 64 KiB/s rd, 5.0 KiB/s wr, 94 op/s 2024-09-14T04:36:59.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:58 smithi039 ceph-mon[106470]: osdmap e1308: 8 total, 8 up, 8 in 2024-09-14T04:36:59.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:36:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:36:58] "GET /metrics HTTP/1.1" 200 36696 "" "Prometheus/2.51.0" 2024-09-14T04:36:59.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:58 smithi039 ceph-mon[111235]: pgmap v2097: 105 pgs: 105 active+clean; 865 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 64 KiB/s rd, 5.0 KiB/s wr, 94 op/s 2024-09-14T04:36:59.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:58 smithi039 ceph-mon[111235]: osdmap e1308: 8 total, 8 up, 8 in 2024-09-14T04:36:59.097 INFO:tasks.workunit.client.0.smithi039.stdout: wrote [0~8388608,12582912~8388608] to clone 2024-09-14T04:36:59.097 INFO:tasks.workunit.client.0.smithi039.stdout: diff was [0~8388608,12582912~8388608] 2024-09-14T04:36:59.101 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/1.DiffIterateIgnoreParent (1142 ms) 2024-09-14T04:36:59.101 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/1.DiffIterateCallbackError 2024-09-14T04:36:59.101 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:36:59.122 INFO:tasks.workunit.client.0.smithi039.stdout: wrote [1696965~61952,3890000~65192,4524493~50430,4662525~93011,6957851~82726,11808580~13234,14368439~89370,15339286~14403,19404872~25841,20751343~91622] 2024-09-14T04:36:59.132 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/1.DiffIterateCallbackError (31 ms) 2024-09-14T04:36:59.132 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/1.DiffIterateParentDiscard 2024-09-14T04:36:59.132 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:36:59.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:58 smithi129 ceph-mon[101316]: pgmap v2097: 105 pgs: 105 active+clean; 865 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 64 KiB/s rd, 5.0 KiB/s wr, 94 op/s 2024-09-14T04:36:59.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:58 smithi129 ceph-mon[101316]: osdmap e1308: 8 total, 8 up, 8 in 2024-09-14T04:37:00.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:36:59 smithi129 ceph-mon[101316]: osdmap e1309: 8 total, 8 up, 8 in 2024-09-14T04:37:00.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:36:59 smithi039 ceph-mon[106470]: osdmap e1309: 8 total, 8 up, 8 in 2024-09-14T04:37:00.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:36:59 smithi039 ceph-mon[111235]: osdmap e1309: 8 total, 8 up, 8 in 2024-09-14T04:37:00.949 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/1.DiffIterateParentDiscard (1817 ms) 2024-09-14T04:37:00.949 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/1.DiffIterateUnalignedSmall 2024-09-14T04:37:00.949 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:37:01.003 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/1.DiffIterateUnalignedSmall (54 ms) 2024-09-14T04:37:01.004 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/1.DiffIterateUnaligned 2024-09-14T04:37:01.004 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:37:01.080 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/1.DiffIterateUnaligned (77 ms) 2024-09-14T04:37:01.080 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/1.DiffIterateTryAcquireLock 2024-09-14T04:37:01.080 INFO:tasks.workunit.client.0.smithi039.stdout:using new format! 2024-09-14T04:37:01.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:00 smithi129 ceph-mon[101316]: pgmap v2100: 105 pgs: 105 active+clean; 793 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 5.0 MiB/s wr, 123 op/s 2024-09-14T04:37:01.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:00 smithi129 ceph-mon[101316]: osdmap e1310: 8 total, 8 up, 8 in 2024-09-14T04:37:01.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:00 smithi129 ceph-mon[101316]: osdmap e1311: 8 total, 8 up, 8 in 2024-09-14T04:37:01.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:00 smithi039 ceph-mon[106470]: pgmap v2100: 105 pgs: 105 active+clean; 793 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 5.0 MiB/s wr, 123 op/s 2024-09-14T04:37:01.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:00 smithi039 ceph-mon[106470]: osdmap e1310: 8 total, 8 up, 8 in 2024-09-14T04:37:01.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:00 smithi039 ceph-mon[106470]: osdmap e1311: 8 total, 8 up, 8 in 2024-09-14T04:37:01.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:00 smithi039 ceph-mon[111235]: pgmap v2100: 105 pgs: 105 active+clean; 793 MiB data, 2.6 GiB used, 713 GiB / 715 GiB avail; 83 KiB/s rd, 5.0 MiB/s wr, 123 op/s 2024-09-14T04:37:01.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:00 smithi039 ceph-mon[111235]: osdmap e1310: 8 total, 8 up, 8 in 2024-09-14T04:37:01.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:00 smithi039 ceph-mon[111235]: osdmap e1311: 8 total, 8 up, 8 in 2024-09-14T04:37:03.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:02 smithi129 ceph-mon[101316]: pgmap v2103: 105 pgs: 105 active+clean; 791 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 68 KiB/s rd, 5.0 MiB/s wr, 95 op/s 2024-09-14T04:37:03.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:02 smithi039 ceph-mon[106470]: pgmap v2103: 105 pgs: 105 active+clean; 791 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 68 KiB/s rd, 5.0 MiB/s wr, 95 op/s 2024-09-14T04:37:03.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:02 smithi039 ceph-mon[111235]: pgmap v2103: 105 pgs: 105 active+clean; 791 MiB data, 2.5 GiB used, 713 GiB / 715 GiB avail; 68 KiB/s rd, 5.0 MiB/s wr, 95 op/s 2024-09-14T04:37:05.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:04 smithi129 ceph-mon[101316]: pgmap v2104: 105 pgs: 105 active+clean; 872 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 9.9 MiB/s wr, 229 op/s 2024-09-14T04:37:05.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:04 smithi039 ceph-mon[106470]: pgmap v2104: 105 pgs: 105 active+clean; 872 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 9.9 MiB/s wr, 229 op/s 2024-09-14T04:37:05.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:04 smithi039 ceph-mon[111235]: pgmap v2104: 105 pgs: 105 active+clean; 872 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.8 MiB/s rd, 9.9 MiB/s wr, 229 op/s 2024-09-14T04:37:06.124 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] DiffIterateTest/1.DiffIterateTryAcquireLock (5043 ms) 2024-09-14T04:37:06.124 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] DiffIterateTest/1.DiffIterateStriping 2024-09-14T04:37:06.124 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/librbd/test_librbd.cc:8958: Skipped 2024-09-14T04:37:06.124 INFO:tasks.workunit.client.0.smithi039.stdout:Skipping due to unmet REQUIRE 2024-09-14T04:37:06.124 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] DiffIterateTest/1.DiffIterateStriping (0 ms) 2024-09-14T04:37:06.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:05 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:37:06.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:05 smithi129 ceph-mon[101316]: osdmap e1312: 8 total, 8 up, 8 in 2024-09-14T04:37:06.289 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:05 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:37:06.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:05 smithi039 ceph-mon[106470]: osdmap e1312: 8 total, 8 up, 8 in 2024-09-14T04:37:06.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:05 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:37:06.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:05 smithi039 ceph-mon[111235]: osdmap e1312: 8 total, 8 up, 8 in 2024-09-14T04:37:07.032 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 19 tests from DiffIterateTest/1 (250961 ms total) 2024-09-14T04:37:07.032 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:37:07.032 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] Global test environment tear-down 2024-09-14T04:37:07.032 INFO:tasks.workunit.client.0.smithi039.stdout:[==========] 490 tests from 24 test suites ran. (2564859 ms total) 2024-09-14T04:37:07.032 INFO:tasks.workunit.client.0.smithi039.stdout:[ PASSED ] 444 tests. 2024-09-14T04:37:07.032 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] 46 tests, listed below: 2024-09-14T04:37:07.032 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalStress.DiscardWithPruneWriteOverlap 2024-09-14T04:37:07.033 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.AioDiscardEvent 2024-09-14T04:37:07.033 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.AioWriteEvent 2024-09-14T04:37:07.033 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.AioFlushEvent 2024-09-14T04:37:07.033 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.SnapCreate 2024-09-14T04:37:07.033 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.SnapProtect 2024-09-14T04:37:07.033 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.SnapUnprotect 2024-09-14T04:37:07.033 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.SnapRename 2024-09-14T04:37:07.033 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.SnapRollback 2024-09-14T04:37:07.033 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.SnapRemove 2024-09-14T04:37:07.033 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.Rename 2024-09-14T04:37:07.033 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.Resize 2024-09-14T04:37:07.033 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.Flatten 2024-09-14T04:37:07.033 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.UpdateFeatures 2024-09-14T04:37:07.034 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.MetadataSet 2024-09-14T04:37:07.034 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.MetadataRemove 2024-09-14T04:37:07.034 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalReplay.ObjectPosition 2024-09-14T04:37:07.034 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalEntries.AioWrite 2024-09-14T04:37:07.034 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalEntries.AioDiscard 2024-09-14T04:37:07.034 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalEntries.AioDiscardWithPrune 2024-09-14T04:37:07.034 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestJournalEntries.AioFlush 2024-09-14T04:37:07.034 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestOperations.DisableJournalingCorrupt 2024-09-14T04:37:07.034 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestMigration.MirroringSamePool 2024-09-14T04:37:07.034 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestMigration.MirroringAbort 2024-09-14T04:37:07.034 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestMigration.MirroringOtherPoolDisabled 2024-09-14T04:37:07.034 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestMigration.MirroringOtherPoolEnabled 2024-09-14T04:37:07.034 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestMigration.MirroringPool 2024-09-14T04:37:07.034 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestDeepCopy.NoSnaps_StrippingLargerDstObjSize 2024-09-14T04:37:07.035 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestDeepCopy.Snaps_StrippingLargerDstObjSize 2024-09-14T04:37:07.035 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestDeepCopy.Clone_StrippingLargerDstObjSize 2024-09-14T04:37:07.035 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestDeepCopy.CloneFlatten_StrippingLargerDstObjSize 2024-09-14T04:37:07.035 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestDeepCopy.Stress_StrippingLargerDstObjSize 2024-09-14T04:37:07.035 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestDeepCopy.NoSnaps_StrippingSmallerDstObjSize 2024-09-14T04:37:07.035 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestDeepCopy.Snaps_StrippingSmallerDstObjSize 2024-09-14T04:37:07.035 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestDeepCopy.Clone_StrippingSmallerDstObjSize 2024-09-14T04:37:07.035 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestDeepCopy.CloneFlatten_StrippingSmallerDstObjSize 2024-09-14T04:37:07.035 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestDeepCopy.Stress_StrippingSmallerDstObjSize 2024-09-14T04:37:07.035 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestMirroring.DisableImageMirrorWithPeer 2024-09-14T04:37:07.035 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestMirroring.DisableJournalingWithPeer 2024-09-14T04:37:07.035 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestInternal.DiscardCopyup 2024-09-14T04:37:07.035 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestInternal.TestCoR 2024-09-14T04:37:07.036 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestLibRBD.EncryptionLoadBadStripePattern 2024-09-14T04:37:07.036 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestLibRBD.EnableJournalingViaLockOwner 2024-09-14T04:37:07.036 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] TestLibRBD.ExclusiveLockReadTransition 2024-09-14T04:37:07.036 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] DiffIterateTest/0.DiffIterateStriping 2024-09-14T04:37:07.036 INFO:tasks.workunit.client.0.smithi039.stdout:[ SKIPPED ] DiffIterateTest/1.DiffIterateStriping 2024-09-14T04:37:07.036 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:37:07.036 INFO:tasks.workunit.client.0.smithi039.stdout: YOU HAVE 2 DISABLED TESTS 2024-09-14T04:37:07.036 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T04:37:07.041 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-09-14T04:37:07.042 DEBUG:teuthology.orchestra.run.smithi039:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-14T04:37:07.105 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:06 smithi039 ceph-mon[106470]: pgmap v2105: 105 pgs: 105 active+clean; 884 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 9.9 MiB/s wr, 249 op/s 2024-09-14T04:37:07.106 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:07 smithi039 ceph-mon[111235]: pgmap v2105: 105 pgs: 105 active+clean; 884 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 9.9 MiB/s wr, 249 op/s 2024-09-14T04:37:07.112 INFO:tasks.workunit:Stopping ['rbd/test_librbd.sh'] on client.0... 2024-09-14T04:37:07.112 DEBUG:teuthology.orchestra.run.smithi039:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-09-14T04:37:07.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:06 smithi129 ceph-mon[101316]: pgmap v2105: 105 pgs: 105 active+clean; 884 MiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 3.3 MiB/s rd, 9.9 MiB/s wr, 249 op/s 2024-09-14T04:37:07.408 DEBUG:teuthology.parallel:result is None 2024-09-14T04:37:07.409 DEBUG:teuthology.orchestra.run.smithi039:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-09-14T04:37:07.435 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-09-14T04:37:07.435 DEBUG:teuthology.orchestra.run.smithi039:> rmdir -- /home/ubuntu/cephtest/mnt.0 2024-09-14T04:37:07.498 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2024-09-14T04:37:07.498 INFO:teuthology.task.sequential:In sequential, running task print... 2024-09-14T04:37:07.498 INFO:teuthology.task.print:**** done rbd/test_librbd.sh 7-workload 2024-09-14T04:37:07.499 DEBUG:teuthology.parallel:result is None 2024-09-14T04:37:07.499 INFO:teuthology.run_tasks:Running task print... 2024-09-14T04:37:07.511 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-09-14T04:37:07.512 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-09-14T04:37:07.528 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-09-14T04:37:07.528 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-09-14T04:37:07.528 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph mgr dump --format=json 2024-09-14T04:37:08.190 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:37:08.313 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:08 smithi039 ceph-mon[106470]: osdmap e1313: 8 total, 8 up, 8 in 2024-09-14T04:37:08.313 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:08 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:37:08.313 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:08 smithi039 ceph-mon[111235]: osdmap e1313: 8 total, 8 up, 8 in 2024-09-14T04:37:08.313 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:08 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:37:08.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:08 smithi129 ceph-mon[101316]: osdmap e1313: 8 total, 8 up, 8 in 2024-09-14T04:37:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:08 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:37:08.972 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:37:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:37:08] "GET /metrics HTTP/1.1" 200 34258 "" "Prometheus/2.51.0" 2024-09-14T04:37:09.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:09 smithi129 ceph-mon[101316]: pgmap v2108: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:37:09.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:09 smithi039 ceph-mon[111235]: pgmap v2108: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:37:09.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:09 smithi039 ceph-mon[106470]: pgmap v2108: 73 pgs: 73 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:37:10.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:10 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:37:10.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:10 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:37:10.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:10 smithi039 ceph-mon[111235]: pgmap v2109: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:37:10.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:10 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:37:10.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:10 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:37:10.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:10 smithi039 ceph-mon[106470]: pgmap v2109: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:37:10.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:10 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:37:10.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:10 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:37:10.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:10 smithi129 ceph-mon[101316]: pgmap v2109: 73 pgs: 73 active+clean; 582 KiB data, 2.5 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:37:11.441 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:37:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:11 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:37:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:11 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:37:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:11 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:37:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:11 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:37:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:11 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:37:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:11 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4154442866' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-09-14T04:37:11.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:11 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:37:11.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:11 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:37:11.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:11 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:37:11.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:11 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:37:11.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:11 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:37:11.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:11 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4154442866' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-09-14T04:37:11.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:11 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:37:11.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:11 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:37:11.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:11 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:37:11.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:11 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:37:11.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:11 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:37:11.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:11 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4154442866' entity='client.admin' cmd={"prefix": "mgr dump", "format": "json"} : dispatch 2024-09-14T04:37:12.121 INFO:teuthology.orchestra.run.smithi039.stdout:{"epoch":41,"flags":0,"active_gid":25157,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6800","nonce":4157380157},{"type":"v1","addr":"172.21.15.39:6801","nonce":4157380157}]},"active_addr":"172.21.15.39:6801/4157380157","active_change":"2024-09-14T03:59:06.707943+0000","active_mgr_features":4540719139924082687,"available":true,"standbys":[{"gid":24976,"name":"x","mgr_features":4540719139924082687,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:sclorg-nginx-126","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.17","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"oauth2-proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba_metrics":{"name":"container_image_samba_metrics","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-metrics:latest","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB metrics exporter container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to, use commas to separate multiple","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.27.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_nginx":{"name":"container_image_nginx","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nginx:sclorg-nginx-126","min":"","max":"","enum_allowed":[],"desc":"Nginx container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.17","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_oauth2_proxy":{"name":"container_image_oauth2_proxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/oauth2-proxy/oauth2-proxy:v7.6.0","min":"","max":"","enum_allowed":[],"desc":"oauth2-proxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba_metrics":{"name":"container_image_samba_metrics","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-metrics:latest","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB metrics exporter container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number.if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"smb","can_run":true,"error_string":"","module_options":{"internal_store_backend":{"name":"internal_store_backend","type":"str","level":"dev","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"set internal store backend. for develoment and testing only","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_orchestration":{"name":"update_orchestration","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically update orchestration when smb resources are changed","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.39:8443/","prometheus":"http://172.21.15.39:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":348,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.39:0","nonce":2106679592}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.39:0","nonce":106076659}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.39:0","nonce":1884661062}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.39:0","nonce":229867376}]}]} 2024-09-14T04:37:12.126 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-09-14T04:37:12.126 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-14T04:37:12.126 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T04:37:12.339 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:37:13.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:13 smithi039 ceph-mon[106470]: pgmap v2110: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:37:13.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:13 smithi039 ceph-mon[111235]: pgmap v2110: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:37:13.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:13 smithi129 ceph-mon[101316]: pgmap v2110: 73 pgs: 73 active+clean; 582 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:37:14.446 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:37:14.447 INFO:teuthology.orchestra.run.smithi039.stdout:{"epoch":1313,"fsid":"bd7c803c-724b-11ef-bceb-c7b262605968","created":"2024-09-14T03:46:13.726476+0000","modified":"2024-09-14T04:37:06.997883+0000","last_up_change":"2024-09-14T04:07:57.405001+0000","last_in_change":"2024-09-14T03:50:56.569390+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":52,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:49:18.022402+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T03:51:45.764151+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-14T03:53:04.166504+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-14T03:53:04.665111+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"72","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"5fdfd478-dd27-4674-a7d6-53177d9e549a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":667,"up_from":671,"up_thru":1165,"down_at":667,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6802","nonce":3832421293},{"type":"v1","addr":"172.21.15.39:6803","nonce":3832421293}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6804","nonce":3832421293},{"type":"v1","addr":"172.21.15.39:6805","nonce":3832421293}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6808","nonce":3832421293},{"type":"v1","addr":"172.21.15.39:6809","nonce":3832421293}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6806","nonce":3832421293},{"type":"v1","addr":"172.21.15.39:6807","nonce":3832421293}]},"public_addr":"172.21.15.39:6803/3832421293","cluster_addr":"172.21.15.39:6805/3832421293","heartbeat_back_addr":"172.21.15.39:6809/3832421293","heartbeat_front_addr":"172.21.15.39:6807/3832421293","state":["exists","up"]},{"osd":1,"uuid":"991c236e-1679-4c1e-a1a9-3894b1b12055","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":677,"up_from":681,"up_thru":1165,"down_at":677,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6810","nonce":1186072955},{"type":"v1","addr":"172.21.15.39:6811","nonce":1186072955}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6812","nonce":1186072955},{"type":"v1","addr":"172.21.15.39:6813","nonce":1186072955}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6816","nonce":1186072955},{"type":"v1","addr":"172.21.15.39:6817","nonce":1186072955}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6814","nonce":1186072955},{"type":"v1","addr":"172.21.15.39:6815","nonce":1186072955}]},"public_addr":"172.21.15.39:6811/1186072955","cluster_addr":"172.21.15.39:6813/1186072955","heartbeat_back_addr":"172.21.15.39:6817/1186072955","heartbeat_front_addr":"172.21.15.39:6815/1186072955","state":["exists","up"]},{"osd":2,"uuid":"f9611ffd-5d37-4465-8114-ecc70d4d6d14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":688,"up_from":694,"up_thru":1165,"down_at":688,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6818","nonce":2359749690},{"type":"v1","addr":"172.21.15.39:6819","nonce":2359749690}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6820","nonce":2359749690},{"type":"v1","addr":"172.21.15.39:6821","nonce":2359749690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6824","nonce":2359749690},{"type":"v1","addr":"172.21.15.39:6825","nonce":2359749690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6822","nonce":2359749690},{"type":"v1","addr":"172.21.15.39:6823","nonce":2359749690}]},"public_addr":"172.21.15.39:6819/2359749690","cluster_addr":"172.21.15.39:6821/2359749690","heartbeat_back_addr":"172.21.15.39:6825/2359749690","heartbeat_front_addr":"172.21.15.39:6823/2359749690","state":["exists","up"]},{"osd":3,"uuid":"56b6877b-2107-47d6-8da9-015db7352288","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":697,"up_from":701,"up_thru":1165,"down_at":697,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6826","nonce":2092194762},{"type":"v1","addr":"172.21.15.39:6827","nonce":2092194762}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6828","nonce":2092194762},{"type":"v1","addr":"172.21.15.39:6829","nonce":2092194762}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6832","nonce":2092194762},{"type":"v1","addr":"172.21.15.39:6833","nonce":2092194762}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6830","nonce":2092194762},{"type":"v1","addr":"172.21.15.39:6831","nonce":2092194762}]},"public_addr":"172.21.15.39:6827/2092194762","cluster_addr":"172.21.15.39:6829/2092194762","heartbeat_back_addr":"172.21.15.39:6833/2092194762","heartbeat_front_addr":"172.21.15.39:6831/2092194762","state":["exists","up"]},{"osd":4,"uuid":"2d5523c4-5871-4fb3-bc0d-62c3b7d8b6a1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":1165,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6800","nonce":168261430},{"type":"v1","addr":"172.21.15.129:6801","nonce":168261430}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6802","nonce":168261430},{"type":"v1","addr":"172.21.15.129:6803","nonce":168261430}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6806","nonce":168261430},{"type":"v1","addr":"172.21.15.129:6807","nonce":168261430}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6804","nonce":168261430},{"type":"v1","addr":"172.21.15.129:6805","nonce":168261430}]},"public_addr":"172.21.15.129:6801/168261430","cluster_addr":"172.21.15.129:6803/168261430","heartbeat_back_addr":"172.21.15.129:6807/168261430","heartbeat_front_addr":"172.21.15.129:6805/168261430","state":["exists","up"]},{"osd":5,"uuid":"a3ba55fd-f32e-4f6d-9b48-d2845a3474cd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":1165,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6808","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6809","nonce":2184993002}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6810","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6811","nonce":2184993002}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6814","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6815","nonce":2184993002}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6812","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6813","nonce":2184993002}]},"public_addr":"172.21.15.129:6809/2184993002","cluster_addr":"172.21.15.129:6811/2184993002","heartbeat_back_addr":"172.21.15.129:6815/2184993002","heartbeat_front_addr":"172.21.15.129:6813/2184993002","state":["exists","up"]},{"osd":6,"uuid":"79e9c0a2-86a4-4267-a799-059d473d7b74","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":1165,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6816","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6817","nonce":3767031353}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6818","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6819","nonce":3767031353}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6822","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6823","nonce":3767031353}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6820","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6821","nonce":3767031353}]},"public_addr":"172.21.15.129:6817/3767031353","cluster_addr":"172.21.15.129:6819/3767031353","heartbeat_back_addr":"172.21.15.129:6823/3767031353","heartbeat_front_addr":"172.21.15.129:6821/3767031353","state":["exists","up"]},{"osd":7,"uuid":"22f41628-4b88-4b8c-82b2-fcd2802da1bf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":1165,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6824","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6825","nonce":2370414690}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6826","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6827","nonce":2370414690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6830","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6831","nonce":2370414690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6828","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6829","nonce":2370414690}]},"public_addr":"172.21.15.129:6825/2370414690","cluster_addr":"172.21.15.129:6827/2370414690","heartbeat_back_addr":"172.21.15.129:6831/2370414690","heartbeat_front_addr":"172.21.15.129:6829/2370414690","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:48:23.355518+0000","dead_epoch":666},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:48:48.715441+0000","dead_epoch":677},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:49:13.415323+0000","dead_epoch":687},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:49:39.935205+0000","dead_epoch":697},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:50:01.649042+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:50:24.001678+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:50:47.985011+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:51:11.584825+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.10","mappings":[{"from":1,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":5}]},{"pgid":"4.1f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.39:0/1864073859":"2024-09-14T05:12:12.455045+0000","172.21.15.39:0/1958359945":"2024-09-14T05:02:41.504544+0000","172.21.15.39:0/2766717202":"2024-09-14T05:02:40.464304+0000","172.21.15.39:0/1825825785":"2024-09-15T03:55:55.288048+0000","172.21.15.129:0/3885028850":"2024-09-15T03:56:26.175569+0000","172.21.15.129:6835/1529931471":"2024-09-15T03:56:26.175569+0000","172.21.15.39:0/822851906":"2024-09-15T03:59:06.707728+0000","172.21.15.39:6800/748677190":"2024-09-15T03:51:44.726801+0000","172.21.15.39:0/3247815370":"2024-09-15T03:59:06.707728+0000","172.21.15.129:0/78252352":"2024-09-15T03:56:26.175569+0000","172.21.15.39:0/1530637819":"2024-09-15T03:46:59.511580+0000","172.21.15.39:6801/628437919":"2024-09-15T03:46:35.805097+0000","172.21.15.39:6801/748677190":"2024-09-15T03:51:44.726801+0000","172.21.15.39:0/2745905728":"2024-09-15T03:51:44.726801+0000","172.21.15.129:0/2660542464":"2024-09-15T03:56:26.175569+0000","172.21.15.39:0/209353267":"2024-09-15T03:46:59.511580+0000","172.21.15.39:6800/3127470450":"2024-09-15T03:46:59.511580+0000","172.21.15.129:0/1000360058":"2024-09-15T03:56:26.175569+0000","172.21.15.39:0/2670041591":"2024-09-15T03:46:35.805097+0000","172.21.15.39:6801/3924974081":"2024-09-15T03:55:55.288048+0000","172.21.15.39:6800/3924974081":"2024-09-15T03:55:55.288048+0000","172.21.15.39:0/2951004223":"2024-09-15T03:51:44.726801+0000","172.21.15.39:0/3196173579":"2024-09-15T03:55:55.288048+0000","172.21.15.39:0/502495021":"2024-09-15T03:55:55.288048+0000","172.21.15.39:0/3161547781":"2024-09-15T03:46:59.511580+0000","172.21.15.39:0/1398136686":"2024-09-15T03:59:06.707728+0000","172.21.15.39:6801/3127470450":"2024-09-15T03:46:59.511580+0000","172.21.15.39:0/2133325947":"2024-09-15T03:46:35.805097+0000","172.21.15.39:6800/1103763596":"2024-09-15T03:59:06.707728+0000","172.21.15.129:6834/1529931471":"2024-09-15T03:56:26.175569+0000","172.21.15.39:0/1746545094":"2024-09-15T03:59:06.707728+0000","172.21.15.129:0/1993195691":"2024-09-15T03:56:26.175569+0000","172.21.15.39:6801/1103763596":"2024-09-15T03:59:06.707728+0000","172.21.15.39:0/754571920":"2024-09-15T03:51:44.726801+0000","172.21.15.39:0/81449629":"2024-09-15T03:46:35.805097+0000","172.21.15.39:6800/628437919":"2024-09-15T03:46:35.805097+0000","172.21.15.129:0/3477521861":"2024-09-15T03:56:26.175569+0000","172.21.15.39:0/2587896825":"2024-09-15T03:51:44.726801+0000","172.21.15.39:0/1766366412":"2024-09-15T03:55:55.288048+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":6,"snaps":[{"begin":2,"length":1}]},{"pool":7,"snaps":[{"begin":2,"length":1}]},{"pool":8,"snaps":[{"begin":2,"length":1}]},{"pool":9,"snaps":[{"begin":2,"length":1}]},{"pool":10,"snaps":[{"begin":2,"length":1}]},{"pool":11,"snaps":[{"begin":2,"length":1}]},{"pool":12,"snaps":[{"begin":233,"length":1}]},{"pool":14,"snaps":[{"begin":6,"length":3}]},{"pool":16,"snaps":[{"begin":35,"length":3},{"begin":45,"length":3}]},{"pool":17,"snaps":[{"begin":22,"length":1}]},{"pool":18,"snaps":[{"begin":2,"length":1}]},{"pool":51,"snaps":[{"begin":104,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T04:37:15.093 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-14T04:37:15.094 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T04:37:15.302 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:37:15.424 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:15 smithi039 ceph-mon[106470]: pgmap v2111: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:15.424 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:15 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1491801890' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T04:37:15.424 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:15 smithi039 ceph-mon[111235]: pgmap v2111: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:15.424 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:15 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1491801890' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T04:37:15.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:15 smithi129 ceph-mon[101316]: pgmap v2111: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:15.643 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:15 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1491801890' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T04:37:17.252 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:37:17.252 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:17 smithi039 ceph-mon[106470]: pgmap v2112: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:17.253 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:17 smithi039 ceph-mon[111235]: pgmap v2112: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:17.253 INFO:teuthology.orchestra.run.smithi039.stdout:{"epoch":1313,"fsid":"bd7c803c-724b-11ef-bceb-c7b262605968","created":"2024-09-14T03:46:13.726476+0000","modified":"2024-09-14T04:37:06.997883+0000","last_up_change":"2024-09-14T04:07:57.405001+0000","last_in_change":"2024-09-14T03:50:56.569390+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":52,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:49:18.022402+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T03:51:45.764151+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-14T03:53:04.166504+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-14T03:53:04.665111+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"72","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"5fdfd478-dd27-4674-a7d6-53177d9e549a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":667,"up_from":671,"up_thru":1165,"down_at":667,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6802","nonce":3832421293},{"type":"v1","addr":"172.21.15.39:6803","nonce":3832421293}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6804","nonce":3832421293},{"type":"v1","addr":"172.21.15.39:6805","nonce":3832421293}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6808","nonce":3832421293},{"type":"v1","addr":"172.21.15.39:6809","nonce":3832421293}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6806","nonce":3832421293},{"type":"v1","addr":"172.21.15.39:6807","nonce":3832421293}]},"public_addr":"172.21.15.39:6803/3832421293","cluster_addr":"172.21.15.39:6805/3832421293","heartbeat_back_addr":"172.21.15.39:6809/3832421293","heartbeat_front_addr":"172.21.15.39:6807/3832421293","state":["exists","up"]},{"osd":1,"uuid":"991c236e-1679-4c1e-a1a9-3894b1b12055","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":677,"up_from":681,"up_thru":1165,"down_at":677,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6810","nonce":1186072955},{"type":"v1","addr":"172.21.15.39:6811","nonce":1186072955}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6812","nonce":1186072955},{"type":"v1","addr":"172.21.15.39:6813","nonce":1186072955}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6816","nonce":1186072955},{"type":"v1","addr":"172.21.15.39:6817","nonce":1186072955}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6814","nonce":1186072955},{"type":"v1","addr":"172.21.15.39:6815","nonce":1186072955}]},"public_addr":"172.21.15.39:6811/1186072955","cluster_addr":"172.21.15.39:6813/1186072955","heartbeat_back_addr":"172.21.15.39:6817/1186072955","heartbeat_front_addr":"172.21.15.39:6815/1186072955","state":["exists","up"]},{"osd":2,"uuid":"f9611ffd-5d37-4465-8114-ecc70d4d6d14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":688,"up_from":694,"up_thru":1165,"down_at":688,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6818","nonce":2359749690},{"type":"v1","addr":"172.21.15.39:6819","nonce":2359749690}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6820","nonce":2359749690},{"type":"v1","addr":"172.21.15.39:6821","nonce":2359749690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6824","nonce":2359749690},{"type":"v1","addr":"172.21.15.39:6825","nonce":2359749690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6822","nonce":2359749690},{"type":"v1","addr":"172.21.15.39:6823","nonce":2359749690}]},"public_addr":"172.21.15.39:6819/2359749690","cluster_addr":"172.21.15.39:6821/2359749690","heartbeat_back_addr":"172.21.15.39:6825/2359749690","heartbeat_front_addr":"172.21.15.39:6823/2359749690","state":["exists","up"]},{"osd":3,"uuid":"56b6877b-2107-47d6-8da9-015db7352288","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":697,"up_from":701,"up_thru":1165,"down_at":697,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6826","nonce":2092194762},{"type":"v1","addr":"172.21.15.39:6827","nonce":2092194762}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6828","nonce":2092194762},{"type":"v1","addr":"172.21.15.39:6829","nonce":2092194762}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6832","nonce":2092194762},{"type":"v1","addr":"172.21.15.39:6833","nonce":2092194762}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6830","nonce":2092194762},{"type":"v1","addr":"172.21.15.39:6831","nonce":2092194762}]},"public_addr":"172.21.15.39:6827/2092194762","cluster_addr":"172.21.15.39:6829/2092194762","heartbeat_back_addr":"172.21.15.39:6833/2092194762","heartbeat_front_addr":"172.21.15.39:6831/2092194762","state":["exists","up"]},{"osd":4,"uuid":"2d5523c4-5871-4fb3-bc0d-62c3b7d8b6a1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":1165,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6800","nonce":168261430},{"type":"v1","addr":"172.21.15.129:6801","nonce":168261430}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6802","nonce":168261430},{"type":"v1","addr":"172.21.15.129:6803","nonce":168261430}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6806","nonce":168261430},{"type":"v1","addr":"172.21.15.129:6807","nonce":168261430}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6804","nonce":168261430},{"type":"v1","addr":"172.21.15.129:6805","nonce":168261430}]},"public_addr":"172.21.15.129:6801/168261430","cluster_addr":"172.21.15.129:6803/168261430","heartbeat_back_addr":"172.21.15.129:6807/168261430","heartbeat_front_addr":"172.21.15.129:6805/168261430","state":["exists","up"]},{"osd":5,"uuid":"a3ba55fd-f32e-4f6d-9b48-d2845a3474cd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":1165,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6808","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6809","nonce":2184993002}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6810","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6811","nonce":2184993002}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6814","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6815","nonce":2184993002}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6812","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6813","nonce":2184993002}]},"public_addr":"172.21.15.129:6809/2184993002","cluster_addr":"172.21.15.129:6811/2184993002","heartbeat_back_addr":"172.21.15.129:6815/2184993002","heartbeat_front_addr":"172.21.15.129:6813/2184993002","state":["exists","up"]},{"osd":6,"uuid":"79e9c0a2-86a4-4267-a799-059d473d7b74","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":1165,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6816","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6817","nonce":3767031353}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6818","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6819","nonce":3767031353}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6822","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6823","nonce":3767031353}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6820","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6821","nonce":3767031353}]},"public_addr":"172.21.15.129:6817/3767031353","cluster_addr":"172.21.15.129:6819/3767031353","heartbeat_back_addr":"172.21.15.129:6823/3767031353","heartbeat_front_addr":"172.21.15.129:6821/3767031353","state":["exists","up"]},{"osd":7,"uuid":"22f41628-4b88-4b8c-82b2-fcd2802da1bf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":1165,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6824","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6825","nonce":2370414690}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6826","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6827","nonce":2370414690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6830","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6831","nonce":2370414690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6828","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6829","nonce":2370414690}]},"public_addr":"172.21.15.129:6825/2370414690","cluster_addr":"172.21.15.129:6827/2370414690","heartbeat_back_addr":"172.21.15.129:6831/2370414690","heartbeat_front_addr":"172.21.15.129:6829/2370414690","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:48:23.355518+0000","dead_epoch":666},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:48:48.715441+0000","dead_epoch":677},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:49:13.415323+0000","dead_epoch":687},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:49:39.935205+0000","dead_epoch":697},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:50:01.649042+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:50:24.001678+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:50:47.985011+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:51:11.584825+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.10","mappings":[{"from":1,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":5}]},{"pgid":"4.1f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.39:0/1864073859":"2024-09-14T05:12:12.455045+0000","172.21.15.39:0/1958359945":"2024-09-14T05:02:41.504544+0000","172.21.15.39:0/2766717202":"2024-09-14T05:02:40.464304+0000","172.21.15.39:0/1825825785":"2024-09-15T03:55:55.288048+0000","172.21.15.129:0/3885028850":"2024-09-15T03:56:26.175569+0000","172.21.15.129:6835/1529931471":"2024-09-15T03:56:26.175569+0000","172.21.15.39:0/822851906":"2024-09-15T03:59:06.707728+0000","172.21.15.39:6800/748677190":"2024-09-15T03:51:44.726801+0000","172.21.15.39:0/3247815370":"2024-09-15T03:59:06.707728+0000","172.21.15.129:0/78252352":"2024-09-15T03:56:26.175569+0000","172.21.15.39:0/1530637819":"2024-09-15T03:46:59.511580+0000","172.21.15.39:6801/628437919":"2024-09-15T03:46:35.805097+0000","172.21.15.39:6801/748677190":"2024-09-15T03:51:44.726801+0000","172.21.15.39:0/2745905728":"2024-09-15T03:51:44.726801+0000","172.21.15.129:0/2660542464":"2024-09-15T03:56:26.175569+0000","172.21.15.39:0/209353267":"2024-09-15T03:46:59.511580+0000","172.21.15.39:6800/3127470450":"2024-09-15T03:46:59.511580+0000","172.21.15.129:0/1000360058":"2024-09-15T03:56:26.175569+0000","172.21.15.39:0/2670041591":"2024-09-15T03:46:35.805097+0000","172.21.15.39:6801/3924974081":"2024-09-15T03:55:55.288048+0000","172.21.15.39:6800/3924974081":"2024-09-15T03:55:55.288048+0000","172.21.15.39:0/2951004223":"2024-09-15T03:51:44.726801+0000","172.21.15.39:0/3196173579":"2024-09-15T03:55:55.288048+0000","172.21.15.39:0/502495021":"2024-09-15T03:55:55.288048+0000","172.21.15.39:0/3161547781":"2024-09-15T03:46:59.511580+0000","172.21.15.39:0/1398136686":"2024-09-15T03:59:06.707728+0000","172.21.15.39:6801/3127470450":"2024-09-15T03:46:59.511580+0000","172.21.15.39:0/2133325947":"2024-09-15T03:46:35.805097+0000","172.21.15.39:6800/1103763596":"2024-09-15T03:59:06.707728+0000","172.21.15.129:6834/1529931471":"2024-09-15T03:56:26.175569+0000","172.21.15.39:0/1746545094":"2024-09-15T03:59:06.707728+0000","172.21.15.129:0/1993195691":"2024-09-15T03:56:26.175569+0000","172.21.15.39:6801/1103763596":"2024-09-15T03:59:06.707728+0000","172.21.15.39:0/754571920":"2024-09-15T03:51:44.726801+0000","172.21.15.39:0/81449629":"2024-09-15T03:46:35.805097+0000","172.21.15.39:6800/628437919":"2024-09-15T03:46:35.805097+0000","172.21.15.129:0/3477521861":"2024-09-15T03:56:26.175569+0000","172.21.15.39:0/2587896825":"2024-09-15T03:51:44.726801+0000","172.21.15.39:0/1766366412":"2024-09-15T03:55:55.288048+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":6,"snaps":[{"begin":2,"length":1}]},{"pool":7,"snaps":[{"begin":2,"length":1}]},{"pool":8,"snaps":[{"begin":2,"length":1}]},{"pool":9,"snaps":[{"begin":2,"length":1}]},{"pool":10,"snaps":[{"begin":2,"length":1}]},{"pool":11,"snaps":[{"begin":2,"length":1}]},{"pool":12,"snaps":[{"begin":233,"length":1}]},{"pool":14,"snaps":[{"begin":6,"length":3}]},{"pool":16,"snaps":[{"begin":35,"length":3},{"begin":45,"length":3}]},{"pool":17,"snaps":[{"begin":22,"length":1}]},{"pool":18,"snaps":[{"begin":2,"length":1}]},{"pool":51,"snaps":[{"begin":104,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T04:37:17.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:17 smithi129 ceph-mon[101316]: pgmap v2112: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:18.000 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-14T04:37:18.001 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-14T04:37:18.001 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-14T04:37:18.001 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-14T04:37:18.002 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-14T04:37:18.002 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-14T04:37:18.003 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-14T04:37:18.003 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-14T04:37:18.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:18 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1595801157' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T04:37:18.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:18 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1595801157' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T04:37:18.490 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:37:18.493 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:37:18.494 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:37:18.499 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:37:18.500 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:37:18.506 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:37:18.516 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:37:18.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:18 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1595801157' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T04:37:18.521 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:37:18.788 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:37:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:37:18] "GET /metrics HTTP/1.1" 200 34258 "" "Prometheus/2.51.0" 2024-09-14T04:37:19.375 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:19 smithi039 ceph-mon[106470]: pgmap v2113: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:19.375 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:19 smithi039 ceph-mon[111235]: pgmap v2113: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:19.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:19 smithi129 ceph-mon[101316]: pgmap v2113: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:21.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:21 smithi129 ceph-mon[101316]: pgmap v2114: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:21.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:21 smithi039 ceph-mon[106470]: pgmap v2114: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:21.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:21 smithi039 ceph-mon[111235]: pgmap v2114: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:23 smithi039 ceph-mon[106470]: pgmap v2115: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:37:23.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:23 smithi039 ceph-mon[111235]: pgmap v2115: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:23.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:37:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:23 smithi129 ceph-mon[101316]: pgmap v2115: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:37:25.364 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:25 smithi039 ceph-mon[106470]: pgmap v2116: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:25.365 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:25 smithi039 ceph-mon[111235]: pgmap v2116: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:25.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:25 smithi129 ceph-mon[101316]: pgmap v2116: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:27.110 INFO:teuthology.orchestra.run.smithi039.stdout:201863463473 2024-09-14T04:37:27.110 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-14T04:37:27.462 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:27 smithi039 ceph-mon[106470]: pgmap v2117: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:27.462 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:27 smithi039 ceph-mon[111235]: pgmap v2117: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:27.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:27 smithi129 ceph-mon[101316]: pgmap v2117: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:28.725 INFO:teuthology.orchestra.run.smithi039.stdout:2881923055996 2024-09-14T04:37:28.726 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-14T04:37:28.945 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:37:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:37:28] "GET /metrics HTTP/1.1" 200 34258 "" "Prometheus/2.51.0" 2024-09-14T04:37:29.216 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:29 smithi039 ceph-mon[106470]: pgmap v2118: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:29.217 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:29 smithi039 ceph-mon[111235]: pgmap v2118: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:29.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:29 smithi129 ceph-mon[101316]: pgmap v2118: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:31.467 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:31 smithi039 ceph-mon[106470]: pgmap v2119: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:31.468 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:31 smithi039 ceph-mon[111235]: pgmap v2119: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:31.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:31 smithi129 ceph-mon[101316]: pgmap v2119: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:32.107 INFO:teuthology.orchestra.run.smithi039.stdout:133143986749 2024-09-14T04:37:32.107 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-14T04:37:32.835 INFO:teuthology.orchestra.run.smithi039.stdout:167503725111 2024-09-14T04:37:32.835 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-14T04:37:33.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:33 smithi129 ceph-mon[101316]: pgmap v2120: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:33.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:33 smithi039 ceph-mon[106470]: pgmap v2120: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:33.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:33 smithi039 ceph-mon[111235]: pgmap v2120: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:33.892 INFO:teuthology.orchestra.run.smithi039.stdout:3010772074853 2024-09-14T04:37:33.893 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-14T04:37:34.519 INFO:teuthology.orchestra.run.smithi039.stdout:2924872728947 2024-09-14T04:37:34.520 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-14T04:37:35.282 INFO:teuthology.orchestra.run.smithi039.stdout:2980707303788 2024-09-14T04:37:35.283 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-14T04:37:35.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:35 smithi129 ceph-mon[101316]: pgmap v2121: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:35.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:35 smithi039 ceph-mon[106470]: pgmap v2121: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:35.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:35 smithi039 ceph-mon[111235]: pgmap v2121: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:35.600 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:37:36.178 INFO:teuthology.orchestra.run.smithi039.stdout:236223201839 2024-09-14T04:37:36.178 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-14T04:37:36.448 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:37:36.878 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:37:36.879 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:37:37.283 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:37 smithi039 ceph-mon[106470]: pgmap v2122: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:37.283 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:37 smithi039 ceph-mon[111235]: pgmap v2122: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:37.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:37 smithi129 ceph-mon[101316]: pgmap v2122: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:38.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:38 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:37:38.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:38 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:37:38.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:38 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:37:38.914 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:37:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:37:38] "GET /metrics HTTP/1.1" 200 34257 "" "Prometheus/2.51.0" 2024-09-14T04:37:39.455 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:39 smithi039 ceph-mon[106470]: pgmap v2123: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:39.455 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:39 smithi039 ceph-mon[111235]: pgmap v2123: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:39.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:39 smithi129 ceph-mon[101316]: pgmap v2123: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:39.861 INFO:teuthology.orchestra.run.smithi039.stdout:2924872728948 2024-09-14T04:37:40.119 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:37:40.119 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:37:40.378 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:40 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2823705982' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-14T04:37:40.378 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:40 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2823705982' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-14T04:37:40.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:40 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2823705982' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-14T04:37:40.822 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:37:41.223 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2924872728947 got 2924872728948 for osd.1 2024-09-14T04:37:41.224 DEBUG:teuthology.parallel:result is None 2024-09-14T04:37:41.470 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:41 smithi039 ceph-mon[106470]: pgmap v2124: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:41.470 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:41 smithi039 ceph-mon[111235]: pgmap v2124: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:41.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:41 smithi129 ceph-mon[101316]: pgmap v2124: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:43.245 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:43 smithi039 ceph-mon[106470]: pgmap v2125: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:43.245 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:43 smithi039 ceph-mon[111235]: pgmap v2125: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:43.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:43 smithi129 ceph-mon[101316]: pgmap v2125: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:43.535 INFO:teuthology.orchestra.run.smithi039.stdout:236223201840 2024-09-14T04:37:44.359 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:44 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/717492257' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-14T04:37:44.360 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:44 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/717492257' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-14T04:37:44.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:44 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/717492257' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-14T04:37:44.557 INFO:teuthology.orchestra.run.smithi039.stdout:201863463477 2024-09-14T04:37:45.250 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863463473 got 201863463477 for osd.6 2024-09-14T04:37:45.250 DEBUG:teuthology.parallel:result is None 2024-09-14T04:37:45.476 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:45 smithi039 ceph-mon[106470]: pgmap v2126: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:45.476 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:45 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3295001030' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-14T04:37:45.476 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:45 smithi039 ceph-mon[111235]: pgmap v2126: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:45.476 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:45 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3295001030' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-14T04:37:45.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:45 smithi129 ceph-mon[101316]: pgmap v2126: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:45.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:45 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3295001030' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-14T04:37:45.862 INFO:teuthology.orchestra.run.smithi039.stdout:2881923056000 2024-09-14T04:37:46.997 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:46 smithi039 ceph-mon[106470]: pgmap v2127: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:46.997 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:46 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4021970080' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-14T04:37:46.998 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:46 smithi039 ceph-mon[111235]: pgmap v2127: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:46.998 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:46 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4021970080' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-14T04:37:47.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:46 smithi129 ceph-mon[101316]: pgmap v2127: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:47.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:46 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4021970080' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-14T04:37:47.284 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201839 got 236223201840 for osd.7 2024-09-14T04:37:47.285 DEBUG:teuthology.parallel:result is None 2024-09-14T04:37:48.024 INFO:teuthology.orchestra.run.smithi039.stdout:133143986752 2024-09-14T04:37:48.318 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2881923055996 got 2881923056000 for osd.0 2024-09-14T04:37:48.318 DEBUG:teuthology.parallel:result is None 2024-09-14T04:37:48.337 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:37:49.036 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:48 smithi039 ceph-mon[106470]: pgmap v2128: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:49.036 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:48 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1691309880' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-14T04:37:49.036 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:37:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:37:48] "GET /metrics HTTP/1.1" 200 34257 "" "Prometheus/2.51.0" 2024-09-14T04:37:49.037 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:48 smithi039 ceph-mon[111235]: pgmap v2128: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:49.037 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:48 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1691309880' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-14T04:37:49.059 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986749 got 133143986752 for osd.4 2024-09-14T04:37:49.059 DEBUG:teuthology.parallel:result is None 2024-09-14T04:37:49.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:48 smithi129 ceph-mon[101316]: pgmap v2128: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:49.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:48 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1691309880' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-14T04:37:50.498 INFO:teuthology.orchestra.run.smithi039.stdout:2980707303791 2024-09-14T04:37:50.685 INFO:teuthology.orchestra.run.smithi039.stdout:167503725115 2024-09-14T04:37:51.012 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:50 smithi039 ceph-mon[106470]: pgmap v2129: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:51.013 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:50 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/312598350' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-14T04:37:51.013 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:50 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2640331769' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-14T04:37:51.013 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:50 smithi039 ceph-mon[111235]: pgmap v2129: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:51.013 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:50 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/312598350' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-14T04:37:51.013 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:50 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2640331769' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-14T04:37:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:50 smithi129 ceph-mon[101316]: pgmap v2129: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:50 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/312598350' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-14T04:37:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:50 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2640331769' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-14T04:37:51.317 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503725111 got 167503725115 for osd.5 2024-09-14T04:37:51.318 DEBUG:teuthology.parallel:result is None 2024-09-14T04:37:51.754 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2980707303788 got 2980707303791 for osd.2 2024-09-14T04:37:51.754 DEBUG:teuthology.parallel:result is None 2024-09-14T04:37:52.990 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:52 smithi039 ceph-mon[106470]: pgmap v2130: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:52.990 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:37:52.990 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:52 smithi039 ceph-mon[111235]: pgmap v2130: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:52.990 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:37:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:52 smithi129 ceph-mon[101316]: pgmap v2130: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:37:53.399 INFO:teuthology.orchestra.run.smithi039.stdout:3010772074856 2024-09-14T04:37:54.020 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:53 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2346768293' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-14T04:37:54.021 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:53 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2346768293' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-14T04:37:54.043 INFO:tasks.cephadm.ceph_manager.ceph:need seq 3010772074853 got 3010772074856 for osd.3 2024-09-14T04:37:54.044 DEBUG:teuthology.parallel:result is None 2024-09-14T04:37:54.044 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-14T04:37:54.044 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:37:54.252 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:37:54.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:53 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2346768293' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-14T04:37:55.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:54 smithi129 ceph-mon[101316]: pgmap v2131: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:55.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:54 smithi039 ceph-mon[111235]: pgmap v2131: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:55.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:54 smithi039 ceph-mon[106470]: pgmap v2131: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:56.174 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:37:56.175 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T04:37:56.795 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":2132,"stamp":"2024-09-14T04:37:55.478601+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":279,"num_write_kb":2449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":208,"ondisk_log_size":208,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":832152,"kb_used_data":117816,"kb_used_omap":288,"kb_used_meta":713951,"kb_avail":749129064,"statfs":{"total":767960285184,"available":767108161536,"internally_reserved":0,"allocated":120643584,"data_stored":76587607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":294978,"internal_metadata":731086782},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003155"},"pg_stats":[{"pgid":"3.1f","version":"72'4","reported_seq":1910,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.029243+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T04:37:07.029243+0000","last_peered":"2024-09-14T04:37:07.029243+0000","last_clean":"2024-09-14T04:37:07.029243+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T04:37:07.029243+0000","last_undegraded":"2024-09-14T04:37:07.029243+0000","last_fullsized":"2024-09-14T04:37:07.029243+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1274,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116018+0000","last_change":"2024-09-14T03:53:45.836449+0000","last_active":"2024-09-14T04:37:06.116018+0000","last_peered":"2024-09-14T04:37:06.116018+0000","last_clean":"2024-09-14T04:37:06.116018+0000","last_became_active":"2024-09-14T03:53:45.836108+0000","last_became_peered":"2024-09-14T03:53:45.836108+0000","last_unstale":"2024-09-14T04:37:06.116018+0000","last_undegraded":"2024-09-14T04:37:06.116018+0000","last_fullsized":"2024-09-14T04:37:06.116018+0000","mapping_epoch":73,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":74,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:52:13.095225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1899,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.835281+0000","last_change":"2024-09-14T04:07:58.504829+0000","last_active":"2024-09-14T04:37:07.835281+0000","last_peered":"2024-09-14T04:37:07.835281+0000","last_clean":"2024-09-14T04:37:07.835281+0000","last_became_active":"2024-09-14T04:07:58.502623+0000","last_became_peered":"2024-09-14T04:07:58.502623+0000","last_unstale":"2024-09-14T04:37:07.835281+0000","last_undegraded":"2024-09-14T04:37:07.835281+0000","last_fullsized":"2024-09-14T04:37:07.835281+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:59.091993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1911,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.833882+0000","last_change":"2024-09-14T04:07:58.507125+0000","last_active":"2024-09-14T04:37:07.833882+0000","last_peered":"2024-09-14T04:37:07.833882+0000","last_clean":"2024-09-14T04:37:07.833882+0000","last_became_active":"2024-09-14T04:07:58.505701+0000","last_became_peered":"2024-09-14T04:07:58.505701+0000","last_unstale":"2024-09-14T04:37:07.833882+0000","last_undegraded":"2024-09-14T04:37:07.833882+0000","last_fullsized":"2024-09-14T04:37:07.833882+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:25:00.847961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1264,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.843540+0000","last_change":"2024-09-14T03:53:05.234444+0000","last_active":"2024-09-14T04:37:05.843540+0000","last_peered":"2024-09-14T04:37:05.843540+0000","last_clean":"2024-09-14T04:37:05.843540+0000","last_became_active":"2024-09-14T03:53:05.234115+0000","last_became_peered":"2024-09-14T03:53:05.234115+0000","last_unstale":"2024-09-14T04:37:05.843540+0000","last_undegraded":"2024-09-14T04:37:05.843540+0000","last_fullsized":"2024-09-14T04:37:05.843540+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:01:23.153937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1294,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116964+0000","last_change":"2024-09-14T04:07:58.547135+0000","last_active":"2024-09-14T04:37:06.116964+0000","last_peered":"2024-09-14T04:37:06.116964+0000","last_clean":"2024-09-14T04:37:06.116964+0000","last_became_active":"2024-09-14T04:07:58.546751+0000","last_became_peered":"2024-09-14T04:07:58.546751+0000","last_unstale":"2024-09-14T04:37:06.116964+0000","last_undegraded":"2024-09-14T04:37:06.116964+0000","last_fullsized":"2024-09-14T04:37:06.116964+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:01.092698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1279,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.843531+0000","last_change":"2024-09-14T04:06:46.875795+0000","last_active":"2024-09-14T04:37:05.843531+0000","last_peered":"2024-09-14T04:37:05.843531+0000","last_clean":"2024-09-14T04:37:05.843531+0000","last_became_active":"2024-09-14T04:06:46.875395+0000","last_became_peered":"2024-09-14T04:06:46.875395+0000","last_unstale":"2024-09-14T04:37:05.843531+0000","last_undegraded":"2024-09-14T04:37:05.843531+0000","last_fullsized":"2024-09-14T04:37:05.843531+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:51:10.713063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1294,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116345+0000","last_change":"2024-09-14T04:07:58.514349+0000","last_active":"2024-09-14T04:37:06.116345+0000","last_peered":"2024-09-14T04:37:06.116345+0000","last_clean":"2024-09-14T04:37:06.116345+0000","last_became_active":"2024-09-14T04:07:58.513485+0000","last_became_peered":"2024-09-14T04:07:58.513485+0000","last_unstale":"2024-09-14T04:37:06.116345+0000","last_undegraded":"2024-09-14T04:37:06.116345+0000","last_fullsized":"2024-09-14T04:37:06.116345+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:33:09.031813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":1892,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.029098+0000","last_change":"2024-09-14T04:05:54.428057+0000","last_active":"2024-09-14T04:37:07.029098+0000","last_peered":"2024-09-14T04:37:07.029098+0000","last_clean":"2024-09-14T04:37:07.029098+0000","last_became_active":"2024-09-14T04:05:54.414895+0000","last_became_peered":"2024-09-14T04:05:54.414895+0000","last_unstale":"2024-09-14T04:37:07.029098+0000","last_undegraded":"2024-09-14T04:37:07.029098+0000","last_fullsized":"2024-09-14T04:37:07.029098+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:45:15.752675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1907,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.028294+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T04:37:07.028294+0000","last_peered":"2024-09-14T04:37:07.028294+0000","last_clean":"2024-09-14T04:37:07.028294+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T04:37:07.028294+0000","last_undegraded":"2024-09-14T04:37:07.028294+0000","last_fullsized":"2024-09-14T04:37:07.028294+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1296,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116813+0000","last_change":"2024-09-14T04:07:21.717071+0000","last_active":"2024-09-14T04:37:06.116813+0000","last_peered":"2024-09-14T04:37:06.116813+0000","last_clean":"2024-09-14T04:37:06.116813+0000","last_became_active":"2024-09-14T04:07:21.716215+0000","last_became_peered":"2024-09-14T04:07:21.716215+0000","last_unstale":"2024-09-14T04:37:06.116813+0000","last_undegraded":"2024-09-14T04:37:06.116813+0000","last_fullsized":"2024-09-14T04:37:06.116813+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:48:47.175490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1879,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.834536+0000","last_change":"2024-09-14T04:07:58.524371+0000","last_active":"2024-09-14T04:37:07.834536+0000","last_peered":"2024-09-14T04:37:07.834536+0000","last_clean":"2024-09-14T04:37:07.834536+0000","last_became_active":"2024-09-14T04:07:58.524088+0000","last_became_peered":"2024-09-14T04:07:58.524088+0000","last_unstale":"2024-09-14T04:37:07.834536+0000","last_undegraded":"2024-09-14T04:37:07.834536+0000","last_fullsized":"2024-09-14T04:37:07.834536+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:42.615569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1874,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.180210+0000","last_change":"2024-09-14T04:07:58.507553+0000","last_active":"2024-09-14T04:37:07.180210+0000","last_peered":"2024-09-14T04:37:07.180210+0000","last_clean":"2024-09-14T04:37:07.180210+0000","last_became_active":"2024-09-14T04:07:58.507139+0000","last_became_peered":"2024-09-14T04:07:58.507139+0000","last_unstale":"2024-09-14T04:37:07.180210+0000","last_undegraded":"2024-09-14T04:37:07.180210+0000","last_fullsized":"2024-09-14T04:37:07.180210+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:11:57.821769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1904,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.028779+0000","last_change":"2024-09-14T04:07:58.561490+0000","last_active":"2024-09-14T04:37:07.028779+0000","last_peered":"2024-09-14T04:37:07.028779+0000","last_clean":"2024-09-14T04:37:07.028779+0000","last_became_active":"2024-09-14T04:07:58.559841+0000","last_became_peered":"2024-09-14T04:07:58.559841+0000","last_unstale":"2024-09-14T04:37:07.028779+0000","last_undegraded":"2024-09-14T04:37:07.028779+0000","last_fullsized":"2024-09-14T04:37:07.028779+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:27:03.795368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1896,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.833696+0000","last_change":"2024-09-14T04:07:58.555731+0000","last_active":"2024-09-14T04:37:07.833696+0000","last_peered":"2024-09-14T04:37:07.833696+0000","last_clean":"2024-09-14T04:37:07.833696+0000","last_became_active":"2024-09-14T04:07:58.549576+0000","last_became_peered":"2024-09-14T04:07:58.549576+0000","last_unstale":"2024-09-14T04:37:07.833696+0000","last_undegraded":"2024-09-14T04:37:07.833696+0000","last_fullsized":"2024-09-14T04:37:07.833696+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:28:42.339589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1288,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.841667+0000","last_change":"2024-09-14T04:07:21.720797+0000","last_active":"2024-09-14T04:37:05.841667+0000","last_peered":"2024-09-14T04:37:05.841667+0000","last_clean":"2024-09-14T04:37:05.841667+0000","last_became_active":"2024-09-14T04:07:21.719331+0000","last_became_peered":"2024-09-14T04:07:21.719331+0000","last_unstale":"2024-09-14T04:37:05.841667+0000","last_undegraded":"2024-09-14T04:37:05.841667+0000","last_fullsized":"2024-09-14T04:37:05.841667+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:35.404023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1905,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.833897+0000","last_change":"2024-09-14T04:07:58.502533+0000","last_active":"2024-09-14T04:37:07.833897+0000","last_peered":"2024-09-14T04:37:07.833897+0000","last_clean":"2024-09-14T04:37:07.833897+0000","last_became_active":"2024-09-14T04:07:58.499849+0000","last_became_peered":"2024-09-14T04:07:58.499849+0000","last_unstale":"2024-09-14T04:37:07.833897+0000","last_undegraded":"2024-09-14T04:37:07.833897+0000","last_fullsized":"2024-09-14T04:37:07.833897+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:15:35.609746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1900,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.029481+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T04:37:07.029481+0000","last_peered":"2024-09-14T04:37:07.029481+0000","last_clean":"2024-09-14T04:37:07.029481+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T04:37:07.029481+0000","last_undegraded":"2024-09-14T04:37:07.029481+0000","last_fullsized":"2024-09-14T04:37:07.029481+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1881,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.835001+0000","last_change":"2024-09-14T04:07:58.507682+0000","last_active":"2024-09-14T04:37:07.835001+0000","last_peered":"2024-09-14T04:37:07.835001+0000","last_clean":"2024-09-14T04:37:07.835001+0000","last_became_active":"2024-09-14T04:07:58.507365+0000","last_became_peered":"2024-09-14T04:07:58.507365+0000","last_unstale":"2024-09-14T04:37:07.835001+0000","last_undegraded":"2024-09-14T04:37:07.835001+0000","last_fullsized":"2024-09-14T04:37:07.835001+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:14:47.294950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1277,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.843922+0000","last_change":"2024-09-14T04:06:46.875668+0000","last_active":"2024-09-14T04:37:05.843922+0000","last_peered":"2024-09-14T04:37:05.843922+0000","last_clean":"2024-09-14T04:37:05.843922+0000","last_became_active":"2024-09-14T04:06:46.875149+0000","last_became_peered":"2024-09-14T04:06:46.875149+0000","last_unstale":"2024-09-14T04:37:05.843922+0000","last_undegraded":"2024-09-14T04:37:05.843922+0000","last_fullsized":"2024-09-14T04:37:05.843922+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:45:59.190047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1278,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.118067+0000","last_change":"2024-09-14T04:07:58.510699+0000","last_active":"2024-09-14T04:37:06.118067+0000","last_peered":"2024-09-14T04:37:06.118067+0000","last_clean":"2024-09-14T04:37:06.118067+0000","last_became_active":"2024-09-14T04:07:58.506240+0000","last_became_peered":"2024-09-14T04:07:58.506240+0000","last_unstale":"2024-09-14T04:37:06.118067+0000","last_undegraded":"2024-09-14T04:37:06.118067+0000","last_fullsized":"2024-09-14T04:37:06.118067+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:27:18.051658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1907,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.181564+0000","last_change":"2024-09-14T04:07:21.687985+0000","last_active":"2024-09-14T04:37:07.181564+0000","last_peered":"2024-09-14T04:37:07.181564+0000","last_clean":"2024-09-14T04:37:07.181564+0000","last_became_active":"2024-09-14T04:07:21.687670+0000","last_became_peered":"2024-09-14T04:07:21.687670+0000","last_unstale":"2024-09-14T04:37:07.181564+0000","last_undegraded":"2024-09-14T04:37:07.181564+0000","last_fullsized":"2024-09-14T04:37:07.181564+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:53:58.295662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":1290,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:08.762974+0000","last_change":"2024-09-14T03:53:05.244055+0000","last_active":"2024-09-14T04:37:08.762974+0000","last_peered":"2024-09-14T04:37:08.762974+0000","last_clean":"2024-09-14T04:37:08.762974+0000","last_became_active":"2024-09-14T03:53:05.243752+0000","last_became_peered":"2024-09-14T03:53:05.243752+0000","last_unstale":"2024-09-14T04:37:08.762974+0000","last_undegraded":"2024-09-14T04:37:08.762974+0000","last_fullsized":"2024-09-14T04:37:08.762974+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:12:31.850064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1281,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:08.763591+0000","last_change":"2024-09-14T04:06:46.895670+0000","last_active":"2024-09-14T04:37:08.763591+0000","last_peered":"2024-09-14T04:37:08.763591+0000","last_clean":"2024-09-14T04:37:08.763591+0000","last_became_active":"2024-09-14T04:06:46.895370+0000","last_became_peered":"2024-09-14T04:06:46.895370+0000","last_unstale":"2024-09-14T04:37:08.763591+0000","last_undegraded":"2024-09-14T04:37:08.763591+0000","last_fullsized":"2024-09-14T04:37:08.763591+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:48:48.040529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1268,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116350+0000","last_change":"2024-09-14T03:53:06.634043+0000","last_active":"2024-09-14T04:37:06.116350+0000","last_peered":"2024-09-14T04:37:06.116350+0000","last_clean":"2024-09-14T04:37:06.116350+0000","last_became_active":"2024-09-14T03:53:06.633638+0000","last_became_peered":"2024-09-14T03:53:06.633638+0000","last_unstale":"2024-09-14T04:37:06.116350+0000","last_undegraded":"2024-09-14T04:37:06.116350+0000","last_fullsized":"2024-09-14T04:37:06.116350+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:15:53.434064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1292,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:08.946361+0000","last_change":"2024-09-14T04:07:21.721242+0000","last_active":"2024-09-14T04:37:08.946361+0000","last_peered":"2024-09-14T04:37:08.946361+0000","last_clean":"2024-09-14T04:37:08.946361+0000","last_became_active":"2024-09-14T04:07:21.720084+0000","last_became_peered":"2024-09-14T04:07:21.720084+0000","last_unstale":"2024-09-14T04:37:08.946361+0000","last_undegraded":"2024-09-14T04:37:08.946361+0000","last_fullsized":"2024-09-14T04:37:08.946361+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:58:41.308643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":1906,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.028465+0000","last_change":"2024-09-14T04:06:46.932567+0000","last_active":"2024-09-14T04:37:07.028465+0000","last_peered":"2024-09-14T04:37:07.028465+0000","last_clean":"2024-09-14T04:37:07.028465+0000","last_became_active":"2024-09-14T04:06:46.930732+0000","last_became_peered":"2024-09-14T04:06:46.930732+0000","last_unstale":"2024-09-14T04:37:07.028465+0000","last_undegraded":"2024-09-14T04:37:07.028465+0000","last_fullsized":"2024-09-14T04:37:07.028465+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:10:02.729499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1873,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.833825+0000","last_change":"2024-09-14T04:07:58.555446+0000","last_active":"2024-09-14T04:37:07.833825+0000","last_peered":"2024-09-14T04:37:07.833825+0000","last_clean":"2024-09-14T04:37:07.833825+0000","last_became_active":"2024-09-14T04:07:58.548355+0000","last_became_peered":"2024-09-14T04:07:58.548355+0000","last_unstale":"2024-09-14T04:37:07.833825+0000","last_undegraded":"2024-09-14T04:37:07.833825+0000","last_fullsized":"2024-09-14T04:37:07.833825+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:56:28.426094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1911,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.180789+0000","last_change":"2024-09-14T04:06:46.915273+0000","last_active":"2024-09-14T04:37:07.180789+0000","last_peered":"2024-09-14T04:37:07.180789+0000","last_clean":"2024-09-14T04:37:07.180789+0000","last_became_active":"2024-09-14T04:06:46.914994+0000","last_became_peered":"2024-09-14T04:06:46.914994+0000","last_unstale":"2024-09-14T04:37:07.180789+0000","last_undegraded":"2024-09-14T04:37:07.180789+0000","last_fullsized":"2024-09-14T04:37:07.180789+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:48:58.802609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1874,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.835129+0000","last_change":"2024-09-14T04:07:58.564377+0000","last_active":"2024-09-14T04:37:07.835129+0000","last_peered":"2024-09-14T04:37:07.835129+0000","last_clean":"2024-09-14T04:37:07.835129+0000","last_became_active":"2024-09-14T04:07:58.546913+0000","last_became_peered":"2024-09-14T04:07:58.546913+0000","last_unstale":"2024-09-14T04:37:07.835129+0000","last_undegraded":"2024-09-14T04:37:07.835129+0000","last_fullsized":"2024-09-14T04:37:07.835129+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:05:53.637115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1893,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.029265+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T04:37:07.029265+0000","last_peered":"2024-09-14T04:37:07.029265+0000","last_clean":"2024-09-14T04:37:07.029265+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T04:37:07.029265+0000","last_undegraded":"2024-09-14T04:37:07.029265+0000","last_fullsized":"2024-09-14T04:37:07.029265+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1291,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.846686+0000","last_change":"2024-09-14T04:05:54.381035+0000","last_active":"2024-09-14T04:37:07.846686+0000","last_peered":"2024-09-14T04:37:07.846686+0000","last_clean":"2024-09-14T04:37:07.846686+0000","last_became_active":"2024-09-14T04:05:54.379861+0000","last_became_peered":"2024-09-14T04:05:54.379861+0000","last_unstale":"2024-09-14T04:37:07.846686+0000","last_undegraded":"2024-09-14T04:37:07.846686+0000","last_fullsized":"2024-09-14T04:37:07.846686+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:07:46.302925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1910,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.182302+0000","last_change":"2024-09-14T04:07:21.686880+0000","last_active":"2024-09-14T04:37:07.182302+0000","last_peered":"2024-09-14T04:37:07.182302+0000","last_clean":"2024-09-14T04:37:07.182302+0000","last_became_active":"2024-09-14T04:07:21.685558+0000","last_became_peered":"2024-09-14T04:07:21.685558+0000","last_unstale":"2024-09-14T04:37:07.182302+0000","last_undegraded":"2024-09-14T04:37:07.182302+0000","last_fullsized":"2024-09-14T04:37:07.182302+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:24:18.693428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1898,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.181385+0000","last_change":"2024-09-14T04:06:46.905931+0000","last_active":"2024-09-14T04:37:07.181385+0000","last_peered":"2024-09-14T04:37:07.181385+0000","last_clean":"2024-09-14T04:37:07.181385+0000","last_became_active":"2024-09-14T04:06:46.905666+0000","last_became_peered":"2024-09-14T04:06:46.905666+0000","last_unstale":"2024-09-14T04:37:07.181385+0000","last_undegraded":"2024-09-14T04:37:07.181385+0000","last_fullsized":"2024-09-14T04:37:07.181385+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:40:57.007948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1927,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.182411+0000","last_change":"2024-09-14T04:06:46.930482+0000","last_active":"2024-09-14T04:37:07.182411+0000","last_peered":"2024-09-14T04:37:07.182411+0000","last_clean":"2024-09-14T04:37:07.182411+0000","last_became_active":"2024-09-14T04:06:46.927372+0000","last_became_peered":"2024-09-14T04:06:46.927372+0000","last_unstale":"2024-09-14T04:37:07.182411+0000","last_undegraded":"2024-09-14T04:37:07.182411+0000","last_fullsized":"2024-09-14T04:37:07.182411+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:07:22.159128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1296,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.843694+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T04:37:05.843694+0000","last_peered":"2024-09-14T04:37:05.843694+0000","last_clean":"2024-09-14T04:37:05.843694+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T04:37:05.843694+0000","last_undegraded":"2024-09-14T04:37:05.843694+0000","last_fullsized":"2024-09-14T04:37:05.843694+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1920,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.182400+0000","last_change":"2024-09-14T04:07:58.510721+0000","last_active":"2024-09-14T04:37:07.182400+0000","last_peered":"2024-09-14T04:37:07.182400+0000","last_clean":"2024-09-14T04:37:07.182400+0000","last_became_active":"2024-09-14T04:07:58.510332+0000","last_became_peered":"2024-09-14T04:07:58.510332+0000","last_unstale":"2024-09-14T04:37:07.182400+0000","last_undegraded":"2024-09-14T04:37:07.182400+0000","last_fullsized":"2024-09-14T04:37:07.182400+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:07:03.926842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1284,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116506+0000","last_change":"2024-09-14T04:05:54.426939+0000","last_active":"2024-09-14T04:37:06.116506+0000","last_peered":"2024-09-14T04:37:06.116506+0000","last_clean":"2024-09-14T04:37:06.116506+0000","last_became_active":"2024-09-14T04:05:54.426427+0000","last_became_peered":"2024-09-14T04:05:54.426427+0000","last_unstale":"2024-09-14T04:37:06.116506+0000","last_undegraded":"2024-09-14T04:37:06.116506+0000","last_fullsized":"2024-09-14T04:37:06.116506+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:53:55.841489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1295,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:08.602276+0000","last_change":"2024-09-14T04:06:46.875340+0000","last_active":"2024-09-14T04:37:08.602276+0000","last_peered":"2024-09-14T04:37:08.602276+0000","last_clean":"2024-09-14T04:37:08.602276+0000","last_became_active":"2024-09-14T04:06:46.874552+0000","last_became_peered":"2024-09-14T04:06:46.874552+0000","last_unstale":"2024-09-14T04:37:08.602276+0000","last_undegraded":"2024-09-14T04:37:08.602276+0000","last_fullsized":"2024-09-14T04:37:08.602276+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:52.257357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1907,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.180965+0000","last_change":"2024-09-14T04:06:46.917681+0000","last_active":"2024-09-14T04:37:07.180965+0000","last_peered":"2024-09-14T04:37:07.180965+0000","last_clean":"2024-09-14T04:37:07.180965+0000","last_became_active":"2024-09-14T04:06:46.917401+0000","last_became_peered":"2024-09-14T04:06:46.917401+0000","last_unstale":"2024-09-14T04:37:07.180965+0000","last_undegraded":"2024-09-14T04:37:07.180965+0000","last_fullsized":"2024-09-14T04:37:07.180965+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:04:39.949490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1911,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.180136+0000","last_change":"2024-09-14T04:07:21.685415+0000","last_active":"2024-09-14T04:37:07.180136+0000","last_peered":"2024-09-14T04:37:07.180136+0000","last_clean":"2024-09-14T04:37:07.180136+0000","last_became_active":"2024-09-14T04:07:21.683504+0000","last_became_peered":"2024-09-14T04:07:21.683504+0000","last_unstale":"2024-09-14T04:37:07.180136+0000","last_undegraded":"2024-09-14T04:37:07.180136+0000","last_fullsized":"2024-09-14T04:37:07.180136+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:21:08.806291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1913,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.027850+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T04:37:07.027850+0000","last_peered":"2024-09-14T04:37:07.027850+0000","last_clean":"2024-09-14T04:37:07.027850+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T04:37:07.027850+0000","last_undegraded":"2024-09-14T04:37:07.027850+0000","last_fullsized":"2024-09-14T04:37:07.027850+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1914,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.028872+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T04:37:07.028872+0000","last_peered":"2024-09-14T04:37:07.028872+0000","last_clean":"2024-09-14T04:37:07.028872+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T04:37:07.028872+0000","last_undegraded":"2024-09-14T04:37:07.028872+0000","last_fullsized":"2024-09-14T04:37:07.028872+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1903,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.028102+0000","last_change":"2024-09-14T04:07:58.521344+0000","last_active":"2024-09-14T04:37:07.028102+0000","last_peered":"2024-09-14T04:37:07.028102+0000","last_clean":"2024-09-14T04:37:07.028102+0000","last_became_active":"2024-09-14T04:07:58.520965+0000","last_became_peered":"2024-09-14T04:07:58.520965+0000","last_unstale":"2024-09-14T04:37:07.028102+0000","last_undegraded":"2024-09-14T04:37:07.028102+0000","last_fullsized":"2024-09-14T04:37:07.028102+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:58:46.029013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1306,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.117356+0000","last_change":"2024-09-14T04:06:46.857436+0000","last_active":"2024-09-14T04:37:06.117356+0000","last_peered":"2024-09-14T04:37:06.117356+0000","last_clean":"2024-09-14T04:37:06.117356+0000","last_became_active":"2024-09-14T04:06:46.857154+0000","last_became_peered":"2024-09-14T04:06:46.857154+0000","last_unstale":"2024-09-14T04:37:06.117356+0000","last_undegraded":"2024-09-14T04:37:06.117356+0000","last_fullsized":"2024-09-14T04:37:06.117356+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:53:28.772152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1264,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.841559+0000","last_change":"2024-09-14T03:53:05.551268+0000","last_active":"2024-09-14T04:37:05.841559+0000","last_peered":"2024-09-14T04:37:05.841559+0000","last_clean":"2024-09-14T04:37:05.841559+0000","last_became_active":"2024-09-14T03:53:05.551015+0000","last_became_peered":"2024-09-14T03:53:05.551015+0000","last_unstale":"2024-09-14T04:37:05.841559+0000","last_undegraded":"2024-09-14T04:37:05.841559+0000","last_fullsized":"2024-09-14T04:37:05.841559+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.905475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1885,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.834431+0000","last_change":"2024-09-14T04:07:58.503896+0000","last_active":"2024-09-14T04:37:07.834431+0000","last_peered":"2024-09-14T04:37:07.834431+0000","last_clean":"2024-09-14T04:37:07.834431+0000","last_became_active":"2024-09-14T04:07:58.499253+0000","last_became_peered":"2024-09-14T04:07:58.499253+0000","last_unstale":"2024-09-14T04:37:07.834431+0000","last_undegraded":"2024-09-14T04:37:07.834431+0000","last_fullsized":"2024-09-14T04:37:07.834431+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:48:27.543818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1294,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:08.602592+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T04:37:08.602592+0000","last_peered":"2024-09-14T04:37:08.602592+0000","last_clean":"2024-09-14T04:37:08.602592+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T04:37:08.602592+0000","last_undegraded":"2024-09-14T04:37:08.602592+0000","last_fullsized":"2024-09-14T04:37:08.602592+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":1609,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.845285+0000","last_change":"2024-09-14T04:05:54.389043+0000","last_active":"2024-09-14T04:37:07.845285+0000","last_peered":"2024-09-14T04:37:07.845285+0000","last_clean":"2024-09-14T04:37:07.845285+0000","last_became_active":"2024-09-14T04:05:54.388655+0000","last_became_peered":"2024-09-14T04:05:54.388655+0000","last_unstale":"2024-09-14T04:37:07.845285+0000","last_undegraded":"2024-09-14T04:37:07.845285+0000","last_fullsized":"2024-09-14T04:37:07.845285+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T03:49:18.237706+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:38:59.615597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1875,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.180822+0000","last_change":"2024-09-14T04:07:58.530406+0000","last_active":"2024-09-14T04:37:07.180822+0000","last_peered":"2024-09-14T04:37:07.180822+0000","last_clean":"2024-09-14T04:37:07.180822+0000","last_became_active":"2024-09-14T04:07:58.530078+0000","last_became_peered":"2024-09-14T04:07:58.530078+0000","last_unstale":"2024-09-14T04:37:07.180822+0000","last_undegraded":"2024-09-14T04:37:07.180822+0000","last_fullsized":"2024-09-14T04:37:07.180822+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:54:53.524001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1883,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.833696+0000","last_change":"2024-09-14T04:07:58.555717+0000","last_active":"2024-09-14T04:37:07.833696+0000","last_peered":"2024-09-14T04:37:07.833696+0000","last_clean":"2024-09-14T04:37:07.833696+0000","last_became_active":"2024-09-14T04:07:58.548894+0000","last_became_peered":"2024-09-14T04:07:58.548894+0000","last_unstale":"2024-09-14T04:37:07.833696+0000","last_undegraded":"2024-09-14T04:37:07.833696+0000","last_fullsized":"2024-09-14T04:37:07.833696+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:42.061551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1281,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116836+0000","last_change":"2024-09-14T04:06:46.894741+0000","last_active":"2024-09-14T04:37:06.116836+0000","last_peered":"2024-09-14T04:37:06.116836+0000","last_clean":"2024-09-14T04:37:06.116836+0000","last_became_active":"2024-09-14T04:06:46.894445+0000","last_became_peered":"2024-09-14T04:06:46.894445+0000","last_unstale":"2024-09-14T04:37:06.116836+0000","last_undegraded":"2024-09-14T04:37:06.116836+0000","last_fullsized":"2024-09-14T04:37:06.116836+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:52:00.055271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1283,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116551+0000","last_change":"2024-09-14T04:07:21.715462+0000","last_active":"2024-09-14T04:37:06.116551+0000","last_peered":"2024-09-14T04:37:06.116551+0000","last_clean":"2024-09-14T04:37:06.116551+0000","last_became_active":"2024-09-14T04:07:21.713032+0000","last_became_peered":"2024-09-14T04:07:21.713032+0000","last_unstale":"2024-09-14T04:37:06.116551+0000","last_undegraded":"2024-09-14T04:37:06.116551+0000","last_fullsized":"2024-09-14T04:37:06.116551+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:37:05.545164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1296,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116624+0000","last_change":"2024-09-14T04:07:21.718037+0000","last_active":"2024-09-14T04:37:06.116624+0000","last_peered":"2024-09-14T04:37:06.116624+0000","last_clean":"2024-09-14T04:37:06.116624+0000","last_became_active":"2024-09-14T04:07:21.717700+0000","last_became_peered":"2024-09-14T04:07:21.717700+0000","last_unstale":"2024-09-14T04:37:06.116624+0000","last_undegraded":"2024-09-14T04:37:06.116624+0000","last_fullsized":"2024-09-14T04:37:06.116624+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:47:33.483785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1279,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.841296+0000","last_change":"2024-09-14T04:06:46.868780+0000","last_active":"2024-09-14T04:37:05.841296+0000","last_peered":"2024-09-14T04:37:05.841296+0000","last_clean":"2024-09-14T04:37:05.841296+0000","last_became_active":"2024-09-14T04:06:46.868483+0000","last_became_peered":"2024-09-14T04:06:46.868483+0000","last_unstale":"2024-09-14T04:37:05.841296+0000","last_undegraded":"2024-09-14T04:37:05.841296+0000","last_fullsized":"2024-09-14T04:37:05.841296+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:03.133158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1281,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.117264+0000","last_change":"2024-09-14T04:07:58.515859+0000","last_active":"2024-09-14T04:37:06.117264+0000","last_peered":"2024-09-14T04:37:06.117264+0000","last_clean":"2024-09-14T04:37:06.117264+0000","last_became_active":"2024-09-14T04:07:58.515594+0000","last_became_peered":"2024-09-14T04:07:58.515594+0000","last_unstale":"2024-09-14T04:37:06.117264+0000","last_undegraded":"2024-09-14T04:37:06.117264+0000","last_fullsized":"2024-09-14T04:37:06.117264+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:45.966798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":1900,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.833729+0000","last_change":"2024-09-14T04:07:58.509833+0000","last_active":"2024-09-14T04:37:07.833729+0000","last_peered":"2024-09-14T04:37:07.833729+0000","last_clean":"2024-09-14T04:37:07.833729+0000","last_became_active":"2024-09-14T04:07:58.509198+0000","last_became_peered":"2024-09-14T04:07:58.509198+0000","last_unstale":"2024-09-14T04:37:07.833729+0000","last_undegraded":"2024-09-14T04:37:07.833729+0000","last_fullsized":"2024-09-14T04:37:07.833729+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:24:40.193090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1903,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.028840+0000","last_change":"2024-09-14T04:06:46.925323+0000","last_active":"2024-09-14T04:37:07.028840+0000","last_peered":"2024-09-14T04:37:07.028840+0000","last_clean":"2024-09-14T04:37:07.028840+0000","last_became_active":"2024-09-14T04:06:46.925043+0000","last_became_peered":"2024-09-14T04:06:46.925043+0000","last_unstale":"2024-09-14T04:37:07.028840+0000","last_undegraded":"2024-09-14T04:37:07.028840+0000","last_fullsized":"2024-09-14T04:37:07.028840+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:39.895453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1279,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.843564+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T04:37:05.843564+0000","last_peered":"2024-09-14T04:37:05.843564+0000","last_clean":"2024-09-14T04:37:05.843564+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T04:37:05.843564+0000","last_undegraded":"2024-09-14T04:37:05.843564+0000","last_fullsized":"2024-09-14T04:37:05.843564+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1278,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.841209+0000","last_change":"2024-09-14T04:06:46.869868+0000","last_active":"2024-09-14T04:37:05.841209+0000","last_peered":"2024-09-14T04:37:05.841209+0000","last_clean":"2024-09-14T04:37:05.841209+0000","last_became_active":"2024-09-14T04:06:46.869126+0000","last_became_peered":"2024-09-14T04:06:46.869126+0000","last_unstale":"2024-09-14T04:37:05.841209+0000","last_undegraded":"2024-09-14T04:37:05.841209+0000","last_fullsized":"2024-09-14T04:37:05.841209+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:41.150138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1262,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.118225+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T04:37:06.118225+0000","last_peered":"2024-09-14T04:37:06.118225+0000","last_clean":"2024-09-14T04:37:06.118225+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T04:37:06.118225+0000","last_undegraded":"2024-09-14T04:37:06.118225+0000","last_fullsized":"2024-09-14T04:37:06.118225+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1297,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116472+0000","last_change":"2024-09-14T04:07:58.508202+0000","last_active":"2024-09-14T04:37:06.116472+0000","last_peered":"2024-09-14T04:37:06.116472+0000","last_clean":"2024-09-14T04:37:06.116472+0000","last_became_active":"2024-09-14T04:07:58.507929+0000","last_became_peered":"2024-09-14T04:07:58.507929+0000","last_unstale":"2024-09-14T04:37:06.116472+0000","last_undegraded":"2024-09-14T04:37:06.116472+0000","last_fullsized":"2024-09-14T04:37:06.116472+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:17:58.160600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1278,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.118124+0000","last_change":"2024-09-14T04:06:46.858890+0000","last_active":"2024-09-14T04:37:06.118124+0000","last_peered":"2024-09-14T04:37:06.118124+0000","last_clean":"2024-09-14T04:37:06.118124+0000","last_became_active":"2024-09-14T04:06:46.858569+0000","last_became_peered":"2024-09-14T04:06:46.858569+0000","last_unstale":"2024-09-14T04:37:06.118124+0000","last_undegraded":"2024-09-14T04:37:06.118124+0000","last_fullsized":"2024-09-14T04:37:06.118124+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:25:46.400115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1264,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.843281+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T04:37:05.843281+0000","last_peered":"2024-09-14T04:37:05.843281+0000","last_clean":"2024-09-14T04:37:05.843281+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T04:37:05.843281+0000","last_undegraded":"2024-09-14T04:37:05.843281+0000","last_fullsized":"2024-09-14T04:37:05.843281+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1277,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.118257+0000","last_change":"2024-09-14T04:05:54.419435+0000","last_active":"2024-09-14T04:37:06.118257+0000","last_peered":"2024-09-14T04:37:06.118257+0000","last_clean":"2024-09-14T04:37:06.118257+0000","last_became_active":"2024-09-14T04:05:54.419094+0000","last_became_peered":"2024-09-14T04:05:54.419094+0000","last_unstale":"2024-09-14T04:37:06.118257+0000","last_undegraded":"2024-09-14T04:37:06.118257+0000","last_fullsized":"2024-09-14T04:37:06.118257+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:00.162629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1286,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:08.945667+0000","last_change":"2024-09-14T04:05:54.418813+0000","last_active":"2024-09-14T04:37:08.945667+0000","last_peered":"2024-09-14T04:37:08.945667+0000","last_clean":"2024-09-14T04:37:08.945667+0000","last_became_active":"2024-09-14T04:05:54.417584+0000","last_became_peered":"2024-09-14T04:05:54.417584+0000","last_unstale":"2024-09-14T04:37:08.945667+0000","last_undegraded":"2024-09-14T04:37:08.945667+0000","last_fullsized":"2024-09-14T04:37:08.945667+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:13:16.209456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1880,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.835033+0000","last_change":"2024-09-14T04:07:58.565442+0000","last_active":"2024-09-14T04:37:07.835033+0000","last_peered":"2024-09-14T04:37:07.835033+0000","last_clean":"2024-09-14T04:37:07.835033+0000","last_became_active":"2024-09-14T04:07:58.552289+0000","last_became_peered":"2024-09-14T04:07:58.552289+0000","last_unstale":"2024-09-14T04:37:07.835033+0000","last_undegraded":"2024-09-14T04:37:07.835033+0000","last_fullsized":"2024-09-14T04:37:07.835033+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:48:02.364960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1261,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.845043+0000","last_change":"2024-09-14T03:53:05.244003+0000","last_active":"2024-09-14T04:37:07.845043+0000","last_peered":"2024-09-14T04:37:07.845043+0000","last_clean":"2024-09-14T04:37:07.845043+0000","last_became_active":"2024-09-14T03:53:05.240743+0000","last_became_peered":"2024-09-14T03:53:05.240743+0000","last_unstale":"2024-09-14T04:37:07.845043+0000","last_undegraded":"2024-09-14T04:37:07.845043+0000","last_fullsized":"2024-09-14T04:37:07.845043+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:30.000189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1901,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.028402+0000","last_change":"2024-09-14T04:07:58.560880+0000","last_active":"2024-09-14T04:37:07.028402+0000","last_peered":"2024-09-14T04:37:07.028402+0000","last_clean":"2024-09-14T04:37:07.028402+0000","last_became_active":"2024-09-14T04:07:58.560132+0000","last_became_peered":"2024-09-14T04:07:58.560132+0000","last_unstale":"2024-09-14T04:37:07.028402+0000","last_undegraded":"2024-09-14T04:37:07.028402+0000","last_fullsized":"2024-09-14T04:37:07.028402+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:46:00.287050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1900,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.031410+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T04:37:07.031410+0000","last_peered":"2024-09-14T04:37:07.031410+0000","last_clean":"2024-09-14T04:37:07.031410+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T04:37:07.031410+0000","last_undegraded":"2024-09-14T04:37:07.031410+0000","last_fullsized":"2024-09-14T04:37:07.031410+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1277,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.843618+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T04:37:05.843618+0000","last_peered":"2024-09-14T04:37:05.843618+0000","last_clean":"2024-09-14T04:37:05.843618+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T04:37:05.843618+0000","last_undegraded":"2024-09-14T04:37:05.843618+0000","last_fullsized":"2024-09-14T04:37:05.843618+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1278,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.845193+0000","last_change":"2024-09-14T04:07:21.718847+0000","last_active":"2024-09-14T04:37:07.845193+0000","last_peered":"2024-09-14T04:37:07.845193+0000","last_clean":"2024-09-14T04:37:07.845193+0000","last_became_active":"2024-09-14T04:07:21.718504+0000","last_became_peered":"2024-09-14T04:07:21.718504+0000","last_unstale":"2024-09-14T04:37:07.845193+0000","last_undegraded":"2024-09-14T04:37:07.845193+0000","last_fullsized":"2024-09-14T04:37:07.845193+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:37:34.405765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1896,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.835037+0000","last_change":"2024-09-14T04:07:58.537946+0000","last_active":"2024-09-14T04:37:07.835037+0000","last_peered":"2024-09-14T04:37:07.835037+0000","last_clean":"2024-09-14T04:37:07.835037+0000","last_became_active":"2024-09-14T04:07:58.536034+0000","last_became_peered":"2024-09-14T04:07:58.536034+0000","last_unstale":"2024-09-14T04:37:07.835037+0000","last_undegraded":"2024-09-14T04:37:07.835037+0000","last_fullsized":"2024-09-14T04:37:07.835037+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:17:18.200470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201843,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":122440,"kb_used_data":15104,"kb_used_omap":1,"kb_used_meta":107326,"kb_avail":93622712,"statfs":{"total":95995035648,"available":95869657088,"internally_reserved":0,"allocated":15466496,"data_stored":9945307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":109902283},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463479,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":126532,"kb_used_data":15096,"kb_used_omap":1,"kb_used_meta":111422,"kb_avail":93618620,"statfs":{"total":95995035648,"available":95865466880,"internally_reserved":0,"allocated":15458304,"data_stored":9945180,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":114096583},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725116,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":113656,"kb_used_data":14508,"kb_used_omap":1,"kb_used_meta":99134,"kb_avail":93631496,"statfs":{"total":95995035648,"available":95878651904,"internally_reserved":0,"allocated":14856192,"data_stored":9350638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":101513670},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986754,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":139908,"kb_used_data":14520,"kb_used_omap":1,"kb_used_meta":125374,"kb_avail":93605244,"statfs":{"total":95995035648,"available":95851769856,"internally_reserved":0,"allocated":14868480,"data_stored":9354869,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":128383430},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772074857,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":86828,"kb_used_data":14500,"kb_used_omap":71,"kb_used_meta":72248,"kb_avail":93658324,"statfs":{"total":95995035648,"available":95906123776,"internally_reserved":0,"allocated":14848000,"data_stored":9350069,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":73982902},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707303793,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":68852,"kb_used_data":14504,"kb_used_omap":68,"kb_used_meta":54267,"kb_avail":93676300,"statfs":{"total":95995035648,"available":95924531200,"internally_reserved":0,"allocated":14852096,"data_stored":9350595,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":55569616},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872728952,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":86772,"kb_used_data":14500,"kb_used_omap":67,"kb_used_meta":72188,"kb_avail":93658380,"statfs":{"total":95995035648,"available":95906181120,"internally_reserved":0,"allocated":14848000,"data_stored":9349998,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":73920869},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056002,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":87164,"kb_used_data":15084,"kb_used_omap":74,"kb_used_meta":71989,"kb_avail":93657988,"statfs":{"total":95995035648,"available":95905779712,"internally_reserved":0,"allocated":15446016,"data_stored":9940951,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":73717429},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:37:56.799 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:37:57.011 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:37:57.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:56 smithi039 ceph-mon[106470]: pgmap v2132: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:57.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:56 smithi039 ceph-mon[106470]: from='client.35040 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:37:57.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:56 smithi039 ceph-mon[111235]: pgmap v2132: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:57.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:56 smithi039 ceph-mon[111235]: from='client.35040 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:37:57.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:56 smithi129 ceph-mon[101316]: pgmap v2132: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:57.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:56 smithi129 ceph-mon[101316]: from='client.35040 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:37:58.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:37:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:37:58] "GET /metrics HTTP/1.1" 200 34259 "" "Prometheus/2.51.0" 2024-09-14T04:37:58.825 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:37:58.826 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T04:37:59.177 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:58 smithi039 ceph-mon[106470]: pgmap v2133: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:59.177 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:58 smithi039 ceph-mon[111235]: pgmap v2133: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:59.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:58 smithi129 ceph-mon[101316]: pgmap v2133: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:37:59.516 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":2133,"stamp":"2024-09-14T04:37:57.478990+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":279,"num_write_kb":2449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":208,"ondisk_log_size":208,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":832152,"kb_used_data":117816,"kb_used_omap":288,"kb_used_meta":713951,"kb_avail":749129064,"statfs":{"total":767960285184,"available":767108161536,"internally_reserved":0,"allocated":120643584,"data_stored":76587607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":294978,"internal_metadata":731086782},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003056"},"pg_stats":[{"pgid":"3.1f","version":"72'4","reported_seq":1910,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.029243+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T04:37:07.029243+0000","last_peered":"2024-09-14T04:37:07.029243+0000","last_clean":"2024-09-14T04:37:07.029243+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T04:37:07.029243+0000","last_undegraded":"2024-09-14T04:37:07.029243+0000","last_fullsized":"2024-09-14T04:37:07.029243+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1274,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116018+0000","last_change":"2024-09-14T03:53:45.836449+0000","last_active":"2024-09-14T04:37:06.116018+0000","last_peered":"2024-09-14T04:37:06.116018+0000","last_clean":"2024-09-14T04:37:06.116018+0000","last_became_active":"2024-09-14T03:53:45.836108+0000","last_became_peered":"2024-09-14T03:53:45.836108+0000","last_unstale":"2024-09-14T04:37:06.116018+0000","last_undegraded":"2024-09-14T04:37:06.116018+0000","last_fullsized":"2024-09-14T04:37:06.116018+0000","mapping_epoch":73,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":74,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:52:13.095225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1899,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.835281+0000","last_change":"2024-09-14T04:07:58.504829+0000","last_active":"2024-09-14T04:37:07.835281+0000","last_peered":"2024-09-14T04:37:07.835281+0000","last_clean":"2024-09-14T04:37:07.835281+0000","last_became_active":"2024-09-14T04:07:58.502623+0000","last_became_peered":"2024-09-14T04:07:58.502623+0000","last_unstale":"2024-09-14T04:37:07.835281+0000","last_undegraded":"2024-09-14T04:37:07.835281+0000","last_fullsized":"2024-09-14T04:37:07.835281+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:59.091993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1911,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.833882+0000","last_change":"2024-09-14T04:07:58.507125+0000","last_active":"2024-09-14T04:37:07.833882+0000","last_peered":"2024-09-14T04:37:07.833882+0000","last_clean":"2024-09-14T04:37:07.833882+0000","last_became_active":"2024-09-14T04:07:58.505701+0000","last_became_peered":"2024-09-14T04:07:58.505701+0000","last_unstale":"2024-09-14T04:37:07.833882+0000","last_undegraded":"2024-09-14T04:37:07.833882+0000","last_fullsized":"2024-09-14T04:37:07.833882+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:25:00.847961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1264,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.843540+0000","last_change":"2024-09-14T03:53:05.234444+0000","last_active":"2024-09-14T04:37:05.843540+0000","last_peered":"2024-09-14T04:37:05.843540+0000","last_clean":"2024-09-14T04:37:05.843540+0000","last_became_active":"2024-09-14T03:53:05.234115+0000","last_became_peered":"2024-09-14T03:53:05.234115+0000","last_unstale":"2024-09-14T04:37:05.843540+0000","last_undegraded":"2024-09-14T04:37:05.843540+0000","last_fullsized":"2024-09-14T04:37:05.843540+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:01:23.153937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1294,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116964+0000","last_change":"2024-09-14T04:07:58.547135+0000","last_active":"2024-09-14T04:37:06.116964+0000","last_peered":"2024-09-14T04:37:06.116964+0000","last_clean":"2024-09-14T04:37:06.116964+0000","last_became_active":"2024-09-14T04:07:58.546751+0000","last_became_peered":"2024-09-14T04:07:58.546751+0000","last_unstale":"2024-09-14T04:37:06.116964+0000","last_undegraded":"2024-09-14T04:37:06.116964+0000","last_fullsized":"2024-09-14T04:37:06.116964+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:01.092698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1279,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.843531+0000","last_change":"2024-09-14T04:06:46.875795+0000","last_active":"2024-09-14T04:37:05.843531+0000","last_peered":"2024-09-14T04:37:05.843531+0000","last_clean":"2024-09-14T04:37:05.843531+0000","last_became_active":"2024-09-14T04:06:46.875395+0000","last_became_peered":"2024-09-14T04:06:46.875395+0000","last_unstale":"2024-09-14T04:37:05.843531+0000","last_undegraded":"2024-09-14T04:37:05.843531+0000","last_fullsized":"2024-09-14T04:37:05.843531+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:51:10.713063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1294,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116345+0000","last_change":"2024-09-14T04:07:58.514349+0000","last_active":"2024-09-14T04:37:06.116345+0000","last_peered":"2024-09-14T04:37:06.116345+0000","last_clean":"2024-09-14T04:37:06.116345+0000","last_became_active":"2024-09-14T04:07:58.513485+0000","last_became_peered":"2024-09-14T04:07:58.513485+0000","last_unstale":"2024-09-14T04:37:06.116345+0000","last_undegraded":"2024-09-14T04:37:06.116345+0000","last_fullsized":"2024-09-14T04:37:06.116345+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:33:09.031813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":1892,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.029098+0000","last_change":"2024-09-14T04:05:54.428057+0000","last_active":"2024-09-14T04:37:07.029098+0000","last_peered":"2024-09-14T04:37:07.029098+0000","last_clean":"2024-09-14T04:37:07.029098+0000","last_became_active":"2024-09-14T04:05:54.414895+0000","last_became_peered":"2024-09-14T04:05:54.414895+0000","last_unstale":"2024-09-14T04:37:07.029098+0000","last_undegraded":"2024-09-14T04:37:07.029098+0000","last_fullsized":"2024-09-14T04:37:07.029098+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:45:15.752675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1907,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.028294+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T04:37:07.028294+0000","last_peered":"2024-09-14T04:37:07.028294+0000","last_clean":"2024-09-14T04:37:07.028294+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T04:37:07.028294+0000","last_undegraded":"2024-09-14T04:37:07.028294+0000","last_fullsized":"2024-09-14T04:37:07.028294+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1296,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116813+0000","last_change":"2024-09-14T04:07:21.717071+0000","last_active":"2024-09-14T04:37:06.116813+0000","last_peered":"2024-09-14T04:37:06.116813+0000","last_clean":"2024-09-14T04:37:06.116813+0000","last_became_active":"2024-09-14T04:07:21.716215+0000","last_became_peered":"2024-09-14T04:07:21.716215+0000","last_unstale":"2024-09-14T04:37:06.116813+0000","last_undegraded":"2024-09-14T04:37:06.116813+0000","last_fullsized":"2024-09-14T04:37:06.116813+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:48:47.175490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1879,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.834536+0000","last_change":"2024-09-14T04:07:58.524371+0000","last_active":"2024-09-14T04:37:07.834536+0000","last_peered":"2024-09-14T04:37:07.834536+0000","last_clean":"2024-09-14T04:37:07.834536+0000","last_became_active":"2024-09-14T04:07:58.524088+0000","last_became_peered":"2024-09-14T04:07:58.524088+0000","last_unstale":"2024-09-14T04:37:07.834536+0000","last_undegraded":"2024-09-14T04:37:07.834536+0000","last_fullsized":"2024-09-14T04:37:07.834536+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:42.615569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1874,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.180210+0000","last_change":"2024-09-14T04:07:58.507553+0000","last_active":"2024-09-14T04:37:07.180210+0000","last_peered":"2024-09-14T04:37:07.180210+0000","last_clean":"2024-09-14T04:37:07.180210+0000","last_became_active":"2024-09-14T04:07:58.507139+0000","last_became_peered":"2024-09-14T04:07:58.507139+0000","last_unstale":"2024-09-14T04:37:07.180210+0000","last_undegraded":"2024-09-14T04:37:07.180210+0000","last_fullsized":"2024-09-14T04:37:07.180210+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:11:57.821769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1904,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.028779+0000","last_change":"2024-09-14T04:07:58.561490+0000","last_active":"2024-09-14T04:37:07.028779+0000","last_peered":"2024-09-14T04:37:07.028779+0000","last_clean":"2024-09-14T04:37:07.028779+0000","last_became_active":"2024-09-14T04:07:58.559841+0000","last_became_peered":"2024-09-14T04:07:58.559841+0000","last_unstale":"2024-09-14T04:37:07.028779+0000","last_undegraded":"2024-09-14T04:37:07.028779+0000","last_fullsized":"2024-09-14T04:37:07.028779+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:27:03.795368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1896,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.833696+0000","last_change":"2024-09-14T04:07:58.555731+0000","last_active":"2024-09-14T04:37:07.833696+0000","last_peered":"2024-09-14T04:37:07.833696+0000","last_clean":"2024-09-14T04:37:07.833696+0000","last_became_active":"2024-09-14T04:07:58.549576+0000","last_became_peered":"2024-09-14T04:07:58.549576+0000","last_unstale":"2024-09-14T04:37:07.833696+0000","last_undegraded":"2024-09-14T04:37:07.833696+0000","last_fullsized":"2024-09-14T04:37:07.833696+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:28:42.339589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1288,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.841667+0000","last_change":"2024-09-14T04:07:21.720797+0000","last_active":"2024-09-14T04:37:05.841667+0000","last_peered":"2024-09-14T04:37:05.841667+0000","last_clean":"2024-09-14T04:37:05.841667+0000","last_became_active":"2024-09-14T04:07:21.719331+0000","last_became_peered":"2024-09-14T04:07:21.719331+0000","last_unstale":"2024-09-14T04:37:05.841667+0000","last_undegraded":"2024-09-14T04:37:05.841667+0000","last_fullsized":"2024-09-14T04:37:05.841667+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:35.404023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1905,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.833897+0000","last_change":"2024-09-14T04:07:58.502533+0000","last_active":"2024-09-14T04:37:07.833897+0000","last_peered":"2024-09-14T04:37:07.833897+0000","last_clean":"2024-09-14T04:37:07.833897+0000","last_became_active":"2024-09-14T04:07:58.499849+0000","last_became_peered":"2024-09-14T04:07:58.499849+0000","last_unstale":"2024-09-14T04:37:07.833897+0000","last_undegraded":"2024-09-14T04:37:07.833897+0000","last_fullsized":"2024-09-14T04:37:07.833897+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:15:35.609746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1900,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.029481+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T04:37:07.029481+0000","last_peered":"2024-09-14T04:37:07.029481+0000","last_clean":"2024-09-14T04:37:07.029481+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T04:37:07.029481+0000","last_undegraded":"2024-09-14T04:37:07.029481+0000","last_fullsized":"2024-09-14T04:37:07.029481+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1881,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.835001+0000","last_change":"2024-09-14T04:07:58.507682+0000","last_active":"2024-09-14T04:37:07.835001+0000","last_peered":"2024-09-14T04:37:07.835001+0000","last_clean":"2024-09-14T04:37:07.835001+0000","last_became_active":"2024-09-14T04:07:58.507365+0000","last_became_peered":"2024-09-14T04:07:58.507365+0000","last_unstale":"2024-09-14T04:37:07.835001+0000","last_undegraded":"2024-09-14T04:37:07.835001+0000","last_fullsized":"2024-09-14T04:37:07.835001+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:14:47.294950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1277,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.843922+0000","last_change":"2024-09-14T04:06:46.875668+0000","last_active":"2024-09-14T04:37:05.843922+0000","last_peered":"2024-09-14T04:37:05.843922+0000","last_clean":"2024-09-14T04:37:05.843922+0000","last_became_active":"2024-09-14T04:06:46.875149+0000","last_became_peered":"2024-09-14T04:06:46.875149+0000","last_unstale":"2024-09-14T04:37:05.843922+0000","last_undegraded":"2024-09-14T04:37:05.843922+0000","last_fullsized":"2024-09-14T04:37:05.843922+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:45:59.190047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1278,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.118067+0000","last_change":"2024-09-14T04:07:58.510699+0000","last_active":"2024-09-14T04:37:06.118067+0000","last_peered":"2024-09-14T04:37:06.118067+0000","last_clean":"2024-09-14T04:37:06.118067+0000","last_became_active":"2024-09-14T04:07:58.506240+0000","last_became_peered":"2024-09-14T04:07:58.506240+0000","last_unstale":"2024-09-14T04:37:06.118067+0000","last_undegraded":"2024-09-14T04:37:06.118067+0000","last_fullsized":"2024-09-14T04:37:06.118067+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:27:18.051658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1907,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.181564+0000","last_change":"2024-09-14T04:07:21.687985+0000","last_active":"2024-09-14T04:37:07.181564+0000","last_peered":"2024-09-14T04:37:07.181564+0000","last_clean":"2024-09-14T04:37:07.181564+0000","last_became_active":"2024-09-14T04:07:21.687670+0000","last_became_peered":"2024-09-14T04:07:21.687670+0000","last_unstale":"2024-09-14T04:37:07.181564+0000","last_undegraded":"2024-09-14T04:37:07.181564+0000","last_fullsized":"2024-09-14T04:37:07.181564+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:53:58.295662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":1290,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:08.762974+0000","last_change":"2024-09-14T03:53:05.244055+0000","last_active":"2024-09-14T04:37:08.762974+0000","last_peered":"2024-09-14T04:37:08.762974+0000","last_clean":"2024-09-14T04:37:08.762974+0000","last_became_active":"2024-09-14T03:53:05.243752+0000","last_became_peered":"2024-09-14T03:53:05.243752+0000","last_unstale":"2024-09-14T04:37:08.762974+0000","last_undegraded":"2024-09-14T04:37:08.762974+0000","last_fullsized":"2024-09-14T04:37:08.762974+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:12:31.850064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1281,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:08.763591+0000","last_change":"2024-09-14T04:06:46.895670+0000","last_active":"2024-09-14T04:37:08.763591+0000","last_peered":"2024-09-14T04:37:08.763591+0000","last_clean":"2024-09-14T04:37:08.763591+0000","last_became_active":"2024-09-14T04:06:46.895370+0000","last_became_peered":"2024-09-14T04:06:46.895370+0000","last_unstale":"2024-09-14T04:37:08.763591+0000","last_undegraded":"2024-09-14T04:37:08.763591+0000","last_fullsized":"2024-09-14T04:37:08.763591+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:48:48.040529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1268,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116350+0000","last_change":"2024-09-14T03:53:06.634043+0000","last_active":"2024-09-14T04:37:06.116350+0000","last_peered":"2024-09-14T04:37:06.116350+0000","last_clean":"2024-09-14T04:37:06.116350+0000","last_became_active":"2024-09-14T03:53:06.633638+0000","last_became_peered":"2024-09-14T03:53:06.633638+0000","last_unstale":"2024-09-14T04:37:06.116350+0000","last_undegraded":"2024-09-14T04:37:06.116350+0000","last_fullsized":"2024-09-14T04:37:06.116350+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:15:53.434064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1292,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:08.946361+0000","last_change":"2024-09-14T04:07:21.721242+0000","last_active":"2024-09-14T04:37:08.946361+0000","last_peered":"2024-09-14T04:37:08.946361+0000","last_clean":"2024-09-14T04:37:08.946361+0000","last_became_active":"2024-09-14T04:07:21.720084+0000","last_became_peered":"2024-09-14T04:07:21.720084+0000","last_unstale":"2024-09-14T04:37:08.946361+0000","last_undegraded":"2024-09-14T04:37:08.946361+0000","last_fullsized":"2024-09-14T04:37:08.946361+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:58:41.308643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":1906,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.028465+0000","last_change":"2024-09-14T04:06:46.932567+0000","last_active":"2024-09-14T04:37:07.028465+0000","last_peered":"2024-09-14T04:37:07.028465+0000","last_clean":"2024-09-14T04:37:07.028465+0000","last_became_active":"2024-09-14T04:06:46.930732+0000","last_became_peered":"2024-09-14T04:06:46.930732+0000","last_unstale":"2024-09-14T04:37:07.028465+0000","last_undegraded":"2024-09-14T04:37:07.028465+0000","last_fullsized":"2024-09-14T04:37:07.028465+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:10:02.729499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1873,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.833825+0000","last_change":"2024-09-14T04:07:58.555446+0000","last_active":"2024-09-14T04:37:07.833825+0000","last_peered":"2024-09-14T04:37:07.833825+0000","last_clean":"2024-09-14T04:37:07.833825+0000","last_became_active":"2024-09-14T04:07:58.548355+0000","last_became_peered":"2024-09-14T04:07:58.548355+0000","last_unstale":"2024-09-14T04:37:07.833825+0000","last_undegraded":"2024-09-14T04:37:07.833825+0000","last_fullsized":"2024-09-14T04:37:07.833825+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:56:28.426094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1911,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.180789+0000","last_change":"2024-09-14T04:06:46.915273+0000","last_active":"2024-09-14T04:37:07.180789+0000","last_peered":"2024-09-14T04:37:07.180789+0000","last_clean":"2024-09-14T04:37:07.180789+0000","last_became_active":"2024-09-14T04:06:46.914994+0000","last_became_peered":"2024-09-14T04:06:46.914994+0000","last_unstale":"2024-09-14T04:37:07.180789+0000","last_undegraded":"2024-09-14T04:37:07.180789+0000","last_fullsized":"2024-09-14T04:37:07.180789+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:48:58.802609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1874,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.835129+0000","last_change":"2024-09-14T04:07:58.564377+0000","last_active":"2024-09-14T04:37:07.835129+0000","last_peered":"2024-09-14T04:37:07.835129+0000","last_clean":"2024-09-14T04:37:07.835129+0000","last_became_active":"2024-09-14T04:07:58.546913+0000","last_became_peered":"2024-09-14T04:07:58.546913+0000","last_unstale":"2024-09-14T04:37:07.835129+0000","last_undegraded":"2024-09-14T04:37:07.835129+0000","last_fullsized":"2024-09-14T04:37:07.835129+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:05:53.637115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1893,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.029265+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T04:37:07.029265+0000","last_peered":"2024-09-14T04:37:07.029265+0000","last_clean":"2024-09-14T04:37:07.029265+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T04:37:07.029265+0000","last_undegraded":"2024-09-14T04:37:07.029265+0000","last_fullsized":"2024-09-14T04:37:07.029265+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1291,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.846686+0000","last_change":"2024-09-14T04:05:54.381035+0000","last_active":"2024-09-14T04:37:07.846686+0000","last_peered":"2024-09-14T04:37:07.846686+0000","last_clean":"2024-09-14T04:37:07.846686+0000","last_became_active":"2024-09-14T04:05:54.379861+0000","last_became_peered":"2024-09-14T04:05:54.379861+0000","last_unstale":"2024-09-14T04:37:07.846686+0000","last_undegraded":"2024-09-14T04:37:07.846686+0000","last_fullsized":"2024-09-14T04:37:07.846686+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:07:46.302925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1910,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.182302+0000","last_change":"2024-09-14T04:07:21.686880+0000","last_active":"2024-09-14T04:37:07.182302+0000","last_peered":"2024-09-14T04:37:07.182302+0000","last_clean":"2024-09-14T04:37:07.182302+0000","last_became_active":"2024-09-14T04:07:21.685558+0000","last_became_peered":"2024-09-14T04:07:21.685558+0000","last_unstale":"2024-09-14T04:37:07.182302+0000","last_undegraded":"2024-09-14T04:37:07.182302+0000","last_fullsized":"2024-09-14T04:37:07.182302+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:24:18.693428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1898,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.181385+0000","last_change":"2024-09-14T04:06:46.905931+0000","last_active":"2024-09-14T04:37:07.181385+0000","last_peered":"2024-09-14T04:37:07.181385+0000","last_clean":"2024-09-14T04:37:07.181385+0000","last_became_active":"2024-09-14T04:06:46.905666+0000","last_became_peered":"2024-09-14T04:06:46.905666+0000","last_unstale":"2024-09-14T04:37:07.181385+0000","last_undegraded":"2024-09-14T04:37:07.181385+0000","last_fullsized":"2024-09-14T04:37:07.181385+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:40:57.007948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1927,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.182411+0000","last_change":"2024-09-14T04:06:46.930482+0000","last_active":"2024-09-14T04:37:07.182411+0000","last_peered":"2024-09-14T04:37:07.182411+0000","last_clean":"2024-09-14T04:37:07.182411+0000","last_became_active":"2024-09-14T04:06:46.927372+0000","last_became_peered":"2024-09-14T04:06:46.927372+0000","last_unstale":"2024-09-14T04:37:07.182411+0000","last_undegraded":"2024-09-14T04:37:07.182411+0000","last_fullsized":"2024-09-14T04:37:07.182411+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:07:22.159128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1296,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.843694+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T04:37:05.843694+0000","last_peered":"2024-09-14T04:37:05.843694+0000","last_clean":"2024-09-14T04:37:05.843694+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T04:37:05.843694+0000","last_undegraded":"2024-09-14T04:37:05.843694+0000","last_fullsized":"2024-09-14T04:37:05.843694+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1920,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.182400+0000","last_change":"2024-09-14T04:07:58.510721+0000","last_active":"2024-09-14T04:37:07.182400+0000","last_peered":"2024-09-14T04:37:07.182400+0000","last_clean":"2024-09-14T04:37:07.182400+0000","last_became_active":"2024-09-14T04:07:58.510332+0000","last_became_peered":"2024-09-14T04:07:58.510332+0000","last_unstale":"2024-09-14T04:37:07.182400+0000","last_undegraded":"2024-09-14T04:37:07.182400+0000","last_fullsized":"2024-09-14T04:37:07.182400+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:07:03.926842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1284,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116506+0000","last_change":"2024-09-14T04:05:54.426939+0000","last_active":"2024-09-14T04:37:06.116506+0000","last_peered":"2024-09-14T04:37:06.116506+0000","last_clean":"2024-09-14T04:37:06.116506+0000","last_became_active":"2024-09-14T04:05:54.426427+0000","last_became_peered":"2024-09-14T04:05:54.426427+0000","last_unstale":"2024-09-14T04:37:06.116506+0000","last_undegraded":"2024-09-14T04:37:06.116506+0000","last_fullsized":"2024-09-14T04:37:06.116506+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:53:55.841489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1295,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:08.602276+0000","last_change":"2024-09-14T04:06:46.875340+0000","last_active":"2024-09-14T04:37:08.602276+0000","last_peered":"2024-09-14T04:37:08.602276+0000","last_clean":"2024-09-14T04:37:08.602276+0000","last_became_active":"2024-09-14T04:06:46.874552+0000","last_became_peered":"2024-09-14T04:06:46.874552+0000","last_unstale":"2024-09-14T04:37:08.602276+0000","last_undegraded":"2024-09-14T04:37:08.602276+0000","last_fullsized":"2024-09-14T04:37:08.602276+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:52.257357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1907,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.180965+0000","last_change":"2024-09-14T04:06:46.917681+0000","last_active":"2024-09-14T04:37:07.180965+0000","last_peered":"2024-09-14T04:37:07.180965+0000","last_clean":"2024-09-14T04:37:07.180965+0000","last_became_active":"2024-09-14T04:06:46.917401+0000","last_became_peered":"2024-09-14T04:06:46.917401+0000","last_unstale":"2024-09-14T04:37:07.180965+0000","last_undegraded":"2024-09-14T04:37:07.180965+0000","last_fullsized":"2024-09-14T04:37:07.180965+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:04:39.949490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1911,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.180136+0000","last_change":"2024-09-14T04:07:21.685415+0000","last_active":"2024-09-14T04:37:07.180136+0000","last_peered":"2024-09-14T04:37:07.180136+0000","last_clean":"2024-09-14T04:37:07.180136+0000","last_became_active":"2024-09-14T04:07:21.683504+0000","last_became_peered":"2024-09-14T04:07:21.683504+0000","last_unstale":"2024-09-14T04:37:07.180136+0000","last_undegraded":"2024-09-14T04:37:07.180136+0000","last_fullsized":"2024-09-14T04:37:07.180136+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:21:08.806291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1913,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.027850+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T04:37:07.027850+0000","last_peered":"2024-09-14T04:37:07.027850+0000","last_clean":"2024-09-14T04:37:07.027850+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T04:37:07.027850+0000","last_undegraded":"2024-09-14T04:37:07.027850+0000","last_fullsized":"2024-09-14T04:37:07.027850+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1914,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.028872+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T04:37:07.028872+0000","last_peered":"2024-09-14T04:37:07.028872+0000","last_clean":"2024-09-14T04:37:07.028872+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T04:37:07.028872+0000","last_undegraded":"2024-09-14T04:37:07.028872+0000","last_fullsized":"2024-09-14T04:37:07.028872+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1903,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.028102+0000","last_change":"2024-09-14T04:07:58.521344+0000","last_active":"2024-09-14T04:37:07.028102+0000","last_peered":"2024-09-14T04:37:07.028102+0000","last_clean":"2024-09-14T04:37:07.028102+0000","last_became_active":"2024-09-14T04:07:58.520965+0000","last_became_peered":"2024-09-14T04:07:58.520965+0000","last_unstale":"2024-09-14T04:37:07.028102+0000","last_undegraded":"2024-09-14T04:37:07.028102+0000","last_fullsized":"2024-09-14T04:37:07.028102+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:58:46.029013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1306,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.117356+0000","last_change":"2024-09-14T04:06:46.857436+0000","last_active":"2024-09-14T04:37:06.117356+0000","last_peered":"2024-09-14T04:37:06.117356+0000","last_clean":"2024-09-14T04:37:06.117356+0000","last_became_active":"2024-09-14T04:06:46.857154+0000","last_became_peered":"2024-09-14T04:06:46.857154+0000","last_unstale":"2024-09-14T04:37:06.117356+0000","last_undegraded":"2024-09-14T04:37:06.117356+0000","last_fullsized":"2024-09-14T04:37:06.117356+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:53:28.772152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1264,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.841559+0000","last_change":"2024-09-14T03:53:05.551268+0000","last_active":"2024-09-14T04:37:05.841559+0000","last_peered":"2024-09-14T04:37:05.841559+0000","last_clean":"2024-09-14T04:37:05.841559+0000","last_became_active":"2024-09-14T03:53:05.551015+0000","last_became_peered":"2024-09-14T03:53:05.551015+0000","last_unstale":"2024-09-14T04:37:05.841559+0000","last_undegraded":"2024-09-14T04:37:05.841559+0000","last_fullsized":"2024-09-14T04:37:05.841559+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.905475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1885,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.834431+0000","last_change":"2024-09-14T04:07:58.503896+0000","last_active":"2024-09-14T04:37:07.834431+0000","last_peered":"2024-09-14T04:37:07.834431+0000","last_clean":"2024-09-14T04:37:07.834431+0000","last_became_active":"2024-09-14T04:07:58.499253+0000","last_became_peered":"2024-09-14T04:07:58.499253+0000","last_unstale":"2024-09-14T04:37:07.834431+0000","last_undegraded":"2024-09-14T04:37:07.834431+0000","last_fullsized":"2024-09-14T04:37:07.834431+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:48:27.543818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1294,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:08.602592+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T04:37:08.602592+0000","last_peered":"2024-09-14T04:37:08.602592+0000","last_clean":"2024-09-14T04:37:08.602592+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T04:37:08.602592+0000","last_undegraded":"2024-09-14T04:37:08.602592+0000","last_fullsized":"2024-09-14T04:37:08.602592+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":1609,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.845285+0000","last_change":"2024-09-14T04:05:54.389043+0000","last_active":"2024-09-14T04:37:07.845285+0000","last_peered":"2024-09-14T04:37:07.845285+0000","last_clean":"2024-09-14T04:37:07.845285+0000","last_became_active":"2024-09-14T04:05:54.388655+0000","last_became_peered":"2024-09-14T04:05:54.388655+0000","last_unstale":"2024-09-14T04:37:07.845285+0000","last_undegraded":"2024-09-14T04:37:07.845285+0000","last_fullsized":"2024-09-14T04:37:07.845285+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T03:49:18.237706+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:38:59.615597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1875,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.180822+0000","last_change":"2024-09-14T04:07:58.530406+0000","last_active":"2024-09-14T04:37:07.180822+0000","last_peered":"2024-09-14T04:37:07.180822+0000","last_clean":"2024-09-14T04:37:07.180822+0000","last_became_active":"2024-09-14T04:07:58.530078+0000","last_became_peered":"2024-09-14T04:07:58.530078+0000","last_unstale":"2024-09-14T04:37:07.180822+0000","last_undegraded":"2024-09-14T04:37:07.180822+0000","last_fullsized":"2024-09-14T04:37:07.180822+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:54:53.524001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1883,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.833696+0000","last_change":"2024-09-14T04:07:58.555717+0000","last_active":"2024-09-14T04:37:07.833696+0000","last_peered":"2024-09-14T04:37:07.833696+0000","last_clean":"2024-09-14T04:37:07.833696+0000","last_became_active":"2024-09-14T04:07:58.548894+0000","last_became_peered":"2024-09-14T04:07:58.548894+0000","last_unstale":"2024-09-14T04:37:07.833696+0000","last_undegraded":"2024-09-14T04:37:07.833696+0000","last_fullsized":"2024-09-14T04:37:07.833696+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:42.061551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1281,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116836+0000","last_change":"2024-09-14T04:06:46.894741+0000","last_active":"2024-09-14T04:37:06.116836+0000","last_peered":"2024-09-14T04:37:06.116836+0000","last_clean":"2024-09-14T04:37:06.116836+0000","last_became_active":"2024-09-14T04:06:46.894445+0000","last_became_peered":"2024-09-14T04:06:46.894445+0000","last_unstale":"2024-09-14T04:37:06.116836+0000","last_undegraded":"2024-09-14T04:37:06.116836+0000","last_fullsized":"2024-09-14T04:37:06.116836+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:52:00.055271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1283,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116551+0000","last_change":"2024-09-14T04:07:21.715462+0000","last_active":"2024-09-14T04:37:06.116551+0000","last_peered":"2024-09-14T04:37:06.116551+0000","last_clean":"2024-09-14T04:37:06.116551+0000","last_became_active":"2024-09-14T04:07:21.713032+0000","last_became_peered":"2024-09-14T04:07:21.713032+0000","last_unstale":"2024-09-14T04:37:06.116551+0000","last_undegraded":"2024-09-14T04:37:06.116551+0000","last_fullsized":"2024-09-14T04:37:06.116551+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:37:05.545164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1296,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116624+0000","last_change":"2024-09-14T04:07:21.718037+0000","last_active":"2024-09-14T04:37:06.116624+0000","last_peered":"2024-09-14T04:37:06.116624+0000","last_clean":"2024-09-14T04:37:06.116624+0000","last_became_active":"2024-09-14T04:07:21.717700+0000","last_became_peered":"2024-09-14T04:07:21.717700+0000","last_unstale":"2024-09-14T04:37:06.116624+0000","last_undegraded":"2024-09-14T04:37:06.116624+0000","last_fullsized":"2024-09-14T04:37:06.116624+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:47:33.483785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1279,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.841296+0000","last_change":"2024-09-14T04:06:46.868780+0000","last_active":"2024-09-14T04:37:05.841296+0000","last_peered":"2024-09-14T04:37:05.841296+0000","last_clean":"2024-09-14T04:37:05.841296+0000","last_became_active":"2024-09-14T04:06:46.868483+0000","last_became_peered":"2024-09-14T04:06:46.868483+0000","last_unstale":"2024-09-14T04:37:05.841296+0000","last_undegraded":"2024-09-14T04:37:05.841296+0000","last_fullsized":"2024-09-14T04:37:05.841296+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:03.133158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1281,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.117264+0000","last_change":"2024-09-14T04:07:58.515859+0000","last_active":"2024-09-14T04:37:06.117264+0000","last_peered":"2024-09-14T04:37:06.117264+0000","last_clean":"2024-09-14T04:37:06.117264+0000","last_became_active":"2024-09-14T04:07:58.515594+0000","last_became_peered":"2024-09-14T04:07:58.515594+0000","last_unstale":"2024-09-14T04:37:06.117264+0000","last_undegraded":"2024-09-14T04:37:06.117264+0000","last_fullsized":"2024-09-14T04:37:06.117264+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:45.966798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":1900,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.833729+0000","last_change":"2024-09-14T04:07:58.509833+0000","last_active":"2024-09-14T04:37:07.833729+0000","last_peered":"2024-09-14T04:37:07.833729+0000","last_clean":"2024-09-14T04:37:07.833729+0000","last_became_active":"2024-09-14T04:07:58.509198+0000","last_became_peered":"2024-09-14T04:07:58.509198+0000","last_unstale":"2024-09-14T04:37:07.833729+0000","last_undegraded":"2024-09-14T04:37:07.833729+0000","last_fullsized":"2024-09-14T04:37:07.833729+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:24:40.193090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1903,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.028840+0000","last_change":"2024-09-14T04:06:46.925323+0000","last_active":"2024-09-14T04:37:07.028840+0000","last_peered":"2024-09-14T04:37:07.028840+0000","last_clean":"2024-09-14T04:37:07.028840+0000","last_became_active":"2024-09-14T04:06:46.925043+0000","last_became_peered":"2024-09-14T04:06:46.925043+0000","last_unstale":"2024-09-14T04:37:07.028840+0000","last_undegraded":"2024-09-14T04:37:07.028840+0000","last_fullsized":"2024-09-14T04:37:07.028840+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:39.895453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1279,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.843564+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T04:37:05.843564+0000","last_peered":"2024-09-14T04:37:05.843564+0000","last_clean":"2024-09-14T04:37:05.843564+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T04:37:05.843564+0000","last_undegraded":"2024-09-14T04:37:05.843564+0000","last_fullsized":"2024-09-14T04:37:05.843564+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1278,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.841209+0000","last_change":"2024-09-14T04:06:46.869868+0000","last_active":"2024-09-14T04:37:05.841209+0000","last_peered":"2024-09-14T04:37:05.841209+0000","last_clean":"2024-09-14T04:37:05.841209+0000","last_became_active":"2024-09-14T04:06:46.869126+0000","last_became_peered":"2024-09-14T04:06:46.869126+0000","last_unstale":"2024-09-14T04:37:05.841209+0000","last_undegraded":"2024-09-14T04:37:05.841209+0000","last_fullsized":"2024-09-14T04:37:05.841209+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:41.150138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1262,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.118225+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T04:37:06.118225+0000","last_peered":"2024-09-14T04:37:06.118225+0000","last_clean":"2024-09-14T04:37:06.118225+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T04:37:06.118225+0000","last_undegraded":"2024-09-14T04:37:06.118225+0000","last_fullsized":"2024-09-14T04:37:06.118225+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1297,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116472+0000","last_change":"2024-09-14T04:07:58.508202+0000","last_active":"2024-09-14T04:37:06.116472+0000","last_peered":"2024-09-14T04:37:06.116472+0000","last_clean":"2024-09-14T04:37:06.116472+0000","last_became_active":"2024-09-14T04:07:58.507929+0000","last_became_peered":"2024-09-14T04:07:58.507929+0000","last_unstale":"2024-09-14T04:37:06.116472+0000","last_undegraded":"2024-09-14T04:37:06.116472+0000","last_fullsized":"2024-09-14T04:37:06.116472+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:17:58.160600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1278,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.118124+0000","last_change":"2024-09-14T04:06:46.858890+0000","last_active":"2024-09-14T04:37:06.118124+0000","last_peered":"2024-09-14T04:37:06.118124+0000","last_clean":"2024-09-14T04:37:06.118124+0000","last_became_active":"2024-09-14T04:06:46.858569+0000","last_became_peered":"2024-09-14T04:06:46.858569+0000","last_unstale":"2024-09-14T04:37:06.118124+0000","last_undegraded":"2024-09-14T04:37:06.118124+0000","last_fullsized":"2024-09-14T04:37:06.118124+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:25:46.400115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1264,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.843281+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T04:37:05.843281+0000","last_peered":"2024-09-14T04:37:05.843281+0000","last_clean":"2024-09-14T04:37:05.843281+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T04:37:05.843281+0000","last_undegraded":"2024-09-14T04:37:05.843281+0000","last_fullsized":"2024-09-14T04:37:05.843281+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1277,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.118257+0000","last_change":"2024-09-14T04:05:54.419435+0000","last_active":"2024-09-14T04:37:06.118257+0000","last_peered":"2024-09-14T04:37:06.118257+0000","last_clean":"2024-09-14T04:37:06.118257+0000","last_became_active":"2024-09-14T04:05:54.419094+0000","last_became_peered":"2024-09-14T04:05:54.419094+0000","last_unstale":"2024-09-14T04:37:06.118257+0000","last_undegraded":"2024-09-14T04:37:06.118257+0000","last_fullsized":"2024-09-14T04:37:06.118257+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:00.162629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1286,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:08.945667+0000","last_change":"2024-09-14T04:05:54.418813+0000","last_active":"2024-09-14T04:37:08.945667+0000","last_peered":"2024-09-14T04:37:08.945667+0000","last_clean":"2024-09-14T04:37:08.945667+0000","last_became_active":"2024-09-14T04:05:54.417584+0000","last_became_peered":"2024-09-14T04:05:54.417584+0000","last_unstale":"2024-09-14T04:37:08.945667+0000","last_undegraded":"2024-09-14T04:37:08.945667+0000","last_fullsized":"2024-09-14T04:37:08.945667+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:13:16.209456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1880,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.835033+0000","last_change":"2024-09-14T04:07:58.565442+0000","last_active":"2024-09-14T04:37:07.835033+0000","last_peered":"2024-09-14T04:37:07.835033+0000","last_clean":"2024-09-14T04:37:07.835033+0000","last_became_active":"2024-09-14T04:07:58.552289+0000","last_became_peered":"2024-09-14T04:07:58.552289+0000","last_unstale":"2024-09-14T04:37:07.835033+0000","last_undegraded":"2024-09-14T04:37:07.835033+0000","last_fullsized":"2024-09-14T04:37:07.835033+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:48:02.364960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1261,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.845043+0000","last_change":"2024-09-14T03:53:05.244003+0000","last_active":"2024-09-14T04:37:07.845043+0000","last_peered":"2024-09-14T04:37:07.845043+0000","last_clean":"2024-09-14T04:37:07.845043+0000","last_became_active":"2024-09-14T03:53:05.240743+0000","last_became_peered":"2024-09-14T03:53:05.240743+0000","last_unstale":"2024-09-14T04:37:07.845043+0000","last_undegraded":"2024-09-14T04:37:07.845043+0000","last_fullsized":"2024-09-14T04:37:07.845043+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:30.000189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1901,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.028402+0000","last_change":"2024-09-14T04:07:58.560880+0000","last_active":"2024-09-14T04:37:07.028402+0000","last_peered":"2024-09-14T04:37:07.028402+0000","last_clean":"2024-09-14T04:37:07.028402+0000","last_became_active":"2024-09-14T04:07:58.560132+0000","last_became_peered":"2024-09-14T04:07:58.560132+0000","last_unstale":"2024-09-14T04:37:07.028402+0000","last_undegraded":"2024-09-14T04:37:07.028402+0000","last_fullsized":"2024-09-14T04:37:07.028402+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:46:00.287050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1900,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.031410+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T04:37:07.031410+0000","last_peered":"2024-09-14T04:37:07.031410+0000","last_clean":"2024-09-14T04:37:07.031410+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T04:37:07.031410+0000","last_undegraded":"2024-09-14T04:37:07.031410+0000","last_fullsized":"2024-09-14T04:37:07.031410+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1277,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.843618+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T04:37:05.843618+0000","last_peered":"2024-09-14T04:37:05.843618+0000","last_clean":"2024-09-14T04:37:05.843618+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T04:37:05.843618+0000","last_undegraded":"2024-09-14T04:37:05.843618+0000","last_fullsized":"2024-09-14T04:37:05.843618+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1278,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.845193+0000","last_change":"2024-09-14T04:07:21.718847+0000","last_active":"2024-09-14T04:37:07.845193+0000","last_peered":"2024-09-14T04:37:07.845193+0000","last_clean":"2024-09-14T04:37:07.845193+0000","last_became_active":"2024-09-14T04:07:21.718504+0000","last_became_peered":"2024-09-14T04:07:21.718504+0000","last_unstale":"2024-09-14T04:37:07.845193+0000","last_undegraded":"2024-09-14T04:37:07.845193+0000","last_fullsized":"2024-09-14T04:37:07.845193+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:37:34.405765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1896,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.835037+0000","last_change":"2024-09-14T04:07:58.537946+0000","last_active":"2024-09-14T04:37:07.835037+0000","last_peered":"2024-09-14T04:37:07.835037+0000","last_clean":"2024-09-14T04:37:07.835037+0000","last_became_active":"2024-09-14T04:07:58.536034+0000","last_became_peered":"2024-09-14T04:07:58.536034+0000","last_unstale":"2024-09-14T04:37:07.835037+0000","last_undegraded":"2024-09-14T04:37:07.835037+0000","last_fullsized":"2024-09-14T04:37:07.835037+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:17:18.200470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201843,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":122440,"kb_used_data":15104,"kb_used_omap":1,"kb_used_meta":107326,"kb_avail":93622712,"statfs":{"total":95995035648,"available":95869657088,"internally_reserved":0,"allocated":15466496,"data_stored":9945307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":109902283},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463479,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":126532,"kb_used_data":15096,"kb_used_omap":1,"kb_used_meta":111422,"kb_avail":93618620,"statfs":{"total":95995035648,"available":95865466880,"internally_reserved":0,"allocated":15458304,"data_stored":9945180,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":114096583},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725116,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":113656,"kb_used_data":14508,"kb_used_omap":1,"kb_used_meta":99134,"kb_avail":93631496,"statfs":{"total":95995035648,"available":95878651904,"internally_reserved":0,"allocated":14856192,"data_stored":9350638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":101513670},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986754,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":139908,"kb_used_data":14520,"kb_used_omap":1,"kb_used_meta":125374,"kb_avail":93605244,"statfs":{"total":95995035648,"available":95851769856,"internally_reserved":0,"allocated":14868480,"data_stored":9354869,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":128383430},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772074858,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":86828,"kb_used_data":14500,"kb_used_omap":71,"kb_used_meta":72248,"kb_avail":93658324,"statfs":{"total":95995035648,"available":95906123776,"internally_reserved":0,"allocated":14848000,"data_stored":9350069,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":73982902},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707303793,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":68852,"kb_used_data":14504,"kb_used_omap":68,"kb_used_meta":54267,"kb_avail":93676300,"statfs":{"total":95995035648,"available":95924531200,"internally_reserved":0,"allocated":14852096,"data_stored":9350595,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":55569616},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872728952,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":86772,"kb_used_data":14500,"kb_used_omap":67,"kb_used_meta":72188,"kb_avail":93658380,"statfs":{"total":95995035648,"available":95906181120,"internally_reserved":0,"allocated":14848000,"data_stored":9349998,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":73920869},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056003,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":87164,"kb_used_data":15084,"kb_used_omap":74,"kb_used_meta":71989,"kb_avail":93657988,"statfs":{"total":95995035648,"available":95905779712,"internally_reserved":0,"allocated":15446016,"data_stored":9940951,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":73717429},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:37:59.520 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-14T04:37:59.520 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-09-14T04:37:59.521 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-09-14T04:37:59.521 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph health --format=json 2024-09-14T04:37:59.728 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:38:00.122 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:37:59 smithi039 ceph-mon[106470]: from='client.35046 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:38:00.122 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:37:59 smithi039 ceph-mon[111235]: from='client.35046 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:38:00.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:37:59 smithi129 ceph-mon[101316]: from='client.35046 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:38:01.194 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:00 smithi039 ceph-mon[106470]: pgmap v2134: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:01.194 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:00 smithi039 ceph-mon[111235]: pgmap v2134: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:01.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:00 smithi129 ceph-mon[101316]: pgmap v2134: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:01.690 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:38:01.690 INFO:teuthology.orchestra.run.smithi039.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-09-14T04:38:02.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:01 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3150692949' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-09-14T04:38:02.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:01 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3150692949' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-09-14T04:38:02.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:01 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3150692949' entity='client.admin' cmd={"prefix": "health", "format": "json"} : dispatch 2024-09-14T04:38:02.344 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-09-14T04:38:02.344 INFO:teuthology.run_tasks:Running task print... 2024-09-14T04:38:02.357 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2024-09-14T04:38:02.357 INFO:teuthology.run_tasks:Running task parallel... 2024-09-14T04:38:02.376 INFO:teuthology.task.parallel:starting parallel... 2024-09-14T04:38:02.376 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-09-14T04:38:02.377 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2024-09-14T04:38:02.379 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2024-09-14T04:38:02.379 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'main'}, 'ceph': {'conf': {'global': {'mon election default strategy': 1}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', 'mons down', 'mon down', 'MON_DOWN', 'out of quorum', 'PG_AVAILABILITY'], 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'a2782315255a3e7d5f6cdc9eb35785187ff6ddf8'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'main', 'sha1': '1c941a66c1b420259b8fc5ac19e792dbbf60aded'}} 2024-09-14T04:38:02.379 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2024-09-14T04:38:02.379 INFO:tasks.thrashosds:Beginning thrashosds... 2024-09-14T04:38:02.379 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-09-14T04:38:02.380 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:38:02.591 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:38:02.988 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:02 smithi039 ceph-mon[106470]: pgmap v2135: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:02.988 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:02 smithi039 ceph-mon[111235]: pgmap v2135: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:03.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:02 smithi129 ceph-mon[101316]: pgmap v2135: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:04.507 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:38:04.507 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T04:38:05.117 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:04 smithi039 ceph-mon[106470]: pgmap v2136: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:05.117 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:04 smithi039 ceph-mon[111235]: pgmap v2136: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:05.138 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":2136,"stamp":"2024-09-14T04:38:03.480411+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":279,"num_write_kb":2449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":208,"ondisk_log_size":208,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":832152,"kb_used_data":117816,"kb_used_omap":288,"kb_used_meta":713951,"kb_avail":749129064,"statfs":{"total":767960285184,"available":767108161536,"internally_reserved":0,"allocated":120643584,"data_stored":76587607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":294978,"internal_metadata":731086782},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002872"},"pg_stats":[{"pgid":"3.1f","version":"72'4","reported_seq":1910,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.029243+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T04:37:07.029243+0000","last_peered":"2024-09-14T04:37:07.029243+0000","last_clean":"2024-09-14T04:37:07.029243+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T04:37:07.029243+0000","last_undegraded":"2024-09-14T04:37:07.029243+0000","last_fullsized":"2024-09-14T04:37:07.029243+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1274,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116018+0000","last_change":"2024-09-14T03:53:45.836449+0000","last_active":"2024-09-14T04:37:06.116018+0000","last_peered":"2024-09-14T04:37:06.116018+0000","last_clean":"2024-09-14T04:37:06.116018+0000","last_became_active":"2024-09-14T03:53:45.836108+0000","last_became_peered":"2024-09-14T03:53:45.836108+0000","last_unstale":"2024-09-14T04:37:06.116018+0000","last_undegraded":"2024-09-14T04:37:06.116018+0000","last_fullsized":"2024-09-14T04:37:06.116018+0000","mapping_epoch":73,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":74,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:52:13.095225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1899,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.835281+0000","last_change":"2024-09-14T04:07:58.504829+0000","last_active":"2024-09-14T04:37:07.835281+0000","last_peered":"2024-09-14T04:37:07.835281+0000","last_clean":"2024-09-14T04:37:07.835281+0000","last_became_active":"2024-09-14T04:07:58.502623+0000","last_became_peered":"2024-09-14T04:07:58.502623+0000","last_unstale":"2024-09-14T04:37:07.835281+0000","last_undegraded":"2024-09-14T04:37:07.835281+0000","last_fullsized":"2024-09-14T04:37:07.835281+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:59.091993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1911,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.833882+0000","last_change":"2024-09-14T04:07:58.507125+0000","last_active":"2024-09-14T04:37:07.833882+0000","last_peered":"2024-09-14T04:37:07.833882+0000","last_clean":"2024-09-14T04:37:07.833882+0000","last_became_active":"2024-09-14T04:07:58.505701+0000","last_became_peered":"2024-09-14T04:07:58.505701+0000","last_unstale":"2024-09-14T04:37:07.833882+0000","last_undegraded":"2024-09-14T04:37:07.833882+0000","last_fullsized":"2024-09-14T04:37:07.833882+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:25:00.847961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1264,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.843540+0000","last_change":"2024-09-14T03:53:05.234444+0000","last_active":"2024-09-14T04:37:05.843540+0000","last_peered":"2024-09-14T04:37:05.843540+0000","last_clean":"2024-09-14T04:37:05.843540+0000","last_became_active":"2024-09-14T03:53:05.234115+0000","last_became_peered":"2024-09-14T03:53:05.234115+0000","last_unstale":"2024-09-14T04:37:05.843540+0000","last_undegraded":"2024-09-14T04:37:05.843540+0000","last_fullsized":"2024-09-14T04:37:05.843540+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:01:23.153937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1295,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:08.762133+0000","last_change":"2024-09-14T04:07:58.547135+0000","last_active":"2024-09-14T04:37:08.762133+0000","last_peered":"2024-09-14T04:37:08.762133+0000","last_clean":"2024-09-14T04:37:08.762133+0000","last_became_active":"2024-09-14T04:07:58.546751+0000","last_became_peered":"2024-09-14T04:07:58.546751+0000","last_unstale":"2024-09-14T04:37:08.762133+0000","last_undegraded":"2024-09-14T04:37:08.762133+0000","last_fullsized":"2024-09-14T04:37:08.762133+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:01.092698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1279,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.843531+0000","last_change":"2024-09-14T04:06:46.875795+0000","last_active":"2024-09-14T04:37:05.843531+0000","last_peered":"2024-09-14T04:37:05.843531+0000","last_clean":"2024-09-14T04:37:05.843531+0000","last_became_active":"2024-09-14T04:06:46.875395+0000","last_became_peered":"2024-09-14T04:06:46.875395+0000","last_unstale":"2024-09-14T04:37:05.843531+0000","last_undegraded":"2024-09-14T04:37:05.843531+0000","last_fullsized":"2024-09-14T04:37:05.843531+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:51:10.713063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1294,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116345+0000","last_change":"2024-09-14T04:07:58.514349+0000","last_active":"2024-09-14T04:37:06.116345+0000","last_peered":"2024-09-14T04:37:06.116345+0000","last_clean":"2024-09-14T04:37:06.116345+0000","last_became_active":"2024-09-14T04:07:58.513485+0000","last_became_peered":"2024-09-14T04:07:58.513485+0000","last_unstale":"2024-09-14T04:37:06.116345+0000","last_undegraded":"2024-09-14T04:37:06.116345+0000","last_fullsized":"2024-09-14T04:37:06.116345+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:33:09.031813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":1892,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.029098+0000","last_change":"2024-09-14T04:05:54.428057+0000","last_active":"2024-09-14T04:37:07.029098+0000","last_peered":"2024-09-14T04:37:07.029098+0000","last_clean":"2024-09-14T04:37:07.029098+0000","last_became_active":"2024-09-14T04:05:54.414895+0000","last_became_peered":"2024-09-14T04:05:54.414895+0000","last_unstale":"2024-09-14T04:37:07.029098+0000","last_undegraded":"2024-09-14T04:37:07.029098+0000","last_fullsized":"2024-09-14T04:37:07.029098+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:45:15.752675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1907,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.028294+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T04:37:07.028294+0000","last_peered":"2024-09-14T04:37:07.028294+0000","last_clean":"2024-09-14T04:37:07.028294+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T04:37:07.028294+0000","last_undegraded":"2024-09-14T04:37:07.028294+0000","last_fullsized":"2024-09-14T04:37:07.028294+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1297,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:08.762241+0000","last_change":"2024-09-14T04:07:21.717071+0000","last_active":"2024-09-14T04:37:08.762241+0000","last_peered":"2024-09-14T04:37:08.762241+0000","last_clean":"2024-09-14T04:37:08.762241+0000","last_became_active":"2024-09-14T04:07:21.716215+0000","last_became_peered":"2024-09-14T04:07:21.716215+0000","last_unstale":"2024-09-14T04:37:08.762241+0000","last_undegraded":"2024-09-14T04:37:08.762241+0000","last_fullsized":"2024-09-14T04:37:08.762241+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:48:47.175490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1880,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.834576+0000","last_change":"2024-09-14T04:07:58.524371+0000","last_active":"2024-09-14T04:37:07.834576+0000","last_peered":"2024-09-14T04:37:07.834576+0000","last_clean":"2024-09-14T04:37:07.834576+0000","last_became_active":"2024-09-14T04:07:58.524088+0000","last_became_peered":"2024-09-14T04:07:58.524088+0000","last_unstale":"2024-09-14T04:37:07.834576+0000","last_undegraded":"2024-09-14T04:37:07.834576+0000","last_fullsized":"2024-09-14T04:37:07.834576+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:42.615569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1874,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.180210+0000","last_change":"2024-09-14T04:07:58.507553+0000","last_active":"2024-09-14T04:37:07.180210+0000","last_peered":"2024-09-14T04:37:07.180210+0000","last_clean":"2024-09-14T04:37:07.180210+0000","last_became_active":"2024-09-14T04:07:58.507139+0000","last_became_peered":"2024-09-14T04:07:58.507139+0000","last_unstale":"2024-09-14T04:37:07.180210+0000","last_undegraded":"2024-09-14T04:37:07.180210+0000","last_fullsized":"2024-09-14T04:37:07.180210+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:11:57.821769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1905,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.028818+0000","last_change":"2024-09-14T04:07:58.561490+0000","last_active":"2024-09-14T04:37:07.028818+0000","last_peered":"2024-09-14T04:37:07.028818+0000","last_clean":"2024-09-14T04:37:07.028818+0000","last_became_active":"2024-09-14T04:07:58.559841+0000","last_became_peered":"2024-09-14T04:07:58.559841+0000","last_unstale":"2024-09-14T04:37:07.028818+0000","last_undegraded":"2024-09-14T04:37:07.028818+0000","last_fullsized":"2024-09-14T04:37:07.028818+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:27:03.795368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1896,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.833696+0000","last_change":"2024-09-14T04:07:58.555731+0000","last_active":"2024-09-14T04:37:07.833696+0000","last_peered":"2024-09-14T04:37:07.833696+0000","last_clean":"2024-09-14T04:37:07.833696+0000","last_became_active":"2024-09-14T04:07:58.549576+0000","last_became_peered":"2024-09-14T04:07:58.549576+0000","last_unstale":"2024-09-14T04:37:07.833696+0000","last_undegraded":"2024-09-14T04:37:07.833696+0000","last_fullsized":"2024-09-14T04:37:07.833696+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:28:42.339589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1288,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.841667+0000","last_change":"2024-09-14T04:07:21.720797+0000","last_active":"2024-09-14T04:37:05.841667+0000","last_peered":"2024-09-14T04:37:05.841667+0000","last_clean":"2024-09-14T04:37:05.841667+0000","last_became_active":"2024-09-14T04:07:21.719331+0000","last_became_peered":"2024-09-14T04:07:21.719331+0000","last_unstale":"2024-09-14T04:37:05.841667+0000","last_undegraded":"2024-09-14T04:37:05.841667+0000","last_fullsized":"2024-09-14T04:37:05.841667+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:35.404023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1905,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.833897+0000","last_change":"2024-09-14T04:07:58.502533+0000","last_active":"2024-09-14T04:37:07.833897+0000","last_peered":"2024-09-14T04:37:07.833897+0000","last_clean":"2024-09-14T04:37:07.833897+0000","last_became_active":"2024-09-14T04:07:58.499849+0000","last_became_peered":"2024-09-14T04:07:58.499849+0000","last_unstale":"2024-09-14T04:37:07.833897+0000","last_undegraded":"2024-09-14T04:37:07.833897+0000","last_fullsized":"2024-09-14T04:37:07.833897+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:15:35.609746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1900,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.029481+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T04:37:07.029481+0000","last_peered":"2024-09-14T04:37:07.029481+0000","last_clean":"2024-09-14T04:37:07.029481+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T04:37:07.029481+0000","last_undegraded":"2024-09-14T04:37:07.029481+0000","last_fullsized":"2024-09-14T04:37:07.029481+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1881,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.835001+0000","last_change":"2024-09-14T04:07:58.507682+0000","last_active":"2024-09-14T04:37:07.835001+0000","last_peered":"2024-09-14T04:37:07.835001+0000","last_clean":"2024-09-14T04:37:07.835001+0000","last_became_active":"2024-09-14T04:07:58.507365+0000","last_became_peered":"2024-09-14T04:07:58.507365+0000","last_unstale":"2024-09-14T04:37:07.835001+0000","last_undegraded":"2024-09-14T04:37:07.835001+0000","last_fullsized":"2024-09-14T04:37:07.835001+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:14:47.294950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1277,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.843922+0000","last_change":"2024-09-14T04:06:46.875668+0000","last_active":"2024-09-14T04:37:05.843922+0000","last_peered":"2024-09-14T04:37:05.843922+0000","last_clean":"2024-09-14T04:37:05.843922+0000","last_became_active":"2024-09-14T04:06:46.875149+0000","last_became_peered":"2024-09-14T04:06:46.875149+0000","last_unstale":"2024-09-14T04:37:05.843922+0000","last_undegraded":"2024-09-14T04:37:05.843922+0000","last_fullsized":"2024-09-14T04:37:05.843922+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:45:59.190047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1278,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.118067+0000","last_change":"2024-09-14T04:07:58.510699+0000","last_active":"2024-09-14T04:37:06.118067+0000","last_peered":"2024-09-14T04:37:06.118067+0000","last_clean":"2024-09-14T04:37:06.118067+0000","last_became_active":"2024-09-14T04:07:58.506240+0000","last_became_peered":"2024-09-14T04:07:58.506240+0000","last_unstale":"2024-09-14T04:37:06.118067+0000","last_undegraded":"2024-09-14T04:37:06.118067+0000","last_fullsized":"2024-09-14T04:37:06.118067+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:27:18.051658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1907,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.181564+0000","last_change":"2024-09-14T04:07:21.687985+0000","last_active":"2024-09-14T04:37:07.181564+0000","last_peered":"2024-09-14T04:37:07.181564+0000","last_clean":"2024-09-14T04:37:07.181564+0000","last_became_active":"2024-09-14T04:07:21.687670+0000","last_became_peered":"2024-09-14T04:07:21.687670+0000","last_unstale":"2024-09-14T04:37:07.181564+0000","last_undegraded":"2024-09-14T04:37:07.181564+0000","last_fullsized":"2024-09-14T04:37:07.181564+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:53:58.295662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":1290,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:08.762974+0000","last_change":"2024-09-14T03:53:05.244055+0000","last_active":"2024-09-14T04:37:08.762974+0000","last_peered":"2024-09-14T04:37:08.762974+0000","last_clean":"2024-09-14T04:37:08.762974+0000","last_became_active":"2024-09-14T03:53:05.243752+0000","last_became_peered":"2024-09-14T03:53:05.243752+0000","last_unstale":"2024-09-14T04:37:08.762974+0000","last_undegraded":"2024-09-14T04:37:08.762974+0000","last_fullsized":"2024-09-14T04:37:08.762974+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:12:31.850064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1281,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:08.763591+0000","last_change":"2024-09-14T04:06:46.895670+0000","last_active":"2024-09-14T04:37:08.763591+0000","last_peered":"2024-09-14T04:37:08.763591+0000","last_clean":"2024-09-14T04:37:08.763591+0000","last_became_active":"2024-09-14T04:06:46.895370+0000","last_became_peered":"2024-09-14T04:06:46.895370+0000","last_unstale":"2024-09-14T04:37:08.763591+0000","last_undegraded":"2024-09-14T04:37:08.763591+0000","last_fullsized":"2024-09-14T04:37:08.763591+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:48:48.040529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1268,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116350+0000","last_change":"2024-09-14T03:53:06.634043+0000","last_active":"2024-09-14T04:37:06.116350+0000","last_peered":"2024-09-14T04:37:06.116350+0000","last_clean":"2024-09-14T04:37:06.116350+0000","last_became_active":"2024-09-14T03:53:06.633638+0000","last_became_peered":"2024-09-14T03:53:06.633638+0000","last_unstale":"2024-09-14T04:37:06.116350+0000","last_undegraded":"2024-09-14T04:37:06.116350+0000","last_fullsized":"2024-09-14T04:37:06.116350+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:15:53.434064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1292,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:08.946361+0000","last_change":"2024-09-14T04:07:21.721242+0000","last_active":"2024-09-14T04:37:08.946361+0000","last_peered":"2024-09-14T04:37:08.946361+0000","last_clean":"2024-09-14T04:37:08.946361+0000","last_became_active":"2024-09-14T04:07:21.720084+0000","last_became_peered":"2024-09-14T04:07:21.720084+0000","last_unstale":"2024-09-14T04:37:08.946361+0000","last_undegraded":"2024-09-14T04:37:08.946361+0000","last_fullsized":"2024-09-14T04:37:08.946361+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:58:41.308643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":1906,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.028465+0000","last_change":"2024-09-14T04:06:46.932567+0000","last_active":"2024-09-14T04:37:07.028465+0000","last_peered":"2024-09-14T04:37:07.028465+0000","last_clean":"2024-09-14T04:37:07.028465+0000","last_became_active":"2024-09-14T04:06:46.930732+0000","last_became_peered":"2024-09-14T04:06:46.930732+0000","last_unstale":"2024-09-14T04:37:07.028465+0000","last_undegraded":"2024-09-14T04:37:07.028465+0000","last_fullsized":"2024-09-14T04:37:07.028465+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:10:02.729499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1873,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.833825+0000","last_change":"2024-09-14T04:07:58.555446+0000","last_active":"2024-09-14T04:37:07.833825+0000","last_peered":"2024-09-14T04:37:07.833825+0000","last_clean":"2024-09-14T04:37:07.833825+0000","last_became_active":"2024-09-14T04:07:58.548355+0000","last_became_peered":"2024-09-14T04:07:58.548355+0000","last_unstale":"2024-09-14T04:37:07.833825+0000","last_undegraded":"2024-09-14T04:37:07.833825+0000","last_fullsized":"2024-09-14T04:37:07.833825+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:56:28.426094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1911,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.180789+0000","last_change":"2024-09-14T04:06:46.915273+0000","last_active":"2024-09-14T04:37:07.180789+0000","last_peered":"2024-09-14T04:37:07.180789+0000","last_clean":"2024-09-14T04:37:07.180789+0000","last_became_active":"2024-09-14T04:06:46.914994+0000","last_became_peered":"2024-09-14T04:06:46.914994+0000","last_unstale":"2024-09-14T04:37:07.180789+0000","last_undegraded":"2024-09-14T04:37:07.180789+0000","last_fullsized":"2024-09-14T04:37:07.180789+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:48:58.802609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1874,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.835129+0000","last_change":"2024-09-14T04:07:58.564377+0000","last_active":"2024-09-14T04:37:07.835129+0000","last_peered":"2024-09-14T04:37:07.835129+0000","last_clean":"2024-09-14T04:37:07.835129+0000","last_became_active":"2024-09-14T04:07:58.546913+0000","last_became_peered":"2024-09-14T04:07:58.546913+0000","last_unstale":"2024-09-14T04:37:07.835129+0000","last_undegraded":"2024-09-14T04:37:07.835129+0000","last_fullsized":"2024-09-14T04:37:07.835129+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:05:53.637115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1893,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.029265+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T04:37:07.029265+0000","last_peered":"2024-09-14T04:37:07.029265+0000","last_clean":"2024-09-14T04:37:07.029265+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T04:37:07.029265+0000","last_undegraded":"2024-09-14T04:37:07.029265+0000","last_fullsized":"2024-09-14T04:37:07.029265+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1291,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.846686+0000","last_change":"2024-09-14T04:05:54.381035+0000","last_active":"2024-09-14T04:37:07.846686+0000","last_peered":"2024-09-14T04:37:07.846686+0000","last_clean":"2024-09-14T04:37:07.846686+0000","last_became_active":"2024-09-14T04:05:54.379861+0000","last_became_peered":"2024-09-14T04:05:54.379861+0000","last_unstale":"2024-09-14T04:37:07.846686+0000","last_undegraded":"2024-09-14T04:37:07.846686+0000","last_fullsized":"2024-09-14T04:37:07.846686+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:07:46.302925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1910,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.182302+0000","last_change":"2024-09-14T04:07:21.686880+0000","last_active":"2024-09-14T04:37:07.182302+0000","last_peered":"2024-09-14T04:37:07.182302+0000","last_clean":"2024-09-14T04:37:07.182302+0000","last_became_active":"2024-09-14T04:07:21.685558+0000","last_became_peered":"2024-09-14T04:07:21.685558+0000","last_unstale":"2024-09-14T04:37:07.182302+0000","last_undegraded":"2024-09-14T04:37:07.182302+0000","last_fullsized":"2024-09-14T04:37:07.182302+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:24:18.693428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1898,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.181385+0000","last_change":"2024-09-14T04:06:46.905931+0000","last_active":"2024-09-14T04:37:07.181385+0000","last_peered":"2024-09-14T04:37:07.181385+0000","last_clean":"2024-09-14T04:37:07.181385+0000","last_became_active":"2024-09-14T04:06:46.905666+0000","last_became_peered":"2024-09-14T04:06:46.905666+0000","last_unstale":"2024-09-14T04:37:07.181385+0000","last_undegraded":"2024-09-14T04:37:07.181385+0000","last_fullsized":"2024-09-14T04:37:07.181385+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:40:57.007948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1927,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.182411+0000","last_change":"2024-09-14T04:06:46.930482+0000","last_active":"2024-09-14T04:37:07.182411+0000","last_peered":"2024-09-14T04:37:07.182411+0000","last_clean":"2024-09-14T04:37:07.182411+0000","last_became_active":"2024-09-14T04:06:46.927372+0000","last_became_peered":"2024-09-14T04:06:46.927372+0000","last_unstale":"2024-09-14T04:37:07.182411+0000","last_undegraded":"2024-09-14T04:37:07.182411+0000","last_fullsized":"2024-09-14T04:37:07.182411+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:07:22.159128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1296,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.843694+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T04:37:05.843694+0000","last_peered":"2024-09-14T04:37:05.843694+0000","last_clean":"2024-09-14T04:37:05.843694+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T04:37:05.843694+0000","last_undegraded":"2024-09-14T04:37:05.843694+0000","last_fullsized":"2024-09-14T04:37:05.843694+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1920,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.182400+0000","last_change":"2024-09-14T04:07:58.510721+0000","last_active":"2024-09-14T04:37:07.182400+0000","last_peered":"2024-09-14T04:37:07.182400+0000","last_clean":"2024-09-14T04:37:07.182400+0000","last_became_active":"2024-09-14T04:07:58.510332+0000","last_became_peered":"2024-09-14T04:07:58.510332+0000","last_unstale":"2024-09-14T04:37:07.182400+0000","last_undegraded":"2024-09-14T04:37:07.182400+0000","last_fullsized":"2024-09-14T04:37:07.182400+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:07:03.926842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1284,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116506+0000","last_change":"2024-09-14T04:05:54.426939+0000","last_active":"2024-09-14T04:37:06.116506+0000","last_peered":"2024-09-14T04:37:06.116506+0000","last_clean":"2024-09-14T04:37:06.116506+0000","last_became_active":"2024-09-14T04:05:54.426427+0000","last_became_peered":"2024-09-14T04:05:54.426427+0000","last_unstale":"2024-09-14T04:37:06.116506+0000","last_undegraded":"2024-09-14T04:37:06.116506+0000","last_fullsized":"2024-09-14T04:37:06.116506+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:53:55.841489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1295,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:08.602276+0000","last_change":"2024-09-14T04:06:46.875340+0000","last_active":"2024-09-14T04:37:08.602276+0000","last_peered":"2024-09-14T04:37:08.602276+0000","last_clean":"2024-09-14T04:37:08.602276+0000","last_became_active":"2024-09-14T04:06:46.874552+0000","last_became_peered":"2024-09-14T04:06:46.874552+0000","last_unstale":"2024-09-14T04:37:08.602276+0000","last_undegraded":"2024-09-14T04:37:08.602276+0000","last_fullsized":"2024-09-14T04:37:08.602276+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:52.257357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1907,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.180965+0000","last_change":"2024-09-14T04:06:46.917681+0000","last_active":"2024-09-14T04:37:07.180965+0000","last_peered":"2024-09-14T04:37:07.180965+0000","last_clean":"2024-09-14T04:37:07.180965+0000","last_became_active":"2024-09-14T04:06:46.917401+0000","last_became_peered":"2024-09-14T04:06:46.917401+0000","last_unstale":"2024-09-14T04:37:07.180965+0000","last_undegraded":"2024-09-14T04:37:07.180965+0000","last_fullsized":"2024-09-14T04:37:07.180965+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:04:39.949490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1911,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.180136+0000","last_change":"2024-09-14T04:07:21.685415+0000","last_active":"2024-09-14T04:37:07.180136+0000","last_peered":"2024-09-14T04:37:07.180136+0000","last_clean":"2024-09-14T04:37:07.180136+0000","last_became_active":"2024-09-14T04:07:21.683504+0000","last_became_peered":"2024-09-14T04:07:21.683504+0000","last_unstale":"2024-09-14T04:37:07.180136+0000","last_undegraded":"2024-09-14T04:37:07.180136+0000","last_fullsized":"2024-09-14T04:37:07.180136+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:21:08.806291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1913,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.027850+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T04:37:07.027850+0000","last_peered":"2024-09-14T04:37:07.027850+0000","last_clean":"2024-09-14T04:37:07.027850+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T04:37:07.027850+0000","last_undegraded":"2024-09-14T04:37:07.027850+0000","last_fullsized":"2024-09-14T04:37:07.027850+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1914,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.028872+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T04:37:07.028872+0000","last_peered":"2024-09-14T04:37:07.028872+0000","last_clean":"2024-09-14T04:37:07.028872+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T04:37:07.028872+0000","last_undegraded":"2024-09-14T04:37:07.028872+0000","last_fullsized":"2024-09-14T04:37:07.028872+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1903,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.028102+0000","last_change":"2024-09-14T04:07:58.521344+0000","last_active":"2024-09-14T04:37:07.028102+0000","last_peered":"2024-09-14T04:37:07.028102+0000","last_clean":"2024-09-14T04:37:07.028102+0000","last_became_active":"2024-09-14T04:07:58.520965+0000","last_became_peered":"2024-09-14T04:07:58.520965+0000","last_unstale":"2024-09-14T04:37:07.028102+0000","last_undegraded":"2024-09-14T04:37:07.028102+0000","last_fullsized":"2024-09-14T04:37:07.028102+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:58:46.029013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1307,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.845573+0000","last_change":"2024-09-14T04:06:46.857436+0000","last_active":"2024-09-14T04:37:07.845573+0000","last_peered":"2024-09-14T04:37:07.845573+0000","last_clean":"2024-09-14T04:37:07.845573+0000","last_became_active":"2024-09-14T04:06:46.857154+0000","last_became_peered":"2024-09-14T04:06:46.857154+0000","last_unstale":"2024-09-14T04:37:07.845573+0000","last_undegraded":"2024-09-14T04:37:07.845573+0000","last_fullsized":"2024-09-14T04:37:07.845573+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:53:28.772152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1264,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.841559+0000","last_change":"2024-09-14T03:53:05.551268+0000","last_active":"2024-09-14T04:37:05.841559+0000","last_peered":"2024-09-14T04:37:05.841559+0000","last_clean":"2024-09-14T04:37:05.841559+0000","last_became_active":"2024-09-14T03:53:05.551015+0000","last_became_peered":"2024-09-14T03:53:05.551015+0000","last_unstale":"2024-09-14T04:37:05.841559+0000","last_undegraded":"2024-09-14T04:37:05.841559+0000","last_fullsized":"2024-09-14T04:37:05.841559+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.905475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1885,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.834431+0000","last_change":"2024-09-14T04:07:58.503896+0000","last_active":"2024-09-14T04:37:07.834431+0000","last_peered":"2024-09-14T04:37:07.834431+0000","last_clean":"2024-09-14T04:37:07.834431+0000","last_became_active":"2024-09-14T04:07:58.499253+0000","last_became_peered":"2024-09-14T04:07:58.499253+0000","last_unstale":"2024-09-14T04:37:07.834431+0000","last_undegraded":"2024-09-14T04:37:07.834431+0000","last_fullsized":"2024-09-14T04:37:07.834431+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:48:27.543818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1294,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:08.602592+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T04:37:08.602592+0000","last_peered":"2024-09-14T04:37:08.602592+0000","last_clean":"2024-09-14T04:37:08.602592+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T04:37:08.602592+0000","last_undegraded":"2024-09-14T04:37:08.602592+0000","last_fullsized":"2024-09-14T04:37:08.602592+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":1609,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.845285+0000","last_change":"2024-09-14T04:05:54.389043+0000","last_active":"2024-09-14T04:37:07.845285+0000","last_peered":"2024-09-14T04:37:07.845285+0000","last_clean":"2024-09-14T04:37:07.845285+0000","last_became_active":"2024-09-14T04:05:54.388655+0000","last_became_peered":"2024-09-14T04:05:54.388655+0000","last_unstale":"2024-09-14T04:37:07.845285+0000","last_undegraded":"2024-09-14T04:37:07.845285+0000","last_fullsized":"2024-09-14T04:37:07.845285+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T03:49:18.237706+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:38:59.615597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1875,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.180822+0000","last_change":"2024-09-14T04:07:58.530406+0000","last_active":"2024-09-14T04:37:07.180822+0000","last_peered":"2024-09-14T04:37:07.180822+0000","last_clean":"2024-09-14T04:37:07.180822+0000","last_became_active":"2024-09-14T04:07:58.530078+0000","last_became_peered":"2024-09-14T04:07:58.530078+0000","last_unstale":"2024-09-14T04:37:07.180822+0000","last_undegraded":"2024-09-14T04:37:07.180822+0000","last_fullsized":"2024-09-14T04:37:07.180822+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:54:53.524001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1883,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.833696+0000","last_change":"2024-09-14T04:07:58.555717+0000","last_active":"2024-09-14T04:37:07.833696+0000","last_peered":"2024-09-14T04:37:07.833696+0000","last_clean":"2024-09-14T04:37:07.833696+0000","last_became_active":"2024-09-14T04:07:58.548894+0000","last_became_peered":"2024-09-14T04:07:58.548894+0000","last_unstale":"2024-09-14T04:37:07.833696+0000","last_undegraded":"2024-09-14T04:37:07.833696+0000","last_fullsized":"2024-09-14T04:37:07.833696+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:42.061551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1281,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116836+0000","last_change":"2024-09-14T04:06:46.894741+0000","last_active":"2024-09-14T04:37:06.116836+0000","last_peered":"2024-09-14T04:37:06.116836+0000","last_clean":"2024-09-14T04:37:06.116836+0000","last_became_active":"2024-09-14T04:06:46.894445+0000","last_became_peered":"2024-09-14T04:06:46.894445+0000","last_unstale":"2024-09-14T04:37:06.116836+0000","last_undegraded":"2024-09-14T04:37:06.116836+0000","last_fullsized":"2024-09-14T04:37:06.116836+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:52:00.055271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1283,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116551+0000","last_change":"2024-09-14T04:07:21.715462+0000","last_active":"2024-09-14T04:37:06.116551+0000","last_peered":"2024-09-14T04:37:06.116551+0000","last_clean":"2024-09-14T04:37:06.116551+0000","last_became_active":"2024-09-14T04:07:21.713032+0000","last_became_peered":"2024-09-14T04:07:21.713032+0000","last_unstale":"2024-09-14T04:37:06.116551+0000","last_undegraded":"2024-09-14T04:37:06.116551+0000","last_fullsized":"2024-09-14T04:37:06.116551+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:37:05.545164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1296,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116624+0000","last_change":"2024-09-14T04:07:21.718037+0000","last_active":"2024-09-14T04:37:06.116624+0000","last_peered":"2024-09-14T04:37:06.116624+0000","last_clean":"2024-09-14T04:37:06.116624+0000","last_became_active":"2024-09-14T04:07:21.717700+0000","last_became_peered":"2024-09-14T04:07:21.717700+0000","last_unstale":"2024-09-14T04:37:06.116624+0000","last_undegraded":"2024-09-14T04:37:06.116624+0000","last_fullsized":"2024-09-14T04:37:06.116624+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:47:33.483785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1279,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.841296+0000","last_change":"2024-09-14T04:06:46.868780+0000","last_active":"2024-09-14T04:37:05.841296+0000","last_peered":"2024-09-14T04:37:05.841296+0000","last_clean":"2024-09-14T04:37:05.841296+0000","last_became_active":"2024-09-14T04:06:46.868483+0000","last_became_peered":"2024-09-14T04:06:46.868483+0000","last_unstale":"2024-09-14T04:37:05.841296+0000","last_undegraded":"2024-09-14T04:37:05.841296+0000","last_fullsized":"2024-09-14T04:37:05.841296+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:03.133158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1281,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.117264+0000","last_change":"2024-09-14T04:07:58.515859+0000","last_active":"2024-09-14T04:37:06.117264+0000","last_peered":"2024-09-14T04:37:06.117264+0000","last_clean":"2024-09-14T04:37:06.117264+0000","last_became_active":"2024-09-14T04:07:58.515594+0000","last_became_peered":"2024-09-14T04:07:58.515594+0000","last_unstale":"2024-09-14T04:37:06.117264+0000","last_undegraded":"2024-09-14T04:37:06.117264+0000","last_fullsized":"2024-09-14T04:37:06.117264+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:45.966798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":1900,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.833729+0000","last_change":"2024-09-14T04:07:58.509833+0000","last_active":"2024-09-14T04:37:07.833729+0000","last_peered":"2024-09-14T04:37:07.833729+0000","last_clean":"2024-09-14T04:37:07.833729+0000","last_became_active":"2024-09-14T04:07:58.509198+0000","last_became_peered":"2024-09-14T04:07:58.509198+0000","last_unstale":"2024-09-14T04:37:07.833729+0000","last_undegraded":"2024-09-14T04:37:07.833729+0000","last_fullsized":"2024-09-14T04:37:07.833729+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:24:40.193090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1903,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.028840+0000","last_change":"2024-09-14T04:06:46.925323+0000","last_active":"2024-09-14T04:37:07.028840+0000","last_peered":"2024-09-14T04:37:07.028840+0000","last_clean":"2024-09-14T04:37:07.028840+0000","last_became_active":"2024-09-14T04:06:46.925043+0000","last_became_peered":"2024-09-14T04:06:46.925043+0000","last_unstale":"2024-09-14T04:37:07.028840+0000","last_undegraded":"2024-09-14T04:37:07.028840+0000","last_fullsized":"2024-09-14T04:37:07.028840+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:39.895453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1279,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.843564+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T04:37:05.843564+0000","last_peered":"2024-09-14T04:37:05.843564+0000","last_clean":"2024-09-14T04:37:05.843564+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T04:37:05.843564+0000","last_undegraded":"2024-09-14T04:37:05.843564+0000","last_fullsized":"2024-09-14T04:37:05.843564+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1278,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.841209+0000","last_change":"2024-09-14T04:06:46.869868+0000","last_active":"2024-09-14T04:37:05.841209+0000","last_peered":"2024-09-14T04:37:05.841209+0000","last_clean":"2024-09-14T04:37:05.841209+0000","last_became_active":"2024-09-14T04:06:46.869126+0000","last_became_peered":"2024-09-14T04:06:46.869126+0000","last_unstale":"2024-09-14T04:37:05.841209+0000","last_undegraded":"2024-09-14T04:37:05.841209+0000","last_fullsized":"2024-09-14T04:37:05.841209+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:41.150138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1262,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.118225+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T04:37:06.118225+0000","last_peered":"2024-09-14T04:37:06.118225+0000","last_clean":"2024-09-14T04:37:06.118225+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T04:37:06.118225+0000","last_undegraded":"2024-09-14T04:37:06.118225+0000","last_fullsized":"2024-09-14T04:37:06.118225+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1297,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116472+0000","last_change":"2024-09-14T04:07:58.508202+0000","last_active":"2024-09-14T04:37:06.116472+0000","last_peered":"2024-09-14T04:37:06.116472+0000","last_clean":"2024-09-14T04:37:06.116472+0000","last_became_active":"2024-09-14T04:07:58.507929+0000","last_became_peered":"2024-09-14T04:07:58.507929+0000","last_unstale":"2024-09-14T04:37:06.116472+0000","last_undegraded":"2024-09-14T04:37:06.116472+0000","last_fullsized":"2024-09-14T04:37:06.116472+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:17:58.160600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1278,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.118124+0000","last_change":"2024-09-14T04:06:46.858890+0000","last_active":"2024-09-14T04:37:06.118124+0000","last_peered":"2024-09-14T04:37:06.118124+0000","last_clean":"2024-09-14T04:37:06.118124+0000","last_became_active":"2024-09-14T04:06:46.858569+0000","last_became_peered":"2024-09-14T04:06:46.858569+0000","last_unstale":"2024-09-14T04:37:06.118124+0000","last_undegraded":"2024-09-14T04:37:06.118124+0000","last_fullsized":"2024-09-14T04:37:06.118124+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:25:46.400115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1264,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.843281+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T04:37:05.843281+0000","last_peered":"2024-09-14T04:37:05.843281+0000","last_clean":"2024-09-14T04:37:05.843281+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T04:37:05.843281+0000","last_undegraded":"2024-09-14T04:37:05.843281+0000","last_fullsized":"2024-09-14T04:37:05.843281+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1277,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.118257+0000","last_change":"2024-09-14T04:05:54.419435+0000","last_active":"2024-09-14T04:37:06.118257+0000","last_peered":"2024-09-14T04:37:06.118257+0000","last_clean":"2024-09-14T04:37:06.118257+0000","last_became_active":"2024-09-14T04:05:54.419094+0000","last_became_peered":"2024-09-14T04:05:54.419094+0000","last_unstale":"2024-09-14T04:37:06.118257+0000","last_undegraded":"2024-09-14T04:37:06.118257+0000","last_fullsized":"2024-09-14T04:37:06.118257+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:00.162629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1286,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:08.945667+0000","last_change":"2024-09-14T04:05:54.418813+0000","last_active":"2024-09-14T04:37:08.945667+0000","last_peered":"2024-09-14T04:37:08.945667+0000","last_clean":"2024-09-14T04:37:08.945667+0000","last_became_active":"2024-09-14T04:05:54.417584+0000","last_became_peered":"2024-09-14T04:05:54.417584+0000","last_unstale":"2024-09-14T04:37:08.945667+0000","last_undegraded":"2024-09-14T04:37:08.945667+0000","last_fullsized":"2024-09-14T04:37:08.945667+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:13:16.209456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1880,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.835033+0000","last_change":"2024-09-14T04:07:58.565442+0000","last_active":"2024-09-14T04:37:07.835033+0000","last_peered":"2024-09-14T04:37:07.835033+0000","last_clean":"2024-09-14T04:37:07.835033+0000","last_became_active":"2024-09-14T04:07:58.552289+0000","last_became_peered":"2024-09-14T04:07:58.552289+0000","last_unstale":"2024-09-14T04:37:07.835033+0000","last_undegraded":"2024-09-14T04:37:07.835033+0000","last_fullsized":"2024-09-14T04:37:07.835033+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:48:02.364960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1261,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.845043+0000","last_change":"2024-09-14T03:53:05.244003+0000","last_active":"2024-09-14T04:37:07.845043+0000","last_peered":"2024-09-14T04:37:07.845043+0000","last_clean":"2024-09-14T04:37:07.845043+0000","last_became_active":"2024-09-14T03:53:05.240743+0000","last_became_peered":"2024-09-14T03:53:05.240743+0000","last_unstale":"2024-09-14T04:37:07.845043+0000","last_undegraded":"2024-09-14T04:37:07.845043+0000","last_fullsized":"2024-09-14T04:37:07.845043+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:30.000189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1901,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.028402+0000","last_change":"2024-09-14T04:07:58.560880+0000","last_active":"2024-09-14T04:37:07.028402+0000","last_peered":"2024-09-14T04:37:07.028402+0000","last_clean":"2024-09-14T04:37:07.028402+0000","last_became_active":"2024-09-14T04:07:58.560132+0000","last_became_peered":"2024-09-14T04:07:58.560132+0000","last_unstale":"2024-09-14T04:37:07.028402+0000","last_undegraded":"2024-09-14T04:37:07.028402+0000","last_fullsized":"2024-09-14T04:37:07.028402+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:46:00.287050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1900,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.031410+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T04:37:07.031410+0000","last_peered":"2024-09-14T04:37:07.031410+0000","last_clean":"2024-09-14T04:37:07.031410+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T04:37:07.031410+0000","last_undegraded":"2024-09-14T04:37:07.031410+0000","last_fullsized":"2024-09-14T04:37:07.031410+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1277,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.843618+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T04:37:05.843618+0000","last_peered":"2024-09-14T04:37:05.843618+0000","last_clean":"2024-09-14T04:37:05.843618+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T04:37:05.843618+0000","last_undegraded":"2024-09-14T04:37:05.843618+0000","last_fullsized":"2024-09-14T04:37:05.843618+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1278,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.845193+0000","last_change":"2024-09-14T04:07:21.718847+0000","last_active":"2024-09-14T04:37:07.845193+0000","last_peered":"2024-09-14T04:37:07.845193+0000","last_clean":"2024-09-14T04:37:07.845193+0000","last_became_active":"2024-09-14T04:07:21.718504+0000","last_became_peered":"2024-09-14T04:07:21.718504+0000","last_unstale":"2024-09-14T04:37:07.845193+0000","last_undegraded":"2024-09-14T04:37:07.845193+0000","last_fullsized":"2024-09-14T04:37:07.845193+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:37:34.405765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1896,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.835037+0000","last_change":"2024-09-14T04:07:58.537946+0000","last_active":"2024-09-14T04:37:07.835037+0000","last_peered":"2024-09-14T04:37:07.835037+0000","last_clean":"2024-09-14T04:37:07.835037+0000","last_became_active":"2024-09-14T04:07:58.536034+0000","last_became_peered":"2024-09-14T04:07:58.536034+0000","last_unstale":"2024-09-14T04:37:07.835037+0000","last_undegraded":"2024-09-14T04:37:07.835037+0000","last_fullsized":"2024-09-14T04:37:07.835037+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:17:18.200470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201845,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":122440,"kb_used_data":15104,"kb_used_omap":1,"kb_used_meta":107326,"kb_avail":93622712,"statfs":{"total":95995035648,"available":95869657088,"internally_reserved":0,"allocated":15466496,"data_stored":9945307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":109902283},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463481,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":126532,"kb_used_data":15096,"kb_used_omap":1,"kb_used_meta":111422,"kb_avail":93618620,"statfs":{"total":95995035648,"available":95865466880,"internally_reserved":0,"allocated":15458304,"data_stored":9945180,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":114096583},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725118,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":113656,"kb_used_data":14508,"kb_used_omap":1,"kb_used_meta":99134,"kb_avail":93631496,"statfs":{"total":95995035648,"available":95878651904,"internally_reserved":0,"allocated":14856192,"data_stored":9350638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":101513670},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986756,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":139908,"kb_used_data":14520,"kb_used_omap":1,"kb_used_meta":125374,"kb_avail":93605244,"statfs":{"total":95995035648,"available":95851769856,"internally_reserved":0,"allocated":14868480,"data_stored":9354869,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":128383430},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772074859,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":86828,"kb_used_data":14500,"kb_used_omap":71,"kb_used_meta":72248,"kb_avail":93658324,"statfs":{"total":95995035648,"available":95906123776,"internally_reserved":0,"allocated":14848000,"data_stored":9350069,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":73982902},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707303794,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":68852,"kb_used_data":14504,"kb_used_omap":68,"kb_used_meta":54267,"kb_avail":93676300,"statfs":{"total":95995035648,"available":95924531200,"internally_reserved":0,"allocated":14852096,"data_stored":9350595,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":55569616},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872728953,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":86772,"kb_used_data":14500,"kb_used_omap":67,"kb_used_meta":72188,"kb_avail":93658380,"statfs":{"total":95995035648,"available":95906181120,"internally_reserved":0,"allocated":14848000,"data_stored":9349998,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":73920869},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056004,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":87164,"kb_used_data":15084,"kb_used_omap":74,"kb_used_meta":71989,"kb_avail":93657988,"statfs":{"total":95995035648,"available":95905779712,"internally_reserved":0,"allocated":15446016,"data_stored":9940951,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":73717429},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:38:05.141 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:38:05.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:04 smithi129 ceph-mon[101316]: pgmap v2136: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:05.349 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:38:06.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:05 smithi039 ceph-mon[106470]: from='client.35058 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:38:06.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:05 smithi039 ceph-mon[111235]: from='client.35058 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:38:06.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:05 smithi129 ceph-mon[101316]: from='client.35058 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:38:07.129 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:06 smithi039 ceph-mon[106470]: pgmap v2137: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:07.130 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:06 smithi039 ceph-mon[111235]: pgmap v2137: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:07.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:06 smithi129 ceph-mon[101316]: pgmap v2137: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:07.412 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:38:07.413 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T04:38:08.055 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":2137,"stamp":"2024-09-14T04:38:05.481016+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":279,"num_write_kb":2449,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":208,"ondisk_log_size":208,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":832152,"kb_used_data":117816,"kb_used_omap":288,"kb_used_meta":713951,"kb_avail":749129064,"statfs":{"total":767960285184,"available":767108161536,"internally_reserved":0,"allocated":120643584,"data_stored":76587607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":294978,"internal_metadata":731086782},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002977"},"pg_stats":[{"pgid":"3.1f","version":"72'4","reported_seq":1910,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.029243+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T04:37:07.029243+0000","last_peered":"2024-09-14T04:37:07.029243+0000","last_clean":"2024-09-14T04:37:07.029243+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T04:37:07.029243+0000","last_undegraded":"2024-09-14T04:37:07.029243+0000","last_fullsized":"2024-09-14T04:37:07.029243+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1274,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116018+0000","last_change":"2024-09-14T03:53:45.836449+0000","last_active":"2024-09-14T04:37:06.116018+0000","last_peered":"2024-09-14T04:37:06.116018+0000","last_clean":"2024-09-14T04:37:06.116018+0000","last_became_active":"2024-09-14T03:53:45.836108+0000","last_became_peered":"2024-09-14T03:53:45.836108+0000","last_unstale":"2024-09-14T04:37:06.116018+0000","last_undegraded":"2024-09-14T04:37:06.116018+0000","last_fullsized":"2024-09-14T04:37:06.116018+0000","mapping_epoch":73,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":74,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:52:13.095225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1899,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.835281+0000","last_change":"2024-09-14T04:07:58.504829+0000","last_active":"2024-09-14T04:37:07.835281+0000","last_peered":"2024-09-14T04:37:07.835281+0000","last_clean":"2024-09-14T04:37:07.835281+0000","last_became_active":"2024-09-14T04:07:58.502623+0000","last_became_peered":"2024-09-14T04:07:58.502623+0000","last_unstale":"2024-09-14T04:37:07.835281+0000","last_undegraded":"2024-09-14T04:37:07.835281+0000","last_fullsized":"2024-09-14T04:37:07.835281+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:59.091993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1911,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.833882+0000","last_change":"2024-09-14T04:07:58.507125+0000","last_active":"2024-09-14T04:37:07.833882+0000","last_peered":"2024-09-14T04:37:07.833882+0000","last_clean":"2024-09-14T04:37:07.833882+0000","last_became_active":"2024-09-14T04:07:58.505701+0000","last_became_peered":"2024-09-14T04:07:58.505701+0000","last_unstale":"2024-09-14T04:37:07.833882+0000","last_undegraded":"2024-09-14T04:37:07.833882+0000","last_fullsized":"2024-09-14T04:37:07.833882+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:25:00.847961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1264,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.843540+0000","last_change":"2024-09-14T03:53:05.234444+0000","last_active":"2024-09-14T04:37:05.843540+0000","last_peered":"2024-09-14T04:37:05.843540+0000","last_clean":"2024-09-14T04:37:05.843540+0000","last_became_active":"2024-09-14T03:53:05.234115+0000","last_became_peered":"2024-09-14T03:53:05.234115+0000","last_unstale":"2024-09-14T04:37:05.843540+0000","last_undegraded":"2024-09-14T04:37:05.843540+0000","last_fullsized":"2024-09-14T04:37:05.843540+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:01:23.153937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1295,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:08.762133+0000","last_change":"2024-09-14T04:07:58.547135+0000","last_active":"2024-09-14T04:37:08.762133+0000","last_peered":"2024-09-14T04:37:08.762133+0000","last_clean":"2024-09-14T04:37:08.762133+0000","last_became_active":"2024-09-14T04:07:58.546751+0000","last_became_peered":"2024-09-14T04:07:58.546751+0000","last_unstale":"2024-09-14T04:37:08.762133+0000","last_undegraded":"2024-09-14T04:37:08.762133+0000","last_fullsized":"2024-09-14T04:37:08.762133+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:01.092698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1279,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.843531+0000","last_change":"2024-09-14T04:06:46.875795+0000","last_active":"2024-09-14T04:37:05.843531+0000","last_peered":"2024-09-14T04:37:05.843531+0000","last_clean":"2024-09-14T04:37:05.843531+0000","last_became_active":"2024-09-14T04:06:46.875395+0000","last_became_peered":"2024-09-14T04:06:46.875395+0000","last_unstale":"2024-09-14T04:37:05.843531+0000","last_undegraded":"2024-09-14T04:37:05.843531+0000","last_fullsized":"2024-09-14T04:37:05.843531+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:51:10.713063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1294,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116345+0000","last_change":"2024-09-14T04:07:58.514349+0000","last_active":"2024-09-14T04:37:06.116345+0000","last_peered":"2024-09-14T04:37:06.116345+0000","last_clean":"2024-09-14T04:37:06.116345+0000","last_became_active":"2024-09-14T04:07:58.513485+0000","last_became_peered":"2024-09-14T04:07:58.513485+0000","last_unstale":"2024-09-14T04:37:06.116345+0000","last_undegraded":"2024-09-14T04:37:06.116345+0000","last_fullsized":"2024-09-14T04:37:06.116345+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:33:09.031813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":1892,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.029098+0000","last_change":"2024-09-14T04:05:54.428057+0000","last_active":"2024-09-14T04:37:07.029098+0000","last_peered":"2024-09-14T04:37:07.029098+0000","last_clean":"2024-09-14T04:37:07.029098+0000","last_became_active":"2024-09-14T04:05:54.414895+0000","last_became_peered":"2024-09-14T04:05:54.414895+0000","last_unstale":"2024-09-14T04:37:07.029098+0000","last_undegraded":"2024-09-14T04:37:07.029098+0000","last_fullsized":"2024-09-14T04:37:07.029098+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:45:15.752675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1907,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.028294+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T04:37:07.028294+0000","last_peered":"2024-09-14T04:37:07.028294+0000","last_clean":"2024-09-14T04:37:07.028294+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T04:37:07.028294+0000","last_undegraded":"2024-09-14T04:37:07.028294+0000","last_fullsized":"2024-09-14T04:37:07.028294+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1297,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:08.762241+0000","last_change":"2024-09-14T04:07:21.717071+0000","last_active":"2024-09-14T04:37:08.762241+0000","last_peered":"2024-09-14T04:37:08.762241+0000","last_clean":"2024-09-14T04:37:08.762241+0000","last_became_active":"2024-09-14T04:07:21.716215+0000","last_became_peered":"2024-09-14T04:07:21.716215+0000","last_unstale":"2024-09-14T04:37:08.762241+0000","last_undegraded":"2024-09-14T04:37:08.762241+0000","last_fullsized":"2024-09-14T04:37:08.762241+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:48:47.175490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1880,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.834576+0000","last_change":"2024-09-14T04:07:58.524371+0000","last_active":"2024-09-14T04:37:07.834576+0000","last_peered":"2024-09-14T04:37:07.834576+0000","last_clean":"2024-09-14T04:37:07.834576+0000","last_became_active":"2024-09-14T04:07:58.524088+0000","last_became_peered":"2024-09-14T04:07:58.524088+0000","last_unstale":"2024-09-14T04:37:07.834576+0000","last_undegraded":"2024-09-14T04:37:07.834576+0000","last_fullsized":"2024-09-14T04:37:07.834576+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:42.615569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1874,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.180210+0000","last_change":"2024-09-14T04:07:58.507553+0000","last_active":"2024-09-14T04:37:07.180210+0000","last_peered":"2024-09-14T04:37:07.180210+0000","last_clean":"2024-09-14T04:37:07.180210+0000","last_became_active":"2024-09-14T04:07:58.507139+0000","last_became_peered":"2024-09-14T04:07:58.507139+0000","last_unstale":"2024-09-14T04:37:07.180210+0000","last_undegraded":"2024-09-14T04:37:07.180210+0000","last_fullsized":"2024-09-14T04:37:07.180210+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:11:57.821769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1905,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.028818+0000","last_change":"2024-09-14T04:07:58.561490+0000","last_active":"2024-09-14T04:37:07.028818+0000","last_peered":"2024-09-14T04:37:07.028818+0000","last_clean":"2024-09-14T04:37:07.028818+0000","last_became_active":"2024-09-14T04:07:58.559841+0000","last_became_peered":"2024-09-14T04:07:58.559841+0000","last_unstale":"2024-09-14T04:37:07.028818+0000","last_undegraded":"2024-09-14T04:37:07.028818+0000","last_fullsized":"2024-09-14T04:37:07.028818+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:27:03.795368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1896,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.833696+0000","last_change":"2024-09-14T04:07:58.555731+0000","last_active":"2024-09-14T04:37:07.833696+0000","last_peered":"2024-09-14T04:37:07.833696+0000","last_clean":"2024-09-14T04:37:07.833696+0000","last_became_active":"2024-09-14T04:07:58.549576+0000","last_became_peered":"2024-09-14T04:07:58.549576+0000","last_unstale":"2024-09-14T04:37:07.833696+0000","last_undegraded":"2024-09-14T04:37:07.833696+0000","last_fullsized":"2024-09-14T04:37:07.833696+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:28:42.339589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1288,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.841667+0000","last_change":"2024-09-14T04:07:21.720797+0000","last_active":"2024-09-14T04:37:05.841667+0000","last_peered":"2024-09-14T04:37:05.841667+0000","last_clean":"2024-09-14T04:37:05.841667+0000","last_became_active":"2024-09-14T04:07:21.719331+0000","last_became_peered":"2024-09-14T04:07:21.719331+0000","last_unstale":"2024-09-14T04:37:05.841667+0000","last_undegraded":"2024-09-14T04:37:05.841667+0000","last_fullsized":"2024-09-14T04:37:05.841667+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:35.404023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1905,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.833897+0000","last_change":"2024-09-14T04:07:58.502533+0000","last_active":"2024-09-14T04:37:07.833897+0000","last_peered":"2024-09-14T04:37:07.833897+0000","last_clean":"2024-09-14T04:37:07.833897+0000","last_became_active":"2024-09-14T04:07:58.499849+0000","last_became_peered":"2024-09-14T04:07:58.499849+0000","last_unstale":"2024-09-14T04:37:07.833897+0000","last_undegraded":"2024-09-14T04:37:07.833897+0000","last_fullsized":"2024-09-14T04:37:07.833897+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:15:35.609746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1900,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.029481+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T04:37:07.029481+0000","last_peered":"2024-09-14T04:37:07.029481+0000","last_clean":"2024-09-14T04:37:07.029481+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T04:37:07.029481+0000","last_undegraded":"2024-09-14T04:37:07.029481+0000","last_fullsized":"2024-09-14T04:37:07.029481+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1881,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.835001+0000","last_change":"2024-09-14T04:07:58.507682+0000","last_active":"2024-09-14T04:37:07.835001+0000","last_peered":"2024-09-14T04:37:07.835001+0000","last_clean":"2024-09-14T04:37:07.835001+0000","last_became_active":"2024-09-14T04:07:58.507365+0000","last_became_peered":"2024-09-14T04:07:58.507365+0000","last_unstale":"2024-09-14T04:37:07.835001+0000","last_undegraded":"2024-09-14T04:37:07.835001+0000","last_fullsized":"2024-09-14T04:37:07.835001+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:14:47.294950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1277,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.843922+0000","last_change":"2024-09-14T04:06:46.875668+0000","last_active":"2024-09-14T04:37:05.843922+0000","last_peered":"2024-09-14T04:37:05.843922+0000","last_clean":"2024-09-14T04:37:05.843922+0000","last_became_active":"2024-09-14T04:06:46.875149+0000","last_became_peered":"2024-09-14T04:06:46.875149+0000","last_unstale":"2024-09-14T04:37:05.843922+0000","last_undegraded":"2024-09-14T04:37:05.843922+0000","last_fullsized":"2024-09-14T04:37:05.843922+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:45:59.190047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1278,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.118067+0000","last_change":"2024-09-14T04:07:58.510699+0000","last_active":"2024-09-14T04:37:06.118067+0000","last_peered":"2024-09-14T04:37:06.118067+0000","last_clean":"2024-09-14T04:37:06.118067+0000","last_became_active":"2024-09-14T04:07:58.506240+0000","last_became_peered":"2024-09-14T04:07:58.506240+0000","last_unstale":"2024-09-14T04:37:06.118067+0000","last_undegraded":"2024-09-14T04:37:06.118067+0000","last_fullsized":"2024-09-14T04:37:06.118067+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:27:18.051658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1907,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.181564+0000","last_change":"2024-09-14T04:07:21.687985+0000","last_active":"2024-09-14T04:37:07.181564+0000","last_peered":"2024-09-14T04:37:07.181564+0000","last_clean":"2024-09-14T04:37:07.181564+0000","last_became_active":"2024-09-14T04:07:21.687670+0000","last_became_peered":"2024-09-14T04:07:21.687670+0000","last_unstale":"2024-09-14T04:37:07.181564+0000","last_undegraded":"2024-09-14T04:37:07.181564+0000","last_fullsized":"2024-09-14T04:37:07.181564+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:53:58.295662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":1290,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:08.762974+0000","last_change":"2024-09-14T03:53:05.244055+0000","last_active":"2024-09-14T04:37:08.762974+0000","last_peered":"2024-09-14T04:37:08.762974+0000","last_clean":"2024-09-14T04:37:08.762974+0000","last_became_active":"2024-09-14T03:53:05.243752+0000","last_became_peered":"2024-09-14T03:53:05.243752+0000","last_unstale":"2024-09-14T04:37:08.762974+0000","last_undegraded":"2024-09-14T04:37:08.762974+0000","last_fullsized":"2024-09-14T04:37:08.762974+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:12:31.850064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1281,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:08.763591+0000","last_change":"2024-09-14T04:06:46.895670+0000","last_active":"2024-09-14T04:37:08.763591+0000","last_peered":"2024-09-14T04:37:08.763591+0000","last_clean":"2024-09-14T04:37:08.763591+0000","last_became_active":"2024-09-14T04:06:46.895370+0000","last_became_peered":"2024-09-14T04:06:46.895370+0000","last_unstale":"2024-09-14T04:37:08.763591+0000","last_undegraded":"2024-09-14T04:37:08.763591+0000","last_fullsized":"2024-09-14T04:37:08.763591+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:48:48.040529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1268,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116350+0000","last_change":"2024-09-14T03:53:06.634043+0000","last_active":"2024-09-14T04:37:06.116350+0000","last_peered":"2024-09-14T04:37:06.116350+0000","last_clean":"2024-09-14T04:37:06.116350+0000","last_became_active":"2024-09-14T03:53:06.633638+0000","last_became_peered":"2024-09-14T03:53:06.633638+0000","last_unstale":"2024-09-14T04:37:06.116350+0000","last_undegraded":"2024-09-14T04:37:06.116350+0000","last_fullsized":"2024-09-14T04:37:06.116350+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:15:53.434064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1292,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:08.946361+0000","last_change":"2024-09-14T04:07:21.721242+0000","last_active":"2024-09-14T04:37:08.946361+0000","last_peered":"2024-09-14T04:37:08.946361+0000","last_clean":"2024-09-14T04:37:08.946361+0000","last_became_active":"2024-09-14T04:07:21.720084+0000","last_became_peered":"2024-09-14T04:07:21.720084+0000","last_unstale":"2024-09-14T04:37:08.946361+0000","last_undegraded":"2024-09-14T04:37:08.946361+0000","last_fullsized":"2024-09-14T04:37:08.946361+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:58:41.308643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":1906,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.028465+0000","last_change":"2024-09-14T04:06:46.932567+0000","last_active":"2024-09-14T04:37:07.028465+0000","last_peered":"2024-09-14T04:37:07.028465+0000","last_clean":"2024-09-14T04:37:07.028465+0000","last_became_active":"2024-09-14T04:06:46.930732+0000","last_became_peered":"2024-09-14T04:06:46.930732+0000","last_unstale":"2024-09-14T04:37:07.028465+0000","last_undegraded":"2024-09-14T04:37:07.028465+0000","last_fullsized":"2024-09-14T04:37:07.028465+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:10:02.729499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1873,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.833825+0000","last_change":"2024-09-14T04:07:58.555446+0000","last_active":"2024-09-14T04:37:07.833825+0000","last_peered":"2024-09-14T04:37:07.833825+0000","last_clean":"2024-09-14T04:37:07.833825+0000","last_became_active":"2024-09-14T04:07:58.548355+0000","last_became_peered":"2024-09-14T04:07:58.548355+0000","last_unstale":"2024-09-14T04:37:07.833825+0000","last_undegraded":"2024-09-14T04:37:07.833825+0000","last_fullsized":"2024-09-14T04:37:07.833825+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:56:28.426094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1911,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.180789+0000","last_change":"2024-09-14T04:06:46.915273+0000","last_active":"2024-09-14T04:37:07.180789+0000","last_peered":"2024-09-14T04:37:07.180789+0000","last_clean":"2024-09-14T04:37:07.180789+0000","last_became_active":"2024-09-14T04:06:46.914994+0000","last_became_peered":"2024-09-14T04:06:46.914994+0000","last_unstale":"2024-09-14T04:37:07.180789+0000","last_undegraded":"2024-09-14T04:37:07.180789+0000","last_fullsized":"2024-09-14T04:37:07.180789+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:48:58.802609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1874,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.835129+0000","last_change":"2024-09-14T04:07:58.564377+0000","last_active":"2024-09-14T04:37:07.835129+0000","last_peered":"2024-09-14T04:37:07.835129+0000","last_clean":"2024-09-14T04:37:07.835129+0000","last_became_active":"2024-09-14T04:07:58.546913+0000","last_became_peered":"2024-09-14T04:07:58.546913+0000","last_unstale":"2024-09-14T04:37:07.835129+0000","last_undegraded":"2024-09-14T04:37:07.835129+0000","last_fullsized":"2024-09-14T04:37:07.835129+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:05:53.637115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1893,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.029265+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T04:37:07.029265+0000","last_peered":"2024-09-14T04:37:07.029265+0000","last_clean":"2024-09-14T04:37:07.029265+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T04:37:07.029265+0000","last_undegraded":"2024-09-14T04:37:07.029265+0000","last_fullsized":"2024-09-14T04:37:07.029265+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1291,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.846686+0000","last_change":"2024-09-14T04:05:54.381035+0000","last_active":"2024-09-14T04:37:07.846686+0000","last_peered":"2024-09-14T04:37:07.846686+0000","last_clean":"2024-09-14T04:37:07.846686+0000","last_became_active":"2024-09-14T04:05:54.379861+0000","last_became_peered":"2024-09-14T04:05:54.379861+0000","last_unstale":"2024-09-14T04:37:07.846686+0000","last_undegraded":"2024-09-14T04:37:07.846686+0000","last_fullsized":"2024-09-14T04:37:07.846686+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:07:46.302925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1911,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.182349+0000","last_change":"2024-09-14T04:07:21.686880+0000","last_active":"2024-09-14T04:37:07.182349+0000","last_peered":"2024-09-14T04:37:07.182349+0000","last_clean":"2024-09-14T04:37:07.182349+0000","last_became_active":"2024-09-14T04:07:21.685558+0000","last_became_peered":"2024-09-14T04:07:21.685558+0000","last_unstale":"2024-09-14T04:37:07.182349+0000","last_undegraded":"2024-09-14T04:37:07.182349+0000","last_fullsized":"2024-09-14T04:37:07.182349+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:24:18.693428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1899,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.181428+0000","last_change":"2024-09-14T04:06:46.905931+0000","last_active":"2024-09-14T04:37:07.181428+0000","last_peered":"2024-09-14T04:37:07.181428+0000","last_clean":"2024-09-14T04:37:07.181428+0000","last_became_active":"2024-09-14T04:06:46.905666+0000","last_became_peered":"2024-09-14T04:06:46.905666+0000","last_unstale":"2024-09-14T04:37:07.181428+0000","last_undegraded":"2024-09-14T04:37:07.181428+0000","last_fullsized":"2024-09-14T04:37:07.181428+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:40:57.007948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1928,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.182492+0000","last_change":"2024-09-14T04:06:46.930482+0000","last_active":"2024-09-14T04:37:07.182492+0000","last_peered":"2024-09-14T04:37:07.182492+0000","last_clean":"2024-09-14T04:37:07.182492+0000","last_became_active":"2024-09-14T04:06:46.927372+0000","last_became_peered":"2024-09-14T04:06:46.927372+0000","last_unstale":"2024-09-14T04:37:07.182492+0000","last_undegraded":"2024-09-14T04:37:07.182492+0000","last_fullsized":"2024-09-14T04:37:07.182492+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:07:22.159128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1296,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.843694+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T04:37:05.843694+0000","last_peered":"2024-09-14T04:37:05.843694+0000","last_clean":"2024-09-14T04:37:05.843694+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T04:37:05.843694+0000","last_undegraded":"2024-09-14T04:37:05.843694+0000","last_fullsized":"2024-09-14T04:37:05.843694+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1921,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.182437+0000","last_change":"2024-09-14T04:07:58.510721+0000","last_active":"2024-09-14T04:37:07.182437+0000","last_peered":"2024-09-14T04:37:07.182437+0000","last_clean":"2024-09-14T04:37:07.182437+0000","last_became_active":"2024-09-14T04:07:58.510332+0000","last_became_peered":"2024-09-14T04:07:58.510332+0000","last_unstale":"2024-09-14T04:37:07.182437+0000","last_undegraded":"2024-09-14T04:37:07.182437+0000","last_fullsized":"2024-09-14T04:37:07.182437+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:07:03.926842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1284,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116506+0000","last_change":"2024-09-14T04:05:54.426939+0000","last_active":"2024-09-14T04:37:06.116506+0000","last_peered":"2024-09-14T04:37:06.116506+0000","last_clean":"2024-09-14T04:37:06.116506+0000","last_became_active":"2024-09-14T04:05:54.426427+0000","last_became_peered":"2024-09-14T04:05:54.426427+0000","last_unstale":"2024-09-14T04:37:06.116506+0000","last_undegraded":"2024-09-14T04:37:06.116506+0000","last_fullsized":"2024-09-14T04:37:06.116506+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:53:55.841489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1295,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:08.602276+0000","last_change":"2024-09-14T04:06:46.875340+0000","last_active":"2024-09-14T04:37:08.602276+0000","last_peered":"2024-09-14T04:37:08.602276+0000","last_clean":"2024-09-14T04:37:08.602276+0000","last_became_active":"2024-09-14T04:06:46.874552+0000","last_became_peered":"2024-09-14T04:06:46.874552+0000","last_unstale":"2024-09-14T04:37:08.602276+0000","last_undegraded":"2024-09-14T04:37:08.602276+0000","last_fullsized":"2024-09-14T04:37:08.602276+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:52.257357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1907,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.180965+0000","last_change":"2024-09-14T04:06:46.917681+0000","last_active":"2024-09-14T04:37:07.180965+0000","last_peered":"2024-09-14T04:37:07.180965+0000","last_clean":"2024-09-14T04:37:07.180965+0000","last_became_active":"2024-09-14T04:06:46.917401+0000","last_became_peered":"2024-09-14T04:06:46.917401+0000","last_unstale":"2024-09-14T04:37:07.180965+0000","last_undegraded":"2024-09-14T04:37:07.180965+0000","last_fullsized":"2024-09-14T04:37:07.180965+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:04:39.949490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1911,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.180136+0000","last_change":"2024-09-14T04:07:21.685415+0000","last_active":"2024-09-14T04:37:07.180136+0000","last_peered":"2024-09-14T04:37:07.180136+0000","last_clean":"2024-09-14T04:37:07.180136+0000","last_became_active":"2024-09-14T04:07:21.683504+0000","last_became_peered":"2024-09-14T04:07:21.683504+0000","last_unstale":"2024-09-14T04:37:07.180136+0000","last_undegraded":"2024-09-14T04:37:07.180136+0000","last_fullsized":"2024-09-14T04:37:07.180136+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:21:08.806291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1913,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.027850+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T04:37:07.027850+0000","last_peered":"2024-09-14T04:37:07.027850+0000","last_clean":"2024-09-14T04:37:07.027850+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T04:37:07.027850+0000","last_undegraded":"2024-09-14T04:37:07.027850+0000","last_fullsized":"2024-09-14T04:37:07.027850+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1914,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.028872+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T04:37:07.028872+0000","last_peered":"2024-09-14T04:37:07.028872+0000","last_clean":"2024-09-14T04:37:07.028872+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T04:37:07.028872+0000","last_undegraded":"2024-09-14T04:37:07.028872+0000","last_fullsized":"2024-09-14T04:37:07.028872+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1903,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.028102+0000","last_change":"2024-09-14T04:07:58.521344+0000","last_active":"2024-09-14T04:37:07.028102+0000","last_peered":"2024-09-14T04:37:07.028102+0000","last_clean":"2024-09-14T04:37:07.028102+0000","last_became_active":"2024-09-14T04:07:58.520965+0000","last_became_peered":"2024-09-14T04:07:58.520965+0000","last_unstale":"2024-09-14T04:37:07.028102+0000","last_undegraded":"2024-09-14T04:37:07.028102+0000","last_fullsized":"2024-09-14T04:37:07.028102+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:58:46.029013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1307,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.845573+0000","last_change":"2024-09-14T04:06:46.857436+0000","last_active":"2024-09-14T04:37:07.845573+0000","last_peered":"2024-09-14T04:37:07.845573+0000","last_clean":"2024-09-14T04:37:07.845573+0000","last_became_active":"2024-09-14T04:06:46.857154+0000","last_became_peered":"2024-09-14T04:06:46.857154+0000","last_unstale":"2024-09-14T04:37:07.845573+0000","last_undegraded":"2024-09-14T04:37:07.845573+0000","last_fullsized":"2024-09-14T04:37:07.845573+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:53:28.772152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1264,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.841559+0000","last_change":"2024-09-14T03:53:05.551268+0000","last_active":"2024-09-14T04:37:05.841559+0000","last_peered":"2024-09-14T04:37:05.841559+0000","last_clean":"2024-09-14T04:37:05.841559+0000","last_became_active":"2024-09-14T03:53:05.551015+0000","last_became_peered":"2024-09-14T03:53:05.551015+0000","last_unstale":"2024-09-14T04:37:05.841559+0000","last_undegraded":"2024-09-14T04:37:05.841559+0000","last_fullsized":"2024-09-14T04:37:05.841559+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.905475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1885,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.834431+0000","last_change":"2024-09-14T04:07:58.503896+0000","last_active":"2024-09-14T04:37:07.834431+0000","last_peered":"2024-09-14T04:37:07.834431+0000","last_clean":"2024-09-14T04:37:07.834431+0000","last_became_active":"2024-09-14T04:07:58.499253+0000","last_became_peered":"2024-09-14T04:07:58.499253+0000","last_unstale":"2024-09-14T04:37:07.834431+0000","last_undegraded":"2024-09-14T04:37:07.834431+0000","last_fullsized":"2024-09-14T04:37:07.834431+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:48:27.543818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1294,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:08.602592+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T04:37:08.602592+0000","last_peered":"2024-09-14T04:37:08.602592+0000","last_clean":"2024-09-14T04:37:08.602592+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T04:37:08.602592+0000","last_undegraded":"2024-09-14T04:37:08.602592+0000","last_fullsized":"2024-09-14T04:37:08.602592+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":1609,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.845285+0000","last_change":"2024-09-14T04:05:54.389043+0000","last_active":"2024-09-14T04:37:07.845285+0000","last_peered":"2024-09-14T04:37:07.845285+0000","last_clean":"2024-09-14T04:37:07.845285+0000","last_became_active":"2024-09-14T04:05:54.388655+0000","last_became_peered":"2024-09-14T04:05:54.388655+0000","last_unstale":"2024-09-14T04:37:07.845285+0000","last_undegraded":"2024-09-14T04:37:07.845285+0000","last_fullsized":"2024-09-14T04:37:07.845285+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T03:49:18.237706+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:38:59.615597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1875,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.180822+0000","last_change":"2024-09-14T04:07:58.530406+0000","last_active":"2024-09-14T04:37:07.180822+0000","last_peered":"2024-09-14T04:37:07.180822+0000","last_clean":"2024-09-14T04:37:07.180822+0000","last_became_active":"2024-09-14T04:07:58.530078+0000","last_became_peered":"2024-09-14T04:07:58.530078+0000","last_unstale":"2024-09-14T04:37:07.180822+0000","last_undegraded":"2024-09-14T04:37:07.180822+0000","last_fullsized":"2024-09-14T04:37:07.180822+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:54:53.524001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1883,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.833696+0000","last_change":"2024-09-14T04:07:58.555717+0000","last_active":"2024-09-14T04:37:07.833696+0000","last_peered":"2024-09-14T04:37:07.833696+0000","last_clean":"2024-09-14T04:37:07.833696+0000","last_became_active":"2024-09-14T04:07:58.548894+0000","last_became_peered":"2024-09-14T04:07:58.548894+0000","last_unstale":"2024-09-14T04:37:07.833696+0000","last_undegraded":"2024-09-14T04:37:07.833696+0000","last_fullsized":"2024-09-14T04:37:07.833696+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:42.061551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1281,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116836+0000","last_change":"2024-09-14T04:06:46.894741+0000","last_active":"2024-09-14T04:37:06.116836+0000","last_peered":"2024-09-14T04:37:06.116836+0000","last_clean":"2024-09-14T04:37:06.116836+0000","last_became_active":"2024-09-14T04:06:46.894445+0000","last_became_peered":"2024-09-14T04:06:46.894445+0000","last_unstale":"2024-09-14T04:37:06.116836+0000","last_undegraded":"2024-09-14T04:37:06.116836+0000","last_fullsized":"2024-09-14T04:37:06.116836+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:52:00.055271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1283,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116551+0000","last_change":"2024-09-14T04:07:21.715462+0000","last_active":"2024-09-14T04:37:06.116551+0000","last_peered":"2024-09-14T04:37:06.116551+0000","last_clean":"2024-09-14T04:37:06.116551+0000","last_became_active":"2024-09-14T04:07:21.713032+0000","last_became_peered":"2024-09-14T04:07:21.713032+0000","last_unstale":"2024-09-14T04:37:06.116551+0000","last_undegraded":"2024-09-14T04:37:06.116551+0000","last_fullsized":"2024-09-14T04:37:06.116551+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:37:05.545164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1296,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116624+0000","last_change":"2024-09-14T04:07:21.718037+0000","last_active":"2024-09-14T04:37:06.116624+0000","last_peered":"2024-09-14T04:37:06.116624+0000","last_clean":"2024-09-14T04:37:06.116624+0000","last_became_active":"2024-09-14T04:07:21.717700+0000","last_became_peered":"2024-09-14T04:07:21.717700+0000","last_unstale":"2024-09-14T04:37:06.116624+0000","last_undegraded":"2024-09-14T04:37:06.116624+0000","last_fullsized":"2024-09-14T04:37:06.116624+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:47:33.483785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1279,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.841296+0000","last_change":"2024-09-14T04:06:46.868780+0000","last_active":"2024-09-14T04:37:05.841296+0000","last_peered":"2024-09-14T04:37:05.841296+0000","last_clean":"2024-09-14T04:37:05.841296+0000","last_became_active":"2024-09-14T04:06:46.868483+0000","last_became_peered":"2024-09-14T04:06:46.868483+0000","last_unstale":"2024-09-14T04:37:05.841296+0000","last_undegraded":"2024-09-14T04:37:05.841296+0000","last_fullsized":"2024-09-14T04:37:05.841296+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:03.133158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1281,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.117264+0000","last_change":"2024-09-14T04:07:58.515859+0000","last_active":"2024-09-14T04:37:06.117264+0000","last_peered":"2024-09-14T04:37:06.117264+0000","last_clean":"2024-09-14T04:37:06.117264+0000","last_became_active":"2024-09-14T04:07:58.515594+0000","last_became_peered":"2024-09-14T04:07:58.515594+0000","last_unstale":"2024-09-14T04:37:06.117264+0000","last_undegraded":"2024-09-14T04:37:06.117264+0000","last_fullsized":"2024-09-14T04:37:06.117264+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:45.966798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":1900,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.833729+0000","last_change":"2024-09-14T04:07:58.509833+0000","last_active":"2024-09-14T04:37:07.833729+0000","last_peered":"2024-09-14T04:37:07.833729+0000","last_clean":"2024-09-14T04:37:07.833729+0000","last_became_active":"2024-09-14T04:07:58.509198+0000","last_became_peered":"2024-09-14T04:07:58.509198+0000","last_unstale":"2024-09-14T04:37:07.833729+0000","last_undegraded":"2024-09-14T04:37:07.833729+0000","last_fullsized":"2024-09-14T04:37:07.833729+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:24:40.193090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1903,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.028840+0000","last_change":"2024-09-14T04:06:46.925323+0000","last_active":"2024-09-14T04:37:07.028840+0000","last_peered":"2024-09-14T04:37:07.028840+0000","last_clean":"2024-09-14T04:37:07.028840+0000","last_became_active":"2024-09-14T04:06:46.925043+0000","last_became_peered":"2024-09-14T04:06:46.925043+0000","last_unstale":"2024-09-14T04:37:07.028840+0000","last_undegraded":"2024-09-14T04:37:07.028840+0000","last_fullsized":"2024-09-14T04:37:07.028840+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:39.895453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1279,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.843564+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T04:37:05.843564+0000","last_peered":"2024-09-14T04:37:05.843564+0000","last_clean":"2024-09-14T04:37:05.843564+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T04:37:05.843564+0000","last_undegraded":"2024-09-14T04:37:05.843564+0000","last_fullsized":"2024-09-14T04:37:05.843564+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1278,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.841209+0000","last_change":"2024-09-14T04:06:46.869868+0000","last_active":"2024-09-14T04:37:05.841209+0000","last_peered":"2024-09-14T04:37:05.841209+0000","last_clean":"2024-09-14T04:37:05.841209+0000","last_became_active":"2024-09-14T04:06:46.869126+0000","last_became_peered":"2024-09-14T04:06:46.869126+0000","last_unstale":"2024-09-14T04:37:05.841209+0000","last_undegraded":"2024-09-14T04:37:05.841209+0000","last_fullsized":"2024-09-14T04:37:05.841209+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:41.150138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1262,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.118225+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T04:37:06.118225+0000","last_peered":"2024-09-14T04:37:06.118225+0000","last_clean":"2024-09-14T04:37:06.118225+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T04:37:06.118225+0000","last_undegraded":"2024-09-14T04:37:06.118225+0000","last_fullsized":"2024-09-14T04:37:06.118225+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1297,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.116472+0000","last_change":"2024-09-14T04:07:58.508202+0000","last_active":"2024-09-14T04:37:06.116472+0000","last_peered":"2024-09-14T04:37:06.116472+0000","last_clean":"2024-09-14T04:37:06.116472+0000","last_became_active":"2024-09-14T04:07:58.507929+0000","last_became_peered":"2024-09-14T04:07:58.507929+0000","last_unstale":"2024-09-14T04:37:06.116472+0000","last_undegraded":"2024-09-14T04:37:06.116472+0000","last_fullsized":"2024-09-14T04:37:06.116472+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:17:58.160600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1278,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.118124+0000","last_change":"2024-09-14T04:06:46.858890+0000","last_active":"2024-09-14T04:37:06.118124+0000","last_peered":"2024-09-14T04:37:06.118124+0000","last_clean":"2024-09-14T04:37:06.118124+0000","last_became_active":"2024-09-14T04:06:46.858569+0000","last_became_peered":"2024-09-14T04:06:46.858569+0000","last_unstale":"2024-09-14T04:37:06.118124+0000","last_undegraded":"2024-09-14T04:37:06.118124+0000","last_fullsized":"2024-09-14T04:37:06.118124+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:25:46.400115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1264,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.843281+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T04:37:05.843281+0000","last_peered":"2024-09-14T04:37:05.843281+0000","last_clean":"2024-09-14T04:37:05.843281+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T04:37:05.843281+0000","last_undegraded":"2024-09-14T04:37:05.843281+0000","last_fullsized":"2024-09-14T04:37:05.843281+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1277,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:06.118257+0000","last_change":"2024-09-14T04:05:54.419435+0000","last_active":"2024-09-14T04:37:06.118257+0000","last_peered":"2024-09-14T04:37:06.118257+0000","last_clean":"2024-09-14T04:37:06.118257+0000","last_became_active":"2024-09-14T04:05:54.419094+0000","last_became_peered":"2024-09-14T04:05:54.419094+0000","last_unstale":"2024-09-14T04:37:06.118257+0000","last_undegraded":"2024-09-14T04:37:06.118257+0000","last_fullsized":"2024-09-14T04:37:06.118257+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:00.162629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1286,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:08.945667+0000","last_change":"2024-09-14T04:05:54.418813+0000","last_active":"2024-09-14T04:37:08.945667+0000","last_peered":"2024-09-14T04:37:08.945667+0000","last_clean":"2024-09-14T04:37:08.945667+0000","last_became_active":"2024-09-14T04:05:54.417584+0000","last_became_peered":"2024-09-14T04:05:54.417584+0000","last_unstale":"2024-09-14T04:37:08.945667+0000","last_undegraded":"2024-09-14T04:37:08.945667+0000","last_fullsized":"2024-09-14T04:37:08.945667+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:13:16.209456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1880,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.835033+0000","last_change":"2024-09-14T04:07:58.565442+0000","last_active":"2024-09-14T04:37:07.835033+0000","last_peered":"2024-09-14T04:37:07.835033+0000","last_clean":"2024-09-14T04:37:07.835033+0000","last_became_active":"2024-09-14T04:07:58.552289+0000","last_became_peered":"2024-09-14T04:07:58.552289+0000","last_unstale":"2024-09-14T04:37:07.835033+0000","last_undegraded":"2024-09-14T04:37:07.835033+0000","last_fullsized":"2024-09-14T04:37:07.835033+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:48:02.364960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1261,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.845043+0000","last_change":"2024-09-14T03:53:05.244003+0000","last_active":"2024-09-14T04:37:07.845043+0000","last_peered":"2024-09-14T04:37:07.845043+0000","last_clean":"2024-09-14T04:37:07.845043+0000","last_became_active":"2024-09-14T03:53:05.240743+0000","last_became_peered":"2024-09-14T03:53:05.240743+0000","last_unstale":"2024-09-14T04:37:07.845043+0000","last_undegraded":"2024-09-14T04:37:07.845043+0000","last_fullsized":"2024-09-14T04:37:07.845043+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:30.000189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1901,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.028402+0000","last_change":"2024-09-14T04:07:58.560880+0000","last_active":"2024-09-14T04:37:07.028402+0000","last_peered":"2024-09-14T04:37:07.028402+0000","last_clean":"2024-09-14T04:37:07.028402+0000","last_became_active":"2024-09-14T04:07:58.560132+0000","last_became_peered":"2024-09-14T04:07:58.560132+0000","last_unstale":"2024-09-14T04:37:07.028402+0000","last_undegraded":"2024-09-14T04:37:07.028402+0000","last_fullsized":"2024-09-14T04:37:07.028402+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:46:00.287050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1900,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.031410+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T04:37:07.031410+0000","last_peered":"2024-09-14T04:37:07.031410+0000","last_clean":"2024-09-14T04:37:07.031410+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T04:37:07.031410+0000","last_undegraded":"2024-09-14T04:37:07.031410+0000","last_fullsized":"2024-09-14T04:37:07.031410+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1277,"reported_epoch":1312,"state":"active+clean","last_fresh":"2024-09-14T04:37:05.843618+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T04:37:05.843618+0000","last_peered":"2024-09-14T04:37:05.843618+0000","last_clean":"2024-09-14T04:37:05.843618+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T04:37:05.843618+0000","last_undegraded":"2024-09-14T04:37:05.843618+0000","last_fullsized":"2024-09-14T04:37:05.843618+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1278,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.845193+0000","last_change":"2024-09-14T04:07:21.718847+0000","last_active":"2024-09-14T04:37:07.845193+0000","last_peered":"2024-09-14T04:37:07.845193+0000","last_clean":"2024-09-14T04:37:07.845193+0000","last_became_active":"2024-09-14T04:07:21.718504+0000","last_became_peered":"2024-09-14T04:07:21.718504+0000","last_unstale":"2024-09-14T04:37:07.845193+0000","last_undegraded":"2024-09-14T04:37:07.845193+0000","last_fullsized":"2024-09-14T04:37:07.845193+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:37:34.405765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1896,"reported_epoch":1313,"state":"active+clean","last_fresh":"2024-09-14T04:37:07.835037+0000","last_change":"2024-09-14T04:07:58.537946+0000","last_active":"2024-09-14T04:37:07.835037+0000","last_peered":"2024-09-14T04:37:07.835037+0000","last_clean":"2024-09-14T04:37:07.835037+0000","last_became_active":"2024-09-14T04:07:58.536034+0000","last_became_peered":"2024-09-14T04:07:58.536034+0000","last_unstale":"2024-09-14T04:37:07.835037+0000","last_undegraded":"2024-09-14T04:37:07.835037+0000","last_fullsized":"2024-09-14T04:37:07.835037+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:17:18.200470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201845,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":122440,"kb_used_data":15104,"kb_used_omap":1,"kb_used_meta":107326,"kb_avail":93622712,"statfs":{"total":95995035648,"available":95869657088,"internally_reserved":0,"allocated":15466496,"data_stored":9945307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":109902283},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463481,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":126532,"kb_used_data":15096,"kb_used_omap":1,"kb_used_meta":111422,"kb_avail":93618620,"statfs":{"total":95995035648,"available":95865466880,"internally_reserved":0,"allocated":15458304,"data_stored":9945180,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":114096583},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725118,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":113656,"kb_used_data":14508,"kb_used_omap":1,"kb_used_meta":99134,"kb_avail":93631496,"statfs":{"total":95995035648,"available":95878651904,"internally_reserved":0,"allocated":14856192,"data_stored":9350638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":101513670},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986756,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":139908,"kb_used_data":14520,"kb_used_omap":1,"kb_used_meta":125374,"kb_avail":93605244,"statfs":{"total":95995035648,"available":95851769856,"internally_reserved":0,"allocated":14868480,"data_stored":9354869,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":128383430},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772074859,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":86828,"kb_used_data":14500,"kb_used_omap":71,"kb_used_meta":72248,"kb_avail":93658324,"statfs":{"total":95995035648,"available":95906123776,"internally_reserved":0,"allocated":14848000,"data_stored":9350069,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":73982902},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707303795,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":68852,"kb_used_data":14504,"kb_used_omap":68,"kb_used_meta":54267,"kb_avail":93676300,"statfs":{"total":95995035648,"available":95924531200,"internally_reserved":0,"allocated":14852096,"data_stored":9350595,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":55569616},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872728954,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":86772,"kb_used_data":14500,"kb_used_omap":67,"kb_used_meta":72188,"kb_avail":93658380,"statfs":{"total":95995035648,"available":95906181120,"internally_reserved":0,"allocated":14848000,"data_stored":9349998,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":73920869},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056004,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":87164,"kb_used_data":15084,"kb_used_omap":74,"kb_used_meta":71989,"kb_avail":93657988,"statfs":{"total":95995035648,"available":95905779712,"internally_reserved":0,"allocated":15446016,"data_stored":9940951,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":73717429},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:38:08.059 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-09-14T04:38:08.059 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd dump 2024-09-14T04:38:08.269 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:38:08.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:38:08.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:38:08.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:38:08.975 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:38:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:38:08] "GET /metrics HTTP/1.1" 200 34259 "" "Prometheus/2.51.0" 2024-09-14T04:38:09.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:08 smithi129 ceph-mon[101316]: from='client.35064 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:38:09.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:08 smithi129 ceph-mon[101316]: pgmap v2138: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:09.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:08 smithi039 ceph-mon[106470]: from='client.35064 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:38:09.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:08 smithi039 ceph-mon[106470]: pgmap v2138: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:09.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:08 smithi039 ceph-mon[111235]: from='client.35064 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:38:09.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:08 smithi039 ceph-mon[111235]: pgmap v2138: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:10.195 INFO:teuthology.orchestra.run.smithi039.stdout:epoch 1313 2024-09-14T04:38:10.195 INFO:teuthology.orchestra.run.smithi039.stdout:fsid bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T04:38:10.196 INFO:teuthology.orchestra.run.smithi039.stdout:created 2024-09-14T03:46:13.726476+0000 2024-09-14T04:38:10.196 INFO:teuthology.orchestra.run.smithi039.stdout:modified 2024-09-14T04:37:06.997883+0000 2024-09-14T04:38:10.196 INFO:teuthology.orchestra.run.smithi039.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2024-09-14T04:38:10.196 INFO:teuthology.orchestra.run.smithi039.stdout:crush_version 22 2024-09-14T04:38:10.196 INFO:teuthology.orchestra.run.smithi039.stdout:full_ratio 0.95 2024-09-14T04:38:10.196 INFO:teuthology.orchestra.run.smithi039.stdout:backfillfull_ratio 0.9 2024-09-14T04:38:10.196 INFO:teuthology.orchestra.run.smithi039.stdout:nearfull_ratio 0.85 2024-09-14T04:38:10.196 INFO:teuthology.orchestra.run.smithi039.stdout:require_min_compat_client luminous 2024-09-14T04:38:10.196 INFO:teuthology.orchestra.run.smithi039.stdout:min_compat_client luminous 2024-09-14T04:38:10.196 INFO:teuthology.orchestra.run.smithi039.stdout:require_osd_release reef 2024-09-14T04:38:10.196 INFO:teuthology.orchestra.run.smithi039.stdout:stretch_mode_enabled false 2024-09-14T04:38:10.196 INFO:teuthology.orchestra.run.smithi039.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 22 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89 2024-09-14T04:38:10.197 INFO:teuthology.orchestra.run.smithi039.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 66 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd read_balance_score 2.00 2024-09-14T04:38:10.197 INFO:teuthology.orchestra.run.smithi039.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 71 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2024-09-14T04:38:10.197 INFO:teuthology.orchestra.run.smithi039.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 72 flags hashpspool,bulk stripe_width 0 application cephfs read_balance_score 2.25 2024-09-14T04:38:10.197 INFO:teuthology.orchestra.run.smithi039.stdout:max_osd 8 2024-09-14T04:38:10.197 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0 up in weight 1 up_from 671 up_thru 1165 down_at 667 last_clean_interval [8,667) [v2:172.21.15.39:6802/3832421293,v1:172.21.15.39:6803/3832421293] [v2:172.21.15.39:6804/3832421293,v1:172.21.15.39:6805/3832421293] exists,up 5fdfd478-dd27-4674-a7d6-53177d9e549a 2024-09-14T04:38:10.197 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1 up in weight 1 up_from 681 up_thru 1165 down_at 677 last_clean_interval [13,677) [v2:172.21.15.39:6810/1186072955,v1:172.21.15.39:6811/1186072955] [v2:172.21.15.39:6812/1186072955,v1:172.21.15.39:6813/1186072955] exists,up 991c236e-1679-4c1e-a1a9-3894b1b12055 2024-09-14T04:38:10.197 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2 up in weight 1 up_from 694 up_thru 1165 down_at 688 last_clean_interval [18,688) [v2:172.21.15.39:6818/2359749690,v1:172.21.15.39:6819/2359749690] [v2:172.21.15.39:6820/2359749690,v1:172.21.15.39:6821/2359749690] exists,up f9611ffd-5d37-4465-8114-ecc70d4d6d14 2024-09-14T04:38:10.197 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3 up in weight 1 up_from 701 up_thru 1165 down_at 697 last_clean_interval [26,697) [v2:172.21.15.39:6826/2092194762,v1:172.21.15.39:6827/2092194762] [v2:172.21.15.39:6828/2092194762,v1:172.21.15.39:6829/2092194762] exists,up 56b6877b-2107-47d6-8da9-015db7352288 2024-09-14T04:38:10.197 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4 up in weight 1 up_from 31 up_thru 1165 down_at 0 last_clean_interval [0,0) [v2:172.21.15.129:6800/168261430,v1:172.21.15.129:6801/168261430] [v2:172.21.15.129:6802/168261430,v1:172.21.15.129:6803/168261430] exists,up 2d5523c4-5871-4fb3-bc0d-62c3b7d8b6a1 2024-09-14T04:38:10.197 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5 up in weight 1 up_from 39 up_thru 1165 down_at 0 last_clean_interval [0,0) [v2:172.21.15.129:6808/2184993002,v1:172.21.15.129:6809/2184993002] [v2:172.21.15.129:6810/2184993002,v1:172.21.15.129:6811/2184993002] exists,up a3ba55fd-f32e-4f6d-9b48-d2845a3474cd 2024-09-14T04:38:10.197 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6 up in weight 1 up_from 47 up_thru 1165 down_at 0 last_clean_interval [0,0) [v2:172.21.15.129:6816/3767031353,v1:172.21.15.129:6817/3767031353] [v2:172.21.15.129:6818/3767031353,v1:172.21.15.129:6819/3767031353] exists,up 79e9c0a2-86a4-4267-a799-059d473d7b74 2024-09-14T04:38:10.197 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7 up in weight 1 up_from 55 up_thru 1165 down_at 0 last_clean_interval [0,0) [v2:172.21.15.129:6824/2370414690,v1:172.21.15.129:6825/2370414690] [v2:172.21.15.129:6826/2370414690,v1:172.21.15.129:6827/2370414690] exists,up 22f41628-4b88-4b8c-82b2-fcd2802da1bf 2024-09-14T04:38:10.197 INFO:teuthology.orchestra.run.smithi039.stdout:pg_upmap_items 4.10 [1,2] 2024-09-14T04:38:10.197 INFO:teuthology.orchestra.run.smithi039.stdout:pg_upmap_items 4.18 [1,5] 2024-09-14T04:38:10.198 INFO:teuthology.orchestra.run.smithi039.stdout:pg_upmap_items 4.1f [1,2] 2024-09-14T04:38:10.198 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.39:0/1864073859 expires 2024-09-14T05:12:12.455045+0000 2024-09-14T04:38:10.198 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.39:0/1958359945 expires 2024-09-14T05:02:41.504544+0000 2024-09-14T04:38:10.198 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.39:0/2766717202 expires 2024-09-14T05:02:40.464304+0000 2024-09-14T04:38:10.198 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.39:0/1825825785 expires 2024-09-15T03:55:55.288048+0000 2024-09-14T04:38:10.198 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.129:0/3885028850 expires 2024-09-15T03:56:26.175569+0000 2024-09-14T04:38:10.198 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.129:6835/1529931471 expires 2024-09-15T03:56:26.175569+0000 2024-09-14T04:38:10.198 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.39:0/822851906 expires 2024-09-15T03:59:06.707728+0000 2024-09-14T04:38:10.198 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.39:6800/748677190 expires 2024-09-15T03:51:44.726801+0000 2024-09-14T04:38:10.198 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.39:0/3247815370 expires 2024-09-15T03:59:06.707728+0000 2024-09-14T04:38:10.198 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.129:0/78252352 expires 2024-09-15T03:56:26.175569+0000 2024-09-14T04:38:10.198 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.39:0/1530637819 expires 2024-09-15T03:46:59.511580+0000 2024-09-14T04:38:10.198 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.39:6801/628437919 expires 2024-09-15T03:46:35.805097+0000 2024-09-14T04:38:10.198 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.39:6801/748677190 expires 2024-09-15T03:51:44.726801+0000 2024-09-14T04:38:10.199 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.39:0/2745905728 expires 2024-09-15T03:51:44.726801+0000 2024-09-14T04:38:10.199 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.129:0/2660542464 expires 2024-09-15T03:56:26.175569+0000 2024-09-14T04:38:10.199 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.39:0/209353267 expires 2024-09-15T03:46:59.511580+0000 2024-09-14T04:38:10.199 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.39:6800/3127470450 expires 2024-09-15T03:46:59.511580+0000 2024-09-14T04:38:10.199 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.129:0/1000360058 expires 2024-09-15T03:56:26.175569+0000 2024-09-14T04:38:10.199 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.39:0/2670041591 expires 2024-09-15T03:46:35.805097+0000 2024-09-14T04:38:10.199 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.39:6801/3924974081 expires 2024-09-15T03:55:55.288048+0000 2024-09-14T04:38:10.199 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.39:6800/3924974081 expires 2024-09-15T03:55:55.288048+0000 2024-09-14T04:38:10.199 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.39:0/2951004223 expires 2024-09-15T03:51:44.726801+0000 2024-09-14T04:38:10.199 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.39:0/3196173579 expires 2024-09-15T03:55:55.288048+0000 2024-09-14T04:38:10.199 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.39:0/502495021 expires 2024-09-15T03:55:55.288048+0000 2024-09-14T04:38:10.199 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.39:0/3161547781 expires 2024-09-15T03:46:59.511580+0000 2024-09-14T04:38:10.199 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.39:0/1398136686 expires 2024-09-15T03:59:06.707728+0000 2024-09-14T04:38:10.200 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.39:6801/3127470450 expires 2024-09-15T03:46:59.511580+0000 2024-09-14T04:38:10.200 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.39:0/2133325947 expires 2024-09-15T03:46:35.805097+0000 2024-09-14T04:38:10.200 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.39:6800/1103763596 expires 2024-09-15T03:59:06.707728+0000 2024-09-14T04:38:10.200 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.129:6834/1529931471 expires 2024-09-15T03:56:26.175569+0000 2024-09-14T04:38:10.200 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.39:0/1746545094 expires 2024-09-15T03:59:06.707728+0000 2024-09-14T04:38:10.200 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.129:0/1993195691 expires 2024-09-15T03:56:26.175569+0000 2024-09-14T04:38:10.200 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.39:6801/1103763596 expires 2024-09-15T03:59:06.707728+0000 2024-09-14T04:38:10.200 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.39:0/754571920 expires 2024-09-15T03:51:44.726801+0000 2024-09-14T04:38:10.200 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.39:0/81449629 expires 2024-09-15T03:46:35.805097+0000 2024-09-14T04:38:10.200 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.39:6800/628437919 expires 2024-09-15T03:46:35.805097+0000 2024-09-14T04:38:10.200 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.129:0/3477521861 expires 2024-09-15T03:56:26.175569+0000 2024-09-14T04:38:10.200 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.39:0/2587896825 expires 2024-09-15T03:51:44.726801+0000 2024-09-14T04:38:10.200 INFO:teuthology.orchestra.run.smithi039.stdout:blocklist 172.21.15.39:0/1766366412 expires 2024-09-15T03:55:55.288048+0000 2024-09-14T04:38:10.872 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 671 up_thru 1165 down_at 667 last_clean_interval [8,667) [v2:172.21.15.39:6802/3832421293,v1:172.21.15.39:6803/3832421293] [v2:172.21.15.39:6804/3832421293,v1:172.21.15.39:6805/3832421293] exists,up 5fdfd478-dd27-4674-a7d6-53177d9e549a', 'osd.1 up in weight 1 up_from 681 up_thru 1165 down_at 677 last_clean_interval [13,677) [v2:172.21.15.39:6810/1186072955,v1:172.21.15.39:6811/1186072955] [v2:172.21.15.39:6812/1186072955,v1:172.21.15.39:6813/1186072955] exists,up 991c236e-1679-4c1e-a1a9-3894b1b12055', 'osd.2 up in weight 1 up_from 694 up_thru 1165 down_at 688 last_clean_interval [18,688) [v2:172.21.15.39:6818/2359749690,v1:172.21.15.39:6819/2359749690] [v2:172.21.15.39:6820/2359749690,v1:172.21.15.39:6821/2359749690] exists,up f9611ffd-5d37-4465-8114-ecc70d4d6d14', 'osd.3 up in weight 1 up_from 701 up_thru 1165 down_at 697 last_clean_interval [26,697) [v2:172.21.15.39:6826/2092194762,v1:172.21.15.39:6827/2092194762] [v2:172.21.15.39:6828/2092194762,v1:172.21.15.39:6829/2092194762] exists,up 56b6877b-2107-47d6-8da9-015db7352288', 'osd.4 up in weight 1 up_from 31 up_thru 1165 down_at 0 last_clean_interval [0,0) [v2:172.21.15.129:6800/168261430,v1:172.21.15.129:6801/168261430] [v2:172.21.15.129:6802/168261430,v1:172.21.15.129:6803/168261430] exists,up 2d5523c4-5871-4fb3-bc0d-62c3b7d8b6a1', 'osd.5 up in weight 1 up_from 39 up_thru 1165 down_at 0 last_clean_interval [0,0) [v2:172.21.15.129:6808/2184993002,v1:172.21.15.129:6809/2184993002] [v2:172.21.15.129:6810/2184993002,v1:172.21.15.129:6811/2184993002] exists,up a3ba55fd-f32e-4f6d-9b48-d2845a3474cd', 'osd.6 up in weight 1 up_from 47 up_thru 1165 down_at 0 last_clean_interval [0,0) [v2:172.21.15.129:6816/3767031353,v1:172.21.15.129:6817/3767031353] [v2:172.21.15.129:6818/3767031353,v1:172.21.15.129:6819/3767031353] exists,up 79e9c0a2-86a4-4267-a799-059d473d7b74', 'osd.7 up in weight 1 up_from 55 up_thru 1165 down_at 0 last_clean_interval [0,0) [v2:172.21.15.129:6824/2370414690,v1:172.21.15.129:6825/2370414690] [v2:172.21.15.129:6826/2370414690,v1:172.21.15.129:6827/2370414690] exists,up 22f41628-4b88-4b8c-82b2-fcd2802da1bf'] 2024-09-14T04:38:10.873 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon mon.a config show 2024-09-14T04:38:11.084 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:38:11.205 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:10 smithi039 ceph-mon[106470]: pgmap v2139: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:11.205 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:10 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1399602548' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-09-14T04:38:11.205 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:10 smithi039 ceph-mon[111235]: pgmap v2139: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:11.205 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:10 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1399602548' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-09-14T04:38:11.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:10 smithi129 ceph-mon[101316]: pgmap v2139: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:11.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:10 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1399602548' entity='client.admin' cmd={"prefix": "osd dump"} : dispatch 2024-09-14T04:38:12.148 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:12 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:38:12.149 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:12 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:38:12.491 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:12 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:38:12.954 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:38:12.955 INFO:teuthology.orchestra.run.smithi039.stdout: "name": "mon.a", 2024-09-14T04:38:12.955 INFO:teuthology.orchestra.run.smithi039.stdout: "cluster": "ceph", 2024-09-14T04:38:12.955 INFO:teuthology.orchestra.run.smithi039.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2024-09-14T04:38:12.955 INFO:teuthology.orchestra.run.smithi039.stdout: "admin_socket_mode": "", 2024-09-14T04:38:12.955 INFO:teuthology.orchestra.run.smithi039.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2024-09-14T04:38:12.955 INFO:teuthology.orchestra.run.smithi039.stdout: "auth_client_required": "cephx, none", 2024-09-14T04:38:12.955 INFO:teuthology.orchestra.run.smithi039.stdout: "auth_cluster_required": "cephx", 2024-09-14T04:38:12.955 INFO:teuthology.orchestra.run.smithi039.stdout: "auth_debug": "true", 2024-09-14T04:38:12.955 INFO:teuthology.orchestra.run.smithi039.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2024-09-14T04:38:12.955 INFO:teuthology.orchestra.run.smithi039.stdout: "auth_mon_ticket_ttl": "660.000000", 2024-09-14T04:38:12.955 INFO:teuthology.orchestra.run.smithi039.stdout: "auth_service_required": "cephx", 2024-09-14T04:38:12.955 INFO:teuthology.orchestra.run.smithi039.stdout: "auth_service_ticket_ttl": "240.000000", 2024-09-14T04:38:12.955 INFO:teuthology.orchestra.run.smithi039.stdout: "auth_supported": "", 2024-09-14T04:38:12.956 INFO:teuthology.orchestra.run.smithi039.stdout: "bdev_aio": "true", 2024-09-14T04:38:12.956 INFO:teuthology.orchestra.run.smithi039.stdout: "bdev_aio_max_queue_depth": "1024", 2024-09-14T04:38:12.956 INFO:teuthology.orchestra.run.smithi039.stdout: "bdev_aio_poll_ms": "250", 2024-09-14T04:38:12.956 INFO:teuthology.orchestra.run.smithi039.stdout: "bdev_aio_reap_max": "16", 2024-09-14T04:38:12.956 INFO:teuthology.orchestra.run.smithi039.stdout: "bdev_async_discard_threads": "0", 2024-09-14T04:38:12.956 INFO:teuthology.orchestra.run.smithi039.stdout: "bdev_block_size": "4096", 2024-09-14T04:38:12.956 INFO:teuthology.orchestra.run.smithi039.stdout: "bdev_debug_aio": "false", 2024-09-14T04:38:12.956 INFO:teuthology.orchestra.run.smithi039.stdout: "bdev_debug_aio_log_age": "5.000000", 2024-09-14T04:38:12.956 INFO:teuthology.orchestra.run.smithi039.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2024-09-14T04:38:12.956 INFO:teuthology.orchestra.run.smithi039.stdout: "bdev_debug_inflight_ios": "false", 2024-09-14T04:38:12.956 INFO:teuthology.orchestra.run.smithi039.stdout: "bdev_enable_discard": "false", 2024-09-14T04:38:12.956 INFO:teuthology.orchestra.run.smithi039.stdout: "bdev_flock_retry": "3", 2024-09-14T04:38:12.956 INFO:teuthology.orchestra.run.smithi039.stdout: "bdev_flock_retry_interval": "0.100000", 2024-09-14T04:38:12.957 INFO:teuthology.orchestra.run.smithi039.stdout: "bdev_inject_crash": "0", 2024-09-14T04:38:12.957 INFO:teuthology.orchestra.run.smithi039.stdout: "bdev_inject_crash_flush_delay": "2", 2024-09-14T04:38:12.957 INFO:teuthology.orchestra.run.smithi039.stdout: "bdev_ioring": "false", 2024-09-14T04:38:12.957 INFO:teuthology.orchestra.run.smithi039.stdout: "bdev_ioring_hipri": "false", 2024-09-14T04:38:12.957 INFO:teuthology.orchestra.run.smithi039.stdout: "bdev_ioring_sqthread_poll": "false", 2024-09-14T04:38:12.957 INFO:teuthology.orchestra.run.smithi039.stdout: "bdev_nvme_unbind_from_kernel": "false", 2024-09-14T04:38:12.957 INFO:teuthology.orchestra.run.smithi039.stdout: "bdev_read_buffer_alignment": "4096", 2024-09-14T04:38:12.957 INFO:teuthology.orchestra.run.smithi039.stdout: "bdev_read_preallocated_huge_buffers": "", 2024-09-14T04:38:12.957 INFO:teuthology.orchestra.run.smithi039.stdout: "bdev_stalled_read_warn_lifetime": "86400", 2024-09-14T04:38:12.957 INFO:teuthology.orchestra.run.smithi039.stdout: "bdev_stalled_read_warn_threshold": "1", 2024-09-14T04:38:12.957 INFO:teuthology.orchestra.run.smithi039.stdout: "bdev_type": "", 2024-09-14T04:38:12.957 INFO:teuthology.orchestra.run.smithi039.stdout: "bluefs_alloc_size": "1048576", 2024-09-14T04:38:12.957 INFO:teuthology.orchestra.run.smithi039.stdout: "bluefs_allocator": "hybrid", 2024-09-14T04:38:12.958 INFO:teuthology.orchestra.run.smithi039.stdout: "bluefs_buffered_io": "true", 2024-09-14T04:38:12.958 INFO:teuthology.orchestra.run.smithi039.stdout: "bluefs_check_for_zeros": "false", 2024-09-14T04:38:12.958 INFO:teuthology.orchestra.run.smithi039.stdout: "bluefs_check_volume_selector_often": "false", 2024-09-14T04:38:12.958 INFO:teuthology.orchestra.run.smithi039.stdout: "bluefs_check_volume_selector_on_umount": "false", 2024-09-14T04:38:12.958 INFO:teuthology.orchestra.run.smithi039.stdout: "bluefs_compact_log_sync": "false", 2024-09-14T04:38:12.958 INFO:teuthology.orchestra.run.smithi039.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2024-09-14T04:38:12.958 INFO:teuthology.orchestra.run.smithi039.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2024-09-14T04:38:12.958 INFO:teuthology.orchestra.run.smithi039.stdout: "bluefs_log_compact_min_size": "16777216", 2024-09-14T04:38:12.958 INFO:teuthology.orchestra.run.smithi039.stdout: "bluefs_log_replay_check_allocations": "true", 2024-09-14T04:38:12.958 INFO:teuthology.orchestra.run.smithi039.stdout: "bluefs_max_log_runway": "4194304", 2024-09-14T04:38:12.958 INFO:teuthology.orchestra.run.smithi039.stdout: "bluefs_max_prefetch": "1048576", 2024-09-14T04:38:12.958 INFO:teuthology.orchestra.run.smithi039.stdout: "bluefs_min_flush_size": "524288", 2024-09-14T04:38:12.958 INFO:teuthology.orchestra.run.smithi039.stdout: "bluefs_min_log_runway": "1048576", 2024-09-14T04:38:12.959 INFO:teuthology.orchestra.run.smithi039.stdout: "bluefs_replay_recovery": "false", 2024-09-14T04:38:12.959 INFO:teuthology.orchestra.run.smithi039.stdout: "bluefs_replay_recovery_disable_compact": "false", 2024-09-14T04:38:12.959 INFO:teuthology.orchestra.run.smithi039.stdout: "bluefs_shared_alloc_size": "65536", 2024-09-14T04:38:12.959 INFO:teuthology.orchestra.run.smithi039.stdout: "bluefs_sync_write": "false", 2024-09-14T04:38:12.959 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2024-09-14T04:38:12.959 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2024-09-14T04:38:12.959 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2024-09-14T04:38:12.959 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_allocation_from_file": "true", 2024-09-14T04:38:12.959 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_allocator": "hybrid", 2024-09-14T04:38:12.959 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_async_db_compaction": "true", 2024-09-14T04:38:12.959 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2024-09-14T04:38:12.959 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2024-09-14T04:38:12.959 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2024-09-14T04:38:12.960 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2024-09-14T04:38:12.960 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_bdev_label_multi": "true", 2024-09-14T04:38:12.960 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_bdev_label_multi_upgrade": "false", 2024-09-14T04:38:12.960 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_bdev_label_require_all": "true", 2024-09-14T04:38:12.960 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2024-09-14T04:38:12.960 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_bitmapallocator_span_size": "1024", 2024-09-14T04:38:12.960 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_blobid_prealloc": "10240", 2024-09-14T04:38:12.960 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_block_create": "true", 2024-09-14T04:38:12.960 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_block_db_create": "false", 2024-09-14T04:38:12.960 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_block_db_path": "", 2024-09-14T04:38:12.960 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_block_db_size": "0", 2024-09-14T04:38:12.960 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_block_path": "", 2024-09-14T04:38:12.960 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_block_preallocate_file": "false", 2024-09-14T04:38:12.961 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_block_size": "107374182400", 2024-09-14T04:38:12.961 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_block_wal_create": "false", 2024-09-14T04:38:12.961 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_block_wal_path": "", 2024-09-14T04:38:12.961 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_block_wal_size": "100663296", 2024-09-14T04:38:12.961 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_bluefs": "true", 2024-09-14T04:38:12.961 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2024-09-14T04:38:12.961 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_bluefs_env_mirror": "false", 2024-09-14T04:38:12.961 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_bluefs_max_free": "10737418240", 2024-09-14T04:38:12.961 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_btree2_alloc_weight_factor": "2.000000", 2024-09-14T04:38:12.961 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2024-09-14T04:38:12.961 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2024-09-14T04:38:12.961 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2024-09-14T04:38:12.961 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2024-09-14T04:38:12.962 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2024-09-14T04:38:12.962 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_cache_autotune": "true", 2024-09-14T04:38:12.962 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_cache_autotune_interval": "5.000000", 2024-09-14T04:38:12.962 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2024-09-14T04:38:12.962 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_cache_kv_ratio": "0.450000", 2024-09-14T04:38:12.962 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_cache_meta_ratio": "0.450000", 2024-09-14T04:38:12.962 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_cache_size": "0", 2024-09-14T04:38:12.962 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_cache_size_hdd": "1073741824", 2024-09-14T04:38:12.962 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_cache_size_ssd": "3221225472", 2024-09-14T04:38:12.962 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_cache_trim_interval": "0.050000", 2024-09-14T04:38:12.962 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2024-09-14T04:38:12.962 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_cache_type": "2q", 2024-09-14T04:38:12.962 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2024-09-14T04:38:12.963 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_clone_cow": "true", 2024-09-14T04:38:12.963 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_compression_algorithm": "snappy", 2024-09-14T04:38:12.963 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_compression_max_blob_size": "0", 2024-09-14T04:38:12.963 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2024-09-14T04:38:12.963 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2024-09-14T04:38:12.963 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_compression_min_blob_size": "0", 2024-09-14T04:38:12.963 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2024-09-14T04:38:12.963 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2024-09-14T04:38:12.963 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_compression_mode": "none", 2024-09-14T04:38:12.963 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_compression_required_ratio": "0.875000", 2024-09-14T04:38:12.963 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_csum_type": "crc32c", 2024-09-14T04:38:12.963 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_debug_enforce_min_alloc_size": "0", 2024-09-14T04:38:12.963 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_debug_enforce_settings": "default", 2024-09-14T04:38:12.963 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_debug_freelist": "false", 2024-09-14T04:38:12.964 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_debug_fsck_abort": "false", 2024-09-14T04:38:12.964 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2024-09-14T04:38:12.964 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2024-09-14T04:38:12.964 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_debug_inject_read_err": "false", 2024-09-14T04:38:12.964 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_debug_legacy_omap": "false", 2024-09-14T04:38:12.964 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_debug_no_reuse_blocks": "false", 2024-09-14T04:38:12.964 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_debug_omit_block_device_write": "false", 2024-09-14T04:38:12.964 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_debug_omit_kv_commit": "false", 2024-09-14T04:38:12.964 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2024-09-14T04:38:12.965 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_debug_prefragment_max": "1048576", 2024-09-14T04:38:12.965 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_debug_random_read_err": "0.000000", 2024-09-14T04:38:12.965 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2024-09-14T04:38:12.965 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_debug_small_allocations": "0", 2024-09-14T04:38:12.965 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2024-09-14T04:38:12.965 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_default_buffered_read": "true", 2024-09-14T04:38:12.965 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_default_buffered_write": "false", 2024-09-14T04:38:12.965 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_deferred_batch_ops": "0", 2024-09-14T04:38:12.965 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2024-09-14T04:38:12.965 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2024-09-14T04:38:12.965 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_elastic_shared_blobs": "true", 2024-09-14T04:38:12.965 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2024-09-14T04:38:12.965 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_extent_map_shard_max_size": "1200", 2024-09-14T04:38:12.966 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_extent_map_shard_min_size": "150", 2024-09-14T04:38:12.966 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_extent_map_shard_target_size": "500", 2024-09-14T04:38:12.966 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2024-09-14T04:38:12.966 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_fail_eio": "false", 2024-09-14T04:38:12.966 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_freelist_blocks_per_key": "128", 2024-09-14T04:38:12.966 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2024-09-14T04:38:12.966 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2024-09-14T04:38:12.966 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2024-09-14T04:38:12.966 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_fsck_on_mkfs": "true", 2024-09-14T04:38:12.966 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2024-09-14T04:38:12.966 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_fsck_on_mount": "false", 2024-09-14T04:38:12.966 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_fsck_on_mount_deep": "false", 2024-09-14T04:38:12.966 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_fsck_on_umount": "false", 2024-09-14T04:38:12.966 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_fsck_on_umount_deep": "false", 2024-09-14T04:38:12.967 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2024-09-14T04:38:12.967 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_fsck_quick_fix_threads": "2", 2024-09-14T04:38:12.967 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2024-09-14T04:38:12.967 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2024-09-14T04:38:12.967 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_gc_enable_blob_threshold": "0", 2024-09-14T04:38:12.967 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_gc_enable_total_threshold": "0", 2024-09-14T04:38:12.967 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2024-09-14T04:38:12.967 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_ignore_data_csum": "false", 2024-09-14T04:38:12.967 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2024-09-14T04:38:12.967 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_kvbackend": "rocksdb", 2024-09-14T04:38:12.967 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_log_collection_list_age": "60.000000", 2024-09-14T04:38:12.967 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2024-09-14T04:38:12.967 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_log_op_age": "5.000000", 2024-09-14T04:38:12.968 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_max_alloc_size": "0", 2024-09-14T04:38:12.968 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_max_blob_size": "0", 2024-09-14T04:38:12.968 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_max_blob_size_hdd": "65536", 2024-09-14T04:38:12.968 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_max_blob_size_ssd": "65536", 2024-09-14T04:38:12.968 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_max_defer_interval": "3.000000", 2024-09-14T04:38:12.968 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_max_deferred_txc": "32", 2024-09-14T04:38:12.968 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_min_alloc_size": "0", 2024-09-14T04:38:12.968 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_min_alloc_size_hdd": "4096", 2024-09-14T04:38:12.968 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_min_alloc_size_ssd": "4096", 2024-09-14T04:38:12.968 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_nid_prealloc": "1024", 2024-09-14T04:38:12.968 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_prefer_deferred_size": "0", 2024-09-14T04:38:12.968 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2024-09-14T04:38:12.968 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2024-09-14T04:38:12.968 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_qfsck_on_mount": "true", 2024-09-14T04:38:12.969 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_retry_disk_reads": "3", 2024-09-14T04:38:12.969 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_rocksdb_cf": "true", 2024-09-14T04:38:12.969 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2024-09-14T04:38:12.969 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_rocksdb_options": "compression=kLZ4Compression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2024-09-14T04:38:12.969 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_rocksdb_options_annex": "", 2024-09-14T04:38:12.969 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_slow_ops_warn_lifetime": "86400", 2024-09-14T04:38:12.969 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_slow_ops_warn_threshold": "1", 2024-09-14T04:38:12.969 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_spdk_coremask": "0x1", 2024-09-14T04:38:12.969 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_spdk_io_sleep": "5", 2024-09-14T04:38:12.969 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_spdk_max_io_completion": "0", 2024-09-14T04:38:12.969 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_spdk_mem": "512", 2024-09-14T04:38:12.969 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_sync_submit_transaction": "false", 2024-09-14T04:38:12.969 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_throttle_bytes": "67108864", 2024-09-14T04:38:12.970 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_throttle_cost_per_io": "0", 2024-09-14T04:38:12.970 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2024-09-14T04:38:12.970 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2024-09-14T04:38:12.970 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2024-09-14T04:38:12.970 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_throttle_trace_rate": "0.000000", 2024-09-14T04:38:12.970 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_tracing": "false", 2024-09-14T04:38:12.970 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2024-09-14T04:38:12.970 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2024-09-14T04:38:12.970 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_volume_selection_reserved": "0", 2024-09-14T04:38:12.970 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2024-09-14T04:38:12.970 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2024-09-14T04:38:12.970 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_warn_on_legacy_statfs": "true", 2024-09-14T04:38:12.970 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2024-09-14T04:38:12.970 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2024-09-14T04:38:12.971 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2024-09-14T04:38:12.971 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_write_v2": "false", 2024-09-14T04:38:12.971 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_write_v2_random": "false", 2024-09-14T04:38:12.971 INFO:teuthology.orchestra.run.smithi039.stdout: "bluestore_zero_block_detection": "false", 2024-09-14T04:38:12.971 INFO:teuthology.orchestra.run.smithi039.stdout: "cephadm_path": "/usr/sbin/cephadm", 2024-09-14T04:38:12.971 INFO:teuthology.orchestra.run.smithi039.stdout: "cephfs_mirror_action_update_interval": "2", 2024-09-14T04:38:12.971 INFO:teuthology.orchestra.run.smithi039.stdout: "cephfs_mirror_directory_scan_interval": "10", 2024-09-14T04:38:12.971 INFO:teuthology.orchestra.run.smithi039.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2024-09-14T04:38:12.971 INFO:teuthology.orchestra.run.smithi039.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2024-09-14T04:38:12.971 INFO:teuthology.orchestra.run.smithi039.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2024-09-14T04:38:12.971 INFO:teuthology.orchestra.run.smithi039.stdout: "cephfs_mirror_mount_timeout": "10", 2024-09-14T04:38:12.971 INFO:teuthology.orchestra.run.smithi039.stdout: "cephfs_mirror_perf_stats_prio": "5", 2024-09-14T04:38:12.971 INFO:teuthology.orchestra.run.smithi039.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2024-09-14T04:38:12.972 INFO:teuthology.orchestra.run.smithi039.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2024-09-14T04:38:12.972 INFO:teuthology.orchestra.run.smithi039.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2024-09-14T04:38:12.972 INFO:teuthology.orchestra.run.smithi039.stdout: "cephsqlite_blocklist_dead_locker": "true", 2024-09-14T04:38:12.972 INFO:teuthology.orchestra.run.smithi039.stdout: "cephsqlite_lock_renewal_interval": "2000", 2024-09-14T04:38:12.972 INFO:teuthology.orchestra.run.smithi039.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2024-09-14T04:38:12.972 INFO:teuthology.orchestra.run.smithi039.stdout: "cephx_cluster_require_signatures": "false", 2024-09-14T04:38:12.972 INFO:teuthology.orchestra.run.smithi039.stdout: "cephx_cluster_require_version": "2", 2024-09-14T04:38:12.972 INFO:teuthology.orchestra.run.smithi039.stdout: "cephx_require_signatures": "false", 2024-09-14T04:38:12.972 INFO:teuthology.orchestra.run.smithi039.stdout: "cephx_require_version": "2", 2024-09-14T04:38:12.972 INFO:teuthology.orchestra.run.smithi039.stdout: "cephx_service_require_signatures": "false", 2024-09-14T04:38:12.972 INFO:teuthology.orchestra.run.smithi039.stdout: "cephx_service_require_version": "2", 2024-09-14T04:38:12.972 INFO:teuthology.orchestra.run.smithi039.stdout: "cephx_sign_messages": "true", 2024-09-14T04:38:12.972 INFO:teuthology.orchestra.run.smithi039.stdout: "chdir": "", 2024-09-14T04:38:12.973 INFO:teuthology.orchestra.run.smithi039.stdout: "client_acl_type": "", 2024-09-14T04:38:12.973 INFO:teuthology.orchestra.run.smithi039.stdout: "client_asio_thread_count": "2", 2024-09-14T04:38:12.973 INFO:teuthology.orchestra.run.smithi039.stdout: "client_cache_mid": "0.750000", 2024-09-14T04:38:12.973 INFO:teuthology.orchestra.run.smithi039.stdout: "client_cache_size": "16384", 2024-09-14T04:38:12.973 INFO:teuthology.orchestra.run.smithi039.stdout: "client_caps_release_delay": "5", 2024-09-14T04:38:12.973 INFO:teuthology.orchestra.run.smithi039.stdout: "client_check_pool_perm": "true", 2024-09-14T04:38:12.973 INFO:teuthology.orchestra.run.smithi039.stdout: "client_collect_and_send_global_metrics": "false", 2024-09-14T04:38:12.973 INFO:teuthology.orchestra.run.smithi039.stdout: "client_debug_force_sync_read": "false", 2024-09-14T04:38:12.973 INFO:teuthology.orchestra.run.smithi039.stdout: "client_debug_getattr_caps": "false", 2024-09-14T04:38:12.973 INFO:teuthology.orchestra.run.smithi039.stdout: "client_debug_inject_features": "", 2024-09-14T04:38:12.973 INFO:teuthology.orchestra.run.smithi039.stdout: "client_debug_inject_tick_delay": "0", 2024-09-14T04:38:12.973 INFO:teuthology.orchestra.run.smithi039.stdout: "client_die_on_failed_dentry_invalidate": "true", 2024-09-14T04:38:12.973 INFO:teuthology.orchestra.run.smithi039.stdout: "client_die_on_failed_remount": "false", 2024-09-14T04:38:12.973 INFO:teuthology.orchestra.run.smithi039.stdout: "client_dirsize_rbytes": "true", 2024-09-14T04:38:12.974 INFO:teuthology.orchestra.run.smithi039.stdout: "client_force_lazyio": "false", 2024-09-14T04:38:12.974 INFO:teuthology.orchestra.run.smithi039.stdout: "client_fs": "", 2024-09-14T04:38:12.974 INFO:teuthology.orchestra.run.smithi039.stdout: "client_inject_fixed_oldest_tid": "false", 2024-09-14T04:38:12.974 INFO:teuthology.orchestra.run.smithi039.stdout: "client_inject_release_failure": "false", 2024-09-14T04:38:12.974 INFO:teuthology.orchestra.run.smithi039.stdout: "client_max_inline_size": "4096", 2024-09-14T04:38:12.974 INFO:teuthology.orchestra.run.smithi039.stdout: "client_max_retries_on_remount_failure": "5", 2024-09-14T04:38:12.974 INFO:teuthology.orchestra.run.smithi039.stdout: "client_mds_namespace": "", 2024-09-14T04:38:12.974 INFO:teuthology.orchestra.run.smithi039.stdout: "client_metadata": "", 2024-09-14T04:38:12.974 INFO:teuthology.orchestra.run.smithi039.stdout: "client_mount_gid": "-1", 2024-09-14T04:38:12.974 INFO:teuthology.orchestra.run.smithi039.stdout: "client_mount_timeout": "300", 2024-09-14T04:38:12.974 INFO:teuthology.orchestra.run.smithi039.stdout: "client_mount_uid": "-1", 2024-09-14T04:38:12.974 INFO:teuthology.orchestra.run.smithi039.stdout: "client_mountpoint": "/", 2024-09-14T04:38:12.974 INFO:teuthology.orchestra.run.smithi039.stdout: "client_notify_timeout": "10", 2024-09-14T04:38:12.975 INFO:teuthology.orchestra.run.smithi039.stdout: "client_oc": "true", 2024-09-14T04:38:12.975 INFO:teuthology.orchestra.run.smithi039.stdout: "client_oc_max_dirty": "104857600", 2024-09-14T04:38:12.975 INFO:teuthology.orchestra.run.smithi039.stdout: "client_oc_max_dirty_age": "5.000000", 2024-09-14T04:38:12.975 INFO:teuthology.orchestra.run.smithi039.stdout: "client_oc_max_objects": "1000", 2024-09-14T04:38:12.975 INFO:teuthology.orchestra.run.smithi039.stdout: "client_oc_size": "209715200", 2024-09-14T04:38:12.975 INFO:teuthology.orchestra.run.smithi039.stdout: "client_oc_target_dirty": "8388608", 2024-09-14T04:38:12.975 INFO:teuthology.orchestra.run.smithi039.stdout: "client_permissions": "true", 2024-09-14T04:38:12.975 INFO:teuthology.orchestra.run.smithi039.stdout: "client_quota": "true", 2024-09-14T04:38:12.975 INFO:teuthology.orchestra.run.smithi039.stdout: "client_quota_df": "true", 2024-09-14T04:38:12.975 INFO:teuthology.orchestra.run.smithi039.stdout: "client_readahead_max_bytes": "0", 2024-09-14T04:38:12.975 INFO:teuthology.orchestra.run.smithi039.stdout: "client_readahead_max_periods": "4", 2024-09-14T04:38:12.975 INFO:teuthology.orchestra.run.smithi039.stdout: "client_readahead_min": "131072", 2024-09-14T04:38:12.975 INFO:teuthology.orchestra.run.smithi039.stdout: "client_reconnect_stale": "false", 2024-09-14T04:38:12.976 INFO:teuthology.orchestra.run.smithi039.stdout: "client_shutdown_timeout": "30", 2024-09-14T04:38:12.976 INFO:teuthology.orchestra.run.smithi039.stdout: "client_snapdir": ".snap", 2024-09-14T04:38:12.976 INFO:teuthology.orchestra.run.smithi039.stdout: "client_tick_interval": "1", 2024-09-14T04:38:12.976 INFO:teuthology.orchestra.run.smithi039.stdout: "client_trace": "", 2024-09-14T04:38:12.976 INFO:teuthology.orchestra.run.smithi039.stdout: "client_try_dentry_invalidate": "false", 2024-09-14T04:38:12.976 INFO:teuthology.orchestra.run.smithi039.stdout: "client_use_faked_inos": "false", 2024-09-14T04:38:12.976 INFO:teuthology.orchestra.run.smithi039.stdout: "client_use_random_mds": "false", 2024-09-14T04:38:12.976 INFO:teuthology.orchestra.run.smithi039.stdout: "clog_to_graylog": "false", 2024-09-14T04:38:12.976 INFO:teuthology.orchestra.run.smithi039.stdout: "clog_to_graylog_host": "127.0.0.1", 2024-09-14T04:38:12.976 INFO:teuthology.orchestra.run.smithi039.stdout: "clog_to_graylog_port": "12201", 2024-09-14T04:38:12.976 INFO:teuthology.orchestra.run.smithi039.stdout: "clog_to_monitors": "default=true", 2024-09-14T04:38:12.976 INFO:teuthology.orchestra.run.smithi039.stdout: "clog_to_syslog": "false", 2024-09-14T04:38:12.976 INFO:teuthology.orchestra.run.smithi039.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2024-09-14T04:38:12.976 INFO:teuthology.orchestra.run.smithi039.stdout: "clog_to_syslog_level": "info", 2024-09-14T04:38:12.977 INFO:teuthology.orchestra.run.smithi039.stdout: "cluster_addr": "-", 2024-09-14T04:38:12.977 INFO:teuthology.orchestra.run.smithi039.stdout: "cluster_network": "", 2024-09-14T04:38:12.977 INFO:teuthology.orchestra.run.smithi039.stdout: "cluster_network_interface": "", 2024-09-14T04:38:12.977 INFO:teuthology.orchestra.run.smithi039.stdout: "compressor_zlib_isal": "false", 2024-09-14T04:38:12.977 INFO:teuthology.orchestra.run.smithi039.stdout: "compressor_zlib_level": "5", 2024-09-14T04:38:12.977 INFO:teuthology.orchestra.run.smithi039.stdout: "compressor_zlib_winsize": "-15", 2024-09-14T04:38:12.977 INFO:teuthology.orchestra.run.smithi039.stdout: "compressor_zstd_level": "1", 2024-09-14T04:38:12.977 INFO:teuthology.orchestra.run.smithi039.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802", 2024-09-14T04:38:12.977 INFO:teuthology.orchestra.run.smithi039.stdout: "crash_dir": "/var/lib/ceph/crash", 2024-09-14T04:38:12.977 INFO:teuthology.orchestra.run.smithi039.stdout: "crimson_alien_op_num_threads": "6", 2024-09-14T04:38:12.977 INFO:teuthology.orchestra.run.smithi039.stdout: "crimson_alien_thread_cpu_cores": "", 2024-09-14T04:38:12.977 INFO:teuthology.orchestra.run.smithi039.stdout: "crimson_osd_obc_lru_size": "512", 2024-09-14T04:38:12.977 INFO:teuthology.orchestra.run.smithi039.stdout: "crimson_osd_scheduler_concurrency": "0", 2024-09-14T04:38:12.978 INFO:teuthology.orchestra.run.smithi039.stdout: "crimson_osd_stat_interval": "0", 2024-09-14T04:38:12.978 INFO:teuthology.orchestra.run.smithi039.stdout: "crimson_seastar_cpu_cores": "", 2024-09-14T04:38:12.978 INFO:teuthology.orchestra.run.smithi039.stdout: "crimson_seastar_num_threads": "0", 2024-09-14T04:38:12.978 INFO:teuthology.orchestra.run.smithi039.stdout: "crush_location": "", 2024-09-14T04:38:12.978 INFO:teuthology.orchestra.run.smithi039.stdout: "crush_location_hook": "", 2024-09-14T04:38:12.978 INFO:teuthology.orchestra.run.smithi039.stdout: "crush_location_hook_timeout": "10", 2024-09-14T04:38:12.978 INFO:teuthology.orchestra.run.smithi039.stdout: "daemonize": "false", 2024-09-14T04:38:12.978 INFO:teuthology.orchestra.run.smithi039.stdout: "daos_pool": "tank", 2024-09-14T04:38:12.978 INFO:teuthology.orchestra.run.smithi039.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2024-09-14T04:38:12.978 INFO:teuthology.orchestra.run.smithi039.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2024-09-14T04:38:12.978 INFO:teuthology.orchestra.run.smithi039.stdout: "dbstore_db_name_prefix": "dbstore", 2024-09-14T04:38:12.978 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_alienstore": "0/5", 2024-09-14T04:38:12.978 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_allow_any_pool_priority": "false", 2024-09-14T04:38:12.979 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_asok": "1/5", 2024-09-14T04:38:12.979 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_asok_assert_abort": "false", 2024-09-14T04:38:12.979 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_asserts_on_shutdown": "true", 2024-09-14T04:38:12.979 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_auth": "1/5", 2024-09-14T04:38:12.979 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_bdev": "1/3", 2024-09-14T04:38:12.979 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_bluefs": "1/5", 2024-09-14T04:38:12.979 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_bluestore": "1/5", 2024-09-14T04:38:12.979 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_buffer": "0/1", 2024-09-14T04:38:12.979 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_ceph_dedup": "0/5", 2024-09-14T04:38:12.979 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_ceph_exporter": "1/5", 2024-09-14T04:38:12.979 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_cephfs_mirror": "0/5", 2024-09-14T04:38:12.979 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_cephsqlite": "0/5", 2024-09-14T04:38:12.980 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_client": "0/5", 2024-09-14T04:38:12.980 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_compressor": "1/5", 2024-09-14T04:38:12.980 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_context": "0/1", 2024-09-14T04:38:12.980 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_crimson_interrupt": "0/5", 2024-09-14T04:38:12.980 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_crush": "1/1", 2024-09-14T04:38:12.980 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_crypto": "1/5", 2024-09-14T04:38:12.980 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_cyanstore": "0/5", 2024-09-14T04:38:12.980 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_deliberately_leak_memory": "false", 2024-09-14T04:38:12.980 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_disable_randomized_ping": "false", 2024-09-14T04:38:12.980 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_dpdk": "1/5", 2024-09-14T04:38:12.980 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_eventtrace": "1/5", 2024-09-14T04:38:12.980 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_filer": "0/1", 2024-09-14T04:38:12.980 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_filestore": "1/3", 2024-09-14T04:38:12.981 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_finisher": "1/1", 2024-09-14T04:38:12.981 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_fuse": "1/5", 2024-09-14T04:38:12.981 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_heartbeat_testing_span": "0", 2024-09-14T04:38:12.981 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_heartbeatmap": "1/5", 2024-09-14T04:38:12.981 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_immutable_obj_cache": "0/5", 2024-09-14T04:38:12.981 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_javaclient": "1/5", 2024-09-14T04:38:12.981 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_journal": "1/3", 2024-09-14T04:38:12.981 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_journaler": "0/5", 2024-09-14T04:38:12.981 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_kstore": "1/5", 2024-09-14T04:38:12.981 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_lockdep": "0/1", 2024-09-14T04:38:12.981 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_mclock": "1/5", 2024-09-14T04:38:12.981 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_mds": "1/5", 2024-09-14T04:38:12.981 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_mds_balancer": "1/5", 2024-09-14T04:38:12.981 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_mds_locker": "1/5", 2024-09-14T04:38:12.982 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_mds_log": "1/5", 2024-09-14T04:38:12.982 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_mds_log_expire": "1/5", 2024-09-14T04:38:12.982 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_mds_migrator": "1/5", 2024-09-14T04:38:12.982 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_mds_quiesce": "3/5", 2024-09-14T04:38:12.982 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_memstore": "1/5", 2024-09-14T04:38:12.982 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_mgr": "2/5", 2024-09-14T04:38:12.982 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_mgrc": "1/5", 2024-09-14T04:38:12.982 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_mon": "20/20", 2024-09-14T04:38:12.982 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_monc": "0/10", 2024-09-14T04:38:12.982 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_ms": "1/1", 2024-09-14T04:38:12.982 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_none": "0/5", 2024-09-14T04:38:12.982 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_objclass": "0/5", 2024-09-14T04:38:12.982 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_objectcacher": "0/5", 2024-09-14T04:38:12.983 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_objecter": "0/1", 2024-09-14T04:38:12.983 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_optracker": "0/5", 2024-09-14T04:38:12.983 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_osd": "1/5", 2024-09-14T04:38:12.983 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_paxos": "20/20", 2024-09-14T04:38:12.983 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_perfcounter": "1/5", 2024-09-14T04:38:12.983 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_prioritycache": "1/5", 2024-09-14T04:38:12.983 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_rados": "0/5", 2024-09-14T04:38:12.983 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_rbd": "0/5", 2024-09-14T04:38:12.983 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_rbd_mirror": "0/5", 2024-09-14T04:38:12.983 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_rbd_pwl": "0/5", 2024-09-14T04:38:12.983 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_rbd_replay": "0/5", 2024-09-14T04:38:12.983 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_refs": "0/0", 2024-09-14T04:38:12.983 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_reserver": "1/1", 2024-09-14T04:38:12.983 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_rgw": "1/5", 2024-09-14T04:38:12.984 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_rgw_access": "1/5", 2024-09-14T04:38:12.984 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_rgw_datacache": "1/5", 2024-09-14T04:38:12.984 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_rgw_dbstore": "1/5", 2024-09-14T04:38:12.984 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_rgw_flight": "1/5", 2024-09-14T04:38:12.984 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_rgw_lifecycle": "1/5", 2024-09-14T04:38:12.984 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_rgw_notification": "1/5", 2024-09-14T04:38:12.984 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_rgw_sync": "1/5", 2024-09-14T04:38:12.984 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_rocksdb": "4/5", 2024-09-14T04:38:12.984 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_seastore": "0/5", 2024-09-14T04:38:12.984 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_seastore_backref": "0/5", 2024-09-14T04:38:12.984 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_seastore_cache": "0/5", 2024-09-14T04:38:12.984 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_seastore_cleaner": "0/5", 2024-09-14T04:38:12.984 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_seastore_device": "0/5", 2024-09-14T04:38:12.985 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_seastore_epm": "0/5", 2024-09-14T04:38:12.985 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_seastore_fixedkv_tree": "0/5", 2024-09-14T04:38:12.985 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_seastore_journal": "0/5", 2024-09-14T04:38:12.985 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_seastore_lba": "0/5", 2024-09-14T04:38:12.985 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_seastore_odata": "0/5", 2024-09-14T04:38:12.985 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_seastore_omap": "0/5", 2024-09-14T04:38:12.985 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_seastore_onode": "0/5", 2024-09-14T04:38:12.985 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_seastore_t": "0/5", 2024-09-14T04:38:12.985 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_seastore_tm": "0/5", 2024-09-14T04:38:12.985 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_striper": "0/1", 2024-09-14T04:38:12.985 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_test": "0/5", 2024-09-14T04:38:12.985 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_throttle": "1/1", 2024-09-14T04:38:12.985 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_timer": "0/1", 2024-09-14T04:38:12.985 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_tp": "0/5", 2024-09-14T04:38:12.986 INFO:teuthology.orchestra.run.smithi039.stdout: "debug_trace": "1/5", 2024-09-14T04:38:12.986 INFO:teuthology.orchestra.run.smithi039.stdout: "defer_client_eviction_on_laggy_osds": "false", 2024-09-14T04:38:12.986 INFO:teuthology.orchestra.run.smithi039.stdout: "device_failure_prediction_mode": "none", 2024-09-14T04:38:12.986 INFO:teuthology.orchestra.run.smithi039.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2024-09-14T04:38:12.986 INFO:teuthology.orchestra.run.smithi039.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2024-09-14T04:38:12.986 INFO:teuthology.orchestra.run.smithi039.stdout: "err_to_graylog": "false", 2024-09-14T04:38:12.986 INFO:teuthology.orchestra.run.smithi039.stdout: "err_to_journald": "false", 2024-09-14T04:38:12.986 INFO:teuthology.orchestra.run.smithi039.stdout: "err_to_stderr": "true", 2024-09-14T04:38:12.986 INFO:teuthology.orchestra.run.smithi039.stdout: "err_to_syslog": "false", 2024-09-14T04:38:12.986 INFO:teuthology.orchestra.run.smithi039.stdout: "event_tracing": "false", 2024-09-14T04:38:12.986 INFO:teuthology.orchestra.run.smithi039.stdout: "exporter_addr": "0.0.0.0", 2024-09-14T04:38:12.986 INFO:teuthology.orchestra.run.smithi039.stdout: "exporter_cert_file": "", 2024-09-14T04:38:12.986 INFO:teuthology.orchestra.run.smithi039.stdout: "exporter_http_port": "9926", 2024-09-14T04:38:12.987 INFO:teuthology.orchestra.run.smithi039.stdout: "exporter_key_file": "", 2024-09-14T04:38:12.987 INFO:teuthology.orchestra.run.smithi039.stdout: "exporter_prio_limit": "5", 2024-09-14T04:38:12.987 INFO:teuthology.orchestra.run.smithi039.stdout: "exporter_sock_dir": "/var/run/ceph/", 2024-09-14T04:38:12.987 INFO:teuthology.orchestra.run.smithi039.stdout: "exporter_sort_metrics": "true", 2024-09-14T04:38:12.987 INFO:teuthology.orchestra.run.smithi039.stdout: "exporter_stats_period": "5", 2024-09-14T04:38:12.987 INFO:teuthology.orchestra.run.smithi039.stdout: "fake_statfs_for_testing": "0", 2024-09-14T04:38:12.987 INFO:teuthology.orchestra.run.smithi039.stdout: "fatal_signal_handlers": "true", 2024-09-14T04:38:12.987 INFO:teuthology.orchestra.run.smithi039.stdout: "filer_max_purge_ops": "10", 2024-09-14T04:38:12.987 INFO:teuthology.orchestra.run.smithi039.stdout: "filer_max_truncate_ops": "128", 2024-09-14T04:38:12.987 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_apply_finisher_threads": "1", 2024-09-14T04:38:12.987 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_blackhole": "false", 2024-09-14T04:38:12.987 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_btrfs_clone_range": "true", 2024-09-14T04:38:12.987 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_btrfs_snap": "true", 2024-09-14T04:38:12.987 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_caller_concurrency": "10", 2024-09-14T04:38:12.988 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_collect_device_partition_information": "true", 2024-09-14T04:38:12.988 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_commit_timeout": "600.000000", 2024-09-14T04:38:12.988 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_debug_inject_read_err": "false", 2024-09-14T04:38:12.988 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_debug_omap_check": "false", 2024-09-14T04:38:12.988 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_debug_verify_split": "false", 2024-09-14T04:38:12.988 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_dump_file": "", 2024-09-14T04:38:12.988 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2024-09-14T04:38:12.988 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_expected_throughput_ops": "200.000000", 2024-09-14T04:38:12.988 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_fadvise": "true", 2024-09-14T04:38:12.988 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_fail_eio": "true", 2024-09-14T04:38:12.988 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_fd_cache_shards": "16", 2024-09-14T04:38:12.988 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_fd_cache_size": "128", 2024-09-14T04:38:12.988 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_fiemap": "false", 2024-09-14T04:38:12.989 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_fiemap_threshold": "4096", 2024-09-14T04:38:12.989 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_fsync_flushes_journal_data": "false", 2024-09-14T04:38:12.989 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_index_retry_probability": "0.000000", 2024-09-14T04:38:12.989 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_inject_stall": "0", 2024-09-14T04:38:12.989 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_journal_parallel": "false", 2024-09-14T04:38:12.989 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_journal_trailing": "false", 2024-09-14T04:38:12.989 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_journal_writeahead": "false", 2024-09-14T04:38:12.989 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_kill_at": "0", 2024-09-14T04:38:12.989 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_max_alloc_hint_size": "1048576", 2024-09-14T04:38:12.989 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_max_inline_xattr_size": "0", 2024-09-14T04:38:12.989 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2024-09-14T04:38:12.989 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_max_inline_xattr_size_other": "512", 2024-09-14T04:38:12.989 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2024-09-14T04:38:12.990 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_max_inline_xattrs": "0", 2024-09-14T04:38:12.990 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2024-09-14T04:38:12.990 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_max_inline_xattrs_other": "2", 2024-09-14T04:38:12.990 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_max_inline_xattrs_xfs": "10", 2024-09-14T04:38:12.990 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_max_sync_interval": "5.000000", 2024-09-14T04:38:12.990 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_max_xattr_value_size": "0", 2024-09-14T04:38:12.990 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2024-09-14T04:38:12.990 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_max_xattr_value_size_other": "1024", 2024-09-14T04:38:12.990 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2024-09-14T04:38:12.990 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_merge_threshold": "-10", 2024-09-14T04:38:12.990 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_min_sync_interval": "0.010000", 2024-09-14T04:38:12.990 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_odsync_write": "false", 2024-09-14T04:38:12.990 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_omap_backend": "rocksdb", 2024-09-14T04:38:12.990 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_omap_backend_path": "", 2024-09-14T04:38:12.991 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_omap_header_cache_size": "1024", 2024-09-14T04:38:12.991 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_ondisk_finisher_threads": "1", 2024-09-14T04:38:12.991 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_op_thread_suicide_timeout": "180", 2024-09-14T04:38:12.991 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_op_thread_timeout": "60", 2024-09-14T04:38:12.991 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_op_threads": "2", 2024-09-14T04:38:12.991 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_punch_hole": "false", 2024-09-14T04:38:12.991 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2024-09-14T04:38:12.991 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2024-09-14T04:38:12.991 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2024-09-14T04:38:12.991 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_queue_high_threshhold": "0.900000", 2024-09-14T04:38:12.991 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_queue_low_threshhold": "0.300000", 2024-09-14T04:38:12.991 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_queue_max_bytes": "104857600", 2024-09-14T04:38:12.991 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2024-09-14T04:38:12.991 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2024-09-14T04:38:12.992 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2024-09-14T04:38:12.992 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_queue_max_ops": "50", 2024-09-14T04:38:12.992 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2024-09-14T04:38:12.992 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_seek_data_hole": "false", 2024-09-14T04:38:12.992 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_sloppy_crc": "false", 2024-09-14T04:38:12.992 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_sloppy_crc_block_size": "65536", 2024-09-14T04:38:12.992 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_splice": "false", 2024-09-14T04:38:12.992 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_split_multiple": "2", 2024-09-14T04:38:12.992 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_split_rand_factor": "20", 2024-09-14T04:38:12.992 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_update_to": "1000", 2024-09-14T04:38:12.992 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2024-09-14T04:38:12.992 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2024-09-14T04:38:12.992 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2024-09-14T04:38:12.993 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2024-09-14T04:38:12.993 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2024-09-14T04:38:12.993 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2024-09-14T04:38:12.993 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_wbthrottle_enable": "true", 2024-09-14T04:38:12.993 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2024-09-14T04:38:12.993 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2024-09-14T04:38:12.993 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2024-09-14T04:38:12.993 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2024-09-14T04:38:12.993 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2024-09-14T04:38:12.993 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2024-09-14T04:38:12.993 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_xfs_extsize": "false", 2024-09-14T04:38:12.993 INFO:teuthology.orchestra.run.smithi039.stdout: "filestore_zfs_snap": "false", 2024-09-14T04:38:12.993 INFO:teuthology.orchestra.run.smithi039.stdout: "fio_dir": "/tmp/fio", 2024-09-14T04:38:12.994 INFO:teuthology.orchestra.run.smithi039.stdout: "fsid": "bd7c803c-724b-11ef-bceb-c7b262605968", 2024-09-14T04:38:12.994 INFO:teuthology.orchestra.run.smithi039.stdout: "fuse_allow_other": "true", 2024-09-14T04:38:12.994 INFO:teuthology.orchestra.run.smithi039.stdout: "fuse_atomic_o_trunc": "true", 2024-09-14T04:38:12.994 INFO:teuthology.orchestra.run.smithi039.stdout: "fuse_big_writes": "true", 2024-09-14T04:38:12.994 INFO:teuthology.orchestra.run.smithi039.stdout: "fuse_debug": "false", 2024-09-14T04:38:12.994 INFO:teuthology.orchestra.run.smithi039.stdout: "fuse_default_permissions": "false", 2024-09-14T04:38:12.994 INFO:teuthology.orchestra.run.smithi039.stdout: "fuse_disable_pagecache": "false", 2024-09-14T04:38:12.994 INFO:teuthology.orchestra.run.smithi039.stdout: "fuse_max_write": "0", 2024-09-14T04:38:12.994 INFO:teuthology.orchestra.run.smithi039.stdout: "fuse_multithreaded": "true", 2024-09-14T04:38:12.994 INFO:teuthology.orchestra.run.smithi039.stdout: "fuse_require_active_mds": "true", 2024-09-14T04:38:12.994 INFO:teuthology.orchestra.run.smithi039.stdout: "fuse_set_user_groups": "true", 2024-09-14T04:38:12.994 INFO:teuthology.orchestra.run.smithi039.stdout: "fuse_splice_move": "true", 2024-09-14T04:38:12.994 INFO:teuthology.orchestra.run.smithi039.stdout: "fuse_splice_read": "true", 2024-09-14T04:38:12.994 INFO:teuthology.orchestra.run.smithi039.stdout: "fuse_splice_write": "true", 2024-09-14T04:38:12.995 INFO:teuthology.orchestra.run.smithi039.stdout: "fuse_syncfs_on_mksnap": "true", 2024-09-14T04:38:12.995 INFO:teuthology.orchestra.run.smithi039.stdout: "fuse_use_invalidate_cb": "true", 2024-09-14T04:38:12.995 INFO:teuthology.orchestra.run.smithi039.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2024-09-14T04:38:12.995 INFO:teuthology.orchestra.run.smithi039.stdout: "gss_target_name": "ceph", 2024-09-14T04:38:12.995 INFO:teuthology.orchestra.run.smithi039.stdout: "heartbeat_file": "", 2024-09-14T04:38:12.995 INFO:teuthology.orchestra.run.smithi039.stdout: "heartbeat_inject_failure": "0", 2024-09-14T04:38:12.995 INFO:teuthology.orchestra.run.smithi039.stdout: "heartbeat_interval": "5", 2024-09-14T04:38:12.995 INFO:teuthology.orchestra.run.smithi039.stdout: "host": "smithi039", 2024-09-14T04:38:12.995 INFO:teuthology.orchestra.run.smithi039.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2024-09-14T04:38:12.995 INFO:teuthology.orchestra.run.smithi039.stdout: "immutable_object_cache_max_inflight_ops": "128", 2024-09-14T04:38:12.995 INFO:teuthology.orchestra.run.smithi039.stdout: "immutable_object_cache_max_size": "1073741824", 2024-09-14T04:38:12.995 INFO:teuthology.orchestra.run.smithi039.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2024-09-14T04:38:12.995 INFO:teuthology.orchestra.run.smithi039.stdout: "immutable_object_cache_qos_bps_burst": "0", 2024-09-14T04:38:12.995 INFO:teuthology.orchestra.run.smithi039.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2024-09-14T04:38:12.995 INFO:teuthology.orchestra.run.smithi039.stdout: "immutable_object_cache_qos_bps_limit": "0", 2024-09-14T04:38:12.996 INFO:teuthology.orchestra.run.smithi039.stdout: "immutable_object_cache_qos_iops_burst": "0", 2024-09-14T04:38:12.996 INFO:teuthology.orchestra.run.smithi039.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2024-09-14T04:38:12.996 INFO:teuthology.orchestra.run.smithi039.stdout: "immutable_object_cache_qos_iops_limit": "0", 2024-09-14T04:38:12.996 INFO:teuthology.orchestra.run.smithi039.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2024-09-14T04:38:12.996 INFO:teuthology.orchestra.run.smithi039.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2024-09-14T04:38:12.996 INFO:teuthology.orchestra.run.smithi039.stdout: "immutable_object_cache_watermark": "0.900000", 2024-09-14T04:38:12.996 INFO:teuthology.orchestra.run.smithi039.stdout: "inject_early_sigterm": "false", 2024-09-14T04:38:12.996 INFO:teuthology.orchestra.run.smithi039.stdout: "jaeger_agent_port": "6799", 2024-09-14T04:38:12.996 INFO:teuthology.orchestra.run.smithi039.stdout: "jaeger_tracing_enable": "false", 2024-09-14T04:38:12.996 INFO:teuthology.orchestra.run.smithi039.stdout: "journal_aio": "true", 2024-09-14T04:38:12.996 INFO:teuthology.orchestra.run.smithi039.stdout: "journal_align_min_size": "65536", 2024-09-14T04:38:12.996 INFO:teuthology.orchestra.run.smithi039.stdout: "journal_block_align": "true", 2024-09-14T04:38:12.996 INFO:teuthology.orchestra.run.smithi039.stdout: "journal_block_size": "4096", 2024-09-14T04:38:12.996 INFO:teuthology.orchestra.run.smithi039.stdout: "journal_dio": "true", 2024-09-14T04:38:12.997 INFO:teuthology.orchestra.run.smithi039.stdout: "journal_discard": "false", 2024-09-14T04:38:12.997 INFO:teuthology.orchestra.run.smithi039.stdout: "journal_force_aio": "false", 2024-09-14T04:38:12.997 INFO:teuthology.orchestra.run.smithi039.stdout: "journal_ignore_corruption": "false", 2024-09-14T04:38:12.997 INFO:teuthology.orchestra.run.smithi039.stdout: "journal_max_write_bytes": "10485760", 2024-09-14T04:38:12.997 INFO:teuthology.orchestra.run.smithi039.stdout: "journal_max_write_entries": "100", 2024-09-14T04:38:12.997 INFO:teuthology.orchestra.run.smithi039.stdout: "journal_replay_from": "0", 2024-09-14T04:38:12.997 INFO:teuthology.orchestra.run.smithi039.stdout: "journal_throttle_high_multiple": "0.000000", 2024-09-14T04:38:12.997 INFO:teuthology.orchestra.run.smithi039.stdout: "journal_throttle_high_threshhold": "0.900000", 2024-09-14T04:38:12.997 INFO:teuthology.orchestra.run.smithi039.stdout: "journal_throttle_low_threshhold": "0.600000", 2024-09-14T04:38:12.997 INFO:teuthology.orchestra.run.smithi039.stdout: "journal_throttle_max_multiple": "0.000000", 2024-09-14T04:38:12.997 INFO:teuthology.orchestra.run.smithi039.stdout: "journal_write_header_frequency": "0", 2024-09-14T04:38:12.997 INFO:teuthology.orchestra.run.smithi039.stdout: "journal_zero_on_create": "false", 2024-09-14T04:38:12.997 INFO:teuthology.orchestra.run.smithi039.stdout: "journaler_prefetch_periods": "10", 2024-09-14T04:38:12.998 INFO:teuthology.orchestra.run.smithi039.stdout: "journaler_prezero_periods": "5", 2024-09-14T04:38:12.998 INFO:teuthology.orchestra.run.smithi039.stdout: "journaler_write_head_interval": "15", 2024-09-14T04:38:12.998 INFO:teuthology.orchestra.run.smithi039.stdout: "key": "", 2024-09-14T04:38:12.998 INFO:teuthology.orchestra.run.smithi039.stdout: "keyfile": "", 2024-09-14T04:38:12.998 INFO:teuthology.orchestra.run.smithi039.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2024-09-14T04:38:12.998 INFO:teuthology.orchestra.run.smithi039.stdout: "kstore_backend": "rocksdb", 2024-09-14T04:38:12.998 INFO:teuthology.orchestra.run.smithi039.stdout: "kstore_default_stripe_size": "65536", 2024-09-14T04:38:12.998 INFO:teuthology.orchestra.run.smithi039.stdout: "kstore_fsck_on_mount": "false", 2024-09-14T04:38:12.998 INFO:teuthology.orchestra.run.smithi039.stdout: "kstore_fsck_on_mount_deep": "true", 2024-09-14T04:38:12.998 INFO:teuthology.orchestra.run.smithi039.stdout: "kstore_max_bytes": "67108864", 2024-09-14T04:38:12.998 INFO:teuthology.orchestra.run.smithi039.stdout: "kstore_max_ops": "512", 2024-09-14T04:38:12.998 INFO:teuthology.orchestra.run.smithi039.stdout: "kstore_nid_prealloc": "1024", 2024-09-14T04:38:12.998 INFO:teuthology.orchestra.run.smithi039.stdout: "kstore_onode_map_size": "1024", 2024-09-14T04:38:12.999 INFO:teuthology.orchestra.run.smithi039.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2024-09-14T04:38:12.999 INFO:teuthology.orchestra.run.smithi039.stdout: "kstore_sync_submit_transaction": "false", 2024-09-14T04:38:12.999 INFO:teuthology.orchestra.run.smithi039.stdout: "kstore_sync_transaction": "false", 2024-09-14T04:38:12.999 INFO:teuthology.orchestra.run.smithi039.stdout: "librados_thread_count": "2", 2024-09-14T04:38:12.999 INFO:teuthology.orchestra.run.smithi039.stdout: "lockdep": "false", 2024-09-14T04:38:12.999 INFO:teuthology.orchestra.run.smithi039.stdout: "lockdep_force_backtrace": "false", 2024-09-14T04:38:12.999 INFO:teuthology.orchestra.run.smithi039.stdout: "log_coarse_timestamps": "true", 2024-09-14T04:38:12.999 INFO:teuthology.orchestra.run.smithi039.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2024-09-14T04:38:12.999 INFO:teuthology.orchestra.run.smithi039.stdout: "log_flush_on_exit": "false", 2024-09-14T04:38:12.999 INFO:teuthology.orchestra.run.smithi039.stdout: "log_graylog_host": "127.0.0.1", 2024-09-14T04:38:12.999 INFO:teuthology.orchestra.run.smithi039.stdout: "log_graylog_port": "12201", 2024-09-14T04:38:12.999 INFO:teuthology.orchestra.run.smithi039.stdout: "log_max_new": "1000", 2024-09-14T04:38:12.999 INFO:teuthology.orchestra.run.smithi039.stdout: "log_max_recent": "10000", 2024-09-14T04:38:12.999 INFO:teuthology.orchestra.run.smithi039.stdout: "log_stderr_prefix": "", 2024-09-14T04:38:13.000 INFO:teuthology.orchestra.run.smithi039.stdout: "log_stop_at_utilization": "0.970000", 2024-09-14T04:38:13.000 INFO:teuthology.orchestra.run.smithi039.stdout: "log_to_file": "true", 2024-09-14T04:38:13.000 INFO:teuthology.orchestra.run.smithi039.stdout: "log_to_graylog": "false", 2024-09-14T04:38:13.000 INFO:teuthology.orchestra.run.smithi039.stdout: "log_to_journald": "false", 2024-09-14T04:38:13.000 INFO:teuthology.orchestra.run.smithi039.stdout: "log_to_stderr": "false", 2024-09-14T04:38:13.000 INFO:teuthology.orchestra.run.smithi039.stdout: "log_to_syslog": "false", 2024-09-14T04:38:13.000 INFO:teuthology.orchestra.run.smithi039.stdout: "max_rotating_auth_attempts": "10", 2024-09-14T04:38:13.000 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2024-09-14T04:38:13.000 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_action_on_write_error": "1", 2024-09-14T04:38:13.000 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_alternate_name_max": "8192", 2024-09-14T04:38:13.000 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_asio_thread_count": "2", 2024-09-14T04:38:13.000 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_bal_export_pin": "true", 2024-09-14T04:38:13.000 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_bal_fragment_dirs": "true", 2024-09-14T04:38:13.000 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2024-09-14T04:38:13.001 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_bal_fragment_interval": "5", 2024-09-14T04:38:13.001 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_bal_fragment_size_max": "100000", 2024-09-14T04:38:13.001 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_bal_idle_threshold": "0.000000", 2024-09-14T04:38:13.001 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_bal_interval": "10", 2024-09-14T04:38:13.001 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_bal_max": "-1", 2024-09-14T04:38:13.001 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_bal_max_until": "-1", 2024-09-14T04:38:13.001 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_bal_merge_size": "50", 2024-09-14T04:38:13.001 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_bal_midchunk": "0.300000", 2024-09-14T04:38:13.001 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_bal_min_rebalance": "0.100000", 2024-09-14T04:38:13.001 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_bal_min_start": "0.200000", 2024-09-14T04:38:13.001 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_bal_minchunk": "0.001000", 2024-09-14T04:38:13.001 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_bal_mode": "0", 2024-09-14T04:38:13.001 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_bal_need_max": "1.200000", 2024-09-14T04:38:13.001 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_bal_need_min": "0.800000", 2024-09-14T04:38:13.002 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_bal_overload_epochs": "2", 2024-09-14T04:38:13.002 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_bal_replicate_threshold": "8000.000000", 2024-09-14T04:38:13.002 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_bal_sample_interval": "3.000000", 2024-09-14T04:38:13.002 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_bal_split_bits": "3", 2024-09-14T04:38:13.002 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_bal_split_rd": "25000.000000", 2024-09-14T04:38:13.002 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_bal_split_size": "10000", 2024-09-14T04:38:13.002 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_bal_split_wr": "10000.000000", 2024-09-14T04:38:13.002 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_bal_target_decay": "10.000000", 2024-09-14T04:38:13.002 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2024-09-14T04:38:13.002 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_beacon_grace": "15.000000", 2024-09-14T04:38:13.002 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_beacon_interval": "4.000000", 2024-09-14T04:38:13.002 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_beacon_mon_down_grace": "60", 2024-09-14T04:38:13.002 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_cache_memory_limit": "4294967296", 2024-09-14T04:38:13.002 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_cache_mid": "0.700000", 2024-09-14T04:38:13.003 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_cache_quiesce_decay_rate": "1.000000", 2024-09-14T04:38:13.003 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_cache_quiesce_delay": "0", 2024-09-14T04:38:13.003 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_cache_quiesce_sleep": "200", 2024-09-14T04:38:13.003 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_cache_quiesce_splitauth": "true", 2024-09-14T04:38:13.003 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_cache_quiesce_threshold": "524288", 2024-09-14T04:38:13.003 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_cache_release_free_interval": "10", 2024-09-14T04:38:13.003 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_cache_reservation": "0.050000", 2024-09-14T04:38:13.003 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_cache_trim_decay_rate": "1.000000", 2024-09-14T04:38:13.003 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_cache_trim_interval": "1", 2024-09-14T04:38:13.003 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_cache_trim_threshold": "262144", 2024-09-14T04:38:13.003 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2024-09-14T04:38:13.003 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2024-09-14T04:38:13.003 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_client_delegate_inos_pct": "50", 2024-09-14T04:38:13.003 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_client_prealloc_inos": "1000", 2024-09-14T04:38:13.004 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2024-09-14T04:38:13.004 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_connect_bootstrapping": "false", 2024-09-14T04:38:13.004 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_damage_table_max_entries": "10000", 2024-09-14T04:38:13.004 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2024-09-14T04:38:13.004 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_debug_auth_pins": "false", 2024-09-14T04:38:13.004 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_debug_frag": "false", 2024-09-14T04:38:13.004 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_debug_scatterstat": "false", 2024-09-14T04:38:13.004 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_debug_subtrees": "false", 2024-09-14T04:38:13.004 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_decay_halflife": "5.000000", 2024-09-14T04:38:13.004 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_default_dir_hash": "2", 2024-09-14T04:38:13.004 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_defer_session_stale": "true", 2024-09-14T04:38:13.004 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_deny_all_reconnect": "false", 2024-09-14T04:38:13.004 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_dir_keys_per_op": "16384", 2024-09-14T04:38:13.005 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_dir_max_commit_size": "10", 2024-09-14T04:38:13.005 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_dir_max_entries": "0", 2024-09-14T04:38:13.005 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_dir_prefetch": "true", 2024-09-14T04:38:13.005 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_dirstat_min_interval": "1.000000", 2024-09-14T04:38:13.005 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_dump_cache_after_rejoin": "false", 2024-09-14T04:38:13.005 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_dump_cache_on_map": "false", 2024-09-14T04:38:13.005 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_dump_cache_threshold_file": "0", 2024-09-14T04:38:13.005 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2024-09-14T04:38:13.005 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_early_reply": "true", 2024-09-14T04:38:13.005 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_enable_op_tracker": "true", 2024-09-14T04:38:13.005 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_enforce_unique_name": "true", 2024-09-14T04:38:13.005 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_export_ephemeral_distributed": "true", 2024-09-14T04:38:13.005 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2024-09-14T04:38:13.006 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_export_ephemeral_random": "true", 2024-09-14T04:38:13.006 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_export_ephemeral_random_max": "0.010000", 2024-09-14T04:38:13.006 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_extraordinary_events_dump_interval": "0", 2024-09-14T04:38:13.006 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_forward_all_requests_to_auth": "false", 2024-09-14T04:38:13.006 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_freeze_tree_timeout": "30.000000", 2024-09-14T04:38:13.006 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_fscrypt_last_block_max_size": "4096", 2024-09-14T04:38:13.006 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_go_bad_corrupt_dentry": "true", 2024-09-14T04:38:13.006 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2024-09-14T04:38:13.006 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_health_cache_threshold": "1.500000", 2024-09-14T04:38:13.006 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_health_summarize_threshold": "10", 2024-09-14T04:38:13.006 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_heartbeat_grace": "15.000000", 2024-09-14T04:38:13.006 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_heartbeat_reset_grace": "1000", 2024-09-14T04:38:13.006 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_inject_health_dummy": "false", 2024-09-14T04:38:13.006 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2024-09-14T04:38:13.007 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_inject_migrator_session_race": "false", 2024-09-14T04:38:13.007 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2024-09-14T04:38:13.007 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_inject_skip_replaying_inotable": "false", 2024-09-14T04:38:13.007 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2024-09-14T04:38:13.007 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_join_fs": "", 2024-09-14T04:38:13.007 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_journal_format": "1", 2024-09-14T04:38:13.007 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_kill_after_journal_logs_flushed": "false", 2024-09-14T04:38:13.007 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_kill_create_at": "0", 2024-09-14T04:38:13.007 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_kill_dirfrag_at": "0", 2024-09-14T04:38:13.007 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_kill_export_at": "0", 2024-09-14T04:38:13.007 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_kill_import_at": "0", 2024-09-14T04:38:13.007 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_kill_journal_at": "0", 2024-09-14T04:38:13.007 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_kill_journal_expire_at": "0", 2024-09-14T04:38:13.007 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_kill_journal_replay_at": "0", 2024-09-14T04:38:13.008 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_kill_link_at": "0", 2024-09-14T04:38:13.008 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_kill_mdstable_at": "0", 2024-09-14T04:38:13.008 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_kill_openc_at": "0", 2024-09-14T04:38:13.008 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_kill_rename_at": "0", 2024-09-14T04:38:13.008 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_kill_shutdown_at": "0", 2024-09-14T04:38:13.008 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_log_event_large_threshold": "524288", 2024-09-14T04:38:13.008 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_log_events_per_segment": "1024", 2024-09-14T04:38:13.008 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_log_major_segment_event_ratio": "12", 2024-09-14T04:38:13.008 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_log_max_events": "-1", 2024-09-14T04:38:13.008 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_log_max_segments": "128", 2024-09-14T04:38:13.008 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_log_pause": "false", 2024-09-14T04:38:13.008 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_log_segment_size": "0", 2024-09-14T04:38:13.008 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_log_skip_corrupt_events": "false", 2024-09-14T04:38:13.009 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_log_skip_unbounded_events": "false", 2024-09-14T04:38:13.009 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_log_trim_decay_rate": "1.000000", 2024-09-14T04:38:13.009 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_log_trim_threshold": "128", 2024-09-14T04:38:13.009 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_log_trim_upkeep_interval": "1000", 2024-09-14T04:38:13.009 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_log_warn_factor": "2.000000", 2024-09-14T04:38:13.009 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_max_caps_per_client": "1048576", 2024-09-14T04:38:13.009 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_max_completed_flushes": "100000", 2024-09-14T04:38:13.009 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_max_completed_requests": "100000", 2024-09-14T04:38:13.009 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_max_export_size": "20971520", 2024-09-14T04:38:13.009 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_max_file_recover": "32", 2024-09-14T04:38:13.009 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_max_purge_files": "64", 2024-09-14T04:38:13.009 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_max_purge_ops": "8192", 2024-09-14T04:38:13.009 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2024-09-14T04:38:13.009 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_max_scrub_ops_in_progress": "5", 2024-09-14T04:38:13.010 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_max_snaps_per_dir": "100", 2024-09-14T04:38:13.010 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_metrics_update_interval": "2", 2024-09-14T04:38:13.010 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_min_caps_per_client": "100", 2024-09-14T04:38:13.010 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_min_caps_working_set": "10000", 2024-09-14T04:38:13.010 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_mon_shutdown_timeout": "5.000000", 2024-09-14T04:38:13.010 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_numa_node": "-1", 2024-09-14T04:38:13.010 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_oft_prefetch_dirfrags": "false", 2024-09-14T04:38:13.010 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_op_complaint_time": "30.000000", 2024-09-14T04:38:13.010 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_op_history_duration": "600", 2024-09-14T04:38:13.010 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_op_history_size": "20", 2024-09-14T04:38:13.010 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_op_history_slow_op_size": "20", 2024-09-14T04:38:13.010 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_op_history_slow_op_threshold": "10", 2024-09-14T04:38:13.010 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_op_log_threshold": "5", 2024-09-14T04:38:13.010 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_ping_grace": "15", 2024-09-14T04:38:13.010 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_ping_interval": "5", 2024-09-14T04:38:13.011 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2024-09-14T04:38:13.011 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_recall_global_max_decay_threshold": "131072", 2024-09-14T04:38:13.011 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_recall_max_caps": "30000", 2024-09-14T04:38:13.011 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_recall_max_decay_rate": "1.500000", 2024-09-14T04:38:13.011 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_recall_max_decay_threshold": "131072", 2024-09-14T04:38:13.011 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_recall_warning_decay_rate": "60.000000", 2024-09-14T04:38:13.011 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_recall_warning_threshold": "262144", 2024-09-14T04:38:13.011 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_reconnect_timeout": "45.000000", 2024-09-14T04:38:13.011 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_replay_interval": "1.000000", 2024-09-14T04:38:13.011 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_replay_unsafe_with_closed_session": "false", 2024-09-14T04:38:13.011 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_request_load_average_decay_rate": "60.000000", 2024-09-14T04:38:13.011 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_root_ino_gid": "0", 2024-09-14T04:38:13.011 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_root_ino_uid": "0", 2024-09-14T04:38:13.011 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_scatter_nudge_interval": "5.000000", 2024-09-14T04:38:13.012 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_scrub_stats_review_period": "1", 2024-09-14T04:38:13.012 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_server_dispatch_client_request_delay": "0", 2024-09-14T04:38:13.012 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_server_dispatch_killpoint_random": "0.000000", 2024-09-14T04:38:13.012 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_session_blocklist_on_evict": "true", 2024-09-14T04:38:13.012 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_session_blocklist_on_timeout": "true", 2024-09-14T04:38:13.012 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2024-09-14T04:38:13.012 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_session_cache_liveness_magnitude": "10", 2024-09-14T04:38:13.012 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2024-09-14T04:38:13.012 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_session_cap_acquisition_throttle": "100000", 2024-09-14T04:38:13.012 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2024-09-14T04:38:13.012 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_session_metadata_threshold": "16777216", 2024-09-14T04:38:13.012 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_sessionmap_keys_per_op": "1024", 2024-09-14T04:38:13.012 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_shutdown_check": "0", 2024-09-14T04:38:13.012 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_skip_ino": "0", 2024-09-14T04:38:13.013 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_sleep_rank_change": "0.000000", 2024-09-14T04:38:13.013 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_snap_max_uid": "4294967294", 2024-09-14T04:38:13.013 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_snap_min_uid": "0", 2024-09-14T04:38:13.013 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_snap_rstat": "false", 2024-09-14T04:38:13.014 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_standby_replay_damaged": "false", 2024-09-14T04:38:13.014 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_symlink_recovery": "true", 2024-09-14T04:38:13.014 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_task_status_update_interval": "2.000000", 2024-09-14T04:38:13.014 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_thrash_exports": "0", 2024-09-14T04:38:13.015 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_thrash_fragments": "0", 2024-09-14T04:38:13.015 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_tick_interval": "5.000000", 2024-09-14T04:38:13.015 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_valgrind_exit": "false", 2024-09-14T04:38:13.015 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_verify_backtrace": "1", 2024-09-14T04:38:13.015 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_verify_scatter": "false", 2024-09-14T04:38:13.015 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_wipe_ino_prealloc": "false", 2024-09-14T04:38:13.015 INFO:teuthology.orchestra.run.smithi039.stdout: "mds_wipe_sessions": "false", 2024-09-14T04:38:13.015 INFO:teuthology.orchestra.run.smithi039.stdout: "mempool_debug": "false", 2024-09-14T04:38:13.015 INFO:teuthology.orchestra.run.smithi039.stdout: "memstore_debug_omit_block_device_write": "false", 2024-09-14T04:38:13.015 INFO:teuthology.orchestra.run.smithi039.stdout: "memstore_device_bytes": "1073741824", 2024-09-14T04:38:13.015 INFO:teuthology.orchestra.run.smithi039.stdout: "memstore_page_set": "false", 2024-09-14T04:38:13.015 INFO:teuthology.orchestra.run.smithi039.stdout: "memstore_page_size": "65536", 2024-09-14T04:38:13.015 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_client_bytes": "134217728", 2024-09-14T04:38:13.016 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_client_messages": "512", 2024-09-14T04:38:13.016 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2024-09-14T04:38:13.016 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_connect_retry_interval": "1.000000", 2024-09-14T04:38:13.016 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2024-09-14T04:38:13.016 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2024-09-14T04:38:13.016 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_disabled_modules": "", 2024-09-14T04:38:13.016 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_enable_op_tracker": "true", 2024-09-14T04:38:13.016 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_initial_modules": "restful iostat nfs", 2024-09-14T04:38:13.016 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_max_pg_creating": "1024", 2024-09-14T04:38:13.016 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_max_pg_num_change": "128", 2024-09-14T04:38:13.016 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_mds_bytes": "134217728", 2024-09-14T04:38:13.016 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_mds_messages": "128", 2024-09-14T04:38:13.016 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2024-09-14T04:38:13.017 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_mon_bytes": "134217728", 2024-09-14T04:38:13.017 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_mon_messages": "128", 2024-09-14T04:38:13.017 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_num_op_tracker_shard": "32", 2024-09-14T04:38:13.017 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_op_complaint_time": "30.000000", 2024-09-14T04:38:13.017 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_op_history_duration": "600", 2024-09-14T04:38:13.017 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_op_history_size": "20", 2024-09-14T04:38:13.017 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_op_history_slow_op_size": "20", 2024-09-14T04:38:13.017 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_op_history_slow_op_threshold": "10.000000", 2024-09-14T04:38:13.017 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_op_log_threshold": "5", 2024-09-14T04:38:13.017 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_osd_bytes": "536870912", 2024-09-14T04:38:13.017 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_osd_messages": "8192", 2024-09-14T04:38:13.017 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_pool": "true", 2024-09-14T04:38:13.018 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_service_beacon_grace": "60.000000", 2024-09-14T04:38:13.018 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_standby_modules": "true", 2024-09-14T04:38:13.018 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_stats_period": "5", 2024-09-14T04:38:13.018 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_stats_threshold": "5", 2024-09-14T04:38:13.018 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_tick_period": "2", 2024-09-14T04:38:13.018 INFO:teuthology.orchestra.run.smithi039.stdout: "mgr_ttl_cache_expire_seconds": "0", 2024-09-14T04:38:13.018 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_accept_timeout_factor": "2.000000", 2024-09-14T04:38:13.018 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_allow_pool_delete": "true", 2024-09-14T04:38:13.018 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_allow_pool_size_one": "false", 2024-09-14T04:38:13.018 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_auth_validate_all_caps": "true", 2024-09-14T04:38:13.018 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2024-09-14T04:38:13.018 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2024-09-14T04:38:13.018 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_client_bytes": "104857600", 2024-09-14T04:38:13.018 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_client_directed_command_retry": "2", 2024-09-14T04:38:13.019 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_client_hunt_interval": "3.000000", 2024-09-14T04:38:13.019 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2024-09-14T04:38:13.019 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2024-09-14T04:38:13.019 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2024-09-14T04:38:13.019 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_client_hunt_parallel": "3", 2024-09-14T04:38:13.019 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_client_log_interval": "1.000000", 2024-09-14T04:38:13.019 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_client_max_log_entries_per_message": "1000", 2024-09-14T04:38:13.019 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_client_ping_interval": "10.000000", 2024-09-14T04:38:13.019 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_client_ping_timeout": "30.000000", 2024-09-14T04:38:13.019 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_client_target_rank": "-1", 2024-09-14T04:38:13.019 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_clock_drift_allowed": "1.000000", 2024-09-14T04:38:13.019 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2024-09-14T04:38:13.019 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2024-09-14T04:38:13.020 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_cluster_log_level": "debug", 2024-09-14T04:38:13.020 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_cluster_log_to_file": "true", 2024-09-14T04:38:13.020 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_cluster_log_to_graylog": "false", 2024-09-14T04:38:13.020 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2024-09-14T04:38:13.020 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_cluster_log_to_graylog_port": "12201", 2024-09-14T04:38:13.020 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_cluster_log_to_journald": "true", 2024-09-14T04:38:13.020 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_cluster_log_to_stderr": "false", 2024-09-14T04:38:13.020 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_cluster_log_to_syslog": "default=false", 2024-09-14T04:38:13.020 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2024-09-14T04:38:13.020 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_compact_on_bootstrap": "false", 2024-09-14T04:38:13.020 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_compact_on_start": "false", 2024-09-14T04:38:13.020 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_compact_on_trim": "true", 2024-09-14T04:38:13.020 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_con_tracker_persist_interval": "10", 2024-09-14T04:38:13.021 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_con_tracker_score_halflife": "43200", 2024-09-14T04:38:13.021 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_config_key_max_entry_size": "65536", 2024-09-14T04:38:13.021 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_cpu_threads": "4", 2024-09-14T04:38:13.021 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_crush_min_required_version": "hammer", 2024-09-14T04:38:13.021 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_daemon_bytes": "419430400", 2024-09-14T04:38:13.021 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2024-09-14T04:38:13.021 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_data_avail_crit": "5", 2024-09-14T04:38:13.021 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_data_avail_warn": "5", 2024-09-14T04:38:13.021 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_data_size_warn": "16106127360", 2024-09-14T04:38:13.021 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_debug_block_osdmap_trim": "false", 2024-09-14T04:38:13.021 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_debug_deprecated_as_obsolete": "false", 2024-09-14T04:38:13.021 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_debug_dump_json": "false", 2024-09-14T04:38:13.021 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2024-09-14T04:38:13.022 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_debug_dump_transactions": "false", 2024-09-14T04:38:13.022 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_debug_extra_checks": "false", 2024-09-14T04:38:13.022 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_debug_no_initial_persistent_features": "false", 2024-09-14T04:38:13.022 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2024-09-14T04:38:13.022 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_debug_no_require_reef": "false", 2024-09-14T04:38:13.022 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_debug_no_require_squid": "false", 2024-09-14T04:38:13.022 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2024-09-14T04:38:13.022 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_delta_reset_interval": "10.000000", 2024-09-14T04:38:13.022 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_dns_srv_name": "ceph-mon", 2024-09-14T04:38:13.022 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_down_mkfs_grace": "60", 2024-09-14T04:38:13.022 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_down_uptime_grace": "60", 2024-09-14T04:38:13.022 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_election_default_strategy": "1", 2024-09-14T04:38:13.022 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_election_timeout": "5.000000", 2024-09-14T04:38:13.023 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2024-09-14T04:38:13.023 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_elector_ping_divisor": "2", 2024-09-14T04:38:13.023 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_elector_ping_timeout": "2.000000", 2024-09-14T04:38:13.023 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_enable_op_tracker": "true", 2024-09-14T04:38:13.023 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_fake_pool_delete": "false", 2024-09-14T04:38:13.023 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_force_quorum_join": "false", 2024-09-14T04:38:13.023 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_fsmap_prune_threshold": "300", 2024-09-14T04:38:13.023 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_globalid_prealloc": "10000", 2024-09-14T04:38:13.023 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_health_detail_to_clog": "true", 2024-09-14T04:38:13.023 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_health_log_update_period": "5", 2024-09-14T04:38:13.023 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_health_max_detail": "50", 2024-09-14T04:38:13.023 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_health_to_clog": "true", 2024-09-14T04:38:13.023 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_health_to_clog_interval": "600", 2024-09-14T04:38:13.024 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2024-09-14T04:38:13.024 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_host": "[v2:172.21.15.39:3300/0,v1:172.21.15.39:6789/0] [v2:172.21.15.129:3300/0,v1:172.21.15.129:6789/0] [v2:172.21.15.39:3301/0,v1:172.21.15.39:6790/0]", 2024-09-14T04:38:13.024 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_host_override": "", 2024-09-14T04:38:13.024 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_initial_members": "", 2024-09-14T04:38:13.024 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2024-09-14T04:38:13.024 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2024-09-14T04:38:13.024 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_inject_transaction_delay_max": "10.000000", 2024-09-14T04:38:13.024 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2024-09-14T04:38:13.024 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_keyvaluedb": "rocksdb", 2024-09-14T04:38:13.024 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_lease": "5.000000", 2024-09-14T04:38:13.024 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2024-09-14T04:38:13.024 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_lease_renew_interval_factor": "0.600000", 2024-09-14T04:38:13.024 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_log_full_interval": "50", 2024-09-14T04:38:13.025 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_log_max": "10000", 2024-09-14T04:38:13.025 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_log_max_summary": "50", 2024-09-14T04:38:13.025 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_max_log_entries_per_event": "4096", 2024-09-14T04:38:13.025 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_max_log_epochs": "500", 2024-09-14T04:38:13.025 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_max_mdsmap_epochs": "500", 2024-09-14T04:38:13.025 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_max_mgrmap_epochs": "500", 2024-09-14T04:38:13.025 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_max_nvmeof_epochs": "500", 2024-09-14T04:38:13.025 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_max_osd": "10000", 2024-09-14T04:38:13.025 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_max_pg_per_osd": "10000", 2024-09-14T04:38:13.025 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_max_pool_pg_num": "65536", 2024-09-14T04:38:13.025 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_max_snap_prune_per_epoch": "100", 2024-09-14T04:38:13.025 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_mds_blocklist_interval": "86400.000000", 2024-09-14T04:38:13.025 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_mds_force_trim_to": "0", 2024-09-14T04:38:13.025 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_mds_skip_sanity": "true", 2024-09-14T04:38:13.026 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_memory_autotune": "true", 2024-09-14T04:38:13.026 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_memory_target": "2147483648", 2024-09-14T04:38:13.026 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_mgr_beacon_grace": "30", 2024-09-14T04:38:13.026 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2024-09-14T04:38:13.026 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_mgr_digest_period": "5", 2024-09-14T04:38:13.026 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_mgr_inactive_grace": "60", 2024-09-14T04:38:13.026 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_mgr_mkfs_grace": "240", 2024-09-14T04:38:13.026 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2024-09-14T04:38:13.026 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_min_osdmap_epochs": "500", 2024-09-14T04:38:13.026 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_nvmeofgw_beacon_grace": "10", 2024-09-14T04:38:13.026 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_nvmeofgw_set_group_id_retry": "1000", 2024-09-14T04:38:13.026 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_op_complaint_time": "30", 2024-09-14T04:38:13.026 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_op_history_duration": "600", 2024-09-14T04:38:13.026 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_op_history_size": "20", 2024-09-14T04:38:13.027 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_op_history_slow_op_size": "20", 2024-09-14T04:38:13.027 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_op_history_slow_op_threshold": "10", 2024-09-14T04:38:13.027 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_op_log_threshold": "5", 2024-09-14T04:38:13.027 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_adjust_down_out_interval": "true", 2024-09-14T04:38:13.027 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2024-09-14T04:38:13.027 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2024-09-14T04:38:13.027 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_auto_mark_in": "false", 2024-09-14T04:38:13.027 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_auto_mark_new_in": "true", 2024-09-14T04:38:13.027 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2024-09-14T04:38:13.027 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2024-09-14T04:38:13.027 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_cache_size": "500", 2024-09-14T04:38:13.027 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_cache_size_min": "134217728", 2024-09-14T04:38:13.027 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_crush_smoke_test": "true", 2024-09-14T04:38:13.027 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_destroyed_out_interval": "600", 2024-09-14T04:38:13.028 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_down_out_interval": "600", 2024-09-14T04:38:13.028 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_down_out_subtree_limit": "rack", 2024-09-14T04:38:13.028 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2024-09-14T04:38:13.028 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_force_trim_to": "0", 2024-09-14T04:38:13.028 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_full_ratio": "0.950000", 2024-09-14T04:38:13.028 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2024-09-14T04:38:13.028 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_laggy_halflife": "3600", 2024-09-14T04:38:13.028 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_laggy_max_interval": "300", 2024-09-14T04:38:13.028 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_laggy_weight": "0.300000", 2024-09-14T04:38:13.028 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2024-09-14T04:38:13.028 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_max_initial_pgs": "1024", 2024-09-14T04:38:13.028 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_min_down_reporters": "2", 2024-09-14T04:38:13.028 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_min_in_ratio": "0.750000", 2024-09-14T04:38:13.029 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_min_up_ratio": "0.300000", 2024-09-14T04:38:13.029 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_nearfull_ratio": "0.850000", 2024-09-14T04:38:13.029 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_prime_pg_temp": "true", 2024-09-14T04:38:13.029 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2024-09-14T04:38:13.029 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2024-09-14T04:38:13.029 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_report_timeout": "900", 2024-09-14T04:38:13.029 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_reporter_subtree_level": "osd", 2024-09-14T04:38:13.029 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2024-09-14T04:38:13.029 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_warn_num_repaired": "10", 2024-09-14T04:38:13.029 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osd_warn_op_age": "32.000000", 2024-09-14T04:38:13.029 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osdmap_full_prune_enabled": "true", 2024-09-14T04:38:13.029 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osdmap_full_prune_interval": "10", 2024-09-14T04:38:13.029 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osdmap_full_prune_min": "10000", 2024-09-14T04:38:13.030 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_osdmap_full_prune_txsize": "100", 2024-09-14T04:38:13.030 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2024-09-14T04:38:13.030 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_pg_stuck_threshold": "60", 2024-09-14T04:38:13.030 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2024-09-14T04:38:13.030 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_pg_warn_min_objects": "10000", 2024-09-14T04:38:13.030 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_pg_warn_min_per_osd": "0", 2024-09-14T04:38:13.030 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_pg_warn_min_pool_objects": "1000", 2024-09-14T04:38:13.030 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_pool_quota_crit_threshold": "0", 2024-09-14T04:38:13.030 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_pool_quota_warn_threshold": "0", 2024-09-14T04:38:13.030 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_probe_timeout": "2.000000", 2024-09-14T04:38:13.030 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_reweight_max_change": "0.050000", 2024-09-14T04:38:13.030 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_reweight_max_osds": "4", 2024-09-14T04:38:13.030 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_reweight_min_bytes_per_osd": "10", 2024-09-14T04:38:13.030 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_reweight_min_pgs_per_osd": "4", 2024-09-14T04:38:13.031 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2024-09-14T04:38:13.031 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2024-09-14T04:38:13.031 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2024-09-14T04:38:13.031 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_scrub_interval": "86400", 2024-09-14T04:38:13.031 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_scrub_max_keys": "100", 2024-09-14T04:38:13.031 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_scrub_timeout": "300", 2024-09-14T04:38:13.031 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_session_timeout": "300", 2024-09-14T04:38:13.031 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_smart_report_timeout": "5", 2024-09-14T04:38:13.031 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_stat_smooth_intervals": "6", 2024-09-14T04:38:13.031 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2024-09-14T04:38:13.031 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_stretch_pool_min_size": "2", 2024-09-14T04:38:13.031 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_stretch_pool_size": "4", 2024-09-14T04:38:13.031 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2024-09-14T04:38:13.031 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_subscribe_interval": "86400.000000", 2024-09-14T04:38:13.031 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_sync_debug": "false", 2024-09-14T04:38:13.032 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_sync_max_payload_keys": "2000", 2024-09-14T04:38:13.032 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_sync_max_payload_size": "1048576", 2024-09-14T04:38:13.032 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_sync_provider_kill_at": "0", 2024-09-14T04:38:13.032 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_sync_requester_kill_at": "0", 2024-09-14T04:38:13.032 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_sync_timeout": "60.000000", 2024-09-14T04:38:13.032 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_target_pg_per_osd": "100", 2024-09-14T04:38:13.032 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_tick_interval": "5", 2024-09-14T04:38:13.032 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_timecheck_interval": "300.000000", 2024-09-14T04:38:13.032 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_timecheck_skew_interval": "30.000000", 2024-09-14T04:38:13.032 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_use_min_delay_socket": "false", 2024-09-14T04:38:13.032 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_warn_older_version_delay": "604800", 2024-09-14T04:38:13.032 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2024-09-14T04:38:13.032 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2024-09-14T04:38:13.032 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2024-09-14T04:38:13.033 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_warn_on_filestore_osds": "true", 2024-09-14T04:38:13.033 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2024-09-14T04:38:13.033 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2024-09-14T04:38:13.033 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2024-09-14T04:38:13.033 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_warn_on_misplaced": "false", 2024-09-14T04:38:13.033 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2024-09-14T04:38:13.033 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_warn_on_older_version": "true", 2024-09-14T04:38:13.033 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2024-09-14T04:38:13.033 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_warn_on_pool_no_app": "true", 2024-09-14T04:38:13.033 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_warn_on_pool_no_redundancy": "true", 2024-09-14T04:38:13.033 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2024-09-14T04:38:13.033 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2024-09-14T04:38:13.033 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2024-09-14T04:38:13.033 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_warn_on_too_few_osds": "false", 2024-09-14T04:38:13.033 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2024-09-14T04:38:13.034 INFO:teuthology.orchestra.run.smithi039.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2024-09-14T04:38:13.034 INFO:teuthology.orchestra.run.smithi039.stdout: "monmap": "", 2024-09-14T04:38:13.034 INFO:teuthology.orchestra.run.smithi039.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2024-09-14T04:38:13.034 INFO:teuthology.orchestra.run.smithi039.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2024-09-14T04:38:13.034 INFO:teuthology.orchestra.run.smithi039.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2024-09-14T04:38:13.034 INFO:teuthology.orchestra.run.smithi039.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2024-09-14T04:38:13.034 INFO:teuthology.orchestra.run.smithi039.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2024-09-14T04:38:13.034 INFO:teuthology.orchestra.run.smithi039.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2024-09-14T04:38:13.034 INFO:teuthology.orchestra.run.smithi039.stdout: "motr_tracing_enabled": "false", 2024-09-14T04:38:13.034 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_async_op_threads": "3", 2024-09-14T04:38:13.034 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_async_rdma_buffer_size": "131072", 2024-09-14T04:38:13.034 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_async_rdma_cm": "false", 2024-09-14T04:38:13.034 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_async_rdma_device_name": "", 2024-09-14T04:38:13.034 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_async_rdma_dscp": "96", 2024-09-14T04:38:13.035 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_async_rdma_enable_hugepage": "false", 2024-09-14T04:38:13.035 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_async_rdma_gid_idx": "0", 2024-09-14T04:38:13.035 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_async_rdma_local_gid": "", 2024-09-14T04:38:13.035 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_async_rdma_polling_us": "1000", 2024-09-14T04:38:13.035 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_async_rdma_port_num": "1", 2024-09-14T04:38:13.035 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_async_rdma_receive_buffers": "32768", 2024-09-14T04:38:13.035 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_async_rdma_receive_queue_len": "4096", 2024-09-14T04:38:13.035 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_async_rdma_roce_ver": "1", 2024-09-14T04:38:13.035 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_async_rdma_send_buffers": "1024", 2024-09-14T04:38:13.035 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_async_rdma_sl": "3", 2024-09-14T04:38:13.035 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_async_rdma_support_srq": "true", 2024-09-14T04:38:13.035 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_async_rdma_type": "ib", 2024-09-14T04:38:13.035 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_async_reap_threshold": "5", 2024-09-14T04:38:13.036 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_bind_before_connect": "false", 2024-09-14T04:38:13.036 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_bind_ipv4": "true", 2024-09-14T04:38:13.036 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_bind_ipv6": "false", 2024-09-14T04:38:13.036 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_bind_msgr1": "true", 2024-09-14T04:38:13.036 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_bind_msgr2": "true", 2024-09-14T04:38:13.036 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_bind_port_max": "7568", 2024-09-14T04:38:13.036 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_bind_port_min": "6800", 2024-09-14T04:38:13.036 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_bind_prefer_ipv4": "false", 2024-09-14T04:38:13.036 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_bind_retry_count": "3", 2024-09-14T04:38:13.036 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_bind_retry_delay": "5", 2024-09-14T04:38:13.036 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_blackhole_client": "false", 2024-09-14T04:38:13.036 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_blackhole_mds": "false", 2024-09-14T04:38:13.036 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_blackhole_mgr": "false", 2024-09-14T04:38:13.036 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_blackhole_mon": "false", 2024-09-14T04:38:13.037 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_blackhole_osd": "false", 2024-09-14T04:38:13.037 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_client_mode": "crc secure", 2024-09-14T04:38:13.037 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_client_throttle_retry_time_interval": "5000", 2024-09-14T04:38:13.037 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_cluster_mode": "crc secure", 2024-09-14T04:38:13.037 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_cluster_type": "", 2024-09-14T04:38:13.037 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_compress_secure": "false", 2024-09-14T04:38:13.037 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_connection_idle_timeout": "900", 2024-09-14T04:38:13.037 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_connection_ready_timeout": "10", 2024-09-14T04:38:13.037 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_crc_data": "true", 2024-09-14T04:38:13.037 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_crc_header": "true", 2024-09-14T04:38:13.037 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_die_on_bad_msg": "false", 2024-09-14T04:38:13.037 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_die_on_bug": "true", 2024-09-14T04:38:13.037 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_die_on_old_message": "true", 2024-09-14T04:38:13.038 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_die_on_skipped_message": "false", 2024-09-14T04:38:13.038 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_die_on_unhandled_msg": "false", 2024-09-14T04:38:13.038 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_dispatch_throttle_bytes": "104857600", 2024-09-14T04:38:13.038 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_dpdk_coremask": "0xF", 2024-09-14T04:38:13.038 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_dpdk_debug_allow_loopback": "false", 2024-09-14T04:38:13.038 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_dpdk_devs_allowlist": "", 2024-09-14T04:38:13.038 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_dpdk_enable_tso": "true", 2024-09-14T04:38:13.038 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2024-09-14T04:38:13.038 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_dpdk_host_ipv4_addr": "", 2024-09-14T04:38:13.038 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_dpdk_hugepages": "", 2024-09-14T04:38:13.038 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_dpdk_hw_flow_control": "true", 2024-09-14T04:38:13.038 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2024-09-14T04:38:13.038 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_dpdk_lro": "true", 2024-09-14T04:38:13.038 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_dpdk_memory_channel": "4", 2024-09-14T04:38:13.039 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2024-09-14T04:38:13.039 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_dpdk_pmd": "", 2024-09-14T04:38:13.039 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_dpdk_port_id": "0", 2024-09-14T04:38:13.039 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2024-09-14T04:38:13.039 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_dump_corrupt_message_level": "1", 2024-09-14T04:38:13.039 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_dump_on_send": "false", 2024-09-14T04:38:13.039 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_initial_backoff": "0.200000", 2024-09-14T04:38:13.039 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_inject_delay_max": "1.000000", 2024-09-14T04:38:13.039 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_inject_delay_probability": "0.000000", 2024-09-14T04:38:13.039 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_inject_delay_type": "", 2024-09-14T04:38:13.039 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_inject_internal_delays": "0.000000", 2024-09-14T04:38:13.039 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_inject_network_congestion": "0", 2024-09-14T04:38:13.039 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_inject_socket_failures": "0", 2024-09-14T04:38:13.040 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_learn_addr_from_peer": "true", 2024-09-14T04:38:13.040 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_max_accept_failures": "4", 2024-09-14T04:38:13.040 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_max_backoff": "15.000000", 2024-09-14T04:38:13.040 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_mon_client_mode": "secure crc", 2024-09-14T04:38:13.040 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_mon_cluster_mode": "secure crc", 2024-09-14T04:38:13.040 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_mon_service_mode": "secure crc", 2024-09-14T04:38:13.040 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_osd_compress_min_size": "1024", 2024-09-14T04:38:13.040 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_osd_compress_mode": "none", 2024-09-14T04:38:13.040 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_osd_compression_algorithm": "snappy", 2024-09-14T04:38:13.040 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2024-09-14T04:38:13.040 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_pq_min_cost": "65536", 2024-09-14T04:38:13.040 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_public_type": "", 2024-09-14T04:38:13.040 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_service_mode": "crc secure", 2024-09-14T04:38:13.040 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_tcp_listen_backlog": "512", 2024-09-14T04:38:13.041 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_tcp_nodelay": "true", 2024-09-14T04:38:13.041 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_tcp_prefetch_max_size": "4096", 2024-09-14T04:38:13.041 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_tcp_rcvbuf": "0", 2024-09-14T04:38:13.041 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_time_events_min_wait_interval": "1000", 2024-09-14T04:38:13.041 INFO:teuthology.orchestra.run.smithi039.stdout: "ms_type": "async+posix", 2024-09-14T04:38:13.041 INFO:teuthology.orchestra.run.smithi039.stdout: "no_config_file": "false", 2024-09-14T04:38:13.041 INFO:teuthology.orchestra.run.smithi039.stdout: "nvmeof_mon_client_disconnect_panic": "100", 2024-09-14T04:38:13.041 INFO:teuthology.orchestra.run.smithi039.stdout: "nvmeof_mon_client_tick_period": "2", 2024-09-14T04:38:13.041 INFO:teuthology.orchestra.run.smithi039.stdout: "objecter_completion_locks_per_session": "32", 2024-09-14T04:38:13.041 INFO:teuthology.orchestra.run.smithi039.stdout: "objecter_debug_inject_relock_delay": "false", 2024-09-14T04:38:13.041 INFO:teuthology.orchestra.run.smithi039.stdout: "objecter_inflight_op_bytes": "104857600", 2024-09-14T04:38:13.041 INFO:teuthology.orchestra.run.smithi039.stdout: "objecter_inflight_ops": "1024", 2024-09-14T04:38:13.041 INFO:teuthology.orchestra.run.smithi039.stdout: "objecter_inject_no_watch_ping": "false", 2024-09-14T04:38:13.041 INFO:teuthology.orchestra.run.smithi039.stdout: "objecter_retry_writes_after_first_reply": "false", 2024-09-14T04:38:13.042 INFO:teuthology.orchestra.run.smithi039.stdout: "objecter_tick_interval": "5.000000", 2024-09-14T04:38:13.042 INFO:teuthology.orchestra.run.smithi039.stdout: "objecter_timeout": "10.000000", 2024-09-14T04:38:13.042 INFO:teuthology.orchestra.run.smithi039.stdout: "objectstore_blackhole": "false", 2024-09-14T04:38:13.042 INFO:teuthology.orchestra.run.smithi039.stdout: "objectstore_debug_throw_on_failed_txc": "false", 2024-09-14T04:38:13.042 INFO:teuthology.orchestra.run.smithi039.stdout: "openssl_engine_opts": "", 2024-09-14T04:38:13.042 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_agent_delay_time": "5.000000", 2024-09-14T04:38:13.042 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_agent_hist_halflife": "1000", 2024-09-14T04:38:13.042 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_agent_max_low_ops": "2", 2024-09-14T04:38:13.042 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_agent_max_ops": "4", 2024-09-14T04:38:13.042 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_agent_min_evict_effort": "0.100000", 2024-09-14T04:38:13.042 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_agent_quantize_effort": "0.100000", 2024-09-14T04:38:13.042 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_agent_slop": "0.020000", 2024-09-14T04:38:13.042 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_aggregated_slow_ops_logging": "true", 2024-09-14T04:38:13.043 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_allow_recovery_below_min_size": "true", 2024-09-14T04:38:13.043 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_asio_thread_count": "2", 2024-09-14T04:38:13.043 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_async_recovery_min_cost": "100", 2024-09-14T04:38:13.043 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_auto_mark_unfound_lost": "false", 2024-09-14T04:38:13.043 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_backfill_retry_interval": "30.000000", 2024-09-14T04:38:13.043 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_backfill_scan_max": "512", 2024-09-14T04:38:13.043 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_backfill_scan_min": "64", 2024-09-14T04:38:13.043 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_backoff_on_degraded": "false", 2024-09-14T04:38:13.043 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_backoff_on_peering": "false", 2024-09-14T04:38:13.043 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_backoff_on_unfound": "true", 2024-09-14T04:38:13.043 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_beacon_report_interval": "300", 2024-09-14T04:38:13.043 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_bench_duration": "30", 2024-09-14T04:38:13.043 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_bench_large_size_max_throughput": "104857600", 2024-09-14T04:38:13.043 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_bench_max_block_size": "67108864", 2024-09-14T04:38:13.044 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_bench_small_size_max_iops": "100", 2024-09-14T04:38:13.044 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_blkin_trace_all": "false", 2024-09-14T04:38:13.044 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_blocked_scrub_grace_period": "120", 2024-09-14T04:38:13.044 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2024-09-14T04:38:13.044 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2024-09-14T04:38:13.044 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2024-09-14T04:38:13.044 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_check_for_log_corruption": "false", 2024-09-14T04:38:13.044 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_check_max_object_name_len_on_startup": "true", 2024-09-14T04:38:13.044 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2024-09-14T04:38:13.044 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2024-09-14T04:38:13.044 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2024-09-14T04:38:13.044 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_class_update_on_start": "true", 2024-09-14T04:38:13.044 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_client_message_cap": "256", 2024-09-14T04:38:13.045 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_client_message_size_cap": "524288000", 2024-09-14T04:38:13.045 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_client_op_priority": "63", 2024-09-14T04:38:13.045 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_client_watch_timeout": "30", 2024-09-14T04:38:13.045 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_command_max_records": "256", 2024-09-14T04:38:13.045 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_compact_on_start": "false", 2024-09-14T04:38:13.045 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_copyfrom_max_chunk": "8388608", 2024-09-14T04:38:13.045 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_crush_chooseleaf_type": "1", 2024-09-14T04:38:13.045 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_crush_initial_weight": "-1.000000", 2024-09-14T04:38:13.045 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_crush_update_on_start": "true", 2024-09-14T04:38:13.045 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_crush_update_weight_set": "true", 2024-09-14T04:38:13.045 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2024-09-14T04:38:13.045 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2024-09-14T04:38:13.046 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2024-09-14T04:38:13.046 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_debug_drop_ping_duration": "0", 2024-09-14T04:38:13.046 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_debug_drop_ping_probability": "0.000000", 2024-09-14T04:38:13.046 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_debug_feed_pullee": "-1", 2024-09-14T04:38:13.046 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_debug_inject_copyfrom_error": "false", 2024-09-14T04:38:13.046 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2024-09-14T04:38:13.046 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2024-09-14T04:38:13.046 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_debug_misdirected_ops": "false", 2024-09-14T04:38:13.046 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_debug_no_acting_change": "false", 2024-09-14T04:38:13.046 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_debug_no_purge_strays": "false", 2024-09-14T04:38:13.046 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_debug_op_order": "false", 2024-09-14T04:38:13.046 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_debug_pg_log_writeout": "false", 2024-09-14T04:38:13.046 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_debug_pretend_recovery_active": "false", 2024-09-14T04:38:13.047 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_debug_random_push_read_error": "0.000000", 2024-09-14T04:38:13.047 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2024-09-14T04:38:13.047 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_debug_shutdown": "false", 2024-09-14T04:38:13.047 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2024-09-14T04:38:13.047 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2024-09-14T04:38:13.047 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_debug_trim_objects": "false", 2024-09-14T04:38:13.047 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_debug_verify_cached_snaps": "false", 2024-09-14T04:38:13.047 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_debug_verify_missing_on_start": "false", 2024-09-14T04:38:13.047 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_debug_verify_snaps": "false", 2024-09-14T04:38:13.047 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_debug_verify_stray_on_activate": "false", 2024-09-14T04:38:13.047 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_deep_scrub_interval": "604800.000000", 2024-09-14T04:38:13.047 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_deep_scrub_keys": "1024", 2024-09-14T04:38:13.047 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2024-09-14T04:38:13.048 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2024-09-14T04:38:13.048 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2024-09-14T04:38:13.048 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_deep_scrub_stride": "524288", 2024-09-14T04:38:13.048 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2024-09-14T04:38:13.048 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_default_data_pool_replay_window": "45", 2024-09-14T04:38:13.048 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_default_notify_timeout": "30", 2024-09-14T04:38:13.048 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_delete_sleep": "0.000000", 2024-09-14T04:38:13.048 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_delete_sleep_hdd": "5.000000", 2024-09-14T04:38:13.048 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_delete_sleep_hybrid": "1.000000", 2024-09-14T04:38:13.048 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_delete_sleep_ssd": "1.000000", 2024-09-14T04:38:13.048 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_discard_disconnected_ops": "true", 2024-09-14T04:38:13.048 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_ec_partial_reads": "true", 2024-09-14T04:38:13.048 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_enable_op_tracker": "true", 2024-09-14T04:38:13.048 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2024-09-14T04:38:13.049 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_extblkdev_plugins": "vdo", 2024-09-14T04:38:13.049 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_failsafe_full_ratio": "0.970000", 2024-09-14T04:38:13.049 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_fast_fail_on_connection_refused": "true", 2024-09-14T04:38:13.049 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_fast_info": "true", 2024-09-14T04:38:13.049 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_fast_shutdown": "true", 2024-09-14T04:38:13.049 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_fast_shutdown_notify_mon": "true", 2024-09-14T04:38:13.049 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_fast_shutdown_timeout": "15", 2024-09-14T04:38:13.049 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_find_best_info_ignore_history_les": "false", 2024-09-14T04:38:13.049 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_force_auth_primary_missing_objects": "100", 2024-09-14T04:38:13.049 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2024-09-14T04:38:13.049 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_function_tracing": "false", 2024-09-14T04:38:13.049 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_heartbeat_grace": "20", 2024-09-14T04:38:13.049 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_heartbeat_interval": "6", 2024-09-14T04:38:13.050 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2024-09-14T04:38:13.050 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_heartbeat_min_peers": "10", 2024-09-14T04:38:13.050 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_heartbeat_min_size": "2000", 2024-09-14T04:38:13.050 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_heartbeat_stale": "600", 2024-09-14T04:38:13.050 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2024-09-14T04:38:13.050 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_hit_set_max_size": "100000", 2024-09-14T04:38:13.050 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_hit_set_min_size": "1000", 2024-09-14T04:38:13.050 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_hit_set_namespace": ".ceph-internal", 2024-09-14T04:38:13.050 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_ignore_stale_divergent_priors": "false", 2024-09-14T04:38:13.050 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2024-09-14T04:38:13.050 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_inject_failure_on_pg_removal": "false", 2024-09-14T04:38:13.050 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2024-09-14T04:38:13.050 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_journal_flush_on_shutdown": "true", 2024-09-14T04:38:13.051 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_journal_size": "5120", 2024-09-14T04:38:13.051 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_kill_backfill_at": "0", 2024-09-14T04:38:13.051 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_loop_before_reset_tphandle": "64", 2024-09-14T04:38:13.051 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_map_cache_size": "50", 2024-09-14T04:38:13.051 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_map_dedup": "true", 2024-09-14T04:38:13.051 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_map_message_max": "40", 2024-09-14T04:38:13.051 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_map_message_max_bytes": "10485760", 2024-09-14T04:38:13.051 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_map_share_max_epochs": "40", 2024-09-14T04:38:13.051 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_max_attr_name_len": "100", 2024-09-14T04:38:13.051 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_max_attr_size": "0", 2024-09-14T04:38:13.051 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_max_backfills": "1", 2024-09-14T04:38:13.051 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_max_markdown_count": "5", 2024-09-14T04:38:13.051 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_max_markdown_period": "600", 2024-09-14T04:38:13.051 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_max_object_name_len": "2048", 2024-09-14T04:38:13.052 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_max_object_namespace_len": "256", 2024-09-14T04:38:13.052 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_max_object_size": "134217728", 2024-09-14T04:38:13.052 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2024-09-14T04:38:13.052 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_max_omap_entries_per_request": "1024", 2024-09-14T04:38:13.052 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_max_pg_blocked_by": "16", 2024-09-14T04:38:13.052 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_max_pg_log_entries": "10000", 2024-09-14T04:38:13.052 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2024-09-14T04:38:13.052 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_max_pgls": "1024", 2024-09-14T04:38:13.052 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_max_push_cost": "8388608", 2024-09-14T04:38:13.052 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_max_push_objects": "10", 2024-09-14T04:38:13.052 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_max_scrubs": "3", 2024-09-14T04:38:13.052 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2024-09-14T04:38:13.052 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_max_trimming_pgs": "2", 2024-09-14T04:38:13.053 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_max_write_op_reply_len": "64", 2024-09-14T04:38:13.053 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_max_write_size": "90", 2024-09-14T04:38:13.053 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2024-09-14T04:38:13.053 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2024-09-14T04:38:13.053 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2024-09-14T04:38:13.053 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2024-09-14T04:38:13.053 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2024-09-14T04:38:13.053 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2024-09-14T04:38:13.053 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2024-09-14T04:38:13.053 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_mclock_override_recovery_settings": "false", 2024-09-14T04:38:13.053 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_mclock_profile": "balanced", 2024-09-14T04:38:13.053 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2024-09-14T04:38:13.053 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2024-09-14T04:38:13.054 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2024-09-14T04:38:13.054 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2024-09-14T04:38:13.054 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2024-09-14T04:38:13.054 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2024-09-14T04:38:13.054 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2024-09-14T04:38:13.054 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2024-09-14T04:38:13.054 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2024-09-14T04:38:13.054 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_mclock_scheduler_client_wgt": "1", 2024-09-14T04:38:13.054 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_mclock_skip_benchmark": "false", 2024-09-14T04:38:13.054 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_memory_base": "805306368", 2024-09-14T04:38:13.054 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_memory_cache_min": "134217728", 2024-09-14T04:38:13.054 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_memory_cache_resize_interval": "1.000000", 2024-09-14T04:38:13.054 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_memory_expected_fragmentation": "0.150000", 2024-09-14T04:38:13.055 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_memory_target": "4294967296", 2024-09-14T04:38:13.055 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_memory_target_autotune": "false", 2024-09-14T04:38:13.055 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2024-09-14T04:38:13.055 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_min_pg_log_entries": "250", 2024-09-14T04:38:13.055 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_min_recovery_priority": "0", 2024-09-14T04:38:13.055 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_mon_heartbeat_interval": "30", 2024-09-14T04:38:13.055 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2024-09-14T04:38:13.055 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_mon_report_interval": "5", 2024-09-14T04:38:13.055 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_mon_report_max_in_flight": "2", 2024-09-14T04:38:13.055 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_mon_shutdown_timeout": "5.000000", 2024-09-14T04:38:13.055 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_num_cache_shards": "32", 2024-09-14T04:38:13.055 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_num_op_tracker_shard": "32", 2024-09-14T04:38:13.055 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_numa_auto_affinity": "true", 2024-09-14T04:38:13.055 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_numa_node": "-1", 2024-09-14T04:38:13.056 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_numa_prefer_iface": "true", 2024-09-14T04:38:13.056 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_object_clean_region_max_num_intervals": "10", 2024-09-14T04:38:13.056 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_objecter_finishers": "1", 2024-09-14T04:38:13.056 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_objectstore": "bluestore", 2024-09-14T04:38:13.056 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_objectstore_fuse": "false", 2024-09-14T04:38:13.056 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_objectstore_ideal_list_max": "64", 2024-09-14T04:38:13.056 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_objectstore_tracing": "false", 2024-09-14T04:38:13.056 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_op_complaint_time": "30.000000", 2024-09-14T04:38:13.056 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_op_history_duration": "600", 2024-09-14T04:38:13.056 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_op_history_size": "20", 2024-09-14T04:38:13.056 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_op_history_slow_op_size": "20", 2024-09-14T04:38:13.056 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2024-09-14T04:38:13.056 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_op_log_threshold": "5", 2024-09-14T04:38:13.057 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_op_num_shards": "0", 2024-09-14T04:38:13.057 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_op_num_shards_hdd": "5", 2024-09-14T04:38:13.057 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_op_num_shards_ssd": "8", 2024-09-14T04:38:13.057 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_op_num_threads_per_shard": "0", 2024-09-14T04:38:13.057 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2024-09-14T04:38:13.057 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2024-09-14T04:38:13.057 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2024-09-14T04:38:13.057 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_op_pq_min_cost": "65536", 2024-09-14T04:38:13.057 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_op_queue": "mclock_scheduler", 2024-09-14T04:38:13.057 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_op_queue_cut_off": "high", 2024-09-14T04:38:13.057 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_op_thread_suicide_timeout": "150", 2024-09-14T04:38:13.057 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_op_thread_timeout": "15", 2024-09-14T04:38:13.057 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_open_classes_on_start": "true", 2024-09-14T04:38:13.058 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_os_flags": "0", 2024-09-14T04:38:13.058 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_peering_op_priority": "255", 2024-09-14T04:38:13.058 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pg_delete_cost": "1048576", 2024-09-14T04:38:13.058 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pg_delete_priority": "5", 2024-09-14T04:38:13.058 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2024-09-14T04:38:13.058 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2024-09-14T04:38:13.058 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pg_log_dups_tracked": "3000", 2024-09-14T04:38:13.058 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pg_log_trim_max": "10000", 2024-09-14T04:38:13.058 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pg_log_trim_min": "100", 2024-09-14T04:38:13.058 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2024-09-14T04:38:13.058 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pg_object_context_cache_count": "64", 2024-09-14T04:38:13.058 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pg_stat_report_interval_max_epochs": "500", 2024-09-14T04:38:13.058 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pg_stat_report_interval_max_seconds": "5", 2024-09-14T04:38:13.059 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2024-09-14T04:38:13.059 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pool_default_cache_min_evict_age": "0", 2024-09-14T04:38:13.059 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pool_default_cache_min_flush_age": "0", 2024-09-14T04:38:13.059 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2024-09-14T04:38:13.059 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2024-09-14T04:38:13.059 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2024-09-14T04:38:13.059 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pool_default_crimson": "false", 2024-09-14T04:38:13.059 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pool_default_crush_rule": "-1", 2024-09-14T04:38:13.059 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pool_default_ec_fast_read": "false", 2024-09-14T04:38:13.059 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2024-09-14T04:38:13.059 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pool_default_flag_bulk": "false", 2024-09-14T04:38:13.059 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pool_default_flag_hashpspool": "true", 2024-09-14T04:38:13.059 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pool_default_flag_nodelete": "false", 2024-09-14T04:38:13.060 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pool_default_flag_nopgchange": "false", 2024-09-14T04:38:13.060 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pool_default_flag_nosizechange": "false", 2024-09-14T04:38:13.060 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pool_default_flags": "0", 2024-09-14T04:38:13.060 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2024-09-14T04:38:13.060 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pool_default_min_size": "0", 2024-09-14T04:38:13.060 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2024-09-14T04:38:13.060 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pool_default_pg_num": "32", 2024-09-14T04:38:13.060 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pool_default_pgp_num": "0", 2024-09-14T04:38:13.060 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2024-09-14T04:38:13.060 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pool_default_read_ratio": "70", 2024-09-14T04:38:13.060 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pool_default_size": "3", 2024-09-14T04:38:13.060 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pool_default_type": "replicated", 2024-09-14T04:38:13.060 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2024-09-14T04:38:13.061 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_pool_use_gmt_hitset": "true", 2024-09-14T04:38:13.061 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_push_per_object_cost": "1000", 2024-09-14T04:38:13.061 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_read_ec_check_for_errors": "false", 2024-09-14T04:38:13.061 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_recover_clone_overlap": "true", 2024-09-14T04:38:13.061 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_recover_clone_overlap_limit": "10", 2024-09-14T04:38:13.061 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_recovery_cost": "20971520", 2024-09-14T04:38:13.061 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_recovery_delay_start": "0.000000", 2024-09-14T04:38:13.061 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_recovery_max_active": "0", 2024-09-14T04:38:13.061 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_recovery_max_active_hdd": "3", 2024-09-14T04:38:13.061 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_recovery_max_active_ssd": "10", 2024-09-14T04:38:13.061 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_recovery_max_chunk": "8388608", 2024-09-14T04:38:13.061 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2024-09-14T04:38:13.061 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_recovery_max_single_start": "1", 2024-09-14T04:38:13.061 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_recovery_op_priority": "3", 2024-09-14T04:38:13.062 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_recovery_op_warn_multiple": "16", 2024-09-14T04:38:13.062 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_recovery_priority": "5", 2024-09-14T04:38:13.062 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_recovery_retry_interval": "30.000000", 2024-09-14T04:38:13.062 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_recovery_sleep": "0.000000", 2024-09-14T04:38:13.062 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_recovery_sleep_hdd": "0.100000", 2024-09-14T04:38:13.062 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2024-09-14T04:38:13.062 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_recovery_sleep_ssd": "0.000000", 2024-09-14T04:38:13.062 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_repair_during_recovery": "false", 2024-09-14T04:38:13.062 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_requested_scrub_priority": "120", 2024-09-14T04:38:13.062 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2024-09-14T04:38:13.062 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_rollback_to_cluster_snap": "", 2024-09-14T04:38:13.062 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_scrub_auto_repair": "false", 2024-09-14T04:38:13.062 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_scrub_auto_repair_num_errors": "5", 2024-09-14T04:38:13.063 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_scrub_backoff_ratio": "0.660000", 2024-09-14T04:38:13.063 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_scrub_begin_hour": "0", 2024-09-14T04:38:13.063 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_scrub_begin_week_day": "0", 2024-09-14T04:38:13.063 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_scrub_chunk_max": "25", 2024-09-14T04:38:13.063 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_scrub_chunk_min": "5", 2024-09-14T04:38:13.063 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_scrub_cost": "52428800", 2024-09-14T04:38:13.063 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_scrub_disable_reservation_queuing": "false", 2024-09-14T04:38:13.063 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_scrub_during_recovery": "false", 2024-09-14T04:38:13.063 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_scrub_end_hour": "0", 2024-09-14T04:38:13.063 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_scrub_end_week_day": "0", 2024-09-14T04:38:13.063 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_scrub_event_cost": "4096", 2024-09-14T04:38:13.063 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_scrub_extended_sleep": "0.000000", 2024-09-14T04:38:13.063 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2024-09-14T04:38:13.064 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_scrub_invalid_stats": "true", 2024-09-14T04:38:13.064 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_scrub_load_threshold": "0.500000", 2024-09-14T04:38:13.064 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_scrub_max_interval": "604800.000000", 2024-09-14T04:38:13.064 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_scrub_max_preemptions": "5", 2024-09-14T04:38:13.064 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_scrub_min_interval": "86400.000000", 2024-09-14T04:38:13.064 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_scrub_priority": "5", 2024-09-14T04:38:13.064 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_scrub_retry_after_noscrub": "60", 2024-09-14T04:38:13.064 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_scrub_retry_delay": "30", 2024-09-14T04:38:13.064 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_scrub_retry_new_interval": "10", 2024-09-14T04:38:13.064 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_scrub_retry_pg_state": "60", 2024-09-14T04:38:13.064 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_scrub_retry_trimming": "10", 2024-09-14T04:38:13.064 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_scrub_sleep": "0.000000", 2024-09-14T04:38:13.064 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_shallow_scrub_chunk_max": "100", 2024-09-14T04:38:13.065 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_shallow_scrub_chunk_min": "50", 2024-09-14T04:38:13.065 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_shutdown_pgref_assert": "false", 2024-09-14T04:38:13.065 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_skip_check_past_interval_bounds": "false", 2024-09-14T04:38:13.065 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_skip_data_digest": "false", 2024-09-14T04:38:13.065 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_smart_report_timeout": "5", 2024-09-14T04:38:13.065 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_snap_trim_cost": "1048576", 2024-09-14T04:38:13.065 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_snap_trim_priority": "5", 2024-09-14T04:38:13.065 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_snap_trim_sleep": "0.000000", 2024-09-14T04:38:13.065 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2024-09-14T04:38:13.065 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2024-09-14T04:38:13.065 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2024-09-14T04:38:13.065 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_stats_update_period_not_scrubbing": "120", 2024-09-14T04:38:13.065 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_stats_update_period_scrubbing": "15", 2024-09-14T04:38:13.066 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2024-09-14T04:38:13.066 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_target_transaction_size": "30", 2024-09-14T04:38:13.066 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_tier_default_cache_hit_set_count": "4", 2024-09-14T04:38:13.066 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2024-09-14T04:38:13.066 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2024-09-14T04:38:13.066 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2024-09-14T04:38:13.066 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2024-09-14T04:38:13.066 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2024-09-14T04:38:13.066 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2024-09-14T04:38:13.066 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_tier_default_cache_mode": "writeback", 2024-09-14T04:38:13.066 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2024-09-14T04:38:13.066 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_tier_promote_max_objects_sec": "25", 2024-09-14T04:38:13.066 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_tracing": "false", 2024-09-14T04:38:13.067 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_use_stale_snap": "false", 2024-09-14T04:38:13.067 INFO:teuthology.orchestra.run.smithi039.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2024-09-14T04:38:13.067 INFO:teuthology.orchestra.run.smithi039.stdout: "osdc_blkin_trace_all": "false", 2024-09-14T04:38:13.067 INFO:teuthology.orchestra.run.smithi039.stdout: "paxos_kill_at": "0", 2024-09-14T04:38:13.067 INFO:teuthology.orchestra.run.smithi039.stdout: "paxos_max_join_drift": "10", 2024-09-14T04:38:13.067 INFO:teuthology.orchestra.run.smithi039.stdout: "paxos_min": "500", 2024-09-14T04:38:13.067 INFO:teuthology.orchestra.run.smithi039.stdout: "paxos_min_wait": "0.050000", 2024-09-14T04:38:13.067 INFO:teuthology.orchestra.run.smithi039.stdout: "paxos_propose_interval": "1.000000", 2024-09-14T04:38:13.067 INFO:teuthology.orchestra.run.smithi039.stdout: "paxos_service_trim_max": "500", 2024-09-14T04:38:13.067 INFO:teuthology.orchestra.run.smithi039.stdout: "paxos_service_trim_max_multiplier": "20", 2024-09-14T04:38:13.067 INFO:teuthology.orchestra.run.smithi039.stdout: "paxos_service_trim_min": "250", 2024-09-14T04:38:13.067 INFO:teuthology.orchestra.run.smithi039.stdout: "paxos_stash_full_interval": "25", 2024-09-14T04:38:13.067 INFO:teuthology.orchestra.run.smithi039.stdout: "paxos_trim_max": "500", 2024-09-14T04:38:13.067 INFO:teuthology.orchestra.run.smithi039.stdout: "paxos_trim_min": "250", 2024-09-14T04:38:13.068 INFO:teuthology.orchestra.run.smithi039.stdout: "perf": "true", 2024-09-14T04:38:13.068 INFO:teuthology.orchestra.run.smithi039.stdout: "pid_file": "", 2024-09-14T04:38:13.068 INFO:teuthology.orchestra.run.smithi039.stdout: "plugin_crypto_accelerator": "crypto_isal", 2024-09-14T04:38:13.068 INFO:teuthology.orchestra.run.smithi039.stdout: "plugin_dir": "/usr/lib64/ceph", 2024-09-14T04:38:13.068 INFO:teuthology.orchestra.run.smithi039.stdout: "public_addr": "-", 2024-09-14T04:38:13.068 INFO:teuthology.orchestra.run.smithi039.stdout: "public_addrv": "", 2024-09-14T04:38:13.068 INFO:teuthology.orchestra.run.smithi039.stdout: "public_bind_addr": "-", 2024-09-14T04:38:13.068 INFO:teuthology.orchestra.run.smithi039.stdout: "public_network": "172.21.0.0/20", 2024-09-14T04:38:13.068 INFO:teuthology.orchestra.run.smithi039.stdout: "public_network_interface": "", 2024-09-14T04:38:13.068 INFO:teuthology.orchestra.run.smithi039.stdout: "qat_compressor_busy_polling": "false", 2024-09-14T04:38:13.068 INFO:teuthology.orchestra.run.smithi039.stdout: "qat_compressor_enabled": "false", 2024-09-14T04:38:13.068 INFO:teuthology.orchestra.run.smithi039.stdout: "qat_compressor_session_max_number": "256", 2024-09-14T04:38:13.068 INFO:teuthology.orchestra.run.smithi039.stdout: "rados_mon_op_timeout": "0", 2024-09-14T04:38:13.068 INFO:teuthology.orchestra.run.smithi039.stdout: "rados_osd_op_timeout": "0", 2024-09-14T04:38:13.069 INFO:teuthology.orchestra.run.smithi039.stdout: "rados_tracing": "false", 2024-09-14T04:38:13.069 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_atime_update_interval": "60", 2024-09-14T04:38:13.069 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2024-09-14T04:38:13.069 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_balance_parent_reads": "false", 2024-09-14T04:38:13.069 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_balance_snap_reads": "false", 2024-09-14T04:38:13.069 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_blkin_trace_all": "false", 2024-09-14T04:38:13.069 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_blocklist_expire_seconds": "0", 2024-09-14T04:38:13.069 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_blocklist_on_break_lock": "true", 2024-09-14T04:38:13.069 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_cache": "true", 2024-09-14T04:38:13.069 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_cache_block_writes_upfront": "false", 2024-09-14T04:38:13.069 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_cache_max_dirty": "25165824", 2024-09-14T04:38:13.069 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_cache_max_dirty_age": "1.000000", 2024-09-14T04:38:13.069 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_cache_max_dirty_object": "0", 2024-09-14T04:38:13.070 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_cache_policy": "writearound", 2024-09-14T04:38:13.070 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_cache_size": "33554432", 2024-09-14T04:38:13.070 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_cache_target_dirty": "16777216", 2024-09-14T04:38:13.070 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_cache_writethrough_until_flush": "true", 2024-09-14T04:38:13.070 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_clone_copy_on_read": "false", 2024-09-14T04:38:13.070 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_compression_hint": "none", 2024-09-14T04:38:13.070 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_concurrent_management_ops": "10", 2024-09-14T04:38:13.070 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_config_pool_override_update_timestamp": "1726287135", 2024-09-14T04:38:13.070 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_default_clone_format": "auto", 2024-09-14T04:38:13.070 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_default_data_pool": "", 2024-09-14T04:38:13.070 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_default_features": "61", 2024-09-14T04:38:13.070 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_default_format": "2", 2024-09-14T04:38:13.070 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_default_map_options": "", 2024-09-14T04:38:13.070 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_default_order": "22", 2024-09-14T04:38:13.071 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_default_pool": "rbd", 2024-09-14T04:38:13.071 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2024-09-14T04:38:13.071 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_default_stripe_count": "0", 2024-09-14T04:38:13.071 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_default_stripe_unit": "0", 2024-09-14T04:38:13.071 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_disable_zero_copy_writes": "true", 2024-09-14T04:38:13.071 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_discard_granularity_bytes": "65536", 2024-09-14T04:38:13.071 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_discard_on_zeroed_write_same": "true", 2024-09-14T04:38:13.071 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_enable_alloc_hint": "true", 2024-09-14T04:38:13.071 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2024-09-14T04:38:13.071 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_io_scheduler": "simple", 2024-09-14T04:38:13.071 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2024-09-14T04:38:13.071 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_journal_commit_age": "5.000000", 2024-09-14T04:38:13.071 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2024-09-14T04:38:13.071 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_journal_max_payload_bytes": "16384", 2024-09-14T04:38:13.072 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_journal_object_flush_age": "0.000000", 2024-09-14T04:38:13.072 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_journal_object_flush_bytes": "1048576", 2024-09-14T04:38:13.072 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_journal_object_flush_interval": "0", 2024-09-14T04:38:13.072 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2024-09-14T04:38:13.072 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2024-09-14T04:38:13.072 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_journal_order": "24", 2024-09-14T04:38:13.072 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_journal_pool": "", 2024-09-14T04:38:13.072 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_journal_splay_width": "4", 2024-09-14T04:38:13.072 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_localize_parent_reads": "false", 2024-09-14T04:38:13.072 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_localize_snap_reads": "false", 2024-09-14T04:38:13.072 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2024-09-14T04:38:13.072 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2024-09-14T04:38:13.072 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2024-09-14T04:38:13.073 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2024-09-14T04:38:13.073 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2024-09-14T04:38:13.073 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2024-09-14T04:38:13.073 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_mirror_image_policy_type": "simple", 2024-09-14T04:38:13.073 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2024-09-14T04:38:13.073 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_mirror_image_state_check_interval": "30", 2024-09-14T04:38:13.073 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2024-09-14T04:38:13.073 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2024-09-14T04:38:13.073 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2024-09-14T04:38:13.073 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2024-09-14T04:38:13.073 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2024-09-14T04:38:13.073 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_mirror_memory_autotune": "true", 2024-09-14T04:38:13.073 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_mirror_memory_base": "805306368", 2024-09-14T04:38:13.074 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2024-09-14T04:38:13.074 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_mirror_memory_cache_min": "134217728", 2024-09-14T04:38:13.074 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2024-09-14T04:38:13.074 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2024-09-14T04:38:13.074 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_mirror_memory_target": "4294967296", 2024-09-14T04:38:13.074 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_mirror_perf_stats_prio": "5", 2024-09-14T04:38:13.074 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2024-09-14T04:38:13.074 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2024-09-14T04:38:13.074 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_mirroring_delete_delay": "0", 2024-09-14T04:38:13.074 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2024-09-14T04:38:13.074 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_mirroring_replay_delay": "0", 2024-09-14T04:38:13.074 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2024-09-14T04:38:13.074 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2024-09-14T04:38:13.074 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_move_to_trash_on_remove": "false", 2024-09-14T04:38:13.075 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2024-09-14T04:38:13.075 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_mtime_update_interval": "60", 2024-09-14T04:38:13.075 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_non_blocking_aio": "true", 2024-09-14T04:38:13.075 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_op_thread_timeout": "60", 2024-09-14T04:38:13.075 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_op_threads": "1", 2024-09-14T04:38:13.075 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_parent_cache_enabled": "false", 2024-09-14T04:38:13.075 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_persistent_cache_mode": "disabled", 2024-09-14T04:38:13.075 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_persistent_cache_path": "/tmp", 2024-09-14T04:38:13.075 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_persistent_cache_size": "1073741824", 2024-09-14T04:38:13.075 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_plugins": "", 2024-09-14T04:38:13.075 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_qos_bps_burst": "0", 2024-09-14T04:38:13.075 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_qos_bps_burst_seconds": "1", 2024-09-14T04:38:13.075 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_qos_bps_limit": "0", 2024-09-14T04:38:13.075 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_qos_exclude_ops": "0", 2024-09-14T04:38:13.075 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_qos_iops_burst": "0", 2024-09-14T04:38:13.076 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_qos_iops_burst_seconds": "1", 2024-09-14T04:38:13.076 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_qos_iops_limit": "0", 2024-09-14T04:38:13.076 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_qos_read_bps_burst": "0", 2024-09-14T04:38:13.076 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2024-09-14T04:38:13.076 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_qos_read_bps_limit": "0", 2024-09-14T04:38:13.076 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_qos_read_iops_burst": "0", 2024-09-14T04:38:13.076 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2024-09-14T04:38:13.076 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_qos_read_iops_limit": "0", 2024-09-14T04:38:13.076 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_qos_schedule_tick_min": "50", 2024-09-14T04:38:13.076 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_qos_write_bps_burst": "0", 2024-09-14T04:38:13.076 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2024-09-14T04:38:13.076 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_qos_write_bps_limit": "0", 2024-09-14T04:38:13.076 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_qos_write_iops_burst": "0", 2024-09-14T04:38:13.076 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2024-09-14T04:38:13.077 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_qos_write_iops_limit": "0", 2024-09-14T04:38:13.077 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_quiesce_notification_attempts": "10", 2024-09-14T04:38:13.077 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_read_from_replica_policy": "default", 2024-09-14T04:38:13.077 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2024-09-14T04:38:13.077 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_readahead_max_bytes": "524288", 2024-09-14T04:38:13.077 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_readahead_trigger_requests": "10", 2024-09-14T04:38:13.077 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_request_timed_out_seconds": "30", 2024-09-14T04:38:13.077 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_skip_partial_discard": "true", 2024-09-14T04:38:13.077 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2024-09-14T04:38:13.077 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_tracing": "false", 2024-09-14T04:38:13.077 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_validate_names": "true", 2024-09-14T04:38:13.077 INFO:teuthology.orchestra.run.smithi039.stdout: "rbd_validate_pool": "true", 2024-09-14T04:38:13.077 INFO:teuthology.orchestra.run.smithi039.stdout: "restapi_base_url": "", 2024-09-14T04:38:13.078 INFO:teuthology.orchestra.run.smithi039.stdout: "restapi_log_level": "", 2024-09-14T04:38:13.078 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_account_default_quota_max_objects": "-1", 2024-09-14T04:38:13.078 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_account_default_quota_max_size": "-1", 2024-09-14T04:38:13.078 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_acl_grants_max_num": "100", 2024-09-14T04:38:13.078 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_admin_entry": "admin", 2024-09-14T04:38:13.078 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2024-09-14T04:38:13.078 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_asio_assert_yielding": "false", 2024-09-14T04:38:13.078 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_backend_store": "rados", 2024-09-14T04:38:13.078 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_barbican_url": "", 2024-09-14T04:38:13.078 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_beast_enable_async": "true", 2024-09-14T04:38:13.078 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_bucket_counters_cache": "false", 2024-09-14T04:38:13.078 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_bucket_counters_cache_size": "10000", 2024-09-14T04:38:13.078 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2024-09-14T04:38:13.078 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_bucket_default_quota_max_size": "-1", 2024-09-14T04:38:13.079 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_bucket_index_max_aio": "128", 2024-09-14T04:38:13.079 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2024-09-14T04:38:13.079 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_bucket_quota_cache_size": "10000", 2024-09-14T04:38:13.079 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_bucket_quota_ttl": "600", 2024-09-14T04:38:13.079 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_bucket_sync_spawn_window": "20", 2024-09-14T04:38:13.079 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_cache_enabled": "true", 2024-09-14T04:38:13.079 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_cache_expiry_interval": "900", 2024-09-14T04:38:13.079 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_cache_lru_size": "25000", 2024-09-14T04:38:13.079 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_config_store": "rados", 2024-09-14T04:38:13.079 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_content_length_compat": "false", 2024-09-14T04:38:13.079 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_copy_obj_progress": "true", 2024-09-14T04:38:13.079 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2024-09-14T04:38:13.080 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_cors_rules_max_num": "100", 2024-09-14T04:38:13.080 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_cross_domain_policy": "", 2024-09-14T04:38:13.080 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_default_encryption_key": "", 2024-09-14T04:38:13.080 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_kmip_addr": "", 2024-09-14T04:38:13.080 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_kmip_ca_path": "", 2024-09-14T04:38:13.080 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_kmip_client_cert": "", 2024-09-14T04:38:13.080 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_kmip_client_key": "", 2024-09-14T04:38:13.080 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_kmip_kms_key_template": "", 2024-09-14T04:38:13.080 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_kmip_password": "", 2024-09-14T04:38:13.080 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2024-09-14T04:38:13.080 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_kmip_username": "", 2024-09-14T04:38:13.080 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_require_ssl": "true", 2024-09-14T04:38:13.080 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2024-09-14T04:38:13.081 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2024-09-14T04:38:13.081 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_sse_s3_backend": "vault", 2024-09-14T04:38:13.081 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2024-09-14T04:38:13.081 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2024-09-14T04:38:13.081 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2024-09-14T04:38:13.081 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2024-09-14T04:38:13.081 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2024-09-14T04:38:13.081 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2024-09-14T04:38:13.081 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2024-09-14T04:38:13.081 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2024-09-14T04:38:13.081 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2024-09-14T04:38:13.081 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2024-09-14T04:38:13.081 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2024-09-14T04:38:13.082 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_suppress_logs": "true", 2024-09-14T04:38:13.082 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_vault_addr": "", 2024-09-14T04:38:13.082 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_vault_auth": "token", 2024-09-14T04:38:13.082 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_vault_namespace": "", 2024-09-14T04:38:13.082 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_vault_prefix": "", 2024-09-14T04:38:13.082 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_vault_secret_engine": "transit", 2024-09-14T04:38:13.082 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_vault_ssl_cacert": "", 2024-09-14T04:38:13.082 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2024-09-14T04:38:13.082 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2024-09-14T04:38:13.082 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_vault_token_file": "", 2024-09-14T04:38:13.082 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_crypt_vault_verify_ssl": "true", 2024-09-14T04:38:13.082 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_curl_buffersize": "524288", 2024-09-14T04:38:13.082 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_curl_low_speed_limit": "1024", 2024-09-14T04:38:13.082 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_curl_low_speed_time": "300", 2024-09-14T04:38:13.082 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_curl_tcp_keepalive": "0", 2024-09-14T04:38:13.083 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_curl_wait_timeout_ms": "1000", 2024-09-14T04:38:13.083 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2024-09-14T04:38:13.083 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2024-09-14T04:38:13.083 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2024-09-14T04:38:13.083 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2024-09-14T04:38:13.083 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_d3n_l1_fadvise": "4", 2024-09-14T04:38:13.083 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2024-09-14T04:38:13.083 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_d3n_libaio_aio_num": "64", 2024-09-14T04:38:13.083 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_d3n_libaio_aio_threads": "20", 2024-09-14T04:38:13.083 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_d4n_address": "127.0.0.1:6379", 2024-09-14T04:38:13.083 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_d4n_l1_datacache_address": "127.0.0.1:6379", 2024-09-14T04:38:13.083 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_d4n_l1_datacache_persistent_path": "/tmp/rgw_d4n_datacache/", 2024-09-14T04:38:13.083 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_d4n_l1_datacache_size": "1073741824", 2024-09-14T04:38:13.083 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_d4n_l1_evict_cache_on_start": "true", 2024-09-14T04:38:13.084 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_d4n_l1_fadvise": "4", 2024-09-14T04:38:13.084 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_d4n_libaio_aio_num": "64", 2024-09-14T04:38:13.084 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_d4n_libaio_aio_threads": "20", 2024-09-14T04:38:13.084 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2024-09-14T04:38:13.084 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_data_log_changes_size": "1000", 2024-09-14T04:38:13.084 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_data_log_num_shards": "128", 2024-09-14T04:38:13.084 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_data_log_obj_prefix": "data_log", 2024-09-14T04:38:13.084 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_data_log_window": "30", 2024-09-14T04:38:13.084 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_data_notify_interval_msec": "0", 2024-09-14T04:38:13.084 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_data_sync_poll_interval": "20", 2024-09-14T04:38:13.084 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_data_sync_spawn_window": "20", 2024-09-14T04:38:13.084 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_debug_inject_latency_bi_unlink": "0", 2024-09-14T04:38:13.084 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2024-09-14T04:38:13.084 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_debug_inject_set_olh_err": "0", 2024-09-14T04:38:13.085 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_default_data_log_backing": "fifo", 2024-09-14T04:38:13.085 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_default_realm_info_oid": "default.realm", 2024-09-14T04:38:13.085 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_default_region_info_oid": "default.region", 2024-09-14T04:38:13.085 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_default_zone_info_oid": "default.zone", 2024-09-14T04:38:13.085 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2024-09-14T04:38:13.085 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_defer_to_bucket_acls": "", 2024-09-14T04:38:13.085 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_delete_multi_obj_max_num": "1000", 2024-09-14T04:38:13.085 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_disable_s3select": "false", 2024-09-14T04:38:13.085 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_dmclock_admin_lim": "0.000000", 2024-09-14T04:38:13.085 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_dmclock_admin_res": "100.000000", 2024-09-14T04:38:13.085 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2024-09-14T04:38:13.085 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_dmclock_auth_lim": "0.000000", 2024-09-14T04:38:13.085 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_dmclock_auth_res": "200.000000", 2024-09-14T04:38:13.085 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2024-09-14T04:38:13.086 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_dmclock_data_lim": "0.000000", 2024-09-14T04:38:13.086 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_dmclock_data_res": "500.000000", 2024-09-14T04:38:13.086 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_dmclock_data_wgt": "500.000000", 2024-09-14T04:38:13.086 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2024-09-14T04:38:13.086 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_dmclock_metadata_res": "500.000000", 2024-09-14T04:38:13.086 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2024-09-14T04:38:13.086 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_dns_name": "", 2024-09-14T04:38:13.086 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_dns_s3website_name": "", 2024-09-14T04:38:13.086 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_dynamic_resharding": "true", 2024-09-14T04:38:13.086 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_dynamic_resharding_may_reduce": "true", 2024-09-14T04:38:13.086 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_dynamic_resharding_reduction_wait": "120", 2024-09-14T04:38:13.086 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2024-09-14T04:38:13.086 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_enable_gc_threads": "true", 2024-09-14T04:38:13.086 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_enable_lc_threads": "true", 2024-09-14T04:38:13.086 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_enable_mdsearch": "true", 2024-09-14T04:38:13.087 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_enable_ops_log": "false", 2024-09-14T04:38:13.087 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_enable_quota_threads": "true", 2024-09-14T04:38:13.087 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_enable_static_website": "false", 2024-09-14T04:38:13.087 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_enable_usage_log": "false", 2024-09-14T04:38:13.087 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_enforce_swift_acls": "true", 2024-09-14T04:38:13.087 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_exit_timeout_secs": "120", 2024-09-14T04:38:13.087 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_expose_bucket": "false", 2024-09-14T04:38:13.087 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_extended_http_attrs": "", 2024-09-14T04:38:13.087 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_filter": "none", 2024-09-14T04:38:13.087 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2024-09-14T04:38:13.087 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_frontends": "beast port=7480", 2024-09-14T04:38:13.087 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_gc_max_concurrent_io": "10", 2024-09-14T04:38:13.087 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_gc_max_deferred": "50", 2024-09-14T04:38:13.087 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2024-09-14T04:38:13.088 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_gc_max_objs": "32", 2024-09-14T04:38:13.088 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_gc_max_queue_size": "134213632", 2024-09-14T04:38:13.088 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_gc_max_trim_chunk": "16", 2024-09-14T04:38:13.088 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_gc_obj_min_wait": "7200", 2024-09-14T04:38:13.088 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_gc_processor_max_time": "3600", 2024-09-14T04:38:13.088 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_gc_processor_period": "3600", 2024-09-14T04:38:13.088 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_get_obj_max_req_size": "4194304", 2024-09-14T04:38:13.088 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_get_obj_window_size": "16777216", 2024-09-14T04:38:13.088 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_healthcheck_disabling_path": "", 2024-09-14T04:38:13.088 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_ignore_get_invalid_range": "false", 2024-09-14T04:38:13.088 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_init_timeout": "300", 2024-09-14T04:38:13.088 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2024-09-14T04:38:13.088 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2024-09-14T04:38:13.088 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_kafka_connection_idle": "300", 2024-09-14T04:38:13.089 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_kafka_message_timeout": "5000", 2024-09-14T04:38:13.089 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_kafka_sleep_timeout": "10", 2024-09-14T04:38:13.089 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_keystone_accepted_admin_roles": "", 2024-09-14T04:38:13.089 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_keystone_accepted_reader_roles": "", 2024-09-14T04:38:13.089 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2024-09-14T04:38:13.089 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_keystone_admin_domain": "", 2024-09-14T04:38:13.089 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_keystone_admin_password": "", 2024-09-14T04:38:13.089 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_keystone_admin_password_path": "", 2024-09-14T04:38:13.089 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_keystone_admin_project": "", 2024-09-14T04:38:13.089 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_keystone_admin_tenant": "", 2024-09-14T04:38:13.089 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_keystone_admin_token": "", 2024-09-14T04:38:13.089 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_keystone_admin_token_path": "", 2024-09-14T04:38:13.090 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_keystone_admin_user": "", 2024-09-14T04:38:13.090 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_keystone_barbican_domain": "", 2024-09-14T04:38:13.090 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_keystone_barbican_password": "", 2024-09-14T04:38:13.090 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_keystone_barbican_project": "", 2024-09-14T04:38:13.090 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_keystone_barbican_tenant": "", 2024-09-14T04:38:13.090 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_keystone_barbican_user": "", 2024-09-14T04:38:13.090 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2024-09-14T04:38:13.090 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_keystone_implicit_tenants": "false", 2024-09-14T04:38:13.090 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2024-09-14T04:38:13.090 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_keystone_service_token_enabled": "false", 2024-09-14T04:38:13.090 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_keystone_token_cache_size": "10000", 2024-09-14T04:38:13.090 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_keystone_url": "", 2024-09-14T04:38:13.090 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_keystone_verify_ssl": "true", 2024-09-14T04:38:13.090 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_lc_debug_interval": "-1", 2024-09-14T04:38:13.090 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_lc_lock_max_time": "90", 2024-09-14T04:38:13.091 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_lc_max_objs": "32", 2024-09-14T04:38:13.091 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_lc_max_rules": "1000", 2024-09-14T04:38:13.091 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_lc_max_worker": "3", 2024-09-14T04:38:13.091 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_lc_max_wp_worker": "3", 2024-09-14T04:38:13.091 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_lc_thread_delay": "0", 2024-09-14T04:38:13.091 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2024-09-14T04:38:13.091 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_ldap_dnattr": "uid", 2024-09-14T04:38:13.091 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2024-09-14T04:38:13.091 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_ldap_searchfilter": "", 2024-09-14T04:38:13.091 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2024-09-14T04:38:13.091 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_ldap_uri": "", 2024-09-14T04:38:13.091 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_lfuda_sync_frequency": "60", 2024-09-14T04:38:13.091 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2024-09-14T04:38:13.092 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_list_bucket_min_readahead": "1000", 2024-09-14T04:38:13.092 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_list_buckets_max_chunk": "1000", 2024-09-14T04:38:13.092 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_log_http_headers": "", 2024-09-14T04:38:13.092 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_log_nonexistent_bucket": "false", 2024-09-14T04:38:13.092 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2024-09-14T04:38:13.092 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_log_object_name_utc": "false", 2024-09-14T04:38:13.092 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_lua_max_memory_per_state": "128000", 2024-09-14T04:38:13.092 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_luarocks_location": "/tmp/rgw_luarocks/mon.a", 2024-09-14T04:38:13.092 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_max_attr_name_len": "0", 2024-09-14T04:38:13.092 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_max_attr_size": "0", 2024-09-14T04:38:13.092 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_max_attrs_num_in_req": "0", 2024-09-14T04:38:13.092 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_max_chunk_size": "4194304", 2024-09-14T04:38:13.093 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_max_concurrent_requests": "1024", 2024-09-14T04:38:13.093 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2024-09-14T04:38:13.093 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_max_dynamic_shards": "1999", 2024-09-14T04:38:13.093 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_max_listing_results": "5000", 2024-09-14T04:38:13.093 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_max_notify_retries": "10", 2024-09-14T04:38:13.093 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_max_objs_per_shard": "100000", 2024-09-14T04:38:13.093 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_max_put_param_size": "1048576", 2024-09-14T04:38:13.093 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_max_put_size": "5368709120", 2024-09-14T04:38:13.093 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_max_slo_entries": "1000", 2024-09-14T04:38:13.093 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_md_log_max_shards": "64", 2024-09-14T04:38:13.093 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_md_notify_interval_msec": "200", 2024-09-14T04:38:13.093 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_meta_sync_poll_interval": "20", 2024-09-14T04:38:13.093 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_meta_sync_spawn_window": "20", 2024-09-14T04:38:13.093 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_mime_types_file": "/etc/mime.types", 2024-09-14T04:38:13.094 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_mp_lock_max_time": "600", 2024-09-14T04:38:13.094 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_multi_obj_del_max_aio": "16", 2024-09-14T04:38:13.094 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_multipart_min_part_size": "5242880", 2024-09-14T04:38:13.094 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_multipart_part_upload_limit": "10000", 2024-09-14T04:38:13.094 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_nfs_fhcache_partitions": "3", 2024-09-14T04:38:13.094 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_nfs_fhcache_size": "2017", 2024-09-14T04:38:13.094 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_nfs_frontends": "rgw-nfs", 2024-09-14T04:38:13.094 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2024-09-14T04:38:13.094 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_nfs_lru_lanes": "5", 2024-09-14T04:38:13.094 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_nfs_max_gc": "300", 2024-09-14T04:38:13.094 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_nfs_namespace_expire_secs": "300", 2024-09-14T04:38:13.094 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_nfs_run_gc_threads": "false", 2024-09-14T04:38:13.094 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_nfs_run_lc_threads": "false", 2024-09-14T04:38:13.094 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_nfs_run_quota_threads": "false", 2024-09-14T04:38:13.095 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_nfs_run_sync_thread": "false", 2024-09-14T04:38:13.095 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_nfs_s3_fast_attrs": "false", 2024-09-14T04:38:13.095 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_nfs_write_completion_interval_s": "10", 2024-09-14T04:38:13.095 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_num_async_rados_threads": "32", 2024-09-14T04:38:13.095 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_num_control_oids": "8", 2024-09-14T04:38:13.095 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_numa_node": "-1", 2024-09-14T04:38:13.095 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_obj_stripe_size": "4194304", 2024-09-14T04:38:13.095 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_obj_tombstone_cache_size": "1000", 2024-09-14T04:38:13.095 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_objexp_chunk_size": "100", 2024-09-14T04:38:13.095 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_objexp_gc_interval": "600", 2024-09-14T04:38:13.095 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_objexp_hints_num_shards": "127", 2024-09-14T04:38:13.095 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_olh_pending_timeout_sec": "3600", 2024-09-14T04:38:13.095 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_op_thread_suicide_timeout": "0", 2024-09-14T04:38:13.095 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_op_thread_timeout": "600", 2024-09-14T04:38:13.096 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_op_tracing": "false", 2024-09-14T04:38:13.096 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_opa_token": "", 2024-09-14T04:38:13.096 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_opa_url": "", 2024-09-14T04:38:13.096 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_opa_verify_ssl": "true", 2024-09-14T04:38:13.096 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_ops_log_data_backlog": "5242880", 2024-09-14T04:38:13.096 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2024-09-14T04:38:13.096 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_ops_log_rados": "false", 2024-09-14T04:38:13.096 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_ops_log_socket_path": "", 2024-09-14T04:38:13.096 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_override_bucket_index_max_shards": "0", 2024-09-14T04:38:13.096 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2024-09-14T04:38:13.096 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2024-09-14T04:38:13.096 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_period_push_interval": "2.000000", 2024-09-14T04:38:13.096 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_period_push_interval_max": "30.000000", 2024-09-14T04:38:13.096 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_period_root_pool": ".rgw.root", 2024-09-14T04:38:13.097 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_policy_reject_invalid_principals": "true", 2024-09-14T04:38:13.097 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_posix_base_path": "/tmp/rgw_posix_driver", 2024-09-14T04:38:13.097 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_posix_cache_lanes": "3", 2024-09-14T04:38:13.097 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_posix_cache_lmdb_count": "3", 2024-09-14T04:38:13.097 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_posix_cache_max_buckets": "100", 2024-09-14T04:38:13.097 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_posix_cache_partitions": "3", 2024-09-14T04:38:13.097 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_posix_database_root": "/var/lib/ceph/radosgw", 2024-09-14T04:38:13.097 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_print_continue": "true", 2024-09-14T04:38:13.097 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_print_prohibited_content_length": "false", 2024-09-14T04:38:13.097 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_put_obj_max_window_size": "67108864", 2024-09-14T04:38:13.097 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_put_obj_min_window_size": "16777216", 2024-09-14T04:38:13.097 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2024-09-14T04:38:13.097 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_rados_pool_recovery_priority": "5", 2024-09-14T04:38:13.097 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_rados_tracing": "false", 2024-09-14T04:38:13.098 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_realm": "", 2024-09-14T04:38:13.098 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_realm_id": "", 2024-09-14T04:38:13.098 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_realm_root_pool": ".rgw.root", 2024-09-14T04:38:13.098 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_region": "", 2024-09-14T04:38:13.098 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_region_root_pool": ".rgw.root", 2024-09-14T04:38:13.098 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_relaxed_region_enforcement": "false", 2024-09-14T04:38:13.098 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_relaxed_s3_bucket_names": "false", 2024-09-14T04:38:13.098 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2024-09-14T04:38:13.098 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_request_uri": "", 2024-09-14T04:38:13.098 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_reshard_batch_size": "64", 2024-09-14T04:38:13.098 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_reshard_bucket_lock_duration": "360", 2024-09-14T04:38:13.098 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_reshard_debug_interval": "-1", 2024-09-14T04:38:13.098 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_reshard_max_aio": "128", 2024-09-14T04:38:13.098 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_reshard_num_logs": "16", 2024-09-14T04:38:13.099 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_reshard_progress_judge_interval": "120", 2024-09-14T04:38:13.099 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_reshard_progress_judge_ratio": "0.500000", 2024-09-14T04:38:13.099 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_reshard_thread_interval": "600", 2024-09-14T04:38:13.099 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_reshardlog_threshold": "30000", 2024-09-14T04:38:13.099 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_resolve_cname": "false", 2024-09-14T04:38:13.099 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_rest_getusage_op_compat": "false", 2024-09-14T04:38:13.099 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_run_sync_thread": "true", 2024-09-14T04:38:13.099 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_s3_auth_disable_signature_url": "false", 2024-09-14T04:38:13.099 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_s3_auth_order": "sts, external, local", 2024-09-14T04:38:13.099 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_s3_auth_use_keystone": "false", 2024-09-14T04:38:13.099 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_s3_auth_use_ldap": "false", 2024-09-14T04:38:13.099 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_s3_auth_use_rados": "true", 2024-09-14T04:38:13.099 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_s3_auth_use_sts": "false", 2024-09-14T04:38:13.100 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_s3_client_max_sig_ver": "-1", 2024-09-14T04:38:13.100 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_s3_success_create_obj_status": "0", 2024-09-14T04:38:13.100 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_safe_max_objects_per_shard": "102400", 2024-09-14T04:38:13.100 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_scheduler_type": "throttler", 2024-09-14T04:38:13.100 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_script_uri": "", 2024-09-14T04:38:13.100 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_service_provider_name": "", 2024-09-14T04:38:13.100 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_shard_warning_threshold": "90.000000", 2024-09-14T04:38:13.100 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_sts_client_id": "", 2024-09-14T04:38:13.100 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_sts_client_secret": "", 2024-09-14T04:38:13.100 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_sts_entry": "sts", 2024-09-14T04:38:13.100 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_sts_key": "sts", 2024-09-14T04:38:13.100 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_sts_max_session_duration": "43200", 2024-09-14T04:38:13.101 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_sts_min_session_duration": "900", 2024-09-14T04:38:13.101 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_sts_token_introspection_url": "", 2024-09-14T04:38:13.101 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_swift_account_in_url": "false", 2024-09-14T04:38:13.101 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_swift_auth_entry": "auth", 2024-09-14T04:38:13.101 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_swift_auth_url": "", 2024-09-14T04:38:13.101 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_swift_custom_header": "", 2024-09-14T04:38:13.101 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_swift_enforce_content_length": "false", 2024-09-14T04:38:13.101 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_swift_need_stats": "true", 2024-09-14T04:38:13.101 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_swift_tenant_name": "", 2024-09-14T04:38:13.101 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_swift_token_expiration": "86400", 2024-09-14T04:38:13.101 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_swift_url": "", 2024-09-14T04:38:13.101 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_swift_url_prefix": "swift", 2024-09-14T04:38:13.101 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_swift_versioning_enabled": "false", 2024-09-14T04:38:13.102 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2024-09-14T04:38:13.102 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2024-09-14T04:38:13.102 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_sync_lease_period": "120", 2024-09-14T04:38:13.102 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2024-09-14T04:38:13.102 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_sync_log_trim_interval": "1200", 2024-09-14T04:38:13.102 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_sync_log_trim_max_buckets": "16", 2024-09-14T04:38:13.102 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2024-09-14T04:38:13.102 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2024-09-14T04:38:13.102 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_sync_obj_etag_verify": "false", 2024-09-14T04:38:13.102 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_sync_trace_history_size": "4096", 2024-09-14T04:38:13.102 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_sync_trace_per_node_log_size": "32", 2024-09-14T04:38:13.102 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2024-09-14T04:38:13.102 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_thread_pool_size": "512", 2024-09-14T04:38:13.102 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_topic_persistency_max_retries": "0", 2024-09-14T04:38:13.103 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_topic_persistency_sleep_duration": "0", 2024-09-14T04:38:13.103 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_topic_persistency_time_to_live": "0", 2024-09-14T04:38:13.103 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_topic_require_publish_policy": "false", 2024-09-14T04:38:13.103 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_torrent_comment": "", 2024-09-14T04:38:13.103 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_torrent_createby": "", 2024-09-14T04:38:13.103 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_torrent_encoding": "", 2024-09-14T04:38:13.103 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_torrent_flag": "false", 2024-09-14T04:38:13.103 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_torrent_max_size": "5368709120", 2024-09-14T04:38:13.103 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_torrent_origin": "", 2024-09-14T04:38:13.103 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_torrent_sha_unit": "524288", 2024-09-14T04:38:13.103 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_torrent_tracker": "", 2024-09-14T04:38:13.103 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_trust_forwarded_https": "false", 2024-09-14T04:38:13.103 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_usage_log_flush_threshold": "1024", 2024-09-14T04:38:13.103 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_usage_log_tick_interval": "30", 2024-09-14T04:38:13.104 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_usage_max_shards": "32", 2024-09-14T04:38:13.104 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_usage_max_user_shards": "1", 2024-09-14T04:38:13.104 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_use_opa_authz": "false", 2024-09-14T04:38:13.104 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_user_counters_cache": "false", 2024-09-14T04:38:13.104 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_user_counters_cache_size": "10000", 2024-09-14T04:38:13.104 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_user_default_quota_max_objects": "-1", 2024-09-14T04:38:13.104 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_user_default_quota_max_size": "-1", 2024-09-14T04:38:13.104 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_user_max_buckets": "1000", 2024-09-14T04:38:13.104 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_user_policies_max_num": "100", 2024-09-14T04:38:13.104 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2024-09-14T04:38:13.104 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_user_quota_sync_idle_users": "false", 2024-09-14T04:38:13.104 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_user_quota_sync_interval": "86400", 2024-09-14T04:38:13.104 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_user_quota_sync_wait_time": "86400", 2024-09-14T04:38:13.105 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_user_unique_email": "true", 2024-09-14T04:38:13.105 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_verify_ssl": "true", 2024-09-14T04:38:13.105 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_website_routing_rules_max_num": "50", 2024-09-14T04:38:13.105 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_zone": "", 2024-09-14T04:38:13.105 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_zone_id": "", 2024-09-14T04:38:13.105 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_zone_root_pool": ".rgw.root", 2024-09-14T04:38:13.105 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_zonegroup": "", 2024-09-14T04:38:13.105 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_zonegroup_id": "", 2024-09-14T04:38:13.105 INFO:teuthology.orchestra.run.smithi039.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2024-09-14T04:38:13.105 INFO:teuthology.orchestra.run.smithi039.stdout: "rgwlc_auto_session_clear": "true", 2024-09-14T04:38:13.105 INFO:teuthology.orchestra.run.smithi039.stdout: "rgwlc_skip_bucket_step": "false", 2024-09-14T04:38:13.105 INFO:teuthology.orchestra.run.smithi039.stdout: "rocksdb_block_size": "4096", 2024-09-14T04:38:13.105 INFO:teuthology.orchestra.run.smithi039.stdout: "rocksdb_bloom_bits_per_key": "20", 2024-09-14T04:38:13.105 INFO:teuthology.orchestra.run.smithi039.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2024-09-14T04:38:13.106 INFO:teuthology.orchestra.run.smithi039.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2024-09-14T04:38:13.106 INFO:teuthology.orchestra.run.smithi039.stdout: "rocksdb_cache_row_ratio": "0.000000", 2024-09-14T04:38:13.106 INFO:teuthology.orchestra.run.smithi039.stdout: "rocksdb_cache_shard_bits": "4", 2024-09-14T04:38:13.106 INFO:teuthology.orchestra.run.smithi039.stdout: "rocksdb_cache_size": "536870912", 2024-09-14T04:38:13.106 INFO:teuthology.orchestra.run.smithi039.stdout: "rocksdb_cache_type": "binned_lru", 2024-09-14T04:38:13.106 INFO:teuthology.orchestra.run.smithi039.stdout: "rocksdb_cf_compact_on_deletion": "true", 2024-09-14T04:38:13.106 INFO:teuthology.orchestra.run.smithi039.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2024-09-14T04:38:13.107 INFO:teuthology.orchestra.run.smithi039.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2024-09-14T04:38:13.107 INFO:teuthology.orchestra.run.smithi039.stdout: "rocksdb_collect_compaction_stats": "false", 2024-09-14T04:38:13.108 INFO:teuthology.orchestra.run.smithi039.stdout: "rocksdb_collect_extended_stats": "false", 2024-09-14T04:38:13.108 INFO:teuthology.orchestra.run.smithi039.stdout: "rocksdb_collect_memory_stats": "false", 2024-09-14T04:38:13.108 INFO:teuthology.orchestra.run.smithi039.stdout: "rocksdb_delete_range_threshold": "1048576", 2024-09-14T04:38:13.108 INFO:teuthology.orchestra.run.smithi039.stdout: "rocksdb_index_type": "binary_search", 2024-09-14T04:38:13.108 INFO:teuthology.orchestra.run.smithi039.stdout: "rocksdb_log_to_ceph_log": "true", 2024-09-14T04:38:13.108 INFO:teuthology.orchestra.run.smithi039.stdout: "rocksdb_metadata_block_size": "4096", 2024-09-14T04:38:13.108 INFO:teuthology.orchestra.run.smithi039.stdout: "rocksdb_partition_filters": "false", 2024-09-14T04:38:13.108 INFO:teuthology.orchestra.run.smithi039.stdout: "rocksdb_perf": "false", 2024-09-14T04:38:13.108 INFO:teuthology.orchestra.run.smithi039.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2024-09-14T04:38:13.108 INFO:teuthology.orchestra.run.smithi039.stdout: "rotating_keys_bootstrap_timeout": "30", 2024-09-14T04:38:13.108 INFO:teuthology.orchestra.run.smithi039.stdout: "rotating_keys_renewal_timeout": "10", 2024-09-14T04:38:13.108 INFO:teuthology.orchestra.run.smithi039.stdout: "run_dir": "/var/run/ceph", 2024-09-14T04:38:13.108 INFO:teuthology.orchestra.run.smithi039.stdout: "seastore_block_create": "true", 2024-09-14T04:38:13.108 INFO:teuthology.orchestra.run.smithi039.stdout: "seastore_cache_lru_size": "67108864", 2024-09-14T04:38:13.109 INFO:teuthology.orchestra.run.smithi039.stdout: "seastore_cbjournal_size": "5368709120", 2024-09-14T04:38:13.109 INFO:teuthology.orchestra.run.smithi039.stdout: "seastore_data_delta_based_overwrite": "0", 2024-09-14T04:38:13.109 INFO:teuthology.orchestra.run.smithi039.stdout: "seastore_default_max_object_size": "16777216", 2024-09-14T04:38:13.109 INFO:teuthology.orchestra.run.smithi039.stdout: "seastore_default_object_metadata_reservation": "16777216", 2024-09-14T04:38:13.109 INFO:teuthology.orchestra.run.smithi039.stdout: "seastore_device_size": "53687091200", 2024-09-14T04:38:13.109 INFO:teuthology.orchestra.run.smithi039.stdout: "seastore_disable_end_to_end_data_protection": "true", 2024-09-14T04:38:13.109 INFO:teuthology.orchestra.run.smithi039.stdout: "seastore_full_integrity_check": "false", 2024-09-14T04:38:13.109 INFO:teuthology.orchestra.run.smithi039.stdout: "seastore_journal_batch_capacity": "16", 2024-09-14T04:38:13.109 INFO:teuthology.orchestra.run.smithi039.stdout: "seastore_journal_batch_flush_size": "16777216", 2024-09-14T04:38:13.109 INFO:teuthology.orchestra.run.smithi039.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2024-09-14T04:38:13.109 INFO:teuthology.orchestra.run.smithi039.stdout: "seastore_journal_iodepth_limit": "5", 2024-09-14T04:38:13.109 INFO:teuthology.orchestra.run.smithi039.stdout: "seastore_main_device_type": "SSD", 2024-09-14T04:38:13.109 INFO:teuthology.orchestra.run.smithi039.stdout: "seastore_max_concurrent_transactions": "8", 2024-09-14T04:38:13.109 INFO:teuthology.orchestra.run.smithi039.stdout: "seastore_max_data_allocation_size": "32768", 2024-09-14T04:38:13.110 INFO:teuthology.orchestra.run.smithi039.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2024-09-14T04:38:13.110 INFO:teuthology.orchestra.run.smithi039.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2024-09-14T04:38:13.110 INFO:teuthology.orchestra.run.smithi039.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2024-09-14T04:38:13.110 INFO:teuthology.orchestra.run.smithi039.stdout: "seastore_obj_data_write_amplification": "1.250000", 2024-09-14T04:38:13.110 INFO:teuthology.orchestra.run.smithi039.stdout: "seastore_segment_size": "67108864", 2024-09-14T04:38:13.110 INFO:teuthology.orchestra.run.smithi039.stdout: "set_keepcaps": "false", 2024-09-14T04:38:13.110 INFO:teuthology.orchestra.run.smithi039.stdout: "setgroup": "ceph", 2024-09-14T04:38:13.110 INFO:teuthology.orchestra.run.smithi039.stdout: "setuser": "ceph", 2024-09-14T04:38:13.110 INFO:teuthology.orchestra.run.smithi039.stdout: "setuser_match_path": "", 2024-09-14T04:38:13.110 INFO:teuthology.orchestra.run.smithi039.stdout: "target_max_misplaced_ratio": "0.050000", 2024-09-14T04:38:13.110 INFO:teuthology.orchestra.run.smithi039.stdout: "thp": "false", 2024-09-14T04:38:13.110 INFO:teuthology.orchestra.run.smithi039.stdout: "threadpool_default_timeout": "60", 2024-09-14T04:38:13.110 INFO:teuthology.orchestra.run.smithi039.stdout: "threadpool_empty_queue_max_wait": "2", 2024-09-14T04:38:13.110 INFO:teuthology.orchestra.run.smithi039.stdout: "throttler_perf_counter": "true", 2024-09-14T04:38:13.111 INFO:teuthology.orchestra.run.smithi039.stdout: "tmp_dir": "/tmp", 2024-09-14T04:38:13.111 INFO:teuthology.orchestra.run.smithi039.stdout: "tmp_file_template": "/tmp/ceph-mon.a.XXXXXX", 2024-09-14T04:38:13.111 INFO:teuthology.orchestra.run.smithi039.stdout: "uadk_compressor_enabled": "false", 2024-09-14T04:38:13.111 INFO:teuthology.orchestra.run.smithi039.stdout: "uadk_wd_sync_ctx_num": "2" 2024-09-14T04:38:13.111 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:38:13.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:13 smithi039 ceph-mon[106470]: pgmap v2140: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:13.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:13 smithi039 ceph-mon[111235]: pgmap v2140: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:13.328 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:13 smithi129 ceph-mon[101316]: pgmap v2140: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:14.144 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2024-09-14T04:38:14.596 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:38:15.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:15 smithi129 ceph-mon[101316]: pgmap v2141: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:15.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:15 smithi039 ceph-mon[106470]: pgmap v2141: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:15.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:15 smithi039 ceph-mon[111235]: pgmap v2141: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:16.339 INFO:teuthology.orchestra.run.smithi039.stdout:mon.a: {} 2024-09-14T04:38:16.339 INFO:teuthology.orchestra.run.smithi039.stderr:mon.a: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-14T04:38:16.348 INFO:teuthology.orchestra.run.smithi039.stdout:mon.b: {} 2024-09-14T04:38:16.348 INFO:teuthology.orchestra.run.smithi039.stderr:mon.b: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-14T04:38:16.358 INFO:teuthology.orchestra.run.smithi039.stdout:mon.c: {} 2024-09-14T04:38:16.358 INFO:teuthology.orchestra.run.smithi039.stderr:mon.c: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-14T04:38:16.680 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:16 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:38:16.681 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:16 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:38:16.681 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:16 smithi039 ceph-mon[106470]: pgmap v2142: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:16.681 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:16 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/41476237' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-14T04:38:16.681 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:16 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:38:16.681 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:16 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:38:16.681 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:16 smithi039 ceph-mon[111235]: pgmap v2142: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:16.682 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:16 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/41476237' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-14T04:38:16.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:16 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:38:16.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:16 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:38:16.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:16 smithi129 ceph-mon[101316]: pgmap v2142: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:16.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:16 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/41476237' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-14T04:38:16.988 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2024-09-14T04:38:16.988 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2024-09-14T04:38:16.989 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-14T04:38:16.991 INFO:tasks.radosbench:Beginning radosbench... 2024-09-14T04:38:16.991 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2024-09-14T04:38:16.992 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_0 16 2024-09-14T04:38:16.992 INFO:tasks.thrashosds.thrasher:starting do_thrash 2024-09-14T04:38:16.992 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-14T04:38:16.992 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:38:16.993 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2024-09-14T04:38:16.993 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:16.993 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2024-09-14T04:38:16.994 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:38:16.994 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2024-09-14T04:38:16.994 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-14T04:38:16.995 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2024-09-14T04:38:16.995 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:38:17.022 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:17.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:17.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:17.124 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:17.149 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:17.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:17.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:17.251 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:17.270 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 04:38:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=cleanup t=2024-09-14T04:38:16.890761205Z level=info msg="Completed cleanup jobs" duration=4.326298ms 2024-09-14T04:38:17.271 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 04:38:17 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=grafana.update.checker t=2024-09-14T04:38:17.034521552Z level=info msg="Update check succeeded" duration=62.828599ms 2024-09-14T04:38:17.271 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 04:38:17 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=plugins.update.checker t=2024-09-14T04:38:17.132703856Z level=info msg="Update check succeeded" duration=64.277954ms 2024-09-14T04:38:17.275 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:17.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:17.276 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:17.376 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:17.399 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:17.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:17.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:17.501 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:17.523 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:17.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:17.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:17.625 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:17.651 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:17.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:17.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:17.753 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:17.775 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:17.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:17.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:17.816 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:38:17.816 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:38:17.817 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:38:17.817 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:38:17.878 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:17.900 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:17.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:17.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:18.002 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:18.028 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:18.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:18.029 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:18.130 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:18.153 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:18.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:18.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:18.255 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:18.276 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:18.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:18.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:18.378 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:18.400 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:18.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:18.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:18.502 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:18.528 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:18.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:18.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:18.630 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:18.648 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:18 smithi039 ceph-mon[106470]: pgmap v2143: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:18.648 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:18 smithi039 ceph-mon[111235]: pgmap v2143: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:18.656 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:18.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:18.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:18.758 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:18.782 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:18.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:18.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:18.883 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:18.909 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:18.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:18.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:19.011 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:19.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:18 smithi129 ceph-mon[101316]: pgmap v2143: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:19.033 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:19.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:19.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:19.039 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:38:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:38:18] "GET /metrics HTTP/1.1" 200 34259 "" "Prometheus/2.51.0" 2024-09-14T04:38:19.135 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:19.161 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:19.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:19.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:19.262 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:19.288 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:19.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:19.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:19.390 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:19.413 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:19.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:19.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:19.514 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:19.540 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:19.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:19.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:19.642 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:19.665 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:19.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:19.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:19.766 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:19.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:19.793 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:19.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:19.894 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:19.916 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:19.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:19.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:20.018 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:20.044 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:20.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:20.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:20.146 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:20.168 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:20.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:20.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:20.270 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:20.292 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:20.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:20.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:20.394 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:20.420 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:20.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:20.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:20.522 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:20.548 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:20.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:20.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:20.650 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:20.675 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:20.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:20.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:20.777 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:20.799 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:20.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:20.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:20.902 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:20.928 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:20.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:20.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:20.980 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:20 smithi039 ceph-mon[106470]: pgmap v2144: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:20.981 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:20 smithi039 ceph-mon[111235]: pgmap v2144: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:21.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:20 smithi129 ceph-mon[101316]: pgmap v2144: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:21.030 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:21.052 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:21.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:21.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:21.154 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:21.176 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:21.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:21.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:21.278 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:21.299 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:21.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:21.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:21.401 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:21.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:21.424 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:21.424 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:21.525 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:21.550 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:21.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:21.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:21.652 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:21.674 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:21.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:21.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:21.776 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:21.802 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:21.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:21.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:21.903 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:21.927 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:21.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:21.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:22.029 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:22.050 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:22.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:22.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:22.152 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:22.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:22.175 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:22.175 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:22.276 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:22.302 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:22.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:22.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:22.404 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:22.429 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:22.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:22.430 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:22.531 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:22.554 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:22.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:22.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:22.656 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:22.678 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:22.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:22.679 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:22.780 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:22.802 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:22.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:22.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:22.904 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:22.929 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:22.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:22.930 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:22.962 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:22 smithi039 ceph-mon[111235]: pgmap v2145: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:22.962 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:38:22.962 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:22 smithi039 ceph-mon[106470]: pgmap v2145: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:22.962 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:38:23.031 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:23.055 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:23.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:23.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:23.157 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:23.181 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:23.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:23.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:23.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:22 smithi129 ceph-mon[101316]: pgmap v2145: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:23.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:38:23.283 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:23.306 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:23.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:23.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:23.408 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:23.430 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:23.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:23.432 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:23.532 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:23.558 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:23.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:23.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:23.660 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:23.685 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:23.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:23.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:23.787 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:23.811 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:23.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:23.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:23.883 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is set 2024-09-14T04:38:23.913 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:23.936 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:23.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:23.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:24.037 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:24.061 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:24.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:24.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:24.079 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:23 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2898916193' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:38:24.079 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:23 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:38:24.079 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:23 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2898916193' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:38:24.079 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:23 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:38:24.163 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:24.189 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:24.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:24.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:24.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:23 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2898916193' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:38:24.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:23 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:38:24.291 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:24.292 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:38:24.292 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [], 2024-09-14T04:38:24.292 INFO:teuthology.orchestra.run.smithi039.stdout: "num_ops": 0 2024-09-14T04:38:24.292 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:38:24.316 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:24.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:24.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:24.418 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:24.443 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:24.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:24.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:24.545 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:24.569 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:24.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:24.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:24.671 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:24.696 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:24.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:24.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:24.800 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:24.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:24.822 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:24.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:24.924 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:24.946 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:24.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:24.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:25.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:24 smithi039 ceph-mon[106470]: pgmap v2146: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:25.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:24 smithi039 ceph-mon[106470]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:38:25.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:24 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:38:25.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:24 smithi039 ceph-mon[106470]: osdmap e1314: 8 total, 8 up, 8 in 2024-09-14T04:38:25.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:24 smithi039 ceph-mon[111235]: pgmap v2146: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:25.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:24 smithi039 ceph-mon[111235]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:38:25.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:24 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:38:25.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:24 smithi039 ceph-mon[111235]: osdmap e1314: 8 total, 8 up, 8 in 2024-09-14T04:38:25.048 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:25.070 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:25.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:25.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:25.171 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:25.180 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-14T04:38:25.197 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:25.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:25.198 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:25.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:24 smithi129 ceph-mon[101316]: pgmap v2146: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:25.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:24 smithi129 ceph-mon[101316]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:38:25.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:24 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:38:25.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:24 smithi129 ceph-mon[101316]: osdmap e1314: 8 total, 8 up, 8 in 2024-09-14T04:38:25.299 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:25.322 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:25.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:25.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:25.424 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:25.450 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:25.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:25.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:25.551 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:25.574 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:25.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:25.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:25.675 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:25.698 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:25.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:25.699 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:25.799 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:25.821 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:25.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:25.822 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:25.890 INFO:teuthology.orchestra.run.smithi039.stderr:pool 'unique_pool_0' created 2024-09-14T04:38:25.923 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:25.947 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:25.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:25.948 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:26.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:25 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2116844664' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-09-14T04:38:26.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:25 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-09-14T04:38:26.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:25 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2116844664' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-09-14T04:38:26.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:25 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-09-14T04:38:26.049 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:26.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:26.071 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:26.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:26.172 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:26.196 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:26.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:26.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:26.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:25 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2116844664' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-09-14T04:38:26.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:25 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16} : dispatch 2024-09-14T04:38:26.298 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:26.320 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:26.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:26.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:26.422 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:26.447 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:26.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:26.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:26.549 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:26.572 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:26.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:26.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:26.674 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:26.700 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:26.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:26.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:26.801 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:26.827 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:26.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:26.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:26.929 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:26.954 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:26.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:26.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:26.967 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2024-09-14T04:38:27.056 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:27.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:27.081 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:27.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:27.182 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:27.213 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:26 smithi039 ceph-mon[106470]: pgmap v2148: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:27.213 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:26 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-09-14T04:38:27.214 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:26 smithi039 ceph-mon[106470]: osdmap e1315: 8 total, 8 up, 8 in 2024-09-14T04:38:27.214 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:26 smithi039 ceph-mon[111235]: pgmap v2148: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:27.214 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:26 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-09-14T04:38:27.214 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:26 smithi039 ceph-mon[111235]: osdmap e1315: 8 total, 8 up, 8 in 2024-09-14T04:38:27.232 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:27.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:27.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:27.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:26 smithi129 ceph-mon[101316]: pgmap v2148: 73 pgs: 73 active+clean; 582 KiB data, 813 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:27.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:26 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2024-09-14T04:38:27.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:26 smithi129 ceph-mon[101316]: osdmap e1315: 8 total, 8 up, 8 in 2024-09-14T04:38:27.334 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:27.359 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:27.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:27.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:27.460 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:38:27.460 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:38:27.460 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:27.471 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:38:27.471 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:38:27.483 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:27.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:27.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:27.489 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:38:27.489 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:38:27.505 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:38:27.506 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:38:27.522 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:38:27.522 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:38:27.541 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:38:27.542 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:38:27.558 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:38:27.559 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:38:27.575 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:38:27.575 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:38:27.585 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:27.608 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:27.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:27.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:27.710 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:27.731 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:27.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:27.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:27.833 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:27.859 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:27.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:27.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:27.961 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:27.982 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:27.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:27.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:28.084 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:28.106 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:28.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:28.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:28.123 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:38:28.171 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:27 smithi039 ceph-mon[106470]: osdmap e1316: 8 total, 8 up, 8 in 2024-09-14T04:38:28.171 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:27 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/364343934' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:38:28.171 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:27 smithi039 ceph-mon[111235]: osdmap e1316: 8 total, 8 up, 8 in 2024-09-14T04:38:28.171 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:27 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/364343934' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:38:28.208 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:28.234 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:28.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:28.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:28.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:27 smithi129 ceph-mon[101316]: osdmap e1316: 8 total, 8 up, 8 in 2024-09-14T04:38:28.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:27 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/364343934' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:38:28.336 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:28.367 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:28.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:28.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:28.469 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:28.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:28.492 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:28.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:28.581 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:38:28.583 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:38:28.586 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:38:28.594 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:28.621 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:28.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:28.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:28.724 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:28.749 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:28.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:28.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:28.851 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:28.874 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:28.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:28.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:28.915 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:38:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:38:28] "GET /metrics HTTP/1.1" 200 34258 "" "Prometheus/2.51.0" 2024-09-14T04:38:28.977 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:28.998 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:28.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:28.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:29.101 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:29.127 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:29.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:29.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:29.167 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:28 smithi039 ceph-mon[106470]: pgmap v2151: 89 pgs: 2 creating+peering, 12 unknown, 75 active+clean; 582 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:29.167 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:28 smithi039 ceph-mon[106470]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:38:29.168 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:28 smithi039 ceph-mon[106470]: osdmap e1317: 8 total, 8 up, 8 in 2024-09-14T04:38:29.168 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:28 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:38:29.168 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:28 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:38:29.168 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:28 smithi039 ceph-mon[111235]: pgmap v2151: 89 pgs: 2 creating+peering, 12 unknown, 75 active+clean; 582 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:29.168 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:28 smithi039 ceph-mon[111235]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:38:29.168 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:28 smithi039 ceph-mon[111235]: osdmap e1317: 8 total, 8 up, 8 in 2024-09-14T04:38:29.168 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:28 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:38:29.168 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:28 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:38:29.172 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:28 smithi129 ceph-mon[101316]: pgmap v2151: 89 pgs: 2 creating+peering, 12 unknown, 75 active+clean; 582 KiB data, 812 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:29.172 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:28 smithi129 ceph-mon[101316]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:38:29.172 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:28 smithi129 ceph-mon[101316]: osdmap e1317: 8 total, 8 up, 8 in 2024-09-14T04:38:29.172 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:28 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:38:29.172 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:28 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:38:29.229 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:29.256 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:29.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:29.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:29.358 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:29.380 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:29.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:29.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:29.481 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:29.503 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:29.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:29.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:29.606 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:29.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:29.630 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:29.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:29.731 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:29.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:29.754 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:29.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:29.855 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:29.878 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:29.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:29.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:29.980 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:30.004 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:30.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:30.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:30.106 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:30.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:30.129 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:30.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:30.230 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:30.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:30.253 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:30.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:30.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:30 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:38:30.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:30 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:38:30.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:30 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:38:30.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:30 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:38:30.306 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:38:30.354 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:30.375 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:30.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:30.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:30.466 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:30 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:38:30.467 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:30 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:38:30.478 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:30.499 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:30.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:30.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:30.601 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:30.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:30.625 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:30.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:30.726 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:30.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:30.750 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:30.750 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:30.851 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:30.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:30.873 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:30.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:30.974 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:30.996 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:30.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:30.997 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:31.097 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:31.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:31.122 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:31.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:31.223 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:31.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:31.248 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:31.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:31.349 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:31.372 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:31.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:31.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:31.471 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:31 smithi039 ceph-mon[106470]: pgmap v2153: 89 pgs: 2 creating+peering, 87 active+clean; 582 KiB data, 809 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:31.471 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:31 smithi039 ceph-mon[111235]: pgmap v2153: 89 pgs: 2 creating+peering, 87 active+clean; 582 KiB data, 809 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:31.473 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:31.497 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:31.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:31.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:31.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:31 smithi129 ceph-mon[101316]: pgmap v2153: 89 pgs: 2 creating+peering, 87 active+clean; 582 KiB data, 809 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:31.599 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:31.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:31.623 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:31.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:31.723 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:31.749 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:31.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:31.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:31.851 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:31.874 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:31.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:31.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:31.976 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:31.998 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:31.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:31.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:32.100 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:32.121 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:32.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:32.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:32.224 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:32.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:32.249 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:32.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:32.350 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:32.361 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:38:32.361 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:38:32.374 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:32.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:32.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:32.478 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:32.503 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:32.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:32.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:32.605 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:32.631 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:32.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:32.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:32.733 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:32.759 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:32.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:32.760 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:32.861 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:32.883 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:32.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:32.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:32.986 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:33.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:33.013 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:33.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:33.114 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:33.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:33.140 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:33.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:33.239 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-14T04:38:33.241 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:33.245 INFO:teuthology.orchestra.run.smithi039.stderr:enabled application 'rados' on pool 'unique_pool_0' 2024-09-14T04:38:33.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:33.267 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:33.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:33.368 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:33.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:33.392 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:33.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:33.493 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:33.519 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:33.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:33.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:33.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:33 smithi129 ceph-mon[101316]: pgmap v2154: 89 pgs: 89 active+clean; 582 KiB data, 808 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:33.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:33 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1641205224' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:38:33.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:33 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:38:33.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:33 smithi039 ceph-mon[106470]: pgmap v2154: 89 pgs: 89 active+clean; 582 KiB data, 808 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:33.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:33 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1641205224' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:38:33.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:33 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:38:33.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:33 smithi039 ceph-mon[111235]: pgmap v2154: 89 pgs: 89 active+clean; 582 KiB data, 808 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:33.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:33 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1641205224' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:38:33.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:33 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:38:33.621 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:33.647 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:33.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:33.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:33.749 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:33.775 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:33.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:33.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:33.876 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:33.901 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:33.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:33.902 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:34.003 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:34.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:34.026 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:34.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:34.127 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:34.150 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:34.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:34.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:34.252 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:34.277 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:34 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:38:34.277 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:34 smithi039 ceph-mon[106470]: osdmap e1318: 8 total, 8 up, 8 in 2024-09-14T04:38:34.278 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:34 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:38:34.278 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:34 smithi039 ceph-mon[111235]: osdmap e1318: 8 total, 8 up, 8 in 2024-09-14T04:38:34.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:34.279 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:34.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:34.380 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:34.405 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:34.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:34.406 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:34.507 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:34.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:34 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:38:34.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:34 smithi129 ceph-mon[101316]: osdmap e1318: 8 total, 8 up, 8 in 2024-09-14T04:38:34.530 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:34.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:34.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:34.631 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:34.656 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:34.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:34.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:34.758 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:34.782 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:34.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:34.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:34.883 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:34.906 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:34.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:34.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:35.008 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:35.030 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:35.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:35.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:35.132 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:35.157 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:35.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:35.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:35.174 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:38:35.259 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:35.282 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is set 2024-09-14T04:38:35.285 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:35.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:35.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:35.387 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:35.411 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:35.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:35.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:35.453 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:35 smithi039 ceph-mon[106470]: pgmap v2156: 89 pgs: 89 active+clean; 582 KiB data, 811 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:35.453 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:35 smithi039 ceph-mon[106470]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:38:35.453 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:35 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2023200658' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:38:35.453 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:35 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:38:35.453 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:35 smithi039 ceph-mon[111235]: pgmap v2156: 89 pgs: 89 active+clean; 582 KiB data, 811 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:35.453 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:35 smithi039 ceph-mon[111235]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:38:35.453 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:35 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2023200658' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:38:35.453 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:35 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:38:35.513 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:35.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:35 smithi129 ceph-mon[101316]: pgmap v2156: 89 pgs: 89 active+clean; 582 KiB data, 811 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:35.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:35 smithi129 ceph-mon[101316]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:38:35.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:35 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2023200658' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:38:35.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:35 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:38:35.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:35.537 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:35.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:35.633 DEBUG:teuthology.orchestra.run.smithi039:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2024-09-14T04:38:35.634 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_0 2024-09-14T04:38:35.634 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:38:35.636 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-14T04:38:35.636 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-14T04:38:35.637 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:35.663 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:35.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:35.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:35.687 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:hints = 1 2024-09-14T04:38:35.688 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-14T04:38:35.688 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Object prefix: benchmark_data_smithi039_224949 2024-09-14T04:38:35.688 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:38:35.688 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 0 0 0 0 0 0 - 0 2024-09-14T04:38:35.766 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:35.802 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:35.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:35.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:35.904 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:35.942 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:35.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:35.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:36.044 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:36.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:36.077 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:36.078 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:36.178 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:36.209 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:36.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:36.210 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:36.311 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:36.344 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:36.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:36.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:36.446 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:36.477 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:36.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:36.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:36.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:36 smithi039 ceph-mon[106470]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:38:36.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:36 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:38:36.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:36 smithi039 ceph-mon[106470]: osdmap e1319: 8 total, 8 up, 8 in 2024-09-14T04:38:36.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:36 smithi039 ceph-mon[111235]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:38:36.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:36 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:38:36.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:36 smithi039 ceph-mon[111235]: osdmap e1319: 8 total, 8 up, 8 in 2024-09-14T04:38:36.579 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:36.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:36.612 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:36.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:36.688 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 1 16 3905 3889 243.048 243.062 0.00206703 0.00410039 2024-09-14T04:38:36.713 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:36.754 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:36.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:36.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:36.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:36 smithi129 ceph-mon[101316]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:38:36.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:36 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:38:36.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:36 smithi129 ceph-mon[101316]: osdmap e1319: 8 total, 8 up, 8 in 2024-09-14T04:38:36.856 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:36.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:36.891 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:36.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:36.972 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:38:36.992 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:37.020 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:37.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:37.021 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:37.122 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:37.156 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:37.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:37.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:37.258 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:37.290 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:37.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:37.291 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:37.392 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:37.425 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:37.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:37.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:37.527 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:37.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:37 smithi039 ceph-mon[106470]: pgmap v2158: 89 pgs: 89 active+clean; 582 KiB data, 811 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:37.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:38:37.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:37 smithi039 ceph-mon[111235]: pgmap v2158: 89 pgs: 89 active+clean; 582 KiB data, 811 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:37.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:38:37.558 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:37.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:37.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:37.659 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:37.688 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 2 16 7540 7524 235.107 227.188 0.00462257 0.00424411 2024-09-14T04:38:37.690 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:37.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:37.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:37.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:37 smithi129 ceph-mon[101316]: pgmap v2158: 89 pgs: 89 active+clean; 582 KiB data, 811 MiB used, 714 GiB / 715 GiB avail 2024-09-14T04:38:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:38:37.793 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:37.832 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:37.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:37.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:37.934 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:37.974 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:37.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:37.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:38.075 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:38.107 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:38.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:38.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:38.209 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:38.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:38.253 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:38.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:38.354 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:38.384 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:38.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:38.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:38.486 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:38.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:38.514 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:38.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:38.604 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:38 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:38:38.604 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:38 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:38:38.614 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:38.664 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:38.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:38.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:38.688 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 3 16 10693 10677 222.419 197.062 0.00309114 0.00448882 2024-09-14T04:38:38.766 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:38.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:38 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:38:38.796 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:38.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:38.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:38.898 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:38.903 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:38:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:38:38] "GET /metrics HTTP/1.1" 200 36670 "" "Prometheus/2.51.0" 2024-09-14T04:38:38.929 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:38.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:38.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:39.016 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:38:39.032 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:39.058 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:39.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:39.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:39.160 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:39.191 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:39.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:39.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:39.293 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:39.325 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:39.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:39.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:39.428 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:39.455 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:39.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:39.457 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:39.558 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:39.577 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:39 smithi039 ceph-mon[106470]: pgmap v2159: 89 pgs: 89 active+clean; 72 MiB data, 999 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s wr, 141 op/s 2024-09-14T04:38:39.578 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:39 smithi039 ceph-mon[111235]: pgmap v2159: 89 pgs: 89 active+clean; 72 MiB data, 999 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s wr, 141 op/s 2024-09-14T04:38:39.591 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:39.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:39.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:39.688 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 4 16 13428 13412 209.545 170.938 0.00538849 0.00476412 2024-09-14T04:38:39.693 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:39.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:39.731 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:39.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:39.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:39 smithi129 ceph-mon[101316]: pgmap v2159: 89 pgs: 89 active+clean; 72 MiB data, 999 MiB used, 714 GiB / 715 GiB avail; 8.9 MiB/s wr, 141 op/s 2024-09-14T04:38:39.832 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:39.861 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:39.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:39.862 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:39.963 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:40.001 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:40.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:40.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:40.103 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:40.143 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:40.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:40.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:40.245 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:40.283 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:40.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:40.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:40.385 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:40.420 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:40.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:40.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:40.522 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:40.563 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:40.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:40.565 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:40.665 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:40.667 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:38:40.690 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 5 16 16067 16051 200.555 164.938 0.00746419 0.004979 2024-09-14T04:38:40.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:40.725 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:40.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:40.826 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:40.858 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:40.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:40.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:40.961 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:41.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:41.008 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:41.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:41.109 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:41.153 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:41.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:41.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:41.243 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:38:41.243 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:38:41.256 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:41.281 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:38:41.281 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:38:41.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:41.310 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:41.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:41.321 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:38:41.321 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:38:41.361 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:38:41.362 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:38:41.378 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:38:41.378 INFO:teuthology.orchestra.run.smithi039.stdout: "size": 20, 2024-09-14T04:38:41.378 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 600, 2024-09-14T04:38:41.378 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [ 2024-09-14T04:38:41.378 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.378 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.34896.0:3281 51.c e1123/1016 51:32cdd098:::rbd_data.88502b0ee4d1.0000000000000033:head v 1123'31, mlcod=1120'29)", 2024-09-14T04:38:41.378 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:32:08.653195+0000", 2024-09-14T04:38:41.378 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 392.66976807600003, 2024-09-14T04:38:41.378 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.041767403000000002, 2024-09-14T04:38:41.378 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:38:41.379 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:38:41.379 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:38:41.379 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:38:41.379 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.379 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:38:41.379 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:08.653195+0000", 2024-09-14T04:38:41.379 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.379 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.379 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.379 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:38:41.379 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:08.653195+0000", 2024-09-14T04:38:41.379 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.379 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.380 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.380 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:38:41.380 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:08.653196+0000", 2024-09-14T04:38:41.380 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3e-06 2024-09-14T04:38:41.380 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.380 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.380 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:38:41.380 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:08.665746+0000", 2024-09-14T04:38:41.380 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.012550357 2024-09-14T04:38:41.380 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.380 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.381 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:38:41.381 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:08.665767+0000", 2024-09-14T04:38:41.381 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1123000000000001e-05 2024-09-14T04:38:41.381 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.381 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.381 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:38:41.381 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:08.665777+0000", 2024-09-14T04:38:41.381 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.6779999999999995e-06 2024-09-14T04:38:41.381 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.381 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.381 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:38:41.382 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:08.665854+0000", 2024-09-14T04:38:41.382 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6859000000000004e-05 2024-09-14T04:38:41.382 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.382 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.382 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:38:41.382 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:08.665881+0000", 2024-09-14T04:38:41.382 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.7300999999999998e-05 2024-09-14T04:38:41.382 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.382 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.382 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:38:41.382 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:08.694906+0000", 2024-09-14T04:38:41.383 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.029025201 2024-09-14T04:38:41.383 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.383 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.383 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:38:41.383 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:08.694962+0000", 2024-09-14T04:38:41.383 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5584000000000002e-05 2024-09-14T04:38:41.383 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.383 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:38:41.383 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.383 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.383 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.384 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.34896.0:3480 51.11 e1127/1016 51:8eb940a2:::rbd_data.88502b0ee4d1.000000000000004c:head v 1127'29, mlcod=1126'26)", 2024-09-14T04:38:41.384 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:32:12.690774+0000", 2024-09-14T04:38:41.384 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 388.63218880699998, 2024-09-14T04:38:41.384 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.051671366000000003, 2024-09-14T04:38:41.384 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:38:41.384 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:38:41.384 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:38:41.384 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:38:41.384 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.384 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:38:41.384 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:12.690774+0000", 2024-09-14T04:38:41.384 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.384 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.385 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.385 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:38:41.385 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:12.690774+0000", 2024-09-14T04:38:41.385 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.385 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.385 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.385 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:38:41.385 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:12.690775+0000", 2024-09-14T04:38:41.385 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.406e-06 2024-09-14T04:38:41.385 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.385 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.385 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:38:41.386 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:12.698559+0000", 2024-09-14T04:38:41.386 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0077842959999999996 2024-09-14T04:38:41.386 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.386 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.386 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:38:41.386 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:12.698590+0000", 2024-09-14T04:38:41.386 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.0349999999999999e-05 2024-09-14T04:38:41.386 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.386 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.387 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:38:41.387 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:12.698601+0000", 2024-09-14T04:38:41.387 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1446e-05 2024-09-14T04:38:41.387 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.387 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.387 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:38:41.387 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:12.698699+0000", 2024-09-14T04:38:41.388 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.8202999999999995e-05 2024-09-14T04:38:41.388 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.388 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.388 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:38:41.388 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:12.698727+0000", 2024-09-14T04:38:41.388 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.7086000000000001e-05 2024-09-14T04:38:41.388 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.388 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.388 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:38:41.388 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:12.742402+0000", 2024-09-14T04:38:41.388 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.043675650000000003 2024-09-14T04:38:41.388 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.388 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.388 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:38:41.389 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:12.742445+0000", 2024-09-14T04:38:41.389 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2929000000000002e-05 2024-09-14T04:38:41.389 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.389 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:38:41.389 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.389 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.389 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.389 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.34896.0:3581 51.1e 51:7e1cf65a:::rbd_data.88502b0ee4d1.0000000000000043:head [write 0~3093868] snapc 70=[70,6f,6e,6d,6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1129)", 2024-09-14T04:38:41.389 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:32:14.723319+0000", 2024-09-14T04:38:41.389 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 386.59964328699999, 2024-09-14T04:38:41.389 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.042505301000000002, 2024-09-14T04:38:41.389 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:38:41.389 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:38:41.390 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:38:41.390 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:38:41.390 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.34896", 2024-09-14T04:38:41.390 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1436074065", 2024-09-14T04:38:41.390 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 3581 2024-09-14T04:38:41.390 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.390 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:38:41.390 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.390 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:38:41.390 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:14.723319+0000", 2024-09-14T04:38:41.390 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.390 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.390 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.390 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:38:41.391 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:14.723319+0000", 2024-09-14T04:38:41.391 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.391 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.391 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.391 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:38:41.391 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:14.723321+0000", 2024-09-14T04:38:41.391 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7799999999999999e-06 2024-09-14T04:38:41.391 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.391 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.391 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:38:41.391 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:14.725669+0000", 2024-09-14T04:38:41.391 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0023482030000000001 2024-09-14T04:38:41.391 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.391 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.391 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:38:41.392 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:14.725693+0000", 2024-09-14T04:38:41.392 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4014000000000001e-05 2024-09-14T04:38:41.392 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.392 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.392 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:38:41.392 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:14.725710+0000", 2024-09-14T04:38:41.392 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6639000000000001e-05 2024-09-14T04:38:41.392 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.392 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.392 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:38:41.392 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:14.725793+0000", 2024-09-14T04:38:41.392 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.2754000000000001e-05 2024-09-14T04:38:41.392 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.393 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.393 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:38:41.393 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:14.725915+0000", 2024-09-14T04:38:41.393 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012202200000000001 2024-09-14T04:38:41.393 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.393 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.393 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,7", 2024-09-14T04:38:41.393 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:14.726135+0000", 2024-09-14T04:38:41.393 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00022040699999999999 2024-09-14T04:38:41.393 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.393 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.393 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:38:41.393 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:14.733055+0000", 2024-09-14T04:38:41.393 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0069195619999999998 2024-09-14T04:38:41.394 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.394 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.394 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.394 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:14.734978+0000", 2024-09-14T04:38:41.394 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001923807 2024-09-14T04:38:41.394 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.394 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.394 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.394 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:14.765715+0000", 2024-09-14T04:38:41.394 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.030736891999999998 2024-09-14T04:38:41.395 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.395 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.395 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:38:41.395 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:14.765771+0000", 2024-09-14T04:38:41.395 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5612000000000002e-05 2024-09-14T04:38:41.395 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.395 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.395 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:38:41.395 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:14.765825+0000", 2024-09-14T04:38:41.395 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3609000000000001e-05 2024-09-14T04:38:41.395 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.395 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:38:41.395 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.395 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.396 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.396 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.34896.0:3849 51.1e 51:7d8e7839:::rbd_data.88502b0ee4d1.0000000000000022:head [write 323674~3850920] snapc 75=[75,74,73,72,71,70,6f,6e,6d,6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1134)", 2024-09-14T04:38:41.396 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:32:19.769141+0000", 2024-09-14T04:38:41.396 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 381.55382111300003, 2024-09-14T04:38:41.396 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.041786140999999999, 2024-09-14T04:38:41.396 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:38:41.396 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:38:41.396 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:38:41.396 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:38:41.396 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.34896", 2024-09-14T04:38:41.396 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1436074065", 2024-09-14T04:38:41.396 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 3849 2024-09-14T04:38:41.396 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.396 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:38:41.397 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.397 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:38:41.397 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:19.769141+0000", 2024-09-14T04:38:41.397 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.397 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.397 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.397 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:38:41.397 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:19.769141+0000", 2024-09-14T04:38:41.397 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.397 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.397 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.397 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:38:41.397 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:19.769143+0000", 2024-09-14T04:38:41.398 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2449999999999999e-06 2024-09-14T04:38:41.398 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.398 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.398 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:38:41.398 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:19.773054+0000", 2024-09-14T04:38:41.398 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0039109230000000002 2024-09-14T04:38:41.398 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.398 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.398 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:38:41.398 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:19.773069+0000", 2024-09-14T04:38:41.398 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5658000000000002e-05 2024-09-14T04:38:41.398 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.398 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.398 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:38:41.399 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:19.773080+0000", 2024-09-14T04:38:41.399 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0519999999999999e-05 2024-09-14T04:38:41.399 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.399 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.399 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:38:41.399 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:19.773152+0000", 2024-09-14T04:38:41.399 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1789e-05 2024-09-14T04:38:41.399 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.399 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.399 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:38:41.399 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:19.773264+0000", 2024-09-14T04:38:41.399 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000112419 2024-09-14T04:38:41.399 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.399 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.400 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,7", 2024-09-14T04:38:41.400 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:19.773903+0000", 2024-09-14T04:38:41.400 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00063878700000000004 2024-09-14T04:38:41.400 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.400 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.400 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:38:41.400 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:19.785981+0000", 2024-09-14T04:38:41.400 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.012078554 2024-09-14T04:38:41.400 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.400 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.400 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.400 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:19.790040+0000", 2024-09-14T04:38:41.400 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.004058163 2024-09-14T04:38:41.400 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.401 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.401 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.401 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:19.810768+0000", 2024-09-14T04:38:41.401 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.020728599 2024-09-14T04:38:41.401 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.404 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.404 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:38:41.404 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:19.810868+0000", 2024-09-14T04:38:41.404 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000100302 2024-09-14T04:38:41.404 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.404 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.405 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:38:41.405 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:19.810928+0000", 2024-09-14T04:38:41.405 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.9181999999999997e-05 2024-09-14T04:38:41.405 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.405 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:38:41.405 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.405 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.405 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.405 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.34896.0:3967 51.1a 51:5a60ec32:::rbd_data.88502b0ee4d1.0000000000000025:head [write 1445048~2749256] snapc 77=[77,76,75,74,73,72,71,70,6f,6e,6d,6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1136)", 2024-09-14T04:38:41.405 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:32:21.798348+0000", 2024-09-14T04:38:41.405 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 379.52461478200001, 2024-09-14T04:38:41.405 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.042104330000000002, 2024-09-14T04:38:41.405 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:38:41.405 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:38:41.406 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:38:41.406 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:38:41.406 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.34896", 2024-09-14T04:38:41.406 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1436074065", 2024-09-14T04:38:41.406 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 3967 2024-09-14T04:38:41.406 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.406 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:38:41.406 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.406 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:38:41.406 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:21.798348+0000", 2024-09-14T04:38:41.406 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.406 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.406 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.406 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:38:41.407 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:21.798348+0000", 2024-09-14T04:38:41.407 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.407 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.407 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.407 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:38:41.407 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:21.798353+0000", 2024-09-14T04:38:41.407 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1880000000000001e-06 2024-09-14T04:38:41.407 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.407 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.407 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:38:41.407 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:21.799646+0000", 2024-09-14T04:38:41.407 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001293169 2024-09-14T04:38:41.407 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.407 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.408 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:38:41.408 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:21.799680+0000", 2024-09-14T04:38:41.408 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.4069999999999997e-05 2024-09-14T04:38:41.408 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.408 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.408 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:38:41.408 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:21.799698+0000", 2024-09-14T04:38:41.408 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7631000000000001e-05 2024-09-14T04:38:41.408 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.408 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.408 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:38:41.408 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:21.799854+0000", 2024-09-14T04:38:41.408 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015622499999999999 2024-09-14T04:38:41.408 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.409 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.409 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:38:41.409 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:21.800027+0000", 2024-09-14T04:38:41.409 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00017256899999999999 2024-09-14T04:38:41.409 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.409 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.409 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:38:41.409 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:21.800194+0000", 2024-09-14T04:38:41.409 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000167677 2024-09-14T04:38:41.409 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.409 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.409 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:38:41.409 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:21.813041+0000", 2024-09-14T04:38:41.409 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.012847127999999999 2024-09-14T04:38:41.410 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.410 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.410 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.410 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:21.825614+0000", 2024-09-14T04:38:41.410 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.012572982 2024-09-14T04:38:41.410 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.410 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.410 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.410 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:21.840327+0000", 2024-09-14T04:38:41.410 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.014712744999999999 2024-09-14T04:38:41.410 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.410 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.410 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:38:41.411 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:21.840398+0000", 2024-09-14T04:38:41.411 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1007999999999995e-05 2024-09-14T04:38:41.411 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.411 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.411 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:38:41.411 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:21.840452+0000", 2024-09-14T04:38:41.411 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3937999999999998e-05 2024-09-14T04:38:41.411 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.411 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:38:41.411 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.411 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.411 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.411 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.34896.0:41369 51.b 51:d05de766:::rbd_data.88501c1a06ae.0000000000000003:head [set-alloc-hint object_size 4194304 write_size 4194304,writefull 0~4194304] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1162)", 2024-09-14T04:38:41.412 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:32:49.072044+0000", 2024-09-14T04:38:41.412 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 352.250918861, 2024-09-14T04:38:41.412 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.049712114000000002, 2024-09-14T04:38:41.412 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:38:41.412 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:38:41.412 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:38:41.412 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:38:41.412 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.34896", 2024-09-14T04:38:41.412 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1436074065", 2024-09-14T04:38:41.412 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 41369 2024-09-14T04:38:41.412 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.412 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:38:41.412 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.412 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:38:41.413 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:49.072044+0000", 2024-09-14T04:38:41.413 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.413 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.413 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.413 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:38:41.413 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:49.072044+0000", 2024-09-14T04:38:41.413 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.413 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.413 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.413 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:38:41.413 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:49.072045+0000", 2024-09-14T04:38:41.413 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.674e-06 2024-09-14T04:38:41.413 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.413 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.414 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:38:41.414 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:49.073326+0000", 2024-09-14T04:38:41.414 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001280092 2024-09-14T04:38:41.414 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.414 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.414 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:38:41.414 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:49.073360+0000", 2024-09-14T04:38:41.414 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.4113999999999999e-05 2024-09-14T04:38:41.414 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.414 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.414 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:38:41.414 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:49.073376+0000", 2024-09-14T04:38:41.415 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5988999999999999e-05 2024-09-14T04:38:41.415 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.415 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.415 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:38:41.415 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:49.073493+0000", 2024-09-14T04:38:41.415 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000117209 2024-09-14T04:38:41.415 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.415 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.415 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:38:41.415 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:49.073642+0000", 2024-09-14T04:38:41.415 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000149538 2024-09-14T04:38:41.415 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.415 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.415 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 4,6", 2024-09-14T04:38:41.416 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:49.073772+0000", 2024-09-14T04:38:41.416 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012926600000000001 2024-09-14T04:38:41.416 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.416 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.416 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:38:41.416 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:49.084491+0000", 2024-09-14T04:38:41.416 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.010719227 2024-09-14T04:38:41.416 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.416 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.416 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.416 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:49.121638+0000", 2024-09-14T04:38:41.416 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.037147322000000003 2024-09-14T04:38:41.416 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.416 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.417 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.417 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:49.121673+0000", 2024-09-14T04:38:41.417 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.4956000000000002e-05 2024-09-14T04:38:41.417 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.417 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.417 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:38:41.417 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:49.121719+0000", 2024-09-14T04:38:41.417 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6060000000000003e-05 2024-09-14T04:38:41.417 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.418 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.418 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:38:41.418 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:32:49.121756+0000", 2024-09-14T04:38:41.418 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.6667e-05 2024-09-14T04:38:41.418 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.418 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:38:41.418 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.418 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.418 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.419 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.44840.0:1100 52.18 52:1b7a50f0:::rbd_data.af28843a9204.0000000000000000:head [set-alloc-hint object_size 4194304 write_size 4194304,writefull 0~4194304] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1229)", 2024-09-14T04:38:41.419 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:34:38.251964+0000", 2024-09-14T04:38:41.419 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 243.07099829699999, 2024-09-14T04:38:41.419 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.030655323000000002, 2024-09-14T04:38:41.419 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:38:41.419 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:38:41.419 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:38:41.419 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:38:41.419 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.44840", 2024-09-14T04:38:41.420 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1436701460", 2024-09-14T04:38:41.420 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 1100 2024-09-14T04:38:41.420 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.420 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:38:41.420 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.420 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:38:41.420 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:34:38.251964+0000", 2024-09-14T04:38:41.420 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.420 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.420 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.420 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:38:41.420 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:34:38.251964+0000", 2024-09-14T04:38:41.420 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.420 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.421 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.421 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:38:41.421 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:34:38.251966+0000", 2024-09-14T04:38:41.421 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0439999999999998e-06 2024-09-14T04:38:41.421 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.421 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.421 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:38:41.421 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:34:38.255038+0000", 2024-09-14T04:38:41.421 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0030721540000000001 2024-09-14T04:38:41.421 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.421 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.421 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:38:41.421 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:34:38.255079+0000", 2024-09-14T04:38:41.422 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.0339999999999997e-05 2024-09-14T04:38:41.422 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.422 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.422 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:38:41.422 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:34:38.255095+0000", 2024-09-14T04:38:41.422 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5845000000000002e-05 2024-09-14T04:38:41.422 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.422 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.422 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:38:41.422 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:34:38.255207+0000", 2024-09-14T04:38:41.422 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000111907 2024-09-14T04:38:41.422 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.422 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.422 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:38:41.422 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:34:38.255416+0000", 2024-09-14T04:38:41.423 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00020953000000000001 2024-09-14T04:38:41.423 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.423 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.423 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:38:41.423 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:34:38.255551+0000", 2024-09-14T04:38:41.423 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013456500000000001 2024-09-14T04:38:41.423 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.423 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.423 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:38:41.423 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:34:38.263167+0000", 2024-09-14T04:38:41.423 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0076160209999999997 2024-09-14T04:38:41.423 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.423 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.423 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.424 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:34:38.282495+0000", 2024-09-14T04:38:41.424 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.019327895000000001 2024-09-14T04:38:41.424 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.424 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.424 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.424 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:34:38.282528+0000", 2024-09-14T04:38:41.424 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.3475000000000001e-05 2024-09-14T04:38:41.424 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.424 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.424 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:38:41.424 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:34:38.282566+0000", 2024-09-14T04:38:41.424 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.8381999999999999e-05 2024-09-14T04:38:41.424 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.424 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.425 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:38:41.425 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:34:38.282620+0000", 2024-09-14T04:38:41.425 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3164999999999997e-05 2024-09-14T04:38:41.425 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.425 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:38:41.426 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.426 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.426 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.426 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.34914.0:1417 52.8 52:1089d5b9:::rbd_data.886237e36aca.0000000000000003:head [set-alloc-hint object_size 4194304 write_size 4194304,writefull 0~4194304] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1250)", 2024-09-14T04:38:41.426 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:35:47.871178+0000", 2024-09-14T04:38:41.426 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 173.451784995, 2024-09-14T04:38:41.426 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.044813933, 2024-09-14T04:38:41.426 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:38:41.427 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:38:41.427 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:38:41.427 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:38:41.427 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.34914", 2024-09-14T04:38:41.427 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1048804342", 2024-09-14T04:38:41.427 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 1417 2024-09-14T04:38:41.427 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.428 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:38:41.428 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.428 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:38:41.428 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:35:47.871178+0000", 2024-09-14T04:38:41.429 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.430 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.430 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.430 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:38:41.430 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:35:47.871178+0000", 2024-09-14T04:38:41.430 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.430 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.430 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.430 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:38:41.430 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:35:47.871181+0000", 2024-09-14T04:38:41.430 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.4999999999999999e-06 2024-09-14T04:38:41.431 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.431 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.431 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:38:41.431 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:35:47.874485+0000", 2024-09-14T04:38:41.431 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0033039440000000001 2024-09-14T04:38:41.431 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.432 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.432 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:38:41.432 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:35:47.874520+0000", 2024-09-14T04:38:41.432 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.4471000000000003e-05 2024-09-14T04:38:41.432 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.432 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.433 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:38:41.433 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:35:47.874535+0000", 2024-09-14T04:38:41.433 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5743000000000002e-05 2024-09-14T04:38:41.433 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.433 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.433 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:38:41.433 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:35:47.874652+0000", 2024-09-14T04:38:41.434 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000116322 2024-09-14T04:38:41.434 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.434 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.434 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:38:41.434 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:35:47.874861+0000", 2024-09-14T04:38:41.434 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000209827 2024-09-14T04:38:41.434 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.435 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.435 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:38:41.435 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:35:47.875068+0000", 2024-09-14T04:38:41.435 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00020611800000000001 2024-09-14T04:38:41.435 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.435 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.436 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:38:41.436 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:35:47.892021+0000", 2024-09-14T04:38:41.436 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.016953333000000001 2024-09-14T04:38:41.436 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.436 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.436 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.436 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:35:47.914956+0000", 2024-09-14T04:38:41.437 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.022934768000000001 2024-09-14T04:38:41.437 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.437 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.437 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.437 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:35:47.915861+0000", 2024-09-14T04:38:41.437 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00090518300000000005 2024-09-14T04:38:41.438 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.438 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.438 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:38:41.438 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:35:47.915955+0000", 2024-09-14T04:38:41.438 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.4652000000000001e-05 2024-09-14T04:38:41.438 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.438 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.438 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:38:41.438 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:35:47.915992+0000", 2024-09-14T04:38:41.438 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.6072000000000003e-05 2024-09-14T04:38:41.439 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.439 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:38:41.439 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.439 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.439 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.439 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.34914.0:2370 52.1d 52:bd61e55e:::rbd_data.8862fb33ee1a.0000000000000033:head [set-alloc-hint object_size 4194304 write_size 4194304,write 962926~3231378] snapc 6a=[6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1272)", 2024-09-14T04:38:41.439 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:36:22.254673+0000", 2024-09-14T04:38:41.439 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 139.068289338, 2024-09-14T04:38:41.439 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.045074972999999997, 2024-09-14T04:38:41.439 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:38:41.439 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:38:41.439 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:38:41.439 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:38:41.440 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.34914", 2024-09-14T04:38:41.440 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1048804342", 2024-09-14T04:38:41.440 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 2370 2024-09-14T04:38:41.440 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.440 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:38:41.440 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.440 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:38:41.440 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:22.254673+0000", 2024-09-14T04:38:41.440 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.440 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.440 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.440 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:38:41.440 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:22.254673+0000", 2024-09-14T04:38:41.441 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.441 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.441 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.441 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:38:41.441 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:22.254675+0000", 2024-09-14T04:38:41.441 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8870000000000001e-06 2024-09-14T04:38:41.441 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.441 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.441 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:38:41.441 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:22.257075+0000", 2024-09-14T04:38:41.441 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.002399591 2024-09-14T04:38:41.441 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.441 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.442 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:38:41.442 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:22.257109+0000", 2024-09-14T04:38:41.442 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.4236999999999998e-05 2024-09-14T04:38:41.442 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.442 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.442 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:38:41.442 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:22.257123+0000", 2024-09-14T04:38:41.442 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4099e-05 2024-09-14T04:38:41.442 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.442 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.442 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:38:41.442 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:22.257203+0000", 2024-09-14T04:38:41.442 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9876000000000006e-05 2024-09-14T04:38:41.443 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.443 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.443 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:38:41.443 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:22.257426+0000", 2024-09-14T04:38:41.443 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000222981 2024-09-14T04:38:41.443 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.443 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.443 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:38:41.443 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:22.257576+0000", 2024-09-14T04:38:41.443 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014989499999999999 2024-09-14T04:38:41.443 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.443 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.443 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:38:41.444 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:22.266018+0000", 2024-09-14T04:38:41.444 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0084421530000000009 2024-09-14T04:38:41.444 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.444 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.444 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.444 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:22.297144+0000", 2024-09-14T04:38:41.444 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.031126339999999999 2024-09-14T04:38:41.444 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.444 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.444 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.444 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:22.299631+0000", 2024-09-14T04:38:41.444 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0024865149999999999 2024-09-14T04:38:41.444 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.445 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.445 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:38:41.445 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:22.299698+0000", 2024-09-14T04:38:41.445 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.6678000000000001e-05 2024-09-14T04:38:41.445 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.445 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.445 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:38:41.445 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:22.299748+0000", 2024-09-14T04:38:41.445 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.0720999999999998e-05 2024-09-14T04:38:41.445 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.445 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:38:41.445 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.445 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.445 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.446 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.34914.0:2417 52.18 52:1bdea9ef:::rbd_data.8862fb33ee1a.000000000000004c:head [set-alloc-hint object_size 4194304 write_size 4194304,write 0~1231238] snapc 6b=[6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1273)", 2024-09-14T04:38:41.446 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:36:23.261257+0000", 2024-09-14T04:38:41.446 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 138.06170555099999, 2024-09-14T04:38:41.446 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.027911515000000001, 2024-09-14T04:38:41.446 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:38:41.446 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:38:41.446 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:38:41.446 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:38:41.446 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.34914", 2024-09-14T04:38:41.446 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1048804342", 2024-09-14T04:38:41.446 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 2417 2024-09-14T04:38:41.446 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.447 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:38:41.447 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.447 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:38:41.447 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:23.261257+0000", 2024-09-14T04:38:41.447 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.447 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.447 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.447 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:38:41.447 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:23.261257+0000", 2024-09-14T04:38:41.447 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.447 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.447 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.448 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:38:41.448 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:23.261263+0000", 2024-09-14T04:38:41.448 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.8309999999999997e-06 2024-09-14T04:38:41.448 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.448 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.448 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:38:41.448 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:23.262573+0000", 2024-09-14T04:38:41.448 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0013102490000000001 2024-09-14T04:38:41.448 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.448 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.448 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:38:41.448 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:23.262623+0000", 2024-09-14T04:38:41.448 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9477000000000001e-05 2024-09-14T04:38:41.449 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.449 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.449 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:38:41.449 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:23.262640+0000", 2024-09-14T04:38:41.449 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6946999999999999e-05 2024-09-14T04:38:41.449 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.449 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.449 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:38:41.449 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:23.262775+0000", 2024-09-14T04:38:41.450 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013564800000000001 2024-09-14T04:38:41.450 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.450 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.450 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op must wait for map", 2024-09-14T04:38:41.450 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:23.262785+0000", 2024-09-14T04:38:41.451 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0212e-05 2024-09-14T04:38:41.451 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.451 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.451 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:38:41.451 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:23.272746+0000", 2024-09-14T04:38:41.451 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0099609989999999999 2024-09-14T04:38:41.452 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.452 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.452 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:38:41.452 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:23.273002+0000", 2024-09-14T04:38:41.452 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00025526500000000002 2024-09-14T04:38:41.452 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.452 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.453 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:38:41.453 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:23.273248+0000", 2024-09-14T04:38:41.453 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00024680399999999999 2024-09-14T04:38:41.453 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.453 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.453 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:38:41.453 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:23.284246+0000", 2024-09-14T04:38:41.453 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.010997552000000001 2024-09-14T04:38:41.453 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.453 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.454 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.454 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:23.288718+0000", 2024-09-14T04:38:41.454 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0044719510000000001 2024-09-14T04:38:41.454 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.454 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.454 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.454 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:23.289035+0000", 2024-09-14T04:38:41.454 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00031751400000000002 2024-09-14T04:38:41.454 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.454 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.454 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:38:41.454 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:23.289106+0000", 2024-09-14T04:38:41.454 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.0031999999999996e-05 2024-09-14T04:38:41.455 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.455 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.455 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:38:41.455 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:23.289169+0000", 2024-09-14T04:38:41.455 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3034000000000007e-05 2024-09-14T04:38:41.455 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.455 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:38:41.455 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.455 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.455 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.455 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.34914.0:2440 52.1d 52:b980f0d4:::rbd_data.8862fb33ee1a.000000000000003d:head [write 0~3043778] snapc 6b=[6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1273)", 2024-09-14T04:38:41.455 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:36:23.306159+0000", 2024-09-14T04:38:41.455 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 138.01680361300001, 2024-09-14T04:38:41.456 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.032503329999999997, 2024-09-14T04:38:41.456 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:38:41.456 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:38:41.456 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:38:41.456 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:38:41.456 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.34914", 2024-09-14T04:38:41.456 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1048804342", 2024-09-14T04:38:41.456 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 2440 2024-09-14T04:38:41.456 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.456 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:38:41.456 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.456 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:38:41.456 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:23.306159+0000", 2024-09-14T04:38:41.457 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.457 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.457 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.457 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:38:41.457 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:23.306159+0000", 2024-09-14T04:38:41.457 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.457 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.457 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.457 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:38:41.457 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:23.306161+0000", 2024-09-14T04:38:41.457 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.829e-06 2024-09-14T04:38:41.457 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.458 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.458 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:38:41.458 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:23.308065+0000", 2024-09-14T04:38:41.458 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0019038340000000001 2024-09-14T04:38:41.458 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.458 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.458 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:38:41.458 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:23.308088+0000", 2024-09-14T04:38:41.458 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3445999999999999e-05 2024-09-14T04:38:41.458 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.458 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.458 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:38:41.458 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:23.308096+0000", 2024-09-14T04:38:41.458 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6699999999999994e-06 2024-09-14T04:38:41.459 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.459 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.459 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:38:41.459 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:23.308172+0000", 2024-09-14T04:38:41.459 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.5753999999999993e-05 2024-09-14T04:38:41.459 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.459 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.459 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:38:41.459 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:23.308308+0000", 2024-09-14T04:38:41.459 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013631 2024-09-14T04:38:41.459 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.459 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.459 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:38:41.459 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:23.308546+0000", 2024-09-14T04:38:41.460 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000238421 2024-09-14T04:38:41.460 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.460 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.460 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:38:41.460 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:23.318132+0000", 2024-09-14T04:38:41.460 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0095857830000000005 2024-09-14T04:38:41.460 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.460 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.461 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.461 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:23.338509+0000", 2024-09-14T04:38:41.461 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.020377072999999999 2024-09-14T04:38:41.461 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.461 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.461 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.461 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:23.338556+0000", 2024-09-14T04:38:41.461 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7005000000000001e-05 2024-09-14T04:38:41.461 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.461 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.462 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:38:41.462 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:23.338612+0000", 2024-09-14T04:38:41.462 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6227999999999998e-05 2024-09-14T04:38:41.462 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.462 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.462 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:38:41.462 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:23.338662+0000", 2024-09-14T04:38:41.462 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9976999999999999e-05 2024-09-14T04:38:41.462 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.462 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:38:41.462 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.462 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.462 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.462 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.34914.0:2584 52.1d 52:bc815abd:::rbd_data.8862fb33ee1a.0000000000000052:head [write 1593698~2600606] snapc 6e=[6e,6d,6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1276)", 2024-09-14T04:38:41.462 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:36:26.288944+0000", 2024-09-14T04:38:41.463 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 135.03401889899999, 2024-09-14T04:38:41.463 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.035350255999999997, 2024-09-14T04:38:41.463 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:38:41.463 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:38:41.463 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:38:41.463 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:38:41.463 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.34914", 2024-09-14T04:38:41.463 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1048804342", 2024-09-14T04:38:41.463 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 2584 2024-09-14T04:38:41.464 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.464 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:38:41.464 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.464 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:38:41.464 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:26.288944+0000", 2024-09-14T04:38:41.464 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.464 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.465 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.465 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:38:41.465 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:26.288944+0000", 2024-09-14T04:38:41.465 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.465 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.465 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.465 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:38:41.465 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:26.288946+0000", 2024-09-14T04:38:41.465 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4569999999999999e-06 2024-09-14T04:38:41.465 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.465 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.465 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:38:41.466 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:26.290963+0000", 2024-09-14T04:38:41.466 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0020165000000000001 2024-09-14T04:38:41.466 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.466 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.466 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:38:41.466 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:26.290987+0000", 2024-09-14T04:38:41.466 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4728000000000001e-05 2024-09-14T04:38:41.466 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.466 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.466 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:38:41.466 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:26.290999+0000", 2024-09-14T04:38:41.467 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1287000000000001e-05 2024-09-14T04:38:41.467 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.467 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.467 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:38:41.467 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:26.291116+0000", 2024-09-14T04:38:41.467 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000116929 2024-09-14T04:38:41.467 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.467 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.467 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op must wait for map", 2024-09-14T04:38:41.467 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:26.291123+0000", 2024-09-14T04:38:41.467 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9870000000000002e-06 2024-09-14T04:38:41.467 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.467 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.468 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:38:41.468 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:26.296794+0000", 2024-09-14T04:38:41.468 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0056717190000000004 2024-09-14T04:38:41.468 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.468 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.468 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:38:41.468 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:26.296878+0000", 2024-09-14T04:38:41.468 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.3945999999999999e-05 2024-09-14T04:38:41.468 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.468 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.468 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:38:41.468 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:26.297058+0000", 2024-09-14T04:38:41.468 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00017956999999999999 2024-09-14T04:38:41.469 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.469 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.469 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:38:41.469 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:26.312157+0000", 2024-09-14T04:38:41.469 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.015098913 2024-09-14T04:38:41.469 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.469 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.469 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.469 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:26.324088+0000", 2024-09-14T04:38:41.469 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.011931659000000001 2024-09-14T04:38:41.469 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.469 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.469 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.470 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:26.324147+0000", 2024-09-14T04:38:41.470 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.8791e-05 2024-09-14T04:38:41.470 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.470 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.470 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:38:41.470 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:26.324216+0000", 2024-09-14T04:38:41.470 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9028000000000004e-05 2024-09-14T04:38:41.470 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.470 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.470 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:38:41.470 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:26.324294+0000", 2024-09-14T04:38:41.470 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7742000000000002e-05 2024-09-14T04:38:41.470 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.471 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:38:41.471 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.471 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.471 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.471 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.34914.0:2693 52.1d 52:b980f0d4:::rbd_data.8862fb33ee1a.000000000000003d:head [write 0~2884975] snapc 70=[70,6f,6e,6d,6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1278)", 2024-09-14T04:38:41.471 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:36:28.310188+0000", 2024-09-14T04:38:41.471 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 133.01277434400001, 2024-09-14T04:38:41.471 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.045969726000000002, 2024-09-14T04:38:41.471 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:38:41.471 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:38:41.471 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:38:41.472 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:38:41.472 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.34914", 2024-09-14T04:38:41.472 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1048804342", 2024-09-14T04:38:41.472 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 2693 2024-09-14T04:38:41.472 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.472 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:38:41.472 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.472 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:38:41.472 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:28.310188+0000", 2024-09-14T04:38:41.472 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.472 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.472 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.473 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:38:41.473 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:28.310188+0000", 2024-09-14T04:38:41.473 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.473 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.473 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.473 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:38:41.473 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:28.310193+0000", 2024-09-14T04:38:41.473 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5510000000000001e-06 2024-09-14T04:38:41.473 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.473 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.473 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:38:41.473 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:28.314405+0000", 2024-09-14T04:38:41.473 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0042119829999999999 2024-09-14T04:38:41.473 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.474 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.474 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:38:41.474 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:28.314429+0000", 2024-09-14T04:38:41.474 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4049999999999998e-05 2024-09-14T04:38:41.474 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.474 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.474 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:38:41.474 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:28.314444+0000", 2024-09-14T04:38:41.474 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4827999999999999e-05 2024-09-14T04:38:41.474 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.474 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.474 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:38:41.474 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:28.315154+0000", 2024-09-14T04:38:41.475 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00070995699999999999 2024-09-14T04:38:41.475 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.475 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.475 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:38:41.475 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:28.315294+0000", 2024-09-14T04:38:41.475 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014081 2024-09-14T04:38:41.475 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.475 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.475 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:38:41.476 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:28.315520+0000", 2024-09-14T04:38:41.476 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00022537300000000001 2024-09-14T04:38:41.476 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.476 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.476 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:38:41.476 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:28.325551+0000", 2024-09-14T04:38:41.476 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.010031632 2024-09-14T04:38:41.476 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.476 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.476 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.476 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:28.352726+0000", 2024-09-14T04:38:41.476 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.027174904999999999 2024-09-14T04:38:41.477 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.477 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.477 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.477 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:28.356056+0000", 2024-09-14T04:38:41.477 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0033292920000000002 2024-09-14T04:38:41.477 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.477 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.477 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:38:41.478 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:28.356106+0000", 2024-09-14T04:38:41.478 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.0856999999999998e-05 2024-09-14T04:38:41.478 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.478 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.478 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:38:41.478 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:28.356158+0000", 2024-09-14T04:38:41.479 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1487999999999999e-05 2024-09-14T04:38:41.479 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.479 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:38:41.479 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.479 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.479 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.479 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.34914.0:2746 52.1d 52:bc815abd:::rbd_data.8862fb33ee1a.0000000000000052:head [write 2994106~1200198] snapc 71=[71,70,6f,6e,6d,6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1279)", 2024-09-14T04:38:41.479 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:36:29.326913+0000", 2024-09-14T04:38:41.479 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 131.99604984300001, 2024-09-14T04:38:41.479 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.024578830999999999, 2024-09-14T04:38:41.479 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:38:41.479 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:38:41.480 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:38:41.480 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:38:41.480 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.34914", 2024-09-14T04:38:41.480 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1048804342", 2024-09-14T04:38:41.480 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 2746 2024-09-14T04:38:41.480 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.481 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:38:41.481 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.481 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:38:41.481 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:29.326913+0000", 2024-09-14T04:38:41.481 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.481 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.481 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.481 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:38:41.481 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:29.326913+0000", 2024-09-14T04:38:41.482 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.482 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.482 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.482 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:38:41.482 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:41.483 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:41 smithi039 ceph-mon[106470]: pgmap v2160: 89 pgs: 89 active+clean; 515 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-14T04:38:41.483 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:41 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/10122336' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:38:41.483 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:41 smithi039 ceph-mon[111235]: pgmap v2160: 89 pgs: 89 active+clean; 515 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-14T04:38:41.483 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:41 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/10122336' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:38:41.484 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:29.326916+0000", 2024-09-14T04:38:41.484 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.858e-06 2024-09-14T04:38:41.484 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.484 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.484 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:38:41.484 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:29.327908+0000", 2024-09-14T04:38:41.484 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00099249300000000011 2024-09-14T04:38:41.484 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.484 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.484 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:38:41.485 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:29.327934+0000", 2024-09-14T04:38:41.485 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.5885e-05 2024-09-14T04:38:41.485 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.485 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.485 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:38:41.485 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:29.327945+0000", 2024-09-14T04:38:41.485 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0538e-05 2024-09-14T04:38:41.485 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.485 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.485 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:38:41.485 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:29.328011+0000", 2024-09-14T04:38:41.485 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.6540999999999999e-05 2024-09-14T04:38:41.485 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.485 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.486 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op must wait for map", 2024-09-14T04:38:41.486 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:29.328018+0000", 2024-09-14T04:38:41.486 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1910000000000003e-06 2024-09-14T04:38:41.486 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.486 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.486 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:38:41.486 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:29.330001+0000", 2024-09-14T04:38:41.486 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001982667 2024-09-14T04:38:41.486 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.486 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.486 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:38:41.486 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:29.330116+0000", 2024-09-14T04:38:41.487 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011475499999999999 2024-09-14T04:38:41.487 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.487 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.487 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:38:41.487 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:29.330376+0000", 2024-09-14T04:38:41.487 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000260604 2024-09-14T04:38:41.487 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.487 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.487 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:38:41.487 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:29.334728+0000", 2024-09-14T04:38:41.487 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.004352145 2024-09-14T04:38:41.487 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.487 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.487 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.487 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:29.343767+0000", 2024-09-14T04:38:41.488 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0090383459999999992 2024-09-14T04:38:41.488 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.488 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.488 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.488 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:29.351354+0000", 2024-09-14T04:38:41.488 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0075873190000000004 2024-09-14T04:38:41.488 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.488 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.488 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:38:41.488 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:29.351433+0000", 2024-09-14T04:38:41.488 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9282000000000004e-05 2024-09-14T04:38:41.488 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.488 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.489 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:38:41.489 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:29.351492+0000", 2024-09-14T04:38:41.489 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.8207e-05 2024-09-14T04:38:41.489 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.489 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:38:41.489 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.489 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.489 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.489 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.34914.0:2738 52.a e1279/1165 52:516b3bd4:::rbd_data.8862fb33ee1a.0000000000000029:head v 1279'23, mlcod=1261'20)", 2024-09-14T04:38:41.489 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:36:29.330061+0000", 2024-09-14T04:38:41.489 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 131.99290152500001, 2024-09-14T04:38:41.489 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.023865382000000001, 2024-09-14T04:38:41.489 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:38:41.490 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:38:41.490 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:38:41.490 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:38:41.490 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.490 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:38:41.490 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:29.330061+0000", 2024-09-14T04:38:41.490 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.490 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.490 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.490 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:38:41.490 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:29.330061+0000", 2024-09-14T04:38:41.490 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.490 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.490 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.491 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:38:41.491 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:29.330061+0000", 2024-09-14T04:38:41.491 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.1e-07 2024-09-14T04:38:41.491 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.491 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.491 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:38:41.491 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:29.347623+0000", 2024-09-14T04:38:41.491 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.017561863 2024-09-14T04:38:41.491 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.491 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.491 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:38:41.491 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:29.347649+0000", 2024-09-14T04:38:41.491 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.5429e-05 2024-09-14T04:38:41.492 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.492 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.492 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:38:41.492 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:29.347661+0000", 2024-09-14T04:38:41.492 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2397e-05 2024-09-14T04:38:41.492 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.492 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.492 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:38:41.492 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:29.347743+0000", 2024-09-14T04:38:41.492 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.1661000000000003e-05 2024-09-14T04:38:41.493 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.493 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.493 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:38:41.493 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:29.347774+0000", 2024-09-14T04:38:41.493 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.1711999999999997e-05 2024-09-14T04:38:41.493 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.494 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.494 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:38:41.494 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:29.353856+0000", 2024-09-14T04:38:41.494 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0060812549999999998 2024-09-14T04:38:41.494 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.494 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.495 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:38:41.495 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:29.353926+0000", 2024-09-14T04:38:41.495 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.0754999999999996e-05 2024-09-14T04:38:41.495 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.495 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:38:41.495 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.495 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.495 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.495 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.34914.0:2853 52.1d 52:bd61e55e:::rbd_data.8862fb33ee1a.0000000000000033:head [writefull 0~4194304] snapc 73=[73,72,71,70,6f,6e,6d,6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1281)", 2024-09-14T04:38:41.496 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:36:31.355916+0000", 2024-09-14T04:38:41.496 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 129.96704616299999, 2024-09-14T04:38:41.496 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.051953660999999998, 2024-09-14T04:38:41.496 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:38:41.496 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:38:41.496 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:38:41.496 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:38:41.497 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.34914", 2024-09-14T04:38:41.497 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1048804342", 2024-09-14T04:38:41.497 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 2853 2024-09-14T04:38:41.497 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.497 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:38:41.497 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.497 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:38:41.497 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.355916+0000", 2024-09-14T04:38:41.497 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.497 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.497 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.497 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:38:41.498 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.355916+0000", 2024-09-14T04:38:41.498 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.498 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.498 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.498 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:38:41.498 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.355919+0000", 2024-09-14T04:38:41.498 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.9040000000000001e-06 2024-09-14T04:38:41.498 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.498 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.498 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:38:41.498 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.359739+0000", 2024-09-14T04:38:41.498 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0038191800000000001 2024-09-14T04:38:41.498 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.498 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.498 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:38:41.499 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.359766+0000", 2024-09-14T04:38:41.499 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.7631e-05 2024-09-14T04:38:41.499 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.499 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.499 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:38:41.499 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.359776+0000", 2024-09-14T04:38:41.499 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.6350000000000001e-06 2024-09-14T04:38:41.499 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.499 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.499 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:38:41.499 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.359979+0000", 2024-09-14T04:38:41.500 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000203415 2024-09-14T04:38:41.500 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.500 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.500 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:38:41.500 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.360079+0000", 2024-09-14T04:38:41.500 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.9874e-05 2024-09-14T04:38:41.500 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.500 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.500 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:38:41.500 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.360265+0000", 2024-09-14T04:38:41.500 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00018635400000000001 2024-09-14T04:38:41.500 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.501 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:38:41.501 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:38:41.501 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:38:41.502 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.502 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:38:41.502 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.369447+0000", 2024-09-14T04:38:41.502 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0091820410000000002 2024-09-14T04:38:41.502 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.502 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.503 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.503 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.405296+0000", 2024-09-14T04:38:41.503 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.035848182999999999 2024-09-14T04:38:41.503 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.503 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.503 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.503 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.407779+0000", 2024-09-14T04:38:41.503 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0024835640000000002 2024-09-14T04:38:41.503 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.503 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.504 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:38:41.504 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.407824+0000", 2024-09-14T04:38:41.504 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4328e-05 2024-09-14T04:38:41.504 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.504 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.505 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:38:41.505 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.407870+0000", 2024-09-14T04:38:41.505 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6551999999999997e-05 2024-09-14T04:38:41.505 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.505 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:38:41.505 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.505 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.506 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.506 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.34914.0:2962 52.1d 52:b980f0d4:::rbd_data.8862fb33ee1a.000000000000003d:head [write 0~3480559] snapc 75=[75,74,73,72,71,70,6f,6e,6d,6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1283)", 2024-09-14T04:38:41.506 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:36:33.397917+0000", 2024-09-14T04:38:41.506 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 127.92504605400001, 2024-09-14T04:38:41.506 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.039805216999999997, 2024-09-14T04:38:41.506 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:38:41.506 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:38:41.506 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:38:41.506 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:38:41.506 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.34914", 2024-09-14T04:38:41.506 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1048804342", 2024-09-14T04:38:41.506 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 2962 2024-09-14T04:38:41.506 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.506 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:38:41.507 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.507 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:38:41.507 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:33.397917+0000", 2024-09-14T04:38:41.507 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.507 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.507 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.507 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:38:41.507 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:33.397917+0000", 2024-09-14T04:38:41.507 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.507 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.507 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.508 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:38:41.508 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:33.397920+0000", 2024-09-14T04:38:41.508 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.4020000000000002e-06 2024-09-14T04:38:41.508 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.508 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.509 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:38:41.509 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:33.402396+0000", 2024-09-14T04:38:41.509 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0044765259999999998 2024-09-14T04:38:41.509 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.509 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.509 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:38:41.509 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:33.402430+0000", 2024-09-14T04:38:41.509 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.3461999999999999e-05 2024-09-14T04:38:41.509 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.509 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.509 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:38:41.510 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:33.402444+0000", 2024-09-14T04:38:41.510 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4469e-05 2024-09-14T04:38:41.510 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.510 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.510 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:38:41.510 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:33.402563+0000", 2024-09-14T04:38:41.510 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000118847 2024-09-14T04:38:41.510 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.510 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.510 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:38:41.510 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:33.402681+0000", 2024-09-14T04:38:41.510 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000117821 2024-09-14T04:38:41.510 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.510 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.511 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:38:41.511 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:33.402962+0000", 2024-09-14T04:38:41.511 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00028061999999999998 2024-09-14T04:38:41.511 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.511 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.511 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:38:41.511 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:33.410701+0000", 2024-09-14T04:38:41.511 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0077389080000000001 2024-09-14T04:38:41.511 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.511 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.511 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.511 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:33.431163+0000", 2024-09-14T04:38:41.512 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.020462734 2024-09-14T04:38:41.512 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.512 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.512 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.512 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:33.437472+0000", 2024-09-14T04:38:41.512 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.006308892 2024-09-14T04:38:41.512 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.512 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.512 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:38:41.512 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:33.437586+0000", 2024-09-14T04:38:41.512 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000114162 2024-09-14T04:38:41.512 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.512 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.513 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:38:41.513 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:33.437722+0000", 2024-09-14T04:38:41.513 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013537400000000001 2024-09-14T04:38:41.513 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.513 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:38:41.513 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.513 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.513 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.513 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.34914.0:3196 52.1d 52:bc815abd:::rbd_data.8862fb33ee1a.0000000000000052:head [write 1499689~1817739] snapc 79=[79,78,77,76,75,74,73,72,71,70,6f,6e,6d,6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1287)", 2024-09-14T04:38:41.513 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:36:37.456935+0000", 2024-09-14T04:38:41.513 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 123.866027766, 2024-09-14T04:38:41.513 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.025043824999999999, 2024-09-14T04:38:41.513 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:38:41.513 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:38:41.514 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:38:41.514 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:38:41.514 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.34914", 2024-09-14T04:38:41.514 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1048804342", 2024-09-14T04:38:41.514 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 3196 2024-09-14T04:38:41.514 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.514 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:38:41.514 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.514 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:38:41.514 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:37.456935+0000", 2024-09-14T04:38:41.514 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.514 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.515 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.515 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:38:41.515 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:37.456935+0000", 2024-09-14T04:38:41.515 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.515 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.515 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.515 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:38:41.515 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:37.456939+0000", 2024-09-14T04:38:41.516 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.9049999999999999e-06 2024-09-14T04:38:41.516 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.516 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.516 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:38:41.516 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:37.458987+0000", 2024-09-14T04:38:41.516 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0020479719999999999 2024-09-14T04:38:41.517 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.517 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.517 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:38:41.517 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:37.459041+0000", 2024-09-14T04:38:41.517 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4759000000000002e-05 2024-09-14T04:38:41.517 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.517 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.517 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:38:41.517 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:37.459062+0000", 2024-09-14T04:38:41.517 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0049999999999999e-05 2024-09-14T04:38:41.517 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.517 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.517 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:38:41.518 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:37.459253+0000", 2024-09-14T04:38:41.518 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00019114499999999999 2024-09-14T04:38:41.518 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.518 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.518 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:38:41.518 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:37.459398+0000", 2024-09-14T04:38:41.519 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014497800000000001 2024-09-14T04:38:41.519 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.519 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.519 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:38:41.519 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:37.459651+0000", 2024-09-14T04:38:41.519 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00025377100000000002 2024-09-14T04:38:41.519 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.519 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.519 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:38:41.519 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:37.466634+0000", 2024-09-14T04:38:41.519 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0069829360000000004 2024-09-14T04:38:41.519 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.519 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.519 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.520 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:37.477867+0000", 2024-09-14T04:38:41.520 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.011233006 2024-09-14T04:38:41.520 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.520 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.520 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.520 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:37.481875+0000", 2024-09-14T04:38:41.520 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0040077769999999997 2024-09-14T04:38:41.520 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.520 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.520 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:38:41.520 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:37.481920+0000", 2024-09-14T04:38:41.520 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4869e-05 2024-09-14T04:38:41.520 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.521 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.521 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:38:41.521 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:37.481979+0000", 2024-09-14T04:38:41.521 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.8656999999999998e-05 2024-09-14T04:38:41.521 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.521 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:38:41.521 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.521 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.521 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.521 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.34914.0:7344 52.1d 52:ba931cc8:::rbd_data.8862ba52cd58.0000000000000000:head [set-alloc-hint object_size 4194304 write_size 4194304,writefull 0~4194304] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1311)", 2024-09-14T04:38:41.521 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:37:01.028179+0000", 2024-09-14T04:38:41.521 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 100.29478405099999, 2024-09-14T04:38:41.521 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.032001193999999997, 2024-09-14T04:38:41.521 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:38:41.522 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:38:41.522 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:38:41.522 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:38:41.522 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.34914", 2024-09-14T04:38:41.522 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1048804342", 2024-09-14T04:38:41.522 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 7344 2024-09-14T04:38:41.522 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.522 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:38:41.522 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.522 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:38:41.522 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.028179+0000", 2024-09-14T04:38:41.522 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.522 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.522 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.523 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:38:41.523 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.028179+0000", 2024-09-14T04:38:41.523 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.523 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.523 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.523 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:38:41.523 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.028180+0000", 2024-09-14T04:38:41.523 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.702e-06 2024-09-14T04:38:41.523 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.523 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.523 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:38:41.523 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.030544+0000", 2024-09-14T04:38:41.523 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0023633759999999999 2024-09-14T04:38:41.523 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.523 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.524 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:38:41.524 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.030573+0000", 2024-09-14T04:38:41.524 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.9513999999999999e-05 2024-09-14T04:38:41.524 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.524 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.524 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:38:41.524 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.030584+0000", 2024-09-14T04:38:41.524 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1119e-05 2024-09-14T04:38:41.524 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.524 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.524 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:38:41.524 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.030670+0000", 2024-09-14T04:38:41.525 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.5964000000000006e-05 2024-09-14T04:38:41.525 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.525 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.525 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:38:41.525 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.030867+0000", 2024-09-14T04:38:41.525 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00019704 2024-09-14T04:38:41.525 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.525 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.525 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:38:41.525 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.031057+0000", 2024-09-14T04:38:41.525 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00019018400000000001 2024-09-14T04:38:41.525 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.526 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.526 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:38:41.526 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.043527+0000", 2024-09-14T04:38:41.526 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.012469310000000001 2024-09-14T04:38:41.526 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.526 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.526 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.526 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.045858+0000", 2024-09-14T04:38:41.526 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0023310850000000001 2024-09-14T04:38:41.526 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.526 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.526 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:38:41.526 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.060081+0000", 2024-09-14T04:38:41.527 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.014222811 2024-09-14T04:38:41.527 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.527 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.527 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:38:41.527 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.060126+0000", 2024-09-14T04:38:41.527 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5633000000000002e-05 2024-09-14T04:38:41.527 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.527 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.527 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:38:41.527 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.060180+0000", 2024-09-14T04:38:41.527 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3455999999999998e-05 2024-09-14T04:38:41.527 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.527 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:38:41.528 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.528 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.528 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.528 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:17479 53.1 e1319/1315 53:81a148aa:::benchmark_data_smithi039_224949_object17478:head v 1319'1142, mlcod=1319'1140)", 2024-09-14T04:38:41.528 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:38:41.273408+0000", 2024-09-14T04:38:41.528 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.049554170000000002, 2024-09-14T04:38:41.528 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.013235999, 2024-09-14T04:38:41.528 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:38:41.528 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:38:41.528 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:38:41.528 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:38:41.528 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.528 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:38:41.529 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:41.273408+0000", 2024-09-14T04:38:41.529 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.529 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.529 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.529 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:38:41.529 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:41.273408+0000", 2024-09-14T04:38:41.529 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:38:41.529 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.529 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.530 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:38:41.530 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:41.273409+0000", 2024-09-14T04:38:41.530 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.8299999999999997e-07 2024-09-14T04:38:41.530 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.530 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.530 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:38:41.530 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:41.273444+0000", 2024-09-14T04:38:41.530 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.4647999999999998e-05 2024-09-14T04:38:41.531 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.531 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.531 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:38:41.531 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:41.273471+0000", 2024-09-14T04:38:41.531 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.7860000000000001e-05 2024-09-14T04:38:41.531 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.531 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.531 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:38:41.531 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:41.273483+0000", 2024-09-14T04:38:41.531 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1516e-05 2024-09-14T04:38:41.531 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.532 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.532 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:38:41.532 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:41.276022+0000", 2024-09-14T04:38:41.532 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.002539464 2024-09-14T04:38:41.532 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.532 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.532 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:38:41.532 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:41.276040+0000", 2024-09-14T04:38:41.532 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7784000000000001e-05 2024-09-14T04:38:41.532 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.533 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.533 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:38:41.533 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:41.286611+0000", 2024-09-14T04:38:41.533 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.010570939 2024-09-14T04:38:41.533 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:38:41.533 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:38:41.534 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:38:41.534 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:41.286644+0000", 2024-09-14T04:38:41.534 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.3305e-05 2024-09-14T04:38:41.534 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.534 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:38:41.534 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.534 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:38:41.535 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:38:41.535 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:38:41.535 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:38:41.536 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:38:41.536 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:38:41.537 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:38:41.538 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:38:41.538 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:38:41.579 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:41.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:41.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:41.681 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:41.690 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 6 16 18428 18412 191.714 147.562 0.00903141 0.00520794 2024-09-14T04:38:41.718 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:41.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:41.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:41 smithi129 ceph-mon[101316]: pgmap v2160: 89 pgs: 89 active+clean; 515 MiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-14T04:38:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:41 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/10122336' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:38:41.821 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:41.861 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:41.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:41.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:41.963 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:42.003 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:42.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:42.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:42.104 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:42.135 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:42.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:42.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:42.237 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:42.277 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:42.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:42.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:42.380 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:42.408 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:42.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:42.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:42.510 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:42.550 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:42.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:42.551 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:42.652 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:42.688 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:42.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:42.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:42.690 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 7 16 21061 21045 187.834 164.562 0.00636707 0.00531833 2024-09-14T04:38:42.790 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:42.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:42.839 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:42.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:42.940 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:42.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:42.973 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:42.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:43.074 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:43.112 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:43.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:43.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:43.214 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:43.251 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:43.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:43.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:43.353 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:43.391 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:43.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:43.392 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:43.493 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:43.529 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:43.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:43.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:43.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:43 smithi039 ceph-mon[106470]: pgmap v2161: 89 pgs: 89 active+clean; 633 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-14T04:38:43.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:43 smithi039 ceph-mon[111235]: pgmap v2161: 89 pgs: 89 active+clean; 633 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-14T04:38:43.631 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:43.663 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:43.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:43.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:43.691 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 8 15 23770 23755 185.525 169.375 0.00798563 0.00538399 2024-09-14T04:38:43.764 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:43.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:43 smithi129 ceph-mon[101316]: pgmap v2161: 89 pgs: 89 active+clean; 633 MiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-14T04:38:43.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:43.801 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:43.802 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:43.902 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:43.932 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:43.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:43.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:44.034 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:44.061 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:44.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:44.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:44.163 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:44.192 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:44.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:44.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:44.294 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:44.326 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:44.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:44.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:44.428 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:44.465 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:44.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:44.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:44.567 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:44.602 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:44.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:44.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:44.691 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 9 16 26400 26384 183.167 164.312 0.00592293 0.00545413 2024-09-14T04:38:44.704 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:44.742 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:44.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:44.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:44.810 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:38:44.845 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:44.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:44.896 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:44.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:44.997 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:45.019 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:45.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:45.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:45.120 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:45.154 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:45.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:45.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:45.256 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:45.297 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:45.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:45.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:45.398 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:45.429 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:45.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:45.430 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:45.531 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:45.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:45.556 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:45.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:45.657 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:45.689 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:45.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:45.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:45.691 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 10 16 28711 28695 179.293 144.438 0.00609997 0.00557223 2024-09-14T04:38:45.708 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:45 smithi039 ceph-mon[106470]: pgmap v2162: 89 pgs: 89 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-09-14T04:38:45.708 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:45 smithi039 ceph-mon[111235]: pgmap v2162: 89 pgs: 89 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-09-14T04:38:45.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:45 smithi129 ceph-mon[101316]: pgmap v2162: 89 pgs: 89 active+clean; 1.1 GiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-09-14T04:38:45.791 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:45.812 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:45.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:45.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:45.914 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:45.951 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:45.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:45.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:46.053 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:46.073 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:46.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:46.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:46.175 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:46.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:46.216 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:46.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:46.317 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:46.350 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:46.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:46.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:46.452 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:46.488 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:46.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:46.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:46.590 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:46.612 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:46.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:46.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:46.691 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 11 16 30622 30606 173.851 119.438 0.00182893 0.00573539 2024-09-14T04:38:46.715 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:46.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:46.753 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:46.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:46.854 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:46.890 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:46.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:46.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:46.992 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:47.026 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:47.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:47.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:47.128 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:47.161 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:47.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:47.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:47.263 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:47.294 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:47.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:47.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:47.396 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:47.435 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:47.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:47.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:47.537 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:47.572 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:47.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:47.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:47.673 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:47.691 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 12 16 32641 32625 169.876 126.188 0.00697998 0.00588058 2024-09-14T04:38:47.714 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:47.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:47.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:47.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:47 smithi129 ceph-mon[101316]: pgmap v2163: 89 pgs: 89 active+clean; 1.3 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s 2024-09-14T04:38:47.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:47 smithi039 ceph-mon[106470]: pgmap v2163: 89 pgs: 89 active+clean; 1.3 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s 2024-09-14T04:38:47.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:47 smithi039 ceph-mon[111235]: pgmap v2163: 89 pgs: 89 active+clean; 1.3 GiB data, 4.7 GiB used, 710 GiB / 715 GiB avail; 133 MiB/s wr, 2.13k op/s 2024-09-14T04:38:47.815 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:47.836 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:47.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:47.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:47.938 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:47.977 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:47.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:47.978 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:48.079 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:48.113 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:48.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:48.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:48.215 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:48.252 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:48.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:48.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:48.354 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:48.386 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:48.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:48.387 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:48.488 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:48.519 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:48.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:48.520 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:48.621 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:48.656 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:48.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:48.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:48.691 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 13 16 34809 34793 167.231 135.5 0.00551862 0.00597485 2024-09-14T04:38:48.758 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:48.798 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:48.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:48.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:48.900 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:48.932 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:48.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:48.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:49.033 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:49.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:38:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:38:48] "GET /metrics HTTP/1.1" 200 36670 "" "Prometheus/2.51.0" 2024-09-14T04:38:49.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:49.062 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:49.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:49.163 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:49.209 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:49.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:49.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:49.310 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:49.333 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:49.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:49.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:49.435 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:49.468 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:49.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:49.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:49.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:49 smithi039 ceph-mon[106470]: pgmap v2164: 89 pgs: 89 active+clean; 1.5 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2024-09-14T04:38:49.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:49 smithi039 ceph-mon[111235]: pgmap v2164: 89 pgs: 89 active+clean; 1.5 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2024-09-14T04:38:49.570 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:49.607 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:49.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:49.608 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:49.691 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 14 15 36736 36721 163.893 120.5 0.0102568 0.00609677 2024-09-14T04:38:49.709 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:49.745 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:49.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:49.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:49.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:49 smithi129 ceph-mon[101316]: pgmap v2164: 89 pgs: 89 active+clean; 1.5 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 129 MiB/s wr, 2.06k op/s 2024-09-14T04:38:49.847 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:49.871 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:49.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:49.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:49.972 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:49.995 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:49.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:49.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:50.096 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:50.133 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:50.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:50.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:50.235 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:50.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:50.282 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:50.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:50.383 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:50.423 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:50.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:50.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:50.525 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:50.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:50.560 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:50.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:50.661 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:50.692 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 15 16 38410 38394 159.938 104.562 0.0136577 0.00624703 2024-09-14T04:38:50.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:50.695 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:50.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:50.796 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:50.819 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:50.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:50.820 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:50.921 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:50.960 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:50.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:50.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:51.020 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-14T04:38:51.063 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:51.094 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:51.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:51.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:51.196 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:51.228 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:51.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:51.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:51.262 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:38:51.329 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:51.363 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:51.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:51.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:51.464 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:51.509 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:51.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:51.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:51.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:51 smithi039 ceph-mon[106470]: pgmap v2165: 89 pgs: 89 active+clean; 2.0 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 167 MiB/s wr, 2.67k op/s 2024-09-14T04:38:51.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:51 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:38:51.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:51 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:38:51.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:51 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:38:51.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:51 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:38:51.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:51 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:38:51.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:51 smithi039 ceph-mon[111235]: pgmap v2165: 89 pgs: 89 active+clean; 2.0 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 167 MiB/s wr, 2.67k op/s 2024-09-14T04:38:51.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:51 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:38:51.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:51 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:38:51.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:51 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:38:51.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:51 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:38:51.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:51 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:38:51.611 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:51.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:51.643 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:51.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:51.692 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 16 16 40297 40281 157.312 117.938 0.00771493 0.00635171 2024-09-14T04:38:51.744 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:51 smithi129 ceph-mon[101316]: pgmap v2165: 89 pgs: 89 active+clean; 2.0 GiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 167 MiB/s wr, 2.67k op/s 2024-09-14T04:38:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:51 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:38:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:51 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:38:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:51 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:38:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:51 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:38:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:51 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:38:51.780 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:51.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:51.781 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:51.882 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:51.915 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:51.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:51.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:52.017 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:52.050 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:52.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:52.051 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:52.152 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:52.182 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:52.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:52.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:52.284 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:52.312 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:52.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:52.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:52.414 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:52.444 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:52.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:52.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:52.546 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:52.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:52.590 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:52.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:52.659 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:38:52.659 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:38:52.692 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:52.693 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 17 16 42328 42312 155.514 126.938 0.0080569 0.00642528 2024-09-14T04:38:52.734 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:52.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:52.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:52.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:38:52.836 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:52.875 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:52.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:52.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:52.977 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:53.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:53.011 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:53.012 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:53.112 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:53.145 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:53.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:53.146 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:53.247 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:53.287 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:53.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:53.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:53.389 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:53.425 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:53.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:53.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:53.528 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:53.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:53 smithi039 ceph-mon[106470]: pgmap v2166: 89 pgs: 89 active+clean; 2.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2024-09-14T04:38:53.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:53 smithi039 ceph-mon[111235]: pgmap v2166: 89 pgs: 89 active+clean; 2.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2024-09-14T04:38:53.558 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:53.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:53.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:53.660 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:53.693 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 18 16 44344 44328 153.874 126 0.00738689 0.0064932 2024-09-14T04:38:53.702 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:53.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:53.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:53.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:53 smithi129 ceph-mon[101316]: pgmap v2166: 89 pgs: 89 active+clean; 2.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2024-09-14T04:38:53.804 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:53.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:53.842 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:53.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:53.943 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:53.982 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:53.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:53.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:54.084 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:54.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:54.115 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:54.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:54.216 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:54.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:54.251 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:54.251 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:54.352 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:54.379 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:54.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:54.380 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:54.481 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:54.526 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:54.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:54.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:54.628 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:54.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:54.674 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:54.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:54.682 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:38:54.687 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T04:38:54.693 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 19 16 46049 46033 151.384 106.562 0.00813933 0.00659876 2024-09-14T04:38:54.774 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:54.806 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:54.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:54.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:54.907 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:54.947 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:54.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:54.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:55.049 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:55.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:55.093 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:55.093 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:55.194 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:55.231 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:55.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:55.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:55.333 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:55.366 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:55.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:55.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:55.393 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is unset 2024-09-14T04:38:55.468 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:55.508 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:55.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:55.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:55.610 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:55.644 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:55.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:55.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:55.693 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:2024-09-14T04:38:55.693626+0000 min lat: 0.00122803 max lat: 0.11781 avg lat: 0.00668994 2024-09-14T04:38:55.694 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:38:55.694 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 20 16 47819 47803 149.344 110.625 0.0128527 0.00668994 2024-09-14T04:38:55.746 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:55 smithi129 ceph-mon[101316]: pgmap v2167: 89 pgs: 89 active+clean; 2.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 158 MiB/s wr, 2.53k op/s 2024-09-14T04:38:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:55 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3852205421' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:38:55.777 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:55.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:55.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:55.787 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:55 smithi039 ceph-mon[106470]: pgmap v2167: 89 pgs: 89 active+clean; 2.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 158 MiB/s wr, 2.53k op/s 2024-09-14T04:38:55.787 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:55 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3852205421' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:38:55.788 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:55 smithi039 ceph-mon[111235]: pgmap v2167: 89 pgs: 89 active+clean; 2.5 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 158 MiB/s wr, 2.53k op/s 2024-09-14T04:38:55.788 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:55 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3852205421' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:38:55.879 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:55.913 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:55.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:55.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:56.015 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:56.050 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:56.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:56.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:56.152 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:56.184 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:56.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:56.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:56.286 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:56.321 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:56.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:56.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:56.420 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:38:56.423 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:56.446 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:56.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:56.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:56.548 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:56.577 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:56.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:56.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:56.644 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:56 smithi039 ceph-mon[106470]: from='client.45062 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:38:56.644 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:56 smithi039 ceph-mon[106470]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:38:56.644 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:56 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3852205421' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:38:56.644 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:56 smithi039 ceph-mon[106470]: osdmap e1320: 8 total, 8 up, 8 in 2024-09-14T04:38:56.644 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:56 smithi039 ceph-mon[111235]: from='client.45062 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:38:56.644 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:56 smithi039 ceph-mon[111235]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:38:56.644 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:56 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3852205421' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:38:56.644 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:56 smithi039 ceph-mon[111235]: osdmap e1320: 8 total, 8 up, 8 in 2024-09-14T04:38:56.679 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:56.694 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 21 16 49578 49562 147.464 109.938 0.00741988 0.00677596 2024-09-14T04:38:56.721 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:56.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:56.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:56.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:56 smithi129 ceph-mon[101316]: from='client.45062 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:38:56.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:56 smithi129 ceph-mon[101316]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:38:56.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:56 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3852205421' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:38:56.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:56 smithi129 ceph-mon[101316]: osdmap e1320: 8 total, 8 up, 8 in 2024-09-14T04:38:56.823 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:56.853 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:56.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:56.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:56.955 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:56.990 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:56.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:56.991 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:57.008 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":2167,"stamp":"2024-09-14T04:38:53.497103+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2652837861,"num_objects":40496,"num_object_clones":0,"num_object_copies":121488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":40496,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":40749,"num_write_kb":2592529,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":40678,"ondisk_log_size":40678,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8461224,"kb_used_data":7613288,"kb_used_omap":288,"kb_used_meta":847583,"kb_avail":741499992,"statfs":{"total":767960285184,"available":759295991808,"internally_reserved":0,"allocated":7796006912,"data_stored":7758684215,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":294978,"internal_metadata":867925950},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[7,7,2,1,2,1,7],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1989148672,"num_objects":30352,"num_object_clones":0,"num_object_copies":91056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":30352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":30352,"num_write_kb":1942528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.006054"},"pg_stats":[{"pgid":"53.f","version":"1319'2661","reported_seq":2674,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:52.765134+0000","last_change":"2024-09-14T04:38:26.895527+0000","last_active":"2024-09-14T04:38:52.765134+0000","last_peered":"2024-09-14T04:38:52.765134+0000","last_clean":"2024-09-14T04:38:52.765134+0000","last_became_active":"2024-09-14T04:38:26.895218+0000","last_became_peered":"2024-09-14T04:38:26.895218+0000","last_unstale":"2024-09-14T04:38:52.765134+0000","last_undegraded":"2024-09-14T04:38:52.765134+0000","last_fullsized":"2024-09-14T04:38:52.765134+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":2661,"log_dups_size":0,"ondisk_log_size":2661,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:07:35.310515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174391296,"num_objects":2661,"num_object_clones":0,"num_object_copies":7983,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2661,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2661,"num_write_kb":170304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"53.e","version":"1319'2555","reported_seq":2567,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:52.806245+0000","last_change":"2024-09-14T04:38:26.894036+0000","last_active":"2024-09-14T04:38:52.806245+0000","last_peered":"2024-09-14T04:38:52.806245+0000","last_clean":"2024-09-14T04:38:52.806245+0000","last_became_active":"2024-09-14T04:38:26.893654+0000","last_became_peered":"2024-09-14T04:38:26.893654+0000","last_unstale":"2024-09-14T04:38:52.806245+0000","last_undegraded":"2024-09-14T04:38:52.806245+0000","last_fullsized":"2024-09-14T04:38:52.806245+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":2555,"log_dups_size":0,"ondisk_log_size":2555,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:39:17.902111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167444480,"num_objects":2555,"num_object_clones":0,"num_object_copies":7665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2555,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2555,"num_write_kb":163520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"53.d","version":"1319'2137","reported_seq":2156,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:49.255138+0000","last_change":"2024-09-14T04:38:27.529546+0000","last_active":"2024-09-14T04:38:49.255138+0000","last_peered":"2024-09-14T04:38:49.255138+0000","last_clean":"2024-09-14T04:38:49.255138+0000","last_became_active":"2024-09-14T04:38:27.529237+0000","last_became_peered":"2024-09-14T04:38:27.529237+0000","last_unstale":"2024-09-14T04:38:49.255138+0000","last_undegraded":"2024-09-14T04:38:49.255138+0000","last_fullsized":"2024-09-14T04:38:49.255138+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":2137,"log_dups_size":0,"ondisk_log_size":2137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:37.769458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140050432,"num_objects":2137,"num_object_clones":0,"num_object_copies":6411,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2137,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2137,"num_write_kb":136768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"53.c","version":"1319'2778","reported_seq":2791,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:52.783235+0000","last_change":"2024-09-14T04:38:26.895807+0000","last_active":"2024-09-14T04:38:52.783235+0000","last_peered":"2024-09-14T04:38:52.783235+0000","last_clean":"2024-09-14T04:38:52.783235+0000","last_became_active":"2024-09-14T04:38:26.895506+0000","last_became_peered":"2024-09-14T04:38:26.895506+0000","last_unstale":"2024-09-14T04:38:52.783235+0000","last_undegraded":"2024-09-14T04:38:52.783235+0000","last_fullsized":"2024-09-14T04:38:52.783235+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":2778,"log_dups_size":0,"ondisk_log_size":2778,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:07:45.042093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182059008,"num_objects":2778,"num_object_clones":0,"num_object_copies":8334,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2778,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2778,"num_write_kb":177792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"53.b","version":"1319'2637","reported_seq":2650,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:52.800824+0000","last_change":"2024-09-14T04:38:26.893051+0000","last_active":"2024-09-14T04:38:52.800824+0000","last_peered":"2024-09-14T04:38:52.800824+0000","last_clean":"2024-09-14T04:38:52.800824+0000","last_became_active":"2024-09-14T04:38:26.892762+0000","last_became_peered":"2024-09-14T04:38:26.892762+0000","last_unstale":"2024-09-14T04:38:52.800824+0000","last_undegraded":"2024-09-14T04:38:52.800824+0000","last_fullsized":"2024-09-14T04:38:52.800824+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":2637,"log_dups_size":0,"ondisk_log_size":2637,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:23:11.179960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172818432,"num_objects":2637,"num_object_clones":0,"num_object_copies":7911,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2637,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2637,"num_write_kb":168768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"53.a","version":"1319'2508","reported_seq":2527,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:51.398867+0000","last_change":"2024-09-14T04:38:26.895294+0000","last_active":"2024-09-14T04:38:51.398867+0000","last_peered":"2024-09-14T04:38:51.398867+0000","last_clean":"2024-09-14T04:38:51.398867+0000","last_became_active":"2024-09-14T04:38:26.894942+0000","last_became_peered":"2024-09-14T04:38:26.894942+0000","last_unstale":"2024-09-14T04:38:51.398867+0000","last_undegraded":"2024-09-14T04:38:51.398867+0000","last_fullsized":"2024-09-14T04:38:51.398867+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":2508,"log_dups_size":0,"ondisk_log_size":2508,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:08:20.107915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164364288,"num_objects":2508,"num_object_clones":0,"num_object_copies":7524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2508,"num_write_kb":160512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"53.9","version":"1319'2577","reported_seq":2596,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:52.363862+0000","last_change":"2024-09-14T04:38:26.899027+0000","last_active":"2024-09-14T04:38:52.363862+0000","last_peered":"2024-09-14T04:38:52.363862+0000","last_clean":"2024-09-14T04:38:52.363862+0000","last_became_active":"2024-09-14T04:38:26.898732+0000","last_became_peered":"2024-09-14T04:38:26.898732+0000","last_unstale":"2024-09-14T04:38:52.363862+0000","last_undegraded":"2024-09-14T04:38:52.363862+0000","last_fullsized":"2024-09-14T04:38:52.363862+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":2577,"log_dups_size":0,"ondisk_log_size":2577,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:59:20.182013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168886272,"num_objects":2577,"num_object_clones":0,"num_object_copies":7731,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2577,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2577,"num_write_kb":164928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"53.8","version":"1319'2700","reported_seq":2713,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:52.786930+0000","last_change":"2024-09-14T04:38:26.897778+0000","last_active":"2024-09-14T04:38:52.786930+0000","last_peered":"2024-09-14T04:38:52.786930+0000","last_clean":"2024-09-14T04:38:52.786930+0000","last_became_active":"2024-09-14T04:38:26.897509+0000","last_became_peered":"2024-09-14T04:38:26.897509+0000","last_unstale":"2024-09-14T04:38:52.786930+0000","last_undegraded":"2024-09-14T04:38:52.786930+0000","last_fullsized":"2024-09-14T04:38:52.786930+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":2700,"log_dups_size":0,"ondisk_log_size":2700,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:51:52.846967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176947200,"num_objects":2700,"num_object_clones":0,"num_object_copies":8100,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2700,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2700,"num_write_kb":172800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"53.7","version":"1319'2317","reported_seq":2336,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:49.267383+0000","last_change":"2024-09-14T04:38:26.897393+0000","last_active":"2024-09-14T04:38:49.267383+0000","last_peered":"2024-09-14T04:38:49.267383+0000","last_clean":"2024-09-14T04:38:49.267383+0000","last_became_active":"2024-09-14T04:38:26.897127+0000","last_became_peered":"2024-09-14T04:38:26.897127+0000","last_unstale":"2024-09-14T04:38:49.267383+0000","last_undegraded":"2024-09-14T04:38:49.267383+0000","last_fullsized":"2024-09-14T04:38:49.267383+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":2317,"log_dups_size":0,"ondisk_log_size":2317,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:30.801131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151846912,"num_objects":2317,"num_object_clones":0,"num_object_copies":6951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2317,"num_write_kb":148288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"53.6","version":"1319'2536","reported_seq":2555,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:51.424822+0000","last_change":"2024-09-14T04:38:27.529579+0000","last_active":"2024-09-14T04:38:51.424822+0000","last_peered":"2024-09-14T04:38:51.424822+0000","last_clean":"2024-09-14T04:38:51.424822+0000","last_became_active":"2024-09-14T04:38:27.529245+0000","last_became_peered":"2024-09-14T04:38:27.529245+0000","last_unstale":"2024-09-14T04:38:51.424822+0000","last_undegraded":"2024-09-14T04:38:51.424822+0000","last_fullsized":"2024-09-14T04:38:51.424822+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":2536,"log_dups_size":0,"ondisk_log_size":2536,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:12:32.432630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166199296,"num_objects":2536,"num_object_clones":0,"num_object_copies":7608,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2536,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2536,"num_write_kb":162304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"53.5","version":"1319'2651","reported_seq":2670,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:52.361386+0000","last_change":"2024-09-14T04:38:26.897570+0000","last_active":"2024-09-14T04:38:52.361386+0000","last_peered":"2024-09-14T04:38:52.361386+0000","last_clean":"2024-09-14T04:38:52.361386+0000","last_became_active":"2024-09-14T04:38:26.897277+0000","last_became_peered":"2024-09-14T04:38:26.897277+0000","last_unstale":"2024-09-14T04:38:52.361386+0000","last_undegraded":"2024-09-14T04:38:52.361386+0000","last_fullsized":"2024-09-14T04:38:52.361386+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":2651,"log_dups_size":0,"ondisk_log_size":2651,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:11:45.104701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173735936,"num_objects":2651,"num_object_clones":0,"num_object_copies":7953,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2651,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2651,"num_write_kb":169664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"53.4","version":"1319'2250","reported_seq":2269,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:49.276653+0000","last_change":"2024-09-14T04:38:27.529509+0000","last_active":"2024-09-14T04:38:49.276653+0000","last_peered":"2024-09-14T04:38:49.276653+0000","last_clean":"2024-09-14T04:38:49.276653+0000","last_became_active":"2024-09-14T04:38:27.529180+0000","last_became_peered":"2024-09-14T04:38:27.529180+0000","last_unstale":"2024-09-14T04:38:49.276653+0000","last_undegraded":"2024-09-14T04:38:49.276653+0000","last_fullsized":"2024-09-14T04:38:49.276653+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":2250,"log_dups_size":0,"ondisk_log_size":2250,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:06:07.380505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147456000,"num_objects":2250,"num_object_clones":0,"num_object_copies":6750,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2250,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2250,"num_write_kb":144000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"53.3","version":"1319'2664","reported_seq":2677,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:52.790958+0000","last_change":"2024-09-14T04:38:26.894968+0000","last_active":"2024-09-14T04:38:52.790958+0000","last_peered":"2024-09-14T04:38:52.790958+0000","last_clean":"2024-09-14T04:38:52.790958+0000","last_became_active":"2024-09-14T04:38:26.894621+0000","last_became_peered":"2024-09-14T04:38:26.894621+0000","last_unstale":"2024-09-14T04:38:52.790958+0000","last_undegraded":"2024-09-14T04:38:52.790958+0000","last_fullsized":"2024-09-14T04:38:52.790958+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":2664,"log_dups_size":0,"ondisk_log_size":2664,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:15:43.660519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174587904,"num_objects":2664,"num_object_clones":0,"num_object_copies":7992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2664,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2664,"num_write_kb":170496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"53.2","version":"1319'2601","reported_seq":2614,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:52.790360+0000","last_change":"2024-09-14T04:38:26.897057+0000","last_active":"2024-09-14T04:38:52.790360+0000","last_peered":"2024-09-14T04:38:52.790360+0000","last_clean":"2024-09-14T04:38:52.790360+0000","last_became_active":"2024-09-14T04:38:26.896677+0000","last_became_peered":"2024-09-14T04:38:26.896677+0000","last_unstale":"2024-09-14T04:38:52.790360+0000","last_undegraded":"2024-09-14T04:38:52.790360+0000","last_fullsized":"2024-09-14T04:38:52.790360+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":2601,"log_dups_size":0,"ondisk_log_size":2601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:15.244766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":170459136,"num_objects":2601,"num_object_clones":0,"num_object_copies":7803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2601,"num_write_kb":166464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"53.1","version":"1319'2278","reported_seq":2296,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:49.268849+0000","last_change":"2024-09-14T04:38:26.897425+0000","last_active":"2024-09-14T04:38:49.268849+0000","last_peered":"2024-09-14T04:38:49.268849+0000","last_clean":"2024-09-14T04:38:49.268849+0000","last_became_active":"2024-09-14T04:38:26.897127+0000","last_became_peered":"2024-09-14T04:38:26.897127+0000","last_unstale":"2024-09-14T04:38:49.268849+0000","last_undegraded":"2024-09-14T04:38:49.268849+0000","last_fullsized":"2024-09-14T04:38:49.268849+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":2278,"log_dups_size":0,"ondisk_log_size":2278,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:46:36.167586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149291008,"num_objects":2278,"num_object_clones":0,"num_object_copies":6834,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2278,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2278,"num_write_kb":145792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"53.0","version":"1319'2620","reported_seq":2632,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:52.799475+0000","last_change":"2024-09-14T04:38:27.529766+0000","last_active":"2024-09-14T04:38:52.799475+0000","last_peered":"2024-09-14T04:38:52.799475+0000","last_clean":"2024-09-14T04:38:52.799475+0000","last_became_active":"2024-09-14T04:38:27.529414+0000","last_became_peered":"2024-09-14T04:38:27.529414+0000","last_unstale":"2024-09-14T04:38:52.799475+0000","last_undegraded":"2024-09-14T04:38:52.799475+0000","last_fullsized":"2024-09-14T04:38:52.799475+0000","mapping_epoch":1315,"log_start":"0'0","ondisk_log_start":"0'0","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":2620,"log_dups_size":0,"ondisk_log_size":2620,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:30:08.912265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171704320,"num_objects":2620,"num_object_clones":0,"num_object_copies":7860,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2620,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2620,"num_write_kb":167680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":1922,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.692708+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T04:38:35.692708+0000","last_peered":"2024-09-14T04:38:35.692708+0000","last_clean":"2024-09-14T04:38:35.692708+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T04:38:35.692708+0000","last_undegraded":"2024-09-14T04:38:35.692708+0000","last_fullsized":"2024-09-14T04:38:35.692708+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1280,"reported_epoch":1318,"state":"active+clean","last_fresh":"2024-09-14T04:38:33.274383+0000","last_change":"2024-09-14T03:53:45.836449+0000","last_active":"2024-09-14T04:38:33.274383+0000","last_peered":"2024-09-14T04:38:33.274383+0000","last_clean":"2024-09-14T04:38:33.274383+0000","last_became_active":"2024-09-14T03:53:45.836108+0000","last_became_peered":"2024-09-14T03:53:45.836108+0000","last_unstale":"2024-09-14T04:38:33.274383+0000","last_undegraded":"2024-09-14T04:38:33.274383+0000","last_fullsized":"2024-09-14T04:38:33.274383+0000","mapping_epoch":73,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":74,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:52:13.095225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1912,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.693007+0000","last_change":"2024-09-14T04:07:58.504829+0000","last_active":"2024-09-14T04:38:35.693007+0000","last_peered":"2024-09-14T04:38:35.693007+0000","last_clean":"2024-09-14T04:38:35.693007+0000","last_became_active":"2024-09-14T04:07:58.502623+0000","last_became_peered":"2024-09-14T04:07:58.502623+0000","last_unstale":"2024-09-14T04:38:35.693007+0000","last_undegraded":"2024-09-14T04:38:35.693007+0000","last_fullsized":"2024-09-14T04:38:35.693007+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:59.091993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1923,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.692151+0000","last_change":"2024-09-14T04:07:58.507125+0000","last_active":"2024-09-14T04:38:35.692151+0000","last_peered":"2024-09-14T04:38:35.692151+0000","last_clean":"2024-09-14T04:38:35.692151+0000","last_became_active":"2024-09-14T04:07:58.505701+0000","last_became_peered":"2024-09-14T04:07:58.505701+0000","last_unstale":"2024-09-14T04:38:35.692151+0000","last_undegraded":"2024-09-14T04:38:35.692151+0000","last_fullsized":"2024-09-14T04:38:35.692151+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:25:00.847961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1271,"reported_epoch":1318,"state":"active+clean","last_fresh":"2024-09-14T04:38:33.244513+0000","last_change":"2024-09-14T03:53:05.234444+0000","last_active":"2024-09-14T04:38:33.244513+0000","last_peered":"2024-09-14T04:38:33.244513+0000","last_clean":"2024-09-14T04:38:33.244513+0000","last_became_active":"2024-09-14T03:53:05.234115+0000","last_became_peered":"2024-09-14T03:53:05.234115+0000","last_unstale":"2024-09-14T04:38:33.244513+0000","last_undegraded":"2024-09-14T04:38:33.244513+0000","last_fullsized":"2024-09-14T04:38:33.244513+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:01:23.153937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1301,"reported_epoch":1318,"state":"active+clean","last_fresh":"2024-09-14T04:38:33.274550+0000","last_change":"2024-09-14T04:07:58.547135+0000","last_active":"2024-09-14T04:38:33.274550+0000","last_peered":"2024-09-14T04:38:33.274550+0000","last_clean":"2024-09-14T04:38:33.274550+0000","last_became_active":"2024-09-14T04:07:58.546751+0000","last_became_peered":"2024-09-14T04:07:58.546751+0000","last_unstale":"2024-09-14T04:38:33.274550+0000","last_undegraded":"2024-09-14T04:38:33.274550+0000","last_fullsized":"2024-09-14T04:38:33.274550+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:01.092698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1286,"reported_epoch":1318,"state":"active+clean","last_fresh":"2024-09-14T04:38:33.244441+0000","last_change":"2024-09-14T04:06:46.875795+0000","last_active":"2024-09-14T04:38:33.244441+0000","last_peered":"2024-09-14T04:38:33.244441+0000","last_clean":"2024-09-14T04:38:33.244441+0000","last_became_active":"2024-09-14T04:06:46.875395+0000","last_became_peered":"2024-09-14T04:06:46.875395+0000","last_unstale":"2024-09-14T04:38:33.244441+0000","last_undegraded":"2024-09-14T04:38:33.244441+0000","last_fullsized":"2024-09-14T04:38:33.244441+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:51:10.713063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1301,"reported_epoch":1318,"state":"active+clean","last_fresh":"2024-09-14T04:38:33.274748+0000","last_change":"2024-09-14T04:07:58.514349+0000","last_active":"2024-09-14T04:38:33.274748+0000","last_peered":"2024-09-14T04:38:33.274748+0000","last_clean":"2024-09-14T04:38:33.274748+0000","last_became_active":"2024-09-14T04:07:58.513485+0000","last_became_peered":"2024-09-14T04:07:58.513485+0000","last_unstale":"2024-09-14T04:38:33.274748+0000","last_undegraded":"2024-09-14T04:38:33.274748+0000","last_fullsized":"2024-09-14T04:38:33.274748+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:33:09.031813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":1905,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.691904+0000","last_change":"2024-09-14T04:05:54.428057+0000","last_active":"2024-09-14T04:38:35.691904+0000","last_peered":"2024-09-14T04:38:35.691904+0000","last_clean":"2024-09-14T04:38:35.691904+0000","last_became_active":"2024-09-14T04:05:54.414895+0000","last_became_peered":"2024-09-14T04:05:54.414895+0000","last_unstale":"2024-09-14T04:38:35.691904+0000","last_undegraded":"2024-09-14T04:38:35.691904+0000","last_fullsized":"2024-09-14T04:38:35.691904+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:45:15.752675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1919,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.561627+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T04:38:35.561627+0000","last_peered":"2024-09-14T04:38:35.561627+0000","last_clean":"2024-09-14T04:38:35.561627+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T04:38:35.561627+0000","last_undegraded":"2024-09-14T04:38:35.561627+0000","last_fullsized":"2024-09-14T04:38:35.561627+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1303,"reported_epoch":1318,"state":"active+clean","last_fresh":"2024-09-14T04:38:33.274702+0000","last_change":"2024-09-14T04:07:21.717071+0000","last_active":"2024-09-14T04:38:33.274702+0000","last_peered":"2024-09-14T04:38:33.274702+0000","last_clean":"2024-09-14T04:38:33.274702+0000","last_became_active":"2024-09-14T04:07:21.716215+0000","last_became_peered":"2024-09-14T04:07:21.716215+0000","last_unstale":"2024-09-14T04:38:33.274702+0000","last_undegraded":"2024-09-14T04:38:33.274702+0000","last_fullsized":"2024-09-14T04:38:33.274702+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:48:47.175490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1892,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.692930+0000","last_change":"2024-09-14T04:07:58.524371+0000","last_active":"2024-09-14T04:38:35.692930+0000","last_peered":"2024-09-14T04:38:35.692930+0000","last_clean":"2024-09-14T04:38:35.692930+0000","last_became_active":"2024-09-14T04:07:58.524088+0000","last_became_peered":"2024-09-14T04:07:58.524088+0000","last_unstale":"2024-09-14T04:38:35.692930+0000","last_undegraded":"2024-09-14T04:38:35.692930+0000","last_fullsized":"2024-09-14T04:38:35.692930+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:42.615569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1887,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.692812+0000","last_change":"2024-09-14T04:07:58.507553+0000","last_active":"2024-09-14T04:38:35.692812+0000","last_peered":"2024-09-14T04:38:35.692812+0000","last_clean":"2024-09-14T04:38:35.692812+0000","last_became_active":"2024-09-14T04:07:58.507139+0000","last_became_peered":"2024-09-14T04:07:58.507139+0000","last_unstale":"2024-09-14T04:38:35.692812+0000","last_undegraded":"2024-09-14T04:38:35.692812+0000","last_fullsized":"2024-09-14T04:38:35.692812+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:11:57.821769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1917,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.693100+0000","last_change":"2024-09-14T04:07:58.561490+0000","last_active":"2024-09-14T04:38:35.693100+0000","last_peered":"2024-09-14T04:38:35.693100+0000","last_clean":"2024-09-14T04:38:35.693100+0000","last_became_active":"2024-09-14T04:07:58.559841+0000","last_became_peered":"2024-09-14T04:07:58.559841+0000","last_unstale":"2024-09-14T04:38:35.693100+0000","last_undegraded":"2024-09-14T04:38:35.693100+0000","last_fullsized":"2024-09-14T04:38:35.693100+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:27:03.795368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1908,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.692199+0000","last_change":"2024-09-14T04:07:58.555731+0000","last_active":"2024-09-14T04:38:35.692199+0000","last_peered":"2024-09-14T04:38:35.692199+0000","last_clean":"2024-09-14T04:38:35.692199+0000","last_became_active":"2024-09-14T04:07:58.549576+0000","last_became_peered":"2024-09-14T04:07:58.549576+0000","last_unstale":"2024-09-14T04:38:35.692199+0000","last_undegraded":"2024-09-14T04:38:35.692199+0000","last_fullsized":"2024-09-14T04:38:35.692199+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:28:42.339589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1295,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.693149+0000","last_change":"2024-09-14T04:07:21.720797+0000","last_active":"2024-09-14T04:38:35.693149+0000","last_peered":"2024-09-14T04:38:35.693149+0000","last_clean":"2024-09-14T04:38:35.693149+0000","last_became_active":"2024-09-14T04:07:21.719331+0000","last_became_peered":"2024-09-14T04:07:21.719331+0000","last_unstale":"2024-09-14T04:38:35.693149+0000","last_undegraded":"2024-09-14T04:38:35.693149+0000","last_fullsized":"2024-09-14T04:38:35.693149+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:35.404023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1918,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.692373+0000","last_change":"2024-09-14T04:07:58.502533+0000","last_active":"2024-09-14T04:38:35.692373+0000","last_peered":"2024-09-14T04:38:35.692373+0000","last_clean":"2024-09-14T04:38:35.692373+0000","last_became_active":"2024-09-14T04:07:58.499849+0000","last_became_peered":"2024-09-14T04:07:58.499849+0000","last_unstale":"2024-09-14T04:38:35.692373+0000","last_undegraded":"2024-09-14T04:38:35.692373+0000","last_fullsized":"2024-09-14T04:38:35.692373+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:15:35.609746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1912,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.692893+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T04:38:35.692893+0000","last_peered":"2024-09-14T04:38:35.692893+0000","last_clean":"2024-09-14T04:38:35.692893+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T04:38:35.692893+0000","last_undegraded":"2024-09-14T04:38:35.692893+0000","last_fullsized":"2024-09-14T04:38:35.692893+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1894,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.692357+0000","last_change":"2024-09-14T04:07:58.507682+0000","last_active":"2024-09-14T04:38:35.692357+0000","last_peered":"2024-09-14T04:38:35.692357+0000","last_clean":"2024-09-14T04:38:35.692357+0000","last_became_active":"2024-09-14T04:07:58.507365+0000","last_became_peered":"2024-09-14T04:07:58.507365+0000","last_unstale":"2024-09-14T04:38:35.692357+0000","last_undegraded":"2024-09-14T04:38:35.692357+0000","last_fullsized":"2024-09-14T04:38:35.692357+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:14:47.294950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1283,"reported_epoch":1318,"state":"active+clean","last_fresh":"2024-09-14T04:38:33.244512+0000","last_change":"2024-09-14T04:06:46.875668+0000","last_active":"2024-09-14T04:38:33.244512+0000","last_peered":"2024-09-14T04:38:33.244512+0000","last_clean":"2024-09-14T04:38:33.244512+0000","last_became_active":"2024-09-14T04:06:46.875149+0000","last_became_peered":"2024-09-14T04:06:46.875149+0000","last_unstale":"2024-09-14T04:38:33.244512+0000","last_undegraded":"2024-09-14T04:38:33.244512+0000","last_fullsized":"2024-09-14T04:38:33.244512+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:45:59.190047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1284,"reported_epoch":1318,"state":"active+clean","last_fresh":"2024-09-14T04:38:33.242568+0000","last_change":"2024-09-14T04:07:58.510699+0000","last_active":"2024-09-14T04:38:33.242568+0000","last_peered":"2024-09-14T04:38:33.242568+0000","last_clean":"2024-09-14T04:38:33.242568+0000","last_became_active":"2024-09-14T04:07:58.506240+0000","last_became_peered":"2024-09-14T04:07:58.506240+0000","last_unstale":"2024-09-14T04:38:33.242568+0000","last_undegraded":"2024-09-14T04:38:33.242568+0000","last_fullsized":"2024-09-14T04:38:33.242568+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:27:18.051658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1919,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.692213+0000","last_change":"2024-09-14T04:07:21.687985+0000","last_active":"2024-09-14T04:38:35.692213+0000","last_peered":"2024-09-14T04:38:35.692213+0000","last_clean":"2024-09-14T04:38:35.692213+0000","last_became_active":"2024-09-14T04:07:21.687670+0000","last_became_peered":"2024-09-14T04:07:21.687670+0000","last_unstale":"2024-09-14T04:38:35.692213+0000","last_undegraded":"2024-09-14T04:38:35.692213+0000","last_fullsized":"2024-09-14T04:38:35.692213+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:53:58.295662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":1296,"reported_epoch":1318,"state":"active+clean","last_fresh":"2024-09-14T04:38:33.275075+0000","last_change":"2024-09-14T03:53:05.244055+0000","last_active":"2024-09-14T04:38:33.275075+0000","last_peered":"2024-09-14T04:38:33.275075+0000","last_clean":"2024-09-14T04:38:33.275075+0000","last_became_active":"2024-09-14T03:53:05.243752+0000","last_became_peered":"2024-09-14T03:53:05.243752+0000","last_unstale":"2024-09-14T04:38:33.275075+0000","last_undegraded":"2024-09-14T04:38:33.275075+0000","last_fullsized":"2024-09-14T04:38:33.275075+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:12:31.850064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1287,"reported_epoch":1318,"state":"active+clean","last_fresh":"2024-09-14T04:38:33.274849+0000","last_change":"2024-09-14T04:06:46.895670+0000","last_active":"2024-09-14T04:38:33.274849+0000","last_peered":"2024-09-14T04:38:33.274849+0000","last_clean":"2024-09-14T04:38:33.274849+0000","last_became_active":"2024-09-14T04:06:46.895370+0000","last_became_peered":"2024-09-14T04:06:46.895370+0000","last_unstale":"2024-09-14T04:38:33.274849+0000","last_undegraded":"2024-09-14T04:38:33.274849+0000","last_fullsized":"2024-09-14T04:38:33.274849+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:48:48.040529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1275,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.281733+0000","last_change":"2024-09-14T03:53:06.634043+0000","last_active":"2024-09-14T04:38:35.281733+0000","last_peered":"2024-09-14T04:38:35.281733+0000","last_clean":"2024-09-14T04:38:35.281733+0000","last_became_active":"2024-09-14T03:53:06.633638+0000","last_became_peered":"2024-09-14T03:53:06.633638+0000","last_unstale":"2024-09-14T04:38:35.281733+0000","last_undegraded":"2024-09-14T04:38:35.281733+0000","last_fullsized":"2024-09-14T04:38:35.281733+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:15:53.434064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1298,"reported_epoch":1318,"state":"active+clean","last_fresh":"2024-09-14T04:38:33.855026+0000","last_change":"2024-09-14T04:07:21.721242+0000","last_active":"2024-09-14T04:38:33.855026+0000","last_peered":"2024-09-14T04:38:33.855026+0000","last_clean":"2024-09-14T04:38:33.855026+0000","last_became_active":"2024-09-14T04:07:21.720084+0000","last_became_peered":"2024-09-14T04:07:21.720084+0000","last_unstale":"2024-09-14T04:38:33.855026+0000","last_undegraded":"2024-09-14T04:38:33.855026+0000","last_fullsized":"2024-09-14T04:38:33.855026+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:58:41.308643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":1919,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.693750+0000","last_change":"2024-09-14T04:06:46.932567+0000","last_active":"2024-09-14T04:38:35.693750+0000","last_peered":"2024-09-14T04:38:35.693750+0000","last_clean":"2024-09-14T04:38:35.693750+0000","last_became_active":"2024-09-14T04:06:46.930732+0000","last_became_peered":"2024-09-14T04:06:46.930732+0000","last_unstale":"2024-09-14T04:38:35.693750+0000","last_undegraded":"2024-09-14T04:38:35.693750+0000","last_fullsized":"2024-09-14T04:38:35.693750+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:10:02.729499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1886,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.692290+0000","last_change":"2024-09-14T04:07:58.555446+0000","last_active":"2024-09-14T04:38:35.692290+0000","last_peered":"2024-09-14T04:38:35.692290+0000","last_clean":"2024-09-14T04:38:35.692290+0000","last_became_active":"2024-09-14T04:07:58.548355+0000","last_became_peered":"2024-09-14T04:07:58.548355+0000","last_unstale":"2024-09-14T04:38:35.692290+0000","last_undegraded":"2024-09-14T04:38:35.692290+0000","last_fullsized":"2024-09-14T04:38:35.692290+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:56:28.426094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1923,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.693967+0000","last_change":"2024-09-14T04:06:46.915273+0000","last_active":"2024-09-14T04:38:35.693967+0000","last_peered":"2024-09-14T04:38:35.693967+0000","last_clean":"2024-09-14T04:38:35.693967+0000","last_became_active":"2024-09-14T04:06:46.914994+0000","last_became_peered":"2024-09-14T04:06:46.914994+0000","last_unstale":"2024-09-14T04:38:35.693967+0000","last_undegraded":"2024-09-14T04:38:35.693967+0000","last_fullsized":"2024-09-14T04:38:35.693967+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:48:58.802609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1887,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.693262+0000","last_change":"2024-09-14T04:07:58.564377+0000","last_active":"2024-09-14T04:38:35.693262+0000","last_peered":"2024-09-14T04:38:35.693262+0000","last_clean":"2024-09-14T04:38:35.693262+0000","last_became_active":"2024-09-14T04:07:58.546913+0000","last_became_peered":"2024-09-14T04:07:58.546913+0000","last_unstale":"2024-09-14T04:38:35.693262+0000","last_undegraded":"2024-09-14T04:38:35.693262+0000","last_fullsized":"2024-09-14T04:38:35.693262+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:05:53.637115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1905,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.693103+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T04:38:35.693103+0000","last_peered":"2024-09-14T04:38:35.693103+0000","last_clean":"2024-09-14T04:38:35.693103+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T04:38:35.693103+0000","last_undegraded":"2024-09-14T04:38:35.693103+0000","last_fullsized":"2024-09-14T04:38:35.693103+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1297,"reported_epoch":1318,"state":"active+clean","last_fresh":"2024-09-14T04:38:33.242690+0000","last_change":"2024-09-14T04:05:54.381035+0000","last_active":"2024-09-14T04:38:33.242690+0000","last_peered":"2024-09-14T04:38:33.242690+0000","last_clean":"2024-09-14T04:38:33.242690+0000","last_became_active":"2024-09-14T04:05:54.379861+0000","last_became_peered":"2024-09-14T04:05:54.379861+0000","last_unstale":"2024-09-14T04:38:33.242690+0000","last_undegraded":"2024-09-14T04:38:33.242690+0000","last_fullsized":"2024-09-14T04:38:33.242690+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:07:46.302925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1923,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.695314+0000","last_change":"2024-09-14T04:07:21.686880+0000","last_active":"2024-09-14T04:38:35.695314+0000","last_peered":"2024-09-14T04:38:35.695314+0000","last_clean":"2024-09-14T04:38:35.695314+0000","last_became_active":"2024-09-14T04:07:21.685558+0000","last_became_peered":"2024-09-14T04:07:21.685558+0000","last_unstale":"2024-09-14T04:38:35.695314+0000","last_undegraded":"2024-09-14T04:38:35.695314+0000","last_fullsized":"2024-09-14T04:38:35.695314+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:24:18.693428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1911,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.692074+0000","last_change":"2024-09-14T04:06:46.905931+0000","last_active":"2024-09-14T04:38:35.692074+0000","last_peered":"2024-09-14T04:38:35.692074+0000","last_clean":"2024-09-14T04:38:35.692074+0000","last_became_active":"2024-09-14T04:06:46.905666+0000","last_became_peered":"2024-09-14T04:06:46.905666+0000","last_unstale":"2024-09-14T04:38:35.692074+0000","last_undegraded":"2024-09-14T04:38:35.692074+0000","last_fullsized":"2024-09-14T04:38:35.692074+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:40:57.007948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1940,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.695557+0000","last_change":"2024-09-14T04:06:46.930482+0000","last_active":"2024-09-14T04:38:35.695557+0000","last_peered":"2024-09-14T04:38:35.695557+0000","last_clean":"2024-09-14T04:38:35.695557+0000","last_became_active":"2024-09-14T04:06:46.927372+0000","last_became_peered":"2024-09-14T04:06:46.927372+0000","last_unstale":"2024-09-14T04:38:35.695557+0000","last_undegraded":"2024-09-14T04:38:35.695557+0000","last_fullsized":"2024-09-14T04:38:35.695557+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:07:22.159128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1302,"reported_epoch":1318,"state":"active+clean","last_fresh":"2024-09-14T04:38:33.244602+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T04:38:33.244602+0000","last_peered":"2024-09-14T04:38:33.244602+0000","last_clean":"2024-09-14T04:38:33.244602+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T04:38:33.244602+0000","last_undegraded":"2024-09-14T04:38:33.244602+0000","last_fullsized":"2024-09-14T04:38:33.244602+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1933,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.695470+0000","last_change":"2024-09-14T04:07:58.510721+0000","last_active":"2024-09-14T04:38:35.695470+0000","last_peered":"2024-09-14T04:38:35.695470+0000","last_clean":"2024-09-14T04:38:35.695470+0000","last_became_active":"2024-09-14T04:07:58.510332+0000","last_became_peered":"2024-09-14T04:07:58.510332+0000","last_unstale":"2024-09-14T04:38:35.695470+0000","last_undegraded":"2024-09-14T04:38:35.695470+0000","last_fullsized":"2024-09-14T04:38:35.695470+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:07:03.926842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1290,"reported_epoch":1318,"state":"active+clean","last_fresh":"2024-09-14T04:38:33.274953+0000","last_change":"2024-09-14T04:05:54.426939+0000","last_active":"2024-09-14T04:38:33.274953+0000","last_peered":"2024-09-14T04:38:33.274953+0000","last_clean":"2024-09-14T04:38:33.274953+0000","last_became_active":"2024-09-14T04:05:54.426427+0000","last_became_peered":"2024-09-14T04:05:54.426427+0000","last_unstale":"2024-09-14T04:38:33.274953+0000","last_undegraded":"2024-09-14T04:38:33.274953+0000","last_fullsized":"2024-09-14T04:38:33.274953+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:53:55.841489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1301,"reported_epoch":1318,"state":"active+clean","last_fresh":"2024-09-14T04:38:33.244225+0000","last_change":"2024-09-14T04:06:46.875340+0000","last_active":"2024-09-14T04:38:33.244225+0000","last_peered":"2024-09-14T04:38:33.244225+0000","last_clean":"2024-09-14T04:38:33.244225+0000","last_became_active":"2024-09-14T04:06:46.874552+0000","last_became_peered":"2024-09-14T04:06:46.874552+0000","last_unstale":"2024-09-14T04:38:33.244225+0000","last_undegraded":"2024-09-14T04:38:33.244225+0000","last_fullsized":"2024-09-14T04:38:33.244225+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:52.257357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1919,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.694411+0000","last_change":"2024-09-14T04:06:46.917681+0000","last_active":"2024-09-14T04:38:35.694411+0000","last_peered":"2024-09-14T04:38:35.694411+0000","last_clean":"2024-09-14T04:38:35.694411+0000","last_became_active":"2024-09-14T04:06:46.917401+0000","last_became_peered":"2024-09-14T04:06:46.917401+0000","last_unstale":"2024-09-14T04:38:35.694411+0000","last_undegraded":"2024-09-14T04:38:35.694411+0000","last_fullsized":"2024-09-14T04:38:35.694411+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:04:39.949490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1923,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.695796+0000","last_change":"2024-09-14T04:07:21.685415+0000","last_active":"2024-09-14T04:38:35.695796+0000","last_peered":"2024-09-14T04:38:35.695796+0000","last_clean":"2024-09-14T04:38:35.695796+0000","last_became_active":"2024-09-14T04:07:21.683504+0000","last_became_peered":"2024-09-14T04:07:21.683504+0000","last_unstale":"2024-09-14T04:38:35.695796+0000","last_undegraded":"2024-09-14T04:38:35.695796+0000","last_fullsized":"2024-09-14T04:38:35.695796+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:21:08.806291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1926,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.561604+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T04:38:35.561604+0000","last_peered":"2024-09-14T04:38:35.561604+0000","last_clean":"2024-09-14T04:38:35.561604+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T04:38:35.561604+0000","last_undegraded":"2024-09-14T04:38:35.561604+0000","last_fullsized":"2024-09-14T04:38:35.561604+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1927,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.693191+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T04:38:35.693191+0000","last_peered":"2024-09-14T04:38:35.693191+0000","last_clean":"2024-09-14T04:38:35.693191+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T04:38:35.693191+0000","last_undegraded":"2024-09-14T04:38:35.693191+0000","last_fullsized":"2024-09-14T04:38:35.693191+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1915,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.693288+0000","last_change":"2024-09-14T04:07:58.521344+0000","last_active":"2024-09-14T04:38:35.693288+0000","last_peered":"2024-09-14T04:38:35.693288+0000","last_clean":"2024-09-14T04:38:35.693288+0000","last_became_active":"2024-09-14T04:07:58.520965+0000","last_became_peered":"2024-09-14T04:07:58.520965+0000","last_unstale":"2024-09-14T04:38:35.693288+0000","last_undegraded":"2024-09-14T04:38:35.693288+0000","last_fullsized":"2024-09-14T04:38:35.693288+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:58:46.029013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1313,"reported_epoch":1318,"state":"active+clean","last_fresh":"2024-09-14T04:38:33.242451+0000","last_change":"2024-09-14T04:06:46.857436+0000","last_active":"2024-09-14T04:38:33.242451+0000","last_peered":"2024-09-14T04:38:33.242451+0000","last_clean":"2024-09-14T04:38:33.242451+0000","last_became_active":"2024-09-14T04:06:46.857154+0000","last_became_peered":"2024-09-14T04:06:46.857154+0000","last_unstale":"2024-09-14T04:38:33.242451+0000","last_undegraded":"2024-09-14T04:38:33.242451+0000","last_fullsized":"2024-09-14T04:38:33.242451+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:53:28.772152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1270,"reported_epoch":1318,"state":"active+clean","last_fresh":"2024-09-14T04:38:33.854990+0000","last_change":"2024-09-14T03:53:05.551268+0000","last_active":"2024-09-14T04:38:33.854990+0000","last_peered":"2024-09-14T04:38:33.854990+0000","last_clean":"2024-09-14T04:38:33.854990+0000","last_became_active":"2024-09-14T03:53:05.551015+0000","last_became_peered":"2024-09-14T03:53:05.551015+0000","last_unstale":"2024-09-14T04:38:33.854990+0000","last_undegraded":"2024-09-14T04:38:33.854990+0000","last_fullsized":"2024-09-14T04:38:33.854990+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.905475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1897,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.692613+0000","last_change":"2024-09-14T04:07:58.503896+0000","last_active":"2024-09-14T04:38:35.692613+0000","last_peered":"2024-09-14T04:38:35.692613+0000","last_clean":"2024-09-14T04:38:35.692613+0000","last_became_active":"2024-09-14T04:07:58.499253+0000","last_became_peered":"2024-09-14T04:07:58.499253+0000","last_unstale":"2024-09-14T04:38:35.692613+0000","last_undegraded":"2024-09-14T04:38:35.692613+0000","last_fullsized":"2024-09-14T04:38:35.692613+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:48:27.543818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1300,"reported_epoch":1318,"state":"active+clean","last_fresh":"2024-09-14T04:38:33.244359+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T04:38:33.244359+0000","last_peered":"2024-09-14T04:38:33.244359+0000","last_clean":"2024-09-14T04:38:33.244359+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T04:38:33.244359+0000","last_undegraded":"2024-09-14T04:38:33.244359+0000","last_fullsized":"2024-09-14T04:38:33.244359+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":1615,"reported_epoch":1318,"state":"active+clean","last_fresh":"2024-09-14T04:38:33.242208+0000","last_change":"2024-09-14T04:05:54.389043+0000","last_active":"2024-09-14T04:38:33.242208+0000","last_peered":"2024-09-14T04:38:33.242208+0000","last_clean":"2024-09-14T04:38:33.242208+0000","last_became_active":"2024-09-14T04:05:54.388655+0000","last_became_peered":"2024-09-14T04:05:54.388655+0000","last_unstale":"2024-09-14T04:38:33.242208+0000","last_undegraded":"2024-09-14T04:38:33.242208+0000","last_fullsized":"2024-09-14T04:38:33.242208+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T03:49:18.237706+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:38:59.615597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1888,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.694329+0000","last_change":"2024-09-14T04:07:58.530406+0000","last_active":"2024-09-14T04:38:35.694329+0000","last_peered":"2024-09-14T04:38:35.694329+0000","last_clean":"2024-09-14T04:38:35.694329+0000","last_became_active":"2024-09-14T04:07:58.530078+0000","last_became_peered":"2024-09-14T04:07:58.530078+0000","last_unstale":"2024-09-14T04:38:35.694329+0000","last_undegraded":"2024-09-14T04:38:35.694329+0000","last_fullsized":"2024-09-14T04:38:35.694329+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:54:53.524001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1896,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.692090+0000","last_change":"2024-09-14T04:07:58.555717+0000","last_active":"2024-09-14T04:38:35.692090+0000","last_peered":"2024-09-14T04:38:35.692090+0000","last_clean":"2024-09-14T04:38:35.692090+0000","last_became_active":"2024-09-14T04:07:58.548894+0000","last_became_peered":"2024-09-14T04:07:58.548894+0000","last_unstale":"2024-09-14T04:38:35.692090+0000","last_undegraded":"2024-09-14T04:38:35.692090+0000","last_fullsized":"2024-09-14T04:38:35.692090+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:42.061551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1288,"reported_epoch":1318,"state":"active+clean","last_fresh":"2024-09-14T04:38:33.274973+0000","last_change":"2024-09-14T04:06:46.894741+0000","last_active":"2024-09-14T04:38:33.274973+0000","last_peered":"2024-09-14T04:38:33.274973+0000","last_clean":"2024-09-14T04:38:33.274973+0000","last_became_active":"2024-09-14T04:06:46.894445+0000","last_became_peered":"2024-09-14T04:06:46.894445+0000","last_unstale":"2024-09-14T04:38:33.274973+0000","last_undegraded":"2024-09-14T04:38:33.274973+0000","last_fullsized":"2024-09-14T04:38:33.274973+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:52:00.055271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1290,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.281898+0000","last_change":"2024-09-14T04:07:21.715462+0000","last_active":"2024-09-14T04:38:35.281898+0000","last_peered":"2024-09-14T04:38:35.281898+0000","last_clean":"2024-09-14T04:38:35.281898+0000","last_became_active":"2024-09-14T04:07:21.713032+0000","last_became_peered":"2024-09-14T04:07:21.713032+0000","last_unstale":"2024-09-14T04:38:35.281898+0000","last_undegraded":"2024-09-14T04:38:35.281898+0000","last_fullsized":"2024-09-14T04:38:35.281898+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:37:05.545164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1303,"reported_epoch":1318,"state":"active+clean","last_fresh":"2024-09-14T04:38:33.274750+0000","last_change":"2024-09-14T04:07:21.718037+0000","last_active":"2024-09-14T04:38:33.274750+0000","last_peered":"2024-09-14T04:38:33.274750+0000","last_clean":"2024-09-14T04:38:33.274750+0000","last_became_active":"2024-09-14T04:07:21.717700+0000","last_became_peered":"2024-09-14T04:07:21.717700+0000","last_unstale":"2024-09-14T04:38:33.274750+0000","last_undegraded":"2024-09-14T04:38:33.274750+0000","last_fullsized":"2024-09-14T04:38:33.274750+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:47:33.483785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1286,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.692851+0000","last_change":"2024-09-14T04:06:46.868780+0000","last_active":"2024-09-14T04:38:35.692851+0000","last_peered":"2024-09-14T04:38:35.692851+0000","last_clean":"2024-09-14T04:38:35.692851+0000","last_became_active":"2024-09-14T04:06:46.868483+0000","last_became_peered":"2024-09-14T04:06:46.868483+0000","last_unstale":"2024-09-14T04:38:35.692851+0000","last_undegraded":"2024-09-14T04:38:35.692851+0000","last_fullsized":"2024-09-14T04:38:35.692851+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:03.133158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1288,"reported_epoch":1318,"state":"active+clean","last_fresh":"2024-09-14T04:38:33.274876+0000","last_change":"2024-09-14T04:07:58.515859+0000","last_active":"2024-09-14T04:38:33.274876+0000","last_peered":"2024-09-14T04:38:33.274876+0000","last_clean":"2024-09-14T04:38:33.274876+0000","last_became_active":"2024-09-14T04:07:58.515594+0000","last_became_peered":"2024-09-14T04:07:58.515594+0000","last_unstale":"2024-09-14T04:38:33.274876+0000","last_undegraded":"2024-09-14T04:38:33.274876+0000","last_fullsized":"2024-09-14T04:38:33.274876+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:45.966798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":1912,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.692582+0000","last_change":"2024-09-14T04:07:58.509833+0000","last_active":"2024-09-14T04:38:35.692582+0000","last_peered":"2024-09-14T04:38:35.692582+0000","last_clean":"2024-09-14T04:38:35.692582+0000","last_became_active":"2024-09-14T04:07:58.509198+0000","last_became_peered":"2024-09-14T04:07:58.509198+0000","last_unstale":"2024-09-14T04:38:35.692582+0000","last_undegraded":"2024-09-14T04:38:35.692582+0000","last_fullsized":"2024-09-14T04:38:35.692582+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:24:40.193090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1915,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.691808+0000","last_change":"2024-09-14T04:06:46.925323+0000","last_active":"2024-09-14T04:38:35.691808+0000","last_peered":"2024-09-14T04:38:35.691808+0000","last_clean":"2024-09-14T04:38:35.691808+0000","last_became_active":"2024-09-14T04:06:46.925043+0000","last_became_peered":"2024-09-14T04:06:46.925043+0000","last_unstale":"2024-09-14T04:38:35.691808+0000","last_undegraded":"2024-09-14T04:38:35.691808+0000","last_fullsized":"2024-09-14T04:38:35.691808+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:39.895453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1286,"reported_epoch":1318,"state":"active+clean","last_fresh":"2024-09-14T04:38:33.244457+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T04:38:33.244457+0000","last_peered":"2024-09-14T04:38:33.244457+0000","last_clean":"2024-09-14T04:38:33.244457+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T04:38:33.244457+0000","last_undegraded":"2024-09-14T04:38:33.244457+0000","last_fullsized":"2024-09-14T04:38:33.244457+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1285,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.692696+0000","last_change":"2024-09-14T04:06:46.869868+0000","last_active":"2024-09-14T04:38:35.692696+0000","last_peered":"2024-09-14T04:38:35.692696+0000","last_clean":"2024-09-14T04:38:35.692696+0000","last_became_active":"2024-09-14T04:06:46.869126+0000","last_became_peered":"2024-09-14T04:06:46.869126+0000","last_unstale":"2024-09-14T04:38:35.692696+0000","last_undegraded":"2024-09-14T04:38:35.692696+0000","last_fullsized":"2024-09-14T04:38:35.692696+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:41.150138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1269,"reported_epoch":1318,"state":"active+clean","last_fresh":"2024-09-14T04:38:33.242618+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T04:38:33.242618+0000","last_peered":"2024-09-14T04:38:33.242618+0000","last_clean":"2024-09-14T04:38:33.242618+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T04:38:33.242618+0000","last_undegraded":"2024-09-14T04:38:33.242618+0000","last_fullsized":"2024-09-14T04:38:33.242618+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1304,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.281812+0000","last_change":"2024-09-14T04:07:58.508202+0000","last_active":"2024-09-14T04:38:35.281812+0000","last_peered":"2024-09-14T04:38:35.281812+0000","last_clean":"2024-09-14T04:38:35.281812+0000","last_became_active":"2024-09-14T04:07:58.507929+0000","last_became_peered":"2024-09-14T04:07:58.507929+0000","last_unstale":"2024-09-14T04:38:35.281812+0000","last_undegraded":"2024-09-14T04:38:35.281812+0000","last_fullsized":"2024-09-14T04:38:35.281812+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:17:58.160600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1285,"reported_epoch":1318,"state":"active+clean","last_fresh":"2024-09-14T04:38:33.242703+0000","last_change":"2024-09-14T04:06:46.858890+0000","last_active":"2024-09-14T04:38:33.242703+0000","last_peered":"2024-09-14T04:38:33.242703+0000","last_clean":"2024-09-14T04:38:33.242703+0000","last_became_active":"2024-09-14T04:06:46.858569+0000","last_became_peered":"2024-09-14T04:06:46.858569+0000","last_unstale":"2024-09-14T04:38:33.242703+0000","last_undegraded":"2024-09-14T04:38:33.242703+0000","last_fullsized":"2024-09-14T04:38:33.242703+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:25:46.400115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1271,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.282229+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T04:38:35.282229+0000","last_peered":"2024-09-14T04:38:35.282229+0000","last_clean":"2024-09-14T04:38:35.282229+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T04:38:35.282229+0000","last_undegraded":"2024-09-14T04:38:35.282229+0000","last_fullsized":"2024-09-14T04:38:35.282229+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1284,"reported_epoch":1318,"state":"active+clean","last_fresh":"2024-09-14T04:38:33.242699+0000","last_change":"2024-09-14T04:05:54.419435+0000","last_active":"2024-09-14T04:38:33.242699+0000","last_peered":"2024-09-14T04:38:33.242699+0000","last_clean":"2024-09-14T04:38:33.242699+0000","last_became_active":"2024-09-14T04:05:54.419094+0000","last_became_peered":"2024-09-14T04:05:54.419094+0000","last_unstale":"2024-09-14T04:38:33.242699+0000","last_undegraded":"2024-09-14T04:38:33.242699+0000","last_fullsized":"2024-09-14T04:38:33.242699+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:00.162629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1292,"reported_epoch":1318,"state":"active+clean","last_fresh":"2024-09-14T04:38:33.855278+0000","last_change":"2024-09-14T04:05:54.418813+0000","last_active":"2024-09-14T04:38:33.855278+0000","last_peered":"2024-09-14T04:38:33.855278+0000","last_clean":"2024-09-14T04:38:33.855278+0000","last_became_active":"2024-09-14T04:05:54.417584+0000","last_became_peered":"2024-09-14T04:05:54.417584+0000","last_unstale":"2024-09-14T04:38:33.855278+0000","last_undegraded":"2024-09-14T04:38:33.855278+0000","last_fullsized":"2024-09-14T04:38:33.855278+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:13:16.209456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1892,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.693175+0000","last_change":"2024-09-14T04:07:58.565442+0000","last_active":"2024-09-14T04:38:35.693175+0000","last_peered":"2024-09-14T04:38:35.693175+0000","last_clean":"2024-09-14T04:38:35.693175+0000","last_became_active":"2024-09-14T04:07:58.552289+0000","last_became_peered":"2024-09-14T04:07:58.552289+0000","last_unstale":"2024-09-14T04:38:35.693175+0000","last_undegraded":"2024-09-14T04:38:35.693175+0000","last_fullsized":"2024-09-14T04:38:35.693175+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:48:02.364960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1267,"reported_epoch":1318,"state":"active+clean","last_fresh":"2024-09-14T04:38:33.242348+0000","last_change":"2024-09-14T03:53:05.244003+0000","last_active":"2024-09-14T04:38:33.242348+0000","last_peered":"2024-09-14T04:38:33.242348+0000","last_clean":"2024-09-14T04:38:33.242348+0000","last_became_active":"2024-09-14T03:53:05.240743+0000","last_became_peered":"2024-09-14T03:53:05.240743+0000","last_unstale":"2024-09-14T04:38:33.242348+0000","last_undegraded":"2024-09-14T04:38:33.242348+0000","last_fullsized":"2024-09-14T04:38:33.242348+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:30.000189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1913,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.693005+0000","last_change":"2024-09-14T04:07:58.560880+0000","last_active":"2024-09-14T04:38:35.693005+0000","last_peered":"2024-09-14T04:38:35.693005+0000","last_clean":"2024-09-14T04:38:35.693005+0000","last_became_active":"2024-09-14T04:07:58.560132+0000","last_became_peered":"2024-09-14T04:07:58.560132+0000","last_unstale":"2024-09-14T04:38:35.693005+0000","last_undegraded":"2024-09-14T04:38:35.693005+0000","last_fullsized":"2024-09-14T04:38:35.693005+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:46:00.287050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1913,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.691927+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T04:38:35.691927+0000","last_peered":"2024-09-14T04:38:35.691927+0000","last_clean":"2024-09-14T04:38:35.691927+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T04:38:35.691927+0000","last_undegraded":"2024-09-14T04:38:35.691927+0000","last_fullsized":"2024-09-14T04:38:35.691927+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1283,"reported_epoch":1318,"state":"active+clean","last_fresh":"2024-09-14T04:38:33.244533+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T04:38:33.244533+0000","last_peered":"2024-09-14T04:38:33.244533+0000","last_clean":"2024-09-14T04:38:33.244533+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T04:38:33.244533+0000","last_undegraded":"2024-09-14T04:38:33.244533+0000","last_fullsized":"2024-09-14T04:38:33.244533+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1284,"reported_epoch":1318,"state":"active+clean","last_fresh":"2024-09-14T04:38:33.242410+0000","last_change":"2024-09-14T04:07:21.718847+0000","last_active":"2024-09-14T04:38:33.242410+0000","last_peered":"2024-09-14T04:38:33.242410+0000","last_clean":"2024-09-14T04:38:33.242410+0000","last_became_active":"2024-09-14T04:07:21.718504+0000","last_became_peered":"2024-09-14T04:07:21.718504+0000","last_unstale":"2024-09-14T04:38:33.242410+0000","last_undegraded":"2024-09-14T04:38:33.242410+0000","last_fullsized":"2024-09-14T04:38:33.242410+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:37:34.405765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1909,"reported_epoch":1319,"state":"active+clean","last_fresh":"2024-09-14T04:38:35.692812+0000","last_change":"2024-09-14T04:07:58.537946+0000","last_active":"2024-09-14T04:38:35.692812+0000","last_peered":"2024-09-14T04:38:35.692812+0000","last_clean":"2024-09-14T04:38:35.692812+0000","last_became_active":"2024-09-14T04:07:58.536034+0000","last_became_peered":"2024-09-14T04:07:58.536034+0000","last_unstale":"2024-09-14T04:38:35.692812+0000","last_undegraded":"2024-09-14T04:38:35.692812+0000","last_fullsized":"2024-09-14T04:38:35.692812+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:17:18.200470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":53,"num_pg":16,"stat_sum":{"num_bytes":2652241920,"num_objects":40470,"num_object_clones":0,"num_object_copies":121410,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":40470,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":40470,"num_write_kb":2590080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7925202944,"data_stored":7925202944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":40470,"ondisk_log_size":40470,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201855,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":304908,"kb_used_data":187716,"kb_used_omap":1,"kb_used_meta":117182,"kb_avail":93440244,"statfs":{"total":95995035648,"available":95682809856,"internally_reserved":0,"allocated":192221184,"data_stored":187787869,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":119994827},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463491,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":770436,"kb_used_data":653244,"kb_used_omap":1,"kb_used_meta":117182,"kb_avail":92974716,"statfs":{"total":95995035648,"available":95206109184,"internally_reserved":0,"allocated":668921856,"data_stored":664496606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":119994823},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725128,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1289912,"kb_used_data":1172720,"kb_used_omap":1,"kb_used_meta":117182,"kb_avail":92455240,"statfs":{"total":95995035648,"available":94674165760,"internally_reserved":0,"allocated":1200865280,"data_stored":1196447600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":119994822},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986766,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1941636,"kb_used_data":1802300,"kb_used_omap":1,"kb_used_meta":139326,"kb_avail":91803516,"statfs":{"total":95995035648,"available":94006800384,"internally_reserved":0,"allocated":1845555200,"data_stored":1841129463,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":142670278},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772074869,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":894644,"kb_used_data":808428,"kb_used_omap":71,"kb_used_meta":86136,"kb_avail":92850508,"statfs":{"total":95995035648,"available":95078920192,"internally_reserved":0,"allocated":827830272,"data_stored":822927787,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":88204214},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707303804,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":683000,"kb_used_data":592624,"kb_used_omap":68,"kb_used_meta":90299,"kb_avail":93062152,"statfs":{"total":95995035648,"available":95295643648,"internally_reserved":0,"allocated":606846976,"data_stored":601940921,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":92466384},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872728963,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1512372,"kb_used_data":1426220,"kb_used_omap":67,"kb_used_meta":86076,"kb_avail":92232780,"statfs":{"total":95995035648,"available":94446366720,"internally_reserved":0,"allocated":1460449280,"data_stored":1455546724,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":88142181},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,6,0,0,0,0,7],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056014,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1064316,"kb_used_data":970036,"kb_used_omap":74,"kb_used_meta":94197,"kb_avail":92680836,"statfs":{"total":95995035648,"available":94905176064,"internally_reserved":0,"allocated":993316864,"data_stored":988407245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":96458421},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1032781824,"data_stored":1032781824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1480785920,"data_stored":1480785920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593428480,"data_stored":593428480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":818610176,"data_stored":818610176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1915224064,"data_stored":1915224064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1212874752,"data_stored":1212874752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":689438720,"data_stored":689438720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":182059008,"data_stored":182059008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:38:57.012 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd pool set unique_pool_0 pgp_num 16 2024-09-14T04:38:57.092 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:57.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:57.139 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:57.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:57.241 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:57.265 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:57.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:57.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:57.367 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:38:57.409 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:57.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:57.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:38:57.507 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:57 smithi039 ceph-mon[111235]: pgmap v2169: 89 pgs: 89 active+clean; 2.6 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s 2024-09-14T04:38:57.508 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:57 smithi039 ceph-mon[106470]: pgmap v2169: 89 pgs: 89 active+clean; 2.6 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s 2024-09-14T04:38:57.511 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:57.542 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:57.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:57.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:57.644 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:57.674 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:57.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:57.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:57.694 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 22 16 51329 51313 145.735 109.438 0.0167436 0.00685647 2024-09-14T04:38:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:57 smithi129 ceph-mon[101316]: pgmap v2169: 89 pgs: 89 active+clean; 2.6 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s 2024-09-14T04:38:57.776 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:57.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:57.812 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:57.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:57.913 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:57.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:57.968 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:57.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:58.069 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:58.100 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:58.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:58.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:58.202 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:58.227 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:58.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:58.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:58.291 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:38:58.291 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:38:58.329 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:58.332 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:38:58.332 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:38:58.371 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:58.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:58.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:58.373 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:38:58.373 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:38:58.405 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:38:58.406 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:38:58.416 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:38:58.439 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:38:58.439 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:38:58.473 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:58.474 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:38:58.474 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:38:58.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:58.505 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:58.506 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:58.509 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:38:58.509 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:38:58.546 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:38:58.546 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:38:58.606 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:38:58.638 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:58.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:58.639 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:38:58.651 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:58 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2470001089' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:38:58.652 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:58 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2470001089' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:38:58.695 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 23 16 52969 52953 143.854 102.5 0.0165895 0.00694524 2024-09-14T04:38:58.739 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:58.767 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:58.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:58.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:58.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:58 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2470001089' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:38:58.870 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:58.912 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:58.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:58.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:59.014 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:38:59.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:38:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:38:58] "GET /metrics HTTP/1.1" 200 36732 "" "Prometheus/2.51.0" 2024-09-14T04:38:59.050 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:59.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:59.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:38:59.151 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:38:59.186 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:59.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:59.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:38:59.288 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:38:59.325 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:59.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:59.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:38:59.426 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:59.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:59.466 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:59.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:59.567 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:38:59.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:59.606 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:59.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:38:59.671 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:38:59 smithi129 ceph-mon[101316]: pgmap v2170: 89 pgs: 89 active+clean; 2.8 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2024-09-14T04:38:59.694 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 24 16 54714 54698 142.404 109.062 0.00372359 0.00701717 2024-09-14T04:38:59.707 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:38:59.753 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:38:59 smithi039 ceph-mon[106470]: pgmap v2170: 89 pgs: 89 active+clean; 2.8 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2024-09-14T04:38:59.753 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:38:59 smithi039 ceph-mon[111235]: pgmap v2170: 89 pgs: 89 active+clean; 2.8 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2024-09-14T04:38:59.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:59.755 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:38:59.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:38:59.856 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:38:59.890 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:38:59.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:38:59.892 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:38:59.992 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:00.020 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:00.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:00.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:00.122 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:00.167 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:00.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:00.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:00.257 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:39:00.269 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:00.295 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:00.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:00.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:00.311 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:39:00.311 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:39:00.333 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:39:00.397 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:00.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:00.449 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:00.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:00.550 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:00.584 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:00.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:00.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:00.686 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:00.694 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 25 16 56344 56328 140.783 101.875 0.00718457 0.00709753 2024-09-14T04:39:00.697 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:00 smithi039 ceph-mon[106470]: pgmap v2171: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-09-14T04:39:00.698 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:00 smithi039 ceph-mon[111235]: pgmap v2171: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-09-14T04:39:00.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:00.719 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:00.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:00.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:00 smithi129 ceph-mon[101316]: pgmap v2171: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s 2024-09-14T04:39:00.820 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:00.856 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:00.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:00.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:00.957 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:00.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:00.998 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:00.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:01.099 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:01.136 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:01.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:01.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:01.238 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:01.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:01.282 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:01.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:01.345 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-14T04:39:01.383 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:01.413 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:01.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:01.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:01.446 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:39:01.514 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:01.572 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:01.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:01.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:01.674 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:01.695 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 26 16 57933 57917 139.187 99.3125 0.0168537 0.00717902 2024-09-14T04:39:01.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:01.718 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:01.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:01.819 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:01.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:01.851 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:01.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:01.951 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:01.983 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:01.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:01.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:02.087 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:02.129 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:02.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:02.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:02.232 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:02.260 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:02.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:02.261 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:02.362 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:02.402 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:02.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:02.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:02.504 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:02.545 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:02.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:02.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:02.647 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:02.679 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:02.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:02.680 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:02.690 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:02 smithi039 ceph-mon[106470]: pgmap v2172: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-09-14T04:39:02.692 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:02 smithi039 ceph-mon[111235]: pgmap v2172: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-09-14T04:39:02.695 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 27 15 59639 59624 137.984 106.688 0.00859789 0.00724183 2024-09-14T04:39:02.781 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:02.826 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:02.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:02.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:02.929 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:02.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:02.963 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:02.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:03.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:02 smithi129 ceph-mon[101316]: pgmap v2172: 89 pgs: 89 active+clean; 3.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-09-14T04:39:03.064 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:03.109 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:03.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:03.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:03.211 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:03.242 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:03.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:03.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:03.344 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:03.386 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:03.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:03.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:03.488 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:03.526 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:03.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:03.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:03.628 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:03.661 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:03.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:03.662 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:03.695 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 28 15 61215 61200 136.573 98.5 0.0104497 0.00731687 2024-09-14T04:39:03.762 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:03.789 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:03.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:03.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:03.891 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:03.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:03.924 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:03.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:04.025 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:04.054 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:04.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:04.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:04.156 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:04.184 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:04.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:04.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:04.286 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:04.326 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:04.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:04.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:04.428 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:04.462 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:04.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:04.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:04.564 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:04.596 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:04.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:04.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:04.632 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:39:04.632 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:39:04.698 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:04.699 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 29 16 62746 62730 135.142 95.625 0.014737 0.00739408 2024-09-14T04:39:04.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:04.753 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:04.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:04.854 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:04.897 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:04.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:04.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:04.930 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:04 smithi039 ceph-mon[106470]: pgmap v2173: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2024-09-14T04:39:04.931 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:04 smithi039 ceph-mon[111235]: pgmap v2173: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2024-09-14T04:39:05.000 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:05.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:04 smithi129 ceph-mon[101316]: pgmap v2173: 89 pgs: 89 active+clean; 3.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2024-09-14T04:39:05.042 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:05.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:05.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:05.144 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:05.180 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:05.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:05.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:05.282 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:05.311 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:05.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:05.312 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:05.413 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:05.440 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:05.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:05.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:05.542 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:05.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:05.578 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:05.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:05.607 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is unset 2024-09-14T04:39:05.679 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:05.700 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 30 16 64269 64253 133.807 95.1875 0.0136095 0.00746915 2024-09-14T04:39:05.720 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:05.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:05.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:05.816 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:05 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2858321642' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:39:05.816 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:05 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:39:05.816 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:05 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2858321642' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:39:05.816 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:05 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:39:05.822 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:05.856 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:05.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:05.857 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:05.958 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:05.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:05.987 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:05.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:06.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:05 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2858321642' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:39:06.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:05 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:39:06.088 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:06.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:06.124 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:06.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:06.225 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:06.260 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:06.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:06.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:06.362 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:06.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:06.391 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:06.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:06.493 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:06.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:06.539 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:06.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:06.640 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:06.669 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:06.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:06.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:06.700 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 31 16 65817 65801 132.612 96.75 0.00401796 0.00753559 2024-09-14T04:39:06.770 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:06.783 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:06 smithi039 ceph-mon[106470]: pgmap v2174: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-09-14T04:39:06.783 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:06 smithi039 ceph-mon[106470]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:39:06.783 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:06 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T04:39:06.783 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:06 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:39:06.783 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:06 smithi039 ceph-mon[106470]: osdmap e1321: 8 total, 8 up, 8 in 2024-09-14T04:39:06.783 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:06 smithi039 ceph-mon[111235]: pgmap v2174: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-09-14T04:39:06.783 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:06 smithi039 ceph-mon[111235]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:39:06.783 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:06 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T04:39:06.783 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:06 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:39:06.783 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:06 smithi039 ceph-mon[111235]: osdmap e1321: 8 total, 8 up, 8 in 2024-09-14T04:39:06.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:06.806 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:06.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:06.907 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:06.951 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:06.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:06.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:07.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:06 smithi129 ceph-mon[101316]: pgmap v2174: 89 pgs: 89 active+clean; 3.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s 2024-09-14T04:39:07.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:06 smithi129 ceph-mon[101316]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:39:07.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:06 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T04:39:07.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:06 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:39:07.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:06 smithi129 ceph-mon[101316]: osdmap e1321: 8 total, 8 up, 8 in 2024-09-14T04:39:07.053 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:07.089 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:07.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:07.091 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:07.191 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:07.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:07.239 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:07.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:07.341 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:07.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:07.380 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:07.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:07.481 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:07.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:07.516 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:07.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:07.618 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:07.640 INFO:teuthology.orchestra.run.smithi039.stderr:set pool 53 pgp_num to 16 2024-09-14T04:39:07.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:07.670 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:07.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:07.700 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 32 16 67221 67205 131.21 87.75 0.0136742 0.00761639 2024-09-14T04:39:07.770 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:07.794 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:07 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2675273742' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"} : dispatch 2024-09-14T04:39:07.794 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:07 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"} : dispatch 2024-09-14T04:39:07.794 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:39:07.794 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:39:07.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:07 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2675273742' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"} : dispatch 2024-09-14T04:39:07.794 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:07 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"} : dispatch 2024-09-14T04:39:07.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:39:07.795 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:39:07.814 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:07.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:07.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:07.915 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:07.951 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:07.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:07.952 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:07 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2675273742' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"} : dispatch 2024-09-14T04:39:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:07 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"} : dispatch 2024-09-14T04:39:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:39:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:39:08.053 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:08.090 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:08.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:08.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:08.192 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:08.234 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:08.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:08.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:08.336 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:08.380 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:08.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:08.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:08.482 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:08.520 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:08.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:08.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:08.621 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:08.656 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:08.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:08.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:08.700 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 33 15 68521 68506 129.698 81.3125 0.0102506 0.00770447 2024-09-14T04:39:08.758 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:08.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:08 smithi039 ceph-mon[106470]: pgmap v2176: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-14T04:39:08.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:08 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]': finished 2024-09-14T04:39:08.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:08 smithi039 ceph-mon[106470]: osdmap e1322: 8 total, 8 up, 8 in 2024-09-14T04:39:08.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:39:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:39:08] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.51.0" 2024-09-14T04:39:08.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:08 smithi039 ceph-mon[111235]: pgmap v2176: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-14T04:39:08.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:08 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]': finished 2024-09-14T04:39:08.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:08 smithi039 ceph-mon[111235]: osdmap e1322: 8 total, 8 up, 8 in 2024-09-14T04:39:08.797 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:08.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:08.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:08.899 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:08.928 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:08.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:08.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:09.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:08 smithi129 ceph-mon[101316]: pgmap v2176: 89 pgs: 89 active+clean; 3.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-14T04:39:09.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:08 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]': finished 2024-09-14T04:39:09.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:08 smithi129 ceph-mon[101316]: osdmap e1322: 8 total, 8 up, 8 in 2024-09-14T04:39:09.030 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:09.058 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:09.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:09.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:09.161 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:09.196 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:09.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:09.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:09.298 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:09.329 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:09.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:09.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:09.431 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:09.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:09.473 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:09.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:09.574 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:09.602 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:09.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:09.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:09.700 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 34 16 69812 69796 128.255 80.625 0.00968126 0.00779129 2024-09-14T04:39:09.704 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:09.743 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:09.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:09.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:09.845 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:09.881 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:09.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:09.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:09.984 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:10.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:10.023 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:10.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:10.124 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:10.160 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:10.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:10.161 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:10.261 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:10.307 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:10.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:10.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:10.409 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:10.444 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:10.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:10.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:10.546 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:10.590 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:10.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:10.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:10.682 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:39:10.682 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [], 2024-09-14T04:39:10.683 INFO:teuthology.orchestra.run.smithi039.stdout: "complaint_time": 30, 2024-09-14T04:39:10.683 INFO:teuthology.orchestra.run.smithi039.stdout: "num_blocked_ops": 0 2024-09-14T04:39:10.683 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:39:10.692 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:10.700 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 35 16 71223 71207 127.111 88.1875 0.00674499 0.00786248 2024-09-14T04:39:10.729 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:39:10.729 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:39:10.730 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:39:10.730 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:39:10.730 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:39:10.730 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:39:10.730 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:39:10.731 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:39:10.732 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:10.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:10.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:10.769 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:39:10.770 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:39:10.770 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:10 smithi039 ceph-mon[106470]: pgmap v2178: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-09-14T04:39:10.770 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:10 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2632896754' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:39:10.770 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:10 smithi039 ceph-mon[111235]: pgmap v2178: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-09-14T04:39:10.770 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:10 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2632896754' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:39:10.802 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:39:10.803 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:39:10.834 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:10.836 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:39:10.836 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:39:10.868 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:39:10.868 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:39:10.870 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:10.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:10.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:10.972 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:11.011 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:11.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:11.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:11.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:10 smithi129 ceph-mon[101316]: pgmap v2178: 89 pgs: 89 active+clean; 4.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-09-14T04:39:11.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:10 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2632896754' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:39:11.106 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:39:11.112 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:11.144 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:11.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:11.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:11.246 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:11.290 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:11.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:11.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:11.392 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:11.413 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:11.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:11.414 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:11.515 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:11.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:11.549 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:11.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:11.650 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:11.683 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:11.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:11.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:11.700 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 36 15 72561 72546 125.904 83.6875 0.00764922 0.00793796 2024-09-14T04:39:11.785 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:11.808 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:11.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:11.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:11.910 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:11.946 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:11.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:11.947 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:12.048 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:12.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:12.070 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:12.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:12.171 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:12.211 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:12.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:12.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:12.313 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:12.350 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:12.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:12.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:12.451 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:12.486 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:12.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:12.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:12.588 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:12.624 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:12.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:12.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:12.700 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 37 16 73836 73820 124.654 79.625 0.00713785 0.00800234 2024-09-14T04:39:12.726 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:12.765 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:12.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:12.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:12.867 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:12.892 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:12.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:12.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:12.994 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:13.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:12 smithi129 ceph-mon[101316]: pgmap v2179: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-14T04:39:13.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:13.026 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:13.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:13.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:12 smithi039 ceph-mon[106470]: pgmap v2179: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-14T04:39:13.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:12 smithi039 ceph-mon[111235]: pgmap v2179: 89 pgs: 89 active+clean; 4.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-14T04:39:13.127 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:13.156 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:13.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:13.157 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:13.258 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:13.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:13.290 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:13.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:13.391 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:13.424 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:13.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:13.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:13.526 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:13.566 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:13.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:13.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:13.668 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:13.690 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:13.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:13.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:13.701 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 38 16 75013 74997 123.309 73.5625 0.00352318 0.00809858 2024-09-14T04:39:13.792 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:13.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:13.833 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:13.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:13.934 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:13.968 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:13.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:13.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:14.070 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:14.111 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:14.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:14.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:14.213 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:14.251 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:14.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:14.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:14.353 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:14.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:14.382 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:14.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:14.483 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:14.514 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:14.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:14.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:14.616 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:14.637 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:14.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:14.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:14.701 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 39 16 76233 76217 122.103 76.25 0.00564462 0.00818465 2024-09-14T04:39:14.739 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:14.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:14.770 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:14.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:14.871 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:14.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:14.910 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:14.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:15.011 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:15.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:14 smithi129 ceph-mon[101316]: pgmap v2180: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-09-14T04:39:15.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:14 smithi039 ceph-mon[106470]: pgmap v2180: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-09-14T04:39:15.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:14 smithi039 ceph-mon[111235]: pgmap v2180: 89 pgs: 89 active+clean; 4.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s 2024-09-14T04:39:15.048 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:15.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:15.049 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:15.149 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:15.176 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:15.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:15.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:15.192 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-14T04:39:15.192 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 0 1 2024-09-14T04:39:15.278 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:15.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:15.317 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:15.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:15.418 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:15.448 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:15.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:15.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:15.550 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:15.585 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:15.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:15.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:15.687 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:15.701 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:2024-09-14T04:39:15.701113+0000 min lat: 0.00122803 max lat: 0.134158 avg lat: 0.00826204 2024-09-14T04:39:15.701 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:39:15.701 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 40 16 77410 77394 120.889 73.5625 0.00532778 0.00826204 2024-09-14T04:39:15.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:15.712 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:15.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:15.813 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:15.845 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:15.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:15.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:15.946 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:15.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:15.979 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:15.979 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:16.080 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:16.109 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:16.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:16.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:16.211 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:16.239 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:16.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:16.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:16.289 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:39:16.341 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:16.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:16.383 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:16.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:16.409 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-14T04:39:16.426 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:39:16.485 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:16.513 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:16.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:16.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:16.615 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:16.643 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:16.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:16.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:16.701 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 41 15 78717 78702 119.934 81.75 0.0180786 0.00833192 2024-09-14T04:39:16.746 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:16.817 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:16.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:16.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:16.919 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:16.949 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:16 smithi039 ceph-mon[106470]: pgmap v2181: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-14T04:39:16.949 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:16 smithi039 ceph-mon[111235]: pgmap v2181: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-14T04:39:16.951 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:16.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:16.952 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:16 smithi129 ceph-mon[101316]: pgmap v2181: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s 2024-09-14T04:39:17.053 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:17.087 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:17.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:17.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:17.189 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:17.221 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:17.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:17.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:17.305 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:39:17.323 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:17.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:17.388 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:17.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:17.489 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:17.528 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:17.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:17.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:17.630 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:17.667 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:17.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:17.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:17.701 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 42 16 80033 80017 119.035 82.1875 0.00951203 0.00839571 2024-09-14T04:39:17.769 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:17.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:17.827 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:17.827 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:17.928 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:17.961 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:17.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:17.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:18.062 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:18.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:18.112 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:18.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:18.193 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:39:18.198 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:39:18.213 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:18.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:18.259 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:18.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:18.361 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:18.394 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:18.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:18.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:18.496 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:18.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:18.540 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:18.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:18.641 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:18.672 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:18.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:18.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:18.701 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 43 16 81312 81296 118.126 79.9375 0.0384159 0.00846036 2024-09-14T04:39:18.774 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:18.811 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:18.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:18.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:18.913 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:18.953 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:18.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:18.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:18.969 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:18 smithi039 ceph-mon[106470]: pgmap v2182: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-09-14T04:39:18.969 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:39:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:39:18] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.51.0" 2024-09-14T04:39:18.970 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:18 smithi039 ceph-mon[111235]: pgmap v2182: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-09-14T04:39:19.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:18 smithi129 ceph-mon[101316]: pgmap v2182: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-09-14T04:39:19.055 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:19.092 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:19.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:19.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:19.195 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:19.238 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:19.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:19.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:19.340 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:19.387 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:19.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:19.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:19.488 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:19.534 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:19.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:19.535 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:19.636 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:19.680 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:19.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:19.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:19.701 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 44 16 82590 82574 117.257 79.875 0.0106777 0.00852369 2024-09-14T04:39:19.782 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:19.812 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:19.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:19.813 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:19.914 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:19.952 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:19.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:19.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:20.054 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:20.091 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:20.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:20.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:20.193 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:20.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:20.230 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:20.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:20.331 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:20.357 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:20.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:20.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:20.459 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:20.485 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:20.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:20.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:20.587 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:20.617 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:20.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:20.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:20.699 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is set 2024-09-14T04:39:20.702 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 45 16 83852 83836 116.404 78.875 0.0100673 0.00858465 2024-09-14T04:39:20.719 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:20.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:20.751 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:20.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:20.852 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:20.888 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:20.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:20.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:20.990 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:21.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:20 smithi129 ceph-mon[101316]: pgmap v2183: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-14T04:39:21.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:20 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3854201642' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:39:21.028 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:21.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:21.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:21.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:20 smithi039 ceph-mon[106470]: pgmap v2183: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-14T04:39:21.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:20 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3854201642' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:39:21.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:20 smithi039 ceph-mon[111235]: pgmap v2183: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-14T04:39:21.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:20 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3854201642' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:39:21.130 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:21.176 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:21.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:21.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:21.278 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:21.309 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:21.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:21.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:21.411 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:21.444 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:21.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:21.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:21.548 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:21.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:21.587 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:21.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:21.688 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:21.701 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 46 16 85135 85119 115.616 80.1875 0.019831 0.00864341 2024-09-14T04:39:21.716 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:21.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:21.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:21.818 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:21.859 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:21.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:21.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:21.953 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:21 smithi039 ceph-mon[106470]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:39:21.953 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:21 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3854201642' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:39:21.953 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:21 smithi039 ceph-mon[106470]: osdmap e1323: 8 total, 8 up, 8 in 2024-09-14T04:39:21.953 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:21 smithi039 ceph-mon[111235]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:39:21.953 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:21 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3854201642' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:39:21.953 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:21 smithi039 ceph-mon[111235]: osdmap e1323: 8 total, 8 up, 8 in 2024-09-14T04:39:21.960 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:21.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:21.990 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:21.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:21 smithi129 ceph-mon[101316]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:39:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:21 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3854201642' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:39:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:21 smithi129 ceph-mon[101316]: osdmap e1323: 8 total, 8 up, 8 in 2024-09-14T04:39:22.091 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:22.133 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:22.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:22.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:22.235 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:22.265 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:22.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:22.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:22.367 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:22.402 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:22.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:22.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:22.504 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:22.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:22.542 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:22.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:22.642 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:22.686 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:22.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:22.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:22.702 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 47 16 86422 86406 114.868 80.4375 0.0112519 0.00870085 2024-09-14T04:39:22.787 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:22.824 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:22.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:22.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:22.926 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:22.963 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:22.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:22.964 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:23.064 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:23.111 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:23.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:23.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:23.213 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:23.242 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:23.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:23.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:23.344 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:23.351 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:23 smithi039 ceph-mon[106470]: pgmap v2185: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-09-14T04:39:23.351 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:23 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:39:23.351 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:39:23.351 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:23 smithi039 ceph-mon[111235]: pgmap v2185: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-09-14T04:39:23.352 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:23 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:39:23.352 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:39:23.374 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:23.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:23.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:23.476 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:23.525 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:23.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:23.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:23.627 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:23.665 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:23.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:23.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:23.702 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 48 16 87648 87632 114.071 76.625 0.0466491 0.00876034 2024-09-14T04:39:23.767 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:23 smithi129 ceph-mon[101316]: pgmap v2185: 89 pgs: 89 active+clean; 5.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 80 MiB/s wr, 1.27k op/s 2024-09-14T04:39:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:23 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:39:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:39:23.805 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:23.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:23.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:23.907 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:23.938 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:23.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:23.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:23.955 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:39:24.040 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:24.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:24.091 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:24.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:24.192 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:24.235 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:24.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:24.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:24.337 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:24.358 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:24.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:24.359 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:24.460 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:24.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:24.489 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:24.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:24.513 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:39:24.513 INFO:teuthology.orchestra.run.smithi039.stdout: "size": 20, 2024-09-14T04:39:24.513 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 600, 2024-09-14T04:39:24.513 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [ 2024-09-14T04:39:24.513 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.513 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:29380 53.4 53:22d4e9fa:::benchmark_data_smithi039_224949_object29379:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-14T04:39:24.514 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:38:45.991258+0000", 2024-09-14T04:39:24.514 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 37.75912666, 2024-09-14T04:39:24.514 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.11345762299999999, 2024-09-14T04:39:24.514 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:39:24.514 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:39:24.514 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:39:24.514 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:39:24.514 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:39:24.514 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:39:24.514 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 29380 2024-09-14T04:39:24.514 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.514 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:39:24.515 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.515 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:39:24.515 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.991258+0000", 2024-09-14T04:39:24.515 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.515 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.515 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.515 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:39:24.515 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.991258+0000", 2024-09-14T04:39:24.515 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.515 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.515 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.515 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:39:24.515 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.991259+0000", 2024-09-14T04:39:24.515 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3850000000000001e-06 2024-09-14T04:39:24.516 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.516 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.516 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:39:24.516 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.991282+0000", 2024-09-14T04:39:24.516 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3303e-05 2024-09-14T04:39:24.516 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.516 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.516 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:39:24.516 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.991453+0000", 2024-09-14T04:39:24.516 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000170631 2024-09-14T04:39:24.516 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.516 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.516 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:39:24.517 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.991468+0000", 2024-09-14T04:39:24.517 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4854999999999999e-05 2024-09-14T04:39:24.517 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.517 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.517 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:39:24.517 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.991565+0000", 2024-09-14T04:39:24.517 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.7254000000000001e-05 2024-09-14T04:39:24.517 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.517 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.517 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:39:24.518 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.991687+0000", 2024-09-14T04:39:24.518 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000121773 2024-09-14T04:39:24.518 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.518 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.518 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:39:24.518 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.991818+0000", 2024-09-14T04:39:24.518 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000131082 2024-09-14T04:39:24.518 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.518 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.518 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:39:24.518 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.994657+0000", 2024-09-14T04:39:24.518 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0028387159999999998 2024-09-14T04:39:24.518 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.519 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.519 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:39:24.519 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.994764+0000", 2024-09-14T04:39:24.519 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00010715800000000001 2024-09-14T04:39:24.519 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.519 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.519 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:39:24.519 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.996626+0000", 2024-09-14T04:39:24.519 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0018625530000000001 2024-09-14T04:39:24.519 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.519 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.519 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:39:24.519 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.104662+0000", 2024-09-14T04:39:24.520 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.10803534400000001 2024-09-14T04:39:24.520 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.520 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.520 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:39:24.520 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.104715+0000", 2024-09-14T04:39:24.520 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3569000000000002e-05 2024-09-14T04:39:24.520 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.520 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:39:24.520 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.520 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.520 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.520 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:29387 53.d 53:b7cefd33:::benchmark_data_smithi039_224949_object29386:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-14T04:39:24.520 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:38:45.994265+0000", 2024-09-14T04:39:24.521 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 37.756119837, 2024-09-14T04:39:24.521 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.112344047, 2024-09-14T04:39:24.521 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:39:24.521 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:39:24.521 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:39:24.521 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:39:24.521 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:39:24.521 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:39:24.521 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 29387 2024-09-14T04:39:24.521 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.521 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:39:24.521 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.521 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:39:24.522 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.994265+0000", 2024-09-14T04:39:24.522 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.522 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.522 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.522 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:39:24.522 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.994265+0000", 2024-09-14T04:39:24.522 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.522 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.522 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.522 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:39:24.522 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.994266+0000", 2024-09-14T04:39:24.522 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4759999999999999e-06 2024-09-14T04:39:24.522 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.522 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.522 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:39:24.523 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.994288+0000", 2024-09-14T04:39:24.523 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2135000000000001e-05 2024-09-14T04:39:24.523 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.523 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.523 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:39:24.523 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.994300+0000", 2024-09-14T04:39:24.523 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2092999999999999e-05 2024-09-14T04:39:24.523 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.523 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.523 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:39:24.523 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.994307+0000", 2024-09-14T04:39:24.524 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.905e-06 2024-09-14T04:39:24.524 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.524 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.524 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:39:24.524 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.994366+0000", 2024-09-14T04:39:24.524 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.8884000000000001e-05 2024-09-14T04:39:24.524 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.524 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.524 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:39:24.524 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.994549+0000", 2024-09-14T04:39:24.524 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000183036 2024-09-14T04:39:24.524 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.524 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.525 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:39:24.525 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.994707+0000", 2024-09-14T04:39:24.525 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015768499999999999 2024-09-14T04:39:24.525 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.525 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.525 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:39:24.525 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.104830+0000", 2024-09-14T04:39:24.525 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.110122964 2024-09-14T04:39:24.525 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.525 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.525 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:39:24.525 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.104886+0000", 2024-09-14T04:39:24.525 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6125000000000002e-05 2024-09-14T04:39:24.525 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.526 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.526 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:39:24.526 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.106488+0000", 2024-09-14T04:39:24.526 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0016017239999999999 2024-09-14T04:39:24.526 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.526 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.526 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:39:24.526 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.106538+0000", 2024-09-14T04:39:24.526 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9988000000000003e-05 2024-09-14T04:39:24.526 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.526 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.526 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:39:24.527 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.106609+0000", 2024-09-14T04:39:24.527 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1031999999999993e-05 2024-09-14T04:39:24.527 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.527 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:39:24.527 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.527 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.527 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.527 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:29390 53.d 53:be337127:::benchmark_data_smithi039_224949_object29389:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-14T04:39:24.527 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:38:45.995468+0000", 2024-09-14T04:39:24.527 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 37.754916002000002, 2024-09-14T04:39:24.527 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.11227029600000001, 2024-09-14T04:39:24.527 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:39:24.527 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:39:24.528 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:39:24.528 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:39:24.528 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:39:24.528 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:39:24.528 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 29390 2024-09-14T04:39:24.528 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.528 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:39:24.528 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.528 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:39:24.528 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.995468+0000", 2024-09-14T04:39:24.528 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.528 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.528 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.529 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:39:24.529 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.995468+0000", 2024-09-14T04:39:24.529 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.529 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.529 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.529 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:39:24.529 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.995469+0000", 2024-09-14T04:39:24.529 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.023e-06 2024-09-14T04:39:24.529 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.529 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.529 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:39:24.529 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.995490+0000", 2024-09-14T04:39:24.529 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0890000000000002e-05 2024-09-14T04:39:24.530 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.530 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.530 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:39:24.530 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.995501+0000", 2024-09-14T04:39:24.530 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0716e-05 2024-09-14T04:39:24.530 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.530 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.530 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:39:24.530 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.995507+0000", 2024-09-14T04:39:24.530 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.4849999999999997e-06 2024-09-14T04:39:24.530 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.530 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.530 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:39:24.530 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.995776+0000", 2024-09-14T04:39:24.531 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00026900999999999997 2024-09-14T04:39:24.531 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.531 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.531 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:39:24.531 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.995907+0000", 2024-09-14T04:39:24.531 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013005499999999999 2024-09-14T04:39:24.531 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.531 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.531 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:39:24.531 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.996040+0000", 2024-09-14T04:39:24.531 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013362000000000001 2024-09-14T04:39:24.531 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.531 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.532 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:39:24.532 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.106444+0000", 2024-09-14T04:39:24.532 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.11040388499999999 2024-09-14T04:39:24.532 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.532 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.532 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:39:24.532 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.106655+0000", 2024-09-14T04:39:24.532 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000210862 2024-09-14T04:39:24.532 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.532 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.532 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:39:24.532 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.107648+0000", 2024-09-14T04:39:24.532 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00099344899999999998 2024-09-14T04:39:24.532 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.533 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.533 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:39:24.533 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.107696+0000", 2024-09-14T04:39:24.533 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7808e-05 2024-09-14T04:39:24.533 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.533 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.533 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:39:24.533 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.107739+0000", 2024-09-14T04:39:24.533 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2493000000000001e-05 2024-09-14T04:39:24.533 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.533 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:39:24.533 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.533 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.534 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.534 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop_reply(client.45053.0:29380 53.4 e1319/1315 ondisk, result = 0)", 2024-09-14T04:39:24.534 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:38:45.995604+0000", 2024-09-14T04:39:24.534 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 37.754780607000001, 2024-09-14T04:39:24.534 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.109120363, 2024-09-14T04:39:24.534 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:39:24.534 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:39:24.534 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "started", 2024-09-14T04:39:24.534 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:39:24.534 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.534 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:39:24.534 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.995604+0000", 2024-09-14T04:39:24.534 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.534 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.534 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.535 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:39:24.535 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.995604+0000", 2024-09-14T04:39:24.535 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.535 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.535 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.535 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:39:24.535 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.995604+0000", 2024-09-14T04:39:24.535 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6700000000000003e-07 2024-09-14T04:39:24.535 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.535 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.535 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:39:24.535 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.995609+0000", 2024-09-14T04:39:24.535 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4190000000000004e-06 2024-09-14T04:39:24.535 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.535 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.536 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:39:24.536 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.995620+0000", 2024-09-14T04:39:24.536 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0875e-05 2024-09-14T04:39:24.536 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.536 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.536 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:39:24.536 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.995626+0000", 2024-09-14T04:39:24.536 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.9989999999999997e-06 2024-09-14T04:39:24.536 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.536 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.536 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:39:24.536 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.996614+0000", 2024-09-14T04:39:24.536 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00098833300000000009 2024-09-14T04:39:24.536 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.536 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.537 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:39:24.537 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.996621+0000", 2024-09-14T04:39:24.537 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.4939999999999997e-06 2024-09-14T04:39:24.537 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.537 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.537 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:39:24.537 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.104724+0000", 2024-09-14T04:39:24.537 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.10810267599999999 2024-09-14T04:39:24.537 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.537 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:39:24.537 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.537 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.537 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.537 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:29383 53.8 e1319/1315 53:11159672:::benchmark_data_smithi039_224949_object29382:head v 1319'1862, mlcod=1319'1860)", 2024-09-14T04:39:24.537 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:38:45.995645+0000", 2024-09-14T04:39:24.538 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 37.754739010000002, 2024-09-14T04:39:24.538 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.10907726399999999, 2024-09-14T04:39:24.538 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:39:24.538 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:39:24.538 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:39:24.538 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:39:24.538 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.538 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:39:24.538 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.995645+0000", 2024-09-14T04:39:24.538 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.538 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.538 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.538 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:39:24.538 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.995645+0000", 2024-09-14T04:39:24.539 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.539 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.539 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.539 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:39:24.539 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.995646+0000", 2024-09-14T04:39:24.539 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.3299999999999998e-07 2024-09-14T04:39:24.539 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.539 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.539 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:39:24.539 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.995692+0000", 2024-09-14T04:39:24.539 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6217999999999997e-05 2024-09-14T04:39:24.539 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.539 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.539 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:39:24.540 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.995709+0000", 2024-09-14T04:39:24.540 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7292999999999999e-05 2024-09-14T04:39:24.540 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.540 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.540 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:39:24.540 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.995716+0000", 2024-09-14T04:39:24.540 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.4629999999999998e-06 2024-09-14T04:39:24.540 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.540 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.540 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:39:24.540 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.995797+0000", 2024-09-14T04:39:24.540 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.1353000000000005e-05 2024-09-14T04:39:24.540 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.541 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.541 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:39:24.541 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.995815+0000", 2024-09-14T04:39:24.541 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7810999999999999e-05 2024-09-14T04:39:24.541 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.541 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.541 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:39:24.541 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.104669+0000", 2024-09-14T04:39:24.541 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.10885404899999999 2024-09-14T04:39:24.541 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.541 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.541 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:39:24.541 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.104723+0000", 2024-09-14T04:39:24.542 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3743999999999999e-05 2024-09-14T04:39:24.542 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.542 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:39:24.542 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.542 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.542 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.542 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop_reply(client.45053.0:29387 53.d e1319/1315 ondisk, result = 0)", 2024-09-14T04:39:24.542 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:38:45.996631+0000", 2024-09-14T04:39:24.542 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 37.753753246999999, 2024-09-14T04:39:24.542 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.108277923, 2024-09-14T04:39:24.542 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:39:24.542 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:39:24.542 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "started", 2024-09-14T04:39:24.543 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:39:24.543 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.543 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:39:24.543 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.996631+0000", 2024-09-14T04:39:24.543 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.543 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.543 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.543 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:39:24.543 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.996631+0000", 2024-09-14T04:39:24.543 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.543 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.543 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.543 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:39:24.544 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.996632+0000", 2024-09-14T04:39:24.544 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.6400000000000001e-07 2024-09-14T04:39:24.544 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.544 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.544 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:39:24.544 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.996636+0000", 2024-09-14T04:39:24.544 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2760000000000002e-06 2024-09-14T04:39:24.544 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.544 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.544 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:39:24.544 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.996646+0000", 2024-09-14T04:39:24.544 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0003999999999999e-05 2024-09-14T04:39:24.544 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.544 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.545 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:39:24.545 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.996653+0000", 2024-09-14T04:39:24.545 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.477e-06 2024-09-14T04:39:24.545 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.545 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.545 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:39:24.545 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.104872+0000", 2024-09-14T04:39:24.545 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.108219232 2024-09-14T04:39:24.545 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.545 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.545 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:39:24.545 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.104880+0000", 2024-09-14T04:39:24.545 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.926e-06 2024-09-14T04:39:24.545 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.546 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.546 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:39:24.546 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.104909+0000", 2024-09-14T04:39:24.546 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.9144000000000001e-05 2024-09-14T04:39:24.546 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.546 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:39:24.546 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.546 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.546 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.546 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:29396 53.d 53:b54b4555:::benchmark_data_smithi039_224949_object29395:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-14T04:39:24.546 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:38:45.996706+0000", 2024-09-14T04:39:24.546 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 37.753678284999999, 2024-09-14T04:39:24.546 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.11733969499999999, 2024-09-14T04:39:24.547 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:39:24.547 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:39:24.547 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:39:24.547 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:39:24.547 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:39:24.547 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:39:24.547 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 29396 2024-09-14T04:39:24.547 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.547 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:39:24.547 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.547 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:39:24.547 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.996706+0000", 2024-09-14T04:39:24.547 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.548 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.548 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.548 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:39:24.548 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.996706+0000", 2024-09-14T04:39:24.548 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.548 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.548 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.548 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:39:24.548 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.996707+0000", 2024-09-14T04:39:24.549 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.6600000000000005e-07 2024-09-14T04:39:24.549 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.549 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.549 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:39:24.549 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:45.996731+0000", 2024-09-14T04:39:24.549 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3612999999999999e-05 2024-09-14T04:39:24.549 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.549 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.549 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:39:24.549 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.104666+0000", 2024-09-14T04:39:24.549 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.107935399 2024-09-14T04:39:24.549 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.549 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.550 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:39:24.550 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.104694+0000", 2024-09-14T04:39:24.550 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.8523000000000001e-05 2024-09-14T04:39:24.550 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.550 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.550 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:39:24.550 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.104935+0000", 2024-09-14T04:39:24.550 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00024009899999999999 2024-09-14T04:39:24.550 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.550 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.550 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:39:24.550 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.105125+0000", 2024-09-14T04:39:24.551 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00019080200000000001 2024-09-14T04:39:24.551 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.551 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.551 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:39:24.551 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.105249+0000", 2024-09-14T04:39:24.551 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000123979 2024-09-14T04:39:24.551 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.551 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.551 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:39:24.551 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.107775+0000", 2024-09-14T04:39:24.551 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0025257529999999999 2024-09-14T04:39:24.551 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.551 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.551 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:39:24.552 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.109077+0000", 2024-09-14T04:39:24.552 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0013019609999999999 2024-09-14T04:39:24.552 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.552 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.552 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:39:24.552 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.113924+0000", 2024-09-14T04:39:24.552 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.004846843 2024-09-14T04:39:24.552 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.552 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.552 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:39:24.552 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.113993+0000", 2024-09-14T04:39:24.552 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9251000000000005e-05 2024-09-14T04:39:24.553 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.553 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.553 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:39:24.553 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.114046+0000", 2024-09-14T04:39:24.553 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.2605999999999997e-05 2024-09-14T04:39:24.553 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.553 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:39:24.553 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.553 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.553 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.553 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:29405 53.8 e1319/1315 53:1eab3930:::benchmark_data_smithi039_224949_object29404:head v 1319'1863, mlcod=1319'1860)", 2024-09-14T04:39:24.553 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:38:46.002462+0000", 2024-09-14T04:39:24.553 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 37.747922699, 2024-09-14T04:39:24.554 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.108579043, 2024-09-14T04:39:24.554 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:39:24.554 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:39:24.554 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:39:24.554 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:39:24.554 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.554 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:39:24.554 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.002462+0000", 2024-09-14T04:39:24.554 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.554 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.554 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.554 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:39:24.554 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.002462+0000", 2024-09-14T04:39:24.555 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.555 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.555 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.555 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:39:24.555 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.002463+0000", 2024-09-14T04:39:24.555 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.175e-06 2024-09-14T04:39:24.555 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.555 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.555 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:39:24.555 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.002520+0000", 2024-09-14T04:39:24.555 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6900000000000001e-05 2024-09-14T04:39:24.555 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.555 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.556 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:39:24.556 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.104640+0000", 2024-09-14T04:39:24.556 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.102120611 2024-09-14T04:39:24.556 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.556 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.556 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:39:24.556 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.104654+0000", 2024-09-14T04:39:24.556 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3295999999999999e-05 2024-09-14T04:39:24.556 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.556 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.556 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:39:24.556 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.104767+0000", 2024-09-14T04:39:24.556 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000113807 2024-09-14T04:39:24.556 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.557 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.557 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:39:24.557 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.104790+0000", 2024-09-14T04:39:24.557 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2887999999999998e-05 2024-09-14T04:39:24.557 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.557 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.557 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:39:24.557 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.110994+0000", 2024-09-14T04:39:24.557 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0062033510000000002 2024-09-14T04:39:24.557 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.557 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.557 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:39:24.558 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:46.111041+0000", 2024-09-14T04:39:24.558 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7015000000000003e-05 2024-09-14T04:39:24.558 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.558 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:39:24.558 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.558 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.558 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.558 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:31602 53.d 53:b7eb2d39:::benchmark_data_smithi039_224949_object31601:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-14T04:39:24.558 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:38:47.156815+0000", 2024-09-14T04:39:24.558 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 36.593569221000003, 2024-09-14T04:39:24.558 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.10448399799999999, 2024-09-14T04:39:24.558 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:39:24.559 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:39:24.559 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:39:24.559 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:39:24.559 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:39:24.559 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:39:24.559 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 31602 2024-09-14T04:39:24.559 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.559 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:39:24.559 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.559 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:39:24.559 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.156815+0000", 2024-09-14T04:39:24.559 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.559 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.560 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.560 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:39:24.560 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.156815+0000", 2024-09-14T04:39:24.560 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.560 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.560 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.560 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:39:24.560 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.156816+0000", 2024-09-14T04:39:24.560 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0020000000000001e-06 2024-09-14T04:39:24.560 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.560 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.560 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:39:24.561 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.156840+0000", 2024-09-14T04:39:24.561 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3439999999999999e-05 2024-09-14T04:39:24.561 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.561 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.561 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:39:24.561 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.156851+0000", 2024-09-14T04:39:24.561 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1243999999999999e-05 2024-09-14T04:39:24.561 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.561 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.561 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:39:24.561 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.156858+0000", 2024-09-14T04:39:24.561 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8460000000000002e-06 2024-09-14T04:39:24.562 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.562 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.562 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:39:24.562 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.254503+0000", 2024-09-14T04:39:24.562 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.097645437000000002 2024-09-14T04:39:24.562 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.562 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.562 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:39:24.562 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.254645+0000", 2024-09-14T04:39:24.562 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014144799999999999 2024-09-14T04:39:24.562 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.562 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.562 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:39:24.562 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.254774+0000", 2024-09-14T04:39:24.563 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000129441 2024-09-14T04:39:24.563 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.563 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.563 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:39:24.563 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.259860+0000", 2024-09-14T04:39:24.563 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0050860790000000003 2024-09-14T04:39:24.563 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.563 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.563 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:39:24.563 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.261049+0000", 2024-09-14T04:39:24.563 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001188485 2024-09-14T04:39:24.563 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.564 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.564 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:39:24.564 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.261201+0000", 2024-09-14T04:39:24.564 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000152399 2024-09-14T04:39:24.564 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.564 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.564 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:39:24.564 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.261242+0000", 2024-09-14T04:39:24.564 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.1176e-05 2024-09-14T04:39:24.564 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.564 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.564 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:39:24.564 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.261299+0000", 2024-09-14T04:39:24.564 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7000999999999999e-05 2024-09-14T04:39:24.565 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.565 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:39:24.565 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.565 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.565 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.565 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:73817 53.1 53:8f66518f:::benchmark_data_smithi039_224949_object73816:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-09-14T04:39:24.565 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:39:12.618936+0000", 2024-09-14T04:39:24.565 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 11.131448096, 2024-09-14T04:39:24.565 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.118828214, 2024-09-14T04:39:24.565 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:39:24.565 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:39:24.565 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:39:24.566 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:39:24.566 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:39:24.566 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:39:24.566 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 73817 2024-09-14T04:39:24.566 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.566 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:39:24.566 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.566 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:39:24.566 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.618936+0000", 2024-09-14T04:39:24.566 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.566 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.566 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.566 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:39:24.566 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.618936+0000", 2024-09-14T04:39:24.566 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.566 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.567 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.567 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:39:24.567 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.618937+0000", 2024-09-14T04:39:24.567 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.127e-06 2024-09-14T04:39:24.567 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.567 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.567 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:39:24.567 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.618963+0000", 2024-09-14T04:39:24.567 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.5437000000000001e-05 2024-09-14T04:39:24.567 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.567 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.567 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:39:24.567 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.618974+0000", 2024-09-14T04:39:24.568 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0696000000000001e-05 2024-09-14T04:39:24.568 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.568 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.568 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:39:24.568 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.618980+0000", 2024-09-14T04:39:24.568 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.956e-06 2024-09-14T04:39:24.568 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.568 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.568 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:39:24.568 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.619309+0000", 2024-09-14T04:39:24.568 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00032890699999999997 2024-09-14T04:39:24.568 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.569 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.569 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:39:24.569 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.619428+0000", 2024-09-14T04:39:24.569 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000118966 2024-09-14T04:39:24.569 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.569 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.569 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:39:24.569 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.619556+0000", 2024-09-14T04:39:24.569 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012732100000000001 2024-09-14T04:39:24.569 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.569 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.569 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:39:24.570 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.622203+0000", 2024-09-14T04:39:24.570 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0026477750000000002 2024-09-14T04:39:24.570 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.570 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.570 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:39:24.570 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.630226+0000", 2024-09-14T04:39:24.570 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.008022085 2024-09-14T04:39:24.570 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.570 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.570 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:39:24.570 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.737646+0000", 2024-09-14T04:39:24.570 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.107420764 2024-09-14T04:39:24.570 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.570 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.570 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:39:24.571 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.737707+0000", 2024-09-14T04:39:24.571 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0322000000000003e-05 2024-09-14T04:39:24.571 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.571 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.571 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:39:24.571 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.737764+0000", 2024-09-14T04:39:24.571 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7858000000000001e-05 2024-09-14T04:39:24.571 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.571 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:39:24.571 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.571 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.571 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.571 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:73823 53.1 53:89730e74:::benchmark_data_smithi039_224949_object73822:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-09-14T04:39:24.572 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:39:12.623946+0000", 2024-09-14T04:39:24.572 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 11.126438152, 2024-09-14T04:39:24.572 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.13397935499999999, 2024-09-14T04:39:24.572 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:39:24.572 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:39:24.572 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:39:24.572 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:39:24.572 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:39:24.572 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:39:24.572 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 73823 2024-09-14T04:39:24.572 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.572 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:39:24.573 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.573 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:39:24.573 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.623946+0000", 2024-09-14T04:39:24.573 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.573 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.573 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.573 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:39:24.573 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.623946+0000", 2024-09-14T04:39:24.574 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.574 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.574 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.574 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:39:24.574 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.623947+0000", 2024-09-14T04:39:24.574 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.1900000000000001e-07 2024-09-14T04:39:24.574 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.574 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.574 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:39:24.574 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.623969+0000", 2024-09-14T04:39:24.574 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1804999999999999e-05 2024-09-14T04:39:24.574 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.575 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.575 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:39:24.575 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.623980+0000", 2024-09-14T04:39:24.575 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.099e-05 2024-09-14T04:39:24.575 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.575 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.575 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:39:24.575 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.623987+0000", 2024-09-14T04:39:24.575 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6559999999999992e-06 2024-09-14T04:39:24.575 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.575 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.575 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:39:24.576 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.624049+0000", 2024-09-14T04:39:24.576 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1267999999999997e-05 2024-09-14T04:39:24.576 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.576 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.576 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:39:24.576 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.624236+0000", 2024-09-14T04:39:24.576 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000186763 2024-09-14T04:39:24.576 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.576 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.576 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:39:24.576 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.624388+0000", 2024-09-14T04:39:24.576 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015237200000000001 2024-09-14T04:39:24.576 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.577 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.577 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:39:24.577 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.628010+0000", 2024-09-14T04:39:24.577 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.003622452 2024-09-14T04:39:24.577 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.577 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.577 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:39:24.577 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.630261+0000", 2024-09-14T04:39:24.577 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0022507249999999999 2024-09-14T04:39:24.577 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.577 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.577 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:39:24.577 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.757814+0000", 2024-09-14T04:39:24.578 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.127552901 2024-09-14T04:39:24.578 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.578 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.578 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:39:24.578 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.757880+0000", 2024-09-14T04:39:24.578 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.6081999999999995e-05 2024-09-14T04:39:24.578 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.578 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.578 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:39:24.578 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.757926+0000", 2024-09-14T04:39:24.578 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5522000000000002e-05 2024-09-14T04:39:24.578 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.578 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:39:24.578 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.579 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.579 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.579 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:73819 53.5 e1322/1315 53:a813ea22:::benchmark_data_smithi039_224949_object73818:head v 1322'4584, mlcod=1322'4581)", 2024-09-14T04:39:24.579 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:39:12.629002+0000", 2024-09-14T04:39:24.579 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 11.121382003000001, 2024-09-14T04:39:24.579 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.10730400399999999, 2024-09-14T04:39:24.579 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:39:24.579 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:39:24.579 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:39:24.579 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:39:24.579 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.579 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:39:24.579 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.629002+0000", 2024-09-14T04:39:24.580 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.580 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.580 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.580 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:39:24.580 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.629002+0000", 2024-09-14T04:39:24.580 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.580 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.580 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.580 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:39:24.580 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.629003+0000", 2024-09-14T04:39:24.580 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5499999999999998e-07 2024-09-14T04:39:24.580 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.580 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.581 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:39:24.581 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.629024+0000", 2024-09-14T04:39:24.581 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1305999999999999e-05 2024-09-14T04:39:24.581 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.581 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.581 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:39:24.581 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.629031+0000", 2024-09-14T04:39:24.581 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.379e-06 2024-09-14T04:39:24.581 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.581 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.581 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:39:24.581 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.629037+0000", 2024-09-14T04:39:24.581 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1250000000000001e-06 2024-09-14T04:39:24.581 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.582 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.582 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:39:24.582 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.630756+0000", 2024-09-14T04:39:24.582 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001719852 2024-09-14T04:39:24.582 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.582 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.582 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:39:24.582 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.630774+0000", 2024-09-14T04:39:24.582 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7669999999999999e-05 2024-09-14T04:39:24.582 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.582 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.582 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:39:24.582 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.632879+0000", 2024-09-14T04:39:24.582 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0021048460000000001 2024-09-14T04:39:24.583 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.583 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.583 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:39:24.583 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.736306+0000", 2024-09-14T04:39:24.583 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.103427371 2024-09-14T04:39:24.583 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.583 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:39:24.583 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.583 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.583 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.583 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:73829 53.f e1322/1315 53:fa329c58:::benchmark_data_smithi039_224949_object73828:head v 1322'4567, mlcod=1322'4565)", 2024-09-14T04:39:24.583 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:39:12.629172+0000", 2024-09-14T04:39:24.583 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 11.121212742999999, 2024-09-14T04:39:24.583 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.10924816800000001, 2024-09-14T04:39:24.583 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:39:24.584 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:39:24.584 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:39:24.584 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:39:24.584 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.584 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:39:24.584 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.629172+0000", 2024-09-14T04:39:24.584 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.584 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.584 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.584 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:39:24.584 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.629172+0000", 2024-09-14T04:39:24.584 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.584 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.585 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.585 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:39:24.585 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.629172+0000", 2024-09-14T04:39:24.585 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.8100000000000003e-07 2024-09-14T04:39:24.585 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.585 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.585 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:39:24.585 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.629199+0000", 2024-09-14T04:39:24.585 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.7370000000000001e-05 2024-09-14T04:39:24.585 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.585 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.585 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:39:24.586 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.629208+0000", 2024-09-14T04:39:24.586 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.5150000000000006e-06 2024-09-14T04:39:24.586 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.586 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.586 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:39:24.586 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.629213+0000", 2024-09-14T04:39:24.586 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3929999999999999e-06 2024-09-14T04:39:24.586 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.586 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.586 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:39:24.586 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.630888+0000", 2024-09-14T04:39:24.586 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0016741390000000001 2024-09-14T04:39:24.587 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.587 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.587 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:39:24.587 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.630905+0000", 2024-09-14T04:39:24.587 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7507e-05 2024-09-14T04:39:24.587 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.587 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.587 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:39:24.587 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.738189+0000", 2024-09-14T04:39:24.587 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.10728370399999999 2024-09-14T04:39:24.587 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.587 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.587 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:39:24.587 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.738420+0000", 2024-09-14T04:39:24.588 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00023105900000000001 2024-09-14T04:39:24.588 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.588 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:39:24.588 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.588 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.588 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.588 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:73830 53.f e1322/1315 53:f0fc3480:::benchmark_data_smithi039_224949_object73829:head v 1322'4568, mlcod=1322'4565)", 2024-09-14T04:39:24.588 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:39:12.631190+0000", 2024-09-14T04:39:24.588 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 11.119193872, 2024-09-14T04:39:24.588 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.10763599, 2024-09-14T04:39:24.588 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:39:24.588 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:39:24.588 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:39:24.589 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:39:24.589 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.589 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:39:24.589 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.631190+0000", 2024-09-14T04:39:24.589 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.589 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.589 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.589 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:39:24.589 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.631190+0000", 2024-09-14T04:39:24.589 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.589 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.589 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.589 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:39:24.590 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.631191+0000", 2024-09-14T04:39:24.590 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2399999999999998e-07 2024-09-14T04:39:24.590 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.590 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.590 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:39:24.590 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.631228+0000", 2024-09-14T04:39:24.590 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.7174e-05 2024-09-14T04:39:24.590 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.590 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.590 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:39:24.590 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.631238+0000", 2024-09-14T04:39:24.590 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.7159999999999998e-06 2024-09-14T04:39:24.590 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.590 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.591 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:39:24.591 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.631244+0000", 2024-09-14T04:39:24.591 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0129999999999999e-06 2024-09-14T04:39:24.591 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.591 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.591 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:39:24.591 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.735236+0000", 2024-09-14T04:39:24.591 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.103991639 2024-09-14T04:39:24.591 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.591 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.591 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:39:24.591 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.735258+0000", 2024-09-14T04:39:24.591 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2651999999999999e-05 2024-09-14T04:39:24.591 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.592 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.592 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:39:24.592 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.738783+0000", 2024-09-14T04:39:24.592 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00352498 2024-09-14T04:39:24.592 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.592 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.592 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:39:24.592 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.738826+0000", 2024-09-14T04:39:24.592 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3192000000000003e-05 2024-09-14T04:39:24.592 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.592 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:39:24.592 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.593 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.593 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.593 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:74031 53.1 53:803d4652:::benchmark_data_smithi039_224949_object74030:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-09-14T04:39:24.593 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:39:12.865631+0000", 2024-09-14T04:39:24.593 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 10.884753751, 2024-09-14T04:39:24.593 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.106661005, 2024-09-14T04:39:24.593 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:39:24.593 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:39:24.593 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:39:24.593 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:39:24.593 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:39:24.593 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:39:24.593 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 74031 2024-09-14T04:39:24.593 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.594 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:39:24.594 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.594 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:39:24.594 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.865631+0000", 2024-09-14T04:39:24.594 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.594 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.594 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.594 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:39:24.594 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.865631+0000", 2024-09-14T04:39:24.594 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.594 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.594 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.594 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:39:24.595 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.865632+0000", 2024-09-14T04:39:24.595 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4890000000000001e-06 2024-09-14T04:39:24.595 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.595 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.595 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:39:24.595 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.865663+0000", 2024-09-14T04:39:24.595 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.0446999999999998e-05 2024-09-14T04:39:24.595 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.595 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.595 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:39:24.595 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.865674+0000", 2024-09-14T04:39:24.595 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1613e-05 2024-09-14T04:39:24.595 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.596 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.596 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:39:24.596 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.865682+0000", 2024-09-14T04:39:24.596 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.5549999999999997e-06 2024-09-14T04:39:24.596 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.596 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.596 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:39:24.596 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.865746+0000", 2024-09-14T04:39:24.596 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.4145000000000003e-05 2024-09-14T04:39:24.596 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.596 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.596 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:39:24.596 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.865860+0000", 2024-09-14T04:39:24.596 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000114312 2024-09-14T04:39:24.597 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.597 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.597 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:39:24.597 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.865982+0000", 2024-09-14T04:39:24.597 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012183700000000001 2024-09-14T04:39:24.597 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.597 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.597 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:39:24.597 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.877341+0000", 2024-09-14T04:39:24.597 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.011359259 2024-09-14T04:39:24.597 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.597 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.597 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:39:24.598 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.877674+0000", 2024-09-14T04:39:24.598 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00033241899999999998 2024-09-14T04:39:24.598 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.598 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.598 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:39:24.598 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.972215+0000", 2024-09-14T04:39:24.598 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.094541737000000001 2024-09-14T04:39:24.598 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.598 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.598 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:39:24.598 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.972259+0000", 2024-09-14T04:39:24.598 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3340000000000002e-05 2024-09-14T04:39:24.598 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.599 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.599 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:39:24.599 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.972292+0000", 2024-09-14T04:39:24.599 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.2852000000000003e-05 2024-09-14T04:39:24.599 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.599 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:39:24.599 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.599 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.599 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.599 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:74036 53.1 53:8a6bcccd:::benchmark_data_smithi039_224949_object74035:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-09-14T04:39:24.599 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:39:12.868111+0000", 2024-09-14T04:39:24.599 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 10.88227307, 2024-09-14T04:39:24.600 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.104703903, 2024-09-14T04:39:24.600 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:39:24.600 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:39:24.600 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:39:24.600 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:39:24.600 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:39:24.600 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:39:24.600 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 74036 2024-09-14T04:39:24.601 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.601 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:39:24.601 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.601 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:39:24.601 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.868111+0000", 2024-09-14T04:39:24.601 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.601 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.601 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.601 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:39:24.601 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.868111+0000", 2024-09-14T04:39:24.601 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.601 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.602 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.602 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:39:24.602 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.868113+0000", 2024-09-14T04:39:24.602 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2410000000000001e-06 2024-09-14T04:39:24.602 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.602 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.602 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:39:24.602 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.868134+0000", 2024-09-14T04:39:24.602 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1591e-05 2024-09-14T04:39:24.602 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.602 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.603 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:39:24.603 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.868147+0000", 2024-09-14T04:39:24.603 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.257e-05 2024-09-14T04:39:24.603 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.603 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.603 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:39:24.603 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.868155+0000", 2024-09-14T04:39:24.603 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.8949999999999998e-06 2024-09-14T04:39:24.603 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.603 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.603 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:39:24.603 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.872447+0000", 2024-09-14T04:39:24.603 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0042922180000000004 2024-09-14T04:39:24.604 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.604 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.604 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:39:24.604 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.872588+0000", 2024-09-14T04:39:24.604 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014105500000000001 2024-09-14T04:39:24.604 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.604 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.604 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:39:24.604 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.872718+0000", 2024-09-14T04:39:24.604 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013039400000000001 2024-09-14T04:39:24.604 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.604 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.605 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:39:24.605 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.877346+0000", 2024-09-14T04:39:24.605 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0046282099999999998 2024-09-14T04:39:24.605 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.605 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.605 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:39:24.605 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.972337+0000", 2024-09-14T04:39:24.605 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.094990527000000005 2024-09-14T04:39:24.606 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.606 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.606 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:39:24.606 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.972724+0000", 2024-09-14T04:39:24.606 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00038750500000000001 2024-09-14T04:39:24.606 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.606 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.606 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:39:24.606 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.972775+0000", 2024-09-14T04:39:24.606 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.0235999999999998e-05 2024-09-14T04:39:24.606 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.606 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.606 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:39:24.606 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.972815+0000", 2024-09-14T04:39:24.607 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.0460999999999998e-05 2024-09-14T04:39:24.607 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.607 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:39:24.607 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.607 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.607 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.607 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:74041 53.7 53:e6dfbd15:::benchmark_data_smithi039_224949_object74040:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-09-14T04:39:24.607 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:39:12.871485+0000", 2024-09-14T04:39:24.607 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 10.878899609999999, 2024-09-14T04:39:24.607 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.106150568, 2024-09-14T04:39:24.607 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:39:24.607 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:39:24.607 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:39:24.607 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:39:24.608 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:39:24.608 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:39:24.608 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 74041 2024-09-14T04:39:24.608 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.608 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:39:24.608 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.608 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:39:24.608 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.871485+0000", 2024-09-14T04:39:24.608 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.608 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.608 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.608 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:39:24.608 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.871485+0000", 2024-09-14T04:39:24.609 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.609 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.609 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.609 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:39:24.609 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.871486+0000", 2024-09-14T04:39:24.609 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5519999999999999e-06 2024-09-14T04:39:24.609 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.609 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.609 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:39:24.609 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.871512+0000", 2024-09-14T04:39:24.609 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.5556000000000001e-05 2024-09-14T04:39:24.609 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.609 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.610 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:39:24.610 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.871523+0000", 2024-09-14T04:39:24.610 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1104000000000001e-05 2024-09-14T04:39:24.610 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.610 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.610 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:39:24.610 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.871530+0000", 2024-09-14T04:39:24.610 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9099999999999999e-06 2024-09-14T04:39:24.610 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.610 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.610 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:39:24.610 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.874512+0000", 2024-09-14T04:39:24.610 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.002981853 2024-09-14T04:39:24.611 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.611 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.611 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:39:24.611 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.874638+0000", 2024-09-14T04:39:24.611 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012608500000000001 2024-09-14T04:39:24.611 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.611 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.611 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 0,3", 2024-09-14T04:39:24.611 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.874771+0000", 2024-09-14T04:39:24.611 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013291900000000001 2024-09-14T04:39:24.611 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.611 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.611 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:39:24.612 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.971890+0000", 2024-09-14T04:39:24.612 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.097119442 2024-09-14T04:39:24.612 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.612 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.612 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:39:24.612 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.972052+0000", 2024-09-14T04:39:24.612 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000161698 2024-09-14T04:39:24.612 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.612 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.612 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:39:24.612 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.976483+0000", 2024-09-14T04:39:24.612 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0044311150000000002 2024-09-14T04:39:24.612 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.612 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.613 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:39:24.613 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.977582+0000", 2024-09-14T04:39:24.613 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0010990399999999999 2024-09-14T04:39:24.613 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.613 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.613 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:39:24.613 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.977635+0000", 2024-09-14T04:39:24.613 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3294000000000002e-05 2024-09-14T04:39:24.613 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.613 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:39:24.613 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.613 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.613 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.614 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:76189 53.4 53:26584c6d:::benchmark_data_smithi039_224949_object76188:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-09-14T04:39:24.614 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:39:14.580754+0000", 2024-09-14T04:39:24.614 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 9.1696300780000009, 2024-09-14T04:39:24.614 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.104843834, 2024-09-14T04:39:24.614 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:39:24.614 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:39:24.614 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:39:24.614 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:39:24.614 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:39:24.614 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:39:24.614 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 76189 2024-09-14T04:39:24.614 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.614 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:39:24.615 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.615 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:39:24.615 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.580754+0000", 2024-09-14T04:39:24.615 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.615 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.615 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.615 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:39:24.615 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.580754+0000", 2024-09-14T04:39:24.615 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.615 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.615 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.615 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:39:24.615 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.580756+0000", 2024-09-14T04:39:24.616 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4699999999999999e-06 2024-09-14T04:39:24.616 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.616 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.616 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:39:24.616 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.580780+0000", 2024-09-14T04:39:24.616 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4417999999999999e-05 2024-09-14T04:39:24.616 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.616 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.616 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:39:24.616 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.580792+0000", 2024-09-14T04:39:24.616 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1698e-05 2024-09-14T04:39:24.616 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.617 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.617 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:39:24.617 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.580799+0000", 2024-09-14T04:39:24.617 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1799999999999999e-06 2024-09-14T04:39:24.617 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.617 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.617 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:39:24.617 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.580860+0000", 2024-09-14T04:39:24.617 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0652000000000001e-05 2024-09-14T04:39:24.617 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.617 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.617 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:39:24.617 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.580993+0000", 2024-09-14T04:39:24.618 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013334499999999999 2024-09-14T04:39:24.618 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.618 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.618 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:39:24.618 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.581086+0000", 2024-09-14T04:39:24.618 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.3226000000000005e-05 2024-09-14T04:39:24.618 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.618 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.618 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:39:24.618 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.583608+0000", 2024-09-14T04:39:24.618 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0025216639999999999 2024-09-14T04:39:24.618 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.618 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.619 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:39:24.619 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.685442+0000", 2024-09-14T04:39:24.619 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.10183428999999999 2024-09-14T04:39:24.619 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.619 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.619 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:39:24.619 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.685499+0000", 2024-09-14T04:39:24.619 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7039000000000001e-05 2024-09-14T04:39:24.619 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.619 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.619 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:39:24.619 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.685553+0000", 2024-09-14T04:39:24.619 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3538999999999997e-05 2024-09-14T04:39:24.620 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.620 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.620 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:39:24.620 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.685598+0000", 2024-09-14T04:39:24.620 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5312999999999998e-05 2024-09-14T04:39:24.620 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.620 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:39:24.620 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.620 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.620 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.620 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:76191 53.4 53:2e60b96a:::benchmark_data_smithi039_224949_object76190:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-09-14T04:39:24.620 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:39:14.583555+0000", 2024-09-14T04:39:24.620 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 9.1668296280000003, 2024-09-14T04:39:24.621 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.106628556, 2024-09-14T04:39:24.621 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:39:24.621 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:39:24.621 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:39:24.621 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:39:24.621 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:39:24.621 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:39:24.621 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 76191 2024-09-14T04:39:24.621 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.621 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:39:24.621 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.621 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:39:24.621 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.583555+0000", 2024-09-14T04:39:24.622 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.622 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.622 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.622 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:39:24.622 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.583555+0000", 2024-09-14T04:39:24.622 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.622 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.622 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.622 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:39:24.622 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.583556+0000", 2024-09-14T04:39:24.622 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4959999999999999e-06 2024-09-14T04:39:24.622 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.622 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.622 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:39:24.623 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.583578+0000", 2024-09-14T04:39:24.623 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1823000000000001e-05 2024-09-14T04:39:24.623 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.623 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.623 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:39:24.623 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.583589+0000", 2024-09-14T04:39:24.623 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1046000000000001e-05 2024-09-14T04:39:24.623 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.623 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.623 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:39:24.623 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.583596+0000", 2024-09-14T04:39:24.623 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9820000000000002e-06 2024-09-14T04:39:24.623 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.623 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.624 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:39:24.624 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.583634+0000", 2024-09-14T04:39:24.624 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.7914999999999999e-05 2024-09-14T04:39:24.624 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.624 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.624 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:39:24.624 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.583792+0000", 2024-09-14T04:39:24.624 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015831100000000001 2024-09-14T04:39:24.624 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.624 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.624 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:39:24.624 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.677223+0000", 2024-09-14T04:39:24.624 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.093430314 2024-09-14T04:39:24.625 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.625 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.625 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:39:24.625 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.685636+0000", 2024-09-14T04:39:24.625 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0084133509999999995 2024-09-14T04:39:24.625 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.625 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.625 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:39:24.625 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.685677+0000", 2024-09-14T04:39:24.625 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.1254000000000002e-05 2024-09-14T04:39:24.625 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.625 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.625 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:39:24.625 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.690078+0000", 2024-09-14T04:39:24.626 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0044006840000000002 2024-09-14T04:39:24.626 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.626 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.626 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:39:24.626 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.690134+0000", 2024-09-14T04:39:24.626 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6329000000000002e-05 2024-09-14T04:39:24.626 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.626 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.626 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:39:24.626 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.690183+0000", 2024-09-14T04:39:24.626 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9051000000000002e-05 2024-09-14T04:39:24.626 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.626 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:39:24.626 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.627 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.627 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.627 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:76192 53.7 53:e7b2853a:::benchmark_data_smithi039_224949_object76191:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-09-14T04:39:24.627 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:39:14.583728+0000", 2024-09-14T04:39:24.627 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 9.1666564130000001, 2024-09-14T04:39:24.627 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.108719867, 2024-09-14T04:39:24.627 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:39:24.627 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:39:24.627 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:39:24.627 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:39:24.627 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:39:24.627 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:39:24.627 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 76192 2024-09-14T04:39:24.628 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.628 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:39:24.628 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.628 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:39:24.628 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.583728+0000", 2024-09-14T04:39:24.628 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.628 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.628 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.628 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:39:24.628 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.583728+0000", 2024-09-14T04:39:24.628 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:39:24.628 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.629 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.629 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:39:24.629 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.583729+0000", 2024-09-14T04:39:24.629 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7499999999999999e-07 2024-09-14T04:39:24.629 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.629 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.629 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:39:24.629 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.583750+0000", 2024-09-14T04:39:24.629 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1432000000000001e-05 2024-09-14T04:39:24.629 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.629 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.629 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:39:24.629 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.583760+0000", 2024-09-14T04:39:24.630 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.6039999999999999e-06 2024-09-14T04:39:24.630 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.630 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.630 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:39:24.630 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.583767+0000", 2024-09-14T04:39:24.630 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8240000000000002e-06 2024-09-14T04:39:24.630 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.630 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.630 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:39:24.630 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.583819+0000", 2024-09-14T04:39:24.630 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.2284999999999998e-05 2024-09-14T04:39:24.630 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.630 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.631 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:39:24.631 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.676997+0000", 2024-09-14T04:39:24.631 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.093177642000000005 2024-09-14T04:39:24.631 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.631 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.631 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 0,3", 2024-09-14T04:39:24.631 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.677254+0000", 2024-09-14T04:39:24.631 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00025704500000000002 2024-09-14T04:39:24.631 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.631 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.631 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:39:24.631 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.688532+0000", 2024-09-14T04:39:24.631 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.011277957999999999 2024-09-14T04:39:24.631 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.632 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.632 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:39:24.632 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.688745+0000", 2024-09-14T04:39:24.632 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00021327 2024-09-14T04:39:24.632 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.632 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.632 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:39:24.632 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.692301+0000", 2024-09-14T04:39:24.632 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0035559379999999998 2024-09-14T04:39:24.632 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.632 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.632 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:39:24.632 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.692402+0000", 2024-09-14T04:39:24.633 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000101591 2024-09-14T04:39:24.633 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:39:24.633 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:39:24.633 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:39:24.633 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.692448+0000", 2024-09-14T04:39:24.633 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5503000000000001e-05 2024-09-14T04:39:24.633 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.633 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:39:24.633 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.633 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:39:24.633 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:39:24.633 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:39:24.634 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:24.674 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:24.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:24.675 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:24.702 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 49 16 88751 88735 113.15 68.9375 0.0101389 0.00883302 2024-09-14T04:39:24.777 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:24.813 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:24.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:24.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:24.915 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:24.949 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:24.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:24.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:25.050 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:25.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:25.080 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:25.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:25.181 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:25.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:25.214 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:25.214 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:25.242 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:39:25.242 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:39:25.276 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:39:25.276 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:39:25.308 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:39:25.309 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:39:25.309 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:25 smithi039 ceph-mon[106470]: pgmap v2186: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-09-14T04:39:25.309 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:25 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/350034674' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:39:25.309 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:25 smithi039 ceph-mon[111235]: pgmap v2186: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-09-14T04:39:25.309 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:25 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/350034674' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:39:25.315 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:25.343 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:39:25.343 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:39:25.355 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:25.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:25.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:25.379 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:39:25.380 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:39:25.419 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:39:25.419 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:39:25.457 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:25.462 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:39:25.462 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:39:25.497 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:25.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:25.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:25.505 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:39:25.506 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:39:25.599 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:25.629 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:25.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:25.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:25.702 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 50 16 89912 89896 112.338 72.5625 0.0100032 0.00889643 2024-09-14T04:39:25.731 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:25 smithi129 ceph-mon[101316]: pgmap v2186: 89 pgs: 89 active+clean; 5.2 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 79 MiB/s wr, 1.27k op/s 2024-09-14T04:39:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:25 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/350034674' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:39:25.772 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:25.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:25.773 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:25.874 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:25.916 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:25.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:25.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:26.018 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:26.054 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:26.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:26.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:26.156 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:26.192 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:26.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:26.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:26.294 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:26.313 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.0 to 1 (10000) 2024-09-14T04:39:26.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:26.327 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:26.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:26.428 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:26.468 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:26.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:26.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:26.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:26 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1865988883' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:39:26.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:26 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:39:26.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:26 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1865988883' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:39:26.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:26 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:39:26.570 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:26.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:26.606 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:26.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:26.702 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 51 16 91077 91061 111.563 72.8125 0.00787659 0.00895822 2024-09-14T04:39:26.707 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:26.744 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:26.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:26.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:26.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:26 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1865988883' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:39:26.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:26 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:39:26.846 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:26.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:26.887 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:26.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:26.988 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:27.026 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:27.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:27.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:27.128 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:27.176 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:27.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:27.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:27.278 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:27.308 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:27.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:27.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:27.410 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:27.443 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:27.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:27.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:27.453 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:27 smithi039 ceph-mon[106470]: pgmap v2187: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-14T04:39:27.453 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:27 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:39:27.453 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:27 smithi039 ceph-mon[106470]: osdmap e1324: 8 total, 8 up, 8 in 2024-09-14T04:39:27.453 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:27 smithi039 ceph-mon[111235]: pgmap v2187: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-14T04:39:27.453 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:27 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:39:27.453 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:27 smithi039 ceph-mon[111235]: osdmap e1324: 8 total, 8 up, 8 in 2024-09-14T04:39:27.491 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-14T04:39:27.544 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:27.574 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:27.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:27.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:27.675 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:27.702 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 52 16 92235 92219 110.81 72.375 0.0130907 0.00901938 2024-09-14T04:39:27.705 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:27.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:27.706 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:27.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:27 smithi129 ceph-mon[101316]: pgmap v2187: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-14T04:39:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:27 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:39:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:27 smithi129 ceph-mon[101316]: osdmap e1324: 8 total, 8 up, 8 in 2024-09-14T04:39:27.806 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:27.850 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:27.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:27.851 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:27.952 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:27.979 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:39:28.015 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 1 1 2024-09-14T04:39:28.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:28.030 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:28.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:28.131 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:28.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:28.161 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:28.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:28.262 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:28.291 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:28.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:28.292 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:28.299 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:39:28.393 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:28.421 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:28.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:28.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:28.523 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:28.566 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:28.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:28.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:28.668 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:28.703 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 53 16 93347 93331 110.028 69.5 0.0086175 0.00908311 2024-09-14T04:39:28.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:28.711 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:28.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:28.812 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:28.844 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:28.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:28.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:28.863 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:39:28.946 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:28.976 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:28.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:28.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:29.015 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:39:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:39:28] "GET /metrics HTTP/1.1" 200 36741 "" "Prometheus/2.51.0" 2024-09-14T04:39:29.079 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:29.123 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:29.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:29.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:29.225 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:29.259 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:29.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:29.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:29.361 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:29.393 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:29.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:29.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:29.455 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:39:29.495 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:29.554 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:29.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:29.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:29.572 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:29 smithi129 ceph-mon[101316]: pgmap v2189: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s 2024-09-14T04:39:29.656 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:29.661 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:29 smithi039 ceph-mon[106470]: pgmap v2189: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s 2024-09-14T04:39:29.662 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:29 smithi039 ceph-mon[111235]: pgmap v2189: 89 pgs: 89 active+clean; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 983 op/s 2024-09-14T04:39:29.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:29.690 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:29.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:29.704 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 54 16 94513 94497 109.339 72.875 0.00787363 0.0091398 2024-09-14T04:39:29.790 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:29.828 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:29.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:29.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:29.930 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:29.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:29.972 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:29.972 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:30.073 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:30.107 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:30.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:30.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:30.209 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:30.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:30.240 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:30.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:30.341 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:30.379 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:30.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:30.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:30.481 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:30.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:30.512 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:30.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:30.613 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:30.642 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:30.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:30.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:30.704 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 55 16 95660 95644 108.655 71.6875 0.0183438 0.0091986 2024-09-14T04:39:30.743 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:30.784 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:30.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:30.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:30.885 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:30.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:30.921 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:30.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:31.022 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:31.053 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:31.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:31.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:31.155 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:31.189 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:31.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:31.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:31.291 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:31.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:31.348 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:31.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:31.355 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is set 2024-09-14T04:39:31.449 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:31.485 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:31.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:31.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:31.528 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:31 smithi039 ceph-mon[106470]: pgmap v2190: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-14T04:39:31.529 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:31 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/874955025' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:39:31.529 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:31 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:39:31.529 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:31 smithi039 ceph-mon[111235]: pgmap v2190: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-14T04:39:31.529 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:31 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/874955025' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:39:31.529 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:31 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:39:31.588 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:31.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:31.621 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:31.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:31.704 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 56 16 96782 96766 107.967 70.125 0.0149882 0.00925647 2024-09-14T04:39:31.722 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:31.751 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:31.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:31.752 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:31 smithi129 ceph-mon[101316]: pgmap v2190: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s 2024-09-14T04:39:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:31 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/874955025' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:39:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:31 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:39:31.853 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:31.881 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:31.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:31.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:31.983 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:32.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:32.018 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:32.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:32.119 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:32.158 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:32.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:32.159 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:32.260 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:32.292 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:32.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:32.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:32.394 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:32.432 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:32.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:32.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:32.534 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:32.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:32.569 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:32.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:32.670 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:32.699 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:32.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:32.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:32.704 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 57 16 97979 97963 107.385 74.8125 0.0128664 0.00930709 2024-09-14T04:39:32.723 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:32 smithi039 ceph-mon[106470]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:39:32.723 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:32 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:39:32.723 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:32 smithi039 ceph-mon[106470]: osdmap e1325: 8 total, 8 up, 8 in 2024-09-14T04:39:32.724 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:32 smithi039 ceph-mon[111235]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:39:32.724 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:32 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:39:32.724 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:32 smithi039 ceph-mon[111235]: osdmap e1325: 8 total, 8 up, 8 in 2024-09-14T04:39:32.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:32 smithi129 ceph-mon[101316]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:39:32.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:32 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:39:32.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:32 smithi129 ceph-mon[101316]: osdmap e1325: 8 total, 8 up, 8 in 2024-09-14T04:39:32.801 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:32.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:32.844 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:32.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:32.946 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:32.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:32.991 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:32.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:33.092 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:33.128 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:33.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:33.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:33.230 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:33.231 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:39:33.265 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:33.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:33.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:33.366 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:33.415 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:33.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:33.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:33.518 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:33.548 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:33.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:33.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:33.649 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:33.686 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:33.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:33.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:33.704 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 58 16 99156 99140 106.802 73.5625 0.00829084 0.00935844 2024-09-14T04:39:33.719 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:33 smithi039 ceph-mon[106470]: pgmap v2192: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-14T04:39:33.720 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:33 smithi039 ceph-mon[111235]: pgmap v2192: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-14T04:39:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:33 smithi129 ceph-mon[101316]: pgmap v2192: 89 pgs: 89 active+clean; 5.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-14T04:39:33.788 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:33.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:33.845 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:33.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:33.946 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:33.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:33.987 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:33.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:34.088 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:34.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:34.119 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:34.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:34.220 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:34.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:34.273 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:34.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:34.338 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:39:34.338 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:39:34.374 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:34.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:34.404 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:34.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:34.505 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:34.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:34.552 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:34.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:34.653 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:34.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:34.694 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:34.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:34.704 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 59 16 100245 100229 106.145 68.0625 0.014896 0.00941544 2024-09-14T04:39:34.761 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:39:34.795 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:34.824 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:34.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:34.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:34.926 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:34.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:34.970 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:34.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:35.071 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:35.105 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:35.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:35.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:35.207 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:35.245 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:35.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:35.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:35.347 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:35.386 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:35.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:35.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:35.489 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:35.522 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:35.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:35.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:35.624 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:35.657 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:35.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:35.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:35.705 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:2024-09-14T04:39:35.704649+0000 min lat: 0.00122803 max lat: 0.134158 avg lat: 0.00947802 2024-09-14T04:39:35.705 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:39:35.705 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 60 16 101271 101255 105.445 64.125 0.00707374 0.00947802 2024-09-14T04:39:35.761 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:35.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:35 smithi129 ceph-mon[101316]: pgmap v2193: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-14T04:39:35.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:35 smithi039 ceph-mon[106470]: pgmap v2193: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-14T04:39:35.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:35 smithi039 ceph-mon[111235]: pgmap v2193: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-14T04:39:35.801 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:35.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:35.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:35.903 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:35.937 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:35.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:35.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:36.038 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:36.060 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:36.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:36.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:36.162 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:36.189 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:36.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:36.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:36.294 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:36.325 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:36.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:36.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:36.426 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:36.463 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:36.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:36.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:36.565 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:36.598 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:36.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:36.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:36.699 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:36.704 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 61 16 102308 102292 104.779 64.8125 0.0162552 0.00953877 2024-09-14T04:39:36.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:36.731 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:36.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:36.832 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:36.853 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:36.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:36.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:36.955 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:36.992 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:36.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:36.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:37.094 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:37.125 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:37.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:37.126 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:37.227 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:37.260 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:37.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:37.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:37.361 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:37.400 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:37.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:37.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:37.502 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:37.526 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:37.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:37.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:37.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:37 smithi039 ceph-mon[106470]: pgmap v2194: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-14T04:39:37.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:39:37.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:37 smithi039 ceph-mon[111235]: pgmap v2194: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-14T04:39:37.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:39:37.629 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:37.680 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:37.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:37.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:37.705 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 62 16 103291 103275 104.079 61.4375 0.018794 0.0096026 2024-09-14T04:39:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:37 smithi129 ceph-mon[101316]: pgmap v2194: 89 pgs: 89 active+clean; 6.0 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-14T04:39:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:39:37.783 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:37.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:37.826 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:37.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:37.927 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:37.967 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:37.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:37.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:38.062 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-14T04:39:38.070 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:38.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:38.102 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:38.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:38.203 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:38.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:38.251 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:38.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:38.352 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:38.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:38.398 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:38.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:38.499 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:38.545 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:38.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:38.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:38.619 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:39:38.647 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:38.685 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:39:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:39:38] "GET /metrics HTTP/1.1" 200 36741 "" "Prometheus/2.51.0" 2024-09-14T04:39:38.706 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 63 16 104141 104125 103.269 53.125 0.0139496 0.00967701 2024-09-14T04:39:38.708 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:38.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:38.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:38.810 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:38.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:38.864 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:38.864 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:38.965 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:39.011 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:39.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:39.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:39.113 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:39.138 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:39.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:39.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:39.240 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:39.290 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:39.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:39.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:39.384 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.1 to 1 (10000) 2024-09-14T04:39:39.393 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:39.442 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:39.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:39.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:39.543 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:39.575 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:39.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:39.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:39.580 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:39 smithi039 ceph-mon[106470]: pgmap v2195: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-14T04:39:39.580 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:39 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1906612985' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:39:39.580 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:39 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/148130833' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:39:39.580 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:39 smithi039 ceph-mon[111235]: pgmap v2195: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-14T04:39:39.581 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:39 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1906612985' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:39:39.581 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:39 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/148130833' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:39:39.676 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:39.706 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 64 16 105069 105053 102.561 58 0.0204803 0.00974556 2024-09-14T04:39:39.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:39.718 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:39.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:39.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:39 smithi129 ceph-mon[101316]: pgmap v2195: 89 pgs: 89 active+clean; 6.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-14T04:39:39.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:39 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1906612985' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:39:39.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:39 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/148130833' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:39:39.818 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:39.855 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:39.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:39.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:39.957 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:39.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:39.999 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:39.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:40.100 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:40.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:40.132 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:40.132 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:40.233 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:40.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:40.277 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:40.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:40.305 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:39:40.306 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:39:40.306 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:39:40.306 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:39:40.306 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:39:40.306 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:39:40.306 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:39:40.306 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:39:40.307 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:39:40.307 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:39:40.307 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:39:40.307 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:39:40.307 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:39:40.307 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:39:40.307 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:39:40.307 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:39:40.379 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:40.415 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:40.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:40.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:40.517 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:40.555 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:40.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:40.556 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:40.657 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:40.686 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:40.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:40.687 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:40.706 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 65 16 106144 106128 102.017 67.1875 0.00920998 0.00979714 2024-09-14T04:39:40.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:40 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1906612985' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-14T04:39:40.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:40 smithi129 ceph-mon[101316]: osdmap e1326: 8 total, 8 up, 8 in 2024-09-14T04:39:40.788 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:40.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:40 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1906612985' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-14T04:39:40.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:40 smithi039 ceph-mon[106470]: osdmap e1326: 8 total, 8 up, 8 in 2024-09-14T04:39:40.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:40 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1906612985' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-14T04:39:40.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:40 smithi039 ceph-mon[111235]: osdmap e1326: 8 total, 8 up, 8 in 2024-09-14T04:39:40.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:40.829 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:40.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:40.930 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:40.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:40.968 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:40.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:41.069 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:41.111 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:41.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:41.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:41.213 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:41.245 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:41.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:41.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:41.347 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:41.375 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:41.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:41.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:41.477 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:41.533 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:41.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:41.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:41.635 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:41.669 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:41.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:41.670 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:41.706 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 66 16 107258 107242 101.526 69.625 0.02818 0.00984438 2024-09-14T04:39:41.764 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:41 smithi039 ceph-mon[106470]: pgmap v2197: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-14T04:39:41.765 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:41 smithi039 ceph-mon[111235]: pgmap v2197: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-14T04:39:41.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:41 smithi129 ceph-mon[101316]: pgmap v2197: 89 pgs: 89 active+clean; 6.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 83 MiB/s wr, 1.33k op/s 2024-09-14T04:39:41.771 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:41.798 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:41.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:41.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:41.900 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:41.942 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:41.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:41.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:42.043 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:42.086 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:42.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:42.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:42.188 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:42.231 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:42.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:42.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:42.333 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:42.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:42.355 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:42.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:42.456 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:42.484 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:42.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:42.485 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:42.586 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:42.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:42.629 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:42.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:42.706 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 67 16 108296 108280 100.979 64.875 0.0101613 0.00989772 2024-09-14T04:39:42.730 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:42.769 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:42.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:42.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:42.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:42 smithi129 ceph-mon[101316]: pgmap v2198: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-09-14T04:39:42.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:42 smithi039 ceph-mon[106470]: pgmap v2198: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-09-14T04:39:42.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:42 smithi039 ceph-mon[111235]: pgmap v2198: 89 pgs: 89 active+clean; 6.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s 2024-09-14T04:39:42.846 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 2 1 2024-09-14T04:39:42.872 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:42.938 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:42.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:42.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:43.039 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:43.069 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:43.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:43.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:43.171 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:43.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:43.210 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:43.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:43.311 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:43.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:43.365 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:43.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:43.393 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:39:43.466 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:43.506 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:43.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:43.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:43.510 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:39:43.609 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:43.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:43.657 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:43.657 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:43.707 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 68 16 109233 109217 100.356 58.5625 0.042491 0.00995799 2024-09-14T04:39:43.758 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:43.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:43.814 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:43.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:43.915 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:43.949 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:43.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:43.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:44.051 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:44.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:44.085 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:44.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:44.186 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:44.227 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:44.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:44.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:44.329 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:44.383 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:44.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:44.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:44.485 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:44.520 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:44.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:44.525 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:44.595 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is unset 2024-09-14T04:39:44.626 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:44.655 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:44.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:44.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:44.707 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 69 16 110253 110237 99.8252 63.75 0.0102791 0.0100124 2024-09-14T04:39:44.757 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:44.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:44 smithi039 ceph-mon[106470]: pgmap v2199: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-14T04:39:44.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:44 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/167776792' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:39:44.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:44 smithi039 ceph-mon[111235]: pgmap v2199: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-14T04:39:44.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:44 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/167776792' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:39:44.793 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:44.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:44.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:44.895 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:44.932 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:44.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:44.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:45.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:44 smithi129 ceph-mon[101316]: pgmap v2199: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-14T04:39:45.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:44 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/167776792' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:39:45.024 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:39:45.034 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:45.061 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:45.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:45.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:45.162 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:45.196 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:45.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:45.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:45.298 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:45.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:45.341 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:45.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:45.442 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:45.477 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:45.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:45.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:45.579 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:45.608 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:45.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:45.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:45.707 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 70 16 111283 111267 99.3188 64.375 0.0157417 0.0100633 2024-09-14T04:39:45.710 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:45.750 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:45.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:45.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:45.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:45 smithi039 ceph-mon[106470]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:39:45.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:45 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/167776792' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:39:45.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:45 smithi039 ceph-mon[106470]: osdmap e1327: 8 total, 8 up, 8 in 2024-09-14T04:39:45.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:45 smithi039 ceph-mon[111235]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:39:45.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:45 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/167776792' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:39:45.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:45 smithi039 ceph-mon[111235]: osdmap e1327: 8 total, 8 up, 8 in 2024-09-14T04:39:45.852 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:45.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:45.925 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:45.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:46.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:45 smithi129 ceph-mon[101316]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:39:46.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:45 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/167776792' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:39:46.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:45 smithi129 ceph-mon[101316]: osdmap e1327: 8 total, 8 up, 8 in 2024-09-14T04:39:46.026 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:46.064 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:46.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:46.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:46.166 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:46.197 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:46.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:46.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:46.300 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:46.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:46.340 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:46.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:46.441 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:46.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:46.477 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:46.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:46.573 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:39:46.579 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:46.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:46.608 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:46.609 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:46.707 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 71 16 112304 112288 98.8187 63.8125 0.0162072 0.0101141 2024-09-14T04:39:46.709 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:46.748 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:46.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:46.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:46.850 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:46.887 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:46 smithi039 ceph-mon[106470]: pgmap v2201: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-14T04:39:46.887 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:46 smithi039 ceph-mon[111235]: pgmap v2201: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-14T04:39:46.891 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:46.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:46.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:46.993 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:47.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:46 smithi129 ceph-mon[101316]: pgmap v2201: 89 pgs: 89 active+clean; 6.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 69 MiB/s wr, 1.11k op/s 2024-09-14T04:39:47.033 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:47.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:47.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:47.135 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:47.180 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:47.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:47.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:47.281 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:47.317 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:47.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:47.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:47.419 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:47.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:47.462 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:47.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:47.513 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:39:47.513 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [], 2024-09-14T04:39:47.513 INFO:teuthology.orchestra.run.smithi039.stdout: "complaint_time": 30, 2024-09-14T04:39:47.513 INFO:teuthology.orchestra.run.smithi039.stdout: "num_blocked_ops": 0 2024-09-14T04:39:47.513 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:39:47.564 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:47.592 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:47.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:47.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:47.694 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:47.707 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 72 16 113283 113267 98.296 61.1875 0.0167305 0.010168 2024-09-14T04:39:47.728 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:47.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:47.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:47.830 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:47.837 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:39:47.862 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:47.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:47.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:47.964 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:48.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:48.039 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:48.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:48.140 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:48.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:48.178 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:48.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:48.278 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:48.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:48.308 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:48.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:48.409 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:48.444 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:48.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:48.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:48.546 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:48.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:48.586 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:48.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:48.687 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:48.707 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 73 16 114286 114270 97.8082 62.6875 0.0111173 0.0102192 2024-09-14T04:39:48.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:48.717 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:48.717 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:48.818 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:48.850 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:48.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:48.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:48.952 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:48.979 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:48.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:48.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:49.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:48 smithi129 ceph-mon[101316]: pgmap v2202: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 926 op/s 2024-09-14T04:39:49.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:48 smithi039 ceph-mon[106470]: pgmap v2202: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 926 op/s 2024-09-14T04:39:49.043 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:39:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:39:48] "GET /metrics HTTP/1.1" 200 36741 "" "Prometheus/2.51.0" 2024-09-14T04:39:49.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:48 smithi039 ceph-mon[111235]: pgmap v2202: 89 pgs: 89 active+clean; 6.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 926 op/s 2024-09-14T04:39:49.081 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:49.108 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:49.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:49.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:49.210 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:49.244 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:49.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:49.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:49.345 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:49.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:49.387 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:49.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:49.444 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-14T04:39:49.488 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:49.549 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:49.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:49.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:49.651 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:49.708 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 74 16 115256 115240 97.3052 60.625 0.00981658 0.0102717 2024-09-14T04:39:49.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:49.709 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:49.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:49.810 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:49.846 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:49.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:49.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:49.947 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:49.976 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:49.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:49.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:50.078 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:50.124 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:50.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:50.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:50.226 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:50.262 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:50.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:50.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:50.364 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:50.404 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:50.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:50.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:50.506 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:50.542 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:50.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:50.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:50.602 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:39:50.644 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:50.705 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:50.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:50.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:50.708 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 75 16 116223 116207 96.8137 60.4375 0.0108833 0.0103222 2024-09-14T04:39:50.807 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:50.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:50.862 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:50.862 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:50.963 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:50.964 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:50 smithi039 ceph-mon[106470]: pgmap v2203: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-14T04:39:50.964 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:50 smithi039 ceph-mon[111235]: pgmap v2203: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-14T04:39:50.995 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:50.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:50.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:51.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:50 smithi129 ceph-mon[101316]: pgmap v2203: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-14T04:39:51.097 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:51.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:51.133 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:51.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:51.234 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:51.277 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:51.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:51.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:51.380 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:51.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:51.449 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:51.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:51.546 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:39:51.546 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:39:51.551 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:51.558 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:39:51.558 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:39:51.587 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:51.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:51.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:51.595 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:39:51.596 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:39:51.630 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:39:51.630 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:39:51.666 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:39:51.667 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:39:51.690 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:51.698 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:39:51.698 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:39:51.709 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 76 16 117039 117023 96.2097 51 0.0135834 0.0103882 2024-09-14T04:39:51.732 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:39:51.733 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:39:51.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:51.740 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:51.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:51.767 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:39:51.768 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:39:51.768 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:51 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:39:51.768 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:51 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:39:51.768 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:51 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:39:51.769 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:51 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3240423998' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:39:51.769 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:51 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:39:51.769 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:51 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:39:51.769 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:51 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:39:51.769 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:51 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1153582161' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:39:51.769 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:51 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:39:51.769 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:51 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:39:51.769 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:51 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:39:51.769 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:51 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3240423998' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:39:51.769 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:51 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:39:51.769 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:51 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:39:51.770 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:51 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:39:51.770 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:51 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1153582161' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:39:51.841 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:51.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:51.878 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:51.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:51.895 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:51 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:39:51.895 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:51 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:39:51.895 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:51 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:39:51.896 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:51 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3240423998' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:39:51.896 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:51 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:39:51.896 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:51 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:39:51.896 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:51 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:39:51.896 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:51 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1153582161' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:39:51.979 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:52.020 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:52.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:52.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:52.033 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.2 to 1 (10000) 2024-09-14T04:39:52.122 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:52.162 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:52.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:52.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:52.264 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:52.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:52.305 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:52.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:52.406 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:52.436 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:52.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:52.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:52.538 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:52.570 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:52.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:52.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:52.672 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:52.709 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 77 16 118054 118038 95.7841 63.4375 0.0104653 0.0104347 2024-09-14T04:39:52.713 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:52.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:52.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:52.814 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:52.852 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:52.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:52.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:52.954 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:52.991 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:52.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:52.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:53 smithi039 ceph-mon[106470]: pgmap v2204: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s 2024-09-14T04:39:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:53 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-14T04:39:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:53 smithi039 ceph-mon[106470]: osdmap e1328: 8 total, 8 up, 8 in 2024-09-14T04:39:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:53 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:39:53.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:53 smithi039 ceph-mon[111235]: pgmap v2204: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s 2024-09-14T04:39:53.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:53 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-14T04:39:53.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:53 smithi039 ceph-mon[111235]: osdmap e1328: 8 total, 8 up, 8 in 2024-09-14T04:39:53.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:53 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:39:53.096 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:53.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:53.139 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:53.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:53.241 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:53 smithi129 ceph-mon[101316]: pgmap v2204: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 998 op/s 2024-09-14T04:39:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:53 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-14T04:39:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:53 smithi129 ceph-mon[101316]: osdmap e1328: 8 total, 8 up, 8 in 2024-09-14T04:39:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:53 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:39:53.290 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:53.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:53.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:53.392 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:53.433 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:53.434 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:39:53.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:53.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:53.535 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:53.576 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:53.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:53.577 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:53.678 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:53.701 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:53.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:53.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:53.709 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 78 16 118983 118967 95.3005 58.0625 0.0671948 0.0104874 2024-09-14T04:39:53.803 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:53.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:53.838 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:53.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:53.939 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:53.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:53.969 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:53.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:54.070 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:54.106 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:54.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:54.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:54.208 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:54.247 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:54.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:54.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:54.348 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:54.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:54.400 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:54.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:54.501 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:54.541 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:54.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:54.542 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:54.642 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:54.651 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 3 1 2024-09-14T04:39:54.666 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:54.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:54.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:54.709 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 79 16 119991 119975 94.8916 63 0.00674594 0.010533 2024-09-14T04:39:54.768 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:54.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:54.837 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:54.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:54.938 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:54.993 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:54.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:54.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:55.095 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:55.126 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:55.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:55.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:55.170 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:39:55.212 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:55 smithi039 ceph-mon[106470]: pgmap v2206: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 963 op/s 2024-09-14T04:39:55.212 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:55 smithi039 ceph-mon[111235]: pgmap v2206: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 963 op/s 2024-09-14T04:39:55.227 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:55.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:55.260 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:55.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:55.361 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:55.391 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:55.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:55.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:55.492 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:55.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:55 smithi129 ceph-mon[101316]: pgmap v2206: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 60 MiB/s wr, 963 op/s 2024-09-14T04:39:55.524 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:55.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:55.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:55.625 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:55.668 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:55.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:55.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:55.709 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:2024-09-14T04:39:55.709447+0000 min lat: 0.00122803 max lat: 0.134158 avg lat: 0.0105901 2024-09-14T04:39:55.709 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:39:55.709 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 80 16 120852 120836 94.3781 53.8125 0.00979547 0.0105901 2024-09-14T04:39:55.770 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:55.808 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:55.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:55.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:55.910 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:55.955 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:55.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:55.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:56.057 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:56.101 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:56.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:56.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:56.203 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:56.235 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:56.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:56.236 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:56.337 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:56.381 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:56.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:56.382 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:56.483 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:56.528 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:56.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:56.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:56.630 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:56.670 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:56.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:56.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:56.709 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 81 16 121794 121778 93.9398 58.875 0.00498274 0.0106393 2024-09-14T04:39:56.772 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:56.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:56.812 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:56.812 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:56.881 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:39:56.913 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:56.968 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:56.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:56.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:57.070 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:57.110 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:57.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:57.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:57.212 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:57.257 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:57.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:57.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:57.282 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:57 smithi039 ceph-mon[106470]: pgmap v2207: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2024-09-14T04:39:57.282 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:57 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:39:57.282 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:57 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:39:57.282 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:57 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3403570186' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:39:57.282 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:57 smithi039 ceph-mon[111235]: pgmap v2207: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2024-09-14T04:39:57.282 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:57 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:39:57.282 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:57 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:39:57.282 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:57 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3403570186' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:39:57.359 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:57.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:57.399 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:57.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:57.500 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:57.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:57 smithi129 ceph-mon[101316]: pgmap v2207: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 61 MiB/s wr, 981 op/s 2024-09-14T04:39:57.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:57 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:39:57.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:57 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:39:57.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:57 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3403570186' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:39:57.534 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:57.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:57.535 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:57.602 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is unset 2024-09-14T04:39:57.636 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:57.693 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:57.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:57.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:57.709 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 82 16 122684 122668 93.4724 55.625 0.00859334 0.0106934 2024-09-14T04:39:57.794 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:57.829 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:57.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:57.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:57.931 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:57.957 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:57.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:57.958 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:57.980 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:39:57.981 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:39:58.059 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:39:58.084 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:58.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:58.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:39:58.185 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:39:58.229 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:58.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:58.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:39:58.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:58 smithi039 ceph-mon[106470]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:39:58.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:58 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T04:39:58.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:58 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3403570186' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:39:58.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:58 smithi039 ceph-mon[106470]: osdmap e1329: 8 total, 8 up, 8 in 2024-09-14T04:39:58.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:58 smithi039 ceph-mon[111235]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:39:58.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:58 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T04:39:58.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:58 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3403570186' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:39:58.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:58 smithi039 ceph-mon[111235]: osdmap e1329: 8 total, 8 up, 8 in 2024-09-14T04:39:58.331 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:58.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:58.356 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:58.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:58.457 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:58.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:58.493 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:58.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:58.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:58 smithi129 ceph-mon[101316]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:39:58.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:58 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T04:39:58.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:58 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3403570186' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:39:58.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:58 smithi129 ceph-mon[101316]: osdmap e1329: 8 total, 8 up, 8 in 2024-09-14T04:39:58.593 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:58.630 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:58.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:58.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:58.710 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 83 16 123613 123597 93.0458 58.0625 0.0129639 0.0107418 2024-09-14T04:39:58.732 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:58.775 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:58.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:58.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:58.876 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:58.912 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:58.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:58.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:59.014 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:39:59.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:39:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:39:58] "GET /metrics HTTP/1.1" 200 36746 "" "Prometheus/2.51.0" 2024-09-14T04:39:59.041 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:59.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:59.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:39:59.143 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:59.179 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:59.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:59.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:59.281 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:39:59.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:59.319 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:59.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:39:59.420 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:59.421 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:39:59 smithi129 ceph-mon[101316]: pgmap v2208: 89 pgs: 89 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 61 MiB/s wr, 968 op/s 2024-09-14T04:39:59.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:59.457 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:59.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:39:59.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:39:59 smithi039 ceph-mon[106470]: pgmap v2208: 89 pgs: 89 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 61 MiB/s wr, 968 op/s 2024-09-14T04:39:59.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:39:59 smithi039 ceph-mon[111235]: pgmap v2208: 89 pgs: 89 active+clean; 7.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 61 MiB/s wr, 968 op/s 2024-09-14T04:39:59.559 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:39:59.597 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:59.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:59.599 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:39:59.699 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:39:59.710 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 84 16 124491 124475 92.5914 54.875 0.0208708 0.0107948 2024-09-14T04:39:59.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:59.732 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:59.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:39:59.833 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:39:59.866 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:39:59.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:59.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:39:59.968 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:39:59.997 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:39:59.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:39:59.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:00.100 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:00.132 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:00.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:00.133 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:00.139 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:00 smithi129 ceph-mon[101316]: overall HEALTH_OK 2024-09-14T04:40:00.234 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:00.271 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:00.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:00.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:00.373 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:00.410 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:00.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:00.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:00.417 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:00 smithi039 ceph-mon[106470]: overall HEALTH_OK 2024-09-14T04:40:00.418 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:00 smithi039 ceph-mon[111235]: overall HEALTH_OK 2024-09-14T04:40:00.511 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:00.528 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:40:00.569 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:00.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:00.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:00.671 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:00.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:00.706 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:00.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:00.710 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 85 16 125413 125397 92.18 57.625 0.00763541 0.0108427 2024-09-14T04:40:00.807 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:00.842 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:00.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:00.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:00.943 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:00.985 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:00.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:00.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:01.086 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:01.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:01.137 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:01.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:01.238 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:01.275 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:01.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:01.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:01.317 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:01 smithi129 ceph-mon[101316]: pgmap v2210: 89 pgs: 89 active+clean; 7.5 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-14T04:40:01.319 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:01 smithi039 ceph-mon[106470]: pgmap v2210: 89 pgs: 89 active+clean; 7.5 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-14T04:40:01.320 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:01 smithi039 ceph-mon[111235]: pgmap v2210: 89 pgs: 89 active+clean; 7.5 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-14T04:40:01.322 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-14T04:40:01.377 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:01.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:01.414 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:01.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:01.515 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:01.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:01.544 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:01.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:01.645 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:01.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:01.685 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:01.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:01.711 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 86 15 126344 126329 91.784 58.25 0.0122718 0.01089 2024-09-14T04:40:01.786 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:01.830 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:01.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:01.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:01.932 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:01.972 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:01.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:01.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:02.074 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:02.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:02.117 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:02.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:02.218 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:02.259 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:02.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:02.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:02.361 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:02.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:02.400 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:02.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:02.501 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:02.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:02.540 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:02.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:02.641 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:02.693 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:02.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:02.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:02.712 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 87 16 127321 127305 91.4302 61 0.00995075 0.0109325 2024-09-14T04:40:02.795 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:02.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:02.841 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:02.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:02.942 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:02.971 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:02.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:02.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:03.072 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:03.098 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:03.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:03.099 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:03.132 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:40:03.200 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:03.230 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:03.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:03.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:03.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:03 smithi039 ceph-mon[106470]: pgmap v2211: 89 pgs: 89 active+clean; 7.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 62 MiB/s wr, 990 op/s 2024-09-14T04:40:03.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:03 smithi039 ceph-mon[111235]: pgmap v2211: 89 pgs: 89 active+clean; 7.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 62 MiB/s wr, 990 op/s 2024-09-14T04:40:03.332 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:03.363 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:03.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:03.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:03.466 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:03.503 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:03.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:03.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:03.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:03 smithi129 ceph-mon[101316]: pgmap v2211: 89 pgs: 89 active+clean; 7.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 62 MiB/s wr, 990 op/s 2024-09-14T04:40:03.605 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:03.634 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:03.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:03.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:03.712 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 88 16 128227 128211 91.0339 56.625 0.0138896 0.0109798 2024-09-14T04:40:03.736 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:03.766 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:03.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:03.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:03.868 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:03.911 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:03.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:03.912 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:04.013 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:04.062 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:04.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:04.063 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:04.093 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.3 to 1 (10000) 2024-09-14T04:40:04.165 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:04.192 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:40:04.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:04.214 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:04.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:04.217 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:04 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/872980891' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:40:04.218 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:04 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/872980891' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:40:04.315 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:04.345 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:04.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:04.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:04.448 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:04.485 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:04.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:04.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:04.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:04 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/872980891' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:40:04.587 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:04.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:04.663 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:04.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:04.712 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 89 16 129148 129132 90.6576 57.5625 0.00839951 0.0110256 2024-09-14T04:40:04.764 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:04.794 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:04.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:04.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:04.896 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:04.939 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:04.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:04.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:05.041 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:05.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:05.079 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:05.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:05.180 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:05.214 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:05.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:05.215 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:05.234 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:05 smithi039 ceph-mon[106470]: pgmap v2212: 89 pgs: 89 active+clean; 7.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 58 MiB/s wr, 926 op/s 2024-09-14T04:40:05.234 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:05 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/872980891' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-14T04:40:05.234 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:05 smithi039 ceph-mon[106470]: osdmap e1330: 8 total, 8 up, 8 in 2024-09-14T04:40:05.234 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:05 smithi039 ceph-mon[111235]: pgmap v2212: 89 pgs: 89 active+clean; 7.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 58 MiB/s wr, 926 op/s 2024-09-14T04:40:05.235 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:05 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/872980891' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-14T04:40:05.235 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:05 smithi039 ceph-mon[111235]: osdmap e1330: 8 total, 8 up, 8 in 2024-09-14T04:40:05.315 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:05.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:05.349 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:05.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:05.450 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:05.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:05.495 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:05.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:05.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:05 smithi129 ceph-mon[101316]: pgmap v2212: 89 pgs: 89 active+clean; 7.7 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 58 MiB/s wr, 926 op/s 2024-09-14T04:40:05.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:05 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/872980891' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-14T04:40:05.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:05 smithi129 ceph-mon[101316]: osdmap e1330: 8 total, 8 up, 8 in 2024-09-14T04:40:05.596 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:05.628 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:05.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:05.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:05.713 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Total time run: 90.0103 2024-09-14T04:40:05.713 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Total writes made: 130090 2024-09-14T04:40:05.713 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Write size: 65536 2024-09-14T04:40:05.713 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Object size: 65536 2024-09-14T04:40:05.713 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Bandwidth (MB/sec): 90.3299 2024-09-14T04:40:05.713 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Stddev Bandwidth: 38.9943 2024-09-14T04:40:05.713 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Max bandwidth (MB/sec): 243.062 2024-09-14T04:40:05.713 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Min bandwidth (MB/sec): 51 2024-09-14T04:40:05.713 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Average IOPS: 1445 2024-09-14T04:40:05.714 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Stddev IOPS: 623.91 2024-09-14T04:40:05.714 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Max IOPS: 3889 2024-09-14T04:40:05.714 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Min IOPS: 816 2024-09-14T04:40:05.714 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Average Latency(s): 0.011066 2024-09-14T04:40:05.714 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Stddev Latency(s): 0.00811048 2024-09-14T04:40:05.714 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Max latency(s): 0.134158 2024-09-14T04:40:05.714 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Min latency(s): 0.00122803 2024-09-14T04:40:05.719 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Cleaning up (deleting benchmark objects) 2024-09-14T04:40:05.729 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:05.765 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:05.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:05.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:05.867 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:05.893 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:05.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:05.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:05.995 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:06.026 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:06.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:06.027 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:06.128 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:06.154 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:06.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:06.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:06.257 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:06.297 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:06.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:06.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:06.400 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:06.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:06.438 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:06.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:06.539 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:06.584 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:06.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:06.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:06.680 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:40:06.680 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:40:06.686 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:06.714 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:40:06.714 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:40:06.715 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:06.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:06.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:06.749 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:40:06.749 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:40:06.796 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:40:06.796 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:40:06.806 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:40:06.806 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:40:06.816 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:06.831 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:40:06.831 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:40:06.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:06.843 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:06.843 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:06.872 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:40:06.872 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:40:06.908 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:40:06.908 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:40:06.943 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:40:06.943 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:40:06.944 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:06.969 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:06.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:06.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:07.071 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:07.102 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:07.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:07.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:07.204 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:07.241 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:07.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:07.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:07.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:07 smithi039 ceph-mon[106470]: pgmap v2214: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-14T04:40:07.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:07 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3709210541' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:40:07.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:07 smithi039 ceph-mon[111235]: pgmap v2214: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-14T04:40:07.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:07 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3709210541' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:40:07.343 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:07.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:07.380 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:07.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:07.481 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:07.507 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:07.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:07.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:07.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:07 smithi129 ceph-mon[101316]: pgmap v2214: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s 2024-09-14T04:40:07.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:07 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3709210541' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:40:07.522 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 4 1 2024-09-14T04:40:07.609 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:07.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:07.657 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:07.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:07.758 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:07.792 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:07.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:07.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:07.894 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:07.922 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:07.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:07.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:08.024 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:08.069 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:08.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:08.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:08.171 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:08.216 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:08.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:08.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:08.318 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:08.353 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:08.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:08.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:08.438 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:40:08.438 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:08 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:40:08.438 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:08 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:40:08.438 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:08 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:40:08.456 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:08.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:08.481 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:08.482 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:08.582 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:08.628 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:08.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:08.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:08.730 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:08.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:08 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:40:08.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:08 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:40:08.775 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:08.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:08.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:08.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:40:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:40:08] "GET /metrics HTTP/1.1" 200 36748 "" "Prometheus/2.51.0" 2024-09-14T04:40:08.877 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:08.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:08.911 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:08.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:09.012 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:09.040 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:09.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:09.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:09.141 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:09.178 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:09.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:09.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:09.275 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-14T04:40:09.279 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:09.298 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:40:09.304 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:09.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:09.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:09.406 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:09.453 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:09.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:09.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:09.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:09 smithi039 ceph-mon[106470]: pgmap v2215: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 56 MiB/s wr, 930 op/s 2024-09-14T04:40:09.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:09 smithi039 ceph-mon[111235]: pgmap v2215: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 56 MiB/s wr, 930 op/s 2024-09-14T04:40:09.556 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:09.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:09.627 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:09.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:09.728 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:09.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:09.768 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:09.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:09.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:09 smithi129 ceph-mon[101316]: pgmap v2215: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 56 MiB/s wr, 930 op/s 2024-09-14T04:40:09.869 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:09.893 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:09.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:09.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:09.996 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:10.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:10.039 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:10.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:10.141 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:10.173 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:10.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:10.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:10.275 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:10.304 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:10.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:10.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:10.406 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:10.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:10.453 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:10.453 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:10.481 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:40:10.554 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:10.602 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:10.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:10.603 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:10.704 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:10.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:10.729 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:10.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:10.830 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:10.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:10.867 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:10.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:10.968 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:10.990 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:10.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:10.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:11.092 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:11.130 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:11.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:11.131 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:11.232 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:11.270 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:11.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:11.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:11.372 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:11.410 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:11.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:11.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:11.512 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:11.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:11 smithi039 ceph-mon[106470]: pgmap v2216: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 44 MiB/s wr, 932 op/s 2024-09-14T04:40:11.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:11 smithi039 ceph-mon[111235]: pgmap v2216: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 44 MiB/s wr, 932 op/s 2024-09-14T04:40:11.553 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:11.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:11.554 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:11.655 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:11.702 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:11.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:11.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:11.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:11 smithi129 ceph-mon[101316]: pgmap v2216: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 44 MiB/s wr, 932 op/s 2024-09-14T04:40:11.805 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:11.852 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:11.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:11.853 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:11.925 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:40:11.949 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:40:11.954 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:11.980 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:11.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:11.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:12.082 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:12.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:12.115 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:12.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:12.216 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:12.255 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:12.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:12.256 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:12.357 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:12.386 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:12.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:12.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:12.488 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:12.531 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:12.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:12.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:12.633 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:12.665 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:12.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:12.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:12.695 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:40:12.767 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:12.802 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:12.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:12.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:12.904 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:12.935 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:12.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:12.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:13.037 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:13.075 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:13.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:13.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:13.177 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:13.206 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:13.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:13.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:13.308 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:13.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:13.345 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:13.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:13.446 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:13.481 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:13.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:13.482 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:13.583 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:13.613 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:13.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:13.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:13.714 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:13.757 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:13.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:13.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:13.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:13 smithi129 ceph-mon[101316]: pgmap v2217: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 41 MiB/s wr, 874 op/s 2024-09-14T04:40:13.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:13 smithi039 ceph-mon[106470]: pgmap v2217: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 41 MiB/s wr, 874 op/s 2024-09-14T04:40:13.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:13 smithi039 ceph-mon[111235]: pgmap v2217: 89 pgs: 89 active+clean; 7.8 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 41 MiB/s wr, 874 op/s 2024-09-14T04:40:13.859 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:13.893 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:13.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:13.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:13.995 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:14.029 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:14.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:14.030 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:14.131 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:14.166 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:14.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:14.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:14.268 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:14.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:14.300 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:14.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:14.401 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:14.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:14.440 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:14.440 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:14.541 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:14.572 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:14.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:14.573 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:14.674 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:14.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:14.727 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:14.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:14.829 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:14.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:14.882 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:14.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:14.983 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:15.018 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:15.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:15.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:15.120 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:15.149 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:15.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:15.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:15.250 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:15.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:15.306 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:15.306 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:15.394 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:15 smithi039 ceph-mon[106470]: pgmap v2218: 89 pgs: 89 active+clean; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 23 MiB/s wr, 921 op/s 2024-09-14T04:40:15.394 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:15 smithi039 ceph-mon[111235]: pgmap v2218: 89 pgs: 89 active+clean; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 23 MiB/s wr, 921 op/s 2024-09-14T04:40:15.407 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:15.449 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:15.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:15.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:15.551 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:15.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:15.583 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:15.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:15.684 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:15.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:15.713 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:15.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:15.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:15 smithi129 ceph-mon[101316]: pgmap v2218: 89 pgs: 89 active+clean; 7.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 102 B/s rd, 23 MiB/s wr, 921 op/s 2024-09-14T04:40:15.814 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:15.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:15.855 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:15.856 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:15.956 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:15.995 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:15.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:15.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:16.097 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:16.131 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:16.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:16.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:16.233 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:16.283 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:16.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:16.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:16.385 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:16.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:16.416 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:16.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:16.517 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:16.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:16.548 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:16.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:16.649 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:16.688 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:16.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:16.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:16.790 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:16.832 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:16.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:16.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:16.934 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:16.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:16.972 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:16.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:17.073 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:17.115 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:17.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:17.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:17.217 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:17.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:17.281 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:17.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:17.382 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:17.437 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:17.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:17.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:17.525 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:17 smithi039 ceph-mon[106470]: pgmap v2219: 89 pgs: 89 active+clean; 7.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 89 B/s rd, 20 MiB/s wr, 903 op/s 2024-09-14T04:40:17.525 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:17 smithi039 ceph-mon[111235]: pgmap v2219: 89 pgs: 89 active+clean; 7.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 89 B/s rd, 20 MiB/s wr, 903 op/s 2024-09-14T04:40:17.539 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:17.588 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:17.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:17.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:17.690 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:17.726 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:17.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:17.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:17.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:17 smithi129 ceph-mon[101316]: pgmap v2219: 89 pgs: 89 active+clean; 7.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 89 B/s rd, 20 MiB/s wr, 903 op/s 2024-09-14T04:40:17.829 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:17.869 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:17.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:17.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:17.970 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:18.004 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:18.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:18.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:18.107 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:18.155 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:18.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:18.157 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:18.257 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:18.286 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:18.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:18.287 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:18.338 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.4 to 1 (10000) 2024-09-14T04:40:18.388 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:18.415 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:18 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2719078259' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:40:18.415 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:18 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:40:18.415 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:18 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2719078259' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:40:18.415 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:18 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:40:18.418 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:18.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:18.419 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:18.520 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:18.548 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:18.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:18.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:18.650 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:18.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:18.688 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:18.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:18.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:18 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2719078259' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:40:18.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:18 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:40:18.788 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:18.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:40:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:40:18] "GET /metrics HTTP/1.1" 200 36748 "" "Prometheus/2.51.0" 2024-09-14T04:40:18.829 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:18.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:18.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:18.931 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:18.976 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:18.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:18.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:19.078 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:19.108 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:19.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:19.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:19.209 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:19.235 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:19.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:19.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:19.340 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:19.371 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:19.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:19.372 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:19.473 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:19.507 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:19.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:19.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:19.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:19 smithi039 ceph-mon[106470]: pgmap v2220: 89 pgs: 89 active+clean; 7.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 13 MiB/s wr, 863 op/s 2024-09-14T04:40:19.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:19 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:40:19.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:19 smithi039 ceph-mon[106470]: osdmap e1331: 8 total, 8 up, 8 in 2024-09-14T04:40:19.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:19 smithi039 ceph-mon[111235]: pgmap v2220: 89 pgs: 89 active+clean; 7.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 13 MiB/s wr, 863 op/s 2024-09-14T04:40:19.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:19 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:40:19.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:19 smithi039 ceph-mon[111235]: osdmap e1331: 8 total, 8 up, 8 in 2024-09-14T04:40:19.609 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:19.636 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:19.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:19.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:19.737 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:19.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:19 smithi129 ceph-mon[101316]: pgmap v2220: 89 pgs: 89 active+clean; 7.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 85 B/s rd, 13 MiB/s wr, 863 op/s 2024-09-14T04:40:19.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:19 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:40:19.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:19 smithi129 ceph-mon[101316]: osdmap e1331: 8 total, 8 up, 8 in 2024-09-14T04:40:19.775 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:19.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:19.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:19.877 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:19.913 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:19.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:19.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:20.015 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:20.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:20.053 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:20.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:20.156 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:20.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:20.198 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:20.198 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:20.257 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 5 1 2024-09-14T04:40:20.299 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:20.329 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:20.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:20.330 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:20.374 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is set 2024-09-14T04:40:20.423 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:20 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3551377568' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:40:20.423 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:20 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:40:20.428 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:20 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3551377568' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:40:20.428 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:20 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:40:20.431 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:20.469 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:20.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:20.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:20.571 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:20.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:20.604 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:20.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:20.704 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:20.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:20.736 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:20.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:20.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:20 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3551377568' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:40:20.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:20 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:40:20.838 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:20.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:20.876 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:20.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:20.977 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:21.017 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:21.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:21.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:21.119 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:21.147 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:21.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:21.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:21.249 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:21.275 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:21.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:21.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:21.377 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:21.418 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:21.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:21.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:21.491 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:21 smithi039 ceph-mon[106470]: pgmap v2222: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 925 op/s 2024-09-14T04:40:21.491 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:21 smithi039 ceph-mon[106470]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:40:21.491 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:21 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:40:21.491 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:21 smithi039 ceph-mon[106470]: osdmap e1332: 8 total, 8 up, 8 in 2024-09-14T04:40:21.491 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:21 smithi039 ceph-mon[111235]: pgmap v2222: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 925 op/s 2024-09-14T04:40:21.491 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:21 smithi039 ceph-mon[111235]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:40:21.491 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:21 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:40:21.491 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:21 smithi039 ceph-mon[111235]: osdmap e1332: 8 total, 8 up, 8 in 2024-09-14T04:40:21.521 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:21.550 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:21.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:21.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:21.652 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:21.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:21.699 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:21.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:21.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:21 smithi129 ceph-mon[101316]: pgmap v2222: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 925 op/s 2024-09-14T04:40:21.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:21 smithi129 ceph-mon[101316]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:40:21.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:21 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:40:21.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:21 smithi129 ceph-mon[101316]: osdmap e1332: 8 total, 8 up, 8 in 2024-09-14T04:40:21.800 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:21.820 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:40:21.820 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [], 2024-09-14T04:40:21.820 INFO:teuthology.orchestra.run.smithi039.stdout: "complaint_time": 30, 2024-09-14T04:40:21.820 INFO:teuthology.orchestra.run.smithi039.stdout: "num_blocked_ops": 0 2024-09-14T04:40:21.820 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:40:21.828 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:21.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:21.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:21.930 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:21.964 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:21.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:21.965 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:22.066 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:22.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:22.101 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:22.102 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:22.202 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:22.245 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:22.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:22.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:22.347 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:22.389 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:22.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:22.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:22.492 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:22.534 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:22.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:22.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:22.636 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:22.666 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:22.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:22.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:22.768 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:22.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:22.806 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:22.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:22.907 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:22.939 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:22.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:22.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:22.972 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-14T04:40:23.040 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:23.071 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:23.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:23.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:23.172 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:23.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:23.232 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:23.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:23.264 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:40:23.333 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:23.382 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:23.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:23.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:23.485 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:23.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:23.561 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:23.561 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:23.662 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:23.689 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:23.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:23.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:23.704 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:23 smithi039 ceph-mon[106470]: pgmap v2224: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 744 op/s 2024-09-14T04:40:23.704 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:23 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:40:23.705 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:40:23.705 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:23 smithi039 ceph-mon[111235]: pgmap v2224: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 744 op/s 2024-09-14T04:40:23.705 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:23 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:40:23.705 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:40:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:23 smithi129 ceph-mon[101316]: pgmap v2224: 89 pgs: 89 active+clean; 7.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 744 op/s 2024-09-14T04:40:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:23 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:40:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:40:23.791 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:23.828 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:23.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:23.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:23.929 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:23.992 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:23.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:23.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:24.094 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:24.126 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:24.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:24.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:24.228 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:24.278 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:24.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:24.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:24.289 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:40:24.293 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:40:24.336 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:40:24.336 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:40:24.380 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:24.387 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:40:24.387 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:40:24.426 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:40:24.427 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:40:24.447 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:24.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:24.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:24.474 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:40:24.474 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:40:24.507 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:40:24.508 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:40:24.544 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:40:24.545 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:40:24.549 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:24.581 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:40:24.581 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:40:24.589 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:24.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:24.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:24.611 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:40:24.612 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:40:24.651 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:24 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:40:24.651 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:24 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:40:24.651 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:24 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:40:24.652 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:24 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:40:24.652 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:24 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:40:24.652 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:24 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1924146982' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:40:24.652 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:24 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:40:24.652 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:24 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:40:24.652 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:24 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:40:24.652 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:24 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:40:24.652 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:24 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:40:24.652 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:24 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1924146982' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:40:24.692 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:24.729 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:24.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:24.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:24.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:24 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:40:24.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:24 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:40:24.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:24 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:40:24.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:24 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:40:24.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:24 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:40:24.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:24 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1924146982' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:40:24.831 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:24.878 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:24.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:24.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:24.980 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:25.005 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:25.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:25.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:25.107 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:25.132 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:25.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:25.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:25.234 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:25.278 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:25.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:25.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:25.380 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:25.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:25.425 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:25.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:25.526 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:25.533 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:25 smithi039 ceph-mon[106470]: pgmap v2225: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-14T04:40:25.533 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:25 smithi039 ceph-mon[111235]: pgmap v2225: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-14T04:40:25.558 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:25.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:25.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:25.660 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:25.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:25.701 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:25.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:25.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:25 smithi129 ceph-mon[101316]: pgmap v2225: 89 pgs: 89 active+clean; 7.0 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-14T04:40:25.802 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:25.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:25.837 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:25.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:25.938 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:25.970 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:25.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:25.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:26.072 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:26.110 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:26.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:26.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:26.212 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:26.249 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:26.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:26.250 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:26.351 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:26.380 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:26.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:26.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:26.482 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:26.516 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:26.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:26.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:26.617 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:26.653 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:26.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:26.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:26.756 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:26.786 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:26.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:26.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:26.887 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:26.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:26.929 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:26.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:27.030 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:27.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:27.071 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:27.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:27.172 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:27.207 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:27.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:27.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:27.309 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:27.365 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:27.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:27.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:27.470 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:27.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:27.516 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:27.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:27.561 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:27 smithi039 ceph-mon[106470]: pgmap v2226: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-14T04:40:27.562 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:27 smithi039 ceph-mon[111235]: pgmap v2226: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-14T04:40:27.616 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:27.658 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:27.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:27.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:27.760 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:27.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:27 smithi129 ceph-mon[101316]: pgmap v2226: 89 pgs: 89 active+clean; 7.0 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-14T04:40:27.789 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:27.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:27.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:27.891 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:27.936 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:27.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:27.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:28.038 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:28.067 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:28.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:28.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:28.076 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:40:28.111 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:40:28.111 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:40:28.169 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:28.209 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:28.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:28.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:28.313 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:28.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:28.366 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:28.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:28.431 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:40:28.467 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:28.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:28.547 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:28.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:28.648 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:28.711 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:28.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:28.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:28.762 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:40:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:40:28] "GET /metrics HTTP/1.1" 200 36746 "" "Prometheus/2.51.0" 2024-09-14T04:40:28.813 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:28.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:28.849 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:28.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:28.950 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:28.987 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:28.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:28.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:29.089 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:29.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:29.125 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:29.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:29.149 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-14T04:40:29.226 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:29.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:29.266 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:29.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:29.368 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:29.425 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:29.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:29.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:29.442 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is set 2024-09-14T04:40:29.453 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:29 smithi129 ceph-mon[101316]: pgmap v2227: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-09-14T04:40:29.454 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:29 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/746523697' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:40:29.454 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:29 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:40:29.493 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:40:29.528 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:29.570 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:29.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:29.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:29.672 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:29.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:29.710 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:29.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:29.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:29 smithi039 ceph-mon[106470]: pgmap v2227: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-09-14T04:40:29.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:29 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/746523697' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:40:29.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:29 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:40:29.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:29 smithi039 ceph-mon[111235]: pgmap v2227: 89 pgs: 89 active+clean; 6.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 700 op/s 2024-09-14T04:40:29.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:29 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/746523697' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:40:29.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:29 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:40:29.812 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:29.862 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:29.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:29.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:29.964 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:30.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:30.008 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:30.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:30.109 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:30.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:30.143 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:30.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:30.244 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:30.285 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:30.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:30.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:30.388 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:30.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:30.420 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:30.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:30.521 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:30.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:30 smithi039 ceph-mon[106470]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:40:30.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:30 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:40:30.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:30 smithi039 ceph-mon[106470]: osdmap e1333: 8 total, 8 up, 8 in 2024-09-14T04:40:30.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:30 smithi039 ceph-mon[106470]: pgmap v2229: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 877 op/s 2024-09-14T04:40:30.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:30 smithi039 ceph-mon[111235]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:40:30.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:30 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:40:30.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:30 smithi039 ceph-mon[111235]: osdmap e1333: 8 total, 8 up, 8 in 2024-09-14T04:40:30.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:30 smithi039 ceph-mon[111235]: pgmap v2229: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 877 op/s 2024-09-14T04:40:30.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:30.558 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:30.558 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:30.659 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:30.693 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:30.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:30.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:30.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:30 smithi129 ceph-mon[101316]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:40:30.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:30 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:40:30.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:30 smithi129 ceph-mon[101316]: osdmap e1333: 8 total, 8 up, 8 in 2024-09-14T04:40:30.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:30 smithi129 ceph-mon[101316]: pgmap v2229: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 877 op/s 2024-09-14T04:40:30.794 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:30.831 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:30.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:30.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:30.933 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:30.986 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:30.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:30.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:31.088 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:31.134 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:40:31.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:31.139 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:31.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:31.240 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:31.270 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:31.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:31.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:31.372 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:31.406 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:31.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:31.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:31.508 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:31.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:31.542 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:31.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:31.643 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:31.679 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:31.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:31.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:31.781 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:31.814 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:40:31.814 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:40:31.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:31.820 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:31.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:31.921 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:31.968 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:31.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:31.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:32.070 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:32.106 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:32.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:32.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:32.208 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:32.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:32.243 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:32.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:32.344 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:32.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:32.378 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:32.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:32.479 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:32.511 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:32.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:32.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:32.567 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-14T04:40:32.613 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:32.614 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.5 to 1 (10000) 2024-09-14T04:40:32.644 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:32.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:32.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:32.746 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:32.786 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:32.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:32.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:32.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:32 smithi039 ceph-mon[106470]: pgmap v2230: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2024-09-14T04:40:32.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:32 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/533046571' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:40:32.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:32 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:40:32.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:32 smithi039 ceph-mon[111235]: pgmap v2230: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2024-09-14T04:40:32.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:32 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/533046571' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:40:32.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:32 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:40:32.887 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:32.907 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:40:32.921 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:32.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:32.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:32 smithi129 ceph-mon[101316]: pgmap v2230: 89 pgs: 89 active+clean; 6.7 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2024-09-14T04:40:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:32 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/533046571' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:40:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:32 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:40:33.023 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:33.048 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:33.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:33.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:33.137 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:40:33.150 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:33.230 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:33.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:33.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:33.332 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:33.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:33.367 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:33.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:33.468 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:33.523 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:33.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:33.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:33.625 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:33.651 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:33.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:33.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:33.752 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:33.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:33.787 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:33.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:33.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:33 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-14T04:40:33.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:33 smithi039 ceph-mon[106470]: osdmap e1334: 8 total, 8 up, 8 in 2024-09-14T04:40:33.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:33 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-14T04:40:33.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:33 smithi039 ceph-mon[111235]: osdmap e1334: 8 total, 8 up, 8 in 2024-09-14T04:40:33.877 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:33 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-14T04:40:33.877 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:33 smithi129 ceph-mon[101316]: osdmap e1334: 8 total, 8 up, 8 in 2024-09-14T04:40:33.888 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:33.926 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:33.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:33.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:33.993 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 6 1 2024-09-14T04:40:34.028 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:34.058 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:34.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:34.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:34.159 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:34.230 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:34.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:34.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:34.269 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:40:34.274 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:40:34.275 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:40:34.329 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:40:34.329 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:40:34.332 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:34.364 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:40:34.364 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:40:34.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:34.386 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:34.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:34.401 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:40:34.402 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:40:34.431 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:40:34.431 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:40:34.462 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:40:34.462 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:40:34.487 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:34.488 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:40:34.488 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:40:34.511 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:40:34.511 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:40:34.523 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:34.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:34.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:34.625 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:34.658 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:34.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:34.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:34.760 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:34.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:34 smithi039 ceph-mon[106470]: pgmap v2232: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 977 op/s 2024-09-14T04:40:34.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:34 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/540225245' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:40:34.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:34 smithi039 ceph-mon[111235]: pgmap v2232: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 977 op/s 2024-09-14T04:40:34.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:34 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/540225245' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:40:34.795 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:34.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:34.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:34.897 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:34.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:34.943 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:34.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:34.998 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:34 smithi129 ceph-mon[101316]: pgmap v2232: 89 pgs: 89 active+clean; 6.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 977 op/s 2024-09-14T04:40:34.998 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:34 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/540225245' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:40:35.044 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:35.077 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:35.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:35.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:35.179 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:35.224 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:35.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:35.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:35.318 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:40:35.318 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [], 2024-09-14T04:40:35.318 INFO:teuthology.orchestra.run.smithi129.stdout: "complaint_time": 30, 2024-09-14T04:40:35.318 INFO:teuthology.orchestra.run.smithi129.stdout: "num_blocked_ops": 0 2024-09-14T04:40:35.318 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:40:35.328 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:35.362 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:35.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:35.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:35.464 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:35.507 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:35.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:35.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:35.609 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:35.653 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:35.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:35.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:35.755 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:35.795 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:35.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:35.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:35.837 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:40:35.897 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:35.932 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:35.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:35.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:36.031 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-14T04:40:36.034 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:36.125 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:36.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:36.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:36.227 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:36.262 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:36.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:36.264 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:36.365 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:36.378 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:40:36.402 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:36.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:36.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:36.504 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:36.538 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:36.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:36.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:36.640 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:36.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:36.678 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:36.678 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:36.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:36 smithi129 ceph-mon[101316]: pgmap v2233: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-14T04:40:36.779 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:36.816 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:36.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:36.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:36.918 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:36.954 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:36.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:36.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:36.997 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:36 smithi039 ceph-mon[106470]: pgmap v2233: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-14T04:40:36.997 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:36 smithi039 ceph-mon[111235]: pgmap v2233: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-14T04:40:37.057 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:37.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:37.096 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:37.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:37.197 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:37.234 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:37.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:37.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:37.336 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:37.368 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:37.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:37.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:37.470 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:37.505 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:37.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:37.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:37.607 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:37.640 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:37.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:37.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:37.742 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:37.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:37.786 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:37.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:37.888 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:37.913 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:40:37.915 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:37.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:37.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:38.017 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:40:38.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:40:38.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:38.041 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:38.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:38.142 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:38.179 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:38.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:38.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:38.281 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:38.330 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:38.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:38.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:38.335 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:40:38.339 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:40:38.339 INFO:teuthology.orchestra.run.smithi129.stdout: "size": 20, 2024-09-14T04:40:38.339 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 600, 2024-09-14T04:40:38.339 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [ 2024-09-14T04:40:38.339 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.339 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.34896.0:3466 51.e 51:77c0a9ef:::rbd_data.88502b0ee4d1.0000000000000053:head [write 0~4093863] snapc 6e=[6e,6d,6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1127)", 2024-09-14T04:40:38.339 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:32:12.669543+0000", 2024-09-14T04:40:38.339 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 505.66484438600003, 2024-09-14T04:40:38.339 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.073055307, 2024-09-14T04:40:38.339 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:38.340 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:38.340 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:38.340 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.34896", 2024-09-14T04:40:38.340 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/1436074065", 2024-09-14T04:40:38.340 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 3466 2024-09-14T04:40:38.340 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.340 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:38.341 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.341 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:38.341 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:32:12.669543+0000", 2024-09-14T04:40:38.341 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.341 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.341 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.341 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:38.341 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:32:12.669543+0000", 2024-09-14T04:40:38.341 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.341 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.341 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.341 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:38.341 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:32:12.669541+0000", 2024-09-14T04:40:38.341 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999976 2024-09-14T04:40:38.342 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.342 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.342 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:38.342 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:32:12.675803+0000", 2024-09-14T04:40:38.342 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0062614669999999997 2024-09-14T04:40:38.342 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.342 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.342 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:38.342 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:32:12.675829+0000", 2024-09-14T04:40:38.342 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.686e-05 2024-09-14T04:40:38.342 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.342 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.342 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:38.342 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:32:12.675841+0000", 2024-09-14T04:40:38.343 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.154e-05 2024-09-14T04:40:38.343 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.343 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.343 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:38.343 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:32:12.675929+0000", 2024-09-14T04:40:38.343 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.7528000000000006e-05 2024-09-14T04:40:38.343 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.343 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.343 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:38.343 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:32:12.676047+0000", 2024-09-14T04:40:38.343 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000118477 2024-09-14T04:40:38.343 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.344 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.344 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,6", 2024-09-14T04:40:38.344 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:32:12.676243+0000", 2024-09-14T04:40:38.344 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000195627 2024-09-14T04:40:38.344 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.344 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.344 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:38.344 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:32:12.719714+0000", 2024-09-14T04:40:38.344 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.043471825999999998 2024-09-14T04:40:38.344 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.344 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.344 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.344 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:32:12.730168+0000", 2024-09-14T04:40:38.345 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.010453087 2024-09-14T04:40:38.345 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.345 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.345 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.345 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:32:12.742499+0000", 2024-09-14T04:40:38.345 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.012331168999999999 2024-09-14T04:40:38.345 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.345 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.345 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:38.345 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:32:12.742559+0000", 2024-09-14T04:40:38.345 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.0195000000000002e-05 2024-09-14T04:40:38.345 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.345 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.345 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:38.346 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:32:12.742599+0000", 2024-09-14T04:40:38.346 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9709999999999998e-05 2024-09-14T04:40:38.346 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.346 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:38.346 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.346 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.346 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.346 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:29383 53.8 53:11159672:::benchmark_data_smithi039_224949_object29382:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-14T04:40:38.346 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:38:45.993261+0000", 2024-09-14T04:40:38.346 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 112.341126876, 2024-09-14T04:40:38.346 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.112018194, 2024-09-14T04:40:38.346 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:38.346 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:38.347 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:38.347 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:38.347 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:38.347 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 29383 2024-09-14T04:40:38.347 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.347 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:38.347 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.347 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:38.347 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.993261+0000", 2024-09-14T04:40:38.347 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.347 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.347 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.347 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:38.347 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.993261+0000", 2024-09-14T04:40:38.348 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.348 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.348 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.348 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:38.348 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.993260+0000", 2024-09-14T04:40:38.348 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:40:38.348 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.348 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.348 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:38.348 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.993406+0000", 2024-09-14T04:40:38.348 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000146365 2024-09-14T04:40:38.348 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.348 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.349 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:38.349 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.993412+0000", 2024-09-14T04:40:38.349 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9290000000000003e-06 2024-09-14T04:40:38.349 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.349 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.349 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:38.349 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.993416+0000", 2024-09-14T04:40:38.349 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8809999999999998e-06 2024-09-14T04:40:38.349 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.349 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.349 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:38.349 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.993453+0000", 2024-09-14T04:40:38.349 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6433000000000002e-05 2024-09-14T04:40:38.349 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.350 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.350 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:38.350 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.993680+0000", 2024-09-14T04:40:38.350 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00022770900000000001 2024-09-14T04:40:38.350 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.350 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.350 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:40:38.350 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.993800+0000", 2024-09-14T04:40:38.350 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011922 2024-09-14T04:40:38.350 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.351 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.351 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:38.351 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.996025+0000", 2024-09-14T04:40:38.351 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0022250809999999998 2024-09-14T04:40:38.351 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.351 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.351 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.351 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.998562+0000", 2024-09-14T04:40:38.351 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0025370190000000002 2024-09-14T04:40:38.351 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.351 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.351 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.351 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.105174+0000", 2024-09-14T04:40:38.351 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.106612754 2024-09-14T04:40:38.352 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.352 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.352 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:38.352 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.105250+0000", 2024-09-14T04:40:38.352 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.5889000000000005e-05 2024-09-14T04:40:38.352 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.352 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.352 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:38.352 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.105279+0000", 2024-09-14T04:40:38.352 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.8640999999999999e-05 2024-09-14T04:40:38.352 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.352 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:38.352 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.353 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.353 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.353 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:29405 53.8 53:1eab3930:::benchmark_data_smithi039_224949_object29404:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-14T04:40:38.353 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:38:46.002040+0000", 2024-09-14T04:40:38.353 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 112.332348117, 2024-09-14T04:40:38.353 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.109250364, 2024-09-14T04:40:38.353 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:38.353 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:38.353 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:38.353 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:38.353 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:38.354 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 29405 2024-09-14T04:40:38.354 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.354 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:38.355 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.355 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:38.355 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.002040+0000", 2024-09-14T04:40:38.355 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.355 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.355 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.355 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:38.355 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.002040+0000", 2024-09-14T04:40:38.355 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.355 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.355 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.355 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:38.356 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.002038+0000", 2024-09-14T04:40:38.356 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:40:38.356 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.356 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.356 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:38.356 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.002088+0000", 2024-09-14T04:40:38.356 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.9236000000000001e-05 2024-09-14T04:40:38.356 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.356 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.356 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:38.356 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.002097+0000", 2024-09-14T04:40:38.356 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.9400000000000008e-06 2024-09-14T04:40:38.356 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.357 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.357 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:38.357 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.002103+0000", 2024-09-14T04:40:38.357 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.5699999999999998e-06 2024-09-14T04:40:38.357 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.357 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.357 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:38.357 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.002145+0000", 2024-09-14T04:40:38.357 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1399999999999997e-05 2024-09-14T04:40:38.357 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.357 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.357 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:38.357 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.002240+0000", 2024-09-14T04:40:38.357 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.5149999999999995e-05 2024-09-14T04:40:38.358 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.358 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.358 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:40:38.358 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.002338+0000", 2024-09-14T04:40:38.358 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.8062999999999999e-05 2024-09-14T04:40:38.358 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.358 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.358 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:38.358 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.003338+0000", 2024-09-14T04:40:38.358 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001000117 2024-09-14T04:40:38.358 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.358 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.358 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.359 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.003652+0000", 2024-09-14T04:40:38.359 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000314059 2024-09-14T04:40:38.359 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.359 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.359 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.361 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.111235+0000", 2024-09-14T04:40:38.361 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.107582651 2024-09-14T04:40:38.361 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.361 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.361 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:38.361 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.111268+0000", 2024-09-14T04:40:38.361 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3822000000000002e-05 2024-09-14T04:40:38.361 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.361 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.361 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:38.361 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.111290+0000", 2024-09-14T04:40:38.362 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.1532999999999999e-05 2024-09-14T04:40:38.362 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.362 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:38.362 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.362 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.362 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.362 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.45053.0:30522 53.9 e1319/1315 53:9d228399:::benchmark_data_smithi039_224949_object30521:head v 1319'1873, mlcod=1319'1871)", 2024-09-14T04:40:38.362 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:38:46.572100+0000", 2024-09-14T04:40:38.362 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 111.762288149, 2024-09-14T04:40:38.362 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.068139837999999994, 2024-09-14T04:40:38.362 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:38.362 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:38.362 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:38.362 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.363 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:38.363 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.572100+0000", 2024-09-14T04:40:38.363 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.363 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.363 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.363 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:38.363 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.572100+0000", 2024-09-14T04:40:38.363 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.364 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.364 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.364 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:38.364 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.572099+0000", 2024-09-14T04:40:38.364 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:40:38.364 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.365 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.365 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:38.365 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.572205+0000", 2024-09-14T04:40:38.365 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000106483 2024-09-14T04:40:38.365 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.365 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.365 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:38.365 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.633935+0000", 2024-09-14T04:40:38.365 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.06172913 2024-09-14T04:40:38.365 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.365 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.365 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:38.365 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.633944+0000", 2024-09-14T04:40:38.366 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.7899999999999994e-06 2024-09-14T04:40:38.366 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.366 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.366 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:38.366 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.634037+0000", 2024-09-14T04:40:38.366 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.2503000000000005e-05 2024-09-14T04:40:38.366 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.366 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.366 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:38.366 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.634054+0000", 2024-09-14T04:40:38.367 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.6952e-05 2024-09-14T04:40:38.367 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.367 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.367 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:38.367 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.640140+0000", 2024-09-14T04:40:38.367 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0060865240000000003 2024-09-14T04:40:38.367 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.367 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.367 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:38.367 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.640239+0000", 2024-09-14T04:40:38.367 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.9065000000000001e-05 2024-09-14T04:40:38.367 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.367 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:38.367 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.368 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.368 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.368 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:49312 53.0 53:038e8115:::benchmark_data_smithi039_224949_object49311:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1320)", 2024-09-14T04:40:38.368 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:38:56.495003+0000", 2024-09-14T04:40:38.368 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 101.839384966, 2024-09-14T04:40:38.368 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.077434717, 2024-09-14T04:40:38.368 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:38.368 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:38.368 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:38.368 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:38.368 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:38.368 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 49312 2024-09-14T04:40:38.368 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.369 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:38.369 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.369 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:38.369 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:56.495003+0000", 2024-09-14T04:40:38.369 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.369 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.369 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.369 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:38.369 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:56.495003+0000", 2024-09-14T04:40:38.369 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.369 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.369 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.370 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:38.370 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:56.495002+0000", 2024-09-14T04:40:38.370 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:40:38.370 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.370 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.370 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:38.370 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:56.495075+0000", 2024-09-14T04:40:38.370 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.3215999999999998e-05 2024-09-14T04:40:38.370 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.370 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.370 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:38.370 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:56.495085+0000", 2024-09-14T04:40:38.371 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.38e-06 2024-09-14T04:40:38.371 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.371 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.371 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:38.371 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:56.495091+0000", 2024-09-14T04:40:38.371 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9880000000000001e-06 2024-09-14T04:40:38.371 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.371 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.371 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:38.371 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:56.558482+0000", 2024-09-14T04:40:38.371 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.063391245999999998 2024-09-14T04:40:38.372 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.372 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.372 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:38.372 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:56.558592+0000", 2024-09-14T04:40:38.372 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010982899999999999 2024-09-14T04:40:38.372 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.372 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.372 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:40:38.372 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:56.558705+0000", 2024-09-14T04:40:38.372 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011283 2024-09-14T04:40:38.372 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.372 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.372 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.373 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:56.562606+0000", 2024-09-14T04:40:38.373 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0039019689999999999 2024-09-14T04:40:38.373 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.373 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.373 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:38.373 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:56.562622+0000", 2024-09-14T04:40:38.373 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5869e-05 2024-09-14T04:40:38.373 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.373 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.373 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.373 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:56.572329+0000", 2024-09-14T04:40:38.373 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.009706381 2024-09-14T04:40:38.373 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.373 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.374 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:38.374 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:56.572397+0000", 2024-09-14T04:40:38.374 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8268000000000004e-05 2024-09-14T04:40:38.374 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.374 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.374 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:38.374 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:56.572437+0000", 2024-09-14T04:40:38.374 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0433999999999999e-05 2024-09-14T04:40:38.374 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.374 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:38.374 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.374 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.374 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.374 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:73824 53.8 53:1c355733:::benchmark_data_smithi039_224949_object73823:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-09-14T04:40:38.375 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:39:12.623979+0000", 2024-09-14T04:40:38.375 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 85.710408556999994, 2024-09-14T04:40:38.375 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.112198508, 2024-09-14T04:40:38.375 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:38.375 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:38.375 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:38.375 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:38.375 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:38.375 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 73824 2024-09-14T04:40:38.375 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.375 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:38.375 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.375 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:38.375 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.623979+0000", 2024-09-14T04:40:38.376 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.376 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.376 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.376 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:38.376 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.623979+0000", 2024-09-14T04:40:38.376 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.376 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.376 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.377 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:38.377 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.623978+0000", 2024-09-14T04:40:38.377 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:40:38.377 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.377 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.377 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:38.377 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.624032+0000", 2024-09-14T04:40:38.377 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.4119999999999997e-05 2024-09-14T04:40:38.377 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.377 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.378 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:38.378 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.624045+0000", 2024-09-14T04:40:38.378 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3247999999999999e-05 2024-09-14T04:40:38.378 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.378 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.378 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:38.378 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.624052+0000", 2024-09-14T04:40:38.378 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.2690000000000002e-06 2024-09-14T04:40:38.378 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.378 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.378 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:38.379 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.624089+0000", 2024-09-14T04:40:38.379 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6451e-05 2024-09-14T04:40:38.379 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.379 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.380 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:38.380 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.624308+0000", 2024-09-14T04:40:38.380 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00021910199999999999 2024-09-14T04:40:38.380 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.380 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.380 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:40:38.380 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.624468+0000", 2024-09-14T04:40:38.380 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000160631 2024-09-14T04:40:38.380 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.381 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.381 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:38.381 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.628826+0000", 2024-09-14T04:40:38.381 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0043573489999999999 2024-09-14T04:40:38.381 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.381 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.381 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.381 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.631057+0000", 2024-09-14T04:40:38.381 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0022309840000000001 2024-09-14T04:40:38.381 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.381 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.381 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.382 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.736086+0000", 2024-09-14T04:40:38.382 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.105029277 2024-09-14T04:40:38.382 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.382 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.382 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:38.382 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.736137+0000", 2024-09-14T04:40:38.382 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.1317999999999999e-05 2024-09-14T04:40:38.382 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.382 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.382 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:38.383 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.736178+0000", 2024-09-14T04:40:38.383 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0256999999999998e-05 2024-09-14T04:40:38.383 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.383 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:38.383 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.383 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.383 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.383 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:73825 53.8 53:16961f53:::benchmark_data_smithi039_224949_object73824:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-09-14T04:40:38.383 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:39:12.624105+0000", 2024-09-14T04:40:38.383 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 85.710282477000007, 2024-09-14T04:40:38.383 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.112162661, 2024-09-14T04:40:38.384 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:38.384 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:38.384 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:38.384 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:38.384 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:38.384 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 73825 2024-09-14T04:40:38.384 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.384 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:38.384 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.384 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:38.384 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.624105+0000", 2024-09-14T04:40:38.384 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.384 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.385 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.385 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:38.385 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.624105+0000", 2024-09-14T04:40:38.385 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.385 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.385 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.385 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:38.385 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.624099+0000", 2024-09-14T04:40:38.385 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999938 2024-09-14T04:40:38.385 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.385 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.385 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:38.385 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.624194+0000", 2024-09-14T04:40:38.386 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.4488999999999996e-05 2024-09-14T04:40:38.386 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.386 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.386 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:38.386 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.624204+0000", 2024-09-14T04:40:38.386 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0477e-05 2024-09-14T04:40:38.386 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.386 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.386 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:38.386 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.624211+0000", 2024-09-14T04:40:38.386 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.742e-06 2024-09-14T04:40:38.386 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.386 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.386 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:38.387 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.626612+0000", 2024-09-14T04:40:38.387 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0024014930000000002 2024-09-14T04:40:38.387 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.387 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.387 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:38.387 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.626753+0000", 2024-09-14T04:40:38.387 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000141163 2024-09-14T04:40:38.387 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.387 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.387 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:40:38.387 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.626904+0000", 2024-09-14T04:40:38.387 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000150927 2024-09-14T04:40:38.387 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.388 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.388 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:38.388 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.628963+0000", 2024-09-14T04:40:38.388 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0020590349999999999 2024-09-14T04:40:38.388 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.388 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.388 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.388 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.632948+0000", 2024-09-14T04:40:38.388 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0039844900000000003 2024-09-14T04:40:38.388 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.388 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.388 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.388 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.736215+0000", 2024-09-14T04:40:38.388 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.103266706 2024-09-14T04:40:38.389 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.389 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.389 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:38.389 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.736246+0000", 2024-09-14T04:40:38.389 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.1427999999999998e-05 2024-09-14T04:40:38.389 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.389 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.389 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:38.389 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.736268+0000", 2024-09-14T04:40:38.389 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.1832999999999999e-05 2024-09-14T04:40:38.389 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.389 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:38.389 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.390 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.390 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.390 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:74033 53.8 53:129e948a:::benchmark_data_smithi039_224949_object74032:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-09-14T04:40:38.390 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:39:12.867117+0000", 2024-09-14T04:40:38.390 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 85.467270892000002, 2024-09-14T04:40:38.390 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.10307910000000001, 2024-09-14T04:40:38.390 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:38.390 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:38.390 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:38.390 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:38.390 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:38.390 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 74033 2024-09-14T04:40:38.390 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.391 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:38.391 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.391 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:38.391 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.867117+0000", 2024-09-14T04:40:38.391 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.391 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.391 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.391 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:38.391 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.867117+0000", 2024-09-14T04:40:38.391 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.391 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.391 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.391 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:38.391 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.867116+0000", 2024-09-14T04:40:38.392 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:40:38.392 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.392 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.392 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:38.392 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.867141+0000", 2024-09-14T04:40:38.392 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.5055999999999999e-05 2024-09-14T04:40:38.392 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.392 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.392 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:38.392 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.867148+0000", 2024-09-14T04:40:38.392 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.4980000000000003e-06 2024-09-14T04:40:38.392 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.392 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.392 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:38.393 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.867152+0000", 2024-09-14T04:40:38.393 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.5730000000000001e-06 2024-09-14T04:40:38.393 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.393 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.393 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:38.393 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.870996+0000", 2024-09-14T04:40:38.393 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0038432169999999999 2024-09-14T04:40:38.393 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.393 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.393 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:38.393 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.871159+0000", 2024-09-14T04:40:38.393 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00016362899999999999 2024-09-14T04:40:38.393 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.394 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.394 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:40:38.394 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.871296+0000", 2024-09-14T04:40:38.394 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013694999999999999 2024-09-14T04:40:38.394 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.394 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.394 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:38.394 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.875268+0000", 2024-09-14T04:40:38.394 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0039720340000000002 2024-09-14T04:40:38.394 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.394 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.394 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.394 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.877641+0000", 2024-09-14T04:40:38.394 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0023731920000000001 2024-09-14T04:40:38.395 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.395 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.395 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.395 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.970112+0000", 2024-09-14T04:40:38.395 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.092470681999999998 2024-09-14T04:40:38.395 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.395 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.395 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:38.395 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.970163+0000", 2024-09-14T04:40:38.396 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.1258999999999998e-05 2024-09-14T04:40:38.396 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.396 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.396 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:38.396 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.970196+0000", 2024-09-14T04:40:38.396 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.2495e-05 2024-09-14T04:40:38.396 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.396 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:38.396 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.396 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.396 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.396 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:74050 53.8 53:1bc7dfaa:::benchmark_data_smithi039_224949_object74049:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-09-14T04:40:38.396 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:39:12.877479+0000", 2024-09-14T04:40:38.396 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 85.456908713999994, 2024-09-14T04:40:38.397 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.099365340999999996, 2024-09-14T04:40:38.397 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:38.397 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:38.397 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:38.397 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:38.397 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:38.397 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 74050 2024-09-14T04:40:38.397 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.397 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:38.397 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.397 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:38.397 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.877479+0000", 2024-09-14T04:40:38.397 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.397 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.398 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.398 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:38.398 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.877479+0000", 2024-09-14T04:40:38.398 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.398 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.398 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.398 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:38.398 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.877478+0000", 2024-09-14T04:40:38.398 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:40:38.398 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.398 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.398 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:38.398 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.877683+0000", 2024-09-14T04:40:38.398 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00020491199999999999 2024-09-14T04:40:38.399 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.399 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.399 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:38.399 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.877691+0000", 2024-09-14T04:40:38.399 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.5480000000000002e-06 2024-09-14T04:40:38.399 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.399 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.399 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:38.399 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.877697+0000", 2024-09-14T04:40:38.399 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.8250000000000001e-06 2024-09-14T04:40:38.400 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.400 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.400 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:38.400 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.877759+0000", 2024-09-14T04:40:38.400 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.2582999999999994e-05 2024-09-14T04:40:38.400 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.400 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.400 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:38.400 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.877883+0000", 2024-09-14T04:40:38.400 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000123974 2024-09-14T04:40:38.400 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.400 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.400 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:40:38.400 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.878213+0000", 2024-09-14T04:40:38.401 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00032952900000000001 2024-09-14T04:40:38.401 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.401 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.401 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:38.401 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.880876+0000", 2024-09-14T04:40:38.401 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0026629919999999999 2024-09-14T04:40:38.401 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.401 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.401 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.401 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.969944+0000", 2024-09-14T04:40:38.401 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.089067563000000002 2024-09-14T04:40:38.401 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.401 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.402 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.402 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.976770+0000", 2024-09-14T04:40:38.402 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0068267240000000002 2024-09-14T04:40:38.402 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.402 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.402 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:38.402 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.976808+0000", 2024-09-14T04:40:38.402 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7993e-05 2024-09-14T04:40:38.402 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.402 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.402 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:38.402 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.976844+0000", 2024-09-14T04:40:38.402 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6062999999999997e-05 2024-09-14T04:40:38.402 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.403 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:38.403 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.403 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.403 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.404 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:74053 53.8 53:1dfce94a:::benchmark_data_smithi039_224949_object74052:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-09-14T04:40:38.404 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:39:12.884135+0000", 2024-09-14T04:40:38.404 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 85.450252667000001, 2024-09-14T04:40:38.404 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.096380848000000005, 2024-09-14T04:40:38.404 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:38.404 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:38.404 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:38.404 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:38.404 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:38.404 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 74053 2024-09-14T04:40:38.404 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.405 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:38.405 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.405 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:38.405 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.884135+0000", 2024-09-14T04:40:38.405 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.405 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.405 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.405 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:38.405 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.884135+0000", 2024-09-14T04:40:38.405 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.405 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.405 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.405 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:38.405 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.884133+0000", 2024-09-14T04:40:38.406 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:40:38.406 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.406 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.406 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:38.406 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.884181+0000", 2024-09-14T04:40:38.406 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7358999999999998e-05 2024-09-14T04:40:38.406 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.406 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.406 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:38.406 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.884197+0000", 2024-09-14T04:40:38.406 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5846e-05 2024-09-14T04:40:38.406 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.406 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.406 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:38.407 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.884205+0000", 2024-09-14T04:40:38.407 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.9169999999999995e-06 2024-09-14T04:40:38.407 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.407 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.407 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:38.407 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.884242+0000", 2024-09-14T04:40:38.407 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6338999999999999e-05 2024-09-14T04:40:38.407 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.407 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.407 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:38.407 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.884339+0000", 2024-09-14T04:40:38.407 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.7424999999999996e-05 2024-09-14T04:40:38.407 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.408 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.408 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:40:38.408 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.884431+0000", 2024-09-14T04:40:38.408 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.1395000000000002e-05 2024-09-14T04:40:38.408 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.408 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.408 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:38.408 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.886351+0000", 2024-09-14T04:40:38.408 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001920284 2024-09-14T04:40:38.408 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.408 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.408 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.408 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.973172+0000", 2024-09-14T04:40:38.408 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.086821105999999995 2024-09-14T04:40:38.409 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.409 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.409 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.409 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.980400+0000", 2024-09-14T04:40:38.409 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0072280299999999999 2024-09-14T04:40:38.409 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.409 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.409 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:38.409 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.980477+0000", 2024-09-14T04:40:38.409 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.6846000000000003e-05 2024-09-14T04:40:38.409 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.409 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.409 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:38.409 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.980516+0000", 2024-09-14T04:40:38.410 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9008999999999999e-05 2024-09-14T04:40:38.410 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.410 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:38.410 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.410 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.410 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.410 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:76204 53.8 53:1fef6cf6:::benchmark_data_smithi039_224949_object76203:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-09-14T04:40:38.410 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:39:14.594303+0000", 2024-09-14T04:40:38.410 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 83.740084804000006, 2024-09-14T04:40:38.410 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.092212248999999996, 2024-09-14T04:40:38.410 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:38.410 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:38.410 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:38.410 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:38.411 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:38.411 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 76204 2024-09-14T04:40:38.411 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.411 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:38.411 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.411 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:38.411 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.594303+0000", 2024-09-14T04:40:38.411 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.411 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.411 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.411 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:38.411 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.594303+0000", 2024-09-14T04:40:38.411 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.411 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.412 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.412 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:38.412 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.594302+0000", 2024-09-14T04:40:38.412 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:40:38.412 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.412 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.412 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:38.412 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.594349+0000", 2024-09-14T04:40:38.412 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7024000000000002e-05 2024-09-14T04:40:38.412 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.412 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.412 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:38.412 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.594360+0000", 2024-09-14T04:40:38.412 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1763000000000001e-05 2024-09-14T04:40:38.413 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.413 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.413 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:38.413 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.594366+0000", 2024-09-14T04:40:38.413 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2290000000000002e-06 2024-09-14T04:40:38.413 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.413 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.413 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:38.413 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.594400+0000", 2024-09-14T04:40:38.413 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.4751e-05 2024-09-14T04:40:38.413 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.413 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.413 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:38.413 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.594497+0000", 2024-09-14T04:40:38.414 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.6669000000000006e-05 2024-09-14T04:40:38.414 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.414 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.414 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:40:38.414 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.594590+0000", 2024-09-14T04:40:38.414 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.3172999999999997e-05 2024-09-14T04:40:38.414 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.414 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.414 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:38.414 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.596450+0000", 2024-09-14T04:40:38.414 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0018601659999999999 2024-09-14T04:40:38.414 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.414 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.414 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.414 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.686248+0000", 2024-09-14T04:40:38.415 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.089797848 2024-09-14T04:40:38.415 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.415 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.415 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.415 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.686382+0000", 2024-09-14T04:40:38.415 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013408499999999999 2024-09-14T04:40:38.415 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.415 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.415 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:38.415 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.686435+0000", 2024-09-14T04:40:38.415 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2698000000000003e-05 2024-09-14T04:40:38.415 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.415 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.415 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:38.416 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.686515+0000", 2024-09-14T04:40:38.416 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.0210999999999995e-05 2024-09-14T04:40:38.416 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.416 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:38.416 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.416 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.416 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.416 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:76555 53.8 53:1955e064:::benchmark_data_smithi039_224949_object76554:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-09-14T04:40:38.416 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:39:14.926677+0000", 2024-09-14T04:40:38.416 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 83.407710508999998, 2024-09-14T04:40:38.416 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.079983242999999996, 2024-09-14T04:40:38.416 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:38.416 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:38.416 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:38.417 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:38.417 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:38.417 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 76555 2024-09-14T04:40:38.417 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.417 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:38.417 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.417 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:38.417 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.926677+0000", 2024-09-14T04:40:38.417 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.417 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.417 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.417 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:38.417 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.926677+0000", 2024-09-14T04:40:38.418 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.418 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.418 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.418 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:38.418 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.926675+0000", 2024-09-14T04:40:38.418 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:40:38.418 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.418 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.418 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:38.418 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.926707+0000", 2024-09-14T04:40:38.418 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.1139000000000001e-05 2024-09-14T04:40:38.418 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.418 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.418 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:38.419 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.926724+0000", 2024-09-14T04:40:38.419 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.7552000000000001e-05 2024-09-14T04:40:38.419 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.419 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.419 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:38.419 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.926731+0000", 2024-09-14T04:40:38.419 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.3810000000000001e-06 2024-09-14T04:40:38.419 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.419 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.419 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:38.419 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.926774+0000", 2024-09-14T04:40:38.419 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.2737000000000001e-05 2024-09-14T04:40:38.419 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.419 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.420 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:38.420 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.926983+0000", 2024-09-14T04:40:38.420 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00020827899999999999 2024-09-14T04:40:38.420 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.420 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.420 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:40:38.420 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.927115+0000", 2024-09-14T04:40:38.420 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013270400000000001 2024-09-14T04:40:38.420 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.420 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.421 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:38.421 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.929755+0000", 2024-09-14T04:40:38.421 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0026400019999999998 2024-09-14T04:40:38.421 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.421 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.421 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.421 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.931657+0000", 2024-09-14T04:40:38.421 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001902001 2024-09-14T04:40:38.421 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.421 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.421 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.421 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:15.006559+0000", 2024-09-14T04:40:38.421 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.074901361999999999 2024-09-14T04:40:38.421 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.422 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.422 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:38.422 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:15.006615+0000", 2024-09-14T04:40:38.422 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.6705e-05 2024-09-14T04:40:38.422 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.422 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.422 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:38.422 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:15.006660+0000", 2024-09-14T04:40:38.422 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.5153e-05 2024-09-14T04:40:38.422 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.422 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:38.422 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.422 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.422 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.423 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:107923 53.8 53:1f22efd2:::benchmark_data_smithi039_224949_object107922:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1326)", 2024-09-14T04:40:38.423 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:39:42.310679+0000", 2024-09-14T04:40:38.423 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 56.023708589000002, 2024-09-14T04:40:38.423 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.075959417000000001, 2024-09-14T04:40:38.424 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:38.424 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:38.424 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:38.424 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:38.424 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:38.425 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 107923 2024-09-14T04:40:38.425 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.425 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:38.425 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.425 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:38.425 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:42.310679+0000", 2024-09-14T04:40:38.425 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.425 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.425 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.425 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:38.425 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:42.310679+0000", 2024-09-14T04:40:38.425 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.425 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.425 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.426 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:38.426 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:42.310678+0000", 2024-09-14T04:40:38.426 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:40:38.426 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.426 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.426 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:38.426 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:42.310739+0000", 2024-09-14T04:40:38.426 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.0365000000000002e-05 2024-09-14T04:40:38.426 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.426 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.426 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:38.426 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:42.310750+0000", 2024-09-14T04:40:38.426 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1474000000000001e-05 2024-09-14T04:40:38.426 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.427 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.427 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:38.427 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:42.310755+0000", 2024-09-14T04:40:38.427 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.3369999999999999e-06 2024-09-14T04:40:38.427 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.427 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.427 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:38.427 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:42.313281+0000", 2024-09-14T04:40:38.427 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0025255189999999999 2024-09-14T04:40:38.427 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.427 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.427 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:38.427 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:42.313465+0000", 2024-09-14T04:40:38.427 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00018450300000000001 2024-09-14T04:40:38.428 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.428 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.428 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:40:38.428 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:42.313559+0000", 2024-09-14T04:40:38.428 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.3608000000000002e-05 2024-09-14T04:40:38.428 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.428 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.428 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:38.428 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:42.316958+0000", 2024-09-14T04:40:38.428 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0033989480000000002 2024-09-14T04:40:38.428 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.428 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.428 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.429 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:42.369115+0000", 2024-09-14T04:40:38.429 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.052157121000000001 2024-09-14T04:40:38.429 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.429 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.429 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.429 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:42.386573+0000", 2024-09-14T04:40:38.429 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.01745824 2024-09-14T04:40:38.429 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.429 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.429 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:38.429 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:42.386614+0000", 2024-09-14T04:40:38.429 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0831000000000003e-05 2024-09-14T04:40:38.429 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.429 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.430 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:38.430 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:42.386639+0000", 2024-09-14T04:40:38.430 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.4375999999999998e-05 2024-09-14T04:40:38.430 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.430 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:38.430 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.430 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.430 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.430 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:107928 53.8 53:16fe20bd:::benchmark_data_smithi039_224949_object107927:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1326)", 2024-09-14T04:40:38.430 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:39:42.317900+0000", 2024-09-14T04:40:38.430 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 56.016487808999997, 2024-09-14T04:40:38.430 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.068859745999999999, 2024-09-14T04:40:38.430 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:38.430 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:38.431 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:38.431 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:38.431 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:38.431 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 107928 2024-09-14T04:40:38.431 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.431 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:38.431 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.431 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:38.431 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:42.317900+0000", 2024-09-14T04:40:38.431 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.431 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.431 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.431 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:38.431 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:42.317900+0000", 2024-09-14T04:40:38.432 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.432 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.432 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.432 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:38.432 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:42.317898+0000", 2024-09-14T04:40:38.432 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:40:38.432 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.432 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.432 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:38.432 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:42.317957+0000", 2024-09-14T04:40:38.432 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.8773999999999997e-05 2024-09-14T04:40:38.432 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.432 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.433 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:38.433 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:42.317971+0000", 2024-09-14T04:40:38.433 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.399e-05 2024-09-14T04:40:38.433 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.433 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.433 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:38.433 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:42.317981+0000", 2024-09-14T04:40:38.433 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.7960000000000007e-06 2024-09-14T04:40:38.433 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.433 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.433 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:38.433 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:42.318012+0000", 2024-09-14T04:40:38.433 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.0902999999999999e-05 2024-09-14T04:40:38.433 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.434 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.434 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:38.434 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:42.318129+0000", 2024-09-14T04:40:38.434 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000116805 2024-09-14T04:40:38.434 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.434 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.434 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:40:38.434 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:42.318242+0000", 2024-09-14T04:40:38.434 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000112997 2024-09-14T04:40:38.434 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.434 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.434 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:38.434 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:42.320778+0000", 2024-09-14T04:40:38.434 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0025358820000000002 2024-09-14T04:40:38.435 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.435 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.435 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.435 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:42.373768+0000", 2024-09-14T04:40:38.435 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.052990322999999999 2024-09-14T04:40:38.435 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.435 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.435 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.435 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:42.386677+0000", 2024-09-14T04:40:38.435 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.012909090999999999 2024-09-14T04:40:38.435 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.435 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.435 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:38.435 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:42.386726+0000", 2024-09-14T04:40:38.436 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8866999999999998e-05 2024-09-14T04:40:38.436 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.436 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.436 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:38.436 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:42.386760+0000", 2024-09-14T04:40:38.436 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3798999999999999e-05 2024-09-14T04:40:38.436 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.436 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:38.436 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.436 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.436 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.436 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:133863 53.8 53:12979ba3:::benchmark_data_smithi039_224949_object3770:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1330)", 2024-09-14T04:40:38.436 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:40:09.864198+0000", 2024-09-14T04:40:38.437 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 28.47019019, 2024-09-14T04:40:38.437 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.105536675, 2024-09-14T04:40:38.437 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:38.437 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:38.437 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:38.437 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:38.437 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:38.437 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 133863 2024-09-14T04:40:38.437 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.437 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:38.437 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.437 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:38.437 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.864198+0000", 2024-09-14T04:40:38.437 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.438 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.438 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.438 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:38.438 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.864198+0000", 2024-09-14T04:40:38.438 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.438 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.438 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.438 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:38.438 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.864196+0000", 2024-09-14T04:40:38.438 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:40:38.438 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.438 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.438 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:38.438 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.864203+0000", 2024-09-14T04:40:38.439 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7989999999999996e-06 2024-09-14T04:40:38.439 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.439 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.439 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:38.439 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.864216+0000", 2024-09-14T04:40:38.439 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3883000000000001e-05 2024-09-14T04:40:38.439 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.439 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.439 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:38.439 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.864224+0000", 2024-09-14T04:40:38.439 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.8490000000000006e-06 2024-09-14T04:40:38.439 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.439 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.439 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:38.440 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.864273+0000", 2024-09-14T04:40:38.440 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8414000000000001e-05 2024-09-14T04:40:38.440 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.440 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.440 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:38.440 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.864391+0000", 2024-09-14T04:40:38.440 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000118324 2024-09-14T04:40:38.440 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.440 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.440 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:40:38.440 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.864508+0000", 2024-09-14T04:40:38.440 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000116592 2024-09-14T04:40:38.440 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.440 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.441 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.441 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.876305+0000", 2024-09-14T04:40:38.441 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.011797782 2024-09-14T04:40:38.441 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.441 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.441 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:38.441 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.876312+0000", 2024-09-14T04:40:38.441 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7120000000000003e-06 2024-09-14T04:40:38.441 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.441 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.441 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.441 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.969675+0000", 2024-09-14T04:40:38.441 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.093362675000000006 2024-09-14T04:40:38.441 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.442 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.442 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:38.442 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.969707+0000", 2024-09-14T04:40:38.442 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.2291e-05 2024-09-14T04:40:38.442 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.442 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.442 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:38.442 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.969734+0000", 2024-09-14T04:40:38.442 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.7073999999999999e-05 2024-09-14T04:40:38.442 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.442 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:38.442 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.442 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.442 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.443 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:133867 53.8 53:19989e9c:::benchmark_data_smithi039_224949_object3774:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1330)", 2024-09-14T04:40:38.443 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:40:09.866383+0000", 2024-09-14T04:40:38.443 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 28.468004964999999, 2024-09-14T04:40:38.443 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.103437502, 2024-09-14T04:40:38.443 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:38.443 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:38.443 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:38.443 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:38.443 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:38.443 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 133867 2024-09-14T04:40:38.443 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.443 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:38.443 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.443 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:38.444 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.866383+0000", 2024-09-14T04:40:38.444 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.444 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.444 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.444 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:38.444 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.866383+0000", 2024-09-14T04:40:38.444 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.444 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.444 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.444 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:38.444 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.866382+0000", 2024-09-14T04:40:38.444 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:40:38.444 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.444 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.445 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:38.445 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.866389+0000", 2024-09-14T04:40:38.445 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.8269999999999998e-06 2024-09-14T04:40:38.445 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.445 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.445 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:38.445 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.866403+0000", 2024-09-14T04:40:38.445 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3847e-05 2024-09-14T04:40:38.445 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.445 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.445 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:38.445 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.866410+0000", 2024-09-14T04:40:38.445 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.17e-06 2024-09-14T04:40:38.445 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.446 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.446 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:38.446 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.867374+0000", 2024-09-14T04:40:38.446 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00096367200000000005 2024-09-14T04:40:38.446 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.446 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.446 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:38.446 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.867485+0000", 2024-09-14T04:40:38.446 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011109599999999999 2024-09-14T04:40:38.446 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.446 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.446 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:40:38.446 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.867618+0000", 2024-09-14T04:40:38.446 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013255400000000001 2024-09-14T04:40:38.447 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.447 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.447 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.447 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.876340+0000", 2024-09-14T04:40:38.447 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0087218360000000002 2024-09-14T04:40:38.447 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.447 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.447 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:38.447 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.877173+0000", 2024-09-14T04:40:38.447 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00083366899999999999 2024-09-14T04:40:38.447 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.447 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.447 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.447 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.969768+0000", 2024-09-14T04:40:38.448 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.092595178 2024-09-14T04:40:38.448 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.448 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.448 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:38.448 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.969799+0000", 2024-09-14T04:40:38.448 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.0537999999999998e-05 2024-09-14T04:40:38.448 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.448 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.448 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:38.448 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.969820+0000", 2024-09-14T04:40:38.448 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.1294e-05 2024-09-14T04:40:38.448 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.448 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:38.448 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.449 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.449 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.449 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:134522 53.8 53:1b637408:::benchmark_data_smithi039_224949_object4429:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1330)", 2024-09-14T04:40:38.449 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:40:10.641150+0000", 2024-09-14T04:40:38.449 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 27.693237356000001, 2024-09-14T04:40:38.449 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.081715386000000001, 2024-09-14T04:40:38.449 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:38.449 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:38.449 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:38.449 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:38.449 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:38.449 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 134522 2024-09-14T04:40:38.449 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.449 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:38.450 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.450 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:38.450 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.641150+0000", 2024-09-14T04:40:38.450 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.450 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.450 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.450 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:38.450 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.641150+0000", 2024-09-14T04:40:38.450 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.450 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.450 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.450 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:38.450 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.641149+0000", 2024-09-14T04:40:38.451 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:40:38.451 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.451 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.451 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:38.451 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.641155+0000", 2024-09-14T04:40:38.451 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.1090000000000003e-06 2024-09-14T04:40:38.451 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.451 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.451 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:38.451 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.641167+0000", 2024-09-14T04:40:38.451 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1996e-05 2024-09-14T04:40:38.451 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.451 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.451 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:38.452 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.641174+0000", 2024-09-14T04:40:38.452 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7220000000000002e-06 2024-09-14T04:40:38.452 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.452 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.452 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:38.452 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.641209+0000", 2024-09-14T04:40:38.452 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.4966999999999999e-05 2024-09-14T04:40:38.452 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.452 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.452 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:38.452 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.641293+0000", 2024-09-14T04:40:38.452 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.4392000000000001e-05 2024-09-14T04:40:38.452 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.452 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.453 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:40:38.453 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.641366+0000", 2024-09-14T04:40:38.453 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.3206000000000003e-05 2024-09-14T04:40:38.453 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.453 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.453 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:38.453 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.645775+0000", 2024-09-14T04:40:38.453 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0044090500000000003 2024-09-14T04:40:38.453 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.453 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.453 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.453 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.645809+0000", 2024-09-14T04:40:38.453 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3507999999999998e-05 2024-09-14T04:40:38.453 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.454 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.454 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.454 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.722782+0000", 2024-09-14T04:40:38.454 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.076973421 2024-09-14T04:40:38.454 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.454 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.454 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:38.454 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.722830+0000", 2024-09-14T04:40:38.454 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7163999999999998e-05 2024-09-14T04:40:38.454 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.454 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.454 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:38.454 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.722866+0000", 2024-09-14T04:40:38.454 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6177000000000003e-05 2024-09-14T04:40:38.455 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.455 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:38.455 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.455 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.455 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.455 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:134523 53.8 53:1aea971f:::benchmark_data_smithi039_224949_object4430:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1330)", 2024-09-14T04:40:38.455 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:40:10.642501+0000", 2024-09-14T04:40:38.455 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 27.691886392000001, 2024-09-14T04:40:38.455 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.088621399000000003, 2024-09-14T04:40:38.455 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:38.455 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:38.455 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:38.455 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:38.455 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:38.456 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 134523 2024-09-14T04:40:38.456 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.456 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:38.456 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.456 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:38.456 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.642501+0000", 2024-09-14T04:40:38.456 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.456 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.456 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.456 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:38.456 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.642501+0000", 2024-09-14T04:40:38.456 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.456 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.457 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.457 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:38.457 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.642501+0000", 2024-09-14T04:40:38.457 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:40:38.457 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.457 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.457 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:38.457 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.642505+0000", 2024-09-14T04:40:38.457 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0879999999999997e-06 2024-09-14T04:40:38.457 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.458 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.458 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:38.458 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.642514+0000", 2024-09-14T04:40:38.458 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.7410000000000004e-06 2024-09-14T04:40:38.458 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.458 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.458 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:38.458 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.642519+0000", 2024-09-14T04:40:38.458 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.775e-06 2024-09-14T04:40:38.458 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.458 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.458 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:38.458 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.643904+0000", 2024-09-14T04:40:38.458 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00138444 2024-09-14T04:40:38.459 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.459 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.459 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:38.459 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.643987+0000", 2024-09-14T04:40:38.459 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.3330000000000003e-05 2024-09-14T04:40:38.459 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.459 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.459 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:40:38.459 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.644052+0000", 2024-09-14T04:40:38.459 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.4591999999999994e-05 2024-09-14T04:40:38.459 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.459 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.459 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.459 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.645833+0000", 2024-09-14T04:40:38.460 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0017817180000000001 2024-09-14T04:40:38.460 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.460 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.460 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:38.460 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.645902+0000", 2024-09-14T04:40:38.460 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:38.461 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.9189000000000005e-05 2024-09-14T04:40:38.461 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.461 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.461 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.461 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.731039+0000", 2024-09-14T04:40:38.461 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.085136292000000002 2024-09-14T04:40:38.461 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.462 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.462 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:38.462 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.731082+0000", 2024-09-14T04:40:38.462 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.3327e-05 2024-09-14T04:40:38.462 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.462 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.462 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:38.462 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.731123+0000", 2024-09-14T04:40:38.462 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0649000000000003e-05 2024-09-14T04:40:38.462 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.462 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:38.463 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.463 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.463 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.463 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:134743 53.8 53:17e49e9d:::benchmark_data_smithi039_224949_object4650:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1330)", 2024-09-14T04:40:38.463 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:40:10.921311+0000", 2024-09-14T04:40:38.463 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 27.413076941, 2024-09-14T04:40:38.463 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.094157052000000005, 2024-09-14T04:40:38.463 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:38.463 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:38.463 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:38.463 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:38.463 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:38.463 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 134743 2024-09-14T04:40:38.463 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.464 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:38.464 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.464 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:38.464 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.921311+0000", 2024-09-14T04:40:38.464 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.464 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.464 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.465 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:38.465 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.921311+0000", 2024-09-14T04:40:38.465 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.465 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.465 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.465 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:38.465 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.921309+0000", 2024-09-14T04:40:38.465 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:40:38.465 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.465 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.465 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:38.465 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.921316+0000", 2024-09-14T04:40:38.465 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.9310000000000002e-06 2024-09-14T04:40:38.465 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.466 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.466 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:38.466 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.921333+0000", 2024-09-14T04:40:38.466 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.7138000000000001e-05 2024-09-14T04:40:38.466 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.466 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.466 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:38.466 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.921341+0000", 2024-09-14T04:40:38.466 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.7540000000000006e-06 2024-09-14T04:40:38.466 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.466 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.466 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:38.466 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.922657+0000", 2024-09-14T04:40:38.466 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0013164990000000001 2024-09-14T04:40:38.467 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.467 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.467 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:38.467 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.922771+0000", 2024-09-14T04:40:38.467 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000113638 2024-09-14T04:40:38.467 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.467 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.467 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:40:38.467 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.922879+0000", 2024-09-14T04:40:38.467 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000108268 2024-09-14T04:40:38.467 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.467 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.467 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:38.467 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.926705+0000", 2024-09-14T04:40:38.468 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.003825742 2024-09-14T04:40:38.468 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.468 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.468 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.468 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.928958+0000", 2024-09-14T04:40:38.468 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.002253222 2024-09-14T04:40:38.468 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.468 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.468 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.468 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:11.015369+0000", 2024-09-14T04:40:38.468 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.086411336000000005 2024-09-14T04:40:38.468 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.468 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.469 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:38.469 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:11.015429+0000", 2024-09-14T04:40:38.469 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9707000000000003e-05 2024-09-14T04:40:38.469 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.469 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.469 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:38.469 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:11.015468+0000", 2024-09-14T04:40:38.469 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8717999999999998e-05 2024-09-14T04:40:38.469 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.469 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:38.469 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.469 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.469 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.469 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:134748 53.8 53:14fea4a1:::benchmark_data_smithi039_224949_object4655:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1330)", 2024-09-14T04:40:38.469 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:40:10.928932+0000", 2024-09-14T04:40:38.470 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 27.40545573, 2024-09-14T04:40:38.470 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.086807507000000006, 2024-09-14T04:40:38.470 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:38.470 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:38.470 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:38.470 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:38.470 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:38.470 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 134748 2024-09-14T04:40:38.470 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.470 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:38.470 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.470 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:38.470 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.928932+0000", 2024-09-14T04:40:38.471 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.471 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.471 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.471 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:38.471 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.928932+0000", 2024-09-14T04:40:38.471 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:38.471 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.471 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.471 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:38.471 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.928930+0000", 2024-09-14T04:40:38.471 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:40:38.471 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.471 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.471 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:38.471 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.928935+0000", 2024-09-14T04:40:38.472 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.079e-06 2024-09-14T04:40:38.472 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.472 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.472 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:38.472 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.928945+0000", 2024-09-14T04:40:38.472 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.251e-06 2024-09-14T04:40:38.472 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.472 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.472 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:38.472 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.928951+0000", 2024-09-14T04:40:38.472 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.2020000000000001e-06 2024-09-14T04:40:38.472 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.472 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.472 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:38.473 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.928980+0000", 2024-09-14T04:40:38.473 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.9334000000000001e-05 2024-09-14T04:40:38.473 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.473 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.473 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:38.473 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.929093+0000", 2024-09-14T04:40:38.473 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000112922 2024-09-14T04:40:38.473 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.473 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.473 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:40:38.473 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.929206+0000", 2024-09-14T04:40:38.473 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000113235 2024-09-14T04:40:38.473 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.473 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.474 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.474 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.932212+0000", 2024-09-14T04:40:38.474 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.003005424 2024-09-14T04:40:38.474 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.474 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.474 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:38.474 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.932294+0000", 2024-09-14T04:40:38.474 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.2650000000000003e-05 2024-09-14T04:40:38.474 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.474 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.474 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:38.474 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:11.015666+0000", 2024-09-14T04:40:38.474 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.083372084999999999 2024-09-14T04:40:38.474 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.475 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.475 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:38.475 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:11.015706+0000", 2024-09-14T04:40:38.475 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9385000000000003e-05 2024-09-14T04:40:38.475 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:38.475 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:38.475 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:38.475 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:11.015739+0000", 2024-09-14T04:40:38.475 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3729000000000002e-05 2024-09-14T04:40:38.475 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.475 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:38.475 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.475 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:38.475 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:38.476 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:40:38.506 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:38.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:38.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:38.608 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:38.668 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:38.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:38.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:38.770 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:38.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:38 smithi129 ceph-mon[101316]: pgmap v2234: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 760 op/s 2024-09-14T04:40:38.812 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:38.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:38.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:38.904 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:38 smithi039 ceph-mon[106470]: pgmap v2234: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 760 op/s 2024-09-14T04:40:38.904 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:40:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:40:38] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.51.0" 2024-09-14T04:40:38.904 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:38 smithi039 ceph-mon[111235]: pgmap v2234: 89 pgs: 89 active+clean; 6.4 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 760 op/s 2024-09-14T04:40:38.914 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:38.942 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:38.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:38.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:39.044 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:39.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:39.085 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:39.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:39.186 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:39.219 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:39.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:39.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:39.321 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:39.326 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-14T04:40:39.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:39.366 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:39.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:39.467 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:39.515 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:39.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:39.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:39.616 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:39.643 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:40:39.662 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:39.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:39.663 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:39.764 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:39.797 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:40:39.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:39.818 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:39.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:39.919 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:39.954 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:39.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:39.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:40.056 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:40.079 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:40.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:40.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:40.181 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:40.222 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:40.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:40.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:40.324 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:40.349 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:40.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:40.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:40.451 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:40.493 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:40.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:40.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:40.595 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:40.622 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:40.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:40.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:40.691 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is unset 2024-09-14T04:40:40.698 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.6 to 1 (10000) 2024-09-14T04:40:40.724 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:40.747 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:40.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:40.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:40.849 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:40.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:40.875 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:40.875 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:40.909 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:40 smithi129 ceph-mon[101316]: pgmap v2235: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 902 op/s 2024-09-14T04:40:40.910 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:40 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/495214440' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:40:40.910 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:40 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:40:40.910 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:40 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2144525538' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:40:40.910 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:40 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:40:40.976 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:41.013 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:41.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:41.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:41.035 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:40 smithi039 ceph-mon[106470]: pgmap v2235: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 902 op/s 2024-09-14T04:40:41.036 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:40 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/495214440' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:40:41.036 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:40 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:40:41.036 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:40 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2144525538' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:40:41.036 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:40 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:40:41.036 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:40 smithi039 ceph-mon[111235]: pgmap v2235: 89 pgs: 89 active+clean; 6.2 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 902 op/s 2024-09-14T04:40:41.036 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:40 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/495214440' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:40:41.036 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:40 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:40:41.036 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:40 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2144525538' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:40:41.036 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:40 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:40:41.115 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:41.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:41.160 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:41.160 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:41.261 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:41.300 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:41.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:41.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:41.402 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:41.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:41.445 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:41.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:41.546 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:41.584 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:41.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:41.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:41.589 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:40:41.589 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [ 2024-09-14T04:40:41.589 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.590 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:162503 53.2 53:4db299d5:::benchmark_data_smithi039_224949_object32410:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1335)", 2024-09-14T04:40:41.590 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:40:41.571110+0000", 2024-09-14T04:40:41.590 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.016395162000000001, 2024-09-14T04:40:41.590 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.016426970999999999, 2024-09-14T04:40:41.590 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:41.590 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:40:41.590 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:41.590 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:41.590 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:41.590 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 162503 2024-09-14T04:40:41.590 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.590 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:41.590 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.591 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:41.591 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.571110+0000", 2024-09-14T04:40:41.591 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:41.591 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.591 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.591 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:41.591 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.571110+0000", 2024-09-14T04:40:41.591 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:41.591 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.591 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.591 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:41.591 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.571108+0000", 2024-09-14T04:40:41.591 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:40:41.592 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.592 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.592 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:41.592 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.571115+0000", 2024-09-14T04:40:41.592 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.9229999999999996e-06 2024-09-14T04:40:41.592 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.592 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.592 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:41.592 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.571131+0000", 2024-09-14T04:40:41.592 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.6339000000000001e-05 2024-09-14T04:40:41.592 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.592 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.593 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:41.593 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.571139+0000", 2024-09-14T04:40:41.593 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.6079999999999998e-06 2024-09-14T04:40:41.593 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.593 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.593 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:41.593 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.571193+0000", 2024-09-14T04:40:41.593 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.4500000000000003e-05 2024-09-14T04:40:41.593 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.593 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.593 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:41.593 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.571318+0000", 2024-09-14T04:40:41.593 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012423800000000001 2024-09-14T04:40:41.594 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.594 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.594 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:40:41.594 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.571431+0000", 2024-09-14T04:40:41.594 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011369100000000001 2024-09-14T04:40:41.594 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.594 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.594 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:41.594 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.576968+0000", 2024-09-14T04:40:41.594 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0055364719999999997 2024-09-14T04:40:41.594 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.594 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.594 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:41.595 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.585139+0000", 2024-09-14T04:40:41.595 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0081708079999999999 2024-09-14T04:40:41.595 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:41.595 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:41.595 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:41.595 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.595 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.595 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:162512 53.b 53:d59afd2f:::benchmark_data_smithi039_224949_object32419:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1335)", 2024-09-14T04:40:41.595 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:40:41.580990+0000", 2024-09-14T04:40:41.595 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.0065150800000000004, 2024-09-14T04:40:41.595 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0066305829999999998, 2024-09-14T04:40:41.595 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:41.595 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:40:41.596 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:41.596 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:41.596 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:41.596 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 162512 2024-09-14T04:40:41.596 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.596 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:41.596 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.596 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:41.596 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.580990+0000", 2024-09-14T04:40:41.596 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:41.597 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.597 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.597 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:41.597 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.580990+0000", 2024-09-14T04:40:41.597 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:41.597 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.597 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.597 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:41.597 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.580988+0000", 2024-09-14T04:40:41.597 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:40:41.597 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.597 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.598 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:41.598 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.580996+0000", 2024-09-14T04:40:41.598 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.1529999999999997e-06 2024-09-14T04:40:41.598 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.598 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.599 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:41.599 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.581015+0000", 2024-09-14T04:40:41.599 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.8343e-05 2024-09-14T04:40:41.599 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.599 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.599 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:41.599 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.581022+0000", 2024-09-14T04:40:41.599 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.1160000000000001e-06 2024-09-14T04:40:41.599 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.599 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.599 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:41.600 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.581067+0000", 2024-09-14T04:40:41.600 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.5179000000000002e-05 2024-09-14T04:40:41.600 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.600 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.600 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:41.600 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.581204+0000", 2024-09-14T04:40:41.600 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000137541 2024-09-14T04:40:41.600 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.600 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.600 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 4,6", 2024-09-14T04:40:41.601 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.581317+0000", 2024-09-14T04:40:41.602 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000112184 2024-09-14T04:40:41.602 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:41.602 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:41.602 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:41.602 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.602 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.602 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:162515 53.b 53:dfe28f5d:::benchmark_data_smithi039_224949_object32422:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1335)", 2024-09-14T04:40:41.602 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:40:41.585396+0000", 2024-09-14T04:40:41.602 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.002108445, 2024-09-14T04:40:41.602 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.002289452, 2024-09-14T04:40:41.602 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:41.602 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:40:41.602 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:41.603 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:41.603 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:41.603 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 162515 2024-09-14T04:40:41.603 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.603 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:41.603 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.603 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:41.603 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.585396+0000", 2024-09-14T04:40:41.603 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:41.603 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.603 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.603 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:41.603 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.585396+0000", 2024-09-14T04:40:41.603 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:41.604 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.604 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.604 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:41.604 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.585395+0000", 2024-09-14T04:40:41.604 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:40:41.604 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.604 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.604 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:41.604 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.585401+0000", 2024-09-14T04:40:41.604 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.3739999999999997e-06 2024-09-14T04:40:41.604 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.604 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.604 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:41.605 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.585410+0000", 2024-09-14T04:40:41.605 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.3649999999999993e-06 2024-09-14T04:40:41.605 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.605 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.605 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:41.605 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.585416+0000", 2024-09-14T04:40:41.605 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.2689999999999996e-06 2024-09-14T04:40:41.605 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.605 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.605 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:41.605 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.586116+0000", 2024-09-14T04:40:41.605 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00069953099999999998 2024-09-14T04:40:41.605 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.605 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.606 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:41.606 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.586217+0000", 2024-09-14T04:40:41.606 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000100698 2024-09-14T04:40:41.606 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.606 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.606 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 4,6", 2024-09-14T04:40:41.606 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.586319+0000", 2024-09-14T04:40:41.606 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000102058 2024-09-14T04:40:41.606 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:41.606 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:41.606 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:41.606 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.606 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.607 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:162516 53.2 53:43f71688:::benchmark_data_smithi039_224949_object32423:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1335)", 2024-09-14T04:40:41.607 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:40:41.586053+0000", 2024-09-14T04:40:41.607 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.0014523540000000001, 2024-09-14T04:40:41.607 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001699978, 2024-09-14T04:40:41.607 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:41.607 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:40:41.607 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:41.607 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:41.607 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:41.607 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 162516 2024-09-14T04:40:41.607 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.607 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:41.607 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.607 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:41.608 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.586053+0000", 2024-09-14T04:40:41.608 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:41.608 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.608 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.608 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:41.608 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.586053+0000", 2024-09-14T04:40:41.608 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:41.608 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.608 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.608 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:41.608 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.586051+0000", 2024-09-14T04:40:41.608 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:40:41.608 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.608 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.609 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:41.609 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.586058+0000", 2024-09-14T04:40:41.609 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.4250000000000003e-06 2024-09-14T04:40:41.609 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.609 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.609 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:41.609 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.586072+0000", 2024-09-14T04:40:41.609 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3997e-05 2024-09-14T04:40:41.609 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.609 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.609 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:41.609 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.586079+0000", 2024-09-14T04:40:41.610 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.6639999999999998e-06 2024-09-14T04:40:41.610 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.610 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.610 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:41.610 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.586129+0000", 2024-09-14T04:40:41.610 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.9271999999999999e-05 2024-09-14T04:40:41.610 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.610 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.610 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:41.610 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.586243+0000", 2024-09-14T04:40:41.610 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000114268 2024-09-14T04:40:41.610 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.610 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.611 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:40:41.611 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.586360+0000", 2024-09-14T04:40:41.611 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000117321 2024-09-14T04:40:41.611 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:41.611 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:41.611 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:41.611 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.611 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.611 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.45053.0:162514 53.0 e1335/1315)", 2024-09-14T04:40:41.611 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:40:41.586931+0000", 2024-09-14T04:40:41.611 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.00057352800000000002, 2024-09-14T04:40:41.611 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00088135700000000004, 2024-09-14T04:40:41.611 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:41.612 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "queued for pg", 2024-09-14T04:40:41.612 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:41.612 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.612 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:41.612 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.586931+0000", 2024-09-14T04:40:41.612 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:41.612 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.612 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.612 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:41.612 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.586931+0000", 2024-09-14T04:40:41.612 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:41.612 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.612 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.613 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:41.613 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.586930+0000", 2024-09-14T04:40:41.613 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:40:41.613 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.613 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.613 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:41.613 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.586938+0000", 2024-09-14T04:40:41.613 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.7130000000000005e-06 2024-09-14T04:40:41.613 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.613 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.613 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:41.613 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.586949+0000", 2024-09-14T04:40:41.613 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1138000000000001e-05 2024-09-14T04:40:41.614 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:41.614 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:41.614 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:41.614 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:41.586955+0000", 2024-09-14T04:40:41.614 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9000000000000003e-06 2024-09-14T04:40:41.614 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:41.614 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:41.614 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:41.614 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:41.614 INFO:teuthology.orchestra.run.smithi129.stdout: ], 2024-09-14T04:40:41.614 INFO:teuthology.orchestra.run.smithi129.stdout: "num_ops": 5 2024-09-14T04:40:41.614 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:40:41.686 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:41.722 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:41.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:41.723 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:41 smithi129 ceph-mon[101316]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:40:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:41 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-14T04:40:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:41 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:40:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:41 smithi129 ceph-mon[101316]: osdmap e1335: 8 total, 8 up, 8 in 2024-09-14T04:40:41.824 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:41.861 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:41.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:41.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:41.963 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:41.999 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:42.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:42.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:42.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:41 smithi039 ceph-mon[106470]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:40:42.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:41 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-14T04:40:42.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:41 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:40:42.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:41 smithi039 ceph-mon[106470]: osdmap e1335: 8 total, 8 up, 8 in 2024-09-14T04:40:42.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:41 smithi039 ceph-mon[111235]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:40:42.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:41 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-14T04:40:42.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:41 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:40:42.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:41 smithi039 ceph-mon[111235]: osdmap e1335: 8 total, 8 up, 8 in 2024-09-14T04:40:42.101 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:42.123 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:42.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:42.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:42.225 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:42.257 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:42.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:42.258 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:42.359 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:42.381 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-14T04:40:42.400 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:42.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:42.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:42.502 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:42.541 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:42.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:42.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:42.644 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:42.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:42.669 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:42.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:42.688 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:42 smithi129 ceph-mon[101316]: pgmap v2237: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 874 op/s 2024-09-14T04:40:42.693 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:40:42.771 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:42.807 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:42.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:42.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:42.909 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:42.943 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:42.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:42.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:43.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:42 smithi039 ceph-mon[106470]: pgmap v2237: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 874 op/s 2024-09-14T04:40:43.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:42 smithi039 ceph-mon[111235]: pgmap v2237: 89 pgs: 89 active+clean; 6.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 874 op/s 2024-09-14T04:40:43.044 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:43.090 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:43.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:43.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:43.192 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:43.229 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:43.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:43.230 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:43.331 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:43.365 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:43.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:43.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:43.467 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:43.520 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:43.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:43.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:43.622 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:43.654 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:43.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:43.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:43.756 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:43.782 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:43.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:43.783 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:43.884 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:43.917 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:43.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:43.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:44.019 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:44.051 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:44.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:44.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:44.153 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:44.187 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:44.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:44.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:44.289 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:44.321 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:44.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:44.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:44.423 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:44.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:44.461 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:44.462 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:44.562 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:44.596 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:44.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:44.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:44.697 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:44.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:44.735 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:44.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:44.746 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:40:44.746 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [], 2024-09-14T04:40:44.746 INFO:teuthology.orchestra.run.smithi129.stdout: "complaint_time": 30, 2024-09-14T04:40:44.746 INFO:teuthology.orchestra.run.smithi129.stdout: "num_blocked_ops": 0 2024-09-14T04:40:44.747 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:40:44.836 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:44.879 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:44.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:44.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:44.981 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:45.013 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:45.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:45.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:45.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:44 smithi129 ceph-mon[101316]: pgmap v2238: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 874 op/s 2024-09-14T04:40:45.024 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:44 smithi039 ceph-mon[106470]: pgmap v2238: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 874 op/s 2024-09-14T04:40:45.024 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:44 smithi039 ceph-mon[111235]: pgmap v2238: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 874 op/s 2024-09-14T04:40:45.115 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:45.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:45.157 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:45.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:45.258 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:45.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:45.305 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:45.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:45.406 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:45.444 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:45.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:45.445 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:45.464 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-14T04:40:45.545 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:45.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:45.581 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:45.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:45.682 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:45.719 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:45.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:45.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:45.822 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:45.831 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 7 1 2024-09-14T04:40:45.835 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:40:45.861 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:45.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:45.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:45.963 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:45.987 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:45.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:45.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:46.079 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:40:46.079 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:40:46.089 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:46.121 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:46.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:46.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:46.124 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:40:46.124 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:40:46.156 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:40:46.157 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:40:46.197 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:40:46.198 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:40:46.223 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:46.237 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:40:46.237 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:40:46.243 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:40:46.270 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:40:46.271 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:40:46.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:46.282 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:46.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:46.302 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:40:46.303 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:40:46.337 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:40:46.337 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:40:46.383 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:46.424 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:46.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:46.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:46.526 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:46.562 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:46.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:46.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:46.664 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:46.711 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:46.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:46.712 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:46.813 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:46.845 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:46.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:46.847 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:46.947 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:46.983 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:46.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:46.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:47.010 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:46 smithi039 ceph-mon[106470]: pgmap v2239: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 866 op/s 2024-09-14T04:40:47.010 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:46 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1994473546' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:40:47.010 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:46 smithi039 ceph-mon[111235]: pgmap v2239: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 866 op/s 2024-09-14T04:40:47.010 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:46 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1994473546' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:40:47.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:46 smithi129 ceph-mon[101316]: pgmap v2239: 89 pgs: 89 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 866 op/s 2024-09-14T04:40:47.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:46 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1994473546' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:40:47.066 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:40:47.085 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:47.149 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:47.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:47.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:47.251 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:47.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:47.298 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:47.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:47.399 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:47.434 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:47.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:47.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:47.535 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:47.565 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:47.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:47.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:47.667 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:47.709 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:47.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:47.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:47.811 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:47.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:47.856 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:47.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:47.957 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:47.989 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:47.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:47.990 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:48.091 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:48.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:48.137 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:48.137 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:48.238 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:48.276 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:40:48.276 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:40:48.279 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:48.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:48.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:48.333 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:40:48.381 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:48.417 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:48.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:48.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:48.519 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:48.554 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:48.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:48.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:48.656 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:48.701 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:48.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:48.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:48.803 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:48.837 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:48.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:48.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:48.920 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:48 smithi129 ceph-mon[101316]: pgmap v2240: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 864 op/s 2024-09-14T04:40:48.939 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:48.951 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:48 smithi039 ceph-mon[106470]: pgmap v2240: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 864 op/s 2024-09-14T04:40:48.952 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:40:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:40:48] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.51.0" 2024-09-14T04:40:48.952 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:48 smithi039 ceph-mon[111235]: pgmap v2240: 89 pgs: 89 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 864 op/s 2024-09-14T04:40:48.962 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-14T04:40:48.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:48.975 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:48.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:49.077 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:49.114 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:49.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:49.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:49.216 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:49.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:49.249 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:49.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:49.324 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:40:49.350 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:49.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:49.386 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:49.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:49.487 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:49.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:49.531 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:49.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:49.632 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:49.676 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:49.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:49.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:49.779 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:49.799 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:40:49.821 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:49.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:49.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:49.923 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:49.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:49.989 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:49.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:50.090 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:50.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:50.134 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:50.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:50.235 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:50.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:50.276 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:50.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:50.377 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:50.405 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:50.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:50.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:50.507 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:50.534 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:50.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:50.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:50.636 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:50.675 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:50.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:50.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:50.708 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:40:50.732 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.7 to 1 (10000) 2024-09-14T04:40:50.777 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:50.804 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:50.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:50.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:50.906 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:50.935 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:50.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:50.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:50.996 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:50 smithi129 ceph-mon[101316]: pgmap v2241: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 852 op/s 2024-09-14T04:40:50.996 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:50 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1584507194' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:40:51.037 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:51.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:50 smithi039 ceph-mon[106470]: pgmap v2241: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 852 op/s 2024-09-14T04:40:51.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:50 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1584507194' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:40:51.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:50 smithi039 ceph-mon[111235]: pgmap v2241: 89 pgs: 89 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 852 op/s 2024-09-14T04:40:51.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:50 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1584507194' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:40:51.067 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:51.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:51.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:51.169 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:51.194 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:51.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:51.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:51.296 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:51.330 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:51.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:51.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:51.430 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:40:51.430 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:40:51.431 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:51.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:51.469 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:51.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:51.570 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:51.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:51.603 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:51.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:51.704 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:51.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:51.737 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:51.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:51.775 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is unset 2024-09-14T04:40:51.838 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:51.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:51.875 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:51.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:51.897 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-14T04:40:51.898 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:40:51.977 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:52.011 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:52.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:52.012 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:52.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:51 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1584507194' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-14T04:40:52.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:51 smithi039 ceph-mon[106470]: osdmap e1336: 8 total, 8 up, 8 in 2024-09-14T04:40:52.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:51 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2340057337' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:40:52.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:51 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1584507194' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-14T04:40:52.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:51 smithi039 ceph-mon[111235]: osdmap e1336: 8 total, 8 up, 8 in 2024-09-14T04:40:52.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:51 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2340057337' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:40:52.078 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:51 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1584507194' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-14T04:40:52.078 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:51 smithi129 ceph-mon[101316]: osdmap e1336: 8 total, 8 up, 8 in 2024-09-14T04:40:52.078 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:51 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2340057337' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:40:52.107 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-14T04:40:52.113 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:52.183 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:52.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:52.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:52.285 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:52.326 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:52.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:52.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:52.427 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:52.441 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:40:52.477 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:52.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:52.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:52.579 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:52.615 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:52.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:52.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:52.716 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:52.755 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:52.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:52.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:52.857 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:52.895 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:52.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:52.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:52.997 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:52 smithi129 ceph-mon[101316]: pgmap v2243: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2024-09-14T04:40:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:52 smithi129 ceph-mon[101316]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:40:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:52 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T04:40:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:52 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2340057337' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:40:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:52 smithi129 ceph-mon[101316]: osdmap e1337: 8 total, 8 up, 8 in 2024-09-14T04:40:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:52 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:40:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:40:53.031 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:53.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:53.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:53.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:52 smithi039 ceph-mon[111235]: pgmap v2243: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2024-09-14T04:40:53.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:52 smithi039 ceph-mon[111235]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:40:53.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:52 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T04:40:53.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:52 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2340057337' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:40:53.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:52 smithi039 ceph-mon[111235]: osdmap e1337: 8 total, 8 up, 8 in 2024-09-14T04:40:53.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:52 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:40:53.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:40:53.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:52 smithi039 ceph-mon[106470]: pgmap v2243: 89 pgs: 89 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 857 op/s 2024-09-14T04:40:53.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:52 smithi039 ceph-mon[106470]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:40:53.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:52 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T04:40:53.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:52 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2340057337' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:40:53.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:52 smithi039 ceph-mon[106470]: osdmap e1337: 8 total, 8 up, 8 in 2024-09-14T04:40:53.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:52 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:40:53.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:40:53.133 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:53.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:53.170 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:53.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:53.271 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:53.298 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:53.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:53.299 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:53.400 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:53.436 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:53.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:53.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:53.538 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:53.568 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:53.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:53.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:53.647 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:40:53.669 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:53.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:53.701 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:53.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:53.802 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:53.832 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:53.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:53.833 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:53.934 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:53.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:53.969 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:53.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:54.070 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:54.107 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:54.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:54.108 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:54.209 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:54.249 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:54.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:54.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:54.351 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:54.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:54.388 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:54.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:54.488 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:40:54.488 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:40:54.489 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:54.521 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:40:54.521 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:40:54.536 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:54.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:54.537 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:54.544 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:40:54.559 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:40:54.559 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:40:54.596 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:40:54.596 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:40:54.629 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:40:54.630 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:40:54.638 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:54.669 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:40:54.669 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:40:54.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:54.683 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:54.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:54.707 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:40:54.708 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:40:54.737 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:40:54.737 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:40:54.785 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:54.830 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:54.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:54.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:54.842 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:54 smithi039 ceph-mon[106470]: pgmap v2245: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 965 op/s 2024-09-14T04:40:54.842 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:54 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3829961399' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:40:54.843 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:54 smithi039 ceph-mon[111235]: pgmap v2245: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 965 op/s 2024-09-14T04:40:54.843 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:54 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3829961399' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:40:54.933 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:54.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:54.974 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:54.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:55.075 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:55.108 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:55.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:55.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:55.210 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:55.245 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:55.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:55.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:55.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:54 smithi129 ceph-mon[101316]: pgmap v2245: 89 pgs: 89 active+clean; 5.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 965 op/s 2024-09-14T04:40:55.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:54 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3829961399' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:40:55.346 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:55.396 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:55.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:55.397 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:55.498 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:55.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:55.538 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:55.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:55.639 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:55.691 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:55.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:55.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:55.793 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:55.826 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:55.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:55.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:55.928 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:55.955 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:55.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:55.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:56.057 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:56.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:56.094 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:56.094 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:56.195 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:56.247 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:56.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:56.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:56.349 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:56.359 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:40:56.380 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:56.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:56.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:56.482 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:56.515 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:56.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:56.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:56.617 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:56.652 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:56.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:56.653 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:56.754 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:56.789 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:56.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:56.790 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:56.835 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:40:56.835 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [], 2024-09-14T04:40:56.835 INFO:teuthology.orchestra.run.smithi129.stdout: "complaint_time": 30, 2024-09-14T04:40:56.835 INFO:teuthology.orchestra.run.smithi129.stdout: "num_blocked_ops": 0 2024-09-14T04:40:56.835 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:40:56.891 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:56.936 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:56.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:56.938 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:56.994 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:56 smithi039 ceph-mon[106470]: pgmap v2246: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-14T04:40:56.995 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:56 smithi039 ceph-mon[111235]: pgmap v2246: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-14T04:40:57.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:56 smithi129 ceph-mon[101316]: pgmap v2246: 89 pgs: 89 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s 2024-09-14T04:40:57.038 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:40:57.085 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:57.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:57.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:40:57.187 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:57.219 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:57.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:57.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:57.321 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:57.356 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:57.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:57.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:57.458 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:57.504 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:57.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:57.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:57.545 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-14T04:40:57.606 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:57.656 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:57.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:57.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:57.758 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:57.794 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:57.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:57.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:57.884 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:40:57.896 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:57.932 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:57.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:57.933 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:58.034 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:58.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:58.079 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:58.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:58.180 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:58.215 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:58.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:58.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:58.312 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:40:58.317 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:58.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:58.414 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:58.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:58.515 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:58.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:58.549 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:58.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:58.650 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:58.697 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:58.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:58.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:58.799 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:40:58.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:58.831 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:58.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:40:58.932 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:58.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:58.976 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:58.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:58.980 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:58 smithi039 ceph-mon[106470]: pgmap v2247: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 766 op/s 2024-09-14T04:40:58.980 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:40:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:40:58] "GET /metrics HTTP/1.1" 200 36746 "" "Prometheus/2.51.0" 2024-09-14T04:40:58.981 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:58 smithi039 ceph-mon[111235]: pgmap v2247: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 766 op/s 2024-09-14T04:40:58.986 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:40:58.987 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T04:40:59.078 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:59.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:59.118 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:59.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:40:59.157 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:58 smithi129 ceph-mon[101316]: pgmap v2247: 89 pgs: 89 active+clean; 5.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 766 op/s 2024-09-14T04:40:59.220 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:40:59.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:59.252 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:59.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:40:59.353 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:40:59.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:59.393 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:59.393 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:40:59.494 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:40:59.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:59.537 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:59.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:40:59.586 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:40:59.638 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:40:59.673 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:40:59.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:59.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:40:59.775 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:40:59.819 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:40:59.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:40:59.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:40:59.879 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:40:59.879 INFO:teuthology.orchestra.run.smithi129.stdout: "size": 20, 2024-09-14T04:40:59.879 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 600, 2024-09-14T04:40:59.879 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [ 2024-09-14T04:40:59.879 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.879 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:29386 53.f 53:fcdef1c5:::benchmark_data_smithi039_224949_object29385:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-14T04:40:59.880 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:38:45.993701+0000", 2024-09-14T04:40:59.880 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 133.836752139, 2024-09-14T04:40:59.880 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.111609922, 2024-09-14T04:40:59.880 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:59.880 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:59.880 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:59.880 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:59.880 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:59.880 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 29386 2024-09-14T04:40:59.880 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.880 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:59.880 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.880 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:59.881 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.993701+0000", 2024-09-14T04:40:59.881 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.881 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.881 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.881 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:59.881 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.993701+0000", 2024-09-14T04:40:59.881 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.881 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.881 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.881 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:59.881 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.993700+0000", 2024-09-14T04:40:59.881 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:40:59.881 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.882 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.882 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:59.882 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.993767+0000", 2024-09-14T04:40:59.882 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7080000000000001e-05 2024-09-14T04:40:59.882 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.882 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.882 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:59.882 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.993784+0000", 2024-09-14T04:40:59.882 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.7564999999999999e-05 2024-09-14T04:40:59.882 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.882 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.882 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:59.882 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.993791+0000", 2024-09-14T04:40:59.883 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.6930000000000001e-06 2024-09-14T04:40:59.883 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.883 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.883 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:59.883 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.993849+0000", 2024-09-14T04:40:59.883 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.8122999999999999e-05 2024-09-14T04:40:59.883 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.883 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.883 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:59.883 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.993973+0000", 2024-09-14T04:40:59.883 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000124146 2024-09-14T04:40:59.883 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.883 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.884 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:40:59.884 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.994109+0000", 2024-09-14T04:40:59.884 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013525899999999999 2024-09-14T04:40:59.884 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.884 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.884 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:59.884 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.995442+0000", 2024-09-14T04:40:59.884 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0013337080000000001 2024-09-14T04:40:59.884 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.884 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.884 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.884 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.998149+0000", 2024-09-14T04:40:59.884 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0027067189999999998 2024-09-14T04:40:59.884 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.885 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.885 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.885 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.105183+0000", 2024-09-14T04:40:59.885 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.10703407199999999 2024-09-14T04:40:59.885 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.885 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.885 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:59.885 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.105273+0000", 2024-09-14T04:40:59.885 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.9530000000000005e-05 2024-09-14T04:40:59.885 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.885 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.885 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:59.885 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.105311+0000", 2024-09-14T04:40:59.886 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8390999999999999e-05 2024-09-14T04:40:59.886 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.886 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:59.886 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.886 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.886 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.886 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:29393 53.f 53:fb85c396:::benchmark_data_smithi039_224949_object29392:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-14T04:40:59.886 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:38:45.995894+0000", 2024-09-14T04:40:59.886 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 133.834559348, 2024-09-14T04:40:59.886 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.115771892, 2024-09-14T04:40:59.886 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:59.886 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:59.886 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:59.886 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:59.887 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:59.887 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 29393 2024-09-14T04:40:59.887 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.887 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:59.887 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.887 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:59.887 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.995894+0000", 2024-09-14T04:40:59.887 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.887 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.887 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.887 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:59.887 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.995894+0000", 2024-09-14T04:40:59.887 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.887 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.887 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.888 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:59.888 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.995893+0000", 2024-09-14T04:40:59.888 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:40:59.888 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.888 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.888 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:59.888 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.995940+0000", 2024-09-14T04:40:59.888 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7593000000000003e-05 2024-09-14T04:40:59.888 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.888 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.888 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:59.889 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.995956+0000", 2024-09-14T04:40:59.889 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5832999999999999e-05 2024-09-14T04:40:59.889 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.889 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.889 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:59.889 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.995962+0000", 2024-09-14T04:40:59.889 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.4189999999999999e-06 2024-09-14T04:40:59.889 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.889 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.889 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:59.889 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.996567+0000", 2024-09-14T04:40:59.889 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00060409799999999996 2024-09-14T04:40:59.889 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.889 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.890 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:59.890 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.996710+0000", 2024-09-14T04:40:59.890 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00014312600000000001 2024-09-14T04:40:59.890 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.890 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.890 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:40:59.890 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.996848+0000", 2024-09-14T04:40:59.890 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000138338 2024-09-14T04:40:59.890 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.890 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.890 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.890 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.999130+0000", 2024-09-14T04:40:59.890 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0022814749999999998 2024-09-14T04:40:59.890 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.891 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.891 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:59.891 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.999276+0000", 2024-09-14T04:40:59.891 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00014641799999999999 2024-09-14T04:40:59.891 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.891 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.891 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.891 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.111584+0000", 2024-09-14T04:40:59.891 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.11230854 2024-09-14T04:40:59.891 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.891 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.891 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:59.891 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.111632+0000", 2024-09-14T04:40:59.891 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7052999999999998e-05 2024-09-14T04:40:59.891 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.892 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.892 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:59.892 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.111666+0000", 2024-09-14T04:40:59.892 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.4384000000000001e-05 2024-09-14T04:40:59.892 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.892 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:59.892 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.892 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.892 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.892 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:29395 53.3 53:cac015a6:::benchmark_data_smithi039_224949_object29394:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-14T04:40:59.892 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:38:45.996438+0000", 2024-09-14T04:40:59.892 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 133.834015448, 2024-09-14T04:40:59.892 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.116360031, 2024-09-14T04:40:59.893 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:59.893 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:59.893 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:59.893 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:59.893 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:59.893 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 29395 2024-09-14T04:40:59.893 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.893 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:59.893 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.893 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:59.893 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.996438+0000", 2024-09-14T04:40:59.893 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.893 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.893 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.894 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:59.894 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.996438+0000", 2024-09-14T04:40:59.894 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.894 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.894 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.894 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:59.894 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.996437+0000", 2024-09-14T04:40:59.894 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:40:59.894 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.894 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.894 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:59.894 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.996483+0000", 2024-09-14T04:40:59.894 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.5992000000000003e-05 2024-09-14T04:40:59.895 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.895 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.895 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:59.895 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.996498+0000", 2024-09-14T04:40:59.895 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5324000000000001e-05 2024-09-14T04:40:59.895 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.895 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.895 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:59.895 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.996506+0000", 2024-09-14T04:40:59.895 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.7139999999999993e-06 2024-09-14T04:40:59.895 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.895 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.895 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:59.895 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.996966+0000", 2024-09-14T04:40:59.895 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00045954600000000002 2024-09-14T04:40:59.896 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.896 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.896 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:59.896 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.997104+0000", 2024-09-14T04:40:59.896 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013867999999999999 2024-09-14T04:40:59.896 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.896 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.896 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:40:59.896 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.997241+0000", 2024-09-14T04:40:59.896 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013638999999999999 2024-09-14T04:40:59.896 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.896 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.896 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:59.896 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.999298+0000", 2024-09-14T04:40:59.897 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0020571750000000001 2024-09-14T04:40:59.897 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.897 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.897 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.897 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.002282+0000", 2024-09-14T04:40:59.897 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0029844680000000001 2024-09-14T04:40:59.897 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.897 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.897 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.897 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.112724+0000", 2024-09-14T04:40:59.897 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.110441568 2024-09-14T04:40:59.897 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.897 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.897 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:59.898 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.112766+0000", 2024-09-14T04:40:59.898 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1758999999999998e-05 2024-09-14T04:40:59.898 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.898 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.898 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:59.898 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.112798+0000", 2024-09-14T04:40:59.898 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.2302999999999999e-05 2024-09-14T04:40:59.898 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.898 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:59.898 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.898 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.898 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.899 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:29399 53.3 53:c3e5d755:::benchmark_data_smithi039_224949_object29398:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-14T04:40:59.899 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:38:45.999557+0000", 2024-09-14T04:40:59.899 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 133.830896779, 2024-09-14T04:40:59.899 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.113332952, 2024-09-14T04:40:59.899 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:59.899 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:59.899 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:59.899 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:59.899 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:59.899 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 29399 2024-09-14T04:40:59.899 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.899 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:59.899 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.900 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:59.900 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.999557+0000", 2024-09-14T04:40:59.900 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.900 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.900 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.900 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:59.900 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.999557+0000", 2024-09-14T04:40:59.900 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.900 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.900 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.900 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:59.900 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.999555+0000", 2024-09-14T04:40:59.900 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:40:59.900 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.901 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.901 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:59.901 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.999629+0000", 2024-09-14T04:40:59.901 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.4873e-05 2024-09-14T04:40:59.901 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.901 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.901 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:59.901 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.999646+0000", 2024-09-14T04:40:59.901 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.6558999999999998e-05 2024-09-14T04:40:59.901 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.901 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.901 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:59.901 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.999654+0000", 2024-09-14T04:40:59.901 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.8439999999999998e-06 2024-09-14T04:40:59.902 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.902 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.902 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:59.902 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.999703+0000", 2024-09-14T04:40:59.902 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8699999999999998e-05 2024-09-14T04:40:59.902 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.902 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.902 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:59.902 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.999840+0000", 2024-09-14T04:40:59.902 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013731299999999999 2024-09-14T04:40:59.902 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.902 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.902 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:40:59.902 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.000003+0000", 2024-09-14T04:40:59.903 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000162869 2024-09-14T04:40:59.903 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.903 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.903 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:59.903 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.002293+0000", 2024-09-14T04:40:59.903 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0022905540000000002 2024-09-14T04:40:59.903 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.903 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.903 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.903 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.002485+0000", 2024-09-14T04:40:59.903 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00019141199999999999 2024-09-14T04:40:59.903 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.903 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.903 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.904 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.112830+0000", 2024-09-14T04:40:59.904 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.110344875 2024-09-14T04:40:59.904 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.904 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.904 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:59.904 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.112862+0000", 2024-09-14T04:40:59.904 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.2162999999999997e-05 2024-09-14T04:40:59.904 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.904 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.904 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:59.904 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.112890+0000", 2024-09-14T04:40:59.904 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.7820000000000001e-05 2024-09-14T04:40:59.904 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.905 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:59.905 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.905 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.905 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.905 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:29400 53.3 53:c771cf78:::benchmark_data_smithi039_224949_object29399:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-14T04:40:59.905 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:38:45.999719+0000", 2024-09-14T04:40:59.905 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 133.83073402299999, 2024-09-14T04:40:59.905 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.113265301, 2024-09-14T04:40:59.905 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:59.905 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:59.905 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:59.905 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:59.905 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:59.905 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 29400 2024-09-14T04:40:59.906 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.906 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:59.906 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.906 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:59.906 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.999719+0000", 2024-09-14T04:40:59.906 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.906 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.906 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.906 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:59.906 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.999719+0000", 2024-09-14T04:40:59.906 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.906 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.906 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.906 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:59.906 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.999719+0000", 2024-09-14T04:40:59.907 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:40:59.907 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.907 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.907 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:59.907 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.999832+0000", 2024-09-14T04:40:59.907 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000113041 2024-09-14T04:40:59.907 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.907 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.907 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:59.907 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.999838+0000", 2024-09-14T04:40:59.907 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.0900000000000001e-06 2024-09-14T04:40:59.907 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.907 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.908 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:59.908 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:45.999842+0000", 2024-09-14T04:40:59.908 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6859999999999999e-06 2024-09-14T04:40:59.908 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.908 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.908 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:59.908 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.001061+0000", 2024-09-14T04:40:59.908 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00121897 2024-09-14T04:40:59.908 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.908 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.908 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:59.908 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.001187+0000", 2024-09-14T04:40:59.908 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012662900000000001 2024-09-14T04:40:59.909 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.909 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.909 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:40:59.909 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.001311+0000", 2024-09-14T04:40:59.909 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012354600000000001 2024-09-14T04:40:59.909 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.909 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.909 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:59.909 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.002347+0000", 2024-09-14T04:40:59.909 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001036297 2024-09-14T04:40:59.909 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.909 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.909 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.909 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.002635+0000", 2024-09-14T04:40:59.910 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00028816099999999999 2024-09-14T04:40:59.910 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.910 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.910 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.910 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.112925+0000", 2024-09-14T04:40:59.910 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.11028962 2024-09-14T04:40:59.910 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.910 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.910 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:59.910 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.112957+0000", 2024-09-14T04:40:59.910 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.1924000000000001e-05 2024-09-14T04:40:59.910 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.910 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.910 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:59.911 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.112985+0000", 2024-09-14T04:40:59.911 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.7852e-05 2024-09-14T04:40:59.911 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.911 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:59.911 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.911 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.911 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.911 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:29408 53.3 53:c293bea9:::benchmark_data_smithi039_224949_object29407:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-14T04:40:59.911 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:38:46.004100+0000", 2024-09-14T04:40:59.911 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 133.82635321000001, 2024-09-14T04:40:59.911 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.10898516900000001, 2024-09-14T04:40:59.911 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:59.911 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:59.912 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:59.912 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:59.912 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:59.912 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 29408 2024-09-14T04:40:59.912 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.912 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:59.912 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.912 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:59.912 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.004100+0000", 2024-09-14T04:40:59.912 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.912 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.912 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.912 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:59.912 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.004100+0000", 2024-09-14T04:40:59.913 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.913 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.913 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.913 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:59.913 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.004099+0000", 2024-09-14T04:40:59.914 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:40:59.914 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.914 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.914 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:59.915 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.004151+0000", 2024-09-14T04:40:59.915 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.1641999999999998e-05 2024-09-14T04:40:59.915 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.915 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.915 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:59.915 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.004162+0000", 2024-09-14T04:40:59.915 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1586000000000001e-05 2024-09-14T04:40:59.915 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.915 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.915 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:59.915 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.004167+0000", 2024-09-14T04:40:59.915 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2499999999999997e-06 2024-09-14T04:40:59.915 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.916 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.916 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:59.916 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.004202+0000", 2024-09-14T04:40:59.916 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.4632000000000003e-05 2024-09-14T04:40:59.916 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.916 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.916 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:59.916 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.004282+0000", 2024-09-14T04:40:59.916 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.9406000000000005e-05 2024-09-14T04:40:59.916 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.916 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.916 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:40:59.916 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.004371+0000", 2024-09-14T04:40:59.917 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.9260999999999998e-05 2024-09-14T04:40:59.917 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.917 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.917 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:59.917 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.005139+0000", 2024-09-14T04:40:59.917 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00076797200000000001 2024-09-14T04:40:59.917 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.917 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.917 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.917 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.005388+0000", 2024-09-14T04:40:59.917 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00024925400000000002 2024-09-14T04:40:59.917 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.917 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.918 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.918 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.113021+0000", 2024-09-14T04:40:59.918 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.107632623 2024-09-14T04:40:59.918 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.918 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.918 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:59.918 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.113057+0000", 2024-09-14T04:40:59.918 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5890999999999999e-05 2024-09-14T04:40:59.918 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.918 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.918 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:59.918 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:38:46.113085+0000", 2024-09-14T04:40:59.918 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.8809000000000001e-05 2024-09-14T04:40:59.918 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.919 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:59.919 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.919 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.919 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.919 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:73829 53.f 53:fa329c58:::benchmark_data_smithi039_224949_object73828:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-09-14T04:40:59.919 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:39:12.628588+0000", 2024-09-14T04:40:59.919 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 107.20186561600001, 2024-09-14T04:40:59.919 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.11013993900000001, 2024-09-14T04:40:59.919 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:59.919 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:59.919 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:59.919 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:59.919 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:59.919 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 73829 2024-09-14T04:40:59.920 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.920 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:59.920 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.920 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:59.920 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.628588+0000", 2024-09-14T04:40:59.920 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.920 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.920 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.920 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:59.920 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.628588+0000", 2024-09-14T04:40:59.920 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.920 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.920 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.921 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:59.921 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.628576+0000", 2024-09-14T04:40:59.921 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999886 2024-09-14T04:40:59.921 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.921 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.921 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:59.921 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.628634+0000", 2024-09-14T04:40:59.921 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.7834000000000002e-05 2024-09-14T04:40:59.921 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.921 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.921 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:59.921 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.628663+0000", 2024-09-14T04:40:59.921 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.9563000000000001e-05 2024-09-14T04:40:59.921 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.922 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.922 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:59.922 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.628683+0000", 2024-09-14T04:40:59.922 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.9735999999999999e-05 2024-09-14T04:40:59.922 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.922 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.922 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:59.922 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.628771+0000", 2024-09-14T04:40:59.922 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.8103000000000007e-05 2024-09-14T04:40:59.922 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.922 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.922 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:59.923 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.628926+0000", 2024-09-14T04:40:59.923 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000155181 2024-09-14T04:40:59.923 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.923 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.923 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:40:59.923 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.629044+0000", 2024-09-14T04:40:59.923 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000117866 2024-09-14T04:40:59.923 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.923 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.923 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:59.923 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.633205+0000", 2024-09-14T04:40:59.923 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.004161049 2024-09-14T04:40:59.923 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.923 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.924 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.924 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.633240+0000", 2024-09-14T04:40:59.924 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.4381999999999997e-05 2024-09-14T04:40:59.924 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.924 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.924 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.924 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.738643+0000", 2024-09-14T04:40:59.924 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.105403102 2024-09-14T04:40:59.924 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.924 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.924 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:59.924 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.738692+0000", 2024-09-14T04:40:59.924 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8868e-05 2024-09-14T04:40:59.925 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.925 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.925 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:59.925 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.738728+0000", 2024-09-14T04:40:59.925 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5918999999999999e-05 2024-09-14T04:40:59.925 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.925 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:59.925 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.925 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.925 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.925 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:73830 53.f 53:f0fc3480:::benchmark_data_smithi039_224949_object73829:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-09-14T04:40:59.925 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:39:12.628724+0000", 2024-09-14T04:40:59.925 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 107.201729113, 2024-09-14T04:40:59.925 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.110432924, 2024-09-14T04:40:59.926 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:59.926 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:59.926 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:59.926 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:59.926 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:59.926 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 73830 2024-09-14T04:40:59.926 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.926 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:59.926 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.926 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:59.926 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.628724+0000", 2024-09-14T04:40:59.926 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.926 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.926 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.927 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:59.927 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.628724+0000", 2024-09-14T04:40:59.927 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.927 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.927 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.927 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:59.927 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.628724+0000", 2024-09-14T04:40:59.927 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:40:59.927 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.927 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.927 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:59.927 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.628749+0000", 2024-09-14T04:40:59.927 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.5848000000000001e-05 2024-09-14T04:40:59.927 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.928 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.928 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:59.928 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.628758+0000", 2024-09-14T04:40:59.928 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.7800000000000006e-06 2024-09-14T04:40:59.928 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.928 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.928 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:59.928 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.628766+0000", 2024-09-14T04:40:59.928 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.0039999999999999e-06 2024-09-14T04:40:59.928 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.928 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.928 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:59.928 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.630851+0000", 2024-09-14T04:40:59.929 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0020851160000000001 2024-09-14T04:40:59.929 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.929 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.929 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:59.929 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.630992+0000", 2024-09-14T04:40:59.929 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00014102099999999999 2024-09-14T04:40:59.929 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.929 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.929 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:40:59.929 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.631127+0000", 2024-09-14T04:40:59.929 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013437799999999999 2024-09-14T04:40:59.929 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.929 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.930 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:59.930 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.633362+0000", 2024-09-14T04:40:59.930 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0022352439999999999 2024-09-14T04:40:59.930 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.930 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.930 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.930 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.633564+0000", 2024-09-14T04:40:59.930 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000201716 2024-09-14T04:40:59.930 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.930 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.930 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.930 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.739074+0000", 2024-09-14T04:40:59.930 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.105510258 2024-09-14T04:40:59.931 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.931 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.931 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:59.931 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.739123+0000", 2024-09-14T04:40:59.931 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8653999999999999e-05 2024-09-14T04:40:59.931 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.931 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.931 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:59.931 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.739157+0000", 2024-09-14T04:40:59.931 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.4521999999999999e-05 2024-09-14T04:40:59.931 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.931 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:59.931 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.931 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.932 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.932 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:73835 53.3 53:c0079b2a:::benchmark_data_smithi039_224949_object73834:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-09-14T04:40:59.932 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:39:12.631319+0000", 2024-09-14T04:40:59.932 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 107.199133973, 2024-09-14T04:40:59.932 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.104263961, 2024-09-14T04:40:59.932 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:59.932 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:59.932 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:59.932 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:59.932 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:59.932 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 73835 2024-09-14T04:40:59.932 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.932 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:59.932 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.933 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:59.933 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.631319+0000", 2024-09-14T04:40:59.933 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.933 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.933 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.933 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:59.933 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.631319+0000", 2024-09-14T04:40:59.933 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.933 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.933 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.933 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:59.933 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.631318+0000", 2024-09-14T04:40:59.933 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:40:59.933 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.934 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.934 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:59.934 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.631386+0000", 2024-09-14T04:40:59.934 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7800999999999997e-05 2024-09-14T04:40:59.934 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.934 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.934 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:59.934 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.631398+0000", 2024-09-14T04:40:59.934 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.235e-05 2024-09-14T04:40:59.934 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.934 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.934 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:59.934 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.631404+0000", 2024-09-14T04:40:59.934 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.4670000000000003e-06 2024-09-14T04:40:59.935 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.935 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.935 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:59.935 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.631450+0000", 2024-09-14T04:40:59.935 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6436000000000001e-05 2024-09-14T04:40:59.935 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.935 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.935 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:59.935 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.631576+0000", 2024-09-14T04:40:59.935 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000125105 2024-09-14T04:40:59.935 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.935 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.935 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:40:59.935 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.631703+0000", 2024-09-14T04:40:59.936 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000127234 2024-09-14T04:40:59.936 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.936 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.936 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:59.936 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.634029+0000", 2024-09-14T04:40:59.936 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0023262859999999999 2024-09-14T04:40:59.936 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.936 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.936 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.936 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.634266+0000", 2024-09-14T04:40:59.936 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00023665599999999999 2024-09-14T04:40:59.936 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.936 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.937 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.937 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.735522+0000", 2024-09-14T04:40:59.937 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.101256314 2024-09-14T04:40:59.937 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.937 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.937 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:59.937 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.735557+0000", 2024-09-14T04:40:59.937 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.451e-05 2024-09-14T04:40:59.937 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.937 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.937 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:59.937 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:12.735583+0000", 2024-09-14T04:40:59.937 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.6837000000000001e-05 2024-09-14T04:40:59.937 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.938 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:59.938 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.938 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.938 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.938 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:76186 53.f 53:f0ee6bfe:::benchmark_data_smithi039_224949_object76185:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-09-14T04:40:59.938 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:39:14.579113+0000", 2024-09-14T04:40:59.938 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 105.251339999, 2024-09-14T04:40:59.938 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.108369623, 2024-09-14T04:40:59.938 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:59.938 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:59.938 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:59.938 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:59.938 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:59.938 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 76186 2024-09-14T04:40:59.938 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.939 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:59.939 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.939 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:59.939 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.579113+0000", 2024-09-14T04:40:59.939 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.939 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.939 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.939 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:59.939 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.579113+0000", 2024-09-14T04:40:59.939 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.939 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.939 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.939 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:59.939 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.579111+0000", 2024-09-14T04:40:59.939 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:40:59.940 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.940 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.940 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:59.940 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.579183+0000", 2024-09-14T04:40:59.940 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.1613999999999996e-05 2024-09-14T04:40:59.940 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.940 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.940 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:59.940 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.579192+0000", 2024-09-14T04:40:59.940 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.4550000000000001e-06 2024-09-14T04:40:59.940 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.940 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.940 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:59.940 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.579199+0000", 2024-09-14T04:40:59.941 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.3609999999999997e-06 2024-09-14T04:40:59.941 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.941 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.941 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:59.941 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.579539+0000", 2024-09-14T04:40:59.941 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00034069599999999998 2024-09-14T04:40:59.941 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.941 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.941 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:59.941 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.579683+0000", 2024-09-14T04:40:59.941 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000143387 2024-09-14T04:40:59.941 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.941 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.941 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:40:59.942 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.582902+0000", 2024-09-14T04:40:59.942 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0032194929999999999 2024-09-14T04:40:59.942 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.942 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.942 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:59.942 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.585944+0000", 2024-09-14T04:40:59.942 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0030411710000000001 2024-09-14T04:40:59.942 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.942 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.942 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.942 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.586574+0000", 2024-09-14T04:40:59.942 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00063000700000000003 2024-09-14T04:40:59.942 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.942 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.943 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.943 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.687368+0000", 2024-09-14T04:40:59.943 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.100794462 2024-09-14T04:40:59.943 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.943 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.943 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:59.943 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.687446+0000", 2024-09-14T04:40:59.943 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.8281999999999994e-05 2024-09-14T04:40:59.943 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.943 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.943 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:59.943 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.687483+0000", 2024-09-14T04:40:59.943 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6677000000000001e-05 2024-09-14T04:40:59.943 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.943 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:59.944 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.944 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.944 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.944 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:76201 53.f 53:f344f31c:::benchmark_data_smithi039_224949_object76200:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-09-14T04:40:59.944 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:39:14.588170+0000", 2024-09-14T04:40:59.944 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 105.242283265, 2024-09-14T04:40:59.944 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.10900038099999999, 2024-09-14T04:40:59.944 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:59.944 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:59.944 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:59.944 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:59.944 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:59.944 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 76201 2024-09-14T04:40:59.945 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.945 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:59.945 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.945 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:59.945 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.588170+0000", 2024-09-14T04:40:59.945 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.945 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.945 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.945 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:59.945 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.588170+0000", 2024-09-14T04:40:59.945 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.945 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.945 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.945 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:59.946 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.588169+0000", 2024-09-14T04:40:59.946 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:40:59.946 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.946 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.946 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:59.946 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.588238+0000", 2024-09-14T04:40:59.946 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8979000000000005e-05 2024-09-14T04:40:59.946 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.946 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.946 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:59.946 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.588253+0000", 2024-09-14T04:40:59.946 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5508000000000001e-05 2024-09-14T04:40:59.946 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.946 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.947 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:59.947 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.588260+0000", 2024-09-14T04:40:59.947 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.4680000000000003e-06 2024-09-14T04:40:59.947 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.947 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.947 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:59.947 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.588313+0000", 2024-09-14T04:40:59.947 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2883999999999997e-05 2024-09-14T04:40:59.947 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.947 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.947 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:59.947 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.588449+0000", 2024-09-14T04:40:59.947 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000135258 2024-09-14T04:40:59.948 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.948 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.948 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:40:59.948 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.588591+0000", 2024-09-14T04:40:59.948 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00014192100000000001 2024-09-14T04:40:59.948 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.948 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.948 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:59.948 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.591692+0000", 2024-09-14T04:40:59.948 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0031014010000000002 2024-09-14T04:40:59.948 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.948 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.948 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.948 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.592158+0000", 2024-09-14T04:40:59.949 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00046655399999999999 2024-09-14T04:40:59.949 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.949 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.949 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.949 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.697018+0000", 2024-09-14T04:40:59.949 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.10485968499999999 2024-09-14T04:40:59.949 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.949 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.949 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:59.950 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.697107+0000", 2024-09-14T04:40:59.950 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.8708000000000005e-05 2024-09-14T04:40:59.950 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.950 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.950 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:59.950 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:39:14.697170+0000", 2024-09-14T04:40:59.950 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.3581e-05 2024-09-14T04:40:59.950 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.950 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:59.950 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.950 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.950 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.950 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:133861 53.3 53:ccb86fcf:::benchmark_data_smithi039_224949_object3768:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1330)", 2024-09-14T04:40:59.951 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:40:09.859471+0000", 2024-09-14T04:40:59.951 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 49.970982691000003, 2024-09-14T04:40:59.951 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.110945197, 2024-09-14T04:40:59.951 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:59.951 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:59.951 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:59.951 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:59.951 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:59.951 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 133861 2024-09-14T04:40:59.951 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.951 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:59.951 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.951 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:59.952 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.859471+0000", 2024-09-14T04:40:59.952 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.952 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.952 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.952 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:59.952 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.859471+0000", 2024-09-14T04:40:59.952 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.952 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.952 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.952 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:59.952 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.859469+0000", 2024-09-14T04:40:59.952 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:40:59.952 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.952 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.953 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:59.953 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.859474+0000", 2024-09-14T04:40:59.953 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.9579999999999998e-06 2024-09-14T04:40:59.953 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.953 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.953 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:59.953 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.859484+0000", 2024-09-14T04:40:59.953 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.6749999999999997e-06 2024-09-14T04:40:59.953 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.953 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.953 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:59.953 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.859490+0000", 2024-09-14T04:40:59.953 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.3799999999999999e-06 2024-09-14T04:40:59.953 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.954 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.954 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:59.954 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.859795+0000", 2024-09-14T04:40:59.954 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00030491900000000001 2024-09-14T04:40:59.954 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.954 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.954 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:59.954 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.859982+0000", 2024-09-14T04:40:59.954 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00018634200000000001 2024-09-14T04:40:59.954 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.954 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.954 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:40:59.954 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.860129+0000", 2024-09-14T04:40:59.954 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000147792 2024-09-14T04:40:59.955 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.955 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.955 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:59.955 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.864133+0000", 2024-09-14T04:40:59.955 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0040039910000000001 2024-09-14T04:40:59.955 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.955 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.955 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.955 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.867934+0000", 2024-09-14T04:40:59.955 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0038002410000000002 2024-09-14T04:40:59.955 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.955 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.955 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.956 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.970329+0000", 2024-09-14T04:40:59.956 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.102395763 2024-09-14T04:40:59.956 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.956 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.956 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:59.956 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.970375+0000", 2024-09-14T04:40:59.956 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.5893000000000003e-05 2024-09-14T04:40:59.956 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.956 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.956 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:59.956 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.970416+0000", 2024-09-14T04:40:59.956 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0535999999999999e-05 2024-09-14T04:40:59.956 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.956 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:59.957 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.957 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.957 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.957 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:133862 53.e 53:707e12ab:::benchmark_data_smithi039_224949_object3769:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1330)", 2024-09-14T04:40:59.957 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:40:09.860186+0000", 2024-09-14T04:40:59.957 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 49.970267401000001, 2024-09-14T04:40:59.957 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.11018760900000001, 2024-09-14T04:40:59.957 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:59.957 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:59.957 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:59.957 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:59.957 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:59.957 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 133862 2024-09-14T04:40:59.957 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.958 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:59.958 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.958 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:59.958 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.860186+0000", 2024-09-14T04:40:59.958 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.958 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.958 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.958 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:59.958 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.860186+0000", 2024-09-14T04:40:59.958 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.958 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.958 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.958 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:59.958 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.860185+0000", 2024-09-14T04:40:59.959 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:40:59.959 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.959 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.959 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:59.959 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.860190+0000", 2024-09-14T04:40:59.959 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8709999999999996e-06 2024-09-14T04:40:59.959 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.959 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.959 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:59.959 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.860198+0000", 2024-09-14T04:40:59.959 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.3210000000000005e-06 2024-09-14T04:40:59.959 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.959 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.959 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:59.960 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.860205+0000", 2024-09-14T04:40:59.960 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.148e-06 2024-09-14T04:40:59.960 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.960 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.960 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:59.960 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.860244+0000", 2024-09-14T04:40:59.960 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8865000000000001e-05 2024-09-14T04:40:59.960 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.960 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.960 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:59.960 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.860350+0000", 2024-09-14T04:40:59.960 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010559400000000001 2024-09-14T04:40:59.960 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.960 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.961 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:40:59.961 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.860441+0000", 2024-09-14T04:40:59.961 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.0746000000000002e-05 2024-09-14T04:40:59.961 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.961 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.961 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:59.961 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.864291+0000", 2024-09-14T04:40:59.961 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0038505700000000002 2024-09-14T04:40:59.961 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.961 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.961 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.961 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.865876+0000", 2024-09-14T04:40:59.961 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001584961 2024-09-14T04:40:59.961 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.962 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.962 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.962 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.970291+0000", 2024-09-14T04:40:59.962 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.10441439199999999 2024-09-14T04:40:59.962 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.962 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.962 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:59.962 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.970337+0000", 2024-09-14T04:40:59.962 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6134000000000002e-05 2024-09-14T04:40:59.962 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.962 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.962 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:59.962 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.970374+0000", 2024-09-14T04:40:59.962 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6811999999999999e-05 2024-09-14T04:40:59.963 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.963 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:59.963 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.963 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.963 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.963 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:133864 53.3 53:cfde2c03:::benchmark_data_smithi039_224949_object3771:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1330)", 2024-09-14T04:40:59.963 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:40:09.864265+0000", 2024-09-14T04:40:59.963 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 49.966188146999997, 2024-09-14T04:40:59.963 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.106274249, 2024-09-14T04:40:59.963 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:59.963 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:59.963 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:59.963 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:59.963 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:59.964 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 133864 2024-09-14T04:40:59.964 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.964 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:59.964 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.964 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:59.964 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.864265+0000", 2024-09-14T04:40:59.964 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.964 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.964 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.964 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:59.964 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.864265+0000", 2024-09-14T04:40:59.964 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.964 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.964 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.965 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:59.965 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.864264+0000", 2024-09-14T04:40:59.965 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:40:59.965 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.965 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.965 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:59.965 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.864270+0000", 2024-09-14T04:40:59.965 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9630000000000003e-06 2024-09-14T04:40:59.965 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.965 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.965 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:59.965 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.864280+0000", 2024-09-14T04:40:59.965 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0465999999999999e-05 2024-09-14T04:40:59.965 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.966 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.966 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:59.966 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.864287+0000", 2024-09-14T04:40:59.966 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.4219999999999997e-06 2024-09-14T04:40:59.966 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.966 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.966 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:59.966 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.864333+0000", 2024-09-14T04:40:59.966 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.5725e-05 2024-09-14T04:40:59.966 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.966 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.966 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:59.966 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.864414+0000", 2024-09-14T04:40:59.966 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.1054e-05 2024-09-14T04:40:59.967 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.967 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.967 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:40:59.967 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.864483+0000", 2024-09-14T04:40:59.967 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.9146000000000005e-05 2024-09-14T04:40:59.967 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.967 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.967 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:59.967 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.868686+0000", 2024-09-14T04:40:59.967 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0042032110000000001 2024-09-14T04:40:59.967 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.967 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.967 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.967 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.876118+0000", 2024-09-14T04:40:59.967 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0074316790000000001 2024-09-14T04:40:59.968 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.968 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.968 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.968 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.970455+0000", 2024-09-14T04:40:59.968 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.094337346000000002 2024-09-14T04:40:59.968 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.968 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.968 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:59.968 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.970503+0000", 2024-09-14T04:40:59.968 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8288000000000003e-05 2024-09-14T04:40:59.968 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.968 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.968 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:59.969 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.970539+0000", 2024-09-14T04:40:59.969 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6229999999999997e-05 2024-09-14T04:40:59.969 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.969 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:59.969 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.969 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.969 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.969 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:133872 53.3 53:c5e47f4e:::benchmark_data_smithi039_224949_object3779:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1330)", 2024-09-14T04:40:59.969 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:40:09.871689+0000", 2024-09-14T04:40:59.969 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 49.958764702000003, 2024-09-14T04:40:59.969 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.107211944, 2024-09-14T04:40:59.969 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:59.969 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:59.969 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:59.970 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:59.970 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:59.970 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 133872 2024-09-14T04:40:59.970 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.970 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:59.970 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.970 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:59.970 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.871689+0000", 2024-09-14T04:40:59.970 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.970 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.970 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.970 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:59.970 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.871689+0000", 2024-09-14T04:40:59.971 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.971 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.971 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.971 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:59.971 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.871687+0000", 2024-09-14T04:40:59.971 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:40:59.971 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.971 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.971 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:59.971 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.871695+0000", 2024-09-14T04:40:59.971 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.8080000000000005e-06 2024-09-14T04:40:59.971 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.972 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.972 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:59.972 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.871711+0000", 2024-09-14T04:40:59.972 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.6413e-05 2024-09-14T04:40:59.972 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.972 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.972 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:59.972 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.871718+0000", 2024-09-14T04:40:59.972 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.9859999999999997e-06 2024-09-14T04:40:59.972 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.972 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.972 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:59.972 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.871824+0000", 2024-09-14T04:40:59.972 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000105926 2024-09-14T04:40:59.973 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.973 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.973 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:59.973 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.871938+0000", 2024-09-14T04:40:59.973 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000114178 2024-09-14T04:40:59.973 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.973 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.973 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:40:59.973 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.872039+0000", 2024-09-14T04:40:59.973 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010052699999999999 2024-09-14T04:40:59.973 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.973 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.973 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:59.973 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.876765+0000", 2024-09-14T04:40:59.974 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0047260920000000003 2024-09-14T04:40:59.974 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.974 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.974 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.974 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.879590+0000", 2024-09-14T04:40:59.974 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.002824624 2024-09-14T04:40:59.974 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.974 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.974 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.974 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.978733+0000", 2024-09-14T04:40:59.974 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.099143389999999998 2024-09-14T04:40:59.974 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.974 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.974 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:59.975 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.978812+0000", 2024-09-14T04:40:59.975 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.9128999999999994e-05 2024-09-14T04:40:59.975 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.975 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.975 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:59.975 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.978901+0000", 2024-09-14T04:40:59.975 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.8529999999999994e-05 2024-09-14T04:40:59.975 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.975 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:59.975 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.975 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.975 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.975 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:133875 53.f 53:ff192831:::benchmark_data_smithi039_224949_object3782:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1330)", 2024-09-14T04:40:59.975 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:40:09.872461+0000", 2024-09-14T04:40:59.975 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 49.957992077, 2024-09-14T04:40:59.976 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.12950610600000001, 2024-09-14T04:40:59.976 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:59.976 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:59.976 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:59.976 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:59.976 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:59.976 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 133875 2024-09-14T04:40:59.976 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.976 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:59.976 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.976 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:59.976 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.872461+0000", 2024-09-14T04:40:59.976 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.976 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.977 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.977 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:59.977 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.872461+0000", 2024-09-14T04:40:59.977 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.977 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.977 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.977 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:40:59.978 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:59.979 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.872460+0000", 2024-09-14T04:40:59.979 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:40:59.979 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.979 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.979 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:59.979 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.872466+0000", 2024-09-14T04:40:59.979 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.4500000000000003e-06 2024-09-14T04:40:59.979 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.979 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.979 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:59.979 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.872478+0000", 2024-09-14T04:40:59.979 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2252999999999999e-05 2024-09-14T04:40:59.979 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.980 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.980 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:59.980 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.872485+0000", 2024-09-14T04:40:59.980 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.844e-06 2024-09-14T04:40:59.980 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.980 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.980 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:59.980 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.872531+0000", 2024-09-14T04:40:59.980 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6610000000000003e-05 2024-09-14T04:40:59.980 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.980 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.980 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:59.980 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.872635+0000", 2024-09-14T04:40:59.981 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010344300000000001 2024-09-14T04:40:59.981 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.981 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.981 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:40:59.981 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.872727+0000", 2024-09-14T04:40:59.981 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.2147999999999999e-05 2024-09-14T04:40:59.981 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.981 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.981 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:59.981 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.880040+0000", 2024-09-14T04:40:59.981 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0073125289999999999 2024-09-14T04:40:59.981 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.981 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.981 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.982 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.883763+0000", 2024-09-14T04:40:59.982 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0037232810000000002 2024-09-14T04:40:59.982 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.982 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.982 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.982 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.001899+0000", 2024-09-14T04:40:59.982 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.118135806 2024-09-14T04:40:59.982 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.983 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.984 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:59.984 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.001940+0000", 2024-09-14T04:40:59.984 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1859e-05 2024-09-14T04:40:59.984 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.984 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.984 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:59.984 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.001967+0000", 2024-09-14T04:40:59.984 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.6902000000000001e-05 2024-09-14T04:40:59.984 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.985 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:59.985 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.985 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.985 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.985 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:133876 53.e 53:7b155125:::benchmark_data_smithi039_224949_object3783:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1330)", 2024-09-14T04:40:59.985 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:40:09.873010+0000", 2024-09-14T04:40:59.985 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 49.957443812000001, 2024-09-14T04:40:59.985 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.12064151400000001, 2024-09-14T04:40:59.985 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:59.985 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:59.985 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:59.985 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:59.985 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:59.986 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 133876 2024-09-14T04:40:59.986 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.986 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:59.986 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.986 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:59.986 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.873010+0000", 2024-09-14T04:40:59.986 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.986 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.986 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.986 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:59.986 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.873010+0000", 2024-09-14T04:40:59.986 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.986 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.987 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.987 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:59.987 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.873009+0000", 2024-09-14T04:40:59.987 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:40:59.987 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.987 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.987 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:59.987 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.873014+0000", 2024-09-14T04:40:59.987 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.9280000000000001e-06 2024-09-14T04:40:59.987 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.987 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.987 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:59.987 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.873033+0000", 2024-09-14T04:40:59.987 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.9440999999999999e-05 2024-09-14T04:40:59.988 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.988 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.988 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:59.988 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.873041+0000", 2024-09-14T04:40:59.988 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.4410000000000004e-06 2024-09-14T04:40:59.988 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.988 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.988 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:59.988 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.873099+0000", 2024-09-14T04:40:59.988 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.8078000000000002e-05 2024-09-14T04:40:59.988 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.988 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.988 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:59.988 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.873204+0000", 2024-09-14T04:40:59.989 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000105476 2024-09-14T04:40:59.989 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.989 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.989 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:40:59.989 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.873297+0000", 2024-09-14T04:40:59.989 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.2545000000000002e-05 2024-09-14T04:40:59.989 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.989 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.989 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.989 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.877209+0000", 2024-09-14T04:40:59.989 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0039119840000000003 2024-09-14T04:40:59.989 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.989 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.990 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:59.990 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.877360+0000", 2024-09-14T04:40:59.990 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0001518 2024-09-14T04:40:59.990 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.990 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.990 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.990 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.993543+0000", 2024-09-14T04:40:59.990 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.11618216200000001 2024-09-14T04:40:59.990 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.990 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.990 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:59.990 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.993613+0000", 2024-09-14T04:40:59.990 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.0701e-05 2024-09-14T04:40:59.990 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.991 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.991 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:59.991 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.993651+0000", 2024-09-14T04:40:59.991 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7787999999999998e-05 2024-09-14T04:40:59.991 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.991 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:59.991 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.991 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.991 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.991 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:133877 53.f 53:f3590d33:::benchmark_data_smithi039_224949_object3784:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1330)", 2024-09-14T04:40:59.991 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:40:09.875509+0000", 2024-09-14T04:40:59.991 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 49.954944288999997, 2024-09-14T04:40:59.991 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.126625354, 2024-09-14T04:40:59.991 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:59.992 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:59.992 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:59.992 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:59.992 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:59.992 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 133877 2024-09-14T04:40:59.992 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.992 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:59.992 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.992 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:59.992 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.875509+0000", 2024-09-14T04:40:59.992 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.992 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.992 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.993 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:59.993 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.875509+0000", 2024-09-14T04:40:59.993 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.993 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.993 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.993 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:59.993 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.875507+0000", 2024-09-14T04:40:59.993 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:40:59.993 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.993 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.993 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:40:59.993 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.875514+0000", 2024-09-14T04:40:59.993 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.2480000000000001e-06 2024-09-14T04:40:59.993 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.994 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.994 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:40:59.994 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.875525+0000", 2024-09-14T04:40:59.994 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1352000000000001e-05 2024-09-14T04:40:59.994 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.994 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.994 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:40:59.994 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.875532+0000", 2024-09-14T04:40:59.994 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.1890000000000001e-06 2024-09-14T04:40:59.994 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.994 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.994 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:40:59.994 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.876410+0000", 2024-09-14T04:40:59.995 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00087783100000000003 2024-09-14T04:40:59.995 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.995 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.995 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:40:59.995 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.876544+0000", 2024-09-14T04:40:59.995 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013356300000000001 2024-09-14T04:40:59.995 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.995 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.995 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:40:59.995 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.876655+0000", 2024-09-14T04:40:59.995 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000111879 2024-09-14T04:40:59.995 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.995 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.995 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:40:59.996 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.883727+0000", 2024-09-14T04:40:59.996 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0070713240000000004 2024-09-14T04:40:59.996 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.996 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.996 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.996 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.884610+0000", 2024-09-14T04:40:59.996 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00088370099999999997 2024-09-14T04:40:59.996 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.996 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.996 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:40:59.996 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.002080+0000", 2024-09-14T04:40:59.996 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.11746994700000001 2024-09-14T04:40:59.996 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.997 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.997 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:40:59.997 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.002111+0000", 2024-09-14T04:40:59.997 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.0765000000000001e-05 2024-09-14T04:40:59.997 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.997 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.997 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:40:59.997 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.002134+0000", 2024-09-14T04:40:59.997 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.3215999999999999e-05 2024-09-14T04:40:59.997 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.997 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:40:59.997 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:40:59.997 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.997 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.997 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:133880 53.f 53:f3324a09:::benchmark_data_smithi039_224949_object3787:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1330)", 2024-09-14T04:40:59.998 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:40:09.876901+0000", 2024-09-14T04:40:59.998 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 49.953552260999999, 2024-09-14T04:40:59.998 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.13501423300000001, 2024-09-14T04:40:59.998 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:40:59.998 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:40:59.998 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:40:59.998 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:40:59.998 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:40:59.998 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 133880 2024-09-14T04:40:59.998 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.998 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:40:59.998 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.998 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:40:59.999 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.876901+0000", 2024-09-14T04:40:59.999 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.999 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.999 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.999 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:40:59.999 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.876901+0000", 2024-09-14T04:40:59.999 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:40:59.999 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.999 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:40:59.999 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:40:59.999 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.876900+0000", 2024-09-14T04:40:59.999 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:40:59.999 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:40:59.999 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:00.000 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:41:00.000 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.876905+0000", 2024-09-14T04:41:00.000 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2780000000000001e-06 2024-09-14T04:41:00.000 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:00.000 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:00.000 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:41:00.000 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.876922+0000", 2024-09-14T04:41:00.000 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.6427e-05 2024-09-14T04:41:00.000 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:00.000 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:00.000 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:41:00.000 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.876928+0000", 2024-09-14T04:41:00.000 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.5950000000000004e-06 2024-09-14T04:41:00.000 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:00.001 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:00.001 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:41:00.001 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.880083+0000", 2024-09-14T04:41:00.001 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0031543420000000001 2024-09-14T04:41:00.001 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:00.001 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:00.001 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:41:00.001 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.880243+0000", 2024-09-14T04:41:00.001 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000160128 2024-09-14T04:41:00.001 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:00.001 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:00.001 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:41:00.002 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.880353+0000", 2024-09-14T04:41:00.002 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000110627 2024-09-14T04:41:00.002 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:00.002 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:00.002 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:41:00.002 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.883828+0000", 2024-09-14T04:41:00.002 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0034744139999999999 2024-09-14T04:41:00.002 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:00.002 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:00.002 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:00.002 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:09.884725+0000", 2024-09-14T04:41:00.002 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00089682100000000001 2024-09-14T04:41:00.002 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:00.003 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:00.003 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:00.003 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.011813+0000", 2024-09-14T04:41:00.003 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.12708794200000001 2024-09-14T04:41:00.003 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:00.003 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:00.003 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:41:00.003 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.011864+0000", 2024-09-14T04:41:00.003 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.1128999999999998e-05 2024-09-14T04:41:00.003 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:00.003 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:00.003 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:41:00.003 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.011915+0000", 2024-09-14T04:41:00.003 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.1619000000000002e-05 2024-09-14T04:41:00.004 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:41:00.004 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:41:00.004 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:41:00.004 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:00.004 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:00.004 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:134741 53.3 53:c982cb6a:::benchmark_data_smithi039_224949_object4648:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1330)", 2024-09-14T04:41:00.004 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:40:10.921222+0000", 2024-09-14T04:41:00.004 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 48.909231525999999, 2024-09-14T04:41:00.004 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.10968191400000001, 2024-09-14T04:41:00.004 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:41:00.004 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:41:00.004 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:41:00.004 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:41:00.005 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:41:00.005 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 134741 2024-09-14T04:41:00.005 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:00.005 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:41:00.005 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:00.005 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:41:00.005 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.921222+0000", 2024-09-14T04:41:00.005 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:41:00.005 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:00.005 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:00.005 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:41:00.005 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.921222+0000", 2024-09-14T04:41:00.005 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:41:00.005 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:00.006 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:00.006 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:41:00.006 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.921221+0000", 2024-09-14T04:41:00.006 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:41:00.006 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:00.006 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:00.006 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:41:00.006 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.921228+0000", 2024-09-14T04:41:00.006 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.9539999999999998e-06 2024-09-14T04:41:00.006 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:00.006 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:00.006 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:41:00.006 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.921243+0000", 2024-09-14T04:41:00.006 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.4807000000000001e-05 2024-09-14T04:41:00.007 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:00.007 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:00.007 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:41:00.007 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.921250+0000", 2024-09-14T04:41:00.007 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.9650000000000002e-06 2024-09-14T04:41:00.007 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:00.007 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:00.007 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:41:00.007 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.921292+0000", 2024-09-14T04:41:00.007 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.2681e-05 2024-09-14T04:41:00.007 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:00.007 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:00.007 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:41:00.007 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.921404+0000", 2024-09-14T04:41:00.007 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011171 2024-09-14T04:41:00.008 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:00.008 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:00.008 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:41:00.008 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.921517+0000", 2024-09-14T04:41:00.008 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000112611 2024-09-14T04:41:00.008 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:00.008 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:00.008 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:41:00.008 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.925814+0000", 2024-09-14T04:41:00.008 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0042977420000000002 2024-09-14T04:41:00.008 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:00.008 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:00.008 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:00.009 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:10.930000+0000", 2024-09-14T04:41:00.009 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0041858360000000001 2024-09-14T04:41:00.009 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:00.009 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:00.009 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:00.009 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:11.030814+0000", 2024-09-14T04:41:00.009 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.10081367099999999 2024-09-14T04:41:00.009 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:00.009 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:00.009 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:41:00.009 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:11.030866+0000", 2024-09-14T04:41:00.009 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.1913000000000003e-05 2024-09-14T04:41:00.009 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:00.009 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:00.010 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:41:00.010 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:40:11.030904+0000", 2024-09-14T04:41:00.010 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7910999999999997e-05 2024-09-14T04:41:00.010 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:41:00.010 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:41:00.010 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:41:00.010 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:41:00.010 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:41:00.010 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:41:00.012 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:40:59 smithi039 ceph-mon[111235]: from='client.35280 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:41:00.012 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:40:59 smithi039 ceph-mon[106470]: from='client.35280 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:41:00.015 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:00.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:00.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:00.047 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":2247,"stamp":"2024-09-14T04:40:57.547020+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5693446141,"num_objects":86893,"num_object_clones":0,"num_object_copies":260679,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":86893,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":173594,"num_write_kb":8328210,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137823,"ondisk_log_size":137823,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18315180,"kb_used_data":16834452,"kb_used_omap":288,"kb_used_meta":1480095,"kb_avail":731646036,"statfs":{"total":767960285184,"available":749205540864,"internally_reserved":0,"allocated":17238478848,"data_stored":17207176350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":294978,"internal_metadata":1515618238},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,4,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-401866752,"num_objects":-6132,"num_object_clones":0,"num_object_copies":-18396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-6132,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6132,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002886"},"pg_stats":[{"pgid":"53.f","version":"1337'10673","reported_seq":10705,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:52.860894+0000","last_change":"2024-09-14T04:38:26.895527+0000","last_active":"2024-09-14T04:40:52.860894+0000","last_peered":"2024-09-14T04:40:52.860894+0000","last_clean":"2024-09-14T04:40:52.860894+0000","last_became_active":"2024-09-14T04:38:26.895218+0000","last_became_peered":"2024-09-14T04:38:26.895218+0000","last_unstale":"2024-09-14T04:40:52.860894+0000","last_undegraded":"2024-09-14T04:40:52.860894+0000","last_fullsized":"2024-09-14T04:40:52.860894+0000","mapping_epoch":1315,"log_start":"1319'900","ondisk_log_start":"1319'900","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":9773,"log_dups_size":0,"ondisk_log_size":9773,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:07:35.310515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351993856,"num_objects":5371,"num_object_clones":0,"num_object_copies":16113,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5371,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10673,"num_write_kb":513408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"53.e","version":"1337'10572","reported_seq":10604,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:52.831828+0000","last_change":"2024-09-14T04:38:26.894036+0000","last_active":"2024-09-14T04:40:52.831828+0000","last_peered":"2024-09-14T04:40:52.831828+0000","last_clean":"2024-09-14T04:40:52.831828+0000","last_became_active":"2024-09-14T04:38:26.893654+0000","last_became_peered":"2024-09-14T04:38:26.893654+0000","last_unstale":"2024-09-14T04:40:52.831828+0000","last_undegraded":"2024-09-14T04:40:52.831828+0000","last_fullsized":"2024-09-14T04:40:52.831828+0000","mapping_epoch":1315,"log_start":"1319'800","ondisk_log_start":"1319'800","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":9772,"log_dups_size":0,"ondisk_log_size":9772,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:39:17.902111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359792640,"num_objects":5490,"num_object_clones":0,"num_object_copies":16470,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5490,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10572,"num_write_kb":513984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"53.d","version":"1337'10768","reported_seq":10826,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:54.599807+0000","last_change":"2024-09-14T04:38:27.529546+0000","last_active":"2024-09-14T04:40:54.599807+0000","last_peered":"2024-09-14T04:40:54.599807+0000","last_clean":"2024-09-14T04:40:54.599807+0000","last_became_active":"2024-09-14T04:38:27.529237+0000","last_became_peered":"2024-09-14T04:38:27.529237+0000","last_unstale":"2024-09-14T04:40:54.599807+0000","last_undegraded":"2024-09-14T04:40:54.599807+0000","last_fullsized":"2024-09-14T04:40:54.599807+0000","mapping_epoch":1315,"log_start":"1320'3600","ondisk_log_start":"1320'3600","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":7168,"log_dups_size":0,"ondisk_log_size":7168,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:37:40.618858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355926040,"num_objects":5432,"num_object_clones":0,"num_object_copies":16296,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5432,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10768,"num_write_kb":518337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"53.c","version":"1337'11044","reported_seq":11075,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:52.965062+0000","last_change":"2024-09-14T04:38:26.895807+0000","last_active":"2024-09-14T04:40:52.965062+0000","last_peered":"2024-09-14T04:40:52.965062+0000","last_clean":"2024-09-14T04:40:52.965062+0000","last_became_active":"2024-09-14T04:38:26.895506+0000","last_became_peered":"2024-09-14T04:38:26.895506+0000","last_unstale":"2024-09-14T04:40:52.965062+0000","last_undegraded":"2024-09-14T04:40:52.965062+0000","last_fullsized":"2024-09-14T04:40:52.965062+0000","mapping_epoch":1315,"log_start":"1319'1000","ondisk_log_start":"1319'1000","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":10044,"log_dups_size":0,"ondisk_log_size":10044,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:07:45.042093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359661568,"num_objects":5488,"num_object_clones":0,"num_object_copies":16464,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5488,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11044,"num_write_kb":529024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"53.b","version":"1337'10668","reported_seq":10696,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:52.989478+0000","last_change":"2024-09-14T04:38:26.893051+0000","last_active":"2024-09-14T04:40:52.989478+0000","last_peered":"2024-09-14T04:40:52.989478+0000","last_clean":"2024-09-14T04:40:52.989478+0000","last_became_active":"2024-09-14T04:38:26.892762+0000","last_became_peered":"2024-09-14T04:38:26.892762+0000","last_unstale":"2024-09-14T04:40:52.989478+0000","last_undegraded":"2024-09-14T04:40:52.989478+0000","last_fullsized":"2024-09-14T04:40:52.989478+0000","mapping_epoch":1315,"log_start":"1319'600","ondisk_log_start":"1319'600","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":10068,"log_dups_size":0,"ondisk_log_size":10068,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:23:11.179960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354025472,"num_objects":5402,"num_object_clones":0,"num_object_copies":16206,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5402,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10668,"num_write_kb":514240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"53.a","version":"1337'11090","reported_seq":11147,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:56.568992+0000","last_change":"2024-09-14T04:38:26.895294+0000","last_active":"2024-09-14T04:40:56.568992+0000","last_peered":"2024-09-14T04:40:56.568992+0000","last_clean":"2024-09-14T04:40:56.568992+0000","last_became_active":"2024-09-14T04:38:26.894942+0000","last_became_peered":"2024-09-14T04:38:26.894942+0000","last_unstale":"2024-09-14T04:40:56.568992+0000","last_undegraded":"2024-09-14T04:40:56.568992+0000","last_fullsized":"2024-09-14T04:40:56.568992+0000","mapping_epoch":1315,"log_start":"1319'2700","ondisk_log_start":"1319'2700","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":8390,"log_dups_size":0,"ondisk_log_size":8390,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:17:54.687919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352452608,"num_objects":5378,"num_object_clones":0,"num_object_copies":16134,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5378,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11090,"num_write_kb":526976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"53.9","version":"1337'10769","reported_seq":10826,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:52.540056+0000","last_change":"2024-09-14T04:38:26.899027+0000","last_active":"2024-09-14T04:40:52.540056+0000","last_peered":"2024-09-14T04:40:52.540056+0000","last_clean":"2024-09-14T04:40:52.540056+0000","last_became_active":"2024-09-14T04:38:26.898732+0000","last_became_peered":"2024-09-14T04:38:26.898732+0000","last_unstale":"2024-09-14T04:40:52.540056+0000","last_undegraded":"2024-09-14T04:40:52.540056+0000","last_fullsized":"2024-09-14T04:40:52.540056+0000","mapping_epoch":1315,"log_start":"1319'1300","ondisk_log_start":"1319'1300","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":9469,"log_dups_size":0,"ondisk_log_size":9469,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:58:20.477950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364576768,"num_objects":5563,"num_object_clones":0,"num_object_copies":16689,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5563,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10769,"num_write_kb":522624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"53.8","version":"1337'10763","reported_seq":10795,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:53.005746+0000","last_change":"2024-09-14T04:38:26.897778+0000","last_active":"2024-09-14T04:40:53.005746+0000","last_peered":"2024-09-14T04:40:53.005746+0000","last_clean":"2024-09-14T04:40:53.005746+0000","last_became_active":"2024-09-14T04:38:26.897509+0000","last_became_peered":"2024-09-14T04:38:26.897509+0000","last_unstale":"2024-09-14T04:40:53.005746+0000","last_undegraded":"2024-09-14T04:40:53.005746+0000","last_fullsized":"2024-09-14T04:40:53.005746+0000","mapping_epoch":1315,"log_start":"1320'3900","ondisk_log_start":"1320'3900","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":6863,"log_dups_size":0,"ondisk_log_size":6863,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:51:52.846967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351600640,"num_objects":5365,"num_object_clones":0,"num_object_copies":16095,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5365,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10763,"num_write_kb":516096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"53.7","version":"1337'11030","reported_seq":11087,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:54.543479+0000","last_change":"2024-09-14T04:38:26.897393+0000","last_active":"2024-09-14T04:40:54.543479+0000","last_peered":"2024-09-14T04:40:54.543479+0000","last_clean":"2024-09-14T04:40:54.543479+0000","last_became_active":"2024-09-14T04:38:26.897127+0000","last_became_peered":"2024-09-14T04:38:26.897127+0000","last_unstale":"2024-09-14T04:40:54.543479+0000","last_undegraded":"2024-09-14T04:40:54.543479+0000","last_fullsized":"2024-09-14T04:40:54.543479+0000","mapping_epoch":1315,"log_start":"1320'3800","ondisk_log_start":"1320'3800","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":7230,"log_dups_size":0,"ondisk_log_size":7230,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:15:57.417163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354156544,"num_objects":5404,"num_object_clones":0,"num_object_copies":16212,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5404,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11030,"num_write_kb":525888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"53.6","version":"1337'11171","reported_seq":11228,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:56.595800+0000","last_change":"2024-09-14T04:38:27.529579+0000","last_active":"2024-09-14T04:40:56.595800+0000","last_peered":"2024-09-14T04:40:56.595800+0000","last_clean":"2024-09-14T04:40:56.595800+0000","last_became_active":"2024-09-14T04:38:27.529245+0000","last_became_peered":"2024-09-14T04:38:27.529245+0000","last_unstale":"2024-09-14T04:40:56.595800+0000","last_undegraded":"2024-09-14T04:40:56.595800+0000","last_fullsized":"2024-09-14T04:40:56.595800+0000","mapping_epoch":1315,"log_start":"1319'2800","ondisk_log_start":"1319'2800","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":8371,"log_dups_size":0,"ondisk_log_size":8371,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:11.351641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348848128,"num_objects":5323,"num_object_clones":0,"num_object_copies":15969,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5323,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11171,"num_write_kb":527808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"53.5","version":"1337'10759","reported_seq":10816,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:52.541333+0000","last_change":"2024-09-14T04:38:26.897570+0000","last_active":"2024-09-14T04:40:52.541333+0000","last_peered":"2024-09-14T04:40:52.541333+0000","last_clean":"2024-09-14T04:40:52.541333+0000","last_became_active":"2024-09-14T04:38:26.897277+0000","last_became_peered":"2024-09-14T04:38:26.897277+0000","last_unstale":"2024-09-14T04:40:52.541333+0000","last_undegraded":"2024-09-14T04:40:52.541333+0000","last_fullsized":"2024-09-14T04:40:52.541333+0000","mapping_epoch":1315,"log_start":"1319'1300","ondisk_log_start":"1319'1300","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":9459,"log_dups_size":0,"ondisk_log_size":9459,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:06:40.318691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355139584,"num_objects":5419,"num_object_clones":0,"num_object_copies":16257,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5419,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10759,"num_write_kb":517696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"53.4","version":"1337'10760","reported_seq":10817,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:54.532680+0000","last_change":"2024-09-14T04:38:27.529509+0000","last_active":"2024-09-14T04:40:54.532680+0000","last_peered":"2024-09-14T04:40:54.532680+0000","last_clean":"2024-09-14T04:40:54.532680+0000","last_became_active":"2024-09-14T04:38:27.529180+0000","last_became_peered":"2024-09-14T04:38:27.529180+0000","last_unstale":"2024-09-14T04:40:54.532680+0000","last_undegraded":"2024-09-14T04:40:54.532680+0000","last_fullsized":"2024-09-14T04:40:54.532680+0000","mapping_epoch":1315,"log_start":"1320'3600","ondisk_log_start":"1320'3600","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":7160,"log_dups_size":0,"ondisk_log_size":7160,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:04:21.631485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":347209728,"num_objects":5298,"num_object_clones":0,"num_object_copies":15894,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5298,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10760,"num_write_kb":513856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"53.3","version":"1337'10775","reported_seq":10807,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:52.844223+0000","last_change":"2024-09-14T04:38:26.894968+0000","last_active":"2024-09-14T04:40:52.844223+0000","last_peered":"2024-09-14T04:40:52.844223+0000","last_clean":"2024-09-14T04:40:52.844223+0000","last_became_active":"2024-09-14T04:38:26.894621+0000","last_became_peered":"2024-09-14T04:38:26.894621+0000","last_unstale":"2024-09-14T04:40:52.844223+0000","last_undegraded":"2024-09-14T04:40:52.844223+0000","last_fullsized":"2024-09-14T04:40:52.844223+0000","mapping_epoch":1315,"log_start":"1319'1000","ondisk_log_start":"1319'1000","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":9775,"log_dups_size":0,"ondisk_log_size":9775,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:15:43.660519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358678528,"num_objects":5473,"num_object_clones":0,"num_object_copies":16419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10775,"num_write_kb":519936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"53.2","version":"1337'10737","reported_seq":10769,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:52.951771+0000","last_change":"2024-09-14T04:38:26.897057+0000","last_active":"2024-09-14T04:40:52.951771+0000","last_peered":"2024-09-14T04:40:52.951771+0000","last_clean":"2024-09-14T04:40:52.951771+0000","last_became_active":"2024-09-14T04:38:26.896677+0000","last_became_peered":"2024-09-14T04:38:26.896677+0000","last_unstale":"2024-09-14T04:40:52.951771+0000","last_undegraded":"2024-09-14T04:40:52.951771+0000","last_fullsized":"2024-09-14T04:40:52.951771+0000","mapping_epoch":1315,"log_start":"1319'700","ondisk_log_start":"1319'700","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":10037,"log_dups_size":0,"ondisk_log_size":10037,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:15.244766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363266048,"num_objects":5543,"num_object_clones":0,"num_object_copies":16629,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5543,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10737,"num_write_kb":520960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"53.1","version":"1337'11011","reported_seq":11068,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:54.588005+0000","last_change":"2024-09-14T04:38:26.897425+0000","last_active":"2024-09-14T04:40:54.588005+0000","last_peered":"2024-09-14T04:40:54.588005+0000","last_clean":"2024-09-14T04:40:54.588005+0000","last_became_active":"2024-09-14T04:38:26.897127+0000","last_became_peered":"2024-09-14T04:38:26.897127+0000","last_unstale":"2024-09-14T04:40:54.588005+0000","last_undegraded":"2024-09-14T04:40:54.588005+0000","last_fullsized":"2024-09-14T04:40:54.588005+0000","mapping_epoch":1315,"log_start":"1320'3800","ondisk_log_start":"1320'3800","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":7211,"log_dups_size":0,"ondisk_log_size":7211,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:49:46.941647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355270656,"num_objects":5421,"num_object_clones":0,"num_object_copies":16263,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5421,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11011,"num_write_kb":525824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"53.0","version":"1337'10725","reported_seq":10757,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:52.969245+0000","last_change":"2024-09-14T04:38:27.529766+0000","last_active":"2024-09-14T04:40:52.969245+0000","last_peered":"2024-09-14T04:40:52.969245+0000","last_clean":"2024-09-14T04:40:52.969245+0000","last_became_active":"2024-09-14T04:38:27.529414+0000","last_became_peered":"2024-09-14T04:38:27.529414+0000","last_unstale":"2024-09-14T04:40:52.969245+0000","last_undegraded":"2024-09-14T04:40:52.969245+0000","last_fullsized":"2024-09-14T04:40:52.969245+0000","mapping_epoch":1315,"log_start":"1320'3900","ondisk_log_start":"1320'3900","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":6825,"log_dups_size":0,"ondisk_log_size":6825,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:30:08.912265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360251392,"num_objects":5497,"num_object_clones":0,"num_object_copies":16491,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5497,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10725,"num_write_kb":519104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":1959,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.785572+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T04:40:51.785572+0000","last_peered":"2024-09-14T04:40:51.785572+0000","last_clean":"2024-09-14T04:40:51.785572+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T04:40:51.785572+0000","last_undegraded":"2024-09-14T04:40:51.785572+0000","last_fullsized":"2024-09-14T04:40:51.785572+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1299,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.743043+0000","last_change":"2024-09-14T03:53:45.836449+0000","last_active":"2024-09-14T04:40:50.743043+0000","last_peered":"2024-09-14T04:40:50.743043+0000","last_clean":"2024-09-14T04:40:50.743043+0000","last_became_active":"2024-09-14T03:53:45.836108+0000","last_became_peered":"2024-09-14T03:53:45.836108+0000","last_unstale":"2024-09-14T04:40:50.743043+0000","last_undegraded":"2024-09-14T04:40:50.743043+0000","last_fullsized":"2024-09-14T04:40:50.743043+0000","mapping_epoch":73,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":74,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:52:13.095225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1949,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.792964+0000","last_change":"2024-09-14T04:07:58.504829+0000","last_active":"2024-09-14T04:40:51.792964+0000","last_peered":"2024-09-14T04:40:51.792964+0000","last_clean":"2024-09-14T04:40:51.792964+0000","last_became_active":"2024-09-14T04:07:58.502623+0000","last_became_peered":"2024-09-14T04:07:58.502623+0000","last_unstale":"2024-09-14T04:40:51.792964+0000","last_undegraded":"2024-09-14T04:40:51.792964+0000","last_fullsized":"2024-09-14T04:40:51.792964+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:59.091993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1960,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.787156+0000","last_change":"2024-09-14T04:07:58.507125+0000","last_active":"2024-09-14T04:40:51.787156+0000","last_peered":"2024-09-14T04:40:51.787156+0000","last_clean":"2024-09-14T04:40:51.787156+0000","last_became_active":"2024-09-14T04:07:58.505701+0000","last_became_peered":"2024-09-14T04:07:58.505701+0000","last_unstale":"2024-09-14T04:40:51.787156+0000","last_undegraded":"2024-09-14T04:40:51.787156+0000","last_fullsized":"2024-09-14T04:40:51.787156+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:25:00.847961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1290,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.739481+0000","last_change":"2024-09-14T03:53:05.234444+0000","last_active":"2024-09-14T04:40:50.739481+0000","last_peered":"2024-09-14T04:40:50.739481+0000","last_clean":"2024-09-14T04:40:50.739481+0000","last_became_active":"2024-09-14T03:53:05.234115+0000","last_became_peered":"2024-09-14T03:53:05.234115+0000","last_unstale":"2024-09-14T04:40:50.739481+0000","last_undegraded":"2024-09-14T04:40:50.739481+0000","last_fullsized":"2024-09-14T04:40:50.739481+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:01:23.153937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1320,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.745865+0000","last_change":"2024-09-14T04:07:58.547135+0000","last_active":"2024-09-14T04:40:50.745865+0000","last_peered":"2024-09-14T04:40:50.745865+0000","last_clean":"2024-09-14T04:40:50.745865+0000","last_became_active":"2024-09-14T04:07:58.546751+0000","last_became_peered":"2024-09-14T04:07:58.546751+0000","last_unstale":"2024-09-14T04:40:50.745865+0000","last_undegraded":"2024-09-14T04:40:50.745865+0000","last_fullsized":"2024-09-14T04:40:50.745865+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:01.092698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1305,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.738491+0000","last_change":"2024-09-14T04:06:46.875795+0000","last_active":"2024-09-14T04:40:50.738491+0000","last_peered":"2024-09-14T04:40:50.738491+0000","last_clean":"2024-09-14T04:40:50.738491+0000","last_became_active":"2024-09-14T04:06:46.875395+0000","last_became_peered":"2024-09-14T04:06:46.875395+0000","last_unstale":"2024-09-14T04:40:50.738491+0000","last_undegraded":"2024-09-14T04:40:50.738491+0000","last_fullsized":"2024-09-14T04:40:50.738491+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:51:10.713063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1320,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.742478+0000","last_change":"2024-09-14T04:07:58.514349+0000","last_active":"2024-09-14T04:40:50.742478+0000","last_peered":"2024-09-14T04:40:50.742478+0000","last_clean":"2024-09-14T04:40:50.742478+0000","last_became_active":"2024-09-14T04:07:58.513485+0000","last_became_peered":"2024-09-14T04:07:58.513485+0000","last_unstale":"2024-09-14T04:40:50.742478+0000","last_undegraded":"2024-09-14T04:40:50.742478+0000","last_fullsized":"2024-09-14T04:40:50.742478+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:33:09.031813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":1941,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.784758+0000","last_change":"2024-09-14T04:05:54.428057+0000","last_active":"2024-09-14T04:40:51.784758+0000","last_peered":"2024-09-14T04:40:51.784758+0000","last_clean":"2024-09-14T04:40:51.784758+0000","last_became_active":"2024-09-14T04:05:54.414895+0000","last_became_peered":"2024-09-14T04:05:54.414895+0000","last_unstale":"2024-09-14T04:40:51.784758+0000","last_undegraded":"2024-09-14T04:40:51.784758+0000","last_fullsized":"2024-09-14T04:40:51.784758+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:45:15.752675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1956,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.793151+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T04:40:51.793151+0000","last_peered":"2024-09-14T04:40:51.793151+0000","last_clean":"2024-09-14T04:40:51.793151+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T04:40:51.793151+0000","last_undegraded":"2024-09-14T04:40:51.793151+0000","last_fullsized":"2024-09-14T04:40:51.793151+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1322,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.745927+0000","last_change":"2024-09-14T04:07:21.717071+0000","last_active":"2024-09-14T04:40:50.745927+0000","last_peered":"2024-09-14T04:40:50.745927+0000","last_clean":"2024-09-14T04:40:50.745927+0000","last_became_active":"2024-09-14T04:07:21.716215+0000","last_became_peered":"2024-09-14T04:07:21.716215+0000","last_unstale":"2024-09-14T04:40:50.745927+0000","last_undegraded":"2024-09-14T04:40:50.745927+0000","last_fullsized":"2024-09-14T04:40:50.745927+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:48:47.175490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1929,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.790805+0000","last_change":"2024-09-14T04:07:58.524371+0000","last_active":"2024-09-14T04:40:51.790805+0000","last_peered":"2024-09-14T04:40:51.790805+0000","last_clean":"2024-09-14T04:40:51.790805+0000","last_became_active":"2024-09-14T04:07:58.524088+0000","last_became_peered":"2024-09-14T04:07:58.524088+0000","last_unstale":"2024-09-14T04:40:51.790805+0000","last_undegraded":"2024-09-14T04:40:51.790805+0000","last_fullsized":"2024-09-14T04:40:51.790805+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:42.615569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1924,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.790171+0000","last_change":"2024-09-14T04:07:58.507553+0000","last_active":"2024-09-14T04:40:51.790171+0000","last_peered":"2024-09-14T04:40:51.790171+0000","last_clean":"2024-09-14T04:40:51.790171+0000","last_became_active":"2024-09-14T04:07:58.507139+0000","last_became_peered":"2024-09-14T04:07:58.507139+0000","last_unstale":"2024-09-14T04:40:51.790171+0000","last_undegraded":"2024-09-14T04:40:51.790171+0000","last_fullsized":"2024-09-14T04:40:51.790171+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:11:57.821769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1954,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.785320+0000","last_change":"2024-09-14T04:07:58.561490+0000","last_active":"2024-09-14T04:40:51.785320+0000","last_peered":"2024-09-14T04:40:51.785320+0000","last_clean":"2024-09-14T04:40:51.785320+0000","last_became_active":"2024-09-14T04:07:58.559841+0000","last_became_peered":"2024-09-14T04:07:58.559841+0000","last_unstale":"2024-09-14T04:40:51.785320+0000","last_undegraded":"2024-09-14T04:40:51.785320+0000","last_fullsized":"2024-09-14T04:40:51.785320+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:27:03.795368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1945,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.786811+0000","last_change":"2024-09-14T04:07:58.555731+0000","last_active":"2024-09-14T04:40:51.786811+0000","last_peered":"2024-09-14T04:40:51.786811+0000","last_clean":"2024-09-14T04:40:51.786811+0000","last_became_active":"2024-09-14T04:07:58.549576+0000","last_became_peered":"2024-09-14T04:07:58.549576+0000","last_unstale":"2024-09-14T04:40:51.786811+0000","last_undegraded":"2024-09-14T04:40:51.786811+0000","last_fullsized":"2024-09-14T04:40:51.786811+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:28:42.339589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1313,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.757433+0000","last_change":"2024-09-14T04:07:21.720797+0000","last_active":"2024-09-14T04:40:50.757433+0000","last_peered":"2024-09-14T04:40:50.757433+0000","last_clean":"2024-09-14T04:40:50.757433+0000","last_became_active":"2024-09-14T04:07:21.719331+0000","last_became_peered":"2024-09-14T04:07:21.719331+0000","last_unstale":"2024-09-14T04:40:50.757433+0000","last_undegraded":"2024-09-14T04:40:50.757433+0000","last_fullsized":"2024-09-14T04:40:50.757433+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:35.404023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1955,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.786995+0000","last_change":"2024-09-14T04:07:58.502533+0000","last_active":"2024-09-14T04:40:51.786995+0000","last_peered":"2024-09-14T04:40:51.786995+0000","last_clean":"2024-09-14T04:40:51.786995+0000","last_became_active":"2024-09-14T04:07:58.499849+0000","last_became_peered":"2024-09-14T04:07:58.499849+0000","last_unstale":"2024-09-14T04:40:51.786995+0000","last_undegraded":"2024-09-14T04:40:51.786995+0000","last_fullsized":"2024-09-14T04:40:51.786995+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:15:35.609746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1949,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.785728+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T04:40:51.785728+0000","last_peered":"2024-09-14T04:40:51.785728+0000","last_clean":"2024-09-14T04:40:51.785728+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T04:40:51.785728+0000","last_undegraded":"2024-09-14T04:40:51.785728+0000","last_fullsized":"2024-09-14T04:40:51.785728+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1931,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.787351+0000","last_change":"2024-09-14T04:07:58.507682+0000","last_active":"2024-09-14T04:40:51.787351+0000","last_peered":"2024-09-14T04:40:51.787351+0000","last_clean":"2024-09-14T04:40:51.787351+0000","last_became_active":"2024-09-14T04:07:58.507365+0000","last_became_peered":"2024-09-14T04:07:58.507365+0000","last_unstale":"2024-09-14T04:40:51.787351+0000","last_undegraded":"2024-09-14T04:40:51.787351+0000","last_fullsized":"2024-09-14T04:40:51.787351+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:14:47.294950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1302,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.738019+0000","last_change":"2024-09-14T04:06:46.875668+0000","last_active":"2024-09-14T04:40:50.738019+0000","last_peered":"2024-09-14T04:40:50.738019+0000","last_clean":"2024-09-14T04:40:50.738019+0000","last_became_active":"2024-09-14T04:06:46.875149+0000","last_became_peered":"2024-09-14T04:06:46.875149+0000","last_unstale":"2024-09-14T04:40:50.738019+0000","last_undegraded":"2024-09-14T04:40:50.738019+0000","last_fullsized":"2024-09-14T04:40:50.738019+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:45:59.190047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1303,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.741162+0000","last_change":"2024-09-14T04:07:58.510699+0000","last_active":"2024-09-14T04:40:50.741162+0000","last_peered":"2024-09-14T04:40:50.741162+0000","last_clean":"2024-09-14T04:40:50.741162+0000","last_became_active":"2024-09-14T04:07:58.506240+0000","last_became_peered":"2024-09-14T04:07:58.506240+0000","last_unstale":"2024-09-14T04:40:50.741162+0000","last_undegraded":"2024-09-14T04:40:50.741162+0000","last_fullsized":"2024-09-14T04:40:50.741162+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:27:18.051658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1956,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.783961+0000","last_change":"2024-09-14T04:07:21.687985+0000","last_active":"2024-09-14T04:40:51.783961+0000","last_peered":"2024-09-14T04:40:51.783961+0000","last_clean":"2024-09-14T04:40:51.783961+0000","last_became_active":"2024-09-14T04:07:21.687670+0000","last_became_peered":"2024-09-14T04:07:21.687670+0000","last_unstale":"2024-09-14T04:40:51.783961+0000","last_undegraded":"2024-09-14T04:40:51.783961+0000","last_fullsized":"2024-09-14T04:40:51.783961+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:53:58.295662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":1315,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.742570+0000","last_change":"2024-09-14T03:53:05.244055+0000","last_active":"2024-09-14T04:40:50.742570+0000","last_peered":"2024-09-14T04:40:50.742570+0000","last_clean":"2024-09-14T04:40:50.742570+0000","last_became_active":"2024-09-14T03:53:05.243752+0000","last_became_peered":"2024-09-14T03:53:05.243752+0000","last_unstale":"2024-09-14T04:40:50.742570+0000","last_undegraded":"2024-09-14T04:40:50.742570+0000","last_fullsized":"2024-09-14T04:40:50.742570+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:12:31.850064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1306,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.742635+0000","last_change":"2024-09-14T04:06:46.895670+0000","last_active":"2024-09-14T04:40:50.742635+0000","last_peered":"2024-09-14T04:40:50.742635+0000","last_clean":"2024-09-14T04:40:50.742635+0000","last_became_active":"2024-09-14T04:06:46.895370+0000","last_became_peered":"2024-09-14T04:06:46.895370+0000","last_unstale":"2024-09-14T04:40:50.742635+0000","last_undegraded":"2024-09-14T04:40:50.742635+0000","last_fullsized":"2024-09-14T04:40:50.742635+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:48:48.040529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1293,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.742263+0000","last_change":"2024-09-14T03:53:06.634043+0000","last_active":"2024-09-14T04:40:50.742263+0000","last_peered":"2024-09-14T04:40:50.742263+0000","last_clean":"2024-09-14T04:40:50.742263+0000","last_became_active":"2024-09-14T03:53:06.633638+0000","last_became_peered":"2024-09-14T03:53:06.633638+0000","last_unstale":"2024-09-14T04:40:50.742263+0000","last_undegraded":"2024-09-14T04:40:50.742263+0000","last_fullsized":"2024-09-14T04:40:50.742263+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:15:53.434064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1317,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.757510+0000","last_change":"2024-09-14T04:07:21.721242+0000","last_active":"2024-09-14T04:40:50.757510+0000","last_peered":"2024-09-14T04:40:50.757510+0000","last_clean":"2024-09-14T04:40:50.757510+0000","last_became_active":"2024-09-14T04:07:21.720084+0000","last_became_peered":"2024-09-14T04:07:21.720084+0000","last_unstale":"2024-09-14T04:40:50.757510+0000","last_undegraded":"2024-09-14T04:40:50.757510+0000","last_fullsized":"2024-09-14T04:40:50.757510+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:58:41.308643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":1956,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.785228+0000","last_change":"2024-09-14T04:06:46.932567+0000","last_active":"2024-09-14T04:40:51.785228+0000","last_peered":"2024-09-14T04:40:51.785228+0000","last_clean":"2024-09-14T04:40:51.785228+0000","last_became_active":"2024-09-14T04:06:46.930732+0000","last_became_peered":"2024-09-14T04:06:46.930732+0000","last_unstale":"2024-09-14T04:40:51.785228+0000","last_undegraded":"2024-09-14T04:40:51.785228+0000","last_fullsized":"2024-09-14T04:40:51.785228+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:10:02.729499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1923,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.786906+0000","last_change":"2024-09-14T04:07:58.555446+0000","last_active":"2024-09-14T04:40:51.786906+0000","last_peered":"2024-09-14T04:40:51.786906+0000","last_clean":"2024-09-14T04:40:51.786906+0000","last_became_active":"2024-09-14T04:07:58.548355+0000","last_became_peered":"2024-09-14T04:07:58.548355+0000","last_unstale":"2024-09-14T04:40:51.786906+0000","last_undegraded":"2024-09-14T04:40:51.786906+0000","last_fullsized":"2024-09-14T04:40:51.786906+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:56:28.426094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1960,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.785264+0000","last_change":"2024-09-14T04:06:46.915273+0000","last_active":"2024-09-14T04:40:51.785264+0000","last_peered":"2024-09-14T04:40:51.785264+0000","last_clean":"2024-09-14T04:40:51.785264+0000","last_became_active":"2024-09-14T04:06:46.914994+0000","last_became_peered":"2024-09-14T04:06:46.914994+0000","last_unstale":"2024-09-14T04:40:51.785264+0000","last_undegraded":"2024-09-14T04:40:51.785264+0000","last_fullsized":"2024-09-14T04:40:51.785264+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:48:58.802609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1924,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.791907+0000","last_change":"2024-09-14T04:07:58.564377+0000","last_active":"2024-09-14T04:40:51.791907+0000","last_peered":"2024-09-14T04:40:51.791907+0000","last_clean":"2024-09-14T04:40:51.791907+0000","last_became_active":"2024-09-14T04:07:58.546913+0000","last_became_peered":"2024-09-14T04:07:58.546913+0000","last_unstale":"2024-09-14T04:40:51.791907+0000","last_undegraded":"2024-09-14T04:40:51.791907+0000","last_fullsized":"2024-09-14T04:40:51.791907+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:05:53.637115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1942,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.784907+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T04:40:51.784907+0000","last_peered":"2024-09-14T04:40:51.784907+0000","last_clean":"2024-09-14T04:40:51.784907+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T04:40:51.784907+0000","last_undegraded":"2024-09-14T04:40:51.784907+0000","last_fullsized":"2024-09-14T04:40:51.784907+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1316,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.741119+0000","last_change":"2024-09-14T04:05:54.381035+0000","last_active":"2024-09-14T04:40:50.741119+0000","last_peered":"2024-09-14T04:40:50.741119+0000","last_clean":"2024-09-14T04:40:50.741119+0000","last_became_active":"2024-09-14T04:05:54.379861+0000","last_became_peered":"2024-09-14T04:05:54.379861+0000","last_unstale":"2024-09-14T04:40:50.741119+0000","last_undegraded":"2024-09-14T04:40:50.741119+0000","last_fullsized":"2024-09-14T04:40:50.741119+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:07:46.302925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1960,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.783983+0000","last_change":"2024-09-14T04:07:21.686880+0000","last_active":"2024-09-14T04:40:51.783983+0000","last_peered":"2024-09-14T04:40:51.783983+0000","last_clean":"2024-09-14T04:40:51.783983+0000","last_became_active":"2024-09-14T04:07:21.685558+0000","last_became_peered":"2024-09-14T04:07:21.685558+0000","last_unstale":"2024-09-14T04:40:51.783983+0000","last_undegraded":"2024-09-14T04:40:51.783983+0000","last_fullsized":"2024-09-14T04:40:51.783983+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:24:18.693428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1948,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.783804+0000","last_change":"2024-09-14T04:06:46.905931+0000","last_active":"2024-09-14T04:40:51.783804+0000","last_peered":"2024-09-14T04:40:51.783804+0000","last_clean":"2024-09-14T04:40:51.783804+0000","last_became_active":"2024-09-14T04:06:46.905666+0000","last_became_peered":"2024-09-14T04:06:46.905666+0000","last_unstale":"2024-09-14T04:40:51.783804+0000","last_undegraded":"2024-09-14T04:40:51.783804+0000","last_fullsized":"2024-09-14T04:40:51.783804+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:40:57.007948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1977,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.784181+0000","last_change":"2024-09-14T04:06:46.930482+0000","last_active":"2024-09-14T04:40:51.784181+0000","last_peered":"2024-09-14T04:40:51.784181+0000","last_clean":"2024-09-14T04:40:51.784181+0000","last_became_active":"2024-09-14T04:06:46.927372+0000","last_became_peered":"2024-09-14T04:06:46.927372+0000","last_unstale":"2024-09-14T04:40:51.784181+0000","last_undegraded":"2024-09-14T04:40:51.784181+0000","last_fullsized":"2024-09-14T04:40:51.784181+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:07:22.159128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1321,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.739529+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T04:40:50.739529+0000","last_peered":"2024-09-14T04:40:50.739529+0000","last_clean":"2024-09-14T04:40:50.739529+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T04:40:50.739529+0000","last_undegraded":"2024-09-14T04:40:50.739529+0000","last_fullsized":"2024-09-14T04:40:50.739529+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1970,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.784091+0000","last_change":"2024-09-14T04:07:58.510721+0000","last_active":"2024-09-14T04:40:51.784091+0000","last_peered":"2024-09-14T04:40:51.784091+0000","last_clean":"2024-09-14T04:40:51.784091+0000","last_became_active":"2024-09-14T04:07:58.510332+0000","last_became_peered":"2024-09-14T04:07:58.510332+0000","last_unstale":"2024-09-14T04:40:51.784091+0000","last_undegraded":"2024-09-14T04:40:51.784091+0000","last_fullsized":"2024-09-14T04:40:51.784091+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:07:03.926842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1309,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.742982+0000","last_change":"2024-09-14T04:05:54.426939+0000","last_active":"2024-09-14T04:40:50.742982+0000","last_peered":"2024-09-14T04:40:50.742982+0000","last_clean":"2024-09-14T04:40:50.742982+0000","last_became_active":"2024-09-14T04:05:54.426427+0000","last_became_peered":"2024-09-14T04:05:54.426427+0000","last_unstale":"2024-09-14T04:40:50.742982+0000","last_undegraded":"2024-09-14T04:40:50.742982+0000","last_fullsized":"2024-09-14T04:40:50.742982+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:53:55.841489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1320,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.738005+0000","last_change":"2024-09-14T04:06:46.875340+0000","last_active":"2024-09-14T04:40:50.738005+0000","last_peered":"2024-09-14T04:40:50.738005+0000","last_clean":"2024-09-14T04:40:50.738005+0000","last_became_active":"2024-09-14T04:06:46.874552+0000","last_became_peered":"2024-09-14T04:06:46.874552+0000","last_unstale":"2024-09-14T04:40:50.738005+0000","last_undegraded":"2024-09-14T04:40:50.738005+0000","last_fullsized":"2024-09-14T04:40:50.738005+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:52.257357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1956,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.789801+0000","last_change":"2024-09-14T04:06:46.917681+0000","last_active":"2024-09-14T04:40:51.789801+0000","last_peered":"2024-09-14T04:40:51.789801+0000","last_clean":"2024-09-14T04:40:51.789801+0000","last_became_active":"2024-09-14T04:06:46.917401+0000","last_became_peered":"2024-09-14T04:06:46.917401+0000","last_unstale":"2024-09-14T04:40:51.789801+0000","last_undegraded":"2024-09-14T04:40:51.789801+0000","last_fullsized":"2024-09-14T04:40:51.789801+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:04:39.949490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1960,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.783706+0000","last_change":"2024-09-14T04:07:21.685415+0000","last_active":"2024-09-14T04:40:51.783706+0000","last_peered":"2024-09-14T04:40:51.783706+0000","last_clean":"2024-09-14T04:40:51.783706+0000","last_became_active":"2024-09-14T04:07:21.683504+0000","last_became_peered":"2024-09-14T04:07:21.683504+0000","last_unstale":"2024-09-14T04:40:51.783706+0000","last_undegraded":"2024-09-14T04:40:51.783706+0000","last_fullsized":"2024-09-14T04:40:51.783706+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:21:08.806291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1963,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.789840+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T04:40:51.789840+0000","last_peered":"2024-09-14T04:40:51.789840+0000","last_clean":"2024-09-14T04:40:51.789840+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T04:40:51.789840+0000","last_undegraded":"2024-09-14T04:40:51.789840+0000","last_fullsized":"2024-09-14T04:40:51.789840+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1964,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.785393+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T04:40:51.785393+0000","last_peered":"2024-09-14T04:40:51.785393+0000","last_clean":"2024-09-14T04:40:51.785393+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T04:40:51.785393+0000","last_undegraded":"2024-09-14T04:40:51.785393+0000","last_fullsized":"2024-09-14T04:40:51.785393+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1952,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.784777+0000","last_change":"2024-09-14T04:07:58.521344+0000","last_active":"2024-09-14T04:40:51.784777+0000","last_peered":"2024-09-14T04:40:51.784777+0000","last_clean":"2024-09-14T04:40:51.784777+0000","last_became_active":"2024-09-14T04:07:58.520965+0000","last_became_peered":"2024-09-14T04:07:58.520965+0000","last_unstale":"2024-09-14T04:40:51.784777+0000","last_undegraded":"2024-09-14T04:40:51.784777+0000","last_fullsized":"2024-09-14T04:40:51.784777+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:58:46.029013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1332,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.740778+0000","last_change":"2024-09-14T04:06:46.857436+0000","last_active":"2024-09-14T04:40:50.740778+0000","last_peered":"2024-09-14T04:40:50.740778+0000","last_clean":"2024-09-14T04:40:50.740778+0000","last_became_active":"2024-09-14T04:06:46.857154+0000","last_became_peered":"2024-09-14T04:06:46.857154+0000","last_unstale":"2024-09-14T04:40:50.740778+0000","last_undegraded":"2024-09-14T04:40:50.740778+0000","last_fullsized":"2024-09-14T04:40:50.740778+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:53:28.772152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1289,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.757402+0000","last_change":"2024-09-14T03:53:05.551268+0000","last_active":"2024-09-14T04:40:50.757402+0000","last_peered":"2024-09-14T04:40:50.757402+0000","last_clean":"2024-09-14T04:40:50.757402+0000","last_became_active":"2024-09-14T03:53:05.551015+0000","last_became_peered":"2024-09-14T03:53:05.551015+0000","last_unstale":"2024-09-14T04:40:50.757402+0000","last_undegraded":"2024-09-14T04:40:50.757402+0000","last_fullsized":"2024-09-14T04:40:50.757402+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.905475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1934,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.787220+0000","last_change":"2024-09-14T04:07:58.503896+0000","last_active":"2024-09-14T04:40:51.787220+0000","last_peered":"2024-09-14T04:40:51.787220+0000","last_clean":"2024-09-14T04:40:51.787220+0000","last_became_active":"2024-09-14T04:07:58.499253+0000","last_became_peered":"2024-09-14T04:07:58.499253+0000","last_unstale":"2024-09-14T04:40:51.787220+0000","last_undegraded":"2024-09-14T04:40:51.787220+0000","last_fullsized":"2024-09-14T04:40:51.787220+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:48:27.543818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1319,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.737971+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T04:40:50.737971+0000","last_peered":"2024-09-14T04:40:50.737971+0000","last_clean":"2024-09-14T04:40:50.737971+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T04:40:50.737971+0000","last_undegraded":"2024-09-14T04:40:50.737971+0000","last_fullsized":"2024-09-14T04:40:50.737971+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":1634,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.740655+0000","last_change":"2024-09-14T04:05:54.389043+0000","last_active":"2024-09-14T04:40:50.740655+0000","last_peered":"2024-09-14T04:40:50.740655+0000","last_clean":"2024-09-14T04:40:50.740655+0000","last_became_active":"2024-09-14T04:05:54.388655+0000","last_became_peered":"2024-09-14T04:05:54.388655+0000","last_unstale":"2024-09-14T04:40:50.740655+0000","last_undegraded":"2024-09-14T04:40:50.740655+0000","last_fullsized":"2024-09-14T04:40:50.740655+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T03:49:18.237706+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:38:59.615597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1925,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.789729+0000","last_change":"2024-09-14T04:07:58.530406+0000","last_active":"2024-09-14T04:40:51.789729+0000","last_peered":"2024-09-14T04:40:51.789729+0000","last_clean":"2024-09-14T04:40:51.789729+0000","last_became_active":"2024-09-14T04:07:58.530078+0000","last_became_peered":"2024-09-14T04:07:58.530078+0000","last_unstale":"2024-09-14T04:40:51.789729+0000","last_undegraded":"2024-09-14T04:40:51.789729+0000","last_fullsized":"2024-09-14T04:40:51.789729+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:54:53.524001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1933,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.786699+0000","last_change":"2024-09-14T04:07:58.555717+0000","last_active":"2024-09-14T04:40:51.786699+0000","last_peered":"2024-09-14T04:40:51.786699+0000","last_clean":"2024-09-14T04:40:51.786699+0000","last_became_active":"2024-09-14T04:07:58.548894+0000","last_became_peered":"2024-09-14T04:07:58.548894+0000","last_unstale":"2024-09-14T04:40:51.786699+0000","last_undegraded":"2024-09-14T04:40:51.786699+0000","last_fullsized":"2024-09-14T04:40:51.786699+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:42.061551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1307,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.742686+0000","last_change":"2024-09-14T04:06:46.894741+0000","last_active":"2024-09-14T04:40:50.742686+0000","last_peered":"2024-09-14T04:40:50.742686+0000","last_clean":"2024-09-14T04:40:50.742686+0000","last_became_active":"2024-09-14T04:06:46.894445+0000","last_became_peered":"2024-09-14T04:06:46.894445+0000","last_unstale":"2024-09-14T04:40:50.742686+0000","last_undegraded":"2024-09-14T04:40:50.742686+0000","last_fullsized":"2024-09-14T04:40:50.742686+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:52:00.055271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1309,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.743030+0000","last_change":"2024-09-14T04:07:21.715462+0000","last_active":"2024-09-14T04:40:50.743030+0000","last_peered":"2024-09-14T04:40:50.743030+0000","last_clean":"2024-09-14T04:40:50.743030+0000","last_became_active":"2024-09-14T04:07:21.713032+0000","last_became_peered":"2024-09-14T04:07:21.713032+0000","last_unstale":"2024-09-14T04:40:50.743030+0000","last_undegraded":"2024-09-14T04:40:50.743030+0000","last_fullsized":"2024-09-14T04:40:50.743030+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:37:05.545164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1322,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.742577+0000","last_change":"2024-09-14T04:07:21.718037+0000","last_active":"2024-09-14T04:40:50.742577+0000","last_peered":"2024-09-14T04:40:50.742577+0000","last_clean":"2024-09-14T04:40:50.742577+0000","last_became_active":"2024-09-14T04:07:21.717700+0000","last_became_peered":"2024-09-14T04:07:21.717700+0000","last_unstale":"2024-09-14T04:40:50.742577+0000","last_undegraded":"2024-09-14T04:40:50.742577+0000","last_fullsized":"2024-09-14T04:40:50.742577+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:47:33.483785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1305,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.775974+0000","last_change":"2024-09-14T04:06:46.868780+0000","last_active":"2024-09-14T04:40:51.775974+0000","last_peered":"2024-09-14T04:40:51.775974+0000","last_clean":"2024-09-14T04:40:51.775974+0000","last_became_active":"2024-09-14T04:06:46.868483+0000","last_became_peered":"2024-09-14T04:06:46.868483+0000","last_unstale":"2024-09-14T04:40:51.775974+0000","last_undegraded":"2024-09-14T04:40:51.775974+0000","last_fullsized":"2024-09-14T04:40:51.775974+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:03.133158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1307,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.742483+0000","last_change":"2024-09-14T04:07:58.515859+0000","last_active":"2024-09-14T04:40:50.742483+0000","last_peered":"2024-09-14T04:40:50.742483+0000","last_clean":"2024-09-14T04:40:50.742483+0000","last_became_active":"2024-09-14T04:07:58.515594+0000","last_became_peered":"2024-09-14T04:07:58.515594+0000","last_unstale":"2024-09-14T04:40:50.742483+0000","last_undegraded":"2024-09-14T04:40:50.742483+0000","last_fullsized":"2024-09-14T04:40:50.742483+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:45.966798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":1949,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.787569+0000","last_change":"2024-09-14T04:07:58.509833+0000","last_active":"2024-09-14T04:40:51.787569+0000","last_peered":"2024-09-14T04:40:51.787569+0000","last_clean":"2024-09-14T04:40:51.787569+0000","last_became_active":"2024-09-14T04:07:58.509198+0000","last_became_peered":"2024-09-14T04:07:58.509198+0000","last_unstale":"2024-09-14T04:40:51.787569+0000","last_undegraded":"2024-09-14T04:40:51.787569+0000","last_fullsized":"2024-09-14T04:40:51.787569+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:24:40.193090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1952,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.784663+0000","last_change":"2024-09-14T04:06:46.925323+0000","last_active":"2024-09-14T04:40:51.784663+0000","last_peered":"2024-09-14T04:40:51.784663+0000","last_clean":"2024-09-14T04:40:51.784663+0000","last_became_active":"2024-09-14T04:06:46.925043+0000","last_became_peered":"2024-09-14T04:06:46.925043+0000","last_unstale":"2024-09-14T04:40:51.784663+0000","last_undegraded":"2024-09-14T04:40:51.784663+0000","last_fullsized":"2024-09-14T04:40:51.784663+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:39.895453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1305,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.738489+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T04:40:50.738489+0000","last_peered":"2024-09-14T04:40:50.738489+0000","last_clean":"2024-09-14T04:40:50.738489+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T04:40:50.738489+0000","last_undegraded":"2024-09-14T04:40:50.738489+0000","last_fullsized":"2024-09-14T04:40:50.738489+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1304,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.775367+0000","last_change":"2024-09-14T04:06:46.869868+0000","last_active":"2024-09-14T04:40:51.775367+0000","last_peered":"2024-09-14T04:40:51.775367+0000","last_clean":"2024-09-14T04:40:51.775367+0000","last_became_active":"2024-09-14T04:06:46.869126+0000","last_became_peered":"2024-09-14T04:06:46.869126+0000","last_unstale":"2024-09-14T04:40:51.775367+0000","last_undegraded":"2024-09-14T04:40:51.775367+0000","last_fullsized":"2024-09-14T04:40:51.775367+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:41.150138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1288,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.741046+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T04:40:50.741046+0000","last_peered":"2024-09-14T04:40:50.741046+0000","last_clean":"2024-09-14T04:40:50.741046+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T04:40:50.741046+0000","last_undegraded":"2024-09-14T04:40:50.741046+0000","last_fullsized":"2024-09-14T04:40:50.741046+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1323,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.742330+0000","last_change":"2024-09-14T04:07:58.508202+0000","last_active":"2024-09-14T04:40:50.742330+0000","last_peered":"2024-09-14T04:40:50.742330+0000","last_clean":"2024-09-14T04:40:50.742330+0000","last_became_active":"2024-09-14T04:07:58.507929+0000","last_became_peered":"2024-09-14T04:07:58.507929+0000","last_unstale":"2024-09-14T04:40:50.742330+0000","last_undegraded":"2024-09-14T04:40:50.742330+0000","last_fullsized":"2024-09-14T04:40:50.742330+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:17:58.160600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1304,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.741057+0000","last_change":"2024-09-14T04:06:46.858890+0000","last_active":"2024-09-14T04:40:50.741057+0000","last_peered":"2024-09-14T04:40:50.741057+0000","last_clean":"2024-09-14T04:40:50.741057+0000","last_became_active":"2024-09-14T04:06:46.858569+0000","last_became_peered":"2024-09-14T04:06:46.858569+0000","last_unstale":"2024-09-14T04:40:50.741057+0000","last_undegraded":"2024-09-14T04:40:50.741057+0000","last_fullsized":"2024-09-14T04:40:50.741057+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:25:46.400115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1290,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.737802+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T04:40:50.737802+0000","last_peered":"2024-09-14T04:40:50.737802+0000","last_clean":"2024-09-14T04:40:50.737802+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T04:40:50.737802+0000","last_undegraded":"2024-09-14T04:40:50.737802+0000","last_fullsized":"2024-09-14T04:40:50.737802+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1303,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.741041+0000","last_change":"2024-09-14T04:05:54.419435+0000","last_active":"2024-09-14T04:40:50.741041+0000","last_peered":"2024-09-14T04:40:50.741041+0000","last_clean":"2024-09-14T04:40:50.741041+0000","last_became_active":"2024-09-14T04:05:54.419094+0000","last_became_peered":"2024-09-14T04:05:54.419094+0000","last_unstale":"2024-09-14T04:40:50.741041+0000","last_undegraded":"2024-09-14T04:40:50.741041+0000","last_fullsized":"2024-09-14T04:40:50.741041+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:00.162629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1311,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.757028+0000","last_change":"2024-09-14T04:05:54.418813+0000","last_active":"2024-09-14T04:40:50.757028+0000","last_peered":"2024-09-14T04:40:50.757028+0000","last_clean":"2024-09-14T04:40:50.757028+0000","last_became_active":"2024-09-14T04:05:54.417584+0000","last_became_peered":"2024-09-14T04:05:54.417584+0000","last_unstale":"2024-09-14T04:40:50.757028+0000","last_undegraded":"2024-09-14T04:40:50.757028+0000","last_fullsized":"2024-09-14T04:40:50.757028+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:13:16.209456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1929,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.791819+0000","last_change":"2024-09-14T04:07:58.565442+0000","last_active":"2024-09-14T04:40:51.791819+0000","last_peered":"2024-09-14T04:40:51.791819+0000","last_clean":"2024-09-14T04:40:51.791819+0000","last_became_active":"2024-09-14T04:07:58.552289+0000","last_became_peered":"2024-09-14T04:07:58.552289+0000","last_unstale":"2024-09-14T04:40:51.791819+0000","last_undegraded":"2024-09-14T04:40:51.791819+0000","last_fullsized":"2024-09-14T04:40:51.791819+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:48:02.364960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1286,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.740773+0000","last_change":"2024-09-14T03:53:05.244003+0000","last_active":"2024-09-14T04:40:50.740773+0000","last_peered":"2024-09-14T04:40:50.740773+0000","last_clean":"2024-09-14T04:40:50.740773+0000","last_became_active":"2024-09-14T03:53:05.240743+0000","last_became_peered":"2024-09-14T03:53:05.240743+0000","last_unstale":"2024-09-14T04:40:50.740773+0000","last_undegraded":"2024-09-14T04:40:50.740773+0000","last_fullsized":"2024-09-14T04:40:50.740773+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:30.000189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1950,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.785139+0000","last_change":"2024-09-14T04:07:58.560880+0000","last_active":"2024-09-14T04:40:51.785139+0000","last_peered":"2024-09-14T04:40:51.785139+0000","last_clean":"2024-09-14T04:40:51.785139+0000","last_became_active":"2024-09-14T04:07:58.560132+0000","last_became_peered":"2024-09-14T04:07:58.560132+0000","last_unstale":"2024-09-14T04:40:51.785139+0000","last_undegraded":"2024-09-14T04:40:51.785139+0000","last_fullsized":"2024-09-14T04:40:51.785139+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:46:00.287050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1949,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.786752+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T04:40:51.786752+0000","last_peered":"2024-09-14T04:40:51.786752+0000","last_clean":"2024-09-14T04:40:51.786752+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T04:40:51.786752+0000","last_undegraded":"2024-09-14T04:40:51.786752+0000","last_fullsized":"2024-09-14T04:40:51.786752+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1302,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.739538+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T04:40:50.739538+0000","last_peered":"2024-09-14T04:40:50.739538+0000","last_clean":"2024-09-14T04:40:50.739538+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T04:40:50.739538+0000","last_undegraded":"2024-09-14T04:40:50.739538+0000","last_fullsized":"2024-09-14T04:40:50.739538+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1303,"reported_epoch":1336,"state":"active+clean","last_fresh":"2024-09-14T04:40:50.741041+0000","last_change":"2024-09-14T04:07:21.718847+0000","last_active":"2024-09-14T04:40:50.741041+0000","last_peered":"2024-09-14T04:40:50.741041+0000","last_clean":"2024-09-14T04:40:50.741041+0000","last_became_active":"2024-09-14T04:07:21.718504+0000","last_became_peered":"2024-09-14T04:07:21.718504+0000","last_unstale":"2024-09-14T04:40:50.741041+0000","last_undegraded":"2024-09-14T04:40:50.741041+0000","last_fullsized":"2024-09-14T04:40:50.741041+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:37:34.405765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1946,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.787777+0000","last_change":"2024-09-14T04:07:58.537946+0000","last_active":"2024-09-14T04:40:51.787777+0000","last_peered":"2024-09-14T04:40:51.787777+0000","last_clean":"2024-09-14T04:40:51.787777+0000","last_became_active":"2024-09-14T04:07:58.536034+0000","last_became_peered":"2024-09-14T04:07:58.536034+0000","last_unstale":"2024-09-14T04:40:51.787777+0000","last_undegraded":"2024-09-14T04:40:51.787777+0000","last_fullsized":"2024-09-14T04:40:51.787777+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:17:18.200470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":53,"num_pg":16,"stat_sum":{"num_bytes":5692850200,"num_objects":86867,"num_object_clones":0,"num_object_copies":260601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":86867,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":173315,"num_write_kb":8325761,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17079480320,"data_stored":17079468104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137615,"ondisk_log_size":137615,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201879,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":484488,"kb_used_data":363196,"kb_used_omap":1,"kb_used_meta":121278,"kb_avail":93260664,"statfs":{"total":95995035648,"available":95498919936,"internally_reserved":0,"allocated":371912704,"data_stored":367986271,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":124189131},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463515,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1590020,"kb_used_data":1406388,"kb_used_omap":1,"kb_used_meta":183614,"kb_avail":92155132,"statfs":{"total":95995035648,"available":94366855168,"internally_reserved":0,"allocated":1440141312,"data_stored":1436222944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":188021191},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725152,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2743740,"kb_used_data":2501868,"kb_used_omap":1,"kb_used_meta":241854,"kb_avail":91001412,"statfs":{"total":95995035648,"available":93185445888,"internally_reserved":0,"allocated":2561912832,"data_stored":2557997962,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":247658950},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986790,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4150844,"kb_used_data":3867124,"kb_used_omap":1,"kb_used_meta":283710,"kb_avail":89594308,"statfs":{"total":95995035648,"available":91744571392,"internally_reserved":0,"allocated":3959934976,"data_stored":3956016121,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":290519494},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772074894,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1848744,"kb_used_data":1718048,"kb_used_omap":71,"kb_used_meta":130552,"kb_avail":91896408,"statfs":{"total":95995035648,"available":94101921792,"internally_reserved":0,"allocated":1759281152,"data_stored":1755377977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":133686198},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707303829,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1523952,"kb_used_data":1397416,"kb_used_omap":68,"kb_used_meta":126395,"kb_avail":92221200,"statfs":{"total":95995035648,"available":94434508800,"internally_reserved":0,"allocated":1430953984,"data_stored":1427043167,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":129428688},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872728988,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3670060,"kb_used_data":3439524,"kb_used_omap":67,"kb_used_meta":230460,"kb_avail":90075092,"statfs":{"total":95995035648,"available":92236894208,"internally_reserved":0,"allocated":3522072576,"data_stored":3518165258,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":235991397},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056038,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2303332,"kb_used_data":2140888,"kb_used_omap":74,"kb_used_meta":162229,"kb_avail":91441820,"statfs":{"total":95995035648,"available":93636423680,"internally_reserved":0,"allocated":2192269312,"data_stored":2188366650,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":166123189},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2162163712,"data_stored":2162163712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3512471552,"data_stored":3512467480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1412763648,"data_stored":1412759576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1734082560,"data_stored":1734082560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3944873984,"data_stored":3944873984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2527596544,"data_stored":2527592472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1424883712,"data_stored":1424883712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":360644608,"data_stored":360644608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:41:00.052 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:41:00.117 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:00.133 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:40:59 smithi129 ceph-mon[101316]: from='client.35280 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:41:00.147 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:00.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:00.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:00.249 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:41:00.279 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:00.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:00.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:41:00.380 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:00.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:00.424 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:00.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:00.525 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:00.564 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:00.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:00.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:00.666 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:00.714 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:00.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:00.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:00.816 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:00.824 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is set 2024-09-14T04:41:00.833 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:00 smithi129 ceph-mon[101316]: pgmap v2248: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2024-09-14T04:41:00.833 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:00 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1376711520' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:41:00.834 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:00 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:41:00.858 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:00.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:00.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:00.961 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:00.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:00.999 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:01.000 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:01.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:00 smithi039 ceph-mon[106470]: pgmap v2248: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2024-09-14T04:41:01.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:00 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1376711520' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:41:01.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:00 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:41:01.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:00 smithi039 ceph-mon[111235]: pgmap v2248: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2024-09-14T04:41:01.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:00 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1376711520' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:41:01.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:00 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:41:01.076 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-14T04:41:01.100 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:01.160 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:01.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:01.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:01.262 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:01.313 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:01.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:01.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:01.415 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:01.431 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:41:01.456 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:01.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:01.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:01.558 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:01.594 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:01.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:01.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:01.697 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:01.724 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:01.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:01.725 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:01.827 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:01.854 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:01 smithi129 ceph-mon[101316]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:41:01.855 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:01 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:41:01.855 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:01 smithi129 ceph-mon[101316]: osdmap e1338: 8 total, 8 up, 8 in 2024-09-14T04:41:01.862 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:01.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:01.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:01.964 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:01.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:01.993 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:01.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:02.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:01 smithi039 ceph-mon[106470]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:41:02.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:01 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:41:02.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:01 smithi039 ceph-mon[106470]: osdmap e1338: 8 total, 8 up, 8 in 2024-09-14T04:41:02.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:01 smithi039 ceph-mon[111235]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:41:02.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:01 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:41:02.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:01 smithi039 ceph-mon[111235]: osdmap e1338: 8 total, 8 up, 8 in 2024-09-14T04:41:02.095 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:02.134 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:02.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:02.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:02.236 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:02.265 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:02.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:02.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:02.367 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:41:02.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:02.407 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:02.408 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:41:02.425 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:41:02.509 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:02.547 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:02.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:02.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:02.649 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:02.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:02.682 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:02.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:02.783 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:02.807 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:02.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:02.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:02.909 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:02.943 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:02.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:02.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:03.031 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:02 smithi039 ceph-mon[106470]: pgmap v2250: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2024-09-14T04:41:03.031 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:02 smithi039 ceph-mon[111235]: pgmap v2250: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2024-09-14T04:41:03.046 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:03.075 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:02 smithi129 ceph-mon[101316]: pgmap v2250: 89 pgs: 89 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2024-09-14T04:41:03.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:03.076 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:03.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:03.177 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:41:03.213 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:03.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:03.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:41:03.315 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:03.356 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:03.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:03.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:03.376 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:41:03.376 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:41:03.413 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:41:03.414 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:41:03.445 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:41:03.446 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:41:03.458 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:03.482 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:41:03.482 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:41:03.499 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:03.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:03.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:03.520 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:41:03.521 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:41:03.531 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:41:03.531 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [ 2024-09-14T04:41:03.531 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:03.531 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.45053.0:181795 53.c e1338/1315 53:3caba86a:::benchmark_data_smithi039_224949_object51702:head v 1338'11629, mlcod=1338'11625)", 2024-09-14T04:41:03.531 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:41:03.527889+0000", 2024-09-14T04:41:03.532 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.0018923099999999999, 2024-09-14T04:41:03.532 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0019313539999999999, 2024-09-14T04:41:03.532 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:41:03.532 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "started", 2024-09-14T04:41:03.532 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:41:03.532 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:03.532 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:41:03.532 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:41:03.527889+0000", 2024-09-14T04:41:03.532 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:41:03.532 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:03.532 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:03.532 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:41:03.533 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:41:03.527889+0000", 2024-09-14T04:41:03.533 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:41:03.533 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:03.533 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:03.533 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:41:03.533 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:41:03.527888+0000", 2024-09-14T04:41:03.533 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:41:03.533 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:03.534 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:03.534 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:41:03.534 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:41:03.527897+0000", 2024-09-14T04:41:03.534 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.6710000000000005e-06 2024-09-14T04:41:03.534 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:03.534 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:03.534 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:41:03.534 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:41:03.527918+0000", 2024-09-14T04:41:03.534 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.0945999999999999e-05 2024-09-14T04:41:03.534 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:03.534 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:03.534 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:41:03.534 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:41:03.527926+0000", 2024-09-14T04:41:03.535 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.7409999999999992e-06 2024-09-14T04:41:03.535 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:03.535 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:03.535 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:41:03.535 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:41:03.528032+0000", 2024-09-14T04:41:03.535 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010685 2024-09-14T04:41:03.535 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:41:03.535 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:41:03.535 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:41:03.535 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:41:03.528093+0000", 2024-09-14T04:41:03.535 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.0137999999999999e-05 2024-09-14T04:41:03.535 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:41:03.535 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:41:03.535 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:41:03.536 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:41:03.536 INFO:teuthology.orchestra.run.smithi129.stdout: ], 2024-09-14T04:41:03.536 INFO:teuthology.orchestra.run.smithi129.stdout: "num_ops": 1 2024-09-14T04:41:03.536 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:41:03.552 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:41:03.552 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:41:03.592 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:41:03.592 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:41:03.601 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:41:03.627 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:41:03.627 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:41:03.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:03.638 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:03.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:41:03.655 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:41:03.739 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:03.781 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:03.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:03.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:03.877 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:03 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/490930624' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:41:03.883 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:03.935 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:03.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:03.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:04.037 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:04.076 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:04.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:04.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:04.180 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:04.184 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:03 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/490930624' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:41:04.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:04.215 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:04.215 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:04.218 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-14T04:41:04.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:03 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/490930624' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:41:04.316 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:04.352 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:04.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:04.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:04.454 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:04.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:04.490 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:04.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:04.557 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:41:04.591 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:04.630 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:04.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:04.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:04.732 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:04.769 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:04.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:04.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:04.871 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:04.907 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:04.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:04.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:05.012 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:05.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:04 smithi129 ceph-mon[101316]: pgmap v2251: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 879 op/s 2024-09-14T04:41:05.051 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:05.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:05.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:05.153 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:05.175 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:04 smithi039 ceph-mon[106470]: pgmap v2251: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 879 op/s 2024-09-14T04:41:05.175 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:04 smithi039 ceph-mon[111235]: pgmap v2251: 89 pgs: 89 active+clean; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 879 op/s 2024-09-14T04:41:05.190 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:05.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:05.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:05.292 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:05.333 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:05.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:05.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:05.435 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:05.468 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:05.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:05.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:05.570 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:05.604 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:05.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:05.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:05.609 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:41:05.706 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:05.751 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:05.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:05.752 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:05.853 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:05.894 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:05.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:05.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:05.996 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:06.039 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:06.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:06.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:06.141 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:06.178 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:06.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:06.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:06.280 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:06.319 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:06.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:06.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:06.421 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:06.458 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:06.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:06.459 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:06.560 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:41:06.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:06.590 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:06.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:41:06.675 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:41:06.676 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T04:41:06.691 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:06.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:06.733 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:06.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:06.834 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:06.867 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:06.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:06.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:06.969 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:06.997 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:06.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:06.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:07.008 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:41:07.036 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:41:07.036 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:41:07.044 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:06 smithi039 ceph-mon[106470]: pgmap v2252: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 874 op/s 2024-09-14T04:41:07.045 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:06 smithi039 ceph-mon[111235]: pgmap v2252: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 874 op/s 2024-09-14T04:41:07.099 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:07.129 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:07.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:07.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:07.231 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:07.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:06 smithi129 ceph-mon[101316]: pgmap v2252: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 874 op/s 2024-09-14T04:41:07.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:07.285 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:07.286 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:07.386 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:41:07.415 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:07.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:07.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:41:07.517 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:07.557 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:07.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:07.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:07.659 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:07.699 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:07.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:07.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:07.749 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-14T04:41:07.801 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:07.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:07.833 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:07.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:07.934 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:07.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:07.965 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:07.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:08.066 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:08.078 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:41:08.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:08.117 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:08.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:08.218 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:08.254 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:08.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:08.255 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:08.356 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:08.387 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:08.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:08.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:08.489 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:08 smithi129 ceph-mon[101316]: from='client.45197 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:41:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:08 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:41:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:08 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:41:08.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:08.540 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:08.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:08.616 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:08 smithi039 ceph-mon[106470]: from='client.45197 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:41:08.616 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:41:08.616 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:08 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:41:08.617 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:08 smithi039 ceph-mon[111235]: from='client.45197 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:41:08.617 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:08 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:41:08.617 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:08 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:41:08.640 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:08.653 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":2252,"stamp":"2024-09-14T04:41:05.550721+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5156575229,"num_objects":78701,"num_object_clones":0,"num_object_copies":236103,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":78701,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":181786,"num_write_kb":8328210,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137515,"ondisk_log_size":137515,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":16698268,"kb_used_data":15246740,"kb_used_omap":288,"kb_used_meta":1451103,"kb_avail":733262948,"statfs":{"total":767960285184,"available":750861258752,"internally_reserved":0,"allocated":15612661760,"data_stored":15581298086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":294978,"internal_metadata":1485930430},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,2,6,7,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-573571072,"num_objects":-8752,"num_object_clones":0,"num_object_copies":-26256,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8752,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8752,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004070"},"pg_stats":[{"pgid":"53.f","version":"1338'11176","reported_seq":11209,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:02.874492+0000","last_change":"2024-09-14T04:38:26.895527+0000","last_active":"2024-09-14T04:41:02.874492+0000","last_peered":"2024-09-14T04:41:02.874492+0000","last_clean":"2024-09-14T04:41:02.874492+0000","last_became_active":"2024-09-14T04:38:26.895218+0000","last_became_peered":"2024-09-14T04:38:26.895218+0000","last_unstale":"2024-09-14T04:41:02.874492+0000","last_undegraded":"2024-09-14T04:41:02.874492+0000","last_fullsized":"2024-09-14T04:41:02.874492+0000","mapping_epoch":1315,"log_start":"1319'1400","ondisk_log_start":"1319'1400","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":9776,"log_dups_size":0,"ondisk_log_size":9776,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:07:35.310515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319029248,"num_objects":4868,"num_object_clones":0,"num_object_copies":14604,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4868,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11176,"num_write_kb":513408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"53.e","version":"1338'11113","reported_seq":11146,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:02.862816+0000","last_change":"2024-09-14T04:38:26.894036+0000","last_active":"2024-09-14T04:41:02.862816+0000","last_peered":"2024-09-14T04:41:02.862816+0000","last_clean":"2024-09-14T04:41:02.862816+0000","last_became_active":"2024-09-14T04:38:26.893654+0000","last_became_peered":"2024-09-14T04:38:26.893654+0000","last_unstale":"2024-09-14T04:41:02.862816+0000","last_undegraded":"2024-09-14T04:41:02.862816+0000","last_fullsized":"2024-09-14T04:41:02.862816+0000","mapping_epoch":1315,"log_start":"1319'1400","ondisk_log_start":"1319'1400","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":9713,"log_dups_size":0,"ondisk_log_size":9713,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:39:17.902111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324337664,"num_objects":4949,"num_object_clones":0,"num_object_copies":14847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11113,"num_write_kb":513984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"53.d","version":"1338'11351","reported_seq":11411,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:04.608117+0000","last_change":"2024-09-14T04:38:27.529546+0000","last_active":"2024-09-14T04:41:04.608117+0000","last_peered":"2024-09-14T04:41:04.608117+0000","last_clean":"2024-09-14T04:41:04.608117+0000","last_became_active":"2024-09-14T04:38:27.529237+0000","last_became_peered":"2024-09-14T04:38:27.529237+0000","last_unstale":"2024-09-14T04:41:04.608117+0000","last_undegraded":"2024-09-14T04:41:04.608117+0000","last_fullsized":"2024-09-14T04:41:04.608117+0000","mapping_epoch":1315,"log_start":"1322'4200","ondisk_log_start":"1322'4200","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":7151,"log_dups_size":0,"ondisk_log_size":7151,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:37:40.618858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317718552,"num_objects":4849,"num_object_clones":0,"num_object_copies":14547,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4849,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11351,"num_write_kb":518337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"53.c","version":"1338'11605","reported_seq":11638,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:02.931112+0000","last_change":"2024-09-14T04:38:26.895807+0000","last_active":"2024-09-14T04:41:02.931112+0000","last_peered":"2024-09-14T04:41:02.931112+0000","last_clean":"2024-09-14T04:41:02.931112+0000","last_became_active":"2024-09-14T04:38:26.895506+0000","last_became_peered":"2024-09-14T04:38:26.895506+0000","last_unstale":"2024-09-14T04:41:02.931112+0000","last_undegraded":"2024-09-14T04:41:02.931112+0000","last_fullsized":"2024-09-14T04:41:02.931112+0000","mapping_epoch":1315,"log_start":"1319'1600","ondisk_log_start":"1319'1600","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":10005,"log_dups_size":0,"ondisk_log_size":10005,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:07:45.042093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322895872,"num_objects":4927,"num_object_clones":0,"num_object_copies":14781,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4927,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11605,"num_write_kb":529024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"53.b","version":"1338'11199","reported_seq":11232,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:02.986248+0000","last_change":"2024-09-14T04:38:26.893051+0000","last_active":"2024-09-14T04:41:02.986248+0000","last_peered":"2024-09-14T04:41:02.986248+0000","last_clean":"2024-09-14T04:41:02.986248+0000","last_became_active":"2024-09-14T04:38:26.892762+0000","last_became_peered":"2024-09-14T04:38:26.892762+0000","last_unstale":"2024-09-14T04:41:02.986248+0000","last_undegraded":"2024-09-14T04:41:02.986248+0000","last_fullsized":"2024-09-14T04:41:02.986248+0000","mapping_epoch":1315,"log_start":"1319'1100","ondisk_log_start":"1319'1100","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":10099,"log_dups_size":0,"ondisk_log_size":10099,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:23:11.179960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319225856,"num_objects":4871,"num_object_clones":0,"num_object_copies":14613,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4871,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11199,"num_write_kb":514240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"53.a","version":"1338'11375","reported_seq":11434,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:01.572600+0000","last_change":"2024-09-14T04:38:26.895294+0000","last_active":"2024-09-14T04:41:01.572600+0000","last_peered":"2024-09-14T04:41:01.572600+0000","last_clean":"2024-09-14T04:41:01.572600+0000","last_became_active":"2024-09-14T04:38:26.894942+0000","last_became_peered":"2024-09-14T04:38:26.894942+0000","last_unstale":"2024-09-14T04:41:01.572600+0000","last_undegraded":"2024-09-14T04:41:01.572600+0000","last_fullsized":"2024-09-14T04:41:01.572600+0000","mapping_epoch":1315,"log_start":"1320'3000","ondisk_log_start":"1320'3000","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":8375,"log_dups_size":0,"ondisk_log_size":8375,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:17:54.687919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333774848,"num_objects":5093,"num_object_clones":0,"num_object_copies":15279,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5093,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11375,"num_write_kb":526976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"53.9","version":"1338'11344","reported_seq":11403,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:02.543407+0000","last_change":"2024-09-14T04:38:26.899027+0000","last_active":"2024-09-14T04:41:02.543407+0000","last_peered":"2024-09-14T04:41:02.543407+0000","last_clean":"2024-09-14T04:41:02.543407+0000","last_became_active":"2024-09-14T04:38:26.898732+0000","last_became_peered":"2024-09-14T04:38:26.898732+0000","last_unstale":"2024-09-14T04:41:02.543407+0000","last_undegraded":"2024-09-14T04:41:02.543407+0000","last_fullsized":"2024-09-14T04:41:02.543407+0000","mapping_epoch":1315,"log_start":"1319'1900","ondisk_log_start":"1319'1900","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":9444,"log_dups_size":0,"ondisk_log_size":9444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:58:20.477950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326893568,"num_objects":4988,"num_object_clones":0,"num_object_copies":14964,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4988,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11344,"num_write_kb":522624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"53.8","version":"1338'11287","reported_seq":11316,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:03.034736+0000","last_change":"2024-09-14T04:38:26.897778+0000","last_active":"2024-09-14T04:41:03.034736+0000","last_peered":"2024-09-14T04:41:03.034736+0000","last_clean":"2024-09-14T04:41:03.034736+0000","last_became_active":"2024-09-14T04:38:26.897509+0000","last_became_peered":"2024-09-14T04:38:26.897509+0000","last_unstale":"2024-09-14T04:41:03.034736+0000","last_undegraded":"2024-09-14T04:41:03.034736+0000","last_fullsized":"2024-09-14T04:41:03.034736+0000","mapping_epoch":1315,"log_start":"1322'4400","ondisk_log_start":"1322'4400","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":6887,"log_dups_size":0,"ondisk_log_size":6887,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:51:52.846967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317259776,"num_objects":4841,"num_object_clones":0,"num_object_copies":14523,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4841,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11287,"num_write_kb":516096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"53.7","version":"1338'11601","reported_seq":11660,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:04.619661+0000","last_change":"2024-09-14T04:38:26.897393+0000","last_active":"2024-09-14T04:41:04.619661+0000","last_peered":"2024-09-14T04:41:04.619661+0000","last_clean":"2024-09-14T04:41:04.619661+0000","last_became_active":"2024-09-14T04:38:26.897127+0000","last_became_peered":"2024-09-14T04:38:26.897127+0000","last_unstale":"2024-09-14T04:41:04.619661+0000","last_undegraded":"2024-09-14T04:41:04.619661+0000","last_fullsized":"2024-09-14T04:41:04.619661+0000","mapping_epoch":1315,"log_start":"1322'4400","ondisk_log_start":"1322'4400","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":7201,"log_dups_size":0,"ondisk_log_size":7201,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:15:57.417163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316735488,"num_objects":4833,"num_object_clones":0,"num_object_copies":14499,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4833,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11601,"num_write_kb":525888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"53.6","version":"1338'11445","reported_seq":11504,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:01.573501+0000","last_change":"2024-09-14T04:38:27.529579+0000","last_active":"2024-09-14T04:41:01.573501+0000","last_peered":"2024-09-14T04:41:01.573501+0000","last_clean":"2024-09-14T04:41:01.573501+0000","last_became_active":"2024-09-14T04:38:27.529245+0000","last_became_peered":"2024-09-14T04:38:27.529245+0000","last_unstale":"2024-09-14T04:41:01.573501+0000","last_undegraded":"2024-09-14T04:41:01.573501+0000","last_fullsized":"2024-09-14T04:41:01.573501+0000","mapping_epoch":1315,"log_start":"1320'3100","ondisk_log_start":"1320'3100","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":8345,"log_dups_size":0,"ondisk_log_size":8345,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:11.351641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330891264,"num_objects":5049,"num_object_clones":0,"num_object_copies":15147,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5049,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11445,"num_write_kb":527808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"53.5","version":"1338'11294","reported_seq":11353,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:02.597617+0000","last_change":"2024-09-14T04:38:26.897570+0000","last_active":"2024-09-14T04:41:02.597617+0000","last_peered":"2024-09-14T04:41:02.597617+0000","last_clean":"2024-09-14T04:41:02.597617+0000","last_became_active":"2024-09-14T04:38:26.897277+0000","last_became_peered":"2024-09-14T04:38:26.897277+0000","last_unstale":"2024-09-14T04:41:02.597617+0000","last_undegraded":"2024-09-14T04:41:02.597617+0000","last_fullsized":"2024-09-14T04:41:02.597617+0000","mapping_epoch":1315,"log_start":"1319'1900","ondisk_log_start":"1319'1900","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":9394,"log_dups_size":0,"ondisk_log_size":9394,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:06:40.318691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320077824,"num_objects":4884,"num_object_clones":0,"num_object_copies":14652,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4884,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11294,"num_write_kb":517696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"53.4","version":"1338'11264","reported_seq":11322,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:04.619330+0000","last_change":"2024-09-14T04:38:27.529509+0000","last_active":"2024-09-14T04:41:04.619330+0000","last_peered":"2024-09-14T04:41:04.619330+0000","last_clean":"2024-09-14T04:41:04.619330+0000","last_became_active":"2024-09-14T04:38:27.529180+0000","last_became_peered":"2024-09-14T04:38:27.529180+0000","last_unstale":"2024-09-14T04:41:04.619330+0000","last_undegraded":"2024-09-14T04:41:04.619330+0000","last_fullsized":"2024-09-14T04:41:04.619330+0000","mapping_epoch":1315,"log_start":"1322'4100","ondisk_log_start":"1322'4100","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":7164,"log_dups_size":0,"ondisk_log_size":7164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:04:21.631485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314179584,"num_objects":4794,"num_object_clones":0,"num_object_copies":14382,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4794,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11264,"num_write_kb":513856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"53.3","version":"1338'11320","reported_seq":11353,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:02.854250+0000","last_change":"2024-09-14T04:38:26.894968+0000","last_active":"2024-09-14T04:41:02.854250+0000","last_peered":"2024-09-14T04:41:02.854250+0000","last_clean":"2024-09-14T04:41:02.854250+0000","last_became_active":"2024-09-14T04:38:26.894621+0000","last_became_peered":"2024-09-14T04:38:26.894621+0000","last_unstale":"2024-09-14T04:41:02.854250+0000","last_undegraded":"2024-09-14T04:41:02.854250+0000","last_fullsized":"2024-09-14T04:41:02.854250+0000","mapping_epoch":1315,"log_start":"1319'1600","ondisk_log_start":"1319'1600","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":9720,"log_dups_size":0,"ondisk_log_size":9720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:15:43.660519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322961408,"num_objects":4928,"num_object_clones":0,"num_object_copies":14784,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4928,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11320,"num_write_kb":519936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"53.2","version":"1338'11307","reported_seq":11340,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:02.969890+0000","last_change":"2024-09-14T04:38:26.897057+0000","last_active":"2024-09-14T04:41:02.969890+0000","last_peered":"2024-09-14T04:41:02.969890+0000","last_clean":"2024-09-14T04:41:02.969890+0000","last_became_active":"2024-09-14T04:38:26.896677+0000","last_became_peered":"2024-09-14T04:38:26.896677+0000","last_unstale":"2024-09-14T04:41:02.969890+0000","last_undegraded":"2024-09-14T04:41:02.969890+0000","last_fullsized":"2024-09-14T04:41:02.969890+0000","mapping_epoch":1315,"log_start":"1319'1300","ondisk_log_start":"1319'1300","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":10007,"log_dups_size":0,"ondisk_log_size":10007,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:15.244766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325910528,"num_objects":4973,"num_object_clones":0,"num_object_copies":14919,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4973,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11307,"num_write_kb":520960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"53.1","version":"1338'11565","reported_seq":11624,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:04.537031+0000","last_change":"2024-09-14T04:38:26.897425+0000","last_active":"2024-09-14T04:41:04.537031+0000","last_peered":"2024-09-14T04:41:04.537031+0000","last_clean":"2024-09-14T04:41:04.537031+0000","last_became_active":"2024-09-14T04:38:26.897127+0000","last_became_peered":"2024-09-14T04:38:26.897127+0000","last_unstale":"2024-09-14T04:41:04.537031+0000","last_undegraded":"2024-09-14T04:41:04.537031+0000","last_fullsized":"2024-09-14T04:41:04.537031+0000","mapping_epoch":1315,"log_start":"1322'4400","ondisk_log_start":"1322'4400","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":7165,"log_dups_size":0,"ondisk_log_size":7165,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:49:46.941647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318963712,"num_objects":4867,"num_object_clones":0,"num_object_copies":14601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11565,"num_write_kb":525824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"53.0","version":"1338'11261","reported_seq":11294,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:02.971796+0000","last_change":"2024-09-14T04:38:27.529766+0000","last_active":"2024-09-14T04:41:02.971796+0000","last_peered":"2024-09-14T04:41:02.971796+0000","last_clean":"2024-09-14T04:41:02.971796+0000","last_became_active":"2024-09-14T04:38:27.529414+0000","last_became_peered":"2024-09-14T04:38:27.529414+0000","last_unstale":"2024-09-14T04:41:02.971796+0000","last_undegraded":"2024-09-14T04:41:02.971796+0000","last_fullsized":"2024-09-14T04:41:02.971796+0000","mapping_epoch":1315,"log_start":"1322'4400","ondisk_log_start":"1322'4400","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":6861,"log_dups_size":0,"ondisk_log_size":6861,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:30:08.912265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325124096,"num_objects":4961,"num_object_clones":0,"num_object_copies":14883,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4961,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11261,"num_write_kb":519104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":1961,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.829221+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T04:41:00.829221+0000","last_peered":"2024-09-14T04:41:00.829221+0000","last_clean":"2024-09-14T04:41:00.829221+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T04:41:00.829221+0000","last_undegraded":"2024-09-14T04:41:00.829221+0000","last_fullsized":"2024-09-14T04:41:00.829221+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1300,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.784254+0000","last_change":"2024-09-14T03:53:45.836449+0000","last_active":"2024-09-14T04:40:51.784254+0000","last_peered":"2024-09-14T04:40:51.784254+0000","last_clean":"2024-09-14T04:40:51.784254+0000","last_became_active":"2024-09-14T03:53:45.836108+0000","last_became_peered":"2024-09-14T03:53:45.836108+0000","last_unstale":"2024-09-14T04:40:51.784254+0000","last_undegraded":"2024-09-14T04:40:51.784254+0000","last_fullsized":"2024-09-14T04:40:51.784254+0000","mapping_epoch":73,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":74,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:52:13.095225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1951,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.848238+0000","last_change":"2024-09-14T04:07:58.504829+0000","last_active":"2024-09-14T04:41:00.848238+0000","last_peered":"2024-09-14T04:41:00.848238+0000","last_clean":"2024-09-14T04:41:00.848238+0000","last_became_active":"2024-09-14T04:07:58.502623+0000","last_became_peered":"2024-09-14T04:07:58.502623+0000","last_unstale":"2024-09-14T04:41:00.848238+0000","last_undegraded":"2024-09-14T04:41:00.848238+0000","last_fullsized":"2024-09-14T04:41:00.848238+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:59.091993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1962,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.845281+0000","last_change":"2024-09-14T04:07:58.507125+0000","last_active":"2024-09-14T04:41:00.845281+0000","last_peered":"2024-09-14T04:41:00.845281+0000","last_clean":"2024-09-14T04:41:00.845281+0000","last_became_active":"2024-09-14T04:07:58.505701+0000","last_became_peered":"2024-09-14T04:07:58.505701+0000","last_unstale":"2024-09-14T04:41:00.845281+0000","last_undegraded":"2024-09-14T04:41:00.845281+0000","last_fullsized":"2024-09-14T04:41:00.845281+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:25:00.847961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1291,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.780777+0000","last_change":"2024-09-14T03:53:05.234444+0000","last_active":"2024-09-14T04:40:51.780777+0000","last_peered":"2024-09-14T04:40:51.780777+0000","last_clean":"2024-09-14T04:40:51.780777+0000","last_became_active":"2024-09-14T03:53:05.234115+0000","last_became_peered":"2024-09-14T03:53:05.234115+0000","last_unstale":"2024-09-14T04:40:51.780777+0000","last_undegraded":"2024-09-14T04:40:51.780777+0000","last_fullsized":"2024-09-14T04:40:51.780777+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:01:23.153937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1321,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.785469+0000","last_change":"2024-09-14T04:07:58.547135+0000","last_active":"2024-09-14T04:40:51.785469+0000","last_peered":"2024-09-14T04:40:51.785469+0000","last_clean":"2024-09-14T04:40:51.785469+0000","last_became_active":"2024-09-14T04:07:58.546751+0000","last_became_peered":"2024-09-14T04:07:58.546751+0000","last_unstale":"2024-09-14T04:40:51.785469+0000","last_undegraded":"2024-09-14T04:40:51.785469+0000","last_fullsized":"2024-09-14T04:40:51.785469+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:01.092698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1306,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.780611+0000","last_change":"2024-09-14T04:06:46.875795+0000","last_active":"2024-09-14T04:40:51.780611+0000","last_peered":"2024-09-14T04:40:51.780611+0000","last_clean":"2024-09-14T04:40:51.780611+0000","last_became_active":"2024-09-14T04:06:46.875395+0000","last_became_peered":"2024-09-14T04:06:46.875395+0000","last_unstale":"2024-09-14T04:40:51.780611+0000","last_undegraded":"2024-09-14T04:40:51.780611+0000","last_fullsized":"2024-09-14T04:40:51.780611+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:51:10.713063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1321,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.783605+0000","last_change":"2024-09-14T04:07:58.514349+0000","last_active":"2024-09-14T04:40:51.783605+0000","last_peered":"2024-09-14T04:40:51.783605+0000","last_clean":"2024-09-14T04:40:51.783605+0000","last_became_active":"2024-09-14T04:07:58.513485+0000","last_became_peered":"2024-09-14T04:07:58.513485+0000","last_unstale":"2024-09-14T04:40:51.783605+0000","last_undegraded":"2024-09-14T04:40:51.783605+0000","last_fullsized":"2024-09-14T04:40:51.783605+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:33:09.031813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":1944,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.829337+0000","last_change":"2024-09-14T04:05:54.428057+0000","last_active":"2024-09-14T04:41:00.829337+0000","last_peered":"2024-09-14T04:41:00.829337+0000","last_clean":"2024-09-14T04:41:00.829337+0000","last_became_active":"2024-09-14T04:05:54.414895+0000","last_became_peered":"2024-09-14T04:05:54.414895+0000","last_unstale":"2024-09-14T04:41:00.829337+0000","last_undegraded":"2024-09-14T04:41:00.829337+0000","last_fullsized":"2024-09-14T04:41:00.829337+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:45:15.752675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1958,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.844080+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T04:41:00.844080+0000","last_peered":"2024-09-14T04:41:00.844080+0000","last_clean":"2024-09-14T04:41:00.844080+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T04:41:00.844080+0000","last_undegraded":"2024-09-14T04:41:00.844080+0000","last_fullsized":"2024-09-14T04:41:00.844080+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1323,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.785591+0000","last_change":"2024-09-14T04:07:21.717071+0000","last_active":"2024-09-14T04:40:51.785591+0000","last_peered":"2024-09-14T04:40:51.785591+0000","last_clean":"2024-09-14T04:40:51.785591+0000","last_became_active":"2024-09-14T04:07:21.716215+0000","last_became_peered":"2024-09-14T04:07:21.716215+0000","last_unstale":"2024-09-14T04:40:51.785591+0000","last_undegraded":"2024-09-14T04:40:51.785591+0000","last_fullsized":"2024-09-14T04:40:51.785591+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:48:47.175490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1931,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.847039+0000","last_change":"2024-09-14T04:07:58.524371+0000","last_active":"2024-09-14T04:41:00.847039+0000","last_peered":"2024-09-14T04:41:00.847039+0000","last_clean":"2024-09-14T04:41:00.847039+0000","last_became_active":"2024-09-14T04:07:58.524088+0000","last_became_peered":"2024-09-14T04:07:58.524088+0000","last_unstale":"2024-09-14T04:41:00.847039+0000","last_undegraded":"2024-09-14T04:41:00.847039+0000","last_fullsized":"2024-09-14T04:41:00.847039+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:42.615569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1926,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.833220+0000","last_change":"2024-09-14T04:07:58.507553+0000","last_active":"2024-09-14T04:41:00.833220+0000","last_peered":"2024-09-14T04:41:00.833220+0000","last_clean":"2024-09-14T04:41:00.833220+0000","last_became_active":"2024-09-14T04:07:58.507139+0000","last_became_peered":"2024-09-14T04:07:58.507139+0000","last_unstale":"2024-09-14T04:41:00.833220+0000","last_undegraded":"2024-09-14T04:41:00.833220+0000","last_fullsized":"2024-09-14T04:41:00.833220+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:11:57.821769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1956,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.829603+0000","last_change":"2024-09-14T04:07:58.561490+0000","last_active":"2024-09-14T04:41:00.829603+0000","last_peered":"2024-09-14T04:41:00.829603+0000","last_clean":"2024-09-14T04:41:00.829603+0000","last_became_active":"2024-09-14T04:07:58.559841+0000","last_became_peered":"2024-09-14T04:07:58.559841+0000","last_unstale":"2024-09-14T04:41:00.829603+0000","last_undegraded":"2024-09-14T04:41:00.829603+0000","last_fullsized":"2024-09-14T04:41:00.829603+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:27:03.795368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1947,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.845331+0000","last_change":"2024-09-14T04:07:58.555731+0000","last_active":"2024-09-14T04:41:00.845331+0000","last_peered":"2024-09-14T04:41:00.845331+0000","last_clean":"2024-09-14T04:41:00.845331+0000","last_became_active":"2024-09-14T04:07:58.549576+0000","last_became_peered":"2024-09-14T04:07:58.549576+0000","last_unstale":"2024-09-14T04:41:00.845331+0000","last_undegraded":"2024-09-14T04:41:00.845331+0000","last_fullsized":"2024-09-14T04:41:00.845331+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:28:42.339589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1315,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:57.882469+0000","last_change":"2024-09-14T04:07:21.720797+0000","last_active":"2024-09-14T04:40:57.882469+0000","last_peered":"2024-09-14T04:40:57.882469+0000","last_clean":"2024-09-14T04:40:57.882469+0000","last_became_active":"2024-09-14T04:07:21.719331+0000","last_became_peered":"2024-09-14T04:07:21.719331+0000","last_unstale":"2024-09-14T04:40:57.882469+0000","last_undegraded":"2024-09-14T04:40:57.882469+0000","last_fullsized":"2024-09-14T04:40:57.882469+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:35.404023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1957,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.845527+0000","last_change":"2024-09-14T04:07:58.502533+0000","last_active":"2024-09-14T04:41:00.845527+0000","last_peered":"2024-09-14T04:41:00.845527+0000","last_clean":"2024-09-14T04:41:00.845527+0000","last_became_active":"2024-09-14T04:07:58.499849+0000","last_became_peered":"2024-09-14T04:07:58.499849+0000","last_unstale":"2024-09-14T04:41:00.845527+0000","last_undegraded":"2024-09-14T04:41:00.845527+0000","last_fullsized":"2024-09-14T04:41:00.845527+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:15:35.609746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1951,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.829742+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T04:41:00.829742+0000","last_peered":"2024-09-14T04:41:00.829742+0000","last_clean":"2024-09-14T04:41:00.829742+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T04:41:00.829742+0000","last_undegraded":"2024-09-14T04:41:00.829742+0000","last_fullsized":"2024-09-14T04:41:00.829742+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1933,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.846214+0000","last_change":"2024-09-14T04:07:58.507682+0000","last_active":"2024-09-14T04:41:00.846214+0000","last_peered":"2024-09-14T04:41:00.846214+0000","last_clean":"2024-09-14T04:41:00.846214+0000","last_became_active":"2024-09-14T04:07:58.507365+0000","last_became_peered":"2024-09-14T04:07:58.507365+0000","last_unstale":"2024-09-14T04:41:00.846214+0000","last_undegraded":"2024-09-14T04:41:00.846214+0000","last_fullsized":"2024-09-14T04:41:00.846214+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:14:47.294950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1303,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.780475+0000","last_change":"2024-09-14T04:06:46.875668+0000","last_active":"2024-09-14T04:40:51.780475+0000","last_peered":"2024-09-14T04:40:51.780475+0000","last_clean":"2024-09-14T04:40:51.780475+0000","last_became_active":"2024-09-14T04:06:46.875149+0000","last_became_peered":"2024-09-14T04:06:46.875149+0000","last_unstale":"2024-09-14T04:40:51.780475+0000","last_undegraded":"2024-09-14T04:40:51.780475+0000","last_fullsized":"2024-09-14T04:40:51.780475+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:45:59.190047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1304,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.801432+0000","last_change":"2024-09-14T04:07:58.510699+0000","last_active":"2024-09-14T04:40:51.801432+0000","last_peered":"2024-09-14T04:40:51.801432+0000","last_clean":"2024-09-14T04:40:51.801432+0000","last_became_active":"2024-09-14T04:07:58.506240+0000","last_became_peered":"2024-09-14T04:07:58.506240+0000","last_unstale":"2024-09-14T04:40:51.801432+0000","last_undegraded":"2024-09-14T04:40:51.801432+0000","last_fullsized":"2024-09-14T04:40:51.801432+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:27:18.051658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1958,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.831768+0000","last_change":"2024-09-14T04:07:21.687985+0000","last_active":"2024-09-14T04:41:00.831768+0000","last_peered":"2024-09-14T04:41:00.831768+0000","last_clean":"2024-09-14T04:41:00.831768+0000","last_became_active":"2024-09-14T04:07:21.687670+0000","last_became_peered":"2024-09-14T04:07:21.687670+0000","last_unstale":"2024-09-14T04:41:00.831768+0000","last_undegraded":"2024-09-14T04:41:00.831768+0000","last_fullsized":"2024-09-14T04:41:00.831768+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:53:58.295662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":1316,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.781906+0000","last_change":"2024-09-14T03:53:05.244055+0000","last_active":"2024-09-14T04:40:51.781906+0000","last_peered":"2024-09-14T04:40:51.781906+0000","last_clean":"2024-09-14T04:40:51.781906+0000","last_became_active":"2024-09-14T03:53:05.243752+0000","last_became_peered":"2024-09-14T03:53:05.243752+0000","last_unstale":"2024-09-14T04:40:51.781906+0000","last_undegraded":"2024-09-14T04:40:51.781906+0000","last_fullsized":"2024-09-14T04:40:51.781906+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:12:31.850064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1307,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.783799+0000","last_change":"2024-09-14T04:06:46.895670+0000","last_active":"2024-09-14T04:40:51.783799+0000","last_peered":"2024-09-14T04:40:51.783799+0000","last_clean":"2024-09-14T04:40:51.783799+0000","last_became_active":"2024-09-14T04:06:46.895370+0000","last_became_peered":"2024-09-14T04:06:46.895370+0000","last_unstale":"2024-09-14T04:40:51.783799+0000","last_undegraded":"2024-09-14T04:40:51.783799+0000","last_fullsized":"2024-09-14T04:40:51.783799+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:48:48.040529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1295,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:58.032860+0000","last_change":"2024-09-14T03:53:06.634043+0000","last_active":"2024-09-14T04:40:58.032860+0000","last_peered":"2024-09-14T04:40:58.032860+0000","last_clean":"2024-09-14T04:40:58.032860+0000","last_became_active":"2024-09-14T03:53:06.633638+0000","last_became_peered":"2024-09-14T03:53:06.633638+0000","last_unstale":"2024-09-14T04:40:58.032860+0000","last_undegraded":"2024-09-14T04:40:58.032860+0000","last_fullsized":"2024-09-14T04:40:58.032860+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:15:53.434064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1318,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.777248+0000","last_change":"2024-09-14T04:07:21.721242+0000","last_active":"2024-09-14T04:40:51.777248+0000","last_peered":"2024-09-14T04:40:51.777248+0000","last_clean":"2024-09-14T04:40:51.777248+0000","last_became_active":"2024-09-14T04:07:21.720084+0000","last_became_peered":"2024-09-14T04:07:21.720084+0000","last_unstale":"2024-09-14T04:40:51.777248+0000","last_undegraded":"2024-09-14T04:40:51.777248+0000","last_fullsized":"2024-09-14T04:40:51.777248+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:58:41.308643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":1958,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.829477+0000","last_change":"2024-09-14T04:06:46.932567+0000","last_active":"2024-09-14T04:41:00.829477+0000","last_peered":"2024-09-14T04:41:00.829477+0000","last_clean":"2024-09-14T04:41:00.829477+0000","last_became_active":"2024-09-14T04:06:46.930732+0000","last_became_peered":"2024-09-14T04:06:46.930732+0000","last_unstale":"2024-09-14T04:41:00.829477+0000","last_undegraded":"2024-09-14T04:41:00.829477+0000","last_fullsized":"2024-09-14T04:41:00.829477+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:10:02.729499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1925,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.845423+0000","last_change":"2024-09-14T04:07:58.555446+0000","last_active":"2024-09-14T04:41:00.845423+0000","last_peered":"2024-09-14T04:41:00.845423+0000","last_clean":"2024-09-14T04:41:00.845423+0000","last_became_active":"2024-09-14T04:07:58.548355+0000","last_became_peered":"2024-09-14T04:07:58.548355+0000","last_unstale":"2024-09-14T04:41:00.845423+0000","last_undegraded":"2024-09-14T04:41:00.845423+0000","last_fullsized":"2024-09-14T04:41:00.845423+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:56:28.426094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1962,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.832227+0000","last_change":"2024-09-14T04:06:46.915273+0000","last_active":"2024-09-14T04:41:00.832227+0000","last_peered":"2024-09-14T04:41:00.832227+0000","last_clean":"2024-09-14T04:41:00.832227+0000","last_became_active":"2024-09-14T04:06:46.914994+0000","last_became_peered":"2024-09-14T04:06:46.914994+0000","last_unstale":"2024-09-14T04:41:00.832227+0000","last_undegraded":"2024-09-14T04:41:00.832227+0000","last_fullsized":"2024-09-14T04:41:00.832227+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:48:58.802609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1926,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.846734+0000","last_change":"2024-09-14T04:07:58.564377+0000","last_active":"2024-09-14T04:41:00.846734+0000","last_peered":"2024-09-14T04:41:00.846734+0000","last_clean":"2024-09-14T04:41:00.846734+0000","last_became_active":"2024-09-14T04:07:58.546913+0000","last_became_peered":"2024-09-14T04:07:58.546913+0000","last_unstale":"2024-09-14T04:41:00.846734+0000","last_undegraded":"2024-09-14T04:41:00.846734+0000","last_fullsized":"2024-09-14T04:41:00.846734+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:05:53.637115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1944,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.829514+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T04:41:00.829514+0000","last_peered":"2024-09-14T04:41:00.829514+0000","last_clean":"2024-09-14T04:41:00.829514+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T04:41:00.829514+0000","last_undegraded":"2024-09-14T04:41:00.829514+0000","last_fullsized":"2024-09-14T04:41:00.829514+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1317,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.801634+0000","last_change":"2024-09-14T04:05:54.381035+0000","last_active":"2024-09-14T04:40:51.801634+0000","last_peered":"2024-09-14T04:40:51.801634+0000","last_clean":"2024-09-14T04:40:51.801634+0000","last_became_active":"2024-09-14T04:05:54.379861+0000","last_became_peered":"2024-09-14T04:05:54.379861+0000","last_unstale":"2024-09-14T04:40:51.801634+0000","last_undegraded":"2024-09-14T04:40:51.801634+0000","last_fullsized":"2024-09-14T04:40:51.801634+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:07:46.302925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1962,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.844489+0000","last_change":"2024-09-14T04:07:21.686880+0000","last_active":"2024-09-14T04:41:00.844489+0000","last_peered":"2024-09-14T04:41:00.844489+0000","last_clean":"2024-09-14T04:41:00.844489+0000","last_became_active":"2024-09-14T04:07:21.685558+0000","last_became_peered":"2024-09-14T04:07:21.685558+0000","last_unstale":"2024-09-14T04:41:00.844489+0000","last_undegraded":"2024-09-14T04:41:00.844489+0000","last_fullsized":"2024-09-14T04:41:00.844489+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:24:18.693428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1950,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.831655+0000","last_change":"2024-09-14T04:06:46.905931+0000","last_active":"2024-09-14T04:41:00.831655+0000","last_peered":"2024-09-14T04:41:00.831655+0000","last_clean":"2024-09-14T04:41:00.831655+0000","last_became_active":"2024-09-14T04:06:46.905666+0000","last_became_peered":"2024-09-14T04:06:46.905666+0000","last_unstale":"2024-09-14T04:41:00.831655+0000","last_undegraded":"2024-09-14T04:41:00.831655+0000","last_fullsized":"2024-09-14T04:41:00.831655+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:40:57.007948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1979,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.844727+0000","last_change":"2024-09-14T04:06:46.930482+0000","last_active":"2024-09-14T04:41:00.844727+0000","last_peered":"2024-09-14T04:41:00.844727+0000","last_clean":"2024-09-14T04:41:00.844727+0000","last_became_active":"2024-09-14T04:06:46.927372+0000","last_became_peered":"2024-09-14T04:06:46.927372+0000","last_unstale":"2024-09-14T04:41:00.844727+0000","last_undegraded":"2024-09-14T04:41:00.844727+0000","last_fullsized":"2024-09-14T04:41:00.844727+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:07:22.159128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1322,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.780929+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T04:40:51.780929+0000","last_peered":"2024-09-14T04:40:51.780929+0000","last_clean":"2024-09-14T04:40:51.780929+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T04:40:51.780929+0000","last_undegraded":"2024-09-14T04:40:51.780929+0000","last_fullsized":"2024-09-14T04:40:51.780929+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1972,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.844638+0000","last_change":"2024-09-14T04:07:58.510721+0000","last_active":"2024-09-14T04:41:00.844638+0000","last_peered":"2024-09-14T04:41:00.844638+0000","last_clean":"2024-09-14T04:41:00.844638+0000","last_became_active":"2024-09-14T04:07:58.510332+0000","last_became_peered":"2024-09-14T04:07:58.510332+0000","last_unstale":"2024-09-14T04:41:00.844638+0000","last_undegraded":"2024-09-14T04:41:00.844638+0000","last_fullsized":"2024-09-14T04:41:00.844638+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:07:03.926842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1311,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.828330+0000","last_change":"2024-09-14T04:05:54.426939+0000","last_active":"2024-09-14T04:41:00.828330+0000","last_peered":"2024-09-14T04:41:00.828330+0000","last_clean":"2024-09-14T04:41:00.828330+0000","last_became_active":"2024-09-14T04:05:54.426427+0000","last_became_peered":"2024-09-14T04:05:54.426427+0000","last_unstale":"2024-09-14T04:41:00.828330+0000","last_undegraded":"2024-09-14T04:41:00.828330+0000","last_fullsized":"2024-09-14T04:41:00.828330+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:53:55.841489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1321,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.781023+0000","last_change":"2024-09-14T04:06:46.875340+0000","last_active":"2024-09-14T04:40:51.781023+0000","last_peered":"2024-09-14T04:40:51.781023+0000","last_clean":"2024-09-14T04:40:51.781023+0000","last_became_active":"2024-09-14T04:06:46.874552+0000","last_became_peered":"2024-09-14T04:06:46.874552+0000","last_unstale":"2024-09-14T04:40:51.781023+0000","last_undegraded":"2024-09-14T04:40:51.781023+0000","last_fullsized":"2024-09-14T04:40:51.781023+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:52.257357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1958,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.843723+0000","last_change":"2024-09-14T04:06:46.917681+0000","last_active":"2024-09-14T04:41:00.843723+0000","last_peered":"2024-09-14T04:41:00.843723+0000","last_clean":"2024-09-14T04:41:00.843723+0000","last_became_active":"2024-09-14T04:06:46.917401+0000","last_became_peered":"2024-09-14T04:06:46.917401+0000","last_unstale":"2024-09-14T04:41:00.843723+0000","last_undegraded":"2024-09-14T04:41:00.843723+0000","last_fullsized":"2024-09-14T04:41:00.843723+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:04:39.949490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1963,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.832769+0000","last_change":"2024-09-14T04:07:21.685415+0000","last_active":"2024-09-14T04:41:00.832769+0000","last_peered":"2024-09-14T04:41:00.832769+0000","last_clean":"2024-09-14T04:41:00.832769+0000","last_became_active":"2024-09-14T04:07:21.683504+0000","last_became_peered":"2024-09-14T04:07:21.683504+0000","last_unstale":"2024-09-14T04:41:00.832769+0000","last_undegraded":"2024-09-14T04:41:00.832769+0000","last_fullsized":"2024-09-14T04:41:00.832769+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:21:08.806291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1965,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.842331+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T04:41:00.842331+0000","last_peered":"2024-09-14T04:41:00.842331+0000","last_clean":"2024-09-14T04:41:00.842331+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T04:41:00.842331+0000","last_undegraded":"2024-09-14T04:41:00.842331+0000","last_fullsized":"2024-09-14T04:41:00.842331+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1966,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.829685+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T04:41:00.829685+0000","last_peered":"2024-09-14T04:41:00.829685+0000","last_clean":"2024-09-14T04:41:00.829685+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T04:41:00.829685+0000","last_undegraded":"2024-09-14T04:41:00.829685+0000","last_fullsized":"2024-09-14T04:41:00.829685+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1954,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.835158+0000","last_change":"2024-09-14T04:07:58.521344+0000","last_active":"2024-09-14T04:41:00.835158+0000","last_peered":"2024-09-14T04:41:00.835158+0000","last_clean":"2024-09-14T04:41:00.835158+0000","last_became_active":"2024-09-14T04:07:58.520965+0000","last_became_peered":"2024-09-14T04:07:58.520965+0000","last_unstale":"2024-09-14T04:41:00.835158+0000","last_undegraded":"2024-09-14T04:41:00.835158+0000","last_fullsized":"2024-09-14T04:41:00.835158+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:58:46.029013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1333,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.801290+0000","last_change":"2024-09-14T04:06:46.857436+0000","last_active":"2024-09-14T04:40:51.801290+0000","last_peered":"2024-09-14T04:40:51.801290+0000","last_clean":"2024-09-14T04:40:51.801290+0000","last_became_active":"2024-09-14T04:06:46.857154+0000","last_became_peered":"2024-09-14T04:06:46.857154+0000","last_unstale":"2024-09-14T04:40:51.801290+0000","last_undegraded":"2024-09-14T04:40:51.801290+0000","last_fullsized":"2024-09-14T04:40:51.801290+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:53:28.772152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1290,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.775828+0000","last_change":"2024-09-14T03:53:05.551268+0000","last_active":"2024-09-14T04:40:51.775828+0000","last_peered":"2024-09-14T04:40:51.775828+0000","last_clean":"2024-09-14T04:40:51.775828+0000","last_became_active":"2024-09-14T03:53:05.551015+0000","last_became_peered":"2024-09-14T03:53:05.551015+0000","last_unstale":"2024-09-14T04:40:51.775828+0000","last_undegraded":"2024-09-14T04:40:51.775828+0000","last_fullsized":"2024-09-14T04:40:51.775828+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.905475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1936,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.846967+0000","last_change":"2024-09-14T04:07:58.503896+0000","last_active":"2024-09-14T04:41:00.846967+0000","last_peered":"2024-09-14T04:41:00.846967+0000","last_clean":"2024-09-14T04:41:00.846967+0000","last_became_active":"2024-09-14T04:07:58.499253+0000","last_became_peered":"2024-09-14T04:07:58.499253+0000","last_unstale":"2024-09-14T04:41:00.846967+0000","last_undegraded":"2024-09-14T04:41:00.846967+0000","last_fullsized":"2024-09-14T04:41:00.846967+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:48:27.543818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1320,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.780323+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T04:40:51.780323+0000","last_peered":"2024-09-14T04:40:51.780323+0000","last_clean":"2024-09-14T04:40:51.780323+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T04:40:51.780323+0000","last_undegraded":"2024-09-14T04:40:51.780323+0000","last_fullsized":"2024-09-14T04:40:51.780323+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":1635,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.801182+0000","last_change":"2024-09-14T04:05:54.389043+0000","last_active":"2024-09-14T04:40:51.801182+0000","last_peered":"2024-09-14T04:40:51.801182+0000","last_clean":"2024-09-14T04:40:51.801182+0000","last_became_active":"2024-09-14T04:05:54.388655+0000","last_became_peered":"2024-09-14T04:05:54.388655+0000","last_unstale":"2024-09-14T04:40:51.801182+0000","last_undegraded":"2024-09-14T04:40:51.801182+0000","last_fullsized":"2024-09-14T04:40:51.801182+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T03:49:18.237706+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:38:59.615597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1927,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.843553+0000","last_change":"2024-09-14T04:07:58.530406+0000","last_active":"2024-09-14T04:41:00.843553+0000","last_peered":"2024-09-14T04:41:00.843553+0000","last_clean":"2024-09-14T04:41:00.843553+0000","last_became_active":"2024-09-14T04:07:58.530078+0000","last_became_peered":"2024-09-14T04:07:58.530078+0000","last_unstale":"2024-09-14T04:41:00.843553+0000","last_undegraded":"2024-09-14T04:41:00.843553+0000","last_fullsized":"2024-09-14T04:41:00.843553+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:54:53.524001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1935,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.845223+0000","last_change":"2024-09-14T04:07:58.555717+0000","last_active":"2024-09-14T04:41:00.845223+0000","last_peered":"2024-09-14T04:41:00.845223+0000","last_clean":"2024-09-14T04:41:00.845223+0000","last_became_active":"2024-09-14T04:07:58.548894+0000","last_became_peered":"2024-09-14T04:07:58.548894+0000","last_unstale":"2024-09-14T04:41:00.845223+0000","last_undegraded":"2024-09-14T04:41:00.845223+0000","last_fullsized":"2024-09-14T04:41:00.845223+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:42.061551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1308,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.782189+0000","last_change":"2024-09-14T04:06:46.894741+0000","last_active":"2024-09-14T04:40:51.782189+0000","last_peered":"2024-09-14T04:40:51.782189+0000","last_clean":"2024-09-14T04:40:51.782189+0000","last_became_active":"2024-09-14T04:06:46.894445+0000","last_became_peered":"2024-09-14T04:06:46.894445+0000","last_unstale":"2024-09-14T04:40:51.782189+0000","last_undegraded":"2024-09-14T04:40:51.782189+0000","last_fullsized":"2024-09-14T04:40:51.782189+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:52:00.055271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1310,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.781678+0000","last_change":"2024-09-14T04:07:21.715462+0000","last_active":"2024-09-14T04:40:51.781678+0000","last_peered":"2024-09-14T04:40:51.781678+0000","last_clean":"2024-09-14T04:40:51.781678+0000","last_became_active":"2024-09-14T04:07:21.713032+0000","last_became_peered":"2024-09-14T04:07:21.713032+0000","last_unstale":"2024-09-14T04:40:51.781678+0000","last_undegraded":"2024-09-14T04:40:51.781678+0000","last_fullsized":"2024-09-14T04:40:51.781678+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:37:05.545164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1323,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.781784+0000","last_change":"2024-09-14T04:07:21.718037+0000","last_active":"2024-09-14T04:40:51.781784+0000","last_peered":"2024-09-14T04:40:51.781784+0000","last_clean":"2024-09-14T04:40:51.781784+0000","last_became_active":"2024-09-14T04:07:21.717700+0000","last_became_peered":"2024-09-14T04:07:21.717700+0000","last_unstale":"2024-09-14T04:40:51.781784+0000","last_undegraded":"2024-09-14T04:40:51.781784+0000","last_fullsized":"2024-09-14T04:40:51.781784+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:47:33.483785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1306,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:52.882002+0000","last_change":"2024-09-14T04:06:46.868780+0000","last_active":"2024-09-14T04:40:52.882002+0000","last_peered":"2024-09-14T04:40:52.882002+0000","last_clean":"2024-09-14T04:40:52.882002+0000","last_became_active":"2024-09-14T04:06:46.868483+0000","last_became_peered":"2024-09-14T04:06:46.868483+0000","last_unstale":"2024-09-14T04:40:52.882002+0000","last_undegraded":"2024-09-14T04:40:52.882002+0000","last_fullsized":"2024-09-14T04:40:52.882002+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:03.133158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1308,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.781831+0000","last_change":"2024-09-14T04:07:58.515859+0000","last_active":"2024-09-14T04:40:51.781831+0000","last_peered":"2024-09-14T04:40:51.781831+0000","last_clean":"2024-09-14T04:40:51.781831+0000","last_became_active":"2024-09-14T04:07:58.515594+0000","last_became_peered":"2024-09-14T04:07:58.515594+0000","last_unstale":"2024-09-14T04:40:51.781831+0000","last_undegraded":"2024-09-14T04:40:51.781831+0000","last_fullsized":"2024-09-14T04:40:51.781831+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:45.966798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":1951,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.845692+0000","last_change":"2024-09-14T04:07:58.509833+0000","last_active":"2024-09-14T04:41:00.845692+0000","last_peered":"2024-09-14T04:41:00.845692+0000","last_clean":"2024-09-14T04:41:00.845692+0000","last_became_active":"2024-09-14T04:07:58.509198+0000","last_became_peered":"2024-09-14T04:07:58.509198+0000","last_unstale":"2024-09-14T04:41:00.845692+0000","last_undegraded":"2024-09-14T04:41:00.845692+0000","last_fullsized":"2024-09-14T04:41:00.845692+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:24:40.193090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1954,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.829163+0000","last_change":"2024-09-14T04:06:46.925323+0000","last_active":"2024-09-14T04:41:00.829163+0000","last_peered":"2024-09-14T04:41:00.829163+0000","last_clean":"2024-09-14T04:41:00.829163+0000","last_became_active":"2024-09-14T04:06:46.925043+0000","last_became_peered":"2024-09-14T04:06:46.925043+0000","last_unstale":"2024-09-14T04:41:00.829163+0000","last_undegraded":"2024-09-14T04:41:00.829163+0000","last_fullsized":"2024-09-14T04:41:00.829163+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:39.895453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1306,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.780659+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T04:40:51.780659+0000","last_peered":"2024-09-14T04:40:51.780659+0000","last_clean":"2024-09-14T04:40:51.780659+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T04:40:51.780659+0000","last_undegraded":"2024-09-14T04:40:51.780659+0000","last_fullsized":"2024-09-14T04:40:51.780659+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1305,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:52.881990+0000","last_change":"2024-09-14T04:06:46.869868+0000","last_active":"2024-09-14T04:40:52.881990+0000","last_peered":"2024-09-14T04:40:52.881990+0000","last_clean":"2024-09-14T04:40:52.881990+0000","last_became_active":"2024-09-14T04:06:46.869126+0000","last_became_peered":"2024-09-14T04:06:46.869126+0000","last_unstale":"2024-09-14T04:40:52.881990+0000","last_undegraded":"2024-09-14T04:40:52.881990+0000","last_fullsized":"2024-09-14T04:40:52.881990+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:41.150138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1289,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.801561+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T04:40:51.801561+0000","last_peered":"2024-09-14T04:40:51.801561+0000","last_clean":"2024-09-14T04:40:51.801561+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T04:40:51.801561+0000","last_undegraded":"2024-09-14T04:40:51.801561+0000","last_fullsized":"2024-09-14T04:40:51.801561+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1324,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.781593+0000","last_change":"2024-09-14T04:07:58.508202+0000","last_active":"2024-09-14T04:40:51.781593+0000","last_peered":"2024-09-14T04:40:51.781593+0000","last_clean":"2024-09-14T04:40:51.781593+0000","last_became_active":"2024-09-14T04:07:58.507929+0000","last_became_peered":"2024-09-14T04:07:58.507929+0000","last_unstale":"2024-09-14T04:40:51.781593+0000","last_undegraded":"2024-09-14T04:40:51.781593+0000","last_fullsized":"2024-09-14T04:40:51.781593+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:17:58.160600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1305,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.801578+0000","last_change":"2024-09-14T04:06:46.858890+0000","last_active":"2024-09-14T04:40:51.801578+0000","last_peered":"2024-09-14T04:40:51.801578+0000","last_clean":"2024-09-14T04:40:51.801578+0000","last_became_active":"2024-09-14T04:06:46.858569+0000","last_became_peered":"2024-09-14T04:06:46.858569+0000","last_unstale":"2024-09-14T04:40:51.801578+0000","last_undegraded":"2024-09-14T04:40:51.801578+0000","last_fullsized":"2024-09-14T04:40:51.801578+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:25:46.400115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1291,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.780263+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T04:40:51.780263+0000","last_peered":"2024-09-14T04:40:51.780263+0000","last_clean":"2024-09-14T04:40:51.780263+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T04:40:51.780263+0000","last_undegraded":"2024-09-14T04:40:51.780263+0000","last_fullsized":"2024-09-14T04:40:51.780263+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1304,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.801769+0000","last_change":"2024-09-14T04:05:54.419435+0000","last_active":"2024-09-14T04:40:51.801769+0000","last_peered":"2024-09-14T04:40:51.801769+0000","last_clean":"2024-09-14T04:40:51.801769+0000","last_became_active":"2024-09-14T04:05:54.419094+0000","last_became_peered":"2024-09-14T04:05:54.419094+0000","last_unstale":"2024-09-14T04:40:51.801769+0000","last_undegraded":"2024-09-14T04:40:51.801769+0000","last_fullsized":"2024-09-14T04:40:51.801769+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:00.162629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1312,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.775708+0000","last_change":"2024-09-14T04:05:54.418813+0000","last_active":"2024-09-14T04:40:51.775708+0000","last_peered":"2024-09-14T04:40:51.775708+0000","last_clean":"2024-09-14T04:40:51.775708+0000","last_became_active":"2024-09-14T04:05:54.417584+0000","last_became_peered":"2024-09-14T04:05:54.417584+0000","last_unstale":"2024-09-14T04:40:51.775708+0000","last_undegraded":"2024-09-14T04:40:51.775708+0000","last_fullsized":"2024-09-14T04:40:51.775708+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:13:16.209456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1931,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.846640+0000","last_change":"2024-09-14T04:07:58.565442+0000","last_active":"2024-09-14T04:41:00.846640+0000","last_peered":"2024-09-14T04:41:00.846640+0000","last_clean":"2024-09-14T04:41:00.846640+0000","last_became_active":"2024-09-14T04:07:58.552289+0000","last_became_peered":"2024-09-14T04:07:58.552289+0000","last_unstale":"2024-09-14T04:41:00.846640+0000","last_undegraded":"2024-09-14T04:41:00.846640+0000","last_fullsized":"2024-09-14T04:41:00.846640+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:48:02.364960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1287,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.801367+0000","last_change":"2024-09-14T03:53:05.244003+0000","last_active":"2024-09-14T04:40:51.801367+0000","last_peered":"2024-09-14T04:40:51.801367+0000","last_clean":"2024-09-14T04:40:51.801367+0000","last_became_active":"2024-09-14T03:53:05.240743+0000","last_became_peered":"2024-09-14T03:53:05.240743+0000","last_unstale":"2024-09-14T04:40:51.801367+0000","last_undegraded":"2024-09-14T04:40:51.801367+0000","last_fullsized":"2024-09-14T04:40:51.801367+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:30.000189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1952,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.829380+0000","last_change":"2024-09-14T04:07:58.560880+0000","last_active":"2024-09-14T04:41:00.829380+0000","last_peered":"2024-09-14T04:41:00.829380+0000","last_clean":"2024-09-14T04:41:00.829380+0000","last_became_active":"2024-09-14T04:07:58.560132+0000","last_became_peered":"2024-09-14T04:07:58.560132+0000","last_unstale":"2024-09-14T04:41:00.829380+0000","last_undegraded":"2024-09-14T04:41:00.829380+0000","last_fullsized":"2024-09-14T04:41:00.829380+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:46:00.287050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1952,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.831508+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T04:41:00.831508+0000","last_peered":"2024-09-14T04:41:00.831508+0000","last_clean":"2024-09-14T04:41:00.831508+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T04:41:00.831508+0000","last_undegraded":"2024-09-14T04:41:00.831508+0000","last_fullsized":"2024-09-14T04:41:00.831508+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1303,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.780851+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T04:40:51.780851+0000","last_peered":"2024-09-14T04:40:51.780851+0000","last_clean":"2024-09-14T04:40:51.780851+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T04:40:51.780851+0000","last_undegraded":"2024-09-14T04:40:51.780851+0000","last_fullsized":"2024-09-14T04:40:51.780851+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1304,"reported_epoch":1337,"state":"active+clean","last_fresh":"2024-09-14T04:40:51.801638+0000","last_change":"2024-09-14T04:07:21.718847+0000","last_active":"2024-09-14T04:40:51.801638+0000","last_peered":"2024-09-14T04:40:51.801638+0000","last_clean":"2024-09-14T04:40:51.801638+0000","last_became_active":"2024-09-14T04:07:21.718504+0000","last_became_peered":"2024-09-14T04:07:21.718504+0000","last_unstale":"2024-09-14T04:40:51.801638+0000","last_undegraded":"2024-09-14T04:40:51.801638+0000","last_fullsized":"2024-09-14T04:40:51.801638+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:37:34.405765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1948,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.845940+0000","last_change":"2024-09-14T04:07:58.537946+0000","last_active":"2024-09-14T04:41:00.845940+0000","last_peered":"2024-09-14T04:41:00.845940+0000","last_clean":"2024-09-14T04:41:00.845940+0000","last_became_active":"2024-09-14T04:07:58.536034+0000","last_became_peered":"2024-09-14T04:07:58.536034+0000","last_unstale":"2024-09-14T04:41:00.845940+0000","last_undegraded":"2024-09-14T04:41:00.845940+0000","last_fullsized":"2024-09-14T04:41:00.845940+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:17:18.200470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":53,"num_pg":16,"stat_sum":{"num_bytes":5155979288,"num_objects":78675,"num_object_clones":0,"num_object_copies":236025,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":78675,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":181507,"num_write_kb":8325761,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15448092672,"data_stored":15448080456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137307,"ondisk_log_size":137307,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201881,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":447512,"kb_used_data":326220,"kb_used_omap":1,"kb_used_meta":121278,"kb_avail":93297640,"statfs":{"total":95995035648,"available":95536783360,"internally_reserved":0,"allocated":334049280,"data_stored":330115200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":124189131},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463517,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1446032,"kb_used_data":1270596,"kb_used_omap":1,"kb_used_meta":175422,"kb_avail":92299120,"statfs":{"total":95995035648,"available":94514298880,"internally_reserved":0,"allocated":1301090304,"data_stored":1297164289,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":179632583},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,4,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725154,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2489796,"kb_used_data":2252028,"kb_used_omap":1,"kb_used_meta":237758,"kb_avail":91255356,"statfs":{"total":95995035648,"available":93445484544,"internally_reserved":0,"allocated":2306076672,"data_stored":2302154155,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":243464646},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986792,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3772044,"kb_used_data":3484228,"kb_used_omap":1,"kb_used_meta":287806,"kb_avail":89973108,"statfs":{"total":95995035648,"available":92132462592,"internally_reserved":0,"allocated":3567849472,"data_stored":3563922970,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":294713798},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,3,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772074895,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1749032,"kb_used_data":1630880,"kb_used_omap":71,"kb_used_meta":118072,"kb_avail":91996120,"statfs":{"total":95995035648,"available":94204026880,"internally_reserved":0,"allocated":1670021120,"data_stored":1666117945,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":120906678},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707303831,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1376704,"kb_used_data":1258552,"kb_used_omap":68,"kb_used_meta":118075,"kb_avail":92368448,"statfs":{"total":95995035648,"available":94585290752,"internally_reserved":0,"allocated":1288757248,"data_stored":1284838784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":120909008},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872728990,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3331260,"kb_used_data":3096500,"kb_used_omap":67,"kb_used_meta":234620,"kb_avail":90413892,"statfs":{"total":95995035648,"available":92583825408,"internally_reserved":0,"allocated":3170816000,"data_stored":3166901035,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":240251237},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056040,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2085888,"kb_used_data":1927736,"kb_used_omap":74,"kb_used_meta":158069,"kb_avail":91659264,"statfs":{"total":95995035648,"available":93859086336,"internally_reserved":0,"allocated":1974001664,"data_stored":1970083708,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":161863349},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1944846336,"data_stored":1944846336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3154841600,"data_stored":3154837528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1269370880,"data_stored":1269366808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1644232704,"data_stored":1644232704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3552706560,"data_stored":3552706560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2272792576,"data_stored":2272788504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1286012928,"data_stored":1286012928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":323289088,"data_stored":323289088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:41:08.657 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-14T04:41:08.658 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:41:08.658 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2024-09-14T04:41:08.658 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:41:08.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:08.684 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:08.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:08.785 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:08.842 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:08.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:08.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:08.873 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:41:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:41:08] "GET /metrics HTTP/1.1" 200 36737 "" "Prometheus/2.51.0" 2024-09-14T04:41:08.944 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:08.976 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:08.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:08.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:09.078 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:41:09.110 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:09.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:09.111 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:41:09.212 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:09.269 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:09.279 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:41:09.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:09.282 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:09.360 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:41:09.383 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:09.406 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:09 smithi129 ceph-mon[101316]: pgmap v2253: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 873 op/s 2024-09-14T04:41:09.408 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:09.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:09.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:09.425 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:09 smithi039 ceph-mon[106470]: pgmap v2253: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 873 op/s 2024-09-14T04:41:09.425 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:09 smithi039 ceph-mon[111235]: pgmap v2253: 89 pgs: 89 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 873 op/s 2024-09-14T04:41:09.510 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:09.563 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:09.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:09.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:09.664 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:09.690 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:09.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:09.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:09.792 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:09.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:09.828 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:09.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:09.929 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:09.967 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:09.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:09.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:10.069 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:41:10.109 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:10.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:10.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:41:10.138 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:41:10.138 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:41:10.210 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:10.238 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:10.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:10.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:10.337 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is set 2024-09-14T04:41:10.339 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:10.364 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:10.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:10.365 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:10.466 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:10.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:10.514 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:10.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:10.615 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:10.646 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:10.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:10.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:10.748 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:10.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:10 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2332440757' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:41:10.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:10 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:41:10.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:10.788 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:10.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:10.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:10 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2332440757' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:41:10.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:10 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:41:10.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:10 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2332440757' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:41:10.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:10 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:41:10.841 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-14T04:41:10.889 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:10.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:10.945 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:10.945 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:11.046 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:11.079 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:11.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:11.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:11.181 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:41:11.215 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:11.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:11.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:41:11.317 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:11.352 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:11.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:11.353 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:11.454 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:11.485 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:11.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:11.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:11.587 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:11.615 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:11.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:11.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:11.717 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:41:11.725 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:11 smithi039 ceph-mon[106470]: pgmap v2254: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 881 op/s 2024-09-14T04:41:11.725 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:11 smithi039 ceph-mon[106470]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:41:11.725 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:11 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:41:11.725 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:11 smithi039 ceph-mon[106470]: osdmap e1339: 8 total, 8 up, 8 in 2024-09-14T04:41:11.726 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:11 smithi039 ceph-mon[111235]: pgmap v2254: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 881 op/s 2024-09-14T04:41:11.726 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:11 smithi039 ceph-mon[111235]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:41:11.726 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:11 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:41:11.726 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:11 smithi039 ceph-mon[111235]: osdmap e1339: 8 total, 8 up, 8 in 2024-09-14T04:41:11.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:11.762 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:11.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:41:11.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:11 smithi129 ceph-mon[101316]: pgmap v2254: 89 pgs: 89 active+clean; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 881 op/s 2024-09-14T04:41:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:11 smithi129 ceph-mon[101316]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:41:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:11 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:41:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:11 smithi129 ceph-mon[101316]: osdmap e1339: 8 total, 8 up, 8 in 2024-09-14T04:41:11.863 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:11.899 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:11.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:11.900 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:12.001 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:12.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:12.040 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:12.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:12.141 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:12.176 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:12.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:12.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:12.277 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:41:12.312 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:12.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:12.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:41:12.414 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:12.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:12.449 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:12.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:12.550 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:12.586 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:12.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:12.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:12.687 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:12.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:12.722 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:12.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:12.823 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:12.860 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:12.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:12.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:12.962 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:41:12.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:12.993 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:12.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:41:13.094 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:13.153 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:13.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:13.154 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:13.254 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:13.292 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:13.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:13.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:13.394 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:13.430 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:13.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:13.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:13.531 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:13.573 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:13.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:13.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:13.579 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:13 smithi039 ceph-mon[106470]: pgmap v2256: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 872 op/s 2024-09-14T04:41:13.579 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:13 smithi039 ceph-mon[111235]: pgmap v2256: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 872 op/s 2024-09-14T04:41:13.676 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:13.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:13.721 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:13.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:13.757 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:41:13.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:13 smithi129 ceph-mon[101316]: pgmap v2256: 89 pgs: 89 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 872 op/s 2024-09-14T04:41:13.800 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:41:13.801 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:41:13.822 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:13.829 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:41:13.829 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:41:13.859 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:13.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:13.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:13.864 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:41:13.864 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:41:13.906 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:41:13.906 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:41:13.940 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:41:13.941 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:41:13.961 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:13.975 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:41:13.975 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:41:13.989 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:13.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:13.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:14.020 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:41:14.020 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:41:14.061 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:41:14.061 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:41:14.091 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:14.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:14.119 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:14.119 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:14.220 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:41:14.253 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:14.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:14.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:41:14.341 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:41:14.354 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:14.401 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:14.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:14.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:14.485 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:41:14.486 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T04:41:14.504 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:41:14.532 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:14.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:14.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:41:14.634 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:14.671 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:14.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:14.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:14.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:14 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2667440507' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:41:14.773 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:14.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:14 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2667440507' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:41:14.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:14 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2667440507' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:41:14.809 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:14.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:14.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:14.911 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:14.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:14.953 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:14.953 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:15.054 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:15.080 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:15.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:15.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:15.181 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:15.210 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:15.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:15.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:15.312 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:15.350 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:15.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:15.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:15.452 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:15.491 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:15.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:15.493 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:15.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:15 smithi039 ceph-mon[106470]: pgmap v2257: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 867 op/s 2024-09-14T04:41:15.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:15 smithi039 ceph-mon[111235]: pgmap v2257: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 867 op/s 2024-09-14T04:41:15.593 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:15.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:15.629 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:15.629 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:15.635 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":2257,"stamp":"2024-09-14T04:41:13.553516+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4659877885,"num_objects":71122,"num_object_clones":0,"num_object_copies":213366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":71122,"num_whiteouts":0,"num_read":393,"num_read_kb":835,"num_write":189365,"num_write_kb":8328210,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137894,"ondisk_log_size":137894,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15323276,"kb_used_data":13832708,"kb_used_omap":288,"kb_used_meta":1489695,"kb_avail":734637940,"statfs":{"total":767960285184,"available":752269250560,"internally_reserved":0,"allocated":14164692992,"data_stored":14133275789,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":294978,"internal_metadata":1525448638},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-568918016,"num_objects":-8681,"num_object_clones":0,"num_object_copies":-26043,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8681,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8681,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004560"},"pg_stats":[{"pgid":"53.f","version":"1339'11744","reported_seq":11777,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:12.842148+0000","last_change":"2024-09-14T04:38:26.895527+0000","last_active":"2024-09-14T04:41:12.842148+0000","last_peered":"2024-09-14T04:41:12.842148+0000","last_clean":"2024-09-14T04:41:12.842148+0000","last_became_active":"2024-09-14T04:38:26.895218+0000","last_became_peered":"2024-09-14T04:38:26.895218+0000","last_unstale":"2024-09-14T04:41:12.842148+0000","last_undegraded":"2024-09-14T04:41:12.842148+0000","last_fullsized":"2024-09-14T04:41:12.842148+0000","mapping_epoch":1315,"log_start":"1319'2000","ondisk_log_start":"1319'2000","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":9744,"log_dups_size":0,"ondisk_log_size":9744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:07:35.310515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":281804800,"num_objects":4300,"num_object_clones":0,"num_object_copies":12900,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4300,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11744,"num_write_kb":513408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"53.e","version":"1339'11696","reported_seq":11730,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:12.833444+0000","last_change":"2024-09-14T04:38:26.894036+0000","last_active":"2024-09-14T04:41:12.833444+0000","last_peered":"2024-09-14T04:41:12.833444+0000","last_clean":"2024-09-14T04:41:12.833444+0000","last_became_active":"2024-09-14T04:38:26.893654+0000","last_became_peered":"2024-09-14T04:38:26.893654+0000","last_unstale":"2024-09-14T04:41:12.833444+0000","last_undegraded":"2024-09-14T04:41:12.833444+0000","last_fullsized":"2024-09-14T04:41:12.833444+0000","mapping_epoch":1315,"log_start":"1319'2000","ondisk_log_start":"1319'2000","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":9696,"log_dups_size":0,"ondisk_log_size":9696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:39:17.902111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286130176,"num_objects":4366,"num_object_clones":0,"num_object_copies":13098,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4366,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11696,"num_write_kb":513984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"53.d","version":"1338'11620","reported_seq":11680,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:09.684003+0000","last_change":"2024-09-14T04:38:27.529546+0000","last_active":"2024-09-14T04:41:09.684003+0000","last_peered":"2024-09-14T04:41:09.684003+0000","last_clean":"2024-09-14T04:41:09.684003+0000","last_became_active":"2024-09-14T04:38:27.529237+0000","last_became_peered":"2024-09-14T04:38:27.529237+0000","last_unstale":"2024-09-14T04:41:09.684003+0000","last_undegraded":"2024-09-14T04:41:09.684003+0000","last_fullsized":"2024-09-14T04:41:09.684003+0000","mapping_epoch":1315,"log_start":"1322'4400","ondisk_log_start":"1322'4400","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":7220,"log_dups_size":0,"ondisk_log_size":7220,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:37:40.618858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300089368,"num_objects":4580,"num_object_clones":0,"num_object_copies":13740,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4580,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11620,"num_write_kb":518337,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"53.c","version":"1339'12180","reported_seq":12214,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:12.989272+0000","last_change":"2024-09-14T04:38:26.895807+0000","last_active":"2024-09-14T04:41:12.989272+0000","last_peered":"2024-09-14T04:41:12.989272+0000","last_clean":"2024-09-14T04:41:12.989272+0000","last_became_active":"2024-09-14T04:38:26.895506+0000","last_became_peered":"2024-09-14T04:38:26.895506+0000","last_unstale":"2024-09-14T04:41:12.989272+0000","last_undegraded":"2024-09-14T04:41:12.989272+0000","last_fullsized":"2024-09-14T04:41:12.989272+0000","mapping_epoch":1315,"log_start":"1319'2100","ondisk_log_start":"1319'2100","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":10080,"log_dups_size":0,"ondisk_log_size":10080,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:07:45.042093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285212672,"num_objects":4352,"num_object_clones":0,"num_object_copies":13056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12180,"num_write_kb":529024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"53.b","version":"1339'11718","reported_seq":11752,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:13.008036+0000","last_change":"2024-09-14T04:38:26.893051+0000","last_active":"2024-09-14T04:41:13.008036+0000","last_peered":"2024-09-14T04:41:13.008036+0000","last_clean":"2024-09-14T04:41:13.008036+0000","last_became_active":"2024-09-14T04:38:26.892762+0000","last_became_peered":"2024-09-14T04:38:26.892762+0000","last_unstale":"2024-09-14T04:41:13.008036+0000","last_undegraded":"2024-09-14T04:41:13.008036+0000","last_fullsized":"2024-09-14T04:41:13.008036+0000","mapping_epoch":1315,"log_start":"1319'1700","ondisk_log_start":"1319'1700","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":10018,"log_dups_size":0,"ondisk_log_size":10018,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:23:11.179960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":285212672,"num_objects":4352,"num_object_clones":0,"num_object_copies":13056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11718,"num_write_kb":514240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"53.a","version":"1339'11905","reported_seq":11966,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:11.602237+0000","last_change":"2024-09-14T04:38:26.895294+0000","last_active":"2024-09-14T04:41:11.602237+0000","last_peered":"2024-09-14T04:41:11.602237+0000","last_clean":"2024-09-14T04:41:11.602237+0000","last_became_active":"2024-09-14T04:38:26.894942+0000","last_became_peered":"2024-09-14T04:38:26.894942+0000","last_unstale":"2024-09-14T04:41:11.602237+0000","last_undegraded":"2024-09-14T04:41:11.602237+0000","last_fullsized":"2024-09-14T04:41:11.602237+0000","mapping_epoch":1315,"log_start":"1320'3500","ondisk_log_start":"1320'3500","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":8405,"log_dups_size":0,"ondisk_log_size":8405,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:17:54.687919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299040768,"num_objects":4563,"num_object_clones":0,"num_object_copies":13689,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4563,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11905,"num_write_kb":526976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,5],"acting":[3,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"53.9","version":"1339'11868","reported_seq":11929,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:12.630108+0000","last_change":"2024-09-14T04:38:26.899027+0000","last_active":"2024-09-14T04:41:12.630108+0000","last_peered":"2024-09-14T04:41:12.630108+0000","last_clean":"2024-09-14T04:41:12.630108+0000","last_became_active":"2024-09-14T04:38:26.898732+0000","last_became_peered":"2024-09-14T04:38:26.898732+0000","last_unstale":"2024-09-14T04:41:12.630108+0000","last_undegraded":"2024-09-14T04:41:12.630108+0000","last_fullsized":"2024-09-14T04:41:12.630108+0000","mapping_epoch":1315,"log_start":"1319'2400","ondisk_log_start":"1319'2400","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":9468,"log_dups_size":0,"ondisk_log_size":9468,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:58:20.477950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292552704,"num_objects":4464,"num_object_clones":0,"num_object_copies":13392,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4464,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11868,"num_write_kb":522624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"53.8","version":"1339'11807","reported_seq":11841,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:13.048129+0000","last_change":"2024-09-14T04:38:26.897778+0000","last_active":"2024-09-14T04:41:13.048129+0000","last_peered":"2024-09-14T04:41:13.048129+0000","last_clean":"2024-09-14T04:41:13.048129+0000","last_became_active":"2024-09-14T04:38:26.897509+0000","last_became_peered":"2024-09-14T04:38:26.897509+0000","last_unstale":"2024-09-14T04:41:13.048129+0000","last_undegraded":"2024-09-14T04:41:13.048129+0000","last_fullsized":"2024-09-14T04:41:13.048129+0000","mapping_epoch":1315,"log_start":"1322'4900","ondisk_log_start":"1322'4900","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":6907,"log_dups_size":0,"ondisk_log_size":6907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:51:52.846967+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283181056,"num_objects":4321,"num_object_clones":0,"num_object_copies":12963,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4321,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11807,"num_write_kb":516096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"53.7","version":"1338'11847","reported_seq":11906,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:09.588902+0000","last_change":"2024-09-14T04:38:26.897393+0000","last_active":"2024-09-14T04:41:09.588902+0000","last_peered":"2024-09-14T04:41:09.588902+0000","last_clean":"2024-09-14T04:41:09.588902+0000","last_became_active":"2024-09-14T04:38:26.897127+0000","last_became_peered":"2024-09-14T04:38:26.897127+0000","last_unstale":"2024-09-14T04:41:09.588902+0000","last_undegraded":"2024-09-14T04:41:09.588902+0000","last_fullsized":"2024-09-14T04:41:09.588902+0000","mapping_epoch":1315,"log_start":"1322'4700","ondisk_log_start":"1322'4700","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":7147,"log_dups_size":0,"ondisk_log_size":7147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:15:57.417163+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300613632,"num_objects":4587,"num_object_clones":0,"num_object_copies":13761,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4587,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11847,"num_write_kb":525888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"53.6","version":"1339'12011","reported_seq":12072,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:11.614896+0000","last_change":"2024-09-14T04:38:27.529579+0000","last_active":"2024-09-14T04:41:11.614896+0000","last_peered":"2024-09-14T04:41:11.614896+0000","last_clean":"2024-09-14T04:41:11.614896+0000","last_became_active":"2024-09-14T04:38:27.529245+0000","last_became_peered":"2024-09-14T04:38:27.529245+0000","last_unstale":"2024-09-14T04:41:11.614896+0000","last_undegraded":"2024-09-14T04:41:11.614896+0000","last_fullsized":"2024-09-14T04:41:11.614896+0000","mapping_epoch":1315,"log_start":"1320'3600","ondisk_log_start":"1320'3600","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":8411,"log_dups_size":0,"ondisk_log_size":8411,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:11.351641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293797888,"num_objects":4483,"num_object_clones":0,"num_object_copies":13449,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4483,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12011,"num_write_kb":527808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"53.5","version":"1339'11837","reported_seq":11898,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:12.615704+0000","last_change":"2024-09-14T04:38:26.897570+0000","last_active":"2024-09-14T04:41:12.615704+0000","last_peered":"2024-09-14T04:41:12.615704+0000","last_clean":"2024-09-14T04:41:12.615704+0000","last_became_active":"2024-09-14T04:38:26.897277+0000","last_became_peered":"2024-09-14T04:38:26.897277+0000","last_unstale":"2024-09-14T04:41:12.615704+0000","last_undegraded":"2024-09-14T04:41:12.615704+0000","last_fullsized":"2024-09-14T04:41:12.615704+0000","mapping_epoch":1315,"log_start":"1319'2400","ondisk_log_start":"1319'2400","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":9437,"log_dups_size":0,"ondisk_log_size":9437,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:06:40.318691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284491776,"num_objects":4341,"num_object_clones":0,"num_object_copies":13023,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4341,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11837,"num_write_kb":517696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"53.4","version":"1338'11507","reported_seq":11566,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:09.658468+0000","last_change":"2024-09-14T04:38:27.529509+0000","last_active":"2024-09-14T04:41:09.658468+0000","last_peered":"2024-09-14T04:41:09.658468+0000","last_clean":"2024-09-14T04:41:09.658468+0000","last_became_active":"2024-09-14T04:38:27.529180+0000","last_became_peered":"2024-09-14T04:38:27.529180+0000","last_unstale":"2024-09-14T04:41:09.658468+0000","last_undegraded":"2024-09-14T04:41:09.658468+0000","last_fullsized":"2024-09-14T04:41:09.658468+0000","mapping_epoch":1315,"log_start":"1322'4300","ondisk_log_start":"1322'4300","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":7207,"log_dups_size":0,"ondisk_log_size":7207,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:04:21.631485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298254336,"num_objects":4551,"num_object_clones":0,"num_object_copies":13653,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4551,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11507,"num_write_kb":513856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"53.3","version":"1339'11866","reported_seq":11898,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:12.873218+0000","last_change":"2024-09-14T04:38:26.894968+0000","last_active":"2024-09-14T04:41:12.873218+0000","last_peered":"2024-09-14T04:41:12.873218+0000","last_clean":"2024-09-14T04:41:12.873218+0000","last_became_active":"2024-09-14T04:38:26.894621+0000","last_became_peered":"2024-09-14T04:38:26.894621+0000","last_unstale":"2024-09-14T04:41:12.873218+0000","last_undegraded":"2024-09-14T04:41:12.873218+0000","last_fullsized":"2024-09-14T04:41:12.873218+0000","mapping_epoch":1315,"log_start":"1319'2100","ondisk_log_start":"1319'2100","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":9766,"log_dups_size":0,"ondisk_log_size":9766,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:15:43.660519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287178752,"num_objects":4382,"num_object_clones":0,"num_object_copies":13146,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4382,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11866,"num_write_kb":519936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"53.2","version":"1339'11854","reported_seq":11888,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:13.007402+0000","last_change":"2024-09-14T04:38:26.897057+0000","last_active":"2024-09-14T04:41:13.007402+0000","last_peered":"2024-09-14T04:41:13.007402+0000","last_clean":"2024-09-14T04:41:13.007402+0000","last_became_active":"2024-09-14T04:38:26.896677+0000","last_became_peered":"2024-09-14T04:38:26.896677+0000","last_unstale":"2024-09-14T04:41:13.007402+0000","last_undegraded":"2024-09-14T04:41:13.007402+0000","last_fullsized":"2024-09-14T04:41:13.007402+0000","mapping_epoch":1315,"log_start":"1319'1800","ondisk_log_start":"1319'1800","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":10054,"log_dups_size":0,"ondisk_log_size":10054,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:15.244766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290062336,"num_objects":4426,"num_object_clones":0,"num_object_copies":13278,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4426,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11854,"num_write_kb":520960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"53.1","version":"1338'11823","reported_seq":11882,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:09.661072+0000","last_change":"2024-09-14T04:38:26.897425+0000","last_active":"2024-09-14T04:41:09.661072+0000","last_peered":"2024-09-14T04:41:09.661072+0000","last_clean":"2024-09-14T04:41:09.661072+0000","last_became_active":"2024-09-14T04:38:26.897127+0000","last_became_peered":"2024-09-14T04:38:26.897127+0000","last_unstale":"2024-09-14T04:41:09.661072+0000","last_undegraded":"2024-09-14T04:41:09.661072+0000","last_fullsized":"2024-09-14T04:41:09.661072+0000","mapping_epoch":1315,"log_start":"1322'4600","ondisk_log_start":"1322'4600","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":7223,"log_dups_size":0,"ondisk_log_size":7223,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:49:46.941647+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":302055424,"num_objects":4609,"num_object_clones":0,"num_object_copies":13827,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4609,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11823,"num_write_kb":525824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"53.0","version":"1339'11803","reported_seq":11837,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:13.048537+0000","last_change":"2024-09-14T04:38:27.529766+0000","last_active":"2024-09-14T04:41:13.048537+0000","last_peered":"2024-09-14T04:41:13.048537+0000","last_clean":"2024-09-14T04:41:13.048537+0000","last_became_active":"2024-09-14T04:38:27.529414+0000","last_became_peered":"2024-09-14T04:38:27.529414+0000","last_unstale":"2024-09-14T04:41:13.048537+0000","last_undegraded":"2024-09-14T04:41:13.048537+0000","last_fullsized":"2024-09-14T04:41:13.048537+0000","mapping_epoch":1315,"log_start":"1322'4900","ondisk_log_start":"1322'4900","created":1315,"last_epoch_clean":1316,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:38:25.872003+0000","last_clean_scrub_stamp":"2024-09-14T04:38:25.872003+0000","objects_scrubbed":0,"log_size":6903,"log_dups_size":0,"ondisk_log_size":6903,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:30:08.912265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289603584,"num_objects":4419,"num_object_clones":0,"num_object_copies":13257,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4419,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11803,"num_write_kb":519104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,2],"acting":[4,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":1963,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:10.331155+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T04:41:10.331155+0000","last_peered":"2024-09-14T04:41:10.331155+0000","last_clean":"2024-09-14T04:41:10.331155+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T04:41:10.331155+0000","last_undegraded":"2024-09-14T04:41:10.331155+0000","last_fullsized":"2024-09-14T04:41:10.331155+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":1301,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.826924+0000","last_change":"2024-09-14T03:53:45.836449+0000","last_active":"2024-09-14T04:41:00.826924+0000","last_peered":"2024-09-14T04:41:00.826924+0000","last_clean":"2024-09-14T04:41:00.826924+0000","last_became_active":"2024-09-14T03:53:45.836108+0000","last_became_peered":"2024-09-14T03:53:45.836108+0000","last_unstale":"2024-09-14T04:41:00.826924+0000","last_undegraded":"2024-09-14T04:41:00.826924+0000","last_fullsized":"2024-09-14T04:41:00.826924+0000","mapping_epoch":73,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":74,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:52:13.095225+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":1953,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:10.338709+0000","last_change":"2024-09-14T04:07:58.504829+0000","last_active":"2024-09-14T04:41:10.338709+0000","last_peered":"2024-09-14T04:41:10.338709+0000","last_clean":"2024-09-14T04:41:10.338709+0000","last_became_active":"2024-09-14T04:07:58.502623+0000","last_became_peered":"2024-09-14T04:07:58.502623+0000","last_unstale":"2024-09-14T04:41:10.338709+0000","last_undegraded":"2024-09-14T04:41:10.338709+0000","last_fullsized":"2024-09-14T04:41:10.338709+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:59.091993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1964,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:10.344295+0000","last_change":"2024-09-14T04:07:58.507125+0000","last_active":"2024-09-14T04:41:10.344295+0000","last_peered":"2024-09-14T04:41:10.344295+0000","last_clean":"2024-09-14T04:41:10.344295+0000","last_became_active":"2024-09-14T04:07:58.505701+0000","last_became_peered":"2024-09-14T04:07:58.505701+0000","last_unstale":"2024-09-14T04:41:10.344295+0000","last_undegraded":"2024-09-14T04:41:10.344295+0000","last_fullsized":"2024-09-14T04:41:10.344295+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:25:00.847961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1292,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.823129+0000","last_change":"2024-09-14T03:53:05.234444+0000","last_active":"2024-09-14T04:41:00.823129+0000","last_peered":"2024-09-14T04:41:00.823129+0000","last_clean":"2024-09-14T04:41:00.823129+0000","last_became_active":"2024-09-14T03:53:05.234115+0000","last_became_peered":"2024-09-14T03:53:05.234115+0000","last_unstale":"2024-09-14T04:41:00.823129+0000","last_undegraded":"2024-09-14T04:41:00.823129+0000","last_fullsized":"2024-09-14T04:41:00.823129+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:01:23.153937+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":1322,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.830425+0000","last_change":"2024-09-14T04:07:58.547135+0000","last_active":"2024-09-14T04:41:00.830425+0000","last_peered":"2024-09-14T04:41:00.830425+0000","last_clean":"2024-09-14T04:41:00.830425+0000","last_became_active":"2024-09-14T04:07:58.546751+0000","last_became_peered":"2024-09-14T04:07:58.546751+0000","last_unstale":"2024-09-14T04:41:00.830425+0000","last_undegraded":"2024-09-14T04:41:00.830425+0000","last_fullsized":"2024-09-14T04:41:00.830425+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:01.092698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1307,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.823702+0000","last_change":"2024-09-14T04:06:46.875795+0000","last_active":"2024-09-14T04:41:00.823702+0000","last_peered":"2024-09-14T04:41:00.823702+0000","last_clean":"2024-09-14T04:41:00.823702+0000","last_became_active":"2024-09-14T04:06:46.875395+0000","last_became_peered":"2024-09-14T04:06:46.875395+0000","last_unstale":"2024-09-14T04:41:00.823702+0000","last_undegraded":"2024-09-14T04:41:00.823702+0000","last_fullsized":"2024-09-14T04:41:00.823702+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:51:10.713063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":1322,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.827270+0000","last_change":"2024-09-14T04:07:58.514349+0000","last_active":"2024-09-14T04:41:00.827270+0000","last_peered":"2024-09-14T04:41:00.827270+0000","last_clean":"2024-09-14T04:41:00.827270+0000","last_became_active":"2024-09-14T04:07:58.513485+0000","last_became_peered":"2024-09-14T04:07:58.513485+0000","last_unstale":"2024-09-14T04:41:00.827270+0000","last_undegraded":"2024-09-14T04:41:00.827270+0000","last_fullsized":"2024-09-14T04:41:00.827270+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:33:09.031813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":1946,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:10.331179+0000","last_change":"2024-09-14T04:05:54.428057+0000","last_active":"2024-09-14T04:41:10.331179+0000","last_peered":"2024-09-14T04:41:10.331179+0000","last_clean":"2024-09-14T04:41:10.331179+0000","last_became_active":"2024-09-14T04:05:54.414895+0000","last_became_peered":"2024-09-14T04:05:54.414895+0000","last_unstale":"2024-09-14T04:41:10.331179+0000","last_undegraded":"2024-09-14T04:41:10.331179+0000","last_fullsized":"2024-09-14T04:41:10.331179+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:45:15.752675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":1958,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.844080+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T04:41:00.844080+0000","last_peered":"2024-09-14T04:41:00.844080+0000","last_clean":"2024-09-14T04:41:00.844080+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T04:41:00.844080+0000","last_undegraded":"2024-09-14T04:41:00.844080+0000","last_fullsized":"2024-09-14T04:41:00.844080+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":1324,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.830563+0000","last_change":"2024-09-14T04:07:21.717071+0000","last_active":"2024-09-14T04:41:00.830563+0000","last_peered":"2024-09-14T04:41:00.830563+0000","last_clean":"2024-09-14T04:41:00.830563+0000","last_became_active":"2024-09-14T04:07:21.716215+0000","last_became_peered":"2024-09-14T04:07:21.716215+0000","last_unstale":"2024-09-14T04:41:00.830563+0000","last_undegraded":"2024-09-14T04:41:00.830563+0000","last_fullsized":"2024-09-14T04:41:00.830563+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:48:47.175490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":1933,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:10.338940+0000","last_change":"2024-09-14T04:07:58.524371+0000","last_active":"2024-09-14T04:41:10.338940+0000","last_peered":"2024-09-14T04:41:10.338940+0000","last_clean":"2024-09-14T04:41:10.338940+0000","last_became_active":"2024-09-14T04:07:58.524088+0000","last_became_peered":"2024-09-14T04:07:58.524088+0000","last_unstale":"2024-09-14T04:41:10.338940+0000","last_undegraded":"2024-09-14T04:41:10.338940+0000","last_fullsized":"2024-09-14T04:41:10.338940+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:42.615569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":1926,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.833220+0000","last_change":"2024-09-14T04:07:58.507553+0000","last_active":"2024-09-14T04:41:00.833220+0000","last_peered":"2024-09-14T04:41:00.833220+0000","last_clean":"2024-09-14T04:41:00.833220+0000","last_became_active":"2024-09-14T04:07:58.507139+0000","last_became_peered":"2024-09-14T04:07:58.507139+0000","last_unstale":"2024-09-14T04:41:00.833220+0000","last_undegraded":"2024-09-14T04:41:00.833220+0000","last_fullsized":"2024-09-14T04:41:00.833220+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:11:57.821769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":1958,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:10.331715+0000","last_change":"2024-09-14T04:07:58.561490+0000","last_active":"2024-09-14T04:41:10.331715+0000","last_peered":"2024-09-14T04:41:10.331715+0000","last_clean":"2024-09-14T04:41:10.331715+0000","last_became_active":"2024-09-14T04:07:58.559841+0000","last_became_peered":"2024-09-14T04:07:58.559841+0000","last_unstale":"2024-09-14T04:41:10.331715+0000","last_undegraded":"2024-09-14T04:41:10.331715+0000","last_fullsized":"2024-09-14T04:41:10.331715+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:27:03.795368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":1949,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:10.345186+0000","last_change":"2024-09-14T04:07:58.555731+0000","last_active":"2024-09-14T04:41:10.345186+0000","last_peered":"2024-09-14T04:41:10.345186+0000","last_clean":"2024-09-14T04:41:10.345186+0000","last_became_active":"2024-09-14T04:07:58.549576+0000","last_became_peered":"2024-09-14T04:07:58.549576+0000","last_unstale":"2024-09-14T04:41:10.345186+0000","last_undegraded":"2024-09-14T04:41:10.345186+0000","last_fullsized":"2024-09-14T04:41:10.345186+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:28:42.339589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1316,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.830248+0000","last_change":"2024-09-14T04:07:21.720797+0000","last_active":"2024-09-14T04:41:00.830248+0000","last_peered":"2024-09-14T04:41:00.830248+0000","last_clean":"2024-09-14T04:41:00.830248+0000","last_became_active":"2024-09-14T04:07:21.719331+0000","last_became_peered":"2024-09-14T04:07:21.719331+0000","last_unstale":"2024-09-14T04:41:00.830248+0000","last_undegraded":"2024-09-14T04:41:00.830248+0000","last_fullsized":"2024-09-14T04:41:00.830248+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:35.404023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":1959,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:10.345336+0000","last_change":"2024-09-14T04:07:58.502533+0000","last_active":"2024-09-14T04:41:10.345336+0000","last_peered":"2024-09-14T04:41:10.345336+0000","last_clean":"2024-09-14T04:41:10.345336+0000","last_became_active":"2024-09-14T04:07:58.499849+0000","last_became_peered":"2024-09-14T04:07:58.499849+0000","last_unstale":"2024-09-14T04:41:10.345336+0000","last_undegraded":"2024-09-14T04:41:10.345336+0000","last_fullsized":"2024-09-14T04:41:10.345336+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:15:35.609746+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,6],"acting":[3,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":1953,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:10.332947+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T04:41:10.332947+0000","last_peered":"2024-09-14T04:41:10.332947+0000","last_clean":"2024-09-14T04:41:10.332947+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T04:41:10.332947+0000","last_undegraded":"2024-09-14T04:41:10.332947+0000","last_fullsized":"2024-09-14T04:41:10.332947+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":1935,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:10.344496+0000","last_change":"2024-09-14T04:07:58.507682+0000","last_active":"2024-09-14T04:41:10.344496+0000","last_peered":"2024-09-14T04:41:10.344496+0000","last_clean":"2024-09-14T04:41:10.344496+0000","last_became_active":"2024-09-14T04:07:58.507365+0000","last_became_peered":"2024-09-14T04:07:58.507365+0000","last_unstale":"2024-09-14T04:41:10.344496+0000","last_undegraded":"2024-09-14T04:41:10.344496+0000","last_fullsized":"2024-09-14T04:41:10.344496+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:14:47.294950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1304,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.822937+0000","last_change":"2024-09-14T04:06:46.875668+0000","last_active":"2024-09-14T04:41:00.822937+0000","last_peered":"2024-09-14T04:41:00.822937+0000","last_clean":"2024-09-14T04:41:00.822937+0000","last_became_active":"2024-09-14T04:06:46.875149+0000","last_became_peered":"2024-09-14T04:06:46.875149+0000","last_unstale":"2024-09-14T04:41:00.822937+0000","last_undegraded":"2024-09-14T04:41:00.822937+0000","last_fullsized":"2024-09-14T04:41:00.822937+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:45:59.190047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1305,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.832217+0000","last_change":"2024-09-14T04:07:58.510699+0000","last_active":"2024-09-14T04:41:00.832217+0000","last_peered":"2024-09-14T04:41:00.832217+0000","last_clean":"2024-09-14T04:41:00.832217+0000","last_became_active":"2024-09-14T04:07:58.506240+0000","last_became_peered":"2024-09-14T04:07:58.506240+0000","last_unstale":"2024-09-14T04:41:00.832217+0000","last_undegraded":"2024-09-14T04:41:00.832217+0000","last_fullsized":"2024-09-14T04:41:00.832217+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:27:18.051658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":1958,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.831768+0000","last_change":"2024-09-14T04:07:21.687985+0000","last_active":"2024-09-14T04:41:00.831768+0000","last_peered":"2024-09-14T04:41:00.831768+0000","last_clean":"2024-09-14T04:41:00.831768+0000","last_became_active":"2024-09-14T04:07:21.687670+0000","last_became_peered":"2024-09-14T04:07:21.687670+0000","last_unstale":"2024-09-14T04:41:00.831768+0000","last_undegraded":"2024-09-14T04:41:00.831768+0000","last_fullsized":"2024-09-14T04:41:00.831768+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:53:58.295662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":1317,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.829439+0000","last_change":"2024-09-14T03:53:05.244055+0000","last_active":"2024-09-14T04:41:00.829439+0000","last_peered":"2024-09-14T04:41:00.829439+0000","last_clean":"2024-09-14T04:41:00.829439+0000","last_became_active":"2024-09-14T03:53:05.243752+0000","last_became_peered":"2024-09-14T03:53:05.243752+0000","last_unstale":"2024-09-14T04:41:00.829439+0000","last_undegraded":"2024-09-14T04:41:00.829439+0000","last_fullsized":"2024-09-14T04:41:00.829439+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:12:31.850064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":1308,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.827333+0000","last_change":"2024-09-14T04:06:46.895670+0000","last_active":"2024-09-14T04:41:00.827333+0000","last_peered":"2024-09-14T04:41:00.827333+0000","last_clean":"2024-09-14T04:41:00.827333+0000","last_became_active":"2024-09-14T04:06:46.895370+0000","last_became_peered":"2024-09-14T04:06:46.895370+0000","last_unstale":"2024-09-14T04:41:00.827333+0000","last_undegraded":"2024-09-14T04:41:00.827333+0000","last_fullsized":"2024-09-14T04:41:00.827333+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:48:48.040529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":1296,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.830901+0000","last_change":"2024-09-14T03:53:06.634043+0000","last_active":"2024-09-14T04:41:00.830901+0000","last_peered":"2024-09-14T04:41:00.830901+0000","last_clean":"2024-09-14T04:41:00.830901+0000","last_became_active":"2024-09-14T03:53:06.633638+0000","last_became_peered":"2024-09-14T03:53:06.633638+0000","last_unstale":"2024-09-14T04:41:00.830901+0000","last_undegraded":"2024-09-14T04:41:00.830901+0000","last_fullsized":"2024-09-14T04:41:00.830901+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:15:53.434064+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1319,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.830326+0000","last_change":"2024-09-14T04:07:21.721242+0000","last_active":"2024-09-14T04:41:00.830326+0000","last_peered":"2024-09-14T04:41:00.830326+0000","last_clean":"2024-09-14T04:41:00.830326+0000","last_became_active":"2024-09-14T04:07:21.720084+0000","last_became_peered":"2024-09-14T04:07:21.720084+0000","last_unstale":"2024-09-14T04:41:00.830326+0000","last_undegraded":"2024-09-14T04:41:00.830326+0000","last_fullsized":"2024-09-14T04:41:00.830326+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:58:41.308643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":1960,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:10.331617+0000","last_change":"2024-09-14T04:06:46.932567+0000","last_active":"2024-09-14T04:41:10.331617+0000","last_peered":"2024-09-14T04:41:10.331617+0000","last_clean":"2024-09-14T04:41:10.331617+0000","last_became_active":"2024-09-14T04:06:46.930732+0000","last_became_peered":"2024-09-14T04:06:46.930732+0000","last_unstale":"2024-09-14T04:41:10.331617+0000","last_undegraded":"2024-09-14T04:41:10.331617+0000","last_fullsized":"2024-09-14T04:41:10.331617+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:10:02.729499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":1927,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:10.345265+0000","last_change":"2024-09-14T04:07:58.555446+0000","last_active":"2024-09-14T04:41:10.345265+0000","last_peered":"2024-09-14T04:41:10.345265+0000","last_clean":"2024-09-14T04:41:10.345265+0000","last_became_active":"2024-09-14T04:07:58.548355+0000","last_became_peered":"2024-09-14T04:07:58.548355+0000","last_unstale":"2024-09-14T04:41:10.345265+0000","last_undegraded":"2024-09-14T04:41:10.345265+0000","last_fullsized":"2024-09-14T04:41:10.345265+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:56:28.426094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":1962,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.832227+0000","last_change":"2024-09-14T04:06:46.915273+0000","last_active":"2024-09-14T04:41:00.832227+0000","last_peered":"2024-09-14T04:41:00.832227+0000","last_clean":"2024-09-14T04:41:00.832227+0000","last_became_active":"2024-09-14T04:06:46.914994+0000","last_became_peered":"2024-09-14T04:06:46.914994+0000","last_unstale":"2024-09-14T04:41:00.832227+0000","last_undegraded":"2024-09-14T04:41:00.832227+0000","last_fullsized":"2024-09-14T04:41:00.832227+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:48:58.802609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":1928,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:10.341474+0000","last_change":"2024-09-14T04:07:58.564377+0000","last_active":"2024-09-14T04:41:10.341474+0000","last_peered":"2024-09-14T04:41:10.341474+0000","last_clean":"2024-09-14T04:41:10.341474+0000","last_became_active":"2024-09-14T04:07:58.546913+0000","last_became_peered":"2024-09-14T04:07:58.546913+0000","last_unstale":"2024-09-14T04:41:10.341474+0000","last_undegraded":"2024-09-14T04:41:10.341474+0000","last_fullsized":"2024-09-14T04:41:10.341474+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:05:53.637115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":1946,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:10.331350+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T04:41:10.331350+0000","last_peered":"2024-09-14T04:41:10.331350+0000","last_clean":"2024-09-14T04:41:10.331350+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T04:41:10.331350+0000","last_undegraded":"2024-09-14T04:41:10.331350+0000","last_fullsized":"2024-09-14T04:41:10.331350+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1318,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.832405+0000","last_change":"2024-09-14T04:05:54.381035+0000","last_active":"2024-09-14T04:41:00.832405+0000","last_peered":"2024-09-14T04:41:00.832405+0000","last_clean":"2024-09-14T04:41:00.832405+0000","last_became_active":"2024-09-14T04:05:54.379861+0000","last_became_peered":"2024-09-14T04:05:54.379861+0000","last_unstale":"2024-09-14T04:41:00.832405+0000","last_undegraded":"2024-09-14T04:41:00.832405+0000","last_fullsized":"2024-09-14T04:41:00.832405+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:07:46.302925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":1962,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.844489+0000","last_change":"2024-09-14T04:07:21.686880+0000","last_active":"2024-09-14T04:41:00.844489+0000","last_peered":"2024-09-14T04:41:00.844489+0000","last_clean":"2024-09-14T04:41:00.844489+0000","last_became_active":"2024-09-14T04:07:21.685558+0000","last_became_peered":"2024-09-14T04:07:21.685558+0000","last_unstale":"2024-09-14T04:41:00.844489+0000","last_undegraded":"2024-09-14T04:41:00.844489+0000","last_fullsized":"2024-09-14T04:41:00.844489+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:24:18.693428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":1950,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.831655+0000","last_change":"2024-09-14T04:06:46.905931+0000","last_active":"2024-09-14T04:41:00.831655+0000","last_peered":"2024-09-14T04:41:00.831655+0000","last_clean":"2024-09-14T04:41:00.831655+0000","last_became_active":"2024-09-14T04:06:46.905666+0000","last_became_peered":"2024-09-14T04:06:46.905666+0000","last_unstale":"2024-09-14T04:41:00.831655+0000","last_undegraded":"2024-09-14T04:41:00.831655+0000","last_fullsized":"2024-09-14T04:41:00.831655+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:40:57.007948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":1979,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.844727+0000","last_change":"2024-09-14T04:06:46.930482+0000","last_active":"2024-09-14T04:41:00.844727+0000","last_peered":"2024-09-14T04:41:00.844727+0000","last_clean":"2024-09-14T04:41:00.844727+0000","last_became_active":"2024-09-14T04:06:46.927372+0000","last_became_peered":"2024-09-14T04:06:46.927372+0000","last_unstale":"2024-09-14T04:41:00.844727+0000","last_undegraded":"2024-09-14T04:41:00.844727+0000","last_fullsized":"2024-09-14T04:41:00.844727+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:07:22.159128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1324,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:08.011805+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T04:41:08.011805+0000","last_peered":"2024-09-14T04:41:08.011805+0000","last_clean":"2024-09-14T04:41:08.011805+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T04:41:08.011805+0000","last_undegraded":"2024-09-14T04:41:08.011805+0000","last_fullsized":"2024-09-14T04:41:08.011805+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1972,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.844638+0000","last_change":"2024-09-14T04:07:58.510721+0000","last_active":"2024-09-14T04:41:00.844638+0000","last_peered":"2024-09-14T04:41:00.844638+0000","last_clean":"2024-09-14T04:41:00.844638+0000","last_became_active":"2024-09-14T04:07:58.510332+0000","last_became_peered":"2024-09-14T04:07:58.510332+0000","last_unstale":"2024-09-14T04:41:00.844638+0000","last_undegraded":"2024-09-14T04:41:00.844638+0000","last_fullsized":"2024-09-14T04:41:00.844638+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:07:03.926842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":1312,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:03.040128+0000","last_change":"2024-09-14T04:05:54.426939+0000","last_active":"2024-09-14T04:41:03.040128+0000","last_peered":"2024-09-14T04:41:03.040128+0000","last_clean":"2024-09-14T04:41:03.040128+0000","last_became_active":"2024-09-14T04:05:54.426427+0000","last_became_peered":"2024-09-14T04:05:54.426427+0000","last_unstale":"2024-09-14T04:41:03.040128+0000","last_undegraded":"2024-09-14T04:41:03.040128+0000","last_fullsized":"2024-09-14T04:41:03.040128+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:53:55.841489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1322,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.823907+0000","last_change":"2024-09-14T04:06:46.875340+0000","last_active":"2024-09-14T04:41:00.823907+0000","last_peered":"2024-09-14T04:41:00.823907+0000","last_clean":"2024-09-14T04:41:00.823907+0000","last_became_active":"2024-09-14T04:06:46.874552+0000","last_became_peered":"2024-09-14T04:06:46.874552+0000","last_unstale":"2024-09-14T04:41:00.823907+0000","last_undegraded":"2024-09-14T04:41:00.823907+0000","last_fullsized":"2024-09-14T04:41:00.823907+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:52.257357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":1958,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.843723+0000","last_change":"2024-09-14T04:06:46.917681+0000","last_active":"2024-09-14T04:41:00.843723+0000","last_peered":"2024-09-14T04:41:00.843723+0000","last_clean":"2024-09-14T04:41:00.843723+0000","last_became_active":"2024-09-14T04:06:46.917401+0000","last_became_peered":"2024-09-14T04:06:46.917401+0000","last_unstale":"2024-09-14T04:41:00.843723+0000","last_undegraded":"2024-09-14T04:41:00.843723+0000","last_fullsized":"2024-09-14T04:41:00.843723+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:04:39.949490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":1963,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.832769+0000","last_change":"2024-09-14T04:07:21.685415+0000","last_active":"2024-09-14T04:41:00.832769+0000","last_peered":"2024-09-14T04:41:00.832769+0000","last_clean":"2024-09-14T04:41:00.832769+0000","last_became_active":"2024-09-14T04:07:21.683504+0000","last_became_peered":"2024-09-14T04:07:21.683504+0000","last_unstale":"2024-09-14T04:41:00.832769+0000","last_undegraded":"2024-09-14T04:41:00.832769+0000","last_fullsized":"2024-09-14T04:41:00.832769+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:21:08.806291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":1965,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.842331+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T04:41:00.842331+0000","last_peered":"2024-09-14T04:41:00.842331+0000","last_clean":"2024-09-14T04:41:00.842331+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T04:41:00.842331+0000","last_undegraded":"2024-09-14T04:41:00.842331+0000","last_fullsized":"2024-09-14T04:41:00.842331+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":1968,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:10.331798+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T04:41:10.331798+0000","last_peered":"2024-09-14T04:41:10.331798+0000","last_clean":"2024-09-14T04:41:10.331798+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T04:41:10.331798+0000","last_undegraded":"2024-09-14T04:41:10.331798+0000","last_fullsized":"2024-09-14T04:41:10.331798+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":1956,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:10.336345+0000","last_change":"2024-09-14T04:07:58.521344+0000","last_active":"2024-09-14T04:41:10.336345+0000","last_peered":"2024-09-14T04:41:10.336345+0000","last_clean":"2024-09-14T04:41:10.336345+0000","last_became_active":"2024-09-14T04:07:58.520965+0000","last_became_peered":"2024-09-14T04:07:58.520965+0000","last_unstale":"2024-09-14T04:41:10.336345+0000","last_undegraded":"2024-09-14T04:41:10.336345+0000","last_fullsized":"2024-09-14T04:41:10.336345+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:58:46.029013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1334,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.831790+0000","last_change":"2024-09-14T04:06:46.857436+0000","last_active":"2024-09-14T04:41:00.831790+0000","last_peered":"2024-09-14T04:41:00.831790+0000","last_clean":"2024-09-14T04:41:00.831790+0000","last_became_active":"2024-09-14T04:06:46.857154+0000","last_became_peered":"2024-09-14T04:06:46.857154+0000","last_unstale":"2024-09-14T04:41:00.831790+0000","last_undegraded":"2024-09-14T04:41:00.831790+0000","last_fullsized":"2024-09-14T04:41:00.831790+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:53:28.772152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1292,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:07.883485+0000","last_change":"2024-09-14T03:53:05.551268+0000","last_active":"2024-09-14T04:41:07.883485+0000","last_peered":"2024-09-14T04:41:07.883485+0000","last_clean":"2024-09-14T04:41:07.883485+0000","last_became_active":"2024-09-14T03:53:05.551015+0000","last_became_peered":"2024-09-14T03:53:05.551015+0000","last_unstale":"2024-09-14T04:41:07.883485+0000","last_undegraded":"2024-09-14T04:41:07.883485+0000","last_fullsized":"2024-09-14T04:41:07.883485+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.905475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":1938,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:10.342824+0000","last_change":"2024-09-14T04:07:58.503896+0000","last_active":"2024-09-14T04:41:10.342824+0000","last_peered":"2024-09-14T04:41:10.342824+0000","last_clean":"2024-09-14T04:41:10.342824+0000","last_became_active":"2024-09-14T04:07:58.499253+0000","last_became_peered":"2024-09-14T04:07:58.499253+0000","last_unstale":"2024-09-14T04:41:10.342824+0000","last_undegraded":"2024-09-14T04:41:10.342824+0000","last_fullsized":"2024-09-14T04:41:10.342824+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:48:27.543818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1321,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.822849+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T04:41:00.822849+0000","last_peered":"2024-09-14T04:41:00.822849+0000","last_clean":"2024-09-14T04:41:00.822849+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T04:41:00.822849+0000","last_undegraded":"2024-09-14T04:41:00.822849+0000","last_fullsized":"2024-09-14T04:41:00.822849+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":1636,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.831571+0000","last_change":"2024-09-14T04:05:54.389043+0000","last_active":"2024-09-14T04:41:00.831571+0000","last_peered":"2024-09-14T04:41:00.831571+0000","last_clean":"2024-09-14T04:41:00.831571+0000","last_became_active":"2024-09-14T04:05:54.388655+0000","last_became_peered":"2024-09-14T04:05:54.388655+0000","last_unstale":"2024-09-14T04:41:00.831571+0000","last_undegraded":"2024-09-14T04:41:00.831571+0000","last_fullsized":"2024-09-14T04:41:00.831571+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T03:49:18.237706+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:38:59.615597+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":1927,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.843553+0000","last_change":"2024-09-14T04:07:58.530406+0000","last_active":"2024-09-14T04:41:00.843553+0000","last_peered":"2024-09-14T04:41:00.843553+0000","last_clean":"2024-09-14T04:41:00.843553+0000","last_became_active":"2024-09-14T04:07:58.530078+0000","last_became_peered":"2024-09-14T04:07:58.530078+0000","last_unstale":"2024-09-14T04:41:00.843553+0000","last_undegraded":"2024-09-14T04:41:00.843553+0000","last_fullsized":"2024-09-14T04:41:00.843553+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:54:53.524001+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":1937,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:10.345092+0000","last_change":"2024-09-14T04:07:58.555717+0000","last_active":"2024-09-14T04:41:10.345092+0000","last_peered":"2024-09-14T04:41:10.345092+0000","last_clean":"2024-09-14T04:41:10.345092+0000","last_became_active":"2024-09-14T04:07:58.548894+0000","last_became_peered":"2024-09-14T04:07:58.548894+0000","last_unstale":"2024-09-14T04:41:10.345092+0000","last_undegraded":"2024-09-14T04:41:10.345092+0000","last_fullsized":"2024-09-14T04:41:10.345092+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:42.061551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":1309,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.827684+0000","last_change":"2024-09-14T04:06:46.894741+0000","last_active":"2024-09-14T04:41:00.827684+0000","last_peered":"2024-09-14T04:41:00.827684+0000","last_clean":"2024-09-14T04:41:00.827684+0000","last_became_active":"2024-09-14T04:06:46.894445+0000","last_became_peered":"2024-09-14T04:06:46.894445+0000","last_unstale":"2024-09-14T04:41:00.827684+0000","last_undegraded":"2024-09-14T04:41:00.827684+0000","last_fullsized":"2024-09-14T04:41:00.827684+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:52:00.055271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":1311,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.831135+0000","last_change":"2024-09-14T04:07:21.715462+0000","last_active":"2024-09-14T04:41:00.831135+0000","last_peered":"2024-09-14T04:41:00.831135+0000","last_clean":"2024-09-14T04:41:00.831135+0000","last_became_active":"2024-09-14T04:07:21.713032+0000","last_became_peered":"2024-09-14T04:07:21.713032+0000","last_unstale":"2024-09-14T04:41:00.831135+0000","last_undegraded":"2024-09-14T04:41:00.831135+0000","last_fullsized":"2024-09-14T04:41:00.831135+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:37:05.545164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":1324,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.828962+0000","last_change":"2024-09-14T04:07:21.718037+0000","last_active":"2024-09-14T04:41:00.828962+0000","last_peered":"2024-09-14T04:41:00.828962+0000","last_clean":"2024-09-14T04:41:00.828962+0000","last_became_active":"2024-09-14T04:07:21.717700+0000","last_became_peered":"2024-09-14T04:07:21.717700+0000","last_unstale":"2024-09-14T04:41:00.828962+0000","last_undegraded":"2024-09-14T04:41:00.828962+0000","last_fullsized":"2024-09-14T04:41:00.828962+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:47:33.483785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1307,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.830104+0000","last_change":"2024-09-14T04:06:46.868780+0000","last_active":"2024-09-14T04:41:00.830104+0000","last_peered":"2024-09-14T04:41:00.830104+0000","last_clean":"2024-09-14T04:41:00.830104+0000","last_became_active":"2024-09-14T04:06:46.868483+0000","last_became_peered":"2024-09-14T04:06:46.868483+0000","last_unstale":"2024-09-14T04:41:00.830104+0000","last_undegraded":"2024-09-14T04:41:00.830104+0000","last_fullsized":"2024-09-14T04:41:00.830104+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:03.133158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":1309,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.829354+0000","last_change":"2024-09-14T04:07:58.515859+0000","last_active":"2024-09-14T04:41:00.829354+0000","last_peered":"2024-09-14T04:41:00.829354+0000","last_clean":"2024-09-14T04:41:00.829354+0000","last_became_active":"2024-09-14T04:07:58.515594+0000","last_became_peered":"2024-09-14T04:07:58.515594+0000","last_unstale":"2024-09-14T04:41:00.829354+0000","last_undegraded":"2024-09-14T04:41:00.829354+0000","last_fullsized":"2024-09-14T04:41:00.829354+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:45.966798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":1953,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:10.344554+0000","last_change":"2024-09-14T04:07:58.509833+0000","last_active":"2024-09-14T04:41:10.344554+0000","last_peered":"2024-09-14T04:41:10.344554+0000","last_clean":"2024-09-14T04:41:10.344554+0000","last_became_active":"2024-09-14T04:07:58.509198+0000","last_became_peered":"2024-09-14T04:07:58.509198+0000","last_unstale":"2024-09-14T04:41:10.344554+0000","last_undegraded":"2024-09-14T04:41:10.344554+0000","last_fullsized":"2024-09-14T04:41:10.344554+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:24:40.193090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":1956,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:10.331069+0000","last_change":"2024-09-14T04:06:46.925323+0000","last_active":"2024-09-14T04:41:10.331069+0000","last_peered":"2024-09-14T04:41:10.331069+0000","last_clean":"2024-09-14T04:41:10.331069+0000","last_became_active":"2024-09-14T04:06:46.925043+0000","last_became_peered":"2024-09-14T04:06:46.925043+0000","last_unstale":"2024-09-14T04:41:10.331069+0000","last_undegraded":"2024-09-14T04:41:10.331069+0000","last_fullsized":"2024-09-14T04:41:10.331069+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:39.895453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1307,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.824076+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T04:41:00.824076+0000","last_peered":"2024-09-14T04:41:00.824076+0000","last_clean":"2024-09-14T04:41:00.824076+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T04:41:00.824076+0000","last_undegraded":"2024-09-14T04:41:00.824076+0000","last_fullsized":"2024-09-14T04:41:00.824076+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1306,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.829947+0000","last_change":"2024-09-14T04:06:46.869868+0000","last_active":"2024-09-14T04:41:00.829947+0000","last_peered":"2024-09-14T04:41:00.829947+0000","last_clean":"2024-09-14T04:41:00.829947+0000","last_became_active":"2024-09-14T04:06:46.869126+0000","last_became_peered":"2024-09-14T04:06:46.869126+0000","last_unstale":"2024-09-14T04:41:00.829947+0000","last_undegraded":"2024-09-14T04:41:00.829947+0000","last_fullsized":"2024-09-14T04:41:00.829947+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:41.150138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1290,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.832341+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T04:41:00.832341+0000","last_peered":"2024-09-14T04:41:00.832341+0000","last_clean":"2024-09-14T04:41:00.832341+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T04:41:00.832341+0000","last_undegraded":"2024-09-14T04:41:00.832341+0000","last_fullsized":"2024-09-14T04:41:00.832341+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":1325,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.831026+0000","last_change":"2024-09-14T04:07:58.508202+0000","last_active":"2024-09-14T04:41:00.831026+0000","last_peered":"2024-09-14T04:41:00.831026+0000","last_clean":"2024-09-14T04:41:00.831026+0000","last_became_active":"2024-09-14T04:07:58.507929+0000","last_became_peered":"2024-09-14T04:07:58.507929+0000","last_unstale":"2024-09-14T04:41:00.831026+0000","last_undegraded":"2024-09-14T04:41:00.831026+0000","last_fullsized":"2024-09-14T04:41:00.831026+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:17:58.160600+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1306,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.832621+0000","last_change":"2024-09-14T04:06:46.858890+0000","last_active":"2024-09-14T04:41:00.832621+0000","last_peered":"2024-09-14T04:41:00.832621+0000","last_clean":"2024-09-14T04:41:00.832621+0000","last_became_active":"2024-09-14T04:06:46.858569+0000","last_became_peered":"2024-09-14T04:06:46.858569+0000","last_unstale":"2024-09-14T04:41:00.832621+0000","last_undegraded":"2024-09-14T04:41:00.832621+0000","last_fullsized":"2024-09-14T04:41:00.832621+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:25:46.400115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1292,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.821303+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T04:41:00.821303+0000","last_peered":"2024-09-14T04:41:00.821303+0000","last_clean":"2024-09-14T04:41:00.821303+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T04:41:00.821303+0000","last_undegraded":"2024-09-14T04:41:00.821303+0000","last_fullsized":"2024-09-14T04:41:00.821303+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1305,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.832693+0000","last_change":"2024-09-14T04:05:54.419435+0000","last_active":"2024-09-14T04:41:00.832693+0000","last_peered":"2024-09-14T04:41:00.832693+0000","last_clean":"2024-09-14T04:41:00.832693+0000","last_became_active":"2024-09-14T04:05:54.419094+0000","last_became_peered":"2024-09-14T04:05:54.419094+0000","last_unstale":"2024-09-14T04:41:00.832693+0000","last_undegraded":"2024-09-14T04:41:00.832693+0000","last_fullsized":"2024-09-14T04:41:00.832693+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:00.162629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1313,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.830487+0000","last_change":"2024-09-14T04:05:54.418813+0000","last_active":"2024-09-14T04:41:00.830487+0000","last_peered":"2024-09-14T04:41:00.830487+0000","last_clean":"2024-09-14T04:41:00.830487+0000","last_became_active":"2024-09-14T04:05:54.417584+0000","last_became_peered":"2024-09-14T04:05:54.417584+0000","last_unstale":"2024-09-14T04:41:00.830487+0000","last_undegraded":"2024-09-14T04:41:00.830487+0000","last_fullsized":"2024-09-14T04:41:00.830487+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:13:16.209456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":1933,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:10.341366+0000","last_change":"2024-09-14T04:07:58.565442+0000","last_active":"2024-09-14T04:41:10.341366+0000","last_peered":"2024-09-14T04:41:10.341366+0000","last_clean":"2024-09-14T04:41:10.341366+0000","last_became_active":"2024-09-14T04:07:58.552289+0000","last_became_peered":"2024-09-14T04:07:58.552289+0000","last_unstale":"2024-09-14T04:41:10.341366+0000","last_undegraded":"2024-09-14T04:41:10.341366+0000","last_fullsized":"2024-09-14T04:41:10.341366+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:48:02.364960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1288,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.831986+0000","last_change":"2024-09-14T03:53:05.244003+0000","last_active":"2024-09-14T04:41:00.831986+0000","last_peered":"2024-09-14T04:41:00.831986+0000","last_clean":"2024-09-14T04:41:00.831986+0000","last_became_active":"2024-09-14T03:53:05.240743+0000","last_became_peered":"2024-09-14T03:53:05.240743+0000","last_unstale":"2024-09-14T04:41:00.831986+0000","last_undegraded":"2024-09-14T04:41:00.831986+0000","last_fullsized":"2024-09-14T04:41:00.831986+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:27:30.000189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":1954,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:10.331526+0000","last_change":"2024-09-14T04:07:58.560880+0000","last_active":"2024-09-14T04:41:10.331526+0000","last_peered":"2024-09-14T04:41:10.331526+0000","last_clean":"2024-09-14T04:41:10.331526+0000","last_became_active":"2024-09-14T04:07:58.560132+0000","last_became_peered":"2024-09-14T04:07:58.560132+0000","last_unstale":"2024-09-14T04:41:10.331526+0000","last_undegraded":"2024-09-14T04:41:10.331526+0000","last_fullsized":"2024-09-14T04:41:10.331526+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:46:00.287050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":1954,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:10.331442+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T04:41:10.331442+0000","last_peered":"2024-09-14T04:41:10.331442+0000","last_clean":"2024-09-14T04:41:10.331442+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T04:41:10.331442+0000","last_undegraded":"2024-09-14T04:41:10.331442+0000","last_fullsized":"2024-09-14T04:41:10.331442+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1304,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.823206+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T04:41:00.823206+0000","last_peered":"2024-09-14T04:41:00.823206+0000","last_clean":"2024-09-14T04:41:00.823206+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T04:41:00.823206+0000","last_undegraded":"2024-09-14T04:41:00.823206+0000","last_fullsized":"2024-09-14T04:41:00.823206+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1305,"reported_epoch":1338,"state":"active+clean","last_fresh":"2024-09-14T04:41:00.832099+0000","last_change":"2024-09-14T04:07:21.718847+0000","last_active":"2024-09-14T04:41:00.832099+0000","last_peered":"2024-09-14T04:41:00.832099+0000","last_clean":"2024-09-14T04:41:00.832099+0000","last_became_active":"2024-09-14T04:07:21.718504+0000","last_became_peered":"2024-09-14T04:07:21.718504+0000","last_unstale":"2024-09-14T04:41:00.832099+0000","last_undegraded":"2024-09-14T04:41:00.832099+0000","last_fullsized":"2024-09-14T04:41:00.832099+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:37:34.405765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":1950,"reported_epoch":1339,"state":"active+clean","last_fresh":"2024-09-14T04:41:10.337902+0000","last_change":"2024-09-14T04:07:58.537946+0000","last_active":"2024-09-14T04:41:10.337902+0000","last_peered":"2024-09-14T04:41:10.337902+0000","last_clean":"2024-09-14T04:41:10.337902+0000","last_became_active":"2024-09-14T04:07:58.536034+0000","last_became_peered":"2024-09-14T04:07:58.536034+0000","last_unstale":"2024-09-14T04:41:10.337902+0000","last_undegraded":"2024-09-14T04:41:10.337902+0000","last_fullsized":"2024-09-14T04:41:10.337902+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:17:18.200470+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":53,"num_pg":16,"stat_sum":{"num_bytes":4659281944,"num_objects":71096,"num_object_clones":0,"num_object_copies":213288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":71096,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":189086,"num_write_kb":8325761,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13998370816,"data_stored":13998358600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":137686,"ondisk_log_size":137686,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":392,"num_read_kb":834,"num_write":215,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":149,"ondisk_log_size":149,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201883,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":416036,"kb_used_data":290652,"kb_used_omap":1,"kb_used_meta":125374,"kb_avail":93329116,"statfs":{"total":95995035648,"available":95569014784,"internally_reserved":0,"allocated":297627648,"data_stored":293685921,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":128383435},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463519,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1313820,"kb_used_data":1130068,"kb_used_omap":1,"kb_used_meta":183614,"kb_avail":92431332,"statfs":{"total":95995035648,"available":94649683968,"internally_reserved":0,"allocated":1157189632,"data_stored":1153255970,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":188021191},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725156,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2267604,"kb_used_data":2011404,"kb_used_omap":1,"kb_used_meta":255806,"kb_avail":91477548,"statfs":{"total":95995035648,"available":93673009152,"internally_reserved":0,"allocated":2059677696,"data_stored":2055747532,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":261945798},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986794,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3398560,"kb_used_data":3106644,"kb_used_omap":1,"kb_used_meta":291902,"kb_avail":90346592,"statfs":{"total":95995035648,"available":92514910208,"internally_reserved":0,"allocated":3181203456,"data_stored":3177269307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":298908102},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772074897,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1587788,"kb_used_data":1457152,"kb_used_omap":71,"kb_used_meta":130552,"kb_avail":92157364,"statfs":{"total":95995035648,"available":94369140736,"internally_reserved":0,"allocated":1492123648,"data_stored":1488205179,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":133686198},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707303832,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1309760,"kb_used_data":1187448,"kb_used_omap":68,"kb_used_meta":122235,"kb_avail":92435392,"statfs":{"total":95995035648,"available":94653841408,"internally_reserved":0,"allocated":1215946752,"data_stored":1212028288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":125168848},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872728991,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3154364,"kb_used_data":2927988,"kb_used_omap":67,"kb_used_meta":226300,"kb_avail":90590788,"statfs":{"total":95995035648,"available":92764966912,"internally_reserved":0,"allocated":2998259712,"data_stored":2994344747,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":231731557},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056042,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1875344,"kb_used_data":1721352,"kb_used_omap":74,"kb_used_meta":153909,"kb_avail":91869808,"statfs":{"total":95995035648,"available":94074683392,"internally_reserved":0,"allocated":1762664448,"data_stored":1758738845,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":157603509},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1735393280,"data_stored":1735393280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2981761024,"data_stored":2981756952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1200164864,"data_stored":1200160792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1465319424,"data_stored":1465319424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3164143616,"data_stored":3164143616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2024411136,"data_stored":2024407064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1140981760,"data_stored":1140981760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":53,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":286195712,"data_stored":286195712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:41:15.638 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-14T04:41:15.639 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pg_num 11 2024-09-14T04:41:15.730 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:15.765 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:15.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:15.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:15.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:15 smithi129 ceph-mon[101316]: pgmap v2257: 89 pgs: 89 active+clean; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 867 op/s 2024-09-14T04:41:15.867 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:15.904 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:15.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:15.905 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:16.006 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:16.052 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:16.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:16.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:16.154 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:16.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:16.191 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:16.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:16.292 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:16.304 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:41:16.323 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:16.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:16.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:16.426 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:16.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:16.466 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:16.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:16.567 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:16.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:16.601 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:16.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:16.702 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:16.739 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:16.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:16.740 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:16.755 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:16 smithi039 ceph-mon[106470]: from='client.45212 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:41:16.755 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:16 smithi039 ceph-mon[111235]: from='client.45212 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:41:16.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:16 smithi129 ceph-mon[101316]: from='client.45212 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:41:16.831 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:41:16.841 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:16.929 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:16.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:16.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:17.031 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:17.097 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:17.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:17.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:17.199 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:17.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:17.246 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:17.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:17.347 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:17.374 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:17.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:17.375 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:17.476 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:17.507 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:17.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:17.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:17.609 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:17.643 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:17.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:17.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:17.744 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:17.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:17 smithi129 ceph-mon[101316]: pgmap v2258: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 865 op/s 2024-09-14T04:41:17.779 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:17.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:17.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:17.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:17 smithi039 ceph-mon[106470]: pgmap v2258: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 865 op/s 2024-09-14T04:41:17.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:17 smithi039 ceph-mon[111235]: pgmap v2258: 89 pgs: 89 active+clean; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 865 op/s 2024-09-14T04:41:17.881 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:17.906 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:17.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:17.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:18.008 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:41:18.037 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:18.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:18.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:41:18.139 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:18.170 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:18.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:18.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:18.273 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:18.296 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:18.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:18.297 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:18.398 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:18.435 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:18.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:18.436 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:18.537 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:18.569 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:18.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:18.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:18.671 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:18.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:18.707 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:18.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:18.808 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:18.848 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:18.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:18.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:18.947 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:41:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:41:18] "GET /metrics HTTP/1.1" 200 36737 "" "Prometheus/2.51.0" 2024-09-14T04:41:18.950 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:18.986 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:18.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:18.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:19.088 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:19.123 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:19.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:19.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:19.225 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:41:19.253 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:19.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:19.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:41:19.354 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:19.391 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:19.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:19.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:19.492 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:19.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:19.525 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:19.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:19.626 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:19.657 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:19.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:19.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:19.675 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:19 smithi039 ceph-mon[106470]: pgmap v2259: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 867 op/s 2024-09-14T04:41:19.677 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:19 smithi039 ceph-mon[111235]: pgmap v2259: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 867 op/s 2024-09-14T04:41:19.760 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:19.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:19 smithi129 ceph-mon[101316]: pgmap v2259: 89 pgs: 89 active+clean; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 867 op/s 2024-09-14T04:41:19.794 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:41:19.803 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:19.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:19.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:19.854 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:41:19.855 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [ 2024-09-14T04:41:19.855 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.855 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:196032 53.2 e1339/1315 53:4f44dabe:::benchmark_data_smithi039_224949_object65939:head v 1339'12220, mlcod=1339'12218)", 2024-09-14T04:41:19.855 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:41:19.834483+0000", 2024-09-14T04:41:19.855 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.01295925, 2024-09-14T04:41:19.855 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.01300101, 2024-09-14T04:41:19.855 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:41:19.855 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:41:19.855 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "started", 2024-09-14T04:41:19.855 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:41:19.855 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.855 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:41:19.855 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.834483+0000", 2024-09-14T04:41:19.856 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:19.856 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.856 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.856 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:41:19.856 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.834483+0000", 2024-09-14T04:41:19.856 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:19.856 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.856 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.856 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:41:19.856 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.834484+0000", 2024-09-14T04:41:19.856 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.8899999999999998e-07 2024-09-14T04:41:19.856 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.856 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.857 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:41:19.857 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.834491+0000", 2024-09-14T04:41:19.857 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9500000000000004e-06 2024-09-14T04:41:19.857 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.857 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.857 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:41:19.857 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.834517+0000", 2024-09-14T04:41:19.857 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.6639999999999999e-05 2024-09-14T04:41:19.857 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.857 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.857 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:41:19.857 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.834524+0000", 2024-09-14T04:41:19.857 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9789999999999996e-06 2024-09-14T04:41:19.858 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.858 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.858 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:41:19.858 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.834614+0000", 2024-09-14T04:41:19.858 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.9421999999999999e-05 2024-09-14T04:41:19.858 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.858 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.858 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:41:19.858 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.834632+0000", 2024-09-14T04:41:19.858 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8349e-05 2024-09-14T04:41:19.858 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:19.858 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:41:19.858 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:19.859 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.859 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.859 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:196033 53.2 e1339/1315 53:4253cca8:::benchmark_data_smithi039_224949_object65940:head v 1339'12221, mlcod=1339'12218)", 2024-09-14T04:41:19.859 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:41:19.836565+0000", 2024-09-14T04:41:19.859 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.010877380000000001, 2024-09-14T04:41:19.859 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.010974368, 2024-09-14T04:41:19.859 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:41:19.859 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:41:19.859 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "started", 2024-09-14T04:41:19.859 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:41:19.859 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.859 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:41:19.859 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.836565+0000", 2024-09-14T04:41:19.859 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:19.860 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.860 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.860 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:41:19.860 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.836565+0000", 2024-09-14T04:41:19.860 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:19.860 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.860 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.860 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:41:19.860 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.836566+0000", 2024-09-14T04:41:19.860 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.1299999999999998e-07 2024-09-14T04:41:19.860 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.860 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.861 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:41:19.861 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.836573+0000", 2024-09-14T04:41:19.861 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7939999999999997e-06 2024-09-14T04:41:19.861 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.861 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.861 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:41:19.861 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.836589+0000", 2024-09-14T04:41:19.861 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6308999999999999e-05 2024-09-14T04:41:19.861 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.861 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.861 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:41:19.861 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.836597+0000", 2024-09-14T04:41:19.861 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.2039999999999997e-06 2024-09-14T04:41:19.861 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.862 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.862 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:41:19.862 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.843748+0000", 2024-09-14T04:41:19.862 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0071506909999999998 2024-09-14T04:41:19.862 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.862 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.862 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:41:19.862 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.843770+0000", 2024-09-14T04:41:19.862 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2158999999999999e-05 2024-09-14T04:41:19.863 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:19.863 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:41:19.863 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:19.863 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.863 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.863 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:196037 53.2 e1339/1315)", 2024-09-14T04:41:19.863 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:41:19.843619+0000", 2024-09-14T04:41:19.863 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.0038233030000000001, 2024-09-14T04:41:19.863 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0039653900000000001, 2024-09-14T04:41:19.863 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:41:19.864 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:41:19.864 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "queued for pg", 2024-09-14T04:41:19.864 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:41:19.864 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.864 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:41:19.864 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.843619+0000", 2024-09-14T04:41:19.864 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:19.864 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.864 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.864 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:41:19.864 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.843619+0000", 2024-09-14T04:41:19.864 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:19.864 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.865 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.865 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:41:19.865 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.843620+0000", 2024-09-14T04:41:19.865 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1200000000000001e-06 2024-09-14T04:41:19.865 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.865 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.865 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:41:19.865 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.843629+0000", 2024-09-14T04:41:19.865 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.7460000000000006e-06 2024-09-14T04:41:19.865 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.865 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.865 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:41:19.865 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.843643+0000", 2024-09-14T04:41:19.865 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4482e-05 2024-09-14T04:41:19.866 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.866 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.866 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:41:19.866 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.843652+0000", 2024-09-14T04:41:19.866 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.3529999999999995e-06 2024-09-14T04:41:19.866 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:19.866 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:41:19.866 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:19.866 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.866 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.866 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:196039 53.5 53:a59c59db:::benchmark_data_smithi039_224949_object65946:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1339)", 2024-09-14T04:41:19.866 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:41:19.843682+0000", 2024-09-14T04:41:19.866 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.0037607679999999998, 2024-09-14T04:41:19.866 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0039448299999999999, 2024-09-14T04:41:19.867 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:41:19.867 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:41:19.867 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:41:19.867 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:41:19.867 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:41:19.867 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:41:19.867 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 196039 2024-09-14T04:41:19.867 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.867 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:41:19.867 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.867 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:41:19.867 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.843682+0000", 2024-09-14T04:41:19.867 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:19.867 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.868 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.868 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:41:19.868 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.843682+0000", 2024-09-14T04:41:19.868 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:19.868 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.868 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.868 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:41:19.868 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.843683+0000", 2024-09-14T04:41:19.868 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.48e-06 2024-09-14T04:41:19.868 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.868 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.868 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:41:19.868 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.843687+0000", 2024-09-14T04:41:19.868 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.495e-06 2024-09-14T04:41:19.869 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.869 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.869 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:41:19.869 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.843696+0000", 2024-09-14T04:41:19.869 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.3770000000000002e-06 2024-09-14T04:41:19.869 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.869 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.869 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:41:19.869 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.843702+0000", 2024-09-14T04:41:19.869 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2990000000000002e-06 2024-09-14T04:41:19.869 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.869 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.869 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:41:19.869 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.845153+0000", 2024-09-14T04:41:19.870 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0014508360000000001 2024-09-14T04:41:19.870 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.870 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.870 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:41:19.870 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.845472+0000", 2024-09-14T04:41:19.870 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00031919799999999998 2024-09-14T04:41:19.870 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.870 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.870 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:41:19.870 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.845853+0000", 2024-09-14T04:41:19.870 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00038029999999999997 2024-09-14T04:41:19.870 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:19.870 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:41:19.870 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:19.872 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.872 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.872 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:196047 53.9 53.5ea82a9 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1339)", 2024-09-14T04:41:19.872 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:41:19.844941+0000", 2024-09-14T04:41:19.872 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.0025013050000000001, 2024-09-14T04:41:19.872 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.002744519, 2024-09-14T04:41:19.872 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:41:19.873 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:41:19.873 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "queued for pg", 2024-09-14T04:41:19.873 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:41:19.873 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:41:19.873 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:41:19.873 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 196047 2024-09-14T04:41:19.873 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.873 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:41:19.873 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.873 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:41:19.874 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.844941+0000", 2024-09-14T04:41:19.874 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:19.874 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.874 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.874 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:41:19.874 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.844941+0000", 2024-09-14T04:41:19.874 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:19.874 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.874 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.874 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:41:19.874 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.844942+0000", 2024-09-14T04:41:19.874 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.09e-07 2024-09-14T04:41:19.874 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.875 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.875 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:41:19.875 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.844946+0000", 2024-09-14T04:41:19.875 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.76e-06 2024-09-14T04:41:19.875 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.875 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.875 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:41:19.875 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.844961+0000", 2024-09-14T04:41:19.875 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5491000000000001e-05 2024-09-14T04:41:19.875 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.875 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.875 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:41:19.875 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.844969+0000", 2024-09-14T04:41:19.875 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6760000000000007e-06 2024-09-14T04:41:19.875 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:19.876 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:41:19.876 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:19.876 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.876 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.876 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:196048 53.8 e1339/1315 53:150fcee6:::benchmark_data_smithi039_224949_object65955:head v 1339'12180, mlcod=1339'12178)", 2024-09-14T04:41:19.876 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:41:19.846720+0000", 2024-09-14T04:41:19.876 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.00072210099999999995, 2024-09-14T04:41:19.876 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001008395, 2024-09-14T04:41:19.876 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:41:19.876 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:41:19.876 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "started", 2024-09-14T04:41:19.876 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:41:19.876 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.877 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:41:19.877 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.846720+0000", 2024-09-14T04:41:19.877 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:19.877 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.877 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.877 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:41:19.877 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.846720+0000", 2024-09-14T04:41:19.877 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:19.877 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.877 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.877 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:41:19.877 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.846721+0000", 2024-09-14T04:41:19.877 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2130000000000001e-06 2024-09-14T04:41:19.877 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.877 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.878 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:41:19.878 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.846732+0000", 2024-09-14T04:41:19.878 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0872e-05 2024-09-14T04:41:19.878 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.878 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.878 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:41:19.878 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.846760+0000", 2024-09-14T04:41:19.878 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.7385e-05 2024-09-14T04:41:19.878 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.878 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.878 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:41:19.878 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.846767+0000", 2024-09-14T04:41:19.878 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7680000000000008e-06 2024-09-14T04:41:19.878 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.879 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.879 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:41:19.879 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.846866+0000", 2024-09-14T04:41:19.879 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.8544999999999999e-05 2024-09-14T04:41:19.879 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.879 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.879 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:41:19.879 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.846889+0000", 2024-09-14T04:41:19.879 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3028999999999999e-05 2024-09-14T04:41:19.879 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:19.879 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:41:19.879 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:19.879 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.879 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.880 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:196040 53.2 e1339/1315)", 2024-09-14T04:41:19.880 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:41:19.846894+0000", 2024-09-14T04:41:19.880 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.00054803899999999995, 2024-09-14T04:41:19.880 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00087590699999999999, 2024-09-14T04:41:19.880 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:41:19.880 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:41:19.880 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "queued for pg", 2024-09-14T04:41:19.880 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:41:19.880 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.880 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:41:19.880 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.846894+0000", 2024-09-14T04:41:19.880 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:19.880 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.880 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.881 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:41:19.881 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.846894+0000", 2024-09-14T04:41:19.881 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:19.881 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.881 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.881 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:41:19.881 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.846895+0000", 2024-09-14T04:41:19.881 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.7700000000000003e-07 2024-09-14T04:41:19.881 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.881 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.881 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:41:19.881 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.846901+0000", 2024-09-14T04:41:19.881 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3619999999999997e-06 2024-09-14T04:41:19.881 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.881 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.882 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:41:19.882 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.846910+0000", 2024-09-14T04:41:19.882 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.8670000000000006e-06 2024-09-14T04:41:19.882 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:19.882 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:19.882 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:41:19.882 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:19.846917+0000", 2024-09-14T04:41:19.882 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3980000000000004e-06 2024-09-14T04:41:19.882 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:19.882 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:41:19.882 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:19.882 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:19.882 INFO:teuthology.orchestra.run.smithi039.stdout: ], 2024-09-14T04:41:19.882 INFO:teuthology.orchestra.run.smithi039.stdout: "num_ops": 7 2024-09-14T04:41:19.883 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:41:19.905 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:41:19.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:19.933 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:19.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:41:20.034 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:20.069 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:20.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:20.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:20.171 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:41:20.205 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:20.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:20.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:41:20.307 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:20.341 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:20.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:20.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:20.443 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:20.489 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:20.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:20.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:20.591 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:20.619 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:20.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:20.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:20.720 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:20.758 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:20.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:20.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:20.860 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:20.899 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:20.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:20.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:21.001 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:21.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:21.033 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:21.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:21.134 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:21.163 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:21.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:21.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:21.265 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:21.309 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:21.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:21.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:21.411 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:21.450 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:21.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:21.451 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:21.551 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:21.588 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:21.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:21.589 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:21.604 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:41:21.690 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:21.717 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:21 smithi039 ceph-mon[106470]: pgmap v2260: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 862 op/s 2024-09-14T04:41:21.718 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:21 smithi039 ceph-mon[111235]: pgmap v2260: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 862 op/s 2024-09-14T04:41:21.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:21.728 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:21.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:21.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:21 smithi129 ceph-mon[101316]: pgmap v2260: 89 pgs: 89 active+clean; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 862 op/s 2024-09-14T04:41:21.829 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:21.874 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:21.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:21.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:21.976 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:22.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:22.018 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:22.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:22.119 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:22.154 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:22.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:22.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:22.256 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:22.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:22.302 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:22.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:22.403 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:22.447 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:22.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:22.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:22.548 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:22.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:22.588 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:22.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:22.689 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:22.717 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:41:22.718 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:41:22.726 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:22.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:22.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:22.756 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-14T04:41:22.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:41:22.828 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:22.859 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:22.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:22.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:22.961 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:22.990 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:22.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:22.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:23.092 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:41:23.126 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:23.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:23.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:41:23.228 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:23.266 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:23.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:23.267 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:23.368 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:23.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:23.404 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:23.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:23.505 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:23.545 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:23.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:23.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:23.646 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:23.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:23.676 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:23.676 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:23.698 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:23 smithi039 ceph-mon[106470]: pgmap v2261: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 775 op/s 2024-09-14T04:41:23.699 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:23 smithi039 ceph-mon[111235]: pgmap v2261: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 775 op/s 2024-09-14T04:41:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:23 smithi129 ceph-mon[101316]: pgmap v2261: 89 pgs: 89 active+clean; 4.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 775 op/s 2024-09-14T04:41:23.777 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:23.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:23.820 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:23.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:23.921 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:23.960 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:23.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:23.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:24.062 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:24.100 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:24.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:24.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:24.202 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:24.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:24.255 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:24.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:24.356 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:41:24.396 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:24.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:24.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:41:24.499 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:24.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:24.543 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:24.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:24.644 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:24.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:24.708 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:24.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:24.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:24 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:41:24.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:24 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:41:24.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:24 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:41:24.809 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:24.848 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:24.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:24.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:24.950 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:24.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:24.992 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:24.993 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:25.093 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:25.129 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:25.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:25.130 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:25.231 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:41:25.264 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:41:25.264 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:41:25.281 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:25.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:25.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:41:25.299 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:41:25.299 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:41:25.331 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:41:25.332 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:41:25.370 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:41:25.370 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:41:25.382 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:25.414 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:25.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:25.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:25.418 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:41:25.418 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:41:25.459 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:41:25.460 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:41:25.465 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:25 smithi039 ceph-mon[106470]: pgmap v2262: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-14T04:41:25.465 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:25 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1724144666' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:41:25.465 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:25 smithi039 ceph-mon[111235]: pgmap v2262: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-14T04:41:25.465 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:25 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1724144666' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:41:25.500 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:41:25.500 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:41:25.516 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:25.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:25 smithi129 ceph-mon[101316]: pgmap v2262: 89 pgs: 89 active+clean; 3.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-14T04:41:25.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:25 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1724144666' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:41:25.546 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:25.546 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:41:25.547 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:41:25.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:25.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:25.648 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:25.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:25.684 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:25.684 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:25.785 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:25.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:25.831 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:25.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:25.932 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:25.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:25.969 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:25.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:26.070 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:26.096 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:26.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:26.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:26.198 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:26.232 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:26.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:26.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:26.334 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:26.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:26.371 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:26.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:26.468 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is unset 2024-09-14T04:41:26.472 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:26.474 INFO:teuthology.orchestra.run.smithi039.stderr:set pool 1 pg_num to 11 2024-09-14T04:41:26.496 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:26.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:26.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:26.532 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:26 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/695894801' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-14T04:41:26.532 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:26 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-14T04:41:26.532 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:26 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2531900258' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:41:26.532 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:26 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/695894801' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-14T04:41:26.532 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:26 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-14T04:41:26.532 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:26 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2531900258' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:41:26.598 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:26.633 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:26.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:26.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:26.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:26 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/695894801' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-14T04:41:26.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:26 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"} : dispatch 2024-09-14T04:41:26.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:26 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2531900258' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:41:26.827 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:26.860 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:26.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:26.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:26.962 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:27.006 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:27.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:27.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:27.108 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:27.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:27.148 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:27.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:27.249 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:27.280 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:27.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:27.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:27.382 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:41:27.421 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:27.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:27.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:41:27.523 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:27.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:27 smithi039 ceph-mon[106470]: pgmap v2263: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 834 op/s 2024-09-14T04:41:27.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:27 smithi039 ceph-mon[106470]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:41:27.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:27 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-09-14T04:41:27.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:27 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2531900258' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:41:27.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:27 smithi039 ceph-mon[106470]: osdmap e1340: 8 total, 8 up, 8 in 2024-09-14T04:41:27.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:27 smithi039 ceph-mon[111235]: pgmap v2263: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 834 op/s 2024-09-14T04:41:27.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:27 smithi039 ceph-mon[111235]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:41:27.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:27 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-09-14T04:41:27.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:27 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2531900258' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:41:27.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:27 smithi039 ceph-mon[111235]: osdmap e1340: 8 total, 8 up, 8 in 2024-09-14T04:41:27.558 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:27.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:27.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:27.660 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:27.696 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:27.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:27.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:27 smithi129 ceph-mon[101316]: pgmap v2263: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 834 op/s 2024-09-14T04:41:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:27 smithi129 ceph-mon[101316]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:41:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:27 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2024-09-14T04:41:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:27 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2531900258' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:41:27.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:27 smithi129 ceph-mon[101316]: osdmap e1340: 8 total, 8 up, 8 in 2024-09-14T04:41:27.798 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:27.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:27.847 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:27.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:27.948 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:27.983 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:27.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:27.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:28.085 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:28.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:28.123 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:28.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:28.224 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:28.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:28.266 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:28.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:28.367 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:28.405 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:28.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:28.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:28.507 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:28.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:28.545 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:28.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:28.646 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:41:28.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:28.697 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:28.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:41:28.741 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:28 smithi039 ceph-mon[106470]: pgmap v2265: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 889 op/s 2024-09-14T04:41:28.742 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:28 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-14T04:41:28.742 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:28 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-14T04:41:28.742 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:41:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:41:28] "GET /metrics HTTP/1.1" 200 36750 "" "Prometheus/2.51.0" 2024-09-14T04:41:28.742 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:28 smithi039 ceph-mon[111235]: pgmap v2265: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 889 op/s 2024-09-14T04:41:28.742 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:28 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-14T04:41:28.743 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:28 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-14T04:41:28.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:28 smithi129 ceph-mon[101316]: pgmap v2265: 89 pgs: 89 active+clean; 3.7 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 889 op/s 2024-09-14T04:41:28.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:28 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-14T04:41:28.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:28 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"} : dispatch 2024-09-14T04:41:28.783 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:41:28.799 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:28.829 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:28.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:28.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:28.931 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:28.967 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:28.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:28.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:29.068 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:29.096 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:29.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:29.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:29.198 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:29.232 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:29.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:29.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:29.265 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:41:29.334 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:29.367 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:29.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:29.368 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:29.468 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:29.512 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:29.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:29.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:29.613 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:29.667 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:29 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-09-14T04:41:29.668 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:29 smithi129 ceph-mon[101316]: osdmap e1341: 8 total, 8 up, 8 in 2024-09-14T04:41:29.686 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:29.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:29.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:29.788 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:29.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:29 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-09-14T04:41:29.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:29 smithi039 ceph-mon[106470]: osdmap e1341: 8 total, 8 up, 8 in 2024-09-14T04:41:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:29 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2024-09-14T04:41:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:29 smithi039 ceph-mon[111235]: osdmap e1341: 8 total, 8 up, 8 in 2024-09-14T04:41:29.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:29.821 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:29.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:29.924 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:29.958 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:29.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:29.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:30.033 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:41:30.060 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:30.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:30.099 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:30.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:30.200 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:30.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:30.245 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:30.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:30.346 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:41:30.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:30.379 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:30.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:41:30.480 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:30.509 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:30.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:30.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:30.536 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:41:30.611 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:30.631 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:30 smithi039 ceph-mon[111235]: osdmap e1342: 8 total, 8 up, 8 in 2024-09-14T04:41:30.631 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:30 smithi039 ceph-mon[111235]: pgmap v2268: 99 pgs: 10 unknown, 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2024-09-14T04:41:30.631 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:30 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:41:30.632 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:30 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:41:30.632 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:30 smithi039 ceph-mon[106470]: osdmap e1342: 8 total, 8 up, 8 in 2024-09-14T04:41:30.632 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:30 smithi039 ceph-mon[106470]: pgmap v2268: 99 pgs: 10 unknown, 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2024-09-14T04:41:30.632 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:30 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:41:30.632 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:30 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:41:30.649 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:30.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:30.650 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:30.752 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:30.789 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:30.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:30.790 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:30.891 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:30.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:30.924 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:30.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:31.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:30 smithi129 ceph-mon[101316]: osdmap e1342: 8 total, 8 up, 8 in 2024-09-14T04:41:31.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:30 smithi129 ceph-mon[101316]: pgmap v2268: 99 pgs: 10 unknown, 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 732 op/s 2024-09-14T04:41:31.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:30 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:41:31.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:30 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:41:31.025 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:31.058 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:31.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:31.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:31.160 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:31.210 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:31.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:31.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:31.314 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:31.346 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:31.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:31.347 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:31.448 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:31.476 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:31.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:31.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:31.578 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:41:31.606 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:31.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:31.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:41:31.649 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:31 smithi039 ceph-mon[106470]: 1.0 scrub starts 2024-09-14T04:41:31.649 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:31 smithi039 ceph-mon[106470]: 1.0 scrub ok 2024-09-14T04:41:31.650 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:31 smithi039 ceph-mon[111235]: 1.0 scrub starts 2024-09-14T04:41:31.650 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:31 smithi039 ceph-mon[111235]: 1.0 scrub ok 2024-09-14T04:41:31.707 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:31.739 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:31.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:31.741 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:31.842 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:31.881 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:31.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:31.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:31.983 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:32.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:31 smithi129 ceph-mon[101316]: 1.0 scrub starts 2024-09-14T04:41:32.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:31 smithi129 ceph-mon[101316]: 1.0 scrub ok 2024-09-14T04:41:32.028 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:32.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:32.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:32.101 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:41:32.101 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:41:32.129 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:32.155 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:32.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:32.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:32.256 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:32.297 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:32.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:32.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:32.340 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:41:32.398 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:32.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:32.439 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:32.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:32.540 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:32.574 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:32.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:32.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:32.676 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:32.716 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:32.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:32.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:32.767 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:32 smithi039 ceph-mon[111235]: pgmap v2269: 99 pgs: 10 unknown, 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-09-14T04:41:32.767 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:32 smithi039 ceph-mon[111235]: 1.3 scrub starts 2024-09-14T04:41:32.767 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:32 smithi039 ceph-mon[111235]: 1.3 scrub ok 2024-09-14T04:41:32.768 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:32 smithi039 ceph-mon[106470]: pgmap v2269: 99 pgs: 10 unknown, 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-09-14T04:41:32.768 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:32 smithi039 ceph-mon[106470]: 1.3 scrub starts 2024-09-14T04:41:32.768 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:32 smithi039 ceph-mon[106470]: 1.3 scrub ok 2024-09-14T04:41:32.804 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-14T04:41:32.818 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:32.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:32.901 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:32.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:33.002 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:33.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:32 smithi129 ceph-mon[101316]: pgmap v2269: 99 pgs: 10 unknown, 89 active+clean; 3.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 716 op/s 2024-09-14T04:41:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:32 smithi129 ceph-mon[101316]: 1.3 scrub starts 2024-09-14T04:41:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:32 smithi129 ceph-mon[101316]: 1.3 scrub ok 2024-09-14T04:41:33.050 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:33.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:33.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:33.152 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:33.180 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:33.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:33.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:33.282 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:33.327 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:33.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:33.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:33.429 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:33.468 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:33.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:33.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:33.571 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:41:33.611 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:33.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:33.612 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:41:33.713 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:33.744 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-14T04:41:33.744 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:41:33.744 INFO:tasks.thrashosds.thrasher:Killing osd 4, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-14T04:41:33.744 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-14T04:41:33.745 DEBUG:teuthology.orchestra.run.smithi129:> sudo systemctl stop ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.4 2024-09-14T04:41:33.749 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:33.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:33.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:33.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:33 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-14T04:41:33.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:33 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-14T04:41:33.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:33 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-14T04:41:33.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:33 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-14T04:41:33.851 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:33.881 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:33.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:33.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:33.983 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:34.021 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:41:33 smithi129 systemd[1]: Stopping Ceph osd.4 for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T04:41:34.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:33 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-14T04:41:34.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:33 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"} : dispatch 2024-09-14T04:41:34.023 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:34.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:34.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:34.125 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:34.161 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:34.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:34.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:34.263 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:34.293 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:34.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:34.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:34.395 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:34.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:34.442 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:34.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:34.521 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:41:34 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4[38530]: 2024-09-14T04:41:34.235+0000 7fcbdcf15640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T04:41:34.521 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:41:34 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4[38530]: 2024-09-14T04:41:34.235+0000 7fcbdcf15640 -1 osd.4 1343 *** Got signal Terminated *** 2024-09-14T04:41:34.521 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:41:34 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4[38530]: 2024-09-14T04:41:34.235+0000 7fcbdcf15640 -1 osd.4 1343 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T04:41:34.543 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:34.589 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:34.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:34.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:34.691 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:34.713 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:34.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:34.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:34.815 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:34.838 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:34.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:34.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:34.912 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:34 smithi039 ceph-mon[106470]: pgmap v2270: 99 pgs: 99 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-14T04:41:34.912 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:34 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-09-14T04:41:34.912 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:34 smithi039 ceph-mon[106470]: osdmap e1343: 8 total, 8 up, 8 in 2024-09-14T04:41:34.912 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:34 smithi039 ceph-mon[106470]: osd.4 marked itself down and dead 2024-09-14T04:41:34.912 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:34 smithi039 ceph-mon[111235]: pgmap v2270: 99 pgs: 99 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-14T04:41:34.913 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:34 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-09-14T04:41:34.913 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:34 smithi039 ceph-mon[111235]: osdmap e1343: 8 total, 8 up, 8 in 2024-09-14T04:41:34.913 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:34 smithi039 ceph-mon[111235]: osd.4 marked itself down and dead 2024-09-14T04:41:34.941 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:34.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:34.964 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:34.964 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:35.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:34 smithi129 ceph-mon[101316]: pgmap v2270: 99 pgs: 99 active+clean; 3.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.08k op/s 2024-09-14T04:41:35.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:34 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "11"}]': finished 2024-09-14T04:41:35.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:34 smithi129 ceph-mon[101316]: osdmap e1343: 8 total, 8 up, 8 in 2024-09-14T04:41:35.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:34 smithi129 ceph-mon[101316]: osd.4 marked itself down and dead 2024-09-14T04:41:35.064 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:35.087 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:35.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:35.088 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:35.189 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:35.206 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:41:35.206 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:41:35.206 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:41:35.207 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:41:35.207 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:41:35.207 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:41:35.207 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:41:35.207 INFO:teuthology.orchestra.run.smithi039.stderr:Error ENXIO: problem getting command descriptions from osd.4 2024-09-14T04:41:35.208 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:41:35.208 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:41:35.208 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:41:35.208 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:41:35.208 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:41:35.208 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:41:35.208 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:41:35.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:35.213 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:35.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:35.314 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:35.335 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:35.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:35.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:35.413 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:41:35.437 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:35.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:35.461 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:35.461 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:35.521 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:41:35 smithi129 podman[145380]: 2024-09-14 04:41:35.250163337 +0000 UTC m=+1.171814247 container died 5b64727311b71ec9c175b7db4b8926dfdb261f3d1acf6536e6a6093296d8ef2d (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=reef-e4e227b) 2024-09-14T04:41:35.562 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:35.584 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:35.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:35.585 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:35.686 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:35.729 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:35.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:35.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:35.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:35 smithi039 ceph-mon[106470]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T04:41:35.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:35 smithi039 ceph-mon[106470]: osdmap e1344: 8 total, 7 up, 8 in 2024-09-14T04:41:35.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:35 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/966631286' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:41:35.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:35 smithi039 ceph-mon[106470]: osdmap e1345: 8 total, 7 up, 8 in 2024-09-14T04:41:35.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:35 smithi039 ceph-mon[111235]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T04:41:35.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:35 smithi039 ceph-mon[111235]: osdmap e1344: 8 total, 7 up, 8 in 2024-09-14T04:41:35.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:35 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/966631286' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:41:35.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:35 smithi039 ceph-mon[111235]: osdmap e1345: 8 total, 7 up, 8 in 2024-09-14T04:41:35.831 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:35.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:35.873 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:35.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:35.909 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:41:35 smithi129 podman[145380]: 2024-09-14 04:41:35.606953357 +0000 UTC m=+1.528604259 container cleanup 5b64727311b71ec9c175b7db4b8926dfdb261f3d1acf6536e6a6093296d8ef2d (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, RELEASE=reef-e4e227b, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2) 2024-09-14T04:41:35.910 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:41:35 smithi129 bash[145380]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4 2024-09-14T04:41:35.910 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:35 smithi129 ceph-mon[101316]: Health check failed: 1 osds down (OSD_DOWN) 2024-09-14T04:41:35.910 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:35 smithi129 ceph-mon[101316]: osdmap e1344: 8 total, 7 up, 8 in 2024-09-14T04:41:35.910 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:35 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/966631286' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:41:35.910 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:35 smithi129 ceph-mon[101316]: osdmap e1345: 8 total, 7 up, 8 in 2024-09-14T04:41:35.974 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:35.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:35.998 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:35.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:36.099 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:36.139 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:36.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:36.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:36.241 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:36.265 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:36.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:36.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:36.270 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:41:35 smithi129 podman[145416]: 2024-09-14 04:41:35.908735692 +0000 UTC m=+0.653688446 container remove 5b64727311b71ec9c175b7db4b8926dfdb261f3d1acf6536e6a6093296d8ef2d (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_CLEAN=True, io.buildah.version=1.37.2, ceph=True, org.label-schema.build-date=20240731, RELEASE=reef-e4e227b, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T04:41:36.367 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:36.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:36.391 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:36.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:36.492 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:36.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:36.521 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:36.522 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:36.589 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:41:36 smithi129 podman[145554]: 2024-09-14 04:41:36.228772575 +0000 UTC m=+0.024484564 image pull 67ddd33fa72f26a51c969a9043e7a3ddf784506b3d1949ad815ae4dd791beec1 quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322 2024-09-14T04:41:36.589 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:41:36 smithi129 podman[145554]: 2024-09-14 04:41:36.335150232 +0000 UTC m=+0.130862219 container create 0a1bcfb89ac46a41df66d69ff2270f873825e8b563c91886a36ce5df109547f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=reef-e4e227b, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-14T04:41:36.623 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:36.652 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:36.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:36.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:36.754 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:36.776 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:36.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:36.777 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:36.879 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:36.905 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:36.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:36.906 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:37.007 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:37.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:36 smithi129 ceph-mon[101316]: pgmap v2273: 99 pgs: 10 peering, 11 stale+active+clean, 78 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 917 op/s 2024-09-14T04:41:37.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:36 smithi129 ceph-mon[101316]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-09-14T04:41:37.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:36 smithi129 ceph-mon[101316]: osdmap e1346: 8 total, 7 up, 8 in 2024-09-14T04:41:37.021 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:41:36 smithi129 podman[145554]: 2024-09-14 04:41:36.588473187 +0000 UTC m=+0.384185189 container init 0a1bcfb89ac46a41df66d69ff2270f873825e8b563c91886a36ce5df109547f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4-deactivate, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , ceph=True, GIT_CLEAN=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=reef-e4e227b, org.label-schema.schema-version=1.0) 2024-09-14T04:41:37.021 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:41:36 smithi129 podman[145554]: 2024-09-14 04:41:36.594516509 +0000 UTC m=+0.390228498 container start 0a1bcfb89ac46a41df66d69ff2270f873825e8b563c91886a36ce5df109547f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, ceph=True, RELEASE=reef-e4e227b, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2) 2024-09-14T04:41:37.021 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:41:36 smithi129 podman[145554]: 2024-09-14 04:41:36.688222815 +0000 UTC m=+0.483934804 container attach 0a1bcfb89ac46a41df66d69ff2270f873825e8b563c91886a36ce5df109547f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-e4e227b, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2) 2024-09-14T04:41:37.021 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:41:36 smithi129 podman[145554]: 2024-09-14 04:41:36.829972139 +0000 UTC m=+0.625684120 container died 0a1bcfb89ac46a41df66d69ff2270f873825e8b563c91886a36ce5df109547f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4-deactivate, RELEASE=reef-e4e227b, io.buildah.version=1.37.2, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-09-14T04:41:37.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:36 smithi039 ceph-mon[106470]: pgmap v2273: 99 pgs: 10 peering, 11 stale+active+clean, 78 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 917 op/s 2024-09-14T04:41:37.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:36 smithi039 ceph-mon[106470]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-09-14T04:41:37.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:36 smithi039 ceph-mon[106470]: osdmap e1346: 8 total, 7 up, 8 in 2024-09-14T04:41:37.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:36 smithi039 ceph-mon[111235]: pgmap v2273: 99 pgs: 10 peering, 11 stale+active+clean, 78 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 917 op/s 2024-09-14T04:41:37.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:36 smithi039 ceph-mon[111235]: Health check failed: Reduced data availability: 4 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-09-14T04:41:37.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:36 smithi039 ceph-mon[111235]: osdmap e1346: 8 total, 7 up, 8 in 2024-09-14T04:41:37.050 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:37.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:37.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:37.152 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:37.184 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:41:37.185 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:41:37.200 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:37.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:37.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:37.302 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:37.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:37.355 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:37.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:37.456 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:37.490 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:37.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:37.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:37.496 DEBUG:teuthology.orchestra.run.smithi129:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.4.service' 2024-09-14T04:41:37.545 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T04:41:37.546 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-14T04:41:37.594 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:37.618 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:37.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:37.619 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:37.720 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:37.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:37.769 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:37.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:37.870 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:37.908 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:37.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:37.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:38.010 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:38.066 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:38.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:38.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:38.168 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:38.214 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:38.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:38.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:37 smithi129 ceph-mon[101316]: osdmap e1347: 8 total, 7 up, 8 in 2024-09-14T04:41:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:41:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:41:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:37 smithi129 ceph-mon[101316]: 1.3 scrub starts 2024-09-14T04:41:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:37 smithi129 ceph-mon[101316]: 1.3 scrub ok 2024-09-14T04:41:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:37 smithi129 ceph-mon[101316]: 1.4 scrub starts 2024-09-14T04:41:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:37 smithi129 ceph-mon[101316]: 1.4 scrub ok 2024-09-14T04:41:38.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:37 smithi039 ceph-mon[106470]: osdmap e1347: 8 total, 7 up, 8 in 2024-09-14T04:41:38.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:41:38.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:41:38.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:37 smithi039 ceph-mon[106470]: 1.3 scrub starts 2024-09-14T04:41:38.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:37 smithi039 ceph-mon[106470]: 1.3 scrub ok 2024-09-14T04:41:38.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:37 smithi039 ceph-mon[106470]: 1.4 scrub starts 2024-09-14T04:41:38.292 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:37 smithi039 ceph-mon[106470]: 1.4 scrub ok 2024-09-14T04:41:38.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:37 smithi039 ceph-mon[111235]: osdmap e1347: 8 total, 7 up, 8 in 2024-09-14T04:41:38.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:41:38.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:41:38.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:37 smithi039 ceph-mon[111235]: 1.3 scrub starts 2024-09-14T04:41:38.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:37 smithi039 ceph-mon[111235]: 1.3 scrub ok 2024-09-14T04:41:38.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:37 smithi039 ceph-mon[111235]: 1.4 scrub starts 2024-09-14T04:41:38.292 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:37 smithi039 ceph-mon[111235]: 1.4 scrub ok 2024-09-14T04:41:38.317 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:38.341 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:38.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:38.342 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:38.442 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:38.483 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:38.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:38.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:38.585 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:38.608 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:38.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:38.609 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:38.710 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:38.755 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:38.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:38.756 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:38.857 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:38.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:38.893 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:38.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:38.927 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:41:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:41:38] "GET /metrics HTTP/1.1" 200 36764 "" "Prometheus/2.51.0" 2024-09-14T04:41:38.994 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:39.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:39.046 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:39.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:39.147 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:39.186 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:41:39.194 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:39.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:39.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:39.217 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:38 smithi039 ceph-mon[106470]: pgmap v2277: 99 pgs: 4 active+undersized, 11 peering, 8 stale+active+clean, 3 active+undersized+degraded, 73 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 411 op/s; 6310/155451 objects degraded (4.059%); 81 KiB/s, 0 objects/s recovering 2024-09-14T04:41:39.217 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:38 smithi039 ceph-mon[106470]: 1.5 scrub starts 2024-09-14T04:41:39.217 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:38 smithi039 ceph-mon[106470]: 1.5 scrub ok 2024-09-14T04:41:39.217 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:38 smithi039 ceph-mon[106470]: 1.8 scrub starts 2024-09-14T04:41:39.217 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:38 smithi039 ceph-mon[106470]: 1.8 scrub ok 2024-09-14T04:41:39.217 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:38 smithi039 ceph-mon[106470]: Health check failed: Degraded data redundancy: 6310/155451 objects degraded (4.059%), 3 pgs degraded (PG_DEGRADED) 2024-09-14T04:41:39.217 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:38 smithi039 ceph-mon[106470]: 1.7 scrub starts 2024-09-14T04:41:39.217 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:38 smithi039 ceph-mon[106470]: 1.7 scrub ok 2024-09-14T04:41:39.218 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:38 smithi039 ceph-mon[111235]: pgmap v2277: 99 pgs: 4 active+undersized, 11 peering, 8 stale+active+clean, 3 active+undersized+degraded, 73 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 411 op/s; 6310/155451 objects degraded (4.059%); 81 KiB/s, 0 objects/s recovering 2024-09-14T04:41:39.218 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:38 smithi039 ceph-mon[111235]: 1.5 scrub starts 2024-09-14T04:41:39.218 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:38 smithi039 ceph-mon[111235]: 1.5 scrub ok 2024-09-14T04:41:39.218 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:38 smithi039 ceph-mon[111235]: 1.8 scrub starts 2024-09-14T04:41:39.218 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:38 smithi039 ceph-mon[111235]: 1.8 scrub ok 2024-09-14T04:41:39.218 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:38 smithi039 ceph-mon[111235]: Health check failed: Degraded data redundancy: 6310/155451 objects degraded (4.059%), 3 pgs degraded (PG_DEGRADED) 2024-09-14T04:41:39.219 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:38 smithi039 ceph-mon[111235]: 1.7 scrub starts 2024-09-14T04:41:39.219 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:38 smithi039 ceph-mon[111235]: 1.7 scrub ok 2024-09-14T04:41:39.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:38 smithi129 ceph-mon[101316]: pgmap v2277: 99 pgs: 4 active+undersized, 11 peering, 8 stale+active+clean, 3 active+undersized+degraded, 73 active+clean; 3.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 411 op/s; 6310/155451 objects degraded (4.059%); 81 KiB/s, 0 objects/s recovering 2024-09-14T04:41:39.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:38 smithi129 ceph-mon[101316]: 1.5 scrub starts 2024-09-14T04:41:39.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:38 smithi129 ceph-mon[101316]: 1.5 scrub ok 2024-09-14T04:41:39.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:38 smithi129 ceph-mon[101316]: 1.8 scrub starts 2024-09-14T04:41:39.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:38 smithi129 ceph-mon[101316]: 1.8 scrub ok 2024-09-14T04:41:39.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:38 smithi129 ceph-mon[101316]: Health check failed: Degraded data redundancy: 6310/155451 objects degraded (4.059%), 3 pgs degraded (PG_DEGRADED) 2024-09-14T04:41:39.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:38 smithi129 ceph-mon[101316]: 1.7 scrub starts 2024-09-14T04:41:39.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:38 smithi129 ceph-mon[101316]: 1.7 scrub ok 2024-09-14T04:41:39.296 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:39.318 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:39.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:39.319 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:39.402 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:41:39.403 INFO:teuthology.orchestra.run.smithi039.stdout: "size": 20, 2024-09-14T04:41:39.403 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 600, 2024-09-14T04:41:39.403 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [ 2024-09-14T04:41:39.403 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.403 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:31587 53.5 53:ad77489c:::benchmark_data_smithi039_224949_object31586:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-14T04:41:39.403 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:38:47.151671+0000", 2024-09-14T04:41:39.403 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 172.24688148300001, 2024-09-14T04:41:39.403 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.099772641999999995, 2024-09-14T04:41:39.403 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:41:39.403 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:41:39.404 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:41:39.404 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:41:39.404 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:41:39.404 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:41:39.404 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 31587 2024-09-14T04:41:39.404 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.404 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:41:39.404 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.404 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:41:39.404 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.151671+0000", 2024-09-14T04:41:39.404 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.404 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.405 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.405 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:41:39.405 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.151671+0000", 2024-09-14T04:41:39.405 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.405 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.405 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.405 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:41:39.405 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.151671+0000", 2024-09-14T04:41:39.405 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.0999999999999999e-07 2024-09-14T04:41:39.405 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.405 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.405 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:41:39.405 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.151693+0000", 2024-09-14T04:41:39.405 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1557999999999999e-05 2024-09-14T04:41:39.406 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.406 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.406 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:41:39.406 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.151700+0000", 2024-09-14T04:41:39.406 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9550000000000003e-06 2024-09-14T04:41:39.406 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.406 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.406 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:41:39.406 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.151705+0000", 2024-09-14T04:41:39.406 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.575e-06 2024-09-14T04:41:39.406 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.406 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.407 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:41:39.407 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.151761+0000", 2024-09-14T04:41:39.407 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5302e-05 2024-09-14T04:41:39.407 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.407 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.407 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:41:39.407 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.151882+0000", 2024-09-14T04:41:39.407 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012132400000000001 2024-09-14T04:41:39.407 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.407 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.407 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:41:39.407 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.152024+0000", 2024-09-14T04:41:39.407 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014214300000000001 2024-09-14T04:41:39.408 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.408 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.408 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:39.408 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.155506+0000", 2024-09-14T04:41:39.408 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0034822579999999998 2024-09-14T04:41:39.408 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.408 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.408 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:41:39.408 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.158596+0000", 2024-09-14T04:41:39.408 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0030897780000000001 2024-09-14T04:41:39.408 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.408 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.409 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:39.409 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.251343+0000", 2024-09-14T04:41:39.409 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.092747034000000006 2024-09-14T04:41:39.409 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.409 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.409 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:41:39.409 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.251403+0000", 2024-09-14T04:41:39.409 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0306999999999997e-05 2024-09-14T04:41:39.409 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.409 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.409 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:41:39.409 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.251443+0000", 2024-09-14T04:41:39.410 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.9898000000000003e-05 2024-09-14T04:41:39.410 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.410 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:41:39.410 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.410 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.410 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.410 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:31588 53.5 53:a90f4fab:::benchmark_data_smithi039_224949_object31587:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-14T04:41:39.410 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:38:47.151718+0000", 2024-09-14T04:41:39.410 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 172.24683378200001, 2024-09-14T04:41:39.410 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.103376312, 2024-09-14T04:41:39.410 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:41:39.410 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:41:39.410 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:41:39.410 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:41:39.410 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:41:39.411 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:41:39.411 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 31588 2024-09-14T04:41:39.411 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.411 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:41:39.411 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.411 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:41:39.411 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.151718+0000", 2024-09-14T04:41:39.411 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.411 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.411 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.411 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:41:39.411 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.151718+0000", 2024-09-14T04:41:39.411 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.412 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.412 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.412 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:41:39.412 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.151719+0000", 2024-09-14T04:41:39.412 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.8599999999999998e-07 2024-09-14T04:41:39.412 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.412 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.412 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:41:39.412 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.151740+0000", 2024-09-14T04:41:39.412 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0871000000000001e-05 2024-09-14T04:41:39.412 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.412 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.412 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:41:39.413 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.151746+0000", 2024-09-14T04:41:39.413 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.72e-06 2024-09-14T04:41:39.413 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.413 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.413 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:41:39.413 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.151753+0000", 2024-09-14T04:41:39.413 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2049999999999999e-06 2024-09-14T04:41:39.413 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.413 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.413 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:41:39.413 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.153501+0000", 2024-09-14T04:41:39.413 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0017482190000000001 2024-09-14T04:41:39.413 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.413 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.414 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:41:39.414 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.153811+0000", 2024-09-14T04:41:39.414 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00031006399999999998 2024-09-14T04:41:39.414 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.414 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.414 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:41:39.414 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.153929+0000", 2024-09-14T04:41:39.414 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000117948 2024-09-14T04:41:39.414 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.414 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.414 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:41:39.414 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.158603+0000", 2024-09-14T04:41:39.414 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0046737640000000004 2024-09-14T04:41:39.414 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.415 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.415 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:39.415 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.210228+0000", 2024-09-14T04:41:39.415 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.051625472999999998 2024-09-14T04:41:39.415 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.415 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.415 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:39.415 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.254938+0000", 2024-09-14T04:41:39.415 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.044709857999999998 2024-09-14T04:41:39.415 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.415 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.415 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:41:39.415 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.255040+0000", 2024-09-14T04:41:39.415 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000101891 2024-09-14T04:41:39.416 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.416 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.416 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:41:39.416 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.255095+0000", 2024-09-14T04:41:39.416 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4812999999999998e-05 2024-09-14T04:41:39.416 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.416 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:41:39.416 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.416 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.416 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.416 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:31594 53.5 53:abeb434f:::benchmark_data_smithi039_224949_object31593:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1319)", 2024-09-14T04:41:39.416 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:38:47.153706+0000", 2024-09-14T04:41:39.416 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 172.24484641699999, 2024-09-14T04:41:39.417 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.101604466, 2024-09-14T04:41:39.417 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:41:39.417 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:41:39.417 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:41:39.417 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:41:39.417 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:41:39.418 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:41:39.418 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 31594 2024-09-14T04:41:39.418 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.418 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:41:39.418 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.418 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:41:39.418 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.153706+0000", 2024-09-14T04:41:39.418 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.418 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.418 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.418 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:41:39.418 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.153706+0000", 2024-09-14T04:41:39.418 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.419 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.419 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.419 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:41:39.419 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.153707+0000", 2024-09-14T04:41:39.419 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3170000000000001e-06 2024-09-14T04:41:39.419 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.419 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.419 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:41:39.419 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.153763+0000", 2024-09-14T04:41:39.419 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6478999999999999e-05 2024-09-14T04:41:39.419 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.419 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.419 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:41:39.420 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.153775+0000", 2024-09-14T04:41:39.420 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1072e-05 2024-09-14T04:41:39.420 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.420 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.420 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:41:39.420 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.153781+0000", 2024-09-14T04:41:39.420 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8920000000000003e-06 2024-09-14T04:41:39.420 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.420 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.420 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:41:39.420 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.155598+0000", 2024-09-14T04:41:39.420 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001816554 2024-09-14T04:41:39.420 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.420 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.421 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:41:39.421 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.155877+0000", 2024-09-14T04:41:39.421 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00027855399999999998 2024-09-14T04:41:39.421 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.421 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.421 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:41:39.421 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.156015+0000", 2024-09-14T04:41:39.421 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000138775 2024-09-14T04:41:39.421 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.421 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.421 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:41:39.421 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.158790+0000", 2024-09-14T04:41:39.421 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.002774964 2024-09-14T04:41:39.421 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.422 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.422 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:39.422 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.210269+0000", 2024-09-14T04:41:39.422 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.051478514000000003 2024-09-14T04:41:39.422 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.422 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.422 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:39.422 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.255145+0000", 2024-09-14T04:41:39.422 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.044876105999999999 2024-09-14T04:41:39.422 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.422 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.422 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:41:39.422 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.255267+0000", 2024-09-14T04:41:39.423 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000122082 2024-09-14T04:41:39.423 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.423 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.423 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:41:39.423 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:47.255310+0000", 2024-09-14T04:41:39.423 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3157e-05 2024-09-14T04:41:39.423 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.423 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:41:39.423 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.423 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.423 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.423 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:73818 53.5 53:a6fdc581:::benchmark_data_smithi039_224949_object73817:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-09-14T04:41:39.423 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:39:12.619036+0000", 2024-09-14T04:41:39.424 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 146.779516567, 2024-09-14T04:41:39.424 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.11966767, 2024-09-14T04:41:39.424 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:41:39.424 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:41:39.424 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:41:39.424 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:41:39.424 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:41:39.424 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:41:39.424 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 73818 2024-09-14T04:41:39.424 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.424 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:41:39.424 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.424 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:41:39.424 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.619036+0000", 2024-09-14T04:41:39.424 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.425 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.425 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.425 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:41:39.425 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.619036+0000", 2024-09-14T04:41:39.425 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.425 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.425 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.425 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:41:39.425 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.619036+0000", 2024-09-14T04:41:39.425 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.2999999999999999e-07 2024-09-14T04:41:39.425 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.425 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.425 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:41:39.426 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.619059+0000", 2024-09-14T04:41:39.426 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3037e-05 2024-09-14T04:41:39.426 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.426 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.426 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:41:39.426 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.619070+0000", 2024-09-14T04:41:39.426 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.079e-05 2024-09-14T04:41:39.426 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.426 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.426 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:41:39.426 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.619077+0000", 2024-09-14T04:41:39.427 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.5119999999999997e-06 2024-09-14T04:41:39.427 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.427 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.427 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:41:39.427 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.625257+0000", 2024-09-14T04:41:39.427 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0061806550000000002 2024-09-14T04:41:39.427 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.427 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.427 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:41:39.427 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.625396+0000", 2024-09-14T04:41:39.427 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013899999999999999 2024-09-14T04:41:39.427 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.427 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.428 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:41:39.428 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.625541+0000", 2024-09-14T04:41:39.428 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014451899999999999 2024-09-14T04:41:39.428 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.428 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.428 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:41:39.428 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.628114+0000", 2024-09-14T04:41:39.428 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0025729609999999999 2024-09-14T04:41:39.428 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.428 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.428 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:39.429 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.738560+0000", 2024-09-14T04:41:39.429 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.11044583099999999 2024-09-14T04:41:39.429 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.429 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.429 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:39.429 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.738604+0000", 2024-09-14T04:41:39.429 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4147999999999998e-05 2024-09-14T04:41:39.429 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.429 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.429 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:41:39.429 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.738659+0000", 2024-09-14T04:41:39.429 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5090000000000003e-05 2024-09-14T04:41:39.429 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.430 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.430 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:41:39.430 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.738703+0000", 2024-09-14T04:41:39.430 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4297e-05 2024-09-14T04:41:39.430 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.430 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:41:39.430 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.430 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.430 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.430 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:73817 53.1 e1322/1315 53:8f66518f:::benchmark_data_smithi039_224949_object73816:head v 1322'4687, mlcod=1322'4685)", 2024-09-14T04:41:39.430 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:39:12.619635+0000", 2024-09-14T04:41:39.430 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 146.778917447, 2024-09-14T04:41:39.430 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.117670881, 2024-09-14T04:41:39.430 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:41:39.431 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:41:39.431 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:41:39.431 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:41:39.431 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.431 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:41:39.431 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.619635+0000", 2024-09-14T04:41:39.431 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.431 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.431 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.432 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:41:39.432 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.619635+0000", 2024-09-14T04:41:39.432 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.432 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.432 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.432 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:41:39.432 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.619635+0000", 2024-09-14T04:41:39.432 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.3600000000000003e-07 2024-09-14T04:41:39.432 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.432 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.432 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:41:39.432 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.619656+0000", 2024-09-14T04:41:39.432 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1032000000000001e-05 2024-09-14T04:41:39.433 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.433 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.433 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:41:39.433 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.619665+0000", 2024-09-14T04:41:39.433 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.833e-06 2024-09-14T04:41:39.433 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.433 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.433 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:41:39.433 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.619671+0000", 2024-09-14T04:41:39.433 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.9070000000000004e-06 2024-09-14T04:41:39.433 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.433 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.434 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:41:39.434 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.621133+0000", 2024-09-14T04:41:39.434 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001462007 2024-09-14T04:41:39.434 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.434 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.434 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:41:39.434 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.621154+0000", 2024-09-14T04:41:39.434 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0336e-05 2024-09-14T04:41:39.434 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.434 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.434 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:41:39.434 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.737260+0000", 2024-09-14T04:41:39.434 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.11610609700000001 2024-09-14T04:41:39.435 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.435 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.435 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:41:39.435 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.737306+0000", 2024-09-14T04:41:39.435 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5933000000000002e-05 2024-09-14T04:41:39.435 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.435 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:41:39.435 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.435 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.435 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.435 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:73819 53.5 53:a813ea22:::benchmark_data_smithi039_224949_object73818:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-09-14T04:41:39.435 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:39:12.620922+0000", 2024-09-14T04:41:39.435 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 146.777630069, 2024-09-14T04:41:39.435 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.117967276, 2024-09-14T04:41:39.436 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:41:39.436 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:41:39.436 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:41:39.436 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:41:39.436 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:41:39.436 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:41:39.436 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 73819 2024-09-14T04:41:39.436 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.436 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:41:39.436 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.436 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:41:39.436 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.620922+0000", 2024-09-14T04:41:39.436 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.437 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.437 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.437 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:41:39.437 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.620922+0000", 2024-09-14T04:41:39.437 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.437 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.437 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.437 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:41:39.437 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.620923+0000", 2024-09-14T04:41:39.437 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.358e-06 2024-09-14T04:41:39.437 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.437 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.437 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:41:39.438 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.620947+0000", 2024-09-14T04:41:39.438 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3895000000000001e-05 2024-09-14T04:41:39.438 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.438 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.438 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:41:39.438 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.620956+0000", 2024-09-14T04:41:39.438 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.9369999999999993e-06 2024-09-14T04:41:39.438 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.438 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.438 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:41:39.438 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.620963+0000", 2024-09-14T04:41:39.438 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3759999999999996e-06 2024-09-14T04:41:39.438 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.439 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.439 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:41:39.439 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.628626+0000", 2024-09-14T04:41:39.439 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0076629419999999998 2024-09-14T04:41:39.439 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.439 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.439 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:41:39.439 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.628756+0000", 2024-09-14T04:41:39.439 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000130044 2024-09-14T04:41:39.439 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.439 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.439 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:41:39.439 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.628900+0000", 2024-09-14T04:41:39.440 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014481100000000001 2024-09-14T04:41:39.440 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.440 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.440 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:41:39.440 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.738517+0000", 2024-09-14T04:41:39.440 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.109616988 2024-09-14T04:41:39.440 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.440 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.440 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:39.440 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.738747+0000", 2024-09-14T04:41:39.440 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00022959099999999999 2024-09-14T04:41:39.441 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.441 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.441 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:39.441 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.738788+0000", 2024-09-14T04:41:39.441 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.1356000000000002e-05 2024-09-14T04:41:39.441 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.441 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.441 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:41:39.441 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.738852+0000", 2024-09-14T04:41:39.441 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.4088e-05 2024-09-14T04:41:39.441 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.441 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.441 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:41:39.441 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.738889+0000", 2024-09-14T04:41:39.442 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.6890000000000001e-05 2024-09-14T04:41:39.442 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.442 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:41:39.442 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.442 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.442 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.442 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:73821 53.9 53:98c9d076:::benchmark_data_smithi039_224949_object73820:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-09-14T04:41:39.442 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:39:12.622445+0000", 2024-09-14T04:41:39.442 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 146.77610747200001, 2024-09-14T04:41:39.442 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.13122450099999999, 2024-09-14T04:41:39.442 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:41:39.443 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:41:39.443 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:41:39.443 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:41:39.443 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:41:39.443 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:41:39.443 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 73821 2024-09-14T04:41:39.443 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.443 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:41:39.443 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.443 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:41:39.443 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.622445+0000", 2024-09-14T04:41:39.443 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.444 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.444 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.444 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:41:39.444 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.622445+0000", 2024-09-14T04:41:39.444 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.444 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.444 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.444 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:41:39.444 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.622446+0000", 2024-09-14T04:41:39.444 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0720000000000001e-06 2024-09-14T04:41:39.444 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.444 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.445 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:41:39.445 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.622472+0000", 2024-09-14T04:41:39.445 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.6656999999999999e-05 2024-09-14T04:41:39.445 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.445 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.445 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:41:39.445 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.622484+0000", 2024-09-14T04:41:39.445 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1199999999999999e-05 2024-09-14T04:41:39.445 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.445 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.445 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:41:39.445 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.622490+0000", 2024-09-14T04:41:39.445 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.4239999999999998e-06 2024-09-14T04:41:39.446 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.446 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.446 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:41:39.446 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.622535+0000", 2024-09-14T04:41:39.446 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5198999999999999e-05 2024-09-14T04:41:39.446 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.446 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.446 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:41:39.446 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.622745+0000", 2024-09-14T04:41:39.446 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00020974399999999999 2024-09-14T04:41:39.446 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.446 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.446 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 4,5", 2024-09-14T04:41:39.447 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.622874+0000", 2024-09-14T04:41:39.447 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012870700000000001 2024-09-14T04:41:39.447 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.447 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.447 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:39.447 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.628335+0000", 2024-09-14T04:41:39.447 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0054617049999999999 2024-09-14T04:41:39.447 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.447 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.447 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:39.447 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:39.449 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.628395+0000", 2024-09-14T04:41:39.449 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0049000000000001e-05 2024-09-14T04:41:39.449 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.449 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.449 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:41:39.449 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.753524+0000", 2024-09-14T04:41:39.449 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.125128407 2024-09-14T04:41:39.449 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.449 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.450 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:41:39.450 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.753621+0000", 2024-09-14T04:41:39.450 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.6989000000000003e-05 2024-09-14T04:41:39.450 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.450 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.450 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:41:39.450 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.753669+0000", 2024-09-14T04:41:39.450 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.8347999999999999e-05 2024-09-14T04:41:39.450 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.450 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:41:39.450 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.450 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.450 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.450 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:73823 53.1 e1322/1315 53:89730e74:::benchmark_data_smithi039_224949_object73822:head v 1322'4688, mlcod=1322'4685)", 2024-09-14T04:41:39.451 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:39:12.626622+0000", 2024-09-14T04:41:39.451 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 146.77192969399999, 2024-09-14T04:41:39.451 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.12933592499999999, 2024-09-14T04:41:39.451 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:41:39.451 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:41:39.451 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:41:39.451 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:41:39.451 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.451 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:41:39.451 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.626622+0000", 2024-09-14T04:41:39.451 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.451 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.451 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.451 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:41:39.452 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.626622+0000", 2024-09-14T04:41:39.452 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.452 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.452 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.452 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:41:39.452 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.626623+0000", 2024-09-14T04:41:39.452 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.4900000000000004e-07 2024-09-14T04:41:39.452 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.452 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.452 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:41:39.452 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.626651+0000", 2024-09-14T04:41:39.452 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.7222000000000001e-05 2024-09-14T04:41:39.453 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.453 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.453 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:41:39.453 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.626661+0000", 2024-09-14T04:41:39.453 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0855e-05 2024-09-14T04:41:39.453 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.453 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.453 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:41:39.453 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.626668+0000", 2024-09-14T04:41:39.453 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8009999999999998e-06 2024-09-14T04:41:39.453 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.453 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.453 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:41:39.453 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.628424+0000", 2024-09-14T04:41:39.454 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001755827 2024-09-14T04:41:39.454 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.454 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.454 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:41:39.454 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.628442+0000", 2024-09-14T04:41:39.454 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7723e-05 2024-09-14T04:41:39.454 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.454 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.454 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:41:39.454 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.755859+0000", 2024-09-14T04:41:39.454 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.12741770499999999 2024-09-14T04:41:39.454 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.454 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.454 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:41:39.455 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.755958+0000", 2024-09-14T04:41:39.455 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.8843000000000002e-05 2024-09-14T04:41:39.455 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.455 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:41:39.455 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.455 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.455 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.455 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:73824 53.8 e1322/1315 53:1c355733:::benchmark_data_smithi039_224949_object73823:head v 1322'4625, mlcod=1322'4622)", 2024-09-14T04:41:39.455 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:39:12.626711+0000", 2024-09-14T04:41:39.455 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 146.77184065599999, 2024-09-14T04:41:39.455 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.109200745, 2024-09-14T04:41:39.455 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:41:39.455 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:41:39.455 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:41:39.456 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:41:39.456 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.456 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:41:39.456 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.626711+0000", 2024-09-14T04:41:39.456 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.456 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.456 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.456 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:41:39.456 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.626711+0000", 2024-09-14T04:41:39.456 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.456 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.456 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.456 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:41:39.457 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.626712+0000", 2024-09-14T04:41:39.457 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.15e-07 2024-09-14T04:41:39.457 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.457 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.457 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:41:39.457 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.626744+0000", 2024-09-14T04:41:39.457 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.1865000000000001e-05 2024-09-14T04:41:39.457 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.457 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.457 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:41:39.457 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.626750+0000", 2024-09-14T04:41:39.457 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.6669999999999999e-06 2024-09-14T04:41:39.457 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.458 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.458 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:41:39.458 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.626755+0000", 2024-09-14T04:41:39.458 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3579999999999996e-06 2024-09-14T04:41:39.458 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.458 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.458 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:41:39.458 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.627011+0000", 2024-09-14T04:41:39.458 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00025667499999999999 2024-09-14T04:41:39.458 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.458 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.458 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:41:39.458 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.627029+0000", 2024-09-14T04:41:39.458 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7631000000000001e-05 2024-09-14T04:41:39.459 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.459 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.459 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:41:39.459 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.735870+0000", 2024-09-14T04:41:39.459 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.108841206 2024-09-14T04:41:39.459 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.459 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.459 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:41:39.459 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.735912+0000", 2024-09-14T04:41:39.459 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.1928000000000002e-05 2024-09-14T04:41:39.459 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.459 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:41:39.459 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.459 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.459 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.460 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:73825 53.8 e1322/1315 53:16961f53:::benchmark_data_smithi039_224949_object73824:head v 1322'4626, mlcod=1322'4622)", 2024-09-14T04:41:39.460 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:39:12.627788+0000", 2024-09-14T04:41:39.461 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 146.77076430400001, 2024-09-14T04:41:39.461 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.108155792, 2024-09-14T04:41:39.461 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:41:39.461 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:41:39.461 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:41:39.461 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:41:39.461 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.461 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:41:39.461 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.627788+0000", 2024-09-14T04:41:39.461 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.461 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.461 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.462 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:41:39.462 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.627788+0000", 2024-09-14T04:41:39.462 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.462 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.462 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.462 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:41:39.462 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.627789+0000", 2024-09-14T04:41:39.462 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.7300000000000005e-07 2024-09-14T04:41:39.462 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.462 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.462 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:41:39.462 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.627821+0000", 2024-09-14T04:41:39.462 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.2021999999999999e-05 2024-09-14T04:41:39.462 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.463 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.463 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:41:39.463 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.627832+0000", 2024-09-14T04:41:39.463 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0823e-05 2024-09-14T04:41:39.463 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.463 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.463 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:41:39.463 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.627838+0000", 2024-09-14T04:41:39.463 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7379999999999997e-06 2024-09-14T04:41:39.463 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.463 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.463 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:41:39.463 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.629288+0000", 2024-09-14T04:41:39.463 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0014495549999999999 2024-09-14T04:41:39.464 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.464 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.464 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:41:39.464 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.629381+0000", 2024-09-14T04:41:39.464 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.344e-05 2024-09-14T04:41:39.464 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.464 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.464 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:41:39.464 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.735916+0000", 2024-09-14T04:41:39.464 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.106534948 2024-09-14T04:41:39.464 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.464 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.464 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:41:39.465 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.735944+0000", 2024-09-14T04:41:39.465 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.7393e-05 2024-09-14T04:41:39.465 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.465 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:41:39.465 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.465 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.465 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.465 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:73828 53.9 53:9cacab43:::benchmark_data_smithi039_224949_object73827:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-09-14T04:41:39.465 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:39:12.628507+0000", 2024-09-14T04:41:39.465 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 146.770045084, 2024-09-14T04:41:39.465 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.12767793399999999, 2024-09-14T04:41:39.465 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:41:39.465 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:41:39.466 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:41:39.466 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:41:39.466 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:41:39.466 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:41:39.466 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 73828 2024-09-14T04:41:39.466 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.466 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:41:39.466 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.466 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:41:39.466 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.628507+0000", 2024-09-14T04:41:39.466 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.466 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.466 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.466 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:41:39.467 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.628507+0000", 2024-09-14T04:41:39.467 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.467 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.467 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.467 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:41:39.467 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.628508+0000", 2024-09-14T04:41:39.467 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.466e-06 2024-09-14T04:41:39.467 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.467 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.467 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:41:39.467 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.628530+0000", 2024-09-14T04:41:39.467 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1569999999999998e-05 2024-09-14T04:41:39.467 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.468 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.468 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:41:39.468 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.628539+0000", 2024-09-14T04:41:39.468 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.0210000000000005e-06 2024-09-14T04:41:39.468 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.468 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.468 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:41:39.468 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.628545+0000", 2024-09-14T04:41:39.468 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.4150000000000004e-06 2024-09-14T04:41:39.468 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.468 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.468 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:41:39.468 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.630464+0000", 2024-09-14T04:41:39.468 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001918665 2024-09-14T04:41:39.469 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.469 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.469 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:41:39.469 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.734578+0000", 2024-09-14T04:41:39.469 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.10411408599999999 2024-09-14T04:41:39.469 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.469 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.469 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 4,5", 2024-09-14T04:41:39.469 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.734747+0000", 2024-09-14T04:41:39.469 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00016916500000000001 2024-09-14T04:41:39.469 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.469 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.469 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:39.470 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.745382+0000", 2024-09-14T04:41:39.470 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.010634719000000001 2024-09-14T04:41:39.470 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.470 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.470 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:39.470 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.745422+0000", 2024-09-14T04:41:39.470 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.0015000000000002e-05 2024-09-14T04:41:39.470 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.470 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.470 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:41:39.471 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.756039+0000", 2024-09-14T04:41:39.471 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.010616626000000001 2024-09-14T04:41:39.471 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.471 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.471 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:41:39.471 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.756141+0000", 2024-09-14T04:41:39.471 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000102638 2024-09-14T04:41:39.471 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.471 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.471 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:41:39.471 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.756185+0000", 2024-09-14T04:41:39.471 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3547999999999997e-05 2024-09-14T04:41:39.471 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.471 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:41:39.471 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.472 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.472 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.472 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:73831 53.5 53:adf88a36:::benchmark_data_smithi039_224949_object73830:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-09-14T04:41:39.472 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:39:12.629011+0000", 2024-09-14T04:41:39.472 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 146.76954104800001, 2024-09-14T04:41:39.472 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.118674189, 2024-09-14T04:41:39.472 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:41:39.472 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:41:39.472 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:41:39.472 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:41:39.472 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:41:39.472 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:41:39.472 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 73831 2024-09-14T04:41:39.473 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.473 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:41:39.473 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.473 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:41:39.473 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.629011+0000", 2024-09-14T04:41:39.473 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.473 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.473 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.473 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:41:39.473 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.629011+0000", 2024-09-14T04:41:39.473 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.473 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.473 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.474 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:41:39.474 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.629012+0000", 2024-09-14T04:41:39.474 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.4600000000000003e-07 2024-09-14T04:41:39.474 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.474 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.474 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:41:39.474 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.629039+0000", 2024-09-14T04:41:39.474 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.7438999999999999e-05 2024-09-14T04:41:39.474 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.474 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.474 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:41:39.474 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.629048+0000", 2024-09-14T04:41:39.474 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.02e-06 2024-09-14T04:41:39.474 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.475 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.475 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:41:39.475 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.629055+0000", 2024-09-14T04:41:39.475 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.5950000000000004e-06 2024-09-14T04:41:39.475 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.475 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.475 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:41:39.475 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.735048+0000", 2024-09-14T04:41:39.475 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.105993294 2024-09-14T04:41:39.475 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.475 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.475 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:41:39.475 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.735191+0000", 2024-09-14T04:41:39.475 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000143017 2024-09-14T04:41:39.476 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.476 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.476 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:41:39.476 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.735331+0000", 2024-09-14T04:41:39.476 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014001899999999999 2024-09-14T04:41:39.476 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.476 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.476 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:41:39.476 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.738942+0000", 2024-09-14T04:41:39.476 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.003611019 2024-09-14T04:41:39.476 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.476 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.476 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:39.476 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.745619+0000", 2024-09-14T04:41:39.477 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0066762979999999998 2024-09-14T04:41:39.477 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.477 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.477 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:39.477 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.747562+0000", 2024-09-14T04:41:39.477 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0019433600000000001 2024-09-14T04:41:39.477 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.477 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.477 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:41:39.477 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.747640+0000", 2024-09-14T04:41:39.477 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7971000000000003e-05 2024-09-14T04:41:39.477 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.477 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.478 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:41:39.478 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.747685+0000", 2024-09-14T04:41:39.478 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5311e-05 2024-09-14T04:41:39.478 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.478 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:41:39.478 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.478 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.478 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.479 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:73832 53.9 53:9a5a1b3a:::benchmark_data_smithi039_224949_object73831:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-09-14T04:41:39.480 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:39:12.630552+0000", 2024-09-14T04:41:39.480 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 146.768000118, 2024-09-14T04:41:39.480 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.12576367799999999, 2024-09-14T04:41:39.480 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:41:39.480 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:41:39.480 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:41:39.480 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:41:39.480 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:41:39.480 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:41:39.480 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 73832 2024-09-14T04:41:39.480 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.481 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:41:39.481 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.481 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:41:39.481 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.630552+0000", 2024-09-14T04:41:39.481 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.481 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.481 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.481 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:41:39.481 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.630552+0000", 2024-09-14T04:41:39.481 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.481 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.481 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.481 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:41:39.482 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.630553+0000", 2024-09-14T04:41:39.482 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.13e-06 2024-09-14T04:41:39.482 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.482 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.482 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:41:39.482 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.630594+0000", 2024-09-14T04:41:39.482 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.0902000000000002e-05 2024-09-14T04:41:39.482 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.482 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.482 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:41:39.482 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.734706+0000", 2024-09-14T04:41:39.482 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.104111487 2024-09-14T04:41:39.482 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.483 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.483 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:41:39.483 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.734721+0000", 2024-09-14T04:41:39.483 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5304000000000001e-05 2024-09-14T04:41:39.483 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.483 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.483 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:41:39.483 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.737297+0000", 2024-09-14T04:41:39.483 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0025764490000000002 2024-09-14T04:41:39.483 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.483 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.483 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:41:39.483 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.737467+0000", 2024-09-14T04:41:39.483 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000169748 2024-09-14T04:41:39.483 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.484 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.484 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 4,5", 2024-09-14T04:41:39.484 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.737922+0000", 2024-09-14T04:41:39.484 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000454711 2024-09-14T04:41:39.484 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.484 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.484 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:39.484 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.745466+0000", 2024-09-14T04:41:39.484 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0075445030000000001 2024-09-14T04:41:39.484 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.485 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.485 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:39.485 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.745510+0000", 2024-09-14T04:41:39.485 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3305e-05 2024-09-14T04:41:39.485 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.485 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.485 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:41:39.485 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.756192+0000", 2024-09-14T04:41:39.485 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.010682844 2024-09-14T04:41:39.485 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.485 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.485 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:41:39.486 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.756272+0000", 2024-09-14T04:41:39.486 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9725e-05 2024-09-14T04:41:39.486 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.486 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.486 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:41:39.486 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.756316+0000", 2024-09-14T04:41:39.486 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3569999999999998e-05 2024-09-14T04:41:39.486 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.486 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:41:39.486 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.486 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.486 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.486 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop_reply(client.45053.0:73818 53.5 e1322/1315 ondisk, result = 0)", 2024-09-14T04:41:39.486 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:39:12.632679+0000", 2024-09-14T04:41:39.486 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 146.765873196, 2024-09-14T04:41:39.487 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.10588924299999999, 2024-09-14T04:41:39.487 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:41:39.487 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:41:39.487 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "started", 2024-09-14T04:41:39.487 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:41:39.487 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.487 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:41:39.487 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.632679+0000", 2024-09-14T04:41:39.487 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.487 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.487 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.488 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:41:39.488 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.632679+0000", 2024-09-14T04:41:39.488 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.488 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.488 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.488 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:41:39.488 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.632680+0000", 2024-09-14T04:41:39.488 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0100000000000004e-07 2024-09-14T04:41:39.488 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.488 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.488 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:41:39.488 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.632684+0000", 2024-09-14T04:41:39.488 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2719999999999999e-06 2024-09-14T04:41:39.488 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.489 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.489 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:41:39.489 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.733648+0000", 2024-09-14T04:41:39.489 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.100963596 2024-09-14T04:41:39.489 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.489 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.489 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:41:39.489 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.733668+0000", 2024-09-14T04:41:39.489 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0397000000000001e-05 2024-09-14T04:41:39.489 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.489 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.489 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:41:39.489 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.738547+0000", 2024-09-14T04:41:39.490 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.004878584 2024-09-14T04:41:39.490 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.490 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.490 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:41:39.490 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.738554+0000", 2024-09-14T04:41:39.490 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.5209999999999997e-06 2024-09-14T04:41:39.490 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.490 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.490 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:41:39.490 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.738568+0000", 2024-09-14T04:41:39.490 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4072e-05 2024-09-14T04:41:39.490 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.490 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:41:39.490 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.491 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.491 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.491 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:74044 53.5 53:a85913f8:::benchmark_data_smithi039_224949_object74043:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-09-14T04:41:39.491 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:39:12.873774+0000", 2024-09-14T04:41:39.491 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 146.524778489, 2024-09-14T04:41:39.491 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.102219587, 2024-09-14T04:41:39.491 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:41:39.491 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:41:39.491 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:41:39.491 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:41:39.491 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:41:39.491 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:41:39.492 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 74044 2024-09-14T04:41:39.492 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.492 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:41:39.492 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.492 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:41:39.492 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.873774+0000", 2024-09-14T04:41:39.492 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.492 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.492 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.493 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:41:39.493 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.873774+0000", 2024-09-14T04:41:39.493 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.493 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.493 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.493 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:41:39.493 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.873775+0000", 2024-09-14T04:41:39.493 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.9000000000000005e-07 2024-09-14T04:41:39.493 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.493 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.493 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:41:39.493 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.873795+0000", 2024-09-14T04:41:39.493 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9949000000000002e-05 2024-09-14T04:41:39.493 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.494 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.494 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:41:39.494 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.873805+0000", 2024-09-14T04:41:39.494 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0562e-05 2024-09-14T04:41:39.494 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.494 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.494 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:41:39.494 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.873812+0000", 2024-09-14T04:41:39.494 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.4670000000000001e-06 2024-09-14T04:41:39.494 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.494 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.494 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:41:39.494 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.874511+0000", 2024-09-14T04:41:39.494 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00069979799999999996 2024-09-14T04:41:39.494 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.495 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.495 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:41:39.495 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.874638+0000", 2024-09-14T04:41:39.495 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012626799999999999 2024-09-14T04:41:39.495 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.495 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.495 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:41:39.495 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.874769+0000", 2024-09-14T04:41:39.495 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013174899999999999 2024-09-14T04:41:39.495 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.496 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.496 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:41:39.496 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.879908+0000", 2024-09-14T04:41:39.496 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0051390510000000004 2024-09-14T04:41:39.496 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.496 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.496 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:39.496 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.969782+0000", 2024-09-14T04:41:39.496 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.089873117000000002 2024-09-14T04:41:39.496 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.496 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.496 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:39.496 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.975880+0000", 2024-09-14T04:41:39.496 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0060980080000000002 2024-09-14T04:41:39.497 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.497 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.497 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:41:39.497 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.975944+0000", 2024-09-14T04:41:39.497 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.4318000000000003e-05 2024-09-14T04:41:39.497 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.497 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.497 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:41:39.497 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.975993+0000", 2024-09-14T04:41:39.497 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9310000000000001e-05 2024-09-14T04:41:39.497 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.497 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:41:39.497 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.497 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.498 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.498 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:74048 53.5 53:aadb1272:::benchmark_data_smithi039_224949_object74047:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-09-14T04:41:39.498 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:39:12.877227+0000", 2024-09-14T04:41:39.498 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 146.52132466200001, 2024-09-14T04:41:39.498 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.098993697000000005, 2024-09-14T04:41:39.498 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:41:39.498 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:41:39.498 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:41:39.498 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:41:39.498 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:41:39.498 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:41:39.498 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 74048 2024-09-14T04:41:39.498 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.499 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:41:39.499 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.499 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:41:39.499 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.877227+0000", 2024-09-14T04:41:39.499 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.499 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.499 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.499 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:41:39.499 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.877227+0000", 2024-09-14T04:41:39.499 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.499 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.499 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.499 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:41:39.499 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.877228+0000", 2024-09-14T04:41:39.499 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5900000000000002e-07 2024-09-14T04:41:39.500 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.500 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.500 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:41:39.500 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.877250+0000", 2024-09-14T04:41:39.500 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2368999999999999e-05 2024-09-14T04:41:39.500 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.500 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.500 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:41:39.500 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.877257+0000", 2024-09-14T04:41:39.500 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1570000000000003e-06 2024-09-14T04:41:39.500 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.500 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.500 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:41:39.500 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.877263+0000", 2024-09-14T04:41:39.500 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4269999999999999e-06 2024-09-14T04:41:39.501 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.501 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.501 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:41:39.501 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.877449+0000", 2024-09-14T04:41:39.501 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00018616600000000001 2024-09-14T04:41:39.501 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.501 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.501 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:41:39.501 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.877631+0000", 2024-09-14T04:41:39.501 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000181743 2024-09-14T04:41:39.501 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.501 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.501 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:41:39.501 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.877775+0000", 2024-09-14T04:41:39.502 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000144275 2024-09-14T04:41:39.502 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.502 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.502 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:41:39.502 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.880033+0000", 2024-09-14T04:41:39.502 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0022583999999999998 2024-09-14T04:41:39.502 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.502 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.502 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:39.502 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.969832+0000", 2024-09-14T04:41:39.502 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.089798065999999996 2024-09-14T04:41:39.502 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.502 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.503 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:39.503 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.976143+0000", 2024-09-14T04:41:39.503 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0063113839999999997 2024-09-14T04:41:39.503 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.503 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.503 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:41:39.503 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.976187+0000", 2024-09-14T04:41:39.503 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4530000000000002e-05 2024-09-14T04:41:39.503 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.503 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.503 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:41:39.503 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.976221+0000", 2024-09-14T04:41:39.503 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.3720999999999998e-05 2024-09-14T04:41:39.503 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.504 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:41:39.504 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.504 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.504 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.504 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:76194 53.5 53:afa54a9e:::benchmark_data_smithi039_224949_object76193:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-09-14T04:41:39.504 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:39:14.583819+0000", 2024-09-14T04:41:39.504 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 144.81473339600001, 2024-09-14T04:41:39.504 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.10545776799999999, 2024-09-14T04:41:39.504 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:41:39.504 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:41:39.504 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:41:39.504 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:41:39.505 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:41:39.505 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:41:39.505 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 76194 2024-09-14T04:41:39.505 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.505 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:41:39.505 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.505 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:41:39.505 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.583819+0000", 2024-09-14T04:41:39.505 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.505 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.505 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.505 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:41:39.505 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.583819+0000", 2024-09-14T04:41:39.506 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.506 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.506 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.506 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:41:39.506 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.583821+0000", 2024-09-14T04:41:39.506 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9099999999999999e-06 2024-09-14T04:41:39.506 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.506 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.506 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:41:39.506 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.583841+0000", 2024-09-14T04:41:39.506 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9953999999999999e-05 2024-09-14T04:41:39.506 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.506 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.507 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:41:39.507 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.583852+0000", 2024-09-14T04:41:39.507 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1343e-05 2024-09-14T04:41:39.507 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.507 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.507 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:41:39.507 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.583858+0000", 2024-09-14T04:41:39.507 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.9560000000000002e-06 2024-09-14T04:41:39.507 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.507 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.507 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:41:39.507 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.583909+0000", 2024-09-14T04:41:39.507 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.0943000000000003e-05 2024-09-14T04:41:39.507 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.507 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.507 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:41:39.508 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.583994+0000", 2024-09-14T04:41:39.508 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.4800000000000001e-05 2024-09-14T04:41:39.508 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.508 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.508 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:41:39.508 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.584075+0000", 2024-09-14T04:41:39.508 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0980000000000001e-05 2024-09-14T04:41:39.508 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.508 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.508 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:41:39.508 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.586610+0000", 2024-09-14T04:41:39.508 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0025353749999999999 2024-09-14T04:41:39.508 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.509 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.509 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:39.509 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.586795+0000", 2024-09-14T04:41:39.509 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00018456000000000001 2024-09-14T04:41:39.509 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.509 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.509 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:39.509 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.689189+0000", 2024-09-14T04:41:39.509 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.10239401300000001 2024-09-14T04:41:39.509 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.509 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.510 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:41:39.510 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.689236+0000", 2024-09-14T04:41:39.510 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7160000000000002e-05 2024-09-14T04:41:39.510 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.510 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.510 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:41:39.510 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.689276+0000", 2024-09-14T04:41:39.510 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.0774e-05 2024-09-14T04:41:39.510 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.510 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:41:39.510 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.510 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.510 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.510 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:76203 53.5 53:ab1a3df3:::benchmark_data_smithi039_224949_object76202:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-09-14T04:41:39.511 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:39:14.592211+0000", 2024-09-14T04:41:39.511 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 144.80634134499999, 2024-09-14T04:41:39.511 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.097639940999999994, 2024-09-14T04:41:39.511 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:41:39.511 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:41:39.511 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:41:39.511 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:41:39.511 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:41:39.511 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:41:39.511 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 76203 2024-09-14T04:41:39.511 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.511 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:41:39.511 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.512 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:41:39.512 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.592211+0000", 2024-09-14T04:41:39.512 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.512 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.512 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.512 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:41:39.512 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.592211+0000", 2024-09-14T04:41:39.512 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.512 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.512 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.512 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:41:39.512 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.592212+0000", 2024-09-14T04:41:39.512 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7700000000000004e-07 2024-09-14T04:41:39.513 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.513 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.513 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:41:39.513 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.592230+0000", 2024-09-14T04:41:39.513 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8587999999999999e-05 2024-09-14T04:41:39.513 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.513 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.513 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:41:39.513 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.592239+0000", 2024-09-14T04:41:39.513 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.3000000000000007e-06 2024-09-14T04:41:39.513 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.513 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.513 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:41:39.513 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.592244+0000", 2024-09-14T04:41:39.514 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9540000000000003e-06 2024-09-14T04:41:39.514 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.514 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.514 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:41:39.514 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.592314+0000", 2024-09-14T04:41:39.514 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9765999999999996e-05 2024-09-14T04:41:39.514 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.514 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.514 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:41:39.514 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.681211+0000", 2024-09-14T04:41:39.514 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.088896817000000003 2024-09-14T04:41:39.514 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.514 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.514 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:41:39.515 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.681393+0000", 2024-09-14T04:41:39.515 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000181831 2024-09-14T04:41:39.515 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.515 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.515 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:41:39.515 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.689094+0000", 2024-09-14T04:41:39.515 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0077015759999999999 2024-09-14T04:41:39.515 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.515 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.515 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:39.515 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.689148+0000", 2024-09-14T04:41:39.515 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3865e-05 2024-09-14T04:41:39.515 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.515 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.515 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:39.515 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.689629+0000", 2024-09-14T04:41:39.516 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000480667 2024-09-14T04:41:39.516 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.516 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.516 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:41:39.516 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.689804+0000", 2024-09-14T04:41:39.516 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00017496800000000001 2024-09-14T04:41:39.516 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.516 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.516 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:41:39.516 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:14.689851+0000", 2024-09-14T04:41:39.516 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6832000000000002e-05 2024-09-14T04:41:39.516 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.516 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:41:39.516 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.517 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.517 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.517 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:134487 53.5 53:a91d6404:::benchmark_data_smithi039_224949_object4394:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1330)", 2024-09-14T04:41:39.517 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:40:10.600863+0000", 2024-09-14T04:41:39.517 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 88.797688780000001, 2024-09-14T04:41:39.517 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.111584462, 2024-09-14T04:41:39.517 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:41:39.517 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:41:39.517 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:41:39.517 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:41:39.517 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:41:39.517 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:41:39.517 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 134487 2024-09-14T04:41:39.517 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.518 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:41:39.518 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.518 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:41:39.518 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:10.600863+0000", 2024-09-14T04:41:39.518 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.518 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.518 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.518 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:41:39.518 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:10.600863+0000", 2024-09-14T04:41:39.518 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.518 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.518 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.518 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:41:39.518 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:10.600864+0000", 2024-09-14T04:41:39.518 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4299999999999998e-07 2024-09-14T04:41:39.518 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.518 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.519 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:41:39.519 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:10.600866+0000", 2024-09-14T04:41:39.520 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.182e-06 2024-09-14T04:41:39.520 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.520 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.520 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:41:39.520 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:10.600872+0000", 2024-09-14T04:41:39.520 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3249999999999996e-06 2024-09-14T04:41:39.521 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.521 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.521 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:41:39.521 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:10.600877+0000", 2024-09-14T04:41:39.521 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.8980000000000004e-06 2024-09-14T04:41:39.521 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.521 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.521 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:41:39.521 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:10.606408+0000", 2024-09-14T04:41:39.521 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0055303349999999999 2024-09-14T04:41:39.521 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.521 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.521 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:41:39.521 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:10.606514+0000", 2024-09-14T04:41:39.521 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000106711 2024-09-14T04:41:39.522 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.522 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.522 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:41:39.522 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:10.606619+0000", 2024-09-14T04:41:39.522 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00010488 2024-09-14T04:41:39.522 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.522 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.522 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:41:39.522 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:10.614448+0000", 2024-09-14T04:41:39.522 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0078294000000000002 2024-09-14T04:41:39.522 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.522 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.522 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:39.522 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:10.615719+0000", 2024-09-14T04:41:39.522 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001270167 2024-09-14T04:41:39.523 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.523 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.523 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:39.523 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:10.712288+0000", 2024-09-14T04:41:39.523 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.096569784000000006 2024-09-14T04:41:39.523 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.523 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.523 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:41:39.523 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:10.712396+0000", 2024-09-14T04:41:39.523 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000107876 2024-09-14T04:41:39.523 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:39.524 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:39.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:39.525 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.525 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:41:39.525 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:10.712448+0000", 2024-09-14T04:41:39.525 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1461000000000001e-05 2024-09-14T04:41:39.525 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.525 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:41:39.525 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.525 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.525 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.525 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:134516 53.5 53:a0630957:::benchmark_data_smithi039_224949_object4423:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1330)", 2024-09-14T04:41:39.525 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:40:10.631028+0000", 2024-09-14T04:41:39.525 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 88.767523750999999, 2024-09-14T04:41:39.525 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.100465073, 2024-09-14T04:41:39.526 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:41:39.526 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:41:39.526 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:41:39.526 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:41:39.526 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:41:39.526 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:41:39.526 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 134516 2024-09-14T04:41:39.526 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.526 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:41:39.527 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.527 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:41:39.527 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:10.631028+0000", 2024-09-14T04:41:39.527 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.527 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.527 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.527 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:41:39.527 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:10.631028+0000", 2024-09-14T04:41:39.527 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:41:39.527 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.528 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.528 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:41:39.528 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:10.631030+0000", 2024-09-14T04:41:39.528 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6029999999999999e-06 2024-09-14T04:41:39.528 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.528 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.528 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:41:39.528 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:10.631035+0000", 2024-09-14T04:41:39.528 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3020000000000002e-06 2024-09-14T04:41:39.528 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.528 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.528 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:41:39.528 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:10.631049+0000", 2024-09-14T04:41:39.528 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3737000000000001e-05 2024-09-14T04:41:39.529 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.529 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.529 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:41:39.529 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:10.631057+0000", 2024-09-14T04:41:39.529 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0679999999999997e-06 2024-09-14T04:41:39.529 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.529 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.529 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:41:39.529 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:10.631129+0000", 2024-09-14T04:41:39.529 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1929999999999997e-05 2024-09-14T04:41:39.529 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.529 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.529 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:41:39.529 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:10.631284+0000", 2024-09-14T04:41:39.530 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000155017 2024-09-14T04:41:39.530 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.530 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.530 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:41:39.530 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:10.631391+0000", 2024-09-14T04:41:39.530 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00010706500000000001 2024-09-14T04:41:39.530 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.530 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.530 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:41:39.530 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:10.637528+0000", 2024-09-14T04:41:39.530 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0061369620000000001 2024-09-14T04:41:39.530 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.530 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.531 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:39.531 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:10.639336+0000", 2024-09-14T04:41:39.531 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0018084290000000001 2024-09-14T04:41:39.531 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.531 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.531 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:41:39.531 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:10.731091+0000", 2024-09-14T04:41:39.531 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.091754207000000004 2024-09-14T04:41:39.531 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.531 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.531 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:41:39.531 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:10.731335+0000", 2024-09-14T04:41:39.532 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00024409900000000001 2024-09-14T04:41:39.532 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:41:39.532 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:41:39.532 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:41:39.532 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:10.731493+0000", 2024-09-14T04:41:39.532 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015865399999999999 2024-09-14T04:41:39.532 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.532 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:41:39.532 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.532 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:41:39.532 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:41:39.533 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:41:39.625 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:39.647 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:39.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:39.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:39.748 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:39.772 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:39.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:39.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:39.874 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:39.898 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:39.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:39.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:40.000 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:40.041 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:40.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:40.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:40.142 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:40.166 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:40.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:40.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:40.268 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:40.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:39 smithi129 ceph-mon[101316]: 1.1 scrub starts 2024-09-14T04:41:40.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:39 smithi129 ceph-mon[101316]: 1.1 scrub ok 2024-09-14T04:41:40.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:39 smithi129 ceph-mon[101316]: 1.6 scrub starts 2024-09-14T04:41:40.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:39 smithi129 ceph-mon[101316]: 1.6 scrub ok 2024-09-14T04:41:40.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:39 smithi039 ceph-mon[106470]: 1.1 scrub starts 2024-09-14T04:41:40.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:39 smithi039 ceph-mon[106470]: 1.1 scrub ok 2024-09-14T04:41:40.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:39 smithi039 ceph-mon[106470]: 1.6 scrub starts 2024-09-14T04:41:40.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:39 smithi039 ceph-mon[106470]: 1.6 scrub ok 2024-09-14T04:41:40.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:39 smithi039 ceph-mon[111235]: 1.1 scrub starts 2024-09-14T04:41:40.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:39 smithi039 ceph-mon[111235]: 1.1 scrub ok 2024-09-14T04:41:40.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:39 smithi039 ceph-mon[111235]: 1.6 scrub starts 2024-09-14T04:41:40.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:39 smithi039 ceph-mon[111235]: 1.6 scrub ok 2024-09-14T04:41:40.306 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:40.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:40.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:40.409 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:40.433 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:40.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:40.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:40.535 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:40.577 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:40.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:40.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:40.678 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:40.714 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:40.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:40.715 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:40.816 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:40.839 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:40.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:40.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:40.941 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:40.978 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:40.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:40.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:41.080 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:41.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:41.120 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:41.120 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:41.221 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:41.262 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:41.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:41.263 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:41.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:40 smithi129 ceph-mon[101316]: pgmap v2278: 99 pgs: 1 active+clean+remapped, 16 active+undersized, 11 peering, 18 active+undersized+degraded, 53 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 27445/148035 objects degraded (18.540%); 1/148035 objects misplaced (0.001%); 33 KiB/s, 0 objects/s recovering 2024-09-14T04:41:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:40 smithi129 ceph-mon[101316]: 1.a scrub starts 2024-09-14T04:41:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:40 smithi129 ceph-mon[101316]: 1.a scrub ok 2024-09-14T04:41:41.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:40 smithi039 ceph-mon[106470]: pgmap v2278: 99 pgs: 1 active+clean+remapped, 16 active+undersized, 11 peering, 18 active+undersized+degraded, 53 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 27445/148035 objects degraded (18.540%); 1/148035 objects misplaced (0.001%); 33 KiB/s, 0 objects/s recovering 2024-09-14T04:41:41.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:40 smithi039 ceph-mon[106470]: 1.a scrub starts 2024-09-14T04:41:41.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:40 smithi039 ceph-mon[106470]: 1.a scrub ok 2024-09-14T04:41:41.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:40 smithi039 ceph-mon[111235]: pgmap v2278: 99 pgs: 1 active+clean+remapped, 16 active+undersized, 11 peering, 18 active+undersized+degraded, 53 active+clean; 3.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 828 op/s; 27445/148035 objects degraded (18.540%); 1/148035 objects misplaced (0.001%); 33 KiB/s, 0 objects/s recovering 2024-09-14T04:41:41.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:40 smithi039 ceph-mon[111235]: 1.a scrub starts 2024-09-14T04:41:41.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:40 smithi039 ceph-mon[111235]: 1.a scrub ok 2024-09-14T04:41:41.364 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:41.386 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:41.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:41.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:41.488 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:41.525 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:41.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:41.526 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:41.626 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:41.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:41.670 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:41.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:41.771 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:41.793 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:41.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:41.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:41.895 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:41.917 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:41.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:41.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:42.019 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:42.041 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:42.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:42.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:42.142 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:42.182 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:42.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:42.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:42.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:41 smithi129 ceph-mon[101316]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2024-09-14T04:41:42.284 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:42.289 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:41 smithi039 ceph-mon[106470]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2024-09-14T04:41:42.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:41 smithi039 ceph-mon[111235]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2024-09-14T04:41:42.326 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:42.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:42.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:42.427 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:42.450 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:42.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:42.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:42.455 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-14T04:41:42.546 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [4] live_osds: [0, 1, 2, 3, 5, 6, 7] 2024-09-14T04:41:42.547 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:41:42.547 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2024-09-14T04:41:42.547 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight-by-utilization 110 1.0 2024-09-14T04:41:42.552 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:42.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:42.646 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:42.646 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:42.747 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:42.769 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:42.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:42.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:42.871 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:42.923 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:42.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:42.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:43.025 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:43.027 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is unset 2024-09-14T04:41:43.080 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:43.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:43.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:43.182 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:43.217 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:43.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:43.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:43.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:43 smithi129 ceph-mon[101316]: pgmap v2279: 99 pgs: 3 active+clean+remapped, 24 active+undersized, 1 peering, 20 active+undersized+degraded, 51 active+clean; 3.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 33377/145263 objects degraded (22.977%); 1/145263 objects misplaced (0.001%); 943 B/s, 0 objects/s recovering 2024-09-14T04:41:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:43 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/537374734' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:41:43.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:43 smithi039 ceph-mon[106470]: pgmap v2279: 99 pgs: 3 active+clean+remapped, 24 active+undersized, 1 peering, 20 active+undersized+degraded, 51 active+clean; 3.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 33377/145263 objects degraded (22.977%); 1/145263 objects misplaced (0.001%); 943 B/s, 0 objects/s recovering 2024-09-14T04:41:43.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:43 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/537374734' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:41:43.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:43 smithi039 ceph-mon[111235]: pgmap v2279: 99 pgs: 3 active+clean+remapped, 24 active+undersized, 1 peering, 20 active+undersized+degraded, 51 active+clean; 3.0 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 639 op/s; 33377/145263 objects degraded (22.977%); 1/145263 objects misplaced (0.001%); 943 B/s, 0 objects/s recovering 2024-09-14T04:41:43.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:43 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/537374734' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:41:43.319 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:43.362 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:43.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:43.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:43.465 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:43.501 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:43.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:43.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:43.603 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:43.644 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:43.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:43.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:43.746 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:43.770 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:43.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:43.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:43.872 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:43.920 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:43.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:43.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:43.962 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:41:44.023 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:44.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:44.081 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:44.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:44.182 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:44.208 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:44.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:44.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:44.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:44 smithi129 ceph-mon[101316]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:41:44.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:44 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/537374734' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:41:44.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:44 smithi129 ceph-mon[101316]: osdmap e1348: 8 total, 7 up, 8 in 2024-09-14T04:41:44.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:44 smithi039 ceph-mon[106470]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:41:44.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:44 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/537374734' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:41:44.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:44 smithi039 ceph-mon[106470]: osdmap e1348: 8 total, 7 up, 8 in 2024-09-14T04:41:44.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:44 smithi039 ceph-mon[111235]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:41:44.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:44 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/537374734' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:41:44.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:44 smithi039 ceph-mon[111235]: osdmap e1348: 8 total, 7 up, 8 in 2024-09-14T04:41:44.310 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:44.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:44.351 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:44.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:44.379 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:41:44.453 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:44.481 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:41:44.505 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:44.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:44.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:44.607 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:44.652 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:44.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:44.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:44.754 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:44.776 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:44.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:44.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:44.878 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:44.901 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:44.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:44.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:45.003 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:45.018 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:45 smithi039 ceph-mon[106470]: pgmap v2281: 99 pgs: 3 active+clean+remapped, 24 active+undersized, 20 active+undersized+degraded, 52 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 937 op/s; 30808/134955 objects degraded (22.828%); 1/134955 objects misplaced (0.001%) 2024-09-14T04:41:45.018 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:45 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 30808/134955 objects degraded (22.828%), 20 pgs degraded (PG_DEGRADED) 2024-09-14T04:41:45.049 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:45.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:45.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:45.151 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:45.173 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:45.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:45.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:45.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:45 smithi129 ceph-mon[101316]: pgmap v2281: 99 pgs: 3 active+clean+remapped, 24 active+undersized, 20 active+undersized+degraded, 52 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 937 op/s; 30808/134955 objects degraded (22.828%); 1/134955 objects misplaced (0.001%) 2024-09-14T04:41:45.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:45 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 30808/134955 objects degraded (22.828%), 20 pgs degraded (PG_DEGRADED) 2024-09-14T04:41:45.275 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:45.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:45 smithi039 ceph-mon[111235]: pgmap v2281: 99 pgs: 3 active+clean+remapped, 24 active+undersized, 20 active+undersized+degraded, 52 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 937 op/s; 30808/134955 objects degraded (22.828%); 1/134955 objects misplaced (0.001%) 2024-09-14T04:41:45.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:45 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 30808/134955 objects degraded (22.828%), 20 pgs degraded (PG_DEGRADED) 2024-09-14T04:41:45.299 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:45.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:45.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:45.401 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:45.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:45.441 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:45.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:45.543 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:45.567 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:45.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:45.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:45.670 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:45.710 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:45.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:45.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:45.812 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:45.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:45.860 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:45.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:45.961 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:45.984 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:45.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:45.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:46.086 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:46.109 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:46.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:46.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:46.211 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:46.249 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:46.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:46.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:46.351 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:46.376 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:46.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:46.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:46.413 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:41:46.477 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:46.522 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:46.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:46.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:46.624 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:46.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:46.679 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:46.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:46.748 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:46 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:41:46.748 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:46 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:41:46.748 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:46 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:41:46.748 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:46 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:41:46.748 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:46 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:41:46.749 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:46 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:41:46.749 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:46 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:41:46.749 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:46 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:41:46.749 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:46 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:41:46.749 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:46 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:41:46.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:46 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:41:46.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:46 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:41:46.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:46 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:41:46.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:46 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:41:46.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:46 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:41:46.780 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:46.818 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:46.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:46.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:46.920 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:46.955 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:46.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:46.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:47.057 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:47.101 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:47.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:47.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:47.203 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:47.242 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:47.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:47.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:47.344 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:47.369 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:47.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:47.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:47.471 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:47.508 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:47.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:47.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:47.610 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:47.654 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:47.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:47.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:47.756 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:47.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:47 smithi129 ceph-mon[101316]: pgmap v2282: 99 pgs: 3 active+clean+remapped, 24 active+undersized, 20 active+undersized+degraded, 52 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 911 op/s; 30221/131400 objects degraded (22.999%); 1/131400 objects misplaced (0.001%) 2024-09-14T04:41:47.779 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:47.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:47.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:47.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:47 smithi039 ceph-mon[106470]: pgmap v2282: 99 pgs: 3 active+clean+remapped, 24 active+undersized, 20 active+undersized+degraded, 52 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 911 op/s; 30221/131400 objects degraded (22.999%); 1/131400 objects misplaced (0.001%) 2024-09-14T04:41:47.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:47 smithi039 ceph-mon[111235]: pgmap v2282: 99 pgs: 3 active+clean+remapped, 24 active+undersized, 20 active+undersized+degraded, 52 active+clean; 2.7 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 911 op/s; 30221/131400 objects degraded (22.999%); 1/131400 objects misplaced (0.001%) 2024-09-14T04:41:47.882 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:47.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:47.921 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:47.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:48.022 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:48.057 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:48.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:48.059 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:48.159 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:48.182 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:48.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:48.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:48.283 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:48.307 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:48.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:48.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:48.408 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:48.453 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:48.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:48.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:48.555 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:48.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:48.603 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:48.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:48.704 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:48.726 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:48.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:48.728 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:48.759 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:48 smithi039 ceph-mon[106470]: pgmap v2283: 99 pgs: 3 active+clean+remapped, 24 active+undersized, 20 active+undersized+degraded, 52 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 29652/129693 objects degraded (22.863%); 1/129693 objects misplaced (0.001%) 2024-09-14T04:41:48.760 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:41:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:41:48] "GET /metrics HTTP/1.1" 200 36764 "" "Prometheus/2.51.0" 2024-09-14T04:41:48.760 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:48 smithi039 ceph-mon[111235]: pgmap v2283: 99 pgs: 3 active+clean+remapped, 24 active+undersized, 20 active+undersized+degraded, 52 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 29652/129693 objects degraded (22.863%); 1/129693 objects misplaced (0.001%) 2024-09-14T04:41:48.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:48 smithi129 ceph-mon[101316]: pgmap v2283: 99 pgs: 3 active+clean+remapped, 24 active+undersized, 20 active+undersized+degraded, 52 active+clean; 2.6 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 29652/129693 objects degraded (22.863%); 1/129693 objects misplaced (0.001%) 2024-09-14T04:41:48.829 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:48.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:48.881 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:48.882 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:48.982 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:49.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:49.034 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:49.035 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:49.135 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:49.157 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:49.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:49.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:49.259 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:49.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:49.288 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:49.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:49.389 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:49.431 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:49.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:49.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:49.532 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:49.559 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:49.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:49.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:49.661 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:49.707 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:49.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:49.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:49.809 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:49.833 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:49.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:49.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:49.935 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:49.971 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:49.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:49.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:50.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:49 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 27520/120603 objects degraded (22.819%), 20 pgs degraded (PG_DEGRADED) 2024-09-14T04:41:50.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:49 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 27520/120603 objects degraded (22.819%), 20 pgs degraded (PG_DEGRADED) 2024-09-14T04:41:50.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:49 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 27520/120603 objects degraded (22.819%), 20 pgs degraded (PG_DEGRADED) 2024-09-14T04:41:50.072 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:50.135 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:50.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:50.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:50.237 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:50.259 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:50.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:50.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:50.360 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:50.377 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:41:50.377 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:41:50.387 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:41:50.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:50.412 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:50.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:50.419 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:41:50.419 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:41:50.454 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:41:50.454 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:41:50.485 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:41:50.485 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:41:50.489 INFO:teuthology.orchestra.run.smithi039.stderr:Error ENXIO: problem getting command descriptions from osd.4 2024-09-14T04:41:50.514 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:50.526 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:41:50.526 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:41:50.566 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:50.566 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:41:50.566 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:41:50.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:50.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:50.607 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:41:50.608 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:41:50.670 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:50.694 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:50.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:50.695 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:50.796 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:50.841 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:50.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:50.842 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:50.943 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:50.971 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:50 smithi039 ceph-mon[106470]: pgmap v2284: 99 pgs: 3 active+clean+remapped, 24 active+undersized, 20 active+undersized+degraded, 52 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 902 op/s; 27520/120603 objects degraded (22.819%); 1/120603 objects misplaced (0.001%) 2024-09-14T04:41:50.972 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:50 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1313959773' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:41:50.972 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:50 smithi039 ceph-mon[111235]: pgmap v2284: 99 pgs: 3 active+clean+remapped, 24 active+undersized, 20 active+undersized+degraded, 52 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 902 op/s; 27520/120603 objects degraded (22.819%); 1/120603 objects misplaced (0.001%) 2024-09-14T04:41:50.972 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:50 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1313959773' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:41:50.980 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:50.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:50.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:51.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:50 smithi129 ceph-mon[101316]: pgmap v2284: 99 pgs: 3 active+clean+remapped, 24 active+undersized, 20 active+undersized+degraded, 52 active+clean; 2.5 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 902 op/s; 27520/120603 objects degraded (22.819%); 1/120603 objects misplaced (0.001%) 2024-09-14T04:41:51.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:50 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1313959773' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:41:51.083 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:51.108 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:51.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:51.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:51.210 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:51.232 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:51.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:51.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:51.334 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:51.369 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:51.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:51.370 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:51.471 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:51.494 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:51.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:51.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:51.596 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:51.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:51.635 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:51.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:51.701 INFO:teuthology.orchestra.run.smithi039.stdout:moved 18 / 297 (6.06061%) 2024-09-14T04:41:51.702 INFO:teuthology.orchestra.run.smithi039.stdout:avg 42.4286 2024-09-14T04:41:51.702 INFO:teuthology.orchestra.run.smithi039.stdout:stddev 8.20279 -> 7.17706 (expected baseline 6.03053) 2024-09-14T04:41:51.702 INFO:teuthology.orchestra.run.smithi039.stdout:min osd.2 with 28 -> 37 pgs (0.659933 -> 0.872054 * mean) 2024-09-14T04:41:51.702 INFO:teuthology.orchestra.run.smithi039.stdout:max osd.1 with 43 -> 30 pgs (1.01347 -> 0.707071 * mean) 2024-09-14T04:41:51.702 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:41:51.702 INFO:teuthology.orchestra.run.smithi039.stdout:oload 110 2024-09-14T04:41:51.702 INFO:teuthology.orchestra.run.smithi039.stdout:max_change 1 2024-09-14T04:41:51.702 INFO:teuthology.orchestra.run.smithi039.stdout:max_change_osds 4 2024-09-14T04:41:51.702 INFO:teuthology.orchestra.run.smithi039.stdout:average_utilization 0.0135 2024-09-14T04:41:51.702 INFO:teuthology.orchestra.run.smithi039.stdout:overload_utilization 0.0148 2024-09-14T04:41:51.702 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4 weight 1.0000 -> 0.4794 2024-09-14T04:41:51.702 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1 weight 1.0000 -> 0.6410 2024-09-14T04:41:51.702 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5 weight 1.0000 -> 0.9052 2024-09-14T04:41:51.736 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:51.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:51.758 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:51.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:51.859 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:51.882 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:51.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:51.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:51.983 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:52.004 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:52.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:52.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:52.018 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:51 smithi039 ceph-mon[106470]: 1.2 scrub starts 2024-09-14T04:41:52.018 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:51 smithi039 ceph-mon[106470]: 1.2 scrub ok 2024-09-14T04:41:52.018 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:51 smithi039 ceph-mon[106470]: from='client.45245 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:41:52.018 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:51 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'42009','4':'31417','5':'59323'}"} : dispatch 2024-09-14T04:41:52.018 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:51 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'42009','4':'31417','5':'59323'}"} : dispatch 2024-09-14T04:41:52.018 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:51 smithi039 ceph-mon[111235]: 1.2 scrub starts 2024-09-14T04:41:52.019 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:51 smithi039 ceph-mon[111235]: 1.2 scrub ok 2024-09-14T04:41:52.019 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:51 smithi039 ceph-mon[111235]: from='client.45245 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:41:52.019 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:51 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'42009','4':'31417','5':'59323'}"} : dispatch 2024-09-14T04:41:52.019 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:51 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'42009','4':'31417','5':'59323'}"} : dispatch 2024-09-14T04:41:52.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:51 smithi129 ceph-mon[101316]: 1.2 scrub starts 2024-09-14T04:41:52.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:51 smithi129 ceph-mon[101316]: 1.2 scrub ok 2024-09-14T04:41:52.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:51 smithi129 ceph-mon[101316]: from='client.45245 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:41:52.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:51 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'42009','4':'31417','5':'59323'}"} : dispatch 2024-09-14T04:41:52.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:51 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'42009','4':'31417','5':'59323'}"} : dispatch 2024-09-14T04:41:52.023 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:41:52.023 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:41:52.106 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:52.129 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:52.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:52.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:52.230 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:52.253 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:52.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:52.254 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:52.355 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:52.378 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:52.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:52.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:52.479 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:52.503 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:52.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:52.504 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:52.605 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:52.629 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:52.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:52.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:52.730 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:52.753 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:52.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:52.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:52.855 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:52.878 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:52.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:52.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:52.979 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:53.002 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:53.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:53.003 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:53.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:52 smithi129 ceph-mon[101316]: pgmap v2285: 99 pgs: 3 active+clean+remapped, 24 active+undersized, 20 active+undersized+degraded, 52 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 937 op/s; 26909/116826 objects degraded (23.033%); 1/116826 objects misplaced (0.001%) 2024-09-14T04:41:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:52 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'42009','4':'31417','5':'59323'}"}]': finished 2024-09-14T04:41:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:52 smithi129 ceph-mon[101316]: osdmap e1349: 8 total, 7 up, 8 in 2024-09-14T04:41:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:52 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:41:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:41:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:52 smithi039 ceph-mon[106470]: pgmap v2285: 99 pgs: 3 active+clean+remapped, 24 active+undersized, 20 active+undersized+degraded, 52 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 937 op/s; 26909/116826 objects degraded (23.033%); 1/116826 objects misplaced (0.001%) 2024-09-14T04:41:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:52 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'42009','4':'31417','5':'59323'}"}]': finished 2024-09-14T04:41:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:52 smithi039 ceph-mon[106470]: osdmap e1349: 8 total, 7 up, 8 in 2024-09-14T04:41:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:52 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:41:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:41:53.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:52 smithi039 ceph-mon[111235]: pgmap v2285: 99 pgs: 3 active+clean+remapped, 24 active+undersized, 20 active+undersized+degraded, 52 active+clean; 2.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 937 op/s; 26909/116826 objects degraded (23.033%); 1/116826 objects misplaced (0.001%) 2024-09-14T04:41:53.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:52 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'42009','4':'31417','5':'59323'}"}]': finished 2024-09-14T04:41:53.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:52 smithi039 ceph-mon[111235]: osdmap e1349: 8 total, 7 up, 8 in 2024-09-14T04:41:53.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:52 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:41:53.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:41:53.104 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:53.131 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:53.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:53.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:53.233 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:53.260 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:53.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:53.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:53.362 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:53.384 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:53.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:53.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:53.486 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:53.509 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:53.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:53.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:53.611 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:53.638 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:53.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:53.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:53.740 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:53.775 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:53.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:53.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:53.809 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:41:53.810 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:41:53.878 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:53.907 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:53.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:53.908 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:54.009 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:54.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:53 smithi129 ceph-mon[101316]: osdmap e1350: 8 total, 7 up, 8 in 2024-09-14T04:41:54.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:53 smithi129 ceph-mon[101316]: osdmap e1351: 8 total, 7 up, 8 in 2024-09-14T04:41:54.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:53 smithi039 ceph-mon[106470]: osdmap e1350: 8 total, 7 up, 8 in 2024-09-14T04:41:54.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:53 smithi039 ceph-mon[106470]: osdmap e1351: 8 total, 7 up, 8 in 2024-09-14T04:41:54.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:53 smithi039 ceph-mon[111235]: osdmap e1350: 8 total, 7 up, 8 in 2024-09-14T04:41:54.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:53 smithi039 ceph-mon[111235]: osdmap e1351: 8 total, 7 up, 8 in 2024-09-14T04:41:54.045 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:54.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:54.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:54.147 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:54.190 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:54.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:54.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:54.292 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:54.351 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-14T04:41:54.354 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:54.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:54.356 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:54.457 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:54.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:54.518 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:54.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:54.619 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:54.645 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:54.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:54.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:54.747 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:54.775 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:54.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:54.776 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:54.876 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:54.902 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:54.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:54.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:55.004 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:55.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:54 smithi129 ceph-mon[101316]: pgmap v2288: 99 pgs: 9 remapped+peering, 8 peering, 2 active+clean+remapped, 15 active+undersized, 10 active+undersized+degraded, 55 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 962 op/s; 9221/108234 objects degraded (8.520%); 0 B/s, 1 keys/s, 1 objects/s recovering 2024-09-14T04:41:55.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:54 smithi129 ceph-mon[101316]: 53.2 starting backfill to osd.6 from (0'0,0'0] MAX to 1348'14043 2024-09-14T04:41:55.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:54 smithi129 ceph-mon[101316]: 53.b starting backfill to osd.0 from (0'0,0'0] MAX to 1348'13898 2024-09-14T04:41:55.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:54 smithi129 ceph-mon[101316]: 53.e starting backfill to osd.0 from (0'0,0'0] MAX to 1348'13786 2024-09-14T04:41:55.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:54 smithi129 ceph-mon[101316]: 53.f starting backfill to osd.3 from (0'0,0'0] MAX to 1348'13858 2024-09-14T04:41:55.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:54 smithi129 ceph-mon[101316]: 53.a starting backfill to osd.2 from (0'0,0'0] MAX to 1348'14178 2024-09-14T04:41:55.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:54 smithi129 ceph-mon[101316]: 53.1 starting backfill to osd.6 from (0'0,0'0] MAX to 1348'14143 2024-09-14T04:41:55.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:54 smithi129 ceph-mon[101316]: 53.9 starting backfill to osd.6 from (0'0,0'0] MAX to 1348'14061 2024-09-14T04:41:55.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:54 smithi129 ceph-mon[101316]: 53.8 starting backfill to osd.7 from (0'0,0'0] MAX to 1348'13965 2024-09-14T04:41:55.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:54 smithi129 ceph-mon[101316]: 53.a starting backfill to osd.6 from (0'0,0'0] MAX to 1348'14178 2024-09-14T04:41:55.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:54 smithi129 ceph-mon[101316]: 53.b starting backfill to osd.7 from (0'0,0'0] MAX to 1348'13898 2024-09-14T04:41:55.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:54 smithi129 ceph-mon[101316]: 53.3 starting backfill to osd.0 from (0'0,0'0] MAX to 1348'14092 2024-09-14T04:41:55.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:54 smithi129 ceph-mon[101316]: 53.3 starting backfill to osd.2 from (0'0,0'0] MAX to 1348'14092 2024-09-14T04:41:55.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:54 smithi129 ceph-mon[101316]: 1.9 scrub starts 2024-09-14T04:41:55.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:54 smithi129 ceph-mon[101316]: 1.9 scrub ok 2024-09-14T04:41:55.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:54 smithi129 ceph-mon[101316]: osdmap e1352: 8 total, 7 up, 8 in 2024-09-14T04:41:55.032 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:55.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:55.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:55.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[106470]: pgmap v2288: 99 pgs: 9 remapped+peering, 8 peering, 2 active+clean+remapped, 15 active+undersized, 10 active+undersized+degraded, 55 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 962 op/s; 9221/108234 objects degraded (8.520%); 0 B/s, 1 keys/s, 1 objects/s recovering 2024-09-14T04:41:55.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[106470]: 53.2 starting backfill to osd.6 from (0'0,0'0] MAX to 1348'14043 2024-09-14T04:41:55.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[106470]: 53.b starting backfill to osd.0 from (0'0,0'0] MAX to 1348'13898 2024-09-14T04:41:55.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[106470]: 53.e starting backfill to osd.0 from (0'0,0'0] MAX to 1348'13786 2024-09-14T04:41:55.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[106470]: 53.f starting backfill to osd.3 from (0'0,0'0] MAX to 1348'13858 2024-09-14T04:41:55.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[106470]: 53.a starting backfill to osd.2 from (0'0,0'0] MAX to 1348'14178 2024-09-14T04:41:55.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[106470]: 53.1 starting backfill to osd.6 from (0'0,0'0] MAX to 1348'14143 2024-09-14T04:41:55.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[106470]: 53.9 starting backfill to osd.6 from (0'0,0'0] MAX to 1348'14061 2024-09-14T04:41:55.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[106470]: 53.8 starting backfill to osd.7 from (0'0,0'0] MAX to 1348'13965 2024-09-14T04:41:55.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[106470]: 53.a starting backfill to osd.6 from (0'0,0'0] MAX to 1348'14178 2024-09-14T04:41:55.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[106470]: 53.b starting backfill to osd.7 from (0'0,0'0] MAX to 1348'13898 2024-09-14T04:41:55.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[106470]: 53.3 starting backfill to osd.0 from (0'0,0'0] MAX to 1348'14092 2024-09-14T04:41:55.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[106470]: 53.3 starting backfill to osd.2 from (0'0,0'0] MAX to 1348'14092 2024-09-14T04:41:55.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[106470]: 1.9 scrub starts 2024-09-14T04:41:55.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[106470]: 1.9 scrub ok 2024-09-14T04:41:55.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[106470]: osdmap e1352: 8 total, 7 up, 8 in 2024-09-14T04:41:55.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[111235]: pgmap v2288: 99 pgs: 9 remapped+peering, 8 peering, 2 active+clean+remapped, 15 active+undersized, 10 active+undersized+degraded, 55 active+clean; 2.2 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 962 op/s; 9221/108234 objects degraded (8.520%); 0 B/s, 1 keys/s, 1 objects/s recovering 2024-09-14T04:41:55.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[111235]: 53.2 starting backfill to osd.6 from (0'0,0'0] MAX to 1348'14043 2024-09-14T04:41:55.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[111235]: 53.b starting backfill to osd.0 from (0'0,0'0] MAX to 1348'13898 2024-09-14T04:41:55.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[111235]: 53.e starting backfill to osd.0 from (0'0,0'0] MAX to 1348'13786 2024-09-14T04:41:55.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[111235]: 53.f starting backfill to osd.3 from (0'0,0'0] MAX to 1348'13858 2024-09-14T04:41:55.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[111235]: 53.a starting backfill to osd.2 from (0'0,0'0] MAX to 1348'14178 2024-09-14T04:41:55.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[111235]: 53.1 starting backfill to osd.6 from (0'0,0'0] MAX to 1348'14143 2024-09-14T04:41:55.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[111235]: 53.9 starting backfill to osd.6 from (0'0,0'0] MAX to 1348'14061 2024-09-14T04:41:55.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[111235]: 53.8 starting backfill to osd.7 from (0'0,0'0] MAX to 1348'13965 2024-09-14T04:41:55.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[111235]: 53.a starting backfill to osd.6 from (0'0,0'0] MAX to 1348'14178 2024-09-14T04:41:55.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[111235]: 53.b starting backfill to osd.7 from (0'0,0'0] MAX to 1348'13898 2024-09-14T04:41:55.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[111235]: 53.3 starting backfill to osd.0 from (0'0,0'0] MAX to 1348'14092 2024-09-14T04:41:55.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[111235]: 53.3 starting backfill to osd.2 from (0'0,0'0] MAX to 1348'14092 2024-09-14T04:41:55.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[111235]: 1.9 scrub starts 2024-09-14T04:41:55.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[111235]: 1.9 scrub ok 2024-09-14T04:41:55.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:54 smithi039 ceph-mon[111235]: osdmap e1352: 8 total, 7 up, 8 in 2024-09-14T04:41:55.123 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2024-09-14T04:41:55.124 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight-by-utilization 110 1.0 2024-09-14T04:41:55.134 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:55.231 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:55.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:55.232 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:55.333 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:55.397 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:55.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:55.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:55.499 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:55.523 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:55.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:55.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:55.624 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:55.652 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:55.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:55.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:55.662 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:41:55.754 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:55.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:55.802 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:55.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:55.811 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:41:55.904 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:55.929 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:55.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:55.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:56.031 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:56.081 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:56.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:56.082 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:56.183 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:56.208 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:56.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:56.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:56.310 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:56.335 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:56.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:56.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:56.437 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:56.485 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:56.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:56.486 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:56.586 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:56.610 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:56.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:56.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:56.712 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:56.736 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:56.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:56.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:56.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:56 smithi039 ceph-mon[106470]: pgmap v2291: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 9 remapped+peering, 11 peering, 10 active+undersized, 9 active+undersized+degraded, 59 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 874 op/s; 8887/106320 objects degraded (8.359%); 65 KiB/s, 2 keys/s, 2 objects/s recovering 2024-09-14T04:41:56.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:56 smithi039 ceph-mon[106470]: osdmap e1353: 8 total, 7 up, 8 in 2024-09-14T04:41:56.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:56 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 8887/106320 objects degraded (8.359%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:41:56.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:56 smithi039 ceph-mon[111235]: pgmap v2291: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 9 remapped+peering, 11 peering, 10 active+undersized, 9 active+undersized+degraded, 59 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 874 op/s; 8887/106320 objects degraded (8.359%); 65 KiB/s, 2 keys/s, 2 objects/s recovering 2024-09-14T04:41:56.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:56 smithi039 ceph-mon[111235]: osdmap e1353: 8 total, 7 up, 8 in 2024-09-14T04:41:56.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:56 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 8887/106320 objects degraded (8.359%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:41:56.838 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:56.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:56.865 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:56.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:56.966 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:57.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:57.006 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:57.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:57.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:56 smithi129 ceph-mon[101316]: pgmap v2291: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 9 remapped+peering, 11 peering, 10 active+undersized, 9 active+undersized+degraded, 59 active+clean; 2.2 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 874 op/s; 8887/106320 objects degraded (8.359%); 65 KiB/s, 2 keys/s, 2 objects/s recovering 2024-09-14T04:41:57.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:56 smithi129 ceph-mon[101316]: osdmap e1353: 8 total, 7 up, 8 in 2024-09-14T04:41:57.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:56 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 8887/106320 objects degraded (8.359%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:41:57.106 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:57.132 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:57.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:57.133 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:57.234 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:57.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:57.292 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:57.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:57.393 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:57.439 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:57.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:57.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:57.541 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:57.604 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:57.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:57.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:57.706 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:57.758 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:57.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:57.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:57.798 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:41:57.860 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:57.920 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:57.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:57.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:58.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:57 smithi129 ceph-mon[101316]: 1.8 scrub starts 2024-09-14T04:41:58.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:57 smithi129 ceph-mon[101316]: 1.8 scrub ok 2024-09-14T04:41:58.022 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:41:58.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:57 smithi039 ceph-mon[106470]: 1.8 scrub starts 2024-09-14T04:41:58.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:57 smithi039 ceph-mon[106470]: 1.8 scrub ok 2024-09-14T04:41:58.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:57 smithi039 ceph-mon[111235]: 1.8 scrub starts 2024-09-14T04:41:58.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:57 smithi039 ceph-mon[111235]: 1.8 scrub ok 2024-09-14T04:41:58.059 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:58.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:58.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:41:58.161 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:58.191 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:58.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:58.192 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:58.293 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:58.317 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:58.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:58.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:58.419 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:58.464 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:58.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:58.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:58.566 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:58.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:58.613 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:58.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:58.715 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:58.742 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is set 2024-09-14T04:41:58.752 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:58.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:58.754 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:58.854 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:58.881 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:58.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:58.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:58.983 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:41:59.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:58 smithi129 ceph-mon[101316]: pgmap v2293: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 9 remapped+peering, 1 active+undersized+degraded+remapped+backfilling, 6 peering, 10 active+undersized, 9 active+undersized+degraded, 63 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 495 op/s; 10700/105513 objects degraded (10.141%); 1947/105513 objects misplaced (1.845%); 2.9 MiB/s, 2 keys/s, 48 objects/s recovering 2024-09-14T04:41:59.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:58 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1993449095' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:41:59.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:58 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:41:59.025 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:59.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:59.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:41:59.035 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:58 smithi039 ceph-mon[106470]: pgmap v2293: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 9 remapped+peering, 1 active+undersized+degraded+remapped+backfilling, 6 peering, 10 active+undersized, 9 active+undersized+degraded, 63 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 495 op/s; 10700/105513 objects degraded (10.141%); 1947/105513 objects misplaced (1.845%); 2.9 MiB/s, 2 keys/s, 48 objects/s recovering 2024-09-14T04:41:59.035 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:58 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1993449095' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:41:59.035 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:58 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:41:59.036 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:41:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:41:58] "GET /metrics HTTP/1.1" 200 36795 "" "Prometheus/2.51.0" 2024-09-14T04:41:59.036 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:58 smithi039 ceph-mon[111235]: pgmap v2293: 99 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 9 remapped+peering, 1 active+undersized+degraded+remapped+backfilling, 6 peering, 10 active+undersized, 9 active+undersized+degraded, 63 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 495 op/s; 10700/105513 objects degraded (10.141%); 1947/105513 objects misplaced (1.845%); 2.9 MiB/s, 2 keys/s, 48 objects/s recovering 2024-09-14T04:41:59.036 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:58 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1993449095' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:41:59.036 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:58 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:41:59.128 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:41:59.177 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:59.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:59.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:41:59.215 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:41:59.215 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:41:59.280 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:41:59.305 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:59.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:59.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:41:59.407 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:59.430 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:59.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:59.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:59.532 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:41:59.574 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:41:59.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:59.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:41:59.676 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:41:59.700 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:59.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:59.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:41:59.802 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:59.828 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:59.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:59.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:41:59.930 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:41:59.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:41:59.960 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:41:59.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:00.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:59 smithi039 ceph-mon[106470]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:42:00.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:59 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:42:00.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:41:59 smithi039 ceph-mon[106470]: osdmap e1354: 8 total, 7 up, 8 in 2024-09-14T04:42:00.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:59 smithi039 ceph-mon[111235]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:42:00.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:59 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:42:00.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:41:59 smithi039 ceph-mon[111235]: osdmap e1354: 8 total, 7 up, 8 in 2024-09-14T04:42:00.061 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:00.111 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:00.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:00.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:00.134 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-14T04:42:00.213 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:00.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:00.264 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:00.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:00.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:59 smithi129 ceph-mon[101316]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:42:00.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:59 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:42:00.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:41:59 smithi129 ceph-mon[101316]: osdmap e1354: 8 total, 7 up, 8 in 2024-09-14T04:42:00.337 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:42:00.366 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:00.395 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:00.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:00.396 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:00.497 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:00.524 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:00.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:00.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:00.626 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:00.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:00.669 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:00.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:00.770 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:00.797 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:00.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:00.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:00.899 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:00.941 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:00.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:00.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:01.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:00 smithi129 ceph-mon[101316]: pgmap v2295: 99 pgs: 3 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 3 peering, 1 active+remapped+backfill_wait, 10 active+undersized, 9 active+undersized+degraded, 65 active+clean; 2.0 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 21827/99591 objects degraded (21.917%); 7971/99591 objects misplaced (8.004%); 10 MiB/s, 161 objects/s recovering 2024-09-14T04:42:01.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:00 smithi039 ceph-mon[106470]: pgmap v2295: 99 pgs: 3 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 3 peering, 1 active+remapped+backfill_wait, 10 active+undersized, 9 active+undersized+degraded, 65 active+clean; 2.0 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 21827/99591 objects degraded (21.917%); 7971/99591 objects misplaced (8.004%); 10 MiB/s, 161 objects/s recovering 2024-09-14T04:42:01.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:00 smithi039 ceph-mon[111235]: pgmap v2295: 99 pgs: 3 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 3 peering, 1 active+remapped+backfill_wait, 10 active+undersized, 9 active+undersized+degraded, 65 active+clean; 2.0 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 491 op/s; 21827/99591 objects degraded (21.917%); 7971/99591 objects misplaced (8.004%); 10 MiB/s, 161 objects/s recovering 2024-09-14T04:42:01.043 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:01.070 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:01.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:01.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:01.172 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:01.197 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:01.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:01.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:01.299 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:01.323 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:01.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:01.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:01.425 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:01.452 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:01.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:01.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:01.554 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:01.610 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:01.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:01.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:01.712 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:01.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:01.737 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:01.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:01.838 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:01.881 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:01.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:01.882 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:01.983 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:02.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:01 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 21827/99591 objects degraded (21.917%), 17 pgs degraded (PG_DEGRADED) 2024-09-14T04:42:02.030 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:02.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:02.031 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:02.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:01 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 21827/99591 objects degraded (21.917%), 17 pgs degraded (PG_DEGRADED) 2024-09-14T04:42:02.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:01 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 21827/99591 objects degraded (21.917%), 17 pgs degraded (PG_DEGRADED) 2024-09-14T04:42:02.132 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:02.176 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:02.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:02.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:02.278 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:02.321 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:02.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:02.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:02.422 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:02.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:02.450 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:02.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:02.551 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:02.579 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:02.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:02.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:02.681 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:02.721 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:02.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:02.722 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:02.774 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:42:02.823 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:02.884 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:02.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:02.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:02.986 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:03.037 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:03.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:03.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:03.139 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:03.167 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:03.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:03.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:03.254 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:02 smithi039 ceph-mon[106470]: pgmap v2296: 99 pgs: 3 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 10 active+undersized, 9 active+undersized+degraded, 68 active+clean; 2.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 21346/96573 objects degraded (22.103%); 7971/96573 objects misplaced (8.254%); 8.6 MiB/s, 138 objects/s recovering 2024-09-14T04:42:03.258 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:02 smithi039 ceph-mon[111235]: pgmap v2296: 99 pgs: 3 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 10 active+undersized, 9 active+undersized+degraded, 68 active+clean; 2.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 21346/96573 objects degraded (22.103%); 7971/96573 objects misplaced (8.254%); 8.6 MiB/s, 138 objects/s recovering 2024-09-14T04:42:03.269 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:03.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:02 smithi129 ceph-mon[101316]: pgmap v2296: 99 pgs: 3 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 10 active+undersized, 9 active+undersized+degraded, 68 active+clean; 2.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 564 op/s; 21346/96573 objects degraded (22.103%); 7971/96573 objects misplaced (8.254%); 8.6 MiB/s, 138 objects/s recovering 2024-09-14T04:42:03.320 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:03.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:03.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:03.423 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:03.449 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:03.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:03.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:03.550 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:03.613 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:03.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:03.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:03.714 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:03.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:03.755 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:03.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:03.856 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:03.909 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:03.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:03.910 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:04.011 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:04.047 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:04.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:04.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:04.149 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:04.174 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:04.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:04.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:04.205 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:42:04.276 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:04.334 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:04.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:04.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:04.436 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:04.483 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:04.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:04.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:04.585 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:04.586 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:42:04.586 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:42:04.634 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:04.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:04.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:04.636 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:42:04.636 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:42:04.682 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:42:04.682 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:42:04.724 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:42:04.724 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:42:04.729 INFO:teuthology.orchestra.run.smithi039.stderr:Error ENXIO: problem getting command descriptions from osd.4 2024-09-14T04:42:04.736 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:04.765 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:42:04.765 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:42:04.782 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:04.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:04.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:04.798 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:42:04.799 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:42:04.815 INFO:teuthology.orchestra.run.smithi039.stdout:moved 9 / 297 (3.0303%) 2024-09-14T04:42:04.816 INFO:teuthology.orchestra.run.smithi039.stdout:avg 42.4286 2024-09-14T04:42:04.816 INFO:teuthology.orchestra.run.smithi039.stdout:stddev 7.17706 -> 9.0283 (expected baseline 6.03053) 2024-09-14T04:42:04.816 INFO:teuthology.orchestra.run.smithi039.stdout:min osd.1 with 30 -> 23 pgs (0.707071 -> 0.542088 * mean) 2024-09-14T04:42:04.816 INFO:teuthology.orchestra.run.smithi039.stdout:max osd.3 with 48 -> 50 pgs (1.13131 -> 1.17845 * mean) 2024-09-14T04:42:04.817 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:42:04.817 INFO:teuthology.orchestra.run.smithi039.stdout:oload 110 2024-09-14T04:42:04.817 INFO:teuthology.orchestra.run.smithi039.stdout:max_change 1 2024-09-14T04:42:04.817 INFO:teuthology.orchestra.run.smithi039.stdout:max_change_osds 4 2024-09-14T04:42:04.817 INFO:teuthology.orchestra.run.smithi039.stdout:average_utilization 0.0115 2024-09-14T04:42:04.817 INFO:teuthology.orchestra.run.smithi039.stdout:overload_utilization 0.0127 2024-09-14T04:42:04.818 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4 weight 0.4794 -> 0.1967 2024-09-14T04:42:04.818 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1 weight 0.6410 -> 0.4612 2024-09-14T04:42:04.843 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:42:04.843 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:42:04.884 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:04.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:04.920 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:04.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:05.021 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:05.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:04 smithi039 ceph-mon[106470]: pgmap v2297: 99 pgs: 3 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 10 active+undersized, 9 active+undersized+degraded, 68 active+clean; 1.8 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 18159/87231 objects degraded (20.817%); 6636/87231 objects misplaced (7.607%); 15 MiB/s, 247 objects/s recovering 2024-09-14T04:42:05.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:04 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/112546553' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:42:05.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:04 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'30228','4':'12891'}"} : dispatch 2024-09-14T04:42:05.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:04 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'30228','4':'12891'}"} : dispatch 2024-09-14T04:42:05.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:04 smithi039 ceph-mon[111235]: pgmap v2297: 99 pgs: 3 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 10 active+undersized, 9 active+undersized+degraded, 68 active+clean; 1.8 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 18159/87231 objects degraded (20.817%); 6636/87231 objects misplaced (7.607%); 15 MiB/s, 247 objects/s recovering 2024-09-14T04:42:05.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:04 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/112546553' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:42:05.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:04 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'30228','4':'12891'}"} : dispatch 2024-09-14T04:42:05.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:04 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'30228','4':'12891'}"} : dispatch 2024-09-14T04:42:05.048 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:05.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:05.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:05.151 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:05.172 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:05.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:05.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:05.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:04 smithi129 ceph-mon[101316]: pgmap v2297: 99 pgs: 3 active+clean+remapped, 6 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 10 active+undersized, 9 active+undersized+degraded, 68 active+clean; 1.8 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 790 op/s; 18159/87231 objects degraded (20.817%); 6636/87231 objects misplaced (7.607%); 15 MiB/s, 247 objects/s recovering 2024-09-14T04:42:05.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:04 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/112546553' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:42:05.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:04 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'30228','4':'12891'}"} : dispatch 2024-09-14T04:42:05.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:04 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'30228','4':'12891'}"} : dispatch 2024-09-14T04:42:05.274 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:05.301 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:05.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:05.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:05.403 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:05.429 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:05.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:05.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:05.532 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:05.557 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:05.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:05.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:05.659 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:05.685 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:05.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:05.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:05.696 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:42:05.786 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:05.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:05.812 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:05.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:05.913 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:05.936 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:05.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:05.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:06.038 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:06.061 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:06.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:06.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:06.163 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:06.187 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:06.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:06.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:06.254 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:05 smithi039 ceph-mon[106470]: from='client.35343 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:42:06.254 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:05 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'30228','4':'12891'}"}]': finished 2024-09-14T04:42:06.254 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:05 smithi039 ceph-mon[106470]: osdmap e1355: 8 total, 7 up, 8 in 2024-09-14T04:42:06.254 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:05 smithi039 ceph-mon[106470]: osdmap e1356: 8 total, 7 up, 8 in 2024-09-14T04:42:06.254 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:05 smithi039 ceph-mon[111235]: from='client.35343 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:42:06.254 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:05 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'30228','4':'12891'}"}]': finished 2024-09-14T04:42:06.254 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:05 smithi039 ceph-mon[111235]: osdmap e1355: 8 total, 7 up, 8 in 2024-09-14T04:42:06.254 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:05 smithi039 ceph-mon[111235]: osdmap e1356: 8 total, 7 up, 8 in 2024-09-14T04:42:06.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:05 smithi129 ceph-mon[101316]: from='client.35343 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:42:06.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:05 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'30228','4':'12891'}"}]': finished 2024-09-14T04:42:06.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:05 smithi129 ceph-mon[101316]: osdmap e1355: 8 total, 7 up, 8 in 2024-09-14T04:42:06.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:05 smithi129 ceph-mon[101316]: osdmap e1356: 8 total, 7 up, 8 in 2024-09-14T04:42:06.289 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:06.312 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:06.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:06.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:06.414 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:06.436 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:06.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:06.438 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:06.539 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:06.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:06.562 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:06.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:06.606 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:42:06.606 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:42:06.664 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:06.687 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:06.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:06.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:06.789 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:06.815 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:06.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:06.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:06.916 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:06.939 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:06.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:06.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:07.040 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:07.074 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:07.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:07.076 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:07.177 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:07.207 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:07.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:07.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:07.214 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:06 smithi039 ceph-mon[106470]: pgmap v2299: 99 pgs: 3 active+clean+remapped, 1 remapped+peering, 5 active+undersized+degraded+remapped+backfill_wait, 1 peering, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 10 active+undersized, 8 active+undersized+degraded, 68 active+clean; 1.7 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 14240/84990 objects degraded (16.755%); 6636/84990 objects misplaced (7.808%); 14 MiB/s, 219 objects/s recovering 2024-09-14T04:42:07.215 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:06 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 14240/84990 objects degraded (16.755%), 15 pgs degraded (PG_DEGRADED) 2024-09-14T04:42:07.215 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:06 smithi039 ceph-mon[106470]: osdmap e1357: 8 total, 7 up, 8 in 2024-09-14T04:42:07.215 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:06 smithi039 ceph-mon[111235]: pgmap v2299: 99 pgs: 3 active+clean+remapped, 1 remapped+peering, 5 active+undersized+degraded+remapped+backfill_wait, 1 peering, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 10 active+undersized, 8 active+undersized+degraded, 68 active+clean; 1.7 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 14240/84990 objects degraded (16.755%); 6636/84990 objects misplaced (7.808%); 14 MiB/s, 219 objects/s recovering 2024-09-14T04:42:07.215 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:06 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 14240/84990 objects degraded (16.755%), 15 pgs degraded (PG_DEGRADED) 2024-09-14T04:42:07.215 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:06 smithi039 ceph-mon[111235]: osdmap e1357: 8 total, 7 up, 8 in 2024-09-14T04:42:07.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:06 smithi129 ceph-mon[101316]: pgmap v2299: 99 pgs: 3 active+clean+remapped, 1 remapped+peering, 5 active+undersized+degraded+remapped+backfill_wait, 1 peering, 2 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 10 active+undersized, 8 active+undersized+degraded, 68 active+clean; 1.7 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 14240/84990 objects degraded (16.755%); 6636/84990 objects misplaced (7.808%); 14 MiB/s, 219 objects/s recovering 2024-09-14T04:42:07.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:06 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 14240/84990 objects degraded (16.755%), 15 pgs degraded (PG_DEGRADED) 2024-09-14T04:42:07.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:06 smithi129 ceph-mon[101316]: osdmap e1357: 8 total, 7 up, 8 in 2024-09-14T04:42:07.309 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:07.355 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:07.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:07.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:07.458 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:07.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:07.490 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:07.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:07.577 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2024-09-14T04:42:07.578 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd test-reweight-by-utilization 110 3.0 2024-09-14T04:42:07.591 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:07.701 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:07.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:07.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:07.803 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:07.852 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:07.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:07.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:07.954 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:07.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:07.991 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:07.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:08.091 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:08.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:08.138 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:08.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:08.239 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:08.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:07 smithi129 ceph-mon[101316]: 53.4 starting backfill to osd.6 from (0'0,0'0] MAX to 1354'14389 2024-09-14T04:42:08.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:07 smithi129 ceph-mon[101316]: 53.1 starting backfill to osd.5 from (0'0,0'0] MAX to 1354'14701 2024-09-14T04:42:08.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:07 smithi129 ceph-mon[101316]: 53.7 starting backfill to osd.7 from (0'0,0'0] MAX to 1354'14762 2024-09-14T04:42:08.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:07 smithi129 ceph-mon[101316]: 53.5 starting backfill to osd.7 from (0'0,0'0] MAX to 1354'14562 2024-09-14T04:42:08.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:07 smithi129 ceph-mon[101316]: 53.1 continuing backfill to osd.6 from (1327'7079,1354'14701] MIN to 1354'14701 2024-09-14T04:42:08.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:42:08.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:42:08.287 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:08.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:08.288 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:08.289 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:07 smithi039 ceph-mon[106470]: 53.4 starting backfill to osd.6 from (0'0,0'0] MAX to 1354'14389 2024-09-14T04:42:08.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:07 smithi039 ceph-mon[106470]: 53.1 starting backfill to osd.5 from (0'0,0'0] MAX to 1354'14701 2024-09-14T04:42:08.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:07 smithi039 ceph-mon[106470]: 53.7 starting backfill to osd.7 from (0'0,0'0] MAX to 1354'14762 2024-09-14T04:42:08.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:07 smithi039 ceph-mon[106470]: 53.5 starting backfill to osd.7 from (0'0,0'0] MAX to 1354'14562 2024-09-14T04:42:08.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:07 smithi039 ceph-mon[106470]: 53.1 continuing backfill to osd.6 from (1327'7079,1354'14701] MIN to 1354'14701 2024-09-14T04:42:08.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:42:08.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:42:08.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:07 smithi039 ceph-mon[111235]: 53.4 starting backfill to osd.6 from (0'0,0'0] MAX to 1354'14389 2024-09-14T04:42:08.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:07 smithi039 ceph-mon[111235]: 53.1 starting backfill to osd.5 from (0'0,0'0] MAX to 1354'14701 2024-09-14T04:42:08.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:07 smithi039 ceph-mon[111235]: 53.7 starting backfill to osd.7 from (0'0,0'0] MAX to 1354'14762 2024-09-14T04:42:08.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:07 smithi039 ceph-mon[111235]: 53.5 starting backfill to osd.7 from (0'0,0'0] MAX to 1354'14562 2024-09-14T04:42:08.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:07 smithi039 ceph-mon[111235]: 53.1 continuing backfill to osd.6 from (1327'7079,1354'14701] MIN to 1354'14701 2024-09-14T04:42:08.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:42:08.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:42:08.389 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:08.418 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:08.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:08.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:08.520 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:08.547 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:08.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:08.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:08.607 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:42:08.650 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:08.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:08.705 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:08.705 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:08.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:42:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:42:08] "GET /metrics HTTP/1.1" 200 36797 "" "Prometheus/2.51.0" 2024-09-14T04:42:08.806 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:08.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:08.833 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:08.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:08.934 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:08.962 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:08.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:08.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:09.064 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:09.093 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:09.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:09.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:09.195 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:09.243 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:09.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:09.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:09.345 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:09.385 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:09.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:09.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:09.487 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:09.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:09.509 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:09.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:09.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:09 smithi039 ceph-mon[106470]: pgmap v2302: 99 pgs: 3 active+clean+remapped, 2 remapped+peering, 6 active+undersized+degraded+remapped+backfill_wait, 1 peering, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 8 active+undersized, 8 active+undersized+degraded, 69 active+clean; 1.7 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 13304/83319 objects degraded (15.968%); 5854/83319 objects misplaced (7.026%); 18 MiB/s, 284 objects/s recovering 2024-09-14T04:42:09.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:09 smithi039 ceph-mon[111235]: pgmap v2302: 99 pgs: 3 active+clean+remapped, 2 remapped+peering, 6 active+undersized+degraded+remapped+backfill_wait, 1 peering, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 8 active+undersized, 8 active+undersized+degraded, 69 active+clean; 1.7 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 13304/83319 objects degraded (15.968%); 5854/83319 objects misplaced (7.026%); 18 MiB/s, 284 objects/s recovering 2024-09-14T04:42:09.610 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:09.646 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:09.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:09.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:09.748 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:09.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:09 smithi129 ceph-mon[101316]: pgmap v2302: 99 pgs: 3 active+clean+remapped, 2 remapped+peering, 6 active+undersized+degraded+remapped+backfill_wait, 1 peering, 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfill_wait, 8 active+undersized, 8 active+undersized+degraded, 69 active+clean; 1.7 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 736 op/s; 13304/83319 objects degraded (15.968%); 5854/83319 objects misplaced (7.026%); 18 MiB/s, 284 objects/s recovering 2024-09-14T04:42:09.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:09.792 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:09.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:09.893 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:09.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:09.917 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:09.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:10.018 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:10.044 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:10.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:10.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:10.146 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:10.172 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:10.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:10.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:10.274 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:10.304 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:10.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:10.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:10.406 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:10.433 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:10.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:10.434 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:10.535 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:10.572 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:10.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:10.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:10.674 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:10.710 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:10.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:10.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:10.812 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:10.838 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:10.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:10.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:10.940 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:10.969 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:10.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:10.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:11.070 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:11.094 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:11.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:11.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:11.196 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:11.222 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:11.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:11.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:11.324 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:11.351 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:11.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:11.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:11.453 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:11.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:11.484 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:11.484 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:11.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:11 smithi039 ceph-mon[106470]: pgmap v2303: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 1 remapped+peering, 4 active+undersized+degraded+remapped+backfill_wait, 1 peering, 2 active+undersized+degraded+remapped+backfilling, 2 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 80 active+clean; 1.6 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 453 op/s; 11672/79068 objects degraded (14.762%); 8541/79068 objects misplaced (10.802%); 33 MiB/s, 533 objects/s recovering 2024-09-14T04:42:11.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:11 smithi039 ceph-mon[106470]: osdmap e1358: 8 total, 7 up, 8 in 2024-09-14T04:42:11.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:11 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 11672/79068 objects degraded (14.762%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:42:11.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:11 smithi039 ceph-mon[111235]: pgmap v2303: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 1 remapped+peering, 4 active+undersized+degraded+remapped+backfill_wait, 1 peering, 2 active+undersized+degraded+remapped+backfilling, 2 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 80 active+clean; 1.6 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 453 op/s; 11672/79068 objects degraded (14.762%); 8541/79068 objects misplaced (10.802%); 33 MiB/s, 533 objects/s recovering 2024-09-14T04:42:11.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:11 smithi039 ceph-mon[111235]: osdmap e1358: 8 total, 7 up, 8 in 2024-09-14T04:42:11.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:11 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 11672/79068 objects degraded (14.762%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:42:11.585 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:11.619 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:11.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:11.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:11.667 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:11 smithi129 ceph-mon[101316]: pgmap v2303: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 1 remapped+peering, 4 active+undersized+degraded+remapped+backfill_wait, 1 peering, 2 active+undersized+degraded+remapped+backfilling, 2 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 80 active+clean; 1.6 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 453 op/s; 11672/79068 objects degraded (14.762%); 8541/79068 objects misplaced (10.802%); 33 MiB/s, 533 objects/s recovering 2024-09-14T04:42:11.667 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:11 smithi129 ceph-mon[101316]: osdmap e1358: 8 total, 7 up, 8 in 2024-09-14T04:42:11.667 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:11 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 11672/79068 objects degraded (14.762%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:42:11.720 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:11.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:11.764 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:11.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:11.868 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:11.919 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:42:11.919 INFO:teuthology.orchestra.run.smithi039.stdout: "size": 20, 2024-09-14T04:42:11.919 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 600, 2024-09-14T04:42:11.919 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [ 2024-09-14T04:42:11.919 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.919 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:73823 53.1 53:89730e74:::benchmark_data_smithi039_224949_object73822:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1322)", 2024-09-14T04:42:11.920 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:39:12.623946+0000", 2024-09-14T04:42:11.920 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 179.290071716, 2024-09-14T04:42:11.920 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.13397935499999999, 2024-09-14T04:42:11.920 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:11.920 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:11.920 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:11.920 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:42:11.920 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:42:11.920 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:42:11.920 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 73823 2024-09-14T04:42:11.920 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.920 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:11.921 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.921 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:11.921 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.623946+0000", 2024-09-14T04:42:11.921 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:11.921 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.921 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.921 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:11.921 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.623946+0000", 2024-09-14T04:42:11.921 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:11.921 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.921 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.921 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:11.921 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.623947+0000", 2024-09-14T04:42:11.922 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.1900000000000001e-07 2024-09-14T04:42:11.922 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.922 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.922 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:11.922 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.623969+0000", 2024-09-14T04:42:11.922 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1804999999999999e-05 2024-09-14T04:42:11.922 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.922 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.922 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:11.922 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.623980+0000", 2024-09-14T04:42:11.922 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.099e-05 2024-09-14T04:42:11.922 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.922 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.922 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:11.923 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.623987+0000", 2024-09-14T04:42:11.923 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6559999999999992e-06 2024-09-14T04:42:11.923 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.923 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.923 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:11.923 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.624049+0000", 2024-09-14T04:42:11.923 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1267999999999997e-05 2024-09-14T04:42:11.923 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.923 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.923 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:11.923 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.624236+0000", 2024-09-14T04:42:11.923 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000186763 2024-09-14T04:42:11.924 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.924 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.924 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:42:11.924 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.624388+0000", 2024-09-14T04:42:11.924 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015237200000000001 2024-09-14T04:42:11.924 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.924 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.924 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:42:11.924 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.628010+0000", 2024-09-14T04:42:11.924 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.003622452 2024-09-14T04:42:11.924 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.925 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.925 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:11.925 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.630261+0000", 2024-09-14T04:42:11.925 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0022507249999999999 2024-09-14T04:42:11.925 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.925 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.925 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:11.925 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.757814+0000", 2024-09-14T04:42:11.925 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.127552901 2024-09-14T04:42:11.925 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.925 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.925 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:11.925 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.757880+0000", 2024-09-14T04:42:11.925 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.6081999999999995e-05 2024-09-14T04:42:11.926 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.926 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.926 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:11.926 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:12.757926+0000", 2024-09-14T04:42:11.926 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5522000000000002e-05 2024-09-14T04:42:11.926 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:11.927 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:11.927 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:11.927 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.927 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.927 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:133875 53.f e1330/1315 53:ff192831:::benchmark_data_smithi039_224949_object3782:head v 1330'8264, mlcod=1330'8262)", 2024-09-14T04:42:11.927 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:40:09.872919+0000", 2024-09-14T04:42:11.927 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 122.04109930200001, 2024-09-14T04:42:11.927 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.128861789, 2024-09-14T04:42:11.927 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:11.928 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:11.928 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:11.928 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:11.928 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.928 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:11.928 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:09.872919+0000", 2024-09-14T04:42:11.928 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:11.928 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.928 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.928 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:11.928 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:09.872919+0000", 2024-09-14T04:42:11.928 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:11.928 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.929 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.929 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:11.929 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:09.872919+0000", 2024-09-14T04:42:11.929 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6599999999999996e-07 2024-09-14T04:42:11.929 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.929 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.929 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:11.929 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:09.872924+0000", 2024-09-14T04:42:11.929 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.8400000000000002e-06 2024-09-14T04:42:11.929 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.929 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.929 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:11.930 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:09.872932+0000", 2024-09-14T04:42:11.930 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.1729999999999995e-06 2024-09-14T04:42:11.930 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.930 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.930 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:11.930 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:09.872938+0000", 2024-09-14T04:42:11.930 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1149999999999999e-06 2024-09-14T04:42:11.930 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.930 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.931 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:11.931 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:09.969586+0000", 2024-09-14T04:42:11.931 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.096647716999999994 2024-09-14T04:42:11.931 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.931 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.931 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:11.931 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:09.969605+0000", 2024-09-14T04:42:11.931 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8510999999999999e-05 2024-09-14T04:42:11.931 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.931 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.932 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:11.932 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:10.001422+0000", 2024-09-14T04:42:11.932 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.031816994000000001 2024-09-14T04:42:11.933 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.933 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.933 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:11.933 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:10.001780+0000", 2024-09-14T04:42:11.933 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000358873 2024-09-14T04:42:11.933 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:11.933 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:11.933 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:11.933 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.933 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.933 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233346 53.4 53:29de5274:::benchmark_data_smithi039_224949_object103253:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1354)", 2024-09-14T04:42:11.934 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.800661+0000", 2024-09-14T04:42:11.934 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 7.1133571030000002, 2024-09-14T04:42:11.934 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1855864270000001, 2024-09-14T04:42:11.934 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:11.934 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:11.934 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T04:42:11.934 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:42:11.934 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:42:11.934 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:42:11.935 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233346 2024-09-14T04:42:11.935 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.935 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:11.935 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.935 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:11.935 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.800661+0000", 2024-09-14T04:42:11.935 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:11.936 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.936 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.936 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:11.936 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.800661+0000", 2024-09-14T04:42:11.936 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:11.936 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.936 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.937 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:11.937 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.800662+0000", 2024-09-14T04:42:11.937 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2890000000000001e-06 2024-09-14T04:42:11.937 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.937 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.937 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:11.937 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.800666+0000", 2024-09-14T04:42:11.937 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.5130000000000001e-06 2024-09-14T04:42:11.937 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.937 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.937 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:11.937 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.800678+0000", 2024-09-14T04:42:11.938 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2146e-05 2024-09-14T04:42:11.938 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.938 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.938 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:11.938 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.800685+0000", 2024-09-14T04:42:11.938 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1319999999999996e-06 2024-09-14T04:42:11.938 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.938 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.938 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:11.938 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.803352+0000", 2024-09-14T04:42:11.938 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.002667333 2024-09-14T04:42:11.939 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.939 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.939 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:11.939 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.803484+0000", 2024-09-14T04:42:11.939 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000131395 2024-09-14T04:42:11.940 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.940 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.940 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2", 2024-09-14T04:42:11.940 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.803596+0000", 2024-09-14T04:42:11.940 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000112693 2024-09-14T04:42:11.940 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.940 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.940 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:11.941 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.831786+0000", 2024-09-14T04:42:11.941 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.028190112 2024-09-14T04:42:11.941 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.941 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.941 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:42:11.941 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.831796+0000", 2024-09-14T04:42:11.941 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.5699999999999999e-06 2024-09-14T04:42:11.942 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.942 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.942 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:11.942 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820032+0000", 2024-09-14T04:42:11.942 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.98823552299999995 2024-09-14T04:42:11.942 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.942 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.942 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:42:11.942 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820042+0000", 2024-09-14T04:42:11.942 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0434999999999999e-05 2024-09-14T04:42:11.942 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.943 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.943 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:11.943 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.986165+0000", 2024-09-14T04:42:11.943 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.166122954 2024-09-14T04:42:11.943 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.943 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.943 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:11.943 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.986247+0000", 2024-09-14T04:42:11.943 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.2331999999999997e-05 2024-09-14T04:42:11.943 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:11.943 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:11.943 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:11.943 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.944 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.944 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233355 53.1 53.9503e781 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1354)", 2024-09-14T04:42:11.944 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.812481+0000", 2024-09-14T04:42:11.944 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 7.1015372269999997, 2024-09-14T04:42:11.944 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.004531539, 2024-09-14T04:42:11.944 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:11.944 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:11.945 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T04:42:11.945 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:42:11.945 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:42:11.945 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:42:11.945 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233355 2024-09-14T04:42:11.945 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.945 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:11.945 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.945 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:11.945 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.812481+0000", 2024-09-14T04:42:11.945 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:11.946 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.946 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.946 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:11.946 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.812481+0000", 2024-09-14T04:42:11.946 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:11.946 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.946 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.946 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:11.946 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.812481+0000", 2024-09-14T04:42:11.946 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.4000000000000001e-07 2024-09-14T04:42:11.946 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.946 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.947 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:11.947 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.812486+0000", 2024-09-14T04:42:11.947 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2590000000000002e-06 2024-09-14T04:42:11.947 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.947 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.947 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:11.947 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.812496+0000", 2024-09-14T04:42:11.947 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0181e-05 2024-09-14T04:42:11.947 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.947 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.947 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:11.948 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.812504+0000", 2024-09-14T04:42:11.948 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9910000000000002e-06 2024-09-14T04:42:11.948 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.948 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.948 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:11.948 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.819855+0000", 2024-09-14T04:42:11.948 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0073515070000000002 2024-09-14T04:42:11.948 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.948 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.948 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:42:11.948 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.819866+0000", 2024-09-14T04:42:11.948 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0739e-05 2024-09-14T04:42:11.949 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.949 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.949 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:11.949 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.816996+0000", 2024-09-14T04:42:11.949 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.99712972799999999 2024-09-14T04:42:11.949 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.949 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.949 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:11.949 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.817012+0000", 2024-09-14T04:42:11.949 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6393999999999999e-05 2024-09-14T04:42:11.949 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:11.949 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:11.949 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:11.949 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.950 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.950 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233370 53.4 53:2070d59a:::benchmark_data_smithi039_224949_object103277:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-09-14T04:42:11.950 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.851143+0000", 2024-09-14T04:42:11.950 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 7.0628747709999997, 2024-09-14T04:42:11.950 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1441897729999999, 2024-09-14T04:42:11.950 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:11.950 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:11.950 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:11.950 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:42:11.950 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:42:11.950 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:42:11.951 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233370 2024-09-14T04:42:11.951 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.951 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:11.951 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.951 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:11.951 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.851143+0000", 2024-09-14T04:42:11.951 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:11.951 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.951 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.951 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:11.951 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.851143+0000", 2024-09-14T04:42:11.951 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:11.952 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.952 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.952 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:11.952 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.851144+0000", 2024-09-14T04:42:11.952 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.212e-06 2024-09-14T04:42:11.952 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.952 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.952 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:11.952 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.851149+0000", 2024-09-14T04:42:11.952 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2209999999999999e-06 2024-09-14T04:42:11.952 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.953 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.953 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:11.953 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.851158+0000", 2024-09-14T04:42:11.953 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.9369999999999993e-06 2024-09-14T04:42:11.953 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.953 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.953 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:11.953 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.851164+0000", 2024-09-14T04:42:11.953 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2360000000000001e-06 2024-09-14T04:42:11.953 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.953 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.953 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:11.954 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.851253+0000", 2024-09-14T04:42:11.954 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.9313000000000004e-05 2024-09-14T04:42:11.954 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.954 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.954 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:42:11.954 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.851263+0000", 2024-09-14T04:42:11.954 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.5380000000000008e-06 2024-09-14T04:42:11.954 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.954 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.954 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:11.954 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820077+0000", 2024-09-14T04:42:11.954 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.96881390199999995 2024-09-14T04:42:11.954 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.955 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.955 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:42:11.955 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820084+0000", 2024-09-14T04:42:11.955 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.661e-06 2024-09-14T04:42:11.955 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.955 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.955 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:11.955 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.986538+0000", 2024-09-14T04:42:11.955 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1664539979999999 2024-09-14T04:42:11.955 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.955 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.955 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:11.955 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.986613+0000", 2024-09-14T04:42:11.956 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.5295000000000003e-05 2024-09-14T04:42:11.956 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.956 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.956 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:42:11.956 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.986688+0000", 2024-09-14T04:42:11.957 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.4932999999999996e-05 2024-09-14T04:42:11.957 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.957 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.957 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:42:11.957 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.995009+0000", 2024-09-14T04:42:11.957 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0083206119999999998 2024-09-14T04:42:11.957 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.957 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.957 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:11.957 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.995057+0000", 2024-09-14T04:42:11.957 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7873000000000001e-05 2024-09-14T04:42:11.957 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.958 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.958 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:11.958 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.995139+0000", 2024-09-14T04:42:11.958 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.2451000000000001e-05 2024-09-14T04:42:11.958 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.958 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.958 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:11.958 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.995214+0000", 2024-09-14T04:42:11.958 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.4501999999999999e-05 2024-09-14T04:42:11.958 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.958 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.958 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:11.958 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.995333+0000", 2024-09-14T04:42:11.959 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000119089 2024-09-14T04:42:11.959 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:11.959 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:11.959 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:11.959 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.959 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.959 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233376 53.4 53:2f4a7748:::benchmark_data_smithi039_224949_object103283:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-09-14T04:42:11.959 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.861746+0000", 2024-09-14T04:42:11.959 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 7.0522717039999998, 2024-09-14T04:42:11.959 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1339306530000002, 2024-09-14T04:42:11.959 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:11.959 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:11.959 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:11.960 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:42:11.960 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:42:11.960 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:42:11.960 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233376 2024-09-14T04:42:11.960 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.960 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:11.960 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.960 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:11.960 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.861746+0000", 2024-09-14T04:42:11.960 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:11.960 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.961 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.961 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:11.961 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.861746+0000", 2024-09-14T04:42:11.961 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:11.961 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.961 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.961 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:11.961 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.861748+0000", 2024-09-14T04:42:11.961 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2299999999999998e-06 2024-09-14T04:42:11.961 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.961 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.961 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:11.962 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.861755+0000", 2024-09-14T04:42:11.962 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1580000000000002e-06 2024-09-14T04:42:11.962 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.962 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.962 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:11.962 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.861767+0000", 2024-09-14T04:42:11.962 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.254e-05 2024-09-14T04:42:11.962 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.962 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.962 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:11.962 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.861776+0000", 2024-09-14T04:42:11.962 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.8850000000000005e-06 2024-09-14T04:42:11.962 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.963 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.963 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:11.963 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.862325+0000", 2024-09-14T04:42:11.963 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00054872099999999995 2024-09-14T04:42:11.963 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.963 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.963 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:42:11.963 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.862333+0000", 2024-09-14T04:42:11.963 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.4689999999999997e-06 2024-09-14T04:42:11.963 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.963 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.963 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:11.963 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820254+0000", 2024-09-14T04:42:11.964 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.95792123699999998 2024-09-14T04:42:11.964 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.964 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.964 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:42:11.964 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820262+0000", 2024-09-14T04:42:11.964 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7780000000000007e-06 2024-09-14T04:42:11.964 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.964 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.964 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:11.964 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.990645+0000", 2024-09-14T04:42:11.965 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.170382351 2024-09-14T04:42:11.965 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.965 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.965 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:11.965 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.990791+0000", 2024-09-14T04:42:11.965 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014623199999999999 2024-09-14T04:42:11.965 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.965 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.965 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:42:11.965 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.990931+0000", 2024-09-14T04:42:11.965 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013971799999999999 2024-09-14T04:42:11.966 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.966 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.966 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:42:11.966 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.995074+0000", 2024-09-14T04:42:11.966 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0041437649999999998 2024-09-14T04:42:11.966 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.966 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.966 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:11.966 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.995400+0000", 2024-09-14T04:42:11.966 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00032538700000000001 2024-09-14T04:42:11.966 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.966 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.966 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:11.967 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.995468+0000", 2024-09-14T04:42:11.967 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8169000000000004e-05 2024-09-14T04:42:11.967 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.967 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.967 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:11.967 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.995546+0000", 2024-09-14T04:42:11.967 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7861000000000006e-05 2024-09-14T04:42:11.967 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.967 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.967 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:11.967 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.995677+0000", 2024-09-14T04:42:11.967 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013115199999999999 2024-09-14T04:42:11.967 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:11.968 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:11.968 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:11.968 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.968 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.968 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233378 53.4 53:2ea9efd4:::benchmark_data_smithi039_224949_object103285:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-09-14T04:42:11.968 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.864442+0000", 2024-09-14T04:42:11.968 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 7.0495756780000001, 2024-09-14T04:42:11.968 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1434582400000002, 2024-09-14T04:42:11.968 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:11.968 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:11.968 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:11.969 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:42:11.969 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:42:11.969 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:42:11.969 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233378 2024-09-14T04:42:11.969 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.969 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:11.969 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.969 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:11.969 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.864442+0000", 2024-09-14T04:42:11.969 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:11.969 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.969 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.969 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:11.969 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.864442+0000", 2024-09-14T04:42:11.970 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:11.970 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.970 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.970 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:11.970 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.864444+0000", 2024-09-14T04:42:11.970 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.968e-06 2024-09-14T04:42:11.970 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.970 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.970 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:11.970 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.864452+0000", 2024-09-14T04:42:11.970 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.452e-06 2024-09-14T04:42:11.970 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.970 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.971 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:11.971 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.864494+0000", 2024-09-14T04:42:11.971 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.1869000000000002e-05 2024-09-14T04:42:11.971 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.971 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.971 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:11.971 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.864503+0000", 2024-09-14T04:42:11.971 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.7100000000000002e-06 2024-09-14T04:42:11.971 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.971 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.971 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:11.971 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.864617+0000", 2024-09-14T04:42:11.971 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011412399999999999 2024-09-14T04:42:11.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:11.973 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:11.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:11.973 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.974 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.974 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:42:11.974 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.864627+0000", 2024-09-14T04:42:11.974 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0011e-05 2024-09-14T04:42:11.974 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.974 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.974 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:11.974 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820558+0000", 2024-09-14T04:42:11.974 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.95593013699999996 2024-09-14T04:42:11.974 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.974 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.975 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:42:11.975 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820565+0000", 2024-09-14T04:42:11.975 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.8900000000000007e-06 2024-09-14T04:42:11.975 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.975 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.975 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:11.975 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.995726+0000", 2024-09-14T04:42:11.975 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1751607719999999 2024-09-14T04:42:11.975 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.975 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.975 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:11.975 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.995867+0000", 2024-09-14T04:42:11.975 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000141297 2024-09-14T04:42:11.976 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.976 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.976 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:42:11.976 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.996016+0000", 2024-09-14T04:42:11.976 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014899100000000001 2024-09-14T04:42:11.976 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.976 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.976 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:11.976 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.003532+0000", 2024-09-14T04:42:11.976 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0075159880000000004 2024-09-14T04:42:11.976 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.977 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.977 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:42:11.977 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.003646+0000", 2024-09-14T04:42:11.977 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000114041 2024-09-14T04:42:11.977 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.977 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.977 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:11.977 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.007735+0000", 2024-09-14T04:42:11.977 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0040889840000000004 2024-09-14T04:42:11.977 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.977 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.977 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:11.978 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.007788+0000", 2024-09-14T04:42:11.978 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.2154999999999998e-05 2024-09-14T04:42:11.978 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.978 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.978 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:11.978 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.007900+0000", 2024-09-14T04:42:11.978 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011285099999999999 2024-09-14T04:42:11.978 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:11.978 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:11.978 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:11.978 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.978 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.979 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233406 53.1 53.a3bc2e61 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-09-14T04:42:11.979 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.908181+0000", 2024-09-14T04:42:11.979 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 7.0058369520000001, 2024-09-14T04:42:11.979 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.90887015999999998, 2024-09-14T04:42:11.979 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:11.979 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:11.979 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T04:42:11.979 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:42:11.979 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:42:11.979 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:42:11.979 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233406 2024-09-14T04:42:11.979 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.979 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:11.980 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.980 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:11.980 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.908181+0000", 2024-09-14T04:42:11.980 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:11.980 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.980 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.980 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:11.980 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.908181+0000", 2024-09-14T04:42:11.980 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:11.980 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.980 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.980 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:11.980 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.908182+0000", 2024-09-14T04:42:11.981 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.437e-06 2024-09-14T04:42:11.981 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.981 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.981 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:11.981 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.908188+0000", 2024-09-14T04:42:11.981 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6529999999999999e-06 2024-09-14T04:42:11.981 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.981 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.981 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:11.981 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.908201+0000", 2024-09-14T04:42:11.981 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2634e-05 2024-09-14T04:42:11.981 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.982 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.982 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:11.982 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.908210+0000", 2024-09-14T04:42:11.982 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.8400000000000001e-06 2024-09-14T04:42:11.982 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.982 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.982 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:11.982 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.908542+0000", 2024-09-14T04:42:11.982 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00033224600000000002 2024-09-14T04:42:11.982 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.982 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.982 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:42:11.982 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.908554+0000", 2024-09-14T04:42:11.982 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1736000000000001e-05 2024-09-14T04:42:11.983 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.983 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.983 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:11.983 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.817042+0000", 2024-09-14T04:42:11.983 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.90848892199999998 2024-09-14T04:42:11.983 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.983 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.983 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:11.983 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.817051+0000", 2024-09-14T04:42:11.983 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.6920000000000008e-06 2024-09-14T04:42:11.983 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:11.983 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:11.983 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:11.984 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.984 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.984 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233411 53.4 53:2cd724b4:::benchmark_data_smithi039_224949_object103318:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-09-14T04:42:11.984 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.920755+0000", 2024-09-14T04:42:11.984 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 6.9932634150000004, 2024-09-14T04:42:11.984 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.087357184, 2024-09-14T04:42:11.984 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:11.984 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:11.984 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:11.984 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:42:11.984 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:42:11.984 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:42:11.985 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233411 2024-09-14T04:42:11.985 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.985 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:11.985 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.985 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:11.985 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.920755+0000", 2024-09-14T04:42:11.985 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:11.985 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.985 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.985 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:11.985 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.920755+0000", 2024-09-14T04:42:11.985 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:11.985 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.986 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.986 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:11.986 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.920757+0000", 2024-09-14T04:42:11.986 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2879999999999999e-06 2024-09-14T04:42:11.986 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.986 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.986 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:11.986 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.920765+0000", 2024-09-14T04:42:11.986 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.3510000000000002e-06 2024-09-14T04:42:11.986 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.986 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.986 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:11.987 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.920791+0000", 2024-09-14T04:42:11.987 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.5904000000000001e-05 2024-09-14T04:42:11.987 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.987 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.987 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:11.987 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.920802+0000", 2024-09-14T04:42:11.987 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0804e-05 2024-09-14T04:42:11.987 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.987 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.987 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:11.987 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.920893+0000", 2024-09-14T04:42:11.987 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.0964000000000005e-05 2024-09-14T04:42:11.987 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.988 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.988 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:42:11.988 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.920904+0000", 2024-09-14T04:42:11.988 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0774000000000001e-05 2024-09-14T04:42:11.988 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.988 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.988 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:11.988 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820656+0000", 2024-09-14T04:42:11.988 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.89975210000000005 2024-09-14T04:42:11.988 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.988 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.988 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:42:11.988 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820665+0000", 2024-09-14T04:42:11.989 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.2860000000000005e-06 2024-09-14T04:42:11.989 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.989 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.989 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:11.989 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.999414+0000", 2024-09-14T04:42:11.989 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1787490410000001 2024-09-14T04:42:11.989 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.989 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.989 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:11.989 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.999538+0000", 2024-09-14T04:42:11.989 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012393500000000001 2024-09-14T04:42:11.990 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.990 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.990 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:42:11.990 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.999662+0000", 2024-09-14T04:42:11.990 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012404199999999999 2024-09-14T04:42:11.990 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.990 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.990 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:11.990 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.003622+0000", 2024-09-14T04:42:11.990 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0039600080000000001 2024-09-14T04:42:11.990 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.990 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.990 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:42:11.990 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.003684+0000", 2024-09-14T04:42:11.991 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2436999999999999e-05 2024-09-14T04:42:11.991 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.991 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.991 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:11.991 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.007955+0000", 2024-09-14T04:42:11.991 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0042703330000000003 2024-09-14T04:42:11.991 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.991 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.991 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:11.991 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.008012+0000", 2024-09-14T04:42:11.991 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7008e-05 2024-09-14T04:42:11.991 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.991 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.991 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:11.992 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.008112+0000", 2024-09-14T04:42:11.992 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.9908999999999995e-05 2024-09-14T04:42:11.992 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:11.992 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:11.992 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:11.992 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.992 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.992 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233412 53.1 53.a2159a51 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-09-14T04:42:11.992 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.921978+0000", 2024-09-14T04:42:11.992 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 6.9920399069999997, 2024-09-14T04:42:11.992 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.89512192400000001, 2024-09-14T04:42:11.992 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:11.993 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:11.993 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T04:42:11.993 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:42:11.993 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:42:11.993 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:42:11.993 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233412 2024-09-14T04:42:11.993 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.994 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:11.994 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.994 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:11.994 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.921978+0000", 2024-09-14T04:42:11.995 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:11.995 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.995 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.995 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:11.995 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.921978+0000", 2024-09-14T04:42:11.995 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:11.995 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.996 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.996 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:11.996 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.921979+0000", 2024-09-14T04:42:11.996 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.540000000000001e-07 2024-09-14T04:42:11.996 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.996 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.996 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:11.996 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.921984+0000", 2024-09-14T04:42:11.996 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6229999999999996e-06 2024-09-14T04:42:11.996 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.997 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.997 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:11.997 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.921994+0000", 2024-09-14T04:42:11.997 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.078e-05 2024-09-14T04:42:11.997 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.997 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.998 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:11.998 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.922000+0000", 2024-09-14T04:42:11.998 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.2449999999999998e-06 2024-09-14T04:42:11.998 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.998 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.998 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:11.998 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.922053+0000", 2024-09-14T04:42:11.998 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3575999999999997e-05 2024-09-14T04:42:11.998 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.998 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.998 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:42:11.998 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.922062+0000", 2024-09-14T04:42:11.999 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.2240000000000001e-06 2024-09-14T04:42:11.999 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.999 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.999 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:11.999 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.817089+0000", 2024-09-14T04:42:11.999 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.89502705100000002 2024-09-14T04:42:11.999 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:11.999 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:11.999 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:11.999 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.817100+0000", 2024-09-14T04:42:11.999 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0471e-05 2024-09-14T04:42:11.999 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:11.999 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:11.999 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:12.000 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.000 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.000 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:235271 53.4 53:2568f8d1:::benchmark_data_smithi039_224949_object105178:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1357)", 2024-09-14T04:42:12.000 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:09.439691+0000", 2024-09-14T04:42:12.000 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 2.4743266429999999, 2024-09-14T04:42:12.000 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.14003636799999999, 2024-09-14T04:42:12.000 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:12.000 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:12.000 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:12.000 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:42:12.000 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:42:12.000 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:42:12.000 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 235271 2024-09-14T04:42:12.001 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.001 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:12.001 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.001 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:12.001 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.439691+0000", 2024-09-14T04:42:12.001 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:12.001 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.001 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.001 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:12.001 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.439691+0000", 2024-09-14T04:42:12.001 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:12.001 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.001 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.002 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:12.002 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.439692+0000", 2024-09-14T04:42:12.002 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.096e-06 2024-09-14T04:42:12.002 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.002 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.002 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:12.002 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.439696+0000", 2024-09-14T04:42:12.002 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.5279999999999999e-06 2024-09-14T04:42:12.002 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.002 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.002 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:12.002 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.439705+0000", 2024-09-14T04:42:12.002 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.8210000000000008e-06 2024-09-14T04:42:12.002 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.003 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.003 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:12.003 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.439711+0000", 2024-09-14T04:42:12.003 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.9599999999999997e-06 2024-09-14T04:42:12.003 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.003 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.003 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:12.003 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.443584+0000", 2024-09-14T04:42:12.003 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0038728510000000001 2024-09-14T04:42:12.004 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.004 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.004 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:12.004 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.443707+0000", 2024-09-14T04:42:12.004 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012386799999999999 2024-09-14T04:42:12.004 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.004 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.004 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:42:12.004 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.443824+0000", 2024-09-14T04:42:12.004 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000116768 2024-09-14T04:42:12.004 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.005 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.005 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:12.005 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.455829+0000", 2024-09-14T04:42:12.005 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.012005006 2024-09-14T04:42:12.005 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.005 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.005 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:42:12.005 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.455842+0000", 2024-09-14T04:42:12.005 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2690999999999999e-05 2024-09-14T04:42:12.005 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.005 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.006 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:12.006 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.579368+0000", 2024-09-14T04:42:12.006 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.12352587800000001 2024-09-14T04:42:12.006 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.006 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.006 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:12.006 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.579581+0000", 2024-09-14T04:42:12.006 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000213494 2024-09-14T04:42:12.006 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.006 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.007 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:12.007 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.579728+0000", 2024-09-14T04:42:12.007 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014640700000000001 2024-09-14T04:42:12.007 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:12.007 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:12.007 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:12.007 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.007 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.008 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:235272 53.d 53:b20bffe8:::benchmark_data_smithi039_224949_object105179:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1357)", 2024-09-14T04:42:12.008 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:09.439729+0000", 2024-09-14T04:42:12.008 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 2.4742885889999999, 2024-09-14T04:42:12.008 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.13600108899999999, 2024-09-14T04:42:12.008 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:12.008 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:12.009 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:12.009 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:42:12.009 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:42:12.009 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:42:12.009 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 235272 2024-09-14T04:42:12.009 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.009 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:12.010 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.010 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:12.010 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.439729+0000", 2024-09-14T04:42:12.010 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:12.010 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.010 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.010 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:12.010 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.439729+0000", 2024-09-14T04:42:12.010 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:12.010 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.010 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.010 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:12.010 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.439730+0000", 2024-09-14T04:42:12.011 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.5100000000000001e-07 2024-09-14T04:42:12.011 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.011 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.011 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:12.011 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.439732+0000", 2024-09-14T04:42:12.011 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9410000000000001e-06 2024-09-14T04:42:12.011 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.011 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.011 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:12.011 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.439737+0000", 2024-09-14T04:42:12.011 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.8150000000000002e-06 2024-09-14T04:42:12.011 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.011 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.012 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:12.012 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.439742+0000", 2024-09-14T04:42:12.012 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5730000000000001e-06 2024-09-14T04:42:12.012 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.012 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.012 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:12.012 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.442340+0000", 2024-09-14T04:42:12.012 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0025984490000000001 2024-09-14T04:42:12.012 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.012 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.012 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:12.012 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.442449+0000", 2024-09-14T04:42:12.013 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000108238 2024-09-14T04:42:12.013 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.013 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.013 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:42:12.013 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.442558+0000", 2024-09-14T04:42:12.013 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000109415 2024-09-14T04:42:12.013 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.013 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.013 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:42:12.013 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.451444+0000", 2024-09-14T04:42:12.013 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0088858089999999997 2024-09-14T04:42:12.013 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.014 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.014 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:12.014 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.451730+0000", 2024-09-14T04:42:12.014 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00028611499999999998 2024-09-14T04:42:12.014 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.014 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.014 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:12.014 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.575641+0000", 2024-09-14T04:42:12.014 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.123910531 2024-09-14T04:42:12.014 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.014 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.015 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:12.015 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.575688+0000", 2024-09-14T04:42:12.015 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7411999999999999e-05 2024-09-14T04:42:12.015 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.015 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.015 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:12.015 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.575730+0000", 2024-09-14T04:42:12.015 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.244e-05 2024-09-14T04:42:12.015 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:12.016 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:12.016 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:12.016 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.016 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.016 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:235274 53.d 53:b5ab9741:::benchmark_data_smithi039_224949_object105181:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1357)", 2024-09-14T04:42:12.017 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:09.440686+0000", 2024-09-14T04:42:12.017 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 2.4733317690000001, 2024-09-14T04:42:12.017 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.13515137599999999, 2024-09-14T04:42:12.017 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:12.017 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:12.017 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:12.018 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:42:12.018 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:42:12.018 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:42:12.018 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 235274 2024-09-14T04:42:12.018 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.018 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:12.018 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.018 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:12.018 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.440686+0000", 2024-09-14T04:42:12.018 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:12.018 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.018 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.019 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:12.019 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.440686+0000", 2024-09-14T04:42:12.019 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:12.019 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.019 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.019 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:12.019 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.440687+0000", 2024-09-14T04:42:12.019 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1289999999999999e-06 2024-09-14T04:42:12.019 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.019 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.019 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:12.019 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.440691+0000", 2024-09-14T04:42:12.019 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.354e-06 2024-09-14T04:42:12.020 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.020 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.020 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:12.020 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.440701+0000", 2024-09-14T04:42:12.020 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0662000000000001e-05 2024-09-14T04:42:12.020 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.020 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.020 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:12.020 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.440709+0000", 2024-09-14T04:42:12.020 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6020000000000002e-06 2024-09-14T04:42:12.020 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.020 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.020 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:12.021 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.446491+0000", 2024-09-14T04:42:12.021 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0057821260000000003 2024-09-14T04:42:12.021 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.021 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.021 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:12.021 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.446727+0000", 2024-09-14T04:42:12.021 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000235784 2024-09-14T04:42:12.021 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.021 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.021 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:42:12.021 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.446873+0000", 2024-09-14T04:42:12.021 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014574 2024-09-14T04:42:12.021 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.021 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.022 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:42:12.022 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.451652+0000", 2024-09-14T04:42:12.022 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0047792590000000001 2024-09-14T04:42:12.022 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.022 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.022 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:12.022 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.451850+0000", 2024-09-14T04:42:12.022 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000197961 2024-09-14T04:42:12.022 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.022 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.022 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:12.022 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.575772+0000", 2024-09-14T04:42:12.023 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.123922217 2024-09-14T04:42:12.023 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.023 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.023 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:12.023 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.575808+0000", 2024-09-14T04:42:12.023 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.6089e-05 2024-09-14T04:42:12.023 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.023 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.023 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:12.023 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.575838+0000", 2024-09-14T04:42:12.023 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.9453000000000001e-05 2024-09-14T04:42:12.023 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:12.023 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:12.024 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:12.024 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.024 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.024 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:235277 53.4 53:2fa50fdc:::benchmark_data_smithi039_224949_object105184:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1357)", 2024-09-14T04:42:12.024 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:09.441494+0000", 2024-09-14T04:42:12.024 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 2.4725241599999999, 2024-09-14T04:42:12.024 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.14754867699999999, 2024-09-14T04:42:12.024 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:12.024 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:12.025 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:12.025 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:42:12.025 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:42:12.025 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:42:12.025 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 235277 2024-09-14T04:42:12.025 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.025 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:12.025 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.026 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:12.026 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.441494+0000", 2024-09-14T04:42:12.026 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:12.026 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.027 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.027 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:12.027 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.441494+0000", 2024-09-14T04:42:12.027 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:12.027 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.027 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.027 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:12.027 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.441495+0000", 2024-09-14T04:42:12.027 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.4300000000000001e-07 2024-09-14T04:42:12.027 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.027 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.027 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:12.027 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.441499+0000", 2024-09-14T04:42:12.027 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6800000000000001e-06 2024-09-14T04:42:12.028 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.028 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.028 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:12.028 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.441511+0000", 2024-09-14T04:42:12.028 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1827999999999999e-05 2024-09-14T04:42:12.028 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.028 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.028 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:12.028 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.441519+0000", 2024-09-14T04:42:12.028 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.5009999999999998e-06 2024-09-14T04:42:12.029 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.029 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.029 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:12.029 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.451474+0000", 2024-09-14T04:42:12.029 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0099551789999999998 2024-09-14T04:42:12.029 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.029 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.029 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:12.029 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.451657+0000", 2024-09-14T04:42:12.029 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000182819 2024-09-14T04:42:12.029 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.029 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.029 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:42:12.030 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.451779+0000", 2024-09-14T04:42:12.030 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012254799999999999 2024-09-14T04:42:12.030 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.030 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.030 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:12.030 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.579771+0000", 2024-09-14T04:42:12.030 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.127992146 2024-09-14T04:42:12.030 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.030 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.030 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:42:12.030 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.584330+0000", 2024-09-14T04:42:12.030 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0045588909999999998 2024-09-14T04:42:12.030 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.030 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.031 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:12.031 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.588878+0000", 2024-09-14T04:42:12.031 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.004547578 2024-09-14T04:42:12.031 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.031 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.031 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:12.031 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.588939+0000", 2024-09-14T04:42:12.031 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1424e-05 2024-09-14T04:42:12.031 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.031 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.031 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:12.031 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.589042+0000", 2024-09-14T04:42:12.031 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00010314000000000001 2024-09-14T04:42:12.031 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:12.032 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:12.032 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:12.032 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.032 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.032 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop_reply(client.45053.0:235271 53.4 e1357/1356 ondisk, result = 0)", 2024-09-14T04:42:12.032 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:09.451329+0000", 2024-09-14T04:42:12.032 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 2.462688574, 2024-09-14T04:42:12.032 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.12840467, 2024-09-14T04:42:12.033 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:12.033 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:12.033 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "started", 2024-09-14T04:42:12.033 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:12.033 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.033 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:12.033 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.451329+0000", 2024-09-14T04:42:12.033 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:12.034 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.034 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.034 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:12.034 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.451329+0000", 2024-09-14T04:42:12.034 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:12.034 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.034 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.034 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:12.034 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.451330+0000", 2024-09-14T04:42:12.034 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0100000000000005e-07 2024-09-14T04:42:12.034 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.034 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.035 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:12.035 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.451333+0000", 2024-09-14T04:42:12.035 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.9500000000000001e-06 2024-09-14T04:42:12.035 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.035 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.035 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:12.035 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.451340+0000", 2024-09-14T04:42:12.035 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.2679999999999997e-06 2024-09-14T04:42:12.035 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.035 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.035 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:12.035 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.451347+0000", 2024-09-14T04:42:12.036 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.5339999999999996e-06 2024-09-14T04:42:12.036 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.036 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.036 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:12.036 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.579352+0000", 2024-09-14T04:42:12.036 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.12800521000000001 2024-09-14T04:42:12.036 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.036 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.036 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:12.036 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.579361+0000", 2024-09-14T04:42:12.036 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.986e-06 2024-09-14T04:42:12.036 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.036 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.037 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:12.037 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.579734+0000", 2024-09-14T04:42:12.037 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00037312100000000002 2024-09-14T04:42:12.037 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:12.037 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:12.037 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:12.037 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.037 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.037 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:235282 53.c e1357/1315 53:3b56e3cb:::benchmark_data_smithi039_224949_object105189:head v 1357'14983, mlcod=1357'14980)", 2024-09-14T04:42:12.037 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:09.451346+0000", 2024-09-14T04:42:12.037 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 2.462671973, 2024-09-14T04:42:12.037 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.144025703, 2024-09-14T04:42:12.038 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:12.038 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:12.038 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:12.038 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:12.038 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.038 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:12.038 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.451346+0000", 2024-09-14T04:42:12.038 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:12.038 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.038 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.038 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:12.038 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.451346+0000", 2024-09-14T04:42:12.038 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:12.038 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.038 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.039 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:12.039 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.451347+0000", 2024-09-14T04:42:12.039 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6799999999999999e-07 2024-09-14T04:42:12.039 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.039 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.039 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:12.039 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.451352+0000", 2024-09-14T04:42:12.039 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.694e-06 2024-09-14T04:42:12.039 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.039 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.039 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:12.039 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.451363+0000", 2024-09-14T04:42:12.039 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0438999999999999e-05 2024-09-14T04:42:12.039 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.040 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.040 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:12.040 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.451369+0000", 2024-09-14T04:42:12.040 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.63e-06 2024-09-14T04:42:12.040 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.040 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.040 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:12.040 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.455897+0000", 2024-09-14T04:42:12.040 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0045271840000000001 2024-09-14T04:42:12.040 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.040 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.040 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:12.040 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.575909+0000", 2024-09-14T04:42:12.041 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.120012534 2024-09-14T04:42:12.041 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.041 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.041 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:12.041 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.595293+0000", 2024-09-14T04:42:12.041 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.019383839 2024-09-14T04:42:12.041 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.041 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.041 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:12.041 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.595372+0000", 2024-09-14T04:42:12.041 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.8615000000000005e-05 2024-09-14T04:42:12.041 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:12.041 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:12.041 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:12.042 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.042 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.042 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:235286 53.e e1357/1350 53:7ced952f:::benchmark_data_smithi039_224949_object105193:head v 1357'14449, mlcod=1357'14447)", 2024-09-14T04:42:12.042 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:09.452062+0000", 2024-09-14T04:42:12.042 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 2.4619557300000001, 2024-09-14T04:42:12.042 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.12731499299999999, 2024-09-14T04:42:12.042 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:12.042 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:12.042 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:12.042 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:12.042 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.042 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:12.042 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.452062+0000", 2024-09-14T04:42:12.043 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:12.043 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.043 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.043 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:12.043 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.452062+0000", 2024-09-14T04:42:12.043 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:12.043 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.043 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.043 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:12.043 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.452063+0000", 2024-09-14T04:42:12.043 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.0900000000000002e-07 2024-09-14T04:42:12.043 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.043 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.043 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:12.044 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.452066+0000", 2024-09-14T04:42:12.044 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.7069999999999998e-06 2024-09-14T04:42:12.044 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.044 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.044 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:12.044 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.452079+0000", 2024-09-14T04:42:12.044 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2475e-05 2024-09-14T04:42:12.044 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.044 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.044 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:12.044 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.452085+0000", 2024-09-14T04:42:12.044 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.8579999999999996e-06 2024-09-14T04:42:12.044 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.044 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.045 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:12.045 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.452153+0000", 2024-09-14T04:42:12.045 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8442999999999995e-05 2024-09-14T04:42:12.045 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.045 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.045 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:12.045 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.452175+0000", 2024-09-14T04:42:12.045 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1518e-05 2024-09-14T04:42:12.045 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.045 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.046 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:12.046 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.579330+0000", 2024-09-14T04:42:12.046 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.12715532900000001 2024-09-14T04:42:12.046 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.046 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.046 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:12.046 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.579377+0000", 2024-09-14T04:42:12.046 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7154000000000003e-05 2024-09-14T04:42:12.046 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:12.046 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:12.046 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:12.046 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.046 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.046 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:235291 53.4 53:24f6a53c:::benchmark_data_smithi039_224949_object105198:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1357)", 2024-09-14T04:42:12.047 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:09.454631+0000", 2024-09-14T04:42:12.047 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 2.459386775, 2024-09-14T04:42:12.047 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.15679042700000001, 2024-09-14T04:42:12.047 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:12.047 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:12.047 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:12.047 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:42:12.047 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:42:12.047 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:42:12.047 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 235291 2024-09-14T04:42:12.047 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.048 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:12.048 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.048 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:12.048 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.454631+0000", 2024-09-14T04:42:12.048 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:12.048 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.048 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.048 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:12.048 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.454631+0000", 2024-09-14T04:42:12.048 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:12.048 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.048 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.048 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:12.048 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.454633+0000", 2024-09-14T04:42:12.049 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7409999999999999e-06 2024-09-14T04:42:12.049 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.049 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.049 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:12.049 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.454638+0000", 2024-09-14T04:42:12.049 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.2279999999999998e-06 2024-09-14T04:42:12.049 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.049 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.049 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:12.049 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.454652+0000", 2024-09-14T04:42:12.049 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3423e-05 2024-09-14T04:42:12.049 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.049 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.050 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:12.050 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.454659+0000", 2024-09-14T04:42:12.050 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.5909999999999998e-06 2024-09-14T04:42:12.050 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.050 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.050 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:12.050 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.589086+0000", 2024-09-14T04:42:12.050 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.13442657799999999 2024-09-14T04:42:12.050 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.050 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.050 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:12.050 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.589254+0000", 2024-09-14T04:42:12.050 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00016805999999999999 2024-09-14T04:42:12.051 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.051 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.051 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:42:12.051 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.589372+0000", 2024-09-14T04:42:12.051 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000117843 2024-09-14T04:42:12.051 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.051 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.051 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:12.051 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.595794+0000", 2024-09-14T04:42:12.051 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0064222359999999996 2024-09-14T04:42:12.051 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.051 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.051 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:12.052 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.602295+0000", 2024-09-14T04:42:12.052 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0065011399999999999 2024-09-14T04:42:12.052 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.052 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.052 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:42:12.052 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.611133+0000", 2024-09-14T04:42:12.052 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0088381169999999995 2024-09-14T04:42:12.052 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.052 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.052 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:12.052 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.611201+0000", 2024-09-14T04:42:12.052 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7929999999999995e-05 2024-09-14T04:42:12.052 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.053 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.053 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:12.053 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.611422+0000", 2024-09-14T04:42:12.053 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00022054 2024-09-14T04:42:12.053 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:12.053 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:12.053 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:12.053 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.053 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.053 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop_reply(client.45053.0:235277 53.4 e1357/1356 ondisk, result = 0)", 2024-09-14T04:42:12.053 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:09.454851+0000", 2024-09-14T04:42:12.053 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 2.459167318, 2024-09-14T04:42:12.053 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.12492805, 2024-09-14T04:42:12.054 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:12.054 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:12.054 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "started", 2024-09-14T04:42:12.054 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:12.054 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.054 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:12.054 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.454851+0000", 2024-09-14T04:42:12.054 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:12.054 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.054 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.054 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:12.054 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.454851+0000", 2024-09-14T04:42:12.054 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:12.054 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.054 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.055 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:12.055 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.454852+0000", 2024-09-14T04:42:12.055 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1209999999999999e-06 2024-09-14T04:42:12.055 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.055 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.055 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:12.055 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.454857+0000", 2024-09-14T04:42:12.055 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9629999999999997e-06 2024-09-14T04:42:12.055 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.055 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.055 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:12.056 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.454867+0000", 2024-09-14T04:42:12.056 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0643e-05 2024-09-14T04:42:12.056 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.056 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.056 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:12.056 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.454874+0000", 2024-09-14T04:42:12.056 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.5520000000000001e-06 2024-09-14T04:42:12.056 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.056 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.056 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:12.056 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.579758+0000", 2024-09-14T04:42:12.056 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.124883705 2024-09-14T04:42:12.056 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.057 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.057 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:12.057 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.579765+0000", 2024-09-14T04:42:12.057 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7179999999999996e-06 2024-09-14T04:42:12.057 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.057 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.057 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:12.057 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.579779+0000", 2024-09-14T04:42:12.057 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3348e-05 2024-09-14T04:42:12.057 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:12.057 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:12.058 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:12.058 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.058 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.058 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop_reply(client.45053.0:235277 53.4 e1357/1356 ondisk, result = 0)", 2024-09-14T04:42:12.058 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:09.456230+0000", 2024-09-14T04:42:12.058 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 2.4577883300000001, 2024-09-14T04:42:12.058 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.13282028000000001, 2024-09-14T04:42:12.058 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:12.058 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:12.058 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "started", 2024-09-14T04:42:12.058 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:12.058 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.058 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:12.059 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.456230+0000", 2024-09-14T04:42:12.059 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:12.059 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.059 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.059 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:12.059 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.456230+0000", 2024-09-14T04:42:12.059 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:12.059 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.059 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.059 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:12.059 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.456230+0000", 2024-09-14T04:42:12.059 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.2699999999999999e-07 2024-09-14T04:42:12.059 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.059 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.059 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:12.060 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.456234+0000", 2024-09-14T04:42:12.060 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.1799999999999998e-06 2024-09-14T04:42:12.060 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.060 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.060 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:12.060 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.575797+0000", 2024-09-14T04:42:12.060 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.119562379 2024-09-14T04:42:12.060 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.060 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.060 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:12.060 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.575806+0000", 2024-09-14T04:42:12.060 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.7669999999999992e-06 2024-09-14T04:42:12.060 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.061 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.061 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:12.061 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.588862+0000", 2024-09-14T04:42:12.061 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.013056676 2024-09-14T04:42:12.061 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.061 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.061 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:12.061 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.588872+0000", 2024-09-14T04:42:12.061 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.4190000000000008e-06 2024-09-14T04:42:12.061 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:12.061 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:12.061 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:12.061 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.589050+0000", 2024-09-14T04:42:12.061 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00017813200000000001 2024-09-14T04:42:12.062 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:12.062 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:12.062 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:12.062 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:12.062 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:12.062 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:42:12.074 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:12.102 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:12.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:12.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:12.205 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:12.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:12.247 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:12.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:12.348 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:12.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:12.391 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:12.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:12.492 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:12.535 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:12.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:12.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:12.637 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:12.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:12.668 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:12.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:12.728 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:12 smithi039 ceph-mon[106470]: osdmap e1359: 8 total, 7 up, 8 in 2024-09-14T04:42:12.728 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:12 smithi039 ceph-mon[111235]: osdmap e1359: 8 total, 7 up, 8 in 2024-09-14T04:42:12.769 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:12.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:12 smithi129 ceph-mon[101316]: osdmap e1359: 8 total, 7 up, 8 in 2024-09-14T04:42:12.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:12.827 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:12.827 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:12.928 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:12.954 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:12.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:12.955 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:13.056 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:13.083 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:13.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:13.084 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:13.184 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:13.210 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:13.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:13.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:13.312 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:13.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:13.340 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:13.341 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:13.441 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:13.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:13.470 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:13.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:13.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:13 smithi039 ceph-mon[106470]: pgmap v2306: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 1 remapped+peering, 5 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 2 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 80 active+clean; 1.6 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 387 op/s; 13208/78276 objects degraded (16.874%); 8541/78276 objects misplaced (10.911%); 35 MiB/s, 554 objects/s recovering 2024-09-14T04:42:13.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:13 smithi039 ceph-mon[111235]: pgmap v2306: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 1 remapped+peering, 5 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 2 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 80 active+clean; 1.6 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 387 op/s; 13208/78276 objects degraded (16.874%); 8541/78276 objects misplaced (10.911%); 35 MiB/s, 554 objects/s recovering 2024-09-14T04:42:13.572 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:13.599 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:13.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:13.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:13.649 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-14T04:42:13.685 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:42:13.701 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:13.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:13.764 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:13.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:13.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:13 smithi129 ceph-mon[101316]: pgmap v2306: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 1 remapped+peering, 5 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 2 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 80 active+clean; 1.6 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 387 op/s; 13208/78276 objects degraded (16.874%); 8541/78276 objects misplaced (10.911%); 35 MiB/s, 554 objects/s recovering 2024-09-14T04:42:13.865 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:13.923 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:13.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:13.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:14.025 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:14.080 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:14.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:14.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:14.182 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:14.223 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:14.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:14.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:14.325 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:14.369 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:14.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:14.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:14.471 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:14.527 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:14.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:14.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:14.629 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:14.656 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:14.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:14.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:14.758 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:14.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:14.802 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:14.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:14.903 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:14.948 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:14.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:14.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:15.032 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:42:15.050 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:15.076 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:15.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:15.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:15.178 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:15.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:15.223 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:15.223 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:15.324 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:15.348 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:15.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:15.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:15.371 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is set 2024-09-14T04:42:15.450 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:15.486 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:15.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:15.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:15.588 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:15.633 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:15.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:15.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:15.735 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:15.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:15 smithi129 ceph-mon[101316]: pgmap v2307: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 81 active+clean; 1.4 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 9994/69243 objects degraded (14.433%); 8194/69243 objects misplaced (11.834%); 40 MiB/s, 640 objects/s recovering 2024-09-14T04:42:15.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:15 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3661441340' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:42:15.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:15 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:42:15.776 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:15.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:15.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:15.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:15 smithi039 ceph-mon[106470]: pgmap v2307: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 81 active+clean; 1.4 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 9994/69243 objects degraded (14.433%); 8194/69243 objects misplaced (11.834%); 40 MiB/s, 640 objects/s recovering 2024-09-14T04:42:15.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:15 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3661441340' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:42:15.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:15 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:42:15.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:15 smithi039 ceph-mon[111235]: pgmap v2307: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 81 active+clean; 1.4 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 9994/69243 objects degraded (14.433%); 8194/69243 objects misplaced (11.834%); 40 MiB/s, 640 objects/s recovering 2024-09-14T04:42:15.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:15 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3661441340' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:42:15.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:15 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:42:15.878 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:15.900 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:15.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:15.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:16.002 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:16.037 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:16.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:16.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:16.139 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:16.162 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:16.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:16.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:16.264 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:16.311 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:16.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:16.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:16.413 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:16.439 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:16.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:16.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:16.542 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:16.567 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:16.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:16.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:16.669 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:16.684 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:16 smithi039 ceph-mon[106470]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:42:16.684 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:16 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:42:16.684 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:16 smithi039 ceph-mon[106470]: osdmap e1360: 8 total, 7 up, 8 in 2024-09-14T04:42:16.684 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:16 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 9994/69243 objects degraded (14.433%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:42:16.684 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:16 smithi039 ceph-mon[111235]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:42:16.684 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:16 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:42:16.684 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:16 smithi039 ceph-mon[111235]: osdmap e1360: 8 total, 7 up, 8 in 2024-09-14T04:42:16.684 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:16 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 9994/69243 objects degraded (14.433%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:42:16.695 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:16.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:16.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:16.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:16 smithi129 ceph-mon[101316]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:42:16.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:16 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:42:16.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:16 smithi129 ceph-mon[101316]: osdmap e1360: 8 total, 7 up, 8 in 2024-09-14T04:42:16.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:16 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 9994/69243 objects degraded (14.433%), 11 pgs degraded (PG_DEGRADED) 2024-09-14T04:42:16.797 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:16.842 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:16.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:16.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:16.944 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:16.973 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:16.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:16.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:17.074 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:17.113 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:17.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:17.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:17.214 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:17.241 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:17.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:17.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:17.343 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:17.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:17.377 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:17.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:17.460 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:42:17.478 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:17.503 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:17.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:17.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:17.605 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:17.654 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:17.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:17.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:17.756 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:17.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:17 smithi129 ceph-mon[101316]: pgmap v2309: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 81 active+clean; 1.4 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 9780/67935 objects degraded (14.396%); 8194/67935 objects misplaced (12.062%); 19 MiB/s, 301 objects/s recovering 2024-09-14T04:42:17.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:17 smithi039 ceph-mon[106470]: pgmap v2309: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 81 active+clean; 1.4 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 9780/67935 objects degraded (14.396%); 8194/67935 objects misplaced (12.062%); 19 MiB/s, 301 objects/s recovering 2024-09-14T04:42:17.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:17 smithi039 ceph-mon[111235]: pgmap v2309: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 81 active+clean; 1.4 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 618 op/s; 9780/67935 objects degraded (14.396%); 8194/67935 objects misplaced (12.062%); 19 MiB/s, 301 objects/s recovering 2024-09-14T04:42:17.798 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:17.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:17.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:17.900 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:17.944 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:17.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:17.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:18.046 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:18.072 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:18.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:18.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:18.174 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:18.203 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:18.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:18.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:18.305 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:18.346 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:18.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:18.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:18.448 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:18.496 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:18.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:18.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:18.598 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:18.631 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:18.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:18.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:18.724 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:42:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:42:18] "GET /metrics HTTP/1.1" 200 36797 "" "Prometheus/2.51.0" 2024-09-14T04:42:18.732 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:18.755 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:18.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:18.756 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:18.857 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:18.880 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:18.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:18.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:18.983 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:19.010 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:19.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:19.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:19.113 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:19.135 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:19.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:19.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:19.236 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:19.264 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:19.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:19.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:19.366 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:19.389 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:19.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:19.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:19.490 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:19.522 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:19.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:19.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:19.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:19 smithi039 ceph-mon[106470]: pgmap v2310: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 81 active+clean; 1.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 9417/65403 objects degraded (14.398%); 7823/65403 objects misplaced (11.961%); 16 MiB/s, 249 objects/s recovering 2024-09-14T04:42:19.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:19 smithi039 ceph-mon[106470]: osdmap e1361: 8 total, 7 up, 8 in 2024-09-14T04:42:19.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:19 smithi039 ceph-mon[111235]: pgmap v2310: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 81 active+clean; 1.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 9417/65403 objects degraded (14.398%); 7823/65403 objects misplaced (11.961%); 16 MiB/s, 249 objects/s recovering 2024-09-14T04:42:19.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:19 smithi039 ceph-mon[111235]: osdmap e1361: 8 total, 7 up, 8 in 2024-09-14T04:42:19.624 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:19.650 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:19.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:19.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:19.752 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:19.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:19 smithi129 ceph-mon[101316]: pgmap v2310: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 4 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 81 active+clean; 1.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 592 op/s; 9417/65403 objects degraded (14.398%); 7823/65403 objects misplaced (11.961%); 16 MiB/s, 249 objects/s recovering 2024-09-14T04:42:19.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:19 smithi129 ceph-mon[101316]: osdmap e1361: 8 total, 7 up, 8 in 2024-09-14T04:42:19.787 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:19.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:19.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:19.889 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:19.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:19.919 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:19.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:20.019 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:20.048 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:20.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:20.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:20.058 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:42:20.059 INFO:teuthology.orchestra.run.smithi039.stdout:moved 8 / 297 (2.6936%) 2024-09-14T04:42:20.059 INFO:teuthology.orchestra.run.smithi039.stdout:avg 42.4286 2024-09-14T04:42:20.059 INFO:teuthology.orchestra.run.smithi039.stdout:stddev 9.0283 -> 8.90769 (expected baseline 6.03053) 2024-09-14T04:42:20.059 INFO:teuthology.orchestra.run.smithi039.stdout:min osd.1 with 23 -> 22 pgs (0.542088 -> 0.518519 * mean) 2024-09-14T04:42:20.059 INFO:teuthology.orchestra.run.smithi039.stdout:max osd.3 with 50 -> 50 pgs (1.17845 -> 1.17845 * mean) 2024-09-14T04:42:20.059 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:42:20.059 INFO:teuthology.orchestra.run.smithi039.stdout:oload 110 2024-09-14T04:42:20.059 INFO:teuthology.orchestra.run.smithi039.stdout:max_change 3 2024-09-14T04:42:20.059 INFO:teuthology.orchestra.run.smithi039.stdout:max_change_osds 4 2024-09-14T04:42:20.060 INFO:teuthology.orchestra.run.smithi039.stdout:average_utilization 0.0101 2024-09-14T04:42:20.060 INFO:teuthology.orchestra.run.smithi039.stdout:overload_utilization 0.0111 2024-09-14T04:42:20.060 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4 weight 0.1967 -> 0.0710 2024-09-14T04:42:20.060 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1 weight 0.4612 -> 0.3878 2024-09-14T04:42:20.060 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5 weight 0.9052 -> 1.0000 2024-09-14T04:42:20.060 INFO:teuthology.orchestra.run.smithi039.stderr:no change 2024-09-14T04:42:20.150 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:20.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:20.178 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:20.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:20.279 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:20.289 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:42:20 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T04:42:20.057+0000 7f268b1da640 -1 mgr.server reply reply (3) No such process no change 2024-09-14T04:42:20.303 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:20.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:20.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:20.404 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:20.440 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:20.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:20.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:20.542 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:20.582 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:20.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:20.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:20.685 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:20.709 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:20.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:20.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:20.715 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:20 smithi039 ceph-mon[106470]: osdmap e1362: 8 total, 7 up, 8 in 2024-09-14T04:42:20.715 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:20 smithi039 ceph-mon[111235]: osdmap e1362: 8 total, 7 up, 8 in 2024-09-14T04:42:20.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:20 smithi129 ceph-mon[101316]: osdmap e1362: 8 total, 7 up, 8 in 2024-09-14T04:42:20.811 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:20.838 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:20.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:20.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:20.939 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:20.967 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:20.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:20.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:21.069 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:21.095 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:21.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:21.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:21.197 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:21.224 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:21.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:21.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:21.325 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:21.349 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:21.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:21.350 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:21.451 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:21.481 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:21.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:21.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:21.583 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:21.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:21.628 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:21.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:21.729 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:21.750 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:21 smithi039 ceph-mon[106470]: pgmap v2313: 99 pgs: 1 active+undersized+remapped, 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 81 active+clean; 1.1 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 7327/56565 objects degraded (12.953%); 6214/56565 objects misplaced (10.986%); 9.9 MiB/s, 158 objects/s recovering 2024-09-14T04:42:21.750 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:21 smithi039 ceph-mon[106470]: from='client.54931 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:42:21.750 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:21 smithi039 ceph-mon[106470]: osdmap e1363: 8 total, 7 up, 8 in 2024-09-14T04:42:21.751 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:21 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 7327/56565 objects degraded (12.953%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:42:21.751 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:21 smithi039 ceph-mon[111235]: pgmap v2313: 99 pgs: 1 active+undersized+remapped, 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 81 active+clean; 1.1 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 7327/56565 objects degraded (12.953%); 6214/56565 objects misplaced (10.986%); 9.9 MiB/s, 158 objects/s recovering 2024-09-14T04:42:21.751 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:21 smithi039 ceph-mon[111235]: from='client.54931 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:42:21.751 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:21 smithi039 ceph-mon[111235]: osdmap e1363: 8 total, 7 up, 8 in 2024-09-14T04:42:21.751 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:21 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 7327/56565 objects degraded (12.953%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:42:21.755 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:21.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:21.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:21.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:21 smithi129 ceph-mon[101316]: pgmap v2313: 99 pgs: 1 active+undersized+remapped, 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 81 active+clean; 1.1 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 704 op/s; 7327/56565 objects degraded (12.953%); 6214/56565 objects misplaced (10.986%); 9.9 MiB/s, 158 objects/s recovering 2024-09-14T04:42:21.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:21 smithi129 ceph-mon[101316]: from='client.54931 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:42:21.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:21 smithi129 ceph-mon[101316]: osdmap e1363: 8 total, 7 up, 8 in 2024-09-14T04:42:21.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:21 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 7327/56565 objects degraded (12.953%), 10 pgs degraded (PG_DEGRADED) 2024-09-14T04:42:21.857 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:21.901 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:21.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:21.902 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:21.919 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2024-09-14T04:42:21.919 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd test-reweight-by-utilization 110 3.0 2024-09-14T04:42:21.937 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:42:21.937 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [ 2024-09-14T04:42:21.937 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:21.937 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:243368 53.0 e1364/1344)", 2024-09-14T04:42:21.937 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:21.930416+0000", 2024-09-14T04:42:21.938 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.00010170000000000001, 2024-09-14T04:42:21.938 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000123433, 2024-09-14T04:42:21.938 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:21.938 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:21.938 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "queued for pg", 2024-09-14T04:42:21.938 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:21.938 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:21.938 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:21.938 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.930416+0000", 2024-09-14T04:42:21.938 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:21.938 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:21.938 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:21.939 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:21.939 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.930416+0000", 2024-09-14T04:42:21.939 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:21.939 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:21.939 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:21.939 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:21.939 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.930417+0000", 2024-09-14T04:42:21.939 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.288e-06 2024-09-14T04:42:21.939 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:21.939 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:21.939 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:21.939 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.930427+0000", 2024-09-14T04:42:21.940 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.2329999999999995e-06 2024-09-14T04:42:21.940 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:21.940 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:21.940 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:21.940 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.930451+0000", 2024-09-14T04:42:21.940 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4280000000000001e-05 2024-09-14T04:42:21.940 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:21.940 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:21.940 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:21.940 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.930470+0000", 2024-09-14T04:42:21.940 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9088000000000001e-05 2024-09-14T04:42:21.940 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:21.940 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:21.940 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:21.941 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:21.941 INFO:teuthology.orchestra.run.smithi039.stdout: ], 2024-09-14T04:42:21.941 INFO:teuthology.orchestra.run.smithi039.stdout: "num_ops": 1 2024-09-14T04:42:21.941 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:42:22.003 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:22.053 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:22.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:22.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:22.156 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:22.182 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:22.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:22.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:22.284 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:22.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:22.338 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:22.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:22.439 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:22.482 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:22.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:22.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:22.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:22 smithi039 ceph-mon[106470]: osdmap e1364: 8 total, 7 up, 8 in 2024-09-14T04:42:22.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:42:22.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:22 smithi039 ceph-mon[111235]: osdmap e1364: 8 total, 7 up, 8 in 2024-09-14T04:42:22.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:42:22.585 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:22.626 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:22.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:22.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:22.728 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:22.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:22 smithi129 ceph-mon[101316]: osdmap e1364: 8 total, 7 up, 8 in 2024-09-14T04:42:22.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:42:22.774 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:22.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:22.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:22.876 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:22.902 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:22.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:22.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:23.004 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:23.030 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:23.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:23.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:23.057 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-14T04:42:23.132 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:23.155 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:23.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:23.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:23.257 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:23.281 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:23.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:23.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:23.352 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:42:23.365 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:42:23.365 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:42:23.383 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:23.409 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:42:23.409 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:42:23.435 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:42:23.450 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:42:23.450 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:42:23.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:23.454 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:23.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:23.503 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:42:23.504 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:42:23.508 INFO:teuthology.orchestra.run.smithi039.stderr:Error ENXIO: problem getting command descriptions from osd.4 2024-09-14T04:42:23.540 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:42:23.541 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:42:23.555 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:23.573 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:42:23.573 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:42:23.609 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:42:23.609 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:42:23.615 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:23 smithi039 ceph-mon[106470]: pgmap v2316: 99 pgs: 1 active+undersized+remapped, 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 81 active+clean; 1.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 7117/55197 objects degraded (12.894%); 6214/55197 objects misplaced (11.258%); 15 MiB/s, 237 objects/s recovering 2024-09-14T04:42:23.616 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:23 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1157648772' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:42:23.616 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:23 smithi039 ceph-mon[111235]: pgmap v2316: 99 pgs: 1 active+undersized+remapped, 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 81 active+clean; 1.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 7117/55197 objects degraded (12.894%); 6214/55197 objects misplaced (11.258%); 15 MiB/s, 237 objects/s recovering 2024-09-14T04:42:23.617 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:23 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1157648772' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:42:23.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:23.628 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:23.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:23.729 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:23.768 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:23.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:23.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:23.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:23 smithi129 ceph-mon[101316]: pgmap v2316: 99 pgs: 1 active+undersized+remapped, 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 2 active+undersized+degraded+remapped+backfilling, 3 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 81 active+clean; 1.1 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 7117/55197 objects degraded (12.894%); 6214/55197 objects misplaced (11.258%); 15 MiB/s, 237 objects/s recovering 2024-09-14T04:42:23.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:23 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1157648772' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:42:23.870 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:23.893 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:23.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:23.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:23.995 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:24.018 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:24.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:24.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:24.120 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:24.143 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:24.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:24.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:24.245 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:24.268 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:24.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:24.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:24.371 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:24.375 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:42:24.398 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:24.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:24.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:24.500 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:24.526 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:24.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:24.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:24.628 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:24.661 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:24.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:24.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:24.763 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:24.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:24 smithi129 ceph-mon[101316]: osdmap e1365: 8 total, 7 up, 8 in 2024-09-14T04:42:24.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:24 smithi129 ceph-mon[101316]: pgmap v2318: 99 pgs: 2 active+remapped+backfilling, 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 85 active+clean; 1021 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 5157/49041 objects degraded (10.516%); 3300/49041 objects misplaced (6.729%); 47 MiB/s, 748 objects/s recovering 2024-09-14T04:42:24.787 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:24.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:24.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:24.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:24 smithi039 ceph-mon[106470]: osdmap e1365: 8 total, 7 up, 8 in 2024-09-14T04:42:24.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:24 smithi039 ceph-mon[106470]: pgmap v2318: 99 pgs: 2 active+remapped+backfilling, 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 85 active+clean; 1021 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 5157/49041 objects degraded (10.516%); 3300/49041 objects misplaced (6.729%); 47 MiB/s, 748 objects/s recovering 2024-09-14T04:42:24.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:24 smithi039 ceph-mon[111235]: osdmap e1365: 8 total, 7 up, 8 in 2024-09-14T04:42:24.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:24 smithi039 ceph-mon[111235]: pgmap v2318: 99 pgs: 2 active+remapped+backfilling, 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 85 active+clean; 1021 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 5157/49041 objects degraded (10.516%); 3300/49041 objects misplaced (6.729%); 47 MiB/s, 748 objects/s recovering 2024-09-14T04:42:24.888 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:24.912 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:24.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:24.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:25.014 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:25.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:25.037 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:25.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:25.138 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:25.161 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:25.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:25.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:25.266 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:25.290 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:25.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:25.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:25.392 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:25.416 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:25.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:25.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:25.518 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:25.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:25.541 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:25.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:25.642 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:25.666 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:25.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:25.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:25.768 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:25 smithi129 ceph-mon[101316]: osdmap e1366: 8 total, 7 up, 8 in 2024-09-14T04:42:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:25 smithi129 ceph-mon[101316]: osdmap e1367: 8 total, 7 up, 8 in 2024-09-14T04:42:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:25 smithi039 ceph-mon[106470]: osdmap e1366: 8 total, 7 up, 8 in 2024-09-14T04:42:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:25 smithi039 ceph-mon[106470]: osdmap e1367: 8 total, 7 up, 8 in 2024-09-14T04:42:25.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:25 smithi039 ceph-mon[111235]: osdmap e1366: 8 total, 7 up, 8 in 2024-09-14T04:42:25.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:25 smithi039 ceph-mon[111235]: osdmap e1367: 8 total, 7 up, 8 in 2024-09-14T04:42:25.799 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:25.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:25.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:25.900 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:25.948 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:25.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:25.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:26.050 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:26.074 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:26.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:26.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:26.080 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:42:26.081 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:42:26.176 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:26.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:26.226 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:26.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:26.327 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:26.372 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:26.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:26.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:26.474 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:26.502 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:26.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:26.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:26.604 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:26.626 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:26.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:26.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:26.728 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:26.756 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:26.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:26.757 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:26.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:26 smithi129 ceph-mon[101316]: pgmap v2321: 99 pgs: 1 peering, 2 active+remapped+backfilling, 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 85 active+clean; 1005 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 553 op/s; 4045/48315 objects degraded (8.372%); 3300/48315 objects misplaced (6.830%); 63 MiB/s, 1.01k objects/s recovering 2024-09-14T04:42:26.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:26 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 5157/49041 objects degraded (10.516%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T04:42:26.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:26 smithi039 ceph-mon[106470]: pgmap v2321: 99 pgs: 1 peering, 2 active+remapped+backfilling, 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 85 active+clean; 1005 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 553 op/s; 4045/48315 objects degraded (8.372%); 3300/48315 objects misplaced (6.830%); 63 MiB/s, 1.01k objects/s recovering 2024-09-14T04:42:26.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:26 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 5157/49041 objects degraded (10.516%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T04:42:26.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:26 smithi039 ceph-mon[111235]: pgmap v2321: 99 pgs: 1 peering, 2 active+remapped+backfilling, 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 85 active+clean; 1005 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 553 op/s; 4045/48315 objects degraded (8.372%); 3300/48315 objects misplaced (6.830%); 63 MiB/s, 1.01k objects/s recovering 2024-09-14T04:42:26.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:26 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 5157/49041 objects degraded (10.516%), 7 pgs degraded (PG_DEGRADED) 2024-09-14T04:42:26.858 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:26.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:26.898 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:26.898 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:26.999 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:27.026 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:27.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:27.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:27.128 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:27.154 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:27.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:27.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:27.256 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:27.282 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:27.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:27.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:27.384 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:27.410 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:27.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:27.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:27.513 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:27.552 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:27.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:27.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:27.654 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:27.698 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:27.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:27.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:27.800 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:27.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:27.843 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:27.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:27.944 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:27.969 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:27.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:27.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:28.071 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:28.470 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:42:28.522 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:28.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:28.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:28.624 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:28.649 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:28.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:28.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:28.751 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:28.779 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:28.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:28.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:28.881 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:28.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:28.917 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:28.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:28.938 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:28 smithi039 ceph-mon[106470]: pgmap v2322: 99 pgs: 1 peering, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 86 active+clean; 973 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 469 op/s; 3873/46740 objects degraded (8.286%); 2998/46740 objects misplaced (6.414%); 47 MiB/s, 744 objects/s recovering 2024-09-14T04:42:28.938 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:42:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:42:28] "GET /metrics HTTP/1.1" 200 36794 "" "Prometheus/2.51.0" 2024-09-14T04:42:28.939 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:28 smithi039 ceph-mon[111235]: pgmap v2322: 99 pgs: 1 peering, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 86 active+clean; 973 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 469 op/s; 3873/46740 objects degraded (8.286%); 2998/46740 objects misplaced (6.414%); 47 MiB/s, 744 objects/s recovering 2024-09-14T04:42:29.018 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:29.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:28 smithi129 ceph-mon[101316]: pgmap v2322: 99 pgs: 1 peering, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 86 active+clean; 973 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 469 op/s; 3873/46740 objects degraded (8.286%); 2998/46740 objects misplaced (6.414%); 47 MiB/s, 744 objects/s recovering 2024-09-14T04:42:29.045 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:29.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:29.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:29.147 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:29.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:29.169 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:29.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:29.205 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:42:29.205 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:42:29.270 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:29.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:29.293 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:29.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:29.394 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:29.417 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:29.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:29.418 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:29.519 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:29.542 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:29.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:29.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:29.644 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:29.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:29.668 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:29.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:29.680 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is unset 2024-09-14T04:42:29.769 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:29.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:29 smithi039 ceph-mon[106470]: osdmap e1368: 8 total, 7 up, 8 in 2024-09-14T04:42:29.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:29 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/421778568' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:42:29.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:29 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:42:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:29 smithi039 ceph-mon[111235]: osdmap e1368: 8 total, 7 up, 8 in 2024-09-14T04:42:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:29 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/421778568' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:42:29.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:29 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:42:29.819 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:29.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:29.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:29.921 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:29.950 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:29.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:29.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:29.992 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-14T04:42:30.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:29 smithi129 ceph-mon[101316]: osdmap e1368: 8 total, 7 up, 8 in 2024-09-14T04:42:30.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:29 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/421778568' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:42:30.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:29 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:42:30.052 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:30.078 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:30.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:30.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:30.181 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:30.207 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:30.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:30.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:30.309 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:30.350 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:30.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:30.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:30.452 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:30.496 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:30.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:30.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:30.598 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:30.634 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:30.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:30.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:30.735 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:30.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:30.777 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:30.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:30.878 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:30.905 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:30.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:30.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:31.007 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:31.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:30 smithi129 ceph-mon[101316]: pgmap v2324: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 1 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 87 active+clean; 840 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 482 op/s; 2466/40362 objects degraded (6.110%); 1555/40362 objects misplaced (3.853%); 24 MiB/s, 388 objects/s recovering 2024-09-14T04:42:31.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:30 smithi129 ceph-mon[101316]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:42:31.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:30 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:42:31.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:30 smithi129 ceph-mon[101316]: osdmap e1369: 8 total, 7 up, 8 in 2024-09-14T04:42:31.029 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:31.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:31.030 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:30 smithi039 ceph-mon[106470]: pgmap v2324: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 1 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 87 active+clean; 840 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 482 op/s; 2466/40362 objects degraded (6.110%); 1555/40362 objects misplaced (3.853%); 24 MiB/s, 388 objects/s recovering 2024-09-14T04:42:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:30 smithi039 ceph-mon[106470]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:42:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:30 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:42:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:30 smithi039 ceph-mon[106470]: osdmap e1369: 8 total, 7 up, 8 in 2024-09-14T04:42:31.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:30 smithi039 ceph-mon[111235]: pgmap v2324: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 1 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 87 active+clean; 840 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 482 op/s; 2466/40362 objects degraded (6.110%); 1555/40362 objects misplaced (3.853%); 24 MiB/s, 388 objects/s recovering 2024-09-14T04:42:31.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:30 smithi039 ceph-mon[111235]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:42:31.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:30 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:42:31.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:30 smithi039 ceph-mon[111235]: osdmap e1369: 8 total, 7 up, 8 in 2024-09-14T04:42:31.131 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:31.154 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:31.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:31.155 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:31.256 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:31.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:31.279 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:31.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:31.380 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:31.407 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:31.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:31.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:31.509 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:31.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:31.531 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:31.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:31.632 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:31.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:31.654 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:31.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:31.755 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:31.786 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:31.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:31.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:31.888 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:31.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:31.921 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:31.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:32.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:31 smithi129 ceph-mon[101316]: osdmap e1370: 8 total, 7 up, 8 in 2024-09-14T04:42:32.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:31 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 2466/40362 objects degraded (6.110%), 6 pgs degraded (PG_DEGRADED) 2024-09-14T04:42:32.022 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:32.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:31 smithi039 ceph-mon[106470]: osdmap e1370: 8 total, 7 up, 8 in 2024-09-14T04:42:32.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:31 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 2466/40362 objects degraded (6.110%), 6 pgs degraded (PG_DEGRADED) 2024-09-14T04:42:32.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:31 smithi039 ceph-mon[111235]: osdmap e1370: 8 total, 7 up, 8 in 2024-09-14T04:42:32.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:31 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 2466/40362 objects degraded (6.110%), 6 pgs degraded (PG_DEGRADED) 2024-09-14T04:42:32.047 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:32.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:32.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:32.149 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:32.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:32.201 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:32.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:32.305 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:32.339 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:32.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:32.340 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:32.441 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:32.465 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:32.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:32.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:32.567 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:32.608 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:32.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:32.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:32.710 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:32.735 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:32.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:32.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:32.837 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:32.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:32.865 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:32.866 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:32.966 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:33.002 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:33.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:33.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:32 smithi129 ceph-mon[101316]: pgmap v2327: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 1 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 88 active+clean; 812 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 2466/39033 objects degraded (6.318%); 1555/39033 objects misplaced (3.984%); 13 MiB/s, 209 objects/s recovering 2024-09-14T04:42:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:32 smithi129 ceph-mon[101316]: osdmap e1371: 8 total, 7 up, 8 in 2024-09-14T04:42:33.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:32 smithi039 ceph-mon[106470]: pgmap v2327: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 1 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 88 active+clean; 812 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 2466/39033 objects degraded (6.318%); 1555/39033 objects misplaced (3.984%); 13 MiB/s, 209 objects/s recovering 2024-09-14T04:42:33.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:32 smithi039 ceph-mon[106470]: osdmap e1371: 8 total, 7 up, 8 in 2024-09-14T04:42:33.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:32 smithi039 ceph-mon[111235]: pgmap v2327: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 1 active+remapped+backfill_wait, 2 active+undersized, 4 active+undersized+degraded, 88 active+clean; 812 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 515 op/s; 2466/39033 objects degraded (6.318%); 1555/39033 objects misplaced (3.984%); 13 MiB/s, 209 objects/s recovering 2024-09-14T04:42:33.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:32 smithi039 ceph-mon[111235]: osdmap e1371: 8 total, 7 up, 8 in 2024-09-14T04:42:33.105 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:33.130 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:33.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:33.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:33.232 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:33.259 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:33.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:33.260 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:33.361 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:33.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:33.404 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:33.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:33.505 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:33.531 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:33.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:33.533 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:33.634 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:33.672 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:33.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:33.673 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:33.774 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:33.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:33.812 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:33.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:33.913 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:33.949 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:33.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:33.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:34.051 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:34.092 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:34.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:34.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:34.194 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:34.221 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:34.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:34.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:34.323 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:34.365 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:34.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:34.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:34.467 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:34.505 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:34.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:34.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:34.607 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:34.629 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:34.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:34.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:34.731 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:34.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:34.774 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:34.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:34.875 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:34.916 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:34.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:34.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:35.018 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:35.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:34 smithi129 ceph-mon[101316]: pgmap v2329: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 2 active+undersized, 4 active+undersized+degraded, 90 active+clean; 638 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 1892/30678 objects degraded (6.167%); 573/30678 objects misplaced (1.868%); 9.2 MiB/s, 147 objects/s recovering 2024-09-14T04:42:35.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:34 smithi039 ceph-mon[106470]: pgmap v2329: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 2 active+undersized, 4 active+undersized+degraded, 90 active+clean; 638 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 1892/30678 objects degraded (6.167%); 573/30678 objects misplaced (1.868%); 9.2 MiB/s, 147 objects/s recovering 2024-09-14T04:42:35.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:34 smithi039 ceph-mon[111235]: pgmap v2329: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 2 active+undersized, 4 active+undersized+degraded, 90 active+clean; 638 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 1892/30678 objects degraded (6.167%); 573/30678 objects misplaced (1.868%); 9.2 MiB/s, 147 objects/s recovering 2024-09-14T04:42:35.044 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:35.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:35.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:35.147 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:35.186 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:35.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:35.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:35.288 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:35.318 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:35.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:35.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:35.419 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:35.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:35.463 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:35.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:35.564 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:35.593 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:35.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:35.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:35.695 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:35.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:35.722 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:35.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:35.823 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:35.862 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:35.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:35.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:35.964 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:35.969 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:42:36.025 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:36.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:36.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:36.128 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:36.159 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:36.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:36.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:36.261 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:36.307 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:36.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:36.308 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:36.409 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:36.439 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:36.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:36.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:36.541 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:36.569 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:36.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:36.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:36.671 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:36.702 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:36.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:36.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:36.804 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:36.853 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:36.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:36.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:36.918 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:36 smithi039 ceph-mon[106470]: pgmap v2330: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 2 active+undersized, 4 active+undersized+degraded, 90 active+clean; 606 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 1892/29157 objects degraded (6.489%); 573/29157 objects misplaced (1.965%); 7.6 MiB/s, 121 objects/s recovering 2024-09-14T04:42:36.918 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:36 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 1892/29157 objects degraded (6.489%), 5 pgs degraded (PG_DEGRADED) 2024-09-14T04:42:36.919 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:36 smithi039 ceph-mon[111235]: pgmap v2330: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 2 active+undersized, 4 active+undersized+degraded, 90 active+clean; 606 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 1892/29157 objects degraded (6.489%); 573/29157 objects misplaced (1.965%); 7.6 MiB/s, 121 objects/s recovering 2024-09-14T04:42:36.919 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:36 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 1892/29157 objects degraded (6.489%), 5 pgs degraded (PG_DEGRADED) 2024-09-14T04:42:36.955 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:36.983 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:36.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:36.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:37.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:36 smithi129 ceph-mon[101316]: pgmap v2330: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 2 active+undersized, 4 active+undersized+degraded, 90 active+clean; 606 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 622 op/s; 1892/29157 objects degraded (6.489%); 573/29157 objects misplaced (1.965%); 7.6 MiB/s, 121 objects/s recovering 2024-09-14T04:42:37.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:36 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 1892/29157 objects degraded (6.489%), 5 pgs degraded (PG_DEGRADED) 2024-09-14T04:42:37.085 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:37.130 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:37.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:37.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:37.232 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:37.261 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:37.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:37.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:37.363 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:37.406 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:37.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:37.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:37.508 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:37.542 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:37.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:37.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:37.645 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:37.671 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:37.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:37.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:37.773 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:37.802 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:37.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:37.803 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:37.904 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:37.949 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:37.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:37.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:38.005 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:42:38.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:42:38.026 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:42:38.026 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:42:38.051 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:38.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:38.079 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:38.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:38.180 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:38.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:38.235 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:38.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:38.336 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:38.359 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:38.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:38.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:38.461 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:38.499 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:38.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:38.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:38.600 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:38.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:38.630 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:38.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:38.731 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:38.757 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:38.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:38.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:38.765 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:42:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:42:38] "GET /metrics HTTP/1.1" 200 36791 "" "Prometheus/2.51.0" 2024-09-14T04:42:38.859 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:38.885 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:38.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:38.886 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:38.987 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:39.011 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:39.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:39.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:39.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:38 smithi129 ceph-mon[101316]: pgmap v2331: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 2 active+undersized, 4 active+undersized+degraded, 90 active+clean; 544 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 1550/26160 objects degraded (5.925%); 573/26160 objects misplaced (2.190%); 5.7 MiB/s, 91 objects/s recovering 2024-09-14T04:42:39.035 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:38 smithi039 ceph-mon[106470]: pgmap v2331: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 2 active+undersized, 4 active+undersized+degraded, 90 active+clean; 544 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 1550/26160 objects degraded (5.925%); 573/26160 objects misplaced (2.190%); 5.7 MiB/s, 91 objects/s recovering 2024-09-14T04:42:39.035 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:38 smithi039 ceph-mon[111235]: pgmap v2331: 99 pgs: 1 active+undersized+degraded+remapped, 2 active+clean+remapped, 2 active+undersized, 4 active+undersized+degraded, 90 active+clean; 544 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 598 op/s; 1550/26160 objects degraded (5.925%); 573/26160 objects misplaced (2.190%); 5.7 MiB/s, 91 objects/s recovering 2024-09-14T04:42:39.071 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:42:39.114 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:39.163 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:39.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:39.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:39.265 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:39.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:39.312 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:39.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:39.413 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:39.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:39.463 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:39.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:39.487 INFO:teuthology.orchestra.run.smithi039.stdout:moved 17 / 297 (5.72391%) 2024-09-14T04:42:39.487 INFO:teuthology.orchestra.run.smithi039.stdout:avg 42.4286 2024-09-14T04:42:39.487 INFO:teuthology.orchestra.run.smithi039.stdout:stddev 9.0283 -> 4.43548 (expected baseline 6.03053) 2024-09-14T04:42:39.487 INFO:teuthology.orchestra.run.smithi039.stdout:min osd.1 with 23 -> 40 pgs (0.542088 -> 0.942761 * mean) 2024-09-14T04:42:39.487 INFO:teuthology.orchestra.run.smithi039.stdout:max osd.3 with 50 -> 50 pgs (1.17845 -> 1.17845 * mean) 2024-09-14T04:42:39.487 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:42:39.487 INFO:teuthology.orchestra.run.smithi039.stdout:oload 110 2024-09-14T04:42:39.487 INFO:teuthology.orchestra.run.smithi039.stdout:max_change 3 2024-09-14T04:42:39.487 INFO:teuthology.orchestra.run.smithi039.stdout:max_change_osds 4 2024-09-14T04:42:39.488 INFO:teuthology.orchestra.run.smithi039.stdout:average_utilization 0.0078 2024-09-14T04:42:39.488 INFO:teuthology.orchestra.run.smithi039.stdout:overload_utilization 0.0086 2024-09-14T04:42:39.488 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4 weight 0.1967 -> 0.0550 2024-09-14T04:42:39.488 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1 weight 0.4612 -> 0.7210 2024-09-14T04:42:39.488 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5 weight 0.9052 -> 1.0000 2024-09-14T04:42:39.488 INFO:teuthology.orchestra.run.smithi039.stderr:no change 2024-09-14T04:42:39.565 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:39.589 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:39.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:39.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:39.647 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:42:39 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T04:42:39.484+0000 7f268b1da640 -1 mgr.server reply reply (3) No such process no change 2024-09-14T04:42:39.691 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:39.731 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:39.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:39.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:39.833 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:39.857 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:39.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:39.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:39.860 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:42:39.860 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:42:39.905 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:42:39.906 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:42:39.944 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:42:39.945 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:42:39.959 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:39.981 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:42:39.981 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:42:39.983 INFO:teuthology.orchestra.run.smithi039.stderr:Error ENXIO: problem getting command descriptions from osd.4 2024-09-14T04:42:40.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:40.002 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:40.002 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:40.015 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:42:40.015 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:42:40.042 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:42:40.043 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:42:40.071 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:42:40.072 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:42:40.103 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:40.132 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:40.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:40.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:40.234 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:40.256 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:40.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:40.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:40.358 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:40.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:40.381 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:40.381 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:40.482 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:40.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:40.506 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:40.506 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:40.607 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:40.629 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:40.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:40.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:40.732 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:40.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:40.755 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:40.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:40.856 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:40.893 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:40.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:40.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:40.995 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:41.025 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:41.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:41.026 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:41.127 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:41.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:41.155 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:41.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:41.256 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:40 smithi129 ceph-mon[101316]: from='client.35367 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:42:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:40 smithi129 ceph-mon[101316]: pgmap v2332: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 2 active+undersized, 4 active+undersized+degraded, 90 active+clean; 328 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 781/15783 objects degraded (4.948%); 125/15783 objects misplaced (0.792%); 6.5 MiB/s, 104 objects/s recovering 2024-09-14T04:42:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:40 smithi129 ceph-mon[101316]: osdmap e1372: 8 total, 7 up, 8 in 2024-09-14T04:42:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:40 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1265294710' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:42:41.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:40 smithi039 ceph-mon[106470]: from='client.35367 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:42:41.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:40 smithi039 ceph-mon[106470]: pgmap v2332: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 2 active+undersized, 4 active+undersized+degraded, 90 active+clean; 328 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 781/15783 objects degraded (4.948%); 125/15783 objects misplaced (0.792%); 6.5 MiB/s, 104 objects/s recovering 2024-09-14T04:42:41.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:40 smithi039 ceph-mon[106470]: osdmap e1372: 8 total, 7 up, 8 in 2024-09-14T04:42:41.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:40 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1265294710' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:42:41.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:40 smithi039 ceph-mon[111235]: from='client.35367 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:42:41.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:40 smithi039 ceph-mon[111235]: pgmap v2332: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 2 active+undersized, 4 active+undersized+degraded, 90 active+clean; 328 MiB data, 5.1 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 781/15783 objects degraded (4.948%); 125/15783 objects misplaced (0.792%); 6.5 MiB/s, 104 objects/s recovering 2024-09-14T04:42:41.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:40 smithi039 ceph-mon[111235]: osdmap e1372: 8 total, 7 up, 8 in 2024-09-14T04:42:41.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:40 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1265294710' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:42:41.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:41.292 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:41.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:41.393 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:41.419 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:42:41.420 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:42:41.440 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:41.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:41.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:41.542 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:41.568 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:41.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:41.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:41.669 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:41.701 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:41.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:41.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:41.804 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:41.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:41.854 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:41.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:41.955 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:41.962 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:41 smithi039 ceph-mon[106470]: osdmap e1373: 8 total, 7 up, 8 in 2024-09-14T04:42:41.962 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:41 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 781/15783 objects degraded (4.948%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-14T04:42:41.964 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:41 smithi039 ceph-mon[111235]: osdmap e1373: 8 total, 7 up, 8 in 2024-09-14T04:42:41.964 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:41 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 781/15783 objects degraded (4.948%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-14T04:42:41.996 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:41.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:41.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:42.098 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:42.125 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:42.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:42.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:42.227 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:42.239 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2024-09-14T04:42:42.240 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight-by-utilization 110 1.0 2024-09-14T04:42:42.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:41 smithi129 ceph-mon[101316]: osdmap e1373: 8 total, 7 up, 8 in 2024-09-14T04:42:42.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:41 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 781/15783 objects degraded (4.948%), 5 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-14T04:42:42.273 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:42.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:42.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:42.375 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:42.435 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:42.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:42.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:42.537 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:42.582 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:42.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:42.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:42.683 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:42.726 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:42.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:42.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:42.761 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:42:42.821 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:42:42.821 INFO:teuthology.orchestra.run.smithi039.stdout: "size": 20, 2024-09-14T04:42:42.821 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 600, 2024-09-14T04:42:42.821 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [ 2024-09-14T04:42:42.821 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.821 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.44840.0:1332 52.1f 52:f9ed7d24:::rbd_data.af28a5612578.0000000000000002:head [set-alloc-hint object_size 4194304 write_size 4194304,writefull 0~4194304] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1235)", 2024-09-14T04:42:42.821 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:34:58.263323+0000", 2024-09-14T04:42:42.821 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 464.55401461500003, 2024-09-14T04:42:42.822 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.038524251000000002, 2024-09-14T04:42:42.822 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:42.822 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:42.822 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:42.822 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:42:42.822 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.44840", 2024-09-14T04:42:42.822 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1436701460", 2024-09-14T04:42:42.822 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 1332 2024-09-14T04:42:42.822 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.822 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:42.822 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.822 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:42.823 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:34:58.263323+0000", 2024-09-14T04:42:42.823 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.823 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.823 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.823 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:42.823 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:34:58.263323+0000", 2024-09-14T04:42:42.823 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.823 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.823 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.823 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:42.823 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:34:58.263326+0000", 2024-09-14T04:42:42.823 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.368e-06 2024-09-14T04:42:42.823 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.823 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.824 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:42.824 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:34:58.265693+0000", 2024-09-14T04:42:42.824 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.002367877 2024-09-14T04:42:42.824 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.824 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.824 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:42.824 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:34:58.265742+0000", 2024-09-14T04:42:42.824 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.8408999999999997e-05 2024-09-14T04:42:42.824 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.824 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.824 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:42.824 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:34:58.265764+0000", 2024-09-14T04:42:42.825 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1790999999999999e-05 2024-09-14T04:42:42.825 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.825 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.825 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:42.825 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:34:58.265902+0000", 2024-09-14T04:42:42.825 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013848199999999999 2024-09-14T04:42:42.825 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.825 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.825 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:42.825 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:34:58.266123+0000", 2024-09-14T04:42:42.825 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000220719 2024-09-14T04:42:42.825 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.825 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.826 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:42:42.826 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:34:58.266317+0000", 2024-09-14T04:42:42.826 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00019432499999999999 2024-09-14T04:42:42.826 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.826 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.826 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:42:42.826 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:34:58.279771+0000", 2024-09-14T04:42:42.826 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.013453784999999999 2024-09-14T04:42:42.826 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.826 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.826 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:42.826 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:34:58.280248+0000", 2024-09-14T04:42:42.826 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000477004 2024-09-14T04:42:42.827 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.827 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.827 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:42.827 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:34:58.301774+0000", 2024-09-14T04:42:42.827 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.021525788000000001 2024-09-14T04:42:42.827 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.827 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.827 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:42.827 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:34:58.301819+0000", 2024-09-14T04:42:42.827 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5531000000000002e-05 2024-09-14T04:42:42.827 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.827 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.827 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:42.828 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:34:58.301847+0000", 2024-09-14T04:42:42.828 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.8172000000000001e-05 2024-09-14T04:42:42.828 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.828 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:42.828 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.828 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.828 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.828 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.34914.0:2209 52.d 52:b2fb3e73:::rbd_data.8862fb33ee1a.0000000000000054:head [set-alloc-hint object_size 4194304 write_size 4194304,write 0~1976798] snapc 67=[67] ondisk+write+known_if_redirected+supports_pool_eio e1269)", 2024-09-14T04:42:42.829 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:36:19.222766+0000", 2024-09-14T04:42:42.829 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 383.59457213799999, 2024-09-14T04:42:42.829 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.064658328000000001, 2024-09-14T04:42:42.829 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:42.829 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:42.829 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:42.829 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:42:42.829 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.34914", 2024-09-14T04:42:42.829 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1048804342", 2024-09-14T04:42:42.829 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 2209 2024-09-14T04:42:42.829 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.829 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:42.829 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.829 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:42.830 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:19.222766+0000", 2024-09-14T04:42:42.830 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.830 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.830 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.830 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:42.830 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:19.222766+0000", 2024-09-14T04:42:42.830 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.830 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.830 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.830 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:42.830 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:19.222768+0000", 2024-09-14T04:42:42.830 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.599e-06 2024-09-14T04:42:42.830 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.831 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.831 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:42.831 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:19.224765+0000", 2024-09-14T04:42:42.831 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001996425 2024-09-14T04:42:42.831 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.831 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.831 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:42.831 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:19.224790+0000", 2024-09-14T04:42:42.831 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.5622e-05 2024-09-14T04:42:42.831 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.831 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.831 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:42.831 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:19.224804+0000", 2024-09-14T04:42:42.831 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3993e-05 2024-09-14T04:42:42.832 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.832 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.832 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:42.832 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:19.224900+0000", 2024-09-14T04:42:42.832 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.5847000000000006e-05 2024-09-14T04:42:42.832 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.832 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.832 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:42.832 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:19.225144+0000", 2024-09-14T04:42:42.832 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00024429200000000001 2024-09-14T04:42:42.832 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.832 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.832 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:42:42.833 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:19.225380+0000", 2024-09-14T04:42:42.833 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000235119 2024-09-14T04:42:42.833 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.833 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.833 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:42:42.833 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:19.230496+0000", 2024-09-14T04:42:42.833 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0051164879999999998 2024-09-14T04:42:42.833 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.833 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.833 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:42.833 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:19.243509+0000", 2024-09-14T04:42:42.833 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.013012998 2024-09-14T04:42:42.833 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.833 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.834 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:42.834 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:19.287295+0000", 2024-09-14T04:42:42.834 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.043786261999999999 2024-09-14T04:42:42.834 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.834 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.834 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:42.834 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:19.287372+0000", 2024-09-14T04:42:42.834 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6697000000000001e-05 2024-09-14T04:42:42.834 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.834 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.834 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:42.834 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:19.287424+0000", 2024-09-14T04:42:42.834 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1986e-05 2024-09-14T04:42:42.834 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.835 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:42.835 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.835 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.835 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.835 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.34914.0:2852 52.1f 52:fae8de57:::rbd_data.8862fb33ee1a.0000000000000032:head [set-alloc-hint object_size 4194304 write_size 4194304,write 2319715~1874589] snapc 73=[73,72,71,70,6f,6e,6d,6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1281)", 2024-09-14T04:42:42.835 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:36:31.355463+0000", 2024-09-14T04:42:42.835 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 371.46187470699999, 2024-09-14T04:42:42.835 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.050093245000000002, 2024-09-14T04:42:42.835 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:42.835 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:42.835 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:42.835 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:42.836 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:42:42.836 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.34914", 2024-09-14T04:42:42.837 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1048804342", 2024-09-14T04:42:42.837 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 2852 2024-09-14T04:42:42.837 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.837 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:42.837 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.837 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:42.837 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.355463+0000", 2024-09-14T04:42:42.837 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.837 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.837 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.837 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:42.837 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.355463+0000", 2024-09-14T04:42:42.837 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.837 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.838 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.838 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:42.838 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.355467+0000", 2024-09-14T04:42:42.838 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.5949999999999999e-06 2024-09-14T04:42:42.838 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.838 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.838 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:42.838 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.357569+0000", 2024-09-14T04:42:42.838 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0021018370000000001 2024-09-14T04:42:42.838 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.838 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.838 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:42.838 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.357597+0000", 2024-09-14T04:42:42.838 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.8283e-05 2024-09-14T04:42:42.839 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.839 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.839 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:42.839 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.357608+0000", 2024-09-14T04:42:42.839 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0693000000000001e-05 2024-09-14T04:42:42.839 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.839 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.839 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:42.839 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.357687+0000", 2024-09-14T04:42:42.839 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9002e-05 2024-09-14T04:42:42.839 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.839 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.839 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op must wait for map", 2024-09-14T04:42:42.839 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.357693+0000", 2024-09-14T04:42:42.840 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8449999999999997e-06 2024-09-14T04:42:42.840 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.840 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.840 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:42.840 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.360429+0000", 2024-09-14T04:42:42.840 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0027357509999999998 2024-09-14T04:42:42.840 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.840 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.840 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:42.840 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.360545+0000", 2024-09-14T04:42:42.840 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011612600000000001 2024-09-14T04:42:42.840 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.840 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.840 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:42:42.841 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.360664+0000", 2024-09-14T04:42:42.841 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000118245 2024-09-14T04:42:42.841 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.841 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.841 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:42:42.841 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.366855+0000", 2024-09-14T04:42:42.841 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0061913899999999997 2024-09-14T04:42:42.841 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.841 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.841 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:42.841 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.367103+0000", 2024-09-14T04:42:42.841 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00024819399999999998 2024-09-14T04:42:42.841 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.841 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.841 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:42.842 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.405476+0000", 2024-09-14T04:42:42.842 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.038372457999999998 2024-09-14T04:42:42.842 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.842 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.842 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:42.842 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.405518+0000", 2024-09-14T04:42:42.842 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2673000000000003e-05 2024-09-14T04:42:42.842 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.842 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.842 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:42.842 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.405556+0000", 2024-09-14T04:42:42.842 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.8152999999999999e-05 2024-09-14T04:42:42.842 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.843 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:42.843 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.843 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.843 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.843 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.34914.0:2910 52.d 52:b2fb3e73:::rbd_data.8862fb33ee1a.0000000000000054:head [write 1277026~2917278] snapc 74=[74,73,72,71,70,6f,6e,6d,6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-09-14T04:42:42.843 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:36:32.376602+0000", 2024-09-14T04:42:42.843 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 370.44073552700002, 2024-09-14T04:42:42.843 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.049396374, 2024-09-14T04:42:42.843 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:42.843 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:42.843 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:42.844 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:42:42.844 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.34914", 2024-09-14T04:42:42.844 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1048804342", 2024-09-14T04:42:42.844 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 2910 2024-09-14T04:42:42.844 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.844 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:42.844 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.844 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:42.844 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.376602+0000", 2024-09-14T04:42:42.844 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.844 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.844 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.845 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:42.845 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.376602+0000", 2024-09-14T04:42:42.845 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.845 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.845 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.845 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:42.845 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.376606+0000", 2024-09-14T04:42:42.845 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.7869999999999999e-06 2024-09-14T04:42:42.845 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.845 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.845 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:42.845 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.379637+0000", 2024-09-14T04:42:42.845 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0030309159999999998 2024-09-14T04:42:42.845 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.846 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.846 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:42.846 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.379672+0000", 2024-09-14T04:42:42.846 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.4740999999999998e-05 2024-09-14T04:42:42.846 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.846 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.846 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:42.846 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.379707+0000", 2024-09-14T04:42:42.846 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.5601e-05 2024-09-14T04:42:42.846 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.846 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.846 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:42.846 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.379775+0000", 2024-09-14T04:42:42.846 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7509999999999996e-05 2024-09-14T04:42:42.847 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.847 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.847 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op must wait for map", 2024-09-14T04:42:42.847 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.379779+0000", 2024-09-14T04:42:42.847 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4190000000000004e-06 2024-09-14T04:42:42.847 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.847 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.847 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:42.847 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.382895+0000", 2024-09-14T04:42:42.847 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0031161439999999999 2024-09-14T04:42:42.847 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.847 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.847 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:42.847 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.383003+0000", 2024-09-14T04:42:42.847 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000107284 2024-09-14T04:42:42.848 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.848 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.848 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:42:42.848 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.383203+0000", 2024-09-14T04:42:42.848 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000200158 2024-09-14T04:42:42.848 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.848 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.848 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:42:42.848 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.389485+0000", 2024-09-14T04:42:42.848 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0062822579999999998 2024-09-14T04:42:42.848 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.848 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.848 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:42.849 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.422592+0000", 2024-09-14T04:42:42.849 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.033107325999999999 2024-09-14T04:42:42.849 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.849 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.849 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:42.849 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.425918+0000", 2024-09-14T04:42:42.849 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.003325864 2024-09-14T04:42:42.849 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.849 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.849 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:42.849 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.425964+0000", 2024-09-14T04:42:42.849 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5235000000000003e-05 2024-09-14T04:42:42.849 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.850 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.850 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:42.850 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.425999+0000", 2024-09-14T04:42:42.850 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.5131e-05 2024-09-14T04:42:42.850 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.850 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:42.850 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.850 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.850 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.850 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.34914.0:2978 52.1f 52:fa5f4250:::rbd_data.8862fb33ee1a.000000000000005e:head [write 0~3797557] snapc 75=[75,74,73,72,71,70,6f,6e,6d,6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1283)", 2024-09-14T04:42:42.850 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:36:33.417353+0000", 2024-09-14T04:42:42.850 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 369.39998458399998, 2024-09-14T04:42:42.850 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.034871017999999997, 2024-09-14T04:42:42.851 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:42.851 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:42.851 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:42.851 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:42:42.851 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.34914", 2024-09-14T04:42:42.851 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1048804342", 2024-09-14T04:42:42.851 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 2978 2024-09-14T04:42:42.851 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.851 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:42.851 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.851 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:42.851 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:33.417353+0000", 2024-09-14T04:42:42.851 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.851 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.851 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.852 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:42.852 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:33.417353+0000", 2024-09-14T04:42:42.852 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.852 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.852 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.852 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:42.852 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:33.417356+0000", 2024-09-14T04:42:42.852 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.6690000000000002e-06 2024-09-14T04:42:42.852 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.852 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.852 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:42.852 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:33.420157+0000", 2024-09-14T04:42:42.852 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0028008080000000001 2024-09-14T04:42:42.852 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.853 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.853 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:42.853 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:33.420179+0000", 2024-09-14T04:42:42.853 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1795999999999999e-05 2024-09-14T04:42:42.853 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.853 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.853 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:42.853 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:33.420187+0000", 2024-09-14T04:42:42.853 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0239999999999997e-06 2024-09-14T04:42:42.853 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.853 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.853 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:42.853 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:33.420251+0000", 2024-09-14T04:42:42.853 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.4635999999999996e-05 2024-09-14T04:42:42.854 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.854 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.854 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:42.854 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:33.420335+0000", 2024-09-14T04:42:42.854 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.3690999999999995e-05 2024-09-14T04:42:42.854 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.854 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.854 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:42:42.854 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:33.420499+0000", 2024-09-14T04:42:42.854 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00016363700000000001 2024-09-14T04:42:42.854 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.854 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.854 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:42:42.854 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:33.435673+0000", 2024-09-14T04:42:42.855 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.015174653999999999 2024-09-14T04:42:42.855 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.855 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.855 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:42.855 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:33.437382+0000", 2024-09-14T04:42:42.855 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0017083720000000001 2024-09-14T04:42:42.855 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.855 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.855 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:42.855 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:33.452139+0000", 2024-09-14T04:42:42.855 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.014757937 2024-09-14T04:42:42.855 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.855 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.855 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:42.855 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:33.452182+0000", 2024-09-14T04:42:42.856 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2571999999999998e-05 2024-09-14T04:42:42.856 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.856 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.856 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:42.856 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:33.452224+0000", 2024-09-14T04:42:42.856 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2222000000000003e-05 2024-09-14T04:42:42.856 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.856 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:42.856 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.856 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.856 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.856 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.34914.0:7346 52.d 52:b6b5c6cb:::rbd_data.8862ba52cd58.0000000000000002:head [set-alloc-hint object_size 4194304 write_size 4194304,writefull 0~4194304] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1311)", 2024-09-14T04:42:42.856 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:37:01.028984+0000", 2024-09-14T04:42:42.857 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 341.78835420500002, 2024-09-14T04:42:42.857 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.047506552000000001, 2024-09-14T04:42:42.857 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:42.857 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:42.857 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:42.857 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:42:42.857 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.34914", 2024-09-14T04:42:42.857 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1048804342", 2024-09-14T04:42:42.857 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 7346 2024-09-14T04:42:42.857 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.857 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:42.857 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.857 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:42.857 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.028984+0000", 2024-09-14T04:42:42.858 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.858 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.858 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.858 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:42.858 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.028984+0000", 2024-09-14T04:42:42.858 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.858 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.858 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.858 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:42.858 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.028987+0000", 2024-09-14T04:42:42.858 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.9129999999999999e-06 2024-09-14T04:42:42.859 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.859 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.859 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:42.859 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.032923+0000", 2024-09-14T04:42:42.859 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0039369410000000002 2024-09-14T04:42:42.859 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.859 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.859 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:42.859 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.032966+0000", 2024-09-14T04:42:42.859 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2792e-05 2024-09-14T04:42:42.859 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.859 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.860 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:42.860 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.032980+0000", 2024-09-14T04:42:42.860 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4093e-05 2024-09-14T04:42:42.860 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.860 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.860 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:42.860 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.033057+0000", 2024-09-14T04:42:42.860 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6240000000000002e-05 2024-09-14T04:42:42.860 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.860 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.860 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:42.860 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.033175+0000", 2024-09-14T04:42:42.860 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011859 2024-09-14T04:42:42.860 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.861 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.861 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:42:42.861 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.033279+0000", 2024-09-14T04:42:42.861 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000103942 2024-09-14T04:42:42.861 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.861 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.861 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:42:42.861 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.043730+0000", 2024-09-14T04:42:42.861 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.010450536999999999 2024-09-14T04:42:42.861 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.861 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.861 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:42.861 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.074688+0000", 2024-09-14T04:42:42.862 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.030958066999999999 2024-09-14T04:42:42.862 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.862 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.862 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:42:42.862 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.076379+0000", 2024-09-14T04:42:42.862 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001691269 2024-09-14T04:42:42.862 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.862 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.862 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:42.862 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.076426+0000", 2024-09-14T04:42:42.862 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6891000000000002e-05 2024-09-14T04:42:42.862 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.862 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.862 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:42.863 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.076490+0000", 2024-09-14T04:42:42.863 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.4276999999999995e-05 2024-09-14T04:42:42.863 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.863 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:42.863 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.863 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.863 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.863 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:24695 53.0 e1319/1315 53:03650f8a:::benchmark_data_smithi039_224949_object24694:head v 1319'1508, mlcod=1319'1505)", 2024-09-14T04:42:42.863 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:38:44.028985+0000", 2024-09-14T04:42:42.863 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 238.78835324600001, 2024-09-14T04:42:42.863 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.052261575999999997, 2024-09-14T04:42:42.863 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:42.863 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:42.863 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:42.864 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:42.864 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.864 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:42.864 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.028985+0000", 2024-09-14T04:42:42.864 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.864 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.864 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.864 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:42.864 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.028985+0000", 2024-09-14T04:42:42.864 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.864 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.865 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.865 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:42.865 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.028985+0000", 2024-09-14T04:42:42.866 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.09e-07 2024-09-14T04:42:42.866 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.866 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.866 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:42.866 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.029059+0000", 2024-09-14T04:42:42.866 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.3288999999999996e-05 2024-09-14T04:42:42.866 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.866 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.866 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:42.866 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.029072+0000", 2024-09-14T04:42:42.866 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2809999999999999e-05 2024-09-14T04:42:42.866 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.866 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.867 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:42.867 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.029078+0000", 2024-09-14T04:42:42.867 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.657e-06 2024-09-14T04:42:42.867 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.867 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.867 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:42.867 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.029229+0000", 2024-09-14T04:42:42.867 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015115899999999999 2024-09-14T04:42:42.867 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.867 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.867 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:42.867 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.029249+0000", 2024-09-14T04:42:42.867 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9919e-05 2024-09-14T04:42:42.867 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.867 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.868 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:42.868 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.081212+0000", 2024-09-14T04:42:42.868 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.051962770999999998 2024-09-14T04:42:42.868 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.868 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.868 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:42.868 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.081246+0000", 2024-09-14T04:42:42.868 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.4162000000000003e-05 2024-09-14T04:42:42.868 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.868 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:42.868 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.868 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.868 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.868 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:24702 53.0 e1319/1315 53:0ad7507c:::benchmark_data_smithi039_224949_object24701:head v 1319'1509, mlcod=1319'1506)", 2024-09-14T04:42:42.869 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:38:44.030119+0000", 2024-09-14T04:42:42.869 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 238.78721847899999, 2024-09-14T04:42:42.869 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.051852284999999998, 2024-09-14T04:42:42.869 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:42.869 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:42.869 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:42.869 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:42.869 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.869 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:42.869 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.030119+0000", 2024-09-14T04:42:42.869 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.869 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.869 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.870 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:42.870 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.030119+0000", 2024-09-14T04:42:42.870 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.870 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.870 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.870 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:42.870 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.030120+0000", 2024-09-14T04:42:42.870 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.1399999999999996e-07 2024-09-14T04:42:42.870 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.870 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.870 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:42.870 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.030159+0000", 2024-09-14T04:42:42.870 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.8470000000000003e-05 2024-09-14T04:42:42.870 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.870 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.870 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:42.871 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.030170+0000", 2024-09-14T04:42:42.871 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1114999999999999e-05 2024-09-14T04:42:42.871 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.871 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.871 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:42.871 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.030177+0000", 2024-09-14T04:42:42.871 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8040000000000004e-06 2024-09-14T04:42:42.871 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.871 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.871 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:42.871 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.080322+0000", 2024-09-14T04:42:42.871 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.050145035999999997 2024-09-14T04:42:42.871 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.871 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.871 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:42.872 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.080345+0000", 2024-09-14T04:42:42.872 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3068999999999999e-05 2024-09-14T04:42:42.872 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.872 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.872 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:42.872 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.081935+0000", 2024-09-14T04:42:42.872 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0015904199999999999 2024-09-14T04:42:42.872 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.872 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.872 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:42.872 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.081972+0000", 2024-09-14T04:42:42.872 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.6557000000000003e-05 2024-09-14T04:42:42.872 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.872 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:42.873 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.873 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.873 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.873 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:24696 53.4 e1319/1315 53:21e22dfb:::benchmark_data_smithi039_224949_object24695:head v 1319'1530, mlcod=1319'1526)", 2024-09-14T04:42:42.873 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:38:44.030684+0000", 2024-09-14T04:42:42.873 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 238.78665389, 2024-09-14T04:42:42.873 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.052337142000000003, 2024-09-14T04:42:42.873 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:42.873 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:42.873 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:42.873 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:42.873 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.873 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:42.873 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.030684+0000", 2024-09-14T04:42:42.874 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.874 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.874 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.874 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:42.874 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.030684+0000", 2024-09-14T04:42:42.874 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.874 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.874 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.874 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:42.874 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.030685+0000", 2024-09-14T04:42:42.874 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.8299999999999996e-07 2024-09-14T04:42:42.874 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.874 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.874 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:42.874 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.030839+0000", 2024-09-14T04:42:42.875 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000154566 2024-09-14T04:42:42.875 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.875 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.875 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:42.875 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.080294+0000", 2024-09-14T04:42:42.875 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.049454867 2024-09-14T04:42:42.875 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.875 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.875 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:42.875 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.080303+0000", 2024-09-14T04:42:42.875 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.5920000000000001e-06 2024-09-14T04:42:42.875 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.875 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.875 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:42.875 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.080355+0000", 2024-09-14T04:42:42.876 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1790999999999999e-05 2024-09-14T04:42:42.876 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.876 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.876 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:42.876 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.080369+0000", 2024-09-14T04:42:42.876 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3964999999999999e-05 2024-09-14T04:42:42.876 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.876 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.876 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:42.876 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.082976+0000", 2024-09-14T04:42:42.876 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0026071620000000001 2024-09-14T04:42:42.876 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.876 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.876 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:42.876 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.083021+0000", 2024-09-14T04:42:42.877 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5416e-05 2024-09-14T04:42:42.877 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.877 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:42.877 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.877 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.877 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.877 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:24703 53.0 e1319/1315 53:08f310e0:::benchmark_data_smithi039_224949_object24702:head v 1319'1510, mlcod=1319'1506)", 2024-09-14T04:42:42.877 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:38:44.031148+0000", 2024-09-14T04:42:42.877 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 238.78618954800001, 2024-09-14T04:42:42.877 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.051003469000000003, 2024-09-14T04:42:42.877 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:42.877 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:42.877 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:42.877 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:42.878 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.878 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:42.878 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.031148+0000", 2024-09-14T04:42:42.878 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.878 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.878 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.878 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:42.878 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.031148+0000", 2024-09-14T04:42:42.878 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.878 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.879 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.879 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:42.879 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.031149+0000", 2024-09-14T04:42:42.879 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.6400000000000002e-07 2024-09-14T04:42:42.879 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.879 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.879 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:42.879 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.031211+0000", 2024-09-14T04:42:42.879 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1780999999999997e-05 2024-09-14T04:42:42.879 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.879 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.879 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:42.879 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.080269+0000", 2024-09-14T04:42:42.880 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.049058483 2024-09-14T04:42:42.880 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.880 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.880 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:42.880 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.080284+0000", 2024-09-14T04:42:42.880 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4654999999999999e-05 2024-09-14T04:42:42.880 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.880 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.880 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:42.880 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.081276+0000", 2024-09-14T04:42:42.880 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00099212600000000008 2024-09-14T04:42:42.880 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.880 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.881 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:42.881 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.081302+0000", 2024-09-14T04:42:42.881 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.6310000000000001e-05 2024-09-14T04:42:42.881 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.881 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.881 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:42.881 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.082123+0000", 2024-09-14T04:42:42.881 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00082041700000000004 2024-09-14T04:42:42.881 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.881 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.881 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:42.881 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.082152+0000", 2024-09-14T04:42:42.882 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.9033000000000001e-05 2024-09-14T04:42:42.882 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.882 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:42.882 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.882 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.882 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.882 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:116785 53.6 e1327/1315 53:60b3bb6f:::benchmark_data_smithi039_224949_object116784:head v 1327'7390, mlcod=1327'7388)", 2024-09-14T04:42:42.882 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:39:51.416057+0000", 2024-09-14T04:42:42.882 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 171.40128125699999, 2024-09-14T04:42:42.882 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.032313394000000002, 2024-09-14T04:42:42.882 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:42.882 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:42.882 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:42.882 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:42.883 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.883 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:42.883 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:51.416057+0000", 2024-09-14T04:42:42.883 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.883 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.883 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.883 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:42.883 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:51.416057+0000", 2024-09-14T04:42:42.883 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.883 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.883 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.883 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:42.883 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:51.416058+0000", 2024-09-14T04:42:42.883 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4419999999999999e-06 2024-09-14T04:42:42.884 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.884 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.884 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:42.884 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:51.416085+0000", 2024-09-14T04:42:42.884 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.7271e-05 2024-09-14T04:42:42.884 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.884 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.884 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:42.884 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:51.416097+0000", 2024-09-14T04:42:42.884 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1378e-05 2024-09-14T04:42:42.884 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.884 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.884 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:42.884 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:51.416104+0000", 2024-09-14T04:42:42.885 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9530000000000002e-06 2024-09-14T04:42:42.885 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.885 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.885 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:42.885 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:51.416547+0000", 2024-09-14T04:42:42.885 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00044382500000000001 2024-09-14T04:42:42.885 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.885 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.885 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:42.885 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:51.416615+0000", 2024-09-14T04:42:42.885 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7883e-05 2024-09-14T04:42:42.885 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.885 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.885 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:42.886 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:51.448230+0000", 2024-09-14T04:42:42.886 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.031614661000000002 2024-09-14T04:42:42.886 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.886 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.886 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:42.886 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:51.448370+0000", 2024-09-14T04:42:42.886 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013998099999999999 2024-09-14T04:42:42.886 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.886 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:42.886 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.886 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.886 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.886 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:116788 53.6 e1327/1315 53:6f7aee81:::benchmark_data_smithi039_224949_object116787:head v 1327'7391, mlcod=1327'7388)", 2024-09-14T04:42:42.886 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:39:51.416126+0000", 2024-09-14T04:42:42.887 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 171.40121160300001, 2024-09-14T04:42:42.887 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.032271978, 2024-09-14T04:42:42.887 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:42.887 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:42.887 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:42.887 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:42.887 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.887 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:42.887 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:51.416126+0000", 2024-09-14T04:42:42.887 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.887 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.887 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.887 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:42.887 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:51.416126+0000", 2024-09-14T04:42:42.888 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.888 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.888 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.888 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:42.888 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:51.416127+0000", 2024-09-14T04:42:42.888 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.3700000000000001e-07 2024-09-14T04:42:42.888 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.888 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.888 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:42.888 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:51.416152+0000", 2024-09-14T04:42:42.888 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4978999999999999e-05 2024-09-14T04:42:42.888 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.888 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.888 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:42.888 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:51.416159+0000", 2024-09-14T04:42:42.889 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.5569999999999998e-06 2024-09-14T04:42:42.889 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.889 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.889 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:42.889 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:51.416164+0000", 2024-09-14T04:42:42.889 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.2159999999999997e-06 2024-09-14T04:42:42.889 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.889 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.889 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:42.889 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:51.426117+0000", 2024-09-14T04:42:42.889 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0099531480000000002 2024-09-14T04:42:42.889 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.889 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.889 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:42.890 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:51.426172+0000", 2024-09-14T04:42:42.890 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4814000000000001e-05 2024-09-14T04:42:42.890 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.890 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.890 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:42.890 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:51.448374+0000", 2024-09-14T04:42:42.890 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.022201990000000001 2024-09-14T04:42:42.890 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.890 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.890 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:42.890 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:39:51.448398+0000", 2024-09-14T04:42:42.890 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3937000000000001e-05 2024-09-14T04:42:42.890 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.890 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:42.890 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.891 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.891 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.891 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:141044 53.4 e1330/1315 53:2ef1f326:::benchmark_data_smithi039_224949_object10951:head v 1330'8684, mlcod=1330'8680)", 2024-09-14T04:42:42.891 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:40:17.444663+0000", 2024-09-14T04:42:42.891 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 145.37267470800001, 2024-09-14T04:42:42.891 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.044830643000000003, 2024-09-14T04:42:42.891 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:42.891 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:42.891 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:42.891 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:42.891 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.891 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:42.891 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:17.444663+0000", 2024-09-14T04:42:42.891 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.891 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.892 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.892 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:42.892 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:17.444663+0000", 2024-09-14T04:42:42.892 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.892 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.892 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.892 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:42.892 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:17.444664+0000", 2024-09-14T04:42:42.892 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0729999999999999e-06 2024-09-14T04:42:42.892 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.892 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.892 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:42.892 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:17.444671+0000", 2024-09-14T04:42:42.892 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.4010000000000002e-06 2024-09-14T04:42:42.893 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.893 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.893 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:42.893 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:17.444689+0000", 2024-09-14T04:42:42.893 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8131999999999999e-05 2024-09-14T04:42:42.893 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.893 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.893 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:42.893 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:17.444697+0000", 2024-09-14T04:42:42.893 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.6659999999999997e-06 2024-09-14T04:42:42.893 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.893 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.893 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:42.894 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:17.455365+0000", 2024-09-14T04:42:42.894 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.010667944 2024-09-14T04:42:42.894 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.894 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.894 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:42.894 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:17.455415+0000", 2024-09-14T04:42:42.894 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.0043999999999997e-05 2024-09-14T04:42:42.894 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.894 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.894 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:42.894 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:17.489311+0000", 2024-09-14T04:42:42.894 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.033895542000000001 2024-09-14T04:42:42.894 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.894 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.895 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:42.895 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:40:17.489494+0000", 2024-09-14T04:42:42.895 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00018284099999999999 2024-09-14T04:42:42.895 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.895 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:42.895 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.895 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.895 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.895 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:223129 53.4 e1348/1344 53:271a56d4:::benchmark_data_smithi039_224949_object93036:head v 1348'13736, mlcod=1348'13733)", 2024-09-14T04:42:42.895 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:41:50.106943+0000", 2024-09-14T04:42:42.895 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 52.710394534999999, 2024-09-14T04:42:42.895 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.034886499000000001, 2024-09-14T04:42:42.895 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:42.895 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:42.895 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:42.896 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:42.896 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.896 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:42.896 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:50.106943+0000", 2024-09-14T04:42:42.896 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.896 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.896 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.896 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:42.896 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:50.106943+0000", 2024-09-14T04:42:42.896 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.896 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.896 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.896 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:42.896 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:50.106945+0000", 2024-09-14T04:42:42.897 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.455e-06 2024-09-14T04:42:42.897 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.897 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.897 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:42.897 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:50.106955+0000", 2024-09-14T04:42:42.897 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0599e-05 2024-09-14T04:42:42.897 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.897 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.897 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:42.897 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:50.106982+0000", 2024-09-14T04:42:42.897 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.6352000000000001e-05 2024-09-14T04:42:42.897 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.897 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.897 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:42.898 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:50.106993+0000", 2024-09-14T04:42:42.898 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0808000000000001e-05 2024-09-14T04:42:42.898 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.898 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.898 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:42.898 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:50.107244+0000", 2024-09-14T04:42:42.898 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00025107900000000003 2024-09-14T04:42:42.898 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.898 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.898 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:42.898 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:50.107306+0000", 2024-09-14T04:42:42.898 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2768000000000006e-05 2024-09-14T04:42:42.898 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.898 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.899 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:42.899 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:50.141786+0000", 2024-09-14T04:42:42.899 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.034479639999999999 2024-09-14T04:42:42.899 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.899 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.899 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:42.899 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:41:50.141830+0000", 2024-09-14T04:42:42.899 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3798000000000003e-05 2024-09-14T04:42:42.899 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.899 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:42.899 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.899 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.899 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.899 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:233349 53.0 e1355/1344 53:03dab870:::benchmark_data_smithi039_224949_object103256:head v 1355'14513, mlcod=1354'14510)", 2024-09-14T04:42:42.900 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.820943+0000", 2024-09-14T04:42:42.900 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 37.996394854999998, 2024-09-14T04:42:42.900 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.040432133000000002, 2024-09-14T04:42:42.900 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:42.900 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:42.900 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:42.900 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:42.900 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.900 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:42.900 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.820943+0000", 2024-09-14T04:42:42.900 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.900 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.900 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.901 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:42.901 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.820943+0000", 2024-09-14T04:42:42.901 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.901 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.901 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.901 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:42.901 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.820943+0000", 2024-09-14T04:42:42.901 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.6100000000000002e-07 2024-09-14T04:42:42.901 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.901 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.901 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:42.901 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.820948+0000", 2024-09-14T04:42:42.901 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3710000000000002e-06 2024-09-14T04:42:42.901 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.902 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.902 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:42.902 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.820956+0000", 2024-09-14T04:42:42.902 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.1880000000000002e-06 2024-09-14T04:42:42.902 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.902 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.902 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:42.902 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.820963+0000", 2024-09-14T04:42:42.902 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.0960000000000003e-06 2024-09-14T04:42:42.902 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.902 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.902 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:42.902 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.824934+0000", 2024-09-14T04:42:42.902 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0039712799999999998 2024-09-14T04:42:42.903 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.903 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.903 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:42.903 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.825231+0000", 2024-09-14T04:42:42.903 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00029706399999999999 2024-09-14T04:42:42.903 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.903 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.903 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:42.903 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.861268+0000", 2024-09-14T04:42:42.903 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.036037048000000002 2024-09-14T04:42:42.903 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.903 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.903 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:42.903 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.861375+0000", 2024-09-14T04:42:42.903 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00010682499999999999 2024-09-14T04:42:42.904 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.904 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:42.904 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.904 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.904 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.904 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:233361 53.0 e1355/1344 53:024c67d1:::benchmark_data_smithi039_224949_object103268:head v 1355'14514, mlcod=1354'14511)", 2024-09-14T04:42:42.904 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.829297+0000", 2024-09-14T04:42:42.904 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 37.988040984999998, 2024-09-14T04:42:42.904 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.032211401000000001, 2024-09-14T04:42:42.904 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:42.904 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:42.904 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:42.904 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:42.904 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.904 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:42.904 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.829297+0000", 2024-09-14T04:42:42.905 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.905 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.905 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.905 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:42.905 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.829297+0000", 2024-09-14T04:42:42.905 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.905 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.905 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.905 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:42.905 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.829298+0000", 2024-09-14T04:42:42.905 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.24e-06 2024-09-14T04:42:42.905 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.905 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.905 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:42.905 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.829307+0000", 2024-09-14T04:42:42.906 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.7309999999999999e-06 2024-09-14T04:42:42.906 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.906 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.906 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:42.906 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.829320+0000", 2024-09-14T04:42:42.906 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2892e-05 2024-09-14T04:42:42.906 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.906 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.906 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:42.906 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.829330+0000", 2024-09-14T04:42:42.906 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0326000000000001e-05 2024-09-14T04:42:42.906 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.906 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.907 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:42.908 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.846710+0000", 2024-09-14T04:42:42.908 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.017379555000000001 2024-09-14T04:42:42.908 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.908 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.908 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:42.908 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.846733+0000", 2024-09-14T04:42:42.908 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3504999999999999e-05 2024-09-14T04:42:42.908 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.908 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.908 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:42.908 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.861380+0000", 2024-09-14T04:42:42.909 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.014647214 2024-09-14T04:42:42.909 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.909 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.909 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:42.909 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.861508+0000", 2024-09-14T04:42:42.909 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012793799999999999 2024-09-14T04:42:42.909 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.909 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:42.909 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.909 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.909 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.909 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:236922 53.4 e1359/1356 53:275b4c2b:::benchmark_data_smithi039_224949_object106829:head v 1359'14594, mlcod=1359'14592)", 2024-09-14T04:42:42.909 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:12.211584+0000", 2024-09-14T04:42:42.910 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 30.605754301000001, 2024-09-14T04:42:42.910 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.039193196, 2024-09-14T04:42:42.910 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:42.910 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:42.910 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:42.910 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:42.910 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.910 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:42.910 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:12.211584+0000", 2024-09-14T04:42:42.910 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.910 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.910 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.910 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:42.910 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:12.211584+0000", 2024-09-14T04:42:42.911 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.911 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.911 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.911 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:42.911 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:12.211585+0000", 2024-09-14T04:42:42.911 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0049999999999999e-06 2024-09-14T04:42:42.911 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.911 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.911 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:42.911 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:12.211591+0000", 2024-09-14T04:42:42.911 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.5440000000000004e-06 2024-09-14T04:42:42.911 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.911 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.911 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:42.911 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:12.211609+0000", 2024-09-14T04:42:42.912 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7649e-05 2024-09-14T04:42:42.912 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.912 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.912 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:42.912 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:12.211618+0000", 2024-09-14T04:42:42.912 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.8659999999999995e-06 2024-09-14T04:42:42.912 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.912 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.912 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:42.912 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:12.211814+0000", 2024-09-14T04:42:42.912 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000196722 2024-09-14T04:42:42.912 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.912 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.912 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:42.913 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:12.211866+0000", 2024-09-14T04:42:42.913 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1511999999999997e-05 2024-09-14T04:42:42.913 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.913 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.913 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:42.913 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:12.250742+0000", 2024-09-14T04:42:42.913 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.038875674999999998 2024-09-14T04:42:42.913 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.913 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.913 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:42.913 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:12.250777+0000", 2024-09-14T04:42:42.913 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.5222999999999998e-05 2024-09-14T04:42:42.913 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.914 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:42.914 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.914 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.914 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.914 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:236924 53.4 e1359/1356 53:2cdffdfc:::benchmark_data_smithi039_224949_object106831:head v 1359'14595, mlcod=1359'14592)", 2024-09-14T04:42:42.914 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:12.213567+0000", 2024-09-14T04:42:42.914 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 30.603770514000001, 2024-09-14T04:42:42.914 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.037237352000000001, 2024-09-14T04:42:42.914 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:42.914 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:42.914 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:42.914 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:42.914 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.915 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:42.915 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:12.213567+0000", 2024-09-14T04:42:42.915 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.915 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.915 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.915 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:42.915 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:12.213567+0000", 2024-09-14T04:42:42.915 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.915 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.915 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.915 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:42.915 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:12.213568+0000", 2024-09-14T04:42:42.915 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.7199999999999997e-07 2024-09-14T04:42:42.915 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.916 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.916 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:42.916 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:12.213575+0000", 2024-09-14T04:42:42.916 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9890000000000003e-06 2024-09-14T04:42:42.916 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.916 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.916 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:42.916 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:12.213589+0000", 2024-09-14T04:42:42.916 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4245e-05 2024-09-14T04:42:42.916 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.916 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.916 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:42.916 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:12.213598+0000", 2024-09-14T04:42:42.917 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0819999999999999e-06 2024-09-14T04:42:42.917 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.917 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.917 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:42.917 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:12.218867+0000", 2024-09-14T04:42:42.917 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0052691129999999997 2024-09-14T04:42:42.917 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.917 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.917 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:42.917 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:12.218937+0000", 2024-09-14T04:42:42.917 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9958999999999999e-05 2024-09-14T04:42:42.917 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.917 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.917 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:42.918 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:12.250782+0000", 2024-09-14T04:42:42.918 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.031845409999999998 2024-09-14T04:42:42.918 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.918 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.918 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:42.918 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:12.250805+0000", 2024-09-14T04:42:42.918 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2682000000000001e-05 2024-09-14T04:42:42.918 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.918 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:42.918 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.918 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.918 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.918 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:238929 53.0 e1359/1344 53:098b9bf5:::benchmark_data_smithi039_224949_object108836:head v 1359'14886, mlcod=1359'14882)", 2024-09-14T04:42:42.918 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:14.460371+0000", 2024-09-14T04:42:42.919 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 28.356966944, 2024-09-14T04:42:42.919 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.030586780000000001, 2024-09-14T04:42:42.919 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:42.919 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:42.919 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:42.919 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:42.919 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.919 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:42.919 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:14.460371+0000", 2024-09-14T04:42:42.919 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.919 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.919 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.919 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:42.920 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:14.460371+0000", 2024-09-14T04:42:42.920 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.920 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.920 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.920 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:42.920 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:14.460372+0000", 2024-09-14T04:42:42.920 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2440000000000001e-06 2024-09-14T04:42:42.920 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.920 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.920 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:42.920 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:14.460379+0000", 2024-09-14T04:42:42.920 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.6039999999999998e-06 2024-09-14T04:42:42.920 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.920 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.921 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:42.921 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:14.460391+0000", 2024-09-14T04:42:42.921 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2232000000000001e-05 2024-09-14T04:42:42.921 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.921 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.921 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:42.921 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:14.460399+0000", 2024-09-14T04:42:42.921 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.678e-06 2024-09-14T04:42:42.921 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.921 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.921 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:42.921 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:14.465124+0000", 2024-09-14T04:42:42.921 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0047255470000000001 2024-09-14T04:42:42.921 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.922 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.922 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:42.922 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:14.465217+0000", 2024-09-14T04:42:42.922 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.2632000000000004e-05 2024-09-14T04:42:42.922 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.922 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.922 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:42.922 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:14.490714+0000", 2024-09-14T04:42:42.922 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.025497586999999999 2024-09-14T04:42:42.922 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.922 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.922 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:42.922 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:14.490958+0000", 2024-09-14T04:42:42.922 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00024325599999999999 2024-09-14T04:42:42.922 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.923 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:42.923 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.923 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.923 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.923 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:243922 53.4 e1364/1356 53:2b5ff389:::benchmark_data_smithi039_224949_object113829:head v 1364'15023, mlcod=1364'15019)", 2024-09-14T04:42:42.923 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:22.626693+0000", 2024-09-14T04:42:42.923 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 20.190644906999999, 2024-09-14T04:42:42.923 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.032361909000000001, 2024-09-14T04:42:42.923 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:42:42.923 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:42:42.923 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:42:42.923 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:42:42.923 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.923 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:42:42.924 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:22.626693+0000", 2024-09-14T04:42:42.924 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.924 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.924 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.924 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:42:42.924 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:22.626693+0000", 2024-09-14T04:42:42.924 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:42:42.924 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.924 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.924 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:42:42.924 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:22.626694+0000", 2024-09-14T04:42:42.924 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.5000000000000002e-07 2024-09-14T04:42:42.924 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.924 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.925 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:42:42.925 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:22.626699+0000", 2024-09-14T04:42:42.925 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4609999999999999e-06 2024-09-14T04:42:42.925 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.925 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.925 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:42:42.925 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:22.626715+0000", 2024-09-14T04:42:42.925 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5489e-05 2024-09-14T04:42:42.925 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.925 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.925 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:42:42.925 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:22.626721+0000", 2024-09-14T04:42:42.925 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.5220000000000004e-06 2024-09-14T04:42:42.925 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.926 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.926 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:42:42.926 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:22.630675+0000", 2024-09-14T04:42:42.926 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0039541070000000001 2024-09-14T04:42:42.926 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.926 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.926 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:42:42.926 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:22.630698+0000", 2024-09-14T04:42:42.926 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2609999999999999e-05 2024-09-14T04:42:42.926 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.926 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.926 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:42:42.926 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:22.659009+0000", 2024-09-14T04:42:42.927 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.028311218999999999 2024-09-14T04:42:42.927 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:42:42.927 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:42:42.927 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:42:42.927 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:22.659055+0000", 2024-09-14T04:42:42.927 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5751000000000003e-05 2024-09-14T04:42:42.927 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.927 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:42.927 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.927 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:42:42.927 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:42:42.927 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:42:42.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:42.928 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:42.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:43.029 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:43.050 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:43.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:43.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:43.150 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:42 smithi039 ceph-mon[106470]: pgmap v2335: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 2 active+undersized, 4 active+undersized+degraded, 90 active+clean; 290 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 781/13992 objects degraded (5.582%); 125/13992 objects misplaced (0.893%); 1.6 MiB/s, 24 objects/s recovering 2024-09-14T04:42:43.150 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:42 smithi039 ceph-mon[111235]: pgmap v2335: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 2 active+undersized, 4 active+undersized+degraded, 90 active+clean; 290 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 781/13992 objects degraded (5.582%); 125/13992 objects misplaced (0.893%); 1.6 MiB/s, 24 objects/s recovering 2024-09-14T04:42:43.152 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:43.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:43.194 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:43.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:43.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:42 smithi129 ceph-mon[101316]: pgmap v2335: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 active+clean+remapped, 2 active+undersized, 4 active+undersized+degraded, 90 active+clean; 290 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 781/13992 objects degraded (5.582%); 125/13992 objects misplaced (0.893%); 1.6 MiB/s, 24 objects/s recovering 2024-09-14T04:42:43.295 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:43.332 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:43.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:43.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:43.420 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:42:43.434 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:43.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:43.462 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:43.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:43.562 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:43.605 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:43.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:43.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:43.683 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Removed 130090 objects 2024-09-14T04:42:43.683 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Clean up completed and total clean up time :157.964 2024-09-14T04:42:43.707 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:43.728 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:43.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:43.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:43.743 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:42:43.830 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:43.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:43.853 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:43.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:43.954 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:43.976 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:43.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:43.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:44.078 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:44.104 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:44.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:44.105 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:44.206 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:44.232 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:44.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:44.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:44.334 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:44.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:44.361 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:44.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:44.439 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-14T04:42:44.462 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:44.488 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:44.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:44.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:44.589 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:44.611 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:44.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:44.612 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:44.713 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:44.738 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:44.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:44.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:44.840 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:44.863 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:44.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:44.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:44.965 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:44.987 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:44.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:44.988 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:45.088 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:45.110 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:45.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:45.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:45.133 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:44 smithi039 ceph-mon[106470]: pgmap v2336: 99 pgs: 2 active+clean+remapped, 2 active+undersized, 4 active+undersized+degraded, 91 active+clean; 76 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 164/3702 objects degraded (4.430%); 1/3702 objects misplaced (0.027%); 2.4 MiB/s, 37 objects/s recovering 2024-09-14T04:42:45.134 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:44 smithi039 ceph-mon[111235]: pgmap v2336: 99 pgs: 2 active+clean+remapped, 2 active+undersized, 4 active+undersized+degraded, 91 active+clean; 76 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 164/3702 objects degraded (4.430%); 1/3702 objects misplaced (0.027%); 2.4 MiB/s, 37 objects/s recovering 2024-09-14T04:42:45.212 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:45.238 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:45.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:45.239 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:45.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:44 smithi129 ceph-mon[101316]: pgmap v2336: 99 pgs: 2 active+clean+remapped, 2 active+undersized, 4 active+undersized+degraded, 91 active+clean; 76 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s; 164/3702 objects degraded (4.430%); 1/3702 objects misplaced (0.027%); 2.4 MiB/s, 37 objects/s recovering 2024-09-14T04:42:45.340 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:45.363 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:45.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:45.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:45.465 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:45.487 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:45.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:45.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:45.589 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:45.604 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:42:45.611 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:45.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:45.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:45.713 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:45.738 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:45.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:45.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:45.840 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:45.862 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:45.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:45.864 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:45.941 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2024-09-14T04:42:45.941 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2024-09-14T04:42:45.964 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:46.006 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:46.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:46.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:46.108 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:46.131 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:46.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:46.132 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:46.232 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:46.255 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:46.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:46.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:46.357 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:46.379 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:46.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:46.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:46.481 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:46.504 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:46.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:46.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:46.606 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:46.629 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:46.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:46.630 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:46.731 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:46.753 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:46.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:46.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:46.855 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:46.879 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:46.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:46.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:46.980 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:47.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:47.004 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:47.004 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:47.105 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:47.128 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:47.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:47.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:47.131 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:46 smithi129 ceph-mon[101316]: pgmap v2337: 99 pgs: 2 active+clean+remapped, 2 active+undersized, 4 active+undersized+degraded, 91 active+clean; 50 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 987 op/s; 164/2445 objects degraded (6.708%); 1/2445 objects misplaced (0.041%); 2.4 MiB/s, 37 objects/s recovering 2024-09-14T04:42:47.132 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:46 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 164/2445 objects degraded (6.708%), 4 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-14T04:42:47.132 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:46 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:42:47.142 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:46 smithi039 ceph-mon[106470]: pgmap v2337: 99 pgs: 2 active+clean+remapped, 2 active+undersized, 4 active+undersized+degraded, 91 active+clean; 50 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 987 op/s; 164/2445 objects degraded (6.708%); 1/2445 objects misplaced (0.041%); 2.4 MiB/s, 37 objects/s recovering 2024-09-14T04:42:47.143 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:46 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 164/2445 objects degraded (6.708%), 4 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-14T04:42:47.143 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:46 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:42:47.143 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:46 smithi039 ceph-mon[111235]: pgmap v2337: 99 pgs: 2 active+clean+remapped, 2 active+undersized, 4 active+undersized+degraded, 91 active+clean; 50 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 987 op/s; 164/2445 objects degraded (6.708%); 1/2445 objects misplaced (0.041%); 2.4 MiB/s, 37 objects/s recovering 2024-09-14T04:42:47.143 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:46 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 164/2445 objects degraded (6.708%), 4 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-14T04:42:47.143 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:46 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:42:47.231 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:47.252 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:47.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:47.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:47.354 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:47.378 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:47.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:47.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:47.480 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:47.502 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:47.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:47.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:47.604 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:47.625 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:47.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:47.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:47.727 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:47.750 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:47.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:47.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:47.854 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:47.877 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:47.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:47.878 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:47.979 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:48.000 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:48.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:48.001 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:48.101 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:48.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:48.124 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:48.124 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:48.225 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:48.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:48.247 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:48.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:48.282 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:42:48.283 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:42:48.348 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:48.371 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:48.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:48.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:48.473 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:48.494 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:48.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:48.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:48.596 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:48.618 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:48.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:48.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:48.720 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:48.740 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:48.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:48.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:48.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:42:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:42:48] "GET /metrics HTTP/1.1" 200 36791 "" "Prometheus/2.51.0" 2024-09-14T04:42:48.843 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:48.865 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:48.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:48.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:48.894 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is unset 2024-09-14T04:42:48.904 INFO:teuthology.orchestra.run.smithi039.stdout:moved 24 / 297 (8.08081%) 2024-09-14T04:42:48.904 INFO:teuthology.orchestra.run.smithi039.stdout:avg 42.4286 2024-09-14T04:42:48.904 INFO:teuthology.orchestra.run.smithi039.stdout:stddev 9.0283 -> 4.74664 (expected baseline 6.03053) 2024-09-14T04:42:48.904 INFO:teuthology.orchestra.run.smithi039.stdout:min osd.1 with 23 -> 47 pgs (0.542088 -> 1.10774 * mean) 2024-09-14T04:42:48.904 INFO:teuthology.orchestra.run.smithi039.stdout:max osd.3 with 50 -> 48 pgs (1.17845 -> 1.13131 * mean) 2024-09-14T04:42:48.904 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:42:48.904 INFO:teuthology.orchestra.run.smithi039.stdout:oload 110 2024-09-14T04:42:48.904 INFO:teuthology.orchestra.run.smithi039.stdout:max_change 1 2024-09-14T04:42:48.905 INFO:teuthology.orchestra.run.smithi039.stdout:max_change_osds 4 2024-09-14T04:42:48.905 INFO:teuthology.orchestra.run.smithi039.stdout:average_utilization 0.0059 2024-09-14T04:42:48.905 INFO:teuthology.orchestra.run.smithi039.stdout:overload_utilization 0.0064 2024-09-14T04:42:48.905 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4 weight 0.1967 -> 0.0411 2024-09-14T04:42:48.905 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1 weight 0.4612 -> 0.9036 2024-09-14T04:42:48.905 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5 weight 0.9052 -> 1.0000 2024-09-14T04:42:48.968 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:48.990 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:48.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:48.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:49.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:48 smithi129 ceph-mon[101316]: pgmap v2338: 99 pgs: 2 active+clean+remapped, 3 active+undersized, 3 active+undersized+degraded, 91 active+clean; 28 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 44/1392 objects degraded (3.161%); 1/1392 objects misplaced (0.072%); 832 KiB/s, 12 objects/s recovering 2024-09-14T04:42:49.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:48 smithi129 ceph-mon[101316]: from='client.45296 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:42:49.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:48 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'59216','4':'2691','5':'65536'}"} : dispatch 2024-09-14T04:42:49.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:48 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'59216','4':'2691','5':'65536'}"} : dispatch 2024-09-14T04:42:49.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:48 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2383796968' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:42:49.093 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:49.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:49.115 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:49.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:49.212 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:48 smithi039 ceph-mon[106470]: pgmap v2338: 99 pgs: 2 active+clean+remapped, 3 active+undersized, 3 active+undersized+degraded, 91 active+clean; 28 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 44/1392 objects degraded (3.161%); 1/1392 objects misplaced (0.072%); 832 KiB/s, 12 objects/s recovering 2024-09-14T04:42:49.212 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:48 smithi039 ceph-mon[106470]: from='client.45296 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:42:49.212 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:48 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'59216','4':'2691','5':'65536'}"} : dispatch 2024-09-14T04:42:49.212 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:48 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'59216','4':'2691','5':'65536'}"} : dispatch 2024-09-14T04:42:49.213 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:48 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2383796968' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:42:49.213 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:48 smithi039 ceph-mon[111235]: pgmap v2338: 99 pgs: 2 active+clean+remapped, 3 active+undersized, 3 active+undersized+degraded, 91 active+clean; 28 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 599 op/s; 44/1392 objects degraded (3.161%); 1/1392 objects misplaced (0.072%); 832 KiB/s, 12 objects/s recovering 2024-09-14T04:42:49.213 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:48 smithi039 ceph-mon[111235]: from='client.45296 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:42:49.213 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:48 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'59216','4':'2691','5':'65536'}"} : dispatch 2024-09-14T04:42:49.213 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:48 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'1':'59216','4':'2691','5':'65536'}"} : dispatch 2024-09-14T04:42:49.213 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:48 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2383796968' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:42:49.217 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:49.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:49.239 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:49.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:49.340 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:49.363 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:49.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:49.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:49.465 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:49.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:49.488 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:49.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:49.589 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:49.611 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:49.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:49.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:49.712 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:49.733 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:49.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:49.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:49.835 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:49.856 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:49.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:49.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:49.958 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:49.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:49.982 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:49.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:49 smithi129 ceph-mon[101316]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:42:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:49 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'59216','4':'2691','5':'65536'}"}]': finished 2024-09-14T04:42:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:49 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2383796968' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:42:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:49 smithi129 ceph-mon[101316]: osdmap e1374: 8 total, 7 up, 8 in 2024-09-14T04:42:50.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:49 smithi039 ceph-mon[106470]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:42:50.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:49 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'59216','4':'2691','5':'65536'}"}]': finished 2024-09-14T04:42:50.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:49 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2383796968' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:42:50.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:49 smithi039 ceph-mon[106470]: osdmap e1374: 8 total, 7 up, 8 in 2024-09-14T04:42:50.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:49 smithi039 ceph-mon[111235]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:42:50.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:49 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'1':'59216','4':'2691','5':'65536'}"}]': finished 2024-09-14T04:42:50.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:49 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2383796968' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:42:50.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:49 smithi039 ceph-mon[111235]: osdmap e1374: 8 total, 7 up, 8 in 2024-09-14T04:42:50.083 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:50.106 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:50.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:50.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:50.208 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:50.231 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:50.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:50.231 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:50.332 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:50.354 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:50.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:50.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:50.456 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:50.478 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:50.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:50.479 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:50.580 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:50.601 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:50.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:50.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:50.702 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:50.724 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:50.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:50.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:50.826 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:50.847 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:50.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:50.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:50.949 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:50.958 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:50 smithi039 ceph-mon[106470]: pgmap v2340: 99 pgs: 2 active+clean+remapped, 3 active+undersized, 3 active+undersized+degraded, 91 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 527 op/s; 2/78 objects degraded (2.564%); 1/78 objects misplaced (1.282%); 757 KiB/s, 11 objects/s recovering 2024-09-14T04:42:50.958 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:50 smithi039 ceph-mon[106470]: osdmap e1375: 8 total, 7 up, 8 in 2024-09-14T04:42:50.958 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:50 smithi039 ceph-mon[111235]: pgmap v2340: 99 pgs: 2 active+clean+remapped, 3 active+undersized, 3 active+undersized+degraded, 91 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 527 op/s; 2/78 objects degraded (2.564%); 1/78 objects misplaced (1.282%); 757 KiB/s, 11 objects/s recovering 2024-09-14T04:42:50.958 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:50 smithi039 ceph-mon[111235]: osdmap e1375: 8 total, 7 up, 8 in 2024-09-14T04:42:51.002 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:51.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:51.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:51.104 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:51.141 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:51.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:51.142 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:51.243 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:51.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:51.265 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:51.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:51.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:50 smithi129 ceph-mon[101316]: pgmap v2340: 99 pgs: 2 active+clean+remapped, 3 active+undersized, 3 active+undersized+degraded, 91 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 527 op/s; 2/78 objects degraded (2.564%); 1/78 objects misplaced (1.282%); 757 KiB/s, 11 objects/s recovering 2024-09-14T04:42:51.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:50 smithi129 ceph-mon[101316]: osdmap e1375: 8 total, 7 up, 8 in 2024-09-14T04:42:51.366 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:51.389 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:51.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:51.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:51.491 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:51.513 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:51.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:51.514 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:51.516 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:42:51.517 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:42:51.517 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:42:51.517 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:42:51.517 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:42:51.517 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:42:51.517 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:42:51.517 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:42:51.518 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:42:51.518 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:42:51.518 INFO:teuthology.orchestra.run.smithi039.stderr:Error ENXIO: problem getting command descriptions from osd.4 2024-09-14T04:42:51.518 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:42:51.518 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:42:51.518 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:42:51.519 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:42:51.615 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:51.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:51.638 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:51.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:51.739 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:51.762 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:51.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:51.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:51.864 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:51.886 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:51.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:51.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:51.988 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:52.010 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:52.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:52.011 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:52.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:51 smithi039 ceph-mon[106470]: 53.0 starting backfill to osd.0 from (0'0,0'0] MAX to 1373'16222 2024-09-14T04:42:52.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:51 smithi039 ceph-mon[106470]: 53.a starting backfill to osd.5 from (0'0,0'0] MAX to 1373'16468 2024-09-14T04:42:52.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:51 smithi039 ceph-mon[106470]: 53.6 starting backfill to osd.1 from (0'0,0'0] MAX to 1373'16494 2024-09-14T04:42:52.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:51 smithi039 ceph-mon[106470]: 53.3 starting backfill to osd.1 from (0'0,0'0] MAX to 1373'16248 2024-09-14T04:42:52.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:51 smithi039 ceph-mon[106470]: 53.1 starting backfill to osd.1 from (0'0,0'0] MAX to 1373'16432 2024-09-14T04:42:52.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:51 smithi039 ceph-mon[106470]: osdmap e1376: 8 total, 7 up, 8 in 2024-09-14T04:42:52.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:51 smithi039 ceph-mon[106470]: 53.e starting backfill to osd.1 from (0'0,0'0] MAX to 1373'16062 2024-09-14T04:42:52.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:51 smithi039 ceph-mon[106470]: 53.7 starting backfill to osd.1 from (0'0,0'0] MAX to 1373'16434 2024-09-14T04:42:52.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:51 smithi039 ceph-mon[106470]: 53.5 starting backfill to osd.1 from (0'0,0'0] MAX to 1373'16178 2024-09-14T04:42:52.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:51 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 2/78 objects degraded (2.564%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-14T04:42:52.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:51 smithi039 ceph-mon[106470]: 53.b starting backfill to osd.5 from (0'0,0'0] MAX to 1373'16070 2024-09-14T04:42:52.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:51 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:42:52.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:51 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:42:52.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:51 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3572057308' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:42:52.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:51 smithi039 ceph-mon[111235]: 53.0 starting backfill to osd.0 from (0'0,0'0] MAX to 1373'16222 2024-09-14T04:42:52.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:51 smithi039 ceph-mon[111235]: 53.a starting backfill to osd.5 from (0'0,0'0] MAX to 1373'16468 2024-09-14T04:42:52.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:51 smithi039 ceph-mon[111235]: 53.6 starting backfill to osd.1 from (0'0,0'0] MAX to 1373'16494 2024-09-14T04:42:52.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:51 smithi039 ceph-mon[111235]: 53.3 starting backfill to osd.1 from (0'0,0'0] MAX to 1373'16248 2024-09-14T04:42:52.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:51 smithi039 ceph-mon[111235]: 53.1 starting backfill to osd.1 from (0'0,0'0] MAX to 1373'16432 2024-09-14T04:42:52.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:51 smithi039 ceph-mon[111235]: osdmap e1376: 8 total, 7 up, 8 in 2024-09-14T04:42:52.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:51 smithi039 ceph-mon[111235]: 53.e starting backfill to osd.1 from (0'0,0'0] MAX to 1373'16062 2024-09-14T04:42:52.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:51 smithi039 ceph-mon[111235]: 53.7 starting backfill to osd.1 from (0'0,0'0] MAX to 1373'16434 2024-09-14T04:42:52.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:51 smithi039 ceph-mon[111235]: 53.5 starting backfill to osd.1 from (0'0,0'0] MAX to 1373'16178 2024-09-14T04:42:52.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:51 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 2/78 objects degraded (2.564%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-14T04:42:52.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:51 smithi039 ceph-mon[111235]: 53.b starting backfill to osd.5 from (0'0,0'0] MAX to 1373'16070 2024-09-14T04:42:52.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:51 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:42:52.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:51 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:42:52.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:51 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3572057308' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:42:52.112 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:52.134 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:52.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:52.135 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:52.236 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:52.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:52.260 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:52.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:51 smithi129 ceph-mon[101316]: 53.0 starting backfill to osd.0 from (0'0,0'0] MAX to 1373'16222 2024-09-14T04:42:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:51 smithi129 ceph-mon[101316]: 53.a starting backfill to osd.5 from (0'0,0'0] MAX to 1373'16468 2024-09-14T04:42:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:51 smithi129 ceph-mon[101316]: 53.6 starting backfill to osd.1 from (0'0,0'0] MAX to 1373'16494 2024-09-14T04:42:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:51 smithi129 ceph-mon[101316]: 53.3 starting backfill to osd.1 from (0'0,0'0] MAX to 1373'16248 2024-09-14T04:42:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:51 smithi129 ceph-mon[101316]: 53.1 starting backfill to osd.1 from (0'0,0'0] MAX to 1373'16432 2024-09-14T04:42:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:51 smithi129 ceph-mon[101316]: osdmap e1376: 8 total, 7 up, 8 in 2024-09-14T04:42:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:51 smithi129 ceph-mon[101316]: 53.e starting backfill to osd.1 from (0'0,0'0] MAX to 1373'16062 2024-09-14T04:42:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:51 smithi129 ceph-mon[101316]: 53.7 starting backfill to osd.1 from (0'0,0'0] MAX to 1373'16434 2024-09-14T04:42:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:51 smithi129 ceph-mon[101316]: 53.5 starting backfill to osd.1 from (0'0,0'0] MAX to 1373'16178 2024-09-14T04:42:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:51 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 2/78 objects degraded (2.564%), 3 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2024-09-14T04:42:52.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:51 smithi129 ceph-mon[101316]: 53.b starting backfill to osd.5 from (0'0,0'0] MAX to 1373'16070 2024-09-14T04:42:52.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:51 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:42:52.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:51 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:42:52.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:51 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3572057308' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:42:52.361 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:52.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:52.385 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:52.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:52.486 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:52.510 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:52.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:52.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:52.613 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:52.638 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:52.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:52.639 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:52.740 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:52.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:52.764 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:52.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:52.865 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:52.888 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:52.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:52.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:52.990 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:53.020 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:53.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:53.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:53.122 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:53.143 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:53.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:53.144 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:53.245 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:53.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:53.268 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:53.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:53.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:52 smithi129 ceph-mon[101316]: pgmap v2343: 99 pgs: 2 unknown, 4 peering, 2 active+clean+remapped, 3 active+undersized, 3 active+undersized+degraded, 85 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2/78 objects degraded (2.564%); 1/78 objects misplaced (1.282%) 2024-09-14T04:42:53.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:52 smithi129 ceph-mon[101316]: osdmap e1377: 8 total, 7 up, 8 in 2024-09-14T04:42:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:52 smithi129 ceph-mon[101316]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-14T04:42:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:52 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:42:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:42:53.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:52 smithi039 ceph-mon[111235]: pgmap v2343: 99 pgs: 2 unknown, 4 peering, 2 active+clean+remapped, 3 active+undersized, 3 active+undersized+degraded, 85 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2/78 objects degraded (2.564%); 1/78 objects misplaced (1.282%) 2024-09-14T04:42:53.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:52 smithi039 ceph-mon[111235]: osdmap e1377: 8 total, 7 up, 8 in 2024-09-14T04:42:53.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:52 smithi039 ceph-mon[111235]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-14T04:42:53.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:52 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:42:53.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:42:53.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:52 smithi039 ceph-mon[106470]: pgmap v2343: 99 pgs: 2 unknown, 4 peering, 2 active+clean+remapped, 3 active+undersized, 3 active+undersized+degraded, 85 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 2/78 objects degraded (2.564%); 1/78 objects misplaced (1.282%) 2024-09-14T04:42:53.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:52 smithi039 ceph-mon[106470]: osdmap e1377: 8 total, 7 up, 8 in 2024-09-14T04:42:53.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:52 smithi039 ceph-mon[106470]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-14T04:42:53.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:52 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:42:53.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:42:53.369 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:53.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:53.393 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:53.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:53.494 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:53.516 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:53.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:53.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:53.618 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:53.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:53.642 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:53.642 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:53.662 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:42:53.663 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:42:53.743 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:53.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:53.767 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:53.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:53.868 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:53.896 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:53.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:53.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:53.998 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:54.019 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:54.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:54.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:54.121 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:54.146 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:54.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:54.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:54.248 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:54.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:53 smithi129 ceph-mon[101316]: osdmap e1378: 8 total, 7 up, 8 in 2024-09-14T04:42:54.271 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:54.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:54.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:54.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:53 smithi039 ceph-mon[106470]: osdmap e1378: 8 total, 7 up, 8 in 2024-09-14T04:42:54.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:53 smithi039 ceph-mon[111235]: osdmap e1378: 8 total, 7 up, 8 in 2024-09-14T04:42:54.373 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:54.395 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:54.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:54.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:54.497 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:54.519 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:54.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:54.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:54.621 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:54.646 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:54.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:54.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:54.748 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:54.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:54.774 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:54.774 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:54.875 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:54.897 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:54.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:54.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:55.000 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:55.013 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:54 smithi039 ceph-mon[106470]: pgmap v2346: 99 pgs: 1 active+undersized+remapped, 2 unknown, 5 peering, 1 active+clean+remapped, 90 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:42:55.014 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:54 smithi039 ceph-mon[106470]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/78 objects degraded (2.564%), 3 pgs degraded, 6 pgs undersized) 2024-09-14T04:42:55.014 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:54 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2194359821' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:42:55.014 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:54 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:42:55.014 INFO:teuthology.orchestra.run.smithi039.stderr:pool 'unique_pool_0' removed 2024-09-14T04:42:55.014 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:54 smithi039 ceph-mon[111235]: pgmap v2346: 99 pgs: 1 active+undersized+remapped, 2 unknown, 5 peering, 1 active+clean+remapped, 90 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:42:55.014 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:54 smithi039 ceph-mon[111235]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/78 objects degraded (2.564%), 3 pgs degraded, 6 pgs undersized) 2024-09-14T04:42:55.014 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:54 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2194359821' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:42:55.015 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:54 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:42:55.026 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:55.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:55.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:55.043 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:42:55.128 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:55.150 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:55.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:55.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:55.252 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:55.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:54 smithi129 ceph-mon[101316]: pgmap v2346: 99 pgs: 1 active+undersized+remapped, 2 unknown, 5 peering, 1 active+clean+remapped, 90 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:42:55.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:54 smithi129 ceph-mon[101316]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/78 objects degraded (2.564%), 3 pgs degraded, 6 pgs undersized) 2024-09-14T04:42:55.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:54 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2194359821' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:42:55.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:54 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:42:55.279 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:55.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:55.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:55.381 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:55.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:55.409 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:55.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:55.510 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:55.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:55.537 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:55.538 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:55.638 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:55.664 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:42:55.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:55.665 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:55.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:55.766 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:55.792 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:55.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:55.793 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:55.893 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:55.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:55.917 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:55.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:56.018 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:56.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:56.042 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:56.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:56.141 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:56 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:42:56.141 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:56 smithi039 ceph-mon[106470]: osdmap e1379: 8 total, 7 up, 8 in 2024-09-14T04:42:56.141 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:56 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:42:56.141 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:56 smithi039 ceph-mon[111235]: osdmap e1379: 8 total, 7 up, 8 in 2024-09-14T04:42:56.143 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:56.160 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-14T04:42:56.161 INFO:tasks.radosbench:Beginning radosbench... 2024-09-14T04:42:56.162 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2024-09-14T04:42:56.162 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_1 16 2024-09-14T04:42:56.167 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:56.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:56.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:56.269 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:56.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:56 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:42:56.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:56 smithi129 ceph-mon[101316]: osdmap e1379: 8 total, 7 up, 8 in 2024-09-14T04:42:56.295 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:56.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:56.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:56.397 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:56.409 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:42:56.409 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:42:56.424 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:56.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:56.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:56.525 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:56.549 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:56.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:56.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:56.636 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:42:56.651 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:56.674 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:56.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:56.675 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:56.775 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:56.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:56.801 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:56.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:56.902 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:56.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:56.925 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:56.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:57.026 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:57.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:57.049 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:57.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:57.150 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:57.172 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:57.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:57.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:57.209 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-14T04:42:57.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:57 smithi129 ceph-mon[101316]: pgmap v2348: 83 pgs: 83 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:42:57.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:57 smithi129 ceph-mon[101316]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-09-14T04:42:57.272 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:42:57.274 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:57.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:57 smithi039 ceph-mon[106470]: pgmap v2348: 83 pgs: 83 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:42:57.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:57 smithi039 ceph-mon[106470]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-09-14T04:42:57.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:57 smithi039 ceph-mon[111235]: pgmap v2348: 83 pgs: 83 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:42:57.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:57 smithi039 ceph-mon[111235]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-09-14T04:42:57.301 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:57.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:57.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:57.403 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:57.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:57.425 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:57.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:57.526 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:57.548 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:57.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:57.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:57.650 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:57.672 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:57.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:57.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:57.755 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [4] live_osds: [0, 1, 2, 3, 5, 6, 7] 2024-09-14T04:42:57.755 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:42:57.755 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-09-14T04:42:57.755 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.698030 2024-09-14T04:42:57.755 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 3 0.6980296320016317 2024-09-14T04:42:57.775 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:57.801 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:57.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:57.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:57.902 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:57.925 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:57.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:57.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:58.027 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:42:58.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:58.050 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:58.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:42:58.151 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:58.177 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:58.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:58.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:58.278 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:58.304 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:58.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:58.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:58.406 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:58.431 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:58.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:58.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:58.533 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:58.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:58.556 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:58.556 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:58.657 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:58.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:58.680 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:58.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:58.781 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:58.788 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:42:58.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:58.807 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:58.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:58.909 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:58.934 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:58.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:58.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:58.987 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:42:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:42:58] "GET /metrics HTTP/1.1" 200 36752 "" "Prometheus/2.51.0" 2024-09-14T04:42:59.036 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:42:59.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:59.094 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:59.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:42:59.195 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:42:59.217 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:59.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:59.218 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:42:59.263 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:42:59 smithi039 ceph-mon[106470]: pgmap v2349: 83 pgs: 83 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:42:59.263 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:42:59 smithi039 ceph-mon[111235]: pgmap v2349: 83 pgs: 83 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:42:59.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:42:59 smithi129 ceph-mon[101316]: pgmap v2349: 83 pgs: 83 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:42:59.319 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:59.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:59.346 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:59.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:42:59.447 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:42:59.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:59.471 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:42:59.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:42:59.573 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:59.599 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:59.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:59.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:59.701 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:42:59.728 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:59.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:59.729 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:42:59.830 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:42:59.855 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:59.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:59.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:42:59.957 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:42:59.982 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:42:59.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:42:59.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:00.039 INFO:teuthology.orchestra.run.smithi039.stderr:pool 'unique_pool_1' created 2024-09-14T04:43:00.084 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:00.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:00.108 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:00.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:00.209 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:00.235 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:00.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:00.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:00.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:00 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1742863599' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-14T04:43:00.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:00 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-14T04:43:00.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:00 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1742863599' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-14T04:43:00.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:00 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-14T04:43:00.337 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:00.359 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:00.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:00.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:00.461 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:00.486 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:00.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:00.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:00.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:00 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1742863599' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-14T04:43:00.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:00 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16} : dispatch 2024-09-14T04:43:00.574 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:43:00.588 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:00.614 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:00.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:00.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:00.715 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:00.741 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:00.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:00.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:00.843 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:00.866 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:00.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:00.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:00.968 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:00.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:00.994 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:00.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:01.095 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:01.121 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:01.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:01.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:01.223 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:01.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:01.249 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:01.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:01.350 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:01.374 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:01.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:01.375 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:01.476 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:01.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:01.502 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:01.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:01.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:01 smithi129 ceph-mon[101316]: pgmap v2350: 83 pgs: 83 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:43:01.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:01 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-09-14T04:43:01.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:01 smithi129 ceph-mon[101316]: osdmap e1380: 8 total, 7 up, 8 in 2024-09-14T04:43:01.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:01 smithi039 ceph-mon[106470]: pgmap v2350: 83 pgs: 83 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:43:01.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:01 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-09-14T04:43:01.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:01 smithi039 ceph-mon[106470]: osdmap e1380: 8 total, 7 up, 8 in 2024-09-14T04:43:01.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:01 smithi039 ceph-mon[111235]: pgmap v2350: 83 pgs: 83 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:43:01.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:01 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2024-09-14T04:43:01.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:01 smithi039 ceph-mon[111235]: osdmap e1380: 8 total, 7 up, 8 in 2024-09-14T04:43:01.603 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:01.629 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2024-09-14T04:43:01.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:01.632 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:01.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:01.733 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:01.755 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:01.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:01.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:01.857 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:01.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:01.880 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:01.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:01.981 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:02.003 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:02.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:02.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:02.105 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:02.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:02.130 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:02.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:02.144 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:02 smithi039 ceph-mon[106470]: osdmap e1381: 8 total, 7 up, 8 in 2024-09-14T04:43:02.145 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:02 smithi039 ceph-mon[111235]: osdmap e1381: 8 total, 7 up, 8 in 2024-09-14T04:43:02.231 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:02.254 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:02.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:02.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:02.356 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:02.379 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:02.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:02.380 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:02.481 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:02.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:02.505 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:02.505 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:02.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:02 smithi129 ceph-mon[101316]: osdmap e1381: 8 total, 7 up, 8 in 2024-09-14T04:43:02.605 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:02.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:02.632 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:02.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:02.733 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:02.755 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:02.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:02.757 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:02.857 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:02.880 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:02.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:02.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:02.981 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:03.004 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:03.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:03.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:03.105 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:03.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:03.129 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:03.129 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:03.230 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:03.257 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:03.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:03.258 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:03.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:03 smithi039 ceph-mon[106470]: pgmap v2353: 99 pgs: 16 unknown, 83 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:43:03.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:03 smithi039 ceph-mon[106470]: osdmap e1382: 8 total, 7 up, 8 in 2024-09-14T04:43:03.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:03 smithi039 ceph-mon[106470]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:43:03.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:03 smithi039 ceph-mon[111235]: pgmap v2353: 99 pgs: 16 unknown, 83 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:43:03.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:03 smithi039 ceph-mon[111235]: osdmap e1382: 8 total, 7 up, 8 in 2024-09-14T04:43:03.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:03 smithi039 ceph-mon[111235]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:43:03.358 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:03.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:03.386 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:03.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:03.487 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:03.512 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:03.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:03.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:03.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:03 smithi129 ceph-mon[101316]: pgmap v2353: 99 pgs: 16 unknown, 83 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:43:03.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:03 smithi129 ceph-mon[101316]: osdmap e1382: 8 total, 7 up, 8 in 2024-09-14T04:43:03.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:03 smithi129 ceph-mon[101316]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:43:03.615 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:03.641 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:03.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:03.642 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:03.744 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:03.769 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:03.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:03.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:03.871 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:03.895 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:03.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:03.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:03.997 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:04.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:04.023 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:04.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:04.124 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:04.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:04.149 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:04.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:04.251 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:04.290 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:04.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:04.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:04.393 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:04.423 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:04.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:04.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:04.525 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:04.549 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:04.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:04.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:04.651 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:04.675 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:04.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:04.676 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:04.689 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:43:04.777 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:04.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:04.806 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:04.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:04.907 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:04.932 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:04.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:04.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:05.034 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:05.057 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:05.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:05.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:05.159 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:05.184 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:05.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:05.185 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:05.286 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:05.312 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:05.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:05.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:05.416 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:05.440 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:05.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:05.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:05.457 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:05 smithi039 ceph-mon[106470]: pgmap v2355: 99 pgs: 3 unknown, 96 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:43:05.457 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:05 smithi039 ceph-mon[111235]: pgmap v2355: 99 pgs: 3 unknown, 96 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:43:05.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:05 smithi129 ceph-mon[101316]: pgmap v2355: 99 pgs: 3 unknown, 96 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:43:05.542 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:05.566 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:05.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:05.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:05.668 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:05.693 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:05.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:05.694 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:05.795 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:05.819 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:05.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:05.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:05.921 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:05.944 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:05.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:05.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:06.046 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:06.072 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:06.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:06.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:06.174 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:06.196 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:06.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:06.197 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:06.298 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:06.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:06.322 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:06.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:06.423 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:06.444 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:06.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:06.445 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:06.546 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:06.568 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:06.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:06.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:06.670 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:06.696 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:06.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:06.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:06.797 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:06.824 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:06.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:06.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:06.926 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:06.951 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:06.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:06.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:07.053 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:07.079 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:07.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:07.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:07.094 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is set 2024-09-14T04:43:07.182 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:07.205 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:07.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:07.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:07.307 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:07.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:07.331 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:07.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:07.334 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:07 smithi039 ceph-mon[106470]: pgmap v2356: 99 pgs: 99 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:43:07.334 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:07 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1803152641' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:43:07.334 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:07 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:43:07.334 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:07 smithi039 ceph-mon[111235]: pgmap v2356: 99 pgs: 99 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:43:07.334 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:07 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1803152641' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:43:07.334 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:07 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:43:07.432 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:07.454 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:07.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:07.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:07.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:07 smithi129 ceph-mon[101316]: pgmap v2356: 99 pgs: 99 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:43:07.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:07 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1803152641' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:43:07.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:07 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:43:07.556 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:07.578 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:07.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:07.579 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:07.588 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:43:07.588 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:43:07.603 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:43:07.603 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:43:07.619 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:43:07.620 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:43:07.635 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:43:07.636 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:43:07.636 INFO:teuthology.orchestra.run.smithi039.stderr:Error ENXIO: problem getting command descriptions from osd.4 2024-09-14T04:43:07.653 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:43:07.653 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:43:07.669 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:43:07.670 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:43:07.680 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:07.686 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:43:07.687 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:43:07.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:07.704 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:07.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:07.805 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:07.831 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:07.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:07.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:07.933 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:07.955 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:07.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:07.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:08.057 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:08.084 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:08.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:08.085 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:08.185 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:08.187 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:08 smithi039 ceph-mon[106470]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:43:08.187 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:08 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:43:08.187 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:08 smithi039 ceph-mon[106470]: osdmap e1383: 8 total, 7 up, 8 in 2024-09-14T04:43:08.187 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:43:08.187 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:08 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:43:08.187 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:08 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/76818855' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:43:08.188 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:08 smithi039 ceph-mon[111235]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:43:08.188 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:08 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:43:08.188 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:08 smithi039 ceph-mon[111235]: osdmap e1383: 8 total, 7 up, 8 in 2024-09-14T04:43:08.188 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:08 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:43:08.188 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:08 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:43:08.188 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:08 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/76818855' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:43:08.210 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:08.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:08.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:08.312 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:08.334 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:08.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:08.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:08.436 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:08.458 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:08.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:08.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:08 smithi129 ceph-mon[101316]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:43:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:08 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:43:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:08 smithi129 ceph-mon[101316]: osdmap e1383: 8 total, 7 up, 8 in 2024-09-14T04:43:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:08 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:43:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:08 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:43:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:08 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/76818855' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:43:08.561 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:08.583 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:08.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:08.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:08.685 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:08.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:08.709 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:08.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:08.810 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:08.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:08.833 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:08.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:08.930 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:43:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:43:08] "GET /metrics HTTP/1.1" 200 36691 "" "Prometheus/2.51.0" 2024-09-14T04:43:08.934 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:08.960 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:08.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:08.961 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:09.062 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:09.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:09.086 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:09.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:09.187 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:09.210 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:09.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:09.211 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:09.312 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:09.335 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:09.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:09.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:09.356 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:43:09.356 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:43:09.437 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:09.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:09.464 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:09.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:09.565 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:09.571 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:09 smithi039 ceph-mon[111235]: pgmap v2358: 99 pgs: 99 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:43:09.571 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:09 smithi039 ceph-mon[106470]: pgmap v2358: 99 pgs: 99 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:43:09.573 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:43:09.591 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:09.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:09.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:09.693 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:09.717 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:09.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:09.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:09 smithi129 ceph-mon[101316]: pgmap v2358: 99 pgs: 99 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:43:09.818 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:09.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:09.844 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:09.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:09.945 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:09.971 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:09.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:09.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:10.073 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:10.099 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:10.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:10.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:10.201 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:10.225 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:10.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:10.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:10.327 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:10.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:10.350 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:10.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:10.451 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:10.477 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:10.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:10.478 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:10.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:10 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [3, 5]} : dispatch 2024-09-14T04:43:10.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:10 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 5]} : dispatch 2024-09-14T04:43:10.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:10 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [3, 5]} : dispatch 2024-09-14T04:43:10.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:10 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 5]} : dispatch 2024-09-14T04:43:10.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:10 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [3, 5]} : dispatch 2024-09-14T04:43:10.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:10 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 5]} : dispatch 2024-09-14T04:43:10.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:10 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [3, 5]} : dispatch 2024-09-14T04:43:10.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:10 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 5]} : dispatch 2024-09-14T04:43:10.579 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:10.604 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:10.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:10.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:10.670 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:43:10.670 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [], 2024-09-14T04:43:10.670 INFO:teuthology.orchestra.run.smithi039.stdout: "complaint_time": 30, 2024-09-14T04:43:10.670 INFO:teuthology.orchestra.run.smithi039.stdout: "num_blocked_ops": 0 2024-09-14T04:43:10.670 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:43:10.707 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:10.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:10.730 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:10.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:10.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:10 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [3, 5]} : dispatch 2024-09-14T04:43:10.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:10 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 5]} : dispatch 2024-09-14T04:43:10.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:10 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [3, 5]} : dispatch 2024-09-14T04:43:10.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:10 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 5]} : dispatch 2024-09-14T04:43:10.830 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:10.853 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:10.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:10.854 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:10.955 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:10.980 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:10.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:10.981 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:11.082 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:11.108 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:11.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:11.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:11.210 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:11.232 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:11.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:11.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:11.334 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:11.357 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:43:11.359 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:11.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:11.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:11.461 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:11.484 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:11.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:11.485 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:11.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:11 smithi039 ceph-mon[106470]: pgmap v2359: 99 pgs: 99 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:43:11.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:11 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [3, 5]}]': finished 2024-09-14T04:43:11.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:11 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 5]}]': finished 2024-09-14T04:43:11.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:11 smithi039 ceph-mon[106470]: osdmap e1384: 8 total, 7 up, 8 in 2024-09-14T04:43:11.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:11 smithi039 ceph-mon[111235]: pgmap v2359: 99 pgs: 99 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:43:11.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:11 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [3, 5]}]': finished 2024-09-14T04:43:11.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:11 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 5]}]': finished 2024-09-14T04:43:11.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:11 smithi039 ceph-mon[111235]: osdmap e1384: 8 total, 7 up, 8 in 2024-09-14T04:43:11.585 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:11.611 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:11.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:11.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:11.713 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:11.736 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:11.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:11.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:11 smithi129 ceph-mon[101316]: pgmap v2359: 99 pgs: 99 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:43:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:11 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.10", "id": [3, 5]}]': finished 2024-09-14T04:43:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:11 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1a", "id": [3, 5]}]': finished 2024-09-14T04:43:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:11 smithi129 ceph-mon[101316]: osdmap e1384: 8 total, 7 up, 8 in 2024-09-14T04:43:11.838 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:11.861 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:11.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:11.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:11.963 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:11.986 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:43:11.989 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:11.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:11.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:12.091 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:12.114 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:12.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:12.115 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:12.216 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:12.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:12.244 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:12.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:12.343 INFO:teuthology.orchestra.run.smithi039.stderr:set osd.3 primary-affinity to 0.69803 (8457462) 2024-09-14T04:43:12.345 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:12.375 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:12.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:12.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:12.430 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-14T04:43:12.477 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:12.499 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:12.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:12.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:12.601 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:12.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:12.625 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:12.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:12.726 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:12.752 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:12.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:12.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:12.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:12 smithi129 ceph-mon[101316]: osdmap e1385: 8 total, 7 up, 8 in 2024-09-14T04:43:12.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:12 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2230763254' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.6980296320016317} : dispatch 2024-09-14T04:43:12.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:12 smithi039 ceph-mon[106470]: osdmap e1385: 8 total, 7 up, 8 in 2024-09-14T04:43:12.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:12 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2230763254' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.6980296320016317} : dispatch 2024-09-14T04:43:12.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:12 smithi039 ceph-mon[111235]: osdmap e1385: 8 total, 7 up, 8 in 2024-09-14T04:43:12.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:12 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2230763254' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.6980296320016317} : dispatch 2024-09-14T04:43:12.854 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:12.876 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:12.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:12.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:12.978 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:13.003 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:13.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:13.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:13.105 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:13.128 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:13.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:13.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:13.230 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:13.251 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:13.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:13.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:13.353 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:13.378 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:13.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:13.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:13.480 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:13.505 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:13.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:13.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:13.606 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:13.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:13.630 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:13.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:13.731 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:13.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:13.754 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:13.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:13.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:13 smithi129 ceph-mon[101316]: pgmap v2362: 99 pgs: 99 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:43:13.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:13 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2230763254' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.6980296320016317}]': finished 2024-09-14T04:43:13.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:13 smithi129 ceph-mon[101316]: osdmap e1386: 8 total, 7 up, 8 in 2024-09-14T04:43:13.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:13 smithi129 ceph-mon[101316]: osdmap e1387: 8 total, 7 up, 8 in 2024-09-14T04:43:13.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:13 smithi039 ceph-mon[106470]: pgmap v2362: 99 pgs: 99 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:43:13.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:13 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2230763254' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.6980296320016317}]': finished 2024-09-14T04:43:13.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:13 smithi039 ceph-mon[106470]: osdmap e1386: 8 total, 7 up, 8 in 2024-09-14T04:43:13.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:13 smithi039 ceph-mon[106470]: osdmap e1387: 8 total, 7 up, 8 in 2024-09-14T04:43:13.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:13 smithi039 ceph-mon[111235]: pgmap v2362: 99 pgs: 99 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:43:13.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:13 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2230763254' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.6980296320016317}]': finished 2024-09-14T04:43:13.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:13 smithi039 ceph-mon[111235]: osdmap e1386: 8 total, 7 up, 8 in 2024-09-14T04:43:13.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:13 smithi039 ceph-mon[111235]: osdmap e1387: 8 total, 7 up, 8 in 2024-09-14T04:43:13.856 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:13.878 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:13.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:13.879 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:13.980 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:14.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:14.003 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:14.003 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:14.104 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:14.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:14.134 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:14.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:14.235 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:14.257 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:14.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:14.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:14.358 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:14.384 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:14.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:14.386 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:14.455 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:14 smithi039 ceph-mon[106470]: Health check failed: Reduced data availability: 2 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-09-14T04:43:14.455 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:14 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:43:14.456 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:14 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:43:14.456 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:14 smithi039 ceph-mon[111235]: Health check failed: Reduced data availability: 2 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-09-14T04:43:14.456 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:14 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:43:14.456 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:14 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:43:14.486 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:14.513 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:14.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:14.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:14.615 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:14.649 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:14.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:14.650 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:14.751 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:14.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:14 smithi129 ceph-mon[101316]: Health check failed: Reduced data availability: 2 pgs inactive, 4 pgs peering (PG_AVAILABILITY) 2024-09-14T04:43:14.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:14 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:43:14.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:14 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:43:14.777 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:14.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:14.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:14.835 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:43:14.835 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:43:14.835 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:43:14.882 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:14.907 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:14.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:14.908 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:15.009 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:15.033 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:15.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:15.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:15.134 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:15.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:15.158 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:15.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:15.259 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:15.282 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:15.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:15.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:15.383 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:15.405 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:15.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:15.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:15.507 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:15.529 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:15.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:15.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:15.631 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:15.657 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:15.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:15.658 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:15.759 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:15.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:15 smithi129 ceph-mon[101316]: pgmap v2365: 99 pgs: 6 peering, 93 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:43:15.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:15 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:43:15.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:15 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:43:15.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:15 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:43:15.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:15 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2611360160' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:43:15.776 INFO:teuthology.orchestra.run.smithi039.stderr:enabled application 'rados' on pool 'unique_pool_1' 2024-09-14T04:43:15.777 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:15 smithi039 ceph-mon[106470]: pgmap v2365: 99 pgs: 6 peering, 93 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:43:15.777 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:15 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:43:15.777 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:15 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:43:15.777 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:15 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:43:15.777 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:15 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2611360160' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:43:15.777 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:15 smithi039 ceph-mon[111235]: pgmap v2365: 99 pgs: 6 peering, 93 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:43:15.777 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:15 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:43:15.778 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:15 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:43:15.778 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:15 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:43:15.778 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:15 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2611360160' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:43:15.785 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:15.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:15.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:15.887 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:15.913 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:15.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:15.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:16.015 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:16.040 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:16.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:16.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:16.142 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:16.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:16.167 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:16.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:16.268 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:16.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:16.293 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:16.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:16.394 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:16.417 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:16.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:16.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:16.519 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:16.542 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:16.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:16.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:16.644 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:16.667 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:16.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:16.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:16.769 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:16.791 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:16.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:16.792 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:16.893 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:16.916 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:16.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:16.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:17.018 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:17.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:16 smithi039 ceph-mon[106470]: pgmap v2366: 99 pgs: 6 peering, 93 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:43:17.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:16 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2611360160' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:43:17.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:16 smithi039 ceph-mon[106470]: osdmap e1388: 8 total, 7 up, 8 in 2024-09-14T04:43:17.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:16 smithi039 ceph-mon[111235]: pgmap v2366: 99 pgs: 6 peering, 93 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:43:17.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:16 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2611360160' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:43:17.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:16 smithi039 ceph-mon[111235]: osdmap e1388: 8 total, 7 up, 8 in 2024-09-14T04:43:17.044 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:17.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:17.045 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:17.146 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:17.172 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:17.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:17.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:17.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:16 smithi129 ceph-mon[101316]: pgmap v2366: 99 pgs: 6 peering, 93 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:43:17.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:16 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2611360160' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:43:17.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:16 smithi129 ceph-mon[101316]: osdmap e1388: 8 total, 7 up, 8 in 2024-09-14T04:43:17.273 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:17.296 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:17.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:17.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:17.398 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:17.421 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:17.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:17.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:17.479 DEBUG:teuthology.orchestra.run.smithi039:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2024-09-14T04:43:17.481 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-14T04:43:17.481 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-14T04:43:17.524 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:17.532 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:hints = 1 2024-09-14T04:43:17.532 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-14T04:43:17.532 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Object prefix: benchmark_data_smithi039_264334 2024-09-14T04:43:17.532 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:43:17.533 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 0 0 0 0 0 0 - 0 2024-09-14T04:43:17.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:17.567 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:17.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:17.669 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:17.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:17.708 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:17.708 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:17.809 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:17.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:17.856 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:17.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:17.957 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:17.997 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:17.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:17.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:18.008 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:43:18.008 INFO:teuthology.orchestra.run.smithi039.stdout: "size": 20, 2024-09-14T04:43:18.009 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 600, 2024-09-14T04:43:18.009 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [ 2024-09-14T04:43:18.009 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.009 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233351 53.7 53:ed868572:::benchmark_data_smithi039_224949_object103258:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1355)", 2024-09-14T04:43:18.009 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.814699+0000", 2024-09-14T04:43:18.009 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 73.190897672999995, 2024-09-14T04:43:18.009 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1969449910000001, 2024-09-14T04:43:18.009 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:43:18.009 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:43:18.009 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T04:43:18.009 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:43:18.009 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:43:18.010 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:43:18.010 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233351 2024-09-14T04:43:18.010 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.010 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:43:18.010 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.010 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:43:18.010 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814699+0000", 2024-09-14T04:43:18.010 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.010 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.010 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.010 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:43:18.010 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814699+0000", 2024-09-14T04:43:18.010 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.010 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.011 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.011 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:43:18.011 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814700+0000", 2024-09-14T04:43:18.011 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6499999999999998e-07 2024-09-14T04:43:18.011 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.011 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.011 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:43:18.011 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814703+0000", 2024-09-14T04:43:18.011 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.1310000000000002e-06 2024-09-14T04:43:18.011 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.011 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.011 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:43:18.011 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814712+0000", 2024-09-14T04:43:18.012 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.4749999999999993e-06 2024-09-14T04:43:18.012 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.012 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.012 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:43:18.012 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814718+0000", 2024-09-14T04:43:18.012 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.5089999999999999e-06 2024-09-14T04:43:18.012 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.012 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.012 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.012 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.815266+0000", 2024-09-14T04:43:18.012 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00054806600000000005 2024-09-14T04:43:18.012 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.012 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.013 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op must wait for map", 2024-09-14T04:43:18.013 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.815274+0000", 2024-09-14T04:43:18.013 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.8630000000000008e-06 2024-09-14T04:43:18.013 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.013 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.013 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.013 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.830888+0000", 2024-09-14T04:43:18.013 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.015613846000000001 2024-09-14T04:43:18.013 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.013 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.013 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:43:18.013 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.830900+0000", 2024-09-14T04:43:18.013 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1911e-05 2024-09-14T04:43:18.013 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.014 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.014 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.014 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.814346+0000", 2024-09-14T04:43:18.014 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.98344564700000003 2024-09-14T04:43:18.014 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.014 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.014 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:43:18.014 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.814359+0000", 2024-09-14T04:43:18.014 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3509999999999999e-05 2024-09-14T04:43:18.014 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.014 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.014 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.014 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.009662+0000", 2024-09-14T04:43:18.014 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.195302284 2024-09-14T04:43:18.015 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.015 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.015 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:43:18.015 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.011644+0000", 2024-09-14T04:43:18.015 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0019829840000000001 2024-09-14T04:43:18.015 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.015 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:43:18.015 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.015 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.015 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.015 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233354 53.7 53:e0779eb3:::benchmark_data_smithi039_224949_object103261:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1355)", 2024-09-14T04:43:18.015 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.814735+0000", 2024-09-14T04:43:18.015 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 73.190861824999999, 2024-09-14T04:43:18.015 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2280154030000001, 2024-09-14T04:43:18.016 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:43:18.016 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:43:18.016 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:43:18.016 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:43:18.016 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:43:18.016 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:43:18.016 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233354 2024-09-14T04:43:18.016 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.016 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:43:18.016 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.016 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:43:18.016 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814735+0000", 2024-09-14T04:43:18.016 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.017 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.017 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.017 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:43:18.017 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814735+0000", 2024-09-14T04:43:18.017 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.017 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.017 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.017 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:43:18.017 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814736+0000", 2024-09-14T04:43:18.017 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.0200000000000002e-07 2024-09-14T04:43:18.017 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.017 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.017 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:43:18.017 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814738+0000", 2024-09-14T04:43:18.017 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.43e-06 2024-09-14T04:43:18.018 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.018 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.018 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:43:18.018 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814747+0000", 2024-09-14T04:43:18.018 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.2390000000000002e-06 2024-09-14T04:43:18.018 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.018 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.018 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:43:18.018 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814753+0000", 2024-09-14T04:43:18.018 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.6510000000000004e-06 2024-09-14T04:43:18.018 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.018 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.018 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.018 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.820795+0000", 2024-09-14T04:43:18.019 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0060417639999999998 2024-09-14T04:43:18.019 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.019 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.019 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting_for_map not empty", 2024-09-14T04:43:18.019 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.820801+0000", 2024-09-14T04:43:18.019 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.9739999999999999e-06 2024-09-14T04:43:18.019 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.019 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.019 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.019 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.830925+0000", 2024-09-14T04:43:18.019 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.010123996 2024-09-14T04:43:18.019 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.019 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.019 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:43:18.020 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.830931+0000", 2024-09-14T04:43:18.020 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.421e-06 2024-09-14T04:43:18.020 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.020 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.020 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.020 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.814390+0000", 2024-09-14T04:43:18.020 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.98345883099999998 2024-09-14T04:43:18.020 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.020 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.020 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:43:18.020 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.814398+0000", 2024-09-14T04:43:18.020 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.5159999999999997e-06 2024-09-14T04:43:18.021 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.021 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.021 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.021 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.011957+0000", 2024-09-14T04:43:18.021 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1975596669999999 2024-09-14T04:43:18.021 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.021 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.021 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:43:18.021 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.012683+0000", 2024-09-14T04:43:18.021 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00072578599999999997 2024-09-14T04:43:18.021 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.021 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.021 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 0,1,7", 2024-09-14T04:43:18.022 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.013085+0000", 2024-09-14T04:43:18.022 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00040193299999999999 2024-09-14T04:43:18.022 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.022 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.022 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.022 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.031842+0000", 2024-09-14T04:43:18.022 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.018757149000000001 2024-09-14T04:43:18.022 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.022 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.022 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:43:18.022 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.035830+0000", 2024-09-14T04:43:18.022 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0039881919999999998 2024-09-14T04:43:18.022 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.023 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.023 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.023 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.038481+0000", 2024-09-14T04:43:18.023 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0026504660000000002 2024-09-14T04:43:18.023 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.023 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.023 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.023 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.042512+0000", 2024-09-14T04:43:18.024 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.004031247 2024-09-14T04:43:18.024 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.024 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.024 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:43:18.024 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.042625+0000", 2024-09-14T04:43:18.024 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000112434 2024-09-14T04:43:18.024 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.024 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.024 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:43:18.024 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.042751+0000", 2024-09-14T04:43:18.024 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000126205 2024-09-14T04:43:18.024 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.024 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:43:18.024 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.025 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.025 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.025 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233359 53.7 53:e8733a52:::benchmark_data_smithi039_224949_object103266:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-09-14T04:43:18.025 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.820832+0000", 2024-09-14T04:43:18.025 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 73.184764959999995, 2024-09-14T04:43:18.025 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2221726209999999, 2024-09-14T04:43:18.025 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:43:18.025 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:43:18.025 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:43:18.025 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:43:18.025 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:43:18.025 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:43:18.025 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233359 2024-09-14T04:43:18.026 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.026 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:43:18.026 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.026 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:43:18.026 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.820832+0000", 2024-09-14T04:43:18.026 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.026 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.026 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.026 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:43:18.026 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.820832+0000", 2024-09-14T04:43:18.026 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.026 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.026 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.026 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:43:18.027 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.820834+0000", 2024-09-14T04:43:18.027 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7010000000000001e-06 2024-09-14T04:43:18.027 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.027 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.027 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:43:18.027 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.820839+0000", 2024-09-14T04:43:18.027 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6890000000000003e-06 2024-09-14T04:43:18.027 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.027 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.027 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:43:18.027 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.820848+0000", 2024-09-14T04:43:18.027 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.5710000000000004e-06 2024-09-14T04:43:18.027 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.027 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.028 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:43:18.028 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.820856+0000", 2024-09-14T04:43:18.028 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6329999999999996e-06 2024-09-14T04:43:18.028 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.028 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.028 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.028 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.830952+0000", 2024-09-14T04:43:18.028 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.010096519 2024-09-14T04:43:18.028 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.028 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.028 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:43:18.028 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.830959+0000", 2024-09-14T04:43:18.028 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.426e-06 2024-09-14T04:43:18.029 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.029 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.029 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.029 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.814422+0000", 2024-09-14T04:43:18.029 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.98346295299999997 2024-09-14T04:43:18.029 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.029 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.029 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:43:18.029 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.814430+0000", 2024-09-14T04:43:18.029 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9899999999999997e-06 2024-09-14T04:43:18.029 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.029 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.029 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.029 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.028595+0000", 2024-09-14T04:43:18.030 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.214164939 2024-09-14T04:43:18.030 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.030 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.030 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:43:18.030 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.028728+0000", 2024-09-14T04:43:18.030 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000133213 2024-09-14T04:43:18.030 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.030 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.030 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 0,1,7", 2024-09-14T04:43:18.030 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.028867+0000", 2024-09-14T04:43:18.030 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000138801 2024-09-14T04:43:18.030 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.030 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.030 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.031 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.031892+0000", 2024-09-14T04:43:18.031 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0030254040000000002 2024-09-14T04:43:18.031 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.031 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.031 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.031 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.040818+0000", 2024-09-14T04:43:18.031 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0089257309999999992 2024-09-14T04:43:18.031 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.031 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.031 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:43:18.031 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.041553+0000", 2024-09-14T04:43:18.031 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00073544599999999995 2024-09-14T04:43:18.031 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.031 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.032 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.032 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.042807+0000", 2024-09-14T04:43:18.032 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0012535210000000001 2024-09-14T04:43:18.032 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.032 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.032 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:43:18.032 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.042903+0000", 2024-09-14T04:43:18.032 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.6471999999999994e-05 2024-09-14T04:43:18.032 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.032 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.032 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:43:18.032 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.043005+0000", 2024-09-14T04:43:18.032 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000101612 2024-09-14T04:43:18.033 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.033 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:43:18.033 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.033 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.033 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.033 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233393 53.7 53:e9d8059a:::benchmark_data_smithi039_224949_object103300:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-09-14T04:43:18.033 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.894168+0000", 2024-09-14T04:43:18.033 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 73.111429357000006, 2024-09-14T04:43:18.033 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.149336516, 2024-09-14T04:43:18.033 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:43:18.033 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:43:18.033 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:43:18.033 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:43:18.033 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:43:18.033 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:43:18.034 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233393 2024-09-14T04:43:18.034 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.034 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:43:18.034 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.034 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:43:18.034 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894168+0000", 2024-09-14T04:43:18.034 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.034 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.034 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.034 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:43:18.034 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894168+0000", 2024-09-14T04:43:18.034 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.034 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.034 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.035 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:43:18.035 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894169+0000", 2024-09-14T04:43:18.035 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.443e-06 2024-09-14T04:43:18.035 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.035 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.035 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:43:18.035 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894173+0000", 2024-09-14T04:43:18.035 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.5149999999999998e-06 2024-09-14T04:43:18.035 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.035 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.035 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:43:18.035 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894182+0000", 2024-09-14T04:43:18.035 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.3360000000000001e-06 2024-09-14T04:43:18.035 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.036 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.036 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:43:18.036 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894188+0000", 2024-09-14T04:43:18.036 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3439999999999997e-06 2024-09-14T04:43:18.036 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.036 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.036 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.036 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894306+0000", 2024-09-14T04:43:18.036 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000117439 2024-09-14T04:43:18.036 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.036 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.036 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:43:18.036 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894316+0000", 2024-09-14T04:43:18.036 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0349e-05 2024-09-14T04:43:18.037 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.037 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.037 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.037 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.814569+0000", 2024-09-14T04:43:18.037 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.92025298899999997 2024-09-14T04:43:18.037 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.037 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.037 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:43:18.037 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.814584+0000", 2024-09-14T04:43:18.037 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.468e-05 2024-09-14T04:43:18.037 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.037 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.037 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.037 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.032234+0000", 2024-09-14T04:43:18.038 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2176502870000001 2024-09-14T04:43:18.038 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.038 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.038 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:43:18.038 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.032359+0000", 2024-09-14T04:43:18.038 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000124533 2024-09-14T04:43:18.038 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.038 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.038 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 0,1,7", 2024-09-14T04:43:18.038 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.032518+0000", 2024-09-14T04:43:18.038 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015903900000000001 2024-09-14T04:43:18.038 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.038 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.039 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.039 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.040779+0000", 2024-09-14T04:43:18.039 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0082608909999999994 2024-09-14T04:43:18.039 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.039 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.039 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.039 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.041107+0000", 2024-09-14T04:43:18.039 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000328778 2024-09-14T04:43:18.039 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.039 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.039 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:43:18.039 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.041689+0000", 2024-09-14T04:43:18.039 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00058127799999999996 2024-09-14T04:43:18.039 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.040 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.040 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.040 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.043303+0000", 2024-09-14T04:43:18.040 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001614297 2024-09-14T04:43:18.040 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.040 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.040 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:43:18.040 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.043390+0000", 2024-09-14T04:43:18.040 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.6940000000000004e-05 2024-09-14T04:43:18.040 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.040 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.040 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:43:18.040 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.043504+0000", 2024-09-14T04:43:18.040 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011437799999999999 2024-09-14T04:43:18.041 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.041 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:43:18.041 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.041 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.041 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.041 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:235290 53.6 53:633ba3b3:::benchmark_data_smithi039_224949_object105197:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1357)", 2024-09-14T04:43:18.041 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:09.453718+0000", 2024-09-14T04:43:18.041 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 68.551879370999998, 2024-09-14T04:43:18.041 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.12523731299999999, 2024-09-14T04:43:18.041 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:43:18.041 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:43:18.041 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:43:18.041 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:43:18.042 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:43:18.042 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:43:18.042 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 235290 2024-09-14T04:43:18.042 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.042 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:43:18.042 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.042 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:43:18.042 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.453718+0000", 2024-09-14T04:43:18.042 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.042 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.042 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.042 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:43:18.042 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.453718+0000", 2024-09-14T04:43:18.042 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.043 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.043 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.043 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:43:18.043 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.453720+0000", 2024-09-14T04:43:18.043 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9420000000000002e-06 2024-09-14T04:43:18.043 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.043 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.044 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:43:18.044 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.453726+0000", 2024-09-14T04:43:18.044 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3579999999999999e-06 2024-09-14T04:43:18.044 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.044 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.044 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:43:18.044 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.453740+0000", 2024-09-14T04:43:18.044 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3920000000000001e-05 2024-09-14T04:43:18.044 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.044 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.044 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:43:18.044 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.453747+0000", 2024-09-14T04:43:18.045 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.2559999999999996e-06 2024-09-14T04:43:18.045 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.045 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.045 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.045 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.453826+0000", 2024-09-14T04:43:18.045 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.8796999999999998e-05 2024-09-14T04:43:18.045 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.045 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.045 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:43:18.045 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.453949+0000", 2024-09-14T04:43:18.045 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012308600000000001 2024-09-14T04:43:18.045 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.045 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.045 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2", 2024-09-14T04:43:18.046 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.454067+0000", 2024-09-14T04:43:18.046 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011754100000000001 2024-09-14T04:43:18.046 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.046 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.046 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.046 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.578806+0000", 2024-09-14T04:43:18.046 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.124738944 2024-09-14T04:43:18.046 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.046 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.046 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:43:18.046 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.578822+0000", 2024-09-14T04:43:18.047 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6626e-05 2024-09-14T04:43:18.047 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.047 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.047 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:43:18.047 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.578857+0000", 2024-09-14T04:43:18.047 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.5210999999999999e-05 2024-09-14T04:43:18.047 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.047 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.047 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:43:18.047 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.578955+0000", 2024-09-14T04:43:18.047 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.7632000000000003e-05 2024-09-14T04:43:18.047 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.047 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:43:18.047 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.048 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.048 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.048 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:235292 53.6 53:63c53da4:::benchmark_data_smithi039_224949_object105199:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1357)", 2024-09-14T04:43:18.048 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:09.455108+0000", 2024-09-14T04:43:18.048 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 68.550488831999999, 2024-09-14T04:43:18.048 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.124119519, 2024-09-14T04:43:18.048 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:43:18.048 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:43:18.048 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:43:18.048 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:43:18.048 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:43:18.048 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:43:18.048 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 235292 2024-09-14T04:43:18.048 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.049 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:43:18.049 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.049 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:43:18.049 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.455108+0000", 2024-09-14T04:43:18.049 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.049 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.049 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.049 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:43:18.049 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.455108+0000", 2024-09-14T04:43:18.049 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.049 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.049 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.049 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:43:18.050 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.455110+0000", 2024-09-14T04:43:18.050 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.499e-06 2024-09-14T04:43:18.050 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.050 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.050 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:43:18.050 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.455115+0000", 2024-09-14T04:43:18.050 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.933e-06 2024-09-14T04:43:18.050 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.050 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.050 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:43:18.050 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.455127+0000", 2024-09-14T04:43:18.050 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2017e-05 2024-09-14T04:43:18.050 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.050 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.050 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:43:18.051 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.455134+0000", 2024-09-14T04:43:18.051 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9779999999999999e-06 2024-09-14T04:43:18.051 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.051 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.051 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.051 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.458949+0000", 2024-09-14T04:43:18.051 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.003814764 2024-09-14T04:43:18.051 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.051 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.051 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:43:18.051 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.459087+0000", 2024-09-14T04:43:18.051 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013837800000000001 2024-09-14T04:43:18.052 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.052 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.052 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2", 2024-09-14T04:43:18.052 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.459359+0000", 2024-09-14T04:43:18.052 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000272482 2024-09-14T04:43:18.052 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.052 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.052 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.053 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.578999+0000", 2024-09-14T04:43:18.053 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.119639959 2024-09-14T04:43:18.053 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.053 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.053 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:43:18.053 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.579018+0000", 2024-09-14T04:43:18.053 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8698999999999998e-05 2024-09-14T04:43:18.053 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.053 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.053 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:43:18.053 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.579100+0000", 2024-09-14T04:43:18.053 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.2046999999999996e-05 2024-09-14T04:43:18.053 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.054 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.054 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:43:18.054 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.579228+0000", 2024-09-14T04:43:18.054 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000127763 2024-09-14T04:43:18.054 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.054 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:43:18.054 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.054 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.054 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.054 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop_reply(client.45053.0:235290 53.6 e1357/1344 ondisk, result = 0)", 2024-09-14T04:43:18.054 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:09.458867+0000", 2024-09-14T04:43:18.054 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 68.546729796999998, 2024-09-14T04:43:18.054 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.119946039, 2024-09-14T04:43:18.054 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:43:18.054 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:43:18.055 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "started", 2024-09-14T04:43:18.055 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:43:18.055 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.055 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:43:18.055 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.458867+0000", 2024-09-14T04:43:18.055 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.055 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.055 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.055 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:43:18.055 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.458867+0000", 2024-09-14T04:43:18.055 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.055 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.055 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.056 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:43:18.056 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.458868+0000", 2024-09-14T04:43:18.056 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.2699999999999999e-07 2024-09-14T04:43:18.056 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.056 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.056 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:43:18.056 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.458871+0000", 2024-09-14T04:43:18.056 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.4970000000000001e-06 2024-09-14T04:43:18.056 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.056 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.056 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:43:18.056 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.458881+0000", 2024-09-14T04:43:18.056 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.5249999999999994e-06 2024-09-14T04:43:18.056 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.057 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.057 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:43:18.057 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.458887+0000", 2024-09-14T04:43:18.057 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1480000000000003e-06 2024-09-14T04:43:18.057 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.057 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.057 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.057 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.578794+0000", 2024-09-14T04:43:18.057 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.119907422 2024-09-14T04:43:18.057 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.057 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.057 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:43:18.057 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.578801+0000", 2024-09-14T04:43:18.057 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.6490000000000002e-06 2024-09-14T04:43:18.058 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.058 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.058 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:43:18.058 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.578813+0000", 2024-09-14T04:43:18.058 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2271e-05 2024-09-14T04:43:18.058 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.058 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:43:18.058 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.058 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.058 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.058 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:235296 53.a 53:53b4c80c:::benchmark_data_smithi039_224949_object105203:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1357)", 2024-09-14T04:43:18.058 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:09.459159+0000", 2024-09-14T04:43:18.059 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 68.546437928000003, 2024-09-14T04:43:18.059 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.129549253, 2024-09-14T04:43:18.059 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:43:18.059 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:43:18.059 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:43:18.059 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:43:18.059 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:43:18.059 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:43:18.059 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 235296 2024-09-14T04:43:18.059 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.059 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:43:18.059 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.059 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:43:18.059 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.459159+0000", 2024-09-14T04:43:18.060 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.060 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.060 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.060 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:43:18.060 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.459159+0000", 2024-09-14T04:43:18.060 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.060 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.060 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.060 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:43:18.060 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.459160+0000", 2024-09-14T04:43:18.060 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.164e-06 2024-09-14T04:43:18.060 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.060 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.060 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:43:18.061 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.459165+0000", 2024-09-14T04:43:18.061 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5449999999999997e-06 2024-09-14T04:43:18.061 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.061 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.061 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:43:18.061 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.459173+0000", 2024-09-14T04:43:18.061 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.2570000000000007e-06 2024-09-14T04:43:18.061 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.061 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.061 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:43:18.061 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.459180+0000", 2024-09-14T04:43:18.061 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1439999999999997e-06 2024-09-14T04:43:18.061 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.061 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.062 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.062 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.459316+0000", 2024-09-14T04:43:18.062 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013531700000000001 2024-09-14T04:43:18.062 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.062 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.062 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:43:18.062 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.459392+0000", 2024-09-14T04:43:18.062 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6427999999999994e-05 2024-09-14T04:43:18.062 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.062 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.062 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,5,6", 2024-09-14T04:43:18.062 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.575917+0000", 2024-09-14T04:43:18.062 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.11652535999999999 2024-09-14T04:43:18.062 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.062 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.063 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.063 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.579276+0000", 2024-09-14T04:43:18.063 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.003358738 2024-09-14T04:43:18.063 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.063 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.063 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.063 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.579323+0000", 2024-09-14T04:43:18.063 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6881000000000001e-05 2024-09-14T04:43:18.063 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.063 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.063 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:43:18.063 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.579544+0000", 2024-09-14T04:43:18.063 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00022113600000000001 2024-09-14T04:43:18.063 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.063 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.064 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.064 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.588387+0000", 2024-09-14T04:43:18.064 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0088430390000000005 2024-09-14T04:43:18.064 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.064 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.064 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:43:18.064 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.588452+0000", 2024-09-14T04:43:18.064 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.4764999999999994e-05 2024-09-14T04:43:18.064 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.064 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.064 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:43:18.064 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.588708+0000", 2024-09-14T04:43:18.064 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000256479 2024-09-14T04:43:18.064 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.065 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:43:18.065 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.065 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.065 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.065 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:235297 53.7 53:e46136f7:::benchmark_data_smithi039_224949_object105204:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1357)", 2024-09-14T04:43:18.065 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:09.459289+0000", 2024-09-14T04:43:18.065 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 68.546308404000001, 2024-09-14T04:43:18.065 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.125531848, 2024-09-14T04:43:18.065 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:43:18.065 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:43:18.065 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:43:18.065 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:43:18.065 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:43:18.066 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:43:18.066 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 235297 2024-09-14T04:43:18.066 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.066 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:43:18.066 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.066 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:43:18.066 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.459289+0000", 2024-09-14T04:43:18.066 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.066 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.066 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.066 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:43:18.066 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.459289+0000", 2024-09-14T04:43:18.066 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.066 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.067 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.067 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:43:18.067 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.459289+0000", 2024-09-14T04:43:18.067 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.2600000000000002e-07 2024-09-14T04:43:18.067 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.067 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.067 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:43:18.067 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.459293+0000", 2024-09-14T04:43:18.067 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.3500000000000001e-06 2024-09-14T04:43:18.067 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.067 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.067 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:43:18.067 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.459301+0000", 2024-09-14T04:43:18.067 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.5439999999999999e-06 2024-09-14T04:43:18.068 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.068 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.068 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:43:18.068 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.459308+0000", 2024-09-14T04:43:18.068 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1399999999999997e-06 2024-09-14T04:43:18.068 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.068 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.068 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.068 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.459351+0000", 2024-09-14T04:43:18.068 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3952999999999997e-05 2024-09-14T04:43:18.068 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.068 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.068 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:43:18.068 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.459428+0000", 2024-09-14T04:43:18.069 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6613999999999995e-05 2024-09-14T04:43:18.069 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.069 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.069 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 0,1,7", 2024-09-14T04:43:18.069 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.576023+0000", 2024-09-14T04:43:18.069 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.116595247 2024-09-14T04:43:18.069 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.069 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.069 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.069 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.579291+0000", 2024-09-14T04:43:18.069 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0032677439999999999 2024-09-14T04:43:18.069 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.069 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.069 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.070 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.582945+0000", 2024-09-14T04:43:18.070 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0036541910000000002 2024-09-14T04:43:18.070 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.070 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.070 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:43:18.070 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.582959+0000", 2024-09-14T04:43:18.070 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3543000000000001e-05 2024-09-14T04:43:18.070 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.070 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.070 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.070 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.584596+0000", 2024-09-14T04:43:18.070 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001637087 2024-09-14T04:43:18.070 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.070 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.071 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:43:18.071 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.584696+0000", 2024-09-14T04:43:18.071 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000100107 2024-09-14T04:43:18.071 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.071 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.071 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:43:18.071 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.584821+0000", 2024-09-14T04:43:18.071 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000124602 2024-09-14T04:43:18.071 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.071 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:43:18.071 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.071 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.071 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.071 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop_reply(client.45053.0:235292 53.6 e1357/1344 ondisk, result = 0)", 2024-09-14T04:43:18.071 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:09.461478+0000", 2024-09-14T04:43:18.072 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 68.544119190000004, 2024-09-14T04:43:18.072 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.117530069, 2024-09-14T04:43:18.072 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:43:18.072 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:43:18.072 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "started", 2024-09-14T04:43:18.072 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:43:18.072 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.072 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:43:18.072 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.461478+0000", 2024-09-14T04:43:18.072 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.072 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.072 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.072 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:43:18.073 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.461478+0000", 2024-09-14T04:43:18.073 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.073 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.073 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.073 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:43:18.073 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.461478+0000", 2024-09-14T04:43:18.073 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.06e-07 2024-09-14T04:43:18.073 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.073 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.073 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:43:18.073 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.461481+0000", 2024-09-14T04:43:18.073 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.9399999999999998e-06 2024-09-14T04:43:18.073 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.073 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.074 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:43:18.074 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.575823+0000", 2024-09-14T04:43:18.074 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.114341712 2024-09-14T04:43:18.074 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.074 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.074 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:43:18.074 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.575832+0000", 2024-09-14T04:43:18.074 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.0189999999999995e-06 2024-09-14T04:43:18.074 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.074 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.074 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.074 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.578986+0000", 2024-09-14T04:43:18.074 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0031541780000000001 2024-09-14T04:43:18.074 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.074 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.075 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:43:18.075 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.578994+0000", 2024-09-14T04:43:18.075 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.2439999999999996e-06 2024-09-14T04:43:18.075 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.075 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.075 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:43:18.075 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.579008+0000", 2024-09-14T04:43:18.075 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.447e-05 2024-09-14T04:43:18.075 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.075 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:43:18.075 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.075 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.075 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.075 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:235508 53.7 53:e7621651:::benchmark_data_smithi039_224949_object105415:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1357)", 2024-09-14T04:43:18.076 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:09.845516+0000", 2024-09-14T04:43:18.076 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 68.160080880999999, 2024-09-14T04:43:18.076 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.109532642, 2024-09-14T04:43:18.076 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:43:18.076 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:43:18.076 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:43:18.076 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:43:18.076 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:43:18.076 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:43:18.076 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 235508 2024-09-14T04:43:18.076 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.076 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:43:18.076 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.076 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:43:18.077 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.845516+0000", 2024-09-14T04:43:18.077 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.077 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.077 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.077 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:43:18.077 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.845516+0000", 2024-09-14T04:43:18.077 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.077 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.077 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.077 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:43:18.077 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.845518+0000", 2024-09-14T04:43:18.077 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5200000000000001e-06 2024-09-14T04:43:18.077 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.077 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.077 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:43:18.078 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.845521+0000", 2024-09-14T04:43:18.078 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.4599999999999999e-06 2024-09-14T04:43:18.078 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.078 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.078 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:43:18.078 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.845534+0000", 2024-09-14T04:43:18.078 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2295e-05 2024-09-14T04:43:18.078 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.078 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.078 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:43:18.078 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.845541+0000", 2024-09-14T04:43:18.078 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9030000000000004e-06 2024-09-14T04:43:18.078 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.078 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.078 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.079 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.845688+0000", 2024-09-14T04:43:18.079 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000146565 2024-09-14T04:43:18.079 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.079 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.079 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:43:18.079 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.845798+0000", 2024-09-14T04:43:18.079 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00010974199999999999 2024-09-14T04:43:18.079 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.079 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.079 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 0,1,7", 2024-09-14T04:43:18.079 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.845914+0000", 2024-09-14T04:43:18.079 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000116477 2024-09-14T04:43:18.079 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.079 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.080 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:43:18.080 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.849444+0000", 2024-09-14T04:43:18.080 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0035297290000000001 2024-09-14T04:43:18.080 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.080 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.081 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.081 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.851693+0000", 2024-09-14T04:43:18.081 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0022486670000000002 2024-09-14T04:43:18.081 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.081 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.081 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.081 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.852781+0000", 2024-09-14T04:43:18.081 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0010883030000000001 2024-09-14T04:43:18.081 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.081 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.081 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.081 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.954869+0000", 2024-09-14T04:43:18.081 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.10208782600000001 2024-09-14T04:43:18.081 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.082 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.082 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:43:18.082 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.954933+0000", 2024-09-14T04:43:18.082 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.4604999999999996e-05 2024-09-14T04:43:18.082 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.082 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.082 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:43:18.082 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:09.955049+0000", 2024-09-14T04:43:18.082 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011555 2024-09-14T04:43:18.082 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.082 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:43:18.082 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.082 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.082 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.083 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:240105 53.7 53:e64188eb:::benchmark_data_smithi039_224949_object110012:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1360)", 2024-09-14T04:43:18.083 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:15.831513+0000", 2024-09-14T04:43:18.083 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 62.174084145999998, 2024-09-14T04:43:18.083 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.10955073, 2024-09-14T04:43:18.083 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:43:18.083 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:43:18.083 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:43:18.083 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:43:18.083 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:43:18.083 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:43:18.083 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 240105 2024-09-14T04:43:18.083 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.083 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:43:18.083 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.084 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:43:18.084 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:15.831513+0000", 2024-09-14T04:43:18.084 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.084 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.084 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.084 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:43:18.084 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:15.831513+0000", 2024-09-14T04:43:18.084 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.084 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.084 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.084 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:43:18.084 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:15.831514+0000", 2024-09-14T04:43:18.084 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.079e-06 2024-09-14T04:43:18.084 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.085 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.085 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:43:18.085 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:15.831519+0000", 2024-09-14T04:43:18.085 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7010000000000004e-06 2024-09-14T04:43:18.085 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.085 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.085 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:43:18.085 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:15.831531+0000", 2024-09-14T04:43:18.085 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1916000000000001e-05 2024-09-14T04:43:18.085 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.085 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.085 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:43:18.085 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:15.831539+0000", 2024-09-14T04:43:18.085 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0360000000000006e-06 2024-09-14T04:43:18.086 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.086 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.086 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.086 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:15.835086+0000", 2024-09-14T04:43:18.086 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0035470469999999998 2024-09-14T04:43:18.086 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.086 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.086 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:43:18.086 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:15.835207+0000", 2024-09-14T04:43:18.086 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000120999 2024-09-14T04:43:18.086 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.086 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.086 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 0,1,7", 2024-09-14T04:43:18.086 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:15.835344+0000", 2024-09-14T04:43:18.086 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000136799 2024-09-14T04:43:18.087 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.087 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.087 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.087 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:15.841925+0000", 2024-09-14T04:43:18.087 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.006581038 2024-09-14T04:43:18.087 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.087 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.087 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:43:18.087 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:15.842207+0000", 2024-09-14T04:43:18.087 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00028274499999999999 2024-09-14T04:43:18.087 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.087 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.087 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.087 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:15.846163+0000", 2024-09-14T04:43:18.088 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0039555889999999998 2024-09-14T04:43:18.088 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.088 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.088 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.088 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:15.938295+0000", 2024-09-14T04:43:18.088 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.092132512999999999 2024-09-14T04:43:18.088 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.088 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.088 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:43:18.088 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:15.940641+0000", 2024-09-14T04:43:18.088 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0023456449999999999 2024-09-14T04:43:18.088 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.088 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.088 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:43:18.089 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:15.941064+0000", 2024-09-14T04:43:18.089 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00042262300000000001 2024-09-14T04:43:18.089 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.089 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:43:18.089 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.089 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.089 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.089 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242867 53.a 53:54ad9484:::benchmark_data_smithi039_224949_object112774:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1363)", 2024-09-14T04:43:18.089 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:20.495372+0000", 2024-09-14T04:43:18.089 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 57.510225247999998, 2024-09-14T04:43:18.089 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.997320294, 2024-09-14T04:43:18.089 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:43:18.089 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:43:18.089 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:43:18.090 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:43:18.090 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:43:18.090 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:43:18.090 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242867 2024-09-14T04:43:18.090 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.090 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:43:18.090 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.090 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:43:18.090 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.495372+0000", 2024-09-14T04:43:18.090 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.090 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.090 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.090 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:43:18.090 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.495372+0000", 2024-09-14T04:43:18.090 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.091 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.091 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.091 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:43:18.091 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.495373+0000", 2024-09-14T04:43:18.091 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.850000000000001e-07 2024-09-14T04:43:18.091 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.091 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.091 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:43:18.091 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.495377+0000", 2024-09-14T04:43:18.091 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5059999999999997e-06 2024-09-14T04:43:18.091 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.091 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.091 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:43:18.091 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.495390+0000", 2024-09-14T04:43:18.092 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2498e-05 2024-09-14T04:43:18.092 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.092 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.092 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:43:18.092 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.495397+0000", 2024-09-14T04:43:18.092 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.2060000000000001e-06 2024-09-14T04:43:18.092 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.092 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.092 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.092 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.495443+0000", 2024-09-14T04:43:18.092 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6139e-05 2024-09-14T04:43:18.092 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.092 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.093 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:43:18.093 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.495449+0000", 2024-09-14T04:43:18.093 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5949999999999998e-06 2024-09-14T04:43:18.093 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.093 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.093 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.093 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.479903+0000", 2024-09-14T04:43:18.093 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.98445407699999998 2024-09-14T04:43:18.093 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.093 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.093 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:43:18.093 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.480015+0000", 2024-09-14T04:43:18.093 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011235600000000001 2024-09-14T04:43:18.093 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.094 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.094 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:43:18.094 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.480135+0000", 2024-09-14T04:43:18.094 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:17 smithi039 ceph-mon[106470]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:43:18.094 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:17 smithi039 ceph-mon[111235]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:43:18.095 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000119462 2024-09-14T04:43:18.095 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.095 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.095 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.095 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.484482+0000", 2024-09-14T04:43:18.095 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0043470779999999999 2024-09-14T04:43:18.095 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.095 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.095 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.095 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.484532+0000", 2024-09-14T04:43:18.095 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9981999999999997e-05 2024-09-14T04:43:18.096 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.096 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.096 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:43:18.096 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.492566+0000", 2024-09-14T04:43:18.096 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0080345810000000007 2024-09-14T04:43:18.096 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.096 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.096 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:43:18.096 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.492646+0000", 2024-09-14T04:43:18.096 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0039999999999999e-05 2024-09-14T04:43:18.096 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.096 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.096 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:43:18.096 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.492692+0000", 2024-09-14T04:43:18.097 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5788999999999998e-05 2024-09-14T04:43:18.097 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.097 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:43:18.097 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.097 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.097 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.097 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242882 53.a 53:5478a5c5:::benchmark_data_smithi039_224949_object112789:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1363)", 2024-09-14T04:43:18.097 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:20.512060+0000", 2024-09-14T04:43:18.097 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 57.493537394999997, 2024-09-14T04:43:18.097 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0040218409999999, 2024-09-14T04:43:18.097 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:43:18.097 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:43:18.097 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:43:18.097 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:43:18.098 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:43:18.098 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:43:18.098 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242882 2024-09-14T04:43:18.098 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.098 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:43:18.098 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.098 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:43:18.098 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.512060+0000", 2024-09-14T04:43:18.098 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.098 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.098 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.098 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:43:18.098 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.512060+0000", 2024-09-14T04:43:18.098 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.099 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.099 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.099 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:43:18.099 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.512061+0000", 2024-09-14T04:43:18.099 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6869999999999999e-06 2024-09-14T04:43:18.099 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.099 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.099 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:43:18.099 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.512067+0000", 2024-09-14T04:43:18.099 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.993e-06 2024-09-14T04:43:18.099 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.099 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.099 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:43:18.099 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.512078+0000", 2024-09-14T04:43:18.100 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0913999999999999e-05 2024-09-14T04:43:18.100 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.100 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.100 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:43:18.100 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.512085+0000", 2024-09-14T04:43:18.100 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7109999999999998e-06 2024-09-14T04:43:18.100 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.100 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.100 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.100 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.512156+0000", 2024-09-14T04:43:18.100 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.0850000000000001e-05 2024-09-14T04:43:18.100 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.100 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.100 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:43:18.101 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.512164+0000", 2024-09-14T04:43:18.101 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.4530000000000002e-06 2024-09-14T04:43:18.101 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.101 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.101 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.101 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.484560+0000", 2024-09-14T04:43:18.101 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.972395491 2024-09-14T04:43:18.101 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.101 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.101 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:43:18.101 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.484663+0000", 2024-09-14T04:43:18.101 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00010302300000000001 2024-09-14T04:43:18.101 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.101 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.102 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:43:18.102 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.484869+0000", 2024-09-14T04:43:18.102 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000206104 2024-09-14T04:43:18.102 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.102 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.102 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.102 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.492736+0000", 2024-09-14T04:43:18.102 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0078666750000000001 2024-09-14T04:43:18.102 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.102 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.102 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.102 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.492783+0000", 2024-09-14T04:43:18.102 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7630999999999998e-05 2024-09-14T04:43:18.102 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.102 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.103 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:43:18.103 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516001+0000", 2024-09-14T04:43:18.103 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.023217924000000001 2024-09-14T04:43:18.103 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.103 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.103 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:43:18.103 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516046+0000", 2024-09-14T04:43:18.103 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4833999999999998e-05 2024-09-14T04:43:18.103 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.103 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.103 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:43:18.103 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516082+0000", 2024-09-14T04:43:18.103 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.5550999999999999e-05 2024-09-14T04:43:18.104 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.104 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:43:18.104 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.104 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.104 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.104 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242920 53.a 53:5584c2e0:::benchmark_data_smithi039_224949_object112827:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1363)", 2024-09-14T04:43:18.104 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:20.566563+0000", 2024-09-14T04:43:18.104 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 57.439034440999997, 2024-09-14T04:43:18.104 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.94971936199999996, 2024-09-14T04:43:18.104 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:43:18.104 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:43:18.104 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:43:18.104 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:43:18.104 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:43:18.105 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:43:18.105 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242920 2024-09-14T04:43:18.105 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.105 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:43:18.105 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.105 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:43:18.105 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.566563+0000", 2024-09-14T04:43:18.105 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.105 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.105 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.105 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:43:18.105 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.566563+0000", 2024-09-14T04:43:18.105 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.105 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.106 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.106 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:43:18.106 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.566564+0000", 2024-09-14T04:43:18.106 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4890000000000001e-06 2024-09-14T04:43:18.106 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.106 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.106 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:43:18.106 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.566569+0000", 2024-09-14T04:43:18.106 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4750000000000004e-06 2024-09-14T04:43:18.106 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.106 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.106 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:43:18.106 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.566579+0000", 2024-09-14T04:43:18.106 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0421000000000001e-05 2024-09-14T04:43:18.107 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.107 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.107 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:43:18.107 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.566586+0000", 2024-09-14T04:43:18.107 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7449999999999998e-06 2024-09-14T04:43:18.107 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.107 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.107 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.107 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.566678+0000", 2024-09-14T04:43:18.107 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.2319999999999997e-05 2024-09-14T04:43:18.107 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.107 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.107 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:43:18.107 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.566686+0000", 2024-09-14T04:43:18.108 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9400000000000002e-06 2024-09-14T04:43:18.108 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.108 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.108 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.108 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.488450+0000", 2024-09-14T04:43:18.108 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.92176433300000005 2024-09-14T04:43:18.108 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.108 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.108 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:43:18.108 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.488608+0000", 2024-09-14T04:43:18.108 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015782400000000001 2024-09-14T04:43:18.108 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.108 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.108 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:43:18.109 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.488674+0000", 2024-09-14T04:43:18.109 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.5457999999999996e-05 2024-09-14T04:43:18.109 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.109 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.109 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.109 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.492855+0000", 2024-09-14T04:43:18.109 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0041807909999999997 2024-09-14T04:43:18.109 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.109 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.109 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.109 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.492909+0000", 2024-09-14T04:43:18.109 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4629999999999997e-05 2024-09-14T04:43:18.109 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.109 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.110 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:43:18.110 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516207+0000", 2024-09-14T04:43:18.110 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.023297549000000001 2024-09-14T04:43:18.110 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.110 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.110 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:43:18.110 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516246+0000", 2024-09-14T04:43:18.110 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.9242000000000001e-05 2024-09-14T04:43:18.110 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.110 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.110 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:43:18.110 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516282+0000", 2024-09-14T04:43:18.110 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.6145e-05 2024-09-14T04:43:18.110 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.111 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:43:18.111 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.111 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.111 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.111 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242925 53.a 53:5601ec35:::benchmark_data_smithi039_224949_object112832:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1363)", 2024-09-14T04:43:18.111 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:20.570988+0000", 2024-09-14T04:43:18.111 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 57.434609426000002, 2024-09-14T04:43:18.111 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.94542205099999999, 2024-09-14T04:43:18.111 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:43:18.111 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:43:18.111 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:43:18.111 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:43:18.111 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:43:18.111 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:43:18.112 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242925 2024-09-14T04:43:18.112 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.112 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:43:18.112 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.112 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:43:18.112 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.570988+0000", 2024-09-14T04:43:18.112 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.112 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.112 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.112 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:43:18.112 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.570988+0000", 2024-09-14T04:43:18.112 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.112 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.113 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.113 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:43:18.113 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.570990+0000", 2024-09-14T04:43:18.113 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8870000000000001e-06 2024-09-14T04:43:18.113 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.113 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.113 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:43:18.113 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.570995+0000", 2024-09-14T04:43:18.113 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7799999999999997e-06 2024-09-14T04:43:18.113 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.113 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.113 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:43:18.113 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.571008+0000", 2024-09-14T04:43:18.113 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2683e-05 2024-09-14T04:43:18.114 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.114 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.114 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:43:18.114 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.571016+0000", 2024-09-14T04:43:18.114 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.1880000000000002e-06 2024-09-14T04:43:18.114 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.114 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.114 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.114 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.571101+0000", 2024-09-14T04:43:18.114 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.4460000000000001e-05 2024-09-14T04:43:18.114 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.114 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.114 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:43:18.114 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.571110+0000", 2024-09-14T04:43:18.115 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.7700000000000007e-06 2024-09-14T04:43:18.115 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.115 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.115 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.115 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.492959+0000", 2024-09-14T04:43:18.115 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.92184951400000004 2024-09-14T04:43:18.115 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.115 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.115 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:43:18.115 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.493043+0000", 2024-09-14T04:43:18.115 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.3777999999999996e-05 2024-09-14T04:43:18.115 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.115 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.115 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:43:18.116 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.493120+0000", 2024-09-14T04:43:18.116 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7489000000000003e-05 2024-09-14T04:43:18.116 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.116 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.116 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.116 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.500548+0000", 2024-09-14T04:43:18.116 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0074275349999999999 2024-09-14T04:43:18.116 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.116 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.116 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.116 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.500616+0000", 2024-09-14T04:43:18.116 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7871000000000001e-05 2024-09-14T04:43:18.116 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.116 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.117 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:43:18.117 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516288+0000", 2024-09-14T04:43:18.117 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.015672654000000001 2024-09-14T04:43:18.117 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.117 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.117 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:43:18.117 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516331+0000", 2024-09-14T04:43:18.117 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2698999999999999e-05 2024-09-14T04:43:18.117 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.117 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.117 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:43:18.117 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516410+0000", 2024-09-14T04:43:18.117 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.8743000000000001e-05 2024-09-14T04:43:18.117 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.118 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:43:18.118 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.118 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.118 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.118 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242952 53.a 53:58b7a1db:::benchmark_data_smithi039_224949_object112859:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1363)", 2024-09-14T04:43:18.118 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:20.604696+0000", 2024-09-14T04:43:18.118 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 57.400901615000002, 2024-09-14T04:43:18.118 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.91180369299999997, 2024-09-14T04:43:18.118 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:43:18.118 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:43:18.118 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:43:18.118 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:43:18.118 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:43:18.118 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:43:18.118 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242952 2024-09-14T04:43:18.119 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.119 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:43:18.119 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.119 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:43:18.119 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.604696+0000", 2024-09-14T04:43:18.119 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.119 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.119 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.119 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:43:18.119 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.604696+0000", 2024-09-14T04:43:18.119 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.119 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.119 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.119 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:43:18.120 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.604697+0000", 2024-09-14T04:43:18.120 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8789999999999999e-06 2024-09-14T04:43:18.120 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.120 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.120 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:43:18.120 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.604703+0000", 2024-09-14T04:43:18.120 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7209999999999999e-06 2024-09-14T04:43:18.120 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.120 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.120 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:43:18.120 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.604721+0000", 2024-09-14T04:43:18.120 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7353000000000001e-05 2024-09-14T04:43:18.120 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.121 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.121 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:43:18.121 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.604728+0000", 2024-09-14T04:43:18.121 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7139999999999993e-06 2024-09-14T04:43:18.121 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.121 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.121 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.121 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.604805+0000", 2024-09-14T04:43:18.121 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6718999999999995e-05 2024-09-14T04:43:18.121 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.121 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.121 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:43:18.121 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.604814+0000", 2024-09-14T04:43:18.121 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.0589999999999992e-06 2024-09-14T04:43:18.121 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.122 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.122 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.122 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.496130+0000", 2024-09-14T04:43:18.122 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.89131583400000003 2024-09-14T04:43:18.122 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:18.123 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.123 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.123 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:43:18.123 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.496260+0000", 2024-09-14T04:43:18.123 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012988 2024-09-14T04:43:18.123 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.123 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.123 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:43:18.124 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.496352+0000", 2024-09-14T04:43:18.124 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.2159999999999999e-05 2024-09-14T04:43:18.124 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.124 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.124 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.124 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.500654+0000", 2024-09-14T04:43:18.124 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0043017810000000002 2024-09-14T04:43:18.124 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.124 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.124 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.124 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.500688+0000", 2024-09-14T04:43:18.124 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.4146000000000001e-05 2024-09-14T04:43:18.124 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.125 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.125 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:43:18.125 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516417+0000", 2024-09-14T04:43:18.125 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.015729097000000001 2024-09-14T04:43:18.125 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.125 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.125 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:43:18.125 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516462+0000", 2024-09-14T04:43:18.125 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5104e-05 2024-09-14T04:43:18.125 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.125 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.125 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:43:18.125 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516499+0000", 2024-09-14T04:43:18.125 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.7246000000000002e-05 2024-09-14T04:43:18.126 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.126 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:43:18.126 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.126 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.126 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.126 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242956 53.a 53:53d3e9bf:::benchmark_data_smithi039_224949_object112863:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1363)", 2024-09-14T04:43:18.126 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:20.610137+0000", 2024-09-14T04:43:18.126 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 57.395459903999999, 2024-09-14T04:43:18.126 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.90651257799999996, 2024-09-14T04:43:18.126 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:43:18.126 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:43:18.126 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:43:18.126 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:43:18.127 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:43:18.127 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:43:18.127 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242956 2024-09-14T04:43:18.127 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.127 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:43:18.127 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.127 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:43:18.127 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.610137+0000", 2024-09-14T04:43:18.127 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.127 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.127 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.127 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:43:18.127 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.610137+0000", 2024-09-14T04:43:18.127 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.128 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.128 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.128 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:43:18.128 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.610139+0000", 2024-09-14T04:43:18.128 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9520000000000001e-06 2024-09-14T04:43:18.128 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.128 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.128 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:43:18.128 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.610148+0000", 2024-09-14T04:43:18.128 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.6459999999999999e-06 2024-09-14T04:43:18.128 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.128 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.128 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:43:18.128 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.610160+0000", 2024-09-14T04:43:18.129 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2574e-05 2024-09-14T04:43:18.129 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.129 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.129 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:43:18.129 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.610168+0000", 2024-09-14T04:43:18.129 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6669999999999996e-06 2024-09-14T04:43:18.129 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.129 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.129 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.129 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.610230+0000", 2024-09-14T04:43:18.129 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1472999999999999e-05 2024-09-14T04:43:18.129 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.129 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.129 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:43:18.130 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.610239+0000", 2024-09-14T04:43:18.130 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.0880000000000001e-06 2024-09-14T04:43:18.130 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.130 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.130 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.130 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.500716+0000", 2024-09-14T04:43:18.130 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.89047721599999996 2024-09-14T04:43:18.130 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.130 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.130 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:43:18.130 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.500820+0000", 2024-09-14T04:43:18.130 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00010407 2024-09-14T04:43:18.130 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.130 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.131 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:43:18.131 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.500928+0000", 2024-09-14T04:43:18.131 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000107909 2024-09-14T04:43:18.131 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.131 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.131 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.131 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.512156+0000", 2024-09-14T04:43:18.131 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.011227849 2024-09-14T04:43:18.131 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.131 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.131 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.131 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.512195+0000", 2024-09-14T04:43:18.131 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.9739e-05 2024-09-14T04:43:18.131 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.131 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.132 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:43:18.132 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516505+0000", 2024-09-14T04:43:18.132 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0043095210000000002 2024-09-14T04:43:18.132 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.132 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.132 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:43:18.132 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516541+0000", 2024-09-14T04:43:18.132 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.5911999999999998e-05 2024-09-14T04:43:18.132 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.132 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.132 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:43:18.132 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516650+0000", 2024-09-14T04:43:18.132 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000108962 2024-09-14T04:43:18.133 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.133 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:43:18.133 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.133 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.133 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.133 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242984 53.a 53:5753476b:::benchmark_data_smithi039_224949_object112891:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1363)", 2024-09-14T04:43:18.133 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:20.654008+0000", 2024-09-14T04:43:18.133 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 57.351589322000002, 2024-09-14T04:43:18.133 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.86274647400000004, 2024-09-14T04:43:18.133 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:43:18.133 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:43:18.133 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:43:18.133 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:43:18.133 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:43:18.133 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:43:18.134 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242984 2024-09-14T04:43:18.134 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.134 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:43:18.134 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.134 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:43:18.136 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.654008+0000", 2024-09-14T04:43:18.136 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.136 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.137 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.137 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:43:18.137 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.654008+0000", 2024-09-14T04:43:18.137 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.137 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.137 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.137 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:43:18.137 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.654010+0000", 2024-09-14T04:43:18.137 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.753e-06 2024-09-14T04:43:18.137 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.137 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.137 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:43:18.137 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.654015+0000", 2024-09-14T04:43:18.137 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4160000000000003e-06 2024-09-14T04:43:18.138 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.138 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.138 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:43:18.138 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.654031+0000", 2024-09-14T04:43:18.138 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6458000000000001e-05 2024-09-14T04:43:18.138 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.138 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.138 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:43:18.138 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.654039+0000", 2024-09-14T04:43:18.138 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1019999999999999e-06 2024-09-14T04:43:18.138 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.138 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.138 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.138 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.654110+0000", 2024-09-14T04:43:18.139 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1265000000000003e-05 2024-09-14T04:43:18.139 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.139 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.139 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:43:18.139 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.654116+0000", 2024-09-14T04:43:18.139 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6910000000000002e-06 2024-09-14T04:43:18.139 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.139 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.139 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.139 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.503852+0000", 2024-09-14T04:43:18.139 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.84973631199999999 2024-09-14T04:43:18.139 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.139 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.139 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:43:18.140 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.503999+0000", 2024-09-14T04:43:18.140 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014705799999999999 2024-09-14T04:43:18.140 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.140 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.140 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:43:18.140 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.504119+0000", 2024-09-14T04:43:18.140 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000119982 2024-09-14T04:43:18.140 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.140 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.140 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.140 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.512233+0000", 2024-09-14T04:43:18.140 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0081145850000000005 2024-09-14T04:43:18.140 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.141 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.141 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.141 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.512271+0000", 2024-09-14T04:43:18.141 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.7440000000000001e-05 2024-09-14T04:43:18.141 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.141 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.141 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:43:18.141 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516656+0000", 2024-09-14T04:43:18.141 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0043848259999999997 2024-09-14T04:43:18.141 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.141 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.141 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:43:18.141 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516720+0000", 2024-09-14T04:43:18.141 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3862000000000006e-05 2024-09-14T04:43:18.141 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.142 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.142 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:43:18.142 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516754+0000", 2024-09-14T04:43:18.142 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.4724000000000002e-05 2024-09-14T04:43:18.142 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.142 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:43:18.142 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.142 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.142 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.142 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242992 53.a 53:5da6abf3:::benchmark_data_smithi039_224949_object112899:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1363)", 2024-09-14T04:43:18.142 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:20.682771+0000", 2024-09-14T04:43:18.142 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 57.322826169999999, 2024-09-14T04:43:18.142 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.83649015400000004, 2024-09-14T04:43:18.142 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:43:18.143 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:43:18.143 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:43:18.143 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:43:18.143 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:43:18.143 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:43:18.143 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242992 2024-09-14T04:43:18.143 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.143 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:43:18.143 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.144 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:43:18.144 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.682771+0000", 2024-09-14T04:43:18.144 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.144 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.144 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.144 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:43:18.144 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.682771+0000", 2024-09-14T04:43:18.144 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:43:18.144 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.144 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.144 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:43:18.145 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.682772+0000", 2024-09-14T04:43:18.145 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.271e-06 2024-09-14T04:43:18.145 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.145 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.145 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:43:18.145 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.682777+0000", 2024-09-14T04:43:18.145 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2849999999999996e-06 2024-09-14T04:43:18.145 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.145 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.145 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:43:18.145 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.682787+0000", 2024-09-14T04:43:18.145 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0787e-05 2024-09-14T04:43:18.145 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.145 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.145 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:43:18.146 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.682793+0000", 2024-09-14T04:43:18.146 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3990000000000003e-06 2024-09-14T04:43:18.146 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.146 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.146 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.146 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.682856+0000", 2024-09-14T04:43:18.146 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2890000000000003e-05 2024-09-14T04:43:18.146 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.146 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.146 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:43:18.146 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.682863+0000", 2024-09-14T04:43:18.146 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9600000000000003e-06 2024-09-14T04:43:18.146 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.146 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.147 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:43:18.147 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.508101+0000", 2024-09-14T04:43:18.147 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.82523816900000002 2024-09-14T04:43:18.147 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.147 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.147 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:43:18.147 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.508347+0000", 2024-09-14T04:43:18.147 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000246629 2024-09-14T04:43:18.147 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.147 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.147 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:43:18.147 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.508454+0000", 2024-09-14T04:43:18.147 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000106424 2024-09-14T04:43:18.147 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.147 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.148 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.148 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.512305+0000", 2024-09-14T04:43:18.148 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.003851508 2024-09-14T04:43:18.148 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.148 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.148 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:43:18.148 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516195+0000", 2024-09-14T04:43:18.148 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0038899450000000001 2024-09-14T04:43:18.148 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.148 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.148 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:43:18.148 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516761+0000", 2024-09-14T04:43:18.148 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00056550800000000005 2024-09-14T04:43:18.148 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.149 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.149 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:43:18.149 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516900+0000", 2024-09-14T04:43:18.149 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000139639 2024-09-14T04:43:18.149 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:43:18.149 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:43:18.149 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:43:18.149 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.519261+0000", 2024-09-14T04:43:18.149 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0023607400000000001 2024-09-14T04:43:18.149 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.149 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:43:18.149 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.149 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:43:18.149 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:43:18.149 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:43:18.187 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:18.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:18.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:18.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:17 smithi129 ceph-mon[101316]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:43:18.289 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:18.322 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:18.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:18.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:18.423 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:18.459 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:18.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:18.461 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:18.532 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 1 16 3552 3536 220.991 221 0.00292246 0.00451127 2024-09-14T04:43:18.562 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:18.604 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:18.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:18.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:18.706 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:18.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:18.746 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:18.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:18.847 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:18.888 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:18.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:18.889 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:18.922 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [4] live_osds: [0, 1, 2, 3, 5, 6, 7] 2024-09-14T04:43:18.922 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:43:18.923 INFO:tasks.thrashosds.thrasher:inject_pause on osd.2 2024-09-14T04:43:18.923 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-14T04:43:18.923 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-14T04:43:18.923 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.2 config set filestore_inject_stall 3 2024-09-14T04:43:18.969 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:18 smithi039 ceph-mon[106470]: pgmap v2368: 99 pgs: 6 peering, 93 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:43:18.969 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:18 smithi039 ceph-mon[111235]: pgmap v2368: 99 pgs: 6 peering, 93 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:43:18.970 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:43:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:43:18] "GET /metrics HTTP/1.1" 200 36691 "" "Prometheus/2.51.0" 2024-09-14T04:43:18.990 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:19.035 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:19.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:19.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:19.137 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:19.167 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:19.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:19.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:19.269 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:19.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:18 smithi129 ceph-mon[101316]: pgmap v2368: 99 pgs: 6 peering, 93 active+clean; 582 KiB data, 4.1 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:43:19.312 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:19.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:19.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:19.414 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:19.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:19.467 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:19.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:19.533 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 2 16 6880 6864 214.486 208 0.00338398 0.00465413 2024-09-14T04:43:19.568 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:19.609 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:19.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:19.610 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:19.711 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:19.751 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:19.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:19.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:19.853 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:19.885 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:19.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:19.887 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:19.987 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:20.018 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:20.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:20.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:20.120 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:20.153 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:20.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:20.155 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:20.256 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:20.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:19 smithi129 ceph-mon[101316]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 4 pgs peering) 2024-09-14T04:43:20.274 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:19 smithi039 ceph-mon[106470]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 4 pgs peering) 2024-09-14T04:43:20.274 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:19 smithi039 ceph-mon[111235]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 4 pgs peering) 2024-09-14T04:43:20.287 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:20.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:20.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:20.317 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-14T04:43:20.389 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:20.433 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:20.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:20.434 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:20.533 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 3 16 10161 10145 211.339 205.062 0.00511802 0.0047255 2024-09-14T04:43:20.535 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:20.574 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:20.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:20.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:20.641 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:43:20.675 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:20.719 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:20.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:20.720 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:20.821 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:20.843 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:20.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:20.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:20.945 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:20.985 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:20.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:20.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:21.087 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:21.109 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:21.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:21.110 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:21.163 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:20 smithi039 ceph-mon[106470]: pgmap v2369: 99 pgs: 99 active+clean; 50 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.9 MiB/s wr, 109 op/s 2024-09-14T04:43:21.163 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:20 smithi039 ceph-mon[111235]: pgmap v2369: 99 pgs: 99 active+clean; 50 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.9 MiB/s wr, 109 op/s 2024-09-14T04:43:21.211 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:21.243 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:21.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:21.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:21.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:20 smithi129 ceph-mon[101316]: pgmap v2369: 99 pgs: 99 active+clean; 50 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 6.9 MiB/s wr, 109 op/s 2024-09-14T04:43:21.345 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:21.361 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:43:21.361 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:43:21.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:21.389 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:21.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:21.398 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:43:21.398 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:43:21.435 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:43:21.436 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:43:21.480 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:43:21.481 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:43:21.481 INFO:teuthology.orchestra.run.smithi039.stderr:Error ENXIO: problem getting command descriptions from osd.4 2024-09-14T04:43:21.490 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:21.516 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:43:21.516 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:43:21.529 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:21.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:21.530 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:21.533 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 4 16 12544 12528 195.735 148.938 0.00320324 0.00509821 2024-09-14T04:43:21.550 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:43:21.550 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:43:21.584 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:43:21.584 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:43:21.631 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:21.661 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:21.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:21.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:21.762 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:21.803 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:21.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:21.804 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:21.905 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:21.942 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:21.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:21.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:22.045 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:22.071 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:22.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:22.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:22.173 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:22.203 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:21 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3712602940' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:43:22.211 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:22.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:22.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:22.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:21 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3712602940' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:43:22.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:21 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3712602940' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:43:22.314 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:22.345 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:22.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:22.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:22.447 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:22.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:22.488 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:22.489 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:22.533 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 5 16 15448 15432 192.885 181.5 0.00366793 0.00517724 2024-09-14T04:43:22.590 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:22.625 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:22.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:22.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:22.726 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:22.764 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:22.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:22.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:22.813 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:43:22.814 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:43:22.866 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:22.904 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:22.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:22.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:23.006 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:23.038 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:23.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:23.039 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:23.140 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:23.176 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:23.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:23.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:23.278 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:23.287 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:43:23.307 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:23.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:23.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:23.409 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:23.451 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:23.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:23.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:23.512 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:23 smithi129 ceph-mon[101316]: pgmap v2370: 99 pgs: 99 active+clean; 144 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 17 MiB/s wr, 278 op/s 2024-09-14T04:43:23.513 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:23 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:43:23.513 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:43:23.533 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 6 16 18254 18238 189.963 175.375 0.00463487 0.00525799 2024-09-14T04:43:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:23 smithi039 ceph-mon[106470]: pgmap v2370: 99 pgs: 99 active+clean; 144 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 17 MiB/s wr, 278 op/s 2024-09-14T04:43:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:23 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:43:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:43:23.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:23 smithi039 ceph-mon[111235]: pgmap v2370: 99 pgs: 99 active+clean; 144 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 17 MiB/s wr, 278 op/s 2024-09-14T04:43:23.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:23 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:43:23.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:43:23.553 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:23.583 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:23.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:23.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:23.685 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:23.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:23.725 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:23.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:23.804 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:43:23.804 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:43:23.826 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:23.863 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:23.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:23.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:23.965 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:23.994 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:23.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:23.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:24.096 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:24.128 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:24.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:24.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:24.230 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:24.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:24.263 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:24.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:24.364 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:24.393 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:24.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:24.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:24.495 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:24.530 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:24.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:24.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:24.533 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 7 16 20804 20788 185.591 159.375 0.00173686 0.00537382 2024-09-14T04:43:24.545 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-14T04:43:24.632 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:24.670 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:24.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:24.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:24.773 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:24.811 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:24.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:24.812 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:24.815 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:43:24.829 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:43:24.914 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:24.947 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:24.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:24.948 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:25.049 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:25.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:25.094 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:25.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:25.195 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:25.239 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:25.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:25.240 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:25.305 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is set 2024-09-14T04:43:25.341 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:25.383 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:25.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:25.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:25.485 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:25.518 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:25.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:25.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:25.533 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 8 16 23092 23076 180.266 143 0.0056814 0.0055419 2024-09-14T04:43:25.553 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:25 smithi039 ceph-mon[106470]: pgmap v2371: 99 pgs: 99 active+clean; 873 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2024-09-14T04:43:25.553 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:25 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/6021321' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:43:25.554 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:25 smithi039 ceph-mon[111235]: pgmap v2371: 99 pgs: 99 active+clean; 873 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2024-09-14T04:43:25.554 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:25 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/6021321' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:43:25.620 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:25.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:25.668 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:25.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:25.770 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:25 smithi129 ceph-mon[101316]: pgmap v2371: 99 pgs: 99 active+clean; 873 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2024-09-14T04:43:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:25 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/6021321' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:43:25.799 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:25.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:25.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:25.901 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:25.935 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:25.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:25.936 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:26.037 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:26.080 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:26.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:26.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:26.182 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:26.223 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:26.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:26.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:26.325 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:26.354 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:26.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:26.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:26.456 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:26.493 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:26.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:26.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:26.533 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 9 16 25234 25218 175.11 133.875 0.00179213 0.0056801 2024-09-14T04:43:26.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:26 smithi039 ceph-mon[106470]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:43:26.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:26 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/6021321' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:43:26.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:26 smithi039 ceph-mon[106470]: osdmap e1389: 8 total, 7 up, 8 in 2024-09-14T04:43:26.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:26 smithi039 ceph-mon[111235]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:43:26.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:26 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/6021321' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:43:26.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:26 smithi039 ceph-mon[111235]: osdmap e1389: 8 total, 7 up, 8 in 2024-09-14T04:43:26.595 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:26.626 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:26.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:26.627 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:26.729 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:26.761 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:26.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:26.762 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:26.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:26 smithi129 ceph-mon[101316]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:43:26.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:26 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/6021321' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:43:26.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:26 smithi129 ceph-mon[101316]: osdmap e1389: 8 total, 7 up, 8 in 2024-09-14T04:43:26.864 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:26.893 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:26.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:26.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:26.995 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:27.016 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:27.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:27.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:27.118 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:27.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:27.152 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:27.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:27.253 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:27.292 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:27.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:27.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:27.394 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:27.436 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:27.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:27.437 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:27.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:27 smithi129 ceph-mon[101316]: pgmap v2373: 99 pgs: 99 active+clean; 1.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-14T04:43:27.533 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 10 16 27547 27531 172.053 144.562 0.00649005 0.00580566 2024-09-14T04:43:27.538 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:27.577 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:27.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:27.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:27.667 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:43:27.667 INFO:teuthology.orchestra.run.smithi039.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-14T04:43:27.667 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:43:27.668 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:27 smithi039 ceph-mon[106470]: pgmap v2373: 99 pgs: 99 active+clean; 1.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-14T04:43:27.668 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:27 smithi039 ceph-mon[111235]: pgmap v2373: 99 pgs: 99 active+clean; 1.0 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 105 MiB/s wr, 1.68k op/s 2024-09-14T04:43:27.678 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:27.700 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:27.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:27.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:27.802 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:27.833 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:27.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:27.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:27.935 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:27.957 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:27.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:27.962 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:28.063 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:28.099 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:28.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:28.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:28.201 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:28.238 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:28.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:28.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:28.340 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:28.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:28.376 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:28.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:28.477 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:28.499 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:28.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:28.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:28.533 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 11 16 29103 29087 165.252 97.25 0.00450198 0.0060459 2024-09-14T04:43:28.601 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:28.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:28.640 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:28.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:28.742 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:28.773 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:28.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:28.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:28.875 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:28.903 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:28.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:28.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:29.005 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:43:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:43:28] "GET /metrics HTTP/1.1" 200 36740 "" "Prometheus/2.51.0" 2024-09-14T04:43:29.041 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:29.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:29.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:29.143 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:29.180 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:29.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:29.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:29.282 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:29.300 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:43:29.300 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:43:29.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:29.326 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:29.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:29.427 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:29.462 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:29.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:29.463 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:29.483 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:43:29.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:29 smithi129 ceph-mon[101316]: pgmap v2374: 99 pgs: 99 active+clean; 1.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-09-14T04:43:29.534 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 12 16 31114 31098 161.952 125.688 0.00960905 0.0061693 2024-09-14T04:43:29.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:29 smithi039 ceph-mon[106470]: pgmap v2374: 99 pgs: 99 active+clean; 1.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-09-14T04:43:29.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:29 smithi039 ceph-mon[111235]: pgmap v2374: 99 pgs: 99 active+clean; 1.3 GiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 128 MiB/s wr, 2.05k op/s 2024-09-14T04:43:29.564 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:29.596 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:43:29.643 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:29.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:29.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:29.746 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:29.794 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:29.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:29.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:29.895 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:29.943 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:29.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:29.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:29.984 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-14T04:43:30.045 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:30.083 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:30.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:30.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:30.185 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:30.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:30.236 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:30.237 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:30.332 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:43:30.337 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:30.362 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:30.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:30.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:30.464 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:30.503 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:30.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:30.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:30.534 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 13 16 33014 32998 158.628 118.75 0.00353575 0.00629848 2024-09-14T04:43:30.605 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:30.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:30.633 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:30.633 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:30.734 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:30.769 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:30.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:30.770 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:30.871 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:30.906 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:30.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:30.907 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:31.008 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:31.042 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:31.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:31.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:31.144 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:31.182 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:31.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:31.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:31.284 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:31.306 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:31.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:31.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:31.408 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:31.445 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:31.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:31.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:31.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:31 smithi129 ceph-mon[101316]: pgmap v2375: 99 pgs: 99 active+clean; 1.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 159 MiB/s wr, 2.54k op/s 2024-09-14T04:43:31.534 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 14 16 34744 34728 155.02 108.125 0.00301397 0.00644519 2024-09-14T04:43:31.547 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:31.587 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:31.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:31.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:31.688 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:31.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:31.728 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:31.729 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:31.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:31 smithi039 ceph-mon[106470]: pgmap v2375: 99 pgs: 99 active+clean; 1.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 159 MiB/s wr, 2.54k op/s 2024-09-14T04:43:31.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:31 smithi039 ceph-mon[111235]: pgmap v2375: 99 pgs: 99 active+clean; 1.6 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 159 MiB/s wr, 2.54k op/s 2024-09-14T04:43:31.830 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:31.871 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:31.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:31.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:31.972 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:32.007 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:32.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:32.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:32.109 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:32.144 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:32.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:32.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:32.246 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:32.286 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:32.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:32.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:32.379 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:43:32.379 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:43:32.390 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:32.425 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:32.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:32.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:32.527 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:32.534 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 15 15 36600 36585 152.422 116.062 0.00336774 0.00655482 2024-09-14T04:43:32.560 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:32.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:32.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:32.662 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:32.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:32.696 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:32.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:32.797 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:32.836 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:32.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:32.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:32.937 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:32.970 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:32.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:32.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:33.055 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-14T04:43:33.071 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:33.107 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:33.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:33.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:33.209 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:33.248 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:33.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:33.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:33.328 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:43:33.350 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:33.388 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:33.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:33.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:33.490 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:33.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:33.529 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:33.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:33.534 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 16 16 38459 38443 150.153 116.125 0.00330079 0.00665494 2024-09-14T04:43:33.617 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:33 smithi129 ceph-mon[101316]: pgmap v2376: 99 pgs: 99 active+clean; 1.7 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 162 MiB/s wr, 2.58k op/s 2024-09-14T04:43:33.631 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:33.667 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:33.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:33.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:33.769 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:33.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:33 smithi039 ceph-mon[106470]: pgmap v2376: 99 pgs: 99 active+clean; 1.7 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 162 MiB/s wr, 2.58k op/s 2024-09-14T04:43:33.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:33 smithi039 ceph-mon[111235]: pgmap v2376: 99 pgs: 99 active+clean; 1.7 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 162 MiB/s wr, 2.58k op/s 2024-09-14T04:43:33.814 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:33.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:33.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:33.916 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:33.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:33.952 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:33.952 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:34.026 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:43:34.053 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:34.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:34.094 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:34.094 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:34.195 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:34.236 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:34.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:34.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:34.338 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:34.376 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:34.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:34.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:34.478 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:34.519 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:34.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:34.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:34.534 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 17 16 40445 40429 148.621 124.125 0.00314869 0.0067234 2024-09-14T04:43:34.622 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:34.624 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [4] live_osds: [0, 1, 2, 3, 5, 6, 7] 2024-09-14T04:43:34.624 INFO:tasks.thrashosds.thrasher:Reviving osd 4 2024-09-14T04:43:34.624 INFO:tasks.cephadm.osd.4:Restarting osd.4 (starting--it wasn't running)... 2024-09-14T04:43:34.624 DEBUG:teuthology.orchestra.run.smithi129:osd.4> sudo journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.4.service 2024-09-14T04:43:34.627 DEBUG:teuthology.orchestra.run.smithi129:> sudo systemctl start ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.4 2024-09-14T04:43:34.658 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:34.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:34.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:34.760 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:34.795 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:34.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:34.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:34.897 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:34.941 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:34.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:34.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:34.947 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:34 smithi129 systemd[1]: Starting Ceph osd.4 for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T04:43:35.043 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:35.082 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:35.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:35.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:35.184 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:35.234 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:35.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:35.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:35.257 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:43:35.257 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:43:35.336 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:35.378 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:35.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:35.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:35.476 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:35 smithi039 ceph-mon[106470]: pgmap v2377: 99 pgs: 99 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2024-09-14T04:43:35.476 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:35 smithi039 ceph-mon[111235]: pgmap v2377: 99 pgs: 99 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2024-09-14T04:43:35.481 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:35.505 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:35 smithi129 ceph-mon[101316]: pgmap v2377: 99 pgs: 99 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 136 MiB/s wr, 2.18k op/s 2024-09-14T04:43:35.505 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:35 smithi129 podman[153341]: 2024-09-14 04:43:35.257900164 +0000 UTC m=+0.030328467 image pull 67ddd33fa72f26a51c969a9043e7a3ddf784506b3d1949ad815ae4dd791beec1 quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322 2024-09-14T04:43:35.505 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:35 smithi129 podman[153341]: 2024-09-14 04:43:35.388395345 +0000 UTC m=+0.160823632 container create 1e32db016b63f88ccdb08234789556fd5d613ad7d0419b08c21ecc37a06500cc (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, RELEASE=reef-e4e227b, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-14T04:43:35.536 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:35.537 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 18 16 42294 42278 146.784 115.562 0.00658932 0.00680703 2024-09-14T04:43:35.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:35.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:35.638 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:35.680 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:35.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:35.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:35.782 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:35.821 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:35.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:35.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:35.923 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:35.962 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:35.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:35.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:36.064 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:36.102 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:36.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:36.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:36.204 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:36.239 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:36.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:36.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:36.260 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:43:36.260 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:43:36.271 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:35 smithi129 podman[153341]: 2024-09-14 04:43:35.826413847 +0000 UTC m=+0.598842142 container init 1e32db016b63f88ccdb08234789556fd5d613ad7d0419b08c21ecc37a06500cc (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-e4e227b, org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-14T04:43:36.271 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:35 smithi129 podman[153341]: 2024-09-14 04:43:35.834185721 +0000 UTC m=+0.606614003 container start 1e32db016b63f88ccdb08234789556fd5d613ad7d0419b08c21ecc37a06500cc (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=reef-e4e227b, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T04:43:36.271 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:35 smithi129 podman[153341]: 2024-09-14 04:43:35.92617228 +0000 UTC m=+0.698600585 container attach 1e32db016b63f88ccdb08234789556fd5d613ad7d0419b08c21ecc37a06500cc (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, RELEASE=reef-e4e227b, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=) 2024-09-14T04:43:36.298 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:43:36.299 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:43:36.336 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:43:36.337 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:43:36.342 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:36.376 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:43:36.376 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:43:36.378 INFO:teuthology.orchestra.run.smithi039.stderr:Error ENXIO: problem getting command descriptions from osd.4 2024-09-14T04:43:36.385 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:36.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:36.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:36.416 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:43:36.417 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:43:36.452 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:43:36.452 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:43:36.459 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-14T04:43:36.486 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:43:36.487 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:43:36.487 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:36.529 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:36.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:36.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:36.534 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 19 16 44115 44099 145.048 113.812 0.0141584 0.00688878 2024-09-14T04:43:36.632 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:36.671 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:36.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:36.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:36.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:36 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2447404188' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:43:36.771 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4-activate[153365]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-14T04:43:36.771 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:36 smithi129 bash[153341]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-14T04:43:36.772 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4-activate[153365]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-14T04:43:36.772 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:36 smithi129 bash[153341]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-09-14T04:43:36.772 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4-activate[153365]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-14T04:43:36.772 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:36 smithi129 bash[153341]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-09-14T04:43:36.772 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4-activate[153365]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-14T04:43:36.772 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:36 smithi129 bash[153341]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-09-14T04:43:36.772 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4-activate[153365]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-14T04:43:36.772 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:36 smithi129 bash[153341]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-09-14T04:43:36.772 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4-activate[153365]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-14T04:43:36.772 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:36 smithi129 bash[153341]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-09-14T04:43:36.772 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4-activate[153365]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-14T04:43:36.772 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:36 smithi129 bash[153341]: --> ceph-volume raw activate successful for osd ID: 4 2024-09-14T04:43:36.772 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:36 smithi129 podman[153341]: 2024-09-14 04:43:36.439514643 +0000 UTC m=+1.211942942 container died 1e32db016b63f88ccdb08234789556fd5d613ad7d0419b08c21ecc37a06500cc (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4-activate, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-e4e227b) 2024-09-14T04:43:36.774 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:36.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:36 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2447404188' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:43:36.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:36 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2447404188' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:43:36.807 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:36.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:36.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:36.909 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:36.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:36.941 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:36.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:37.042 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:37.084 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:37.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:37.085 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:37.186 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:37.224 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:37.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:37.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:37.326 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:37.358 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:43:37.363 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:37.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:37.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:37.465 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:37.495 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:37 smithi129 ceph-mon[101316]: pgmap v2378: 99 pgs: 99 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2024-09-14T04:43:37.496 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:43:37.496 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:37 smithi129 podman[153543]: 2024-09-14 04:43:37.227383928 +0000 UTC m=+0.833641860 container remove 1e32db016b63f88ccdb08234789556fd5d613ad7d0419b08c21ecc37a06500cc (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4-activate, GIT_CLEAN=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , RELEASE=reef-e4e227b, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T04:43:37.501 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:37.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:37.503 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:37.534 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:2024-09-14T04:43:37.534673+0000 min lat: 0.00134858 max lat: 0.144682 avg lat: 0.0069572 2024-09-14T04:43:37.534 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:43:37.535 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 20 16 45981 45965 143.626 116.625 0.00382472 0.0069572 2024-09-14T04:43:37.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:37 smithi039 ceph-mon[106470]: pgmap v2378: 99 pgs: 99 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2024-09-14T04:43:37.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:43:37.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:37 smithi039 ceph-mon[111235]: pgmap v2378: 99 pgs: 99 active+clean; 2.3 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 127 MiB/s wr, 2.03k op/s 2024-09-14T04:43:37.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:43:37.604 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:37.620 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:43:37.620 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:43:37.641 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:37.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:37.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:37.743 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:37.765 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:37.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:37.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:37.867 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:37.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:37.906 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:37.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:38.007 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:38.021 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:37 smithi129 podman[153699]: 2024-09-14 04:43:37.670496501 +0000 UTC m=+0.266551592 image pull 67ddd33fa72f26a51c969a9043e7a3ddf784506b3d1949ad815ae4dd791beec1 quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322 2024-09-14T04:43:38.021 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:37 smithi129 podman[153699]: 2024-09-14 04:43:37.806089637 +0000 UTC m=+0.402144720 container create c25f04f58bc6ef694053d7b1d5fba3948cdfdb2f38e0a83dd4f9e8ecffa7da95 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , RELEASE=reef-e4e227b, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-09-14T04:43:38.053 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:38.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:38.054 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:38.155 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:38.184 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:38.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:38.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:38.286 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:38.327 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:38.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:38.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:38.429 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:38.457 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:38.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:38.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:38.487 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:38 smithi129 podman[153699]: 2024-09-14 04:43:38.177225015 +0000 UTC m=+0.773280109 container init c25f04f58bc6ef694053d7b1d5fba3948cdfdb2f38e0a83dd4f9e8ecffa7da95 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4, org.label-schema.build-date=20240731, RELEASE=reef-e4e227b, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD) 2024-09-14T04:43:38.488 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:38 smithi129 podman[153699]: 2024-09-14 04:43:38.186661611 +0000 UTC m=+0.782716684 container start c25f04f58bc6ef694053d7b1d5fba3948cdfdb2f38e0a83dd4f9e8ecffa7da95 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, GIT_CLEAN=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, RELEASE=reef-e4e227b, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=) 2024-09-14T04:43:38.488 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:38 smithi129 ceph-osd[153825]: -- 172.21.15.129:0/2010014177 <== mon.0 v2:172.21.15.39:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559a339771e0 con 0x559a33aea400 2024-09-14T04:43:38.534 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 21 16 47529 47513 141.394 96.75 0.0204397 0.00706639 2024-09-14T04:43:38.559 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:38.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:38.600 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:38.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:38.627 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-14T04:43:38.702 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:38.746 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:38.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:38.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:38.758 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:38 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4[153765]: 2024-09-14T04:43:38.485+0000 7fa981293640 -1 Falling back to public interface 2024-09-14T04:43:38.759 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:38 smithi129 bash[153699]: c25f04f58bc6ef694053d7b1d5fba3948cdfdb2f38e0a83dd4f9e8ecffa7da95 2024-09-14T04:43:38.759 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:38 smithi129 systemd[1]: Started Ceph osd.4 for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T04:43:38.849 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:38.888 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:38.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:38.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:38.990 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:39.030 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:39.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:39.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:39.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:43:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:43:38] "GET /metrics HTTP/1.1" 200 36754 "" "Prometheus/2.51.0" 2024-09-14T04:43:39.131 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:39.171 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:39.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:39.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:39.272 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:39.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:39.314 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:39.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:39.410 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is unset 2024-09-14T04:43:39.416 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:39.462 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:39.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:39.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:39.535 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 22 15 49026 49011 139.222 93.625 0.00898827 0.00717793 2024-09-14T04:43:39.564 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:39.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:39.602 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:39.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:39.621 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:43:39.639 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:39 smithi129 ceph-mon[101316]: pgmap v2379: 99 pgs: 99 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2024-09-14T04:43:39.640 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:39 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/357686308' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:43:39.640 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:39 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:43:39.664 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:39 smithi039 ceph-mon[106470]: pgmap v2379: 99 pgs: 99 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2024-09-14T04:43:39.664 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:39 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/357686308' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:43:39.664 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:39 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:43:39.664 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:39 smithi039 ceph-mon[111235]: pgmap v2379: 99 pgs: 99 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 124 MiB/s wr, 1.99k op/s 2024-09-14T04:43:39.664 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:39 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/357686308' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:43:39.664 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:39 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:43:39.703 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:39.733 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:39.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:39.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:39.792 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:43:39.793 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [], 2024-09-14T04:43:39.793 INFO:teuthology.orchestra.run.smithi129.stdout: "complaint_time": 30, 2024-09-14T04:43:39.793 INFO:teuthology.orchestra.run.smithi129.stdout: "num_blocked_ops": 0 2024-09-14T04:43:39.793 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:43:39.802 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:43:39.835 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:39.867 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:39.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:39.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:39.969 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:40.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:40.019 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:40.019 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:40.120 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:40.150 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:40.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:40.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:40.252 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:40.292 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:40.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:40.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:40.394 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:40.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:40.432 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:40.432 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:40.439 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:43:40.498 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-14T04:43:40.534 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:40.535 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 23 15 50600 50585 137.446 98.375 0.0073679 0.00727083 2024-09-14T04:43:40.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:40 smithi039 ceph-mon[106470]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:43:40.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:40 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:43:40.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:40 smithi039 ceph-mon[106470]: osdmap e1390: 8 total, 7 up, 8 in 2024-09-14T04:43:40.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:40 smithi039 ceph-mon[111235]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:43:40.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:40 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:43:40.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:40 smithi039 ceph-mon[111235]: osdmap e1390: 8 total, 7 up, 8 in 2024-09-14T04:43:40.570 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:40.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:40.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:40.673 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:40.709 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:40 smithi129 ceph-mon[101316]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:43:40.710 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:40 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:43:40.710 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:40 smithi129 ceph-mon[101316]: osdmap e1390: 8 total, 7 up, 8 in 2024-09-14T04:43:40.711 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:40.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:40.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:40.813 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:40.847 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:40.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:40.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:40.949 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:40.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:40.984 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:40.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:41.085 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:41.123 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:41.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:41.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:41.225 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:41.267 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:41.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:41.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:41.369 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:41.405 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:41.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:41.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:41.507 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:41.535 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 24 16 52264 52248 136.049 103.938 0.0106569 0.00734533 2024-09-14T04:43:41.554 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:41.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:41.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:41.656 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:41.694 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:41.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:41.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:41 smithi129 ceph-mon[101316]: pgmap v2381: 99 pgs: 99 active+clean; 2.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-09-14T04:43:41.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:41 smithi039 ceph-mon[106470]: pgmap v2381: 99 pgs: 99 active+clean; 2.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-09-14T04:43:41.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:41 smithi039 ceph-mon[111235]: pgmap v2381: 99 pgs: 99 active+clean; 2.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s 2024-09-14T04:43:41.796 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:41.821 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:41.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:41.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:41.924 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:41.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:41.950 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:41.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:42.052 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:42.089 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:42.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:42.093 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:42.194 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:42.227 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:42.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:42.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:42.293 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:43:42.329 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:42.405 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:42.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:42.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:42.507 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:42.535 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:42.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:42.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:42.538 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 25 16 53658 53642 134.085 87.125 0.00856814 0.00745281 2024-09-14T04:43:42.637 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:42.673 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:42.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:42.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:42.775 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:42.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:42.821 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:42.821 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:42.841 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:43:42.922 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:42.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:42.962 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:42.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:43.030 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:43:43.031 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:43:43.031 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:43:43.063 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:43.071 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:43:43.071 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:43:43.096 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:43.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:43.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:43.106 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:43:43.106 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:43:43.147 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:43:43.147 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:43:43.147 INFO:teuthology.orchestra.run.smithi039.stderr:Error ENXIO: problem getting command descriptions from osd.4 2024-09-14T04:43:43.177 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:43:43.177 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:43:43.197 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:43.215 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:43:43.215 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:43:43.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:43.241 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:43.241 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:43.255 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:43:43.255 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:43:43.343 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:43.385 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:43.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:43.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:43.487 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:43.532 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:43.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:43.533 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:43.536 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 26 16 55164 55148 132.548 94.125 0.00974745 0.00753873 2024-09-14T04:43:43.634 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:43.670 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:43 smithi039 ceph-mon[106470]: pgmap v2382: 99 pgs: 99 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-09-14T04:43:43.670 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:43 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2477659345' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:43:43.670 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:43 smithi039 ceph-mon[111235]: pgmap v2382: 99 pgs: 99 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-09-14T04:43:43.670 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:43 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2477659345' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:43:43.677 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:43.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:43.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:43.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:43 smithi129 ceph-mon[101316]: pgmap v2382: 99 pgs: 99 active+clean; 2.8 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 115 MiB/s wr, 1.84k op/s 2024-09-14T04:43:43.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:43 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2477659345' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:43:43.779 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:43.804 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:43.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:43.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:43.906 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:43.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:43.934 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:43.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:44.035 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:44.076 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:44.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:44.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:44.136 INFO:teuthology.orchestra.run.smithi129.stderr:no valid command found; 10 closest matches: 2024-09-14T04:43:44.137 INFO:teuthology.orchestra.run.smithi129.stderr:0 2024-09-14T04:43:44.137 INFO:teuthology.orchestra.run.smithi129.stderr:1 2024-09-14T04:43:44.137 INFO:teuthology.orchestra.run.smithi129.stderr:2 2024-09-14T04:43:44.137 INFO:teuthology.orchestra.run.smithi129.stderr:abort 2024-09-14T04:43:44.137 INFO:teuthology.orchestra.run.smithi129.stderr:assert 2024-09-14T04:43:44.137 INFO:teuthology.orchestra.run.smithi129.stderr:bluefs debug_inject_read_zeros 2024-09-14T04:43:44.137 INFO:teuthology.orchestra.run.smithi129.stderr:bluefs files list 2024-09-14T04:43:44.137 INFO:teuthology.orchestra.run.smithi129.stderr:bluefs stats 2024-09-14T04:43:44.138 INFO:teuthology.orchestra.run.smithi129.stderr:bluestore allocator dump block 2024-09-14T04:43:44.138 INFO:teuthology.orchestra.run.smithi129.stderr:bluestore allocator fragmentation block 2024-09-14T04:43:44.138 INFO:teuthology.orchestra.run.smithi129.stderr:admin_socket: invalid command 2024-09-14T04:43:44.178 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:44.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:44.221 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:44.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:44.322 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:44.365 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:44.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:44.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:44.466 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:44.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:44.507 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:44.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:44.536 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 27 16 56566 56550 130.883 87.625 0.00384266 0.00762951 2024-09-14T04:43:44.609 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:44.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:44.642 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:44.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:44.743 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:44.777 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:44.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:44.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:44.853 DEBUG:teuthology.orchestra.run:got remote process result: 6 2024-09-14T04:43:44.854 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2024-09-14T04:43:44.880 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:44.918 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:44.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:44.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:45.020 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:45.060 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:45.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:45.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:45.162 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:45.199 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:45.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:45.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:45.301 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:45.336 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:45.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:45.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:45.386 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-14T04:43:45.386 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-4, ['dump_ops_in_flight'] 2024-09-14T04:43:45.438 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:45.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:45.477 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:45.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:45.537 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 28 16 57923 57907 129.238 84.8125 0.0043606 0.00773255 2024-09-14T04:43:45.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:45 smithi039 ceph-mon[106470]: pgmap v2383: 99 pgs: 99 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2024-09-14T04:43:45.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:45 smithi039 ceph-mon[111235]: pgmap v2383: 99 pgs: 99 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2024-09-14T04:43:45.580 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:45.618 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:45.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:45.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:45.720 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:45.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:45.761 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:45.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:45.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:45 smithi129 ceph-mon[101316]: pgmap v2383: 99 pgs: 99 active+clean; 3.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s 2024-09-14T04:43:45.862 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:45.903 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:45.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:45.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:46.008 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:46.058 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:46.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:46.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:46.160 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:46.198 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:46.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:46.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:46.300 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:46.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:46.337 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:46.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:46.438 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:46.477 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:46.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:46.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:46.536 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 29 16 59446 59430 128.064 95.1875 0.0084144 0.00780324 2024-09-14T04:43:46.559 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:43:46.559 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:43:46.579 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:46.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:46.626 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:46.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:46.727 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:46.766 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:46.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:46.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:46.854 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:43:46.868 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:46.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:46.939 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:46.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:47.021 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:46 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4[153765]: 2024-09-14T04:43:46.695+0000 7fa981293640 -1 osd.4 1344 log_to_monitors true 2024-09-14T04:43:47.039 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:47.085 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:47.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:47.086 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:47.187 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:47.197 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-14T04:43:47.246 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:47.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:47.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:47.293 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:43:47.348 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:47.380 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:47.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:47.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:47.437 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is unset 2024-09-14T04:43:47.482 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:47.503 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:43:47.537 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 30 16 60740 60724 126.491 80.875 0.00449196 0.00789921 2024-09-14T04:43:47.554 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:47.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:47.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:47.656 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:47.696 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:47.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:47.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:47.771 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 04:43:47 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4[153765]: 2024-09-14T04:43:47.563+0000 7fa97903d640 -1 osd.4 1391 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-09-14T04:43:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:47 smithi129 ceph-mon[101316]: pgmap v2384: 99 pgs: 99 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 105 MiB/s wr, 1.69k op/s 2024-09-14T04:43:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:47 smithi129 ceph-mon[101316]: from='osd.4 [v2:172.21.15.129:6800/532739938,v1:172.21.15.129:6801/532739938]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-14T04:43:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:47 smithi129 ceph-mon[101316]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-14T04:43:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:47 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4221485768' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:43:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:47 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:43:47.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:47 smithi039 ceph-mon[106470]: pgmap v2384: 99 pgs: 99 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 105 MiB/s wr, 1.69k op/s 2024-09-14T04:43:47.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:47 smithi039 ceph-mon[106470]: from='osd.4 [v2:172.21.15.129:6800/532739938,v1:172.21.15.129:6801/532739938]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-14T04:43:47.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:47 smithi039 ceph-mon[106470]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-14T04:43:47.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:47 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4221485768' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:43:47.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:47 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:43:47.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:47 smithi039 ceph-mon[111235]: pgmap v2384: 99 pgs: 99 active+clean; 3.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 105 MiB/s wr, 1.69k op/s 2024-09-14T04:43:47.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:47 smithi039 ceph-mon[111235]: from='osd.4 [v2:172.21.15.129:6800/532739938,v1:172.21.15.129:6801/532739938]' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-14T04:43:47.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:47 smithi039 ceph-mon[111235]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]} : dispatch 2024-09-14T04:43:47.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:47 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4221485768' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:43:47.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:47 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:43:47.798 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:47.838 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:47.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:47.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:47.941 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:47.988 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:47.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:47.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:48.090 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:48.125 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:48.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:48.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:48.227 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:48.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:48.266 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:48.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:48.367 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:48.407 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:48.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:48.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:48.509 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:48.536 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 31 16 62251 62235 125.456 94.4375 0.00625561 0.00796543 2024-09-14T04:43:48.550 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:48.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:48.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:48.652 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:48.682 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:48.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:48.683 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:48.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:48 smithi129 ceph-mon[101316]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:43:48.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:48 smithi129 ceph-mon[101316]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-14T04:43:48.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:48 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:43:48.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:48 smithi129 ceph-mon[101316]: from='osd.4 [v2:172.21.15.129:6800/532739938,v1:172.21.15.129:6801/532739938]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi129", "root=default"]} : dispatch 2024-09-14T04:43:48.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:48 smithi129 ceph-mon[101316]: osdmap e1391: 8 total, 7 up, 8 in 2024-09-14T04:43:48.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:48 smithi129 ceph-mon[101316]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi129", "root=default"]} : dispatch 2024-09-14T04:43:48.782 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:43:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:43:48] "GET /metrics HTTP/1.1" 200 36754 "" "Prometheus/2.51.0" 2024-09-14T04:43:48.782 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:48 smithi039 ceph-mon[111235]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:43:48.782 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:48 smithi039 ceph-mon[111235]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-14T04:43:48.783 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:48 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:43:48.783 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:48 smithi039 ceph-mon[111235]: from='osd.4 [v2:172.21.15.129:6800/532739938,v1:172.21.15.129:6801/532739938]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi129", "root=default"]} : dispatch 2024-09-14T04:43:48.783 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:48 smithi039 ceph-mon[111235]: osdmap e1391: 8 total, 7 up, 8 in 2024-09-14T04:43:48.783 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:48 smithi039 ceph-mon[111235]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi129", "root=default"]} : dispatch 2024-09-14T04:43:48.783 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:48 smithi039 ceph-mon[106470]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:43:48.783 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:48 smithi039 ceph-mon[106470]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-09-14T04:43:48.783 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:48 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:43:48.783 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:48 smithi039 ceph-mon[106470]: from='osd.4 [v2:172.21.15.129:6800/532739938,v1:172.21.15.129:6801/532739938]' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi129", "root=default"]} : dispatch 2024-09-14T04:43:48.783 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:48 smithi039 ceph-mon[106470]: osdmap e1391: 8 total, 7 up, 8 in 2024-09-14T04:43:48.784 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:48 smithi039 ceph-mon[106470]: from='osd.4 ' entity='osd.4' cmd={"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi129", "root=default"]} : dispatch 2024-09-14T04:43:48.784 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:48.830 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:48.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:48.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:48.932 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:48.968 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:48.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:48.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:49.070 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:49.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:49.105 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:49.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:49.206 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:49.245 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:49.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:49.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:49.347 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:49.375 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:49.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:49.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:49.477 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:49.497 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:43:49.497 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:43:49.506 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:49.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:49.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:49.537 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 32 16 63674 63658 124.315 88.9375 0.00863341 0.00803877 2024-09-14T04:43:49.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:49 smithi039 ceph-mon[106470]: pgmap v2386: 99 pgs: 99 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2024-09-14T04:43:49.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:49 smithi039 ceph-mon[106470]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T04:43:49.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:49 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T04:43:49.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:49 smithi039 ceph-mon[106470]: osd.4 [v2:172.21.15.129:6800/532739938,v1:172.21.15.129:6801/532739938] boot 2024-09-14T04:43:49.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:49 smithi039 ceph-mon[106470]: osdmap e1392: 8 total, 8 up, 8 in 2024-09-14T04:43:49.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:49 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-14T04:43:49.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:49 smithi039 ceph-mon[111235]: pgmap v2386: 99 pgs: 99 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2024-09-14T04:43:49.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:49 smithi039 ceph-mon[111235]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T04:43:49.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:49 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T04:43:49.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:49 smithi039 ceph-mon[111235]: osd.4 [v2:172.21.15.129:6800/532739938,v1:172.21.15.129:6801/532739938] boot 2024-09-14T04:43:49.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:49 smithi039 ceph-mon[111235]: osdmap e1392: 8 total, 8 up, 8 in 2024-09-14T04:43:49.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:49 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-14T04:43:49.608 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:49.644 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:49.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:49.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:49.746 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:49.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:49 smithi129 ceph-mon[101316]: pgmap v2386: 99 pgs: 99 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s 2024-09-14T04:43:49.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:49 smithi129 ceph-mon[101316]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-09-14T04:43:49.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:49 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T04:43:49.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:49 smithi129 ceph-mon[101316]: osd.4 [v2:172.21.15.129:6800/532739938,v1:172.21.15.129:6801/532739938] boot 2024-09-14T04:43:49.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:49 smithi129 ceph-mon[101316]: osdmap e1392: 8 total, 8 up, 8 in 2024-09-14T04:43:49.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:49 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd metadata", "id": 4} : dispatch 2024-09-14T04:43:49.776 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:49.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:49.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:49.878 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:49.900 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:49.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:49.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:50.001 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:50.040 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:50.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:50.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:50.142 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:50.188 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:50.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:50.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:50.290 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:50.317 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:50.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:50.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:50.324 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:43:50.325 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:43:50.341 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:43:50.341 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:43:50.370 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:43:50.370 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:43:50.387 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-14T04:43:50.388 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:43:50.388 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:43:50.418 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:43:50.418 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:43:50.419 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:50.454 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:43:50.454 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:43:50.468 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:50.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:50.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:50.485 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:43:50.485 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:43:50.517 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:43:50.518 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:43:50.537 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 33 16 64954 64938 122.972 80 0.00708563 0.0081263 2024-09-14T04:43:50.569 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:50.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:50.595 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:50.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:50.696 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:50.735 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:50.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:50.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:50.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:50 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3553141922' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:43:50.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:50 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3553141922' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:43:50.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:50 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3553141922' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:43:50.823 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:43:50.837 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:50.904 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:50.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:50.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:51.006 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:51.054 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:51.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:51.055 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:51.156 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:51.178 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:51.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:51.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:51.280 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:51.315 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:51.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:51.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:51.417 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:51.449 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:51.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:51.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:51.537 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 34 16 66079 66063 121.423 70.3125 0.0163172 0.00823047 2024-09-14T04:43:51.550 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:51.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:51.572 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:51.572 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:51.673 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:51.716 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:51.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:51.717 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:51 smithi129 ceph-mon[101316]: pgmap v2388: 99 pgs: 99 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 105 MiB/s wr, 1.69k op/s 2024-09-14T04:43:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:51 smithi039 ceph-mon[106470]: pgmap v2388: 99 pgs: 99 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 105 MiB/s wr, 1.69k op/s 2024-09-14T04:43:51.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:51 smithi039 ceph-mon[111235]: pgmap v2388: 99 pgs: 99 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 105 MiB/s wr, 1.69k op/s 2024-09-14T04:43:51.817 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:51.839 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:51.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:51.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:51.941 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:51.962 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:51.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:51.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:52.064 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:52.104 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:52.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:52.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:52.206 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:52.248 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:52.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:52.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:52.351 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:52.379 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:52.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:52.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:52.481 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:52.501 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:52.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:52.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:52.537 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 35 16 67224 67208 119.998 71.5625 0.00303825 0.00832268 2024-09-14T04:43:52.604 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:52.644 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:52.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:52.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:52.746 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:52.771 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:43:52.773 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:52.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:52.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:52.875 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:52.908 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:52.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:52.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:53.009 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:53.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:53.042 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:53.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:53.143 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:53.171 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:53.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:53.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:53.273 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:53.304 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:53.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:53.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:53.406 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:53.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:53.434 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:53.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:53.535 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:53.538 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 36 16 68481 68465 118.847 78.5625 0.00489826 0.00840911 2024-09-14T04:43:53.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:53 smithi039 ceph-mon[106470]: pgmap v2389: 99 pgs: 99 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-14T04:43:53.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:53 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:43:53.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:53 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:43:53.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:53 smithi039 ceph-mon[111235]: pgmap v2389: 99 pgs: 99 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-14T04:43:53.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:53 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:43:53.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:53 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:43:53.573 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:53.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:53.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:53.675 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:53.709 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:53.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:53.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:53 smithi129 ceph-mon[101316]: pgmap v2389: 99 pgs: 99 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-14T04:43:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:53 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:43:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:53 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:43:53.812 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:53.857 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:53.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:53.858 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:53.959 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:53.992 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:53.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:53.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:54.022 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-14T04:43:54.094 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:54.106 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:43:54.134 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:54.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:54.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:54.236 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:54.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:54.272 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:54.273 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:54.373 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:54.425 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:54.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:54.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:54.527 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:54.537 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 37 16 69907 69891 118.044 89.125 0.00537376 0.00846549 2024-09-14T04:43:54.558 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:54.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:54.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:54.660 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:54.690 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:54.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:54.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:54.696 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:43:54.792 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:54.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:54.846 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:54.846 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:54.918 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:43:54.946 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:55.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:55.000 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:55.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:55.088 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:43:55.101 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:55.154 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:55.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:55.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:55.255 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:55.296 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:55.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:55.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:55.400 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:55.432 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:55.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:55.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:55.533 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:55.537 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 38 16 71150 71134 116.981 77.6875 0.00934912 0.00854312 2024-09-14T04:43:55.539 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:55 smithi039 ceph-mon[106470]: pgmap v2390: 99 pgs: 99 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-14T04:43:55.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:55 smithi039 ceph-mon[111235]: pgmap v2390: 99 pgs: 99 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-14T04:43:55.567 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:55.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:55.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:55.609 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:55 smithi129 ceph-mon[101316]: pgmap v2390: 99 pgs: 99 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 97 MiB/s wr, 1.55k op/s 2024-09-14T04:43:55.670 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:55.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:55.701 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:55.701 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:55.802 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:55.837 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:55.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:55.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:55.939 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:55.976 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:55.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:55.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:56.078 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:56.119 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:56.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:56.120 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:56.221 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:56.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:56.257 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:56.257 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:56.326 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is set 2024-09-14T04:43:56.358 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:56.399 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:56.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:56.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:56.501 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:56.537 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 39 16 72447 72431 116.06 81.0625 0.0166012 0.0086108 2024-09-14T04:43:56.543 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:56.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:56.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:56.644 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:56.666 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:56.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:56.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:56.728 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:56 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/98877569' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:43:56.729 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:56 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:43:56.768 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:56.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:56 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/98877569' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:43:56.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:56 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:43:56.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:56 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/98877569' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:43:56.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:56 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:43:56.804 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:56.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:56.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:56.906 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:56.945 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:56.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:56.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:57.047 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:57.072 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:43:57.072 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [], 2024-09-14T04:43:57.073 INFO:teuthology.orchestra.run.smithi129.stdout: "num_ops": 0 2024-09-14T04:43:57.073 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:43:57.085 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:57.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:57.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:57.187 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:57.219 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:57.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:57.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:57.321 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:57.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:57.353 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:57.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:57.454 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:57.485 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:57.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:57.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:57.537 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:2024-09-14T04:43:57.537794+0000 min lat: 0.00134858 max lat: 0.144682 avg lat: 0.00867106 2024-09-14T04:43:57.538 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:43:57.538 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 40 16 73791 73775 115.259 84 0.00429896 0.00867106 2024-09-14T04:43:57.587 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:57.616 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:57.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:57.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:57.693 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:57 smithi129 ceph-mon[101316]: pgmap v2391: 99 pgs: 99 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 87 MiB/s wr, 1.38k op/s 2024-09-14T04:43:57.693 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:57 smithi129 ceph-mon[101316]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:43:57.693 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:57 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:43:57.693 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:57 smithi129 ceph-mon[101316]: osdmap e1393: 8 total, 8 up, 8 in 2024-09-14T04:43:57.717 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:57.763 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:57.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:57.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:57.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:57 smithi039 ceph-mon[106470]: pgmap v2391: 99 pgs: 99 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 87 MiB/s wr, 1.38k op/s 2024-09-14T04:43:57.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:57 smithi039 ceph-mon[106470]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:43:57.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:57 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:43:57.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:57 smithi039 ceph-mon[106470]: osdmap e1393: 8 total, 8 up, 8 in 2024-09-14T04:43:57.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:57 smithi039 ceph-mon[111235]: pgmap v2391: 99 pgs: 99 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 87 MiB/s wr, 1.38k op/s 2024-09-14T04:43:57.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:57 smithi039 ceph-mon[111235]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:43:57.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:57 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:43:57.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:57 smithi039 ceph-mon[111235]: osdmap e1393: 8 total, 8 up, 8 in 2024-09-14T04:43:57.846 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:43:57.846 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [], 2024-09-14T04:43:57.846 INFO:teuthology.orchestra.run.smithi129.stdout: "complaint_time": 30, 2024-09-14T04:43:57.846 INFO:teuthology.orchestra.run.smithi129.stdout: "num_blocked_ops": 0 2024-09-14T04:43:57.846 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:43:57.865 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:57.907 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:57.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:57.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:58.009 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:58.045 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:58.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:58.046 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:58.147 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:43:58.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:58.178 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:58.178 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:43:58.279 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:43:58.319 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:58.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:58.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:43:58.421 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:58.458 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:58.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:58.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:58.538 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 41 15 74942 74927 114.203 72 0.0137241 0.00875095 2024-09-14T04:43:58.560 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:43:58.600 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:58.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:58.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:43:58.702 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:58.712 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 0 1 2024-09-14T04:43:58.735 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:58.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:58.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:58.837 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:43:58.867 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:58.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:58.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:43:58.969 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:43:59.021 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:59.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:59.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:43:59.037 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:43:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:43:58] "GET /metrics HTTP/1.1" 200 36610 "" "Prometheus/2.51.0" 2024-09-14T04:43:59.122 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:43:59.159 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:59.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:59.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:43:59.261 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:59.303 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:59.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:59.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:59.365 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-14T04:43:59.405 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:43:59.448 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:43:59.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:59.449 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:43:59.538 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 42 16 76236 76220 113.408 80.8125 0.00755705 0.0088125 2024-09-14T04:43:59.550 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:59.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:59.604 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:59.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:59.625 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:43:59.705 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:43:59.711 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:43:59.728 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:59.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:59.729 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:43:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:43:59 smithi129 ceph-mon[101316]: pgmap v2393: 99 pgs: 99 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-14T04:43:59.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:43:59 smithi039 ceph-mon[106470]: pgmap v2393: 99 pgs: 99 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-14T04:43:59.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:43:59 smithi039 ceph-mon[111235]: pgmap v2393: 99 pgs: 99 active+clean; 4.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-14T04:43:59.830 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:43:59.865 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:43:59.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:43:59.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:43:59.967 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:00.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:00.002 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:00.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:00.103 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:00.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:00.140 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:00.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:00.242 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:00.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:00.280 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:00.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:00.381 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:00.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:00.426 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:00.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:00.527 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:00.538 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 43 16 77437 77421 112.516 75.0625 0.0218528 0.00888216 2024-09-14T04:44:00.566 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:00.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:00.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:00.668 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:00.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:00.708 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:00.708 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:00.809 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:00.848 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:00.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:00.849 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:00.950 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:00.987 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:00.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:00.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:01.079 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:44:01.088 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:01.173 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:01.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:01.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:01.275 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:01.303 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:01.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:01.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:01.405 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:01.449 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:01.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:01.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:01.538 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 44 16 78672 78656 111.713 77.1875 0.0153558 0.00894611 2024-09-14T04:44:01.551 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:01.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:01.597 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:01.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:01.633 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:01 smithi129 ceph-mon[101316]: pgmap v2394: 99 pgs: 99 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-09-14T04:44:01.647 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:01 smithi039 ceph-mon[106470]: pgmap v2394: 99 pgs: 99 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-09-14T04:44:01.647 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:01 smithi039 ceph-mon[111235]: pgmap v2394: 99 pgs: 99 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 81 MiB/s wr, 1.30k op/s 2024-09-14T04:44:01.668 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:44:01.668 INFO:teuthology.orchestra.run.smithi129.stdout: "size": 20, 2024-09-14T04:44:01.668 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 600, 2024-09-14T04:44:01.669 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [ 2024-09-14T04:44:01.669 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.669 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.44840.0:1217 52.6 52:645af7d8:::rbd_data.af28c493695c.0000000000000002:head [set-alloc-hint object_size 4194304 write_size 4194304,writefull 0~4194304] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1232)", 2024-09-14T04:44:01.669 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:34:48.256995+0000", 2024-09-14T04:44:01.669 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 553.40772446400001, 2024-09-14T04:44:01.669 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.042029234999999998, 2024-09-14T04:44:01.669 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:44:01.669 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:44:01.669 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:44:01.669 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.44840", 2024-09-14T04:44:01.670 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/1436701460", 2024-09-14T04:44:01.670 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 1217 2024-09-14T04:44:01.670 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.670 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:44:01.670 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.670 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:44:01.670 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:34:48.256995+0000", 2024-09-14T04:44:01.670 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.670 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.671 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.671 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:44:01.671 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:34:48.256995+0000", 2024-09-14T04:44:01.671 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.671 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.671 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.671 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:44:01.671 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:34:48.256962+0000", 2024-09-14T04:44:01.671 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999666 2024-09-14T04:44:01.671 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.672 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.672 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:44:01.672 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:34:48.268350+0000", 2024-09-14T04:44:01.672 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.011388191000000001 2024-09-14T04:44:01.672 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.672 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.672 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:44:01.672 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:34:48.268377+0000", 2024-09-14T04:44:01.672 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.7158999999999999e-05 2024-09-14T04:44:01.672 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.672 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.672 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:44:01.672 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:34:48.268393+0000", 2024-09-14T04:44:01.673 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5642e-05 2024-09-14T04:44:01.673 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.673 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.673 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:44:01.673 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:34:48.268452+0000", 2024-09-14T04:44:01.673 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.8708000000000001e-05 2024-09-14T04:44:01.673 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.673 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.673 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:44:01.673 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:34:48.268595+0000", 2024-09-14T04:44:01.674 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00014380099999999999 2024-09-14T04:44:01.674 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.674 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.674 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 5,6", 2024-09-14T04:44:01.674 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:34:48.268756+0000", 2024-09-14T04:44:01.674 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000160369 2024-09-14T04:44:01.674 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.674 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.674 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:44:01.674 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:34:48.294755+0000", 2024-09-14T04:44:01.674 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.025998929 2024-09-14T04:44:01.674 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.674 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.675 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:44:01.675 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:34:48.298856+0000", 2024-09-14T04:44:01.675 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0041017420000000002 2024-09-14T04:44:01.675 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.675 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.675 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:44:01.675 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:34:48.298955+0000", 2024-09-14T04:44:01.675 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.8162e-05 2024-09-14T04:44:01.675 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.675 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.675 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:44:01.676 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:34:48.298995+0000", 2024-09-14T04:44:01.676 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0342000000000001e-05 2024-09-14T04:44:01.676 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.676 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.676 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:44:01.676 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:34:48.299024+0000", 2024-09-14T04:44:01.676 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.9458000000000002e-05 2024-09-14T04:44:01.676 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.676 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:44:01.676 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.676 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.676 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.676 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.44840.0:1216 52.5 e1232/1165 52:a69d48fe:::rbd_data.af28c493695c.0000000000000001:head v 1232'14, mlcod=1190'12)", 2024-09-14T04:44:01.677 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:34:48.259968+0000", 2024-09-14T04:44:01.677 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 553.404751852, 2024-09-14T04:44:01.677 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.038750299000000002, 2024-09-14T04:44:01.677 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:44:01.677 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:44:01.677 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:44:01.677 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.677 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:44:01.677 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:34:48.259968+0000", 2024-09-14T04:44:01.678 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.678 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.678 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.678 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:44:01.678 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:34:48.259968+0000", 2024-09-14T04:44:01.678 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.678 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.678 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.678 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:44:01.678 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:34:48.259966+0000", 2024-09-14T04:44:01.678 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:44:01.678 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.678 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.678 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:44:01.678 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:34:48.271855+0000", 2024-09-14T04:44:01.679 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.011888783 2024-09-14T04:44:01.679 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.679 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.679 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:44:01.679 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:34:48.271935+0000", 2024-09-14T04:44:01.679 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.0739000000000001e-05 2024-09-14T04:44:01.679 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.679 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.679 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:44:01.679 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:34:48.271950+0000", 2024-09-14T04:44:01.679 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.4956999999999999e-05 2024-09-14T04:44:01.679 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.679 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.680 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:44:01.680 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:34:48.272015+0000", 2024-09-14T04:44:01.680 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.4465e-05 2024-09-14T04:44:01.680 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.680 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.680 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:44:01.680 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:34:48.272039+0000", 2024-09-14T04:44:01.680 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.4000999999999999e-05 2024-09-14T04:44:01.680 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.680 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.680 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:44:01.680 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:34:48.298666+0000", 2024-09-14T04:44:01.680 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.026627074000000001 2024-09-14T04:44:01.681 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.681 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.681 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:44:01.681 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:34:48.298718+0000", 2024-09-14T04:44:01.681 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2123999999999998e-05 2024-09-14T04:44:01.681 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.681 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:44:01.681 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.681 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.681 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.681 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.34914.0:1417 52.8 e1250/1165 52:1089d5b9:::rbd_data.886237e36aca.0000000000000003:head v 1250'27, mlcod=1238'25)", 2024-09-14T04:44:01.681 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:35:47.876918+0000", 2024-09-14T04:44:01.681 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 493.787801683, 2024-09-14T04:44:01.682 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.037751221000000001, 2024-09-14T04:44:01.682 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:44:01.682 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:44:01.682 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:44:01.682 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.682 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:44:01.682 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:35:47.876918+0000", 2024-09-14T04:44:01.682 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.682 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.682 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.682 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:44:01.682 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:35:47.876918+0000", 2024-09-14T04:44:01.682 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.682 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.683 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.683 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:44:01.683 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:35:47.876917+0000", 2024-09-14T04:44:01.683 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:44:01.683 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.683 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.683 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:44:01.683 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:35:47.889248+0000", 2024-09-14T04:44:01.683 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.012330829 2024-09-14T04:44:01.683 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.683 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.683 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:44:01.683 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:35:47.889279+0000", 2024-09-14T04:44:01.683 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.1245999999999999e-05 2024-09-14T04:44:01.684 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.684 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.684 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:44:01.684 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:35:47.889287+0000", 2024-09-14T04:44:01.684 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.9510000000000006e-06 2024-09-14T04:44:01.684 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.684 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.684 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:44:01.685 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:35:47.889333+0000", 2024-09-14T04:44:01.685 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6566000000000001e-05 2024-09-14T04:44:01.685 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.685 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.685 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:44:01.685 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:35:47.889356+0000", 2024-09-14T04:44:01.685 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.2221999999999998e-05 2024-09-14T04:44:01.685 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.685 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.685 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:44:01.686 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:35:47.914620+0000", 2024-09-14T04:44:01.686 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.02526403 2024-09-14T04:44:01.686 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.686 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.686 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:44:01.686 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:35:47.914669+0000", 2024-09-14T04:44:01.686 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.9403000000000001e-05 2024-09-14T04:44:01.686 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.686 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:44:01.686 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.686 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.686 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.686 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.34914.0:2209 52.d e1269/1165 52:b2fb3e73:::rbd_data.8862fb33ee1a.0000000000000054:head v 1269'38, mlcod=1264'36)", 2024-09-14T04:44:01.686 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:36:19.227157+0000", 2024-09-14T04:44:01.686 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 462.437562345, 2024-09-14T04:44:01.687 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.059920946000000003, 2024-09-14T04:44:01.687 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:44:01.687 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:44:01.687 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:44:01.687 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.687 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:44:01.687 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:19.227157+0000", 2024-09-14T04:44:01.687 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.687 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.687 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.687 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:44:01.687 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:19.227157+0000", 2024-09-14T04:44:01.687 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.687 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.688 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.688 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:44:01.688 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:19.227157+0000", 2024-09-14T04:44:01.688 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:44:01.688 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.688 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.688 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:44:01.688 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:19.249496+0000", 2024-09-14T04:44:01.688 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.022339214 2024-09-14T04:44:01.688 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.688 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.688 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:44:01.688 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:19.249529+0000", 2024-09-14T04:44:01.688 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.2583999999999998e-05 2024-09-14T04:44:01.688 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.689 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.689 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:44:01.689 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:19.249540+0000", 2024-09-14T04:44:01.689 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1054e-05 2024-09-14T04:44:01.689 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.689 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.689 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:44:01.689 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:19.249652+0000", 2024-09-14T04:44:01.689 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000112187 2024-09-14T04:44:01.689 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.689 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.689 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:44:01.689 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:19.249676+0000", 2024-09-14T04:44:01.690 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.4110000000000001e-05 2024-09-14T04:44:01.690 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.690 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.690 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:44:01.690 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:19.287021+0000", 2024-09-14T04:44:01.690 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.037345110000000001 2024-09-14T04:44:01.690 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.690 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.690 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:44:01.690 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:19.287078+0000", 2024-09-14T04:44:01.690 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.7071000000000003e-05 2024-09-14T04:44:01.690 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.690 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:44:01.690 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.690 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.690 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.691 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.34914.0:2213 52.6 52:66670c87:::rbd_data.8862fb33ee1a.000000000000001b:head [set-alloc-hint object_size 4194304 write_size 4194304,write 2152051~2042253] snapc 67=[67] ondisk+write+known_if_redirected+supports_pool_eio e1269)", 2024-09-14T04:44:01.691 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:36:19.231069+0000", 2024-09-14T04:44:01.691 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 462.43365048300001, 2024-09-14T04:44:01.691 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.053958543999999997, 2024-09-14T04:44:01.691 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:44:01.691 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:44:01.691 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:44:01.691 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.34914", 2024-09-14T04:44:01.691 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/1048804342", 2024-09-14T04:44:01.691 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 2213 2024-09-14T04:44:01.691 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.691 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:44:01.691 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.692 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:44:01.692 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:19.231069+0000", 2024-09-14T04:44:01.692 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.692 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.692 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.692 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:44:01.692 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:19.231069+0000", 2024-09-14T04:44:01.692 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.692 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.692 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.692 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:44:01.692 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:19.231068+0000", 2024-09-14T04:44:01.692 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:44:01.692 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.693 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.693 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:44:01.693 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:19.243929+0000", 2024-09-14T04:44:01.693 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.012861155000000001 2024-09-14T04:44:01.693 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.693 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.693 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:44:01.693 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:19.243959+0000", 2024-09-14T04:44:01.693 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.0527000000000001e-05 2024-09-14T04:44:01.693 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.693 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.693 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:44:01.693 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:19.243974+0000", 2024-09-14T04:44:01.694 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.4256e-05 2024-09-14T04:44:01.694 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.694 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.694 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:44:01.694 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:19.244040+0000", 2024-09-14T04:44:01.694 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.6632999999999997e-05 2024-09-14T04:44:01.694 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.694 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.694 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:44:01.694 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:19.244309+0000", 2024-09-14T04:44:01.695 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00026847299999999998 2024-09-14T04:44:01.695 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.695 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.695 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 5,6", 2024-09-14T04:44:01.695 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:19.244537+0000", 2024-09-14T04:44:01.695 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00022878899999999999 2024-09-14T04:44:01.695 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.695 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.695 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:44:01.695 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:19.255763+0000", 2024-09-14T04:44:01.695 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.011225133999999999 2024-09-14T04:44:01.695 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.695 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.695 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:44:01.696 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:19.267025+0000", 2024-09-14T04:44:01.696 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.011262619 2024-09-14T04:44:01.696 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.696 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.696 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:44:01.696 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:19.284929+0000", 2024-09-14T04:44:01.696 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.017903750999999999 2024-09-14T04:44:01.696 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.696 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.696 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:44:01.696 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:19.284986+0000", 2024-09-14T04:44:01.696 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.6622000000000001e-05 2024-09-14T04:44:01.696 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.696 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.697 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:44:01.697 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:19.285028+0000", 2024-09-14T04:44:01.697 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.2049999999999999e-05 2024-09-14T04:44:01.697 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.697 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:44:01.697 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.697 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.697 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.697 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.34914.0:2363 52.9 52:90057332:::rbd_data.8862fb33ee1a.0000000000000006:head [set-alloc-hint object_size 4194304 write_size 4194304,write 535985~3658319] snapc 6a=[6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1272)", 2024-09-14T04:44:01.697 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:36:22.250891+0000", 2024-09-14T04:44:01.697 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 459.41382868199997, 2024-09-14T04:44:01.697 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.042418879999999999, 2024-09-14T04:44:01.697 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:44:01.697 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:44:01.698 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:44:01.698 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.34914", 2024-09-14T04:44:01.698 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/1048804342", 2024-09-14T04:44:01.698 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 2363 2024-09-14T04:44:01.698 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.698 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:44:01.698 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.698 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:44:01.698 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:22.250891+0000", 2024-09-14T04:44:01.698 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.698 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.698 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.698 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:44:01.699 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:22.250891+0000", 2024-09-14T04:44:01.699 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.699 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.699 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.699 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:44:01.699 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:22.250889+0000", 2024-09-14T04:44:01.699 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:44:01.699 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.699 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.699 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:44:01.699 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:22.263309+0000", 2024-09-14T04:44:01.700 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.012419384 2024-09-14T04:44:01.700 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.700 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.700 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:44:01.700 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:22.263338+0000", 2024-09-14T04:44:01.700 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.9805999999999999e-05 2024-09-14T04:44:01.700 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.700 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.700 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:44:01.700 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:22.263350+0000", 2024-09-14T04:44:01.700 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1442e-05 2024-09-14T04:44:01.700 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.700 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.701 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:44:01.701 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:22.263629+0000", 2024-09-14T04:44:01.701 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00027957999999999998 2024-09-14T04:44:01.701 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.701 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.701 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:44:01.701 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:22.263817+0000", 2024-09-14T04:44:01.701 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00018734099999999999 2024-09-14T04:44:01.701 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.701 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.701 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:44:01.701 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:22.264011+0000", 2024-09-14T04:44:01.702 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00019373799999999999 2024-09-14T04:44:01.702 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.702 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.702 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:44:01.702 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:22.281887+0000", 2024-09-14T04:44:01.702 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.017876736000000001 2024-09-14T04:44:01.702 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.703 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.703 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:44:01.703 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:22.286280+0000", 2024-09-14T04:44:01.703 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0043922969999999999 2024-09-14T04:44:01.703 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.703 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.703 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:44:01.703 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:22.293202+0000", 2024-09-14T04:44:01.703 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0069224710000000004 2024-09-14T04:44:01.703 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.703 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.703 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:44:01.704 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:22.293263+0000", 2024-09-14T04:44:01.704 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.1223000000000007e-05 2024-09-14T04:44:01.704 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.704 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.704 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:44:01.704 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:22.293310+0000", 2024-09-14T04:44:01.704 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6480999999999998e-05 2024-09-14T04:44:01.704 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.704 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:44:01.705 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.705 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.705 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.705 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.34914.0:2370 52.1d e1272/1165 52:bd61e55e:::rbd_data.8862fb33ee1a.0000000000000033:head v 1272'28, mlcod=1262'26)", 2024-09-14T04:44:01.705 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:36:22.259040+0000", 2024-09-14T04:44:01.705 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 459.40567905900002, 2024-09-14T04:44:01.705 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.040416186, 2024-09-14T04:44:01.705 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:44:01.705 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:44:01.705 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:44:01.705 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.705 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:44:01.706 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:22.259040+0000", 2024-09-14T04:44:01.706 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.706 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.706 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.706 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:44:01.706 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:22.259040+0000", 2024-09-14T04:44:01.706 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.706 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.707 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.707 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:44:01.707 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:22.259040+0000", 2024-09-14T04:44:01.707 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:44:01.707 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.707 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.707 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:44:01.707 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:22.276998+0000", 2024-09-14T04:44:01.707 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.017958439 2024-09-14T04:44:01.707 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.707 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.708 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:44:01.708 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:22.277025+0000", 2024-09-14T04:44:01.708 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.7046000000000002e-05 2024-09-14T04:44:01.708 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.708 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.708 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:44:01.708 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:22.277037+0000", 2024-09-14T04:44:01.708 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2308e-05 2024-09-14T04:44:01.708 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.708 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.709 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:44:01.709 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:22.277112+0000", 2024-09-14T04:44:01.709 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.4233999999999994e-05 2024-09-14T04:44:01.709 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.709 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.709 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:44:01.709 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:22.277138+0000", 2024-09-14T04:44:01.709 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.6746000000000001e-05 2024-09-14T04:44:01.709 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.709 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.710 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:44:01.710 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:22.299426+0000", 2024-09-14T04:44:01.710 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.02228714 2024-09-14T04:44:01.710 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.710 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.710 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:44:01.710 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:22.299457+0000", 2024-09-14T04:44:01.710 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.1065999999999997e-05 2024-09-14T04:44:01.710 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.711 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:44:01.711 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.711 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.711 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.711 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.34914.0:2541 52.15 52:abdec082:::rbd_data.8862fb33ee1a.0000000000000030:head [write 1197373~2996931] snapc 6d=[6d,6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1275)", 2024-09-14T04:44:01.711 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:36:25.293579+0000", 2024-09-14T04:44:01.711 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 456.37114094100002, 2024-09-14T04:44:01.711 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.052552912, 2024-09-14T04:44:01.711 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:44:01.711 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:44:01.711 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:44:01.711 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.34914", 2024-09-14T04:44:01.712 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/1048804342", 2024-09-14T04:44:01.712 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 2541 2024-09-14T04:44:01.712 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.712 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:44:01.712 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.712 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:44:01.712 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:25.293579+0000", 2024-09-14T04:44:01.712 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.712 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.712 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.712 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:44:01.712 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:25.293579+0000", 2024-09-14T04:44:01.712 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.713 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.713 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.713 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:44:01.713 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:25.293578+0000", 2024-09-14T04:44:01.713 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:44:01.713 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.713 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.713 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:44:01.713 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:25.309713+0000", 2024-09-14T04:44:01.713 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.016135689000000002 2024-09-14T04:44:01.713 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.713 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.713 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:44:01.713 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:25.309736+0000", 2024-09-14T04:44:01.714 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.2728999999999999e-05 2024-09-14T04:44:01.714 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.714 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.714 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:44:01.714 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:25.309747+0000", 2024-09-14T04:44:01.714 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0583e-05 2024-09-14T04:44:01.714 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.714 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.714 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:44:01.714 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:25.309809+0000", 2024-09-14T04:44:01.714 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.1962000000000001e-05 2024-09-14T04:44:01.714 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.714 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.714 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:44:01.715 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:25.309932+0000", 2024-09-14T04:44:01.715 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000122905 2024-09-14T04:44:01.715 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.715 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.715 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:44:01.715 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:25.310119+0000", 2024-09-14T04:44:01.715 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000187474 2024-09-14T04:44:01.715 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.715 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.715 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:44:01.715 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:25.318860+0000", 2024-09-14T04:44:01.715 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0087413130000000006 2024-09-14T04:44:01.715 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.715 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.715 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:44:01.715 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:25.337023+0000", 2024-09-14T04:44:01.716 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.018162676999999999 2024-09-14T04:44:01.716 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.716 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.716 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:44:01.716 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:25.345935+0000", 2024-09-14T04:44:01.716 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0089121549999999997 2024-09-14T04:44:01.716 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.716 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.716 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:44:01.716 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:25.346039+0000", 2024-09-14T04:44:01.716 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000103959 2024-09-14T04:44:01.716 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.716 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.716 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:44:01.716 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:25.346132+0000", 2024-09-14T04:44:01.717 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.2435000000000005e-05 2024-09-14T04:44:01.717 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.717 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:44:01.717 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.717 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.717 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.717 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.34914.0:2696 52.5 e1278/1165 52:a562b873:::rbd_data.8862fb33ee1a.000000000000003b:head v 1278'42, mlcod=1273'38)", 2024-09-14T04:44:01.717 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:36:28.322589+0000", 2024-09-14T04:44:01.717 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 453.34213045899997, 2024-09-14T04:44:01.717 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.043867906999999998, 2024-09-14T04:44:01.717 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:44:01.717 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:44:01.717 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:44:01.717 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.718 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:44:01.718 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:28.322589+0000", 2024-09-14T04:44:01.718 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.718 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.718 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.718 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:44:01.718 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:28.322589+0000", 2024-09-14T04:44:01.718 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.718 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.718 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.718 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:44:01.718 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:28.322588+0000", 2024-09-14T04:44:01.718 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:44:01.718 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.718 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.719 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:44:01.719 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:28.338372+0000", 2024-09-14T04:44:01.719 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.015784156000000001 2024-09-14T04:44:01.719 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.719 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.719 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:44:01.719 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:28.338391+0000", 2024-09-14T04:44:01.719 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.8692999999999999e-05 2024-09-14T04:44:01.719 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.719 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.719 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:44:01.720 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:28.338400+0000", 2024-09-14T04:44:01.720 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.3519999999999996e-06 2024-09-14T04:44:01.720 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.720 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.720 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:44:01.720 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:28.338635+0000", 2024-09-14T04:44:01.720 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00023469499999999999 2024-09-14T04:44:01.720 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.720 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.720 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:44:01.721 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:28.338658+0000", 2024-09-14T04:44:01.721 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.2804000000000001e-05 2024-09-14T04:44:01.721 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.721 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.721 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:44:01.721 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:28.366412+0000", 2024-09-14T04:44:01.721 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.027754095999999999 2024-09-14T04:44:01.721 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.721 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.721 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:44:01.721 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:28.366457+0000", 2024-09-14T04:44:01.721 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.5232999999999999e-05 2024-09-14T04:44:01.722 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.722 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:44:01.722 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.722 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.722 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.722 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.34914.0:2853 52.1d e1281/1165 52:bd61e55e:::rbd_data.8862fb33ee1a.0000000000000033:head v 1281'42, mlcod=1278'38)", 2024-09-14T04:44:01.722 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:36:31.363661+0000", 2024-09-14T04:44:01.722 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 450.30105846100003, 2024-09-14T04:44:01.722 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.043798205999999999, 2024-09-14T04:44:01.722 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:44:01.722 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:44:01.722 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:44:01.722 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.723 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:44:01.723 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:31.363661+0000", 2024-09-14T04:44:01.723 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.723 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.723 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.723 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:44:01.723 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:31.363661+0000", 2024-09-14T04:44:01.723 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.723 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.723 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.723 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:44:01.723 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:31.363661+0000", 2024-09-14T04:44:01.723 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967296 2024-09-14T04:44:01.724 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.724 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.724 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:44:01.724 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:31.381784+0000", 2024-09-14T04:44:01.724 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.018122995999999999 2024-09-14T04:44:01.724 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.724 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.724 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:44:01.724 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:31.381816+0000", 2024-09-14T04:44:01.724 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.2270000000000001e-05 2024-09-14T04:44:01.725 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.725 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.725 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:44:01.725 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:31.381828+0000", 2024-09-14T04:44:01.725 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1919999999999999e-05 2024-09-14T04:44:01.725 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.725 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.725 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:44:01.725 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:31.381919+0000", 2024-09-14T04:44:01.725 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.0623000000000003e-05 2024-09-14T04:44:01.725 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.726 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.726 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:44:01.726 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:31.381941+0000", 2024-09-14T04:44:01.726 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.2725e-05 2024-09-14T04:44:01.726 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.726 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.726 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:44:01.726 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:31.407399+0000", 2024-09-14T04:44:01.726 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.025457362000000001 2024-09-14T04:44:01.726 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.726 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.726 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:44:01.726 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:31.407459+0000", 2024-09-14T04:44:01.726 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.0494000000000001e-05 2024-09-14T04:44:01.727 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.727 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:44:01.727 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.727 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.727 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.727 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.34914.0:3085 52.9 52:9045add6:::rbd_data.8862fb33ee1a.0000000000000027:head [writefull 0~4194304] snapc 77=[77,76,75,74,73,72,71,70,6f,6e,6d,6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1285)", 2024-09-14T04:44:01.728 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:36:35.419020+0000", 2024-09-14T04:44:01.728 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 446.24569982000003, 2024-09-14T04:44:01.728 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.046306130000000001, 2024-09-14T04:44:01.728 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:44:01.728 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:44:01.728 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:44:01.728 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.34914", 2024-09-14T04:44:01.728 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/1048804342", 2024-09-14T04:44:01.728 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 3085 2024-09-14T04:44:01.728 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.728 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:44:01.728 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.729 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:44:01.729 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:35.419020+0000", 2024-09-14T04:44:01.729 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.729 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.729 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.729 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:44:01.729 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:35.419020+0000", 2024-09-14T04:44:01.729 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.729 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.729 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.729 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:44:01.730 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:35.419017+0000", 2024-09-14T04:44:01.730 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999976 2024-09-14T04:44:01.730 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.730 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.730 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:44:01.730 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:35.432004+0000", 2024-09-14T04:44:01.730 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.012987138000000001 2024-09-14T04:44:01.730 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.730 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.730 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:44:01.730 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:35.432033+0000", 2024-09-14T04:44:01.730 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.9133999999999999e-05 2024-09-14T04:44:01.730 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.731 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.731 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:44:01.731 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:35.432043+0000", 2024-09-14T04:44:01.731 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.8479999999999996e-06 2024-09-14T04:44:01.731 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.731 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.731 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:44:01.731 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:35.432106+0000", 2024-09-14T04:44:01.731 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.2691e-05 2024-09-14T04:44:01.731 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.731 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.732 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:01.733 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:44:01.733 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:35.432238+0000", 2024-09-14T04:44:01.733 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000131574 2024-09-14T04:44:01.733 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.733 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.733 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:44:01.733 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:35.432461+0000", 2024-09-14T04:44:01.733 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000223618 2024-09-14T04:44:01.734 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.734 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.734 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:44:01.734 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:35.459696+0000", 2024-09-14T04:44:01.734 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.027234988000000002 2024-09-14T04:44:01.734 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.734 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.734 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:44:01.734 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:35.459814+0000", 2024-09-14T04:44:01.734 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000118205 2024-09-14T04:44:01.734 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.734 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.734 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:44:01.735 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:35.465214+0000", 2024-09-14T04:44:01.735 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0053994899999999998 2024-09-14T04:44:01.735 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.735 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.735 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:44:01.735 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:35.465271+0000", 2024-09-14T04:44:01.735 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.7379000000000001e-05 2024-09-14T04:44:01.735 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.735 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.735 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:44:01.736 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:36:35.465326+0000", 2024-09-14T04:44:01.736 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.4672000000000001e-05 2024-09-14T04:44:01.736 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.736 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:44:01.736 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.736 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.736 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.736 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.34914.0:7346 52.d e1311/1165 52:b6b5c6cb:::rbd_data.8862ba52cd58.0000000000000002:head v 1311'76, mlcod=1306'73)", 2024-09-14T04:44:01.736 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:37:01.034269+0000", 2024-09-14T04:44:01.736 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 420.63045049700003, 2024-09-14T04:44:01.736 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.040092709999999997, 2024-09-14T04:44:01.736 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:44:01.737 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:44:01.737 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:44:01.737 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.737 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:44:01.737 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:37:01.034269+0000", 2024-09-14T04:44:01.737 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.737 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.737 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.737 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:44:01.737 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:37:01.034269+0000", 2024-09-14T04:44:01.738 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.738 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.738 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.738 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:44:01.738 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:37:01.034268+0000", 2024-09-14T04:44:01.738 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:44:01.738 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.738 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.738 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:44:01.738 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:37:01.049343+0000", 2024-09-14T04:44:01.738 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.015075280999999999 2024-09-14T04:44:01.738 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.738 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.739 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:44:01.739 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:37:01.049362+0000", 2024-09-14T04:44:01.739 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.8952000000000001e-05 2024-09-14T04:44:01.739 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.739 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.739 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:44:01.739 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:37:01.049372+0000", 2024-09-14T04:44:01.739 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0006e-05 2024-09-14T04:44:01.739 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.739 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.739 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:44:01.739 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:37:01.049420+0000", 2024-09-14T04:44:01.739 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7914000000000002e-05 2024-09-14T04:44:01.740 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.740 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.740 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:44:01.740 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:37:01.049444+0000", 2024-09-14T04:44:01.740 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.37e-05 2024-09-14T04:44:01.740 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.740 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.740 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:44:01.740 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:37:01.074334+0000", 2024-09-14T04:44:01.740 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.024890385000000001 2024-09-14T04:44:01.741 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.741 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.741 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:44:01.741 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:37:01.074362+0000", 2024-09-14T04:44:01.741 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.7311e-05 2024-09-14T04:44:01.741 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.741 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:44:01.741 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.741 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.741 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.741 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:11496 54.6 54:65dd3c56:::benchmark_data_smithi039_264334_object11495:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1388)", 2024-09-14T04:44:01.742 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:43:21.062393+0000", 2024-09-14T04:44:01.742 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 40.602326732999998, 2024-09-14T04:44:01.742 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.098479161999999995, 2024-09-14T04:44:01.742 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:44:01.742 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:44:01.742 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:44:01.742 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:44:01.742 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:44:01.742 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 11496 2024-09-14T04:44:01.742 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.742 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:44:01.743 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.743 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:44:01.743 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:21.062393+0000", 2024-09-14T04:44:01.743 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.743 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.743 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.743 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:44:01.743 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:21.062393+0000", 2024-09-14T04:44:01.743 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.743 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.743 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.744 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:44:01.744 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:21.062391+0000", 2024-09-14T04:44:01.744 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:44:01.744 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.744 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.744 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:44:01.744 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:21.062443+0000", 2024-09-14T04:44:01.744 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.1796999999999999e-05 2024-09-14T04:44:01.744 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.744 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.744 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:44:01.744 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:21.062453+0000", 2024-09-14T04:44:01.745 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0131e-05 2024-09-14T04:44:01.745 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.745 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.745 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:44:01.745 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:21.062458+0000", 2024-09-14T04:44:01.745 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6600000000000003e-06 2024-09-14T04:44:01.745 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.745 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.745 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:44:01.745 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:21.062489+0000", 2024-09-14T04:44:01.745 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.0828999999999999e-05 2024-09-14T04:44:01.746 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.746 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.746 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:44:01.746 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:21.062564+0000", 2024-09-14T04:44:01.746 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.4744000000000001e-05 2024-09-14T04:44:01.746 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.746 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.746 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:44:01.746 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:21.062643+0000", 2024-09-14T04:44:01.746 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.9103999999999993e-05 2024-09-14T04:44:01.746 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.747 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.747 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:44:01.747 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:21.063172+0000", 2024-09-14T04:44:01.747 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00052912800000000002 2024-09-14T04:44:01.747 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.747 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.747 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:44:01.747 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:21.063585+0000", 2024-09-14T04:44:01.747 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00041343400000000001 2024-09-14T04:44:01.748 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.748 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.748 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:44:01.748 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:21.160768+0000", 2024-09-14T04:44:01.748 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.097182828999999998 2024-09-14T04:44:01.748 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.748 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.748 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:44:01.748 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:21.160830+0000", 2024-09-14T04:44:01.748 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.2117000000000002e-05 2024-09-14T04:44:01.749 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.749 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.749 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:44:01.749 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:21.160872+0000", 2024-09-14T04:44:01.749 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1817000000000003e-05 2024-09-14T04:44:01.749 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.749 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:44:01.749 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.749 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.749 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.749 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:66556 54.6 54:647e80e0:::benchmark_data_smithi039_264334_object66555:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1392)", 2024-09-14T04:44:01.749 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:43:51.929593+0000", 2024-09-14T04:44:01.750 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 9.7351264929999992, 2024-09-14T04:44:01.750 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.084587792999999994, 2024-09-14T04:44:01.750 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:44:01.750 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:44:01.750 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:44:01.750 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:44:01.750 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:44:01.750 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 66556 2024-09-14T04:44:01.750 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.750 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:44:01.750 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.750 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:44:01.750 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:51.929593+0000", 2024-09-14T04:44:01.750 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.751 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.751 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.751 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:44:01.751 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:51.929593+0000", 2024-09-14T04:44:01.751 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.751 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.751 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.751 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:44:01.751 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:51.929591+0000", 2024-09-14T04:44:01.751 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:44:01.751 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.751 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.751 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:44:01.752 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:51.929626+0000", 2024-09-14T04:44:01.752 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.4984999999999998e-05 2024-09-14T04:44:01.752 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.752 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.752 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:44:01.752 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:51.929638+0000", 2024-09-14T04:44:01.752 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1646e-05 2024-09-14T04:44:01.752 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.752 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.752 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:44:01.752 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:51.929643+0000", 2024-09-14T04:44:01.752 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.3900000000000001e-06 2024-09-14T04:44:01.753 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.753 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.753 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:44:01.753 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:51.929672+0000", 2024-09-14T04:44:01.753 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.8442e-05 2024-09-14T04:44:01.753 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.753 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.753 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:44:01.753 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:51.929757+0000", 2024-09-14T04:44:01.753 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.5463999999999994e-05 2024-09-14T04:44:01.753 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.753 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.753 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:44:01.753 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:51.929842+0000", 2024-09-14T04:44:01.754 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.4493999999999994e-05 2024-09-14T04:44:01.754 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.754 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.754 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:44:01.754 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:51.931640+0000", 2024-09-14T04:44:01.754 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0017979890000000001 2024-09-14T04:44:01.754 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.754 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.754 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:44:01.754 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:51.932203+0000", 2024-09-14T04:44:01.754 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00056333299999999995 2024-09-14T04:44:01.754 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.754 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.754 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:44:01.754 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:52.014088+0000", 2024-09-14T04:44:01.755 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.081884578999999999 2024-09-14T04:44:01.755 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.755 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.755 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:44:01.755 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:52.014140+0000", 2024-09-14T04:44:01.755 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2669e-05 2024-09-14T04:44:01.755 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.755 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.755 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:44:01.755 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:52.014181+0000", 2024-09-14T04:44:01.755 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0590999999999998e-05 2024-09-14T04:44:01.755 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.755 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:44:01.755 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.756 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.756 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.756 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:68400 54.6 54:68328e13:::benchmark_data_smithi039_264334_object68399:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1392)", 2024-09-14T04:44:01.756 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:43:53.403956+0000", 2024-09-14T04:44:01.756 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 8.2607637119999993, 2024-09-14T04:44:01.756 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.097056130000000004, 2024-09-14T04:44:01.756 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:44:01.756 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:44:01.756 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:44:01.756 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:44:01.756 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:44:01.756 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 68400 2024-09-14T04:44:01.756 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.757 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:44:01.757 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.757 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:44:01.757 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.403956+0000", 2024-09-14T04:44:01.757 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.757 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.757 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.757 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:44:01.757 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.403956+0000", 2024-09-14T04:44:01.757 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.757 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.757 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.757 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:44:01.757 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.403954+0000", 2024-09-14T04:44:01.758 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:44:01.758 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.758 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.758 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:44:01.758 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404019+0000", 2024-09-14T04:44:01.758 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.4386000000000003e-05 2024-09-14T04:44:01.758 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.758 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.758 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:44:01.758 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404029+0000", 2024-09-14T04:44:01.758 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0414000000000001e-05 2024-09-14T04:44:01.758 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.758 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.758 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:44:01.759 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404034+0000", 2024-09-14T04:44:01.759 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.057e-06 2024-09-14T04:44:01.759 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.759 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.759 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:44:01.759 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404068+0000", 2024-09-14T04:44:01.759 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3380999999999998e-05 2024-09-14T04:44:01.759 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.759 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.759 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:44:01.759 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404195+0000", 2024-09-14T04:44:01.759 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012756099999999999 2024-09-14T04:44:01.759 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.760 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.760 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:44:01.760 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404326+0000", 2024-09-14T04:44:01.760 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013119499999999999 2024-09-14T04:44:01.760 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.760 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.760 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:44:01.760 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.410437+0000", 2024-09-14T04:44:01.760 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0061107419999999997 2024-09-14T04:44:01.760 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.760 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.760 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:44:01.760 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.410475+0000", 2024-09-14T04:44:01.760 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8169e-05 2024-09-14T04:44:01.761 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.761 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.761 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:44:01.761 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.500921+0000", 2024-09-14T04:44:01.761 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.090445708999999999 2024-09-14T04:44:01.761 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.761 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.761 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:44:01.761 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.500974+0000", 2024-09-14T04:44:01.761 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2923999999999997e-05 2024-09-14T04:44:01.761 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.761 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.761 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:44:01.761 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.501012+0000", 2024-09-14T04:44:01.762 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8012999999999997e-05 2024-09-14T04:44:01.762 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.762 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:44:01.762 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.762 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.762 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.762 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:68403 54.6 54:6443ddab:::benchmark_data_smithi039_264334_object68402:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1392)", 2024-09-14T04:44:01.762 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:43:53.404446+0000", 2024-09-14T04:44:01.762 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 8.2602732920000008, 2024-09-14T04:44:01.762 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.096669750999999998, 2024-09-14T04:44:01.762 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:44:01.762 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:44:01.762 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:44:01.762 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:44:01.763 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:44:01.763 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 68403 2024-09-14T04:44:01.763 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.763 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:44:01.763 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.763 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:44:01.763 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404446+0000", 2024-09-14T04:44:01.763 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.763 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.763 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.763 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:44:01.763 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404446+0000", 2024-09-14T04:44:01.763 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.763 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.764 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.764 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:44:01.764 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404445+0000", 2024-09-14T04:44:01.764 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:44:01.764 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.764 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.764 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:44:01.764 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404498+0000", 2024-09-14T04:44:01.764 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.346e-05 2024-09-14T04:44:01.764 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.764 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.764 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:44:01.764 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404510+0000", 2024-09-14T04:44:01.765 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1194e-05 2024-09-14T04:44:01.765 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.765 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.765 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:44:01.765 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404517+0000", 2024-09-14T04:44:01.765 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.0530000000000001e-06 2024-09-14T04:44:01.765 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.765 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.765 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:44:01.765 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.406468+0000", 2024-09-14T04:44:01.765 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0019508500000000001 2024-09-14T04:44:01.765 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.765 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.765 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:44:01.766 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.406575+0000", 2024-09-14T04:44:01.766 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000107556 2024-09-14T04:44:01.766 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.766 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.768 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:44:01.768 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.406697+0000", 2024-09-14T04:44:01.768 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012212 2024-09-14T04:44:01.768 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.769 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.769 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:44:01.769 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.410469+0000", 2024-09-14T04:44:01.769 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0037713019999999998 2024-09-14T04:44:01.769 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.769 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.769 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:44:01.769 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.410479+0000", 2024-09-14T04:44:01.769 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0847e-05 2024-09-14T04:44:01.769 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.769 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.769 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:44:01.769 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.501046+0000", 2024-09-14T04:44:01.769 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.090566728999999999 2024-09-14T04:44:01.770 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.770 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.770 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:44:01.770 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.501087+0000", 2024-09-14T04:44:01.770 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1026999999999999e-05 2024-09-14T04:44:01.770 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.770 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.770 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:44:01.770 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.501116+0000", 2024-09-14T04:44:01.770 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.8852000000000001e-05 2024-09-14T04:44:01.770 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.770 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:44:01.770 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.770 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.771 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.771 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:68404 54.6 54:6bb4aa50:::benchmark_data_smithi039_264334_object68403:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1392)", 2024-09-14T04:44:01.771 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:43:53.404843+0000", 2024-09-14T04:44:01.771 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 8.2598761439999997, 2024-09-14T04:44:01.771 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.096363689000000002, 2024-09-14T04:44:01.771 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:44:01.771 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:44:01.771 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:44:01.771 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:44:01.771 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:44:01.771 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 68404 2024-09-14T04:44:01.771 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.771 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:44:01.771 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.772 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:44:01.772 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404843+0000", 2024-09-14T04:44:01.772 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.772 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.772 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.772 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:44:01.772 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404843+0000", 2024-09-14T04:44:01.772 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.772 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.772 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.772 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:44:01.772 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404843+0000", 2024-09-14T04:44:01.772 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:44:01.773 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.773 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.773 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:44:01.773 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404873+0000", 2024-09-14T04:44:01.773 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.9960000000000001e-05 2024-09-14T04:44:01.773 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.773 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.773 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:44:01.773 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404882+0000", 2024-09-14T04:44:01.773 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.4490000000000005e-06 2024-09-14T04:44:01.773 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.773 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.773 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:44:01.774 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404888+0000", 2024-09-14T04:44:01.774 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.1789999999999996e-06 2024-09-14T04:44:01.774 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.774 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.774 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:44:01.774 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.408713+0000", 2024-09-14T04:44:01.774 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0038248610000000001 2024-09-14T04:44:01.774 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.774 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.774 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:44:01.774 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.408795+0000", 2024-09-14T04:44:01.774 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.1738999999999998e-05 2024-09-14T04:44:01.775 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.775 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.775 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:44:01.775 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.408873+0000", 2024-09-14T04:44:01.775 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.8032000000000001e-05 2024-09-14T04:44:01.775 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.775 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.775 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:44:01.775 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.410526+0000", 2024-09-14T04:44:01.775 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0016527740000000001 2024-09-14T04:44:01.775 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.776 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.776 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:44:01.776 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.411065+0000", 2024-09-14T04:44:01.776 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00053945900000000005 2024-09-14T04:44:01.776 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.776 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.776 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:44:01.776 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.501144+0000", 2024-09-14T04:44:01.776 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.090079318000000005 2024-09-14T04:44:01.776 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.776 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.776 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:44:01.776 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.501181+0000", 2024-09-14T04:44:01.776 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6458000000000002e-05 2024-09-14T04:44:01.776 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.777 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.777 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:44:01.777 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.501207+0000", 2024-09-14T04:44:01.777 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.6276999999999999e-05 2024-09-14T04:44:01.777 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.777 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:44:01.777 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.777 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.777 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.778 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.35436.0:68414 54.f e1392/1380 54:f4108b15:::benchmark_data_smithi039_264334_object68413:head v 1392'4287, mlcod=1392'4285)", 2024-09-14T04:44:01.778 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:43:53.412852+0000", 2024-09-14T04:44:01.778 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 8.2518673469999992, 2024-09-14T04:44:01.778 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.093915778000000005, 2024-09-14T04:44:01.778 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:44:01.778 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:44:01.778 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:44:01.778 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.778 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:44:01.778 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.412852+0000", 2024-09-14T04:44:01.778 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.778 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.778 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.778 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:44:01.779 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.412852+0000", 2024-09-14T04:44:01.779 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.779 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.779 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.779 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:44:01.779 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.412852+0000", 2024-09-14T04:44:01.779 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:44:01.779 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.779 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.779 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:44:01.779 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.412921+0000", 2024-09-14T04:44:01.780 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.9048999999999996e-05 2024-09-14T04:44:01.780 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.780 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.780 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:44:01.780 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.412928+0000", 2024-09-14T04:44:01.780 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.2509999999999997e-06 2024-09-14T04:44:01.780 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.780 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.780 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:44:01.780 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.412932+0000", 2024-09-14T04:44:01.780 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8130000000000002e-06 2024-09-14T04:44:01.780 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.781 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.781 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:44:01.781 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.500774+0000", 2024-09-14T04:44:01.781 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.087842240000000002 2024-09-14T04:44:01.781 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.781 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.781 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:44:01.781 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.500798+0000", 2024-09-14T04:44:01.781 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.3510999999999999e-05 2024-09-14T04:44:01.781 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.781 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.781 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:44:01.781 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.506732+0000", 2024-09-14T04:44:01.781 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0059339570000000001 2024-09-14T04:44:01.782 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.782 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.782 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:44:01.782 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.506768+0000", 2024-09-14T04:44:01.782 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6448000000000001e-05 2024-09-14T04:44:01.782 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.782 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:44:01.782 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.782 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.782 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.782 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.35436.0:68412 54.e e1392/1380 54:7391dba8:::benchmark_data_smithi039_264334_object68411:head v 1392'4283, mlcod=1392'4281)", 2024-09-14T04:44:01.782 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:43:53.414570+0000", 2024-09-14T04:44:01.782 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 8.2501490929999992, 2024-09-14T04:44:01.783 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.095679800999999995, 2024-09-14T04:44:01.783 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:44:01.783 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:44:01.783 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:44:01.783 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.783 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:44:01.783 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.414570+0000", 2024-09-14T04:44:01.783 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.783 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.783 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.783 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:44:01.783 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.414570+0000", 2024-09-14T04:44:01.783 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.783 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.784 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.784 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:44:01.784 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.414570+0000", 2024-09-14T04:44:01.784 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:44:01.784 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.784 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.784 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:44:01.784 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.414619+0000", 2024-09-14T04:44:01.784 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.9243999999999998e-05 2024-09-14T04:44:01.784 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.784 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.784 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:44:01.785 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.500758+0000", 2024-09-14T04:44:01.785 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.086138951000000005 2024-09-14T04:44:01.785 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.785 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.785 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:44:01.785 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.500769+0000", 2024-09-14T04:44:01.785 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.095e-05 2024-09-14T04:44:01.785 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.785 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.786 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:44:01.786 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.501229+0000", 2024-09-14T04:44:01.786 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00045990199999999999 2024-09-14T04:44:01.786 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.786 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.786 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:44:01.786 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.501245+0000", 2024-09-14T04:44:01.786 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5956000000000001e-05 2024-09-14T04:44:01.786 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.786 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.786 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:44:01.786 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.510168+0000", 2024-09-14T04:44:01.786 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.008923149 2024-09-14T04:44:01.787 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.787 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.787 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:44:01.787 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.510250+0000", 2024-09-14T04:44:01.787 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.2233999999999999e-05 2024-09-14T04:44:01.787 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.787 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:44:01.787 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.787 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.788 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.788 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.35436.0:68417 54.f e1392/1380 54:f223c55f:::benchmark_data_smithi039_264334_object68416:head v 1392'4288, mlcod=1392'4285)", 2024-09-14T04:44:01.788 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:43:53.415332+0000", 2024-09-14T04:44:01.788 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 8.2493875840000008, 2024-09-14T04:44:01.788 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.091473889000000003, 2024-09-14T04:44:01.788 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:44:01.788 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:44:01.788 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:44:01.788 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.788 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:44:01.789 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.415332+0000", 2024-09-14T04:44:01.789 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.789 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.789 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.789 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:44:01.789 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.415332+0000", 2024-09-14T04:44:01.789 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:44:01.789 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.789 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.789 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:44:01.790 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.415332+0000", 2024-09-14T04:44:01.790 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:44:01.790 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.790 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.790 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:44:01.790 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.415401+0000", 2024-09-14T04:44:01.790 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.9407999999999997e-05 2024-09-14T04:44:01.790 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.790 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.791 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:44:01.791 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.500754+0000", 2024-09-14T04:44:01.791 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.085353164999999995 2024-09-14T04:44:01.791 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.791 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.791 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:44:01.791 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.500766+0000", 2024-09-14T04:44:01.791 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1855000000000001e-05 2024-09-14T04:44:01.791 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.792 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.792 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:44:01.792 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.504126+0000", 2024-09-14T04:44:01.792 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0033599020000000001 2024-09-14T04:44:01.792 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.792 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.792 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:44:01.792 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.504195+0000", 2024-09-14T04:44:01.792 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8804000000000001e-05 2024-09-14T04:44:01.792 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.792 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.792 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:44:01.792 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.506789+0000", 2024-09-14T04:44:01.793 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0025946649999999999 2024-09-14T04:44:01.793 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:44:01.793 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:44:01.793 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:44:01.793 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.506806+0000", 2024-09-14T04:44:01.793 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.6381000000000001e-05 2024-09-14T04:44:01.793 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.793 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:44:01.793 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.793 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:44:01.793 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:44:01.793 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:44:01.826 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:01.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:01.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:01.929 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:01.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:01.981 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:01.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:02.045 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:44:02.045 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:44:02.083 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:02.085 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:44:02.085 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:44:02.111 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:02.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:02.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:02.117 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:44:02.117 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:44:02.149 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:44:02.149 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:44:02.183 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:44:02.183 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:44:02.214 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:02.217 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:44:02.217 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:44:02.251 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:44:02.251 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:44:02.255 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:02.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:02.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:02.287 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:44:02.287 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:44:02.357 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:02.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:02.402 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:02.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:02.503 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:02.537 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:02.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:02.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:02.538 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 45 16 79899 79883 110.935 76.6875 0.0100686 0.00900961 2024-09-14T04:44:02.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:02 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/647758250' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:44:02.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:02 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/647758250' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:44:02.583 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-14T04:44:02.619 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:02 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/647758250' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:44:02.638 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:02.678 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:02.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:02.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:02.780 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:02.818 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:02.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:02.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:02.900 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:44:02.919 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:02.953 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:02.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:02.954 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:03.055 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:03.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:03.089 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:03.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:03.190 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:03.229 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:03.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:03.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:03.331 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:03.369 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:03.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:03.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:03.471 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:03.502 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:03.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:03.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:03.538 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 46 16 81169 81153 110.249 79.375 0.0083554 0.00906557 2024-09-14T04:44:03.539 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:03 smithi039 ceph-mon[106470]: pgmap v2395: 99 pgs: 99 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-14T04:44:03.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:03 smithi039 ceph-mon[111235]: pgmap v2395: 99 pgs: 99 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-14T04:44:03.604 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:03.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:03.638 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:03.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:03.739 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:03 smithi129 ceph-mon[101316]: pgmap v2395: 99 pgs: 99 active+clean; 4.5 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s 2024-09-14T04:44:03.778 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:03.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:03.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:03.880 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:03.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:03.916 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:03.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:04.017 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:04.056 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:04.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:04.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:04.159 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:04.199 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:04.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:04.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:04.301 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:04.337 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:04.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:04.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:04.438 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:04.471 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:04.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:04.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:04.538 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 47 16 82361 82345 109.488 74.5 0.00861712 0.00912756 2024-09-14T04:44:04.573 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:04.602 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:04.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:04.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:04.629 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:44:04.704 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:04.753 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:04.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:04.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:04.855 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:04.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:04.892 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:04.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:04.993 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:05.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:05.030 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:05.030 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:05.131 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:05.168 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:05.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:05.169 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:05.270 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:05.308 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:05.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:05.309 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:05.397 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.0 to 1 (10000) 2024-09-14T04:44:05.409 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:05.417 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:05 smithi039 ceph-mon[106470]: pgmap v2396: 99 pgs: 99 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-14T04:44:05.418 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:05 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3084003578' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:44:05.418 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:05 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:44:05.418 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:05 smithi039 ceph-mon[111235]: pgmap v2396: 99 pgs: 99 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-14T04:44:05.418 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:05 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3084003578' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:44:05.418 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:05 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:44:05.450 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:05.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:05.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:05.530 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:05 smithi129 ceph-mon[101316]: pgmap v2396: 99 pgs: 99 active+clean; 4.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-14T04:44:05.530 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:05 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3084003578' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:44:05.530 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:05 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:44:05.538 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 48 16 83521 83505 108.717 72.5 0.010111 0.00919303 2024-09-14T04:44:05.552 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:05.585 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:44:05.585 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:44:05.587 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:05.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:05.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:05.689 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:05.721 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:05.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:05.723 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:05.823 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:05.860 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:05.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:05.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:05.962 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:06.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:06.000 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:06.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:06.094 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:44:06.101 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:06.130 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:06.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:06.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:06.232 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:06.266 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:06.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:06.267 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:06.282 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-14T04:44:06.368 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:06.400 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:06.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:06.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:06.421 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:06 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:44:06.502 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:06.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:06 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:44:06.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:06 smithi129 ceph-mon[101316]: osdmap e1394: 8 total, 8 up, 8 in 2024-09-14T04:44:06.539 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 49 16 84724 84708 108.031 75.1875 0.00747614 0.00925132 2024-09-14T04:44:06.544 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:06.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:06.546 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:06.637 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:44:06.647 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:06.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:06.686 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:06.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:06.787 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:06.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:06 smithi039 ceph-mon[106470]: osdmap e1394: 8 total, 8 up, 8 in 2024-09-14T04:44:06.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:06 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:44:06.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:06 smithi039 ceph-mon[111235]: osdmap e1394: 8 total, 8 up, 8 in 2024-09-14T04:44:06.831 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:06.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:06.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:06.933 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:06.975 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:06.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:06.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:07.001 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 1 1 2024-09-14T04:44:07.077 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:07.108 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:07.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:07.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:07.210 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:07.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:07.274 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:07.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:07.375 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:07.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:07.420 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:07.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:07.521 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:07.539 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 50 16 85886 85870 107.323 72.625 0.0201884 0.00931252 2024-09-14T04:44:07.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:07.553 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:07.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:07.654 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:07.679 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:07.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:07.680 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:07.753 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:07 smithi039 ceph-mon[106470]: pgmap v2398: 99 pgs: 99 active+clean; 4.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-14T04:44:07.753 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:44:07.753 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:44:07.753 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:07 smithi039 ceph-mon[111235]: pgmap v2398: 99 pgs: 99 active+clean; 4.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-14T04:44:07.754 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:44:07.754 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:44:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:07 smithi129 ceph-mon[101316]: pgmap v2398: 99 pgs: 99 active+clean; 4.9 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-14T04:44:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:44:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:44:07.781 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:07.807 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:07.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:07.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:07.908 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:44:07.910 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:07.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:07.943 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:07.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:07.993 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:44:08.043 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:08.093 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:08.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:08.095 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:08.195 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:08.236 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:08.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:08.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:08.338 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:08.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:08.369 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:08.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:08.442 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is set 2024-09-14T04:44:08.470 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:08.483 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:08 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1726223882' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:44:08.514 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:08.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:08.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:08.515 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:44:08.515 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:44:08.541 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 51 16 86828 86812 106.373 58.875 0.00929477 0.00939539 2024-09-14T04:44:08.616 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:08.646 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:08.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:08.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:08.652 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:08 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1726223882' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:44:08.653 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:08 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1726223882' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:44:08.748 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:08.787 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:08.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:08.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:08.889 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:08.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:08.919 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:08.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:08.929 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:44:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:44:08] "GET /metrics HTTP/1.1" 200 36749 "" "Prometheus/2.51.0" 2024-09-14T04:44:09.020 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:09.053 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:09.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:09.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:09.155 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:09.202 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:09.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:09.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:09.303 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:09.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:09.342 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:09.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:09.435 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-14T04:44:09.442 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:09.516 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:09.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:09.517 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:09.539 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 52 16 87893 87877 105.607 66.5625 0.0108943 0.00946409 2024-09-14T04:44:09.618 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:09.657 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:09.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:09.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:09.748 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:44:09.758 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:09.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:09 smithi129 ceph-mon[101316]: pgmap v2399: 99 pgs: 99 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-14T04:44:09.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:09 smithi129 ceph-mon[101316]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:44:09.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:09 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1726223882' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:44:09.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:09 smithi129 ceph-mon[101316]: osdmap e1395: 8 total, 8 up, 8 in 2024-09-14T04:44:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:09 smithi039 ceph-mon[106470]: pgmap v2399: 99 pgs: 99 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-14T04:44:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:09 smithi039 ceph-mon[106470]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:44:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:09 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1726223882' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:44:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:09 smithi039 ceph-mon[106470]: osdmap e1395: 8 total, 8 up, 8 in 2024-09-14T04:44:09.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:09 smithi039 ceph-mon[111235]: pgmap v2399: 99 pgs: 99 active+clean; 5.0 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s 2024-09-14T04:44:09.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:09 smithi039 ceph-mon[111235]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:44:09.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:09 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1726223882' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:44:09.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:09 smithi039 ceph-mon[111235]: osdmap e1395: 8 total, 8 up, 8 in 2024-09-14T04:44:09.794 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:09.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:09.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:09.896 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:09.934 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:09.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:09.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:10.036 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:10.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:10.076 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:10.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:10.177 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:10.211 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:10.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:10.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:10.313 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:10.342 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:10.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:10.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:10.444 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:10.492 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:10.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:10.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:10.540 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 53 16 89093 89077 105.029 75 0.0160546 0.009516 2024-09-14T04:44:10.594 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:10.640 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:10.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:10.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:10.742 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:10.781 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:10.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:10.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:10.883 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:10.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:10.928 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:10.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:11.029 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:11.058 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:11.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:11.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:11.160 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:11.204 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:11.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:11.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:11.306 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:11.341 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:11.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:11.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:11.443 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:11.487 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:11.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:11.488 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:11.540 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 54 16 90267 90251 104.443 73.375 0.0224941 0.00957012 2024-09-14T04:44:11.589 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:11.619 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:11.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:11.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:11.721 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:11.726 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:11 smithi129 ceph-mon[101316]: pgmap v2401: 99 pgs: 99 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-09-14T04:44:11.757 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:11.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:11.758 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:11.784 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:44:11.784 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:44:11.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:11 smithi039 ceph-mon[106470]: pgmap v2401: 99 pgs: 99 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-09-14T04:44:11.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:11 smithi039 ceph-mon[111235]: pgmap v2401: 99 pgs: 99 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-09-14T04:44:11.859 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:11.886 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:11.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:11.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:11.988 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:12.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:12.029 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:12.030 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:12.130 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:12.160 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:12.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:12.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:12.262 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:12.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:12.303 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:12.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:12.404 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:12.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:12.435 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:12.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:12.460 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-14T04:44:12.537 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:12.540 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 55 16 91369 91353 103.796 68.875 0.0148615 0.00962799 2024-09-14T04:44:12.562 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:44:12.562 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:44:12.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:12.566 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:12.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:12.611 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:44:12.611 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:44:12.651 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:44:12.651 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:44:12.667 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:12.693 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:44:12.693 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:44:12.694 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:12.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:12.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:12.727 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:44:12.727 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:44:12.762 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:44:12.763 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:44:12.796 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:12.799 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:44:12.799 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:44:12.824 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:12.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:12.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:12.835 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:44:12.835 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:44:12.925 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:12.967 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:12.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:12.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:13.069 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:13.111 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:13.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:13.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:13.213 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:13.251 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:13.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:13.252 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:13.353 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:13.378 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:13.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:13.380 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:13.460 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.1 to 1 (10000) 2024-09-14T04:44:13.481 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:13.524 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:13.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:13.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:13.540 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 56 16 92396 92380 103.089 64.1875 0.0121487 0.00969523 2024-09-14T04:44:13.626 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:13.670 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:13.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:13.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:13.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:13 smithi129 ceph-mon[101316]: pgmap v2402: 99 pgs: 99 active+clean; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 889 op/s 2024-09-14T04:44:13.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:13 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/397284397' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:44:13.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:13 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3527385211' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:44:13.772 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:13.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:13 smithi039 ceph-mon[106470]: pgmap v2402: 99 pgs: 99 active+clean; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 889 op/s 2024-09-14T04:44:13.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:13 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/397284397' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:44:13.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:13 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3527385211' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:44:13.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:13 smithi039 ceph-mon[111235]: pgmap v2402: 99 pgs: 99 active+clean; 5.3 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 889 op/s 2024-09-14T04:44:13.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:13 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/397284397' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:44:13.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:13 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3527385211' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:44:13.806 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:13.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:13.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:13.908 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:13.946 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:13.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:13.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:14.048 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:14.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:14.082 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:14.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:14.182 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:14.218 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:14.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:14.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:14.321 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:14.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:14.359 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:14.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:14.460 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:14.488 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:14.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:14.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:14.518 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:44:14.540 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 57 15 93576 93561 102.575 73.8125 0.0111041 0.00974295 2024-09-14T04:44:14.590 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:14.626 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:14.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:14.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:14.655 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:44:14.728 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:14.760 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:14.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:14.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:14.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:14 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3527385211' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-14T04:44:14.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:14 smithi129 ceph-mon[101316]: osdmap e1396: 8 total, 8 up, 8 in 2024-09-14T04:44:14.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:14 smithi129 ceph-mon[101316]: osdmap e1397: 8 total, 8 up, 8 in 2024-09-14T04:44:14.779 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:14 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3527385211' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-14T04:44:14.779 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:14 smithi039 ceph-mon[106470]: osdmap e1396: 8 total, 8 up, 8 in 2024-09-14T04:44:14.779 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:14 smithi039 ceph-mon[106470]: osdmap e1397: 8 total, 8 up, 8 in 2024-09-14T04:44:14.779 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:14 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3527385211' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-14T04:44:14.779 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:14 smithi039 ceph-mon[111235]: osdmap e1396: 8 total, 8 up, 8 in 2024-09-14T04:44:14.779 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:14 smithi039 ceph-mon[111235]: osdmap e1397: 8 total, 8 up, 8 in 2024-09-14T04:44:14.862 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:14.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:14.894 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:14.894 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:14.995 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:15.045 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:15.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:15.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:15.147 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:15.189 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:15.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:15.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:15.274 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 2 1 2024-09-14T04:44:15.291 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:15.366 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:15.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:15.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:15.468 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:15.523 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:15.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:15.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:15.540 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 58 16 94616 94600 101.926 64.9375 0.00586434 0.00980475 2024-09-14T04:44:15.625 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:15.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:15.683 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:15.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:15.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:15 smithi129 ceph-mon[101316]: pgmap v2404: 99 pgs: 99 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-14T04:44:15.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:15 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:44:15.783 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:15.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:15 smithi039 ceph-mon[106470]: pgmap v2404: 99 pgs: 99 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-14T04:44:15.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:15 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:44:15.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:15 smithi039 ceph-mon[111235]: pgmap v2404: 99 pgs: 99 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 81 MiB/s wr, 1.29k op/s 2024-09-14T04:44:15.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:15 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:44:15.832 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:15.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:15.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:15.934 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:15.962 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:15.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:15.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:16.064 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:16.097 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:16.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:16.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:16.198 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:16.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:16.244 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:16.244 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:16.345 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:16.379 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:16.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:16.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:16.481 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:16.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:16.511 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:16.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:16.540 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 59 15 95692 95677 101.339 67.3125 0.0155662 0.00986257 2024-09-14T04:44:16.613 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:16.653 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:16.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:16.654 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:16.755 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:16.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:16.798 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:16.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:16.800 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:44:16.900 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:16.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:16.946 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:16.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:17.047 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:17.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:17.082 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:17.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:17.183 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:17.228 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:17.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:17.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:17.330 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:17.365 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:17.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:17.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:17.467 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:17.500 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:17.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:17.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:17.540 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:2024-09-14T04:44:17.540787+0000 min lat: 0.00134858 max lat: 0.144682 avg lat: 0.00991474 2024-09-14T04:44:17.541 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:44:17.541 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 60 16 96807 96791 100.81 69.625 0.0129107 0.00991474 2024-09-14T04:44:17.580 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:44:17.602 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:17.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:17.668 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:17.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:17.703 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:44:17.769 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:17.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:17 smithi129 ceph-mon[101316]: pgmap v2406: 99 pgs: 99 active+clean; 5.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 919 op/s 2024-09-14T04:44:17.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:17 smithi039 ceph-mon[106470]: pgmap v2406: 99 pgs: 99 active+clean; 5.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 919 op/s 2024-09-14T04:44:17.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:17 smithi039 ceph-mon[111235]: pgmap v2406: 99 pgs: 99 active+clean; 5.6 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 57 MiB/s wr, 919 op/s 2024-09-14T04:44:17.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:17.820 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:17.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:17.921 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:17.950 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:17.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:17.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:18.051 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:18.079 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:18.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:18.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:18.181 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:18.223 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:18.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:18.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:18.325 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:18.365 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:18.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:18.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:18.467 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:18.511 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:18.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:18.514 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:18.541 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 61 16 97808 97792 100.183 62.5625 0.02439 0.00997627 2024-09-14T04:44:18.615 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:18.645 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:18.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:18.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:18.650 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:18 smithi039 ceph-mon[106470]: pgmap v2407: 99 pgs: 99 active+clean; 5.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-09-14T04:44:18.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:18 smithi039 ceph-mon[111235]: pgmap v2407: 99 pgs: 99 active+clean; 5.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-09-14T04:44:18.747 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:18.777 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:18.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:18.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:18.879 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:18.906 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:18.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:18.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:19.008 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:19.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:18 smithi129 ceph-mon[101316]: pgmap v2407: 99 pgs: 99 active+clean; 5.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-09-14T04:44:19.033 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:44:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:44:18] "GET /metrics HTTP/1.1" 200 36749 "" "Prometheus/2.51.0" 2024-09-14T04:44:19.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:19.041 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:19.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:19.142 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:19.180 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:19.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:19.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:19.282 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:19.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:19.311 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:19.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:19.412 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:19.448 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:19.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:19.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:19.541 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 62 16 98864 98848 99.6315 66 0.022524 0.0100316 2024-09-14T04:44:19.550 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:19.581 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:19.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:19.582 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:19.683 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:19.724 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:19.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:19.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:19.825 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:19.868 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:19.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:19.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:19.970 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:20.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:20.016 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:20.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:20.117 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:20.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:20.151 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:20.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:20.252 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:20.280 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:20.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:20.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:20.382 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:20.419 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:20.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:20.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:20.521 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:20.541 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 63 16 99859 99843 99.037 62.1875 0.00756904 0.010079 2024-09-14T04:44:20.546 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:20.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:20.547 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:20.648 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:20.681 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:20.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:20.683 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:20.783 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:20.822 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:20.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:20.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:20.924 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:20.968 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:20.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:20.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:21.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:20 smithi129 ceph-mon[101316]: pgmap v2408: 99 pgs: 99 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-09-14T04:44:21.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:20 smithi039 ceph-mon[106470]: pgmap v2408: 99 pgs: 99 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-09-14T04:44:21.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:20 smithi039 ceph-mon[111235]: pgmap v2408: 99 pgs: 99 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-09-14T04:44:21.070 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:21.106 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:21.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:21.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:21.208 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:21.252 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:21.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:21.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:21.354 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:21.388 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:21.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:21.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:21.490 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:21.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:21.528 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:21.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:21.541 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 64 16 100885 100869 98.4915 64.125 0.0111657 0.0101485 2024-09-14T04:44:21.630 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:21.659 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:21.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:21.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:21.761 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:21.807 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:21.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:21.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:21.909 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:21.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:21.940 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:21.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:22.041 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:22.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:22.079 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:22.079 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:22.180 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:22.220 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:22.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:22.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:22.322 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:22.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:22.384 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:22.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:22.485 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:22.522 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:22.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:22.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:22.541 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 65 16 101922 101906 97.9733 64.8125 0.00678591 0.0102017 2024-09-14T04:44:22.624 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:22.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:22.669 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:22.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:22.770 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:22.813 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:22.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:22.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:22.916 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:22.944 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:44:22.944 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [ 2024-09-14T04:44:22.944 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.944 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.35436.0:102319 54.d e1397/1380 54:b37a7133:::benchmark_data_smithi039_264334_object102318:head v 1397'6380, mlcod=1397'6377)", 2024-09-14T04:44:22.944 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:22.937485+0000", 2024-09-14T04:44:22.944 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.0045450270000000001, 2024-09-14T04:44:22.944 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0045755570000000001, 2024-09-14T04:44:22.945 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:44:22.945 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:44:22.945 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "started", 2024-09-14T04:44:22.945 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:44:22.945 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.945 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:44:22.945 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.937485+0000", 2024-09-14T04:44:22.945 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:44:22.945 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.945 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.945 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:44:22.946 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.937485+0000", 2024-09-14T04:44:22.946 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:44:22.946 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.946 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.946 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:44:22.946 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.937486+0000", 2024-09-14T04:44:22.946 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.159e-06 2024-09-14T04:44:22.946 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.946 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.946 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:44:22.946 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.937516+0000", 2024-09-14T04:44:22.947 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.0134999999999999e-05 2024-09-14T04:44:22.947 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.947 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.947 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:44:22.947 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.937534+0000", 2024-09-14T04:44:22.947 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7584999999999999e-05 2024-09-14T04:44:22.947 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.947 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.947 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:44:22.947 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.937543+0000", 2024-09-14T04:44:22.947 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.1290000000000002e-06 2024-09-14T04:44:22.948 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.948 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.948 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:44:22.948 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.937747+0000", 2024-09-14T04:44:22.948 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00020415 2024-09-14T04:44:22.948 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.948 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.948 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:44:22.948 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.937799+0000", 2024-09-14T04:44:22.948 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.2187e-05 2024-09-14T04:44:22.948 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:44:22.948 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:44:22.949 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:44:22.949 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.949 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.949 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.35436.0:102321 54.d e1397/1380)", 2024-09-14T04:44:22.949 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:22.940413+0000", 2024-09-14T04:44:22.949 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.001616749, 2024-09-14T04:44:22.949 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0016990729999999999, 2024-09-14T04:44:22.949 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:44:22.949 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:44:22.949 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "queued for pg", 2024-09-14T04:44:22.949 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:44:22.950 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.950 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:44:22.950 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.940413+0000", 2024-09-14T04:44:22.950 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:44:22.950 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.950 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.950 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:44:22.950 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.940413+0000", 2024-09-14T04:44:22.950 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:44:22.950 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.950 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.950 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:44:22.951 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.940422+0000", 2024-09-14T04:44:22.951 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.7069999999999998e-06 2024-09-14T04:44:22.951 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.951 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.951 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:44:22.951 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.940511+0000", 2024-09-14T04:44:22.951 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.8931000000000006e-05 2024-09-14T04:44:22.951 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.951 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.951 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:44:22.951 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.940528+0000", 2024-09-14T04:44:22.952 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7189000000000001e-05 2024-09-14T04:44:22.952 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.952 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.952 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:44:22.952 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.940537+0000", 2024-09-14T04:44:22.952 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.5110000000000003e-06 2024-09-14T04:44:22.952 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:44:22.952 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:44:22.952 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:44:22.953 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.953 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.953 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.35436.0:102311 54.5 e1397/1380 54:aa96fee3:::benchmark_data_smithi039_264334_object102310:head v 1397'6417, mlcod=1397'6415)", 2024-09-14T04:44:22.953 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:22.928850+0000", 2024-09-14T04:44:22.953 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.013179491999999999, 2024-09-14T04:44:22.953 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.013304160000000001, 2024-09-14T04:44:22.953 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:44:22.953 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:44:22.953 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "started", 2024-09-14T04:44:22.953 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:44:22.953 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.954 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:44:22.954 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.928850+0000", 2024-09-14T04:44:22.954 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:44:22.954 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.954 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.954 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:44:22.954 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.928850+0000", 2024-09-14T04:44:22.954 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:44:22.954 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.955 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.955 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:44:22.955 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.928852+0000", 2024-09-14T04:44:22.955 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.432e-06 2024-09-14T04:44:22.955 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.955 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.955 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:44:22.955 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.928883+0000", 2024-09-14T04:44:22.955 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.1343000000000001e-05 2024-09-14T04:44:22.955 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.956 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.956 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:44:22.956 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.928903+0000", 2024-09-14T04:44:22.956 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0129999999999999e-05 2024-09-14T04:44:22.956 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.956 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.956 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:44:22.956 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.928912+0000", 2024-09-14T04:44:22.957 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.4570000000000005e-06 2024-09-14T04:44:22.957 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.957 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.957 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:44:22.957 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.928997+0000", 2024-09-14T04:44:22.957 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.5609e-05 2024-09-14T04:44:22.957 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.957 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.957 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:44:22.958 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.929019+0000", 2024-09-14T04:44:22.958 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1787e-05 2024-09-14T04:44:22.958 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:44:22.958 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:44:22.958 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:44:22.958 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.958 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.958 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:102313 54.b 54:db52109b:::benchmark_data_smithi039_264334_object102312:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1397)", 2024-09-14T04:44:22.958 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:22.929302+0000", 2024-09-14T04:44:22.959 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.012727686, 2024-09-14T04:44:22.959 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.012911067, 2024-09-14T04:44:22.959 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:44:22.959 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:44:22.959 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:44:22.959 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:44:22.959 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:44:22.959 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:44:22.960 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 102313 2024-09-14T04:44:22.960 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.960 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:44:22.960 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.960 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:44:22.960 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.929302+0000", 2024-09-14T04:44:22.960 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:44:22.960 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.961 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.961 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:44:22.961 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.929302+0000", 2024-09-14T04:44:22.961 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:44:22.961 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.961 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.961 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:44:22.961 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.929304+0000", 2024-09-14T04:44:22.961 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4109999999999999e-06 2024-09-14T04:44:22.961 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.961 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.961 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:44:22.962 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.929326+0000", 2024-09-14T04:44:22.962 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2310000000000002e-05 2024-09-14T04:44:22.962 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.962 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.962 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:44:22.962 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.929336+0000", 2024-09-14T04:44:22.962 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0349e-05 2024-09-14T04:44:22.962 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.962 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.962 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:44:22.962 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.929343+0000", 2024-09-14T04:44:22.962 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1690000000000003e-06 2024-09-14T04:44:22.963 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.963 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.963 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:44:22.963 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.933462+0000", 2024-09-14T04:44:22.963 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0041188010000000001 2024-09-14T04:44:22.963 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.963 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.963 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:44:22.963 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.933604+0000", 2024-09-14T04:44:22.963 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014185700000000001 2024-09-14T04:44:22.963 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.963 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.963 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,5", 2024-09-14T04:44:22.964 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.933743+0000", 2024-09-14T04:44:22.964 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000139221 2024-09-14T04:44:22.964 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.964 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.964 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:44:22.964 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.938807+0000", 2024-09-14T04:44:22.964 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0050633969999999999 2024-09-14T04:44:22.964 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.964 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.964 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:44:22.964 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.939537+0000", 2024-09-14T04:44:22.964 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000729947 2024-09-14T04:44:22.964 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:44:22.964 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:44:22.965 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:44:22.965 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.965 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.965 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.35436.0:102315 54.d e1397/1380 54:b864d42f:::benchmark_data_smithi039_264334_object102314:head v 1397'6379, mlcod=1397'6377)", 2024-09-14T04:44:22.965 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:22.932948+0000", 2024-09-14T04:44:22.965 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.0090818229999999993, 2024-09-14T04:44:22.965 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0093381929999999998, 2024-09-14T04:44:22.965 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:44:22.965 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:44:22.965 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "started", 2024-09-14T04:44:22.965 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:44:22.965 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.965 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:44:22.965 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.932948+0000", 2024-09-14T04:44:22.966 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:44:22.966 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.966 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.966 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:44:22.966 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.932948+0000", 2024-09-14T04:44:22.966 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:44:22.966 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.966 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.966 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:44:22.966 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.932950+0000", 2024-09-14T04:44:22.966 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0719999999999998e-06 2024-09-14T04:44:22.966 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.967 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.967 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:44:22.967 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.932991+0000", 2024-09-14T04:44:22.967 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.0537000000000002e-05 2024-09-14T04:44:22.967 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.967 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.967 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:44:22.967 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.933005+0000", 2024-09-14T04:44:22.967 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3823e-05 2024-09-14T04:44:22.967 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.967 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.967 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:44:22.968 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.933012+0000", 2024-09-14T04:44:22.968 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.8839999999999994e-06 2024-09-14T04:44:22.968 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.968 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.968 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:44:22.968 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.933094+0000", 2024-09-14T04:44:22.968 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.1922000000000006e-05 2024-09-14T04:44:22.968 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:22.968 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:22.968 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:44:22.968 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:22.933115+0000", 2024-09-14T04:44:22.968 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0826999999999999e-05 2024-09-14T04:44:22.969 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:44:22.969 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:44:22.969 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:44:22.969 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:44:22.969 INFO:teuthology.orchestra.run.smithi039.stdout: ], 2024-09-14T04:44:22.969 INFO:teuthology.orchestra.run.smithi039.stdout: "num_ops": 5 2024-09-14T04:44:22.969 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:44:22.970 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:22 smithi039 ceph-mon[106470]: pgmap v2409: 99 pgs: 99 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 788 op/s 2024-09-14T04:44:22.970 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:44:22.970 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:22 smithi039 ceph-mon[111235]: pgmap v2409: 99 pgs: 99 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 788 op/s 2024-09-14T04:44:22.970 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:44:22.971 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:22.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:22.972 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:23.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:22 smithi129 ceph-mon[101316]: pgmap v2409: 99 pgs: 99 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 49 MiB/s wr, 788 op/s 2024-09-14T04:44:23.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:44:23.073 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:23.112 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:23.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:23.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:23.214 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:23.254 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:23.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:23.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:23.356 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:23.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:23.390 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:23.390 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:23.491 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:23.529 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:23.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:23.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:23.541 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 66 16 102973 102957 97.484 65.6875 0.0164242 0.0102534 2024-09-14T04:44:23.631 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:23.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:23.682 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:23.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:23.783 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:23.811 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:23.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:23.812 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:23.913 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:23.953 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:23.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:23.954 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:24.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:23 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:44:24.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:23 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:44:24.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:23 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:44:24.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:23 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:44:24.055 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:24.085 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:24.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:24.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:24.187 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:24.224 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:24.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:24.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:24.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:23 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:44:24.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:23 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:44:24.326 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:24.356 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-14T04:44:24.366 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:24.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:24.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:24.468 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:24.497 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:24.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:24.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:24.541 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 67 16 104012 103996 96.9982 64.9375 0.0167308 0.0103039 2024-09-14T04:44:24.598 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:24.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:24.669 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:24.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:24.770 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:24.802 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:24.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:24.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:24.903 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:24.924 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:24.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:24.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:25.026 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:25.067 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:25.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:25.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:25.162 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:24 smithi039 ceph-mon[106470]: pgmap v2410: 99 pgs: 99 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-09-14T04:44:25.163 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:24 smithi039 ceph-mon[111235]: pgmap v2410: 99 pgs: 99 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-09-14T04:44:25.169 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:25.201 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:25.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:25.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:25.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:24 smithi129 ceph-mon[101316]: pgmap v2410: 99 pgs: 99 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s 2024-09-14T04:44:25.303 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:25.343 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:25.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:25.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:25.444 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:25.475 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:25.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:25.476 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:25.541 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 68 16 105019 105003 96.4973 62.9375 0.00727515 0.0103582 2024-09-14T04:44:25.577 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:25.614 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:25.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:25.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:25.716 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:25.773 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:25.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:25.774 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:25.875 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:25.906 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:25.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:25.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:26.007 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:26.040 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:26.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:26.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:26.142 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:26.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:26.194 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:26.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:26.295 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:26.322 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:26.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:26.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:26.424 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:26.452 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:26.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:26.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:26.528 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:44:26.542 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 69 16 105943 105927 95.9357 57.75 0.0267721 0.0104163 2024-09-14T04:44:26.554 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:26.559 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:44:26.579 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:26.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:26.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:26.681 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:26.711 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:26.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:26.712 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:26.813 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:26.849 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is unset 2024-09-14T04:44:26.869 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:26.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:26.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:26.971 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:27.001 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:27.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:27.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:27.069 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:26 smithi039 ceph-mon[106470]: pgmap v2411: 99 pgs: 99 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-09-14T04:44:27.069 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:26 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4141151262' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:44:27.070 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:26 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:44:27.070 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:26 smithi039 ceph-mon[111235]: pgmap v2411: 99 pgs: 99 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-09-14T04:44:27.070 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:26 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4141151262' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:44:27.070 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:26 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:44:27.103 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:27.129 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:27.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:27.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:27.231 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:27.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:26 smithi129 ceph-mon[101316]: pgmap v2411: 99 pgs: 99 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-09-14T04:44:27.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:26 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4141151262' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:44:27.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:26 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:44:27.277 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:27.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:27.279 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:27.379 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:27.421 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:27.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:27.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:27.523 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:27.541 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 70 16 106893 106877 95.4133 59.375 0.00955121 0.0104757 2024-09-14T04:44:27.553 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:27.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:27.554 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:27.655 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:27.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:27.698 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:27.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:27.799 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:27.827 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:27.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:27.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:27.877 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.2 to 1 (10000) 2024-09-14T04:44:27.929 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:27.958 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:27.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:27.959 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:28.060 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:28.093 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:28.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:28.094 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:28.195 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:28.226 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:28.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:28.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:28.246 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:27 smithi039 ceph-mon[106470]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:44:28.246 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:27 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:44:28.246 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:27 smithi039 ceph-mon[106470]: osdmap e1398: 8 total, 8 up, 8 in 2024-09-14T04:44:28.246 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:27 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3554363014' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:44:28.246 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:27 smithi039 ceph-mon[111235]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:44:28.246 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:27 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:44:28.246 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:27 smithi039 ceph-mon[111235]: osdmap e1398: 8 total, 8 up, 8 in 2024-09-14T04:44:28.246 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:27 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3554363014' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:44:28.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:27 smithi129 ceph-mon[101316]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:44:28.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:27 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:44:28.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:27 smithi129 ceph-mon[101316]: osdmap e1398: 8 total, 8 up, 8 in 2024-09-14T04:44:28.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:27 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3554363014' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:44:28.328 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:28.382 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:28.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:28.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:28.484 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:28.522 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:28.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:28.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:28.542 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 71 16 107949 107933 94.999 66 0.0277883 0.0105214 2024-09-14T04:44:28.624 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:28.653 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:28.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:28.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:28.755 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:28.785 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:28.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:28.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:28.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:44:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:44:28] "GET /metrics HTTP/1.1" 200 36741 "" "Prometheus/2.51.0" 2024-09-14T04:44:28.886 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:28.930 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:28.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:28.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:29.032 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:29.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:29.065 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:29.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:29.166 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:29.171 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:28 smithi129 ceph-mon[101316]: pgmap v2413: 99 pgs: 99 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-14T04:44:29.171 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:28 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3554363014' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-14T04:44:29.171 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:28 smithi129 ceph-mon[101316]: osdmap e1399: 8 total, 8 up, 8 in 2024-09-14T04:44:29.198 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:29.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:29.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:29.275 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:28 smithi039 ceph-mon[106470]: pgmap v2413: 99 pgs: 99 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-14T04:44:29.275 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:28 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3554363014' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-14T04:44:29.275 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:28 smithi039 ceph-mon[106470]: osdmap e1399: 8 total, 8 up, 8 in 2024-09-14T04:44:29.275 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:28 smithi039 ceph-mon[111235]: pgmap v2413: 99 pgs: 99 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-14T04:44:29.275 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:28 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3554363014' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-14T04:44:29.275 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:28 smithi039 ceph-mon[111235]: osdmap e1399: 8 total, 8 up, 8 in 2024-09-14T04:44:29.300 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:29.336 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:29.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:29.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:29.438 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:29.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:29.479 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:29.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:29.542 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 72 16 109054 109038 94.6387 69.0625 0.0117291 0.010561 2024-09-14T04:44:29.580 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:29.618 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:29.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:29.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:29.720 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:29.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:29.761 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:29.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:29.862 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:29.919 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:29.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:29.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:30.021 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:30.063 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:30.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:30.064 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:30.164 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:30.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:30.196 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:30.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:30.297 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:30.328 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:30.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:30.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:30.430 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:30.465 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:30.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:30.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:30.546 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 73 16 110096 110080 94.229 65.125 0.011976 0.0106072 2024-09-14T04:44:30.567 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:30.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:30.612 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:30.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:30.713 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:30.746 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:30.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:30.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:30.829 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 3 1 2024-09-14T04:44:30.847 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:30.879 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:30.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:30.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:30.981 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:31.048 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:30 smithi039 ceph-mon[106470]: pgmap v2415: 99 pgs: 99 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-14T04:44:31.049 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:30 smithi039 ceph-mon[111235]: pgmap v2415: 99 pgs: 99 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-14T04:44:31.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:31.058 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:31.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:31.159 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:31.212 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:31.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:31.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:31.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:30 smithi129 ceph-mon[101316]: pgmap v2415: 99 pgs: 99 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-14T04:44:31.314 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:31.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:31.354 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:31.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:31.455 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:31.490 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:31.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:31.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:31.546 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 74 16 111057 111041 93.7672 60.0625 0.00704947 0.0106594 2024-09-14T04:44:31.592 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:31.618 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:31.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:31.619 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:31.680 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:44:31.681 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:44:31.681 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:44:31.681 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:44:31.681 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:44:31.681 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:44:31.682 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:44:31.682 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:44:31.682 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:44:31.682 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:44:31.682 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:44:31.683 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:44:31.683 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:44:31.683 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:44:31.683 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:44:31.683 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:44:31.720 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:31.765 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:31.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:31.769 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:31.869 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:31.896 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:31.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:31.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:31.998 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:32.030 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:31 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/384481917' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:44:32.030 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:31 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/384481917' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:44:32.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:32.038 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:32.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:32.139 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:32.179 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:32.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:32.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:32.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:31 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/384481917' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:44:32.281 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:32.310 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:32.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:32.311 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:32.411 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:32.442 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:32.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:32.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:32.512 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:44:32.544 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:32.546 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 75 16 112113 112097 93.397 66 0.0143417 0.010702 2024-09-14T04:44:32.572 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:32.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:32.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:32.673 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:32.699 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:32.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:32.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:32.802 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:32.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:32.833 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:32.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:32.915 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:32 smithi039 ceph-mon[106470]: pgmap v2416: 99 pgs: 99 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s 2024-09-14T04:44:32.935 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:32.966 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:32.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:32.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:33.067 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:33.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:33.112 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:33.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:33.213 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:33.218 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:32 smithi039 ceph-mon[111235]: pgmap v2416: 99 pgs: 99 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s 2024-09-14T04:44:33.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:33.266 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:33.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:33.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:32 smithi129 ceph-mon[101316]: pgmap v2416: 99 pgs: 99 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 47 MiB/s wr, 748 op/s 2024-09-14T04:44:33.366 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:33.400 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:44:33.401 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:44:33.408 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:33.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:33.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:33.510 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:33.547 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 76 16 113091 113075 92.9714 61.125 0.00870774 0.010751 2024-09-14T04:44:33.549 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:33.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:33.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:33.650 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:33.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:33.684 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:33.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:33.785 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:33.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:33.828 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:33.829 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:33.932 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:33.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:33.977 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:33.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:34.025 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:44:34.079 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:34.111 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:34.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:34.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:34.213 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:34.246 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:34.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:34.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:34.348 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:34.388 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:34.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:34.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:34.400 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:44:34.490 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:34.518 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:34.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:34.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:34.547 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 77 16 114107 114091 92.5887 63.5 0.0199518 0.0107955 2024-09-14T04:44:34.619 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:34.644 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:34.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:34.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:34.647 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-14T04:44:34.746 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:34.793 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:34.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:34.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:34.895 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:34.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:34.962 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:34.963 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:35.063 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:35.101 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:35.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:35.102 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:35.203 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:35.234 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:35.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:35.235 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:35.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:34 smithi129 ceph-mon[101316]: pgmap v2417: 99 pgs: 99 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-14T04:44:35.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:34 smithi039 ceph-mon[106470]: pgmap v2417: 99 pgs: 99 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-14T04:44:35.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:34 smithi039 ceph-mon[111235]: pgmap v2417: 99 pgs: 99 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-14T04:44:35.336 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:35.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:35.361 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:35.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:35.462 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:35.503 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:35.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:35.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:35.547 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 78 16 115001 114985 92.1179 55.875 0.0123455 0.0108502 2024-09-14T04:44:35.605 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:35.626 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:35.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:35.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:35.728 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:35.765 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:35.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:35.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:35.867 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:35.899 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:35.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:35.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:36.001 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:36.022 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:36.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:36.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:36.124 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:36.160 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:36.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:36.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:36.262 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:36.288 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:36.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:36.289 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:36.389 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:36.419 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:36.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:36.420 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:36.521 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:36.547 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 79 16 115993 115977 91.7364 62 0.0121322 0.0108955 2024-09-14T04:44:36.552 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:36.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:36.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:36.577 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:44:36.654 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:36.671 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:44:36.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:36.680 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:36.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:36.781 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:36.808 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:36.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:36.809 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:36.910 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:36.946 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:36.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:36.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:37.010 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:36 smithi039 ceph-mon[106470]: pgmap v2418: 99 pgs: 99 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-14T04:44:37.011 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:36 smithi039 ceph-mon[111235]: pgmap v2418: 99 pgs: 99 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-14T04:44:37.048 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:37.084 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:37.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:37.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:37.186 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:37.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:37.233 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:37.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:37.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:36 smithi129 ceph-mon[101316]: pgmap v2418: 99 pgs: 99 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-14T04:44:37.334 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:37.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:37.367 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:37.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:37.468 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:37.496 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:37.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:37.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:37.548 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:2024-09-14T04:44:37.547875+0000 min lat: 0.00134858 max lat: 0.144682 avg lat: 0.0109416 2024-09-14T04:44:37.548 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:44:37.548 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 80 16 116968 116952 91.3514 60.9375 0.0177833 0.0109416 2024-09-14T04:44:37.598 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:37.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:37.640 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:37.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:37.741 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:37.772 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:37.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:37.773 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:37.873 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:37.907 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:37.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:37.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:38.009 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:38.047 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:38.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:38.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:38.149 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:38.178 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:38.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:38.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:44:38.280 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:38.289 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:44:38.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:44:38.302 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:38.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:38.303 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:38.403 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:38.445 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:38.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:38.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:38.546 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:38.547 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 81 16 117982 117966 91.006 63.375 0.0138394 0.0109835 2024-09-14T04:44:38.581 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:38.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:38.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:38.683 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:38.721 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:38.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:38.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:38.823 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:38.855 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:38.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:38.856 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:38.946 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:44:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:44:38] "GET /metrics HTTP/1.1" 200 36746 "" "Prometheus/2.51.0" 2024-09-14T04:44:38.957 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:39.007 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:39.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:39.008 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:39.109 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:39.135 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:39.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:39.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:39.203 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:38 smithi039 ceph-mon[106470]: pgmap v2419: 99 pgs: 99 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-14T04:44:39.204 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:38 smithi039 ceph-mon[111235]: pgmap v2419: 99 pgs: 99 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-14T04:44:39.236 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:39.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:38 smithi129 ceph-mon[101316]: pgmap v2419: 99 pgs: 99 active+clean; 7.0 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s 2024-09-14T04:44:39.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:39.292 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:39.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:39.393 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:39.436 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:39.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:39.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:39.538 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:39.548 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 82 16 118931 118915 90.6194 59.3125 0.0200331 0.0110299 2024-09-14T04:44:39.564 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:39.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:39.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:39.666 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:39.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:39.697 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:39.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:39.798 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:39.847 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:39.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:39.848 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:39.949 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:39.996 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:39.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:39.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:40.097 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:40.129 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:40.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:40.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:40.230 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:40.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:40.271 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:40.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:40.372 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:40.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:40.408 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:40.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:40.509 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:40.548 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 83 16 119875 119859 90.2384 59 0.0145416 0.0110764 2024-09-14T04:44:40.554 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:40.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:40.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:40.657 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:40.703 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:40.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:40.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:40.805 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:40.837 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:40.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:40.838 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:40.938 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:40.968 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:40.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:40.969 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:41.069 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:41.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:41.103 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:41.103 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:41.203 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:41.226 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:41.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:41.227 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:41.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:40 smithi129 ceph-mon[101316]: pgmap v2420: 99 pgs: 99 active+clean; 7.1 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-14T04:44:41.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:40 smithi039 ceph-mon[106470]: pgmap v2420: 99 pgs: 99 active+clean; 7.1 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-14T04:44:41.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:40 smithi039 ceph-mon[111235]: pgmap v2420: 99 pgs: 99 active+clean; 7.1 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 67 MiB/s wr, 1.07k op/s 2024-09-14T04:44:41.328 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:41.359 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:41.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:41.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:41.461 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:41.489 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:41.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:41.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:41.549 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 84 16 120786 120770 89.8412 56.9375 0.0136575 0.0111254 2024-09-14T04:44:41.590 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:41.626 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:41.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:41.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:41.728 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:41.763 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:41.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:41.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:41.855 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:44:41.864 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:41.889 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:41.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:41.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:41.991 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:42.038 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:42.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:42.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:42.139 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:42.171 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:42.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:42.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:42.272 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:42.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:42.324 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:42.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:42.425 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:42.455 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:42.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:42.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:42.549 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 85 16 121747 121731 89.4901 60.0625 0.0162913 0.0111685 2024-09-14T04:44:42.556 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:42.621 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:42.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:42.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:42.725 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:42.755 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:42.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:42.756 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:42.857 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:42.884 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:42.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:42.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:42.987 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:43.033 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:43.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:43.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:43.135 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:43.178 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:43.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:43.179 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:43.269 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:42 smithi039 ceph-mon[106470]: pgmap v2421: 99 pgs: 99 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 57 MiB/s wr, 915 op/s 2024-09-14T04:44:43.269 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:42 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:44:43.269 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:42 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:44:43.269 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:42 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:44:43.269 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:42 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:44:43.269 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:42 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:44:43.269 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:42 smithi039 ceph-mon[111235]: pgmap v2421: 99 pgs: 99 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 57 MiB/s wr, 915 op/s 2024-09-14T04:44:43.269 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:42 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:44:43.270 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:42 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:44:43.270 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:42 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:44:43.270 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:42 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:44:43.270 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:42 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:44:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:42 smithi129 ceph-mon[101316]: pgmap v2421: 99 pgs: 99 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 57 MiB/s wr, 915 op/s 2024-09-14T04:44:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:42 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:44:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:42 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:44:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:42 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:44:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:42 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:44:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:42 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:44:43.280 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:43.306 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:43.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:43.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:43.407 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:43.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:43.432 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:43.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:43.533 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:43.550 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 86 16 122611 122595 89.0769 54 0.0105658 0.0112203 2024-09-14T04:44:43.572 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:43.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:43.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:43.674 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:43.711 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:43.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:43.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:43.813 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:43.840 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:43.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:43.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:43.925 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is unset 2024-09-14T04:44:43.943 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:43.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:43.987 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:43.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:44.089 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:44.129 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:44.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:44.130 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:44.231 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:44.255 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:44.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:44.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:44.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:43 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3197928810' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:44:44.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:43 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:44:44.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:43 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3197928810' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:44:44.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:43 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:44:44.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:43 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3197928810' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:44:44.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:43 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:44:44.357 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:44.387 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:44:44.388 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:44:44.393 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:44.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:44.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:44.496 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:44.520 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:44.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:44.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:44.550 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 87 16 123436 123420 88.6457 51.5625 0.00408847 0.0112706 2024-09-14T04:44:44.622 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:44.662 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:44.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:44.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:44.764 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:44.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:44.788 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:44.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:44.889 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:44.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:44.928 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:44.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:45.029 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:45.057 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:45.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:45.058 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:45.159 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:45.200 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:45.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:45.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:45.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:44 smithi129 ceph-mon[101316]: pgmap v2422: 99 pgs: 99 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-14T04:44:45.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:44 smithi129 ceph-mon[101316]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:44:45.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:44 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T04:44:45.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:44 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:44:45.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:44 smithi129 ceph-mon[101316]: osdmap e1400: 8 total, 8 up, 8 in 2024-09-14T04:44:45.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:44 smithi039 ceph-mon[106470]: pgmap v2422: 99 pgs: 99 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-14T04:44:45.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:44 smithi039 ceph-mon[106470]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:44:45.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:44 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T04:44:45.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:44 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:44:45.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:44 smithi039 ceph-mon[106470]: osdmap e1400: 8 total, 8 up, 8 in 2024-09-14T04:44:45.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:44 smithi039 ceph-mon[111235]: pgmap v2422: 99 pgs: 99 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-14T04:44:45.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:44 smithi039 ceph-mon[111235]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:44:45.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:44 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T04:44:45.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:44 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:44:45.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:44 smithi039 ceph-mon[111235]: osdmap e1400: 8 total, 8 up, 8 in 2024-09-14T04:44:45.302 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:45.327 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:45.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:45.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:45.429 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:45.472 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:45.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:45.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:45.550 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 88 16 124390 124374 88.3159 59.625 0.020989 0.0113178 2024-09-14T04:44:45.574 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:45.605 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:45.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:45.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:45.707 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:45.727 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:45.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:45.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:45.829 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:45.855 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:45.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:45.856 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:45.957 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:45.986 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:45.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:45.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:46.088 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:46.133 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:46.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:46.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:46.235 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:46.265 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:46.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:46.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:46.367 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:46.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:46.400 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:46.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:46.500 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:46.533 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:46.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:46.534 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:46.550 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 89 16 125404 125388 88.0356 63.375 0.00890962 0.0113538 2024-09-14T04:44:46.636 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:46.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:46.676 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:46.676 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:46.777 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:46.805 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:46.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:46.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:46.906 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:46.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:46.943 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:46.943 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:47.044 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:47.076 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:47.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:47.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:47.179 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:47.195 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:46 smithi039 ceph-mon[106470]: pgmap v2424: 99 pgs: 99 active+clean; 7.5 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2024-09-14T04:44:47.195 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:46 smithi039 ceph-mon[111235]: pgmap v2424: 99 pgs: 99 active+clean; 7.5 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2024-09-14T04:44:47.217 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:47.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:47.218 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:47.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:46 smithi129 ceph-mon[101316]: pgmap v2424: 99 pgs: 99 active+clean; 7.5 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s 2024-09-14T04:44:47.319 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:47.349 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:47.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:47.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:47.451 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:47.484 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:47.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:47.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:47.552 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Total time run: 90.0104 2024-09-14T04:44:47.552 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Total writes made: 126394 2024-09-14T04:44:47.552 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Write size: 65536 2024-09-14T04:44:47.552 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Object size: 65536 2024-09-14T04:44:47.552 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Bandwidth (MB/sec): 87.7634 2024-09-14T04:44:47.552 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Stddev Bandwidth: 36.1819 2024-09-14T04:44:47.552 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Max bandwidth (MB/sec): 221 2024-09-14T04:44:47.552 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Min bandwidth (MB/sec): 51.5625 2024-09-14T04:44:47.552 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Average IOPS: 1404 2024-09-14T04:44:47.552 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Stddev IOPS: 578.91 2024-09-14T04:44:47.552 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Max IOPS: 3536 2024-09-14T04:44:47.552 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Min IOPS: 825 2024-09-14T04:44:47.553 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Average Latency(s): 0.0113896 2024-09-14T04:44:47.553 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Stddev Latency(s): 0.00882177 2024-09-14T04:44:47.553 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Max latency(s): 0.144682 2024-09-14T04:44:47.553 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Min latency(s): 0.00134858 2024-09-14T04:44:47.555 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Cleaning up (deleting benchmark objects) 2024-09-14T04:44:47.585 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:47.618 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:47.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:47.619 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:47.720 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:47.764 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:47.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:47.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:47.866 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:47.905 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:47.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:47.906 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:47.943 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.3 to 1 (10000) 2024-09-14T04:44:48.007 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:48.033 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:47 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4021559274' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:44:48.033 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:47 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4021559274' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:44:48.046 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:48.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:48.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:48.148 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:48.177 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:48.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:48.178 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:48.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:47 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4021559274' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:44:48.278 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:48.315 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:48.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:48.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:48.417 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:48.452 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:48.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:48.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:48.554 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:48.602 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:48.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:48.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:48.704 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:48.740 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:48.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:48.741 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:48.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:44:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:44:48] "GET /metrics HTTP/1.1" 200 36746 "" "Prometheus/2.51.0" 2024-09-14T04:44:48.842 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:48.869 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:48.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:48.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:48.971 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:49.007 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:49.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:49.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:49.109 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:49.150 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:49.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:49.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:49.215 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:48 smithi039 ceph-mon[106470]: pgmap v2425: 99 pgs: 99 active+clean; 7.5 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 59 MiB/s wr, 951 op/s 2024-09-14T04:44:49.215 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:48 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4021559274' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-14T04:44:49.215 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:48 smithi039 ceph-mon[106470]: osdmap e1401: 8 total, 8 up, 8 in 2024-09-14T04:44:49.215 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:48 smithi039 ceph-mon[111235]: pgmap v2425: 99 pgs: 99 active+clean; 7.5 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 59 MiB/s wr, 951 op/s 2024-09-14T04:44:49.215 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:48 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4021559274' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-14T04:44:49.216 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:48 smithi039 ceph-mon[111235]: osdmap e1401: 8 total, 8 up, 8 in 2024-09-14T04:44:49.252 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:49.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:48 smithi129 ceph-mon[101316]: pgmap v2425: 99 pgs: 99 active+clean; 7.5 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 59 MiB/s wr, 951 op/s 2024-09-14T04:44:49.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:48 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4021559274' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-14T04:44:49.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:48 smithi129 ceph-mon[101316]: osdmap e1401: 8 total, 8 up, 8 in 2024-09-14T04:44:49.289 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:49.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:49.290 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:49.391 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:49.431 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:49.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:49.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:49.533 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:49.575 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:49.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:49.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:49.677 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:49.682 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-14T04:44:49.694 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 4 1 2024-09-14T04:44:49.702 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:49.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:49.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:49.803 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:49.870 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:49.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:49.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:49.972 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:50.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:50.040 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:50.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:50.141 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:50.182 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:50.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:50.184 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:50.284 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:50.325 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:50.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:50.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:50.427 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:50.478 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:50.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:50.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:50.579 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:50.610 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:50.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:50.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:50.712 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:50.746 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:44:50.747 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:44:50.747 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:44:50.786 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:44:50.791 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:44:50.791 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:44:50.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:50.812 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:50.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:50.829 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:44:50.830 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:44:50.861 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:44:50.862 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:44:50.891 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:44:50.892 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:44:50.914 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:50.924 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:44:50.924 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:44:50.945 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:50.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:50.946 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:50.957 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:44:50.957 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:44:50.996 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:44:50.996 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:44:51.046 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:51.091 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:51.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:51.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:51.193 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:51.212 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:50 smithi039 ceph-mon[106470]: pgmap v2427: 99 pgs: 99 active+clean; 7.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 127 B/s rd, 64 MiB/s wr, 1.06k op/s 2024-09-14T04:44:51.212 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:50 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3074133252' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:44:51.213 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:50 smithi039 ceph-mon[111235]: pgmap v2427: 99 pgs: 99 active+clean; 7.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 127 B/s rd, 64 MiB/s wr, 1.06k op/s 2024-09-14T04:44:51.213 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:50 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3074133252' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:44:51.233 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:51.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:51.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:50 smithi129 ceph-mon[101316]: pgmap v2427: 99 pgs: 99 active+clean; 7.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 127 B/s rd, 64 MiB/s wr, 1.06k op/s 2024-09-14T04:44:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:50 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3074133252' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:44:51.335 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:51.366 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:51.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:51.367 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:51.468 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:51.496 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:51.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:51.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:51.598 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:51.628 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:51.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:51.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:51.730 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:51.756 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:51.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:51.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:51.859 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:51.889 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:51.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:51.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:51.991 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:52.036 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:52.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:52.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:52.138 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:52.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:52.178 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:52.178 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:52.279 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:52.316 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:52.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:52.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:52.419 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:52.445 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:44:52.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:52.449 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:52.449 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:52.550 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:52.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:52.592 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:52.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:52.694 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:52.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:52.724 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:52.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:52.825 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:52.864 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:52.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:52.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:52.966 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:53.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:53.004 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:53.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:53.105 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:53.141 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:53.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:53.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:53.243 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:53.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:53.285 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:53.286 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:53.386 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:53.415 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:53.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:53.416 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:53.516 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:53.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:53 smithi039 ceph-mon[106470]: pgmap v2428: 99 pgs: 99 active+clean; 7.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 127 B/s rd, 38 MiB/s wr, 703 op/s 2024-09-14T04:44:53.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:53 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:44:53.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:53 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:44:53.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:53 smithi039 ceph-mon[111235]: pgmap v2428: 99 pgs: 99 active+clean; 7.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 127 B/s rd, 38 MiB/s wr, 703 op/s 2024-09-14T04:44:53.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:53 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:44:53.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:53 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:44:53.554 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:53.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:53.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:53.656 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:53.690 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:53.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:53.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:53 smithi129 ceph-mon[101316]: pgmap v2428: 99 pgs: 99 active+clean; 7.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 127 B/s rd, 38 MiB/s wr, 703 op/s 2024-09-14T04:44:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:53 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:44:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:53 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:44:53.792 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:53.819 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:53.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:53.820 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:53.854 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:44:53.921 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:53.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:53.993 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:53.993 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:54.094 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:54.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:54.128 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:54.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:54.229 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:54.256 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:54.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:54.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:54.358 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:54.389 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:54.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:54.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:54.491 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:54.531 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:54.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:54.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:54.633 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:54.655 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:54.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:54.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:54.757 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:54.790 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:54.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:54.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:54.892 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:54.925 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:54.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:54.926 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:55.026 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:55.054 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:55.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:55.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:55.156 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:55.189 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:55.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:55.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:55.291 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:55.324 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:55.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:55.325 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:55.426 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:55.464 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:55.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:55.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:55.566 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:55.612 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:55 smithi039 ceph-mon[106470]: pgmap v2429: 99 pgs: 99 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 105 B/s rd, 32 MiB/s wr, 1.02k op/s 2024-09-14T04:44:55.612 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:55 smithi039 ceph-mon[111235]: pgmap v2429: 99 pgs: 99 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 105 B/s rd, 32 MiB/s wr, 1.02k op/s 2024-09-14T04:44:55.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:55.617 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:55.618 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:55.718 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:55.719 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:44:55.720 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [ 2024-09-14T04:44:55.720 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:55.720 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:135014 54.3 54:c9d5c994:::benchmark_data_smithi039_264334_object8617:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1401)", 2024-09-14T04:44:55.720 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:55.672497+0000", 2024-09-14T04:44:55.720 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.0044582229999999999, 2024-09-14T04:44:55.720 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0044925199999999998, 2024-09-14T04:44:55.720 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:44:55.720 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:44:55.720 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:44:55.720 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:44:55.720 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:44:55.721 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:44:55.721 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 135014 2024-09-14T04:44:55.721 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:55.721 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:44:55.721 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:55.721 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:44:55.721 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:55.672497+0000", 2024-09-14T04:44:55.721 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:44:55.721 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:55.721 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:55.721 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:44:55.721 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:55.672497+0000", 2024-09-14T04:44:55.721 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:44:55.722 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:55.722 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:55.722 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:44:55.722 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:55.672499+0000", 2024-09-14T04:44:55.722 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9080000000000002e-06 2024-09-14T04:44:55.722 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:55.722 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:55.722 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:44:55.722 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:55.672506+0000", 2024-09-14T04:44:55.722 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8789999999999997e-06 2024-09-14T04:44:55.722 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:55.722 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:55.722 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:44:55.723 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:55.672525+0000", 2024-09-14T04:44:55.723 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9403e-05 2024-09-14T04:44:55.723 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:55.723 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:55.723 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:44:55.723 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:55.672535+0000", 2024-09-14T04:44:55.723 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.1230000000000006e-06 2024-09-14T04:44:55.723 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:55.723 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:55.723 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:44:55.723 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:55.672608+0000", 2024-09-14T04:44:55.723 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.3251999999999996e-05 2024-09-14T04:44:55.723 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:55.723 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:55.724 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:44:55.724 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:55.672724+0000", 2024-09-14T04:44:55.724 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011671999999999999 2024-09-14T04:44:55.724 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:55.724 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:55.724 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 0,3", 2024-09-14T04:44:55.724 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:55.672840+0000", 2024-09-14T04:44:55.724 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011513799999999999 2024-09-14T04:44:55.724 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:44:55.724 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:44:55.724 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:44:55.724 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:55.724 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:55.725 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:135017 54.3 54:c21bad38:::benchmark_data_smithi039_264334_object8620:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1401)", 2024-09-14T04:44:55.725 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:55.676429+0000", 2024-09-14T04:44:55.725 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.00052629900000000004, 2024-09-14T04:44:55.725 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00064564799999999997, 2024-09-14T04:44:55.725 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:44:55.725 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:44:55.725 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:44:55.725 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:44:55.725 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:44:55.725 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:44:55.725 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 135017 2024-09-14T04:44:55.726 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:55.726 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:44:55.726 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:55.726 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:44:55.726 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:55.676429+0000", 2024-09-14T04:44:55.726 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:44:55.726 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:55.726 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:55.726 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:44:55.726 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:55.676429+0000", 2024-09-14T04:44:55.726 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:44:55.726 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:55.726 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:55.727 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:44:55.727 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:55.676431+0000", 2024-09-14T04:44:55.727 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5379999999999999e-06 2024-09-14T04:44:55.727 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:55.727 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:55.727 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:44:55.727 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:55.676436+0000", 2024-09-14T04:44:55.727 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4469999999999997e-06 2024-09-14T04:44:55.727 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:55.727 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:55.727 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:44:55.727 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:55.676453+0000", 2024-09-14T04:44:55.727 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6463000000000001e-05 2024-09-14T04:44:55.727 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:55.728 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:55.728 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:44:55.728 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:55.676467+0000", 2024-09-14T04:44:55.728 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.448e-05 2024-09-14T04:44:55.728 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:55.728 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:55.728 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:44:55.728 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:55.676526+0000", 2024-09-14T04:44:55.728 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.9048000000000001e-05 2024-09-14T04:44:55.729 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:55.729 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:55.729 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:44:55.729 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:55.676632+0000", 2024-09-14T04:44:55.729 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000105565 2024-09-14T04:44:55.729 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:44:55.729 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:44:55.729 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 0,3", 2024-09-14T04:44:55.729 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:55.676749+0000", 2024-09-14T04:44:55.729 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000117811 2024-09-14T04:44:55.729 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:44:55.729 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:44:55.729 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:44:55.730 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:44:55.730 INFO:teuthology.orchestra.run.smithi039.stdout: ], 2024-09-14T04:44:55.730 INFO:teuthology.orchestra.run.smithi039.stdout: "num_ops": 2 2024-09-14T04:44:55.730 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:44:55.767 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:55.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:55.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:55.776 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:55 smithi129 ceph-mon[101316]: pgmap v2429: 99 pgs: 99 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 105 B/s rd, 32 MiB/s wr, 1.02k op/s 2024-09-14T04:44:55.869 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:55.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:55.912 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:55.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:56.013 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:56.040 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:56.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:56.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:56.141 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:56.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:56.178 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:56.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:56.279 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:56.323 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:56.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:56.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:56.425 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:56.459 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:56.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:56.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:56.561 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:56.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:56.584 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:56.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:56.685 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:56.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:56.708 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:56.708 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:56.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:56 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4196449729' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:44:56.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:56 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:44:56.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:56 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4196449729' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:44:56.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:56 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:44:56.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:56 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4196449729' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:44:56.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:56 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:44:56.809 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:56.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:56.833 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:56.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:56.934 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:56.958 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:56.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:56.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:56.998 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.4 to 1 (10000) 2024-09-14T04:44:57.060 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:57.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:57.084 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:57.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:57.186 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:57.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:57.210 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:57.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:57.311 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:57.344 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:57.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:57.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:57.446 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:44:57.472 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:57.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:57.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:44:57.574 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:57.596 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:57.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:57.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:57.699 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:57.721 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:57.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:57.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:57 smithi129 ceph-mon[101316]: pgmap v2430: 99 pgs: 99 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 26 MiB/s wr, 1.00k op/s 2024-09-14T04:44:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:57 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:44:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:57 smithi129 ceph-mon[101316]: osdmap e1402: 8 total, 8 up, 8 in 2024-09-14T04:44:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:57 smithi129 ceph-mon[101316]: osdmap e1403: 8 total, 8 up, 8 in 2024-09-14T04:44:57.771 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:57 smithi039 ceph-mon[106470]: pgmap v2430: 99 pgs: 99 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 26 MiB/s wr, 1.00k op/s 2024-09-14T04:44:57.772 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:57 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:44:57.772 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:57 smithi039 ceph-mon[106470]: osdmap e1402: 8 total, 8 up, 8 in 2024-09-14T04:44:57.772 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:57 smithi039 ceph-mon[106470]: osdmap e1403: 8 total, 8 up, 8 in 2024-09-14T04:44:57.772 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:57 smithi039 ceph-mon[111235]: pgmap v2430: 99 pgs: 99 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 26 MiB/s wr, 1.00k op/s 2024-09-14T04:44:57.772 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:57 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:44:57.772 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:57 smithi039 ceph-mon[111235]: osdmap e1402: 8 total, 8 up, 8 in 2024-09-14T04:44:57.772 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:57 smithi039 ceph-mon[111235]: osdmap e1403: 8 total, 8 up, 8 in 2024-09-14T04:44:57.823 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:57.845 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:57.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:57.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:57.946 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:57.968 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:57.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:57.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:58.070 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:58.097 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:58.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:58.099 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:58.199 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:44:58.226 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:58.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:58.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:44:58.328 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:58.346 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 5 1 2024-09-14T04:44:58.353 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:58.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:58.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:58.455 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:58.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:58.498 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:58.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:58.599 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:58.628 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:58.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:58.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:58.730 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:44:58.766 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:58.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:58.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:44:58.869 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:44:58.877 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-14T04:44:58.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:58.924 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:58.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:44:58.958 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:44:58.975 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:44:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:44:58] "GET /metrics HTTP/1.1" 200 36747 "" "Prometheus/2.51.0" 2024-09-14T04:44:59.025 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:44:59.031 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:44:59.050 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:59.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:59.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:44:59.151 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:44:59.181 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:59.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:59.183 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:44:59.283 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:59.304 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:44:59.309 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:59.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:59.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:59.411 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:44:59.455 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:44:59.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:59.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:44:59.557 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:59.597 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:59 smithi039 ceph-mon[106470]: pgmap v2433: 99 pgs: 1 peering, 98 active+clean; 7.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 5.3 MiB/s wr, 925 op/s 2024-09-14T04:44:59.598 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:59 smithi039 ceph-mon[106470]: 54.0 starting backfill to osd.4 from (0'0,0'0] MAX to 1401'8408 2024-09-14T04:44:59.598 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:59 smithi039 ceph-mon[106470]: 54.8 starting backfill to osd.4 from (0'0,0'0] MAX to 1401'8499 2024-09-14T04:44:59.598 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:59 smithi039 ceph-mon[106470]: osdmap e1404: 8 total, 8 up, 8 in 2024-09-14T04:44:59.598 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:44:59 smithi039 ceph-mon[106470]: 54.c starting backfill to osd.4 from (0'0,0'0] MAX to 1401'8422 2024-09-14T04:44:59.598 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:59 smithi039 ceph-mon[111235]: pgmap v2433: 99 pgs: 1 peering, 98 active+clean; 7.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 5.3 MiB/s wr, 925 op/s 2024-09-14T04:44:59.598 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:59 smithi039 ceph-mon[111235]: 54.0 starting backfill to osd.4 from (0'0,0'0] MAX to 1401'8408 2024-09-14T04:44:59.598 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:59 smithi039 ceph-mon[111235]: 54.8 starting backfill to osd.4 from (0'0,0'0] MAX to 1401'8499 2024-09-14T04:44:59.598 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:59 smithi039 ceph-mon[111235]: osdmap e1404: 8 total, 8 up, 8 in 2024-09-14T04:44:59.598 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:44:59 smithi039 ceph-mon[111235]: 54.c starting backfill to osd.4 from (0'0,0'0] MAX to 1401'8422 2024-09-14T04:44:59.610 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:59.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:59.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:59.711 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:59.756 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:59.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:59.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:59 smithi129 ceph-mon[101316]: pgmap v2433: 99 pgs: 1 peering, 98 active+clean; 7.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 5.3 MiB/s wr, 925 op/s 2024-09-14T04:44:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:59 smithi129 ceph-mon[101316]: 54.0 starting backfill to osd.4 from (0'0,0'0] MAX to 1401'8408 2024-09-14T04:44:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:59 smithi129 ceph-mon[101316]: 54.8 starting backfill to osd.4 from (0'0,0'0] MAX to 1401'8499 2024-09-14T04:44:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:59 smithi129 ceph-mon[101316]: osdmap e1404: 8 total, 8 up, 8 in 2024-09-14T04:44:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:44:59 smithi129 ceph-mon[101316]: 54.c starting backfill to osd.4 from (0'0,0'0] MAX to 1401'8422 2024-09-14T04:44:59.859 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:44:59.892 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:44:59.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:44:59.893 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:44:59.994 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:00.034 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:00.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:00.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:00.136 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:00.175 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:00.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:00.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:00.277 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:00.314 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:00.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:00.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:00.416 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:00.451 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:00.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:00.452 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:00.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:00 smithi039 ceph-mon[106470]: osdmap e1405: 8 total, 8 up, 8 in 2024-09-14T04:45:00.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:00 smithi039 ceph-mon[106470]: osdmap e1406: 8 total, 8 up, 8 in 2024-09-14T04:45:00.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:00 smithi039 ceph-mon[111235]: osdmap e1405: 8 total, 8 up, 8 in 2024-09-14T04:45:00.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:00 smithi039 ceph-mon[111235]: osdmap e1406: 8 total, 8 up, 8 in 2024-09-14T04:45:00.553 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:00.593 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:00.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:00.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:00.695 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:00.713 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:00 smithi129 ceph-mon[101316]: osdmap e1405: 8 total, 8 up, 8 in 2024-09-14T04:45:00.713 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:00 smithi129 ceph-mon[101316]: osdmap e1406: 8 total, 8 up, 8 in 2024-09-14T04:45:00.730 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:00.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:00.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:00.832 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:00.860 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:00.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:00.861 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:00.962 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:01.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:01.009 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:01.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:01.110 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:01.146 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:01.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:01.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:01.248 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:01.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:01.289 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:01.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:01.390 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:01.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:01.447 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:01.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:01.548 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:01.592 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:01.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:01.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:01.694 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:01.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:01.728 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:01.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:01 smithi129 ceph-mon[101316]: pgmap v2436: 99 pgs: 3 remapped+peering, 1 peering, 95 active+clean; 7.2 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 36 B/s, 2 keys/s, 2 objects/s recovering 2024-09-14T04:45:01.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:01 smithi039 ceph-mon[111235]: pgmap v2436: 99 pgs: 3 remapped+peering, 1 peering, 95 active+clean; 7.2 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 36 B/s, 2 keys/s, 2 objects/s recovering 2024-09-14T04:45:01.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:01 smithi039 ceph-mon[106470]: pgmap v2436: 99 pgs: 3 remapped+peering, 1 peering, 95 active+clean; 7.2 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 36 B/s, 2 keys/s, 2 objects/s recovering 2024-09-14T04:45:01.828 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:01.868 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:01.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:01.869 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:01.970 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:02.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:02.004 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:02.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:02.104 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:02.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:02.146 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:02.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:02.247 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:02.288 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:02.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:02.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:02.391 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:02.427 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:02.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:02.428 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:02.529 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:02.558 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:02.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:02.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:02.660 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:02.697 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:02.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:02.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:02.799 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:02.802 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:45:02.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:02.829 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:02.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:02.930 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:02.958 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:02.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:02.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:03.059 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:03.089 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:03.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:03.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:03.191 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:03.217 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:03.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:03.218 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:03.319 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:03.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:03.365 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:03.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:03.466 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:03.505 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:03.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:03.506 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:03.607 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:03.653 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:03.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:03.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:03.741 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:03 smithi039 ceph-mon[106470]: pgmap v2438: 99 pgs: 1 active+recovering+remapped, 3 remapped+peering, 2 peering, 93 active+clean; 7.1 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 7309/350634 objects misplaced (2.085%); 237 KiB/s, 2 keys/s, 6 objects/s recovering 2024-09-14T04:45:03.741 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:03 smithi039 ceph-mon[111235]: pgmap v2438: 99 pgs: 1 active+recovering+remapped, 3 remapped+peering, 2 peering, 93 active+clean; 7.1 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 7309/350634 objects misplaced (2.085%); 237 KiB/s, 2 keys/s, 6 objects/s recovering 2024-09-14T04:45:03.755 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:03.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:03 smithi129 ceph-mon[101316]: pgmap v2438: 99 pgs: 1 active+recovering+remapped, 3 remapped+peering, 2 peering, 93 active+clean; 7.1 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 591 op/s; 7309/350634 objects misplaced (2.085%); 237 KiB/s, 2 keys/s, 6 objects/s recovering 2024-09-14T04:45:03.781 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:03.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:03.782 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:03.883 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:03.913 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:03.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:03.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:04.014 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:04.059 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:04.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:04.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:04.161 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:04.199 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:04.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:04.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:04.301 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:04.329 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:04.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:04.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:04.408 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is set 2024-09-14T04:45:04.431 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:04.482 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:04.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:04.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:04.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:04 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1088724355' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:45:04.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:04 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1088724355' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:45:04.583 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:04.620 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:04.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:04.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:04.722 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:04.753 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:04.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:04.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:04.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:04 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1088724355' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:45:04.855 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:04.881 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:04.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:04.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:04.983 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:05.013 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:05.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:05.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:05.115 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:05.157 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:05.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:05.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:05.259 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:05.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:05.310 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:05.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:05.368 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:45:05.368 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:45:05.410 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:05.412 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:45:05.412 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:45:05.422 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.5 to 1 (10000) 2024-09-14T04:45:05.446 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:45:05.447 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:45:05.453 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:05.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:05.458 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:05.489 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:45:05.489 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:45:05.523 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:45:05.524 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:45:05.556 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:45:05.556 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:45:05.560 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:05.596 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:45:05.596 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:45:05.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:05.597 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:05.598 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:05.637 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:45:05.637 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:45:05.699 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:05.727 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:05.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:05.728 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:05 smithi129 ceph-mon[101316]: pgmap v2439: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+remapped, 1 peering, 94 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 956 op/s; 28480/341091 objects misplaced (8.350%); 512 KiB/s, 1 keys/s, 9 objects/s recovering 2024-09-14T04:45:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:05 smithi129 ceph-mon[101316]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:45:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:05 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1088724355' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:45:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:05 smithi129 ceph-mon[101316]: osdmap e1407: 8 total, 8 up, 8 in 2024-09-14T04:45:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:05 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2327931034' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:45:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:05 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:45:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:05 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/277207287' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:45:05.784 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:05 smithi039 ceph-mon[106470]: pgmap v2439: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+remapped, 1 peering, 94 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 956 op/s; 28480/341091 objects misplaced (8.350%); 512 KiB/s, 1 keys/s, 9 objects/s recovering 2024-09-14T04:45:05.784 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:05 smithi039 ceph-mon[106470]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:45:05.784 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:05 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1088724355' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:45:05.784 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:05 smithi039 ceph-mon[106470]: osdmap e1407: 8 total, 8 up, 8 in 2024-09-14T04:45:05.784 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:05 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2327931034' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:45:05.785 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:05 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:45:05.785 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:05 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/277207287' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:45:05.785 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:05 smithi039 ceph-mon[111235]: pgmap v2439: 99 pgs: 3 active+remapped+backfill_wait, 1 active+recovering+remapped, 1 peering, 94 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 956 op/s; 28480/341091 objects misplaced (8.350%); 512 KiB/s, 1 keys/s, 9 objects/s recovering 2024-09-14T04:45:05.785 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:05 smithi039 ceph-mon[111235]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:45:05.785 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:05 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1088724355' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:45:05.785 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:05 smithi039 ceph-mon[111235]: osdmap e1407: 8 total, 8 up, 8 in 2024-09-14T04:45:05.785 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:05 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2327931034' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:45:05.785 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:05 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:45:05.785 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:05 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/277207287' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:45:05.829 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:05.859 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:05.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:05.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:05.961 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:05.999 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:06.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:06.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:06.101 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:06.132 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:06.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:06.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:06.233 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:06.270 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:06.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:06.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:06.371 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:06.406 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:06.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:06.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:06.507 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:06.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:06.557 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:06.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:06.658 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:06.694 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:06.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:06.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:06.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:06 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-14T04:45:06.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:06 smithi129 ceph-mon[101316]: osdmap e1408: 8 total, 8 up, 8 in 2024-09-14T04:45:06.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:06 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-14T04:45:06.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:06 smithi039 ceph-mon[106470]: osdmap e1408: 8 total, 8 up, 8 in 2024-09-14T04:45:06.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:06 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-14T04:45:06.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:06 smithi039 ceph-mon[111235]: osdmap e1408: 8 total, 8 up, 8 in 2024-09-14T04:45:06.795 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:06.835 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:06.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:06.836 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:06.937 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:06.974 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:06.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:06.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:07.076 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:07.117 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:07.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:07.118 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:07.219 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:07.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:07.262 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:07.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:07.364 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:07.401 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:07.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:07.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:07.503 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:07.542 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:07.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:07.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:07.645 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:07.670 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:07.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:07.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:07.748 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:07 smithi039 ceph-mon[106470]: pgmap v2442: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 28410/338445 objects misplaced (8.394%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:45:07.748 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:45:07.748 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:45:07.749 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:07 smithi039 ceph-mon[111235]: pgmap v2442: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 28410/338445 objects misplaced (8.394%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:45:07.749 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:45:07.749 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:45:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:07 smithi129 ceph-mon[101316]: pgmap v2442: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 28410/338445 objects misplaced (8.394%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:45:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:45:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:45:07.772 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:07.793 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 6 1 2024-09-14T04:45:07.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:07.812 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:07.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:07.913 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:07.943 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:07.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:07.945 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:08.045 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:08.073 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:08.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:08.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:08.175 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:08.222 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:08.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:08.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:08.324 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:08.350 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:08.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:08.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:08.451 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:08.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:08.497 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:08.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:08.598 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:08.631 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:08.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:08.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:08.733 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:08.763 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:08.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:08.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:08.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:45:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:45:08] "GET /metrics HTTP/1.1" 200 36756 "" "Prometheus/2.51.0" 2024-09-14T04:45:08.865 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:08.900 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:08.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:08.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:09.003 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:09.017 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:45:09.018 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:45:09.031 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:09.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:09.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:09.133 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:09.173 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:09.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:09.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:09.193 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:45:09.276 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:09.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:09.338 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:09.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:09.439 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:09.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:09.467 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:09.467 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:09.535 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:09 smithi039 ceph-mon[106470]: pgmap v2443: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 28094/335055 objects misplaced (8.385%); 893 KiB/s, 13 objects/s recovering 2024-09-14T04:45:09.536 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:09 smithi039 ceph-mon[111235]: pgmap v2443: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 28094/335055 objects misplaced (8.385%); 893 KiB/s, 13 objects/s recovering 2024-09-14T04:45:09.567 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:09.599 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:09.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:09.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:09.701 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:09.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:09.738 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:09.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:09.766 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-14T04:45:09.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:09 smithi129 ceph-mon[101316]: pgmap v2443: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 6.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 710 op/s; 28094/335055 objects misplaced (8.385%); 893 KiB/s, 13 objects/s recovering 2024-09-14T04:45:09.823 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:45:09.838 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:09.864 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:09.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:09.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:09.966 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:10.044 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:10.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:10.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:10.146 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:10.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:10.205 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:10.205 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:10.306 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:10.340 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:10.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:10.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:10.442 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:10.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:10.483 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:10.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:10.584 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:10.617 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:10.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:10.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:10.676 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:45:10.719 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:10.733 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:45:10.744 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:10.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:10.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:10.846 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:10.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:10.885 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:10.885 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:10.931 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:45:10.986 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:11.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:11.041 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:11.042 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:11.142 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:11.190 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:11.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:11.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:11.291 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:11.336 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:11.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:11.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:11.438 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:11.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:11.483 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:11.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:11.583 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:11.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:11.620 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:11.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:11.721 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:11.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:11.753 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:11.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:11.757 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:11 smithi039 ceph-mon[106470]: pgmap v2444: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 953 op/s; 27588/327735 objects misplaced (8.418%); 816 KiB/s, 12 objects/s recovering 2024-09-14T04:45:11.758 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:11 smithi039 ceph-mon[111235]: pgmap v2444: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 953 op/s; 27588/327735 objects misplaced (8.418%); 816 KiB/s, 12 objects/s recovering 2024-09-14T04:45:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:11 smithi129 ceph-mon[101316]: pgmap v2444: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 6.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 953 op/s; 27588/327735 objects misplaced (8.418%); 816 KiB/s, 12 objects/s recovering 2024-09-14T04:45:11.854 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:11.884 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:11.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:11.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:11.986 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:12.019 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:12.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:12.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:12.121 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:12.151 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:12.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:12.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:12.253 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:12.293 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:12.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:12.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:12.395 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:12.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:12.426 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:12.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:12.527 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:12.561 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:12.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:12.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:12.663 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:12.695 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:12.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:12.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:12.797 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:12.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:12.848 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:12.849 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:12.949 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:12.984 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:12.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:12.985 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:13.086 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:13.117 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:13.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:13.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:13.219 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:13.251 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:13.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:13.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:13.353 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:13.389 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:13.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:13.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:13.491 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:13.516 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.6 to 1 (10000) 2024-09-14T04:45:13.531 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:13.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:13.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:13.633 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:13.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:13.673 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:13.674 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:13.774 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:13.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:13 smithi039 ceph-mon[106470]: pgmap v2445: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 27521/325386 objects misplaced (8.458%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:45:13.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:13 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3576686240' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:45:13.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:13 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:45:13.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:13 smithi039 ceph-mon[111235]: pgmap v2445: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 27521/325386 objects misplaced (8.458%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:45:13.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:13 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3576686240' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:45:13.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:13 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:45:13.808 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:13.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:13.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:13.910 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:13.947 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:13.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:13.949 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:14.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:13 smithi129 ceph-mon[101316]: pgmap v2445: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 653 op/s; 27521/325386 objects misplaced (8.458%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:45:14.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:13 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3576686240' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:45:14.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:13 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:45:14.049 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:14.087 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:14.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:14.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:14.190 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:14.230 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:14.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:14.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:14.331 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:14.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:14.363 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:14.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:14.464 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:14.494 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:14.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:14.495 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:14.596 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:14.635 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:14.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:14.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:14.739 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:14.774 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:14.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:14.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:14.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:14 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-14T04:45:14.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:14 smithi039 ceph-mon[106470]: osdmap e1409: 8 total, 8 up, 8 in 2024-09-14T04:45:14.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:14 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-14T04:45:14.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:14 smithi039 ceph-mon[111235]: osdmap e1409: 8 total, 8 up, 8 in 2024-09-14T04:45:14.876 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:14.907 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:14.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:14.908 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:15.009 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:15.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:14 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-14T04:45:15.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:14 smithi129 ceph-mon[101316]: osdmap e1409: 8 total, 8 up, 8 in 2024-09-14T04:45:15.037 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:15.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:15.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:15.139 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:15.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:15.180 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:15.180 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:15.281 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:15.313 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:15.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:15.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:15.415 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:15.439 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:15.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:15.440 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:15.540 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:15.575 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:15.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:15.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:15.677 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:15.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:15.709 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:15.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:15.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:15 smithi039 ceph-mon[106470]: pgmap v2447: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 942 op/s; 26674/315183 objects misplaced (8.463%); 521 KiB/s, 8 objects/s recovering 2024-09-14T04:45:15.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:15 smithi039 ceph-mon[111235]: pgmap v2447: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 942 op/s; 26674/315183 objects misplaced (8.463%); 521 KiB/s, 8 objects/s recovering 2024-09-14T04:45:15.810 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:15.841 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:15.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:15.842 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:15.923 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:45:15.943 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:15.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:15.995 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:15.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:16.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:15 smithi129 ceph-mon[101316]: pgmap v2447: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 942 op/s; 26674/315183 objects misplaced (8.463%); 521 KiB/s, 8 objects/s recovering 2024-09-14T04:45:16.096 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:16.127 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:16.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:16.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:16.230 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:16.271 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:16.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:16.272 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:16.373 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:16.407 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 7 1 2024-09-14T04:45:16.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:16.426 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:16.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:16.527 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:16.561 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:16.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:16.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:16.663 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:16.712 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:16.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:16.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:16.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:16 smithi039 ceph-mon[106470]: pgmap v2448: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 856 op/s; 26589/312723 objects misplaced (8.502%); 972 KiB/s, 15 objects/s recovering 2024-09-14T04:45:16.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:16 smithi039 ceph-mon[111235]: pgmap v2448: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 856 op/s; 26589/312723 objects misplaced (8.502%); 972 KiB/s, 15 objects/s recovering 2024-09-14T04:45:16.815 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:16.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:16.855 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:16.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:16.956 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:16.991 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:16.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:16.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:16 smithi129 ceph-mon[101316]: pgmap v2448: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 6.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 856 op/s; 26589/312723 objects misplaced (8.502%); 972 KiB/s, 15 objects/s recovering 2024-09-14T04:45:17.092 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:17.129 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:17.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:17.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:17.231 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:17.267 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:17.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:17.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:17.369 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:17.407 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:17.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:17.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:17.509 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:17.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:17.540 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:17.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:17.641 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:17.675 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:17.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:17.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:17.777 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:17.807 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:17.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:17.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:17.909 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:17.944 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:17.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:17.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:18.046 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:18.087 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:18.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:18.089 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:18.190 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:18.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:18.223 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:18.223 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:18.324 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:18.353 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:18.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:18.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:18.455 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:18.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:18.493 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:18.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:18.594 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:18.636 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:18.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:18.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:18.703 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:45:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:45:18] "GET /metrics HTTP/1.1" 200 36756 "" "Prometheus/2.51.0" 2024-09-14T04:45:18.738 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:18.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:18.770 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:18.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:18.871 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:18.912 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:18.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:18.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:19.014 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:19.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:18 smithi129 ceph-mon[101316]: pgmap v2449: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 849 op/s; 26308/309549 objects misplaced (8.499%); 972 KiB/s, 15 objects/s recovering 2024-09-14T04:45:19.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:18 smithi039 ceph-mon[106470]: pgmap v2449: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 849 op/s; 26308/309549 objects misplaced (8.499%); 972 KiB/s, 15 objects/s recovering 2024-09-14T04:45:19.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:18 smithi039 ceph-mon[111235]: pgmap v2449: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 849 op/s; 26308/309549 objects misplaced (8.499%); 972 KiB/s, 15 objects/s recovering 2024-09-14T04:45:19.053 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:19.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:19.054 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:19.154 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:19.185 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:19.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:19.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:19.287 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:19.308 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:19.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:19.309 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:19.410 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:19.447 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:19.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:19.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:19.550 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:19.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:19.574 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:19.574 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:19.675 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:19.699 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:19.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:19.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:19.801 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:19.822 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:19.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:19.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:19.923 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:19.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:19.961 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:19.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:20.062 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:20.094 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:20.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:20.095 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:20.196 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:20.226 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:20.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:20.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:20.328 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:20.367 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:20.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:20.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:20.469 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:20.501 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:20.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:20.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:20.602 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:20.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:20.627 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:20.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:20.728 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:20.760 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:20.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:20.760 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:20.861 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:20.887 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:20.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:20.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:20.988 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:21.010 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:21.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:21.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:21.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:20 smithi129 ceph-mon[101316]: pgmap v2450: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 25811/302373 objects misplaced (8.536%); 972 KiB/s, 15 objects/s recovering 2024-09-14T04:45:21.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:20 smithi039 ceph-mon[106470]: pgmap v2450: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 25811/302373 objects misplaced (8.536%); 972 KiB/s, 15 objects/s recovering 2024-09-14T04:45:21.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:20 smithi039 ceph-mon[111235]: pgmap v2450: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 6.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 25811/302373 objects misplaced (8.536%); 972 KiB/s, 15 objects/s recovering 2024-09-14T04:45:21.113 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:21.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:21.148 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:21.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:21.249 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:21.277 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:21.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:21.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:21.379 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:21.413 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:21.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:21.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:21.515 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:21.538 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:21.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:21.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:21.639 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:21.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:21.686 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:21.686 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:21.787 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:21.823 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:21.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:21.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:21.925 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:21.947 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:21.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:21.948 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:22.049 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:22.079 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:22.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:22.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:22.181 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:22.228 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:22.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:22.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:22.330 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:22.356 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:22.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:22.357 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:22.458 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:22.495 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:22.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:22.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:22.597 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:22.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:22.633 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:22.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:22.735 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:22.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:22.765 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:22.765 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:22.781 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:22 smithi039 ceph-mon[106470]: pgmap v2451: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 25723/300048 objects misplaced (8.573%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:45:22.781 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:45:22.782 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:22 smithi039 ceph-mon[111235]: pgmap v2451: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 25723/300048 objects misplaced (8.573%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:45:22.782 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:45:22.866 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:22.905 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:22.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:22.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:23.007 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:23.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:22 smithi129 ceph-mon[101316]: pgmap v2451: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 6.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 844 op/s; 25723/300048 objects misplaced (8.573%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:45:23.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:45:23.064 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:23.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:23.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:23.166 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:23.217 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:23.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:23.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:23.319 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:23.356 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:23.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:23.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:23.458 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:23.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:23.494 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:23.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:23.595 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:23.629 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:23.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:23.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:23.731 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:23.757 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is set 2024-09-14T04:45:23.795 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:23.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:23.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:23.897 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:23.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:23.939 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:23.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:24.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:23 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/258038641' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:45:24.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:23 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:45:24.033 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:23 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/258038641' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:45:24.033 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:23 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:45:24.033 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:23 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/258038641' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:45:24.033 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:23 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:45:24.033 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:45:24.034 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:45:24.040 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:24.065 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:24.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:24.066 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:24.167 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:24.199 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:24.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:24.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:24.301 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:24.331 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:24.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:24.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:24.433 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:24.473 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:24.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:24.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:24.575 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:24.614 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:24.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:24.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:24.715 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:24.751 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:24.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:24.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:24.837 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:45:24.838 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:45:24.853 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:24.875 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:45:24.876 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:45:24.891 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:24.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:24.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:24.908 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:45:24.908 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:45:24.944 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:45:24.944 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:45:24.974 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:45:24.974 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:45:24.993 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:25.020 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:45:25.020 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:45:25.020 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:24 smithi039 ceph-mon[111235]: pgmap v2452: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 796 op/s; 25023/290967 objects misplaced (8.600%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:45:25.020 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:24 smithi039 ceph-mon[111235]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:45:25.021 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:24 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:45:25.021 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:24 smithi039 ceph-mon[111235]: osdmap e1410: 8 total, 8 up, 8 in 2024-09-14T04:45:25.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:24 smithi129 ceph-mon[101316]: pgmap v2452: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 796 op/s; 25023/290967 objects misplaced (8.600%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:45:25.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:24 smithi129 ceph-mon[101316]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:45:25.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:24 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:45:25.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:24 smithi129 ceph-mon[101316]: osdmap e1410: 8 total, 8 up, 8 in 2024-09-14T04:45:25.022 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:24 smithi039 ceph-mon[106470]: pgmap v2452: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 796 op/s; 25023/290967 objects misplaced (8.600%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:45:25.022 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:24 smithi039 ceph-mon[106470]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:45:25.022 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:24 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:45:25.022 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:24 smithi039 ceph-mon[106470]: osdmap e1410: 8 total, 8 up, 8 in 2024-09-14T04:45:25.029 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:25.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:25.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:25.061 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:45:25.062 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:45:25.099 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:45:25.099 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:45:25.131 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:25.186 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:25.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:25.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:25.288 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:25.315 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:25.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:25.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:25.417 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:25.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:25.449 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:25.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:25.550 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:25.573 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:25.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:25.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:25.675 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:25.701 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:25.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:25.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:25.803 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:25.835 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:25.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:25.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:25.937 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:25.972 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:25.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:25.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:25 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1992318042' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:45:26.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:25 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1992318042' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:45:26.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:25 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1992318042' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:45:26.074 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:26.116 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:26.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:26.117 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:26.150 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:45:26.218 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:26.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:26.260 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:26.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:26.361 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:26.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:26.396 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:26.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:26.497 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:26.533 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:26.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:26.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:26.635 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:26.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:26.672 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:26.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:26.735 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-14T04:45:26.773 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:26.837 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:26.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:26.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:26.939 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:26.966 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:26 smithi039 ceph-mon[106470]: pgmap v2454: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 24941/288669 objects misplaced (8.640%); 1.1 MiB/s, 16 objects/s recovering 2024-09-14T04:45:26.966 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:26 smithi039 ceph-mon[111235]: pgmap v2454: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 24941/288669 objects misplaced (8.640%); 1.1 MiB/s, 16 objects/s recovering 2024-09-14T04:45:26.967 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:26.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:26.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:27.069 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:27.114 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:27.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:27.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:27.174 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:45:27.216 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:27.270 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:27.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:26 smithi129 ceph-mon[101316]: pgmap v2454: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 5.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 24941/288669 objects misplaced (8.640%); 1.1 MiB/s, 16 objects/s recovering 2024-09-14T04:45:27.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:27.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:27.372 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:27.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:27.434 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:27.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:27.536 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:27.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:27.566 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:27.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:27.667 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:27.699 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:27.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:27.700 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:27.801 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:27.836 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:27.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:27.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:27.937 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:27.963 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:27.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:27.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:28.065 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:28.100 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:28.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:28.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:28.202 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:28.239 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:28.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:28.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:28.341 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:28.374 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:28.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:28.375 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:28.475 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:28.515 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:28.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:28.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:28.617 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:28.650 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:28.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:28.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:28.752 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:28.779 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:28.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:28.780 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:28.881 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:28.911 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:28.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:28.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:29.013 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:29.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:28 smithi039 ceph-mon[106470]: pgmap v2455: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 24650/285399 objects misplaced (8.637%); 1.1 MiB/s, 16 objects/s recovering 2024-09-14T04:45:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:45:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:45:28] "GET /metrics HTTP/1.1" 200 36750 "" "Prometheus/2.51.0" 2024-09-14T04:45:29.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:28 smithi039 ceph-mon[111235]: pgmap v2455: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 24650/285399 objects misplaced (8.637%); 1.1 MiB/s, 16 objects/s recovering 2024-09-14T04:45:29.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:29.041 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:29.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:29.072 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:28 smithi129 ceph-mon[101316]: pgmap v2455: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 5.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 24650/285399 objects misplaced (8.637%); 1.1 MiB/s, 16 objects/s recovering 2024-09-14T04:45:29.143 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:29.170 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:29.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:29.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:29.173 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:45:29.272 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:29.310 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:29.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:29.311 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:29.411 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:29.444 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:29.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:29.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:29.451 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:45:29.546 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:29.586 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:29.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:29.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:29.688 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:29.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:29.729 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:29.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:29.806 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:45:29.830 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:29.900 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:29.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:29.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:30.002 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:30.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:30.038 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:30.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:30.139 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:30.169 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:30.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:30.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:30.270 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:30.304 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:30.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:30.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:30.406 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:30.452 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:30.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:30.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:30.554 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:30.579 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:30.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:30.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:30.681 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:30.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:30.712 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:30.713 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:30.813 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:30.852 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:30.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:30.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:30.955 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:30.990 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:30.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:30.991 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:31.092 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:31.124 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:31.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:31.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:31.226 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:31.254 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:30 smithi039 ceph-mon[106470]: pgmap v2456: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 24086/278319 objects misplaced (8.654%); 1.1 MiB/s, 16 objects/s recovering 2024-09-14T04:45:31.255 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:30 smithi039 ceph-mon[111235]: pgmap v2456: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 24086/278319 objects misplaced (8.654%); 1.1 MiB/s, 16 objects/s recovering 2024-09-14T04:45:31.267 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:31.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:31.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:31.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:30 smithi129 ceph-mon[101316]: pgmap v2456: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 5.7 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 24086/278319 objects misplaced (8.654%); 1.1 MiB/s, 16 objects/s recovering 2024-09-14T04:45:31.369 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:31.397 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:31.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:31.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:31.499 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:31.523 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:31.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:31.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:31.626 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:31.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:31.673 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:31.673 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:31.774 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:31.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:31.821 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:31.821 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:31.922 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:31.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:31.964 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:31.965 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:32.065 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:32.099 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:32.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:32.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:32.201 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:32.244 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:32.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:32.245 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:32.346 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:32.393 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:32.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:32.394 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:32.495 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:32.530 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:32.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:32.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:32.632 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:32.662 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:32.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:32.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:32.764 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:32.804 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:32.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:32.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:32.906 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:32.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:32.939 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:32.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:33.040 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:33.082 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:33.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:33.083 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:33.122 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:32 smithi039 ceph-mon[106470]: pgmap v2457: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 23994/276072 objects misplaced (8.691%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:45:33.122 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:32 smithi039 ceph-mon[111235]: pgmap v2457: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 23994/276072 objects misplaced (8.691%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:45:33.184 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:33.211 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:33.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:33.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:33.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:32 smithi129 ceph-mon[101316]: pgmap v2457: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 5.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 798 op/s; 23994/276072 objects misplaced (8.691%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:45:33.313 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:33.351 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:33.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:33.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:33.453 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:33.496 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:33.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:33.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:33.598 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:33.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:33.649 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:33.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:33.750 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:33.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:33.775 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:33.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:33.877 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:33.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:33.922 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:33.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:34.023 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:34.057 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:34.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:34.058 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:34.159 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:34.223 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:34.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:34.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:34.325 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:34.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:34.385 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:34.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:34.486 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:34.534 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:34.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:34.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:34.636 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:34.682 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:34.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:34.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:34.720 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:45:34.776 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:45:34.776 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [], 2024-09-14T04:45:34.777 INFO:teuthology.orchestra.run.smithi039.stdout: "num_ops": 0 2024-09-14T04:45:34.777 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:45:34.784 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:34.813 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:34.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:34.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:34.824 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.7 to 1 (10000) 2024-09-14T04:45:34.851 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:34 smithi039 ceph-mon[106470]: pgmap v2458: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 23183/265602 objects misplaced (8.728%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:45:34.852 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:34 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3451116321' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:45:34.852 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:34 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:45:34.852 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:34 smithi039 ceph-mon[111235]: pgmap v2458: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 23183/265602 objects misplaced (8.728%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:45:34.852 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:34 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3451116321' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:45:34.852 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:34 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:45:34.915 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:34.956 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:34.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:34.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:35.058 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:35.108 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:35.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:35.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:35.210 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:35.248 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:35.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:35.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:35.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:34 smithi129 ceph-mon[101316]: pgmap v2458: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 5.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 845 op/s; 23183/265602 objects misplaced (8.728%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:45:35.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:34 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3451116321' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:45:35.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:34 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:45:35.352 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:35.386 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:35.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:35.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:35.488 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:35.520 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:35.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:35.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:35.622 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:35.659 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:35.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:35.660 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:35.760 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:35.795 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:35.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:35.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:35.875 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-14T04:45:35.876 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:45:35.897 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:35.928 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:35.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:35.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:36.029 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:36.078 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:36.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:36.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:36.102 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:35 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-14T04:45:36.102 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:35 smithi039 ceph-mon[106470]: osdmap e1411: 8 total, 8 up, 8 in 2024-09-14T04:45:36.102 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:35 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-14T04:45:36.102 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:35 smithi039 ceph-mon[111235]: osdmap e1411: 8 total, 8 up, 8 in 2024-09-14T04:45:36.180 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:36.224 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:36.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:36.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:36.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:35 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-14T04:45:36.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:35 smithi129 ceph-mon[101316]: osdmap e1411: 8 total, 8 up, 8 in 2024-09-14T04:45:36.326 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:36.373 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:36.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:36.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:36.474 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:36.512 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:36.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:36.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:36.614 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:36.651 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:36.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:36.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:36.753 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:36.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:36.787 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:36.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:36.888 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:36.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:36.928 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:36.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:37.029 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:37.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:37.078 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:37.078 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:37.179 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:37.216 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:37.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:37.218 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:37.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:36 smithi129 ceph-mon[101316]: pgmap v2460: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 848 op/s; 23095/263193 objects misplaced (8.775%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:45:37.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:36 smithi039 ceph-mon[106470]: pgmap v2460: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 848 op/s; 23095/263193 objects misplaced (8.775%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:45:37.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:36 smithi039 ceph-mon[111235]: pgmap v2460: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 5.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 848 op/s; 23095/263193 objects misplaced (8.775%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:45:37.319 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:37.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:37.369 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:37.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:37.471 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:37.512 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:37.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:37.513 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:37.614 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:37.648 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:37.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:37.649 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:37.750 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:37.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:37.788 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:37.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:37.881 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is unset 2024-09-14T04:45:37.889 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:37.916 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:37.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:37.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:37.942 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-14T04:45:38.018 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:38.060 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:38.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:38.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:38.162 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:38.186 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:38.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:38.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:38.252 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:45:38.252 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:37 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2444143722' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:45:38.252 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:37 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:45:38.253 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:45:38.253 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:37 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2444143722' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:45:38.253 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:37 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:45:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:45:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:37 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2444143722' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:45:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:37 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:45:38.288 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:38.317 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:38.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:38.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:38.419 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:38.452 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:38.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:38.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:38.554 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:38.599 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:38.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:38.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:38.701 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:38.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:38.757 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:38.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:38.858 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:38.882 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:45:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:45:38] "GET /metrics HTTP/1.1" 200 36751 "" "Prometheus/2.51.0" 2024-09-14T04:45:38.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:38.908 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:38.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:39.009 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:39.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:39.049 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:39.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:39.121 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:45:39.150 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:39.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:39.202 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:39.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:39.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:38 smithi129 ceph-mon[101316]: pgmap v2461: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 22827/260196 objects misplaced (8.773%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:45:39.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:38 smithi129 ceph-mon[101316]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:45:39.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:38 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:45:39.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:38 smithi129 ceph-mon[101316]: osdmap e1412: 8 total, 8 up, 8 in 2024-09-14T04:45:39.280 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:38 smithi039 ceph-mon[106470]: pgmap v2461: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 22827/260196 objects misplaced (8.773%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:45:39.280 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:38 smithi039 ceph-mon[106470]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:45:39.281 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:38 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:45:39.281 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:38 smithi039 ceph-mon[106470]: osdmap e1412: 8 total, 8 up, 8 in 2024-09-14T04:45:39.281 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:38 smithi039 ceph-mon[111235]: pgmap v2461: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 5.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 839 op/s; 22827/260196 objects misplaced (8.773%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:45:39.281 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:38 smithi039 ceph-mon[111235]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:45:39.281 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:38 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:45:39.281 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:38 smithi039 ceph-mon[111235]: osdmap e1412: 8 total, 8 up, 8 in 2024-09-14T04:45:39.303 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:39.346 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:39.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:39.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:39.448 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:39.479 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:39.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:39.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:39.581 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:39.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:39.641 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:39.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:39.742 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:39.786 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:39.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:39.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:39.887 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:39.916 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:39.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:39.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:40.018 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:40.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:40.060 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:40.060 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:40.160 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:40.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:40.200 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:40.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:40.301 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:40.326 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:40.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:40.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:40.429 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:40.434 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:45:40.434 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:45:40.434 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:45:40.434 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:45:40.434 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:45:40.434 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:45:40.435 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:45:40.435 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:45:40.435 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:45:40.435 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:45:40.435 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:45:40.435 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:45:40.435 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:45:40.435 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:45:40.449 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:45:40.450 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:45:40.471 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:40.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:40.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:40.573 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:40.610 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:40.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:40.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:40.617 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:45:40.711 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:40.737 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:40.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:40.739 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:40.840 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:40.881 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:40.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:40.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:40.983 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:41.019 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:41.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:41.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:41.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:40 smithi039 ceph-mon[106470]: pgmap v2463: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 956 op/s; 22328/253119 objects misplaced (8.821%); 704 KiB/s, 10 objects/s recovering 2024-09-14T04:45:41.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:40 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/237531850' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:45:41.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:40 smithi039 ceph-mon[111235]: pgmap v2463: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 956 op/s; 22328/253119 objects misplaced (8.821%); 704 KiB/s, 10 objects/s recovering 2024-09-14T04:45:41.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:40 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/237531850' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:45:41.122 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:41.153 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:41.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:41.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:41.191 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:45:41.255 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:41.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:40 smithi129 ceph-mon[101316]: pgmap v2463: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 956 op/s; 22328/253119 objects misplaced (8.821%); 704 KiB/s, 10 objects/s recovering 2024-09-14T04:45:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:40 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/237531850' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:45:41.285 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:41.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:41.286 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:41.387 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:41.418 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:41.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:41.419 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:41.519 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:41.548 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:41.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:41.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:41.649 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:41.687 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:41.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:41.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:41.789 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:41.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:41.833 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:41.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:41.934 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:41.965 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:41.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:41.967 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:42.067 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:42.096 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:42.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:42.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:42.198 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:42.230 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:42.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:42.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:42.332 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:42.358 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:42.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:42.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:42.460 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:42.506 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:42.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:42.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:42.608 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:42.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:42.647 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:42.647 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:42.748 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:42.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:42.786 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:42.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:42.886 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:42.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:42.921 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:42.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:43.022 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:43.069 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:43.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:43.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:43.171 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:43.218 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:43.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:43.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:43.235 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:45:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:42 smithi129 ceph-mon[101316]: pgmap v2464: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 22235/250998 objects misplaced (8.859%); 1.4 MiB/s, 22 objects/s recovering 2024-09-14T04:45:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:42 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:45:43.275 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:42 smithi039 ceph-mon[106470]: pgmap v2464: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 22235/250998 objects misplaced (8.859%); 1.4 MiB/s, 22 objects/s recovering 2024-09-14T04:45:43.275 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:42 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:45:43.275 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:42 smithi039 ceph-mon[111235]: pgmap v2464: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 5.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 608 op/s; 22235/250998 objects misplaced (8.859%); 1.4 MiB/s, 22 objects/s recovering 2024-09-14T04:45:43.275 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:42 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:45:43.321 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:43.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:43.405 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:43.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:43.506 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:43.550 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:43.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:43.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:43.652 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:43.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:43.694 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:43.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:43.795 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:43.842 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:43.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:43.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:43.944 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:44.008 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:44.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:44.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:44.110 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:44.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:44.151 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:44.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:44.252 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:44.273 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:44.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:44.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:44.375 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:44.408 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:44.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:44.410 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:44.435 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:45:44.436 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T04:45:44.510 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:44.552 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:44.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:44.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:44.654 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:44.694 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:44.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:44.695 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:44.796 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:44.829 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:44.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:44.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:44.931 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:44.972 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:44.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:44.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:45.060 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:44 smithi039 ceph-mon[106470]: pgmap v2465: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 920 op/s; 21468/241194 objects misplaced (8.901%); 1.3 MiB/s, 20 objects/s recovering 2024-09-14T04:45:45.060 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:44 smithi039 ceph-mon[106470]: from='client.35577 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:45:45.061 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:44 smithi039 ceph-mon[111235]: pgmap v2465: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 920 op/s; 21468/241194 objects misplaced (8.901%); 1.3 MiB/s, 20 objects/s recovering 2024-09-14T04:45:45.061 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:44 smithi039 ceph-mon[111235]: from='client.35577 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:45:45.074 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:45.117 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:45.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:45.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:45.221 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:45.259 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:45.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:45.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:45.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:44 smithi129 ceph-mon[101316]: pgmap v2465: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 920 op/s; 21468/241194 objects misplaced (8.901%); 1.3 MiB/s, 20 objects/s recovering 2024-09-14T04:45:45.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:44 smithi129 ceph-mon[101316]: from='client.35577 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:45:45.361 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:45.396 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:45.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:45.397 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:45.439 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":2465,"stamp":"2024-09-14T04:45:43.654941+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5267789821,"num_objects":80398,"num_object_clones":0,"num_object_copies":241194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":21468,"num_objects_unfound":0,"num_objects_dirty":80398,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":172482,"num_write_kb":8089250,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":747,"num_bytes_recovered":46994926,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123633,"ondisk_log_size":123633,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":301,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17985036,"kb_used_data":15548684,"kb_used_omap":404,"kb_used_meta":2435563,"kb_avail":731976180,"statfs":{"total":767960285184,"available":749543608320,"internally_reserved":0,"allocated":15921852416,"data_stored":15886145363,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":414022,"internal_metadata":2494017210},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,5,3,2,5,2,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-533200896,"num_objects":-8136,"num_object_clones":0,"num_object_copies":-24408,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1715,"num_objects_unfound":0,"num_objects_dirty":-8136,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8136,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":181,"num_bytes_recovered":11862016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.836186"},"pg_stats":[{"pgid":"54.f","version":"1412'10779","reported_seq":10855,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:42.192807+0000","last_change":"2024-09-14T04:43:01.044742+0000","last_active":"2024-09-14T04:45:42.192807+0000","last_peered":"2024-09-14T04:45:42.192807+0000","last_clean":"2024-09-14T04:45:42.192807+0000","last_became_active":"2024-09-14T04:43:01.044421+0000","last_became_peered":"2024-09-14T04:43:01.044421+0000","last_unstale":"2024-09-14T04:45:42.192807+0000","last_undegraded":"2024-09-14T04:45:42.192807+0000","last_fullsized":"2024-09-14T04:45:42.192807+0000","mapping_epoch":1380,"log_start":"1392'4100","ondisk_log_start":"1392'4100","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":6679,"log_dups_size":0,"ondisk_log_size":6679,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:28.897426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317784064,"num_objects":4849,"num_object_clones":0,"num_object_copies":14547,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4849,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10779,"num_write_kb":500096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"54.e","version":"1412'10680","reported_seq":12122,"reported_epoch":1412,"state":"active+recovering+remapped","last_fresh":"2024-09-14T04:45:40.442368+0000","last_change":"2024-09-14T04:44:58.690021+0000","last_active":"2024-09-14T04:45:40.442368+0000","last_peered":"2024-09-14T04:45:40.442368+0000","last_clean":"2024-09-14T04:44:56.302285+0000","last_became_active":"2024-09-14T04:44:58.579754+0000","last_became_peered":"2024-09-14T04:44:58.579754+0000","last_unstale":"2024-09-14T04:45:40.442368+0000","last_undegraded":"2024-09-14T04:45:40.442368+0000","last_fullsized":"2024-09-14T04:45:40.442368+0000","mapping_epoch":1403,"log_start":"1389'2783","ondisk_log_start":"1389'2783","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7897,"log_dups_size":0,"ondisk_log_size":7897,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335675392,"num_objects":5122,"num_object_clones":0,"num_object_copies":15366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6644,"num_objects_unfound":0,"num_objects_dirty":5122,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10680,"num_write_kb":505664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":682,"num_bytes_recovered":44630016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[1,2,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":-1522}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"54.d","version":"1412'10881","reported_seq":10924,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:43.169511+0000","last_change":"2024-09-14T04:43:01.050000+0000","last_active":"2024-09-14T04:45:43.169511+0000","last_peered":"2024-09-14T04:45:43.169511+0000","last_clean":"2024-09-14T04:45:43.169511+0000","last_became_active":"2024-09-14T04:43:01.049752+0000","last_became_peered":"2024-09-14T04:43:01.049752+0000","last_unstale":"2024-09-14T04:45:43.169511+0000","last_undegraded":"2024-09-14T04:45:43.169511+0000","last_fullsized":"2024-09-14T04:45:43.169511+0000","mapping_epoch":1380,"log_start":"1390'3300","ondisk_log_start":"1390'3300","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7581,"log_dups_size":0,"ondisk_log_size":7581,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:35:06.578523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327811096,"num_objects":5003,"num_object_clones":0,"num_object_copies":15009,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5003,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10881,"num_write_kb":508225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"54.c","version":"1412'10819","reported_seq":10899,"reported_epoch":1412,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:45:42.205998+0000","last_change":"2024-09-14T04:44:58.455281+0000","last_active":"2024-09-14T04:45:42.205998+0000","last_peered":"2024-09-14T04:45:42.205998+0000","last_clean":"2024-09-14T04:44:56.285515+0000","last_became_active":"2024-09-14T04:44:58.445654+0000","last_became_peered":"2024-09-14T04:44:58.445654+0000","last_unstale":"2024-09-14T04:45:42.205998+0000","last_undegraded":"2024-09-14T04:45:42.205998+0000","last_fullsized":"2024-09-14T04:45:42.205998+0000","mapping_epoch":1403,"log_start":"1392'4100","ondisk_log_start":"1392'4100","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":6719,"log_dups_size":0,"ondisk_log_size":6719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324337664,"num_objects":4949,"num_object_clones":0,"num_object_copies":14847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4949,"num_objects_unfound":0,"num_objects_dirty":4949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10819,"num_write_kb":504576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[{"shards":"3,5,6","objects":4949}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"54.b","version":"1412'10754","reported_seq":10819,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:43.081800+0000","last_change":"2024-09-14T04:43:13.355592+0000","last_active":"2024-09-14T04:45:43.081800+0000","last_peered":"2024-09-14T04:45:43.081800+0000","last_clean":"2024-09-14T04:45:43.081800+0000","last_became_active":"2024-09-14T04:43:13.355130+0000","last_became_peered":"2024-09-14T04:43:13.355130+0000","last_unstale":"2024-09-14T04:45:43.081800+0000","last_undegraded":"2024-09-14T04:45:43.081800+0000","last_fullsized":"2024-09-14T04:45:43.081800+0000","mapping_epoch":1386,"log_start":"1389'2400","ondisk_log_start":"1389'2400","created":1380,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":8354,"log_dups_size":0,"ondisk_log_size":8354,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:42:44.062290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324534272,"num_objects":4952,"num_object_clones":0,"num_object_copies":14856,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4952,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10754,"num_write_kb":502592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"54.a","version":"1412'10765","reported_seq":10807,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:43.143155+0000","last_change":"2024-09-14T04:43:01.043585+0000","last_active":"2024-09-14T04:45:43.143155+0000","last_peered":"2024-09-14T04:45:43.143155+0000","last_clean":"2024-09-14T04:45:43.143155+0000","last_became_active":"2024-09-14T04:43:01.043295+0000","last_became_peered":"2024-09-14T04:43:01.043295+0000","last_unstale":"2024-09-14T04:45:43.143155+0000","last_undegraded":"2024-09-14T04:45:43.143155+0000","last_fullsized":"2024-09-14T04:45:43.143155+0000","mapping_epoch":1380,"log_start":"1390'3200","ondisk_log_start":"1390'3200","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7565,"log_dups_size":0,"ondisk_log_size":7565,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:58:35.934308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319619072,"num_objects":4877,"num_object_clones":0,"num_object_copies":14631,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4877,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10765,"num_write_kb":500544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"54.9","version":"1412'10653","reported_seq":10729,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:40.447023+0000","last_change":"2024-09-14T04:43:01.048728+0000","last_active":"2024-09-14T04:45:40.447023+0000","last_peered":"2024-09-14T04:45:40.447023+0000","last_clean":"2024-09-14T04:45:40.447023+0000","last_became_active":"2024-09-14T04:43:01.048381+0000","last_became_peered":"2024-09-14T04:43:01.048381+0000","last_unstale":"2024-09-14T04:45:40.447023+0000","last_undegraded":"2024-09-14T04:45:40.447023+0000","last_fullsized":"2024-09-14T04:45:40.447023+0000","mapping_epoch":1380,"log_start":"1389'2700","ondisk_log_start":"1389'2700","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7953,"log_dups_size":0,"ondisk_log_size":7953,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:47:27.860166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338886656,"num_objects":5171,"num_object_clones":0,"num_object_copies":15513,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5171,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10653,"num_write_kb":506368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"54.8","version":"1412'10864","reported_seq":10942,"reported_epoch":1412,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:45:43.016424+0000","last_change":"2024-09-14T04:44:58.458770+0000","last_active":"2024-09-14T04:45:43.016424+0000","last_peered":"2024-09-14T04:45:43.016424+0000","last_clean":"2024-09-14T04:44:56.294675+0000","last_became_active":"2024-09-14T04:44:58.445646+0000","last_became_peered":"2024-09-14T04:44:58.445646+0000","last_unstale":"2024-09-14T04:45:43.016424+0000","last_undegraded":"2024-09-14T04:45:43.016424+0000","last_fullsized":"2024-09-14T04:45:43.016424+0000","mapping_epoch":1403,"log_start":"1389'2500","ondisk_log_start":"1389'2500","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":8364,"log_dups_size":0,"ondisk_log_size":8364,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326631424,"num_objects":4984,"num_object_clones":0,"num_object_copies":14952,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4984,"num_objects_unfound":0,"num_objects_dirty":4984,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10864,"num_write_kb":507136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[0,3,7],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3,7","objects":4984}],"blocked_by":[],"up_primary":4,"acting_primary":0,"purged_snaps":[]},{"pgid":"54.7","version":"1412'10724","reported_seq":10800,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:42.194236+0000","last_change":"2024-09-14T04:43:01.050691+0000","last_active":"2024-09-14T04:45:42.194236+0000","last_peered":"2024-09-14T04:45:42.194236+0000","last_clean":"2024-09-14T04:45:42.194236+0000","last_became_active":"2024-09-14T04:43:01.050334+0000","last_became_peered":"2024-09-14T04:43:01.050334+0000","last_unstale":"2024-09-14T04:45:42.194236+0000","last_undegraded":"2024-09-14T04:45:42.194236+0000","last_fullsized":"2024-09-14T04:45:42.194236+0000","mapping_epoch":1380,"log_start":"1392'4000","ondisk_log_start":"1392'4000","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":6724,"log_dups_size":0,"ondisk_log_size":6724,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:22:30.806651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333578240,"num_objects":5090,"num_object_clones":0,"num_object_copies":15270,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5090,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10724,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"54.6","version":"1412'10853","reported_seq":10895,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:42.919550+0000","last_change":"2024-09-14T04:43:01.043477+0000","last_active":"2024-09-14T04:45:42.919550+0000","last_peered":"2024-09-14T04:45:42.919550+0000","last_clean":"2024-09-14T04:45:42.919550+0000","last_became_active":"2024-09-14T04:43:01.043136+0000","last_became_peered":"2024-09-14T04:43:01.043136+0000","last_unstale":"2024-09-14T04:45:42.919550+0000","last_undegraded":"2024-09-14T04:45:42.919550+0000","last_fullsized":"2024-09-14T04:45:42.919550+0000","mapping_epoch":1380,"log_start":"1390'3500","ondisk_log_start":"1390'3500","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7353,"log_dups_size":0,"ondisk_log_size":7353,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:00:57.211271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328269824,"num_objects":5009,"num_object_clones":0,"num_object_copies":15027,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5009,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10853,"num_write_kb":507584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"54.5","version":"1412'10754","reported_seq":10796,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:43.110353+0000","last_change":"2024-09-14T04:43:01.048975+0000","last_active":"2024-09-14T04:45:43.110353+0000","last_peered":"2024-09-14T04:45:43.110353+0000","last_clean":"2024-09-14T04:45:43.110353+0000","last_became_active":"2024-09-14T04:43:01.048677+0000","last_became_peered":"2024-09-14T04:43:01.048677+0000","last_unstale":"2024-09-14T04:45:43.110353+0000","last_undegraded":"2024-09-14T04:45:43.110353+0000","last_fullsized":"2024-09-14T04:45:43.110353+0000","mapping_epoch":1380,"log_start":"1390'3200","ondisk_log_start":"1390'3200","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7554,"log_dups_size":0,"ondisk_log_size":7554,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:58:04.849606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332398592,"num_objects":5072,"num_object_clones":0,"num_object_copies":15216,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5072,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10754,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"54.4","version":"1412'10757","reported_seq":10832,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:42.160038+0000","last_change":"2024-09-14T04:43:01.043603+0000","last_active":"2024-09-14T04:45:42.160038+0000","last_peered":"2024-09-14T04:45:42.160038+0000","last_clean":"2024-09-14T04:45:42.160038+0000","last_became_active":"2024-09-14T04:43:01.042757+0000","last_became_peered":"2024-09-14T04:43:01.042757+0000","last_unstale":"2024-09-14T04:45:42.160038+0000","last_undegraded":"2024-09-14T04:45:42.160038+0000","last_fullsized":"2024-09-14T04:45:42.160038+0000","mapping_epoch":1380,"log_start":"1392'4000","ondisk_log_start":"1392'4000","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":6757,"log_dups_size":0,"ondisk_log_size":6757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:46:20.719227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333905920,"num_objects":5095,"num_object_clones":0,"num_object_copies":15285,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5095,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10757,"num_write_kb":507264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"54.3","version":"1412'10821","reported_seq":10897,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:40.423586+0000","last_change":"2024-09-14T04:43:01.048551+0000","last_active":"2024-09-14T04:45:40.423586+0000","last_peered":"2024-09-14T04:45:40.423586+0000","last_clean":"2024-09-14T04:45:40.423586+0000","last_became_active":"2024-09-14T04:43:01.048218+0000","last_became_peered":"2024-09-14T04:43:01.048218+0000","last_unstale":"2024-09-14T04:45:40.423586+0000","last_undegraded":"2024-09-14T04:45:40.423586+0000","last_fullsized":"2024-09-14T04:45:40.423586+0000","mapping_epoch":1380,"log_start":"1389'2900","ondisk_log_start":"1389'2900","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7921,"log_dups_size":0,"ondisk_log_size":7921,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:27:10.233265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343474176,"num_objects":5241,"num_object_clones":0,"num_object_copies":15723,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5241,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10821,"num_write_kb":513984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"54.2","version":"1412'10775","reported_seq":10850,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:43.069078+0000","last_change":"2024-09-14T04:43:01.049088+0000","last_active":"2024-09-14T04:45:43.069078+0000","last_peered":"2024-09-14T04:45:43.069078+0000","last_clean":"2024-09-14T04:45:43.069078+0000","last_became_active":"2024-09-14T04:43:01.048764+0000","last_became_peered":"2024-09-14T04:43:01.048764+0000","last_unstale":"2024-09-14T04:45:43.069078+0000","last_undegraded":"2024-09-14T04:45:43.069078+0000","last_fullsized":"2024-09-14T04:45:43.069078+0000","mapping_epoch":1380,"log_start":"1389'2400","ondisk_log_start":"1389'2400","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":8375,"log_dups_size":0,"ondisk_log_size":8375,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:03:50.532719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326565888,"num_objects":4983,"num_object_clones":0,"num_object_copies":14949,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4983,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10775,"num_write_kb":504256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"54.1","version":"1412'10816","reported_seq":10858,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:43.173546+0000","last_change":"2024-09-14T04:43:01.047658+0000","last_active":"2024-09-14T04:45:43.173546+0000","last_peered":"2024-09-14T04:45:43.173546+0000","last_clean":"2024-09-14T04:45:43.173546+0000","last_became_active":"2024-09-14T04:43:01.047401+0000","last_became_peered":"2024-09-14T04:43:01.047401+0000","last_unstale":"2024-09-14T04:45:43.173546+0000","last_undegraded":"2024-09-14T04:45:43.173546+0000","last_fullsized":"2024-09-14T04:45:43.173546+0000","mapping_epoch":1380,"log_start":"1389'2200","ondisk_log_start":"1389'2200","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":8616,"log_dups_size":0,"ondisk_log_size":8616,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:02:05.987375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332005376,"num_objects":5066,"num_object_clones":0,"num_object_copies":15198,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5066,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10816,"num_write_kb":508224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"54.0","version":"1412'10723","reported_seq":10842,"reported_epoch":1412,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:45:43.131538+0000","last_change":"2024-09-14T04:44:58.640486+0000","last_active":"2024-09-14T04:45:43.131538+0000","last_peered":"2024-09-14T04:45:43.131538+0000","last_clean":"2024-09-14T04:44:56.291188+0000","last_became_active":"2024-09-14T04:44:58.441092+0000","last_became_peered":"2024-09-14T04:44:58.441092+0000","last_unstale":"2024-09-14T04:45:43.131538+0000","last_undegraded":"2024-09-14T04:45:43.131538+0000","last_fullsized":"2024-09-14T04:45:43.131538+0000","mapping_epoch":1403,"log_start":"1390'3200","ondisk_log_start":"1390'3200","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7523,"log_dups_size":0,"ondisk_log_size":7523,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:13:27.337351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321716224,"num_objects":4909,"num_object_clones":0,"num_object_copies":14727,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4891,"num_objects_unfound":0,"num_objects_dirty":4909,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10723,"num_write_kb":500224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":2097152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,3,5],"avail_no_missing":["6","3","5"],"object_location_counts":[{"shards":"3,5,6","objects":4909}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":2112,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.885553+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T04:45:37.885553+0000","last_peered":"2024-09-14T04:45:37.885553+0000","last_clean":"2024-09-14T04:45:37.885553+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T04:45:37.885553+0000","last_undegraded":"2024-09-14T04:45:37.885553+0000","last_fullsized":"2024-09-14T04:45:37.885553+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":20,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.814442+0000","last_change":"2024-09-14T04:44:57.325109+0000","last_active":"2024-09-14T04:45:34.814442+0000","last_peered":"2024-09-14T04:45:34.814442+0000","last_clean":"2024-09-14T04:45:34.814442+0000","last_became_active":"2024-09-14T04:44:57.324791+0000","last_became_peered":"2024-09-14T04:44:57.324791+0000","last_unstale":"2024-09-14T04:45:34.814442+0000","last_undegraded":"2024-09-14T04:45:34.814442+0000","last_fullsized":"2024-09-14T04:45:34.814442+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:03.593275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":2101,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.888543+0000","last_change":"2024-09-14T04:07:58.504829+0000","last_active":"2024-09-14T04:45:37.888543+0000","last_peered":"2024-09-14T04:45:37.888543+0000","last_clean":"2024-09-14T04:45:37.888543+0000","last_became_active":"2024-09-14T04:07:58.502623+0000","last_became_peered":"2024-09-14T04:07:58.502623+0000","last_unstale":"2024-09-14T04:45:37.888543+0000","last_undegraded":"2024-09-14T04:45:37.888543+0000","last_fullsized":"2024-09-14T04:45:37.888543+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:59.091993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":734,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.890618+0000","last_change":"2024-09-14T04:43:13.355281+0000","last_active":"2024-09-14T04:45:37.890618+0000","last_peered":"2024-09-14T04:45:37.890618+0000","last_clean":"2024-09-14T04:45:37.890618+0000","last_became_active":"2024-09-14T04:43:13.354967+0000","last_became_peered":"2024-09-14T04:43:13.354967+0000","last_unstale":"2024-09-14T04:45:37.890618+0000","last_undegraded":"2024-09-14T04:45:37.890618+0000","last_fullsized":"2024-09-14T04:45:37.890618+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:23:19.332075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1399,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.885885+0000","last_change":"2024-09-14T04:44:57.342918+0000","last_active":"2024-09-14T04:45:37.885885+0000","last_peered":"2024-09-14T04:45:37.885885+0000","last_clean":"2024-09-14T04:45:37.885885+0000","last_became_active":"2024-09-14T04:44:57.329550+0000","last_became_peered":"2024-09-14T04:44:57.329550+0000","last_unstale":"2024-09-14T04:45:37.885885+0000","last_undegraded":"2024-09-14T04:45:37.885885+0000","last_fullsized":"2024-09-14T04:45:37.885885+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:52:52.435893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":20,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.814583+0000","last_change":"2024-09-14T04:44:57.325537+0000","last_active":"2024-09-14T04:45:34.814583+0000","last_peered":"2024-09-14T04:45:34.814583+0000","last_clean":"2024-09-14T04:45:34.814583+0000","last_became_active":"2024-09-14T04:44:57.325057+0000","last_became_peered":"2024-09-14T04:44:57.325057+0000","last_unstale":"2024-09-14T04:45:34.814583+0000","last_undegraded":"2024-09-14T04:45:34.814583+0000","last_fullsized":"2024-09-14T04:45:34.814583+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:43:23.166708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1414,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.886272+0000","last_change":"2024-09-14T04:44:57.337033+0000","last_active":"2024-09-14T04:45:37.886272+0000","last_peered":"2024-09-14T04:45:37.886272+0000","last_clean":"2024-09-14T04:45:37.886272+0000","last_became_active":"2024-09-14T04:44:57.328437+0000","last_became_peered":"2024-09-14T04:44:57.328437+0000","last_unstale":"2024-09-14T04:45:37.886272+0000","last_undegraded":"2024-09-14T04:45:37.886272+0000","last_fullsized":"2024-09-14T04:45:37.886272+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.109086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":20,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.814871+0000","last_change":"2024-09-14T04:44:57.332243+0000","last_active":"2024-09-14T04:45:34.814871+0000","last_peered":"2024-09-14T04:45:34.814871+0000","last_clean":"2024-09-14T04:45:34.814871+0000","last_became_active":"2024-09-14T04:44:57.330413+0000","last_became_peered":"2024-09-14T04:44:57.330413+0000","last_unstale":"2024-09-14T04:45:34.814871+0000","last_undegraded":"2024-09-14T04:45:34.814871+0000","last_fullsized":"2024-09-14T04:45:34.814871+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:25:01.338549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":2118,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.888136+0000","last_change":"2024-09-14T04:44:57.333097+0000","last_active":"2024-09-14T04:45:37.888136+0000","last_peered":"2024-09-14T04:45:37.888136+0000","last_clean":"2024-09-14T04:45:37.888136+0000","last_became_active":"2024-09-14T04:44:57.332294+0000","last_became_peered":"2024-09-14T04:44:57.332294+0000","last_unstale":"2024-09-14T04:45:37.888136+0000","last_undegraded":"2024-09-14T04:45:37.888136+0000","last_fullsized":"2024-09-14T04:45:37.888136+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:57:44.426280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2108,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.891060+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T04:45:37.891060+0000","last_peered":"2024-09-14T04:45:37.891060+0000","last_clean":"2024-09-14T04:45:37.891060+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T04:45:37.891060+0000","last_undegraded":"2024-09-14T04:45:37.891060+0000","last_fullsized":"2024-09-14T04:45:37.891060+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":20,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.814570+0000","last_change":"2024-09-14T04:44:57.325392+0000","last_active":"2024-09-14T04:45:34.814570+0000","last_peered":"2024-09-14T04:45:34.814570+0000","last_clean":"2024-09-14T04:45:34.814570+0000","last_became_active":"2024-09-14T04:44:57.324624+0000","last_became_peered":"2024-09-14T04:44:57.324624+0000","last_unstale":"2024-09-14T04:45:34.814570+0000","last_undegraded":"2024-09-14T04:45:34.814570+0000","last_fullsized":"2024-09-14T04:45:34.814570+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:33:18.810199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2076,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.892685+0000","last_change":"2024-09-14T04:44:57.329113+0000","last_active":"2024-09-14T04:45:37.892685+0000","last_peered":"2024-09-14T04:45:37.892685+0000","last_clean":"2024-09-14T04:45:37.892685+0000","last_became_active":"2024-09-14T04:44:57.328727+0000","last_became_peered":"2024-09-14T04:44:57.328727+0000","last_unstale":"2024-09-14T04:45:37.892685+0000","last_undegraded":"2024-09-14T04:45:37.892685+0000","last_fullsized":"2024-09-14T04:45:37.892685+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:34.939874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":2094,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.892269+0000","last_change":"2024-09-14T04:44:57.329277+0000","last_active":"2024-09-14T04:45:37.892269+0000","last_peered":"2024-09-14T04:45:37.892269+0000","last_clean":"2024-09-14T04:45:37.892269+0000","last_became_active":"2024-09-14T04:44:57.328804+0000","last_became_peered":"2024-09-14T04:44:57.328804+0000","last_unstale":"2024-09-14T04:45:37.892269+0000","last_undegraded":"2024-09-14T04:45:37.892269+0000","last_fullsized":"2024-09-14T04:45:37.892269+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:42:46.055240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2106,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.884047+0000","last_change":"2024-09-14T04:07:58.561490+0000","last_active":"2024-09-14T04:45:37.884047+0000","last_peered":"2024-09-14T04:45:37.884047+0000","last_clean":"2024-09-14T04:45:37.884047+0000","last_became_active":"2024-09-14T04:07:58.559841+0000","last_became_peered":"2024-09-14T04:07:58.559841+0000","last_unstale":"2024-09-14T04:45:37.884047+0000","last_undegraded":"2024-09-14T04:45:37.884047+0000","last_fullsized":"2024-09-14T04:45:37.884047+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:27:03.795368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2090,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.884597+0000","last_change":"2024-09-14T04:43:13.357092+0000","last_active":"2024-09-14T04:45:37.884597+0000","last_peered":"2024-09-14T04:45:37.884597+0000","last_clean":"2024-09-14T04:45:37.884597+0000","last_became_active":"2024-09-14T04:43:13.356694+0000","last_became_peered":"2024-09-14T04:43:13.356694+0000","last_unstale":"2024-09-14T04:45:37.884597+0000","last_undegraded":"2024-09-14T04:45:37.884597+0000","last_fullsized":"2024-09-14T04:45:37.884597+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:11.167574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1391,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.816376+0000","last_change":"2024-09-14T04:07:21.720797+0000","last_active":"2024-09-14T04:45:34.816376+0000","last_peered":"2024-09-14T04:45:34.816376+0000","last_clean":"2024-09-14T04:45:34.816376+0000","last_became_active":"2024-09-14T04:07:21.719331+0000","last_became_peered":"2024-09-14T04:07:21.719331+0000","last_unstale":"2024-09-14T04:45:34.816376+0000","last_undegraded":"2024-09-14T04:45:34.816376+0000","last_fullsized":"2024-09-14T04:45:34.816376+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:35.404023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":39,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.817696+0000","last_change":"2024-09-14T04:43:11.641514+0000","last_active":"2024-09-14T04:45:34.817696+0000","last_peered":"2024-09-14T04:45:34.817696+0000","last_clean":"2024-09-14T04:45:34.817696+0000","last_became_active":"2024-09-14T04:43:11.641151+0000","last_became_peered":"2024-09-14T04:43:11.641151+0000","last_unstale":"2024-09-14T04:45:34.817696+0000","last_undegraded":"2024-09-14T04:45:34.817696+0000","last_fullsized":"2024-09-14T04:45:34.817696+0000","mapping_epoch":1384,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:06:45.045430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2101,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.885685+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T04:45:37.885685+0000","last_peered":"2024-09-14T04:45:37.885685+0000","last_clean":"2024-09-14T04:45:37.885685+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T04:45:37.885685+0000","last_undegraded":"2024-09-14T04:45:37.885685+0000","last_fullsized":"2024-09-14T04:45:37.885685+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":2083,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.890436+0000","last_change":"2024-09-14T04:07:58.507682+0000","last_active":"2024-09-14T04:45:37.890436+0000","last_peered":"2024-09-14T04:45:37.890436+0000","last_clean":"2024-09-14T04:45:37.890436+0000","last_became_active":"2024-09-14T04:07:58.507365+0000","last_became_peered":"2024-09-14T04:07:58.507365+0000","last_unstale":"2024-09-14T04:45:37.890436+0000","last_undegraded":"2024-09-14T04:45:37.890436+0000","last_fullsized":"2024-09-14T04:45:37.890436+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:14:47.294950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1397,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.818916+0000","last_change":"2024-09-14T04:42:49.938505+0000","last_active":"2024-09-14T04:45:34.818916+0000","last_peered":"2024-09-14T04:45:34.818916+0000","last_clean":"2024-09-14T04:45:34.818916+0000","last_became_active":"2024-09-14T04:42:49.938180+0000","last_became_peered":"2024-09-14T04:42:49.938180+0000","last_unstale":"2024-09-14T04:45:34.818916+0000","last_undegraded":"2024-09-14T04:45:34.818916+0000","last_fullsized":"2024-09-14T04:45:34.818916+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:05.929479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1416,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.821457+0000","last_change":"2024-09-14T04:44:57.346582+0000","last_active":"2024-09-14T04:45:34.821457+0000","last_peered":"2024-09-14T04:45:34.821457+0000","last_clean":"2024-09-14T04:45:34.821457+0000","last_became_active":"2024-09-14T04:44:57.330446+0000","last_became_peered":"2024-09-14T04:44:57.330446+0000","last_unstale":"2024-09-14T04:45:34.821457+0000","last_undegraded":"2024-09-14T04:45:34.821457+0000","last_fullsized":"2024-09-14T04:45:34.821457+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:29.238585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":42,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.891811+0000","last_change":"2024-09-14T04:44:14.480667+0000","last_active":"2024-09-14T04:45:37.891811+0000","last_peered":"2024-09-14T04:45:37.891811+0000","last_clean":"2024-09-14T04:45:37.891811+0000","last_became_active":"2024-09-14T04:44:14.480365+0000","last_became_peered":"2024-09-14T04:44:14.480365+0000","last_unstale":"2024-09-14T04:45:37.891811+0000","last_undegraded":"2024-09-14T04:45:37.891811+0000","last_fullsized":"2024-09-14T04:45:37.891811+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:00.611771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":30,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.815746+0000","last_change":"2024-09-14T04:44:57.341426+0000","last_active":"2024-09-14T04:45:34.815746+0000","last_peered":"2024-09-14T04:45:34.815746+0000","last_clean":"2024-09-14T04:45:34.815746+0000","last_became_active":"2024-09-14T04:44:57.328763+0000","last_became_peered":"2024-09-14T04:44:57.328763+0000","last_unstale":"2024-09-14T04:45:34.815746+0000","last_undegraded":"2024-09-14T04:45:34.815746+0000","last_fullsized":"2024-09-14T04:45:34.815746+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:34:40.066761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":20,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.815058+0000","last_change":"2024-09-14T04:44:57.331337+0000","last_active":"2024-09-14T04:45:34.815058+0000","last_peered":"2024-09-14T04:45:34.815058+0000","last_clean":"2024-09-14T04:45:34.815058+0000","last_became_active":"2024-09-14T04:44:57.329336+0000","last_became_peered":"2024-09-14T04:44:57.329336+0000","last_unstale":"2024-09-14T04:45:34.815058+0000","last_undegraded":"2024-09-14T04:45:34.815058+0000","last_fullsized":"2024-09-14T04:45:34.815058+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:52:35.980869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"349'149","reported_seq":770,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.890891+0000","last_change":"2024-09-14T04:41:37.755174+0000","last_active":"2024-09-14T04:45:37.890891+0000","last_peered":"2024-09-14T04:45:37.890891+0000","last_clean":"2024-09-14T04:45:37.890891+0000","last_became_active":"2024-09-14T04:41:36.930263+0000","last_became_peered":"2024-09-14T04:41:36.930263+0000","last_unstale":"2024-09-14T04:45:37.890891+0000","last_undegraded":"2024-09-14T04:45:37.890891+0000","last_fullsized":"2024-09-14T04:45:37.890891+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.755105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.755105+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:27:03.161713+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":20,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.814520+0000","last_change":"2024-09-14T04:44:57.329870+0000","last_active":"2024-09-14T04:45:34.814520+0000","last_peered":"2024-09-14T04:45:34.814520+0000","last_clean":"2024-09-14T04:45:34.814520+0000","last_became_active":"2024-09-14T04:44:57.329353+0000","last_became_peered":"2024-09-14T04:44:57.329353+0000","last_unstale":"2024-09-14T04:45:34.814520+0000","last_undegraded":"2024-09-14T04:45:34.814520+0000","last_fullsized":"2024-09-14T04:45:34.814520+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:46.432181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1394,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.816438+0000","last_change":"2024-09-14T04:07:21.721242+0000","last_active":"2024-09-14T04:45:34.816438+0000","last_peered":"2024-09-14T04:45:34.816438+0000","last_clean":"2024-09-14T04:45:34.816438+0000","last_became_active":"2024-09-14T04:07:21.720084+0000","last_became_peered":"2024-09-14T04:07:21.720084+0000","last_unstale":"2024-09-14T04:45:34.816438+0000","last_undegraded":"2024-09-14T04:45:34.816438+0000","last_fullsized":"2024-09-14T04:45:34.816438+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:58:41.308643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":2139,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.883961+0000","last_change":"2024-09-14T04:44:57.344614+0000","last_active":"2024-09-14T04:45:37.883961+0000","last_peered":"2024-09-14T04:45:37.883961+0000","last_clean":"2024-09-14T04:45:37.883961+0000","last_became_active":"2024-09-14T04:44:57.332256+0000","last_became_peered":"2024-09-14T04:44:57.332256+0000","last_unstale":"2024-09-14T04:45:37.883961+0000","last_undegraded":"2024-09-14T04:45:37.883961+0000","last_fullsized":"2024-09-14T04:45:37.883961+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:23.438417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"349'149","reported_seq":778,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.815410+0000","last_change":"2024-09-14T04:45:00.331854+0000","last_active":"2024-09-14T04:45:34.815410+0000","last_peered":"2024-09-14T04:45:34.815410+0000","last_clean":"2024-09-14T04:45:34.815410+0000","last_became_active":"2024-09-14T04:45:00.331460+0000","last_became_peered":"2024-09-14T04:45:00.331460+0000","last_unstale":"2024-09-14T04:45:34.815410+0000","last_undegraded":"2024-09-14T04:45:34.815410+0000","last_fullsized":"2024-09-14T04:45:34.815410+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.567535+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.567535+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:45.095729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":2075,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.895047+0000","last_change":"2024-09-14T04:07:58.555446+0000","last_active":"2024-09-14T04:45:37.895047+0000","last_peered":"2024-09-14T04:45:37.895047+0000","last_clean":"2024-09-14T04:45:37.895047+0000","last_became_active":"2024-09-14T04:07:58.548355+0000","last_became_peered":"2024-09-14T04:07:58.548355+0000","last_unstale":"2024-09-14T04:45:37.895047+0000","last_undegraded":"2024-09-14T04:45:37.895047+0000","last_fullsized":"2024-09-14T04:45:37.895047+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:56:28.426094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":94,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.911357+0000","last_change":"2024-09-14T04:44:57.330746+0000","last_active":"2024-09-14T04:45:37.911357+0000","last_peered":"2024-09-14T04:45:37.911357+0000","last_clean":"2024-09-14T04:45:37.911357+0000","last_became_active":"2024-09-14T04:44:57.330327+0000","last_became_peered":"2024-09-14T04:44:57.330327+0000","last_unstale":"2024-09-14T04:45:37.911357+0000","last_undegraded":"2024-09-14T04:45:37.911357+0000","last_fullsized":"2024-09-14T04:45:37.911357+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:53.122771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":60,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.822425+0000","last_change":"2024-09-14T04:43:13.355296+0000","last_active":"2024-09-14T04:45:34.822425+0000","last_peered":"2024-09-14T04:45:34.822425+0000","last_clean":"2024-09-14T04:45:34.822425+0000","last_became_active":"2024-09-14T04:43:13.354950+0000","last_became_peered":"2024-09-14T04:43:13.354950+0000","last_unstale":"2024-09-14T04:45:34.822425+0000","last_undegraded":"2024-09-14T04:45:34.822425+0000","last_fullsized":"2024-09-14T04:45:34.822425+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:20:08.968818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"349'149","reported_seq":716,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.816068+0000","last_change":"2024-09-14T04:41:39.471393+0000","last_active":"2024-09-14T04:45:34.816068+0000","last_peered":"2024-09-14T04:45:34.816068+0000","last_clean":"2024-09-14T04:45:34.816068+0000","last_became_active":"2024-09-14T04:41:36.931768+0000","last_became_peered":"2024-09-14T04:41:36.931768+0000","last_unstale":"2024-09-14T04:45:34.816068+0000","last_undegraded":"2024-09-14T04:45:34.816068+0000","last_fullsized":"2024-09-14T04:45:34.816068+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.471248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.471248+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:56.210375+0000","scrub_duration":0.021758344999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2095,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.888605+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T04:45:37.888605+0000","last_peered":"2024-09-14T04:45:37.888605+0000","last_clean":"2024-09-14T04:45:37.888605+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T04:45:37.888605+0000","last_undegraded":"2024-09-14T04:45:37.888605+0000","last_fullsized":"2024-09-14T04:45:37.888605+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1426,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.821644+0000","last_change":"2024-09-14T04:44:57.331436+0000","last_active":"2024-09-14T04:45:34.821644+0000","last_peered":"2024-09-14T04:45:34.821644+0000","last_clean":"2024-09-14T04:45:34.821644+0000","last_became_active":"2024-09-14T04:44:57.330957+0000","last_became_peered":"2024-09-14T04:44:57.330957+0000","last_unstale":"2024-09-14T04:45:34.821644+0000","last_undegraded":"2024-09-14T04:45:34.821644+0000","last_fullsized":"2024-09-14T04:45:34.821644+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:09.516688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":87,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.892341+0000","last_change":"2024-09-14T04:42:49.940968+0000","last_active":"2024-09-14T04:45:37.892341+0000","last_peered":"2024-09-14T04:45:37.892341+0000","last_clean":"2024-09-14T04:45:37.892341+0000","last_became_active":"2024-09-14T04:42:49.940667+0000","last_became_peered":"2024-09-14T04:42:49.940667+0000","last_unstale":"2024-09-14T04:45:37.892341+0000","last_undegraded":"2024-09-14T04:45:37.892341+0000","last_fullsized":"2024-09-14T04:45:37.892341+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:52:49.623815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"349'149","reported_seq":716,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.816308+0000","last_change":"2024-09-14T04:41:38.479980+0000","last_active":"2024-09-14T04:45:34.816308+0000","last_peered":"2024-09-14T04:45:34.816308+0000","last_clean":"2024-09-14T04:45:34.816308+0000","last_became_active":"2024-09-14T04:41:36.933392+0000","last_became_peered":"2024-09-14T04:41:36.933392+0000","last_unstale":"2024-09-14T04:45:34.816308+0000","last_undegraded":"2024-09-14T04:45:34.816308+0000","last_fullsized":"2024-09-14T04:45:34.816308+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:38.479887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:38.479887+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:01:26.102279+0000","scrub_duration":0.0062728319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":49,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.891715+0000","last_change":"2024-09-14T04:44:57.329983+0000","last_active":"2024-09-14T04:45:37.891715+0000","last_peered":"2024-09-14T04:45:37.891715+0000","last_clean":"2024-09-14T04:45:37.891715+0000","last_became_active":"2024-09-14T04:44:57.329597+0000","last_became_peered":"2024-09-14T04:44:57.329597+0000","last_unstale":"2024-09-14T04:45:37.891715+0000","last_undegraded":"2024-09-14T04:45:37.891715+0000","last_fullsized":"2024-09-14T04:45:37.891715+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:45.718039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":2129,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.892574+0000","last_change":"2024-09-14T04:06:46.930482+0000","last_active":"2024-09-14T04:45:37.892574+0000","last_peered":"2024-09-14T04:45:37.892574+0000","last_clean":"2024-09-14T04:45:37.892574+0000","last_became_active":"2024-09-14T04:06:46.927372+0000","last_became_peered":"2024-09-14T04:06:46.927372+0000","last_unstale":"2024-09-14T04:45:37.892574+0000","last_undegraded":"2024-09-14T04:45:37.892574+0000","last_fullsized":"2024-09-14T04:45:37.892574+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:07:22.159128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1399,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.817124+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T04:45:34.817124+0000","last_peered":"2024-09-14T04:45:34.817124+0000","last_clean":"2024-09-14T04:45:34.817124+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T04:45:34.817124+0000","last_undegraded":"2024-09-14T04:45:34.817124+0000","last_fullsized":"2024-09-14T04:45:34.817124+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"349'149","reported_seq":708,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.816829+0000","last_change":"2024-09-14T04:41:39.012977+0000","last_active":"2024-09-14T04:45:34.816829+0000","last_peered":"2024-09-14T04:45:34.816829+0000","last_clean":"2024-09-14T04:45:34.816829+0000","last_became_active":"2024-09-14T04:41:36.932552+0000","last_became_peered":"2024-09-14T04:41:36.932552+0000","last_unstale":"2024-09-14T04:45:34.816829+0000","last_undegraded":"2024-09-14T04:45:34.816829+0000","last_fullsized":"2024-09-14T04:45:34.816829+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.012879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.012879+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:16:40.080262+0000","scrub_duration":0.0051721620000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":87,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.892244+0000","last_change":"2024-09-14T04:42:49.945069+0000","last_active":"2024-09-14T04:45:37.892244+0000","last_peered":"2024-09-14T04:45:37.892244+0000","last_clean":"2024-09-14T04:45:37.892244+0000","last_became_active":"2024-09-14T04:42:49.943527+0000","last_became_peered":"2024-09-14T04:42:49.943527+0000","last_unstale":"2024-09-14T04:45:37.892244+0000","last_undegraded":"2024-09-14T04:45:37.892244+0000","last_fullsized":"2024-09-14T04:45:37.892244+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:39:10.524631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":20,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.814945+0000","last_change":"2024-09-14T04:44:57.329652+0000","last_active":"2024-09-14T04:45:34.814945+0000","last_peered":"2024-09-14T04:45:34.814945+0000","last_clean":"2024-09-14T04:45:34.814945+0000","last_became_active":"2024-09-14T04:44:57.328888+0000","last_became_peered":"2024-09-14T04:44:57.328888+0000","last_unstale":"2024-09-14T04:45:34.814945+0000","last_undegraded":"2024-09-14T04:45:34.814945+0000","last_fullsized":"2024-09-14T04:45:34.814945+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:10.170068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1397,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.817037+0000","last_change":"2024-09-14T04:06:46.875340+0000","last_active":"2024-09-14T04:45:34.817037+0000","last_peered":"2024-09-14T04:45:34.817037+0000","last_clean":"2024-09-14T04:45:34.817037+0000","last_became_active":"2024-09-14T04:06:46.874552+0000","last_became_peered":"2024-09-14T04:06:46.874552+0000","last_unstale":"2024-09-14T04:45:34.817037+0000","last_undegraded":"2024-09-14T04:45:34.817037+0000","last_fullsized":"2024-09-14T04:45:34.817037+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:52.257357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"349'149","reported_seq":1779,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.887977+0000","last_change":"2024-09-14T04:45:00.346627+0000","last_active":"2024-09-14T04:45:37.887977+0000","last_peered":"2024-09-14T04:45:37.887977+0000","last_clean":"2024-09-14T04:45:37.887977+0000","last_became_active":"2024-09-14T04:45:00.346286+0000","last_became_peered":"2024-09-14T04:45:00.346286+0000","last_unstale":"2024-09-14T04:45:37.887977+0000","last_undegraded":"2024-09-14T04:45:37.887977+0000","last_fullsized":"2024-09-14T04:45:37.887977+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:50.784648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:50.784648+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:07:58.441372+0000","scrub_duration":0.01141873,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":42,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.892851+0000","last_change":"2024-09-14T04:44:14.489235+0000","last_active":"2024-09-14T04:45:37.892851+0000","last_peered":"2024-09-14T04:45:37.892851+0000","last_clean":"2024-09-14T04:45:37.892851+0000","last_became_active":"2024-09-14T04:44:14.488950+0000","last_became_peered":"2024-09-14T04:44:14.488950+0000","last_unstale":"2024-09-14T04:45:37.892851+0000","last_undegraded":"2024-09-14T04:45:37.892851+0000","last_fullsized":"2024-09-14T04:45:37.892851+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:54.441144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":2113,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.891699+0000","last_change":"2024-09-14T04:07:21.685415+0000","last_active":"2024-09-14T04:45:37.891699+0000","last_peered":"2024-09-14T04:45:37.891699+0000","last_clean":"2024-09-14T04:45:37.891699+0000","last_became_active":"2024-09-14T04:07:21.683504+0000","last_became_peered":"2024-09-14T04:07:21.683504+0000","last_unstale":"2024-09-14T04:45:37.891699+0000","last_undegraded":"2024-09-14T04:45:37.891699+0000","last_fullsized":"2024-09-14T04:45:37.891699+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:21:08.806291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2115,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.890669+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T04:45:37.890669+0000","last_peered":"2024-09-14T04:45:37.890669+0000","last_clean":"2024-09-14T04:45:37.890669+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T04:45:37.890669+0000","last_undegraded":"2024-09-14T04:45:37.890669+0000","last_fullsized":"2024-09-14T04:45:37.890669+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"349'149","reported_seq":790,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.892501+0000","last_change":"2024-09-14T04:45:00.342826+0000","last_active":"2024-09-14T04:45:37.892501+0000","last_peered":"2024-09-14T04:45:37.892501+0000","last_clean":"2024-09-14T04:45:37.892501+0000","last_became_active":"2024-09-14T04:45:00.342013+0000","last_became_peered":"2024-09-14T04:45:00.342013+0000","last_unstale":"2024-09-14T04:45:37.892501+0000","last_undegraded":"2024-09-14T04:45:37.892501+0000","last_fullsized":"2024-09-14T04:45:37.892501+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.326129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.326129+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:54:28.021603+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2116,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.884119+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T04:45:37.884119+0000","last_peered":"2024-09-14T04:45:37.884119+0000","last_clean":"2024-09-14T04:45:37.884119+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T04:45:37.884119+0000","last_undegraded":"2024-09-14T04:45:37.884119+0000","last_fullsized":"2024-09-14T04:45:37.884119+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2123,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.883701+0000","last_change":"2024-09-14T04:44:57.335783+0000","last_active":"2024-09-14T04:45:37.883701+0000","last_peered":"2024-09-14T04:45:37.883701+0000","last_clean":"2024-09-14T04:45:37.883701+0000","last_became_active":"2024-09-14T04:44:57.332780+0000","last_became_peered":"2024-09-14T04:44:57.332780+0000","last_unstale":"2024-09-14T04:45:37.883701+0000","last_undegraded":"2024-09-14T04:45:37.883701+0000","last_fullsized":"2024-09-14T04:45:37.883701+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:57.457593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1427,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.820503+0000","last_change":"2024-09-14T04:42:49.939790+0000","last_active":"2024-09-14T04:45:34.820503+0000","last_peered":"2024-09-14T04:45:34.820503+0000","last_clean":"2024-09-14T04:45:34.820503+0000","last_became_active":"2024-09-14T04:42:49.939591+0000","last_became_peered":"2024-09-14T04:42:49.939591+0000","last_unstale":"2024-09-14T04:45:34.820503+0000","last_undegraded":"2024-09-14T04:45:34.820503+0000","last_fullsized":"2024-09-14T04:45:34.820503+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:23:16.790593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1391,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.817550+0000","last_change":"2024-09-14T04:44:57.326008+0000","last_active":"2024-09-14T04:45:34.817550+0000","last_peered":"2024-09-14T04:45:34.817550+0000","last_clean":"2024-09-14T04:45:34.817550+0000","last_became_active":"2024-09-14T04:44:57.325617+0000","last_became_peered":"2024-09-14T04:44:57.325617+0000","last_unstale":"2024-09-14T04:45:34.817550+0000","last_undegraded":"2024-09-14T04:45:34.817550+0000","last_fullsized":"2024-09-14T04:45:34.817550+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:15.637508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":680,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.817174+0000","last_change":"2024-09-14T04:43:13.354846+0000","last_active":"2024-09-14T04:45:34.817174+0000","last_peered":"2024-09-14T04:45:34.817174+0000","last_clean":"2024-09-14T04:45:34.817174+0000","last_became_active":"2024-09-14T04:43:13.354515+0000","last_became_peered":"2024-09-14T04:43:13.354515+0000","last_unstale":"2024-09-14T04:45:34.817174+0000","last_undegraded":"2024-09-14T04:45:34.817174+0000","last_fullsized":"2024-09-14T04:45:34.817174+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:17.889341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1396,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.817663+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T04:45:34.817663+0000","last_peered":"2024-09-14T04:45:34.817663+0000","last_clean":"2024-09-14T04:45:34.817663+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T04:45:34.817663+0000","last_undegraded":"2024-09-14T04:45:34.817663+0000","last_fullsized":"2024-09-14T04:45:34.817663+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":1734,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.820373+0000","last_change":"2024-09-14T04:41:30.640651+0000","last_active":"2024-09-14T04:45:34.820373+0000","last_peered":"2024-09-14T04:45:34.820373+0000","last_clean":"2024-09-14T04:45:34.820373+0000","last_became_active":"2024-09-14T04:41:29.532466+0000","last_became_peered":"2024-09-14T04:41:29.532466+0000","last_unstale":"2024-09-14T04:45:34.820373+0000","last_undegraded":"2024-09-14T04:45:34.820373+0000","last_fullsized":"2024-09-14T04:45:34.820373+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":0,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:30.640520+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:30.640520+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:27.776724+0000","scrub_duration":0.0028653139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"349'149","reported_seq":708,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.816955+0000","last_change":"2024-09-14T04:41:39.978080+0000","last_active":"2024-09-14T04:45:34.816955+0000","last_peered":"2024-09-14T04:45:34.816955+0000","last_clean":"2024-09-14T04:45:34.816955+0000","last_became_active":"2024-09-14T04:41:36.929279+0000","last_became_peered":"2024-09-14T04:41:36.929279+0000","last_unstale":"2024-09-14T04:45:34.816955+0000","last_undegraded":"2024-09-14T04:45:34.816955+0000","last_fullsized":"2024-09-14T04:45:34.816955+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.977999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.977999+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:44:06.383827+0000","scrub_duration":0.00192122,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":2095,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.892942+0000","last_change":"2024-09-14T04:44:57.330745+0000","last_active":"2024-09-14T04:45:37.892942+0000","last_peered":"2024-09-14T04:45:37.892942+0000","last_clean":"2024-09-14T04:45:37.892942+0000","last_became_active":"2024-09-14T04:44:57.330261+0000","last_became_peered":"2024-09-14T04:44:57.330261+0000","last_unstale":"2024-09-14T04:45:37.892942+0000","last_undegraded":"2024-09-14T04:45:37.892942+0000","last_fullsized":"2024-09-14T04:45:37.892942+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:24:08.870976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2099,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.894880+0000","last_change":"2024-09-14T04:42:49.941397+0000","last_active":"2024-09-14T04:45:37.894880+0000","last_peered":"2024-09-14T04:45:37.894880+0000","last_clean":"2024-09-14T04:45:37.894880+0000","last_became_active":"2024-09-14T04:42:49.941066+0000","last_became_peered":"2024-09-14T04:42:49.941066+0000","last_unstale":"2024-09-14T04:45:37.894880+0000","last_undegraded":"2024-09-14T04:45:37.894880+0000","last_fullsized":"2024-09-14T04:45:37.894880+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:30.777822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":20,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.815716+0000","last_change":"2024-09-14T04:44:57.326724+0000","last_active":"2024-09-14T04:45:34.815716+0000","last_peered":"2024-09-14T04:45:34.815716+0000","last_clean":"2024-09-14T04:45:34.815716+0000","last_became_active":"2024-09-14T04:44:57.326191+0000","last_became_peered":"2024-09-14T04:44:57.326191+0000","last_unstale":"2024-09-14T04:45:34.815716+0000","last_undegraded":"2024-09-14T04:45:34.815716+0000","last_fullsized":"2024-09-14T04:45:34.815716+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:46.060269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":20,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.814514+0000","last_change":"2024-09-14T04:44:57.328528+0000","last_active":"2024-09-14T04:45:34.814514+0000","last_peered":"2024-09-14T04:45:34.814514+0000","last_clean":"2024-09-14T04:45:34.814514+0000","last_became_active":"2024-09-14T04:44:57.327739+0000","last_became_peered":"2024-09-14T04:44:57.327739+0000","last_unstale":"2024-09-14T04:45:34.814514+0000","last_undegraded":"2024-09-14T04:45:34.814514+0000","last_fullsized":"2024-09-14T04:45:34.814514+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:04:58.294690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"349'149","reported_seq":740,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.817840+0000","last_change":"2024-09-14T04:42:53.013201+0000","last_active":"2024-09-14T04:45:34.817840+0000","last_peered":"2024-09-14T04:45:34.817840+0000","last_clean":"2024-09-14T04:45:34.817840+0000","last_became_active":"2024-09-14T04:42:53.012818+0000","last_became_peered":"2024-09-14T04:42:53.012818+0000","last_unstale":"2024-09-14T04:45:34.817840+0000","last_undegraded":"2024-09-14T04:45:34.817840+0000","last_fullsized":"2024-09-14T04:45:34.817840+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:56.738636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:56.738636+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:39:19.056067+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":20,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.815287+0000","last_change":"2024-09-14T04:44:57.323436+0000","last_active":"2024-09-14T04:45:34.815287+0000","last_peered":"2024-09-14T04:45:34.815287+0000","last_clean":"2024-09-14T04:45:34.815287+0000","last_became_active":"2024-09-14T04:44:57.323022+0000","last_became_peered":"2024-09-14T04:44:57.323022+0000","last_unstale":"2024-09-14T04:45:34.815287+0000","last_undegraded":"2024-09-14T04:45:34.815287+0000","last_fullsized":"2024-09-14T04:45:34.815287+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:22.775261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1426,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.817118+0000","last_change":"2024-09-14T04:44:57.322357+0000","last_active":"2024-09-14T04:45:34.817118+0000","last_peered":"2024-09-14T04:45:34.817118+0000","last_clean":"2024-09-14T04:45:34.817118+0000","last_became_active":"2024-09-14T04:44:57.321990+0000","last_became_peered":"2024-09-14T04:44:57.321990+0000","last_unstale":"2024-09-14T04:45:34.817118+0000","last_undegraded":"2024-09-14T04:45:34.817118+0000","last_fullsized":"2024-09-14T04:45:34.817118+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:09.658008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"349'149","reported_seq":1750,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.820423+0000","last_change":"2024-09-14T04:41:53.858076+0000","last_active":"2024-09-14T04:45:34.820423+0000","last_peered":"2024-09-14T04:45:34.820423+0000","last_clean":"2024-09-14T04:45:34.820423+0000","last_became_active":"2024-09-14T04:41:36.931585+0000","last_became_peered":"2024-09-14T04:41:36.931585+0000","last_unstale":"2024-09-14T04:45:34.820423+0000","last_undegraded":"2024-09-14T04:45:34.820423+0000","last_fullsized":"2024-09-14T04:45:34.820423+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:53.857924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:53.857924+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:10:20.658986+0000","scrub_duration":0.0022355919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":20,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.815518+0000","last_change":"2024-09-14T04:44:57.329198+0000","last_active":"2024-09-14T04:45:34.815518+0000","last_peered":"2024-09-14T04:45:34.815518+0000","last_clean":"2024-09-14T04:45:34.815518+0000","last_became_active":"2024-09-14T04:44:57.327754+0000","last_became_peered":"2024-09-14T04:44:57.327754+0000","last_unstale":"2024-09-14T04:45:34.815518+0000","last_undegraded":"2024-09-14T04:45:34.815518+0000","last_fullsized":"2024-09-14T04:45:34.815518+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:10:27.944476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":100,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.888688+0000","last_change":"2024-09-14T04:44:57.348687+0000","last_active":"2024-09-14T04:45:37.888688+0000","last_peered":"2024-09-14T04:45:37.888688+0000","last_clean":"2024-09-14T04:45:37.888688+0000","last_became_active":"2024-09-14T04:44:57.332845+0000","last_became_peered":"2024-09-14T04:44:57.332845+0000","last_unstale":"2024-09-14T04:45:37.888688+0000","last_undegraded":"2024-09-14T04:45:37.888688+0000","last_fullsized":"2024-09-14T04:45:37.888688+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:12:18.762737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2122,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.888053+0000","last_change":"2024-09-14T04:44:57.333468+0000","last_active":"2024-09-14T04:45:37.888053+0000","last_peered":"2024-09-14T04:45:37.888053+0000","last_clean":"2024-09-14T04:45:37.888053+0000","last_became_active":"2024-09-14T04:44:57.332626+0000","last_became_peered":"2024-09-14T04:44:57.332626+0000","last_unstale":"2024-09-14T04:45:37.888053+0000","last_undegraded":"2024-09-14T04:45:37.888053+0000","last_fullsized":"2024-09-14T04:45:37.888053+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:57:26.366438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1382,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.817344+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T04:45:34.817344+0000","last_peered":"2024-09-14T04:45:34.817344+0000","last_clean":"2024-09-14T04:45:34.817344+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T04:45:34.817344+0000","last_undegraded":"2024-09-14T04:45:34.817344+0000","last_fullsized":"2024-09-14T04:45:34.817344+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1381,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.816774+0000","last_change":"2024-09-14T04:06:46.869868+0000","last_active":"2024-09-14T04:45:34.816774+0000","last_peered":"2024-09-14T04:45:34.816774+0000","last_clean":"2024-09-14T04:45:34.816774+0000","last_became_active":"2024-09-14T04:06:46.869126+0000","last_became_peered":"2024-09-14T04:06:46.869126+0000","last_unstale":"2024-09-14T04:45:34.816774+0000","last_undegraded":"2024-09-14T04:45:34.816774+0000","last_fullsized":"2024-09-14T04:45:34.816774+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:41.150138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1365,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.821578+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T04:45:34.821578+0000","last_peered":"2024-09-14T04:45:34.821578+0000","last_clean":"2024-09-14T04:45:34.821578+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T04:45:34.821578+0000","last_undegraded":"2024-09-14T04:45:34.821578+0000","last_fullsized":"2024-09-14T04:45:34.821578+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":20,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.814608+0000","last_change":"2024-09-14T04:44:57.332112+0000","last_active":"2024-09-14T04:45:34.814608+0000","last_peered":"2024-09-14T04:45:34.814608+0000","last_clean":"2024-09-14T04:45:34.814608+0000","last_became_active":"2024-09-14T04:44:57.331575+0000","last_became_peered":"2024-09-14T04:44:57.331575+0000","last_unstale":"2024-09-14T04:45:34.814608+0000","last_undegraded":"2024-09-14T04:45:34.814608+0000","last_fullsized":"2024-09-14T04:45:34.814608+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:50:00.391283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1413,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.821766+0000","last_change":"2024-09-14T04:44:57.351460+0000","last_active":"2024-09-14T04:45:34.821766+0000","last_peered":"2024-09-14T04:45:34.821766+0000","last_clean":"2024-09-14T04:45:34.821766+0000","last_became_active":"2024-09-14T04:44:57.334662+0000","last_became_peered":"2024-09-14T04:44:57.334662+0000","last_unstale":"2024-09-14T04:45:34.821766+0000","last_undegraded":"2024-09-14T04:45:34.821766+0000","last_fullsized":"2024-09-14T04:45:34.821766+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:02.399062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1367,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.817954+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T04:45:34.817954+0000","last_peered":"2024-09-14T04:45:34.817954+0000","last_clean":"2024-09-14T04:45:34.817954+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T04:45:34.817954+0000","last_undegraded":"2024-09-14T04:45:34.817954+0000","last_fullsized":"2024-09-14T04:45:34.817954+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1425,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.822363+0000","last_change":"2024-09-14T04:44:57.349690+0000","last_active":"2024-09-14T04:45:34.822363+0000","last_peered":"2024-09-14T04:45:34.822363+0000","last_clean":"2024-09-14T04:45:34.822363+0000","last_became_active":"2024-09-14T04:44:57.333579+0000","last_became_peered":"2024-09-14T04:44:57.333579+0000","last_unstale":"2024-09-14T04:45:34.822363+0000","last_undegraded":"2024-09-14T04:45:34.822363+0000","last_fullsized":"2024-09-14T04:45:34.822363+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:17:55.265141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1426,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.816385+0000","last_change":"2024-09-14T04:42:49.960838+0000","last_active":"2024-09-14T04:45:34.816385+0000","last_peered":"2024-09-14T04:45:34.816385+0000","last_clean":"2024-09-14T04:45:34.816385+0000","last_became_active":"2024-09-14T04:42:49.954736+0000","last_became_peered":"2024-09-14T04:42:49.954736+0000","last_unstale":"2024-09-14T04:45:34.816385+0000","last_undegraded":"2024-09-14T04:45:34.816385+0000","last_fullsized":"2024-09-14T04:45:34.816385+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:41:13.540983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":2081,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.891169+0000","last_change":"2024-09-14T04:07:58.565442+0000","last_active":"2024-09-14T04:45:37.891169+0000","last_peered":"2024-09-14T04:45:37.891169+0000","last_clean":"2024-09-14T04:45:37.891169+0000","last_became_active":"2024-09-14T04:07:58.552289+0000","last_became_peered":"2024-09-14T04:07:58.552289+0000","last_unstale":"2024-09-14T04:45:37.891169+0000","last_undegraded":"2024-09-14T04:45:37.891169+0000","last_fullsized":"2024-09-14T04:45:37.891169+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:48:02.364960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1387,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.821163+0000","last_change":"2024-09-14T04:44:57.332511+0000","last_active":"2024-09-14T04:45:34.821163+0000","last_peered":"2024-09-14T04:45:34.821163+0000","last_clean":"2024-09-14T04:45:34.821163+0000","last_became_active":"2024-09-14T04:44:57.332166+0000","last_became_peered":"2024-09-14T04:44:57.332166+0000","last_unstale":"2024-09-14T04:45:34.821163+0000","last_undegraded":"2024-09-14T04:45:34.821163+0000","last_fullsized":"2024-09-14T04:45:34.821163+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:53.984639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2102,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.883883+0000","last_change":"2024-09-14T04:07:58.560880+0000","last_active":"2024-09-14T04:45:37.883883+0000","last_peered":"2024-09-14T04:45:37.883883+0000","last_clean":"2024-09-14T04:45:37.883883+0000","last_became_active":"2024-09-14T04:07:58.560132+0000","last_became_peered":"2024-09-14T04:07:58.560132+0000","last_unstale":"2024-09-14T04:45:37.883883+0000","last_undegraded":"2024-09-14T04:45:37.883883+0000","last_fullsized":"2024-09-14T04:45:37.883883+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:46:00.287050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2102,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.885631+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T04:45:37.885631+0000","last_peered":"2024-09-14T04:45:37.885631+0000","last_clean":"2024-09-14T04:45:37.885631+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T04:45:37.885631+0000","last_undegraded":"2024-09-14T04:45:37.885631+0000","last_fullsized":"2024-09-14T04:45:37.885631+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1379,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.817051+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T04:45:34.817051+0000","last_peered":"2024-09-14T04:45:34.817051+0000","last_clean":"2024-09-14T04:45:34.817051+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T04:45:34.817051+0000","last_undegraded":"2024-09-14T04:45:34.817051+0000","last_fullsized":"2024-09-14T04:45:34.817051+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1416,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.820903+0000","last_change":"2024-09-14T04:44:57.339993+0000","last_active":"2024-09-14T04:45:34.820903+0000","last_peered":"2024-09-14T04:45:34.820903+0000","last_clean":"2024-09-14T04:45:34.820903+0000","last_became_active":"2024-09-14T04:44:57.330472+0000","last_became_peered":"2024-09-14T04:44:57.330472+0000","last_unstale":"2024-09-14T04:45:34.820903+0000","last_undegraded":"2024-09-14T04:45:34.820903+0000","last_fullsized":"2024-09-14T04:45:34.820903+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:19:27.046988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":42,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.892146+0000","last_change":"2024-09-14T04:44:14.479373+0000","last_active":"2024-09-14T04:45:37.892146+0000","last_peered":"2024-09-14T04:45:37.892146+0000","last_clean":"2024-09-14T04:45:37.892146+0000","last_became_active":"2024-09-14T04:44:14.479084+0000","last_became_peered":"2024-09-14T04:44:14.479084+0000","last_unstale":"2024-09-14T04:45:37.892146+0000","last_undegraded":"2024-09-14T04:45:37.892146+0000","last_fullsized":"2024-09-14T04:45:37.892146+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:49:23.412806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":54,"num_pg":16,"stat_sum":{"num_bytes":5267193880,"num_objects":80372,"num_object_clones":0,"num_object_copies":241116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":21468,"num_objects_unfound":0,"num_objects_dirty":80372,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":172418,"num_write_kb":8089217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":714,"num_bytes_recovered":46727168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15785865216,"data_stored":15785853000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":121935,"ondisk_log_size":121935,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":5070,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4250,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201937,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2548140,"kb_used_data":2250340,"kb_used_omap":1,"kb_used_meta":297662,"kb_avail":91197012,"statfs":{"total":95995035648,"available":93385740288,"internally_reserved":0,"allocated":2304348160,"data_stored":2299876534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":304806347},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463573,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2968808,"kb_used_data":2568544,"kb_used_omap":1,"kb_used_meta":400190,"kb_avail":90776344,"statfs":{"total":95995035648,"available":92954976256,"internally_reserved":0,"allocated":2630189056,"data_stored":2625721423,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":409795015},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725210,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2931936,"kb_used_data":2543896,"kb_used_omap":1,"kb_used_meta":387902,"kb_avail":90813216,"statfs":{"total":95995035648,"available":92992733184,"internally_reserved":0,"allocated":2604949504,"data_stored":2600486409,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":397212102},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":1392,"seq":5978594476056,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":96876,"kb_used_data":11620,"kb_used_omap":117,"kb_used_meta":85130,"kb_avail":93648276,"statfs":{"total":95995035648,"available":95895834624,"internally_reserved":0,"allocated":11898880,"data_stored":7427806,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120638,"internal_metadata":87173314},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772074951,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3927648,"kb_used_data":3548504,"kb_used_omap":71,"kb_used_meta":379064,"kb_avail":89817504,"statfs":{"total":95995035648,"available":91973124096,"internally_reserved":0,"allocated":3633668096,"data_stored":3629205078,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":388162486},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707303886,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1215644,"kb_used_data":1007252,"kb_used_omap":68,"kb_used_meta":208315,"kb_avail":92529508,"statfs":{"total":95995035648,"available":94750216192,"internally_reserved":0,"allocated":1031426048,"data_stored":1026967006,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":213314768},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872729045,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1675924,"kb_used_data":1337100,"kb_used_omap":67,"kb_used_meta":338748,"kb_avail":92069228,"statfs":{"total":95995035648,"available":94278889472,"internally_reserved":0,"allocated":1369190400,"data_stored":1364738409,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":346878309},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1,1,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056096,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2620060,"kb_used_data":2281428,"kb_used_omap":74,"kb_used_meta":338549,"kb_avail":91125092,"statfs":{"total":95995035648,"available":93312094208,"internally_reserved":0,"allocated":2336182272,"data_stored":2331722698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":346674869},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1901,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2303987712,"data_stored":2303983640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1358102528,"data_stored":1358102528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1017970688,"data_stored":1017970688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3616411648,"data_stored":3616407576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1245184,"data_stored":1245184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2586836992,"data_stored":2586836992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2608926720,"data_stored":2608922648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2292383744,"data_stored":2292383744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:45:45.445 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:45:45.499 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:45.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:45.551 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:45.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:45.652 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:45.681 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:45.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:45.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:45.783 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:45.813 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:45.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:45.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:45.838 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:45:45.878 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:45:45.878 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:45:45.915 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:45.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:45.941 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:45.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:46.042 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:46.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:46.084 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:46.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:46.185 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:46.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:46.227 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:46.228 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:46.328 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:46.372 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:46.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:46.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:46.473 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:46.513 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:46.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:46.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:46.615 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:46.653 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:46.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:46.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:46.721 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:45:46.723 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:45:46.755 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:46.768 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-14T04:45:46.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:46.823 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:46.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:46.924 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:46.959 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:46.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:46.960 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:47.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:46 smithi039 ceph-mon[106470]: pgmap v2466: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 21367/238935 objects misplaced (8.943%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:45:47.043 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:46 smithi039 ceph-mon[111235]: pgmap v2466: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 21367/238935 objects misplaced (8.943%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:45:47.057 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:46 smithi129 ceph-mon[101316]: pgmap v2466: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 4.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 21367/238935 objects misplaced (8.943%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:45:47.061 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:47.091 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:47.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:47.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:47.192 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:47.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:47.237 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:47.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:47.338 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:47.379 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:47.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:47.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:47.481 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:47.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:47.520 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:47.521 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:47.621 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:47.656 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:47.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:47.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:47.757 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:47.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:47.796 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:47.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:47.897 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:47.919 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:47.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:47.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:48.021 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:48.049 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:48.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:48.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:48.150 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:48.189 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:48.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:48.190 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:48.290 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:48.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:48.343 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:48.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:48.444 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:48.482 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:48.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:48.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:48.584 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:48.621 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:48.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:48.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:48.723 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:48.767 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:48.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:48.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:48.869 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:48.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:48.901 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:48.901 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:48.913 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:45:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:45:48] "GET /metrics HTTP/1.1" 200 36751 "" "Prometheus/2.51.0" 2024-09-14T04:45:49.002 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:49.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:49.033 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:49.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:49.134 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:49.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:49.178 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:49.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:49.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:48 smithi129 ceph-mon[101316]: pgmap v2467: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 21136/235989 objects misplaced (8.956%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:45:49.278 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:48 smithi039 ceph-mon[106470]: pgmap v2467: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 21136/235989 objects misplaced (8.956%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:45:49.279 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:49.280 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:48 smithi039 ceph-mon[111235]: pgmap v2467: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 4.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 21136/235989 objects misplaced (8.956%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:45:49.309 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:49.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:49.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:49.411 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:49.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:49.449 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:49.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:49.550 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:49.581 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:49.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:49.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:49.682 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:49.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:49.727 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:49.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:49.828 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:49.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:49.868 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:49.868 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:49.969 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:50.007 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:50.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:50.008 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:50.109 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:50.149 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:50.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:50.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:50.250 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:50.291 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:50.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:50.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:50.393 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:50.431 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:50.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:50.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:50.533 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:50.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:50 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:45:50.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:50 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:45:50.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:50 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:45:50.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:50 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:45:50.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:50.568 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:50.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:50.669 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:50.705 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:50.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:50.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:50.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:50 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:45:50.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:50 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:45:50.807 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:50.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:50.840 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:50.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:50.941 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:50.965 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:50.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:50.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:51.067 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:51.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:51.099 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:51.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:51.200 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:51.231 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:51.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:51.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:51.333 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:51.356 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:51.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:51.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:51.458 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:51.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:51.498 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:51.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:51.599 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:51.648 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:51.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:51.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:51.750 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:51 smithi129 ceph-mon[101316]: pgmap v2468: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 20633/229359 objects misplaced (8.996%); 1.0 MiB/s, 16 objects/s recovering 2024-09-14T04:45:51.783 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:51.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:51.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:51 smithi039 ceph-mon[106470]: pgmap v2468: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 20633/229359 objects misplaced (8.996%); 1.0 MiB/s, 16 objects/s recovering 2024-09-14T04:45:51.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:51 smithi039 ceph-mon[111235]: pgmap v2468: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 20633/229359 objects misplaced (8.996%); 1.0 MiB/s, 16 objects/s recovering 2024-09-14T04:45:51.886 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:51.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:51.912 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:51.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:52.013 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:52.046 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:52.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:52.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:52.148 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:52.176 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:52.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:52.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:52.278 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:52.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:52.319 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:52.319 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:52.419 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:52.457 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:52.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:52.458 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:52.559 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:45:52.595 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:52.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:52.596 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:45:52.697 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:52.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:52.737 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:52.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:52.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:45:52.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:45:52.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:45:52.838 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:52.868 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:52.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:52.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:52.970 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:53.002 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:53.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:53.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:53.104 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:53.135 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:53.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:53.136 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:53.237 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:53.278 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:53.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:53.279 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:53.380 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:53.405 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:53.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:53.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:53.507 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:53.546 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:53.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:53.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:53.649 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:53.676 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:53.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:53.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:53 smithi129 ceph-mon[101316]: pgmap v2469: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 20540/227127 objects misplaced (9.043%); 1.5 MiB/s, 23 objects/s recovering 2024-09-14T04:45:53.778 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:53.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:53 smithi039 ceph-mon[106470]: pgmap v2469: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 20540/227127 objects misplaced (9.043%); 1.5 MiB/s, 23 objects/s recovering 2024-09-14T04:45:53.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:53 smithi039 ceph-mon[111235]: pgmap v2469: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 4.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 20540/227127 objects misplaced (9.043%); 1.5 MiB/s, 23 objects/s recovering 2024-09-14T04:45:53.812 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:53.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:53.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:53.913 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:53.948 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:53.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:53.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:54.050 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:54.081 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:54.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:54.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:54.182 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:54.214 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:54.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:54.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:54.316 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:54.356 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:54.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:54.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:54.458 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:54.506 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:54.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:54.507 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:54.608 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:54.647 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:54.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:54.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:54.749 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:54.788 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:54.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:54.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:54.890 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:54.922 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:54.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:54.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:55.024 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:55.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:55.063 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:55.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:55.164 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:55.196 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:55.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:55.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:55.297 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:55.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:55.333 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:55.333 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:55.434 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:55.463 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:55.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:55.464 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:55.565 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:55.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:55.599 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:55.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:55.700 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:55.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:55.732 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:55.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:55.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:55 smithi129 ceph-mon[101316]: pgmap v2470: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 944 op/s; 19774/216996 objects misplaced (9.113%); 1.0 MiB/s, 16 objects/s recovering 2024-09-14T04:45:55.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:55 smithi039 ceph-mon[106470]: pgmap v2470: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 944 op/s; 19774/216996 objects misplaced (9.113%); 1.0 MiB/s, 16 objects/s recovering 2024-09-14T04:45:55.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:55 smithi039 ceph-mon[111235]: pgmap v2470: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 944 op/s; 19774/216996 objects misplaced (9.113%); 1.0 MiB/s, 16 objects/s recovering 2024-09-14T04:45:55.833 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:55.872 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:55.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:55.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:55.974 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:56.004 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:56.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:56.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:56.106 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:56.129 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:56.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:56.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:56.231 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:56.276 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:56.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:56.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:56.378 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:56.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:56.409 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:56.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:56.510 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:56.536 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:56.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:56.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:56.638 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:56.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:56.680 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:56.680 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:56.781 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:56.829 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:56.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:56.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:56.931 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:56.995 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:56.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:56.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:57.097 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:57.130 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:57.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:57.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:57.232 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:57.273 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:57.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:57.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:57.375 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:45:57.412 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:57.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:57.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:45:57.514 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:57.562 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:57.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:57.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:57.664 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:57.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:57.696 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:57.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:57.701 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:57 smithi039 ceph-mon[106470]: pgmap v2471: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 19692/214641 objects misplaced (9.174%); 1.4 MiB/s, 22 objects/s recovering 2024-09-14T04:45:57.701 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:57 smithi039 ceph-mon[111235]: pgmap v2471: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 19692/214641 objects misplaced (9.174%); 1.4 MiB/s, 22 objects/s recovering 2024-09-14T04:45:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:57 smithi129 ceph-mon[101316]: pgmap v2471: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 4.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 737 op/s; 19692/214641 objects misplaced (9.174%); 1.4 MiB/s, 22 objects/s recovering 2024-09-14T04:45:57.796 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:57.841 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:57.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:57.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:57.943 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:57.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:57.976 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:57.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:58.077 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:58.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:58.116 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:58.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:58.217 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:45:58.257 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:58.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:58.259 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:45:58.359 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:45:58.393 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:58.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:58.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:45:58.495 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:58.522 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:58.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:58.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:58.623 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:58.630 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:45:58.632 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T04:45:58.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:58.650 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:58.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:58.661 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:45:58.751 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:58.777 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:58.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:58.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:58.878 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:58.928 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:58.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:58.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:58.991 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:45:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:45:58] "GET /metrics HTTP/1.1" 200 36755 "" "Prometheus/2.51.0" 2024-09-14T04:45:59.030 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:59.065 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:59.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:59.068 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:59.169 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:59.208 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:59.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:59.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:59.310 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:45:59.350 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:59.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:59.351 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:45:59.452 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:45:59.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:59.477 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:45:59.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:45:59.578 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:59.609 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:59.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:59.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:59.711 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:45:59.748 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:59.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:59.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:45:59.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:59 smithi129 ceph-mon[101316]: pgmap v2472: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 19435/211620 objects misplaced (9.184%); 933 KiB/s, 14 objects/s recovering 2024-09-14T04:45:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:45:59 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3291711008' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:45:59.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:59 smithi039 ceph-mon[106470]: pgmap v2472: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 19435/211620 objects misplaced (9.184%); 933 KiB/s, 14 objects/s recovering 2024-09-14T04:45:59.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:45:59 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3291711008' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:45:59.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:59 smithi039 ceph-mon[111235]: pgmap v2472: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 4.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 758 op/s; 19435/211620 objects misplaced (9.184%); 933 KiB/s, 14 objects/s recovering 2024-09-14T04:45:59.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:45:59 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3291711008' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:45:59.850 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:45:59.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:45:59.889 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:45:59.889 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:45:59.990 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:00.017 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:00.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:00.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:00.119 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:00.158 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:00.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:00.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:00.260 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:00.295 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:00.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:00.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:00.324 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":2472,"stamp":"2024-09-14T04:45:57.660548+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4621735933,"num_objects":70540,"num_object_clones":0,"num_object_copies":211620,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":19435,"num_objects_unfound":0,"num_objects_dirty":70540,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":182340,"num_write_kb":8089250,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1023,"num_bytes_recovered":65082862,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123791,"ondisk_log_size":123791,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":301,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":16206220,"kb_used_data":13748044,"kb_used_omap":404,"kb_used_meta":2457643,"kb_avail":733754996,"statfs":{"total":767960285184,"available":751365115904,"internally_reserved":0,"allocated":14077997056,"data_stored":14042290003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":414022,"internal_metadata":2516627130},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-596705280,"num_objects":-9105,"num_object_clones":0,"num_object_copies":-27315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1932,"num_objects_unfound":0,"num_objects_dirty":-9105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9105,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":175,"num_bytes_recovered":11468800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004766"},"pg_stats":[{"pgid":"54.f","version":"1412'11486","reported_seq":11562,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:57.211506+0000","last_change":"2024-09-14T04:43:01.044742+0000","last_active":"2024-09-14T04:45:57.211506+0000","last_peered":"2024-09-14T04:45:57.211506+0000","last_clean":"2024-09-14T04:45:57.211506+0000","last_became_active":"2024-09-14T04:43:01.044421+0000","last_became_peered":"2024-09-14T04:43:01.044421+0000","last_unstale":"2024-09-14T04:45:57.211506+0000","last_undegraded":"2024-09-14T04:45:57.211506+0000","last_fullsized":"2024-09-14T04:45:57.211506+0000","mapping_epoch":1380,"log_start":"1393'4800","ondisk_log_start":"1393'4800","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":6686,"log_dups_size":0,"ondisk_log_size":6686,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:28.897426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271450112,"num_objects":4142,"num_object_clones":0,"num_object_copies":12426,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4142,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11486,"num_write_kb":500096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"54.e","version":"1412'11467","reported_seq":13460,"reported_epoch":1412,"state":"active+recovering+remapped","last_fresh":"2024-09-14T04:45:55.551542+0000","last_change":"2024-09-14T04:44:58.690021+0000","last_active":"2024-09-14T04:45:55.551542+0000","last_peered":"2024-09-14T04:45:55.551542+0000","last_clean":"2024-09-14T04:44:56.302285+0000","last_became_active":"2024-09-14T04:44:58.579754+0000","last_became_peered":"2024-09-14T04:44:58.579754+0000","last_unstale":"2024-09-14T04:45:55.551542+0000","last_undegraded":"2024-09-14T04:45:55.551542+0000","last_fullsized":"2024-09-14T04:45:55.551542+0000","mapping_epoch":1403,"log_start":"1390'3483","ondisk_log_start":"1390'3483","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7984,"log_dups_size":0,"ondisk_log_size":7984,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284098560,"num_objects":4335,"num_object_clones":0,"num_object_copies":13005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6368,"num_objects_unfound":0,"num_objects_dirty":4335,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11467,"num_write_kb":505664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":958,"num_bytes_recovered":62717952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[1,2,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":-2033}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"54.d","version":"1412'11382","reported_seq":11425,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:53.192443+0000","last_change":"2024-09-14T04:43:01.050000+0000","last_active":"2024-09-14T04:45:53.192443+0000","last_peered":"2024-09-14T04:45:53.192443+0000","last_clean":"2024-09-14T04:45:53.192443+0000","last_became_active":"2024-09-14T04:43:01.049752+0000","last_became_peered":"2024-09-14T04:43:01.049752+0000","last_unstale":"2024-09-14T04:45:53.192443+0000","last_undegraded":"2024-09-14T04:45:53.192443+0000","last_fullsized":"2024-09-14T04:45:53.192443+0000","mapping_epoch":1380,"log_start":"1390'3800","ondisk_log_start":"1390'3800","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7582,"log_dups_size":0,"ondisk_log_size":7582,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:35:06.578523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294977560,"num_objects":4502,"num_object_clones":0,"num_object_copies":13506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4502,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11382,"num_write_kb":508225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"54.c","version":"1412'11564","reported_seq":11644,"reported_epoch":1412,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:45:57.221215+0000","last_change":"2024-09-14T04:44:58.455281+0000","last_active":"2024-09-14T04:45:57.221215+0000","last_peered":"2024-09-14T04:45:57.221215+0000","last_clean":"2024-09-14T04:44:56.285515+0000","last_became_active":"2024-09-14T04:44:58.445654+0000","last_became_peered":"2024-09-14T04:44:58.445654+0000","last_unstale":"2024-09-14T04:45:57.221215+0000","last_undegraded":"2024-09-14T04:45:57.221215+0000","last_fullsized":"2024-09-14T04:45:57.221215+0000","mapping_epoch":1403,"log_start":"1393'4800","ondisk_log_start":"1393'4800","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":6764,"log_dups_size":0,"ondisk_log_size":6764,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275513344,"num_objects":4204,"num_object_clones":0,"num_object_copies":12612,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4204,"num_objects_unfound":0,"num_objects_dirty":4204,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11564,"num_write_kb":504576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[{"shards":"3,5,6","objects":4204}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"54.b","version":"1412'11248","reported_seq":11313,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:53.093002+0000","last_change":"2024-09-14T04:43:13.355592+0000","last_active":"2024-09-14T04:45:53.093002+0000","last_peered":"2024-09-14T04:45:53.093002+0000","last_clean":"2024-09-14T04:45:53.093002+0000","last_became_active":"2024-09-14T04:43:13.355130+0000","last_became_peered":"2024-09-14T04:43:13.355130+0000","last_unstale":"2024-09-14T04:45:53.093002+0000","last_undegraded":"2024-09-14T04:45:53.093002+0000","last_fullsized":"2024-09-14T04:45:53.093002+0000","mapping_epoch":1386,"log_start":"1389'2900","ondisk_log_start":"1389'2900","created":1380,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":8348,"log_dups_size":0,"ondisk_log_size":8348,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:42:44.062290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292159488,"num_objects":4458,"num_object_clones":0,"num_object_copies":13374,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4458,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11248,"num_write_kb":502592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"54.a","version":"1412'11249","reported_seq":11291,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:53.169984+0000","last_change":"2024-09-14T04:43:01.043585+0000","last_active":"2024-09-14T04:45:53.169984+0000","last_peered":"2024-09-14T04:45:53.169984+0000","last_clean":"2024-09-14T04:45:53.169984+0000","last_became_active":"2024-09-14T04:43:01.043295+0000","last_became_peered":"2024-09-14T04:43:01.043295+0000","last_unstale":"2024-09-14T04:45:53.169984+0000","last_undegraded":"2024-09-14T04:45:53.169984+0000","last_fullsized":"2024-09-14T04:45:53.169984+0000","mapping_epoch":1380,"log_start":"1390'3700","ondisk_log_start":"1390'3700","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7549,"log_dups_size":0,"ondisk_log_size":7549,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:58:35.934308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287899648,"num_objects":4393,"num_object_clones":0,"num_object_copies":13179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11249,"num_write_kb":500544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"54.9","version":"1412'11398","reported_seq":11474,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:55.510821+0000","last_change":"2024-09-14T04:43:01.048728+0000","last_active":"2024-09-14T04:45:55.510821+0000","last_peered":"2024-09-14T04:45:55.510821+0000","last_clean":"2024-09-14T04:45:55.510821+0000","last_became_active":"2024-09-14T04:43:01.048381+0000","last_became_peered":"2024-09-14T04:43:01.048381+0000","last_unstale":"2024-09-14T04:45:55.510821+0000","last_undegraded":"2024-09-14T04:45:55.510821+0000","last_fullsized":"2024-09-14T04:45:55.510821+0000","mapping_epoch":1380,"log_start":"1390'3500","ondisk_log_start":"1390'3500","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7898,"log_dups_size":0,"ondisk_log_size":7898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:47:27.860166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290062336,"num_objects":4426,"num_object_clones":0,"num_object_copies":13278,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4426,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11398,"num_write_kb":506368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"54.8","version":"1412'11357","reported_seq":11435,"reported_epoch":1412,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:45:53.092572+0000","last_change":"2024-09-14T04:44:58.458770+0000","last_active":"2024-09-14T04:45:53.092572+0000","last_peered":"2024-09-14T04:45:53.092572+0000","last_clean":"2024-09-14T04:44:56.294675+0000","last_became_active":"2024-09-14T04:44:58.445646+0000","last_became_peered":"2024-09-14T04:44:58.445646+0000","last_unstale":"2024-09-14T04:45:53.092572+0000","last_undegraded":"2024-09-14T04:45:53.092572+0000","last_fullsized":"2024-09-14T04:45:53.092572+0000","mapping_epoch":1403,"log_start":"1389'3000","ondisk_log_start":"1389'3000","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":8357,"log_dups_size":0,"ondisk_log_size":8357,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294322176,"num_objects":4491,"num_object_clones":0,"num_object_copies":13473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4491,"num_objects_unfound":0,"num_objects_dirty":4491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11357,"num_write_kb":507136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[0,3,7],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3,7","objects":4491}],"blocked_by":[],"up_primary":4,"acting_primary":0,"purged_snaps":[]},{"pgid":"54.7","version":"1412'11530","reported_seq":11606,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:57.193657+0000","last_change":"2024-09-14T04:43:01.050691+0000","last_active":"2024-09-14T04:45:57.193657+0000","last_peered":"2024-09-14T04:45:57.193657+0000","last_clean":"2024-09-14T04:45:57.193657+0000","last_became_active":"2024-09-14T04:43:01.050334+0000","last_became_peered":"2024-09-14T04:43:01.050334+0000","last_unstale":"2024-09-14T04:45:57.193657+0000","last_undegraded":"2024-09-14T04:45:57.193657+0000","last_fullsized":"2024-09-14T04:45:57.193657+0000","mapping_epoch":1380,"log_start":"1393'4800","ondisk_log_start":"1393'4800","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":6730,"log_dups_size":0,"ondisk_log_size":6730,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:22:30.806651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280756224,"num_objects":4284,"num_object_clones":0,"num_object_copies":12852,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4284,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11530,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"54.6","version":"1412'11355","reported_seq":11397,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:52.908636+0000","last_change":"2024-09-14T04:43:01.043477+0000","last_active":"2024-09-14T04:45:52.908636+0000","last_peered":"2024-09-14T04:45:52.908636+0000","last_clean":"2024-09-14T04:45:52.908636+0000","last_became_active":"2024-09-14T04:43:01.043136+0000","last_became_peered":"2024-09-14T04:43:01.043136+0000","last_unstale":"2024-09-14T04:45:52.908636+0000","last_undegraded":"2024-09-14T04:45:52.908636+0000","last_fullsized":"2024-09-14T04:45:52.908636+0000","mapping_epoch":1380,"log_start":"1392'4000","ondisk_log_start":"1392'4000","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7355,"log_dups_size":0,"ondisk_log_size":7355,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:00:57.211271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295370752,"num_objects":4507,"num_object_clones":0,"num_object_copies":13521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11355,"num_write_kb":507584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"54.5","version":"1412'11276","reported_seq":11318,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:53.167487+0000","last_change":"2024-09-14T04:43:01.048975+0000","last_active":"2024-09-14T04:45:53.167487+0000","last_peered":"2024-09-14T04:45:53.167487+0000","last_clean":"2024-09-14T04:45:53.167487+0000","last_became_active":"2024-09-14T04:43:01.048677+0000","last_became_peered":"2024-09-14T04:43:01.048677+0000","last_unstale":"2024-09-14T04:45:53.167487+0000","last_undegraded":"2024-09-14T04:45:53.167487+0000","last_fullsized":"2024-09-14T04:45:53.167487+0000","mapping_epoch":1380,"log_start":"1390'3700","ondisk_log_start":"1390'3700","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7576,"log_dups_size":0,"ondisk_log_size":7576,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:58:04.849606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298188800,"num_objects":4550,"num_object_clones":0,"num_object_copies":13650,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4550,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11276,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"54.4","version":"1412'11535","reported_seq":11611,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:57.242215+0000","last_change":"2024-09-14T04:43:01.043603+0000","last_active":"2024-09-14T04:45:57.242215+0000","last_peered":"2024-09-14T04:45:57.242215+0000","last_clean":"2024-09-14T04:45:57.242215+0000","last_became_active":"2024-09-14T04:43:01.042757+0000","last_became_peered":"2024-09-14T04:43:01.042757+0000","last_unstale":"2024-09-14T04:45:57.242215+0000","last_undegraded":"2024-09-14T04:45:57.242215+0000","last_fullsized":"2024-09-14T04:45:57.242215+0000","mapping_epoch":1380,"log_start":"1393'4800","ondisk_log_start":"1393'4800","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":6735,"log_dups_size":0,"ondisk_log_size":6735,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:46:20.719227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282918912,"num_objects":4317,"num_object_clones":0,"num_object_copies":12951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11535,"num_write_kb":507264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"54.3","version":"1412'11571","reported_seq":11647,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:55.538424+0000","last_change":"2024-09-14T04:43:01.048551+0000","last_active":"2024-09-14T04:45:55.538424+0000","last_peered":"2024-09-14T04:45:55.538424+0000","last_clean":"2024-09-14T04:45:55.538424+0000","last_became_active":"2024-09-14T04:43:01.048218+0000","last_became_peered":"2024-09-14T04:43:01.048218+0000","last_unstale":"2024-09-14T04:45:55.538424+0000","last_undegraded":"2024-09-14T04:45:55.538424+0000","last_fullsized":"2024-09-14T04:45:55.538424+0000","mapping_epoch":1380,"log_start":"1390'3600","ondisk_log_start":"1390'3600","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7971,"log_dups_size":0,"ondisk_log_size":7971,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:27:10.233265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294322176,"num_objects":4491,"num_object_clones":0,"num_object_copies":13473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11571,"num_write_kb":513984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"54.2","version":"1412'11287","reported_seq":11363,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:53.088183+0000","last_change":"2024-09-14T04:43:01.049088+0000","last_active":"2024-09-14T04:45:53.088183+0000","last_peered":"2024-09-14T04:45:53.088183+0000","last_clean":"2024-09-14T04:45:53.088183+0000","last_became_active":"2024-09-14T04:43:01.048764+0000","last_became_peered":"2024-09-14T04:43:01.048764+0000","last_unstale":"2024-09-14T04:45:53.088183+0000","last_undegraded":"2024-09-14T04:45:53.088183+0000","last_fullsized":"2024-09-14T04:45:53.088183+0000","mapping_epoch":1380,"log_start":"1389'2900","ondisk_log_start":"1389'2900","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":8387,"log_dups_size":0,"ondisk_log_size":8387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:03:50.532719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293011456,"num_objects":4471,"num_object_clones":0,"num_object_copies":13413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4471,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11287,"num_write_kb":504256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"54.1","version":"1412'11327","reported_seq":11369,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:53.163181+0000","last_change":"2024-09-14T04:43:01.047658+0000","last_active":"2024-09-14T04:45:53.163181+0000","last_peered":"2024-09-14T04:45:53.163181+0000","last_clean":"2024-09-14T04:45:53.163181+0000","last_became_active":"2024-09-14T04:43:01.047401+0000","last_became_peered":"2024-09-14T04:43:01.047401+0000","last_unstale":"2024-09-14T04:45:53.163181+0000","last_undegraded":"2024-09-14T04:45:53.163181+0000","last_fullsized":"2024-09-14T04:45:53.163181+0000","mapping_epoch":1380,"log_start":"1389'2700","ondisk_log_start":"1389'2700","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":8627,"log_dups_size":0,"ondisk_log_size":8627,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:02:05.987375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298516480,"num_objects":4555,"num_object_clones":0,"num_object_copies":13665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4555,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11327,"num_write_kb":508224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"54.0","version":"1412'11244","reported_seq":11363,"reported_epoch":1412,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:45:53.171502+0000","last_change":"2024-09-14T04:44:58.640486+0000","last_active":"2024-09-14T04:45:53.171502+0000","last_peered":"2024-09-14T04:45:53.171502+0000","last_clean":"2024-09-14T04:44:56.291188+0000","last_became_active":"2024-09-14T04:44:58.441092+0000","last_became_peered":"2024-09-14T04:44:58.441092+0000","last_unstale":"2024-09-14T04:45:53.171502+0000","last_undegraded":"2024-09-14T04:45:53.171502+0000","last_fullsized":"2024-09-14T04:45:53.171502+0000","mapping_epoch":1403,"log_start":"1390'3700","ondisk_log_start":"1390'3700","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7544,"log_dups_size":0,"ondisk_log_size":7544,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:13:27.337351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287571968,"num_objects":4388,"num_object_clones":0,"num_object_copies":13164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4372,"num_objects_unfound":0,"num_objects_dirty":4388,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11244,"num_write_kb":500224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":2097152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,3,5],"avail_no_missing":["6","3","5"],"object_location_counts":[{"shards":"3,5,6","objects":4388}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":2112,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.885553+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T04:45:37.885553+0000","last_peered":"2024-09-14T04:45:37.885553+0000","last_clean":"2024-09-14T04:45:37.885553+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T04:45:37.885553+0000","last_undegraded":"2024-09-14T04:45:37.885553+0000","last_fullsized":"2024-09-14T04:45:37.885553+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":20,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.814442+0000","last_change":"2024-09-14T04:44:57.325109+0000","last_active":"2024-09-14T04:45:34.814442+0000","last_peered":"2024-09-14T04:45:34.814442+0000","last_clean":"2024-09-14T04:45:34.814442+0000","last_became_active":"2024-09-14T04:44:57.324791+0000","last_became_peered":"2024-09-14T04:44:57.324791+0000","last_unstale":"2024-09-14T04:45:34.814442+0000","last_undegraded":"2024-09-14T04:45:34.814442+0000","last_fullsized":"2024-09-14T04:45:34.814442+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:03.593275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":2101,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.888543+0000","last_change":"2024-09-14T04:07:58.504829+0000","last_active":"2024-09-14T04:45:37.888543+0000","last_peered":"2024-09-14T04:45:37.888543+0000","last_clean":"2024-09-14T04:45:37.888543+0000","last_became_active":"2024-09-14T04:07:58.502623+0000","last_became_peered":"2024-09-14T04:07:58.502623+0000","last_unstale":"2024-09-14T04:45:37.888543+0000","last_undegraded":"2024-09-14T04:45:37.888543+0000","last_fullsized":"2024-09-14T04:45:37.888543+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:59.091993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":734,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.890618+0000","last_change":"2024-09-14T04:43:13.355281+0000","last_active":"2024-09-14T04:45:37.890618+0000","last_peered":"2024-09-14T04:45:37.890618+0000","last_clean":"2024-09-14T04:45:37.890618+0000","last_became_active":"2024-09-14T04:43:13.354967+0000","last_became_peered":"2024-09-14T04:43:13.354967+0000","last_unstale":"2024-09-14T04:45:37.890618+0000","last_undegraded":"2024-09-14T04:45:37.890618+0000","last_fullsized":"2024-09-14T04:45:37.890618+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:23:19.332075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1399,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.885885+0000","last_change":"2024-09-14T04:44:57.342918+0000","last_active":"2024-09-14T04:45:37.885885+0000","last_peered":"2024-09-14T04:45:37.885885+0000","last_clean":"2024-09-14T04:45:37.885885+0000","last_became_active":"2024-09-14T04:44:57.329550+0000","last_became_peered":"2024-09-14T04:44:57.329550+0000","last_unstale":"2024-09-14T04:45:37.885885+0000","last_undegraded":"2024-09-14T04:45:37.885885+0000","last_fullsized":"2024-09-14T04:45:37.885885+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:52:52.435893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":20,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.814583+0000","last_change":"2024-09-14T04:44:57.325537+0000","last_active":"2024-09-14T04:45:34.814583+0000","last_peered":"2024-09-14T04:45:34.814583+0000","last_clean":"2024-09-14T04:45:34.814583+0000","last_became_active":"2024-09-14T04:44:57.325057+0000","last_became_peered":"2024-09-14T04:44:57.325057+0000","last_unstale":"2024-09-14T04:45:34.814583+0000","last_undegraded":"2024-09-14T04:45:34.814583+0000","last_fullsized":"2024-09-14T04:45:34.814583+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:43:23.166708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1414,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.886272+0000","last_change":"2024-09-14T04:44:57.337033+0000","last_active":"2024-09-14T04:45:37.886272+0000","last_peered":"2024-09-14T04:45:37.886272+0000","last_clean":"2024-09-14T04:45:37.886272+0000","last_became_active":"2024-09-14T04:44:57.328437+0000","last_became_peered":"2024-09-14T04:44:57.328437+0000","last_unstale":"2024-09-14T04:45:37.886272+0000","last_undegraded":"2024-09-14T04:45:37.886272+0000","last_fullsized":"2024-09-14T04:45:37.886272+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.109086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":20,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.814871+0000","last_change":"2024-09-14T04:44:57.332243+0000","last_active":"2024-09-14T04:45:34.814871+0000","last_peered":"2024-09-14T04:45:34.814871+0000","last_clean":"2024-09-14T04:45:34.814871+0000","last_became_active":"2024-09-14T04:44:57.330413+0000","last_became_peered":"2024-09-14T04:44:57.330413+0000","last_unstale":"2024-09-14T04:45:34.814871+0000","last_undegraded":"2024-09-14T04:45:34.814871+0000","last_fullsized":"2024-09-14T04:45:34.814871+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:25:01.338549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":2118,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.888136+0000","last_change":"2024-09-14T04:44:57.333097+0000","last_active":"2024-09-14T04:45:37.888136+0000","last_peered":"2024-09-14T04:45:37.888136+0000","last_clean":"2024-09-14T04:45:37.888136+0000","last_became_active":"2024-09-14T04:44:57.332294+0000","last_became_peered":"2024-09-14T04:44:57.332294+0000","last_unstale":"2024-09-14T04:45:37.888136+0000","last_undegraded":"2024-09-14T04:45:37.888136+0000","last_fullsized":"2024-09-14T04:45:37.888136+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:57:44.426280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2108,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.891060+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T04:45:37.891060+0000","last_peered":"2024-09-14T04:45:37.891060+0000","last_clean":"2024-09-14T04:45:37.891060+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T04:45:37.891060+0000","last_undegraded":"2024-09-14T04:45:37.891060+0000","last_fullsized":"2024-09-14T04:45:37.891060+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":20,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.814570+0000","last_change":"2024-09-14T04:44:57.325392+0000","last_active":"2024-09-14T04:45:34.814570+0000","last_peered":"2024-09-14T04:45:34.814570+0000","last_clean":"2024-09-14T04:45:34.814570+0000","last_became_active":"2024-09-14T04:44:57.324624+0000","last_became_peered":"2024-09-14T04:44:57.324624+0000","last_unstale":"2024-09-14T04:45:34.814570+0000","last_undegraded":"2024-09-14T04:45:34.814570+0000","last_fullsized":"2024-09-14T04:45:34.814570+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:33:18.810199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2077,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.892724+0000","last_change":"2024-09-14T04:44:57.329113+0000","last_active":"2024-09-14T04:45:37.892724+0000","last_peered":"2024-09-14T04:45:37.892724+0000","last_clean":"2024-09-14T04:45:37.892724+0000","last_became_active":"2024-09-14T04:44:57.328727+0000","last_became_peered":"2024-09-14T04:44:57.328727+0000","last_unstale":"2024-09-14T04:45:37.892724+0000","last_undegraded":"2024-09-14T04:45:37.892724+0000","last_fullsized":"2024-09-14T04:45:37.892724+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:34.939874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":2094,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.892269+0000","last_change":"2024-09-14T04:44:57.329277+0000","last_active":"2024-09-14T04:45:37.892269+0000","last_peered":"2024-09-14T04:45:37.892269+0000","last_clean":"2024-09-14T04:45:37.892269+0000","last_became_active":"2024-09-14T04:44:57.328804+0000","last_became_peered":"2024-09-14T04:44:57.328804+0000","last_unstale":"2024-09-14T04:45:37.892269+0000","last_undegraded":"2024-09-14T04:45:37.892269+0000","last_fullsized":"2024-09-14T04:45:37.892269+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:42:46.055240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2106,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.884047+0000","last_change":"2024-09-14T04:07:58.561490+0000","last_active":"2024-09-14T04:45:37.884047+0000","last_peered":"2024-09-14T04:45:37.884047+0000","last_clean":"2024-09-14T04:45:37.884047+0000","last_became_active":"2024-09-14T04:07:58.559841+0000","last_became_peered":"2024-09-14T04:07:58.559841+0000","last_unstale":"2024-09-14T04:45:37.884047+0000","last_undegraded":"2024-09-14T04:45:37.884047+0000","last_fullsized":"2024-09-14T04:45:37.884047+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:27:03.795368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2090,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.884597+0000","last_change":"2024-09-14T04:43:13.357092+0000","last_active":"2024-09-14T04:45:37.884597+0000","last_peered":"2024-09-14T04:45:37.884597+0000","last_clean":"2024-09-14T04:45:37.884597+0000","last_became_active":"2024-09-14T04:43:13.356694+0000","last_became_peered":"2024-09-14T04:43:13.356694+0000","last_unstale":"2024-09-14T04:45:37.884597+0000","last_undegraded":"2024-09-14T04:45:37.884597+0000","last_fullsized":"2024-09-14T04:45:37.884597+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:11.167574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1391,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.816376+0000","last_change":"2024-09-14T04:07:21.720797+0000","last_active":"2024-09-14T04:45:34.816376+0000","last_peered":"2024-09-14T04:45:34.816376+0000","last_clean":"2024-09-14T04:45:34.816376+0000","last_became_active":"2024-09-14T04:07:21.719331+0000","last_became_peered":"2024-09-14T04:07:21.719331+0000","last_unstale":"2024-09-14T04:45:34.816376+0000","last_undegraded":"2024-09-14T04:45:34.816376+0000","last_fullsized":"2024-09-14T04:45:34.816376+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:35.404023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":39,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.817696+0000","last_change":"2024-09-14T04:43:11.641514+0000","last_active":"2024-09-14T04:45:34.817696+0000","last_peered":"2024-09-14T04:45:34.817696+0000","last_clean":"2024-09-14T04:45:34.817696+0000","last_became_active":"2024-09-14T04:43:11.641151+0000","last_became_peered":"2024-09-14T04:43:11.641151+0000","last_unstale":"2024-09-14T04:45:34.817696+0000","last_undegraded":"2024-09-14T04:45:34.817696+0000","last_fullsized":"2024-09-14T04:45:34.817696+0000","mapping_epoch":1384,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:06:45.045430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2101,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.885685+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T04:45:37.885685+0000","last_peered":"2024-09-14T04:45:37.885685+0000","last_clean":"2024-09-14T04:45:37.885685+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T04:45:37.885685+0000","last_undegraded":"2024-09-14T04:45:37.885685+0000","last_fullsized":"2024-09-14T04:45:37.885685+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":2083,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.890436+0000","last_change":"2024-09-14T04:07:58.507682+0000","last_active":"2024-09-14T04:45:37.890436+0000","last_peered":"2024-09-14T04:45:37.890436+0000","last_clean":"2024-09-14T04:45:37.890436+0000","last_became_active":"2024-09-14T04:07:58.507365+0000","last_became_peered":"2024-09-14T04:07:58.507365+0000","last_unstale":"2024-09-14T04:45:37.890436+0000","last_undegraded":"2024-09-14T04:45:37.890436+0000","last_fullsized":"2024-09-14T04:45:37.890436+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:14:47.294950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1397,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.818916+0000","last_change":"2024-09-14T04:42:49.938505+0000","last_active":"2024-09-14T04:45:34.818916+0000","last_peered":"2024-09-14T04:45:34.818916+0000","last_clean":"2024-09-14T04:45:34.818916+0000","last_became_active":"2024-09-14T04:42:49.938180+0000","last_became_peered":"2024-09-14T04:42:49.938180+0000","last_unstale":"2024-09-14T04:45:34.818916+0000","last_undegraded":"2024-09-14T04:45:34.818916+0000","last_fullsized":"2024-09-14T04:45:34.818916+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:05.929479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1416,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.821457+0000","last_change":"2024-09-14T04:44:57.346582+0000","last_active":"2024-09-14T04:45:34.821457+0000","last_peered":"2024-09-14T04:45:34.821457+0000","last_clean":"2024-09-14T04:45:34.821457+0000","last_became_active":"2024-09-14T04:44:57.330446+0000","last_became_peered":"2024-09-14T04:44:57.330446+0000","last_unstale":"2024-09-14T04:45:34.821457+0000","last_undegraded":"2024-09-14T04:45:34.821457+0000","last_fullsized":"2024-09-14T04:45:34.821457+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:29.238585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":42,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.891811+0000","last_change":"2024-09-14T04:44:14.480667+0000","last_active":"2024-09-14T04:45:37.891811+0000","last_peered":"2024-09-14T04:45:37.891811+0000","last_clean":"2024-09-14T04:45:37.891811+0000","last_became_active":"2024-09-14T04:44:14.480365+0000","last_became_peered":"2024-09-14T04:44:14.480365+0000","last_unstale":"2024-09-14T04:45:37.891811+0000","last_undegraded":"2024-09-14T04:45:37.891811+0000","last_fullsized":"2024-09-14T04:45:37.891811+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:00.611771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":30,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.815746+0000","last_change":"2024-09-14T04:44:57.341426+0000","last_active":"2024-09-14T04:45:34.815746+0000","last_peered":"2024-09-14T04:45:34.815746+0000","last_clean":"2024-09-14T04:45:34.815746+0000","last_became_active":"2024-09-14T04:44:57.328763+0000","last_became_peered":"2024-09-14T04:44:57.328763+0000","last_unstale":"2024-09-14T04:45:34.815746+0000","last_undegraded":"2024-09-14T04:45:34.815746+0000","last_fullsized":"2024-09-14T04:45:34.815746+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:34:40.066761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":20,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.815058+0000","last_change":"2024-09-14T04:44:57.331337+0000","last_active":"2024-09-14T04:45:34.815058+0000","last_peered":"2024-09-14T04:45:34.815058+0000","last_clean":"2024-09-14T04:45:34.815058+0000","last_became_active":"2024-09-14T04:44:57.329336+0000","last_became_peered":"2024-09-14T04:44:57.329336+0000","last_unstale":"2024-09-14T04:45:34.815058+0000","last_undegraded":"2024-09-14T04:45:34.815058+0000","last_fullsized":"2024-09-14T04:45:34.815058+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:52:35.980869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"349'149","reported_seq":771,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.891032+0000","last_change":"2024-09-14T04:41:37.755174+0000","last_active":"2024-09-14T04:45:37.891032+0000","last_peered":"2024-09-14T04:45:37.891032+0000","last_clean":"2024-09-14T04:45:37.891032+0000","last_became_active":"2024-09-14T04:41:36.930263+0000","last_became_peered":"2024-09-14T04:41:36.930263+0000","last_unstale":"2024-09-14T04:45:37.891032+0000","last_undegraded":"2024-09-14T04:45:37.891032+0000","last_fullsized":"2024-09-14T04:45:37.891032+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.755105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.755105+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:27:03.161713+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":20,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.814520+0000","last_change":"2024-09-14T04:44:57.329870+0000","last_active":"2024-09-14T04:45:34.814520+0000","last_peered":"2024-09-14T04:45:34.814520+0000","last_clean":"2024-09-14T04:45:34.814520+0000","last_became_active":"2024-09-14T04:44:57.329353+0000","last_became_peered":"2024-09-14T04:44:57.329353+0000","last_unstale":"2024-09-14T04:45:34.814520+0000","last_undegraded":"2024-09-14T04:45:34.814520+0000","last_fullsized":"2024-09-14T04:45:34.814520+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:46.432181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1394,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.816438+0000","last_change":"2024-09-14T04:07:21.721242+0000","last_active":"2024-09-14T04:45:34.816438+0000","last_peered":"2024-09-14T04:45:34.816438+0000","last_clean":"2024-09-14T04:45:34.816438+0000","last_became_active":"2024-09-14T04:07:21.720084+0000","last_became_peered":"2024-09-14T04:07:21.720084+0000","last_unstale":"2024-09-14T04:45:34.816438+0000","last_undegraded":"2024-09-14T04:45:34.816438+0000","last_fullsized":"2024-09-14T04:45:34.816438+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:58:41.308643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":2139,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.883961+0000","last_change":"2024-09-14T04:44:57.344614+0000","last_active":"2024-09-14T04:45:37.883961+0000","last_peered":"2024-09-14T04:45:37.883961+0000","last_clean":"2024-09-14T04:45:37.883961+0000","last_became_active":"2024-09-14T04:44:57.332256+0000","last_became_peered":"2024-09-14T04:44:57.332256+0000","last_unstale":"2024-09-14T04:45:37.883961+0000","last_undegraded":"2024-09-14T04:45:37.883961+0000","last_fullsized":"2024-09-14T04:45:37.883961+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:23.438417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"349'149","reported_seq":778,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.815410+0000","last_change":"2024-09-14T04:45:00.331854+0000","last_active":"2024-09-14T04:45:34.815410+0000","last_peered":"2024-09-14T04:45:34.815410+0000","last_clean":"2024-09-14T04:45:34.815410+0000","last_became_active":"2024-09-14T04:45:00.331460+0000","last_became_peered":"2024-09-14T04:45:00.331460+0000","last_unstale":"2024-09-14T04:45:34.815410+0000","last_undegraded":"2024-09-14T04:45:34.815410+0000","last_fullsized":"2024-09-14T04:45:34.815410+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.567535+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.567535+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:45.095729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":2075,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.895047+0000","last_change":"2024-09-14T04:07:58.555446+0000","last_active":"2024-09-14T04:45:37.895047+0000","last_peered":"2024-09-14T04:45:37.895047+0000","last_clean":"2024-09-14T04:45:37.895047+0000","last_became_active":"2024-09-14T04:07:58.548355+0000","last_became_peered":"2024-09-14T04:07:58.548355+0000","last_unstale":"2024-09-14T04:45:37.895047+0000","last_undegraded":"2024-09-14T04:45:37.895047+0000","last_fullsized":"2024-09-14T04:45:37.895047+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:56:28.426094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":94,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.911357+0000","last_change":"2024-09-14T04:44:57.330746+0000","last_active":"2024-09-14T04:45:37.911357+0000","last_peered":"2024-09-14T04:45:37.911357+0000","last_clean":"2024-09-14T04:45:37.911357+0000","last_became_active":"2024-09-14T04:44:57.330327+0000","last_became_peered":"2024-09-14T04:44:57.330327+0000","last_unstale":"2024-09-14T04:45:37.911357+0000","last_undegraded":"2024-09-14T04:45:37.911357+0000","last_fullsized":"2024-09-14T04:45:37.911357+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:53.122771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":60,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.822425+0000","last_change":"2024-09-14T04:43:13.355296+0000","last_active":"2024-09-14T04:45:34.822425+0000","last_peered":"2024-09-14T04:45:34.822425+0000","last_clean":"2024-09-14T04:45:34.822425+0000","last_became_active":"2024-09-14T04:43:13.354950+0000","last_became_peered":"2024-09-14T04:43:13.354950+0000","last_unstale":"2024-09-14T04:45:34.822425+0000","last_undegraded":"2024-09-14T04:45:34.822425+0000","last_fullsized":"2024-09-14T04:45:34.822425+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:20:08.968818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"349'149","reported_seq":716,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.816068+0000","last_change":"2024-09-14T04:41:39.471393+0000","last_active":"2024-09-14T04:45:34.816068+0000","last_peered":"2024-09-14T04:45:34.816068+0000","last_clean":"2024-09-14T04:45:34.816068+0000","last_became_active":"2024-09-14T04:41:36.931768+0000","last_became_peered":"2024-09-14T04:41:36.931768+0000","last_unstale":"2024-09-14T04:45:34.816068+0000","last_undegraded":"2024-09-14T04:45:34.816068+0000","last_fullsized":"2024-09-14T04:45:34.816068+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.471248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.471248+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:56.210375+0000","scrub_duration":0.021758344999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2095,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.888605+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T04:45:37.888605+0000","last_peered":"2024-09-14T04:45:37.888605+0000","last_clean":"2024-09-14T04:45:37.888605+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T04:45:37.888605+0000","last_undegraded":"2024-09-14T04:45:37.888605+0000","last_fullsized":"2024-09-14T04:45:37.888605+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1426,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.821644+0000","last_change":"2024-09-14T04:44:57.331436+0000","last_active":"2024-09-14T04:45:34.821644+0000","last_peered":"2024-09-14T04:45:34.821644+0000","last_clean":"2024-09-14T04:45:34.821644+0000","last_became_active":"2024-09-14T04:44:57.330957+0000","last_became_peered":"2024-09-14T04:44:57.330957+0000","last_unstale":"2024-09-14T04:45:34.821644+0000","last_undegraded":"2024-09-14T04:45:34.821644+0000","last_fullsized":"2024-09-14T04:45:34.821644+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:09.516688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":87,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.892341+0000","last_change":"2024-09-14T04:42:49.940968+0000","last_active":"2024-09-14T04:45:37.892341+0000","last_peered":"2024-09-14T04:45:37.892341+0000","last_clean":"2024-09-14T04:45:37.892341+0000","last_became_active":"2024-09-14T04:42:49.940667+0000","last_became_peered":"2024-09-14T04:42:49.940667+0000","last_unstale":"2024-09-14T04:45:37.892341+0000","last_undegraded":"2024-09-14T04:45:37.892341+0000","last_fullsized":"2024-09-14T04:45:37.892341+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:52:49.623815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"349'149","reported_seq":716,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.816308+0000","last_change":"2024-09-14T04:41:38.479980+0000","last_active":"2024-09-14T04:45:34.816308+0000","last_peered":"2024-09-14T04:45:34.816308+0000","last_clean":"2024-09-14T04:45:34.816308+0000","last_became_active":"2024-09-14T04:41:36.933392+0000","last_became_peered":"2024-09-14T04:41:36.933392+0000","last_unstale":"2024-09-14T04:45:34.816308+0000","last_undegraded":"2024-09-14T04:45:34.816308+0000","last_fullsized":"2024-09-14T04:45:34.816308+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:38.479887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:38.479887+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:01:26.102279+0000","scrub_duration":0.0062728319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":49,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.891715+0000","last_change":"2024-09-14T04:44:57.329983+0000","last_active":"2024-09-14T04:45:37.891715+0000","last_peered":"2024-09-14T04:45:37.891715+0000","last_clean":"2024-09-14T04:45:37.891715+0000","last_became_active":"2024-09-14T04:44:57.329597+0000","last_became_peered":"2024-09-14T04:44:57.329597+0000","last_unstale":"2024-09-14T04:45:37.891715+0000","last_undegraded":"2024-09-14T04:45:37.891715+0000","last_fullsized":"2024-09-14T04:45:37.891715+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:45.718039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":2129,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.892574+0000","last_change":"2024-09-14T04:06:46.930482+0000","last_active":"2024-09-14T04:45:37.892574+0000","last_peered":"2024-09-14T04:45:37.892574+0000","last_clean":"2024-09-14T04:45:37.892574+0000","last_became_active":"2024-09-14T04:06:46.927372+0000","last_became_peered":"2024-09-14T04:06:46.927372+0000","last_unstale":"2024-09-14T04:45:37.892574+0000","last_undegraded":"2024-09-14T04:45:37.892574+0000","last_fullsized":"2024-09-14T04:45:37.892574+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:07:22.159128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1399,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.817124+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T04:45:34.817124+0000","last_peered":"2024-09-14T04:45:34.817124+0000","last_clean":"2024-09-14T04:45:34.817124+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T04:45:34.817124+0000","last_undegraded":"2024-09-14T04:45:34.817124+0000","last_fullsized":"2024-09-14T04:45:34.817124+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"349'149","reported_seq":709,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.885878+0000","last_change":"2024-09-14T04:41:39.012977+0000","last_active":"2024-09-14T04:45:37.885878+0000","last_peered":"2024-09-14T04:45:37.885878+0000","last_clean":"2024-09-14T04:45:37.885878+0000","last_became_active":"2024-09-14T04:41:36.932552+0000","last_became_peered":"2024-09-14T04:41:36.932552+0000","last_unstale":"2024-09-14T04:45:37.885878+0000","last_undegraded":"2024-09-14T04:45:37.885878+0000","last_fullsized":"2024-09-14T04:45:37.885878+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.012879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.012879+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:16:40.080262+0000","scrub_duration":0.0051721620000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":87,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.892244+0000","last_change":"2024-09-14T04:42:49.945069+0000","last_active":"2024-09-14T04:45:37.892244+0000","last_peered":"2024-09-14T04:45:37.892244+0000","last_clean":"2024-09-14T04:45:37.892244+0000","last_became_active":"2024-09-14T04:42:49.943527+0000","last_became_peered":"2024-09-14T04:42:49.943527+0000","last_unstale":"2024-09-14T04:45:37.892244+0000","last_undegraded":"2024-09-14T04:45:37.892244+0000","last_fullsized":"2024-09-14T04:45:37.892244+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:39:10.524631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":20,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.814945+0000","last_change":"2024-09-14T04:44:57.329652+0000","last_active":"2024-09-14T04:45:34.814945+0000","last_peered":"2024-09-14T04:45:34.814945+0000","last_clean":"2024-09-14T04:45:34.814945+0000","last_became_active":"2024-09-14T04:44:57.328888+0000","last_became_peered":"2024-09-14T04:44:57.328888+0000","last_unstale":"2024-09-14T04:45:34.814945+0000","last_undegraded":"2024-09-14T04:45:34.814945+0000","last_fullsized":"2024-09-14T04:45:34.814945+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:10.170068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1397,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.817037+0000","last_change":"2024-09-14T04:06:46.875340+0000","last_active":"2024-09-14T04:45:34.817037+0000","last_peered":"2024-09-14T04:45:34.817037+0000","last_clean":"2024-09-14T04:45:34.817037+0000","last_became_active":"2024-09-14T04:06:46.874552+0000","last_became_peered":"2024-09-14T04:06:46.874552+0000","last_unstale":"2024-09-14T04:45:34.817037+0000","last_undegraded":"2024-09-14T04:45:34.817037+0000","last_fullsized":"2024-09-14T04:45:34.817037+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:52.257357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"349'149","reported_seq":1779,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.887977+0000","last_change":"2024-09-14T04:45:00.346627+0000","last_active":"2024-09-14T04:45:37.887977+0000","last_peered":"2024-09-14T04:45:37.887977+0000","last_clean":"2024-09-14T04:45:37.887977+0000","last_became_active":"2024-09-14T04:45:00.346286+0000","last_became_peered":"2024-09-14T04:45:00.346286+0000","last_unstale":"2024-09-14T04:45:37.887977+0000","last_undegraded":"2024-09-14T04:45:37.887977+0000","last_fullsized":"2024-09-14T04:45:37.887977+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:50.784648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:50.784648+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:07:58.441372+0000","scrub_duration":0.01141873,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":42,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.892851+0000","last_change":"2024-09-14T04:44:14.489235+0000","last_active":"2024-09-14T04:45:37.892851+0000","last_peered":"2024-09-14T04:45:37.892851+0000","last_clean":"2024-09-14T04:45:37.892851+0000","last_became_active":"2024-09-14T04:44:14.488950+0000","last_became_peered":"2024-09-14T04:44:14.488950+0000","last_unstale":"2024-09-14T04:45:37.892851+0000","last_undegraded":"2024-09-14T04:45:37.892851+0000","last_fullsized":"2024-09-14T04:45:37.892851+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:54.441144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":2113,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.891699+0000","last_change":"2024-09-14T04:07:21.685415+0000","last_active":"2024-09-14T04:45:37.891699+0000","last_peered":"2024-09-14T04:45:37.891699+0000","last_clean":"2024-09-14T04:45:37.891699+0000","last_became_active":"2024-09-14T04:07:21.683504+0000","last_became_peered":"2024-09-14T04:07:21.683504+0000","last_unstale":"2024-09-14T04:45:37.891699+0000","last_undegraded":"2024-09-14T04:45:37.891699+0000","last_fullsized":"2024-09-14T04:45:37.891699+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:21:08.806291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2115,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.890669+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T04:45:37.890669+0000","last_peered":"2024-09-14T04:45:37.890669+0000","last_clean":"2024-09-14T04:45:37.890669+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T04:45:37.890669+0000","last_undegraded":"2024-09-14T04:45:37.890669+0000","last_fullsized":"2024-09-14T04:45:37.890669+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"349'149","reported_seq":791,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.892551+0000","last_change":"2024-09-14T04:45:00.342826+0000","last_active":"2024-09-14T04:45:37.892551+0000","last_peered":"2024-09-14T04:45:37.892551+0000","last_clean":"2024-09-14T04:45:37.892551+0000","last_became_active":"2024-09-14T04:45:00.342013+0000","last_became_peered":"2024-09-14T04:45:00.342013+0000","last_unstale":"2024-09-14T04:45:37.892551+0000","last_undegraded":"2024-09-14T04:45:37.892551+0000","last_fullsized":"2024-09-14T04:45:37.892551+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.326129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.326129+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:54:28.021603+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2116,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.884119+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T04:45:37.884119+0000","last_peered":"2024-09-14T04:45:37.884119+0000","last_clean":"2024-09-14T04:45:37.884119+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T04:45:37.884119+0000","last_undegraded":"2024-09-14T04:45:37.884119+0000","last_fullsized":"2024-09-14T04:45:37.884119+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2123,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.883701+0000","last_change":"2024-09-14T04:44:57.335783+0000","last_active":"2024-09-14T04:45:37.883701+0000","last_peered":"2024-09-14T04:45:37.883701+0000","last_clean":"2024-09-14T04:45:37.883701+0000","last_became_active":"2024-09-14T04:44:57.332780+0000","last_became_peered":"2024-09-14T04:44:57.332780+0000","last_unstale":"2024-09-14T04:45:37.883701+0000","last_undegraded":"2024-09-14T04:45:37.883701+0000","last_fullsized":"2024-09-14T04:45:37.883701+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:57.457593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1427,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.820503+0000","last_change":"2024-09-14T04:42:49.939790+0000","last_active":"2024-09-14T04:45:34.820503+0000","last_peered":"2024-09-14T04:45:34.820503+0000","last_clean":"2024-09-14T04:45:34.820503+0000","last_became_active":"2024-09-14T04:42:49.939591+0000","last_became_peered":"2024-09-14T04:42:49.939591+0000","last_unstale":"2024-09-14T04:45:34.820503+0000","last_undegraded":"2024-09-14T04:45:34.820503+0000","last_fullsized":"2024-09-14T04:45:34.820503+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:23:16.790593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1391,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.817550+0000","last_change":"2024-09-14T04:44:57.326008+0000","last_active":"2024-09-14T04:45:34.817550+0000","last_peered":"2024-09-14T04:45:34.817550+0000","last_clean":"2024-09-14T04:45:34.817550+0000","last_became_active":"2024-09-14T04:44:57.325617+0000","last_became_peered":"2024-09-14T04:44:57.325617+0000","last_unstale":"2024-09-14T04:45:34.817550+0000","last_undegraded":"2024-09-14T04:45:34.817550+0000","last_fullsized":"2024-09-14T04:45:34.817550+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:15.637508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":680,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.817174+0000","last_change":"2024-09-14T04:43:13.354846+0000","last_active":"2024-09-14T04:45:34.817174+0000","last_peered":"2024-09-14T04:45:34.817174+0000","last_clean":"2024-09-14T04:45:34.817174+0000","last_became_active":"2024-09-14T04:43:13.354515+0000","last_became_peered":"2024-09-14T04:43:13.354515+0000","last_unstale":"2024-09-14T04:45:34.817174+0000","last_undegraded":"2024-09-14T04:45:34.817174+0000","last_fullsized":"2024-09-14T04:45:34.817174+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:17.889341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1396,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.817663+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T04:45:34.817663+0000","last_peered":"2024-09-14T04:45:34.817663+0000","last_clean":"2024-09-14T04:45:34.817663+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T04:45:34.817663+0000","last_undegraded":"2024-09-14T04:45:34.817663+0000","last_fullsized":"2024-09-14T04:45:34.817663+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":1734,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.820373+0000","last_change":"2024-09-14T04:41:30.640651+0000","last_active":"2024-09-14T04:45:34.820373+0000","last_peered":"2024-09-14T04:45:34.820373+0000","last_clean":"2024-09-14T04:45:34.820373+0000","last_became_active":"2024-09-14T04:41:29.532466+0000","last_became_peered":"2024-09-14T04:41:29.532466+0000","last_unstale":"2024-09-14T04:45:34.820373+0000","last_undegraded":"2024-09-14T04:45:34.820373+0000","last_fullsized":"2024-09-14T04:45:34.820373+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":0,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:30.640520+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:30.640520+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:27.776724+0000","scrub_duration":0.0028653139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"349'149","reported_seq":708,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.816955+0000","last_change":"2024-09-14T04:41:39.978080+0000","last_active":"2024-09-14T04:45:34.816955+0000","last_peered":"2024-09-14T04:45:34.816955+0000","last_clean":"2024-09-14T04:45:34.816955+0000","last_became_active":"2024-09-14T04:41:36.929279+0000","last_became_peered":"2024-09-14T04:41:36.929279+0000","last_unstale":"2024-09-14T04:45:34.816955+0000","last_undegraded":"2024-09-14T04:45:34.816955+0000","last_fullsized":"2024-09-14T04:45:34.816955+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.977999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.977999+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:44:06.383827+0000","scrub_duration":0.00192122,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":2095,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.892942+0000","last_change":"2024-09-14T04:44:57.330745+0000","last_active":"2024-09-14T04:45:37.892942+0000","last_peered":"2024-09-14T04:45:37.892942+0000","last_clean":"2024-09-14T04:45:37.892942+0000","last_became_active":"2024-09-14T04:44:57.330261+0000","last_became_peered":"2024-09-14T04:44:57.330261+0000","last_unstale":"2024-09-14T04:45:37.892942+0000","last_undegraded":"2024-09-14T04:45:37.892942+0000","last_fullsized":"2024-09-14T04:45:37.892942+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:24:08.870976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2099,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.894880+0000","last_change":"2024-09-14T04:42:49.941397+0000","last_active":"2024-09-14T04:45:37.894880+0000","last_peered":"2024-09-14T04:45:37.894880+0000","last_clean":"2024-09-14T04:45:37.894880+0000","last_became_active":"2024-09-14T04:42:49.941066+0000","last_became_peered":"2024-09-14T04:42:49.941066+0000","last_unstale":"2024-09-14T04:45:37.894880+0000","last_undegraded":"2024-09-14T04:45:37.894880+0000","last_fullsized":"2024-09-14T04:45:37.894880+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:30.777822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":20,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.815716+0000","last_change":"2024-09-14T04:44:57.326724+0000","last_active":"2024-09-14T04:45:34.815716+0000","last_peered":"2024-09-14T04:45:34.815716+0000","last_clean":"2024-09-14T04:45:34.815716+0000","last_became_active":"2024-09-14T04:44:57.326191+0000","last_became_peered":"2024-09-14T04:44:57.326191+0000","last_unstale":"2024-09-14T04:45:34.815716+0000","last_undegraded":"2024-09-14T04:45:34.815716+0000","last_fullsized":"2024-09-14T04:45:34.815716+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:46.060269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":20,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.814514+0000","last_change":"2024-09-14T04:44:57.328528+0000","last_active":"2024-09-14T04:45:34.814514+0000","last_peered":"2024-09-14T04:45:34.814514+0000","last_clean":"2024-09-14T04:45:34.814514+0000","last_became_active":"2024-09-14T04:44:57.327739+0000","last_became_peered":"2024-09-14T04:44:57.327739+0000","last_unstale":"2024-09-14T04:45:34.814514+0000","last_undegraded":"2024-09-14T04:45:34.814514+0000","last_fullsized":"2024-09-14T04:45:34.814514+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:04:58.294690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"349'149","reported_seq":740,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.817840+0000","last_change":"2024-09-14T04:42:53.013201+0000","last_active":"2024-09-14T04:45:34.817840+0000","last_peered":"2024-09-14T04:45:34.817840+0000","last_clean":"2024-09-14T04:45:34.817840+0000","last_became_active":"2024-09-14T04:42:53.012818+0000","last_became_peered":"2024-09-14T04:42:53.012818+0000","last_unstale":"2024-09-14T04:45:34.817840+0000","last_undegraded":"2024-09-14T04:45:34.817840+0000","last_fullsized":"2024-09-14T04:45:34.817840+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:56.738636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:56.738636+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:39:19.056067+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":20,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.815287+0000","last_change":"2024-09-14T04:44:57.323436+0000","last_active":"2024-09-14T04:45:34.815287+0000","last_peered":"2024-09-14T04:45:34.815287+0000","last_clean":"2024-09-14T04:45:34.815287+0000","last_became_active":"2024-09-14T04:44:57.323022+0000","last_became_peered":"2024-09-14T04:44:57.323022+0000","last_unstale":"2024-09-14T04:45:34.815287+0000","last_undegraded":"2024-09-14T04:45:34.815287+0000","last_fullsized":"2024-09-14T04:45:34.815287+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:22.775261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1426,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.817118+0000","last_change":"2024-09-14T04:44:57.322357+0000","last_active":"2024-09-14T04:45:34.817118+0000","last_peered":"2024-09-14T04:45:34.817118+0000","last_clean":"2024-09-14T04:45:34.817118+0000","last_became_active":"2024-09-14T04:44:57.321990+0000","last_became_peered":"2024-09-14T04:44:57.321990+0000","last_unstale":"2024-09-14T04:45:34.817118+0000","last_undegraded":"2024-09-14T04:45:34.817118+0000","last_fullsized":"2024-09-14T04:45:34.817118+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:09.658008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"349'149","reported_seq":1751,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.887755+0000","last_change":"2024-09-14T04:41:53.858076+0000","last_active":"2024-09-14T04:45:37.887755+0000","last_peered":"2024-09-14T04:45:37.887755+0000","last_clean":"2024-09-14T04:45:37.887755+0000","last_became_active":"2024-09-14T04:41:36.931585+0000","last_became_peered":"2024-09-14T04:41:36.931585+0000","last_unstale":"2024-09-14T04:45:37.887755+0000","last_undegraded":"2024-09-14T04:45:37.887755+0000","last_fullsized":"2024-09-14T04:45:37.887755+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:53.857924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:53.857924+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:10:20.658986+0000","scrub_duration":0.0022355919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":20,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.815518+0000","last_change":"2024-09-14T04:44:57.329198+0000","last_active":"2024-09-14T04:45:34.815518+0000","last_peered":"2024-09-14T04:45:34.815518+0000","last_clean":"2024-09-14T04:45:34.815518+0000","last_became_active":"2024-09-14T04:44:57.327754+0000","last_became_peered":"2024-09-14T04:44:57.327754+0000","last_unstale":"2024-09-14T04:45:34.815518+0000","last_undegraded":"2024-09-14T04:45:34.815518+0000","last_fullsized":"2024-09-14T04:45:34.815518+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:10:27.944476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":100,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.888688+0000","last_change":"2024-09-14T04:44:57.348687+0000","last_active":"2024-09-14T04:45:37.888688+0000","last_peered":"2024-09-14T04:45:37.888688+0000","last_clean":"2024-09-14T04:45:37.888688+0000","last_became_active":"2024-09-14T04:44:57.332845+0000","last_became_peered":"2024-09-14T04:44:57.332845+0000","last_unstale":"2024-09-14T04:45:37.888688+0000","last_undegraded":"2024-09-14T04:45:37.888688+0000","last_fullsized":"2024-09-14T04:45:37.888688+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:12:18.762737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2122,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.888053+0000","last_change":"2024-09-14T04:44:57.333468+0000","last_active":"2024-09-14T04:45:37.888053+0000","last_peered":"2024-09-14T04:45:37.888053+0000","last_clean":"2024-09-14T04:45:37.888053+0000","last_became_active":"2024-09-14T04:44:57.332626+0000","last_became_peered":"2024-09-14T04:44:57.332626+0000","last_unstale":"2024-09-14T04:45:37.888053+0000","last_undegraded":"2024-09-14T04:45:37.888053+0000","last_fullsized":"2024-09-14T04:45:37.888053+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:57:26.366438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1382,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.817344+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T04:45:34.817344+0000","last_peered":"2024-09-14T04:45:34.817344+0000","last_clean":"2024-09-14T04:45:34.817344+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T04:45:34.817344+0000","last_undegraded":"2024-09-14T04:45:34.817344+0000","last_fullsized":"2024-09-14T04:45:34.817344+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1381,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.816774+0000","last_change":"2024-09-14T04:06:46.869868+0000","last_active":"2024-09-14T04:45:34.816774+0000","last_peered":"2024-09-14T04:45:34.816774+0000","last_clean":"2024-09-14T04:45:34.816774+0000","last_became_active":"2024-09-14T04:06:46.869126+0000","last_became_peered":"2024-09-14T04:06:46.869126+0000","last_unstale":"2024-09-14T04:45:34.816774+0000","last_undegraded":"2024-09-14T04:45:34.816774+0000","last_fullsized":"2024-09-14T04:45:34.816774+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:41.150138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1365,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.821578+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T04:45:34.821578+0000","last_peered":"2024-09-14T04:45:34.821578+0000","last_clean":"2024-09-14T04:45:34.821578+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T04:45:34.821578+0000","last_undegraded":"2024-09-14T04:45:34.821578+0000","last_fullsized":"2024-09-14T04:45:34.821578+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":20,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.814608+0000","last_change":"2024-09-14T04:44:57.332112+0000","last_active":"2024-09-14T04:45:34.814608+0000","last_peered":"2024-09-14T04:45:34.814608+0000","last_clean":"2024-09-14T04:45:34.814608+0000","last_became_active":"2024-09-14T04:44:57.331575+0000","last_became_peered":"2024-09-14T04:44:57.331575+0000","last_unstale":"2024-09-14T04:45:34.814608+0000","last_undegraded":"2024-09-14T04:45:34.814608+0000","last_fullsized":"2024-09-14T04:45:34.814608+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:50:00.391283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1413,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.821766+0000","last_change":"2024-09-14T04:44:57.351460+0000","last_active":"2024-09-14T04:45:34.821766+0000","last_peered":"2024-09-14T04:45:34.821766+0000","last_clean":"2024-09-14T04:45:34.821766+0000","last_became_active":"2024-09-14T04:44:57.334662+0000","last_became_peered":"2024-09-14T04:44:57.334662+0000","last_unstale":"2024-09-14T04:45:34.821766+0000","last_undegraded":"2024-09-14T04:45:34.821766+0000","last_fullsized":"2024-09-14T04:45:34.821766+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:02.399062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1367,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.817954+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T04:45:34.817954+0000","last_peered":"2024-09-14T04:45:34.817954+0000","last_clean":"2024-09-14T04:45:34.817954+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T04:45:34.817954+0000","last_undegraded":"2024-09-14T04:45:34.817954+0000","last_fullsized":"2024-09-14T04:45:34.817954+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1425,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.822363+0000","last_change":"2024-09-14T04:44:57.349690+0000","last_active":"2024-09-14T04:45:34.822363+0000","last_peered":"2024-09-14T04:45:34.822363+0000","last_clean":"2024-09-14T04:45:34.822363+0000","last_became_active":"2024-09-14T04:44:57.333579+0000","last_became_peered":"2024-09-14T04:44:57.333579+0000","last_unstale":"2024-09-14T04:45:34.822363+0000","last_undegraded":"2024-09-14T04:45:34.822363+0000","last_fullsized":"2024-09-14T04:45:34.822363+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:17:55.265141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1426,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.816385+0000","last_change":"2024-09-14T04:42:49.960838+0000","last_active":"2024-09-14T04:45:34.816385+0000","last_peered":"2024-09-14T04:45:34.816385+0000","last_clean":"2024-09-14T04:45:34.816385+0000","last_became_active":"2024-09-14T04:42:49.954736+0000","last_became_peered":"2024-09-14T04:42:49.954736+0000","last_unstale":"2024-09-14T04:45:34.816385+0000","last_undegraded":"2024-09-14T04:45:34.816385+0000","last_fullsized":"2024-09-14T04:45:34.816385+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:41:13.540983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":2081,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.891169+0000","last_change":"2024-09-14T04:07:58.565442+0000","last_active":"2024-09-14T04:45:37.891169+0000","last_peered":"2024-09-14T04:45:37.891169+0000","last_clean":"2024-09-14T04:45:37.891169+0000","last_became_active":"2024-09-14T04:07:58.552289+0000","last_became_peered":"2024-09-14T04:07:58.552289+0000","last_unstale":"2024-09-14T04:45:37.891169+0000","last_undegraded":"2024-09-14T04:45:37.891169+0000","last_fullsized":"2024-09-14T04:45:37.891169+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:48:02.364960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1387,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.821163+0000","last_change":"2024-09-14T04:44:57.332511+0000","last_active":"2024-09-14T04:45:34.821163+0000","last_peered":"2024-09-14T04:45:34.821163+0000","last_clean":"2024-09-14T04:45:34.821163+0000","last_became_active":"2024-09-14T04:44:57.332166+0000","last_became_peered":"2024-09-14T04:44:57.332166+0000","last_unstale":"2024-09-14T04:45:34.821163+0000","last_undegraded":"2024-09-14T04:45:34.821163+0000","last_fullsized":"2024-09-14T04:45:34.821163+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:53.984639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2102,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.883883+0000","last_change":"2024-09-14T04:07:58.560880+0000","last_active":"2024-09-14T04:45:37.883883+0000","last_peered":"2024-09-14T04:45:37.883883+0000","last_clean":"2024-09-14T04:45:37.883883+0000","last_became_active":"2024-09-14T04:07:58.560132+0000","last_became_peered":"2024-09-14T04:07:58.560132+0000","last_unstale":"2024-09-14T04:45:37.883883+0000","last_undegraded":"2024-09-14T04:45:37.883883+0000","last_fullsized":"2024-09-14T04:45:37.883883+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:46:00.287050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2102,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.885631+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T04:45:37.885631+0000","last_peered":"2024-09-14T04:45:37.885631+0000","last_clean":"2024-09-14T04:45:37.885631+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T04:45:37.885631+0000","last_undegraded":"2024-09-14T04:45:37.885631+0000","last_fullsized":"2024-09-14T04:45:37.885631+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1379,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.817051+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T04:45:34.817051+0000","last_peered":"2024-09-14T04:45:34.817051+0000","last_clean":"2024-09-14T04:45:34.817051+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T04:45:34.817051+0000","last_undegraded":"2024-09-14T04:45:34.817051+0000","last_fullsized":"2024-09-14T04:45:34.817051+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1416,"reported_epoch":1411,"state":"active+clean","last_fresh":"2024-09-14T04:45:34.820903+0000","last_change":"2024-09-14T04:44:57.339993+0000","last_active":"2024-09-14T04:45:34.820903+0000","last_peered":"2024-09-14T04:45:34.820903+0000","last_clean":"2024-09-14T04:45:34.820903+0000","last_became_active":"2024-09-14T04:44:57.330472+0000","last_became_peered":"2024-09-14T04:44:57.330472+0000","last_unstale":"2024-09-14T04:45:34.820903+0000","last_undegraded":"2024-09-14T04:45:34.820903+0000","last_fullsized":"2024-09-14T04:45:34.820903+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:19:27.046988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":42,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.892146+0000","last_change":"2024-09-14T04:44:14.479373+0000","last_active":"2024-09-14T04:45:37.892146+0000","last_peered":"2024-09-14T04:45:37.892146+0000","last_clean":"2024-09-14T04:45:37.892146+0000","last_became_active":"2024-09-14T04:44:14.479084+0000","last_became_peered":"2024-09-14T04:44:14.479084+0000","last_unstale":"2024-09-14T04:45:37.892146+0000","last_undegraded":"2024-09-14T04:45:37.892146+0000","last_fullsized":"2024-09-14T04:45:37.892146+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:49:23.412806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":54,"num_pg":16,"stat_sum":{"num_bytes":4621139992,"num_objects":70514,"num_object_clones":0,"num_object_copies":211542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":19435,"num_objects_unfound":0,"num_objects_dirty":70514,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":182276,"num_write_kb":8089217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":990,"num_bytes_recovered":64815104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13901508608,"data_stored":13901496392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122093,"ondisk_log_size":122093,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":5070,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4250,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201939,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2353388,"kb_used_data":2043428,"kb_used_omap":1,"kb_used_meta":309950,"kb_avail":91391764,"statfs":{"total":95995035648,"available":93585166336,"internally_reserved":0,"allocated":2092470272,"data_stored":2087998646,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":317389259},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463575,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2715240,"kb_used_data":2315040,"kb_used_omap":1,"kb_used_meta":400190,"kb_avail":91029912,"statfs":{"total":95995035648,"available":93214629888,"internally_reserved":0,"allocated":2370600960,"data_stored":2366133327,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":409795015},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725212,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2680864,"kb_used_data":2292952,"kb_used_omap":1,"kb_used_meta":387902,"kb_avail":91064288,"statfs":{"total":95995035648,"available":93249830912,"internally_reserved":0,"allocated":2347982848,"data_stored":2343519753,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":397212102},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":1392,"seq":5978594476059,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":92588,"kb_used_data":11428,"kb_used_omap":117,"kb_used_meta":81034,"kb_avail":93652564,"statfs":{"total":95995035648,"available":95900225536,"internally_reserved":0,"allocated":11702272,"data_stored":7231198,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120638,"internal_metadata":82979010},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772074954,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3443168,"kb_used_data":3050136,"kb_used_omap":71,"kb_used_meta":392952,"kb_avail":90301984,"statfs":{"total":95995035648,"available":92469231616,"internally_reserved":0,"allocated":3123339264,"data_stored":3118876246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":402383798},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707303889,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1078748,"kb_used_data":862036,"kb_used_omap":68,"kb_used_meta":216635,"kb_avail":92666404,"statfs":{"total":95995035648,"available":94890397696,"internally_reserved":0,"allocated":882724864,"data_stored":878265822,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":221834448},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872729048,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1475668,"kb_used_data":1141004,"kb_used_omap":67,"kb_used_meta":334588,"kb_avail":92269484,"statfs":{"total":95995035648,"available":94483951616,"internally_reserved":0,"allocated":1168388096,"data_stored":1163936105,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":342618469},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056098,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2366556,"kb_used_data":2032020,"kb_used_omap":74,"kb_used_meta":334389,"kb_avail":91378596,"statfs":{"total":95995035648,"available":93571682304,"internally_reserved":0,"allocated":2080788480,"data_stored":2076328906,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":342415029},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1901,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2069893120,"data_stored":2069889048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1157627904,"data_stored":1157627904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":868810752,"data_stored":868810752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3071873024,"data_stored":3071868952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1114112,"data_stored":1114112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2326134784,"data_stored":2326134784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2344620032,"data_stored":2344615960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2061434880,"data_stored":2061434880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:46:00.329 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph status --format=json 2024-09-14T04:46:00.397 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:00.430 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:00.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:00.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:00.532 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:00.539 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:00 smithi039 ceph-mon[106470]: from='client.35580 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:46:00.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:00 smithi039 ceph-mon[111235]: from='client.35580 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:46:00.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:00.567 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:00.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:00.668 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:00.707 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:00.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:00.708 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:00 smithi129 ceph-mon[101316]: from='client.35580 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:46:00.809 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:00.838 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:00.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:00.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:00.940 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:00.962 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:00.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:00.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:01.063 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:01.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:01.099 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:01.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:01.200 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:01.242 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:01.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:01.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:01.344 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:01.384 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:01.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:01.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:01.486 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:01.514 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:01.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:01.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:01.616 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:01.657 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:01.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:01.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:01.759 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:01 smithi129 ceph-mon[101316]: pgmap v2473: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 869 op/s; 18945/204672 objects misplaced (9.256%); 933 KiB/s, 14 objects/s recovering 2024-09-14T04:46:01.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:01 smithi039 ceph-mon[106470]: pgmap v2473: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 869 op/s; 18945/204672 objects misplaced (9.256%); 933 KiB/s, 14 objects/s recovering 2024-09-14T04:46:01.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:01 smithi039 ceph-mon[111235]: pgmap v2473: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 869 op/s; 18945/204672 objects misplaced (9.256%); 933 KiB/s, 14 objects/s recovering 2024-09-14T04:46:01.793 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:01.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:01.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:01.895 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:01.920 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:01.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:01.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:02.022 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:02.055 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:02.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:02.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:02.157 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:02.185 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:02.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:02.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:02.287 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:02.318 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:02.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:02.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:02.379 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is unset 2024-09-14T04:46:02.420 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:02.449 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:46:02.449 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:46:02.449 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:46:02.449 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:46:02.449 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:46:02.449 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:46:02.449 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:46:02.449 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:46:02.450 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:46:02.450 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:46:02.450 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:46:02.450 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:46:02.450 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:46:02.450 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:46:02.450 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:46:02.450 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:46:02.458 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:02.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:02.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:02.561 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:02.603 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:02.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:02.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:02.705 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:02.749 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:02.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:02.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:02.759 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:02 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1679915595' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:46:02.759 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:02 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:46:02.759 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:02 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1679915595' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:46:02.760 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:02 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:46:02.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:02 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1679915595' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:46:02.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:02 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:46:02.854 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:02.883 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:02.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:02.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:02.984 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:03.022 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:03.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:03.023 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:03.124 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:03.160 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:03.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:03.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:03.261 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:03.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:03.300 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:03.300 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:03.401 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:03.440 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:03.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:03.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:03.542 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:03.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:03.595 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:03.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:03.696 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:03.736 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:03.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:03.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:03.743 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:03 smithi039 ceph-mon[106470]: pgmap v2474: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 18846/202449 objects misplaced (9.309%); 1.4 MiB/s, 22 objects/s recovering 2024-09-14T04:46:03.743 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:03 smithi039 ceph-mon[106470]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:46:03.743 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:03 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T04:46:03.743 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:03 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:46:03.743 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:03 smithi039 ceph-mon[106470]: osdmap e1413: 8 total, 8 up, 8 in 2024-09-14T04:46:03.743 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:03 smithi039 ceph-mon[111235]: pgmap v2474: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 18846/202449 objects misplaced (9.309%); 1.4 MiB/s, 22 objects/s recovering 2024-09-14T04:46:03.743 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:03 smithi039 ceph-mon[111235]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:46:03.744 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:03 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T04:46:03.744 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:03 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:46:03.744 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:03 smithi039 ceph-mon[111235]: osdmap e1413: 8 total, 8 up, 8 in 2024-09-14T04:46:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:03 smithi129 ceph-mon[101316]: pgmap v2474: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 747 op/s; 18846/202449 objects misplaced (9.309%); 1.4 MiB/s, 22 objects/s recovering 2024-09-14T04:46:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:03 smithi129 ceph-mon[101316]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:46:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:03 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T04:46:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:03 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:46:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:03 smithi129 ceph-mon[101316]: osdmap e1413: 8 total, 8 up, 8 in 2024-09-14T04:46:03.837 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:03.882 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:03.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:03.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:03.984 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:04.020 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:04.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:04.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:04.122 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:04.150 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:04.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:04.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:04.252 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:04.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:04.304 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:04.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:04.405 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:04.441 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:04.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:04.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:04.543 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:04.577 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:04.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:04.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:04.679 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:04.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:04.722 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:04.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:04.823 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:04.864 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:04.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:04.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:04.966 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:05.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:05.014 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:05.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:05.114 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:05.155 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:05.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:05.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:05.257 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:05.292 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:05.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:05.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:05.393 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:05.435 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:05.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:05.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:05.536 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:05.572 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:05.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:05.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:05.585 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:46:05.624 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:05 smithi039 ceph-mon[106470]: pgmap v2476: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 18158/193392 objects misplaced (9.389%); 1.1 MiB/s, 18 objects/s recovering 2024-09-14T04:46:05.624 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:05 smithi039 ceph-mon[111235]: pgmap v2476: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 18158/193392 objects misplaced (9.389%); 1.1 MiB/s, 18 objects/s recovering 2024-09-14T04:46:05.674 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:05.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:05.723 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:05.723 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:05 smithi129 ceph-mon[101316]: pgmap v2476: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 786 op/s; 18158/193392 objects misplaced (9.389%); 1.1 MiB/s, 18 objects/s recovering 2024-09-14T04:46:05.824 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:05.863 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:05.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:05.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:05.965 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:06.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:06.005 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:06.005 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:06.107 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:06.166 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:06.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:06.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:06.242 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:46:06.269 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:06.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:06.344 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:06.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:06.445 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:06.483 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:06.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:06.484 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:06.584 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:06.636 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:06.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:06.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:06.738 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:06.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:06.786 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:06.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:06.887 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:06.926 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:06.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:06.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:06.943 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:46:07.028 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:07.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:07.102 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:07.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:07.203 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:07.250 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:07.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:07.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:07.352 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:07.382 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:07.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:07.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:07.483 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:07.517 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:07.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:07.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:07.618 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:07.652 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:07.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:07.653 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:07.680 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:46:07.721 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:07 smithi039 ceph-mon[106470]: pgmap v2477: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 18158/193392 objects misplaced (9.389%); 633 KiB/s, 9 objects/s recovering 2024-09-14T04:46:07.721 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:46:07.722 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:46:07.722 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:07 smithi039 ceph-mon[111235]: pgmap v2477: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 18158/193392 objects misplaced (9.389%); 633 KiB/s, 9 objects/s recovering 2024-09-14T04:46:07.722 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:46:07.722 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:46:07.753 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:07.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:07 smithi129 ceph-mon[101316]: pgmap v2477: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 707 op/s; 18158/193392 objects misplaced (9.389%); 633 KiB/s, 9 objects/s recovering 2024-09-14T04:46:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:46:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:46:07.780 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:07.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:07.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:07.881 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:07.903 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:07.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:07.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:08.005 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:08.040 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:08.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:08.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:08.142 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:08.165 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:08.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:08.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:08.267 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:08.297 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:08.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:08.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:08.399 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:08.436 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:08.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:08.437 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:08.538 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:08.567 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:08.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:08.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:08.668 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:08.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:08.707 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:08.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:08.807 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:08.836 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:08.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:08.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:08.939 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:08.977 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:08.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:08.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:09.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:46:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:46:08] "GET /metrics HTTP/1.1" 200 36749 "" "Prometheus/2.51.0" 2024-09-14T04:46:09.079 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:09.129 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:09.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:09.130 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:09.231 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:09.262 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:09.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:09.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:09.364 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:09.397 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:09.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:09.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:09.408 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:09 smithi039 ceph-mon[106470]: pgmap v2478: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 17784/187980 objects misplaced (9.461%); 1.3 MiB/s, 20 objects/s recovering 2024-09-14T04:46:09.408 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:09 smithi039 ceph-mon[111235]: pgmap v2478: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 17784/187980 objects misplaced (9.461%); 1.3 MiB/s, 20 objects/s recovering 2024-09-14T04:46:09.499 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:09.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:09.560 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:09.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:09.661 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:09.690 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:09.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:09.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:09 smithi129 ceph-mon[101316]: pgmap v2478: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 17784/187980 objects misplaced (9.461%); 1.3 MiB/s, 20 objects/s recovering 2024-09-14T04:46:09.792 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:09.822 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:09.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:09.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:09.923 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:09.947 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:09.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:09.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:10.049 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:10.090 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:10.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:10.091 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:10.192 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:10.225 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:10.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:10.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:10.327 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:10.353 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:10.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:10.354 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:10.455 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:10.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:10.497 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:10.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:10.598 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:10.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:10.646 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:10.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:10.747 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:10.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:10.800 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:10.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:10.901 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:10.932 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:10.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:10.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:11.034 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:11.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:11.066 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:11.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:11.167 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:11.203 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:11.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:11.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:11.306 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:11.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:11.350 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:11.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:11.451 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:11.485 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:11.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:11.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:11.587 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:11.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:11.623 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:11.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:11.724 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:11.760 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:11.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:11.761 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:11.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:11 smithi129 ceph-mon[101316]: pgmap v2479: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 17265/180987 objects misplaced (9.539%); 1.3 MiB/s, 20 objects/s recovering 2024-09-14T04:46:11.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:11 smithi039 ceph-mon[106470]: pgmap v2479: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 17265/180987 objects misplaced (9.539%); 1.3 MiB/s, 20 objects/s recovering 2024-09-14T04:46:11.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:11 smithi039 ceph-mon[111235]: pgmap v2479: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 17265/180987 objects misplaced (9.539%); 1.3 MiB/s, 20 objects/s recovering 2024-09-14T04:46:11.862 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:11.900 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:11.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:11.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:12.002 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:12.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:12.039 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:12.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:12.140 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:12.177 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:12.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:12.178 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:12.279 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:12.318 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:12.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:12.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:12.420 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:12.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:12.452 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:12.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:12.553 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:12.590 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:12.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:12.591 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:12.692 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:12.725 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:12.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:12.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:12.826 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:12.874 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:12.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:12.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:12.914 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:46:12.914 INFO:teuthology.orchestra.run.smithi039.stdout: "size": 20, 2024-09-14T04:46:12.914 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 600, 2024-09-14T04:46:12.915 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [ 2024-09-14T04:46:12.915 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.915 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.34914.0:2209 52.d 52:b2fb3e73:::rbd_data.8862fb33ee1a.0000000000000054:head [set-alloc-hint object_size 4194304 write_size 4194304,write 0~1976798] snapc 67=[67] ondisk+write+known_if_redirected+supports_pool_eio e1269)", 2024-09-14T04:46:12.915 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:36:19.222766+0000", 2024-09-14T04:46:12.915 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 591.718476775, 2024-09-14T04:46:12.915 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.064658328000000001, 2024-09-14T04:46:12.915 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:12.915 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:12.915 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:12.915 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:46:12.916 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.34914", 2024-09-14T04:46:12.916 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1048804342", 2024-09-14T04:46:12.916 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 2209 2024-09-14T04:46:12.916 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.916 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:12.916 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.916 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:12.916 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:19.222766+0000", 2024-09-14T04:46:12.916 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:12.916 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.916 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.916 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:12.916 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:19.222766+0000", 2024-09-14T04:46:12.916 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:12.917 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.917 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.917 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:12.917 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:19.222768+0000", 2024-09-14T04:46:12.917 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.599e-06 2024-09-14T04:46:12.917 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.917 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.917 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:12.917 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:19.224765+0000", 2024-09-14T04:46:12.917 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001996425 2024-09-14T04:46:12.917 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.917 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.917 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:12.918 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:19.224790+0000", 2024-09-14T04:46:12.918 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.5622e-05 2024-09-14T04:46:12.918 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.918 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.918 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:12.918 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:19.224804+0000", 2024-09-14T04:46:12.918 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3993e-05 2024-09-14T04:46:12.918 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.918 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.918 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:12.918 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:19.224900+0000", 2024-09-14T04:46:12.918 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.5847000000000006e-05 2024-09-14T04:46:12.918 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.919 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.919 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:12.919 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:19.225144+0000", 2024-09-14T04:46:12.919 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00024429200000000001 2024-09-14T04:46:12.919 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.919 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.919 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:46:12.919 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:19.225380+0000", 2024-09-14T04:46:12.919 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000235119 2024-09-14T04:46:12.919 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.919 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.919 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:46:12.919 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:19.230496+0000", 2024-09-14T04:46:12.920 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0051164879999999998 2024-09-14T04:46:12.920 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.920 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.920 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:12.920 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:19.243509+0000", 2024-09-14T04:46:12.920 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.013012998 2024-09-14T04:46:12.920 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.920 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.920 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:12.920 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:19.287295+0000", 2024-09-14T04:46:12.920 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.043786261999999999 2024-09-14T04:46:12.920 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.920 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.921 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:12.921 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:19.287372+0000", 2024-09-14T04:46:12.921 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6697000000000001e-05 2024-09-14T04:46:12.921 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.921 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.921 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:12.921 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:19.287424+0000", 2024-09-14T04:46:12.921 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1986e-05 2024-09-14T04:46:12.921 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:12.921 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:12.921 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:12.921 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.922 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.922 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.34914.0:2852 52.1f 52:fae8de57:::rbd_data.8862fb33ee1a.0000000000000032:head [set-alloc-hint object_size 4194304 write_size 4194304,write 2319715~1874589] snapc 73=[73,72,71,70,6f,6e,6d,6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1281)", 2024-09-14T04:46:12.922 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:36:31.355463+0000", 2024-09-14T04:46:12.922 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 579.585779344, 2024-09-14T04:46:12.922 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.050093245000000002, 2024-09-14T04:46:12.922 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:12.922 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:12.922 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:12.922 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:46:12.922 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.34914", 2024-09-14T04:46:12.922 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1048804342", 2024-09-14T04:46:12.922 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 2852 2024-09-14T04:46:12.922 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.922 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:12.923 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.923 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:12.923 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.355463+0000", 2024-09-14T04:46:12.923 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:12.923 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.923 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.923 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:12.923 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.355463+0000", 2024-09-14T04:46:12.923 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:12.923 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.923 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.923 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:12.923 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.355467+0000", 2024-09-14T04:46:12.924 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.5949999999999999e-06 2024-09-14T04:46:12.924 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.924 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.924 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:12.924 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.357569+0000", 2024-09-14T04:46:12.924 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0021018370000000001 2024-09-14T04:46:12.924 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.924 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.924 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:12.924 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.357597+0000", 2024-09-14T04:46:12.924 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.8283e-05 2024-09-14T04:46:12.924 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.924 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.924 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:12.925 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.357608+0000", 2024-09-14T04:46:12.925 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0693000000000001e-05 2024-09-14T04:46:12.925 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.925 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.925 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:12.925 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.357687+0000", 2024-09-14T04:46:12.925 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9002e-05 2024-09-14T04:46:12.925 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.925 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.925 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op must wait for map", 2024-09-14T04:46:12.925 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.357693+0000", 2024-09-14T04:46:12.925 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8449999999999997e-06 2024-09-14T04:46:12.926 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.926 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.926 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:12.926 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.360429+0000", 2024-09-14T04:46:12.926 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0027357509999999998 2024-09-14T04:46:12.926 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.926 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.926 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:12.926 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.360545+0000", 2024-09-14T04:46:12.926 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011612600000000001 2024-09-14T04:46:12.926 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.926 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.926 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,4", 2024-09-14T04:46:12.926 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.360664+0000", 2024-09-14T04:46:12.927 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000118245 2024-09-14T04:46:12.927 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.927 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.927 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:46:12.927 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.366855+0000", 2024-09-14T04:46:12.927 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0061913899999999997 2024-09-14T04:46:12.927 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.927 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.927 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:12.927 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.367103+0000", 2024-09-14T04:46:12.927 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00024819399999999998 2024-09-14T04:46:12.927 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.927 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.927 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:12.928 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.405476+0000", 2024-09-14T04:46:12.928 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.038372457999999998 2024-09-14T04:46:12.928 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.928 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.928 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:12.928 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.405518+0000", 2024-09-14T04:46:12.928 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2673000000000003e-05 2024-09-14T04:46:12.928 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.928 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.928 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:12.928 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:31.405556+0000", 2024-09-14T04:46:12.928 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.8152999999999999e-05 2024-09-14T04:46:12.928 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:12.929 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:12.929 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:12.929 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.929 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.929 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.34914.0:2910 52.d 52:b2fb3e73:::rbd_data.8862fb33ee1a.0000000000000054:head [write 1277026~2917278] snapc 74=[74,73,72,71,70,6f,6e,6d,6c,6b,6a,69,68,67] ondisk+write+known_if_redirected+supports_pool_eio e1282)", 2024-09-14T04:46:12.929 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:36:32.376602+0000", 2024-09-14T04:46:12.929 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 578.56464016400002, 2024-09-14T04:46:12.929 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.049396374, 2024-09-14T04:46:12.929 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:12.929 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:12.929 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:12.929 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:46:12.929 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.34914", 2024-09-14T04:46:12.930 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1048804342", 2024-09-14T04:46:12.930 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 2910 2024-09-14T04:46:12.930 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.930 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:12.930 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.930 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:12.930 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.376602+0000", 2024-09-14T04:46:12.930 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:12.930 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.930 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.930 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:12.930 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.376602+0000", 2024-09-14T04:46:12.930 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:12.930 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.931 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.931 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:12.931 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.376606+0000", 2024-09-14T04:46:12.931 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.7869999999999999e-06 2024-09-14T04:46:12.931 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.931 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.931 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:12.931 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.379637+0000", 2024-09-14T04:46:12.931 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0030309159999999998 2024-09-14T04:46:12.931 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.932 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.932 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:12.932 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.379672+0000", 2024-09-14T04:46:12.932 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.4740999999999998e-05 2024-09-14T04:46:12.932 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.932 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.932 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:12.932 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.379707+0000", 2024-09-14T04:46:12.932 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.5601e-05 2024-09-14T04:46:12.932 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.932 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.932 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:12.932 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.379775+0000", 2024-09-14T04:46:12.932 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7509999999999996e-05 2024-09-14T04:46:12.933 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.933 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.933 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op must wait for map", 2024-09-14T04:46:12.933 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.379779+0000", 2024-09-14T04:46:12.933 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4190000000000004e-06 2024-09-14T04:46:12.933 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.933 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.933 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:12.933 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.382895+0000", 2024-09-14T04:46:12.933 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0031161439999999999 2024-09-14T04:46:12.933 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.933 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.933 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:12.934 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.383003+0000", 2024-09-14T04:46:12.934 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000107284 2024-09-14T04:46:12.934 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.934 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.934 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:46:12.934 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.383203+0000", 2024-09-14T04:46:12.934 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000200158 2024-09-14T04:46:12.934 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.934 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.934 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:46:12.934 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.389485+0000", 2024-09-14T04:46:12.934 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0062822579999999998 2024-09-14T04:46:12.934 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.935 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.935 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:12.935 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.422592+0000", 2024-09-14T04:46:12.935 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.033107325999999999 2024-09-14T04:46:12.935 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.935 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.935 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:12.935 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.425918+0000", 2024-09-14T04:46:12.935 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.003325864 2024-09-14T04:46:12.935 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.935 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.935 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:12.935 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.425964+0000", 2024-09-14T04:46:12.935 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5235000000000003e-05 2024-09-14T04:46:12.936 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.936 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.936 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:12.936 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:36:32.425999+0000", 2024-09-14T04:46:12.936 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.5131e-05 2024-09-14T04:46:12.936 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:12.936 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:12.936 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:12.936 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.937 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.937 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.34914.0:7346 52.d 52:b6b5c6cb:::rbd_data.8862ba52cd58.0000000000000002:head [set-alloc-hint object_size 4194304 write_size 4194304,writefull 0~4194304] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1311)", 2024-09-14T04:46:12.937 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:37:01.028984+0000", 2024-09-14T04:46:12.937 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 549.91225884200003, 2024-09-14T04:46:12.937 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.047506552000000001, 2024-09-14T04:46:12.937 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:12.937 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:12.937 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:12.937 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:46:12.937 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.34914", 2024-09-14T04:46:12.937 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1048804342", 2024-09-14T04:46:12.937 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 7346 2024-09-14T04:46:12.937 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.938 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:12.938 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.938 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:12.938 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.028984+0000", 2024-09-14T04:46:12.938 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:12.938 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.938 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.938 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:12.938 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.028984+0000", 2024-09-14T04:46:12.938 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:12.938 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.938 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.939 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:12.939 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.028987+0000", 2024-09-14T04:46:12.939 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.9129999999999999e-06 2024-09-14T04:46:12.939 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.939 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.939 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:12.939 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.032923+0000", 2024-09-14T04:46:12.939 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0039369410000000002 2024-09-14T04:46:12.939 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.939 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.939 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:12.940 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.032966+0000", 2024-09-14T04:46:12.940 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2792e-05 2024-09-14T04:46:12.940 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.940 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.940 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:12.940 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.032980+0000", 2024-09-14T04:46:12.940 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4093e-05 2024-09-14T04:46:12.940 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.940 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.940 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:12.941 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.033057+0000", 2024-09-14T04:46:12.941 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6240000000000002e-05 2024-09-14T04:46:12.941 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.941 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.941 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:12.941 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.033175+0000", 2024-09-14T04:46:12.941 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011859 2024-09-14T04:46:12.941 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.941 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.941 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:46:12.941 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.033279+0000", 2024-09-14T04:46:12.941 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000103942 2024-09-14T04:46:12.941 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.941 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.942 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:46:12.942 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.043730+0000", 2024-09-14T04:46:12.942 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.010450536999999999 2024-09-14T04:46:12.942 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.942 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.942 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:12.942 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.074688+0000", 2024-09-14T04:46:12.942 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.030958066999999999 2024-09-14T04:46:12.942 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.942 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.942 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:12.942 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.076379+0000", 2024-09-14T04:46:12.942 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001691269 2024-09-14T04:46:12.942 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.943 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.943 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:12.943 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.076426+0000", 2024-09-14T04:46:12.943 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6891000000000002e-05 2024-09-14T04:46:12.943 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.943 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.943 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:12.943 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:37:01.076490+0000", 2024-09-14T04:46:12.943 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.4276999999999995e-05 2024-09-14T04:46:12.943 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:12.943 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:12.943 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:12.943 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.943 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.944 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:24695 53.0 e1319/1315 53:03650f8a:::benchmark_data_smithi039_224949_object24694:head v 1319'1508, mlcod=1319'1505)", 2024-09-14T04:46:12.944 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:38:44.028985+0000", 2024-09-14T04:46:12.944 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 446.912257883, 2024-09-14T04:46:12.944 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.052261575999999997, 2024-09-14T04:46:12.944 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:12.944 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:12.944 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:12.944 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:12.944 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.944 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:12.945 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.028985+0000", 2024-09-14T04:46:12.945 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:12.945 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.945 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.945 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:12.945 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.028985+0000", 2024-09-14T04:46:12.945 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:12.945 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.945 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.945 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:12.945 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.028985+0000", 2024-09-14T04:46:12.945 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.09e-07 2024-09-14T04:46:12.946 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.946 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.946 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:12.946 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.029059+0000", 2024-09-14T04:46:12.946 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.3288999999999996e-05 2024-09-14T04:46:12.946 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.946 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.946 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:12.946 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.029072+0000", 2024-09-14T04:46:12.946 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2809999999999999e-05 2024-09-14T04:46:12.946 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.946 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.946 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:12.947 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.029078+0000", 2024-09-14T04:46:12.947 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.657e-06 2024-09-14T04:46:12.947 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.947 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.947 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:12.947 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.029229+0000", 2024-09-14T04:46:12.947 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015115899999999999 2024-09-14T04:46:12.947 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.947 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.947 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:12.947 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.029249+0000", 2024-09-14T04:46:12.947 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9919e-05 2024-09-14T04:46:12.947 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.947 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.948 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:12.948 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.081212+0000", 2024-09-14T04:46:12.948 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.051962770999999998 2024-09-14T04:46:12.948 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.948 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.948 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:12.948 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.081246+0000", 2024-09-14T04:46:12.948 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.4162000000000003e-05 2024-09-14T04:46:12.948 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:12.948 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:12.948 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:12.948 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.948 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.948 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:24702 53.0 e1319/1315 53:0ad7507c:::benchmark_data_smithi039_224949_object24701:head v 1319'1509, mlcod=1319'1506)", 2024-09-14T04:46:12.949 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:38:44.030119+0000", 2024-09-14T04:46:12.949 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 446.911123116, 2024-09-14T04:46:12.949 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.051852284999999998, 2024-09-14T04:46:12.949 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:12.949 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:12.949 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:12.949 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:12.949 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.949 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:12.949 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.030119+0000", 2024-09-14T04:46:12.949 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:12.949 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.950 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.950 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:12.950 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.030119+0000", 2024-09-14T04:46:12.950 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:12.950 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.950 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.950 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:12.950 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.030120+0000", 2024-09-14T04:46:12.950 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.1399999999999996e-07 2024-09-14T04:46:12.950 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.950 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.951 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:12.951 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.030159+0000", 2024-09-14T04:46:12.951 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.8470000000000003e-05 2024-09-14T04:46:12.951 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.951 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.951 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:12.951 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.030170+0000", 2024-09-14T04:46:12.951 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1114999999999999e-05 2024-09-14T04:46:12.951 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.951 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.951 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:12.951 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.030177+0000", 2024-09-14T04:46:12.951 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8040000000000004e-06 2024-09-14T04:46:12.952 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.952 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.952 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:12.952 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.080322+0000", 2024-09-14T04:46:12.952 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.050145035999999997 2024-09-14T04:46:12.952 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.952 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.952 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:12.952 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.080345+0000", 2024-09-14T04:46:12.952 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3068999999999999e-05 2024-09-14T04:46:12.952 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.952 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.952 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:12.953 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.081935+0000", 2024-09-14T04:46:12.953 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0015904199999999999 2024-09-14T04:46:12.953 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.953 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.953 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:12.953 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.081972+0000", 2024-09-14T04:46:12.953 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.6557000000000003e-05 2024-09-14T04:46:12.953 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:12.953 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:12.953 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:12.953 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.953 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.953 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:24696 53.4 e1319/1315 53:21e22dfb:::benchmark_data_smithi039_224949_object24695:head v 1319'1530, mlcod=1319'1526)", 2024-09-14T04:46:12.954 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:38:44.030684+0000", 2024-09-14T04:46:12.954 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 446.91055852699998, 2024-09-14T04:46:12.954 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.052337142000000003, 2024-09-14T04:46:12.954 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:12.954 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:12.954 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:12.954 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:12.954 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.954 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:12.954 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.030684+0000", 2024-09-14T04:46:12.955 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:12.955 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.955 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.955 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:12.955 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.030684+0000", 2024-09-14T04:46:12.955 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:12.955 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.955 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.955 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:12.955 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.030685+0000", 2024-09-14T04:46:12.956 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.8299999999999996e-07 2024-09-14T04:46:12.956 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.956 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.956 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:12.956 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.030839+0000", 2024-09-14T04:46:12.956 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000154566 2024-09-14T04:46:12.956 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.956 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.956 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:12.956 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.080294+0000", 2024-09-14T04:46:12.956 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.049454867 2024-09-14T04:46:12.956 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.957 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.957 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:12.957 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.080303+0000", 2024-09-14T04:46:12.957 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.5920000000000001e-06 2024-09-14T04:46:12.957 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.957 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.957 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:12.957 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.080355+0000", 2024-09-14T04:46:12.957 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1790999999999999e-05 2024-09-14T04:46:12.957 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.957 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.957 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:12.957 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.080369+0000", 2024-09-14T04:46:12.958 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3964999999999999e-05 2024-09-14T04:46:12.958 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.958 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.958 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:12.958 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.082976+0000", 2024-09-14T04:46:12.958 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0026071620000000001 2024-09-14T04:46:12.958 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.958 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.958 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:12.958 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.083021+0000", 2024-09-14T04:46:12.958 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5416e-05 2024-09-14T04:46:12.958 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:12.958 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:12.959 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:12.959 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.959 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.959 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45053.0:24703 53.0 e1319/1315 53:08f310e0:::benchmark_data_smithi039_224949_object24702:head v 1319'1510, mlcod=1319'1506)", 2024-09-14T04:46:12.959 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:38:44.031148+0000", 2024-09-14T04:46:12.959 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 446.91009418499999, 2024-09-14T04:46:12.959 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.051003469000000003, 2024-09-14T04:46:12.959 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:12.959 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:12.959 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:12.959 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:12.959 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.959 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:12.959 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.031148+0000", 2024-09-14T04:46:12.960 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:12.960 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.960 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.960 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:12.960 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.031148+0000", 2024-09-14T04:46:12.960 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:12.960 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.960 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.960 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:12.960 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.031149+0000", 2024-09-14T04:46:12.960 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.6400000000000002e-07 2024-09-14T04:46:12.961 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.961 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.961 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:12.961 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.031211+0000", 2024-09-14T04:46:12.961 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1780999999999997e-05 2024-09-14T04:46:12.961 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.961 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.961 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:12.961 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.080269+0000", 2024-09-14T04:46:12.961 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.049058483 2024-09-14T04:46:12.961 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.961 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.961 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:12.962 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.080284+0000", 2024-09-14T04:46:12.962 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4654999999999999e-05 2024-09-14T04:46:12.962 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.962 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.962 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:12.962 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.081276+0000", 2024-09-14T04:46:12.962 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00099212600000000008 2024-09-14T04:46:12.962 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.962 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.962 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:12.962 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.081302+0000", 2024-09-14T04:46:12.962 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.6310000000000001e-05 2024-09-14T04:46:12.962 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.963 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.963 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:12.963 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.082123+0000", 2024-09-14T04:46:12.963 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00082041700000000004 2024-09-14T04:46:12.963 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.963 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.963 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:12.963 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:38:44.082152+0000", 2024-09-14T04:46:12.963 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.9033000000000001e-05 2024-09-14T04:46:12.963 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:12.963 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:12.963 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:12.963 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.964 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.964 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:68412 54.e 54:7391dba8:::benchmark_data_smithi039_264334_object68411:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1392)", 2024-09-14T04:46:12.964 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:43:53.411219+0000", 2024-09-14T04:46:12.964 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 137.53002347099999, 2024-09-14T04:46:12.964 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.099827346999999997, 2024-09-14T04:46:12.964 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:12.964 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:12.964 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:12.964 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:46:12.964 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:46:12.964 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:46:12.964 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 68412 2024-09-14T04:46:12.964 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.965 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:12.965 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.965 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:12.965 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:43:53.411219+0000", 2024-09-14T04:46:12.965 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:12.965 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.965 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.965 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:12.965 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:43:53.411219+0000", 2024-09-14T04:46:12.965 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:12.965 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.965 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.965 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:12.966 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:43:53.411221+0000", 2024-09-14T04:46:12.966 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9439999999999999e-06 2024-09-14T04:46:12.966 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.966 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.966 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:12.966 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:43:53.411245+0000", 2024-09-14T04:46:12.966 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3856e-05 2024-09-14T04:46:12.966 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.966 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.966 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:12.966 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:43:53.411264+0000", 2024-09-14T04:46:12.966 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9207000000000001e-05 2024-09-14T04:46:12.966 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.967 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.967 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:12.967 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:43:53.411273+0000", 2024-09-14T04:46:12.967 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.6549999999999993e-06 2024-09-14T04:46:12.967 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.967 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.967 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:12.967 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:43:53.413582+0000", 2024-09-14T04:46:12.967 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0023093699999999998 2024-09-14T04:46:12.967 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.967 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.967 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:12.967 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:43:53.414345+0000", 2024-09-14T04:46:12.967 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00076310099999999997 2024-09-14T04:46:12.968 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.968 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.968 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,7", 2024-09-14T04:46:12.968 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:43:53.414525+0000", 2024-09-14T04:46:12.968 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00017988100000000001 2024-09-14T04:46:12.968 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.968 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.968 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:46:12.968 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:43:53.417077+0000", 2024-09-14T04:46:12.968 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0025516200000000001 2024-09-14T04:46:12.968 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.968 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.968 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:12.969 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:43:53.417438+0000", 2024-09-14T04:46:12.969 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00036087400000000001 2024-09-14T04:46:12.969 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.969 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.969 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:12.969 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:43:53.510650+0000", 2024-09-14T04:46:12.969 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.093212739000000003 2024-09-14T04:46:12.969 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.969 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.969 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:12.969 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:43:53.510842+0000", 2024-09-14T04:46:12.969 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000191841 2024-09-14T04:46:12.970 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.970 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.970 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:12.970 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:43:53.511046+0000", 2024-09-14T04:46:12.970 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00020425900000000001 2024-09-14T04:46:12.970 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:12.970 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:12.970 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:12.970 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.970 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.971 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:133118 54.e 54:73d0dac8:::benchmark_data_smithi039_264334_object6721:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1401)", 2024-09-14T04:46:12.971 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:53.888608+0000", 2024-09-14T04:46:12.971 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 77.052634565000005, 2024-09-14T04:46:12.971 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.047716445000000003, 2024-09-14T04:46:12.971 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:12.971 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:12.971 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:12.971 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:46:12.971 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:46:12.971 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:46:12.971 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 133118 2024-09-14T04:46:12.971 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.972 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:12.972 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.972 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:12.972 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:53.888608+0000", 2024-09-14T04:46:12.972 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:12.972 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.972 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.972 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:12.972 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:53.888608+0000", 2024-09-14T04:46:12.972 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:12.972 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.972 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.972 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:12.973 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:53.888610+0000", 2024-09-14T04:46:12.973 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6080000000000001e-06 2024-09-14T04:46:12.973 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.973 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.973 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:12.973 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:53.888615+0000", 2024-09-14T04:46:12.973 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1760000000000001e-06 2024-09-14T04:46:12.973 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.976 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.976 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:12.976 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:53.888627+0000", 2024-09-14T04:46:12.976 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2285e-05 2024-09-14T04:46:12.976 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.976 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.976 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:12.977 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:53.888634+0000", 2024-09-14T04:46:12.977 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1509999999999998e-06 2024-09-14T04:46:12.977 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.977 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.977 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:12.977 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:53.907258+0000", 2024-09-14T04:46:12.977 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.018623527000000001 2024-09-14T04:46:12.977 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.977 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.978 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:12.978 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:53.907749+0000", 2024-09-14T04:46:12.978 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00049084900000000002 2024-09-14T04:46:12.978 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.978 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.978 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,7", 2024-09-14T04:46:12.978 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:53.908073+0000", 2024-09-14T04:46:12.978 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00032480399999999999 2024-09-14T04:46:12.978 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.978 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.978 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:12.978 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:53.920867+0000", 2024-09-14T04:46:12.979 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.012794135 2024-09-14T04:46:12.979 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.979 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.979 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:46:12.979 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:53.921228+0000", 2024-09-14T04:46:12.979 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00036042399999999998 2024-09-14T04:46:12.979 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.979 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.979 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:12.979 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:53.935919+0000", 2024-09-14T04:46:12.979 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.014690725999999999 2024-09-14T04:46:12.979 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.979 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.979 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:12.980 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:53.936162+0000", 2024-09-14T04:46:12.980 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00024334700000000001 2024-09-14T04:46:12.980 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.980 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.980 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:12.980 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:53.936324+0000", 2024-09-14T04:46:12.980 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00016241300000000001 2024-09-14T04:46:12.980 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:12.980 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:12.980 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:12.980 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.980 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.980 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.35436.0:190139 54.1 e1413/1380 54:824d3df3:::benchmark_data_smithi039_264334_object63742:head v 1413'11886, mlcod=1413'11882)", 2024-09-14T04:46:12.981 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:46:04.495298+0000", 2024-09-14T04:46:12.981 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 6.4459441069999999, 2024-09-14T04:46:12.981 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.143291582, 2024-09-14T04:46:12.981 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:12.981 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:12.981 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:12.981 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:12.981 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.981 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:12.981 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.495298+0000", 2024-09-14T04:46:12.981 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:12.981 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.981 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.981 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:12.982 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.495298+0000", 2024-09-14T04:46:12.982 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:12.982 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.982 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.982 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:12.982 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.495299+0000", 2024-09-14T04:46:12.982 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.4799999999999997e-07 2024-09-14T04:46:12.982 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.982 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.982 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:12.982 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.495306+0000", 2024-09-14T04:46:12.982 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.058e-06 2024-09-14T04:46:12.982 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.983 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.983 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:12.983 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.495318+0000", 2024-09-14T04:46:12.983 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1623e-05 2024-09-14T04:46:12.983 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.983 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.983 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:12.983 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.495325+0000", 2024-09-14T04:46:12.983 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9589999999999998e-06 2024-09-14T04:46:12.983 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.983 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.983 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:12.983 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.495389+0000", 2024-09-14T04:46:12.983 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3657000000000004e-05 2024-09-14T04:46:12.984 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.984 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.984 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:12.984 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.495409+0000", 2024-09-14T04:46:12.984 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9888e-05 2024-09-14T04:46:12.984 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.984 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.984 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:12.984 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.638561+0000", 2024-09-14T04:46:12.984 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.143152103 2024-09-14T04:46:12.985 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.985 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.985 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:12.985 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.638590+0000", 2024-09-14T04:46:12.985 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.9346e-05 2024-09-14T04:46:12.985 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:12.985 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:12.985 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:12.985 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.985 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.985 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.35436.0:190140 54.1 e1413/1380 54:89558b4e:::benchmark_data_smithi039_264334_object63743:head v 1413'11887, mlcod=1413'11884)", 2024-09-14T04:46:12.985 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:46:04.500201+0000", 2024-09-14T04:46:12.985 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 6.4410414060000001, 2024-09-14T04:46:12.985 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.14532013599999999, 2024-09-14T04:46:12.986 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:12.986 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:12.986 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:12.986 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:12.986 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.986 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:12.986 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.500201+0000", 2024-09-14T04:46:12.986 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:12.986 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.986 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.986 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:12.986 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.500201+0000", 2024-09-14T04:46:12.986 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:12.987 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.987 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.987 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:12.987 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.500202+0000", 2024-09-14T04:46:12.987 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.7199999999999997e-07 2024-09-14T04:46:12.987 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.987 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.987 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:12.987 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.500209+0000", 2024-09-14T04:46:12.987 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8539999999999999e-06 2024-09-14T04:46:12.987 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.987 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.988 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:12.988 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.500221+0000", 2024-09-14T04:46:12.988 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2023e-05 2024-09-14T04:46:12.988 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.988 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.988 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:12.989 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.500228+0000", 2024-09-14T04:46:12.989 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1110000000000002e-06 2024-09-14T04:46:12.989 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.989 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.989 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:12.989 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.500294+0000", 2024-09-14T04:46:12.990 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.6187999999999997e-05 2024-09-14T04:46:12.990 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.990 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.990 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:12.990 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.500314+0000", 2024-09-14T04:46:12.990 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9749999999999999e-05 2024-09-14T04:46:12.990 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.991 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.991 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:12.991 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.645445+0000", 2024-09-14T04:46:12.991 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.145131121 2024-09-14T04:46:12.991 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.991 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.992 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:12.992 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.645521+0000", 2024-09-14T04:46:12.992 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6117000000000003e-05 2024-09-14T04:46:12.992 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:12.992 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:12.992 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:12.992 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.992 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.993 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.35436.0:190159 54.1 e1413/1380 54:823ff782:::benchmark_data_smithi039_264334_object63762:head v 1413'11888, mlcod=1413'11884)", 2024-09-14T04:46:12.993 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:46:04.519482+0000", 2024-09-14T04:46:12.993 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 6.4217608589999999, 2024-09-14T04:46:12.993 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.126075985, 2024-09-14T04:46:12.993 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:12.993 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:12.994 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:12.994 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:12.994 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.994 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:12.994 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.519482+0000", 2024-09-14T04:46:12.994 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:12.994 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.995 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.995 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:12.995 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.519482+0000", 2024-09-14T04:46:12.995 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:12.995 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.996 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.996 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:12.996 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.519483+0000", 2024-09-14T04:46:12.996 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6649999999999999e-06 2024-09-14T04:46:12.996 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.996 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.996 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:12.996 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.519493+0000", 2024-09-14T04:46:12.996 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.6029999999999994e-06 2024-09-14T04:46:12.997 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.997 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.997 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:12.997 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.630001+0000", 2024-09-14T04:46:12.997 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.110508211 2024-09-14T04:46:12.997 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.997 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.997 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:12.997 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.630012+0000", 2024-09-14T04:46:12.997 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1239e-05 2024-09-14T04:46:12.998 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.998 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.998 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:12.998 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.633008+0000", 2024-09-14T04:46:12.998 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.002995359 2024-09-14T04:46:12.998 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.998 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.998 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:12.998 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.633023+0000", 2024-09-14T04:46:12.998 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4837999999999999e-05 2024-09-14T04:46:12.998 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.998 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.998 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:12.999 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.645526+0000", 2024-09-14T04:46:12.999 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.012503515 2024-09-14T04:46:12.999 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.999 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:12.999 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:12.999 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.645558+0000", 2024-09-14T04:46:12.999 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.1554999999999999e-05 2024-09-14T04:46:12.999 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:12.999 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:12.999 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:12.999 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:12.999 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.000 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.35436.0:190151 54.e e1413/1403 54:7af82fd8:::benchmark_data_smithi039_264334_object63754:head v 1413'11897, mlcod=1392'3995)", 2024-09-14T04:46:13.000 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:46:04.519554+0000", 2024-09-14T04:46:13.000 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 6.421688123, 2024-09-14T04:46:13.000 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.120386806, 2024-09-14T04:46:13.000 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:13.000 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:13.000 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:13.000 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:13.000 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.000 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:13.000 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.519554+0000", 2024-09-14T04:46:13.000 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:13.001 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.001 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.001 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:13.001 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.519554+0000", 2024-09-14T04:46:13.001 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:13.001 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.001 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.001 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:13.001 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.519555+0000", 2024-09-14T04:46:13.001 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.2099999999999995e-07 2024-09-14T04:46:13.001 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.001 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.002 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:13.002 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.519562+0000", 2024-09-14T04:46:13.002 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.0079999999999996e-06 2024-09-14T04:46:13.002 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.002 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.002 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:13.002 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.629998+0000", 2024-09-14T04:46:13.002 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.110435494 2024-09-14T04:46:13.002 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.002 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.002 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:13.002 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.630013+0000", 2024-09-14T04:46:13.002 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4817999999999999e-05 2024-09-14T04:46:13.003 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.003 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.003 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:13.003 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.630071+0000", 2024-09-14T04:46:13.003 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.8214999999999997e-05 2024-09-14T04:46:13.003 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.003 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.003 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:13.003 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.630098+0000", 2024-09-14T04:46:13.003 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.6845000000000001e-05 2024-09-14T04:46:13.003 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.003 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.003 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:13.004 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.639895+0000", 2024-09-14T04:46:13.004 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0097977050000000003 2024-09-14T04:46:13.004 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.004 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.004 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:13.004 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:04.639941+0000", 2024-09-14T04:46:13.004 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5800000000000002e-05 2024-09-14T04:46:13.005 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:13.005 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:13.005 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:13.005 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.005 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.005 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.35436.0:192405 54.6 e1413/1380 54:65bc8903:::benchmark_data_smithi039_264334_object66008:head v 1413'12027, mlcod=1413'12025)", 2024-09-14T04:46:13.005 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:46:07.335655+0000", 2024-09-14T04:46:13.005 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 3.6055870429999999, 2024-09-14T04:46:13.005 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.070332895000000006, 2024-09-14T04:46:13.005 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:13.005 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:13.005 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:13.006 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:13.006 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.006 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:13.006 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:07.335655+0000", 2024-09-14T04:46:13.006 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:13.006 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.006 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.006 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:13.006 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:07.335655+0000", 2024-09-14T04:46:13.006 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:13.006 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.006 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.007 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:13.007 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:07.335657+0000", 2024-09-14T04:46:13.007 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0750000000000001e-06 2024-09-14T04:46:13.007 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.007 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.007 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:13.007 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:07.335663+0000", 2024-09-14T04:46:13.008 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.5139999999999998e-06 2024-09-14T04:46:13.008 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.008 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.008 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:13.008 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:07.335674+0000", 2024-09-14T04:46:13.008 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0494e-05 2024-09-14T04:46:13.008 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.008 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.008 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:13.008 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:07.335681+0000", 2024-09-14T04:46:13.008 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.5000000000000002e-06 2024-09-14T04:46:13.008 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.009 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.009 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:13.009 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:07.335743+0000", 2024-09-14T04:46:13.009 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2416999999999995e-05 2024-09-14T04:46:13.009 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.009 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.009 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:13.009 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:07.335761+0000", 2024-09-14T04:46:13.009 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7422999999999999e-05 2024-09-14T04:46:13.009 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.009 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.009 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:13.009 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:07.405710+0000", 2024-09-14T04:46:13.010 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.069949487000000005 2024-09-14T04:46:13.010 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.010 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.010 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:13.010 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:07.405988+0000", 2024-09-14T04:46:13.010 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00027798500000000002 2024-09-14T04:46:13.010 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:13.010 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:13.010 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:13.010 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.011 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.011 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.35436.0:192423 54.1 e1413/1380 54:8fd6b6b3:::benchmark_data_smithi039_264334_object66026:head v 1413'12031, mlcod=1413'12029)", 2024-09-14T04:46:13.011 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:46:07.351540+0000", 2024-09-14T04:46:13.011 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 3.589702242, 2024-09-14T04:46:13.011 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.049027262000000002, 2024-09-14T04:46:13.011 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:13.011 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:13.011 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:13.011 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:13.011 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.011 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:13.011 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:07.351540+0000", 2024-09-14T04:46:13.011 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:13.012 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.012 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.012 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:13.012 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:07.351540+0000", 2024-09-14T04:46:13.012 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:13.012 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.012 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.012 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:13.012 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:07.351542+0000", 2024-09-14T04:46:13.012 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.455e-06 2024-09-14T04:46:13.012 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.012 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.012 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:13.013 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:07.351550+0000", 2024-09-14T04:46:13.013 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.5830000000000007e-06 2024-09-14T04:46:13.013 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.013 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.013 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:13.013 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:07.393685+0000", 2024-09-14T04:46:13.013 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.042134484 2024-09-14T04:46:13.013 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.013 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.013 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:13.013 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:07.393701+0000", 2024-09-14T04:46:13.014 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6183000000000001e-05 2024-09-14T04:46:13.014 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.014 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.014 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:13.014 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:07.393762+0000", 2024-09-14T04:46:13.014 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0958000000000001e-05 2024-09-14T04:46:13.014 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.014 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.014 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:13.014 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:07.393782+0000", 2024-09-14T04:46:13.014 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0449999999999999e-05 2024-09-14T04:46:13.014 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.014 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.015 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:13.015 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:07.400353+0000", 2024-09-14T04:46:13.015 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.006571042 2024-09-14T04:46:13.015 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.015 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.015 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:13.015 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:07.400567+0000", 2024-09-14T04:46:13.015 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000214107 2024-09-14T04:46:13.015 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:13.015 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:13.016 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:13.016 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.016 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.016 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.35436.0:192411 54.e e1413/1403 54:7d4cdcce:::benchmark_data_smithi039_264334_object66014:head v 1413'12031, mlcod=1392'4126)", 2024-09-14T04:46:13.016 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:46:07.357541+0000", 2024-09-14T04:46:13.016 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 3.5837017339999999, 2024-09-14T04:46:13.016 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.061082951000000003, 2024-09-14T04:46:13.016 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:13.016 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:13.016 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:13.016 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:13.016 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.016 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:13.017 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:07.357541+0000", 2024-09-14T04:46:13.017 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:13.017 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.017 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.017 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:13.017 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:07.357541+0000", 2024-09-14T04:46:13.017 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:13.017 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.017 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.017 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:13.017 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:07.357542+0000", 2024-09-14T04:46:13.017 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2979999999999999e-06 2024-09-14T04:46:13.017 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.018 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.018 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:13.018 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:07.357561+0000", 2024-09-14T04:46:13.018 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8525999999999998e-05 2024-09-14T04:46:13.018 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.018 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.018 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:13.018 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:07.393841+0000", 2024-09-14T04:46:13.018 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.036280517999999998 2024-09-14T04:46:13.018 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.018 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.018 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:13.018 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:07.393847+0000", 2024-09-14T04:46:13.018 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.4099999999999996e-06 2024-09-14T04:46:13.018 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.019 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.019 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:13.019 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:07.398297+0000", 2024-09-14T04:46:13.019 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0044492009999999999 2024-09-14T04:46:13.019 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.019 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.019 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:13.019 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:07.398369+0000", 2024-09-14T04:46:13.019 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1954999999999998e-05 2024-09-14T04:46:13.019 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.019 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.019 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:13.019 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:07.418104+0000", 2024-09-14T04:46:13.020 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.019735143 2024-09-14T04:46:13.020 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.020 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.020 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:13.020 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:07.418624+0000", 2024-09-14T04:46:13.020 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00051990000000000001 2024-09-14T04:46:13.020 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:13.020 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:13.020 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:13.020 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.020 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.020 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.35436.0:195070 54.e e1413/1403 54:7728253a:::benchmark_data_smithi039_264334_object68673:head v 1413'12202, mlcod=1392'4298)", 2024-09-14T04:46:13.020 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:46:10.758176+0000", 2024-09-14T04:46:13.021 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.18306634999999999, 2024-09-14T04:46:13.021 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.052482450999999999, 2024-09-14T04:46:13.021 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:13.021 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:13.021 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:13.021 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:13.021 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.021 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:13.021 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:10.758176+0000", 2024-09-14T04:46:13.021 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:13.021 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.021 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.021 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:13.022 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:10.758176+0000", 2024-09-14T04:46:13.022 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:13.022 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.022 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.022 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:13.022 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:10.758177+0000", 2024-09-14T04:46:13.022 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2950000000000001e-06 2024-09-14T04:46:13.022 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.022 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.022 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:13.022 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:10.758186+0000", 2024-09-14T04:46:13.022 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.5210000000000002e-06 2024-09-14T04:46:13.022 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.022 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.023 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:13.023 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:10.758207+0000", 2024-09-14T04:46:13.023 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0755999999999999e-05 2024-09-14T04:46:13.023 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.023 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.023 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:13.023 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:10.758215+0000", 2024-09-14T04:46:13.023 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.2889999999999998e-06 2024-09-14T04:46:13.023 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.023 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.023 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:13.023 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:10.759065+0000", 2024-09-14T04:46:13.024 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00084959599999999999 2024-09-14T04:46:13.024 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.024 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.024 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:13.024 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:10.759219+0000", 2024-09-14T04:46:13.024 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000154587 2024-09-14T04:46:13.024 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.024 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.024 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:13.024 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:10.810605+0000", 2024-09-14T04:46:13.024 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.051385360999999997 2024-09-14T04:46:13.024 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.025 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.025 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:13.025 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:10.810659+0000", 2024-09-14T04:46:13.025 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4045999999999997e-05 2024-09-14T04:46:13.025 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:13.025 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:13.025 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:13.025 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.025 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.025 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.35436.0:195077 54.6 e1413/1380 54:6ab49f44:::benchmark_data_smithi039_264334_object68680:head v 1413'12192, mlcod=1413'12187)", 2024-09-14T04:46:13.025 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:46:10.758415+0000", 2024-09-14T04:46:13.025 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.18282794799999999, 2024-09-14T04:46:13.025 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.052279036000000001, 2024-09-14T04:46:13.026 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:13.026 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:13.026 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:13.026 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:13.026 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.026 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:13.026 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:10.758415+0000", 2024-09-14T04:46:13.026 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:13.026 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.026 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.026 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:13.026 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:10.758415+0000", 2024-09-14T04:46:13.026 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:13.026 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.027 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.027 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:13.027 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:10.758415+0000", 2024-09-14T04:46:13.027 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.8700000000000005e-07 2024-09-14T04:46:13.027 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.027 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.027 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:13.027 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:10.758422+0000", 2024-09-14T04:46:13.027 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.5379999999999999e-06 2024-09-14T04:46:13.027 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.027 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.027 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:13.027 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:10.758437+0000", 2024-09-14T04:46:13.027 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5118e-05 2024-09-14T04:46:13.027 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.028 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.028 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:13.028 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:10.758444+0000", 2024-09-14T04:46:13.028 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7240000000000003e-06 2024-09-14T04:46:13.028 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.028 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.028 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:13.028 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:10.765612+0000", 2024-09-14T04:46:13.028 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0071686750000000002 2024-09-14T04:46:13.028 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.028 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.028 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:13.028 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:10.765664+0000", 2024-09-14T04:46:13.029 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1591999999999997e-05 2024-09-14T04:46:13.029 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.029 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.029 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:13.029 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:10.810664+0000", 2024-09-14T04:46:13.029 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.044999590999999999 2024-09-14T04:46:13.029 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.029 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.029 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:13.029 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:10.810694+0000", 2024-09-14T04:46:13.029 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.0011000000000001e-05 2024-09-14T04:46:13.029 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:13.030 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:13.030 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:13.030 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.030 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.030 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.35436.0:195085 54.6 e1413/1380 54:69663e81:::benchmark_data_smithi039_264334_object68688:head v 1413'12193, mlcod=1413'12187)", 2024-09-14T04:46:13.030 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:46:10.765208+0000", 2024-09-14T04:46:13.030 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.176034626, 2024-09-14T04:46:13.030 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.045508528999999999, 2024-09-14T04:46:13.030 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:13.030 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:13.030 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:13.030 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:13.030 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.030 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:13.031 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:10.765208+0000", 2024-09-14T04:46:13.031 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:13.031 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.031 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.031 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:13.031 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:10.765208+0000", 2024-09-14T04:46:13.031 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:13.031 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.031 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.031 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:13.031 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:10.765209+0000", 2024-09-14T04:46:13.031 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3319999999999999e-06 2024-09-14T04:46:13.032 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.032 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.032 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:13.032 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:10.765219+0000", 2024-09-14T04:46:13.032 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.499e-06 2024-09-14T04:46:13.032 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.032 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.032 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:13.032 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:10.765240+0000", 2024-09-14T04:46:13.032 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0976e-05 2024-09-14T04:46:13.032 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.033 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.033 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:13.033 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:10.765248+0000", 2024-09-14T04:46:13.033 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.8359999999999998e-06 2024-09-14T04:46:13.033 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.033 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.033 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:13.033 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:10.769576+0000", 2024-09-14T04:46:13.033 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0043276149999999999 2024-09-14T04:46:13.033 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.033 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.033 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:13.033 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:10.769595+0000", 2024-09-14T04:46:13.033 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8459000000000001e-05 2024-09-14T04:46:13.034 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.034 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.034 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:13.034 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:10.810696+0000", 2024-09-14T04:46:13.034 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.041101021000000001 2024-09-14T04:46:13.034 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:13.034 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:13.034 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:13.034 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:10.810716+0000", 2024-09-14T04:46:13.034 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0791000000000001e-05 2024-09-14T04:46:13.034 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:13.034 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:13.034 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:13.034 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:13.035 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:13.035 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:46:13.035 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:13.068 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:13.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:13.069 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:13.169 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:13.193 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:13.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:13.194 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:13.295 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:13.336 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:13.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:13.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:13.438 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:13.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:13.473 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:13.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:13.574 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:13.615 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:13.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:13.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:13.717 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:13.737 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:13 smithi039 ceph-mon[106470]: pgmap v2480: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 17176/178782 objects misplaced (9.607%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:46:13.737 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:13 smithi039 ceph-mon[111235]: pgmap v2480: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 17176/178782 objects misplaced (9.607%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:46:13.750 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:13.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:13.752 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:13.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:13 smithi129 ceph-mon[101316]: pgmap v2480: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 3.6 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 17176/178782 objects misplaced (9.607%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:46:13.852 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:13.890 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:13.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:13.891 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:13.992 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:14.027 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:14.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:14.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:14.129 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:14.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:14.176 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:14.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:14.277 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:14.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:14.322 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:14.322 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:14.424 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:14.463 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:14.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:14.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:14.565 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:14.599 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:14.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:14.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:14.701 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:14.748 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:14.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:14.749 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:14.850 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:14.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:14.892 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:14.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:14.993 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:15.024 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:15.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:15.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:15.111 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-14T04:46:15.125 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:15.200 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:46:15.200 INFO:teuthology.orchestra.run.smithi039.stdout:{"fsid":"bd7c803c-724b-11ef-bceb-c7b262605968","health":{"status":"HEALTH_OK","checks":{},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2690,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1413,"num_osds":8,"num_up_osds":8,"osd_up_since":1726289028,"num_in_osds":8,"osd_in_since":1726285856,"num_remapped_pgs":4},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":95},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":56221,"data_bytes":3683325949,"bytes_used":13900263424,"bytes_avail":754060021760,"bytes_total":767960285184,"misplaced_objects":16398,"misplaced_total":168663,"misplaced_ratio":0.097223457426940108,"recovering_objects_per_sec":17,"recovering_bytes_per_sec":1148712,"recovering_keys_per_sec":0,"num_objects_recovered":198,"num_bytes_recovered":12976128,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":914},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi039.bbwiwf","status":"up:active","gid":14646}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.39:8443/","prometheus":"http://172.21.15.39:9283/"}},"servicemap":{"epoch":37,"modified":"2024-09-14T04:45:23.646879+0000","services":{}},"progress_events":{}} 2024-09-14T04:46:15.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:15.228 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:15.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:15.329 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:15.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:15.361 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:15.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:15.462 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:15.495 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:15.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:15.496 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:15.597 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:15.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:15.635 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:15.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:15.726 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:15 smithi039 ceph-mon[106470]: pgmap v2481: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 914 op/s; 16398/168663 objects misplaced (9.722%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:46:15.727 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:15 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3912573901' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-14T04:46:15.727 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:15 smithi039 ceph-mon[111235]: pgmap v2481: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 914 op/s; 16398/168663 objects misplaced (9.722%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:46:15.727 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:15 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3912573901' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-14T04:46:15.737 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:15.771 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:15.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:15 smithi129 ceph-mon[101316]: pgmap v2481: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 914 op/s; 16398/168663 objects misplaced (9.722%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:46:15.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:15 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3912573901' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-14T04:46:15.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:15.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:15.874 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:15.917 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:15.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:15.918 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:16.019 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:16.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:16.057 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:16.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:16.117 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-14T04:46:16.118 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:46:16.159 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:16.205 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:16.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:16.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:16.307 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:16.362 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:16.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:16.364 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:16.413 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:46:16.437 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:46:16.437 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:46:16.465 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:16.489 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:16.489 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:46:16.490 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:46:16.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:16.490 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:16.527 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:46:16.528 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:46:16.579 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:46:16.580 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:46:16.591 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:16.611 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:46:16.611 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:46:16.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:16.630 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:16.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:16.640 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:46:16.646 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:46:16.646 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:46:16.681 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:46:16.681 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:46:16.714 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:16 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1710600296' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:46:16.714 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:16 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1710600296' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:46:16.714 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:46:16.715 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:46:16.731 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:16.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:16 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1710600296' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:46:16.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:16.782 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:16.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:16.883 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:16.924 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:16.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:16.925 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:17.026 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:17.064 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:17.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:17.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:17.166 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:17.188 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:17.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:17.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:17.290 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:17.327 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:17.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:17.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:17.429 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:17.470 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:17.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:17.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:17.571 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:17.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:17.594 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:17.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:17.695 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:17.726 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:17.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:17.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:17.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:17 smithi129 ceph-mon[101316]: pgmap v2482: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 16398/168663 objects misplaced (9.722%); 1.0 MiB/s, 16 objects/s recovering 2024-09-14T04:46:17.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:17 smithi039 ceph-mon[106470]: pgmap v2482: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 16398/168663 objects misplaced (9.722%); 1.0 MiB/s, 16 objects/s recovering 2024-09-14T04:46:17.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:17 smithi039 ceph-mon[111235]: pgmap v2482: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 686 op/s; 16398/168663 objects misplaced (9.722%); 1.0 MiB/s, 16 objects/s recovering 2024-09-14T04:46:17.828 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:17.866 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:17.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:17.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:17.968 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:18.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:18.005 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:18.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:18.105 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:18.148 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:18.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:18.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:18.250 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:18.271 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:18.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:18.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:18.372 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:18.410 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:18.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:18.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:18.512 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:18.543 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:18.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:18.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:18.645 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:18.682 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:18.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:18.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:18.784 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:18.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:18.829 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:18.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:18.930 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:18.967 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:18.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:18.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:19.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:46:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:46:18] "GET /metrics HTTP/1.1" 200 36749 "" "Prometheus/2.51.0" 2024-09-14T04:46:19.068 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:19.107 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:19.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:19.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:19.209 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:19.237 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:19.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:19.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:19.339 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:19.360 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:19.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:19.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:19.462 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:19.486 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:19.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:19.487 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:19.587 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:19.622 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:19.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:19.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:19.724 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:19.760 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:19.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:19.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:19.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:19 smithi129 ceph-mon[101316]: pgmap v2483: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 16051/163182 objects misplaced (9.836%); 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:46:19.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:19 smithi039 ceph-mon[106470]: pgmap v2483: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 16051/163182 objects misplaced (9.836%); 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:46:19.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:19 smithi039 ceph-mon[111235]: pgmap v2483: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 838 op/s; 16051/163182 objects misplaced (9.836%); 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:46:19.862 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:19.893 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:19.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:19.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:19.995 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:20.026 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:20.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:20.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:20.128 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:20.172 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:20.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:20.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:20.274 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:20.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:20.307 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:20.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:20.407 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:20.451 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:20.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:20.453 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:20.554 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:20.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:20.586 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:20.586 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:20.687 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:20.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:20.721 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:20.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:20.822 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:20.852 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:20.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:20.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:20.954 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:20.983 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:20.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:20.984 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:21.084 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:21.109 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:21.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:21.110 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:21.211 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:21.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:21.257 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:21.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:21.358 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:21.385 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:21.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:21.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:21.487 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:21.525 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:21.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:21.526 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:21.627 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:21.654 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:21.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:21.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:21.756 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:21.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:21 smithi129 ceph-mon[101316]: pgmap v2484: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 875 op/s; 15528/156465 objects misplaced (9.924%); 986 KiB/s, 15 objects/s recovering 2024-09-14T04:46:21.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:21 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:46:21.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:21 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:46:21.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:21 smithi039 ceph-mon[106470]: pgmap v2484: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 875 op/s; 15528/156465 objects misplaced (9.924%); 986 KiB/s, 15 objects/s recovering 2024-09-14T04:46:21.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:21 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:46:21.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:21 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:46:21.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:21 smithi039 ceph-mon[111235]: pgmap v2484: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 875 op/s; 15528/156465 objects misplaced (9.924%); 986 KiB/s, 15 objects/s recovering 2024-09-14T04:46:21.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:21 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:46:21.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:21 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:46:21.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:21.791 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:21.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:21.892 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:21.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:21.921 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:21.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:22.023 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:22.068 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:22.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:22.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:22.170 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:22.215 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:22.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:22.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:22.316 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:22.338 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:22.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:22.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:22.440 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:22.477 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:22.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:22.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:22.579 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:22.617 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:22.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:22.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:22.719 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:22.759 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:22.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:22.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:22.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:46:22.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:46:22.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:46:22.861 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:22.905 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:46:22.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:22.906 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:22.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:23.007 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:23.041 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:23.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:23.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:23.143 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:23.188 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:23.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:23.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:23.290 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:23.324 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:23.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:23.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:23.426 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:23.466 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:23.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:23.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:23.568 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:23.602 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:23.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:23.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:23.704 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:23.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:23.743 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:23.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:23.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:23 smithi129 ceph-mon[101316]: pgmap v2485: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 15418/154275 objects misplaced (9.994%); 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:46:23.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:23 smithi039 ceph-mon[106470]: pgmap v2485: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 15418/154275 objects misplaced (9.994%); 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:46:23.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:23 smithi039 ceph-mon[111235]: pgmap v2485: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 741 op/s; 15418/154275 objects misplaced (9.994%); 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:46:23.844 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:23.871 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:23.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:23.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:23.973 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:24.004 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:24.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:24.005 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:24.106 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:24.139 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:24.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:24.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:24.241 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:24.278 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:24.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:24.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:24.380 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:24.417 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:24.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:24.418 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:24.519 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:24.555 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:24.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:24.556 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:24.657 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:24.688 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:24.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:24.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:24.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:24 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:46:24.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:24 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:46:24.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:24 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:46:24.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:24 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:46:24.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:24 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:46:24.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:24 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:46:24.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:24 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:46:24.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:24 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:46:24.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:24 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:46:24.791 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:24.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:24.828 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:24.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:24.929 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:24.967 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:24.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:24.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:25.069 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:25.109 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:25.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:25.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:25.211 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:25.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:25.245 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:25.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:25.346 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:25.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:25.381 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:25.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:25.482 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:25.525 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:25.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:25.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:25.627 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:25.659 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:25.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:25.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:25.761 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:25 smithi129 ceph-mon[101316]: pgmap v2486: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 948 op/s; 14671/144633 objects misplaced (10.144%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:46:25.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:25 smithi039 ceph-mon[106470]: pgmap v2486: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 948 op/s; 14671/144633 objects misplaced (10.144%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:46:25.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:25 smithi039 ceph-mon[111235]: pgmap v2486: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 948 op/s; 14671/144633 objects misplaced (10.144%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:46:25.792 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:25.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:25.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:25.888 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:46:25.894 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:25.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:25.920 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:25.920 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:26.021 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:26.049 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:26.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:26.050 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:26.151 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:26.181 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:26.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:26.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:26.283 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:26.319 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:26.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:26.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:26.421 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:26.457 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:26.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:26.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:26.559 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:26.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:26.587 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:26.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:26.688 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:26.710 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:46:26.710 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:46:26.731 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:26.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:26.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:26.833 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:26.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:26.897 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:26.897 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:26.998 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:27.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:27.041 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:27.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:27.142 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:27.176 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:27.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:27.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:27.277 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:27.315 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:27.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:27.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:27.417 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:27.448 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:27.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:27.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:27.550 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:27.583 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:27.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:27.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:27.658 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-14T04:46:27.685 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:27.742 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:27.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:27.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:27.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:27 smithi039 ceph-mon[106470]: pgmap v2487: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 14671/144633 objects misplaced (10.144%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:46:27.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:27 smithi039 ceph-mon[111235]: pgmap v2487: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 14671/144633 objects misplaced (10.144%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:46:27.844 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:27.883 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:27.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:27.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:27.984 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:28.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:27 smithi129 ceph-mon[101316]: pgmap v2487: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 667 op/s; 14671/144633 objects misplaced (10.144%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:46:28.028 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:28.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:28.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:28.130 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:28.168 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:28.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:28.169 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:28.270 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:28.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:28.304 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:28.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:28.405 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:28.470 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:28.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:28.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:28.572 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:28.625 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:28.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:28.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:28.726 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:28.761 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:28.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:28.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:28.863 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:28.898 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:28.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:28.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:29.000 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:29.034 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:29.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:29.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:46:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:46:28] "GET /metrics HTTP/1.1" 200 36753 "" "Prometheus/2.51.0" 2024-09-14T04:46:29.135 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:29.176 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:29.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:29.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:29.278 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:29.309 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:29.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:29.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:29.411 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:29.449 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:29.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:29.450 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:29.549 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is set 2024-09-14T04:46:29.551 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:29.579 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:29.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:29.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:29.681 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:29.715 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:29.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:29.716 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:29.817 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:29.822 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:29 smithi129 ceph-mon[101316]: pgmap v2488: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 14323/139083 objects misplaced (10.298%); 1.6 MiB/s, 26 objects/s recovering 2024-09-14T04:46:29.822 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:29 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/240509763' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:46:29.822 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:29 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:46:29.858 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:29.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:29.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:29.960 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:30.000 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:30.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:30.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:30.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:29 smithi039 ceph-mon[106470]: pgmap v2488: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 14323/139083 objects misplaced (10.298%); 1.6 MiB/s, 26 objects/s recovering 2024-09-14T04:46:30.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:29 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/240509763' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:46:30.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:29 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:46:30.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:29 smithi039 ceph-mon[111235]: pgmap v2488: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 821 op/s; 14323/139083 objects misplaced (10.298%); 1.6 MiB/s, 26 objects/s recovering 2024-09-14T04:46:30.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:29 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/240509763' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:46:30.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:29 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:46:30.102 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:30.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:30.131 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:30.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:30.232 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:30.260 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:30.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:30.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:30.362 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:30.393 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:30.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:30.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:30.495 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:30.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:30.527 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:30.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:30.628 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:30.665 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:30.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:30.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:30.768 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:30.811 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:30.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:30.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:30.912 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:30.947 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:30.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:30.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:31.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:30 smithi129 ceph-mon[101316]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:46:31.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:30 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:46:31.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:30 smithi129 ceph-mon[101316]: osdmap e1414: 8 total, 8 up, 8 in 2024-09-14T04:46:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:30 smithi039 ceph-mon[106470]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:46:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:30 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:46:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:30 smithi039 ceph-mon[106470]: osdmap e1414: 8 total, 8 up, 8 in 2024-09-14T04:46:31.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:30 smithi039 ceph-mon[111235]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:46:31.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:30 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:46:31.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:30 smithi039 ceph-mon[111235]: osdmap e1414: 8 total, 8 up, 8 in 2024-09-14T04:46:31.049 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:31.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:31.100 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:31.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:31.201 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:31.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:31.237 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:31.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:31.338 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:31.375 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:31.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:31.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:31.477 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:31.505 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:31.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:31.506 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:31.607 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:31.627 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:31.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:31.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:31.729 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:31.752 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:31.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:31.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:31.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:31 smithi039 ceph-mon[106470]: pgmap v2490: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 13819/132339 objects misplaced (10.442%); 1.4 MiB/s, 21 objects/s recovering 2024-09-14T04:46:31.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:31 smithi039 ceph-mon[111235]: pgmap v2490: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 13819/132339 objects misplaced (10.442%); 1.4 MiB/s, 21 objects/s recovering 2024-09-14T04:46:31.854 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:31.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:31.876 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:31.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:31.977 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:32.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:31 smithi129 ceph-mon[101316]: pgmap v2490: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 803 op/s; 13819/132339 objects misplaced (10.442%); 1.4 MiB/s, 21 objects/s recovering 2024-09-14T04:46:32.023 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:32.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:32.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:32.125 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:32.155 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:32.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:32.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:32.257 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:32.282 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:32.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:32.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:32.384 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:32.425 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:32.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:32.426 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:32.526 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:32.558 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:32.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:32.559 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:32.660 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:32.687 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:32.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:32.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:32.689 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:32 smithi039 ceph-mon[106470]: pgmap v2491: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 13714/129894 objects misplaced (10.558%); 1.3 MiB/s, 21 objects/s recovering 2024-09-14T04:46:32.689 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:32 smithi039 ceph-mon[111235]: pgmap v2491: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 13714/129894 objects misplaced (10.558%); 1.3 MiB/s, 21 objects/s recovering 2024-09-14T04:46:32.789 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:32.827 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:32.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:32.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:32.929 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:32.964 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:32.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:32.965 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:32 smithi129 ceph-mon[101316]: pgmap v2491: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 13714/129894 objects misplaced (10.558%); 1.3 MiB/s, 21 objects/s recovering 2024-09-14T04:46:33.066 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:33.108 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:33.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:33.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:33.129 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:46:33.130 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T04:46:33.211 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:33.247 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:33.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:33.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:33.349 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:33.411 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:33.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:33.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:33.513 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:33.542 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:33.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:33.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:33.643 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:33.655 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:46:33.694 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:33.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:33.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:33.767 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:33 smithi039 ceph-mon[111235]: from='client.35610 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:46:33.769 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:33 smithi039 ceph-mon[106470]: from='client.35610 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:46:33.796 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:33.831 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:33.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:33.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:33.933 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:33.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:33.977 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:33.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:34.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:33 smithi129 ceph-mon[101316]: from='client.35610 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:46:34.078 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:34.106 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:34.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:34.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:34.151 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:46:34.170 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":2491,"stamp":"2024-09-14T04:46:31.676730+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2836404221,"num_objects":43298,"num_object_clones":0,"num_object_copies":129894,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":13714,"num_objects_unfound":0,"num_objects_dirty":43298,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":209582,"num_write_kb":8089250,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1741,"num_bytes_recovered":112137710,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123833,"ondisk_log_size":123833,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":301,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":11100400,"kb_used_data":8516716,"kb_used_omap":404,"kb_used_meta":2582891,"kb_avail":738860816,"statfs":{"total":767960285184,"available":756593475584,"internally_reserved":0,"allocated":8721117184,"data_stored":8685335591,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":414022,"internal_metadata":2644881082},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-532611072,"num_objects":-8127,"num_object_clones":0,"num_object_copies":-24381,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1704,"num_objects_unfound":0,"num_objects_dirty":-8127,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8127,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":215,"num_bytes_recovered":14090240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003709"},"pg_stats":[{"pgid":"54.f","version":"1413'12951","reported_seq":13029,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:27.344454+0000","last_change":"2024-09-14T04:43:01.044742+0000","last_active":"2024-09-14T04:46:27.344454+0000","last_peered":"2024-09-14T04:46:27.344454+0000","last_clean":"2024-09-14T04:46:27.344454+0000","last_became_active":"2024-09-14T04:43:01.044421+0000","last_became_peered":"2024-09-14T04:43:01.044421+0000","last_unstale":"2024-09-14T04:46:27.344454+0000","last_undegraded":"2024-09-14T04:46:27.344454+0000","last_fullsized":"2024-09-14T04:46:27.344454+0000","mapping_epoch":1380,"log_start":"1397'6200","ondisk_log_start":"1397'6200","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":6751,"log_dups_size":0,"ondisk_log_size":6751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:28.897426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175439872,"num_objects":2677,"num_object_clones":0,"num_object_copies":8031,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2677,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12951,"num_write_kb":500096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"54.e","version":"1414'13242","reported_seq":16675,"reported_epoch":1414,"state":"active+recovering+remapped","last_fresh":"2024-09-14T04:46:30.745917+0000","last_change":"2024-09-14T04:44:58.690021+0000","last_active":"2024-09-14T04:46:30.745917+0000","last_peered":"2024-09-14T04:46:30.745917+0000","last_clean":"2024-09-14T04:44:56.302285+0000","last_became_active":"2024-09-14T04:44:58.579754+0000","last_became_peered":"2024-09-14T04:44:58.579754+0000","last_unstale":"2024-09-14T04:46:30.745917+0000","last_undegraded":"2024-09-14T04:46:30.745917+0000","last_fullsized":"2024-09-14T04:46:30.745917+0000","mapping_epoch":1403,"log_start":"1394'5283","ondisk_log_start":"1394'5283","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7959,"log_dups_size":0,"ondisk_log_size":7959,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":167772160,"num_objects":2560,"num_object_clones":0,"num_object_copies":7680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5650,"num_objects_unfound":0,"num_objects_dirty":2560,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13242,"num_write_kb":505664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1676,"num_bytes_recovered":109772800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[1,2,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":-3090}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"54.d","version":"1413'13184","reported_seq":13228,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:28.212575+0000","last_change":"2024-09-14T04:43:01.050000+0000","last_active":"2024-09-14T04:46:28.212575+0000","last_peered":"2024-09-14T04:46:28.212575+0000","last_clean":"2024-09-14T04:46:28.212575+0000","last_became_active":"2024-09-14T04:43:01.049752+0000","last_became_peered":"2024-09-14T04:43:01.049752+0000","last_unstale":"2024-09-14T04:46:28.212575+0000","last_undegraded":"2024-09-14T04:46:28.212575+0000","last_fullsized":"2024-09-14T04:46:28.212575+0000","mapping_epoch":1380,"log_start":"1395'5600","ondisk_log_start":"1395'5600","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7584,"log_dups_size":0,"ondisk_log_size":7584,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:35:06.578523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176881688,"num_objects":2700,"num_object_clones":0,"num_object_copies":8100,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2700,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":13184,"num_write_kb":508225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"54.c","version":"1413'13044","reported_seq":13126,"reported_epoch":1413,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:46:27.292642+0000","last_change":"2024-09-14T04:44:58.455281+0000","last_active":"2024-09-14T04:46:27.292642+0000","last_peered":"2024-09-14T04:46:27.292642+0000","last_clean":"2024-09-14T04:44:56.285515+0000","last_became_active":"2024-09-14T04:44:58.445654+0000","last_became_peered":"2024-09-14T04:44:58.445654+0000","last_unstale":"2024-09-14T04:46:27.292642+0000","last_undegraded":"2024-09-14T04:46:27.292642+0000","last_fullsized":"2024-09-14T04:46:27.292642+0000","mapping_epoch":1403,"log_start":"1397'6300","ondisk_log_start":"1397'6300","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":6744,"log_dups_size":0,"ondisk_log_size":6744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178520064,"num_objects":2724,"num_object_clones":0,"num_object_copies":8172,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2724,"num_objects_unfound":0,"num_objects_dirty":2724,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13044,"num_write_kb":504576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[{"shards":"3,5,6","objects":2724}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"54.b","version":"1413'13047","reported_seq":13114,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:28.144879+0000","last_change":"2024-09-14T04:43:13.355592+0000","last_active":"2024-09-14T04:46:28.144879+0000","last_peered":"2024-09-14T04:46:28.144879+0000","last_clean":"2024-09-14T04:46:28.144879+0000","last_became_active":"2024-09-14T04:43:13.355130+0000","last_became_peered":"2024-09-14T04:43:13.355130+0000","last_unstale":"2024-09-14T04:46:28.144879+0000","last_undegraded":"2024-09-14T04:46:28.144879+0000","last_fullsized":"2024-09-14T04:46:28.144879+0000","mapping_epoch":1386,"log_start":"1393'4700","ondisk_log_start":"1393'4700","created":1380,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":8347,"log_dups_size":0,"ondisk_log_size":8347,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:42:44.062290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174260224,"num_objects":2659,"num_object_clones":0,"num_object_copies":7977,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2659,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13047,"num_write_kb":502592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"54.a","version":"1413'12953","reported_seq":12996,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:28.213028+0000","last_change":"2024-09-14T04:43:01.043585+0000","last_active":"2024-09-14T04:46:28.213028+0000","last_peered":"2024-09-14T04:46:28.213028+0000","last_clean":"2024-09-14T04:46:28.213028+0000","last_became_active":"2024-09-14T04:43:01.043295+0000","last_became_peered":"2024-09-14T04:43:01.043295+0000","last_unstale":"2024-09-14T04:46:28.213028+0000","last_undegraded":"2024-09-14T04:46:28.213028+0000","last_fullsized":"2024-09-14T04:46:28.213028+0000","mapping_epoch":1380,"log_start":"1395'5400","ondisk_log_start":"1395'5400","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7553,"log_dups_size":0,"ondisk_log_size":7553,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:58:35.934308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176226304,"num_objects":2689,"num_object_clones":0,"num_object_copies":8067,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2689,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12953,"num_write_kb":500544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"54.9","version":"1414'13243","reported_seq":13323,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:30.735792+0000","last_change":"2024-09-14T04:43:01.048728+0000","last_active":"2024-09-14T04:46:30.735792+0000","last_peered":"2024-09-14T04:46:30.735792+0000","last_clean":"2024-09-14T04:46:30.735792+0000","last_became_active":"2024-09-14T04:43:01.048381+0000","last_became_peered":"2024-09-14T04:43:01.048381+0000","last_unstale":"2024-09-14T04:46:30.735792+0000","last_undegraded":"2024-09-14T04:46:30.735792+0000","last_fullsized":"2024-09-14T04:46:30.735792+0000","mapping_epoch":1380,"log_start":"1394'5300","ondisk_log_start":"1394'5300","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7943,"log_dups_size":0,"ondisk_log_size":7943,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:47:27.860166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":169148416,"num_objects":2581,"num_object_clones":0,"num_object_copies":7743,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2581,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13243,"num_write_kb":506368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"54.8","version":"1413'13124","reported_seq":13204,"reported_epoch":1413,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:46:28.116520+0000","last_change":"2024-09-14T04:44:58.458770+0000","last_active":"2024-09-14T04:46:28.116520+0000","last_peered":"2024-09-14T04:46:28.116520+0000","last_clean":"2024-09-14T04:44:56.294675+0000","last_became_active":"2024-09-14T04:44:58.445646+0000","last_became_peered":"2024-09-14T04:44:58.445646+0000","last_unstale":"2024-09-14T04:46:28.116520+0000","last_undegraded":"2024-09-14T04:46:28.116520+0000","last_fullsized":"2024-09-14T04:46:28.116520+0000","mapping_epoch":1403,"log_start":"1393'4700","ondisk_log_start":"1393'4700","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":8424,"log_dups_size":0,"ondisk_log_size":8424,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178520064,"num_objects":2724,"num_object_clones":0,"num_object_copies":8172,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2724,"num_objects_unfound":0,"num_objects_dirty":2724,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13124,"num_write_kb":507136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[0,3,7],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3,7","objects":2724}],"blocked_by":[],"up_primary":4,"acting_primary":0,"purged_snaps":[]},{"pgid":"54.7","version":"1413'13039","reported_seq":13117,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:27.347696+0000","last_change":"2024-09-14T04:43:01.050691+0000","last_active":"2024-09-14T04:46:27.347696+0000","last_peered":"2024-09-14T04:46:27.347696+0000","last_clean":"2024-09-14T04:46:27.347696+0000","last_became_active":"2024-09-14T04:43:01.050334+0000","last_became_peered":"2024-09-14T04:43:01.050334+0000","last_unstale":"2024-09-14T04:46:27.347696+0000","last_undegraded":"2024-09-14T04:46:27.347696+0000","last_fullsized":"2024-09-14T04:46:27.347696+0000","mapping_epoch":1380,"log_start":"1397'6300","ondisk_log_start":"1397'6300","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":6739,"log_dups_size":0,"ondisk_log_size":6739,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:22:30.806651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181862400,"num_objects":2775,"num_object_clones":0,"num_object_copies":8325,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2775,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13039,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"54.6","version":"1413'13083","reported_seq":13126,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:27.958955+0000","last_change":"2024-09-14T04:43:01.043477+0000","last_active":"2024-09-14T04:46:27.958955+0000","last_peered":"2024-09-14T04:46:27.958955+0000","last_clean":"2024-09-14T04:46:27.958955+0000","last_became_active":"2024-09-14T04:43:01.043136+0000","last_became_peered":"2024-09-14T04:43:01.043136+0000","last_unstale":"2024-09-14T04:46:27.958955+0000","last_undegraded":"2024-09-14T04:46:27.958955+0000","last_fullsized":"2024-09-14T04:46:27.958955+0000","mapping_epoch":1380,"log_start":"1395'5700","ondisk_log_start":"1395'5700","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7383,"log_dups_size":0,"ondisk_log_size":7383,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:00:57.211271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182124544,"num_objects":2779,"num_object_clones":0,"num_object_copies":8337,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2779,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13083,"num_write_kb":507584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"54.5","version":"1413'13060","reported_seq":13103,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:28.249673+0000","last_change":"2024-09-14T04:43:01.048975+0000","last_active":"2024-09-14T04:46:28.249673+0000","last_peered":"2024-09-14T04:46:28.249673+0000","last_clean":"2024-09-14T04:46:28.249673+0000","last_became_active":"2024-09-14T04:43:01.048677+0000","last_became_peered":"2024-09-14T04:43:01.048677+0000","last_unstale":"2024-09-14T04:46:28.249673+0000","last_undegraded":"2024-09-14T04:46:28.249673+0000","last_fullsized":"2024-09-14T04:46:28.249673+0000","mapping_epoch":1380,"log_start":"1395'5500","ondisk_log_start":"1395'5500","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7560,"log_dups_size":0,"ondisk_log_size":7560,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:58:04.849606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181272576,"num_objects":2766,"num_object_clones":0,"num_object_copies":8298,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2766,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13060,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"54.4","version":"1413'13093","reported_seq":13170,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:27.351075+0000","last_change":"2024-09-14T04:43:01.043603+0000","last_active":"2024-09-14T04:46:27.351075+0000","last_peered":"2024-09-14T04:46:27.351075+0000","last_clean":"2024-09-14T04:46:27.351075+0000","last_became_active":"2024-09-14T04:43:01.042757+0000","last_became_peered":"2024-09-14T04:43:01.042757+0000","last_unstale":"2024-09-14T04:46:27.351075+0000","last_undegraded":"2024-09-14T04:46:27.351075+0000","last_fullsized":"2024-09-14T04:46:27.351075+0000","mapping_epoch":1380,"log_start":"1397'6400","ondisk_log_start":"1397'6400","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":6693,"log_dups_size":0,"ondisk_log_size":6693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:46:20.719227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180813824,"num_objects":2759,"num_object_clones":0,"num_object_copies":8277,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2759,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13093,"num_write_kb":507264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"54.3","version":"1414'13352","reported_seq":13432,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:30.724996+0000","last_change":"2024-09-14T04:43:01.048551+0000","last_active":"2024-09-14T04:46:30.724996+0000","last_peered":"2024-09-14T04:46:30.724996+0000","last_clean":"2024-09-14T04:46:30.724996+0000","last_became_active":"2024-09-14T04:43:01.048218+0000","last_became_peered":"2024-09-14T04:43:01.048218+0000","last_unstale":"2024-09-14T04:46:30.724996+0000","last_undegraded":"2024-09-14T04:46:30.724996+0000","last_fullsized":"2024-09-14T04:46:30.724996+0000","mapping_epoch":1380,"log_start":"1394'5400","ondisk_log_start":"1394'5400","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7952,"log_dups_size":0,"ondisk_log_size":7952,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:27:10.233265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177602560,"num_objects":2710,"num_object_clones":0,"num_object_copies":8130,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2710,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13352,"num_write_kb":513984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"54.2","version":"1413'13041","reported_seq":13119,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:28.164492+0000","last_change":"2024-09-14T04:43:01.049088+0000","last_active":"2024-09-14T04:46:28.164492+0000","last_peered":"2024-09-14T04:46:28.164492+0000","last_clean":"2024-09-14T04:46:28.164492+0000","last_became_active":"2024-09-14T04:43:01.048764+0000","last_became_peered":"2024-09-14T04:43:01.048764+0000","last_unstale":"2024-09-14T04:46:28.164492+0000","last_undegraded":"2024-09-14T04:46:28.164492+0000","last_fullsized":"2024-09-14T04:46:28.164492+0000","mapping_epoch":1380,"log_start":"1393'4700","ondisk_log_start":"1393'4700","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":8341,"log_dups_size":0,"ondisk_log_size":8341,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:03:50.532719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178061312,"num_objects":2717,"num_object_clones":0,"num_object_copies":8151,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2717,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13041,"num_write_kb":504256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"54.1","version":"1413'13056","reported_seq":13099,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:28.136903+0000","last_change":"2024-09-14T04:43:01.047658+0000","last_active":"2024-09-14T04:46:28.136903+0000","last_peered":"2024-09-14T04:46:28.136903+0000","last_clean":"2024-09-14T04:46:28.136903+0000","last_became_active":"2024-09-14T04:43:01.047401+0000","last_became_peered":"2024-09-14T04:43:01.047401+0000","last_unstale":"2024-09-14T04:46:28.136903+0000","last_undegraded":"2024-09-14T04:46:28.136903+0000","last_fullsized":"2024-09-14T04:46:28.136903+0000","mapping_epoch":1380,"log_start":"1392'4400","ondisk_log_start":"1392'4400","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":8656,"log_dups_size":0,"ondisk_log_size":8656,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:02:05.987375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185204736,"num_objects":2826,"num_object_clones":0,"num_object_copies":8478,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2826,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13056,"num_write_kb":508224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"54.0","version":"1413'13006","reported_seq":13126,"reported_epoch":1413,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:46:28.161086+0000","last_change":"2024-09-14T04:44:58.640486+0000","last_active":"2024-09-14T04:46:28.161086+0000","last_peered":"2024-09-14T04:46:28.161086+0000","last_clean":"2024-09-14T04:44:56.291188+0000","last_became_active":"2024-09-14T04:44:58.441092+0000","last_became_peered":"2024-09-14T04:44:58.441092+0000","last_unstale":"2024-09-14T04:46:28.161086+0000","last_undegraded":"2024-09-14T04:46:28.161086+0000","last_fullsized":"2024-09-14T04:46:28.161086+0000","mapping_epoch":1403,"log_start":"1395'5500","ondisk_log_start":"1395'5500","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7506,"log_dups_size":0,"ondisk_log_size":7506,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:13:27.337351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":172097536,"num_objects":2626,"num_object_clones":0,"num_object_copies":7878,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2616,"num_objects_unfound":0,"num_objects_dirty":2626,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13006,"num_write_kb":500224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":2097152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,3,5],"avail_no_missing":["6","3","5"],"object_location_counts":[{"shards":"3,5,6","objects":2626}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":2114,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.394282+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T04:46:02.394282+0000","last_peered":"2024-09-14T04:46:02.394282+0000","last_clean":"2024-09-14T04:46:02.394282+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T04:46:02.394282+0000","last_undegraded":"2024-09-14T04:46:02.394282+0000","last_fullsized":"2024-09-14T04:46:02.394282+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":21,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.879280+0000","last_change":"2024-09-14T04:44:57.325109+0000","last_active":"2024-09-14T04:45:37.879280+0000","last_peered":"2024-09-14T04:45:37.879280+0000","last_clean":"2024-09-14T04:45:37.879280+0000","last_became_active":"2024-09-14T04:44:57.324791+0000","last_became_peered":"2024-09-14T04:44:57.324791+0000","last_unstale":"2024-09-14T04:45:37.879280+0000","last_undegraded":"2024-09-14T04:45:37.879280+0000","last_fullsized":"2024-09-14T04:45:37.879280+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:03.593275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":2104,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.381176+0000","last_change":"2024-09-14T04:07:58.504829+0000","last_active":"2024-09-14T04:46:02.381176+0000","last_peered":"2024-09-14T04:46:02.381176+0000","last_clean":"2024-09-14T04:46:02.381176+0000","last_became_active":"2024-09-14T04:07:58.502623+0000","last_became_peered":"2024-09-14T04:07:58.502623+0000","last_unstale":"2024-09-14T04:46:02.381176+0000","last_undegraded":"2024-09-14T04:46:02.381176+0000","last_fullsized":"2024-09-14T04:46:02.381176+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:59.091993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":738,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.560228+0000","last_change":"2024-09-14T04:43:13.355281+0000","last_active":"2024-09-14T04:46:29.560228+0000","last_peered":"2024-09-14T04:46:29.560228+0000","last_clean":"2024-09-14T04:46:29.560228+0000","last_became_active":"2024-09-14T04:43:13.354967+0000","last_became_peered":"2024-09-14T04:43:13.354967+0000","last_unstale":"2024-09-14T04:46:29.560228+0000","last_undegraded":"2024-09-14T04:46:29.560228+0000","last_fullsized":"2024-09-14T04:46:29.560228+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:23:19.332075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1400,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:38.178432+0000","last_change":"2024-09-14T04:44:57.342918+0000","last_active":"2024-09-14T04:45:38.178432+0000","last_peered":"2024-09-14T04:45:38.178432+0000","last_clean":"2024-09-14T04:45:38.178432+0000","last_became_active":"2024-09-14T04:44:57.329550+0000","last_became_peered":"2024-09-14T04:44:57.329550+0000","last_unstale":"2024-09-14T04:45:38.178432+0000","last_undegraded":"2024-09-14T04:45:38.178432+0000","last_fullsized":"2024-09-14T04:45:38.178432+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:52:52.435893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":21,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.877060+0000","last_change":"2024-09-14T04:44:57.325537+0000","last_active":"2024-09-14T04:45:37.877060+0000","last_peered":"2024-09-14T04:45:37.877060+0000","last_clean":"2024-09-14T04:45:37.877060+0000","last_became_active":"2024-09-14T04:44:57.325057+0000","last_became_peered":"2024-09-14T04:44:57.325057+0000","last_unstale":"2024-09-14T04:45:37.877060+0000","last_undegraded":"2024-09-14T04:45:37.877060+0000","last_fullsized":"2024-09-14T04:45:37.877060+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:43:23.166708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1415,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:38.178401+0000","last_change":"2024-09-14T04:44:57.337033+0000","last_active":"2024-09-14T04:45:38.178401+0000","last_peered":"2024-09-14T04:45:38.178401+0000","last_clean":"2024-09-14T04:45:38.178401+0000","last_became_active":"2024-09-14T04:44:57.328437+0000","last_became_peered":"2024-09-14T04:44:57.328437+0000","last_unstale":"2024-09-14T04:45:38.178401+0000","last_undegraded":"2024-09-14T04:45:38.178401+0000","last_fullsized":"2024-09-14T04:45:38.178401+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.109086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":21,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.876897+0000","last_change":"2024-09-14T04:44:57.332243+0000","last_active":"2024-09-14T04:45:37.876897+0000","last_peered":"2024-09-14T04:45:37.876897+0000","last_clean":"2024-09-14T04:45:37.876897+0000","last_became_active":"2024-09-14T04:44:57.330413+0000","last_became_peered":"2024-09-14T04:44:57.330413+0000","last_unstale":"2024-09-14T04:45:37.876897+0000","last_undegraded":"2024-09-14T04:45:37.876897+0000","last_fullsized":"2024-09-14T04:45:37.876897+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:25:01.338549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":2121,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.384362+0000","last_change":"2024-09-14T04:44:57.333097+0000","last_active":"2024-09-14T04:46:02.384362+0000","last_peered":"2024-09-14T04:46:02.384362+0000","last_clean":"2024-09-14T04:46:02.384362+0000","last_became_active":"2024-09-14T04:44:57.332294+0000","last_became_peered":"2024-09-14T04:44:57.332294+0000","last_unstale":"2024-09-14T04:46:02.384362+0000","last_undegraded":"2024-09-14T04:46:02.384362+0000","last_fullsized":"2024-09-14T04:46:02.384362+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:57:44.426280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2112,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.559407+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T04:46:29.559407+0000","last_peered":"2024-09-14T04:46:29.559407+0000","last_clean":"2024-09-14T04:46:29.559407+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T04:46:29.559407+0000","last_undegraded":"2024-09-14T04:46:29.559407+0000","last_fullsized":"2024-09-14T04:46:29.559407+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":21,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.876987+0000","last_change":"2024-09-14T04:44:57.325392+0000","last_active":"2024-09-14T04:45:37.876987+0000","last_peered":"2024-09-14T04:45:37.876987+0000","last_clean":"2024-09-14T04:45:37.876987+0000","last_became_active":"2024-09-14T04:44:57.324624+0000","last_became_peered":"2024-09-14T04:44:57.324624+0000","last_unstale":"2024-09-14T04:45:37.876987+0000","last_undegraded":"2024-09-14T04:45:37.876987+0000","last_fullsized":"2024-09-14T04:45:37.876987+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:33:18.810199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2081,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.550404+0000","last_change":"2024-09-14T04:44:57.329113+0000","last_active":"2024-09-14T04:46:29.550404+0000","last_peered":"2024-09-14T04:46:29.550404+0000","last_clean":"2024-09-14T04:46:29.550404+0000","last_became_active":"2024-09-14T04:44:57.328727+0000","last_became_peered":"2024-09-14T04:44:57.328727+0000","last_unstale":"2024-09-14T04:46:29.550404+0000","last_undegraded":"2024-09-14T04:46:29.550404+0000","last_fullsized":"2024-09-14T04:46:29.550404+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:34.939874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":2098,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.550082+0000","last_change":"2024-09-14T04:44:57.329277+0000","last_active":"2024-09-14T04:46:29.550082+0000","last_peered":"2024-09-14T04:46:29.550082+0000","last_clean":"2024-09-14T04:46:29.550082+0000","last_became_active":"2024-09-14T04:44:57.328804+0000","last_became_peered":"2024-09-14T04:44:57.328804+0000","last_unstale":"2024-09-14T04:46:29.550082+0000","last_undegraded":"2024-09-14T04:46:29.550082+0000","last_fullsized":"2024-09-14T04:46:29.550082+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:42:46.055240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2109,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.379346+0000","last_change":"2024-09-14T04:07:58.561490+0000","last_active":"2024-09-14T04:46:02.379346+0000","last_peered":"2024-09-14T04:46:02.379346+0000","last_clean":"2024-09-14T04:46:02.379346+0000","last_became_active":"2024-09-14T04:07:58.559841+0000","last_became_peered":"2024-09-14T04:07:58.559841+0000","last_unstale":"2024-09-14T04:46:02.379346+0000","last_undegraded":"2024-09-14T04:46:02.379346+0000","last_fullsized":"2024-09-14T04:46:02.379346+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:27:03.795368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2092,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.379893+0000","last_change":"2024-09-14T04:43:13.357092+0000","last_active":"2024-09-14T04:46:02.379893+0000","last_peered":"2024-09-14T04:46:02.379893+0000","last_clean":"2024-09-14T04:46:02.379893+0000","last_became_active":"2024-09-14T04:43:13.356694+0000","last_became_peered":"2024-09-14T04:43:13.356694+0000","last_unstale":"2024-09-14T04:46:02.379893+0000","last_undegraded":"2024-09-14T04:46:02.379893+0000","last_fullsized":"2024-09-14T04:46:02.379893+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:11.167574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1392,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.882083+0000","last_change":"2024-09-14T04:07:21.720797+0000","last_active":"2024-09-14T04:45:37.882083+0000","last_peered":"2024-09-14T04:45:37.882083+0000","last_clean":"2024-09-14T04:45:37.882083+0000","last_became_active":"2024-09-14T04:07:21.719331+0000","last_became_peered":"2024-09-14T04:07:21.719331+0000","last_unstale":"2024-09-14T04:45:37.882083+0000","last_undegraded":"2024-09-14T04:45:37.882083+0000","last_fullsized":"2024-09-14T04:45:37.882083+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:35.404023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":40,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.885722+0000","last_change":"2024-09-14T04:43:11.641514+0000","last_active":"2024-09-14T04:45:37.885722+0000","last_peered":"2024-09-14T04:45:37.885722+0000","last_clean":"2024-09-14T04:45:37.885722+0000","last_became_active":"2024-09-14T04:43:11.641151+0000","last_became_peered":"2024-09-14T04:43:11.641151+0000","last_unstale":"2024-09-14T04:45:37.885722+0000","last_undegraded":"2024-09-14T04:45:37.885722+0000","last_fullsized":"2024-09-14T04:45:37.885722+0000","mapping_epoch":1384,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:06:45.045430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2103,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.394453+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T04:46:02.394453+0000","last_peered":"2024-09-14T04:46:02.394453+0000","last_clean":"2024-09-14T04:46:02.394453+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T04:46:02.394453+0000","last_undegraded":"2024-09-14T04:46:02.394453+0000","last_fullsized":"2024-09-14T04:46:02.394453+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":2085,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.390569+0000","last_change":"2024-09-14T04:07:58.507682+0000","last_active":"2024-09-14T04:46:02.390569+0000","last_peered":"2024-09-14T04:46:02.390569+0000","last_clean":"2024-09-14T04:46:02.390569+0000","last_became_active":"2024-09-14T04:07:58.507365+0000","last_became_peered":"2024-09-14T04:07:58.507365+0000","last_unstale":"2024-09-14T04:46:02.390569+0000","last_undegraded":"2024-09-14T04:46:02.390569+0000","last_fullsized":"2024-09-14T04:46:02.390569+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:14:47.294950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1399,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.373597+0000","last_change":"2024-09-14T04:42:49.938505+0000","last_active":"2024-09-14T04:46:02.373597+0000","last_peered":"2024-09-14T04:46:02.373597+0000","last_clean":"2024-09-14T04:46:02.373597+0000","last_became_active":"2024-09-14T04:42:49.938180+0000","last_became_peered":"2024-09-14T04:42:49.938180+0000","last_unstale":"2024-09-14T04:46:02.373597+0000","last_undegraded":"2024-09-14T04:46:02.373597+0000","last_fullsized":"2024-09-14T04:46:02.373597+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:05.929479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1418,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.374844+0000","last_change":"2024-09-14T04:44:57.346582+0000","last_active":"2024-09-14T04:46:02.374844+0000","last_peered":"2024-09-14T04:46:02.374844+0000","last_clean":"2024-09-14T04:46:02.374844+0000","last_became_active":"2024-09-14T04:44:57.330446+0000","last_became_peered":"2024-09-14T04:44:57.330446+0000","last_unstale":"2024-09-14T04:46:02.374844+0000","last_undegraded":"2024-09-14T04:46:02.374844+0000","last_fullsized":"2024-09-14T04:46:02.374844+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:29.238585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":46,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.549773+0000","last_change":"2024-09-14T04:44:14.480667+0000","last_active":"2024-09-14T04:46:29.549773+0000","last_peered":"2024-09-14T04:46:29.549773+0000","last_clean":"2024-09-14T04:46:29.549773+0000","last_became_active":"2024-09-14T04:44:14.480365+0000","last_became_peered":"2024-09-14T04:44:14.480365+0000","last_unstale":"2024-09-14T04:46:29.549773+0000","last_undegraded":"2024-09-14T04:46:29.549773+0000","last_fullsized":"2024-09-14T04:46:29.549773+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:00.611771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":31,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.877334+0000","last_change":"2024-09-14T04:44:57.341426+0000","last_active":"2024-09-14T04:45:37.877334+0000","last_peered":"2024-09-14T04:45:37.877334+0000","last_clean":"2024-09-14T04:45:37.877334+0000","last_became_active":"2024-09-14T04:44:57.328763+0000","last_became_peered":"2024-09-14T04:44:57.328763+0000","last_unstale":"2024-09-14T04:45:37.877334+0000","last_undegraded":"2024-09-14T04:45:37.877334+0000","last_fullsized":"2024-09-14T04:45:37.877334+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:34:40.066761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":21,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.877056+0000","last_change":"2024-09-14T04:44:57.331337+0000","last_active":"2024-09-14T04:45:37.877056+0000","last_peered":"2024-09-14T04:45:37.877056+0000","last_clean":"2024-09-14T04:45:37.877056+0000","last_became_active":"2024-09-14T04:44:57.329336+0000","last_became_peered":"2024-09-14T04:44:57.329336+0000","last_unstale":"2024-09-14T04:45:37.877056+0000","last_undegraded":"2024-09-14T04:45:37.877056+0000","last_fullsized":"2024-09-14T04:45:37.877056+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:52:35.980869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"349'149","reported_seq":773,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.393762+0000","last_change":"2024-09-14T04:41:37.755174+0000","last_active":"2024-09-14T04:46:02.393762+0000","last_peered":"2024-09-14T04:46:02.393762+0000","last_clean":"2024-09-14T04:46:02.393762+0000","last_became_active":"2024-09-14T04:41:36.930263+0000","last_became_peered":"2024-09-14T04:41:36.930263+0000","last_unstale":"2024-09-14T04:46:02.393762+0000","last_undegraded":"2024-09-14T04:46:02.393762+0000","last_fullsized":"2024-09-14T04:46:02.393762+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.755105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.755105+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:27:03.161713+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":21,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.876707+0000","last_change":"2024-09-14T04:44:57.329870+0000","last_active":"2024-09-14T04:45:37.876707+0000","last_peered":"2024-09-14T04:45:37.876707+0000","last_clean":"2024-09-14T04:45:37.876707+0000","last_became_active":"2024-09-14T04:44:57.329353+0000","last_became_peered":"2024-09-14T04:44:57.329353+0000","last_unstale":"2024-09-14T04:45:37.876707+0000","last_undegraded":"2024-09-14T04:45:37.876707+0000","last_fullsized":"2024-09-14T04:45:37.876707+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:46.432181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1396,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.372666+0000","last_change":"2024-09-14T04:07:21.721242+0000","last_active":"2024-09-14T04:46:02.372666+0000","last_peered":"2024-09-14T04:46:02.372666+0000","last_clean":"2024-09-14T04:46:02.372666+0000","last_became_active":"2024-09-14T04:07:21.720084+0000","last_became_peered":"2024-09-14T04:07:21.720084+0000","last_unstale":"2024-09-14T04:46:02.372666+0000","last_undegraded":"2024-09-14T04:46:02.372666+0000","last_fullsized":"2024-09-14T04:46:02.372666+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:58:41.308643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":2141,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.379228+0000","last_change":"2024-09-14T04:44:57.344614+0000","last_active":"2024-09-14T04:46:02.379228+0000","last_peered":"2024-09-14T04:46:02.379228+0000","last_clean":"2024-09-14T04:46:02.379228+0000","last_became_active":"2024-09-14T04:44:57.332256+0000","last_became_peered":"2024-09-14T04:44:57.332256+0000","last_unstale":"2024-09-14T04:46:02.379228+0000","last_undegraded":"2024-09-14T04:46:02.379228+0000","last_fullsized":"2024-09-14T04:46:02.379228+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:23.438417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"349'149","reported_seq":779,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.877141+0000","last_change":"2024-09-14T04:45:00.331854+0000","last_active":"2024-09-14T04:45:37.877141+0000","last_peered":"2024-09-14T04:45:37.877141+0000","last_clean":"2024-09-14T04:45:37.877141+0000","last_became_active":"2024-09-14T04:45:00.331460+0000","last_became_peered":"2024-09-14T04:45:00.331460+0000","last_unstale":"2024-09-14T04:45:37.877141+0000","last_undegraded":"2024-09-14T04:45:37.877141+0000","last_fullsized":"2024-09-14T04:45:37.877141+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.567535+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.567535+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:45.095729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":2078,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.381680+0000","last_change":"2024-09-14T04:07:58.555446+0000","last_active":"2024-09-14T04:46:02.381680+0000","last_peered":"2024-09-14T04:46:02.381680+0000","last_clean":"2024-09-14T04:46:02.381680+0000","last_became_active":"2024-09-14T04:07:58.548355+0000","last_became_peered":"2024-09-14T04:07:58.548355+0000","last_unstale":"2024-09-14T04:46:02.381680+0000","last_undegraded":"2024-09-14T04:46:02.381680+0000","last_fullsized":"2024-09-14T04:46:02.381680+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:56:28.426094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":98,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.562342+0000","last_change":"2024-09-14T04:44:57.330746+0000","last_active":"2024-09-14T04:46:29.562342+0000","last_peered":"2024-09-14T04:46:29.562342+0000","last_clean":"2024-09-14T04:46:29.562342+0000","last_became_active":"2024-09-14T04:44:57.330327+0000","last_became_peered":"2024-09-14T04:44:57.330327+0000","last_unstale":"2024-09-14T04:46:29.562342+0000","last_undegraded":"2024-09-14T04:46:29.562342+0000","last_fullsized":"2024-09-14T04:46:29.562342+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:53.122771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":61,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.888504+0000","last_change":"2024-09-14T04:43:13.355296+0000","last_active":"2024-09-14T04:45:37.888504+0000","last_peered":"2024-09-14T04:45:37.888504+0000","last_clean":"2024-09-14T04:45:37.888504+0000","last_became_active":"2024-09-14T04:43:13.354950+0000","last_became_peered":"2024-09-14T04:43:13.354950+0000","last_unstale":"2024-09-14T04:45:37.888504+0000","last_undegraded":"2024-09-14T04:45:37.888504+0000","last_fullsized":"2024-09-14T04:45:37.888504+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:20:08.968818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"349'149","reported_seq":718,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:58.211411+0000","last_change":"2024-09-14T04:41:39.471393+0000","last_active":"2024-09-14T04:45:58.211411+0000","last_peered":"2024-09-14T04:45:58.211411+0000","last_clean":"2024-09-14T04:45:58.211411+0000","last_became_active":"2024-09-14T04:41:36.931768+0000","last_became_peered":"2024-09-14T04:41:36.931768+0000","last_unstale":"2024-09-14T04:45:58.211411+0000","last_undegraded":"2024-09-14T04:45:58.211411+0000","last_fullsized":"2024-09-14T04:45:58.211411+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.471248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.471248+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:56.210375+0000","scrub_duration":0.021758344999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2097,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.384515+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T04:46:02.384515+0000","last_peered":"2024-09-14T04:46:02.384515+0000","last_clean":"2024-09-14T04:46:02.384515+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T04:46:02.384515+0000","last_undegraded":"2024-09-14T04:46:02.384515+0000","last_fullsized":"2024-09-14T04:46:02.384515+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1428,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.374985+0000","last_change":"2024-09-14T04:44:57.331436+0000","last_active":"2024-09-14T04:46:02.374985+0000","last_peered":"2024-09-14T04:46:02.374985+0000","last_clean":"2024-09-14T04:46:02.374985+0000","last_became_active":"2024-09-14T04:44:57.330957+0000","last_became_peered":"2024-09-14T04:44:57.330957+0000","last_unstale":"2024-09-14T04:46:02.374985+0000","last_undegraded":"2024-09-14T04:46:02.374985+0000","last_fullsized":"2024-09-14T04:46:02.374985+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:09.516688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":91,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.550369+0000","last_change":"2024-09-14T04:42:49.940968+0000","last_active":"2024-09-14T04:46:29.550369+0000","last_peered":"2024-09-14T04:46:29.550369+0000","last_clean":"2024-09-14T04:46:29.550369+0000","last_became_active":"2024-09-14T04:42:49.940667+0000","last_became_peered":"2024-09-14T04:42:49.940667+0000","last_unstale":"2024-09-14T04:46:29.550369+0000","last_undegraded":"2024-09-14T04:46:29.550369+0000","last_fullsized":"2024-09-14T04:46:29.550369+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:52:49.623815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"349'149","reported_seq":718,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:58.211432+0000","last_change":"2024-09-14T04:41:38.479980+0000","last_active":"2024-09-14T04:45:58.211432+0000","last_peered":"2024-09-14T04:45:58.211432+0000","last_clean":"2024-09-14T04:45:58.211432+0000","last_became_active":"2024-09-14T04:41:36.933392+0000","last_became_peered":"2024-09-14T04:41:36.933392+0000","last_unstale":"2024-09-14T04:45:58.211432+0000","last_undegraded":"2024-09-14T04:45:58.211432+0000","last_fullsized":"2024-09-14T04:45:58.211432+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:38.479887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:38.479887+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:01:26.102279+0000","scrub_duration":0.0062728319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":54,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.549691+0000","last_change":"2024-09-14T04:44:57.329983+0000","last_active":"2024-09-14T04:46:29.549691+0000","last_peered":"2024-09-14T04:46:29.549691+0000","last_clean":"2024-09-14T04:46:29.549691+0000","last_became_active":"2024-09-14T04:44:57.329597+0000","last_became_peered":"2024-09-14T04:44:57.329597+0000","last_unstale":"2024-09-14T04:46:29.549691+0000","last_undegraded":"2024-09-14T04:46:29.549691+0000","last_fullsized":"2024-09-14T04:46:29.549691+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:45.718039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":2134,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.550594+0000","last_change":"2024-09-14T04:06:46.930482+0000","last_active":"2024-09-14T04:46:29.550594+0000","last_peered":"2024-09-14T04:46:29.550594+0000","last_clean":"2024-09-14T04:46:29.550594+0000","last_became_active":"2024-09-14T04:06:46.927372+0000","last_became_peered":"2024-09-14T04:06:46.927372+0000","last_unstale":"2024-09-14T04:46:29.550594+0000","last_undegraded":"2024-09-14T04:46:29.550594+0000","last_fullsized":"2024-09-14T04:46:29.550594+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:07:22.159128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1400,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.886105+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T04:45:37.886105+0000","last_peered":"2024-09-14T04:45:37.886105+0000","last_clean":"2024-09-14T04:45:37.886105+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T04:45:37.886105+0000","last_undegraded":"2024-09-14T04:45:37.886105+0000","last_fullsized":"2024-09-14T04:45:37.886105+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"349'149","reported_seq":710,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:48.188120+0000","last_change":"2024-09-14T04:41:39.012977+0000","last_active":"2024-09-14T04:45:48.188120+0000","last_peered":"2024-09-14T04:45:48.188120+0000","last_clean":"2024-09-14T04:45:48.188120+0000","last_became_active":"2024-09-14T04:41:36.932552+0000","last_became_peered":"2024-09-14T04:41:36.932552+0000","last_unstale":"2024-09-14T04:45:48.188120+0000","last_undegraded":"2024-09-14T04:45:48.188120+0000","last_fullsized":"2024-09-14T04:45:48.188120+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.012879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.012879+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:16:40.080262+0000","scrub_duration":0.0051721620000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":91,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.550281+0000","last_change":"2024-09-14T04:42:49.945069+0000","last_active":"2024-09-14T04:46:29.550281+0000","last_peered":"2024-09-14T04:46:29.550281+0000","last_clean":"2024-09-14T04:46:29.550281+0000","last_became_active":"2024-09-14T04:42:49.943527+0000","last_became_peered":"2024-09-14T04:42:49.943527+0000","last_unstale":"2024-09-14T04:46:29.550281+0000","last_undegraded":"2024-09-14T04:46:29.550281+0000","last_fullsized":"2024-09-14T04:46:29.550281+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:39:10.524631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":21,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.876983+0000","last_change":"2024-09-14T04:44:57.329652+0000","last_active":"2024-09-14T04:45:37.876983+0000","last_peered":"2024-09-14T04:45:37.876983+0000","last_clean":"2024-09-14T04:45:37.876983+0000","last_became_active":"2024-09-14T04:44:57.328888+0000","last_became_peered":"2024-09-14T04:44:57.328888+0000","last_unstale":"2024-09-14T04:45:37.876983+0000","last_undegraded":"2024-09-14T04:45:37.876983+0000","last_fullsized":"2024-09-14T04:45:37.876983+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:10.170068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1399,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.373724+0000","last_change":"2024-09-14T04:06:46.875340+0000","last_active":"2024-09-14T04:46:02.373724+0000","last_peered":"2024-09-14T04:46:02.373724+0000","last_clean":"2024-09-14T04:46:02.373724+0000","last_became_active":"2024-09-14T04:06:46.874552+0000","last_became_peered":"2024-09-14T04:06:46.874552+0000","last_unstale":"2024-09-14T04:46:02.373724+0000","last_undegraded":"2024-09-14T04:46:02.373724+0000","last_fullsized":"2024-09-14T04:46:02.373724+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:52.257357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"349'149","reported_seq":1780,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:42.962508+0000","last_change":"2024-09-14T04:45:00.346627+0000","last_active":"2024-09-14T04:45:42.962508+0000","last_peered":"2024-09-14T04:45:42.962508+0000","last_clean":"2024-09-14T04:45:42.962508+0000","last_became_active":"2024-09-14T04:45:00.346286+0000","last_became_peered":"2024-09-14T04:45:00.346286+0000","last_unstale":"2024-09-14T04:45:42.962508+0000","last_undegraded":"2024-09-14T04:45:42.962508+0000","last_fullsized":"2024-09-14T04:45:42.962508+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:50.784648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:50.784648+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:07:58.441372+0000","scrub_duration":0.01141873,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":47,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.550582+0000","last_change":"2024-09-14T04:44:14.489235+0000","last_active":"2024-09-14T04:46:29.550582+0000","last_peered":"2024-09-14T04:46:29.550582+0000","last_clean":"2024-09-14T04:46:29.550582+0000","last_became_active":"2024-09-14T04:44:14.488950+0000","last_became_peered":"2024-09-14T04:44:14.488950+0000","last_unstale":"2024-09-14T04:46:29.550582+0000","last_undegraded":"2024-09-14T04:46:29.550582+0000","last_fullsized":"2024-09-14T04:46:29.550582+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:54.441144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":2117,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.549538+0000","last_change":"2024-09-14T04:07:21.685415+0000","last_active":"2024-09-14T04:46:29.549538+0000","last_peered":"2024-09-14T04:46:29.549538+0000","last_clean":"2024-09-14T04:46:29.549538+0000","last_became_active":"2024-09-14T04:07:21.683504+0000","last_became_peered":"2024-09-14T04:07:21.683504+0000","last_unstale":"2024-09-14T04:46:29.549538+0000","last_undegraded":"2024-09-14T04:46:29.549538+0000","last_fullsized":"2024-09-14T04:46:29.549538+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:21:08.806291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2119,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.560315+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T04:46:29.560315+0000","last_peered":"2024-09-14T04:46:29.560315+0000","last_clean":"2024-09-14T04:46:29.560315+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T04:46:29.560315+0000","last_undegraded":"2024-09-14T04:46:29.560315+0000","last_fullsized":"2024-09-14T04:46:29.560315+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"349'149","reported_seq":795,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.550742+0000","last_change":"2024-09-14T04:45:00.342826+0000","last_active":"2024-09-14T04:46:29.550742+0000","last_peered":"2024-09-14T04:46:29.550742+0000","last_clean":"2024-09-14T04:46:29.550742+0000","last_became_active":"2024-09-14T04:45:00.342013+0000","last_became_peered":"2024-09-14T04:45:00.342013+0000","last_unstale":"2024-09-14T04:46:29.550742+0000","last_undegraded":"2024-09-14T04:46:29.550742+0000","last_fullsized":"2024-09-14T04:46:29.550742+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.326129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.326129+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:54:28.021603+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2118,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.379387+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T04:46:02.379387+0000","last_peered":"2024-09-14T04:46:02.379387+0000","last_clean":"2024-09-14T04:46:02.379387+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T04:46:02.379387+0000","last_undegraded":"2024-09-14T04:46:02.379387+0000","last_fullsized":"2024-09-14T04:46:02.379387+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2125,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.380035+0000","last_change":"2024-09-14T04:44:57.335783+0000","last_active":"2024-09-14T04:46:02.380035+0000","last_peered":"2024-09-14T04:46:02.380035+0000","last_clean":"2024-09-14T04:46:02.380035+0000","last_became_active":"2024-09-14T04:44:57.332780+0000","last_became_peered":"2024-09-14T04:44:57.332780+0000","last_unstale":"2024-09-14T04:46:02.380035+0000","last_undegraded":"2024-09-14T04:46:02.380035+0000","last_fullsized":"2024-09-14T04:46:02.380035+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:57.457593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1429,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:57.975686+0000","last_change":"2024-09-14T04:42:49.939790+0000","last_active":"2024-09-14T04:45:57.975686+0000","last_peered":"2024-09-14T04:45:57.975686+0000","last_clean":"2024-09-14T04:45:57.975686+0000","last_became_active":"2024-09-14T04:42:49.939591+0000","last_became_peered":"2024-09-14T04:42:49.939591+0000","last_unstale":"2024-09-14T04:45:57.975686+0000","last_undegraded":"2024-09-14T04:45:57.975686+0000","last_fullsized":"2024-09-14T04:45:57.975686+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:23:16.790593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1392,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.882043+0000","last_change":"2024-09-14T04:44:57.326008+0000","last_active":"2024-09-14T04:45:37.882043+0000","last_peered":"2024-09-14T04:45:37.882043+0000","last_clean":"2024-09-14T04:45:37.882043+0000","last_became_active":"2024-09-14T04:44:57.325617+0000","last_became_peered":"2024-09-14T04:44:57.325617+0000","last_unstale":"2024-09-14T04:45:37.882043+0000","last_undegraded":"2024-09-14T04:45:37.882043+0000","last_fullsized":"2024-09-14T04:45:37.882043+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:15.637508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":681,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.889221+0000","last_change":"2024-09-14T04:43:13.354846+0000","last_active":"2024-09-14T04:45:37.889221+0000","last_peered":"2024-09-14T04:45:37.889221+0000","last_clean":"2024-09-14T04:45:37.889221+0000","last_became_active":"2024-09-14T04:43:13.354515+0000","last_became_peered":"2024-09-14T04:43:13.354515+0000","last_unstale":"2024-09-14T04:45:37.889221+0000","last_undegraded":"2024-09-14T04:45:37.889221+0000","last_fullsized":"2024-09-14T04:45:37.889221+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:17.889341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1398,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.373634+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T04:46:02.373634+0000","last_peered":"2024-09-14T04:46:02.373634+0000","last_clean":"2024-09-14T04:46:02.373634+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T04:46:02.373634+0000","last_undegraded":"2024-09-14T04:46:02.373634+0000","last_fullsized":"2024-09-14T04:46:02.373634+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":1735,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.887692+0000","last_change":"2024-09-14T04:41:30.640651+0000","last_active":"2024-09-14T04:45:37.887692+0000","last_peered":"2024-09-14T04:45:37.887692+0000","last_clean":"2024-09-14T04:45:37.887692+0000","last_became_active":"2024-09-14T04:41:29.532466+0000","last_became_peered":"2024-09-14T04:41:29.532466+0000","last_unstale":"2024-09-14T04:45:37.887692+0000","last_undegraded":"2024-09-14T04:45:37.887692+0000","last_fullsized":"2024-09-14T04:45:37.887692+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":0,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:30.640520+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:30.640520+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:27.776724+0000","scrub_duration":0.0028653139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"349'149","reported_seq":710,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:58.214933+0000","last_change":"2024-09-14T04:41:39.978080+0000","last_active":"2024-09-14T04:45:58.214933+0000","last_peered":"2024-09-14T04:45:58.214933+0000","last_clean":"2024-09-14T04:45:58.214933+0000","last_became_active":"2024-09-14T04:41:36.929279+0000","last_became_peered":"2024-09-14T04:41:36.929279+0000","last_unstale":"2024-09-14T04:45:58.214933+0000","last_undegraded":"2024-09-14T04:45:58.214933+0000","last_fullsized":"2024-09-14T04:45:58.214933+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.977999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.977999+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:44:06.383827+0000","scrub_duration":0.00192122,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":2099,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.550773+0000","last_change":"2024-09-14T04:44:57.330745+0000","last_active":"2024-09-14T04:46:29.550773+0000","last_peered":"2024-09-14T04:46:29.550773+0000","last_clean":"2024-09-14T04:46:29.550773+0000","last_became_active":"2024-09-14T04:44:57.330261+0000","last_became_peered":"2024-09-14T04:44:57.330261+0000","last_unstale":"2024-09-14T04:46:29.550773+0000","last_undegraded":"2024-09-14T04:46:29.550773+0000","last_fullsized":"2024-09-14T04:46:29.550773+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:24:08.870976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2101,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.381512+0000","last_change":"2024-09-14T04:42:49.941397+0000","last_active":"2024-09-14T04:46:02.381512+0000","last_peered":"2024-09-14T04:46:02.381512+0000","last_clean":"2024-09-14T04:46:02.381512+0000","last_became_active":"2024-09-14T04:42:49.941066+0000","last_became_peered":"2024-09-14T04:42:49.941066+0000","last_unstale":"2024-09-14T04:46:02.381512+0000","last_undegraded":"2024-09-14T04:46:02.381512+0000","last_fullsized":"2024-09-14T04:46:02.381512+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:30.777822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":21,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.877881+0000","last_change":"2024-09-14T04:44:57.326724+0000","last_active":"2024-09-14T04:45:37.877881+0000","last_peered":"2024-09-14T04:45:37.877881+0000","last_clean":"2024-09-14T04:45:37.877881+0000","last_became_active":"2024-09-14T04:44:57.326191+0000","last_became_peered":"2024-09-14T04:44:57.326191+0000","last_unstale":"2024-09-14T04:45:37.877881+0000","last_undegraded":"2024-09-14T04:45:37.877881+0000","last_fullsized":"2024-09-14T04:45:37.877881+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:46.060269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":21,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.876534+0000","last_change":"2024-09-14T04:44:57.328528+0000","last_active":"2024-09-14T04:45:37.876534+0000","last_peered":"2024-09-14T04:45:37.876534+0000","last_clean":"2024-09-14T04:45:37.876534+0000","last_became_active":"2024-09-14T04:44:57.327739+0000","last_became_peered":"2024-09-14T04:44:57.327739+0000","last_unstale":"2024-09-14T04:45:37.876534+0000","last_undegraded":"2024-09-14T04:45:37.876534+0000","last_fullsized":"2024-09-14T04:45:37.876534+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:04:58.294690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"349'149","reported_seq":741,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.885960+0000","last_change":"2024-09-14T04:42:53.013201+0000","last_active":"2024-09-14T04:45:37.885960+0000","last_peered":"2024-09-14T04:45:37.885960+0000","last_clean":"2024-09-14T04:45:37.885960+0000","last_became_active":"2024-09-14T04:42:53.012818+0000","last_became_peered":"2024-09-14T04:42:53.012818+0000","last_unstale":"2024-09-14T04:45:37.885960+0000","last_undegraded":"2024-09-14T04:45:37.885960+0000","last_fullsized":"2024-09-14T04:45:37.885960+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:56.738636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:56.738636+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:39:19.056067+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":21,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.876983+0000","last_change":"2024-09-14T04:44:57.323436+0000","last_active":"2024-09-14T04:45:37.876983+0000","last_peered":"2024-09-14T04:45:37.876983+0000","last_clean":"2024-09-14T04:45:37.876983+0000","last_became_active":"2024-09-14T04:44:57.323022+0000","last_became_peered":"2024-09-14T04:44:57.323022+0000","last_unstale":"2024-09-14T04:45:37.876983+0000","last_undegraded":"2024-09-14T04:45:37.876983+0000","last_fullsized":"2024-09-14T04:45:37.876983+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:22.775261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1427,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.885030+0000","last_change":"2024-09-14T04:44:57.322357+0000","last_active":"2024-09-14T04:45:37.885030+0000","last_peered":"2024-09-14T04:45:37.885030+0000","last_clean":"2024-09-14T04:45:37.885030+0000","last_became_active":"2024-09-14T04:44:57.321990+0000","last_became_peered":"2024-09-14T04:44:57.321990+0000","last_unstale":"2024-09-14T04:45:37.885030+0000","last_undegraded":"2024-09-14T04:45:37.885030+0000","last_fullsized":"2024-09-14T04:45:37.885030+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:09.658008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"349'149","reported_seq":1752,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:52.974299+0000","last_change":"2024-09-14T04:41:53.858076+0000","last_active":"2024-09-14T04:45:52.974299+0000","last_peered":"2024-09-14T04:45:52.974299+0000","last_clean":"2024-09-14T04:45:52.974299+0000","last_became_active":"2024-09-14T04:41:36.931585+0000","last_became_peered":"2024-09-14T04:41:36.931585+0000","last_unstale":"2024-09-14T04:45:52.974299+0000","last_undegraded":"2024-09-14T04:45:52.974299+0000","last_fullsized":"2024-09-14T04:45:52.974299+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:53.857924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:53.857924+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:10:20.658986+0000","scrub_duration":0.0022355919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":21,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.877266+0000","last_change":"2024-09-14T04:44:57.329198+0000","last_active":"2024-09-14T04:45:37.877266+0000","last_peered":"2024-09-14T04:45:37.877266+0000","last_clean":"2024-09-14T04:45:37.877266+0000","last_became_active":"2024-09-14T04:44:57.327754+0000","last_became_peered":"2024-09-14T04:44:57.327754+0000","last_unstale":"2024-09-14T04:45:37.877266+0000","last_undegraded":"2024-09-14T04:45:37.877266+0000","last_fullsized":"2024-09-14T04:45:37.877266+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:10:27.944476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":102,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.384595+0000","last_change":"2024-09-14T04:44:57.348687+0000","last_active":"2024-09-14T04:46:02.384595+0000","last_peered":"2024-09-14T04:46:02.384595+0000","last_clean":"2024-09-14T04:46:02.384595+0000","last_became_active":"2024-09-14T04:44:57.332845+0000","last_became_peered":"2024-09-14T04:44:57.332845+0000","last_unstale":"2024-09-14T04:46:02.384595+0000","last_undegraded":"2024-09-14T04:46:02.384595+0000","last_fullsized":"2024-09-14T04:46:02.384595+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:12:18.762737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2125,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.384307+0000","last_change":"2024-09-14T04:44:57.333468+0000","last_active":"2024-09-14T04:46:02.384307+0000","last_peered":"2024-09-14T04:46:02.384307+0000","last_clean":"2024-09-14T04:46:02.384307+0000","last_became_active":"2024-09-14T04:44:57.332626+0000","last_became_peered":"2024-09-14T04:44:57.332626+0000","last_unstale":"2024-09-14T04:46:02.384307+0000","last_undegraded":"2024-09-14T04:46:02.384307+0000","last_fullsized":"2024-09-14T04:46:02.384307+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:57:26.366438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1383,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.886352+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T04:45:37.886352+0000","last_peered":"2024-09-14T04:45:37.886352+0000","last_clean":"2024-09-14T04:45:37.886352+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T04:45:37.886352+0000","last_undegraded":"2024-09-14T04:45:37.886352+0000","last_fullsized":"2024-09-14T04:45:37.886352+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1382,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.884833+0000","last_change":"2024-09-14T04:06:46.869868+0000","last_active":"2024-09-14T04:45:37.884833+0000","last_peered":"2024-09-14T04:45:37.884833+0000","last_clean":"2024-09-14T04:45:37.884833+0000","last_became_active":"2024-09-14T04:06:46.869126+0000","last_became_peered":"2024-09-14T04:06:46.869126+0000","last_unstale":"2024-09-14T04:45:37.884833+0000","last_undegraded":"2024-09-14T04:45:37.884833+0000","last_fullsized":"2024-09-14T04:45:37.884833+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:41.150138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1366,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.888535+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T04:45:37.888535+0000","last_peered":"2024-09-14T04:45:37.888535+0000","last_clean":"2024-09-14T04:45:37.888535+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T04:45:37.888535+0000","last_undegraded":"2024-09-14T04:45:37.888535+0000","last_fullsized":"2024-09-14T04:45:37.888535+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":21,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.876825+0000","last_change":"2024-09-14T04:44:57.332112+0000","last_active":"2024-09-14T04:45:37.876825+0000","last_peered":"2024-09-14T04:45:37.876825+0000","last_clean":"2024-09-14T04:45:37.876825+0000","last_became_active":"2024-09-14T04:44:57.331575+0000","last_became_peered":"2024-09-14T04:44:57.331575+0000","last_unstale":"2024-09-14T04:45:37.876825+0000","last_undegraded":"2024-09-14T04:45:37.876825+0000","last_fullsized":"2024-09-14T04:45:37.876825+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:50:00.391283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1414,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.888406+0000","last_change":"2024-09-14T04:44:57.351460+0000","last_active":"2024-09-14T04:45:37.888406+0000","last_peered":"2024-09-14T04:45:37.888406+0000","last_clean":"2024-09-14T04:45:37.888406+0000","last_became_active":"2024-09-14T04:44:57.334662+0000","last_became_peered":"2024-09-14T04:44:57.334662+0000","last_unstale":"2024-09-14T04:45:37.888406+0000","last_undegraded":"2024-09-14T04:45:37.888406+0000","last_fullsized":"2024-09-14T04:45:37.888406+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:02.399062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1368,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.886332+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T04:45:37.886332+0000","last_peered":"2024-09-14T04:45:37.886332+0000","last_clean":"2024-09-14T04:45:37.886332+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T04:45:37.886332+0000","last_undegraded":"2024-09-14T04:45:37.886332+0000","last_fullsized":"2024-09-14T04:45:37.886332+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1426,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.888426+0000","last_change":"2024-09-14T04:44:57.349690+0000","last_active":"2024-09-14T04:45:37.888426+0000","last_peered":"2024-09-14T04:45:37.888426+0000","last_clean":"2024-09-14T04:45:37.888426+0000","last_became_active":"2024-09-14T04:44:57.333579+0000","last_became_peered":"2024-09-14T04:44:57.333579+0000","last_unstale":"2024-09-14T04:45:37.888426+0000","last_undegraded":"2024-09-14T04:45:37.888426+0000","last_fullsized":"2024-09-14T04:45:37.888426+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:17:55.265141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1427,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.881845+0000","last_change":"2024-09-14T04:42:49.960838+0000","last_active":"2024-09-14T04:45:37.881845+0000","last_peered":"2024-09-14T04:45:37.881845+0000","last_clean":"2024-09-14T04:45:37.881845+0000","last_became_active":"2024-09-14T04:42:49.954736+0000","last_became_peered":"2024-09-14T04:42:49.954736+0000","last_unstale":"2024-09-14T04:45:37.881845+0000","last_undegraded":"2024-09-14T04:45:37.881845+0000","last_fullsized":"2024-09-14T04:45:37.881845+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:41:13.540983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":2083,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.391745+0000","last_change":"2024-09-14T04:07:58.565442+0000","last_active":"2024-09-14T04:46:02.391745+0000","last_peered":"2024-09-14T04:46:02.391745+0000","last_clean":"2024-09-14T04:46:02.391745+0000","last_became_active":"2024-09-14T04:07:58.552289+0000","last_became_peered":"2024-09-14T04:07:58.552289+0000","last_unstale":"2024-09-14T04:46:02.391745+0000","last_undegraded":"2024-09-14T04:46:02.391745+0000","last_fullsized":"2024-09-14T04:46:02.391745+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:48:02.364960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1388,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.887917+0000","last_change":"2024-09-14T04:44:57.332511+0000","last_active":"2024-09-14T04:45:37.887917+0000","last_peered":"2024-09-14T04:45:37.887917+0000","last_clean":"2024-09-14T04:45:37.887917+0000","last_became_active":"2024-09-14T04:44:57.332166+0000","last_became_peered":"2024-09-14T04:44:57.332166+0000","last_unstale":"2024-09-14T04:45:37.887917+0000","last_undegraded":"2024-09-14T04:45:37.887917+0000","last_fullsized":"2024-09-14T04:45:37.887917+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:53.984639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2104,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.379136+0000","last_change":"2024-09-14T04:07:58.560880+0000","last_active":"2024-09-14T04:46:02.379136+0000","last_peered":"2024-09-14T04:46:02.379136+0000","last_clean":"2024-09-14T04:46:02.379136+0000","last_became_active":"2024-09-14T04:07:58.560132+0000","last_became_peered":"2024-09-14T04:07:58.560132+0000","last_unstale":"2024-09-14T04:46:02.379136+0000","last_undegraded":"2024-09-14T04:46:02.379136+0000","last_fullsized":"2024-09-14T04:46:02.379136+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:46:00.287050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2104,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.383798+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T04:46:02.383798+0000","last_peered":"2024-09-14T04:46:02.383798+0000","last_clean":"2024-09-14T04:46:02.383798+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T04:46:02.383798+0000","last_undegraded":"2024-09-14T04:46:02.383798+0000","last_fullsized":"2024-09-14T04:46:02.383798+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1381,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.373534+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T04:46:02.373534+0000","last_peered":"2024-09-14T04:46:02.373534+0000","last_clean":"2024-09-14T04:46:02.373534+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T04:46:02.373534+0000","last_undegraded":"2024-09-14T04:46:02.373534+0000","last_fullsized":"2024-09-14T04:46:02.373534+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1417,"reported_epoch":1412,"state":"active+clean","last_fresh":"2024-09-14T04:45:37.888005+0000","last_change":"2024-09-14T04:44:57.339993+0000","last_active":"2024-09-14T04:45:37.888005+0000","last_peered":"2024-09-14T04:45:37.888005+0000","last_clean":"2024-09-14T04:45:37.888005+0000","last_became_active":"2024-09-14T04:44:57.330472+0000","last_became_peered":"2024-09-14T04:44:57.330472+0000","last_unstale":"2024-09-14T04:45:37.888005+0000","last_undegraded":"2024-09-14T04:45:37.888005+0000","last_fullsized":"2024-09-14T04:45:37.888005+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:19:27.046988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":46,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.549316+0000","last_change":"2024-09-14T04:44:14.479373+0000","last_active":"2024-09-14T04:46:29.549316+0000","last_peered":"2024-09-14T04:46:29.549316+0000","last_clean":"2024-09-14T04:46:29.549316+0000","last_became_active":"2024-09-14T04:44:14.479084+0000","last_became_peered":"2024-09-14T04:44:14.479084+0000","last_unstale":"2024-09-14T04:46:29.549316+0000","last_undegraded":"2024-09-14T04:46:29.549316+0000","last_fullsized":"2024-09-14T04:46:29.549316+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:49:23.412806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":54,"num_pg":16,"stat_sum":{"num_bytes":2835808280,"num_objects":43272,"num_object_clones":0,"num_object_copies":129816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":13714,"num_objects_unfound":0,"num_objects_dirty":43272,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":209518,"num_write_kb":8089217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1708,"num_bytes_recovered":111869952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8531095552,"data_stored":8531083336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122135,"ondisk_log_size":122135,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":5070,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4250,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201946,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1580028,"kb_used_data":1256116,"kb_used_omap":1,"kb_used_meta":323902,"kb_avail":92165124,"statfs":{"total":95995035648,"available":94377086976,"internally_reserved":0,"allocated":1286262784,"data_stored":1281783704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":331676107},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463582,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1810424,"kb_used_data":1404464,"kb_used_omap":1,"kb_used_meta":405950,"kb_avail":91934728,"statfs":{"total":95995035648,"available":94141161472,"internally_reserved":0,"allocated":1438171136,"data_stored":1433696049,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":415693255},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725219,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1813488,"kb_used_data":1403432,"kb_used_omap":1,"kb_used_meta":410046,"kb_avail":91931664,"statfs":{"total":95995035648,"available":94138023936,"internally_reserved":0,"allocated":1437114368,"data_stored":1432643819,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":419887558},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":1392,"seq":5978594476065,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":102204,"kb_used_data":11188,"kb_used_omap":117,"kb_used_meta":90890,"kb_avail":93642948,"statfs":{"total":95995035648,"available":95890378752,"internally_reserved":0,"allocated":11456512,"data_stored":6977984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120638,"internal_metadata":93071554},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772074960,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2374512,"kb_used_data":1973032,"kb_used_omap":71,"kb_used_meta":401272,"kb_avail":91370640,"statfs":{"total":95995035648,"available":93563535360,"internally_reserved":0,"allocated":2020384768,"data_stored":2015914296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":410903478},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707303896,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":748864,"kb_used_data":527988,"kb_used_omap":68,"kb_used_meta":220795,"kb_avail":92996288,"statfs":{"total":95995035648,"available":95228198912,"internally_reserved":0,"allocated":540659712,"data_stored":536185762,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":226094288},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872729055,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1032948,"kb_used_data":685740,"kb_used_omap":67,"kb_used_meta":347068,"kb_avail":92712204,"statfs":{"total":95995035648,"available":94937296896,"internally_reserved":0,"allocated":702197760,"data_stored":697730861,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":355397989},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056105,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1637932,"kb_used_data":1254756,"kb_used_omap":74,"kb_used_meta":382965,"kb_avail":92107220,"statfs":{"total":95995035648,"available":94317793280,"internally_reserved":0,"allocated":1284870144,"data_stored":1280403116,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":392156853},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1901,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1255215104,"data_stored":1255211032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":691404800,"data_stored":691404800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":526057472,"data_stored":526057472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1979125760,"data_stored":1979121688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":720896,"data_stored":720896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1413021696,"data_stored":1413021696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1412435968,"data_stored":1412431896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1253113856,"data_stored":1253113856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:46:34.208 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:34.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:34.262 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:34.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:34.363 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:34.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:34.408 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:34.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:34.509 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:34.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:34.548 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:34.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:34.649 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:34.688 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:34.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:34.689 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:34.709 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:34 smithi039 ceph-mon[106470]: pgmap v2492: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 12973/120426 objects misplaced (10.773%); 1.3 MiB/s, 21 objects/s recovering 2024-09-14T04:46:34.709 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:34 smithi039 ceph-mon[111235]: pgmap v2492: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 12973/120426 objects misplaced (10.773%); 1.3 MiB/s, 21 objects/s recovering 2024-09-14T04:46:34.790 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:34.831 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:34.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:34.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:34.933 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:34.976 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:34.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:34.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:35.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:34 smithi129 ceph-mon[101316]: pgmap v2492: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 12973/120426 objects misplaced (10.773%); 1.3 MiB/s, 21 objects/s recovering 2024-09-14T04:46:35.078 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:35.100 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:35.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:35.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:35.202 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:35.223 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:35.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:35.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:35.325 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:35.358 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:35.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:35.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:35.393 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:46:35.460 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:35.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:35.488 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:35.488 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:35.589 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:35.702 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:35.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:35.703 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:35.804 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:35.846 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:35.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:35.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:35.948 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:35.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:35.980 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:35.980 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:36.081 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:36.112 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:36.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:36.113 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:36.214 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:36.250 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:36.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:36.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:36.352 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:36.389 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:36.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:36.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:36.491 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:36.512 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:36.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:36.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:36.613 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:36.657 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:36.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:36.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:36.759 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:36.797 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:36.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:36.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:36.899 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:36.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:36.932 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:36.932 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:37.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:36 smithi129 ceph-mon[101316]: pgmap v2493: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.4 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 12973/120426 objects misplaced (10.773%); 1.3 MiB/s, 21 objects/s recovering 2024-09-14T04:46:37.033 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:37.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:36 smithi039 ceph-mon[106470]: pgmap v2493: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.4 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 12973/120426 objects misplaced (10.773%); 1.3 MiB/s, 21 objects/s recovering 2024-09-14T04:46:37.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:36 smithi039 ceph-mon[111235]: pgmap v2493: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.4 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 806 op/s; 12973/120426 objects misplaced (10.773%); 1.3 MiB/s, 21 objects/s recovering 2024-09-14T04:46:37.070 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:37.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:37.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:37.172 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:37.176 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:46:37.209 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:37.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:37.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:37.311 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:37.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:37.362 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:37.362 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:37.463 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:37.497 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:37.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:37.498 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:37.599 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:37.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:37.653 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:37.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:37.754 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:37.808 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:37.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:37.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:37.910 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:37.944 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:37.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:37.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:38.046 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:38.071 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:38.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:38.072 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:38.173 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:38.202 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:38.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:38.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:38.304 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:38.332 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:38.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:38.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:38.434 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:38.468 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:38.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:38.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:38.570 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:38.600 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:38.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:38.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:38.649 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:38 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:46:38.649 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:38 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:46:38.649 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:38 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:46:38.649 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:38 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:46:38.702 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:38.738 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:38.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:38.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:38.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:38 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:46:38.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:38 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:46:38.840 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:38.877 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:38.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:38.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:38.910 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:46:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:46:38] "GET /metrics HTTP/1.1" 200 36746 "" "Prometheus/2.51.0" 2024-09-14T04:46:38.979 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:39.019 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:39.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:39.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:39.120 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:39.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:39.143 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:39.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:39.244 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:39.281 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:39.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:39.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:39.382 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:39.412 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:39.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:39.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:39.514 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:39.540 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:39.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:39.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:39.632 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:39 smithi039 ceph-mon[106470]: pgmap v2494: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.3 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 12598/115434 objects misplaced (10.914%); 1.4 MiB/s, 21 objects/s recovering 2024-09-14T04:46:39.632 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:39 smithi039 ceph-mon[111235]: pgmap v2494: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.3 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 12598/115434 objects misplaced (10.914%); 1.4 MiB/s, 21 objects/s recovering 2024-09-14T04:46:39.642 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:39.681 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:39.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:39.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:39.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:39 smithi129 ceph-mon[101316]: pgmap v2494: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.3 GiB data, 9.7 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 788 op/s; 12598/115434 objects misplaced (10.914%); 1.4 MiB/s, 21 objects/s recovering 2024-09-14T04:46:39.782 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:39.834 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:39.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:39.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:39.936 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:39.961 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:39.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:39.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:40.063 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:40.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:40.100 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:40.101 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:40.202 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:40.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:40.245 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:40.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:40.346 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:40.391 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:40.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:40.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:40.494 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:40.546 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:40.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:40.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:40.648 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:40.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:40.694 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:40.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:40.795 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:40.829 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:40.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:40.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:40.931 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:40.960 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:40.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:40.961 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:41.062 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:41.091 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:41.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:41.092 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:41.192 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:41.223 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:46:41.224 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:46:41.229 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:41.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:41.230 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:41.253 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:46:41.253 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:46:41.266 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:46:41.267 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:46:41.303 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:46:41.304 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:46:41.331 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:41.336 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:46:41.336 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:46:41.370 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:46:41.370 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:46:41.373 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:41.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:41.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:41.400 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:46:41.400 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:46:41.426 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:46:41.426 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:46:41.460 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:46:41.461 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:46:41.476 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:41 smithi039 ceph-mon[106470]: pgmap v2495: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.2 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 12110/108675 objects misplaced (11.143%); 1.4 MiB/s, 21 objects/s recovering 2024-09-14T04:46:41.476 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:41 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/296573037' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:46:41.477 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:41 smithi039 ceph-mon[111235]: pgmap v2495: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.2 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 12110/108675 objects misplaced (11.143%); 1.4 MiB/s, 21 objects/s recovering 2024-09-14T04:46:41.477 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:41 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/296573037' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:46:41.477 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:41.508 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:41.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:41.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:41.610 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:41.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:41.649 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:41.649 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:41.750 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:41 smithi129 ceph-mon[101316]: pgmap v2495: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.2 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 778 op/s; 12110/108675 objects misplaced (11.143%); 1.4 MiB/s, 21 objects/s recovering 2024-09-14T04:46:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:41 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/296573037' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:46:41.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:41.794 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:41.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:41.895 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:41.924 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:41.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:41.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:42.026 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:42.063 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:42.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:42.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:42.165 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:42.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:42.211 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:42.211 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:42.312 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:42.348 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:42.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:42.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:42.450 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:42.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:42.487 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:42.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:42.588 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:42.624 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:42.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:42.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:42.726 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:42.766 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:42.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:42.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:42.868 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:42.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:42.911 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:42.911 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:43.012 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:43.051 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:43.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:43.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:43.153 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:43.187 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:43.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:43.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:43.289 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:43.329 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:43.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:43.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:43.431 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:43.468 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:43.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:43.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:43.570 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:43.603 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:43.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:43.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:43.705 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:43.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:43.742 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:43.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:43.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:43 smithi129 ceph-mon[101316]: pgmap v2496: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 11995/106386 objects misplaced (11.275%); 1.7 MiB/s, 27 objects/s recovering 2024-09-14T04:46:43.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:43 smithi039 ceph-mon[106470]: pgmap v2496: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 11995/106386 objects misplaced (11.275%); 1.7 MiB/s, 27 objects/s recovering 2024-09-14T04:46:43.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:43 smithi039 ceph-mon[111235]: pgmap v2496: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 11995/106386 objects misplaced (11.275%); 1.7 MiB/s, 27 objects/s recovering 2024-09-14T04:46:43.843 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:43.864 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:43.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:43.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:43.966 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:43.987 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:43.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:43.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:44.089 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:44.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:44.119 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:44.119 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:44.220 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:44.258 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:44.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:44.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:44.360 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:44.391 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:44.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:44.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:44.493 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:44.525 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:44.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:44.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:44.609 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-14T04:46:44.627 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:44.710 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:44.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:44.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:44.812 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:44.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:44.846 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:44.846 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:44.947 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:44.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:44.979 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:44.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:45.011 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:46:45.080 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:45.136 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:45.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:45.137 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:45.238 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:45.282 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:45.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:45.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:45.383 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:45.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:45.425 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:45.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:45.526 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:45.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:45 smithi039 ceph-mon[106470]: pgmap v2497: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 926 op/s; 11257/96534 objects misplaced (11.661%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:46:45.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:45 smithi039 ceph-mon[111235]: pgmap v2497: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 926 op/s; 11257/96534 objects misplaced (11.661%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:46:45.551 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:45.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:45.552 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:45.654 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:45.684 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:45.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:45.685 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:45.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:45 smithi129 ceph-mon[101316]: pgmap v2497: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 926 op/s; 11257/96534 objects misplaced (11.661%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:46:45.786 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:45.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:45.825 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:45.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:45.921 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:46:45.927 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:45.982 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:45.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:45.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:46.084 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:46.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:46.132 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:46.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:46.233 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:46.277 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:46.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:46.278 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:46.379 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:46.414 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:46.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:46.415 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:46.516 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:46.544 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:46.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:46.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:46.646 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:46.691 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:46.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:46.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:46.793 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:46.830 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:46.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:46.831 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:46.932 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:46.962 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:46.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:46.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:47.064 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:47.099 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:47.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:47.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:47.201 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:47.229 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:47.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:47.230 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:47.331 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:47.362 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:47.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:47.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:47.464 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:47.509 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:47.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:47.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:47.611 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:47.647 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:47.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:47.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:47.749 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:47.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:47 smithi129 ceph-mon[101316]: pgmap v2498: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 11257/96534 objects misplaced (11.661%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:46:47.779 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:47.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:47.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:47.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:47 smithi039 ceph-mon[106470]: pgmap v2498: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 11257/96534 objects misplaced (11.661%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:46:47.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:47 smithi039 ceph-mon[111235]: pgmap v2498: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 2.0 GiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 663 op/s; 11257/96534 objects misplaced (11.661%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:46:47.880 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:47.881 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:46:47.928 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:47.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:47.930 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:47.930 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:46:48.031 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:48.056 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:48.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:48.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:48.158 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:48.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:48.212 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:48.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:48.313 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:48.342 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:48.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:48.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:48.444 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:48.499 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:48.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:48.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:48.602 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:48.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:48.648 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:48.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:48.721 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:46:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:46:48] "GET /metrics HTTP/1.1" 200 36746 "" "Prometheus/2.51.0" 2024-09-14T04:46:48.749 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:48.782 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:48.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:48.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:48.884 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:48.926 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:48.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:48.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:49.028 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:49.065 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:49.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:49.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:49.167 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:49.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:49.200 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:49.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:49.301 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:49.326 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:49.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:49.327 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:49.428 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:49.455 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:49.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:49.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:49.556 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:49.600 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:49.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:49.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:49.702 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:49.751 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:49.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:49.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:49.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:49 smithi129 ceph-mon[101316]: pgmap v2499: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 10903/91458 objects misplaced (11.921%); 1.7 MiB/s, 27 objects/s recovering 2024-09-14T04:46:49.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:49 smithi039 ceph-mon[106470]: pgmap v2499: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 10903/91458 objects misplaced (11.921%); 1.7 MiB/s, 27 objects/s recovering 2024-09-14T04:46:49.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:49 smithi039 ceph-mon[111235]: pgmap v2499: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 1.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 804 op/s; 10903/91458 objects misplaced (11.921%); 1.7 MiB/s, 27 objects/s recovering 2024-09-14T04:46:49.853 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:49.905 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:49.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:49.907 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:50.007 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:50.045 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:50.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:50.046 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:50.147 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:50.188 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:50.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:50.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:50.290 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:50.315 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:50.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:50.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:50.417 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:50.462 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:50.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:50.463 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:50.470 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is set 2024-09-14T04:46:50.564 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:50.585 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:50.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:50.586 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:50.687 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:50.725 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:50.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:50.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:50.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:50 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2536745889' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:46:50.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:50 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:46:50.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:50 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2536745889' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:46:50.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:50 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:46:50.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:50 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2536745889' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:46:50.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:50 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:46:50.827 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:50.861 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:50.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:50.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:50.963 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:50.984 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:50.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:50.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:51.085 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:51.109 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:51.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:51.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:51.211 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:51.243 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:51.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:51.244 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:51.344 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:51.384 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:51.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:51.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:51.486 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:51.514 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:51.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:51.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:51.616 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:51.658 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:51.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:51.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:51.760 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:51 smithi129 ceph-mon[101316]: pgmap v2500: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 1.7 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 10440/85077 objects misplaced (12.271%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:46:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:51 smithi129 ceph-mon[101316]: from='client.45539 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:46:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:51 smithi129 ceph-mon[101316]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:46:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:51 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:46:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:51 smithi129 ceph-mon[101316]: osdmap e1415: 8 total, 8 up, 8 in 2024-09-14T04:46:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:51 smithi039 ceph-mon[106470]: pgmap v2500: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 1.7 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 10440/85077 objects misplaced (12.271%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:46:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:51 smithi039 ceph-mon[106470]: from='client.45539 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:46:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:51 smithi039 ceph-mon[106470]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:46:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:51 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:46:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:51 smithi039 ceph-mon[106470]: osdmap e1415: 8 total, 8 up, 8 in 2024-09-14T04:46:51.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:51 smithi039 ceph-mon[111235]: pgmap v2500: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 1.7 GiB data, 7.8 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 842 op/s; 10440/85077 objects misplaced (12.271%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:46:51.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:51 smithi039 ceph-mon[111235]: from='client.45539 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:46:51.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:51 smithi039 ceph-mon[111235]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:46:51.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:51 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:46:51.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:51 smithi039 ceph-mon[111235]: osdmap e1415: 8 total, 8 up, 8 in 2024-09-14T04:46:51.802 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:51.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:51.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:51.904 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:51.943 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:51.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:51.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:52.045 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:52.080 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:52.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:52.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:52.182 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:52.216 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:52.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:52.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:52.318 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:52.348 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:52.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:52.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:52.450 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:52.479 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:52.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:52.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:52.581 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:52.630 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:52.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:52.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:52.731 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:52.765 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:52.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:52.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:52.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:46:52.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:46:52.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:46:52.867 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:52.904 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:52.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:52.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:53.006 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:53.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:53.040 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:53.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:53.141 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:53.175 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:53.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:53.176 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:53.277 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:53.311 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:53.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:53.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:53.413 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:53.421 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:53 smithi039 ceph-mon[106470]: pgmap v2502: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 1.7 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 10311/82890 objects misplaced (12.439%); 1.4 MiB/s, 22 objects/s recovering 2024-09-14T04:46:53.449 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:53.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:53.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:53.550 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:53.582 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:53.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:53.583 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:53.685 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:53.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:53.722 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:53.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:53 smithi129 ceph-mon[101316]: pgmap v2502: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 1.7 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 10311/82890 objects misplaced (12.439%); 1.4 MiB/s, 22 objects/s recovering 2024-09-14T04:46:53.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:53 smithi039 ceph-mon[111235]: pgmap v2502: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 1.7 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 782 op/s; 10311/82890 objects misplaced (12.439%); 1.4 MiB/s, 22 objects/s recovering 2024-09-14T04:46:53.823 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:53.856 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:53.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:53.857 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:53.958 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:53.991 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:46:53.993 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T04:46:53.996 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:53.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:53.997 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:54.098 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:54.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:54.141 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:54.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:54.242 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:54.270 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:54.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:54.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:54.372 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:54.409 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:54.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:54.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:54.511 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:54.552 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:54.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:54.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:54.654 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:54.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:54.694 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:54.694 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:54.795 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:54.862 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:54.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:54.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:54.964 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:55.006 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:55.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:55.007 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:55.108 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:46:55.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:55.152 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:55.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:46:55.253 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:55.289 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:55.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:55.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:55.346 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:46:55.368 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":2500,"stamp":"2024-09-14T04:46:49.687521+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1857361917,"num_objects":28359,"num_object_clones":0,"num_object_copies":85077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":10440,"num_objects_unfound":0,"num_objects_dirty":28359,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":224521,"num_write_kb":8089250,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2069,"num_bytes_recovered":133633518,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123772,"ondisk_log_size":123772,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":301,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8203212,"kb_used_data":5618380,"kb_used_omap":404,"kb_used_meta":2584235,"kb_avail":741758004,"statfs":{"total":767960285184,"available":759560196096,"internally_reserved":0,"allocated":5753221120,"data_stored":5717394803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":414022,"internal_metadata":2646257338},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,5,3,7,5,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-663158784,"num_objects":-10119,"num_object_clones":0,"num_object_copies":-30357,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-2158,"num_objects_unfound":0,"num_objects_dirty":-10119,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10119,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":214,"num_bytes_recovered":14024704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.008342"},"pg_stats":[{"pgid":"54.f","version":"1414'13923","reported_seq":14003,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:47.424486+0000","last_change":"2024-09-14T04:43:01.044742+0000","last_active":"2024-09-14T04:46:47.424486+0000","last_peered":"2024-09-14T04:46:47.424486+0000","last_clean":"2024-09-14T04:46:47.424486+0000","last_became_active":"2024-09-14T04:43:01.044421+0000","last_became_peered":"2024-09-14T04:43:01.044421+0000","last_unstale":"2024-09-14T04:46:47.424486+0000","last_undegraded":"2024-09-14T04:46:47.424486+0000","last_fullsized":"2024-09-14T04:46:47.424486+0000","mapping_epoch":1380,"log_start":"1399'7200","ondisk_log_start":"1399'7200","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":6723,"log_dups_size":0,"ondisk_log_size":6723,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:28.897426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111738880,"num_objects":1705,"num_object_clones":0,"num_object_copies":5115,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1705,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13923,"num_write_kb":500096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"54.e","version":"1414'13972","reported_seq":18063,"reported_epoch":1414,"state":"active+recovering+remapped","last_fresh":"2024-09-14T04:46:45.808806+0000","last_change":"2024-09-14T04:44:58.690021+0000","last_active":"2024-09-14T04:46:45.808806+0000","last_peered":"2024-09-14T04:46:45.808806+0000","last_clean":"2024-09-14T04:44:56.302285+0000","last_became_active":"2024-09-14T04:44:58.579754+0000","last_became_peered":"2024-09-14T04:44:58.579754+0000","last_unstale":"2024-09-14T04:46:45.808806+0000","last_undegraded":"2024-09-14T04:46:45.808806+0000","last_fullsized":"2024-09-14T04:46:45.808806+0000","mapping_epoch":1403,"log_start":"1397'5983","ondisk_log_start":"1397'5983","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7989,"log_dups_size":0,"ondisk_log_size":7989,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119930880,"num_objects":1830,"num_object_clones":0,"num_object_copies":5490,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5322,"num_objects_unfound":0,"num_objects_dirty":1830,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13972,"num_write_kb":505664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2004,"num_bytes_recovered":131268608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[1,2,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":-3492}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"54.d","version":"1414'14099","reported_seq":14142,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:48.255550+0000","last_change":"2024-09-14T04:43:01.050000+0000","last_active":"2024-09-14T04:46:48.255550+0000","last_peered":"2024-09-14T04:46:48.255550+0000","last_clean":"2024-09-14T04:46:48.255550+0000","last_became_active":"2024-09-14T04:43:01.049752+0000","last_became_peered":"2024-09-14T04:43:01.049752+0000","last_unstale":"2024-09-14T04:46:48.255550+0000","last_undegraded":"2024-09-14T04:46:48.255550+0000","last_fullsized":"2024-09-14T04:46:48.255550+0000","mapping_epoch":1380,"log_start":"1397'6500","ondisk_log_start":"1397'6500","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7599,"log_dups_size":0,"ondisk_log_size":7599,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:35:06.578523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116916248,"num_objects":1785,"num_object_clones":0,"num_object_copies":5355,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1785,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":14099,"num_write_kb":508225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"54.c","version":"1414'14069","reported_seq":14153,"reported_epoch":1414,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:46:47.391299+0000","last_change":"2024-09-14T04:44:58.455281+0000","last_active":"2024-09-14T04:46:47.391299+0000","last_peered":"2024-09-14T04:46:47.391299+0000","last_clean":"2024-09-14T04:44:56.285515+0000","last_became_active":"2024-09-14T04:44:58.445654+0000","last_became_peered":"2024-09-14T04:44:58.445654+0000","last_unstale":"2024-09-14T04:46:47.391299+0000","last_undegraded":"2024-09-14T04:46:47.391299+0000","last_fullsized":"2024-09-14T04:46:47.391299+0000","mapping_epoch":1403,"log_start":"1399'7400","ondisk_log_start":"1399'7400","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":6669,"log_dups_size":0,"ondisk_log_size":6669,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111345664,"num_objects":1699,"num_object_clones":0,"num_object_copies":5097,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1699,"num_objects_unfound":0,"num_objects_dirty":1699,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14069,"num_write_kb":504576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[{"shards":"3,5,6","objects":1699}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"54.b","version":"1414'14006","reported_seq":14075,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:48.171629+0000","last_change":"2024-09-14T04:43:13.355592+0000","last_active":"2024-09-14T04:46:48.171629+0000","last_peered":"2024-09-14T04:46:48.171629+0000","last_clean":"2024-09-14T04:46:48.171629+0000","last_became_active":"2024-09-14T04:43:13.355130+0000","last_became_peered":"2024-09-14T04:43:13.355130+0000","last_unstale":"2024-09-14T04:46:48.171629+0000","last_undegraded":"2024-09-14T04:46:48.171629+0000","last_fullsized":"2024-09-14T04:46:48.171629+0000","mapping_epoch":1386,"log_start":"1395'5600","ondisk_log_start":"1395'5600","created":1380,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":8406,"log_dups_size":0,"ondisk_log_size":8406,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:42:44.062290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111411200,"num_objects":1700,"num_object_clones":0,"num_object_copies":5100,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1700,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14006,"num_write_kb":502592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"54.a","version":"1414'13895","reported_seq":13939,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:48.236601+0000","last_change":"2024-09-14T04:43:01.043585+0000","last_active":"2024-09-14T04:46:48.236601+0000","last_peered":"2024-09-14T04:46:48.236601+0000","last_clean":"2024-09-14T04:46:48.236601+0000","last_became_active":"2024-09-14T04:43:01.043295+0000","last_became_peered":"2024-09-14T04:43:01.043295+0000","last_unstale":"2024-09-14T04:46:48.236601+0000","last_undegraded":"2024-09-14T04:46:48.236601+0000","last_fullsized":"2024-09-14T04:46:48.236601+0000","mapping_epoch":1380,"log_start":"1397'6300","ondisk_log_start":"1397'6300","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7595,"log_dups_size":0,"ondisk_log_size":7595,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:58:35.934308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114491392,"num_objects":1747,"num_object_clones":0,"num_object_copies":5241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1747,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13895,"num_write_kb":500544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"54.9","version":"1414'13924","reported_seq":14004,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:45.748692+0000","last_change":"2024-09-14T04:43:01.048728+0000","last_active":"2024-09-14T04:46:45.748692+0000","last_peered":"2024-09-14T04:46:45.748692+0000","last_clean":"2024-09-14T04:46:45.748692+0000","last_became_active":"2024-09-14T04:43:01.048381+0000","last_became_peered":"2024-09-14T04:43:01.048381+0000","last_unstale":"2024-09-14T04:46:45.748692+0000","last_undegraded":"2024-09-14T04:46:45.748692+0000","last_fullsized":"2024-09-14T04:46:45.748692+0000","mapping_epoch":1380,"log_start":"1397'6000","ondisk_log_start":"1397'6000","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7924,"log_dups_size":0,"ondisk_log_size":7924,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:47:27.860166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":124518400,"num_objects":1900,"num_object_clones":0,"num_object_copies":5700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1900,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13924,"num_write_kb":506368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"54.8","version":"1414'14073","reported_seq":14155,"reported_epoch":1414,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:46:48.208631+0000","last_change":"2024-09-14T04:44:58.458770+0000","last_active":"2024-09-14T04:46:48.208631+0000","last_peered":"2024-09-14T04:46:48.208631+0000","last_clean":"2024-09-14T04:44:56.294675+0000","last_became_active":"2024-09-14T04:44:58.445646+0000","last_became_peered":"2024-09-14T04:44:58.445646+0000","last_unstale":"2024-09-14T04:46:48.208631+0000","last_undegraded":"2024-09-14T04:46:48.208631+0000","last_fullsized":"2024-09-14T04:46:48.208631+0000","mapping_epoch":1403,"log_start":"1395'5700","ondisk_log_start":"1395'5700","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":8373,"log_dups_size":0,"ondisk_log_size":8373,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116326400,"num_objects":1775,"num_object_clones":0,"num_object_copies":5325,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1775,"num_objects_unfound":0,"num_objects_dirty":1775,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14073,"num_write_kb":507136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[0,3,7],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3,7","objects":1775}],"blocked_by":[],"up_primary":4,"acting_primary":0,"purged_snaps":[]},{"pgid":"54.7","version":"1414'13996","reported_seq":14076,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:47.429392+0000","last_change":"2024-09-14T04:43:01.050691+0000","last_active":"2024-09-14T04:46:47.429392+0000","last_peered":"2024-09-14T04:46:47.429392+0000","last_clean":"2024-09-14T04:46:47.429392+0000","last_became_active":"2024-09-14T04:43:01.050334+0000","last_became_peered":"2024-09-14T04:43:01.050334+0000","last_unstale":"2024-09-14T04:46:47.429392+0000","last_undegraded":"2024-09-14T04:46:47.429392+0000","last_fullsized":"2024-09-14T04:46:47.429392+0000","mapping_epoch":1380,"log_start":"1399'7300","ondisk_log_start":"1399'7300","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":6696,"log_dups_size":0,"ondisk_log_size":6696,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:22:30.806651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":119144448,"num_objects":1818,"num_object_clones":0,"num_object_copies":5454,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1818,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13996,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"54.6","version":"1414'14068","reported_seq":14112,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:47.973567+0000","last_change":"2024-09-14T04:43:01.043477+0000","last_active":"2024-09-14T04:46:47.973567+0000","last_peered":"2024-09-14T04:46:47.973567+0000","last_clean":"2024-09-14T04:46:47.973567+0000","last_became_active":"2024-09-14T04:43:01.043136+0000","last_became_peered":"2024-09-14T04:43:01.043136+0000","last_unstale":"2024-09-14T04:46:47.973567+0000","last_undegraded":"2024-09-14T04:46:47.973567+0000","last_fullsized":"2024-09-14T04:46:47.973567+0000","mapping_epoch":1380,"log_start":"1399'6700","ondisk_log_start":"1399'6700","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7368,"log_dups_size":0,"ondisk_log_size":7368,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:00:57.211271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117571584,"num_objects":1794,"num_object_clones":0,"num_object_copies":5382,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1794,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14068,"num_write_kb":507584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"54.5","version":"1414'14065","reported_seq":14109,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:48.287212+0000","last_change":"2024-09-14T04:43:01.048975+0000","last_active":"2024-09-14T04:46:48.287212+0000","last_peered":"2024-09-14T04:46:48.287212+0000","last_clean":"2024-09-14T04:46:48.287212+0000","last_became_active":"2024-09-14T04:43:01.048677+0000","last_became_peered":"2024-09-14T04:43:01.048677+0000","last_unstale":"2024-09-14T04:46:48.287212+0000","last_undegraded":"2024-09-14T04:46:48.287212+0000","last_fullsized":"2024-09-14T04:46:48.287212+0000","mapping_epoch":1380,"log_start":"1397'6500","ondisk_log_start":"1397'6500","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7565,"log_dups_size":0,"ondisk_log_size":7565,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:58:04.849606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115408896,"num_objects":1761,"num_object_clones":0,"num_object_copies":5283,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1761,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14065,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"54.4","version":"1414'14078","reported_seq":14158,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:47.387600+0000","last_change":"2024-09-14T04:43:01.043603+0000","last_active":"2024-09-14T04:46:47.387600+0000","last_peered":"2024-09-14T04:46:47.387600+0000","last_clean":"2024-09-14T04:46:47.387600+0000","last_became_active":"2024-09-14T04:43:01.042757+0000","last_became_peered":"2024-09-14T04:43:01.042757+0000","last_unstale":"2024-09-14T04:46:47.387600+0000","last_undegraded":"2024-09-14T04:46:47.387600+0000","last_fullsized":"2024-09-14T04:46:47.387600+0000","mapping_epoch":1380,"log_start":"1399'7400","ondisk_log_start":"1399'7400","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":6678,"log_dups_size":0,"ondisk_log_size":6678,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:46:20.719227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":116260864,"num_objects":1774,"num_object_clones":0,"num_object_copies":5322,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1774,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14078,"num_write_kb":507264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"54.3","version":"1414'14125","reported_seq":14205,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:45.801045+0000","last_change":"2024-09-14T04:43:01.048551+0000","last_active":"2024-09-14T04:46:45.801045+0000","last_peered":"2024-09-14T04:46:45.801045+0000","last_clean":"2024-09-14T04:46:45.801045+0000","last_became_active":"2024-09-14T04:43:01.048218+0000","last_became_peered":"2024-09-14T04:43:01.048218+0000","last_unstale":"2024-09-14T04:46:45.801045+0000","last_undegraded":"2024-09-14T04:46:45.801045+0000","last_fullsized":"2024-09-14T04:46:45.801045+0000","mapping_epoch":1380,"log_start":"1397'6200","ondisk_log_start":"1397'6200","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7925,"log_dups_size":0,"ondisk_log_size":7925,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:27:10.233265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126943232,"num_objects":1937,"num_object_clones":0,"num_object_copies":5811,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1937,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14125,"num_write_kb":513984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"54.2","version":"1414'14062","reported_seq":14142,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:48.184190+0000","last_change":"2024-09-14T04:43:01.049088+0000","last_active":"2024-09-14T04:46:48.184190+0000","last_peered":"2024-09-14T04:46:48.184190+0000","last_clean":"2024-09-14T04:46:48.184190+0000","last_became_active":"2024-09-14T04:43:01.048764+0000","last_became_peered":"2024-09-14T04:43:01.048764+0000","last_unstale":"2024-09-14T04:46:48.184190+0000","last_undegraded":"2024-09-14T04:46:48.184190+0000","last_fullsized":"2024-09-14T04:46:48.184190+0000","mapping_epoch":1380,"log_start":"1395'5700","ondisk_log_start":"1395'5700","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":8362,"log_dups_size":0,"ondisk_log_size":8362,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:03:50.532719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":111149056,"num_objects":1696,"num_object_clones":0,"num_object_copies":5088,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1696,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14062,"num_write_kb":504256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"54.1","version":"1414'14118","reported_seq":14162,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:48.242637+0000","last_change":"2024-09-14T04:43:01.047658+0000","last_active":"2024-09-14T04:46:48.242637+0000","last_peered":"2024-09-14T04:46:48.242637+0000","last_clean":"2024-09-14T04:46:48.242637+0000","last_became_active":"2024-09-14T04:43:01.047401+0000","last_became_peered":"2024-09-14T04:43:01.047401+0000","last_unstale":"2024-09-14T04:46:48.242637+0000","last_undegraded":"2024-09-14T04:46:48.242637+0000","last_fullsized":"2024-09-14T04:46:48.242637+0000","mapping_epoch":1380,"log_start":"1395'5500","ondisk_log_start":"1395'5500","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":8618,"log_dups_size":0,"ondisk_log_size":8618,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:02:05.987375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115605504,"num_objects":1764,"num_object_clones":0,"num_object_copies":5292,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1764,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14118,"num_write_kb":508224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"54.0","version":"1414'13984","reported_seq":14105,"reported_epoch":1414,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:46:48.270773+0000","last_change":"2024-09-14T04:44:58.640486+0000","last_active":"2024-09-14T04:46:48.270773+0000","last_peered":"2024-09-14T04:46:48.270773+0000","last_clean":"2024-09-14T04:44:56.291188+0000","last_became_active":"2024-09-14T04:44:58.441092+0000","last_became_peered":"2024-09-14T04:44:58.441092+0000","last_unstale":"2024-09-14T04:46:48.270773+0000","last_undegraded":"2024-09-14T04:46:48.270773+0000","last_fullsized":"2024-09-14T04:46:48.270773+0000","mapping_epoch":1403,"log_start":"1397'6400","ondisk_log_start":"1397'6400","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7584,"log_dups_size":0,"ondisk_log_size":7584,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:13:27.337351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108003328,"num_objects":1648,"num_object_clones":0,"num_object_copies":4944,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1644,"num_objects_unfound":0,"num_objects_dirty":1648,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13984,"num_write_kb":500224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":2097152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,3,5],"avail_no_missing":["6","3","5"],"object_location_counts":[{"shards":"3,5,6","objects":1648}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":2116,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.560283+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T04:46:29.560283+0000","last_peered":"2024-09-14T04:46:29.560283+0000","last_clean":"2024-09-14T04:46:29.560283+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T04:46:29.560283+0000","last_undegraded":"2024-09-14T04:46:29.560283+0000","last_fullsized":"2024-09-14T04:46:29.560283+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":22,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.371970+0000","last_change":"2024-09-14T04:44:57.325109+0000","last_active":"2024-09-14T04:46:02.371970+0000","last_peered":"2024-09-14T04:46:02.371970+0000","last_clean":"2024-09-14T04:46:02.371970+0000","last_became_active":"2024-09-14T04:44:57.324791+0000","last_became_peered":"2024-09-14T04:44:57.324791+0000","last_unstale":"2024-09-14T04:46:02.371970+0000","last_undegraded":"2024-09-14T04:46:02.371970+0000","last_fullsized":"2024-09-14T04:46:02.371970+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:03.593275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":2106,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.547674+0000","last_change":"2024-09-14T04:07:58.504829+0000","last_active":"2024-09-14T04:46:29.547674+0000","last_peered":"2024-09-14T04:46:29.547674+0000","last_clean":"2024-09-14T04:46:29.547674+0000","last_became_active":"2024-09-14T04:07:58.502623+0000","last_became_peered":"2024-09-14T04:07:58.502623+0000","last_unstale":"2024-09-14T04:46:29.547674+0000","last_undegraded":"2024-09-14T04:46:29.547674+0000","last_fullsized":"2024-09-14T04:46:29.547674+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:59.091993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":738,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.560228+0000","last_change":"2024-09-14T04:43:13.355281+0000","last_active":"2024-09-14T04:46:29.560228+0000","last_peered":"2024-09-14T04:46:29.560228+0000","last_clean":"2024-09-14T04:46:29.560228+0000","last_became_active":"2024-09-14T04:43:13.354967+0000","last_became_peered":"2024-09-14T04:43:13.354967+0000","last_unstale":"2024-09-14T04:46:29.560228+0000","last_undegraded":"2024-09-14T04:46:29.560228+0000","last_fullsized":"2024-09-14T04:46:29.560228+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:23:19.332075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1401,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.373483+0000","last_change":"2024-09-14T04:44:57.342918+0000","last_active":"2024-09-14T04:46:02.373483+0000","last_peered":"2024-09-14T04:46:02.373483+0000","last_clean":"2024-09-14T04:46:02.373483+0000","last_became_active":"2024-09-14T04:44:57.329550+0000","last_became_peered":"2024-09-14T04:44:57.329550+0000","last_unstale":"2024-09-14T04:46:02.373483+0000","last_undegraded":"2024-09-14T04:46:02.373483+0000","last_fullsized":"2024-09-14T04:46:02.373483+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:52:52.435893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":22,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.372067+0000","last_change":"2024-09-14T04:44:57.325537+0000","last_active":"2024-09-14T04:46:02.372067+0000","last_peered":"2024-09-14T04:46:02.372067+0000","last_clean":"2024-09-14T04:46:02.372067+0000","last_became_active":"2024-09-14T04:44:57.325057+0000","last_became_peered":"2024-09-14T04:44:57.325057+0000","last_unstale":"2024-09-14T04:46:02.372067+0000","last_undegraded":"2024-09-14T04:46:02.372067+0000","last_fullsized":"2024-09-14T04:46:02.372067+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:43:23.166708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1416,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.373668+0000","last_change":"2024-09-14T04:44:57.337033+0000","last_active":"2024-09-14T04:46:02.373668+0000","last_peered":"2024-09-14T04:46:02.373668+0000","last_clean":"2024-09-14T04:46:02.373668+0000","last_became_active":"2024-09-14T04:44:57.328437+0000","last_became_peered":"2024-09-14T04:44:57.328437+0000","last_unstale":"2024-09-14T04:46:02.373668+0000","last_undegraded":"2024-09-14T04:46:02.373668+0000","last_fullsized":"2024-09-14T04:46:02.373668+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.109086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":22,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.372072+0000","last_change":"2024-09-14T04:44:57.332243+0000","last_active":"2024-09-14T04:46:02.372072+0000","last_peered":"2024-09-14T04:46:02.372072+0000","last_clean":"2024-09-14T04:46:02.372072+0000","last_became_active":"2024-09-14T04:44:57.330413+0000","last_became_peered":"2024-09-14T04:44:57.330413+0000","last_unstale":"2024-09-14T04:46:02.372072+0000","last_undegraded":"2024-09-14T04:46:02.372072+0000","last_fullsized":"2024-09-14T04:46:02.372072+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:25:01.338549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":2123,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.560880+0000","last_change":"2024-09-14T04:44:57.333097+0000","last_active":"2024-09-14T04:46:29.560880+0000","last_peered":"2024-09-14T04:46:29.560880+0000","last_clean":"2024-09-14T04:46:29.560880+0000","last_became_active":"2024-09-14T04:44:57.332294+0000","last_became_peered":"2024-09-14T04:44:57.332294+0000","last_unstale":"2024-09-14T04:46:29.560880+0000","last_undegraded":"2024-09-14T04:46:29.560880+0000","last_fullsized":"2024-09-14T04:46:29.560880+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:57:44.426280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2112,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.559407+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T04:46:29.559407+0000","last_peered":"2024-09-14T04:46:29.559407+0000","last_clean":"2024-09-14T04:46:29.559407+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T04:46:29.559407+0000","last_undegraded":"2024-09-14T04:46:29.559407+0000","last_fullsized":"2024-09-14T04:46:29.559407+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":22,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.371995+0000","last_change":"2024-09-14T04:44:57.325392+0000","last_active":"2024-09-14T04:46:02.371995+0000","last_peered":"2024-09-14T04:46:02.371995+0000","last_clean":"2024-09-14T04:46:02.371995+0000","last_became_active":"2024-09-14T04:44:57.324624+0000","last_became_peered":"2024-09-14T04:44:57.324624+0000","last_unstale":"2024-09-14T04:46:02.371995+0000","last_undegraded":"2024-09-14T04:46:02.371995+0000","last_fullsized":"2024-09-14T04:46:02.371995+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:33:18.810199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2081,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.550404+0000","last_change":"2024-09-14T04:44:57.329113+0000","last_active":"2024-09-14T04:46:29.550404+0000","last_peered":"2024-09-14T04:46:29.550404+0000","last_clean":"2024-09-14T04:46:29.550404+0000","last_became_active":"2024-09-14T04:44:57.328727+0000","last_became_peered":"2024-09-14T04:44:57.328727+0000","last_unstale":"2024-09-14T04:46:29.550404+0000","last_undegraded":"2024-09-14T04:46:29.550404+0000","last_fullsized":"2024-09-14T04:46:29.550404+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:34.939874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":2098,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.550082+0000","last_change":"2024-09-14T04:44:57.329277+0000","last_active":"2024-09-14T04:46:29.550082+0000","last_peered":"2024-09-14T04:46:29.550082+0000","last_clean":"2024-09-14T04:46:29.550082+0000","last_became_active":"2024-09-14T04:44:57.328804+0000","last_became_peered":"2024-09-14T04:44:57.328804+0000","last_unstale":"2024-09-14T04:46:29.550082+0000","last_undegraded":"2024-09-14T04:46:29.550082+0000","last_fullsized":"2024-09-14T04:46:29.550082+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:42:46.055240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2111,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.560006+0000","last_change":"2024-09-14T04:07:58.561490+0000","last_active":"2024-09-14T04:46:29.560006+0000","last_peered":"2024-09-14T04:46:29.560006+0000","last_clean":"2024-09-14T04:46:29.560006+0000","last_became_active":"2024-09-14T04:07:58.559841+0000","last_became_peered":"2024-09-14T04:07:58.559841+0000","last_unstale":"2024-09-14T04:46:29.560006+0000","last_undegraded":"2024-09-14T04:46:29.560006+0000","last_fullsized":"2024-09-14T04:46:29.560006+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:27:03.795368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2094,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.560043+0000","last_change":"2024-09-14T04:43:13.357092+0000","last_active":"2024-09-14T04:46:29.560043+0000","last_peered":"2024-09-14T04:46:29.560043+0000","last_clean":"2024-09-14T04:46:29.560043+0000","last_became_active":"2024-09-14T04:43:13.356694+0000","last_became_peered":"2024-09-14T04:43:13.356694+0000","last_unstale":"2024-09-14T04:46:29.560043+0000","last_undegraded":"2024-09-14T04:46:29.560043+0000","last_fullsized":"2024-09-14T04:46:29.560043+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:11.167574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1393,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.372620+0000","last_change":"2024-09-14T04:07:21.720797+0000","last_active":"2024-09-14T04:46:02.372620+0000","last_peered":"2024-09-14T04:46:02.372620+0000","last_clean":"2024-09-14T04:46:02.372620+0000","last_became_active":"2024-09-14T04:07:21.719331+0000","last_became_peered":"2024-09-14T04:07:21.719331+0000","last_unstale":"2024-09-14T04:46:02.372620+0000","last_undegraded":"2024-09-14T04:46:02.372620+0000","last_fullsized":"2024-09-14T04:46:02.372620+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:35.404023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":41,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.373148+0000","last_change":"2024-09-14T04:43:11.641514+0000","last_active":"2024-09-14T04:46:02.373148+0000","last_peered":"2024-09-14T04:46:02.373148+0000","last_clean":"2024-09-14T04:46:02.373148+0000","last_became_active":"2024-09-14T04:43:11.641151+0000","last_became_peered":"2024-09-14T04:43:11.641151+0000","last_unstale":"2024-09-14T04:46:02.373148+0000","last_undegraded":"2024-09-14T04:46:02.373148+0000","last_fullsized":"2024-09-14T04:46:02.373148+0000","mapping_epoch":1384,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:06:45.045430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2105,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.560427+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T04:46:29.560427+0000","last_peered":"2024-09-14T04:46:29.560427+0000","last_clean":"2024-09-14T04:46:29.560427+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T04:46:29.560427+0000","last_undegraded":"2024-09-14T04:46:29.560427+0000","last_fullsized":"2024-09-14T04:46:29.560427+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":2087,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.550454+0000","last_change":"2024-09-14T04:07:58.507682+0000","last_active":"2024-09-14T04:46:29.550454+0000","last_peered":"2024-09-14T04:46:29.550454+0000","last_clean":"2024-09-14T04:46:29.550454+0000","last_became_active":"2024-09-14T04:07:58.507365+0000","last_became_peered":"2024-09-14T04:07:58.507365+0000","last_unstale":"2024-09-14T04:46:29.550454+0000","last_undegraded":"2024-09-14T04:46:29.550454+0000","last_fullsized":"2024-09-14T04:46:29.550454+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:14:47.294950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1400,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:18.233456+0000","last_change":"2024-09-14T04:42:49.938505+0000","last_active":"2024-09-14T04:46:18.233456+0000","last_peered":"2024-09-14T04:46:18.233456+0000","last_clean":"2024-09-14T04:46:18.233456+0000","last_became_active":"2024-09-14T04:42:49.938180+0000","last_became_peered":"2024-09-14T04:42:49.938180+0000","last_unstale":"2024-09-14T04:46:18.233456+0000","last_undegraded":"2024-09-14T04:46:18.233456+0000","last_fullsized":"2024-09-14T04:46:18.233456+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:05.929479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1419,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:12.985617+0000","last_change":"2024-09-14T04:44:57.346582+0000","last_active":"2024-09-14T04:46:12.985617+0000","last_peered":"2024-09-14T04:46:12.985617+0000","last_clean":"2024-09-14T04:46:12.985617+0000","last_became_active":"2024-09-14T04:44:57.330446+0000","last_became_peered":"2024-09-14T04:44:57.330446+0000","last_unstale":"2024-09-14T04:46:12.985617+0000","last_undegraded":"2024-09-14T04:46:12.985617+0000","last_fullsized":"2024-09-14T04:46:12.985617+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:29.238585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.549810+0000","last_change":"2024-09-14T04:44:14.480667+0000","last_active":"2024-09-14T04:46:29.549810+0000","last_peered":"2024-09-14T04:46:29.549810+0000","last_clean":"2024-09-14T04:46:29.549810+0000","last_became_active":"2024-09-14T04:44:14.480365+0000","last_became_peered":"2024-09-14T04:44:14.480365+0000","last_unstale":"2024-09-14T04:46:29.549810+0000","last_undegraded":"2024-09-14T04:46:29.549810+0000","last_fullsized":"2024-09-14T04:46:29.549810+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:00.611771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":32,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.372627+0000","last_change":"2024-09-14T04:44:57.341426+0000","last_active":"2024-09-14T04:46:02.372627+0000","last_peered":"2024-09-14T04:46:02.372627+0000","last_clean":"2024-09-14T04:46:02.372627+0000","last_became_active":"2024-09-14T04:44:57.328763+0000","last_became_peered":"2024-09-14T04:44:57.328763+0000","last_unstale":"2024-09-14T04:46:02.372627+0000","last_undegraded":"2024-09-14T04:46:02.372627+0000","last_fullsized":"2024-09-14T04:46:02.372627+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:34:40.066761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":22,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.372232+0000","last_change":"2024-09-14T04:44:57.331337+0000","last_active":"2024-09-14T04:46:02.372232+0000","last_peered":"2024-09-14T04:46:02.372232+0000","last_clean":"2024-09-14T04:46:02.372232+0000","last_became_active":"2024-09-14T04:44:57.329336+0000","last_became_peered":"2024-09-14T04:44:57.329336+0000","last_unstale":"2024-09-14T04:46:02.372232+0000","last_undegraded":"2024-09-14T04:46:02.372232+0000","last_fullsized":"2024-09-14T04:46:02.372232+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:52:35.980869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"349'149","reported_seq":775,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.550889+0000","last_change":"2024-09-14T04:41:37.755174+0000","last_active":"2024-09-14T04:46:29.550889+0000","last_peered":"2024-09-14T04:46:29.550889+0000","last_clean":"2024-09-14T04:46:29.550889+0000","last_became_active":"2024-09-14T04:41:36.930263+0000","last_became_peered":"2024-09-14T04:41:36.930263+0000","last_unstale":"2024-09-14T04:46:29.550889+0000","last_undegraded":"2024-09-14T04:46:29.550889+0000","last_fullsized":"2024-09-14T04:46:29.550889+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.755105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.755105+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:27:03.161713+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":22,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.372022+0000","last_change":"2024-09-14T04:44:57.329870+0000","last_active":"2024-09-14T04:46:02.372022+0000","last_peered":"2024-09-14T04:46:02.372022+0000","last_clean":"2024-09-14T04:46:02.372022+0000","last_became_active":"2024-09-14T04:44:57.329353+0000","last_became_peered":"2024-09-14T04:44:57.329353+0000","last_unstale":"2024-09-14T04:46:02.372022+0000","last_undegraded":"2024-09-14T04:46:02.372022+0000","last_fullsized":"2024-09-14T04:46:02.372022+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:46.432181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1397,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:03.218276+0000","last_change":"2024-09-14T04:07:21.721242+0000","last_active":"2024-09-14T04:46:03.218276+0000","last_peered":"2024-09-14T04:46:03.218276+0000","last_clean":"2024-09-14T04:46:03.218276+0000","last_became_active":"2024-09-14T04:07:21.720084+0000","last_became_peered":"2024-09-14T04:07:21.720084+0000","last_unstale":"2024-09-14T04:46:03.218276+0000","last_undegraded":"2024-09-14T04:46:03.218276+0000","last_fullsized":"2024-09-14T04:46:03.218276+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:58:41.308643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":2143,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.559918+0000","last_change":"2024-09-14T04:44:57.344614+0000","last_active":"2024-09-14T04:46:29.559918+0000","last_peered":"2024-09-14T04:46:29.559918+0000","last_clean":"2024-09-14T04:46:29.559918+0000","last_became_active":"2024-09-14T04:44:57.332256+0000","last_became_peered":"2024-09-14T04:44:57.332256+0000","last_unstale":"2024-09-14T04:46:29.559918+0000","last_undegraded":"2024-09-14T04:46:29.559918+0000","last_fullsized":"2024-09-14T04:46:29.559918+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:23.438417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"349'149","reported_seq":780,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.372571+0000","last_change":"2024-09-14T04:45:00.331854+0000","last_active":"2024-09-14T04:46:02.372571+0000","last_peered":"2024-09-14T04:46:02.372571+0000","last_clean":"2024-09-14T04:46:02.372571+0000","last_became_active":"2024-09-14T04:45:00.331460+0000","last_became_peered":"2024-09-14T04:45:00.331460+0000","last_unstale":"2024-09-14T04:46:02.372571+0000","last_undegraded":"2024-09-14T04:46:02.372571+0000","last_fullsized":"2024-09-14T04:46:02.372571+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.567535+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.567535+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:45.095729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":2080,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.548303+0000","last_change":"2024-09-14T04:07:58.555446+0000","last_active":"2024-09-14T04:46:29.548303+0000","last_peered":"2024-09-14T04:46:29.548303+0000","last_clean":"2024-09-14T04:46:29.548303+0000","last_became_active":"2024-09-14T04:07:58.548355+0000","last_became_peered":"2024-09-14T04:07:58.548355+0000","last_unstale":"2024-09-14T04:46:29.548303+0000","last_undegraded":"2024-09-14T04:46:29.548303+0000","last_fullsized":"2024-09-14T04:46:29.548303+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:56:28.426094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":98,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.562342+0000","last_change":"2024-09-14T04:44:57.330746+0000","last_active":"2024-09-14T04:46:29.562342+0000","last_peered":"2024-09-14T04:46:29.562342+0000","last_clean":"2024-09-14T04:46:29.562342+0000","last_became_active":"2024-09-14T04:44:57.330327+0000","last_became_peered":"2024-09-14T04:44:57.330327+0000","last_unstale":"2024-09-14T04:46:29.562342+0000","last_undegraded":"2024-09-14T04:46:29.562342+0000","last_fullsized":"2024-09-14T04:46:29.562342+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:53.122771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":62,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.375097+0000","last_change":"2024-09-14T04:43:13.355296+0000","last_active":"2024-09-14T04:46:02.375097+0000","last_peered":"2024-09-14T04:46:02.375097+0000","last_clean":"2024-09-14T04:46:02.375097+0000","last_became_active":"2024-09-14T04:43:13.354950+0000","last_became_peered":"2024-09-14T04:43:13.354950+0000","last_unstale":"2024-09-14T04:46:02.375097+0000","last_undegraded":"2024-09-14T04:46:02.375097+0000","last_fullsized":"2024-09-14T04:46:02.375097+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:20:08.968818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"349'149","reported_seq":719,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.372590+0000","last_change":"2024-09-14T04:41:39.471393+0000","last_active":"2024-09-14T04:46:02.372590+0000","last_peered":"2024-09-14T04:46:02.372590+0000","last_clean":"2024-09-14T04:46:02.372590+0000","last_became_active":"2024-09-14T04:41:36.931768+0000","last_became_peered":"2024-09-14T04:41:36.931768+0000","last_unstale":"2024-09-14T04:46:02.372590+0000","last_undegraded":"2024-09-14T04:46:02.372590+0000","last_fullsized":"2024-09-14T04:46:02.372590+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.471248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.471248+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:56.210375+0000","scrub_duration":0.021758344999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2099,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.561023+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T04:46:29.561023+0000","last_peered":"2024-09-14T04:46:29.561023+0000","last_clean":"2024-09-14T04:46:29.561023+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T04:46:29.561023+0000","last_undegraded":"2024-09-14T04:46:29.561023+0000","last_fullsized":"2024-09-14T04:46:29.561023+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1429,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:07.984448+0000","last_change":"2024-09-14T04:44:57.331436+0000","last_active":"2024-09-14T04:46:07.984448+0000","last_peered":"2024-09-14T04:46:07.984448+0000","last_clean":"2024-09-14T04:46:07.984448+0000","last_became_active":"2024-09-14T04:44:57.330957+0000","last_became_peered":"2024-09-14T04:44:57.330957+0000","last_unstale":"2024-09-14T04:46:07.984448+0000","last_undegraded":"2024-09-14T04:46:07.984448+0000","last_fullsized":"2024-09-14T04:46:07.984448+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:09.516688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":91,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.550369+0000","last_change":"2024-09-14T04:42:49.940968+0000","last_active":"2024-09-14T04:46:29.550369+0000","last_peered":"2024-09-14T04:46:29.550369+0000","last_clean":"2024-09-14T04:46:29.550369+0000","last_became_active":"2024-09-14T04:42:49.940667+0000","last_became_peered":"2024-09-14T04:42:49.940667+0000","last_unstale":"2024-09-14T04:46:29.550369+0000","last_undegraded":"2024-09-14T04:46:29.550369+0000","last_fullsized":"2024-09-14T04:46:29.550369+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:52:49.623815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"349'149","reported_seq":719,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.372593+0000","last_change":"2024-09-14T04:41:38.479980+0000","last_active":"2024-09-14T04:46:02.372593+0000","last_peered":"2024-09-14T04:46:02.372593+0000","last_clean":"2024-09-14T04:46:02.372593+0000","last_became_active":"2024-09-14T04:41:36.933392+0000","last_became_peered":"2024-09-14T04:41:36.933392+0000","last_unstale":"2024-09-14T04:46:02.372593+0000","last_undegraded":"2024-09-14T04:46:02.372593+0000","last_fullsized":"2024-09-14T04:46:02.372593+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:38.479887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:38.479887+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:01:26.102279+0000","scrub_duration":0.0062728319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":54,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.549691+0000","last_change":"2024-09-14T04:44:57.329983+0000","last_active":"2024-09-14T04:46:29.549691+0000","last_peered":"2024-09-14T04:46:29.549691+0000","last_clean":"2024-09-14T04:46:29.549691+0000","last_became_active":"2024-09-14T04:44:57.329597+0000","last_became_peered":"2024-09-14T04:44:57.329597+0000","last_unstale":"2024-09-14T04:46:29.549691+0000","last_undegraded":"2024-09-14T04:46:29.549691+0000","last_fullsized":"2024-09-14T04:46:29.549691+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:45.718039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":2134,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.550594+0000","last_change":"2024-09-14T04:06:46.930482+0000","last_active":"2024-09-14T04:46:29.550594+0000","last_peered":"2024-09-14T04:46:29.550594+0000","last_clean":"2024-09-14T04:46:29.550594+0000","last_became_active":"2024-09-14T04:06:46.927372+0000","last_became_peered":"2024-09-14T04:06:46.927372+0000","last_unstale":"2024-09-14T04:46:29.550594+0000","last_undegraded":"2024-09-14T04:46:29.550594+0000","last_fullsized":"2024-09-14T04:46:29.550594+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:07:22.159128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1401,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.373587+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T04:46:02.373587+0000","last_peered":"2024-09-14T04:46:02.373587+0000","last_clean":"2024-09-14T04:46:02.373587+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T04:46:02.373587+0000","last_undegraded":"2024-09-14T04:46:02.373587+0000","last_fullsized":"2024-09-14T04:46:02.373587+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"349'149","reported_seq":711,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.373344+0000","last_change":"2024-09-14T04:41:39.012977+0000","last_active":"2024-09-14T04:46:02.373344+0000","last_peered":"2024-09-14T04:46:02.373344+0000","last_clean":"2024-09-14T04:46:02.373344+0000","last_became_active":"2024-09-14T04:41:36.932552+0000","last_became_peered":"2024-09-14T04:41:36.932552+0000","last_unstale":"2024-09-14T04:46:02.373344+0000","last_undegraded":"2024-09-14T04:46:02.373344+0000","last_fullsized":"2024-09-14T04:46:02.373344+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.012879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.012879+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:16:40.080262+0000","scrub_duration":0.0051721620000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":91,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.550281+0000","last_change":"2024-09-14T04:42:49.945069+0000","last_active":"2024-09-14T04:46:29.550281+0000","last_peered":"2024-09-14T04:46:29.550281+0000","last_clean":"2024-09-14T04:46:29.550281+0000","last_became_active":"2024-09-14T04:42:49.943527+0000","last_became_peered":"2024-09-14T04:42:49.943527+0000","last_unstale":"2024-09-14T04:46:29.550281+0000","last_undegraded":"2024-09-14T04:46:29.550281+0000","last_fullsized":"2024-09-14T04:46:29.550281+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:39:10.524631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":22,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.372152+0000","last_change":"2024-09-14T04:44:57.329652+0000","last_active":"2024-09-14T04:46:02.372152+0000","last_peered":"2024-09-14T04:46:02.372152+0000","last_clean":"2024-09-14T04:46:02.372152+0000","last_became_active":"2024-09-14T04:44:57.328888+0000","last_became_peered":"2024-09-14T04:44:57.328888+0000","last_unstale":"2024-09-14T04:46:02.372152+0000","last_undegraded":"2024-09-14T04:46:02.372152+0000","last_fullsized":"2024-09-14T04:46:02.372152+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:10.170068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1400,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:03.218555+0000","last_change":"2024-09-14T04:06:46.875340+0000","last_active":"2024-09-14T04:46:03.218555+0000","last_peered":"2024-09-14T04:46:03.218555+0000","last_clean":"2024-09-14T04:46:03.218555+0000","last_became_active":"2024-09-14T04:06:46.874552+0000","last_became_peered":"2024-09-14T04:06:46.874552+0000","last_unstale":"2024-09-14T04:46:03.218555+0000","last_undegraded":"2024-09-14T04:46:03.218555+0000","last_fullsized":"2024-09-14T04:46:03.218555+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:52.257357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"349'149","reported_seq":1781,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.374574+0000","last_change":"2024-09-14T04:45:00.346627+0000","last_active":"2024-09-14T04:46:02.374574+0000","last_peered":"2024-09-14T04:46:02.374574+0000","last_clean":"2024-09-14T04:46:02.374574+0000","last_became_active":"2024-09-14T04:45:00.346286+0000","last_became_peered":"2024-09-14T04:45:00.346286+0000","last_unstale":"2024-09-14T04:46:02.374574+0000","last_undegraded":"2024-09-14T04:46:02.374574+0000","last_fullsized":"2024-09-14T04:46:02.374574+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:50.784648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:50.784648+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:07:58.441372+0000","scrub_duration":0.01141873,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":47,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.550582+0000","last_change":"2024-09-14T04:44:14.489235+0000","last_active":"2024-09-14T04:46:29.550582+0000","last_peered":"2024-09-14T04:46:29.550582+0000","last_clean":"2024-09-14T04:46:29.550582+0000","last_became_active":"2024-09-14T04:44:14.488950+0000","last_became_peered":"2024-09-14T04:44:14.488950+0000","last_unstale":"2024-09-14T04:46:29.550582+0000","last_undegraded":"2024-09-14T04:46:29.550582+0000","last_fullsized":"2024-09-14T04:46:29.550582+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:54.441144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":2117,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.549538+0000","last_change":"2024-09-14T04:07:21.685415+0000","last_active":"2024-09-14T04:46:29.549538+0000","last_peered":"2024-09-14T04:46:29.549538+0000","last_clean":"2024-09-14T04:46:29.549538+0000","last_became_active":"2024-09-14T04:07:21.683504+0000","last_became_peered":"2024-09-14T04:07:21.683504+0000","last_unstale":"2024-09-14T04:46:29.549538+0000","last_undegraded":"2024-09-14T04:46:29.549538+0000","last_fullsized":"2024-09-14T04:46:29.549538+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:21:08.806291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2119,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.560315+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T04:46:29.560315+0000","last_peered":"2024-09-14T04:46:29.560315+0000","last_clean":"2024-09-14T04:46:29.560315+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T04:46:29.560315+0000","last_undegraded":"2024-09-14T04:46:29.560315+0000","last_fullsized":"2024-09-14T04:46:29.560315+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"349'149","reported_seq":795,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.550742+0000","last_change":"2024-09-14T04:45:00.342826+0000","last_active":"2024-09-14T04:46:29.550742+0000","last_peered":"2024-09-14T04:46:29.550742+0000","last_clean":"2024-09-14T04:46:29.550742+0000","last_became_active":"2024-09-14T04:45:00.342013+0000","last_became_peered":"2024-09-14T04:45:00.342013+0000","last_unstale":"2024-09-14T04:46:29.550742+0000","last_undegraded":"2024-09-14T04:46:29.550742+0000","last_fullsized":"2024-09-14T04:46:29.550742+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.326129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.326129+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:54:28.021603+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2120,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.560080+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T04:46:29.560080+0000","last_peered":"2024-09-14T04:46:29.560080+0000","last_clean":"2024-09-14T04:46:29.560080+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T04:46:29.560080+0000","last_undegraded":"2024-09-14T04:46:29.560080+0000","last_fullsized":"2024-09-14T04:46:29.560080+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2127,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.559616+0000","last_change":"2024-09-14T04:44:57.335783+0000","last_active":"2024-09-14T04:46:29.559616+0000","last_peered":"2024-09-14T04:46:29.559616+0000","last_clean":"2024-09-14T04:46:29.559616+0000","last_became_active":"2024-09-14T04:44:57.332780+0000","last_became_peered":"2024-09-14T04:44:57.332780+0000","last_unstale":"2024-09-14T04:46:29.559616+0000","last_undegraded":"2024-09-14T04:46:29.559616+0000","last_fullsized":"2024-09-14T04:46:29.559616+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:57.457593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1430,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.374573+0000","last_change":"2024-09-14T04:42:49.939790+0000","last_active":"2024-09-14T04:46:02.374573+0000","last_peered":"2024-09-14T04:46:02.374573+0000","last_clean":"2024-09-14T04:46:02.374573+0000","last_became_active":"2024-09-14T04:42:49.939591+0000","last_became_peered":"2024-09-14T04:42:49.939591+0000","last_unstale":"2024-09-14T04:46:02.374573+0000","last_undegraded":"2024-09-14T04:46:02.374573+0000","last_fullsized":"2024-09-14T04:46:02.374573+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:23:16.790593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1393,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.372873+0000","last_change":"2024-09-14T04:44:57.326008+0000","last_active":"2024-09-14T04:46:02.372873+0000","last_peered":"2024-09-14T04:46:02.372873+0000","last_clean":"2024-09-14T04:46:02.372873+0000","last_became_active":"2024-09-14T04:44:57.325617+0000","last_became_peered":"2024-09-14T04:44:57.325617+0000","last_unstale":"2024-09-14T04:46:02.372873+0000","last_undegraded":"2024-09-14T04:46:02.372873+0000","last_fullsized":"2024-09-14T04:46:02.372873+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:15.637508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":682,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.373804+0000","last_change":"2024-09-14T04:43:13.354846+0000","last_active":"2024-09-14T04:46:02.373804+0000","last_peered":"2024-09-14T04:46:02.373804+0000","last_clean":"2024-09-14T04:46:02.373804+0000","last_became_active":"2024-09-14T04:43:13.354515+0000","last_became_peered":"2024-09-14T04:43:13.354515+0000","last_unstale":"2024-09-14T04:46:02.373804+0000","last_undegraded":"2024-09-14T04:46:02.373804+0000","last_fullsized":"2024-09-14T04:46:02.373804+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:17.889341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1399,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:03.218589+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T04:46:03.218589+0000","last_peered":"2024-09-14T04:46:03.218589+0000","last_clean":"2024-09-14T04:46:03.218589+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T04:46:03.218589+0000","last_undegraded":"2024-09-14T04:46:03.218589+0000","last_fullsized":"2024-09-14T04:46:03.218589+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":1736,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.374431+0000","last_change":"2024-09-14T04:41:30.640651+0000","last_active":"2024-09-14T04:46:02.374431+0000","last_peered":"2024-09-14T04:46:02.374431+0000","last_clean":"2024-09-14T04:46:02.374431+0000","last_became_active":"2024-09-14T04:41:29.532466+0000","last_became_peered":"2024-09-14T04:41:29.532466+0000","last_unstale":"2024-09-14T04:46:02.374431+0000","last_undegraded":"2024-09-14T04:46:02.374431+0000","last_fullsized":"2024-09-14T04:46:02.374431+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":0,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:30.640520+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:30.640520+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:27.776724+0000","scrub_duration":0.0028653139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"349'149","reported_seq":711,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.373701+0000","last_change":"2024-09-14T04:41:39.978080+0000","last_active":"2024-09-14T04:46:02.373701+0000","last_peered":"2024-09-14T04:46:02.373701+0000","last_clean":"2024-09-14T04:46:02.373701+0000","last_became_active":"2024-09-14T04:41:36.929279+0000","last_became_peered":"2024-09-14T04:41:36.929279+0000","last_unstale":"2024-09-14T04:46:02.373701+0000","last_undegraded":"2024-09-14T04:46:02.373701+0000","last_fullsized":"2024-09-14T04:46:02.373701+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.977999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.977999+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:44:06.383827+0000","scrub_duration":0.00192122,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":2099,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.550773+0000","last_change":"2024-09-14T04:44:57.330745+0000","last_active":"2024-09-14T04:46:29.550773+0000","last_peered":"2024-09-14T04:46:29.550773+0000","last_clean":"2024-09-14T04:46:29.550773+0000","last_became_active":"2024-09-14T04:44:57.330261+0000","last_became_peered":"2024-09-14T04:44:57.330261+0000","last_unstale":"2024-09-14T04:46:29.550773+0000","last_undegraded":"2024-09-14T04:46:29.550773+0000","last_fullsized":"2024-09-14T04:46:29.550773+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:24:08.870976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2103,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.548164+0000","last_change":"2024-09-14T04:42:49.941397+0000","last_active":"2024-09-14T04:46:29.548164+0000","last_peered":"2024-09-14T04:46:29.548164+0000","last_clean":"2024-09-14T04:46:29.548164+0000","last_became_active":"2024-09-14T04:42:49.941066+0000","last_became_peered":"2024-09-14T04:42:49.941066+0000","last_unstale":"2024-09-14T04:46:29.548164+0000","last_undegraded":"2024-09-14T04:46:29.548164+0000","last_fullsized":"2024-09-14T04:46:29.548164+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:30.777822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":22,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.372415+0000","last_change":"2024-09-14T04:44:57.326724+0000","last_active":"2024-09-14T04:46:02.372415+0000","last_peered":"2024-09-14T04:46:02.372415+0000","last_clean":"2024-09-14T04:46:02.372415+0000","last_became_active":"2024-09-14T04:44:57.326191+0000","last_became_peered":"2024-09-14T04:44:57.326191+0000","last_unstale":"2024-09-14T04:46:02.372415+0000","last_undegraded":"2024-09-14T04:46:02.372415+0000","last_fullsized":"2024-09-14T04:46:02.372415+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:46.060269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":22,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.372650+0000","last_change":"2024-09-14T04:44:57.328528+0000","last_active":"2024-09-14T04:46:02.372650+0000","last_peered":"2024-09-14T04:46:02.372650+0000","last_clean":"2024-09-14T04:46:02.372650+0000","last_became_active":"2024-09-14T04:44:57.327739+0000","last_became_peered":"2024-09-14T04:44:57.327739+0000","last_unstale":"2024-09-14T04:46:02.372650+0000","last_undegraded":"2024-09-14T04:46:02.372650+0000","last_fullsized":"2024-09-14T04:46:02.372650+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:04:58.294690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"349'149","reported_seq":742,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.373291+0000","last_change":"2024-09-14T04:42:53.013201+0000","last_active":"2024-09-14T04:46:02.373291+0000","last_peered":"2024-09-14T04:46:02.373291+0000","last_clean":"2024-09-14T04:46:02.373291+0000","last_became_active":"2024-09-14T04:42:53.012818+0000","last_became_peered":"2024-09-14T04:42:53.012818+0000","last_unstale":"2024-09-14T04:46:02.373291+0000","last_undegraded":"2024-09-14T04:46:02.373291+0000","last_fullsized":"2024-09-14T04:46:02.373291+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:56.738636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:56.738636+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:39:19.056067+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":22,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.372446+0000","last_change":"2024-09-14T04:44:57.323436+0000","last_active":"2024-09-14T04:46:02.372446+0000","last_peered":"2024-09-14T04:46:02.372446+0000","last_clean":"2024-09-14T04:46:02.372446+0000","last_became_active":"2024-09-14T04:44:57.323022+0000","last_became_peered":"2024-09-14T04:44:57.323022+0000","last_unstale":"2024-09-14T04:46:02.372446+0000","last_undegraded":"2024-09-14T04:46:02.372446+0000","last_fullsized":"2024-09-14T04:46:02.372446+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:22.775261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1428,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.372766+0000","last_change":"2024-09-14T04:44:57.322357+0000","last_active":"2024-09-14T04:46:02.372766+0000","last_peered":"2024-09-14T04:46:02.372766+0000","last_clean":"2024-09-14T04:46:02.372766+0000","last_became_active":"2024-09-14T04:44:57.321990+0000","last_became_peered":"2024-09-14T04:44:57.321990+0000","last_unstale":"2024-09-14T04:46:02.372766+0000","last_undegraded":"2024-09-14T04:46:02.372766+0000","last_fullsized":"2024-09-14T04:46:02.372766+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:09.658008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"349'149","reported_seq":1753,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.374511+0000","last_change":"2024-09-14T04:41:53.858076+0000","last_active":"2024-09-14T04:46:02.374511+0000","last_peered":"2024-09-14T04:46:02.374511+0000","last_clean":"2024-09-14T04:46:02.374511+0000","last_became_active":"2024-09-14T04:41:36.931585+0000","last_became_peered":"2024-09-14T04:41:36.931585+0000","last_unstale":"2024-09-14T04:46:02.374511+0000","last_undegraded":"2024-09-14T04:46:02.374511+0000","last_fullsized":"2024-09-14T04:46:02.374511+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:53.857924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:53.857924+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:10:20.658986+0000","scrub_duration":0.0022355919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":22,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.372551+0000","last_change":"2024-09-14T04:44:57.329198+0000","last_active":"2024-09-14T04:46:02.372551+0000","last_peered":"2024-09-14T04:46:02.372551+0000","last_clean":"2024-09-14T04:46:02.372551+0000","last_became_active":"2024-09-14T04:44:57.327754+0000","last_became_peered":"2024-09-14T04:44:57.327754+0000","last_unstale":"2024-09-14T04:46:02.372551+0000","last_undegraded":"2024-09-14T04:46:02.372551+0000","last_fullsized":"2024-09-14T04:46:02.372551+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:10:27.944476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":104,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.561100+0000","last_change":"2024-09-14T04:44:57.348687+0000","last_active":"2024-09-14T04:46:29.561100+0000","last_peered":"2024-09-14T04:46:29.561100+0000","last_clean":"2024-09-14T04:46:29.561100+0000","last_became_active":"2024-09-14T04:44:57.332845+0000","last_became_peered":"2024-09-14T04:44:57.332845+0000","last_unstale":"2024-09-14T04:46:29.561100+0000","last_undegraded":"2024-09-14T04:46:29.561100+0000","last_fullsized":"2024-09-14T04:46:29.561100+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:12:18.762737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2127,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.560784+0000","last_change":"2024-09-14T04:44:57.333468+0000","last_active":"2024-09-14T04:46:29.560784+0000","last_peered":"2024-09-14T04:46:29.560784+0000","last_clean":"2024-09-14T04:46:29.560784+0000","last_became_active":"2024-09-14T04:44:57.332626+0000","last_became_peered":"2024-09-14T04:44:57.332626+0000","last_unstale":"2024-09-14T04:46:29.560784+0000","last_undegraded":"2024-09-14T04:46:29.560784+0000","last_fullsized":"2024-09-14T04:46:29.560784+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:57:26.366438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1384,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.373725+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T04:46:02.373725+0000","last_peered":"2024-09-14T04:46:02.373725+0000","last_clean":"2024-09-14T04:46:02.373725+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T04:46:02.373725+0000","last_undegraded":"2024-09-14T04:46:02.373725+0000","last_fullsized":"2024-09-14T04:46:02.373725+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1383,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.372621+0000","last_change":"2024-09-14T04:06:46.869868+0000","last_active":"2024-09-14T04:46:02.372621+0000","last_peered":"2024-09-14T04:46:02.372621+0000","last_clean":"2024-09-14T04:46:02.372621+0000","last_became_active":"2024-09-14T04:06:46.869126+0000","last_became_peered":"2024-09-14T04:06:46.869126+0000","last_unstale":"2024-09-14T04:46:02.372621+0000","last_undegraded":"2024-09-14T04:46:02.372621+0000","last_fullsized":"2024-09-14T04:46:02.372621+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:41.150138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1367,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.375017+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T04:46:02.375017+0000","last_peered":"2024-09-14T04:46:02.375017+0000","last_clean":"2024-09-14T04:46:02.375017+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T04:46:02.375017+0000","last_undegraded":"2024-09-14T04:46:02.375017+0000","last_fullsized":"2024-09-14T04:46:02.375017+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":22,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.372141+0000","last_change":"2024-09-14T04:44:57.332112+0000","last_active":"2024-09-14T04:46:02.372141+0000","last_peered":"2024-09-14T04:46:02.372141+0000","last_clean":"2024-09-14T04:46:02.372141+0000","last_became_active":"2024-09-14T04:44:57.331575+0000","last_became_peered":"2024-09-14T04:44:57.331575+0000","last_unstale":"2024-09-14T04:46:02.372141+0000","last_undegraded":"2024-09-14T04:46:02.372141+0000","last_fullsized":"2024-09-14T04:46:02.372141+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:50:00.391283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1415,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.375030+0000","last_change":"2024-09-14T04:44:57.351460+0000","last_active":"2024-09-14T04:46:02.375030+0000","last_peered":"2024-09-14T04:46:02.375030+0000","last_clean":"2024-09-14T04:46:02.375030+0000","last_became_active":"2024-09-14T04:44:57.334662+0000","last_became_peered":"2024-09-14T04:44:57.334662+0000","last_unstale":"2024-09-14T04:46:02.375030+0000","last_undegraded":"2024-09-14T04:46:02.375030+0000","last_fullsized":"2024-09-14T04:46:02.375030+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:02.399062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1369,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.373372+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T04:46:02.373372+0000","last_peered":"2024-09-14T04:46:02.373372+0000","last_clean":"2024-09-14T04:46:02.373372+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T04:46:02.373372+0000","last_undegraded":"2024-09-14T04:46:02.373372+0000","last_fullsized":"2024-09-14T04:46:02.373372+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1427,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:02.374939+0000","last_change":"2024-09-14T04:44:57.349690+0000","last_active":"2024-09-14T04:46:02.374939+0000","last_peered":"2024-09-14T04:46:02.374939+0000","last_clean":"2024-09-14T04:46:02.374939+0000","last_became_active":"2024-09-14T04:44:57.333579+0000","last_became_peered":"2024-09-14T04:44:57.333579+0000","last_unstale":"2024-09-14T04:46:02.374939+0000","last_undegraded":"2024-09-14T04:46:02.374939+0000","last_fullsized":"2024-09-14T04:46:02.374939+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:17:55.265141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1429,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.554933+0000","last_change":"2024-09-14T04:42:49.960838+0000","last_active":"2024-09-14T04:46:29.554933+0000","last_peered":"2024-09-14T04:46:29.554933+0000","last_clean":"2024-09-14T04:46:29.554933+0000","last_became_active":"2024-09-14T04:42:49.954736+0000","last_became_peered":"2024-09-14T04:42:49.954736+0000","last_unstale":"2024-09-14T04:46:29.554933+0000","last_undegraded":"2024-09-14T04:46:29.554933+0000","last_fullsized":"2024-09-14T04:46:29.554933+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:41:13.540983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":2086,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.556052+0000","last_change":"2024-09-14T04:07:58.565442+0000","last_active":"2024-09-14T04:46:29.556052+0000","last_peered":"2024-09-14T04:46:29.556052+0000","last_clean":"2024-09-14T04:46:29.556052+0000","last_became_active":"2024-09-14T04:07:58.552289+0000","last_became_peered":"2024-09-14T04:07:58.552289+0000","last_unstale":"2024-09-14T04:46:29.556052+0000","last_undegraded":"2024-09-14T04:46:29.556052+0000","last_fullsized":"2024-09-14T04:46:29.556052+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:48:02.364960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1390,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.546752+0000","last_change":"2024-09-14T04:44:57.332511+0000","last_active":"2024-09-14T04:46:29.546752+0000","last_peered":"2024-09-14T04:46:29.546752+0000","last_clean":"2024-09-14T04:46:29.546752+0000","last_became_active":"2024-09-14T04:44:57.332166+0000","last_became_peered":"2024-09-14T04:44:57.332166+0000","last_unstale":"2024-09-14T04:46:29.546752+0000","last_undegraded":"2024-09-14T04:46:29.546752+0000","last_fullsized":"2024-09-14T04:46:29.546752+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:53.984639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2106,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.559839+0000","last_change":"2024-09-14T04:07:58.560880+0000","last_active":"2024-09-14T04:46:29.559839+0000","last_peered":"2024-09-14T04:46:29.559839+0000","last_clean":"2024-09-14T04:46:29.559839+0000","last_became_active":"2024-09-14T04:07:58.560132+0000","last_became_peered":"2024-09-14T04:07:58.560132+0000","last_unstale":"2024-09-14T04:46:29.559839+0000","last_undegraded":"2024-09-14T04:46:29.559839+0000","last_fullsized":"2024-09-14T04:46:29.559839+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:46:00.287050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2106,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.559598+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T04:46:29.559598+0000","last_peered":"2024-09-14T04:46:29.559598+0000","last_clean":"2024-09-14T04:46:29.559598+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T04:46:29.559598+0000","last_undegraded":"2024-09-14T04:46:29.559598+0000","last_fullsized":"2024-09-14T04:46:29.559598+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1382,"reported_epoch":1413,"state":"active+clean","last_fresh":"2024-09-14T04:46:13.227338+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T04:46:13.227338+0000","last_peered":"2024-09-14T04:46:13.227338+0000","last_clean":"2024-09-14T04:46:13.227338+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T04:46:13.227338+0000","last_undegraded":"2024-09-14T04:46:13.227338+0000","last_fullsized":"2024-09-14T04:46:13.227338+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1419,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.547494+0000","last_change":"2024-09-14T04:44:57.339993+0000","last_active":"2024-09-14T04:46:29.547494+0000","last_peered":"2024-09-14T04:46:29.547494+0000","last_clean":"2024-09-14T04:46:29.547494+0000","last_became_active":"2024-09-14T04:44:57.330472+0000","last_became_peered":"2024-09-14T04:44:57.330472+0000","last_unstale":"2024-09-14T04:46:29.547494+0000","last_undegraded":"2024-09-14T04:46:29.547494+0000","last_fullsized":"2024-09-14T04:46:29.547494+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:19:27.046988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":47,"reported_epoch":1414,"state":"active+clean","last_fresh":"2024-09-14T04:46:29.549807+0000","last_change":"2024-09-14T04:44:14.479373+0000","last_active":"2024-09-14T04:46:29.549807+0000","last_peered":"2024-09-14T04:46:29.549807+0000","last_clean":"2024-09-14T04:46:29.549807+0000","last_became_active":"2024-09-14T04:44:14.479084+0000","last_became_peered":"2024-09-14T04:44:14.479084+0000","last_unstale":"2024-09-14T04:46:29.549807+0000","last_undegraded":"2024-09-14T04:46:29.549807+0000","last_fullsized":"2024-09-14T04:46:29.549807+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:49:23.412806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":54,"num_pg":16,"stat_sum":{"num_bytes":1856765976,"num_objects":28333,"num_object_clones":0,"num_object_copies":84999,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":10440,"num_objects_unfound":0,"num_objects_dirty":28333,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":224457,"num_write_kb":8089217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2036,"num_bytes_recovered":133365760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5559169024,"data_stored":5559156808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122074,"ondisk_log_size":122074,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":5070,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4250,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201950,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1138828,"kb_used_data":819012,"kb_used_omap":1,"kb_used_meta":319806,"kb_avail":92606324,"statfs":{"total":95995035648,"available":94828875776,"internally_reserved":0,"allocated":838668288,"data_stored":834181754,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":327481803},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463586,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1322056,"kb_used_data":903680,"kb_used_omap":1,"kb_used_meta":418238,"kb_avail":92423096,"statfs":{"total":95995035648,"available":94641250304,"internally_reserved":0,"allocated":925368320,"data_stored":920885779,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":428276167},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725223,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1305216,"kb_used_data":899256,"kb_used_omap":1,"kb_used_meta":405950,"kb_avail":92439936,"statfs":{"total":95995035648,"available":94658494464,"internally_reserved":0,"allocated":920838144,"data_stored":916360141,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":415693254},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":1392,"seq":5978594476069,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":101836,"kb_used_data":10820,"kb_used_omap":117,"kb_used_meta":90890,"kb_avail":93643316,"statfs":{"total":95995035648,"available":95890755584,"internally_reserved":0,"allocated":11079680,"data_stored":6593698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120638,"internal_metadata":93071554},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772074964,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1708224,"kb_used_data":1284664,"kb_used_omap":71,"kb_used_meta":423480,"kb_avail":92036928,"statfs":{"total":95995035648,"available":94245814272,"internally_reserved":0,"allocated":1315495936,"data_stored":1311018010,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":433644470},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707303899,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":593404,"kb_used_data":380852,"kb_used_omap":68,"kb_used_meta":212475,"kb_avail":93151748,"statfs":{"total":95995035648,"available":95387389952,"internally_reserved":0,"allocated":389992448,"data_stored":385518498,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":217574608},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872729058,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":843060,"kb_used_data":504236,"kb_used_omap":67,"kb_used_meta":338748,"kb_avail":92902092,"statfs":{"total":95995035648,"available":95131742208,"internally_reserved":0,"allocated":516337664,"data_stored":511870765,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":346878309},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056109,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1190588,"kb_used_data":815860,"kb_used_omap":74,"kb_used_meta":374645,"kb_avail":92554564,"statfs":{"total":95995035648,"available":94775873536,"internally_reserved":0,"allocated":835440640,"data_stored":830966158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":383637173},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,2,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1901,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":805507072,"data_stored":805503000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":495190016,"data_stored":495190016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":377487360,"data_stored":377487360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1278873600,"data_stored":1278869528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":458752,"data_stored":458752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":893648896,"data_stored":893648896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":903024640,"data_stored":903020568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":804978688,"data_stored":804978688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:46:55.419 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph status --format=json 2024-09-14T04:46:55.419 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:55.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:55.450 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:55.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:55.550 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:55.590 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:55 smithi039 ceph-mon[106470]: pgmap v2503: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 9549/73314 objects misplaced (13.025%); 1.4 MiB/s, 22 objects/s recovering 2024-09-14T04:46:55.590 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:55 smithi039 ceph-mon[111235]: pgmap v2503: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 9549/73314 objects misplaced (13.025%); 1.4 MiB/s, 22 objects/s recovering 2024-09-14T04:46:55.599 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:55.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:55.600 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:55.700 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:55.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:55.743 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:55.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:55.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:55 smithi129 ceph-mon[101316]: pgmap v2503: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 9549/73314 objects misplaced (13.025%); 1.4 MiB/s, 22 objects/s recovering 2024-09-14T04:46:55.844 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:55.870 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:46:55.879 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:55.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:55.880 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:55.981 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:56.007 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:56.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:56.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:56.059 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:46:56.059 INFO:teuthology.orchestra.run.smithi039.stdout: "size": 20, 2024-09-14T04:46:56.059 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 600, 2024-09-14T04:46:56.059 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [ 2024-09-14T04:46:56.060 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.060 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233351 53.7 53:ed868572:::benchmark_data_smithi039_224949_object103258:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1355)", 2024-09-14T04:46:56.060 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.814699+0000", 2024-09-14T04:46:56.060 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 291.24001043499999, 2024-09-14T04:46:56.060 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1969449910000001, 2024-09-14T04:46:56.060 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:56.060 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:56.060 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T04:46:56.060 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:46:56.060 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:46:56.060 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:46:56.060 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233351 2024-09-14T04:46:56.060 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.061 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:56.061 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.061 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:56.061 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814699+0000", 2024-09-14T04:46:56.061 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.061 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.061 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.061 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:56.061 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814699+0000", 2024-09-14T04:46:56.061 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.061 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.061 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.062 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:56.062 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814700+0000", 2024-09-14T04:46:56.062 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6499999999999998e-07 2024-09-14T04:46:56.062 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.062 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.062 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:56.062 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814703+0000", 2024-09-14T04:46:56.062 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.1310000000000002e-06 2024-09-14T04:46:56.062 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.062 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.062 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:56.062 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814712+0000", 2024-09-14T04:46:56.062 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.4749999999999993e-06 2024-09-14T04:46:56.062 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.063 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.063 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:56.063 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814718+0000", 2024-09-14T04:46:56.063 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.5089999999999999e-06 2024-09-14T04:46:56.063 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.063 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.063 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.063 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.815266+0000", 2024-09-14T04:46:56.063 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00054806600000000005 2024-09-14T04:46:56.063 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.063 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.063 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op must wait for map", 2024-09-14T04:46:56.063 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.815274+0000", 2024-09-14T04:46:56.063 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.8630000000000008e-06 2024-09-14T04:46:56.064 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.064 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.064 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.064 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.830888+0000", 2024-09-14T04:46:56.064 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.015613846000000001 2024-09-14T04:46:56.064 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.064 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.064 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:46:56.064 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.830900+0000", 2024-09-14T04:46:56.064 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1911e-05 2024-09-14T04:46:56.064 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.064 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.064 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.065 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.814346+0000", 2024-09-14T04:46:56.065 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.98344564700000003 2024-09-14T04:46:56.065 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.065 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.065 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:46:56.065 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.814359+0000", 2024-09-14T04:46:56.065 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3509999999999999e-05 2024-09-14T04:46:56.065 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.065 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.170 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.170 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.009662+0000", 2024-09-14T04:46:56.170 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.195302284 2024-09-14T04:46:56.170 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.170 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.170 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:56.170 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.011644+0000", 2024-09-14T04:46:56.171 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0019829840000000001 2024-09-14T04:46:56.171 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.171 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:56.171 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.171 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.171 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.171 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233354 53.7 53:e0779eb3:::benchmark_data_smithi039_224949_object103261:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1355)", 2024-09-14T04:46:56.171 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.814735+0000", 2024-09-14T04:46:56.171 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 291.23997458700001, 2024-09-14T04:46:56.171 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2280154030000001, 2024-09-14T04:46:56.171 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:56.171 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:56.171 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:56.171 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:46:56.171 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:46:56.172 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:46:56.172 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233354 2024-09-14T04:46:56.172 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.172 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:56.172 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.172 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:56.172 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814735+0000", 2024-09-14T04:46:56.172 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.172 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.172 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.172 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:56.172 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814735+0000", 2024-09-14T04:46:56.172 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.173 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.173 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.173 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:56.173 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814736+0000", 2024-09-14T04:46:56.173 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.0200000000000002e-07 2024-09-14T04:46:56.173 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.173 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.173 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:56.173 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814738+0000", 2024-09-14T04:46:56.173 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.43e-06 2024-09-14T04:46:56.174 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.174 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.174 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:56.174 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814747+0000", 2024-09-14T04:46:56.174 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.2390000000000002e-06 2024-09-14T04:46:56.174 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.174 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.174 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:56.174 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814753+0000", 2024-09-14T04:46:56.175 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.6510000000000004e-06 2024-09-14T04:46:56.175 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.175 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.175 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.175 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.820795+0000", 2024-09-14T04:46:56.175 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0060417639999999998 2024-09-14T04:46:56.175 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.175 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.175 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting_for_map not empty", 2024-09-14T04:46:56.175 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.820801+0000", 2024-09-14T04:46:56.175 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.9739999999999999e-06 2024-09-14T04:46:56.175 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.175 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.175 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.176 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.830925+0000", 2024-09-14T04:46:56.176 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.010123996 2024-09-14T04:46:56.176 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.176 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.176 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:46:56.176 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.830931+0000", 2024-09-14T04:46:56.176 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.421e-06 2024-09-14T04:46:56.176 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.176 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.176 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.176 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.814390+0000", 2024-09-14T04:46:56.176 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.98345883099999998 2024-09-14T04:46:56.176 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.176 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.177 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:46:56.177 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.814398+0000", 2024-09-14T04:46:56.177 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.5159999999999997e-06 2024-09-14T04:46:56.177 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.177 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.177 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.177 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.011957+0000", 2024-09-14T04:46:56.177 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1975596669999999 2024-09-14T04:46:56.177 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.177 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.177 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:56.178 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.012683+0000", 2024-09-14T04:46:56.178 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00072578599999999997 2024-09-14T04:46:56.178 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.178 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.178 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 0,1,7", 2024-09-14T04:46:56.178 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.013085+0000", 2024-09-14T04:46:56.178 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00040193299999999999 2024-09-14T04:46:56.178 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.178 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.178 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.179 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.031842+0000", 2024-09-14T04:46:56.179 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.018757149000000001 2024-09-14T04:46:56.179 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.179 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.179 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:46:56.179 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.035830+0000", 2024-09-14T04:46:56.179 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0039881919999999998 2024-09-14T04:46:56.179 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.179 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.179 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.179 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.038481+0000", 2024-09-14T04:46:56.180 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0026504660000000002 2024-09-14T04:46:56.180 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.180 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.180 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.180 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.042512+0000", 2024-09-14T04:46:56.180 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.004031247 2024-09-14T04:46:56.180 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.180 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.180 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:56.181 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.042625+0000", 2024-09-14T04:46:56.181 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000112434 2024-09-14T04:46:56.181 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.181 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.181 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:56.181 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.042751+0000", 2024-09-14T04:46:56.181 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000126205 2024-09-14T04:46:56.181 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.181 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:56.181 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.181 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.181 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.181 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233359 53.7 53:e8733a52:::benchmark_data_smithi039_224949_object103266:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-09-14T04:46:56.182 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.820832+0000", 2024-09-14T04:46:56.182 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 291.23387772199999, 2024-09-14T04:46:56.182 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2221726209999999, 2024-09-14T04:46:56.182 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:56.182 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:56.182 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:56.182 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:46:56.182 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:46:56.182 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:46:56.182 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233359 2024-09-14T04:46:56.182 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.183 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:56.183 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.183 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:56.183 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.820832+0000", 2024-09-14T04:46:56.183 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.183 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.183 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.183 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:56.183 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.820832+0000", 2024-09-14T04:46:56.183 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.183 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.183 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.183 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:56.183 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.820834+0000", 2024-09-14T04:46:56.184 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7010000000000001e-06 2024-09-14T04:46:56.184 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.184 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.184 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:56.184 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.820839+0000", 2024-09-14T04:46:56.184 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6890000000000003e-06 2024-09-14T04:46:56.184 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.184 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.184 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:56.184 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.820848+0000", 2024-09-14T04:46:56.184 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.5710000000000004e-06 2024-09-14T04:46:56.184 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.185 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.185 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:56.185 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.820856+0000", 2024-09-14T04:46:56.185 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6329999999999996e-06 2024-09-14T04:46:56.185 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.185 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.185 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.185 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.830952+0000", 2024-09-14T04:46:56.185 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.010096519 2024-09-14T04:46:56.185 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.185 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.185 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:46:56.185 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.830959+0000", 2024-09-14T04:46:56.186 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.426e-06 2024-09-14T04:46:56.186 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.186 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.186 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.186 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.814422+0000", 2024-09-14T04:46:56.186 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.98346295299999997 2024-09-14T04:46:56.186 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.186 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.186 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:46:56.186 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.814430+0000", 2024-09-14T04:46:56.186 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9899999999999997e-06 2024-09-14T04:46:56.187 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.187 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.187 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.187 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.028595+0000", 2024-09-14T04:46:56.187 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.214164939 2024-09-14T04:46:56.187 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.187 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.187 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:56.187 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.028728+0000", 2024-09-14T04:46:56.187 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000133213 2024-09-14T04:46:56.187 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.187 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.187 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 0,1,7", 2024-09-14T04:46:56.187 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.028867+0000", 2024-09-14T04:46:56.187 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000138801 2024-09-14T04:46:56.188 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.188 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.188 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.188 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.031892+0000", 2024-09-14T04:46:56.188 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0030254040000000002 2024-09-14T04:46:56.188 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.188 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.188 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.188 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.040818+0000", 2024-09-14T04:46:56.188 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0089257309999999992 2024-09-14T04:46:56.188 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.188 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.188 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:46:56.189 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.041553+0000", 2024-09-14T04:46:56.189 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00073544599999999995 2024-09-14T04:46:56.189 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.189 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.189 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.189 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.042807+0000", 2024-09-14T04:46:56.189 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0012535210000000001 2024-09-14T04:46:56.189 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.189 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.189 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:56.189 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.042903+0000", 2024-09-14T04:46:56.189 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.6471999999999994e-05 2024-09-14T04:46:56.189 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.189 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.190 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:56.190 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.043005+0000", 2024-09-14T04:46:56.190 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000101612 2024-09-14T04:46:56.190 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.190 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:56.190 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.190 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.190 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.190 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233393 53.7 53:e9d8059a:::benchmark_data_smithi039_224949_object103300:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-09-14T04:46:56.190 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.894168+0000", 2024-09-14T04:46:56.190 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 291.16054211900001, 2024-09-14T04:46:56.190 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.149336516, 2024-09-14T04:46:56.191 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:56.191 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:56.191 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:56.191 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:46:56.191 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:46:56.191 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:46:56.191 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233393 2024-09-14T04:46:56.191 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.191 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:56.191 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.191 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:56.191 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894168+0000", 2024-09-14T04:46:56.191 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.191 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.191 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.192 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:56.192 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894168+0000", 2024-09-14T04:46:56.192 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.192 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.192 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.192 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:56.192 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894169+0000", 2024-09-14T04:46:56.192 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.443e-06 2024-09-14T04:46:56.192 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.192 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.192 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:56.192 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894173+0000", 2024-09-14T04:46:56.192 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.5149999999999998e-06 2024-09-14T04:46:56.193 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.193 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.193 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:56.193 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894182+0000", 2024-09-14T04:46:56.193 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.3360000000000001e-06 2024-09-14T04:46:56.193 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.193 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.193 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:56.193 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894188+0000", 2024-09-14T04:46:56.193 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3439999999999997e-06 2024-09-14T04:46:56.193 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.193 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.193 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.193 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894306+0000", 2024-09-14T04:46:56.194 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000117439 2024-09-14T04:46:56.194 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.194 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.194 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:46:56.194 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894316+0000", 2024-09-14T04:46:56.194 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0349e-05 2024-09-14T04:46:56.194 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.194 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.194 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.194 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:56.197 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.814569+0000", 2024-09-14T04:46:56.197 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.92025298899999997 2024-09-14T04:46:56.197 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.197 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.197 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:46:56.197 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.814584+0000", 2024-09-14T04:46:56.197 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.468e-05 2024-09-14T04:46:56.197 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.197 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.197 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.198 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.032234+0000", 2024-09-14T04:46:56.198 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2176502870000001 2024-09-14T04:46:56.198 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.198 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.198 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:56.198 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.032359+0000", 2024-09-14T04:46:56.198 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000124533 2024-09-14T04:46:56.198 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.198 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.198 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 0,1,7", 2024-09-14T04:46:56.198 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.032518+0000", 2024-09-14T04:46:56.198 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015903900000000001 2024-09-14T04:46:56.198 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.198 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.199 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.199 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.040779+0000", 2024-09-14T04:46:56.199 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0082608909999999994 2024-09-14T04:46:56.199 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.199 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.199 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.199 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.041107+0000", 2024-09-14T04:46:56.199 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000328778 2024-09-14T04:46:56.199 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.199 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.199 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:46:56.199 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.041689+0000", 2024-09-14T04:46:56.200 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00058127799999999996 2024-09-14T04:46:56.200 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.200 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.200 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.200 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.043303+0000", 2024-09-14T04:46:56.200 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001614297 2024-09-14T04:46:56.200 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.200 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.200 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:56.200 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.043390+0000", 2024-09-14T04:46:56.200 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.6940000000000004e-05 2024-09-14T04:46:56.200 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.200 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.200 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:56.201 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.043504+0000", 2024-09-14T04:46:56.201 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011437799999999999 2024-09-14T04:46:56.201 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.201 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:56.201 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.201 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.201 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.201 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242867 53.a 53:54ad9484:::benchmark_data_smithi039_224949_object112774:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1363)", 2024-09-14T04:46:56.201 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:20.495372+0000", 2024-09-14T04:46:56.201 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 275.55933800999998, 2024-09-14T04:46:56.201 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.997320294, 2024-09-14T04:46:56.201 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:56.202 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:56.202 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:56.202 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:46:56.202 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:46:56.202 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:46:56.202 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242867 2024-09-14T04:46:56.202 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.202 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:56.202 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.202 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:56.202 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.495372+0000", 2024-09-14T04:46:56.202 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.202 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.202 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.203 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:56.203 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.495372+0000", 2024-09-14T04:46:56.203 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.203 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.203 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.203 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:56.203 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.495373+0000", 2024-09-14T04:46:56.203 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.850000000000001e-07 2024-09-14T04:46:56.203 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.203 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.203 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:56.203 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.495377+0000", 2024-09-14T04:46:56.203 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5059999999999997e-06 2024-09-14T04:46:56.203 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.203 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.204 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:56.204 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.495390+0000", 2024-09-14T04:46:56.204 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2498e-05 2024-09-14T04:46:56.204 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.204 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.204 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:56.204 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.495397+0000", 2024-09-14T04:46:56.204 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.2060000000000001e-06 2024-09-14T04:46:56.204 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.204 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.204 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.204 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.495443+0000", 2024-09-14T04:46:56.204 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6139e-05 2024-09-14T04:46:56.204 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.205 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.205 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:46:56.205 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.495449+0000", 2024-09-14T04:46:56.205 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5949999999999998e-06 2024-09-14T04:46:56.205 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.205 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.205 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.205 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.479903+0000", 2024-09-14T04:46:56.205 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.98445407699999998 2024-09-14T04:46:56.205 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.205 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.205 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:56.205 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.480015+0000", 2024-09-14T04:46:56.205 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011235600000000001 2024-09-14T04:46:56.205 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.206 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.206 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:46:56.206 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.480135+0000", 2024-09-14T04:46:56.206 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000119462 2024-09-14T04:46:56.206 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.206 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.206 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.206 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.484482+0000", 2024-09-14T04:46:56.206 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0043470779999999999 2024-09-14T04:46:56.206 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.206 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.206 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.206 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.484532+0000", 2024-09-14T04:46:56.206 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9981999999999997e-05 2024-09-14T04:46:56.206 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.207 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.207 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:46:56.207 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.492566+0000", 2024-09-14T04:46:56.207 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0080345810000000007 2024-09-14T04:46:56.207 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.207 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.207 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:56.207 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.492646+0000", 2024-09-14T04:46:56.207 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0039999999999999e-05 2024-09-14T04:46:56.207 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.207 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.207 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:56.207 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.492692+0000", 2024-09-14T04:46:56.207 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5788999999999998e-05 2024-09-14T04:46:56.207 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.208 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:56.208 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.208 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.208 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.208 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242882 53.a 53:5478a5c5:::benchmark_data_smithi039_224949_object112789:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1363)", 2024-09-14T04:46:56.208 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:20.512060+0000", 2024-09-14T04:46:56.208 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 275.54265015700003, 2024-09-14T04:46:56.208 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0040218409999999, 2024-09-14T04:46:56.208 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:56.208 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:56.208 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:56.208 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:46:56.208 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:46:56.208 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:46:56.209 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242882 2024-09-14T04:46:56.209 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.209 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:56.209 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.209 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:56.209 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.512060+0000", 2024-09-14T04:46:56.209 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.209 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.209 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.209 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:56.209 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.512060+0000", 2024-09-14T04:46:56.209 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.209 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.209 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.210 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:56.210 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.512061+0000", 2024-09-14T04:46:56.210 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6869999999999999e-06 2024-09-14T04:46:56.210 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.210 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.210 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:56.210 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.512067+0000", 2024-09-14T04:46:56.210 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.993e-06 2024-09-14T04:46:56.210 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.210 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.210 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:56.210 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.512078+0000", 2024-09-14T04:46:56.210 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0913999999999999e-05 2024-09-14T04:46:56.210 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.210 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.211 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:56.211 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.512085+0000", 2024-09-14T04:46:56.211 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7109999999999998e-06 2024-09-14T04:46:56.211 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.211 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.211 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.211 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.512156+0000", 2024-09-14T04:46:56.211 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.0850000000000001e-05 2024-09-14T04:46:56.211 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.211 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.211 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:46:56.211 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.512164+0000", 2024-09-14T04:46:56.211 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.4530000000000002e-06 2024-09-14T04:46:56.211 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.211 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.212 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.212 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.484560+0000", 2024-09-14T04:46:56.212 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.972395491 2024-09-14T04:46:56.212 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.212 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.212 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:56.212 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.484663+0000", 2024-09-14T04:46:56.212 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00010302300000000001 2024-09-14T04:46:56.212 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.212 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.212 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:46:56.212 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.484869+0000", 2024-09-14T04:46:56.212 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000206104 2024-09-14T04:46:56.212 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.212 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.213 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.213 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.492736+0000", 2024-09-14T04:46:56.213 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0078666750000000001 2024-09-14T04:46:56.213 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.213 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.213 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.213 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.492783+0000", 2024-09-14T04:46:56.213 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7630999999999998e-05 2024-09-14T04:46:56.213 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.213 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.213 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:46:56.213 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516001+0000", 2024-09-14T04:46:56.213 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.023217924000000001 2024-09-14T04:46:56.213 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.213 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.214 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:56.214 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516046+0000", 2024-09-14T04:46:56.214 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4833999999999998e-05 2024-09-14T04:46:56.214 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.214 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.214 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:56.214 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516082+0000", 2024-09-14T04:46:56.214 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.5550999999999999e-05 2024-09-14T04:46:56.214 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.214 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:56.214 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.214 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.214 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.214 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242920 53.a 53:5584c2e0:::benchmark_data_smithi039_224949_object112827:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1363)", 2024-09-14T04:46:56.214 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:20.566563+0000", 2024-09-14T04:46:56.215 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 275.48814720299998, 2024-09-14T04:46:56.215 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.94971936199999996, 2024-09-14T04:46:56.215 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:56.215 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:56.215 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:56.215 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:46:56.215 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:46:56.215 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:46:56.215 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242920 2024-09-14T04:46:56.215 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.215 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:56.215 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.215 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:56.215 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.566563+0000", 2024-09-14T04:46:56.216 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.216 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.216 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.216 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:56.216 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.566563+0000", 2024-09-14T04:46:56.216 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.216 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.216 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.216 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:56.216 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.566564+0000", 2024-09-14T04:46:56.216 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4890000000000001e-06 2024-09-14T04:46:56.216 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.216 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.216 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:56.216 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.566569+0000", 2024-09-14T04:46:56.217 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4750000000000004e-06 2024-09-14T04:46:56.217 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.217 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.217 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:56.217 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.566579+0000", 2024-09-14T04:46:56.217 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0421000000000001e-05 2024-09-14T04:46:56.217 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.217 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.217 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:56.217 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.566586+0000", 2024-09-14T04:46:56.217 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7449999999999998e-06 2024-09-14T04:46:56.217 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.217 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.217 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.217 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.566678+0000", 2024-09-14T04:46:56.218 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.2319999999999997e-05 2024-09-14T04:46:56.218 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.218 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.218 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:46:56.218 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.566686+0000", 2024-09-14T04:46:56.218 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9400000000000002e-06 2024-09-14T04:46:56.218 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.218 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.218 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.218 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.488450+0000", 2024-09-14T04:46:56.218 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.92176433300000005 2024-09-14T04:46:56.218 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.218 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.218 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:56.218 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.488608+0000", 2024-09-14T04:46:56.219 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015782400000000001 2024-09-14T04:46:56.219 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.219 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.219 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:46:56.219 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.488674+0000", 2024-09-14T04:46:56.219 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.5457999999999996e-05 2024-09-14T04:46:56.219 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.219 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.219 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.219 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.492855+0000", 2024-09-14T04:46:56.219 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0041807909999999997 2024-09-14T04:46:56.219 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.219 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.219 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.220 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.492909+0000", 2024-09-14T04:46:56.220 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4629999999999997e-05 2024-09-14T04:46:56.220 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.220 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.220 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:46:56.220 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516207+0000", 2024-09-14T04:46:56.220 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.023297549000000001 2024-09-14T04:46:56.220 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.220 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.220 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:56.220 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516246+0000", 2024-09-14T04:46:56.220 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.9242000000000001e-05 2024-09-14T04:46:56.220 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.220 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.221 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:56.221 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516282+0000", 2024-09-14T04:46:56.221 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.6145e-05 2024-09-14T04:46:56.221 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.221 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:56.221 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.221 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.221 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.221 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242925 53.a 53:5601ec35:::benchmark_data_smithi039_224949_object112832:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1363)", 2024-09-14T04:46:56.221 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:20.570988+0000", 2024-09-14T04:46:56.221 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 275.483722188, 2024-09-14T04:46:56.221 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.94542205099999999, 2024-09-14T04:46:56.221 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:56.221 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:56.222 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:56.222 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:46:56.222 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:46:56.222 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:46:56.222 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242925 2024-09-14T04:46:56.222 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.222 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:56.222 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.222 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:56.222 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.570988+0000", 2024-09-14T04:46:56.222 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.222 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.222 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.222 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:56.223 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.570988+0000", 2024-09-14T04:46:56.223 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.223 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.223 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.223 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:56.223 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.570990+0000", 2024-09-14T04:46:56.223 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8870000000000001e-06 2024-09-14T04:46:56.223 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.223 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.223 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:56.223 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.570995+0000", 2024-09-14T04:46:56.223 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7799999999999997e-06 2024-09-14T04:46:56.223 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.224 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.224 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:56.224 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.571008+0000", 2024-09-14T04:46:56.224 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2683e-05 2024-09-14T04:46:56.224 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.224 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.224 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:56.224 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.571016+0000", 2024-09-14T04:46:56.224 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.1880000000000002e-06 2024-09-14T04:46:56.224 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.224 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.224 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.224 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.571101+0000", 2024-09-14T04:46:56.224 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.4460000000000001e-05 2024-09-14T04:46:56.225 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.225 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.225 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:46:56.225 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.571110+0000", 2024-09-14T04:46:56.225 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.7700000000000007e-06 2024-09-14T04:46:56.225 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.225 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.225 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.225 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.492959+0000", 2024-09-14T04:46:56.226 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.92184951400000004 2024-09-14T04:46:56.226 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.226 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.226 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:56.226 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.493043+0000", 2024-09-14T04:46:56.226 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.3777999999999996e-05 2024-09-14T04:46:56.226 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.226 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.226 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:46:56.226 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.493120+0000", 2024-09-14T04:46:56.226 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7489000000000003e-05 2024-09-14T04:46:56.226 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.226 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.227 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.227 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.500548+0000", 2024-09-14T04:46:56.227 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0074275349999999999 2024-09-14T04:46:56.227 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.227 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.227 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.227 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.500616+0000", 2024-09-14T04:46:56.227 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7871000000000001e-05 2024-09-14T04:46:56.227 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.227 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.227 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:46:56.227 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516288+0000", 2024-09-14T04:46:56.227 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.015672654000000001 2024-09-14T04:46:56.228 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.228 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.228 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:56.228 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516331+0000", 2024-09-14T04:46:56.228 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2698999999999999e-05 2024-09-14T04:46:56.228 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.228 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.228 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:56.228 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516410+0000", 2024-09-14T04:46:56.228 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.8743000000000001e-05 2024-09-14T04:46:56.228 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.228 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:56.228 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.228 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.228 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.229 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242952 53.a 53:58b7a1db:::benchmark_data_smithi039_224949_object112859:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1363)", 2024-09-14T04:46:56.229 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:20.604696+0000", 2024-09-14T04:46:56.229 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 275.450014377, 2024-09-14T04:46:56.229 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.91180369299999997, 2024-09-14T04:46:56.229 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:56.229 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:56.229 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:56.229 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:46:56.229 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:46:56.229 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:46:56.229 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242952 2024-09-14T04:46:56.229 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.229 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:56.230 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.230 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:56.230 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.604696+0000", 2024-09-14T04:46:56.230 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.230 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.230 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.230 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:56.230 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.604696+0000", 2024-09-14T04:46:56.230 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.230 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.230 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.230 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:56.230 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.604697+0000", 2024-09-14T04:46:56.230 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8789999999999999e-06 2024-09-14T04:46:56.230 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.231 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.231 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:56.231 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.604703+0000", 2024-09-14T04:46:56.231 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7209999999999999e-06 2024-09-14T04:46:56.231 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.231 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.231 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:56.231 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.604721+0000", 2024-09-14T04:46:56.231 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7353000000000001e-05 2024-09-14T04:46:56.231 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.231 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.231 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:56.231 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.604728+0000", 2024-09-14T04:46:56.231 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7139999999999993e-06 2024-09-14T04:46:56.231 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.231 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.232 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.232 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.604805+0000", 2024-09-14T04:46:56.232 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6718999999999995e-05 2024-09-14T04:46:56.232 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.232 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.232 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:46:56.232 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.604814+0000", 2024-09-14T04:46:56.232 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.0589999999999992e-06 2024-09-14T04:46:56.232 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.232 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.232 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.232 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.496130+0000", 2024-09-14T04:46:56.232 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.89131583400000003 2024-09-14T04:46:56.232 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.232 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.233 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:56.233 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.496260+0000", 2024-09-14T04:46:56.233 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012988 2024-09-14T04:46:56.233 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.233 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.233 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:46:56.233 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.496352+0000", 2024-09-14T04:46:56.233 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.2159999999999999e-05 2024-09-14T04:46:56.233 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.233 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.233 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.233 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.500654+0000", 2024-09-14T04:46:56.233 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0043017810000000002 2024-09-14T04:46:56.234 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.234 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.234 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.234 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.500688+0000", 2024-09-14T04:46:56.234 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.4146000000000001e-05 2024-09-14T04:46:56.234 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.234 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.234 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:46:56.234 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516417+0000", 2024-09-14T04:46:56.234 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.015729097000000001 2024-09-14T04:46:56.234 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.234 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.234 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:56.234 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516462+0000", 2024-09-14T04:46:56.234 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5104e-05 2024-09-14T04:46:56.235 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.235 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.235 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:56.235 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516499+0000", 2024-09-14T04:46:56.235 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.7246000000000002e-05 2024-09-14T04:46:56.235 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.235 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:56.235 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.235 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.235 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.235 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242956 53.a 53:53d3e9bf:::benchmark_data_smithi039_224949_object112863:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1363)", 2024-09-14T04:46:56.235 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:20.610137+0000", 2024-09-14T04:46:56.235 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 275.444572666, 2024-09-14T04:46:56.235 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.90651257799999996, 2024-09-14T04:46:56.235 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:56.235 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:56.236 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:56.236 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:46:56.236 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:46:56.236 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:46:56.236 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242956 2024-09-14T04:46:56.236 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.236 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:56.236 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.236 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:56.236 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.610137+0000", 2024-09-14T04:46:56.236 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.236 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.236 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.236 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:56.236 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.610137+0000", 2024-09-14T04:46:56.237 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.237 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.237 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.237 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:56.237 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.610139+0000", 2024-09-14T04:46:56.237 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9520000000000001e-06 2024-09-14T04:46:56.237 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.237 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.237 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:56.237 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.610148+0000", 2024-09-14T04:46:56.237 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.6459999999999999e-06 2024-09-14T04:46:56.237 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.237 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.237 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:56.238 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.610160+0000", 2024-09-14T04:46:56.238 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2574e-05 2024-09-14T04:46:56.238 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.238 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.238 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:56.238 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.610168+0000", 2024-09-14T04:46:56.238 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6669999999999996e-06 2024-09-14T04:46:56.238 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.238 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.238 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.238 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.610230+0000", 2024-09-14T04:46:56.238 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1472999999999999e-05 2024-09-14T04:46:56.238 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.239 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.239 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:46:56.239 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.610239+0000", 2024-09-14T04:46:56.239 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.0880000000000001e-06 2024-09-14T04:46:56.239 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.239 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.239 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.239 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.500716+0000", 2024-09-14T04:46:56.239 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.89047721599999996 2024-09-14T04:46:56.239 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.239 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.239 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:56.239 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.500820+0000", 2024-09-14T04:46:56.239 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00010407 2024-09-14T04:46:56.240 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.240 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.240 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:46:56.240 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.500928+0000", 2024-09-14T04:46:56.240 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000107909 2024-09-14T04:46:56.240 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.240 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.240 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.240 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.512156+0000", 2024-09-14T04:46:56.240 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.011227849 2024-09-14T04:46:56.240 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.240 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.240 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.241 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.512195+0000", 2024-09-14T04:46:56.241 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.9739e-05 2024-09-14T04:46:56.241 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.241 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.241 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:46:56.241 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516505+0000", 2024-09-14T04:46:56.241 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0043095210000000002 2024-09-14T04:46:56.241 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.241 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.241 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:56.241 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516541+0000", 2024-09-14T04:46:56.241 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.5911999999999998e-05 2024-09-14T04:46:56.241 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.241 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.241 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:56.242 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516650+0000", 2024-09-14T04:46:56.242 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000108962 2024-09-14T04:46:56.242 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.242 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:56.242 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.242 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.242 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.242 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242984 53.a 53:5753476b:::benchmark_data_smithi039_224949_object112891:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1363)", 2024-09-14T04:46:56.242 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:20.654008+0000", 2024-09-14T04:46:56.242 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 275.40070208399999, 2024-09-14T04:46:56.242 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.86274647400000004, 2024-09-14T04:46:56.242 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:56.242 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:56.243 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:56.243 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:46:56.243 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:46:56.243 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:46:56.243 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242984 2024-09-14T04:46:56.243 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.243 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:56.243 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.243 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:56.243 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.654008+0000", 2024-09-14T04:46:56.243 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.243 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.243 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.243 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:56.243 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.654008+0000", 2024-09-14T04:46:56.244 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.244 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.244 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.244 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:56.244 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.654010+0000", 2024-09-14T04:46:56.244 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.753e-06 2024-09-14T04:46:56.244 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.244 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.244 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:56.244 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.654015+0000", 2024-09-14T04:46:56.244 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4160000000000003e-06 2024-09-14T04:46:56.244 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.244 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.244 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:56.245 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.654031+0000", 2024-09-14T04:46:56.245 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6458000000000001e-05 2024-09-14T04:46:56.245 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.245 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.245 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:56.245 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.654039+0000", 2024-09-14T04:46:56.245 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1019999999999999e-06 2024-09-14T04:46:56.245 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.245 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.245 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.245 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.654110+0000", 2024-09-14T04:46:56.245 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1265000000000003e-05 2024-09-14T04:46:56.245 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.245 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.245 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:46:56.246 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.654116+0000", 2024-09-14T04:46:56.246 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6910000000000002e-06 2024-09-14T04:46:56.246 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.246 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.246 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.246 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.503852+0000", 2024-09-14T04:46:56.246 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.84973631199999999 2024-09-14T04:46:56.246 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.246 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.246 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:56.246 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.503999+0000", 2024-09-14T04:46:56.246 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014705799999999999 2024-09-14T04:46:56.246 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.246 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.247 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:46:56.247 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.504119+0000", 2024-09-14T04:46:56.247 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000119982 2024-09-14T04:46:56.247 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.247 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.247 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.247 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.512233+0000", 2024-09-14T04:46:56.247 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0081145850000000005 2024-09-14T04:46:56.247 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.247 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.247 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.247 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.512271+0000", 2024-09-14T04:46:56.247 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.7440000000000001e-05 2024-09-14T04:46:56.247 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.248 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.248 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:46:56.248 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516656+0000", 2024-09-14T04:46:56.248 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0043848259999999997 2024-09-14T04:46:56.248 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.248 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.248 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:56.248 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516720+0000", 2024-09-14T04:46:56.248 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3862000000000006e-05 2024-09-14T04:46:56.248 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.248 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.248 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:56.248 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516754+0000", 2024-09-14T04:46:56.248 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.4724000000000002e-05 2024-09-14T04:46:56.249 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.249 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:56.249 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.249 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.249 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.249 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242992 53.a 53:5da6abf3:::benchmark_data_smithi039_224949_object112899:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1363)", 2024-09-14T04:46:56.249 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:20.682771+0000", 2024-09-14T04:46:56.249 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 275.37193893199998, 2024-09-14T04:46:56.249 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.83649015400000004, 2024-09-14T04:46:56.249 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:56.249 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:56.249 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:56.249 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:46:56.250 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:46:56.250 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:46:56.250 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242992 2024-09-14T04:46:56.250 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.250 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:56.250 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.250 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:56.250 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.682771+0000", 2024-09-14T04:46:56.250 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.250 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.250 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.250 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:56.250 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.682771+0000", 2024-09-14T04:46:56.250 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.251 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.251 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.251 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:56.251 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.682772+0000", 2024-09-14T04:46:56.251 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.271e-06 2024-09-14T04:46:56.251 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.251 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.251 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:56.251 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.682777+0000", 2024-09-14T04:46:56.251 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2849999999999996e-06 2024-09-14T04:46:56.251 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.251 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.251 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:56.252 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.682787+0000", 2024-09-14T04:46:56.252 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0787e-05 2024-09-14T04:46:56.252 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.252 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.252 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:56.252 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.682793+0000", 2024-09-14T04:46:56.252 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3990000000000003e-06 2024-09-14T04:46:56.252 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.252 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.252 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.252 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.682856+0000", 2024-09-14T04:46:56.252 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2890000000000003e-05 2024-09-14T04:46:56.252 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.253 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.253 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:46:56.253 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.682863+0000", 2024-09-14T04:46:56.253 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9600000000000003e-06 2024-09-14T04:46:56.253 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.253 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.253 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.253 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.508101+0000", 2024-09-14T04:46:56.253 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.82523816900000002 2024-09-14T04:46:56.253 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.253 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.253 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:56.253 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.508347+0000", 2024-09-14T04:46:56.253 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000246629 2024-09-14T04:46:56.254 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.254 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.254 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:46:56.254 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.508454+0000", 2024-09-14T04:46:56.254 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000106424 2024-09-14T04:46:56.254 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.254 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.254 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.254 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.512305+0000", 2024-09-14T04:46:56.254 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.003851508 2024-09-14T04:46:56.254 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.254 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.254 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.255 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516195+0000", 2024-09-14T04:46:56.255 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0038899450000000001 2024-09-14T04:46:56.255 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.255 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.255 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:46:56.255 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516761+0000", 2024-09-14T04:46:56.255 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00056550800000000005 2024-09-14T04:46:56.255 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.255 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.255 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:56.255 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516900+0000", 2024-09-14T04:46:56.256 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000139639 2024-09-14T04:46:56.256 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.256 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.256 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:56.256 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.519261+0000", 2024-09-14T04:46:56.256 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0023607400000000001 2024-09-14T04:46:56.256 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.256 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:56.256 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.257 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.257 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.257 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:135589 54.c 54:356406db:::benchmark_data_smithi039_264334_object9192:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1401)", 2024-09-14T04:46:56.257 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:56.300102+0000", 2024-09-14T04:46:56.257 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 119.75460837999999, 2024-09-14T04:46:56.257 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1549550009999998, 2024-09-14T04:46:56.257 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:56.257 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:56.257 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T04:46:56.257 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:46:56.257 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:46:56.257 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:46:56.257 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 135589 2024-09-14T04:46:56.258 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.258 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:56.258 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.258 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:56.258 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.300102+0000", 2024-09-14T04:46:56.258 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.258 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.258 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.258 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:56.258 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.300102+0000", 2024-09-14T04:46:56.258 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.258 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.258 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.258 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:56.259 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.300103+0000", 2024-09-14T04:46:56.259 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.939e-06 2024-09-14T04:46:56.259 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.259 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.259 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:56.259 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.300109+0000", 2024-09-14T04:46:56.259 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7389999999999996e-06 2024-09-14T04:46:56.259 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.259 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.259 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:56.259 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.300125+0000", 2024-09-14T04:46:56.259 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6076999999999999e-05 2024-09-14T04:46:56.259 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.260 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.260 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:56.260 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.300133+0000", 2024-09-14T04:46:56.260 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.2230000000000001e-06 2024-09-14T04:46:56.260 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.260 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.260 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.260 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.301312+0000", 2024-09-14T04:46:56.260 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0011795480000000001 2024-09-14T04:46:56.260 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.260 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.260 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:56.261 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.302565+0000", 2024-09-14T04:46:56.261 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0012534099999999999 2024-09-14T04:46:56.261 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.261 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.261 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 5,6", 2024-09-14T04:46:56.261 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.302670+0000", 2024-09-14T04:46:56.261 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0001047 2024-09-14T04:46:56.261 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.261 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.261 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.261 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.306221+0000", 2024-09-14T04:46:56.261 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00355105 2024-09-14T04:46:56.261 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.261 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.262 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:46:56.262 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.306468+0000", 2024-09-14T04:46:56.262 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00024643100000000003 2024-09-14T04:46:56.262 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.262 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.262 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.262 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.420868+0000", 2024-09-14T04:46:56.262 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.114400113 2024-09-14T04:46:56.262 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.262 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.262 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:46:56.263 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.420879+0000", 2024-09-14T04:46:56.263 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0720000000000001e-05 2024-09-14T04:46:56.263 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.263 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.263 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.263 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.324315+0000", 2024-09-14T04:46:56.263 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.90343673899999999 2024-09-14T04:46:56.263 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.263 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.263 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:46:56.263 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.324338+0000", 2024-09-14T04:46:56.263 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2510999999999998e-05 2024-09-14T04:46:56.263 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.263 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.264 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.264 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.454966+0000", 2024-09-14T04:46:56.264 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1306284820000001 2024-09-14T04:46:56.264 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.264 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.264 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:56.264 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.455057+0000", 2024-09-14T04:46:56.264 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.0319e-05 2024-09-14T04:46:56.264 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.264 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:56.264 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.264 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.264 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.265 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:135618 54.c 54:3384db48:::benchmark_data_smithi039_264334_object9221:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:46:56.265 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:56.374671+0000", 2024-09-14T04:46:56.265 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 119.680038931, 2024-09-14T04:46:56.265 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.091442705, 2024-09-14T04:46:56.265 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:56.265 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:56.265 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:56.265 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:46:56.265 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:46:56.265 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:46:56.265 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 135618 2024-09-14T04:46:56.265 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.265 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:56.266 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.266 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:56.266 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.374671+0000", 2024-09-14T04:46:56.266 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.266 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.266 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.266 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:56.266 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.374671+0000", 2024-09-14T04:46:56.266 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.267 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.267 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.267 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:56.267 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.374672+0000", 2024-09-14T04:46:56.267 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.5000000000000001e-07 2024-09-14T04:46:56.267 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.267 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.267 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:56.267 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.374676+0000", 2024-09-14T04:46:56.267 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3590000000000001e-06 2024-09-14T04:46:56.267 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.267 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.268 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:56.268 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.374686+0000", 2024-09-14T04:46:56.268 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.9350000000000006e-06 2024-09-14T04:46:56.268 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.268 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.268 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:56.268 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.374693+0000", 2024-09-14T04:46:56.268 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9659999999999999e-06 2024-09-14T04:46:56.268 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.268 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.268 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.268 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.429534+0000", 2024-09-14T04:46:56.269 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.054841049000000003 2024-09-14T04:46:56.269 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.269 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.269 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:46:56.269 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.429544+0000", 2024-09-14T04:46:56.269 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.5289999999999997e-06 2024-09-14T04:46:56.269 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.269 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.269 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.269 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.325910+0000", 2024-09-14T04:46:56.269 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.89636666499999995 2024-09-14T04:46:56.269 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.269 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.270 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:46:56.270 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.325921+0000", 2024-09-14T04:46:56.270 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0747000000000001e-05 2024-09-14T04:46:56.270 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.270 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.270 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.270 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.455428+0000", 2024-09-14T04:46:56.270 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1295071729999999 2024-09-14T04:46:56.270 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.270 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.271 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:56.271 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.455553+0000", 2024-09-14T04:46:56.271 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000125041 2024-09-14T04:46:56.271 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.271 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.271 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 4,5,6", 2024-09-14T04:46:56.271 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.455674+0000", 2024-09-14T04:46:56.271 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000120759 2024-09-14T04:46:56.271 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.271 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.271 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:46:56.271 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.465735+0000", 2024-09-14T04:46:56.271 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.010060955 2024-09-14T04:46:56.272 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.272 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.272 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.272 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.465794+0000", 2024-09-14T04:46:56.272 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.9268000000000002e-05 2024-09-14T04:46:56.272 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.272 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.272 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.272 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.465869+0000", 2024-09-14T04:46:56.272 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.4383999999999998e-05 2024-09-14T04:46:56.272 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.272 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.272 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.273 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.465918+0000", 2024-09-14T04:46:56.273 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9656e-05 2024-09-14T04:46:56.273 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.273 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.273 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:56.273 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.465988+0000", 2024-09-14T04:46:56.273 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9436000000000004e-05 2024-09-14T04:46:56.273 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.273 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.273 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:56.273 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.466114+0000", 2024-09-14T04:46:56.273 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000125833 2024-09-14T04:46:56.273 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.274 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:56.274 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.274 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.274 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.274 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:135633 54.c 54:33252ab7:::benchmark_data_smithi039_264334_object9236:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:46:56.274 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:56.431333+0000", 2024-09-14T04:46:56.274 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 119.623377351, 2024-09-14T04:46:56.274 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0351089770000002, 2024-09-14T04:46:56.274 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:56.274 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:56.274 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:56.274 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:46:56.274 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:46:56.275 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:46:56.275 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 135633 2024-09-14T04:46:56.275 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.275 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:56.275 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.275 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:56.275 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431333+0000", 2024-09-14T04:46:56.275 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.276 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.276 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.276 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:56.276 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431333+0000", 2024-09-14T04:46:56.276 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.276 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.276 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.276 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:56.276 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431335+0000", 2024-09-14T04:46:56.276 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9479999999999998e-06 2024-09-14T04:46:56.276 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.276 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.276 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:56.276 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431340+0000", 2024-09-14T04:46:56.277 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5559999999999998e-06 2024-09-14T04:46:56.277 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.277 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.277 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:56.277 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431354+0000", 2024-09-14T04:46:56.277 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3772e-05 2024-09-14T04:46:56.277 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.277 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.277 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:56.277 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431362+0000", 2024-09-14T04:46:56.277 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.5390000000000007e-06 2024-09-14T04:46:56.277 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.278 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.278 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.278 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.447068+0000", 2024-09-14T04:46:56.278 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.015705666 2024-09-14T04:46:56.278 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.278 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.278 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:46:56.278 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.447080+0000", 2024-09-14T04:46:56.278 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1506e-05 2024-09-14T04:46:56.278 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.278 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.278 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.279 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.325950+0000", 2024-09-14T04:46:56.279 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.87887031699999996 2024-09-14T04:46:56.279 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.279 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.279 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:46:56.279 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.325958+0000", 2024-09-14T04:46:56.279 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.1540000000000002e-06 2024-09-14T04:46:56.279 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.279 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.279 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.279 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.458856+0000", 2024-09-14T04:46:56.280 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1328979669999999 2024-09-14T04:46:56.280 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.280 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.280 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:56.280 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.459013+0000", 2024-09-14T04:46:56.280 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015676700000000001 2024-09-14T04:46:56.280 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.280 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.280 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 4,5,6", 2024-09-14T04:46:56.280 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.459124+0000", 2024-09-14T04:46:56.280 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000111482 2024-09-14T04:46:56.280 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.280 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.281 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:46:56.281 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.466127+0000", 2024-09-14T04:46:56.281 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0070024850000000001 2024-09-14T04:46:56.281 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.281 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.281 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.281 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.466181+0000", 2024-09-14T04:46:56.281 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4088000000000001e-05 2024-09-14T04:46:56.281 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.281 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.281 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.281 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.466226+0000", 2024-09-14T04:46:56.281 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4965000000000001e-05 2024-09-14T04:46:56.281 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.282 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.282 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.282 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.466287+0000", 2024-09-14T04:46:56.282 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1079000000000003e-05 2024-09-14T04:46:56.282 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.282 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.282 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:56.282 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.466341+0000", 2024-09-14T04:46:56.282 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4379000000000002e-05 2024-09-14T04:46:56.282 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.282 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.282 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:56.282 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.466442+0000", 2024-09-14T04:46:56.283 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000100307 2024-09-14T04:46:56.283 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.283 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:56.283 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.283 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.283 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.283 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:135635 54.c 54:31b8983c:::benchmark_data_smithi039_264334_object9238:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:46:56.283 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:56.431574+0000", 2024-09-14T04:46:56.283 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 119.623135898, 2024-09-14T04:46:56.283 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0581618759999998, 2024-09-14T04:46:56.283 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:56.283 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:56.283 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:56.283 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:46:56.284 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:46:56.284 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:46:56.284 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 135635 2024-09-14T04:46:56.284 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.284 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:56.284 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.284 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:56.284 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431574+0000", 2024-09-14T04:46:56.284 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.284 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.284 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.284 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:56.284 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431574+0000", 2024-09-14T04:46:56.285 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.285 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.285 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.285 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:56.285 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431575+0000", 2024-09-14T04:46:56.285 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.2600000000000001e-07 2024-09-14T04:46:56.285 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.285 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.285 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:56.285 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431579+0000", 2024-09-14T04:46:56.285 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5249999999999999e-06 2024-09-14T04:46:56.285 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.285 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.285 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:56.285 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431591+0000", 2024-09-14T04:46:56.286 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1314999999999999e-05 2024-09-14T04:46:56.286 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.286 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.286 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:56.286 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431598+0000", 2024-09-14T04:46:56.286 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.2420000000000003e-06 2024-09-14T04:46:56.286 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.286 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.286 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.286 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.447535+0000", 2024-09-14T04:46:56.286 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.015937337999999999 2024-09-14T04:46:56.286 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.286 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.286 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:46:56.287 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.447545+0000", 2024-09-14T04:46:56.287 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.4890000000000001e-06 2024-09-14T04:46:56.287 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.287 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.287 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.287 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.325982+0000", 2024-09-14T04:46:56.287 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.87843713400000001 2024-09-14T04:46:56.287 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.287 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.287 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:46:56.287 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.325990+0000", 2024-09-14T04:46:56.287 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9400000000000002e-06 2024-09-14T04:46:56.288 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.288 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.288 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.288 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.466504+0000", 2024-09-14T04:46:56.288 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1405139260000001 2024-09-14T04:46:56.288 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.288 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.288 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:56.288 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.466628+0000", 2024-09-14T04:46:56.288 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012408699999999999 2024-09-14T04:46:56.288 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.288 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.288 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 4,5,6", 2024-09-14T04:46:56.289 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.466745+0000", 2024-09-14T04:46:56.289 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000117447 2024-09-14T04:46:56.289 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.289 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.289 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.289 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.476281+0000", 2024-09-14T04:46:56.289 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0095351040000000008 2024-09-14T04:46:56.289 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.289 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.289 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.289 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.476349+0000", 2024-09-14T04:46:56.289 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8342000000000004e-05 2024-09-14T04:46:56.290 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.290 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.290 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:46:56.290 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.479362+0000", 2024-09-14T04:46:56.290 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0030131540000000001 2024-09-14T04:46:56.290 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.290 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.290 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.290 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.488661+0000", 2024-09-14T04:46:56.290 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0092988040000000008 2024-09-14T04:46:56.290 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.290 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.290 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:56.290 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.489603+0000", 2024-09-14T04:46:56.290 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00094184399999999995 2024-09-14T04:46:56.291 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.291 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.291 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:56.291 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.489736+0000", 2024-09-14T04:46:56.291 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013325899999999999 2024-09-14T04:46:56.291 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.291 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:56.291 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.291 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.291 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.291 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:135644 54.c 54:35c60476:::benchmark_data_smithi039_264334_object9247:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:46:56.291 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:56.446718+0000", 2024-09-14T04:46:56.291 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 119.607991751, 2024-09-14T04:46:56.291 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.043331561, 2024-09-14T04:46:56.291 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:56.292 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:56.292 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:56.292 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:46:56.292 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:46:56.292 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:46:56.292 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 135644 2024-09-14T04:46:56.292 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.292 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:56.292 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.292 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:56.292 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.446718+0000", 2024-09-14T04:46:56.292 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.292 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.292 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.293 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:56.293 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.446718+0000", 2024-09-14T04:46:56.293 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.293 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.293 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.293 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:56.293 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.446720+0000", 2024-09-14T04:46:56.293 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3540000000000001e-06 2024-09-14T04:46:56.293 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.293 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.293 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:56.293 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.446724+0000", 2024-09-14T04:46:56.293 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7249999999999997e-06 2024-09-14T04:46:56.294 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.294 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.294 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:56.294 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.446734+0000", 2024-09-14T04:46:56.294 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.5920000000000007e-06 2024-09-14T04:46:56.294 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.294 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.294 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:56.294 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.446742+0000", 2024-09-14T04:46:56.294 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0520000000000002e-06 2024-09-14T04:46:56.294 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.294 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.294 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.294 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.447585+0000", 2024-09-14T04:46:56.295 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00084300300000000005 2024-09-14T04:46:56.295 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.295 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.295 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:46:56.295 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.447597+0000", 2024-09-14T04:46:56.295 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2302e-05 2024-09-14T04:46:56.295 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.295 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.295 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.295 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.326086+0000", 2024-09-14T04:46:56.295 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.87848874099999996 2024-09-14T04:46:56.295 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.295 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.295 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:46:56.296 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.326095+0000", 2024-09-14T04:46:56.296 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.1360000000000003e-06 2024-09-14T04:46:56.296 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.296 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.296 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.296 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.469679+0000", 2024-09-14T04:46:56.296 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.143583765 2024-09-14T04:46:56.296 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.296 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.296 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:56.296 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.469760+0000", 2024-09-14T04:46:56.296 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0669999999999998e-05 2024-09-14T04:46:56.296 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.296 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.297 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 4,5,6", 2024-09-14T04:46:56.297 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.469836+0000", 2024-09-14T04:46:56.297 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6048000000000001e-05 2024-09-14T04:46:56.297 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.297 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.297 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:46:56.297 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.479370+0000", 2024-09-14T04:46:56.297 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.009534275 2024-09-14T04:46:56.297 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.297 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.297 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.297 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.481815+0000", 2024-09-14T04:46:56.297 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.002444786 2024-09-14T04:46:56.298 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.298 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.298 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.298 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.481869+0000", 2024-09-14T04:46:56.298 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4113000000000002e-05 2024-09-14T04:46:56.298 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.298 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.298 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.298 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.489843+0000", 2024-09-14T04:46:56.298 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0079742140000000003 2024-09-14T04:46:56.298 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.298 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.298 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:56.298 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.489933+0000", 2024-09-14T04:46:56.299 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.0115e-05 2024-09-14T04:46:56.299 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.299 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.299 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:56.299 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.490050+0000", 2024-09-14T04:46:56.299 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011667000000000001 2024-09-14T04:46:56.299 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.299 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:56.299 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.299 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.299 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.299 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:135655 54.c 54:3bab8a03:::benchmark_data_smithi039_264334_object9258:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:46:56.299 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:56.456620+0000", 2024-09-14T04:46:56.300 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 119.598090328, 2024-09-14T04:46:56.300 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0336829170000001, 2024-09-14T04:46:56.300 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:56.300 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:56.300 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:56.300 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:46:56.300 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:46:56.300 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:46:56.300 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 135655 2024-09-14T04:46:56.300 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.300 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:56.300 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.300 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:56.300 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.456620+0000", 2024-09-14T04:46:56.301 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.301 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.301 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.301 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:56.301 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.456620+0000", 2024-09-14T04:46:56.301 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.301 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.301 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.301 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:56.301 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.456621+0000", 2024-09-14T04:46:56.301 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7719999999999999e-06 2024-09-14T04:46:56.301 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.301 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.301 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:56.302 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.456628+0000", 2024-09-14T04:46:56.302 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3679999999999998e-06 2024-09-14T04:46:56.302 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.302 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.302 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:56.302 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.456644+0000", 2024-09-14T04:46:56.302 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6297999999999999e-05 2024-09-14T04:46:56.302 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.302 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.302 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:56.302 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.456652+0000", 2024-09-14T04:46:56.302 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9519999999999994e-06 2024-09-14T04:46:56.302 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.302 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.303 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.303 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.456746+0000", 2024-09-14T04:46:56.303 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.3617999999999997e-05 2024-09-14T04:46:56.303 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.303 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.303 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:46:56.303 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.456760+0000", 2024-09-14T04:46:56.303 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4185999999999999e-05 2024-09-14T04:46:56.303 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.303 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.303 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.303 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.326122+0000", 2024-09-14T04:46:56.303 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.86936199400000003 2024-09-14T04:46:56.303 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.304 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.304 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:46:56.304 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.326129+0000", 2024-09-14T04:46:56.304 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.5730000000000002e-06 2024-09-14T04:46:56.304 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.304 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.304 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.304 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.473189+0000", 2024-09-14T04:46:56.304 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.14706011 2024-09-14T04:46:56.304 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.304 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.304 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:56.304 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.473313+0000", 2024-09-14T04:46:56.304 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000123662 2024-09-14T04:46:56.305 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.305 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.305 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 4,5,6", 2024-09-14T04:46:56.305 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.473443+0000", 2024-09-14T04:46:56.305 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012988400000000001 2024-09-14T04:46:56.305 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.305 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.305 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:46:56.305 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.481877+0000", 2024-09-14T04:46:56.305 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0084343060000000008 2024-09-14T04:46:56.305 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.305 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.305 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.305 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.481911+0000", 2024-09-14T04:46:56.306 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.3553000000000002e-05 2024-09-14T04:46:56.306 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.306 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.306 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.306 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.489798+0000", 2024-09-14T04:46:56.306 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0078867659999999999 2024-09-14T04:46:56.306 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.306 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.306 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.306 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.490128+0000", 2024-09-14T04:46:56.306 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00033039400000000002 2024-09-14T04:46:56.306 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.306 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.306 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:56.307 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.490211+0000", 2024-09-14T04:46:56.307 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.2700000000000004e-05 2024-09-14T04:46:56.307 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.307 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.307 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:56.307 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.490303+0000", 2024-09-14T04:46:56.307 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.1780999999999994e-05 2024-09-14T04:46:56.307 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.307 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:56.307 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.307 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.307 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.307 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:187181 54.c 54:349c28b3:::benchmark_data_smithi039_264334_object60784:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1412)", 2024-09-14T04:46:56.307 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:46:00.774068+0000", 2024-09-14T04:46:56.307 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 55.280642344, 2024-09-14T04:46:56.308 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.222606416, 2024-09-14T04:46:56.308 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:56.308 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:56.308 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:56.308 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:46:56.308 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:46:56.308 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:46:56.308 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 187181 2024-09-14T04:46:56.308 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.308 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:56.308 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.308 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:56.308 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:00.774068+0000", 2024-09-14T04:46:56.309 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.309 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.309 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.309 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:56.309 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:00.774068+0000", 2024-09-14T04:46:56.309 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.309 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.309 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.309 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:56.309 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:00.774069+0000", 2024-09-14T04:46:56.309 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5570000000000001e-06 2024-09-14T04:46:56.309 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.309 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.310 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:56.310 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:00.774074+0000", 2024-09-14T04:46:56.310 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4549999999999997e-06 2024-09-14T04:46:56.310 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.310 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.310 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:56.310 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:00.821154+0000", 2024-09-14T04:46:56.310 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.047080601 2024-09-14T04:46:56.310 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.310 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.310 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:56.310 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:00.821171+0000", 2024-09-14T04:46:56.310 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7116e-05 2024-09-14T04:46:56.310 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.311 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.311 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.311 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:00.821463+0000", 2024-09-14T04:46:56.311 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00029207499999999998 2024-09-14T04:46:56.311 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.311 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.311 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:56.311 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:00.821651+0000", 2024-09-14T04:46:56.311 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00018748699999999999 2024-09-14T04:46:56.311 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.311 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.311 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 4,5,6", 2024-09-14T04:46:56.311 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:00.821778+0000", 2024-09-14T04:46:56.312 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012703700000000001 2024-09-14T04:46:56.312 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.312 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.312 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:46:56.312 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:00.826263+0000", 2024-09-14T04:46:56.312 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0044847120000000001 2024-09-14T04:46:56.312 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.312 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.312 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.312 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:00.830861+0000", 2024-09-14T04:46:56.312 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0045983960000000003 2024-09-14T04:46:56.312 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.312 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.312 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.312 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:00.834313+0000", 2024-09-14T04:46:56.313 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0034524730000000002 2024-09-14T04:46:56.313 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.313 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.313 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.313 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:00.996500+0000", 2024-09-14T04:46:56.313 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.16218627399999999 2024-09-14T04:46:56.313 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.313 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.313 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:56.313 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:00.996558+0000", 2024-09-14T04:46:56.313 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.8007999999999998e-05 2024-09-14T04:46:56.313 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.313 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.313 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:56.313 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:00.996674+0000", 2024-09-14T04:46:56.314 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011622500000000001 2024-09-14T04:46:56.314 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.314 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:56.314 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.314 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.314 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.314 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:227084 54.7 54:efb3822d:::benchmark_data_smithi039_264334_object100687:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1415)", 2024-09-14T04:46:56.314 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:46:50.921709+0000", 2024-09-14T04:46:56.314 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 5.1330006130000001, 2024-09-14T04:46:56.314 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.266603494, 2024-09-14T04:46:56.314 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:46:56.314 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:46:56.314 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:46:56.314 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:46:56.314 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:46:56.314 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:46:56.315 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 227084 2024-09-14T04:46:56.315 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.315 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:46:56.315 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.315 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:46:56.315 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:50.921709+0000", 2024-09-14T04:46:56.315 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.315 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.315 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.315 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:46:56.315 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:50.921709+0000", 2024-09-14T04:46:56.315 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:46:56.315 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.315 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.315 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:46:56.315 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:50.921710+0000", 2024-09-14T04:46:56.316 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9999999999999998e-07 2024-09-14T04:46:56.316 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.316 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.316 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:46:56.316 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:50.921713+0000", 2024-09-14T04:46:56.316 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.9799999999999998e-06 2024-09-14T04:46:56.316 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.316 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.316 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:46:56.316 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:50.921719+0000", 2024-09-14T04:46:56.316 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3219999999999998e-06 2024-09-14T04:46:56.316 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.316 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.316 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:46:56.317 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:50.921724+0000", 2024-09-14T04:46:56.317 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1460000000000004e-06 2024-09-14T04:46:56.317 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.317 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.317 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:46:56.317 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:50.926092+0000", 2024-09-14T04:46:56.317 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0043681789999999998 2024-09-14T04:46:56.317 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.317 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.317 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:46:56.317 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:50.926200+0000", 2024-09-14T04:46:56.317 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000107512 2024-09-14T04:46:56.317 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.318 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.318 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 0,7", 2024-09-14T04:46:56.318 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:50.926308+0000", 2024-09-14T04:46:56.318 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00010832699999999999 2024-09-14T04:46:56.318 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.318 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.318 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.318 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:51.103357+0000", 2024-09-14T04:46:56.318 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.17704826800000001 2024-09-14T04:46:56.318 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.318 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.318 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:46:56.318 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:51.112920+0000", 2024-09-14T04:46:56.318 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0095635779999999997 2024-09-14T04:46:56.318 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.318 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.319 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:46:56.319 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:51.188168+0000", 2024-09-14T04:46:56.319 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.075248104999999996 2024-09-14T04:46:56.319 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.319 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.319 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:46:56.319 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:51.188267+0000", 2024-09-14T04:46:56.319 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.9198999999999997e-05 2024-09-14T04:46:56.319 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:46:56.319 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:46:56.319 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:46:56.319 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:51.188313+0000", 2024-09-14T04:46:56.319 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5377999999999998e-05 2024-09-14T04:46:56.319 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.320 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:56.320 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.320 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:46:56.320 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:46:56.320 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:46:56.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:56.322 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:56.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:56.423 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:56.467 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:56.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:56.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:56.568 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:56.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:56.613 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:56.613 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:56.714 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:56.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:56.759 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:56.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:56.860 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:56.914 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:56.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:56.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:57.016 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:57.054 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:57.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:57.055 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:57.156 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:57.193 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:57.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:57.194 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:57.295 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:57.340 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:57.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:57.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:57.442 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:57.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:57.470 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:57.471 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:57.571 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:57.601 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:57.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:57.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:57.703 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:57.738 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:57.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:57.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:57.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:57 smithi129 ceph-mon[101316]: pgmap v2504: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 9549/73314 objects misplaced (13.025%); 1.4 MiB/s, 22 objects/s recovering 2024-09-14T04:46:57.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:57 smithi039 ceph-mon[106470]: pgmap v2504: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 9549/73314 objects misplaced (13.025%); 1.4 MiB/s, 22 objects/s recovering 2024-09-14T04:46:57.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:57 smithi039 ceph-mon[111235]: pgmap v2504: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 1.5 GiB data, 7.1 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 773 op/s; 9549/73314 objects misplaced (13.025%); 1.4 MiB/s, 22 objects/s recovering 2024-09-14T04:46:57.840 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:57.876 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:57.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:57.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:57.977 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:58.014 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:58.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:58.015 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:58.116 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:58.156 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:58.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:58.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:58.258 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:58.293 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:58.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:58.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:58.395 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:46:58.431 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:58.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:58.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:46:58.532 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:58.565 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:58.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:58.566 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:58.667 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:58.690 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:58.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:58.691 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:58.792 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:58.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:58.841 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:58.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:58.942 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:46:58.983 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:58.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:58.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:46:59.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:46:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:46:58] "GET /metrics HTTP/1.1" 200 36746 "" "Prometheus/2.51.0" 2024-09-14T04:46:59.085 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:59.132 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:59.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:59.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:59.234 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:46:59.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:59.268 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:59.268 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:46:59.369 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:46:59.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:59.405 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:59.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:46:59.506 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:46:59.549 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:59.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:59.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:46:59.651 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:46:59.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:59.685 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:46:59.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:46:59.718 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:46:59 smithi039 ceph-mon[106470]: pgmap v2505: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 9148/67884 objects misplaced (13.476%); 1.6 MiB/s, 26 objects/s recovering 2024-09-14T04:46:59.720 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:46:59 smithi039 ceph-mon[111235]: pgmap v2505: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 9148/67884 objects misplaced (13.476%); 1.6 MiB/s, 26 objects/s recovering 2024-09-14T04:46:59.755 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-14T04:46:59.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:46:59 smithi129 ceph-mon[101316]: pgmap v2505: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 1.4 GiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 785 op/s; 9148/67884 objects misplaced (13.476%); 1.6 MiB/s, 26 objects/s recovering 2024-09-14T04:46:59.786 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:59.833 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:59.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:59.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:46:59.935 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:46:59.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:46:59.976 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:46:59.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:00.077 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:00.113 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:47:00.130 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:00.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:00.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:00.231 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:00.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:00.272 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:00.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:00.373 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:00.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:00.445 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:00.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:00.483 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:47:00.487 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:47:00.546 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:00.605 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:00.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:00.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:00.707 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:00.742 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:00.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:00.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:00.844 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:00.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:00.880 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:00.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:00.981 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:01.015 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:01.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:01.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:01.117 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:01.155 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:01.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:01.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:01.257 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:01.289 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:01.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:01.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:01.391 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:01.426 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:01.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:01.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:01.528 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:01.570 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:01.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:01.571 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:01.581 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:01 smithi039 ceph-mon[106470]: pgmap v2506: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 1.2 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 8618/60639 objects misplaced (14.212%); 1.6 MiB/s, 26 objects/s recovering 2024-09-14T04:47:01.581 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:01 smithi039 ceph-mon[111235]: pgmap v2506: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 1.2 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 8618/60639 objects misplaced (14.212%); 1.6 MiB/s, 26 objects/s recovering 2024-09-14T04:47:01.672 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:01.718 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:01.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:01.721 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:01.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:01 smithi129 ceph-mon[101316]: pgmap v2506: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 1.2 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 814 op/s; 8618/60639 objects misplaced (14.212%); 1.6 MiB/s, 26 objects/s recovering 2024-09-14T04:47:01.822 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:01.858 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:01.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:01.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:01.960 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:01.999 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:01.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:02.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:02.101 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:02.153 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:47:02.153 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [ 2024-09-14T04:47:02.153 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:02.153 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.35436.0:236393 54.f e1415/1380 54:f0a8b2f5:::benchmark_data_smithi039_264334_object109996:head v 1415'14642, mlcod=1415'14640)", 2024-09-14T04:47:02.153 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:47:02.146446+0000", 2024-09-14T04:47:02.153 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.0049029970000000001, 2024-09-14T04:47:02.153 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0049347039999999998, 2024-09-14T04:47:02.153 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:02.154 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:02.154 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:02.154 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:02.154 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:02.154 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:02.146446+0000", 2024-09-14T04:47:02.154 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:02.154 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:02.154 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:02.154 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:02.154 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:02.146446+0000", 2024-09-14T04:47:02.154 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:02.154 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:02.154 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:02.154 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:02.155 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:02.146445+0000", 2024-09-14T04:47:02.155 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:47:02.155 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:02.155 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:02.155 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:02.155 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:02.146455+0000", 2024-09-14T04:47:02.155 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0084e-05 2024-09-14T04:47:02.155 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:02.155 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:02.155 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:02.155 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:02.146469+0000", 2024-09-14T04:47:02.155 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3644e-05 2024-09-14T04:47:02.155 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:02.155 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:02.156 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:02.156 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:02.146476+0000", 2024-09-14T04:47:02.156 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.7619999999999995e-06 2024-09-14T04:47:02.156 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:02.156 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:02.156 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:02.156 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:02.146624+0000", 2024-09-14T04:47:02.156 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00014775699999999999 2024-09-14T04:47:02.156 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:02.156 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:02.156 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:02.156 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:02.146675+0000", 2024-09-14T04:47:02.157 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.1109000000000001e-05 2024-09-14T04:47:02.157 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:02.157 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:02.157 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:02.157 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:02.151347+0000", 2024-09-14T04:47:02.157 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0046719109999999999 2024-09-14T04:47:02.157 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:02.157 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:02.157 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:02.157 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:02.157 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:02.157 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.35436.0:236386 54.6 e1415/1380 54:69867642:::benchmark_data_smithi039_264334_object109989:head v 1415'14781, mlcod=1415'14779)", 2024-09-14T04:47:02.157 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:47:02.149852+0000", 2024-09-14T04:47:02.158 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.0014970280000000001, 2024-09-14T04:47:02.158 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0016078049999999999, 2024-09-14T04:47:02.158 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:02.158 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "started", 2024-09-14T04:47:02.158 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:02.158 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:02.158 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:02.158 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:02.149852+0000", 2024-09-14T04:47:02.158 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:02.158 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:02.158 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:02.158 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:02.158 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:02.149852+0000", 2024-09-14T04:47:02.158 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:02.158 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:02.158 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:02.159 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:02.159 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:02.149851+0000", 2024-09-14T04:47:02.159 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:47:02.159 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:02.159 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:02.159 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:02.159 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:02.149860+0000", 2024-09-14T04:47:02.159 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.9160000000000007e-06 2024-09-14T04:47:02.159 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:02.159 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:02.159 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:02.159 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:02.149871+0000", 2024-09-14T04:47:02.159 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0865e-05 2024-09-14T04:47:02.159 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:02.159 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:02.160 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:02.160 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:02.149878+0000", 2024-09-14T04:47:02.160 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.2269999999999996e-06 2024-09-14T04:47:02.160 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:02.160 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:02.160 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:02.160 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:02.150007+0000", 2024-09-14T04:47:02.160 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000129369 2024-09-14T04:47:02.160 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:02.160 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:02.160 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:02.160 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:02.150028+0000", 2024-09-14T04:47:02.161 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.0893000000000001e-05 2024-09-14T04:47:02.161 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:02.161 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:02.161 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:02.161 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:02.161 INFO:teuthology.orchestra.run.smithi129.stdout: ], 2024-09-14T04:47:02.161 INFO:teuthology.orchestra.run.smithi129.stdout: "num_ops": 2 2024-09-14T04:47:02.161 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:47:02.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:02.162 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:02.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:02.263 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:02.292 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:02.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:02.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:02.394 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:02.444 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:02.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:02.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:02.546 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:02.586 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:02.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:02.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:02.687 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:02.735 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:02.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:02.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:02.837 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:02.871 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:02.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:02.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:02.973 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:02.995 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:02.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:02.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:02.997 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-14T04:47:03.096 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:03.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:03.167 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:03.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:03.268 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:03.316 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:03.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:03.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:03.352 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:47:03.418 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:03.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:03.446 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:03.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:03.547 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:03.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:03.588 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:03.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:03.689 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:03.724 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:03.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:03.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:03 smithi129 ceph-mon[101316]: pgmap v2507: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 1.2 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 8494/58311 objects misplaced (14.567%); 2.1 MiB/s, 34 objects/s recovering 2024-09-14T04:47:03.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:03 smithi039 ceph-mon[106470]: pgmap v2507: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 1.2 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 8494/58311 objects misplaced (14.567%); 2.1 MiB/s, 34 objects/s recovering 2024-09-14T04:47:03.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:03 smithi039 ceph-mon[111235]: pgmap v2507: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 1.2 GiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 789 op/s; 8494/58311 objects misplaced (14.567%); 2.1 MiB/s, 34 objects/s recovering 2024-09-14T04:47:03.826 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:03.867 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:03.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:03.868 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:03.969 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:03.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:03.996 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:03.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:04.098 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:04.141 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:04.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:04.142 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:04.243 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:04.264 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:04.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:04.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:04.366 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:04.400 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:04.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:04.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:04.502 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:04.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:04.541 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:04.541 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:04.642 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:04.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:04.689 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:04.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:04.791 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:04.831 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:04.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:04.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:04.933 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:04.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:04.975 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:04.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:05.076 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:05.125 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:05.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:05.126 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:05.227 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:05.262 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:05.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:05.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:05.363 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:05.410 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:05.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:05.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:05.512 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:05.540 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:05.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:05.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:05.587 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:05 smithi129 ceph-mon[101316]: pgmap v2508: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 1005 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 960 op/s; 7734/48285 objects misplaced (16.017%); 1.3 MiB/s, 21 objects/s recovering 2024-09-14T04:47:05.642 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:05.666 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:05.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:05.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:05.700 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:47:05.700 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [], 2024-09-14T04:47:05.700 INFO:teuthology.orchestra.run.smithi129.stdout: "complaint_time": 30, 2024-09-14T04:47:05.700 INFO:teuthology.orchestra.run.smithi129.stdout: "num_blocked_ops": 0 2024-09-14T04:47:05.700 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:47:05.767 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:05.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:05 smithi039 ceph-mon[106470]: pgmap v2508: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 1005 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 960 op/s; 7734/48285 objects misplaced (16.017%); 1.3 MiB/s, 21 objects/s recovering 2024-09-14T04:47:05.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:05 smithi039 ceph-mon[111235]: pgmap v2508: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 1005 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 960 op/s; 7734/48285 objects misplaced (16.017%); 1.3 MiB/s, 21 objects/s recovering 2024-09-14T04:47:05.791 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:05.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:05.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:05.893 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:05.922 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:05.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:05.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:06.024 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:06.064 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:06.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:06.065 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:06.166 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:06.199 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:06.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:06.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:06.301 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:06.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:06.335 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:06.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:06.436 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:06.475 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:06.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:06.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:06.576 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:06.599 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:06.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:06.600 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:06.660 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-14T04:47:06.700 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:06.749 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:06.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:06.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:06.850 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:06.909 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:06.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:06.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:06.912 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:47:06.912 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:47:06.912 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:47:06.912 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:47:06.912 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:47:06.912 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:47:06.912 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:47:06.912 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:47:06.913 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:47:06.913 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:47:06.913 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:47:06.913 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:47:06.913 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:47:06.913 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:47:06.920 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:47:06.921 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:47:06.987 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:47:07.011 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:07.042 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:07.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:07.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:07.144 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:07.185 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:07.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:07.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:07.287 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:07.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:07.331 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:07.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:07.432 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:07.462 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:07.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:07.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:07.563 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:07.608 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:07.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:07.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:07.710 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:07.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:07.758 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:07.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:07 smithi129 ceph-mon[101316]: pgmap v2509: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 1005 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 7734/48285 objects misplaced (16.017%); 1.3 MiB/s, 21 objects/s recovering 2024-09-14T04:47:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:07 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2001983094' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:47:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:47:07.777 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:07 smithi039 ceph-mon[106470]: pgmap v2509: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 1005 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 7734/48285 objects misplaced (16.017%); 1.3 MiB/s, 21 objects/s recovering 2024-09-14T04:47:07.777 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:07 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2001983094' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:47:07.777 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:47:07.777 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:07 smithi039 ceph-mon[111235]: pgmap v2509: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 1005 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 694 op/s; 7734/48285 objects misplaced (16.017%); 1.3 MiB/s, 21 objects/s recovering 2024-09-14T04:47:07.777 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:07 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2001983094' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:47:07.777 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:47:07.860 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:07.896 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:07.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:07.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:07.997 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:08.040 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:08.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:08.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:08.142 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:08.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:08.180 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:08.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:08.281 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:08.320 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:08.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:08.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:08.421 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:08.455 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:08.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:08.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:08.558 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:08.602 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:08.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:08.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:08.703 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:08.747 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:08.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:08.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:08.849 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:08.874 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:08.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:08.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:08.976 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:09.000 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:09.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:09.001 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:09.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:47:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:47:08] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.51.0" 2024-09-14T04:47:09.102 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:09.143 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:09.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:09.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:09.245 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:09.282 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:09.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:09.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:09.316 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:47:09.316 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:47:09.383 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:09.424 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:09.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:09.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:09.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:09 smithi129 ceph-mon[101316]: pgmap v2510: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 902 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 7390/43338 objects misplaced (17.052%); 2.1 MiB/s, 33 objects/s recovering 2024-09-14T04:47:09.526 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:09.572 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:09.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:09.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:09.674 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:09.713 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:09.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:09.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:09 smithi039 ceph-mon[106470]: pgmap v2510: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 902 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 7390/43338 objects misplaced (17.052%); 2.1 MiB/s, 33 objects/s recovering 2024-09-14T04:47:09.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:09 smithi039 ceph-mon[111235]: pgmap v2510: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 902 MiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 832 op/s; 7390/43338 objects misplaced (17.052%); 2.1 MiB/s, 33 objects/s recovering 2024-09-14T04:47:09.816 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:09.847 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:09.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:09.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:09.948 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:09.993 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:09.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:09.994 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:10.004 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-14T04:47:10.095 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:10.129 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:10.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:10.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:10.231 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:10.275 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:10.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:10.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:10.363 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:47:10.377 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:10.427 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:10.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:10.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:10.528 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:10.564 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:10.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:10.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:10.666 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:10.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:10.697 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:10.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:10.798 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:10.833 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:10.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:10.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:10.935 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:10.978 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:10.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:10.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:11.080 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:11.126 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:11.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:11.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:11.228 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:11.280 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:11.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:11.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:11.382 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:11.416 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:11.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:11.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:11.517 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:11.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:11.547 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:11.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:11.648 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:11.668 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:11 smithi039 ceph-mon[106470]: pgmap v2511: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 769 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 858 op/s; 6937/36966 objects misplaced (18.766%); 1.4 MiB/s, 22 objects/s recovering 2024-09-14T04:47:11.668 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:11 smithi039 ceph-mon[111235]: pgmap v2511: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 769 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 858 op/s; 6937/36966 objects misplaced (18.766%); 1.4 MiB/s, 22 objects/s recovering 2024-09-14T04:47:11.702 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:11.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:11.703 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:11.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:11 smithi129 ceph-mon[101316]: pgmap v2511: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 769 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 858 op/s; 6937/36966 objects misplaced (18.766%); 1.4 MiB/s, 22 objects/s recovering 2024-09-14T04:47:11.804 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:11.846 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:11.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:11.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:11.914 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:47:11.915 INFO:teuthology.orchestra.run.smithi039.stdout:{"fsid":"bd7c803c-724b-11ef-bceb-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":2747,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1415,"num_osds":8,"num_up_osds":8,"osd_up_since":1726289028,"num_in_osds":8,"osd_in_since":1726285856,"num_remapped_pgs":4},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":95},{"state_name":"active+remapped+backfill_wait","count":3},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":12322,"data_bytes":806361085,"bytes_used":5329334272,"bytes_avail":762630950912,"bytes_total":767960285184,"misplaced_objects":6937,"misplaced_total":36966,"misplaced_ratio":0.18765892982740898,"recovering_objects_per_sec":22,"recovering_bytes_per_sec":1490366,"recovering_keys_per_sec":0,"num_objects_recovered":273,"num_bytes_recovered":17891328,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":858},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi039.bbwiwf","status":"up:active","gid":14646}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.39:8443/","prometheus":"http://172.21.15.39:9283/"}},"servicemap":{"epoch":37,"modified":"2024-09-14T04:45:23.646879+0000","services":{}},"progress_events":{}} 2024-09-14T04:47:11.947 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:11.974 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:11.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:11.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:12.075 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:12.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:12.116 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:12.116 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:12.217 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:12.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:12.258 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:12.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:12.359 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:12.386 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:12.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:12.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:12.488 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:12.523 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:47:12.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:12.528 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:12.529 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:12.545 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is unset 2024-09-14T04:47:12.629 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:12.692 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:12.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:12.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:12.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:12 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/741260154' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-14T04:47:12.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:12 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/628984841' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:47:12.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:12 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:47:12.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:12 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/741260154' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-14T04:47:12.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:12 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/628984841' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:47:12.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:12 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:47:12.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:12 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/741260154' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-14T04:47:12.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:12 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/628984841' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:47:12.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:12 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:47:12.794 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:12.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:12.830 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:12.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:12.931 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:12.975 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:12.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:12.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:13.077 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:13.133 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:13.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:13.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:13.235 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:13.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:13.260 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:13.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:13.361 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:13.392 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:13.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:13.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:13.494 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:13.528 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:13 smithi039 ceph-mon[106470]: pgmap v2512: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 721 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 6783/34641 objects misplaced (19.581%); 2.2 MiB/s, 35 objects/s recovering 2024-09-14T04:47:13.528 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:13 smithi039 ceph-mon[106470]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:47:13.528 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:13.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:13.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:13.630 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:13.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:13.664 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:13.664 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:13.765 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:13.780 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-14T04:47:13.780 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:47:13.780 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:13 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:47:13.781 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:13 smithi039 ceph-mon[106470]: osdmap e1416: 8 total, 8 up, 8 in 2024-09-14T04:47:13.781 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:13 smithi039 ceph-mon[111235]: pgmap v2512: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 721 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 6783/34641 objects misplaced (19.581%); 2.2 MiB/s, 35 objects/s recovering 2024-09-14T04:47:13.781 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:13 smithi039 ceph-mon[111235]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:47:13.781 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:13 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:47:13.781 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:13 smithi039 ceph-mon[111235]: osdmap e1416: 8 total, 8 up, 8 in 2024-09-14T04:47:13.791 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:13.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:13.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:13.894 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:13.943 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:13.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:13.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:14.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:13 smithi129 ceph-mon[101316]: pgmap v2512: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 721 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 721 op/s; 6783/34641 objects misplaced (19.581%); 2.2 MiB/s, 35 objects/s recovering 2024-09-14T04:47:14.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:13 smithi129 ceph-mon[101316]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:47:14.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:13 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:47:14.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:13 smithi129 ceph-mon[101316]: osdmap e1416: 8 total, 8 up, 8 in 2024-09-14T04:47:14.044 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:14.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:14.079 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:14.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:14.180 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:14.208 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:14.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:14.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:14.310 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:14.342 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:14.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:14.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:14.444 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:14.476 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:14.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:14.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:14.578 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:14.606 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:14.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:14.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:14.664 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:47:14.665 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:47:14.708 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:14.741 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:14.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:14.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:14.842 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:14.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:14.886 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:14.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:14.987 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:15.014 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:15.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:15.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:15.116 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:15.156 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:15.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:15.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:15.258 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:15.293 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:15.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:15.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:15.352 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-14T04:47:15.394 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:15.420 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:15.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:15.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:15.522 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:15.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:15.562 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:15.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:15.663 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:15.700 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:47:15.702 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:15.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:15.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:15.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:15 smithi129 ceph-mon[101316]: pgmap v2514: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 523 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 6089/25146 objects misplaced (24.215%); 1.9 MiB/s, 30 objects/s recovering 2024-09-14T04:47:15.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:15 smithi039 ceph-mon[106470]: pgmap v2514: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 523 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 6089/25146 objects misplaced (24.215%); 1.9 MiB/s, 30 objects/s recovering 2024-09-14T04:47:15.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:15 smithi039 ceph-mon[111235]: pgmap v2514: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 523 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 6089/25146 objects misplaced (24.215%); 1.9 MiB/s, 30 objects/s recovering 2024-09-14T04:47:15.805 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:15.841 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:15.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:15.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:15.942 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:15.976 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:15.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:15.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:16.078 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:16.116 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:16.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:16.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:16.218 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:16.249 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:16.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:16.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:16.351 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:16.378 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:16.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:16.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:16.479 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:16.523 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:47:16.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:16.533 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:16.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:16.634 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:16.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:16.664 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:16.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:16.765 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:16.802 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:16.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:16.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:16.904 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:16.930 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:16.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:16.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:17.032 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:17.057 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:47:17.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:17.089 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:17.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:17.190 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:17.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:17.235 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:17.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:17.336 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:17.379 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:17.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:17.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:17.480 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:17.520 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:17.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:17.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:17.622 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:17.663 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:17.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:17.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:17.696 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:17 smithi129 ceph-mon[101316]: pgmap v2515: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 523 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 6089/25146 objects misplaced (24.215%); 1.9 MiB/s, 30 objects/s recovering 2024-09-14T04:47:17.764 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:17.796 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:17 smithi039 ceph-mon[106470]: pgmap v2515: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 523 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 6089/25146 objects misplaced (24.215%); 1.9 MiB/s, 30 objects/s recovering 2024-09-14T04:47:17.796 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:17 smithi039 ceph-mon[111235]: pgmap v2515: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 523 MiB data, 4.3 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 770 op/s; 6089/25146 objects misplaced (24.215%); 1.9 MiB/s, 30 objects/s recovering 2024-09-14T04:47:17.813 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:17.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:17.814 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:17.842 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:47:17.842 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:47:17.915 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:17.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:17.956 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:17.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:18.057 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:18.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:18.099 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:18.099 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:18.200 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:18.242 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:18.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:18.243 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:18.344 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:18.384 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:18.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:18.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:18.486 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:18.486 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:47:18.510 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:18.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:18.511 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:18.612 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:18.629 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-14T04:47:18.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:18.640 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:18.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:18.698 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:47:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:47:18] "GET /metrics HTTP/1.1" 200 36744 "" "Prometheus/2.51.0" 2024-09-14T04:47:18.741 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:18.786 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:18.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:18.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:18.888 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:18.925 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:18.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:18.927 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:19.004 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:47:19.028 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:19.069 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:19.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:19.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:19.171 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:19.204 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:19.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:19.206 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:19.306 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:19.343 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:19.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:19.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:19.445 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:19.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:19.498 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:19.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:19.599 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:19.634 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:19.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:19.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:19.736 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:19.770 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:19.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:19.771 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:19.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:19 smithi129 ceph-mon[101316]: pgmap v2516: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 418 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 5713/20097 objects misplaced (28.427%); 1.8 MiB/s, 29 objects/s recovering 2024-09-14T04:47:19.824 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:19 smithi039 ceph-mon[106470]: pgmap v2516: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 418 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 5713/20097 objects misplaced (28.427%); 1.8 MiB/s, 29 objects/s recovering 2024-09-14T04:47:19.825 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:19 smithi039 ceph-mon[111235]: pgmap v2516: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 418 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 774 op/s; 5713/20097 objects misplaced (28.427%); 1.8 MiB/s, 29 objects/s recovering 2024-09-14T04:47:19.873 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:19.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:19.922 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:19.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:20.023 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:20.068 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:20.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:20.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:20.170 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:20.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:20.203 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:20.203 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:20.304 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:20.362 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:20.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:20.364 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:20.464 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:20.495 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:20.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:20.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:20.597 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:20.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:20.660 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:20.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:20.754 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:20 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2083347149' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:47:20.754 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:20 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2083347149' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:47:20.755 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:47:20.755 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:47:20.755 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:47:20.755 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:47:20.755 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:47:20.755 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:47:20.755 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:47:20.756 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:47:20.756 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:47:20.756 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:47:20.756 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:47:20.756 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:47:20.756 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:47:20.756 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:47:20.756 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:47:20.756 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:47:20.761 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:20.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:20 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2083347149' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:47:20.803 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:20.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:20.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:20.905 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:20.937 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:20.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:20.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:20.956 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:47:20.956 INFO:teuthology.orchestra.run.smithi129.stdout: "size": 20, 2024-09-14T04:47:20.956 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 600, 2024-09-14T04:47:20.957 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [ 2024-09-14T04:47:20.957 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.957 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:242354 53.b 53:d6da5931:::benchmark_data_smithi039_224949_object112261:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1360)", 2024-09-14T04:47:20.957 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:18.417150+0000", 2024-09-14T04:47:20.957 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 302.53412156600001, 2024-09-14T04:47:20.957 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.040568178, 2024-09-14T04:47:20.957 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:20.957 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "reached pg", 2024-09-14T04:47:20.957 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:20.957 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:47:20.957 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:47:20.957 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 242354 2024-09-14T04:47:20.958 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.958 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:20.958 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.958 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:20.958 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.417150+0000", 2024-09-14T04:47:20.958 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:20.958 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.958 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.958 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:20.958 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.417150+0000", 2024-09-14T04:47:20.958 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:20.958 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.958 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.958 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:20.959 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.417149+0000", 2024-09-14T04:47:20.959 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:47:20.959 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.959 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.959 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:20.959 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.417155+0000", 2024-09-14T04:47:20.959 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.8270000000000003e-06 2024-09-14T04:47:20.959 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.959 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.959 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:20.959 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.417165+0000", 2024-09-14T04:47:20.959 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.9129999999999998e-06 2024-09-14T04:47:20.960 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.960 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.960 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:20.960 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.417172+0000", 2024-09-14T04:47:20.960 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.99e-06 2024-09-14T04:47:20.960 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.960 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.960 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:20.960 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.417225+0000", 2024-09-14T04:47:20.960 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.3115999999999998e-05 2024-09-14T04:47:20.960 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.961 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.961 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:20.961 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.417349+0000", 2024-09-14T04:47:20.961 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012388000000000001 2024-09-14T04:47:20.961 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.961 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.961 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,5,7", 2024-09-14T04:47:20.961 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.417490+0000", 2024-09-14T04:47:20.961 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00014062600000000001 2024-09-14T04:47:20.961 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.961 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.961 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:20.961 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.420836+0000", 2024-09-14T04:47:20.962 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0033468450000000002 2024-09-14T04:47:20.962 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.962 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.962 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:20.962 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.423608+0000", 2024-09-14T04:47:20.962 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.002771286 2024-09-14T04:47:20.962 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.962 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.962 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:47:20.962 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.423617+0000", 2024-09-14T04:47:20.962 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.217e-06 2024-09-14T04:47:20.962 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.963 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.963 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:20.963 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.457659+0000", 2024-09-14T04:47:20.963 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.034042058 2024-09-14T04:47:20.963 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.963 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.963 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:20.963 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.457718+0000", 2024-09-14T04:47:20.963 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9401000000000003e-05 2024-09-14T04:47:20.963 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:20.963 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:20.963 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:20.964 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.964 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.964 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:242361 53.b 53:d8e2cfe0:::benchmark_data_smithi039_224949_object112268:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1361)", 2024-09-14T04:47:20.964 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:18.428956+0000", 2024-09-14T04:47:20.964 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 302.52231566, 2024-09-14T04:47:20.964 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0342326209999999, 2024-09-14T04:47:20.964 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:20.964 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:20.964 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:20.964 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:47:20.964 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:47:20.964 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 242361 2024-09-14T04:47:20.964 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.965 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:20.965 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.965 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:20.965 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.428956+0000", 2024-09-14T04:47:20.965 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:20.965 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.965 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.965 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:20.965 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.428956+0000", 2024-09-14T04:47:20.965 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:20.965 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.965 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.965 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:20.966 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.428955+0000", 2024-09-14T04:47:20.966 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:47:20.966 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.966 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.966 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:20.966 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.428959+0000", 2024-09-14T04:47:20.966 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0330000000000002e-06 2024-09-14T04:47:20.966 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.966 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.966 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:20.966 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.428969+0000", 2024-09-14T04:47:20.966 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0000000000000001e-05 2024-09-14T04:47:20.966 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.967 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.967 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:20.967 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.428974+0000", 2024-09-14T04:47:20.967 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.091e-06 2024-09-14T04:47:20.967 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.967 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.967 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:20.967 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.429010+0000", 2024-09-14T04:47:20.967 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5657000000000001e-05 2024-09-14T04:47:20.967 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.967 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.967 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:47:20.967 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.429018+0000", 2024-09-14T04:47:20.968 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.7789999999999996e-06 2024-09-14T04:47:20.968 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.968 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.968 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:20.968 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.458043+0000", 2024-09-14T04:47:20.968 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0290252070000001 2024-09-14T04:47:20.968 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.968 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.968 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:20.968 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.458132+0000", 2024-09-14T04:47:20.968 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.9197000000000007e-05 2024-09-14T04:47:20.968 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.968 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.969 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,7", 2024-09-14T04:47:20.969 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.458222+0000", 2024-09-14T04:47:20.969 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.9803999999999996e-05 2024-09-14T04:47:20.969 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.969 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.969 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:20.969 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.463013+0000", 2024-09-14T04:47:20.969 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0047913160000000003 2024-09-14T04:47:20.969 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.969 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.969 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:20.969 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.463044+0000", 2024-09-14T04:47:20.969 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.1273999999999999e-05 2024-09-14T04:47:20.970 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.970 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.970 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:20.970 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.463109+0000", 2024-09-14T04:47:20.970 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.4573000000000007e-05 2024-09-14T04:47:20.970 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.970 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.970 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:20.970 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.463150+0000", 2024-09-14T04:47:20.970 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0914999999999997e-05 2024-09-14T04:47:20.970 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.970 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.970 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:20.971 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.463189+0000", 2024-09-14T04:47:20.971 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8797999999999997e-05 2024-09-14T04:47:20.971 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:20.971 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:20.971 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:20.971 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.971 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.971 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:242368 53.b 53:d65b13d1:::benchmark_data_smithi039_224949_object112275:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1361)", 2024-09-14T04:47:20.971 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:18.435347+0000", 2024-09-14T04:47:20.972 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 302.51592497299998, 2024-09-14T04:47:20.972 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.027992153, 2024-09-14T04:47:20.972 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:20.972 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:20.972 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:20.972 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:47:20.972 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:47:20.973 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 242368 2024-09-14T04:47:20.973 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.973 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:20.973 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.973 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:20.973 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.435347+0000", 2024-09-14T04:47:20.973 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:20.973 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.973 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.973 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:20.973 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.435347+0000", 2024-09-14T04:47:20.973 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:20.973 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.973 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.974 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:20.974 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.435346+0000", 2024-09-14T04:47:20.974 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:47:20.974 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.974 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.974 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:20.974 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.435349+0000", 2024-09-14T04:47:20.974 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.9720000000000001e-06 2024-09-14T04:47:20.974 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.974 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.974 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:20.974 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.435356+0000", 2024-09-14T04:47:20.974 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.1709999999999999e-06 2024-09-14T04:47:20.974 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.975 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.975 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:20.975 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.435361+0000", 2024-09-14T04:47:20.975 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.0329999999999999e-06 2024-09-14T04:47:20.975 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.975 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.975 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:20.975 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.435408+0000", 2024-09-14T04:47:20.975 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7414999999999999e-05 2024-09-14T04:47:20.975 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.975 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.975 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:47:20.975 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.435415+0000", 2024-09-14T04:47:20.975 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.1690000000000003e-06 2024-09-14T04:47:20.976 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.976 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.976 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:20.976 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.460686+0000", 2024-09-14T04:47:20.976 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.025271252 2024-09-14T04:47:20.976 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.976 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.976 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:20.976 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.460781+0000", 2024-09-14T04:47:20.976 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.4660999999999994e-05 2024-09-14T04:47:20.976 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.976 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.976 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,7", 2024-09-14T04:47:20.977 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.460912+0000", 2024-09-14T04:47:20.977 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000130648 2024-09-14T04:47:20.977 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.977 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.977 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:20.977 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.463242+0000", 2024-09-14T04:47:20.977 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0023300450000000002 2024-09-14T04:47:20.977 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.977 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.977 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:20.977 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.463279+0000", 2024-09-14T04:47:20.977 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6857000000000003e-05 2024-09-14T04:47:20.977 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.978 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.978 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:20.978 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.463283+0000", 2024-09-14T04:47:20.978 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.758e-06 2024-09-14T04:47:20.978 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.978 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.978 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:20.978 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.463307+0000", 2024-09-14T04:47:20.978 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.3796000000000001e-05 2024-09-14T04:47:20.978 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.978 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.978 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:20.978 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.463339+0000", 2024-09-14T04:47:20.979 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.1772e-05 2024-09-14T04:47:20.979 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:20.979 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:20.979 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:20.979 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.979 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.979 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:242393 53.b 53:da66bab0:::benchmark_data_smithi039_224949_object112300:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1361)", 2024-09-14T04:47:20.979 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:18.475483+0000", 2024-09-14T04:47:20.979 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 302.47578910200002, 2024-09-14T04:47:20.979 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.99403772300000004, 2024-09-14T04:47:20.979 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:20.979 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:20.979 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:20.980 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:47:20.980 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:47:20.980 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 242393 2024-09-14T04:47:20.980 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.980 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:20.980 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.980 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:20.980 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.475483+0000", 2024-09-14T04:47:20.980 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:20.980 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.980 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.980 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:20.980 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.475483+0000", 2024-09-14T04:47:20.981 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:20.981 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.981 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.981 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:20.981 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.475482+0000", 2024-09-14T04:47:20.981 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:47:20.981 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.981 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.981 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:20.981 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.475484+0000", 2024-09-14T04:47:20.981 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.1919999999999999e-06 2024-09-14T04:47:20.981 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.981 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.981 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:20.982 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.475491+0000", 2024-09-14T04:47:20.982 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.0680000000000002e-06 2024-09-14T04:47:20.982 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.982 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.982 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:20.982 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.475495+0000", 2024-09-14T04:47:20.982 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.4789999999999998e-06 2024-09-14T04:47:20.982 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.982 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.982 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:20.982 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.475522+0000", 2024-09-14T04:47:20.982 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.7087999999999999e-05 2024-09-14T04:47:20.982 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.982 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.982 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:47:20.982 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.475533+0000", 2024-09-14T04:47:20.983 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0959e-05 2024-09-14T04:47:20.983 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.983 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.983 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:20.983 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.463375+0000", 2024-09-14T04:47:20.983 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.98784208600000001 2024-09-14T04:47:20.983 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.983 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.983 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:20.983 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.463484+0000", 2024-09-14T04:47:20.983 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000108383 2024-09-14T04:47:20.983 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.983 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.983 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,7", 2024-09-14T04:47:20.984 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.463597+0000", 2024-09-14T04:47:20.984 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011325299999999999 2024-09-14T04:47:20.984 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.984 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.984 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:20.984 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.469383+0000", 2024-09-14T04:47:20.984 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0057862460000000001 2024-09-14T04:47:20.984 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.984 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.984 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:20.984 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.469426+0000", 2024-09-14T04:47:20.984 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.2719999999999998e-05 2024-09-14T04:47:20.984 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.984 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.985 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:20.985 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.469434+0000", 2024-09-14T04:47:20.985 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.5299999999999996e-06 2024-09-14T04:47:20.985 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.985 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.985 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:20.985 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.469483+0000", 2024-09-14T04:47:20.985 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8987999999999999e-05 2024-09-14T04:47:20.985 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.985 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.985 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:20.985 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.469520+0000", 2024-09-14T04:47:20.986 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7109e-05 2024-09-14T04:47:20.986 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:20.986 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:20.986 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:20.986 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.986 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.986 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:242405 53.b 53:dba4dc8d:::benchmark_data_smithi039_224949_object112312:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1361)", 2024-09-14T04:47:20.986 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:18.494349+0000", 2024-09-14T04:47:20.986 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 302.456922943, 2024-09-14T04:47:20.986 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.97533815000000001, 2024-09-14T04:47:20.986 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:20.986 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:20.986 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:20.986 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:47:20.986 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:47:20.987 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 242405 2024-09-14T04:47:20.987 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.987 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:20.987 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.987 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:20.987 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.494349+0000", 2024-09-14T04:47:20.987 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:20.987 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.987 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.987 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:20.987 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.494349+0000", 2024-09-14T04:47:20.987 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:20.987 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.987 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.988 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:20.988 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.494347+0000", 2024-09-14T04:47:20.988 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:47:20.988 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.988 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.988 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:20.988 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.494354+0000", 2024-09-14T04:47:20.988 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7490000000000001e-06 2024-09-14T04:47:20.988 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.988 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.988 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:20.988 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.494367+0000", 2024-09-14T04:47:20.988 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2754e-05 2024-09-14T04:47:20.989 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.989 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.989 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:20.989 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.494374+0000", 2024-09-14T04:47:20.989 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7379999999999997e-06 2024-09-14T04:47:20.989 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.989 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.989 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:20.989 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.494401+0000", 2024-09-14T04:47:20.989 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.7146999999999999e-05 2024-09-14T04:47:20.989 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.989 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.989 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:47:20.990 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.494406+0000", 2024-09-14T04:47:20.990 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8409999999999999e-06 2024-09-14T04:47:20.990 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.990 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.990 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:20.990 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.466391+0000", 2024-09-14T04:47:20.990 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.97198570100000004 2024-09-14T04:47:20.990 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.990 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.990 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:20.990 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.466475+0000", 2024-09-14T04:47:20.990 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.4085999999999994e-05 2024-09-14T04:47:20.990 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.991 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.991 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,7", 2024-09-14T04:47:20.991 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.466556+0000", 2024-09-14T04:47:20.991 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.0914999999999994e-05 2024-09-14T04:47:20.991 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.991 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.991 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:20.991 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.469559+0000", 2024-09-14T04:47:20.991 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0030027360000000002 2024-09-14T04:47:20.991 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.991 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.991 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:20.991 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.469611+0000", 2024-09-14T04:47:20.992 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2157999999999998e-05 2024-09-14T04:47:20.992 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.992 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.992 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:20.992 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.469622+0000", 2024-09-14T04:47:20.992 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0708e-05 2024-09-14T04:47:20.992 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.992 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.992 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:20.992 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.469656+0000", 2024-09-14T04:47:20.992 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.4192000000000001e-05 2024-09-14T04:47:20.992 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.992 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.992 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:20.993 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.469687+0000", 2024-09-14T04:47:20.993 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.0895999999999997e-05 2024-09-14T04:47:20.993 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:20.993 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:20.993 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:20.993 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.993 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.993 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:242418 53.b 53:d24c660c:::benchmark_data_smithi039_224949_object112325:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1361)", 2024-09-14T04:47:20.993 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:18.509160+0000", 2024-09-14T04:47:20.993 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 302.442111458, 2024-09-14T04:47:20.993 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.96705065000000001, 2024-09-14T04:47:20.993 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:20.993 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:20.994 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:20.994 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:47:20.994 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:47:20.994 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 242418 2024-09-14T04:47:20.994 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.994 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:20.994 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.994 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:20.994 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.509160+0000", 2024-09-14T04:47:20.994 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:20.994 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.994 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.995 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:20.995 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.509160+0000", 2024-09-14T04:47:20.995 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:20.995 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.995 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.995 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:20.995 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.509159+0000", 2024-09-14T04:47:20.995 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:47:20.995 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.995 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.995 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:20.995 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.509167+0000", 2024-09-14T04:47:20.995 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.6799999999999993e-06 2024-09-14T04:47:20.995 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.995 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.996 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:20.996 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.509176+0000", 2024-09-14T04:47:20.996 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.7900000000000005e-06 2024-09-14T04:47:20.996 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.996 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.996 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:20.996 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.509182+0000", 2024-09-14T04:47:20.996 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.4609999999999996e-06 2024-09-14T04:47:20.996 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.996 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.996 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:20.996 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.509218+0000", 2024-09-14T04:47:20.996 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6387999999999998e-05 2024-09-14T04:47:20.996 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.996 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.996 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:47:20.997 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.509225+0000", 2024-09-14T04:47:20.997 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8519999999999998e-06 2024-09-14T04:47:20.997 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.997 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.997 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:20.997 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.469727+0000", 2024-09-14T04:47:20.997 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.96050160699999998 2024-09-14T04:47:20.997 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.997 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.997 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:20.997 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.469881+0000", 2024-09-14T04:47:20.997 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00015403500000000001 2024-09-14T04:47:20.997 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.997 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.998 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,7", 2024-09-14T04:47:20.998 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.469992+0000", 2024-09-14T04:47:20.998 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000111032 2024-09-14T04:47:20.998 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.998 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.998 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:20.998 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.476032+0000", 2024-09-14T04:47:20.998 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0060400009999999997 2024-09-14T04:47:20.998 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.998 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.998 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:20.998 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.476074+0000", 2024-09-14T04:47:20.998 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.2287999999999999e-05 2024-09-14T04:47:20.998 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.998 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.998 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:20.999 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.476123+0000", 2024-09-14T04:47:20.999 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8888999999999999e-05 2024-09-14T04:47:20.999 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.999 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.999 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:20.999 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.476170+0000", 2024-09-14T04:47:20.999 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6896e-05 2024-09-14T04:47:20.999 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:20.999 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:20.999 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:20.999 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.476211+0000", 2024-09-14T04:47:20.999 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0904e-05 2024-09-14T04:47:20.999 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:20.999 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:21.000 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:21.000 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.000 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.000 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:242435 53.b 53:d0890965:::benchmark_data_smithi039_224949_object112342:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1361)", 2024-09-14T04:47:21.000 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:18.537973+0000", 2024-09-14T04:47:21.000 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 302.41329886900002, 2024-09-14T04:47:21.000 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.94175468799999995, 2024-09-14T04:47:21.000 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:21.000 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:21.000 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:21.000 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:47:21.000 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:47:21.000 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 242435 2024-09-14T04:47:21.001 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.001 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:21.001 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.001 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:21.001 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.537973+0000", 2024-09-14T04:47:21.001 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:21.001 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.001 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.001 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:21.001 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.537973+0000", 2024-09-14T04:47:21.001 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:21.001 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.001 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.002 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:21.002 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.537972+0000", 2024-09-14T04:47:21.002 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:47:21.002 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.002 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.002 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:21.002 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.537975+0000", 2024-09-14T04:47:21.002 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3189999999999999e-06 2024-09-14T04:47:21.002 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.002 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.002 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:21.002 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.537984+0000", 2024-09-14T04:47:21.002 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.3559999999999993e-06 2024-09-14T04:47:21.002 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.002 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.003 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:21.003 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.537989+0000", 2024-09-14T04:47:21.003 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.0370000000000002e-06 2024-09-14T04:47:21.003 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.003 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.003 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:21.003 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.538021+0000", 2024-09-14T04:47:21.003 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.2191999999999999e-05 2024-09-14T04:47:21.003 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.003 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.003 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:47:21.003 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.538027+0000", 2024-09-14T04:47:21.003 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.3790000000000003e-06 2024-09-14T04:47:21.003 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.004 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.004 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:21.004 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.472855+0000", 2024-09-14T04:47:21.004 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.934827886 2024-09-14T04:47:21.004 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.004 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.004 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:21.004 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.472930+0000", 2024-09-14T04:47:21.004 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.5102e-05 2024-09-14T04:47:21.004 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.004 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.004 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,7", 2024-09-14T04:47:21.004 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.473040+0000", 2024-09-14T04:47:21.004 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000109338 2024-09-14T04:47:21.004 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.005 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.005 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:21.005 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.476114+0000", 2024-09-14T04:47:21.005 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0030742320000000001 2024-09-14T04:47:21.005 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.005 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.005 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:21.005 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.476260+0000", 2024-09-14T04:47:21.005 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000146577 2024-09-14T04:47:21.005 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.005 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.005 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:21.005 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.479646+0000", 2024-09-14T04:47:21.006 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.003385824 2024-09-14T04:47:21.006 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.006 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.006 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:21.006 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.479691+0000", 2024-09-14T04:47:21.006 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.4574999999999999e-05 2024-09-14T04:47:21.006 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.006 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.006 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:21.006 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.479728+0000", 2024-09-14T04:47:21.006 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6785000000000001e-05 2024-09-14T04:47:21.006 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:21.006 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:21.006 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:21.006 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.007 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.007 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:242465 53.b 53:dede790f:::benchmark_data_smithi039_224949_object112372:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1361)", 2024-09-14T04:47:21.007 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:18.575117+0000", 2024-09-14T04:47:21.007 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 302.37615517199998, 2024-09-14T04:47:21.007 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.90764941899999996, 2024-09-14T04:47:21.007 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:21.007 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:21.007 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:21.007 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:47:21.007 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:47:21.007 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 242465 2024-09-14T04:47:21.007 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.007 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:21.007 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.008 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:21.008 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.575117+0000", 2024-09-14T04:47:21.008 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:21.008 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.008 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.008 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:21.008 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.575117+0000", 2024-09-14T04:47:21.008 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:21.008 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.008 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.008 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:21.008 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.575115+0000", 2024-09-14T04:47:21.008 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:47:21.009 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.009 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.009 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:21.009 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.575120+0000", 2024-09-14T04:47:21.009 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.062e-06 2024-09-14T04:47:21.009 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.009 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.009 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:21.009 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.575129+0000", 2024-09-14T04:47:21.009 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.6400000000000003e-06 2024-09-14T04:47:21.009 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.009 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.009 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:21.009 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.575135+0000", 2024-09-14T04:47:21.010 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9699999999999996e-06 2024-09-14T04:47:21.010 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.010 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.010 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:21.010 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.575172+0000", 2024-09-14T04:47:21.010 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6658e-05 2024-09-14T04:47:21.010 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.010 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.010 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:47:21.010 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.575179+0000", 2024-09-14T04:47:21.010 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.8800000000000008e-06 2024-09-14T04:47:21.010 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.010 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.010 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:21.011 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.476297+0000", 2024-09-14T04:47:21.011 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.90111766999999998 2024-09-14T04:47:21.011 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.011 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.011 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:21.011 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.476415+0000", 2024-09-14T04:47:21.011 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000118299 2024-09-14T04:47:21.011 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.011 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.011 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,7", 2024-09-14T04:47:21.011 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.476523+0000", 2024-09-14T04:47:21.011 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000107892 2024-09-14T04:47:21.011 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.011 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.011 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:21.011 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.482618+0000", 2024-09-14T04:47:21.012 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0060950479999999996 2024-09-14T04:47:21.012 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.012 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.012 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:21.012 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.482670+0000", 2024-09-14T04:47:21.012 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.1765999999999999e-05 2024-09-14T04:47:21.012 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.012 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.012 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:21.012 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.482681+0000", 2024-09-14T04:47:21.012 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0641e-05 2024-09-14T04:47:21.012 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.012 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.012 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:21.013 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.482727+0000", 2024-09-14T04:47:21.013 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6388999999999999e-05 2024-09-14T04:47:21.013 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.013 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.013 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:21.013 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.482766+0000", 2024-09-14T04:47:21.013 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8819000000000002e-05 2024-09-14T04:47:21.013 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:21.013 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:21.013 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:21.013 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.013 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.014 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:242468 53.b 53:d7a745df:::benchmark_data_smithi039_224949_object112375:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1361)", 2024-09-14T04:47:21.014 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:18.576916+0000", 2024-09-14T04:47:21.014 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 302.37435601499999, 2024-09-14T04:47:21.014 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.90601413900000005, 2024-09-14T04:47:21.014 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:21.014 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:21.014 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:21.015 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:47:21.015 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:47:21.015 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 242468 2024-09-14T04:47:21.015 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.015 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:21.015 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.015 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:21.015 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.576916+0000", 2024-09-14T04:47:21.015 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:21.015 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.015 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.015 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:21.015 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.576916+0000", 2024-09-14T04:47:21.016 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:21.016 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.016 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.016 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:21.016 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.576915+0000", 2024-09-14T04:47:21.016 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:47:21.016 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.016 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.016 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:21.016 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.576920+0000", 2024-09-14T04:47:21.017 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.9060000000000001e-06 2024-09-14T04:47:21.017 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.017 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.017 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:21.017 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.576929+0000", 2024-09-14T04:47:21.017 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.3759999999999997e-06 2024-09-14T04:47:21.017 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.017 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.017 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:21.017 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.576935+0000", 2024-09-14T04:47:21.017 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.4559999999999997e-06 2024-09-14T04:47:21.017 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.017 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.018 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:21.018 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.576975+0000", 2024-09-14T04:47:21.018 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9749999999999997e-05 2024-09-14T04:47:21.018 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.018 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.018 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:47:21.018 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.576983+0000", 2024-09-14T04:47:21.018 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.9400000000000002e-06 2024-09-14T04:47:21.018 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.018 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.018 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:21.018 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.479759+0000", 2024-09-14T04:47:21.018 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.90277581299999998 2024-09-14T04:47:21.019 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.019 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.019 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:21.019 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.479861+0000", 2024-09-14T04:47:21.019 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000102144 2024-09-14T04:47:21.019 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.019 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.019 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,7", 2024-09-14T04:47:21.019 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.479979+0000", 2024-09-14T04:47:21.019 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000117478 2024-09-14T04:47:21.019 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.019 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.019 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:21.020 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.482806+0000", 2024-09-14T04:47:21.020 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0028272219999999999 2024-09-14T04:47:21.020 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.020 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.020 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:21.020 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.482813+0000", 2024-09-14T04:47:21.020 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.9979999999999997e-06 2024-09-14T04:47:21.020 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.020 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.020 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:21.020 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.482848+0000", 2024-09-14T04:47:21.020 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5005000000000001e-05 2024-09-14T04:47:21.020 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.021 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.021 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:21.021 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.482886+0000", 2024-09-14T04:47:21.021 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7744000000000003e-05 2024-09-14T04:47:21.021 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.021 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.021 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:21.021 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.482930+0000", 2024-09-14T04:47:21.021 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.4332999999999997e-05 2024-09-14T04:47:21.021 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:21.021 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:21.021 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:21.021 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.022 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.022 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:242470 53.b 53:d9e9853e:::benchmark_data_smithi039_224949_object112377:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1361)", 2024-09-14T04:47:21.022 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:18.577075+0000", 2024-09-14T04:47:21.022 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 302.37419719600001, 2024-09-14T04:47:21.022 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.91221598500000001, 2024-09-14T04:47:21.022 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:21.022 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:21.022 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:21.022 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:47:21.022 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:47:21.022 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 242470 2024-09-14T04:47:21.022 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.022 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:21.022 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.023 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:21.023 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.577075+0000", 2024-09-14T04:47:21.023 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:21.023 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.023 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.023 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:21.023 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.577075+0000", 2024-09-14T04:47:21.023 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:21.023 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.023 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.023 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:21.023 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.577074+0000", 2024-09-14T04:47:21.023 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:47:21.024 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.024 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.024 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:21.024 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.577078+0000", 2024-09-14T04:47:21.024 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9369999999999997e-06 2024-09-14T04:47:21.024 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.024 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.024 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:21.024 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.577089+0000", 2024-09-14T04:47:21.024 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0681e-05 2024-09-14T04:47:21.024 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.024 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.025 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:21.025 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.577095+0000", 2024-09-14T04:47:21.025 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.0270000000000001e-06 2024-09-14T04:47:21.025 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.025 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.025 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:21.025 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.577134+0000", 2024-09-14T04:47:21.025 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9765999999999998e-05 2024-09-14T04:47:21.025 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.025 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.025 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:47:21.025 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.577142+0000", 2024-09-14T04:47:21.025 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.7770000000000002e-06 2024-09-14T04:47:21.026 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.026 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.026 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:21.026 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.482959+0000", 2024-09-14T04:47:21.026 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.90581725000000002 2024-09-14T04:47:21.026 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.026 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.026 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:21.026 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.483059+0000", 2024-09-14T04:47:21.026 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.9296999999999995e-05 2024-09-14T04:47:21.026 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.026 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.026 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,7", 2024-09-14T04:47:21.027 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.483173+0000", 2024-09-14T04:47:21.027 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000114776 2024-09-14T04:47:21.027 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.027 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.027 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:21.027 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.489117+0000", 2024-09-14T04:47:21.027 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0059437750000000001 2024-09-14T04:47:21.027 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.027 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.027 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:21.027 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.489175+0000", 2024-09-14T04:47:21.027 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.7401000000000002e-05 2024-09-14T04:47:21.027 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.028 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.028 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:21.028 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.489183+0000", 2024-09-14T04:47:21.028 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.6880000000000005e-06 2024-09-14T04:47:21.028 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.028 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.028 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:21.028 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.489228+0000", 2024-09-14T04:47:21.028 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.5062000000000003e-05 2024-09-14T04:47:21.028 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.028 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.028 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:21.028 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.489291+0000", 2024-09-14T04:47:21.029 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.2215e-05 2024-09-14T04:47:21.029 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:21.029 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:21.029 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:21.029 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.029 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.029 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:242479 53.b 53:d7ce29be:::benchmark_data_smithi039_224949_object112386:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1361)", 2024-09-14T04:47:21.029 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:18.588778+0000", 2024-09-14T04:47:21.029 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 302.36249393600002, 2024-09-14T04:47:21.029 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.90070344400000002, 2024-09-14T04:47:21.029 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:21.029 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:21.029 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:21.030 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:47:21.030 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:47:21.030 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 242479 2024-09-14T04:47:21.030 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.030 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:21.030 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.030 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:21.030 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.588778+0000", 2024-09-14T04:47:21.030 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:21.030 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.030 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.030 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:21.030 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.588778+0000", 2024-09-14T04:47:21.031 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:21.031 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.031 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.031 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:21.031 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.588777+0000", 2024-09-14T04:47:21.031 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:47:21.031 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.031 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.031 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:21.031 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.588782+0000", 2024-09-14T04:47:21.031 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.519e-06 2024-09-14T04:47:21.031 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.031 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.031 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:21.032 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.588796+0000", 2024-09-14T04:47:21.032 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3577999999999999e-05 2024-09-14T04:47:21.032 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.032 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.032 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:21.032 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.588801+0000", 2024-09-14T04:47:21.032 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.6319999999999996e-06 2024-09-14T04:47:21.032 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.032 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.032 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:21.032 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.588844+0000", 2024-09-14T04:47:21.032 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.2589999999999997e-05 2024-09-14T04:47:21.032 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.032 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.033 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:47:21.033 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.588850+0000", 2024-09-14T04:47:21.033 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.8760000000000002e-06 2024-09-14T04:47:21.033 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.033 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.033 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:21.033 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.485864+0000", 2024-09-14T04:47:21.033 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.89701419400000004 2024-09-14T04:47:21.033 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.033 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.033 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:21.033 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.485980+0000", 2024-09-14T04:47:21.033 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000116445 2024-09-14T04:47:21.034 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.034 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.034 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,7", 2024-09-14T04:47:21.034 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.486102+0000", 2024-09-14T04:47:21.034 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000121067 2024-09-14T04:47:21.034 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.034 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.034 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:21.034 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.489340+0000", 2024-09-14T04:47:21.034 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0032381430000000002 2024-09-14T04:47:21.034 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.034 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.034 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:21.034 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.489394+0000", 2024-09-14T04:47:21.035 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.4245e-05 2024-09-14T04:47:21.035 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.035 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.035 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:21.035 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.489403+0000", 2024-09-14T04:47:21.035 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.3079999999999997e-06 2024-09-14T04:47:21.035 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.035 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.035 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:21.035 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.489445+0000", 2024-09-14T04:47:21.035 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1498000000000001e-05 2024-09-14T04:47:21.035 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.035 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.035 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:21.036 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.489481+0000", 2024-09-14T04:47:21.036 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6520000000000003e-05 2024-09-14T04:47:21.036 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:21.036 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:21.036 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:21.036 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.036 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.036 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:242483 53.b 53:dfc667bf:::benchmark_data_smithi039_224949_object112390:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1361)", 2024-09-14T04:47:21.036 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:18.593502+0000", 2024-09-14T04:47:21.036 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 302.35776991699998, 2024-09-14T04:47:21.036 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.913349623, 2024-09-14T04:47:21.036 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:21.036 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:21.037 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:21.037 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:47:21.037 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:47:21.037 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 242483 2024-09-14T04:47:21.037 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.037 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:21.037 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.037 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:21.037 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.593502+0000", 2024-09-14T04:47:21.037 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:21.037 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.037 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.037 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:21.037 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.593502+0000", 2024-09-14T04:47:21.038 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:21.038 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.038 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.038 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:21.038 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.593500+0000", 2024-09-14T04:47:21.038 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:47:21.038 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.038 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.038 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:21.038 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.593507+0000", 2024-09-14T04:47:21.038 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.4810000000000003e-06 2024-09-14T04:47:21.038 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.038 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.038 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:21.039 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.593521+0000", 2024-09-14T04:47:21.039 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3875e-05 2024-09-14T04:47:21.039 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.039 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.039 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:21.039 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.593529+0000", 2024-09-14T04:47:21.039 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.3739999999999998e-06 2024-09-14T04:47:21.039 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.039 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.039 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:21.039 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.593574+0000", 2024-09-14T04:47:21.039 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.4861000000000003e-05 2024-09-14T04:47:21.039 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.039 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.040 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:47:21.040 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.593582+0000", 2024-09-14T04:47:21.040 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.4120000000000001e-06 2024-09-14T04:47:21.040 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.040 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.040 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:21.040 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.489521+0000", 2024-09-14T04:47:21.040 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.89593884899999998 2024-09-14T04:47:21.040 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.040 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.040 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:21.040 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.489623+0000", 2024-09-14T04:47:21.040 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000101576 2024-09-14T04:47:21.041 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.041 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.041 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,7", 2024-09-14T04:47:21.041 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.489725+0000", 2024-09-14T04:47:21.041 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000102738 2024-09-14T04:47:21.041 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.041 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.041 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:21.041 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.499702+0000", 2024-09-14T04:47:21.041 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0099765649999999997 2024-09-14T04:47:21.041 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.041 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.041 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:21.042 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.499739+0000", 2024-09-14T04:47:21.042 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.693e-05 2024-09-14T04:47:21.042 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.042 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.042 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:21.042 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.506768+0000", 2024-09-14T04:47:21.042 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0070286790000000004 2024-09-14T04:47:21.042 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.042 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.042 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:21.042 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.506813+0000", 2024-09-14T04:47:21.042 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.566e-05 2024-09-14T04:47:21.042 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.042 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.042 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:21.043 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.506851+0000", 2024-09-14T04:47:21.043 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8151000000000001e-05 2024-09-14T04:47:21.043 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:21.043 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:21.043 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:21.043 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.043 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.043 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:242490 53.b 53:da67a072:::benchmark_data_smithi039_224949_object112397:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1361)", 2024-09-14T04:47:21.043 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:18.605054+0000", 2024-09-14T04:47:21.043 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 302.34621793999997, 2024-09-14T04:47:21.043 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.90192349199999999, 2024-09-14T04:47:21.043 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:21.043 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:21.044 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:21.044 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:47:21.044 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:47:21.044 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 242490 2024-09-14T04:47:21.044 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.044 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:21.044 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.044 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:21.044 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.605054+0000", 2024-09-14T04:47:21.044 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:21.044 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.044 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.044 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:21.044 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.605054+0000", 2024-09-14T04:47:21.044 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:21.045 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.045 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.045 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:21.045 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.605053+0000", 2024-09-14T04:47:21.045 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:47:21.045 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.045 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.045 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:21.045 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.605058+0000", 2024-09-14T04:47:21.045 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6589999999999998e-06 2024-09-14T04:47:21.045 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.045 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.045 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:21.045 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.605068+0000", 2024-09-14T04:47:21.046 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0118999999999999e-05 2024-09-14T04:47:21.046 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.046 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.046 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:21.046 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.605075+0000", 2024-09-14T04:47:21.046 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.3640000000000001e-06 2024-09-14T04:47:21.046 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.046 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.046 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:21.046 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.605103+0000", 2024-09-14T04:47:21.046 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.8405999999999999e-05 2024-09-14T04:47:21.046 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.046 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.046 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:47:21.046 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:18.605109+0000", 2024-09-14T04:47:21.047 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.4829999999999999e-06 2024-09-14T04:47:21.047 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.047 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.047 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:21.047 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.492589+0000", 2024-09-14T04:47:21.047 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.887479711 2024-09-14T04:47:21.047 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.047 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.047 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:21.047 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.492697+0000", 2024-09-14T04:47:21.047 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000108318 2024-09-14T04:47:21.047 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.047 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.047 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,7", 2024-09-14T04:47:21.048 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.492863+0000", 2024-09-14T04:47:21.048 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00016623899999999999 2024-09-14T04:47:21.048 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.048 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.048 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:21.048 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.499732+0000", 2024-09-14T04:47:21.048 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0068684949999999996 2024-09-14T04:47:21.048 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.048 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.048 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:21.048 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.499745+0000", 2024-09-14T04:47:21.048 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3271999999999999e-05 2024-09-14T04:47:21.048 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.048 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.049 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:21.049 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.506891+0000", 2024-09-14T04:47:21.049 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0071456949999999997 2024-09-14T04:47:21.049 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.049 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.049 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:21.049 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.506934+0000", 2024-09-14T04:47:21.049 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.2852999999999998e-05 2024-09-14T04:47:21.049 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.049 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.049 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:21.049 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:19.506977+0000", 2024-09-14T04:47:21.049 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.3767000000000003e-05 2024-09-14T04:47:21.049 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:21.050 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:21.050 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:21.050 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.050 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.050 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:242884 53.f 53:f45bf103:::benchmark_data_smithi039_224949_object112791:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1363)", 2024-09-14T04:47:21.050 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:20.516136+0000", 2024-09-14T04:47:21.050 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 300.43513616899997, 2024-09-14T04:47:21.050 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.970750369, 2024-09-14T04:47:21.050 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:21.050 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:21.050 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:21.050 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:47:21.050 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:47:21.050 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 242884 2024-09-14T04:47:21.051 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.051 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:21.051 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.051 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:21.051 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.516136+0000", 2024-09-14T04:47:21.051 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:21.051 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.051 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.051 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:21.051 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.516136+0000", 2024-09-14T04:47:21.051 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:21.051 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.051 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.051 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:21.052 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.516134+0000", 2024-09-14T04:47:21.052 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:47:21.052 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.052 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.052 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:21.052 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.516141+0000", 2024-09-14T04:47:21.052 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.6590000000000001e-06 2024-09-14T04:47:21.052 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.052 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.052 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:21.052 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.516153+0000", 2024-09-14T04:47:21.052 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2283000000000001e-05 2024-09-14T04:47:21.052 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.052 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.053 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:21.053 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.516160+0000", 2024-09-14T04:47:21.053 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.216e-06 2024-09-14T04:47:21.053 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.053 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.053 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:21.053 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.516217+0000", 2024-09-14T04:47:21.053 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.6669999999999998e-05 2024-09-14T04:47:21.053 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.053 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.053 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:47:21.053 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.516226+0000", 2024-09-14T04:47:21.053 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.4420000000000004e-06 2024-09-14T04:47:21.054 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.054 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.054 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:21.054 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.478878+0000", 2024-09-14T04:47:21.054 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.96265180900000002 2024-09-14T04:47:21.054 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.054 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.054 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:21.054 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.478985+0000", 2024-09-14T04:47:21.054 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000107157 2024-09-14T04:47:21.054 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.054 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.054 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:47:21.054 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.479064+0000", 2024-09-14T04:47:21.055 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.8378000000000001e-05 2024-09-14T04:47:21.055 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.055 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.055 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:21.055 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.486721+0000", 2024-09-14T04:47:21.055 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0076568080000000002 2024-09-14T04:47:21.055 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.055 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.055 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:21.055 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.486727+0000", 2024-09-14T04:47:21.055 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.6309999999999997e-06 2024-09-14T04:47:21.055 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.055 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.055 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:21.056 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.486800+0000", 2024-09-14T04:47:21.056 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.2910000000000005e-05 2024-09-14T04:47:21.056 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.056 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.056 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:21.056 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.486847+0000", 2024-09-14T04:47:21.056 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:21.057 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7067000000000002e-05 2024-09-14T04:47:21.057 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.058 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.058 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:21.058 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.486886+0000", 2024-09-14T04:47:21.058 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8828999999999997e-05 2024-09-14T04:47:21.058 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:21.058 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:21.058 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:21.058 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.058 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.058 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:242892 53.f 53:ffb20b22:::benchmark_data_smithi039_224949_object112799:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1363)", 2024-09-14T04:47:21.058 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:20.523049+0000", 2024-09-14T04:47:21.058 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 300.42822285199998, 2024-09-14T04:47:21.059 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.97132038600000004, 2024-09-14T04:47:21.059 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:21.059 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:21.059 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:21.059 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:47:21.059 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:47:21.059 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 242892 2024-09-14T04:47:21.059 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.059 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:21.059 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.059 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:21.059 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.523049+0000", 2024-09-14T04:47:21.059 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:21.059 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.060 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.060 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:21.060 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.523049+0000", 2024-09-14T04:47:21.060 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:21.060 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.060 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.060 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:21.060 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.523048+0000", 2024-09-14T04:47:21.060 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:47:21.060 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.060 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.060 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:21.060 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.523053+0000", 2024-09-14T04:47:21.060 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7249999999999997e-06 2024-09-14T04:47:21.061 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.061 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.061 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:21.061 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.523062+0000", 2024-09-14T04:47:21.061 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.9910000000000008e-06 2024-09-14T04:47:21.061 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.061 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.061 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:21.061 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.523067+0000", 2024-09-14T04:47:21.061 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.9470000000000002e-06 2024-09-14T04:47:21.061 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.061 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.061 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:21.062 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.523107+0000", 2024-09-14T04:47:21.062 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0212000000000001e-05 2024-09-14T04:47:21.062 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.062 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.062 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:47:21.062 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.523115+0000", 2024-09-14T04:47:21.062 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.4770000000000003e-06 2024-09-14T04:47:21.062 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.062 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.062 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:21.062 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.481925+0000", 2024-09-14T04:47:21.062 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.95880975999999996 2024-09-14T04:47:21.062 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.062 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.063 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:21.063 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.482039+0000", 2024-09-14T04:47:21.063 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011445799999999999 2024-09-14T04:47:21.063 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.063 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.063 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:47:21.063 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.482171+0000", 2024-09-14T04:47:21.063 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000131581 2024-09-14T04:47:21.063 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.063 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.063 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:21.063 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.488724+0000", 2024-09-14T04:47:21.063 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0065528540000000003 2024-09-14T04:47:21.064 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.064 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.064 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:21.064 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.490817+0000", 2024-09-14T04:47:21.064 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.002093471 2024-09-14T04:47:21.064 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.064 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.064 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:21.064 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.494306+0000", 2024-09-14T04:47:21.064 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0034888100000000002 2024-09-14T04:47:21.064 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.064 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.064 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:21.064 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.494336+0000", 2024-09-14T04:47:21.065 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.0077999999999999e-05 2024-09-14T04:47:21.065 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.065 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.065 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:21.065 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.494369+0000", 2024-09-14T04:47:21.065 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3059000000000003e-05 2024-09-14T04:47:21.065 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:21.065 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:21.065 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:21.065 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.065 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.065 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:242910 53.f 53:fbabfba5:::benchmark_data_smithi039_224949_object112817:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1363)", 2024-09-14T04:47:21.065 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:20.551432+0000", 2024-09-14T04:47:21.066 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 300.39983953699999, 2024-09-14T04:47:21.066 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.94703268799999996, 2024-09-14T04:47:21.066 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:21.066 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:21.066 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:21.066 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:47:21.066 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:47:21.066 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 242910 2024-09-14T04:47:21.066 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.066 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:21.066 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.066 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:21.066 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.551432+0000", 2024-09-14T04:47:21.066 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:21.067 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.067 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.067 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:21.067 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.551432+0000", 2024-09-14T04:47:21.067 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:21.067 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.067 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.067 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:21.067 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.551431+0000", 2024-09-14T04:47:21.067 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:47:21.067 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.067 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.067 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:21.068 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.551436+0000", 2024-09-14T04:47:21.068 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.3070000000000002e-06 2024-09-14T04:47:21.068 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.068 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.068 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:21.068 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.551449+0000", 2024-09-14T04:47:21.068 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2353e-05 2024-09-14T04:47:21.068 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.068 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.068 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:21.068 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.551472+0000", 2024-09-14T04:47:21.068 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.3465e-05 2024-09-14T04:47:21.068 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.069 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.069 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:21.069 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.551518+0000", 2024-09-14T04:47:21.069 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.5584999999999998e-05 2024-09-14T04:47:21.069 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.069 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.069 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:47:21.069 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.551526+0000", 2024-09-14T04:47:21.069 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.9629999999999998e-06 2024-09-14T04:47:21.069 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.069 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.069 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:21.069 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.484460+0000", 2024-09-14T04:47:21.069 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.93293427200000001 2024-09-14T04:47:21.070 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.070 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.070 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:21.070 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.484533+0000", 2024-09-14T04:47:21.070 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.3404000000000004e-05 2024-09-14T04:47:21.070 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.070 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.070 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:47:21.070 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.484604+0000", 2024-09-14T04:47:21.070 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.0519999999999996e-05 2024-09-14T04:47:21.070 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.070 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.070 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:21.071 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.490824+0000", 2024-09-14T04:47:21.071 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.006220387 2024-09-14T04:47:21.071 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.071 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.071 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:21.071 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.490875+0000", 2024-09-14T04:47:21.071 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.0297999999999999e-05 2024-09-14T04:47:21.071 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.071 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.071 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:21.071 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.498408+0000", 2024-09-14T04:47:21.071 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0075339680000000003 2024-09-14T04:47:21.071 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.072 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.072 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:21.072 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.498437+0000", 2024-09-14T04:47:21.072 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.8237000000000001e-05 2024-09-14T04:47:21.072 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.072 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.072 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:21.072 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.498465+0000", 2024-09-14T04:47:21.072 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.8178999999999999e-05 2024-09-14T04:47:21.072 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:21.072 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:21.072 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:21.072 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.072 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.073 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:242947 53.f 53:f6f070f3:::benchmark_data_smithi039_224949_object112854:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1363)", 2024-09-14T04:47:21.073 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:20.598238+0000", 2024-09-14T04:47:21.073 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 300.353033616, 2024-09-14T04:47:21.073 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.90330255500000001, 2024-09-14T04:47:21.073 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:21.073 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:21.073 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:21.073 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:47:21.073 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:47:21.073 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 242947 2024-09-14T04:47:21.073 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.073 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:21.073 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.074 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:21.074 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.598238+0000", 2024-09-14T04:47:21.074 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:21.074 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.074 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.074 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:21.074 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.598238+0000", 2024-09-14T04:47:21.074 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:21.074 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.074 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.074 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:21.074 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.598237+0000", 2024-09-14T04:47:21.074 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:47:21.074 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.075 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.075 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:21.075 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.598242+0000", 2024-09-14T04:47:21.075 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.305e-06 2024-09-14T04:47:21.075 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.075 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.075 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:21.075 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.598253+0000", 2024-09-14T04:47:21.075 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0387999999999999e-05 2024-09-14T04:47:21.075 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.075 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.075 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:21.075 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.598259+0000", 2024-09-14T04:47:21.075 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.6030000000000001e-06 2024-09-14T04:47:21.075 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.076 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.076 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:21.076 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.598301+0000", 2024-09-14T04:47:21.076 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.2113999999999997e-05 2024-09-14T04:47:21.076 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.076 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.076 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:47:21.076 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:20.598307+0000", 2024-09-14T04:47:21.076 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.6529999999999999e-06 2024-09-14T04:47:21.076 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.076 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.076 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:21.076 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.486924+0000", 2024-09-14T04:47:21.076 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.88861657900000002 2024-09-14T04:47:21.077 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.077 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.077 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:21.077 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.486991+0000", 2024-09-14T04:47:21.077 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7797000000000002e-05 2024-09-14T04:47:21.077 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.077 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.077 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:47:21.077 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.487092+0000", 2024-09-14T04:47:21.077 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000100208 2024-09-14T04:47:21.077 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.077 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.077 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:21.077 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.492524+0000", 2024-09-14T04:47:21.078 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0054328650000000003 2024-09-14T04:47:21.078 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.078 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.078 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:21.078 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.498501+0000", 2024-09-14T04:47:21.078 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0059763510000000004 2024-09-14T04:47:21.078 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.078 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.078 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:21.078 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.501487+0000", 2024-09-14T04:47:21.078 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0029864959999999999 2024-09-14T04:47:21.078 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.078 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.078 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:21.079 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.501516+0000", 2024-09-14T04:47:21.079 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.8839999999999998e-05 2024-09-14T04:47:21.079 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.079 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.079 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:21.079 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:21.501541+0000", 2024-09-14T04:47:21.079 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.457e-05 2024-09-14T04:47:21.079 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:21.079 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:21.079 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:21.079 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.079 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.079 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:135622 54.0 54:07c04d6e:::benchmark_data_smithi039_264334_object9225:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:47:21.079 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:44:56.380507+0000", 2024-09-14T04:47:21.080 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 144.570764833, 2024-09-14T04:47:21.080 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.0723913839999999, 2024-09-14T04:47:21.080 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:21.080 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:21.080 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:21.080 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:47:21.080 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:47:21.080 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 135622 2024-09-14T04:47:21.080 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.080 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:21.080 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.080 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:21.080 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.380507+0000", 2024-09-14T04:47:21.080 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:21.080 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.081 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.081 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:21.081 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.380507+0000", 2024-09-14T04:47:21.081 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:21.081 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.081 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.081 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:21.081 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.380505+0000", 2024-09-14T04:47:21.081 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:47:21.081 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.081 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.081 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:21.081 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.380516+0000", 2024-09-14T04:47:21.082 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0756e-05 2024-09-14T04:47:21.082 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.082 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.082 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:21.082 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.380531+0000", 2024-09-14T04:47:21.082 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5228000000000001e-05 2024-09-14T04:47:21.082 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.082 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.082 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:21.082 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.380539+0000", 2024-09-14T04:47:21.082 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.9410000000000007e-06 2024-09-14T04:47:21.082 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.082 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.082 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:21.083 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.380598+0000", 2024-09-14T04:47:21.083 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.8492000000000002e-05 2024-09-14T04:47:21.083 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.083 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.083 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:47:21.083 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.380610+0000", 2024-09-14T04:47:21.083 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2656999999999999e-05 2024-09-14T04:47:21.083 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.083 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.083 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:21.083 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.313832+0000", 2024-09-14T04:47:21.083 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.93322119800000003 2024-09-14T04:47:21.083 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.083 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.084 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:47:21.084 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.313844+0000", 2024-09-14T04:47:21.084 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2429000000000001e-05 2024-09-14T04:47:21.084 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.084 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.084 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:21.084 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.448844+0000", 2024-09-14T04:47:21.084 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1350004300000001 2024-09-14T04:47:21.084 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.084 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.084 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:21.084 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.448996+0000", 2024-09-14T04:47:21.084 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00015165799999999999 2024-09-14T04:47:21.084 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.085 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.085 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,4,5", 2024-09-14T04:47:21.085 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.449141+0000", 2024-09-14T04:47:21.085 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00014442 2024-09-14T04:47:21.085 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.085 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.085 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:21.085 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.452710+0000", 2024-09-14T04:47:21.085 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00356975 2024-09-14T04:47:21.085 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.085 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.085 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:21.085 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.452770+0000", 2024-09-14T04:47:21.086 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9957999999999998e-05 2024-09-14T04:47:21.086 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.086 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.086 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:21.086 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.452781+0000", 2024-09-14T04:47:21.086 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0878e-05 2024-09-14T04:47:21.086 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.086 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.086 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:21.086 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.452803+0000", 2024-09-14T04:47:21.086 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.1488000000000002e-05 2024-09-14T04:47:21.086 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.086 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.086 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:21.087 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.452837+0000", 2024-09-14T04:47:21.087 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.434e-05 2024-09-14T04:47:21.087 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.087 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.087 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:21.087 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.452898+0000", 2024-09-14T04:47:21.087 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.1377000000000005e-05 2024-09-14T04:47:21.087 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:21.087 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:21.087 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:21.087 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.087 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.087 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:135641 54.0 54:06c56606:::benchmark_data_smithi039_264334_object9244:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:47:21.087 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:44:56.444135+0000", 2024-09-14T04:47:21.088 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 144.507136605, 2024-09-14T04:47:21.088 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.0238239230000001, 2024-09-14T04:47:21.088 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:21.088 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:21.088 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:21.088 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:47:21.088 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:47:21.088 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 135641 2024-09-14T04:47:21.088 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.088 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:21.088 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.088 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:21.088 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.444135+0000", 2024-09-14T04:47:21.088 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:21.089 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.089 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.089 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:21.089 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.444135+0000", 2024-09-14T04:47:21.089 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:21.089 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.089 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.089 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:21.089 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.444134+0000", 2024-09-14T04:47:21.089 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:47:21.089 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.089 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.089 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:21.089 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.444140+0000", 2024-09-14T04:47:21.090 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.5490000000000003e-06 2024-09-14T04:47:21.090 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.090 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.090 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:21.090 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.444152+0000", 2024-09-14T04:47:21.090 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1613999999999999e-05 2024-09-14T04:47:21.090 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.090 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.090 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:21.090 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.444159+0000", 2024-09-14T04:47:21.090 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.5279999999999998e-06 2024-09-14T04:47:21.090 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.090 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.090 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:21.091 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.444216+0000", 2024-09-14T04:47:21.091 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.6968000000000001e-05 2024-09-14T04:47:21.091 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.091 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.091 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:47:21.091 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.444228+0000", 2024-09-14T04:47:21.091 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1647e-05 2024-09-14T04:47:21.091 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.091 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.091 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:21.091 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.313867+0000", 2024-09-14T04:47:21.091 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.86963858100000002 2024-09-14T04:47:21.091 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.092 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.092 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:47:21.092 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.313875+0000", 2024-09-14T04:47:21.092 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.9689999999999994e-06 2024-09-14T04:47:21.092 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.092 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.092 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:21.092 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.453104+0000", 2024-09-14T04:47:21.092 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1392290009999999 2024-09-14T04:47:21.092 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.092 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.092 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:21.092 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.453212+0000", 2024-09-14T04:47:21.092 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000108163 2024-09-14T04:47:21.093 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.093 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.093 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,4,5", 2024-09-14T04:47:21.093 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.453314+0000", 2024-09-14T04:47:21.093 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000102532 2024-09-14T04:47:21.093 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.093 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.093 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:21.093 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.459566+0000", 2024-09-14T04:47:21.093 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0062515440000000004 2024-09-14T04:47:21.093 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.093 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.093 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:21.094 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.467414+0000", 2024-09-14T04:47:21.094 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0078484320000000007 2024-09-14T04:47:21.094 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.094 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.094 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:21.094 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.467424+0000", 2024-09-14T04:47:21.094 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.3840000000000003e-06 2024-09-14T04:47:21.094 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.094 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.094 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:21.094 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.467763+0000", 2024-09-14T04:47:21.094 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00033969600000000001 2024-09-14T04:47:21.094 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.095 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.095 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:21.095 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.467827+0000", 2024-09-14T04:47:21.095 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.3794999999999995e-05 2024-09-14T04:47:21.095 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.095 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.095 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:21.095 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.467959+0000", 2024-09-14T04:47:21.095 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000131898 2024-09-14T04:47:21.095 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:21.095 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:21.095 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:21.095 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.095 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.096 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:135658 54.0 54:05f5e225:::benchmark_data_smithi039_264334_object9261:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:47:21.096 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:44:56.461210+0000", 2024-09-14T04:47:21.096 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 144.49006212899999, 2024-09-14T04:47:21.096 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.0069541119999998, 2024-09-14T04:47:21.096 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:21.096 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:21.096 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:21.096 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:47:21.096 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:47:21.096 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 135658 2024-09-14T04:47:21.096 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.096 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:21.096 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.097 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:21.097 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.461210+0000", 2024-09-14T04:47:21.097 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:21.097 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.097 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.097 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:21.097 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.461210+0000", 2024-09-14T04:47:21.097 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:21.097 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.097 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.097 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:21.097 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.461208+0000", 2024-09-14T04:47:21.097 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:47:21.098 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.098 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.098 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:21.098 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.461215+0000", 2024-09-14T04:47:21.098 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7059999999999998e-06 2024-09-14T04:47:21.098 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.098 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.098 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:21.098 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.461228+0000", 2024-09-14T04:47:21.098 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3264e-05 2024-09-14T04:47:21.098 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.098 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.098 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:21.098 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.461235+0000", 2024-09-14T04:47:21.099 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.319e-06 2024-09-14T04:47:21.099 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.099 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.099 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:21.099 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.461282+0000", 2024-09-14T04:47:21.099 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7506000000000002e-05 2024-09-14T04:47:21.099 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.099 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.099 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:47:21.099 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.461291+0000", 2024-09-14T04:47:21.099 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.5930000000000006e-06 2024-09-14T04:47:21.099 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.099 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.099 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:21.100 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.313897+0000", 2024-09-14T04:47:21.100 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.852606007 2024-09-14T04:47:21.100 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.100 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.100 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:47:21.100 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.313905+0000", 2024-09-14T04:47:21.100 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.3410000000000003e-06 2024-09-14T04:47:21.100 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.100 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.100 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:21.100 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.455954+0000", 2024-09-14T04:47:21.100 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1420486839999999 2024-09-14T04:47:21.100 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.100 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.101 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:21.101 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.456064+0000", 2024-09-14T04:47:21.101 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000110536 2024-09-14T04:47:21.101 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.101 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.101 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,4,5", 2024-09-14T04:47:21.101 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.456182+0000", 2024-09-14T04:47:21.101 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011799900000000001 2024-09-14T04:47:21.101 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.101 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.101 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:21.101 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.467430+0000", 2024-09-14T04:47:21.101 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.011247577999999999 2024-09-14T04:47:21.101 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.102 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.102 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:21.102 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.467463+0000", 2024-09-14T04:47:21.102 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3269000000000003e-05 2024-09-14T04:47:21.102 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.102 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.102 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:21.102 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.467500+0000", 2024-09-14T04:47:21.102 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6508999999999999e-05 2024-09-14T04:47:21.102 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.102 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.102 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:21.102 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.467994+0000", 2024-09-14T04:47:21.102 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00049403999999999995 2024-09-14T04:47:21.102 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.103 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.103 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:21.103 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.468070+0000", 2024-09-14T04:47:21.103 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.6661000000000003e-05 2024-09-14T04:47:21.103 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:21.103 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:21.103 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:21.103 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.468164+0000", 2024-09-14T04:47:21.103 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.3315999999999999e-05 2024-09-14T04:47:21.103 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:21.103 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:21.103 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:21.103 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:21.103 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:21.104 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:47:21.150 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:21.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:21.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:21.252 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:21.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:21.293 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:21.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:21.394 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:21.438 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:21.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:21.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:21.540 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:21.581 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:21.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:21.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:21.683 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:21.691 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:47:21.703 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:21 smithi039 ceph-mon[106470]: pgmap v2517: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 279 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 5216/13440 objects misplaced (38.810%); 1.8 MiB/s, 29 objects/s recovering 2024-09-14T04:47:21.703 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:21 smithi039 ceph-mon[111235]: pgmap v2517: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 279 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 5216/13440 objects misplaced (38.810%); 1.8 MiB/s, 29 objects/s recovering 2024-09-14T04:47:21.716 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:21.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:21.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:21.818 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:21.847 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:21.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:21.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:21.949 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:21.986 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:21.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:21.987 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:21 smithi129 ceph-mon[101316]: pgmap v2517: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 279 MiB data, 3.6 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 5216/13440 objects misplaced (38.810%); 1.8 MiB/s, 29 objects/s recovering 2024-09-14T04:47:22.087 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:22.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:22.124 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:22.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:22.146 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-14T04:47:22.225 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:22.267 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:22.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:22.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:22.369 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:22.407 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:22.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:22.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:22.496 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:47:22.509 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:22.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:22.552 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:22.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:22.653 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:22.663 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:47:22.664 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:22 smithi039 ceph-mon[106470]: pgmap v2518: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 231 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 5077/11127 objects misplaced (45.628%); 1.7 MiB/s, 27 objects/s recovering 2024-09-14T04:47:22.664 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:47:22.664 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:22 smithi039 ceph-mon[111235]: pgmap v2518: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 231 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 5077/11127 objects misplaced (45.628%); 1.7 MiB/s, 27 objects/s recovering 2024-09-14T04:47:22.665 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:47:22.669 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T04:47:22.682 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:22.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:22.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:22.784 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:22.822 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:22.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:22.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:22.924 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:22.960 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:22.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:22.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:22.973 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:22 smithi129 ceph-mon[101316]: pgmap v2518: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 231 MiB data, 3.5 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 5077/11127 objects misplaced (45.628%); 1.7 MiB/s, 27 objects/s recovering 2024-09-14T04:47:22.973 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:47:23.062 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:23.100 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:23.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:23.101 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:23.162 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Removed 126394 objects 2024-09-14T04:47:23.162 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Clean up completed and total clean up time :155.607 2024-09-14T04:47:23.202 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:23.226 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:23.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:23.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:23.328 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:23.349 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:23.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:23.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:23.445 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:47:23.450 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:23.471 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:23.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:23.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:23.572 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:23.593 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:23.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:23.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:23.695 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:23.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:23.717 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:23.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:23.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:23 smithi039 ceph-mon[106470]: from='client.35646 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:47:23.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:23 smithi039 ceph-mon[111235]: from='client.35646 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:47:23.818 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:23.841 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:23.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:23.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:23.937 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:23 smithi129 ceph-mon[101316]: from='client.35646 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:47:23.942 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:23.965 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:23.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:23.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:24.066 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:24.088 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:24.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:24.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:24.189 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:24.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:24.212 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:24.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:24.313 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:24.335 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:24.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:24.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:24.436 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:24.459 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:24.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:24.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:24.507 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:47:24.507 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [], 2024-09-14T04:47:24.507 INFO:teuthology.orchestra.run.smithi129.stdout: "num_ops": 0 2024-09-14T04:47:24.507 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:47:24.561 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:24.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:24.583 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:24.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:24.683 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:24.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:24.708 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:24.708 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:24.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:24 smithi129 ceph-mon[101316]: pgmap v2519: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 32 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 897 op/s; 4377/1569 objects misplaced (278.967%); 1.6 MiB/s, 24 objects/s recovering 2024-09-14T04:47:24.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:24 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:47:24.808 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:24.829 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:24.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:24.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:24.933 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:24.956 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:24.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:24.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:25.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:24 smithi039 ceph-mon[106470]: pgmap v2519: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 32 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 897 op/s; 4377/1569 objects misplaced (278.967%); 1.6 MiB/s, 24 objects/s recovering 2024-09-14T04:47:25.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:24 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:47:25.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:24 smithi039 ceph-mon[111235]: pgmap v2519: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 32 MiB data, 2.9 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 897 op/s; 4377/1569 objects misplaced (278.967%); 1.6 MiB/s, 24 objects/s recovering 2024-09-14T04:47:25.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:24 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:47:25.058 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:25.080 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:25.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:25.081 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:25.166 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-14T04:47:25.182 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:25.231 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:25.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:25.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:25.333 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:25.356 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:25.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:25.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:25.393 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:47:25.458 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:25.479 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:25.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:25.480 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:25.581 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:25.602 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:25.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:25.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:25.612 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":2518,"stamp":"2024-09-14T04:47:21.701200+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":241899517,"num_objects":3709,"num_object_clones":0,"num_object_copies":11127,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5077,"num_objects_unfound":0,"num_objects_dirty":3709,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":249171,"num_write_kb":8089250,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3038,"num_bytes_recovered":197137902,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123922,"ondisk_log_size":123922,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":301,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3676300,"kb_used_data":909516,"kb_used_omap":404,"kb_used_meta":2765675,"kb_avail":746284916,"statfs":{"total":767960285184,"available":764195753984,"internally_reserved":0,"allocated":931344384,"data_stored":895398803,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":414022,"internal_metadata":2832051898},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-513671168,"num_objects":-7838,"num_object_clones":0,"num_object_copies":-23514,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-1706,"num_objects_unfound":0,"num_objects_dirty":-7838,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7838,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":279,"num_bytes_recovered":18284544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003553"},"pg_stats":[{"pgid":"54.f","version":"1416'15357","reported_seq":15441,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:17.451570+0000","last_change":"2024-09-14T04:43:01.044742+0000","last_active":"2024-09-14T04:47:17.451570+0000","last_peered":"2024-09-14T04:47:17.451570+0000","last_clean":"2024-09-14T04:47:17.451570+0000","last_became_active":"2024-09-14T04:43:01.044421+0000","last_became_peered":"2024-09-14T04:43:01.044421+0000","last_unstale":"2024-09-14T04:47:17.451570+0000","last_undegraded":"2024-09-14T04:47:17.451570+0000","last_fullsized":"2024-09-14T04:47:17.451570+0000","mapping_epoch":1380,"log_start":"1406'8600","ondisk_log_start":"1406'8600","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":6757,"log_dups_size":0,"ondisk_log_size":6757,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:28.897426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17760256,"num_objects":271,"num_object_clones":0,"num_object_copies":813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":271,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15357,"num_write_kb":500096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"54.e","version":"1416'15670","reported_seq":21703,"reported_epoch":1416,"state":"active+recovering+remapped","last_fresh":"2024-09-14T04:47:20.922887+0000","last_change":"2024-09-14T04:44:58.690021+0000","last_active":"2024-09-14T04:47:20.922887+0000","last_peered":"2024-09-14T04:47:20.922887+0000","last_clean":"2024-09-14T04:44:56.302285+0000","last_became_active":"2024-09-14T04:44:58.579754+0000","last_became_peered":"2024-09-14T04:44:58.579754+0000","last_unstale":"2024-09-14T04:47:20.922887+0000","last_undegraded":"2024-09-14T04:47:20.922887+0000","last_fullsized":"2024-09-14T04:47:20.922887+0000","mapping_epoch":1403,"log_start":"1400'7683","ondisk_log_start":"1400'7683","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7987,"log_dups_size":0,"ondisk_log_size":7987,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8650752,"num_objects":132,"num_object_clones":0,"num_object_copies":396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4353,"num_objects_unfound":0,"num_objects_dirty":132,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15670,"num_write_kb":505664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2973,"num_bytes_recovered":194772992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[1,2,7],"avail_no_missing":["1","2","7"],"object_location_counts":[{"shards":"1,2,7","objects":-4221}],"blocked_by":[],"up_primary":4,"acting_primary":1,"purged_snaps":[]},{"pgid":"54.d","version":"1416'15637","reported_seq":15684,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:18.327897+0000","last_change":"2024-09-14T04:43:01.050000+0000","last_active":"2024-09-14T04:47:18.327897+0000","last_peered":"2024-09-14T04:47:18.327897+0000","last_clean":"2024-09-14T04:47:18.327897+0000","last_became_active":"2024-09-14T04:43:01.049752+0000","last_became_peered":"2024-09-14T04:43:01.049752+0000","last_unstale":"2024-09-14T04:47:18.327897+0000","last_undegraded":"2024-09-14T04:47:18.327897+0000","last_fullsized":"2024-09-14T04:47:18.327897+0000","mapping_epoch":1380,"log_start":"1401'8100","ondisk_log_start":"1401'8100","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7537,"log_dups_size":0,"ondisk_log_size":7537,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:35:06.578523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16121880,"num_objects":247,"num_object_clones":0,"num_object_copies":741,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":247,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":15637,"num_write_kb":508225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"54.c","version":"1416'15526","reported_seq":15614,"reported_epoch":1416,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:47:17.451294+0000","last_change":"2024-09-14T04:44:58.455281+0000","last_active":"2024-09-14T04:47:17.451294+0000","last_peered":"2024-09-14T04:47:17.451294+0000","last_clean":"2024-09-14T04:44:56.285515+0000","last_became_active":"2024-09-14T04:44:58.445654+0000","last_became_peered":"2024-09-14T04:44:58.445654+0000","last_unstale":"2024-09-14T04:47:17.451294+0000","last_undegraded":"2024-09-14T04:47:17.451294+0000","last_fullsized":"2024-09-14T04:47:17.451294+0000","mapping_epoch":1403,"log_start":"1407'8800","ondisk_log_start":"1407'8800","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":6726,"log_dups_size":0,"ondisk_log_size":6726,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15859712,"num_objects":242,"num_object_clones":0,"num_object_copies":726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":242,"num_objects_unfound":0,"num_objects_dirty":242,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15526,"num_write_kb":504576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,4],"acting":[3,5,6],"avail_no_missing":["3","5","6"],"object_location_counts":[{"shards":"3,5,6","objects":242}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"54.b","version":"1416'15487","reported_seq":15560,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:18.236751+0000","last_change":"2024-09-14T04:43:13.355592+0000","last_active":"2024-09-14T04:47:18.236751+0000","last_peered":"2024-09-14T04:47:18.236751+0000","last_clean":"2024-09-14T04:47:18.236751+0000","last_became_active":"2024-09-14T04:43:13.355130+0000","last_became_peered":"2024-09-14T04:43:13.355130+0000","last_unstale":"2024-09-14T04:47:18.236751+0000","last_undegraded":"2024-09-14T04:47:18.236751+0000","last_fullsized":"2024-09-14T04:47:18.236751+0000","mapping_epoch":1386,"log_start":"1399'7100","ondisk_log_start":"1399'7100","created":1380,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":8387,"log_dups_size":0,"ondisk_log_size":8387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:42:44.062290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14352384,"num_objects":219,"num_object_clones":0,"num_object_copies":657,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":219,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15487,"num_write_kb":502592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"54.a","version":"1416'15423","reported_seq":15469,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:18.326880+0000","last_change":"2024-09-14T04:43:01.043585+0000","last_active":"2024-09-14T04:47:18.326880+0000","last_peered":"2024-09-14T04:47:18.326880+0000","last_clean":"2024-09-14T04:47:18.326880+0000","last_became_active":"2024-09-14T04:43:01.043295+0000","last_became_peered":"2024-09-14T04:43:01.043295+0000","last_unstale":"2024-09-14T04:47:18.326880+0000","last_undegraded":"2024-09-14T04:47:18.326880+0000","last_fullsized":"2024-09-14T04:47:18.326880+0000","mapping_epoch":1380,"log_start":"1401'7900","ondisk_log_start":"1401'7900","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7523,"log_dups_size":0,"ondisk_log_size":7523,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:58:35.934308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":14352384,"num_objects":219,"num_object_clones":0,"num_object_copies":657,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":219,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15423,"num_write_kb":500544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"54.9","version":"1416'15703","reported_seq":15787,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:20.905057+0000","last_change":"2024-09-14T04:43:01.048728+0000","last_active":"2024-09-14T04:47:20.905057+0000","last_peered":"2024-09-14T04:47:20.905057+0000","last_clean":"2024-09-14T04:47:20.905057+0000","last_became_active":"2024-09-14T04:43:01.048381+0000","last_became_peered":"2024-09-14T04:43:01.048381+0000","last_unstale":"2024-09-14T04:47:20.905057+0000","last_undegraded":"2024-09-14T04:47:20.905057+0000","last_fullsized":"2024-09-14T04:47:20.905057+0000","mapping_epoch":1380,"log_start":"1400'7800","ondisk_log_start":"1400'7800","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7903,"log_dups_size":0,"ondisk_log_size":7903,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:47:27.860166+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7929856,"num_objects":121,"num_object_clones":0,"num_object_copies":363,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":121,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15703,"num_write_kb":506368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"54.8","version":"1416'15614","reported_seq":15701,"reported_epoch":1416,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:47:18.270385+0000","last_change":"2024-09-14T04:44:58.458770+0000","last_active":"2024-09-14T04:47:18.270385+0000","last_peered":"2024-09-14T04:47:18.270385+0000","last_clean":"2024-09-14T04:44:56.294675+0000","last_became_active":"2024-09-14T04:44:58.445646+0000","last_became_peered":"2024-09-14T04:44:58.445646+0000","last_unstale":"2024-09-14T04:47:18.270385+0000","last_undegraded":"2024-09-14T04:47:18.270385+0000","last_fullsized":"2024-09-14T04:47:18.270385+0000","mapping_epoch":1403,"log_start":"1399'7200","ondisk_log_start":"1399'7200","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":8414,"log_dups_size":0,"ondisk_log_size":8414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15335424,"num_objects":234,"num_object_clones":0,"num_object_copies":702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":234,"num_objects_unfound":0,"num_objects_dirty":234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15614,"num_write_kb":507136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[0,3,7],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3,7","objects":234}],"blocked_by":[],"up_primary":4,"acting_primary":0,"purged_snaps":[]},{"pgid":"54.7","version":"1416'15503","reported_seq":15587,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:17.444966+0000","last_change":"2024-09-14T04:43:01.050691+0000","last_active":"2024-09-14T04:47:17.444966+0000","last_peered":"2024-09-14T04:47:17.444966+0000","last_clean":"2024-09-14T04:47:17.444966+0000","last_became_active":"2024-09-14T04:43:01.050334+0000","last_became_peered":"2024-09-14T04:43:01.050334+0000","last_unstale":"2024-09-14T04:47:17.444966+0000","last_undegraded":"2024-09-14T04:47:17.444966+0000","last_fullsized":"2024-09-14T04:47:17.444966+0000","mapping_epoch":1380,"log_start":"1406'8800","ondisk_log_start":"1406'8800","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":6703,"log_dups_size":0,"ondisk_log_size":6703,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:22:30.806651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20381696,"num_objects":311,"num_object_clones":0,"num_object_copies":933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15503,"num_write_kb":506048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,7],"acting":[3,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"54.6","version":"1416'15575","reported_seq":15621,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:17.994238+0000","last_change":"2024-09-14T04:43:01.043477+0000","last_active":"2024-09-14T04:47:17.994238+0000","last_peered":"2024-09-14T04:47:17.994238+0000","last_clean":"2024-09-14T04:47:17.994238+0000","last_became_active":"2024-09-14T04:43:01.043136+0000","last_became_peered":"2024-09-14T04:43:01.043136+0000","last_unstale":"2024-09-14T04:47:17.994238+0000","last_undegraded":"2024-09-14T04:47:17.994238+0000","last_fullsized":"2024-09-14T04:47:17.994238+0000","mapping_epoch":1380,"log_start":"1401'8200","ondisk_log_start":"1401'8200","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7375,"log_dups_size":0,"ondisk_log_size":7375,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:00:57.211271+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18808832,"num_objects":287,"num_object_clones":0,"num_object_copies":861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":287,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15575,"num_write_kb":507584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"54.5","version":"1416'15573","reported_seq":15620,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:18.327792+0000","last_change":"2024-09-14T04:43:01.048975+0000","last_active":"2024-09-14T04:47:18.327792+0000","last_peered":"2024-09-14T04:47:18.327792+0000","last_clean":"2024-09-14T04:47:18.327792+0000","last_became_active":"2024-09-14T04:43:01.048677+0000","last_became_peered":"2024-09-14T04:43:01.048677+0000","last_unstale":"2024-09-14T04:47:18.327792+0000","last_undegraded":"2024-09-14T04:47:18.327792+0000","last_fullsized":"2024-09-14T04:47:18.327792+0000","mapping_epoch":1380,"log_start":"1401'8000","ondisk_log_start":"1401'8000","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7573,"log_dups_size":0,"ondisk_log_size":7573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:58:04.849606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16580608,"num_objects":253,"num_object_clones":0,"num_object_copies":759,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":253,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15573,"num_write_kb":506432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"54.4","version":"1416'15565","reported_seq":15650,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:17.500114+0000","last_change":"2024-09-14T04:43:01.043603+0000","last_active":"2024-09-14T04:47:17.500114+0000","last_peered":"2024-09-14T04:47:17.500114+0000","last_clean":"2024-09-14T04:47:17.500114+0000","last_became_active":"2024-09-14T04:43:01.042757+0000","last_became_peered":"2024-09-14T04:43:01.042757+0000","last_unstale":"2024-09-14T04:47:17.500114+0000","last_undegraded":"2024-09-14T04:47:17.500114+0000","last_fullsized":"2024-09-14T04:47:17.500114+0000","mapping_epoch":1380,"log_start":"1406'8800","ondisk_log_start":"1406'8800","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":6765,"log_dups_size":0,"ondisk_log_size":6765,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:46:20.719227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18808832,"num_objects":287,"num_object_clones":0,"num_object_copies":861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":287,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15565,"num_write_kb":507264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"54.3","version":"1416'15948","reported_seq":16033,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:20.865322+0000","last_change":"2024-09-14T04:43:01.048551+0000","last_active":"2024-09-14T04:47:20.865322+0000","last_peered":"2024-09-14T04:47:20.865322+0000","last_clean":"2024-09-14T04:47:20.865322+0000","last_became_active":"2024-09-14T04:43:01.048218+0000","last_became_peered":"2024-09-14T04:43:01.048218+0000","last_unstale":"2024-09-14T04:47:20.865322+0000","last_undegraded":"2024-09-14T04:47:20.865322+0000","last_fullsized":"2024-09-14T04:47:20.865322+0000","mapping_epoch":1380,"log_start":"1400'8000","ondisk_log_start":"1400'8000","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7948,"log_dups_size":0,"ondisk_log_size":7948,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:27:10.233265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":7471104,"num_objects":114,"num_object_clones":0,"num_object_copies":342,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":114,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15948,"num_write_kb":513984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"54.2","version":"1416'15522","reported_seq":15606,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:18.288792+0000","last_change":"2024-09-14T04:43:01.049088+0000","last_active":"2024-09-14T04:47:18.288792+0000","last_peered":"2024-09-14T04:47:18.288792+0000","last_clean":"2024-09-14T04:47:18.288792+0000","last_became_active":"2024-09-14T04:43:01.048764+0000","last_became_peered":"2024-09-14T04:43:01.048764+0000","last_unstale":"2024-09-14T04:47:18.288792+0000","last_undegraded":"2024-09-14T04:47:18.288792+0000","last_fullsized":"2024-09-14T04:47:18.288792+0000","mapping_epoch":1380,"log_start":"1399'7100","ondisk_log_start":"1399'7100","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":8422,"log_dups_size":0,"ondisk_log_size":8422,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:03:50.532719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15466496,"num_objects":236,"num_object_clones":0,"num_object_copies":708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15522,"num_write_kb":504256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"54.1","version":"1416'15621","reported_seq":15666,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:18.315914+0000","last_change":"2024-09-14T04:43:01.047658+0000","last_active":"2024-09-14T04:47:18.315914+0000","last_peered":"2024-09-14T04:47:18.315914+0000","last_clean":"2024-09-14T04:47:18.315914+0000","last_became_active":"2024-09-14T04:43:01.047401+0000","last_became_peered":"2024-09-14T04:43:01.047401+0000","last_unstale":"2024-09-14T04:47:18.315914+0000","last_undegraded":"2024-09-14T04:47:18.315914+0000","last_fullsized":"2024-09-14T04:47:18.315914+0000","mapping_epoch":1380,"log_start":"1399'7000","ondisk_log_start":"1399'7000","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":8621,"log_dups_size":0,"ondisk_log_size":8621,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:02:05.987375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17104896,"num_objects":261,"num_object_clones":0,"num_object_copies":783,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":261,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15621,"num_write_kb":508224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"54.0","version":"1416'15383","reported_seq":15507,"reported_epoch":1416,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:47:18.309724+0000","last_change":"2024-09-14T04:44:58.640486+0000","last_active":"2024-09-14T04:47:18.309724+0000","last_peered":"2024-09-14T04:47:18.309724+0000","last_clean":"2024-09-14T04:44:56.291188+0000","last_became_active":"2024-09-14T04:44:58.441092+0000","last_became_peered":"2024-09-14T04:44:58.441092+0000","last_unstale":"2024-09-14T04:47:18.309724+0000","last_undegraded":"2024-09-14T04:47:18.309724+0000","last_fullsized":"2024-09-14T04:47:18.309724+0000","mapping_epoch":1403,"log_start":"1400'7800","ondisk_log_start":"1400'7800","created":1380,"last_epoch_clean":1381,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:43:00.019579+0000","last_clean_scrub_stamp":"2024-09-14T04:43:00.019579+0000","objects_scrubbed":0,"log_size":7583,"log_dups_size":0,"ondisk_log_size":7583,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:13:27.337351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16318464,"num_objects":249,"num_object_clones":0,"num_object_copies":747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":248,"num_objects_unfound":0,"num_objects_dirty":249,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":15383,"num_write_kb":500224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":2097152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,3,5],"avail_no_missing":["6","3","5"],"object_location_counts":[{"shards":"3,5,6","objects":249}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":2120,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.538284+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T04:47:12.538284+0000","last_peered":"2024-09-14T04:47:12.538284+0000","last_clean":"2024-09-14T04:47:12.538284+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T04:47:12.538284+0000","last_undegraded":"2024-09-14T04:47:12.538284+0000","last_fullsized":"2024-09-14T04:47:12.538284+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":24,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.393452+0000","last_change":"2024-09-14T04:44:57.325109+0000","last_active":"2024-09-14T04:46:50.393452+0000","last_peered":"2024-09-14T04:46:50.393452+0000","last_clean":"2024-09-14T04:46:50.393452+0000","last_became_active":"2024-09-14T04:44:57.324791+0000","last_became_peered":"2024-09-14T04:44:57.324791+0000","last_unstale":"2024-09-14T04:46:50.393452+0000","last_undegraded":"2024-09-14T04:46:50.393452+0000","last_fullsized":"2024-09-14T04:46:50.393452+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:03.593275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":2110,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.546444+0000","last_change":"2024-09-14T04:07:58.504829+0000","last_active":"2024-09-14T04:47:12.546444+0000","last_peered":"2024-09-14T04:47:12.546444+0000","last_clean":"2024-09-14T04:47:12.546444+0000","last_became_active":"2024-09-14T04:07:58.502623+0000","last_became_peered":"2024-09-14T04:07:58.502623+0000","last_unstale":"2024-09-14T04:47:12.546444+0000","last_undegraded":"2024-09-14T04:47:12.546444+0000","last_fullsized":"2024-09-14T04:47:12.546444+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:59.091993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":742,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.536132+0000","last_change":"2024-09-14T04:43:13.355281+0000","last_active":"2024-09-14T04:47:12.536132+0000","last_peered":"2024-09-14T04:47:12.536132+0000","last_clean":"2024-09-14T04:47:12.536132+0000","last_became_active":"2024-09-14T04:43:13.354967+0000","last_became_peered":"2024-09-14T04:43:13.354967+0000","last_unstale":"2024-09-14T04:47:12.536132+0000","last_undegraded":"2024-09-14T04:47:12.536132+0000","last_fullsized":"2024-09-14T04:47:12.536132+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:23:19.332075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1403,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.395919+0000","last_change":"2024-09-14T04:44:57.342918+0000","last_active":"2024-09-14T04:46:50.395919+0000","last_peered":"2024-09-14T04:46:50.395919+0000","last_clean":"2024-09-14T04:46:50.395919+0000","last_became_active":"2024-09-14T04:44:57.329550+0000","last_became_peered":"2024-09-14T04:44:57.329550+0000","last_unstale":"2024-09-14T04:46:50.395919+0000","last_undegraded":"2024-09-14T04:46:50.395919+0000","last_fullsized":"2024-09-14T04:46:50.395919+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:52:52.435893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":24,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.393254+0000","last_change":"2024-09-14T04:44:57.325537+0000","last_active":"2024-09-14T04:46:50.393254+0000","last_peered":"2024-09-14T04:46:50.393254+0000","last_clean":"2024-09-14T04:46:50.393254+0000","last_became_active":"2024-09-14T04:44:57.325057+0000","last_became_peered":"2024-09-14T04:44:57.325057+0000","last_unstale":"2024-09-14T04:46:50.393254+0000","last_undegraded":"2024-09-14T04:46:50.393254+0000","last_fullsized":"2024-09-14T04:46:50.393254+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:43:23.166708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1418,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.396455+0000","last_change":"2024-09-14T04:44:57.337033+0000","last_active":"2024-09-14T04:46:50.396455+0000","last_peered":"2024-09-14T04:46:50.396455+0000","last_clean":"2024-09-14T04:46:50.396455+0000","last_became_active":"2024-09-14T04:44:57.328437+0000","last_became_peered":"2024-09-14T04:44:57.328437+0000","last_unstale":"2024-09-14T04:46:50.396455+0000","last_undegraded":"2024-09-14T04:46:50.396455+0000","last_fullsized":"2024-09-14T04:46:50.396455+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.109086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":24,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.393621+0000","last_change":"2024-09-14T04:44:57.332243+0000","last_active":"2024-09-14T04:46:50.393621+0000","last_peered":"2024-09-14T04:46:50.393621+0000","last_clean":"2024-09-14T04:46:50.393621+0000","last_became_active":"2024-09-14T04:44:57.330413+0000","last_became_peered":"2024-09-14T04:44:57.330413+0000","last_unstale":"2024-09-14T04:46:50.393621+0000","last_undegraded":"2024-09-14T04:46:50.393621+0000","last_fullsized":"2024-09-14T04:46:50.393621+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:25:01.338549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":2127,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.539603+0000","last_change":"2024-09-14T04:44:57.333097+0000","last_active":"2024-09-14T04:47:12.539603+0000","last_peered":"2024-09-14T04:47:12.539603+0000","last_clean":"2024-09-14T04:47:12.539603+0000","last_became_active":"2024-09-14T04:44:57.332294+0000","last_became_peered":"2024-09-14T04:44:57.332294+0000","last_unstale":"2024-09-14T04:47:12.539603+0000","last_undegraded":"2024-09-14T04:47:12.539603+0000","last_fullsized":"2024-09-14T04:47:12.539603+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:57:44.426280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2116,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.535361+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T04:47:12.535361+0000","last_peered":"2024-09-14T04:47:12.535361+0000","last_clean":"2024-09-14T04:47:12.535361+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T04:47:12.535361+0000","last_undegraded":"2024-09-14T04:47:12.535361+0000","last_fullsized":"2024-09-14T04:47:12.535361+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":24,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.393182+0000","last_change":"2024-09-14T04:44:57.325392+0000","last_active":"2024-09-14T04:46:50.393182+0000","last_peered":"2024-09-14T04:46:50.393182+0000","last_clean":"2024-09-14T04:46:50.393182+0000","last_became_active":"2024-09-14T04:44:57.324624+0000","last_became_peered":"2024-09-14T04:44:57.324624+0000","last_unstale":"2024-09-14T04:46:50.393182+0000","last_undegraded":"2024-09-14T04:46:50.393182+0000","last_fullsized":"2024-09-14T04:46:50.393182+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:33:18.810199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2085,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.543790+0000","last_change":"2024-09-14T04:44:57.329113+0000","last_active":"2024-09-14T04:47:12.543790+0000","last_peered":"2024-09-14T04:47:12.543790+0000","last_clean":"2024-09-14T04:47:12.543790+0000","last_became_active":"2024-09-14T04:44:57.328727+0000","last_became_peered":"2024-09-14T04:44:57.328727+0000","last_unstale":"2024-09-14T04:47:12.543790+0000","last_undegraded":"2024-09-14T04:47:12.543790+0000","last_fullsized":"2024-09-14T04:47:12.543790+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:34.939874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":2102,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.544310+0000","last_change":"2024-09-14T04:44:57.329277+0000","last_active":"2024-09-14T04:47:12.544310+0000","last_peered":"2024-09-14T04:47:12.544310+0000","last_clean":"2024-09-14T04:47:12.544310+0000","last_became_active":"2024-09-14T04:44:57.328804+0000","last_became_peered":"2024-09-14T04:44:57.328804+0000","last_unstale":"2024-09-14T04:47:12.544310+0000","last_undegraded":"2024-09-14T04:47:12.544310+0000","last_fullsized":"2024-09-14T04:47:12.544310+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:42:46.055240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2115,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.538047+0000","last_change":"2024-09-14T04:07:58.561490+0000","last_active":"2024-09-14T04:47:12.538047+0000","last_peered":"2024-09-14T04:47:12.538047+0000","last_clean":"2024-09-14T04:47:12.538047+0000","last_became_active":"2024-09-14T04:07:58.559841+0000","last_became_peered":"2024-09-14T04:07:58.559841+0000","last_unstale":"2024-09-14T04:47:12.538047+0000","last_undegraded":"2024-09-14T04:47:12.538047+0000","last_fullsized":"2024-09-14T04:47:12.538047+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:27:03.795368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2098,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.538498+0000","last_change":"2024-09-14T04:43:13.357092+0000","last_active":"2024-09-14T04:47:12.538498+0000","last_peered":"2024-09-14T04:47:12.538498+0000","last_clean":"2024-09-14T04:47:12.538498+0000","last_became_active":"2024-09-14T04:43:13.356694+0000","last_became_peered":"2024-09-14T04:43:13.356694+0000","last_unstale":"2024-09-14T04:47:12.538498+0000","last_undegraded":"2024-09-14T04:47:12.538498+0000","last_fullsized":"2024-09-14T04:47:12.538498+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:11.167574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1396,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.531333+0000","last_change":"2024-09-14T04:07:21.720797+0000","last_active":"2024-09-14T04:47:12.531333+0000","last_peered":"2024-09-14T04:47:12.531333+0000","last_clean":"2024-09-14T04:47:12.531333+0000","last_became_active":"2024-09-14T04:07:21.719331+0000","last_became_peered":"2024-09-14T04:07:21.719331+0000","last_unstale":"2024-09-14T04:47:12.531333+0000","last_undegraded":"2024-09-14T04:47:12.531333+0000","last_fullsized":"2024-09-14T04:47:12.531333+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:35.404023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":43,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.394696+0000","last_change":"2024-09-14T04:43:11.641514+0000","last_active":"2024-09-14T04:46:50.394696+0000","last_peered":"2024-09-14T04:46:50.394696+0000","last_clean":"2024-09-14T04:46:50.394696+0000","last_became_active":"2024-09-14T04:43:11.641151+0000","last_became_peered":"2024-09-14T04:43:11.641151+0000","last_unstale":"2024-09-14T04:46:50.394696+0000","last_undegraded":"2024-09-14T04:46:50.394696+0000","last_fullsized":"2024-09-14T04:46:50.394696+0000","mapping_epoch":1384,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:06:45.045430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2110,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.538440+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T04:47:12.538440+0000","last_peered":"2024-09-14T04:47:12.538440+0000","last_clean":"2024-09-14T04:47:12.538440+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T04:47:12.538440+0000","last_undegraded":"2024-09-14T04:47:12.538440+0000","last_fullsized":"2024-09-14T04:47:12.538440+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":2091,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.550328+0000","last_change":"2024-09-14T04:07:58.507682+0000","last_active":"2024-09-14T04:47:12.550328+0000","last_peered":"2024-09-14T04:47:12.550328+0000","last_clean":"2024-09-14T04:47:12.550328+0000","last_became_active":"2024-09-14T04:07:58.507365+0000","last_became_peered":"2024-09-14T04:07:58.507365+0000","last_unstale":"2024-09-14T04:47:12.550328+0000","last_undegraded":"2024-09-14T04:47:12.550328+0000","last_fullsized":"2024-09-14T04:47:12.550328+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:14:47.294950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1402,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.396157+0000","last_change":"2024-09-14T04:42:49.938505+0000","last_active":"2024-09-14T04:46:50.396157+0000","last_peered":"2024-09-14T04:46:50.396157+0000","last_clean":"2024-09-14T04:46:50.396157+0000","last_became_active":"2024-09-14T04:42:49.938180+0000","last_became_peered":"2024-09-14T04:42:49.938180+0000","last_unstale":"2024-09-14T04:46:50.396157+0000","last_undegraded":"2024-09-14T04:46:50.396157+0000","last_fullsized":"2024-09-14T04:46:50.396157+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:05.929479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1421,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.400955+0000","last_change":"2024-09-14T04:44:57.346582+0000","last_active":"2024-09-14T04:46:50.400955+0000","last_peered":"2024-09-14T04:46:50.400955+0000","last_clean":"2024-09-14T04:46:50.400955+0000","last_became_active":"2024-09-14T04:44:57.330446+0000","last_became_peered":"2024-09-14T04:44:57.330446+0000","last_unstale":"2024-09-14T04:46:50.400955+0000","last_undegraded":"2024-09-14T04:46:50.400955+0000","last_fullsized":"2024-09-14T04:46:50.400955+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:29.238585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":51,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.542855+0000","last_change":"2024-09-14T04:44:14.480667+0000","last_active":"2024-09-14T04:47:12.542855+0000","last_peered":"2024-09-14T04:47:12.542855+0000","last_clean":"2024-09-14T04:47:12.542855+0000","last_became_active":"2024-09-14T04:44:14.480365+0000","last_became_peered":"2024-09-14T04:44:14.480365+0000","last_unstale":"2024-09-14T04:47:12.542855+0000","last_undegraded":"2024-09-14T04:47:12.542855+0000","last_fullsized":"2024-09-14T04:47:12.542855+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:00.611771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":35,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.529370+0000","last_change":"2024-09-14T04:44:57.341426+0000","last_active":"2024-09-14T04:47:12.529370+0000","last_peered":"2024-09-14T04:47:12.529370+0000","last_clean":"2024-09-14T04:47:12.529370+0000","last_became_active":"2024-09-14T04:44:57.328763+0000","last_became_peered":"2024-09-14T04:44:57.328763+0000","last_unstale":"2024-09-14T04:47:12.529370+0000","last_undegraded":"2024-09-14T04:47:12.529370+0000","last_fullsized":"2024-09-14T04:47:12.529370+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:34:40.066761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":25,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.529158+0000","last_change":"2024-09-14T04:44:57.331337+0000","last_active":"2024-09-14T04:47:12.529158+0000","last_peered":"2024-09-14T04:47:12.529158+0000","last_clean":"2024-09-14T04:47:12.529158+0000","last_became_active":"2024-09-14T04:44:57.329336+0000","last_became_peered":"2024-09-14T04:44:57.329336+0000","last_unstale":"2024-09-14T04:47:12.529158+0000","last_undegraded":"2024-09-14T04:47:12.529158+0000","last_fullsized":"2024-09-14T04:47:12.529158+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:52:35.980869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"349'149","reported_seq":779,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.547915+0000","last_change":"2024-09-14T04:41:37.755174+0000","last_active":"2024-09-14T04:47:12.547915+0000","last_peered":"2024-09-14T04:47:12.547915+0000","last_clean":"2024-09-14T04:47:12.547915+0000","last_became_active":"2024-09-14T04:41:36.930263+0000","last_became_peered":"2024-09-14T04:41:36.930263+0000","last_unstale":"2024-09-14T04:47:12.547915+0000","last_undegraded":"2024-09-14T04:47:12.547915+0000","last_fullsized":"2024-09-14T04:47:12.547915+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.755105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.755105+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:27:03.161713+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":25,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:47:02.688408+0000","last_change":"2024-09-14T04:44:57.329870+0000","last_active":"2024-09-14T04:47:02.688408+0000","last_peered":"2024-09-14T04:47:02.688408+0000","last_clean":"2024-09-14T04:47:02.688408+0000","last_became_active":"2024-09-14T04:44:57.329353+0000","last_became_peered":"2024-09-14T04:44:57.329353+0000","last_unstale":"2024-09-14T04:47:02.688408+0000","last_undegraded":"2024-09-14T04:47:02.688408+0000","last_fullsized":"2024-09-14T04:47:02.688408+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:46.432181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1399,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.397467+0000","last_change":"2024-09-14T04:07:21.721242+0000","last_active":"2024-09-14T04:46:50.397467+0000","last_peered":"2024-09-14T04:46:50.397467+0000","last_clean":"2024-09-14T04:46:50.397467+0000","last_became_active":"2024-09-14T04:07:21.720084+0000","last_became_peered":"2024-09-14T04:07:21.720084+0000","last_unstale":"2024-09-14T04:46:50.397467+0000","last_undegraded":"2024-09-14T04:46:50.397467+0000","last_fullsized":"2024-09-14T04:46:50.397467+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:58:41.308643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":2148,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.537945+0000","last_change":"2024-09-14T04:44:57.344614+0000","last_active":"2024-09-14T04:47:12.537945+0000","last_peered":"2024-09-14T04:47:12.537945+0000","last_clean":"2024-09-14T04:47:12.537945+0000","last_became_active":"2024-09-14T04:44:57.332256+0000","last_became_peered":"2024-09-14T04:44:57.332256+0000","last_unstale":"2024-09-14T04:47:12.537945+0000","last_undegraded":"2024-09-14T04:47:12.537945+0000","last_fullsized":"2024-09-14T04:47:12.537945+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:23.438417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"349'149","reported_seq":783,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:47:07.698697+0000","last_change":"2024-09-14T04:45:00.331854+0000","last_active":"2024-09-14T04:47:07.698697+0000","last_peered":"2024-09-14T04:47:07.698697+0000","last_clean":"2024-09-14T04:47:07.698697+0000","last_became_active":"2024-09-14T04:45:00.331460+0000","last_became_peered":"2024-09-14T04:45:00.331460+0000","last_unstale":"2024-09-14T04:47:07.698697+0000","last_undegraded":"2024-09-14T04:47:07.698697+0000","last_fullsized":"2024-09-14T04:47:07.698697+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.567535+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.567535+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:45.095729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":2084,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.550843+0000","last_change":"2024-09-14T04:07:58.555446+0000","last_active":"2024-09-14T04:47:12.550843+0000","last_peered":"2024-09-14T04:47:12.550843+0000","last_clean":"2024-09-14T04:47:12.550843+0000","last_became_active":"2024-09-14T04:07:58.548355+0000","last_became_peered":"2024-09-14T04:07:58.548355+0000","last_unstale":"2024-09-14T04:47:12.550843+0000","last_undegraded":"2024-09-14T04:47:12.550843+0000","last_fullsized":"2024-09-14T04:47:12.550843+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:56:28.426094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":103,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.558324+0000","last_change":"2024-09-14T04:44:57.330746+0000","last_active":"2024-09-14T04:47:12.558324+0000","last_peered":"2024-09-14T04:47:12.558324+0000","last_clean":"2024-09-14T04:47:12.558324+0000","last_became_active":"2024-09-14T04:44:57.330327+0000","last_became_peered":"2024-09-14T04:44:57.330327+0000","last_unstale":"2024-09-14T04:47:12.558324+0000","last_undegraded":"2024-09-14T04:47:12.558324+0000","last_fullsized":"2024-09-14T04:47:12.558324+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:53.122771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":64,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.401795+0000","last_change":"2024-09-14T04:43:13.355296+0000","last_active":"2024-09-14T04:46:50.401795+0000","last_peered":"2024-09-14T04:46:50.401795+0000","last_clean":"2024-09-14T04:46:50.401795+0000","last_became_active":"2024-09-14T04:43:13.354950+0000","last_became_peered":"2024-09-14T04:43:13.354950+0000","last_unstale":"2024-09-14T04:46:50.401795+0000","last_undegraded":"2024-09-14T04:46:50.401795+0000","last_fullsized":"2024-09-14T04:46:50.401795+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:20:08.968818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"349'149","reported_seq":721,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.397094+0000","last_change":"2024-09-14T04:41:39.471393+0000","last_active":"2024-09-14T04:46:50.397094+0000","last_peered":"2024-09-14T04:46:50.397094+0000","last_clean":"2024-09-14T04:46:50.397094+0000","last_became_active":"2024-09-14T04:41:36.931768+0000","last_became_peered":"2024-09-14T04:41:36.931768+0000","last_unstale":"2024-09-14T04:46:50.397094+0000","last_undegraded":"2024-09-14T04:46:50.397094+0000","last_fullsized":"2024-09-14T04:46:50.397094+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.471248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.471248+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:56.210375+0000","scrub_duration":0.021758344999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2103,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.539748+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T04:47:12.539748+0000","last_peered":"2024-09-14T04:47:12.539748+0000","last_clean":"2024-09-14T04:47:12.539748+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T04:47:12.539748+0000","last_undegraded":"2024-09-14T04:47:12.539748+0000","last_fullsized":"2024-09-14T04:47:12.539748+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1431,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.401139+0000","last_change":"2024-09-14T04:44:57.331436+0000","last_active":"2024-09-14T04:46:50.401139+0000","last_peered":"2024-09-14T04:46:50.401139+0000","last_clean":"2024-09-14T04:46:50.401139+0000","last_became_active":"2024-09-14T04:44:57.330957+0000","last_became_peered":"2024-09-14T04:44:57.330957+0000","last_unstale":"2024-09-14T04:46:50.401139+0000","last_undegraded":"2024-09-14T04:46:50.401139+0000","last_fullsized":"2024-09-14T04:46:50.401139+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:09.516688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":96,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.543431+0000","last_change":"2024-09-14T04:42:49.940968+0000","last_active":"2024-09-14T04:47:12.543431+0000","last_peered":"2024-09-14T04:47:12.543431+0000","last_clean":"2024-09-14T04:47:12.543431+0000","last_became_active":"2024-09-14T04:42:49.940667+0000","last_became_peered":"2024-09-14T04:42:49.940667+0000","last_unstale":"2024-09-14T04:47:12.543431+0000","last_undegraded":"2024-09-14T04:47:12.543431+0000","last_fullsized":"2024-09-14T04:47:12.543431+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:52:49.623815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"349'149","reported_seq":721,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.397106+0000","last_change":"2024-09-14T04:41:38.479980+0000","last_active":"2024-09-14T04:46:50.397106+0000","last_peered":"2024-09-14T04:46:50.397106+0000","last_clean":"2024-09-14T04:46:50.397106+0000","last_became_active":"2024-09-14T04:41:36.933392+0000","last_became_peered":"2024-09-14T04:41:36.933392+0000","last_unstale":"2024-09-14T04:46:50.397106+0000","last_undegraded":"2024-09-14T04:46:50.397106+0000","last_fullsized":"2024-09-14T04:46:50.397106+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:38.479887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:38.479887+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:01:26.102279+0000","scrub_duration":0.0062728319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":58,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.542760+0000","last_change":"2024-09-14T04:44:57.329983+0000","last_active":"2024-09-14T04:47:12.542760+0000","last_peered":"2024-09-14T04:47:12.542760+0000","last_clean":"2024-09-14T04:47:12.542760+0000","last_became_active":"2024-09-14T04:44:57.329597+0000","last_became_peered":"2024-09-14T04:44:57.329597+0000","last_unstale":"2024-09-14T04:47:12.542760+0000","last_undegraded":"2024-09-14T04:47:12.542760+0000","last_fullsized":"2024-09-14T04:47:12.542760+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:45.718039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":2138,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.543675+0000","last_change":"2024-09-14T04:06:46.930482+0000","last_active":"2024-09-14T04:47:12.543675+0000","last_peered":"2024-09-14T04:47:12.543675+0000","last_clean":"2024-09-14T04:47:12.543675+0000","last_became_active":"2024-09-14T04:06:46.927372+0000","last_became_peered":"2024-09-14T04:06:46.927372+0000","last_unstale":"2024-09-14T04:47:12.543675+0000","last_undegraded":"2024-09-14T04:47:12.543675+0000","last_fullsized":"2024-09-14T04:47:12.543675+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:07:22.159128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1403,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.396064+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T04:46:50.396064+0000","last_peered":"2024-09-14T04:46:50.396064+0000","last_clean":"2024-09-14T04:46:50.396064+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T04:46:50.396064+0000","last_undegraded":"2024-09-14T04:46:50.396064+0000","last_fullsized":"2024-09-14T04:46:50.396064+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"349'149","reported_seq":713,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.394818+0000","last_change":"2024-09-14T04:41:39.012977+0000","last_active":"2024-09-14T04:46:50.394818+0000","last_peered":"2024-09-14T04:46:50.394818+0000","last_clean":"2024-09-14T04:46:50.394818+0000","last_became_active":"2024-09-14T04:41:36.932552+0000","last_became_peered":"2024-09-14T04:41:36.932552+0000","last_unstale":"2024-09-14T04:46:50.394818+0000","last_undegraded":"2024-09-14T04:46:50.394818+0000","last_fullsized":"2024-09-14T04:46:50.394818+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.012879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.012879+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:16:40.080262+0000","scrub_duration":0.0051721620000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":96,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.543313+0000","last_change":"2024-09-14T04:42:49.945069+0000","last_active":"2024-09-14T04:47:12.543313+0000","last_peered":"2024-09-14T04:47:12.543313+0000","last_clean":"2024-09-14T04:47:12.543313+0000","last_became_active":"2024-09-14T04:42:49.943527+0000","last_became_peered":"2024-09-14T04:42:49.943527+0000","last_unstale":"2024-09-14T04:47:12.543313+0000","last_undegraded":"2024-09-14T04:47:12.543313+0000","last_fullsized":"2024-09-14T04:47:12.543313+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:39:10.524631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":25,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:47:02.688472+0000","last_change":"2024-09-14T04:44:57.329652+0000","last_active":"2024-09-14T04:47:02.688472+0000","last_peered":"2024-09-14T04:47:02.688472+0000","last_clean":"2024-09-14T04:47:02.688472+0000","last_became_active":"2024-09-14T04:44:57.328888+0000","last_became_peered":"2024-09-14T04:44:57.328888+0000","last_unstale":"2024-09-14T04:47:02.688472+0000","last_undegraded":"2024-09-14T04:47:02.688472+0000","last_fullsized":"2024-09-14T04:47:02.688472+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:10.170068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1402,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.395843+0000","last_change":"2024-09-14T04:06:46.875340+0000","last_active":"2024-09-14T04:46:50.395843+0000","last_peered":"2024-09-14T04:46:50.395843+0000","last_clean":"2024-09-14T04:46:50.395843+0000","last_became_active":"2024-09-14T04:06:46.874552+0000","last_became_peered":"2024-09-14T04:06:46.874552+0000","last_unstale":"2024-09-14T04:46:50.395843+0000","last_undegraded":"2024-09-14T04:46:50.395843+0000","last_fullsized":"2024-09-14T04:46:50.395843+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:52.257357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"349'149","reported_seq":1783,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.400875+0000","last_change":"2024-09-14T04:45:00.346627+0000","last_active":"2024-09-14T04:46:50.400875+0000","last_peered":"2024-09-14T04:46:50.400875+0000","last_clean":"2024-09-14T04:46:50.400875+0000","last_became_active":"2024-09-14T04:45:00.346286+0000","last_became_peered":"2024-09-14T04:45:00.346286+0000","last_unstale":"2024-09-14T04:46:50.400875+0000","last_undegraded":"2024-09-14T04:46:50.400875+0000","last_fullsized":"2024-09-14T04:46:50.400875+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:50.784648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:50.784648+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:07:58.441372+0000","scrub_duration":0.01141873,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":51,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.543961+0000","last_change":"2024-09-14T04:44:14.489235+0000","last_active":"2024-09-14T04:47:12.543961+0000","last_peered":"2024-09-14T04:47:12.543961+0000","last_clean":"2024-09-14T04:47:12.543961+0000","last_became_active":"2024-09-14T04:44:14.488950+0000","last_became_peered":"2024-09-14T04:44:14.488950+0000","last_unstale":"2024-09-14T04:47:12.543961+0000","last_undegraded":"2024-09-14T04:47:12.543961+0000","last_fullsized":"2024-09-14T04:47:12.543961+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:54.441144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":2122,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.542252+0000","last_change":"2024-09-14T04:07:21.685415+0000","last_active":"2024-09-14T04:47:12.542252+0000","last_peered":"2024-09-14T04:47:12.542252+0000","last_clean":"2024-09-14T04:47:12.542252+0000","last_became_active":"2024-09-14T04:07:21.683504+0000","last_became_peered":"2024-09-14T04:07:21.683504+0000","last_unstale":"2024-09-14T04:47:12.542252+0000","last_undegraded":"2024-09-14T04:47:12.542252+0000","last_fullsized":"2024-09-14T04:47:12.542252+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:21:08.806291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2124,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.536226+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T04:47:12.536226+0000","last_peered":"2024-09-14T04:47:12.536226+0000","last_clean":"2024-09-14T04:47:12.536226+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T04:47:12.536226+0000","last_undegraded":"2024-09-14T04:47:12.536226+0000","last_fullsized":"2024-09-14T04:47:12.536226+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"349'149","reported_seq":799,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.543720+0000","last_change":"2024-09-14T04:45:00.342826+0000","last_active":"2024-09-14T04:47:12.543720+0000","last_peered":"2024-09-14T04:47:12.543720+0000","last_clean":"2024-09-14T04:47:12.543720+0000","last_became_active":"2024-09-14T04:45:00.342013+0000","last_became_peered":"2024-09-14T04:45:00.342013+0000","last_unstale":"2024-09-14T04:47:12.543720+0000","last_undegraded":"2024-09-14T04:47:12.543720+0000","last_fullsized":"2024-09-14T04:47:12.543720+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.326129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.326129+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:54:28.021603+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2125,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.538125+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T04:47:12.538125+0000","last_peered":"2024-09-14T04:47:12.538125+0000","last_clean":"2024-09-14T04:47:12.538125+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T04:47:12.538125+0000","last_undegraded":"2024-09-14T04:47:12.538125+0000","last_fullsized":"2024-09-14T04:47:12.538125+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2131,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.537828+0000","last_change":"2024-09-14T04:44:57.335783+0000","last_active":"2024-09-14T04:47:12.537828+0000","last_peered":"2024-09-14T04:47:12.537828+0000","last_clean":"2024-09-14T04:47:12.537828+0000","last_became_active":"2024-09-14T04:44:57.332780+0000","last_became_peered":"2024-09-14T04:44:57.332780+0000","last_unstale":"2024-09-14T04:47:12.537828+0000","last_undegraded":"2024-09-14T04:47:12.537828+0000","last_fullsized":"2024-09-14T04:47:12.537828+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:57.457593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1432,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.400758+0000","last_change":"2024-09-14T04:42:49.939790+0000","last_active":"2024-09-14T04:46:50.400758+0000","last_peered":"2024-09-14T04:46:50.400758+0000","last_clean":"2024-09-14T04:46:50.400758+0000","last_became_active":"2024-09-14T04:42:49.939591+0000","last_became_peered":"2024-09-14T04:42:49.939591+0000","last_unstale":"2024-09-14T04:46:50.400758+0000","last_undegraded":"2024-09-14T04:46:50.400758+0000","last_fullsized":"2024-09-14T04:46:50.400758+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:23:16.790593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1395,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.397361+0000","last_change":"2024-09-14T04:44:57.326008+0000","last_active":"2024-09-14T04:46:50.397361+0000","last_peered":"2024-09-14T04:46:50.397361+0000","last_clean":"2024-09-14T04:46:50.397361+0000","last_became_active":"2024-09-14T04:44:57.325617+0000","last_became_peered":"2024-09-14T04:44:57.325617+0000","last_unstale":"2024-09-14T04:46:50.397361+0000","last_undegraded":"2024-09-14T04:46:50.397361+0000","last_fullsized":"2024-09-14T04:46:50.397361+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:15.637508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":684,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.395983+0000","last_change":"2024-09-14T04:43:13.354846+0000","last_active":"2024-09-14T04:46:50.395983+0000","last_peered":"2024-09-14T04:46:50.395983+0000","last_clean":"2024-09-14T04:46:50.395983+0000","last_became_active":"2024-09-14T04:43:13.354515+0000","last_became_peered":"2024-09-14T04:43:13.354515+0000","last_unstale":"2024-09-14T04:46:50.395983+0000","last_undegraded":"2024-09-14T04:46:50.395983+0000","last_fullsized":"2024-09-14T04:46:50.395983+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:17.889341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1401,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.396285+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T04:46:50.396285+0000","last_peered":"2024-09-14T04:46:50.396285+0000","last_clean":"2024-09-14T04:46:50.396285+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T04:46:50.396285+0000","last_undegraded":"2024-09-14T04:46:50.396285+0000","last_fullsized":"2024-09-14T04:46:50.396285+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":1738,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.400563+0000","last_change":"2024-09-14T04:41:30.640651+0000","last_active":"2024-09-14T04:46:50.400563+0000","last_peered":"2024-09-14T04:46:50.400563+0000","last_clean":"2024-09-14T04:46:50.400563+0000","last_became_active":"2024-09-14T04:41:29.532466+0000","last_became_peered":"2024-09-14T04:41:29.532466+0000","last_unstale":"2024-09-14T04:46:50.400563+0000","last_undegraded":"2024-09-14T04:46:50.400563+0000","last_fullsized":"2024-09-14T04:46:50.400563+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":0,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:30.640520+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:30.640520+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:27.776724+0000","scrub_duration":0.0028653139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"349'149","reported_seq":713,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.395767+0000","last_change":"2024-09-14T04:41:39.978080+0000","last_active":"2024-09-14T04:46:50.395767+0000","last_peered":"2024-09-14T04:46:50.395767+0000","last_clean":"2024-09-14T04:46:50.395767+0000","last_became_active":"2024-09-14T04:41:36.929279+0000","last_became_peered":"2024-09-14T04:41:36.929279+0000","last_unstale":"2024-09-14T04:46:50.395767+0000","last_undegraded":"2024-09-14T04:46:50.395767+0000","last_fullsized":"2024-09-14T04:46:50.395767+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.977999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.977999+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:44:06.383827+0000","scrub_duration":0.00192122,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":2103,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.544064+0000","last_change":"2024-09-14T04:44:57.330745+0000","last_active":"2024-09-14T04:47:12.544064+0000","last_peered":"2024-09-14T04:47:12.544064+0000","last_clean":"2024-09-14T04:47:12.544064+0000","last_became_active":"2024-09-14T04:44:57.330261+0000","last_became_peered":"2024-09-14T04:44:57.330261+0000","last_unstale":"2024-09-14T04:47:12.544064+0000","last_undegraded":"2024-09-14T04:47:12.544064+0000","last_fullsized":"2024-09-14T04:47:12.544064+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:24:08.870976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2108,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.550520+0000","last_change":"2024-09-14T04:42:49.941397+0000","last_active":"2024-09-14T04:47:12.550520+0000","last_peered":"2024-09-14T04:47:12.550520+0000","last_clean":"2024-09-14T04:47:12.550520+0000","last_became_active":"2024-09-14T04:42:49.941066+0000","last_became_peered":"2024-09-14T04:42:49.941066+0000","last_unstale":"2024-09-14T04:47:12.550520+0000","last_undegraded":"2024-09-14T04:47:12.550520+0000","last_fullsized":"2024-09-14T04:47:12.550520+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:30.777822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":25,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.531059+0000","last_change":"2024-09-14T04:44:57.326724+0000","last_active":"2024-09-14T04:47:12.531059+0000","last_peered":"2024-09-14T04:47:12.531059+0000","last_clean":"2024-09-14T04:47:12.531059+0000","last_became_active":"2024-09-14T04:44:57.326191+0000","last_became_peered":"2024-09-14T04:44:57.326191+0000","last_unstale":"2024-09-14T04:47:12.531059+0000","last_undegraded":"2024-09-14T04:47:12.531059+0000","last_fullsized":"2024-09-14T04:47:12.531059+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:46.060269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":25,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:47:07.698558+0000","last_change":"2024-09-14T04:44:57.328528+0000","last_active":"2024-09-14T04:47:07.698558+0000","last_peered":"2024-09-14T04:47:07.698558+0000","last_clean":"2024-09-14T04:47:07.698558+0000","last_became_active":"2024-09-14T04:44:57.327739+0000","last_became_peered":"2024-09-14T04:44:57.327739+0000","last_unstale":"2024-09-14T04:47:07.698558+0000","last_undegraded":"2024-09-14T04:47:07.698558+0000","last_fullsized":"2024-09-14T04:47:07.698558+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:04:58.294690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"349'149","reported_seq":745,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.535735+0000","last_change":"2024-09-14T04:42:53.013201+0000","last_active":"2024-09-14T04:47:12.535735+0000","last_peered":"2024-09-14T04:47:12.535735+0000","last_clean":"2024-09-14T04:47:12.535735+0000","last_became_active":"2024-09-14T04:42:53.012818+0000","last_became_peered":"2024-09-14T04:42:53.012818+0000","last_unstale":"2024-09-14T04:47:12.535735+0000","last_undegraded":"2024-09-14T04:47:12.535735+0000","last_fullsized":"2024-09-14T04:47:12.535735+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:56.738636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:56.738636+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:39:19.056067+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":25,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.530119+0000","last_change":"2024-09-14T04:44:57.323436+0000","last_active":"2024-09-14T04:47:12.530119+0000","last_peered":"2024-09-14T04:47:12.530119+0000","last_clean":"2024-09-14T04:47:12.530119+0000","last_became_active":"2024-09-14T04:44:57.323022+0000","last_became_peered":"2024-09-14T04:44:57.323022+0000","last_unstale":"2024-09-14T04:47:12.530119+0000","last_undegraded":"2024-09-14T04:47:12.530119+0000","last_fullsized":"2024-09-14T04:47:12.530119+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:22.775261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1430,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.397988+0000","last_change":"2024-09-14T04:44:57.322357+0000","last_active":"2024-09-14T04:46:50.397988+0000","last_peered":"2024-09-14T04:46:50.397988+0000","last_clean":"2024-09-14T04:46:50.397988+0000","last_became_active":"2024-09-14T04:44:57.321990+0000","last_became_peered":"2024-09-14T04:44:57.321990+0000","last_unstale":"2024-09-14T04:46:50.397988+0000","last_undegraded":"2024-09-14T04:46:50.397988+0000","last_fullsized":"2024-09-14T04:46:50.397988+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:09.658008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"349'149","reported_seq":1755,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.400561+0000","last_change":"2024-09-14T04:41:53.858076+0000","last_active":"2024-09-14T04:46:50.400561+0000","last_peered":"2024-09-14T04:46:50.400561+0000","last_clean":"2024-09-14T04:46:50.400561+0000","last_became_active":"2024-09-14T04:41:36.931585+0000","last_became_peered":"2024-09-14T04:41:36.931585+0000","last_unstale":"2024-09-14T04:46:50.400561+0000","last_undegraded":"2024-09-14T04:46:50.400561+0000","last_fullsized":"2024-09-14T04:46:50.400561+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:53.857924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:53.857924+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:10:20.658986+0000","scrub_duration":0.0022355919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":25,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.529297+0000","last_change":"2024-09-14T04:44:57.329198+0000","last_active":"2024-09-14T04:47:12.529297+0000","last_peered":"2024-09-14T04:47:12.529297+0000","last_clean":"2024-09-14T04:47:12.529297+0000","last_became_active":"2024-09-14T04:44:57.327754+0000","last_became_peered":"2024-09-14T04:44:57.327754+0000","last_unstale":"2024-09-14T04:47:12.529297+0000","last_undegraded":"2024-09-14T04:47:12.529297+0000","last_fullsized":"2024-09-14T04:47:12.529297+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:10:27.944476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":108,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.539831+0000","last_change":"2024-09-14T04:44:57.348687+0000","last_active":"2024-09-14T04:47:12.539831+0000","last_peered":"2024-09-14T04:47:12.539831+0000","last_clean":"2024-09-14T04:47:12.539831+0000","last_became_active":"2024-09-14T04:44:57.332845+0000","last_became_peered":"2024-09-14T04:44:57.332845+0000","last_unstale":"2024-09-14T04:47:12.539831+0000","last_undegraded":"2024-09-14T04:47:12.539831+0000","last_fullsized":"2024-09-14T04:47:12.539831+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:12:18.762737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2131,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.539521+0000","last_change":"2024-09-14T04:44:57.333468+0000","last_active":"2024-09-14T04:47:12.539521+0000","last_peered":"2024-09-14T04:47:12.539521+0000","last_clean":"2024-09-14T04:47:12.539521+0000","last_became_active":"2024-09-14T04:44:57.332626+0000","last_became_peered":"2024-09-14T04:44:57.332626+0000","last_unstale":"2024-09-14T04:47:12.539521+0000","last_undegraded":"2024-09-14T04:47:12.539521+0000","last_fullsized":"2024-09-14T04:47:12.539521+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:57:26.366438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1387,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:47:08.337356+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T04:47:08.337356+0000","last_peered":"2024-09-14T04:47:08.337356+0000","last_clean":"2024-09-14T04:47:08.337356+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T04:47:08.337356+0000","last_undegraded":"2024-09-14T04:47:08.337356+0000","last_fullsized":"2024-09-14T04:47:08.337356+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1385,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.397871+0000","last_change":"2024-09-14T04:06:46.869868+0000","last_active":"2024-09-14T04:46:50.397871+0000","last_peered":"2024-09-14T04:46:50.397871+0000","last_clean":"2024-09-14T04:46:50.397871+0000","last_became_active":"2024-09-14T04:06:46.869126+0000","last_became_peered":"2024-09-14T04:06:46.869126+0000","last_unstale":"2024-09-14T04:46:50.397871+0000","last_undegraded":"2024-09-14T04:46:50.397871+0000","last_fullsized":"2024-09-14T04:46:50.397871+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:41.150138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1370,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:47:08.011810+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T04:47:08.011810+0000","last_peered":"2024-09-14T04:47:08.011810+0000","last_clean":"2024-09-14T04:47:08.011810+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T04:47:08.011810+0000","last_undegraded":"2024-09-14T04:47:08.011810+0000","last_fullsized":"2024-09-14T04:47:08.011810+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":25,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:47:07.698588+0000","last_change":"2024-09-14T04:44:57.332112+0000","last_active":"2024-09-14T04:47:07.698588+0000","last_peered":"2024-09-14T04:47:07.698588+0000","last_clean":"2024-09-14T04:47:07.698588+0000","last_became_active":"2024-09-14T04:44:57.331575+0000","last_became_peered":"2024-09-14T04:44:57.331575+0000","last_unstale":"2024-09-14T04:47:07.698588+0000","last_undegraded":"2024-09-14T04:47:07.698588+0000","last_fullsized":"2024-09-14T04:47:07.698588+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:50:00.391283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1418,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:47:08.011847+0000","last_change":"2024-09-14T04:44:57.351460+0000","last_active":"2024-09-14T04:47:08.011847+0000","last_peered":"2024-09-14T04:47:08.011847+0000","last_clean":"2024-09-14T04:47:08.011847+0000","last_became_active":"2024-09-14T04:44:57.334662+0000","last_became_peered":"2024-09-14T04:44:57.334662+0000","last_unstale":"2024-09-14T04:47:08.011847+0000","last_undegraded":"2024-09-14T04:47:08.011847+0000","last_fullsized":"2024-09-14T04:47:08.011847+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:02.399062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1372,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.535801+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T04:47:12.535801+0000","last_peered":"2024-09-14T04:47:12.535801+0000","last_clean":"2024-09-14T04:47:12.535801+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T04:47:12.535801+0000","last_undegraded":"2024-09-14T04:47:12.535801+0000","last_fullsized":"2024-09-14T04:47:12.535801+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1430,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.550046+0000","last_change":"2024-09-14T04:44:57.349690+0000","last_active":"2024-09-14T04:47:12.550046+0000","last_peered":"2024-09-14T04:47:12.550046+0000","last_clean":"2024-09-14T04:47:12.550046+0000","last_became_active":"2024-09-14T04:44:57.333579+0000","last_became_peered":"2024-09-14T04:44:57.333579+0000","last_unstale":"2024-09-14T04:47:12.550046+0000","last_undegraded":"2024-09-14T04:47:12.550046+0000","last_fullsized":"2024-09-14T04:47:12.550046+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:17:55.265141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1431,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.399717+0000","last_change":"2024-09-14T04:42:49.960838+0000","last_active":"2024-09-14T04:46:50.399717+0000","last_peered":"2024-09-14T04:46:50.399717+0000","last_clean":"2024-09-14T04:46:50.399717+0000","last_became_active":"2024-09-14T04:42:49.954736+0000","last_became_peered":"2024-09-14T04:42:49.954736+0000","last_unstale":"2024-09-14T04:46:50.399717+0000","last_undegraded":"2024-09-14T04:46:50.399717+0000","last_fullsized":"2024-09-14T04:46:50.399717+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:41:13.540983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":2090,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.547168+0000","last_change":"2024-09-14T04:07:58.565442+0000","last_active":"2024-09-14T04:47:12.547168+0000","last_peered":"2024-09-14T04:47:12.547168+0000","last_clean":"2024-09-14T04:47:12.547168+0000","last_became_active":"2024-09-14T04:07:58.552289+0000","last_became_peered":"2024-09-14T04:07:58.552289+0000","last_unstale":"2024-09-14T04:47:12.547168+0000","last_undegraded":"2024-09-14T04:47:12.547168+0000","last_fullsized":"2024-09-14T04:47:12.547168+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:48:02.364960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1392,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.401254+0000","last_change":"2024-09-14T04:44:57.332511+0000","last_active":"2024-09-14T04:46:50.401254+0000","last_peered":"2024-09-14T04:46:50.401254+0000","last_clean":"2024-09-14T04:46:50.401254+0000","last_became_active":"2024-09-14T04:44:57.332166+0000","last_became_peered":"2024-09-14T04:44:57.332166+0000","last_unstale":"2024-09-14T04:46:50.401254+0000","last_undegraded":"2024-09-14T04:46:50.401254+0000","last_fullsized":"2024-09-14T04:46:50.401254+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:53.984639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2111,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.537753+0000","last_change":"2024-09-14T04:07:58.560880+0000","last_active":"2024-09-14T04:47:12.537753+0000","last_peered":"2024-09-14T04:47:12.537753+0000","last_clean":"2024-09-14T04:47:12.537753+0000","last_became_active":"2024-09-14T04:07:58.560132+0000","last_became_peered":"2024-09-14T04:07:58.560132+0000","last_unstale":"2024-09-14T04:47:12.537753+0000","last_undegraded":"2024-09-14T04:47:12.537753+0000","last_fullsized":"2024-09-14T04:47:12.537753+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:46:00.287050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2110,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.540255+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T04:47:12.540255+0000","last_peered":"2024-09-14T04:47:12.540255+0000","last_clean":"2024-09-14T04:47:12.540255+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T04:47:12.540255+0000","last_undegraded":"2024-09-14T04:47:12.540255+0000","last_fullsized":"2024-09-14T04:47:12.540255+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1384,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.395996+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T04:46:50.395996+0000","last_peered":"2024-09-14T04:46:50.395996+0000","last_clean":"2024-09-14T04:46:50.395996+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T04:46:50.395996+0000","last_undegraded":"2024-09-14T04:46:50.395996+0000","last_fullsized":"2024-09-14T04:46:50.395996+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1421,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.401198+0000","last_change":"2024-09-14T04:44:57.339993+0000","last_active":"2024-09-14T04:46:50.401198+0000","last_peered":"2024-09-14T04:46:50.401198+0000","last_clean":"2024-09-14T04:46:50.401198+0000","last_became_active":"2024-09-14T04:44:57.330472+0000","last_became_peered":"2024-09-14T04:44:57.330472+0000","last_unstale":"2024-09-14T04:46:50.401198+0000","last_undegraded":"2024-09-14T04:46:50.401198+0000","last_fullsized":"2024-09-14T04:46:50.401198+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:19:27.046988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":51,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.543226+0000","last_change":"2024-09-14T04:44:14.479373+0000","last_active":"2024-09-14T04:47:12.543226+0000","last_peered":"2024-09-14T04:47:12.543226+0000","last_clean":"2024-09-14T04:47:12.543226+0000","last_became_active":"2024-09-14T04:44:14.479084+0000","last_became_peered":"2024-09-14T04:44:14.479084+0000","last_unstale":"2024-09-14T04:47:12.543226+0000","last_undegraded":"2024-09-14T04:47:12.543226+0000","last_fullsized":"2024-09-14T04:47:12.543226+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:49:23.412806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":54,"num_pg":16,"stat_sum":{"num_bytes":241303576,"num_objects":3683,"num_object_clones":0,"num_object_copies":11049,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5077,"num_objects_unfound":0,"num_objects_dirty":3683,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":249107,"num_write_kb":8089217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3005,"num_bytes_recovered":196870144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":753086464,"data_stored":753074248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":122224,"ondisk_log_size":122224,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":5070,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4250,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201956,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":490732,"kb_used_data":148772,"kb_used_omap":1,"kb_used_meta":341950,"kb_avail":93254420,"statfs":{"total":95995035648,"available":95492526080,"internally_reserved":0,"allocated":152342528,"data_stored":147841086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":350157259},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463592,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":574888,"kb_used_data":142560,"kb_used_omap":1,"kb_used_meta":432190,"kb_avail":93170264,"statfs":{"total":95995035648,"available":95406350336,"internally_reserved":0,"allocated":145981440,"data_stored":141483991,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":442563015},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725229,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":583392,"kb_used_data":146968,"kb_used_omap":1,"kb_used_meta":436286,"kb_avail":93161760,"statfs":{"total":95995035648,"available":95397642240,"internally_reserved":0,"allocated":150495232,"data_stored":146002321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":446757318},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":1392,"seq":5978594476075,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":132012,"kb_used_data":10660,"kb_used_omap":117,"kb_used_meta":121226,"kb_avail":93613140,"statfs":{"total":95995035648,"available":95859855360,"internally_reserved":0,"allocated":10915840,"data_stored":6414950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120638,"internal_metadata":124135618},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772074970,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":660704,"kb_used_data":231384,"kb_used_omap":71,"kb_used_meta":429048,"kb_avail":93084448,"statfs":{"total":95995035648,"available":95318474752,"internally_reserved":0,"allocated":236937216,"data_stored":232444382,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":439346102},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707303906,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":283932,"kb_used_data":49172,"kb_used_omap":68,"kb_used_meta":234683,"kb_avail":93461220,"statfs":{"total":95995035648,"available":95704289280,"internally_reserved":0,"allocated":50352128,"data_stored":45863270,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":240315600},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872729065,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":421908,"kb_used_data":52364,"kb_used_omap":67,"kb_used_meta":369276,"kb_avail":93323244,"statfs":{"total":95995035648,"available":95563001856,"internally_reserved":0,"allocated":53620736,"data_stored":49138929,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":378138981},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056115,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":528732,"kb_used_data":127636,"kb_used_omap":74,"kb_used_meta":401013,"kb_avail":93216420,"statfs":{"total":95995035648,"available":95453614080,"internally_reserved":0,"allocated":130699264,"data_stored":126209874,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":410638005},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1901,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":113250304,"data_stored":113246232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":31784960,"data_stored":31784960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":34406400,"data_stored":34406400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":203034624,"data_stored":203030552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":131072,"data_stored":131072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":122748928,"data_stored":122748928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":124981248,"data_stored":124977176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":54,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":122748928,"data_stored":122748928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:47:25.704 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:25.726 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:25.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:25.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:25.828 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:25.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:25.850 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:25.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:25.951 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:25.974 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:25.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:25.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:26.076 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:26.098 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:26.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:26.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:26.200 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:26.223 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:26.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:26.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:26.325 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:26.347 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:26.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:26.348 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:26.449 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:26.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:26.473 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:26.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:26.575 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:26.597 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:26.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:26.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:26.699 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:26.722 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:26.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:26.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:26.824 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:26.846 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:26.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:26.847 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:26.884 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:26 smithi129 ceph-mon[101316]: pgmap v2520: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 32 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 4377/1569 objects misplaced (278.967%); 1.5 MiB/s, 23 objects/s recovering 2024-09-14T04:47:26.948 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:26.970 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:26.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:26.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:27.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:26 smithi039 ceph-mon[106470]: pgmap v2520: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 32 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 4377/1569 objects misplaced (278.967%); 1.5 MiB/s, 23 objects/s recovering 2024-09-14T04:47:27.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:26 smithi039 ceph-mon[111235]: pgmap v2520: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 32 MiB data, 2.8 GiB used, 712 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 4377/1569 objects misplaced (278.967%); 1.5 MiB/s, 23 objects/s recovering 2024-09-14T04:47:27.071 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:27.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:27.096 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:27.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:27.197 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:27.218 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:27.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:27.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:27.320 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:27.342 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:27.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:27.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:27.444 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:27.465 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:27.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:27.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:27.567 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:27.589 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:27.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:27.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:27.636 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2024-09-14T04:47:27.637 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2024-09-14T04:47:27.691 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:27.713 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:27.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:27.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:27.739 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:47:27.739 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [], 2024-09-14T04:47:27.739 INFO:teuthology.orchestra.run.smithi129.stdout: "complaint_time": 30, 2024-09-14T04:47:27.740 INFO:teuthology.orchestra.run.smithi129.stdout: "num_blocked_ops": 0 2024-09-14T04:47:27.740 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:47:27.815 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:27.838 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:27.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:27.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:27.940 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:27.962 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:27.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:27.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:28.064 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:28.087 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:28.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:28.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:28.189 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:28.212 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:28.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:28.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:28.314 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:28.335 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:28.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:28.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:28.437 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:28.459 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:28.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:28.460 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:28.560 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:28.583 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:28.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:28.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:28.618 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:47:28.686 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:28.708 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:28.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:28.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:28.810 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:28.833 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:28.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:28.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:28.935 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:28.957 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:28.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:28.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:29.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:28 smithi129 ceph-mon[101316]: pgmap v2521: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 966 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 4296/96 objects misplaced (4475.000%); 1.7 MiB/s, 52 objects/s recovering 2024-09-14T04:47:29.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:28 smithi039 ceph-mon[106470]: pgmap v2521: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 966 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 4296/96 objects misplaced (4475.000%); 1.7 MiB/s, 52 objects/s recovering 2024-09-14T04:47:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:47:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:47:28] "GET /metrics HTTP/1.1" 200 36740 "" "Prometheus/2.51.0" 2024-09-14T04:47:29.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:28 smithi039 ceph-mon[111235]: pgmap v2521: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 966 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 695 op/s; 4296/96 objects misplaced (4475.000%); 1.7 MiB/s, 52 objects/s recovering 2024-09-14T04:47:29.060 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:29.083 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:29.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:29.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:29.185 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:29.210 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:29.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:29.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:29.311 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:29.334 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:29.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:29.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:29.387 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-14T04:47:29.435 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:29.458 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:29.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:29.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:29.560 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:29.586 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:29.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:29.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:29.670 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:47:29.687 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:29.709 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:29.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:29.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:29.811 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:29.834 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:29.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:29.835 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:29.936 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:29.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:29.957 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:29.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:29.971 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:47:30.059 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:30.080 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:30.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:30.081 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:30.182 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:30.206 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:30.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:30.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:30.308 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:30.332 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:30.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:30.333 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:30.433 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:30.454 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:30.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:30.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:30.556 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:30.577 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:30.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:30.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:30.679 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:30.702 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:30.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:30.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:30.804 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:30.828 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:30.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:30.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:30.930 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:30.952 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:30.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:30.953 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:30.958 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:30 smithi129 ceph-mon[101316]: pgmap v2522: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 4296/78 objects misplaced (5507.692%); 1.0 MiB/s, 41 objects/s recovering 2024-09-14T04:47:31.055 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:31.077 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:31.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:31.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:31.179 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:31.202 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:31.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:31.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:31.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:30 smithi039 ceph-mon[106470]: pgmap v2522: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 4296/78 objects misplaced (5507.692%); 1.0 MiB/s, 41 objects/s recovering 2024-09-14T04:47:31.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:30 smithi039 ceph-mon[111235]: pgmap v2522: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 555 op/s; 4296/78 objects misplaced (5507.692%); 1.0 MiB/s, 41 objects/s recovering 2024-09-14T04:47:31.305 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:31.328 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:31.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:31.329 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:31.430 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:31.452 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:31.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:31.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:31.554 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:31.575 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:31.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:31.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:31.677 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:31.701 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:31.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:31.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:31.803 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:31.827 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:31.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:31.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:31.928 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:32.003 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:32.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:32.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:32.105 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:32.127 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:32.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:32.128 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:32.225 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:47:32.225 INFO:teuthology.orchestra.run.smithi129.stdout: "size": 20, 2024-09-14T04:47:32.225 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 600, 2024-09-14T04:47:32.225 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [ 2024-09-14T04:47:32.225 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.225 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:11496 54.6 54:65dd3c56:::benchmark_data_smithi039_264334_object11495:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1388)", 2024-09-14T04:47:32.225 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:43:21.062393+0000", 2024-09-14T04:47:32.225 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 251.158564321, 2024-09-14T04:47:32.226 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.098479161999999995, 2024-09-14T04:47:32.226 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:32.226 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:32.226 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:32.226 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:47:32.226 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:47:32.226 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 11496 2024-09-14T04:47:32.226 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.226 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:32.226 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.226 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:32.226 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:21.062393+0000", 2024-09-14T04:47:32.226 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.227 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.227 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.227 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:32.227 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:21.062393+0000", 2024-09-14T04:47:32.227 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.227 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.227 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.227 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:32.227 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:21.062391+0000", 2024-09-14T04:47:32.227 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:47:32.227 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.227 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.227 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:32.227 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:21.062443+0000", 2024-09-14T04:47:32.227 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.1796999999999999e-05 2024-09-14T04:47:32.228 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.228 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.228 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:32.228 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:21.062453+0000", 2024-09-14T04:47:32.228 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0131e-05 2024-09-14T04:47:32.228 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.228 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.228 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:32.228 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:21.062458+0000", 2024-09-14T04:47:32.228 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6600000000000003e-06 2024-09-14T04:47:32.228 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.228 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.228 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:32.229 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:21.062489+0000", 2024-09-14T04:47:32.229 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.0828999999999999e-05 2024-09-14T04:47:32.229 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.229 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.229 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:32.229 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:21.062564+0000", 2024-09-14T04:47:32.229 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.4744000000000001e-05 2024-09-14T04:47:32.229 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.229 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.229 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:47:32.229 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:21.062643+0000", 2024-09-14T04:47:32.229 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.9103999999999993e-05 2024-09-14T04:47:32.229 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.229 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.230 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:32.230 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:21.063172+0000", 2024-09-14T04:47:32.230 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00052912800000000002 2024-09-14T04:47:32.230 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.230 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.230 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.230 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:21.063585+0000", 2024-09-14T04:47:32.230 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00041343400000000001 2024-09-14T04:47:32.230 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.230 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.230 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.230 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:21.160768+0000", 2024-09-14T04:47:32.230 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.097182828999999998 2024-09-14T04:47:32.230 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.231 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.231 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:32.231 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:21.160830+0000", 2024-09-14T04:47:32.231 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.2117000000000002e-05 2024-09-14T04:47:32.231 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.231 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.231 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:32.231 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:21.160872+0000", 2024-09-14T04:47:32.231 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1817000000000003e-05 2024-09-14T04:47:32.231 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.231 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:32.231 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.231 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.231 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.232 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:66556 54.6 54:647e80e0:::benchmark_data_smithi039_264334_object66555:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1392)", 2024-09-14T04:47:32.232 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:43:51.929593+0000", 2024-09-14T04:47:32.232 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 220.29136408100001, 2024-09-14T04:47:32.232 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.084587792999999994, 2024-09-14T04:47:32.232 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:32.232 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:32.232 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:32.232 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:47:32.232 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:47:32.232 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 66556 2024-09-14T04:47:32.232 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.232 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:32.232 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.233 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:32.233 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:51.929593+0000", 2024-09-14T04:47:32.233 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.233 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.233 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.233 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:32.233 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:51.929593+0000", 2024-09-14T04:47:32.233 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.233 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.233 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.233 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:32.233 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:51.929591+0000", 2024-09-14T04:47:32.233 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:47:32.233 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.234 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.234 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:32.234 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:51.929626+0000", 2024-09-14T04:47:32.234 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.4984999999999998e-05 2024-09-14T04:47:32.234 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.234 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.234 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:32.234 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:51.929638+0000", 2024-09-14T04:47:32.234 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1646e-05 2024-09-14T04:47:32.234 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.235 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.235 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:32.235 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:51.929643+0000", 2024-09-14T04:47:32.235 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.3900000000000001e-06 2024-09-14T04:47:32.235 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.235 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.235 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:32.235 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:51.929672+0000", 2024-09-14T04:47:32.235 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.8442e-05 2024-09-14T04:47:32.235 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.235 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.235 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:32.235 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:51.929757+0000", 2024-09-14T04:47:32.236 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.5463999999999994e-05 2024-09-14T04:47:32.236 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.236 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.236 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:47:32.236 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:51.929842+0000", 2024-09-14T04:47:32.236 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.4493999999999994e-05 2024-09-14T04:47:32.236 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.236 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.236 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:32.236 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:51.931640+0000", 2024-09-14T04:47:32.236 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0017979890000000001 2024-09-14T04:47:32.236 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.236 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.236 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.237 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:51.932203+0000", 2024-09-14T04:47:32.237 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00056333299999999995 2024-09-14T04:47:32.237 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.237 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.237 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.237 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:52.014088+0000", 2024-09-14T04:47:32.237 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.081884578999999999 2024-09-14T04:47:32.237 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.237 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.237 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:32.237 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:52.014140+0000", 2024-09-14T04:47:32.237 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2669e-05 2024-09-14T04:47:32.237 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.238 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.238 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:32.238 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:52.014181+0000", 2024-09-14T04:47:32.238 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0590999999999998e-05 2024-09-14T04:47:32.238 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.238 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:32.238 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.238 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.238 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.238 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:68400 54.6 54:68328e13:::benchmark_data_smithi039_264334_object68399:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1392)", 2024-09-14T04:47:32.238 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:43:53.403956+0000", 2024-09-14T04:47:32.238 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 218.81700129999999, 2024-09-14T04:47:32.238 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.097056130000000004, 2024-09-14T04:47:32.238 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:32.239 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:32.239 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:32.239 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:47:32.239 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:47:32.239 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 68400 2024-09-14T04:47:32.239 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.239 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:32.239 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.239 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:32.239 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.403956+0000", 2024-09-14T04:47:32.239 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.239 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.239 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.240 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:32.240 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.403956+0000", 2024-09-14T04:47:32.240 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.240 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.240 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.240 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:32.240 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.403954+0000", 2024-09-14T04:47:32.240 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:47:32.240 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.240 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.240 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:32.240 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404019+0000", 2024-09-14T04:47:32.240 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.4386000000000003e-05 2024-09-14T04:47:32.240 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.241 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.241 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:32.241 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404029+0000", 2024-09-14T04:47:32.241 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0414000000000001e-05 2024-09-14T04:47:32.241 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.241 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.241 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:32.241 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404034+0000", 2024-09-14T04:47:32.241 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.057e-06 2024-09-14T04:47:32.241 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.241 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.241 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:32.241 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404068+0000", 2024-09-14T04:47:32.241 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3380999999999998e-05 2024-09-14T04:47:32.242 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.242 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.242 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:32.242 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404195+0000", 2024-09-14T04:47:32.242 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012756099999999999 2024-09-14T04:47:32.242 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.242 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.242 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:47:32.242 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404326+0000", 2024-09-14T04:47:32.242 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013119499999999999 2024-09-14T04:47:32.242 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.242 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.242 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.242 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.410437+0000", 2024-09-14T04:47:32.243 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0061107419999999997 2024-09-14T04:47:32.243 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.243 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.243 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:32.243 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.410475+0000", 2024-09-14T04:47:32.243 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8169e-05 2024-09-14T04:47:32.243 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.243 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.243 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.243 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.500921+0000", 2024-09-14T04:47:32.243 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.090445708999999999 2024-09-14T04:47:32.243 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.243 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.243 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:32.244 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.500974+0000", 2024-09-14T04:47:32.244 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2923999999999997e-05 2024-09-14T04:47:32.244 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.244 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.244 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:32.244 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.501012+0000", 2024-09-14T04:47:32.244 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8012999999999997e-05 2024-09-14T04:47:32.244 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.244 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:32.244 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.244 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.244 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.244 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:68403 54.6 54:6443ddab:::benchmark_data_smithi039_264334_object68402:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1392)", 2024-09-14T04:47:32.244 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:43:53.404446+0000", 2024-09-14T04:47:32.245 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 218.81651088000001, 2024-09-14T04:47:32.245 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.096669750999999998, 2024-09-14T04:47:32.245 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:32.245 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:32.245 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:32.245 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:47:32.245 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:47:32.245 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 68403 2024-09-14T04:47:32.245 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.245 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:32.245 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.246 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:32.246 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404446+0000", 2024-09-14T04:47:32.246 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.246 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.246 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.246 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:32.246 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404446+0000", 2024-09-14T04:47:32.246 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.246 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.246 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.246 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:32.246 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404445+0000", 2024-09-14T04:47:32.246 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:47:32.246 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.247 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.247 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:32.247 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404498+0000", 2024-09-14T04:47:32.247 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.346e-05 2024-09-14T04:47:32.247 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.247 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.247 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:32.247 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404510+0000", 2024-09-14T04:47:32.247 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1194e-05 2024-09-14T04:47:32.247 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.247 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.247 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:32.247 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404517+0000", 2024-09-14T04:47:32.247 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.0530000000000001e-06 2024-09-14T04:47:32.248 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.248 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.248 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:32.248 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.406468+0000", 2024-09-14T04:47:32.248 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0019508500000000001 2024-09-14T04:47:32.248 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.248 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.248 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:32.248 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.406575+0000", 2024-09-14T04:47:32.248 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000107556 2024-09-14T04:47:32.248 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.248 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.248 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:47:32.249 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.406697+0000", 2024-09-14T04:47:32.249 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012212 2024-09-14T04:47:32.249 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.249 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.249 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.249 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.410469+0000", 2024-09-14T04:47:32.249 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0037713019999999998 2024-09-14T04:47:32.249 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.249 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.249 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:32.249 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.410479+0000", 2024-09-14T04:47:32.249 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0847e-05 2024-09-14T04:47:32.249 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.249 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.250 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.250 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.501046+0000", 2024-09-14T04:47:32.250 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.090566728999999999 2024-09-14T04:47:32.250 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.250 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.250 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:32.250 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.501087+0000", 2024-09-14T04:47:32.250 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1026999999999999e-05 2024-09-14T04:47:32.250 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.250 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.250 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:32.250 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.501116+0000", 2024-09-14T04:47:32.250 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.8852000000000001e-05 2024-09-14T04:47:32.250 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.251 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:32.251 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.251 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.251 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.251 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:68404 54.6 54:6bb4aa50:::benchmark_data_smithi039_264334_object68403:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1392)", 2024-09-14T04:47:32.251 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:43:53.404843+0000", 2024-09-14T04:47:32.251 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 218.81611373199999, 2024-09-14T04:47:32.251 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.096363689000000002, 2024-09-14T04:47:32.251 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:32.251 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:32.251 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:32.251 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:47:32.251 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:47:32.251 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 68404 2024-09-14T04:47:32.252 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.252 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:32.252 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.252 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:32.252 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404843+0000", 2024-09-14T04:47:32.252 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.252 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.252 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.252 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:32.252 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404843+0000", 2024-09-14T04:47:32.252 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.252 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.252 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.253 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:32.253 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404843+0000", 2024-09-14T04:47:32.253 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:32.254 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:47:32.254 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.254 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.255 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:32.255 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404873+0000", 2024-09-14T04:47:32.255 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.9960000000000001e-05 2024-09-14T04:47:32.255 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.255 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.255 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:32.255 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404882+0000", 2024-09-14T04:47:32.255 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.4490000000000005e-06 2024-09-14T04:47:32.255 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.255 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.255 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:32.255 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.404888+0000", 2024-09-14T04:47:32.255 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.1789999999999996e-06 2024-09-14T04:47:32.255 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.256 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.256 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:32.256 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.408713+0000", 2024-09-14T04:47:32.256 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0038248610000000001 2024-09-14T04:47:32.256 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.256 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.256 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:32.256 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.408795+0000", 2024-09-14T04:47:32.256 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.1738999999999998e-05 2024-09-14T04:47:32.256 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.256 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.256 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:47:32.256 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.408873+0000", 2024-09-14T04:47:32.257 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.8032000000000001e-05 2024-09-14T04:47:32.257 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.257 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.257 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:32.257 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.410526+0000", 2024-09-14T04:47:32.257 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0016527740000000001 2024-09-14T04:47:32.257 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.257 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.257 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.257 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.411065+0000", 2024-09-14T04:47:32.257 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00053945900000000005 2024-09-14T04:47:32.258 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.258 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.258 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.258 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.501144+0000", 2024-09-14T04:47:32.259 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.090079318000000005 2024-09-14T04:47:32.259 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.259 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.259 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:32.259 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.501181+0000", 2024-09-14T04:47:32.259 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6458000000000002e-05 2024-09-14T04:47:32.259 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.259 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.259 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:32.259 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.501207+0000", 2024-09-14T04:47:32.259 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.6276999999999999e-05 2024-09-14T04:47:32.259 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.259 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:32.260 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.260 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.260 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.260 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.35436.0:68414 54.f e1392/1380 54:f4108b15:::benchmark_data_smithi039_264334_object68413:head v 1392'4287, mlcod=1392'4285)", 2024-09-14T04:47:32.260 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:43:53.412852+0000", 2024-09-14T04:47:32.260 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 218.80810493499999, 2024-09-14T04:47:32.260 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.093915778000000005, 2024-09-14T04:47:32.260 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:32.260 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:32.260 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:32.260 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.260 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:32.260 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.412852+0000", 2024-09-14T04:47:32.260 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.261 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.261 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.261 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:32.261 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.412852+0000", 2024-09-14T04:47:32.261 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.261 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.261 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.261 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:32.261 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.412852+0000", 2024-09-14T04:47:32.261 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:47:32.261 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.261 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.261 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:32.262 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.412921+0000", 2024-09-14T04:47:32.262 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.9048999999999996e-05 2024-09-14T04:47:32.262 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.262 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.262 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:32.262 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.412928+0000", 2024-09-14T04:47:32.262 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.2509999999999997e-06 2024-09-14T04:47:32.262 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.262 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.262 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:32.262 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.412932+0000", 2024-09-14T04:47:32.262 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8130000000000002e-06 2024-09-14T04:47:32.262 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.263 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.263 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:32.263 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.500774+0000", 2024-09-14T04:47:32.263 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.087842240000000002 2024-09-14T04:47:32.263 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.263 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.263 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:32.263 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.500798+0000", 2024-09-14T04:47:32.263 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.3510999999999999e-05 2024-09-14T04:47:32.263 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.263 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.263 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:32.263 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.506732+0000", 2024-09-14T04:47:32.263 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0059339570000000001 2024-09-14T04:47:32.264 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.264 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.264 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:32.264 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.506768+0000", 2024-09-14T04:47:32.264 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6448000000000001e-05 2024-09-14T04:47:32.264 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.264 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:32.264 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.264 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.264 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.264 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.35436.0:68412 54.e e1392/1380 54:7391dba8:::benchmark_data_smithi039_264334_object68411:head v 1392'4283, mlcod=1392'4281)", 2024-09-14T04:47:32.264 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:43:53.414570+0000", 2024-09-14T04:47:32.264 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 218.80638668099999, 2024-09-14T04:47:32.264 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.095679800999999995, 2024-09-14T04:47:32.265 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:32.265 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:32.265 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:32.265 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.265 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:32.265 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.414570+0000", 2024-09-14T04:47:32.265 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.265 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.265 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.265 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:32.265 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.414570+0000", 2024-09-14T04:47:32.265 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.265 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.265 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.266 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:32.266 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.414570+0000", 2024-09-14T04:47:32.266 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:47:32.266 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.266 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.266 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:32.266 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.414619+0000", 2024-09-14T04:47:32.266 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.9243999999999998e-05 2024-09-14T04:47:32.266 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.266 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.266 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:32.266 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.500758+0000", 2024-09-14T04:47:32.266 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.086138951000000005 2024-09-14T04:47:32.266 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.267 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.267 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:32.267 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.500769+0000", 2024-09-14T04:47:32.267 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.095e-05 2024-09-14T04:47:32.267 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.267 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.267 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:32.267 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.501229+0000", 2024-09-14T04:47:32.267 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00045990199999999999 2024-09-14T04:47:32.267 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.267 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.267 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:32.267 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.501245+0000", 2024-09-14T04:47:32.268 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5956000000000001e-05 2024-09-14T04:47:32.268 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.268 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.268 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:32.268 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.510168+0000", 2024-09-14T04:47:32.268 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.008923149 2024-09-14T04:47:32.268 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.268 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.268 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:32.268 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.510250+0000", 2024-09-14T04:47:32.268 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.2233999999999999e-05 2024-09-14T04:47:32.268 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.269 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:32.269 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.269 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.269 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.269 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.35436.0:68417 54.f e1392/1380 54:f223c55f:::benchmark_data_smithi039_264334_object68416:head v 1392'4288, mlcod=1392'4285)", 2024-09-14T04:47:32.269 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:43:53.415332+0000", 2024-09-14T04:47:32.269 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 218.80562517199999, 2024-09-14T04:47:32.269 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.091473889000000003, 2024-09-14T04:47:32.269 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:32.269 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:32.269 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:32.269 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.269 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:32.269 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.415332+0000", 2024-09-14T04:47:32.270 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.270 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.270 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.270 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:32.270 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.415332+0000", 2024-09-14T04:47:32.270 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.270 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.270 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.270 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:32.270 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.415332+0000", 2024-09-14T04:47:32.270 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:47:32.270 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.270 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.271 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:32.271 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.415401+0000", 2024-09-14T04:47:32.271 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.9407999999999997e-05 2024-09-14T04:47:32.271 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.271 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.271 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:32.271 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.500754+0000", 2024-09-14T04:47:32.271 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.085353164999999995 2024-09-14T04:47:32.271 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.271 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.271 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:32.272 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.500766+0000", 2024-09-14T04:47:32.272 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1855000000000001e-05 2024-09-14T04:47:32.272 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.272 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.272 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:32.272 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.504126+0000", 2024-09-14T04:47:32.272 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0033599020000000001 2024-09-14T04:47:32.272 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.272 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.272 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:32.272 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.504195+0000", 2024-09-14T04:47:32.272 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8804000000000001e-05 2024-09-14T04:47:32.272 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.273 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.273 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:32.273 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.506789+0000", 2024-09-14T04:47:32.273 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0025946649999999999 2024-09-14T04:47:32.273 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.273 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.273 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:32.273 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:43:53.506806+0000", 2024-09-14T04:47:32.273 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.6381000000000001e-05 2024-09-14T04:47:32.273 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.273 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:32.273 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.273 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.273 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.274 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:190173 54.6 54:60239bc8:::benchmark_data_smithi039_264334_object63776:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1413)", 2024-09-14T04:47:32.274 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:46:04.533359+0000", 2024-09-14T04:47:32.274 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 87.687597779000001, 2024-09-14T04:47:32.274 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.112311283, 2024-09-14T04:47:32.274 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:32.274 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:32.274 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:32.274 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:47:32.274 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:47:32.274 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 190173 2024-09-14T04:47:32.274 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.274 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:32.274 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.275 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:32.275 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:04.533359+0000", 2024-09-14T04:47:32.275 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.275 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.275 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.275 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:32.275 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:04.533359+0000", 2024-09-14T04:47:32.275 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.275 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.275 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.275 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:32.275 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:04.533358+0000", 2024-09-14T04:47:32.275 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:47:32.275 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.276 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.276 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:32.276 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:04.533365+0000", 2024-09-14T04:47:32.276 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.0389999999999999e-06 2024-09-14T04:47:32.276 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.276 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.276 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:32.276 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:04.533377+0000", 2024-09-14T04:47:32.276 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2532e-05 2024-09-14T04:47:32.276 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.276 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.276 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:32.276 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:04.533383+0000", 2024-09-14T04:47:32.277 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9399999999999999e-06 2024-09-14T04:47:32.277 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.277 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.277 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:32.277 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:04.533429+0000", 2024-09-14T04:47:32.277 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.5738000000000002e-05 2024-09-14T04:47:32.277 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.277 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.277 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:32.277 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:04.533553+0000", 2024-09-14T04:47:32.277 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012347899999999999 2024-09-14T04:47:32.277 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.277 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.277 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:47:32.278 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:04.533680+0000", 2024-09-14T04:47:32.278 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012697399999999999 2024-09-14T04:47:32.278 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.278 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.278 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.278 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:04.538686+0000", 2024-09-14T04:47:32.278 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0050067339999999997 2024-09-14T04:47:32.278 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.278 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.278 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:32.278 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:04.540946+0000", 2024-09-14T04:47:32.278 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00226005 2024-09-14T04:47:32.278 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.278 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.279 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.279 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:04.645575+0000", 2024-09-14T04:47:32.279 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.104628321 2024-09-14T04:47:32.279 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.279 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.279 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:32.279 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:04.645629+0000", 2024-09-14T04:47:32.279 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.4175999999999998e-05 2024-09-14T04:47:32.279 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.279 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.279 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:32.279 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:04.645671+0000", 2024-09-14T04:47:32.280 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1737999999999999e-05 2024-09-14T04:47:32.280 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.280 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:32.280 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.280 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.280 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.280 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:190224 54.6 54:6e3c8ee8:::benchmark_data_smithi039_264334_object63827:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1413)", 2024-09-14T04:47:32.280 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:46:04.609091+0000", 2024-09-14T04:47:32.280 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 87.611866101999993, 2024-09-14T04:47:32.280 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.049173701, 2024-09-14T04:47:32.280 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:32.280 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:32.280 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:32.280 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:47:32.281 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:47:32.281 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 190224 2024-09-14T04:47:32.281 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.281 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:32.281 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.281 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:32.281 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:04.609091+0000", 2024-09-14T04:47:32.281 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.281 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.281 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.281 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:32.281 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:04.609091+0000", 2024-09-14T04:47:32.281 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.282 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.282 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.282 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:32.282 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:04.609089+0000", 2024-09-14T04:47:32.282 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:47:32.282 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.282 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.282 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:32.282 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:04.609100+0000", 2024-09-14T04:47:32.282 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0934999999999999e-05 2024-09-14T04:47:32.282 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.282 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.282 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:32.283 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:04.609116+0000", 2024-09-14T04:47:32.283 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.6030999999999999e-05 2024-09-14T04:47:32.283 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.283 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.283 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:32.283 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:04.609122+0000", 2024-09-14T04:47:32.283 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.3029999999999996e-06 2024-09-14T04:47:32.283 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.283 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.283 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:32.283 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:04.609170+0000", 2024-09-14T04:47:32.283 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8143999999999998e-05 2024-09-14T04:47:32.283 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.283 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.283 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:32.284 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:04.609303+0000", 2024-09-14T04:47:32.284 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000132199 2024-09-14T04:47:32.284 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.284 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.284 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:47:32.284 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:04.609427+0000", 2024-09-14T04:47:32.284 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012400700000000001 2024-09-14T04:47:32.284 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.284 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.284 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.284 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:04.612926+0000", 2024-09-14T04:47:32.284 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0034996459999999999 2024-09-14T04:47:32.284 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.285 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.285 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:32.285 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:04.613046+0000", 2024-09-14T04:47:32.285 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000119435 2024-09-14T04:47:32.285 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.285 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.285 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.285 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:04.658132+0000", 2024-09-14T04:47:32.285 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.045086385999999999 2024-09-14T04:47:32.285 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.285 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.285 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:32.285 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:04.658220+0000", 2024-09-14T04:47:32.285 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.8315999999999999e-05 2024-09-14T04:47:32.286 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.286 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.286 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:32.286 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:04.658265+0000", 2024-09-14T04:47:32.286 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.439e-05 2024-09-14T04:47:32.286 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.286 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:32.286 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.286 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.286 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.286 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:192405 54.6 54:65bc8903:::benchmark_data_smithi039_264334_object66008:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1413)", 2024-09-14T04:47:32.286 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:46:07.335227+0000", 2024-09-14T04:47:32.286 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 84.885729721000004, 2024-09-14T04:47:32.287 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.070835035000000005, 2024-09-14T04:47:32.287 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:32.287 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:32.287 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:32.287 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:47:32.287 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:47:32.287 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 192405 2024-09-14T04:47:32.287 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.287 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:32.287 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.287 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:32.287 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.335227+0000", 2024-09-14T04:47:32.287 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.288 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.288 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.288 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:32.288 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.335227+0000", 2024-09-14T04:47:32.288 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.288 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.288 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.288 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:32.288 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.335226+0000", 2024-09-14T04:47:32.288 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:47:32.288 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.289 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.289 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:32.289 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.335233+0000", 2024-09-14T04:47:32.289 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.4080000000000001e-06 2024-09-14T04:47:32.289 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.289 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.289 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:32.289 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.335246+0000", 2024-09-14T04:47:32.289 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.343e-05 2024-09-14T04:47:32.289 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.289 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.289 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:32.289 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.335253+0000", 2024-09-14T04:47:32.289 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.6279999999999999e-06 2024-09-14T04:47:32.290 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.290 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.290 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:32.290 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.335302+0000", 2024-09-14T04:47:32.290 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.9314000000000003e-05 2024-09-14T04:47:32.290 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.290 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.290 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:32.290 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.335402+0000", 2024-09-14T04:47:32.290 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.9742999999999997e-05 2024-09-14T04:47:32.290 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.290 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.290 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:47:32.291 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.335483+0000", 2024-09-14T04:47:32.291 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.0673999999999994e-05 2024-09-14T04:47:32.291 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.291 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.291 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:32.291 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.338380+0000", 2024-09-14T04:47:32.291 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.002897602 2024-09-14T04:47:32.291 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.291 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.291 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.291 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.338631+0000", 2024-09-14T04:47:32.291 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000250936 2024-09-14T04:47:32.291 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.291 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.292 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.292 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.406002+0000", 2024-09-14T04:47:32.292 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.067370823999999996 2024-09-14T04:47:32.292 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.292 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.292 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:32.292 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.406037+0000", 2024-09-14T04:47:32.292 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5349999999999999e-05 2024-09-14T04:47:32.292 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.292 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.292 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:32.292 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.406062+0000", 2024-09-14T04:47:32.292 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.4978999999999999e-05 2024-09-14T04:47:32.292 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.293 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:32.293 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.293 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.293 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.293 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:192427 54.6 54:6d612468:::benchmark_data_smithi039_264334_object66030:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1413)", 2024-09-14T04:47:32.293 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:46:07.354257+0000", 2024-09-14T04:47:32.293 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 84.866700582999997, 2024-09-14T04:47:32.293 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.061399100999999998, 2024-09-14T04:47:32.293 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:32.293 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:32.293 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:32.293 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:47:32.293 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:47:32.293 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 192427 2024-09-14T04:47:32.294 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.294 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:32.294 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.294 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:32.294 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.354257+0000", 2024-09-14T04:47:32.294 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.294 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.294 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.294 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:32.294 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.354257+0000", 2024-09-14T04:47:32.294 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.294 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.294 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.295 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:32.295 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.354255+0000", 2024-09-14T04:47:32.295 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:47:32.295 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.295 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.295 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:32.295 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.354263+0000", 2024-09-14T04:47:32.295 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.7390000000000005e-06 2024-09-14T04:47:32.295 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.295 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.295 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:32.295 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.354282+0000", 2024-09-14T04:47:32.295 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.9055e-05 2024-09-14T04:47:32.295 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.296 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.296 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:32.296 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.354293+0000", 2024-09-14T04:47:32.296 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0891e-05 2024-09-14T04:47:32.296 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.296 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.296 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:32.296 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.354345+0000", 2024-09-14T04:47:32.296 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.1397000000000003e-05 2024-09-14T04:47:32.296 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.296 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.296 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:32.297 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.354480+0000", 2024-09-14T04:47:32.297 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013511299999999999 2024-09-14T04:47:32.297 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.297 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.297 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:47:32.297 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.354617+0000", 2024-09-14T04:47:32.297 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013687199999999999 2024-09-14T04:47:32.297 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.297 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.297 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.298 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.359239+0000", 2024-09-14T04:47:32.298 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0046226599999999998 2024-09-14T04:47:32.298 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.298 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.298 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:32.298 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.360780+0000", 2024-09-14T04:47:32.298 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0015404489999999999 2024-09-14T04:47:32.298 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.298 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.298 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.298 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.415525+0000", 2024-09-14T04:47:32.298 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.054745004999999999 2024-09-14T04:47:32.298 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.299 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.299 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:32.299 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.415606+0000", 2024-09-14T04:47:32.299 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.1100999999999995e-05 2024-09-14T04:47:32.299 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.299 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.299 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:32.299 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.415656+0000", 2024-09-14T04:47:32.299 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.9840999999999999e-05 2024-09-14T04:47:32.299 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.299 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:32.299 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.299 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.300 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.300 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:192430 54.6 54:643db441:::benchmark_data_smithi039_264334_object66033:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1413)", 2024-09-14T04:47:32.300 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:46:07.361883+0000", 2024-09-14T04:47:32.300 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 84.859073741000003, 2024-09-14T04:47:32.300 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.064590907000000003, 2024-09-14T04:47:32.300 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:32.300 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:32.300 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:32.300 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:47:32.300 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:47:32.300 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 192430 2024-09-14T04:47:32.300 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.300 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:32.301 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.301 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:32.301 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.361883+0000", 2024-09-14T04:47:32.301 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.301 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.301 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.301 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:32.301 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.361883+0000", 2024-09-14T04:47:32.301 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.301 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.301 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.301 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:32.301 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.361882+0000", 2024-09-14T04:47:32.302 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:47:32.302 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.302 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.302 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:32.302 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.361889+0000", 2024-09-14T04:47:32.302 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.3300000000000001e-06 2024-09-14T04:47:32.302 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.302 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.302 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:32.302 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.361906+0000", 2024-09-14T04:47:32.302 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.7112000000000002e-05 2024-09-14T04:47:32.302 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.302 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.302 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:32.303 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.361918+0000", 2024-09-14T04:47:32.303 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.19e-05 2024-09-14T04:47:32.303 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.303 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.303 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:32.303 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.361978+0000", 2024-09-14T04:47:32.303 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9487999999999997e-05 2024-09-14T04:47:32.303 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.303 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.303 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:32.303 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.362135+0000", 2024-09-14T04:47:32.303 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00015768800000000001 2024-09-14T04:47:32.304 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.304 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.304 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:47:32.304 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.362277+0000", 2024-09-14T04:47:32.304 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00014223999999999999 2024-09-14T04:47:32.304 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.304 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.304 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.304 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.366147+0000", 2024-09-14T04:47:32.304 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.003869431 2024-09-14T04:47:32.304 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.304 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.305 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:32.305 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.366672+0000", 2024-09-14T04:47:32.305 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00052507300000000003 2024-09-14T04:47:32.305 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.305 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.305 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.305 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.426403+0000", 2024-09-14T04:47:32.305 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.059730531000000003 2024-09-14T04:47:32.305 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.305 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.305 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:32.305 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.426436+0000", 2024-09-14T04:47:32.305 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3306999999999998e-05 2024-09-14T04:47:32.305 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.306 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.306 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:32.306 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:07.426474+0000", 2024-09-14T04:47:32.306 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8498999999999999e-05 2024-09-14T04:47:32.306 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.306 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:32.306 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.306 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.306 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.306 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:195068 54.6 54:6feecde7:::benchmark_data_smithi039_264334_object68671:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1413)", 2024-09-14T04:47:32.306 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:46:10.733663+0000", 2024-09-14T04:47:32.306 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 81.487294394000003, 2024-09-14T04:47:32.306 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.061674553999999999, 2024-09-14T04:47:32.306 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:32.306 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:32.307 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:32.307 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:47:32.307 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:47:32.307 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 195068 2024-09-14T04:47:32.307 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.307 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:32.307 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.307 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:32.307 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.733663+0000", 2024-09-14T04:47:32.307 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.307 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.307 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.307 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:32.307 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.733663+0000", 2024-09-14T04:47:32.308 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.308 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.308 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.308 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:32.308 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.733662+0000", 2024-09-14T04:47:32.308 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:47:32.308 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.308 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.308 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:32.308 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.733667+0000", 2024-09-14T04:47:32.308 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.4e-06 2024-09-14T04:47:32.308 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.308 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.308 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:32.309 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.733680+0000", 2024-09-14T04:47:32.309 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2828e-05 2024-09-14T04:47:32.309 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.309 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.309 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:32.309 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.733687+0000", 2024-09-14T04:47:32.309 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.1600000000000001e-06 2024-09-14T04:47:32.309 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.309 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.309 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:32.309 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.749306+0000", 2024-09-14T04:47:32.309 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0156188 2024-09-14T04:47:32.309 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.310 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.310 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:32.310 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.749440+0000", 2024-09-14T04:47:32.310 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000133891 2024-09-14T04:47:32.310 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.310 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.310 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:47:32.310 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.749573+0000", 2024-09-14T04:47:32.310 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000133282 2024-09-14T04:47:32.310 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.310 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.310 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:32.310 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.758025+0000", 2024-09-14T04:47:32.310 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0084520710000000002 2024-09-14T04:47:32.311 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.311 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.311 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.311 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.766226+0000", 2024-09-14T04:47:32.311 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0082010680000000006 2024-09-14T04:47:32.311 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.311 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.311 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.311 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.795254+0000", 2024-09-14T04:47:32.311 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.029027846999999999 2024-09-14T04:47:32.311 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.311 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.311 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:32.311 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.795301+0000", 2024-09-14T04:47:32.312 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6737999999999999e-05 2024-09-14T04:47:32.312 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.312 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.312 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:32.312 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.795337+0000", 2024-09-14T04:47:32.312 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.629e-05 2024-09-14T04:47:32.312 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.312 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:32.312 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.312 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.312 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.312 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:195071 54.6 54:69e23093:::benchmark_data_smithi039_264334_object68674:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1413)", 2024-09-14T04:47:32.312 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:46:10.734801+0000", 2024-09-14T04:47:32.313 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 81.486155917000005, 2024-09-14T04:47:32.313 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.060683570999999999, 2024-09-14T04:47:32.313 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:32.313 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:32.313 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:32.313 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:47:32.313 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:47:32.313 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 195071 2024-09-14T04:47:32.313 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.313 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:32.313 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.313 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:32.313 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.734801+0000", 2024-09-14T04:47:32.313 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.314 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.314 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.314 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:32.314 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.734801+0000", 2024-09-14T04:47:32.314 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.314 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.314 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.314 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:32.314 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.734800+0000", 2024-09-14T04:47:32.314 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:47:32.314 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.314 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.314 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:32.314 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.734809+0000", 2024-09-14T04:47:32.315 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.7269999999999996e-06 2024-09-14T04:47:32.315 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.315 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.315 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:32.315 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.734825+0000", 2024-09-14T04:47:32.315 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.6158999999999999e-05 2024-09-14T04:47:32.315 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.315 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.315 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:32.315 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.734834+0000", 2024-09-14T04:47:32.316 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.037e-06 2024-09-14T04:47:32.316 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.316 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.316 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:32.316 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.753245+0000", 2024-09-14T04:47:32.316 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.018410797 2024-09-14T04:47:32.316 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.316 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.316 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:32.316 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.753544+0000", 2024-09-14T04:47:32.316 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00029907000000000002 2024-09-14T04:47:32.316 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.317 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.317 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:47:32.317 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.753889+0000", 2024-09-14T04:47:32.317 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00034539599999999998 2024-09-14T04:47:32.317 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.317 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.317 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:32.317 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.766109+0000", 2024-09-14T04:47:32.317 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.012219633000000001 2024-09-14T04:47:32.317 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.318 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.318 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.318 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.774416+0000", 2024-09-14T04:47:32.318 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0083072279999999998 2024-09-14T04:47:32.318 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.318 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.318 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.318 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.795395+0000", 2024-09-14T04:47:32.318 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.020979162999999999 2024-09-14T04:47:32.318 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.318 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.318 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:32.318 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.795443+0000", 2024-09-14T04:47:32.318 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8100999999999999e-05 2024-09-14T04:47:32.319 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.319 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.319 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:32.319 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.795485+0000", 2024-09-14T04:47:32.319 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1328999999999997e-05 2024-09-14T04:47:32.319 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.319 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:32.319 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.319 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.319 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.319 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:195077 54.6 54:6ab49f44:::benchmark_data_smithi039_264334_object68680:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1413)", 2024-09-14T04:47:32.319 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:46:10.740607+0000", 2024-09-14T04:47:32.319 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 81.480350319999999, 2024-09-14T04:47:32.319 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.070204979000000001, 2024-09-14T04:47:32.320 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:32.320 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:32.320 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:32.320 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:47:32.320 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:47:32.320 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 195077 2024-09-14T04:47:32.320 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.320 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:32.320 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.320 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:32.320 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.740607+0000", 2024-09-14T04:47:32.320 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.320 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.321 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.321 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:32.321 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.740607+0000", 2024-09-14T04:47:32.321 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.321 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.321 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.321 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:32.321 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.740606+0000", 2024-09-14T04:47:32.321 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:47:32.321 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.321 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.321 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:32.322 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.740612+0000", 2024-09-14T04:47:32.322 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.0499999999999997e-06 2024-09-14T04:47:32.322 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.322 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.322 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:32.322 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.740645+0000", 2024-09-14T04:47:32.322 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3621999999999998e-05 2024-09-14T04:47:32.322 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.322 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.322 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:32.322 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.740652+0000", 2024-09-14T04:47:32.322 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.2320000000000004e-06 2024-09-14T04:47:32.322 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.322 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.323 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:32.323 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.758103+0000", 2024-09-14T04:47:32.323 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.017450050000000002 2024-09-14T04:47:32.323 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.323 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.323 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:32.323 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.758180+0000", 2024-09-14T04:47:32.323 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.7305999999999995e-05 2024-09-14T04:47:32.323 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.323 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.323 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:47:32.323 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.758244+0000", 2024-09-14T04:47:32.323 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.3922000000000002e-05 2024-09-14T04:47:32.323 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.324 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.324 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:32.324 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.766234+0000", 2024-09-14T04:47:32.324 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.007990324 2024-09-14T04:47:32.324 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.324 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.324 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.324 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.774440+0000", 2024-09-14T04:47:32.324 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0082059779999999992 2024-09-14T04:47:32.324 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.324 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.324 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.324 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.810754+0000", 2024-09-14T04:47:32.325 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.036313955000000002 2024-09-14T04:47:32.325 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.325 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.325 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:32.325 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.810786+0000", 2024-09-14T04:47:32.325 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.2039999999999998e-05 2024-09-14T04:47:32.325 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.325 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.325 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:32.325 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.810812+0000", 2024-09-14T04:47:32.325 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.5774000000000001e-05 2024-09-14T04:47:32.325 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.325 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:32.325 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.325 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.326 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.326 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:195085 54.6 54:69663e81:::benchmark_data_smithi039_264334_object68688:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1413)", 2024-09-14T04:47:32.326 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:46:10.751273+0000", 2024-09-14T04:47:32.326 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 81.469684616999999, 2024-09-14T04:47:32.326 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.059690890000000003, 2024-09-14T04:47:32.326 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:32.326 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:32.326 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:32.326 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:47:32.326 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:47:32.326 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 195085 2024-09-14T04:47:32.326 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.326 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:32.326 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.327 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:32.327 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.751273+0000", 2024-09-14T04:47:32.327 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.327 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.327 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.327 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:32.328 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.751273+0000", 2024-09-14T04:47:32.328 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.328 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.328 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.328 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:32.328 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.751271+0000", 2024-09-14T04:47:32.328 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:47:32.329 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.329 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.329 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:32.329 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.751279+0000", 2024-09-14T04:47:32.329 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.0530000000000007e-06 2024-09-14T04:47:32.329 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.329 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.329 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:32.329 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.751291+0000", 2024-09-14T04:47:32.329 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1523e-05 2024-09-14T04:47:32.329 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.329 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.329 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:32.329 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.751296+0000", 2024-09-14T04:47:32.330 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.4729999999999999e-06 2024-09-14T04:47:32.330 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.330 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.330 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:32.330 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.762279+0000", 2024-09-14T04:47:32.330 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.010982771000000001 2024-09-14T04:47:32.330 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.330 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.330 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:32.330 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.762403+0000", 2024-09-14T04:47:32.330 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012392300000000001 2024-09-14T04:47:32.330 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.330 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.330 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:47:32.331 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.762522+0000", 2024-09-14T04:47:32.331 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011902500000000001 2024-09-14T04:47:32.331 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.331 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.331 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:32.331 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.766317+0000", 2024-09-14T04:47:32.331 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0037948750000000001 2024-09-14T04:47:32.331 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.331 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.331 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.331 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.777164+0000", 2024-09-14T04:47:32.331 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.010846925 2024-09-14T04:47:32.331 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.331 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.332 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.332 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.810865+0000", 2024-09-14T04:47:32.332 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.033701226000000001 2024-09-14T04:47:32.332 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.332 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.332 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:32.332 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.810920+0000", 2024-09-14T04:47:32.332 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.524e-05 2024-09-14T04:47:32.332 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.332 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.332 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:32.332 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:10.810963+0000", 2024-09-14T04:47:32.332 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.3053999999999998e-05 2024-09-14T04:47:32.333 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.333 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:32.333 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.333 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.333 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.333 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:221641 54.6 54:661b3e3c:::benchmark_data_smithi039_264334_object95244:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1414)", 2024-09-14T04:47:32.333 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:46:43.784784+0000", 2024-09-14T04:47:32.333 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 48.436172659999997, 2024-09-14T04:47:32.333 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.039359185999999997, 2024-09-14T04:47:32.333 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:32.333 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:32.333 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:32.333 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:47:32.333 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:47:32.334 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 221641 2024-09-14T04:47:32.334 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.334 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:32.334 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.334 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:32.334 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:43.784784+0000", 2024-09-14T04:47:32.334 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.334 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.334 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.334 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:32.334 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:43.784784+0000", 2024-09-14T04:47:32.334 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.334 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.334 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.335 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:32.335 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:43.784783+0000", 2024-09-14T04:47:32.335 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:47:32.335 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.335 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.335 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:32.335 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:43.784790+0000", 2024-09-14T04:47:32.335 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.5650000000000004e-06 2024-09-14T04:47:32.335 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.335 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.335 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:32.335 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:43.784803+0000", 2024-09-14T04:47:32.335 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.259e-05 2024-09-14T04:47:32.335 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.336 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.336 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:32.336 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:43.784810+0000", 2024-09-14T04:47:32.336 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.063e-06 2024-09-14T04:47:32.336 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.336 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.336 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:32.336 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:43.784851+0000", 2024-09-14T04:47:32.336 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1026000000000003e-05 2024-09-14T04:47:32.336 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.336 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.336 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:32.336 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:43.784970+0000", 2024-09-14T04:47:32.336 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011921699999999999 2024-09-14T04:47:32.337 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.337 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.337 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:47:32.337 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:43.785086+0000", 2024-09-14T04:47:32.337 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000116087 2024-09-14T04:47:32.337 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.337 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.337 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:32.337 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:43.788882+0000", 2024-09-14T04:47:32.337 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0037958179999999998 2024-09-14T04:47:32.337 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.337 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.337 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.338 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:43.796118+0000", 2024-09-14T04:47:32.338 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0072362579999999998 2024-09-14T04:47:32.338 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.338 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.338 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.338 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:43.824055+0000", 2024-09-14T04:47:32.338 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.027936829 2024-09-14T04:47:32.338 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.338 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.338 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:32.338 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:43.824105+0000", 2024-09-14T04:47:32.338 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.9673999999999999e-05 2024-09-14T04:47:32.338 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.338 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.339 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:32.339 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:46:43.824144+0000", 2024-09-14T04:47:32.339 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8757999999999997e-05 2024-09-14T04:47:32.339 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.339 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:32.339 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.339 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.339 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.339 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:237162 54.6 54:651adb17:::benchmark_data_smithi039_264334_object110765:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1415)", 2024-09-14T04:47:32.339 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:47:03.130009+0000", 2024-09-14T04:47:32.339 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 29.090947720999999, 2024-09-14T04:47:32.339 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.038784405000000001, 2024-09-14T04:47:32.339 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:32.340 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:32.340 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:32.340 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:47:32.340 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:47:32.340 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 237162 2024-09-14T04:47:32.340 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.340 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:32.340 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.340 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:32.340 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:03.130009+0000", 2024-09-14T04:47:32.340 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.340 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.340 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.340 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:32.341 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:03.130009+0000", 2024-09-14T04:47:32.341 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.341 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.341 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.341 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:32.341 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:03.130008+0000", 2024-09-14T04:47:32.341 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:47:32.341 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.341 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.341 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:32.341 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:03.130014+0000", 2024-09-14T04:47:32.341 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.1550000000000004e-06 2024-09-14T04:47:32.341 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.342 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.342 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:32.342 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:03.130023+0000", 2024-09-14T04:47:32.342 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.4290000000000007e-06 2024-09-14T04:47:32.342 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.342 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.342 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:32.342 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:03.130029+0000", 2024-09-14T04:47:32.342 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.395e-06 2024-09-14T04:47:32.342 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.342 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.342 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:32.342 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:03.140099+0000", 2024-09-14T04:47:32.343 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.010070233 2024-09-14T04:47:32.343 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.343 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.343 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:32.343 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:03.140217+0000", 2024-09-14T04:47:32.343 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000118187 2024-09-14T04:47:32.343 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.343 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.343 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:47:32.343 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:03.140334+0000", 2024-09-14T04:47:32.343 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000116891 2024-09-14T04:47:32.343 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.343 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.343 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.344 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:03.145890+0000", 2024-09-14T04:47:32.344 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0055553759999999999 2024-09-14T04:47:32.344 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.344 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.344 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:32.344 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:03.151081+0000", 2024-09-14T04:47:32.344 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0051918880000000004 2024-09-14T04:47:32.344 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.344 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.344 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.344 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:03.168711+0000", 2024-09-14T04:47:32.344 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.017629221 2024-09-14T04:47:32.344 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.345 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.345 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:32.345 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:03.168759+0000", 2024-09-14T04:47:32.345 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8327e-05 2024-09-14T04:47:32.345 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.345 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.345 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:32.345 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:03.168794+0000", 2024-09-14T04:47:32.345 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.4839000000000003e-05 2024-09-14T04:47:32.345 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.345 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:32.345 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.345 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.345 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.346 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:250928 54.6 54:6ebf47e7:::benchmark_data_smithi039_264334_object124531:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1416)", 2024-09-14T04:47:32.346 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:47:20.874398+0000", 2024-09-14T04:47:32.346 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 11.346559263, 2024-09-14T04:47:32.346 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.040907647999999998, 2024-09-14T04:47:32.346 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:47:32.346 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:47:32.346 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:47:32.346 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:47:32.346 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:47:32.346 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 250928 2024-09-14T04:47:32.346 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.346 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:47:32.346 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.346 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:47:32.347 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:20.874398+0000", 2024-09-14T04:47:32.347 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.347 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.347 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.347 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:47:32.347 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:20.874398+0000", 2024-09-14T04:47:32.347 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:47:32.347 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.347 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.347 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:47:32.347 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:20.874396+0000", 2024-09-14T04:47:32.347 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:47:32.347 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.348 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.348 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:47:32.348 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:20.874404+0000", 2024-09-14T04:47:32.348 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.303e-06 2024-09-14T04:47:32.348 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.348 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.348 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:47:32.348 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:20.874419+0000", 2024-09-14T04:47:32.348 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5267000000000001e-05 2024-09-14T04:47:32.348 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.348 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.348 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:47:32.348 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:20.874428+0000", 2024-09-14T04:47:32.348 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.5629999999999992e-06 2024-09-14T04:47:32.349 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.349 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.349 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:47:32.349 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:20.877705+0000", 2024-09-14T04:47:32.349 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.003277187 2024-09-14T04:47:32.349 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.349 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.349 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:47:32.349 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:20.877836+0000", 2024-09-14T04:47:32.349 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000130426 2024-09-14T04:47:32.349 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.349 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.349 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,5", 2024-09-14T04:47:32.349 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:20.878738+0000", 2024-09-14T04:47:32.350 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00090210099999999999 2024-09-14T04:47:32.350 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.350 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.350 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.350 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:20.890108+0000", 2024-09-14T04:47:32.350 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.011370038000000001 2024-09-14T04:47:32.350 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.350 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.350 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:47:32.350 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:20.890339+0000", 2024-09-14T04:47:32.350 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00023090599999999999 2024-09-14T04:47:32.350 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.350 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.350 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:47:32.351 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:20.915071+0000", 2024-09-14T04:47:32.351 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.024732067999999999 2024-09-14T04:47:32.351 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.351 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.351 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:47:32.351 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:20.915191+0000", 2024-09-14T04:47:32.351 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000120036 2024-09-14T04:47:32.351 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:47:32.351 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:47:32.351 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:47:32.351 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:47:20.915306+0000", 2024-09-14T04:47:32.351 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000114739 2024-09-14T04:47:32.351 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.352 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:32.352 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.352 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:47:32.352 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:47:32.352 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:47:32.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:32.353 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:32.354 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:32.454 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:32.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:32.479 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:32.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:32.580 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:32.604 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:32.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:32.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:32.706 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:32.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:32.731 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:32.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:32.824 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is unset 2024-09-14T04:47:32.833 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:32.862 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:32.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:32.863 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:32.964 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:32.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:32.988 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:32.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:32 smithi129 ceph-mon[101316]: pgmap v2523: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 371 op/s; 3980/78 objects misplaced (5102.564%); 1.0 MiB/s, 90 objects/s recovering 2024-09-14T04:47:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:32 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3691039625' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:47:33.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:32 smithi039 ceph-mon[106470]: pgmap v2523: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 371 op/s; 3980/78 objects misplaced (5102.564%); 1.0 MiB/s, 90 objects/s recovering 2024-09-14T04:47:33.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:32 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3691039625' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:47:33.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:32 smithi039 ceph-mon[111235]: pgmap v2523: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 371 op/s; 3980/78 objects misplaced (5102.564%); 1.0 MiB/s, 90 objects/s recovering 2024-09-14T04:47:33.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:32 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3691039625' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:47:33.089 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:33.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:33.113 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:33.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:33.205 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-14T04:47:33.214 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:33.269 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:33.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:33.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:33.371 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:33.395 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:33.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:33.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:33.497 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:33.519 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:47:33.520 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:33.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:33.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:33.544 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:47:33.622 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:33.645 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:33.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:33.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:33.747 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:33.768 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:33.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:33.769 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:33.839 INFO:teuthology.orchestra.run.smithi039.stderr:pool 'unique_pool_1' removed 2024-09-14T04:47:33.870 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:33.924 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:33.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:33.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:34.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:33 smithi129 ceph-mon[101316]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:47:34.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:33 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T04:47:34.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:33 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3691039625' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:47:34.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:33 smithi129 ceph-mon[101316]: osdmap e1417: 8 total, 8 up, 8 in 2024-09-14T04:47:34.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:33 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/471649683' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:47:34.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:33 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:47:34.026 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:34.045 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:47:34.049 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:34.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:34.050 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:34.101 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:33 smithi039 ceph-mon[106470]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:47:34.101 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:33 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T04:47:34.101 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:33 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3691039625' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:47:34.101 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:33 smithi039 ceph-mon[106470]: osdmap e1417: 8 total, 8 up, 8 in 2024-09-14T04:47:34.101 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:33 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/471649683' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:47:34.101 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:33 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:47:34.102 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:33 smithi039 ceph-mon[111235]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:47:34.102 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:33 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T04:47:34.102 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:33 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3691039625' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:47:34.102 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:33 smithi039 ceph-mon[111235]: osdmap e1417: 8 total, 8 up, 8 in 2024-09-14T04:47:34.102 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:33 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/471649683' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:47:34.102 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:33 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:47:34.151 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:34.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:34.173 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:34.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:34.274 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:34.297 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:34.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:34.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:34.399 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:34.421 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:34.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:34.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:34.523 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:34.546 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:34.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:34.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:34.648 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:34.670 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:34.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:34.671 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:34.756 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-14T04:47:34.757 INFO:tasks.radosbench:Beginning radosbench... 2024-09-14T04:47:34.757 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2024-09-14T04:47:34.757 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_2 16 2024-09-14T04:47:34.772 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:34.793 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:34.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:34.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:34.894 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:34 smithi129 ceph-mon[101316]: pgmap v2525: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 49 op/s; 3980/78 objects misplaced (5102.564%); 365 KiB/s, 94 objects/s recovering 2024-09-14T04:47:34.895 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:34 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:47:34.895 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:34 smithi129 ceph-mon[101316]: osdmap e1418: 8 total, 8 up, 8 in 2024-09-14T04:47:34.895 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:34.915 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:34.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:34.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:35.017 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:35.038 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:35.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:35.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:35.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:34 smithi039 ceph-mon[106470]: pgmap v2525: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 49 op/s; 3980/78 objects misplaced (5102.564%); 365 KiB/s, 94 objects/s recovering 2024-09-14T04:47:35.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:34 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:47:35.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:34 smithi039 ceph-mon[106470]: osdmap e1418: 8 total, 8 up, 8 in 2024-09-14T04:47:35.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:34 smithi039 ceph-mon[111235]: pgmap v2525: 99 pgs: 1 active+recovering+remapped, 3 active+remapped+backfill_wait, 95 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 49 op/s; 3980/78 objects misplaced (5102.564%); 365 KiB/s, 94 objects/s recovering 2024-09-14T04:47:35.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:34 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:47:35.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:34 smithi039 ceph-mon[111235]: osdmap e1418: 8 total, 8 up, 8 in 2024-09-14T04:47:35.140 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:35.163 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:35.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:35.164 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:35.265 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:35.286 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:35.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:35.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:35.388 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:35.411 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:35.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:35.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:35.513 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:35.532 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:47:35.534 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:35.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:35.535 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:35.636 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:35.658 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:35.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:35.659 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:35.759 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:35.780 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:35.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:35.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:35.882 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:35.905 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:35.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:35.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:36.006 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:36.028 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:36.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:36.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:36.130 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:36.153 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:36.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:36.153 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:36.254 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:36.278 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:36.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:36.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:36.380 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:36.401 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:36.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:36.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:36.405 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:47:36.405 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [], 2024-09-14T04:47:36.405 INFO:teuthology.orchestra.run.smithi129.stdout: "num_ops": 0 2024-09-14T04:47:36.405 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:47:36.504 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:36.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:36.528 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:36.528 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:36.629 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:36.650 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:36.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:36.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:36.752 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:36.758 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:36 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:47:36.758 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:36 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:47:36.773 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:47:36.773 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:36 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:47:36.773 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:47:36.773 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:36 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:47:36.774 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:36.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:36.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:36.876 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:36.898 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:36.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:36.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:37.000 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:37.027 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:37.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:37.028 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:37.063 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-14T04:47:37.129 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:37.154 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:37.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:37.155 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:37.256 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:37.258 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:47:37.278 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:37.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:37.279 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:37.339 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:47:37.380 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:37.401 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:37.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:37.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:37.503 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:37.526 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:37.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:37.527 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:37.628 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:37.650 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:37.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:37.651 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:37.671 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:37 smithi129 ceph-mon[101316]: pgmap v2527: 83 pgs: 83 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:37.671 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:47:37.671 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:47:37.692 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:37 smithi039 ceph-mon[106470]: pgmap v2527: 83 pgs: 83 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:37.692 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:47:37.692 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:47:37.692 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:37 smithi039 ceph-mon[111235]: pgmap v2527: 83 pgs: 83 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:37.692 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:47:37.692 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:47:37.717 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:47:37.717 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:47:37.717 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:47:37.717 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:47:37.726 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:47:37.726 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:47:37.743 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:47:37.743 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:47:37.751 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:37.759 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:47:37.759 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:47:37.773 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:37.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:37.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:37.776 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:47:37.776 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:47:37.792 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:47:37.793 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:47:37.809 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:47:37.810 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:47:37.875 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:37.896 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:37.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:37.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:37.998 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:38.020 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:38.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:38.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:38.121 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:38.144 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:38.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:38.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:38.246 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:38.255 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:47:38.267 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:38.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:38.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:38.369 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:38.391 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:38.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:38.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:38.493 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:38.516 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:38.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:38.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:38.617 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:38.639 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:38.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:38.640 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:38.648 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:38 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/190463816' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:47:38.648 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:38 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/190463816' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:47:38.741 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:38.763 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:38.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:38.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:38.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:38 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/190463816' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:47:38.865 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:38.887 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:38.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:38.888 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:38.972 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:47:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:47:38] "GET /metrics HTTP/1.1" 200 34315 "" "Prometheus/2.51.0" 2024-09-14T04:47:38.989 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:39.011 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:39.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:39.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:39.113 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:39.120 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:47:39.121 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:47:39.130 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:47:39.131 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T04:47:39.136 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:39.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:39.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:39.238 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:39.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:39.260 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:39.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:39.361 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:39.383 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:39.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:39.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:39.484 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:39.508 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:39.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:39.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:39.610 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:39.631 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:39.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:39.632 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:39.733 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:39.754 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:39.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:39.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:39.758 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:39 smithi129 ceph-mon[101316]: pgmap v2528: 83 pgs: 83 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:39.778 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-14T04:47:39.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:39 smithi039 ceph-mon[106470]: pgmap v2528: 83 pgs: 83 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:39.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:39 smithi039 ceph-mon[111235]: pgmap v2528: 83 pgs: 83 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:39.856 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:39.881 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:39.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:39.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:39.983 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:40.006 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:40.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:40.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:40.015 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:47:40.108 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:40.129 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:40.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:40.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:40.231 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:40.252 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:40.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:40.253 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:40.354 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:40.377 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:40.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:40.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:40.479 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:40.501 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:40.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:40.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:40.603 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:40.625 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:40.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:40.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:40.701 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:47:40.727 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:40.766 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:40.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:40.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:40.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:40 smithi129 ceph-mon[101316]: from='client.45587 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:47:40.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:40 smithi039 ceph-mon[106470]: from='client.45587 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:47:40.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:40 smithi039 ceph-mon[111235]: from='client.45587 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:47:40.868 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:40.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:40.890 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:40.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:40.991 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:41.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:41.017 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:41.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:41.118 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:41.141 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:41.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:41.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:41.242 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:41.263 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:41.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:41.264 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:41.365 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:41.388 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:41.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:41.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:41.490 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:41.512 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:41.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:41.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:41.613 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:41.635 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:41.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:41.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:41.716 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:41 smithi129 ceph-mon[101316]: pgmap v2529: 83 pgs: 83 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:41.737 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:41.758 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:41.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:41.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:41.779 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":2528,"stamp":"2024-09-14T04:47:37.705207+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":267758,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1698,"ondisk_log_size":1698,"up":249,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":287,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2809740,"kb_used_data":84976,"kb_used_omap":404,"kb_used_meta":2724267,"kb_avail":747151476,"statfs":{"total":767960285184,"available":765083111424,"internally_reserved":0,"allocated":87015424,"data_stored":51029761,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":414022,"internal_metadata":2789650106},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":42,"apply_latency_ms":42,"commit_latency_ns":42000000,"apply_latency_ns":42000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-4296,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-252790,"num_write_kb":-8089217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-3361,"num_bytes_recovered":-200605696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001862"},"pg_stats":[{"pgid":"3.1f","version":"72'4","reported_seq":2122,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.201032+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T04:47:33.201032+0000","last_peered":"2024-09-14T04:47:33.201032+0000","last_clean":"2024-09-14T04:47:33.201032+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T04:47:33.201032+0000","last_undegraded":"2024-09-14T04:47:33.201032+0000","last_fullsized":"2024-09-14T04:47:33.201032+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":25,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.529471+0000","last_change":"2024-09-14T04:44:57.325109+0000","last_active":"2024-09-14T04:47:12.529471+0000","last_peered":"2024-09-14T04:47:12.529471+0000","last_clean":"2024-09-14T04:47:12.529471+0000","last_became_active":"2024-09-14T04:44:57.324791+0000","last_became_peered":"2024-09-14T04:44:57.324791+0000","last_unstale":"2024-09-14T04:47:12.529471+0000","last_undegraded":"2024-09-14T04:47:12.529471+0000","last_fullsized":"2024-09-14T04:47:12.529471+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:03.593275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":2114,"reported_epoch":1418,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.872280+0000","last_change":"2024-09-14T04:07:58.504829+0000","last_active":"2024-09-14T04:47:33.872280+0000","last_peered":"2024-09-14T04:47:33.872280+0000","last_clean":"2024-09-14T04:47:33.872280+0000","last_became_active":"2024-09-14T04:07:58.502623+0000","last_became_peered":"2024-09-14T04:07:58.502623+0000","last_unstale":"2024-09-14T04:47:33.872280+0000","last_undegraded":"2024-09-14T04:47:33.872280+0000","last_fullsized":"2024-09-14T04:47:33.872280+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:59.091993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":746,"reported_epoch":1418,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.867679+0000","last_change":"2024-09-14T04:43:13.355281+0000","last_active":"2024-09-14T04:47:33.867679+0000","last_peered":"2024-09-14T04:47:33.867679+0000","last_clean":"2024-09-14T04:47:33.867679+0000","last_became_active":"2024-09-14T04:43:13.354967+0000","last_became_peered":"2024-09-14T04:43:13.354967+0000","last_unstale":"2024-09-14T04:47:33.867679+0000","last_undegraded":"2024-09-14T04:47:33.867679+0000","last_fullsized":"2024-09-14T04:47:33.867679+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:23:19.332075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1404,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.533166+0000","last_change":"2024-09-14T04:44:57.342918+0000","last_active":"2024-09-14T04:47:12.533166+0000","last_peered":"2024-09-14T04:47:12.533166+0000","last_clean":"2024-09-14T04:47:12.533166+0000","last_became_active":"2024-09-14T04:44:57.329550+0000","last_became_peered":"2024-09-14T04:44:57.329550+0000","last_unstale":"2024-09-14T04:47:12.533166+0000","last_undegraded":"2024-09-14T04:47:12.533166+0000","last_fullsized":"2024-09-14T04:47:12.533166+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:52:52.435893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":25,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.530175+0000","last_change":"2024-09-14T04:44:57.325537+0000","last_active":"2024-09-14T04:47:12.530175+0000","last_peered":"2024-09-14T04:47:12.530175+0000","last_clean":"2024-09-14T04:47:12.530175+0000","last_became_active":"2024-09-14T04:44:57.325057+0000","last_became_peered":"2024-09-14T04:44:57.325057+0000","last_unstale":"2024-09-14T04:47:12.530175+0000","last_undegraded":"2024-09-14T04:47:12.530175+0000","last_fullsized":"2024-09-14T04:47:12.530175+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:43:23.166708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1419,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.533248+0000","last_change":"2024-09-14T04:44:57.337033+0000","last_active":"2024-09-14T04:47:12.533248+0000","last_peered":"2024-09-14T04:47:12.533248+0000","last_clean":"2024-09-14T04:47:12.533248+0000","last_became_active":"2024-09-14T04:44:57.328437+0000","last_became_peered":"2024-09-14T04:44:57.328437+0000","last_unstale":"2024-09-14T04:47:12.533248+0000","last_undegraded":"2024-09-14T04:47:12.533248+0000","last_fullsized":"2024-09-14T04:47:12.533248+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.109086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":25,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.529016+0000","last_change":"2024-09-14T04:44:57.332243+0000","last_active":"2024-09-14T04:47:12.529016+0000","last_peered":"2024-09-14T04:47:12.529016+0000","last_clean":"2024-09-14T04:47:12.529016+0000","last_became_active":"2024-09-14T04:44:57.330413+0000","last_became_peered":"2024-09-14T04:44:57.330413+0000","last_unstale":"2024-09-14T04:47:12.529016+0000","last_undegraded":"2024-09-14T04:47:12.529016+0000","last_fullsized":"2024-09-14T04:47:12.529016+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:25:01.338549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":2129,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.200822+0000","last_change":"2024-09-14T04:44:57.333097+0000","last_active":"2024-09-14T04:47:33.200822+0000","last_peered":"2024-09-14T04:47:33.200822+0000","last_clean":"2024-09-14T04:47:33.200822+0000","last_became_active":"2024-09-14T04:44:57.332294+0000","last_became_peered":"2024-09-14T04:44:57.332294+0000","last_unstale":"2024-09-14T04:47:33.200822+0000","last_undegraded":"2024-09-14T04:47:33.200822+0000","last_fullsized":"2024-09-14T04:47:33.200822+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:57:44.426280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2121,"reported_epoch":1418,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.865305+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T04:47:33.865305+0000","last_peered":"2024-09-14T04:47:33.865305+0000","last_clean":"2024-09-14T04:47:33.865305+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T04:47:33.865305+0000","last_undegraded":"2024-09-14T04:47:33.865305+0000","last_fullsized":"2024-09-14T04:47:33.865305+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":25,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.530100+0000","last_change":"2024-09-14T04:44:57.325392+0000","last_active":"2024-09-14T04:47:12.530100+0000","last_peered":"2024-09-14T04:47:12.530100+0000","last_clean":"2024-09-14T04:47:12.530100+0000","last_became_active":"2024-09-14T04:44:57.324624+0000","last_became_peered":"2024-09-14T04:44:57.324624+0000","last_unstale":"2024-09-14T04:47:12.530100+0000","last_undegraded":"2024-09-14T04:47:12.530100+0000","last_fullsized":"2024-09-14T04:47:12.530100+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:33:18.810199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2089,"reported_epoch":1418,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.838236+0000","last_change":"2024-09-14T04:44:57.329113+0000","last_active":"2024-09-14T04:47:33.838236+0000","last_peered":"2024-09-14T04:47:33.838236+0000","last_clean":"2024-09-14T04:47:33.838236+0000","last_became_active":"2024-09-14T04:44:57.328727+0000","last_became_peered":"2024-09-14T04:44:57.328727+0000","last_unstale":"2024-09-14T04:47:33.838236+0000","last_undegraded":"2024-09-14T04:47:33.838236+0000","last_fullsized":"2024-09-14T04:47:33.838236+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:34.939874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":2106,"reported_epoch":1418,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.836275+0000","last_change":"2024-09-14T04:44:57.329277+0000","last_active":"2024-09-14T04:47:33.836275+0000","last_peered":"2024-09-14T04:47:33.836275+0000","last_clean":"2024-09-14T04:47:33.836275+0000","last_became_active":"2024-09-14T04:44:57.328804+0000","last_became_peered":"2024-09-14T04:44:57.328804+0000","last_unstale":"2024-09-14T04:47:33.836275+0000","last_undegraded":"2024-09-14T04:47:33.836275+0000","last_fullsized":"2024-09-14T04:47:33.836275+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:42:46.055240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2117,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.200968+0000","last_change":"2024-09-14T04:07:58.561490+0000","last_active":"2024-09-14T04:47:33.200968+0000","last_peered":"2024-09-14T04:47:33.200968+0000","last_clean":"2024-09-14T04:47:33.200968+0000","last_became_active":"2024-09-14T04:07:58.559841+0000","last_became_peered":"2024-09-14T04:07:58.559841+0000","last_unstale":"2024-09-14T04:47:33.200968+0000","last_undegraded":"2024-09-14T04:47:33.200968+0000","last_fullsized":"2024-09-14T04:47:33.200968+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:27:03.795368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2100,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.199102+0000","last_change":"2024-09-14T04:43:13.357092+0000","last_active":"2024-09-14T04:47:33.199102+0000","last_peered":"2024-09-14T04:47:33.199102+0000","last_clean":"2024-09-14T04:47:33.199102+0000","last_became_active":"2024-09-14T04:43:13.356694+0000","last_became_peered":"2024-09-14T04:43:13.356694+0000","last_unstale":"2024-09-14T04:47:33.199102+0000","last_undegraded":"2024-09-14T04:47:33.199102+0000","last_fullsized":"2024-09-14T04:47:33.199102+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:11.167574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1397,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:13.346568+0000","last_change":"2024-09-14T04:07:21.720797+0000","last_active":"2024-09-14T04:47:13.346568+0000","last_peered":"2024-09-14T04:47:13.346568+0000","last_clean":"2024-09-14T04:47:13.346568+0000","last_became_active":"2024-09-14T04:07:21.719331+0000","last_became_peered":"2024-09-14T04:07:21.719331+0000","last_unstale":"2024-09-14T04:47:13.346568+0000","last_undegraded":"2024-09-14T04:47:13.346568+0000","last_fullsized":"2024-09-14T04:47:13.346568+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:35.404023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":44,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.532926+0000","last_change":"2024-09-14T04:43:11.641514+0000","last_active":"2024-09-14T04:47:12.532926+0000","last_peered":"2024-09-14T04:47:12.532926+0000","last_clean":"2024-09-14T04:47:12.532926+0000","last_became_active":"2024-09-14T04:43:11.641151+0000","last_became_peered":"2024-09-14T04:43:11.641151+0000","last_unstale":"2024-09-14T04:47:12.532926+0000","last_undegraded":"2024-09-14T04:47:12.532926+0000","last_fullsized":"2024-09-14T04:47:12.532926+0000","mapping_epoch":1384,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:06:45.045430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2112,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.201120+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T04:47:33.201120+0000","last_peered":"2024-09-14T04:47:33.201120+0000","last_clean":"2024-09-14T04:47:33.201120+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T04:47:33.201120+0000","last_undegraded":"2024-09-14T04:47:33.201120+0000","last_fullsized":"2024-09-14T04:47:33.201120+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":2095,"reported_epoch":1418,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.872387+0000","last_change":"2024-09-14T04:07:58.507682+0000","last_active":"2024-09-14T04:47:33.872387+0000","last_peered":"2024-09-14T04:47:33.872387+0000","last_clean":"2024-09-14T04:47:33.872387+0000","last_became_active":"2024-09-14T04:07:58.507365+0000","last_became_peered":"2024-09-14T04:07:58.507365+0000","last_unstale":"2024-09-14T04:47:33.872387+0000","last_undegraded":"2024-09-14T04:47:33.872387+0000","last_fullsized":"2024-09-14T04:47:33.872387+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:14:47.294950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1403,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.533460+0000","last_change":"2024-09-14T04:42:49.938505+0000","last_active":"2024-09-14T04:47:12.533460+0000","last_peered":"2024-09-14T04:47:12.533460+0000","last_clean":"2024-09-14T04:47:12.533460+0000","last_became_active":"2024-09-14T04:42:49.938180+0000","last_became_peered":"2024-09-14T04:42:49.938180+0000","last_unstale":"2024-09-14T04:47:12.533460+0000","last_undegraded":"2024-09-14T04:47:12.533460+0000","last_fullsized":"2024-09-14T04:47:12.533460+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:05.929479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1421,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.400955+0000","last_change":"2024-09-14T04:44:57.346582+0000","last_active":"2024-09-14T04:46:50.400955+0000","last_peered":"2024-09-14T04:46:50.400955+0000","last_clean":"2024-09-14T04:46:50.400955+0000","last_became_active":"2024-09-14T04:44:57.330446+0000","last_became_peered":"2024-09-14T04:44:57.330446+0000","last_unstale":"2024-09-14T04:46:50.400955+0000","last_undegraded":"2024-09-14T04:46:50.400955+0000","last_fullsized":"2024-09-14T04:46:50.400955+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:29.238585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":55,"reported_epoch":1418,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.833316+0000","last_change":"2024-09-14T04:44:14.480667+0000","last_active":"2024-09-14T04:47:33.833316+0000","last_peered":"2024-09-14T04:47:33.833316+0000","last_clean":"2024-09-14T04:47:33.833316+0000","last_became_active":"2024-09-14T04:44:14.480365+0000","last_became_peered":"2024-09-14T04:44:14.480365+0000","last_unstale":"2024-09-14T04:47:33.833316+0000","last_undegraded":"2024-09-14T04:47:33.833316+0000","last_fullsized":"2024-09-14T04:47:33.833316+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:00.611771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":35,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.529370+0000","last_change":"2024-09-14T04:44:57.341426+0000","last_active":"2024-09-14T04:47:12.529370+0000","last_peered":"2024-09-14T04:47:12.529370+0000","last_clean":"2024-09-14T04:47:12.529370+0000","last_became_active":"2024-09-14T04:44:57.328763+0000","last_became_peered":"2024-09-14T04:44:57.328763+0000","last_unstale":"2024-09-14T04:47:12.529370+0000","last_undegraded":"2024-09-14T04:47:12.529370+0000","last_fullsized":"2024-09-14T04:47:12.529370+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:34:40.066761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":25,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.529158+0000","last_change":"2024-09-14T04:44:57.331337+0000","last_active":"2024-09-14T04:47:12.529158+0000","last_peered":"2024-09-14T04:47:12.529158+0000","last_clean":"2024-09-14T04:47:12.529158+0000","last_became_active":"2024-09-14T04:44:57.329336+0000","last_became_peered":"2024-09-14T04:44:57.329336+0000","last_unstale":"2024-09-14T04:47:12.529158+0000","last_undegraded":"2024-09-14T04:47:12.529158+0000","last_fullsized":"2024-09-14T04:47:12.529158+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:52:35.980869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"349'149","reported_seq":783,"reported_epoch":1418,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.898187+0000","last_change":"2024-09-14T04:41:37.755174+0000","last_active":"2024-09-14T04:47:33.898187+0000","last_peered":"2024-09-14T04:47:33.898187+0000","last_clean":"2024-09-14T04:47:33.898187+0000","last_became_active":"2024-09-14T04:41:36.930263+0000","last_became_peered":"2024-09-14T04:41:36.930263+0000","last_unstale":"2024-09-14T04:47:33.898187+0000","last_undegraded":"2024-09-14T04:47:33.898187+0000","last_fullsized":"2024-09-14T04:47:33.898187+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.755105+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.755105+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:27:03.161713+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":25,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:47:02.688408+0000","last_change":"2024-09-14T04:44:57.329870+0000","last_active":"2024-09-14T04:47:02.688408+0000","last_peered":"2024-09-14T04:47:02.688408+0000","last_clean":"2024-09-14T04:47:02.688408+0000","last_became_active":"2024-09-14T04:44:57.329353+0000","last_became_peered":"2024-09-14T04:44:57.329353+0000","last_unstale":"2024-09-14T04:47:02.688408+0000","last_undegraded":"2024-09-14T04:47:02.688408+0000","last_fullsized":"2024-09-14T04:47:02.688408+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:46.432181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1400,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.531486+0000","last_change":"2024-09-14T04:07:21.721242+0000","last_active":"2024-09-14T04:47:12.531486+0000","last_peered":"2024-09-14T04:47:12.531486+0000","last_clean":"2024-09-14T04:47:12.531486+0000","last_became_active":"2024-09-14T04:07:21.720084+0000","last_became_peered":"2024-09-14T04:07:21.720084+0000","last_unstale":"2024-09-14T04:47:12.531486+0000","last_undegraded":"2024-09-14T04:47:12.531486+0000","last_fullsized":"2024-09-14T04:47:12.531486+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:58:41.308643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":2150,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.200884+0000","last_change":"2024-09-14T04:44:57.344614+0000","last_active":"2024-09-14T04:47:33.200884+0000","last_peered":"2024-09-14T04:47:33.200884+0000","last_clean":"2024-09-14T04:47:33.200884+0000","last_became_active":"2024-09-14T04:44:57.332256+0000","last_became_peered":"2024-09-14T04:44:57.332256+0000","last_unstale":"2024-09-14T04:47:33.200884+0000","last_undegraded":"2024-09-14T04:47:33.200884+0000","last_fullsized":"2024-09-14T04:47:33.200884+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:23.438417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"349'149","reported_seq":783,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:47:07.698697+0000","last_change":"2024-09-14T04:45:00.331854+0000","last_active":"2024-09-14T04:47:07.698697+0000","last_peered":"2024-09-14T04:47:07.698697+0000","last_clean":"2024-09-14T04:47:07.698697+0000","last_became_active":"2024-09-14T04:45:00.331460+0000","last_became_peered":"2024-09-14T04:45:00.331460+0000","last_unstale":"2024-09-14T04:47:07.698697+0000","last_undegraded":"2024-09-14T04:47:07.698697+0000","last_fullsized":"2024-09-14T04:47:07.698697+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.567535+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.567535+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:45.095729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":2088,"reported_epoch":1418,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.873806+0000","last_change":"2024-09-14T04:07:58.555446+0000","last_active":"2024-09-14T04:47:33.873806+0000","last_peered":"2024-09-14T04:47:33.873806+0000","last_clean":"2024-09-14T04:47:33.873806+0000","last_became_active":"2024-09-14T04:07:58.548355+0000","last_became_peered":"2024-09-14T04:07:58.548355+0000","last_unstale":"2024-09-14T04:47:33.873806+0000","last_undegraded":"2024-09-14T04:47:33.873806+0000","last_fullsized":"2024-09-14T04:47:33.873806+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:56:28.426094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":107,"reported_epoch":1418,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.835187+0000","last_change":"2024-09-14T04:44:57.330746+0000","last_active":"2024-09-14T04:47:33.835187+0000","last_peered":"2024-09-14T04:47:33.835187+0000","last_clean":"2024-09-14T04:47:33.835187+0000","last_became_active":"2024-09-14T04:44:57.330327+0000","last_became_peered":"2024-09-14T04:44:57.330327+0000","last_unstale":"2024-09-14T04:47:33.835187+0000","last_undegraded":"2024-09-14T04:47:33.835187+0000","last_fullsized":"2024-09-14T04:47:33.835187+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:53.122771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":65,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.550119+0000","last_change":"2024-09-14T04:43:13.355296+0000","last_active":"2024-09-14T04:47:12.550119+0000","last_peered":"2024-09-14T04:47:12.550119+0000","last_clean":"2024-09-14T04:47:12.550119+0000","last_became_active":"2024-09-14T04:43:13.354950+0000","last_became_peered":"2024-09-14T04:43:13.354950+0000","last_unstale":"2024-09-14T04:47:12.550119+0000","last_undegraded":"2024-09-14T04:47:12.550119+0000","last_fullsized":"2024-09-14T04:47:12.550119+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:20:08.968818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"349'149","reported_seq":722,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.531238+0000","last_change":"2024-09-14T04:41:39.471393+0000","last_active":"2024-09-14T04:47:12.531238+0000","last_peered":"2024-09-14T04:47:12.531238+0000","last_clean":"2024-09-14T04:47:12.531238+0000","last_became_active":"2024-09-14T04:41:36.931768+0000","last_became_peered":"2024-09-14T04:41:36.931768+0000","last_unstale":"2024-09-14T04:47:12.531238+0000","last_undegraded":"2024-09-14T04:47:12.531238+0000","last_fullsized":"2024-09-14T04:47:12.531238+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.471248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.471248+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:56.210375+0000","scrub_duration":0.021758344999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2105,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.201127+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T04:47:33.201127+0000","last_peered":"2024-09-14T04:47:33.201127+0000","last_clean":"2024-09-14T04:47:33.201127+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T04:47:33.201127+0000","last_undegraded":"2024-09-14T04:47:33.201127+0000","last_fullsized":"2024-09-14T04:47:33.201127+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1431,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.401139+0000","last_change":"2024-09-14T04:44:57.331436+0000","last_active":"2024-09-14T04:46:50.401139+0000","last_peered":"2024-09-14T04:46:50.401139+0000","last_clean":"2024-09-14T04:46:50.401139+0000","last_became_active":"2024-09-14T04:44:57.330957+0000","last_became_peered":"2024-09-14T04:44:57.330957+0000","last_unstale":"2024-09-14T04:46:50.401139+0000","last_undegraded":"2024-09-14T04:46:50.401139+0000","last_fullsized":"2024-09-14T04:46:50.401139+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:09.516688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":100,"reported_epoch":1418,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.838176+0000","last_change":"2024-09-14T04:42:49.940968+0000","last_active":"2024-09-14T04:47:33.838176+0000","last_peered":"2024-09-14T04:47:33.838176+0000","last_clean":"2024-09-14T04:47:33.838176+0000","last_became_active":"2024-09-14T04:42:49.940667+0000","last_became_peered":"2024-09-14T04:42:49.940667+0000","last_unstale":"2024-09-14T04:47:33.838176+0000","last_undegraded":"2024-09-14T04:47:33.838176+0000","last_fullsized":"2024-09-14T04:47:33.838176+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:52:49.623815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"349'149","reported_seq":722,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.531248+0000","last_change":"2024-09-14T04:41:38.479980+0000","last_active":"2024-09-14T04:47:12.531248+0000","last_peered":"2024-09-14T04:47:12.531248+0000","last_clean":"2024-09-14T04:47:12.531248+0000","last_became_active":"2024-09-14T04:41:36.933392+0000","last_became_peered":"2024-09-14T04:41:36.933392+0000","last_unstale":"2024-09-14T04:47:12.531248+0000","last_undegraded":"2024-09-14T04:47:12.531248+0000","last_fullsized":"2024-09-14T04:47:12.531248+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:38.479887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:38.479887+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:01:26.102279+0000","scrub_duration":0.0062728319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":62,"reported_epoch":1418,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.833226+0000","last_change":"2024-09-14T04:44:57.329983+0000","last_active":"2024-09-14T04:47:33.833226+0000","last_peered":"2024-09-14T04:47:33.833226+0000","last_clean":"2024-09-14T04:47:33.833226+0000","last_became_active":"2024-09-14T04:44:57.329597+0000","last_became_peered":"2024-09-14T04:44:57.329597+0000","last_unstale":"2024-09-14T04:47:33.833226+0000","last_undegraded":"2024-09-14T04:47:33.833226+0000","last_fullsized":"2024-09-14T04:47:33.833226+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:45.718039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":2142,"reported_epoch":1418,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.838415+0000","last_change":"2024-09-14T04:06:46.930482+0000","last_active":"2024-09-14T04:47:33.838415+0000","last_peered":"2024-09-14T04:47:33.838415+0000","last_clean":"2024-09-14T04:47:33.838415+0000","last_became_active":"2024-09-14T04:06:46.927372+0000","last_became_peered":"2024-09-14T04:06:46.927372+0000","last_unstale":"2024-09-14T04:47:33.838415+0000","last_undegraded":"2024-09-14T04:47:33.838415+0000","last_fullsized":"2024-09-14T04:47:33.838415+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:07:22.159128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1405,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:23.345777+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T04:47:23.345777+0000","last_peered":"2024-09-14T04:47:23.345777+0000","last_clean":"2024-09-14T04:47:23.345777+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T04:47:23.345777+0000","last_undegraded":"2024-09-14T04:47:23.345777+0000","last_fullsized":"2024-09-14T04:47:23.345777+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"349'149","reported_seq":714,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.533292+0000","last_change":"2024-09-14T04:41:39.012977+0000","last_active":"2024-09-14T04:47:12.533292+0000","last_peered":"2024-09-14T04:47:12.533292+0000","last_clean":"2024-09-14T04:47:12.533292+0000","last_became_active":"2024-09-14T04:41:36.932552+0000","last_became_peered":"2024-09-14T04:41:36.932552+0000","last_unstale":"2024-09-14T04:47:12.533292+0000","last_undegraded":"2024-09-14T04:47:12.533292+0000","last_fullsized":"2024-09-14T04:47:12.533292+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.012879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.012879+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:16:40.080262+0000","scrub_duration":0.0051721620000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":100,"reported_epoch":1418,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.838137+0000","last_change":"2024-09-14T04:42:49.945069+0000","last_active":"2024-09-14T04:47:33.838137+0000","last_peered":"2024-09-14T04:47:33.838137+0000","last_clean":"2024-09-14T04:47:33.838137+0000","last_became_active":"2024-09-14T04:42:49.943527+0000","last_became_peered":"2024-09-14T04:42:49.943527+0000","last_unstale":"2024-09-14T04:47:33.838137+0000","last_undegraded":"2024-09-14T04:47:33.838137+0000","last_fullsized":"2024-09-14T04:47:33.838137+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:39:10.524631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":25,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:47:02.688472+0000","last_change":"2024-09-14T04:44:57.329652+0000","last_active":"2024-09-14T04:47:02.688472+0000","last_peered":"2024-09-14T04:47:02.688472+0000","last_clean":"2024-09-14T04:47:02.688472+0000","last_became_active":"2024-09-14T04:44:57.328888+0000","last_became_peered":"2024-09-14T04:44:57.328888+0000","last_unstale":"2024-09-14T04:47:02.688472+0000","last_undegraded":"2024-09-14T04:47:02.688472+0000","last_fullsized":"2024-09-14T04:47:02.688472+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:10.170068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1403,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.533668+0000","last_change":"2024-09-14T04:06:46.875340+0000","last_active":"2024-09-14T04:47:12.533668+0000","last_peered":"2024-09-14T04:47:12.533668+0000","last_clean":"2024-09-14T04:47:12.533668+0000","last_became_active":"2024-09-14T04:06:46.874552+0000","last_became_peered":"2024-09-14T04:06:46.874552+0000","last_unstale":"2024-09-14T04:47:12.533668+0000","last_undegraded":"2024-09-14T04:47:12.533668+0000","last_fullsized":"2024-09-14T04:47:12.533668+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:52.257357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"349'149","reported_seq":1783,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.400875+0000","last_change":"2024-09-14T04:45:00.346627+0000","last_active":"2024-09-14T04:46:50.400875+0000","last_peered":"2024-09-14T04:46:50.400875+0000","last_clean":"2024-09-14T04:46:50.400875+0000","last_became_active":"2024-09-14T04:45:00.346286+0000","last_became_peered":"2024-09-14T04:45:00.346286+0000","last_unstale":"2024-09-14T04:46:50.400875+0000","last_undegraded":"2024-09-14T04:46:50.400875+0000","last_fullsized":"2024-09-14T04:46:50.400875+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:50.784648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:50.784648+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:07:58.441372+0000","scrub_duration":0.01141873,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":55,"reported_epoch":1418,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.838536+0000","last_change":"2024-09-14T04:44:14.489235+0000","last_active":"2024-09-14T04:47:33.838536+0000","last_peered":"2024-09-14T04:47:33.838536+0000","last_clean":"2024-09-14T04:47:33.838536+0000","last_became_active":"2024-09-14T04:44:14.488950+0000","last_became_peered":"2024-09-14T04:44:14.488950+0000","last_unstale":"2024-09-14T04:47:33.838536+0000","last_undegraded":"2024-09-14T04:47:33.838536+0000","last_fullsized":"2024-09-14T04:47:33.838536+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:54.441144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":2126,"reported_epoch":1418,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.833059+0000","last_change":"2024-09-14T04:07:21.685415+0000","last_active":"2024-09-14T04:47:33.833059+0000","last_peered":"2024-09-14T04:47:33.833059+0000","last_clean":"2024-09-14T04:47:33.833059+0000","last_became_active":"2024-09-14T04:07:21.683504+0000","last_became_peered":"2024-09-14T04:07:21.683504+0000","last_unstale":"2024-09-14T04:47:33.833059+0000","last_undegraded":"2024-09-14T04:47:33.833059+0000","last_fullsized":"2024-09-14T04:47:33.833059+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:21:08.806291+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2128,"reported_epoch":1418,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.867766+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T04:47:33.867766+0000","last_peered":"2024-09-14T04:47:33.867766+0000","last_clean":"2024-09-14T04:47:33.867766+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T04:47:33.867766+0000","last_undegraded":"2024-09-14T04:47:33.867766+0000","last_fullsized":"2024-09-14T04:47:33.867766+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"349'149","reported_seq":803,"reported_epoch":1418,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.837969+0000","last_change":"2024-09-14T04:45:00.342826+0000","last_active":"2024-09-14T04:47:33.837969+0000","last_peered":"2024-09-14T04:47:33.837969+0000","last_clean":"2024-09-14T04:47:33.837969+0000","last_became_active":"2024-09-14T04:45:00.342013+0000","last_became_peered":"2024-09-14T04:45:00.342013+0000","last_unstale":"2024-09-14T04:47:33.837969+0000","last_undegraded":"2024-09-14T04:47:33.837969+0000","last_fullsized":"2024-09-14T04:47:33.837969+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.326129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.326129+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:54:28.021603+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2127,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.201011+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T04:47:33.201011+0000","last_peered":"2024-09-14T04:47:33.201011+0000","last_clean":"2024-09-14T04:47:33.201011+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T04:47:33.201011+0000","last_undegraded":"2024-09-14T04:47:33.201011+0000","last_fullsized":"2024-09-14T04:47:33.201011+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2133,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.200411+0000","last_change":"2024-09-14T04:44:57.335783+0000","last_active":"2024-09-14T04:47:33.200411+0000","last_peered":"2024-09-14T04:47:33.200411+0000","last_clean":"2024-09-14T04:47:33.200411+0000","last_became_active":"2024-09-14T04:44:57.332780+0000","last_became_peered":"2024-09-14T04:44:57.332780+0000","last_unstale":"2024-09-14T04:47:33.200411+0000","last_undegraded":"2024-09-14T04:47:33.200411+0000","last_fullsized":"2024-09-14T04:47:33.200411+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:57.457593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1432,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.400758+0000","last_change":"2024-09-14T04:42:49.939790+0000","last_active":"2024-09-14T04:46:50.400758+0000","last_peered":"2024-09-14T04:46:50.400758+0000","last_clean":"2024-09-14T04:46:50.400758+0000","last_became_active":"2024-09-14T04:42:49.939591+0000","last_became_peered":"2024-09-14T04:42:49.939591+0000","last_unstale":"2024-09-14T04:46:50.400758+0000","last_undegraded":"2024-09-14T04:46:50.400758+0000","last_fullsized":"2024-09-14T04:46:50.400758+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:23:16.790593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1397,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:23.351230+0000","last_change":"2024-09-14T04:44:57.326008+0000","last_active":"2024-09-14T04:47:23.351230+0000","last_peered":"2024-09-14T04:47:23.351230+0000","last_clean":"2024-09-14T04:47:23.351230+0000","last_became_active":"2024-09-14T04:44:57.325617+0000","last_became_peered":"2024-09-14T04:44:57.325617+0000","last_unstale":"2024-09-14T04:47:23.351230+0000","last_undegraded":"2024-09-14T04:47:23.351230+0000","last_fullsized":"2024-09-14T04:47:23.351230+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:15.637508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":686,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:23.345735+0000","last_change":"2024-09-14T04:43:13.354846+0000","last_active":"2024-09-14T04:47:23.345735+0000","last_peered":"2024-09-14T04:47:23.345735+0000","last_clean":"2024-09-14T04:47:23.345735+0000","last_became_active":"2024-09-14T04:43:13.354515+0000","last_became_peered":"2024-09-14T04:43:13.354515+0000","last_unstale":"2024-09-14T04:47:23.345735+0000","last_undegraded":"2024-09-14T04:47:23.345735+0000","last_fullsized":"2024-09-14T04:47:23.345735+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:17.889341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1402,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.533206+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T04:47:12.533206+0000","last_peered":"2024-09-14T04:47:12.533206+0000","last_clean":"2024-09-14T04:47:12.533206+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T04:47:12.533206+0000","last_undegraded":"2024-09-14T04:47:12.533206+0000","last_fullsized":"2024-09-14T04:47:12.533206+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":1739,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.548968+0000","last_change":"2024-09-14T04:41:30.640651+0000","last_active":"2024-09-14T04:47:12.548968+0000","last_peered":"2024-09-14T04:47:12.548968+0000","last_clean":"2024-09-14T04:47:12.548968+0000","last_became_active":"2024-09-14T04:41:29.532466+0000","last_became_peered":"2024-09-14T04:41:29.532466+0000","last_unstale":"2024-09-14T04:47:12.548968+0000","last_undegraded":"2024-09-14T04:47:12.548968+0000","last_fullsized":"2024-09-14T04:47:12.548968+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":0,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:30.640520+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:30.640520+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:27.776724+0000","scrub_duration":0.0028653139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"349'149","reported_seq":714,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.533731+0000","last_change":"2024-09-14T04:41:39.978080+0000","last_active":"2024-09-14T04:47:12.533731+0000","last_peered":"2024-09-14T04:47:12.533731+0000","last_clean":"2024-09-14T04:47:12.533731+0000","last_became_active":"2024-09-14T04:41:36.929279+0000","last_became_peered":"2024-09-14T04:41:36.929279+0000","last_unstale":"2024-09-14T04:47:12.533731+0000","last_undegraded":"2024-09-14T04:47:12.533731+0000","last_fullsized":"2024-09-14T04:47:12.533731+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.977999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.977999+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:44:06.383827+0000","scrub_duration":0.00192122,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":2108,"reported_epoch":1418,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.838543+0000","last_change":"2024-09-14T04:44:57.330745+0000","last_active":"2024-09-14T04:47:33.838543+0000","last_peered":"2024-09-14T04:47:33.838543+0000","last_clean":"2024-09-14T04:47:33.838543+0000","last_became_active":"2024-09-14T04:44:57.330261+0000","last_became_peered":"2024-09-14T04:44:57.330261+0000","last_unstale":"2024-09-14T04:47:33.838543+0000","last_undegraded":"2024-09-14T04:47:33.838543+0000","last_fullsized":"2024-09-14T04:47:33.838543+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:24:08.870976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2112,"reported_epoch":1418,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.873649+0000","last_change":"2024-09-14T04:42:49.941397+0000","last_active":"2024-09-14T04:47:33.873649+0000","last_peered":"2024-09-14T04:47:33.873649+0000","last_clean":"2024-09-14T04:47:33.873649+0000","last_became_active":"2024-09-14T04:42:49.941066+0000","last_became_peered":"2024-09-14T04:42:49.941066+0000","last_unstale":"2024-09-14T04:47:33.873649+0000","last_undegraded":"2024-09-14T04:47:33.873649+0000","last_fullsized":"2024-09-14T04:47:33.873649+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:30.777822+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":25,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.531059+0000","last_change":"2024-09-14T04:44:57.326724+0000","last_active":"2024-09-14T04:47:12.531059+0000","last_peered":"2024-09-14T04:47:12.531059+0000","last_clean":"2024-09-14T04:47:12.531059+0000","last_became_active":"2024-09-14T04:44:57.326191+0000","last_became_peered":"2024-09-14T04:44:57.326191+0000","last_unstale":"2024-09-14T04:47:12.531059+0000","last_undegraded":"2024-09-14T04:47:12.531059+0000","last_fullsized":"2024-09-14T04:47:12.531059+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:46.060269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":25,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:47:07.698558+0000","last_change":"2024-09-14T04:44:57.328528+0000","last_active":"2024-09-14T04:47:07.698558+0000","last_peered":"2024-09-14T04:47:07.698558+0000","last_clean":"2024-09-14T04:47:07.698558+0000","last_became_active":"2024-09-14T04:44:57.327739+0000","last_became_peered":"2024-09-14T04:44:57.327739+0000","last_unstale":"2024-09-14T04:47:07.698558+0000","last_undegraded":"2024-09-14T04:47:07.698558+0000","last_fullsized":"2024-09-14T04:47:07.698558+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:04:58.294690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"349'149","reported_seq":746,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:13.338688+0000","last_change":"2024-09-14T04:42:53.013201+0000","last_active":"2024-09-14T04:47:13.338688+0000","last_peered":"2024-09-14T04:47:13.338688+0000","last_clean":"2024-09-14T04:47:13.338688+0000","last_became_active":"2024-09-14T04:42:53.012818+0000","last_became_peered":"2024-09-14T04:42:53.012818+0000","last_unstale":"2024-09-14T04:47:13.338688+0000","last_undegraded":"2024-09-14T04:47:13.338688+0000","last_fullsized":"2024-09-14T04:47:13.338688+0000","mapping_epoch":1377,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1378,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:56.738636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:56.738636+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:39:19.056067+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":25,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.530119+0000","last_change":"2024-09-14T04:44:57.323436+0000","last_active":"2024-09-14T04:47:12.530119+0000","last_peered":"2024-09-14T04:47:12.530119+0000","last_clean":"2024-09-14T04:47:12.530119+0000","last_became_active":"2024-09-14T04:44:57.323022+0000","last_became_peered":"2024-09-14T04:44:57.323022+0000","last_unstale":"2024-09-14T04:47:12.530119+0000","last_undegraded":"2024-09-14T04:47:12.530119+0000","last_fullsized":"2024-09-14T04:47:12.530119+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:22.775261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1432,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:23.351205+0000","last_change":"2024-09-14T04:44:57.322357+0000","last_active":"2024-09-14T04:47:23.351205+0000","last_peered":"2024-09-14T04:47:23.351205+0000","last_clean":"2024-09-14T04:47:23.351205+0000","last_became_active":"2024-09-14T04:44:57.321990+0000","last_became_peered":"2024-09-14T04:44:57.321990+0000","last_unstale":"2024-09-14T04:47:23.351205+0000","last_undegraded":"2024-09-14T04:47:23.351205+0000","last_fullsized":"2024-09-14T04:47:23.351205+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:09.658008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"349'149","reported_seq":1755,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.400561+0000","last_change":"2024-09-14T04:41:53.858076+0000","last_active":"2024-09-14T04:46:50.400561+0000","last_peered":"2024-09-14T04:46:50.400561+0000","last_clean":"2024-09-14T04:46:50.400561+0000","last_became_active":"2024-09-14T04:41:36.931585+0000","last_became_peered":"2024-09-14T04:41:36.931585+0000","last_unstale":"2024-09-14T04:46:50.400561+0000","last_undegraded":"2024-09-14T04:46:50.400561+0000","last_fullsized":"2024-09-14T04:46:50.400561+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:53.857924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:53.857924+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:10:20.658986+0000","scrub_duration":0.0022355919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":25,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.529297+0000","last_change":"2024-09-14T04:44:57.329198+0000","last_active":"2024-09-14T04:47:12.529297+0000","last_peered":"2024-09-14T04:47:12.529297+0000","last_clean":"2024-09-14T04:47:12.529297+0000","last_became_active":"2024-09-14T04:44:57.327754+0000","last_became_peered":"2024-09-14T04:44:57.327754+0000","last_unstale":"2024-09-14T04:47:12.529297+0000","last_undegraded":"2024-09-14T04:47:12.529297+0000","last_fullsized":"2024-09-14T04:47:12.529297+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:10:27.944476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":110,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.201046+0000","last_change":"2024-09-14T04:44:57.348687+0000","last_active":"2024-09-14T04:47:33.201046+0000","last_peered":"2024-09-14T04:47:33.201046+0000","last_clean":"2024-09-14T04:47:33.201046+0000","last_became_active":"2024-09-14T04:44:57.332845+0000","last_became_peered":"2024-09-14T04:44:57.332845+0000","last_unstale":"2024-09-14T04:47:33.201046+0000","last_undegraded":"2024-09-14T04:47:33.201046+0000","last_fullsized":"2024-09-14T04:47:33.201046+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:12:18.762737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2133,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.200822+0000","last_change":"2024-09-14T04:44:57.333468+0000","last_active":"2024-09-14T04:47:33.200822+0000","last_peered":"2024-09-14T04:47:33.200822+0000","last_clean":"2024-09-14T04:47:33.200822+0000","last_became_active":"2024-09-14T04:44:57.332626+0000","last_became_peered":"2024-09-14T04:44:57.332626+0000","last_unstale":"2024-09-14T04:47:33.200822+0000","last_undegraded":"2024-09-14T04:47:33.200822+0000","last_fullsized":"2024-09-14T04:47:33.200822+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:57:26.366438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1388,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.533319+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T04:47:12.533319+0000","last_peered":"2024-09-14T04:47:12.533319+0000","last_clean":"2024-09-14T04:47:12.533319+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T04:47:12.533319+0000","last_undegraded":"2024-09-14T04:47:12.533319+0000","last_fullsized":"2024-09-14T04:47:12.533319+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1387,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:23.351195+0000","last_change":"2024-09-14T04:06:46.869868+0000","last_active":"2024-09-14T04:47:23.351195+0000","last_peered":"2024-09-14T04:47:23.351195+0000","last_clean":"2024-09-14T04:47:23.351195+0000","last_became_active":"2024-09-14T04:06:46.869126+0000","last_became_peered":"2024-09-14T04:06:46.869126+0000","last_unstale":"2024-09-14T04:47:23.351195+0000","last_undegraded":"2024-09-14T04:47:23.351195+0000","last_fullsized":"2024-09-14T04:47:23.351195+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:41.150138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1370,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:47:08.011810+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T04:47:08.011810+0000","last_peered":"2024-09-14T04:47:08.011810+0000","last_clean":"2024-09-14T04:47:08.011810+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T04:47:08.011810+0000","last_undegraded":"2024-09-14T04:47:08.011810+0000","last_fullsized":"2024-09-14T04:47:08.011810+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":25,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:47:07.698588+0000","last_change":"2024-09-14T04:44:57.332112+0000","last_active":"2024-09-14T04:47:07.698588+0000","last_peered":"2024-09-14T04:47:07.698588+0000","last_clean":"2024-09-14T04:47:07.698588+0000","last_became_active":"2024-09-14T04:44:57.331575+0000","last_became_peered":"2024-09-14T04:44:57.331575+0000","last_unstale":"2024-09-14T04:47:07.698588+0000","last_undegraded":"2024-09-14T04:47:07.698588+0000","last_fullsized":"2024-09-14T04:47:07.698588+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:50:00.391283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1418,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:47:08.011847+0000","last_change":"2024-09-14T04:44:57.351460+0000","last_active":"2024-09-14T04:47:08.011847+0000","last_peered":"2024-09-14T04:47:08.011847+0000","last_clean":"2024-09-14T04:47:08.011847+0000","last_became_active":"2024-09-14T04:44:57.334662+0000","last_became_peered":"2024-09-14T04:44:57.334662+0000","last_unstale":"2024-09-14T04:47:08.011847+0000","last_undegraded":"2024-09-14T04:47:08.011847+0000","last_fullsized":"2024-09-14T04:47:08.011847+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:02.399062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1373,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:18.345134+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T04:47:18.345134+0000","last_peered":"2024-09-14T04:47:18.345134+0000","last_clean":"2024-09-14T04:47:18.345134+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T04:47:18.345134+0000","last_undegraded":"2024-09-14T04:47:18.345134+0000","last_fullsized":"2024-09-14T04:47:18.345134+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1430,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.550046+0000","last_change":"2024-09-14T04:44:57.349690+0000","last_active":"2024-09-14T04:47:12.550046+0000","last_peered":"2024-09-14T04:47:12.550046+0000","last_clean":"2024-09-14T04:47:12.550046+0000","last_became_active":"2024-09-14T04:44:57.333579+0000","last_became_peered":"2024-09-14T04:44:57.333579+0000","last_unstale":"2024-09-14T04:47:12.550046+0000","last_undegraded":"2024-09-14T04:47:12.550046+0000","last_fullsized":"2024-09-14T04:47:12.550046+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:17:55.265141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1432,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.531082+0000","last_change":"2024-09-14T04:42:49.960838+0000","last_active":"2024-09-14T04:47:12.531082+0000","last_peered":"2024-09-14T04:47:12.531082+0000","last_clean":"2024-09-14T04:47:12.531082+0000","last_became_active":"2024-09-14T04:42:49.954736+0000","last_became_peered":"2024-09-14T04:42:49.954736+0000","last_unstale":"2024-09-14T04:47:12.531082+0000","last_undegraded":"2024-09-14T04:47:12.531082+0000","last_fullsized":"2024-09-14T04:47:12.531082+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:41:13.540983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":2094,"reported_epoch":1418,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.899284+0000","last_change":"2024-09-14T04:07:58.565442+0000","last_active":"2024-09-14T04:47:33.899284+0000","last_peered":"2024-09-14T04:47:33.899284+0000","last_clean":"2024-09-14T04:47:33.899284+0000","last_became_active":"2024-09-14T04:07:58.552289+0000","last_became_peered":"2024-09-14T04:07:58.552289+0000","last_unstale":"2024-09-14T04:47:33.899284+0000","last_undegraded":"2024-09-14T04:47:33.899284+0000","last_fullsized":"2024-09-14T04:47:33.899284+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:48:02.364960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1392,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.401254+0000","last_change":"2024-09-14T04:44:57.332511+0000","last_active":"2024-09-14T04:46:50.401254+0000","last_peered":"2024-09-14T04:46:50.401254+0000","last_clean":"2024-09-14T04:46:50.401254+0000","last_became_active":"2024-09-14T04:44:57.332166+0000","last_became_peered":"2024-09-14T04:44:57.332166+0000","last_unstale":"2024-09-14T04:46:50.401254+0000","last_undegraded":"2024-09-14T04:46:50.401254+0000","last_fullsized":"2024-09-14T04:46:50.401254+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:53.984639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2113,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.200820+0000","last_change":"2024-09-14T04:07:58.560880+0000","last_active":"2024-09-14T04:47:33.200820+0000","last_peered":"2024-09-14T04:47:33.200820+0000","last_clean":"2024-09-14T04:47:33.200820+0000","last_became_active":"2024-09-14T04:07:58.560132+0000","last_became_peered":"2024-09-14T04:07:58.560132+0000","last_unstale":"2024-09-14T04:47:33.200820+0000","last_undegraded":"2024-09-14T04:47:33.200820+0000","last_fullsized":"2024-09-14T04:47:33.200820+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:46:00.287050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2112,"reported_epoch":1417,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.200769+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T04:47:33.200769+0000","last_peered":"2024-09-14T04:47:33.200769+0000","last_clean":"2024-09-14T04:47:33.200769+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T04:47:33.200769+0000","last_undegraded":"2024-09-14T04:47:33.200769+0000","last_fullsized":"2024-09-14T04:47:33.200769+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1385,"reported_epoch":1416,"state":"active+clean","last_fresh":"2024-09-14T04:47:12.533241+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T04:47:12.533241+0000","last_peered":"2024-09-14T04:47:12.533241+0000","last_clean":"2024-09-14T04:47:12.533241+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T04:47:12.533241+0000","last_undegraded":"2024-09-14T04:47:12.533241+0000","last_fullsized":"2024-09-14T04:47:12.533241+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1421,"reported_epoch":1415,"state":"active+clean","last_fresh":"2024-09-14T04:46:50.401198+0000","last_change":"2024-09-14T04:44:57.339993+0000","last_active":"2024-09-14T04:46:50.401198+0000","last_peered":"2024-09-14T04:46:50.401198+0000","last_clean":"2024-09-14T04:46:50.401198+0000","last_became_active":"2024-09-14T04:44:57.330472+0000","last_became_peered":"2024-09-14T04:44:57.330472+0000","last_unstale":"2024-09-14T04:46:50.401198+0000","last_undegraded":"2024-09-14T04:46:50.401198+0000","last_fullsized":"2024-09-14T04:46:50.401198+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:19:27.046988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":55,"reported_epoch":1418,"state":"active+clean","last_fresh":"2024-09-14T04:47:33.837974+0000","last_change":"2024-09-14T04:44:14.479373+0000","last_active":"2024-09-14T04:47:33.837974+0000","last_peered":"2024-09-14T04:47:33.837974+0000","last_clean":"2024-09-14T04:47:33.837974+0000","last_became_active":"2024-09-14T04:44:14.479084+0000","last_became_peered":"2024-09-14T04:44:14.479084+0000","last_unstale":"2024-09-14T04:47:33.837974+0000","last_undegraded":"2024-09-14T04:47:33.837974+0000","last_fullsized":"2024-09-14T04:47:33.837974+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:49:23.412806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":5070,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4250,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201959,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":344492,"kb_used_data":10724,"kb_used_omap":1,"kb_used_meta":333758,"kb_avail":93400660,"statfs":{"total":95995035648,"available":95642275840,"internally_reserved":0,"allocated":10981376,"data_stored":6479934,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":341768651},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463595,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":450808,"kb_used_data":10412,"kb_used_omap":1,"kb_used_meta":440382,"kb_avail":93294344,"statfs":{"total":95995035648,"available":95533408256,"internally_reserved":0,"allocated":10661888,"data_stored":6161057,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":450951623},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725232,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":439072,"kb_used_data":10968,"kb_used_omap":1,"kb_used_meta":428094,"kb_avail":93306080,"statfs":{"total":95995035648,"available":95545425920,"internally_reserved":0,"allocated":11231232,"data_stored":6738321,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":438368710},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":1392,"seq":5978594476078,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":127660,"kb_used_data":10404,"kb_used_omap":117,"kb_used_meta":117130,"kb_avail":93617492,"statfs":{"total":95995035648,"available":95864311808,"internally_reserved":0,"allocated":10653696,"data_stored":6152806,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120638,"internal_metadata":119941314},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772074974,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":435972,"kb_used_data":10996,"kb_used_omap":71,"kb_used_meta":424888,"kb_avail":93309180,"statfs":{"total":95995035648,"available":95548600320,"internally_reserved":0,"allocated":11259904,"data_stored":6756194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":435086262},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":19,"apply_latency_ms":19,"commit_latency_ns":19000000,"apply_latency_ns":19000000},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707303909,"num_pgs":23,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":249604,"kb_used_data":10676,"kb_used_omap":68,"kb_used_meta":238843,"kb_avail":93495548,"statfs":{"total":95995035648,"available":95739441152,"internally_reserved":0,"allocated":10932224,"data_stored":6428418,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":244575440},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872729068,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":363132,"kb_used_data":10412,"kb_used_omap":67,"kb_used_meta":352636,"kb_avail":93382020,"statfs":{"total":95995035648,"available":95623188480,"internally_reserved":0,"allocated":10661888,"data_stored":6165133,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":361099621},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":12,"apply_latency_ms":12,"commit_latency_ns":12000000,"apply_latency_ns":12000000},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056118,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":399000,"kb_used_data":10384,"kb_used_omap":74,"kb_used_meta":388533,"kb_avail":93346152,"statfs":{"total":95995035648,"available":95586459648,"internally_reserved":0,"allocated":10633216,"data_stored":6147898,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":397858485},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1901,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:47:41.783 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-14T04:47:41.783 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:47:41.784 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2024-09-14T04:47:41.784 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight-by-utilization 110 0.05 2024-09-14T04:47:41.841 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:41 smithi039 ceph-mon[106470]: pgmap v2529: 83 pgs: 83 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:41.842 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:41 smithi039 ceph-mon[111235]: pgmap v2529: 83 pgs: 83 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:41.860 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:41.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:41.883 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:41.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:41.943 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:47:41.943 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:47:41.984 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:42.005 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:42.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:42.006 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:42.107 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:42.128 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:42.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:42.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:42.229 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:42.254 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:42.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:42.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:42.355 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:42.378 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:42.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:42.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:42.479 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:42.502 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:42.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:42.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:42.562 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-14T04:47:42.604 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:42.626 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:42.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:42.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:42.728 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:42.749 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:42.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:42.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:42.851 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:42.873 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:42.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:42.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:42.975 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:42.998 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:42.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:42.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:43.100 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:43.124 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:43.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:43.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:43.226 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:43.250 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:43.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:43.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:43.352 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:43.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:43.377 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:43.377 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:43.478 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:43.502 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:43.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:43.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:43.604 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:43.612 INFO:teuthology.orchestra.run.smithi039.stderr:pool 'unique_pool_2' created 2024-09-14T04:47:43.629 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:43.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:43.630 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:43.730 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:43.756 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:43.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:43.757 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:43.803 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:43 smithi039 ceph-mon[106470]: pgmap v2530: 83 pgs: 83 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:43.803 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:43 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2303797040' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-14T04:47:43.803 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:43 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-14T04:47:43.803 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:43 smithi039 ceph-mon[111235]: pgmap v2530: 83 pgs: 83 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:43.803 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:43 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2303797040' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-14T04:47:43.804 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:43 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-14T04:47:43.858 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:43.883 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:43.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:43.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:43.985 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:44.011 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:44.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:44.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:44.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:43 smithi129 ceph-mon[101316]: pgmap v2530: 83 pgs: 83 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:44.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:43 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2303797040' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-14T04:47:44.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:43 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16} : dispatch 2024-09-14T04:47:44.113 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:44.138 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:44.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:44.139 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:44.239 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:44.263 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:44.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:44.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:44.365 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:44.390 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:44.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:44.391 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:44.491 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:44.514 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:44.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:44.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:44.607 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is set 2024-09-14T04:47:44.616 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:44.643 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:44.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:44.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:44.745 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:44.767 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:44.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:44.768 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:44.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:44 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-09-14T04:47:44.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:44 smithi039 ceph-mon[106470]: osdmap e1419: 8 total, 8 up, 8 in 2024-09-14T04:47:44.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:44 smithi039 ceph-mon[106470]: pgmap v2532: 99 pgs: 16 unknown, 83 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:44.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:44 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3510577134' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:47:44.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:44 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-09-14T04:47:44.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:44 smithi039 ceph-mon[111235]: osdmap e1419: 8 total, 8 up, 8 in 2024-09-14T04:47:44.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:44 smithi039 ceph-mon[111235]: pgmap v2532: 99 pgs: 16 unknown, 83 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:44.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:44 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3510577134' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:47:44.869 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:44.886 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2024-09-14T04:47:44.895 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:44.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:44.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:44.997 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:45.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:44 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2024-09-14T04:47:45.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:44 smithi129 ceph-mon[101316]: osdmap e1419: 8 total, 8 up, 8 in 2024-09-14T04:47:45.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:44 smithi129 ceph-mon[101316]: pgmap v2532: 99 pgs: 16 unknown, 83 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:45.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:44 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3510577134' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:47:45.022 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:45.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:45.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:45.124 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:45.146 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:45.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:45.147 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:45.248 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:45.272 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:47:45.274 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:45.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:45.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:45.376 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:45.400 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:45.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:45.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:45.502 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:45.527 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:45.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:45.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:45.628 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:45.655 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:45.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:45.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:45.756 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:45.778 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:47:45.780 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:45.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:45.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:45.789 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:45 smithi039 ceph-mon[111235]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:47:45.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:45 smithi039 ceph-mon[111235]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:47:45.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:45 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3510577134' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:47:45.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:45 smithi039 ceph-mon[111235]: osdmap e1420: 8 total, 8 up, 8 in 2024-09-14T04:47:45.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:45 smithi039 ceph-mon[106470]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:47:45.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:45 smithi039 ceph-mon[106470]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:47:45.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:45 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3510577134' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:47:45.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:45 smithi039 ceph-mon[106470]: osdmap e1420: 8 total, 8 up, 8 in 2024-09-14T04:47:45.883 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:45.908 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:45.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:45.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:46.010 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:46.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:45 smithi129 ceph-mon[101316]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:47:46.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:45 smithi129 ceph-mon[101316]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:47:46.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:45 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3510577134' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:47:46.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:45 smithi129 ceph-mon[101316]: osdmap e1420: 8 total, 8 up, 8 in 2024-09-14T04:47:46.036 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:46.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:46.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:46.138 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:46.162 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:46.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:46.163 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:46.263 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:46.289 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:46.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:46.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:46.391 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:46.417 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:46.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:46.418 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:46.518 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:46.543 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:46.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:46.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:46.645 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:46.667 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:46.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:46.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:46.769 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:46.791 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:46.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:46.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:46.893 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:46.904 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:46 smithi039 ceph-mon[106470]: osdmap e1421: 8 total, 8 up, 8 in 2024-09-14T04:47:46.904 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:46 smithi039 ceph-mon[106470]: pgmap v2535: 99 pgs: 15 unknown, 84 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:46.904 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:46 smithi039 ceph-mon[111235]: osdmap e1421: 8 total, 8 up, 8 in 2024-09-14T04:47:46.904 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:46 smithi039 ceph-mon[111235]: pgmap v2535: 99 pgs: 15 unknown, 84 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:46.919 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:46.920 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:46.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:47.021 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:47.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:46 smithi129 ceph-mon[101316]: osdmap e1421: 8 total, 8 up, 8 in 2024-09-14T04:47:47.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:46 smithi129 ceph-mon[101316]: pgmap v2535: 99 pgs: 15 unknown, 84 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:47.047 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:47.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:47.048 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:47.149 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:47.174 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:47.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:47.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:47.275 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:47.300 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:47.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:47.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:47.402 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:47.427 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:47.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:47.428 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:47.529 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:47.556 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:47.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:47.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:47.658 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:47.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:47.685 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:47.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:47.786 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:47.812 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:47.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:47.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:47.914 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:47.937 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:47.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:47.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:48.039 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:48.061 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:48.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:48.062 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:48.163 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:48.190 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:48.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:48.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:48.291 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:48.317 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:48.317 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:47:48.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:48.319 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:48.419 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:48.443 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:48.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:48.444 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:48.545 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:48.569 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:48.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:48.570 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:48.671 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:48.695 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:48.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:48.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:48.797 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:48.821 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:48.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:48.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:48.923 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:48.946 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:48.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:48.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:49.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:48 smithi039 ceph-mon[106470]: pgmap v2536: 99 pgs: 9 unknown, 90 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:49.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:47:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:47:48] "GET /metrics HTTP/1.1" 200 34315 "" "Prometheus/2.51.0" 2024-09-14T04:47:49.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:48 smithi039 ceph-mon[111235]: pgmap v2536: 99 pgs: 9 unknown, 90 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:49.048 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:49.070 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:49.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:49.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:49.172 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:49.198 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:49.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:49.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:49.262 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:47:49.262 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:47:49.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:48 smithi129 ceph-mon[101316]: pgmap v2536: 99 pgs: 9 unknown, 90 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:49.300 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:49.326 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:49.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:49.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:49.428 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:49.453 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:49.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:49.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:49.555 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:49.580 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:49.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:49.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:49.682 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:49.707 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:49.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:49.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:49.809 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:49.834 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:49.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:49.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:49.936 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:49.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:49.959 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:49.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:50.060 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:50.083 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:50.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:50.084 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:50.185 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:50.211 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:50.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:50.212 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:50.234 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-14T04:47:50.313 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:50.335 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:50.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:50.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:50.437 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:50.453 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:47:50.453 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:47:50.462 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:50.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:50.463 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:50.470 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:47:50.471 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:47:50.488 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:47:50.489 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:47:50.505 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:47:50.505 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:47:50.524 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:47:50.524 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:47:50.541 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:47:50.541 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:47:50.558 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:47:50.558 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:47:50.563 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:50.574 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:47:50.575 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:47:50.586 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:50.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:50.587 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:50.688 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:50.713 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:50.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:50.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:50.815 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:50.838 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:50.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:50.839 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:50.939 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:50.966 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:50.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:50.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:51.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:50 smithi039 ceph-mon[106470]: pgmap v2537: 99 pgs: 99 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:51.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:50 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/424517021' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:47:51.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:50 smithi039 ceph-mon[111235]: pgmap v2537: 99 pgs: 99 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:51.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:50 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/424517021' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:47:51.069 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:51.094 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:51.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:51.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:51.196 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:51.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:51.223 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:51.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:50 smithi129 ceph-mon[101316]: pgmap v2537: 99 pgs: 99 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:50 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/424517021' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:47:51.324 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:51.350 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:51.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:51.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:51.452 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:51.477 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:51.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:51.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:51.579 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:51.602 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:51.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:51.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:51.703 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:51.728 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:51.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:51.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:51.830 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:51.853 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:51.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:51.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:51.887 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:47:51.927 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:47:51.955 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:51.980 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:51.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:51.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:52.082 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:52.104 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:52.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:52.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:52.184 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:52 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:47:52.184 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:52 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:47:52.184 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:47:52.184 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:47:52.184 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:52 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:47:52.184 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:52 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:47:52.184 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:52 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:47:52.185 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:47:52.185 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:47:52.185 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:52 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:47:52.206 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:52.232 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:52.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:52.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:52.334 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:52.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:52.357 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:52.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:52.458 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:52.484 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:52.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:52.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:52.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:52 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:47:52.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:52 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:47:52.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:47:52.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:47:52.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:52 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:47:52.586 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:52.611 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:52.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:52.612 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:52.712 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:52.737 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:52.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:52.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:52.839 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:52.863 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:52.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:52.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:52.910 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:47:52.911 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:47:52.965 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:52.987 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:52.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:52.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:53.089 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:53.111 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:53.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:53.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:53.213 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:53.236 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:53.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:53.237 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:53.337 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:53.363 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:53.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:53.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:53.465 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:53.487 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:53.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:53.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:53.588 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:53.613 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:53.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:53.614 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:53.715 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:53.735 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:53 smithi039 ceph-mon[106470]: pgmap v2538: 99 pgs: 99 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:53.736 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:53 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:47:53.736 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:53 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:47:53.736 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:53 smithi039 ceph-mon[111235]: pgmap v2538: 99 pgs: 99 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:53.736 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:53 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:47:53.736 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:53 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:47:53.738 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:53.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:53.739 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:53.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:53 smithi129 ceph-mon[101316]: pgmap v2538: 99 pgs: 99 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:53.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:53 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:47:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:53 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:47:53.840 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:53.864 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:53.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:53.865 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:53.911 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:47:53.965 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:53.987 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:53.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:53.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:54.088 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:54.111 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:54.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:54.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:54.213 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:54.234 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:54.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:54.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:54.336 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:54.362 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:54.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:54.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:54.464 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:54.489 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:54.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:54.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:54.591 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:54.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:54.614 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:54.614 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:54.715 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:54.740 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:54.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:54.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:54.842 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:54.868 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:54.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:54.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:54.970 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:54.995 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:54.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:54.996 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:55.097 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:55.122 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:55.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:55.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:55.224 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:55.248 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:55.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:55.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:55.350 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:55.373 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:55.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:55.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:55.474 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:55.496 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:55.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:55.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:55.539 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:55 smithi039 ceph-mon[106470]: pgmap v2539: 99 pgs: 99 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:55.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:55 smithi039 ceph-mon[111235]: pgmap v2539: 99 pgs: 99 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:55.598 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:55.620 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:55.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:55.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:55.722 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:55.748 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:55.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:55.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:55.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:55 smithi129 ceph-mon[101316]: pgmap v2539: 99 pgs: 99 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:55.849 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:55.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:55.873 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:55.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:55.974 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:56.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:56.001 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:56.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:56.102 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:56.126 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:56.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:56.127 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:56.228 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:56.251 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:56.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:56.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:56.353 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:56.374 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:56.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:56.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:56.476 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:56.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:56.503 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:56.503 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:56.604 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:56.626 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:56.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:56.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:56.728 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:56.753 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:56.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:56.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:56.855 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:56.880 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:56.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:56.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:56.982 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:57.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:57.005 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:57.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:57.110 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:57.133 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:57.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:57.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:57.234 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:57.256 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:57.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:57.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:57.358 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:57.382 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:57.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:57.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:57.484 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:57.497 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:57 smithi039 ceph-mon[106470]: pgmap v2540: 99 pgs: 99 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:57.497 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:57 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1723571852' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:47:57.497 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:57 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:47:57.497 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:57 smithi039 ceph-mon[111235]: pgmap v2540: 99 pgs: 99 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:57.497 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:57 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1723571852' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:47:57.497 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:57 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:47:57.507 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:57.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:57.508 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:57.609 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:57.635 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:57.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:57.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:57.737 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:57.763 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:57.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:57.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:57.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:57 smithi129 ceph-mon[101316]: pgmap v2540: 99 pgs: 99 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:57.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:57 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1723571852' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:47:57.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:57 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:47:57.857 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is set 2024-09-14T04:47:57.865 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:57.887 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:57.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:57.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:57.989 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:58.011 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:58.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:58.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:58.113 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:58.134 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:58.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:58.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:58.236 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:58.258 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:58.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:58.259 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:58.360 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:58.362 INFO:teuthology.orchestra.run.smithi039.stderr:enabled application 'rados' on pool 'unique_pool_2' 2024-09-14T04:47:58.372 INFO:teuthology.orchestra.run.smithi039.stdout:moved 4 / 297 (1.3468%) 2024-09-14T04:47:58.372 INFO:teuthology.orchestra.run.smithi039.stdout:avg 37.125 2024-09-14T04:47:58.372 INFO:teuthology.orchestra.run.smithi039.stdout:stddev 4.88461 -> 5.66651 (expected baseline 5.69951) 2024-09-14T04:47:58.373 INFO:teuthology.orchestra.run.smithi039.stdout:min osd.2 with 30 -> 30 pgs (0.808081 -> 0.808081 * mean) 2024-09-14T04:47:58.373 INFO:teuthology.orchestra.run.smithi039.stdout:max osd.7 with 45 -> 46 pgs (1.21212 -> 1.23906 * mean) 2024-09-14T04:47:58.373 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:47:58.373 INFO:teuthology.orchestra.run.smithi039.stdout:oload 110 2024-09-14T04:47:58.373 INFO:teuthology.orchestra.run.smithi039.stdout:max_change 0.05 2024-09-14T04:47:58.373 INFO:teuthology.orchestra.run.smithi039.stdout:max_change_osds 4 2024-09-14T04:47:58.373 INFO:teuthology.orchestra.run.smithi039.stdout:average_utilization 0.0037 2024-09-14T04:47:58.373 INFO:teuthology.orchestra.run.smithi039.stdout:overload_utilization 0.0041 2024-09-14T04:47:58.373 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6 weight 1.0000 -> 0.9500 2024-09-14T04:47:58.373 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5 weight 1.0000 -> 0.9500 2024-09-14T04:47:58.373 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3 weight 1.0000 -> 0.9500 2024-09-14T04:47:58.373 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0 weight 1.0000 -> 0.9500 2024-09-14T04:47:58.388 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:58.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:58.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:58.490 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:58.515 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:58.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:58.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:58.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:58 smithi039 ceph-mon[106470]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:47:58.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:58 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:47:58.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:58 smithi039 ceph-mon[106470]: osdmap e1422: 8 total, 8 up, 8 in 2024-09-14T04:47:58.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:58 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260','5':'62260','6':'62260'}"} : dispatch 2024-09-14T04:47:58.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:58 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260','5':'62260','6':'62260'}"} : dispatch 2024-09-14T04:47:58.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:58 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/45874277' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:47:58.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:58 smithi039 ceph-mon[111235]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:47:58.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:58 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:47:58.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:58 smithi039 ceph-mon[111235]: osdmap e1422: 8 total, 8 up, 8 in 2024-09-14T04:47:58.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:58 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260','5':'62260','6':'62260'}"} : dispatch 2024-09-14T04:47:58.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:58 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260','5':'62260','6':'62260'}"} : dispatch 2024-09-14T04:47:58.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:58 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/45874277' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:47:58.617 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:58.641 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:58.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:58.642 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:58.742 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:58.764 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:58.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:58.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:58 smithi129 ceph-mon[101316]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:47:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:58 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:47:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:58 smithi129 ceph-mon[101316]: osdmap e1422: 8 total, 8 up, 8 in 2024-09-14T04:47:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:58 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260','5':'62260','6':'62260'}"} : dispatch 2024-09-14T04:47:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:58 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260','5':'62260','6':'62260'}"} : dispatch 2024-09-14T04:47:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:58 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/45874277' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:47:58.866 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:47:58.893 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:58.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:58.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:47:58.942 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:47:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:47:58] "GET /metrics HTTP/1.1" 200 36663 "" "Prometheus/2.51.0" 2024-09-14T04:47:58.994 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:59.020 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:59.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:59.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:59.122 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:47:59.146 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:59.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:59.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:47:59.248 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:47:59.272 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:59.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:59.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:47:59.306 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2024-09-14T04:47:59.306 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd test-reweight-by-utilization 110 0.05 2024-09-14T04:47:59.374 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:47:59.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:59.402 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:59.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:47:59.502 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:47:59.529 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:59.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:59.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:47:59.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:59 smithi039 ceph-mon[106470]: from='client.45617 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:47:59.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:59 smithi039 ceph-mon[106470]: pgmap v2542: 99 pgs: 99 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:59.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260','5':'62260','6':'62260'}"}]': finished 2024-09-14T04:47:59.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:59 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/45874277' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:47:59.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:47:59 smithi039 ceph-mon[106470]: osdmap e1423: 8 total, 8 up, 8 in 2024-09-14T04:47:59.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:59 smithi039 ceph-mon[111235]: from='client.45617 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:47:59.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:59 smithi039 ceph-mon[111235]: pgmap v2542: 99 pgs: 99 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:59.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260','5':'62260','6':'62260'}"}]': finished 2024-09-14T04:47:59.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:59 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/45874277' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:47:59.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:47:59 smithi039 ceph-mon[111235]: osdmap e1423: 8 total, 8 up, 8 in 2024-09-14T04:47:59.632 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:47:59.654 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:59.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:59.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:47:59.672 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:59 smithi129 ceph-mon[101316]: from='client.45617 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:47:59.673 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:59 smithi129 ceph-mon[101316]: pgmap v2542: 99 pgs: 99 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:47:59.673 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'62260','3':'62260','5':'62260','6':'62260'}"}]': finished 2024-09-14T04:47:59.673 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:59 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/45874277' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:47:59.673 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:47:59 smithi129 ceph-mon[101316]: osdmap e1423: 8 total, 8 up, 8 in 2024-09-14T04:47:59.755 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:47:59.783 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:47:59.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:59.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:47:59.884 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:47:59.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:47:59.908 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:47:59.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:47:59.984 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:48:00.009 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:00.031 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:00.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:00.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:00.133 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:00.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:00.155 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:00.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:00.256 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:00.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:00.278 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:00.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:00.379 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:00.400 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:00.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:00.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:00.502 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:00.523 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:00.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:00.524 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:00.625 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:00.636 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:48:00.653 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:00.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:00.654 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:00.676 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:00 smithi039 ceph-mon[106470]: osdmap e1424: 8 total, 8 up, 8 in 2024-09-14T04:48:00.677 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:00 smithi039 ceph-mon[106470]: osdmap e1425: 8 total, 8 up, 8 in 2024-09-14T04:48:00.677 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:00 smithi039 ceph-mon[111235]: osdmap e1424: 8 total, 8 up, 8 in 2024-09-14T04:48:00.677 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:00 smithi039 ceph-mon[111235]: osdmap e1425: 8 total, 8 up, 8 in 2024-09-14T04:48:00.755 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:00.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:00 smithi129 ceph-mon[101316]: osdmap e1424: 8 total, 8 up, 8 in 2024-09-14T04:48:00.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:00 smithi129 ceph-mon[101316]: osdmap e1425: 8 total, 8 up, 8 in 2024-09-14T04:48:00.777 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:00.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:00.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:00.879 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:00.900 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:00.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:00.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:00.969 DEBUG:teuthology.orchestra.run.smithi039:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2024-09-14T04:48:00.972 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-14T04:48:00.972 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-14T04:48:01.001 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:01.022 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:01.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:01.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:01.124 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:01.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:01.150 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:01.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:01.251 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:01.274 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:01.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:01.275 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:01.376 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:01.400 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:01.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:01.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:01.502 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:01.524 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:01.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:01.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:01.544 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:hints = 1 2024-09-14T04:48:01.544 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-14T04:48:01.544 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Object prefix: benchmark_data_smithi039_297155 2024-09-14T04:48:01.544 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:48:01.545 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 0 0 0 0 0 0 - 0 2024-09-14T04:48:01.626 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:01.665 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:01.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:01.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:01.767 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:01 smithi129 ceph-mon[101316]: pgmap v2545: 99 pgs: 99 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:48:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:01 smithi129 ceph-mon[101316]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:48:01.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:01 smithi039 ceph-mon[106470]: pgmap v2545: 99 pgs: 99 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:48:01.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:01 smithi039 ceph-mon[106470]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:48:01.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:01 smithi039 ceph-mon[111235]: pgmap v2545: 99 pgs: 99 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:48:01.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:01 smithi039 ceph-mon[111235]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:48:01.800 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:01.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:01.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:01.902 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:01.937 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:01.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:01.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:01.968 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:48:02.039 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:02.083 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:02.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:02.084 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:02.185 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:02.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:02.226 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:02.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:02.327 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:02.360 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:02.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:02.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:02.461 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:02.501 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:02.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:02.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:02.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:02 smithi039 ceph-mon[106470]: osdmap e1426: 8 total, 8 up, 8 in 2024-09-14T04:48:02.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:02 smithi039 ceph-mon[106470]: osdmap e1427: 8 total, 8 up, 8 in 2024-09-14T04:48:02.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:02 smithi039 ceph-mon[111235]: osdmap e1426: 8 total, 8 up, 8 in 2024-09-14T04:48:02.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:02 smithi039 ceph-mon[111235]: osdmap e1427: 8 total, 8 up, 8 in 2024-09-14T04:48:02.544 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 1 16 3659 3643 227.674 227.688 0.00408681 0.00437522 2024-09-14T04:48:02.604 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:02.643 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:02.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:02.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:02.745 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:02.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:02 smithi129 ceph-mon[101316]: osdmap e1426: 8 total, 8 up, 8 in 2024-09-14T04:48:02.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:02 smithi129 ceph-mon[101316]: osdmap e1427: 8 total, 8 up, 8 in 2024-09-14T04:48:02.785 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:02.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:02.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:02.887 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:02.925 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:02.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:02.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:03.027 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:03.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:03.066 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:03.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:03.167 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:03.202 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:03.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:03.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:03.304 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:03.316 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:48:03.346 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:03.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:03.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:03.450 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:03.475 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:03.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:03.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:03.545 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 2 16 6988 6972 217.859 208.062 0.00457376 0.00458109 2024-09-14T04:48:03.562 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:03 smithi039 ceph-mon[106470]: pgmap v2548: 99 pgs: 1 remapped+peering, 98 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:48:03.562 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:03 smithi039 ceph-mon[106470]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:48:03.563 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:03 smithi039 ceph-mon[111235]: pgmap v2548: 99 pgs: 1 remapped+peering, 98 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:48:03.563 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:03 smithi039 ceph-mon[111235]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:48:03.577 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:03.617 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:03.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:03.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:03.719 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:03.757 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:03.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:03.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:03 smithi129 ceph-mon[101316]: pgmap v2548: 99 pgs: 1 remapped+peering, 98 active+clean; 582 KiB data, 2.7 GiB used, 713 GiB / 715 GiB avail 2024-09-14T04:48:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:03 smithi129 ceph-mon[101316]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:48:03.859 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:03.899 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:03.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:03.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:04.001 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:04.039 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:04.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:04.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:04.141 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:04.173 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:04.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:04.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:04.275 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:04.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:04.320 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:04.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:04.421 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:04.459 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:04.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:04.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:04.545 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 3 16 10373 10357 215.754 211.562 0.00326983 0.00462894 2024-09-14T04:48:04.561 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:04.595 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:04.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:04.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:04.696 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:04.732 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:04.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:04.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:04.835 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:04.869 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:04.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:04.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:04.971 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:04.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:04.993 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:04.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:05.094 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:05.132 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:05.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:05.133 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:05.235 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:05.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:05.268 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:05.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:05.369 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:05.390 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:05.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:05.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:05.494 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:05.519 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:48:05.519 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [], 2024-09-14T04:48:05.519 INFO:teuthology.orchestra.run.smithi039.stdout: "complaint_time": 30, 2024-09-14T04:48:05.519 INFO:teuthology.orchestra.run.smithi039.stdout: "num_blocked_ops": 0 2024-09-14T04:48:05.519 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:48:05.528 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:05.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:05.529 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:05.545 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 4 16 12997 12981 202.811 164 0.00367945 0.0049241 2024-09-14T04:48:05.630 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:05.666 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:05.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:05.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:05.768 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:05 smithi129 ceph-mon[101316]: pgmap v2550: 99 pgs: 1 remapped+peering, 98 active+clean; 266 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s; 0 B/s, 0 objects/s recovering 2024-09-14T04:48:05.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:05 smithi039 ceph-mon[106470]: pgmap v2550: 99 pgs: 1 remapped+peering, 98 active+clean; 266 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s; 0 B/s, 0 objects/s recovering 2024-09-14T04:48:05.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:05 smithi039 ceph-mon[111235]: pgmap v2550: 99 pgs: 1 remapped+peering, 98 active+clean; 266 MiB data, 3.2 GiB used, 712 GiB / 715 GiB avail; 61 MiB/s wr, 982 op/s; 0 B/s, 0 objects/s recovering 2024-09-14T04:48:05.806 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:05.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:05.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:05.908 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:05.942 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:05.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:05.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:06.043 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:06.084 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:06.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:06.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:06.186 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:06.224 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:06.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:06.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:06.325 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:06.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:06.360 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:06.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:06.461 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:06.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:06.495 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:06.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:06.545 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 5 16 16196 16180 202.231 199.938 0.00254249 0.00493871 2024-09-14T04:48:06.597 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:06.636 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:06.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:06.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:06.738 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:06.761 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:06.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:06.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:06.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:06 smithi129 ceph-mon[101316]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-14T04:48:06.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:06 smithi039 ceph-mon[106470]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-14T04:48:06.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:06 smithi039 ceph-mon[111235]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-14T04:48:06.863 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:06.901 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:06.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:06.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:07.003 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:07.040 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:07.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:07.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:07.142 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:07.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:07.177 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:07.177 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:07.278 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:07.314 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:07.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:07.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:07.417 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:07.455 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:07.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:07.456 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:07.545 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 6 16 18645 18629 194.034 153.062 0.00585539 0.00514797 2024-09-14T04:48:07.557 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:07.590 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:07.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:07.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:07.692 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:07.729 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:07.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:07.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:07 smithi129 ceph-mon[101316]: pgmap v2551: 99 pgs: 99 active+clean; 312 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 52 MiB/s wr, 830 op/s; 43 KiB/s, 0 objects/s recovering 2024-09-14T04:48:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:48:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:48:07.781 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:07 smithi039 ceph-mon[106470]: pgmap v2551: 99 pgs: 99 active+clean; 312 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 52 MiB/s wr, 830 op/s; 43 KiB/s, 0 objects/s recovering 2024-09-14T04:48:07.781 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:48:07.781 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:48:07.781 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:07 smithi039 ceph-mon[111235]: pgmap v2551: 99 pgs: 99 active+clean; 312 MiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 52 MiB/s wr, 830 op/s; 43 KiB/s, 0 objects/s recovering 2024-09-14T04:48:07.781 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:48:07.781 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:48:07.831 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:07.870 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:07.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:07.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:07.972 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:08.010 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:08.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:08.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:08.112 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:08.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:08.134 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:08.134 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:08.235 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:08.270 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:08.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:08.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:08.372 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:08.409 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:08.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:08.410 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:08.511 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:08.545 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 7 16 21236 21220 189.447 161.938 0.00452393 0.00527324 2024-09-14T04:48:08.547 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:08.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:08.548 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:08.649 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:08.684 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:08.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:08.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:08.786 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:08.828 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:08.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:08.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:08.930 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:08.965 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:08.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:08.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:09.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:48:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:48:08] "GET /metrics HTTP/1.1" 200 36782 "" "Prometheus/2.51.0" 2024-09-14T04:48:09.066 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:09.103 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:09.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:09.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:09.205 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:09.242 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:09.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:09.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:09.344 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:09.381 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:09.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:09.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:09.484 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:09.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:09.529 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:09.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:09.545 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 8 16 23767 23751 185.538 158.188 0.00582389 0.00538234 2024-09-14T04:48:09.631 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:09.667 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:09.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:09.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:09.770 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:09 smithi129 ceph-mon[101316]: pgmap v2552: 99 pgs: 99 active+clean; 656 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s; 35 KiB/s, 0 objects/s recovering 2024-09-14T04:48:09.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:09 smithi039 ceph-mon[106470]: pgmap v2552: 99 pgs: 99 active+clean; 656 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s; 35 KiB/s, 0 objects/s recovering 2024-09-14T04:48:09.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:09 smithi039 ceph-mon[111235]: pgmap v2552: 99 pgs: 99 active+clean; 656 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s; 35 KiB/s, 0 objects/s recovering 2024-09-14T04:48:09.803 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:09.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:09.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:09.905 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:09.939 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:09.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:09.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:10.041 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:10.063 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:10.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:10.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:10.164 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:10.198 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:10.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:10.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:10.300 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:10.335 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:10.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:10.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:10.438 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:10.472 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:10.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:10.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:10.545 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 9 16 26051 26035 180.781 142.75 0.00175718 0.00550306 2024-09-14T04:48:10.574 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:10.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:10.602 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:10.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:10.703 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:10.725 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:10.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:10.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:10.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:10 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]} : dispatch 2024-09-14T04:48:10.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:10 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "55.1", "id": [7, 0]} : dispatch 2024-09-14T04:48:10.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:10 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]} : dispatch 2024-09-14T04:48:10.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:10 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "55.1", "id": [7, 0]} : dispatch 2024-09-14T04:48:10.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:10 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]} : dispatch 2024-09-14T04:48:10.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:10 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "55.1", "id": [7, 0]} : dispatch 2024-09-14T04:48:10.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:10 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]} : dispatch 2024-09-14T04:48:10.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:10 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "55.1", "id": [7, 0]} : dispatch 2024-09-14T04:48:10.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:10 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]} : dispatch 2024-09-14T04:48:10.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:10 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "55.1", "id": [7, 0]} : dispatch 2024-09-14T04:48:10.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:10 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]} : dispatch 2024-09-14T04:48:10.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:10 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "55.1", "id": [7, 0]} : dispatch 2024-09-14T04:48:10.827 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:10.848 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:10.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:10.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:10.950 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:10.976 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:10.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:10.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:11.078 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:11.104 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:11.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:11.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:11.205 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:11.228 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:11.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:11.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:11.330 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:11.354 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:11.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:11.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:11.456 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:11.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:11.480 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:11.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:11.546 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 10 16 26189 26173 163.565 8.625 0.0016459 0.00551613 2024-09-14T04:48:11.581 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:11.606 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:11.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:11.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:11.708 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:11.734 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:11.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:11.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:11 smithi129 ceph-mon[101316]: pgmap v2553: 99 pgs: 99 active+clean; 1.1 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s; 31 KiB/s, 0 objects/s recovering 2024-09-14T04:48:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:11 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-09-14T04:48:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:11 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "55.1", "id": [7, 0]}]': finished 2024-09-14T04:48:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:11 smithi129 ceph-mon[101316]: osdmap e1428: 8 total, 8 up, 8 in 2024-09-14T04:48:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:11 smithi129 ceph-mon[101316]: osdmap e1429: 8 total, 8 up, 8 in 2024-09-14T04:48:11.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:11 smithi039 ceph-mon[106470]: pgmap v2553: 99 pgs: 99 active+clean; 1.1 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s; 31 KiB/s, 0 objects/s recovering 2024-09-14T04:48:11.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:11 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-09-14T04:48:11.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:11 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "55.1", "id": [7, 0]}]': finished 2024-09-14T04:48:11.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:11 smithi039 ceph-mon[106470]: osdmap e1428: 8 total, 8 up, 8 in 2024-09-14T04:48:11.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:11 smithi039 ceph-mon[106470]: osdmap e1429: 8 total, 8 up, 8 in 2024-09-14T04:48:11.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:11 smithi039 ceph-mon[111235]: pgmap v2553: 99 pgs: 99 active+clean; 1.1 GiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s; 31 KiB/s, 0 objects/s recovering 2024-09-14T04:48:11.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:11 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-09-14T04:48:11.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:11 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "55.1", "id": [7, 0]}]': finished 2024-09-14T04:48:11.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:11 smithi039 ceph-mon[111235]: osdmap e1428: 8 total, 8 up, 8 in 2024-09-14T04:48:11.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:11 smithi039 ceph-mon[111235]: osdmap e1429: 8 total, 8 up, 8 in 2024-09-14T04:48:11.837 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:11.861 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:11.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:11.862 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:11.963 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:11.988 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:11.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:11.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:11.994 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:48:11.994 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:48:12.012 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:48:12.012 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:48:12.029 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:48:12.029 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:48:12.046 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:48:12.046 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:48:12.066 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:48:12.066 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:48:12.083 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:48:12.083 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:48:12.090 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:12.099 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:48:12.099 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:48:12.113 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:12.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:12.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:12.117 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:48:12.118 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:48:12.215 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:12.241 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:12.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:12.242 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:12.343 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:12.365 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:12.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:12.366 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:12.467 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:12.490 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:12.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:12.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:12.545 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 11 16 26189 26173 148.695 0 - 0.00551613 2024-09-14T04:48:12.592 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:12.615 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:12.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:12.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:12.717 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:12.742 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:12.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:12.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:12.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:12 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1692899041' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:48:12.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:12 smithi129 ceph-mon[101316]: osdmap e1430: 8 total, 8 up, 8 in 2024-09-14T04:48:12.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:12 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1692899041' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:48:12.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:12 smithi039 ceph-mon[111235]: osdmap e1430: 8 total, 8 up, 8 in 2024-09-14T04:48:12.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:12 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1692899041' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:48:12.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:12 smithi039 ceph-mon[106470]: osdmap e1430: 8 total, 8 up, 8 in 2024-09-14T04:48:12.844 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:12.867 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:12.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:12.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:12.969 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:13.011 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:13.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:13.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:13.113 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:13.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:13.149 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:13.149 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:13.250 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:13.283 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:13.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:13.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:13.385 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:13.418 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:13.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:13.419 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:13.520 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:13.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:13 smithi039 ceph-mon[111235]: pgmap v2556: 99 pgs: 99 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 129 MiB/s wr, 2.07k op/s; 32 KiB/s, 0 objects/s recovering 2024-09-14T04:48:13.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:13 smithi039 ceph-mon[106470]: pgmap v2556: 99 pgs: 99 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 129 MiB/s wr, 2.07k op/s; 32 KiB/s, 0 objects/s recovering 2024-09-14T04:48:13.546 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 12 16 27597 27581 143.634 44 0.00767393 0.00695504 2024-09-14T04:48:13.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:13.569 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:13.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:13.670 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:13.709 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:13.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:13.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:13.810 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:13.853 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:13.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:13.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:13.908 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-14T04:48:13.955 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:14.008 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:14.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:14.009 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:14.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:13 smithi129 ceph-mon[101316]: pgmap v2556: 99 pgs: 99 active+clean; 1.3 GiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 129 MiB/s wr, 2.07k op/s; 32 KiB/s, 0 objects/s recovering 2024-09-14T04:48:14.110 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:14.152 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:14.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:14.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:14.254 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:14.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:14.286 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:14.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:14.387 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:14.414 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:14.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:14.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:14.516 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:14.545 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:14.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:14.546 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:14.547 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 13 16 29592 29576 142.171 124.688 0.0180092 0.00702675 2024-09-14T04:48:14.647 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:14.676 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:14.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:14.677 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:14.778 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:14.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:14 smithi039 ceph-mon[106470]: Health check failed: Degraded data redundancy: 1607/79293 objects degraded (2.027%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:48:14.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:14 smithi039 ceph-mon[111235]: Health check failed: Degraded data redundancy: 1607/79293 objects degraded (2.027%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:48:14.825 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:14.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:14.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:14.842 INFO:teuthology.orchestra.run.smithi039.stdout:moved 9 / 297 (3.0303%) 2024-09-14T04:48:14.842 INFO:teuthology.orchestra.run.smithi039.stdout:avg 37.125 2024-09-14T04:48:14.842 INFO:teuthology.orchestra.run.smithi039.stdout:stddev 5.13414 -> 3.47985 (expected baseline 5.69951) 2024-09-14T04:48:14.842 INFO:teuthology.orchestra.run.smithi039.stdout:min osd.5 with 30 -> 33 pgs (0.808081 -> 0.888889 * mean) 2024-09-14T04:48:14.842 INFO:teuthology.orchestra.run.smithi039.stdout:max osd.7 with 45 -> 42 pgs (1.21212 -> 1.13131 * mean) 2024-09-14T04:48:14.842 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:48:14.842 INFO:teuthology.orchestra.run.smithi039.stdout:oload 110 2024-09-14T04:48:14.843 INFO:teuthology.orchestra.run.smithi039.stdout:max_change 0.05 2024-09-14T04:48:14.843 INFO:teuthology.orchestra.run.smithi039.stdout:max_change_osds 4 2024-09-14T04:48:14.843 INFO:teuthology.orchestra.run.smithi039.stdout:average_utilization 0.0106 2024-09-14T04:48:14.843 INFO:teuthology.orchestra.run.smithi039.stdout:overload_utilization 0.0116 2024-09-14T04:48:14.843 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7 weight 1.0000 -> 0.9500 2024-09-14T04:48:14.843 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0 weight 0.9500 -> 1.0000 2024-09-14T04:48:14.843 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3 weight 0.9500 -> 0.9000 2024-09-14T04:48:14.843 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6 weight 0.9500 -> 0.9000 2024-09-14T04:48:14.843 INFO:teuthology.orchestra.run.smithi039.stderr:no change 2024-09-14T04:48:14.927 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:14.965 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:14.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:14.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:15.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:14 smithi129 ceph-mon[101316]: Health check failed: Degraded data redundancy: 1607/79293 objects degraded (2.027%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:48:15.067 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:15.102 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:15.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:15.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:15.203 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:15.238 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:15.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:15.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:15.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:48:14 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T04:48:14.838+0000 7f268b1da640 -1 mgr.server reply reply (4) Interrupted system call no change 2024-09-14T04:48:15.340 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:15.381 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:15.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:15.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:15.482 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:15.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:15.522 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:15.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:15.546 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 14 16 31575 31559 140.868 123.938 0.00676874 0.00709235 2024-09-14T04:48:15.595 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is unset 2024-09-14T04:48:15.623 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:15.666 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:15.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:15.667 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:15.768 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:15.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:15 smithi039 ceph-mon[106470]: pgmap v2558: 99 pgs: 1 activating+undersized+degraded+remapped, 98 active+clean; 1.6 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 166 MiB/s wr, 2.65k op/s; 1607/79293 objects degraded (2.027%) 2024-09-14T04:48:15.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:15 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3652499116' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:48:15.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:15 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:48:15.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:15 smithi039 ceph-mon[111235]: pgmap v2558: 99 pgs: 1 activating+undersized+degraded+remapped, 98 active+clean; 1.6 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 166 MiB/s wr, 2.65k op/s; 1607/79293 objects degraded (2.027%) 2024-09-14T04:48:15.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:15 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3652499116' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:48:15.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:15 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:48:15.809 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:15.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:15.810 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:15.877 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2024-09-14T04:48:15.878 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight-by-utilization 1000 3.0 2024-09-14T04:48:15.911 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:15.941 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:15.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:15.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:16.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:15 smithi129 ceph-mon[101316]: pgmap v2558: 99 pgs: 1 activating+undersized+degraded+remapped, 98 active+clean; 1.6 GiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 166 MiB/s wr, 2.65k op/s; 1607/79293 objects degraded (2.027%) 2024-09-14T04:48:16.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:15 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3652499116' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:48:16.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:15 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:48:16.042 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:16.087 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:16.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:16.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:16.190 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:16.222 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:16.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:16.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:16.323 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:16.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:16.364 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:16.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:16.465 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:16.501 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:16.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:16.501 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:16.546 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 15 16 33479 33463 139.409 119 0.00734147 0.00716644 2024-09-14T04:48:16.603 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:16.641 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:16.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:16.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:16.743 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:16.777 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:16.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:16.778 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:16.878 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:16.905 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:16 smithi129 ceph-mon[101316]: from='client.45644 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:48:16.905 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:16 smithi129 ceph-mon[101316]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:48:16.905 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:16 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:48:16.906 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:16 smithi129 ceph-mon[101316]: osdmap e1431: 8 total, 8 up, 8 in 2024-09-14T04:48:16.913 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:16.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:16.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:17.015 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:17.017 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:48:17.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:16 smithi039 ceph-mon[106470]: from='client.45644 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:48:17.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:16 smithi039 ceph-mon[106470]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:48:17.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:16 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:48:17.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:16 smithi039 ceph-mon[106470]: osdmap e1431: 8 total, 8 up, 8 in 2024-09-14T04:48:17.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:16 smithi039 ceph-mon[111235]: from='client.45644 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:48:17.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:16 smithi039 ceph-mon[111235]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:48:17.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:16 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:48:17.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:16 smithi039 ceph-mon[111235]: osdmap e1431: 8 total, 8 up, 8 in 2024-09-14T04:48:17.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:17.061 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:17.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:17.161 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:17.209 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:17.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:17.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:17.271 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 04:48:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=cleanup t=2024-09-14T04:48:16.903574949Z level=info msg="Completed cleanup jobs" duration=16.804479ms 2024-09-14T04:48:17.271 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 04:48:17 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=grafana.update.checker t=2024-09-14T04:48:17.039872097Z level=info msg="Update check succeeded" duration=67.464815ms 2024-09-14T04:48:17.271 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 04:48:17 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=plugins.update.checker t=2024-09-14T04:48:17.126716264Z level=info msg="Update check succeeded" duration=58.022988ms 2024-09-14T04:48:17.311 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:17.333 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:17.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:17.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:17.434 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:17.468 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:17.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:17.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:17.546 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 16 16 35194 35178 137.395 107.188 0.00212861 0.00726555 2024-09-14T04:48:17.570 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:17.608 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:17.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:17.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:17.710 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:17.731 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:17.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:17.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:17.833 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:17.871 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:17.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:17.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:17.973 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:18.010 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:18.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:18.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:18.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:17 smithi129 ceph-mon[101316]: pgmap v2560: 99 pgs: 1 activating+undersized+degraded+remapped, 98 active+clean; 1.6 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s; 1607/80325 objects degraded (2.001%) 2024-09-14T04:48:18.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:17 smithi039 ceph-mon[106470]: pgmap v2560: 99 pgs: 1 activating+undersized+degraded+remapped, 98 active+clean; 1.6 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s; 1607/80325 objects degraded (2.001%) 2024-09-14T04:48:18.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:17 smithi039 ceph-mon[111235]: pgmap v2560: 99 pgs: 1 activating+undersized+degraded+remapped, 98 active+clean; 1.6 GiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s; 1607/80325 objects degraded (2.001%) 2024-09-14T04:48:18.112 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:18.147 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:18.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:18.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:18.249 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:18.273 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:18.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:18.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:18.375 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:18.412 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:18.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:18.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:18.514 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:18.547 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 17 16 37137 37121 136.456 121.438 0.0091402 0.00732262 2024-09-14T04:48:18.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:18.548 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:18.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:18.649 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:18.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:18.690 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:18.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:18.792 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:18.813 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:18.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:18.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:18.915 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:18.949 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:18.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:18.950 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:19.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:18 smithi129 ceph-mon[101316]: pgmap v2561: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s; 1334/91182 objects misplaced (1.463%); 2.6 MiB/s, 40 objects/s recovering 2024-09-14T04:48:19.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:18 smithi039 ceph-mon[106470]: pgmap v2561: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s; 1334/91182 objects misplaced (1.463%); 2.6 MiB/s, 40 objects/s recovering 2024-09-14T04:48:19.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:48:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:48:18] "GET /metrics HTTP/1.1" 200 36782 "" "Prometheus/2.51.0" 2024-09-14T04:48:19.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:18 smithi039 ceph-mon[111235]: pgmap v2561: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 1.9 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s; 1334/91182 objects misplaced (1.463%); 2.6 MiB/s, 40 objects/s recovering 2024-09-14T04:48:19.051 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:19.074 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:19.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:19.075 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:19.176 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:19.207 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:19.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:19.208 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:19.216 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:48:19.309 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:19.350 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:19.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:19.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:19.452 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:19.489 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:19.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:19.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:19.547 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 18 15 38846 38831 134.812 106.875 0.00749189 0.00741272 2024-09-14T04:48:19.590 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:19.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:19.635 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:19.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:19.736 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:19.780 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:19.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:19.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:19.882 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:19.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:19.922 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:19.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:20.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:19 smithi129 ceph-mon[101316]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1607/80325 objects degraded (2.001%), 1 pg degraded) 2024-09-14T04:48:20.023 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:20.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:19 smithi039 ceph-mon[106470]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1607/80325 objects degraded (2.001%), 1 pg degraded) 2024-09-14T04:48:20.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:19 smithi039 ceph-mon[111235]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1607/80325 objects degraded (2.001%), 1 pg degraded) 2024-09-14T04:48:20.058 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:20.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:20.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:20.160 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:20.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:20.193 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:20.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:20.294 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:20.330 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:20.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:20.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:20.432 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:20.470 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:20.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:20.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:20.547 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 19 16 40629 40613 133.578 111.375 0.00553637 0.00748048 2024-09-14T04:48:20.572 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:20.606 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:20.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:20.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:20.615 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:48:20.615 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:48:20.708 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:20.743 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:20.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:20.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:20.754 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:20 smithi039 ceph-mon[106470]: pgmap v2562: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 2.1 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s; 1334/105336 objects misplaced (1.266%); 2.1 MiB/s, 34 objects/s recovering 2024-09-14T04:48:20.755 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:20 smithi039 ceph-mon[111235]: pgmap v2562: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 2.1 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s; 1334/105336 objects misplaced (1.266%); 2.1 MiB/s, 34 objects/s recovering 2024-09-14T04:48:20.845 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:20.887 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:20.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:20.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:20.989 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:21.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:20 smithi129 ceph-mon[101316]: pgmap v2562: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 2.1 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s; 1334/105336 objects misplaced (1.266%); 2.1 MiB/s, 34 objects/s recovering 2024-09-14T04:48:21.021 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:21.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:21.022 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:21.123 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:21.163 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:21.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:21.164 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:21.248 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:48:21.265 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:21.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:21.300 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:21.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:21.401 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:21.428 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:21.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:21.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:21.530 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:21.548 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:2024-09-14T04:48:21.548578+0000 min lat: 0.00109434 max lat: 2.39824 avg lat: 0.00754511 2024-09-14T04:48:21.548 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:48:21.548 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 20 16 42407 42391 132.446 111.125 0.00927358 0.00754511 2024-09-14T04:48:21.561 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:21.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:21.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:21.663 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:21.696 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:21.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:21.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:21.798 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:21.822 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:21.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:21.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:21.924 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:21.962 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:21.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:21.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:22.064 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:22.096 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:22.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:22.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:22.198 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:22.218 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:22.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:22.219 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:22.320 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:22.341 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:22.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:22.342 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:22.442 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:22.463 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:22.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:22.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:22.548 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 21 16 43753 43737 130.145 84.125 0.00788055 0.00767885 2024-09-14T04:48:22.565 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:22.592 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:22.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:22.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:22.694 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:22.731 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:22.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:22.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:22.833 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:22.866 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:22.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:22.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:22.968 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:23.003 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:23.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:23.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:23.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:22 smithi039 ceph-mon[106470]: pgmap v2563: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 2.2 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 1334/110433 objects misplaced (1.208%); 1.8 MiB/s, 28 objects/s recovering 2024-09-14T04:48:23.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:48:23.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:22 smithi039 ceph-mon[111235]: pgmap v2563: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 2.2 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 1334/110433 objects misplaced (1.208%); 1.8 MiB/s, 28 objects/s recovering 2024-09-14T04:48:23.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:48:23.106 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:23.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:23.189 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:23.189 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:23.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:22 smithi129 ceph-mon[101316]: pgmap v2563: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 2.2 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 103 MiB/s wr, 1.65k op/s; 1334/110433 objects misplaced (1.208%); 1.8 MiB/s, 28 objects/s recovering 2024-09-14T04:48:23.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:48:23.290 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:23.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:23.327 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:23.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:23.428 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:23.463 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:23.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:23.464 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:23.548 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 22 16 45525 45509 129.263 110.75 0.00729135 0.00773116 2024-09-14T04:48:23.565 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:23.598 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:23.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:23.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:23.700 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:23.732 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:23.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:23.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:23.833 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:23.866 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:23.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:23.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:23.968 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:24.001 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:24.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:24.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:24.105 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:24.148 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:24.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:24.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:24.249 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:24.286 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:24.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:24.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:24.388 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:24.430 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:24.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:24.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:24.532 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:24.548 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 23 16 47285 47269 128.425 110 0.00648409 0.00778096 2024-09-14T04:48:24.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:24.563 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:24.564 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:24.664 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:24.704 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:24.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:24.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:24.806 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:24.840 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:24.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:24.841 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:24.942 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:24.975 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:24.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:24.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:25.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:24 smithi039 ceph-mon[106470]: pgmap v2564: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s; 978/131175 objects misplaced (0.746%); 3.9 MiB/s, 62 objects/s recovering 2024-09-14T04:48:25.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:24 smithi039 ceph-mon[111235]: pgmap v2564: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s; 978/131175 objects misplaced (0.746%); 3.9 MiB/s, 62 objects/s recovering 2024-09-14T04:48:25.076 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:25.098 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:25.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:25.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:25.200 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:25.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:25.244 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:25.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:25.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:24 smithi129 ceph-mon[101316]: pgmap v2564: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s; 978/131175 objects misplaced (0.746%); 3.9 MiB/s, 62 objects/s recovering 2024-09-14T04:48:25.345 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:25.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:25.388 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:25.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:25.489 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:25.521 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:25.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:25.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:25.549 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 24 16 48957 48941 127.423 104.5 0.00700476 0.00784154 2024-09-14T04:48:25.623 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:25.658 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:25.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:25.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:25.759 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:25.798 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:25.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:25.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:25.900 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:25.931 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:25.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:25.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:26.033 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:26.066 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:26.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:26.067 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:26.168 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:26.211 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:26.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:26.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:26.313 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:26.348 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:26.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:26.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:26.450 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:26.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:26.483 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:26.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:26.550 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 25 16 50740 50724 126.783 111.438 0.00817875 0.00788234 2024-09-14T04:48:26.584 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:26.624 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:26.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:26.625 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:26.711 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:48:26.726 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:26.753 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:26.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:26.754 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:26.854 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:26.887 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:26.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:26.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:26.989 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:27.016 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:26 smithi039 ceph-mon[106470]: pgmap v2565: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s; 978/132819 objects misplaced (0.736%); 3.9 MiB/s, 62 objects/s recovering 2024-09-14T04:48:27.016 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:26 smithi039 ceph-mon[111235]: pgmap v2565: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s; 978/132819 objects misplaced (0.736%); 3.9 MiB/s, 62 objects/s recovering 2024-09-14T04:48:27.018 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:27.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:27.019 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:27.120 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:27.165 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:27.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:27.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:27.267 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:27.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:26 smithi129 ceph-mon[101316]: pgmap v2565: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 108 MiB/s wr, 1.73k op/s; 978/132819 objects misplaced (0.736%); 3.9 MiB/s, 62 objects/s recovering 2024-09-14T04:48:27.307 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:27.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:27.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:27.409 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:27.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:27.443 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:27.443 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:27.467 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:48:27.467 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:48:27.509 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:48:27.509 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:48:27.544 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:27.550 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 26 16 52388 52372 125.868 103 0.0191223 0.00793893 2024-09-14T04:48:27.563 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:48:27.563 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:48:27.587 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:27.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:27.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:27.608 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:48:27.608 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:48:27.643 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:48:27.643 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:48:27.676 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:48:27.677 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:48:27.689 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:27.711 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:48:27.712 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:48:27.723 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:27.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:27.724 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:27.754 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:48:27.754 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:48:27.824 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:27.866 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:27.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:27.867 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:27.968 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:27.998 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:27.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:27.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:28.099 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:28.144 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:28.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:28.145 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:28.160 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:27 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1586951451' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:48:28.160 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:27 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1586951451' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:48:28.227 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:48:28.227 INFO:teuthology.orchestra.run.smithi039.stdout: "size": 20, 2024-09-14T04:48:28.227 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 600, 2024-09-14T04:48:28.227 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [ 2024-09-14T04:48:28.227 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.227 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233353 53.5 53:a6a852bc:::benchmark_data_smithi039_224949_object103260:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1354)", 2024-09-14T04:48:28.227 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.817844+0000", 2024-09-14T04:48:28.227 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 383.40365822899997, 2024-09-14T04:48:28.228 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.249583763, 2024-09-14T04:48:28.228 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:48:28.228 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:48:28.228 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T04:48:28.228 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:48:28.228 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:48:28.228 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:48:28.228 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233353 2024-09-14T04:48:28.228 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.228 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:48:28.228 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.228 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:48:28.229 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.817844+0000", 2024-09-14T04:48:28.229 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.229 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.229 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.229 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:48:28.229 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.817844+0000", 2024-09-14T04:48:28.229 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.229 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.229 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.229 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:48:28.229 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.817845+0000", 2024-09-14T04:48:28.229 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.502e-06 2024-09-14T04:48:28.229 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.230 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.230 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:48:28.230 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.817849+0000", 2024-09-14T04:48:28.230 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.9750000000000001e-06 2024-09-14T04:48:28.230 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.230 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.230 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:48:28.230 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.817868+0000", 2024-09-14T04:48:28.230 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9162000000000001e-05 2024-09-14T04:48:28.230 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.230 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.230 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:48:28.230 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.817877+0000", 2024-09-14T04:48:28.230 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.7919999999999998e-06 2024-09-14T04:48:28.231 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.231 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.231 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.231 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.817984+0000", 2024-09-14T04:48:28.231 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000106606 2024-09-14T04:48:28.231 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.231 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.231 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:48:28.231 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.818094+0000", 2024-09-14T04:48:28.231 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000110072 2024-09-14T04:48:28.231 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.231 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.231 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:48:28.232 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.822688+0000", 2024-09-14T04:48:28.232 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0045945530000000004 2024-09-14T04:48:28.232 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.232 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.232 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.232 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.843026+0000", 2024-09-14T04:48:28.232 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.020337233 2024-09-14T04:48:28.232 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.232 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.232 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:48:28.232 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.843037+0000", 2024-09-14T04:48:28.232 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1022000000000001e-05 2024-09-14T04:48:28.233 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.233 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.233 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.233 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.813667+0000", 2024-09-14T04:48:28.233 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.97063038499999998 2024-09-14T04:48:28.233 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.233 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.233 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:48:28.233 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.813680+0000", 2024-09-14T04:48:28.233 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3013000000000001e-05 2024-09-14T04:48:28.233 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.233 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.234 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.234 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.015538+0000", 2024-09-14T04:48:28.234 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.201857916 2024-09-14T04:48:28.234 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.234 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.234 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for missing object", 2024-09-14T04:48:28.234 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.015727+0000", 2024-09-14T04:48:28.234 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000188863 2024-09-14T04:48:28.234 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.234 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.234 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.234 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.067130+0000", 2024-09-14T04:48:28.235 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.051403345000000003 2024-09-14T04:48:28.235 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.235 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.235 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:48:28.235 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.067427+0000", 2024-09-14T04:48:28.235 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00029732400000000001 2024-09-14T04:48:28.235 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.235 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:48:28.235 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.235 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.235 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.235 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233394 53.5 53:ae57f18f:::benchmark_data_smithi039_224949_object103301:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-09-14T04:48:28.235 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.894024+0000", 2024-09-14T04:48:28.235 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 383.327478348, 2024-09-14T04:48:28.236 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1667164959999998, 2024-09-14T04:48:28.236 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:48:28.236 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:48:28.236 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:48:28.236 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:48:28.236 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:48:28.236 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:48:28.236 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233394 2024-09-14T04:48:28.236 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.236 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:48:28.236 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.236 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:48:28.236 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894024+0000", 2024-09-14T04:48:28.237 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.237 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.237 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.237 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:48:28.237 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894024+0000", 2024-09-14T04:48:28.237 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.237 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.237 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.237 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:48:28.237 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894025+0000", 2024-09-14T04:48:28.238 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.548e-06 2024-09-14T04:48:28.238 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.238 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.238 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:48:28.238 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894031+0000", 2024-09-14T04:48:28.238 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5790000000000003e-06 2024-09-14T04:48:28.238 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.238 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.238 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:48:28.238 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894042+0000", 2024-09-14T04:48:28.238 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1494e-05 2024-09-14T04:48:28.238 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.238 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.238 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:48:28.239 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894050+0000", 2024-09-14T04:48:28.239 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6969999999999993e-06 2024-09-14T04:48:28.239 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.239 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.239 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.239 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894114+0000", 2024-09-14T04:48:28.239 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3689000000000006e-05 2024-09-14T04:48:28.239 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.239 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.239 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:48:28.239 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894123+0000", 2024-09-14T04:48:28.239 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.7480000000000005e-06 2024-09-14T04:48:28.239 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.239 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.240 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.240 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.813712+0000", 2024-09-14T04:48:28.240 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.91958880200000004 2024-09-14T04:48:28.240 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.240 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.240 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:48:28.240 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.813719+0000", 2024-09-14T04:48:28.240 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.6320000000000002e-06 2024-09-14T04:48:28.240 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.240 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.240 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.240 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.021411+0000", 2024-09-14T04:48:28.240 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.207691874 2024-09-14T04:48:28.241 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.241 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.241 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:48:28.241 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.021549+0000", 2024-09-14T04:48:28.241 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013877299999999999 2024-09-14T04:48:28.241 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.241 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.241 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,3,7", 2024-09-14T04:48:28.241 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.022037+0000", 2024-09-14T04:48:28.241 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00048770299999999998 2024-09-14T04:48:28.241 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.241 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.241 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:48:28.241 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.037131+0000", 2024-09-14T04:48:28.242 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.015093766999999999 2024-09-14T04:48:28.242 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.242 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.242 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.242 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.037173+0000", 2024-09-14T04:48:28.242 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2518999999999997e-05 2024-09-14T04:48:28.242 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.242 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.242 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.242 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.049744+0000", 2024-09-14T04:48:28.242 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.012570789000000001 2024-09-14T04:48:28.242 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.242 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.243 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.243 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.060452+0000", 2024-09-14T04:48:28.243 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.010707836 2024-09-14T04:48:28.243 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.243 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.243 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:48:28.243 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.060604+0000", 2024-09-14T04:48:28.243 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000152032 2024-09-14T04:48:28.243 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.243 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.243 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:48:28.243 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.060740+0000", 2024-09-14T04:48:28.244 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000136014 2024-09-14T04:48:28.244 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.244 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:48:28.244 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.244 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.244 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.244 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233404 53.5 53:a80a488e:::benchmark_data_smithi039_224949_object103311:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-09-14T04:48:28.244 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.904066+0000", 2024-09-14T04:48:28.244 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 383.31743571200002, 2024-09-14T04:48:28.244 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1569935720000002, 2024-09-14T04:48:28.244 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:48:28.245 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:48:28.245 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:48:28.245 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:48:28.245 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:48:28.245 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:48:28.245 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233404 2024-09-14T04:48:28.245 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.245 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:48:28.245 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.245 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:48:28.245 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.904066+0000", 2024-09-14T04:48:28.245 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.245 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.245 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.246 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:48:28.246 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.904066+0000", 2024-09-14T04:48:28.246 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.246 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.246 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.246 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:48:28.246 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.904068+0000", 2024-09-14T04:48:28.246 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3939999999999999e-06 2024-09-14T04:48:28.246 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.247 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.247 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:48:28.247 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.904073+0000", 2024-09-14T04:48:28.247 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1410000000000004e-06 2024-09-14T04:48:28.247 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.247 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.247 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:48:28.247 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.904089+0000", 2024-09-14T04:48:28.247 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6464999999999999e-05 2024-09-14T04:48:28.247 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.247 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.247 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:48:28.248 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.904097+0000", 2024-09-14T04:48:28.248 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.3299999999999999e-06 2024-09-14T04:48:28.248 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.248 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.248 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.248 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.904162+0000", 2024-09-14T04:48:28.248 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.4561999999999996e-05 2024-09-14T04:48:28.248 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.248 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.248 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:48:28.248 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.904171+0000", 2024-09-14T04:48:28.249 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.4010000000000003e-06 2024-09-14T04:48:28.249 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.249 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.249 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.249 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.813742+0000", 2024-09-14T04:48:28.249 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.90957067000000003 2024-09-14T04:48:28.249 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.249 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.249 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:48:28.249 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.813749+0000", 2024-09-14T04:48:28.249 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.3000000000000004e-06 2024-09-14T04:48:28.249 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.250 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.250 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.250 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.030045+0000", 2024-09-14T04:48:28.250 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.216295345 2024-09-14T04:48:28.250 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.250 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.250 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:48:28.251 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.030205+0000", 2024-09-14T04:48:28.251 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00016016500000000001 2024-09-14T04:48:28.251 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.251 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.251 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,3,7", 2024-09-14T04:48:28.251 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.030346+0000", 2024-09-14T04:48:28.251 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000140822 2024-09-14T04:48:28.251 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.251 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.251 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.251 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.037216+0000", 2024-09-14T04:48:28.251 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0068704179999999997 2024-09-14T04:48:28.251 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.251 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.252 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:48:28.252 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.043582+0000", 2024-09-14T04:48:28.252 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0063658120000000002 2024-09-14T04:48:28.252 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.252 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.252 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.252 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.050030+0000", 2024-09-14T04:48:28.252 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.006448311 2024-09-14T04:48:28.252 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.252 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.252 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.252 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.060809+0000", 2024-09-14T04:48:28.252 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.010778375999999999 2024-09-14T04:48:28.252 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.253 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.253 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:48:28.253 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.060876+0000", 2024-09-14T04:48:28.253 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7692000000000002e-05 2024-09-14T04:48:28.253 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.253 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.253 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:48:28.253 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.061060+0000", 2024-09-14T04:48:28.253 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000183368 2024-09-14T04:48:28.253 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.253 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:48:28.253 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.254 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.254 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.254 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233415 53.5 53:a93a97f2:::benchmark_data_smithi039_224949_object103322:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-09-14T04:48:28.254 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.931528+0000", 2024-09-14T04:48:28.254 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 383.289973573, 2024-09-14T04:48:28.254 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1304777339999998, 2024-09-14T04:48:28.254 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:48:28.254 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:48:28.254 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:48:28.254 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:48:28.254 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:48:28.254 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:48:28.254 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233415 2024-09-14T04:48:28.254 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.254 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:48:28.255 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.255 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:48:28.255 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.931528+0000", 2024-09-14T04:48:28.255 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.255 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.255 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.255 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:48:28.255 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.931528+0000", 2024-09-14T04:48:28.255 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.255 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.255 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.255 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:48:28.256 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.931530+0000", 2024-09-14T04:48:28.256 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3790000000000001e-06 2024-09-14T04:48:28.256 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.256 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.256 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:48:28.256 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.931535+0000", 2024-09-14T04:48:28.256 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1320000000000002e-06 2024-09-14T04:48:28.256 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.256 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.256 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:48:28.256 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.931547+0000", 2024-09-14T04:48:28.257 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1756000000000001e-05 2024-09-14T04:48:28.257 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.257 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.257 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:48:28.257 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.931552+0000", 2024-09-14T04:48:28.257 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4199999999999998e-06 2024-09-14T04:48:28.257 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.257 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.257 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.257 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.931631+0000", 2024-09-14T04:48:28.257 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9358999999999997e-05 2024-09-14T04:48:28.257 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.257 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.257 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:48:28.258 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:28.259 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.931643+0000", 2024-09-14T04:48:28.259 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1990999999999999e-05 2024-09-14T04:48:28.259 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.259 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.259 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.259 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.813776+0000", 2024-09-14T04:48:28.259 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.88213241600000003 2024-09-14T04:48:28.259 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.259 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.259 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:48:28.259 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.813781+0000", 2024-09-14T04:48:28.260 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6760000000000004e-06 2024-09-14T04:48:28.260 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.260 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.260 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.260 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.033700+0000", 2024-09-14T04:48:28.260 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.219918396 2024-09-14T04:48:28.260 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.260 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.260 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:48:28.260 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.033813+0000", 2024-09-14T04:48:28.260 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000113175 2024-09-14T04:48:28.260 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.260 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.261 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,3,7", 2024-09-14T04:48:28.261 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.033941+0000", 2024-09-14T04:48:28.261 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012790300000000001 2024-09-14T04:48:28.261 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.261 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.261 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.261 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.041351+0000", 2024-09-14T04:48:28.261 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0074097670000000003 2024-09-14T04:48:28.261 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.261 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.261 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:48:28.261 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.043590+0000", 2024-09-14T04:48:28.261 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0022397670000000001 2024-09-14T04:48:28.262 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.262 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.262 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.262 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.052263+0000", 2024-09-14T04:48:28.262 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0086722819999999999 2024-09-14T04:48:28.262 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.262 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.262 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.262 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.061172+0000", 2024-09-14T04:48:28.262 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0089090799999999998 2024-09-14T04:48:28.262 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.262 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.262 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:48:28.262 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.061224+0000", 2024-09-14T04:48:28.263 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.198e-05 2024-09-14T04:48:28.263 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.263 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.263 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:48:28.263 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.062006+0000", 2024-09-14T04:48:28.263 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00078225500000000004 2024-09-14T04:48:28.263 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.263 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:48:28.263 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.263 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.263 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.263 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233355 53.1 53:81e7c0a9:::benchmark_data_smithi039_224949_object103262:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1356)", 2024-09-14T04:48:28.263 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:05.820399+0000", 2024-09-14T04:48:28.263 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 382.40110253500001, 2024-09-14T04:48:28.264 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2541154749999999, 2024-09-14T04:48:28.264 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:48:28.264 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:48:28.264 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:48:28.264 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:48:28.264 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:48:28.264 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:48:28.264 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233355 2024-09-14T04:48:28.264 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.264 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:48:28.264 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.264 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:48:28.264 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820399+0000", 2024-09-14T04:48:28.265 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.265 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.265 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.265 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:48:28.265 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820399+0000", 2024-09-14T04:48:28.265 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.265 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.265 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.265 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:48:28.265 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820401+0000", 2024-09-14T04:48:28.265 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.44e-06 2024-09-14T04:48:28.265 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.266 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.266 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:48:28.266 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820405+0000", 2024-09-14T04:48:28.266 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5040000000000004e-06 2024-09-14T04:48:28.266 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.266 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.266 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:48:28.266 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820418+0000", 2024-09-14T04:48:28.266 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2515e-05 2024-09-14T04:48:28.266 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.266 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.266 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:48:28.266 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820426+0000", 2024-09-14T04:48:28.266 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.5199999999999997e-06 2024-09-14T04:48:28.267 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.267 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.267 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.267 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.821400+0000", 2024-09-14T04:48:28.267 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00097377300000000004 2024-09-14T04:48:28.267 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.267 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.267 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:48:28.267 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.821409+0000", 2024-09-14T04:48:28.267 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.2920000000000001e-06 2024-09-14T04:48:28.267 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.267 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.267 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.267 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.041896+0000", 2024-09-14T04:48:28.268 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2204865730000001 2024-09-14T04:48:28.268 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.268 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.268 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:48:28.268 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.045201+0000", 2024-09-14T04:48:28.268 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0033054400000000002 2024-09-14T04:48:28.268 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.268 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.268 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,5,6", 2024-09-14T04:48:28.268 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.045657+0000", 2024-09-14T04:48:28.268 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000455861 2024-09-14T04:48:28.268 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.268 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.269 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.269 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.065205+0000", 2024-09-14T04:48:28.269 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.019547563 2024-09-14T04:48:28.269 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.269 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.269 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:48:28.269 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.065220+0000", 2024-09-14T04:48:28.269 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4728e-05 2024-09-14T04:48:28.269 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.269 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.269 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.269 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.073947+0000", 2024-09-14T04:48:28.269 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.008727304 2024-09-14T04:48:28.270 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.270 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.270 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.270 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.074187+0000", 2024-09-14T04:48:28.270 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00024006799999999999 2024-09-14T04:48:28.270 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.270 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.270 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:48:28.270 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.074326+0000", 2024-09-14T04:48:28.270 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000139216 2024-09-14T04:48:28.270 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.270 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.270 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:48:28.271 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.074515+0000", 2024-09-14T04:48:28.271 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00018867800000000001 2024-09-14T04:48:28.271 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.271 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:48:28.271 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.271 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.271 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.271 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233406 53.1 53:86743dc5:::benchmark_data_smithi039_224949_object103313:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1356)", 2024-09-14T04:48:28.271 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:05.820440+0000", 2024-09-14T04:48:28.271 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 382.40106186100002, 2024-09-14T04:48:28.271 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2550835579999999, 2024-09-14T04:48:28.271 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:48:28.271 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:48:28.272 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:48:28.272 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:48:28.272 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:48:28.272 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:48:28.272 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233406 2024-09-14T04:48:28.272 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.272 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:48:28.272 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.272 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:48:28.273 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820440+0000", 2024-09-14T04:48:28.273 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.273 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.273 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.273 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:48:28.273 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820440+0000", 2024-09-14T04:48:28.273 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.273 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.273 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.273 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:48:28.273 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820440+0000", 2024-09-14T04:48:28.274 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.8999999999999997e-07 2024-09-14T04:48:28.274 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.274 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.274 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:48:28.274 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820443+0000", 2024-09-14T04:48:28.274 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4420000000000001e-06 2024-09-14T04:48:28.274 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.274 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.274 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:48:28.274 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820460+0000", 2024-09-14T04:48:28.274 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7183000000000001e-05 2024-09-14T04:48:28.274 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.275 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.275 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:48:28.275 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820467+0000", 2024-09-14T04:48:28.275 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1810000000000004e-06 2024-09-14T04:48:28.275 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.275 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.275 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.275 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.821435+0000", 2024-09-14T04:48:28.275 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00096804599999999997 2024-09-14T04:48:28.275 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.275 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.275 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:48:28.275 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.821443+0000", 2024-09-14T04:48:28.275 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.2010000000000002e-06 2024-09-14T04:48:28.276 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.276 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.276 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.276 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.058430+0000", 2024-09-14T04:48:28.276 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.236987088 2024-09-14T04:48:28.276 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.276 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.276 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:48:28.276 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.058568+0000", 2024-09-14T04:48:28.276 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013826100000000001 2024-09-14T04:48:28.276 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.276 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.276 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,5,6", 2024-09-14T04:48:28.276 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.058665+0000", 2024-09-14T04:48:28.277 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.6904999999999995e-05 2024-09-14T04:48:28.277 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.277 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.277 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:48:28.277 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.065226+0000", 2024-09-14T04:48:28.277 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0065607410000000001 2024-09-14T04:48:28.277 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.277 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.277 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.277 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.075244+0000", 2024-09-14T04:48:28.277 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.010018216 2024-09-14T04:48:28.277 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.277 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.277 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.277 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.075286+0000", 2024-09-14T04:48:28.278 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.1987000000000003e-05 2024-09-14T04:48:28.278 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.278 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.278 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.278 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.075346+0000", 2024-09-14T04:48:28.278 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0649999999999997e-05 2024-09-14T04:48:28.278 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.278 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.278 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:48:28.278 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.075409+0000", 2024-09-14T04:48:28.278 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2948999999999996e-05 2024-09-14T04:48:28.278 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.278 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.278 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:48:28.278 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.075524+0000", 2024-09-14T04:48:28.279 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000114218 2024-09-14T04:48:28.279 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.279 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:48:28.279 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.279 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.279 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.279 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233412 53.1 53:8a59a845:::benchmark_data_smithi039_224949_object103319:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1356)", 2024-09-14T04:48:28.279 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:05.820476+0000", 2024-09-14T04:48:28.279 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 382.40102578099999, 2024-09-14T04:48:28.279 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2554734700000001, 2024-09-14T04:48:28.279 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:48:28.279 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:48:28.279 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:48:28.279 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:48:28.280 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:48:28.280 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:48:28.280 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233412 2024-09-14T04:48:28.280 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.280 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:48:28.280 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.280 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:48:28.280 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820476+0000", 2024-09-14T04:48:28.280 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.280 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.280 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.280 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:48:28.280 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820476+0000", 2024-09-14T04:48:28.281 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.281 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.281 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.281 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:48:28.281 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820477+0000", 2024-09-14T04:48:28.281 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.0900000000000002e-07 2024-09-14T04:48:28.281 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.281 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.281 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:48:28.281 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820479+0000", 2024-09-14T04:48:28.281 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.7410000000000001e-06 2024-09-14T04:48:28.281 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.281 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.281 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:48:28.282 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820486+0000", 2024-09-14T04:48:28.282 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8789999999999997e-06 2024-09-14T04:48:28.282 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.282 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.282 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:48:28.282 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820492+0000", 2024-09-14T04:48:28.282 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5060000000000003e-06 2024-09-14T04:48:28.282 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.282 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.282 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.282 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.821477+0000", 2024-09-14T04:48:28.282 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00098510799999999995 2024-09-14T04:48:28.282 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.282 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.283 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:48:28.283 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.821484+0000", 2024-09-14T04:48:28.283 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.34e-06 2024-09-14T04:48:28.283 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.283 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.283 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.283 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.061870+0000", 2024-09-14T04:48:28.283 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.240385472 2024-09-14T04:48:28.283 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.283 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.283 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:48:28.283 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.061995+0000", 2024-09-14T04:48:28.283 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012508899999999999 2024-09-14T04:48:28.283 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.284 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.284 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,5,6", 2024-09-14T04:48:28.284 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.062117+0000", 2024-09-14T04:48:28.284 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000122459 2024-09-14T04:48:28.284 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.284 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.284 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:48:28.284 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.073961+0000", 2024-09-14T04:48:28.284 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.011843907000000001 2024-09-14T04:48:28.284 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.285 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.285 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.285 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.075568+0000", 2024-09-14T04:48:28.285 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001607362 2024-09-14T04:48:28.285 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.285 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.286 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.286 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.075662+0000", 2024-09-14T04:48:28.286 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.3374000000000004e-05 2024-09-14T04:48:28.286 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.286 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.286 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.286 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.075708+0000", 2024-09-14T04:48:28.286 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6300000000000001e-05 2024-09-14T04:48:28.286 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.286 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.286 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:48:28.286 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.075763+0000", 2024-09-14T04:48:28.287 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4898000000000002e-05 2024-09-14T04:48:28.287 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.287 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.287 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:48:28.287 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.075950+0000", 2024-09-14T04:48:28.287 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00018652600000000001 2024-09-14T04:48:28.288 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.288 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:48:28.288 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.288 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.288 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.288 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242571 53.9 53:9bbdb0f8:::benchmark_data_smithi039_224949_object112478:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1362)", 2024-09-14T04:48:28.288 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:19.469714+0000", 2024-09-14T04:48:28.288 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 368.75178741500002, 2024-09-14T04:48:28.288 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0060407760000001, 2024-09-14T04:48:28.288 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:48:28.289 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:48:28.289 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:48:28.289 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:48:28.289 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:48:28.289 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:48:28.289 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242571 2024-09-14T04:48:28.289 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.289 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:48:28.289 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.289 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:48:28.290 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.469714+0000", 2024-09-14T04:48:28.290 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.290 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.290 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.290 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:48:28.290 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.469714+0000", 2024-09-14T04:48:28.290 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.290 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.290 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.290 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:48:28.290 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.469716+0000", 2024-09-14T04:48:28.290 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.367e-06 2024-09-14T04:48:28.291 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.291 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.291 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:48:28.291 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.469721+0000", 2024-09-14T04:48:28.291 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4219999999999999e-06 2024-09-14T04:48:28.291 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.291 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.291 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:48:28.291 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.469734+0000", 2024-09-14T04:48:28.291 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2826999999999999e-05 2024-09-14T04:48:28.291 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.291 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.291 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:48:28.291 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.469740+0000", 2024-09-14T04:48:28.291 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.9800000000000003e-06 2024-09-14T04:48:28.292 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.292 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.292 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.292 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.469802+0000", 2024-09-14T04:48:28.292 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1816999999999994e-05 2024-09-14T04:48:28.292 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.292 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.292 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:48:28.292 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.469818+0000", 2024-09-14T04:48:28.292 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5829999999999999e-05 2024-09-14T04:48:28.292 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.292 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.293 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.293 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.469925+0000", 2024-09-14T04:48:28.293 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0001073389999999 2024-09-14T04:48:28.293 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.293 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.293 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:48:28.293 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.470024+0000", 2024-09-14T04:48:28.293 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.8766999999999998e-05 2024-09-14T04:48:28.293 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.293 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.293 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 5,6", 2024-09-14T04:48:28.293 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.470131+0000", 2024-09-14T04:48:28.294 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000106746 2024-09-14T04:48:28.294 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.294 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.294 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.294 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.473316+0000", 2024-09-14T04:48:28.294 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0031856670000000001 2024-09-14T04:48:28.294 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.294 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.294 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.294 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.473374+0000", 2024-09-14T04:48:28.294 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7612999999999999e-05 2024-09-14T04:48:28.294 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.294 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.294 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:48:28.295 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.475684+0000", 2024-09-14T04:48:28.295 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0023106250000000002 2024-09-14T04:48:28.295 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.295 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.295 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:48:28.295 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.475716+0000", 2024-09-14T04:48:28.295 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.1748000000000002e-05 2024-09-14T04:48:28.295 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.295 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.295 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:48:28.295 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.475755+0000", 2024-09-14T04:48:28.295 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.9028e-05 2024-09-14T04:48:28.295 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.296 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:48:28.296 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.296 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.296 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.296 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242577 53.9 53:9813f1d4:::benchmark_data_smithi039_224949_object112484:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1362)", 2024-09-14T04:48:28.296 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:19.476824+0000", 2024-09-14T04:48:28.296 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 368.74467762299997, 2024-09-14T04:48:28.296 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0070515010000001, 2024-09-14T04:48:28.296 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:48:28.296 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:48:28.296 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:48:28.296 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:48:28.297 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:48:28.297 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:48:28.297 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242577 2024-09-14T04:48:28.297 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.297 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:48:28.297 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.297 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:48:28.297 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.476824+0000", 2024-09-14T04:48:28.297 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.297 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.297 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.297 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:48:28.298 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.476824+0000", 2024-09-14T04:48:28.298 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.298 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.298 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.298 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:48:28.298 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.476825+0000", 2024-09-14T04:48:28.298 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2470000000000001e-06 2024-09-14T04:48:28.298 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.298 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.298 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:48:28.298 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.476830+0000", 2024-09-14T04:48:28.299 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3019999999999996e-06 2024-09-14T04:48:28.299 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.299 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.299 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:48:28.299 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.476841+0000", 2024-09-14T04:48:28.299 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1508999999999999e-05 2024-09-14T04:48:28.299 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.299 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.299 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:48:28.299 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.476849+0000", 2024-09-14T04:48:28.299 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0010000000000001e-06 2024-09-14T04:48:28.299 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.299 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.299 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.300 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.476908+0000", 2024-09-14T04:48:28.300 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.8530999999999999e-05 2024-09-14T04:48:28.300 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.300 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.300 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:48:28.300 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.476917+0000", 2024-09-14T04:48:28.300 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.3479999999999993e-06 2024-09-14T04:48:28.300 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.300 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.300 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.300 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.473414+0000", 2024-09-14T04:48:28.300 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.99649714599999994 2024-09-14T04:48:28.300 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.300 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.301 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:48:28.301 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.473490+0000", 2024-09-14T04:48:28.301 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.5517000000000002e-05 2024-09-14T04:48:28.301 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.301 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.301 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 5,6", 2024-09-14T04:48:28.301 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.473584+0000", 2024-09-14T04:48:28.301 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.3749e-05 2024-09-14T04:48:28.301 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.301 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.301 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.301 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.478047+0000", 2024-09-14T04:48:28.301 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0044635760000000003 2024-09-14T04:48:28.302 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.302 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.302 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:48:28.302 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.480787+0000", 2024-09-14T04:48:28.302 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0027394759999999998 2024-09-14T04:48:28.302 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.302 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.302 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.302 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.483767+0000", 2024-09-14T04:48:28.302 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.002980496 2024-09-14T04:48:28.302 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.302 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.302 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:48:28.302 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.483820+0000", 2024-09-14T04:48:28.303 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3022999999999997e-05 2024-09-14T04:48:28.303 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.303 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.303 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:48:28.303 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.483876+0000", 2024-09-14T04:48:28.303 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.558e-05 2024-09-14T04:48:28.303 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.303 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:48:28.303 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.303 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.303 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.303 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242621 53.9 53:95316125:::benchmark_data_smithi039_224949_object112528:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1362)", 2024-09-14T04:48:28.303 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:19.531111+0000", 2024-09-14T04:48:28.303 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 368.69039097899997, 2024-09-14T04:48:28.304 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.95593541000000004, 2024-09-14T04:48:28.304 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:48:28.304 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:48:28.305 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:27 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1586951451' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:48:28.305 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:48:28.305 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:48:28.306 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:48:28.306 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:48:28.306 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242621 2024-09-14T04:48:28.306 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.306 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:48:28.306 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.306 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:48:28.306 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.531111+0000", 2024-09-14T04:48:28.306 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.306 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.306 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.307 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:48:28.307 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.531111+0000", 2024-09-14T04:48:28.307 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.307 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.307 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.307 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:48:28.307 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.531112+0000", 2024-09-14T04:48:28.307 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4169999999999999e-06 2024-09-14T04:48:28.307 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.307 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.308 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:48:28.308 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.531116+0000", 2024-09-14T04:48:28.308 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.749e-06 2024-09-14T04:48:28.308 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.308 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.308 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:48:28.308 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.531131+0000", 2024-09-14T04:48:28.308 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5288e-05 2024-09-14T04:48:28.308 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.308 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.308 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:48:28.309 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.531139+0000", 2024-09-14T04:48:28.309 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.4800000000000004e-06 2024-09-14T04:48:28.309 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.309 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.309 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.310 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.531207+0000", 2024-09-14T04:48:28.310 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8552000000000004e-05 2024-09-14T04:48:28.310 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.310 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.310 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:48:28.311 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.531216+0000", 2024-09-14T04:48:28.311 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.1519999999999992e-06 2024-09-14T04:48:28.311 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.311 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.311 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.311 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.475804+0000", 2024-09-14T04:48:28.311 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.94458857399999996 2024-09-14T04:48:28.311 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.311 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.312 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:48:28.312 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.475887+0000", 2024-09-14T04:48:28.312 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.3177999999999995e-05 2024-09-14T04:48:28.312 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.312 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.312 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 5,6", 2024-09-14T04:48:28.312 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.475960+0000", 2024-09-14T04:48:28.312 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.2587000000000001e-05 2024-09-14T04:48:28.312 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.312 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.312 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.312 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.478124+0000", 2024-09-14T04:48:28.313 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.002164179 2024-09-14T04:48:28.313 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.313 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.313 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.313 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.483951+0000", 2024-09-14T04:48:28.313 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0058273999999999999 2024-09-14T04:48:28.313 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.313 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.313 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:48:28.313 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.486948+0000", 2024-09-14T04:48:28.313 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0029961509999999998 2024-09-14T04:48:28.313 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.313 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.314 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:48:28.314 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.487003+0000", 2024-09-14T04:48:28.314 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5179000000000001e-05 2024-09-14T04:48:28.314 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.314 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.314 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:48:28.314 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.487046+0000", 2024-09-14T04:48:28.314 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3523999999999999e-05 2024-09-14T04:48:28.314 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.314 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:48:28.314 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.314 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.314 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.314 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242632 53.9 53:9195c169:::benchmark_data_smithi039_224949_object112539:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1362)", 2024-09-14T04:48:28.314 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:19.544391+0000", 2024-09-14T04:48:28.315 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 368.677110685, 2024-09-14T04:48:28.315 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.94285940199999996, 2024-09-14T04:48:28.315 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:48:28.315 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:48:28.315 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:48:28.315 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:48:28.315 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:48:28.315 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:48:28.315 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242632 2024-09-14T04:48:28.315 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.315 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:48:28.315 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.315 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:48:28.315 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.544391+0000", 2024-09-14T04:48:28.315 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.316 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.316 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.316 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:48:28.316 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.544391+0000", 2024-09-14T04:48:28.316 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.316 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.316 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.316 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:48:28.316 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.544393+0000", 2024-09-14T04:48:28.316 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.686e-06 2024-09-14T04:48:28.316 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.316 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.316 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:48:28.316 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.544398+0000", 2024-09-14T04:48:28.317 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7650000000000001e-06 2024-09-14T04:48:28.317 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.317 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.317 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:48:28.317 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.544410+0000", 2024-09-14T04:48:28.317 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.206e-05 2024-09-14T04:48:28.317 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.317 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.317 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:48:28.317 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.544417+0000", 2024-09-14T04:48:28.317 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.0820000000000001e-06 2024-09-14T04:48:28.317 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.317 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.317 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.318 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.545142+0000", 2024-09-14T04:48:28.318 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00072522999999999999 2024-09-14T04:48:28.318 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.318 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.318 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:48:28.318 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.545151+0000", 2024-09-14T04:48:28.318 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.9840000000000007e-06 2024-09-14T04:48:28.318 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.318 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.318 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.318 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.478163+0000", 2024-09-14T04:48:28.318 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.93301250199999997 2024-09-14T04:48:28.318 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.318 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.318 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:48:28.319 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.478279+0000", 2024-09-14T04:48:28.319 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000115438 2024-09-14T04:48:28.319 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.319 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.319 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 5,6", 2024-09-14T04:48:28.319 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.478413+0000", 2024-09-14T04:48:28.319 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000134447 2024-09-14T04:48:28.319 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.319 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.319 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.319 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.483916+0000", 2024-09-14T04:48:28.319 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.005502547 2024-09-14T04:48:28.319 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.320 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.320 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.320 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.487095+0000", 2024-09-14T04:48:28.320 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0031794470000000002 2024-09-14T04:48:28.320 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.320 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.320 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:48:28.320 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.487158+0000", 2024-09-14T04:48:28.320 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3082000000000004e-05 2024-09-14T04:48:28.320 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.320 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.320 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:48:28.320 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.487209+0000", 2024-09-14T04:48:28.321 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.0562999999999997e-05 2024-09-14T04:48:28.321 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.321 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.321 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:48:28.321 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.487251+0000", 2024-09-14T04:48:28.321 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.1569000000000001e-05 2024-09-14T04:48:28.321 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.321 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:48:28.321 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.321 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.321 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.321 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242677 53.9 53:9e1c48cc:::benchmark_data_smithi039_224949_object112584:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1362)", 2024-09-14T04:48:28.321 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:19.589100+0000", 2024-09-14T04:48:28.322 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 368.63240207699999, 2024-09-14T04:48:28.322 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.89828418799999998, 2024-09-14T04:48:28.322 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:48:28.322 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:48:28.322 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:48:28.322 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:48:28.322 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:48:28.322 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:48:28.322 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242677 2024-09-14T04:48:28.322 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.322 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:48:28.322 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.322 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:48:28.323 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.589100+0000", 2024-09-14T04:48:28.323 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.323 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.323 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.323 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:48:28.323 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.589100+0000", 2024-09-14T04:48:28.323 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.323 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.323 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.323 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:48:28.323 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.589101+0000", 2024-09-14T04:48:28.323 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.677e-06 2024-09-14T04:48:28.323 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.324 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.324 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:48:28.324 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.589106+0000", 2024-09-14T04:48:28.324 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2819999999999998e-06 2024-09-14T04:48:28.324 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.324 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.324 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:48:28.324 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.589117+0000", 2024-09-14T04:48:28.324 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1618e-05 2024-09-14T04:48:28.324 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.324 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.324 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:48:28.324 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.589125+0000", 2024-09-14T04:48:28.325 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1799999999999999e-06 2024-09-14T04:48:28.325 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.325 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.325 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.325 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.589178+0000", 2024-09-14T04:48:28.325 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3315e-05 2024-09-14T04:48:28.325 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.325 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.325 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:48:28.325 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.589186+0000", 2024-09-14T04:48:28.325 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.8369999999999997e-06 2024-09-14T04:48:28.325 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.325 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.325 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.326 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.480814+0000", 2024-09-14T04:48:28.326 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.89162806800000005 2024-09-14T04:48:28.326 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.326 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.326 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:48:28.326 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.480909+0000", 2024-09-14T04:48:28.326 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.4790999999999994e-05 2024-09-14T04:48:28.326 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.326 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.326 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 5,6", 2024-09-14T04:48:28.326 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.481014+0000", 2024-09-14T04:48:28.326 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000105525 2024-09-14T04:48:28.326 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.326 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.327 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.327 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.483991+0000", 2024-09-14T04:48:28.327 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0029772900000000001 2024-09-14T04:48:28.327 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.327 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.327 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:48:28.327 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.487257+0000", 2024-09-14T04:48:28.327 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0032661190000000001 2024-09-14T04:48:28.327 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.327 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.327 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.327 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.487315+0000", 2024-09-14T04:48:28.327 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7408999999999999e-05 2024-09-14T04:48:28.327 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.328 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.328 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:48:28.328 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.487345+0000", 2024-09-14T04:48:28.328 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.9709e-05 2024-09-14T04:48:28.328 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.328 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.328 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:48:28.328 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.487384+0000", 2024-09-14T04:48:28.328 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.9368e-05 2024-09-14T04:48:28.329 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.329 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:48:28.329 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.329 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.329 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.329 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242680 53.9 53:966d43da:::benchmark_data_smithi039_224949_object112587:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1362)", 2024-09-14T04:48:28.329 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:19.593251+0000", 2024-09-14T04:48:28.329 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 368.62825045099999, 2024-09-14T04:48:28.329 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.89431749000000005, 2024-09-14T04:48:28.329 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:48:28.329 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:48:28.329 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:48:28.329 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:48:28.330 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:48:28.330 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:48:28.330 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242680 2024-09-14T04:48:28.330 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.330 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:48:28.330 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.330 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:48:28.330 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.593251+0000", 2024-09-14T04:48:28.330 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.330 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.330 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.330 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:48:28.330 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.593251+0000", 2024-09-14T04:48:28.330 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.331 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.331 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.331 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:48:28.331 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.593253+0000", 2024-09-14T04:48:28.331 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1909999999999999e-06 2024-09-14T04:48:28.331 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.331 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.331 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:48:28.331 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.593257+0000", 2024-09-14T04:48:28.331 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3170000000000003e-06 2024-09-14T04:48:28.331 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.331 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.331 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:48:28.332 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.593268+0000", 2024-09-14T04:48:28.332 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1065e-05 2024-09-14T04:48:28.332 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.332 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.332 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:48:28.332 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.593276+0000", 2024-09-14T04:48:28.332 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9799999999999998e-06 2024-09-14T04:48:28.332 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.332 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.332 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.332 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.593320+0000", 2024-09-14T04:48:28.332 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4073000000000003e-05 2024-09-14T04:48:28.332 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.333 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.333 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:48:28.333 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.593325+0000", 2024-09-14T04:48:28.333 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.361e-06 2024-09-14T04:48:28.333 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.333 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.333 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.333 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.484037+0000", 2024-09-14T04:48:28.333 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.89071176699999999 2024-09-14T04:48:28.333 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.333 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.333 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:48:28.333 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.484149+0000", 2024-09-14T04:48:28.333 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000112237 2024-09-14T04:48:28.334 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.334 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.334 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 5,6", 2024-09-14T04:48:28.334 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.484274+0000", 2024-09-14T04:48:28.334 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000124914 2024-09-14T04:48:28.334 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.334 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.334 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.334 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.487146+0000", 2024-09-14T04:48:28.334 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0028716710000000001 2024-09-14T04:48:28.334 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.334 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.334 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.335 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.487438+0000", 2024-09-14T04:48:28.335 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000292465 2024-09-14T04:48:28.335 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.335 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.335 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:48:28.335 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.487487+0000", 2024-09-14T04:48:28.335 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.8780999999999999e-05 2024-09-14T04:48:28.335 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.335 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.335 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:48:28.335 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.487535+0000", 2024-09-14T04:48:28.335 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7750999999999997e-05 2024-09-14T04:48:28.335 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.335 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.336 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:48:28.336 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.487569+0000", 2024-09-14T04:48:28.336 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.3917000000000001e-05 2024-09-14T04:48:28.336 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.336 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:48:28.336 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.336 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.336 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.336 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242713 53.9 53:9fbaa905:::benchmark_data_smithi039_224949_object112620:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1362)", 2024-09-14T04:48:28.336 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:19.632065+0000", 2024-09-14T04:48:28.336 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 368.58943715100003, 2024-09-14T04:48:28.336 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.86213940700000002, 2024-09-14T04:48:28.336 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:48:28.337 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:48:28.337 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:48:28.337 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:48:28.337 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:48:28.337 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:48:28.337 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242713 2024-09-14T04:48:28.337 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.337 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:48:28.337 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.337 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:48:28.337 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.632065+0000", 2024-09-14T04:48:28.337 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.337 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.337 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.338 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:48:28.338 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.632065+0000", 2024-09-14T04:48:28.338 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.338 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.338 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.338 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:48:28.338 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.632066+0000", 2024-09-14T04:48:28.338 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3489999999999999e-06 2024-09-14T04:48:28.338 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.338 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.338 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:48:28.338 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.632071+0000", 2024-09-14T04:48:28.338 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9690000000000002e-06 2024-09-14T04:48:28.339 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.339 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.339 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:48:28.339 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.632085+0000", 2024-09-14T04:48:28.339 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3789e-05 2024-09-14T04:48:28.339 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.339 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.339 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:48:28.339 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.632093+0000", 2024-09-14T04:48:28.339 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9540000000000004e-06 2024-09-14T04:48:28.339 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.339 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.339 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.339 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.632154+0000", 2024-09-14T04:48:28.340 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1087999999999995e-05 2024-09-14T04:48:28.340 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.340 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.340 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:48:28.340 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.632162+0000", 2024-09-14T04:48:28.340 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.6200000000000005e-06 2024-09-14T04:48:28.340 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.340 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.340 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.340 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.487602+0000", 2024-09-14T04:48:28.340 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.85543967200000004 2024-09-14T04:48:28.340 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.340 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.341 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:48:28.341 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.487695+0000", 2024-09-14T04:48:28.341 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.3202000000000007e-05 2024-09-14T04:48:28.341 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.341 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.341 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 5,6", 2024-09-14T04:48:28.341 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.487795+0000", 2024-09-14T04:48:28.341 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.9955000000000001e-05 2024-09-14T04:48:28.341 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.341 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.341 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.342 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.494024+0000", 2024-09-14T04:48:28.342 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0062291439999999998 2024-09-14T04:48:28.342 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.342 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.342 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.342 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.494088+0000", 2024-09-14T04:48:28.342 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3811000000000003e-05 2024-09-14T04:48:28.342 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.342 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.342 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:48:28.342 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.494103+0000", 2024-09-14T04:48:28.342 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4966e-05 2024-09-14T04:48:28.342 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.343 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.343 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:48:28.343 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.494164+0000", 2024-09-14T04:48:28.343 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0448999999999997e-05 2024-09-14T04:48:28.343 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.343 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.343 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:48:28.343 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.494204+0000", 2024-09-14T04:48:28.343 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.0438999999999997e-05 2024-09-14T04:48:28.343 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.343 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:48:28.343 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.343 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.344 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.344 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242739 53.9 53:9f3d4fe1:::benchmark_data_smithi039_224949_object112646:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1362)", 2024-09-14T04:48:28.344 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:19.671329+0000", 2024-09-14T04:48:28.344 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 368.55017335600002, 2024-09-14T04:48:28.344 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.83323918699999999, 2024-09-14T04:48:28.344 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:48:28.344 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:48:28.344 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:48:28.344 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:48:28.344 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:48:28.345 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:48:28.345 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242739 2024-09-14T04:48:28.345 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.345 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:48:28.345 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.345 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:48:28.345 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.671329+0000", 2024-09-14T04:48:28.345 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.345 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.345 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.345 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:48:28.345 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.671329+0000", 2024-09-14T04:48:28.345 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.346 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.346 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.346 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:48:28.346 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.671330+0000", 2024-09-14T04:48:28.346 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7859999999999999e-06 2024-09-14T04:48:28.346 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.346 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.346 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:48:28.346 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.671335+0000", 2024-09-14T04:48:28.346 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1950000000000002e-06 2024-09-14T04:48:28.346 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.346 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.346 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:48:28.346 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.671345+0000", 2024-09-14T04:48:28.346 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.9979999999999998e-06 2024-09-14T04:48:28.347 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.347 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.347 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:48:28.347 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.671353+0000", 2024-09-14T04:48:28.347 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.306e-06 2024-09-14T04:48:28.347 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.347 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.347 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.347 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.671512+0000", 2024-09-14T04:48:28.347 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015923900000000001 2024-09-14T04:48:28.347 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.347 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.347 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:48:28.347 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.671521+0000", 2024-09-14T04:48:28.348 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.8079999999999994e-06 2024-09-14T04:48:28.348 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.348 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.348 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.348 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.490980+0000", 2024-09-14T04:48:28.348 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.81945951400000006 2024-09-14T04:48:28.348 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.348 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.348 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:48:28.348 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.491049+0000", 2024-09-14T04:48:28.348 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8763000000000006e-05 2024-09-14T04:48:28.349 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.349 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.349 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 5,6", 2024-09-14T04:48:28.349 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.491119+0000", 2024-09-14T04:48:28.349 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9400000000000006e-05 2024-09-14T04:48:28.349 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.349 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.349 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.349 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.494252+0000", 2024-09-14T04:48:28.349 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0031334359999999999 2024-09-14T04:48:28.349 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.349 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.349 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.350 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.501174+0000", 2024-09-14T04:48:28.350 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0069219149999999998 2024-09-14T04:48:28.350 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.350 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.350 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:48:28.350 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.504470+0000", 2024-09-14T04:48:28.350 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.003295718 2024-09-14T04:48:28.350 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.350 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.350 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:48:28.350 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.504523+0000", 2024-09-14T04:48:28.350 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3567999999999999e-05 2024-09-14T04:48:28.350 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.351 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.351 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:48:28.351 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.504568+0000", 2024-09-14T04:48:28.351 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4540999999999999e-05 2024-09-14T04:48:28.352 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.352 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:48:28.352 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.352 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.353 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.353 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242769 53.9 53:969e0ef1:::benchmark_data_smithi039_224949_object112676:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1362)", 2024-09-14T04:48:28.353 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:19.707867+0000", 2024-09-14T04:48:28.353 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 368.51363443100001, 2024-09-14T04:48:28.353 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.79684844099999996, 2024-09-14T04:48:28.353 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:48:28.353 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:48:28.353 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:48:28.353 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:48:28.353 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:48:28.353 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:48:28.353 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242769 2024-09-14T04:48:28.353 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.353 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:48:28.353 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.354 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:48:28.354 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.707867+0000", 2024-09-14T04:48:28.354 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.354 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.354 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.354 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:48:28.354 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.707867+0000", 2024-09-14T04:48:28.354 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.354 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.354 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.354 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:48:28.354 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.707869+0000", 2024-09-14T04:48:28.354 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2559999999999999e-06 2024-09-14T04:48:28.354 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.355 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.355 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:48:28.355 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.707873+0000", 2024-09-14T04:48:28.355 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3170000000000003e-06 2024-09-14T04:48:28.355 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.355 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.355 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:48:28.355 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.707885+0000", 2024-09-14T04:48:28.355 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1593000000000001e-05 2024-09-14T04:48:28.355 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.355 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.355 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:48:28.355 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.707892+0000", 2024-09-14T04:48:28.355 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.5859999999999999e-06 2024-09-14T04:48:28.356 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.356 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.356 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.356 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.708317+0000", 2024-09-14T04:48:28.356 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00042500699999999998 2024-09-14T04:48:28.356 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.356 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.356 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:48:28.356 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.708326+0000", 2024-09-14T04:48:28.356 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.5169999999999999e-06 2024-09-14T04:48:28.356 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.356 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.357 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.357 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.494284+0000", 2024-09-14T04:48:28.357 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.78595822000000004 2024-09-14T04:48:28.357 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.357 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.357 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:48:28.357 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.494448+0000", 2024-09-14T04:48:28.357 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00016446000000000001 2024-09-14T04:48:28.357 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.357 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.357 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 5,6", 2024-09-14T04:48:28.357 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.494587+0000", 2024-09-14T04:48:28.357 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013836299999999999 2024-09-14T04:48:28.357 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.358 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.358 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.358 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.501214+0000", 2024-09-14T04:48:28.358 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0066277269999999999 2024-09-14T04:48:28.358 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.358 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.358 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.358 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.501298+0000", 2024-09-14T04:48:28.358 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.3778999999999999e-05 2024-09-14T04:48:28.358 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.358 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.358 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:48:28.358 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.504631+0000", 2024-09-14T04:48:28.358 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0033328490000000001 2024-09-14T04:48:28.359 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.359 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.359 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:48:28.359 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.504684+0000", 2024-09-14T04:48:28.359 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3009e-05 2024-09-14T04:48:28.359 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.359 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.359 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:48:28.359 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.504716+0000", 2024-09-14T04:48:28.359 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.1758000000000003e-05 2024-09-14T04:48:28.359 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.360 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:48:28.360 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.360 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.360 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.360 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:135592 54.8 54:199531d3:::benchmark_data_smithi039_264334_object9195:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e1403)", 2024-09-14T04:48:28.360 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:57.312996+0000", 2024-09-14T04:48:28.360 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 210.908505572, 2024-09-14T04:48:28.360 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1455672880000001, 2024-09-14T04:48:28.360 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:48:28.360 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:48:28.360 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T04:48:28.360 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:48:28.360 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:48:28.361 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:48:28.361 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 135592 2024-09-14T04:48:28.361 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.361 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:48:28.361 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.361 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:48:28.361 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.312996+0000", 2024-09-14T04:48:28.361 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.361 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.361 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.361 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:48:28.361 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.312996+0000", 2024-09-14T04:48:28.361 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.362 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.362 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.362 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:48:28.362 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.312998+0000", 2024-09-14T04:48:28.362 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9309999999999998e-06 2024-09-14T04:48:28.362 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.362 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.362 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:48:28.362 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313006+0000", 2024-09-14T04:48:28.362 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.3520000000000001e-06 2024-09-14T04:48:28.362 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.362 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.362 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:48:28.363 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313025+0000", 2024-09-14T04:48:28.363 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9437999999999999e-05 2024-09-14T04:48:28.363 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.363 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.363 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:48:28.363 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313035+0000", 2024-09-14T04:48:28.363 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0179e-05 2024-09-14T04:48:28.363 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.363 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.363 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.363 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313130+0000", 2024-09-14T04:48:28.363 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.4851000000000004e-05 2024-09-14T04:48:28.363 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.363 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.363 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op must wait for map", 2024-09-14T04:48:28.364 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313136+0000", 2024-09-14T04:48:28.364 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.156e-06 2024-09-14T04:48:28.364 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.364 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.364 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.364 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.319910+0000", 2024-09-14T04:48:28.364 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0067736519999999998 2024-09-14T04:48:28.364 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.364 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.364 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:48:28.364 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.319919+0000", 2024-09-14T04:48:28.364 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.3440000000000007e-06 2024-09-14T04:48:28.365 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.365 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.365 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.365 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.456466+0000", 2024-09-14T04:48:28.365 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.136546662 2024-09-14T04:48:28.365 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.365 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.365 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:48:28.365 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.458564+0000", 2024-09-14T04:48:28.365 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0020977230000000001 2024-09-14T04:48:28.365 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.365 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:48:28.365 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.365 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.366 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.366 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:135646 54.8 54:17b125bc:::benchmark_data_smithi039_264334_object9249:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1403)", 2024-09-14T04:48:28.366 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:57.313060+0000", 2024-09-14T04:48:28.366 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 210.90844154300001, 2024-09-14T04:48:28.366 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1613178790000001, 2024-09-14T04:48:28.366 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:48:28.366 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:48:28.366 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:48:28.366 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:48:28.366 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:48:28.366 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:48:28.366 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 135646 2024-09-14T04:48:28.366 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.366 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:48:28.367 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.367 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:48:28.367 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313060+0000", 2024-09-14T04:48:28.367 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.367 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.367 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.367 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:48:28.367 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313060+0000", 2024-09-14T04:48:28.367 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.367 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.367 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.367 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:48:28.367 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313061+0000", 2024-09-14T04:48:28.368 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1099999999999995e-07 2024-09-14T04:48:28.368 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.368 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.368 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:48:28.368 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313064+0000", 2024-09-14T04:48:28.368 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.3380000000000001e-06 2024-09-14T04:48:28.368 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.368 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.368 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:48:28.368 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313083+0000", 2024-09-14T04:48:28.368 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8858000000000001e-05 2024-09-14T04:48:28.368 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.368 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.368 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:48:28.369 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313091+0000", 2024-09-14T04:48:28.369 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.8909999999999995e-06 2024-09-14T04:48:28.369 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.369 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.369 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.369 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313270+0000", 2024-09-14T04:48:28.369 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000179235 2024-09-14T04:48:28.369 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.369 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.369 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting_for_map not empty", 2024-09-14T04:48:28.369 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313276+0000", 2024-09-14T04:48:28.369 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.9070000000000004e-06 2024-09-14T04:48:28.369 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.369 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.370 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.370 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.320755+0000", 2024-09-14T04:48:28.370 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0074782829999999996 2024-09-14T04:48:28.370 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.370 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.370 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:48:28.370 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.320766+0000", 2024-09-14T04:48:28.370 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1898e-05 2024-09-14T04:48:28.370 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.370 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.370 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.370 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.459100+0000", 2024-09-14T04:48:28.370 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.138333673 2024-09-14T04:48:28.370 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.370 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.371 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:48:28.371 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.459693+0000", 2024-09-14T04:48:28.371 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000592408 2024-09-14T04:48:28.371 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.371 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.371 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4,7", 2024-09-14T04:48:28.371 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.459811+0000", 2024-09-14T04:48:28.371 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000118398 2024-09-14T04:48:28.371 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.371 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.371 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.371 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.468895+0000", 2024-09-14T04:48:28.371 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0090842330000000006 2024-09-14T04:48:28.371 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.372 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.372 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:48:28.372 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.468915+0000", 2024-09-14T04:48:28.372 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9643000000000001e-05 2024-09-14T04:48:28.372 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.372 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.372 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.372 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.468963+0000", 2024-09-14T04:48:28.372 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.8112000000000003e-05 2024-09-14T04:48:28.372 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.372 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.372 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.372 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.474201+0000", 2024-09-14T04:48:28.373 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00523822 2024-09-14T04:48:28.373 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.373 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.373 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:48:28.373 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.474263+0000", 2024-09-14T04:48:28.373 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1756999999999998e-05 2024-09-14T04:48:28.373 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.373 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.373 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:48:28.373 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.474378+0000", 2024-09-14T04:48:28.373 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000115314 2024-09-14T04:48:28.373 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.373 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:48:28.374 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.374 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.374 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.374 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:135649 54.8 54:10793023:::benchmark_data_smithi039_264334_object9252:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1403)", 2024-09-14T04:48:28.374 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:57.313190+0000", 2024-09-14T04:48:28.374 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 210.90831152300001, 2024-09-14T04:48:28.374 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1635118710000001, 2024-09-14T04:48:28.374 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:48:28.374 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:48:28.374 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:48:28.374 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:48:28.374 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:48:28.374 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:48:28.374 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 135649 2024-09-14T04:48:28.375 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.375 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:48:28.375 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.375 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:48:28.375 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313190+0000", 2024-09-14T04:48:28.375 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.375 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.375 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.375 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:48:28.375 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313190+0000", 2024-09-14T04:48:28.375 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.375 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.375 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.376 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:48:28.376 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313191+0000", 2024-09-14T04:48:28.376 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9500000000000002e-07 2024-09-14T04:48:28.376 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.376 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.376 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:48:28.376 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313195+0000", 2024-09-14T04:48:28.376 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.963e-06 2024-09-14T04:48:28.376 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.376 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.377 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:48:28.377 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313209+0000", 2024-09-14T04:48:28.377 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4440999999999999e-05 2024-09-14T04:48:28.377 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.377 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.377 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:48:28.377 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313218+0000", 2024-09-14T04:48:28.377 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.2369999999999992e-06 2024-09-14T04:48:28.377 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.377 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.377 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.377 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313307+0000", 2024-09-14T04:48:28.377 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.9746000000000004e-05 2024-09-14T04:48:28.378 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.378 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.378 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting_for_map not empty", 2024-09-14T04:48:28.378 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313312+0000", 2024-09-14T04:48:28.378 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7500000000000003e-06 2024-09-14T04:48:28.378 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.378 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.378 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.378 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.320795+0000", 2024-09-14T04:48:28.378 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0074831480000000002 2024-09-14T04:48:28.378 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.378 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.378 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:48:28.378 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.320802+0000", 2024-09-14T04:48:28.379 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8680000000000001e-06 2024-09-14T04:48:28.379 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.379 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.379 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.379 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.462371+0000", 2024-09-14T04:48:28.379 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.14156909 2024-09-14T04:48:28.379 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.379 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.379 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:48:28.379 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.462509+0000", 2024-09-14T04:48:28.379 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013791500000000001 2024-09-14T04:48:28.379 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.379 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.379 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4,7", 2024-09-14T04:48:28.380 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.462639+0000", 2024-09-14T04:48:28.380 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013007000000000001 2024-09-14T04:48:28.380 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.380 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.380 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:48:28.380 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.468921+0000", 2024-09-14T04:48:28.380 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0062813490000000003 2024-09-14T04:48:28.380 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.380 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.380 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.380 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.472860+0000", 2024-09-14T04:48:28.380 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0039389639999999997 2024-09-14T04:48:28.380 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.381 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.381 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.381 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.475650+0000", 2024-09-14T04:48:28.381 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0027903950000000002 2024-09-14T04:48:28.381 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.381 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.381 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.381 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.476501+0000", 2024-09-14T04:48:28.381 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00085151499999999995 2024-09-14T04:48:28.381 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.381 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.381 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:48:28.381 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.476573+0000", 2024-09-14T04:48:28.382 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1353000000000006e-05 2024-09-14T04:48:28.382 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.382 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.382 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:48:28.382 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.476702+0000", 2024-09-14T04:48:28.382 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012937199999999999 2024-09-14T04:48:28.382 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.382 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:48:28.382 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.382 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.383 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.383 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:135671 54.8 54:152b5606:::benchmark_data_smithi039_264334_object9274:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1403)", 2024-09-14T04:48:28.383 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:57.313280+0000", 2024-09-14T04:48:28.383 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 210.90822163000001, 2024-09-14T04:48:28.383 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.163696319, 2024-09-14T04:48:28.383 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:48:28.383 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:48:28.384 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:48:28.384 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:48:28.384 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:48:28.384 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:48:28.384 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 135671 2024-09-14T04:48:28.384 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.385 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:48:28.385 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.385 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:48:28.385 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313280+0000", 2024-09-14T04:48:28.385 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.385 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.385 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.386 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:48:28.386 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313280+0000", 2024-09-14T04:48:28.386 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:48:28.386 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.386 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.386 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:48:28.386 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313281+0000", 2024-09-14T04:48:28.387 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8400000000000004e-07 2024-09-14T04:48:28.387 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.387 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.387 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:48:28.387 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313285+0000", 2024-09-14T04:48:28.387 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.8820000000000003e-06 2024-09-14T04:48:28.387 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.387 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.387 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:48:28.387 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313294+0000", 2024-09-14T04:48:28.387 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.1749999999999994e-06 2024-09-14T04:48:28.387 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.388 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.388 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:48:28.388 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313301+0000", 2024-09-14T04:48:28.388 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.058e-06 2024-09-14T04:48:28.388 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.388 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.388 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.388 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313332+0000", 2024-09-14T04:48:28.388 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.0491999999999999e-05 2024-09-14T04:48:28.388 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.388 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.388 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting_for_map not empty", 2024-09-14T04:48:28.388 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313334+0000", 2024-09-14T04:48:28.389 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.7039999999999999e-06 2024-09-14T04:48:28.389 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.389 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.389 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.389 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.320826+0000", 2024-09-14T04:48:28.389 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0074915509999999999 2024-09-14T04:48:28.389 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.389 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.389 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:48:28.389 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.320832+0000", 2024-09-14T04:48:28.389 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.5520000000000001e-06 2024-09-14T04:48:28.389 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.389 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.390 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:48:28.390 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.465552+0000", 2024-09-14T04:48:28.390 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.144719316 2024-09-14T04:48:28.390 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.390 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.390 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:48:28.390 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.465658+0000", 2024-09-14T04:48:28.390 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000105969 2024-09-14T04:48:28.390 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.390 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.390 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4,7", 2024-09-14T04:48:28.390 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.465778+0000", 2024-09-14T04:48:28.390 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000120418 2024-09-14T04:48:28.391 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.391 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.391 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.391 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.472910+0000", 2024-09-14T04:48:28.391 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0071319640000000002 2024-09-14T04:48:28.391 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.391 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.391 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:48:28.391 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.473814+0000", 2024-09-14T04:48:28.391 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000904333 2024-09-14T04:48:28.391 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.391 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.391 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.391 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.475701+0000", 2024-09-14T04:48:28.392 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001886609 2024-09-14T04:48:28.392 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.392 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.392 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:48:28.392 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.476793+0000", 2024-09-14T04:48:28.392 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001092118 2024-09-14T04:48:28.392 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.392 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.392 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:48:28.392 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.476858+0000", 2024-09-14T04:48:28.392 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.4665999999999994e-05 2024-09-14T04:48:28.392 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:48:28.392 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:48:28.393 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:48:28.393 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.476977+0000", 2024-09-14T04:48:28.393 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011882799999999999 2024-09-14T04:48:28.393 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.393 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:48:28.393 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.393 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:48:28.393 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:48:28.393 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:48:28.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:28.394 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:28.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:28.495 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:28.526 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:28.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:28.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:28.550 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 27 16 53861 53845 124.616 92.0625 0.00931167 0.00801947 2024-09-14T04:48:28.628 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:28.668 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:28.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:28.669 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:28.770 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:28.806 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:28.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:28.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:28.841 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:28 smithi039 ceph-mon[106470]: pgmap v2566: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s; 640/144111 objects misplaced (0.444%); 5.0 MiB/s, 80 objects/s recovering 2024-09-14T04:48:28.841 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:28 smithi039 ceph-mon[106470]: from='client.35733 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:48:28.841 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:28 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'65536','5':'65536'}"} : dispatch 2024-09-14T04:48:28.841 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:28 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'65536','5':'65536'}"} : dispatch 2024-09-14T04:48:28.841 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:48:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:48:28] "GET /metrics HTTP/1.1" 200 36798 "" "Prometheus/2.51.0" 2024-09-14T04:48:28.842 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:28 smithi039 ceph-mon[111235]: pgmap v2566: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s; 640/144111 objects misplaced (0.444%); 5.0 MiB/s, 80 objects/s recovering 2024-09-14T04:48:28.842 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:28 smithi039 ceph-mon[111235]: from='client.35733 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:48:28.842 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:28 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'65536','5':'65536'}"} : dispatch 2024-09-14T04:48:28.842 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:28 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'65536','5':'65536'}"} : dispatch 2024-09-14T04:48:28.843 INFO:teuthology.orchestra.run.smithi039.stdout:moved 3 / 297 (1.0101%) 2024-09-14T04:48:28.843 INFO:teuthology.orchestra.run.smithi039.stdout:avg 37.125 2024-09-14T04:48:28.843 INFO:teuthology.orchestra.run.smithi039.stdout:stddev 5.13414 -> 4.51214 (expected baseline 5.69951) 2024-09-14T04:48:28.844 INFO:teuthology.orchestra.run.smithi039.stdout:min osd.5 with 30 -> 32 pgs (0.808081 -> 0.861953 * mean) 2024-09-14T04:48:28.844 INFO:teuthology.orchestra.run.smithi039.stdout:max osd.7 with 45 -> 45 pgs (1.21212 -> 1.21212 * mean) 2024-09-14T04:48:28.844 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:48:28.844 INFO:teuthology.orchestra.run.smithi039.stdout:oload 1000 2024-09-14T04:48:28.844 INFO:teuthology.orchestra.run.smithi039.stdout:max_change 3 2024-09-14T04:48:28.844 INFO:teuthology.orchestra.run.smithi039.stdout:max_change_osds 4 2024-09-14T04:48:28.844 INFO:teuthology.orchestra.run.smithi039.stdout:average_utilization 0.0160 2024-09-14T04:48:28.844 INFO:teuthology.orchestra.run.smithi039.stdout:overload_utilization 0.1603 2024-09-14T04:48:28.844 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0 weight 0.9500 -> 1.0000 2024-09-14T04:48:28.844 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5 weight 0.9500 -> 1.0000 2024-09-14T04:48:28.907 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:28.942 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:28.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:28.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:29.044 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:29.087 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:29.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:29.088 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:29.171 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:28 smithi129 ceph-mon[101316]: pgmap v2566: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s; 640/144111 objects misplaced (0.444%); 5.0 MiB/s, 80 objects/s recovering 2024-09-14T04:48:29.172 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:28 smithi129 ceph-mon[101316]: from='client.35733 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:48:29.172 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:28 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'65536','5':'65536'}"} : dispatch 2024-09-14T04:48:29.172 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:28 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'0':'65536','5':'65536'}"} : dispatch 2024-09-14T04:48:29.189 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:29.229 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:29.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:29.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:29.331 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:29.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:29.366 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:29.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:29.467 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:29.505 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:29.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:29.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:29.550 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 28 16 55475 55459 123.768 100.875 0.00742489 0.00807403 2024-09-14T04:48:29.607 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:29.644 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:29.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:29.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:29.746 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:29.780 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:29.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:29.781 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:29.882 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:29.916 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:29.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:29.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:30.018 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:30.041 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:30.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:30.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:30.143 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:30.186 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:30.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:30.187 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:30.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:29 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'65536','5':'65536'}"}]': finished 2024-09-14T04:48:30.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:29 smithi129 ceph-mon[101316]: osdmap e1432: 8 total, 8 up, 8 in 2024-09-14T04:48:30.288 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:30.289 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:29 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'65536','5':'65536'}"}]': finished 2024-09-14T04:48:30.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:29 smithi039 ceph-mon[106470]: osdmap e1432: 8 total, 8 up, 8 in 2024-09-14T04:48:30.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:29 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'65536','5':'65536'}"}]': finished 2024-09-14T04:48:30.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:29 smithi039 ceph-mon[111235]: osdmap e1432: 8 total, 8 up, 8 in 2024-09-14T04:48:30.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:30.335 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:30.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:30.435 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:30.458 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:30.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:30.459 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:30.550 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 29 16 56749 56733 122.245 79.625 0.00978911 0.00817484 2024-09-14T04:48:30.559 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:30.596 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:30.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:30.597 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:30.698 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:30.731 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:30.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:30.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:30.833 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:30.865 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:30.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:30.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:30.967 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:31.005 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:31.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:31.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:31.106 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:31.139 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:31.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:31.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:31.241 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:31.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:30 smithi129 ceph-mon[101316]: pgmap v2568: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s; 640/156843 objects misplaced (0.408%); 4.3 MiB/s, 69 objects/s recovering 2024-09-14T04:48:31.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:30 smithi129 ceph-mon[101316]: osdmap e1433: 8 total, 8 up, 8 in 2024-09-14T04:48:31.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:31.276 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:31.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:31.289 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:30 smithi039 ceph-mon[106470]: pgmap v2568: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s; 640/156843 objects misplaced (0.408%); 4.3 MiB/s, 69 objects/s recovering 2024-09-14T04:48:31.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:30 smithi039 ceph-mon[106470]: osdmap e1433: 8 total, 8 up, 8 in 2024-09-14T04:48:31.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:30 smithi039 ceph-mon[111235]: pgmap v2568: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 107 MiB/s wr, 1.72k op/s; 640/156843 objects misplaced (0.408%); 4.3 MiB/s, 69 objects/s recovering 2024-09-14T04:48:31.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:30 smithi039 ceph-mon[111235]: osdmap e1433: 8 total, 8 up, 8 in 2024-09-14T04:48:31.296 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:48:31.377 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:31.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:31.407 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:31.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:31.509 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:31.550 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 30 16 58168 58152 121.127 88.6875 0.0088261 0.00825 2024-09-14T04:48:31.554 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:31.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:31.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:31.656 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:31.691 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:31.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:31.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:31.793 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:31.832 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:31.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:31.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:31.933 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:31.969 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:31.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:31.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:32.071 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:32.097 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:32.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:32.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:32.198 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:32.236 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:32.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:32.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:32.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:31 smithi129 ceph-mon[101316]: osdmap e1434: 8 total, 8 up, 8 in 2024-09-14T04:48:32.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:31 smithi129 ceph-mon[101316]: osdmap e1435: 8 total, 8 up, 8 in 2024-09-14T04:48:32.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:31 smithi039 ceph-mon[106470]: osdmap e1434: 8 total, 8 up, 8 in 2024-09-14T04:48:32.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:31 smithi039 ceph-mon[106470]: osdmap e1435: 8 total, 8 up, 8 in 2024-09-14T04:48:32.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:31 smithi039 ceph-mon[111235]: osdmap e1434: 8 total, 8 up, 8 in 2024-09-14T04:48:32.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:31 smithi039 ceph-mon[111235]: osdmap e1435: 8 total, 8 up, 8 in 2024-09-14T04:48:32.338 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:32.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:32.375 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:32.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:32.476 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:32.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:32.504 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:32.505 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:32.550 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 31 16 59393 59377 119.689 76.5625 0.0072998 0.0083501 2024-09-14T04:48:32.606 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:32.627 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:32.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:32.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:32.729 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:32.768 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:32.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:32.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:32.870 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:32.902 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:32.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:32.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:33.004 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:33.041 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:33.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:33.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:33.143 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:33.181 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:33.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:33.182 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:33.283 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:33.316 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:33.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:33.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:33.418 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:33.457 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:33.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:33.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:33.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:33 smithi129 ceph-mon[101316]: pgmap v2572: 99 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 97 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s; 640/161274 objects misplaced (0.397%) 2024-09-14T04:48:33.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:33 smithi129 ceph-mon[101316]: osdmap e1436: 8 total, 8 up, 8 in 2024-09-14T04:48:33.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:33 smithi039 ceph-mon[106470]: pgmap v2572: 99 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 97 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s; 640/161274 objects misplaced (0.397%) 2024-09-14T04:48:33.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:33 smithi039 ceph-mon[106470]: osdmap e1436: 8 total, 8 up, 8 in 2024-09-14T04:48:33.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:33 smithi039 ceph-mon[111235]: pgmap v2572: 99 pgs: 1 remapped+peering, 1 active+recovering+undersized+remapped, 97 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 89 MiB/s wr, 1.43k op/s; 640/161274 objects misplaced (0.397%) 2024-09-14T04:48:33.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:33 smithi039 ceph-mon[111235]: osdmap e1436: 8 total, 8 up, 8 in 2024-09-14T04:48:33.550 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 32 16 60780 60764 118.658 86.6875 0.0171365 0.00842295 2024-09-14T04:48:33.559 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:33.591 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:33.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:33.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:33.693 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:33.713 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:33.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:33.715 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:33.815 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:33.844 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:33.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:33.845 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:33.945 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:33.985 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:33.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:33.986 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:34.087 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:34.124 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:34.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:34.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:34.226 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:34.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:34.267 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:34.267 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:34.368 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:34.407 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:34.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:34.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:34.509 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:34.551 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 33 16 62188 62172 117.728 88 0.0160819 0.00848899 2024-09-14T04:48:34.558 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:34.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:34.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:34.660 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:34.694 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:34.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:34.695 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:34.795 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:34.832 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:34.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:34.833 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:34.934 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:34.965 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:34.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:34.966 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:35.067 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:35.105 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:35.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:35.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:35.207 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:35.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:35.244 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:35.244 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:35.345 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:35.383 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:35.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:35.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:35.484 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:35.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:35 smithi129 ceph-mon[101316]: pgmap v2574: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s; 233/178233 objects misplaced (0.131%); 6.4 MiB/s, 102 objects/s recovering 2024-09-14T04:48:35.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:35.522 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:35.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:35.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:35 smithi039 ceph-mon[106470]: pgmap v2574: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s; 233/178233 objects misplaced (0.131%); 6.4 MiB/s, 102 objects/s recovering 2024-09-14T04:48:35.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:35 smithi039 ceph-mon[111235]: pgmap v2574: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s; 233/178233 objects misplaced (0.131%); 6.4 MiB/s, 102 objects/s recovering 2024-09-14T04:48:35.550 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 34 16 63749 63733 117.135 97.5625 0.00818706 0.00853176 2024-09-14T04:48:35.624 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:35.664 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:35.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:35.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:35.766 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:35.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:35.807 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:35.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:35.907 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:35.945 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:35.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:35.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:36.046 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:36.078 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:36.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:36.080 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:36.180 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:36.206 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:36.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:36.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:36.286 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-14T04:48:36.308 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:36.350 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:36.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:36.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:36.452 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:36.474 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:36.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:36.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:36.551 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 35 16 64798 64782 115.662 65.5625 0.0024369 0.00854262 2024-09-14T04:48:36.576 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:36.598 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:36.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:36.599 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:36.699 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:36.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:36.726 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:36.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:36.827 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:36.852 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:36.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:36.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:36.954 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:36.981 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:36.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:36.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:37.058 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is unset 2024-09-14T04:48:37.083 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:37.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:37.117 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:37.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:37.218 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:37.258 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:37.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:37.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:37.296 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:37 smithi039 ceph-mon[111235]: pgmap v2575: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 233/179490 objects misplaced (0.130%); 4.4 MiB/s, 70 objects/s recovering 2024-09-14T04:48:37.296 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:37 smithi039 ceph-mon[111235]: osdmap e1437: 8 total, 8 up, 8 in 2024-09-14T04:48:37.296 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:37 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2627044314' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:48:37.303 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:37 smithi039 ceph-mon[106470]: pgmap v2575: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 233/179490 objects misplaced (0.130%); 4.4 MiB/s, 70 objects/s recovering 2024-09-14T04:48:37.303 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:37 smithi039 ceph-mon[106470]: osdmap e1437: 8 total, 8 up, 8 in 2024-09-14T04:48:37.303 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:37 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2627044314' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:48:37.337 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2024-09-14T04:48:37.337 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd test-reweight-by-utilization 1000 3.0 2024-09-14T04:48:37.360 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:37.395 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:37.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:37.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:37.497 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:37.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:37 smithi129 ceph-mon[101316]: pgmap v2575: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 233/179490 objects misplaced (0.130%); 4.4 MiB/s, 70 objects/s recovering 2024-09-14T04:48:37.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:37 smithi129 ceph-mon[101316]: osdmap e1437: 8 total, 8 up, 8 in 2024-09-14T04:48:37.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:37 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2627044314' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:48:37.528 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:48:37.555 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 36 16 65463 65447 113.598 41.5625 0.022137 0.008796 2024-09-14T04:48:37.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:37.556 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:37.556 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:37.657 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:37.697 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:37.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:37.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:37.799 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:37.834 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:37.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:37.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:37.936 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:37.977 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:37.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:37.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:38.079 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:38.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:38.129 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:38.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:38.230 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:38.264 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:38.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:38.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:38.366 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:38.407 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:38.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:38.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:38.430 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:38 smithi039 ceph-mon[106470]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:48:38.430 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:38 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T04:48:38.430 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:38 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2627044314' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:48:38.430 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:38 smithi039 ceph-mon[106470]: osdmap e1438: 8 total, 8 up, 8 in 2024-09-14T04:48:38.431 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:38 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:48:38.431 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:38 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:48:38.431 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:38 smithi039 ceph-mon[111235]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:48:38.431 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:38 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T04:48:38.431 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:38 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2627044314' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:48:38.431 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:38 smithi039 ceph-mon[111235]: osdmap e1438: 8 total, 8 up, 8 in 2024-09-14T04:48:38.432 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:38 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:48:38.432 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:38 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:48:38.509 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:38.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:38 smithi129 ceph-mon[101316]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:48:38.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:38 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T04:48:38.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:38 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2627044314' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:48:38.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:38 smithi129 ceph-mon[101316]: osdmap e1438: 8 total, 8 up, 8 in 2024-09-14T04:48:38.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:38 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:48:38.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:38 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:48:38.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:38.545 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:38.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:38.552 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 37 16 66958 66942 113.053 93.4375 0.00543902 0.00884017 2024-09-14T04:48:38.645 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:38.678 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:38.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:38.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:38.723 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:48:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:48:38] "GET /metrics HTTP/1.1" 200 36783 "" "Prometheus/2.51.0" 2024-09-14T04:48:38.781 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:38.816 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:38.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:38.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:38.919 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:38.944 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:48:38.945 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:48:38.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:38.971 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:38.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:39.072 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:39.111 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:39.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:39.112 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:39.213 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:39.242 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:39.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:39.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:39.344 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:39.347 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:39 smithi039 ceph-mon[106470]: pgmap v2578: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 233/186819 objects misplaced (0.125%); 4.3 MiB/s, 68 objects/s recovering 2024-09-14T04:48:39.348 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:39 smithi039 ceph-mon[111235]: pgmap v2578: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 233/186819 objects misplaced (0.125%); 4.3 MiB/s, 68 objects/s recovering 2024-09-14T04:48:39.392 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:39.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:39.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:39.494 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:39.531 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:39.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:39.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:39.553 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 38 16 68442 68426 112.519 92.75 0.0224578 0.00888226 2024-09-14T04:48:39.633 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:39.671 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:39.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:39.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:39.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:39 smithi129 ceph-mon[101316]: pgmap v2578: 99 pgs: 1 active+recovering+undersized+remapped, 98 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s; 233/186819 objects misplaced (0.125%); 4.3 MiB/s, 68 objects/s recovering 2024-09-14T04:48:39.773 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:39.822 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:39.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:39.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:39.923 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:39.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:39.966 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:39.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:40.067 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:40.091 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:48:40.112 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:40.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:40.114 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:40.215 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:40.267 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:40.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:40.270 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:40.371 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:40.403 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:40.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:40.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:40.505 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:40.543 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:40.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:40.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:40.553 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 39 16 69848 69832 111.887 87.875 0.00777911 0.00893288 2024-09-14T04:48:40.645 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:40.687 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:40.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:40.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:40.789 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:40.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:40.828 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:40.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:40.929 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:40.970 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:40.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:40.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:41.072 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:41.113 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:41.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:41.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:41.215 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:41.258 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:41.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:41.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:41.360 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:41.396 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:41.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:41.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:41.497 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:41.518 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:41.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:41.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:41.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:41 smithi129 ceph-mon[101316]: pgmap v2579: 99 pgs: 99 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s; 5.2 MiB/s, 83 objects/s recovering 2024-09-14T04:48:41.553 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:2024-09-14T04:48:41.553091+0000 min lat: 0.00109434 max lat: 2.39824 avg lat: 0.00898687 2024-09-14T04:48:41.553 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:48:41.553 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 40 16 71096 71080 111.039 78 0.00311219 0.00898687 2024-09-14T04:48:41.620 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:41.649 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:41 smithi039 ceph-mon[106470]: pgmap v2579: 99 pgs: 99 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s; 5.2 MiB/s, 83 objects/s recovering 2024-09-14T04:48:41.650 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:41 smithi039 ceph-mon[111235]: pgmap v2579: 99 pgs: 99 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s; 5.2 MiB/s, 83 objects/s recovering 2024-09-14T04:48:41.660 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:41.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:41.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:41.761 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:41.800 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:41.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:41.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:41.902 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:41.933 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:41.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:41.934 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:42.034 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:42.072 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:42.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:42.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:42.174 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:42.215 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:42.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:42.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:42.317 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:42.353 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:42.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:42.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:42.455 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:42.489 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:42.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:42.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:42.553 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 41 16 72552 72536 110.55 91 0.00727906 0.00904089 2024-09-14T04:48:42.591 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:42.608 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:48:42.608 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:48:42.631 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:42.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:42.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:42.642 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:48:42.643 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:48:42.683 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:48:42.684 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:48:42.721 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:48:42.721 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:48:42.735 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:42.755 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:48:42.755 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:48:42.767 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:42.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:42.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:42.786 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:48:42.787 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:48:42.822 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:48:42.823 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:48:42.854 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:48:42.855 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:48:42.869 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:42.910 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:42.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:42.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:43.012 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:43.050 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:43.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:43.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:43.153 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:43.192 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:43.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:43.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:43.293 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:43.332 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:43.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:43.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:43.435 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:43.473 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:43.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:43.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:43.553 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 42 16 73967 73951 110.024 88.4375 0.0171698 0.00908387 2024-09-14T04:48:43.575 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:43.611 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:43.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:43.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:43.701 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:43 smithi039 ceph-mon[106470]: pgmap v2580: 99 pgs: 99 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s; 1.8 MiB/s, 29 objects/s recovering 2024-09-14T04:48:43.701 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:43 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3582703545' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:48:43.701 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:43 smithi039 ceph-mon[111235]: pgmap v2580: 99 pgs: 99 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s; 1.8 MiB/s, 29 objects/s recovering 2024-09-14T04:48:43.701 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:43 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3582703545' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:48:43.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:43 smithi129 ceph-mon[101316]: pgmap v2580: 99 pgs: 99 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s; 1.8 MiB/s, 29 objects/s recovering 2024-09-14T04:48:43.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:43 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3582703545' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:48:43.778 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:43.833 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:43.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:43.834 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:43.934 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:43.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:43.976 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:43.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:44.077 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:44.109 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:44.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:44.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:44.211 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:44.248 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:44.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:44.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:44.350 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:44.390 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:44.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:44.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:44.492 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:44.544 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:44.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:44.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:44.553 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 43 16 75350 75334 109.475 86.4375 0.0066476 0.0091292 2024-09-14T04:48:44.649 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:44.699 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:44.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:44.700 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:44.801 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:44.840 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:44.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:44.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:44.942 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:44.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:44.974 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:44.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:45.074 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:45.108 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:45.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:45.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:45.158 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:48:45.191 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:48:45.191 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:48:45.211 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:45.253 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:45.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:45.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:45.354 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:45.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:45.402 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:45.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:45.407 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:45 smithi039 ceph-mon[106470]: pgmap v2581: 99 pgs: 99 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s; 1.8 MiB/s, 29 objects/s recovering 2024-09-14T04:48:45.408 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:45 smithi039 ceph-mon[111235]: pgmap v2581: 99 pgs: 99 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s; 1.8 MiB/s, 29 objects/s recovering 2024-09-14T04:48:45.503 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:45.539 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:45.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:45.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:45.553 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 44 16 76655 76639 108.841 81.5625 0.0101647 0.00918172 2024-09-14T04:48:45.642 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:45.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:45.665 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:45.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:45.766 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:45.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:45 smithi129 ceph-mon[101316]: pgmap v2581: 99 pgs: 99 active+clean; 4.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 100 MiB/s wr, 1.60k op/s; 1.8 MiB/s, 29 objects/s recovering 2024-09-14T04:48:45.801 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:45.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:45.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:45.903 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:45.941 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:45.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:45.942 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:46.043 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:46.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:46.073 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:46.074 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:46.174 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:46.212 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:46.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:46.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:46.263 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:48:46.314 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:46.345 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:46.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:46.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:46.447 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:46.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:46.470 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:46.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:46.554 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 45 16 77715 77699 107.893 66.25 0.0168853 0.00926321 2024-09-14T04:48:46.571 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:46.605 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:46.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:46.606 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:46.707 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:46.746 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:46.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:46.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:46.802 INFO:teuthology.orchestra.run.smithi039.stdout:moved 0 / 297 (0%) 2024-09-14T04:48:46.802 INFO:teuthology.orchestra.run.smithi039.stdout:avg 37.125 2024-09-14T04:48:46.802 INFO:teuthology.orchestra.run.smithi039.stdout:stddev 4.51214 -> 4.51214 (expected baseline 5.69951) 2024-09-14T04:48:46.802 INFO:teuthology.orchestra.run.smithi039.stdout:min osd.2 with 31 -> 31 pgs (0.835017 -> 0.835017 * mean) 2024-09-14T04:48:46.802 INFO:teuthology.orchestra.run.smithi039.stdout:max osd.7 with 45 -> 45 pgs (1.21212 -> 1.21212 * mean) 2024-09-14T04:48:46.802 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:48:46.802 INFO:teuthology.orchestra.run.smithi039.stdout:oload 1000 2024-09-14T04:48:46.803 INFO:teuthology.orchestra.run.smithi039.stdout:max_change 3 2024-09-14T04:48:46.803 INFO:teuthology.orchestra.run.smithi039.stdout:max_change_osds 4 2024-09-14T04:48:46.803 INFO:teuthology.orchestra.run.smithi039.stdout:average_utilization 0.0229 2024-09-14T04:48:46.803 INFO:teuthology.orchestra.run.smithi039.stdout:overload_utilization 0.2288 2024-09-14T04:48:46.803 INFO:teuthology.orchestra.run.smithi039.stderr:no change 2024-09-14T04:48:46.848 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:46.870 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:46.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:46.871 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:46.972 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:47.013 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:47.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:47.014 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:47.115 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:47.148 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:47.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:47.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:47.250 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:47.283 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:47.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:47.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:47.385 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:47.413 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:47.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:47.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:47.515 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:47.536 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:47.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:47.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:47.554 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 46 16 78959 78943 107.237 77.75 0.0993492 0.00931173 2024-09-14T04:48:47.637 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:47.672 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:47.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:47.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:47 smithi129 ceph-mon[101316]: from='client.35751 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:48:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:47 smithi129 ceph-mon[101316]: pgmap v2582: 99 pgs: 99 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:48:47.774 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:47.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:47 smithi039 ceph-mon[106470]: from='client.35751 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:48:47.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:47 smithi039 ceph-mon[106470]: pgmap v2582: 99 pgs: 99 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:48:47.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:47 smithi039 ceph-mon[111235]: from='client.35751 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:48:47.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:47 smithi039 ceph-mon[111235]: pgmap v2582: 99 pgs: 99 active+clean; 4.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 80 MiB/s wr, 1.28k op/s; 1.5 MiB/s, 24 objects/s recovering 2024-09-14T04:48:47.809 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:47.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:47.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:47.910 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:47.947 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:47.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:47.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:48.049 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:48.089 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:48.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:48.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:48.191 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:48.229 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:48.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:48.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:48.332 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:48.380 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:48.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:48.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:48.482 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:48.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:48.524 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:48.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:48.554 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 47 16 80319 80303 106.764 85 0.00886643 0.00936061 2024-09-14T04:48:48.625 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:48.667 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:48.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:48.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:48.769 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:48.791 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:48:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:48:48] "GET /metrics HTTP/1.1" 200 36783 "" "Prometheus/2.51.0" 2024-09-14T04:48:48.805 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:48.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:48.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:48.907 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:48.947 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:48.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:48.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:48.999 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-14T04:48:49.049 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:49.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:49.113 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:49.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:49.214 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:49.264 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:49.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:49.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:49.366 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:49.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:49.403 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:49.404 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:49.504 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:49.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:49 smithi039 ceph-mon[106470]: pgmap v2583: 99 pgs: 99 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 1.4 MiB/s, 21 objects/s recovering 2024-09-14T04:48:49.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:49 smithi039 ceph-mon[111235]: pgmap v2583: 99 pgs: 99 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 1.4 MiB/s, 21 objects/s recovering 2024-09-14T04:48:49.541 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:49.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:49.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:49.554 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 48 16 81614 81598 106.226 80.9375 0.00833317 0.0094089 2024-09-14T04:48:49.644 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:49.683 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:49.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:49.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:49.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:49 smithi129 ceph-mon[101316]: pgmap v2583: 99 pgs: 99 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 1.4 MiB/s, 21 objects/s recovering 2024-09-14T04:48:49.776 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2024-09-14T04:48:49.776 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd test-reweight-by-utilization 110 3.0 2024-09-14T04:48:49.785 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:49.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:49.858 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:49.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:49.959 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:49.992 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:49.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:49.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:50.093 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:50.129 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:50.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:50.130 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:50.231 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:50.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:50.264 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:50.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:50.273 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:48:50.339 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:48:50.366 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:50.412 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:50.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:50.413 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:50.513 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:50.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:50.554 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:50.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:50.554 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 49 16 82894 82878 105.69 80 0.00762019 0.00945633 2024-09-14T04:48:50.655 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:50.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:50.697 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:50.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:50.798 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:50.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:50.834 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:50.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:50.935 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:50.974 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:50.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:50.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:51.076 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:51.114 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:51.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:51.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:51.216 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:51.280 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:51.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:51.281 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:51.382 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:51.416 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:51.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:51.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:51.459 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:51 smithi039 ceph-mon[106470]: pgmap v2584: 99 pgs: 99 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s; 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:48:51.459 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:51 smithi039 ceph-mon[111235]: pgmap v2584: 99 pgs: 99 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s; 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:48:51.518 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:51.555 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 50 16 84198 84182 105.205 81.5 0.0074474 0.00949922 2024-09-14T04:48:51.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:51.561 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:51.561 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:51.662 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:51.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:51.700 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:51.701 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:51.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:51 smithi129 ceph-mon[101316]: pgmap v2584: 99 pgs: 99 active+clean; 4.8 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 87 MiB/s wr, 1.40k op/s; 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:48:51.804 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:51.826 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:51.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:51.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:51.928 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:51.968 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:51.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:51.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:52.070 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:52.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:52.122 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:52.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:52.223 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:52.265 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:52.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:52.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:52.366 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:52.408 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:52.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:52.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:52.510 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:52.550 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:52.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:52.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:52.555 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 51 16 85277 85261 104.464 67.4375 0.0135697 0.00956716 2024-09-14T04:48:52.638 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:48:52.638 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:48:52.652 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:52.693 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:52.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:52.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:52.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:48:52.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:48:52.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:48:52.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:48:52.795 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:52.833 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:52.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:52.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:52.935 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:52.971 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:52.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:52.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:53.074 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:53.110 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:53.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:53.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:53.212 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:53.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:53.252 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:53.253 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:53.353 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:53.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:53.393 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:53.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:53.494 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:53.528 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:53.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:53.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:53.555 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 52 16 86462 86446 103.88 74.0625 0.00799649 0.0096211 2024-09-14T04:48:53.630 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:53.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:53.673 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:53.673 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:53 smithi129 ceph-mon[101316]: pgmap v2585: 99 pgs: 99 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-14T04:48:53.774 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:53.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:53 smithi039 ceph-mon[106470]: pgmap v2585: 99 pgs: 99 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-14T04:48:53.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:53 smithi039 ceph-mon[111235]: pgmap v2585: 99 pgs: 99 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-14T04:48:53.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:53.815 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:53.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:53.916 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:53.952 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:53.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:53.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:54.054 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:54.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:54.107 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:54.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:54.208 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:54.230 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:48:54.252 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:54.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:54.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:54.354 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:54.389 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:54.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:54.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:54.491 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:54.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:54.524 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:54.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:54.555 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 53 16 87626 87610 103.292 72.75 0.00894051 0.00967555 2024-09-14T04:48:54.626 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:54.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:54.674 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:54.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:54.775 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:54.808 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:54.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:54.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:54.909 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:54.960 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:54.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:54.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:55.062 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:55.093 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:55.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:55.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:55.195 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:55.234 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:55.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:55.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:55.336 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:55.370 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:55.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:55.371 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:55.390 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is set 2024-09-14T04:48:55.472 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:55.508 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:55.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:55.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:55.556 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 54 16 88799 88783 102.737 73.3125 0.00979745 0.00972888 2024-09-14T04:48:55.610 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:55.647 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:55.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:55.648 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:55.749 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:55 smithi129 ceph-mon[101316]: pgmap v2586: 99 pgs: 99 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-09-14T04:48:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:55 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2070924203' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:48:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:55 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:48:55.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:55 smithi039 ceph-mon[106470]: pgmap v2586: 99 pgs: 99 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-09-14T04:48:55.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:55 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2070924203' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:48:55.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:55 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:48:55.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:55 smithi039 ceph-mon[111235]: pgmap v2586: 99 pgs: 99 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-09-14T04:48:55.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:55 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2070924203' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:48:55.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:55 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:48:55.794 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:55.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:55.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:55.897 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:55.961 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:55.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:55.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:56.063 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:56.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:56.101 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:56.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:56.202 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:56.252 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:56.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:56.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:56.354 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:56.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:56.387 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:56.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:56.478 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:56 smithi039 ceph-mon[106470]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:48:56.478 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:56 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:48:56.478 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:56 smithi039 ceph-mon[106470]: osdmap e1439: 8 total, 8 up, 8 in 2024-09-14T04:48:56.478 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:56 smithi039 ceph-mon[111235]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:48:56.479 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:56 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:48:56.479 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:56 smithi039 ceph-mon[111235]: osdmap e1439: 8 total, 8 up, 8 in 2024-09-14T04:48:56.488 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:56.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:56 smithi129 ceph-mon[101316]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:48:56.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:56 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:48:56.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:56 smithi129 ceph-mon[101316]: osdmap e1439: 8 total, 8 up, 8 in 2024-09-14T04:48:56.524 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:56.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:56.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:56.556 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 55 16 89975 89959 102.204 73.5 0.00837381 0.00977928 2024-09-14T04:48:56.625 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:56.675 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:56.676 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:56.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:56.777 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:56.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:56.810 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:56.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:56.911 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:56.966 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:56.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:56.967 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:57.068 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:57.102 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:57.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:57.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:57.203 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:57.233 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:57.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:57.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:57.335 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:57.349 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:48:57.350 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:48:57.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:57.375 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:57.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:57.476 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:57.506 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:57.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:57.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:57.556 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 56 16 91118 91102 101.655 71.4375 0.00783052 0.00983203 2024-09-14T04:48:57.609 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:57.644 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:57.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:57.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:57.746 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:57.764 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:57 smithi039 ceph-mon[106470]: pgmap v2588: 99 pgs: 99 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-14T04:48:57.764 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:57 smithi039 ceph-mon[111235]: pgmap v2588: 99 pgs: 99 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-14T04:48:57.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:57 smithi129 ceph-mon[101316]: pgmap v2588: 99 pgs: 99 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s 2024-09-14T04:48:57.771 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:57.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:57.772 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:57.873 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:57.904 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:57.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:57.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:58.006 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:58.041 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:58.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:58.042 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:58.144 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:58.167 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:58.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:58.168 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:58.269 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:48:58.307 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:58.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:58.308 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:48:58.409 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:48:58.446 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:58.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:58.448 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:48:58.548 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:58.556 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 57 16 92021 92005 100.862 56.4375 0.0125336 0.00990901 2024-09-14T04:48:58.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:58.582 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:58.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:58.683 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:58.705 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:58.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:58.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:58 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1559370030' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:48:58.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:58 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1559370030' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:48:58.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:58 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1559370030' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:48:58.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:48:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:48:58] "GET /metrics HTTP/1.1" 200 36753 "" "Prometheus/2.51.0" 2024-09-14T04:48:58.806 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:48:58.835 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:58.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:58.836 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:48:58.937 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:48:58.972 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:58.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:58.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:48:59.074 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:48:59.112 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:59.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:59.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:48:59.213 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:59.259 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:59.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:59.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:59.360 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:59.405 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:59.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:59.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:59.507 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:48:59.551 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:59.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:59.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:48:59.556 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 58 16 92841 92825 100.006 51.25 0.00842479 0.00999396 2024-09-14T04:48:59.653 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:48:59.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:59.684 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:48:59.685 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:48:59.722 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:48:59 smithi039 ceph-mon[106470]: pgmap v2589: 99 pgs: 99 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-14T04:48:59.722 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:48:59 smithi039 ceph-mon[111235]: pgmap v2589: 99 pgs: 99 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-14T04:48:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:48:59 smithi129 ceph-mon[101316]: pgmap v2589: 99 pgs: 99 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-14T04:48:59.786 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:59.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:59.827 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:59.827 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:48:59.928 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:48:59.969 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:48:59.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:48:59.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:00.071 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:00.093 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:00.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:00.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:00.194 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:00.235 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:00.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:00.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:00.337 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:00.359 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:00.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:00.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:00.460 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:00.496 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:00.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:00.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:00.557 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 59 16 93863 93847 99.3938 63.875 0.00866896 0.0100558 2024-09-14T04:49:00.598 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:00.628 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:00.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:00.629 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:00.730 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:00.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:00.773 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:00.773 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:00.789 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:49:00.789 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:49:00.789 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:49:00.789 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:49:00.789 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:49:00.789 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:49:00.790 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:49:00.790 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:49:00.790 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:49:00.790 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:49:00.790 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:49:00.790 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:49:00.790 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:49:00.790 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:49:00.790 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:49:00.790 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:49:00.874 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:00.912 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:00.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:00.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:01.014 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:01.057 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:01.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:01.058 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:01.062 INFO:teuthology.orchestra.run.smithi039.stdout:moved 33 / 297 (11.1111%) 2024-09-14T04:49:01.062 INFO:teuthology.orchestra.run.smithi039.stdout:avg 37.125 2024-09-14T04:49:01.062 INFO:teuthology.orchestra.run.smithi039.stdout:stddev 4.51214 -> 6.98995 (expected baseline 5.69951) 2024-09-14T04:49:01.062 INFO:teuthology.orchestra.run.smithi039.stdout:min osd.2 with 31 -> 39 pgs (0.835017 -> 1.05051 * mean) 2024-09-14T04:49:01.062 INFO:teuthology.orchestra.run.smithi039.stdout:max osd.7 with 45 -> 20 pgs (1.21212 -> 0.538721 * mean) 2024-09-14T04:49:01.062 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:49:01.062 INFO:teuthology.orchestra.run.smithi039.stdout:oload 110 2024-09-14T04:49:01.062 INFO:teuthology.orchestra.run.smithi039.stdout:max_change 3 2024-09-14T04:49:01.063 INFO:teuthology.orchestra.run.smithi039.stdout:max_change_osds 4 2024-09-14T04:49:01.063 INFO:teuthology.orchestra.run.smithi039.stdout:average_utilization 0.0277 2024-09-14T04:49:01.063 INFO:teuthology.orchestra.run.smithi039.stdout:overload_utilization 0.0305 2024-09-14T04:49:01.063 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7 weight 1.0000 -> 0.5784 2024-09-14T04:49:01.063 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3 weight 0.9500 -> 0.7941 2024-09-14T04:49:01.063 INFO:teuthology.orchestra.run.smithi039.stderr:no change 2024-09-14T04:49:01.159 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:01.202 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:01.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:01.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:01.304 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:01.341 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:01.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:01.342 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:01.403 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:49:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T04:49:01.053+0000 7f268b1da640 -1 mgr.server reply reply (2) No such file or directory no change 2024-09-14T04:49:01.443 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:01.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:01.487 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:01.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:01.557 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:2024-09-14T04:49:01.557538+0000 min lat: 0.00109434 max lat: 2.39824 avg lat: 0.0101162 2024-09-14T04:49:01.557 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:49:01.557 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 60 16 94885 94869 98.8008 63.875 0.0192175 0.0101162 2024-09-14T04:49:01.588 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:01.627 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:01.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:01.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:01.728 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:01 smithi129 ceph-mon[101316]: pgmap v2590: 99 pgs: 99 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-14T04:49:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:01 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:49:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:01 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:49:01.776 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:01 smithi039 ceph-mon[106470]: pgmap v2590: 99 pgs: 99 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-14T04:49:01.777 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:01 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:49:01.777 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:01 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:49:01.778 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:01 smithi039 ceph-mon[111235]: pgmap v2590: 99 pgs: 99 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 75 MiB/s wr, 1.20k op/s 2024-09-14T04:49:01.778 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:01 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:49:01.778 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:01 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:49:01.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:01.779 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:01.779 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:01.880 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:01.914 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:01.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:01.915 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:02.016 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:02.058 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:02.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:02.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:02.160 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:02.177 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:49:02.198 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:02.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:02.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:02.300 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:02.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:02.342 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:02.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:02.443 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:02.476 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:02.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:02.478 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:02.557 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 61 16 96066 96050 98.391 73.8125 0.0146606 0.0101582 2024-09-14T04:49:02.579 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:02.613 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:02.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:02.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:02.665 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-14T04:49:02.715 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:02.724 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:02 smithi039 ceph-mon[106470]: from='client.45692 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:49:02.724 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:02 smithi039 ceph-mon[111235]: from='client.45692 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:49:02.742 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:02.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:02.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:02.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:02 smithi129 ceph-mon[101316]: from='client.45692 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:49:02.844 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:02.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:02.876 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:02.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:02.977 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:03.006 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:03.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:03.007 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:03.108 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:03.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:03.168 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:03.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:03.210 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:49:03.216 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:49:03.269 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:03.307 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:03.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:03.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:03.409 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:03.453 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:49:03.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:03.462 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:03.462 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:03.557 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 62 16 97153 97137 97.8998 67.9375 0.0105384 0.0102093 2024-09-14T04:49:03.563 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:03.596 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:03.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:03.597 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:03.698 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:03.718 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:03 smithi039 ceph-mon[106470]: pgmap v2591: 99 pgs: 99 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-14T04:49:03.719 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:03 smithi039 ceph-mon[111235]: pgmap v2591: 99 pgs: 99 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-14T04:49:03.731 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:03.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:03.732 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:03 smithi129 ceph-mon[101316]: pgmap v2591: 99 pgs: 99 active+clean; 5.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 74 MiB/s wr, 1.18k op/s 2024-09-14T04:49:03.833 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:03.882 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:03.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:03.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:03.984 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:04.024 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:04.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:04.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:04.126 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:04.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:04.166 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:04.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:04.267 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:04.307 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:04.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:04.308 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:04.409 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:04.452 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:04.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:04.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:04.554 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:04.557 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 63 16 98226 98210 97.4103 67.0625 0.00722911 0.0102608 2024-09-14T04:49:04.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:04.583 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:04.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:04.684 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:04.722 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:04.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:04.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:04.824 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:04.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:04.859 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:04.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:04.960 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:04.997 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:04.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:04.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:05.098 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:05.138 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:05.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:05.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:05.239 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:05.261 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:05.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:05.262 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:05.363 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:05.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:05.403 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:05.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:05.504 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:05.547 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:05.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:05.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:05.557 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 64 16 99320 99304 96.9566 68.375 0.0227582 0.0103088 2024-09-14T04:49:05.649 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:05.695 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:05.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:05.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:05.760 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:05 smithi039 ceph-mon[106470]: pgmap v2592: 99 pgs: 99 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-09-14T04:49:05.760 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:05 smithi039 ceph-mon[111235]: pgmap v2592: 99 pgs: 99 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-09-14T04:49:05.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:05 smithi129 ceph-mon[101316]: pgmap v2592: 99 pgs: 99 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s 2024-09-14T04:49:05.798 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:05.834 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:05.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:05.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:05.936 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:05.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:05.976 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:05.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:06.077 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:06.109 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:06.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:06.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:06.211 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:06.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:06.249 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:06.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:06.350 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:06.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:06.391 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:06.391 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:06.492 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:06.526 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:06.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:06.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:06.558 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 65 16 100476 100460 96.5764 72.25 0.0125592 0.0103492 2024-09-14T04:49:06.628 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:06.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:06.668 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:06.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:06.769 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:06.804 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:06.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:06.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:06.905 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:06.938 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:06.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:06.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:07.040 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:07.086 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:07.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:07.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:07.132 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-14T04:49:07.132 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:49:07.132 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2024-09-14T04:49:07.133 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-14T04:49:07.133 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-14T04:49:07.133 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 config set filestore_inject_stall 3 2024-09-14T04:49:07.188 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:07.235 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:07.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:07.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:07.337 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:07.369 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:07.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:07.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:07.471 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:07.513 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:49:07.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:07.539 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:07.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:07.558 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 66 16 101585 101569 96.1633 69.3125 0.0210854 0.0103941 2024-09-14T04:49:07.640 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:07.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:07.664 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:07.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:07.765 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:07 smithi129 ceph-mon[101316]: pgmap v2593: 99 pgs: 99 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-14T04:49:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:49:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:49:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:07 smithi039 ceph-mon[106470]: pgmap v2593: 99 pgs: 99 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-14T04:49:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:49:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:49:07.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:07 smithi039 ceph-mon[111235]: pgmap v2593: 99 pgs: 99 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-14T04:49:07.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:49:07.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:49:07.803 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:07.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:07.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:07.906 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:07.955 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:07.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:07.956 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:08.057 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:08.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:08.096 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:08.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:08.197 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:08.238 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:08.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:08.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:08.340 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:08.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:08.383 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:08.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:08.484 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:08.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:08.528 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:08.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:08.558 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 67 16 102566 102550 95.643 61.3125 0.0145783 0.0104503 2024-09-14T04:49:08.629 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:08.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:08.685 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:08.686 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:08.706 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:49:08.706 INFO:teuthology.orchestra.run.smithi039.stdout: "size": 20, 2024-09-14T04:49:08.706 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 600, 2024-09-14T04:49:08.706 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [ 2024-09-14T04:49:08.706 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.706 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233346 53.4 53:29de5274:::benchmark_data_smithi039_224949_object103253:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1354)", 2024-09-14T04:49:08.706 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.800661+0000", 2024-09-14T04:49:08.706 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 423.82938184599999, 2024-09-14T04:49:08.707 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1855864270000001, 2024-09-14T04:49:08.707 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:49:08.707 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:49:08.707 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T04:49:08.707 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:49:08.707 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:49:08.707 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:49:08.707 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233346 2024-09-14T04:49:08.707 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.707 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:49:08.707 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.707 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:49:08.707 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.800661+0000", 2024-09-14T04:49:08.707 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.708 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.708 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.708 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:49:08.708 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.800661+0000", 2024-09-14T04:49:08.708 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.708 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.708 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.708 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:49:08.708 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.800662+0000", 2024-09-14T04:49:08.708 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2890000000000001e-06 2024-09-14T04:49:08.708 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.708 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.708 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:49:08.708 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.800666+0000", 2024-09-14T04:49:08.708 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.5130000000000001e-06 2024-09-14T04:49:08.709 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.709 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.709 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:49:08.709 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.800678+0000", 2024-09-14T04:49:08.709 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2146e-05 2024-09-14T04:49:08.709 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.709 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.709 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:49:08.709 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.800685+0000", 2024-09-14T04:49:08.709 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1319999999999996e-06 2024-09-14T04:49:08.709 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.709 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.710 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.710 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.803352+0000", 2024-09-14T04:49:08.710 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.002667333 2024-09-14T04:49:08.710 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.710 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.710 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:49:08.710 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.803484+0000", 2024-09-14T04:49:08.710 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000131395 2024-09-14T04:49:08.710 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.710 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.710 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2", 2024-09-14T04:49:08.710 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.803596+0000", 2024-09-14T04:49:08.711 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000112693 2024-09-14T04:49:08.711 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.711 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.711 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.711 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.831786+0000", 2024-09-14T04:49:08.711 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.028190112 2024-09-14T04:49:08.711 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.711 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.711 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:49:08.711 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.831796+0000", 2024-09-14T04:49:08.711 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.5699999999999999e-06 2024-09-14T04:49:08.711 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.711 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.711 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.712 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820032+0000", 2024-09-14T04:49:08.712 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.98823552299999995 2024-09-14T04:49:08.712 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.712 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.712 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:49:08.712 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820042+0000", 2024-09-14T04:49:08.712 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0434999999999999e-05 2024-09-14T04:49:08.712 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.712 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.712 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.712 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.986165+0000", 2024-09-14T04:49:08.712 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.166122954 2024-09-14T04:49:08.712 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.713 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.713 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:49:08.713 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.986247+0000", 2024-09-14T04:49:08.713 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.2331999999999997e-05 2024-09-14T04:49:08.713 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.713 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:49:08.713 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.713 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.713 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.713 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233355 53.1 53.9503e781 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1354)", 2024-09-14T04:49:08.713 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.812481+0000", 2024-09-14T04:49:08.713 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 423.81756196999999, 2024-09-14T04:49:08.713 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.004531539, 2024-09-14T04:49:08.713 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:49:08.714 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:49:08.714 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T04:49:08.714 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:49:08.714 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:49:08.714 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:49:08.714 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233355 2024-09-14T04:49:08.714 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.714 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:49:08.714 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.714 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:49:08.714 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.812481+0000", 2024-09-14T04:49:08.714 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.714 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.714 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.714 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:49:08.714 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.812481+0000", 2024-09-14T04:49:08.715 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.715 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.715 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.715 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:49:08.715 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.812481+0000", 2024-09-14T04:49:08.715 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.4000000000000001e-07 2024-09-14T04:49:08.715 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.715 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.715 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:49:08.715 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.812486+0000", 2024-09-14T04:49:08.715 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2590000000000002e-06 2024-09-14T04:49:08.715 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.715 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.715 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:49:08.716 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.812496+0000", 2024-09-14T04:49:08.716 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0181e-05 2024-09-14T04:49:08.716 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.716 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.716 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:49:08.716 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.812504+0000", 2024-09-14T04:49:08.716 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9910000000000002e-06 2024-09-14T04:49:08.716 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.716 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.716 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.716 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.819855+0000", 2024-09-14T04:49:08.716 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0073515070000000002 2024-09-14T04:49:08.716 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.716 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.716 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:49:08.717 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.819866+0000", 2024-09-14T04:49:08.717 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0739e-05 2024-09-14T04:49:08.717 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.717 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.717 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.717 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.816996+0000", 2024-09-14T04:49:08.717 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.99712972799999999 2024-09-14T04:49:08.717 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.717 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.717 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:49:08.717 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.817012+0000", 2024-09-14T04:49:08.717 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6393999999999999e-05 2024-09-14T04:49:08.717 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.717 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:49:08.717 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.717 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.718 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.718 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233370 53.4 53:2070d59a:::benchmark_data_smithi039_224949_object103277:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-09-14T04:49:08.718 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.851143+0000", 2024-09-14T04:49:08.718 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 423.77889951399999, 2024-09-14T04:49:08.718 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1441897729999999, 2024-09-14T04:49:08.718 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:49:08.718 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:49:08.718 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:49:08.718 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:49:08.718 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:49:08.718 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:49:08.718 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233370 2024-09-14T04:49:08.718 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.718 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:49:08.718 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.718 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:49:08.719 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.851143+0000", 2024-09-14T04:49:08.719 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.719 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.719 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.719 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:49:08.719 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.851143+0000", 2024-09-14T04:49:08.719 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.719 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.719 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.719 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:49:08.719 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.851144+0000", 2024-09-14T04:49:08.719 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.212e-06 2024-09-14T04:49:08.719 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.719 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.720 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:49:08.720 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.851149+0000", 2024-09-14T04:49:08.720 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2209999999999999e-06 2024-09-14T04:49:08.720 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.720 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.720 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:49:08.720 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.851158+0000", 2024-09-14T04:49:08.720 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.9369999999999993e-06 2024-09-14T04:49:08.720 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.720 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.720 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:49:08.720 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.851164+0000", 2024-09-14T04:49:08.720 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2360000000000001e-06 2024-09-14T04:49:08.720 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.721 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.721 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.721 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.851253+0000", 2024-09-14T04:49:08.721 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.9313000000000004e-05 2024-09-14T04:49:08.721 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.721 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.721 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:49:08.721 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.851263+0000", 2024-09-14T04:49:08.721 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.5380000000000008e-06 2024-09-14T04:49:08.721 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.721 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.721 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.721 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820077+0000", 2024-09-14T04:49:08.721 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.96881390199999995 2024-09-14T04:49:08.722 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.722 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.722 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:49:08.722 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820084+0000", 2024-09-14T04:49:08.722 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.661e-06 2024-09-14T04:49:08.722 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.722 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.722 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.722 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.986538+0000", 2024-09-14T04:49:08.722 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1664539979999999 2024-09-14T04:49:08.722 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.722 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.722 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:49:08.722 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.986613+0000", 2024-09-14T04:49:08.722 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.5295000000000003e-05 2024-09-14T04:49:08.723 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.723 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.723 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:49:08.723 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.986688+0000", 2024-09-14T04:49:08.723 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.4932999999999996e-05 2024-09-14T04:49:08.723 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.723 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.723 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:49:08.723 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.995009+0000", 2024-09-14T04:49:08.723 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0083206119999999998 2024-09-14T04:49:08.723 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.723 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.723 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:49:08.723 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.995057+0000", 2024-09-14T04:49:08.723 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7873000000000001e-05 2024-09-14T04:49:08.724 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.724 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.724 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:49:08.724 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.995139+0000", 2024-09-14T04:49:08.724 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.2451000000000001e-05 2024-09-14T04:49:08.724 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.724 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.724 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:49:08.724 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.995214+0000", 2024-09-14T04:49:08.724 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.4501999999999999e-05 2024-09-14T04:49:08.724 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.724 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.724 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:49:08.724 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.995333+0000", 2024-09-14T04:49:08.725 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000119089 2024-09-14T04:49:08.725 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.725 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:49:08.725 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.725 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.725 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.725 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233376 53.4 53:2f4a7748:::benchmark_data_smithi039_224949_object103283:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-09-14T04:49:08.725 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.861746+0000", 2024-09-14T04:49:08.725 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 423.76829644700001, 2024-09-14T04:49:08.725 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1339306530000002, 2024-09-14T04:49:08.725 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:49:08.725 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:49:08.726 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:49:08.726 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:49:08.726 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:49:08.726 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:49:08.726 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233376 2024-09-14T04:49:08.726 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.726 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:49:08.726 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.726 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:49:08.726 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.861746+0000", 2024-09-14T04:49:08.726 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.726 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.726 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.726 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:49:08.726 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.861746+0000", 2024-09-14T04:49:08.726 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.727 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.727 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.727 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:49:08.727 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.861748+0000", 2024-09-14T04:49:08.727 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2299999999999998e-06 2024-09-14T04:49:08.727 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.727 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.727 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:49:08.727 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.861755+0000", 2024-09-14T04:49:08.727 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1580000000000002e-06 2024-09-14T04:49:08.727 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.727 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.727 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:49:08.727 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.861767+0000", 2024-09-14T04:49:08.727 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.254e-05 2024-09-14T04:49:08.727 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.728 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.728 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:49:08.728 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.861776+0000", 2024-09-14T04:49:08.728 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.8850000000000005e-06 2024-09-14T04:49:08.728 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.728 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.728 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.728 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.862325+0000", 2024-09-14T04:49:08.728 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00054872099999999995 2024-09-14T04:49:08.728 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.728 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.728 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:49:08.728 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.862333+0000", 2024-09-14T04:49:08.728 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.4689999999999997e-06 2024-09-14T04:49:08.728 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.728 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.729 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.729 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820254+0000", 2024-09-14T04:49:08.729 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.95792123699999998 2024-09-14T04:49:08.729 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.729 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.729 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:49:08.729 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820262+0000", 2024-09-14T04:49:08.729 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7780000000000007e-06 2024-09-14T04:49:08.729 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.729 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.729 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.729 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.990645+0000", 2024-09-14T04:49:08.729 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.170382351 2024-09-14T04:49:08.730 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.730 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.730 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:49:08.730 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.990791+0000", 2024-09-14T04:49:08.730 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014623199999999999 2024-09-14T04:49:08.730 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.730 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.730 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:49:08.730 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.990931+0000", 2024-09-14T04:49:08.730 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013971799999999999 2024-09-14T04:49:08.730 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.730 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.730 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:49:08.730 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.995074+0000", 2024-09-14T04:49:08.730 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0041437649999999998 2024-09-14T04:49:08.730 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.731 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.731 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:49:08.731 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.995400+0000", 2024-09-14T04:49:08.731 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00032538700000000001 2024-09-14T04:49:08.731 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.731 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.731 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:49:08.731 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.995468+0000", 2024-09-14T04:49:08.731 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8169000000000004e-05 2024-09-14T04:49:08.731 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.731 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.731 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:49:08.731 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.995546+0000", 2024-09-14T04:49:08.731 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7861000000000006e-05 2024-09-14T04:49:08.731 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.731 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.732 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:49:08.732 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.995677+0000", 2024-09-14T04:49:08.732 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013115199999999999 2024-09-14T04:49:08.732 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.732 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:49:08.732 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.732 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.732 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.732 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233378 53.4 53:2ea9efd4:::benchmark_data_smithi039_224949_object103285:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-09-14T04:49:08.732 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.864442+0000", 2024-09-14T04:49:08.732 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 423.76560042099999, 2024-09-14T04:49:08.732 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1434582400000002, 2024-09-14T04:49:08.732 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:49:08.732 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:49:08.733 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:49:08.733 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:49:08.733 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:49:08.733 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:49:08.733 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233378 2024-09-14T04:49:08.733 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.733 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:49:08.733 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.733 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:49:08.733 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.864442+0000", 2024-09-14T04:49:08.733 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.733 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.733 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.734 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:49:08.734 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.864442+0000", 2024-09-14T04:49:08.734 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.734 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.734 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.734 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:49:08.734 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.864444+0000", 2024-09-14T04:49:08.734 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.968e-06 2024-09-14T04:49:08.734 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.734 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.734 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:49:08.734 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.864452+0000", 2024-09-14T04:49:08.735 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.452e-06 2024-09-14T04:49:08.735 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.735 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.735 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:49:08.735 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.864494+0000", 2024-09-14T04:49:08.735 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.1869000000000002e-05 2024-09-14T04:49:08.735 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.735 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.735 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:49:08.735 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.864503+0000", 2024-09-14T04:49:08.735 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.7100000000000002e-06 2024-09-14T04:49:08.735 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.735 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.735 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.735 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.864617+0000", 2024-09-14T04:49:08.736 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011412399999999999 2024-09-14T04:49:08.736 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.736 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.736 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:49:08.736 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.864627+0000", 2024-09-14T04:49:08.736 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0011e-05 2024-09-14T04:49:08.736 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.736 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.736 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.736 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820558+0000", 2024-09-14T04:49:08.736 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.95593013699999996 2024-09-14T04:49:08.736 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.737 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.737 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:49:08.737 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820565+0000", 2024-09-14T04:49:08.737 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.8900000000000007e-06 2024-09-14T04:49:08.737 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.737 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.737 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.737 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.995726+0000", 2024-09-14T04:49:08.737 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1751607719999999 2024-09-14T04:49:08.737 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.737 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.737 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:49:08.737 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.995867+0000", 2024-09-14T04:49:08.738 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000141297 2024-09-14T04:49:08.738 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.738 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.738 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:49:08.738 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.996016+0000", 2024-09-14T04:49:08.738 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014899100000000001 2024-09-14T04:49:08.738 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.738 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.738 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:49:08.738 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.003532+0000", 2024-09-14T04:49:08.738 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0075159880000000004 2024-09-14T04:49:08.738 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.738 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.738 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:49:08.738 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.003646+0000", 2024-09-14T04:49:08.739 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000114041 2024-09-14T04:49:08.739 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.739 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.739 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:49:08.739 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.007735+0000", 2024-09-14T04:49:08.739 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0040889840000000004 2024-09-14T04:49:08.739 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.739 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.739 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:49:08.739 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.007788+0000", 2024-09-14T04:49:08.739 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.2154999999999998e-05 2024-09-14T04:49:08.739 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.739 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.739 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:49:08.740 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.007900+0000", 2024-09-14T04:49:08.740 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011285099999999999 2024-09-14T04:49:08.740 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.740 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:49:08.740 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.740 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.740 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.740 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233406 53.1 53.a3bc2e61 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-09-14T04:49:08.740 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.908181+0000", 2024-09-14T04:49:08.740 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 423.72186169499997, 2024-09-14T04:49:08.740 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.90887015999999998, 2024-09-14T04:49:08.740 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:49:08.740 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:49:08.740 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T04:49:08.741 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:49:08.741 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:49:08.741 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:49:08.741 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233406 2024-09-14T04:49:08.741 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.741 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:49:08.741 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.741 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:49:08.741 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.908181+0000", 2024-09-14T04:49:08.741 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.741 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.741 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.741 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:49:08.742 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.908181+0000", 2024-09-14T04:49:08.742 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.742 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.742 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.742 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:49:08.742 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.908182+0000", 2024-09-14T04:49:08.742 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.437e-06 2024-09-14T04:49:08.742 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.742 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.742 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:49:08.742 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.908188+0000", 2024-09-14T04:49:08.742 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6529999999999999e-06 2024-09-14T04:49:08.742 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.742 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.743 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:49:08.743 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.908201+0000", 2024-09-14T04:49:08.743 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2634e-05 2024-09-14T04:49:08.743 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.743 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.743 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:49:08.743 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.908210+0000", 2024-09-14T04:49:08.743 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.8400000000000001e-06 2024-09-14T04:49:08.743 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.743 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.743 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.743 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.908542+0000", 2024-09-14T04:49:08.743 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00033224600000000002 2024-09-14T04:49:08.743 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.744 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.744 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:49:08.744 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.908554+0000", 2024-09-14T04:49:08.744 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1736000000000001e-05 2024-09-14T04:49:08.744 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.744 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.744 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.744 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.817042+0000", 2024-09-14T04:49:08.744 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.90848892199999998 2024-09-14T04:49:08.744 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.744 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.744 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:49:08.744 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.817051+0000", 2024-09-14T04:49:08.744 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.6920000000000008e-06 2024-09-14T04:49:08.745 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.745 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:49:08.745 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.745 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.745 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.745 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233411 53.4 53:2cd724b4:::benchmark_data_smithi039_224949_object103318:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-09-14T04:49:08.745 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.920755+0000", 2024-09-14T04:49:08.745 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 423.70928815799999, 2024-09-14T04:49:08.745 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.087357184, 2024-09-14T04:49:08.745 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:49:08.745 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:49:08.745 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:49:08.745 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:49:08.745 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:49:08.746 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:49:08.746 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233411 2024-09-14T04:49:08.746 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.746 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:49:08.746 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.746 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:49:08.746 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.920755+0000", 2024-09-14T04:49:08.746 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.746 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.746 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.746 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:49:08.746 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.920755+0000", 2024-09-14T04:49:08.746 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.746 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.747 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.747 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:49:08.747 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.920757+0000", 2024-09-14T04:49:08.747 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2879999999999999e-06 2024-09-14T04:49:08.747 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.747 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.747 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:49:08.747 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.920765+0000", 2024-09-14T04:49:08.747 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.3510000000000002e-06 2024-09-14T04:49:08.747 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.747 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.747 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:49:08.747 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.920791+0000", 2024-09-14T04:49:08.747 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.5904000000000001e-05 2024-09-14T04:49:08.748 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.748 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.748 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:49:08.748 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.920802+0000", 2024-09-14T04:49:08.748 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0804e-05 2024-09-14T04:49:08.748 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.748 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.748 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.748 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.920893+0000", 2024-09-14T04:49:08.748 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.0964000000000005e-05 2024-09-14T04:49:08.748 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.748 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.748 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:49:08.749 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.920904+0000", 2024-09-14T04:49:08.749 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0774000000000001e-05 2024-09-14T04:49:08.749 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.749 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.749 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.749 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820656+0000", 2024-09-14T04:49:08.749 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.89975210000000005 2024-09-14T04:49:08.749 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.749 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.749 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:49:08.749 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820665+0000", 2024-09-14T04:49:08.749 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.2860000000000005e-06 2024-09-14T04:49:08.749 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.749 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.750 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.750 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.999414+0000", 2024-09-14T04:49:08.750 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1787490410000001 2024-09-14T04:49:08.750 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.750 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.750 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:49:08.750 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.999538+0000", 2024-09-14T04:49:08.750 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012393500000000001 2024-09-14T04:49:08.750 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.750 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.750 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:49:08.750 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:06.999662+0000", 2024-09-14T04:49:08.750 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012404199999999999 2024-09-14T04:49:08.750 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.751 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.751 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:49:08.751 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.003622+0000", 2024-09-14T04:49:08.751 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0039600080000000001 2024-09-14T04:49:08.751 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.751 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.751 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:49:08.751 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.003684+0000", 2024-09-14T04:49:08.751 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2436999999999999e-05 2024-09-14T04:49:08.751 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.751 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.751 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:49:08.751 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.007955+0000", 2024-09-14T04:49:08.751 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0042703330000000003 2024-09-14T04:49:08.752 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.752 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.752 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:49:08.752 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.008012+0000", 2024-09-14T04:49:08.752 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7008e-05 2024-09-14T04:49:08.752 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.752 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.752 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:49:08.752 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.008112+0000", 2024-09-14T04:49:08.752 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.9908999999999995e-05 2024-09-14T04:49:08.752 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.752 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:49:08.752 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.753 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.753 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.753 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233412 53.1 53.a2159a51 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-09-14T04:49:08.753 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.921978+0000", 2024-09-14T04:49:08.753 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 423.70806464999998, 2024-09-14T04:49:08.753 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.89512192400000001, 2024-09-14T04:49:08.753 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:49:08.753 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:49:08.753 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T04:49:08.753 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:49:08.753 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:49:08.753 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:49:08.753 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233412 2024-09-14T04:49:08.754 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.754 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:49:08.754 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.754 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:49:08.754 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.921978+0000", 2024-09-14T04:49:08.754 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.754 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.754 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.754 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:49:08.754 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.921978+0000", 2024-09-14T04:49:08.754 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.754 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.754 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.754 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:49:08.755 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.921979+0000", 2024-09-14T04:49:08.755 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.540000000000001e-07 2024-09-14T04:49:08.755 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.755 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.755 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:49:08.755 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.921984+0000", 2024-09-14T04:49:08.755 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6229999999999996e-06 2024-09-14T04:49:08.755 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.755 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.755 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:49:08.755 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.921994+0000", 2024-09-14T04:49:08.755 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.078e-05 2024-09-14T04:49:08.755 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.756 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.756 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:49:08.756 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.922000+0000", 2024-09-14T04:49:08.756 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.2449999999999998e-06 2024-09-14T04:49:08.756 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.756 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.756 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.756 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.922053+0000", 2024-09-14T04:49:08.756 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3575999999999997e-05 2024-09-14T04:49:08.756 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.756 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.756 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:49:08.756 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.922062+0000", 2024-09-14T04:49:08.757 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.2240000000000001e-06 2024-09-14T04:49:08.757 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.757 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.757 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.757 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.817089+0000", 2024-09-14T04:49:08.757 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.89502705100000002 2024-09-14T04:49:08.757 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.757 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.757 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:49:08.757 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.817100+0000", 2024-09-14T04:49:08.757 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0471e-05 2024-09-14T04:49:08.757 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.757 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:49:08.758 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.758 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.758 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.758 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:135594 54.e 54:7b36e427:::benchmark_data_smithi039_264334_object9197:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e1403)", 2024-09-14T04:49:08.758 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:57.312967+0000", 2024-09-14T04:49:08.758 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 251.31707612599999, 2024-09-14T04:49:08.758 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3028954079999999, 2024-09-14T04:49:08.758 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:49:08.758 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:49:08.758 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T04:49:08.758 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:49:08.758 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:49:08.758 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:49:08.758 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 135594 2024-09-14T04:49:08.759 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.759 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:49:08.759 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.759 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:49:08.759 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.312967+0000", 2024-09-14T04:49:08.759 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.759 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.759 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.759 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:49:08.759 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.312967+0000", 2024-09-14T04:49:08.759 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.759 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.759 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.760 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:49:08.760 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.312968+0000", 2024-09-14T04:49:08.760 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.894e-06 2024-09-14T04:49:08.760 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.760 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.760 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:49:08.760 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.312976+0000", 2024-09-14T04:49:08.760 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.2429999999999999e-06 2024-09-14T04:49:08.760 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.760 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.760 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:49:08.760 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.312995+0000", 2024-09-14T04:49:08.760 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9769e-05 2024-09-14T04:49:08.760 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.760 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.761 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:49:08.761 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313008+0000", 2024-09-14T04:49:08.761 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2729e-05 2024-09-14T04:49:08.761 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.761 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.761 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.761 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313092+0000", 2024-09-14T04:49:08.761 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.4084000000000003e-05 2024-09-14T04:49:08.761 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.761 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.761 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op must wait for map", 2024-09-14T04:49:08.761 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313101+0000", 2024-09-14T04:49:08.761 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.1880000000000008e-06 2024-09-14T04:49:08.762 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.762 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.762 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.762 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.318631+0000", 2024-09-14T04:49:08.762 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0055294619999999997 2024-09-14T04:49:08.762 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.762 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.762 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:49:08.762 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.318643+0000", 2024-09-14T04:49:08.762 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.166e-05 2024-09-14T04:49:08.762 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.762 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.762 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.762 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.596944+0000", 2024-09-14T04:49:08.762 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2783012810000001 2024-09-14T04:49:08.762 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.763 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.763 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for missing object", 2024-09-14T04:49:08.763 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.597066+0000", 2024-09-14T04:49:08.763 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012249099999999999 2024-09-14T04:49:08.763 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.763 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.763 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.763 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.615768+0000", 2024-09-14T04:49:08.763 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.018701998000000001 2024-09-14T04:49:08.763 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.763 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.763 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:49:08.763 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.615862+0000", 2024-09-14T04:49:08.763 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.3609000000000004e-05 2024-09-14T04:49:08.764 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.764 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:49:08.764 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.764 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.764 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.764 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:135662 54.e 54:7adb4039:::benchmark_data_smithi039_264334_object9265:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1403)", 2024-09-14T04:49:08.764 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:57.313037+0000", 2024-09-14T04:49:08.764 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 251.31700562899999, 2024-09-14T04:49:08.764 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.306597802, 2024-09-14T04:49:08.764 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:49:08.764 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:49:08.764 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:49:08.764 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:49:08.764 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:49:08.765 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:49:08.765 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 135662 2024-09-14T04:49:08.765 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.765 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:49:08.765 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.765 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:49:08.765 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313037+0000", 2024-09-14T04:49:08.765 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.765 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.765 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.765 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:49:08.765 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313037+0000", 2024-09-14T04:49:08.765 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.765 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.766 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.766 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:49:08.766 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313037+0000", 2024-09-14T04:49:08.766 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5699999999999998e-07 2024-09-14T04:49:08.766 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.766 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.766 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:49:08.766 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313040+0000", 2024-09-14T04:49:08.766 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.9849999999999998e-06 2024-09-14T04:49:08.766 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.766 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.766 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:49:08.766 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313049+0000", 2024-09-14T04:49:08.767 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.1790000000000008e-06 2024-09-14T04:49:08.767 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.767 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.767 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:49:08.767 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313053+0000", 2024-09-14T04:49:08.767 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6310000000000002e-06 2024-09-14T04:49:08.767 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.767 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.767 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.767 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313197+0000", 2024-09-14T04:49:08.767 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000144148 2024-09-14T04:49:08.767 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.767 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.768 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting_for_map not empty", 2024-09-14T04:49:08.768 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313204+0000", 2024-09-14T04:49:08.768 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8569999999999997e-06 2024-09-14T04:49:08.768 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.768 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.768 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.768 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.319670+0000", 2024-09-14T04:49:08.768 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0064654730000000002 2024-09-14T04:49:08.768 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.768 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.768 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:49:08.768 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.319679+0000", 2024-09-14T04:49:08.768 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.9819999999999997e-06 2024-09-14T04:49:08.769 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.769 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.769 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.769 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.600703+0000", 2024-09-14T04:49:08.769 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.281024452 2024-09-14T04:49:08.769 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.769 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.769 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:49:08.769 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.603324+0000", 2024-09-14T04:49:08.769 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.002620832 2024-09-14T04:49:08.769 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.769 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.769 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,4,7", 2024-09-14T04:49:08.769 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.603691+0000", 2024-09-14T04:49:08.770 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00036688200000000002 2024-09-14T04:49:08.770 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.770 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.770 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:49:08.770 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.615959+0000", 2024-09-14T04:49:08.770 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.012267887 2024-09-14T04:49:08.770 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.770 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.770 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:49:08.770 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.616023+0000", 2024-09-14T04:49:08.770 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.4488999999999999e-05 2024-09-14T04:49:08.770 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.770 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.771 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:49:08.771 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.616154+0000", 2024-09-14T04:49:08.771 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013044099999999999 2024-09-14T04:49:08.771 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.771 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.771 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:49:08.771 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.616216+0000", 2024-09-14T04:49:08.771 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1947999999999997e-05 2024-09-14T04:49:08.771 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.771 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.771 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:49:08.771 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.616291+0000", 2024-09-14T04:49:08.771 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.4832000000000005e-05 2024-09-14T04:49:08.771 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.772 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.772 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:49:08.772 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.619635+0000", 2024-09-14T04:49:08.772 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0033443269999999998 2024-09-14T04:49:08.772 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.772 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:49:08.772 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.772 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.772 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.772 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:135674 54.e 54:760a8d5e:::benchmark_data_smithi039_264334_object9277:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1403)", 2024-09-14T04:49:08.772 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:57.313082+0000", 2024-09-14T04:49:08.772 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 251.31696055099999, 2024-09-14T04:49:08.772 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3323649710000001, 2024-09-14T04:49:08.772 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:49:08.773 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:49:08.773 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:49:08.773 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:49:08.773 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:49:08.773 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:49:08.773 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 135674 2024-09-14T04:49:08.773 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.773 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:49:08.773 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.773 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:49:08.773 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313082+0000", 2024-09-14T04:49:08.773 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.773 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.774 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.774 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:49:08.774 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313082+0000", 2024-09-14T04:49:08.774 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.774 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.774 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.774 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:49:08.774 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313083+0000", 2024-09-14T04:49:08.774 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9100000000000004e-07 2024-09-14T04:49:08.774 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.774 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.774 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:49:08.774 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313086+0000", 2024-09-14T04:49:08.775 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.0199999999999999e-06 2024-09-14T04:49:08.775 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.775 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.775 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:49:08.775 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313095+0000", 2024-09-14T04:49:08.775 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.9090000000000006e-06 2024-09-14T04:49:08.775 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.775 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.775 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:49:08.775 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313102+0000", 2024-09-14T04:49:08.775 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.3069999999999997e-06 2024-09-14T04:49:08.775 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.775 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.775 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.776 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313242+0000", 2024-09-14T04:49:08.776 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013966200000000001 2024-09-14T04:49:08.776 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.776 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.776 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting_for_map not empty", 2024-09-14T04:49:08.776 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313247+0000", 2024-09-14T04:49:08.776 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.0270000000000003e-06 2024-09-14T04:49:08.776 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.776 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.776 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.776 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.319701+0000", 2024-09-14T04:49:08.776 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0064543880000000001 2024-09-14T04:49:08.776 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.776 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.776 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:49:08.777 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.320481+0000", 2024-09-14T04:49:08.777 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000779746 2024-09-14T04:49:08.777 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.777 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.777 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.777 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.619691+0000", 2024-09-14T04:49:08.777 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2992098439999999 2024-09-14T04:49:08.777 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.777 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.777 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:49:08.777 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.619806+0000", 2024-09-14T04:49:08.777 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011595999999999999 2024-09-14T04:49:08.777 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.778 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.778 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,4,7", 2024-09-14T04:49:08.778 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.619989+0000", 2024-09-14T04:49:08.778 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00018227599999999999 2024-09-14T04:49:08.778 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.778 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.778 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:49:08.778 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.634377+0000", 2024-09-14T04:49:08.778 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.014388471999999999 2024-09-14T04:49:08.778 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.778 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.778 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:49:08.778 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.634453+0000", 2024-09-14T04:49:08.778 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.5547e-05 2024-09-14T04:49:08.779 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.779 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.779 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:49:08.779 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.641603+0000", 2024-09-14T04:49:08.779 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0071500160000000004 2024-09-14T04:49:08.779 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.779 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.779 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:49:08.779 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.641683+0000", 2024-09-14T04:49:08.779 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9748999999999998e-05 2024-09-14T04:49:08.779 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.779 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.779 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:49:08.779 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.641730+0000", 2024-09-14T04:49:08.780 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7474e-05 2024-09-14T04:49:08.780 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.780 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.780 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:49:08.780 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.645447+0000", 2024-09-14T04:49:08.780 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0037170829999999999 2024-09-14T04:49:08.780 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.780 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:49:08.780 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.780 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.780 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.780 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:221622 54.e 54:7f78e682:::benchmark_data_smithi039_264334_object95225:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1414)", 2024-09-14T04:49:08.780 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:46:43.763601+0000", 2024-09-14T04:49:08.780 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 144.86644144900001, 2024-09-14T04:49:08.781 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.37214667099999998, 2024-09-14T04:49:08.781 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:49:08.781 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:49:08.781 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:49:08.781 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:49:08.781 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:49:08.781 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:49:08.781 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 221622 2024-09-14T04:49:08.781 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.781 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:49:08.781 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.781 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:49:08.781 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.763601+0000", 2024-09-14T04:49:08.782 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.782 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.782 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.782 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:49:08.782 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.763601+0000", 2024-09-14T04:49:08.782 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.782 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.782 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.782 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:49:08.782 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.763603+0000", 2024-09-14T04:49:08.782 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2409999999999998e-06 2024-09-14T04:49:08.782 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.782 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.782 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:49:08.782 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.763609+0000", 2024-09-14T04:49:08.783 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3739999999999997e-06 2024-09-14T04:49:08.783 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.783 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.783 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:49:08.783 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.763622+0000", 2024-09-14T04:49:08.783 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2825e-05 2024-09-14T04:49:08.783 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.783 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.783 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:49:08.783 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.763629+0000", 2024-09-14T04:49:08.783 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.058e-06 2024-09-14T04:49:08.783 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.783 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.783 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.784 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.776965+0000", 2024-09-14T04:49:08.784 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.013336705000000001 2024-09-14T04:49:08.784 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.784 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.784 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for rw locks", 2024-09-14T04:49:08.784 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.777067+0000", 2024-09-14T04:49:08.784 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000101835 2024-09-14T04:49:08.784 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.784 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.784 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.784 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.094743+0000", 2024-09-14T04:49:08.784 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.31767551700000002 2024-09-14T04:49:08.785 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.785 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.785 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:49:08.785 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.094812+0000", 2024-09-14T04:49:08.785 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8900999999999996e-05 2024-09-14T04:49:08.785 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.785 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.785 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,4,7", 2024-09-14T04:49:08.785 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.094905+0000", 2024-09-14T04:49:08.785 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.3271999999999997e-05 2024-09-14T04:49:08.785 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.785 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.785 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:49:08.785 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.104187+0000", 2024-09-14T04:49:08.786 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0092817850000000007 2024-09-14T04:49:08.786 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.786 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.786 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:49:08.786 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.104237+0000", 2024-09-14T04:49:08.786 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.0448999999999998e-05 2024-09-14T04:49:08.786 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.786 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.786 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:49:08.786 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.104298+0000", 2024-09-14T04:49:08.786 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1105000000000005e-05 2024-09-14T04:49:08.786 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.786 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.786 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:49:08.787 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.132891+0000", 2024-09-14T04:49:08.787 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.028592474 2024-09-14T04:49:08.787 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.787 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.787 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:49:08.787 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.132979+0000", 2024-09-14T04:49:08.787 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.8424000000000006e-05 2024-09-14T04:49:08.787 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.787 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.787 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:49:08.787 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.135748+0000", 2024-09-14T04:49:08.787 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0027687060000000001 2024-09-14T04:49:08.787 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.788 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:49:08.788 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.788 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.788 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.788 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:221626 54.e 54:7a533153:::benchmark_data_smithi039_264334_object95229:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1414)", 2024-09-14T04:49:08.788 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:46:43.766483+0000", 2024-09-14T04:49:08.788 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 144.86355940000001, 2024-09-14T04:49:08.788 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.33506649199999999, 2024-09-14T04:49:08.788 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:49:08.788 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:49:08.788 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:49:08.788 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:49:08.788 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:49:08.788 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:49:08.789 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 221626 2024-09-14T04:49:08.789 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.789 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:49:08.789 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.789 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:49:08.789 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.766483+0000", 2024-09-14T04:49:08.789 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.789 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.789 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.789 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:49:08.789 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.766483+0000", 2024-09-14T04:49:08.789 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.789 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.790 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.790 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:49:08.790 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.766485+0000", 2024-09-14T04:49:08.790 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3400000000000001e-06 2024-09-14T04:49:08.790 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.790 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.790 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:49:08.790 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.766489+0000", 2024-09-14T04:49:08.790 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.8060000000000002e-06 2024-09-14T04:49:08.790 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.790 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.790 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:49:08.790 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.766501+0000", 2024-09-14T04:49:08.790 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1764e-05 2024-09-14T04:49:08.790 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.791 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.791 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:49:08.791 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.766509+0000", 2024-09-14T04:49:08.791 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9759999999999995e-06 2024-09-14T04:49:08.791 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.791 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.791 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.791 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.777093+0000", 2024-09-14T04:49:08.791 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.010584145999999999 2024-09-14T04:49:08.791 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.791 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.791 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:49:08.791 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.777219+0000", 2024-09-14T04:49:08.792 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000125733 2024-09-14T04:49:08.792 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.792 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.792 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,4,7", 2024-09-14T04:49:08.792 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.777366+0000", 2024-09-14T04:49:08.792 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014667700000000001 2024-09-14T04:49:08.792 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.792 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.792 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:49:08.792 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.785560+0000", 2024-09-14T04:49:08.792 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0081939179999999997 2024-09-14T04:49:08.792 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.792 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.793 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:49:08.793 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.788126+0000", 2024-09-14T04:49:08.793 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0025660409999999998 2024-09-14T04:49:08.793 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.793 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.793 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:49:08.793 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.094695+0000", 2024-09-14T04:49:08.793 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.306569698 2024-09-14T04:49:08.793 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.793 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.793 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:49:08.793 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.098206+0000", 2024-09-14T04:49:08.793 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0035103119999999998 2024-09-14T04:49:08.793 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.794 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.794 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:49:08.794 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.098256+0000", 2024-09-14T04:49:08.794 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.0528999999999997e-05 2024-09-14T04:49:08.794 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.794 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.794 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:49:08.794 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.101550+0000", 2024-09-14T04:49:08.794 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0032935519999999999 2024-09-14T04:49:08.794 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.794 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:49:08.794 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.794 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.795 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.795 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:221632 54.e 54:7a51d552:::benchmark_data_smithi039_264334_object95235:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1414)", 2024-09-14T04:49:08.795 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:46:43.774068+0000", 2024-09-14T04:49:08.795 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 144.85597466499999, 2024-09-14T04:49:08.795 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.330061361, 2024-09-14T04:49:08.795 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:49:08.795 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:49:08.795 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:49:08.795 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:49:08.795 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:49:08.795 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:49:08.795 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 221632 2024-09-14T04:49:08.795 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.796 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:49:08.796 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.796 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:49:08.796 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.774068+0000", 2024-09-14T04:49:08.796 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.796 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.796 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.796 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:49:08.796 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.774068+0000", 2024-09-14T04:49:08.796 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.796 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.796 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.796 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:49:08.796 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.774070+0000", 2024-09-14T04:49:08.797 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.719e-06 2024-09-14T04:49:08.797 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.797 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.797 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:49:08.797 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.774075+0000", 2024-09-14T04:49:08.797 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4550000000000003e-06 2024-09-14T04:49:08.797 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.797 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.797 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:49:08.797 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.774089+0000", 2024-09-14T04:49:08.797 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3382e-05 2024-09-14T04:49:08.797 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.797 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.797 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:49:08.798 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.774097+0000", 2024-09-14T04:49:08.798 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0679999999999997e-06 2024-09-14T04:49:08.798 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.798 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.798 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.798 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.781425+0000", 2024-09-14T04:49:08.798 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0073284120000000003 2024-09-14T04:49:08.798 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.798 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.798 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:49:08.798 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.781579+0000", 2024-09-14T04:49:08.798 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000153774 2024-09-14T04:49:08.798 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.798 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.799 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,4,7", 2024-09-14T04:49:08.799 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.781710+0000", 2024-09-14T04:49:08.799 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013134 2024-09-14T04:49:08.799 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.799 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.799 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:49:08.799 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.094706+0000", 2024-09-14T04:49:08.799 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.31299630099999998 2024-09-14T04:49:08.799 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.799 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.799 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:49:08.799 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.098165+0000", 2024-09-14T04:49:08.799 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.003458329 2024-09-14T04:49:08.800 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.800 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.800 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:49:08.800 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.101605+0000", 2024-09-14T04:49:08.800 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.003440598 2024-09-14T04:49:08.800 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.800 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.800 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:49:08.800 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.101653+0000", 2024-09-14T04:49:08.800 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7920000000000002e-05 2024-09-14T04:49:08.800 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.800 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.800 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:49:08.800 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.101713+0000", 2024-09-14T04:49:08.800 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.9905999999999999e-05 2024-09-14T04:49:08.801 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.801 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.801 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:49:08.801 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.104129+0000", 2024-09-14T04:49:08.801 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0024161569999999999 2024-09-14T04:49:08.801 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.801 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:49:08.801 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.801 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.801 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.801 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:221643 54.3 54:c3e67225:::benchmark_data_smithi039_264334_object95246:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1414)", 2024-09-14T04:49:08.801 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:46:43.785843+0000", 2024-09-14T04:49:08.801 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 144.84419985599999, 2024-09-14T04:49:08.802 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.31179128299999997, 2024-09-14T04:49:08.802 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:49:08.802 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:49:08.802 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:49:08.802 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:49:08.802 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:49:08.802 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:49:08.802 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 221643 2024-09-14T04:49:08.802 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.802 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:49:08.802 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.802 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:08.804 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:49:08.804 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.785843+0000", 2024-09-14T04:49:08.804 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.804 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.804 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.804 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:49:08.804 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.785843+0000", 2024-09-14T04:49:08.804 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.804 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.804 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.804 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:49:08.805 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.785844+0000", 2024-09-14T04:49:08.805 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.22e-06 2024-09-14T04:49:08.805 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.805 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.805 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:49:08.805 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.785848+0000", 2024-09-14T04:49:08.805 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4390000000000002e-06 2024-09-14T04:49:08.805 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.805 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.805 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:49:08.805 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.785860+0000", 2024-09-14T04:49:08.805 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1199000000000001e-05 2024-09-14T04:49:08.805 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.806 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.806 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:49:08.806 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.785867+0000", 2024-09-14T04:49:08.806 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1430000000000001e-06 2024-09-14T04:49:08.806 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.806 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.806 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.806 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.785920+0000", 2024-09-14T04:49:08.806 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3343000000000001e-05 2024-09-14T04:49:08.806 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.806 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.806 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:49:08.806 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.786026+0000", 2024-09-14T04:49:08.806 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000105801 2024-09-14T04:49:08.807 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.807 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.807 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 0,3", 2024-09-14T04:49:08.807 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.786122+0000", 2024-09-14T04:49:08.807 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.5799999999999998e-05 2024-09-14T04:49:08.807 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.807 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.807 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:49:08.807 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.097375+0000", 2024-09-14T04:49:08.807 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.31125290999999999 2024-09-14T04:49:08.807 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.807 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.807 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:49:08.808 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.097475+0000", 2024-09-14T04:49:08.808 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000100437 2024-09-14T04:49:08.808 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.808 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.808 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:49:08.808 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.097520+0000", 2024-09-14T04:49:08.808 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5012999999999997e-05 2024-09-14T04:49:08.808 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.808 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.808 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:49:08.808 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.097578+0000", 2024-09-14T04:49:08.808 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7463000000000002e-05 2024-09-14T04:49:08.808 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.808 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.808 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:49:08.808 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.097634+0000", 2024-09-14T04:49:08.809 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6514999999999997e-05 2024-09-14T04:49:08.809 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.809 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:49:08.809 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.809 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.809 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.809 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "MOSDPGPushReply(54.e 1414/1403 [PushReplyOp(54:7f78e682:::benchmark_data_smithi039_264334_object95225:head)])", 2024-09-14T04:49:08.809 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:46:43.787754+0000", 2024-09-14T04:49:08.809 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 144.842289002, 2024-09-14T04:49:08.809 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.30695541100000001, 2024-09-14T04:49:08.809 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:49:08.809 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:49:08.809 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T04:49:08.809 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:49:08.809 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.810 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:49:08.810 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.787754+0000", 2024-09-14T04:49:08.810 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.810 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.810 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.810 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:49:08.810 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.787754+0000", 2024-09-14T04:49:08.810 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.810 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.810 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.810 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:49:08.810 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.787754+0000", 2024-09-14T04:49:08.810 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9999999999999998e-07 2024-09-14T04:49:08.811 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.811 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.811 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:49:08.811 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.787758+0000", 2024-09-14T04:49:08.811 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.9400000000000004e-06 2024-09-14T04:49:08.811 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.811 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.811 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:49:08.811 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.787766+0000", 2024-09-14T04:49:08.811 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6580000000000002e-06 2024-09-14T04:49:08.811 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.811 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.811 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:49:08.811 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.787771+0000", 2024-09-14T04:49:08.812 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.2220000000000001e-06 2024-09-14T04:49:08.812 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.812 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.812 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.812 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.788200+0000", 2024-09-14T04:49:08.812 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00042895999999999999 2024-09-14T04:49:08.812 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.812 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.812 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:49:08.812 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.094709+0000", 2024-09-14T04:49:08.812 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.30650913099999999 2024-09-14T04:49:08.812 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.812 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:49:08.812 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.812 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.813 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.813 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop_reply(client.35436.0:221632 54.e e1414/1403 ondisk, result = 0)", 2024-09-14T04:49:08.813 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:46:43.788375+0000", 2024-09-14T04:49:08.813 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 144.841667831, 2024-09-14T04:49:08.813 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.30979790800000001, 2024-09-14T04:49:08.813 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:49:08.813 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:49:08.813 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "started", 2024-09-14T04:49:08.813 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:49:08.813 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.813 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:49:08.813 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.788375+0000", 2024-09-14T04:49:08.813 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.813 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.814 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.814 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:49:08.814 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.788375+0000", 2024-09-14T04:49:08.814 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.814 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.814 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.814 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:49:08.814 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.788375+0000", 2024-09-14T04:49:08.814 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6000000000000004e-07 2024-09-14T04:49:08.814 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.814 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.814 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:49:08.814 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.788378+0000", 2024-09-14T04:49:08.814 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.0599999999999999e-06 2024-09-14T04:49:08.814 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.815 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.815 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:49:08.815 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.788385+0000", 2024-09-14T04:49:08.815 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9059999999999996e-06 2024-09-14T04:49:08.815 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.815 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.815 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:49:08.815 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.788389+0000", 2024-09-14T04:49:08.815 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.0899999999999998e-06 2024-09-14T04:49:08.815 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.815 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.815 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.815 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.098150+0000", 2024-09-14T04:49:08.815 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.30976060100000002 2024-09-14T04:49:08.816 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.816 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.816 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:49:08.816 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.098158+0000", 2024-09-14T04:49:08.816 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0390000000000005e-06 2024-09-14T04:49:08.816 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.816 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.816 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:49:08.816 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.098173+0000", 2024-09-14T04:49:08.816 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4652e-05 2024-09-14T04:49:08.816 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.816 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:49:08.816 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.816 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.816 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.817 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:221645 54.3 54:cf626b17:::benchmark_data_smithi039_264334_object95248:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1414)", 2024-09-14T04:49:08.817 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:46:43.789694+0000", 2024-09-14T04:49:08.817 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 144.84034826000001, 2024-09-14T04:49:08.817 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.312450332, 2024-09-14T04:49:08.817 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:49:08.817 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:49:08.817 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:49:08.817 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:49:08.817 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:49:08.817 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:49:08.817 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 221645 2024-09-14T04:49:08.817 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.817 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:49:08.817 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.818 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:49:08.818 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.789694+0000", 2024-09-14T04:49:08.818 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.818 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.818 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.818 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:49:08.818 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.789694+0000", 2024-09-14T04:49:08.818 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.818 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.818 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.818 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:49:08.818 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.789696+0000", 2024-09-14T04:49:08.818 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.497e-06 2024-09-14T04:49:08.818 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.818 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.819 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:49:08.819 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.789702+0000", 2024-09-14T04:49:08.819 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2779999999999999e-06 2024-09-14T04:49:08.819 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.819 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.819 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:49:08.819 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.789730+0000", 2024-09-14T04:49:08.819 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.7549e-05 2024-09-14T04:49:08.819 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.819 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.819 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:49:08.819 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.789737+0000", 2024-09-14T04:49:08.819 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.4830000000000002e-06 2024-09-14T04:49:08.820 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.820 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.820 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.820 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.790121+0000", 2024-09-14T04:49:08.820 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00038405699999999999 2024-09-14T04:49:08.820 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.820 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.820 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:49:08.820 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.790230+0000", 2024-09-14T04:49:08.820 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000109002 2024-09-14T04:49:08.820 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.820 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.820 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 0,3", 2024-09-14T04:49:08.821 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.790375+0000", 2024-09-14T04:49:08.821 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000144995 2024-09-14T04:49:08.821 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.821 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.821 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:49:08.821 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.097449+0000", 2024-09-14T04:49:08.821 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.30707343500000001 2024-09-14T04:49:08.821 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.821 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.821 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:49:08.821 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.097701+0000", 2024-09-14T04:49:08.821 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000251969 2024-09-14T04:49:08.821 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.821 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.822 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:49:08.822 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.102048+0000", 2024-09-14T04:49:08.822 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0043475629999999996 2024-09-14T04:49:08.822 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.822 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.822 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:49:08.822 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.102101+0000", 2024-09-14T04:49:08.822 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.2689999999999999e-05 2024-09-14T04:49:08.822 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.822 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.822 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:49:08.822 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.102145+0000", 2024-09-14T04:49:08.822 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3813999999999998e-05 2024-09-14T04:49:08.822 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.822 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:49:08.823 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.823 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.823 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.823 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:221647 54.9 54:90543e7e:::benchmark_data_smithi039_264334_object95250:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1414)", 2024-09-14T04:49:08.823 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:46:43.789783+0000", 2024-09-14T04:49:08.823 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 144.840259997, 2024-09-14T04:49:08.823 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.31175572800000001, 2024-09-14T04:49:08.823 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:49:08.823 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:49:08.823 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:49:08.823 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:49:08.823 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:49:08.823 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:49:08.824 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 221647 2024-09-14T04:49:08.824 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.824 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:49:08.824 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.824 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:49:08.824 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.789783+0000", 2024-09-14T04:49:08.824 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.824 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.824 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.824 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:49:08.824 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.789783+0000", 2024-09-14T04:49:08.824 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.825 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.825 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.825 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:49:08.825 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.789783+0000", 2024-09-14T04:49:08.825 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.37e-07 2024-09-14T04:49:08.825 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.825 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.825 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:49:08.825 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.789788+0000", 2024-09-14T04:49:08.825 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5020000000000003e-06 2024-09-14T04:49:08.825 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.825 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.825 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:49:08.825 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.789795+0000", 2024-09-14T04:49:08.826 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.3089999999999998e-06 2024-09-14T04:49:08.826 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.826 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.826 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:49:08.826 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.789801+0000", 2024-09-14T04:49:08.826 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7119999999999997e-06 2024-09-14T04:49:08.826 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.826 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.826 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.826 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.789858+0000", 2024-09-14T04:49:08.826 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7188999999999997e-05 2024-09-14T04:49:08.826 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.826 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.826 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:49:08.826 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.789976+0000", 2024-09-14T04:49:08.827 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000118349 2024-09-14T04:49:08.827 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.827 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.827 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,6", 2024-09-14T04:49:08.827 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.790103+0000", 2024-09-14T04:49:08.827 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000127126 2024-09-14T04:49:08.827 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.827 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.827 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:49:08.827 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.097201+0000", 2024-09-14T04:49:08.827 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.30709784600000001 2024-09-14T04:49:08.827 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.827 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.827 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:49:08.828 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.097232+0000", 2024-09-14T04:49:08.828 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.1096000000000002e-05 2024-09-14T04:49:08.828 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.828 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.828 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:49:08.828 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.101387+0000", 2024-09-14T04:49:08.828 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0041547509999999999 2024-09-14T04:49:08.828 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.828 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.828 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:49:08.828 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.101450+0000", 2024-09-14T04:49:08.828 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2748000000000003e-05 2024-09-14T04:49:08.828 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.828 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.829 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:49:08.829 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.101538+0000", 2024-09-14T04:49:08.829 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.8462999999999996e-05 2024-09-14T04:49:08.829 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.829 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:49:08.829 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.829 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.829 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.829 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.35436.0:221651 54.1 e1414/1380 54:87e58355:::benchmark_data_smithi039_264334_object95254:head v 1414'13890, mlcod=1414'13888)", 2024-09-14T04:49:08.829 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:46:43.798660+0000", 2024-09-14T04:49:08.829 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 144.831382526, 2024-09-14T04:49:08.829 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.31284135600000001, 2024-09-14T04:49:08.829 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:49:08.829 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:49:08.830 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:49:08.830 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:49:08.830 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.830 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:49:08.830 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.798660+0000", 2024-09-14T04:49:08.830 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.830 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.830 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.830 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:49:08.830 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.798660+0000", 2024-09-14T04:49:08.830 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:49:08.830 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.830 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.830 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:49:08.831 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.798662+0000", 2024-09-14T04:49:08.831 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8029999999999999e-06 2024-09-14T04:49:08.831 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.831 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.831 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:49:08.831 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:43.798673+0000", 2024-09-14T04:49:08.831 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0913999999999999e-05 2024-09-14T04:49:08.831 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.831 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.831 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:49:08.831 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.093571+0000", 2024-09-14T04:49:08.831 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.29489777499999997 2024-09-14T04:49:08.831 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.831 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.831 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:49:08.832 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.093584+0000", 2024-09-14T04:49:08.832 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.323e-05 2024-09-14T04:49:08.832 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.832 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.832 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:49:08.832 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.093647+0000", 2024-09-14T04:49:08.832 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2912999999999999e-05 2024-09-14T04:49:08.832 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.832 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.832 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:49:08.832 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.093674+0000", 2024-09-14T04:49:08.832 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.7648e-05 2024-09-14T04:49:08.832 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.832 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.832 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:49:08.833 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.111061+0000", 2024-09-14T04:49:08.833 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.017386578 2024-09-14T04:49:08.833 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:49:08.833 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:49:08.833 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:49:08.833 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:46:44.111501+0000", 2024-09-14T04:49:08.833 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000440495 2024-09-14T04:49:08.833 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.833 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:49:08.833 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.833 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:49:08.833 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:49:08.833 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:49:08.887 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:49:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:49:08] "GET /metrics HTTP/1.1" 200 36756 "" "Prometheus/2.51.0" 2024-09-14T04:49:08.888 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:08.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:08.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:08.991 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:09.023 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:09.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:09.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:09.125 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:09.160 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:09.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:09.161 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:09.262 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:09.314 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:09.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:09.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:09.415 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:09.453 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:09.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:09.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:09.474 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is set 2024-09-14T04:49:09.555 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:09.558 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 68 16 103540 103524 95.1317 60.875 0.00761514 0.0105069 2024-09-14T04:49:09.598 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:09.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:09.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:09.633 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:09 smithi129 ceph-mon[101316]: pgmap v2594: 99 pgs: 99 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s 2024-09-14T04:49:09.633 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:09 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1460428926' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:49:09.633 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:09 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:49:09.700 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:09.732 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:09.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:09.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:09 smithi039 ceph-mon[106470]: pgmap v2594: 99 pgs: 99 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s 2024-09-14T04:49:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:09 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1460428926' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:49:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:09 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:49:09.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:09 smithi039 ceph-mon[111235]: pgmap v2594: 99 pgs: 99 active+clean; 5.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s 2024-09-14T04:49:09.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:09 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1460428926' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:49:09.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:09 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:49:09.834 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:09.871 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:09.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:09.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:09.973 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:10.005 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:10.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:10.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:10.108 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:10.109 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:49:10.109 INFO:teuthology.orchestra.run.smithi129.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-14T04:49:10.109 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:49:10.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:10.148 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:10.148 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:10.249 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:10.260 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-14T04:49:10.269 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:10.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:10.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:10.371 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:10.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:10.393 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:10.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:10.494 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:10.515 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:10.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:10.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:10.558 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 69 16 104294 104278 94.4359 47.125 0.00532389 0.010545 2024-09-14T04:49:10.617 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:10.624 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:49:10.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:10.651 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:10.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:10.752 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:10.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:10 smithi129 ceph-mon[101316]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:49:10.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:10 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:49:10.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:10 smithi129 ceph-mon[101316]: osdmap e1440: 8 total, 8 up, 8 in 2024-09-14T04:49:10.789 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:10.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:10 smithi039 ceph-mon[106470]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:49:10.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:10 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:49:10.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:10 smithi039 ceph-mon[106470]: osdmap e1440: 8 total, 8 up, 8 in 2024-09-14T04:49:10.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:10 smithi039 ceph-mon[111235]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:49:10.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:10 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:49:10.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:10 smithi039 ceph-mon[111235]: osdmap e1440: 8 total, 8 up, 8 in 2024-09-14T04:49:10.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:10.792 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:10.893 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:10.924 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:10.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:10.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:11.027 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:11.070 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:11.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:11.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:11.172 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:11.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:11.217 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:11.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:11.318 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:11.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:11.356 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:11.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:11.457 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:11.497 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:11.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:11.499 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:11.559 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 70 16 105304 105288 93.9877 63.125 0.0101175 0.0106345 2024-09-14T04:49:11.599 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:11.641 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:11.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:11.643 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:11.736 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:11 smithi129 ceph-mon[101316]: pgmap v2596: 99 pgs: 99 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-14T04:49:11.743 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:11.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:11.775 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:11.776 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:11.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:11 smithi039 ceph-mon[106470]: pgmap v2596: 99 pgs: 99 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-14T04:49:11.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:11 smithi039 ceph-mon[111235]: pgmap v2596: 99 pgs: 99 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-14T04:49:11.876 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:11.934 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:11.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:11.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:12.036 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:12.073 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:12.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:12.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:12.175 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:12.207 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:12.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:12.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:12.309 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:12.343 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:12.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:12.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:12.444 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:12.478 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:12.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:12.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:12.560 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 71 16 106401 106385 93.6283 68.5625 0.0113016 0.010675 2024-09-14T04:49:12.579 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:12.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:12.622 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:12.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:12.723 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:12.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:12.754 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:12.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:12.855 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:12.886 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:12.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:12.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:12.988 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:13.024 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:13.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:13.025 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:13.108 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:49:13.125 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:13.192 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:13.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:13.193 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:13.294 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:13.334 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:13.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:13.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:13.436 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:13.464 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:13.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:13.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:13.560 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 72 16 107435 107419 93.2254 64.625 0.0132588 0.0107213 2024-09-14T04:49:13.561 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:13 smithi039 ceph-mon[106470]: pgmap v2597: 99 pgs: 99 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2024-09-14T04:49:13.561 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:13 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2939929476' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:49:13.561 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:13 smithi039 ceph-mon[111235]: pgmap v2597: 99 pgs: 99 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2024-09-14T04:49:13.561 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:13 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2939929476' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:49:13.565 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:13.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:13.606 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:13.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:13.707 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:13.738 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:13.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:13.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:13.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:13 smithi129 ceph-mon[101316]: pgmap v2597: 99 pgs: 99 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 65 MiB/s wr, 1.05k op/s 2024-09-14T04:49:13.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:13 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2939929476' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:49:13.839 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:13.873 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:13.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:13.874 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:13.975 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:14.015 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:14.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:14.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:14.117 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:14.134 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:49:14.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:14.147 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:14.147 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:14.248 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:14.251 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:49:14.251 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:49:14.251 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:49:14.251 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:49:14.251 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:49:14.251 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:49:14.252 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:49:14.252 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:49:14.252 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:49:14.252 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:49:14.252 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:49:14.252 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:49:14.253 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:49:14.253 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:49:14.253 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:49:14.253 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:49:14.280 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:14.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:14.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:14.382 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:14.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:14.420 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:14.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:14.521 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:14.560 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 73 16 108498 108482 92.8584 66.4375 0.00724719 0.0107638 2024-09-14T04:49:14.570 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:14.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:14.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:14.672 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:14.708 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:14.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:14.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:14.810 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:14.849 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:14.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:14.849 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:14.950 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:14.990 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:14.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:14.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:15.092 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:15.126 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:15.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:15.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:15.147 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:49:15.227 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:15.271 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:15.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:15.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:15.373 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:15.408 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:15.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:15.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:15.510 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:15.550 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:15.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:15.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:15.560 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 74 16 109557 109541 92.498 66.1875 0.022098 0.010806 2024-09-14T04:49:15.652 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:15.688 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:15.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:15.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:15.736 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:15 smithi039 ceph-mon[106470]: pgmap v2598: 99 pgs: 99 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-14T04:49:15.737 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:15 smithi039 ceph-mon[111235]: pgmap v2598: 99 pgs: 99 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-14T04:49:15.791 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:15.840 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:15.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:15.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:15.942 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:15.975 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:15.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:15.976 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:16.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:15 smithi129 ceph-mon[101316]: pgmap v2598: 99 pgs: 99 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-14T04:49:16.077 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:16.109 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:16.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:16.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:16.211 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:16.247 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:16.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:16.249 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:16.349 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:16.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:16.397 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:16.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:16.497 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:16.533 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:16.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:16.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:16.560 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 75 16 110607 110591 92.1396 65.625 0.023804 0.0108486 2024-09-14T04:49:16.635 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:16.671 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:16.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:16.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:16.772 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:16.777 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-14T04:49:16.777 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:49:16.777 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-09-14T04:49:16.778 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.000000 2024-09-14T04:49:16.778 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 3 0 2024-09-14T04:49:16.793 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:16.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:16.794 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:16.895 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:16.925 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:16.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:16.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:17.027 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:17.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:17.064 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:17.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:17.105 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:49:17.165 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:17.202 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:17.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:17.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:17.304 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:17.339 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:17.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:17.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:17.440 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:17.461 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:17.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:17.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:17.560 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 76 16 111397 111381 91.5769 49.375 0.0121743 0.0109146 2024-09-14T04:49:17.563 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:17.588 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:17.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:17.589 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:17.690 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:17.729 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:17.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:17.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:17.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:17 smithi039 ceph-mon[106470]: pgmap v2599: 99 pgs: 99 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-14T04:49:17.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:17 smithi039 ceph-mon[111235]: pgmap v2599: 99 pgs: 99 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-14T04:49:17.831 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:17.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:17.864 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:17.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:17.965 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:17.999 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:17.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:18.000 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:18.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:17 smithi129 ceph-mon[101316]: pgmap v2599: 99 pgs: 99 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-14T04:49:18.101 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:18.140 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:18.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:18.141 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:18.241 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:18.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:18.280 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:18.280 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:18.381 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:18.414 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:18.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:18.415 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:18.516 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:18.548 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:18.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:18.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:18.560 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 77 16 112352 112336 91.1628 59.6875 0.0217551 0.0109646 2024-09-14T04:49:18.650 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:18.683 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:18.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:18.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:18.785 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:18.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:18.821 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:18.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:18.922 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:18.957 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:18.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:18.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:19.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:49:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:49:18] "GET /metrics HTTP/1.1" 200 36756 "" "Prometheus/2.51.0" 2024-09-14T04:49:19.060 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:19.093 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:19.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:19.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:19.195 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:19.230 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:19.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:19.232 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:19.332 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:19.368 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:19.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:19.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:19.470 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:19.508 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:19.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:19.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:19.560 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 78 16 113336 113320 90.7824 61.5 0.00526444 0.0110095 2024-09-14T04:49:19.610 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:19.648 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:19.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:19.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:19.750 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:19.779 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:19.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:19.780 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:19.881 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:19.913 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:19.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:19.914 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:20.014 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:20.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:19 smithi129 ceph-mon[101316]: pgmap v2600: 99 pgs: 99 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-09-14T04:49:20.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:19 smithi039 ceph-mon[106470]: pgmap v2600: 99 pgs: 99 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-09-14T04:49:20.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:19 smithi039 ceph-mon[111235]: pgmap v2600: 99 pgs: 99 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s 2024-09-14T04:49:20.055 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:20.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:20.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:20.157 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:20.190 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:20.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:20.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:20.293 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:20.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:20.354 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:20.354 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:20.455 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:20.490 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:20.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:20.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:20.561 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 79 16 114280 114264 90.3801 59 0.00545542 0.0110552 2024-09-14T04:49:20.591 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:20.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:20.638 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:20.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:20.739 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:20.776 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:20.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:20.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:20.878 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:20.920 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:20.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:20.921 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:21.022 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:21.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:21.058 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:21.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:21.159 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:21.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:21.195 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:21.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:21.296 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:21.356 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:21.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:21.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:21.457 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:21.519 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:21.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:21.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:21.552 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:21 smithi039 ceph-mon[106470]: pgmap v2601: 99 pgs: 99 active+clean; 6.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 60 MiB/s wr, 962 op/s 2024-09-14T04:49:21.553 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:21 smithi039 ceph-mon[111235]: pgmap v2601: 99 pgs: 99 active+clean; 6.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 60 MiB/s wr, 962 op/s 2024-09-14T04:49:21.561 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:2024-09-14T04:49:21.561083+0000 min lat: 0.00109434 max lat: 2.39824 avg lat: 0.0111004 2024-09-14T04:49:21.561 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:49:21.561 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 80 16 115269 115253 90.023 61.8125 0.00457017 0.0111004 2024-09-14T04:49:21.620 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:21.661 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:21.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:21.662 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:21.762 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:21.799 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:21.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:21.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:21.901 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:21.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:21.952 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:21.952 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:22.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:21 smithi129 ceph-mon[101316]: pgmap v2601: 99 pgs: 99 active+clean; 6.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 60 MiB/s wr, 962 op/s 2024-09-14T04:49:22.053 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:22.085 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:22.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:22.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:22.187 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:22.223 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:22.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:22.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:22.324 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:22.355 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:22.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:22.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:22.457 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:22.488 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:22.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:22.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:22.554 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:49:22.554 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:49:22.561 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 81 16 116215 116199 89.6415 59.125 0.00827757 0.0111502 2024-09-14T04:49:22.589 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:22.629 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:22.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:22.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:22.732 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:22.767 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:22.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:22.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:22.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:49:22.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:49:22.810 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:49:22.868 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:22.901 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:22.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:22.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:23.002 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:23.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:49:23.047 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:23.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:23.048 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:23.149 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:23.194 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:23.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:23.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:23.296 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:23.329 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:23.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:23.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:23.431 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:23.477 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:23.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:23.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:23.561 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 82 16 117197 117181 89.2968 61.375 0.0182978 0.0111935 2024-09-14T04:49:23.578 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:23.617 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:23.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:23.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:23.719 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:23.761 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:23.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:23.763 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:23.778 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:23 smithi039 ceph-mon[106470]: pgmap v2602: 99 pgs: 99 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s 2024-09-14T04:49:23.779 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:23 smithi039 ceph-mon[111235]: pgmap v2602: 99 pgs: 99 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s 2024-09-14T04:49:23.818 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-14T04:49:23.863 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:23.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:23.944 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:23.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:24.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:23 smithi129 ceph-mon[101316]: pgmap v2602: 99 pgs: 99 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 56 MiB/s wr, 896 op/s 2024-09-14T04:49:24.045 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:24.079 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:24.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:24.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:24.181 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:24.219 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:24.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:24.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:24.321 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:24.357 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:24.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:24.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:24.459 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:24.499 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:24.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:24.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:24.561 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 83 16 118127 118111 88.9212 58.125 0.0140287 0.0112408 2024-09-14T04:49:24.604 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:24.606 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is unset 2024-09-14T04:49:24.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:24.644 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:24.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:24.745 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:24.782 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:24.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:24.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:24.875 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:24 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4221315168' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:49:24.876 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:24 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4221315168' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:49:24.884 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:24.921 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:24.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:24.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:25.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:24 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4221315168' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:49:25.023 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:25.074 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:25.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:25.075 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:25.175 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:25.219 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:25.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:25.221 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:25.321 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:25.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:25.371 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:25.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:25.472 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:25.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:25.520 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:25.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:25.561 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 84 16 119142 119126 88.6178 63.4375 0.011495 0.0112794 2024-09-14T04:49:25.621 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:25.655 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:25.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:25.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:25.756 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:25 smithi039 ceph-mon[106470]: pgmap v2603: 99 pgs: 99 active+clean; 7.1 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-14T04:49:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:25 smithi039 ceph-mon[106470]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:49:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:25 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4221315168' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:49:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:25 smithi039 ceph-mon[106470]: osdmap e1441: 8 total, 8 up, 8 in 2024-09-14T04:49:25.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:25 smithi039 ceph-mon[111235]: pgmap v2603: 99 pgs: 99 active+clean; 7.1 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-14T04:49:25.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:25 smithi039 ceph-mon[111235]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:49:25.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:25 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4221315168' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:49:25.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:25 smithi039 ceph-mon[111235]: osdmap e1441: 8 total, 8 up, 8 in 2024-09-14T04:49:25.799 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:25.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:25.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:25.901 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:25.938 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:25.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:25.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:25 smithi129 ceph-mon[101316]: pgmap v2603: 99 pgs: 99 active+clean; 7.1 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-14T04:49:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:25 smithi129 ceph-mon[101316]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:49:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:25 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4221315168' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:49:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:25 smithi129 ceph-mon[101316]: osdmap e1441: 8 total, 8 up, 8 in 2024-09-14T04:49:26.040 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:26.078 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:26.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:26.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:26.180 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:26.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:26.224 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:26.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:26.325 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:26.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:26.366 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:26.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:26.468 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:26.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:26.509 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:26.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:26.561 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 85 16 120144 120128 88.3119 62.625 0.0100398 0.0113183 2024-09-14T04:49:26.610 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:26.628 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:49:26.642 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:26.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:26.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:26.744 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:26.787 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:26.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:26.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:26.889 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:26.918 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:26.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:26.919 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:27.020 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:27.052 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:27.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:27.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:27.069 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:49:27.154 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:27.187 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:27.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:27.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:27.289 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:27.338 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:27.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:27.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:27.440 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:27.479 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:27.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:27.480 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:27.561 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 86 16 121101 121085 87.9805 59.8125 0.0327909 0.0113606 2024-09-14T04:49:27.581 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:27.610 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:27.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:27.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:27.711 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:27 smithi039 ceph-mon[106470]: pgmap v2605: 99 pgs: 99 active+clean; 7.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2024-09-14T04:49:27.711 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:27 smithi039 ceph-mon[111235]: pgmap v2605: 99 pgs: 99 active+clean; 7.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2024-09-14T04:49:27.713 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:27.753 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:27.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:27.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:27.854 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:27.892 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:27.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:27.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:27.994 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:28.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:27 smithi129 ceph-mon[101316]: pgmap v2605: 99 pgs: 99 active+clean; 7.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2024-09-14T04:49:28.029 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:28.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:28.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:28.131 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:28.169 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:28.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:28.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:28.270 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:28.306 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:28.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:28.307 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:28.408 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:28.446 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:28.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:28.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:28.548 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:28.562 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 87 16 122096 122080 87.683 62.1875 0.0118941 0.0113996 2024-09-14T04:49:28.582 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:28.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:28.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:28.617 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:49:28.624 INFO:teuthology.orchestra.run.smithi039.stderr:set osd.3 primary-affinity to 0 (802) 2024-09-14T04:49:28.685 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:28.707 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:28.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:28.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:28.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:28 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4162566040' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.0} : dispatch 2024-09-14T04:49:28.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:28 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.0} : dispatch 2024-09-14T04:49:28.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:49:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:49:28] "GET /metrics HTTP/1.1" 200 36755 "" "Prometheus/2.51.0" 2024-09-14T04:49:28.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:28 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4162566040' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.0} : dispatch 2024-09-14T04:49:28.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:28 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.0} : dispatch 2024-09-14T04:49:28.809 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:28.830 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:28.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:28.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:28.931 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:28.955 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:28.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:28.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:29.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:28 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4162566040' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.0} : dispatch 2024-09-14T04:49:29.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:28 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 0.0} : dispatch 2024-09-14T04:49:29.056 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:29.078 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:29.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:29.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:29.180 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:29.203 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:29.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:29.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:29.305 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:29.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:29.328 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:29.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:29.429 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:29.453 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:29.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:29.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:29.554 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:29.562 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 88 16 122204 122188 86.7633 6.75 0.00504787 0.0114027 2024-09-14T04:49:29.577 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:29.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:29.578 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:29.679 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:29.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:29.718 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:29.718 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:29.819 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:29.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:29.855 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:29.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:29.921 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:29 smithi129 ceph-mon[101316]: pgmap v2606: 99 pgs: 99 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2024-09-14T04:49:29.921 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:29 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]': finished 2024-09-14T04:49:29.921 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:29 smithi129 ceph-mon[101316]: osdmap e1442: 8 total, 8 up, 8 in 2024-09-14T04:49:29.957 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:29.994 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:29.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:29.995 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:30.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:29 smithi039 ceph-mon[106470]: pgmap v2606: 99 pgs: 99 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2024-09-14T04:49:30.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:29 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]': finished 2024-09-14T04:49:30.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:29 smithi039 ceph-mon[106470]: osdmap e1442: 8 total, 8 up, 8 in 2024-09-14T04:49:30.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:29 smithi039 ceph-mon[111235]: pgmap v2606: 99 pgs: 99 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 61 MiB/s wr, 972 op/s 2024-09-14T04:49:30.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:29 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.0}]': finished 2024-09-14T04:49:30.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:29 smithi039 ceph-mon[111235]: osdmap e1442: 8 total, 8 up, 8 in 2024-09-14T04:49:30.096 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:30.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:30.135 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:30.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:30.236 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:30.277 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:30.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:30.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:30.378 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:30.415 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:30.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:30.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:30.517 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:30.547 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:30.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:30.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:30.562 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 89 16 123095 123079 86.4141 55.6875 0.0331342 0.0115675 2024-09-14T04:49:30.649 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:30.682 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:30.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:30.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:30.783 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:30.821 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:30.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:30.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:30.910 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:30 smithi039 ceph-mon[111235]: osdmap e1443: 8 total, 8 up, 8 in 2024-09-14T04:49:30.910 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:30 smithi039 ceph-mon[111235]: pgmap v2609: 99 pgs: 99 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s 2024-09-14T04:49:30.910 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:30 smithi039 ceph-mon[111235]: 1.4 scrub starts 2024-09-14T04:49:30.910 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:30 smithi039 ceph-mon[111235]: 1.4 scrub ok 2024-09-14T04:49:30.911 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:30 smithi039 ceph-mon[106470]: osdmap e1443: 8 total, 8 up, 8 in 2024-09-14T04:49:30.911 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:30 smithi039 ceph-mon[106470]: pgmap v2609: 99 pgs: 99 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s 2024-09-14T04:49:30.911 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:30 smithi039 ceph-mon[106470]: 1.4 scrub starts 2024-09-14T04:49:30.911 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:30 smithi039 ceph-mon[106470]: 1.4 scrub ok 2024-09-14T04:49:30.923 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:30.955 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:30.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:30.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:31.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:30 smithi129 ceph-mon[101316]: osdmap e1443: 8 total, 8 up, 8 in 2024-09-14T04:49:31.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:30 smithi129 ceph-mon[101316]: pgmap v2609: 99 pgs: 99 active+clean; 7.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s 2024-09-14T04:49:31.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:30 smithi129 ceph-mon[101316]: 1.4 scrub starts 2024-09-14T04:49:31.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:30 smithi129 ceph-mon[101316]: 1.4 scrub ok 2024-09-14T04:49:31.057 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:31.102 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:31.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:31.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:31.204 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:31.240 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:31.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:31.241 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:31.342 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:31.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:31.386 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:31.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:31.487 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:31.519 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:31.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:31.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:31.563 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Total time run: 90.0086 2024-09-14T04:49:31.563 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Total writes made: 123930 2024-09-14T04:49:31.563 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Write size: 65536 2024-09-14T04:49:31.563 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Object size: 65536 2024-09-14T04:49:31.563 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Bandwidth (MB/sec): 86.0543 2024-09-14T04:49:31.563 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Stddev Bandwidth: 40.4375 2024-09-14T04:49:31.563 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Max bandwidth (MB/sec): 227.688 2024-09-14T04:49:31.563 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Min bandwidth (MB/sec): 0 2024-09-14T04:49:31.563 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Average IOPS: 1376 2024-09-14T04:49:31.564 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Stddev IOPS: 647.001 2024-09-14T04:49:31.564 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Max IOPS: 3643 2024-09-14T04:49:31.564 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Min IOPS: 0 2024-09-14T04:49:31.564 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Average Latency(s): 0.0116162 2024-09-14T04:49:31.564 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Stddev Latency(s): 0.0324861 2024-09-14T04:49:31.564 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Max latency(s): 2.39824 2024-09-14T04:49:31.564 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Min latency(s): 0.00109434 2024-09-14T04:49:31.570 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Cleaning up (deleting benchmark objects) 2024-09-14T04:49:31.621 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:31.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:31.660 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:31.660 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:31.761 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:31.769 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:49:31.769 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:49:31.793 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:49:31.794 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:49:31.802 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:31.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:31.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:31.818 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:49:31.818 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:49:31.849 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:49:31.849 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:49:31.885 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:49:31.885 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:49:31.904 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:31.921 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:49:31.921 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:49:31.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:31.943 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:31.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:31.952 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:49:31.952 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:49:32.006 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:49:32.007 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:49:32.044 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:32.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:32.089 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:32.089 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:32.190 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:32.227 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:32.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:32.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:32.329 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:32.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:32.371 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:32.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:32.473 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:32.505 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:32.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:32.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:32.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:32 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:49:32.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:32 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:49:32.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:32 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2352850630' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:49:32.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:32 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:49:32.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:32 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:49:32.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:32 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:49:32.608 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:32.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:32.649 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:32.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:32.750 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:32.759 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:32 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:49:32.759 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:32 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:49:32.759 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:32 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2352850630' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:49:32.759 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:32 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:49:32.759 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:32 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:49:32.759 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:32 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:49:32.760 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:32 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:49:32.760 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:32 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:49:32.760 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:32 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2352850630' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:49:32.760 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:32 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:49:32.760 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:32 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:49:32.760 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:32 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:49:32.789 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:32.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:32.790 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:32.891 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:32.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:32.928 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:32.929 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:33.029 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:33.053 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:49:33.054 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [], 2024-09-14T04:49:33.054 INFO:teuthology.orchestra.run.smithi039.stdout: "complaint_time": 30, 2024-09-14T04:49:33.054 INFO:teuthology.orchestra.run.smithi039.stdout: "num_blocked_ops": 0 2024-09-14T04:49:33.054 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:49:33.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:33.094 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:33.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:33.195 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:33.228 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:33.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:33.229 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:33.330 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:33.364 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:33.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:33.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:33.467 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:33.503 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:33.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:33.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:33.597 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:33 smithi039 ceph-mon[106470]: pgmap v2610: 99 pgs: 99 active+clean; 7.4 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 49 MiB/s wr, 786 op/s 2024-09-14T04:49:33.597 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:33 smithi039 ceph-mon[111235]: pgmap v2610: 99 pgs: 99 active+clean; 7.4 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 49 MiB/s wr, 786 op/s 2024-09-14T04:49:33.605 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:33.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:33.644 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:33.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:33.746 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:33.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:33 smithi129 ceph-mon[101316]: pgmap v2610: 99 pgs: 99 active+clean; 7.4 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 49 MiB/s wr, 786 op/s 2024-09-14T04:49:33.778 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:33.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:33.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:33.880 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:33.915 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:33.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:33.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:34.017 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:34.049 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:34.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:34.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:34.151 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:34.180 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:34.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:34.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:34.282 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:34.318 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:34.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:34.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:34.420 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:34.458 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:34.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:34.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:34.560 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:34.604 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:34.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:34.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:34.706 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:34.750 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:34.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:34.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:34.852 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:34.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:34.888 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:34.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:34.989 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:35.022 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:35.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:35.023 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:35.124 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:35.165 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:35.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:35.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:35.267 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:35.287 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:35.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:35.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:35.389 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:35.423 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:35.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:35.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:35.524 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:35.556 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:35.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:35.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:35.637 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:49:35.657 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:35.677 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:35 smithi039 ceph-mon[106470]: pgmap v2611: 99 pgs: 99 active+clean; 7.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 127 B/s rd, 56 MiB/s wr, 1.07k op/s 2024-09-14T04:49:35.677 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:35 smithi039 ceph-mon[111235]: pgmap v2611: 99 pgs: 99 active+clean; 7.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 127 B/s rd, 56 MiB/s wr, 1.07k op/s 2024-09-14T04:49:35.692 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:35.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:35.693 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:35.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:35 smithi129 ceph-mon[101316]: pgmap v2611: 99 pgs: 99 active+clean; 7.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 127 B/s rd, 56 MiB/s wr, 1.07k op/s 2024-09-14T04:49:35.794 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:35.818 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-14T04:49:35.822 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:49:35.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:35.870 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:35.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:35.971 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:36.023 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:36.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:36.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:36.125 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:36.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:36.181 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:36.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:36.282 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:36.317 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:36.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:36.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:36.418 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:36.452 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:36.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:36.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:36.554 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:36.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:36.585 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:36.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:36.686 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:36.741 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:36.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:36.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:36.742 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:49:36.752 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:49:36.843 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:36.883 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:36.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:36.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:36.985 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:37.030 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:37.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:37.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:37.132 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:37.172 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:37.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:37.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:37.274 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:37.309 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:37.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:37.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:37.411 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:37.454 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:37.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:37.455 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:37.511 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:37 smithi039 ceph-mon[106470]: pgmap v2612: 99 pgs: 99 active+clean; 7.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 127 B/s rd, 52 MiB/s wr, 1.03k op/s 2024-09-14T04:49:37.511 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:49:37.511 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:37 smithi039 ceph-mon[111235]: pgmap v2612: 99 pgs: 99 active+clean; 7.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 127 B/s rd, 52 MiB/s wr, 1.03k op/s 2024-09-14T04:49:37.511 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:49:37.556 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:37.602 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:37.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:37.604 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:37.705 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:37.750 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:37.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:37.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:37.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:37 smithi129 ceph-mon[101316]: pgmap v2612: 99 pgs: 99 active+clean; 7.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 127 B/s rd, 52 MiB/s wr, 1.03k op/s 2024-09-14T04:49:37.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:49:37.852 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:37.894 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:37.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:37.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:37.996 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:38.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:38.035 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:38.035 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:38.041 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-14T04:49:38.042 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:49:38.042 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-09-14T04:49:38.042 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.605287 2024-09-14T04:49:38.042 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 1 0.6052867485352801 2024-09-14T04:49:38.136 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:38.172 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:38.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:38.173 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:38.274 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:38.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:38.318 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:38.318 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:38.419 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:38.453 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:38.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:38.454 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:38.555 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:38.589 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:38.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:38.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:38.689 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:49:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:49:38] "GET /metrics HTTP/1.1" 200 36753 "" "Prometheus/2.51.0" 2024-09-14T04:49:38.691 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:38.724 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:38.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:38.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:38.826 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:38.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:38.867 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:38.867 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:38.968 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:39.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:39.005 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:39.005 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:39.106 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:39.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:39.140 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:39.140 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:39.241 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:39.279 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:39.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:39.280 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:39.381 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:39.389 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:39 smithi039 ceph-mon[106470]: pgmap v2613: 99 pgs: 99 active+clean; 7.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 112 B/s rd, 19 MiB/s wr, 533 op/s 2024-09-14T04:49:39.389 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:39 smithi039 ceph-mon[111235]: pgmap v2613: 99 pgs: 99 active+clean; 7.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 112 B/s rd, 19 MiB/s wr, 533 op/s 2024-09-14T04:49:39.416 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:39.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:39.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:39.518 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:39.558 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:39.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:39.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:39.660 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:39.688 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:39.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:39.689 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:39.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:39 smithi129 ceph-mon[101316]: pgmap v2613: 99 pgs: 99 active+clean; 7.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 112 B/s rd, 19 MiB/s wr, 533 op/s 2024-09-14T04:49:39.790 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:39.829 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:39.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:39.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:39.931 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:39.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:39.967 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:39.967 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:40.068 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:40.103 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:40.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:40.104 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:40.205 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:40.239 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:40.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:40.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:40.341 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:40.377 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is unset 2024-09-14T04:49:40.381 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:40.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:40.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:40.483 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:40.531 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:40.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:40.532 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:40.632 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:40.671 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:40.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:40.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:40.711 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:40 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2182901263' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:49:40.711 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:40 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2182901263' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:49:40.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:40 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2182901263' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:49:40.773 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:40.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:40.806 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:40.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:40.907 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:40.943 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:40.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:40.944 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:41.045 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:41.079 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:41.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:41.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:41.181 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:41.214 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:41.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:41.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:41.316 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:41.361 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:41.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:41.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:41.463 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:41.498 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:41.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:41.499 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:41.600 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:41.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:41.635 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:41.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:41.665 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:41 smithi039 ceph-mon[106470]: pgmap v2614: 99 pgs: 99 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 100 B/s rd, 17 MiB/s wr, 878 op/s 2024-09-14T04:49:41.665 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:41 smithi039 ceph-mon[106470]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:49:41.665 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:41 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T04:49:41.665 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:41 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2182901263' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:49:41.665 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:41 smithi039 ceph-mon[106470]: osdmap e1444: 8 total, 8 up, 8 in 2024-09-14T04:49:41.665 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:41 smithi039 ceph-mon[111235]: pgmap v2614: 99 pgs: 99 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 100 B/s rd, 17 MiB/s wr, 878 op/s 2024-09-14T04:49:41.665 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:41 smithi039 ceph-mon[111235]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:49:41.665 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:41 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T04:49:41.666 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:41 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2182901263' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:49:41.666 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:41 smithi039 ceph-mon[111235]: osdmap e1444: 8 total, 8 up, 8 in 2024-09-14T04:49:41.736 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:41.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:41 smithi129 ceph-mon[101316]: pgmap v2614: 99 pgs: 99 active+clean; 7.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 100 B/s rd, 17 MiB/s wr, 878 op/s 2024-09-14T04:49:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:41 smithi129 ceph-mon[101316]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:49:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:41 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T04:49:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:41 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2182901263' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:49:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:41 smithi129 ceph-mon[101316]: osdmap e1444: 8 total, 8 up, 8 in 2024-09-14T04:49:41.773 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:41.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:41.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:41.784 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:49:41.850 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:49:41.851 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:49:41.875 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:41.883 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:49:41.883 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:49:41.910 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:41.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:41.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:41.914 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:49:41.914 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:49:41.947 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:49:41.948 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:49:41.979 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:49:41.979 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:49:42.011 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:49:42.012 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:49:42.012 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:42.049 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:49:42.049 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:49:42.062 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:42.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:42.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:42.088 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:49:42.088 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:49:42.164 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:42.199 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:42.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:42.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:42.301 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:42.335 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:42.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:42.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:42.437 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:42.457 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:42 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2369316346' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:49:42.458 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:42 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2369316346' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:49:42.478 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:42.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:42.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:42.580 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:42.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:42.627 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:42.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:42.728 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:42.763 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:42.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:42.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:42.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:42 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2369316346' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:49:42.865 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:42.898 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:42.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:42.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:43.000 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:43.036 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:43.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:43.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:43.139 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:43.174 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:43.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:43.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:43.276 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:43.306 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:43.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:43.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:43.352 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:49:43.408 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:43.447 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:43.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:43.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:43.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:43 smithi039 ceph-mon[106470]: pgmap v2616: 99 pgs: 99 active+clean; 7.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 13 MiB/s wr, 914 op/s 2024-09-14T04:49:43.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:43 smithi039 ceph-mon[111235]: pgmap v2616: 99 pgs: 99 active+clean; 7.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 13 MiB/s wr, 914 op/s 2024-09-14T04:49:43.549 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:43.595 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:43.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:43.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:43.697 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:43.703 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:49:43.704 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:49:43.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:43.739 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:43.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:43.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:43 smithi129 ceph-mon[101316]: pgmap v2616: 99 pgs: 99 active+clean; 7.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 13 MiB/s wr, 914 op/s 2024-09-14T04:49:43.840 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:43.874 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:43.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:43.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:43.976 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:44.010 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:44.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:44.011 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:44.112 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:44.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:44.152 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:44.152 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:44.253 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:44.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:44.297 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:44.297 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:44.398 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:44.431 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:44.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:44.432 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:44.496 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-14T04:49:44.532 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:44.564 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:44.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:44.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:44.666 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:44.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:44.703 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:44.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:44.804 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:44.840 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:44.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:44.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:44.942 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:44.980 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:44.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:44.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:45.081 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:45.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:45.115 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:45.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:45.216 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:45.274 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:45.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:45.275 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:45.304 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:49:45.376 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:45.410 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:45.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:45.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:45.512 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:45.515 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:45 smithi039 ceph-mon[106470]: pgmap v2617: 99 pgs: 99 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 908 KiB/s wr, 978 op/s 2024-09-14T04:49:45.515 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:45 smithi039 ceph-mon[111235]: pgmap v2617: 99 pgs: 99 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 908 KiB/s wr, 978 op/s 2024-09-14T04:49:45.529 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:49:45.545 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:45.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:45.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:45.625 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:49:45.646 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:45.672 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:45.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:45.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:45.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:45 smithi129 ceph-mon[101316]: pgmap v2617: 99 pgs: 99 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 908 KiB/s wr, 978 op/s 2024-09-14T04:49:45.774 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:45.821 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:45.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:45.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:45.923 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:45.956 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:45.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:45.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:46.058 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:46.085 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:46.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:46.086 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:46.187 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:46.223 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:46.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:46.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:46.324 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:46.363 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:46.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:46.365 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:46.465 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:46.499 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:46.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:46.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:46.601 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:46.633 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:46.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:46.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:46.735 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:46.774 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:46.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:46.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:46.876 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:46.916 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:46.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:46.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:47.018 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:47.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:47.057 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:47.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:47.158 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:47.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:47.194 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:47.194 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:47.295 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:47.326 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:47.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:47.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:47.428 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:47.460 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:47.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:47.461 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:47.562 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:47.593 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:47.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:47.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:47.696 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:47.735 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:47.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:47.736 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:47.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:47 smithi129 ceph-mon[101316]: pgmap v2618: 99 pgs: 99 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 352 KiB/s wr, 981 op/s 2024-09-14T04:49:47.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:47 smithi039 ceph-mon[106470]: pgmap v2618: 99 pgs: 99 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 352 KiB/s wr, 981 op/s 2024-09-14T04:49:47.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:47 smithi039 ceph-mon[111235]: pgmap v2618: 99 pgs: 99 active+clean; 6.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 352 KiB/s wr, 981 op/s 2024-09-14T04:49:47.837 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:47.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:47.871 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:47.872 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:47.972 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:48.008 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:48.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:48.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:48.110 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:48.148 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:48.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:48.149 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:48.249 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:48.286 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:48.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:48.287 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:48.388 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:48.429 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:48.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:48.430 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:48.431 INFO:teuthology.orchestra.run.smithi039.stderr:set osd.1 primary-affinity to 0.605287 (8396682) 2024-09-14T04:49:48.531 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:48.564 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:48.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:48.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:48.668 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:48.712 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:48.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:48.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:48.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:48 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3497506692' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 0.6052867485352801} : dispatch 2024-09-14T04:49:48.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:48 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 0.6052867485352801} : dispatch 2024-09-14T04:49:48.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:48 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3497506692' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 0.6052867485352801} : dispatch 2024-09-14T04:49:48.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:48 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 0.6052867485352801} : dispatch 2024-09-14T04:49:48.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:49:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:49:48] "GET /metrics HTTP/1.1" 200 36753 "" "Prometheus/2.51.0" 2024-09-14T04:49:48.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:48 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3497506692' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 0.6052867485352801} : dispatch 2024-09-14T04:49:48.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:48 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 0.6052867485352801} : dispatch 2024-09-14T04:49:48.814 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:48.860 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:48.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:48.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:48.962 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:49.003 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:49.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:49.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:49.105 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:49.149 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:49.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:49.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:49.251 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:49.285 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:49.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:49.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:49.386 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:49.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:49.424 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:49.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:49.525 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:49.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:49 smithi039 ceph-mon[106470]: pgmap v2619: 99 pgs: 99 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 986 op/s 2024-09-14T04:49:49.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:49 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.6052867485352801}]': finished 2024-09-14T04:49:49.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:49 smithi039 ceph-mon[106470]: osdmap e1445: 8 total, 8 up, 8 in 2024-09-14T04:49:49.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:49 smithi039 ceph-mon[111235]: pgmap v2619: 99 pgs: 99 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 986 op/s 2024-09-14T04:49:49.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:49 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.6052867485352801}]': finished 2024-09-14T04:49:49.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:49 smithi039 ceph-mon[111235]: osdmap e1445: 8 total, 8 up, 8 in 2024-09-14T04:49:49.559 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:49.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:49.560 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:49.661 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:49.690 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:49.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:49.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:49.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:49 smithi129 ceph-mon[101316]: pgmap v2619: 99 pgs: 99 active+clean; 6.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 986 op/s 2024-09-14T04:49:49.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:49 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.6052867485352801}]': finished 2024-09-14T04:49:49.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:49 smithi129 ceph-mon[101316]: osdmap e1445: 8 total, 8 up, 8 in 2024-09-14T04:49:49.792 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:49.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:49.843 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:49.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:49.944 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:49.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:49.983 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:49.984 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:50.084 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:50.122 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:50.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:50.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:50.224 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:50.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:50.261 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:50.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:50.362 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:50.409 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:50.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:50.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:50.455 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is set 2024-09-14T04:49:50.511 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:50.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:50.550 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:50.551 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:50.651 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:50.698 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:50.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:50.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:50.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:50 smithi129 ceph-mon[101316]: osdmap e1446: 8 total, 8 up, 8 in 2024-09-14T04:49:50.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:50 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1935672456' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:49:50.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:50 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:49:50.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:50 smithi039 ceph-mon[106470]: osdmap e1446: 8 total, 8 up, 8 in 2024-09-14T04:49:50.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:50 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1935672456' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:49:50.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:50 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:49:50.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:50 smithi039 ceph-mon[111235]: osdmap e1446: 8 total, 8 up, 8 in 2024-09-14T04:49:50.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:50 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1935672456' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:49:50.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:50 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:49:50.800 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:50.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:50.833 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:50.833 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:50.934 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:50.972 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:50.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:50.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:51.074 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:51.106 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:51.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:51.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:51.208 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:51.242 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:51.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:51.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:51.344 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:51.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:51.381 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:51.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:51.482 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:51.520 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:51.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:51.521 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:51.622 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:51.654 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:51.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:51.655 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:51.677 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:51 smithi039 ceph-mon[106470]: pgmap v2622: 99 pgs: 1 peering, 98 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 9.4 KiB/s rd, 27 KiB/s wr, 1.11k op/s; 20 KiB/s, 0 objects/s recovering 2024-09-14T04:49:51.677 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:51 smithi039 ceph-mon[106470]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:49:51.677 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:51 smithi039 ceph-mon[106470]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:49:51.678 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:51 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:49:51.678 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:51 smithi039 ceph-mon[106470]: osdmap e1447: 8 total, 8 up, 8 in 2024-09-14T04:49:51.678 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:51 smithi039 ceph-mon[111235]: pgmap v2622: 99 pgs: 1 peering, 98 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 9.4 KiB/s rd, 27 KiB/s wr, 1.11k op/s; 20 KiB/s, 0 objects/s recovering 2024-09-14T04:49:51.678 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:51 smithi039 ceph-mon[111235]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:49:51.678 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:51 smithi039 ceph-mon[111235]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:49:51.678 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:51 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:49:51.678 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:51 smithi039 ceph-mon[111235]: osdmap e1447: 8 total, 8 up, 8 in 2024-09-14T04:49:51.688 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:49:51.732 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:49:51.732 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:49:51.756 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:51.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:51 smithi129 ceph-mon[101316]: pgmap v2622: 99 pgs: 1 peering, 98 active+clean; 6.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 9.4 KiB/s rd, 27 KiB/s wr, 1.11k op/s; 20 KiB/s, 0 objects/s recovering 2024-09-14T04:49:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:51 smithi129 ceph-mon[101316]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:49:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:51 smithi129 ceph-mon[101316]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:49:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:51 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:49:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:51 smithi129 ceph-mon[101316]: osdmap e1447: 8 total, 8 up, 8 in 2024-09-14T04:49:51.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:51.810 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:51.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:51.911 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:51.944 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:51.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:51.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:52.046 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:52.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:52.087 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:52.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:52.188 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:52.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:52.226 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:52.227 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:52.327 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:52.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:52.362 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:52.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:52.463 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:52.497 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:52.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:52.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:52.599 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:52.637 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:52.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:52.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:52.739 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:52.774 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:52.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:52.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:52.876 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:52.910 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:52.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:52.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:52.958 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-14T04:49:53.012 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:53.065 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:53.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:53.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:53.166 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:53.210 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:53.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:53.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:53.312 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:53.342 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:53.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:53.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:53.425 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:49:53.444 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:53.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:53.509 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:53.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:53.609 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:53.631 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:53.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:53.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:53.733 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:53.766 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:53.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:53.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:53.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:53 smithi129 ceph-mon[101316]: pgmap v2624: 99 pgs: 1 peering, 98 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 12 KiB/s rd, 37 KiB/s wr, 931 op/s; 26 KiB/s, 0 objects/s recovering 2024-09-14T04:49:53.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:53 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:49:53.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:53 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:49:53.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:53 smithi039 ceph-mon[106470]: pgmap v2624: 99 pgs: 1 peering, 98 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 12 KiB/s rd, 37 KiB/s wr, 931 op/s; 26 KiB/s, 0 objects/s recovering 2024-09-14T04:49:53.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:53 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:49:53.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:53 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:49:53.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:53 smithi039 ceph-mon[111235]: pgmap v2624: 99 pgs: 1 peering, 98 active+clean; 6.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 12 KiB/s rd, 37 KiB/s wr, 931 op/s; 26 KiB/s, 0 objects/s recovering 2024-09-14T04:49:53.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:53 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:49:53.794 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:53 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:49:53.868 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:53.905 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:53.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:53.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:54.007 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:54.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:54.056 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:54.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:54.157 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:54.195 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:54.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:54.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:54.297 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:54.332 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:54.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:54.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:54.434 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:54.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:54.470 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:54.470 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:54.570 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:54.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:54.611 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:54.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:54.712 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:54.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:54.750 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:54.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:54.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:54 smithi129 ceph-mon[101316]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-14T04:49:54.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:54 smithi039 ceph-mon[106470]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-14T04:49:54.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:54 smithi039 ceph-mon[111235]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-14T04:49:54.852 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:54.886 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:54.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:54.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:54.988 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:55.010 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:55.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:55.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:55.112 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:55.142 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:55.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:55.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:55.244 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:49:55.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:55.282 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:55.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:49:55.370 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-14T04:49:55.371 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:49:55.371 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_2 2024-09-14T04:49:55.372 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:49:55.384 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:55.452 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:55.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:55.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:55.554 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:55.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:55.601 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:55.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:55.702 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:55.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:55.735 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:55.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:55.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:55 smithi129 ceph-mon[101316]: pgmap v2625: 99 pgs: 99 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 12 KiB/s rd, 37 KiB/s wr, 1.46k op/s; 26 KiB/s, 0 objects/s recovering 2024-09-14T04:49:55.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:55 smithi039 ceph-mon[106470]: pgmap v2625: 99 pgs: 99 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 12 KiB/s rd, 37 KiB/s wr, 1.46k op/s; 26 KiB/s, 0 objects/s recovering 2024-09-14T04:49:55.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:55 smithi039 ceph-mon[111235]: pgmap v2625: 99 pgs: 99 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 12 KiB/s rd, 37 KiB/s wr, 1.46k op/s; 26 KiB/s, 0 objects/s recovering 2024-09-14T04:49:55.836 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:55.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:55.869 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:55.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:55.970 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:56.004 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:56.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:56.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:56.106 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:56.138 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:56.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:56.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:56.239 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:56.278 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:56.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:56.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:56.379 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:56.421 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:56.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:56.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:56.523 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:56.566 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:56.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:56.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:56.653 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:49:56.669 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:56.678 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:49:56.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:56.755 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:56.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:56.855 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:56.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:56.897 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:56.897 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:56.998 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:57.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:57.037 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:57.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:57.138 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:57.169 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:57.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:57.170 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:57.271 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:57.303 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:57.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:57.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:57.404 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:57.448 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:49:57.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:57.454 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:57.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:57.526 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:49:57.526 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:49:57.556 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:57.572 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:49:57.572 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:49:57.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:57.601 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:57.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:57.608 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:49:57.609 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:49:57.645 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:49:57.645 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:49:57.645 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:57 smithi039 ceph-mon[106470]: pgmap v2626: 99 pgs: 99 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 10 KiB/s rd, 30 KiB/s wr, 979 op/s; 21 KiB/s, 0 objects/s recovering 2024-09-14T04:49:57.646 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:57 smithi039 ceph-mon[111235]: pgmap v2626: 99 pgs: 99 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 10 KiB/s rd, 30 KiB/s wr, 979 op/s; 21 KiB/s, 0 objects/s recovering 2024-09-14T04:49:57.674 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:49:57.674 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:49:57.702 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:57.705 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:49:57.705 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:49:57.746 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:49:57.746 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:49:57.746 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:57.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:57.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:57.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:57 smithi129 ceph-mon[101316]: pgmap v2626: 99 pgs: 99 active+clean; 6.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 10 KiB/s rd, 30 KiB/s wr, 979 op/s; 21 KiB/s, 0 objects/s recovering 2024-09-14T04:49:57.776 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:49:57.777 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:49:57.848 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:57.884 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:57.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:57.885 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:57.985 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:58.024 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:58.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:58.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:58.126 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:58.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:58.155 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:58.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:58.256 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:58.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:58.302 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:58.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:58.403 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:58.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:58.435 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:58.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:58.536 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:58.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:58 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2092856726' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:49:58.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:58 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2092856726' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:49:58.557 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:58.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:58.558 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:58.659 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:58.682 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:58.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:58.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:58.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:58 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2092856726' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:49:58.784 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:49:58.805 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:58.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:58.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:49:58.907 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:49:58.941 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:58.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:58.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:49:59.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:49:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:49:58] "GET /metrics HTTP/1.1" 200 36782 "" "Prometheus/2.51.0" 2024-09-14T04:49:59.043 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:59.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:59.079 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:59.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:59.180 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:49:59.215 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:59.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:59.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:49:59.317 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:49:59.351 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:59.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:59.352 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:49:59.453 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:49:59.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:59.495 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:59.495 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:49:59.596 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:49:59.641 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:49:59.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:59.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:49:59.671 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:49:59 smithi129 ceph-mon[101316]: pgmap v2627: 99 pgs: 99 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2024-09-14T04:49:59.743 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:59.787 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:59.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:59.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:49:59.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:49:59 smithi039 ceph-mon[106470]: pgmap v2627: 99 pgs: 99 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2024-09-14T04:49:59.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:49:59 smithi039 ceph-mon[111235]: pgmap v2627: 99 pgs: 99 active+clean; 6.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 670 op/s 2024-09-14T04:49:59.889 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:49:59.921 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:49:59.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:49:59.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:00.023 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:00.068 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:00.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:00.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:00.169 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:00.205 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:00.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:00.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:00.307 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:00.340 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:00.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:00.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:00.443 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:00.479 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:00.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:00.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:00.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:00 smithi039 ceph-mon[106470]: Health detail: HEALTH_WARN noscrub flag(s) set 2024-09-14T04:50:00.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:00 smithi039 ceph-mon[106470]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-09-14T04:50:00.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:00 smithi039 ceph-mon[111235]: Health detail: HEALTH_WARN noscrub flag(s) set 2024-09-14T04:50:00.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:00 smithi039 ceph-mon[111235]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-09-14T04:50:00.581 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:00.618 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:00.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:00.620 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:00.720 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:00.765 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:00.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:00.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:00.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:00 smithi129 ceph-mon[101316]: Health detail: HEALTH_WARN noscrub flag(s) set 2024-09-14T04:50:00.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:00 smithi129 ceph-mon[101316]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2024-09-14T04:50:00.867 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:00.904 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:00.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:00.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:01.006 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:01.043 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:01.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:01.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:01.145 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:01.195 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:01.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:01.196 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:01.297 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:01.342 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:01.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:01.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:01.444 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:01.485 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:01.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:01.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:01.539 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:01 smithi039 ceph-mon[106470]: pgmap v2628: 99 pgs: 99 active+clean; 6.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 922 op/s 2024-09-14T04:50:01.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:01 smithi039 ceph-mon[111235]: pgmap v2628: 99 pgs: 99 active+clean; 6.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 922 op/s 2024-09-14T04:50:01.587 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:01.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:01.636 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:01.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:01.737 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:01.769 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:01.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:01.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:01 smithi129 ceph-mon[101316]: pgmap v2628: 99 pgs: 99 active+clean; 6.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 922 op/s 2024-09-14T04:50:01.871 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:01.910 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:01.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:01.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:02.012 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:02.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:02.055 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:02.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:02.156 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:02.187 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:02.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:02.188 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:02.289 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:02.324 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:02.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:02.325 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:02.426 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:02.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:02.470 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:02.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:02.570 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:02.614 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:02.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:02.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:02.716 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:02.743 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:02.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:02.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:02.776 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:50:02.845 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:02.892 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:02.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:02.894 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:02.922 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:50:02.923 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T04:50:02.994 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:03.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:03.065 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:03.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:03.168 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:03.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:03.209 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:03.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:03.310 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:03.346 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:03.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:03.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:03.448 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:03.483 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:03.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:03.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:03.585 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:03.623 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:03.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:03.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:03.703 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:03 smithi039 ceph-mon[106470]: pgmap v2629: 99 pgs: 99 active+clean; 6.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2024-09-14T04:50:03.703 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:03 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1466015815' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:50:03.704 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:03 smithi039 ceph-mon[111235]: pgmap v2629: 99 pgs: 99 active+clean; 6.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2024-09-14T04:50:03.704 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:03 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1466015815' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:50:03.724 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:03.738 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":2629,"stamp":"2024-09-14T04:50:01.757524+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6414854563,"num_objects":97900,"num_object_clones":0,"num_object_copies":293700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":97900,"num_whiteouts":0,"num_read":36,"num_read_kb":76,"num_write":150072,"num_write_kb":7931774,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1651,"num_bytes_recovered":106270496,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":130488,"ondisk_log_size":130488,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":22546636,"kb_used_data":18992844,"kb_used_omap":404,"kb_used_meta":3553131,"kb_avail":727414580,"statfs":{"total":767960285184,"available":744872529920,"internally_reserved":0,"allocated":19448672256,"data_stored":19410872563,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":414022,"internal_metadata":3638406842},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-638779392,"num_objects":-9747,"num_object_clones":0,"num_object_copies":-29241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-9747,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9747,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"11.308732"},"pg_stats":[{"pgid":"55.f","version":"1447'9340","reported_seq":9377,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:58.462513+0000","last_change":"2024-09-14T04:47:44.619393+0000","last_active":"2024-09-14T04:49:58.462513+0000","last_peered":"2024-09-14T04:49:58.462513+0000","last_clean":"2024-09-14T04:49:58.462513+0000","last_became_active":"2024-09-14T04:47:44.619068+0000","last_became_peered":"2024-09-14T04:47:44.619068+0000","last_unstale":"2024-09-14T04:49:58.462513+0000","last_undegraded":"2024-09-14T04:49:58.462513+0000","last_fullsized":"2024-09-14T04:49:58.462513+0000","mapping_epoch":1419,"log_start":"0'0","ondisk_log_start":"0'0","created":1419,"last_epoch_clean":1420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":9340,"log_dups_size":0,"ondisk_log_size":9340,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:24:11.584916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399769600,"num_objects":6100,"num_object_clones":0,"num_object_copies":18300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6100,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9340,"num_write_kb":494080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,7],"acting":[5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"55.e","version":"1447'9301","reported_seq":9368,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:58.461706+0000","last_change":"2024-09-14T04:49:29.627836+0000","last_active":"2024-09-14T04:49:58.461706+0000","last_peered":"2024-09-14T04:49:58.461706+0000","last_clean":"2024-09-14T04:49:58.461706+0000","last_became_active":"2024-09-14T04:49:29.616603+0000","last_became_peered":"2024-09-14T04:49:29.616603+0000","last_unstale":"2024-09-14T04:49:58.461706+0000","last_undegraded":"2024-09-14T04:49:58.461706+0000","last_fullsized":"2024-09-14T04:49:58.461706+0000","mapping_epoch":1442,"log_start":"1427'1400","ondisk_log_start":"1427'1400","created":1419,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":7901,"log_dups_size":0,"ondisk_log_size":7901,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:25:14.159610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393936896,"num_objects":6011,"num_object_clones":0,"num_object_copies":18033,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6011,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9301,"num_write_kb":489984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"55.d","version":"1447'9415","reported_seq":9453,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:58.271662+0000","last_change":"2024-09-14T04:47:44.732561+0000","last_active":"2024-09-14T04:49:58.271662+0000","last_peered":"2024-09-14T04:49:58.271662+0000","last_clean":"2024-09-14T04:49:58.271662+0000","last_became_active":"2024-09-14T04:47:44.732146+0000","last_became_peered":"2024-09-14T04:47:44.732146+0000","last_unstale":"2024-09-14T04:49:58.271662+0000","last_undegraded":"2024-09-14T04:49:58.271662+0000","last_fullsized":"2024-09-14T04:49:58.271662+0000","mapping_epoch":1419,"log_start":"1431'2700","ondisk_log_start":"1431'2700","created":1419,"last_epoch_clean":1420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":6715,"log_dups_size":0,"ondisk_log_size":6715,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:08:21.134275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":409468952,"num_objects":6249,"num_object_clones":0,"num_object_copies":18747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6249,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9415,"num_write_kb":501185,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"55.c","version":"1447'9291","reported_seq":9323,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:57.716261+0000","last_change":"2024-09-14T04:47:44.623726+0000","last_active":"2024-09-14T04:49:57.716261+0000","last_peered":"2024-09-14T04:49:57.716261+0000","last_clean":"2024-09-14T04:49:57.716261+0000","last_became_active":"2024-09-14T04:47:44.623263+0000","last_became_peered":"2024-09-14T04:47:44.623263+0000","last_unstale":"2024-09-14T04:49:57.716261+0000","last_undegraded":"2024-09-14T04:49:57.716261+0000","last_fullsized":"2024-09-14T04:49:57.716261+0000","mapping_epoch":1419,"log_start":"1427'1500","ondisk_log_start":"1427'1500","created":1419,"last_epoch_clean":1420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":7791,"log_dups_size":0,"ondisk_log_size":7791,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:34:24.614254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399441920,"num_objects":6095,"num_object_clones":0,"num_object_copies":18285,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6095,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9291,"num_write_kb":492352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"55.b","version":"1447'9328","reported_seq":9365,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:58.449301+0000","last_change":"2024-09-14T04:47:44.620800+0000","last_active":"2024-09-14T04:49:58.449301+0000","last_peered":"2024-09-14T04:49:58.449301+0000","last_clean":"2024-09-14T04:49:58.449301+0000","last_became_active":"2024-09-14T04:47:44.620531+0000","last_became_peered":"2024-09-14T04:47:44.620531+0000","last_unstale":"2024-09-14T04:49:58.449301+0000","last_undegraded":"2024-09-14T04:49:58.449301+0000","last_fullsized":"2024-09-14T04:49:58.449301+0000","mapping_epoch":1419,"log_start":"1427'1400","ondisk_log_start":"1427'1400","created":1419,"last_epoch_clean":1420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":7928,"log_dups_size":0,"ondisk_log_size":7928,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:11:20.790717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399638528,"num_objects":6098,"num_object_clones":0,"num_object_copies":18294,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6098,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9328,"num_write_kb":493632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"55.a","version":"1447'9309","reported_seq":9377,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:58.243420+0000","last_change":"2024-09-14T04:49:29.644757+0000","last_active":"2024-09-14T04:49:58.243420+0000","last_peered":"2024-09-14T04:49:58.243420+0000","last_clean":"2024-09-14T04:49:58.243420+0000","last_became_active":"2024-09-14T04:49:29.621926+0000","last_became_peered":"2024-09-14T04:49:29.621926+0000","last_unstale":"2024-09-14T04:49:58.243420+0000","last_undegraded":"2024-09-14T04:49:58.243420+0000","last_fullsized":"2024-09-14T04:49:58.243420+0000","mapping_epoch":1442,"log_start":"1431'2598","ondisk_log_start":"1431'2598","created":1419,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":6711,"log_dups_size":0,"ondisk_log_size":6711,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:20.701618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403767296,"num_objects":6161,"num_object_clones":0,"num_object_copies":18483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9309,"num_write_kb":495040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"55.9","version":"1447'9345","reported_seq":9411,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:58.462119+0000","last_change":"2024-09-14T04:49:29.640509+0000","last_active":"2024-09-14T04:49:58.462119+0000","last_peered":"2024-09-14T04:49:58.462119+0000","last_clean":"2024-09-14T04:49:58.462119+0000","last_became_active":"2024-09-14T04:49:29.621747+0000","last_became_peered":"2024-09-14T04:49:29.621747+0000","last_unstale":"2024-09-14T04:49:58.462119+0000","last_undegraded":"2024-09-14T04:49:58.462119+0000","last_fullsized":"2024-09-14T04:49:58.462119+0000","mapping_epoch":1442,"log_start":"1427'300","ondisk_log_start":"1427'300","created":1419,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":9045,"log_dups_size":0,"ondisk_log_size":9045,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:56:32.339858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":404029440,"num_objects":6165,"num_object_clones":0,"num_object_copies":18495,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6165,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9345,"num_write_kb":496320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"55.8","version":"1447'9385","reported_seq":9423,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:58.471473+0000","last_change":"2024-09-14T04:47:44.732510+0000","last_active":"2024-09-14T04:49:58.471473+0000","last_peered":"2024-09-14T04:49:58.471473+0000","last_clean":"2024-09-14T04:49:58.471473+0000","last_became_active":"2024-09-14T04:47:44.731755+0000","last_became_peered":"2024-09-14T04:47:44.731755+0000","last_unstale":"2024-09-14T04:49:58.471473+0000","last_undegraded":"2024-09-14T04:49:58.471473+0000","last_fullsized":"2024-09-14T04:49:58.471473+0000","mapping_epoch":1419,"log_start":"0'0","ondisk_log_start":"0'0","created":1419,"last_epoch_clean":1420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":9385,"log_dups_size":0,"ondisk_log_size":9385,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:51:34.295649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":402325504,"num_objects":6139,"num_object_clones":0,"num_object_copies":18417,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6139,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9385,"num_write_kb":496768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"55.7","version":"1447'9509","reported_seq":9546,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:58.313081+0000","last_change":"2024-09-14T04:47:44.620751+0000","last_active":"2024-09-14T04:49:58.313081+0000","last_peered":"2024-09-14T04:49:58.313081+0000","last_clean":"2024-09-14T04:49:58.313081+0000","last_became_active":"2024-09-14T04:47:44.620468+0000","last_became_peered":"2024-09-14T04:47:44.620468+0000","last_unstale":"2024-09-14T04:49:58.313081+0000","last_undegraded":"2024-09-14T04:49:58.313081+0000","last_fullsized":"2024-09-14T04:49:58.313081+0000","mapping_epoch":1419,"log_start":"1431'2800","ondisk_log_start":"1431'2800","created":1419,"last_epoch_clean":1420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":6709,"log_dups_size":0,"ondisk_log_size":6709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:14.021795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":405471232,"num_objects":6187,"num_object_clones":0,"num_object_copies":18561,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6187,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9509,"num_write_kb":502272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"55.6","version":"1447'9544","reported_seq":9610,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:50:00.123904+0000","last_change":"2024-09-14T04:47:44.620668+0000","last_active":"2024-09-14T04:50:00.123904+0000","last_peered":"2024-09-14T04:50:00.123904+0000","last_clean":"2024-09-14T04:50:00.123904+0000","last_became_active":"2024-09-14T04:47:44.620248+0000","last_became_peered":"2024-09-14T04:47:44.620248+0000","last_unstale":"2024-09-14T04:50:00.123904+0000","last_undegraded":"2024-09-14T04:50:00.123904+0000","last_fullsized":"2024-09-14T04:50:00.123904+0000","mapping_epoch":1419,"log_start":"0'0","ondisk_log_start":"0'0","created":1419,"last_epoch_clean":1420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":9544,"log_dups_size":0,"ondisk_log_size":9544,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:28:41.791546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":405405696,"num_objects":6186,"num_object_clones":0,"num_object_copies":18558,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6186,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9544,"num_write_kb":503360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"55.5","version":"1447'9167","reported_seq":9205,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:58.431896+0000","last_change":"2024-09-14T04:47:44.732608+0000","last_active":"2024-09-14T04:49:58.431896+0000","last_peered":"2024-09-14T04:49:58.431896+0000","last_clean":"2024-09-14T04:49:58.431896+0000","last_became_active":"2024-09-14T04:47:44.732055+0000","last_became_peered":"2024-09-14T04:47:44.732055+0000","last_unstale":"2024-09-14T04:49:58.431896+0000","last_undegraded":"2024-09-14T04:49:58.431896+0000","last_fullsized":"2024-09-14T04:49:58.431896+0000","mapping_epoch":1419,"log_start":"1427'1200","ondisk_log_start":"1427'1200","created":1419,"last_epoch_clean":1420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":7967,"log_dups_size":0,"ondisk_log_size":7967,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:27:27.167314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399310848,"num_objects":6093,"num_object_clones":0,"num_object_copies":18279,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6093,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9167,"num_write_kb":488320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"55.4","version":"1447'9166","reported_seq":9204,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:58.270974+0000","last_change":"2024-09-14T04:47:44.624288+0000","last_active":"2024-09-14T04:49:58.270974+0000","last_peered":"2024-09-14T04:49:58.270974+0000","last_clean":"2024-09-14T04:49:58.270974+0000","last_became_active":"2024-09-14T04:47:44.623792+0000","last_became_peered":"2024-09-14T04:47:44.623792+0000","last_unstale":"2024-09-14T04:49:58.270974+0000","last_undegraded":"2024-09-14T04:49:58.270974+0000","last_fullsized":"2024-09-14T04:49:58.270974+0000","mapping_epoch":1419,"log_start":"1431'2400","ondisk_log_start":"1431'2400","created":1419,"last_epoch_clean":1420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":6766,"log_dups_size":0,"ondisk_log_size":6766,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:02:35.092982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390725632,"num_objects":5962,"num_object_clones":0,"num_object_copies":17886,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5962,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9166,"num_write_kb":484096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"55.3","version":"1447'9795","reported_seq":9863,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:50:01.024606+0000","last_change":"2024-09-14T04:47:44.620809+0000","last_active":"2024-09-14T04:50:01.024606+0000","last_peered":"2024-09-14T04:50:01.024606+0000","last_clean":"2024-09-14T04:50:01.024606+0000","last_became_active":"2024-09-14T04:47:44.620523+0000","last_became_peered":"2024-09-14T04:47:44.620523+0000","last_unstale":"2024-09-14T04:50:01.024606+0000","last_undegraded":"2024-09-14T04:50:01.024606+0000","last_fullsized":"2024-09-14T04:50:01.024606+0000","mapping_epoch":1419,"log_start":"1430'1900","ondisk_log_start":"1430'1900","created":1419,"last_epoch_clean":1420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":7895,"log_dups_size":0,"ondisk_log_size":7895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:04:28.783772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":399835136,"num_objects":6101,"num_object_clones":0,"num_object_copies":18303,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6101,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9795,"num_write_kb":508672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"55.2","version":"1447'9392","reported_seq":9466,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:50:01.038605+0000","last_change":"2024-09-14T04:47:59.382691+0000","last_active":"2024-09-14T04:50:01.038605+0000","last_peered":"2024-09-14T04:50:01.038605+0000","last_clean":"2024-09-14T04:50:01.038605+0000","last_became_active":"2024-09-14T04:47:59.382325+0000","last_became_peered":"2024-09-14T04:47:59.382325+0000","last_unstale":"2024-09-14T04:50:01.038605+0000","last_undegraded":"2024-09-14T04:50:01.038605+0000","last_fullsized":"2024-09-14T04:50:01.038605+0000","mapping_epoch":1423,"log_start":"1427'1400","ondisk_log_start":"1427'1400","created":1419,"last_epoch_clean":1424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":7992,"log_dups_size":0,"ondisk_log_size":7992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:38:22.106396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":393871360,"num_objects":6010,"num_object_clones":0,"num_object_copies":18030,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6010,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9392,"num_write_kb":492864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"55.1","version":"1447'9356","reported_seq":12642,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:58.419068+0000","last_change":"2024-09-14T04:48:37.067853+0000","last_active":"2024-09-14T04:49:58.419068+0000","last_peered":"2024-09-14T04:49:58.419068+0000","last_clean":"2024-09-14T04:49:58.419068+0000","last_became_active":"2024-09-14T04:48:37.064586+0000","last_became_peered":"2024-09-14T04:48:37.064586+0000","last_unstale":"2024-09-14T04:49:58.419068+0000","last_undegraded":"2024-09-14T04:49:58.419068+0000","last_fullsized":"2024-09-14T04:49:58.419068+0000","mapping_epoch":1437,"log_start":"1426'200","ondisk_log_start":"1426'200","created":1419,"last_epoch_clean":1438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":9156,"log_dups_size":0,"ondisk_log_size":9156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:12:57.808206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":404750336,"num_objects":6176,"num_object_clones":0,"num_object_copies":18528,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6176,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9356,"num_write_kb":497024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1607,"num_bytes_recovered":105316352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"55.0","version":"1447'9345","reported_seq":9414,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:58.492260+0000","last_change":"2024-09-14T04:49:29.626446+0000","last_active":"2024-09-14T04:49:58.492260+0000","last_peered":"2024-09-14T04:49:58.492260+0000","last_clean":"2024-09-14T04:49:58.492260+0000","last_became_active":"2024-09-14T04:49:29.616617+0000","last_became_peered":"2024-09-14T04:49:29.616617+0000","last_unstale":"2024-09-14T04:49:58.492260+0000","last_undegraded":"2024-09-14T04:49:58.492260+0000","last_fullsized":"2024-09-14T04:49:58.492260+0000","mapping_epoch":1442,"log_start":"1427'1400","ondisk_log_start":"1427'1400","created":1419,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":7945,"log_dups_size":0,"ondisk_log_size":7945,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:38:53.404238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":402456576,"num_objects":6141,"num_object_clones":0,"num_object_copies":18423,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6141,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9345,"num_write_kb":495552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":2184,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.459654+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T04:49:50.459654+0000","last_peered":"2024-09-14T04:49:50.459654+0000","last_clean":"2024-09-14T04:49:50.459654+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T04:49:50.459654+0000","last_undegraded":"2024-09-14T04:49:50.459654+0000","last_fullsized":"2024-09-14T04:49:50.459654+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":57,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.428410+0000","last_change":"2024-09-14T04:44:57.325109+0000","last_active":"2024-09-14T04:49:49.428410+0000","last_peered":"2024-09-14T04:49:49.428410+0000","last_clean":"2024-09-14T04:49:49.428410+0000","last_became_active":"2024-09-14T04:44:57.324791+0000","last_became_peered":"2024-09-14T04:44:57.324791+0000","last_unstale":"2024-09-14T04:49:49.428410+0000","last_undegraded":"2024-09-14T04:49:49.428410+0000","last_fullsized":"2024-09-14T04:49:49.428410+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:03.593275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":26,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.432349+0000","last_change":"2024-09-14T04:49:29.628435+0000","last_active":"2024-09-14T04:49:49.432349+0000","last_peered":"2024-09-14T04:49:49.432349+0000","last_clean":"2024-09-14T04:49:49.432349+0000","last_became_active":"2024-09-14T04:49:29.628081+0000","last_became_peered":"2024-09-14T04:49:29.628081+0000","last_unstale":"2024-09-14T04:49:49.432349+0000","last_undegraded":"2024-09-14T04:49:49.432349+0000","last_fullsized":"2024-09-14T04:49:49.432349+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:01.520334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":805,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.449237+0000","last_change":"2024-09-14T04:43:13.355281+0000","last_active":"2024-09-14T04:49:50.449237+0000","last_peered":"2024-09-14T04:49:50.449237+0000","last_clean":"2024-09-14T04:49:50.449237+0000","last_became_active":"2024-09-14T04:43:13.354967+0000","last_became_peered":"2024-09-14T04:43:13.354967+0000","last_unstale":"2024-09-14T04:49:50.449237+0000","last_undegraded":"2024-09-14T04:49:50.449237+0000","last_fullsized":"2024-09-14T04:49:50.449237+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:23:19.332075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1435,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.430492+0000","last_change":"2024-09-14T04:44:57.342918+0000","last_active":"2024-09-14T04:49:49.430492+0000","last_peered":"2024-09-14T04:49:49.430492+0000","last_clean":"2024-09-14T04:49:49.430492+0000","last_became_active":"2024-09-14T04:44:57.329550+0000","last_became_peered":"2024-09-14T04:44:57.329550+0000","last_unstale":"2024-09-14T04:49:49.430492+0000","last_undegraded":"2024-09-14T04:49:49.430492+0000","last_fullsized":"2024-09-14T04:49:49.430492+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:52:52.435893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":57,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.452509+0000","last_change":"2024-09-14T04:44:57.325537+0000","last_active":"2024-09-14T04:49:50.452509+0000","last_peered":"2024-09-14T04:49:50.452509+0000","last_clean":"2024-09-14T04:49:50.452509+0000","last_became_active":"2024-09-14T04:44:57.325057+0000","last_became_peered":"2024-09-14T04:44:57.325057+0000","last_unstale":"2024-09-14T04:49:50.452509+0000","last_undegraded":"2024-09-14T04:49:50.452509+0000","last_fullsized":"2024-09-14T04:49:50.452509+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:43:23.166708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1450,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.430050+0000","last_change":"2024-09-14T04:44:57.337033+0000","last_active":"2024-09-14T04:49:49.430050+0000","last_peered":"2024-09-14T04:49:49.430050+0000","last_clean":"2024-09-14T04:49:49.430050+0000","last_became_active":"2024-09-14T04:44:57.328437+0000","last_became_peered":"2024-09-14T04:44:57.328437+0000","last_unstale":"2024-09-14T04:49:49.430050+0000","last_undegraded":"2024-09-14T04:49:49.430050+0000","last_fullsized":"2024-09-14T04:49:49.430050+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.109086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":57,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.452843+0000","last_change":"2024-09-14T04:44:57.332243+0000","last_active":"2024-09-14T04:49:50.452843+0000","last_peered":"2024-09-14T04:49:50.452843+0000","last_clean":"2024-09-14T04:49:50.452843+0000","last_became_active":"2024-09-14T04:44:57.330413+0000","last_became_peered":"2024-09-14T04:44:57.330413+0000","last_unstale":"2024-09-14T04:49:50.452843+0000","last_undegraded":"2024-09-14T04:49:50.452843+0000","last_fullsized":"2024-09-14T04:49:50.452843+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:25:01.338549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":40,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.457526+0000","last_change":"2024-09-14T04:48:29.862929+0000","last_active":"2024-09-14T04:49:50.457526+0000","last_peered":"2024-09-14T04:49:50.457526+0000","last_clean":"2024-09-14T04:49:50.457526+0000","last_became_active":"2024-09-14T04:48:29.862581+0000","last_became_peered":"2024-09-14T04:48:29.862581+0000","last_unstale":"2024-09-14T04:49:50.457526+0000","last_undegraded":"2024-09-14T04:49:50.457526+0000","last_fullsized":"2024-09-14T04:49:50.457526+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:16:43.060088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2180,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.450324+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T04:49:50.450324+0000","last_peered":"2024-09-14T04:49:50.450324+0000","last_clean":"2024-09-14T04:49:50.450324+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T04:49:50.450324+0000","last_undegraded":"2024-09-14T04:49:50.450324+0000","last_fullsized":"2024-09-14T04:49:50.450324+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":57,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.452440+0000","last_change":"2024-09-14T04:44:57.325392+0000","last_active":"2024-09-14T04:49:50.452440+0000","last_peered":"2024-09-14T04:49:50.452440+0000","last_clean":"2024-09-14T04:49:50.452440+0000","last_became_active":"2024-09-14T04:44:57.324624+0000","last_became_peered":"2024-09-14T04:44:57.324624+0000","last_unstale":"2024-09-14T04:49:50.452440+0000","last_undegraded":"2024-09-14T04:49:50.452440+0000","last_fullsized":"2024-09-14T04:49:50.452440+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:33:18.810199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2148,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.468527+0000","last_change":"2024-09-14T04:44:57.329113+0000","last_active":"2024-09-14T04:49:50.468527+0000","last_peered":"2024-09-14T04:49:50.468527+0000","last_clean":"2024-09-14T04:49:50.468527+0000","last_became_active":"2024-09-14T04:44:57.328727+0000","last_became_peered":"2024-09-14T04:44:57.328727+0000","last_unstale":"2024-09-14T04:49:50.468527+0000","last_undegraded":"2024-09-14T04:49:50.468527+0000","last_fullsized":"2024-09-14T04:49:50.468527+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:34.939874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":2165,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.468866+0000","last_change":"2024-09-14T04:44:57.329277+0000","last_active":"2024-09-14T04:49:50.468866+0000","last_peered":"2024-09-14T04:49:50.468866+0000","last_clean":"2024-09-14T04:49:50.468866+0000","last_became_active":"2024-09-14T04:44:57.328804+0000","last_became_peered":"2024-09-14T04:44:57.328804+0000","last_unstale":"2024-09-14T04:49:50.468866+0000","last_undegraded":"2024-09-14T04:49:50.468866+0000","last_fullsized":"2024-09-14T04:49:50.468866+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:42:46.055240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2178,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.459469+0000","last_change":"2024-09-14T04:07:58.561490+0000","last_active":"2024-09-14T04:49:50.459469+0000","last_peered":"2024-09-14T04:49:50.459469+0000","last_clean":"2024-09-14T04:49:50.459469+0000","last_became_active":"2024-09-14T04:07:58.559841+0000","last_became_peered":"2024-09-14T04:07:58.559841+0000","last_unstale":"2024-09-14T04:49:50.459469+0000","last_undegraded":"2024-09-14T04:49:50.459469+0000","last_fullsized":"2024-09-14T04:49:50.459469+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:27:03.795368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2161,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.457492+0000","last_change":"2024-09-14T04:43:13.357092+0000","last_active":"2024-09-14T04:49:50.457492+0000","last_peered":"2024-09-14T04:49:50.457492+0000","last_clean":"2024-09-14T04:49:50.457492+0000","last_became_active":"2024-09-14T04:43:13.356694+0000","last_became_peered":"2024-09-14T04:43:13.356694+0000","last_unstale":"2024-09-14T04:49:50.457492+0000","last_undegraded":"2024-09-14T04:49:50.457492+0000","last_fullsized":"2024-09-14T04:49:50.457492+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:11.167574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1428,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.432382+0000","last_change":"2024-09-14T04:07:21.720797+0000","last_active":"2024-09-14T04:49:49.432382+0000","last_peered":"2024-09-14T04:49:49.432382+0000","last_clean":"2024-09-14T04:49:49.432382+0000","last_became_active":"2024-09-14T04:07:21.719331+0000","last_became_peered":"2024-09-14T04:07:21.719331+0000","last_unstale":"2024-09-14T04:49:49.432382+0000","last_undegraded":"2024-09-14T04:49:49.432382+0000","last_fullsized":"2024-09-14T04:49:49.432382+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:35.404023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":75,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.430683+0000","last_change":"2024-09-14T04:43:11.641514+0000","last_active":"2024-09-14T04:49:49.430683+0000","last_peered":"2024-09-14T04:49:49.430683+0000","last_clean":"2024-09-14T04:49:49.430683+0000","last_became_active":"2024-09-14T04:43:11.641151+0000","last_became_peered":"2024-09-14T04:43:11.641151+0000","last_unstale":"2024-09-14T04:49:49.430683+0000","last_undegraded":"2024-09-14T04:49:49.430683+0000","last_fullsized":"2024-09-14T04:49:49.430683+0000","mapping_epoch":1384,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:06:45.045430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2173,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.459798+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T04:49:50.459798+0000","last_peered":"2024-09-14T04:49:50.459798+0000","last_clean":"2024-09-14T04:49:50.459798+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T04:49:50.459798+0000","last_undegraded":"2024-09-14T04:49:50.459798+0000","last_fullsized":"2024-09-14T04:49:50.459798+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":656,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.433902+0000","last_change":"2024-09-14T04:49:29.618477+0000","last_active":"2024-09-14T04:49:49.433902+0000","last_peered":"2024-09-14T04:49:49.433902+0000","last_clean":"2024-09-14T04:49:49.433902+0000","last_became_active":"2024-09-14T04:49:29.618210+0000","last_became_peered":"2024-09-14T04:49:29.618210+0000","last_unstale":"2024-09-14T04:49:49.433902+0000","last_undegraded":"2024-09-14T04:49:49.433902+0000","last_fullsized":"2024-09-14T04:49:49.433902+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:13:23.895512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1434,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.429782+0000","last_change":"2024-09-14T04:42:49.938505+0000","last_active":"2024-09-14T04:49:49.429782+0000","last_peered":"2024-09-14T04:49:49.429782+0000","last_clean":"2024-09-14T04:49:49.429782+0000","last_became_active":"2024-09-14T04:42:49.938180+0000","last_became_peered":"2024-09-14T04:42:49.938180+0000","last_unstale":"2024-09-14T04:49:49.429782+0000","last_undegraded":"2024-09-14T04:49:49.429782+0000","last_fullsized":"2024-09-14T04:49:49.429782+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:05.929479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1453,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.434817+0000","last_change":"2024-09-14T04:44:57.346582+0000","last_active":"2024-09-14T04:49:49.434817+0000","last_peered":"2024-09-14T04:49:49.434817+0000","last_clean":"2024-09-14T04:49:49.434817+0000","last_became_active":"2024-09-14T04:44:57.330446+0000","last_became_peered":"2024-09-14T04:44:57.330446+0000","last_unstale":"2024-09-14T04:49:49.434817+0000","last_undegraded":"2024-09-14T04:49:49.434817+0000","last_fullsized":"2024-09-14T04:49:49.434817+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:29.238585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":114,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.467866+0000","last_change":"2024-09-14T04:44:14.480667+0000","last_active":"2024-09-14T04:49:50.467866+0000","last_peered":"2024-09-14T04:49:50.467866+0000","last_clean":"2024-09-14T04:49:50.467866+0000","last_became_active":"2024-09-14T04:44:14.480365+0000","last_became_peered":"2024-09-14T04:44:14.480365+0000","last_unstale":"2024-09-14T04:49:50.467866+0000","last_undegraded":"2024-09-14T04:49:50.467866+0000","last_fullsized":"2024-09-14T04:49:50.467866+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:00.611771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":67,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.430850+0000","last_change":"2024-09-14T04:44:57.341426+0000","last_active":"2024-09-14T04:49:49.430850+0000","last_peered":"2024-09-14T04:49:49.430850+0000","last_clean":"2024-09-14T04:49:49.430850+0000","last_became_active":"2024-09-14T04:44:57.328763+0000","last_became_peered":"2024-09-14T04:44:57.328763+0000","last_unstale":"2024-09-14T04:49:49.430850+0000","last_undegraded":"2024-09-14T04:49:49.430850+0000","last_fullsized":"2024-09-14T04:49:49.430850+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:34:40.066761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":57,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.429822+0000","last_change":"2024-09-14T04:44:57.331337+0000","last_active":"2024-09-14T04:49:49.429822+0000","last_peered":"2024-09-14T04:49:49.429822+0000","last_clean":"2024-09-14T04:49:49.429822+0000","last_became_active":"2024-09-14T04:44:57.329336+0000","last_became_peered":"2024-09-14T04:44:57.329336+0000","last_unstale":"2024-09-14T04:49:49.429822+0000","last_undegraded":"2024-09-14T04:49:49.429822+0000","last_fullsized":"2024-09-14T04:49:49.429822+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:52:35.980869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"349'149","reported_seq":645,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.429767+0000","last_change":"2024-09-14T04:49:30.085274+0000","last_active":"2024-09-14T04:49:49.429767+0000","last_peered":"2024-09-14T04:49:49.429767+0000","last_clean":"2024-09-14T04:49:49.429767+0000","last_became_active":"2024-09-14T04:49:29.623295+0000","last_became_peered":"2024-09-14T04:49:29.623295+0000","last_unstale":"2024-09-14T04:49:49.429767+0000","last_undegraded":"2024-09-14T04:49:49.429767+0000","last_fullsized":"2024-09-14T04:49:49.429767+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:49:30.085089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:49:30.085089+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:10.012963+0000","scrub_duration":0.0071928790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":57,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.428662+0000","last_change":"2024-09-14T04:44:57.329870+0000","last_active":"2024-09-14T04:49:49.428662+0000","last_peered":"2024-09-14T04:49:49.428662+0000","last_clean":"2024-09-14T04:49:49.428662+0000","last_became_active":"2024-09-14T04:44:57.329353+0000","last_became_peered":"2024-09-14T04:44:57.329353+0000","last_unstale":"2024-09-14T04:49:49.428662+0000","last_undegraded":"2024-09-14T04:49:49.428662+0000","last_fullsized":"2024-09-14T04:49:49.428662+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:46.432181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1431,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.432462+0000","last_change":"2024-09-14T04:07:21.721242+0000","last_active":"2024-09-14T04:49:49.432462+0000","last_peered":"2024-09-14T04:49:49.432462+0000","last_clean":"2024-09-14T04:49:49.432462+0000","last_became_active":"2024-09-14T04:07:21.720084+0000","last_became_peered":"2024-09-14T04:07:21.720084+0000","last_unstale":"2024-09-14T04:49:49.432462+0000","last_undegraded":"2024-09-14T04:49:49.432462+0000","last_fullsized":"2024-09-14T04:49:49.432462+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:58:41.308643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":2211,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.459368+0000","last_change":"2024-09-14T04:44:57.344614+0000","last_active":"2024-09-14T04:49:50.459368+0000","last_peered":"2024-09-14T04:49:50.459368+0000","last_clean":"2024-09-14T04:49:50.459368+0000","last_became_active":"2024-09-14T04:44:57.332256+0000","last_became_peered":"2024-09-14T04:44:57.332256+0000","last_unstale":"2024-09-14T04:49:50.459368+0000","last_undegraded":"2024-09-14T04:49:50.459368+0000","last_fullsized":"2024-09-14T04:49:50.459368+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:23.438417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"349'149","reported_seq":815,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.431551+0000","last_change":"2024-09-14T04:45:00.331854+0000","last_active":"2024-09-14T04:49:49.431551+0000","last_peered":"2024-09-14T04:49:49.431551+0000","last_clean":"2024-09-14T04:49:49.431551+0000","last_became_active":"2024-09-14T04:45:00.331460+0000","last_became_peered":"2024-09-14T04:45:00.331460+0000","last_unstale":"2024-09-14T04:49:49.431551+0000","last_undegraded":"2024-09-14T04:49:49.431551+0000","last_fullsized":"2024-09-14T04:49:49.431551+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.567535+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.567535+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:45.095729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":646,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.433801+0000","last_change":"2024-09-14T04:49:29.616831+0000","last_active":"2024-09-14T04:49:49.433801+0000","last_peered":"2024-09-14T04:49:49.433801+0000","last_clean":"2024-09-14T04:49:49.433801+0000","last_became_active":"2024-09-14T04:49:29.616546+0000","last_became_peered":"2024-09-14T04:49:29.616546+0000","last_unstale":"2024-09-14T04:49:49.433801+0000","last_undegraded":"2024-09-14T04:49:49.433801+0000","last_fullsized":"2024-09-14T04:49:49.433801+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:27.882929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":166,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.467637+0000","last_change":"2024-09-14T04:44:57.330746+0000","last_active":"2024-09-14T04:49:50.467637+0000","last_peered":"2024-09-14T04:49:50.467637+0000","last_clean":"2024-09-14T04:49:50.467637+0000","last_became_active":"2024-09-14T04:44:57.330327+0000","last_became_peered":"2024-09-14T04:44:57.330327+0000","last_unstale":"2024-09-14T04:49:50.467637+0000","last_undegraded":"2024-09-14T04:49:50.467637+0000","last_fullsized":"2024-09-14T04:49:50.467637+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:53.122771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":97,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.434286+0000","last_change":"2024-09-14T04:43:13.355296+0000","last_active":"2024-09-14T04:49:49.434286+0000","last_peered":"2024-09-14T04:49:49.434286+0000","last_clean":"2024-09-14T04:49:49.434286+0000","last_became_active":"2024-09-14T04:43:13.354950+0000","last_became_peered":"2024-09-14T04:43:13.354950+0000","last_unstale":"2024-09-14T04:49:49.434286+0000","last_undegraded":"2024-09-14T04:49:49.434286+0000","last_fullsized":"2024-09-14T04:49:49.434286+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:20:08.968818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"349'149","reported_seq":753,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.432212+0000","last_change":"2024-09-14T04:41:39.471393+0000","last_active":"2024-09-14T04:49:49.432212+0000","last_peered":"2024-09-14T04:49:49.432212+0000","last_clean":"2024-09-14T04:49:49.432212+0000","last_became_active":"2024-09-14T04:41:36.931768+0000","last_became_peered":"2024-09-14T04:41:36.931768+0000","last_unstale":"2024-09-14T04:49:49.432212+0000","last_undegraded":"2024-09-14T04:49:49.432212+0000","last_fullsized":"2024-09-14T04:49:49.432212+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.471248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.471248+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:56.210375+0000","scrub_duration":0.021758344999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2167,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.460058+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T04:49:50.460058+0000","last_peered":"2024-09-14T04:49:50.460058+0000","last_clean":"2024-09-14T04:49:50.460058+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T04:49:50.460058+0000","last_undegraded":"2024-09-14T04:49:50.460058+0000","last_fullsized":"2024-09-14T04:49:50.460058+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1463,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.434914+0000","last_change":"2024-09-14T04:44:57.331436+0000","last_active":"2024-09-14T04:49:49.434914+0000","last_peered":"2024-09-14T04:49:49.434914+0000","last_clean":"2024-09-14T04:49:49.434914+0000","last_became_active":"2024-09-14T04:44:57.330957+0000","last_became_peered":"2024-09-14T04:44:57.330957+0000","last_unstale":"2024-09-14T04:49:49.434914+0000","last_undegraded":"2024-09-14T04:49:49.434914+0000","last_fullsized":"2024-09-14T04:49:49.434914+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:09.516688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":716,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.450285+0000","last_change":"2024-09-14T04:49:49.460401+0000","last_active":"2024-09-14T04:49:50.450285+0000","last_peered":"2024-09-14T04:49:50.450285+0000","last_clean":"2024-09-14T04:49:50.450285+0000","last_became_active":"2024-09-14T04:49:49.458530+0000","last_became_peered":"2024-09-14T04:49:49.458530+0000","last_unstale":"2024-09-14T04:49:50.450285+0000","last_undegraded":"2024-09-14T04:49:50.450285+0000","last_fullsized":"2024-09-14T04:49:50.450285+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:11:29.143239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"349'149","reported_seq":753,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.432293+0000","last_change":"2024-09-14T04:41:38.479980+0000","last_active":"2024-09-14T04:49:49.432293+0000","last_peered":"2024-09-14T04:49:49.432293+0000","last_clean":"2024-09-14T04:49:49.432293+0000","last_became_active":"2024-09-14T04:41:36.933392+0000","last_became_peered":"2024-09-14T04:41:36.933392+0000","last_unstale":"2024-09-14T04:49:49.432293+0000","last_undegraded":"2024-09-14T04:49:49.432293+0000","last_fullsized":"2024-09-14T04:49:49.432293+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:38.479887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:38.479887+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:01:26.102279+0000","scrub_duration":0.0062728319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":121,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.467788+0000","last_change":"2024-09-14T04:44:57.329983+0000","last_active":"2024-09-14T04:49:50.467788+0000","last_peered":"2024-09-14T04:49:50.467788+0000","last_clean":"2024-09-14T04:49:50.467788+0000","last_became_active":"2024-09-14T04:44:57.329597+0000","last_became_peered":"2024-09-14T04:44:57.329597+0000","last_unstale":"2024-09-14T04:49:50.467788+0000","last_undegraded":"2024-09-14T04:49:50.467788+0000","last_fullsized":"2024-09-14T04:49:50.467788+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:45.718039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":660,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.459145+0000","last_change":"2024-09-14T04:49:49.443631+0000","last_active":"2024-09-14T04:49:50.459145+0000","last_peered":"2024-09-14T04:49:50.459145+0000","last_clean":"2024-09-14T04:49:50.459145+0000","last_became_active":"2024-09-14T04:49:49.443313+0000","last_became_peered":"2024-09-14T04:49:49.443313+0000","last_unstale":"2024-09-14T04:49:50.459145+0000","last_undegraded":"2024-09-14T04:49:50.459145+0000","last_fullsized":"2024-09-14T04:49:50.459145+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:06:54.562292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1436,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.430636+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T04:49:49.430636+0000","last_peered":"2024-09-14T04:49:49.430636+0000","last_clean":"2024-09-14T04:49:49.430636+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T04:49:49.430636+0000","last_undegraded":"2024-09-14T04:49:49.430636+0000","last_fullsized":"2024-09-14T04:49:49.430636+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"349'149","reported_seq":746,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.455343+0000","last_change":"2024-09-14T04:41:39.012977+0000","last_active":"2024-09-14T04:49:50.455343+0000","last_peered":"2024-09-14T04:49:50.455343+0000","last_clean":"2024-09-14T04:49:50.455343+0000","last_became_active":"2024-09-14T04:41:36.932552+0000","last_became_peered":"2024-09-14T04:41:36.932552+0000","last_unstale":"2024-09-14T04:49:50.455343+0000","last_undegraded":"2024-09-14T04:49:50.455343+0000","last_fullsized":"2024-09-14T04:49:50.455343+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.012879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.012879+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:16:40.080262+0000","scrub_duration":0.0051721620000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":752,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.450400+0000","last_change":"2024-09-14T04:49:49.459590+0000","last_active":"2024-09-14T04:49:50.450400+0000","last_peered":"2024-09-14T04:49:50.450400+0000","last_clean":"2024-09-14T04:49:50.450400+0000","last_became_active":"2024-09-14T04:49:49.456263+0000","last_became_peered":"2024-09-14T04:49:49.456263+0000","last_unstale":"2024-09-14T04:49:50.450400+0000","last_undegraded":"2024-09-14T04:49:50.450400+0000","last_fullsized":"2024-09-14T04:49:50.450400+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:20:44.666805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":57,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.429746+0000","last_change":"2024-09-14T04:44:57.329652+0000","last_active":"2024-09-14T04:49:49.429746+0000","last_peered":"2024-09-14T04:49:49.429746+0000","last_clean":"2024-09-14T04:49:49.429746+0000","last_became_active":"2024-09-14T04:44:57.328888+0000","last_became_peered":"2024-09-14T04:44:57.328888+0000","last_unstale":"2024-09-14T04:49:49.429746+0000","last_undegraded":"2024-09-14T04:49:49.429746+0000","last_fullsized":"2024-09-14T04:49:49.429746+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:10.170068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1434,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.430283+0000","last_change":"2024-09-14T04:06:46.875340+0000","last_active":"2024-09-14T04:49:49.430283+0000","last_peered":"2024-09-14T04:49:49.430283+0000","last_clean":"2024-09-14T04:49:49.430283+0000","last_became_active":"2024-09-14T04:06:46.874552+0000","last_became_peered":"2024-09-14T04:06:46.874552+0000","last_unstale":"2024-09-14T04:49:49.430283+0000","last_undegraded":"2024-09-14T04:49:49.430283+0000","last_fullsized":"2024-09-14T04:49:49.430283+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:52.257357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"349'149","reported_seq":1816,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.465057+0000","last_change":"2024-09-14T04:45:00.346627+0000","last_active":"2024-09-14T04:49:50.465057+0000","last_peered":"2024-09-14T04:49:50.465057+0000","last_clean":"2024-09-14T04:49:50.465057+0000","last_became_active":"2024-09-14T04:45:00.346286+0000","last_became_peered":"2024-09-14T04:45:00.346286+0000","last_unstale":"2024-09-14T04:49:50.465057+0000","last_undegraded":"2024-09-14T04:49:50.465057+0000","last_fullsized":"2024-09-14T04:49:50.465057+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:50.784648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:50.784648+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:07:58.441372+0000","scrub_duration":0.01141873,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":114,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.468779+0000","last_change":"2024-09-14T04:44:14.489235+0000","last_active":"2024-09-14T04:49:50.468779+0000","last_peered":"2024-09-14T04:49:50.468779+0000","last_clean":"2024-09-14T04:49:50.468779+0000","last_became_active":"2024-09-14T04:44:14.488950+0000","last_became_peered":"2024-09-14T04:44:14.488950+0000","last_unstale":"2024-09-14T04:49:50.468779+0000","last_undegraded":"2024-09-14T04:49:50.468779+0000","last_fullsized":"2024-09-14T04:49:50.468779+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:54.441144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":670,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.449166+0000","last_change":"2024-09-14T04:49:49.456317+0000","last_active":"2024-09-14T04:49:50.449166+0000","last_peered":"2024-09-14T04:49:50.449166+0000","last_clean":"2024-09-14T04:49:50.449166+0000","last_became_active":"2024-09-14T04:49:49.456029+0000","last_became_peered":"2024-09-14T04:49:49.456029+0000","last_unstale":"2024-09-14T04:49:50.449166+0000","last_undegraded":"2024-09-14T04:49:50.449166+0000","last_fullsized":"2024-09-14T04:49:50.449166+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:46:05.319676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2187,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.449430+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T04:49:50.449430+0000","last_peered":"2024-09-14T04:49:50.449430+0000","last_clean":"2024-09-14T04:49:50.449430+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T04:49:50.449430+0000","last_undegraded":"2024-09-14T04:49:50.449430+0000","last_fullsized":"2024-09-14T04:49:50.449430+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"349'149","reported_seq":628,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.450606+0000","last_change":"2024-09-14T04:49:49.448561+0000","last_active":"2024-09-14T04:49:49.450606+0000","last_peered":"2024-09-14T04:49:49.450606+0000","last_clean":"2024-09-14T04:49:49.450606+0000","last_became_active":"2024-09-14T04:49:49.448278+0000","last_became_peered":"2024-09-14T04:49:49.448278+0000","last_unstale":"2024-09-14T04:49:49.450606+0000","last_undegraded":"2024-09-14T04:49:49.450606+0000","last_fullsized":"2024-09-14T04:49:49.450606+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.326129+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.326129+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":35,"num_read_kb":75,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2188,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.459558+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T04:49:50.459558+0000","last_peered":"2024-09-14T04:49:50.459558+0000","last_clean":"2024-09-14T04:49:50.459558+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T04:49:50.459558+0000","last_undegraded":"2024-09-14T04:49:50.459558+0000","last_fullsized":"2024-09-14T04:49:50.459558+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2195,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.461192+0000","last_change":"2024-09-14T04:44:57.335783+0000","last_active":"2024-09-14T04:49:50.461192+0000","last_peered":"2024-09-14T04:49:50.461192+0000","last_clean":"2024-09-14T04:49:50.461192+0000","last_became_active":"2024-09-14T04:44:57.332780+0000","last_became_peered":"2024-09-14T04:44:57.332780+0000","last_unstale":"2024-09-14T04:49:50.461192+0000","last_undegraded":"2024-09-14T04:49:50.461192+0000","last_fullsized":"2024-09-14T04:49:50.461192+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:57.457593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1465,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.458941+0000","last_change":"2024-09-14T04:42:49.939790+0000","last_active":"2024-09-14T04:49:50.458941+0000","last_peered":"2024-09-14T04:49:50.458941+0000","last_clean":"2024-09-14T04:49:50.458941+0000","last_became_active":"2024-09-14T04:42:49.939591+0000","last_became_peered":"2024-09-14T04:42:49.939591+0000","last_unstale":"2024-09-14T04:49:50.458941+0000","last_undegraded":"2024-09-14T04:49:50.458941+0000","last_fullsized":"2024-09-14T04:49:50.458941+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:23:16.790593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1428,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.437222+0000","last_change":"2024-09-14T04:44:57.326008+0000","last_active":"2024-09-14T04:49:49.437222+0000","last_peered":"2024-09-14T04:49:49.437222+0000","last_clean":"2024-09-14T04:49:49.437222+0000","last_became_active":"2024-09-14T04:44:57.325617+0000","last_became_peered":"2024-09-14T04:44:57.325617+0000","last_unstale":"2024-09-14T04:49:49.437222+0000","last_undegraded":"2024-09-14T04:49:49.437222+0000","last_fullsized":"2024-09-14T04:49:49.437222+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:15.637508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":717,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.430362+0000","last_change":"2024-09-14T04:43:13.354846+0000","last_active":"2024-09-14T04:49:49.430362+0000","last_peered":"2024-09-14T04:49:49.430362+0000","last_clean":"2024-09-14T04:49:49.430362+0000","last_became_active":"2024-09-14T04:43:13.354515+0000","last_became_peered":"2024-09-14T04:43:13.354515+0000","last_unstale":"2024-09-14T04:49:49.430362+0000","last_undegraded":"2024-09-14T04:49:49.430362+0000","last_fullsized":"2024-09-14T04:49:49.430362+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:17.889341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1433,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.430589+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T04:49:49.430589+0000","last_peered":"2024-09-14T04:49:49.430589+0000","last_clean":"2024-09-14T04:49:49.430589+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T04:49:49.430589+0000","last_undegraded":"2024-09-14T04:49:49.430589+0000","last_fullsized":"2024-09-14T04:49:49.430589+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":1771,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.433729+0000","last_change":"2024-09-14T04:41:30.640651+0000","last_active":"2024-09-14T04:49:49.433729+0000","last_peered":"2024-09-14T04:49:49.433729+0000","last_clean":"2024-09-14T04:49:49.433729+0000","last_became_active":"2024-09-14T04:41:29.532466+0000","last_became_peered":"2024-09-14T04:41:29.532466+0000","last_unstale":"2024-09-14T04:49:49.433729+0000","last_undegraded":"2024-09-14T04:49:49.433729+0000","last_fullsized":"2024-09-14T04:49:49.433729+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":0,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:30.640520+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:30.640520+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:27.776724+0000","scrub_duration":0.0028653139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"349'149","reported_seq":745,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.430202+0000","last_change":"2024-09-14T04:41:39.978080+0000","last_active":"2024-09-14T04:49:49.430202+0000","last_peered":"2024-09-14T04:49:49.430202+0000","last_clean":"2024-09-14T04:49:49.430202+0000","last_became_active":"2024-09-14T04:41:36.929279+0000","last_became_peered":"2024-09-14T04:41:36.929279+0000","last_unstale":"2024-09-14T04:49:49.430202+0000","last_undegraded":"2024-09-14T04:49:49.430202+0000","last_fullsized":"2024-09-14T04:49:49.430202+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.977999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.977999+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:44:06.383827+0000","scrub_duration":0.00192122,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":48,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.449539+0000","last_change":"2024-09-14T04:48:11.040133+0000","last_active":"2024-09-14T04:49:50.449539+0000","last_peered":"2024-09-14T04:49:50.449539+0000","last_clean":"2024-09-14T04:49:50.449539+0000","last_became_active":"2024-09-14T04:48:11.039787+0000","last_became_peered":"2024-09-14T04:48:11.039787+0000","last_unstale":"2024-09-14T04:49:50.449539+0000","last_undegraded":"2024-09-14T04:49:50.449539+0000","last_fullsized":"2024-09-14T04:49:50.449539+0000","mapping_epoch":1428,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:53.520672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2035,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.467084+0000","last_change":"2024-09-14T04:49:29.616443+0000","last_active":"2024-09-14T04:49:50.467084+0000","last_peered":"2024-09-14T04:49:50.467084+0000","last_clean":"2024-09-14T04:49:50.467084+0000","last_became_active":"2024-09-14T04:49:29.616066+0000","last_became_peered":"2024-09-14T04:49:29.616066+0000","last_unstale":"2024-09-14T04:49:50.467084+0000","last_undegraded":"2024-09-14T04:49:50.467084+0000","last_fullsized":"2024-09-14T04:49:50.467084+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:09:15.897957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":57,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.431728+0000","last_change":"2024-09-14T04:44:57.326724+0000","last_active":"2024-09-14T04:49:49.431728+0000","last_peered":"2024-09-14T04:49:49.431728+0000","last_clean":"2024-09-14T04:49:49.431728+0000","last_became_active":"2024-09-14T04:44:57.326191+0000","last_became_peered":"2024-09-14T04:44:57.326191+0000","last_unstale":"2024-09-14T04:49:49.431728+0000","last_undegraded":"2024-09-14T04:49:49.431728+0000","last_fullsized":"2024-09-14T04:49:49.431728+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:46.060269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":57,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.428547+0000","last_change":"2024-09-14T04:44:57.328528+0000","last_active":"2024-09-14T04:49:49.428547+0000","last_peered":"2024-09-14T04:49:49.428547+0000","last_clean":"2024-09-14T04:49:49.428547+0000","last_became_active":"2024-09-14T04:44:57.327739+0000","last_became_peered":"2024-09-14T04:44:57.327739+0000","last_unstale":"2024-09-14T04:49:49.428547+0000","last_undegraded":"2024-09-14T04:49:49.428547+0000","last_fullsized":"2024-09-14T04:49:49.428547+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:04:58.294690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"349'149","reported_seq":761,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.430546+0000","last_change":"2024-09-14T04:48:32.059177+0000","last_active":"2024-09-14T04:49:49.430546+0000","last_peered":"2024-09-14T04:49:49.430546+0000","last_clean":"2024-09-14T04:49:49.430546+0000","last_became_active":"2024-09-14T04:48:32.058741+0000","last_became_peered":"2024-09-14T04:48:32.058741+0000","last_unstale":"2024-09-14T04:49:49.430546+0000","last_undegraded":"2024-09-14T04:49:49.430546+0000","last_fullsized":"2024-09-14T04:49:49.430546+0000","mapping_epoch":1435,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1436,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:56.738636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:56.738636+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:20.932276+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":57,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.431112+0000","last_change":"2024-09-14T04:44:57.323436+0000","last_active":"2024-09-14T04:49:49.431112+0000","last_peered":"2024-09-14T04:49:49.431112+0000","last_clean":"2024-09-14T04:49:49.431112+0000","last_became_active":"2024-09-14T04:44:57.323022+0000","last_became_peered":"2024-09-14T04:44:57.323022+0000","last_unstale":"2024-09-14T04:49:49.431112+0000","last_undegraded":"2024-09-14T04:49:49.431112+0000","last_fullsized":"2024-09-14T04:49:49.431112+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:22.775261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1463,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.432229+0000","last_change":"2024-09-14T04:44:57.322357+0000","last_active":"2024-09-14T04:49:49.432229+0000","last_peered":"2024-09-14T04:49:49.432229+0000","last_clean":"2024-09-14T04:49:49.432229+0000","last_became_active":"2024-09-14T04:44:57.321990+0000","last_became_peered":"2024-09-14T04:44:57.321990+0000","last_unstale":"2024-09-14T04:49:49.432229+0000","last_undegraded":"2024-09-14T04:49:49.432229+0000","last_fullsized":"2024-09-14T04:49:49.432229+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:09.658008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"349'149","reported_seq":1787,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.433559+0000","last_change":"2024-09-14T04:41:53.858076+0000","last_active":"2024-09-14T04:49:49.433559+0000","last_peered":"2024-09-14T04:49:49.433559+0000","last_clean":"2024-09-14T04:49:49.433559+0000","last_became_active":"2024-09-14T04:41:36.931585+0000","last_became_peered":"2024-09-14T04:41:36.931585+0000","last_unstale":"2024-09-14T04:49:49.433559+0000","last_undegraded":"2024-09-14T04:49:49.433559+0000","last_fullsized":"2024-09-14T04:49:49.433559+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:53.857924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:53.857924+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:10:20.658986+0000","scrub_duration":0.0022355919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":57,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.430760+0000","last_change":"2024-09-14T04:44:57.329198+0000","last_active":"2024-09-14T04:49:49.430760+0000","last_peered":"2024-09-14T04:49:49.430760+0000","last_clean":"2024-09-14T04:49:49.430760+0000","last_became_active":"2024-09-14T04:44:57.327754+0000","last_became_peered":"2024-09-14T04:44:57.327754+0000","last_unstale":"2024-09-14T04:49:49.430760+0000","last_undegraded":"2024-09-14T04:49:49.430760+0000","last_fullsized":"2024-09-14T04:49:49.430760+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:10:27.944476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":172,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.460184+0000","last_change":"2024-09-14T04:44:57.348687+0000","last_active":"2024-09-14T04:49:50.460184+0000","last_peered":"2024-09-14T04:49:50.460184+0000","last_clean":"2024-09-14T04:49:50.460184+0000","last_became_active":"2024-09-14T04:44:57.332845+0000","last_became_peered":"2024-09-14T04:44:57.332845+0000","last_unstale":"2024-09-14T04:49:50.460184+0000","last_undegraded":"2024-09-14T04:49:50.460184+0000","last_fullsized":"2024-09-14T04:49:50.460184+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:12:18.762737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2194,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.457650+0000","last_change":"2024-09-14T04:44:57.333468+0000","last_active":"2024-09-14T04:49:50.457650+0000","last_peered":"2024-09-14T04:49:50.457650+0000","last_clean":"2024-09-14T04:49:50.457650+0000","last_became_active":"2024-09-14T04:44:57.332626+0000","last_became_peered":"2024-09-14T04:44:57.332626+0000","last_unstale":"2024-09-14T04:49:50.457650+0000","last_undegraded":"2024-09-14T04:49:50.457650+0000","last_fullsized":"2024-09-14T04:49:50.457650+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:57:26.366438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1419,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.430081+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T04:49:49.430081+0000","last_peered":"2024-09-14T04:49:49.430081+0000","last_clean":"2024-09-14T04:49:49.430081+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T04:49:49.430081+0000","last_undegraded":"2024-09-14T04:49:49.430081+0000","last_fullsized":"2024-09-14T04:49:49.430081+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1418,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.432095+0000","last_change":"2024-09-14T04:06:46.869868+0000","last_active":"2024-09-14T04:49:49.432095+0000","last_peered":"2024-09-14T04:49:49.432095+0000","last_clean":"2024-09-14T04:49:49.432095+0000","last_became_active":"2024-09-14T04:06:46.869126+0000","last_became_peered":"2024-09-14T04:06:46.869126+0000","last_unstale":"2024-09-14T04:49:49.432095+0000","last_undegraded":"2024-09-14T04:49:49.432095+0000","last_fullsized":"2024-09-14T04:49:49.432095+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:41.150138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1402,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.434987+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T04:49:49.434987+0000","last_peered":"2024-09-14T04:49:49.434987+0000","last_clean":"2024-09-14T04:49:49.434987+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T04:49:49.434987+0000","last_undegraded":"2024-09-14T04:49:49.434987+0000","last_fullsized":"2024-09-14T04:49:49.434987+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":57,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.428771+0000","last_change":"2024-09-14T04:44:57.332112+0000","last_active":"2024-09-14T04:49:49.428771+0000","last_peered":"2024-09-14T04:49:49.428771+0000","last_clean":"2024-09-14T04:49:49.428771+0000","last_became_active":"2024-09-14T04:44:57.331575+0000","last_became_peered":"2024-09-14T04:44:57.331575+0000","last_unstale":"2024-09-14T04:49:49.428771+0000","last_undegraded":"2024-09-14T04:49:49.428771+0000","last_fullsized":"2024-09-14T04:49:49.428771+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:50:00.391283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1450,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.435411+0000","last_change":"2024-09-14T04:44:57.351460+0000","last_active":"2024-09-14T04:49:49.435411+0000","last_peered":"2024-09-14T04:49:49.435411+0000","last_clean":"2024-09-14T04:49:49.435411+0000","last_became_active":"2024-09-14T04:44:57.334662+0000","last_became_peered":"2024-09-14T04:44:57.334662+0000","last_unstale":"2024-09-14T04:49:49.435411+0000","last_undegraded":"2024-09-14T04:49:49.435411+0000","last_fullsized":"2024-09-14T04:49:49.435411+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:02.399062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1404,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.430783+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T04:49:49.430783+0000","last_peered":"2024-09-14T04:49:49.430783+0000","last_clean":"2024-09-14T04:49:49.430783+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T04:49:49.430783+0000","last_undegraded":"2024-09-14T04:49:49.430783+0000","last_fullsized":"2024-09-14T04:49:49.430783+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1462,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.434200+0000","last_change":"2024-09-14T04:44:57.349690+0000","last_active":"2024-09-14T04:49:49.434200+0000","last_peered":"2024-09-14T04:49:49.434200+0000","last_clean":"2024-09-14T04:49:49.434200+0000","last_became_active":"2024-09-14T04:44:57.333579+0000","last_became_peered":"2024-09-14T04:44:57.333579+0000","last_unstale":"2024-09-14T04:49:49.434200+0000","last_undegraded":"2024-09-14T04:49:49.434200+0000","last_fullsized":"2024-09-14T04:49:49.434200+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:17:55.265141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1501,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.430357+0000","last_change":"2024-09-14T04:48:29.893179+0000","last_active":"2024-09-14T04:49:49.430357+0000","last_peered":"2024-09-14T04:49:49.430357+0000","last_clean":"2024-09-14T04:49:49.430357+0000","last_became_active":"2024-09-14T04:48:29.864724+0000","last_became_peered":"2024-09-14T04:48:29.864724+0000","last_unstale":"2024-09-14T04:49:49.430357+0000","last_undegraded":"2024-09-14T04:49:49.430357+0000","last_fullsized":"2024-09-14T04:49:49.430357+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:19:15.120740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":655,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.434178+0000","last_change":"2024-09-14T04:49:29.616761+0000","last_active":"2024-09-14T04:49:49.434178+0000","last_peered":"2024-09-14T04:49:49.434178+0000","last_clean":"2024-09-14T04:49:49.434178+0000","last_became_active":"2024-09-14T04:49:29.616072+0000","last_became_peered":"2024-09-14T04:49:29.616072+0000","last_unstale":"2024-09-14T04:49:49.434178+0000","last_undegraded":"2024-09-14T04:49:49.434178+0000","last_fullsized":"2024-09-14T04:49:49.434178+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:49:52.358949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1424,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.434048+0000","last_change":"2024-09-14T04:44:57.332511+0000","last_active":"2024-09-14T04:49:49.434048+0000","last_peered":"2024-09-14T04:49:49.434048+0000","last_clean":"2024-09-14T04:49:49.434048+0000","last_became_active":"2024-09-14T04:44:57.332166+0000","last_became_peered":"2024-09-14T04:44:57.332166+0000","last_unstale":"2024-09-14T04:49:49.434048+0000","last_undegraded":"2024-09-14T04:49:49.434048+0000","last_fullsized":"2024-09-14T04:49:49.434048+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:53.984639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2174,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.459281+0000","last_change":"2024-09-14T04:07:58.560880+0000","last_active":"2024-09-14T04:49:50.459281+0000","last_peered":"2024-09-14T04:49:50.459281+0000","last_clean":"2024-09-14T04:49:50.459281+0000","last_became_active":"2024-09-14T04:07:58.560132+0000","last_became_peered":"2024-09-14T04:07:58.560132+0000","last_unstale":"2024-09-14T04:49:50.459281+0000","last_undegraded":"2024-09-14T04:49:50.459281+0000","last_fullsized":"2024-09-14T04:49:50.459281+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:46:00.287050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2173,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.457694+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T04:49:50.457694+0000","last_peered":"2024-09-14T04:49:50.457694+0000","last_clean":"2024-09-14T04:49:50.457694+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T04:49:50.457694+0000","last_undegraded":"2024-09-14T04:49:50.457694+0000","last_fullsized":"2024-09-14T04:49:50.457694+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1416,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.430565+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T04:49:49.430565+0000","last_peered":"2024-09-14T04:49:49.430565+0000","last_clean":"2024-09-14T04:49:49.430565+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T04:49:49.430565+0000","last_undegraded":"2024-09-14T04:49:49.430565+0000","last_fullsized":"2024-09-14T04:49:49.430565+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1453,"reported_epoch":1446,"state":"active+clean","last_fresh":"2024-09-14T04:49:49.435643+0000","last_change":"2024-09-14T04:44:57.339993+0000","last_active":"2024-09-14T04:49:49.435643+0000","last_peered":"2024-09-14T04:49:49.435643+0000","last_clean":"2024-09-14T04:49:49.435643+0000","last_became_active":"2024-09-14T04:44:57.330472+0000","last_became_peered":"2024-09-14T04:44:57.330472+0000","last_unstale":"2024-09-14T04:49:49.435643+0000","last_undegraded":"2024-09-14T04:49:49.435643+0000","last_fullsized":"2024-09-14T04:49:49.435643+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:19:27.046988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2001,"reported_epoch":1447,"state":"active+clean","last_fresh":"2024-09-14T04:49:50.458782+0000","last_change":"2024-09-14T04:49:49.449687+0000","last_active":"2024-09-14T04:49:50.458782+0000","last_peered":"2024-09-14T04:49:50.458782+0000","last_clean":"2024-09-14T04:49:50.458782+0000","last_became_active":"2024-09-14T04:49:49.449144+0000","last_became_peered":"2024-09-14T04:49:49.449144+0000","last_unstale":"2024-09-14T04:49:50.458782+0000","last_undegraded":"2024-09-14T04:49:50.458782+0000","last_fullsized":"2024-09-14T04:49:50.458782+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:59:06.779730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":55,"num_pg":16,"stat_sum":{"num_bytes":6414204952,"num_objects":97874,"num_object_clones":0,"num_object_copies":293622,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":97874,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":149988,"num_write_kb":7931521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1607,"num_bytes_recovered":105316352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":19249377280,"data_stored":19249365064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128790,"ondisk_log_size":128790,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":644038,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":35,"num_read_kb":75,"num_write":20,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":949074,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":5070,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4250,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201988,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4876000,"kb_used_data":4379800,"kb_used_omap":1,"kb_used_meta":496190,"kb_avail":88869152,"statfs":{"total":95995035648,"available":91002011648,"internally_reserved":0,"allocated":4484915200,"data_stored":4480177922,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":508099019},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463624,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2901460,"kb_used_data":2365004,"kb_used_omap":1,"kb_used_meta":536382,"kb_avail":90843692,"statfs":{"total":95995035648,"available":93023940608,"internally_reserved":0,"allocated":2421764096,"data_stored":2417038955,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":549255623},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725261,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2541328,"kb_used_data":1992648,"kb_used_omap":1,"kb_used_meta":548670,"kb_avail":91203824,"statfs":{"total":95995035648,"available":93392715776,"internally_reserved":0,"allocated":2040471552,"data_stored":2035746877,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":561838534},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":1392,"seq":5978594476107,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1352800,"kb_used_data":1203544,"kb_used_omap":117,"kb_used_meta":149130,"kb_avail":92392352,"statfs":{"total":95995035648,"available":94609768448,"internally_reserved":0,"allocated":1232429056,"data_stored":1227692330,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120638,"internal_metadata":152709314},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772075002,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3362252,"kb_used_data":2780420,"kb_used_omap":71,"kb_used_meta":581752,"kb_avail":90382900,"statfs":{"total":95995035648,"available":92552089600,"internally_reserved":0,"allocated":2847150080,"data_stored":2842429554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":595714998},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707303938,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3058636,"kb_used_data":2723844,"kb_used_omap":68,"kb_used_meta":334651,"kb_avail":90686516,"statfs":{"total":95995035648,"available":92862992384,"internally_reserved":0,"allocated":2789216256,"data_stored":2784495634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":342682832},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872729097,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2402372,"kb_used_data":1951036,"kb_used_omap":67,"kb_used_meta":451196,"kb_avail":91342780,"statfs":{"total":95995035648,"available":93535006720,"internally_reserved":0,"allocated":1997860864,"data_stored":1993147293,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":462025061},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056147,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2051788,"kb_used_data":1596548,"kb_used_omap":74,"kb_used_meta":455157,"kb_avail":91693364,"statfs":{"total":95995035648,"available":93894004736,"internally_reserved":0,"allocated":1634865152,"data_stored":1630143998,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":466081461},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1901,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":55,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1615335424,"data_stored":1615331352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":55,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1968766976,"data_stored":1968766976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":55,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2776170496,"data_stored":2776170496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":55,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2812674048,"data_stored":2812674048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":55,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1217990656,"data_stored":1217986584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":55,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2026438656,"data_stored":2026438656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":55,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2397241344,"data_stored":2397241344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":55,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4434759680,"data_stored":4434755608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:50:03.743 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd pool set unique_pool_2 pgp_num 16 2024-09-14T04:50:03.754 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:03.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:03.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:03.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:03 smithi129 ceph-mon[101316]: pgmap v2629: 99 pgs: 99 active+clean; 6.0 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2024-09-14T04:50:03.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:03 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1466015815' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:50:03.856 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:03.894 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:03.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:03.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:03.996 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:04.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:04.048 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:04.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:04.096 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is set 2024-09-14T04:50:04.150 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:04.183 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:04.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:04.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:04.285 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:04.331 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:04.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:04.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:04.433 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:04.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:04.467 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:04.467 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:04.568 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:04.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:04.607 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:04.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:04.699 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:04 smithi039 ceph-mon[106470]: from='client.55315 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:50:04.699 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:04 smithi039 ceph-mon[106470]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:50:04.699 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:04 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1466015815' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:50:04.699 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:04 smithi039 ceph-mon[106470]: osdmap e1448: 8 total, 8 up, 8 in 2024-09-14T04:50:04.699 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:04 smithi039 ceph-mon[111235]: from='client.55315 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:50:04.699 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:04 smithi039 ceph-mon[111235]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:50:04.699 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:04 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1466015815' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:50:04.699 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:04 smithi039 ceph-mon[111235]: osdmap e1448: 8 total, 8 up, 8 in 2024-09-14T04:50:04.708 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:04.751 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:04.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:04.752 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:04.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:04 smithi129 ceph-mon[101316]: from='client.55315 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:50:04.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:04 smithi129 ceph-mon[101316]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:50:04.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:04 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1466015815' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:50:04.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:04 smithi129 ceph-mon[101316]: osdmap e1448: 8 total, 8 up, 8 in 2024-09-14T04:50:04.853 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:04.885 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:04.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:04.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:04.987 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:05.020 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:05.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:05.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:05.064 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:50:05.064 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [], 2024-09-14T04:50:05.064 INFO:teuthology.orchestra.run.smithi039.stdout: "complaint_time": 30, 2024-09-14T04:50:05.064 INFO:teuthology.orchestra.run.smithi039.stdout: "num_blocked_ops": 0 2024-09-14T04:50:05.064 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:50:05.122 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:05.156 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:05.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:05.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:05.258 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:05.299 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:05.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:05.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:05.401 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:05.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:05.443 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:05.444 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:05.544 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:05.575 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:05.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:05.576 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:05.677 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:05.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:05.708 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:05.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:05.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:05 smithi129 ceph-mon[101316]: pgmap v2631: 99 pgs: 99 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 888 op/s 2024-09-14T04:50:05.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:05 smithi039 ceph-mon[106470]: pgmap v2631: 99 pgs: 99 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 888 op/s 2024-09-14T04:50:05.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:05 smithi039 ceph-mon[111235]: pgmap v2631: 99 pgs: 99 active+clean; 5.8 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 888 op/s 2024-09-14T04:50:05.810 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:05.842 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:05.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:05.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:05.882 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-14T04:50:05.916 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:50:05.944 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:05.972 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:05.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:05.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:06.076 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:06.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:06.123 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:06.123 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:06.224 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:06.279 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:06.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:06.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:06.382 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:06.423 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:06.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:06.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:06.525 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:06.563 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:06.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:06.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:06.666 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:06.697 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:06.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:06.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:06.733 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:50:06.798 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:50:06.798 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:06.830 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:06.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:06.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:06.931 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:06.968 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:06.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:06.970 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:07.000 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:50:07.070 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:07.153 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:07.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:07.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:07.255 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:07.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:07.290 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:07.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:07.391 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:07.445 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:07.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:07.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:07.547 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:07.578 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:07.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:07.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:07.680 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:07.713 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:07.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:07.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:07.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:07 smithi129 ceph-mon[101316]: pgmap v2632: 99 pgs: 99 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2024-09-14T04:50:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:50:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:07 smithi039 ceph-mon[106470]: pgmap v2632: 99 pgs: 99 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2024-09-14T04:50:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:50:07.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:07 smithi039 ceph-mon[111235]: pgmap v2632: 99 pgs: 99 active+clean; 5.7 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 891 op/s 2024-09-14T04:50:07.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:07.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:50:07.815 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:07.859 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:07.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:07.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:07.961 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:08.003 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:08.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:08.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:08.105 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:08.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:08.140 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:08.140 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:08.241 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:08.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:08.277 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:08.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:08.378 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:08.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:08.419 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:08.419 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:08.520 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:08.556 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:08.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:08.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:08.655 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:50:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:50:08] "GET /metrics HTTP/1.1" 200 36766 "" "Prometheus/2.51.0" 2024-09-14T04:50:08.657 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:08.689 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:08.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:08.690 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:08.791 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:08.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:08.823 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:08.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:08.925 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:08.956 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:08.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:08.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:09.058 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:09.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:09.098 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:09.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:09.200 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:09.239 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:09.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:09.241 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:09.341 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:09.377 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:09.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:09.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:09.479 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:09.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:09.530 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:09.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:09.631 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:09.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:09.672 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:09.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:09 smithi129 ceph-mon[101316]: pgmap v2633: 99 pgs: 99 active+clean; 5.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2024-09-14T04:50:09.773 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:09 smithi039 ceph-mon[106470]: pgmap v2633: 99 pgs: 99 active+clean; 5.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2024-09-14T04:50:09.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:09 smithi039 ceph-mon[111235]: pgmap v2633: 99 pgs: 99 active+clean; 5.7 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2024-09-14T04:50:09.806 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:09.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:09.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:09.909 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:09.956 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:09.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:09.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:09.964 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:50:10.058 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:10.098 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:10.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:10.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:10.200 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:10.239 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:10.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:10.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:10.341 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:10.374 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:10.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:10.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:10.476 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:10.520 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:10.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:10.521 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:10.622 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:10.676 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:10.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:10.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:10.780 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:10.811 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:10.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:10.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:10.912 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:10.938 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:10.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:10.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:10.967 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:50:10.967 INFO:teuthology.orchestra.run.smithi039.stdout: "size": 20, 2024-09-14T04:50:10.967 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 600, 2024-09-14T04:50:10.967 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [ 2024-09-14T04:50:10.968 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:10.968 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233351 53.7 53:ed868572:::benchmark_data_smithi039_224949_object103258:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1355)", 2024-09-14T04:50:10.968 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.814699+0000", 2024-09-14T04:50:10.968 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 486.14636330399998, 2024-09-14T04:50:10.968 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1969449910000001, 2024-09-14T04:50:10.968 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:50:10.969 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:50:10.969 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T04:50:10.969 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:50:10.969 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:50:10.969 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:50:10.969 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233351 2024-09-14T04:50:10.970 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:10.970 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:50:10.970 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:10.970 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:50:10.970 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814699+0000", 2024-09-14T04:50:10.971 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:10.971 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:10.971 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:10.971 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:50:10.971 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814699+0000", 2024-09-14T04:50:10.971 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:10.972 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:10.972 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:10.972 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:50:10.972 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814700+0000", 2024-09-14T04:50:10.973 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6499999999999998e-07 2024-09-14T04:50:10.973 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:10.973 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:10.973 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:50:10.973 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814703+0000", 2024-09-14T04:50:10.974 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.1310000000000002e-06 2024-09-14T04:50:10.974 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:10.974 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:10.974 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:50:10.974 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814712+0000", 2024-09-14T04:50:10.974 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.4749999999999993e-06 2024-09-14T04:50:10.975 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:10.975 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:10.975 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:50:10.975 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814718+0000", 2024-09-14T04:50:10.975 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.5089999999999999e-06 2024-09-14T04:50:10.975 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:10.976 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:10.976 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:10.976 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.815266+0000", 2024-09-14T04:50:10.976 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00054806600000000005 2024-09-14T04:50:10.976 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:10.976 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:10.977 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op must wait for map", 2024-09-14T04:50:10.977 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.815274+0000", 2024-09-14T04:50:10.977 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.8630000000000008e-06 2024-09-14T04:50:10.977 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:10.978 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:10.978 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:10.978 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.830888+0000", 2024-09-14T04:50:10.978 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.015613846000000001 2024-09-14T04:50:10.978 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:10.978 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:10.979 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:50:10.979 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.830900+0000", 2024-09-14T04:50:10.979 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1911e-05 2024-09-14T04:50:10.979 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:10.979 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:10.979 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:10.980 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.814346+0000", 2024-09-14T04:50:10.980 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.98344564700000003 2024-09-14T04:50:10.980 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:10.980 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:10.980 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:50:10.981 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.814359+0000", 2024-09-14T04:50:10.981 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3509999999999999e-05 2024-09-14T04:50:10.981 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:10.981 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:10.981 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:10.981 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.009662+0000", 2024-09-14T04:50:10.982 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.195302284 2024-09-14T04:50:10.982 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:10.982 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:10.982 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:50:10.982 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.011644+0000", 2024-09-14T04:50:10.983 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0019829840000000001 2024-09-14T04:50:10.983 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:10.983 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:50:10.983 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:10.983 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:10.983 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:10.984 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233354 53.7 53:e0779eb3:::benchmark_data_smithi039_224949_object103261:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1355)", 2024-09-14T04:50:10.984 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.814735+0000", 2024-09-14T04:50:10.984 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 486.14632745599999, 2024-09-14T04:50:10.984 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2280154030000001, 2024-09-14T04:50:10.985 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:50:10.985 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:50:10.985 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:10.985 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:50:10.985 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:50:10.985 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:50:10.986 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233354 2024-09-14T04:50:10.986 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:10.986 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:50:10.986 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:10.986 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:50:10.987 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814735+0000", 2024-09-14T04:50:10.987 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:10.987 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:10.987 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:10.987 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:50:10.987 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814735+0000", 2024-09-14T04:50:10.988 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:10.988 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:10.988 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:10.988 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:50:10.988 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814736+0000", 2024-09-14T04:50:10.988 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.0200000000000002e-07 2024-09-14T04:50:10.989 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:10.989 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:10.989 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:50:10.989 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814738+0000", 2024-09-14T04:50:10.989 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.43e-06 2024-09-14T04:50:10.989 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:10.990 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:10.990 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:50:10.990 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814747+0000", 2024-09-14T04:50:10.990 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.2390000000000002e-06 2024-09-14T04:50:10.990 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:10.991 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:10.991 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:50:10.991 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.814753+0000", 2024-09-14T04:50:10.991 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.6510000000000004e-06 2024-09-14T04:50:10.991 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:10.991 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:10.992 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:10.992 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.820795+0000", 2024-09-14T04:50:10.992 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0060417639999999998 2024-09-14T04:50:10.992 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:10.992 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:10.993 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting_for_map not empty", 2024-09-14T04:50:10.993 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.820801+0000", 2024-09-14T04:50:10.993 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.9739999999999999e-06 2024-09-14T04:50:10.993 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:10.993 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:10.993 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:10.994 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.830925+0000", 2024-09-14T04:50:10.994 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.010123996 2024-09-14T04:50:10.994 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:10.994 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:10.994 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:50:10.994 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.830931+0000", 2024-09-14T04:50:10.995 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.421e-06 2024-09-14T04:50:10.995 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:10.995 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:10.995 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:10.995 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.814390+0000", 2024-09-14T04:50:10.996 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.98345883099999998 2024-09-14T04:50:10.996 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:10.996 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:10.996 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:50:10.996 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.814398+0000", 2024-09-14T04:50:10.996 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.5159999999999997e-06 2024-09-14T04:50:10.997 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:10.997 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:10.997 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:10.997 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.011957+0000", 2024-09-14T04:50:10.998 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1975596669999999 2024-09-14T04:50:10.998 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:10.998 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:10.998 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:50:10.998 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.012683+0000", 2024-09-14T04:50:10.998 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00072578599999999997 2024-09-14T04:50:10.999 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:10.999 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:10.999 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 0,1,7", 2024-09-14T04:50:10.999 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.013085+0000", 2024-09-14T04:50:10.999 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00040193299999999999 2024-09-14T04:50:10.999 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:10.999 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.000 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.000 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.031842+0000", 2024-09-14T04:50:11.000 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.018757149000000001 2024-09-14T04:50:11.000 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.000 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.000 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:50:11.000 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.035830+0000", 2024-09-14T04:50:11.000 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0039881919999999998 2024-09-14T04:50:11.000 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.000 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.000 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.000 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.038481+0000", 2024-09-14T04:50:11.001 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0026504660000000002 2024-09-14T04:50:11.001 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.001 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.001 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.001 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.042512+0000", 2024-09-14T04:50:11.001 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.004031247 2024-09-14T04:50:11.001 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.001 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.001 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:50:11.001 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.042625+0000", 2024-09-14T04:50:11.001 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000112434 2024-09-14T04:50:11.001 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.001 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.002 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:50:11.002 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.042751+0000", 2024-09-14T04:50:11.002 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000126205 2024-09-14T04:50:11.002 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.002 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:50:11.002 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.002 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.002 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.002 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233359 53.7 53:e8733a52:::benchmark_data_smithi039_224949_object103266:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-09-14T04:50:11.002 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.820832+0000", 2024-09-14T04:50:11.002 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 486.14023059099998, 2024-09-14T04:50:11.002 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2221726209999999, 2024-09-14T04:50:11.002 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:50:11.003 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:50:11.003 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:11.003 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:50:11.003 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:50:11.003 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:50:11.003 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233359 2024-09-14T04:50:11.003 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.003 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:50:11.003 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.003 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:50:11.003 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.820832+0000", 2024-09-14T04:50:11.004 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.004 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.004 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.004 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:50:11.004 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.820832+0000", 2024-09-14T04:50:11.004 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.004 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.004 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.004 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:50:11.004 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.820834+0000", 2024-09-14T04:50:11.004 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7010000000000001e-06 2024-09-14T04:50:11.004 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.004 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.004 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:50:11.005 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.820839+0000", 2024-09-14T04:50:11.005 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6890000000000003e-06 2024-09-14T04:50:11.005 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.005 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.005 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:50:11.005 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.820848+0000", 2024-09-14T04:50:11.005 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.5710000000000004e-06 2024-09-14T04:50:11.005 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.005 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.005 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:50:11.005 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.820856+0000", 2024-09-14T04:50:11.005 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6329999999999996e-06 2024-09-14T04:50:11.005 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.006 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.006 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.006 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.830952+0000", 2024-09-14T04:50:11.006 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.010096519 2024-09-14T04:50:11.006 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.006 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.006 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:50:11.006 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.830959+0000", 2024-09-14T04:50:11.006 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.426e-06 2024-09-14T04:50:11.006 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.006 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.006 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.007 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.814422+0000", 2024-09-14T04:50:11.007 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.98346295299999997 2024-09-14T04:50:11.007 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.007 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.007 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:50:11.007 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.814430+0000", 2024-09-14T04:50:11.007 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9899999999999997e-06 2024-09-14T04:50:11.007 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.007 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.007 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.008 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.028595+0000", 2024-09-14T04:50:11.008 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.214164939 2024-09-14T04:50:11.008 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.008 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.008 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:50:11.008 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.028728+0000", 2024-09-14T04:50:11.008 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000133213 2024-09-14T04:50:11.008 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.008 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.008 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 0,1,7", 2024-09-14T04:50:11.009 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.028867+0000", 2024-09-14T04:50:11.009 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000138801 2024-09-14T04:50:11.009 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.009 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.009 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.009 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.031892+0000", 2024-09-14T04:50:11.009 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0030254040000000002 2024-09-14T04:50:11.009 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.009 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.009 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.009 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.040818+0000", 2024-09-14T04:50:11.009 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0089257309999999992 2024-09-14T04:50:11.009 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.009 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.010 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:50:11.010 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.041553+0000", 2024-09-14T04:50:11.010 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00073544599999999995 2024-09-14T04:50:11.010 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.010 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.010 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.010 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.042807+0000", 2024-09-14T04:50:11.010 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0012535210000000001 2024-09-14T04:50:11.010 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.010 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.011 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:50:11.011 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.042903+0000", 2024-09-14T04:50:11.011 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.6471999999999994e-05 2024-09-14T04:50:11.011 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.011 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.011 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:50:11.011 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.043005+0000", 2024-09-14T04:50:11.012 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000101612 2024-09-14T04:50:11.012 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.012 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:50:11.012 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.012 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.012 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.012 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233393 53.7 53:e9d8059a:::benchmark_data_smithi039_224949_object103300:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-09-14T04:50:11.012 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.894168+0000", 2024-09-14T04:50:11.012 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 486.066894988, 2024-09-14T04:50:11.012 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.149336516, 2024-09-14T04:50:11.012 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:50:11.012 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:50:11.012 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:11.013 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:50:11.013 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:50:11.013 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:50:11.013 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233393 2024-09-14T04:50:11.013 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.014 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:50:11.014 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.014 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:50:11.014 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894168+0000", 2024-09-14T04:50:11.014 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.014 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.015 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.015 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:50:11.015 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894168+0000", 2024-09-14T04:50:11.015 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.015 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.015 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.015 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:50:11.016 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894169+0000", 2024-09-14T04:50:11.016 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.443e-06 2024-09-14T04:50:11.016 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.016 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.016 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:50:11.016 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894173+0000", 2024-09-14T04:50:11.016 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.5149999999999998e-06 2024-09-14T04:50:11.016 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.016 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.016 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:50:11.016 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894182+0000", 2024-09-14T04:50:11.017 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.3360000000000001e-06 2024-09-14T04:50:11.017 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.017 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.017 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:50:11.017 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894188+0000", 2024-09-14T04:50:11.017 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3439999999999997e-06 2024-09-14T04:50:11.018 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.018 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.018 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.018 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894306+0000", 2024-09-14T04:50:11.018 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000117439 2024-09-14T04:50:11.018 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.018 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.018 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:50:11.018 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894316+0000", 2024-09-14T04:50:11.018 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0349e-05 2024-09-14T04:50:11.018 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.019 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.019 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.020 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.814569+0000", 2024-09-14T04:50:11.020 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.92025298899999997 2024-09-14T04:50:11.020 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.020 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.020 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:50:11.020 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.814584+0000", 2024-09-14T04:50:11.021 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.468e-05 2024-09-14T04:50:11.021 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.021 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.021 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.021 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.032234+0000", 2024-09-14T04:50:11.021 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2176502870000001 2024-09-14T04:50:11.021 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.022 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.022 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:50:11.022 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.032359+0000", 2024-09-14T04:50:11.022 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000124533 2024-09-14T04:50:11.023 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.023 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.023 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 0,1,7", 2024-09-14T04:50:11.023 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.032518+0000", 2024-09-14T04:50:11.023 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015903900000000001 2024-09-14T04:50:11.023 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.023 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.023 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.024 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.040779+0000", 2024-09-14T04:50:11.024 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0082608909999999994 2024-09-14T04:50:11.024 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.024 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.024 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.024 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.041107+0000", 2024-09-14T04:50:11.024 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000328778 2024-09-14T04:50:11.024 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.024 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.024 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:50:11.024 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.041689+0000", 2024-09-14T04:50:11.024 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00058127799999999996 2024-09-14T04:50:11.024 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.025 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.025 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.025 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.043303+0000", 2024-09-14T04:50:11.025 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001614297 2024-09-14T04:50:11.025 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.025 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.025 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:50:11.025 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.043390+0000", 2024-09-14T04:50:11.025 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.6940000000000004e-05 2024-09-14T04:50:11.025 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.025 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.025 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:50:11.025 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.043504+0000", 2024-09-14T04:50:11.026 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011437799999999999 2024-09-14T04:50:11.026 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.026 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:50:11.026 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.026 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.026 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.026 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242867 53.a 53:54ad9484:::benchmark_data_smithi039_224949_object112774:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1363)", 2024-09-14T04:50:11.026 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:20.495372+0000", 2024-09-14T04:50:11.026 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 470.46569087900002, 2024-09-14T04:50:11.026 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.997320294, 2024-09-14T04:50:11.026 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:50:11.026 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:50:11.026 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:11.026 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:50:11.026 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:50:11.027 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:50:11.027 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242867 2024-09-14T04:50:11.027 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.027 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:50:11.027 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.027 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:50:11.027 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.495372+0000", 2024-09-14T04:50:11.027 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.027 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.027 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.028 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:50:11.028 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.495372+0000", 2024-09-14T04:50:11.028 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.028 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.028 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.028 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:50:11.028 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.495373+0000", 2024-09-14T04:50:11.028 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.850000000000001e-07 2024-09-14T04:50:11.028 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.028 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.028 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:50:11.028 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.495377+0000", 2024-09-14T04:50:11.028 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5059999999999997e-06 2024-09-14T04:50:11.029 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.029 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.029 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:50:11.029 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.495390+0000", 2024-09-14T04:50:11.029 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2498e-05 2024-09-14T04:50:11.029 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.029 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.029 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:50:11.029 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.495397+0000", 2024-09-14T04:50:11.029 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.2060000000000001e-06 2024-09-14T04:50:11.029 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.029 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.029 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.029 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.495443+0000", 2024-09-14T04:50:11.030 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6139e-05 2024-09-14T04:50:11.030 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.030 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.030 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:50:11.030 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.495449+0000", 2024-09-14T04:50:11.030 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5949999999999998e-06 2024-09-14T04:50:11.030 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.030 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.030 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.030 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.479903+0000", 2024-09-14T04:50:11.030 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.98445407699999998 2024-09-14T04:50:11.030 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.030 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.030 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:50:11.030 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.480015+0000", 2024-09-14T04:50:11.031 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011235600000000001 2024-09-14T04:50:11.031 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.031 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.031 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:50:11.031 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.480135+0000", 2024-09-14T04:50:11.031 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000119462 2024-09-14T04:50:11.031 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.031 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.031 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.032 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.484482+0000", 2024-09-14T04:50:11.032 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0043470779999999999 2024-09-14T04:50:11.032 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.032 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.032 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.032 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.484532+0000", 2024-09-14T04:50:11.032 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9981999999999997e-05 2024-09-14T04:50:11.032 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.032 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.032 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:50:11.032 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.492566+0000", 2024-09-14T04:50:11.032 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0080345810000000007 2024-09-14T04:50:11.032 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.033 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.033 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:50:11.033 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.492646+0000", 2024-09-14T04:50:11.033 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0039999999999999e-05 2024-09-14T04:50:11.033 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.033 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.033 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:50:11.033 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.492692+0000", 2024-09-14T04:50:11.033 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5788999999999998e-05 2024-09-14T04:50:11.033 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.033 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:50:11.033 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.033 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.034 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.034 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242882 53.a 53:5478a5c5:::benchmark_data_smithi039_224949_object112789:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1363)", 2024-09-14T04:50:11.034 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:20.512060+0000", 2024-09-14T04:50:11.034 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 470.44900302600001, 2024-09-14T04:50:11.034 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0040218409999999, 2024-09-14T04:50:11.034 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:50:11.034 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:50:11.034 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:11.034 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:50:11.034 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:50:11.034 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:50:11.034 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242882 2024-09-14T04:50:11.034 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.034 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:50:11.035 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.035 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:50:11.035 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.512060+0000", 2024-09-14T04:50:11.035 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.035 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.035 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.035 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:50:11.035 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.512060+0000", 2024-09-14T04:50:11.035 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.035 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.035 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.035 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:50:11.035 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.512061+0000", 2024-09-14T04:50:11.035 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6869999999999999e-06 2024-09-14T04:50:11.036 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.036 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.036 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:50:11.036 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.512067+0000", 2024-09-14T04:50:11.036 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.993e-06 2024-09-14T04:50:11.036 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.036 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.036 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:50:11.036 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.512078+0000", 2024-09-14T04:50:11.036 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0913999999999999e-05 2024-09-14T04:50:11.036 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.036 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.036 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:50:11.036 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.512085+0000", 2024-09-14T04:50:11.037 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7109999999999998e-06 2024-09-14T04:50:11.037 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.037 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.037 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.037 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.512156+0000", 2024-09-14T04:50:11.037 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.0850000000000001e-05 2024-09-14T04:50:11.037 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.037 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.037 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:50:11.037 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.512164+0000", 2024-09-14T04:50:11.037 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.4530000000000002e-06 2024-09-14T04:50:11.037 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.037 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.038 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.038 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.484560+0000", 2024-09-14T04:50:11.038 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.972395491 2024-09-14T04:50:11.038 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.038 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.038 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:50:11.038 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.484663+0000", 2024-09-14T04:50:11.038 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00010302300000000001 2024-09-14T04:50:11.038 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.038 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.039 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:50:11.039 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.484869+0000", 2024-09-14T04:50:11.039 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000206104 2024-09-14T04:50:11.039 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.039 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.039 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.039 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.492736+0000", 2024-09-14T04:50:11.039 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0078666750000000001 2024-09-14T04:50:11.039 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.039 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.039 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.039 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.492783+0000", 2024-09-14T04:50:11.039 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7630999999999998e-05 2024-09-14T04:50:11.039 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.040 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.040 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:50:11.040 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516001+0000", 2024-09-14T04:50:11.040 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.023217924000000001 2024-09-14T04:50:11.040 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.040 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.040 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:50:11.040 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516046+0000", 2024-09-14T04:50:11.040 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4833999999999998e-05 2024-09-14T04:50:11.040 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.040 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.040 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:50:11.041 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516082+0000", 2024-09-14T04:50:11.041 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.5550999999999999e-05 2024-09-14T04:50:11.041 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.041 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:50:11.041 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.041 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.041 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.041 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242920 53.a 53:5584c2e0:::benchmark_data_smithi039_224949_object112827:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1363)", 2024-09-14T04:50:11.041 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:20.566563+0000", 2024-09-14T04:50:11.041 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 470.39450007200003, 2024-09-14T04:50:11.041 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.94971936199999996, 2024-09-14T04:50:11.041 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:50:11.042 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:50:11.042 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:11.042 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:50:11.042 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:50:11.042 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:50:11.042 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242920 2024-09-14T04:50:11.042 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.042 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:50:11.042 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.042 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:50:11.042 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.566563+0000", 2024-09-14T04:50:11.042 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.042 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.042 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.042 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:50:11.043 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.566563+0000", 2024-09-14T04:50:11.043 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.043 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.043 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.043 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:50:11.043 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.566564+0000", 2024-09-14T04:50:11.043 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4890000000000001e-06 2024-09-14T04:50:11.043 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.043 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.043 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:50:11.043 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.566569+0000", 2024-09-14T04:50:11.043 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4750000000000004e-06 2024-09-14T04:50:11.043 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.044 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.044 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:50:11.044 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.566579+0000", 2024-09-14T04:50:11.044 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0421000000000001e-05 2024-09-14T04:50:11.044 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.044 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.044 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:50:11.044 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.566586+0000", 2024-09-14T04:50:11.045 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7449999999999998e-06 2024-09-14T04:50:11.045 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.045 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.045 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.045 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.566678+0000", 2024-09-14T04:50:11.045 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.2319999999999997e-05 2024-09-14T04:50:11.045 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.046 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.046 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:50:11.046 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.566686+0000", 2024-09-14T04:50:11.046 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9400000000000002e-06 2024-09-14T04:50:11.046 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.046 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.046 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.046 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.488450+0000", 2024-09-14T04:50:11.046 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.92176433300000005 2024-09-14T04:50:11.046 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.046 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.046 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:50:11.046 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.488608+0000", 2024-09-14T04:50:11.046 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015782400000000001 2024-09-14T04:50:11.047 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.047 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.047 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:50:11.047 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.488674+0000", 2024-09-14T04:50:11.047 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.5457999999999996e-05 2024-09-14T04:50:11.047 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.047 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.047 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.047 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.492855+0000", 2024-09-14T04:50:11.047 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0041807909999999997 2024-09-14T04:50:11.047 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.047 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.047 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.048 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.492909+0000", 2024-09-14T04:50:11.048 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4629999999999997e-05 2024-09-14T04:50:11.048 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.048 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.048 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:50:11.048 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516207+0000", 2024-09-14T04:50:11.048 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.023297549000000001 2024-09-14T04:50:11.048 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.048 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.048 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:50:11.049 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516246+0000", 2024-09-14T04:50:11.049 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.9242000000000001e-05 2024-09-14T04:50:11.049 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.049 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.049 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:50:11.049 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516282+0000", 2024-09-14T04:50:11.049 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.6145e-05 2024-09-14T04:50:11.049 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.049 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:50:11.049 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.049 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.049 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.049 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242925 53.a 53:5601ec35:::benchmark_data_smithi039_224949_object112832:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1363)", 2024-09-14T04:50:11.049 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:20.570988+0000", 2024-09-14T04:50:11.050 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 470.39007505699999, 2024-09-14T04:50:11.050 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.94542205099999999, 2024-09-14T04:50:11.050 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:50:11.050 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:50:11.050 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:11.050 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:50:11.050 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:50:11.050 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:50:11.050 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242925 2024-09-14T04:50:11.050 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.050 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:50:11.050 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.050 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:50:11.050 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.570988+0000", 2024-09-14T04:50:11.051 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.051 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.051 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.051 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:50:11.051 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.570988+0000", 2024-09-14T04:50:11.051 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.051 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.051 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.051 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:50:11.051 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.570990+0000", 2024-09-14T04:50:11.051 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8870000000000001e-06 2024-09-14T04:50:11.051 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.051 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.051 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:50:11.052 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.570995+0000", 2024-09-14T04:50:11.052 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7799999999999997e-06 2024-09-14T04:50:11.052 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.052 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.052 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:50:11.052 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.571008+0000", 2024-09-14T04:50:11.052 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2683e-05 2024-09-14T04:50:11.052 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.052 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.052 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:50:11.052 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.571016+0000", 2024-09-14T04:50:11.052 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.1880000000000002e-06 2024-09-14T04:50:11.052 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.053 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.053 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.053 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.571101+0000", 2024-09-14T04:50:11.053 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.4460000000000001e-05 2024-09-14T04:50:11.053 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.053 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.053 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:50:11.053 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.571110+0000", 2024-09-14T04:50:11.053 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.7700000000000007e-06 2024-09-14T04:50:11.053 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.053 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.053 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.053 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.492959+0000", 2024-09-14T04:50:11.054 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.92184951400000004 2024-09-14T04:50:11.054 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.054 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.054 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:50:11.054 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.493043+0000", 2024-09-14T04:50:11.054 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.3777999999999996e-05 2024-09-14T04:50:11.054 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.054 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.054 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:50:11.055 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.493120+0000", 2024-09-14T04:50:11.055 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7489000000000003e-05 2024-09-14T04:50:11.055 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.055 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.055 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.055 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.500548+0000", 2024-09-14T04:50:11.055 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0074275349999999999 2024-09-14T04:50:11.055 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.055 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.055 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.055 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.500616+0000", 2024-09-14T04:50:11.055 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7871000000000001e-05 2024-09-14T04:50:11.055 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.055 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.056 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:50:11.056 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516288+0000", 2024-09-14T04:50:11.056 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.015672654000000001 2024-09-14T04:50:11.056 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.056 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.056 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:50:11.056 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516331+0000", 2024-09-14T04:50:11.056 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2698999999999999e-05 2024-09-14T04:50:11.056 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.056 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.056 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:50:11.056 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516410+0000", 2024-09-14T04:50:11.056 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.8743000000000001e-05 2024-09-14T04:50:11.057 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.057 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:50:11.057 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.057 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.057 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.057 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242952 53.a 53:58b7a1db:::benchmark_data_smithi039_224949_object112859:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1363)", 2024-09-14T04:50:11.057 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:20.604696+0000", 2024-09-14T04:50:11.057 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 470.35636724599999, 2024-09-14T04:50:11.057 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.91180369299999997, 2024-09-14T04:50:11.057 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:50:11.057 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:50:11.058 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:11.058 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:50:11.058 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:50:11.058 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:50:11.058 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242952 2024-09-14T04:50:11.058 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.058 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:50:11.058 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.058 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:50:11.058 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.604696+0000", 2024-09-14T04:50:11.058 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.058 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.058 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.059 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:50:11.059 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.604696+0000", 2024-09-14T04:50:11.059 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.059 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.059 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.059 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:50:11.059 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.604697+0000", 2024-09-14T04:50:11.059 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8789999999999999e-06 2024-09-14T04:50:11.059 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.059 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.059 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:50:11.060 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.604703+0000", 2024-09-14T04:50:11.060 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7209999999999999e-06 2024-09-14T04:50:11.060 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.060 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.060 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:50:11.060 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.604721+0000", 2024-09-14T04:50:11.060 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7353000000000001e-05 2024-09-14T04:50:11.060 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.060 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.060 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:50:11.060 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.604728+0000", 2024-09-14T04:50:11.060 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7139999999999993e-06 2024-09-14T04:50:11.060 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.060 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.061 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.061 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.604805+0000", 2024-09-14T04:50:11.061 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6718999999999995e-05 2024-09-14T04:50:11.061 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.061 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.061 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:50:11.061 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.604814+0000", 2024-09-14T04:50:11.061 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.0589999999999992e-06 2024-09-14T04:50:11.061 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.061 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.061 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.061 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.496130+0000", 2024-09-14T04:50:11.062 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.89131583400000003 2024-09-14T04:50:11.062 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.062 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.062 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:50:11.062 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.496260+0000", 2024-09-14T04:50:11.062 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012988 2024-09-14T04:50:11.063 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.063 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.063 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:50:11.063 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.496352+0000", 2024-09-14T04:50:11.063 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.2159999999999999e-05 2024-09-14T04:50:11.063 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.063 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.063 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.063 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.500654+0000", 2024-09-14T04:50:11.063 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0043017810000000002 2024-09-14T04:50:11.063 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.063 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.063 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.064 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.500688+0000", 2024-09-14T04:50:11.064 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.4146000000000001e-05 2024-09-14T04:50:11.064 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.064 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.064 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:50:11.064 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516417+0000", 2024-09-14T04:50:11.064 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.015729097000000001 2024-09-14T04:50:11.064 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.064 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.064 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:50:11.064 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516462+0000", 2024-09-14T04:50:11.064 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5104e-05 2024-09-14T04:50:11.064 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.064 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.065 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:50:11.065 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516499+0000", 2024-09-14T04:50:11.065 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.7246000000000002e-05 2024-09-14T04:50:11.065 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.065 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:50:11.065 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.065 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.065 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.065 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242956 53.a 53:53d3e9bf:::benchmark_data_smithi039_224949_object112863:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1363)", 2024-09-14T04:50:11.065 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:20.610137+0000", 2024-09-14T04:50:11.065 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 470.35092553499999, 2024-09-14T04:50:11.066 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.90651257799999996, 2024-09-14T04:50:11.066 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:50:11.066 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:50:11.066 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:11.066 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:50:11.066 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:50:11.066 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:50:11.066 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242956 2024-09-14T04:50:11.066 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.066 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:50:11.066 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.066 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:50:11.066 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.610137+0000", 2024-09-14T04:50:11.067 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.067 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.067 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.067 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:50:11.067 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.610137+0000", 2024-09-14T04:50:11.067 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.067 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.067 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.067 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:50:11.067 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.610139+0000", 2024-09-14T04:50:11.067 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9520000000000001e-06 2024-09-14T04:50:11.067 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.067 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.067 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:50:11.068 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.610148+0000", 2024-09-14T04:50:11.068 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.6459999999999999e-06 2024-09-14T04:50:11.068 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.068 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.068 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:50:11.068 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.610160+0000", 2024-09-14T04:50:11.068 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2574e-05 2024-09-14T04:50:11.068 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.068 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.068 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:50:11.068 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.610168+0000", 2024-09-14T04:50:11.068 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6669999999999996e-06 2024-09-14T04:50:11.068 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.069 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.069 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.069 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.610230+0000", 2024-09-14T04:50:11.069 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1472999999999999e-05 2024-09-14T04:50:11.069 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.069 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.069 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:50:11.069 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.610239+0000", 2024-09-14T04:50:11.069 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.0880000000000001e-06 2024-09-14T04:50:11.069 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.069 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.069 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.069 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.500716+0000", 2024-09-14T04:50:11.069 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.89047721599999996 2024-09-14T04:50:11.070 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.070 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.070 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:50:11.070 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.500820+0000", 2024-09-14T04:50:11.070 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00010407 2024-09-14T04:50:11.070 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.070 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.070 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:50:11.070 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.500928+0000", 2024-09-14T04:50:11.070 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000107909 2024-09-14T04:50:11.070 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.070 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.071 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.071 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.512156+0000", 2024-09-14T04:50:11.071 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.011227849 2024-09-14T04:50:11.071 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.071 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.071 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.071 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.512195+0000", 2024-09-14T04:50:11.071 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.9739e-05 2024-09-14T04:50:11.071 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.071 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.071 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:50:11.072 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516505+0000", 2024-09-14T04:50:11.072 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0043095210000000002 2024-09-14T04:50:11.072 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.072 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.072 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:50:11.072 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516541+0000", 2024-09-14T04:50:11.072 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.5911999999999998e-05 2024-09-14T04:50:11.072 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.072 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.072 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:50:11.072 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516650+0000", 2024-09-14T04:50:11.072 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000108962 2024-09-14T04:50:11.072 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.073 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:50:11.073 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.073 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.073 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.073 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242984 53.a 53:5753476b:::benchmark_data_smithi039_224949_object112891:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1363)", 2024-09-14T04:50:11.073 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:20.654008+0000", 2024-09-14T04:50:11.073 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 470.30705495299998, 2024-09-14T04:50:11.073 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.86274647400000004, 2024-09-14T04:50:11.073 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:50:11.073 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:50:11.073 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:11.073 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:50:11.073 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:50:11.074 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:50:11.074 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242984 2024-09-14T04:50:11.074 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.074 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:50:11.074 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.074 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:50:11.074 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.654008+0000", 2024-09-14T04:50:11.074 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.075 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.075 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.075 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:50:11.075 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.654008+0000", 2024-09-14T04:50:11.075 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.075 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.075 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.075 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:50:11.075 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.654010+0000", 2024-09-14T04:50:11.076 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.753e-06 2024-09-14T04:50:11.076 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.076 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.076 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:50:11.076 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.654015+0000", 2024-09-14T04:50:11.076 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4160000000000003e-06 2024-09-14T04:50:11.076 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.076 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.076 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:50:11.076 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.654031+0000", 2024-09-14T04:50:11.076 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6458000000000001e-05 2024-09-14T04:50:11.076 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.076 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.077 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:50:11.077 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.654039+0000", 2024-09-14T04:50:11.077 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1019999999999999e-06 2024-09-14T04:50:11.077 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.077 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.077 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.077 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.654110+0000", 2024-09-14T04:50:11.077 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1265000000000003e-05 2024-09-14T04:50:11.077 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.077 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.077 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:50:11.077 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.654116+0000", 2024-09-14T04:50:11.077 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6910000000000002e-06 2024-09-14T04:50:11.077 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.078 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.078 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.078 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.503852+0000", 2024-09-14T04:50:11.078 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.84973631199999999 2024-09-14T04:50:11.078 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.078 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.078 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:50:11.078 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.503999+0000", 2024-09-14T04:50:11.078 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014705799999999999 2024-09-14T04:50:11.078 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.078 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.078 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:50:11.078 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.504119+0000", 2024-09-14T04:50:11.078 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000119982 2024-09-14T04:50:11.079 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.079 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.079 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.079 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.512233+0000", 2024-09-14T04:50:11.079 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0081145850000000005 2024-09-14T04:50:11.079 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.079 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.079 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.079 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.512271+0000", 2024-09-14T04:50:11.079 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.7440000000000001e-05 2024-09-14T04:50:11.079 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.079 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.079 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:50:11.079 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516656+0000", 2024-09-14T04:50:11.080 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0043848259999999997 2024-09-14T04:50:11.080 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.080 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.080 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:50:11.080 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516720+0000", 2024-09-14T04:50:11.080 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3862000000000006e-05 2024-09-14T04:50:11.080 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.080 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.080 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:50:11.080 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516754+0000", 2024-09-14T04:50:11.080 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.4724000000000002e-05 2024-09-14T04:50:11.080 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.080 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:50:11.080 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.081 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.081 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.081 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242992 53.a 53:5da6abf3:::benchmark_data_smithi039_224949_object112899:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1363)", 2024-09-14T04:50:11.081 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:20.682771+0000", 2024-09-14T04:50:11.081 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 470.27829180100002, 2024-09-14T04:50:11.081 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.83649015400000004, 2024-09-14T04:50:11.081 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:50:11.081 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:50:11.081 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:11.081 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:50:11.081 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:50:11.081 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:50:11.081 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242992 2024-09-14T04:50:11.081 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.082 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:50:11.082 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.082 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:50:11.082 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.682771+0000", 2024-09-14T04:50:11.082 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.082 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.082 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.082 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:50:11.082 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.682771+0000", 2024-09-14T04:50:11.082 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.082 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.082 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.082 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:50:11.083 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.682772+0000", 2024-09-14T04:50:11.083 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.271e-06 2024-09-14T04:50:11.083 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.083 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.083 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:50:11.083 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.682777+0000", 2024-09-14T04:50:11.083 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2849999999999996e-06 2024-09-14T04:50:11.083 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.083 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.083 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:50:11.083 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.682787+0000", 2024-09-14T04:50:11.083 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0787e-05 2024-09-14T04:50:11.083 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.083 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.084 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:50:11.084 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.682793+0000", 2024-09-14T04:50:11.084 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3990000000000003e-06 2024-09-14T04:50:11.084 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.084 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.084 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.084 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.682856+0000", 2024-09-14T04:50:11.084 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2890000000000003e-05 2024-09-14T04:50:11.084 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.084 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.084 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:50:11.084 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.682863+0000", 2024-09-14T04:50:11.084 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9600000000000003e-06 2024-09-14T04:50:11.085 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.085 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.085 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.085 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.508101+0000", 2024-09-14T04:50:11.085 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.82523816900000002 2024-09-14T04:50:11.085 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.085 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.085 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:50:11.086 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.508347+0000", 2024-09-14T04:50:11.086 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000246629 2024-09-14T04:50:11.086 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.086 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.086 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:50:11.086 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.508454+0000", 2024-09-14T04:50:11.086 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000106424 2024-09-14T04:50:11.086 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.086 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.086 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.086 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.512305+0000", 2024-09-14T04:50:11.086 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.003851508 2024-09-14T04:50:11.086 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.086 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.087 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.087 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516195+0000", 2024-09-14T04:50:11.087 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0038899450000000001 2024-09-14T04:50:11.087 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.087 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.087 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:50:11.087 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516761+0000", 2024-09-14T04:50:11.087 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00056550800000000005 2024-09-14T04:50:11.087 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.087 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.087 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:50:11.088 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.516900+0000", 2024-09-14T04:50:11.088 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000139639 2024-09-14T04:50:11.088 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.088 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.088 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:50:11.088 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:21.519261+0000", 2024-09-14T04:50:11.088 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0023607400000000001 2024-09-14T04:50:11.088 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.088 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:50:11.088 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.088 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.088 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.088 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:135589 54.c 54:356406db:::benchmark_data_smithi039_264334_object9192:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1401)", 2024-09-14T04:50:11.089 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:56.300102+0000", 2024-09-14T04:50:11.089 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 314.66096124900002, 2024-09-14T04:50:11.089 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1549550009999998, 2024-09-14T04:50:11.089 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:50:11.089 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:50:11.089 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T04:50:11.089 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:50:11.089 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:50:11.089 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:50:11.089 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 135589 2024-09-14T04:50:11.089 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.089 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:50:11.089 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.090 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:50:11.090 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.300102+0000", 2024-09-14T04:50:11.090 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.090 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.090 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.090 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:50:11.090 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.300102+0000", 2024-09-14T04:50:11.090 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.090 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.090 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.090 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:50:11.090 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.300103+0000", 2024-09-14T04:50:11.090 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.939e-06 2024-09-14T04:50:11.091 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.091 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.091 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:50:11.091 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.300109+0000", 2024-09-14T04:50:11.091 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7389999999999996e-06 2024-09-14T04:50:11.091 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.091 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.091 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:50:11.091 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.300125+0000", 2024-09-14T04:50:11.091 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6076999999999999e-05 2024-09-14T04:50:11.091 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.092 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.092 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:50:11.092 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.300133+0000", 2024-09-14T04:50:11.092 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.2230000000000001e-06 2024-09-14T04:50:11.092 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.092 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.092 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.092 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.301312+0000", 2024-09-14T04:50:11.092 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0011795480000000001 2024-09-14T04:50:11.092 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.092 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.092 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:50:11.092 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.302565+0000", 2024-09-14T04:50:11.092 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0012534099999999999 2024-09-14T04:50:11.093 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.093 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.093 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 5,6", 2024-09-14T04:50:11.093 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.302670+0000", 2024-09-14T04:50:11.093 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0001047 2024-09-14T04:50:11.093 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.093 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.093 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.093 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.306221+0000", 2024-09-14T04:50:11.093 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00355105 2024-09-14T04:50:11.093 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.093 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.093 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:50:11.094 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.306468+0000", 2024-09-14T04:50:11.094 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00024643100000000003 2024-09-14T04:50:11.094 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.094 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.094 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.094 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.420868+0000", 2024-09-14T04:50:11.094 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.114400113 2024-09-14T04:50:11.094 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.094 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.094 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:50:11.094 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.420879+0000", 2024-09-14T04:50:11.094 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0720000000000001e-05 2024-09-14T04:50:11.094 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.094 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.095 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.095 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.324315+0000", 2024-09-14T04:50:11.095 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.90343673899999999 2024-09-14T04:50:11.095 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.095 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.095 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:50:11.095 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.324338+0000", 2024-09-14T04:50:11.095 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2510999999999998e-05 2024-09-14T04:50:11.095 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.095 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.095 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.095 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.454966+0000", 2024-09-14T04:50:11.095 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1306284820000001 2024-09-14T04:50:11.095 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.095 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.096 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:50:11.096 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.455057+0000", 2024-09-14T04:50:11.096 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.0319e-05 2024-09-14T04:50:11.096 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.096 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:50:11.096 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.096 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.096 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.096 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:135618 54.c 54:3384db48:::benchmark_data_smithi039_264334_object9221:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:50:11.096 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:56.374671+0000", 2024-09-14T04:50:11.096 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 314.5863918, 2024-09-14T04:50:11.097 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.091442705, 2024-09-14T04:50:11.097 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:50:11.097 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:50:11.097 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:11.097 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:50:11.097 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:50:11.097 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:50:11.097 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 135618 2024-09-14T04:50:11.097 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.097 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:50:11.097 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.098 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:50:11.098 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:11.099 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.374671+0000", 2024-09-14T04:50:11.099 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.099 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.099 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.099 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:50:11.100 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.374671+0000", 2024-09-14T04:50:11.100 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.100 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.100 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.100 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:50:11.100 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.374672+0000", 2024-09-14T04:50:11.100 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.5000000000000001e-07 2024-09-14T04:50:11.100 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.100 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.100 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:50:11.100 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.374676+0000", 2024-09-14T04:50:11.100 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3590000000000001e-06 2024-09-14T04:50:11.100 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.100 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.101 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:50:11.101 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.374686+0000", 2024-09-14T04:50:11.101 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.9350000000000006e-06 2024-09-14T04:50:11.101 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.101 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.101 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:50:11.101 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.374693+0000", 2024-09-14T04:50:11.101 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9659999999999999e-06 2024-09-14T04:50:11.101 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.101 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.101 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.101 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.429534+0000", 2024-09-14T04:50:11.101 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.054841049000000003 2024-09-14T04:50:11.102 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.102 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.102 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:50:11.102 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.429544+0000", 2024-09-14T04:50:11.102 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.5289999999999997e-06 2024-09-14T04:50:11.102 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.102 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.102 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.102 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.325910+0000", 2024-09-14T04:50:11.102 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.89636666499999995 2024-09-14T04:50:11.102 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.102 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.102 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:50:11.102 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.325921+0000", 2024-09-14T04:50:11.103 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0747000000000001e-05 2024-09-14T04:50:11.103 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.103 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.103 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.103 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.455428+0000", 2024-09-14T04:50:11.103 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1295071729999999 2024-09-14T04:50:11.103 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.103 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.103 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:50:11.103 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.455553+0000", 2024-09-14T04:50:11.104 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000125041 2024-09-14T04:50:11.104 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.104 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.104 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 4,5,6", 2024-09-14T04:50:11.104 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.455674+0000", 2024-09-14T04:50:11.104 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000120759 2024-09-14T04:50:11.104 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.104 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.104 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:50:11.105 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.465735+0000", 2024-09-14T04:50:11.105 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.010060955 2024-09-14T04:50:11.105 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.105 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.105 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.105 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.465794+0000", 2024-09-14T04:50:11.105 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.9268000000000002e-05 2024-09-14T04:50:11.105 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.105 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.105 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.105 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.465869+0000", 2024-09-14T04:50:11.105 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.4383999999999998e-05 2024-09-14T04:50:11.105 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.106 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.106 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.106 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.465918+0000", 2024-09-14T04:50:11.106 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9656e-05 2024-09-14T04:50:11.106 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.107 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.107 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:50:11.107 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.465988+0000", 2024-09-14T04:50:11.107 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9436000000000004e-05 2024-09-14T04:50:11.107 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.107 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.108 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:50:11.108 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.466114+0000", 2024-09-14T04:50:11.108 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000125833 2024-09-14T04:50:11.108 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.108 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:50:11.109 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.109 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.109 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.109 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:135633 54.c 54:33252ab7:::benchmark_data_smithi039_264334_object9236:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:50:11.109 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:56.431333+0000", 2024-09-14T04:50:11.109 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 314.52973021999998, 2024-09-14T04:50:11.109 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0351089770000002, 2024-09-14T04:50:11.109 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:50:11.110 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:50:11.110 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:11.110 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:50:11.110 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:50:11.110 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:50:11.110 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 135633 2024-09-14T04:50:11.110 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.110 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:50:11.110 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.110 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:50:11.110 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431333+0000", 2024-09-14T04:50:11.110 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.111 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.111 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.111 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:50:11.111 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431333+0000", 2024-09-14T04:50:11.111 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.111 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.112 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.112 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:50:11.112 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431335+0000", 2024-09-14T04:50:11.112 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9479999999999998e-06 2024-09-14T04:50:11.112 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.112 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.112 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:50:11.112 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431340+0000", 2024-09-14T04:50:11.112 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5559999999999998e-06 2024-09-14T04:50:11.112 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.112 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.112 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:50:11.112 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431354+0000", 2024-09-14T04:50:11.113 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3772e-05 2024-09-14T04:50:11.113 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.113 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.113 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:50:11.113 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431362+0000", 2024-09-14T04:50:11.113 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.5390000000000007e-06 2024-09-14T04:50:11.113 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.113 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.113 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.113 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.447068+0000", 2024-09-14T04:50:11.113 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.015705666 2024-09-14T04:50:11.113 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.114 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.114 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:50:11.114 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.447080+0000", 2024-09-14T04:50:11.114 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1506e-05 2024-09-14T04:50:11.114 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.114 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.114 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.114 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.325950+0000", 2024-09-14T04:50:11.114 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.87887031699999996 2024-09-14T04:50:11.114 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.114 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.114 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:50:11.115 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.325958+0000", 2024-09-14T04:50:11.115 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.1540000000000002e-06 2024-09-14T04:50:11.115 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.115 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.115 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.115 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.458856+0000", 2024-09-14T04:50:11.115 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1328979669999999 2024-09-14T04:50:11.115 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.115 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.115 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:50:11.115 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.459013+0000", 2024-09-14T04:50:11.115 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015676700000000001 2024-09-14T04:50:11.115 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.116 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.116 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 4,5,6", 2024-09-14T04:50:11.116 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.459124+0000", 2024-09-14T04:50:11.116 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000111482 2024-09-14T04:50:11.116 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.116 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.116 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:50:11.116 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.466127+0000", 2024-09-14T04:50:11.116 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0070024850000000001 2024-09-14T04:50:11.116 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.117 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.117 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.117 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.466181+0000", 2024-09-14T04:50:11.117 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4088000000000001e-05 2024-09-14T04:50:11.117 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.117 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.117 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.117 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.466226+0000", 2024-09-14T04:50:11.117 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4965000000000001e-05 2024-09-14T04:50:11.117 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.117 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.117 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.117 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.466287+0000", 2024-09-14T04:50:11.117 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1079000000000003e-05 2024-09-14T04:50:11.118 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.118 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.118 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:50:11.118 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.466341+0000", 2024-09-14T04:50:11.118 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4379000000000002e-05 2024-09-14T04:50:11.118 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.118 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.118 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:50:11.118 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.466442+0000", 2024-09-14T04:50:11.118 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000100307 2024-09-14T04:50:11.118 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.118 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:50:11.118 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.119 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.119 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.119 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:135635 54.c 54:31b8983c:::benchmark_data_smithi039_264334_object9238:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:50:11.119 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:56.431574+0000", 2024-09-14T04:50:11.119 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 314.52948876699998, 2024-09-14T04:50:11.119 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0581618759999998, 2024-09-14T04:50:11.119 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:50:11.119 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:50:11.119 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:11.119 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:50:11.119 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:50:11.120 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:50:11.120 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 135635 2024-09-14T04:50:11.120 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.120 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:50:11.120 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.120 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:50:11.120 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431574+0000", 2024-09-14T04:50:11.120 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.120 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.120 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.121 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:50:11.121 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431574+0000", 2024-09-14T04:50:11.121 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.121 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.121 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.121 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:50:11.121 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431575+0000", 2024-09-14T04:50:11.121 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.2600000000000001e-07 2024-09-14T04:50:11.121 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.121 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.121 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:50:11.121 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431579+0000", 2024-09-14T04:50:11.121 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5249999999999999e-06 2024-09-14T04:50:11.122 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.122 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.122 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:50:11.122 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431591+0000", 2024-09-14T04:50:11.122 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1314999999999999e-05 2024-09-14T04:50:11.122 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.122 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.122 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:50:11.122 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431598+0000", 2024-09-14T04:50:11.122 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.2420000000000003e-06 2024-09-14T04:50:11.122 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.122 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.122 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.123 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.447535+0000", 2024-09-14T04:50:11.123 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.015937337999999999 2024-09-14T04:50:11.123 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.123 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.123 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:50:11.123 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.447545+0000", 2024-09-14T04:50:11.123 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.4890000000000001e-06 2024-09-14T04:50:11.123 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.123 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.123 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.123 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.325982+0000", 2024-09-14T04:50:11.123 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.87843713400000001 2024-09-14T04:50:11.124 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.124 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.124 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:50:11.124 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.325990+0000", 2024-09-14T04:50:11.124 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9400000000000002e-06 2024-09-14T04:50:11.124 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.124 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.124 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.124 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.466504+0000", 2024-09-14T04:50:11.124 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1405139260000001 2024-09-14T04:50:11.124 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.124 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.125 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:50:11.125 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.466628+0000", 2024-09-14T04:50:11.125 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012408699999999999 2024-09-14T04:50:11.125 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.125 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.125 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 4,5,6", 2024-09-14T04:50:11.125 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.466745+0000", 2024-09-14T04:50:11.125 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000117447 2024-09-14T04:50:11.125 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.125 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.125 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.125 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.476281+0000", 2024-09-14T04:50:11.125 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0095351040000000008 2024-09-14T04:50:11.126 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.126 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.126 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.126 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.476349+0000", 2024-09-14T04:50:11.126 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8342000000000004e-05 2024-09-14T04:50:11.126 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.126 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.126 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:50:11.126 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.479362+0000", 2024-09-14T04:50:11.126 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0030131540000000001 2024-09-14T04:50:11.126 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.126 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.126 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.126 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.488661+0000", 2024-09-14T04:50:11.127 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0092988040000000008 2024-09-14T04:50:11.127 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.127 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.127 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:50:11.127 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.489603+0000", 2024-09-14T04:50:11.127 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00094184399999999995 2024-09-14T04:50:11.127 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.127 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.127 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:50:11.127 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.489736+0000", 2024-09-14T04:50:11.127 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013325899999999999 2024-09-14T04:50:11.127 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.127 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:50:11.127 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.128 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.128 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.128 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:135644 54.c 54:35c60476:::benchmark_data_smithi039_264334_object9247:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:50:11.128 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:56.446718+0000", 2024-09-14T04:50:11.128 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 314.51434461999997, 2024-09-14T04:50:11.128 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.043331561, 2024-09-14T04:50:11.129 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:50:11.129 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:50:11.129 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:11.129 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:50:11.129 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:50:11.129 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:50:11.129 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 135644 2024-09-14T04:50:11.129 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.129 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:50:11.129 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.129 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:50:11.129 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.446718+0000", 2024-09-14T04:50:11.129 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.129 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.130 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.130 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:50:11.130 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.446718+0000", 2024-09-14T04:50:11.130 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.130 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.130 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.130 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:50:11.130 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.446720+0000", 2024-09-14T04:50:11.130 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3540000000000001e-06 2024-09-14T04:50:11.130 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.130 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.130 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:50:11.130 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.446724+0000", 2024-09-14T04:50:11.130 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7249999999999997e-06 2024-09-14T04:50:11.131 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.131 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.131 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:50:11.131 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.446734+0000", 2024-09-14T04:50:11.131 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.5920000000000007e-06 2024-09-14T04:50:11.131 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.131 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.131 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:50:11.131 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.446742+0000", 2024-09-14T04:50:11.131 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0520000000000002e-06 2024-09-14T04:50:11.131 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.131 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.131 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.131 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.447585+0000", 2024-09-14T04:50:11.131 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00084300300000000005 2024-09-14T04:50:11.132 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.132 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.132 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:50:11.132 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.447597+0000", 2024-09-14T04:50:11.132 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2302e-05 2024-09-14T04:50:11.132 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.132 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.132 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.132 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.326086+0000", 2024-09-14T04:50:11.132 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.87848874099999996 2024-09-14T04:50:11.132 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.133 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.133 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:50:11.133 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.326095+0000", 2024-09-14T04:50:11.133 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.1360000000000003e-06 2024-09-14T04:50:11.133 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.133 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.133 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.133 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.469679+0000", 2024-09-14T04:50:11.133 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.143583765 2024-09-14T04:50:11.133 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.133 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.133 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:50:11.133 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.469760+0000", 2024-09-14T04:50:11.134 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0669999999999998e-05 2024-09-14T04:50:11.134 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.134 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.134 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 4,5,6", 2024-09-14T04:50:11.134 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.469836+0000", 2024-09-14T04:50:11.134 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6048000000000001e-05 2024-09-14T04:50:11.134 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.134 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.134 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:50:11.134 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.479370+0000", 2024-09-14T04:50:11.134 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.009534275 2024-09-14T04:50:11.134 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.134 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.134 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.135 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.481815+0000", 2024-09-14T04:50:11.135 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.002444786 2024-09-14T04:50:11.135 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.135 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.135 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.135 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.481869+0000", 2024-09-14T04:50:11.135 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4113000000000002e-05 2024-09-14T04:50:11.135 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.135 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.135 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.135 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.489843+0000", 2024-09-14T04:50:11.135 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0079742140000000003 2024-09-14T04:50:11.135 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.135 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.136 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:50:11.136 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.489933+0000", 2024-09-14T04:50:11.136 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.0115e-05 2024-09-14T04:50:11.136 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.136 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.136 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:50:11.136 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.490050+0000", 2024-09-14T04:50:11.136 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011667000000000001 2024-09-14T04:50:11.136 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.136 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:50:11.136 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.136 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.136 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.137 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:135655 54.c 54:3bab8a03:::benchmark_data_smithi039_264334_object9258:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:50:11.137 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:56.456620+0000", 2024-09-14T04:50:11.137 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 314.504443197, 2024-09-14T04:50:11.137 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0336829170000001, 2024-09-14T04:50:11.137 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:50:11.137 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:50:11.137 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:11.137 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:50:11.137 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:50:11.137 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:50:11.137 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 135655 2024-09-14T04:50:11.137 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.137 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:50:11.137 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.138 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:50:11.138 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.456620+0000", 2024-09-14T04:50:11.138 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.138 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.138 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.138 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:50:11.138 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.456620+0000", 2024-09-14T04:50:11.138 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.138 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.138 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.139 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:50:11.139 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.456621+0000", 2024-09-14T04:50:11.139 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7719999999999999e-06 2024-09-14T04:50:11.139 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.139 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.139 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:50:11.139 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.456628+0000", 2024-09-14T04:50:11.139 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3679999999999998e-06 2024-09-14T04:50:11.139 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.139 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.140 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:50:11.140 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.456644+0000", 2024-09-14T04:50:11.140 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6297999999999999e-05 2024-09-14T04:50:11.140 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.140 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.140 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:50:11.140 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.456652+0000", 2024-09-14T04:50:11.140 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9519999999999994e-06 2024-09-14T04:50:11.140 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.140 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.140 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.140 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.456746+0000", 2024-09-14T04:50:11.140 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.3617999999999997e-05 2024-09-14T04:50:11.141 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.141 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.141 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:50:11.141 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.456760+0000", 2024-09-14T04:50:11.141 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4185999999999999e-05 2024-09-14T04:50:11.141 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.141 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.141 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.141 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.326122+0000", 2024-09-14T04:50:11.141 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.86936199400000003 2024-09-14T04:50:11.141 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.142 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.142 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:50:11.142 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.326129+0000", 2024-09-14T04:50:11.142 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.5730000000000002e-06 2024-09-14T04:50:11.142 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.142 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.142 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.142 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.473189+0000", 2024-09-14T04:50:11.142 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.14706011 2024-09-14T04:50:11.142 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.142 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.142 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:50:11.142 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.473313+0000", 2024-09-14T04:50:11.142 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000123662 2024-09-14T04:50:11.143 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.143 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.143 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 4,5,6", 2024-09-14T04:50:11.143 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.473443+0000", 2024-09-14T04:50:11.143 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012988400000000001 2024-09-14T04:50:11.143 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.143 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.143 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:50:11.143 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.481877+0000", 2024-09-14T04:50:11.143 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0084343060000000008 2024-09-14T04:50:11.143 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.143 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.143 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.143 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.481911+0000", 2024-09-14T04:50:11.143 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.3553000000000002e-05 2024-09-14T04:50:11.144 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.144 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.144 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.144 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.489798+0000", 2024-09-14T04:50:11.144 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0078867659999999999 2024-09-14T04:50:11.144 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.144 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.144 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.144 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.490128+0000", 2024-09-14T04:50:11.144 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00033039400000000002 2024-09-14T04:50:11.144 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.144 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.144 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:50:11.144 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.490211+0000", 2024-09-14T04:50:11.145 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.2700000000000004e-05 2024-09-14T04:50:11.145 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.145 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.145 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:50:11.145 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.490303+0000", 2024-09-14T04:50:11.145 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.1780999999999994e-05 2024-09-14T04:50:11.145 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.145 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:50:11.145 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.145 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.145 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.145 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45632.0:104260 55.e 55:78317f85:::benchmark_data_smithi039_297155_object104259:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1440)", 2024-09-14T04:50:11.146 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:49:10.237868+0000", 2024-09-14T04:50:11.146 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 60.723195191000002, 2024-09-14T04:50:11.146 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.382777745, 2024-09-14T04:50:11.146 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:50:11.146 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:50:11.146 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:11.146 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:50:11.146 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45632", 2024-09-14T04:50:11.146 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:11.146 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 104260 2024-09-14T04:50:11.146 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.146 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:50:11.146 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.147 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:50:11.147 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.237868+0000", 2024-09-14T04:50:11.147 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.147 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.147 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.147 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:50:11.147 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.237868+0000", 2024-09-14T04:50:11.147 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.147 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.147 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.147 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:50:11.147 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.237869+0000", 2024-09-14T04:50:11.147 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5120000000000001e-06 2024-09-14T04:50:11.147 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.148 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.148 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:50:11.148 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.237893+0000", 2024-09-14T04:50:11.148 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4287999999999999e-05 2024-09-14T04:50:11.148 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.148 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.148 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:50:11.148 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.237909+0000", 2024-09-14T04:50:11.148 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5637999999999998e-05 2024-09-14T04:50:11.148 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.148 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.148 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:50:11.148 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.237918+0000", 2024-09-14T04:50:11.149 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.8300000000000002e-06 2024-09-14T04:50:11.149 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.149 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.149 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.149 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.238288+0000", 2024-09-14T04:50:11.149 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000369725 2024-09-14T04:50:11.149 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.149 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.149 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:50:11.150 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.238451+0000", 2024-09-14T04:50:11.150 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000163859 2024-09-14T04:50:11.150 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.150 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.150 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:50:11.150 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.238603+0000", 2024-09-14T04:50:11.150 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015151699999999999 2024-09-14T04:50:11.150 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.150 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.150 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:50:11.150 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.242832+0000", 2024-09-14T04:50:11.150 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0042295029999999999 2024-09-14T04:50:11.150 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.151 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.151 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.151 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.243043+0000", 2024-09-14T04:50:11.151 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00021068500000000001 2024-09-14T04:50:11.151 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.151 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.151 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.151 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.620547+0000", 2024-09-14T04:50:11.151 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.37750365000000002 2024-09-14T04:50:11.151 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.151 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.151 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:50:11.151 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.620594+0000", 2024-09-14T04:50:11.151 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6699999999999997e-05 2024-09-14T04:50:11.151 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.152 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.152 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:50:11.152 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.620645+0000", 2024-09-14T04:50:11.152 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1838000000000001e-05 2024-09-14T04:50:11.152 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.152 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:50:11.152 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.152 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.152 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.152 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45632.0:104268 55.0 55:03b0a67b:::benchmark_data_smithi039_297155_object104267:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1440)", 2024-09-14T04:50:11.152 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:49:10.326212+0000", 2024-09-14T04:50:11.152 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 60.634850303, 2024-09-14T04:50:11.152 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.31380261300000001, 2024-09-14T04:50:11.153 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:50:11.153 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:50:11.153 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:11.153 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:50:11.153 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45632", 2024-09-14T04:50:11.153 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:11.153 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 104268 2024-09-14T04:50:11.153 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.153 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:50:11.154 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.154 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:50:11.154 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.326212+0000", 2024-09-14T04:50:11.154 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.154 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.154 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.154 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:50:11.154 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.326212+0000", 2024-09-14T04:50:11.154 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:50:11.154 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.154 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.155 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:50:11.155 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.326215+0000", 2024-09-14T04:50:11.155 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0260000000000002e-06 2024-09-14T04:50:11.155 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.155 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.155 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:50:11.155 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.326240+0000", 2024-09-14T04:50:11.155 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.5248e-05 2024-09-14T04:50:11.155 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.155 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.155 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:50:11.155 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.326257+0000", 2024-09-14T04:50:11.155 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7141000000000001e-05 2024-09-14T04:50:11.155 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.156 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.156 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:50:11.156 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.326265+0000", 2024-09-14T04:50:11.156 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.1370000000000002e-06 2024-09-14T04:50:11.156 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.156 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.156 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:50:11.156 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.326327+0000", 2024-09-14T04:50:11.156 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1739e-05 2024-09-14T04:50:11.156 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.156 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.156 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:50:11.156 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.326513+0000", 2024-09-14T04:50:11.156 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00018593900000000001 2024-09-14T04:50:11.157 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.157 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.157 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:50:11.157 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.326671+0000", 2024-09-14T04:50:11.157 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015817399999999999 2024-09-14T04:50:11.157 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.157 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.157 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:50:11.157 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.331777+0000", 2024-09-14T04:50:11.157 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0051062290000000003 2024-09-14T04:50:11.157 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.158 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.158 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.158 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.334887+0000", 2024-09-14T04:50:11.158 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0031100759999999998 2024-09-14T04:50:11.158 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.158 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.158 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:11.158 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.639904+0000", 2024-09-14T04:50:11.158 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.30501712800000003 2024-09-14T04:50:11.158 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.158 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.158 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:50:11.158 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.639962+0000", 2024-09-14T04:50:11.158 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7657e-05 2024-09-14T04:50:11.159 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:50:11.159 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:50:11.159 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:50:11.159 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.640015+0000", 2024-09-14T04:50:11.159 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3118999999999997e-05 2024-09-14T04:50:11.159 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.159 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:50:11.159 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.159 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:50:11.159 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:50:11.159 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:50:11.194 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:11.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:11.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:11.296 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:11.333 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:11.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:11.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:11.435 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:11.464 INFO:teuthology.orchestra.run.smithi039.stderr:set pool 55 pgp_num to 16 2024-09-14T04:50:11.469 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:11.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:11.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:11.571 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:11.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:11.617 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:11.618 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:11.718 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:11.753 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:11.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:11.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:11.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:11 smithi129 ceph-mon[101316]: pgmap v2634: 99 pgs: 99 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 916 op/s 2024-09-14T04:50:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:11 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1964204977' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"} : dispatch 2024-09-14T04:50:11.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:11 smithi039 ceph-mon[106470]: pgmap v2634: 99 pgs: 99 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 916 op/s 2024-09-14T04:50:11.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:11 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1964204977' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"} : dispatch 2024-09-14T04:50:11.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:11 smithi039 ceph-mon[111235]: pgmap v2634: 99 pgs: 99 active+clean; 5.5 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 916 op/s 2024-09-14T04:50:11.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:11 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1964204977' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"} : dispatch 2024-09-14T04:50:11.855 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:11.892 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:11.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:11.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:11.994 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:12.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:12.031 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:12.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:12.132 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:12.177 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:12.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:12.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:12.279 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:12.311 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:12.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:12.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:12.413 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:12.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:12.455 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:12.455 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:12.556 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:12.592 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:12.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:12.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:12.694 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:12.726 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:12.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:12.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:12.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:12 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1964204977' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]': finished 2024-09-14T04:50:12.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:12 smithi129 ceph-mon[101316]: osdmap e1449: 8 total, 8 up, 8 in 2024-09-14T04:50:12.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:12 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1964204977' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]': finished 2024-09-14T04:50:12.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:12 smithi039 ceph-mon[106470]: osdmap e1449: 8 total, 8 up, 8 in 2024-09-14T04:50:12.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:12 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1964204977' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pgp_num", "val": "16"}]': finished 2024-09-14T04:50:12.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:12 smithi039 ceph-mon[111235]: osdmap e1449: 8 total, 8 up, 8 in 2024-09-14T04:50:12.828 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:12.863 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:12.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:12.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:12.964 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:12.999 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:12.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:13.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:13.069 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-14T04:50:13.101 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:13.132 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:13.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:13.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:13.233 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:13.275 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:13.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:13.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:13.325 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:50:13.326 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:50:13.356 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:50:13.356 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:50:13.377 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:13.386 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:50:13.386 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:50:13.414 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:50:13.415 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:50:13.425 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:13.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:13.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:13.439 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:50:13.439 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:50:13.459 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:50:13.466 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:50:13.466 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:50:13.494 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:50:13.494 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:50:13.500 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:13 smithi129 ceph-mon[101316]: pgmap v2636: 99 pgs: 99 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 942 op/s 2024-09-14T04:50:13.500 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:13 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4148097454' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:50:13.527 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:13.540 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:50:13.540 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:50:13.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:13.570 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:13.570 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:13.614 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:13 smithi039 ceph-mon[106470]: pgmap v2636: 99 pgs: 99 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 942 op/s 2024-09-14T04:50:13.614 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:13 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4148097454' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:50:13.615 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:13 smithi039 ceph-mon[111235]: pgmap v2636: 99 pgs: 99 active+clean; 5.4 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 942 op/s 2024-09-14T04:50:13.615 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:13 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4148097454' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:50:13.671 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:13.710 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:13.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:13.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:13.812 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:13.846 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:13.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:13.847 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:13.948 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:13.975 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:13.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:13.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:14.078 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:14.117 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:14.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:14.118 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:14.218 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:14.251 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:14.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:14.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:14.353 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:14.386 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:14.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:14.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:14.488 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:14.524 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:14.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:14.526 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:14.626 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:14.654 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:14.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:14.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:14.756 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:14.793 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:14.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:14.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:14.895 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:14.931 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:14.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:14.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:15.033 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:15.055 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:15.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:15.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:15.157 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:15.192 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:15.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:15.193 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:15.294 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:15.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:15.331 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:15.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:15.432 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:15.476 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:15.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:15.477 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:15.578 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:15.586 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:15 smithi039 ceph-mon[106470]: pgmap v2637: 99 pgs: 99 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 951 op/s 2024-09-14T04:50:15.586 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:15 smithi039 ceph-mon[111235]: pgmap v2637: 99 pgs: 99 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 951 op/s 2024-09-14T04:50:15.615 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:15.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:15.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:15.717 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:15.757 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:15.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:15.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:15.767 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:15 smithi129 ceph-mon[101316]: pgmap v2637: 99 pgs: 99 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 951 op/s 2024-09-14T04:50:15.860 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:15.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:15.897 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:15.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:15.999 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:16.037 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:16.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:16.038 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:16.139 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:16.168 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:16.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:16.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:16.251 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:50:16.251 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:50:16.270 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:16.304 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:16.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:16.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:16.406 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:16.444 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:16.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:16.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:16.545 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:16.590 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:16.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:16.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:16.692 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:16.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:16.738 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:16.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:16.839 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:16.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:16.880 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:16.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:16.962 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-14T04:50:16.981 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:17.019 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:17.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:17.020 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:17.120 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:17.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:17.166 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:17.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:17.266 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:17.308 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:50:17.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:17.329 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:17.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:17.429 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:17.460 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:17.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:17.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:17.518 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is unset 2024-09-14T04:50:17.563 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:17.601 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:17.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:17.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:17.703 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:17.737 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:17.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:17.738 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:17.740 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:17 smithi129 ceph-mon[101316]: pgmap v2638: 99 pgs: 99 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 951 op/s 2024-09-14T04:50:17.740 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:17 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/979676085' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:50:17.740 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:17 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:50:17.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:17 smithi039 ceph-mon[106470]: pgmap v2638: 99 pgs: 99 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 951 op/s 2024-09-14T04:50:17.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:17 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/979676085' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:50:17.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:17 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:50:17.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:17 smithi039 ceph-mon[111235]: pgmap v2638: 99 pgs: 99 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 951 op/s 2024-09-14T04:50:17.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:17 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/979676085' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:50:17.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:17 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:50:17.839 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:17.876 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:17.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:17.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:17.974 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:50:17.978 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:18.067 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:18.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:18.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:18.169 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:18.201 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:18.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:18.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:18.304 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:18.327 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:18.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:18.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:18.429 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:18.451 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:18.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:18.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:18.553 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:18.575 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:18.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:18.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:18.676 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:18.716 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:18.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:18.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:18.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:18 smithi129 ceph-mon[101316]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:50:18.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:18 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:50:18.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:18 smithi129 ceph-mon[101316]: osdmap e1450: 8 total, 8 up, 8 in 2024-09-14T04:50:18.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:18 smithi039 ceph-mon[106470]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:50:18.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:18 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:50:18.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:18 smithi039 ceph-mon[106470]: osdmap e1450: 8 total, 8 up, 8 in 2024-09-14T04:50:18.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:50:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:50:18] "GET /metrics HTTP/1.1" 200 36766 "" "Prometheus/2.51.0" 2024-09-14T04:50:18.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:18 smithi039 ceph-mon[111235]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:50:18.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:18 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:50:18.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:18 smithi039 ceph-mon[111235]: osdmap e1450: 8 total, 8 up, 8 in 2024-09-14T04:50:18.819 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:18.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:18.863 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:18.863 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:18.963 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:19.003 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:19.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:19.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:19.088 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-14T04:50:19.088 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:50:19.088 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-14T04:50:19.088 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:50:19.105 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:19.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:19.170 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:19.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:19.171 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:50:19.171 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:50:19.272 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:19.305 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:19.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:19.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:19.407 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:19.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:19.447 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:19.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:19.548 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:19.586 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:19.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:19.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:19.689 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:19.752 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:19.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:19.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:19.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:19 smithi039 ceph-mon[106470]: pgmap v2640: 99 pgs: 99 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-09-14T04:50:19.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:19 smithi039 ceph-mon[111235]: pgmap v2640: 99 pgs: 99 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-09-14T04:50:19.832 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:19 smithi129 ceph-mon[101316]: pgmap v2640: 99 pgs: 99 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 703 op/s 2024-09-14T04:50:19.854 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:19.869 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-14T04:50:19.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:19.900 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:19.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:20.001 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:20.065 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:20.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:20.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:20.167 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:20.219 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:20.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:20.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:20.255 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:50:20.266 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:50:20.318 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:50:20.321 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:20.355 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:20.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:20.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:20.457 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:20.493 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:20.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:20.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:20.595 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:20.617 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:20.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:20.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:20.719 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:20.757 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:20.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:20.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:20.858 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:20.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:20.916 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:20.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:21.017 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:21.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:21.054 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:21.054 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:21.155 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:21.186 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:21.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:21.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:21.287 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:21.329 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:21.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:21.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:21.431 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:21.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:21.478 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:21.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:21.579 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:21.614 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:21.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:21.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:21.715 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:21.746 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:21.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:21.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:21.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:21 smithi039 ceph-mon[106470]: pgmap v2641: 99 pgs: 99 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 984 op/s 2024-09-14T04:50:21.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:21 smithi039 ceph-mon[111235]: pgmap v2641: 99 pgs: 99 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 984 op/s 2024-09-14T04:50:21.847 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:21.870 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:21.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:21.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:21.972 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:21.996 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:21.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:21.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:21 smithi129 ceph-mon[101316]: pgmap v2641: 99 pgs: 99 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 984 op/s 2024-09-14T04:50:22.099 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:22.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:22.135 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:22.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:22.211 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:50:22.236 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:22.279 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:22.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:22.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:22.380 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:22.401 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:22.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:22.402 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:22.503 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:22.541 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:22.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:22.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:22.643 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:22.676 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:22.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:22.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:22.778 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:22.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:50:22.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:50:22.819 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:22.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:22.820 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:22.921 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:22.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:22.959 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:22.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:23.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:50:23.060 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:23.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:23.093 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:23.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:23.194 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:23.237 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:23.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:23.238 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:23.339 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:23.378 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:23.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:23.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:23.480 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:23.519 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:23.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:23.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:23.622 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:23.644 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:23.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:23.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:23.695 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:23 smithi039 ceph-mon[106470]: pgmap v2642: 99 pgs: 99 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 892 op/s 2024-09-14T04:50:23.695 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:23 smithi039 ceph-mon[106470]: 1.3 scrub starts 2024-09-14T04:50:23.696 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:23 smithi039 ceph-mon[106470]: 1.3 scrub ok 2024-09-14T04:50:23.696 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:23 smithi039 ceph-mon[111235]: pgmap v2642: 99 pgs: 99 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 892 op/s 2024-09-14T04:50:23.696 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:23 smithi039 ceph-mon[111235]: 1.3 scrub starts 2024-09-14T04:50:23.696 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:23 smithi039 ceph-mon[111235]: 1.3 scrub ok 2024-09-14T04:50:23.728 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:50:23.728 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:50:23.729 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:50:23.729 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:50:23.729 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:50:23.729 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:50:23.745 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:50:23.745 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:50:23.747 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:23.775 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:50:23.775 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:50:23.781 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:23.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:23.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:23.808 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:50:23.808 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:50:23.841 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:50:23.841 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:50:23.878 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:50:23.878 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:50:23.883 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:23.912 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:23.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:23.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:24.014 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:24.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:23 smithi129 ceph-mon[101316]: pgmap v2642: 99 pgs: 99 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 892 op/s 2024-09-14T04:50:24.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:23 smithi129 ceph-mon[101316]: 1.3 scrub starts 2024-09-14T04:50:24.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:23 smithi129 ceph-mon[101316]: 1.3 scrub ok 2024-09-14T04:50:24.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:24.047 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:24.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:24.148 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:24.198 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:24.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:24.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:24.300 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:24.342 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:24.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:24.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:24.363 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:50:24.364 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T04:50:24.444 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:24.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:24.483 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:24.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:24.584 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:24.619 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:24.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:24.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:24.721 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:24.753 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:24.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:24.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:24.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:24 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3633098014' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:50:24.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:24 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3633098014' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:50:24.855 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:24.876 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:24.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:24.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:24.978 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:25.009 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:25.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:25.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:25.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:24 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3633098014' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:50:25.111 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:25.146 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:25.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:25.147 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:25.248 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:25.275 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:25.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:25.276 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:25.377 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:25.417 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:25.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:25.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:25.519 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:25.552 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:25.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:25.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:25.654 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:25.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:25.694 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:25.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:25.737 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:50:25.761 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":2643,"stamp":"2024-09-14T04:50:23.764737+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5016000509,"num_objects":76556,"num_object_clones":0,"num_object_copies":229668,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":76556,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":171396,"num_write_kb":7931554,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1650,"num_bytes_recovered":106109486,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":131232,"ondisk_log_size":131232,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18545484,"kb_used_data":14893324,"kb_used_omap":404,"kb_used_meta":3651691,"kb_avail":731415732,"statfs":{"total":767960285184,"available":748969709568,"internally_reserved":0,"allocated":15250763776,"data_stored":15212786131,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":414022,"internal_metadata":3739332282},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-541249958,"num_objects":-8258,"num_object_clones":0,"num_object_copies":-24774,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8258,"num_whiteouts":0,"num_read":-35,"num_read_kb":-75,"num_write":8238,"num_write_kb":-220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-1,"num_bytes_recovered":-161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002551"},"pg_stats":[{"pgid":"55.f","version":"1450'10710","reported_seq":10751,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:23.505742+0000","last_change":"2024-09-14T04:47:44.619393+0000","last_active":"2024-09-14T04:50:23.505742+0000","last_peered":"2024-09-14T04:50:23.505742+0000","last_clean":"2024-09-14T04:50:23.505742+0000","last_became_active":"2024-09-14T04:47:44.619068+0000","last_became_peered":"2024-09-14T04:47:44.619068+0000","last_unstale":"2024-09-14T04:50:23.505742+0000","last_undegraded":"2024-09-14T04:50:23.505742+0000","last_fullsized":"2024-09-14T04:50:23.505742+0000","mapping_epoch":1419,"log_start":"1427'1300","ondisk_log_start":"1427'1300","created":1419,"last_epoch_clean":1420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":9410,"log_dups_size":0,"ondisk_log_size":9410,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:24:11.584916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309985280,"num_objects":4730,"num_object_clones":0,"num_object_copies":14190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10710,"num_write_kb":494080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,7],"acting":[5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"55.e","version":"1450'10629","reported_seq":10699,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:23.522070+0000","last_change":"2024-09-14T04:49:29.627836+0000","last_active":"2024-09-14T04:50:23.522070+0000","last_peered":"2024-09-14T04:50:23.522070+0000","last_clean":"2024-09-14T04:50:23.522070+0000","last_became_active":"2024-09-14T04:49:29.616603+0000","last_became_peered":"2024-09-14T04:49:29.616603+0000","last_unstale":"2024-09-14T04:50:23.522070+0000","last_undegraded":"2024-09-14T04:50:23.522070+0000","last_fullsized":"2024-09-14T04:50:23.522070+0000","mapping_epoch":1442,"log_start":"1431'2700","ondisk_log_start":"1431'2700","created":1419,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":7929,"log_dups_size":0,"ondisk_log_size":7929,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:25:14.159610+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306905088,"num_objects":4683,"num_object_clones":0,"num_object_copies":14049,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4683,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10629,"num_write_kb":489984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"55.d","version":"1450'10810","reported_seq":10852,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:23.267709+0000","last_change":"2024-09-14T04:47:44.732561+0000","last_active":"2024-09-14T04:50:23.267709+0000","last_peered":"2024-09-14T04:50:23.267709+0000","last_clean":"2024-09-14T04:50:23.267709+0000","last_became_active":"2024-09-14T04:47:44.732146+0000","last_became_peered":"2024-09-14T04:47:44.732146+0000","last_unstale":"2024-09-14T04:50:23.267709+0000","last_undegraded":"2024-09-14T04:50:23.267709+0000","last_fullsized":"2024-09-14T04:50:23.267709+0000","mapping_epoch":1419,"log_start":"1438'4100","ondisk_log_start":"1438'4100","created":1419,"last_epoch_clean":1420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":6710,"log_dups_size":0,"ondisk_log_size":6710,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:08:21.134275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318046232,"num_objects":4854,"num_object_clones":0,"num_object_copies":14562,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4854,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10810,"num_write_kb":501185,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"55.c","version":"1450'10678","reported_seq":10719,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:22.740015+0000","last_change":"2024-09-14T04:47:44.623726+0000","last_active":"2024-09-14T04:50:22.740015+0000","last_peered":"2024-09-14T04:50:22.740015+0000","last_clean":"2024-09-14T04:50:22.740015+0000","last_became_active":"2024-09-14T04:47:44.623263+0000","last_became_peered":"2024-09-14T04:47:44.623263+0000","last_unstale":"2024-09-14T04:50:22.740015+0000","last_undegraded":"2024-09-14T04:50:22.740015+0000","last_fullsized":"2024-09-14T04:50:22.740015+0000","mapping_epoch":1419,"log_start":"1431'2900","ondisk_log_start":"1431'2900","created":1419,"last_epoch_clean":1420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":7778,"log_dups_size":0,"ondisk_log_size":7778,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:34:24.614254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":308543488,"num_objects":4708,"num_object_clones":0,"num_object_copies":14124,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4708,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10678,"num_write_kb":492352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"55.b","version":"1450'10694","reported_seq":10735,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:23.555017+0000","last_change":"2024-09-14T04:47:44.620800+0000","last_active":"2024-09-14T04:50:23.555017+0000","last_peered":"2024-09-14T04:50:23.555017+0000","last_clean":"2024-09-14T04:50:23.555017+0000","last_became_active":"2024-09-14T04:47:44.620531+0000","last_became_peered":"2024-09-14T04:47:44.620531+0000","last_unstale":"2024-09-14T04:50:23.555017+0000","last_undegraded":"2024-09-14T04:50:23.555017+0000","last_fullsized":"2024-09-14T04:50:23.555017+0000","mapping_epoch":1419,"log_start":"1431'2700","ondisk_log_start":"1431'2700","created":1419,"last_epoch_clean":1420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":7994,"log_dups_size":0,"ondisk_log_size":7994,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:11:20.790717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310116352,"num_objects":4732,"num_object_clones":0,"num_object_copies":14196,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4732,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10694,"num_write_kb":493632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"55.a","version":"1450'10690","reported_seq":10761,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:23.339030+0000","last_change":"2024-09-14T04:49:29.644757+0000","last_active":"2024-09-14T04:50:23.339030+0000","last_peered":"2024-09-14T04:50:23.339030+0000","last_clean":"2024-09-14T04:50:23.339030+0000","last_became_active":"2024-09-14T04:49:29.621926+0000","last_became_peered":"2024-09-14T04:49:29.621926+0000","last_unstale":"2024-09-14T04:50:23.339030+0000","last_undegraded":"2024-09-14T04:50:23.339030+0000","last_fullsized":"2024-09-14T04:50:23.339030+0000","mapping_epoch":1442,"log_start":"1436'3998","ondisk_log_start":"1436'3998","created":1419,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":6692,"log_dups_size":0,"ondisk_log_size":6692,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:20.701618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":313262080,"num_objects":4780,"num_object_clones":0,"num_object_copies":14340,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4780,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10690,"num_write_kb":495040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"55.9","version":"1450'10735","reported_seq":10803,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:23.480820+0000","last_change":"2024-09-14T04:49:29.640509+0000","last_active":"2024-09-14T04:50:23.480820+0000","last_peered":"2024-09-14T04:50:23.480820+0000","last_clean":"2024-09-14T04:50:23.480820+0000","last_became_active":"2024-09-14T04:49:29.621747+0000","last_became_peered":"2024-09-14T04:49:29.621747+0000","last_unstale":"2024-09-14T04:50:23.480820+0000","last_undegraded":"2024-09-14T04:50:23.480820+0000","last_fullsized":"2024-09-14T04:50:23.480820+0000","mapping_epoch":1442,"log_start":"1427'1300","ondisk_log_start":"1427'1300","created":1419,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":9435,"log_dups_size":0,"ondisk_log_size":9435,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:56:32.339858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312934400,"num_objects":4775,"num_object_clones":0,"num_object_copies":14325,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4775,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10735,"num_write_kb":496320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"55.8","version":"1450'10796","reported_seq":10837,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:23.503713+0000","last_change":"2024-09-14T04:47:44.732510+0000","last_active":"2024-09-14T04:50:23.503713+0000","last_peered":"2024-09-14T04:50:23.503713+0000","last_clean":"2024-09-14T04:50:23.503713+0000","last_became_active":"2024-09-14T04:47:44.731755+0000","last_became_peered":"2024-09-14T04:47:44.731755+0000","last_unstale":"2024-09-14T04:50:23.503713+0000","last_undegraded":"2024-09-14T04:50:23.503713+0000","last_fullsized":"2024-09-14T04:50:23.503713+0000","mapping_epoch":1419,"log_start":"1427'1400","ondisk_log_start":"1427'1400","created":1419,"last_epoch_clean":1420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":9396,"log_dups_size":0,"ondisk_log_size":9396,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:51:34.295649+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309854208,"num_objects":4728,"num_object_clones":0,"num_object_copies":14184,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4728,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10796,"num_write_kb":496768,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"55.7","version":"1450'10844","reported_seq":10883,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:23.347979+0000","last_change":"2024-09-14T04:47:44.620751+0000","last_active":"2024-09-14T04:50:23.347979+0000","last_peered":"2024-09-14T04:50:23.347979+0000","last_clean":"2024-09-14T04:50:23.347979+0000","last_became_active":"2024-09-14T04:47:44.620468+0000","last_became_peered":"2024-09-14T04:47:44.620468+0000","last_unstale":"2024-09-14T04:50:23.347979+0000","last_undegraded":"2024-09-14T04:50:23.347979+0000","last_fullsized":"2024-09-14T04:50:23.347979+0000","mapping_epoch":1419,"log_start":"1438'4100","ondisk_log_start":"1438'4100","created":1419,"last_epoch_clean":1420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":6744,"log_dups_size":0,"ondisk_log_size":6744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:14.021795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317980672,"num_objects":4852,"num_object_clones":0,"num_object_copies":14556,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4852,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10844,"num_write_kb":502272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"55.6","version":"1450'10728","reported_seq":10803,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:20.120664+0000","last_change":"2024-09-14T04:47:44.620668+0000","last_active":"2024-09-14T04:50:20.120664+0000","last_peered":"2024-09-14T04:50:20.120664+0000","last_clean":"2024-09-14T04:50:20.120664+0000","last_became_active":"2024-09-14T04:47:44.620248+0000","last_became_peered":"2024-09-14T04:47:44.620248+0000","last_unstale":"2024-09-14T04:50:20.120664+0000","last_undegraded":"2024-09-14T04:50:20.120664+0000","last_fullsized":"2024-09-14T04:50:20.120664+0000","mapping_epoch":1419,"log_start":"1427'1000","ondisk_log_start":"1427'1000","created":1419,"last_epoch_clean":1420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":9728,"log_dups_size":0,"ondisk_log_size":9728,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:38:43.241120+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327811072,"num_objects":5002,"num_object_clones":0,"num_object_copies":15006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10728,"num_write_kb":503360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"55.5","version":"1450'10453","reported_seq":10491,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:23.563522+0000","last_change":"2024-09-14T04:47:44.732608+0000","last_active":"2024-09-14T04:50:23.563522+0000","last_peered":"2024-09-14T04:50:23.563522+0000","last_clean":"2024-09-14T04:50:23.563522+0000","last_became_active":"2024-09-14T04:47:44.732055+0000","last_became_peered":"2024-09-14T04:47:44.732055+0000","last_unstale":"2024-09-14T04:50:23.563522+0000","last_undegraded":"2024-09-14T04:50:23.563522+0000","last_fullsized":"2024-09-14T04:50:23.563522+0000","mapping_epoch":1419,"log_start":"1431'2500","ondisk_log_start":"1431'2500","created":1419,"last_epoch_clean":1420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":7953,"log_dups_size":0,"ondisk_log_size":7953,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:27:27.167314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315031552,"num_objects":4807,"num_object_clones":0,"num_object_copies":14421,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4807,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10453,"num_write_kb":488320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"55.4","version":"1450'10550","reported_seq":10591,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:23.298893+0000","last_change":"2024-09-14T04:47:44.624288+0000","last_active":"2024-09-14T04:50:23.298893+0000","last_peered":"2024-09-14T04:50:23.298893+0000","last_clean":"2024-09-14T04:50:23.298893+0000","last_became_active":"2024-09-14T04:47:44.623792+0000","last_became_peered":"2024-09-14T04:47:44.623792+0000","last_unstale":"2024-09-14T04:50:23.298893+0000","last_undegraded":"2024-09-14T04:50:23.298893+0000","last_fullsized":"2024-09-14T04:50:23.298893+0000","mapping_epoch":1419,"log_start":"1436'3800","ondisk_log_start":"1436'3800","created":1419,"last_epoch_clean":1420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":6750,"log_dups_size":0,"ondisk_log_size":6750,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:02:35.092982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300023808,"num_objects":4578,"num_object_clones":0,"num_object_copies":13734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10550,"num_write_kb":484096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"55.3","version":"1450'10920","reported_seq":10995,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:21.014801+0000","last_change":"2024-09-14T04:47:44.620809+0000","last_active":"2024-09-14T04:50:21.014801+0000","last_peered":"2024-09-14T04:50:21.014801+0000","last_clean":"2024-09-14T04:50:21.014801+0000","last_became_active":"2024-09-14T04:47:44.620523+0000","last_became_peered":"2024-09-14T04:47:44.620523+0000","last_unstale":"2024-09-14T04:50:21.014801+0000","last_undegraded":"2024-09-14T04:50:21.014801+0000","last_fullsized":"2024-09-14T04:50:21.014801+0000","mapping_epoch":1419,"log_start":"1431'3000","ondisk_log_start":"1431'3000","created":1419,"last_epoch_clean":1420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":7920,"log_dups_size":0,"ondisk_log_size":7920,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:39:25.802609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326107136,"num_objects":4976,"num_object_clones":0,"num_object_copies":14928,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4976,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10920,"num_write_kb":508672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"55.2","version":"1450'10537","reported_seq":10618,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:21.044095+0000","last_change":"2024-09-14T04:47:59.382691+0000","last_active":"2024-09-14T04:50:21.044095+0000","last_peered":"2024-09-14T04:50:21.044095+0000","last_clean":"2024-09-14T04:50:21.044095+0000","last_became_active":"2024-09-14T04:47:59.382325+0000","last_became_peered":"2024-09-14T04:47:59.382325+0000","last_unstale":"2024-09-14T04:50:21.044095+0000","last_undegraded":"2024-09-14T04:50:21.044095+0000","last_fullsized":"2024-09-14T04:50:21.044095+0000","mapping_epoch":1423,"log_start":"1431'2600","ondisk_log_start":"1431'2600","created":1419,"last_epoch_clean":1424,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":7937,"log_dups_size":0,"ondisk_log_size":7937,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:51:34.837512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318832640,"num_objects":4865,"num_object_clones":0,"num_object_copies":14595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4865,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10537,"num_write_kb":492864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"55.1","version":"1450'10764","reported_seq":14057,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:23.448855+0000","last_change":"2024-09-14T04:48:37.067853+0000","last_active":"2024-09-14T04:50:23.448855+0000","last_peered":"2024-09-14T04:50:23.448855+0000","last_clean":"2024-09-14T04:50:23.448855+0000","last_became_active":"2024-09-14T04:48:37.064586+0000","last_became_peered":"2024-09-14T04:48:37.064586+0000","last_unstale":"2024-09-14T04:50:23.448855+0000","last_undegraded":"2024-09-14T04:50:23.448855+0000","last_fullsized":"2024-09-14T04:50:23.448855+0000","mapping_epoch":1437,"log_start":"1427'1600","ondisk_log_start":"1427'1600","created":1419,"last_epoch_clean":1438,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":9164,"log_dups_size":0,"ondisk_log_size":9164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:49:56.633540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":312475648,"num_objects":4768,"num_object_clones":0,"num_object_copies":14304,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4768,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10764,"num_write_kb":497024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1607,"num_bytes_recovered":105316352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"55.0","version":"1450'10794","reported_seq":10866,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:23.555052+0000","last_change":"2024-09-14T04:49:29.626446+0000","last_active":"2024-09-14T04:50:23.555052+0000","last_peered":"2024-09-14T04:50:23.555052+0000","last_clean":"2024-09-14T04:50:23.555052+0000","last_became_active":"2024-09-14T04:49:29.616617+0000","last_became_peered":"2024-09-14T04:49:29.616617+0000","last_unstale":"2024-09-14T04:50:23.555052+0000","last_undegraded":"2024-09-14T04:50:23.555052+0000","last_fullsized":"2024-09-14T04:50:23.555052+0000","mapping_epoch":1442,"log_start":"1431'2800","ondisk_log_start":"1431'2800","created":1419,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:47:43.565010+0000","last_clean_scrub_stamp":"2024-09-14T04:47:43.565010+0000","objects_scrubbed":0,"log_size":7994,"log_dups_size":0,"ondisk_log_size":7994,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:38:53.404238+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307494912,"num_objects":4692,"num_object_clones":0,"num_object_copies":14076,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4692,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10794,"num_write_kb":495552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":2190,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.520215+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T04:50:17.520215+0000","last_peered":"2024-09-14T04:50:17.520215+0000","last_clean":"2024-09-14T04:50:17.520215+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T04:50:17.520215+0000","last_undegraded":"2024-09-14T04:50:17.520215+0000","last_fullsized":"2024-09-14T04:50:17.520215+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":60,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.466087+0000","last_change":"2024-09-14T04:44:57.325109+0000","last_active":"2024-09-14T04:50:11.466087+0000","last_peered":"2024-09-14T04:50:11.466087+0000","last_clean":"2024-09-14T04:50:11.466087+0000","last_became_active":"2024-09-14T04:44:57.324791+0000","last_became_peered":"2024-09-14T04:44:57.324791+0000","last_unstale":"2024-09-14T04:50:11.466087+0000","last_undegraded":"2024-09-14T04:50:11.466087+0000","last_fullsized":"2024-09-14T04:50:11.466087+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:03.593275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":29,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.470203+0000","last_change":"2024-09-14T04:49:29.628435+0000","last_active":"2024-09-14T04:50:11.470203+0000","last_peered":"2024-09-14T04:50:11.470203+0000","last_clean":"2024-09-14T04:50:11.470203+0000","last_became_active":"2024-09-14T04:49:29.628081+0000","last_became_peered":"2024-09-14T04:49:29.628081+0000","last_unstale":"2024-09-14T04:50:11.470203+0000","last_undegraded":"2024-09-14T04:50:11.470203+0000","last_fullsized":"2024-09-14T04:50:11.470203+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:01.520334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":811,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.526514+0000","last_change":"2024-09-14T04:43:13.355281+0000","last_active":"2024-09-14T04:50:17.526514+0000","last_peered":"2024-09-14T04:50:17.526514+0000","last_clean":"2024-09-14T04:50:17.526514+0000","last_became_active":"2024-09-14T04:43:13.354967+0000","last_became_peered":"2024-09-14T04:43:13.354967+0000","last_unstale":"2024-09-14T04:50:17.526514+0000","last_undegraded":"2024-09-14T04:50:17.526514+0000","last_fullsized":"2024-09-14T04:50:17.526514+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:23:19.332075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1438,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.471874+0000","last_change":"2024-09-14T04:44:57.342918+0000","last_active":"2024-09-14T04:50:11.471874+0000","last_peered":"2024-09-14T04:50:11.471874+0000","last_clean":"2024-09-14T04:50:11.471874+0000","last_became_active":"2024-09-14T04:44:57.329550+0000","last_became_peered":"2024-09-14T04:44:57.329550+0000","last_unstale":"2024-09-14T04:50:11.471874+0000","last_undegraded":"2024-09-14T04:50:11.471874+0000","last_fullsized":"2024-09-14T04:50:11.471874+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:52:52.435893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":60,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.470423+0000","last_change":"2024-09-14T04:44:57.325537+0000","last_active":"2024-09-14T04:50:11.470423+0000","last_peered":"2024-09-14T04:50:11.470423+0000","last_clean":"2024-09-14T04:50:11.470423+0000","last_became_active":"2024-09-14T04:44:57.325057+0000","last_became_peered":"2024-09-14T04:44:57.325057+0000","last_unstale":"2024-09-14T04:50:11.470423+0000","last_undegraded":"2024-09-14T04:50:11.470423+0000","last_fullsized":"2024-09-14T04:50:11.470423+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:43:23.166708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1453,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.472188+0000","last_change":"2024-09-14T04:44:57.337033+0000","last_active":"2024-09-14T04:50:11.472188+0000","last_peered":"2024-09-14T04:50:11.472188+0000","last_clean":"2024-09-14T04:50:11.472188+0000","last_became_active":"2024-09-14T04:44:57.328437+0000","last_became_peered":"2024-09-14T04:44:57.328437+0000","last_unstale":"2024-09-14T04:50:11.472188+0000","last_undegraded":"2024-09-14T04:50:11.472188+0000","last_fullsized":"2024-09-14T04:50:11.472188+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.109086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":60,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.470600+0000","last_change":"2024-09-14T04:44:57.332243+0000","last_active":"2024-09-14T04:50:11.470600+0000","last_peered":"2024-09-14T04:50:11.470600+0000","last_clean":"2024-09-14T04:50:11.470600+0000","last_became_active":"2024-09-14T04:44:57.330413+0000","last_became_peered":"2024-09-14T04:44:57.330413+0000","last_unstale":"2024-09-14T04:50:11.470600+0000","last_undegraded":"2024-09-14T04:50:11.470600+0000","last_fullsized":"2024-09-14T04:50:11.470600+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:25:01.338549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":46,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.519665+0000","last_change":"2024-09-14T04:48:29.862929+0000","last_active":"2024-09-14T04:50:17.519665+0000","last_peered":"2024-09-14T04:50:17.519665+0000","last_clean":"2024-09-14T04:50:17.519665+0000","last_became_active":"2024-09-14T04:48:29.862581+0000","last_became_peered":"2024-09-14T04:48:29.862581+0000","last_unstale":"2024-09-14T04:50:17.519665+0000","last_undegraded":"2024-09-14T04:50:17.519665+0000","last_fullsized":"2024-09-14T04:50:17.519665+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:16:43.060088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2186,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.521744+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T04:50:17.521744+0000","last_peered":"2024-09-14T04:50:17.521744+0000","last_clean":"2024-09-14T04:50:17.521744+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T04:50:17.521744+0000","last_undegraded":"2024-09-14T04:50:17.521744+0000","last_fullsized":"2024-09-14T04:50:17.521744+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":60,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.470406+0000","last_change":"2024-09-14T04:44:57.325392+0000","last_active":"2024-09-14T04:50:11.470406+0000","last_peered":"2024-09-14T04:50:11.470406+0000","last_clean":"2024-09-14T04:50:11.470406+0000","last_became_active":"2024-09-14T04:44:57.324624+0000","last_became_peered":"2024-09-14T04:44:57.324624+0000","last_unstale":"2024-09-14T04:50:11.470406+0000","last_undegraded":"2024-09-14T04:50:11.470406+0000","last_fullsized":"2024-09-14T04:50:11.470406+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:33:18.810199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2154,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.511577+0000","last_change":"2024-09-14T04:44:57.329113+0000","last_active":"2024-09-14T04:50:17.511577+0000","last_peered":"2024-09-14T04:50:17.511577+0000","last_clean":"2024-09-14T04:50:17.511577+0000","last_became_active":"2024-09-14T04:44:57.328727+0000","last_became_peered":"2024-09-14T04:44:57.328727+0000","last_unstale":"2024-09-14T04:50:17.511577+0000","last_undegraded":"2024-09-14T04:50:17.511577+0000","last_fullsized":"2024-09-14T04:50:17.511577+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:34.939874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":2171,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.511266+0000","last_change":"2024-09-14T04:44:57.329277+0000","last_active":"2024-09-14T04:50:17.511266+0000","last_peered":"2024-09-14T04:50:17.511266+0000","last_clean":"2024-09-14T04:50:17.511266+0000","last_became_active":"2024-09-14T04:44:57.328804+0000","last_became_peered":"2024-09-14T04:44:57.328804+0000","last_unstale":"2024-09-14T04:50:17.511266+0000","last_undegraded":"2024-09-14T04:50:17.511266+0000","last_fullsized":"2024-09-14T04:50:17.511266+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:42:46.055240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2185,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.520398+0000","last_change":"2024-09-14T04:07:58.561490+0000","last_active":"2024-09-14T04:50:17.520398+0000","last_peered":"2024-09-14T04:50:17.520398+0000","last_clean":"2024-09-14T04:50:17.520398+0000","last_became_active":"2024-09-14T04:07:58.559841+0000","last_became_peered":"2024-09-14T04:07:58.559841+0000","last_unstale":"2024-09-14T04:50:17.520398+0000","last_undegraded":"2024-09-14T04:50:17.520398+0000","last_fullsized":"2024-09-14T04:50:17.520398+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:27:03.795368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2167,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.521959+0000","last_change":"2024-09-14T04:43:13.357092+0000","last_active":"2024-09-14T04:50:17.521959+0000","last_peered":"2024-09-14T04:50:17.521959+0000","last_clean":"2024-09-14T04:50:17.521959+0000","last_became_active":"2024-09-14T04:43:13.356694+0000","last_became_peered":"2024-09-14T04:43:13.356694+0000","last_unstale":"2024-09-14T04:50:17.521959+0000","last_undegraded":"2024-09-14T04:50:17.521959+0000","last_fullsized":"2024-09-14T04:50:17.521959+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:11.167574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1431,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.470130+0000","last_change":"2024-09-14T04:07:21.720797+0000","last_active":"2024-09-14T04:50:11.470130+0000","last_peered":"2024-09-14T04:50:11.470130+0000","last_clean":"2024-09-14T04:50:11.470130+0000","last_became_active":"2024-09-14T04:07:21.719331+0000","last_became_peered":"2024-09-14T04:07:21.719331+0000","last_unstale":"2024-09-14T04:50:11.470130+0000","last_undegraded":"2024-09-14T04:50:11.470130+0000","last_fullsized":"2024-09-14T04:50:11.470130+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:35.404023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":79,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:13.511351+0000","last_change":"2024-09-14T04:43:11.641514+0000","last_active":"2024-09-14T04:50:13.511351+0000","last_peered":"2024-09-14T04:50:13.511351+0000","last_clean":"2024-09-14T04:50:13.511351+0000","last_became_active":"2024-09-14T04:43:11.641151+0000","last_became_peered":"2024-09-14T04:43:11.641151+0000","last_unstale":"2024-09-14T04:50:13.511351+0000","last_undegraded":"2024-09-14T04:50:13.511351+0000","last_fullsized":"2024-09-14T04:50:13.511351+0000","mapping_epoch":1384,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:06:45.045430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2179,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.520372+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T04:50:17.520372+0000","last_peered":"2024-09-14T04:50:17.520372+0000","last_clean":"2024-09-14T04:50:17.520372+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T04:50:17.520372+0000","last_undegraded":"2024-09-14T04:50:17.520372+0000","last_fullsized":"2024-09-14T04:50:17.520372+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":659,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.474338+0000","last_change":"2024-09-14T04:49:29.618477+0000","last_active":"2024-09-14T04:50:11.474338+0000","last_peered":"2024-09-14T04:50:11.474338+0000","last_clean":"2024-09-14T04:50:11.474338+0000","last_became_active":"2024-09-14T04:49:29.618210+0000","last_became_peered":"2024-09-14T04:49:29.618210+0000","last_unstale":"2024-09-14T04:50:11.474338+0000","last_undegraded":"2024-09-14T04:50:11.474338+0000","last_fullsized":"2024-09-14T04:50:11.474338+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:13:23.895512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1437,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.472321+0000","last_change":"2024-09-14T04:42:49.938505+0000","last_active":"2024-09-14T04:50:11.472321+0000","last_peered":"2024-09-14T04:50:11.472321+0000","last_clean":"2024-09-14T04:50:11.472321+0000","last_became_active":"2024-09-14T04:42:49.938180+0000","last_became_peered":"2024-09-14T04:42:49.938180+0000","last_unstale":"2024-09-14T04:50:11.472321+0000","last_undegraded":"2024-09-14T04:50:11.472321+0000","last_fullsized":"2024-09-14T04:50:11.472321+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:05.929479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1456,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.474826+0000","last_change":"2024-09-14T04:44:57.346582+0000","last_active":"2024-09-14T04:50:11.474826+0000","last_peered":"2024-09-14T04:50:11.474826+0000","last_clean":"2024-09-14T04:50:11.474826+0000","last_became_active":"2024-09-14T04:44:57.330446+0000","last_became_peered":"2024-09-14T04:44:57.330446+0000","last_unstale":"2024-09-14T04:50:11.474826+0000","last_undegraded":"2024-09-14T04:50:11.474826+0000","last_fullsized":"2024-09-14T04:50:11.474826+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:29.238585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":120,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.512648+0000","last_change":"2024-09-14T04:44:14.480667+0000","last_active":"2024-09-14T04:50:17.512648+0000","last_peered":"2024-09-14T04:50:17.512648+0000","last_clean":"2024-09-14T04:50:17.512648+0000","last_became_active":"2024-09-14T04:44:14.480365+0000","last_became_peered":"2024-09-14T04:44:14.480365+0000","last_unstale":"2024-09-14T04:50:17.512648+0000","last_undegraded":"2024-09-14T04:50:17.512648+0000","last_fullsized":"2024-09-14T04:50:17.512648+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:00.611771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":70,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.471113+0000","last_change":"2024-09-14T04:44:57.341426+0000","last_active":"2024-09-14T04:50:11.471113+0000","last_peered":"2024-09-14T04:50:11.471113+0000","last_clean":"2024-09-14T04:50:11.471113+0000","last_became_active":"2024-09-14T04:44:57.328763+0000","last_became_peered":"2024-09-14T04:44:57.328763+0000","last_unstale":"2024-09-14T04:50:11.471113+0000","last_undegraded":"2024-09-14T04:50:11.471113+0000","last_fullsized":"2024-09-14T04:50:11.471113+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:34:40.066761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":60,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.470805+0000","last_change":"2024-09-14T04:44:57.331337+0000","last_active":"2024-09-14T04:50:11.470805+0000","last_peered":"2024-09-14T04:50:11.470805+0000","last_clean":"2024-09-14T04:50:11.470805+0000","last_became_active":"2024-09-14T04:44:57.329336+0000","last_became_peered":"2024-09-14T04:44:57.329336+0000","last_unstale":"2024-09-14T04:50:11.470805+0000","last_undegraded":"2024-09-14T04:50:11.470805+0000","last_fullsized":"2024-09-14T04:50:11.470805+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:52:35.980869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"349'149","reported_seq":648,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.472053+0000","last_change":"2024-09-14T04:49:30.085274+0000","last_active":"2024-09-14T04:50:11.472053+0000","last_peered":"2024-09-14T04:50:11.472053+0000","last_clean":"2024-09-14T04:50:11.472053+0000","last_became_active":"2024-09-14T04:49:29.623295+0000","last_became_peered":"2024-09-14T04:49:29.623295+0000","last_unstale":"2024-09-14T04:50:11.472053+0000","last_undegraded":"2024-09-14T04:50:11.472053+0000","last_fullsized":"2024-09-14T04:50:11.472053+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:49:30.085089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:49:30.085089+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:10.012963+0000","scrub_duration":0.0071928790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":60,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.470279+0000","last_change":"2024-09-14T04:44:57.329870+0000","last_active":"2024-09-14T04:50:11.470279+0000","last_peered":"2024-09-14T04:50:11.470279+0000","last_clean":"2024-09-14T04:50:11.470279+0000","last_became_active":"2024-09-14T04:44:57.329353+0000","last_became_peered":"2024-09-14T04:44:57.329353+0000","last_unstale":"2024-09-14T04:50:11.470279+0000","last_undegraded":"2024-09-14T04:50:11.470279+0000","last_fullsized":"2024-09-14T04:50:11.470279+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:46.432181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1435,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.535432+0000","last_change":"2024-09-14T04:07:21.721242+0000","last_active":"2024-09-14T04:50:17.535432+0000","last_peered":"2024-09-14T04:50:17.535432+0000","last_clean":"2024-09-14T04:50:17.535432+0000","last_became_active":"2024-09-14T04:07:21.720084+0000","last_became_peered":"2024-09-14T04:07:21.720084+0000","last_unstale":"2024-09-14T04:50:17.535432+0000","last_undegraded":"2024-09-14T04:50:17.535432+0000","last_fullsized":"2024-09-14T04:50:17.535432+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:58:41.308643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":2217,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.520309+0000","last_change":"2024-09-14T04:44:57.344614+0000","last_active":"2024-09-14T04:50:17.520309+0000","last_peered":"2024-09-14T04:50:17.520309+0000","last_clean":"2024-09-14T04:50:17.520309+0000","last_became_active":"2024-09-14T04:44:57.332256+0000","last_became_peered":"2024-09-14T04:44:57.332256+0000","last_unstale":"2024-09-14T04:50:17.520309+0000","last_undegraded":"2024-09-14T04:50:17.520309+0000","last_fullsized":"2024-09-14T04:50:17.520309+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:23.438417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"349'149","reported_seq":818,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.472985+0000","last_change":"2024-09-14T04:45:00.331854+0000","last_active":"2024-09-14T04:50:11.472985+0000","last_peered":"2024-09-14T04:50:11.472985+0000","last_clean":"2024-09-14T04:50:11.472985+0000","last_became_active":"2024-09-14T04:45:00.331460+0000","last_became_peered":"2024-09-14T04:45:00.331460+0000","last_unstale":"2024-09-14T04:50:11.472985+0000","last_undegraded":"2024-09-14T04:50:11.472985+0000","last_fullsized":"2024-09-14T04:50:11.472985+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.567535+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.567535+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:45.095729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":649,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.474336+0000","last_change":"2024-09-14T04:49:29.616831+0000","last_active":"2024-09-14T04:50:11.474336+0000","last_peered":"2024-09-14T04:50:11.474336+0000","last_clean":"2024-09-14T04:50:11.474336+0000","last_became_active":"2024-09-14T04:49:29.616546+0000","last_became_peered":"2024-09-14T04:49:29.616546+0000","last_unstale":"2024-09-14T04:50:11.474336+0000","last_undegraded":"2024-09-14T04:50:11.474336+0000","last_fullsized":"2024-09-14T04:50:11.474336+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:27.882929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":172,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.512372+0000","last_change":"2024-09-14T04:44:57.330746+0000","last_active":"2024-09-14T04:50:17.512372+0000","last_peered":"2024-09-14T04:50:17.512372+0000","last_clean":"2024-09-14T04:50:17.512372+0000","last_became_active":"2024-09-14T04:44:57.330327+0000","last_became_peered":"2024-09-14T04:44:57.330327+0000","last_unstale":"2024-09-14T04:50:17.512372+0000","last_undegraded":"2024-09-14T04:50:17.512372+0000","last_fullsized":"2024-09-14T04:50:17.512372+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:53.122771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":100,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.475594+0000","last_change":"2024-09-14T04:43:13.355296+0000","last_active":"2024-09-14T04:50:11.475594+0000","last_peered":"2024-09-14T04:50:11.475594+0000","last_clean":"2024-09-14T04:50:11.475594+0000","last_became_active":"2024-09-14T04:43:13.354950+0000","last_became_peered":"2024-09-14T04:43:13.354950+0000","last_unstale":"2024-09-14T04:50:11.475594+0000","last_undegraded":"2024-09-14T04:50:11.475594+0000","last_fullsized":"2024-09-14T04:50:11.475594+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:20:08.968818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"349'149","reported_seq":757,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.521222+0000","last_change":"2024-09-14T04:41:39.471393+0000","last_active":"2024-09-14T04:50:17.521222+0000","last_peered":"2024-09-14T04:50:17.521222+0000","last_clean":"2024-09-14T04:50:17.521222+0000","last_became_active":"2024-09-14T04:41:36.931768+0000","last_became_peered":"2024-09-14T04:41:36.931768+0000","last_unstale":"2024-09-14T04:50:17.521222+0000","last_undegraded":"2024-09-14T04:50:17.521222+0000","last_fullsized":"2024-09-14T04:50:17.521222+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.471248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.471248+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:56.210375+0000","scrub_duration":0.021758344999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2173,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.519912+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T04:50:17.519912+0000","last_peered":"2024-09-14T04:50:17.519912+0000","last_clean":"2024-09-14T04:50:17.519912+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T04:50:17.519912+0000","last_undegraded":"2024-09-14T04:50:17.519912+0000","last_fullsized":"2024-09-14T04:50:17.519912+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1467,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.515372+0000","last_change":"2024-09-14T04:44:57.331436+0000","last_active":"2024-09-14T04:50:17.515372+0000","last_peered":"2024-09-14T04:50:17.515372+0000","last_clean":"2024-09-14T04:50:17.515372+0000","last_became_active":"2024-09-14T04:44:57.330957+0000","last_became_peered":"2024-09-14T04:44:57.330957+0000","last_unstale":"2024-09-14T04:50:17.515372+0000","last_undegraded":"2024-09-14T04:50:17.515372+0000","last_fullsized":"2024-09-14T04:50:17.515372+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:09.516688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":722,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.521585+0000","last_change":"2024-09-14T04:49:49.460401+0000","last_active":"2024-09-14T04:50:17.521585+0000","last_peered":"2024-09-14T04:50:17.521585+0000","last_clean":"2024-09-14T04:50:17.521585+0000","last_became_active":"2024-09-14T04:49:49.458530+0000","last_became_peered":"2024-09-14T04:49:49.458530+0000","last_unstale":"2024-09-14T04:50:17.521585+0000","last_undegraded":"2024-09-14T04:50:17.521585+0000","last_fullsized":"2024-09-14T04:50:17.521585+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:11:29.143239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"349'149","reported_seq":757,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.534748+0000","last_change":"2024-09-14T04:41:38.479980+0000","last_active":"2024-09-14T04:50:17.534748+0000","last_peered":"2024-09-14T04:50:17.534748+0000","last_clean":"2024-09-14T04:50:17.534748+0000","last_became_active":"2024-09-14T04:41:36.933392+0000","last_became_peered":"2024-09-14T04:41:36.933392+0000","last_unstale":"2024-09-14T04:50:17.534748+0000","last_undegraded":"2024-09-14T04:50:17.534748+0000","last_fullsized":"2024-09-14T04:50:17.534748+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:38.479887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:38.479887+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:01:26.102279+0000","scrub_duration":0.0062728319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":127,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.512565+0000","last_change":"2024-09-14T04:44:57.329983+0000","last_active":"2024-09-14T04:50:17.512565+0000","last_peered":"2024-09-14T04:50:17.512565+0000","last_clean":"2024-09-14T04:50:17.512565+0000","last_became_active":"2024-09-14T04:44:57.329597+0000","last_became_peered":"2024-09-14T04:44:57.329597+0000","last_unstale":"2024-09-14T04:50:17.512565+0000","last_undegraded":"2024-09-14T04:50:17.512565+0000","last_fullsized":"2024-09-14T04:50:17.512565+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:45.718039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":666,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.520089+0000","last_change":"2024-09-14T04:49:49.443631+0000","last_active":"2024-09-14T04:50:17.520089+0000","last_peered":"2024-09-14T04:50:17.520089+0000","last_clean":"2024-09-14T04:50:17.520089+0000","last_became_active":"2024-09-14T04:49:49.443313+0000","last_became_peered":"2024-09-14T04:49:49.443313+0000","last_unstale":"2024-09-14T04:50:17.520089+0000","last_undegraded":"2024-09-14T04:50:17.520089+0000","last_fullsized":"2024-09-14T04:50:17.520089+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:06:54.562292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1439,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.472025+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T04:50:11.472025+0000","last_peered":"2024-09-14T04:50:11.472025+0000","last_clean":"2024-09-14T04:50:11.472025+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T04:50:11.472025+0000","last_undegraded":"2024-09-14T04:50:11.472025+0000","last_fullsized":"2024-09-14T04:50:11.472025+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"349'149","reported_seq":749,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.475126+0000","last_change":"2024-09-14T04:41:39.012977+0000","last_active":"2024-09-14T04:50:11.475126+0000","last_peered":"2024-09-14T04:50:11.475126+0000","last_clean":"2024-09-14T04:50:11.475126+0000","last_became_active":"2024-09-14T04:41:36.932552+0000","last_became_peered":"2024-09-14T04:41:36.932552+0000","last_unstale":"2024-09-14T04:50:11.475126+0000","last_undegraded":"2024-09-14T04:50:11.475126+0000","last_fullsized":"2024-09-14T04:50:11.475126+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.012879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.012879+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:16:40.080262+0000","scrub_duration":0.0051721620000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":758,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.521829+0000","last_change":"2024-09-14T04:49:49.459590+0000","last_active":"2024-09-14T04:50:17.521829+0000","last_peered":"2024-09-14T04:50:17.521829+0000","last_clean":"2024-09-14T04:50:17.521829+0000","last_became_active":"2024-09-14T04:49:49.456263+0000","last_became_peered":"2024-09-14T04:49:49.456263+0000","last_unstale":"2024-09-14T04:50:17.521829+0000","last_undegraded":"2024-09-14T04:50:17.521829+0000","last_fullsized":"2024-09-14T04:50:17.521829+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:20:44.666805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":60,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.470733+0000","last_change":"2024-09-14T04:44:57.329652+0000","last_active":"2024-09-14T04:50:11.470733+0000","last_peered":"2024-09-14T04:50:11.470733+0000","last_clean":"2024-09-14T04:50:11.470733+0000","last_became_active":"2024-09-14T04:44:57.328888+0000","last_became_peered":"2024-09-14T04:44:57.328888+0000","last_unstale":"2024-09-14T04:50:11.470733+0000","last_undegraded":"2024-09-14T04:50:11.470733+0000","last_fullsized":"2024-09-14T04:50:11.470733+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:10.170068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1438,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:13.518995+0000","last_change":"2024-09-14T04:06:46.875340+0000","last_active":"2024-09-14T04:50:13.518995+0000","last_peered":"2024-09-14T04:50:13.518995+0000","last_clean":"2024-09-14T04:50:13.518995+0000","last_became_active":"2024-09-14T04:06:46.874552+0000","last_became_peered":"2024-09-14T04:06:46.874552+0000","last_unstale":"2024-09-14T04:50:13.518995+0000","last_undegraded":"2024-09-14T04:50:13.518995+0000","last_fullsized":"2024-09-14T04:50:13.518995+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:52.257357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"349'149","reported_seq":1819,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.477200+0000","last_change":"2024-09-14T04:45:00.346627+0000","last_active":"2024-09-14T04:50:11.477200+0000","last_peered":"2024-09-14T04:50:11.477200+0000","last_clean":"2024-09-14T04:50:11.477200+0000","last_became_active":"2024-09-14T04:45:00.346286+0000","last_became_peered":"2024-09-14T04:45:00.346286+0000","last_unstale":"2024-09-14T04:50:11.477200+0000","last_undegraded":"2024-09-14T04:50:11.477200+0000","last_fullsized":"2024-09-14T04:50:11.477200+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:50.784648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:50.784648+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:07:58.441372+0000","scrub_duration":0.01141873,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":120,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.511767+0000","last_change":"2024-09-14T04:44:14.489235+0000","last_active":"2024-09-14T04:50:17.511767+0000","last_peered":"2024-09-14T04:50:17.511767+0000","last_clean":"2024-09-14T04:50:17.511767+0000","last_became_active":"2024-09-14T04:44:14.488950+0000","last_became_peered":"2024-09-14T04:44:14.488950+0000","last_unstale":"2024-09-14T04:50:17.511767+0000","last_undegraded":"2024-09-14T04:50:17.511767+0000","last_fullsized":"2024-09-14T04:50:17.511767+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:54.441144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":676,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.521390+0000","last_change":"2024-09-14T04:49:49.456317+0000","last_active":"2024-09-14T04:50:17.521390+0000","last_peered":"2024-09-14T04:50:17.521390+0000","last_clean":"2024-09-14T04:50:17.521390+0000","last_became_active":"2024-09-14T04:49:49.456029+0000","last_became_peered":"2024-09-14T04:49:49.456029+0000","last_unstale":"2024-09-14T04:50:17.521390+0000","last_undegraded":"2024-09-14T04:50:17.521390+0000","last_fullsized":"2024-09-14T04:50:17.521390+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:46:05.319676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2193,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.522099+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T04:50:17.522099+0000","last_peered":"2024-09-14T04:50:17.522099+0000","last_clean":"2024-09-14T04:50:17.522099+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T04:50:17.522099+0000","last_undegraded":"2024-09-14T04:50:17.522099+0000","last_fullsized":"2024-09-14T04:50:17.522099+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"349'149","reported_seq":643,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:22.002602+0000","last_change":"2024-09-14T04:50:22.002798+0000","last_active":"2024-09-14T04:50:22.002602+0000","last_peered":"2024-09-14T04:50:22.002602+0000","last_clean":"2024-09-14T04:50:22.002602+0000","last_became_active":"2024-09-14T04:49:49.448278+0000","last_became_peered":"2024-09-14T04:49:49.448278+0000","last_unstale":"2024-09-14T04:50:22.002602+0000","last_undegraded":"2024-09-14T04:50:22.002602+0000","last_fullsized":"2024-09-14T04:50:22.002602+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:50:22.002568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:50:22.002568+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:01:57.489223+0000","scrub_duration":0.010888326,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2194,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.520483+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T04:50:17.520483+0000","last_peered":"2024-09-14T04:50:17.520483+0000","last_clean":"2024-09-14T04:50:17.520483+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T04:50:17.520483+0000","last_undegraded":"2024-09-14T04:50:17.520483+0000","last_fullsized":"2024-09-14T04:50:17.520483+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2201,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.525333+0000","last_change":"2024-09-14T04:44:57.335783+0000","last_active":"2024-09-14T04:50:17.525333+0000","last_peered":"2024-09-14T04:50:17.525333+0000","last_clean":"2024-09-14T04:50:17.525333+0000","last_became_active":"2024-09-14T04:44:57.332780+0000","last_became_peered":"2024-09-14T04:44:57.332780+0000","last_unstale":"2024-09-14T04:50:17.525333+0000","last_undegraded":"2024-09-14T04:50:17.525333+0000","last_fullsized":"2024-09-14T04:50:17.525333+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:57.457593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1468,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.474427+0000","last_change":"2024-09-14T04:42:49.939790+0000","last_active":"2024-09-14T04:50:11.474427+0000","last_peered":"2024-09-14T04:50:11.474427+0000","last_clean":"2024-09-14T04:50:11.474427+0000","last_became_active":"2024-09-14T04:42:49.939591+0000","last_became_peered":"2024-09-14T04:42:49.939591+0000","last_unstale":"2024-09-14T04:50:11.474427+0000","last_undegraded":"2024-09-14T04:50:11.474427+0000","last_fullsized":"2024-09-14T04:50:11.474427+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:23:16.790593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1431,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.472838+0000","last_change":"2024-09-14T04:44:57.326008+0000","last_active":"2024-09-14T04:50:11.472838+0000","last_peered":"2024-09-14T04:50:11.472838+0000","last_clean":"2024-09-14T04:50:11.472838+0000","last_became_active":"2024-09-14T04:44:57.325617+0000","last_became_peered":"2024-09-14T04:44:57.325617+0000","last_unstale":"2024-09-14T04:50:11.472838+0000","last_undegraded":"2024-09-14T04:50:11.472838+0000","last_fullsized":"2024-09-14T04:50:11.472838+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:15.637508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":720,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.471895+0000","last_change":"2024-09-14T04:43:13.354846+0000","last_active":"2024-09-14T04:50:11.471895+0000","last_peered":"2024-09-14T04:50:11.471895+0000","last_clean":"2024-09-14T04:50:11.471895+0000","last_became_active":"2024-09-14T04:43:13.354515+0000","last_became_peered":"2024-09-14T04:43:13.354515+0000","last_unstale":"2024-09-14T04:50:11.471895+0000","last_undegraded":"2024-09-14T04:50:11.471895+0000","last_fullsized":"2024-09-14T04:50:11.471895+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:17.889341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1437,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:13.519025+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T04:50:13.519025+0000","last_peered":"2024-09-14T04:50:13.519025+0000","last_clean":"2024-09-14T04:50:13.519025+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T04:50:13.519025+0000","last_undegraded":"2024-09-14T04:50:13.519025+0000","last_fullsized":"2024-09-14T04:50:13.519025+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":1774,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.474246+0000","last_change":"2024-09-14T04:41:30.640651+0000","last_active":"2024-09-14T04:50:11.474246+0000","last_peered":"2024-09-14T04:50:11.474246+0000","last_clean":"2024-09-14T04:50:11.474246+0000","last_became_active":"2024-09-14T04:41:29.532466+0000","last_became_peered":"2024-09-14T04:41:29.532466+0000","last_unstale":"2024-09-14T04:50:11.474246+0000","last_undegraded":"2024-09-14T04:50:11.474246+0000","last_fullsized":"2024-09-14T04:50:11.474246+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":0,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:30.640520+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:30.640520+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:27.776724+0000","scrub_duration":0.0028653139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"349'149","reported_seq":749,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.508343+0000","last_change":"2024-09-14T04:41:39.978080+0000","last_active":"2024-09-14T04:50:17.508343+0000","last_peered":"2024-09-14T04:50:17.508343+0000","last_clean":"2024-09-14T04:50:17.508343+0000","last_became_active":"2024-09-14T04:41:36.929279+0000","last_became_peered":"2024-09-14T04:41:36.929279+0000","last_unstale":"2024-09-14T04:50:17.508343+0000","last_undegraded":"2024-09-14T04:50:17.508343+0000","last_fullsized":"2024-09-14T04:50:17.508343+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.977999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.977999+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:44:06.383827+0000","scrub_duration":0.00192122,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":54,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.521319+0000","last_change":"2024-09-14T04:48:11.040133+0000","last_active":"2024-09-14T04:50:17.521319+0000","last_peered":"2024-09-14T04:50:17.521319+0000","last_clean":"2024-09-14T04:50:17.521319+0000","last_became_active":"2024-09-14T04:48:11.039787+0000","last_became_peered":"2024-09-14T04:48:11.039787+0000","last_unstale":"2024-09-14T04:50:17.521319+0000","last_undegraded":"2024-09-14T04:50:17.521319+0000","last_fullsized":"2024-09-14T04:50:17.521319+0000","mapping_epoch":1428,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:53.520672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2041,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.511727+0000","last_change":"2024-09-14T04:49:29.616443+0000","last_active":"2024-09-14T04:50:17.511727+0000","last_peered":"2024-09-14T04:50:17.511727+0000","last_clean":"2024-09-14T04:50:17.511727+0000","last_became_active":"2024-09-14T04:49:29.616066+0000","last_became_peered":"2024-09-14T04:49:29.616066+0000","last_unstale":"2024-09-14T04:50:17.511727+0000","last_undegraded":"2024-09-14T04:50:17.511727+0000","last_fullsized":"2024-09-14T04:50:17.511727+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:09:15.897957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":60,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.471254+0000","last_change":"2024-09-14T04:44:57.326724+0000","last_active":"2024-09-14T04:50:11.471254+0000","last_peered":"2024-09-14T04:50:11.471254+0000","last_clean":"2024-09-14T04:50:11.471254+0000","last_became_active":"2024-09-14T04:44:57.326191+0000","last_became_peered":"2024-09-14T04:44:57.326191+0000","last_unstale":"2024-09-14T04:50:11.471254+0000","last_undegraded":"2024-09-14T04:50:11.471254+0000","last_fullsized":"2024-09-14T04:50:11.471254+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:46.060269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":60,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.470054+0000","last_change":"2024-09-14T04:44:57.328528+0000","last_active":"2024-09-14T04:50:11.470054+0000","last_peered":"2024-09-14T04:50:11.470054+0000","last_clean":"2024-09-14T04:50:11.470054+0000","last_became_active":"2024-09-14T04:44:57.327739+0000","last_became_peered":"2024-09-14T04:44:57.327739+0000","last_unstale":"2024-09-14T04:50:11.470054+0000","last_undegraded":"2024-09-14T04:50:11.470054+0000","last_fullsized":"2024-09-14T04:50:11.470054+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:04:58.294690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"349'149","reported_seq":764,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.473133+0000","last_change":"2024-09-14T04:48:32.059177+0000","last_active":"2024-09-14T04:50:11.473133+0000","last_peered":"2024-09-14T04:50:11.473133+0000","last_clean":"2024-09-14T04:50:11.473133+0000","last_became_active":"2024-09-14T04:48:32.058741+0000","last_became_peered":"2024-09-14T04:48:32.058741+0000","last_unstale":"2024-09-14T04:50:11.473133+0000","last_undegraded":"2024-09-14T04:50:11.473133+0000","last_fullsized":"2024-09-14T04:50:11.473133+0000","mapping_epoch":1435,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1436,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:56.738636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:56.738636+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:20.932276+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":60,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.472252+0000","last_change":"2024-09-14T04:44:57.323436+0000","last_active":"2024-09-14T04:50:11.472252+0000","last_peered":"2024-09-14T04:50:11.472252+0000","last_clean":"2024-09-14T04:50:11.472252+0000","last_became_active":"2024-09-14T04:44:57.323022+0000","last_became_peered":"2024-09-14T04:44:57.323022+0000","last_unstale":"2024-09-14T04:50:11.472252+0000","last_undegraded":"2024-09-14T04:50:11.472252+0000","last_fullsized":"2024-09-14T04:50:11.472252+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:22.775261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1466,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.469835+0000","last_change":"2024-09-14T04:44:57.322357+0000","last_active":"2024-09-14T04:50:11.469835+0000","last_peered":"2024-09-14T04:50:11.469835+0000","last_clean":"2024-09-14T04:50:11.469835+0000","last_became_active":"2024-09-14T04:44:57.321990+0000","last_became_peered":"2024-09-14T04:44:57.321990+0000","last_unstale":"2024-09-14T04:50:11.469835+0000","last_undegraded":"2024-09-14T04:50:11.469835+0000","last_fullsized":"2024-09-14T04:50:11.469835+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:09.658008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"349'149","reported_seq":1791,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:13.352563+0000","last_change":"2024-09-14T04:41:53.858076+0000","last_active":"2024-09-14T04:50:13.352563+0000","last_peered":"2024-09-14T04:50:13.352563+0000","last_clean":"2024-09-14T04:50:13.352563+0000","last_became_active":"2024-09-14T04:41:36.931585+0000","last_became_peered":"2024-09-14T04:41:36.931585+0000","last_unstale":"2024-09-14T04:50:13.352563+0000","last_undegraded":"2024-09-14T04:50:13.352563+0000","last_fullsized":"2024-09-14T04:50:13.352563+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:53.857924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:53.857924+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:10:20.658986+0000","scrub_duration":0.0022355919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":60,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.471023+0000","last_change":"2024-09-14T04:44:57.329198+0000","last_active":"2024-09-14T04:50:11.471023+0000","last_peered":"2024-09-14T04:50:11.471023+0000","last_clean":"2024-09-14T04:50:11.471023+0000","last_became_active":"2024-09-14T04:44:57.327754+0000","last_became_peered":"2024-09-14T04:44:57.327754+0000","last_unstale":"2024-09-14T04:50:11.471023+0000","last_undegraded":"2024-09-14T04:50:11.471023+0000","last_fullsized":"2024-09-14T04:50:11.471023+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:10:27.944476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":178,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.519990+0000","last_change":"2024-09-14T04:44:57.348687+0000","last_active":"2024-09-14T04:50:17.519990+0000","last_peered":"2024-09-14T04:50:17.519990+0000","last_clean":"2024-09-14T04:50:17.519990+0000","last_became_active":"2024-09-14T04:44:57.332845+0000","last_became_peered":"2024-09-14T04:44:57.332845+0000","last_unstale":"2024-09-14T04:50:17.519990+0000","last_undegraded":"2024-09-14T04:50:17.519990+0000","last_fullsized":"2024-09-14T04:50:17.519990+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:12:18.762737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2200,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.519784+0000","last_change":"2024-09-14T04:44:57.333468+0000","last_active":"2024-09-14T04:50:17.519784+0000","last_peered":"2024-09-14T04:50:17.519784+0000","last_clean":"2024-09-14T04:50:17.519784+0000","last_became_active":"2024-09-14T04:44:57.332626+0000","last_became_peered":"2024-09-14T04:44:57.332626+0000","last_unstale":"2024-09-14T04:50:17.519784+0000","last_undegraded":"2024-09-14T04:50:17.519784+0000","last_fullsized":"2024-09-14T04:50:17.519784+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:57:26.366438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1422,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.472258+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T04:50:11.472258+0000","last_peered":"2024-09-14T04:50:11.472258+0000","last_clean":"2024-09-14T04:50:11.472258+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T04:50:11.472258+0000","last_undegraded":"2024-09-14T04:50:11.472258+0000","last_fullsized":"2024-09-14T04:50:11.472258+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1421,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.469706+0000","last_change":"2024-09-14T04:06:46.869868+0000","last_active":"2024-09-14T04:50:11.469706+0000","last_peered":"2024-09-14T04:50:11.469706+0000","last_clean":"2024-09-14T04:50:11.469706+0000","last_became_active":"2024-09-14T04:06:46.869126+0000","last_became_peered":"2024-09-14T04:06:46.869126+0000","last_unstale":"2024-09-14T04:50:11.469706+0000","last_undegraded":"2024-09-14T04:50:11.469706+0000","last_fullsized":"2024-09-14T04:50:11.469706+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:41.150138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1405,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.474954+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T04:50:11.474954+0000","last_peered":"2024-09-14T04:50:11.474954+0000","last_clean":"2024-09-14T04:50:11.474954+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T04:50:11.474954+0000","last_undegraded":"2024-09-14T04:50:11.474954+0000","last_fullsized":"2024-09-14T04:50:11.474954+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":60,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.470413+0000","last_change":"2024-09-14T04:44:57.332112+0000","last_active":"2024-09-14T04:50:11.470413+0000","last_peered":"2024-09-14T04:50:11.470413+0000","last_clean":"2024-09-14T04:50:11.470413+0000","last_became_active":"2024-09-14T04:44:57.331575+0000","last_became_peered":"2024-09-14T04:44:57.331575+0000","last_unstale":"2024-09-14T04:50:11.470413+0000","last_undegraded":"2024-09-14T04:50:11.470413+0000","last_fullsized":"2024-09-14T04:50:11.470413+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:50:00.391283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1453,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.475063+0000","last_change":"2024-09-14T04:44:57.351460+0000","last_active":"2024-09-14T04:50:11.475063+0000","last_peered":"2024-09-14T04:50:11.475063+0000","last_clean":"2024-09-14T04:50:11.475063+0000","last_became_active":"2024-09-14T04:44:57.334662+0000","last_became_peered":"2024-09-14T04:44:57.334662+0000","last_unstale":"2024-09-14T04:50:11.475063+0000","last_undegraded":"2024-09-14T04:50:11.475063+0000","last_fullsized":"2024-09-14T04:50:11.475063+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:02.399062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1407,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.473343+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T04:50:11.473343+0000","last_peered":"2024-09-14T04:50:11.473343+0000","last_clean":"2024-09-14T04:50:11.473343+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T04:50:11.473343+0000","last_undegraded":"2024-09-14T04:50:11.473343+0000","last_fullsized":"2024-09-14T04:50:11.473343+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1465,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.475540+0000","last_change":"2024-09-14T04:44:57.349690+0000","last_active":"2024-09-14T04:50:11.475540+0000","last_peered":"2024-09-14T04:50:11.475540+0000","last_clean":"2024-09-14T04:50:11.475540+0000","last_became_active":"2024-09-14T04:44:57.333579+0000","last_became_peered":"2024-09-14T04:44:57.333579+0000","last_unstale":"2024-09-14T04:50:11.475540+0000","last_undegraded":"2024-09-14T04:50:11.475540+0000","last_fullsized":"2024-09-14T04:50:11.475540+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:17:55.265141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1504,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.469553+0000","last_change":"2024-09-14T04:48:29.893179+0000","last_active":"2024-09-14T04:50:11.469553+0000","last_peered":"2024-09-14T04:50:11.469553+0000","last_clean":"2024-09-14T04:50:11.469553+0000","last_became_active":"2024-09-14T04:48:29.864724+0000","last_became_peered":"2024-09-14T04:48:29.864724+0000","last_unstale":"2024-09-14T04:50:11.469553+0000","last_undegraded":"2024-09-14T04:50:11.469553+0000","last_fullsized":"2024-09-14T04:50:11.469553+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:19:15.120740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":658,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.475475+0000","last_change":"2024-09-14T04:49:29.616761+0000","last_active":"2024-09-14T04:50:11.475475+0000","last_peered":"2024-09-14T04:50:11.475475+0000","last_clean":"2024-09-14T04:50:11.475475+0000","last_became_active":"2024-09-14T04:49:29.616072+0000","last_became_peered":"2024-09-14T04:49:29.616072+0000","last_unstale":"2024-09-14T04:50:11.475475+0000","last_undegraded":"2024-09-14T04:50:11.475475+0000","last_fullsized":"2024-09-14T04:50:11.475475+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:49:52.358949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1427,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.474578+0000","last_change":"2024-09-14T04:44:57.332511+0000","last_active":"2024-09-14T04:50:11.474578+0000","last_peered":"2024-09-14T04:50:11.474578+0000","last_clean":"2024-09-14T04:50:11.474578+0000","last_became_active":"2024-09-14T04:44:57.332166+0000","last_became_peered":"2024-09-14T04:44:57.332166+0000","last_unstale":"2024-09-14T04:50:11.474578+0000","last_undegraded":"2024-09-14T04:50:11.474578+0000","last_fullsized":"2024-09-14T04:50:11.474578+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:53.984639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2180,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.520229+0000","last_change":"2024-09-14T04:07:58.560880+0000","last_active":"2024-09-14T04:50:17.520229+0000","last_peered":"2024-09-14T04:50:17.520229+0000","last_clean":"2024-09-14T04:50:17.520229+0000","last_became_active":"2024-09-14T04:07:58.560132+0000","last_became_peered":"2024-09-14T04:07:58.560132+0000","last_unstale":"2024-09-14T04:50:17.520229+0000","last_undegraded":"2024-09-14T04:50:17.520229+0000","last_fullsized":"2024-09-14T04:50:17.520229+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:46:00.287050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2180,"reported_epoch":1450,"state":"active+clean","last_fresh":"2024-09-14T04:50:17.519841+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T04:50:17.519841+0000","last_peered":"2024-09-14T04:50:17.519841+0000","last_clean":"2024-09-14T04:50:17.519841+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T04:50:17.519841+0000","last_undegraded":"2024-09-14T04:50:17.519841+0000","last_fullsized":"2024-09-14T04:50:17.519841+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1419,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.471953+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T04:50:11.471953+0000","last_peered":"2024-09-14T04:50:11.471953+0000","last_clean":"2024-09-14T04:50:11.471953+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T04:50:11.471953+0000","last_undegraded":"2024-09-14T04:50:11.471953+0000","last_fullsized":"2024-09-14T04:50:11.471953+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1456,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.474836+0000","last_change":"2024-09-14T04:44:57.339993+0000","last_active":"2024-09-14T04:50:11.474836+0000","last_peered":"2024-09-14T04:50:11.474836+0000","last_clean":"2024-09-14T04:50:11.474836+0000","last_became_active":"2024-09-14T04:44:57.330472+0000","last_became_peered":"2024-09-14T04:44:57.330472+0000","last_unstale":"2024-09-14T04:50:11.474836+0000","last_undegraded":"2024-09-14T04:50:11.474836+0000","last_fullsized":"2024-09-14T04:50:11.474836+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:19:27.046988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2004,"reported_epoch":1449,"state":"active+clean","last_fresh":"2024-09-14T04:50:11.475394+0000","last_change":"2024-09-14T04:49:49.449687+0000","last_active":"2024-09-14T04:50:11.475394+0000","last_peered":"2024-09-14T04:50:11.475394+0000","last_clean":"2024-09-14T04:50:11.475394+0000","last_became_active":"2024-09-14T04:49:49.449144+0000","last_became_peered":"2024-09-14T04:49:49.449144+0000","last_unstale":"2024-09-14T04:50:11.475394+0000","last_undegraded":"2024-09-14T04:50:11.475394+0000","last_fullsized":"2024-09-14T04:50:11.475394+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:59:06.779730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":55,"num_pg":16,"stat_sum":{"num_bytes":5015404568,"num_objects":76530,"num_object_clones":0,"num_object_copies":229590,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":76530,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":171332,"num_write_kb":7931521,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1607,"num_bytes_recovered":105316352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15093280768,"data_stored":15093268552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":129534,"ondisk_log_size":129534,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":5070,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4250,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201993,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3908816,"kb_used_data":3368328,"kb_used_omap":1,"kb_used_meta":540478,"kb_avail":89836336,"statfs":{"total":95995035648,"available":91992408064,"internally_reserved":0,"allocated":3449167872,"data_stored":3444408350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":553449931},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463629,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2374980,"kb_used_data":1838588,"kb_used_omap":1,"kb_used_meta":536382,"kb_avail":91370172,"statfs":{"total":95995035648,"available":93563056128,"internally_reserved":0,"allocated":1882714112,"data_stored":1877966727,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":549255623},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725266,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2086208,"kb_used_data":1545720,"kb_used_omap":1,"kb_used_meta":540478,"kb_avail":91658944,"statfs":{"total":95995035648,"available":93858758656,"internally_reserved":0,"allocated":1582817280,"data_stored":1578070361,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":553449926},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":1392,"seq":5978594476112,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1085264,"kb_used_data":936008,"kb_used_omap":117,"kb_used_meta":149130,"kb_avail":92659888,"statfs":{"total":95995035648,"available":94883725312,"internally_reserved":0,"allocated":958472192,"data_stored":953713222,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120638,"internal_metadata":152709314},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772075007,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2750460,"kb_used_data":2150580,"kb_used_omap":71,"kb_used_meta":599800,"kb_avail":90994692,"statfs":{"total":95995035648,"available":93178564608,"internally_reserved":0,"allocated":2202193920,"data_stored":2197451150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":614196150},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707303942,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2562684,"kb_used_data":2215476,"kb_used_omap":68,"kb_used_meta":347131,"kb_avail":91182468,"statfs":{"total":95995035648,"available":93370847232,"internally_reserved":0,"allocated":2268647424,"data_stored":2263904558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":355462352},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872729101,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2070836,"kb_used_data":1593196,"kb_used_omap":67,"kb_used_meta":477564,"kb_avail":91674316,"statfs":{"total":95995035648,"available":93874499584,"internally_reserved":0,"allocated":1631432704,"data_stored":1626696889,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":489025893},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056152,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1706236,"kb_used_data":1245428,"kb_used_omap":74,"kb_used_meta":460725,"kb_avail":92038916,"statfs":{"total":95995035648,"available":94247849984,"internally_reserved":0,"allocated":1275318272,"data_stored":1270574874,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":471783093},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1901,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":55,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1256198144,"data_stored":1256194072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":55,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1607729152,"data_stored":1607729152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":55,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2254307328,"data_stored":2254307328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":55,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2172911616,"data_stored":2172911616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":55,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":942673920,"data_stored":942669848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":55,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1567358976,"data_stored":1567358976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":55,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1858994176,"data_stored":1858994176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":55,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":3433107456,"data_stored":3433103384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:50:25.767 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pgp_num 11 2024-09-14T04:50:25.794 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:25.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:25.825 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:25.826 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:25.881 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:25 smithi039 ceph-mon[106470]: pgmap v2643: 99 pgs: 99 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2024-09-14T04:50:25.881 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:25 smithi039 ceph-mon[106470]: from='client.35838 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:50:25.881 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:25 smithi039 ceph-mon[111235]: pgmap v2643: 99 pgs: 99 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2024-09-14T04:50:25.881 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:25 smithi039 ceph-mon[111235]: from='client.35838 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:50:25.926 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:25.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:25.963 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:25.963 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:25 smithi129 ceph-mon[101316]: pgmap v2643: 99 pgs: 99 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 823 op/s 2024-09-14T04:50:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:25 smithi129 ceph-mon[101316]: from='client.35838 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:50:26.064 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:26.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:26.112 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:26.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:26.213 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:26.247 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:26.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:26.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:26.348 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:26.387 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:26.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:26.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:26.489 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:26.541 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:26.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:26.542 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:26.610 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:50:26.643 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:26.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:26.687 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:26.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:26.788 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:26.789 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:50:26.790 INFO:teuthology.orchestra.run.smithi129.stdout: "size": 20, 2024-09-14T04:50:26.790 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 600, 2024-09-14T04:50:26.790 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [ 2024-09-14T04:50:26.790 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.790 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:235830 53.2 53:42936e73:::benchmark_data_smithi039_224949_object105737:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1357)", 2024-09-14T04:50:26.790 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:10.328754+0000", 2024-09-14T04:50:26.791 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 496.45557605400001, 2024-09-14T04:50:26.791 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0652573279999999, 2024-09-14T04:50:26.791 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:26.791 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "reached pg", 2024-09-14T04:50:26.791 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:26.791 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:50:26.791 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:50:26.791 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 235830 2024-09-14T04:50:26.791 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.791 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:26.791 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.791 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:26.792 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.328754+0000", 2024-09-14T04:50:26.792 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.792 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.792 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.792 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:26.792 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.328754+0000", 2024-09-14T04:50:26.792 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.792 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.792 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.792 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:26.792 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.328753+0000", 2024-09-14T04:50:26.792 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:50:26.792 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.793 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.793 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:26.793 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.328759+0000", 2024-09-14T04:50:26.793 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9109999999999998e-06 2024-09-14T04:50:26.793 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.793 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.793 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:26.793 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.328774+0000", 2024-09-14T04:50:26.793 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5123000000000001e-05 2024-09-14T04:50:26.793 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.793 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.793 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:26.794 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.328782+0000", 2024-09-14T04:50:26.794 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.7030000000000006e-06 2024-09-14T04:50:26.794 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.794 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.794 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.794 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.328828+0000", 2024-09-14T04:50:26.794 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6276000000000002e-05 2024-09-14T04:50:26.794 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.794 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.794 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:26.794 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.328972+0000", 2024-09-14T04:50:26.794 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00014447 2024-09-14T04:50:26.794 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.794 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.794 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,6", 2024-09-14T04:50:26.794 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.329103+0000", 2024-09-14T04:50:26.795 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000130748 2024-09-14T04:50:26.795 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.795 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.795 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.795 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.336426+0000", 2024-09-14T04:50:26.795 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0073225010000000004 2024-09-14T04:50:26.795 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.795 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.795 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:26.795 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.336456+0000", 2024-09-14T04:50:26.795 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.9944999999999999e-05 2024-09-14T04:50:26.795 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.795 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.795 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.796 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.341020+0000", 2024-09-14T04:50:26.796 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0045638450000000004 2024-09-14T04:50:26.796 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.796 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.796 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:50:26.796 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.341029+0000", 2024-09-14T04:50:26.796 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.9830000000000002e-06 2024-09-14T04:50:26.796 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.796 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.796 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.796 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.393943+0000", 2024-09-14T04:50:26.796 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.05291465 2024-09-14T04:50:26.796 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.796 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.796 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:26.797 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.394012+0000", 2024-09-14T04:50:26.797 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8505999999999998e-05 2024-09-14T04:50:26.797 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.797 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:26.797 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.797 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.797 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.797 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:235833 53.2 53:4ef9278c:::benchmark_data_smithi039_224949_object105740:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1357)", 2024-09-14T04:50:26.797 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:10.330442+0000", 2024-09-14T04:50:26.797 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 496.45388836299998, 2024-09-14T04:50:26.797 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.064089794, 2024-09-14T04:50:26.798 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:26.798 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "reached pg", 2024-09-14T04:50:26.798 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:26.798 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:50:26.798 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:50:26.798 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 235833 2024-09-14T04:50:26.798 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.798 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:26.798 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.799 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:26.799 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.330442+0000", 2024-09-14T04:50:26.799 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.799 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.799 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.799 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:26.799 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.330442+0000", 2024-09-14T04:50:26.799 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.799 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.799 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.799 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:26.799 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.330440+0000", 2024-09-14T04:50:26.799 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:50:26.799 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.800 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.800 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:26.800 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.330450+0000", 2024-09-14T04:50:26.800 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.1099999999999992e-06 2024-09-14T04:50:26.800 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.800 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.800 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:26.800 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.330466+0000", 2024-09-14T04:50:26.801 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.6412000000000001e-05 2024-09-14T04:50:26.801 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.801 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.801 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:26.801 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.330474+0000", 2024-09-14T04:50:26.801 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.9470000000000003e-06 2024-09-14T04:50:26.801 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.801 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.801 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.801 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.332953+0000", 2024-09-14T04:50:26.801 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0024788100000000001 2024-09-14T04:50:26.801 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.802 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.802 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:26.802 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.333089+0000", 2024-09-14T04:50:26.802 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000136079 2024-09-14T04:50:26.802 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.802 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.802 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,6", 2024-09-14T04:50:26.802 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.333224+0000", 2024-09-14T04:50:26.802 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013539100000000001 2024-09-14T04:50:26.802 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.802 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.802 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.802 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.336444+0000", 2024-09-14T04:50:26.803 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0032200359999999999 2024-09-14T04:50:26.803 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.803 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.803 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:26.803 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.336540+0000", 2024-09-14T04:50:26.803 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.5446999999999996e-05 2024-09-14T04:50:26.803 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.803 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.803 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.803 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.341067+0000", 2024-09-14T04:50:26.803 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0045271699999999996 2024-09-14T04:50:26.803 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.803 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.804 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:50:26.804 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.341075+0000", 2024-09-14T04:50:26.804 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.4780000000000008e-06 2024-09-14T04:50:26.804 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.804 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.804 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.804 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.394428+0000", 2024-09-14T04:50:26.804 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0533524380000001 2024-09-14T04:50:26.804 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.804 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.804 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:26.804 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.394532+0000", 2024-09-14T04:50:26.805 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010402400000000001 2024-09-14T04:50:26.805 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.805 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:26.805 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.805 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.805 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.805 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:235848 53.2 53:4dde4333:::benchmark_data_smithi039_224949_object105755:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1358)", 2024-09-14T04:50:26.805 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:10.351232+0000", 2024-09-14T04:50:26.805 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 496.43309807399999, 2024-09-14T04:50:26.805 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0509252659999999, 2024-09-14T04:50:26.805 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:26.805 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:26.805 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:26.806 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:50:26.806 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:50:26.806 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 235848 2024-09-14T04:50:26.806 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.806 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:26.806 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.806 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:26.806 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.351232+0000", 2024-09-14T04:50:26.806 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.806 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.807 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.807 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:26.807 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.351232+0000", 2024-09-14T04:50:26.807 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.807 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.807 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.807 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:26.807 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.351231+0000", 2024-09-14T04:50:26.807 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:50:26.807 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.807 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.807 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:26.807 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.351238+0000", 2024-09-14T04:50:26.808 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8179999999999998e-06 2024-09-14T04:50:26.808 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.808 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.808 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:26.808 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.351251+0000", 2024-09-14T04:50:26.808 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3655000000000001e-05 2024-09-14T04:50:26.808 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.808 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.808 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:26.808 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.351259+0000", 2024-09-14T04:50:26.808 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.8860000000000004e-06 2024-09-14T04:50:26.808 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.808 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.809 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.809 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.351297+0000", 2024-09-14T04:50:26.809 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7657999999999998e-05 2024-09-14T04:50:26.809 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.809 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.809 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:50:26.809 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.351303+0000", 2024-09-14T04:50:26.809 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.2870000000000001e-06 2024-09-14T04:50:26.809 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.809 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.809 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.809 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.394558+0000", 2024-09-14T04:50:26.809 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0432548580000001 2024-09-14T04:50:26.810 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.810 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.810 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:26.810 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.394663+0000", 2024-09-14T04:50:26.810 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010546499999999999 2024-09-14T04:50:26.810 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.810 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.810 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,6", 2024-09-14T04:50:26.810 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.394798+0000", 2024-09-14T04:50:26.810 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013500400000000001 2024-09-14T04:50:26.810 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.810 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.810 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.811 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.401893+0000", 2024-09-14T04:50:26.811 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0070940719999999999 2024-09-14T04:50:26.811 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.811 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.811 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.811 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.401993+0000", 2024-09-14T04:50:26.811 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010001300000000001 2024-09-14T04:50:26.811 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.811 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.811 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:26.812 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.402036+0000", 2024-09-14T04:50:26.812 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.3757999999999997e-05 2024-09-14T04:50:26.812 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.812 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.812 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:26.812 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.402105+0000", 2024-09-14T04:50:26.812 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8547000000000006e-05 2024-09-14T04:50:26.812 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.812 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.812 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:26.812 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.402158+0000", 2024-09-14T04:50:26.812 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2757000000000003e-05 2024-09-14T04:50:26.812 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.813 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:26.813 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.813 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.813 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.813 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:235851 53.2 53:454fe5c3:::benchmark_data_smithi039_224949_object105758:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1358)", 2024-09-14T04:50:26.813 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:10.353081+0000", 2024-09-14T04:50:26.813 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 496.431249552, 2024-09-14T04:50:26.813 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.053068511, 2024-09-14T04:50:26.813 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:26.813 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:26.813 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:26.813 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:50:26.813 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:50:26.813 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 235851 2024-09-14T04:50:26.814 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.814 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:26.814 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.814 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:26.814 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.353081+0000", 2024-09-14T04:50:26.814 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.814 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.814 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.814 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:26.814 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.353081+0000", 2024-09-14T04:50:26.814 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.815 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.815 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.815 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:26.815 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.353080+0000", 2024-09-14T04:50:26.815 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:50:26.815 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.815 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.815 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:26.815 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.353085+0000", 2024-09-14T04:50:26.815 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7550000000000002e-06 2024-09-14T04:50:26.815 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.815 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.816 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:26.816 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.353096+0000", 2024-09-14T04:50:26.816 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1043000000000001e-05 2024-09-14T04:50:26.816 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.816 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.816 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:26.816 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.353100+0000", 2024-09-14T04:50:26.816 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.578e-06 2024-09-14T04:50:26.816 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.816 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.816 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.816 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.353129+0000", 2024-09-14T04:50:26.816 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.8388999999999999e-05 2024-09-14T04:50:26.817 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.817 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.817 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:50:26.817 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.353133+0000", 2024-09-14T04:50:26.817 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7090000000000001e-06 2024-09-14T04:50:26.817 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.817 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.817 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.817 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.398294+0000", 2024-09-14T04:50:26.817 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.045161002 2024-09-14T04:50:26.817 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.817 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.817 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:26.818 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.398390+0000", 2024-09-14T04:50:26.818 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.6065999999999998e-05 2024-09-14T04:50:26.818 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.818 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.818 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,6", 2024-09-14T04:50:26.818 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.398501+0000", 2024-09-14T04:50:26.818 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000110795 2024-09-14T04:50:26.818 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.818 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.818 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.818 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.401954+0000", 2024-09-14T04:50:26.818 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0034530300000000002 2024-09-14T04:50:26.818 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.818 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.819 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.819 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.406033+0000", 2024-09-14T04:50:26.819 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00407874 2024-09-14T04:50:26.819 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.819 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.819 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:26.819 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.406043+0000", 2024-09-14T04:50:26.819 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.026e-05 2024-09-14T04:50:26.819 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.819 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.819 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:26.819 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.406090+0000", 2024-09-14T04:50:26.819 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6844000000000001e-05 2024-09-14T04:50:26.820 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.820 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.820 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:26.820 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.406149+0000", 2024-09-14T04:50:26.820 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9208e-05 2024-09-14T04:50:26.820 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.820 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:26.820 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.820 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.820 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.821 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:235895 53.2 53:4a78facd:::benchmark_data_smithi039_224949_object105802:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1358)", 2024-09-14T04:50:26.821 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:10.400551+0000", 2024-09-14T04:50:26.822 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 496.38377910299999, 2024-09-14T04:50:26.822 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.014094031, 2024-09-14T04:50:26.822 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:26.822 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:26.822 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:26.822 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:50:26.822 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:50:26.822 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 235895 2024-09-14T04:50:26.822 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.822 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:26.822 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.822 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:26.822 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.400551+0000", 2024-09-14T04:50:26.823 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.823 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.823 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.823 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:26.823 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.400551+0000", 2024-09-14T04:50:26.823 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.823 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.823 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.823 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:26.823 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.400550+0000", 2024-09-14T04:50:26.823 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:50:26.823 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.823 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.824 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:26.824 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.400555+0000", 2024-09-14T04:50:26.824 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.3070000000000002e-06 2024-09-14T04:50:26.824 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.824 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.824 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:26.824 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.400572+0000", 2024-09-14T04:50:26.824 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.7059e-05 2024-09-14T04:50:26.824 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.824 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.824 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:26.824 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.400578+0000", 2024-09-14T04:50:26.824 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.4840000000000003e-06 2024-09-14T04:50:26.824 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.825 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.825 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.825 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.400622+0000", 2024-09-14T04:50:26.825 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.4226e-05 2024-09-14T04:50:26.825 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.825 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.825 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:50:26.825 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.400644+0000", 2024-09-14T04:50:26.825 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.2010000000000001e-05 2024-09-14T04:50:26.825 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.825 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.825 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.826 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.402204+0000", 2024-09-14T04:50:26.826 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0015593460000001 2024-09-14T04:50:26.826 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.826 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.826 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:26.826 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.402316+0000", 2024-09-14T04:50:26.826 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000112893 2024-09-14T04:50:26.826 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.826 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.826 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,6", 2024-09-14T04:50:26.826 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.402441+0000", 2024-09-14T04:50:26.826 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012472500000000001 2024-09-14T04:50:26.826 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.827 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.827 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.827 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.406205+0000", 2024-09-14T04:50:26.827 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0037634309999999998 2024-09-14T04:50:26.827 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.827 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.827 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:26.827 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.409887+0000", 2024-09-14T04:50:26.827 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.003682205 2024-09-14T04:50:26.827 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.827 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.827 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.828 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.414550+0000", 2024-09-14T04:50:26.828 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0046629530000000001 2024-09-14T04:50:26.828 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.828 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.828 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:26.828 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.414600+0000", 2024-09-14T04:50:26.828 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.0538999999999998e-05 2024-09-14T04:50:26.828 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.828 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.828 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:26.828 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.414645+0000", 2024-09-14T04:50:26.829 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.5040000000000002e-05 2024-09-14T04:50:26.829 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.829 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:26.829 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.829 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.829 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.829 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:235909 53.2 53:46dbb868:::benchmark_data_smithi039_224949_object105816:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1358)", 2024-09-14T04:50:26.829 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:10.416744+0000", 2024-09-14T04:50:26.829 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 496.36758624100003, 2024-09-14T04:50:26.829 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.998072921, 2024-09-14T04:50:26.829 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:26.829 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:26.829 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:26.830 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:50:26.830 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:50:26.830 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 235909 2024-09-14T04:50:26.830 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.830 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:26.830 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.830 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:26.830 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.416744+0000", 2024-09-14T04:50:26.830 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.830 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.830 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.830 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:26.830 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.416744+0000", 2024-09-14T04:50:26.831 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.831 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.831 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.831 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:26.831 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.416743+0000", 2024-09-14T04:50:26.831 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:50:26.831 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.831 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.831 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:26.831 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.416751+0000", 2024-09-14T04:50:26.831 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.2539999999999992e-06 2024-09-14T04:50:26.831 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.831 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.831 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:26.831 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.416771+0000", 2024-09-14T04:50:26.832 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.0452e-05 2024-09-14T04:50:26.832 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.832 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.832 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:26.832 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.416779+0000", 2024-09-14T04:50:26.832 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.9060000000000002e-06 2024-09-14T04:50:26.832 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.832 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.832 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.833 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.416842+0000", 2024-09-14T04:50:26.833 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.3295999999999998e-05 2024-09-14T04:50:26.833 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.833 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.833 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:50:26.833 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.416852+0000", 2024-09-14T04:50:26.833 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.6520000000000001e-06 2024-09-14T04:50:26.833 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.833 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.833 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.833 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.406221+0000", 2024-09-14T04:50:26.833 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.98936869000000005 2024-09-14T04:50:26.833 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.833 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.833 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:26.834 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.406290+0000", 2024-09-14T04:50:26.834 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.9319000000000005e-05 2024-09-14T04:50:26.834 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.834 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.834 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,6", 2024-09-14T04:50:26.834 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.406356+0000", 2024-09-14T04:50:26.834 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.6291000000000006e-05 2024-09-14T04:50:26.834 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.834 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.834 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.834 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.414697+0000", 2024-09-14T04:50:26.834 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0083410619999999998 2024-09-14T04:50:26.834 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.834 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.834 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:26.834 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.414704+0000", 2024-09-14T04:50:26.835 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.3810000000000004e-06 2024-09-14T04:50:26.835 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.835 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.835 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.835 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.414745+0000", 2024-09-14T04:50:26.835 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1659000000000002e-05 2024-09-14T04:50:26.835 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.835 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.835 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:26.835 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.414785+0000", 2024-09-14T04:50:26.835 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9499000000000002e-05 2024-09-14T04:50:26.835 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.835 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.835 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:26.835 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.414817+0000", 2024-09-14T04:50:26.836 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.2089999999999999e-05 2024-09-14T04:50:26.836 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.836 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:26.836 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.836 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.836 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.836 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:235954 53.2 53:46a9f388:::benchmark_data_smithi039_224949_object105861:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1358)", 2024-09-14T04:50:26.836 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:10.463737+0000", 2024-09-14T04:50:26.836 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 496.320593363, 2024-09-14T04:50:26.836 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.95122717899999998, 2024-09-14T04:50:26.837 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:26.837 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:26.837 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:26.837 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:50:26.837 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:50:26.837 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 235954 2024-09-14T04:50:26.837 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.837 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:26.837 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.837 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:26.837 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.463737+0000", 2024-09-14T04:50:26.837 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.837 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.837 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.837 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:26.838 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.463737+0000", 2024-09-14T04:50:26.838 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.838 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.838 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.838 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:26.838 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.463736+0000", 2024-09-14T04:50:26.838 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:50:26.838 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.838 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.838 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:26.838 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.463743+0000", 2024-09-14T04:50:26.838 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.99e-06 2024-09-14T04:50:26.838 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.838 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.838 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:26.839 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.463754+0000", 2024-09-14T04:50:26.839 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1447999999999999e-05 2024-09-14T04:50:26.839 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.839 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.839 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:26.839 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.463759+0000", 2024-09-14T04:50:26.839 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.4210000000000003e-06 2024-09-14T04:50:26.839 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.839 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.839 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.839 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.463796+0000", 2024-09-14T04:50:26.839 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6769e-05 2024-09-14T04:50:26.839 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.839 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.839 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:50:26.840 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.463803+0000", 2024-09-14T04:50:26.840 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7850000000000003e-06 2024-09-14T04:50:26.840 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.840 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.840 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.840 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.409917+0000", 2024-09-14T04:50:26.840 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.94611380899999997 2024-09-14T04:50:26.840 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.840 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.840 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:26.840 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.410011+0000", 2024-09-14T04:50:26.840 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.4535000000000002e-05 2024-09-14T04:50:26.840 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.840 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.841 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,6", 2024-09-14T04:50:26.841 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.410115+0000", 2024-09-14T04:50:26.841 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000103816 2024-09-14T04:50:26.841 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.841 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.841 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.841 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.414850+0000", 2024-09-14T04:50:26.841 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0047353129999999997 2024-09-14T04:50:26.841 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.841 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.841 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:26.841 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.414855+0000", 2024-09-14T04:50:26.841 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.5190000000000003e-06 2024-09-14T04:50:26.841 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.841 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.842 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.842 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.414892+0000", 2024-09-14T04:50:26.842 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6872000000000002e-05 2024-09-14T04:50:26.842 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.842 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.842 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:26.842 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.414935+0000", 2024-09-14T04:50:26.842 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.2910000000000001e-05 2024-09-14T04:50:26.842 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.842 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.842 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:26.842 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.414964+0000", 2024-09-14T04:50:26.842 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.9456000000000001e-05 2024-09-14T04:50:26.842 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.843 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:26.843 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.843 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.843 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.843 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:235986 53.2 53:49605ae2:::benchmark_data_smithi039_224949_object105893:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1358)", 2024-09-14T04:50:26.843 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:10.496597+0000", 2024-09-14T04:50:26.843 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 496.28773307300003, 2024-09-14T04:50:26.844 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.92793495100000001, 2024-09-14T04:50:26.844 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:26.844 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:26.844 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:26.844 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:50:26.844 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:50:26.844 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 235986 2024-09-14T04:50:26.844 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.844 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:26.844 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.845 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:26.845 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.496597+0000", 2024-09-14T04:50:26.845 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.845 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.846 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.846 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:26.846 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.496597+0000", 2024-09-14T04:50:26.846 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.846 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.846 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.846 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:26.846 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.496596+0000", 2024-09-14T04:50:26.846 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:50:26.846 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.846 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.846 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:26.847 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.496601+0000", 2024-09-14T04:50:26.847 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.3900000000000001e-06 2024-09-14T04:50:26.847 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.847 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.847 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:26.847 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.496612+0000", 2024-09-14T04:50:26.847 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0560000000000001e-05 2024-09-14T04:50:26.847 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.847 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.847 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:26.847 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.496617+0000", 2024-09-14T04:50:26.847 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.1880000000000001e-06 2024-09-14T04:50:26.847 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.848 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.848 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.848 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.496662+0000", 2024-09-14T04:50:26.848 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.5059999999999999e-05 2024-09-14T04:50:26.848 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.848 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.848 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:50:26.848 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.496669+0000", 2024-09-14T04:50:26.848 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.472e-06 2024-09-14T04:50:26.848 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.848 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.848 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.848 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.414999+0000", 2024-09-14T04:50:26.848 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.91832989300000001 2024-09-14T04:50:26.849 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.849 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.849 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:26.849 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.415086+0000", 2024-09-14T04:50:26.849 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.7601000000000004e-05 2024-09-14T04:50:26.849 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.849 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.849 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,6", 2024-09-14T04:50:26.849 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.415197+0000", 2024-09-14T04:50:26.849 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000111171 2024-09-14T04:50:26.849 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.849 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.849 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.849 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.424319+0000", 2024-09-14T04:50:26.850 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0091215089999999999 2024-09-14T04:50:26.850 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.850 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.850 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.850 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.424376+0000", 2024-09-14T04:50:26.850 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.7303999999999999e-05 2024-09-14T04:50:26.850 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.850 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.850 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:26.850 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.424452+0000", 2024-09-14T04:50:26.850 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.5959999999999998e-05 2024-09-14T04:50:26.850 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.850 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.850 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:26.850 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.424495+0000", 2024-09-14T04:50:26.851 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.2568999999999998e-05 2024-09-14T04:50:26.851 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.851 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.851 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:26.851 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.424532+0000", 2024-09-14T04:50:26.851 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7570000000000001e-05 2024-09-14T04:50:26.851 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.851 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:26.851 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.851 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.851 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.851 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:236061 53.2 53:40dd0bbd:::benchmark_data_smithi039_224949_object105968:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1358)", 2024-09-14T04:50:26.851 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:10.599837+0000", 2024-09-14T04:50:26.851 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 496.18449322800001, 2024-09-14T04:50:26.851 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.832678052, 2024-09-14T04:50:26.851 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:26.852 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:26.852 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:26.852 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:50:26.852 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:50:26.852 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 236061 2024-09-14T04:50:26.852 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.852 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:26.852 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.852 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:26.852 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.599837+0000", 2024-09-14T04:50:26.852 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.852 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.852 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.853 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:26.853 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.599837+0000", 2024-09-14T04:50:26.853 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.853 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.853 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.853 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:26.853 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.599836+0000", 2024-09-14T04:50:26.853 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:50:26.853 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.853 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.853 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:26.853 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.599842+0000", 2024-09-14T04:50:26.854 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.6309999999999997e-06 2024-09-14T04:50:26.854 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.854 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.854 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:26.854 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.599855+0000", 2024-09-14T04:50:26.854 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3002e-05 2024-09-14T04:50:26.854 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.854 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.854 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:26.854 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.599861+0000", 2024-09-14T04:50:26.855 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.366e-06 2024-09-14T04:50:26.855 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.855 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.855 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.855 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.599898+0000", 2024-09-14T04:50:26.855 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7361999999999999e-05 2024-09-14T04:50:26.855 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.855 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.855 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:50:26.855 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.599904+0000", 2024-09-14T04:50:26.855 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.8270000000000003e-06 2024-09-14T04:50:26.855 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.856 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.856 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.856 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.420271+0000", 2024-09-14T04:50:26.856 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.82036698699999999 2024-09-14T04:50:26.856 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.856 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.856 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:26.856 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.420349+0000", 2024-09-14T04:50:26.857 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.7624000000000001e-05 2024-09-14T04:50:26.857 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.857 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.857 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,6", 2024-09-14T04:50:26.857 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.420422+0000", 2024-09-14T04:50:26.857 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.3077999999999994e-05 2024-09-14T04:50:26.857 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.858 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.858 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.858 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.424443+0000", 2024-09-14T04:50:26.858 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0040210649999999999 2024-09-14T04:50:26.858 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.858 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.859 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:26.859 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.432399+0000", 2024-09-14T04:50:26.859 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0079561200000000006 2024-09-14T04:50:26.859 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.859 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.859 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.860 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.432436+0000", 2024-09-14T04:50:26.860 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6820999999999999e-05 2024-09-14T04:50:26.860 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.860 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.860 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:26.860 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.432480+0000", 2024-09-14T04:50:26.861 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.4438999999999999e-05 2024-09-14T04:50:26.861 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.861 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.861 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:26.861 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.432515+0000", 2024-09-14T04:50:26.861 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5111999999999999e-05 2024-09-14T04:50:26.861 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.862 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:26.862 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.862 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.862 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.862 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:236073 53.2 53:4d31e253:::benchmark_data_smithi039_224949_object105980:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1358)", 2024-09-14T04:50:26.863 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:10.616660+0000", 2024-09-14T04:50:26.863 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 496.167670429, 2024-09-14T04:50:26.863 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.82373883800000003, 2024-09-14T04:50:26.863 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:26.864 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:26.864 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:26.864 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:50:26.864 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:50:26.864 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 236073 2024-09-14T04:50:26.864 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.865 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:26.865 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.865 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:26.865 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.616660+0000", 2024-09-14T04:50:26.865 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.865 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.866 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.866 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:26.866 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.616660+0000", 2024-09-14T04:50:26.866 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.866 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.866 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.867 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:26.867 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.616659+0000", 2024-09-14T04:50:26.867 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:50:26.867 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.867 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.867 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:26.868 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.616665+0000", 2024-09-14T04:50:26.868 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.4200000000000004e-06 2024-09-14T04:50:26.868 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.868 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.868 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:26.868 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.616677+0000", 2024-09-14T04:50:26.869 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2407e-05 2024-09-14T04:50:26.869 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.869 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.869 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:26.869 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.616684+0000", 2024-09-14T04:50:26.869 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.55e-06 2024-09-14T04:50:26.870 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.870 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.870 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.870 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.616717+0000", 2024-09-14T04:50:26.870 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.256e-05 2024-09-14T04:50:26.871 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.871 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.871 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:50:26.871 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.616723+0000", 2024-09-14T04:50:26.871 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.0789999999999997e-06 2024-09-14T04:50:26.871 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.872 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.872 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.872 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.424583+0000", 2024-09-14T04:50:26.872 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.80786079200000005 2024-09-14T04:50:26.872 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.872 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.873 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:26.873 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.424688+0000", 2024-09-14T04:50:26.873 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000104856 2024-09-14T04:50:26.873 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.873 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.873 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,6", 2024-09-14T04:50:26.874 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.424781+0000", 2024-09-14T04:50:26.874 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.2703999999999999e-05 2024-09-14T04:50:26.874 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.874 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.874 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:26.874 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.432525+0000", 2024-09-14T04:50:26.874 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0077435189999999999 2024-09-14T04:50:26.875 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.875 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.875 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.875 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.432606+0000", 2024-09-14T04:50:26.875 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.1834000000000003e-05 2024-09-14T04:50:26.875 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.875 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.876 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.876 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.440300+0000", 2024-09-14T04:50:26.876 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0076939640000000002 2024-09-14T04:50:26.876 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.876 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.876 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:26.876 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.440353+0000", 2024-09-14T04:50:26.876 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2394e-05 2024-09-14T04:50:26.876 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.876 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.876 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:26.876 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.440399+0000", 2024-09-14T04:50:26.877 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:50:26.878 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6096e-05 2024-09-14T04:50:26.878 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.879 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:26.879 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.879 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.879 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.879 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:236080 53.2 53:4ead5129:::benchmark_data_smithi039_224949_object105987:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1358)", 2024-09-14T04:50:26.879 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:10.623606+0000", 2024-09-14T04:50:26.879 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 496.16072424200001, 2024-09-14T04:50:26.879 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.82051554999999998, 2024-09-14T04:50:26.879 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:26.879 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:26.879 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:26.880 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:50:26.880 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:50:26.880 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 236080 2024-09-14T04:50:26.880 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.880 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:26.880 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.880 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:26.880 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.623606+0000", 2024-09-14T04:50:26.880 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.880 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.880 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.881 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:26.881 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.623606+0000", 2024-09-14T04:50:26.881 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.881 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.881 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.881 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:26.881 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.623605+0000", 2024-09-14T04:50:26.881 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:50:26.881 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.881 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.881 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:26.881 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.623611+0000", 2024-09-14T04:50:26.882 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.485e-06 2024-09-14T04:50:26.882 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.882 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.882 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:26.882 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.623624+0000", 2024-09-14T04:50:26.882 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3273e-05 2024-09-14T04:50:26.882 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.882 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.882 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:26.882 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.623631+0000", 2024-09-14T04:50:26.882 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.1659999999999997e-06 2024-09-14T04:50:26.882 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.882 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.882 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.882 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.626331+0000", 2024-09-14T04:50:26.883 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0026999960000000001 2024-09-14T04:50:26.883 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.883 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.883 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:50:26.883 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.626340+0000", 2024-09-14T04:50:26.883 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.969e-06 2024-09-14T04:50:26.883 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.883 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.883 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.883 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.428504+0000", 2024-09-14T04:50:26.883 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.80216416700000004 2024-09-14T04:50:26.883 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.883 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.883 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:26.884 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.428623+0000", 2024-09-14T04:50:26.884 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000118954 2024-09-14T04:50:26.884 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.884 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.884 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,6", 2024-09-14T04:50:26.884 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.428747+0000", 2024-09-14T04:50:26.884 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012388000000000001 2024-09-14T04:50:26.884 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.884 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.885 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.885 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.432639+0000", 2024-09-14T04:50:26.885 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0038913979999999999 2024-09-14T04:50:26.885 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.885 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.885 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:26.885 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.440415+0000", 2024-09-14T04:50:26.885 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0077763909999999997 2024-09-14T04:50:26.885 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.885 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.886 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.886 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.444035+0000", 2024-09-14T04:50:26.886 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.003620327 2024-09-14T04:50:26.886 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.886 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.886 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:26.886 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.444082+0000", 2024-09-14T04:50:26.886 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6374e-05 2024-09-14T04:50:26.886 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.886 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.886 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:26.886 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.444122+0000", 2024-09-14T04:50:26.886 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0006999999999998e-05 2024-09-14T04:50:26.886 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.887 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:26.887 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.887 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.887 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.887 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:236124 53.2 53:40279bf1:::benchmark_data_smithi039_224949_object106031:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1358)", 2024-09-14T04:50:26.887 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:10.688581+0000", 2024-09-14T04:50:26.887 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 496.09574909399998, 2024-09-14T04:50:26.887 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.76278635800000005, 2024-09-14T04:50:26.887 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:26.887 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:26.887 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:26.887 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:50:26.887 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:50:26.887 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 236124 2024-09-14T04:50:26.888 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.888 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:26.888 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.888 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:26.888 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.688581+0000", 2024-09-14T04:50:26.888 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.888 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.888 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.888 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:26.888 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.688581+0000", 2024-09-14T04:50:26.888 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.888 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.888 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.888 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:26.889 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.688580+0000", 2024-09-14T04:50:26.889 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:50:26.889 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.889 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.889 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:26.889 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.688586+0000", 2024-09-14T04:50:26.889 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.1099999999999999e-06 2024-09-14T04:50:26.889 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.889 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.889 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:26.889 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.688598+0000", 2024-09-14T04:50:26.889 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1201e-05 2024-09-14T04:50:26.889 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.890 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.890 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:26.890 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.688603+0000", 2024-09-14T04:50:26.890 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.412e-06 2024-09-14T04:50:26.890 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.890 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.890 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.890 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.688645+0000", 2024-09-14T04:50:26.890 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.2216999999999999e-05 2024-09-14T04:50:26.890 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.890 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.890 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:50:26.890 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.688653+0000", 2024-09-14T04:50:26.890 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.622e-06 2024-09-14T04:50:26.891 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.891 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.891 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.891 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.432658+0000", 2024-09-14T04:50:26.891 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.74400565600000002 2024-09-14T04:50:26.891 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.891 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.891 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:26.891 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.432784+0000", 2024-09-14T04:50:26.891 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000125878 2024-09-14T04:50:26.891 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.891 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.891 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,6", 2024-09-14T04:50:26.891 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.432900+0000", 2024-09-14T04:50:26.891 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011533500000000001 2024-09-14T04:50:26.892 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.892 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.892 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.892 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.440475+0000", 2024-09-14T04:50:26.892 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0075753620000000004 2024-09-14T04:50:26.892 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.892 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.892 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:26.892 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.443985+0000", 2024-09-14T04:50:26.892 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0035098719999999998 2024-09-14T04:50:26.892 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.892 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.892 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.892 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.451289+0000", 2024-09-14T04:50:26.893 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.007304385 2024-09-14T04:50:26.893 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.893 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.893 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:26.893 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.451336+0000", 2024-09-14T04:50:26.893 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6925999999999998e-05 2024-09-14T04:50:26.893 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.893 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.893 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:26.893 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.451368+0000", 2024-09-14T04:50:26.893 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.1485000000000001e-05 2024-09-14T04:50:26.893 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.893 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:26.894 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.894 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.894 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.894 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:236153 53.2 53:4cd9f70a:::benchmark_data_smithi039_224949_object106060:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1358)", 2024-09-14T04:50:26.894 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:10.733312+0000", 2024-09-14T04:50:26.894 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 496.05101804399999, 2024-09-14T04:50:26.894 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.718164099, 2024-09-14T04:50:26.894 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:26.894 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:26.894 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:26.894 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:50:26.894 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:50:26.894 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 236153 2024-09-14T04:50:26.894 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.895 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:26.895 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.895 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:26.895 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.733312+0000", 2024-09-14T04:50:26.895 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.895 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.895 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.895 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:26.895 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.733312+0000", 2024-09-14T04:50:26.895 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.895 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.895 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.895 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:26.895 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.733311+0000", 2024-09-14T04:50:26.896 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:50:26.896 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.896 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.896 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:26.896 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.733317+0000", 2024-09-14T04:50:26.896 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.0499999999999997e-06 2024-09-14T04:50:26.896 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.896 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.896 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:26.896 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.733329+0000", 2024-09-14T04:50:26.896 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1542999999999999e-05 2024-09-14T04:50:26.896 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.896 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.897 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:26.897 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.733334+0000", 2024-09-14T04:50:26.897 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.5670000000000002e-06 2024-09-14T04:50:26.897 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.897 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.897 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.897 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.733383+0000", 2024-09-14T04:50:26.897 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8479999999999997e-05 2024-09-14T04:50:26.897 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.897 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.897 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:50:26.897 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.733395+0000", 2024-09-14T04:50:26.897 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1838000000000001e-05 2024-09-14T04:50:26.898 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.898 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.898 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.898 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.436601+0000", 2024-09-14T04:50:26.898 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.70320648399999997 2024-09-14T04:50:26.898 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.898 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.898 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:26.899 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.436685+0000", 2024-09-14T04:50:26.899 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.3458e-05 2024-09-14T04:50:26.899 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.899 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.899 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,6", 2024-09-14T04:50:26.899 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.436809+0000", 2024-09-14T04:50:26.899 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000124046 2024-09-14T04:50:26.899 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.899 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.899 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.899 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.440514+0000", 2024-09-14T04:50:26.899 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0037058299999999998 2024-09-14T04:50:26.899 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.900 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.900 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:26.900 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.443998+0000", 2024-09-14T04:50:26.900 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0034835040000000001 2024-09-14T04:50:26.900 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.900 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.900 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.900 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.451413+0000", 2024-09-14T04:50:26.900 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0074155549999999999 2024-09-14T04:50:26.900 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.900 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.900 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:26.900 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.451441+0000", 2024-09-14T04:50:26.901 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.7603e-05 2024-09-14T04:50:26.901 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.901 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.901 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:26.901 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.451476+0000", 2024-09-14T04:50:26.901 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5363e-05 2024-09-14T04:50:26.901 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.901 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:26.901 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.901 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.901 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.901 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:236163 53.2 53:44cffcae:::benchmark_data_smithi039_224949_object106070:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1358)", 2024-09-14T04:50:26.901 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:10.814149+0000", 2024-09-14T04:50:26.901 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 495.970181895, 2024-09-14T04:50:26.902 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.637431945, 2024-09-14T04:50:26.902 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:26.902 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:26.902 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:26.902 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:50:26.902 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:50:26.902 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 236163 2024-09-14T04:50:26.902 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.902 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:26.902 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.902 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:26.902 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.814149+0000", 2024-09-14T04:50:26.902 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.902 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.903 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.903 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:26.903 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.814149+0000", 2024-09-14T04:50:26.903 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.903 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.903 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.903 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:26.903 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.814147+0000", 2024-09-14T04:50:26.903 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:50:26.903 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.903 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.903 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:26.903 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.814154+0000", 2024-09-14T04:50:26.903 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.2799999999999998e-06 2024-09-14T04:50:26.904 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.904 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.904 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:26.904 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.814167+0000", 2024-09-14T04:50:26.904 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2716e-05 2024-09-14T04:50:26.904 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.904 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.904 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:26.904 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.814174+0000", 2024-09-14T04:50:26.904 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.922e-06 2024-09-14T04:50:26.904 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.904 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.904 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.905 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.814218+0000", 2024-09-14T04:50:26.905 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.3721999999999999e-05 2024-09-14T04:50:26.905 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.905 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.905 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:50:26.905 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.814227+0000", 2024-09-14T04:50:26.905 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.5699999999999999e-06 2024-09-14T04:50:26.905 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.905 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.905 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.905 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.440538+0000", 2024-09-14T04:50:26.905 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.62631039300000002 2024-09-14T04:50:26.905 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.905 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.906 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:26.906 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.440675+0000", 2024-09-14T04:50:26.906 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000137491 2024-09-14T04:50:26.906 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.906 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.906 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,6", 2024-09-14T04:50:26.906 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.440752+0000", 2024-09-14T04:50:26.906 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.6443e-05 2024-09-14T04:50:26.907 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.907 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.907 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.907 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.444180+0000", 2024-09-14T04:50:26.907 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0034289189999999999 2024-09-14T04:50:26.907 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.907 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.907 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:26.907 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.451246+0000", 2024-09-14T04:50:26.907 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0070654029999999996 2024-09-14T04:50:26.908 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.908 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.908 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.908 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.451523+0000", 2024-09-14T04:50:26.908 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00027719700000000002 2024-09-14T04:50:26.908 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.908 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.908 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:26.908 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.451550+0000", 2024-09-14T04:50:26.908 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.72e-05 2024-09-14T04:50:26.908 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.908 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.908 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:26.909 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.451580+0000", 2024-09-14T04:50:26.909 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.0171999999999998e-05 2024-09-14T04:50:26.909 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.909 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:26.909 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.909 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.909 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.909 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:236193 53.2 53:496c3490:::benchmark_data_smithi039_224949_object106100:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1358)", 2024-09-14T04:50:26.909 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:10.878075+0000", 2024-09-14T04:50:26.909 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 495.90625533899998, 2024-09-14T04:50:26.909 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.58090754099999997, 2024-09-14T04:50:26.909 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:26.910 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:26.910 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:26.910 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:50:26.910 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:50:26.910 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 236193 2024-09-14T04:50:26.910 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.910 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:26.910 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.910 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:26.910 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.878075+0000", 2024-09-14T04:50:26.910 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.910 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.911 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.911 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:26.911 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.878075+0000", 2024-09-14T04:50:26.911 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.911 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.911 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.911 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:26.911 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.878074+0000", 2024-09-14T04:50:26.911 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:50:26.911 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.911 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.912 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:26.912 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.878080+0000", 2024-09-14T04:50:26.912 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9920000000000004e-06 2024-09-14T04:50:26.912 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.912 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.912 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:26.912 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.878092+0000", 2024-09-14T04:50:26.912 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1989e-05 2024-09-14T04:50:26.912 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.912 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.912 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:26.912 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.878099+0000", 2024-09-14T04:50:26.912 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.3010000000000001e-06 2024-09-14T04:50:26.913 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.913 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.913 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.913 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.878148+0000", 2024-09-14T04:50:26.913 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.9234000000000003e-05 2024-09-14T04:50:26.913 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.913 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.913 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:50:26.913 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.878158+0000", 2024-09-14T04:50:26.913 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.9150000000000008e-06 2024-09-14T04:50:26.913 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.913 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.913 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.914 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.444216+0000", 2024-09-14T04:50:26.914 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.56605803499999996 2024-09-14T04:50:26.914 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.914 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.914 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:26.914 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.444330+0000", 2024-09-14T04:50:26.914 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000114058 2024-09-14T04:50:26.914 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.914 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.914 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,6", 2024-09-14T04:50:26.914 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.444450+0000", 2024-09-14T04:50:26.914 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011963 2024-09-14T04:50:26.914 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.914 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.915 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:26.915 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.451489+0000", 2024-09-14T04:50:26.915 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0070390749999999997 2024-09-14T04:50:26.915 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.915 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.915 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.915 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.451614+0000", 2024-09-14T04:50:26.915 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012502100000000001 2024-09-14T04:50:26.915 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.915 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.915 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.915 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.458842+0000", 2024-09-14T04:50:26.915 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0072282830000000003 2024-09-14T04:50:26.916 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.916 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.916 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:26.916 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.458939+0000", 2024-09-14T04:50:26.916 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.6545000000000005e-05 2024-09-14T04:50:26.916 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.916 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.916 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:26.916 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.458983+0000", 2024-09-14T04:50:26.916 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.3612000000000002e-05 2024-09-14T04:50:26.916 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.916 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:26.916 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.917 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.917 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.917 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45053.0:236211 53.2 53:45d474a9:::benchmark_data_smithi039_224949_object106118:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1358)", 2024-09-14T04:50:26.917 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:42:10.950313+0000", 2024-09-14T04:50:26.917 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 495.83401707500002, 2024-09-14T04:50:26.917 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.51294055500000002, 2024-09-14T04:50:26.917 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:26.917 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:26.918 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:26.918 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45053", 2024-09-14T04:50:26.918 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:50:26.918 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 236211 2024-09-14T04:50:26.918 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.918 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:26.918 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.918 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:26.918 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.950313+0000", 2024-09-14T04:50:26.918 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.918 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.918 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.919 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:26.919 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.950313+0000", 2024-09-14T04:50:26.919 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.919 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.919 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.919 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:26.919 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.950312+0000", 2024-09-14T04:50:26.919 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:50:26.919 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.919 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.919 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:26.919 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.950320+0000", 2024-09-14T04:50:26.919 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.6600000000000001e-06 2024-09-14T04:50:26.920 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.920 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.920 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:26.920 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.950337+0000", 2024-09-14T04:50:26.920 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.6846000000000001e-05 2024-09-14T04:50:26.920 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.920 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.920 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:26.920 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.950345+0000", 2024-09-14T04:50:26.920 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.0690000000000002e-06 2024-09-14T04:50:26.920 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.920 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.920 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.921 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.950396+0000", 2024-09-14T04:50:26.921 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.0846000000000001e-05 2024-09-14T04:50:26.921 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.921 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.921 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:50:26.921 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:10.950403+0000", 2024-09-14T04:50:26.921 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.2180000000000002e-06 2024-09-14T04:50:26.921 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.921 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.921 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.921 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.447712+0000", 2024-09-14T04:50:26.921 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.49730823200000002 2024-09-14T04:50:26.922 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.922 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.922 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:26.922 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.447802+0000", 2024-09-14T04:50:26.922 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.0816999999999995e-05 2024-09-14T04:50:26.922 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.922 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.922 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,6", 2024-09-14T04:50:26.922 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.447880+0000", 2024-09-14T04:50:26.922 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.7788999999999997e-05 2024-09-14T04:50:26.922 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.922 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.922 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:26.922 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.451619+0000", 2024-09-14T04:50:26.923 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0037387119999999999 2024-09-14T04:50:26.923 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.923 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.923 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.923 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.451652+0000", 2024-09-14T04:50:26.923 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3206e-05 2024-09-14T04:50:26.923 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.923 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.923 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.923 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.463159+0000", 2024-09-14T04:50:26.923 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.011507089999999999 2024-09-14T04:50:26.923 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.923 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.923 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:26.924 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.463214+0000", 2024-09-14T04:50:26.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:26.925 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:26.925 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:26.925 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.4604000000000001e-05 2024-09-14T04:50:26.925 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.926 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.926 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:26.926 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:42:11.463254+0000", 2024-09-14T04:50:26.926 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0142000000000003e-05 2024-09-14T04:50:26.926 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.926 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:26.926 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.926 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.926 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.926 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:104264 55.8 55:1fc585f2:::benchmark_data_smithi039_297155_object104263:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1440)", 2024-09-14T04:50:26.926 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:49:10.240109+0000", 2024-09-14T04:50:26.926 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 76.544221668000006, 2024-09-14T04:50:26.926 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.39979108000000002, 2024-09-14T04:50:26.926 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:26.927 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:26.927 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:26.927 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:50:26.927 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:26.927 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 104264 2024-09-14T04:50:26.927 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.927 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:26.927 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.927 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:26.927 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.240109+0000", 2024-09-14T04:50:26.927 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.927 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.927 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.928 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:26.928 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.240109+0000", 2024-09-14T04:50:26.928 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.928 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.928 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.928 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:26.929 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.240107+0000", 2024-09-14T04:50:26.929 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:50:26.929 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.929 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.929 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:26.929 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.240140+0000", 2024-09-14T04:50:26.929 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.2925000000000001e-05 2024-09-14T04:50:26.929 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.929 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.929 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:26.929 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.240153+0000", 2024-09-14T04:50:26.929 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3074000000000001e-05 2024-09-14T04:50:26.929 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.930 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.930 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:26.930 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.240160+0000", 2024-09-14T04:50:26.930 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.0439999999999998e-06 2024-09-14T04:50:26.930 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.930 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.930 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.930 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.240203+0000", 2024-09-14T04:50:26.930 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.2797999999999999e-05 2024-09-14T04:50:26.930 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.930 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.930 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:26.930 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.240306+0000", 2024-09-14T04:50:26.930 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000102959 2024-09-14T04:50:26.931 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.931 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.931 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,6", 2024-09-14T04:50:26.931 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.240386+0000", 2024-09-14T04:50:26.931 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.9819000000000003e-05 2024-09-14T04:50:26.931 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.931 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.931 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:26.931 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.322851+0000", 2024-09-14T04:50:26.931 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.082465262999999997 2024-09-14T04:50:26.931 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.931 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.931 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.931 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.322882+0000", 2024-09-14T04:50:26.932 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.0682000000000002e-05 2024-09-14T04:50:26.932 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.932 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.932 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.932 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.639772+0000", 2024-09-14T04:50:26.932 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.31689024900000001 2024-09-14T04:50:26.932 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.932 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.932 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:26.932 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.639859+0000", 2024-09-14T04:50:26.932 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.7014000000000004e-05 2024-09-14T04:50:26.932 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.932 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.933 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:26.933 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.639900+0000", 2024-09-14T04:50:26.933 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0748999999999999e-05 2024-09-14T04:50:26.933 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.933 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:26.933 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.933 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.933 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.933 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:104267 55.8 55:19de0ba9:::benchmark_data_smithi039_297155_object104266:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1440)", 2024-09-14T04:50:26.933 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:49:10.323499+0000", 2024-09-14T04:50:26.933 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 76.460831251000002, 2024-09-14T04:50:26.933 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.316520142, 2024-09-14T04:50:26.933 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:26.933 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:26.934 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:26.934 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:50:26.934 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:26.934 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 104267 2024-09-14T04:50:26.934 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.934 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:26.934 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.934 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:26.934 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.323499+0000", 2024-09-14T04:50:26.934 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.934 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.934 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.934 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:26.935 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.323499+0000", 2024-09-14T04:50:26.935 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.935 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.935 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.935 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:26.935 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.323494+0000", 2024-09-14T04:50:26.935 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999948 2024-09-14T04:50:26.935 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.935 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.935 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:26.935 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.323584+0000", 2024-09-14T04:50:26.935 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.0680999999999995e-05 2024-09-14T04:50:26.935 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.935 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.936 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:26.936 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.323602+0000", 2024-09-14T04:50:26.936 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.7265000000000002e-05 2024-09-14T04:50:26.936 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.936 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.936 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:26.936 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.323610+0000", 2024-09-14T04:50:26.936 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.0139999999999998e-06 2024-09-14T04:50:26.936 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.936 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.936 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.937 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.325579+0000", 2024-09-14T04:50:26.937 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0019690760000000002 2024-09-14T04:50:26.937 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.937 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.937 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:26.937 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.325786+0000", 2024-09-14T04:50:26.937 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00020740600000000001 2024-09-14T04:50:26.937 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.937 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.937 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,6", 2024-09-14T04:50:26.938 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.325940+0000", 2024-09-14T04:50:26.938 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00015370299999999999 2024-09-14T04:50:26.938 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.938 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.938 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:26.938 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.329395+0000", 2024-09-14T04:50:26.938 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0034548270000000002 2024-09-14T04:50:26.938 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.938 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.938 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.938 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.332624+0000", 2024-09-14T04:50:26.939 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0032294889999999999 2024-09-14T04:50:26.939 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.939 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.939 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.939 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.639938+0000", 2024-09-14T04:50:26.939 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.307313581 2024-09-14T04:50:26.939 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.939 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.940 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:26.940 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.639985+0000", 2024-09-14T04:50:26.940 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6761999999999997e-05 2024-09-14T04:50:26.940 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.940 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.940 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:26.940 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.640019+0000", 2024-09-14T04:50:26.940 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.4668000000000001e-05 2024-09-14T04:50:26.940 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.940 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:26.940 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.941 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.941 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.941 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:149721 55.9 55:963f2b16:::benchmark_data_smithi039_297155_object25788:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1447)", 2024-09-14T04:50:26.941 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:49:58.468180+0000", 2024-09-14T04:50:26.941 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 28.316150263000001, 2024-09-14T04:50:26.941 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.34284567700000002, 2024-09-14T04:50:26.941 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:26.941 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:26.941 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:26.941 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:50:26.941 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:26.941 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 149721 2024-09-14T04:50:26.941 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.942 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:26.942 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.942 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:26.942 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.468180+0000", 2024-09-14T04:50:26.942 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.942 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.942 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.942 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:26.942 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.468180+0000", 2024-09-14T04:50:26.942 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.942 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.942 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.942 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:26.942 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.468179+0000", 2024-09-14T04:50:26.943 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:50:26.943 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.943 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.943 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:26.943 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.468188+0000", 2024-09-14T04:50:26.943 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.1759999999999999e-06 2024-09-14T04:50:26.943 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.943 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.943 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:26.943 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.468200+0000", 2024-09-14T04:50:26.943 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2089000000000001e-05 2024-09-14T04:50:26.943 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.943 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.943 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:26.944 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.468207+0000", 2024-09-14T04:50:26.944 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7549999999999997e-06 2024-09-14T04:50:26.944 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.944 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.944 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.944 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.468475+0000", 2024-09-14T04:50:26.944 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00026856999999999998 2024-09-14T04:50:26.944 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.944 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.944 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:26.944 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.469536+0000", 2024-09-14T04:50:26.944 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0010609160000000001 2024-09-14T04:50:26.944 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.945 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.945 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,3", 2024-09-14T04:50:26.945 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.469643+0000", 2024-09-14T04:50:26.945 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010696000000000001 2024-09-14T04:50:26.945 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.945 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.945 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:26.945 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.479119+0000", 2024-09-14T04:50:26.945 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.009476356 2024-09-14T04:50:26.946 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.946 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.946 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.946 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.479464+0000", 2024-09-14T04:50:26.946 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00034406999999999998 2024-09-14T04:50:26.946 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.946 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.946 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.946 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.810925+0000", 2024-09-14T04:50:26.946 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33146173600000001 2024-09-14T04:50:26.946 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.947 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.947 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:26.947 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.810982+0000", 2024-09-14T04:50:26.947 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.6569e-05 2024-09-14T04:50:26.947 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.947 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.947 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:26.947 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.811026+0000", 2024-09-14T04:50:26.947 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.3993999999999999e-05 2024-09-14T04:50:26.947 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.947 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:26.947 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.947 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.948 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.948 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:149725 55.9 55:90743abc:::benchmark_data_smithi039_297155_object25792:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1447)", 2024-09-14T04:50:26.948 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:49:58.471639+0000", 2024-09-14T04:50:26.948 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 28.312690973999999, 2024-09-14T04:50:26.948 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.34457423300000001, 2024-09-14T04:50:26.948 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:26.948 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:26.948 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:26.948 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:50:26.948 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:26.948 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 149725 2024-09-14T04:50:26.948 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.948 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:26.949 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.949 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:26.949 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471639+0000", 2024-09-14T04:50:26.949 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.949 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.949 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.949 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:26.949 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471639+0000", 2024-09-14T04:50:26.949 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:26.949 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.949 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.950 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:26.950 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471638+0000", 2024-09-14T04:50:26.950 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:50:26.950 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.950 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.950 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:26.950 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471644+0000", 2024-09-14T04:50:26.950 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.6960000000000002e-06 2024-09-14T04:50:26.950 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.950 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.951 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:26.951 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471656+0000", 2024-09-14T04:50:26.951 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2234e-05 2024-09-14T04:50:26.951 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.951 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.951 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:26.951 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471661+0000", 2024-09-14T04:50:26.951 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2299999999999999e-06 2024-09-14T04:50:26.951 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.951 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.951 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:26.951 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.474615+0000", 2024-09-14T04:50:26.952 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.002953893 2024-09-14T04:50:26.952 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.952 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.952 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:26.952 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.474746+0000", 2024-09-14T04:50:26.952 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013021800000000001 2024-09-14T04:50:26.952 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.952 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.952 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,3", 2024-09-14T04:50:26.952 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.474861+0000", 2024-09-14T04:50:26.952 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000115696 2024-09-14T04:50:26.952 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.952 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.953 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:26.953 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.479246+0000", 2024-09-14T04:50:26.953 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0043852259999999999 2024-09-14T04:50:26.953 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.953 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.953 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.953 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.496383+0000", 2024-09-14T04:50:26.953 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.017136703999999999 2024-09-14T04:50:26.953 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.953 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.953 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:26.953 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.816111+0000", 2024-09-14T04:50:26.954 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.31972738299999998 2024-09-14T04:50:26.954 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.954 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.954 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:26.954 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.816168+0000", 2024-09-14T04:50:26.954 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.7682000000000001e-05 2024-09-14T04:50:26.954 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:26.954 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:26.954 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:26.954 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.816214+0000", 2024-09-14T04:50:26.954 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.5391e-05 2024-09-14T04:50:26.954 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.954 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:26.955 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.955 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:26.955 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:26.955 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:50:27.026 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:27.080 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:27.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:27.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:27.184 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:27.220 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:27.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:27.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:27.322 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:27.360 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:27.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:27.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:27.462 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:27.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:27.508 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:27.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:27.609 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:27.672 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:27.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:27.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:27.730 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:27 smithi039 ceph-mon[111235]: pgmap v2644: 99 pgs: 99 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 819 op/s 2024-09-14T04:50:27.731 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:27 smithi039 ceph-mon[106470]: pgmap v2644: 99 pgs: 99 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 819 op/s 2024-09-14T04:50:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:27 smithi129 ceph-mon[101316]: pgmap v2644: 99 pgs: 99 active+clean; 4.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 819 op/s 2024-09-14T04:50:27.774 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:27.809 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:27.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:27.810 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:27.911 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:27.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:27.944 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:27.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:27.991 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-14T04:50:28.045 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:28.074 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:28.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:28.075 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:28.176 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:28.238 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:28.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:28.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:28.340 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:28.355 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:50:28.380 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:28.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:28.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:28.482 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:28.520 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:28.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:28.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:28.622 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:28.658 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:28.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:28.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:28.760 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:28.791 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:50:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:50:28] "GET /metrics HTTP/1.1" 200 36769 "" "Prometheus/2.51.0" 2024-09-14T04:50:28.813 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:28.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:28.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:28.915 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:28.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:28.959 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:28.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:29.060 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:29.098 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:29.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:29.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:29.110 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:50:29.200 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:29.242 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:29.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:29.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:29.344 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:29.373 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:29.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:29.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:29.474 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:29.518 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:29.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:29.519 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:29.612 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is unset 2024-09-14T04:50:29.620 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:29.664 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:29.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:29.665 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:29.766 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:29.787 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:29.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:29.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:29.889 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:29.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:29.925 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:29.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:30.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:29 smithi129 ceph-mon[101316]: pgmap v2645: 99 pgs: 99 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 792 op/s 2024-09-14T04:50:30.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:29 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3594043354' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:50:30.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:29 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:50:30.026 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:30.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:29 smithi039 ceph-mon[106470]: pgmap v2645: 99 pgs: 99 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 792 op/s 2024-09-14T04:50:30.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:29 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3594043354' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:50:30.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:29 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:50:30.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:29 smithi039 ceph-mon[111235]: pgmap v2645: 99 pgs: 99 active+clean; 4.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 792 op/s 2024-09-14T04:50:30.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:29 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3594043354' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:50:30.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:29 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:50:30.047 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:30.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:30.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:30.149 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:30.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:30.188 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:30.188 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:30.289 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:30.330 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:30.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:30.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:30.432 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:30.470 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:30.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:30.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:30.572 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:30.609 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:30.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:30.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:30.711 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:30.741 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:30 smithi039 ceph-mon[106470]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:50:30.741 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:30 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T04:50:30.741 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:30 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:50:30.741 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:30 smithi039 ceph-mon[106470]: osdmap e1451: 8 total, 8 up, 8 in 2024-09-14T04:50:30.742 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:30 smithi039 ceph-mon[111235]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:50:30.742 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:30 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T04:50:30.742 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:30 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:50:30.742 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:30 smithi039 ceph-mon[111235]: osdmap e1451: 8 total, 8 up, 8 in 2024-09-14T04:50:30.745 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:30.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:30.746 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:30.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:30 smithi129 ceph-mon[101316]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:50:30.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:30 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T04:50:30.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:30 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:50:30.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:30 smithi129 ceph-mon[101316]: osdmap e1451: 8 total, 8 up, 8 in 2024-09-14T04:50:30.847 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:30.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:30.886 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:30.886 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:30.987 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:31.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:31.021 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:31.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:31.122 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:31.180 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:31.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:31.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:31.282 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:31.331 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:31.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:31.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:31.433 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:31.462 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:31.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:31.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:31.564 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:31.604 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:31.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:31.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:31.646 INFO:teuthology.orchestra.run.smithi039.stderr:set pool 1 pgp_num to 11 2024-09-14T04:50:31.706 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:31.747 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:31.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:31.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:31.788 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:50:31.789 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [ 2024-09-14T04:50:31.789 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.789 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.45632.0:178774 55.8 e1452/1419 55:14ae32df:::benchmark_data_smithi039_297155_object54841:head v 1452'11236, mlcod=1452'11234)", 2024-09-14T04:50:31.789 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:50:31.769121+0000", 2024-09-14T04:50:31.789 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.015775978, 2024-09-14T04:50:31.789 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.015812030000000001, 2024-09-14T04:50:31.789 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:31.789 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "started", 2024-09-14T04:50:31.789 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:31.789 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.789 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:31.790 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.769121+0000", 2024-09-14T04:50:31.790 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:31.790 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.790 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.790 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:31.790 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.769121+0000", 2024-09-14T04:50:31.790 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:31.790 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.790 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.790 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:31.790 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.769120+0000", 2024-09-14T04:50:31.790 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:50:31.790 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.791 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.791 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:31.791 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.769128+0000", 2024-09-14T04:50:31.791 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.3739999999999998e-06 2024-09-14T04:50:31.791 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.791 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.791 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:31.791 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.769143+0000", 2024-09-14T04:50:31.791 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5367999999999999e-05 2024-09-14T04:50:31.791 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.791 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.791 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:31.791 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.769151+0000", 2024-09-14T04:50:31.791 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.9459999999999998e-06 2024-09-14T04:50:31.791 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.792 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.792 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:31.792 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.769681+0000", 2024-09-14T04:50:31.792 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00052983399999999997 2024-09-14T04:50:31.792 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.792 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.792 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:31.792 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.769727+0000", 2024-09-14T04:50:31.792 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6020000000000003e-05 2024-09-14T04:50:31.792 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:31.792 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:31.792 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:31.792 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.793 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.793 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:178779 55.e 55:7c345b6b:::benchmark_data_smithi039_297155_object54846:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1452)", 2024-09-14T04:50:31.793 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:50:31.770321+0000", 2024-09-14T04:50:31.793 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.014575209, 2024-09-14T04:50:31.793 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.014681068, 2024-09-14T04:50:31.793 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:31.793 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:50:31.793 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:31.793 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:50:31.793 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:31.793 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 178779 2024-09-14T04:50:31.793 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.793 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:31.794 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.794 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:31.794 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.770321+0000", 2024-09-14T04:50:31.794 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:31.794 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.794 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.794 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:31.794 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.770321+0000", 2024-09-14T04:50:31.794 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:31.794 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.794 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.794 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:31.794 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.770320+0000", 2024-09-14T04:50:31.794 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:50:31.795 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.795 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.795 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:31.795 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.770326+0000", 2024-09-14T04:50:31.795 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.7010000000000001e-06 2024-09-14T04:50:31.795 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.795 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.795 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:31.795 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.770334+0000", 2024-09-14T04:50:31.795 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.4670000000000004e-06 2024-09-14T04:50:31.795 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.795 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.795 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:31.796 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.770341+0000", 2024-09-14T04:50:31.796 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.2249999999999997e-06 2024-09-14T04:50:31.796 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.796 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.796 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:31.796 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.774809+0000", 2024-09-14T04:50:31.796 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0044687629999999997 2024-09-14T04:50:31.796 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.796 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.796 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:31.796 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.774933+0000", 2024-09-14T04:50:31.796 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012390099999999999 2024-09-14T04:50:31.796 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.796 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.797 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:50:31.797 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.775035+0000", 2024-09-14T04:50:31.797 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000101221 2024-09-14T04:50:31.797 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.797 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.797 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:31.797 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.782554+0000", 2024-09-14T04:50:31.797 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0075190930000000001 2024-09-14T04:50:31.797 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:31.797 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:31.797 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:31.797 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.797 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.797 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:178784 55.5 55:ab6bd8e4:::benchmark_data_smithi039_297155_object54851:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1452)", 2024-09-14T04:50:31.798 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:50:31.775938+0000", 2024-09-14T04:50:31.798 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.0089590680000000006, 2024-09-14T04:50:31.798 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0091483069999999996, 2024-09-14T04:50:31.798 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:31.798 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:50:31.798 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:31.798 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:50:31.798 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:31.798 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 178784 2024-09-14T04:50:31.798 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.798 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:31.798 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.798 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:31.799 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.775938+0000", 2024-09-14T04:50:31.799 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:31.799 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.799 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.799 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:31.799 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.775938+0000", 2024-09-14T04:50:31.799 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:31.799 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.799 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.799 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:31.800 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.775936+0000", 2024-09-14T04:50:31.800 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:50:31.800 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.800 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.800 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:31.800 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.775948+0000", 2024-09-14T04:50:31.800 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1807999999999999e-05 2024-09-14T04:50:31.800 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.800 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.800 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:31.800 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.775963+0000", 2024-09-14T04:50:31.800 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5302999999999999e-05 2024-09-14T04:50:31.800 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.801 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.801 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:31.801 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.775975+0000", 2024-09-14T04:50:31.801 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1938e-05 2024-09-14T04:50:31.801 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.801 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.801 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:31.801 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.776570+0000", 2024-09-14T04:50:31.801 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00059460700000000003 2024-09-14T04:50:31.801 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.802 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.802 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:31.802 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.776866+0000", 2024-09-14T04:50:31.802 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00029585899999999999 2024-09-14T04:50:31.802 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.802 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.802 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:50:31.802 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.777249+0000", 2024-09-14T04:50:31.802 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00038321200000000002 2024-09-14T04:50:31.802 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:31.802 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:31.802 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:31.802 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.803 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.803 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:178785 55.0 55:0ec8a14a:::benchmark_data_smithi039_297155_object54852:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1452)", 2024-09-14T04:50:31.803 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:50:31.775985+0000", 2024-09-14T04:50:31.803 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.0089117350000000005, 2024-09-14T04:50:31.803 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0091721040000000004, 2024-09-14T04:50:31.803 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:31.803 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:50:31.803 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:31.803 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:50:31.803 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:31.803 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 178785 2024-09-14T04:50:31.803 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.803 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:31.804 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.804 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:31.804 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.775985+0000", 2024-09-14T04:50:31.804 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:31.804 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.804 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.804 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:31.804 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.775985+0000", 2024-09-14T04:50:31.804 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:31.804 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.804 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.804 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:31.804 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.775984+0000", 2024-09-14T04:50:31.805 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:50:31.805 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.805 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.805 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:31.805 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.775988+0000", 2024-09-14T04:50:31.805 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5109999999999999e-06 2024-09-14T04:50:31.805 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.805 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.805 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:31.805 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.775997+0000", 2024-09-14T04:50:31.805 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.1349999999999998e-06 2024-09-14T04:50:31.805 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.806 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.806 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:31.806 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.776002+0000", 2024-09-14T04:50:31.806 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.3369999999999999e-06 2024-09-14T04:50:31.806 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.806 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.806 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:31.806 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.778430+0000", 2024-09-14T04:50:31.806 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0024273760000000002 2024-09-14T04:50:31.806 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.806 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.806 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:31.806 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.778561+0000", 2024-09-14T04:50:31.807 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013097799999999999 2024-09-14T04:50:31.807 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.807 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.807 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:50:31.807 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.778667+0000", 2024-09-14T04:50:31.807 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000106496 2024-09-14T04:50:31.807 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:31.807 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:31.807 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:31.807 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.807 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.807 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:178788 55.5 55:a067c1a9:::benchmark_data_smithi039_297155_object54855:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1452)", 2024-09-14T04:50:31.807 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:50:31.776013+0000", 2024-09-14T04:50:31.808 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.0088841019999999996, 2024-09-14T04:50:31.808 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0092128969999999994, 2024-09-14T04:50:31.808 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:31.808 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:50:31.808 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:31.808 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:50:31.808 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:31.808 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 178788 2024-09-14T04:50:31.808 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.808 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:31.808 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.808 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:31.808 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.776013+0000", 2024-09-14T04:50:31.809 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:31.809 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.809 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.809 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:31.809 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.776013+0000", 2024-09-14T04:50:31.809 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:31.809 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.809 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.809 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:31.809 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.776012+0000", 2024-09-14T04:50:31.809 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:50:31.809 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.809 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.809 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:31.810 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.776015+0000", 2024-09-14T04:50:31.810 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.6419999999999999e-06 2024-09-14T04:50:31.810 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.810 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.810 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:31.810 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.776028+0000", 2024-09-14T04:50:31.810 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2850000000000001e-05 2024-09-14T04:50:31.810 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.810 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.810 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:31.810 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.776033+0000", 2024-09-14T04:50:31.810 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.6829999999999996e-06 2024-09-14T04:50:31.810 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.811 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.811 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:31.811 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.782287+0000", 2024-09-14T04:50:31.811 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0062541469999999998 2024-09-14T04:50:31.811 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.811 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.811 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:31.811 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.782398+0000", 2024-09-14T04:50:31.811 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000110441 2024-09-14T04:50:31.811 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.811 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.811 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:50:31.811 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.782762+0000", 2024-09-14T04:50:31.811 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000363889 2024-09-14T04:50:31.812 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:31.812 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:31.812 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:31.812 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.812 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.812 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop_reply(client.45632.0:178769 55.5 e1452/1419)", 2024-09-14T04:50:31.812 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:50:31.777884+0000", 2024-09-14T04:50:31.812 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.0070129270000000004, 2024-09-14T04:50:31.812 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0074027440000000002, 2024-09-14T04:50:31.812 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:31.812 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "queued for pg", 2024-09-14T04:50:31.812 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:31.813 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.813 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:31.813 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.777884+0000", 2024-09-14T04:50:31.813 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:31.813 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.813 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.813 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:31.813 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.777884+0000", 2024-09-14T04:50:31.813 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:31.813 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.813 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.813 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:31.813 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.777883+0000", 2024-09-14T04:50:31.813 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:50:31.814 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.814 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.814 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:31.814 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.777890+0000", 2024-09-14T04:50:31.814 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.0890000000000002e-06 2024-09-14T04:50:31.814 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.814 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.814 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:31.814 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.777906+0000", 2024-09-14T04:50:31.814 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.6171000000000002e-05 2024-09-14T04:50:31.814 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.814 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.814 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:31.814 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.777919+0000", 2024-09-14T04:50:31.815 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2721999999999999e-05 2024-09-14T04:50:31.815 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:31.815 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:31.815 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:31.815 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.815 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.815 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop_reply(client.45632.0:178784 55.5 e1452/1419)", 2024-09-14T04:50:31.815 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:50:31.782455+0000", 2024-09-14T04:50:31.815 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.002441992, 2024-09-14T04:50:31.815 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0028715469999999999, 2024-09-14T04:50:31.815 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:31.815 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "queued for pg", 2024-09-14T04:50:31.815 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:31.816 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.816 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:31.816 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.782455+0000", 2024-09-14T04:50:31.816 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:31.816 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.816 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.816 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:31.816 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.782455+0000", 2024-09-14T04:50:31.816 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:31.816 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.816 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.816 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:31.816 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.782454+0000", 2024-09-14T04:50:31.816 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:50:31.817 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.817 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.817 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:31.817 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.782460+0000", 2024-09-14T04:50:31.817 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.4840000000000001e-06 2024-09-14T04:50:31.817 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.817 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.817 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:31.817 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.782471+0000", 2024-09-14T04:50:31.817 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0776e-05 2024-09-14T04:50:31.817 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.817 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.817 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:31.817 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.782478+0000", 2024-09-14T04:50:31.818 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.1559999999999998e-06 2024-09-14T04:50:31.818 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:31.818 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:31.818 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:31.818 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.818 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.818 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop_reply(client.45632.0:178785 55.0 e1452/1442)", 2024-09-14T04:50:31.818 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:50:31.782779+0000", 2024-09-14T04:50:31.818 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.0021174449999999999, 2024-09-14T04:50:31.819 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0025857789999999999, 2024-09-14T04:50:31.819 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:31.819 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "queued for pg", 2024-09-14T04:50:31.819 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:31.819 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.819 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:31.819 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.782779+0000", 2024-09-14T04:50:31.819 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:31.819 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.819 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.819 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:31.819 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.782779+0000", 2024-09-14T04:50:31.819 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:31.820 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.820 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.820 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:31.820 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.782779+0000", 2024-09-14T04:50:31.820 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:50:31.820 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.820 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.820 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:31.820 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.782783+0000", 2024-09-14T04:50:31.820 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8500000000000004e-06 2024-09-14T04:50:31.820 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.820 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.820 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:31.820 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.782791+0000", 2024-09-14T04:50:31.821 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.2390000000000002e-06 2024-09-14T04:50:31.821 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.821 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.821 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:31.821 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.782796+0000", 2024-09-14T04:50:31.821 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.0170000000000004e-06 2024-09-14T04:50:31.821 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:31.821 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:31.821 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:31.821 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.821 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.821 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop_reply(client.45632.0:178779 55.e e1452/1442)", 2024-09-14T04:50:31.821 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:50:31.783943+0000", 2024-09-14T04:50:31.822 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.00095399099999999995, 2024-09-14T04:50:31.822 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001463661, 2024-09-14T04:50:31.822 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:31.822 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "queued for pg", 2024-09-14T04:50:31.822 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:31.822 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.822 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:31.822 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.783943+0000", 2024-09-14T04:50:31.822 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:31.822 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.822 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.822 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:31.822 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.783943+0000", 2024-09-14T04:50:31.823 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:31.823 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.823 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.823 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:31.823 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.783942+0000", 2024-09-14T04:50:31.823 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:50:31.823 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.823 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.823 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:31.823 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.783947+0000", 2024-09-14T04:50:31.823 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8489999999999997e-06 2024-09-14T04:50:31.823 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.823 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.823 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:31.824 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.783957+0000", 2024-09-14T04:50:31.824 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.7119999999999995e-06 2024-09-14T04:50:31.824 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.824 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.824 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:31.824 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.783962+0000", 2024-09-14T04:50:31.824 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.8080000000000001e-06 2024-09-14T04:50:31.824 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:31.824 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:31.824 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:31.824 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.824 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.824 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:178769 55.5 55:a49fe0da:::benchmark_data_smithi039_297155_object54836:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1452)", 2024-09-14T04:50:31.825 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:50:31.751962+0000", 2024-09-14T04:50:31.825 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.032935015999999998, 2024-09-14T04:50:31.825 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.033489309000000002, 2024-09-14T04:50:31.825 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:31.825 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:50:31.825 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:31.825 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:50:31.825 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:31.825 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 178769 2024-09-14T04:50:31.825 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.825 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:31.825 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.825 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:31.825 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.751962+0000", 2024-09-14T04:50:31.826 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:31.826 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.826 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.826 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:31.826 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.751962+0000", 2024-09-14T04:50:31.826 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:31.826 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.826 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.826 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:31.826 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.751960+0000", 2024-09-14T04:50:31.826 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:50:31.826 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.826 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.826 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:31.826 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.751969+0000", 2024-09-14T04:50:31.827 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.9800000000000004e-06 2024-09-14T04:50:31.827 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.827 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.827 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:31.827 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.751984+0000", 2024-09-14T04:50:31.827 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.4714e-05 2024-09-14T04:50:31.827 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.827 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.827 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:31.827 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.751992+0000", 2024-09-14T04:50:31.827 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.8560000000000007e-06 2024-09-14T04:50:31.827 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.827 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.827 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:31.827 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.765118+0000", 2024-09-14T04:50:31.827 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.013126083 2024-09-14T04:50:31.828 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.828 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.828 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:31.828 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.765218+0000", 2024-09-14T04:50:31.828 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000100036 2024-09-14T04:50:31.828 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.828 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.828 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,1", 2024-09-14T04:50:31.828 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.765334+0000", 2024-09-14T04:50:31.828 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000116731 2024-09-14T04:50:31.828 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.828 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.828 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:31.829 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.776452+0000", 2024-09-14T04:50:31.829 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.011117141000000001 2024-09-14T04:50:31.829 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:31.829 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:31.829 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:31.829 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.829 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.829 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop_reply(client.45632.0:178785 55.0 e1452/1442)", 2024-09-14T04:50:31.829 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:50:31.784672+0000", 2024-09-14T04:50:31.829 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.00022478600000000001, 2024-09-14T04:50:31.829 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00084424800000000003, 2024-09-14T04:50:31.829 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:31.829 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "queued for pg", 2024-09-14T04:50:31.829 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:31.830 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.830 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:31.830 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.784672+0000", 2024-09-14T04:50:31.830 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:31.830 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.830 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.830 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:31.830 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.784672+0000", 2024-09-14T04:50:31.830 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:31.830 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.830 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.830 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:31.830 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.784671+0000", 2024-09-14T04:50:31.830 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:50:31.830 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.831 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.831 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:31.831 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.784677+0000", 2024-09-14T04:50:31.831 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.5960000000000003e-06 2024-09-14T04:50:31.831 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.831 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.831 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:31.831 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.784692+0000", 2024-09-14T04:50:31.831 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.4865000000000001e-05 2024-09-14T04:50:31.831 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:31.831 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:31.831 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:31.831 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:31.784699+0000", 2024-09-14T04:50:31.831 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.9179999999999997e-06 2024-09-14T04:50:31.832 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:31.832 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:31.832 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:31.832 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:31.832 INFO:teuthology.orchestra.run.smithi129.stdout: ], 2024-09-14T04:50:31.832 INFO:teuthology.orchestra.run.smithi129.stdout: "num_ops": 11 2024-09-14T04:50:31.832 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:50:31.849 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:31.865 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:31 smithi129 ceph-mon[101316]: pgmap v2647: 99 pgs: 99 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2024-09-14T04:50:31.865 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:31 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1287789469' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-14T04:50:31.881 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:31.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:31.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:31.983 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:32.024 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:32.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:32.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:32.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:31 smithi039 ceph-mon[106470]: pgmap v2647: 99 pgs: 99 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2024-09-14T04:50:32.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:31 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1287789469' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-14T04:50:32.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:31 smithi039 ceph-mon[111235]: pgmap v2647: 99 pgs: 99 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 816 op/s 2024-09-14T04:50:32.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:31 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1287789469' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-14T04:50:32.125 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:32.162 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:32.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:32.163 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:32.264 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:32.296 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:32.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:32.298 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:32.398 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:32.433 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:32.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:32.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:32.534 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:32.575 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:32.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:32.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:32.650 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-14T04:50:32.678 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:32.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:32.741 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:32.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:32.777 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:32 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1287789469' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-14T04:50:32.777 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:32 smithi039 ceph-mon[106470]: osdmap e1452: 8 total, 8 up, 8 in 2024-09-14T04:50:32.778 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:32 smithi039 ceph-mon[106470]: pgmap v2649: 99 pgs: 99 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2024-09-14T04:50:32.778 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:32 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:50:32.778 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:32 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1287789469' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-14T04:50:32.778 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:32 smithi039 ceph-mon[111235]: osdmap e1452: 8 total, 8 up, 8 in 2024-09-14T04:50:32.778 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:32 smithi039 ceph-mon[111235]: pgmap v2649: 99 pgs: 99 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2024-09-14T04:50:32.779 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:32 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:50:32.842 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:32.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:32.907 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:32.907 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:33.008 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:33.025 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:32 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1287789469' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-14T04:50:33.025 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:32 smithi129 ceph-mon[101316]: osdmap e1452: 8 total, 8 up, 8 in 2024-09-14T04:50:33.025 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:32 smithi129 ceph-mon[101316]: pgmap v2649: 99 pgs: 99 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2024-09-14T04:50:33.025 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:32 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:50:33.025 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:50:33.043 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:33.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:33.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:33.145 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:33.193 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:33.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:33.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:33.295 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:33.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:33.337 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:33.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:33.378 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:50:33.379 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:50:33.410 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:50:33.411 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:50:33.438 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:33.444 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:50:33.444 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:50:33.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:33.476 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:33.477 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:33.480 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:50:33.480 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:50:33.524 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:50:33.524 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:50:33.568 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:50:33.568 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:50:33.577 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:33.600 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:50:33.600 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:50:33.608 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:33.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:33.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:33.637 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:50:33.637 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:50:33.710 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:33.746 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:33.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:33.747 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:33.848 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:33.891 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:33.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:33.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:33.993 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:34.034 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:34.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:34.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:34.132 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:50:34.136 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:34.176 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:33 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:34.176 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:33 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:34.177 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:33 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:34.177 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:33 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:34.177 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:33 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3446143795' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:50:34.177 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:33 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:34.177 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:33 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:34.177 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:33 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:34.178 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:33 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:34.178 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:33 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3446143795' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:50:34.219 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:34.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:34.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:34.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:33 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:34.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:33 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:34.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:33 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:34.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:33 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:34.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:33 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3446143795' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:50:34.321 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:34.354 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:34.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:34.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:34.456 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:34.487 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:34.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:34.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:34.590 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:34.623 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:34.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:34.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:34.725 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:34.755 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:34.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:34.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:34.857 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:34.899 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:34.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:34.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:34.923 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:50:35.001 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:35.035 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:35.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:35.036 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:35.088 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:50:35.088 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:50:35.138 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:35.171 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:35.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:35.172 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:35.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:34 smithi129 ceph-mon[101316]: pgmap v2650: 99 pgs: 99 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-14T04:50:35.272 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:35.289 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:34 smithi039 ceph-mon[106470]: pgmap v2650: 99 pgs: 99 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-14T04:50:35.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:34 smithi039 ceph-mon[111235]: pgmap v2650: 99 pgs: 99 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s 2024-09-14T04:50:35.332 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:35.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:35.334 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:35.434 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:35.472 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:35.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:35.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:35.574 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:35.618 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:35.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:35.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:35.720 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:35.763 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:35.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:35.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:35.865 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:35.902 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:35.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:35.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:36.004 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:36.049 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:36.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:36.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:36.150 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:36.187 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:36.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:36.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:36.288 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:36.327 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:36.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:36.328 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:36.400 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:50:36.428 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:36.495 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:36.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:36.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:36.543 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-14T04:50:36.596 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:36.665 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:36.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:36.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:36.766 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:36.828 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:36.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:36.829 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:36.930 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:36.964 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:36.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:36.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:37.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:36 smithi039 ceph-mon[106470]: pgmap v2651: 99 pgs: 99 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 987 op/s 2024-09-14T04:50:37.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:36 smithi039 ceph-mon[111235]: pgmap v2651: 99 pgs: 99 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 987 op/s 2024-09-14T04:50:37.066 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:37.096 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:37.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:37.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:37.126 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:50:37.198 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:37.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:37.239 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:37.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:37.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:36 smithi129 ceph-mon[101316]: pgmap v2651: 99 pgs: 99 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 987 op/s 2024-09-14T04:50:37.340 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:37.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:37.377 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:37.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:37.478 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:37.508 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:37.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:37.509 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:37.610 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:37.651 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:37.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:37.652 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:37.752 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:37.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:37.801 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:37.801 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:37.809 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-14T04:50:37.810 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 0 1 2024-09-14T04:50:37.901 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:37.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:37.938 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:37.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:38.039 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:38.050 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:50:38.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:38.068 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:38.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:38.170 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:38.200 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:38.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:38.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:38.302 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:38.343 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:38.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:38.344 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:38.445 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:38.478 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:38.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:38.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:38.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:38 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:38.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:38 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:50:38.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:38 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:38.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:38 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:50:38.580 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:38.611 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:38.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:38.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:38.713 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:38.748 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:38.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:38.750 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:38.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:38 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:38.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:38 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:50:38.851 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:38.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:38.887 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:38.887 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:38.934 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:50:38.987 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:39.016 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:39.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:39.017 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:39.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:50:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:50:38] "GET /metrics HTTP/1.1" 200 36762 "" "Prometheus/2.51.0" 2024-09-14T04:50:39.117 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:39.151 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:39.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:39.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:39.253 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:39.281 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:39.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:39.282 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:39.315 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is set 2024-09-14T04:50:39.383 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:39.418 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:39.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:39.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:39.520 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:39.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:39.559 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:39.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:39.613 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:39 smithi039 ceph-mon[106470]: pgmap v2652: 99 pgs: 99 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2024-09-14T04:50:39.613 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:39 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/247692077' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:50:39.613 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:39 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:50:39.614 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:39 smithi039 ceph-mon[111235]: pgmap v2652: 99 pgs: 99 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2024-09-14T04:50:39.614 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:39 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/247692077' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:50:39.614 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:39 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:50:39.661 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:39.706 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:39.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:39.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:39.758 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:39 smithi129 ceph-mon[101316]: pgmap v2652: 99 pgs: 99 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 605 op/s 2024-09-14T04:50:39.758 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:39 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/247692077' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:50:39.758 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:39 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:50:39.809 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:39.841 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:39.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:39.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:39.942 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:39.989 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:39.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:39.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:40.090 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:40.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:40.141 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:40.142 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:40.243 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:40.280 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:40.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:40.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:40.382 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:40.411 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:40 smithi129 ceph-mon[101316]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:50:40.411 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:40 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:50:40.411 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:40 smithi129 ceph-mon[101316]: osdmap e1453: 8 total, 8 up, 8 in 2024-09-14T04:50:40.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:40.419 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:40.420 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:40.469 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:50:40.469 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:50:40.488 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:40 smithi039 ceph-mon[106470]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:50:40.488 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:40 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:50:40.488 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:40 smithi039 ceph-mon[106470]: osdmap e1453: 8 total, 8 up, 8 in 2024-09-14T04:50:40.488 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:40 smithi039 ceph-mon[111235]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:50:40.488 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:40 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:50:40.488 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:40 smithi039 ceph-mon[111235]: osdmap e1453: 8 total, 8 up, 8 in 2024-09-14T04:50:40.521 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:40.574 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:40.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:40.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:40.676 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:40.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:40.711 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:40.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:40.812 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:40.842 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:40.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:40.843 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:40.870 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:50:40.871 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:50:40.904 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:50:40.904 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:50:40.934 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:50:40.934 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:50:40.944 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:40.973 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:50:40.973 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:50:40.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:40.988 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:40.988 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:41.005 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:50:41.006 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:50:41.042 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:50:41.042 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:50:41.083 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:50:41.083 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:50:41.088 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:41.124 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:50:41.124 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:50:41.137 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:41.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:41.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:41.239 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:41.271 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:41.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:41.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:41.373 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:41.412 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:41.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:41.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:41.514 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:41.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:41 smithi129 ceph-mon[101316]: pgmap v2654: 99 pgs: 99 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2024-09-14T04:50:41.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:41 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4082127466' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:50:41.546 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:41.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:41.547 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:41.648 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:41.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:41.682 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:41.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:41.783 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:41.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:41 smithi039 ceph-mon[106470]: pgmap v2654: 99 pgs: 99 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2024-09-14T04:50:41.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:41 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4082127466' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:50:41.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:41 smithi039 ceph-mon[111235]: pgmap v2654: 99 pgs: 99 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 950 op/s 2024-09-14T04:50:41.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:41 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4082127466' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:50:41.817 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:41.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:41.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:41.868 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-14T04:50:41.919 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:41.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:41.977 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:41.977 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:42.078 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:42.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:42.137 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:42.138 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:42.216 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:50:42.238 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:42.281 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:42.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:42.283 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:42.383 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:42.418 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:42.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:42.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:42.520 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:42.552 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:42.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:42.553 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:42.560 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:50:42.654 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:42.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:42.712 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:42.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:42.813 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:42.854 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:42.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:42.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:42.956 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:43.000 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:43.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:43.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:43.102 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:43.143 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:43.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:43.144 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:43.244 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:43.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:43.299 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:43.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:43.401 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:43.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:43.437 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:43.437 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:43.538 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:43.573 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:43.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:43.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:43.675 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:43.713 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:43.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:43.715 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:43.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:43 smithi129 ceph-mon[101316]: pgmap v2655: 99 pgs: 99 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2024-09-14T04:50:43.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:43 smithi039 ceph-mon[106470]: pgmap v2655: 99 pgs: 99 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2024-09-14T04:50:43.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:43 smithi039 ceph-mon[111235]: pgmap v2655: 99 pgs: 99 active+clean; 3.8 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 853 op/s 2024-09-14T04:50:43.815 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:43.847 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:43.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:43.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:43.948 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:43.990 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:43.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:43.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:44.092 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:44.120 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:44.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:44.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:44.221 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:44.267 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:44.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:44.268 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:44.369 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:44.392 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:44.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:44.393 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:44.494 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:44.515 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:44.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:44.516 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:44.617 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:44.664 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:44.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:44.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:44.766 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:44.803 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:44.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:44.804 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:44.905 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:44.939 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:44.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:44.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:45.041 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:45.078 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:45.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:45.079 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:45.180 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:45.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:45.212 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:45.212 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:45.313 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:45.341 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:45.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:45.342 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:45.443 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:45.477 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:45.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:45.478 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:45.579 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:45.623 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:45.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:45.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:45.672 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:45 smithi039 ceph-mon[106470]: pgmap v2656: 99 pgs: 99 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 878 op/s 2024-09-14T04:50:45.673 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:45 smithi039 ceph-mon[111235]: pgmap v2656: 99 pgs: 99 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 878 op/s 2024-09-14T04:50:45.725 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:45.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:45.769 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:45.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:45.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:45 smithi129 ceph-mon[101316]: pgmap v2656: 99 pgs: 99 active+clean; 3.6 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 878 op/s 2024-09-14T04:50:45.869 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:45.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:45.918 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:45.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:46.019 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:46.057 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:46.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:46.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:46.158 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:46.192 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:46.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:46.193 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:46.294 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:46.334 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:46.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:46.335 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:46.436 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:46.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:46.469 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:46.469 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:46.570 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:46.613 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:46.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:46.614 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:46.715 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:46.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:46.760 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:46.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:46.861 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:46.900 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:46.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:46.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:47.002 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:47.023 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:47.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:47.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:47.125 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:47.170 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:47.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:47.171 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:47.272 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:47.310 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:47.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:47.311 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:47.387 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.0 to 1 (10000) 2024-09-14T04:50:47.413 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:47.445 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:47.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:47.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:47.547 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:47.586 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:47.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:47.587 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:47.688 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:47.710 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:47 smithi129 ceph-mon[101316]: pgmap v2657: 99 pgs: 99 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 879 op/s 2024-09-14T04:50:47.710 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:47 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/406506191' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:50:47.710 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:47 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:50:47.736 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:47 smithi039 ceph-mon[106470]: pgmap v2657: 99 pgs: 99 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 879 op/s 2024-09-14T04:50:47.736 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:47 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/406506191' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:50:47.736 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:47 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:50:47.737 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:47 smithi039 ceph-mon[111235]: pgmap v2657: 99 pgs: 99 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 879 op/s 2024-09-14T04:50:47.737 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:47 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/406506191' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:50:47.737 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:47 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:50:47.744 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:47.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:47.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:47.846 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:47.891 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:47.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:47.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:47.992 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:48.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:48.021 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:48.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:48.122 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:48.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:48.156 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:48.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:48.221 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:50:48.221 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [ 2024-09-14T04:50:48.222 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.222 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:192841 55.7 55:e33e3213:::benchmark_data_smithi039_297155_object68908:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1454)", 2024-09-14T04:50:48.222 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:50:48.210698+0000", 2024-09-14T04:50:48.222 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.0083023190000000007, 2024-09-14T04:50:48.222 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0083325570000000009, 2024-09-14T04:50:48.222 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:48.223 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:50:48.223 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:48.223 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:50:48.223 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:48.223 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 192841 2024-09-14T04:50:48.223 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.223 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:48.224 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.224 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:48.224 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.210698+0000", 2024-09-14T04:50:48.224 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:48.224 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.224 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.224 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:48.224 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.210698+0000", 2024-09-14T04:50:48.225 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:48.225 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.225 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.225 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:48.225 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.210696+0000", 2024-09-14T04:50:48.225 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:50:48.226 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.226 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.226 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:48.226 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.210705+0000", 2024-09-14T04:50:48.226 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.4489999999999999e-06 2024-09-14T04:50:48.226 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.227 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.227 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:48.227 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.210722+0000", 2024-09-14T04:50:48.227 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.7076999999999999e-05 2024-09-14T04:50:48.227 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.227 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.228 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:48.228 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.210730+0000", 2024-09-14T04:50:48.228 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.8539999999999997e-06 2024-09-14T04:50:48.228 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.228 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.228 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:48.228 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.210772+0000", 2024-09-14T04:50:48.229 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1987999999999998e-05 2024-09-14T04:50:48.229 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.229 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.229 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:48.229 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.210891+0000", 2024-09-14T04:50:48.229 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011911199999999999 2024-09-14T04:50:48.229 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.230 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.230 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:50:48.230 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.211086+0000", 2024-09-14T04:50:48.230 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000195668 2024-09-14T04:50:48.230 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.230 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.230 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:48.231 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.216697+0000", 2024-09-14T04:50:48.231 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0056110789999999997 2024-09-14T04:50:48.231 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.231 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.231 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:48.231 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.218632+0000", 2024-09-14T04:50:48.231 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0019345320000000001 2024-09-14T04:50:48.232 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:48.232 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:48.232 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:48.232 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.232 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.233 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:192842 55.d 55:be5017e0:::benchmark_data_smithi039_297155_object68909:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1454)", 2024-09-14T04:50:48.233 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:50:48.210922+0000", 2024-09-14T04:50:48.233 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.0080779730000000004, 2024-09-14T04:50:48.233 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0081995560000000002, 2024-09-14T04:50:48.233 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:48.233 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:50:48.233 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:48.233 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:50:48.234 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:48.234 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 192842 2024-09-14T04:50:48.234 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.234 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:48.234 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.234 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:48.234 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.210922+0000", 2024-09-14T04:50:48.234 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:48.234 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.234 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.234 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:48.234 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.210922+0000", 2024-09-14T04:50:48.234 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:48.235 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.235 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.235 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:48.235 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.210922+0000", 2024-09-14T04:50:48.235 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:50:48.235 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.235 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.235 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:48.235 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.210926+0000", 2024-09-14T04:50:48.235 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.4340000000000002e-06 2024-09-14T04:50:48.235 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.235 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.235 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:48.235 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.210934+0000", 2024-09-14T04:50:48.236 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.2260000000000005e-06 2024-09-14T04:50:48.236 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.236 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.236 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:48.236 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.210940+0000", 2024-09-14T04:50:48.236 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.0129999999999999e-06 2024-09-14T04:50:48.236 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.236 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.236 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:48.236 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.211072+0000", 2024-09-14T04:50:48.236 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000131409 2024-09-14T04:50:48.236 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.237 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.237 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:48.237 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.211196+0000", 2024-09-14T04:50:48.237 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012450800000000001 2024-09-14T04:50:48.237 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.237 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.237 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:50:48.237 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.211316+0000", 2024-09-14T04:50:48.237 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011998999999999999 2024-09-14T04:50:48.237 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.237 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.237 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:48.237 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.218296+0000", 2024-09-14T04:50:48.237 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0069796570000000002 2024-09-14T04:50:48.238 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.238 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.238 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:48.238 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.218590+0000", 2024-09-14T04:50:48.238 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00029374000000000002 2024-09-14T04:50:48.238 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:48.238 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:48.238 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:48.238 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.238 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.238 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.45632.0:192843 55.6 e1454/1419 55:6218c90a:::benchmark_data_smithi039_297155_object68910:head v 1454'12251, mlcod=1454'12249)", 2024-09-14T04:50:48.238 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:50:48.214602+0000", 2024-09-14T04:50:48.238 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.0043979559999999997, 2024-09-14T04:50:48.239 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0046051, 2024-09-14T04:50:48.239 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:48.239 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "started", 2024-09-14T04:50:48.239 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:48.239 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.239 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:48.239 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.214602+0000", 2024-09-14T04:50:48.239 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:48.240 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.240 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.240 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:48.240 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.214602+0000", 2024-09-14T04:50:48.240 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:48.240 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.240 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.240 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:48.241 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.214602+0000", 2024-09-14T04:50:48.241 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:50:48.241 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.241 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.241 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:48.241 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.214609+0000", 2024-09-14T04:50:48.241 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.0119999999999999e-06 2024-09-14T04:50:48.241 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.241 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.241 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:48.242 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.214623+0000", 2024-09-14T04:50:48.242 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.4783000000000001e-05 2024-09-14T04:50:48.242 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.242 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.242 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:48.242 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.214630+0000", 2024-09-14T04:50:48.242 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.6959999999999999e-06 2024-09-14T04:50:48.242 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.242 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.242 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:48.242 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.214698+0000", 2024-09-14T04:50:48.243 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8027000000000004e-05 2024-09-14T04:50:48.243 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.243 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.243 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:48.244 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.214737+0000", 2024-09-14T04:50:48.244 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8733000000000003e-05 2024-09-14T04:50:48.244 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:48.244 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:48.244 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:48.244 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.244 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.244 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.45632.0:192844 55.6 e1454/1419)", 2024-09-14T04:50:48.245 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:50:48.218070+0000", 2024-09-14T04:50:48.245 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.00093052499999999997, 2024-09-14T04:50:48.245 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0011926790000000001, 2024-09-14T04:50:48.245 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:48.245 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "queued for pg", 2024-09-14T04:50:48.245 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:48.245 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.245 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:48.245 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.218070+0000", 2024-09-14T04:50:48.245 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:48.245 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.245 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.245 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:48.246 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.218070+0000", 2024-09-14T04:50:48.246 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:48.246 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.246 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.246 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:48.246 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.218069+0000", 2024-09-14T04:50:48.246 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:50:48.246 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.246 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.246 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:48.246 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.218076+0000", 2024-09-14T04:50:48.246 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.5309999999999998e-06 2024-09-14T04:50:48.246 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.246 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.247 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:48.247 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.218090+0000", 2024-09-14T04:50:48.247 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3767e-05 2024-09-14T04:50:48.247 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.247 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.247 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:48.247 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.218096+0000", 2024-09-14T04:50:48.247 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.3409999999999999e-06 2024-09-14T04:50:48.247 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:48.247 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:48.247 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:48.247 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.247 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.247 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.45632.0:192849 55.c e1454/1419 55:323ce53f:::benchmark_data_smithi039_297155_object68916:head v 1454'11925, mlcod=1454'11922)", 2024-09-14T04:50:48.248 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:50:48.218193+0000", 2024-09-14T04:50:48.248 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.000807696, 2024-09-14T04:50:48.248 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001116969, 2024-09-14T04:50:48.248 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:48.248 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "started", 2024-09-14T04:50:48.248 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:48.248 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.248 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:48.248 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.218193+0000", 2024-09-14T04:50:48.248 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:48.248 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.248 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.248 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:48.249 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.218193+0000", 2024-09-14T04:50:48.249 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:48.249 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.249 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.249 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:48.249 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.218192+0000", 2024-09-14T04:50:48.249 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967296 2024-09-14T04:50:48.249 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.249 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.249 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:48.249 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.218196+0000", 2024-09-14T04:50:48.249 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6859999999999999e-06 2024-09-14T04:50:48.249 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.250 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.250 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:48.250 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.218201+0000", 2024-09-14T04:50:48.250 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.9459999999999997e-06 2024-09-14T04:50:48.250 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.250 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.250 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:48.250 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.218208+0000", 2024-09-14T04:50:48.250 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.1969999999999999e-06 2024-09-14T04:50:48.250 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.250 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.250 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:48.250 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.218662+0000", 2024-09-14T04:50:48.251 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00045415100000000002 2024-09-14T04:50:48.251 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.251 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.251 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:48.251 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.218680+0000", 2024-09-14T04:50:48.251 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.7757e-05 2024-09-14T04:50:48.251 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:48.251 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:48.251 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:48.251 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.251 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.251 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.45632.0:192852 55.3 e1454/1419)", 2024-09-14T04:50:48.251 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:50:48.219162+0000", 2024-09-14T04:50:48.251 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 4294967295.9998384, 2024-09-14T04:50:48.252 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00020336, 2024-09-14T04:50:48.252 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:48.252 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "queued for pg", 2024-09-14T04:50:48.252 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:48.252 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.252 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:48.252 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.219162+0000", 2024-09-14T04:50:48.252 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:48.252 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.252 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.252 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:48.252 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.219162+0000", 2024-09-14T04:50:48.252 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:48.253 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.253 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.253 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:48.253 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.219161+0000", 2024-09-14T04:50:48.253 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:50:48.253 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.253 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.253 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:48.253 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.219169+0000", 2024-09-14T04:50:48.253 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.3370000000000002e-06 2024-09-14T04:50:48.253 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.253 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.253 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:48.254 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.219180+0000", 2024-09-14T04:50:48.254 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1025e-05 2024-09-14T04:50:48.254 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.254 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.254 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:48.254 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.219186+0000", 2024-09-14T04:50:48.254 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.5899999999999996e-06 2024-09-14T04:50:48.254 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:48.254 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:48.254 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:48.254 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.254 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.254 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.45632.0:192853 55.2 e1454/1423 55:460708ee:::benchmark_data_smithi039_297155_object68920:head v 1454'11988, mlcod=1454'11986)", 2024-09-14T04:50:48.254 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:50:48.219203+0000", 2024-09-14T04:50:48.255 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 4294967295.9997978, 2024-09-14T04:50:48.255 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00020745200000000001, 2024-09-14T04:50:48.255 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:48.255 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "started", 2024-09-14T04:50:48.255 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:48.255 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.255 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:48.255 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.219203+0000", 2024-09-14T04:50:48.255 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:48.255 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.255 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.255 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:48.255 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.219203+0000", 2024-09-14T04:50:48.255 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:48.256 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.256 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.256 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:48.256 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.219202+0000", 2024-09-14T04:50:48.256 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:50:48.256 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.256 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.256 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:48.256 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.219206+0000", 2024-09-14T04:50:48.256 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9840000000000003e-06 2024-09-14T04:50:48.256 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.256 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.256 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:48.257 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.219213+0000", 2024-09-14T04:50:48.257 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.511e-06 2024-09-14T04:50:48.257 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.257 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.257 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:48.257 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.219217+0000", 2024-09-14T04:50:48.257 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6079999999999998e-06 2024-09-14T04:50:48.257 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.257 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.257 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:48.257 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.219275+0000", 2024-09-14T04:50:48.257 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.8038000000000002e-05 2024-09-14T04:50:48.257 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.257 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.258 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:48.258 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.219296+0000", 2024-09-14T04:50:48.258 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.0701000000000001e-05 2024-09-14T04:50:48.258 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:48.258 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:48.258 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:48.258 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.258 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.258 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.45632.0:192831 55.c e1454/1419 55:3733d476:::benchmark_data_smithi039_297155_object68898:head v 1454'11924, mlcod=1454'11922)", 2024-09-14T04:50:48.258 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:50:48.197339+0000", 2024-09-14T04:50:48.258 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.021661438000000002, 2024-09-14T04:50:48.258 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.022123804, 2024-09-14T04:50:48.258 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:48.258 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "started", 2024-09-14T04:50:48.259 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:48.259 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.259 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:48.259 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.197339+0000", 2024-09-14T04:50:48.259 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:48.259 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.259 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.259 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:48.259 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.197339+0000", 2024-09-14T04:50:48.259 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:48.259 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.259 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.259 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:48.260 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.197338+0000", 2024-09-14T04:50:48.260 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:50:48.260 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.260 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.260 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:48.260 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.197347+0000", 2024-09-14T04:50:48.260 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.3719999999999994e-06 2024-09-14T04:50:48.260 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.260 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.260 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:48.260 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.197450+0000", 2024-09-14T04:50:48.260 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010323900000000001 2024-09-14T04:50:48.260 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.261 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.261 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:48.261 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.197459+0000", 2024-09-14T04:50:48.261 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.7409999999999998e-06 2024-09-14T04:50:48.261 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.261 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.261 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:48.261 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.201316+0000", 2024-09-14T04:50:48.261 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0038565119999999999 2024-09-14T04:50:48.261 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:48.261 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:48.261 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:48.261 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:48.201333+0000", 2024-09-14T04:50:48.262 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.7822999999999999e-05 2024-09-14T04:50:48.262 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:48.262 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:48.262 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:48.262 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:48.262 INFO:teuthology.orchestra.run.smithi129.stdout: ], 2024-09-14T04:50:48.262 INFO:teuthology.orchestra.run.smithi129.stdout: "num_ops": 8 2024-09-14T04:50:48.262 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:50:48.262 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:48.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:48.297 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:48.297 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:48.398 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:48.438 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:48.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:48.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:48.540 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:48.553 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 1 1 2024-09-14T04:50:48.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:48.580 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:48.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:48.600 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:50:48.681 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:48.691 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:48 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:50:48.691 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:48 smithi039 ceph-mon[106470]: osdmap e1454: 8 total, 8 up, 8 in 2024-09-14T04:50:48.691 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:48 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:48.691 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:48 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:48.691 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:48 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:50:48.691 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:48 smithi039 ceph-mon[111235]: osdmap e1454: 8 total, 8 up, 8 in 2024-09-14T04:50:48.691 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:48 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:48.691 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:48 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:48.734 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:48.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:48.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:48.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:48 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:50:48.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:48 smithi129 ceph-mon[101316]: osdmap e1454: 8 total, 8 up, 8 in 2024-09-14T04:50:48.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:48 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:48.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:48 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:48.835 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:48.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:48.869 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:48.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:48.958 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-14T04:50:48.971 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:49.002 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:50:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:50:48] "GET /metrics HTTP/1.1" 200 36762 "" "Prometheus/2.51.0" 2024-09-14T04:50:49.046 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:49.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:49.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:49.147 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:49.192 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:49.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:49.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:49.294 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:49.348 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:49.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:49.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:49.359 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:50:49.360 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:50:49.451 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:49.493 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:49.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:49.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:49.595 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:49.634 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:49.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:49.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:49.675 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:49 smithi129 ceph-mon[101316]: pgmap v2659: 99 pgs: 99 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2024-09-14T04:50:49.737 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:49.777 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:49.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:49.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:49.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:49 smithi039 ceph-mon[106470]: pgmap v2659: 99 pgs: 99 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2024-09-14T04:50:49.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:49 smithi039 ceph-mon[111235]: pgmap v2659: 99 pgs: 99 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 613 op/s 2024-09-14T04:50:49.880 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:49.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:49.924 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:49.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:50.025 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:50.060 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:50.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:50.062 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:50.163 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:50.201 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:50.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:50.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:50.303 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:50.336 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:50.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:50.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:50.438 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:50.468 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:50.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:50.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:50.570 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:50.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:50.607 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:50.607 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:50.708 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:50.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:50.742 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:50.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:50.843 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:50.875 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:50.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:50.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:50.976 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:51.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:51.020 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:51.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:51.121 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:51.161 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:51.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:51.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:51.263 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:51.307 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:51.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:51.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:51.346 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:50:51.346 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [], 2024-09-14T04:50:51.346 INFO:teuthology.orchestra.run.smithi129.stdout: "complaint_time": 30, 2024-09-14T04:50:51.346 INFO:teuthology.orchestra.run.smithi129.stdout: "num_blocked_ops": 0 2024-09-14T04:50:51.346 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:50:51.408 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:51.446 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:51.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:51.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:51.548 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:51.578 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:51.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:51.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:51.667 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:50:51.681 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:51.707 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:51 smithi129 ceph-mon[101316]: pgmap v2660: 99 pgs: 99 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 848 op/s 2024-09-14T04:50:51.708 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:50:51.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:51.763 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:51.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:51 smithi039 ceph-mon[106470]: pgmap v2660: 99 pgs: 99 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 848 op/s 2024-09-14T04:50:51.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:51 smithi039 ceph-mon[111235]: pgmap v2660: 99 pgs: 99 active+clean; 3.4 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 848 op/s 2024-09-14T04:50:51.864 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:51.906 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:51.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:51.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:52.008 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:52.032 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-14T04:50:52.048 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:52.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:52.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:52.150 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:52.201 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:52.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:52.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:52.303 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:52.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:52.348 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:52.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:52.386 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:50:52.450 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:52.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:52.491 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:52.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:52.592 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:52.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:52.634 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:52.634 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:52.735 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:52.774 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:52.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:52.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:52.876 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:52.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:52.913 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:52.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:53.014 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:53.056 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:53.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:53.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:53.158 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:53.202 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:53.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:53.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:53.304 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:53.340 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:53.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:53.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:53.442 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:53.470 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:53.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:53.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:53.572 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:53.598 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:53 smithi039 ceph-mon[111235]: pgmap v2661: 99 pgs: 99 active+clean; 3.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 844 op/s 2024-09-14T04:50:53.598 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:53 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:53.598 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:53 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:50:53.598 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:53 smithi039 ceph-mon[106470]: pgmap v2661: 99 pgs: 99 active+clean; 3.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 844 op/s 2024-09-14T04:50:53.598 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:53 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:53.598 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:53 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:50:53.617 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:53.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:53.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:53.719 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:53.752 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:53.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:53.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:53.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:53 smithi129 ceph-mon[101316]: pgmap v2661: 99 pgs: 99 active+clean; 3.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 844 op/s 2024-09-14T04:50:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:53 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:53 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:50:53.854 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:53.889 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:53.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:53.890 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:53.991 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:54.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:54.032 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:54.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:54.133 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:54.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:54.168 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:54.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:54.269 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:54.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:54.314 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:54.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:54.415 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:54.459 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is set 2024-09-14T04:50:54.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:54.476 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:54.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:54.530 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:54 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:54.530 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:54 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:54.530 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:54 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2405735835' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:50:54.530 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:54 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:50:54.578 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:54.609 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:54.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:54.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:54.711 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:54.741 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:54 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:54.741 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:54 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:54.741 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:54 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2405735835' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:50:54.741 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:54 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:50:54.742 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:54 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:54.742 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:54 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:54.742 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:54 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2405735835' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:50:54.742 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:54 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:50:54.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:54.744 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:54.745 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:54.845 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:54.897 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:54.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:54.898 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:54.989 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:50:54.989 INFO:teuthology.orchestra.run.smithi129.stdout: "size": 20, 2024-09-14T04:50:54.989 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 600, 2024-09-14T04:50:54.989 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [ 2024-09-14T04:50:54.989 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:54.989 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:16557 55.1 55:8e242178:::benchmark_data_smithi039_297155_object16556:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1427)", 2024-09-14T04:50:54.989 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:48:06.681562+0000", 2024-09-14T04:50:54.989 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 168.303412493, 2024-09-14T04:50:54.989 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.119333601, 2024-09-14T04:50:54.989 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:54.990 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:54.990 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:54.990 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:50:54.990 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:54.990 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 16557 2024-09-14T04:50:54.990 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:54.990 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:54.990 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:54.990 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:54.990 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.681562+0000", 2024-09-14T04:50:54.990 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:54.990 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:54.990 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:54.991 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:54.991 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.681562+0000", 2024-09-14T04:50:54.991 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:54.991 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:54.991 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:54.991 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:54.991 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.681560+0000", 2024-09-14T04:50:54.991 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:50:54.991 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:54.991 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:54.991 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:54.991 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.681730+0000", 2024-09-14T04:50:54.991 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00016961100000000001 2024-09-14T04:50:54.991 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:54.992 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:54.992 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:54.992 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.681754+0000", 2024-09-14T04:50:54.992 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.4235000000000001e-05 2024-09-14T04:50:54.992 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:54.992 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:54.992 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:54.992 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.681761+0000", 2024-09-14T04:50:54.992 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.9700000000000002e-06 2024-09-14T04:50:54.992 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:54.992 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:54.992 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:54.992 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.682084+0000", 2024-09-14T04:50:54.993 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00032267999999999999 2024-09-14T04:50:54.993 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:54.993 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:54.993 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:54.993 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.682276+0000", 2024-09-14T04:50:54.993 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00019212400000000001 2024-09-14T04:50:54.993 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:54.993 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:54.993 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:50:54.993 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.682473+0000", 2024-09-14T04:50:54.993 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00019693099999999999 2024-09-14T04:50:54.993 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:54.993 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:54.994 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:54.994 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.685472+0000", 2024-09-14T04:50:54.994 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0029992500000000002 2024-09-14T04:50:54.994 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:54.994 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:54.994 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:54.994 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.685506+0000", 2024-09-14T04:50:54.994 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.4440000000000002e-05 2024-09-14T04:50:54.994 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:54.994 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:54.994 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:54.994 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.800768+0000", 2024-09-14T04:50:54.994 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.115261698 2024-09-14T04:50:54.994 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:54.995 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:54.995 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:54.995 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.800828+0000", 2024-09-14T04:50:54.995 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.0381999999999999e-05 2024-09-14T04:50:54.995 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:54.995 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:54.995 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:54.995 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.800895+0000", 2024-09-14T04:50:54.995 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.6755999999999996e-05 2024-09-14T04:50:54.995 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:54.995 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:54.995 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:54.995 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:54.995 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:54.996 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:16559 55.1 55:8d030f1d:::benchmark_data_smithi039_297155_object16558:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1427)", 2024-09-14T04:50:54.996 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:48:06.681781+0000", 2024-09-14T04:50:54.996 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 168.30319292600001, 2024-09-14T04:50:54.996 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.11931874200000001, 2024-09-14T04:50:54.996 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:54.996 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:54.996 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:54.996 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:50:54.996 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:54.996 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 16559 2024-09-14T04:50:54.996 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:54.996 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:54.996 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:54.996 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:54.997 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.681781+0000", 2024-09-14T04:50:54.997 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:54.997 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:54.997 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:54.997 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:54.997 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.681781+0000", 2024-09-14T04:50:54.997 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:54.997 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:54.997 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:54.997 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:54.997 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.681780+0000", 2024-09-14T04:50:54.997 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:50:54.997 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:54.997 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:54.998 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:54.998 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.681805+0000", 2024-09-14T04:50:54.998 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.4532000000000002e-05 2024-09-14T04:50:54.998 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:54.998 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:54.998 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:54.998 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.681820+0000", 2024-09-14T04:50:54.998 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5522000000000001e-05 2024-09-14T04:50:54.998 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:54.998 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:54.998 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:54.998 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.681826+0000", 2024-09-14T04:50:54.998 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.5559999999999998e-06 2024-09-14T04:50:54.999 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:54.999 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:54.999 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:54.999 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.683410+0000", 2024-09-14T04:50:54.999 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0015836909999999999 2024-09-14T04:50:54.999 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:54.999 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:54.999 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:54.999 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.683525+0000", 2024-09-14T04:50:54.999 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000115287 2024-09-14T04:50:54.999 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:54.999 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:54.999 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:50:55.000 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.683646+0000", 2024-09-14T04:50:55.000 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000120901 2024-09-14T04:50:55.000 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.000 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.000 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:55.000 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.685477+0000", 2024-09-14T04:50:55.000 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0018310100000000001 2024-09-14T04:50:55.000 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.000 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.000 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.000 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.687962+0000", 2024-09-14T04:50:55.000 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0024851420000000001 2024-09-14T04:50:55.000 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.000 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.001 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.001 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.800943+0000", 2024-09-14T04:50:55.001 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.112980646 2024-09-14T04:50:55.001 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.001 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.001 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:55.001 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.801047+0000", 2024-09-14T04:50:55.001 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000104824 2024-09-14T04:50:55.001 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.001 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.001 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:55.001 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.801100+0000", 2024-09-14T04:50:55.002 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2373999999999997e-05 2024-09-14T04:50:55.002 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.002 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:55.002 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.002 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.002 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.002 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:16562 55.1 55:810c3ea5:::benchmark_data_smithi039_297155_object16561:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1427)", 2024-09-14T04:50:55.002 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:48:06.682083+0000", 2024-09-14T04:50:55.002 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 168.30289062400001, 2024-09-14T04:50:55.002 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.119138888, 2024-09-14T04:50:55.002 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:55.002 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:55.002 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:55.003 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:50:55.003 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:55.003 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 16562 2024-09-14T04:50:55.003 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.003 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:55.003 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.003 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:55.003 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.682083+0000", 2024-09-14T04:50:55.003 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.003 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.003 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.003 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:55.003 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.682083+0000", 2024-09-14T04:50:55.004 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.004 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.004 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.004 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:55.004 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.682083+0000", 2024-09-14T04:50:55.004 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:50:55.004 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.004 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.004 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:55.004 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.682120+0000", 2024-09-14T04:50:55.004 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6857999999999998e-05 2024-09-14T04:50:55.004 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.004 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.005 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:55.005 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.682341+0000", 2024-09-14T04:50:55.005 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00022156500000000001 2024-09-14T04:50:55.005 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.005 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.005 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:55.005 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.682350+0000", 2024-09-14T04:50:55.005 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.5420000000000005e-06 2024-09-14T04:50:55.005 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.005 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.005 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:55.005 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.684350+0000", 2024-09-14T04:50:55.005 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0020004910000000001 2024-09-14T04:50:55.005 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.006 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.006 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:55.006 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.684784+0000", 2024-09-14T04:50:55.006 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00043395700000000001 2024-09-14T04:50:55.006 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.006 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.006 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:50:55.006 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.684920+0000", 2024-09-14T04:50:55.006 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013569400000000001 2024-09-14T04:50:55.006 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.006 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.006 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:55.006 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.685953+0000", 2024-09-14T04:50:55.006 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0010332920000000001 2024-09-14T04:50:55.007 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.007 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.007 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.007 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.688037+0000", 2024-09-14T04:50:55.007 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0020838409999999999 2024-09-14T04:50:55.007 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.007 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.007 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.007 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.801140+0000", 2024-09-14T04:50:55.007 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.113103381 2024-09-14T04:50:55.007 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.007 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.007 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:55.007 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.801186+0000", 2024-09-14T04:50:55.008 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.5927999999999998e-05 2024-09-14T04:50:55.008 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.008 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.008 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:55.008 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.801222+0000", 2024-09-14T04:50:55.008 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6009e-05 2024-09-14T04:50:55.008 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.008 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:55.008 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.008 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.008 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.008 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:56672 55.7 55:e78523a6:::benchmark_data_smithi039_297155_object56671:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1433)", 2024-09-14T04:50:55.008 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:48:30.378028+0000", 2024-09-14T04:50:55.008 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 144.60694649999999, 2024-09-14T04:50:55.009 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.12829610399999999, 2024-09-14T04:50:55.009 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:55.009 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:55.009 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:55.009 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:50:55.009 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:55.009 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 56672 2024-09-14T04:50:55.009 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.009 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:55.009 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.009 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:55.009 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.378028+0000", 2024-09-14T04:50:55.009 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.010 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.010 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.010 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:55.010 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.378028+0000", 2024-09-14T04:50:55.010 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.010 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.010 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.010 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:55.010 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.378026+0000", 2024-09-14T04:50:55.010 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:50:55.010 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.010 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.010 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:55.011 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.378082+0000", 2024-09-14T04:50:55.011 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.5429000000000001e-05 2024-09-14T04:50:55.011 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.011 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.011 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:55.011 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.378095+0000", 2024-09-14T04:50:55.011 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3189e-05 2024-09-14T04:50:55.011 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.011 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.011 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:55.011 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.378102+0000", 2024-09-14T04:50:55.011 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.498e-06 2024-09-14T04:50:55.012 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.012 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.012 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:55.012 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.378144+0000", 2024-09-14T04:50:55.012 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.2110000000000002e-05 2024-09-14T04:50:55.012 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.012 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.012 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:55.012 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.378278+0000", 2024-09-14T04:50:55.012 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000134127 2024-09-14T04:50:55.012 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.012 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.012 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:50:55.013 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.378414+0000", 2024-09-14T04:50:55.013 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000135279 2024-09-14T04:50:55.013 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.013 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.013 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.013 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.380890+0000", 2024-09-14T04:50:55.013 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0024766329999999998 2024-09-14T04:50:55.013 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.013 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.013 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:55.013 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.383168+0000", 2024-09-14T04:50:55.013 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0022781310000000001 2024-09-14T04:50:55.014 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.014 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.014 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.014 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.506225+0000", 2024-09-14T04:50:55.014 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.12305627399999999 2024-09-14T04:50:55.014 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.014 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.014 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:55.014 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.506284+0000", 2024-09-14T04:50:55.014 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9608000000000003e-05 2024-09-14T04:50:55.014 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.014 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.014 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:55.014 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.506324+0000", 2024-09-14T04:50:55.015 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9273999999999997e-05 2024-09-14T04:50:55.015 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.015 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:55.015 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.015 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.015 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.015 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:56674 55.7 55:e000770d:::benchmark_data_smithi039_297155_object56673:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1433)", 2024-09-14T04:50:55.015 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:48:30.380959+0000", 2024-09-14T04:50:55.015 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 144.60401495599999, 2024-09-14T04:50:55.015 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.127488041, 2024-09-14T04:50:55.015 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:55.015 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:55.015 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:55.016 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:50:55.016 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:55.016 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 56674 2024-09-14T04:50:55.016 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.016 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:55.016 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.016 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:55.016 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.380959+0000", 2024-09-14T04:50:55.016 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.016 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.016 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.016 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:55.016 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.380959+0000", 2024-09-14T04:50:55.017 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.017 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.017 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.017 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:55.017 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.380957+0000", 2024-09-14T04:50:55.017 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:55.018 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:50:55.018 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.018 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.019 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:55.019 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.380988+0000", 2024-09-14T04:50:55.019 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.0224999999999999e-05 2024-09-14T04:50:55.019 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.019 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.019 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:55.019 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.380997+0000", 2024-09-14T04:50:55.019 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.0429999999999996e-06 2024-09-14T04:50:55.019 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.019 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.019 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:55.019 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.381003+0000", 2024-09-14T04:50:55.019 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.2700000000000001e-06 2024-09-14T04:50:55.020 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.020 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.020 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:55.020 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.381055+0000", 2024-09-14T04:50:55.020 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2259999999999998e-05 2024-09-14T04:50:55.020 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.020 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.020 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:55.020 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.381154+0000", 2024-09-14T04:50:55.020 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.8317000000000001e-05 2024-09-14T04:50:55.020 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.020 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.020 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:50:55.021 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.381241+0000", 2024-09-14T04:50:55.021 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.7866000000000002e-05 2024-09-14T04:50:55.021 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.021 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.021 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.021 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.383232+0000", 2024-09-14T04:50:55.021 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00199096 2024-09-14T04:50:55.021 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.021 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.021 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:55.021 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.383401+0000", 2024-09-14T04:50:55.021 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00016826300000000001 2024-09-14T04:50:55.021 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.022 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.022 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.022 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.508360+0000", 2024-09-14T04:50:55.022 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.124959548 2024-09-14T04:50:55.022 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.022 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.022 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:55.022 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.508411+0000", 2024-09-14T04:50:55.022 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.0946000000000003e-05 2024-09-14T04:50:55.022 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.022 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.022 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:55.022 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.508447+0000", 2024-09-14T04:50:55.023 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5920000000000002e-05 2024-09-14T04:50:55.023 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.023 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:55.023 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.023 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.023 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.023 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:92037 55.7 55:e6c7ac93:::benchmark_data_smithi039_297155_object92036:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1439)", 2024-09-14T04:50:55.023 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:48:58.569106+0000", 2024-09-14T04:50:55.023 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 116.41586842300001, 2024-09-14T04:50:55.023 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.1991202, 2024-09-14T04:50:55.023 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:55.023 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:55.023 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:55.023 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:50:55.023 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:55.024 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 92037 2024-09-14T04:50:55.024 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.024 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:55.024 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.024 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:55.024 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.569106+0000", 2024-09-14T04:50:55.024 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.024 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.024 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.024 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:55.024 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.569106+0000", 2024-09-14T04:50:55.024 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.024 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.025 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.025 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:55.025 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.569104+0000", 2024-09-14T04:50:55.025 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:50:55.025 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.025 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.025 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:55.025 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.569135+0000", 2024-09-14T04:50:55.025 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.0352e-05 2024-09-14T04:50:55.025 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.025 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.025 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:55.025 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.569146+0000", 2024-09-14T04:50:55.025 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1778e-05 2024-09-14T04:50:55.026 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.026 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.026 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:55.026 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.569153+0000", 2024-09-14T04:50:55.026 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.5050000000000004e-06 2024-09-14T04:50:55.026 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.026 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.026 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:55.026 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.575078+0000", 2024-09-14T04:50:55.026 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0059247040000000003 2024-09-14T04:50:55.026 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.026 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.026 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:55.027 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.575248+0000", 2024-09-14T04:50:55.027 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00017037500000000001 2024-09-14T04:50:55.027 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.027 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.027 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:50:55.027 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.575400+0000", 2024-09-14T04:50:55.027 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00015239799999999999 2024-09-14T04:50:55.027 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.027 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.027 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.027 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.578349+0000", 2024-09-14T04:50:55.027 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.002948754 2024-09-14T04:50:55.027 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.027 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.028 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:55.028 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.581730+0000", 2024-09-14T04:50:55.028 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.003380723 2024-09-14T04:50:55.028 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.028 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.028 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.028 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.768120+0000", 2024-09-14T04:50:55.028 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.18639050300000001 2024-09-14T04:50:55.028 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.028 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.028 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:55.028 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.768187+0000", 2024-09-14T04:50:55.028 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.6332000000000001e-05 2024-09-14T04:50:55.028 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.029 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.029 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:55.029 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.768226+0000", 2024-09-14T04:50:55.029 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9112000000000001e-05 2024-09-14T04:50:55.029 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.029 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:55.029 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.029 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.029 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.029 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:92038 55.4 55:2508621a:::benchmark_data_smithi039_297155_object92037:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1439)", 2024-09-14T04:50:55.029 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:48:58.569228+0000", 2024-09-14T04:50:55.029 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 116.415746242, 2024-09-14T04:50:55.029 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.20580523000000001, 2024-09-14T04:50:55.030 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:55.030 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:55.030 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:55.030 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:50:55.030 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:55.030 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 92038 2024-09-14T04:50:55.030 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.030 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:55.030 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.030 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:55.030 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.569228+0000", 2024-09-14T04:50:55.030 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.030 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.030 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.031 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:55.031 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.569228+0000", 2024-09-14T04:50:55.031 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.031 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.031 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.031 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:55.031 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.569227+0000", 2024-09-14T04:50:55.031 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:50:55.031 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.031 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.031 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:55.031 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.569283+0000", 2024-09-14T04:50:55.031 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.6246000000000003e-05 2024-09-14T04:50:55.031 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.032 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.032 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:55.032 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.569291+0000", 2024-09-14T04:50:55.032 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.7870000000000002e-06 2024-09-14T04:50:55.032 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.032 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.032 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:55.032 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.569306+0000", 2024-09-14T04:50:55.032 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.525e-05 2024-09-14T04:50:55.032 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.032 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.032 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:55.032 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.569500+0000", 2024-09-14T04:50:55.033 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00019347799999999999 2024-09-14T04:50:55.033 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.033 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.033 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:55.033 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.569629+0000", 2024-09-14T04:50:55.033 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012876599999999999 2024-09-14T04:50:55.033 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.033 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.033 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,3", 2024-09-14T04:50:55.033 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.569745+0000", 2024-09-14T04:50:55.033 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000116031 2024-09-14T04:50:55.033 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.033 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.033 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.034 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.572895+0000", 2024-09-14T04:50:55.034 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0031508320000000001 2024-09-14T04:50:55.034 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.034 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.034 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:55.034 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.573137+0000", 2024-09-14T04:50:55.034 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00024122 2024-09-14T04:50:55.034 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.034 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.034 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.034 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.774948+0000", 2024-09-14T04:50:55.034 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.20181154900000001 2024-09-14T04:50:55.034 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.035 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.035 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:55.035 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.774998+0000", 2024-09-14T04:50:55.035 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.9942999999999999e-05 2024-09-14T04:50:55.035 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.035 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.035 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:55.035 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.775033+0000", 2024-09-14T04:50:55.035 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.4851999999999998e-05 2024-09-14T04:50:55.035 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.035 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:55.035 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.035 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.036 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.036 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:92039 55.7 55:e4b1559c:::benchmark_data_smithi039_297155_object92038:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1439)", 2024-09-14T04:50:55.036 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:48:58.569333+0000", 2024-09-14T04:50:55.036 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 116.415640771, 2024-09-14T04:50:55.036 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.21139920700000001, 2024-09-14T04:50:55.036 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:55.036 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:55.036 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:55.036 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:50:55.036 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:55.036 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 92039 2024-09-14T04:50:55.036 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.037 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:55.037 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.037 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:55.037 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.569333+0000", 2024-09-14T04:50:55.037 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.037 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.037 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.037 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:55.037 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.569333+0000", 2024-09-14T04:50:55.037 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.037 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.037 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.038 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:55.038 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.569333+0000", 2024-09-14T04:50:55.038 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:50:55.038 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.038 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.038 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:55.038 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.569449+0000", 2024-09-14T04:50:55.038 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000116632 2024-09-14T04:50:55.038 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.038 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.038 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:55.038 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.569458+0000", 2024-09-14T04:50:55.038 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.5210000000000002e-06 2024-09-14T04:50:55.039 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.039 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.039 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:55.039 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.569465+0000", 2024-09-14T04:50:55.039 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8190000000000003e-06 2024-09-14T04:50:55.039 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.039 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.039 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:55.039 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.578375+0000", 2024-09-14T04:50:55.039 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0089103859999999993 2024-09-14T04:50:55.039 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.039 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.039 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:55.040 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.578510+0000", 2024-09-14T04:50:55.040 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013535700000000001 2024-09-14T04:50:55.040 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.040 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.040 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:50:55.040 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.578655+0000", 2024-09-14T04:50:55.040 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00014447 2024-09-14T04:50:55.040 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.040 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.040 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.040 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.581778+0000", 2024-09-14T04:50:55.041 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.003123139 2024-09-14T04:50:55.041 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.041 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.041 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:55.041 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.584360+0000", 2024-09-14T04:50:55.041 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0025822800000000002 2024-09-14T04:50:55.041 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.041 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.041 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.041 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.780676+0000", 2024-09-14T04:50:55.041 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.19631622800000001 2024-09-14T04:50:55.041 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.041 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.042 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:55.042 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.780710+0000", 2024-09-14T04:50:55.042 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3321999999999997e-05 2024-09-14T04:50:55.042 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.042 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.042 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:55.042 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.780732+0000", 2024-09-14T04:50:55.042 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.2708999999999999e-05 2024-09-14T04:50:55.042 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.042 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:55.042 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.042 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.043 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.043 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:131291 55.d 55:b9ad60fd:::benchmark_data_smithi039_297155_object7358:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1443)", 2024-09-14T04:50:55.043 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:49:38.986867+0000", 2024-09-14T04:50:55.043 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 75.998107337999997, 2024-09-14T04:50:55.043 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.118317478, 2024-09-14T04:50:55.043 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:55.043 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:55.043 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:55.043 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:50:55.043 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:55.043 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 131291 2024-09-14T04:50:55.043 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.043 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:55.044 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.044 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:55.044 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.986867+0000", 2024-09-14T04:50:55.044 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.044 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.044 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.044 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:55.044 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.986867+0000", 2024-09-14T04:50:55.044 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.044 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.044 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.045 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:55.045 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.986865+0000", 2024-09-14T04:50:55.045 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:50:55.045 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.045 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.045 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:55.045 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.986871+0000", 2024-09-14T04:50:55.045 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.3300000000000004e-06 2024-09-14T04:50:55.045 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.045 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.045 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:55.045 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.986884+0000", 2024-09-14T04:50:55.046 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2191000000000001e-05 2024-09-14T04:50:55.046 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.046 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.046 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:55.046 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.986890+0000", 2024-09-14T04:50:55.046 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.4409999999999998e-06 2024-09-14T04:50:55.046 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.046 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.046 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:55.046 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.986937+0000", 2024-09-14T04:50:55.046 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7079000000000001e-05 2024-09-14T04:50:55.047 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.047 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.047 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:55.047 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.987045+0000", 2024-09-14T04:50:55.047 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000107741 2024-09-14T04:50:55.047 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.047 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.047 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:50:55.047 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.987144+0000", 2024-09-14T04:50:55.047 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.9537000000000006e-05 2024-09-14T04:50:55.047 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.047 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.047 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:55.047 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.990691+0000", 2024-09-14T04:50:55.048 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0035470100000000002 2024-09-14T04:50:55.048 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.048 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.048 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.048 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.997799+0000", 2024-09-14T04:50:55.048 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0071071349999999997 2024-09-14T04:50:55.048 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.048 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.048 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.048 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.105101+0000", 2024-09-14T04:50:55.048 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.107302144 2024-09-14T04:50:55.048 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.048 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.049 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:55.049 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.105147+0000", 2024-09-14T04:50:55.049 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.5970999999999997e-05 2024-09-14T04:50:55.049 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.049 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.049 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:55.049 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.105184+0000", 2024-09-14T04:50:55.049 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7419000000000002e-05 2024-09-14T04:50:55.049 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.049 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:55.049 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.049 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.049 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.050 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:131297 55.d 55:bed42fa5:::benchmark_data_smithi039_297155_object7364:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1443)", 2024-09-14T04:50:55.050 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:49:38.993550+0000", 2024-09-14T04:50:55.050 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 75.991423562999998, 2024-09-14T04:50:55.050 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.122918954, 2024-09-14T04:50:55.050 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:55.050 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:55.050 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:55.050 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:50:55.050 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:55.050 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 131297 2024-09-14T04:50:55.050 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.050 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:55.050 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.051 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:55.051 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.993550+0000", 2024-09-14T04:50:55.051 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.051 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.051 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.051 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:55.051 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.993550+0000", 2024-09-14T04:50:55.051 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.051 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.051 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.051 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:55.051 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.993549+0000", 2024-09-14T04:50:55.052 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:50:55.052 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.052 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.052 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:55.052 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.993555+0000", 2024-09-14T04:50:55.052 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.4790000000000001e-06 2024-09-14T04:50:55.052 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.052 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.052 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:55.052 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.993570+0000", 2024-09-14T04:50:55.052 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.4698000000000001e-05 2024-09-14T04:50:55.052 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.052 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.052 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:55.053 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.993577+0000", 2024-09-14T04:50:55.053 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.3730000000000004e-06 2024-09-14T04:50:55.053 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.053 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.053 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:55.053 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.993616+0000", 2024-09-14T04:50:55.053 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8130000000000003e-05 2024-09-14T04:50:55.053 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.053 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.053 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:55.053 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.993715+0000", 2024-09-14T04:50:55.053 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.9098000000000006e-05 2024-09-14T04:50:55.054 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.054 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.054 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:50:55.054 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.993815+0000", 2024-09-14T04:50:55.054 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000100525 2024-09-14T04:50:55.054 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.054 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.054 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:55.054 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.997956+0000", 2024-09-14T04:50:55.054 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0041404229999999998 2024-09-14T04:50:55.054 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.054 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.054 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.054 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.011890+0000", 2024-09-14T04:50:55.055 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.013934528999999999 2024-09-14T04:50:55.055 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.055 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.055 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.055 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.116388+0000", 2024-09-14T04:50:55.055 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.104497731 2024-09-14T04:50:55.055 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.055 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.055 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:55.055 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.116434+0000", 2024-09-14T04:50:55.055 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6042999999999999e-05 2024-09-14T04:50:55.055 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.055 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.055 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:55.056 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.116469+0000", 2024-09-14T04:50:55.056 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5444999999999997e-05 2024-09-14T04:50:55.056 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.056 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:55.056 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.056 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.056 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.056 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:131306 55.d 55:be20f68d:::benchmark_data_smithi039_297155_object7373:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1443)", 2024-09-14T04:50:55.056 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:49:39.003915+0000", 2024-09-14T04:50:55.056 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 75.981059314999996, 2024-09-14T04:50:55.056 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.115943805, 2024-09-14T04:50:55.056 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:55.056 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:55.057 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:55.057 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:50:55.057 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:55.057 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 131306 2024-09-14T04:50:55.057 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.057 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:55.057 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.057 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:55.057 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.003915+0000", 2024-09-14T04:50:55.057 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.057 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.057 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.057 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:55.058 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.003915+0000", 2024-09-14T04:50:55.058 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.058 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.058 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.058 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:55.058 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.003913+0000", 2024-09-14T04:50:55.058 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:50:55.058 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.058 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.058 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:55.058 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.003919+0000", 2024-09-14T04:50:55.058 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.254e-06 2024-09-14T04:50:55.058 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.058 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.058 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:55.058 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.003928+0000", 2024-09-14T04:50:55.059 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.2699999999999993e-06 2024-09-14T04:50:55.059 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.059 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.059 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:55.059 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.003933+0000", 2024-09-14T04:50:55.059 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.0440000000000003e-06 2024-09-14T04:50:55.059 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.059 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.059 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:55.059 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.003968+0000", 2024-09-14T04:50:55.059 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5401000000000002e-05 2024-09-14T04:50:55.059 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.059 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.060 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:55.060 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.004049+0000", 2024-09-14T04:50:55.060 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.0302000000000005e-05 2024-09-14T04:50:55.060 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.060 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.060 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:50:55.060 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.004122+0000", 2024-09-14T04:50:55.060 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.3386999999999994e-05 2024-09-14T04:50:55.060 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.060 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.060 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:55.060 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.008765+0000", 2024-09-14T04:50:55.060 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0046425980000000004 2024-09-14T04:50:55.061 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.061 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.061 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.061 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.012679+0000", 2024-09-14T04:50:55.061 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0039144829999999999 2024-09-14T04:50:55.061 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.061 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.061 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.061 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.119775+0000", 2024-09-14T04:50:55.061 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.10709581 2024-09-14T04:50:55.061 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.061 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.061 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:55.061 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.119821+0000", 2024-09-14T04:50:55.063 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.5784000000000001e-05 2024-09-14T04:50:55.063 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.063 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.063 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:55.063 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.119859+0000", 2024-09-14T04:50:55.063 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7824999999999998e-05 2024-09-14T04:50:55.063 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.063 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:55.063 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.063 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.064 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.064 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:149714 55.a 55:585bdded:::benchmark_data_smithi039_297155_object25781:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1447)", 2024-09-14T04:50:55.064 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:49:58.462329+0000", 2024-09-14T04:50:55.064 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 56.522644733, 2024-09-14T04:50:55.064 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.35080457599999998, 2024-09-14T04:50:55.064 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:55.064 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:55.064 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:55.064 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:50:55.064 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:55.064 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 149714 2024-09-14T04:50:55.064 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.064 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:55.065 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.065 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:55.065 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.462329+0000", 2024-09-14T04:50:55.065 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.065 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.065 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.065 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:55.065 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.462329+0000", 2024-09-14T04:50:55.065 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.065 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.065 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.065 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:55.065 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.462327+0000", 2024-09-14T04:50:55.065 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:50:55.066 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.066 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.066 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:55.066 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.462334+0000", 2024-09-14T04:50:55.066 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.888e-06 2024-09-14T04:50:55.066 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.066 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.067 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:55.068 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.462348+0000", 2024-09-14T04:50:55.068 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3735e-05 2024-09-14T04:50:55.068 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.068 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.068 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:55.068 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.462355+0000", 2024-09-14T04:50:55.068 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.9870000000000002e-06 2024-09-14T04:50:55.068 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.068 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.068 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:55.068 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.463626+0000", 2024-09-14T04:50:55.068 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0012708610000000001 2024-09-14T04:50:55.069 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.069 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.069 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:55.069 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.463774+0000", 2024-09-14T04:50:55.069 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00014832100000000001 2024-09-14T04:50:55.069 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.069 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.069 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,3", 2024-09-14T04:50:55.069 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.463895+0000", 2024-09-14T04:50:55.069 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012069399999999999 2024-09-14T04:50:55.069 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.069 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.070 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.070 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471453+0000", 2024-09-14T04:50:55.070 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0075585490000000003 2024-09-14T04:50:55.070 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.070 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.070 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:55.070 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471460+0000", 2024-09-14T04:50:55.070 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8719999999999996e-06 2024-09-14T04:50:55.070 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.070 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.070 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.070 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.813005+0000", 2024-09-14T04:50:55.070 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.34154447599999999 2024-09-14T04:50:55.071 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.071 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.071 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:55.071 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.813087+0000", 2024-09-14T04:50:55.071 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.2387999999999998e-05 2024-09-14T04:50:55.071 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.071 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.071 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:55.071 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.813134+0000", 2024-09-14T04:50:55.071 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6684000000000002e-05 2024-09-14T04:50:55.071 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.071 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:55.071 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.072 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.072 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.072 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:149716 55.a 55:53418830:::benchmark_data_smithi039_297155_object25783:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1447)", 2024-09-14T04:50:55.072 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:49:58.462361+0000", 2024-09-14T04:50:55.072 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 56.522613096000001, 2024-09-14T04:50:55.072 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.35768450099999999, 2024-09-14T04:50:55.072 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:55.072 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:55.072 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:55.072 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:50:55.072 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:55.072 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 149716 2024-09-14T04:50:55.072 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.073 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:55.073 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.073 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:55.073 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.462361+0000", 2024-09-14T04:50:55.073 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.073 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.073 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.073 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:55.073 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.462361+0000", 2024-09-14T04:50:55.073 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.073 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.073 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.073 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:55.074 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.462361+0000", 2024-09-14T04:50:55.074 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:50:55.074 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.074 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.074 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:55.074 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.462363+0000", 2024-09-14T04:50:55.074 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.2649999999999999e-06 2024-09-14T04:50:55.074 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.074 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.074 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:55.074 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.462368+0000", 2024-09-14T04:50:55.074 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.3539999999999999e-06 2024-09-14T04:50:55.074 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.075 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.075 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:55.075 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.462373+0000", 2024-09-14T04:50:55.075 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.3939999999999998e-06 2024-09-14T04:50:55.075 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.075 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.075 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:55.075 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.467268+0000", 2024-09-14T04:50:55.075 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.004895234 2024-09-14T04:50:55.075 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.075 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.075 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:55.076 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.467997+0000", 2024-09-14T04:50:55.076 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00072953799999999995 2024-09-14T04:50:55.076 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.076 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.076 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,3", 2024-09-14T04:50:55.076 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.468102+0000", 2024-09-14T04:50:55.076 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000104978 2024-09-14T04:50:55.076 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.076 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.076 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.076 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.473040+0000", 2024-09-14T04:50:55.076 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0049380559999999997 2024-09-14T04:50:55.076 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.077 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.077 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:55.077 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.475341+0000", 2024-09-14T04:50:55.077 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0023002750000000001 2024-09-14T04:50:55.077 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.077 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.077 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.077 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.819968+0000", 2024-09-14T04:50:55.077 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.34462760799999997 2024-09-14T04:50:55.077 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.077 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.077 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:55.077 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.820010+0000", 2024-09-14T04:50:55.077 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1309e-05 2024-09-14T04:50:55.077 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.078 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.078 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:55.078 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.820045+0000", 2024-09-14T04:50:55.078 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5831000000000003e-05 2024-09-14T04:50:55.078 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.078 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:55.078 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.078 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.078 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.078 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:149720 55.4 55:25ea9b58:::benchmark_data_smithi039_297155_object25787:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1447)", 2024-09-14T04:50:55.078 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:49:58.467387+0000", 2024-09-14T04:50:55.078 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 56.517586987000001, 2024-09-14T04:50:55.078 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.34366680700000002, 2024-09-14T04:50:55.079 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:55.079 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:55.079 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:55.079 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:50:55.079 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:55.079 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 149720 2024-09-14T04:50:55.079 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.079 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:55.079 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.079 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:55.079 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.467387+0000", 2024-09-14T04:50:55.079 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.079 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.080 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.080 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:55.080 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.467387+0000", 2024-09-14T04:50:55.080 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.080 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.080 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.080 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:55.080 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.467386+0000", 2024-09-14T04:50:55.080 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:50:55.080 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.080 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.080 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:55.080 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.467392+0000", 2024-09-14T04:50:55.080 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.0809999999999999e-06 2024-09-14T04:50:55.081 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.081 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.081 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:55.081 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.467401+0000", 2024-09-14T04:50:55.081 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.4609999999999997e-06 2024-09-14T04:50:55.081 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.081 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.081 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:55.081 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.467408+0000", 2024-09-14T04:50:55.081 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.3570000000000003e-06 2024-09-14T04:50:55.081 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.081 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.082 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:55.082 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.467448+0000", 2024-09-14T04:50:55.082 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0255e-05 2024-09-14T04:50:55.082 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.082 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.082 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:55.082 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.467545+0000", 2024-09-14T04:50:55.082 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.7424999999999996e-05 2024-09-14T04:50:55.082 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.082 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.082 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,3", 2024-09-14T04:50:55.082 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.469890+0000", 2024-09-14T04:50:55.082 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.002344984 2024-09-14T04:50:55.083 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.083 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.083 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:55.083 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.477279+0000", 2024-09-14T04:50:55.083 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0073887639999999999 2024-09-14T04:50:55.083 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.083 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.083 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.083 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.477318+0000", 2024-09-14T04:50:55.083 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8495999999999999e-05 2024-09-14T04:50:55.083 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.083 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.084 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.084 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.810969+0000", 2024-09-14T04:50:55.084 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33365140500000001 2024-09-14T04:50:55.084 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.084 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.084 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:55.084 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.811017+0000", 2024-09-14T04:50:55.084 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7828999999999999e-05 2024-09-14T04:50:55.084 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.084 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.084 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:55.084 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.811054+0000", 2024-09-14T04:50:55.084 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7015999999999999e-05 2024-09-14T04:50:55.085 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.085 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:55.085 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.085 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.085 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.085 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:149723 55.4 55:2fc5bef1:::benchmark_data_smithi039_297155_object25790:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1447)", 2024-09-14T04:50:55.085 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:49:58.471232+0000", 2024-09-14T04:50:55.085 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 56.513741930999998, 2024-09-14T04:50:55.085 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.34963855999999999, 2024-09-14T04:50:55.085 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:55.085 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:55.085 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:55.085 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:50:55.086 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:55.086 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 149723 2024-09-14T04:50:55.086 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.086 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:55.086 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.086 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:55.086 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471232+0000", 2024-09-14T04:50:55.086 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.086 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.086 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.086 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:55.086 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471232+0000", 2024-09-14T04:50:55.086 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.087 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.087 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.087 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:55.087 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471231+0000", 2024-09-14T04:50:55.087 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:50:55.087 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.087 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.087 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:55.087 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471237+0000", 2024-09-14T04:50:55.087 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8730000000000001e-06 2024-09-14T04:50:55.087 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.087 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.087 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:55.087 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471252+0000", 2024-09-14T04:50:55.088 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.4996e-05 2024-09-14T04:50:55.088 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.088 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.088 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:55.088 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471259+0000", 2024-09-14T04:50:55.088 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.956e-06 2024-09-14T04:50:55.088 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.088 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.088 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:55.088 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.473268+0000", 2024-09-14T04:50:55.088 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.002008489 2024-09-14T04:50:55.088 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.088 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.088 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:55.089 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.473377+0000", 2024-09-14T04:50:55.089 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010955699999999999 2024-09-14T04:50:55.089 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.089 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.089 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,3", 2024-09-14T04:50:55.089 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.473485+0000", 2024-09-14T04:50:55.089 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010726999999999999 2024-09-14T04:50:55.089 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.089 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.089 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:55.089 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.477416+0000", 2024-09-14T04:50:55.089 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0039313739999999996 2024-09-14T04:50:55.089 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.090 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.090 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.090 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.492154+0000", 2024-09-14T04:50:55.090 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.014738335 2024-09-14T04:50:55.090 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.090 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.090 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.090 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.820787+0000", 2024-09-14T04:50:55.090 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.328632704 2024-09-14T04:50:55.090 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.090 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.090 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:55.090 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.820833+0000", 2024-09-14T04:50:55.090 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6109999999999997e-05 2024-09-14T04:50:55.090 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.090 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.091 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:55.091 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.820871+0000", 2024-09-14T04:50:55.091 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7437000000000001e-05 2024-09-14T04:50:55.091 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.091 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:55.091 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.091 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.091 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.091 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:149727 55.7 55:e5894293:::benchmark_data_smithi039_297155_object25794:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1447)", 2024-09-14T04:50:55.091 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:49:58.471708+0000", 2024-09-14T04:50:55.091 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 56.513265548, 2024-09-14T04:50:55.091 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.34548536699999999, 2024-09-14T04:50:55.091 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:55.091 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:55.091 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:55.092 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:50:55.092 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:55.092 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 149727 2024-09-14T04:50:55.092 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.092 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:55.092 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.092 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:55.092 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471708+0000", 2024-09-14T04:50:55.092 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.092 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.092 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.092 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:55.092 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471708+0000", 2024-09-14T04:50:55.092 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.092 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.093 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.093 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:55.093 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471708+0000", 2024-09-14T04:50:55.093 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:50:55.093 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.093 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.093 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:55.093 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471712+0000", 2024-09-14T04:50:55.093 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.3649999999999997e-06 2024-09-14T04:50:55.093 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.093 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.093 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:55.093 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471723+0000", 2024-09-14T04:50:55.093 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0825000000000001e-05 2024-09-14T04:50:55.094 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.094 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.094 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:55.094 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471728+0000", 2024-09-14T04:50:55.094 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.5589999999999999e-06 2024-09-14T04:50:55.094 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.094 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.094 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:55.094 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471755+0000", 2024-09-14T04:50:55.094 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.7868000000000002e-05 2024-09-14T04:50:55.094 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.094 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.094 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:55.095 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471830+0000", 2024-09-14T04:50:55.095 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.5081999999999997e-05 2024-09-14T04:50:55.095 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.095 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.095 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:50:55.095 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471899+0000", 2024-09-14T04:50:55.095 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8860000000000001e-05 2024-09-14T04:50:55.095 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.095 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.095 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:55.095 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.478613+0000", 2024-09-14T04:50:55.095 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.006713498 2024-09-14T04:50:55.095 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.096 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.096 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.096 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.480092+0000", 2024-09-14T04:50:55.096 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001479181 2024-09-14T04:50:55.096 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.096 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.096 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.096 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.817114+0000", 2024-09-14T04:50:55.096 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33702223599999998 2024-09-14T04:50:55.096 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.096 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.096 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:55.096 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.817160+0000", 2024-09-14T04:50:55.097 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6199999999999998e-05 2024-09-14T04:50:55.097 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.097 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.097 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:55.097 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.817194+0000", 2024-09-14T04:50:55.097 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3414999999999998e-05 2024-09-14T04:50:55.097 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.097 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:55.097 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.097 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.097 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.097 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:149731 55.7 55:ec89c09c:::benchmark_data_smithi039_297155_object25798:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1447)", 2024-09-14T04:50:55.097 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:49:58.474658+0000", 2024-09-14T04:50:55.098 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 56.510315704999996, 2024-09-14T04:50:55.098 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.34318062199999999, 2024-09-14T04:50:55.098 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:55.098 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:55.098 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:55.098 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:50:55.098 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:55.098 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 149731 2024-09-14T04:50:55.098 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.098 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:55.098 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.098 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:55.098 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.474658+0000", 2024-09-14T04:50:55.099 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.099 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.099 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.099 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:55.099 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.474658+0000", 2024-09-14T04:50:55.099 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.099 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.099 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.099 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:55.099 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.474657+0000", 2024-09-14T04:50:55.099 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:50:55.099 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.099 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.099 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:55.100 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.474663+0000", 2024-09-14T04:50:55.100 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.0390000000000001e-06 2024-09-14T04:50:55.100 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.100 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.100 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:55.100 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.474673+0000", 2024-09-14T04:50:55.100 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0284e-05 2024-09-14T04:50:55.100 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.100 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.100 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:55.100 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.474679+0000", 2024-09-14T04:50:55.100 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.1720000000000004e-06 2024-09-14T04:50:55.100 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.101 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.101 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:55.101 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.475315+0000", 2024-09-14T04:50:55.101 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00063524000000000002 2024-09-14T04:50:55.101 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.101 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.101 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:55.101 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.475413+0000", 2024-09-14T04:50:55.101 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.8039000000000001e-05 2024-09-14T04:50:55.101 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.101 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.101 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:50:55.101 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.475511+0000", 2024-09-14T04:50:55.101 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.8388999999999996e-05 2024-09-14T04:50:55.102 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.102 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.102 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:55.102 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.478928+0000", 2024-09-14T04:50:55.102 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0034173020000000001 2024-09-14T04:50:55.102 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.102 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.102 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.102 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.480126+0000", 2024-09-14T04:50:55.102 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0011971449999999999 2024-09-14T04:50:55.102 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.102 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.102 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.103 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.817755+0000", 2024-09-14T04:50:55.103 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.337629293 2024-09-14T04:50:55.103 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.103 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.103 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:55.103 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.817810+0000", 2024-09-14T04:50:55.103 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.5318000000000001e-05 2024-09-14T04:50:55.103 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.103 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.103 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:55.103 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.817839+0000", 2024-09-14T04:50:55.103 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.8728999999999999e-05 2024-09-14T04:50:55.103 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.104 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:55.104 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.104 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.104 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.104 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:149732 55.a 55:5aa14efc:::benchmark_data_smithi039_297155_object25799:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1447)", 2024-09-14T04:50:55.104 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:49:58.492517+0000", 2024-09-14T04:50:55.104 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 56.492456728999997, 2024-09-14T04:50:55.104 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33853844900000002, 2024-09-14T04:50:55.104 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:55.104 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:55.104 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:55.104 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:50:55.104 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:55.105 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 149732 2024-09-14T04:50:55.105 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.105 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:55.105 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.105 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:55.105 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.492517+0000", 2024-09-14T04:50:55.105 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.105 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.105 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.105 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:55.105 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.492517+0000", 2024-09-14T04:50:55.105 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.106 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.106 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.106 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:55.106 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.492516+0000", 2024-09-14T04:50:55.106 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:50:55.106 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.106 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.106 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:55.106 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.492522+0000", 2024-09-14T04:50:55.106 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.8669999999999999e-06 2024-09-14T04:50:55.106 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.106 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.106 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:55.107 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.492533+0000", 2024-09-14T04:50:55.107 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0891e-05 2024-09-14T04:50:55.107 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.107 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.107 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:55.107 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.492540+0000", 2024-09-14T04:50:55.107 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8709999999999999e-06 2024-09-14T04:50:55.107 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.107 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.107 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:55.107 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.492582+0000", 2024-09-14T04:50:55.107 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1859e-05 2024-09-14T04:50:55.107 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.107 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.108 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:55.108 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.492669+0000", 2024-09-14T04:50:55.108 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.7410000000000005e-05 2024-09-14T04:50:55.108 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.108 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.108 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,3", 2024-09-14T04:50:55.108 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.492745+0000", 2024-09-14T04:50:55.108 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.6487000000000001e-05 2024-09-14T04:50:55.108 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.108 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.108 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:55.109 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.495567+0000", 2024-09-14T04:50:55.109 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.002821287 2024-09-14T04:50:55.109 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.109 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.109 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.109 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.495890+0000", 2024-09-14T04:50:55.109 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00032340899999999998 2024-09-14T04:50:55.109 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.109 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.109 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.109 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.830975+0000", 2024-09-14T04:50:55.109 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33508471000000001 2024-09-14T04:50:55.109 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.110 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.110 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:55.110 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.831020+0000", 2024-09-14T04:50:55.110 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.4799999999999998e-05 2024-09-14T04:50:55.110 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.110 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.110 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:55.110 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.831056+0000", 2024-09-14T04:50:55.110 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6112000000000003e-05 2024-09-14T04:50:55.110 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.110 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:55.110 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.110 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.110 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.111 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:189801 55.d 55:b1b98788:::benchmark_data_smithi039_297155_object65868:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1453)", 2024-09-14T04:50:55.111 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:50:44.333453+0000", 2024-09-14T04:50:55.111 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 10.651521321000001, 2024-09-14T04:50:55.111 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.26339609400000003, 2024-09-14T04:50:55.111 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:55.111 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:55.111 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:55.111 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:50:55.111 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:55.111 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 189801 2024-09-14T04:50:55.111 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.111 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:55.111 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.112 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:55.112 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.333453+0000", 2024-09-14T04:50:55.112 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.112 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.112 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.112 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:55.112 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.333453+0000", 2024-09-14T04:50:55.112 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.112 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.112 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.112 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:55.112 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.333452+0000", 2024-09-14T04:50:55.113 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:50:55.113 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.113 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.113 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:55.113 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.333457+0000", 2024-09-14T04:50:55.113 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.0069999999999997e-06 2024-09-14T04:50:55.113 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.113 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.113 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:55.113 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.333475+0000", 2024-09-14T04:50:55.113 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.8292999999999999e-05 2024-09-14T04:50:55.113 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.114 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.114 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:55.114 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.333482+0000", 2024-09-14T04:50:55.114 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.9079999999999998e-06 2024-09-14T04:50:55.114 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.114 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.114 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:55.114 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.333523+0000", 2024-09-14T04:50:55.114 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1199000000000003e-05 2024-09-14T04:50:55.114 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.114 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.114 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:55.114 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.333630+0000", 2024-09-14T04:50:55.115 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010705999999999999 2024-09-14T04:50:55.115 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.115 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.115 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:50:55.115 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.333726+0000", 2024-09-14T04:50:55.115 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.5357000000000002e-05 2024-09-14T04:50:55.115 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.115 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.115 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:55.115 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.337315+0000", 2024-09-14T04:50:55.115 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0035892530000000001 2024-09-14T04:50:55.115 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.116 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.116 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.116 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.343266+0000", 2024-09-14T04:50:55.116 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0059506280000000003 2024-09-14T04:50:55.116 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.116 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.116 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.116 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.596770+0000", 2024-09-14T04:50:55.116 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.25350475900000002 2024-09-14T04:50:55.116 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.116 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.116 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:55.117 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.596808+0000", 2024-09-14T04:50:55.117 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7327999999999999e-05 2024-09-14T04:50:55.117 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.117 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.117 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:55.117 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.596849+0000", 2024-09-14T04:50:55.117 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1109999999999998e-05 2024-09-14T04:50:55.117 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.117 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:55.117 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.117 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.118 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.118 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:189832 55.d 55:b8a02156:::benchmark_data_smithi039_297155_object65899:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1453)", 2024-09-14T04:50:55.118 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:50:44.370566+0000", 2024-09-14T04:50:55.118 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 10.614408178, 2024-09-14T04:50:55.118 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.226419179, 2024-09-14T04:50:55.118 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:50:55.118 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:50:55.118 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:50:55.118 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:50:55.118 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:50:55.118 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 189832 2024-09-14T04:50:55.118 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.119 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:50:55.119 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.119 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:50:55.119 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.370566+0000", 2024-09-14T04:50:55.119 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.119 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.119 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.119 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:50:55.119 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.370566+0000", 2024-09-14T04:50:55.119 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:50:55.119 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.119 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.119 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:50:55.120 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.370564+0000", 2024-09-14T04:50:55.120 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:50:55.120 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.120 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.120 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:50:55.120 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.370570+0000", 2024-09-14T04:50:55.120 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.0340000000000002e-06 2024-09-14T04:50:55.120 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.120 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.120 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:50:55.120 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.370582+0000", 2024-09-14T04:50:55.120 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1272e-05 2024-09-14T04:50:55.120 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.121 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.121 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:50:55.121 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.370588+0000", 2024-09-14T04:50:55.121 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7440000000000001e-06 2024-09-14T04:50:55.121 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.121 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.121 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:50:55.121 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.370635+0000", 2024-09-14T04:50:55.121 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6743000000000003e-05 2024-09-14T04:50:55.121 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.121 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.121 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:50:55.121 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.370739+0000", 2024-09-14T04:50:55.121 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000103544 2024-09-14T04:50:55.121 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.122 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.122 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:50:55.122 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.370842+0000", 2024-09-14T04:50:55.122 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010326400000000001 2024-09-14T04:50:55.122 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.122 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.122 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:50:55.122 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.374247+0000", 2024-09-14T04:50:55.122 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0034054570000000002 2024-09-14T04:50:55.122 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.122 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.122 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.122 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.374880+0000", 2024-09-14T04:50:55.123 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00063245499999999997 2024-09-14T04:50:55.123 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.123 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.123 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:50:55.123 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.596889+0000", 2024-09-14T04:50:55.123 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.22200866 2024-09-14T04:50:55.123 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.123 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.123 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:50:55.123 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.596947+0000", 2024-09-14T04:50:55.123 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.8053999999999997e-05 2024-09-14T04:50:55.123 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:50:55.123 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:50:55.124 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:50:55.124 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.596985+0000", 2024-09-14T04:50:55.124 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8367e-05 2024-09-14T04:50:55.124 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.124 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:55.124 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.124 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:50:55.124 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:50:55.124 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:50:55.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:55.126 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:55.126 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:55.227 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:55.262 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:55.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:55.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:55.364 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:55.413 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:55.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:55.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:55.515 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:55.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:55 smithi039 ceph-mon[106470]: pgmap v2662: 99 pgs: 99 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 848 op/s 2024-09-14T04:50:55.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:55 smithi039 ceph-mon[106470]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:50:55.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:55 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:50:55.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:55 smithi039 ceph-mon[106470]: osdmap e1455: 8 total, 8 up, 8 in 2024-09-14T04:50:55.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:50:55.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:50:55.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:55.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:55 smithi039 ceph-mon[111235]: pgmap v2662: 99 pgs: 99 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 848 op/s 2024-09-14T04:50:55.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:55 smithi039 ceph-mon[111235]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:50:55.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:55 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:50:55.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:55 smithi039 ceph-mon[111235]: osdmap e1455: 8 total, 8 up, 8 in 2024-09-14T04:50:55.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:50:55.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:50:55.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:55.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:55.553 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:55.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:55.654 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:55.698 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:55.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:55.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:55 smithi129 ceph-mon[101316]: pgmap v2662: 99 pgs: 99 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 848 op/s 2024-09-14T04:50:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:55 smithi129 ceph-mon[101316]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:50:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:55 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:50:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:55 smithi129 ceph-mon[101316]: osdmap e1455: 8 total, 8 up, 8 in 2024-09-14T04:50:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:50:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:50:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:50:55.800 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:55.839 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:55.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:55.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:55.941 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:55.975 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:55.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:55.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:56.078 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:56.097 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-14T04:50:56.117 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:56.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:56.118 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:56.219 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:56.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:56.261 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:56.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:56.361 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:56.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:56.399 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:56.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:56.450 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:50:56.500 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:56.531 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:56.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:56.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:56.576 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:50:56.577 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:50:56.615 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:50:56.615 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:50:56.633 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:56.652 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:50:56.652 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:50:56.675 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:56.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:56.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:56.694 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:50:56.694 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:50:56.728 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:50:56.728 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:50:56.767 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:50:56.767 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:50:56.777 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:56.794 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:50:56.794 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:50:56.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:56.823 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:56.823 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:56.838 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:50:56.838 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:50:56.924 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:56.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:56.958 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:56.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:57.059 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:57.095 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:57.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:57.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:57.197 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:57.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:57.241 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:57.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:57.342 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:57.382 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:57.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:57.383 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:57.484 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:57.523 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:57.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:57.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:57.624 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:50:57.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:57.663 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:57.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:50:57.745 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:57 smithi039 ceph-mon[106470]: pgmap v2664: 99 pgs: 99 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 953 op/s 2024-09-14T04:50:57.745 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:57 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3128068396' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:50:57.745 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:57 smithi039 ceph-mon[111235]: pgmap v2664: 99 pgs: 99 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 953 op/s 2024-09-14T04:50:57.746 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:57 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3128068396' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:50:57.764 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:57.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:57 smithi129 ceph-mon[101316]: pgmap v2664: 99 pgs: 99 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 953 op/s 2024-09-14T04:50:57.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:57 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3128068396' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:50:57.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:57.802 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:57.802 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:57.903 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:57.940 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:57.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:57.941 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:58.042 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:58.078 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:58.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:58.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:58.179 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:58.216 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:58.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:58.217 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:58.318 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:58.326 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:50:58.353 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:58.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:58.355 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:58.448 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.1 to 1 (10000) 2024-09-14T04:50:58.455 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:50:58.483 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:58.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:58.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:50:58.530 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:50:58.530 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:50:58.585 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:58.616 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:58.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:58.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:58.649 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:58 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1631221989' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:50:58.649 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:58 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1631221989' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:50:58.718 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:58.760 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:58.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:58.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:58.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:58 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1631221989' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:50:58.862 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:58.895 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:58.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:58.897 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:58.937 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:50:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:50:58] "GET /metrics HTTP/1.1" 200 36755 "" "Prometheus/2.51.0" 2024-09-14T04:50:58.998 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:59.025 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:59.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:59.026 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:59.127 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:50:59.158 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:59.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:59.162 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:50:59.262 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:50:59.303 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:59.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:59.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:50:59.314 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 2 1 2024-09-14T04:50:59.405 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:50:59.428 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:59.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:59.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:50:59.432 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:50:59.531 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:50:59.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:59 smithi039 ceph-mon[106470]: pgmap v2665: 99 pgs: 99 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2024-09-14T04:50:59.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:59 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1631221989' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-14T04:50:59.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:50:59 smithi039 ceph-mon[106470]: osdmap e1456: 8 total, 8 up, 8 in 2024-09-14T04:50:59.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:59 smithi039 ceph-mon[111235]: pgmap v2665: 99 pgs: 99 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2024-09-14T04:50:59.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:59 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1631221989' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-14T04:50:59.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:50:59 smithi039 ceph-mon[111235]: osdmap e1456: 8 total, 8 up, 8 in 2024-09-14T04:50:59.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:59.561 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:50:59.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:50:59.662 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:50:59.672 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:59 smithi129 ceph-mon[101316]: pgmap v2665: 99 pgs: 99 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2024-09-14T04:50:59.672 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:59 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1631221989' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-14T04:50:59.672 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:50:59 smithi129 ceph-mon[101316]: osdmap e1456: 8 total, 8 up, 8 in 2024-09-14T04:50:59.686 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:59.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:59.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:50:59.779 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:50:59.788 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:50:59.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:50:59.865 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:50:59.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:50:59.966 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:00.007 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:00.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:00.008 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:00.109 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:00.147 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:00.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:00.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:00.249 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:00.282 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:51:00.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:00.305 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:00.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:00.406 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:00.440 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:00.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:00.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:00.542 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:00.574 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:00.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:00.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:00.676 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:00.711 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:00.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:00.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:00.754 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:51:00.813 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:00.842 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:00.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:00.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:00.944 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:00.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:00.970 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:00.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:01.071 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:01.107 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:01.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:01.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:01.209 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:01.247 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:01.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:01.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:01.349 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:01.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:01.380 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:01.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:01.482 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:01.523 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:01.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:01.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:01.625 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:01.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:01.664 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:01.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:01.683 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:01 smithi039 ceph-mon[106470]: pgmap v2667: 99 pgs: 99 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-14T04:51:01.683 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:01 smithi039 ceph-mon[111235]: pgmap v2667: 99 pgs: 99 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-14T04:51:01.765 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:01.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:01 smithi129 ceph-mon[101316]: pgmap v2667: 99 pgs: 99 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-14T04:51:01.801 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:01.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:01.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:01.903 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:01.937 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:01.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:01.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:02.039 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:02.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:02.078 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:02.078 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:02.103 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-14T04:51:02.179 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:02.209 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:02.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:02.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:02.311 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:02.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:02.370 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:02.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:02.471 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:02.472 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:51:02.475 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is unset 2024-09-14T04:51:02.516 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:02.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:02.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:02.618 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:02.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:02.653 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:02.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:02.754 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:02.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:02 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3760905077' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:51:02.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:02 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:51:02.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:02 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3760905077' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:51:02.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:02 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:51:02.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:02 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3760905077' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:51:02.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:02 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:51:02.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:02.804 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:02.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:02.905 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:02.939 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:02.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:02.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:03.044 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:03.074 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:03.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:03.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:03.176 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:03.216 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:03.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:03.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:03.318 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:03.353 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:03.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:03.354 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:03.456 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:03.464 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:03 smithi129 ceph-mon[101316]: pgmap v2668: 99 pgs: 99 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2024-09-14T04:51:03.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:03.492 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:03.492 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:03.593 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:03.626 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:03.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:03.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:03.728 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:03.757 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:03.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:03.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:03.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:03 smithi129 ceph-mon[101316]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:51:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:03 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:51:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:03 smithi129 ceph-mon[101316]: osdmap e1457: 8 total, 8 up, 8 in 2024-09-14T04:51:03.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:03 smithi039 ceph-mon[106470]: pgmap v2668: 99 pgs: 99 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2024-09-14T04:51:03.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:03 smithi039 ceph-mon[106470]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:51:03.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:03 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:51:03.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:03 smithi039 ceph-mon[106470]: osdmap e1457: 8 total, 8 up, 8 in 2024-09-14T04:51:03.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:03 smithi039 ceph-mon[111235]: pgmap v2668: 99 pgs: 99 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 665 op/s 2024-09-14T04:51:03.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:03 smithi039 ceph-mon[111235]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:51:03.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:03 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:51:03.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:03 smithi039 ceph-mon[111235]: osdmap e1457: 8 total, 8 up, 8 in 2024-09-14T04:51:03.859 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:03.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:03.889 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:03.889 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:03.990 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:04.033 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:04.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:04.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:04.135 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:04.169 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:04.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:04.170 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:04.271 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:04.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:04.314 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:04.314 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:04.415 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:04.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:04.455 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:04.455 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:04.501 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:51:04.501 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:51:04.556 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:04.599 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:04.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:04.600 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:04.701 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:04.735 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:04.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:04.736 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:04.837 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:04.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:04.875 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:04.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:04.976 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:05.010 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:05.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:05.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:05.112 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:05.146 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:05.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:05.146 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:05.211 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-14T04:51:05.247 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:05.316 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:05.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:05.317 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:05.417 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:05.458 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:05.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:05.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:05.472 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:05 smithi129 ceph-mon[101316]: pgmap v2670: 99 pgs: 99 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T04:51:05.473 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:05 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3549093121' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:51:05.485 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.2 to 1 (10000) 2024-09-14T04:51:05.560 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:05.569 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:51:05.594 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:05.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:05.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:05.684 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:05 smithi039 ceph-mon[106470]: pgmap v2670: 99 pgs: 99 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T04:51:05.684 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:05 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3549093121' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:51:05.685 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:05 smithi039 ceph-mon[111235]: pgmap v2670: 99 pgs: 99 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T04:51:05.685 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:05 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3549093121' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:51:05.695 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:05.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:05.735 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:05.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:05.836 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:05.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:05.876 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:05.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:05.977 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:06.015 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:06.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:06.016 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:06.117 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:06.157 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:06.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:06.158 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:06.217 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:51:06.217 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:51:06.254 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:51:06.254 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:51:06.259 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:06.288 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:51:06.288 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:51:06.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:06.299 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:06.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:06.322 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:51:06.322 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:51:06.351 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:51:06.351 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:51:06.388 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:51:06.388 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:51:06.400 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:06.414 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:51:06.415 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:51:06.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:06.431 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:06.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:06.445 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:51:06.445 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:51:06.532 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:06.539 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:06 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3549093121' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-14T04:51:06.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:06 smithi039 ceph-mon[106470]: osdmap e1458: 8 total, 8 up, 8 in 2024-09-14T04:51:06.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:06 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1630884509' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:51:06.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:06 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3549093121' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-14T04:51:06.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:06 smithi039 ceph-mon[111235]: osdmap e1458: 8 total, 8 up, 8 in 2024-09-14T04:51:06.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:06 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1630884509' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:51:06.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:06.566 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:06.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:06.667 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:06.690 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 3 1 2024-09-14T04:51:06.709 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:06.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:06.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:06.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:06 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3549093121' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-14T04:51:06.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:06 smithi129 ceph-mon[101316]: osdmap e1458: 8 total, 8 up, 8 in 2024-09-14T04:51:06.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:06 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1630884509' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:51:06.811 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:06.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:06.861 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:06.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:06.871 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:51:06.962 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:07.005 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:07.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:07.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:07.107 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:07.148 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:07.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:07.150 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:07.250 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:07.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:07.291 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:07.291 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:07.392 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:07.428 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:07.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:07.429 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:07.521 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:51:07.531 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:07.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:07.587 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:07.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:07.617 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:51:07.618 INFO:teuthology.orchestra.run.smithi129.stdout: "size": 20, 2024-09-14T04:51:07.618 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 600, 2024-09-14T04:51:07.618 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [ 2024-09-14T04:51:07.618 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.618 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:135592 54.8 54.cb8ca998 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:51:07.618 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:44:56.328570+0000", 2024-09-14T04:51:07.618 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 371.28612722399998, 2024-09-14T04:51:07.618 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.98625274299999999, 2024-09-14T04:51:07.618 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:51:07.618 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "reached pg", 2024-09-14T04:51:07.618 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:51:07.619 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:51:07.619 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:51:07.619 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 135592 2024-09-14T04:51:07.619 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.619 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:51:07.619 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.619 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:51:07.619 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328570+0000", 2024-09-14T04:51:07.619 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.619 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.619 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.619 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:51:07.619 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328570+0000", 2024-09-14T04:51:07.620 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.620 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.620 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.620 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:51:07.620 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328569+0000", 2024-09-14T04:51:07.620 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:51:07.620 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.620 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.620 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:51:07.620 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328575+0000", 2024-09-14T04:51:07.620 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7700000000000004e-06 2024-09-14T04:51:07.620 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.620 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.620 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:51:07.621 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328588+0000", 2024-09-14T04:51:07.621 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2832e-05 2024-09-14T04:51:07.621 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.621 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.621 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:51:07.621 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328596+0000", 2024-09-14T04:51:07.621 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.2490000000000001e-06 2024-09-14T04:51:07.621 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.621 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.621 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:51:07.621 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328692+0000", 2024-09-14T04:51:07.621 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.5469000000000004e-05 2024-09-14T04:51:07.621 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.622 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.622 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:51:07.622 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328697+0000", 2024-09-14T04:51:07.622 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.5099999999999998e-06 2024-09-14T04:51:07.622 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.622 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.622 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:51:07.622 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.314812+0000", 2024-09-14T04:51:07.622 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.98611423499999995 2024-09-14T04:51:07.622 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.622 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.622 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:51:07.622 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.314823+0000", 2024-09-14T04:51:07.623 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1259e-05 2024-09-14T04:51:07.623 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.623 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:51:07.623 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.623 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.623 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.623 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:135594 54.e 54.e4276cde (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:51:07.623 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:44:56.328609+0000", 2024-09-14T04:51:07.623 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 371.286088108, 2024-09-14T04:51:07.623 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.98386206200000004, 2024-09-14T04:51:07.623 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:51:07.623 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "reached pg", 2024-09-14T04:51:07.623 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:51:07.623 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:51:07.623 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:51:07.624 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 135594 2024-09-14T04:51:07.624 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.624 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:51:07.624 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.624 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:51:07.624 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328609+0000", 2024-09-14T04:51:07.624 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.624 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.624 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.624 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:51:07.624 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328609+0000", 2024-09-14T04:51:07.624 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.624 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.625 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.625 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:51:07.625 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328609+0000", 2024-09-14T04:51:07.625 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:51:07.625 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.625 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.625 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:51:07.625 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328611+0000", 2024-09-14T04:51:07.625 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.5799999999999999e-06 2024-09-14T04:51:07.625 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.625 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.625 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:51:07.625 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328617+0000", 2024-09-14T04:51:07.625 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.6080000000000003e-06 2024-09-14T04:51:07.626 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.626 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.626 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:51:07.626 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328622+0000", 2024-09-14T04:51:07.626 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.685e-06 2024-09-14T04:51:07.626 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.626 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.626 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:51:07.626 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328658+0000", 2024-09-14T04:51:07.626 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6219e-05 2024-09-14T04:51:07.626 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.626 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.626 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:51:07.626 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328665+0000", 2024-09-14T04:51:07.626 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.0779999999999998e-06 2024-09-14T04:51:07.627 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.627 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.627 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:51:07.627 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.312460+0000", 2024-09-14T04:51:07.627 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.98379467799999998 2024-09-14T04:51:07.627 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.627 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.627 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:51:07.627 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.312471+0000", 2024-09-14T04:51:07.627 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1620000000000001e-05 2024-09-14T04:51:07.627 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.627 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:51:07.627 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.627 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.627 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.628 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:135646 54.8 54.3da48de8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:51:07.628 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:44:56.446796+0000", 2024-09-14T04:51:07.628 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 371.16790099600001, 2024-09-14T04:51:07.628 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.86805825299999995, 2024-09-14T04:51:07.628 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:51:07.628 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "reached pg", 2024-09-14T04:51:07.628 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:51:07.628 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:51:07.628 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:51:07.628 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 135646 2024-09-14T04:51:07.628 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.628 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:51:07.628 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.628 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:51:07.628 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.446796+0000", 2024-09-14T04:51:07.629 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.629 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.629 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.629 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:51:07.629 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.446796+0000", 2024-09-14T04:51:07.629 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.629 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.629 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.629 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:51:07.629 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.446794+0000", 2024-09-14T04:51:07.629 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:51:07.629 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.629 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.629 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:51:07.630 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.446806+0000", 2024-09-14T04:51:07.630 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1218e-05 2024-09-14T04:51:07.630 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.630 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.630 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:51:07.630 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.446829+0000", 2024-09-14T04:51:07.630 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.2872999999999999e-05 2024-09-14T04:51:07.630 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.630 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.630 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:51:07.630 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.446839+0000", 2024-09-14T04:51:07.630 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0163000000000001e-05 2024-09-14T04:51:07.630 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.630 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.631 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:51:07.631 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.446888+0000", 2024-09-14T04:51:07.631 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8918999999999997e-05 2024-09-14T04:51:07.631 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.631 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.631 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:51:07.631 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.446896+0000", 2024-09-14T04:51:07.631 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.9440000000000005e-06 2024-09-14T04:51:07.631 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.631 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.631 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:51:07.631 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.314847+0000", 2024-09-14T04:51:07.631 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.86795141499999995 2024-09-14T04:51:07.631 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.632 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.632 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:51:07.632 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.314855+0000", 2024-09-14T04:51:07.632 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.6020000000000002e-06 2024-09-14T04:51:07.632 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.633 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:51:07.633 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.633 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.633 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.633 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:135649 54.8 54.c40c9e08 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:51:07.633 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:44:56.448432+0000", 2024-09-14T04:51:07.633 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 371.166265791, 2024-09-14T04:51:07.633 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.86648267700000003, 2024-09-14T04:51:07.633 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:51:07.633 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "reached pg", 2024-09-14T04:51:07.633 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:51:07.633 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:51:07.633 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:51:07.634 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 135649 2024-09-14T04:51:07.634 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.634 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:51:07.634 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.634 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:51:07.634 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.448432+0000", 2024-09-14T04:51:07.634 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.634 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.634 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.634 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:51:07.634 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.448432+0000", 2024-09-14T04:51:07.634 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.634 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.634 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.635 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:51:07.635 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.448431+0000", 2024-09-14T04:51:07.635 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:51:07.635 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.635 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.635 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:51:07.635 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.448437+0000", 2024-09-14T04:51:07.635 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.5490000000000003e-06 2024-09-14T04:51:07.635 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.635 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.635 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:51:07.635 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.448451+0000", 2024-09-14T04:51:07.635 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3910999999999999e-05 2024-09-14T04:51:07.636 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.636 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.636 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:51:07.636 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.448459+0000", 2024-09-14T04:51:07.636 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.9929999999999995e-06 2024-09-14T04:51:07.636 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.636 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.636 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:51:07.636 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.448500+0000", 2024-09-14T04:51:07.636 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1527999999999999e-05 2024-09-14T04:51:07.636 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.636 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.637 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:51:07.637 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.448509+0000", 2024-09-14T04:51:07.637 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.0000000000000002e-06 2024-09-14T04:51:07.637 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.637 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.637 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:51:07.637 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.314892+0000", 2024-09-14T04:51:07.637 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.86638208000000005 2024-09-14T04:51:07.637 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.637 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.637 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:51:07.637 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.314914+0000", 2024-09-14T04:51:07.637 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.2685999999999999e-05 2024-09-14T04:51:07.638 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.638 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:51:07.638 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.638 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.638 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.638 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:135662 54.e 54.9c02db5e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:51:07.638 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:44:56.465929+0000", 2024-09-14T04:51:07.638 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 371.148768115, 2024-09-14T04:51:07.638 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.84657111200000001, 2024-09-14T04:51:07.638 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:51:07.639 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "reached pg", 2024-09-14T04:51:07.639 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:51:07.639 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:51:07.639 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:51:07.639 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 135662 2024-09-14T04:51:07.639 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.639 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:51:07.639 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.639 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:51:07.639 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.465929+0000", 2024-09-14T04:51:07.639 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.639 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.639 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.639 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:51:07.640 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.465929+0000", 2024-09-14T04:51:07.640 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.640 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.640 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.640 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:51:07.640 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.465927+0000", 2024-09-14T04:51:07.640 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:51:07.640 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.640 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.640 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:51:07.640 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.465936+0000", 2024-09-14T04:51:07.640 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.6009999999999995e-06 2024-09-14T04:51:07.640 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.640 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.641 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:51:07.641 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.465954+0000", 2024-09-14T04:51:07.641 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.8406e-05 2024-09-14T04:51:07.641 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.641 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.641 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:51:07.641 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.465961+0000", 2024-09-14T04:51:07.641 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8600000000000004e-06 2024-09-14T04:51:07.641 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.641 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.641 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:51:07.641 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.466010+0000", 2024-09-14T04:51:07.641 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8504999999999997e-05 2024-09-14T04:51:07.641 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.642 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.642 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:51:07.642 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.466019+0000", 2024-09-14T04:51:07.642 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.2129999999999997e-06 2024-09-14T04:51:07.642 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.642 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.642 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:51:07.642 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.312493+0000", 2024-09-14T04:51:07.642 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.84647388999999995 2024-09-14T04:51:07.642 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.642 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.642 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:51:07.642 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.312500+0000", 2024-09-14T04:51:07.642 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.4109999999999999e-06 2024-09-14T04:51:07.643 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.643 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:51:07.643 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.643 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.643 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.643 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:135671 54.8 54.606ad4a8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:51:07.643 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:44:56.481939+0000", 2024-09-14T04:51:07.643 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 371.13275810599998, 2024-09-14T04:51:07.643 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.83301307300000005, 2024-09-14T04:51:07.643 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:51:07.643 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "reached pg", 2024-09-14T04:51:07.643 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:51:07.643 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:51:07.643 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:51:07.644 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 135671 2024-09-14T04:51:07.644 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.644 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:51:07.644 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.644 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:51:07.644 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.481939+0000", 2024-09-14T04:51:07.644 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.644 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.644 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.644 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:51:07.644 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.481939+0000", 2024-09-14T04:51:07.644 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.644 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.645 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.645 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:51:07.645 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.481938+0000", 2024-09-14T04:51:07.645 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:51:07.645 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.645 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.645 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:51:07.645 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.481952+0000", 2024-09-14T04:51:07.645 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3995000000000001e-05 2024-09-14T04:51:07.645 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.645 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.645 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:51:07.645 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.481974+0000", 2024-09-14T04:51:07.646 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.2124999999999999e-05 2024-09-14T04:51:07.646 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.646 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.646 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:51:07.646 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.481987+0000", 2024-09-14T04:51:07.646 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3174e-05 2024-09-14T04:51:07.646 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.646 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.646 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:51:07.646 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.482047+0000", 2024-09-14T04:51:07.646 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9987e-05 2024-09-14T04:51:07.646 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.646 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.646 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:51:07.646 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.482056+0000", 2024-09-14T04:51:07.647 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.0699999999999996e-06 2024-09-14T04:51:07.647 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.647 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.647 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:51:07.647 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.314944+0000", 2024-09-14T04:51:07.647 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.83288777000000003 2024-09-14T04:51:07.647 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.647 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.647 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:51:07.647 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.314952+0000", 2024-09-14T04:51:07.647 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.4020000000000002e-06 2024-09-14T04:51:07.647 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.647 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:51:07.648 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.648 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.648 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.648 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:135674 54.e 54.7ab1506e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:51:07.648 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:44:56.489558+0000", 2024-09-14T04:51:07.648 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 371.12513951, 2024-09-14T04:51:07.648 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.82296867500000004, 2024-09-14T04:51:07.648 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:51:07.648 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "reached pg", 2024-09-14T04:51:07.648 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:51:07.648 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:51:07.648 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:51:07.648 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 135674 2024-09-14T04:51:07.648 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.649 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:51:07.649 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.649 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:51:07.649 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.489558+0000", 2024-09-14T04:51:07.649 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.649 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.649 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.649 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:51:07.649 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.489558+0000", 2024-09-14T04:51:07.649 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.649 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.649 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.649 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:51:07.650 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.489556+0000", 2024-09-14T04:51:07.650 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:51:07.650 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.650 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.650 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:51:07.650 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.489562+0000", 2024-09-14T04:51:07.650 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9370000000000001e-06 2024-09-14T04:51:07.650 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.650 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.650 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:51:07.650 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.489573+0000", 2024-09-14T04:51:07.650 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0942e-05 2024-09-14T04:51:07.650 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.651 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.651 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:51:07.651 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.489580+0000", 2024-09-14T04:51:07.651 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.4280000000000001e-06 2024-09-14T04:51:07.651 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.651 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.651 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:51:07.651 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.489616+0000", 2024-09-14T04:51:07.651 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6606000000000002e-05 2024-09-14T04:51:07.651 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.651 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.651 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:51:07.651 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.489622+0000", 2024-09-14T04:51:07.652 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.5169999999999999e-06 2024-09-14T04:51:07.652 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.652 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.652 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:51:07.652 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.312519+0000", 2024-09-14T04:51:07.652 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.82289769999999995 2024-09-14T04:51:07.652 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.652 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.652 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:51:07.652 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.312527+0000", 2024-09-14T04:51:07.652 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.1380000000000001e-06 2024-09-14T04:51:07.652 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.652 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:51:07.653 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.653 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.653 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.653 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.35436.0:135838 54.e e1404/1403 54:78ffc273:::benchmark_data_smithi039_264334_object9441:head v 1404'8492, mlcod=1388'576)", 2024-09-14T04:51:07.653 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:44:58.682129+0000", 2024-09-14T04:51:07.653 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 368.93256873199999, 2024-09-14T04:51:07.653 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.104093137, 2024-09-14T04:51:07.653 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:51:07.653 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:51:07.653 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:51:07.653 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.653 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:51:07.653 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.682129+0000", 2024-09-14T04:51:07.654 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.654 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.654 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.654 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:51:07.654 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.682129+0000", 2024-09-14T04:51:07.654 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.654 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.654 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.654 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:51:07.654 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.682127+0000", 2024-09-14T04:51:07.654 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:51:07.654 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.654 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.655 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:51:07.655 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.682137+0000", 2024-09-14T04:51:07.655 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0149999999999999e-05 2024-09-14T04:51:07.655 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.655 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.655 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:51:07.655 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.682155+0000", 2024-09-14T04:51:07.655 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.7909000000000001e-05 2024-09-14T04:51:07.655 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.655 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.655 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:51:07.655 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.682163+0000", 2024-09-14T04:51:07.655 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.4139999999999994e-06 2024-09-14T04:51:07.655 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.656 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.656 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:51:07.656 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.737181+0000", 2024-09-14T04:51:07.656 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.055017513999999997 2024-09-14T04:51:07.656 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.656 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.656 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:51:07.656 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.737208+0000", 2024-09-14T04:51:07.656 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.7328e-05 2024-09-14T04:51:07.656 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.656 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.656 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:51:07.657 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.786194+0000", 2024-09-14T04:51:07.657 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.048985446000000002 2024-09-14T04:51:07.657 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.657 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.657 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:51:07.657 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:58.786222+0000", 2024-09-14T04:51:07.657 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.8005e-05 2024-09-14T04:51:07.657 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.657 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:51:07.657 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.657 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.657 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.657 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:16570 55.c 55:3cbead5e:::benchmark_data_smithi039_297155_object16569:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1427)", 2024-09-14T04:51:07.658 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:48:06.685530+0000", 2024-09-14T04:51:07.658 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 180.92916687100001, 2024-09-14T04:51:07.658 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.114227415, 2024-09-14T04:51:07.658 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:51:07.658 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:51:07.658 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:51:07.658 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:51:07.658 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:51:07.658 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 16570 2024-09-14T04:51:07.658 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.658 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:51:07.658 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.658 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:51:07.658 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.685530+0000", 2024-09-14T04:51:07.659 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.659 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.659 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.659 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:51:07.659 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.685530+0000", 2024-09-14T04:51:07.659 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.659 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.659 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.659 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:51:07.659 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.685529+0000", 2024-09-14T04:51:07.659 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:51:07.659 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.659 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.660 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:51:07.660 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.685584+0000", 2024-09-14T04:51:07.660 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.4898999999999997e-05 2024-09-14T04:51:07.660 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.660 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.660 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:51:07.660 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.685597+0000", 2024-09-14T04:51:07.660 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2819e-05 2024-09-14T04:51:07.660 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.660 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.660 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:51:07.660 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.685604+0000", 2024-09-14T04:51:07.660 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7920000000000004e-06 2024-09-14T04:51:07.660 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.661 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.661 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:51:07.661 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.685642+0000", 2024-09-14T04:51:07.661 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8435e-05 2024-09-14T04:51:07.661 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.661 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.661 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:51:07.661 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.685749+0000", 2024-09-14T04:51:07.661 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010686799999999999 2024-09-14T04:51:07.661 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.661 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.661 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:51:07.661 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.685867+0000", 2024-09-14T04:51:07.661 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000118318 2024-09-14T04:51:07.662 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.662 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.662 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:51:07.662 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.687160+0000", 2024-09-14T04:51:07.662 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0012927749999999999 2024-09-14T04:51:07.662 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.663 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.663 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:07.663 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.690301+0000", 2024-09-14T04:51:07.663 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0031406009999999998 2024-09-14T04:51:07.663 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.663 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.663 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:07.663 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.799678+0000", 2024-09-14T04:51:07.663 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.109377443 2024-09-14T04:51:07.663 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.663 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.663 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:51:07.663 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.799716+0000", 2024-09-14T04:51:07.663 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8031999999999998e-05 2024-09-14T04:51:07.664 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.664 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.664 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:51:07.664 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.799758+0000", 2024-09-14T04:51:07.664 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1899e-05 2024-09-14T04:51:07.664 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.664 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:51:07.664 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.664 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.664 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.664 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:16578 55.c 55:3d36d81d:::benchmark_data_smithi039_297155_object16577:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1427)", 2024-09-14T04:51:07.664 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:48:06.687918+0000", 2024-09-14T04:51:07.665 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 180.92677930400001, 2024-09-14T04:51:07.665 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.111929998, 2024-09-14T04:51:07.665 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:51:07.665 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:51:07.665 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:51:07.665 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:51:07.665 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:51:07.665 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 16578 2024-09-14T04:51:07.665 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.665 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:51:07.665 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.665 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:51:07.665 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.687918+0000", 2024-09-14T04:51:07.665 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.666 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.666 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.666 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:51:07.666 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.687918+0000", 2024-09-14T04:51:07.666 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.666 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.666 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.666 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:51:07.666 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.687917+0000", 2024-09-14T04:51:07.666 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:51:07.666 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.666 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.666 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:51:07.666 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.687951+0000", 2024-09-14T04:51:07.666 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.4554000000000002e-05 2024-09-14T04:51:07.667 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.667 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.667 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:51:07.667 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.687968+0000", 2024-09-14T04:51:07.667 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.7007000000000002e-05 2024-09-14T04:51:07.667 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.667 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.667 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:51:07.667 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.687976+0000", 2024-09-14T04:51:07.667 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.1000000000000004e-06 2024-09-14T04:51:07.667 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.667 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.667 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:51:07.667 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.688113+0000", 2024-09-14T04:51:07.668 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000136174 2024-09-14T04:51:07.668 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.668 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.668 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:51:07.668 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.688239+0000", 2024-09-14T04:51:07.668 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012652599999999999 2024-09-14T04:51:07.668 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.668 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.668 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:51:07.668 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.688372+0000", 2024-09-14T04:51:07.668 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013341000000000001 2024-09-14T04:51:07.668 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.668 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.668 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:51:07.668 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.689834+0000", 2024-09-14T04:51:07.669 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00146166 2024-09-14T04:51:07.669 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.669 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.669 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:07.669 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.691085+0000", 2024-09-14T04:51:07.669 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0012508300000000001 2024-09-14T04:51:07.669 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.669 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.669 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:07.669 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.799798+0000", 2024-09-14T04:51:07.669 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.108712709 2024-09-14T04:51:07.669 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.670 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.670 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:51:07.670 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.799827+0000", 2024-09-14T04:51:07.670 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.9524000000000001e-05 2024-09-14T04:51:07.670 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.670 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.670 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:51:07.670 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.799848+0000", 2024-09-14T04:51:07.670 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.0883999999999999e-05 2024-09-14T04:51:07.670 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.670 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:51:07.670 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.670 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.670 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.670 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:16579 55.c 55:316a9903:::benchmark_data_smithi039_297155_object16578:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1427)", 2024-09-14T04:51:07.671 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:48:06.687988+0000", 2024-09-14T04:51:07.671 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 180.926709002, 2024-09-14T04:51:07.671 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.11195281, 2024-09-14T04:51:07.671 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:51:07.671 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:51:07.671 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:51:07.671 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:51:07.671 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:51:07.671 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 16579 2024-09-14T04:51:07.671 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.671 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:51:07.671 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.671 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:51:07.671 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.687988+0000", 2024-09-14T04:51:07.671 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.672 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.672 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.672 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:51:07.672 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.687988+0000", 2024-09-14T04:51:07.672 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.672 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.672 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.672 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:51:07.672 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.687988+0000", 2024-09-14T04:51:07.672 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:51:07.672 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.672 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.672 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:51:07.672 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.688018+0000", 2024-09-14T04:51:07.673 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.9737e-05 2024-09-14T04:51:07.673 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.673 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.673 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:51:07.673 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.688025+0000", 2024-09-14T04:51:07.673 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.7889999999999995e-06 2024-09-14T04:51:07.673 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.673 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.673 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:51:07.673 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.688077+0000", 2024-09-14T04:51:07.673 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.1449999999999997e-05 2024-09-14T04:51:07.673 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.673 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.673 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:51:07.673 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.689060+0000", 2024-09-14T04:51:07.674 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00098351199999999997 2024-09-14T04:51:07.674 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.674 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.674 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:51:07.674 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.689189+0000", 2024-09-14T04:51:07.674 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012850900000000001 2024-09-14T04:51:07.674 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.674 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.674 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:51:07.674 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.689334+0000", 2024-09-14T04:51:07.674 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00014553799999999999 2024-09-14T04:51:07.674 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.674 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.674 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:51:07.675 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.689996+0000", 2024-09-14T04:51:07.675 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00066150699999999998 2024-09-14T04:51:07.675 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.675 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.675 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:07.675 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.692297+0000", 2024-09-14T04:51:07.675 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.002300823 2024-09-14T04:51:07.675 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.675 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.675 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:07.675 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.799874+0000", 2024-09-14T04:51:07.675 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.10757739199999999 2024-09-14T04:51:07.675 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.676 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.676 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:51:07.676 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.799904+0000", 2024-09-14T04:51:07.676 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.9867999999999999e-05 2024-09-14T04:51:07.676 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.676 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.676 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:51:07.676 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.799941+0000", 2024-09-14T04:51:07.676 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7208e-05 2024-09-14T04:51:07.676 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.676 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:51:07.676 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.676 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.676 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.677 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:16587 55.c 55:3d9e0cdb:::benchmark_data_smithi039_297155_object16586:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1427)", 2024-09-14T04:51:07.677 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:48:06.691238+0000", 2024-09-14T04:51:07.677 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 180.92345953700001, 2024-09-14T04:51:07.677 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.108972739, 2024-09-14T04:51:07.677 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:51:07.677 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:51:07.677 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:51:07.677 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:51:07.677 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:51:07.677 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 16587 2024-09-14T04:51:07.677 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.677 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:51:07.677 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.678 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:51:07.678 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.691238+0000", 2024-09-14T04:51:07.678 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.678 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.678 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.678 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:51:07.678 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.691238+0000", 2024-09-14T04:51:07.678 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.678 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.678 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.678 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:51:07.678 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.691233+0000", 2024-09-14T04:51:07.678 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999948 2024-09-14T04:51:07.678 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.679 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.679 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:51:07.679 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.691287+0000", 2024-09-14T04:51:07.679 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.4666999999999997e-05 2024-09-14T04:51:07.679 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.679 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.679 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:51:07.679 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.691303+0000", 2024-09-14T04:51:07.679 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5457000000000001e-05 2024-09-14T04:51:07.679 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.679 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.679 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:51:07.679 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.691310+0000", 2024-09-14T04:51:07.680 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.0720000000000002e-06 2024-09-14T04:51:07.680 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.680 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.680 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:51:07.680 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.691350+0000", 2024-09-14T04:51:07.680 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9926999999999999e-05 2024-09-14T04:51:07.680 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.680 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.680 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:51:07.680 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.691484+0000", 2024-09-14T04:51:07.680 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013383000000000001 2024-09-14T04:51:07.680 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.680 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.681 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:51:07.681 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.691642+0000", 2024-09-14T04:51:07.681 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00015854399999999999 2024-09-14T04:51:07.681 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.681 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.681 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:51:07.681 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.692403+0000", 2024-09-14T04:51:07.681 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00076062000000000005 2024-09-14T04:51:07.682 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.682 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.682 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:07.682 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.692713+0000", 2024-09-14T04:51:07.682 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00031010899999999998 2024-09-14T04:51:07.682 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.682 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.682 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:07.682 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.800153+0000", 2024-09-14T04:51:07.682 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.107440671 2024-09-14T04:51:07.682 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.682 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.682 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:51:07.683 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.800180+0000", 2024-09-14T04:51:07.683 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.6502000000000001e-05 2024-09-14T04:51:07.683 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.683 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.683 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:51:07.683 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:06.800211+0000", 2024-09-14T04:51:07.683 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.0605999999999998e-05 2024-09-14T04:51:07.683 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.683 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:51:07.683 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.683 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.683 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.683 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:56662 55.c 55:3b7b606b:::benchmark_data_smithi039_297155_object56661:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1433)", 2024-09-14T04:51:07.683 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:48:30.373134+0000", 2024-09-14T04:51:07.684 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 157.241563716, 2024-09-14T04:51:07.684 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.141525453, 2024-09-14T04:51:07.684 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:51:07.684 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:51:07.684 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:51:07.684 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:51:07.684 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:51:07.684 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 56662 2024-09-14T04:51:07.684 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.684 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:51:07.684 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.684 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:51:07.685 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.373134+0000", 2024-09-14T04:51:07.685 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.685 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.685 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.685 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:51:07.685 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.373134+0000", 2024-09-14T04:51:07.685 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.685 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.685 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.685 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:51:07.685 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.373132+0000", 2024-09-14T04:51:07.685 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:51:07.685 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.685 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.686 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:51:07.686 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.373180+0000", 2024-09-14T04:51:07.686 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8718999999999999e-05 2024-09-14T04:51:07.686 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.686 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.686 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:51:07.686 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.373203+0000", 2024-09-14T04:51:07.686 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.296e-05 2024-09-14T04:51:07.686 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.686 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.686 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:51:07.686 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.373212+0000", 2024-09-14T04:51:07.686 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.9129999999999992e-06 2024-09-14T04:51:07.686 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.687 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.687 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:51:07.687 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.373264+0000", 2024-09-14T04:51:07.687 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.1756999999999999e-05 2024-09-14T04:51:07.687 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.687 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.687 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:51:07.687 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.373407+0000", 2024-09-14T04:51:07.687 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00014326700000000001 2024-09-14T04:51:07.687 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.687 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.687 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:51:07.687 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.373549+0000", 2024-09-14T04:51:07.687 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00014176799999999999 2024-09-14T04:51:07.688 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.688 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.688 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:51:07.688 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.376148+0000", 2024-09-14T04:51:07.688 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.002598908 2024-09-14T04:51:07.688 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.688 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.688 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:07.688 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.379195+0000", 2024-09-14T04:51:07.688 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0030473409999999999 2024-09-14T04:51:07.688 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.688 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.688 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:07.688 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.514504+0000", 2024-09-14T04:51:07.689 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.13530854 2024-09-14T04:51:07.689 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.689 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.689 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:51:07.689 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.514620+0000", 2024-09-14T04:51:07.689 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000116042 2024-09-14T04:51:07.689 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.689 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.689 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:51:07.689 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.514659+0000", 2024-09-14T04:51:07.689 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9206999999999999e-05 2024-09-14T04:51:07.689 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.690 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:51:07.690 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.690 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.690 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.690 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:56679 55.c 55:3bf8c327:::benchmark_data_smithi039_297155_object56678:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1433)", 2024-09-14T04:51:07.690 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:48:30.383285+0000", 2024-09-14T04:51:07.690 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 157.23141276199999, 2024-09-14T04:51:07.690 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.13148783999999999, 2024-09-14T04:51:07.690 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:51:07.690 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:51:07.690 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:51:07.690 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:51:07.690 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:51:07.690 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 56679 2024-09-14T04:51:07.690 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.691 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:51:07.691 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.691 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:51:07.691 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.383285+0000", 2024-09-14T04:51:07.691 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.691 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.691 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.691 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:51:07.691 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.383285+0000", 2024-09-14T04:51:07.691 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.691 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.691 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.691 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:51:07.691 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.383279+0000", 2024-09-14T04:51:07.692 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999948 2024-09-14T04:51:07.692 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.692 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.692 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:51:07.692 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.383328+0000", 2024-09-14T04:51:07.692 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.905e-05 2024-09-14T04:51:07.692 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.692 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.692 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:51:07.692 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.383349+0000", 2024-09-14T04:51:07.692 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.0727999999999999e-05 2024-09-14T04:51:07.692 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.692 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.693 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:51:07.693 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.383367+0000", 2024-09-14T04:51:07.693 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.7818000000000001e-05 2024-09-14T04:51:07.693 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.693 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.693 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:51:07.693 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.383428+0000", 2024-09-14T04:51:07.693 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.0961999999999997e-05 2024-09-14T04:51:07.693 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.693 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.693 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:51:07.693 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.383563+0000", 2024-09-14T04:51:07.693 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013574299999999999 2024-09-14T04:51:07.694 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.694 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.694 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:51:07.694 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.383657+0000", 2024-09-14T04:51:07.694 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.3778999999999998e-05 2024-09-14T04:51:07.694 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.694 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.694 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:51:07.694 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.385347+0000", 2024-09-14T04:51:07.694 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0016898709999999999 2024-09-14T04:51:07.694 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.694 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.694 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:07.694 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.385980+0000", 2024-09-14T04:51:07.695 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00063324799999999999 2024-09-14T04:51:07.695 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.695 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.695 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:07.695 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.514701+0000", 2024-09-14T04:51:07.695 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.12872082000000001 2024-09-14T04:51:07.695 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.695 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.695 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:51:07.695 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.514743+0000", 2024-09-14T04:51:07.695 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1696999999999997e-05 2024-09-14T04:51:07.695 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.695 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.695 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:51:07.695 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:30.514772+0000", 2024-09-14T04:51:07.696 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.9539999999999998e-05 2024-09-14T04:51:07.696 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.696 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:51:07.696 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.696 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.696 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.696 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:58368 55.c 55:34aba172:::benchmark_data_smithi039_297155_object58367:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1435)", 2024-09-14T04:51:07.696 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:48:31.674369+0000", 2024-09-14T04:51:07.696 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 155.94032850599999, 2024-09-14T04:51:07.696 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.102345909, 2024-09-14T04:51:07.696 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:51:07.696 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:51:07.696 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:51:07.696 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:51:07.696 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:51:07.697 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 58368 2024-09-14T04:51:07.697 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.697 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:51:07.697 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.697 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:51:07.697 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:31.674369+0000", 2024-09-14T04:51:07.697 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.697 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.697 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.697 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:51:07.697 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:31.674369+0000", 2024-09-14T04:51:07.697 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.697 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.697 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.697 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:51:07.698 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:31.674368+0000", 2024-09-14T04:51:07.698 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:51:07.698 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.698 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.698 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:51:07.698 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:31.674395+0000", 2024-09-14T04:51:07.698 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.6244000000000001e-05 2024-09-14T04:51:07.698 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.698 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.698 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:51:07.698 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:31.674399+0000", 2024-09-14T04:51:07.698 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6099999999999999e-06 2024-09-14T04:51:07.698 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.698 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.699 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:51:07.699 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:31.674402+0000", 2024-09-14T04:51:07.699 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.089e-06 2024-09-14T04:51:07.699 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.699 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.699 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:51:07.699 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:31.677080+0000", 2024-09-14T04:51:07.699 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.002677905 2024-09-14T04:51:07.699 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.699 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.700 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:51:07.700 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:31.677235+0000", 2024-09-14T04:51:07.700 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00015502699999999999 2024-09-14T04:51:07.700 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.700 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.700 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:51:07.700 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:31.677398+0000", 2024-09-14T04:51:07.700 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00016227200000000001 2024-09-14T04:51:07.700 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.700 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.700 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:07.700 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:31.681066+0000", 2024-09-14T04:51:07.701 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0036686000000000002 2024-09-14T04:51:07.701 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.701 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.701 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:51:07.701 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:31.690046+0000", 2024-09-14T04:51:07.701 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0089799979999999995 2024-09-14T04:51:07.701 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.701 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.701 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:07.701 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:31.776614+0000", 2024-09-14T04:51:07.701 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.086567370000000005 2024-09-14T04:51:07.701 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.701 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.701 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:51:07.701 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:31.776675+0000", 2024-09-14T04:51:07.702 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.1482000000000005e-05 2024-09-14T04:51:07.702 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.702 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.702 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:51:07.702 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:31.776715+0000", 2024-09-14T04:51:07.702 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9702e-05 2024-09-14T04:51:07.702 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.702 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:51:07.702 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.702 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.702 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.702 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:58374 55.c 55:3851899b:::benchmark_data_smithi039_297155_object58373:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1435)", 2024-09-14T04:51:07.702 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:48:31.674405+0000", 2024-09-14T04:51:07.702 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 155.940291957, 2024-09-14T04:51:07.702 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.102445992, 2024-09-14T04:51:07.703 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:51:07.703 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:51:07.703 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:51:07.703 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:51:07.703 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:51:07.703 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 58374 2024-09-14T04:51:07.703 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.703 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:51:07.703 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.703 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:51:07.703 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:31.674405+0000", 2024-09-14T04:51:07.703 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.703 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.704 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:07.704 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.704 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:51:07.705 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:31.674405+0000", 2024-09-14T04:51:07.705 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.705 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.705 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.705 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:51:07.705 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:31.674405+0000", 2024-09-14T04:51:07.705 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:51:07.705 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.705 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.705 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:51:07.705 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:31.674683+0000", 2024-09-14T04:51:07.705 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00027787599999999998 2024-09-14T04:51:07.705 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.705 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.706 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:51:07.706 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:31.674691+0000", 2024-09-14T04:51:07.706 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.0669999999999992e-06 2024-09-14T04:51:07.706 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.706 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.706 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:51:07.706 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:31.674722+0000", 2024-09-14T04:51:07.706 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.1183999999999998e-05 2024-09-14T04:51:07.706 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.706 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.706 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:51:07.706 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:31.679200+0000", 2024-09-14T04:51:07.706 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.004477806 2024-09-14T04:51:07.706 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.706 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.706 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:51:07.707 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:31.679304+0000", 2024-09-14T04:51:07.707 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010384599999999999 2024-09-14T04:51:07.707 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.707 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.707 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:51:07.707 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:31.679420+0000", 2024-09-14T04:51:07.707 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000116555 2024-09-14T04:51:07.707 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.707 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.707 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:07.707 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:31.687430+0000", 2024-09-14T04:51:07.707 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0080092270000000007 2024-09-14T04:51:07.707 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.707 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.707 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:51:07.708 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:31.690051+0000", 2024-09-14T04:51:07.708 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0026215079999999998 2024-09-14T04:51:07.708 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.708 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.708 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:07.708 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:31.776761+0000", 2024-09-14T04:51:07.708 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.086709788999999995 2024-09-14T04:51:07.708 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.708 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.708 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:51:07.708 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:31.776807+0000", 2024-09-14T04:51:07.708 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6402000000000001e-05 2024-09-14T04:51:07.708 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.708 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.708 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:51:07.709 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:31.776851+0000", 2024-09-14T04:51:07.709 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.4016e-05 2024-09-14T04:51:07.709 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.709 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:51:07.709 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.709 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.709 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.709 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:92033 55.c 55:37adb734:::benchmark_data_smithi039_297155_object92032:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1439)", 2024-09-14T04:51:07.709 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:48:58.568215+0000", 2024-09-14T04:51:07.709 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 129.046482234, 2024-09-14T04:51:07.709 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.20783289799999999, 2024-09-14T04:51:07.709 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:51:07.709 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:51:07.710 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:51:07.710 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:51:07.710 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:51:07.710 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 92033 2024-09-14T04:51:07.710 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.710 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:51:07.710 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.710 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:51:07.710 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.568215+0000", 2024-09-14T04:51:07.710 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.710 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.710 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.710 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:51:07.710 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.568215+0000", 2024-09-14T04:51:07.711 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.711 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.711 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.711 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:51:07.711 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.568213+0000", 2024-09-14T04:51:07.711 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999976 2024-09-14T04:51:07.711 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.711 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.711 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:51:07.711 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.568250+0000", 2024-09-14T04:51:07.711 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7233000000000001e-05 2024-09-14T04:51:07.711 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.711 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.711 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:51:07.711 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.568274+0000", 2024-09-14T04:51:07.712 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.3632999999999999e-05 2024-09-14T04:51:07.712 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.712 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.712 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:51:07.712 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.568284+0000", 2024-09-14T04:51:07.712 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.9399999999999997e-06 2024-09-14T04:51:07.712 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.712 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.712 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:51:07.712 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.568334+0000", 2024-09-14T04:51:07.712 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.0340999999999998e-05 2024-09-14T04:51:07.712 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.712 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.713 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:51:07.713 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.568483+0000", 2024-09-14T04:51:07.713 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00014916199999999999 2024-09-14T04:51:07.713 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.713 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.713 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:51:07.713 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.568613+0000", 2024-09-14T04:51:07.713 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000129467 2024-09-14T04:51:07.713 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.713 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.713 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:51:07.713 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.571701+0000", 2024-09-14T04:51:07.713 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0030888909999999999 2024-09-14T04:51:07.713 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.714 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.714 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:07.714 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.575626+0000", 2024-09-14T04:51:07.714 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0039244670000000001 2024-09-14T04:51:07.714 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.714 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.714 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:07.714 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.775900+0000", 2024-09-14T04:51:07.714 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.20027356599999999 2024-09-14T04:51:07.715 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.715 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.715 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:51:07.715 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.776008+0000", 2024-09-14T04:51:07.715 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010897 2024-09-14T04:51:07.715 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.715 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.715 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:51:07.715 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:48:58.776048+0000", 2024-09-14T04:51:07.715 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9548000000000002e-05 2024-09-14T04:51:07.715 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.715 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:51:07.715 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.715 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.716 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.716 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.45632.0:131291 55.d e1443/1419 55:b9ad60fd:::benchmark_data_smithi039_297155_object7358:head v 1443'8299, mlcod=1443'8296)", 2024-09-14T04:51:07.716 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:49:38.987654+0000", 2024-09-14T04:51:07.716 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 88.627043259999994, 2024-09-14T04:51:07.716 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.116035096, 2024-09-14T04:51:07.716 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:51:07.716 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:51:07.716 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:51:07.716 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.716 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:51:07.716 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.987654+0000", 2024-09-14T04:51:07.716 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.717 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.717 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.717 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:51:07.717 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.987654+0000", 2024-09-14T04:51:07.717 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.717 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.717 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.717 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:51:07.717 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.987653+0000", 2024-09-14T04:51:07.717 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:51:07.717 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.717 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.717 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:51:07.717 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.987661+0000", 2024-09-14T04:51:07.718 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.5150000000000006e-06 2024-09-14T04:51:07.718 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.718 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.718 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:51:07.718 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.987677+0000", 2024-09-14T04:51:07.718 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5424e-05 2024-09-14T04:51:07.718 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.718 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.718 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:51:07.718 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.987684+0000", 2024-09-14T04:51:07.718 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.4970000000000004e-06 2024-09-14T04:51:07.718 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.718 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.718 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:51:07.719 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.987734+0000", 2024-09-14T04:51:07.719 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.9888000000000001e-05 2024-09-14T04:51:07.719 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.719 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.719 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:51:07.719 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.987756+0000", 2024-09-14T04:51:07.719 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.1268999999999999e-05 2024-09-14T04:51:07.719 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.719 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.719 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:51:07.719 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.990691+0000", 2024-09-14T04:51:07.719 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0029356270000000001 2024-09-14T04:51:07.719 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.719 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.719 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:51:07.719 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.103689+0000", 2024-09-14T04:51:07.720 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.112998022 2024-09-14T04:51:07.720 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.720 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:51:07.720 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.720 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.720 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.720 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.45632.0:131297 55.d e1443/1419 55:bed42fa5:::benchmark_data_smithi039_297155_object7364:head v 1443'8300, mlcod=1443'8297)", 2024-09-14T04:51:07.720 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:49:38.997240+0000", 2024-09-14T04:51:07.720 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 88.617456966999995, 2024-09-14T04:51:07.720 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.119010659, 2024-09-14T04:51:07.720 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:51:07.720 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:51:07.720 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:51:07.720 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.721 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:51:07.721 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.997240+0000", 2024-09-14T04:51:07.721 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.721 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.721 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.721 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:51:07.721 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.997240+0000", 2024-09-14T04:51:07.721 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.721 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.721 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.721 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:51:07.721 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.997239+0000", 2024-09-14T04:51:07.721 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:51:07.721 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.721 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.722 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:51:07.722 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:38.997249+0000", 2024-09-14T04:51:07.722 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.4700000000000008e-06 2024-09-14T04:51:07.722 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.722 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.722 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:51:07.722 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.103165+0000", 2024-09-14T04:51:07.722 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.105915967 2024-09-14T04:51:07.722 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.722 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.722 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:51:07.722 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.103181+0000", 2024-09-14T04:51:07.722 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.6568000000000001e-05 2024-09-14T04:51:07.722 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.722 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.723 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:51:07.723 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.103720+0000", 2024-09-14T04:51:07.723 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00053891900000000001 2024-09-14T04:51:07.723 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.723 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.723 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:51:07.723 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.103740+0000", 2024-09-14T04:51:07.723 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.0239000000000001e-05 2024-09-14T04:51:07.723 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.723 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.723 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:51:07.723 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.116201+0000", 2024-09-14T04:51:07.723 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.012460219 2024-09-14T04:51:07.723 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.724 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.724 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:51:07.724 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.116251+0000", 2024-09-14T04:51:07.724 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.0368000000000003e-05 2024-09-14T04:51:07.724 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.724 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:51:07.724 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.724 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.724 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.724 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.45632.0:131311 55.1 e1443/1437 55:8be6fca8:::benchmark_data_smithi039_297155_object7378:head v 1443'8237, mlcod=1443'8235)", 2024-09-14T04:51:07.724 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:49:39.009653+0000", 2024-09-14T04:51:07.724 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 88.605044077000002, 2024-09-14T04:51:07.724 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.10595510399999999, 2024-09-14T04:51:07.724 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:51:07.724 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:51:07.725 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:51:07.725 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.725 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:51:07.725 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.009653+0000", 2024-09-14T04:51:07.725 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.725 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.725 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.725 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:51:07.725 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.009653+0000", 2024-09-14T04:51:07.725 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:51:07.725 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.725 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.725 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:51:07.725 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.009652+0000", 2024-09-14T04:51:07.725 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:51:07.726 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.726 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.726 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:51:07.726 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.009661+0000", 2024-09-14T04:51:07.726 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.1640000000000007e-06 2024-09-14T04:51:07.726 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.726 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.726 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:51:07.726 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.103394+0000", 2024-09-14T04:51:07.726 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.093733261999999998 2024-09-14T04:51:07.726 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.726 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.726 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:51:07.726 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.103406+0000", 2024-09-14T04:51:07.727 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1348999999999999e-05 2024-09-14T04:51:07.727 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.727 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.727 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:51:07.727 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.103608+0000", 2024-09-14T04:51:07.727 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00020250199999999999 2024-09-14T04:51:07.727 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.727 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.727 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:51:07.727 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.103629+0000", 2024-09-14T04:51:07.727 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.1038000000000001e-05 2024-09-14T04:51:07.727 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.727 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.727 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:51:07.727 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.115559+0000", 2024-09-14T04:51:07.727 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.011929245 2024-09-14T04:51:07.728 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:51:07.728 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:51:07.728 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:51:07.728 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:39.115608+0000", 2024-09-14T04:51:07.728 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.9801e-05 2024-09-14T04:51:07.728 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.728 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:51:07.728 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.728 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:51:07.728 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:51:07.728 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:51:07.729 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:07 smithi129 ceph-mon[101316]: pgmap v2672: 99 pgs: 99 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2024-09-14T04:51:07.729 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:51:07.743 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:07 smithi039 ceph-mon[106470]: pgmap v2672: 99 pgs: 99 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2024-09-14T04:51:07.743 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:51:07.744 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:07 smithi039 ceph-mon[111235]: pgmap v2672: 99 pgs: 99 active+clean; 2.6 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2024-09-14T04:51:07.744 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:51:07.749 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:07.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:07.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:07.851 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:07.888 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:07.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:07.889 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:07.990 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:08.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:08.035 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:08.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:08.136 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:08.176 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:08.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:08.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:08.278 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:08.309 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:08.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:08.310 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:08.390 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:51:08.411 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:08.446 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:08.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:08.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:08.548 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:08.593 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:08.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:08.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:08.695 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:08.733 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:08.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:08.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:08.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:51:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:51:08] "GET /metrics HTTP/1.1" 200 36754 "" "Prometheus/2.51.0" 2024-09-14T04:51:08.835 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:08.874 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:08.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:08.875 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:08.976 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:09.008 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:09.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:09.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:09.110 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:09.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:09.142 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:09.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:09.243 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:09.247 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-14T04:51:09.283 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:09.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:09.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:09.385 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:09.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:09.428 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:09.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:09.476 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:51:09.529 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:09.539 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:09 smithi039 ceph-mon[106470]: pgmap v2673: 99 pgs: 99 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2024-09-14T04:51:09.575 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:09.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:09.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:09.676 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:09.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:09.737 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:09.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:09.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:09 smithi039 ceph-mon[111235]: pgmap v2673: 99 pgs: 99 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2024-09-14T04:51:09.839 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:09.879 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:09.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:09.880 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:09.981 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:10.019 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:10.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:10.020 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:10.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:09 smithi129 ceph-mon[101316]: pgmap v2673: 99 pgs: 99 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 634 op/s 2024-09-14T04:51:10.121 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:10.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:10.157 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:10.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:10.259 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:10.301 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:10.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:10.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:10.403 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:10.446 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:10.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:10.447 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:10.548 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:10.584 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:10.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:10.585 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:10.686 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:10.725 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:10.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:10.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:10.826 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:10.858 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:10.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:10.859 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:10.960 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:10.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:10.991 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:10.991 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:11.092 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:11.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:11.138 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:11.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:11.239 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:11.278 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:11.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:11.279 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:11.379 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:11.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:11.413 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:11.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:11.514 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:11.554 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:11.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:11.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:11.656 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:11.687 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:11.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:11.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:11.761 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:11 smithi039 ceph-mon[106470]: pgmap v2674: 99 pgs: 99 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 979 op/s 2024-09-14T04:51:11.761 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:11 smithi039 ceph-mon[111235]: pgmap v2674: 99 pgs: 99 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 979 op/s 2024-09-14T04:51:11.789 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:11.824 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:11.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:11.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:11.926 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:11.956 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:11.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:11.958 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:11.965 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:51:12.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:11 smithi129 ceph-mon[101316]: pgmap v2674: 99 pgs: 99 active+clean; 2.3 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 979 op/s 2024-09-14T04:51:12.040 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:51:12.059 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:12.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:12.107 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:12.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:12.208 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:12.247 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:12.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:12.248 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:12.349 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:12.389 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:12.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:12.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:12.490 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:12.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:12.531 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:12.531 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:12.566 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is unset 2024-09-14T04:51:12.572 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.3 to 1 (10000) 2024-09-14T04:51:12.632 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:12.657 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:12 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1561075843' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:51:12.657 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:12 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2771203950' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:51:12.657 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:12 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:51:12.657 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:12 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1561075843' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:51:12.657 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:12 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2771203950' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:51:12.657 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:12 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:51:12.671 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:12.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:12.672 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:12.773 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:12.810 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:12.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:12.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:12.912 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:12.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:12.956 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:12.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:13.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:12 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1561075843' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:51:13.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:12 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2771203950' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:51:13.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:12 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:51:13.057 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:13.094 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:13.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:13.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:13.196 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:13.226 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:13.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:13.227 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:13.327 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:13.359 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:13.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:13.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:13.461 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:13.500 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:13.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:13.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:13.602 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:13.634 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:13.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:13.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:13.736 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:13.766 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:13.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:13.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:13.868 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:13.916 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:13.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:13.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:14.018 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:14.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:13 smithi129 ceph-mon[101316]: pgmap v2675: 99 pgs: 99 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 933 op/s 2024-09-14T04:51:14.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:13 smithi129 ceph-mon[101316]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:51:14.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:13 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T04:51:14.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:13 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1561075843' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:51:14.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:13 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-14T04:51:14.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:13 smithi129 ceph-mon[101316]: osdmap e1459: 8 total, 8 up, 8 in 2024-09-14T04:51:14.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:13 smithi039 ceph-mon[106470]: pgmap v2675: 99 pgs: 99 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 933 op/s 2024-09-14T04:51:14.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:13 smithi039 ceph-mon[106470]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:51:14.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:13 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T04:51:14.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:13 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1561075843' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:51:14.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:13 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-14T04:51:14.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:13 smithi039 ceph-mon[106470]: osdmap e1459: 8 total, 8 up, 8 in 2024-09-14T04:51:14.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:13 smithi039 ceph-mon[111235]: pgmap v2675: 99 pgs: 99 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 933 op/s 2024-09-14T04:51:14.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:13 smithi039 ceph-mon[111235]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:51:14.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:13 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T04:51:14.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:13 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1561075843' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:51:14.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:13 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-14T04:51:14.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:13 smithi039 ceph-mon[111235]: osdmap e1459: 8 total, 8 up, 8 in 2024-09-14T04:51:14.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:14.042 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:14.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:14.143 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:14.165 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:14.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:14.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:14.267 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:14.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:14.308 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:14.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:14.409 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:14.447 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:14.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:14.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:14.549 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:14.581 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:14.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:14.582 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:14.683 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:14.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:14.726 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:14.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:14.827 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:14.868 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:14.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:14.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:14.969 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:15.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:15.008 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:15.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:15.109 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:15.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:15.149 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:15.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:15.250 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:15.293 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:15.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:15.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:15.395 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:15.438 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:15.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:15.439 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:15.539 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:15.592 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:15.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:15.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:15.694 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:15.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:15.736 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:15.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:15.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:15 smithi039 ceph-mon[106470]: pgmap v2677: 99 pgs: 99 active+clean; 2.0 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T04:51:15.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:15 smithi039 ceph-mon[111235]: pgmap v2677: 99 pgs: 99 active+clean; 2.0 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T04:51:15.837 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:15.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:15.872 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:15.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:15.952 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 4 1 2024-09-14T04:51:15.973 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:16.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:16.013 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:16.013 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:16.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:15 smithi129 ceph-mon[101316]: pgmap v2677: 99 pgs: 99 active+clean; 2.0 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s 2024-09-14T04:51:16.114 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:16.157 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:16.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:16.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:16.259 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:16.284 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:16.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:16.285 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:16.386 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:16.423 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:16.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:16.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:16.525 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:16.564 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:16.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:16.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:16.665 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:16.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:16.698 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:16.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:16.799 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:16.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:16.838 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:16.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:16.939 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:16.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:16.977 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:16.977 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:17.059 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:51:17.078 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:17.115 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:17.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:17.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:17.216 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:17.272 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:17.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:17.273 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:17.376 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:17.439 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:17.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:17.440 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:17.540 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:17.582 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:17.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:17.583 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:17.666 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:17 smithi039 ceph-mon[106470]: pgmap v2678: 99 pgs: 99 active+clean; 2.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2024-09-14T04:51:17.666 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:17 smithi039 ceph-mon[111235]: pgmap v2678: 99 pgs: 99 active+clean; 2.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2024-09-14T04:51:17.683 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:17.712 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:17.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:17.713 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:17.813 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:17.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:17.848 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:17.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:17.876 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:51:17.876 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:51:17.922 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:51:17.922 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:51:17.949 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:17.954 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:51:17.954 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:51:17.981 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:17.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:17.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:17.992 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:51:17.992 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:51:18.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:17 smithi129 ceph-mon[101316]: pgmap v2678: 99 pgs: 99 active+clean; 2.0 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2024-09-14T04:51:18.022 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:51:18.022 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:51:18.053 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:51:18.053 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:51:18.084 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:18.090 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:51:18.090 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:51:18.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:18.127 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:18.127 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:18.128 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:51:18.128 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:51:18.229 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:18.270 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:18.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:18.271 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:18.372 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:18.406 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:18.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:18.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:18.508 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:18.543 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:18.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:18.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:18.645 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:18.683 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:18.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:18.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:18.784 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:18.819 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:18.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:18.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:18.891 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:18 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1133258899' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:51:18.891 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:51:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:51:18] "GET /metrics HTTP/1.1" 200 36754 "" "Prometheus/2.51.0" 2024-09-14T04:51:18.891 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:18 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1133258899' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:51:18.922 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:18.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:18.962 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:18.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:19.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:18 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1133258899' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:51:19.063 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:19.105 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:19.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:19.106 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:19.207 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:19.238 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:19.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:19.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:19.340 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:19.371 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:19.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:19.372 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:19.472 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:19.511 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:19.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:19.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:19.613 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:19.647 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:19.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:19.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:19.749 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:19.782 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:19.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:19.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:19.879 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:19 smithi039 ceph-mon[106470]: pgmap v2679: 99 pgs: 99 active+clean; 2.0 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 904 op/s 2024-09-14T04:51:19.879 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:19 smithi039 ceph-mon[111235]: pgmap v2679: 99 pgs: 99 active+clean; 2.0 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 904 op/s 2024-09-14T04:51:19.884 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:19.927 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:19.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:19.928 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:20.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:19 smithi129 ceph-mon[101316]: pgmap v2679: 99 pgs: 99 active+clean; 2.0 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 904 op/s 2024-09-14T04:51:20.029 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:20.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:20.066 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:20.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:20.167 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:20.204 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:20.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:20.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:20.306 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:20.346 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:20.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:20.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:20.448 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:20.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:20.481 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:20.482 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:20.582 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:20.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:20.618 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:20.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:20.719 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:20.754 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:20.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:20.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:20.856 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:20.886 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:20.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:20.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:20.988 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:21.022 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:21.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:21.023 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:21.123 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:21.166 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:21.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:21.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:21.268 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:21.294 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:21.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:21.295 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:21.396 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:21.431 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:21.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:21.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:21.533 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:21.570 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:21.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:21.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:21.672 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:21.705 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:21.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:21.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:21.807 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:21.835 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:21.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:21.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:21.937 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:21.973 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:21.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:21.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:21 smithi129 ceph-mon[101316]: pgmap v2680: 99 pgs: 99 active+clean; 1.7 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 908 op/s 2024-09-14T04:51:22.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:21 smithi039 ceph-mon[106470]: pgmap v2680: 99 pgs: 99 active+clean; 1.7 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 908 op/s 2024-09-14T04:51:22.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:21 smithi039 ceph-mon[111235]: pgmap v2680: 99 pgs: 99 active+clean; 1.7 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 908 op/s 2024-09-14T04:51:22.075 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:22.106 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:22.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:22.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:22.208 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:22.238 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:22.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:22.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:22.340 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:22.398 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:22.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:22.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:22.500 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:22.522 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:22.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:22.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:22.624 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:22.665 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:22.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:22.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:22.712 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:51:22.712 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:51:22.767 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:22.804 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:22.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:22.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:22.906 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:22.937 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:22.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:22.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:23.038 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:23.071 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:23.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:23.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:23.173 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:23.195 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:23.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:23.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:23.297 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:23.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:23.332 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:23.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:23.433 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:23.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:23.466 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:23.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:23 smithi039 ceph-mon[106470]: pgmap v2681: 99 pgs: 99 active+clean; 1.7 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 914 op/s 2024-09-14T04:51:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:23 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:51:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:51:23.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:23 smithi039 ceph-mon[111235]: pgmap v2681: 99 pgs: 99 active+clean; 1.7 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 914 op/s 2024-09-14T04:51:23.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:23 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:51:23.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:51:23.567 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:23.601 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:23.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:23.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:23.703 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:23.736 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:23.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:23.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:23 smithi129 ceph-mon[101316]: pgmap v2681: 99 pgs: 99 active+clean; 1.7 GiB data, 8.9 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 914 op/s 2024-09-14T04:51:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:23 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:51:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:51:23.838 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:23.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:23.869 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:23.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:23.971 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:24.015 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:24.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:24.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:24.055 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-14T04:51:24.100 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:51:24.100 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:51:24.117 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:24.153 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:24.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:24.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:24.255 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:24.311 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:24.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:24.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:24.413 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:24.451 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:24.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:24.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:24.552 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:24.588 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:24.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:24.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:24.690 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:24.735 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:24.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:24.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:24.836 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:24.874 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:24.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:24.875 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:24.976 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:25.014 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:25.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:25.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:25.116 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:25.146 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:25.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:25.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:25.248 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:25.283 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:25.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:25.284 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:25.293 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:51:25.385 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:25.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:25.421 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:25.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:25.523 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:25.568 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:25.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:25.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:25.670 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:25.710 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:25.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:25.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:25 smithi129 ceph-mon[101316]: pgmap v2682: 99 pgs: 99 active+clean; 1.5 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s 2024-09-14T04:51:25.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:25 smithi039 ceph-mon[106470]: pgmap v2682: 99 pgs: 99 active+clean; 1.5 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s 2024-09-14T04:51:25.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:25 smithi039 ceph-mon[111235]: pgmap v2682: 99 pgs: 99 active+clean; 1.5 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.14k op/s 2024-09-14T04:51:25.812 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:25.851 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:25.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:25.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:25.953 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:25.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:25.975 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:25.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:26.076 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:26.114 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:26.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:26.115 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:26.216 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:26.250 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:26.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:26.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:26.352 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:26.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:26.390 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:26.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:26.491 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:26.526 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:26.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:26.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:26.628 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:26.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:26.668 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:26.668 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:26.769 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:26.805 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:26.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:26.806 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:26.907 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:26.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:26.939 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:26.940 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:27.041 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:27.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:27.073 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:27.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:27.174 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:27.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:27.216 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:27.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:27.317 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:27.351 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:27.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:27.353 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:27.453 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:27.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:27.494 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:27.494 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:27.595 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:27.598 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:27 smithi039 ceph-mon[106470]: pgmap v2683: 99 pgs: 99 active+clean; 1.4 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 778 op/s 2024-09-14T04:51:27.598 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:27 smithi039 ceph-mon[111235]: pgmap v2683: 99 pgs: 99 active+clean; 1.4 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 778 op/s 2024-09-14T04:51:27.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:27.628 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:27.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:27.729 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:27.767 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:27.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:27.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:27.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:27 smithi129 ceph-mon[101316]: pgmap v2683: 99 pgs: 99 active+clean; 1.4 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 778 op/s 2024-09-14T04:51:27.869 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:27.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:27.902 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:27.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:28.004 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:28.039 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:28.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:28.040 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:28.141 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:28.170 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:28.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:28.171 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:28.183 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:51:28.273 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:28.308 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:28.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:28.313 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:28.414 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:28.455 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:28.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:28.456 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:28.557 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:28.585 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:28.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:28.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:28.687 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:28.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:28.744 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:28.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:28.845 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:28.881 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:51:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:51:28] "GET /metrics HTTP/1.1" 200 36738 "" "Prometheus/2.51.0" 2024-09-14T04:51:28.906 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:28.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:28.907 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:29.008 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:29.046 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:29.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:29.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:29.148 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:29.196 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:29.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:29.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:29.297 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:29.319 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:29.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:29.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:29.346 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is set 2024-09-14T04:51:29.349 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.4 to 1 (10000) 2024-09-14T04:51:29.421 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:29.462 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:29.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:29.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:29.563 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:29.573 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:29 smithi129 ceph-mon[101316]: pgmap v2684: 99 pgs: 99 active+clean; 1.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2024-09-14T04:51:29.574 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:29 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2604597747' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:51:29.574 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:29 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1925007644' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:51:29.604 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:29.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:29.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:29.705 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:29.722 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:29 smithi039 ceph-mon[106470]: pgmap v2684: 99 pgs: 99 active+clean; 1.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2024-09-14T04:51:29.722 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:29 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2604597747' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:51:29.722 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:29 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1925007644' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:51:29.723 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:29 smithi039 ceph-mon[111235]: pgmap v2684: 99 pgs: 99 active+clean; 1.4 GiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 802 op/s 2024-09-14T04:51:29.723 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:29 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2604597747' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:51:29.723 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:29 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1925007644' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:51:29.743 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:29.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:29.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:29.845 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:29.875 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:29.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:29.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:29.976 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:30.009 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:30.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:30.011 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:30.111 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:30.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:30.149 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:30.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:30.250 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:30.282 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:30.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:30.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:30.384 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:30.417 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:30.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:30.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:30.519 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:30.527 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:51:30.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:30 smithi039 ceph-mon[106470]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:51:30.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:30 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2604597747' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:51:30.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:30 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1925007644' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:51:30.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:30 smithi039 ceph-mon[106470]: osdmap e1460: 8 total, 8 up, 8 in 2024-09-14T04:51:30.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:30 smithi039 ceph-mon[111235]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:51:30.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:30 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2604597747' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:51:30.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:30 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1925007644' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:51:30.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:30 smithi039 ceph-mon[111235]: osdmap e1460: 8 total, 8 up, 8 in 2024-09-14T04:51:30.576 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:30.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:30.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:30.678 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:30.714 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:30.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:30.716 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:30.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:30 smithi129 ceph-mon[101316]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:51:30.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:30 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2604597747' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:51:30.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:30 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1925007644' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:51:30.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:30 smithi129 ceph-mon[101316]: osdmap e1460: 8 total, 8 up, 8 in 2024-09-14T04:51:30.817 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:30.852 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:30.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:30.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:30.954 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:30.986 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:30.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:30.987 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:31.089 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:31.115 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:31.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:31.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:31.217 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:31.258 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:31.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:31.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:31.360 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:31.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:31.408 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:31.408 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:31.509 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:31.543 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:31.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:31.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:31.645 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:31.678 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:31.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:31.679 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:31.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:31 smithi129 ceph-mon[101316]: pgmap v2686: 99 pgs: 99 active+clean; 1.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2024-09-14T04:51:31.780 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:31.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:31 smithi039 ceph-mon[106470]: pgmap v2686: 99 pgs: 99 active+clean; 1.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2024-09-14T04:51:31.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:31 smithi039 ceph-mon[111235]: pgmap v2686: 99 pgs: 99 active+clean; 1.2 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 907 op/s 2024-09-14T04:51:31.823 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:31.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:31.824 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:31.925 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:31.927 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 5 1 2024-09-14T04:51:31.963 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:31.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:31.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:32.065 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:32.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:32.109 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:32.109 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:32.210 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:32.249 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:32.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:32.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:32.351 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:32.380 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:32.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:32.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:32.482 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:32.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:32.527 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:32.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:32.628 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:32.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:32.669 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:32.669 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:32.771 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:32.812 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:32.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:32.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:32.914 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:32.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:32.968 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:32.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:33.069 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:33.075 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:51:33.104 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:33.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:33.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:33.206 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:33.245 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:33.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:33.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:33.347 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:33.358 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:33 smithi039 ceph-mon[106470]: pgmap v2687: 99 pgs: 99 active+clean; 1.1 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 897 op/s 2024-09-14T04:51:33.391 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:33.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:33.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:33.493 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:33.529 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:33.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:33.530 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:33.610 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:51:33.631 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:33.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:33.684 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:33.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:33.750 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:33 smithi039 ceph-mon[111235]: pgmap v2687: 99 pgs: 99 active+clean; 1.1 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 897 op/s 2024-09-14T04:51:33.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:33 smithi129 ceph-mon[101316]: pgmap v2687: 99 pgs: 99 active+clean; 1.1 GiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 897 op/s 2024-09-14T04:51:33.786 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:33.830 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:33.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:33.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:33.885 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:51:33.932 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:33.981 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:33.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:33.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:34.084 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:34.122 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:34.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:34.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:34.224 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:34.264 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:34.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:34.265 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:34.365 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:34.413 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:34.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:34.414 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:34.514 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:34.547 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:34.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:34.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:34.649 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:34.683 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:34.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:34.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:34.785 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:34.822 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:34.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:34.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:34.924 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:34.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:34.962 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:34.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:35.063 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:35.104 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:35.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:35.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:35.206 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:35.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:35.242 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:35.242 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:35.343 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:35.386 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:35.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:35.387 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:35.488 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:35.493 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:51:35.493 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:51:35.517 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:35.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:35.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:35.619 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:35.654 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:35.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:35.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:35.756 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:35.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:35 smithi129 ceph-mon[101316]: pgmap v2688: 99 pgs: 99 active+clean; 949 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 882 op/s 2024-09-14T04:51:35.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:35 smithi039 ceph-mon[106470]: pgmap v2688: 99 pgs: 99 active+clean; 949 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 882 op/s 2024-09-14T04:51:35.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:35 smithi039 ceph-mon[111235]: pgmap v2688: 99 pgs: 99 active+clean; 949 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 882 op/s 2024-09-14T04:51:35.795 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:35.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:35.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:35.897 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:35.933 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:35.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:35.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:36.035 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:36.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:36.072 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:36.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:36.173 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:36.209 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:36.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:36.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:36.311 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:36.349 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:36.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:36.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:36.451 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:36.487 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:36.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:36.488 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:36.589 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:36.622 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:36.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:36.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:36.724 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:36.757 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:36.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:36.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:36.859 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:36.899 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:36.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:36.900 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:36.933 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-14T04:51:37.001 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:37.032 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:37.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:37.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:37.135 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:37.165 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:37.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:37.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:37.267 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:37.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:37.306 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:37.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:37.407 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:37.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:37.440 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:37.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:37.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:37 smithi039 ceph-mon[106470]: pgmap v2689: 99 pgs: 99 active+clean; 931 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 885 op/s 2024-09-14T04:51:37.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:51:37.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:51:37.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:37 smithi039 ceph-mon[111235]: pgmap v2689: 99 pgs: 99 active+clean; 931 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 885 op/s 2024-09-14T04:51:37.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:51:37.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:51:37.542 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:37.581 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:37.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:37.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:37.683 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:37.705 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:37.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:37.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:37 smithi129 ceph-mon[101316]: pgmap v2689: 99 pgs: 99 active+clean; 931 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 885 op/s 2024-09-14T04:51:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:51:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:51:37.807 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:37.831 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:37.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:37.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:37.933 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:37.970 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:37.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:37.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:38.072 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:38.107 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:38.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:38.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:38.209 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:38.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:38.255 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:38.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:38.356 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:38.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:38.396 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:38.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:38.497 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:38.535 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:38.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:38.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:38.628 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:51:38.629 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:51:38.629 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:51:38.629 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:51:38.629 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:51:38.629 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:51:38.629 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:51:38.629 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:51:38.630 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:51:38.630 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:51:38.630 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:51:38.630 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:51:38.630 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:51:38.630 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:51:38.638 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:38.642 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:51:38.643 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:51:38.643 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:38 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2888771685' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:51:38.643 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:38 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2888771685' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:51:38.678 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:38.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:38.679 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:38.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:38 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2888771685' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:51:38.780 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:38.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:38.814 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:38.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:38.915 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:38.949 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:38.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:38.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:38.990 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:51:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:51:38] "GET /metrics HTTP/1.1" 200 36740 "" "Prometheus/2.51.0" 2024-09-14T04:51:39.051 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:39.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:39.085 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:39.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:39.187 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:39.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:39.221 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:39.221 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:39.322 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:39.352 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:39.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:39.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:39.454 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:39.488 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:39.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:39.489 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:39.590 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:39.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:39.634 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:39.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:39.735 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:39.763 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:39.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:39.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:39.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:39 smithi129 ceph-mon[101316]: pgmap v2690: 99 pgs: 99 active+clean; 892 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2024-09-14T04:51:39.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:39 smithi039 ceph-mon[106470]: pgmap v2690: 99 pgs: 99 active+clean; 892 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2024-09-14T04:51:39.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:39 smithi039 ceph-mon[111235]: pgmap v2690: 99 pgs: 99 active+clean; 892 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2024-09-14T04:51:39.864 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:39.900 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:39.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:39.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:40.002 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:40.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:40.039 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:40.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:40.140 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:40.170 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:40.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:40.171 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:40.272 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:40.316 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:40.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:40.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:40.418 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:40.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:40.452 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:40.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:40.554 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:40.591 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:40.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:40.593 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:40.694 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:40.738 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:40.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:40.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:40.839 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:40.874 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:40.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:40.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:40.976 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:41.021 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:41.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:41.022 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:41.123 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:41.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:41.157 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:41.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:41.174 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:51:41.258 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:41.291 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:41.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:41.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:41.393 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:41.420 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:41.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:41.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:41.422 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is set 2024-09-14T04:51:41.468 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:51:41.522 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:41.569 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:41 smithi039 ceph-mon[106470]: pgmap v2691: 99 pgs: 99 active+clean; 660 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 858 op/s 2024-09-14T04:51:41.569 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:41 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2273833306' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:51:41.569 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:41 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:51:41.570 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:41 smithi039 ceph-mon[111235]: pgmap v2691: 99 pgs: 99 active+clean; 660 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 858 op/s 2024-09-14T04:51:41.570 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:41 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2273833306' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:51:41.570 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:41 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:51:41.580 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:41.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:41.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:41.682 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:41.711 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:41.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:41.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:41 smithi129 ceph-mon[101316]: pgmap v2691: 99 pgs: 99 active+clean; 660 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 858 op/s 2024-09-14T04:51:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:41 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2273833306' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:51:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:41 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:51:41.813 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:41.845 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:41.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:41.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:41.947 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:41.987 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:41.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:41.989 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:42.089 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:42.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:42.130 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:42.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:42.231 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:42.270 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:42.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:42.271 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:42.372 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:42.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:42.409 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:42.409 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:42.430 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.5 to 1 (10000) 2024-09-14T04:51:42.510 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:42.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:42 smithi039 ceph-mon[106470]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:51:42.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:42 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:51:42.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:42 smithi039 ceph-mon[106470]: osdmap e1461: 8 total, 8 up, 8 in 2024-09-14T04:51:42.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:42 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3692848428' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:51:42.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:42 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:51:42.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:42 smithi039 ceph-mon[111235]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:51:42.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:42 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:51:42.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:42 smithi039 ceph-mon[111235]: osdmap e1461: 8 total, 8 up, 8 in 2024-09-14T04:51:42.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:42 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3692848428' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:51:42.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:42 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:51:42.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:42.550 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:42.550 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:42.651 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:42.687 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:42.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:42.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:42.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:42 smithi129 ceph-mon[101316]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:51:42.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:42 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:51:42.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:42 smithi129 ceph-mon[101316]: osdmap e1461: 8 total, 8 up, 8 in 2024-09-14T04:51:42.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:42 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3692848428' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:51:42.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:42 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:51:42.788 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:42.819 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:42.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:42.820 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:42.921 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:42.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:42.961 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:42.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:43.062 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:43.095 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:43.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:43.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:43.197 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:43.217 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:43.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:43.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:43.319 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:43.343 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:43.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:43.344 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:43.444 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:43.484 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:43.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:43.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:43.586 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:43.614 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:43.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:43.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:43.717 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:43.761 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:43.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:43.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:43.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:43 smithi129 ceph-mon[101316]: pgmap v2693: 99 pgs: 99 active+clean; 607 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 899 op/s 2024-09-14T04:51:43.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:43 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-14T04:51:43.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:43 smithi129 ceph-mon[101316]: osdmap e1462: 8 total, 8 up, 8 in 2024-09-14T04:51:43.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:43 smithi039 ceph-mon[106470]: pgmap v2693: 99 pgs: 99 active+clean; 607 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 899 op/s 2024-09-14T04:51:43.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:43 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-14T04:51:43.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:43 smithi039 ceph-mon[106470]: osdmap e1462: 8 total, 8 up, 8 in 2024-09-14T04:51:43.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:43 smithi039 ceph-mon[111235]: pgmap v2693: 99 pgs: 99 active+clean; 607 MiB data, 5.6 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 899 op/s 2024-09-14T04:51:43.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:43 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-14T04:51:43.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:43 smithi039 ceph-mon[111235]: osdmap e1462: 8 total, 8 up, 8 in 2024-09-14T04:51:43.863 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:43.900 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:43.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:43.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:44.002 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:44.037 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:44.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:44.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:44.139 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:44.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:44.182 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:44.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:44.283 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:44.320 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:44.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:44.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:44.422 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:44.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:44.480 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:44.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:44.581 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:44.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:44.614 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:44.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:44.708 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:51:44.715 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:44.793 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:44.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:44.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:44.874 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 6 1 2024-09-14T04:51:44.895 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:44.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:44.963 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:44.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:45.064 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:45.138 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:45.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:45.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:45.240 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:45.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:45.282 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:45.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:45.383 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:45.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:45.418 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:45.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:45.425 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:51:45.475 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:51:45.476 INFO:teuthology.orchestra.run.smithi039.stdout: "size": 20, 2024-09-14T04:51:45.476 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 600, 2024-09-14T04:51:45.476 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [ 2024-09-14T04:51:45.476 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.476 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233353 53.5 53:a6a852bc:::benchmark_data_smithi039_224949_object103260:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1354)", 2024-09-14T04:51:45.476 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.817844+0000", 2024-09-14T04:51:45.476 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 580.653680915, 2024-09-14T04:51:45.476 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.249583763, 2024-09-14T04:51:45.476 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:51:45.476 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:51:45.476 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T04:51:45.477 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:51:45.477 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:51:45.477 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:51:45.477 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233353 2024-09-14T04:51:45.477 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.477 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:51:45.477 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.477 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:51:45.477 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.817844+0000", 2024-09-14T04:51:45.477 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.477 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.477 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.477 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:51:45.478 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.817844+0000", 2024-09-14T04:51:45.478 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.478 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.478 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.478 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:51:45.478 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.817845+0000", 2024-09-14T04:51:45.478 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.502e-06 2024-09-14T04:51:45.478 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.478 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.478 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:51:45.478 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.817849+0000", 2024-09-14T04:51:45.478 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.9750000000000001e-06 2024-09-14T04:51:45.478 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.478 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.478 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:51:45.479 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.817868+0000", 2024-09-14T04:51:45.479 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9162000000000001e-05 2024-09-14T04:51:45.479 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.479 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.479 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:51:45.479 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.817877+0000", 2024-09-14T04:51:45.479 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.7919999999999998e-06 2024-09-14T04:51:45.479 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.479 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.479 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.479 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.817984+0000", 2024-09-14T04:51:45.479 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000106606 2024-09-14T04:51:45.479 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.480 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.480 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:51:45.480 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.818094+0000", 2024-09-14T04:51:45.480 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000110072 2024-09-14T04:51:45.480 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.480 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.480 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:51:45.480 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.822688+0000", 2024-09-14T04:51:45.480 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0045945530000000004 2024-09-14T04:51:45.480 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.480 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.480 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.480 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.843026+0000", 2024-09-14T04:51:45.481 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.020337233 2024-09-14T04:51:45.481 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.481 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.481 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:51:45.481 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.843037+0000", 2024-09-14T04:51:45.481 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1022000000000001e-05 2024-09-14T04:51:45.481 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.481 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.481 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.481 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.813667+0000", 2024-09-14T04:51:45.481 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.97063038499999998 2024-09-14T04:51:45.481 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.481 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.482 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:51:45.482 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.813680+0000", 2024-09-14T04:51:45.482 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3013000000000001e-05 2024-09-14T04:51:45.482 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.482 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.482 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.482 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.015538+0000", 2024-09-14T04:51:45.482 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.201857916 2024-09-14T04:51:45.482 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.482 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.482 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for missing object", 2024-09-14T04:51:45.482 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.015727+0000", 2024-09-14T04:51:45.482 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000188863 2024-09-14T04:51:45.483 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.483 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.483 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.483 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.067130+0000", 2024-09-14T04:51:45.483 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.051403345000000003 2024-09-14T04:51:45.483 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.483 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.483 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:51:45.483 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.067427+0000", 2024-09-14T04:51:45.483 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00029732400000000001 2024-09-14T04:51:45.483 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.483 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:51:45.483 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.483 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.483 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.484 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233394 53.5 53:ae57f18f:::benchmark_data_smithi039_224949_object103301:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-09-14T04:51:45.484 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.894024+0000", 2024-09-14T04:51:45.484 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 580.57750103399997, 2024-09-14T04:51:45.484 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1667164959999998, 2024-09-14T04:51:45.484 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:51:45.484 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:51:45.484 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:51:45.484 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:51:45.484 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:51:45.484 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:51:45.484 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233394 2024-09-14T04:51:45.484 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.484 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:51:45.485 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.485 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:51:45.485 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894024+0000", 2024-09-14T04:51:45.485 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.485 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.485 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.485 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:51:45.485 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894024+0000", 2024-09-14T04:51:45.485 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.485 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.485 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.485 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:51:45.485 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894025+0000", 2024-09-14T04:51:45.485 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.548e-06 2024-09-14T04:51:45.485 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.486 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.486 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:51:45.486 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894031+0000", 2024-09-14T04:51:45.486 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5790000000000003e-06 2024-09-14T04:51:45.486 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.486 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.486 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:51:45.486 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894042+0000", 2024-09-14T04:51:45.486 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1494e-05 2024-09-14T04:51:45.486 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.486 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.486 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:51:45.486 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894050+0000", 2024-09-14T04:51:45.486 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6969999999999993e-06 2024-09-14T04:51:45.486 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.487 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.487 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.487 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894114+0000", 2024-09-14T04:51:45.487 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3689000000000006e-05 2024-09-14T04:51:45.487 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.487 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.487 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:51:45.487 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.894123+0000", 2024-09-14T04:51:45.487 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.7480000000000005e-06 2024-09-14T04:51:45.487 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.487 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.487 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.487 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.813712+0000", 2024-09-14T04:51:45.488 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.91958880200000004 2024-09-14T04:51:45.488 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.488 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.488 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:51:45.488 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.813719+0000", 2024-09-14T04:51:45.488 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.6320000000000002e-06 2024-09-14T04:51:45.488 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.488 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.488 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.488 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.021411+0000", 2024-09-14T04:51:45.488 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.207691874 2024-09-14T04:51:45.489 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.489 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.489 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:51:45.489 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.021549+0000", 2024-09-14T04:51:45.489 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013877299999999999 2024-09-14T04:51:45.489 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.489 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.489 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,3,7", 2024-09-14T04:51:45.489 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.022037+0000", 2024-09-14T04:51:45.489 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00048770299999999998 2024-09-14T04:51:45.489 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.490 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.490 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:51:45.490 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.037131+0000", 2024-09-14T04:51:45.490 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.015093766999999999 2024-09-14T04:51:45.490 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.490 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.490 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.491 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.037173+0000", 2024-09-14T04:51:45.491 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2518999999999997e-05 2024-09-14T04:51:45.491 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.491 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.491 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.491 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.049744+0000", 2024-09-14T04:51:45.491 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.012570789000000001 2024-09-14T04:51:45.491 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.491 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.491 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.491 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.060452+0000", 2024-09-14T04:51:45.491 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.010707836 2024-09-14T04:51:45.492 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.492 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.492 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:51:45.492 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.060604+0000", 2024-09-14T04:51:45.492 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000152032 2024-09-14T04:51:45.492 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.492 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.492 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:51:45.492 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.060740+0000", 2024-09-14T04:51:45.492 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000136014 2024-09-14T04:51:45.492 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.492 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:51:45.492 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.493 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.493 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.493 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233404 53.5 53:a80a488e:::benchmark_data_smithi039_224949_object103311:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-09-14T04:51:45.493 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.904066+0000", 2024-09-14T04:51:45.493 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 580.56745839799999, 2024-09-14T04:51:45.493 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1569935720000002, 2024-09-14T04:51:45.493 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:51:45.493 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:51:45.493 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:51:45.493 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:51:45.493 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:51:45.493 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:51:45.493 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233404 2024-09-14T04:51:45.494 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.494 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:51:45.494 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.494 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:51:45.494 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.904066+0000", 2024-09-14T04:51:45.494 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.494 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.494 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.494 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:51:45.494 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.904066+0000", 2024-09-14T04:51:45.494 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.494 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.494 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.495 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:51:45.495 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.904068+0000", 2024-09-14T04:51:45.495 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3939999999999999e-06 2024-09-14T04:51:45.495 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.495 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.495 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:51:45.495 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.904073+0000", 2024-09-14T04:51:45.495 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1410000000000004e-06 2024-09-14T04:51:45.495 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.495 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.495 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:51:45.495 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.904089+0000", 2024-09-14T04:51:45.495 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6464999999999999e-05 2024-09-14T04:51:45.496 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.496 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.496 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:51:45.496 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.904097+0000", 2024-09-14T04:51:45.496 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.3299999999999999e-06 2024-09-14T04:51:45.496 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.496 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.496 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.496 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.904162+0000", 2024-09-14T04:51:45.496 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.4561999999999996e-05 2024-09-14T04:51:45.496 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.497 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.497 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:51:45.497 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.904171+0000", 2024-09-14T04:51:45.497 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.4010000000000003e-06 2024-09-14T04:51:45.497 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.497 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.497 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.497 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.813742+0000", 2024-09-14T04:51:45.497 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.90957067000000003 2024-09-14T04:51:45.497 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.497 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.497 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:51:45.497 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.813749+0000", 2024-09-14T04:51:45.498 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.3000000000000004e-06 2024-09-14T04:51:45.498 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.498 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.498 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.498 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.030045+0000", 2024-09-14T04:51:45.498 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.216295345 2024-09-14T04:51:45.498 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.498 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.498 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:51:45.498 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.030205+0000", 2024-09-14T04:51:45.498 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00016016500000000001 2024-09-14T04:51:45.499 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.499 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.499 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,3,7", 2024-09-14T04:51:45.499 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.030346+0000", 2024-09-14T04:51:45.499 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000140822 2024-09-14T04:51:45.499 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.499 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.499 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.499 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.037216+0000", 2024-09-14T04:51:45.499 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0068704179999999997 2024-09-14T04:51:45.499 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.499 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.499 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:51:45.500 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.043582+0000", 2024-09-14T04:51:45.500 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0063658120000000002 2024-09-14T04:51:45.500 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.500 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.500 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.500 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.050030+0000", 2024-09-14T04:51:45.500 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.006448311 2024-09-14T04:51:45.500 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.500 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.500 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.500 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.060809+0000", 2024-09-14T04:51:45.501 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.010778375999999999 2024-09-14T04:51:45.501 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.501 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.501 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:51:45.501 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.060876+0000", 2024-09-14T04:51:45.501 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7692000000000002e-05 2024-09-14T04:51:45.501 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.501 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.501 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:51:45.501 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.061060+0000", 2024-09-14T04:51:45.501 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000183368 2024-09-14T04:51:45.501 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.501 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:51:45.502 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.502 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.502 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.502 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233415 53.5 53:a93a97f2:::benchmark_data_smithi039_224949_object103322:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1355)", 2024-09-14T04:51:45.502 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:04.931528+0000", 2024-09-14T04:51:45.502 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 580.53999625899996, 2024-09-14T04:51:45.502 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1304777339999998, 2024-09-14T04:51:45.502 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:51:45.502 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:51:45.502 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:51:45.502 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:51:45.502 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:51:45.503 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:51:45.503 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233415 2024-09-14T04:51:45.503 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.503 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:51:45.503 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.503 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:51:45.503 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.931528+0000", 2024-09-14T04:51:45.503 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.503 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.503 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.503 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:51:45.503 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.931528+0000", 2024-09-14T04:51:45.503 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.503 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.504 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.504 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:51:45.504 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.931530+0000", 2024-09-14T04:51:45.504 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3790000000000001e-06 2024-09-14T04:51:45.504 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.504 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.504 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:51:45.504 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.931535+0000", 2024-09-14T04:51:45.504 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1320000000000002e-06 2024-09-14T04:51:45.504 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.504 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.504 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:51:45.505 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.931547+0000", 2024-09-14T04:51:45.505 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1756000000000001e-05 2024-09-14T04:51:45.505 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.505 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.505 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:51:45.505 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.931552+0000", 2024-09-14T04:51:45.505 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4199999999999998e-06 2024-09-14T04:51:45.505 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.505 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.505 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.505 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.931631+0000", 2024-09-14T04:51:45.505 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9358999999999997e-05 2024-09-14T04:51:45.505 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.505 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.506 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:51:45.506 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:04.931643+0000", 2024-09-14T04:51:45.506 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1990999999999999e-05 2024-09-14T04:51:45.506 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.506 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.506 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.506 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.813776+0000", 2024-09-14T04:51:45.506 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.88213241600000003 2024-09-14T04:51:45.506 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.506 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.506 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:51:45.506 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.813781+0000", 2024-09-14T04:51:45.506 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6760000000000004e-06 2024-09-14T04:51:45.507 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.507 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.507 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.507 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.033700+0000", 2024-09-14T04:51:45.507 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.219918396 2024-09-14T04:51:45.507 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.507 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.507 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:51:45.507 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.033813+0000", 2024-09-14T04:51:45.507 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000113175 2024-09-14T04:51:45.507 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.507 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.507 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,3,7", 2024-09-14T04:51:45.508 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.033941+0000", 2024-09-14T04:51:45.508 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012790300000000001 2024-09-14T04:51:45.508 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.508 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.508 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.508 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.041351+0000", 2024-09-14T04:51:45.508 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0074097670000000003 2024-09-14T04:51:45.508 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.508 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.508 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:51:45.508 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.043590+0000", 2024-09-14T04:51:45.508 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0022397670000000001 2024-09-14T04:51:45.509 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.509 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.509 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.509 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.052263+0000", 2024-09-14T04:51:45.509 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0086722819999999999 2024-09-14T04:51:45.509 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.509 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.509 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.509 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.061172+0000", 2024-09-14T04:51:45.509 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0089090799999999998 2024-09-14T04:51:45.509 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.509 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.510 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:51:45.510 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.061224+0000", 2024-09-14T04:51:45.510 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.198e-05 2024-09-14T04:51:45.510 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.510 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.510 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:51:45.510 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.062006+0000", 2024-09-14T04:51:45.510 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00078225500000000004 2024-09-14T04:51:45.510 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.510 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:51:45.510 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.510 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.510 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.511 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233355 53.1 53:81e7c0a9:::benchmark_data_smithi039_224949_object103262:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1356)", 2024-09-14T04:51:45.511 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:05.820399+0000", 2024-09-14T04:51:45.511 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 579.65112522100003, 2024-09-14T04:51:45.511 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2541154749999999, 2024-09-14T04:51:45.511 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:51:45.511 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:51:45.511 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:51:45.511 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:51:45.511 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:51:45.511 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:51:45.511 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233355 2024-09-14T04:51:45.511 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.511 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:51:45.511 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.511 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:51:45.512 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820399+0000", 2024-09-14T04:51:45.512 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.512 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.512 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.512 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:51:45.512 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820399+0000", 2024-09-14T04:51:45.512 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.512 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.512 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.512 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:51:45.512 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820401+0000", 2024-09-14T04:51:45.512 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.44e-06 2024-09-14T04:51:45.512 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.513 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.513 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:51:45.513 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820405+0000", 2024-09-14T04:51:45.513 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5040000000000004e-06 2024-09-14T04:51:45.513 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.513 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.513 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:51:45.513 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820418+0000", 2024-09-14T04:51:45.513 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2515e-05 2024-09-14T04:51:45.513 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.513 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.513 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:51:45.513 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820426+0000", 2024-09-14T04:51:45.513 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.5199999999999997e-06 2024-09-14T04:51:45.513 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.513 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.514 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.514 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.821400+0000", 2024-09-14T04:51:45.514 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00097377300000000004 2024-09-14T04:51:45.514 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.514 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.514 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:51:45.514 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.821409+0000", 2024-09-14T04:51:45.514 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.2920000000000001e-06 2024-09-14T04:51:45.514 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.514 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.514 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.514 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.041896+0000", 2024-09-14T04:51:45.514 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2204865730000001 2024-09-14T04:51:45.514 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.515 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.515 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:51:45.515 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.045201+0000", 2024-09-14T04:51:45.515 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0033054400000000002 2024-09-14T04:51:45.515 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.515 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.515 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,5,6", 2024-09-14T04:51:45.515 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.045657+0000", 2024-09-14T04:51:45.515 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000455861 2024-09-14T04:51:45.515 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.515 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.515 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.515 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.065205+0000", 2024-09-14T04:51:45.516 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.019547563 2024-09-14T04:51:45.516 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.516 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.516 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:51:45.516 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.065220+0000", 2024-09-14T04:51:45.516 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4728e-05 2024-09-14T04:51:45.516 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.516 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.516 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.516 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.073947+0000", 2024-09-14T04:51:45.516 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.008727304 2024-09-14T04:51:45.516 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.516 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.516 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.517 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.074187+0000", 2024-09-14T04:51:45.517 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00024006799999999999 2024-09-14T04:51:45.517 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.517 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.517 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:51:45.517 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.074326+0000", 2024-09-14T04:51:45.517 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000139216 2024-09-14T04:51:45.517 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.517 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.517 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:51:45.517 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.074515+0000", 2024-09-14T04:51:45.517 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00018867800000000001 2024-09-14T04:51:45.517 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.517 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:51:45.518 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.518 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.518 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.518 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233406 53.1 53:86743dc5:::benchmark_data_smithi039_224949_object103313:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1356)", 2024-09-14T04:51:45.518 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:05.820440+0000", 2024-09-14T04:51:45.518 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 579.65108454699998, 2024-09-14T04:51:45.518 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2550835579999999, 2024-09-14T04:51:45.518 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:51:45.518 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:51:45.518 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:51:45.518 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:51:45.518 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:51:45.518 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:51:45.518 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233406 2024-09-14T04:51:45.518 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.518 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:51:45.519 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.519 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:51:45.519 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820440+0000", 2024-09-14T04:51:45.519 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.519 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.519 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.519 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:51:45.519 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820440+0000", 2024-09-14T04:51:45.519 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.519 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.519 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.519 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:51:45.519 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820440+0000", 2024-09-14T04:51:45.520 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:45.521 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.8999999999999997e-07 2024-09-14T04:51:45.521 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.521 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.521 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:51:45.521 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820443+0000", 2024-09-14T04:51:45.521 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4420000000000001e-06 2024-09-14T04:51:45.521 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.521 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.522 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:51:45.522 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820460+0000", 2024-09-14T04:51:45.522 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7183000000000001e-05 2024-09-14T04:51:45.522 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.522 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.522 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:51:45.522 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820467+0000", 2024-09-14T04:51:45.522 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1810000000000004e-06 2024-09-14T04:51:45.522 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.522 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.522 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.522 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.821435+0000", 2024-09-14T04:51:45.522 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00096804599999999997 2024-09-14T04:51:45.523 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.523 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.523 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:51:45.523 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.821443+0000", 2024-09-14T04:51:45.523 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.2010000000000002e-06 2024-09-14T04:51:45.523 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.523 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.523 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.523 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.058430+0000", 2024-09-14T04:51:45.523 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.236987088 2024-09-14T04:51:45.523 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.523 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.523 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:51:45.523 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.058568+0000", 2024-09-14T04:51:45.523 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013826100000000001 2024-09-14T04:51:45.524 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.524 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.524 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,5,6", 2024-09-14T04:51:45.524 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.058665+0000", 2024-09-14T04:51:45.524 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.6904999999999995e-05 2024-09-14T04:51:45.524 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.524 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.524 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:51:45.524 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.065226+0000", 2024-09-14T04:51:45.524 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0065607410000000001 2024-09-14T04:51:45.524 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.525 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.525 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.525 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.075244+0000", 2024-09-14T04:51:45.525 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.010018216 2024-09-14T04:51:45.525 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.525 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.525 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.525 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.075286+0000", 2024-09-14T04:51:45.525 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.1987000000000003e-05 2024-09-14T04:51:45.525 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.525 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.525 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.525 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.075346+0000", 2024-09-14T04:51:45.525 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0649999999999997e-05 2024-09-14T04:51:45.526 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.526 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.526 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:51:45.526 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.075409+0000", 2024-09-14T04:51:45.526 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2948999999999996e-05 2024-09-14T04:51:45.526 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.526 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.526 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:51:45.526 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.075524+0000", 2024-09-14T04:51:45.526 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000114218 2024-09-14T04:51:45.526 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.526 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:51:45.526 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.526 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.526 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.526 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:233412 53.1 53:8a59a845:::benchmark_data_smithi039_224949_object103319:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1356)", 2024-09-14T04:51:45.527 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:05.820476+0000", 2024-09-14T04:51:45.527 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 579.65104846700001, 2024-09-14T04:51:45.527 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2554734700000001, 2024-09-14T04:51:45.527 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:51:45.527 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:51:45.527 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:51:45.527 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:51:45.527 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:51:45.527 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:51:45.527 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 233412 2024-09-14T04:51:45.527 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.527 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:51:45.527 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.527 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:51:45.528 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820476+0000", 2024-09-14T04:51:45.528 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.528 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.528 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.528 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:51:45.528 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820476+0000", 2024-09-14T04:51:45.528 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.528 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.528 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.528 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:51:45.528 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820477+0000", 2024-09-14T04:51:45.528 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.0900000000000002e-07 2024-09-14T04:51:45.528 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.528 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.528 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:51:45.529 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820479+0000", 2024-09-14T04:51:45.529 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.7410000000000001e-06 2024-09-14T04:51:45.529 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.529 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.529 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:51:45.529 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820486+0000", 2024-09-14T04:51:45.529 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8789999999999997e-06 2024-09-14T04:51:45.529 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.529 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.529 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:51:45.529 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.820492+0000", 2024-09-14T04:51:45.529 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5060000000000003e-06 2024-09-14T04:51:45.529 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.530 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.530 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.530 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.821477+0000", 2024-09-14T04:51:45.530 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00098510799999999995 2024-09-14T04:51:45.530 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.530 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.530 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:51:45.530 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:05.821484+0000", 2024-09-14T04:51:45.530 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.34e-06 2024-09-14T04:51:45.530 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.530 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.530 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.531 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.061870+0000", 2024-09-14T04:51:45.531 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.240385472 2024-09-14T04:51:45.531 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.531 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.531 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:51:45.531 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.061995+0000", 2024-09-14T04:51:45.531 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012508899999999999 2024-09-14T04:51:45.531 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.531 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.531 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,5,6", 2024-09-14T04:51:45.531 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.062117+0000", 2024-09-14T04:51:45.531 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000122459 2024-09-14T04:51:45.531 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.532 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.532 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:51:45.532 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.073961+0000", 2024-09-14T04:51:45.532 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.011843907000000001 2024-09-14T04:51:45.532 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.532 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.532 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.532 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.075568+0000", 2024-09-14T04:51:45.532 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001607362 2024-09-14T04:51:45.532 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.532 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.532 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.532 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.075662+0000", 2024-09-14T04:51:45.532 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.3374000000000004e-05 2024-09-14T04:51:45.533 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.533 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.533 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.533 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.075708+0000", 2024-09-14T04:51:45.533 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6300000000000001e-05 2024-09-14T04:51:45.533 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.533 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.533 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:51:45.533 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.075763+0000", 2024-09-14T04:51:45.533 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4898000000000002e-05 2024-09-14T04:51:45.533 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.533 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.533 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:51:45.533 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:07.075950+0000", 2024-09-14T04:51:45.534 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00018652600000000001 2024-09-14T04:51:45.534 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.534 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:51:45.534 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.534 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.534 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.534 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242571 53.9 53:9bbdb0f8:::benchmark_data_smithi039_224949_object112478:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1362)", 2024-09-14T04:51:45.534 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:19.469714+0000", 2024-09-14T04:51:45.534 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 566.00181010100005, 2024-09-14T04:51:45.534 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0060407760000001, 2024-09-14T04:51:45.534 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:51:45.534 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:51:45.534 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:51:45.534 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:51:45.534 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:51:45.535 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:51:45.535 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242571 2024-09-14T04:51:45.535 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.535 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:51:45.535 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.535 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:51:45.535 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.469714+0000", 2024-09-14T04:51:45.535 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.535 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.535 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.535 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:51:45.535 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.469714+0000", 2024-09-14T04:51:45.535 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.535 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.536 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.536 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:51:45.536 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.469716+0000", 2024-09-14T04:51:45.536 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.367e-06 2024-09-14T04:51:45.536 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.536 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.536 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:51:45.536 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.469721+0000", 2024-09-14T04:51:45.536 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4219999999999999e-06 2024-09-14T04:51:45.536 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.536 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.536 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:51:45.536 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.469734+0000", 2024-09-14T04:51:45.537 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2826999999999999e-05 2024-09-14T04:51:45.537 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.537 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.537 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:51:45.537 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.469740+0000", 2024-09-14T04:51:45.537 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.9800000000000003e-06 2024-09-14T04:51:45.537 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.537 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.537 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.537 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.469802+0000", 2024-09-14T04:51:45.537 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1816999999999994e-05 2024-09-14T04:51:45.538 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.539 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.539 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:51:45.539 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.469818+0000", 2024-09-14T04:51:45.539 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5829999999999999e-05 2024-09-14T04:51:45.539 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.539 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.539 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.539 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.469925+0000", 2024-09-14T04:51:45.540 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0001073389999999 2024-09-14T04:51:45.540 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.540 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.541 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:51:45.541 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.470024+0000", 2024-09-14T04:51:45.541 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.8766999999999998e-05 2024-09-14T04:51:45.541 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.541 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.541 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 5,6", 2024-09-14T04:51:45.541 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.470131+0000", 2024-09-14T04:51:45.541 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000106746 2024-09-14T04:51:45.541 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.541 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.541 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.542 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.473316+0000", 2024-09-14T04:51:45.542 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0031856670000000001 2024-09-14T04:51:45.542 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.542 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.542 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.542 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.473374+0000", 2024-09-14T04:51:45.542 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7612999999999999e-05 2024-09-14T04:51:45.542 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.542 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.542 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:51:45.542 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.475684+0000", 2024-09-14T04:51:45.542 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0023106250000000002 2024-09-14T04:51:45.543 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.543 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.543 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:51:45.543 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.475716+0000", 2024-09-14T04:51:45.543 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.1748000000000002e-05 2024-09-14T04:51:45.543 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.543 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.543 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:51:45.543 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.475755+0000", 2024-09-14T04:51:45.543 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.9028e-05 2024-09-14T04:51:45.543 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.543 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:51:45.543 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.543 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.543 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.544 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45053.0:242577 53.9 53:9813f1d4:::benchmark_data_smithi039_224949_object112484:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1362)", 2024-09-14T04:51:45.544 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:42:19.476824+0000", 2024-09-14T04:51:45.544 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 565.994700309, 2024-09-14T04:51:45.544 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0070515010000001, 2024-09-14T04:51:45.544 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:51:45.544 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:51:45.544 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:51:45.544 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:51:45.544 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45053", 2024-09-14T04:51:45.544 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2905340658", 2024-09-14T04:51:45.544 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 242577 2024-09-14T04:51:45.544 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.544 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:51:45.544 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.544 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:51:45.545 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.476824+0000", 2024-09-14T04:51:45.545 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.545 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.545 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.545 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:51:45.545 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.476824+0000", 2024-09-14T04:51:45.545 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.545 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.545 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.545 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:51:45.545 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.476825+0000", 2024-09-14T04:51:45.545 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2470000000000001e-06 2024-09-14T04:51:45.545 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.546 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.546 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:51:45.546 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.476830+0000", 2024-09-14T04:51:45.546 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3019999999999996e-06 2024-09-14T04:51:45.546 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.546 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.546 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:51:45.546 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.476841+0000", 2024-09-14T04:51:45.546 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1508999999999999e-05 2024-09-14T04:51:45.546 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.546 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.546 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:51:45.546 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.476849+0000", 2024-09-14T04:51:45.546 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0010000000000001e-06 2024-09-14T04:51:45.546 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.546 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.547 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.547 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.476908+0000", 2024-09-14T04:51:45.547 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.8530999999999999e-05 2024-09-14T04:51:45.547 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.547 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.547 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:51:45.547 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:19.476917+0000", 2024-09-14T04:51:45.547 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.3479999999999993e-06 2024-09-14T04:51:45.547 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.547 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.547 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.547 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.473414+0000", 2024-09-14T04:51:45.547 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.99649714599999994 2024-09-14T04:51:45.548 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.548 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.548 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:51:45.548 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.473490+0000", 2024-09-14T04:51:45.548 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.5517000000000002e-05 2024-09-14T04:51:45.548 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.548 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.548 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 5,6", 2024-09-14T04:51:45.548 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.473584+0000", 2024-09-14T04:51:45.548 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.3749e-05 2024-09-14T04:51:45.548 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.548 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.548 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.548 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.478047+0000", 2024-09-14T04:51:45.549 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0044635760000000003 2024-09-14T04:51:45.549 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.549 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.549 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:51:45.549 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.480787+0000", 2024-09-14T04:51:45.549 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0027394759999999998 2024-09-14T04:51:45.549 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.549 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.549 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.549 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.483767+0000", 2024-09-14T04:51:45.549 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.002980496 2024-09-14T04:51:45.549 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.549 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.549 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:51:45.550 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.483820+0000", 2024-09-14T04:51:45.550 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3022999999999997e-05 2024-09-14T04:51:45.550 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.550 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.550 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:51:45.550 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:42:20.483876+0000", 2024-09-14T04:51:45.550 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.558e-05 2024-09-14T04:51:45.550 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.550 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:51:45.550 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.550 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.550 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.550 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:135592 54.8 54:199531d3:::benchmark_data_smithi039_264334_object9195:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e1403)", 2024-09-14T04:51:45.550 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:57.312996+0000", 2024-09-14T04:51:45.550 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 408.15852825799999, 2024-09-14T04:51:45.551 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1455672880000001, 2024-09-14T04:51:45.551 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:51:45.551 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:51:45.551 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T04:51:45.551 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:51:45.551 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:51:45.551 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:51:45.551 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 135592 2024-09-14T04:51:45.551 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.551 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:51:45.551 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.551 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:51:45.551 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.312996+0000", 2024-09-14T04:51:45.551 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.551 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.552 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.552 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:51:45.552 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.312996+0000", 2024-09-14T04:51:45.552 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.552 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.552 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.552 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:51:45.552 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.312998+0000", 2024-09-14T04:51:45.552 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9309999999999998e-06 2024-09-14T04:51:45.552 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.552 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.552 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:51:45.552 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313006+0000", 2024-09-14T04:51:45.552 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.3520000000000001e-06 2024-09-14T04:51:45.552 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.553 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.553 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:51:45.553 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313025+0000", 2024-09-14T04:51:45.553 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9437999999999999e-05 2024-09-14T04:51:45.553 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.553 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.553 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:51:45.553 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313035+0000", 2024-09-14T04:51:45.553 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0179e-05 2024-09-14T04:51:45.553 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.553 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.553 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.553 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313130+0000", 2024-09-14T04:51:45.553 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.4851000000000004e-05 2024-09-14T04:51:45.553 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.554 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.554 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op must wait for map", 2024-09-14T04:51:45.554 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313136+0000", 2024-09-14T04:51:45.554 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.156e-06 2024-09-14T04:51:45.554 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.554 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.554 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.554 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.319910+0000", 2024-09-14T04:51:45.554 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0067736519999999998 2024-09-14T04:51:45.554 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.554 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.554 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:51:45.554 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.319919+0000", 2024-09-14T04:51:45.554 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.3440000000000007e-06 2024-09-14T04:51:45.554 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.554 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.555 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.555 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.456466+0000", 2024-09-14T04:51:45.555 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.136546662 2024-09-14T04:51:45.555 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.555 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.555 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:51:45.555 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.458564+0000", 2024-09-14T04:51:45.555 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0020977230000000001 2024-09-14T04:51:45.555 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.555 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:51:45.555 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.555 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.555 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.555 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:135646 54.8 54:17b125bc:::benchmark_data_smithi039_264334_object9249:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1403)", 2024-09-14T04:51:45.555 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:57.313060+0000", 2024-09-14T04:51:45.556 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 408.158464229, 2024-09-14T04:51:45.556 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1613178790000001, 2024-09-14T04:51:45.556 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:51:45.556 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:51:45.556 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:51:45.556 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:51:45.556 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:51:45.556 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:51:45.556 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 135646 2024-09-14T04:51:45.556 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.556 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:51:45.556 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.556 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:51:45.557 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313060+0000", 2024-09-14T04:51:45.557 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.557 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.557 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.557 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:51:45.557 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313060+0000", 2024-09-14T04:51:45.557 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.557 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.557 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.557 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:51:45.557 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313061+0000", 2024-09-14T04:51:45.557 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1099999999999995e-07 2024-09-14T04:51:45.557 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.557 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.558 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:51:45.558 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313064+0000", 2024-09-14T04:51:45.558 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.3380000000000001e-06 2024-09-14T04:51:45.558 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.558 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.558 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:51:45.558 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313083+0000", 2024-09-14T04:51:45.558 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8858000000000001e-05 2024-09-14T04:51:45.558 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.558 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.558 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:51:45.558 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313091+0000", 2024-09-14T04:51:45.558 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.8909999999999995e-06 2024-09-14T04:51:45.558 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.558 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.559 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.559 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313270+0000", 2024-09-14T04:51:45.559 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000179235 2024-09-14T04:51:45.559 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.559 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.559 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting_for_map not empty", 2024-09-14T04:51:45.559 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313276+0000", 2024-09-14T04:51:45.559 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.9070000000000004e-06 2024-09-14T04:51:45.559 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.559 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.559 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.559 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.320755+0000", 2024-09-14T04:51:45.559 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0074782829999999996 2024-09-14T04:51:45.560 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.560 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.560 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:51:45.560 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.320766+0000", 2024-09-14T04:51:45.560 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1898e-05 2024-09-14T04:51:45.560 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.560 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.560 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.560 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.459100+0000", 2024-09-14T04:51:45.560 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.138333673 2024-09-14T04:51:45.560 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.560 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.560 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:51:45.560 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.459693+0000", 2024-09-14T04:51:45.561 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000592408 2024-09-14T04:51:45.561 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.561 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.561 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4,7", 2024-09-14T04:51:45.561 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.459811+0000", 2024-09-14T04:51:45.561 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000118398 2024-09-14T04:51:45.561 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.561 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.561 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.561 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.468895+0000", 2024-09-14T04:51:45.561 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0090842330000000006 2024-09-14T04:51:45.561 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.561 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.562 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:51:45.562 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.468915+0000", 2024-09-14T04:51:45.562 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9643000000000001e-05 2024-09-14T04:51:45.562 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.562 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.562 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.562 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.468963+0000", 2024-09-14T04:51:45.562 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.8112000000000003e-05 2024-09-14T04:51:45.562 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.562 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.562 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.563 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.474201+0000", 2024-09-14T04:51:45.563 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00523822 2024-09-14T04:51:45.563 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.563 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.563 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:51:45.563 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.474263+0000", 2024-09-14T04:51:45.563 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1756999999999998e-05 2024-09-14T04:51:45.563 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.563 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.563 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:51:45.563 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.474378+0000", 2024-09-14T04:51:45.563 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000115314 2024-09-14T04:51:45.563 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.563 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:51:45.563 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.564 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.564 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.564 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:135649 54.8 54:10793023:::benchmark_data_smithi039_264334_object9252:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1403)", 2024-09-14T04:51:45.564 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:57.313190+0000", 2024-09-14T04:51:45.564 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 408.15833420899997, 2024-09-14T04:51:45.564 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1635118710000001, 2024-09-14T04:51:45.564 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:51:45.564 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:51:45.564 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:51:45.565 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:51:45.566 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:51:45.566 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:51:45.566 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 135649 2024-09-14T04:51:45.566 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.566 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:51:45.566 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.566 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:51:45.566 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313190+0000", 2024-09-14T04:51:45.566 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.566 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.566 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.566 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:51:45.566 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313190+0000", 2024-09-14T04:51:45.566 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.567 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.567 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.567 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:51:45.567 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313191+0000", 2024-09-14T04:51:45.567 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9500000000000002e-07 2024-09-14T04:51:45.567 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.567 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.567 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:51:45.567 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313195+0000", 2024-09-14T04:51:45.567 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.963e-06 2024-09-14T04:51:45.567 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.567 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.567 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:51:45.567 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313209+0000", 2024-09-14T04:51:45.567 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4440999999999999e-05 2024-09-14T04:51:45.567 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.568 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.568 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:51:45.568 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313218+0000", 2024-09-14T04:51:45.568 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.2369999999999992e-06 2024-09-14T04:51:45.568 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.568 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.568 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.568 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313307+0000", 2024-09-14T04:51:45.568 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.9746000000000004e-05 2024-09-14T04:51:45.568 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.568 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.568 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting_for_map not empty", 2024-09-14T04:51:45.568 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313312+0000", 2024-09-14T04:51:45.568 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7500000000000003e-06 2024-09-14T04:51:45.569 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.569 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.569 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.569 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.320795+0000", 2024-09-14T04:51:45.569 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0074831480000000002 2024-09-14T04:51:45.569 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.569 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.569 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:51:45.569 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.320802+0000", 2024-09-14T04:51:45.569 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8680000000000001e-06 2024-09-14T04:51:45.570 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.570 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.570 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.570 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.462371+0000", 2024-09-14T04:51:45.570 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.14156909 2024-09-14T04:51:45.570 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.570 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.570 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:51:45.570 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.462509+0000", 2024-09-14T04:51:45.570 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013791500000000001 2024-09-14T04:51:45.570 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.570 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.570 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4,7", 2024-09-14T04:51:45.570 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.462639+0000", 2024-09-14T04:51:45.571 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013007000000000001 2024-09-14T04:51:45.571 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.571 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.571 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:51:45.571 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.468921+0000", 2024-09-14T04:51:45.571 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0062813490000000003 2024-09-14T04:51:45.571 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.571 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.571 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.571 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.472860+0000", 2024-09-14T04:51:45.571 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0039389639999999997 2024-09-14T04:51:45.571 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.571 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.571 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.571 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.475650+0000", 2024-09-14T04:51:45.572 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0027903950000000002 2024-09-14T04:51:45.572 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.572 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.572 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.572 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.476501+0000", 2024-09-14T04:51:45.572 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00085151499999999995 2024-09-14T04:51:45.572 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.572 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.572 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:51:45.572 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.476573+0000", 2024-09-14T04:51:45.572 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1353000000000006e-05 2024-09-14T04:51:45.572 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.572 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.572 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:51:45.572 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.476702+0000", 2024-09-14T04:51:45.572 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012937199999999999 2024-09-14T04:51:45.573 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.573 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:51:45.573 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.573 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.573 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.573 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:135671 54.8 54:152b5606:::benchmark_data_smithi039_264334_object9274:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1403)", 2024-09-14T04:51:45.573 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:57.313280+0000", 2024-09-14T04:51:45.573 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 408.15824431599998, 2024-09-14T04:51:45.573 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.163696319, 2024-09-14T04:51:45.573 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:51:45.573 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:51:45.573 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:51:45.573 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:51:45.573 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:51:45.573 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:51:45.574 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 135671 2024-09-14T04:51:45.574 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.574 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:51:45.574 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.574 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:51:45.574 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313280+0000", 2024-09-14T04:51:45.574 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.574 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.574 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.574 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:51:45.574 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313280+0000", 2024-09-14T04:51:45.574 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.574 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.574 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.575 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:51:45.575 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313281+0000", 2024-09-14T04:51:45.575 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8400000000000004e-07 2024-09-14T04:51:45.575 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.575 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.575 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:51:45.575 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313285+0000", 2024-09-14T04:51:45.575 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.8820000000000003e-06 2024-09-14T04:51:45.575 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.575 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.575 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:51:45.575 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313294+0000", 2024-09-14T04:51:45.575 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.1749999999999994e-06 2024-09-14T04:51:45.575 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.575 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.575 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:51:45.576 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313301+0000", 2024-09-14T04:51:45.576 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.058e-06 2024-09-14T04:51:45.576 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.576 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.576 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.576 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313332+0000", 2024-09-14T04:51:45.576 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.0491999999999999e-05 2024-09-14T04:51:45.576 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.576 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.576 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting_for_map not empty", 2024-09-14T04:51:45.576 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.313334+0000", 2024-09-14T04:51:45.576 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.7039999999999999e-06 2024-09-14T04:51:45.576 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.576 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.577 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.577 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.320826+0000", 2024-09-14T04:51:45.577 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0074915509999999999 2024-09-14T04:51:45.577 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.577 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.577 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:51:45.577 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.320832+0000", 2024-09-14T04:51:45.577 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.5520000000000001e-06 2024-09-14T04:51:45.577 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.577 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.577 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.577 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.465552+0000", 2024-09-14T04:51:45.577 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.144719316 2024-09-14T04:51:45.578 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.578 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.578 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:51:45.578 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.465658+0000", 2024-09-14T04:51:45.578 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000105969 2024-09-14T04:51:45.578 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.578 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.578 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4,7", 2024-09-14T04:51:45.578 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.465778+0000", 2024-09-14T04:51:45.578 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000120418 2024-09-14T04:51:45.578 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.578 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.578 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.578 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.472910+0000", 2024-09-14T04:51:45.579 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0071319640000000002 2024-09-14T04:51:45.579 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.579 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.579 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:51:45.579 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.473814+0000", 2024-09-14T04:51:45.579 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000904333 2024-09-14T04:51:45.579 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.579 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.579 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.579 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.475701+0000", 2024-09-14T04:51:45.579 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001886609 2024-09-14T04:51:45.579 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.580 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.580 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.580 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.476793+0000", 2024-09-14T04:51:45.580 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001092118 2024-09-14T04:51:45.580 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.580 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.580 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:51:45.580 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.476858+0000", 2024-09-14T04:51:45.580 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.4665999999999994e-05 2024-09-14T04:51:45.580 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.580 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.580 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:51:45.580 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.476977+0000", 2024-09-14T04:51:45.580 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011882799999999999 2024-09-14T04:51:45.581 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.581 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:51:45.581 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.581 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.581 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.581 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45632.0:64534 55.1 55:81f19a9e:::benchmark_data_smithi039_297155_object64533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1437)", 2024-09-14T04:51:45.581 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:48:36.046591+0000", 2024-09-14T04:51:45.581 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 189.42493336300001, 2024-09-14T04:51:45.581 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0210583499999999, 2024-09-14T04:51:45.581 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:51:45.581 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:51:45.581 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T04:51:45.582 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:51:45.582 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45632", 2024-09-14T04:51:45.582 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:51:45.582 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 64534 2024-09-14T04:51:45.582 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.582 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:51:45.582 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.582 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:51:45.582 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046591+0000", 2024-09-14T04:51:45.582 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.582 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.582 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.583 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:51:45.583 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046591+0000", 2024-09-14T04:51:45.583 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.583 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.583 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.583 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:51:45.583 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046593+0000", 2024-09-14T04:51:45.583 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.917e-06 2024-09-14T04:51:45.583 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.583 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.583 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:51:45.583 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046616+0000", 2024-09-14T04:51:45.583 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3010000000000002e-05 2024-09-14T04:51:45.584 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.584 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.584 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:51:45.584 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046638+0000", 2024-09-14T04:51:45.584 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2059e-05 2024-09-14T04:51:45.584 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.584 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.584 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:51:45.584 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046660+0000", 2024-09-14T04:51:45.584 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1642e-05 2024-09-14T04:51:45.584 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.584 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.585 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.585 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046764+0000", 2024-09-14T04:51:45.585 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000104226 2024-09-14T04:51:45.585 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.585 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.585 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op must wait for map", 2024-09-14T04:51:45.585 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046772+0000", 2024-09-14T04:51:45.585 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7030000000000006e-06 2024-09-14T04:51:45.585 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.585 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.586 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.586 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.050944+0000", 2024-09-14T04:51:45.586 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0041725319999999996 2024-09-14T04:51:45.586 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.586 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.586 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:51:45.586 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.051119+0000", 2024-09-14T04:51:45.586 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000174458 2024-09-14T04:51:45.587 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.587 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.587 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.587 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.067560+0000", 2024-09-14T04:51:45.587 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0164416080000001 2024-09-14T04:51:45.587 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.587 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.587 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:51:45.587 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.067650+0000", 2024-09-14T04:51:45.587 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.9195000000000002e-05 2024-09-14T04:51:45.587 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.587 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:51:45.587 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.587 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.588 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.588 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45632.0:64543 55.1 55:84b0ffa6:::benchmark_data_smithi039_297155_object64542:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1437)", 2024-09-14T04:51:45.588 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:48:36.046692+0000", 2024-09-14T04:51:45.588 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 189.42483263700001, 2024-09-14T04:51:45.588 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0291501249999999, 2024-09-14T04:51:45.588 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:51:45.588 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:51:45.588 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:51:45.588 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:51:45.588 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45632", 2024-09-14T04:51:45.588 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:51:45.588 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 64543 2024-09-14T04:51:45.589 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.589 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:51:45.589 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.589 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:51:45.589 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046692+0000", 2024-09-14T04:51:45.589 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.589 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.589 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.589 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:51:45.589 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046692+0000", 2024-09-14T04:51:45.589 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.590 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.590 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.590 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:51:45.590 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046693+0000", 2024-09-14T04:51:45.590 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0900000000000001e-07 2024-09-14T04:51:45.590 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.590 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.590 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:51:45.591 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046714+0000", 2024-09-14T04:51:45.591 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1467999999999998e-05 2024-09-14T04:51:45.591 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.591 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.591 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:51:45.591 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046733+0000", 2024-09-14T04:51:45.591 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9341999999999999e-05 2024-09-14T04:51:45.591 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.591 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.592 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:51:45.592 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046743+0000", 2024-09-14T04:51:45.592 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.4080000000000004e-06 2024-09-14T04:51:45.592 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.592 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.592 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.592 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.047732+0000", 2024-09-14T04:51:45.592 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00098887600000000008 2024-09-14T04:51:45.593 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.593 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.593 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting_for_map not empty", 2024-09-14T04:51:45.593 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.047738+0000", 2024-09-14T04:51:45.593 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8079999999999999e-06 2024-09-14T04:51:45.593 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.593 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.593 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.593 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.051980+0000", 2024-09-14T04:51:45.594 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0042414929999999998 2024-09-14T04:51:45.596 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.596 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.596 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:51:45.596 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.051988+0000", 2024-09-14T04:51:45.596 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.3180000000000007e-06 2024-09-14T04:51:45.597 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.597 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.597 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.597 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.068072+0000", 2024-09-14T04:51:45.597 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0160837920000001 2024-09-14T04:51:45.597 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.597 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.598 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:51:45.598 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.068183+0000", 2024-09-14T04:51:45.598 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011097 2024-09-14T04:51:45.598 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.598 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.598 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:51:45.599 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.068290+0000", 2024-09-14T04:51:45.599 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000107227 2024-09-14T04:51:45.599 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.599 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.599 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.599 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.075604+0000", 2024-09-14T04:51:45.599 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0073141589999999998 2024-09-14T04:51:45.600 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.600 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.600 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:45 smithi039 ceph-mon[106470]: pgmap v2695: 99 pgs: 99 active+clean; 382 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-14T04:51:45.601 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:45 smithi039 ceph-mon[111235]: pgmap v2695: 99 pgs: 99 active+clean; 382 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-14T04:51:45.602 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:51:45.602 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.075707+0000", 2024-09-14T04:51:45.602 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000102948 2024-09-14T04:51:45.603 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.603 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.603 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.603 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.075759+0000", 2024-09-14T04:51:45.603 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1276999999999997e-05 2024-09-14T04:51:45.603 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.604 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.604 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:51:45.604 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.075815+0000", 2024-09-14T04:51:45.604 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5902000000000001e-05 2024-09-14T04:51:45.604 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.604 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.605 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:51:45.605 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.075842+0000", 2024-09-14T04:51:45.605 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.7528000000000001e-05 2024-09-14T04:51:45.605 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.605 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:51:45.606 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.606 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.606 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.606 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45632.0:64554 55.1 55:80cc57e1:::benchmark_data_smithi039_297155_object64553:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1437)", 2024-09-14T04:51:45.606 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:48:36.053143+0000", 2024-09-14T04:51:45.606 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 189.41838119600001, 2024-09-14T04:51:45.607 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0228815819999999, 2024-09-14T04:51:45.607 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:51:45.607 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:51:45.607 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:51:45.607 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:51:45.607 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45632", 2024-09-14T04:51:45.608 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:51:45.608 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 64554 2024-09-14T04:51:45.608 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.608 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:51:45.608 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.608 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:51:45.609 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.053143+0000", 2024-09-14T04:51:45.609 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.609 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.609 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.609 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:51:45.609 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.053143+0000", 2024-09-14T04:51:45.610 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.610 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.610 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.610 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:51:45.610 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.053145+0000", 2024-09-14T04:51:45.610 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.533e-06 2024-09-14T04:51:45.611 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.611 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.611 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:51:45.611 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.053169+0000", 2024-09-14T04:51:45.611 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4329e-05 2024-09-14T04:51:45.611 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.612 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.612 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:51:45.612 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.053180+0000", 2024-09-14T04:51:45.612 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0888e-05 2024-09-14T04:51:45.612 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.612 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.613 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:51:45.613 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.053188+0000", 2024-09-14T04:51:45.613 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6429999999999995e-06 2024-09-14T04:51:45.613 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.613 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.613 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.613 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.053797+0000", 2024-09-14T04:51:45.613 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00060973199999999998 2024-09-14T04:51:45.614 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.614 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.614 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:51:45.614 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.053833+0000", 2024-09-14T04:51:45.614 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.5256000000000003e-05 2024-09-14T04:51:45.614 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.615 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.615 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.615 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.070510+0000", 2024-09-14T04:51:45.615 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.016677018 2024-09-14T04:51:45.615 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.615 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.615 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:51:45.616 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.070645+0000", 2024-09-14T04:51:45.616 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000135358 2024-09-14T04:51:45.616 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.616 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.616 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:51:45.616 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.070801+0000", 2024-09-14T04:51:45.616 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015627499999999999 2024-09-14T04:51:45.616 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.616 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.616 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.616 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.075687+0000", 2024-09-14T04:51:45.616 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0048856430000000003 2024-09-14T04:51:45.617 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.617 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.617 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:51:45.617 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.075712+0000", 2024-09-14T04:51:45.617 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.5140999999999999e-05 2024-09-14T04:51:45.617 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.617 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.617 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.617 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.075915+0000", 2024-09-14T04:51:45.617 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000202919 2024-09-14T04:51:45.617 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.617 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.617 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:51:45.618 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.075980+0000", 2024-09-14T04:51:45.618 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.5073000000000005e-05 2024-09-14T04:51:45.618 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.618 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.618 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:51:45.618 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.076025+0000", 2024-09-14T04:51:45.618 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4774000000000002e-05 2024-09-14T04:51:45.618 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.618 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:51:45.618 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.618 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.618 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.618 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45632.0:64601 55.1 55:819f926e:::benchmark_data_smithi039_297155_object64600:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1437)", 2024-09-14T04:51:45.618 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:48:36.084827+0000", 2024-09-14T04:51:45.619 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 189.38669746900001, 2024-09-14T04:51:45.619 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.99141031400000001, 2024-09-14T04:51:45.619 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:51:45.619 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:51:45.619 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:51:45.619 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:51:45.619 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45632", 2024-09-14T04:51:45.619 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:51:45.619 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 64601 2024-09-14T04:51:45.619 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.619 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:51:45.619 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.619 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:51:45.619 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.084827+0000", 2024-09-14T04:51:45.619 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.620 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.620 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.620 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:51:45.620 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.084827+0000", 2024-09-14T04:51:45.620 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.620 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.620 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.620 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:51:45.620 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.084829+0000", 2024-09-14T04:51:45.620 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.742e-06 2024-09-14T04:51:45.620 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.620 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.620 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:51:45.621 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.084851+0000", 2024-09-14T04:51:45.621 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1676999999999999e-05 2024-09-14T04:51:45.621 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.621 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.621 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:51:45.621 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.084884+0000", 2024-09-14T04:51:45.621 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.3543000000000001e-05 2024-09-14T04:51:45.621 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.621 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.621 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:51:45.621 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.084893+0000", 2024-09-14T04:51:45.621 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.3889999999999994e-06 2024-09-14T04:51:45.621 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.621 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.622 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.622 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.084957+0000", 2024-09-14T04:51:45.622 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3336000000000005e-05 2024-09-14T04:51:45.622 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.622 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.622 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:51:45.622 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.084969+0000", 2024-09-14T04:51:45.622 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2411e-05 2024-09-14T04:51:45.622 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.622 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.622 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.622 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.073108+0000", 2024-09-14T04:51:45.622 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.98813857500000002 2024-09-14T04:51:45.622 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.623 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.623 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:51:45.623 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.073271+0000", 2024-09-14T04:51:45.623 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00016303199999999999 2024-09-14T04:51:45.623 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.623 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.623 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:51:45.623 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.073434+0000", 2024-09-14T04:51:45.623 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000162755 2024-09-14T04:51:45.623 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:45.625 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:45.625 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:45.625 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.625 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:51:45.626 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.075857+0000", 2024-09-14T04:51:45.626 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.002423209 2024-09-14T04:51:45.626 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.626 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.626 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.626 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.076080+0000", 2024-09-14T04:51:45.626 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00022328100000000001 2024-09-14T04:51:45.626 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.626 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.626 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.626 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.076130+0000", 2024-09-14T04:51:45.626 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9743000000000001e-05 2024-09-14T04:51:45.626 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.627 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.627 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:51:45.627 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.076192+0000", 2024-09-14T04:51:45.627 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1795e-05 2024-09-14T04:51:45.627 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.627 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.627 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:51:45.627 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.076237+0000", 2024-09-14T04:51:45.627 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5825999999999998e-05 2024-09-14T04:51:45.627 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.627 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:51:45.627 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.627 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.628 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.628 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45632.0:64662 55.1 55:80119e9b:::benchmark_data_smithi039_297155_object64661:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1437)", 2024-09-14T04:51:45.628 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:48:36.120270+0000", 2024-09-14T04:51:45.628 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 189.35125492700001, 2024-09-14T04:51:45.628 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.96407522300000004, 2024-09-14T04:51:45.628 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:51:45.628 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:51:45.628 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:51:45.628 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:51:45.628 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45632", 2024-09-14T04:51:45.628 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:51:45.628 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 64662 2024-09-14T04:51:45.628 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.629 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:51:45.629 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.629 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:51:45.629 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.120270+0000", 2024-09-14T04:51:45.629 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.629 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.629 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.629 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:51:45.629 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.120270+0000", 2024-09-14T04:51:45.629 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.629 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.630 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.630 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:51:45.630 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.120271+0000", 2024-09-14T04:51:45.630 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.688e-06 2024-09-14T04:51:45.630 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.630 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.630 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:51:45.630 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.120299+0000", 2024-09-14T04:51:45.630 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.8153e-05 2024-09-14T04:51:45.630 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.630 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.630 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:51:45.631 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.120316+0000", 2024-09-14T04:51:45.631 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6735000000000002e-05 2024-09-14T04:51:45.631 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.631 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.631 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:51:45.631 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.120324+0000", 2024-09-14T04:51:45.631 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7759999999999997e-06 2024-09-14T04:51:45.631 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.631 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.631 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.631 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.120531+0000", 2024-09-14T04:51:45.631 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000207426 2024-09-14T04:51:45.632 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.632 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.632 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:51:45.632 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.120546+0000", 2024-09-14T04:51:45.632 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4630000000000001e-05 2024-09-14T04:51:45.632 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.632 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.632 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.632 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.076283+0000", 2024-09-14T04:51:45.632 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.95573650200000004 2024-09-14T04:51:45.632 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.632 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.633 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:51:45.633 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.076402+0000", 2024-09-14T04:51:45.633 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000119165 2024-09-14T04:51:45.633 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.633 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.633 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:51:45.633 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.076557+0000", 2024-09-14T04:51:45.633 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015563099999999999 2024-09-14T04:51:45.633 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.633 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.633 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.633 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.084154+0000", 2024-09-14T04:51:45.633 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0075969749999999997 2024-09-14T04:51:45.634 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.634 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.634 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:51:45.634 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.084229+0000", 2024-09-14T04:51:45.634 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.4558e-05 2024-09-14T04:51:45.634 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.634 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.634 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.634 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.084273+0000", 2024-09-14T04:51:45.634 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3788999999999997e-05 2024-09-14T04:51:45.634 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.634 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.634 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:51:45.635 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.084314+0000", 2024-09-14T04:51:45.635 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.1477999999999998e-05 2024-09-14T04:51:45.635 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.635 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.635 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:51:45.635 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.084345+0000", 2024-09-14T04:51:45.635 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.0716999999999997e-05 2024-09-14T04:51:45.635 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.635 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:51:45.635 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.635 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.635 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.635 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45632.0:64703 55.1 55:85591b41:::benchmark_data_smithi039_297155_object64702:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1437)", 2024-09-14T04:51:45.635 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:48:36.148377+0000", 2024-09-14T04:51:45.636 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 189.32314788799999, 2024-09-14T04:51:45.636 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.96002874999999999, 2024-09-14T04:51:45.636 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:51:45.636 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:51:45.636 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:51:45.636 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:51:45.636 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45632", 2024-09-14T04:51:45.636 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:51:45.636 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 64703 2024-09-14T04:51:45.636 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.636 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:51:45.636 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.636 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:51:45.637 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.148377+0000", 2024-09-14T04:51:45.637 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.637 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.637 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.637 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:51:45.637 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.148377+0000", 2024-09-14T04:51:45.637 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.637 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.637 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.637 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:51:45.637 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.148378+0000", 2024-09-14T04:51:45.638 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7320000000000001e-06 2024-09-14T04:51:45.638 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.638 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.638 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:51:45.638 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.148403+0000", 2024-09-14T04:51:45.638 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4119000000000001e-05 2024-09-14T04:51:45.638 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.638 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.638 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:51:45.638 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.148419+0000", 2024-09-14T04:51:45.638 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6721000000000001e-05 2024-09-14T04:51:45.638 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.638 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.638 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:51:45.639 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.148427+0000", 2024-09-14T04:51:45.639 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6559999999999992e-06 2024-09-14T04:51:45.639 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.639 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.639 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.639 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.148496+0000", 2024-09-14T04:51:45.639 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9337000000000004e-05 2024-09-14T04:51:45.639 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.639 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.639 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:51:45.639 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.148508+0000", 2024-09-14T04:51:45.639 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1364e-05 2024-09-14T04:51:45.639 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.639 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.640 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.640 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.081665+0000", 2024-09-14T04:51:45.640 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.93315764199999995 2024-09-14T04:51:45.640 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.640 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.640 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:51:45.640 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.081808+0000", 2024-09-14T04:51:45.640 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014285799999999999 2024-09-14T04:51:45.640 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.640 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.641 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:51:45.641 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.081961+0000", 2024-09-14T04:51:45.641 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000153089 2024-09-14T04:51:45.641 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.641 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.641 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.641 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.089885+0000", 2024-09-14T04:51:45.641 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0079243109999999999 2024-09-14T04:51:45.641 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.641 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.641 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.642 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.090165+0000", 2024-09-14T04:51:45.642 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000279795 2024-09-14T04:51:45.642 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.642 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.642 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:51:45.642 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.108306+0000", 2024-09-14T04:51:45.642 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.018140763000000001 2024-09-14T04:51:45.642 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.642 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.642 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:51:45.642 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.108360+0000", 2024-09-14T04:51:45.642 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4262999999999999e-05 2024-09-14T04:51:45.643 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.643 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.643 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:51:45.643 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.108405+0000", 2024-09-14T04:51:45.643 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5099999999999998e-05 2024-09-14T04:51:45.643 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.643 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:51:45.643 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.643 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.643 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.643 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45632.0:64708 55.1 55:8b4837e3:::benchmark_data_smithi039_297155_object64707:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1437)", 2024-09-14T04:51:45.643 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:48:36.152492+0000", 2024-09-14T04:51:45.643 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 189.31903281500001, 2024-09-14T04:51:45.644 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.95604835200000005, 2024-09-14T04:51:45.644 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:51:45.644 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:51:45.644 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:51:45.644 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:51:45.644 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45632", 2024-09-14T04:51:45.644 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:51:45.645 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 64708 2024-09-14T04:51:45.645 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.645 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:51:45.645 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.645 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:51:45.645 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.152492+0000", 2024-09-14T04:51:45.645 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.645 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.645 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.645 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:51:45.645 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.152492+0000", 2024-09-14T04:51:45.645 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:51:45.645 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.645 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.646 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:51:45.646 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.152494+0000", 2024-09-14T04:51:45.646 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3709999999999998e-06 2024-09-14T04:51:45.646 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.646 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.646 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:51:45.646 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.152521+0000", 2024-09-14T04:51:45.646 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.6580999999999998e-05 2024-09-14T04:51:45.646 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.646 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.647 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:51:45.647 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.152539+0000", 2024-09-14T04:51:45.647 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8518000000000001e-05 2024-09-14T04:51:45.647 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.647 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.647 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:51:45.647 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.152549+0000", 2024-09-14T04:51:45.647 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.7710000000000001e-06 2024-09-14T04:51:45.647 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.647 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.647 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.648 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.152631+0000", 2024-09-14T04:51:45.648 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.2509999999999994e-05 2024-09-14T04:51:45.648 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.648 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.648 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:51:45.648 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.152643+0000", 2024-09-14T04:51:45.648 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1398e-05 2024-09-14T04:51:45.648 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.648 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.648 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:51:45.648 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.084389+0000", 2024-09-14T04:51:45.648 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.93174628999999998 2024-09-14T04:51:45.648 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.649 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.649 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:51:45.649 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.084495+0000", 2024-09-14T04:51:45.649 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00010586699999999999 2024-09-14T04:51:45.649 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.649 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.649 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:51:45.649 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.084661+0000", 2024-09-14T04:51:45.649 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000166298 2024-09-14T04:51:45.649 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.649 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.649 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.649 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.090110+0000", 2024-09-14T04:51:45.650 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0054491050000000001 2024-09-14T04:51:45.650 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.650 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.650 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:51:45.650 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.092751+0000", 2024-09-14T04:51:45.650 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0026402259999999999 2024-09-14T04:51:45.650 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.650 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.650 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:51:45.650 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.108464+0000", 2024-09-14T04:51:45.650 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.01571299 2024-09-14T04:51:45.650 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.651 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.651 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:51:45.651 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.108508+0000", 2024-09-14T04:51:45.651 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4283000000000003e-05 2024-09-14T04:51:45.651 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:51:45.651 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:51:45.651 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:51:45.651 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.108540+0000", 2024-09-14T04:51:45.651 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.2144000000000003e-05 2024-09-14T04:51:45.651 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.651 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:51:45.651 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.651 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:51:45.652 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:51:45.652 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:51:45.726 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:45.764 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:45.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:45.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:45.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:45 smithi129 ceph-mon[101316]: pgmap v2695: 99 pgs: 99 active+clean; 382 MiB data, 5.0 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s 2024-09-14T04:51:45.866 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:45.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:45.906 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:45.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:46.007 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:46.057 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:46.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:46.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:46.159 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:46.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:46.209 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:46.209 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:46.310 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:46.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:46.345 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:46.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:46.446 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:46.486 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:46.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:46.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:46.588 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:46.624 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:46.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:46.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:46.726 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:46.757 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:46.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:46.758 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:46.858 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:46.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:46.904 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:46.904 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:47.005 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:47.049 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:47.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:47.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:47.151 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:47.185 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:47.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:47.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:47.287 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:47.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:47.325 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:47.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:47.427 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:47.467 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:47.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:47.468 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:47.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:47 smithi039 ceph-mon[106470]: pgmap v2696: 99 pgs: 99 active+clean; 345 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-14T04:51:47.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:47 smithi039 ceph-mon[111235]: pgmap v2696: 99 pgs: 99 active+clean; 345 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-14T04:51:47.569 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:47.590 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:51:47.605 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:47.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:47.606 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:47.707 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:47.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:47.744 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:47.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:47.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:47 smithi129 ceph-mon[101316]: pgmap v2696: 99 pgs: 99 active+clean; 345 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s 2024-09-14T04:51:47.846 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:47.882 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:47.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:47.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:47.984 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:48.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:48.019 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:48.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:48.120 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:48.156 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:51:48.164 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:48.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:48.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:48.266 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:48.303 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:48.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:48.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:48.406 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:48.445 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:48.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:48.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:48.547 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:48.579 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:48.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:48.582 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:48.682 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:48.720 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:48.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:48.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:48.789 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:51:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:51:48] "GET /metrics HTTP/1.1" 200 36740 "" "Prometheus/2.51.0" 2024-09-14T04:51:48.822 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:48.855 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:48.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:48.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:48.893 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-14T04:51:48.956 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:49.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:49.029 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:49.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:49.130 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:49.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:49.187 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:49.187 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:49.288 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:49.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:49.332 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:49.332 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:49.433 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:49.471 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:49.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:49.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:49.573 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:49.603 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:49.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:49.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:49.699 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:49 smithi039 ceph-mon[106470]: pgmap v2697: 99 pgs: 99 active+clean; 345 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2024-09-14T04:51:49.699 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:49 smithi039 ceph-mon[111235]: pgmap v2697: 99 pgs: 99 active+clean; 345 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2024-09-14T04:51:49.705 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:49.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:49.744 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:49.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:49.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:49 smithi129 ceph-mon[101316]: pgmap v2697: 99 pgs: 99 active+clean; 345 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 629 op/s 2024-09-14T04:51:49.806 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Removed 123930 objects 2024-09-14T04:51:49.806 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Clean up completed and total clean up time :138.236 2024-09-14T04:51:49.845 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:49.868 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:49.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:49.869 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:49.970 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:49.991 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:49.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:49.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:50.093 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:50.115 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:50.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:50.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:50.217 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:50.240 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:50.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:50.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:50.342 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:50.365 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:50.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:50.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:50.466 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:50.492 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:50.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:50.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:50.594 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:50.619 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:50.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:50.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:50.633 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:51:50.633 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:51:50.650 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:51:50.650 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:51:50.666 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:51:50.666 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:51:50.683 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:51:50.683 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:51:50.699 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:51:50.700 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:51:50.716 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:51:50.717 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:51:50.721 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:50.733 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:51:50.734 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:51:50.746 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:50.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:50.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:50.751 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:51:50.751 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:51:50.847 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:50.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:50.872 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:50.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:50.973 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:50.998 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:50.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:50.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:51.099 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:51.124 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:51.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:51.125 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:51.226 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:51.250 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:51.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:51.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:51.352 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:51.376 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:51.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:51.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:51.478 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:51.503 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:51.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:51.504 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:51.605 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:51.630 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:51.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:51.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:51.728 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:51 smithi039 ceph-mon[106470]: pgmap v2698: 99 pgs: 99 active+clean; 81 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-14T04:51:51.728 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:51 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3783856778' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:51:51.728 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:51 smithi039 ceph-mon[111235]: pgmap v2698: 99 pgs: 99 active+clean; 81 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-14T04:51:51.728 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:51 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3783856778' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:51:51.732 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:51.759 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:51.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:51.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:51.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:51 smithi129 ceph-mon[101316]: pgmap v2698: 99 pgs: 99 active+clean; 81 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1.00k op/s 2024-09-14T04:51:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:51 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3783856778' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:51:51.861 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:51.887 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:51.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:51.888 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:51.989 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:52.014 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:52.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:52.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:52.116 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:52.140 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:52.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:52.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:52.242 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:52.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:52.267 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:52.268 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:52.369 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:52.393 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:52.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:52.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:52.494 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:52.520 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:52.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:52.521 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:52.622 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:52.647 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:52.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:52.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:52.749 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:52.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:51:52.777 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:52.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:52.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:52.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:51:52.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:51:52.818 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2024-09-14T04:51:52.818 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2024-09-14T04:51:52.879 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:52.904 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:52.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:52.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:53.006 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:53.032 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:53.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:53.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:53.171 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:53.195 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:53.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:53.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:53.297 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:53.319 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:53.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:53.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:53.421 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:53.445 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:53.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:53.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:53.470 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:53 smithi039 ceph-mon[106470]: pgmap v2699: 99 pgs: 99 active+clean; 63 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 870 op/s 2024-09-14T04:51:53.547 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:53.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:53.571 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:53.572 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:53.673 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:53.695 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:53.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:53.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:53.731 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:53 smithi039 ceph-mon[111235]: pgmap v2699: 99 pgs: 99 active+clean; 63 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 870 op/s 2024-09-14T04:51:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:53 smithi129 ceph-mon[101316]: pgmap v2699: 99 pgs: 99 active+clean; 63 MiB data, 4.0 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 870 op/s 2024-09-14T04:51:53.797 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:53.820 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:53.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:53.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:53.922 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:53.944 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:53.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:53.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:54.007 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:51:54.010 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:51:54.047 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:54.069 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:54.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:54.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:54.171 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:54.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:54.194 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:54.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:54.211 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:51:54.295 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:54.318 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:54.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:54.320 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:54.420 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:54.446 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:54.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:54.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:54.480 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.6 to 1 (10000) 2024-09-14T04:51:54.487 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is unset 2024-09-14T04:51:54.548 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:54.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:54.571 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:54.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:54.672 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:54.699 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:54.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:54.700 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:54.738 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:54 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/561182423' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:51:54.739 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:54 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:51:54.739 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:54 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2779783435' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:51:54.739 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:54 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/561182423' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:51:54.739 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:54 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:51:54.739 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:54 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2779783435' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:51:54.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:54 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/561182423' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:51:54.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:54 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:51:54.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:54 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2779783435' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:51:54.800 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:54.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:54.824 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:54.824 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:54.925 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:54.947 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:54.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:54.948 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:55.049 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:55.071 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:55.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:55.072 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:55.173 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:55.194 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:55.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:55.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:55.295 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:55.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:55.317 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:55.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:55.418 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:55.441 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:55.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:55.442 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:55.543 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:55.566 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:55.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:55.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:55.668 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:55.690 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:55.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:55.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:55 smithi129 ceph-mon[101316]: pgmap v2700: 99 pgs: 99 active+clean; 4.1 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 847 op/s 2024-09-14T04:51:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:55 smithi129 ceph-mon[101316]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:51:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:55 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:51:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:55 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2779783435' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-14T04:51:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:55 smithi129 ceph-mon[101316]: osdmap e1463: 8 total, 8 up, 8 in 2024-09-14T04:51:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:55 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:51:55.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:55 smithi039 ceph-mon[111235]: pgmap v2700: 99 pgs: 99 active+clean; 4.1 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 847 op/s 2024-09-14T04:51:55.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:55 smithi039 ceph-mon[111235]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:51:55.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:55 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:51:55.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:55 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2779783435' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-14T04:51:55.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:55 smithi039 ceph-mon[111235]: osdmap e1463: 8 total, 8 up, 8 in 2024-09-14T04:51:55.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:55 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:51:55.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:55 smithi039 ceph-mon[106470]: pgmap v2700: 99 pgs: 99 active+clean; 4.1 MiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 847 op/s 2024-09-14T04:51:55.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:55 smithi039 ceph-mon[106470]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:51:55.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:55 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:51:55.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:55 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2779783435' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-14T04:51:55.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:55 smithi039 ceph-mon[106470]: osdmap e1463: 8 total, 8 up, 8 in 2024-09-14T04:51:55.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:55 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:51:55.792 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:55.814 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:55.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:55.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:55.915 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:55.937 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:55.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:55.938 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:56.039 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:56.062 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:56.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:56.063 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:56.163 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:56.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:56.188 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:56.188 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:56.289 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:56.312 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:56.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:56.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:56.414 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:56.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:56.437 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:56.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:56.538 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:56.560 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:56.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:56.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:56.662 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:56.682 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:56.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:56.683 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:56.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:56 smithi129 ceph-mon[101316]: osdmap e1464: 8 total, 8 up, 8 in 2024-09-14T04:51:56.784 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:56.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:56 smithi039 ceph-mon[106470]: osdmap e1464: 8 total, 8 up, 8 in 2024-09-14T04:51:56.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:56 smithi039 ceph-mon[111235]: osdmap e1464: 8 total, 8 up, 8 in 2024-09-14T04:51:56.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:56.807 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:56.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:56.908 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:56.931 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:56.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:56.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:57.033 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:57.054 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:57.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:57.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:57.156 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:57.178 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:57.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:57.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:57.280 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:57.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:57.302 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:57.302 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:57.403 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:57.426 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:57.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:57.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:57.515 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 7 1 2024-09-14T04:51:57.528 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:57.580 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:57.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:57.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:57.682 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:57.704 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:57.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:57.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:57 smithi129 ceph-mon[101316]: pgmap v2703: 99 pgs: 99 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2024-09-14T04:51:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:57 smithi129 ceph-mon[101316]: osdmap e1465: 8 total, 8 up, 8 in 2024-09-14T04:51:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:57 smithi129 ceph-mon[101316]: 55.2 starting backfill to osd.6 from (0'0,0'0] MAX to 1462'15402 2024-09-14T04:51:57.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:57 smithi039 ceph-mon[106470]: pgmap v2703: 99 pgs: 99 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2024-09-14T04:51:57.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:57 smithi039 ceph-mon[106470]: osdmap e1465: 8 total, 8 up, 8 in 2024-09-14T04:51:57.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:57 smithi039 ceph-mon[106470]: 55.2 starting backfill to osd.6 from (0'0,0'0] MAX to 1462'15402 2024-09-14T04:51:57.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:57 smithi039 ceph-mon[111235]: pgmap v2703: 99 pgs: 99 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 688 op/s 2024-09-14T04:51:57.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:57 smithi039 ceph-mon[111235]: osdmap e1465: 8 total, 8 up, 8 in 2024-09-14T04:51:57.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:57 smithi039 ceph-mon[111235]: 55.2 starting backfill to osd.6 from (0'0,0'0] MAX to 1462'15402 2024-09-14T04:51:57.806 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:57.828 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:57.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:57.829 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:57.930 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:51:57.952 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:57.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:57.953 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:51:58.053 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:51:58.054 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:51:58.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:58.081 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:58.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:51:58.182 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:58.203 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:58.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:58.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:58.305 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:58.328 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:58.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:58.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:51:58.430 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:58.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:58.453 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:58.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:58.554 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:51:58.577 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:58.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:58.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:51:58.591 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:58 smithi039 ceph-mon[106470]: osdmap e1466: 8 total, 8 up, 8 in 2024-09-14T04:51:58.592 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:58 smithi039 ceph-mon[111235]: osdmap e1466: 8 total, 8 up, 8 in 2024-09-14T04:51:58.679 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:58.700 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:58.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:58.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:58.704 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:58 smithi129 ceph-mon[101316]: osdmap e1466: 8 total, 8 up, 8 in 2024-09-14T04:51:58.803 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:58.824 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:58.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:58.825 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:58.926 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:58.950 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:58.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:58.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:59.007 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:51:59.030 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:51:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:51:58] "GET /metrics HTTP/1.1" 200 36735 "" "Prometheus/2.51.0" 2024-09-14T04:51:59.051 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:59.075 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:59.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:59.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:59.177 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:51:59.198 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:59.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:59.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:51:59.300 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:59.321 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:59.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:59.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:59.423 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:59.447 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:59.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:59.448 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:59.549 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:51:59.570 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:59.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:59.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:51:59.672 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:51:59.694 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:59 smithi129 ceph-mon[101316]: pgmap v2706: 99 pgs: 99 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 14 op/s 2024-09-14T04:51:59.694 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:51:59 smithi129 ceph-mon[101316]: osdmap e1467: 8 total, 8 up, 8 in 2024-09-14T04:51:59.696 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:59.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:59.697 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:51:59.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:59 smithi039 ceph-mon[106470]: pgmap v2706: 99 pgs: 99 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 14 op/s 2024-09-14T04:51:59.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:51:59 smithi039 ceph-mon[106470]: osdmap e1467: 8 total, 8 up, 8 in 2024-09-14T04:51:59.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:59 smithi039 ceph-mon[111235]: pgmap v2706: 99 pgs: 99 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 14 op/s 2024-09-14T04:51:59.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:51:59 smithi039 ceph-mon[111235]: osdmap e1467: 8 total, 8 up, 8 in 2024-09-14T04:51:59.797 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:51:59.819 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:51:59.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:59.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:51:59.921 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:51:59.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:51:59.946 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:51:59.946 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:00.047 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:00.070 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:00.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:00.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:00.102 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:52:00.102 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:52:00.171 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:00.196 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:00.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:00.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:00.298 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:00.323 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:00.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:00.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:00.425 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:00.447 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:00.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:00.448 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:00.549 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:00.551 INFO:teuthology.orchestra.run.smithi039.stderr:pool 'unique_pool_2' removed 2024-09-14T04:52:00.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:00.579 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:00.579 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:00.680 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:00.707 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:00.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:00.708 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:00 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2627040507' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:52:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:00 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:52:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:00 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:52:00.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:00 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2627040507' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:52:00.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:00 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:52:00.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:00 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:52:00.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:00 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2627040507' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:52:00.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:00 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:52:00.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:00 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:52:00.810 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:00.831 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:00.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:00.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:00.933 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:00.957 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:00.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:00.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:01.059 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:01.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:01.081 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:01.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:01.141 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:52:01.146 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:52:01.146 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:52:01.176 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:52:01.177 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:52:01.182 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:01.201 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:52:01.201 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:52:01.204 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:01.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:01.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:01.221 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:52:01.222 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:52:01.238 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:52:01.239 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:52:01.255 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:52:01.256 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:52:01.272 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:52:01.272 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:52:01.288 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:52:01.288 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:52:01.306 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:01.327 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:01.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:01.329 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:01.430 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:01.453 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:01.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:01.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:01.554 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:01.576 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:01.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:01.577 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:01.609 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-14T04:52:01.678 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:01.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:01.702 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:01.702 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:01.804 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:01.826 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:01.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:01.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:01.857 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:01 smithi039 ceph-mon[111235]: pgmap v2708: 99 pgs: 99 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:01.857 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:01 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2627040507' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:52:01.857 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:01 smithi039 ceph-mon[111235]: osdmap e1468: 8 total, 8 up, 8 in 2024-09-14T04:52:01.857 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:01 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2023925838' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:52:01.857 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:01 smithi039 ceph-mon[106470]: pgmap v2708: 99 pgs: 99 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:01.857 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:01 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2627040507' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:52:01.857 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:01 smithi039 ceph-mon[106470]: osdmap e1468: 8 total, 8 up, 8 in 2024-09-14T04:52:01.858 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:01 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2023925838' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:52:01.928 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:01.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:01.951 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:01.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:02.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:01 smithi129 ceph-mon[101316]: pgmap v2708: 99 pgs: 99 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:02.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:01 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2627040507' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:52:02.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:01 smithi129 ceph-mon[101316]: osdmap e1468: 8 total, 8 up, 8 in 2024-09-14T04:52:02.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:01 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2023925838' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:52:02.053 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:02.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:02.075 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:02.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:02.176 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:02.197 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:02.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:02.198 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:02.299 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:02.322 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:02.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:02.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:02.424 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:02.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:02.448 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:02.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:02.549 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:02.572 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:02.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:02.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:02.673 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:02.687 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-14T04:52:02.688 INFO:tasks.radosbench:Beginning radosbench... 2024-09-14T04:52:02.688 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2024-09-14T04:52:02.689 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_3 16 2024-09-14T04:52:02.695 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:02.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:02.696 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:02.797 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:02.820 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:02.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:02.821 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:02.922 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:02.946 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:02.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:02.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:03.048 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:03.070 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:03.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:03.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:03.172 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:03.195 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:03.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:03.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:03.298 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:03.321 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:03.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:03.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:03.423 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:03.448 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:03.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:03.449 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:03.550 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:03.574 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:03.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:03.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:03.676 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:03.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:03.701 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:03.702 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:03.706 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:03 smithi039 ceph-mon[106470]: pgmap v2710: 83 pgs: 83 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:03.706 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:03 smithi039 ceph-mon[111235]: pgmap v2710: 83 pgs: 83 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:03.802 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:03.825 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:03.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:03.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:03.927 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:03.951 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:03.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:03.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:04.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:03 smithi129 ceph-mon[101316]: pgmap v2710: 83 pgs: 83 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:04.054 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:04.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:04.078 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:04.079 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:04.132 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:52:04.176 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:52:04.180 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:04.206 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:04.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:04.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:04.308 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:04.330 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:04.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:04.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:04.432 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:04.457 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:04.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:04.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:04.559 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:04.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:04.585 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:04.586 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:04.686 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:04.709 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:04.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:04.710 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:04.811 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:04.837 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:04.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:04.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:04.939 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:04.962 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:04.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:04.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:05.064 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:05.089 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:05.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:05.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:05.191 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:05.214 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:05.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:05.216 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:05.317 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:05.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:05.344 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:05.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:05.446 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:05.469 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:05.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:05.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:05.570 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:05.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:05.597 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:05.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:05.698 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:05.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:05.722 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:05.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:05.731 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:52:05.823 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:05.831 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:05 smithi039 ceph-mon[106470]: pgmap v2711: 83 pgs: 83 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:05.831 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:05 smithi039 ceph-mon[111235]: pgmap v2711: 83 pgs: 83 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:05.849 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:05.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:05.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:05.953 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:05.975 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:05.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:05.976 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:06.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:05 smithi129 ceph-mon[101316]: pgmap v2711: 83 pgs: 83 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:06.077 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:06.103 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:06.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:06.104 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:06.205 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:06.228 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:06.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:06.229 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:06.330 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:06.355 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:06.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:06.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:06.457 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:06.479 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:06.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:06.481 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:06.581 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:06.605 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:06.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:06.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:06.707 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:06.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:06.730 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:06.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:06.832 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:06.858 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:06.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:06.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:06.960 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:06.983 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:06.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:06.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:07.084 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:07.110 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:07.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:07.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:07.212 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:07.237 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:07.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:07.238 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:07.339 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:07.362 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:07.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:07.363 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:07.464 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:07.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:07.487 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:07.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:07.588 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:07.610 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:07.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:07.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:07.712 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:07.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:07.735 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:07.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:07 smithi039 ceph-mon[106470]: pgmap v2712: 83 pgs: 83 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:52:07.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:07 smithi039 ceph-mon[111235]: pgmap v2712: 83 pgs: 83 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:07.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:52:07.837 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:07.861 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:07.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:07.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:07.963 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:07.990 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:07.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:07.991 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:07 smithi129 ceph-mon[101316]: pgmap v2712: 83 pgs: 83 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:52:08.092 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:08.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:08.121 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:08.121 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:08.222 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:08.244 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:08.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:08.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:08.307 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:52:08.307 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:52:08.346 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:08.368 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:08.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:08.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:08.470 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:08.493 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:08.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:08.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:08.595 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:08.603 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.7 to 1 (10000) 2024-09-14T04:52:08.603 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is unset 2024-09-14T04:52:08.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:08.623 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:08.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:08.724 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:08.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:08.750 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:08.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:08.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:52:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:52:08] "GET /metrics HTTP/1.1" 200 34323 "" "Prometheus/2.51.0" 2024-09-14T04:52:08.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:08 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3656835658' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:52:08.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:08 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:52:08.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:08 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1320847176' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:52:08.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:08 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:52:08.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:08 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3656835658' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:52:08.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:08 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:52:08.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:08 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1320847176' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:52:08.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:08 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:52:08.851 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:08.876 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:08.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:08.877 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:08.978 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:09.002 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:09.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:09.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:09.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:08 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3656835658' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:52:09.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:08 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:52:09.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:08 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1320847176' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:52:09.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:08 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:52:09.104 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:09.129 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:09.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:09.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:09.231 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:09.255 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:09.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:09.256 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:09.356 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:09.379 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:09.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:09.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:09.480 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:09.503 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:09.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:09.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:09.605 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:09.627 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:09.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:09.628 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:09.729 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:09.751 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:09.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:09.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:09 smithi039 ceph-mon[106470]: pgmap v2713: 83 pgs: 83 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:09 smithi039 ceph-mon[106470]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:52:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:09 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T04:52:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:09 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-14T04:52:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:09 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:52:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:09 smithi039 ceph-mon[106470]: osdmap e1469: 8 total, 8 up, 8 in 2024-09-14T04:52:09.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:09 smithi039 ceph-mon[111235]: pgmap v2713: 83 pgs: 83 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:09.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:09 smithi039 ceph-mon[111235]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:52:09.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:09 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T04:52:09.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:09 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-14T04:52:09.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:09 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:52:09.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:09 smithi039 ceph-mon[111235]: osdmap e1469: 8 total, 8 up, 8 in 2024-09-14T04:52:09.854 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:09.880 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:09.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:09.881 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:09.967 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:09 smithi129 ceph-mon[101316]: pgmap v2713: 83 pgs: 83 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:09.968 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:09 smithi129 ceph-mon[101316]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:52:09.968 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:09 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T04:52:09.968 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:09 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-14T04:52:09.968 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:09 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:52:09.968 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:09 smithi129 ceph-mon[101316]: osdmap e1469: 8 total, 8 up, 8 in 2024-09-14T04:52:09.983 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:10.005 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:10.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:10.006 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:10.107 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:10.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:10.131 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:10.132 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:10.232 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:10.254 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:10.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:10.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:10.356 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:10.378 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:10.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:10.378 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:10.479 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:10.502 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:10.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:10.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:10.604 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:10.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:10.628 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:10.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:10.729 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:10.753 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:10.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:10.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:10.855 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:10.879 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:10.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:10.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:10.981 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:11.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:11.006 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:11.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:11.107 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:11.133 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:11.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:11.134 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:11.234 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:11.260 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:11.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:11.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:11.362 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:11.386 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:11.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:11.387 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:11.488 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:11.511 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:11.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:11.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:11.613 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:11.638 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:11.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:11.639 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:11.739 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:11.755 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:52:11.756 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [], 2024-09-14T04:52:11.756 INFO:teuthology.orchestra.run.smithi039.stdout: "complaint_time": 30, 2024-09-14T04:52:11.756 INFO:teuthology.orchestra.run.smithi039.stdout: "num_blocked_ops": 0 2024-09-14T04:52:11.756 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:52:11.763 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:11.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:11.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:11.864 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:11.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:11.888 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:11.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:11.989 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:12.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:12.012 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:12.013 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:12.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:11 smithi129 ceph-mon[101316]: pgmap v2715: 83 pgs: 83 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:12.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:11 smithi039 ceph-mon[106470]: pgmap v2715: 83 pgs: 83 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:12.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:11 smithi039 ceph-mon[111235]: pgmap v2715: 83 pgs: 83 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:12.114 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:12.140 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:12.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:12.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:12.242 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:12.267 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:12.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:12.269 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:12.369 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:12.394 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:12.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:12.395 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:12.494 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-14T04:52:12.494 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:52:12.496 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:12.521 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:12.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:12.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:12.624 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:12.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:12.647 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:12.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:12.748 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:12.774 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:12.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:12.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:12.876 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:12.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:12.900 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:12.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:13.001 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:13.027 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:13.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:13.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:13.129 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:13.151 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:13.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:13.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:13.211 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-14T04:52:13.254 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:13.280 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:13.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:13.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:13.382 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:13.404 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:13.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:13.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:13.506 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:13.528 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:13.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:13.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:13.630 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:13.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:13.653 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:13.653 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:13.670 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:52:13.754 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:13.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:13.781 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:13.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:13.861 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:13 smithi039 ceph-mon[106470]: pgmap v2716: 83 pgs: 83 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:13.861 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:13 smithi039 ceph-mon[111235]: pgmap v2716: 83 pgs: 83 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:13.883 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:13.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:13.910 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:13.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:14.011 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:14.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:13 smithi129 ceph-mon[101316]: pgmap v2716: 83 pgs: 83 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:14.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:14.038 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:14.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:14.139 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:14.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:14.165 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:14.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:14.266 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:14.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:14.292 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:14.292 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:14.393 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:14.419 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:14.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:14.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:14.521 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:14.547 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:14.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:14.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:14.649 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:14.676 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:14.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:14.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:14.778 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:14.804 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:14.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:14.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:14.852 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:52:14.852 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:52:14.868 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:52:14.868 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:52:14.885 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:52:14.886 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:52:14.902 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:52:14.902 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:52:14.906 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:14.919 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:52:14.919 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:52:14.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:14.930 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:14.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:14.937 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:52:14.937 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:52:14.954 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:52:14.955 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:52:14.971 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:52:14.971 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:52:15.030 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:15.052 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:15.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:15.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:15.154 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:15.176 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:15.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:15.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:15.267 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:52:15.278 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:15.303 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:15.304 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:52:15.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:15.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:15.406 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:15.431 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:15.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:15.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:15.533 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:15.557 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:15.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:15.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:15.633 INFO:teuthology.orchestra.run.smithi039.stderr:pool 'unique_pool_3' created 2024-09-14T04:52:15.659 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:15.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:15.683 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:15.684 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:15.785 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:15.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:15.810 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:15.810 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:15.911 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:15.934 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:15.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:15.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:15.987 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:15 smithi039 ceph-mon[106470]: pgmap v2717: 83 pgs: 83 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:15.987 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:15 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3532724534' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:52:15.987 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:15 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/674842255' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-14T04:52:15.987 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:15 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-14T04:52:15.987 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:15 smithi039 ceph-mon[111235]: pgmap v2717: 83 pgs: 83 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:15.987 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:15 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3532724534' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:52:15.988 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:15 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/674842255' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-14T04:52:15.988 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:15 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-14T04:52:16.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:15 smithi129 ceph-mon[101316]: pgmap v2717: 83 pgs: 83 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:16.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:15 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3532724534' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:52:16.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:15 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/674842255' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-14T04:52:16.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:15 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16} : dispatch 2024-09-14T04:52:16.036 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:16.062 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:16.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:16.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:16.164 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:16.190 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:16.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:16.191 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:16.292 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:16.317 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:16.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:16.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:16.419 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:16.441 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:16.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:16.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:16.543 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:16.569 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:16.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:16.570 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:16.671 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:16.693 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:16.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:16.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:16.795 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:16.818 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:16.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:16.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:16.920 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:16.942 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:16.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:16.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:17.020 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:16 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-09-14T04:52:17.021 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:16 smithi039 ceph-mon[106470]: osdmap e1470: 8 total, 8 up, 8 in 2024-09-14T04:52:17.021 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:16 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-09-14T04:52:17.021 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:16 smithi039 ceph-mon[111235]: osdmap e1470: 8 total, 8 up, 8 in 2024-09-14T04:52:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:16 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2024-09-14T04:52:17.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:16 smithi129 ceph-mon[101316]: osdmap e1470: 8 total, 8 up, 8 in 2024-09-14T04:52:17.045 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:17.069 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:17.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:17.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:17.079 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2024-09-14T04:52:17.171 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:17.195 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:17.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:17.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:17.297 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:17.320 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:17.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:17.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:17.421 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:17.445 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:17.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:17.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:17.547 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:17.569 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:17.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:17.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:17.671 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:17.693 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:17.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:17.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:17.795 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:17.818 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:17.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:17.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:17.920 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:17.942 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:17.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:17.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:17.965 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:17 smithi039 ceph-mon[106470]: pgmap v2719: 99 pgs: 16 unknown, 83 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:17.965 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:17 smithi039 ceph-mon[106470]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:52:17.965 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:17 smithi039 ceph-mon[106470]: osdmap e1471: 8 total, 8 up, 8 in 2024-09-14T04:52:17.966 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:17 smithi039 ceph-mon[111235]: pgmap v2719: 99 pgs: 16 unknown, 83 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:17.966 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:17 smithi039 ceph-mon[111235]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:52:17.966 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:17 smithi039 ceph-mon[111235]: osdmap e1471: 8 total, 8 up, 8 in 2024-09-14T04:52:18.012 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:52:18.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:17 smithi129 ceph-mon[101316]: pgmap v2719: 99 pgs: 16 unknown, 83 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:18.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:17 smithi129 ceph-mon[101316]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:52:18.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:17 smithi129 ceph-mon[101316]: osdmap e1471: 8 total, 8 up, 8 in 2024-09-14T04:52:18.044 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:18.068 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:18.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:18.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:18.170 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:18.193 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:18.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:18.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:18.294 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:18.319 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:18.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:18.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:18.421 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:18.444 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:18.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:18.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:18.546 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:18.569 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:18.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:18.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:18.670 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:18.696 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:18.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:18.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:18.798 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:18.821 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:18.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:18.822 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:18.923 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:18.936 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:18 smithi039 ceph-mon[106470]: osdmap e1472: 8 total, 8 up, 8 in 2024-09-14T04:52:18.936 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:18 smithi039 ceph-mon[106470]: pgmap v2722: 99 pgs: 16 unknown, 83 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:18.937 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:52:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:52:18] "GET /metrics HTTP/1.1" 200 34323 "" "Prometheus/2.51.0" 2024-09-14T04:52:18.937 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:18 smithi039 ceph-mon[111235]: osdmap e1472: 8 total, 8 up, 8 in 2024-09-14T04:52:18.937 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:18 smithi039 ceph-mon[111235]: pgmap v2722: 99 pgs: 16 unknown, 83 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:18.946 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:18.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:18.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:19.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:18 smithi129 ceph-mon[101316]: osdmap e1472: 8 total, 8 up, 8 in 2024-09-14T04:52:19.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:18 smithi129 ceph-mon[101316]: pgmap v2722: 99 pgs: 16 unknown, 83 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:19.048 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:19.074 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:19.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:19.075 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:19.176 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:19.200 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:19.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:19.201 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:19.301 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:19.327 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:19.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:19.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:19.429 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:19.453 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:19.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:19.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:19.535 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:52:19.535 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:52:19.555 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:19.577 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:19.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:19.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:19.679 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:19.701 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:19.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:19.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:19.803 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:19.825 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:19.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:19.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:19.927 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:19.953 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:19.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:19.954 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:20.054 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:20.078 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:20.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:20.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:20.180 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:20.202 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:20.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:20.203 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:20.304 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:20.327 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:20.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:20.328 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:20.429 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:20.455 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:20.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:20.456 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:20.557 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:20.583 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:20.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:20.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:20.685 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:20.710 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:20.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:20.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:20.812 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:20.834 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:52:20.839 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:20.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:20.840 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:20.941 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:20.967 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:20.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:20.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:21.070 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:21.093 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:21.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:21.095 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:21.196 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:21.197 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:20 smithi039 ceph-mon[106470]: pgmap v2723: 99 pgs: 99 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:21.197 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:20 smithi039 ceph-mon[111235]: pgmap v2723: 99 pgs: 99 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:21.221 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:21.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:21.223 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:21.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:20 smithi129 ceph-mon[101316]: pgmap v2723: 99 pgs: 99 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:21.323 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:21.349 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:21.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:21.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:21.451 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:21.476 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:21.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:21.477 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:21.578 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:21.602 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:21.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:21.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:21.703 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:21.729 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:21.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:21.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:21.781 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-14T04:52:21.831 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:21.857 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:21.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:21.859 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:21.959 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:21.984 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:21.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:21.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:22.055 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:52:22.086 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:22.111 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:22.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:22.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:22.212 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:22.235 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:22.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:22.236 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:22.336 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:22.361 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:22.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:22.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:22.466 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:52:22.467 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T04:52:22.469 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:22.494 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:22.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:22.495 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:22.596 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:22.621 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:22.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:22.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:22.723 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:22.747 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:22.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:22.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:22.849 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:22.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:22.877 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:22.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:22.978 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:23.002 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:23.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:23.003 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:23.104 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:23.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:23.131 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:23.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:23.232 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:23.257 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:23.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:23.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:23.358 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:23.383 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:23.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:23.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:23.485 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:23.508 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:23.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:23.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:23.596 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:52:23.610 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:23.633 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:23.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:23.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:23.700 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:23 smithi039 ceph-mon[106470]: pgmap v2724: 99 pgs: 99 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:23.700 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:23 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:52:23.700 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:52:23.700 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:23 smithi039 ceph-mon[106470]: from='client.35985 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:52:23.700 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:23 smithi039 ceph-mon[111235]: pgmap v2724: 99 pgs: 99 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:23.700 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:23 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:52:23.700 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:52:23.700 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:23 smithi039 ceph-mon[111235]: from='client.35985 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:52:23.736 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:23.758 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:23.781 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:23 smithi129 ceph-mon[101316]: pgmap v2724: 99 pgs: 99 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:23.782 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:23 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:52:23.782 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:52:23.782 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:23 smithi129 ceph-mon[101316]: from='client.35985 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:52:23.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:23.782 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:23.883 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:23.910 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:23.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:23.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:24.011 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:24.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:24.036 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:24.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:24.137 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:24.163 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:24.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:24.164 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:24.265 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:24.289 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:24.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:24.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:24.391 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:24.417 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:24.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:24.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:24.519 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:24.544 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:24.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:24.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:24.646 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:24.670 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:24.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:24.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:24.771 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:24.798 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:24.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:24.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:24.900 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:24.926 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:24.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:24.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:25.028 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:25.051 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:25.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:25.052 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:25.153 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:25.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:25.176 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:25.176 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:25.277 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:25.300 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:25.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:25.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:25.402 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:25.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:25.427 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:25.427 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:25.528 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:25.551 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:25.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:25.552 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:25.653 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:25.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:25.676 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:25.676 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:25.777 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:25 smithi039 ceph-mon[106470]: pgmap v2725: 99 pgs: 99 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:25 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:52:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:25 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:52:25.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:25 smithi039 ceph-mon[111235]: pgmap v2725: 99 pgs: 99 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:25.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:25 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:52:25.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:25 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:52:25.799 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:25.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:25.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:25.901 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:25.923 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:25.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:25.924 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:25 smithi129 ceph-mon[101316]: pgmap v2725: 99 pgs: 99 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:25 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:52:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:25 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:52:26.024 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:26.047 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:26.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:26.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:26.149 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:26.173 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:26.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:26.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:26.276 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:26.299 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:26.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:26.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:26.401 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:26.406 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":2724,"stamp":"2024-09-14T04:52:21.808100+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":793134,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1698,"ondisk_log_size":1698,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":276,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":3991212,"kb_used_data":92016,"kb_used_omap":404,"kb_used_meta":3898667,"kb_avail":745970004,"statfs":{"total":767960285184,"available":763873284096,"internally_reserved":0,"allocated":94224384,"data_stored":54929500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":414022,"internal_metadata":3992235706},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.177586"},"pg_stats":[{"pgid":"56.f","version":"0'0","reported_seq":11,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.660874+0000","last_change":"2024-09-14T04:52:16.661278+0000","last_active":"2024-09-14T04:52:17.660874+0000","last_peered":"2024-09-14T04:52:17.660874+0000","last_clean":"2024-09-14T04:52:17.660874+0000","last_became_active":"2024-09-14T04:52:16.660906+0000","last_became_peered":"2024-09-14T04:52:16.660906+0000","last_unstale":"2024-09-14T04:52:17.660874+0000","last_undegraded":"2024-09-14T04:52:17.660874+0000","last_fullsized":"2024-09-14T04:52:17.660874+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:55:16.908898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"56.e","version":"0'0","reported_seq":11,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.960516+0000","last_change":"2024-09-14T04:52:16.723745+0000","last_active":"2024-09-14T04:52:17.960516+0000","last_peered":"2024-09-14T04:52:17.960516+0000","last_clean":"2024-09-14T04:52:17.960516+0000","last_became_active":"2024-09-14T04:52:16.723416+0000","last_became_peered":"2024-09-14T04:52:16.723416+0000","last_unstale":"2024-09-14T04:52:17.960516+0000","last_undegraded":"2024-09-14T04:52:17.960516+0000","last_fullsized":"2024-09-14T04:52:17.960516+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:20:17.253535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"56.d","version":"0'0","reported_seq":14,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.993216+0000","last_change":"2024-09-14T04:52:16.871824+0000","last_active":"2024-09-14T04:52:17.993216+0000","last_peered":"2024-09-14T04:52:17.993216+0000","last_clean":"2024-09-14T04:52:17.993216+0000","last_became_active":"2024-09-14T04:52:16.871446+0000","last_became_peered":"2024-09-14T04:52:16.871446+0000","last_unstale":"2024-09-14T04:52:17.993216+0000","last_undegraded":"2024-09-14T04:52:17.993216+0000","last_fullsized":"2024-09-14T04:52:17.993216+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:03:11.522195+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"56.c","version":"0'0","reported_seq":11,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.662388+0000","last_change":"2024-09-14T04:52:16.723896+0000","last_active":"2024-09-14T04:52:17.662388+0000","last_peered":"2024-09-14T04:52:17.662388+0000","last_clean":"2024-09-14T04:52:17.662388+0000","last_became_active":"2024-09-14T04:52:16.723597+0000","last_became_peered":"2024-09-14T04:52:16.723597+0000","last_unstale":"2024-09-14T04:52:17.662388+0000","last_undegraded":"2024-09-14T04:52:17.662388+0000","last_fullsized":"2024-09-14T04:52:17.662388+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:10:31.058435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.b","version":"0'0","reported_seq":11,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.660450+0000","last_change":"2024-09-14T04:52:16.660625+0000","last_active":"2024-09-14T04:52:17.660450+0000","last_peered":"2024-09-14T04:52:17.660450+0000","last_clean":"2024-09-14T04:52:17.660450+0000","last_became_active":"2024-09-14T04:52:16.660277+0000","last_became_peered":"2024-09-14T04:52:16.660277+0000","last_unstale":"2024-09-14T04:52:17.660450+0000","last_undegraded":"2024-09-14T04:52:17.660450+0000","last_fullsized":"2024-09-14T04:52:17.660450+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:43:12.659725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"56.a","version":"0'0","reported_seq":11,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.660372+0000","last_change":"2024-09-14T04:52:16.915139+0000","last_active":"2024-09-14T04:52:17.660372+0000","last_peered":"2024-09-14T04:52:17.660372+0000","last_clean":"2024-09-14T04:52:17.660372+0000","last_became_active":"2024-09-14T04:52:16.914776+0000","last_became_peered":"2024-09-14T04:52:16.914776+0000","last_unstale":"2024-09-14T04:52:17.660372+0000","last_undegraded":"2024-09-14T04:52:17.660372+0000","last_fullsized":"2024-09-14T04:52:17.660372+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:08:58.079681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"56.9","version":"0'0","reported_seq":11,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.662078+0000","last_change":"2024-09-14T04:52:16.723789+0000","last_active":"2024-09-14T04:52:17.662078+0000","last_peered":"2024-09-14T04:52:17.662078+0000","last_clean":"2024-09-14T04:52:17.662078+0000","last_became_active":"2024-09-14T04:52:16.723394+0000","last_became_peered":"2024-09-14T04:52:16.723394+0000","last_unstale":"2024-09-14T04:52:17.662078+0000","last_undegraded":"2024-09-14T04:52:17.662078+0000","last_fullsized":"2024-09-14T04:52:17.662078+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:45:07.892366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.8","version":"0'0","reported_seq":14,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.993026+0000","last_change":"2024-09-14T04:52:16.915103+0000","last_active":"2024-09-14T04:52:17.993026+0000","last_peered":"2024-09-14T04:52:17.993026+0000","last_clean":"2024-09-14T04:52:17.993026+0000","last_became_active":"2024-09-14T04:52:16.914657+0000","last_became_peered":"2024-09-14T04:52:16.914657+0000","last_unstale":"2024-09-14T04:52:17.993026+0000","last_undegraded":"2024-09-14T04:52:17.993026+0000","last_fullsized":"2024-09-14T04:52:17.993026+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:53:00.220575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"56.7","version":"0'0","reported_seq":14,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.993630+0000","last_change":"2024-09-14T04:52:16.723670+0000","last_active":"2024-09-14T04:52:17.993630+0000","last_peered":"2024-09-14T04:52:17.993630+0000","last_clean":"2024-09-14T04:52:17.993630+0000","last_became_active":"2024-09-14T04:52:16.723292+0000","last_became_peered":"2024-09-14T04:52:16.723292+0000","last_unstale":"2024-09-14T04:52:17.993630+0000","last_undegraded":"2024-09-14T04:52:17.993630+0000","last_fullsized":"2024-09-14T04:52:17.993630+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:22:10.784637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"56.6","version":"0'0","reported_seq":11,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.662519+0000","last_change":"2024-09-14T04:52:16.872248+0000","last_active":"2024-09-14T04:52:17.662519+0000","last_peered":"2024-09-14T04:52:17.662519+0000","last_clean":"2024-09-14T04:52:17.662519+0000","last_became_active":"2024-09-14T04:52:16.871513+0000","last_became_peered":"2024-09-14T04:52:16.871513+0000","last_unstale":"2024-09-14T04:52:17.662519+0000","last_undegraded":"2024-09-14T04:52:17.662519+0000","last_fullsized":"2024-09-14T04:52:17.662519+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:45:00.266398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.5","version":"0'0","reported_seq":11,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.960454+0000","last_change":"2024-09-14T04:52:16.871791+0000","last_active":"2024-09-14T04:52:17.960454+0000","last_peered":"2024-09-14T04:52:17.960454+0000","last_clean":"2024-09-14T04:52:17.960454+0000","last_became_active":"2024-09-14T04:52:16.871407+0000","last_became_peered":"2024-09-14T04:52:16.871407+0000","last_unstale":"2024-09-14T04:52:17.960454+0000","last_undegraded":"2024-09-14T04:52:17.960454+0000","last_fullsized":"2024-09-14T04:52:17.960454+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:47:15.419674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"56.4","version":"0'0","reported_seq":11,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.660737+0000","last_change":"2024-09-14T04:52:16.724077+0000","last_active":"2024-09-14T04:52:17.660737+0000","last_peered":"2024-09-14T04:52:17.660737+0000","last_clean":"2024-09-14T04:52:17.660737+0000","last_became_active":"2024-09-14T04:52:16.723597+0000","last_became_peered":"2024-09-14T04:52:16.723597+0000","last_unstale":"2024-09-14T04:52:17.660737+0000","last_undegraded":"2024-09-14T04:52:17.660737+0000","last_fullsized":"2024-09-14T04:52:17.660737+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:23:49.557180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"56.3","version":"0'0","reported_seq":11,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.960271+0000","last_change":"2024-09-14T04:52:16.915388+0000","last_active":"2024-09-14T04:52:17.960271+0000","last_peered":"2024-09-14T04:52:17.960271+0000","last_clean":"2024-09-14T04:52:17.960271+0000","last_became_active":"2024-09-14T04:52:16.915082+0000","last_became_peered":"2024-09-14T04:52:16.915082+0000","last_unstale":"2024-09-14T04:52:17.960271+0000","last_undegraded":"2024-09-14T04:52:17.960271+0000","last_fullsized":"2024-09-14T04:52:17.960271+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:52:13.211784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"56.2","version":"0'0","reported_seq":11,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.662173+0000","last_change":"2024-09-14T04:52:16.871874+0000","last_active":"2024-09-14T04:52:17.662173+0000","last_peered":"2024-09-14T04:52:17.662173+0000","last_clean":"2024-09-14T04:52:17.662173+0000","last_became_active":"2024-09-14T04:52:16.871509+0000","last_became_peered":"2024-09-14T04:52:16.871509+0000","last_unstale":"2024-09-14T04:52:17.662173+0000","last_undegraded":"2024-09-14T04:52:17.662173+0000","last_fullsized":"2024-09-14T04:52:17.662173+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:04:09.707950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.1","version":"0'0","reported_seq":11,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.662069+0000","last_change":"2024-09-14T04:52:16.872703+0000","last_active":"2024-09-14T04:52:17.662069+0000","last_peered":"2024-09-14T04:52:17.662069+0000","last_clean":"2024-09-14T04:52:17.662069+0000","last_became_active":"2024-09-14T04:52:16.871779+0000","last_became_peered":"2024-09-14T04:52:16.871779+0000","last_unstale":"2024-09-14T04:52:17.662069+0000","last_undegraded":"2024-09-14T04:52:17.662069+0000","last_fullsized":"2024-09-14T04:52:17.662069+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:23:30.260775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.0","version":"0'0","reported_seq":11,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.661974+0000","last_change":"2024-09-14T04:52:16.915325+0000","last_active":"2024-09-14T04:52:17.661974+0000","last_peered":"2024-09-14T04:52:17.661974+0000","last_clean":"2024-09-14T04:52:17.661974+0000","last_became_active":"2024-09-14T04:52:16.914980+0000","last_became_peered":"2024-09-14T04:52:16.914980+0000","last_unstale":"2024-09-14T04:52:17.661974+0000","last_undegraded":"2024-09-14T04:52:17.661974+0000","last_fullsized":"2024-09-14T04:52:17.661974+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:55:08.752960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":2235,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.992836+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T04:52:17.992836+0000","last_peered":"2024-09-14T04:52:17.992836+0000","last_clean":"2024-09-14T04:52:17.992836+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T04:52:17.992836+0000","last_undegraded":"2024-09-14T04:52:17.992836+0000","last_fullsized":"2024-09-14T04:52:17.992836+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":83,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.656599+0000","last_change":"2024-09-14T04:44:57.325109+0000","last_active":"2024-09-14T04:52:16.656599+0000","last_peered":"2024-09-14T04:52:16.656599+0000","last_clean":"2024-09-14T04:52:16.656599+0000","last_became_active":"2024-09-14T04:44:57.324791+0000","last_became_peered":"2024-09-14T04:44:57.324791+0000","last_unstale":"2024-09-14T04:52:16.656599+0000","last_undegraded":"2024-09-14T04:52:16.656599+0000","last_fullsized":"2024-09-14T04:52:16.656599+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:03.593275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":52,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.914348+0000","last_change":"2024-09-14T04:49:29.628435+0000","last_active":"2024-09-14T04:52:16.914348+0000","last_peered":"2024-09-14T04:52:16.914348+0000","last_clean":"2024-09-14T04:52:16.914348+0000","last_became_active":"2024-09-14T04:49:29.628081+0000","last_became_peered":"2024-09-14T04:49:29.628081+0000","last_unstale":"2024-09-14T04:52:16.914348+0000","last_undegraded":"2024-09-14T04:52:16.914348+0000","last_fullsized":"2024-09-14T04:52:16.914348+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:01.520334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":856,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:18.482770+0000","last_change":"2024-09-14T04:43:13.355281+0000","last_active":"2024-09-14T04:52:18.482770+0000","last_peered":"2024-09-14T04:52:18.482770+0000","last_clean":"2024-09-14T04:52:18.482770+0000","last_became_active":"2024-09-14T04:43:13.354967+0000","last_became_peered":"2024-09-14T04:43:13.354967+0000","last_unstale":"2024-09-14T04:52:18.482770+0000","last_undegraded":"2024-09-14T04:52:18.482770+0000","last_fullsized":"2024-09-14T04:52:18.482770+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:23:19.332075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1461,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.654227+0000","last_change":"2024-09-14T04:44:57.342918+0000","last_active":"2024-09-14T04:52:16.654227+0000","last_peered":"2024-09-14T04:52:16.654227+0000","last_clean":"2024-09-14T04:52:16.654227+0000","last_became_active":"2024-09-14T04:44:57.329550+0000","last_became_peered":"2024-09-14T04:44:57.329550+0000","last_unstale":"2024-09-14T04:52:16.654227+0000","last_undegraded":"2024-09-14T04:52:16.654227+0000","last_fullsized":"2024-09-14T04:52:16.654227+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:52:52.435893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":82,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.656876+0000","last_change":"2024-09-14T04:44:57.325537+0000","last_active":"2024-09-14T04:52:16.656876+0000","last_peered":"2024-09-14T04:52:16.656876+0000","last_clean":"2024-09-14T04:52:16.656876+0000","last_became_active":"2024-09-14T04:44:57.325057+0000","last_became_peered":"2024-09-14T04:44:57.325057+0000","last_unstale":"2024-09-14T04:52:16.656876+0000","last_undegraded":"2024-09-14T04:52:16.656876+0000","last_fullsized":"2024-09-14T04:52:16.656876+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:43:23.166708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1476,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.654377+0000","last_change":"2024-09-14T04:44:57.337033+0000","last_active":"2024-09-14T04:52:16.654377+0000","last_peered":"2024-09-14T04:52:16.654377+0000","last_clean":"2024-09-14T04:52:16.654377+0000","last_became_active":"2024-09-14T04:44:57.328437+0000","last_became_peered":"2024-09-14T04:44:57.328437+0000","last_unstale":"2024-09-14T04:52:16.654377+0000","last_undegraded":"2024-09-14T04:52:16.654377+0000","last_fullsized":"2024-09-14T04:52:16.654377+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.109086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":82,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.657504+0000","last_change":"2024-09-14T04:44:57.332243+0000","last_active":"2024-09-14T04:52:16.657504+0000","last_peered":"2024-09-14T04:52:16.657504+0000","last_clean":"2024-09-14T04:52:16.657504+0000","last_became_active":"2024-09-14T04:44:57.330413+0000","last_became_peered":"2024-09-14T04:44:57.330413+0000","last_unstale":"2024-09-14T04:52:16.657504+0000","last_undegraded":"2024-09-14T04:52:16.657504+0000","last_fullsized":"2024-09-14T04:52:16.657504+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:25:01.338549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":91,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.993181+0000","last_change":"2024-09-14T04:48:29.862929+0000","last_active":"2024-09-14T04:52:17.993181+0000","last_peered":"2024-09-14T04:52:17.993181+0000","last_clean":"2024-09-14T04:52:17.993181+0000","last_became_active":"2024-09-14T04:48:29.862581+0000","last_became_peered":"2024-09-14T04:48:29.862581+0000","last_unstale":"2024-09-14T04:52:17.993181+0000","last_undegraded":"2024-09-14T04:52:17.993181+0000","last_fullsized":"2024-09-14T04:52:17.993181+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:16:43.060088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2230,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:18.482839+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T04:52:18.482839+0000","last_peered":"2024-09-14T04:52:18.482839+0000","last_clean":"2024-09-14T04:52:18.482839+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T04:52:18.482839+0000","last_undegraded":"2024-09-14T04:52:18.482839+0000","last_fullsized":"2024-09-14T04:52:18.482839+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":82,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.656767+0000","last_change":"2024-09-14T04:44:57.325392+0000","last_active":"2024-09-14T04:52:16.656767+0000","last_peered":"2024-09-14T04:52:16.656767+0000","last_clean":"2024-09-14T04:52:16.656767+0000","last_became_active":"2024-09-14T04:44:57.324624+0000","last_became_peered":"2024-09-14T04:44:57.324624+0000","last_unstale":"2024-09-14T04:52:16.656767+0000","last_undegraded":"2024-09-14T04:52:16.656767+0000","last_fullsized":"2024-09-14T04:52:16.656767+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:33:18.810199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2199,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:18.010307+0000","last_change":"2024-09-14T04:44:57.329113+0000","last_active":"2024-09-14T04:52:18.010307+0000","last_peered":"2024-09-14T04:52:18.010307+0000","last_clean":"2024-09-14T04:52:18.010307+0000","last_became_active":"2024-09-14T04:44:57.328727+0000","last_became_peered":"2024-09-14T04:44:57.328727+0000","last_unstale":"2024-09-14T04:52:18.010307+0000","last_undegraded":"2024-09-14T04:52:18.010307+0000","last_fullsized":"2024-09-14T04:52:18.010307+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:34.939874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":2216,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:18.009970+0000","last_change":"2024-09-14T04:44:57.329277+0000","last_active":"2024-09-14T04:52:18.009970+0000","last_peered":"2024-09-14T04:52:18.009970+0000","last_clean":"2024-09-14T04:52:18.009970+0000","last_became_active":"2024-09-14T04:44:57.328804+0000","last_became_peered":"2024-09-14T04:44:57.328804+0000","last_unstale":"2024-09-14T04:52:18.009970+0000","last_undegraded":"2024-09-14T04:52:18.009970+0000","last_fullsized":"2024-09-14T04:52:18.009970+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:42:46.055240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2230,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.993461+0000","last_change":"2024-09-14T04:07:58.561490+0000","last_active":"2024-09-14T04:52:17.993461+0000","last_peered":"2024-09-14T04:52:17.993461+0000","last_clean":"2024-09-14T04:52:17.993461+0000","last_became_active":"2024-09-14T04:07:58.559841+0000","last_became_peered":"2024-09-14T04:07:58.559841+0000","last_unstale":"2024-09-14T04:52:17.993461+0000","last_undegraded":"2024-09-14T04:52:17.993461+0000","last_fullsized":"2024-09-14T04:52:17.993461+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:27:03.795368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2212,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.993009+0000","last_change":"2024-09-14T04:43:13.357092+0000","last_active":"2024-09-14T04:52:17.993009+0000","last_peered":"2024-09-14T04:52:17.993009+0000","last_clean":"2024-09-14T04:52:17.993009+0000","last_became_active":"2024-09-14T04:43:13.356694+0000","last_became_peered":"2024-09-14T04:43:13.356694+0000","last_unstale":"2024-09-14T04:52:17.993009+0000","last_undegraded":"2024-09-14T04:52:17.993009+0000","last_fullsized":"2024-09-14T04:52:17.993009+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:11.167574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1454,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.914318+0000","last_change":"2024-09-14T04:07:21.720797+0000","last_active":"2024-09-14T04:52:16.914318+0000","last_peered":"2024-09-14T04:52:16.914318+0000","last_clean":"2024-09-14T04:52:16.914318+0000","last_became_active":"2024-09-14T04:07:21.719331+0000","last_became_peered":"2024-09-14T04:07:21.719331+0000","last_unstale":"2024-09-14T04:52:16.914318+0000","last_undegraded":"2024-09-14T04:52:16.914318+0000","last_fullsized":"2024-09-14T04:52:16.914318+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:35.404023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":101,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.654772+0000","last_change":"2024-09-14T04:43:11.641514+0000","last_active":"2024-09-14T04:52:16.654772+0000","last_peered":"2024-09-14T04:52:16.654772+0000","last_clean":"2024-09-14T04:52:16.654772+0000","last_became_active":"2024-09-14T04:43:11.641151+0000","last_became_peered":"2024-09-14T04:43:11.641151+0000","last_unstale":"2024-09-14T04:52:16.654772+0000","last_undegraded":"2024-09-14T04:52:16.654772+0000","last_fullsized":"2024-09-14T04:52:16.654772+0000","mapping_epoch":1384,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:06:45.045430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2224,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.993519+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T04:52:17.993519+0000","last_peered":"2024-09-14T04:52:17.993519+0000","last_clean":"2024-09-14T04:52:17.993519+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T04:52:17.993519+0000","last_undegraded":"2024-09-14T04:52:17.993519+0000","last_fullsized":"2024-09-14T04:52:17.993519+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":682,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.657641+0000","last_change":"2024-09-14T04:49:29.618477+0000","last_active":"2024-09-14T04:52:16.657641+0000","last_peered":"2024-09-14T04:52:16.657641+0000","last_clean":"2024-09-14T04:52:16.657641+0000","last_became_active":"2024-09-14T04:49:29.618210+0000","last_became_peered":"2024-09-14T04:49:29.618210+0000","last_unstale":"2024-09-14T04:52:16.657641+0000","last_undegraded":"2024-09-14T04:52:16.657641+0000","last_fullsized":"2024-09-14T04:52:16.657641+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:13:23.895512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1460,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.654394+0000","last_change":"2024-09-14T04:42:49.938505+0000","last_active":"2024-09-14T04:52:16.654394+0000","last_peered":"2024-09-14T04:52:16.654394+0000","last_clean":"2024-09-14T04:52:16.654394+0000","last_became_active":"2024-09-14T04:42:49.938180+0000","last_became_peered":"2024-09-14T04:42:49.938180+0000","last_unstale":"2024-09-14T04:52:16.654394+0000","last_undegraded":"2024-09-14T04:52:16.654394+0000","last_fullsized":"2024-09-14T04:52:16.654394+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:05.929479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1479,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.658801+0000","last_change":"2024-09-14T04:44:57.346582+0000","last_active":"2024-09-14T04:52:16.658801+0000","last_peered":"2024-09-14T04:52:16.658801+0000","last_clean":"2024-09-14T04:52:16.658801+0000","last_became_active":"2024-09-14T04:44:57.330446+0000","last_became_peered":"2024-09-14T04:44:57.330446+0000","last_unstale":"2024-09-14T04:52:16.658801+0000","last_undegraded":"2024-09-14T04:52:16.658801+0000","last_fullsized":"2024-09-14T04:52:16.658801+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:29.238585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":165,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:18.010040+0000","last_change":"2024-09-14T04:44:14.480667+0000","last_active":"2024-09-14T04:52:18.010040+0000","last_peered":"2024-09-14T04:52:18.010040+0000","last_clean":"2024-09-14T04:52:18.010040+0000","last_became_active":"2024-09-14T04:44:14.480365+0000","last_became_peered":"2024-09-14T04:44:14.480365+0000","last_unstale":"2024-09-14T04:52:18.010040+0000","last_undegraded":"2024-09-14T04:52:18.010040+0000","last_fullsized":"2024-09-14T04:52:18.010040+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:00.611771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":93,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.657800+0000","last_change":"2024-09-14T04:44:57.341426+0000","last_active":"2024-09-14T04:52:16.657800+0000","last_peered":"2024-09-14T04:52:16.657800+0000","last_clean":"2024-09-14T04:52:16.657800+0000","last_became_active":"2024-09-14T04:44:57.328763+0000","last_became_peered":"2024-09-14T04:44:57.328763+0000","last_unstale":"2024-09-14T04:52:16.657800+0000","last_undegraded":"2024-09-14T04:52:16.657800+0000","last_fullsized":"2024-09-14T04:52:16.657800+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:34:40.066761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":83,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.657721+0000","last_change":"2024-09-14T04:44:57.331337+0000","last_active":"2024-09-14T04:52:16.657721+0000","last_peered":"2024-09-14T04:52:16.657721+0000","last_clean":"2024-09-14T04:52:16.657721+0000","last_became_active":"2024-09-14T04:44:57.329336+0000","last_became_peered":"2024-09-14T04:44:57.329336+0000","last_unstale":"2024-09-14T04:52:16.657721+0000","last_undegraded":"2024-09-14T04:52:16.657721+0000","last_fullsized":"2024-09-14T04:52:16.657721+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:52:35.980869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"349'149","reported_seq":671,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.654165+0000","last_change":"2024-09-14T04:49:30.085274+0000","last_active":"2024-09-14T04:52:16.654165+0000","last_peered":"2024-09-14T04:52:16.654165+0000","last_clean":"2024-09-14T04:52:16.654165+0000","last_became_active":"2024-09-14T04:49:29.623295+0000","last_became_peered":"2024-09-14T04:49:29.623295+0000","last_unstale":"2024-09-14T04:52:16.654165+0000","last_undegraded":"2024-09-14T04:52:16.654165+0000","last_fullsized":"2024-09-14T04:52:16.654165+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:49:30.085089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:49:30.085089+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:10.012963+0000","scrub_duration":0.0071928790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":83,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.656571+0000","last_change":"2024-09-14T04:44:57.329870+0000","last_active":"2024-09-14T04:52:16.656571+0000","last_peered":"2024-09-14T04:52:16.656571+0000","last_clean":"2024-09-14T04:52:16.656571+0000","last_became_active":"2024-09-14T04:44:57.329353+0000","last_became_peered":"2024-09-14T04:44:57.329353+0000","last_unstale":"2024-09-14T04:52:16.656571+0000","last_undegraded":"2024-09-14T04:52:16.656571+0000","last_fullsized":"2024-09-14T04:52:16.656571+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:46.432181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1457,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.914440+0000","last_change":"2024-09-14T04:07:21.721242+0000","last_active":"2024-09-14T04:52:16.914440+0000","last_peered":"2024-09-14T04:52:16.914440+0000","last_clean":"2024-09-14T04:52:16.914440+0000","last_became_active":"2024-09-14T04:07:21.720084+0000","last_became_peered":"2024-09-14T04:07:21.720084+0000","last_unstale":"2024-09-14T04:52:16.914440+0000","last_undegraded":"2024-09-14T04:52:16.914440+0000","last_fullsized":"2024-09-14T04:52:16.914440+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:58:41.308643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":2262,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.993387+0000","last_change":"2024-09-14T04:44:57.344614+0000","last_active":"2024-09-14T04:52:17.993387+0000","last_peered":"2024-09-14T04:52:17.993387+0000","last_clean":"2024-09-14T04:52:17.993387+0000","last_became_active":"2024-09-14T04:44:57.332256+0000","last_became_peered":"2024-09-14T04:44:57.332256+0000","last_unstale":"2024-09-14T04:52:17.993387+0000","last_undegraded":"2024-09-14T04:52:17.993387+0000","last_fullsized":"2024-09-14T04:52:17.993387+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:23.438417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"349'149","reported_seq":841,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.657218+0000","last_change":"2024-09-14T04:45:00.331854+0000","last_active":"2024-09-14T04:52:16.657218+0000","last_peered":"2024-09-14T04:52:16.657218+0000","last_clean":"2024-09-14T04:52:16.657218+0000","last_became_active":"2024-09-14T04:45:00.331460+0000","last_became_peered":"2024-09-14T04:45:00.331460+0000","last_unstale":"2024-09-14T04:52:16.657218+0000","last_undegraded":"2024-09-14T04:52:16.657218+0000","last_fullsized":"2024-09-14T04:52:16.657218+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.567535+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.567535+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:45.095729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":672,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.657266+0000","last_change":"2024-09-14T04:49:29.616831+0000","last_active":"2024-09-14T04:52:16.657266+0000","last_peered":"2024-09-14T04:52:16.657266+0000","last_clean":"2024-09-14T04:52:16.657266+0000","last_became_active":"2024-09-14T04:49:29.616546+0000","last_became_peered":"2024-09-14T04:49:29.616546+0000","last_unstale":"2024-09-14T04:52:16.657266+0000","last_undegraded":"2024-09-14T04:52:16.657266+0000","last_fullsized":"2024-09-14T04:52:16.657266+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:27.882929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":217,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:18.010449+0000","last_change":"2024-09-14T04:44:57.330746+0000","last_active":"2024-09-14T04:52:18.010449+0000","last_peered":"2024-09-14T04:52:18.010449+0000","last_clean":"2024-09-14T04:52:18.010449+0000","last_became_active":"2024-09-14T04:44:57.330327+0000","last_became_peered":"2024-09-14T04:44:57.330327+0000","last_unstale":"2024-09-14T04:52:18.010449+0000","last_undegraded":"2024-09-14T04:52:18.010449+0000","last_fullsized":"2024-09-14T04:52:18.010449+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:53.122771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":123,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.659599+0000","last_change":"2024-09-14T04:43:13.355296+0000","last_active":"2024-09-14T04:52:16.659599+0000","last_peered":"2024-09-14T04:52:16.659599+0000","last_clean":"2024-09-14T04:52:16.659599+0000","last_became_active":"2024-09-14T04:43:13.354950+0000","last_became_peered":"2024-09-14T04:43:13.354950+0000","last_unstale":"2024-09-14T04:52:16.659599+0000","last_undegraded":"2024-09-14T04:52:16.659599+0000","last_fullsized":"2024-09-14T04:52:16.659599+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:20:08.968818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"349'149","reported_seq":779,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.914216+0000","last_change":"2024-09-14T04:41:39.471393+0000","last_active":"2024-09-14T04:52:16.914216+0000","last_peered":"2024-09-14T04:52:16.914216+0000","last_clean":"2024-09-14T04:52:16.914216+0000","last_became_active":"2024-09-14T04:41:36.931768+0000","last_became_peered":"2024-09-14T04:41:36.931768+0000","last_unstale":"2024-09-14T04:52:16.914216+0000","last_undegraded":"2024-09-14T04:52:16.914216+0000","last_fullsized":"2024-09-14T04:52:16.914216+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.471248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.471248+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:56.210375+0000","scrub_duration":0.021758344999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2218,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.993380+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T04:52:17.993380+0000","last_peered":"2024-09-14T04:52:17.993380+0000","last_clean":"2024-09-14T04:52:17.993380+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T04:52:17.993380+0000","last_undegraded":"2024-09-14T04:52:17.993380+0000","last_fullsized":"2024-09-14T04:52:17.993380+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1489,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.659087+0000","last_change":"2024-09-14T04:44:57.331436+0000","last_active":"2024-09-14T04:52:16.659087+0000","last_peered":"2024-09-14T04:52:16.659087+0000","last_clean":"2024-09-14T04:52:16.659087+0000","last_became_active":"2024-09-14T04:44:57.330957+0000","last_became_peered":"2024-09-14T04:44:57.330957+0000","last_unstale":"2024-09-14T04:52:16.659087+0000","last_undegraded":"2024-09-14T04:52:16.659087+0000","last_fullsized":"2024-09-14T04:52:16.659087+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:09.516688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":767,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:18.482742+0000","last_change":"2024-09-14T04:49:49.460401+0000","last_active":"2024-09-14T04:52:18.482742+0000","last_peered":"2024-09-14T04:52:18.482742+0000","last_clean":"2024-09-14T04:52:18.482742+0000","last_became_active":"2024-09-14T04:49:49.458530+0000","last_became_peered":"2024-09-14T04:49:49.458530+0000","last_unstale":"2024-09-14T04:52:18.482742+0000","last_undegraded":"2024-09-14T04:52:18.482742+0000","last_fullsized":"2024-09-14T04:52:18.482742+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:11:29.143239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"349'149","reported_seq":779,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.914308+0000","last_change":"2024-09-14T04:41:38.479980+0000","last_active":"2024-09-14T04:52:16.914308+0000","last_peered":"2024-09-14T04:52:16.914308+0000","last_clean":"2024-09-14T04:52:16.914308+0000","last_became_active":"2024-09-14T04:41:36.933392+0000","last_became_peered":"2024-09-14T04:41:36.933392+0000","last_unstale":"2024-09-14T04:52:16.914308+0000","last_undegraded":"2024-09-14T04:52:16.914308+0000","last_fullsized":"2024-09-14T04:52:16.914308+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:38.479887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:38.479887+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:01:26.102279+0000","scrub_duration":0.0062728319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":172,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:18.009970+0000","last_change":"2024-09-14T04:44:57.329983+0000","last_active":"2024-09-14T04:52:18.009970+0000","last_peered":"2024-09-14T04:52:18.009970+0000","last_clean":"2024-09-14T04:52:18.009970+0000","last_became_active":"2024-09-14T04:44:57.329597+0000","last_became_peered":"2024-09-14T04:44:57.329597+0000","last_unstale":"2024-09-14T04:52:18.009970+0000","last_undegraded":"2024-09-14T04:52:18.009970+0000","last_fullsized":"2024-09-14T04:52:18.009970+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:45.718039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":711,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.993141+0000","last_change":"2024-09-14T04:49:49.443631+0000","last_active":"2024-09-14T04:52:17.993141+0000","last_peered":"2024-09-14T04:52:17.993141+0000","last_clean":"2024-09-14T04:52:17.993141+0000","last_became_active":"2024-09-14T04:49:49.443313+0000","last_became_peered":"2024-09-14T04:49:49.443313+0000","last_unstale":"2024-09-14T04:52:17.993141+0000","last_undegraded":"2024-09-14T04:52:17.993141+0000","last_fullsized":"2024-09-14T04:52:17.993141+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:06:54.562292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1462,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.654450+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T04:52:16.654450+0000","last_peered":"2024-09-14T04:52:16.654450+0000","last_clean":"2024-09-14T04:52:16.654450+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T04:52:16.654450+0000","last_undegraded":"2024-09-14T04:52:16.654450+0000","last_fullsized":"2024-09-14T04:52:16.654450+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"349'149","reported_seq":772,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.654765+0000","last_change":"2024-09-14T04:41:39.012977+0000","last_active":"2024-09-14T04:52:16.654765+0000","last_peered":"2024-09-14T04:52:16.654765+0000","last_clean":"2024-09-14T04:52:16.654765+0000","last_became_active":"2024-09-14T04:41:36.932552+0000","last_became_peered":"2024-09-14T04:41:36.932552+0000","last_unstale":"2024-09-14T04:52:16.654765+0000","last_undegraded":"2024-09-14T04:52:16.654765+0000","last_fullsized":"2024-09-14T04:52:16.654765+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.012879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.012879+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:16:40.080262+0000","scrub_duration":0.0051721620000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":803,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:18.482937+0000","last_change":"2024-09-14T04:49:49.459590+0000","last_active":"2024-09-14T04:52:18.482937+0000","last_peered":"2024-09-14T04:52:18.482937+0000","last_clean":"2024-09-14T04:52:18.482937+0000","last_became_active":"2024-09-14T04:49:49.456263+0000","last_became_peered":"2024-09-14T04:49:49.456263+0000","last_unstale":"2024-09-14T04:52:18.482937+0000","last_undegraded":"2024-09-14T04:52:18.482937+0000","last_fullsized":"2024-09-14T04:52:18.482937+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:20:44.666805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":83,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.657638+0000","last_change":"2024-09-14T04:44:57.329652+0000","last_active":"2024-09-14T04:52:16.657638+0000","last_peered":"2024-09-14T04:52:16.657638+0000","last_clean":"2024-09-14T04:52:16.657638+0000","last_became_active":"2024-09-14T04:44:57.328888+0000","last_became_peered":"2024-09-14T04:44:57.328888+0000","last_unstale":"2024-09-14T04:52:16.657638+0000","last_undegraded":"2024-09-14T04:52:16.657638+0000","last_fullsized":"2024-09-14T04:52:16.657638+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:10.170068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1461,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.662285+0000","last_change":"2024-09-14T04:06:46.875340+0000","last_active":"2024-09-14T04:52:17.662285+0000","last_peered":"2024-09-14T04:52:17.662285+0000","last_clean":"2024-09-14T04:52:17.662285+0000","last_became_active":"2024-09-14T04:06:46.874552+0000","last_became_peered":"2024-09-14T04:06:46.874552+0000","last_unstale":"2024-09-14T04:52:17.662285+0000","last_undegraded":"2024-09-14T04:52:17.662285+0000","last_fullsized":"2024-09-14T04:52:17.662285+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:52.257357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"349'149","reported_seq":1842,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.657947+0000","last_change":"2024-09-14T04:45:00.346627+0000","last_active":"2024-09-14T04:52:16.657947+0000","last_peered":"2024-09-14T04:52:16.657947+0000","last_clean":"2024-09-14T04:52:16.657947+0000","last_became_active":"2024-09-14T04:45:00.346286+0000","last_became_peered":"2024-09-14T04:45:00.346286+0000","last_unstale":"2024-09-14T04:52:16.657947+0000","last_undegraded":"2024-09-14T04:52:16.657947+0000","last_fullsized":"2024-09-14T04:52:16.657947+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:50.784648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:50.784648+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:07:58.441372+0000","scrub_duration":0.01141873,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":165,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:18.010453+0000","last_change":"2024-09-14T04:44:14.489235+0000","last_active":"2024-09-14T04:52:18.010453+0000","last_peered":"2024-09-14T04:52:18.010453+0000","last_clean":"2024-09-14T04:52:18.010453+0000","last_became_active":"2024-09-14T04:44:14.488950+0000","last_became_peered":"2024-09-14T04:44:14.488950+0000","last_unstale":"2024-09-14T04:52:18.010453+0000","last_undegraded":"2024-09-14T04:52:18.010453+0000","last_fullsized":"2024-09-14T04:52:18.010453+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:54.441144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":721,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:18.482384+0000","last_change":"2024-09-14T04:49:49.456317+0000","last_active":"2024-09-14T04:52:18.482384+0000","last_peered":"2024-09-14T04:52:18.482384+0000","last_clean":"2024-09-14T04:52:18.482384+0000","last_became_active":"2024-09-14T04:49:49.456029+0000","last_became_peered":"2024-09-14T04:49:49.456029+0000","last_unstale":"2024-09-14T04:52:18.482384+0000","last_undegraded":"2024-09-14T04:52:18.482384+0000","last_fullsized":"2024-09-14T04:52:18.482384+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:46:05.319676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2238,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:18.482859+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T04:52:18.482859+0000","last_peered":"2024-09-14T04:52:18.482859+0000","last_clean":"2024-09-14T04:52:18.482859+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T04:52:18.482859+0000","last_undegraded":"2024-09-14T04:52:18.482859+0000","last_fullsized":"2024-09-14T04:52:18.482859+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"349'149","reported_seq":666,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.914058+0000","last_change":"2024-09-14T04:50:22.002798+0000","last_active":"2024-09-14T04:52:16.914058+0000","last_peered":"2024-09-14T04:52:16.914058+0000","last_clean":"2024-09-14T04:52:16.914058+0000","last_became_active":"2024-09-14T04:49:49.448278+0000","last_became_peered":"2024-09-14T04:49:49.448278+0000","last_unstale":"2024-09-14T04:52:16.914058+0000","last_undegraded":"2024-09-14T04:52:16.914058+0000","last_fullsized":"2024-09-14T04:52:16.914058+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:50:22.002568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:50:22.002568+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:01:57.489223+0000","scrub_duration":0.010888326,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2239,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.993507+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T04:52:17.993507+0000","last_peered":"2024-09-14T04:52:17.993507+0000","last_clean":"2024-09-14T04:52:17.993507+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T04:52:17.993507+0000","last_undegraded":"2024-09-14T04:52:17.993507+0000","last_fullsized":"2024-09-14T04:52:17.993507+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2246,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.993049+0000","last_change":"2024-09-14T04:44:57.335783+0000","last_active":"2024-09-14T04:52:17.993049+0000","last_peered":"2024-09-14T04:52:17.993049+0000","last_clean":"2024-09-14T04:52:17.993049+0000","last_became_active":"2024-09-14T04:44:57.332780+0000","last_became_peered":"2024-09-14T04:44:57.332780+0000","last_unstale":"2024-09-14T04:52:17.993049+0000","last_undegraded":"2024-09-14T04:52:17.993049+0000","last_fullsized":"2024-09-14T04:52:17.993049+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:57.457593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1491,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.657892+0000","last_change":"2024-09-14T04:42:49.939790+0000","last_active":"2024-09-14T04:52:16.657892+0000","last_peered":"2024-09-14T04:52:16.657892+0000","last_clean":"2024-09-14T04:52:16.657892+0000","last_became_active":"2024-09-14T04:42:49.939591+0000","last_became_peered":"2024-09-14T04:42:49.939591+0000","last_unstale":"2024-09-14T04:52:16.657892+0000","last_undegraded":"2024-09-14T04:52:16.657892+0000","last_fullsized":"2024-09-14T04:52:16.657892+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:23:16.790593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1454,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.914646+0000","last_change":"2024-09-14T04:44:57.326008+0000","last_active":"2024-09-14T04:52:16.914646+0000","last_peered":"2024-09-14T04:52:16.914646+0000","last_clean":"2024-09-14T04:52:16.914646+0000","last_became_active":"2024-09-14T04:44:57.325617+0000","last_became_peered":"2024-09-14T04:44:57.325617+0000","last_unstale":"2024-09-14T04:52:16.914646+0000","last_undegraded":"2024-09-14T04:52:16.914646+0000","last_fullsized":"2024-09-14T04:52:16.914646+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:15.637508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":743,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.654675+0000","last_change":"2024-09-14T04:43:13.354846+0000","last_active":"2024-09-14T04:52:16.654675+0000","last_peered":"2024-09-14T04:52:16.654675+0000","last_clean":"2024-09-14T04:52:16.654675+0000","last_became_active":"2024-09-14T04:43:13.354515+0000","last_became_peered":"2024-09-14T04:43:13.354515+0000","last_unstale":"2024-09-14T04:52:16.654675+0000","last_undegraded":"2024-09-14T04:52:16.654675+0000","last_fullsized":"2024-09-14T04:52:16.654675+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:17.889341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1460,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.662360+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T04:52:17.662360+0000","last_peered":"2024-09-14T04:52:17.662360+0000","last_clean":"2024-09-14T04:52:17.662360+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T04:52:17.662360+0000","last_undegraded":"2024-09-14T04:52:17.662360+0000","last_fullsized":"2024-09-14T04:52:17.662360+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":1797,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.657176+0000","last_change":"2024-09-14T04:41:30.640651+0000","last_active":"2024-09-14T04:52:16.657176+0000","last_peered":"2024-09-14T04:52:16.657176+0000","last_clean":"2024-09-14T04:52:16.657176+0000","last_became_active":"2024-09-14T04:41:29.532466+0000","last_became_peered":"2024-09-14T04:41:29.532466+0000","last_unstale":"2024-09-14T04:52:16.657176+0000","last_undegraded":"2024-09-14T04:52:16.657176+0000","last_fullsized":"2024-09-14T04:52:16.657176+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":0,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:30.640520+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:30.640520+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:27.776724+0000","scrub_duration":0.0028653139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"349'149","reported_seq":771,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.654460+0000","last_change":"2024-09-14T04:41:39.978080+0000","last_active":"2024-09-14T04:52:16.654460+0000","last_peered":"2024-09-14T04:52:16.654460+0000","last_clean":"2024-09-14T04:52:16.654460+0000","last_became_active":"2024-09-14T04:41:36.929279+0000","last_became_peered":"2024-09-14T04:41:36.929279+0000","last_unstale":"2024-09-14T04:52:16.654460+0000","last_undegraded":"2024-09-14T04:52:16.654460+0000","last_fullsized":"2024-09-14T04:52:16.654460+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.977999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.977999+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:44:06.383827+0000","scrub_duration":0.00192122,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":99,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:18.482793+0000","last_change":"2024-09-14T04:48:11.040133+0000","last_active":"2024-09-14T04:52:18.482793+0000","last_peered":"2024-09-14T04:52:18.482793+0000","last_clean":"2024-09-14T04:52:18.482793+0000","last_became_active":"2024-09-14T04:48:11.039787+0000","last_became_peered":"2024-09-14T04:48:11.039787+0000","last_unstale":"2024-09-14T04:52:18.482793+0000","last_undegraded":"2024-09-14T04:52:18.482793+0000","last_fullsized":"2024-09-14T04:52:18.482793+0000","mapping_epoch":1428,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:53.520672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2086,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:18.009936+0000","last_change":"2024-09-14T04:49:29.616443+0000","last_active":"2024-09-14T04:52:18.009936+0000","last_peered":"2024-09-14T04:52:18.009936+0000","last_clean":"2024-09-14T04:52:18.009936+0000","last_became_active":"2024-09-14T04:49:29.616066+0000","last_became_peered":"2024-09-14T04:49:29.616066+0000","last_unstale":"2024-09-14T04:52:18.009936+0000","last_undegraded":"2024-09-14T04:52:18.009936+0000","last_fullsized":"2024-09-14T04:52:18.009936+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:09:15.897957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":83,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.657166+0000","last_change":"2024-09-14T04:44:57.326724+0000","last_active":"2024-09-14T04:52:16.657166+0000","last_peered":"2024-09-14T04:52:16.657166+0000","last_clean":"2024-09-14T04:52:16.657166+0000","last_became_active":"2024-09-14T04:44:57.326191+0000","last_became_peered":"2024-09-14T04:44:57.326191+0000","last_unstale":"2024-09-14T04:52:16.657166+0000","last_undegraded":"2024-09-14T04:52:16.657166+0000","last_fullsized":"2024-09-14T04:52:16.657166+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:46.060269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":83,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.656356+0000","last_change":"2024-09-14T04:44:57.328528+0000","last_active":"2024-09-14T04:52:16.656356+0000","last_peered":"2024-09-14T04:52:16.656356+0000","last_clean":"2024-09-14T04:52:16.656356+0000","last_became_active":"2024-09-14T04:44:57.327739+0000","last_became_peered":"2024-09-14T04:44:57.327739+0000","last_unstale":"2024-09-14T04:52:16.656356+0000","last_undegraded":"2024-09-14T04:52:16.656356+0000","last_fullsized":"2024-09-14T04:52:16.656356+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:04:58.294690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"349'149","reported_seq":787,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.654589+0000","last_change":"2024-09-14T04:48:32.059177+0000","last_active":"2024-09-14T04:52:16.654589+0000","last_peered":"2024-09-14T04:52:16.654589+0000","last_clean":"2024-09-14T04:52:16.654589+0000","last_became_active":"2024-09-14T04:48:32.058741+0000","last_became_peered":"2024-09-14T04:48:32.058741+0000","last_unstale":"2024-09-14T04:52:16.654589+0000","last_undegraded":"2024-09-14T04:52:16.654589+0000","last_fullsized":"2024-09-14T04:52:16.654589+0000","mapping_epoch":1435,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1436,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:56.738636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:56.738636+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:20.932276+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":83,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.657852+0000","last_change":"2024-09-14T04:44:57.323436+0000","last_active":"2024-09-14T04:52:16.657852+0000","last_peered":"2024-09-14T04:52:16.657852+0000","last_clean":"2024-09-14T04:52:16.657852+0000","last_became_active":"2024-09-14T04:44:57.323022+0000","last_became_peered":"2024-09-14T04:44:57.323022+0000","last_unstale":"2024-09-14T04:52:16.657852+0000","last_undegraded":"2024-09-14T04:52:16.657852+0000","last_fullsized":"2024-09-14T04:52:16.657852+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:22.775261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1489,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.914142+0000","last_change":"2024-09-14T04:44:57.322357+0000","last_active":"2024-09-14T04:52:16.914142+0000","last_peered":"2024-09-14T04:52:16.914142+0000","last_clean":"2024-09-14T04:52:16.914142+0000","last_became_active":"2024-09-14T04:44:57.321990+0000","last_became_peered":"2024-09-14T04:44:57.321990+0000","last_unstale":"2024-09-14T04:52:16.914142+0000","last_undegraded":"2024-09-14T04:52:16.914142+0000","last_fullsized":"2024-09-14T04:52:16.914142+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:09.658008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"349'149","reported_seq":1813,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.657554+0000","last_change":"2024-09-14T04:41:53.858076+0000","last_active":"2024-09-14T04:52:16.657554+0000","last_peered":"2024-09-14T04:52:16.657554+0000","last_clean":"2024-09-14T04:52:16.657554+0000","last_became_active":"2024-09-14T04:41:36.931585+0000","last_became_peered":"2024-09-14T04:41:36.931585+0000","last_unstale":"2024-09-14T04:52:16.657554+0000","last_undegraded":"2024-09-14T04:52:16.657554+0000","last_fullsized":"2024-09-14T04:52:16.657554+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:53.857924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:53.857924+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:10:20.658986+0000","scrub_duration":0.0022355919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":83,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.657705+0000","last_change":"2024-09-14T04:44:57.329198+0000","last_active":"2024-09-14T04:52:16.657705+0000","last_peered":"2024-09-14T04:52:16.657705+0000","last_clean":"2024-09-14T04:52:16.657705+0000","last_became_active":"2024-09-14T04:44:57.327754+0000","last_became_peered":"2024-09-14T04:44:57.327754+0000","last_unstale":"2024-09-14T04:52:16.657705+0000","last_undegraded":"2024-09-14T04:52:16.657705+0000","last_fullsized":"2024-09-14T04:52:16.657705+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:10:27.944476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":223,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.993412+0000","last_change":"2024-09-14T04:44:57.348687+0000","last_active":"2024-09-14T04:52:17.993412+0000","last_peered":"2024-09-14T04:52:17.993412+0000","last_clean":"2024-09-14T04:52:17.993412+0000","last_became_active":"2024-09-14T04:44:57.332845+0000","last_became_peered":"2024-09-14T04:44:57.332845+0000","last_unstale":"2024-09-14T04:52:17.993412+0000","last_undegraded":"2024-09-14T04:52:17.993412+0000","last_fullsized":"2024-09-14T04:52:17.993412+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:12:18.762737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2245,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.993320+0000","last_change":"2024-09-14T04:44:57.333468+0000","last_active":"2024-09-14T04:52:17.993320+0000","last_peered":"2024-09-14T04:52:17.993320+0000","last_clean":"2024-09-14T04:52:17.993320+0000","last_became_active":"2024-09-14T04:44:57.332626+0000","last_became_peered":"2024-09-14T04:44:57.332626+0000","last_unstale":"2024-09-14T04:52:17.993320+0000","last_undegraded":"2024-09-14T04:52:17.993320+0000","last_fullsized":"2024-09-14T04:52:17.993320+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:57:26.366438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1445,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.654500+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T04:52:16.654500+0000","last_peered":"2024-09-14T04:52:16.654500+0000","last_clean":"2024-09-14T04:52:16.654500+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T04:52:16.654500+0000","last_undegraded":"2024-09-14T04:52:16.654500+0000","last_fullsized":"2024-09-14T04:52:16.654500+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1444,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.913982+0000","last_change":"2024-09-14T04:06:46.869868+0000","last_active":"2024-09-14T04:52:16.913982+0000","last_peered":"2024-09-14T04:52:16.913982+0000","last_clean":"2024-09-14T04:52:16.913982+0000","last_became_active":"2024-09-14T04:06:46.869126+0000","last_became_peered":"2024-09-14T04:06:46.869126+0000","last_unstale":"2024-09-14T04:52:16.913982+0000","last_undegraded":"2024-09-14T04:52:16.913982+0000","last_fullsized":"2024-09-14T04:52:16.913982+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:41.150138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1428,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.658991+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T04:52:16.658991+0000","last_peered":"2024-09-14T04:52:16.658991+0000","last_clean":"2024-09-14T04:52:16.658991+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T04:52:16.658991+0000","last_undegraded":"2024-09-14T04:52:16.658991+0000","last_fullsized":"2024-09-14T04:52:16.658991+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":83,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.656663+0000","last_change":"2024-09-14T04:44:57.332112+0000","last_active":"2024-09-14T04:52:16.656663+0000","last_peered":"2024-09-14T04:52:16.656663+0000","last_clean":"2024-09-14T04:52:16.656663+0000","last_became_active":"2024-09-14T04:44:57.331575+0000","last_became_peered":"2024-09-14T04:44:57.331575+0000","last_unstale":"2024-09-14T04:52:16.656663+0000","last_undegraded":"2024-09-14T04:52:16.656663+0000","last_fullsized":"2024-09-14T04:52:16.656663+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:50:00.391283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1476,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.659470+0000","last_change":"2024-09-14T04:44:57.351460+0000","last_active":"2024-09-14T04:52:16.659470+0000","last_peered":"2024-09-14T04:52:16.659470+0000","last_clean":"2024-09-14T04:52:16.659470+0000","last_became_active":"2024-09-14T04:44:57.334662+0000","last_became_peered":"2024-09-14T04:44:57.334662+0000","last_unstale":"2024-09-14T04:52:16.659470+0000","last_undegraded":"2024-09-14T04:52:16.659470+0000","last_fullsized":"2024-09-14T04:52:16.659470+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:02.399062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1430,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.654985+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T04:52:16.654985+0000","last_peered":"2024-09-14T04:52:16.654985+0000","last_clean":"2024-09-14T04:52:16.654985+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T04:52:16.654985+0000","last_undegraded":"2024-09-14T04:52:16.654985+0000","last_fullsized":"2024-09-14T04:52:16.654985+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1488,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.659364+0000","last_change":"2024-09-14T04:44:57.349690+0000","last_active":"2024-09-14T04:52:16.659364+0000","last_peered":"2024-09-14T04:52:16.659364+0000","last_clean":"2024-09-14T04:52:16.659364+0000","last_became_active":"2024-09-14T04:44:57.333579+0000","last_became_peered":"2024-09-14T04:44:57.333579+0000","last_unstale":"2024-09-14T04:52:16.659364+0000","last_undegraded":"2024-09-14T04:52:16.659364+0000","last_fullsized":"2024-09-14T04:52:16.659364+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:17:55.265141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1527,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.914014+0000","last_change":"2024-09-14T04:48:29.893179+0000","last_active":"2024-09-14T04:52:16.914014+0000","last_peered":"2024-09-14T04:52:16.914014+0000","last_clean":"2024-09-14T04:52:16.914014+0000","last_became_active":"2024-09-14T04:48:29.864724+0000","last_became_peered":"2024-09-14T04:48:29.864724+0000","last_unstale":"2024-09-14T04:52:16.914014+0000","last_undegraded":"2024-09-14T04:52:16.914014+0000","last_fullsized":"2024-09-14T04:52:16.914014+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:19:15.120740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":681,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.659223+0000","last_change":"2024-09-14T04:49:29.616761+0000","last_active":"2024-09-14T04:52:16.659223+0000","last_peered":"2024-09-14T04:52:16.659223+0000","last_clean":"2024-09-14T04:52:16.659223+0000","last_became_active":"2024-09-14T04:49:29.616072+0000","last_became_peered":"2024-09-14T04:49:29.616072+0000","last_unstale":"2024-09-14T04:52:16.659223+0000","last_undegraded":"2024-09-14T04:52:16.659223+0000","last_fullsized":"2024-09-14T04:52:16.659223+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:49:52.358949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1450,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.657787+0000","last_change":"2024-09-14T04:44:57.332511+0000","last_active":"2024-09-14T04:52:16.657787+0000","last_peered":"2024-09-14T04:52:16.657787+0000","last_clean":"2024-09-14T04:52:16.657787+0000","last_became_active":"2024-09-14T04:44:57.332166+0000","last_became_peered":"2024-09-14T04:44:57.332166+0000","last_unstale":"2024-09-14T04:52:16.657787+0000","last_undegraded":"2024-09-14T04:52:16.657787+0000","last_fullsized":"2024-09-14T04:52:16.657787+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:53.984639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2225,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.993353+0000","last_change":"2024-09-14T04:07:58.560880+0000","last_active":"2024-09-14T04:52:17.993353+0000","last_peered":"2024-09-14T04:52:17.993353+0000","last_clean":"2024-09-14T04:52:17.993353+0000","last_became_active":"2024-09-14T04:07:58.560132+0000","last_became_peered":"2024-09-14T04:07:58.560132+0000","last_unstale":"2024-09-14T04:52:17.993353+0000","last_undegraded":"2024-09-14T04:52:17.993353+0000","last_fullsized":"2024-09-14T04:52:17.993353+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:46:00.287050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2224,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.993227+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T04:52:17.993227+0000","last_peered":"2024-09-14T04:52:17.993227+0000","last_clean":"2024-09-14T04:52:17.993227+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T04:52:17.993227+0000","last_undegraded":"2024-09-14T04:52:17.993227+0000","last_fullsized":"2024-09-14T04:52:17.993227+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1442,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.654334+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T04:52:16.654334+0000","last_peered":"2024-09-14T04:52:16.654334+0000","last_clean":"2024-09-14T04:52:16.654334+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T04:52:16.654334+0000","last_undegraded":"2024-09-14T04:52:16.654334+0000","last_fullsized":"2024-09-14T04:52:16.654334+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1479,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.658269+0000","last_change":"2024-09-14T04:44:57.339993+0000","last_active":"2024-09-14T04:52:16.658269+0000","last_peered":"2024-09-14T04:52:16.658269+0000","last_clean":"2024-09-14T04:52:16.658269+0000","last_became_active":"2024-09-14T04:44:57.330472+0000","last_became_peered":"2024-09-14T04:44:57.330472+0000","last_unstale":"2024-09-14T04:52:16.658269+0000","last_undegraded":"2024-09-14T04:52:16.658269+0000","last_fullsized":"2024-09-14T04:52:16.658269+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:19:27.046988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2027,"reported_epoch":1471,"state":"active+clean","last_fresh":"2024-09-14T04:52:16.657619+0000","last_change":"2024-09-14T04:49:49.449687+0000","last_active":"2024-09-14T04:52:16.657619+0000","last_peered":"2024-09-14T04:52:16.657619+0000","last_clean":"2024-09-14T04:52:16.657619+0000","last_became_active":"2024-09-14T04:49:49.449144+0000","last_became_peered":"2024-09-14T04:49:49.449144+0000","last_unstale":"2024-09-14T04:52:16.657619+0000","last_undegraded":"2024-09-14T04:52:16.657619+0000","last_fullsized":"2024-09-14T04:52:16.657619+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:59:06.779730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":56,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":5070,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4250,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223202016,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":588200,"kb_used_data":11608,"kb_used_omap":1,"kb_used_meta":576574,"kb_avail":93156952,"statfs":{"total":95995035648,"available":95392718848,"internally_reserved":0,"allocated":11886592,"data_stored":6969487,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":590412235},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463652,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":610036,"kb_used_data":11296,"kb_used_omap":1,"kb_used_meta":598718,"kb_avail":93135116,"statfs":{"total":95995035648,"available":95370358784,"internally_reserved":0,"allocated":11567104,"data_stored":6650821,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":613087687},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725289,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":562216,"kb_used_data":11868,"kb_used_omap":1,"kb_used_meta":550334,"kb_avail":93182936,"statfs":{"total":95995035648,"available":95419326464,"internally_reserved":0,"allocated":12152832,"data_stored":7237015,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":563542470},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":1392,"seq":5978594476135,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":204860,"kb_used_data":11304,"kb_used_omap":117,"kb_used_meta":193418,"kb_avail":93540292,"statfs":{"total":95995035648,"available":95785259008,"internally_reserved":0,"allocated":11575296,"data_stored":6651500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120638,"internal_metadata":198060226},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772075030,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":625624,"kb_used_data":11848,"kb_used_omap":71,"kb_used_meta":613688,"kb_avail":93119528,"statfs":{"total":95995035648,"available":95354396672,"internally_reserved":0,"allocated":12132352,"data_stored":7227927,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":628417462},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707303966,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":394852,"kb_used_data":11544,"kb_used_omap":68,"kb_used_meta":383227,"kb_avail":93350300,"statfs":{"total":95995035648,"available":95590707200,"internally_reserved":0,"allocated":11821056,"data_stored":6909292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":392424656},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872729125,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":502816,"kb_used_data":11280,"kb_used_omap":67,"kb_used_meta":491452,"kb_avail":93242336,"statfs":{"total":95995035648,"available":95480152064,"internally_reserved":0,"allocated":11550720,"data_stored":6646007,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":503247205},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056175,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":502608,"kb_used_data":11268,"kb_used_omap":74,"kb_used_meta":491253,"kb_avail":93242544,"statfs":{"total":95995035648,"available":95480365056,"internally_reserved":0,"allocated":11538432,"data_stored":6637451,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":503043765},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1901,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:52:26.412 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:52:26.425 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:26.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:26.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:26.526 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:26.550 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:26.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:26.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:26.651 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:26.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:26.674 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:26.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:26.711 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:26 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:52:26.711 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:26 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:52:26.711 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:26 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:52:26.711 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:26 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:52:26.711 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:26 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:52:26.711 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:26 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:52:26.776 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:26.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:26.799 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:26.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:26.900 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:26.926 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:26.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:26.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:26.959 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:52:26.959 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:52:26.976 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:52:26.976 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:52:26.994 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:52:26.994 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:52:27.010 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:52:27.011 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:52:27.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:26 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:52:27.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:26 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:52:27.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:26 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:52:27.027 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:52:27.028 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:52:27.028 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:27.045 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:52:27.045 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:52:27.053 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:27.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:27.054 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:27.062 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:52:27.062 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:52:27.078 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:52:27.079 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:52:27.155 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:27.179 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:27.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:27.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:27.281 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:27.306 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:27.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:27.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:27.408 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:27.431 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:27.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:27.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:27.533 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:27.559 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:27.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:27.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:27.661 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:27.687 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:27.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:27.688 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:27.789 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:27.815 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:27.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:27.816 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:27.899 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:27 smithi039 ceph-mon[106470]: pgmap v2726: 99 pgs: 99 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:27.899 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:27 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/815496647' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:52:27.900 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:27 smithi039 ceph-mon[111235]: pgmap v2726: 99 pgs: 99 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:27.900 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:27 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/815496647' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:52:27.917 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:27.940 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:27.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:27.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:28.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:27 smithi129 ceph-mon[101316]: pgmap v2726: 99 pgs: 99 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:28.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:27 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/815496647' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:52:28.042 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:28.068 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:28.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:28.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:28.170 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:28.196 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:28.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:28.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:28.298 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:28.321 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:28.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:28.322 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:28.423 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:28.449 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:28.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:28.450 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:28.551 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:28.558 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:52:28.576 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:28.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:28.577 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:28.593 INFO:teuthology.orchestra.run.smithi039.stderr:enabled application 'rados' on pool 'unique_pool_3' 2024-09-14T04:52:28.593 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is set 2024-09-14T04:52:28.596 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:52:28.678 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:28.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:28.707 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:28.707 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:28.808 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:28.833 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:28.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:28.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:28.839 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:52:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:52:28] "GET /metrics HTTP/1.1" 200 36663 "" "Prometheus/2.51.0" 2024-09-14T04:52:28.839 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:28 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2727366788' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:52:28.840 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:28 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4011596391' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:52:28.840 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:28 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2727366788' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:52:28.840 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:28 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4011596391' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:52:28.936 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:28.958 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:28.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:28.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:29.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:28 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2727366788' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:52:29.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:28 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4011596391' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:52:29.060 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:29.086 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:29.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:29.088 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:29.188 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:29.212 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:29.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:29.213 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:29.314 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:29.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:29.338 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:29.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:29.439 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:29.465 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:29.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:29.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:29.567 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:29.593 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:29.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:29.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:29.695 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:29.718 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:29.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:29.719 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:29.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:29 smithi039 ceph-mon[106470]: pgmap v2727: 99 pgs: 99 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:29.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:29 smithi039 ceph-mon[106470]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:52:29.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:29 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2727366788' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:52:29.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:29 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4011596391' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:52:29.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:29 smithi039 ceph-mon[106470]: osdmap e1473: 8 total, 8 up, 8 in 2024-09-14T04:52:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:29 smithi039 ceph-mon[111235]: pgmap v2727: 99 pgs: 99 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:29 smithi039 ceph-mon[111235]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:52:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:29 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2727366788' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:52:29.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:29 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4011596391' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:52:29.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:29 smithi039 ceph-mon[111235]: osdmap e1473: 8 total, 8 up, 8 in 2024-09-14T04:52:29.820 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:29.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:29.849 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:29.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:29.922 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:29 smithi129 ceph-mon[101316]: pgmap v2727: 99 pgs: 99 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:29.922 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:29 smithi129 ceph-mon[101316]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:52:29.922 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:29 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2727366788' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:52:29.922 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:29 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4011596391' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:52:29.922 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:29 smithi129 ceph-mon[101316]: osdmap e1473: 8 total, 8 up, 8 in 2024-09-14T04:52:29.950 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:29.976 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:29.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:29.977 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:30.078 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:30.102 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:30.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:30.103 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:30.203 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:30.225 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:30.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:30.226 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:30.327 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:30.350 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:30.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:30.351 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:30.452 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:30.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:30.474 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:30.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:30.575 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:30.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:30.602 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:30.602 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:30.703 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:30.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:30.725 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:30.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:30.827 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:30.851 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:30.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:30.852 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:30.945 DEBUG:teuthology.orchestra.run.smithi039:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2024-09-14T04:52:30.947 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-14T04:52:30.947 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-14T04:52:30.953 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:30.964 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:30 smithi039 ceph-mon[106470]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:52:30.965 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:30 smithi039 ceph-mon[111235]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:52:30.974 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:30.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:30.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:31.005 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:hints = 1 2024-09-14T04:52:31.005 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-14T04:52:31.005 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Object prefix: benchmark_data_smithi039_330847 2024-09-14T04:52:31.005 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:52:31.005 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 0 0 0 0 0 0 - 0 2024-09-14T04:52:31.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:30 smithi129 ceph-mon[101316]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:52:31.076 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:31.113 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:31.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:31.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:31.215 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:31.250 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:31.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:31.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:31.352 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:31.391 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:31.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:31.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:31.493 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:31.526 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:31.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:31.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:31.628 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:31.665 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:31.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:31.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:31.766 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:31.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:31 smithi039 ceph-mon[106470]: pgmap v2729: 99 pgs: 99 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:31.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:31 smithi039 ceph-mon[111235]: pgmap v2729: 99 pgs: 99 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:31.802 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:31.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:31.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:31.904 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:31.940 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:31.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:31.942 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:32.005 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 1 16 3951 3935 245.924 245.938 0.00262106 0.00405178 2024-09-14T04:52:32.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:31 smithi129 ceph-mon[101316]: pgmap v2729: 99 pgs: 99 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:32.042 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:32.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:32.077 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:32.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:32.178 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:32.216 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:32.217 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:32.217 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:32.318 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:32.355 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:32.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:32.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:32.457 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:32.495 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:32.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:32.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:32.597 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:32.635 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:32.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:32.636 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:32.737 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:32.766 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:52:32.774 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:32.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:32.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:32.876 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:32.915 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:32.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:32.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:33.006 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 2 16 7657 7641 238.759 231.625 0.00362977 0.00417854 2024-09-14T04:52:33.016 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:33.051 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:33.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:33.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:33.153 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:33.193 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:33.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:33.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:33.295 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:33.333 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:33.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:33.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:33.435 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:33.473 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:33.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:33.474 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:33.575 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:33.613 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:33.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:33.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:33.714 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:33.747 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:33.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:33.748 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:33.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:33 smithi039 ceph-mon[106470]: pgmap v2730: 99 pgs: 99 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:33.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:33 smithi039 ceph-mon[111235]: pgmap v2730: 99 pgs: 99 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:33.849 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:33.891 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:33.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:33.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:33.952 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:52:33.952 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [ 2024-09-14T04:52:33.953 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:52:33.953 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:10320 56.1 e1473/1470 56:83d59207:::benchmark_data_smithi039_330847_object10319:head v 1473'625, mlcod=1473'623)", 2024-09-14T04:52:33.953 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:52:33.801419+0000", 2024-09-14T04:52:33.953 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.0019756399999999999, 2024-09-14T04:52:33.953 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0020037470000000002, 2024-09-14T04:52:33.953 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:52:33.953 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:52:33.953 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "started", 2024-09-14T04:52:33.953 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:52:33.953 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:52:33.953 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:52:33.953 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:33.801419+0000", 2024-09-14T04:52:33.953 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:52:33.954 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:52:33.954 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:52:33.954 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:52:33.954 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:33.801419+0000", 2024-09-14T04:52:33.954 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:52:33.954 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:52:33.954 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:52:33.954 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:52:33.954 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:33.801420+0000", 2024-09-14T04:52:33.954 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.8999999999999995e-07 2024-09-14T04:52:33.954 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:52:33.954 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:52:33.954 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:52:33.954 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:33.802252+0000", 2024-09-14T04:52:33.955 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00083229799999999998 2024-09-14T04:52:33.955 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:52:33.955 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:52:33.955 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:52:33.955 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:33.802267+0000", 2024-09-14T04:52:33.955 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5131e-05 2024-09-14T04:52:33.955 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:52:33.955 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:52:33.955 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:52:33.955 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:33.802275+0000", 2024-09-14T04:52:33.955 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1450000000000002e-06 2024-09-14T04:52:33.955 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:52:33.955 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:52:33.955 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:52:33.956 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:33.802325+0000", 2024-09-14T04:52:33.956 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.0479999999999998e-05 2024-09-14T04:52:33.956 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:52:33.956 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:52:33.956 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:52:33.956 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:33.802345+0000", 2024-09-14T04:52:33.956 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0072999999999999e-05 2024-09-14T04:52:33.956 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:52:33.956 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:52:33.956 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:52:33.956 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:52:33.956 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:52:33.956 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:10317 56.9 e1473/1470 56:907fe892:::benchmark_data_smithi039_330847_object10316:head v 1473'677, mlcod=1473'674)", 2024-09-14T04:52:33.956 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:52:33.802291+0000", 2024-09-14T04:52:33.957 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.001103997, 2024-09-14T04:52:33.957 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0011987600000000001, 2024-09-14T04:52:33.957 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:52:33.957 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:52:33.957 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "started", 2024-09-14T04:52:33.957 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:52:33.957 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:52:33.957 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:52:33.957 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:33.802291+0000", 2024-09-14T04:52:33.957 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:52:33.957 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:52:33.958 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:52:33.958 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:52:33.958 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:33.802291+0000", 2024-09-14T04:52:33.958 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:52:33.958 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:52:33.958 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:52:33.958 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:52:33.958 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:33.802291+0000", 2024-09-14T04:52:33.958 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2100000000000002e-07 2024-09-14T04:52:33.958 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:52:33.958 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:52:33.958 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:52:33.958 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:33.802320+0000", 2024-09-14T04:52:33.958 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.8804999999999999e-05 2024-09-14T04:52:33.959 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:52:33.959 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:52:33.959 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:52:33.959 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:33.802327+0000", 2024-09-14T04:52:33.959 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.4869999999999996e-06 2024-09-14T04:52:33.959 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:52:33.959 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:52:33.959 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:52:33.959 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:33.802333+0000", 2024-09-14T04:52:33.959 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6019999999999999e-06 2024-09-14T04:52:33.959 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:52:33.959 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:52:33.960 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:52:33.960 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:33.803027+0000", 2024-09-14T04:52:33.960 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00069393799999999996 2024-09-14T04:52:33.960 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:52:33.960 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:52:33.960 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:52:33.960 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:33.803046+0000", 2024-09-14T04:52:33.960 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9083999999999999e-05 2024-09-14T04:52:33.960 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:52:33.960 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:52:33.960 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:52:33.960 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:52:33.960 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:52:33.961 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:10321 56.7 e1473/1470 56:e96b8df7:::benchmark_data_smithi039_330847_object10320:head v 1473'669, mlcod=1473'667)", 2024-09-14T04:52:33.961 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:52:33.803250+0000", 2024-09-14T04:52:33.961 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.00014494200000000001, 2024-09-14T04:52:33.961 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00028545799999999999, 2024-09-14T04:52:33.961 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:52:33.961 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:52:33.961 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "started", 2024-09-14T04:52:33.961 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:52:33.961 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:52:33.961 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:52:33.961 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:33.803250+0000", 2024-09-14T04:52:33.961 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:52:33.961 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:52:33.961 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:52:33.962 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:52:33.962 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:33.803250+0000", 2024-09-14T04:52:33.962 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:52:33.962 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:52:33.962 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:52:33.962 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:52:33.962 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:33.803251+0000", 2024-09-14T04:52:33.962 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0979999999999999e-06 2024-09-14T04:52:33.963 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:52:33.963 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:52:33.963 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:52:33.963 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:33.803277+0000", 2024-09-14T04:52:33.963 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.5996e-05 2024-09-14T04:52:33.963 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:52:33.963 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:52:33.963 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:52:33.963 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:33.803289+0000", 2024-09-14T04:52:33.963 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1747e-05 2024-09-14T04:52:33.963 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:52:33.963 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:52:33.963 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:52:33.964 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:33.803296+0000", 2024-09-14T04:52:33.964 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9789999999999996e-06 2024-09-14T04:52:33.964 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:52:33.964 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:52:33.964 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:52:33.964 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:33.803361+0000", 2024-09-14T04:52:33.964 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.5526999999999998e-05 2024-09-14T04:52:33.964 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:52:33.964 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:52:33.964 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:52:33.964 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:33.803381+0000", 2024-09-14T04:52:33.964 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0239000000000001e-05 2024-09-14T04:52:33.964 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:52:33.964 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:52:33.965 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:52:33.965 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:52:33.965 INFO:teuthology.orchestra.run.smithi039.stdout: ], 2024-09-14T04:52:33.965 INFO:teuthology.orchestra.run.smithi039.stdout: "num_ops": 3 2024-09-14T04:52:33.965 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:52:33.991 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:52:33.993 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:34.006 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 3 16 10950 10934 227.741 205.812 0.0121839 0.0043823 2024-09-14T04:52:34.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:33 smithi129 ceph-mon[101316]: pgmap v2730: 99 pgs: 99 active+clean; 582 KiB data, 3.8 GiB used, 711 GiB / 715 GiB avail 2024-09-14T04:52:34.035 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:34.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:34.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:34.137 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:34.169 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:34.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:34.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:34.271 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:34.278 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:52:34.280 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T04:52:34.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:34.307 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:34.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:34.408 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:34.443 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:34.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:34.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:34.545 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:34.579 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:34.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:34.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:34.681 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:34.720 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:34.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:34.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:34.822 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:34.860 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:34.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:34.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:34.964 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:35.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:35.001 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:35.002 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:35.006 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 4 16 14088 14072 219.834 196.125 0.00320144 0.00454206 2024-09-14T04:52:35.071 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:52:35.103 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:35.105 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-14T04:52:35.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:35.145 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:35.145 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:35.246 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:35.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:35.290 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:35.291 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:35.392 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:35.424 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:35.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:35.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:35.526 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:35.561 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:35.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:35.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:35.663 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:35.699 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:35.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:35.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:35.801 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:35.838 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:35.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:35.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:35.940 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:35.975 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:35.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:35.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:36.006 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 5 16 17038 17022 212.739 184.375 0.0034072 0.00469411 2024-09-14T04:52:36.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:35 smithi129 ceph-mon[101316]: pgmap v2731: 99 pgs: 99 active+clean; 567 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s 2024-09-14T04:52:36.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:35 smithi129 ceph-mon[101316]: from='client.36018 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:52:36.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:35 smithi039 ceph-mon[106470]: pgmap v2731: 99 pgs: 99 active+clean; 567 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s 2024-09-14T04:52:36.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:35 smithi039 ceph-mon[106470]: from='client.36018 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:52:36.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:35 smithi039 ceph-mon[111235]: pgmap v2731: 99 pgs: 99 active+clean; 567 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s 2024-09-14T04:52:36.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:35 smithi039 ceph-mon[111235]: from='client.36018 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:52:36.077 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:36.111 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:36.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:36.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:36.180 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":2731,"stamp":"2024-09-14T04:52:33.810989+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":594221029,"num_objects":9084,"num_object_clones":0,"num_object_copies":27252,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9084,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9122,"num_write_kb":579745,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":43,"num_bytes_recovered":793134,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":10756,"ondisk_log_size":10756,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":297,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5004124,"kb_used_data":1117264,"kb_used_omap":404,"kb_used_meta":3886379,"kb_avail":744957092,"statfs":{"total":767960285184,"available":762836062208,"internally_reserved":0,"allocated":1144078336,"data_stored":1104711204,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":414022,"internal_metadata":3979652794},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[12,6,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":593625088,"num_objects":9058,"num_object_clones":0,"num_object_copies":27174,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9058,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9058,"num_write_kb":579712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.002575"},"pg_stats":[{"pgid":"56.f","version":"1473'455","reported_seq":467,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:32.805562+0000","last_change":"2024-09-14T04:52:16.661278+0000","last_active":"2024-09-14T04:52:32.805562+0000","last_peered":"2024-09-14T04:52:32.805562+0000","last_clean":"2024-09-14T04:52:32.805562+0000","last_became_active":"2024-09-14T04:52:16.660906+0000","last_became_peered":"2024-09-14T04:52:16.660906+0000","last_unstale":"2024-09-14T04:52:32.805562+0000","last_undegraded":"2024-09-14T04:52:32.805562+0000","last_fullsized":"2024-09-14T04:52:32.805562+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":455,"log_dups_size":0,"ondisk_log_size":455,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:55:16.908898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29818880,"num_objects":455,"num_object_clones":0,"num_object_copies":1365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":455,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":455,"num_write_kb":29120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"56.e","version":"1473'629","reported_seq":641,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:33.619990+0000","last_change":"2024-09-14T04:52:16.723745+0000","last_active":"2024-09-14T04:52:33.619990+0000","last_peered":"2024-09-14T04:52:33.619990+0000","last_clean":"2024-09-14T04:52:33.619990+0000","last_became_active":"2024-09-14T04:52:16.723416+0000","last_became_peered":"2024-09-14T04:52:16.723416+0000","last_unstale":"2024-09-14T04:52:33.619990+0000","last_undegraded":"2024-09-14T04:52:33.619990+0000","last_fullsized":"2024-09-14T04:52:33.619990+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":629,"log_dups_size":0,"ondisk_log_size":629,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:20:17.253535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41222144,"num_objects":629,"num_object_clones":0,"num_object_copies":1887,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":629,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":629,"num_write_kb":40256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"56.d","version":"1473'626","reported_seq":641,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:33.557375+0000","last_change":"2024-09-14T04:52:16.871824+0000","last_active":"2024-09-14T04:52:33.557375+0000","last_peered":"2024-09-14T04:52:33.557375+0000","last_clean":"2024-09-14T04:52:33.557375+0000","last_became_active":"2024-09-14T04:52:16.871446+0000","last_became_peered":"2024-09-14T04:52:16.871446+0000","last_unstale":"2024-09-14T04:52:33.557375+0000","last_undegraded":"2024-09-14T04:52:33.557375+0000","last_fullsized":"2024-09-14T04:52:33.557375+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":626,"log_dups_size":0,"ondisk_log_size":626,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:44:00.576543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":41025536,"num_objects":626,"num_object_clones":0,"num_object_copies":1878,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":626,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":626,"num_write_kb":40064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"56.c","version":"1473'617","reported_seq":629,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:33.636395+0000","last_change":"2024-09-14T04:52:16.723896+0000","last_active":"2024-09-14T04:52:33.636395+0000","last_peered":"2024-09-14T04:52:33.636395+0000","last_clean":"2024-09-14T04:52:33.636395+0000","last_became_active":"2024-09-14T04:52:16.723597+0000","last_became_peered":"2024-09-14T04:52:16.723597+0000","last_unstale":"2024-09-14T04:52:33.636395+0000","last_undegraded":"2024-09-14T04:52:33.636395+0000","last_fullsized":"2024-09-14T04:52:33.636395+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":617,"log_dups_size":0,"ondisk_log_size":617,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:10:31.058435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40435712,"num_objects":617,"num_object_clones":0,"num_object_copies":1851,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":617,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":617,"num_write_kb":39488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.b","version":"1473'446","reported_seq":458,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:32.813350+0000","last_change":"2024-09-14T04:52:16.660625+0000","last_active":"2024-09-14T04:52:32.813350+0000","last_peered":"2024-09-14T04:52:32.813350+0000","last_clean":"2024-09-14T04:52:32.813350+0000","last_became_active":"2024-09-14T04:52:16.660277+0000","last_became_peered":"2024-09-14T04:52:16.660277+0000","last_unstale":"2024-09-14T04:52:32.813350+0000","last_undegraded":"2024-09-14T04:52:32.813350+0000","last_fullsized":"2024-09-14T04:52:32.813350+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":446,"log_dups_size":0,"ondisk_log_size":446,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:43:12.659725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29229056,"num_objects":446,"num_object_clones":0,"num_object_copies":1338,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":446,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":446,"num_write_kb":28544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"56.a","version":"1473'450","reported_seq":462,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:32.816195+0000","last_change":"2024-09-14T04:52:16.915139+0000","last_active":"2024-09-14T04:52:32.816195+0000","last_peered":"2024-09-14T04:52:32.816195+0000","last_clean":"2024-09-14T04:52:32.816195+0000","last_became_active":"2024-09-14T04:52:16.914776+0000","last_became_peered":"2024-09-14T04:52:16.914776+0000","last_unstale":"2024-09-14T04:52:32.816195+0000","last_undegraded":"2024-09-14T04:52:32.816195+0000","last_fullsized":"2024-09-14T04:52:32.816195+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":450,"log_dups_size":0,"ondisk_log_size":450,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:08:58.079681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29491200,"num_objects":450,"num_object_clones":0,"num_object_copies":1350,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":450,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":450,"num_write_kb":28800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"56.9","version":"1473'642","reported_seq":654,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:33.628811+0000","last_change":"2024-09-14T04:52:16.723789+0000","last_active":"2024-09-14T04:52:33.628811+0000","last_peered":"2024-09-14T04:52:33.628811+0000","last_clean":"2024-09-14T04:52:33.628811+0000","last_became_active":"2024-09-14T04:52:16.723394+0000","last_became_peered":"2024-09-14T04:52:16.723394+0000","last_unstale":"2024-09-14T04:52:33.628811+0000","last_undegraded":"2024-09-14T04:52:33.628811+0000","last_fullsized":"2024-09-14T04:52:33.628811+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":642,"log_dups_size":0,"ondisk_log_size":642,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:45:07.892366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42074112,"num_objects":642,"num_object_clones":0,"num_object_copies":1926,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":642,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":642,"num_write_kb":41088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.8","version":"1473'562","reported_seq":578,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:33.553773+0000","last_change":"2024-09-14T04:52:16.915103+0000","last_active":"2024-09-14T04:52:33.553773+0000","last_peered":"2024-09-14T04:52:33.553773+0000","last_clean":"2024-09-14T04:52:33.553773+0000","last_became_active":"2024-09-14T04:52:16.914657+0000","last_became_peered":"2024-09-14T04:52:16.914657+0000","last_unstale":"2024-09-14T04:52:33.553773+0000","last_undegraded":"2024-09-14T04:52:33.553773+0000","last_fullsized":"2024-09-14T04:52:33.553773+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":562,"log_dups_size":0,"ondisk_log_size":562,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:08:43.898740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36831232,"num_objects":562,"num_object_clones":0,"num_object_copies":1686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":562,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":562,"num_write_kb":35968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"56.7","version":"1473'616","reported_seq":633,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:33.557104+0000","last_change":"2024-09-14T04:52:16.723670+0000","last_active":"2024-09-14T04:52:33.557104+0000","last_peered":"2024-09-14T04:52:33.557104+0000","last_clean":"2024-09-14T04:52:33.557104+0000","last_became_active":"2024-09-14T04:52:16.723292+0000","last_became_peered":"2024-09-14T04:52:16.723292+0000","last_unstale":"2024-09-14T04:52:33.557104+0000","last_undegraded":"2024-09-14T04:52:33.557104+0000","last_fullsized":"2024-09-14T04:52:33.557104+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":616,"log_dups_size":0,"ondisk_log_size":616,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:55:51.122910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40370176,"num_objects":616,"num_object_clones":0,"num_object_copies":1848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":616,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":616,"num_write_kb":39424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"56.6","version":"1473'564","reported_seq":576,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:33.628555+0000","last_change":"2024-09-14T04:52:16.872248+0000","last_active":"2024-09-14T04:52:33.628555+0000","last_peered":"2024-09-14T04:52:33.628555+0000","last_clean":"2024-09-14T04:52:33.628555+0000","last_became_active":"2024-09-14T04:52:16.871513+0000","last_became_peered":"2024-09-14T04:52:16.871513+0000","last_unstale":"2024-09-14T04:52:33.628555+0000","last_undegraded":"2024-09-14T04:52:33.628555+0000","last_fullsized":"2024-09-14T04:52:33.628555+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":564,"log_dups_size":0,"ondisk_log_size":564,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:45:00.266398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36962304,"num_objects":564,"num_object_clones":0,"num_object_copies":1692,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":564,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":564,"num_write_kb":36096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.5","version":"1473'614","reported_seq":626,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:33.621038+0000","last_change":"2024-09-14T04:52:16.871791+0000","last_active":"2024-09-14T04:52:33.621038+0000","last_peered":"2024-09-14T04:52:33.621038+0000","last_clean":"2024-09-14T04:52:33.621038+0000","last_became_active":"2024-09-14T04:52:16.871407+0000","last_became_peered":"2024-09-14T04:52:16.871407+0000","last_unstale":"2024-09-14T04:52:33.621038+0000","last_undegraded":"2024-09-14T04:52:33.621038+0000","last_fullsized":"2024-09-14T04:52:33.621038+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":614,"log_dups_size":0,"ondisk_log_size":614,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:47:15.419674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40239104,"num_objects":614,"num_object_clones":0,"num_object_copies":1842,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":614,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":614,"num_write_kb":39296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"56.4","version":"1473'463","reported_seq":475,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:32.802829+0000","last_change":"2024-09-14T04:52:16.724077+0000","last_active":"2024-09-14T04:52:32.802829+0000","last_peered":"2024-09-14T04:52:32.802829+0000","last_clean":"2024-09-14T04:52:32.802829+0000","last_became_active":"2024-09-14T04:52:16.723597+0000","last_became_peered":"2024-09-14T04:52:16.723597+0000","last_unstale":"2024-09-14T04:52:32.802829+0000","last_undegraded":"2024-09-14T04:52:32.802829+0000","last_fullsized":"2024-09-14T04:52:32.802829+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":463,"log_dups_size":0,"ondisk_log_size":463,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:23:49.557180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30343168,"num_objects":463,"num_object_clones":0,"num_object_copies":1389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":463,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":463,"num_write_kb":29632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"56.3","version":"1473'605","reported_seq":617,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:33.621762+0000","last_change":"2024-09-14T04:52:16.915388+0000","last_active":"2024-09-14T04:52:33.621762+0000","last_peered":"2024-09-14T04:52:33.621762+0000","last_clean":"2024-09-14T04:52:33.621762+0000","last_became_active":"2024-09-14T04:52:16.915082+0000","last_became_peered":"2024-09-14T04:52:16.915082+0000","last_unstale":"2024-09-14T04:52:33.621762+0000","last_undegraded":"2024-09-14T04:52:33.621762+0000","last_fullsized":"2024-09-14T04:52:33.621762+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":605,"log_dups_size":0,"ondisk_log_size":605,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:52:13.211784+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39649280,"num_objects":605,"num_object_clones":0,"num_object_copies":1815,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":605,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":605,"num_write_kb":38720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"56.2","version":"1473'599","reported_seq":611,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:33.636768+0000","last_change":"2024-09-14T04:52:16.871874+0000","last_active":"2024-09-14T04:52:33.636768+0000","last_peered":"2024-09-14T04:52:33.636768+0000","last_clean":"2024-09-14T04:52:33.636768+0000","last_became_active":"2024-09-14T04:52:16.871509+0000","last_became_peered":"2024-09-14T04:52:16.871509+0000","last_unstale":"2024-09-14T04:52:33.636768+0000","last_undegraded":"2024-09-14T04:52:33.636768+0000","last_fullsized":"2024-09-14T04:52:33.636768+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":599,"log_dups_size":0,"ondisk_log_size":599,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:04:09.707950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39256064,"num_objects":599,"num_object_clones":0,"num_object_copies":1797,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":599,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":599,"num_write_kb":38336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.1","version":"1473'601","reported_seq":613,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:33.632014+0000","last_change":"2024-09-14T04:52:16.872703+0000","last_active":"2024-09-14T04:52:33.632014+0000","last_peered":"2024-09-14T04:52:33.632014+0000","last_clean":"2024-09-14T04:52:33.632014+0000","last_became_active":"2024-09-14T04:52:16.871779+0000","last_became_peered":"2024-09-14T04:52:16.871779+0000","last_unstale":"2024-09-14T04:52:33.632014+0000","last_undegraded":"2024-09-14T04:52:33.632014+0000","last_fullsized":"2024-09-14T04:52:33.632014+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":601,"log_dups_size":0,"ondisk_log_size":601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:23:30.260775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39387136,"num_objects":601,"num_object_clones":0,"num_object_copies":1803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":601,"num_write_kb":38464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.0","version":"1473'569","reported_seq":581,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:33.640543+0000","last_change":"2024-09-14T04:52:16.915325+0000","last_active":"2024-09-14T04:52:33.640543+0000","last_peered":"2024-09-14T04:52:33.640543+0000","last_clean":"2024-09-14T04:52:33.640543+0000","last_became_active":"2024-09-14T04:52:16.914980+0000","last_became_peered":"2024-09-14T04:52:16.914980+0000","last_unstale":"2024-09-14T04:52:33.640543+0000","last_undegraded":"2024-09-14T04:52:33.640543+0000","last_fullsized":"2024-09-14T04:52:33.640543+0000","mapping_epoch":1470,"log_start":"0'0","ondisk_log_start":"0'0","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":569,"log_dups_size":0,"ondisk_log_size":569,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:55:08.752960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37289984,"num_objects":569,"num_object_clones":0,"num_object_copies":1707,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":569,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":569,"num_write_kb":36416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":2237,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:29.485471+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T04:52:29.485471+0000","last_peered":"2024-09-14T04:52:29.485471+0000","last_clean":"2024-09-14T04:52:29.485471+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T04:52:29.485471+0000","last_undegraded":"2024-09-14T04:52:29.485471+0000","last_fullsized":"2024-09-14T04:52:29.485471+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":84,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.660271+0000","last_change":"2024-09-14T04:44:57.325109+0000","last_active":"2024-09-14T04:52:17.660271+0000","last_peered":"2024-09-14T04:52:17.660271+0000","last_clean":"2024-09-14T04:52:17.660271+0000","last_became_active":"2024-09-14T04:44:57.324791+0000","last_became_peered":"2024-09-14T04:44:57.324791+0000","last_unstale":"2024-09-14T04:52:17.660271+0000","last_undegraded":"2024-09-14T04:52:17.660271+0000","last_fullsized":"2024-09-14T04:52:17.660271+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:03.593275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":53,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.659641+0000","last_change":"2024-09-14T04:49:29.628435+0000","last_active":"2024-09-14T04:52:17.659641+0000","last_peered":"2024-09-14T04:52:17.659641+0000","last_clean":"2024-09-14T04:52:17.659641+0000","last_became_active":"2024-09-14T04:49:29.628081+0000","last_became_peered":"2024-09-14T04:49:29.628081+0000","last_unstale":"2024-09-14T04:52:17.659641+0000","last_undegraded":"2024-09-14T04:52:17.659641+0000","last_fullsized":"2024-09-14T04:52:17.659641+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:01.520334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":858,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:28.595506+0000","last_change":"2024-09-14T04:43:13.355281+0000","last_active":"2024-09-14T04:52:28.595506+0000","last_peered":"2024-09-14T04:52:28.595506+0000","last_clean":"2024-09-14T04:52:28.595506+0000","last_became_active":"2024-09-14T04:43:13.354967+0000","last_became_peered":"2024-09-14T04:43:13.354967+0000","last_unstale":"2024-09-14T04:52:28.595506+0000","last_undegraded":"2024-09-14T04:52:28.595506+0000","last_fullsized":"2024-09-14T04:52:28.595506+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:23:19.332075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1462,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.662477+0000","last_change":"2024-09-14T04:44:57.342918+0000","last_active":"2024-09-14T04:52:17.662477+0000","last_peered":"2024-09-14T04:52:17.662477+0000","last_clean":"2024-09-14T04:52:17.662477+0000","last_became_active":"2024-09-14T04:44:57.329550+0000","last_became_peered":"2024-09-14T04:44:57.329550+0000","last_unstale":"2024-09-14T04:52:17.662477+0000","last_undegraded":"2024-09-14T04:52:17.662477+0000","last_fullsized":"2024-09-14T04:52:17.662477+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:52:52.435893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":84,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:27.816876+0000","last_change":"2024-09-14T04:44:57.325537+0000","last_active":"2024-09-14T04:52:27.816876+0000","last_peered":"2024-09-14T04:52:27.816876+0000","last_clean":"2024-09-14T04:52:27.816876+0000","last_became_active":"2024-09-14T04:44:57.325057+0000","last_became_peered":"2024-09-14T04:44:57.325057+0000","last_unstale":"2024-09-14T04:52:27.816876+0000","last_undegraded":"2024-09-14T04:52:27.816876+0000","last_fullsized":"2024-09-14T04:52:27.816876+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:43:23.166708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1477,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.662517+0000","last_change":"2024-09-14T04:44:57.337033+0000","last_active":"2024-09-14T04:52:17.662517+0000","last_peered":"2024-09-14T04:52:17.662517+0000","last_clean":"2024-09-14T04:52:17.662517+0000","last_became_active":"2024-09-14T04:44:57.328437+0000","last_became_peered":"2024-09-14T04:44:57.328437+0000","last_unstale":"2024-09-14T04:52:17.662517+0000","last_undegraded":"2024-09-14T04:52:17.662517+0000","last_fullsized":"2024-09-14T04:52:17.662517+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.109086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":84,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:27.816935+0000","last_change":"2024-09-14T04:44:57.332243+0000","last_active":"2024-09-14T04:52:27.816935+0000","last_peered":"2024-09-14T04:52:27.816935+0000","last_clean":"2024-09-14T04:52:27.816935+0000","last_became_active":"2024-09-14T04:44:57.330413+0000","last_became_peered":"2024-09-14T04:44:57.330413+0000","last_unstale":"2024-09-14T04:52:27.816935+0000","last_undegraded":"2024-09-14T04:52:27.816935+0000","last_fullsized":"2024-09-14T04:52:27.816935+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:25:01.338549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":93,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:29.485100+0000","last_change":"2024-09-14T04:48:29.862929+0000","last_active":"2024-09-14T04:52:29.485100+0000","last_peered":"2024-09-14T04:52:29.485100+0000","last_clean":"2024-09-14T04:52:29.485100+0000","last_became_active":"2024-09-14T04:48:29.862581+0000","last_became_peered":"2024-09-14T04:48:29.862581+0000","last_unstale":"2024-09-14T04:52:29.485100+0000","last_undegraded":"2024-09-14T04:52:29.485100+0000","last_fullsized":"2024-09-14T04:52:29.485100+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:16:43.060088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2233,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:28.596214+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T04:52:28.596214+0000","last_peered":"2024-09-14T04:52:28.596214+0000","last_clean":"2024-09-14T04:52:28.596214+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T04:52:28.596214+0000","last_undegraded":"2024-09-14T04:52:28.596214+0000","last_fullsized":"2024-09-14T04:52:28.596214+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":84,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:27.816851+0000","last_change":"2024-09-14T04:44:57.325392+0000","last_active":"2024-09-14T04:52:27.816851+0000","last_peered":"2024-09-14T04:52:27.816851+0000","last_clean":"2024-09-14T04:52:27.816851+0000","last_became_active":"2024-09-14T04:44:57.324624+0000","last_became_peered":"2024-09-14T04:44:57.324624+0000","last_unstale":"2024-09-14T04:52:27.816851+0000","last_undegraded":"2024-09-14T04:52:27.816851+0000","last_fullsized":"2024-09-14T04:52:27.816851+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:33:18.810199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2199,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:18.010307+0000","last_change":"2024-09-14T04:44:57.329113+0000","last_active":"2024-09-14T04:52:18.010307+0000","last_peered":"2024-09-14T04:52:18.010307+0000","last_clean":"2024-09-14T04:52:18.010307+0000","last_became_active":"2024-09-14T04:44:57.328727+0000","last_became_peered":"2024-09-14T04:44:57.328727+0000","last_unstale":"2024-09-14T04:52:18.010307+0000","last_undegraded":"2024-09-14T04:52:18.010307+0000","last_fullsized":"2024-09-14T04:52:18.010307+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:34.939874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":2216,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:18.009970+0000","last_change":"2024-09-14T04:44:57.329277+0000","last_active":"2024-09-14T04:52:18.009970+0000","last_peered":"2024-09-14T04:52:18.009970+0000","last_clean":"2024-09-14T04:52:18.009970+0000","last_became_active":"2024-09-14T04:44:57.328804+0000","last_became_peered":"2024-09-14T04:44:57.328804+0000","last_unstale":"2024-09-14T04:52:18.009970+0000","last_undegraded":"2024-09-14T04:52:18.009970+0000","last_fullsized":"2024-09-14T04:52:18.009970+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:42:46.055240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2232,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:29.484950+0000","last_change":"2024-09-14T04:07:58.561490+0000","last_active":"2024-09-14T04:52:29.484950+0000","last_peered":"2024-09-14T04:52:29.484950+0000","last_clean":"2024-09-14T04:52:29.484950+0000","last_became_active":"2024-09-14T04:07:58.559841+0000","last_became_peered":"2024-09-14T04:07:58.559841+0000","last_unstale":"2024-09-14T04:52:29.484950+0000","last_undegraded":"2024-09-14T04:52:29.484950+0000","last_fullsized":"2024-09-14T04:52:29.484950+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:27:03.795368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2214,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:29.484098+0000","last_change":"2024-09-14T04:43:13.357092+0000","last_active":"2024-09-14T04:52:29.484098+0000","last_peered":"2024-09-14T04:52:29.484098+0000","last_clean":"2024-09-14T04:52:29.484098+0000","last_became_active":"2024-09-14T04:43:13.356694+0000","last_became_peered":"2024-09-14T04:43:13.356694+0000","last_unstale":"2024-09-14T04:52:29.484098+0000","last_undegraded":"2024-09-14T04:52:29.484098+0000","last_fullsized":"2024-09-14T04:52:29.484098+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:11.167574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1455,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.659583+0000","last_change":"2024-09-14T04:07:21.720797+0000","last_active":"2024-09-14T04:52:17.659583+0000","last_peered":"2024-09-14T04:52:17.659583+0000","last_clean":"2024-09-14T04:52:17.659583+0000","last_became_active":"2024-09-14T04:07:21.719331+0000","last_became_peered":"2024-09-14T04:07:21.719331+0000","last_unstale":"2024-09-14T04:52:17.659583+0000","last_undegraded":"2024-09-14T04:52:17.659583+0000","last_fullsized":"2024-09-14T04:52:17.659583+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:35.404023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":103,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:28.596277+0000","last_change":"2024-09-14T04:43:11.641514+0000","last_active":"2024-09-14T04:52:28.596277+0000","last_peered":"2024-09-14T04:52:28.596277+0000","last_clean":"2024-09-14T04:52:28.596277+0000","last_became_active":"2024-09-14T04:43:11.641151+0000","last_became_peered":"2024-09-14T04:43:11.641151+0000","last_unstale":"2024-09-14T04:52:28.596277+0000","last_undegraded":"2024-09-14T04:52:28.596277+0000","last_fullsized":"2024-09-14T04:52:28.596277+0000","mapping_epoch":1384,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:06:45.045430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2226,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:29.485551+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T04:52:29.485551+0000","last_peered":"2024-09-14T04:52:29.485551+0000","last_clean":"2024-09-14T04:52:29.485551+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T04:52:29.485551+0000","last_undegraded":"2024-09-14T04:52:29.485551+0000","last_fullsized":"2024-09-14T04:52:29.485551+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":683,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.960270+0000","last_change":"2024-09-14T04:49:29.618477+0000","last_active":"2024-09-14T04:52:17.960270+0000","last_peered":"2024-09-14T04:52:17.960270+0000","last_clean":"2024-09-14T04:52:17.960270+0000","last_became_active":"2024-09-14T04:49:29.618210+0000","last_became_peered":"2024-09-14T04:49:29.618210+0000","last_unstale":"2024-09-14T04:52:17.960270+0000","last_undegraded":"2024-09-14T04:52:17.960270+0000","last_fullsized":"2024-09-14T04:52:17.960270+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:13:23.895512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1461,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.662589+0000","last_change":"2024-09-14T04:42:49.938505+0000","last_active":"2024-09-14T04:52:17.662589+0000","last_peered":"2024-09-14T04:52:17.662589+0000","last_clean":"2024-09-14T04:52:17.662589+0000","last_became_active":"2024-09-14T04:42:49.938180+0000","last_became_peered":"2024-09-14T04:42:49.938180+0000","last_unstale":"2024-09-14T04:52:17.662589+0000","last_undegraded":"2024-09-14T04:52:17.662589+0000","last_fullsized":"2024-09-14T04:52:17.662589+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:05.929479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1480,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.960567+0000","last_change":"2024-09-14T04:44:57.346582+0000","last_active":"2024-09-14T04:52:17.960567+0000","last_peered":"2024-09-14T04:52:17.960567+0000","last_clean":"2024-09-14T04:52:17.960567+0000","last_became_active":"2024-09-14T04:44:57.330446+0000","last_became_peered":"2024-09-14T04:44:57.330446+0000","last_unstale":"2024-09-14T04:52:17.960567+0000","last_undegraded":"2024-09-14T04:52:17.960567+0000","last_fullsized":"2024-09-14T04:52:17.960567+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:29.238585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":165,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:18.010040+0000","last_change":"2024-09-14T04:44:14.480667+0000","last_active":"2024-09-14T04:52:18.010040+0000","last_peered":"2024-09-14T04:52:18.010040+0000","last_clean":"2024-09-14T04:52:18.010040+0000","last_became_active":"2024-09-14T04:44:14.480365+0000","last_became_peered":"2024-09-14T04:44:14.480365+0000","last_unstale":"2024-09-14T04:52:18.010040+0000","last_undegraded":"2024-09-14T04:52:18.010040+0000","last_fullsized":"2024-09-14T04:52:18.010040+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:00.611771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":94,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.661119+0000","last_change":"2024-09-14T04:44:57.341426+0000","last_active":"2024-09-14T04:52:17.661119+0000","last_peered":"2024-09-14T04:52:17.661119+0000","last_clean":"2024-09-14T04:52:17.661119+0000","last_became_active":"2024-09-14T04:44:57.328763+0000","last_became_peered":"2024-09-14T04:44:57.328763+0000","last_unstale":"2024-09-14T04:52:17.661119+0000","last_undegraded":"2024-09-14T04:52:17.661119+0000","last_fullsized":"2024-09-14T04:52:17.661119+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:34:40.066761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":84,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.660827+0000","last_change":"2024-09-14T04:44:57.331337+0000","last_active":"2024-09-14T04:52:17.660827+0000","last_peered":"2024-09-14T04:52:17.660827+0000","last_clean":"2024-09-14T04:52:17.660827+0000","last_became_active":"2024-09-14T04:44:57.329336+0000","last_became_peered":"2024-09-14T04:44:57.329336+0000","last_unstale":"2024-09-14T04:52:17.660827+0000","last_undegraded":"2024-09-14T04:52:17.660827+0000","last_fullsized":"2024-09-14T04:52:17.660827+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:52:35.980869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"349'149","reported_seq":672,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.662394+0000","last_change":"2024-09-14T04:49:30.085274+0000","last_active":"2024-09-14T04:52:17.662394+0000","last_peered":"2024-09-14T04:52:17.662394+0000","last_clean":"2024-09-14T04:52:17.662394+0000","last_became_active":"2024-09-14T04:49:29.623295+0000","last_became_peered":"2024-09-14T04:49:29.623295+0000","last_unstale":"2024-09-14T04:52:17.662394+0000","last_undegraded":"2024-09-14T04:52:17.662394+0000","last_fullsized":"2024-09-14T04:52:17.662394+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:49:30.085089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:49:30.085089+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:10.012963+0000","scrub_duration":0.0071928790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":84,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.660389+0000","last_change":"2024-09-14T04:44:57.329870+0000","last_active":"2024-09-14T04:52:17.660389+0000","last_peered":"2024-09-14T04:52:17.660389+0000","last_clean":"2024-09-14T04:52:17.660389+0000","last_became_active":"2024-09-14T04:44:57.329353+0000","last_became_peered":"2024-09-14T04:44:57.329353+0000","last_unstale":"2024-09-14T04:52:17.660389+0000","last_undegraded":"2024-09-14T04:52:17.660389+0000","last_fullsized":"2024-09-14T04:52:17.660389+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:46.432181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1459,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:29.484804+0000","last_change":"2024-09-14T04:07:21.721242+0000","last_active":"2024-09-14T04:52:29.484804+0000","last_peered":"2024-09-14T04:52:29.484804+0000","last_clean":"2024-09-14T04:52:29.484804+0000","last_became_active":"2024-09-14T04:07:21.720084+0000","last_became_peered":"2024-09-14T04:07:21.720084+0000","last_unstale":"2024-09-14T04:52:29.484804+0000","last_undegraded":"2024-09-14T04:52:29.484804+0000","last_fullsized":"2024-09-14T04:52:29.484804+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:58:41.308643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":2264,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:29.484833+0000","last_change":"2024-09-14T04:44:57.344614+0000","last_active":"2024-09-14T04:52:29.484833+0000","last_peered":"2024-09-14T04:52:29.484833+0000","last_clean":"2024-09-14T04:52:29.484833+0000","last_became_active":"2024-09-14T04:44:57.332256+0000","last_became_peered":"2024-09-14T04:44:57.332256+0000","last_unstale":"2024-09-14T04:52:29.484833+0000","last_undegraded":"2024-09-14T04:52:29.484833+0000","last_fullsized":"2024-09-14T04:52:29.484833+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:23.438417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"349'149","reported_seq":842,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.660966+0000","last_change":"2024-09-14T04:45:00.331854+0000","last_active":"2024-09-14T04:52:17.660966+0000","last_peered":"2024-09-14T04:52:17.660966+0000","last_clean":"2024-09-14T04:52:17.660966+0000","last_became_active":"2024-09-14T04:45:00.331460+0000","last_became_peered":"2024-09-14T04:45:00.331460+0000","last_unstale":"2024-09-14T04:52:17.660966+0000","last_undegraded":"2024-09-14T04:52:17.660966+0000","last_fullsized":"2024-09-14T04:52:17.660966+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.567535+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.567535+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:45.095729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":673,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.959980+0000","last_change":"2024-09-14T04:49:29.616831+0000","last_active":"2024-09-14T04:52:17.959980+0000","last_peered":"2024-09-14T04:52:17.959980+0000","last_clean":"2024-09-14T04:52:17.959980+0000","last_became_active":"2024-09-14T04:49:29.616546+0000","last_became_peered":"2024-09-14T04:49:29.616546+0000","last_unstale":"2024-09-14T04:52:17.959980+0000","last_undegraded":"2024-09-14T04:52:17.959980+0000","last_fullsized":"2024-09-14T04:52:17.959980+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:27.882929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":217,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:18.010449+0000","last_change":"2024-09-14T04:44:57.330746+0000","last_active":"2024-09-14T04:52:18.010449+0000","last_peered":"2024-09-14T04:52:18.010449+0000","last_clean":"2024-09-14T04:52:18.010449+0000","last_became_active":"2024-09-14T04:44:57.330327+0000","last_became_peered":"2024-09-14T04:44:57.330327+0000","last_unstale":"2024-09-14T04:52:18.010449+0000","last_undegraded":"2024-09-14T04:52:18.010449+0000","last_fullsized":"2024-09-14T04:52:18.010449+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:53.122771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":124,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.960744+0000","last_change":"2024-09-14T04:43:13.355296+0000","last_active":"2024-09-14T04:52:17.960744+0000","last_peered":"2024-09-14T04:52:17.960744+0000","last_clean":"2024-09-14T04:52:17.960744+0000","last_became_active":"2024-09-14T04:43:13.354950+0000","last_became_peered":"2024-09-14T04:43:13.354950+0000","last_unstale":"2024-09-14T04:52:17.960744+0000","last_undegraded":"2024-09-14T04:52:17.960744+0000","last_fullsized":"2024-09-14T04:52:17.960744+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:20:08.968818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"349'149","reported_seq":781,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:28.698559+0000","last_change":"2024-09-14T04:41:39.471393+0000","last_active":"2024-09-14T04:52:28.698559+0000","last_peered":"2024-09-14T04:52:28.698559+0000","last_clean":"2024-09-14T04:52:28.698559+0000","last_became_active":"2024-09-14T04:41:36.931768+0000","last_became_peered":"2024-09-14T04:41:36.931768+0000","last_unstale":"2024-09-14T04:52:28.698559+0000","last_undegraded":"2024-09-14T04:52:28.698559+0000","last_fullsized":"2024-09-14T04:52:28.698559+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.471248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.471248+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:56.210375+0000","scrub_duration":0.021758344999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2220,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:29.485348+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T04:52:29.485348+0000","last_peered":"2024-09-14T04:52:29.485348+0000","last_clean":"2024-09-14T04:52:29.485348+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T04:52:29.485348+0000","last_undegraded":"2024-09-14T04:52:29.485348+0000","last_fullsized":"2024-09-14T04:52:29.485348+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1491,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:23.623426+0000","last_change":"2024-09-14T04:44:57.331436+0000","last_active":"2024-09-14T04:52:23.623426+0000","last_peered":"2024-09-14T04:52:23.623426+0000","last_clean":"2024-09-14T04:52:23.623426+0000","last_became_active":"2024-09-14T04:44:57.330957+0000","last_became_peered":"2024-09-14T04:44:57.330957+0000","last_unstale":"2024-09-14T04:52:23.623426+0000","last_undegraded":"2024-09-14T04:52:23.623426+0000","last_fullsized":"2024-09-14T04:52:23.623426+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:09.516688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":769,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:28.596131+0000","last_change":"2024-09-14T04:49:49.460401+0000","last_active":"2024-09-14T04:52:28.596131+0000","last_peered":"2024-09-14T04:52:28.596131+0000","last_clean":"2024-09-14T04:52:28.596131+0000","last_became_active":"2024-09-14T04:49:49.458530+0000","last_became_peered":"2024-09-14T04:49:49.458530+0000","last_unstale":"2024-09-14T04:52:28.596131+0000","last_undegraded":"2024-09-14T04:52:28.596131+0000","last_fullsized":"2024-09-14T04:52:28.596131+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:11:29.143239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"349'149","reported_seq":781,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:28.698585+0000","last_change":"2024-09-14T04:41:38.479980+0000","last_active":"2024-09-14T04:52:28.698585+0000","last_peered":"2024-09-14T04:52:28.698585+0000","last_clean":"2024-09-14T04:52:28.698585+0000","last_became_active":"2024-09-14T04:41:36.933392+0000","last_became_peered":"2024-09-14T04:41:36.933392+0000","last_unstale":"2024-09-14T04:52:28.698585+0000","last_undegraded":"2024-09-14T04:52:28.698585+0000","last_fullsized":"2024-09-14T04:52:28.698585+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:38.479887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:38.479887+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:01:26.102279+0000","scrub_duration":0.0062728319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":172,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:18.009970+0000","last_change":"2024-09-14T04:44:57.329983+0000","last_active":"2024-09-14T04:52:18.009970+0000","last_peered":"2024-09-14T04:52:18.009970+0000","last_clean":"2024-09-14T04:52:18.009970+0000","last_became_active":"2024-09-14T04:44:57.329597+0000","last_became_peered":"2024-09-14T04:44:57.329597+0000","last_unstale":"2024-09-14T04:52:18.009970+0000","last_undegraded":"2024-09-14T04:52:18.009970+0000","last_fullsized":"2024-09-14T04:52:18.009970+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:45.718039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":713,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:29.484851+0000","last_change":"2024-09-14T04:49:49.443631+0000","last_active":"2024-09-14T04:52:29.484851+0000","last_peered":"2024-09-14T04:52:29.484851+0000","last_clean":"2024-09-14T04:52:29.484851+0000","last_became_active":"2024-09-14T04:49:49.443313+0000","last_became_peered":"2024-09-14T04:49:49.443313+0000","last_unstale":"2024-09-14T04:52:29.484851+0000","last_undegraded":"2024-09-14T04:52:29.484851+0000","last_fullsized":"2024-09-14T04:52:29.484851+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:06:54.562292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1463,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.662580+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T04:52:17.662580+0000","last_peered":"2024-09-14T04:52:17.662580+0000","last_clean":"2024-09-14T04:52:17.662580+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T04:52:17.662580+0000","last_undegraded":"2024-09-14T04:52:17.662580+0000","last_fullsized":"2024-09-14T04:52:17.662580+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"349'149","reported_seq":773,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.662204+0000","last_change":"2024-09-14T04:41:39.012977+0000","last_active":"2024-09-14T04:52:17.662204+0000","last_peered":"2024-09-14T04:52:17.662204+0000","last_clean":"2024-09-14T04:52:17.662204+0000","last_became_active":"2024-09-14T04:41:36.932552+0000","last_became_peered":"2024-09-14T04:41:36.932552+0000","last_unstale":"2024-09-14T04:52:17.662204+0000","last_undegraded":"2024-09-14T04:52:17.662204+0000","last_fullsized":"2024-09-14T04:52:17.662204+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.012879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.012879+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:16:40.080262+0000","scrub_duration":0.0051721620000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":805,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:28.596288+0000","last_change":"2024-09-14T04:49:49.459590+0000","last_active":"2024-09-14T04:52:28.596288+0000","last_peered":"2024-09-14T04:52:28.596288+0000","last_clean":"2024-09-14T04:52:28.596288+0000","last_became_active":"2024-09-14T04:49:49.456263+0000","last_became_peered":"2024-09-14T04:49:49.456263+0000","last_unstale":"2024-09-14T04:52:28.596288+0000","last_undegraded":"2024-09-14T04:52:28.596288+0000","last_fullsized":"2024-09-14T04:52:28.596288+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:20:44.666805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":84,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.660985+0000","last_change":"2024-09-14T04:44:57.329652+0000","last_active":"2024-09-14T04:52:17.660985+0000","last_peered":"2024-09-14T04:52:17.660985+0000","last_clean":"2024-09-14T04:52:17.660985+0000","last_became_active":"2024-09-14T04:44:57.328888+0000","last_became_peered":"2024-09-14T04:44:57.328888+0000","last_unstale":"2024-09-14T04:52:17.660985+0000","last_undegraded":"2024-09-14T04:52:17.660985+0000","last_fullsized":"2024-09-14T04:52:17.660985+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:10.170068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1462,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:18.640550+0000","last_change":"2024-09-14T04:06:46.875340+0000","last_active":"2024-09-14T04:52:18.640550+0000","last_peered":"2024-09-14T04:52:18.640550+0000","last_clean":"2024-09-14T04:52:18.640550+0000","last_became_active":"2024-09-14T04:06:46.874552+0000","last_became_peered":"2024-09-14T04:06:46.874552+0000","last_unstale":"2024-09-14T04:52:18.640550+0000","last_undegraded":"2024-09-14T04:52:18.640550+0000","last_fullsized":"2024-09-14T04:52:18.640550+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:52.257357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"349'149","reported_seq":1843,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.960009+0000","last_change":"2024-09-14T04:45:00.346627+0000","last_active":"2024-09-14T04:52:17.960009+0000","last_peered":"2024-09-14T04:52:17.960009+0000","last_clean":"2024-09-14T04:52:17.960009+0000","last_became_active":"2024-09-14T04:45:00.346286+0000","last_became_peered":"2024-09-14T04:45:00.346286+0000","last_unstale":"2024-09-14T04:52:17.960009+0000","last_undegraded":"2024-09-14T04:52:17.960009+0000","last_fullsized":"2024-09-14T04:52:17.960009+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:50.784648+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:50.784648+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:07:58.441372+0000","scrub_duration":0.01141873,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":165,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:18.010453+0000","last_change":"2024-09-14T04:44:14.489235+0000","last_active":"2024-09-14T04:52:18.010453+0000","last_peered":"2024-09-14T04:52:18.010453+0000","last_clean":"2024-09-14T04:52:18.010453+0000","last_became_active":"2024-09-14T04:44:14.488950+0000","last_became_peered":"2024-09-14T04:44:14.488950+0000","last_unstale":"2024-09-14T04:52:18.010453+0000","last_undegraded":"2024-09-14T04:52:18.010453+0000","last_fullsized":"2024-09-14T04:52:18.010453+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:54.441144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":723,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:28.595153+0000","last_change":"2024-09-14T04:49:49.456317+0000","last_active":"2024-09-14T04:52:28.595153+0000","last_peered":"2024-09-14T04:52:28.595153+0000","last_clean":"2024-09-14T04:52:28.595153+0000","last_became_active":"2024-09-14T04:49:49.456029+0000","last_became_peered":"2024-09-14T04:49:49.456029+0000","last_unstale":"2024-09-14T04:52:28.595153+0000","last_undegraded":"2024-09-14T04:52:28.595153+0000","last_fullsized":"2024-09-14T04:52:28.595153+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:46:05.319676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2240,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:28.595649+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T04:52:28.595649+0000","last_peered":"2024-09-14T04:52:28.595649+0000","last_clean":"2024-09-14T04:52:28.595649+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T04:52:28.595649+0000","last_undegraded":"2024-09-14T04:52:28.595649+0000","last_fullsized":"2024-09-14T04:52:28.595649+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"349'149","reported_seq":667,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.659360+0000","last_change":"2024-09-14T04:50:22.002798+0000","last_active":"2024-09-14T04:52:17.659360+0000","last_peered":"2024-09-14T04:52:17.659360+0000","last_clean":"2024-09-14T04:52:17.659360+0000","last_became_active":"2024-09-14T04:49:49.448278+0000","last_became_peered":"2024-09-14T04:49:49.448278+0000","last_unstale":"2024-09-14T04:52:17.659360+0000","last_undegraded":"2024-09-14T04:52:17.659360+0000","last_fullsized":"2024-09-14T04:52:17.659360+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:50:22.002568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:50:22.002568+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:01:57.489223+0000","scrub_duration":0.010888326,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2241,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:29.485038+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T04:52:29.485038+0000","last_peered":"2024-09-14T04:52:29.485038+0000","last_clean":"2024-09-14T04:52:29.485038+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T04:52:29.485038+0000","last_undegraded":"2024-09-14T04:52:29.485038+0000","last_fullsized":"2024-09-14T04:52:29.485038+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2248,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:29.484569+0000","last_change":"2024-09-14T04:44:57.335783+0000","last_active":"2024-09-14T04:52:29.484569+0000","last_peered":"2024-09-14T04:52:29.484569+0000","last_clean":"2024-09-14T04:52:29.484569+0000","last_became_active":"2024-09-14T04:44:57.332780+0000","last_became_peered":"2024-09-14T04:44:57.332780+0000","last_unstale":"2024-09-14T04:52:29.484569+0000","last_undegraded":"2024-09-14T04:52:29.484569+0000","last_fullsized":"2024-09-14T04:52:29.484569+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:57.457593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1492,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.960195+0000","last_change":"2024-09-14T04:42:49.939790+0000","last_active":"2024-09-14T04:52:17.960195+0000","last_peered":"2024-09-14T04:52:17.960195+0000","last_clean":"2024-09-14T04:52:17.960195+0000","last_became_active":"2024-09-14T04:42:49.939591+0000","last_became_peered":"2024-09-14T04:42:49.939591+0000","last_unstale":"2024-09-14T04:52:17.960195+0000","last_undegraded":"2024-09-14T04:52:17.960195+0000","last_fullsized":"2024-09-14T04:52:17.960195+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:23:16.790593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1455,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.659810+0000","last_change":"2024-09-14T04:44:57.326008+0000","last_active":"2024-09-14T04:52:17.659810+0000","last_peered":"2024-09-14T04:52:17.659810+0000","last_clean":"2024-09-14T04:52:17.659810+0000","last_became_active":"2024-09-14T04:44:57.325617+0000","last_became_peered":"2024-09-14T04:44:57.325617+0000","last_unstale":"2024-09-14T04:52:17.659810+0000","last_undegraded":"2024-09-14T04:52:17.659810+0000","last_fullsized":"2024-09-14T04:52:17.659810+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:15.637508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":744,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.662307+0000","last_change":"2024-09-14T04:43:13.354846+0000","last_active":"2024-09-14T04:52:17.662307+0000","last_peered":"2024-09-14T04:52:17.662307+0000","last_clean":"2024-09-14T04:52:17.662307+0000","last_became_active":"2024-09-14T04:43:13.354515+0000","last_became_peered":"2024-09-14T04:43:13.354515+0000","last_unstale":"2024-09-14T04:52:17.662307+0000","last_undegraded":"2024-09-14T04:52:17.662307+0000","last_fullsized":"2024-09-14T04:52:17.662307+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:17.889341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1461,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:18.640569+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T04:52:18.640569+0000","last_peered":"2024-09-14T04:52:18.640569+0000","last_clean":"2024-09-14T04:52:18.640569+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T04:52:18.640569+0000","last_undegraded":"2024-09-14T04:52:18.640569+0000","last_fullsized":"2024-09-14T04:52:18.640569+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":1798,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.959931+0000","last_change":"2024-09-14T04:41:30.640651+0000","last_active":"2024-09-14T04:52:17.959931+0000","last_peered":"2024-09-14T04:52:17.959931+0000","last_clean":"2024-09-14T04:52:17.959931+0000","last_became_active":"2024-09-14T04:41:29.532466+0000","last_became_peered":"2024-09-14T04:41:29.532466+0000","last_unstale":"2024-09-14T04:52:17.959931+0000","last_undegraded":"2024-09-14T04:52:17.959931+0000","last_fullsized":"2024-09-14T04:52:17.959931+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":0,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:30.640520+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:30.640520+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:27.776724+0000","scrub_duration":0.0028653139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"349'149","reported_seq":773,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:28.596253+0000","last_change":"2024-09-14T04:41:39.978080+0000","last_active":"2024-09-14T04:52:28.596253+0000","last_peered":"2024-09-14T04:52:28.596253+0000","last_clean":"2024-09-14T04:52:28.596253+0000","last_became_active":"2024-09-14T04:41:36.929279+0000","last_became_peered":"2024-09-14T04:41:36.929279+0000","last_unstale":"2024-09-14T04:52:28.596253+0000","last_undegraded":"2024-09-14T04:52:28.596253+0000","last_fullsized":"2024-09-14T04:52:28.596253+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.977999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.977999+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:44:06.383827+0000","scrub_duration":0.00192122,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":101,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:28.595810+0000","last_change":"2024-09-14T04:48:11.040133+0000","last_active":"2024-09-14T04:52:28.595810+0000","last_peered":"2024-09-14T04:52:28.595810+0000","last_clean":"2024-09-14T04:52:28.595810+0000","last_became_active":"2024-09-14T04:48:11.039787+0000","last_became_peered":"2024-09-14T04:48:11.039787+0000","last_unstale":"2024-09-14T04:52:28.595810+0000","last_undegraded":"2024-09-14T04:52:28.595810+0000","last_fullsized":"2024-09-14T04:52:28.595810+0000","mapping_epoch":1428,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:53.520672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2086,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:18.009936+0000","last_change":"2024-09-14T04:49:29.616443+0000","last_active":"2024-09-14T04:52:18.009936+0000","last_peered":"2024-09-14T04:52:18.009936+0000","last_clean":"2024-09-14T04:52:18.009936+0000","last_became_active":"2024-09-14T04:49:29.616066+0000","last_became_peered":"2024-09-14T04:49:29.616066+0000","last_unstale":"2024-09-14T04:52:18.009936+0000","last_undegraded":"2024-09-14T04:52:18.009936+0000","last_fullsized":"2024-09-14T04:52:18.009936+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:09:15.897957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":84,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.660966+0000","last_change":"2024-09-14T04:44:57.326724+0000","last_active":"2024-09-14T04:52:17.660966+0000","last_peered":"2024-09-14T04:52:17.660966+0000","last_clean":"2024-09-14T04:52:17.660966+0000","last_became_active":"2024-09-14T04:44:57.326191+0000","last_became_peered":"2024-09-14T04:44:57.326191+0000","last_unstale":"2024-09-14T04:52:17.660966+0000","last_undegraded":"2024-09-14T04:52:17.660966+0000","last_fullsized":"2024-09-14T04:52:17.660966+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:46.060269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":84,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.660301+0000","last_change":"2024-09-14T04:44:57.328528+0000","last_active":"2024-09-14T04:52:17.660301+0000","last_peered":"2024-09-14T04:52:17.660301+0000","last_clean":"2024-09-14T04:52:17.660301+0000","last_became_active":"2024-09-14T04:44:57.327739+0000","last_became_peered":"2024-09-14T04:44:57.327739+0000","last_unstale":"2024-09-14T04:52:17.660301+0000","last_undegraded":"2024-09-14T04:52:17.660301+0000","last_fullsized":"2024-09-14T04:52:17.660301+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:04:58.294690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"349'149","reported_seq":788,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.662064+0000","last_change":"2024-09-14T04:48:32.059177+0000","last_active":"2024-09-14T04:52:17.662064+0000","last_peered":"2024-09-14T04:52:17.662064+0000","last_clean":"2024-09-14T04:52:17.662064+0000","last_became_active":"2024-09-14T04:48:32.058741+0000","last_became_peered":"2024-09-14T04:48:32.058741+0000","last_unstale":"2024-09-14T04:52:17.662064+0000","last_undegraded":"2024-09-14T04:52:17.662064+0000","last_fullsized":"2024-09-14T04:52:17.662064+0000","mapping_epoch":1435,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1436,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:56.738636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:56.738636+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:46:20.932276+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":84,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.660848+0000","last_change":"2024-09-14T04:44:57.323436+0000","last_active":"2024-09-14T04:52:17.660848+0000","last_peered":"2024-09-14T04:52:17.660848+0000","last_clean":"2024-09-14T04:52:17.660848+0000","last_became_active":"2024-09-14T04:44:57.323022+0000","last_became_peered":"2024-09-14T04:44:57.323022+0000","last_unstale":"2024-09-14T04:52:17.660848+0000","last_undegraded":"2024-09-14T04:52:17.660848+0000","last_fullsized":"2024-09-14T04:52:17.660848+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:22.775261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1490,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.659634+0000","last_change":"2024-09-14T04:44:57.322357+0000","last_active":"2024-09-14T04:52:17.659634+0000","last_peered":"2024-09-14T04:52:17.659634+0000","last_clean":"2024-09-14T04:52:17.659634+0000","last_became_active":"2024-09-14T04:44:57.321990+0000","last_became_peered":"2024-09-14T04:44:57.321990+0000","last_unstale":"2024-09-14T04:52:17.659634+0000","last_undegraded":"2024-09-14T04:52:17.659634+0000","last_fullsized":"2024-09-14T04:52:17.659634+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:09.658008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"349'149","reported_seq":1815,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:23.623342+0000","last_change":"2024-09-14T04:41:53.858076+0000","last_active":"2024-09-14T04:52:23.623342+0000","last_peered":"2024-09-14T04:52:23.623342+0000","last_clean":"2024-09-14T04:52:23.623342+0000","last_became_active":"2024-09-14T04:41:36.931585+0000","last_became_peered":"2024-09-14T04:41:36.931585+0000","last_unstale":"2024-09-14T04:52:23.623342+0000","last_undegraded":"2024-09-14T04:52:23.623342+0000","last_fullsized":"2024-09-14T04:52:23.623342+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:53.857924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:53.857924+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:10:20.658986+0000","scrub_duration":0.0022355919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":84,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.661013+0000","last_change":"2024-09-14T04:44:57.329198+0000","last_active":"2024-09-14T04:52:17.661013+0000","last_peered":"2024-09-14T04:52:17.661013+0000","last_clean":"2024-09-14T04:52:17.661013+0000","last_became_active":"2024-09-14T04:44:57.327754+0000","last_became_peered":"2024-09-14T04:44:57.327754+0000","last_unstale":"2024-09-14T04:52:17.661013+0000","last_undegraded":"2024-09-14T04:52:17.661013+0000","last_fullsized":"2024-09-14T04:52:17.661013+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:10:27.944476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":225,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:29.485438+0000","last_change":"2024-09-14T04:44:57.348687+0000","last_active":"2024-09-14T04:52:29.485438+0000","last_peered":"2024-09-14T04:52:29.485438+0000","last_clean":"2024-09-14T04:52:29.485438+0000","last_became_active":"2024-09-14T04:44:57.332845+0000","last_became_peered":"2024-09-14T04:44:57.332845+0000","last_unstale":"2024-09-14T04:52:29.485438+0000","last_undegraded":"2024-09-14T04:52:29.485438+0000","last_fullsized":"2024-09-14T04:52:29.485438+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:12:18.762737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2247,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:29.485205+0000","last_change":"2024-09-14T04:44:57.333468+0000","last_active":"2024-09-14T04:52:29.485205+0000","last_peered":"2024-09-14T04:52:29.485205+0000","last_clean":"2024-09-14T04:52:29.485205+0000","last_became_active":"2024-09-14T04:44:57.332626+0000","last_became_peered":"2024-09-14T04:44:57.332626+0000","last_unstale":"2024-09-14T04:52:29.485205+0000","last_undegraded":"2024-09-14T04:52:29.485205+0000","last_fullsized":"2024-09-14T04:52:29.485205+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:57:26.366438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1446,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.662571+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T04:52:17.662571+0000","last_peered":"2024-09-14T04:52:17.662571+0000","last_clean":"2024-09-14T04:52:17.662571+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T04:52:17.662571+0000","last_undegraded":"2024-09-14T04:52:17.662571+0000","last_fullsized":"2024-09-14T04:52:17.662571+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1445,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.659319+0000","last_change":"2024-09-14T04:06:46.869868+0000","last_active":"2024-09-14T04:52:17.659319+0000","last_peered":"2024-09-14T04:52:17.659319+0000","last_clean":"2024-09-14T04:52:17.659319+0000","last_became_active":"2024-09-14T04:06:46.869126+0000","last_became_peered":"2024-09-14T04:06:46.869126+0000","last_unstale":"2024-09-14T04:52:17.659319+0000","last_undegraded":"2024-09-14T04:52:17.659319+0000","last_fullsized":"2024-09-14T04:52:17.659319+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:41.150138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1429,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.960644+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T04:52:17.960644+0000","last_peered":"2024-09-14T04:52:17.960644+0000","last_clean":"2024-09-14T04:52:17.960644+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T04:52:17.960644+0000","last_undegraded":"2024-09-14T04:52:17.960644+0000","last_fullsized":"2024-09-14T04:52:17.960644+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":84,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.660508+0000","last_change":"2024-09-14T04:44:57.332112+0000","last_active":"2024-09-14T04:52:17.660508+0000","last_peered":"2024-09-14T04:52:17.660508+0000","last_clean":"2024-09-14T04:52:17.660508+0000","last_became_active":"2024-09-14T04:44:57.331575+0000","last_became_peered":"2024-09-14T04:44:57.331575+0000","last_unstale":"2024-09-14T04:52:17.660508+0000","last_undegraded":"2024-09-14T04:52:17.660508+0000","last_fullsized":"2024-09-14T04:52:17.660508+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:50:00.391283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1477,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.960690+0000","last_change":"2024-09-14T04:44:57.351460+0000","last_active":"2024-09-14T04:52:17.960690+0000","last_peered":"2024-09-14T04:52:17.960690+0000","last_clean":"2024-09-14T04:52:17.960690+0000","last_became_active":"2024-09-14T04:44:57.334662+0000","last_became_peered":"2024-09-14T04:44:57.334662+0000","last_unstale":"2024-09-14T04:52:17.960690+0000","last_undegraded":"2024-09-14T04:52:17.960690+0000","last_fullsized":"2024-09-14T04:52:17.960690+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:02.399062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1431,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.662183+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T04:52:17.662183+0000","last_peered":"2024-09-14T04:52:17.662183+0000","last_clean":"2024-09-14T04:52:17.662183+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T04:52:17.662183+0000","last_undegraded":"2024-09-14T04:52:17.662183+0000","last_fullsized":"2024-09-14T04:52:17.662183+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1489,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.960691+0000","last_change":"2024-09-14T04:44:57.349690+0000","last_active":"2024-09-14T04:52:17.960691+0000","last_peered":"2024-09-14T04:52:17.960691+0000","last_clean":"2024-09-14T04:52:17.960691+0000","last_became_active":"2024-09-14T04:44:57.333579+0000","last_became_peered":"2024-09-14T04:44:57.333579+0000","last_unstale":"2024-09-14T04:52:17.960691+0000","last_undegraded":"2024-09-14T04:52:17.960691+0000","last_fullsized":"2024-09-14T04:52:17.960691+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:17:55.265141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1528,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.659445+0000","last_change":"2024-09-14T04:48:29.893179+0000","last_active":"2024-09-14T04:52:17.659445+0000","last_peered":"2024-09-14T04:52:17.659445+0000","last_clean":"2024-09-14T04:52:17.659445+0000","last_became_active":"2024-09-14T04:48:29.864724+0000","last_became_peered":"2024-09-14T04:48:29.864724+0000","last_unstale":"2024-09-14T04:52:17.659445+0000","last_undegraded":"2024-09-14T04:52:17.659445+0000","last_fullsized":"2024-09-14T04:52:17.659445+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:19:15.120740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":682,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.960654+0000","last_change":"2024-09-14T04:49:29.616761+0000","last_active":"2024-09-14T04:52:17.960654+0000","last_peered":"2024-09-14T04:52:17.960654+0000","last_clean":"2024-09-14T04:52:17.960654+0000","last_became_active":"2024-09-14T04:49:29.616072+0000","last_became_peered":"2024-09-14T04:49:29.616072+0000","last_unstale":"2024-09-14T04:52:17.960654+0000","last_undegraded":"2024-09-14T04:52:17.960654+0000","last_fullsized":"2024-09-14T04:52:17.960654+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:49:52.358949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1451,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.960373+0000","last_change":"2024-09-14T04:44:57.332511+0000","last_active":"2024-09-14T04:52:17.960373+0000","last_peered":"2024-09-14T04:52:17.960373+0000","last_clean":"2024-09-14T04:52:17.960373+0000","last_became_active":"2024-09-14T04:44:57.332166+0000","last_became_peered":"2024-09-14T04:44:57.332166+0000","last_unstale":"2024-09-14T04:52:17.960373+0000","last_undegraded":"2024-09-14T04:52:17.960373+0000","last_fullsized":"2024-09-14T04:52:17.960373+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:53.984639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2227,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:29.484827+0000","last_change":"2024-09-14T04:07:58.560880+0000","last_active":"2024-09-14T04:52:29.484827+0000","last_peered":"2024-09-14T04:52:29.484827+0000","last_clean":"2024-09-14T04:52:29.484827+0000","last_became_active":"2024-09-14T04:07:58.560132+0000","last_became_peered":"2024-09-14T04:07:58.560132+0000","last_unstale":"2024-09-14T04:52:29.484827+0000","last_undegraded":"2024-09-14T04:52:29.484827+0000","last_fullsized":"2024-09-14T04:52:29.484827+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:46:00.287050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2226,"reported_epoch":1473,"state":"active+clean","last_fresh":"2024-09-14T04:52:29.484954+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T04:52:29.484954+0000","last_peered":"2024-09-14T04:52:29.484954+0000","last_clean":"2024-09-14T04:52:29.484954+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T04:52:29.484954+0000","last_undegraded":"2024-09-14T04:52:29.484954+0000","last_fullsized":"2024-09-14T04:52:29.484954+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1443,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.662558+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T04:52:17.662558+0000","last_peered":"2024-09-14T04:52:17.662558+0000","last_clean":"2024-09-14T04:52:17.662558+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T04:52:17.662558+0000","last_undegraded":"2024-09-14T04:52:17.662558+0000","last_fullsized":"2024-09-14T04:52:17.662558+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1480,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.960368+0000","last_change":"2024-09-14T04:44:57.339993+0000","last_active":"2024-09-14T04:52:17.960368+0000","last_peered":"2024-09-14T04:52:17.960368+0000","last_clean":"2024-09-14T04:52:17.960368+0000","last_became_active":"2024-09-14T04:44:57.330472+0000","last_became_peered":"2024-09-14T04:44:57.330472+0000","last_unstale":"2024-09-14T04:52:17.960368+0000","last_undegraded":"2024-09-14T04:52:17.960368+0000","last_fullsized":"2024-09-14T04:52:17.960368+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:19:27.046988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2028,"reported_epoch":1472,"state":"active+clean","last_fresh":"2024-09-14T04:52:17.960316+0000","last_change":"2024-09-14T04:49:49.449687+0000","last_active":"2024-09-14T04:52:17.960316+0000","last_peered":"2024-09-14T04:52:17.960316+0000","last_clean":"2024-09-14T04:52:17.960316+0000","last_became_active":"2024-09-14T04:49:49.449144+0000","last_became_peered":"2024-09-14T04:49:49.449144+0000","last_unstale":"2024-09-14T04:52:17.960316+0000","last_undegraded":"2024-09-14T04:52:17.960316+0000","last_fullsized":"2024-09-14T04:52:17.960316+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:59:06.779730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":56,"num_pg":16,"stat_sum":{"num_bytes":593625088,"num_objects":9058,"num_object_clones":0,"num_object_copies":27174,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9058,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9058,"num_write_kb":579712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1272971264,"data_stored":1272971264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":9058,"ondisk_log_size":9058,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":5070,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4250,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223202019,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":803520,"kb_used_data":231032,"kb_used_omap":1,"kb_used_meta":572478,"kb_avail":92941632,"statfs":{"total":95995035648,"available":95172231168,"internally_reserved":0,"allocated":236576768,"data_stored":231645203,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":586217931},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463655,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":709048,"kb_used_data":122608,"kb_used_omap":1,"kb_used_meta":586430,"kb_avail":93036104,"statfs":{"total":95995035648,"available":95268970496,"internally_reserved":0,"allocated":125550592,"data_stored":120627092,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":600504775},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725292,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":831028,"kb_used_data":268396,"kb_used_omap":1,"kb_used_meta":562622,"kb_avail":92914124,"statfs":{"total":95995035648,"available":95144062976,"internally_reserved":0,"allocated":274837504,"data_stored":269914470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":576125382},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":1392,"seq":5978594476138,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":353344,"kb_used_data":167992,"kb_used_omap":117,"kb_used_meta":185226,"kb_avail":93391808,"statfs":{"total":95995035648,"available":95633211392,"internally_reserved":0,"allocated":172023808,"data_stored":167092795,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120638,"internal_metadata":189671618},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[5,2,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772075033,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":756592,"kb_used_data":134504,"kb_used_omap":71,"kb_used_meta":622008,"kb_avail":92988560,"statfs":{"total":95995035648,"available":95220285440,"internally_reserved":0,"allocated":137732096,"data_stored":132813211,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":636937142},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707303968,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":394868,"kb_used_data":11560,"kb_used_omap":68,"kb_used_meta":383227,"kb_avail":93350284,"statfs":{"total":95995035648,"available":95590690816,"internally_reserved":0,"allocated":11837440,"data_stored":6918459,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":392424656},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872729127,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":502816,"kb_used_data":11280,"kb_used_omap":67,"kb_used_meta":491452,"kb_avail":93242336,"statfs":{"total":95995035648,"available":95480152064,"internally_reserved":0,"allocated":11550720,"data_stored":6646007,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":503247205},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056178,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":652908,"kb_used_data":169892,"kb_used_omap":74,"kb_used_meta":482933,"kb_avail":93092244,"statfs":{"total":95995035648,"available":95326457856,"internally_reserved":0,"allocated":173969408,"data_stored":169053967,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":494524085},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1901,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":200146944,"data_stored":200146944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":181403648,"data_stored":181403648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":176357376,"data_stored":176357376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":277676032,"data_stored":277676032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":156893184,"data_stored":156893184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":280494080,"data_stored":280494080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:52:36.184 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-14T04:52:36.184 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:52:36.185 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2024-09-14T04:52:36.185 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-14T04:52:36.185 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-14T04:52:36.185 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.7 config set filestore_inject_stall 3 2024-09-14T04:52:36.213 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:36.246 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:36.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:36.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:36.348 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:36.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:36.386 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:36.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:36.488 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:36.522 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:36.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:36.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:36.540 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:52:36.624 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:36.663 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:36.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:36.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:36.765 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:36.804 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:36.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:36.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:36.906 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:36.945 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:36.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:36.947 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:37.006 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 6 16 19715 19699 205.166 167.312 0.00523606 0.00486802 2024-09-14T04:52:37.007 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:36 smithi039 ceph-mon[106470]: pgmap v2732: 99 pgs: 99 active+clean; 567 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s 2024-09-14T04:52:37.008 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:36 smithi039 ceph-mon[111235]: pgmap v2732: 99 pgs: 99 active+clean; 567 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s 2024-09-14T04:52:37.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:36 smithi129 ceph-mon[101316]: pgmap v2732: 99 pgs: 99 active+clean; 567 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s 2024-09-14T04:52:37.048 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:37.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:37.086 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:37.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:37.187 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:37.222 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:37.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:37.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:37.228 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:52:37.303 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:52:37.324 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:37.366 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:37.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:37.368 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:37.468 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:37.507 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:37.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:37.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:37.609 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:37.644 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:37.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:37.645 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:37.746 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:37.780 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:37.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:37.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:37.882 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:37.918 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:37.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:37.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:38.007 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 7 16 22395 22379 199.783 167.5 0.00692228 0.00499951 2024-09-14T04:52:38.020 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:38.063 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:38.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:38.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:38.165 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:38.202 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:38.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:38.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:38.305 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:38.343 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:38.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:38.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:38.445 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:38.483 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:38.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:38.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:38.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:38 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:52:38.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:38 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:52:38.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:38 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:52:38.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:38 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:52:38.578 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:38 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:52:38.578 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:38 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:52:38.585 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:38.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:38.629 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:38.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:38.650 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:52:38.650 INFO:teuthology.orchestra.run.smithi129.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-14T04:52:38.651 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:52:38.731 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:38.768 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:38.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:38.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:38.870 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:38.907 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:38.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:38.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:39.007 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 8 16 24879 24863 194.215 155.25 0.00330603 0.00511962 2024-09-14T04:52:39.009 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:39.030 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:39.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:39.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:39.039 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:52:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:52:38] "GET /metrics HTTP/1.1" 200 36719 "" "Prometheus/2.51.0" 2024-09-14T04:52:39.132 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:39.167 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:39.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:39.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:39.269 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:39.308 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:39.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:39.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:39.410 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:39.441 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:39.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:39.443 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:39.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:39 smithi039 ceph-mon[106470]: pgmap v2733: 99 pgs: 99 active+clean; 567 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s 2024-09-14T04:52:39.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:39 smithi039 ceph-mon[111235]: pgmap v2733: 99 pgs: 99 active+clean; 567 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s 2024-09-14T04:52:39.543 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:39.579 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:39.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:39.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:39.681 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:39.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:39.723 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:39.723 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:39.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:39 smithi129 ceph-mon[101316]: pgmap v2733: 99 pgs: 99 active+clean; 567 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 57 MiB/s wr, 905 op/s 2024-09-14T04:52:39.824 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:39.859 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:39.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:39.860 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:39.961 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:39.992 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:39.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:39.993 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:40.007 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 9 16 26913 26897 186.76 127.125 0.00822815 0.00534884 2024-09-14T04:52:40.093 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:40.118 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:40.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:40.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:40.220 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:40.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:40.245 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:40.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:40.346 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:40.385 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:40.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:40.386 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:40.487 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:40.526 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:40.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:40.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:40.628 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:40.666 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:40.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:40.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:40.768 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:40.806 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:40.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:40.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:40.908 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:40.945 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:40.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:40.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:41.008 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 10 16 28825 28809 180.032 119.5 0.0025912 0.0055297 2024-09-14T04:52:41.047 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:41.094 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:41.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:41.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:41.196 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:41.227 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:52:41.227 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [], 2024-09-14T04:52:41.227 INFO:teuthology.orchestra.run.smithi039.stdout: "complaint_time": 30, 2024-09-14T04:52:41.227 INFO:teuthology.orchestra.run.smithi039.stdout: "num_blocked_ops": 0 2024-09-14T04:52:41.227 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:52:41.239 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:41.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:41.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:41.341 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:41.373 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:41.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:41.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:41.475 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:41.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:41.517 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:41.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:41.539 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:41 smithi039 ceph-mon[106470]: pgmap v2734: 99 pgs: 99 active+clean; 1.4 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s 2024-09-14T04:52:41.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:41 smithi039 ceph-mon[111235]: pgmap v2734: 99 pgs: 99 active+clean; 1.4 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s 2024-09-14T04:52:41.602 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:41 smithi129 ceph-mon[101316]: pgmap v2734: 99 pgs: 99 active+clean; 1.4 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s 2024-09-14T04:52:41.618 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:41.657 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:41.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:41.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:41.759 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:41.799 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:41.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:41.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:41.901 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:41.940 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:41.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:41.941 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:42.007 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 11 16 30864 30848 175.25 127.438 0.00419459 0.00570121 2024-09-14T04:52:42.041 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:42.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:42.070 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:42.070 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:42.171 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:42.194 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:42.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:42.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:42.296 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:42.322 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:42.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:42.324 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:42.337 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is set 2024-09-14T04:52:42.425 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:42.458 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:42.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:42.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:42.560 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:42.583 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:42.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:42.584 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:42.685 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:42.720 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:42.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:42.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:42.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:42 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3960223149' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:52:42.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:42 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:52:42.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:42 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3960223149' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:52:42.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:42 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:52:42.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:42 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3960223149' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:52:42.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:42 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:52:42.823 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:42.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:42.861 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:42.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:42.962 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:42.988 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:42.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:42.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:43.007 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 12 16 32568 32552 169.52 106.5 0.00545099 0.00587652 2024-09-14T04:52:43.091 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:43.120 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:43.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:43.121 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:43.222 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:43.259 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:43.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:43.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:43.361 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:43.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:43.383 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:43.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:43.484 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:43.522 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:43.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:43.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:43.625 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:43.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:43.652 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:43.652 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:43.753 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:43.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:43 smithi129 ceph-mon[101316]: pgmap v2735: 99 pgs: 99 active+clean; 1.4 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2024-09-14T04:52:43.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:43 smithi129 ceph-mon[101316]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:52:43.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:43 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:52:43.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:43 smithi129 ceph-mon[101316]: osdmap e1474: 8 total, 8 up, 8 in 2024-09-14T04:52:43.788 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:43.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:43.789 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:43.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:43 smithi039 ceph-mon[106470]: pgmap v2735: 99 pgs: 99 active+clean; 1.4 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2024-09-14T04:52:43.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:43 smithi039 ceph-mon[106470]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:52:43.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:43 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:52:43.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:43 smithi039 ceph-mon[106470]: osdmap e1474: 8 total, 8 up, 8 in 2024-09-14T04:52:43.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:43 smithi039 ceph-mon[111235]: pgmap v2735: 99 pgs: 99 active+clean; 1.4 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 122 MiB/s wr, 1.95k op/s 2024-09-14T04:52:43.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:43 smithi039 ceph-mon[111235]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:52:43.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:43 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:52:43.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:43 smithi039 ceph-mon[111235]: osdmap e1474: 8 total, 8 up, 8 in 2024-09-14T04:52:43.890 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:43.926 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:43.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:43.927 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:44.007 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 13 16 34360 34344 165.095 112 0.0022689 0.00603841 2024-09-14T04:52:44.029 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:44.063 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:44.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:44.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:44.165 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:44.206 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:44.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:44.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:44.308 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:44.344 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:44.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:44.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:44.446 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:44.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:44.481 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:44.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:44.582 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:44.604 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:44.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:44.605 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:44.706 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:44.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:44.728 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:44.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:44.829 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:44.855 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:44.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:44.856 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:44.957 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:44.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:44.992 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:44.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:45.007 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 14 16 35688 35672 159.231 83 0.00496146 0.00627529 2024-09-14T04:52:45.092 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:45.128 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:45.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:45.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:45.230 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:45.268 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:45.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:45.269 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:45.369 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:45.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:45.409 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:45.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:45.510 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:45.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:45.548 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:45.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:45.649 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:45.671 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:45.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:45.672 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:45.691 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:45 smithi039 ceph-mon[106470]: pgmap v2737: 99 pgs: 99 active+clean; 2.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 151 MiB/s wr, 2.42k op/s 2024-09-14T04:52:45.691 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:45 smithi039 ceph-mon[111235]: pgmap v2737: 99 pgs: 99 active+clean; 2.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 151 MiB/s wr, 2.42k op/s 2024-09-14T04:52:45.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:45 smithi129 ceph-mon[101316]: pgmap v2737: 99 pgs: 99 active+clean; 2.0 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 151 MiB/s wr, 2.42k op/s 2024-09-14T04:52:45.773 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:45.793 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:45.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:45.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:45.895 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:45.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:45.917 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:45.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:46.007 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 15 16 37095 37079 154.477 87.9375 0.00300318 0.0064086 2024-09-14T04:52:46.019 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:46.040 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:46.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:46.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:46.142 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:46.164 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:46.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:46.165 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:46.266 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:46.301 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:46.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:46.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:46.403 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:46.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:46.436 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:46.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:46.537 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:46.573 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:46.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:46.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:46.675 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:46.713 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:46.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:46.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:46.815 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:46.837 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:46.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:46.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:46.939 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:46.975 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:46.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:46.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:47.007 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 16 16 38650 38634 150.896 97.1875 0.00689119 0.00662248 2024-09-14T04:52:47.078 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:47.113 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:47.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:47.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:47.215 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:47.254 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:47.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:47.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:47.357 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:47.394 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:47.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:47.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:47.496 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:47.533 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:47.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:47.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:47.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:47 smithi039 ceph-mon[106470]: pgmap v2738: 99 pgs: 99 active+clean; 2.0 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 151 MiB/s wr, 2.42k op/s 2024-09-14T04:52:47.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:47 smithi039 ceph-mon[111235]: pgmap v2738: 99 pgs: 99 active+clean; 2.0 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 151 MiB/s wr, 2.42k op/s 2024-09-14T04:52:47.634 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:47.675 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:47.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:47.677 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:47.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:47 smithi129 ceph-mon[101316]: pgmap v2738: 99 pgs: 99 active+clean; 2.0 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 151 MiB/s wr, 2.42k op/s 2024-09-14T04:52:47.777 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:47.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:47.815 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:47.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:47.915 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:47.953 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:47.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:47.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:48.007 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 17 16 40480 40464 148.747 114.375 0.00209288 0.00670919 2024-09-14T04:52:48.055 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:48.092 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:48.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:48.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:48.194 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:48.215 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:48.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:48.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:48.317 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:48.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:48.355 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:48.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:48.456 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:48.492 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:48.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:48.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:48.594 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:48.632 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:48.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:48.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:48.734 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:48.773 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:48.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:48.775 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:48.837 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-14T04:52:48.875 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:48.901 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:52:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:52:48] "GET /metrics HTTP/1.1" 200 36719 "" "Prometheus/2.51.0" 2024-09-14T04:52:48.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:48.923 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:48.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:49.008 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 18 16 42102 42086 146.114 101.375 0.010161 0.00683898 2024-09-14T04:52:49.023 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:49.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:49.067 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:49.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:49.168 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:49.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:49.202 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:49.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:49.303 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:49.343 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:49.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:49.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:49.445 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:49.479 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:49.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:49.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:49.581 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:49.615 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:49.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:49.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:49.670 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:49 smithi039 ceph-mon[106470]: pgmap v2739: 99 pgs: 99 active+clean; 2.0 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 151 MiB/s wr, 2.42k op/s 2024-09-14T04:52:49.671 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:49 smithi039 ceph-mon[111235]: pgmap v2739: 99 pgs: 99 active+clean; 2.0 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 151 MiB/s wr, 2.42k op/s 2024-09-14T04:52:49.720 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:49.752 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:49.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:49.753 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:49.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:49 smithi129 ceph-mon[101316]: pgmap v2739: 99 pgs: 99 active+clean; 2.0 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 151 MiB/s wr, 2.42k op/s 2024-09-14T04:52:49.854 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:49.876 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:52:49.877 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:52:49.885 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:52:49.903 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:49.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:49.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:49.921 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:52:49.921 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:52:49.956 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:52:49.956 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:52:49.992 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:52:49.992 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:52:50.006 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:50.008 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 19 16 43944 43928 144.482 115.125 0.0056212 0.00691644 2024-09-14T04:52:50.025 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:52:50.026 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:52:50.042 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:50.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:50.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:50.058 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:52:50.059 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:52:50.090 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:52:50.090 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:52:50.127 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:52:50.127 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:52:50.144 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:50.183 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:50.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:50.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:50.285 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:50.320 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:50.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:50.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:50.422 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:50.459 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:50.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:50.460 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:50.509 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:50 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2659824100' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:52:50.510 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:50 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2659824100' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:52:50.560 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:50.598 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:50.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:50.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:50.700 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:50.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:50.739 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:50.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:50.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:50 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2659824100' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:52:50.840 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:50.874 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:50.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:50.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:50.976 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:51.008 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:2024-09-14T04:52:51.008221+0000 min lat: 0.00136201 max lat: 0.343033 avg lat: 0.00697613 2024-09-14T04:52:51.008 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:52:51.008 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 20 16 45858 45842 143.239 119.625 0.00609344 0.00697613 2024-09-14T04:52:51.013 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:51.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:51.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:51.115 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:51.143 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-14T04:52:51.143 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:52:51.144 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2024-09-14T04:52:51.144 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight-by-utilization 110 1.0 2024-09-14T04:52:51.148 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:51.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:51.149 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:51.250 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:51.284 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:51.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:51.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:51.386 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:51.429 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:51.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:51.430 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:51.531 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:51.539 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:51 smithi039 ceph-mon[106470]: pgmap v2740: 99 pgs: 99 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2024-09-14T04:52:51.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:51 smithi039 ceph-mon[111235]: pgmap v2740: 99 pgs: 99 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2024-09-14T04:52:51.552 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:52:51.575 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:51.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:51.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:51.677 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:51.710 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:51.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:51.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:51.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:51 smithi129 ceph-mon[101316]: pgmap v2740: 99 pgs: 99 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2024-09-14T04:52:51.812 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:51.849 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:51.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:51.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:51.951 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:51.984 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:51.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:51.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:52.008 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 21 16 47714 47698 141.941 116 0.0100972 0.00704021 2024-09-14T04:52:52.086 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:52.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:52.129 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:52.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:52.230 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:52.263 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:52.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:52.264 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:52.365 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:52.402 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:52.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:52.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:52.504 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:52.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:52.541 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:52.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:52.642 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:52.690 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:52.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:52.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:52.715 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:52:52.715 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:52:52.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:52:52.793 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:52.830 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:52.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:52.831 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:52.932 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:52.968 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:52.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:52.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:53.008 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 22 16 49547 49531 140.696 114.562 0.00636374 0.00710244 2024-09-14T04:52:53.070 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:53.106 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:53.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:53.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:53.204 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:52:53.208 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:53.213 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:52:53.230 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:52:53.231 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:52:53.236 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:53.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:53.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:53.339 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:53.376 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:53.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:53.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:53.478 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:53.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:53.512 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:53.512 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:53.613 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:53.615 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:53 smithi039 ceph-mon[106470]: pgmap v2741: 99 pgs: 99 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2024-09-14T04:52:53.615 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:53 smithi039 ceph-mon[111235]: pgmap v2741: 99 pgs: 99 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2024-09-14T04:52:53.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:53.659 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:53.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:53.760 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:53.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:53 smithi129 ceph-mon[101316]: pgmap v2741: 99 pgs: 99 active+clean; 2.5 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 110 MiB/s wr, 1.76k op/s 2024-09-14T04:52:53.804 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:53.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:53.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:53.906 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:53.940 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:53.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:53.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:54.008 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 23 16 51368 51352 139.527 113.812 0.00533866 0.0071615 2024-09-14T04:52:54.043 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:54.078 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:54.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:54.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:54.180 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:54.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:54.221 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:54.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:54.323 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:54.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:54.361 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:54.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:54.462 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:54.590 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:54.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:54.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:54.597 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:52:54.692 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:54.737 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:54.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:54.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:54.839 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:54.873 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:54.874 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:54.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:54.975 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:55.001 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:55.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:55.002 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:55.009 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 24 16 53080 53064 138.172 107 0.00284329 0.00722716 2024-09-14T04:52:55.103 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:55.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:55.142 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:55.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:55.243 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:55.278 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:55.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:55.280 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:55.381 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:55.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:55.420 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:55.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:55.521 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:55.558 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:55.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:55.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:55.641 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:55 smithi039 ceph-mon[106470]: pgmap v2742: 99 pgs: 99 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 146 MiB/s wr, 2.34k op/s 2024-09-14T04:52:55.641 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:55 smithi039 ceph-mon[111235]: pgmap v2742: 99 pgs: 99 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 146 MiB/s wr, 2.34k op/s 2024-09-14T04:52:55.660 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:55.693 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:55.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:55.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:55.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:55 smithi129 ceph-mon[101316]: pgmap v2742: 99 pgs: 99 active+clean; 3.1 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 146 MiB/s wr, 2.34k op/s 2024-09-14T04:52:55.795 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:55.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:55.830 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:55.830 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:55.931 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:55.957 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:55.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:55.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:56.008 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 25 16 54701 54685 136.697 101.312 0.0110175 0.00731089 2024-09-14T04:52:56.060 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:56.095 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:56.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:56.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:56.197 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:56.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:56.231 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:56.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:56.332 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:56.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:56.375 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:56.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:56.476 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:56.512 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:56.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:56.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:56.614 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:52:56.650 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:56.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:56.651 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:52:56.751 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:56.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:56.796 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:56.796 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:56.897 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:56.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:56.932 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:56.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:57.008 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 26 16 56289 56273 135.256 99.25 0.00324153 0.00736827 2024-09-14T04:52:57.034 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:57.055 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:57.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:57.056 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:57.157 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:57.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:57.181 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:57.181 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:57.282 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:57.318 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:57.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:57.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:57.421 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:57.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:57.461 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:57.461 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:57.562 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:57.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:57.600 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:57.601 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:57.686 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:57 smithi039 ceph-mon[106470]: pgmap v2743: 99 pgs: 99 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-14T04:52:57.686 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:57 smithi039 ceph-mon[111235]: pgmap v2743: 99 pgs: 99 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-14T04:52:57.701 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:52:57.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:57.737 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:57.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:52:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:57 smithi129 ceph-mon[101316]: pgmap v2743: 99 pgs: 99 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-14T04:52:57.839 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:57.878 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:57.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:57.879 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:57.979 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:58.008 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 27 16 57605 57589 133.293 82.25 0.0073315 0.00749691 2024-09-14T04:52:58.017 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:58.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:58.018 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:58.119 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:58.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:58.142 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:58.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:58.243 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:52:58.280 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:58.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:58.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:52:58.382 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:58.404 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:58 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'3':'57736','5':'56377','7':'56762'}"} : dispatch 2024-09-14T04:52:58.404 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:58 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'3':'57736','5':'56377','7':'56762'}"} : dispatch 2024-09-14T04:52:58.412 INFO:teuthology.orchestra.run.smithi039.stdout:moved 14 / 297 (4.7138%) 2024-09-14T04:52:58.412 INFO:teuthology.orchestra.run.smithi039.stdout:avg 37.125 2024-09-14T04:52:58.412 INFO:teuthology.orchestra.run.smithi039.stdout:stddev 3.68909 -> 2.8477 (expected baseline 5.69951) 2024-09-14T04:52:58.412 INFO:teuthology.orchestra.run.smithi039.stdout:min osd.2 with 31 -> 34 pgs (0.835017 -> 0.915825 * mean) 2024-09-14T04:52:58.412 INFO:teuthology.orchestra.run.smithi039.stdout:max osd.4 with 42 -> 43 pgs (1.13131 -> 1.15825 * mean) 2024-09-14T04:52:58.413 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:52:58.413 INFO:teuthology.orchestra.run.smithi039.stdout:oload 110 2024-09-14T04:52:58.413 INFO:teuthology.orchestra.run.smithi039.stdout:max_change 1 2024-09-14T04:52:58.413 INFO:teuthology.orchestra.run.smithi039.stdout:max_change_osds 4 2024-09-14T04:52:58.413 INFO:teuthology.orchestra.run.smithi039.stdout:average_utilization 0.0186 2024-09-14T04:52:58.413 INFO:teuthology.orchestra.run.smithi039.stdout:overload_utilization 0.0205 2024-09-14T04:52:58.413 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5 weight 1.0000 -> 0.8602 2024-09-14T04:52:58.413 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7 weight 1.0000 -> 0.8661 2024-09-14T04:52:58.413 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3 weight 1.0000 -> 0.8810 2024-09-14T04:52:58.418 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:58.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:58.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:58.520 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:58.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:58.544 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:58.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:58.645 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:58.667 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:58.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:58.669 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:58.770 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:58 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'3':'57736','5':'56377','7':'56762'}"} : dispatch 2024-09-14T04:52:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:58 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'3':'57736','5':'56377','7':'56762'}"} : dispatch 2024-09-14T04:52:58.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:58 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'3':'57736','5':'56377','7':'56762'}"} : dispatch 2024-09-14T04:52:58.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:58 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd reweightn", "weights": "{'3':'57736','5':'56377','7':'56762'}"} : dispatch 2024-09-14T04:52:58.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:52:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:52:58] "GET /metrics HTTP/1.1" 200 36729 "" "Prometheus/2.51.0" 2024-09-14T04:52:58.793 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:58.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:58.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:58.895 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:58.917 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:58.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:58.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:59.009 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 28 16 58124 58108 129.691 32.4375 0.00253155 0.00754916 2024-09-14T04:52:59.019 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:59.043 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:59.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:59.044 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:59.145 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:52:59.167 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:59.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:59.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:52:59.222 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2024-09-14T04:52:59.223 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd test-reweight-by-utilization 110 3.0 2024-09-14T04:52:59.269 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:52:59.292 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:59.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:59.293 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:52:59.394 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:52:59.424 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:52:59.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:59.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:52:59.526 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:59.539 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:59 smithi039 ceph-mon[106470]: pgmap v2744: 99 pgs: 99 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-14T04:52:59.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:59 smithi039 ceph-mon[106470]: from='client.36030 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:52:59.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'3':'57736','5':'56377','7':'56762'}"}]': finished 2024-09-14T04:52:59.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:52:59 smithi039 ceph-mon[106470]: osdmap e1475: 8 total, 8 up, 8 in 2024-09-14T04:52:59.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:59 smithi039 ceph-mon[111235]: pgmap v2744: 99 pgs: 99 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-14T04:52:59.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:59 smithi039 ceph-mon[111235]: from='client.36030 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:52:59.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'3':'57736','5':'56377','7':'56762'}"}]': finished 2024-09-14T04:52:59.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:52:59 smithi039 ceph-mon[111235]: osdmap e1475: 8 total, 8 up, 8 in 2024-09-14T04:52:59.549 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:59.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:59.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:59.654 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:59.678 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:59.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:59.679 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:52:59.681 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-14T04:52:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:59 smithi129 ceph-mon[101316]: pgmap v2744: 99 pgs: 99 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 89 MiB/s wr, 1.42k op/s 2024-09-14T04:52:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:59 smithi129 ceph-mon[101316]: from='client.36030 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:52:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'3':'57736','5':'56377','7':'56762'}"}]': finished 2024-09-14T04:52:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:52:59 smithi129 ceph-mon[101316]: osdmap e1475: 8 total, 8 up, 8 in 2024-09-14T04:52:59.780 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:52:59.803 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:59.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:59.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:52:59.905 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:52:59.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:52:59.930 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:52:59.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:00.009 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 29 16 58124 58108 125.218 0 - 0.00754916 2024-09-14T04:53:00.031 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:00.054 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:00.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:00.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:00.155 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:00.182 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:00.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:00.183 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:00.232 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:53:00.232 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:53:00.284 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:00.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:00.308 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:00.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:00.409 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:00.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:00.437 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:00.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:00.538 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:00.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:00.569 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:00.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:00.670 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:00.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:00.705 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:00.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:00.743 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:00 smithi039 ceph-mon[106470]: osdmap e1476: 8 total, 8 up, 8 in 2024-09-14T04:53:00.743 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:00 smithi039 ceph-mon[106470]: osdmap e1477: 8 total, 8 up, 8 in 2024-09-14T04:53:00.743 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:00 smithi039 ceph-mon[111235]: osdmap e1476: 8 total, 8 up, 8 in 2024-09-14T04:53:00.743 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:00 smithi039 ceph-mon[111235]: osdmap e1477: 8 total, 8 up, 8 in 2024-09-14T04:53:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:00 smithi129 ceph-mon[101316]: osdmap e1476: 8 total, 8 up, 8 in 2024-09-14T04:53:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:00 smithi129 ceph-mon[101316]: osdmap e1477: 8 total, 8 up, 8 in 2024-09-14T04:53:00.806 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:00.835 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:00.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:00.836 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:00.937 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:00.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:00.974 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:00.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:01.009 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 30 16 58571 58555 121.976 13.9688 0.0393202 0.00819241 2024-09-14T04:53:01.075 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:01.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:01.109 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:01.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:01.210 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:01.246 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:01.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:01.247 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:01.348 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:01.383 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:01.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:01.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:01.484 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:01.518 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:01.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:01.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:01.620 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:01.650 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:01.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:01.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:01.752 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:01 smithi129 ceph-mon[101316]: pgmap v2747: 99 pgs: 8 peering, 1 remapped+peering, 2 unknown, 88 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 9.4 KiB/s rd, 131 MiB/s wr, 2.11k op/s; 20 KiB/s, 0 objects/s recovering 2024-09-14T04:53:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:01 smithi129 ceph-mon[101316]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:53:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:01 smithi129 ceph-mon[101316]: osdmap e1478: 8 total, 8 up, 8 in 2024-09-14T04:53:01.779 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:01.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:01.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:01.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:01 smithi039 ceph-mon[106470]: pgmap v2747: 99 pgs: 8 peering, 1 remapped+peering, 2 unknown, 88 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 9.4 KiB/s rd, 131 MiB/s wr, 2.11k op/s; 20 KiB/s, 0 objects/s recovering 2024-09-14T04:53:01.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:01 smithi039 ceph-mon[106470]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:53:01.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:01 smithi039 ceph-mon[106470]: osdmap e1478: 8 total, 8 up, 8 in 2024-09-14T04:53:01.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:01 smithi039 ceph-mon[111235]: pgmap v2747: 99 pgs: 8 peering, 1 remapped+peering, 2 unknown, 88 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 9.4 KiB/s rd, 131 MiB/s wr, 2.11k op/s; 20 KiB/s, 0 objects/s recovering 2024-09-14T04:53:01.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:01 smithi039 ceph-mon[111235]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-14T04:53:01.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:01 smithi039 ceph-mon[111235]: osdmap e1478: 8 total, 8 up, 8 in 2024-09-14T04:53:01.881 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:01.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:01.916 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:01.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:02.009 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 31 16 59620 59604 120.156 65.5625 0.0165326 0.00831615 2024-09-14T04:53:02.018 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:02.050 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:02.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:02.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:02.152 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:02.186 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:02.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:02.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:02.288 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:02.323 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:02.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:02.324 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:02.425 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:02.459 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:02.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:02.460 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:02.561 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:02.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:02.604 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:02.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:02.705 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:02.736 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:02.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:02.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:02.839 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:02.887 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:02.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:02.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:02.989 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:03.009 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 32 16 60786 60770 118.678 72.875 0.00433714 0.00841952 2024-09-14T04:53:03.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:03.030 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:03.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:03.131 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:03.167 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:03.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:03.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:03.269 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:03.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:03 smithi039 ceph-mon[106470]: pgmap v2750: 99 pgs: 8 peering, 1 remapped+peering, 2 unknown, 88 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 19 KiB/s rd, 118 MiB/s wr, 1.90k op/s; 39 KiB/s, 0 objects/s recovering 2024-09-14T04:53:03.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:03 smithi039 ceph-mon[106470]: osdmap e1479: 8 total, 8 up, 8 in 2024-09-14T04:53:03.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:03 smithi039 ceph-mon[106470]: 1.2 scrub starts 2024-09-14T04:53:03.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:03 smithi039 ceph-mon[111235]: pgmap v2750: 99 pgs: 8 peering, 1 remapped+peering, 2 unknown, 88 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 19 KiB/s rd, 118 MiB/s wr, 1.90k op/s; 39 KiB/s, 0 objects/s recovering 2024-09-14T04:53:03.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:03 smithi039 ceph-mon[111235]: osdmap e1479: 8 total, 8 up, 8 in 2024-09-14T04:53:03.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:03 smithi039 ceph-mon[111235]: 1.2 scrub starts 2024-09-14T04:53:03.308 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:03.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:03.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:03.410 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:03.440 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:03.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:03.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:03.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:03 smithi129 ceph-mon[101316]: pgmap v2750: 99 pgs: 8 peering, 1 remapped+peering, 2 unknown, 88 active+clean; 3.5 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 19 KiB/s rd, 118 MiB/s wr, 1.90k op/s; 39 KiB/s, 0 objects/s recovering 2024-09-14T04:53:03.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:03 smithi129 ceph-mon[101316]: osdmap e1479: 8 total, 8 up, 8 in 2024-09-14T04:53:03.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:03 smithi129 ceph-mon[101316]: 1.2 scrub starts 2024-09-14T04:53:03.541 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:03.569 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:03.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:03.570 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:03.671 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:03.708 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:03.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:03.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:03.810 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:03.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:03.851 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:03.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:03.952 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:03.987 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:03.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:03.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:04.009 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 33 16 61953 61937 117.292 72.9375 0.00743535 0.00852017 2024-09-14T04:53:04.089 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:04.120 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:04.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:04.122 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:04.123 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is unset 2024-09-14T04:53:04.168 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:04 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/512801646' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:53:04.168 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:04 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/512801646' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:53:04.224 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:04.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:04.262 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:04.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:04.364 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:04.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:04.403 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:04.403 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:04.504 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:04.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:04 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/512801646' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:53:04.539 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:04.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:04.540 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:04.630 INFO:teuthology.orchestra.run.smithi039.stdout:moved 17 / 297 (5.72391%) 2024-09-14T04:53:04.630 INFO:teuthology.orchestra.run.smithi039.stdout:avg 37.125 2024-09-14T04:53:04.630 INFO:teuthology.orchestra.run.smithi039.stdout:stddev 2.8477 -> 6.27371 (expected baseline 5.69951) 2024-09-14T04:53:04.630 INFO:teuthology.orchestra.run.smithi039.stdout:min osd.2 with 34 -> 35 pgs (0.915825 -> 0.942761 * mean) 2024-09-14T04:53:04.630 INFO:teuthology.orchestra.run.smithi039.stdout:max osd.4 with 43 -> 44 pgs (1.15825 -> 1.18519 * mean) 2024-09-14T04:53:04.630 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:53:04.630 INFO:teuthology.orchestra.run.smithi039.stdout:oload 110 2024-09-14T04:53:04.630 INFO:teuthology.orchestra.run.smithi039.stdout:max_change 3 2024-09-14T04:53:04.630 INFO:teuthology.orchestra.run.smithi039.stdout:max_change_osds 4 2024-09-14T04:53:04.630 INFO:teuthology.orchestra.run.smithi039.stdout:average_utilization 0.0210 2024-09-14T04:53:04.631 INFO:teuthology.orchestra.run.smithi039.stdout:overload_utilization 0.0231 2024-09-14T04:53:04.631 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5 weight 0.8602 -> 0.7182 2024-09-14T04:53:04.631 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7 weight 0.8661 -> 0.7362 2024-09-14T04:53:04.631 INFO:teuthology.orchestra.run.smithi039.stderr:no change 2024-09-14T04:53:04.641 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:04.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:04.680 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:04.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:04.705 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:53:04 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T04:53:04.628+0000 7f268b1da640 -1 mgr.server reply reply (2) No such file or directory no change 2024-09-14T04:53:04.781 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:04.814 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:04.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:04.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:04.917 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:04.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:04.956 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:04.956 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:05.009 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 34 16 63122 63106 115.991 73.0625 0.00350274 0.0086149 2024-09-14T04:53:05.057 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:05.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:05.095 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:05.095 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:05.196 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:05.225 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:05.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:05.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:05.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:05 smithi039 ceph-mon[106470]: pgmap v2752: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 1 remapped+peering, 94 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 49 MiB/s wr, 780 op/s; 3571/184020 objects degraded (1.941%); 7143/184020 objects misplaced (3.882%); 2.4 MiB/s, 38 objects/s recovering 2024-09-14T04:53:05.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:05 smithi039 ceph-mon[106470]: Health check failed: Degraded data redundancy: 3571/184020 objects degraded (1.941%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:05.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:05 smithi039 ceph-mon[106470]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:53:05.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:05 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/512801646' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:53:05.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:05 smithi039 ceph-mon[106470]: osdmap e1480: 8 total, 8 up, 8 in 2024-09-14T04:53:05.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:05 smithi039 ceph-mon[106470]: from='client.45980 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:53:05.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:05 smithi039 ceph-mon[111235]: pgmap v2752: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 1 remapped+peering, 94 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 49 MiB/s wr, 780 op/s; 3571/184020 objects degraded (1.941%); 7143/184020 objects misplaced (3.882%); 2.4 MiB/s, 38 objects/s recovering 2024-09-14T04:53:05.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:05 smithi039 ceph-mon[111235]: Health check failed: Degraded data redundancy: 3571/184020 objects degraded (1.941%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:05.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:05 smithi039 ceph-mon[111235]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:53:05.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:05 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/512801646' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:53:05.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:05 smithi039 ceph-mon[111235]: osdmap e1480: 8 total, 8 up, 8 in 2024-09-14T04:53:05.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:05 smithi039 ceph-mon[111235]: from='client.45980 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:53:05.327 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:05.364 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:05.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:05.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:05.466 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:05.502 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:05.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:05.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:05.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:05 smithi129 ceph-mon[101316]: pgmap v2752: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 1 remapped+peering, 94 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 49 MiB/s wr, 780 op/s; 3571/184020 objects degraded (1.941%); 7143/184020 objects misplaced (3.882%); 2.4 MiB/s, 38 objects/s recovering 2024-09-14T04:53:05.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:05 smithi129 ceph-mon[101316]: Health check failed: Degraded data redundancy: 3571/184020 objects degraded (1.941%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:05.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:05 smithi129 ceph-mon[101316]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:53:05.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:05 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/512801646' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:53:05.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:05 smithi129 ceph-mon[101316]: osdmap e1480: 8 total, 8 up, 8 in 2024-09-14T04:53:05.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:05 smithi129 ceph-mon[101316]: from='client.45980 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:53:05.604 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:05.634 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:05.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:05.635 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:05.736 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:05.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:05.779 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:05.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:05.880 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:05.918 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:05.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:05.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:05.923 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:53:05.956 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2024-09-14T04:53:05.957 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd test-reweight-by-utilization 1000 1.0 2024-09-14T04:53:06.009 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 35 16 64254 64238 114.698 70.75 0.0155412 0.00871371 2024-09-14T04:53:06.021 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:06.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:06.062 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:06.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:06.163 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:06.195 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:06 smithi039 ceph-mon[106470]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-14T04:53:06.195 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:06 smithi039 ceph-mon[111235]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-14T04:53:06.203 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:06.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:06.204 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:06.304 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:06.338 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:06.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:06.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:06.440 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:06.483 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:06.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:06.484 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:06.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:06 smithi129 ceph-mon[101316]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-14T04:53:06.585 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:06.617 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:06.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:06.618 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:06.718 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:06.757 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:06.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:06.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:06.859 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:53:06.859 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:53:06.860 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:53:06.860 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:53:06.860 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:53:06.860 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:53:06.860 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:53:06.860 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:53:06.860 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:53:06.860 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:53:06.860 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:53:06.861 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:53:06.861 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:53:06.861 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:53:06.861 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:53:06.861 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:53:06.861 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:06.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:06.898 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:06.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:07.000 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:07.009 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 36 16 65414 65398 113.526 72.5 0.0152394 0.00880201 2024-09-14T04:53:07.035 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:07.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:07.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:07.137 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:07.179 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:07.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:07.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:07.281 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:07.317 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:07.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:07.318 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:07.419 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:07.453 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:07.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:07.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:07.477 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:53:07.507 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:07 smithi039 ceph-mon[106470]: pgmap v2754: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 95 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 40 MiB/s wr, 635 op/s; 3571/184020 objects degraded (1.941%); 7143/184020 objects misplaced (3.882%); 2.0 MiB/s, 31 objects/s recovering 2024-09-14T04:53:07.507 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:07 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2010485065' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:53:07.508 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:07 smithi039 ceph-mon[111235]: pgmap v2754: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 95 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 40 MiB/s wr, 635 op/s; 3571/184020 objects degraded (1.941%); 7143/184020 objects misplaced (3.882%); 2.0 MiB/s, 31 objects/s recovering 2024-09-14T04:53:07.508 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:07 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2010485065' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:53:07.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:07 smithi129 ceph-mon[101316]: pgmap v2754: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 95 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 40 MiB/s wr, 635 op/s; 3571/184020 objects degraded (1.941%); 7143/184020 objects misplaced (3.882%); 2.0 MiB/s, 31 objects/s recovering 2024-09-14T04:53:07.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:07 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2010485065' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:53:07.555 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:07.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:07.604 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:07.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:07.705 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:07.736 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:07.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:07.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:07.839 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:07.878 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:07.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:07.879 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:07.979 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:08.009 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 37 16 66570 66554 112.41 72.25 0.0213243 0.00889074 2024-09-14T04:53:08.014 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:08.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:08.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:08.116 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:08.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:08.169 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:08.169 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:08.171 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:53:08.270 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:08.302 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:08.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:08.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:08.311 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:53:08.404 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:08.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:08.442 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:08.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:08.468 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:53:08.543 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:08.576 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:08.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:08.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:08.591 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:08 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:53:08.592 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:08 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:53:08.592 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:08 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:53:08.678 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:08.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:08.718 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:08.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:08.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:08 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:53:08.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:08 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:53:08.818 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:08.857 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:08.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:08.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:08.903 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:53:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:53:08] "GET /metrics HTTP/1.1" 200 36811 "" "Prometheus/2.51.0" 2024-09-14T04:53:08.951 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:53:08.951 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [], 2024-09-14T04:53:08.951 INFO:teuthology.orchestra.run.smithi039.stdout: "num_ops": 0 2024-09-14T04:53:08.951 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:53:08.958 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:08.991 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:08.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:08.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:09.010 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 38 16 67702 67686 111.313 70.75 0.0450558 0.00897739 2024-09-14T04:53:09.092 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:09.121 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:09.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:09.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:09.223 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:09.276 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:09.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:09.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:09.378 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:09.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:09.415 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:09.416 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:09.516 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:09.517 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:09 smithi039 ceph-mon[106470]: pgmap v2755: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 95 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 32 MiB/s wr, 510 op/s; 3571/184020 objects degraded (1.941%); 7143/184020 objects misplaced (3.882%); 1.6 MiB/s, 25 objects/s recovering 2024-09-14T04:53:09.518 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:09 smithi039 ceph-mon[111235]: pgmap v2755: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 95 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 32 MiB/s wr, 510 op/s; 3571/184020 objects degraded (1.941%); 7143/184020 objects misplaced (3.882%); 1.6 MiB/s, 25 objects/s recovering 2024-09-14T04:53:09.556 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:09.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:09.557 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:09.610 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:53:09.658 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:09.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:09.705 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:09.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:09 smithi129 ceph-mon[101316]: pgmap v2755: 99 pgs: 2 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 95 active+clean; 3.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 32 MiB/s wr, 510 op/s; 3571/184020 objects degraded (1.941%); 7143/184020 objects misplaced (3.882%); 1.6 MiB/s, 25 objects/s recovering 2024-09-14T04:53:09.806 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:09.843 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:09.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:09.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:09.945 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:09.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:09.978 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:09.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:10.010 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 39 16 68859 68843 110.313 72.3125 0.0359933 0.00905978 2024-09-14T04:53:10.079 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:10.109 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:10.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:10.111 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:10.212 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:10.243 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:10.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:10.244 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:10.345 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:10.382 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:10.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:10.383 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:10.484 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:10.525 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:10.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:10.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:10.627 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:10.652 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:10 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 3571/201321 objects degraded (1.774%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:10.652 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:10 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 3571/201321 objects degraded (1.774%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:10.657 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:10.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:10.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:10.760 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:10.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:10 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 3571/201321 objects degraded (1.774%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:10.792 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:10.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:10.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:10.894 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:10.931 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:10.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:10.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:11.010 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:2024-09-14T04:53:11.010145+0000 min lat: 0.00136201 max lat: 2.17883 avg lat: 0.00914428 2024-09-14T04:53:11.010 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:53:11.010 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 40 16 69955 69939 109.268 68.5 0.00659492 0.00914428 2024-09-14T04:53:11.032 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:11.072 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:11.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:11.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:11.174 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:11.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:11.210 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:11.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:11.311 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:11.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:11.351 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:11.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:11.452 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:11.466 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-14T04:53:11.495 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:11.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:11.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:11.596 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:11.637 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:11.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:11.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:11.709 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:11 smithi039 ceph-mon[111235]: pgmap v2756: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 3571/201321 objects degraded (1.774%); 6861/201321 objects misplaced (3.408%); 3.5 MiB/s, 56 objects/s recovering 2024-09-14T04:53:11.709 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:11 smithi039 ceph-mon[106470]: pgmap v2756: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 3571/201321 objects degraded (1.774%); 6861/201321 objects misplaced (3.408%); 3.5 MiB/s, 56 objects/s recovering 2024-09-14T04:53:11.740 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:11.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:11 smithi129 ceph-mon[101316]: pgmap v2756: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s; 3571/201321 objects degraded (1.774%); 6861/201321 objects misplaced (3.408%); 3.5 MiB/s, 56 objects/s recovering 2024-09-14T04:53:11.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:11.790 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:11.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:11.891 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:11.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:11.926 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:11.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:12.010 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 41 16 71030 71014 108.241 67.1875 0.0097223 0.00923324 2024-09-14T04:53:12.028 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:12.059 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:12.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:12.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:12.161 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:12.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:12.194 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:12.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:12.295 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:12.329 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:12.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:12.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:12.410 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:53:12.431 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:12.473 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:12.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:12.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:12.575 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:12.615 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:12.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:12.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:12.717 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:12.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:12.765 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:12.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:12.793 INFO:teuthology.orchestra.run.smithi039.stdout:moved 0 / 297 (0%) 2024-09-14T04:53:12.793 INFO:teuthology.orchestra.run.smithi039.stdout:avg 37.125 2024-09-14T04:53:12.794 INFO:teuthology.orchestra.run.smithi039.stdout:stddev 2.8477 -> 2.8477 (expected baseline 5.69951) 2024-09-14T04:53:12.794 INFO:teuthology.orchestra.run.smithi039.stdout:min osd.2 with 34 -> 34 pgs (0.915825 -> 0.915825 * mean) 2024-09-14T04:53:12.794 INFO:teuthology.orchestra.run.smithi039.stdout:max osd.4 with 43 -> 43 pgs (1.15825 -> 1.15825 * mean) 2024-09-14T04:53:12.794 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:53:12.794 INFO:teuthology.orchestra.run.smithi039.stdout:oload 1000 2024-09-14T04:53:12.794 INFO:teuthology.orchestra.run.smithi039.stdout:max_change 1 2024-09-14T04:53:12.794 INFO:teuthology.orchestra.run.smithi039.stdout:max_change_osds 4 2024-09-14T04:53:12.794 INFO:teuthology.orchestra.run.smithi039.stdout:average_utilization 0.0229 2024-09-14T04:53:12.794 INFO:teuthology.orchestra.run.smithi039.stdout:overload_utilization 0.2293 2024-09-14T04:53:12.794 INFO:teuthology.orchestra.run.smithi039.stderr:no change 2024-09-14T04:53:12.866 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:12.900 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:12.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:12.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:13.001 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:13.010 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 42 16 72101 72085 107.258 66.9375 0.042341 0.00931783 2024-09-14T04:53:13.035 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:13.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:13.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:13.138 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:13.178 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:13.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:13.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:13.280 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:13.315 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:13.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:13.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:13.417 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:13.451 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:13.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:13.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:13.553 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:13.578 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:13.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:13.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:13.680 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:13.716 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:13.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:13.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:13.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:13 smithi129 ceph-mon[101316]: pgmap v2757: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 946 op/s; 3571/201321 objects degraded (1.774%); 6861/201321 objects misplaced (3.408%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T04:53:13.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:13 smithi039 ceph-mon[106470]: pgmap v2757: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 946 op/s; 3571/201321 objects degraded (1.774%); 6861/201321 objects misplaced (3.408%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T04:53:13.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:13 smithi039 ceph-mon[111235]: pgmap v2757: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 946 op/s; 3571/201321 objects degraded (1.774%); 6861/201321 objects misplaced (3.408%); 2.9 MiB/s, 46 objects/s recovering 2024-09-14T04:53:13.818 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:13.857 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:13.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:13.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:13.959 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:14.007 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:14.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:14.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:14.010 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 43 16 73074 73058 106.178 60.8125 0.00581528 0.00941238 2024-09-14T04:53:14.109 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:14.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:14.148 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:14.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:14.249 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:14.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:14.282 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:14.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:14.383 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:14.390 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is unset 2024-09-14T04:53:14.436 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:14.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:14.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:14.539 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:14.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:14.582 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:14.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:14.683 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:14.715 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:14.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:14.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:14.769 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:14 smithi039 ceph-mon[106470]: from='client.36048 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:53:14.769 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:14 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/824369538' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:53:14.770 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:14 smithi039 ceph-mon[111235]: from='client.36048 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:53:14.770 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:14 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/824369538' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:53:14.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:14 smithi129 ceph-mon[101316]: from='client.36048 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:53:14.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:14 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/824369538' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:53:14.817 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:14.849 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:14.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:14.850 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:14.951 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:14.989 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:14.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:14.990 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:15.010 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 44 16 74135 74119 105.271 66.3125 0.00559545 0.00949311 2024-09-14T04:53:15.091 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:15.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:15.129 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:15.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:15.229 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:15.269 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:15.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:15.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:15.371 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:15.413 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:15.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:15.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:15.515 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:15.542 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:15.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:15.543 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:15.620 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:15 smithi039 ceph-mon[106470]: pgmap v2758: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 3571/217671 objects degraded (1.641%); 6582/217671 objects misplaced (3.024%); 3.5 MiB/s, 56 objects/s recovering 2024-09-14T04:53:15.620 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:15 smithi039 ceph-mon[106470]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:53:15.621 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:15 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/824369538' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:53:15.621 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:15 smithi039 ceph-mon[106470]: osdmap e1481: 8 total, 8 up, 8 in 2024-09-14T04:53:15.621 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:15 smithi039 ceph-mon[111235]: pgmap v2758: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 3571/217671 objects degraded (1.641%); 6582/217671 objects misplaced (3.024%); 3.5 MiB/s, 56 objects/s recovering 2024-09-14T04:53:15.621 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:15 smithi039 ceph-mon[111235]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:53:15.621 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:15 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/824369538' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:53:15.621 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:15 smithi039 ceph-mon[111235]: osdmap e1481: 8 total, 8 up, 8 in 2024-09-14T04:53:15.644 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:15.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:15.682 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:15.682 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:15.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:15 smithi129 ceph-mon[101316]: pgmap v2758: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 3571/217671 objects degraded (1.641%); 6582/217671 objects misplaced (3.024%); 3.5 MiB/s, 56 objects/s recovering 2024-09-14T04:53:15.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:15 smithi129 ceph-mon[101316]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:53:15.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:15 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/824369538' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:53:15.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:15 smithi129 ceph-mon[101316]: osdmap e1481: 8 total, 8 up, 8 in 2024-09-14T04:53:15.782 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:15.791 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:53:15.791 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:53:15.807 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:15.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:15.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:15.809 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:53:15.810 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:53:15.826 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:53:15.827 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:53:15.843 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:53:15.844 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:53:15.884 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:53:15.884 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:53:15.910 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:15.921 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:53:15.921 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:53:15.938 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:15.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:15.939 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:15.955 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:53:15.956 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:53:15.985 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:53:15.985 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:53:16.010 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 45 16 75121 75105 104.301 61.625 0.0291757 0.0095819 2024-09-14T04:53:16.041 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:16.071 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:16.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:16.072 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:16.172 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:16.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:16.209 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:16.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:16.310 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:16.347 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:16.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:16.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:16.448 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:16.484 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:16.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:16.485 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:16.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:16 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2666228682' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:53:16.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:16 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 3571/217671 objects degraded (1.641%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:16.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:16 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2666228682' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:53:16.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:16 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 3571/217671 objects degraded (1.641%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:16.586 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:16.622 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:16.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:16.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:16.724 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:16.755 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:16.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:16.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:16.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:16 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2666228682' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:53:16.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:16 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 3571/217671 objects degraded (1.641%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:16.857 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:16.887 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:16.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:16.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:16.989 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:17.010 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 46 16 76191 76175 103.488 66.875 0.012288 0.00965611 2024-09-14T04:53:17.027 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:17.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:17.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:17.129 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:17.139 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:53:17.171 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2024-09-14T04:53:17.171 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd test-reweight-by-utilization 1000 0.05 2024-09-14T04:53:17.181 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:53:17.184 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:17.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:17.185 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:17.286 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:17.321 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:17.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:17.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:17.423 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:17.447 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:17.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:17.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:17.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:17 smithi039 ceph-mon[106470]: pgmap v2760: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 3571/217671 objects degraded (1.641%); 6582/217671 objects misplaced (3.024%); 3.5 MiB/s, 56 objects/s recovering 2024-09-14T04:53:17.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:17 smithi039 ceph-mon[111235]: pgmap v2760: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 3571/217671 objects degraded (1.641%); 6582/217671 objects misplaced (3.024%); 3.5 MiB/s, 56 objects/s recovering 2024-09-14T04:53:17.549 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:17.587 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:17.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:17.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:17.689 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:17.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:17.725 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:17.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:17.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:17 smithi129 ceph-mon[101316]: pgmap v2760: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 3571/217671 objects degraded (1.641%); 6582/217671 objects misplaced (3.024%); 3.5 MiB/s, 56 objects/s recovering 2024-09-14T04:53:17.826 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:17.860 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:17.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:17.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:17.962 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:17.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:18.000 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:18.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:18.010 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 47 16 77176 77160 102.596 61.5625 0.00890766 0.00974113 2024-09-14T04:53:18.101 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:18.142 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:18.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:18.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:18.244 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:18.280 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:18.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:18.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:18.382 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:18.423 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:18.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:18.424 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:18.525 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:18.559 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:18.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:18.560 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:18.630 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:53:18.631 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:53:18.661 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:18.705 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:18.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:18.706 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:18.807 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:18.845 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:18.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:18.846 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:18.947 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:18.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:18.986 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:18.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:19.010 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 48 16 78178 78162 101.763 62.625 0.0133786 0.00982145 2024-09-14T04:53:19.012 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:53:19.041 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:53:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:53:18] "GET /metrics HTTP/1.1" 200 36811 "" "Prometheus/2.51.0" 2024-09-14T04:53:19.086 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:19.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:19.122 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:19.122 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:19.223 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:19.272 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:19.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:19.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:19.374 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:19.408 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:19.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:19.409 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:19.511 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:19.550 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:19.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:19.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:19.652 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:19.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:19.686 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:19.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:19.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:19 smithi129 ceph-mon[101316]: pgmap v2761: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 3571/217671 objects degraded (1.641%); 6582/217671 objects misplaced (3.024%); 3.5 MiB/s, 56 objects/s recovering 2024-09-14T04:53:19.787 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:19.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:19 smithi039 ceph-mon[106470]: pgmap v2761: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 3571/217671 objects degraded (1.641%); 6582/217671 objects misplaced (3.024%); 3.5 MiB/s, 56 objects/s recovering 2024-09-14T04:53:19.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:19 smithi039 ceph-mon[111235]: pgmap v2761: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s; 3571/217671 objects degraded (1.641%); 6582/217671 objects misplaced (3.024%); 3.5 MiB/s, 56 objects/s recovering 2024-09-14T04:53:19.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:19.830 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:19.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:19.932 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:19.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:19.969 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:19.969 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:20.011 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 49 16 79242 79226 101.043 66.5 0.00781096 0.00989173 2024-09-14T04:53:20.070 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:20.108 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:20.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:20.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:20.210 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:20.244 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:20.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:20.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:20.346 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:20.384 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:20.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:20.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:20.486 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:20.537 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:20.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:20.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:20.639 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:20.670 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:20.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:20.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:20.771 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:20.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:20.807 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:20.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:20.908 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:20.945 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:20.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:20.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:21.011 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 50 16 80271 80255 100.308 64.3125 0.00691027 0.00996422 2024-09-14T04:53:21.047 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:21.083 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:21.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:21.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:21.186 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:21.223 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:21.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:21.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:21.325 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:21.354 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:21.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:21.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:21.456 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:21.490 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:21.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:21.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:21.591 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:21.620 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:21.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:21.621 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:21.722 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:21.755 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:21 smithi039 ceph-mon[106470]: pgmap v2762: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 3571/233109 objects degraded (1.532%); 6271/233109 objects misplaced (2.690%); 3.7 MiB/s, 58 objects/s recovering 2024-09-14T04:53:21.755 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:21 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 3571/233109 objects degraded (1.532%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:21.755 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:21 smithi039 ceph-mon[111235]: pgmap v2762: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 3571/233109 objects degraded (1.532%); 6271/233109 objects misplaced (2.690%); 3.7 MiB/s, 58 objects/s recovering 2024-09-14T04:53:21.755 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:21 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 3571/233109 objects degraded (1.532%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:21.758 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:21.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:21.759 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:21.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:21 smithi129 ceph-mon[101316]: pgmap v2762: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 3571/233109 objects degraded (1.532%); 6271/233109 objects misplaced (2.690%); 3.7 MiB/s, 58 objects/s recovering 2024-09-14T04:53:21.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:21 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 3571/233109 objects degraded (1.532%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:21.860 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:21.897 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:21.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:21.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:21.999 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:53:21.999 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:22.011 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 51 16 81295 81279 99.5963 64 0.041865 0.010035 2024-09-14T04:53:22.030 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:22.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:22.031 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:22.052 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:53:22.052 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:53:22.132 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:22.174 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:22.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:22.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:22.276 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:22.311 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:22.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:22.312 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:22.413 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:22.453 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:22.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:22.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:22.555 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:22.584 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:22.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:22.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:22.686 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:22.717 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:22.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:22.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:22.819 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:22.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:22.857 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:22.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:22.958 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:22.995 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:22.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:22.996 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:23.011 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 52 16 82300 82284 98.8889 62.8125 0.015158 0.0101068 2024-09-14T04:53:23.097 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:23.128 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:23.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:23.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:23.229 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:23.266 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:23.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:23.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:23.368 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:23.401 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:23.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:23.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:23.503 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:23.513 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:23 smithi039 ceph-mon[106470]: pgmap v2763: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 3571/233109 objects degraded (1.532%); 6271/233109 objects misplaced (2.690%); 3.7 MiB/s, 58 objects/s recovering 2024-09-14T04:53:23.513 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:23 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:53:23.513 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:53:23.513 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:23 smithi039 ceph-mon[111235]: pgmap v2763: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 3571/233109 objects degraded (1.532%); 6271/233109 objects misplaced (2.690%); 3.7 MiB/s, 58 objects/s recovering 2024-09-14T04:53:23.513 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:23 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:53:23.513 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:53:23.538 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:23.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:23.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:23.639 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:23.679 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:23.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:23.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:23 smithi129 ceph-mon[101316]: pgmap v2763: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 66 MiB/s wr, 1.06k op/s; 3571/233109 objects degraded (1.532%); 6271/233109 objects misplaced (2.690%); 3.7 MiB/s, 58 objects/s recovering 2024-09-14T04:53:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:23 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:53:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:53:23.781 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:23.816 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:23.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:23.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:23.918 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:23.953 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:23.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:23.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:24.011 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 53 16 83333 83317 98.2411 64.5625 0.0111987 0.0101734 2024-09-14T04:53:24.056 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:24.093 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:24.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:24.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:24.195 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:24.225 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:24.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:24.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:24.326 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:24.365 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:24.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:24.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:24.467 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:24.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:24.503 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:24.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:24.604 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:24.634 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:24.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:24.635 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:24.736 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:24.769 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:24.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:24.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:24.871 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:24.898 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:24.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:24.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:25.000 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:25.011 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 54 16 84352 84336 97.6011 63.6875 0.00980017 0.0102397 2024-09-14T04:53:25.032 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:25.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:25.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:25.134 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:25.158 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:25.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:25.159 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:25.260 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:25.290 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:25.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:25.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:25.392 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:25.424 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:25.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:25.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:25.526 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:25.557 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:25.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:25.558 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:25.659 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:25.699 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:25.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:25.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:25 smithi129 ceph-mon[101316]: pgmap v2764: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s; 3571/242790 objects degraded (1.471%); 6123/242790 objects misplaced (2.522%); 2.9 MiB/s, 45 objects/s recovering 2024-09-14T04:53:25.776 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:25 smithi039 ceph-mon[106470]: pgmap v2764: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s; 3571/242790 objects degraded (1.471%); 6123/242790 objects misplaced (2.522%); 2.9 MiB/s, 45 objects/s recovering 2024-09-14T04:53:25.776 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:25 smithi039 ceph-mon[111235]: pgmap v2764: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 4.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 52 MiB/s wr, 837 op/s; 3571/242790 objects degraded (1.471%); 6123/242790 objects misplaced (2.522%); 2.9 MiB/s, 45 objects/s recovering 2024-09-14T04:53:25.849 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:25.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:25.879 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:25.879 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:25.980 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:26.010 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:26.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:26.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:26.011 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 55 16 85349 85333 96.9594 62.3125 0.0176734 0.0103085 2024-09-14T04:53:26.112 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:26.150 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:26.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:26.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:26.252 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:26.287 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:26.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:26.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:26.390 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:26.424 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:26.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:26.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:26.526 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:26.563 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:26.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:26.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:26.664 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:26.698 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:26.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:26.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:26.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:26 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 3571/242790 objects degraded (1.471%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:26.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:26 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 3571/242790 objects degraded (1.471%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:26.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:26 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 3571/242790 objects degraded (1.471%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:26.799 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:26.847 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:26.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:26.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:26.948 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:26.977 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:26.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:26.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:27.011 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 56 16 86274 86258 96.2603 57.8125 0.00534546 0.0103831 2024-09-14T04:53:27.079 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:27.110 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:27.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:27.111 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:27.212 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:27.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:27.280 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:27.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:27.320 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-14T04:53:27.381 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:27.424 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:27.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:27.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:27.431 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:27 smithi039 ceph-mon[106470]: pgmap v2765: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s; 3571/248547 objects degraded (1.437%); 5969/248547 objects misplaced (2.402%); 3.3 MiB/s, 53 objects/s recovering 2024-09-14T04:53:27.431 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:27 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:53:27.431 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:27 smithi039 ceph-mon[111235]: pgmap v2765: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s; 3571/248547 objects degraded (1.437%); 5969/248547 objects misplaced (2.402%); 3.3 MiB/s, 53 objects/s recovering 2024-09-14T04:53:27.431 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:27 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:53:27.527 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:27.566 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:27.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:27.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:27.647 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:27 smithi129 ceph-mon[101316]: pgmap v2765: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 56 MiB/s wr, 899 op/s; 3571/248547 objects degraded (1.437%); 5969/248547 objects misplaced (2.402%); 3.3 MiB/s, 53 objects/s recovering 2024-09-14T04:53:27.648 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:27 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:53:27.668 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:27.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:27.702 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:27.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:27.803 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:27.847 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:27.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:27.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:27.949 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:27.978 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:27.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:27.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:28.011 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 57 16 87198 87182 95.5846 57.75 0.00407982 0.0104533 2024-09-14T04:53:28.080 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:28.114 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:28.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:28.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:28.216 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:28.249 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:28.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:28.250 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:28.351 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:28.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:28.389 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:28.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:28.490 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:28.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:28.521 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:28.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:28.587 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:28 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3030567091' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:53:28.587 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:28 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3030567091' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:53:28.623 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:28.673 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:28.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:28.674 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:28.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:28 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3030567091' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:53:28.775 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:28.837 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:28.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:28.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:28.851 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:53:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:53:28] "GET /metrics HTTP/1.1" 200 36802 "" "Prometheus/2.51.0" 2024-09-14T04:53:28.939 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:28.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:28.973 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:28.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:29.011 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 58 16 87960 87944 94.7576 47.625 0.0117818 0.0105477 2024-09-14T04:53:29.074 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:29.078 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:53:29.078 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:53:29.078 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:53:29.078 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:53:29.078 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:53:29.078 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:53:29.078 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:53:29.078 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:53:29.078 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:53:29.078 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:53:29.078 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:53:29.079 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:53:29.079 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:53:29.079 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:53:29.079 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:53:29.079 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:53:29.114 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:29.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:29.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:29.216 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:29.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:29.272 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:29.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:29.292 INFO:teuthology.orchestra.run.smithi039.stdout:moved 1 / 297 (0.3367%) 2024-09-14T04:53:29.292 INFO:teuthology.orchestra.run.smithi039.stdout:avg 37.125 2024-09-14T04:53:29.292 INFO:teuthology.orchestra.run.smithi039.stdout:stddev 2.8477 -> 2.80346 (expected baseline 5.69951) 2024-09-14T04:53:29.293 INFO:teuthology.orchestra.run.smithi039.stdout:min osd.2 with 34 -> 34 pgs (0.915825 -> 0.915825 * mean) 2024-09-14T04:53:29.293 INFO:teuthology.orchestra.run.smithi039.stdout:max osd.4 with 43 -> 43 pgs (1.15825 -> 1.15825 * mean) 2024-09-14T04:53:29.293 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:53:29.293 INFO:teuthology.orchestra.run.smithi039.stdout:oload 1000 2024-09-14T04:53:29.293 INFO:teuthology.orchestra.run.smithi039.stdout:max_change 0.05 2024-09-14T04:53:29.293 INFO:teuthology.orchestra.run.smithi039.stdout:max_change_osds 4 2024-09-14T04:53:29.293 INFO:teuthology.orchestra.run.smithi039.stdout:average_utilization 0.0272 2024-09-14T04:53:29.294 INFO:teuthology.orchestra.run.smithi039.stdout:overload_utilization 0.2725 2024-09-14T04:53:29.294 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3 weight 0.8810 -> 0.8944 2024-09-14T04:53:29.294 INFO:teuthology.orchestra.run.smithi039.stderr:no change 2024-09-14T04:53:29.373 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:29.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:29.424 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:29.424 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:29.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:29 smithi129 ceph-mon[101316]: pgmap v2766: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s; 3571/248547 objects degraded (1.437%); 5969/248547 objects misplaced (2.402%); 3.2 MiB/s, 51 objects/s recovering 2024-09-14T04:53:29.525 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:29.572 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:29.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:29.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:29.674 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:29.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:29.707 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:29.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:29.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:29 smithi039 ceph-mon[106470]: pgmap v2766: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s; 3571/248547 objects degraded (1.437%); 5969/248547 objects misplaced (2.402%); 3.2 MiB/s, 51 objects/s recovering 2024-09-14T04:53:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:29 smithi039 ceph-mon[111235]: pgmap v2766: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 5.1 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 54 MiB/s wr, 857 op/s; 3571/248547 objects degraded (1.437%); 5969/248547 objects misplaced (2.402%); 3.2 MiB/s, 51 objects/s recovering 2024-09-14T04:53:29.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:53:29 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T04:53:29.289+0000 7f268b1da640 -1 mgr.server reply reply (1) Operation not permitted no change 2024-09-14T04:53:29.807 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:29.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:29.844 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:29.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:29.945 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:29.976 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:29.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:29.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:30.011 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 59 16 88854 88838 94.0985 55.875 0.012455 0.0106206 2024-09-14T04:53:30.078 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:30.153 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:30.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:30.154 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:30.255 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:30.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:30.310 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:30.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:30.413 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:30.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:30.486 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:30.486 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:30.587 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:30.618 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:30.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:30.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:30.641 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:30 smithi039 ceph-mon[106470]: from='client.36069 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:53:30.641 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:30 smithi039 ceph-mon[111235]: from='client.36069 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:53:30.720 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:30.736 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:30 smithi129 ceph-mon[101316]: from='client.36069 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:53:30.752 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:30.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:30.753 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:30.854 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:30.897 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:30.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:30.899 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:31.000 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:31.013 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:2024-09-14T04:53:31.011868+0000 min lat: 0.00136201 max lat: 2.17883 avg lat: 0.0107101 2024-09-14T04:53:31.013 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:53:31.013 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 60 16 89612 89596 93.3197 47.375 0.0168614 0.0107101 2024-09-14T04:53:31.050 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:31.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:31.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:31.152 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:31.195 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:31.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:31.196 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:31.229 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:53:31.261 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2024-09-14T04:53:31.261 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd test-reweight-by-utilization 1000 1.0 2024-09-14T04:53:31.297 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:31.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:31.346 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:31.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:31.384 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is set 2024-09-14T04:53:31.393 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:31 smithi129 ceph-mon[101316]: pgmap v2767: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 3571/262827 objects degraded (1.359%); 5675/262827 objects misplaced (2.159%); 4.7 MiB/s, 75 objects/s recovering 2024-09-14T04:53:31.394 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:31 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/479284498' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:53:31.447 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:31.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:31.476 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:31.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:31.533 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:31 smithi039 ceph-mon[111235]: pgmap v2767: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 3571/262827 objects degraded (1.359%); 5675/262827 objects misplaced (2.159%); 4.7 MiB/s, 75 objects/s recovering 2024-09-14T04:53:31.533 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:31 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/479284498' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:53:31.533 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:31 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:53:31.533 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:31 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 3571/262827 objects degraded (1.359%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:31.533 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:31 smithi039 ceph-mon[106470]: pgmap v2767: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 78 MiB/s wr, 1.25k op/s; 3571/262827 objects degraded (1.359%); 5675/262827 objects misplaced (2.159%); 4.7 MiB/s, 75 objects/s recovering 2024-09-14T04:53:31.533 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:31 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/479284498' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:53:31.534 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:31 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:53:31.534 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:31 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 3571/262827 objects degraded (1.359%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:31.578 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:31.610 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:31.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:31.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:31.712 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:31.722 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:31 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:53:31.722 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:31 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 3571/262827 objects degraded (1.359%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:31.750 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:31.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:31.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:31.852 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:31.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:31.882 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:31.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:31.905 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:53:31.983 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:32.011 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 61 16 90426 90410 92.6238 50.875 0.015169 0.0107914 2024-09-14T04:53:32.020 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:32.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:32.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:32.122 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:32.165 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:32.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:32.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:32.267 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:32.306 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:32.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:32.307 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:32.408 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:32.447 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:32.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:32.449 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:32.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:32 smithi039 ceph-mon[106470]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:53:32.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:32 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:53:32.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:32 smithi039 ceph-mon[106470]: osdmap e1482: 8 total, 8 up, 8 in 2024-09-14T04:53:32.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:32 smithi039 ceph-mon[111235]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:53:32.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:32 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:53:32.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:32 smithi039 ceph-mon[111235]: osdmap e1482: 8 total, 8 up, 8 in 2024-09-14T04:53:32.549 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:32.583 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:32.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:32.584 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:32.685 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:32.718 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:32.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:32.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:32.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:32 smithi129 ceph-mon[101316]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:53:32.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:32 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:53:32.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:32 smithi129 ceph-mon[101316]: osdmap e1482: 8 total, 8 up, 8 in 2024-09-14T04:53:32.820 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:32.849 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:32.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:32.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:32.950 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:32.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:32.983 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:32.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:33.012 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 62 16 91312 91296 92.023 55.375 0.0206565 0.0108613 2024-09-14T04:53:33.085 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:33.146 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:33.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:33.149 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:33.250 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:33.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:33.293 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:33.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:33.394 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:33.422 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:33.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:33.425 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:33.526 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:33.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:33.568 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:33.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:33.669 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:33.700 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:33.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:33.702 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:33 smithi129 ceph-mon[101316]: pgmap v2769: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 990 op/s; 3571/262827 objects degraded (1.359%); 5675/262827 objects misplaced (2.159%); 3.7 MiB/s, 59 objects/s recovering 2024-09-14T04:53:33.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:33 smithi129 ceph-mon[101316]: 1.2 scrub starts 2024-09-14T04:53:33.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:33 smithi039 ceph-mon[111235]: pgmap v2769: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 990 op/s; 3571/262827 objects degraded (1.359%); 5675/262827 objects misplaced (2.159%); 3.7 MiB/s, 59 objects/s recovering 2024-09-14T04:53:33.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:33 smithi039 ceph-mon[111235]: 1.2 scrub starts 2024-09-14T04:53:33.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:33 smithi039 ceph-mon[106470]: pgmap v2769: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 5.3 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 62 MiB/s wr, 990 op/s; 3571/262827 objects degraded (1.359%); 5675/262827 objects misplaced (2.159%); 3.7 MiB/s, 59 objects/s recovering 2024-09-14T04:53:33.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:33 smithi039 ceph-mon[106470]: 1.2 scrub starts 2024-09-14T04:53:33.803 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:33.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:33.841 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:33.841 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:33.942 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:33.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:33.974 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:33.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:34.012 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 63 16 92232 92216 91.4749 57.5 0.0172862 0.0109272 2024-09-14T04:53:34.076 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:34.104 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:34.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:34.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:34.210 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:34.271 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:53:34.291 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:34.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:34.292 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:34.307 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:53:34.393 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:34.426 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:34.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:34.428 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:34.528 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:34.568 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:34.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:34.569 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:34.669 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:34.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:34.706 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:34.706 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:34.803 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:53:34.807 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:34.899 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:34.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:34.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:34.934 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:34 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:53:34.934 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:34 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:53:34.934 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:34 smithi039 ceph-mon[106470]: pgmap v2770: 99 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 928 op/s; 3571/270639 objects degraded (1.319%); 5529/270639 objects misplaced (2.043%); 3.7 MiB/s, 59 objects/s recovering 2024-09-14T04:53:34.935 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:34 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:53:34.935 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:34 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:53:34.935 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:34 smithi039 ceph-mon[111235]: pgmap v2770: 99 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 928 op/s; 3571/270639 objects degraded (1.319%); 5529/270639 objects misplaced (2.043%); 3.7 MiB/s, 59 objects/s recovering 2024-09-14T04:53:35.001 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:35.012 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 64 16 93141 93125 90.9332 56.8125 0.0156839 0.0109917 2024-09-14T04:53:35.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:34 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:53:35.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:34 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:53:35.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:34 smithi129 ceph-mon[101316]: pgmap v2770: 99 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 5.5 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 58 MiB/s wr, 928 op/s; 3571/270639 objects degraded (1.319%); 5529/270639 objects misplaced (2.043%); 3.7 MiB/s, 59 objects/s recovering 2024-09-14T04:53:35.043 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:35.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:35.044 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:35.144 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:35.186 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:35.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:35.187 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:35.288 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:35.326 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:35.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:35.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:35.428 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:35.459 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:35.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:35.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:35.561 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:35.601 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:35.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:35.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:35.703 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:35.745 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:35.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:35.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:35.847 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:35.893 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:35.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:35.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:35.995 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:36.012 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 65 16 94080 94064 90.4371 58.6875 0.0353195 0.0110508 2024-09-14T04:53:36.039 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:36.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:36.040 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:36.141 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:36.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:36.180 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:36.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:36.282 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:36.318 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:36.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:36.320 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:36.421 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:36.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:36.459 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:36.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:36.560 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:36.603 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:36.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:36.604 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:36.705 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:36.740 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:36.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:36.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:36.842 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:36.874 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:36.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:36.875 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:36.976 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:37.012 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 66 16 95041 95025 89.9768 60.0625 0.0135678 0.0111082 2024-09-14T04:53:37.013 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:37.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:37.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:37.115 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:37.151 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:37.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:37.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:37.153 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:36 smithi039 ceph-mon[111235]: pgmap v2771: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 95 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s; 3571/275556 objects degraded (1.296%); 5380/275556 objects misplaced (1.952%); 3.7 MiB/s, 58 objects/s recovering 2024-09-14T04:53:37.153 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:36 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 3571/275556 objects degraded (1.296%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:37.154 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:36 smithi039 ceph-mon[106470]: pgmap v2771: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 95 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s; 3571/275556 objects degraded (1.296%); 5380/275556 objects misplaced (1.952%); 3.7 MiB/s, 58 objects/s recovering 2024-09-14T04:53:37.154 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:36 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 3571/275556 objects degraded (1.296%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:37.252 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:37.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:36 smithi129 ceph-mon[101316]: pgmap v2771: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 95 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s; 3571/275556 objects degraded (1.296%); 5380/275556 objects misplaced (1.952%); 3.7 MiB/s, 58 objects/s recovering 2024-09-14T04:53:37.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:36 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 3571/275556 objects degraded (1.296%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:37.289 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:37.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:37.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:37.391 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:37.423 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:53:37.424 INFO:teuthology.orchestra.run.smithi039.stdout: "size": 20, 2024-09-14T04:53:37.424 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 600, 2024-09-14T04:53:37.424 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [ 2024-09-14T04:53:37.424 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.424 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:135589 54.c 54:356406db:::benchmark_data_smithi039_264334_object9192:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1401)", 2024-09-14T04:53:37.424 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:56.300102+0000", 2024-09-14T04:53:37.424 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 521.00253828200005, 2024-09-14T04:53:37.424 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1549550009999998, 2024-09-14T04:53:37.424 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:53:37.424 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:53:37.424 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T04:53:37.424 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:53:37.425 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:53:37.425 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:53:37.425 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 135589 2024-09-14T04:53:37.425 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.425 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:53:37.425 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.425 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:53:37.425 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.300102+0000", 2024-09-14T04:53:37.425 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.425 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.425 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.425 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:53:37.425 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.300102+0000", 2024-09-14T04:53:37.426 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.426 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.426 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.426 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:53:37.426 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.300103+0000", 2024-09-14T04:53:37.426 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.939e-06 2024-09-14T04:53:37.426 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.426 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.426 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:53:37.426 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.300109+0000", 2024-09-14T04:53:37.426 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7389999999999996e-06 2024-09-14T04:53:37.426 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.426 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.426 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:53:37.427 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.300125+0000", 2024-09-14T04:53:37.427 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6076999999999999e-05 2024-09-14T04:53:37.427 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.427 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.427 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:53:37.427 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.300133+0000", 2024-09-14T04:53:37.427 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.2230000000000001e-06 2024-09-14T04:53:37.427 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.427 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.427 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.427 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.301312+0000", 2024-09-14T04:53:37.427 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0011795480000000001 2024-09-14T04:53:37.427 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.428 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.428 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:53:37.428 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.302565+0000", 2024-09-14T04:53:37.428 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0012534099999999999 2024-09-14T04:53:37.428 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.428 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.428 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 5,6", 2024-09-14T04:53:37.428 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.302670+0000", 2024-09-14T04:53:37.428 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0001047 2024-09-14T04:53:37.428 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.428 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.428 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:37.429 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.306221+0000", 2024-09-14T04:53:37.429 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00355105 2024-09-14T04:53:37.429 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.429 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.429 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:53:37.429 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.306468+0000", 2024-09-14T04:53:37.429 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00024643100000000003 2024-09-14T04:53:37.429 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.429 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.429 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.429 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.420868+0000", 2024-09-14T04:53:37.429 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.114400113 2024-09-14T04:53:37.430 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.430 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.430 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:53:37.430 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.420879+0000", 2024-09-14T04:53:37.430 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0720000000000001e-05 2024-09-14T04:53:37.430 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.430 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.430 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.430 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.324315+0000", 2024-09-14T04:53:37.430 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.90343673899999999 2024-09-14T04:53:37.430 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.430 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.431 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:53:37.431 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.324338+0000", 2024-09-14T04:53:37.431 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2510999999999998e-05 2024-09-14T04:53:37.431 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.431 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.431 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.431 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.454966+0000", 2024-09-14T04:53:37.431 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1306284820000001 2024-09-14T04:53:37.431 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.431 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.431 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:53:37.431 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.455057+0000", 2024-09-14T04:53:37.431 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.0319e-05 2024-09-14T04:53:37.432 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.432 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:53:37.432 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.432 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.432 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.432 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:135618 54.c 54:3384db48:::benchmark_data_smithi039_264334_object9221:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:53:37.432 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:56.374671+0000", 2024-09-14T04:53:37.432 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 520.92796883300002, 2024-09-14T04:53:37.432 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.091442705, 2024-09-14T04:53:37.432 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:53:37.432 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:53:37.433 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:37.433 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:53:37.433 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:53:37.433 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:53:37.433 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 135618 2024-09-14T04:53:37.433 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.433 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:53:37.433 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.433 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:53:37.433 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.374671+0000", 2024-09-14T04:53:37.433 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.433 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.433 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.434 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:53:37.434 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.374671+0000", 2024-09-14T04:53:37.434 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.434 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.434 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.434 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:53:37.434 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.374672+0000", 2024-09-14T04:53:37.434 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.5000000000000001e-07 2024-09-14T04:53:37.434 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.434 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.434 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:53:37.434 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.374676+0000", 2024-09-14T04:53:37.434 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3590000000000001e-06 2024-09-14T04:53:37.435 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.435 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.435 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:53:37.435 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.374686+0000", 2024-09-14T04:53:37.435 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.9350000000000006e-06 2024-09-14T04:53:37.435 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.435 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.435 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:53:37.435 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.374693+0000", 2024-09-14T04:53:37.435 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9659999999999999e-06 2024-09-14T04:53:37.435 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.435 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.435 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.435 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.429534+0000", 2024-09-14T04:53:37.436 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.054841049000000003 2024-09-14T04:53:37.436 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.436 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.436 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:53:37.436 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.429544+0000", 2024-09-14T04:53:37.436 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.5289999999999997e-06 2024-09-14T04:53:37.436 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.436 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.436 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.436 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.325910+0000", 2024-09-14T04:53:37.436 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.89636666499999995 2024-09-14T04:53:37.436 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.436 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.437 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:53:37.437 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.325921+0000", 2024-09-14T04:53:37.437 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0747000000000001e-05 2024-09-14T04:53:37.437 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.437 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.437 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.437 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.455428+0000", 2024-09-14T04:53:37.437 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1295071729999999 2024-09-14T04:53:37.437 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.437 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.437 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:53:37.437 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.455553+0000", 2024-09-14T04:53:37.437 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000125041 2024-09-14T04:53:37.437 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.438 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.438 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 4,5,6", 2024-09-14T04:53:37.438 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.455674+0000", 2024-09-14T04:53:37.438 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000120759 2024-09-14T04:53:37.438 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.438 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.438 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:53:37.438 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.465735+0000", 2024-09-14T04:53:37.438 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.010060955 2024-09-14T04:53:37.438 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.438 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.438 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:37.438 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.465794+0000", 2024-09-14T04:53:37.438 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.9268000000000002e-05 2024-09-14T04:53:37.439 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.439 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.439 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:37.439 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.465869+0000", 2024-09-14T04:53:37.439 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.4383999999999998e-05 2024-09-14T04:53:37.439 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.439 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.439 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:37.439 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.465918+0000", 2024-09-14T04:53:37.439 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9656e-05 2024-09-14T04:53:37.439 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.439 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.439 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:53:37.440 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.465988+0000", 2024-09-14T04:53:37.440 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9436000000000004e-05 2024-09-14T04:53:37.440 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.440 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.440 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:53:37.440 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.466114+0000", 2024-09-14T04:53:37.440 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000125833 2024-09-14T04:53:37.440 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.440 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:53:37.440 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.440 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.440 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.440 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:135633 54.c 54:33252ab7:::benchmark_data_smithi039_264334_object9236:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:53:37.441 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:56.431333+0000", 2024-09-14T04:53:37.441 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 520.87130725300005, 2024-09-14T04:53:37.441 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0351089770000002, 2024-09-14T04:53:37.441 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:53:37.441 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:53:37.441 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:37.441 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:53:37.441 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:53:37.441 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:53:37.441 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 135633 2024-09-14T04:53:37.441 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.441 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:53:37.441 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.441 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:53:37.442 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431333+0000", 2024-09-14T04:53:37.442 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.442 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.442 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.442 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:53:37.442 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431333+0000", 2024-09-14T04:53:37.442 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.442 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.442 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.442 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:53:37.442 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431335+0000", 2024-09-14T04:53:37.442 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9479999999999998e-06 2024-09-14T04:53:37.442 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.442 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.443 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:53:37.443 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431340+0000", 2024-09-14T04:53:37.443 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5559999999999998e-06 2024-09-14T04:53:37.443 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.443 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.443 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:53:37.443 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431354+0000", 2024-09-14T04:53:37.443 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3772e-05 2024-09-14T04:53:37.443 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.443 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.443 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:53:37.443 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431362+0000", 2024-09-14T04:53:37.443 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.5390000000000007e-06 2024-09-14T04:53:37.443 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.444 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.444 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.444 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.447068+0000", 2024-09-14T04:53:37.444 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.015705666 2024-09-14T04:53:37.444 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.444 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.444 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:53:37.444 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.447080+0000", 2024-09-14T04:53:37.444 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1506e-05 2024-09-14T04:53:37.444 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.444 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.444 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.445 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.325950+0000", 2024-09-14T04:53:37.445 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.87887031699999996 2024-09-14T04:53:37.445 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.445 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.445 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:53:37.445 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.325958+0000", 2024-09-14T04:53:37.445 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.1540000000000002e-06 2024-09-14T04:53:37.445 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.445 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.445 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.445 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.458856+0000", 2024-09-14T04:53:37.445 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1328979669999999 2024-09-14T04:53:37.446 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.446 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.446 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:53:37.446 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.459013+0000", 2024-09-14T04:53:37.446 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015676700000000001 2024-09-14T04:53:37.446 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.446 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.446 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 4,5,6", 2024-09-14T04:53:37.446 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.459124+0000", 2024-09-14T04:53:37.446 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000111482 2024-09-14T04:53:37.446 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.447 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.447 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:53:37.447 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.466127+0000", 2024-09-14T04:53:37.447 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0070024850000000001 2024-09-14T04:53:37.447 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.447 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.447 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:37.447 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.466181+0000", 2024-09-14T04:53:37.447 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4088000000000001e-05 2024-09-14T04:53:37.447 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.447 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.447 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:37.447 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.466226+0000", 2024-09-14T04:53:37.447 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4965000000000001e-05 2024-09-14T04:53:37.448 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.448 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.448 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:37.448 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.466287+0000", 2024-09-14T04:53:37.448 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1079000000000003e-05 2024-09-14T04:53:37.448 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.448 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.448 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:53:37.448 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.466341+0000", 2024-09-14T04:53:37.448 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4379000000000002e-05 2024-09-14T04:53:37.448 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.448 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.448 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:53:37.448 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.466442+0000", 2024-09-14T04:53:37.449 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000100307 2024-09-14T04:53:37.449 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.449 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:53:37.449 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.449 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.449 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.449 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:135635 54.c 54:31b8983c:::benchmark_data_smithi039_264334_object9238:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:53:37.449 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:56.431574+0000", 2024-09-14T04:53:37.449 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 520.8710658, 2024-09-14T04:53:37.449 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0581618759999998, 2024-09-14T04:53:37.449 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:53:37.449 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:53:37.450 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:37.450 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:53:37.450 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:53:37.450 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:53:37.450 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 135635 2024-09-14T04:53:37.450 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.450 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:53:37.450 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.450 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:53:37.450 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431574+0000", 2024-09-14T04:53:37.450 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.450 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.450 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.450 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:53:37.451 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431574+0000", 2024-09-14T04:53:37.451 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.451 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.451 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.451 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:53:37.451 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431575+0000", 2024-09-14T04:53:37.451 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.2600000000000001e-07 2024-09-14T04:53:37.451 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.451 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.451 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:53:37.451 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431579+0000", 2024-09-14T04:53:37.451 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5249999999999999e-06 2024-09-14T04:53:37.451 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.451 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.452 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:53:37.452 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431591+0000", 2024-09-14T04:53:37.452 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1314999999999999e-05 2024-09-14T04:53:37.452 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.452 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.452 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:53:37.452 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.431598+0000", 2024-09-14T04:53:37.452 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.2420000000000003e-06 2024-09-14T04:53:37.452 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.452 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.452 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.452 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.447535+0000", 2024-09-14T04:53:37.452 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.015937337999999999 2024-09-14T04:53:37.453 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.453 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.453 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:53:37.453 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.447545+0000", 2024-09-14T04:53:37.453 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.4890000000000001e-06 2024-09-14T04:53:37.453 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.453 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.453 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.453 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.325982+0000", 2024-09-14T04:53:37.453 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.87843713400000001 2024-09-14T04:53:37.453 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.453 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.453 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:53:37.454 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.325990+0000", 2024-09-14T04:53:37.454 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9400000000000002e-06 2024-09-14T04:53:37.454 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.454 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.454 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.454 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.466504+0000", 2024-09-14T04:53:37.454 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1405139260000001 2024-09-14T04:53:37.454 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.454 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.454 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:53:37.454 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.466628+0000", 2024-09-14T04:53:37.454 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012408699999999999 2024-09-14T04:53:37.454 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.454 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.455 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 4,5,6", 2024-09-14T04:53:37.455 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.466745+0000", 2024-09-14T04:53:37.455 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000117447 2024-09-14T04:53:37.455 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.455 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.455 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:37.455 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.476281+0000", 2024-09-14T04:53:37.455 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0095351040000000008 2024-09-14T04:53:37.455 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.455 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.455 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:37.455 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.476349+0000", 2024-09-14T04:53:37.455 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8342000000000004e-05 2024-09-14T04:53:37.456 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.456 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.456 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:53:37.456 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.479362+0000", 2024-09-14T04:53:37.456 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0030131540000000001 2024-09-14T04:53:37.456 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.456 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.456 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:37.456 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.488661+0000", 2024-09-14T04:53:37.456 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0092988040000000008 2024-09-14T04:53:37.456 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.456 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.456 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:53:37.456 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.489603+0000", 2024-09-14T04:53:37.457 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00094184399999999995 2024-09-14T04:53:37.457 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.457 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.457 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:53:37.457 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.489736+0000", 2024-09-14T04:53:37.457 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013325899999999999 2024-09-14T04:53:37.457 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.457 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:53:37.457 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.457 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.457 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.457 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:135644 54.c 54:35c60476:::benchmark_data_smithi039_264334_object9247:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:53:37.457 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:56.446718+0000", 2024-09-14T04:53:37.458 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 520.855921653, 2024-09-14T04:53:37.458 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.043331561, 2024-09-14T04:53:37.458 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:53:37.458 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:53:37.458 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:37.458 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:53:37.458 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:53:37.458 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:53:37.458 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 135644 2024-09-14T04:53:37.458 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.458 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:53:37.458 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.458 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:53:37.458 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.446718+0000", 2024-09-14T04:53:37.459 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.459 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.459 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.459 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:53:37.459 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.446718+0000", 2024-09-14T04:53:37.459 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.459 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.459 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.459 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:53:37.459 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.446720+0000", 2024-09-14T04:53:37.459 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3540000000000001e-06 2024-09-14T04:53:37.459 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.459 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.460 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:53:37.460 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.446724+0000", 2024-09-14T04:53:37.460 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7249999999999997e-06 2024-09-14T04:53:37.460 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.460 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.460 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:53:37.460 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.446734+0000", 2024-09-14T04:53:37.460 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.5920000000000007e-06 2024-09-14T04:53:37.460 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.460 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.460 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:53:37.460 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.446742+0000", 2024-09-14T04:53:37.460 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0520000000000002e-06 2024-09-14T04:53:37.460 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.461 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.461 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.461 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.447585+0000", 2024-09-14T04:53:37.461 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00084300300000000005 2024-09-14T04:53:37.461 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.461 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.461 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:53:37.461 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.447597+0000", 2024-09-14T04:53:37.461 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2302e-05 2024-09-14T04:53:37.461 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.461 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.461 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.461 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.326086+0000", 2024-09-14T04:53:37.461 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.87848874099999996 2024-09-14T04:53:37.462 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.462 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.462 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:53:37.462 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.326095+0000", 2024-09-14T04:53:37.462 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.1360000000000003e-06 2024-09-14T04:53:37.462 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.462 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.462 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.462 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.469679+0000", 2024-09-14T04:53:37.462 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.143583765 2024-09-14T04:53:37.462 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.462 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.462 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:53:37.462 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.469760+0000", 2024-09-14T04:53:37.462 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0669999999999998e-05 2024-09-14T04:53:37.463 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.463 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.463 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 4,5,6", 2024-09-14T04:53:37.463 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.469836+0000", 2024-09-14T04:53:37.463 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6048000000000001e-05 2024-09-14T04:53:37.463 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.463 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.463 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:53:37.463 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.479370+0000", 2024-09-14T04:53:37.463 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.009534275 2024-09-14T04:53:37.463 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.463 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.463 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:37.464 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.481815+0000", 2024-09-14T04:53:37.464 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.002444786 2024-09-14T04:53:37.464 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.464 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.464 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:37.464 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.481869+0000", 2024-09-14T04:53:37.464 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4113000000000002e-05 2024-09-14T04:53:37.464 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.464 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.464 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:37.464 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.489843+0000", 2024-09-14T04:53:37.464 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0079742140000000003 2024-09-14T04:53:37.464 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.464 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.465 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:53:37.465 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.489933+0000", 2024-09-14T04:53:37.465 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.0115e-05 2024-09-14T04:53:37.465 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.465 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.465 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:53:37.465 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.490050+0000", 2024-09-14T04:53:37.465 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011667000000000001 2024-09-14T04:53:37.465 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.465 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:53:37.465 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.465 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.465 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.465 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.35436.0:135655 54.c 54:3bab8a03:::benchmark_data_smithi039_264334_object9258:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:53:37.466 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:44:56.456620+0000", 2024-09-14T04:53:37.466 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 520.84602023000002, 2024-09-14T04:53:37.466 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0336829170000001, 2024-09-14T04:53:37.466 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:53:37.466 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:53:37.466 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:37.466 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:53:37.466 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.35436", 2024-09-14T04:53:37.466 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:53:37.466 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 135655 2024-09-14T04:53:37.466 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.466 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:53:37.466 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.466 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:53:37.467 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.456620+0000", 2024-09-14T04:53:37.467 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.467 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.467 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.467 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:53:37.467 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.456620+0000", 2024-09-14T04:53:37.467 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.467 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.467 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.467 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:53:37.467 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.456621+0000", 2024-09-14T04:53:37.467 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7719999999999999e-06 2024-09-14T04:53:37.467 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.468 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.468 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:53:37.468 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.456628+0000", 2024-09-14T04:53:37.468 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3679999999999998e-06 2024-09-14T04:53:37.468 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.468 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.468 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:53:37.468 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.456644+0000", 2024-09-14T04:53:37.468 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6297999999999999e-05 2024-09-14T04:53:37.468 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.468 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.468 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:53:37.468 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.456652+0000", 2024-09-14T04:53:37.469 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9519999999999994e-06 2024-09-14T04:53:37.469 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.469 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.469 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.469 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.456746+0000", 2024-09-14T04:53:37.469 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.3617999999999997e-05 2024-09-14T04:53:37.469 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.469 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.469 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:53:37.469 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:56.456760+0000", 2024-09-14T04:53:37.469 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4185999999999999e-05 2024-09-14T04:53:37.469 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.469 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.469 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.470 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.326122+0000", 2024-09-14T04:53:37.470 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.86936199400000003 2024-09-14T04:53:37.470 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.470 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.470 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:53:37.470 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:57.326129+0000", 2024-09-14T04:53:37.470 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.5730000000000002e-06 2024-09-14T04:53:37.470 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.470 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.470 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.470 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.473189+0000", 2024-09-14T04:53:37.470 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.14706011 2024-09-14T04:53:37.470 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.470 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.471 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:53:37.471 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.473313+0000", 2024-09-14T04:53:37.471 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000123662 2024-09-14T04:53:37.471 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.471 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.471 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 4,5,6", 2024-09-14T04:53:37.471 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.473443+0000", 2024-09-14T04:53:37.471 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012988400000000001 2024-09-14T04:53:37.471 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.471 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.471 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:53:37.471 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.481877+0000", 2024-09-14T04:53:37.471 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0084343060000000008 2024-09-14T04:53:37.471 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.472 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.472 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:37.472 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.481911+0000", 2024-09-14T04:53:37.472 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.3553000000000002e-05 2024-09-14T04:53:37.472 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.472 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.472 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:37.472 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.489798+0000", 2024-09-14T04:53:37.472 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0078867659999999999 2024-09-14T04:53:37.472 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.472 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.472 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:37.472 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.490128+0000", 2024-09-14T04:53:37.473 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00033039400000000002 2024-09-14T04:53:37.473 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.473 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.473 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:53:37.473 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.490211+0000", 2024-09-14T04:53:37.473 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.2700000000000004e-05 2024-09-14T04:53:37.473 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.473 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.473 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:53:37.473 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:44:58.490303+0000", 2024-09-14T04:53:37.473 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.1780999999999994e-05 2024-09-14T04:53:37.473 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.473 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:53:37.473 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.474 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.474 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.474 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45632.0:104260 55.e 55:78317f85:::benchmark_data_smithi039_297155_object104259:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1440)", 2024-09-14T04:53:37.474 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:49:10.237868+0000", 2024-09-14T04:53:37.474 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 267.06477222400002, 2024-09-14T04:53:37.474 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.382777745, 2024-09-14T04:53:37.474 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:53:37.474 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:53:37.474 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:37.474 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:53:37.474 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45632", 2024-09-14T04:53:37.474 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:53:37.474 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 104260 2024-09-14T04:53:37.474 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.475 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:53:37.475 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.475 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:53:37.475 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.237868+0000", 2024-09-14T04:53:37.475 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.475 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.475 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.475 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:53:37.475 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.237868+0000", 2024-09-14T04:53:37.475 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.475 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.475 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.475 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:53:37.475 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.237869+0000", 2024-09-14T04:53:37.476 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5120000000000001e-06 2024-09-14T04:53:37.476 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.476 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.476 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:53:37.476 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.237893+0000", 2024-09-14T04:53:37.476 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4287999999999999e-05 2024-09-14T04:53:37.476 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.476 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.476 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:53:37.476 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.237909+0000", 2024-09-14T04:53:37.476 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5637999999999998e-05 2024-09-14T04:53:37.476 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.476 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.476 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:53:37.477 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.237918+0000", 2024-09-14T04:53:37.477 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.8300000000000002e-06 2024-09-14T04:53:37.477 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.477 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.477 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.477 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.238288+0000", 2024-09-14T04:53:37.477 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000369725 2024-09-14T04:53:37.477 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.477 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.477 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:53:37.477 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.238451+0000", 2024-09-14T04:53:37.477 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000163859 2024-09-14T04:53:37.477 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.477 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.478 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:53:37.478 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.238603+0000", 2024-09-14T04:53:37.478 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015151699999999999 2024-09-14T04:53:37.478 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.478 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.478 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:53:37.478 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.242832+0000", 2024-09-14T04:53:37.478 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0042295029999999999 2024-09-14T04:53:37.478 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.478 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.478 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:37.478 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.243043+0000", 2024-09-14T04:53:37.478 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00021068500000000001 2024-09-14T04:53:37.478 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.479 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.479 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:37.479 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.620547+0000", 2024-09-14T04:53:37.479 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.37750365000000002 2024-09-14T04:53:37.479 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.479 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.479 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:53:37.479 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.620594+0000", 2024-09-14T04:53:37.479 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6699999999999997e-05 2024-09-14T04:53:37.479 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.479 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.479 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:53:37.479 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.620645+0000", 2024-09-14T04:53:37.479 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1838000000000001e-05 2024-09-14T04:53:37.480 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.480 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:53:37.480 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.480 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.480 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.480 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45632.0:104268 55.0 55:03b0a67b:::benchmark_data_smithi039_297155_object104267:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1440)", 2024-09-14T04:53:37.480 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:49:10.326212+0000", 2024-09-14T04:53:37.480 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 266.97642733599997, 2024-09-14T04:53:37.480 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.31380261300000001, 2024-09-14T04:53:37.480 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:53:37.480 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:53:37.480 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:37.480 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:53:37.481 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45632", 2024-09-14T04:53:37.481 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:53:37.481 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 104268 2024-09-14T04:53:37.481 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.481 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:53:37.481 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.481 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:53:37.481 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.326212+0000", 2024-09-14T04:53:37.481 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.481 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.481 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.481 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:53:37.481 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.326212+0000", 2024-09-14T04:53:37.482 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.482 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.482 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.482 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:53:37.482 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.326215+0000", 2024-09-14T04:53:37.482 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0260000000000002e-06 2024-09-14T04:53:37.482 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.482 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.482 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:53:37.482 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.326240+0000", 2024-09-14T04:53:37.482 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.5248e-05 2024-09-14T04:53:37.482 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.482 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.482 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:53:37.483 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.326257+0000", 2024-09-14T04:53:37.483 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7141000000000001e-05 2024-09-14T04:53:37.483 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.483 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.483 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:53:37.483 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.326265+0000", 2024-09-14T04:53:37.483 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.1370000000000002e-06 2024-09-14T04:53:37.483 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.483 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.483 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.483 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.326327+0000", 2024-09-14T04:53:37.483 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1739e-05 2024-09-14T04:53:37.483 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.483 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.484 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:53:37.484 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.326513+0000", 2024-09-14T04:53:37.484 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00018593900000000001 2024-09-14T04:53:37.484 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.484 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.484 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T04:53:37.484 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.326671+0000", 2024-09-14T04:53:37.484 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015817399999999999 2024-09-14T04:53:37.484 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.484 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.484 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:53:37.484 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.331777+0000", 2024-09-14T04:53:37.484 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0051062290000000003 2024-09-14T04:53:37.485 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.485 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.485 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:37.485 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.334887+0000", 2024-09-14T04:53:37.485 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0031100759999999998 2024-09-14T04:53:37.485 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.485 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.485 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:37.485 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.639904+0000", 2024-09-14T04:53:37.485 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.30501712800000003 2024-09-14T04:53:37.485 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.485 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.485 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:53:37.485 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.639962+0000", 2024-09-14T04:53:37.486 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7657e-05 2024-09-14T04:53:37.486 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.486 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.486 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:53:37.486 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:10.640015+0000", 2024-09-14T04:53:37.486 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3118999999999997e-05 2024-09-14T04:53:37.486 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.486 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:53:37.486 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.486 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.486 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.486 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:35472 56.d e1474/1470 56:b70fbc08:::benchmark_data_smithi039_330847_object35471:head v 1474'2243, mlcod=1474'2240)", 2024-09-14T04:53:37.486 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:52:44.887261+0000", 2024-09-14T04:53:37.486 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 52.415378990999997, 2024-09-14T04:53:37.487 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.036227354000000003, 2024-09-14T04:53:37.487 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:53:37.487 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:53:37.487 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:37.487 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:53:37.487 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.487 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:53:37.487 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.887261+0000", 2024-09-14T04:53:37.487 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.487 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.487 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.487 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:53:37.487 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.887261+0000", 2024-09-14T04:53:37.487 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.488 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.488 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.488 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:53:37.488 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.887262+0000", 2024-09-14T04:53:37.488 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.2099999999999995e-07 2024-09-14T04:53:37.488 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.488 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.488 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:53:37.488 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.887294+0000", 2024-09-14T04:53:37.488 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.1764999999999998e-05 2024-09-14T04:53:37.488 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.488 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.488 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:53:37.489 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.887305+0000", 2024-09-14T04:53:37.489 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1929999999999999e-05 2024-09-14T04:53:37.489 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.489 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.489 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:53:37.489 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.887313+0000", 2024-09-14T04:53:37.489 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.2429999999999999e-06 2024-09-14T04:53:37.489 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.489 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.489 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.489 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.887374+0000", 2024-09-14T04:53:37.489 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1184000000000002e-05 2024-09-14T04:53:37.489 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.489 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.489 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:53:37.490 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.887392+0000", 2024-09-14T04:53:37.490 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8099999999999999e-05 2024-09-14T04:53:37.490 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.490 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.490 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:53:37.490 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.923438+0000", 2024-09-14T04:53:37.490 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.036046158000000002 2024-09-14T04:53:37.490 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.490 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.490 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:53:37.490 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.923488+0000", 2024-09-14T04:53:37.491 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.0052999999999997e-05 2024-09-14T04:53:37.491 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.491 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:53:37.491 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.491 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.491 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.491 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:35478 56.0 e1474/1470 56:0b036616:::benchmark_data_smithi039_330847_object35477:head v 1474'2166, mlcod=1474'2163)", 2024-09-14T04:53:37.491 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:52:44.887716+0000", 2024-09-14T04:53:37.491 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 52.414923619, 2024-09-14T04:53:37.491 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.038092978999999999, 2024-09-14T04:53:37.491 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:53:37.491 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:53:37.491 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:37.491 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:53:37.492 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.492 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:53:37.492 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.887716+0000", 2024-09-14T04:53:37.492 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.492 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.492 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.492 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:53:37.492 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.887716+0000", 2024-09-14T04:53:37.492 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.492 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.492 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.492 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:53:37.492 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.887717+0000", 2024-09-14T04:53:37.492 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.3699999999999999e-07 2024-09-14T04:53:37.493 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.493 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.493 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:53:37.493 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.887838+0000", 2024-09-14T04:53:37.493 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000120526 2024-09-14T04:53:37.493 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.493 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.493 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:53:37.493 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.887847+0000", 2024-09-14T04:53:37.493 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.8120000000000002e-06 2024-09-14T04:53:37.493 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.493 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.493 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:53:37.493 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.887854+0000", 2024-09-14T04:53:37.494 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.754e-06 2024-09-14T04:53:37.494 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.494 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.494 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.494 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.887907+0000", 2024-09-14T04:53:37.494 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.2951999999999997e-05 2024-09-14T04:53:37.494 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.494 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.494 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:53:37.494 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.887925+0000", 2024-09-14T04:53:37.494 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8175000000000001e-05 2024-09-14T04:53:37.494 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.494 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.494 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:53:37.495 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.925712+0000", 2024-09-14T04:53:37.495 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.037787045999999998 2024-09-14T04:53:37.495 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.495 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.495 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:53:37.495 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.925809+0000", 2024-09-14T04:53:37.495 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.6877000000000001e-05 2024-09-14T04:53:37.495 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.495 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:53:37.495 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.495 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.495 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.495 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:35480 56.1 e1474/1470 56:87a070db:::benchmark_data_smithi039_330847_object35479:head v 1474'2195, mlcod=1474'2192)", 2024-09-14T04:53:37.495 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:52:44.888173+0000", 2024-09-14T04:53:37.496 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 52.414466462, 2024-09-14T04:53:37.496 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.039000142000000002, 2024-09-14T04:53:37.496 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:53:37.496 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:53:37.496 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:37.496 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:53:37.496 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.496 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:53:37.496 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.888173+0000", 2024-09-14T04:53:37.496 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.496 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.496 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.496 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:53:37.496 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.888173+0000", 2024-09-14T04:53:37.497 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.497 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.497 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.497 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:53:37.497 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.888174+0000", 2024-09-14T04:53:37.497 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8500000000000001e-07 2024-09-14T04:53:37.497 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.497 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.497 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:53:37.497 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.888217+0000", 2024-09-14T04:53:37.497 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3368000000000002e-05 2024-09-14T04:53:37.497 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.497 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.497 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:53:37.498 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.888228+0000", 2024-09-14T04:53:37.498 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0197000000000001e-05 2024-09-14T04:53:37.498 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.498 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.498 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:53:37.498 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.888234+0000", 2024-09-14T04:53:37.498 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0079999999999999e-06 2024-09-14T04:53:37.498 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.498 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.498 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.498 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.888277+0000", 2024-09-14T04:53:37.498 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2948999999999998e-05 2024-09-14T04:53:37.498 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.498 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.499 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:53:37.499 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.888293+0000", 2024-09-14T04:53:37.499 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.66e-05 2024-09-14T04:53:37.499 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.499 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.499 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:53:37.499 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.927130+0000", 2024-09-14T04:53:37.499 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.038836672000000003 2024-09-14T04:53:37.499 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.499 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.499 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:53:37.499 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.927174+0000", 2024-09-14T04:53:37.499 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3662999999999999e-05 2024-09-14T04:53:37.499 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.500 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:53:37.500 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.500 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.500 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.500 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:35479 56.0 e1474/1470 56:02704f1a:::benchmark_data_smithi039_330847_object35478:head v 1474'2167, mlcod=1474'2163)", 2024-09-14T04:53:37.500 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:52:44.889501+0000", 2024-09-14T04:53:37.500 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 52.413138955999997, 2024-09-14T04:53:37.500 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.036340045000000001, 2024-09-14T04:53:37.500 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:53:37.500 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:53:37.500 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:37.500 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:53:37.500 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.500 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:53:37.501 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.889501+0000", 2024-09-14T04:53:37.501 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.501 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.501 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.501 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:53:37.501 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.889501+0000", 2024-09-14T04:53:37.501 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.501 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.501 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.501 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:53:37.501 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.889501+0000", 2024-09-14T04:53:37.501 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.75e-07 2024-09-14T04:53:37.501 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.501 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.502 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:53:37.502 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.889531+0000", 2024-09-14T04:53:37.502 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.9827000000000001e-05 2024-09-14T04:53:37.502 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.502 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.502 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:53:37.502 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.923391+0000", 2024-09-14T04:53:37.502 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.033859325000000003 2024-09-14T04:53:37.502 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.502 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.502 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:53:37.502 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.923405+0000", 2024-09-14T04:53:37.502 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4799e-05 2024-09-14T04:53:37.502 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.503 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.503 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.503 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.923552+0000", 2024-09-14T04:53:37.503 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014648499999999999 2024-09-14T04:53:37.503 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.503 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.503 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:53:37.503 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.923570+0000", 2024-09-14T04:53:37.503 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7926000000000001e-05 2024-09-14T04:53:37.503 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.503 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.503 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:53:37.503 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.925813+0000", 2024-09-14T04:53:37.504 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0022433359999999999 2024-09-14T04:53:37.504 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.504 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.504 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:53:37.504 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.925841+0000", 2024-09-14T04:53:37.504 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.7772000000000001e-05 2024-09-14T04:53:37.504 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.504 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:53:37.504 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.504 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.504 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.504 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:35488 56.d e1474/1470 56:bb015cc1:::benchmark_data_smithi039_330847_object35487:head v 1474'2244, mlcod=1474'2241)", 2024-09-14T04:53:37.504 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:52:44.890718+0000", 2024-09-14T04:53:37.504 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 52.411921862, 2024-09-14T04:53:37.505 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.036293010000000001, 2024-09-14T04:53:37.505 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:53:37.505 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:53:37.505 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:37.505 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:53:37.505 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.505 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:53:37.505 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.890718+0000", 2024-09-14T04:53:37.505 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.505 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.505 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.505 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:53:37.505 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.890718+0000", 2024-09-14T04:53:37.505 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.506 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.506 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.506 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:53:37.506 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.890719+0000", 2024-09-14T04:53:37.506 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.6400000000000001e-07 2024-09-14T04:53:37.506 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.506 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.506 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:53:37.506 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.890744+0000", 2024-09-14T04:53:37.506 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4952000000000001e-05 2024-09-14T04:53:37.506 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.506 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.506 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:53:37.507 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.923389+0000", 2024-09-14T04:53:37.507 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.032645497000000002 2024-09-14T04:53:37.507 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.507 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.507 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:53:37.507 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.923405+0000", 2024-09-14T04:53:37.507 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6096999999999999e-05 2024-09-14T04:53:37.507 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.507 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.507 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.507 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.923513+0000", 2024-09-14T04:53:37.507 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00010763400000000001 2024-09-14T04:53:37.507 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.507 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.508 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:53:37.508 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.923534+0000", 2024-09-14T04:53:37.508 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1225000000000001e-05 2024-09-14T04:53:37.508 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.508 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.508 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:53:37.508 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.926926+0000", 2024-09-14T04:53:37.508 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0033919890000000002 2024-09-14T04:53:37.508 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.508 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.508 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:53:37.508 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.927011+0000", 2024-09-14T04:53:37.508 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.4752000000000004e-05 2024-09-14T04:53:37.509 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.509 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:53:37.509 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.509 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.509 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.509 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:40551 56.0 e1474/1470 56:0d68702d:::benchmark_data_smithi039_330847_object40550:head v 1474'2482, mlcod=1474'2480)", 2024-09-14T04:53:37.509 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:52:48.082357+0000", 2024-09-14T04:53:37.509 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 49.220282791999999, 2024-09-14T04:53:37.509 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.109014311, 2024-09-14T04:53:37.509 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:53:37.509 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:53:37.509 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:37.509 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:53:37.509 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.510 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:53:37.510 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082357+0000", 2024-09-14T04:53:37.510 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.510 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.510 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.510 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:53:37.510 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082357+0000", 2024-09-14T04:53:37.510 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.510 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.510 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.510 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:53:37.510 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082358+0000", 2024-09-14T04:53:37.510 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9999999999999996e-07 2024-09-14T04:53:37.510 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.511 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.511 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:53:37.511 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082422+0000", 2024-09-14T04:53:37.511 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3972000000000003e-05 2024-09-14T04:53:37.511 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.511 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.511 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:53:37.511 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082432+0000", 2024-09-14T04:53:37.511 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.04e-05 2024-09-14T04:53:37.511 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.511 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.511 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:53:37.511 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082438+0000", 2024-09-14T04:53:37.511 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.9950000000000002e-06 2024-09-14T04:53:37.512 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.512 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.512 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.512 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082493+0000", 2024-09-14T04:53:37.512 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4939999999999999e-05 2024-09-14T04:53:37.512 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.512 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.512 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:53:37.512 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082511+0000", 2024-09-14T04:53:37.512 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8190999999999999e-05 2024-09-14T04:53:37.512 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.512 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.512 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:53:37.512 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.191342+0000", 2024-09-14T04:53:37.513 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.10883108599999999 2024-09-14T04:53:37.513 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.513 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.513 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:53:37.513 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.191371+0000", 2024-09-14T04:53:37.513 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.8926999999999999e-05 2024-09-14T04:53:37.513 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.513 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:53:37.513 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.513 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.513 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.513 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:40553 56.2 e1474/1470 56:472e3c32:::benchmark_data_smithi039_330847_object40552:head v 1474'2557, mlcod=1474'2553)", 2024-09-14T04:53:37.513 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:52:48.082671+0000", 2024-09-14T04:53:37.513 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 49.219968668999996, 2024-09-14T04:53:37.514 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.10865520100000001, 2024-09-14T04:53:37.514 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:53:37.514 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:53:37.514 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:37.514 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:53:37.514 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.514 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:53:37.514 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082671+0000", 2024-09-14T04:53:37.514 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.514 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.514 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.514 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:53:37.514 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082671+0000", 2024-09-14T04:53:37.514 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.515 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.515 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.515 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:53:37.515 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082672+0000", 2024-09-14T04:53:37.515 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9299999999999997e-07 2024-09-14T04:53:37.515 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.515 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.515 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:53:37.515 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082756+0000", 2024-09-14T04:53:37.515 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.4225000000000001e-05 2024-09-14T04:53:37.515 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.515 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.515 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:53:37.515 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082766+0000", 2024-09-14T04:53:37.516 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.8539999999999992e-06 2024-09-14T04:53:37.516 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.516 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.516 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:53:37.516 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082771+0000", 2024-09-14T04:53:37.516 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.8180000000000003e-06 2024-09-14T04:53:37.516 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.516 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.516 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.516 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082841+0000", 2024-09-14T04:53:37.516 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9877999999999997e-05 2024-09-14T04:53:37.516 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.516 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.516 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:53:37.517 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082880+0000", 2024-09-14T04:53:37.517 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.9356000000000001e-05 2024-09-14T04:53:37.517 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.517 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.517 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:53:37.517 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.191280+0000", 2024-09-14T04:53:37.517 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.10840013900000001 2024-09-14T04:53:37.517 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.517 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.517 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:53:37.518 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.191326+0000", 2024-09-14T04:53:37.518 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6238e-05 2024-09-14T04:53:37.518 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.518 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:53:37.518 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.518 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.518 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.518 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:40555 56.1 e1474/1470 56:8f6c7fc0:::benchmark_data_smithi039_330847_object40554:head v 1474'2517, mlcod=1474'2515)", 2024-09-14T04:53:37.518 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:52:48.082815+0000", 2024-09-14T04:53:37.518 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 49.219824973000001, 2024-09-14T04:53:37.518 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.108515476, 2024-09-14T04:53:37.518 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:53:37.518 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:53:37.518 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:37.519 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:53:37.519 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.519 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:53:37.519 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082815+0000", 2024-09-14T04:53:37.519 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.519 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.519 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.519 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:53:37.519 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082815+0000", 2024-09-14T04:53:37.519 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.519 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.519 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.519 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:53:37.519 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082815+0000", 2024-09-14T04:53:37.520 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9100000000000004e-07 2024-09-14T04:53:37.520 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.520 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.520 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:53:37.520 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082879+0000", 2024-09-14T04:53:37.520 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3402e-05 2024-09-14T04:53:37.520 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.520 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.520 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:53:37.520 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082888+0000", 2024-09-14T04:53:37.520 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.1859999999999998e-06 2024-09-14T04:53:37.520 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.520 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.521 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:53:37.521 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082894+0000", 2024-09-14T04:53:37.521 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.9510000000000003e-06 2024-09-14T04:53:37.521 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.521 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.521 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.521 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082979+0000", 2024-09-14T04:53:37.521 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.4847999999999998e-05 2024-09-14T04:53:37.521 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.521 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.521 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:53:37.521 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082996+0000", 2024-09-14T04:53:37.521 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6914000000000001e-05 2024-09-14T04:53:37.521 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.521 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.522 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:53:37.522 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.191291+0000", 2024-09-14T04:53:37.522 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.108295473 2024-09-14T04:53:37.522 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.522 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.522 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:53:37.522 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.191330+0000", 2024-09-14T04:53:37.522 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.9211000000000001e-05 2024-09-14T04:53:37.522 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.522 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:53:37.522 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.522 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.522 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.522 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:40554 56.0 e1474/1470 56:02216655:::benchmark_data_smithi039_330847_object40553:head v 1474'2483, mlcod=1474'2480)", 2024-09-14T04:53:37.523 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:52:48.085185+0000", 2024-09-14T04:53:37.523 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 49.217454562, 2024-09-14T04:53:37.523 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.10836931399999999, 2024-09-14T04:53:37.523 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:53:37.523 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:53:37.523 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:37.523 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:53:37.523 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.523 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:53:37.523 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.085185+0000", 2024-09-14T04:53:37.523 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.523 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.523 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.524 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:53:37.524 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.085185+0000", 2024-09-14T04:53:37.524 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.524 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.524 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.524 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:53:37.524 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.085186+0000", 2024-09-14T04:53:37.524 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.8299999999999995e-07 2024-09-14T04:53:37.524 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.524 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.524 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:53:37.524 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.085247+0000", 2024-09-14T04:53:37.524 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0677000000000002e-05 2024-09-14T04:53:37.524 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.525 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.525 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:53:37.525 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.191274+0000", 2024-09-14T04:53:37.525 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.106027363 2024-09-14T04:53:37.525 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.525 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.525 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:53:37.525 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.191286+0000", 2024-09-14T04:53:37.525 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2101e-05 2024-09-14T04:53:37.525 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.525 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.525 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.525 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.191396+0000", 2024-09-14T04:53:37.525 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000109954 2024-09-14T04:53:37.526 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.526 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.526 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:53:37.526 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.191416+0000", 2024-09-14T04:53:37.526 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9295000000000001e-05 2024-09-14T04:53:37.526 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.526 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.526 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:53:37.526 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.193524+0000", 2024-09-14T04:53:37.526 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0021085409999999998 2024-09-14T04:53:37.526 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.526 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.526 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:53:37.527 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.193555+0000", 2024-09-14T04:53:37.527 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.04e-05 2024-09-14T04:53:37.527 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.527 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:53:37.527 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.527 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.527 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.527 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:43086 56.1 e1474/1470 56:8f6c52b1:::benchmark_data_smithi039_330847_object43085:head v 1474'2678, mlcod=1474'2674)", 2024-09-14T04:53:37.527 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:52:49.514491+0000", 2024-09-14T04:53:37.527 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 47.788148888000002, 2024-09-14T04:53:37.527 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.021626408, 2024-09-14T04:53:37.527 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:53:37.527 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:53:37.527 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:37.528 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:53:37.528 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.528 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:53:37.528 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:49.514491+0000", 2024-09-14T04:53:37.528 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.528 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.528 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.528 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:53:37.528 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:49.514491+0000", 2024-09-14T04:53:37.528 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.528 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.528 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.528 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:53:37.528 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:49.514491+0000", 2024-09-14T04:53:37.529 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.7099999999999998e-07 2024-09-14T04:53:37.529 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.529 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.529 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:53:37.529 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:49.514528+0000", 2024-09-14T04:53:37.529 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.7030000000000003e-05 2024-09-14T04:53:37.529 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.529 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.529 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:53:37.529 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:49.514536+0000", 2024-09-14T04:53:37.529 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.4340000000000003e-06 2024-09-14T04:53:37.529 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.529 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.530 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:53:37.530 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:49.514541+0000", 2024-09-14T04:53:37.530 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1340000000000003e-06 2024-09-14T04:53:37.530 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.530 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.530 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.530 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:49.517273+0000", 2024-09-14T04:53:37.530 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0027324070000000001 2024-09-14T04:53:37.530 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.530 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.530 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:53:37.530 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:49.517293+0000", 2024-09-14T04:53:37.530 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0123999999999999e-05 2024-09-14T04:53:37.530 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.530 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.531 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:53:37.531 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:49.536073+0000", 2024-09-14T04:53:37.531 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.018779884 2024-09-14T04:53:37.531 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.531 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.531 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:53:37.531 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:49.536117+0000", 2024-09-14T04:53:37.531 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4124e-05 2024-09-14T04:53:37.531 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.531 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:53:37.531 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.531 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.531 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.531 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:43674 56.d e1474/1470 56:b225c2bb:::benchmark_data_smithi039_330847_object43673:head v 1474'2759, mlcod=1474'2757)", 2024-09-14T04:53:37.532 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:52:49.841039+0000", 2024-09-14T04:53:37.532 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 47.461600900999997, 2024-09-14T04:53:37.532 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.021630132999999999, 2024-09-14T04:53:37.532 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:53:37.532 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:53:37.532 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:37.532 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:53:37.532 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.532 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:53:37.532 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:49.841039+0000", 2024-09-14T04:53:37.532 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.532 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.532 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.532 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:53:37.533 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:49.841039+0000", 2024-09-14T04:53:37.533 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.533 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.533 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.533 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:53:37.533 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:49.841040+0000", 2024-09-14T04:53:37.533 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.6700000000000002e-07 2024-09-14T04:53:37.533 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.533 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.533 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:53:37.534 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:49.841074+0000", 2024-09-14T04:53:37.534 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.3828999999999997e-05 2024-09-14T04:53:37.534 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.534 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.534 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:53:37.534 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:49.841087+0000", 2024-09-14T04:53:37.534 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3580999999999999e-05 2024-09-14T04:53:37.534 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.534 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.534 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:53:37.534 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:49.841095+0000", 2024-09-14T04:53:37.534 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.1499999999999999e-06 2024-09-14T04:53:37.534 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.534 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.534 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.535 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:49.841710+0000", 2024-09-14T04:53:37.535 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00061497099999999996 2024-09-14T04:53:37.535 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.535 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.535 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:53:37.535 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:49.841728+0000", 2024-09-14T04:53:37.535 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7444000000000001e-05 2024-09-14T04:53:37.535 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.535 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.535 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:53:37.535 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:49.859638+0000", 2024-09-14T04:53:37.535 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.017910605 2024-09-14T04:53:37.535 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.535 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.536 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:53:37.536 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:49.862669+0000", 2024-09-14T04:53:37.536 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.003030586 2024-09-14T04:53:37.536 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.536 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:53:37.536 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.536 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.536 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.536 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:92382 56.0 e1482/1470 56:0050e9f1:::benchmark_data_smithi039_330847_object92381:head v 1482'5702, mlcod=1482'5698)", 2024-09-14T04:53:37.536 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:53:34.198336+0000", 2024-09-14T04:53:37.536 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 3.1043036690000001, 2024-09-14T04:53:37.536 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.021867500000000002, 2024-09-14T04:53:37.536 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:53:37.536 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:53:37.537 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:37.537 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:53:37.537 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.537 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:53:37.537 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:53:34.198336+0000", 2024-09-14T04:53:37.537 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.537 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.537 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.537 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:53:37.537 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:53:34.198336+0000", 2024-09-14T04:53:37.537 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:53:37.537 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.537 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.538 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:53:37.538 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:53:34.198337+0000", 2024-09-14T04:53:37.538 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.5999999999999991e-07 2024-09-14T04:53:37.538 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.538 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.538 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:53:37.538 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:53:34.198405+0000", 2024-09-14T04:53:37.538 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7510999999999998e-05 2024-09-14T04:53:37.538 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.538 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.538 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:53:37.538 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:53:34.198424+0000", 2024-09-14T04:53:37.538 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9655000000000001e-05 2024-09-14T04:53:37.538 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.539 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.539 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:53:37.539 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:53:34.198432+0000", 2024-09-14T04:53:37.539 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.4590000000000001e-06 2024-09-14T04:53:37.539 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.539 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.539 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:53:37.539 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:53:34.200175+0000", 2024-09-14T04:53:37.539 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001742923 2024-09-14T04:53:37.539 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.539 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.539 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:53:37.539 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:53:34.200209+0000", 2024-09-14T04:53:37.539 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.4477000000000002e-05 2024-09-14T04:53:37.540 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.540 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.540 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:53:37.540 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:53:34.220023+0000", 2024-09-14T04:53:37.540 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.019814156999999999 2024-09-14T04:53:37.540 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:53:37.540 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:53:37.540 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:53:37.540 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:53:34.220204+0000", 2024-09-14T04:53:37.540 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00018035800000000001 2024-09-14T04:53:37.540 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.540 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:53:37.540 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.540 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:53:37.541 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:53:37.541 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:53:37.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:37.542 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:37.542 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:37.643 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:37.676 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:37.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:37.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:37.778 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:37.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:37.817 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:37.817 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:37.918 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:37.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:37.954 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:37.954 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:38.012 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 67 16 95969 95953 89.4995 58 0.0326804 0.0111683 2024-09-14T04:53:38.055 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:38.094 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:38.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:38.095 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:38.196 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:38.234 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:38.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:38.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:38.335 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:38.369 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:38.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:38.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:38.471 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:38.508 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:38.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:38.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:38.609 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:38.615 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:38 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:53:38.615 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:38 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:53:38.615 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:38 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:53:38.615 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:38 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:53:38.644 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:38.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:38.646 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-14T04:53:38.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:38.729 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:38 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:53:38.730 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:38 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:53:38.747 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:38.816 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:38.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:38.817 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:38.918 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:38.926 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:53:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:53:38] "GET /metrics HTTP/1.1" 200 36800 "" "Prometheus/2.51.0" 2024-09-14T04:53:38.960 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:38.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:38.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:39.012 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 68 16 96874 96858 89.0151 56.5625 0.0129701 0.0112276 2024-09-14T04:53:39.063 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:39.073 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:53:39.097 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:39.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:39.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:39.199 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:39.227 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:39.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:39.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:39.266 INFO:teuthology.orchestra.run.smithi039.stdout:moved 2 / 297 (0.673401%) 2024-09-14T04:53:39.266 INFO:teuthology.orchestra.run.smithi039.stdout:avg 37.125 2024-09-14T04:53:39.267 INFO:teuthology.orchestra.run.smithi039.stdout:stddev 2.8477 -> 2.89126 (expected baseline 5.69951) 2024-09-14T04:53:39.267 INFO:teuthology.orchestra.run.smithi039.stdout:min osd.2 with 34 -> 34 pgs (0.915825 -> 0.915825 * mean) 2024-09-14T04:53:39.267 INFO:teuthology.orchestra.run.smithi039.stdout:max osd.4 with 43 -> 43 pgs (1.15825 -> 1.15825 * mean) 2024-09-14T04:53:39.267 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:53:39.267 INFO:teuthology.orchestra.run.smithi039.stdout:oload 1000 2024-09-14T04:53:39.267 INFO:teuthology.orchestra.run.smithi039.stdout:max_change 1 2024-09-14T04:53:39.267 INFO:teuthology.orchestra.run.smithi039.stdout:max_change_osds 4 2024-09-14T04:53:39.267 INFO:teuthology.orchestra.run.smithi039.stdout:average_utilization 0.0297 2024-09-14T04:53:39.267 INFO:teuthology.orchestra.run.smithi039.stdout:overload_utilization 0.2967 2024-09-14T04:53:39.267 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3 weight 0.8810 -> 0.9236 2024-09-14T04:53:39.267 INFO:teuthology.orchestra.run.smithi039.stderr:no change 2024-09-14T04:53:39.329 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:39.375 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:39.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:39.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:39.477 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:39.496 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:39 smithi129 ceph-mon[101316]: pgmap v2772: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 95 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s; 3571/275556 objects degraded (1.296%); 5380/275556 objects misplaced (1.952%); 3.7 MiB/s, 58 objects/s recovering 2024-09-14T04:53:39.511 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:39.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:39.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:39.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:39 smithi039 ceph-mon[106470]: pgmap v2772: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 95 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s; 3571/275556 objects degraded (1.296%); 5380/275556 objects misplaced (1.952%); 3.7 MiB/s, 58 objects/s recovering 2024-09-14T04:53:39.540 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:53:39 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T04:53:39.264+0000 7f268b1da640 -1 mgr.server reply reply (1) Operation not permitted no change 2024-09-14T04:53:39.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:39 smithi039 ceph-mon[111235]: pgmap v2772: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 95 active+clean; 5.6 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s; 3571/275556 objects degraded (1.296%); 5380/275556 objects misplaced (1.952%); 3.7 MiB/s, 58 objects/s recovering 2024-09-14T04:53:39.614 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:39.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:39.646 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:39.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:39.747 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:39.780 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:39.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:39.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:39.882 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:39.918 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:39.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:39.920 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:40.012 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 69 16 97788 97772 88.5528 57.125 0.00917617 0.0112866 2024-09-14T04:53:40.021 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:40.055 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:40.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:40.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:40.157 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:40.196 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:40.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:40.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:40.298 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:40.326 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:53:40.332 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:40.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:40.333 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:40.353 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:40 smithi039 ceph-mon[106470]: from='client.55567 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:53:40.435 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:40.467 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:40.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:40.468 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:40.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:40 smithi129 ceph-mon[101316]: from='client.55567 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:53:40.569 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:40.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:40.623 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:40.623 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:40.673 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:40 smithi039 ceph-mon[111235]: from='client.55567 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2024-09-14T04:53:40.723 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:40.754 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:40.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:40.755 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:40.856 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:40.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:40.903 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:40.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:41.004 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:41.012 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 70 16 98694 98678 88.0967 56.625 0.0252746 0.0113459 2024-09-14T04:53:41.046 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:41.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:41.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:41.150 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:41.190 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:41.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:41.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:41.292 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:41.324 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:41.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:41.325 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:41.426 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:41.462 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:41.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:41.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:41.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:41 smithi039 ceph-mon[106470]: pgmap v2773: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 95 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 890 op/s; 3571/289539 objects degraded (1.233%); 5052/289539 objects misplaced (1.745%); 3.9 MiB/s, 62 objects/s recovering 2024-09-14T04:53:41.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:41 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 3571/289539 objects degraded (1.233%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:41.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:41 smithi039 ceph-mon[111235]: pgmap v2773: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 95 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 890 op/s; 3571/289539 objects degraded (1.233%); 5052/289539 objects misplaced (1.745%); 3.9 MiB/s, 62 objects/s recovering 2024-09-14T04:53:41.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:41 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 3571/289539 objects degraded (1.233%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:41.565 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:41.610 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:41.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:41.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:41.625 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:41 smithi129 ceph-mon[101316]: pgmap v2773: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 95 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 56 MiB/s wr, 890 op/s; 3571/289539 objects degraded (1.233%); 5052/289539 objects misplaced (1.745%); 3.9 MiB/s, 62 objects/s recovering 2024-09-14T04:53:41.625 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:41 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 3571/289539 objects degraded (1.233%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:41.712 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:41.742 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:41.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:41.743 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:41.845 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:41.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:41.892 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:41.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:41.993 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:42.012 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 71 16 99456 99440 87.5266 47.625 0.0197608 0.0114169 2024-09-14T04:53:42.025 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:42.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:42.026 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:42.127 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:42.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:42.164 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:42.164 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:42.265 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:42.302 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:42.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:42.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:42.404 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:42.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:42.439 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:42.439 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:42.540 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:42.548 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:53:42.548 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:53:42.548 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:53:42.548 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:53:42.549 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:53:42.549 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:53:42.573 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:53:42.574 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:53:42.590 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:42.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:42.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:42.611 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:53:42.611 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:53:42.642 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:53:42.642 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:53:42.674 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:53:42.674 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:53:42.692 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:42.710 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:53:42.710 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:53:42.725 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:42.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:42.726 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:42.827 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:42.875 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:42.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:42.876 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:42.977 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:43.000 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:43.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:43.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:43.012 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 72 16 100334 100318 87.073 54.875 0.02461 0.011475 2024-09-14T04:53:43.102 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:43.140 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:43.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:43.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:43.242 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:43.280 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:43.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:43.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:43.383 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:43.424 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:43.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:43.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:43.455 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:43 smithi039 ceph-mon[106470]: pgmap v2774: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 95 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 851 op/s; 3571/289539 objects degraded (1.233%); 5052/289539 objects misplaced (1.745%); 3.7 MiB/s, 59 objects/s recovering 2024-09-14T04:53:43.456 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:43 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2071748583' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:53:43.456 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:43 smithi039 ceph-mon[111235]: pgmap v2774: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 95 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 851 op/s; 3571/289539 objects degraded (1.233%); 5052/289539 objects misplaced (1.745%); 3.7 MiB/s, 59 objects/s recovering 2024-09-14T04:53:43.456 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:43 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2071748583' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:53:43.525 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:43.559 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:43.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:43.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:43.661 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:43.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:43.705 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:43.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:43.719 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:43 smithi129 ceph-mon[101316]: pgmap v2774: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 95 active+clean; 5.9 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 53 MiB/s wr, 851 op/s; 3571/289539 objects degraded (1.233%); 5052/289539 objects misplaced (1.745%); 3.7 MiB/s, 59 objects/s recovering 2024-09-14T04:53:43.719 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:43 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2071748583' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:53:43.806 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:43.847 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:43.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:43.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:43.949 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:43.982 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:43.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:43.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:44.016 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 73 16 101220 101204 86.6376 55.375 0.0160053 0.0115366 2024-09-14T04:53:44.084 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:44.120 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:44.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:44.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:44.222 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:44.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:44.259 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:44.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:44.360 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:44.389 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is set 2024-09-14T04:53:44.391 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:44.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:44.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:44.493 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:44.538 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:44.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:44.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:44.640 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:44.666 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:44 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1997606949' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:53:44.667 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:44 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:53:44.667 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:44 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1997606949' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:53:44.667 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:44 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:53:44.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:44.685 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:44.685 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:44.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:44 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1997606949' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:53:44.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:44 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:53:44.786 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:44.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:44.809 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:44.809 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:44.910 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:44.941 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:44.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:44.943 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:45.004 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:53:45.004 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:53:45.013 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 74 16 102048 102032 86.1661 51.75 0.0118096 0.0115997 2024-09-14T04:53:45.043 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:45.073 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:45.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:45.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:45.093 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-14T04:53:45.093 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 0 1 2024-09-14T04:53:45.174 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:45.209 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:45.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:45.210 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:45.311 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:45.334 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:45.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:45.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:45.436 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:45.482 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:45.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:45.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:45.584 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:45.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:45.620 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:45.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:45.721 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:45.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:45.754 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:45.754 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:45.766 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:45 smithi039 ceph-mon[106470]: pgmap v2775: 99 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s; 3571/297798 objects degraded (1.199%); 4899/297798 objects misplaced (1.645%); 4.0 MiB/s, 64 objects/s recovering 2024-09-14T04:53:45.766 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:45 smithi039 ceph-mon[106470]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:53:45.766 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:45 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:53:45.766 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:45 smithi039 ceph-mon[106470]: osdmap e1483: 8 total, 8 up, 8 in 2024-09-14T04:53:45.766 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:45 smithi039 ceph-mon[111235]: pgmap v2775: 99 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s; 3571/297798 objects degraded (1.199%); 4899/297798 objects misplaced (1.645%); 4.0 MiB/s, 64 objects/s recovering 2024-09-14T04:53:45.766 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:45 smithi039 ceph-mon[111235]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:53:45.766 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:45 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:53:45.766 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:45 smithi039 ceph-mon[111235]: osdmap e1483: 8 total, 8 up, 8 in 2024-09-14T04:53:45.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:45 smithi129 ceph-mon[101316]: pgmap v2775: 99 pgs: 1 active+clean+scrubbing, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 95 active+clean; 6.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 61 MiB/s wr, 971 op/s; 3571/297798 objects degraded (1.199%); 4899/297798 objects misplaced (1.645%); 4.0 MiB/s, 64 objects/s recovering 2024-09-14T04:53:45.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:45 smithi129 ceph-mon[101316]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:53:45.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:45 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:53:45.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:45 smithi129 ceph-mon[101316]: osdmap e1483: 8 total, 8 up, 8 in 2024-09-14T04:53:45.855 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:45.893 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:45.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:45.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:45.996 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:46.014 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 75 16 102772 102756 85.6205 45.25 0.126228 0.0116693 2024-09-14T04:53:46.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:46.035 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:46.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:46.136 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:46.188 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:46.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:46.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:46.290 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:46.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:46.313 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:46.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:46.414 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:46.449 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:46.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:46.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:46.551 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:46.590 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:46.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:46.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:46.661 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:46 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 3571/297798 objects degraded (1.199%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:46.662 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:46 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 3571/297798 objects degraded (1.199%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:46.692 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:46.705 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:53:46.725 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:46.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:46.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:46.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:46 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 3571/297798 objects degraded (1.199%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:46.827 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:46.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:46.865 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:46.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:46.966 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:47.017 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 76 16 103453 103437 85.052 42.5625 0.0101163 0.0117518 2024-09-14T04:53:47.020 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:53:47.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:47.023 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:47.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:47.123 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:47.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:47.156 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:47.156 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:47.257 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:47.294 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:47.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:47.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:47.396 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:47.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:47.439 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:47.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:47.540 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:47.575 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:47.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:47.576 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:47.677 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:47.703 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:47.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:47.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:47 smithi129 ceph-mon[101316]: pgmap v2777: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 95 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s; 3571/302607 objects degraded (1.180%); 4744/302607 objects misplaced (1.568%); 4.0 MiB/s, 63 objects/s recovering 2024-09-14T04:53:47.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:47 smithi039 ceph-mon[106470]: pgmap v2777: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 95 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s; 3571/302607 objects degraded (1.180%); 4744/302607 objects misplaced (1.568%); 4.0 MiB/s, 63 objects/s recovering 2024-09-14T04:53:47.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:47 smithi039 ceph-mon[111235]: pgmap v2777: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 95 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s; 3571/302607 objects degraded (1.180%); 4744/302607 objects misplaced (1.568%); 4.0 MiB/s, 63 objects/s recovering 2024-09-14T04:53:47.801 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:53:47.805 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:47.851 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:47.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:47.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:47.952 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:47.984 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:47.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:47.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:48.016 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 77 16 104213 104197 84.5643 47.5 0.00460959 0.0118135 2024-09-14T04:53:48.086 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:48.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:48.115 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:48.115 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:48.216 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:48.249 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:48.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:48.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:48.351 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:48.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:48.392 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:48.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:48.493 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:48.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:48.530 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:48.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:48.632 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:48.670 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:48.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:48.672 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:48.772 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:48.808 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:48.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:48.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:48.910 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:48.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:48.940 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:48.940 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:49.016 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 78 16 105004 104988 84.1139 49.4375 0.0122202 0.0118837 2024-09-14T04:53:49.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:53:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:53:48] "GET /metrics HTTP/1.1" 200 36800 "" "Prometheus/2.51.0" 2024-09-14T04:53:49.042 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:49.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:49.074 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:49.074 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:49.176 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:49.209 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:49.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:49.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:49.311 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:49.346 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:49.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:49.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:49.449 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:49.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:49.478 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:49.478 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:49.579 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:49.611 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:49.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:49.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:49.713 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:49.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:49.751 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:49.751 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:49.768 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:49 smithi039 ceph-mon[106470]: pgmap v2778: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 95 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s; 3571/302607 objects degraded (1.180%); 4744/302607 objects misplaced (1.568%); 4.0 MiB/s, 63 objects/s recovering 2024-09-14T04:53:49.769 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:49 smithi039 ceph-mon[111235]: pgmap v2778: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 95 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s; 3571/302607 objects degraded (1.180%); 4744/302607 objects misplaced (1.568%); 4.0 MiB/s, 63 objects/s recovering 2024-09-14T04:53:49.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:49 smithi129 ceph-mon[101316]: pgmap v2778: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+clean+scrubbing, 2 active+recovering+undersized+remapped, 95 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s; 3571/302607 objects degraded (1.180%); 4744/302607 objects misplaced (1.568%); 4.0 MiB/s, 63 objects/s recovering 2024-09-14T04:53:49.853 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:49.886 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:49.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:49.887 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:49.988 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:50.016 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 79 16 105780 105764 83.663 48.5 0.0116154 0.0119472 2024-09-14T04:53:50.025 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:50.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:50.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:50.127 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:50.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:50.165 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:50.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:50.266 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:50.299 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:50.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:50.301 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:50.403 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:50.445 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:50.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:50.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:50.547 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:50.583 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:50.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:50.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:50.603 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-14T04:53:50.685 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:50.726 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:50.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:50.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:50.819 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:53:50.828 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:50.875 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:53:50.903 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:50.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:50.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:50.947 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:53:51.005 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:51.016 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:2024-09-14T04:53:51.016467+0000 min lat: 0.00136201 max lat: 2.17883 avg lat: 0.0120061 2024-09-14T04:53:51.016 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:53:51.016 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 80 16 106598 106582 83.2561 51.125 0.0109124 0.0120061 2024-09-14T04:53:51.055 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:51.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:51.056 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:51.157 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:51.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:51.194 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:51.194 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:51.261 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:51 smithi039 ceph-mon[106470]: pgmap v2779: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.4 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 51 MiB/s wr, 816 op/s; 3571/314052 objects degraded (1.137%); 4420/314052 objects misplaced (1.407%); 3.9 MiB/s, 63 objects/s recovering 2024-09-14T04:53:51.261 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:51 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:53:51.261 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:51 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:53:51.261 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:51 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:53:51.262 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:51 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:53:51.262 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:51 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:53:51.262 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:51 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 3571/314052 objects degraded (1.137%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:51.262 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:51 smithi039 ceph-mon[111235]: pgmap v2779: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.4 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 51 MiB/s wr, 816 op/s; 3571/314052 objects degraded (1.137%); 4420/314052 objects misplaced (1.407%); 3.9 MiB/s, 63 objects/s recovering 2024-09-14T04:53:51.262 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:51 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:53:51.262 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:51 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:53:51.262 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:51 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:53:51.262 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:51 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:53:51.262 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:51 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:53:51.263 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:51 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 3571/314052 objects degraded (1.137%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:51.295 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:51.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:51.327 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:51.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:51.339 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:51 smithi129 ceph-mon[101316]: pgmap v2779: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.4 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 51 MiB/s wr, 816 op/s; 3571/314052 objects degraded (1.137%); 4420/314052 objects misplaced (1.407%); 3.9 MiB/s, 63 objects/s recovering 2024-09-14T04:53:51.339 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:51 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:53:51.339 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:51 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:53:51.339 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:51 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:53:51.339 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:51 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:53:51.339 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:51 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:53:51.339 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:51 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 3571/314052 objects degraded (1.137%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:51.428 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:51.465 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:51.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:51.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:51.567 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:51.604 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:51.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:51.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:51.706 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:51.739 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:51.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:51.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:51.841 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:51.869 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:51.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:51.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:51.970 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:52.016 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 81 16 107409 107393 82.854 50.6875 0.0220625 0.012063 2024-09-14T04:53:52.023 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:52.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:52.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:52.124 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:52.148 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:52.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:52.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:52.250 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:52.280 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:52.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:52.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:52.382 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:52.416 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:52.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:52.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:52.518 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:52.551 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:52.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:52.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:52.653 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:52.697 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:52.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:52.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:52.799 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:52.843 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:52.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:52.844 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:52.945 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:53.007 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:53.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:53.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:53.016 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 82 16 108162 108146 82.4175 47.0625 0.0194737 0.0121286 2024-09-14T04:53:53.109 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:53.154 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:53.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:53.155 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:53.161 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.0 to 1 (10000) 2024-09-14T04:53:53.255 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:53.295 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:53.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:53.296 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:53.397 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:53.400 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:53 smithi129 ceph-mon[101316]: pgmap v2780: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.4 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 51 MiB/s wr, 816 op/s; 3571/314052 objects degraded (1.137%); 4420/314052 objects misplaced (1.407%); 3.9 MiB/s, 63 objects/s recovering 2024-09-14T04:53:53.400 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:53 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/572375048' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:53:53.400 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:53 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:53:53.444 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:53.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:53.445 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:53.521 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:53 smithi039 ceph-mon[106470]: pgmap v2780: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.4 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 51 MiB/s wr, 816 op/s; 3571/314052 objects degraded (1.137%); 4420/314052 objects misplaced (1.407%); 3.9 MiB/s, 63 objects/s recovering 2024-09-14T04:53:53.521 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:53 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/572375048' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:53:53.521 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:53 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:53:53.521 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:53 smithi039 ceph-mon[111235]: pgmap v2780: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.4 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 51 MiB/s wr, 816 op/s; 3571/314052 objects degraded (1.137%); 4420/314052 objects misplaced (1.407%); 3.9 MiB/s, 63 objects/s recovering 2024-09-14T04:53:53.522 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:53 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/572375048' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:53:53.522 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:53 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:53:53.545 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:53.570 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:53:53.570 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:53:53.594 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:53.595 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:53.595 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:53.696 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:53.741 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:53.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:53.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:53.843 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:53.877 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:53.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:53.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:53.979 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:54.016 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 83 16 109006 108990 82.06 52.75 0.0177267 0.0121811 2024-09-14T04:53:54.017 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:54.018 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:54.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:54.119 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:54.159 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:54.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:54.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:54.231 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:54 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/572375048' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:53:54.231 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:54 smithi129 ceph-mon[101316]: osdmap e1484: 8 total, 8 up, 8 in 2024-09-14T04:53:54.262 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:54.273 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-14T04:53:54.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:54 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/572375048' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:53:54.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:54 smithi039 ceph-mon[106470]: osdmap e1484: 8 total, 8 up, 8 in 2024-09-14T04:53:54.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:54 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/572375048' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:53:54.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:54 smithi039 ceph-mon[111235]: osdmap e1484: 8 total, 8 up, 8 in 2024-09-14T04:53:54.306 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:54.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:54.309 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:54.410 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:54.452 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:54.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:54.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:54.554 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:54.602 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:54.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:54.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:54.685 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:53:54.704 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:54.744 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:54.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:54.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:54.846 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:54.893 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:54.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:54.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:54.995 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:55.017 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 84 16 109845 109829 81.7073 52.4375 0.0226796 0.0122334 2024-09-14T04:53:55.030 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:55.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:55.032 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:55.133 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:55.178 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:55.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:55.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:55.188 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is unset 2024-09-14T04:53:55.280 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:55.289 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:55 smithi039 ceph-mon[106470]: pgmap v2782: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 42 MiB/s wr, 668 op/s; 3571/321558 objects degraded (1.111%); 4225/321558 objects misplaced (1.314%); 3.4 MiB/s, 54 objects/s recovering 2024-09-14T04:53:55.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:55 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1056104597' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:53:55.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:55 smithi039 ceph-mon[111235]: pgmap v2782: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 42 MiB/s wr, 668 op/s; 3571/321558 objects degraded (1.111%); 4225/321558 objects misplaced (1.314%); 3.4 MiB/s, 54 objects/s recovering 2024-09-14T04:53:55.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:55 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1056104597' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:53:55.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:55.318 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:55.318 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:55.419 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:55.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:55.454 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:55.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:55.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:55 smithi129 ceph-mon[101316]: pgmap v2782: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 42 MiB/s wr, 668 op/s; 3571/321558 objects degraded (1.111%); 4225/321558 objects misplaced (1.314%); 3.4 MiB/s, 54 objects/s recovering 2024-09-14T04:53:55.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:55 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1056104597' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:53:55.554 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:55.589 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:55.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:55.590 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:55.690 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:55.735 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:55.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:55.736 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:55.837 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:55.875 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:55.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:55.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:55.898 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 1 1 2024-09-14T04:53:55.977 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:55.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:55.999 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:55.999 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:56.017 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 85 16 110653 110637 81.3401 50.5 0.0485893 0.0122883 2024-09-14T04:53:56.100 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:56.140 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:56.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:56.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:56.242 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:56.270 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:56.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:56.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:56.373 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:56.414 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:56.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:56.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:56.435 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:56 smithi039 ceph-mon[106470]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:53:56.435 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:56 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1056104597' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:53:56.436 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:56 smithi039 ceph-mon[106470]: osdmap e1485: 8 total, 8 up, 8 in 2024-09-14T04:53:56.436 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:56 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 3571/321558 objects degraded (1.111%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:56.436 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:56 smithi039 ceph-mon[111235]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:53:56.436 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:56 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1056104597' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:53:56.436 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:56 smithi039 ceph-mon[111235]: osdmap e1485: 8 total, 8 up, 8 in 2024-09-14T04:53:56.436 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:56 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 3571/321558 objects degraded (1.111%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:56.517 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:56.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:56 smithi129 ceph-mon[101316]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:53:56.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:56 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1056104597' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:53:56.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:56 smithi129 ceph-mon[101316]: osdmap e1485: 8 total, 8 up, 8 in 2024-09-14T04:53:56.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:56 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 3571/321558 objects degraded (1.111%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:53:56.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:56.574 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:56.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:56.675 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:56.776 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:56.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:56.777 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:56.878 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:56.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:56.923 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:56.923 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:56.929 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:53:56.929 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:53:56.965 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:53:56.965 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:53:56.974 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:53:56.999 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:53:56.999 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:53:57.019 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 86 16 111507 111491 81.0144 53.375 0.0193137 0.0123382 2024-09-14T04:53:57.024 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:57.032 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:53:57.032 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:53:57.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:57.057 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:57.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:57.060 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:53:57.060 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:53:57.085 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:53:57.086 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:53:57.128 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:53:57.129 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:53:57.156 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:53:57.157 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:53:57.158 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:57.181 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:57.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:57.182 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:57.270 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:57 smithi039 ceph-mon[106470]: pgmap v2784: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s; 3571/326052 objects degraded (1.095%); 4066/326052 objects misplaced (1.247%); 5.3 MiB/s, 84 objects/s recovering 2024-09-14T04:53:57.270 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:57 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1726239238' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:53:57.270 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:57 smithi039 ceph-mon[111235]: pgmap v2784: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s; 3571/326052 objects degraded (1.095%); 4066/326052 objects misplaced (1.247%); 5.3 MiB/s, 84 objects/s recovering 2024-09-14T04:53:57.271 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:57 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1726239238' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:53:57.283 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:57.315 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:57.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:57.316 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:57.417 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:57.449 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:57.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:57.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:57.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:57 smithi129 ceph-mon[101316]: pgmap v2784: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s; 3571/326052 objects degraded (1.095%); 4066/326052 objects misplaced (1.247%); 5.3 MiB/s, 84 objects/s recovering 2024-09-14T04:53:57.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:57 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1726239238' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:53:57.551 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:53:57.586 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:57.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:57.588 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:53:57.689 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:57.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:57.752 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:57.752 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:57.853 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:57.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:57.896 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:57.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:57.997 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:58.018 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 87 16 112260 112244 80.6242 47.0625 0.0208245 0.0123974 2024-09-14T04:53:58.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:58.041 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:58.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:58.142 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:58.181 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:58.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:58.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:58.283 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:58.315 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:58.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:58.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:58.339 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:53:58.339 INFO:teuthology.orchestra.run.smithi129.stdout: "size": 20, 2024-09-14T04:53:58.339 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 600, 2024-09-14T04:53:58.339 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [ 2024-09-14T04:53:58.339 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.340 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:104264 55.8 55:1fc585f2:::benchmark_data_smithi039_297155_object104263:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1440)", 2024-09-14T04:53:58.340 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:49:10.240109+0000", 2024-09-14T04:53:58.340 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 288.09581189699998, 2024-09-14T04:53:58.340 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.39979108000000002, 2024-09-14T04:53:58.340 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:53:58.340 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:58.340 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:53:58.340 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:53:58.340 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:53:58.340 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 104264 2024-09-14T04:53:58.340 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.341 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:53:58.341 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.341 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:53:58.341 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.240109+0000", 2024-09-14T04:53:58.341 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.341 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.341 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.341 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:53:58.341 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.240109+0000", 2024-09-14T04:53:58.341 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.341 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.341 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.341 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:53:58.342 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.240107+0000", 2024-09-14T04:53:58.342 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:53:58.342 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.342 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.342 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:53:58.342 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.240140+0000", 2024-09-14T04:53:58.342 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.2925000000000001e-05 2024-09-14T04:53:58.342 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.342 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.342 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:53:58.342 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.240153+0000", 2024-09-14T04:53:58.342 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3074000000000001e-05 2024-09-14T04:53:58.342 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.343 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.343 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:53:58.343 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.240160+0000", 2024-09-14T04:53:58.343 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.0439999999999998e-06 2024-09-14T04:53:58.343 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.343 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.343 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:53:58.343 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.240203+0000", 2024-09-14T04:53:58.343 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.2797999999999999e-05 2024-09-14T04:53:58.343 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.343 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.343 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:53:58.344 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.240306+0000", 2024-09-14T04:53:58.344 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000102959 2024-09-14T04:53:58.344 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.344 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.344 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,6", 2024-09-14T04:53:58.344 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.240386+0000", 2024-09-14T04:53:58.344 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.9819000000000003e-05 2024-09-14T04:53:58.344 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.344 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.344 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:53:58.344 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.322851+0000", 2024-09-14T04:53:58.344 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.082465262999999997 2024-09-14T04:53:58.345 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.345 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.345 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.345 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.322882+0000", 2024-09-14T04:53:58.345 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.0682000000000002e-05 2024-09-14T04:53:58.345 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.345 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.345 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.345 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.639772+0000", 2024-09-14T04:53:58.346 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.31689024900000001 2024-09-14T04:53:58.346 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.346 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.346 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:53:58.346 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.639859+0000", 2024-09-14T04:53:58.346 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.7014000000000004e-05 2024-09-14T04:53:58.346 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.346 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.346 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:53:58.347 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.639900+0000", 2024-09-14T04:53:58.347 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0748999999999999e-05 2024-09-14T04:53:58.347 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.347 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:53:58.347 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.347 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.347 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.347 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:104267 55.8 55:19de0ba9:::benchmark_data_smithi039_297155_object104266:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1440)", 2024-09-14T04:53:58.347 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:49:10.323499+0000", 2024-09-14T04:53:58.348 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 288.01242148, 2024-09-14T04:53:58.348 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.316520142, 2024-09-14T04:53:58.348 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:53:58.348 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:58.348 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:53:58.348 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:53:58.349 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:53:58.349 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 104267 2024-09-14T04:53:58.349 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.349 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:53:58.349 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.349 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:53:58.349 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.323499+0000", 2024-09-14T04:53:58.349 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.349 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.349 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.349 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:53:58.349 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.323499+0000", 2024-09-14T04:53:58.350 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.350 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.350 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.350 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:53:58.350 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.323494+0000", 2024-09-14T04:53:58.350 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999948 2024-09-14T04:53:58.350 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.350 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.350 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:53:58.350 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.323584+0000", 2024-09-14T04:53:58.350 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.0680999999999995e-05 2024-09-14T04:53:58.351 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.351 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.351 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:53:58.351 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.323602+0000", 2024-09-14T04:53:58.351 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.7265000000000002e-05 2024-09-14T04:53:58.351 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.351 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.351 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:53:58.351 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.323610+0000", 2024-09-14T04:53:58.351 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.0139999999999998e-06 2024-09-14T04:53:58.352 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.352 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.352 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:53:58.352 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.325579+0000", 2024-09-14T04:53:58.352 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0019690760000000002 2024-09-14T04:53:58.352 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.352 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.352 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:53:58.352 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.325786+0000", 2024-09-14T04:53:58.352 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00020740600000000001 2024-09-14T04:53:58.352 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.353 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.353 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,6", 2024-09-14T04:53:58.353 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.325940+0000", 2024-09-14T04:53:58.353 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00015370299999999999 2024-09-14T04:53:58.353 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.353 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.353 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:53:58.353 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.329395+0000", 2024-09-14T04:53:58.354 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0034548270000000002 2024-09-14T04:53:58.354 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.354 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.354 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.354 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.332624+0000", 2024-09-14T04:53:58.354 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0032294889999999999 2024-09-14T04:53:58.354 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.354 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.354 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.354 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.639938+0000", 2024-09-14T04:53:58.354 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.307313581 2024-09-14T04:53:58.355 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.355 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.355 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:53:58.355 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.639985+0000", 2024-09-14T04:53:58.355 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6761999999999997e-05 2024-09-14T04:53:58.355 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.355 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.355 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:53:58.356 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.640019+0000", 2024-09-14T04:53:58.356 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.4668000000000001e-05 2024-09-14T04:53:58.356 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.356 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:53:58.356 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.356 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.356 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.357 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:149721 55.9 55:963f2b16:::benchmark_data_smithi039_297155_object25788:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1447)", 2024-09-14T04:53:58.357 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:49:58.468180+0000", 2024-09-14T04:53:58.357 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 239.867740492, 2024-09-14T04:53:58.357 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.34284567700000002, 2024-09-14T04:53:58.357 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:53:58.357 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:58.357 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:53:58.357 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:53:58.358 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:53:58.358 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 149721 2024-09-14T04:53:58.358 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.358 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:53:58.359 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.359 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:53:58.359 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.468180+0000", 2024-09-14T04:53:58.359 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.359 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.359 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.359 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:53:58.359 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.468180+0000", 2024-09-14T04:53:58.359 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.360 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.360 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.360 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:53:58.360 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.468179+0000", 2024-09-14T04:53:58.360 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:53:58.360 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.360 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.360 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:53:58.360 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.468188+0000", 2024-09-14T04:53:58.360 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.1759999999999999e-06 2024-09-14T04:53:58.360 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.360 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.360 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:53:58.360 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.468200+0000", 2024-09-14T04:53:58.361 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2089000000000001e-05 2024-09-14T04:53:58.361 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.361 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.361 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:53:58.361 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.468207+0000", 2024-09-14T04:53:58.361 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7549999999999997e-06 2024-09-14T04:53:58.361 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.361 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.361 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:53:58.361 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.468475+0000", 2024-09-14T04:53:58.361 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00026856999999999998 2024-09-14T04:53:58.361 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.361 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.362 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:53:58.362 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.469536+0000", 2024-09-14T04:53:58.362 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0010609160000000001 2024-09-14T04:53:58.362 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.362 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.362 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,3", 2024-09-14T04:53:58.362 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.469643+0000", 2024-09-14T04:53:58.362 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010696000000000001 2024-09-14T04:53:58.362 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.362 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.362 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:53:58.362 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.479119+0000", 2024-09-14T04:53:58.362 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.009476356 2024-09-14T04:53:58.362 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.363 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.363 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.363 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.479464+0000", 2024-09-14T04:53:58.363 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00034406999999999998 2024-09-14T04:53:58.363 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.363 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.363 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.363 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.810925+0000", 2024-09-14T04:53:58.363 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33146173600000001 2024-09-14T04:53:58.363 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.363 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.363 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:53:58.363 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.810982+0000", 2024-09-14T04:53:58.364 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.6569e-05 2024-09-14T04:53:58.364 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.364 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.364 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:53:58.364 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.811026+0000", 2024-09-14T04:53:58.364 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.3993999999999999e-05 2024-09-14T04:53:58.364 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.364 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:53:58.364 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.364 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.364 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.364 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:149725 55.9 55:90743abc:::benchmark_data_smithi039_297155_object25792:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1447)", 2024-09-14T04:53:58.364 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:49:58.471639+0000", 2024-09-14T04:53:58.365 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 239.86428120299999, 2024-09-14T04:53:58.365 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.34457423300000001, 2024-09-14T04:53:58.365 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:53:58.365 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:58.365 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:53:58.365 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:53:58.365 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:53:58.365 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 149725 2024-09-14T04:53:58.365 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.365 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:53:58.365 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.365 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:53:58.365 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471639+0000", 2024-09-14T04:53:58.365 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.366 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.366 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.366 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:53:58.366 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471639+0000", 2024-09-14T04:53:58.366 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.366 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.366 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.366 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:53:58.366 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471638+0000", 2024-09-14T04:53:58.366 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:53:58.367 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.367 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.367 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:53:58.367 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471644+0000", 2024-09-14T04:53:58.367 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.6960000000000002e-06 2024-09-14T04:53:58.367 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.367 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.367 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:53:58.367 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471656+0000", 2024-09-14T04:53:58.367 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2234e-05 2024-09-14T04:53:58.367 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.367 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.367 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:53:58.368 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471661+0000", 2024-09-14T04:53:58.368 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2299999999999999e-06 2024-09-14T04:53:58.368 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.368 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.368 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:53:58.368 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.474615+0000", 2024-09-14T04:53:58.368 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.002953893 2024-09-14T04:53:58.369 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.369 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.369 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:53:58.369 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.474746+0000", 2024-09-14T04:53:58.369 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013021800000000001 2024-09-14T04:53:58.369 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.369 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.369 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,3", 2024-09-14T04:53:58.369 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.474861+0000", 2024-09-14T04:53:58.369 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000115696 2024-09-14T04:53:58.369 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.369 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.369 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:53:58.369 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.479246+0000", 2024-09-14T04:53:58.370 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0043852259999999999 2024-09-14T04:53:58.370 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.370 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.370 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.370 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.496383+0000", 2024-09-14T04:53:58.370 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.017136703999999999 2024-09-14T04:53:58.370 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.370 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.370 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.370 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.816111+0000", 2024-09-14T04:53:58.370 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.31972738299999998 2024-09-14T04:53:58.370 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.370 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.371 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:53:58.371 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.816168+0000", 2024-09-14T04:53:58.371 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.7682000000000001e-05 2024-09-14T04:53:58.371 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.371 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.371 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:53:58.371 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.816214+0000", 2024-09-14T04:53:58.371 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.5391e-05 2024-09-14T04:53:58.371 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.371 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:53:58.371 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.371 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.371 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.371 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:27141 56.2 56:43cfe10a:::benchmark_data_smithi039_330847_object27140:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1473)", 2024-09-14T04:53:58.372 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:40.092917+0000", 2024-09-14T04:53:58.372 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 78.243003212000005, 2024-09-14T04:53:58.372 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.15434695600000001, 2024-09-14T04:53:58.372 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:53:58.372 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:58.372 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:53:58.372 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:53:58.372 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:53:58.372 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 27141 2024-09-14T04:53:58.372 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.372 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:53:58.372 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.372 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:53:58.372 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:40.092917+0000", 2024-09-14T04:53:58.373 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.373 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.373 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.373 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:53:58.373 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:40.092917+0000", 2024-09-14T04:53:58.373 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.373 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.373 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.373 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:53:58.373 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:40.092916+0000", 2024-09-14T04:53:58.373 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:53:58.373 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.373 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.374 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:53:58.375 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:53:58.375 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:40.092950+0000", 2024-09-14T04:53:58.375 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3748000000000003e-05 2024-09-14T04:53:58.376 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.376 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.376 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:53:58.376 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:40.092963+0000", 2024-09-14T04:53:58.376 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2425e-05 2024-09-14T04:53:58.376 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.376 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.376 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:53:58.376 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:40.092970+0000", 2024-09-14T04:53:58.376 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.1250000000000004e-06 2024-09-14T04:53:58.376 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.376 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.377 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:53:58.377 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:40.240657+0000", 2024-09-14T04:53:58.377 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.14768695000000001 2024-09-14T04:53:58.378 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.378 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.378 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:53:58.378 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:40.240832+0000", 2024-09-14T04:53:58.378 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00017547999999999999 2024-09-14T04:53:58.378 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.379 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.379 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:53:58.379 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:40.240980+0000", 2024-09-14T04:53:58.379 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000148262 2024-09-14T04:53:58.379 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.379 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.379 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.379 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:40.244406+0000", 2024-09-14T04:53:58.379 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0034257070000000001 2024-09-14T04:53:58.379 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.380 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.380 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:53:58.380 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:40.244575+0000", 2024-09-14T04:53:58.380 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0001685 2024-09-14T04:53:58.380 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.380 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.380 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.380 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:40.247188+0000", 2024-09-14T04:53:58.380 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0026134280000000001 2024-09-14T04:53:58.381 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.381 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.381 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:53:58.381 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:40.247228+0000", 2024-09-14T04:53:58.381 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9765000000000003e-05 2024-09-14T04:53:58.381 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.381 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.381 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:53:58.381 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:40.247264+0000", 2024-09-14T04:53:58.381 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6563999999999998e-05 2024-09-14T04:53:58.382 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.382 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:53:58.382 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.382 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.382 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.382 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:27148 56.c 56:37157259:::benchmark_data_smithi039_330847_object27147:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1473)", 2024-09-14T04:53:58.382 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:40.094679+0000", 2024-09-14T04:53:58.382 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 78.241241333999994, 2024-09-14T04:53:58.383 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.15394894100000001, 2024-09-14T04:53:58.383 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:53:58.383 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:58.383 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:53:58.383 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:53:58.383 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:53:58.383 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 27148 2024-09-14T04:53:58.383 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.383 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:53:58.383 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.383 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:53:58.384 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:40.094679+0000", 2024-09-14T04:53:58.384 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.384 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.384 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.384 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:53:58.384 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:40.094679+0000", 2024-09-14T04:53:58.384 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.384 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.384 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.384 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:53:58.384 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:40.094679+0000", 2024-09-14T04:53:58.384 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:53:58.384 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.385 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.385 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:53:58.385 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:40.094708+0000", 2024-09-14T04:53:58.385 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.9899999999999998e-05 2024-09-14T04:53:58.385 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.385 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.385 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:53:58.385 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:40.094715+0000", 2024-09-14T04:53:58.385 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.4840000000000001e-06 2024-09-14T04:53:58.385 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.385 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.385 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:53:58.385 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:40.094720+0000", 2024-09-14T04:53:58.386 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8289999999999998e-06 2024-09-14T04:53:58.386 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.386 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.386 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:53:58.386 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:40.242752+0000", 2024-09-14T04:53:58.386 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.14803176900000001 2024-09-14T04:53:58.386 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.386 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.386 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:53:58.386 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:40.242882+0000", 2024-09-14T04:53:58.386 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000130543 2024-09-14T04:53:58.386 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.386 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.386 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:53:58.387 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:40.243026+0000", 2024-09-14T04:53:58.387 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00014396799999999999 2024-09-14T04:53:58.387 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.387 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.387 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:53:58.387 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:40.244381+0000", 2024-09-14T04:53:58.387 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0013545059999999999 2024-09-14T04:53:58.387 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.387 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.387 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.387 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:40.245919+0000", 2024-09-14T04:53:58.387 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0015381150000000001 2024-09-14T04:53:58.387 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.387 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.387 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.388 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:40.248410+0000", 2024-09-14T04:53:58.388 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.002491413 2024-09-14T04:53:58.388 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.388 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.388 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:53:58.388 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:40.248454+0000", 2024-09-14T04:53:58.388 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.4320999999999998e-05 2024-09-14T04:53:58.388 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.388 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.388 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:53:58.388 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:40.248628+0000", 2024-09-14T04:53:58.388 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00017379900000000001 2024-09-14T04:53:58.388 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.389 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:53:58.389 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.389 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.389 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.389 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:31914 56.9 56:9f6ae494:::benchmark_data_smithi039_330847_object31913:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:53:58.389 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:42.540366+0000", 2024-09-14T04:53:58.389 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 75.795554447000001, 2024-09-14T04:53:58.389 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.15317641600000001, 2024-09-14T04:53:58.389 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:53:58.389 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:58.389 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:53:58.389 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:53:58.389 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:53:58.389 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 31914 2024-09-14T04:53:58.389 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.390 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:53:58.390 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.390 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:53:58.390 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:42.540366+0000", 2024-09-14T04:53:58.390 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.390 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.390 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.390 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:53:58.390 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:42.540366+0000", 2024-09-14T04:53:58.390 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.391 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.391 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.391 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:53:58.391 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:42.540365+0000", 2024-09-14T04:53:58.391 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:53:58.391 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.391 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.391 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:53:58.391 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:42.540439+0000", 2024-09-14T04:53:58.392 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.4682000000000001e-05 2024-09-14T04:53:58.392 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.392 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.392 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:53:58.392 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:42.540452+0000", 2024-09-14T04:53:58.392 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2288e-05 2024-09-14T04:53:58.392 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.392 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.392 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:53:58.392 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:42.540460+0000", 2024-09-14T04:53:58.392 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.4689999999999997e-06 2024-09-14T04:53:58.393 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.393 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.393 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:53:58.393 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:42.540503+0000", 2024-09-14T04:53:58.393 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.2450999999999997e-05 2024-09-14T04:53:58.393 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.393 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.393 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:53:58.393 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:42.540630+0000", 2024-09-14T04:53:58.393 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000127059 2024-09-14T04:53:58.393 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.393 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.394 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,2", 2024-09-14T04:53:58.394 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:42.540759+0000", 2024-09-14T04:53:58.394 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012968499999999999 2024-09-14T04:53:58.394 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.394 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.394 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.394 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:42.692063+0000", 2024-09-14T04:53:58.394 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.15130418100000001 2024-09-14T04:53:58.394 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.394 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.394 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.394 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:42.692100+0000", 2024-09-14T04:53:58.394 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6464000000000002e-05 2024-09-14T04:53:58.394 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.395 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.395 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:53:58.395 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:42.693462+0000", 2024-09-14T04:53:58.395 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0013623890000000001 2024-09-14T04:53:58.395 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.395 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.395 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:53:58.395 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:42.693509+0000", 2024-09-14T04:53:58.395 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6492000000000001e-05 2024-09-14T04:53:58.395 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.395 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.395 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:53:58.395 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:42.693543+0000", 2024-09-14T04:53:58.395 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3776999999999999e-05 2024-09-14T04:53:58.396 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.396 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:53:58.396 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.396 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.396 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.396 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:35313 56.0 56:0ae4ee63:::benchmark_data_smithi039_330847_object35312:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:53:58.396 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:44.506172+0000", 2024-09-14T04:53:58.396 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 73.829748222000006, 2024-09-14T04:53:58.396 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.342138263, 2024-09-14T04:53:58.396 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:53:58.396 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:58.396 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:53:58.396 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:53:58.397 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:53:58.397 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 35313 2024-09-14T04:53:58.397 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.397 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:53:58.397 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.397 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:53:58.397 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506172+0000", 2024-09-14T04:53:58.397 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.397 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.397 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.397 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:53:58.397 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506172+0000", 2024-09-14T04:53:58.397 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.397 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.398 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.398 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:53:58.398 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506172+0000", 2024-09-14T04:53:58.398 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:53:58.398 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.398 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.398 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:53:58.398 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506322+0000", 2024-09-14T04:53:58.398 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00015055799999999999 2024-09-14T04:53:58.398 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.398 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.398 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:53:58.398 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506335+0000", 2024-09-14T04:53:58.398 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2669e-05 2024-09-14T04:53:58.399 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.399 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.399 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:53:58.399 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506342+0000", 2024-09-14T04:53:58.399 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8129999999999998e-06 2024-09-14T04:53:58.399 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.399 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.399 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:53:58.399 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508580+0000", 2024-09-14T04:53:58.399 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.002237836 2024-09-14T04:53:58.399 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.399 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.400 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:53:58.400 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508708+0000", 2024-09-14T04:53:58.400 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012841999999999999 2024-09-14T04:53:58.400 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.400 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.400 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,6", 2024-09-14T04:53:58.400 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508830+0000", 2024-09-14T04:53:58.400 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012218400000000001 2024-09-14T04:53:58.400 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.400 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.400 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:53:58.401 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.510755+0000", 2024-09-14T04:53:58.401 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0019247159999999999 2024-09-14T04:53:58.401 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.401 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.401 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.401 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.511004+0000", 2024-09-14T04:53:58.401 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00024943099999999999 2024-09-14T04:53:58.401 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.401 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.401 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.401 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.848211+0000", 2024-09-14T04:53:58.401 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33720719500000002 2024-09-14T04:53:58.401 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.402 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.402 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:53:58.402 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.848276+0000", 2024-09-14T04:53:58.402 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.4605999999999998e-05 2024-09-14T04:53:58.402 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.402 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.402 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:53:58.402 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.848311+0000", 2024-09-14T04:53:58.402 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.4588999999999997e-05 2024-09-14T04:53:58.402 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.402 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:53:58.402 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.402 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.402 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.403 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:35362 56.0 56:013453d6:::benchmark_data_smithi039_330847_object35361:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:53:58.403 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:44.530645+0000", 2024-09-14T04:53:58.403 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 73.805276043999996, 2024-09-14T04:53:58.403 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.317765195, 2024-09-14T04:53:58.403 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:53:58.403 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:58.403 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:53:58.403 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:53:58.403 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:53:58.403 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 35362 2024-09-14T04:53:58.403 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.403 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:53:58.403 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.403 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:53:58.404 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.530645+0000", 2024-09-14T04:53:58.404 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.404 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.404 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.404 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:53:58.404 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.530645+0000", 2024-09-14T04:53:58.404 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.404 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.404 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.404 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:53:58.404 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.530644+0000", 2024-09-14T04:53:58.404 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:53:58.404 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.405 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.405 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:53:58.405 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.530695+0000", 2024-09-14T04:53:58.405 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.1685999999999999e-05 2024-09-14T04:53:58.405 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.405 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.405 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:53:58.405 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.530708+0000", 2024-09-14T04:53:58.405 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3019e-05 2024-09-14T04:53:58.405 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.405 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.405 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:53:58.405 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.530715+0000", 2024-09-14T04:53:58.405 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.6510000000000004e-06 2024-09-14T04:53:58.405 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.406 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.406 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:53:58.406 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.530763+0000", 2024-09-14T04:53:58.406 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8068999999999997e-05 2024-09-14T04:53:58.406 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.406 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.406 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:53:58.406 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.530926+0000", 2024-09-14T04:53:58.406 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00016308699999999999 2024-09-14T04:53:58.406 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.406 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.406 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,6", 2024-09-14T04:53:58.406 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.531072+0000", 2024-09-14T04:53:58.406 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000145401 2024-09-14T04:53:58.407 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.407 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.407 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.407 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.532923+0000", 2024-09-14T04:53:58.407 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0018516419999999999 2024-09-14T04:53:58.407 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.407 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.407 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:53:58.407 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.532937+0000", 2024-09-14T04:53:58.407 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.4236999999999999e-05 2024-09-14T04:53:58.407 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.407 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.407 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.408 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.848343+0000", 2024-09-14T04:53:58.408 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.31540504699999999 2024-09-14T04:53:58.408 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.408 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.408 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:53:58.408 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.848382+0000", 2024-09-14T04:53:58.408 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9468000000000002e-05 2024-09-14T04:53:58.408 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.408 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.408 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:53:58.408 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.848410+0000", 2024-09-14T04:53:58.408 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.7804e-05 2024-09-14T04:53:58.408 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.409 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:53:58.409 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.409 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.409 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.409 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:35373 56.0 56:032d996c:::benchmark_data_smithi039_330847_object35372:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:53:58.410 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:44.537746+0000", 2024-09-14T04:53:58.410 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 73.798175052000005, 2024-09-14T04:53:58.410 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.31079336499999999, 2024-09-14T04:53:58.410 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:53:58.410 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:58.410 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:53:58.410 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:53:58.410 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:53:58.410 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 35373 2024-09-14T04:53:58.410 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.410 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:53:58.410 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.410 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:53:58.411 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.537746+0000", 2024-09-14T04:53:58.411 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.411 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.411 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.411 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:53:58.411 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.537746+0000", 2024-09-14T04:53:58.411 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.411 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.411 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.411 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:53:58.411 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.537745+0000", 2024-09-14T04:53:58.411 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:53:58.411 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.412 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.412 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:53:58.412 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.537798+0000", 2024-09-14T04:53:58.412 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.3387999999999998e-05 2024-09-14T04:53:58.412 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.412 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.412 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:53:58.412 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.537809+0000", 2024-09-14T04:53:58.412 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0567999999999999e-05 2024-09-14T04:53:58.412 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.412 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.413 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:53:58.413 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.537813+0000", 2024-09-14T04:53:58.413 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.3610000000000003e-06 2024-09-14T04:53:58.413 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.413 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.413 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:53:58.413 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.537853+0000", 2024-09-14T04:53:58.413 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9873999999999998e-05 2024-09-14T04:53:58.413 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.413 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.413 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:53:58.413 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.537948+0000", 2024-09-14T04:53:58.413 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.4563999999999998e-05 2024-09-14T04:53:58.414 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.414 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.414 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,6", 2024-09-14T04:53:58.414 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.538031+0000", 2024-09-14T04:53:58.414 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.3444999999999998e-05 2024-09-14T04:53:58.414 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.414 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.414 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:53:58.414 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.539130+0000", 2024-09-14T04:53:58.414 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0010991620000000001 2024-09-14T04:53:58.414 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.414 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.414 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.414 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.539564+0000", 2024-09-14T04:53:58.415 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00043424199999999999 2024-09-14T04:53:58.415 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.415 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.415 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.415 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.848453+0000", 2024-09-14T04:53:58.415 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.30888875900000001 2024-09-14T04:53:58.415 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.415 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.415 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:53:58.415 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.848502+0000", 2024-09-14T04:53:58.415 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8578999999999997e-05 2024-09-14T04:53:58.415 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.415 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.416 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:53:58.416 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.848539+0000", 2024-09-14T04:53:58.416 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7128000000000001e-05 2024-09-14T04:53:58.416 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.416 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:53:58.416 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.416 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.416 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.416 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:37087 56.c 56:3620d308:::benchmark_data_smithi039_330847_object37086:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:53:58.416 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:45.867186+0000", 2024-09-14T04:53:58.416 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 72.468734841, 2024-09-14T04:53:58.416 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.30037860300000002, 2024-09-14T04:53:58.416 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:53:58.416 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:58.417 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:53:58.417 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:53:58.417 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:53:58.417 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 37087 2024-09-14T04:53:58.417 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.417 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:53:58.417 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.417 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:53:58.417 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.867186+0000", 2024-09-14T04:53:58.417 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.417 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.417 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.417 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:53:58.417 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.867186+0000", 2024-09-14T04:53:58.418 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.418 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.418 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.418 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:53:58.418 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.867185+0000", 2024-09-14T04:53:58.418 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:53:58.418 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.418 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.418 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:53:58.418 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.867257+0000", 2024-09-14T04:53:58.418 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.2452000000000004e-05 2024-09-14T04:53:58.418 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.418 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.419 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:53:58.419 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.867274+0000", 2024-09-14T04:53:58.419 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.7274000000000001e-05 2024-09-14T04:53:58.419 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.419 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.419 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:53:58.419 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.867281+0000", 2024-09-14T04:53:58.419 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.9779999999999999e-06 2024-09-14T04:53:58.419 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.419 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.419 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:53:58.419 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.867393+0000", 2024-09-14T04:53:58.419 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011156 2024-09-14T04:53:58.419 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.420 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.420 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:53:58.420 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.867516+0000", 2024-09-14T04:53:58.420 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012327599999999999 2024-09-14T04:53:58.420 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.420 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.420 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:53:58.420 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.867634+0000", 2024-09-14T04:53:58.420 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011731999999999999 2024-09-14T04:53:58.420 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.420 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.420 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:53:58.420 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.869172+0000", 2024-09-14T04:53:58.421 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0015381839999999999 2024-09-14T04:53:58.421 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.421 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.421 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.421 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:46.165058+0000", 2024-09-14T04:53:58.421 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.295886119 2024-09-14T04:53:58.421 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.421 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.421 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.421 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:46.167480+0000", 2024-09-14T04:53:58.421 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0024220779999999998 2024-09-14T04:53:58.421 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.421 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.421 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:53:58.422 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:46.167530+0000", 2024-09-14T04:53:58.422 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.9932999999999998e-05 2024-09-14T04:53:58.422 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.422 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.422 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:53:58.422 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:46.167564+0000", 2024-09-14T04:53:58.422 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.4516e-05 2024-09-14T04:53:58.422 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.422 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:53:58.422 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.422 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.422 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.422 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:37088 56.9 56:9552b3ae:::benchmark_data_smithi039_330847_object37087:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:53:58.422 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:45.868780+0000", 2024-09-14T04:53:58.423 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 72.467140395000001, 2024-09-14T04:53:58.423 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.29755063700000001, 2024-09-14T04:53:58.423 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:53:58.423 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:58.423 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:53:58.423 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:53:58.423 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:53:58.423 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 37088 2024-09-14T04:53:58.423 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.423 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:53:58.423 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.423 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:53:58.423 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.868780+0000", 2024-09-14T04:53:58.423 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.424 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.424 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.424 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:53:58.424 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.868780+0000", 2024-09-14T04:53:58.424 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.424 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.424 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.424 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:53:58.424 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.868779+0000", 2024-09-14T04:53:58.424 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:53:58.424 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.424 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.424 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:53:58.425 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.868839+0000", 2024-09-14T04:53:58.425 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9540999999999998e-05 2024-09-14T04:53:58.425 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.425 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.425 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:53:58.425 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.868852+0000", 2024-09-14T04:53:58.425 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3733999999999999e-05 2024-09-14T04:53:58.425 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.425 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.425 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:53:58.425 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.868859+0000", 2024-09-14T04:53:58.425 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.6660000000000002e-06 2024-09-14T04:53:58.425 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.425 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.426 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:53:58.426 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.868926+0000", 2024-09-14T04:53:58.426 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7006000000000002e-05 2024-09-14T04:53:58.426 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.426 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.426 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:53:58.426 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.869060+0000", 2024-09-14T04:53:58.426 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000133671 2024-09-14T04:53:58.426 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.426 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.426 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,2", 2024-09-14T04:53:58.426 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.869202+0000", 2024-09-14T04:53:58.426 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00014203499999999999 2024-09-14T04:53:58.426 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.427 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.427 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.427 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.871044+0000", 2024-09-14T04:53:58.427 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001842406 2024-09-14T04:53:58.427 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.427 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.427 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:53:58.427 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.871575+0000", 2024-09-14T04:53:58.427 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00053110699999999995 2024-09-14T04:53:58.427 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.427 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.427 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.427 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:46.166253+0000", 2024-09-14T04:53:58.427 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.29467800799999999 2024-09-14T04:53:58.428 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.428 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.428 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:53:58.428 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:46.166298+0000", 2024-09-14T04:53:58.428 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.4435999999999999e-05 2024-09-14T04:53:58.428 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.428 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.428 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:53:58.428 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:46.166331+0000", 2024-09-14T04:53:58.428 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3114999999999997e-05 2024-09-14T04:53:58.428 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.428 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:53:58.429 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.429 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.429 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.429 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:40539 56.c 56:3cc2ec4e:::benchmark_data_smithi039_330847_object40538:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:53:58.429 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:48.074755+0000", 2024-09-14T04:53:58.429 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 70.261165649000006, 2024-09-14T04:53:58.429 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.183810472, 2024-09-14T04:53:58.429 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:53:58.429 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:58.430 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:53:58.430 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:53:58.430 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:53:58.430 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 40539 2024-09-14T04:53:58.430 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.430 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:53:58.430 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.430 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:53:58.430 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.074755+0000", 2024-09-14T04:53:58.430 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.430 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.430 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.430 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:53:58.430 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.074755+0000", 2024-09-14T04:53:58.431 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.431 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.431 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.431 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:53:58.431 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.074754+0000", 2024-09-14T04:53:58.431 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:53:58.431 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.431 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.431 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:53:58.431 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.074828+0000", 2024-09-14T04:53:58.431 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.4546000000000001e-05 2024-09-14T04:53:58.431 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.431 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.431 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:53:58.432 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.074841+0000", 2024-09-14T04:53:58.432 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2519e-05 2024-09-14T04:53:58.432 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.432 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.432 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:53:58.432 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.074848+0000", 2024-09-14T04:53:58.432 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.1230000000000002e-06 2024-09-14T04:53:58.432 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.432 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.432 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:53:58.432 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.077229+0000", 2024-09-14T04:53:58.432 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0023808119999999999 2024-09-14T04:53:58.432 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.432 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.433 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:53:58.433 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.077382+0000", 2024-09-14T04:53:58.433 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00015366499999999999 2024-09-14T04:53:58.433 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.433 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.433 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:53:58.433 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.077522+0000", 2024-09-14T04:53:58.433 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000139863 2024-09-14T04:53:58.433 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.433 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.433 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.433 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.080212+0000", 2024-09-14T04:53:58.433 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0026899620000000002 2024-09-14T04:53:58.433 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.434 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.434 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:53:58.434 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.083877+0000", 2024-09-14T04:53:58.434 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.003665289 2024-09-14T04:53:58.434 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.434 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.434 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.434 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.258480+0000", 2024-09-14T04:53:58.434 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.17460240699999999 2024-09-14T04:53:58.434 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.434 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.434 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:53:58.434 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.258530+0000", 2024-09-14T04:53:58.434 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.0201999999999998e-05 2024-09-14T04:53:58.435 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.435 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.435 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:53:58.435 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.258565+0000", 2024-09-14T04:53:58.435 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5539e-05 2024-09-14T04:53:58.435 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.435 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:53:58.435 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.435 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.435 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.435 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:40548 56.c 56:3c571652:::benchmark_data_smithi039_330847_object40547:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:53:58.435 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:48.079695+0000", 2024-09-14T04:53:58.435 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 70.256225915000002, 2024-09-14T04:53:58.435 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.181339682, 2024-09-14T04:53:58.436 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:53:58.436 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:58.436 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:53:58.436 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:53:58.436 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:53:58.436 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 40548 2024-09-14T04:53:58.436 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.436 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:53:58.436 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.436 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:53:58.436 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.079695+0000", 2024-09-14T04:53:58.436 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.437 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.437 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.437 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:53:58.437 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.079695+0000", 2024-09-14T04:53:58.437 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.437 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.437 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.437 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:53:58.437 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.079693+0000", 2024-09-14T04:53:58.437 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:53:58.437 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.437 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.437 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:53:58.437 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.079763+0000", 2024-09-14T04:53:58.438 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.9976999999999998e-05 2024-09-14T04:53:58.438 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.438 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.438 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:53:58.438 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.079986+0000", 2024-09-14T04:53:58.438 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00022270100000000001 2024-09-14T04:53:58.438 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.438 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.438 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:53:58.438 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.079994+0000", 2024-09-14T04:53:58.438 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.5369999999999997e-06 2024-09-14T04:53:58.438 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.438 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.439 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:53:58.439 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.080240+0000", 2024-09-14T04:53:58.439 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000246433 2024-09-14T04:53:58.439 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.439 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.439 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:53:58.439 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.080357+0000", 2024-09-14T04:53:58.439 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000116129 2024-09-14T04:53:58.439 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.439 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.439 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:53:58.439 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.080475+0000", 2024-09-14T04:53:58.440 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011867900000000001 2024-09-14T04:53:58.440 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.440 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.440 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.440 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.085544+0000", 2024-09-14T04:53:58.440 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0050682030000000003 2024-09-14T04:53:58.440 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.440 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.440 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:53:58.440 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.085592+0000", 2024-09-14T04:53:58.440 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.846e-05 2024-09-14T04:53:58.440 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.440 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.441 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.441 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.260930+0000", 2024-09-14T04:53:58.441 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.175337727 2024-09-14T04:53:58.441 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.441 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.441 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:53:58.441 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.260999+0000", 2024-09-14T04:53:58.441 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.9361000000000002e-05 2024-09-14T04:53:58.441 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.441 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.441 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:53:58.441 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.261034+0000", 2024-09-14T04:53:58.441 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5342999999999997e-05 2024-09-14T04:53:58.441 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.442 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:53:58.442 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.442 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.442 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.442 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:40549 56.c 56:366fc36f:::benchmark_data_smithi039_330847_object40548:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:53:58.442 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:48.080004+0000", 2024-09-14T04:53:58.442 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 70.255916517000003, 2024-09-14T04:53:58.442 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.18306823999999999, 2024-09-14T04:53:58.442 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:53:58.442 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:58.442 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:53:58.442 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:53:58.442 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:53:58.443 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 40549 2024-09-14T04:53:58.443 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.443 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:53:58.443 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.443 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:53:58.443 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.080004+0000", 2024-09-14T04:53:58.443 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.443 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.443 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.443 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:53:58.443 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.080004+0000", 2024-09-14T04:53:58.443 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.444 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.444 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.444 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:53:58.444 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.080003+0000", 2024-09-14T04:53:58.444 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:53:58.444 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.444 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.444 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:53:58.444 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.080060+0000", 2024-09-14T04:53:58.444 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.6467e-05 2024-09-14T04:53:58.444 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.444 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.444 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:53:58.444 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.080070+0000", 2024-09-14T04:53:58.445 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.6590000000000002e-06 2024-09-14T04:53:58.445 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.445 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.445 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:53:58.445 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.080076+0000", 2024-09-14T04:53:58.445 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.4409999999999998e-06 2024-09-14T04:53:58.445 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.445 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.445 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:53:58.445 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.082160+0000", 2024-09-14T04:53:58.445 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0020835139999999999 2024-09-14T04:53:58.445 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.445 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.445 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:53:58.446 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.082290+0000", 2024-09-14T04:53:58.446 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000130109 2024-09-14T04:53:58.446 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.446 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.446 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:53:58.446 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.082429+0000", 2024-09-14T04:53:58.446 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013966200000000001 2024-09-14T04:53:58.446 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.446 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.446 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.446 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.085573+0000", 2024-09-14T04:53:58.446 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0031437729999999999 2024-09-14T04:53:58.446 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.447 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.447 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:53:58.447 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.085598+0000", 2024-09-14T04:53:58.447 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.5199999999999999e-05 2024-09-14T04:53:58.447 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.447 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.447 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.447 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.262878+0000", 2024-09-14T04:53:58.447 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.17727994399999999 2024-09-14T04:53:58.447 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.447 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.448 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:53:58.448 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.262927+0000", 2024-09-14T04:53:58.448 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.9200000000000003e-05 2024-09-14T04:53:58.448 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.448 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.448 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:53:58.448 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.263072+0000", 2024-09-14T04:53:58.448 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00014490299999999999 2024-09-14T04:53:58.448 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.449 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:53:58.449 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.449 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.449 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.449 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:40550 56.9 56:9281d539:::benchmark_data_smithi039_330847_object40549:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:53:58.449 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:48.080248+0000", 2024-09-14T04:53:58.449 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 70.255672723000004, 2024-09-14T04:53:58.449 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.178523139, 2024-09-14T04:53:58.449 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:53:58.449 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:58.449 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:53:58.449 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:53:58.449 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:53:58.449 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 40550 2024-09-14T04:53:58.450 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.450 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:53:58.450 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.450 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:53:58.450 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.080248+0000", 2024-09-14T04:53:58.450 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.450 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:58.452 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.452 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.452 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:53:58.452 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.080248+0000", 2024-09-14T04:53:58.452 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.452 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.452 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.452 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:53:58.452 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.080247+0000", 2024-09-14T04:53:58.452 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:53:58.452 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.453 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.453 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:53:58.453 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.080303+0000", 2024-09-14T04:53:58.453 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.5603000000000003e-05 2024-09-14T04:53:58.453 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.453 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.453 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:53:58.453 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.080313+0000", 2024-09-14T04:53:58.453 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0669999999999999e-05 2024-09-14T04:53:58.453 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.453 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.453 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:53:58.453 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.080320+0000", 2024-09-14T04:53:58.453 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8530000000000003e-06 2024-09-14T04:53:58.454 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.454 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.454 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:53:58.454 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.080416+0000", 2024-09-14T04:53:58.454 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.5301000000000001e-05 2024-09-14T04:53:58.454 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.454 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.454 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:53:58.454 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.080544+0000", 2024-09-14T04:53:58.454 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012883 2024-09-14T04:53:58.454 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.454 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.454 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,2", 2024-09-14T04:53:58.454 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.080675+0000", 2024-09-14T04:53:58.455 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013019600000000001 2024-09-14T04:53:58.455 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.455 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.455 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:53:58.455 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.084388+0000", 2024-09-14T04:53:58.455 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0037132469999999998 2024-09-14T04:53:58.455 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.455 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.455 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.455 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.085227+0000", 2024-09-14T04:53:58.455 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00083923600000000004 2024-09-14T04:53:58.455 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.455 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.455 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.456 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.258701+0000", 2024-09-14T04:53:58.456 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.173473816 2024-09-14T04:53:58.456 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.456 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.456 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:53:58.456 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.258739+0000", 2024-09-14T04:53:58.456 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7761e-05 2024-09-14T04:53:58.456 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.456 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.456 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:53:58.456 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.258771+0000", 2024-09-14T04:53:58.456 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.2425999999999998e-05 2024-09-14T04:53:58.456 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.456 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:53:58.457 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.457 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.457 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.457 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:40552 56.c 56:3d7b4efd:::benchmark_data_smithi039_330847_object40551:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:53:58.457 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:48.081019+0000", 2024-09-14T04:53:58.457 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 70.254901400999998, 2024-09-14T04:53:58.457 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.18216908700000001, 2024-09-14T04:53:58.457 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:53:58.457 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:58.457 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:53:58.457 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:53:58.457 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:53:58.457 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 40552 2024-09-14T04:53:58.458 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.458 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:53:58.458 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.458 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:53:58.458 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.081019+0000", 2024-09-14T04:53:58.458 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.458 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.458 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.458 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:53:58.458 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.081019+0000", 2024-09-14T04:53:58.458 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.458 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.458 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.458 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:53:58.459 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.081018+0000", 2024-09-14T04:53:58.459 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:53:58.459 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.459 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.459 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:53:58.459 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.081076+0000", 2024-09-14T04:53:58.459 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.7411999999999999e-05 2024-09-14T04:53:58.459 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.459 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.459 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:53:58.459 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.081088+0000", 2024-09-14T04:53:58.459 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2738999999999999e-05 2024-09-14T04:53:58.459 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.459 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.460 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:53:58.460 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.081096+0000", 2024-09-14T04:53:58.460 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.1999999999999997e-06 2024-09-14T04:53:58.460 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.460 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.460 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:53:58.460 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.083901+0000", 2024-09-14T04:53:58.460 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00280554 2024-09-14T04:53:58.460 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.460 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.460 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:53:58.460 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.084043+0000", 2024-09-14T04:53:58.460 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000141929 2024-09-14T04:53:58.460 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.461 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.461 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:53:58.461 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.084157+0000", 2024-09-14T04:53:58.461 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000114108 2024-09-14T04:53:58.461 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.461 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.461 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:53:58.461 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.086349+0000", 2024-09-14T04:53:58.461 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.002191971 2024-09-14T04:53:58.461 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.461 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.461 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.461 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.086828+0000", 2024-09-14T04:53:58.461 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00047831000000000002 2024-09-14T04:53:58.462 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.462 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.462 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.462 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.263121+0000", 2024-09-14T04:53:58.462 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.17629368500000001 2024-09-14T04:53:58.462 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.462 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.462 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:53:58.462 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.263157+0000", 2024-09-14T04:53:58.462 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5587999999999999e-05 2024-09-14T04:53:58.462 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.462 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.462 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:53:58.462 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.263188+0000", 2024-09-14T04:53:58.463 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.1482000000000001e-05 2024-09-14T04:53:58.463 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.463 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:53:58.463 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.463 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.463 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.463 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:40555 56.1 56:8f6c7fc0:::benchmark_data_smithi039_330847_object40554:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:53:58.463 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:48.082083+0000", 2024-09-14T04:53:58.463 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 70.253837367000003, 2024-09-14T04:53:58.463 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.178460438, 2024-09-14T04:53:58.463 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:53:58.463 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:58.463 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:53:58.463 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:53:58.464 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:53:58.464 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 40555 2024-09-14T04:53:58.464 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.464 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:53:58.464 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.464 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:53:58.464 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.082083+0000", 2024-09-14T04:53:58.464 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.464 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.464 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.464 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:53:58.464 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.082083+0000", 2024-09-14T04:53:58.464 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.464 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.465 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.465 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:53:58.465 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.082082+0000", 2024-09-14T04:53:58.465 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:53:58.465 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.465 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.465 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:53:58.465 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.082150+0000", 2024-09-14T04:53:58.465 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7716e-05 2024-09-14T04:53:58.465 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.465 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.465 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:53:58.465 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.082161+0000", 2024-09-14T04:53:58.465 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0917999999999999e-05 2024-09-14T04:53:58.466 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.466 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.466 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:53:58.466 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.082168+0000", 2024-09-14T04:53:58.466 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7889999999999997e-06 2024-09-14T04:53:58.466 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.466 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.466 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:53:58.466 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.082302+0000", 2024-09-14T04:53:58.466 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000134136 2024-09-14T04:53:58.466 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.466 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.466 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:53:58.467 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.082438+0000", 2024-09-14T04:53:58.467 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013563299999999999 2024-09-14T04:53:58.467 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.467 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.467 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,3", 2024-09-14T04:53:58.467 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.082574+0000", 2024-09-14T04:53:58.467 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013603199999999999 2024-09-14T04:53:58.467 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.467 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.467 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:53:58.467 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.088138+0000", 2024-09-14T04:53:58.467 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0055645039999999996 2024-09-14T04:53:58.467 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.467 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.467 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.468 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.191907+0000", 2024-09-14T04:53:58.468 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.10376845899999999 2024-09-14T04:53:58.468 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.468 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.468 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.468 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.260457+0000", 2024-09-14T04:53:58.468 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.068549991000000005 2024-09-14T04:53:58.468 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.468 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.468 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:53:58.468 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.260507+0000", 2024-09-14T04:53:58.468 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.0658000000000002e-05 2024-09-14T04:53:58.468 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.469 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.469 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:53:58.469 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.260544+0000", 2024-09-14T04:53:58.469 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6489999999999998e-05 2024-09-14T04:53:58.469 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.469 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:53:58.469 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.469 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.469 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.469 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:40560 56.1 56:8f8e0b81:::benchmark_data_smithi039_330847_object40559:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:53:58.469 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:48.086212+0000", 2024-09-14T04:53:58.469 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 70.249709056, 2024-09-14T04:53:58.469 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.17444117100000001, 2024-09-14T04:53:58.469 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:53:58.470 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:58.470 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:53:58.470 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:53:58.470 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:53:58.470 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 40560 2024-09-14T04:53:58.470 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.470 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:53:58.470 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.470 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:53:58.470 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.086212+0000", 2024-09-14T04:53:58.470 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.470 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.470 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.470 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:53:58.471 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.086212+0000", 2024-09-14T04:53:58.471 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.471 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.471 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.471 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:53:58.471 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.086211+0000", 2024-09-14T04:53:58.471 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:53:58.471 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.471 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.471 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:53:58.471 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.086270+0000", 2024-09-14T04:53:58.471 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9778000000000003e-05 2024-09-14T04:53:58.471 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.471 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.472 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:53:58.472 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.086280+0000", 2024-09-14T04:53:58.472 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.7480000000000005e-06 2024-09-14T04:53:58.472 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.472 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.472 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:53:58.472 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.086287+0000", 2024-09-14T04:53:58.472 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.6089999999999997e-06 2024-09-14T04:53:58.472 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.472 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.472 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:53:58.472 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.086334+0000", 2024-09-14T04:53:58.472 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6835999999999997e-05 2024-09-14T04:53:58.472 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.473 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.473 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:53:58.473 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.086448+0000", 2024-09-14T04:53:58.473 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000114572 2024-09-14T04:53:58.473 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.473 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.473 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,3", 2024-09-14T04:53:58.473 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.086567+0000", 2024-09-14T04:53:58.473 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000119104 2024-09-14T04:53:58.473 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.473 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.473 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:53:58.473 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.088502+0000", 2024-09-14T04:53:58.473 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0019342890000000001 2024-09-14T04:53:58.474 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.474 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.474 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.474 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.193250+0000", 2024-09-14T04:53:58.474 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.10474829500000001 2024-09-14T04:53:58.474 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.474 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.474 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.474 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.260583+0000", 2024-09-14T04:53:58.474 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.067332766000000002 2024-09-14T04:53:58.474 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.474 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.474 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:53:58.474 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.260621+0000", 2024-09-14T04:53:58.474 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8353999999999999e-05 2024-09-14T04:53:58.475 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.475 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.475 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:53:58.475 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:48.260653+0000", 2024-09-14T04:53:58.475 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.1770000000000002e-05 2024-09-14T04:53:58.475 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.475 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:53:58.475 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.475 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.475 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.475 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:72600 56.6 56:63b53840:::benchmark_data_smithi039_330847_object72599:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1480)", 2024-09-14T04:53:58.475 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:53:13.471996+0000", 2024-09-14T04:53:58.475 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 44.863924333, 2024-09-14T04:53:58.475 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.15540298799999999, 2024-09-14T04:53:58.476 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:53:58.476 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:53:58.476 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:53:58.476 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:53:58.476 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:53:58.476 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 72600 2024-09-14T04:53:58.476 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.476 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:53:58.476 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.476 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:53:58.476 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:53:13.471996+0000", 2024-09-14T04:53:58.476 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.476 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.476 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.477 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:53:58.477 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:53:13.471996+0000", 2024-09-14T04:53:58.477 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:53:58.477 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.477 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.477 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:53:58.477 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:53:13.471996+0000", 2024-09-14T04:53:58.477 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:53:58.477 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.477 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.477 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:53:58.477 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:53:13.472024+0000", 2024-09-14T04:53:58.477 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.7912e-05 2024-09-14T04:53:58.477 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.478 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.478 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:53:58.478 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:53:13.472030+0000", 2024-09-14T04:53:58.478 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.3600000000000001e-06 2024-09-14T04:53:58.478 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.478 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.478 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:53:58.478 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:53:13.472035+0000", 2024-09-14T04:53:58.478 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8119999999999998e-06 2024-09-14T04:53:58.478 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.478 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.478 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:53:58.478 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:53:13.481081+0000", 2024-09-14T04:53:58.478 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0090457590000000004 2024-09-14T04:53:58.479 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.479 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.479 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:53:58.479 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:53:13.481249+0000", 2024-09-14T04:53:58.479 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00016792599999999999 2024-09-14T04:53:58.479 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.479 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.479 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 4,7", 2024-09-14T04:53:58.479 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:53:13.481399+0000", 2024-09-14T04:53:58.479 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00015072299999999999 2024-09-14T04:53:58.479 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.479 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.479 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:53:58.479 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:53:13.486776+0000", 2024-09-14T04:53:58.480 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.005377059 2024-09-14T04:53:58.480 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.480 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.480 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.480 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:53:13.612077+0000", 2024-09-14T04:53:58.480 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.125301053 2024-09-14T04:53:58.480 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.480 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.480 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:53:58.480 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:53:13.624665+0000", 2024-09-14T04:53:58.480 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.012587302 2024-09-14T04:53:58.480 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.480 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.480 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:53:58.481 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:53:13.624718+0000", 2024-09-14T04:53:58.481 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2933999999999998e-05 2024-09-14T04:53:58.481 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:53:58.481 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:53:58.481 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:53:58.481 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:53:13.627399+0000", 2024-09-14T04:53:58.481 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0026816409999999998 2024-09-14T04:53:58.481 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.481 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:53:58.481 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.481 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:53:58.481 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:53:58.481 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:53:58.520 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:58.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:58.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:58.622 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:58.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:58.666 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:58.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:58.768 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:53:58.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:53:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:53:58] "GET /metrics HTTP/1.1" 200 36807 "" "Prometheus/2.51.0" 2024-09-14T04:53:58.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:58.807 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:58.807 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:53:58.909 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:53:58.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:58.951 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:58.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:53:59.017 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 88 16 112982 112966 80.2207 45.125 0.00938154 0.0124608 2024-09-14T04:53:59.052 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:59.084 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:59.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:59.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:59.186 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:53:59.228 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:59.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:59.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:53:59.288 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:53:59 smithi039 ceph-mon[106470]: pgmap v2785: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 31 MiB/s wr, 499 op/s; 3571/326052 objects degraded (1.095%); 4066/326052 objects misplaced (1.247%); 2.8 MiB/s, 44 objects/s recovering 2024-09-14T04:53:59.289 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:53:59 smithi039 ceph-mon[111235]: pgmap v2785: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 31 MiB/s wr, 499 op/s; 3571/326052 objects degraded (1.095%); 4066/326052 objects misplaced (1.247%); 2.8 MiB/s, 44 objects/s recovering 2024-09-14T04:53:59.330 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:59.365 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:59.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:59.366 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:53:59.388 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-14T04:53:59.467 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:53:59.500 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:59.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:59.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:53:59.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:53:59 smithi129 ceph-mon[101316]: pgmap v2785: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.6 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 31 MiB/s wr, 499 op/s; 3571/326052 objects degraded (1.095%); 4066/326052 objects misplaced (1.247%); 2.8 MiB/s, 44 objects/s recovering 2024-09-14T04:53:59.602 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:53:59.666 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:59.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:59.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:53:59.768 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:53:59.796 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:53:59.824 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:53:59.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:59.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:53:59.926 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:53:59.956 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:53:59.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:53:59.957 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:00.017 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 89 16 113792 113776 79.8882 50.625 0.0149125 0.0125119 2024-09-14T04:54:00.058 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:00.100 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:54:00.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:00.103 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:00.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:00.204 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:00.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:00.245 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:00.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:00.346 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:00.380 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:00.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:00.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:00.482 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:00.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:00.518 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:00.518 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:00.619 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:00.655 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:00.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:00.657 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:00.757 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:00.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:00.805 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:00.805 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:00.906 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:00.948 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:00.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:00.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:01.018 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 90 4 114521 114517 79.5151 46.3125 0.0295558 0.0125709 2024-09-14T04:54:01.018 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:54:01.050 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:01.085 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:01.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:01.086 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:01.187 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:01.210 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:01.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:01.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:01.312 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:01.336 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:01.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:01.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:01.438 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:01.461 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:01.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:01.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:01.471 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:01 smithi129 ceph-mon[101316]: pgmap v2786: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 3571/338124 objects degraded (1.056%); 3676/338124 objects misplaced (1.087%); 5.8 MiB/s, 92 objects/s recovering 2024-09-14T04:54:01.471 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:01 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 3571/338124 objects degraded (1.056%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:54:01.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:01 smithi039 ceph-mon[106470]: pgmap v2786: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 3571/338124 objects degraded (1.056%); 3676/338124 objects misplaced (1.087%); 5.8 MiB/s, 92 objects/s recovering 2024-09-14T04:54:01.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:01 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 3571/338124 objects degraded (1.056%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:54:01.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:01 smithi039 ceph-mon[111235]: pgmap v2786: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 3571/338124 objects degraded (1.056%); 3676/338124 objects misplaced (1.087%); 5.8 MiB/s, 92 objects/s recovering 2024-09-14T04:54:01.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:01 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 3571/338124 objects degraded (1.056%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:54:01.563 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:01.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:01.585 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:01.586 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:01.686 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:01.708 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:01.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:01.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:01.813 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:01.836 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:01.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:01.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:01.937 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:01.961 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:01.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:01.962 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:01.991 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:54:02.018 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Total time run: 90.0223 2024-09-14T04:54:02.018 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Total writes made: 114521 2024-09-14T04:54:02.018 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Write size: 65536 2024-09-14T04:54:02.018 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Object size: 65536 2024-09-14T04:54:02.018 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Bandwidth (MB/sec): 79.5087 2024-09-14T04:54:02.018 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Stddev Bandwidth: 44.3513 2024-09-14T04:54:02.018 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Max bandwidth (MB/sec): 245.938 2024-09-14T04:54:02.018 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Min bandwidth (MB/sec): 0 2024-09-14T04:54:02.018 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Average IOPS: 1272 2024-09-14T04:54:02.018 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Stddev IOPS: 709.629 2024-09-14T04:54:02.018 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Max IOPS: 3935 2024-09-14T04:54:02.019 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Min IOPS: 0 2024-09-14T04:54:02.019 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Average Latency(s): 0.0125714 2024-09-14T04:54:02.019 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Stddev Latency(s): 0.028345 2024-09-14T04:54:02.019 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Max latency(s): 2.17883 2024-09-14T04:54:02.019 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Min latency(s): 0.00136201 2024-09-14T04:54:02.027 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Cleaning up (deleting benchmark objects) 2024-09-14T04:54:02.062 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:02.101 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:02.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:02.102 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:02.203 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:02.247 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:02.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:02.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:02.349 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:02.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:02.385 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:02.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:02.486 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:02.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:02.524 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:02.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:02.625 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:02.666 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:02.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:02.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:02.768 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:02.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:02.792 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:02.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:02.893 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:02.917 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:02.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:02.918 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:03.019 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:03.041 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:03.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:03.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:03.143 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:03.176 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:03.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:03.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:03.223 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is unset 2024-09-14T04:54:03.237 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.1 to 1 (10000) 2024-09-14T04:54:03.279 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:03.301 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:03.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:03.302 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:03.403 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:03.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:03.441 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:03.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:03.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:03 smithi129 ceph-mon[101316]: pgmap v2787: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 51 MiB/s wr, 812 op/s; 3571/338124 objects degraded (1.056%); 3676/338124 objects misplaced (1.087%); 5.4 MiB/s, 85 objects/s recovering 2024-09-14T04:54:03.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:03 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/78754317' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:54:03.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:03 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:54:03.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:03 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2975558312' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:54:03.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:03 smithi039 ceph-mon[106470]: pgmap v2787: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 51 MiB/s wr, 812 op/s; 3571/338124 objects degraded (1.056%); 3676/338124 objects misplaced (1.087%); 5.4 MiB/s, 85 objects/s recovering 2024-09-14T04:54:03.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:03 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/78754317' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:54:03.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:03 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:54:03.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:03 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2975558312' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:54:03.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:03 smithi039 ceph-mon[111235]: pgmap v2787: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 51 MiB/s wr, 812 op/s; 3571/338124 objects degraded (1.056%); 3676/338124 objects misplaced (1.087%); 5.4 MiB/s, 85 objects/s recovering 2024-09-14T04:54:03.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:03 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/78754317' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:54:03.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:03 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:54:03.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:03 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2975558312' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:54:03.542 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:03.572 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:03.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:03.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:03.674 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:03.706 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:03.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:03.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:03.808 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:03.850 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:03.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:03.851 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:03.952 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:03.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:03.974 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:03.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:04.075 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:04.104 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:04.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:04.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:04.206 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:04.245 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:04.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:04.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:04.347 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:04.370 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:04.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:04.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:04.472 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:04.511 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:04.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:04.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:04.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:04 smithi129 ceph-mon[101316]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:54:04.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:04 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-14T04:54:04.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:04 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2975558312' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:54:04.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:04 smithi129 ceph-mon[101316]: osdmap e1486: 8 total, 8 up, 8 in 2024-09-14T04:54:04.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:04 smithi129 ceph-mon[101316]: 1.2 scrub starts 2024-09-14T04:54:04.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:04 smithi129 ceph-mon[101316]: 1.2 scrub ok 2024-09-14T04:54:04.526 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:04 smithi039 ceph-mon[106470]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:54:04.526 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:04 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-14T04:54:04.527 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:04 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2975558312' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:54:04.527 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:04 smithi039 ceph-mon[106470]: osdmap e1486: 8 total, 8 up, 8 in 2024-09-14T04:54:04.527 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:04 smithi039 ceph-mon[106470]: 1.2 scrub starts 2024-09-14T04:54:04.527 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:04 smithi039 ceph-mon[106470]: 1.2 scrub ok 2024-09-14T04:54:04.527 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:04 smithi039 ceph-mon[111235]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:54:04.527 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:04 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-14T04:54:04.527 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:04 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2975558312' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:54:04.527 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:04 smithi039 ceph-mon[111235]: osdmap e1486: 8 total, 8 up, 8 in 2024-09-14T04:54:04.527 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:04 smithi039 ceph-mon[111235]: 1.2 scrub starts 2024-09-14T04:54:04.528 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:04 smithi039 ceph-mon[111235]: 1.2 scrub ok 2024-09-14T04:54:04.613 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:04.634 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:04.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:04.635 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:04.736 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:04.760 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:04.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:04.761 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:04.862 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:04.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:04.884 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:04.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:04.985 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:05.008 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:05.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:05.009 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:05.110 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:05.139 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:05.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:05.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:05.182 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 2 1 2024-09-14T04:54:05.241 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:05.279 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:05.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:05.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:05.381 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:05.430 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:05.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:05.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:05.460 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:54:05.460 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:54:05.461 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:05 smithi039 ceph-mon[106470]: pgmap v2789: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 118 B/s rd, 38 MiB/s wr, 675 op/s; 3571/340440 objects degraded (1.049%); 3365/340440 objects misplaced (0.988%); 5.1 MiB/s, 81 objects/s recovering 2024-09-14T04:54:05.461 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:05 smithi039 ceph-mon[111235]: pgmap v2789: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 118 B/s rd, 38 MiB/s wr, 675 op/s; 3571/340440 objects degraded (1.049%); 3365/340440 objects misplaced (0.988%); 5.1 MiB/s, 81 objects/s recovering 2024-09-14T04:54:05.489 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:54:05.490 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:54:05.511 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:54:05.512 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:54:05.523 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:05 smithi129 ceph-mon[101316]: pgmap v2789: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 118 B/s rd, 38 MiB/s wr, 675 op/s; 3571/340440 objects degraded (1.049%); 3365/340440 objects misplaced (0.988%); 5.1 MiB/s, 81 objects/s recovering 2024-09-14T04:54:05.532 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:05.538 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:54:05.539 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:54:05.570 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:54:05.570 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:54:05.571 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:05.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:05.572 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:05.603 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:54:05.603 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:54:05.632 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:54:05.632 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:54:05.669 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:54:05.669 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:54:05.673 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:05.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:05.704 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:05.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:05.805 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:05.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:05.832 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:05.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:05.934 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:05.970 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:05.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:05.971 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:06.072 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:06.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:06.112 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:06.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:06.213 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:06.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:06.254 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:06.254 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:06.355 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:06.387 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:06.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:06.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:06.489 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:06.520 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:06.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:06 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1365598693' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:54:06.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:06 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 3571/340440 objects degraded (1.049%), 1 pg degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:54:06.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:06.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:06.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:06 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1365598693' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:54:06.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:06 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 3571/340440 objects degraded (1.049%), 1 pg degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:54:06.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:06 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1365598693' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:54:06.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:06 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 3571/340440 objects degraded (1.049%), 1 pg degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:54:06.622 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:06.658 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:06.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:06.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:06.760 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:06.804 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:06.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:06.805 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:06.824 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:54:06.906 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:06.933 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:06.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:06.934 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:07.017 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:54:07.017 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:54:07.034 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:07.075 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:07.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:07.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:07.178 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:07.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:07.211 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:07.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:07.312 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:07.354 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:07.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:07.355 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:07.456 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:07.490 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:07.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:07.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:07.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:07 smithi039 ceph-mon[106470]: pgmap v2790: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 37 MiB/s wr, 688 op/s; 3571/340572 objects degraded (1.049%); 3055/340572 objects misplaced (0.897%); 6.3 MiB/s, 101 objects/s recovering 2024-09-14T04:54:07.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:07 smithi039 ceph-mon[111235]: pgmap v2790: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 37 MiB/s wr, 688 op/s; 3571/340572 objects degraded (1.049%); 3055/340572 objects misplaced (0.897%); 6.3 MiB/s, 101 objects/s recovering 2024-09-14T04:54:07.592 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:07.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:07.637 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:07.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:07.732 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:07 smithi129 ceph-mon[101316]: pgmap v2790: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 37 MiB/s wr, 688 op/s; 3571/340572 objects degraded (1.049%); 3055/340572 objects misplaced (0.897%); 6.3 MiB/s, 101 objects/s recovering 2024-09-14T04:54:07.738 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:07.774 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-14T04:54:07.784 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:07.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:07.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:07.886 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:07.891 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:54:07.918 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:07.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:07.919 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:08.020 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:08.078 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:08.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:08.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:08.148 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:54:08.180 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:08.238 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:08.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:08.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:08.341 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:08.382 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:08.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:08.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:08.483 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:08.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:08.513 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:08.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:08.517 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:54:08.517 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:08 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:54:08.520 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:08 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:54:08.520 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:08 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:54:08.614 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:08.651 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:08.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:08.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:08.653 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:08 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:54:08.653 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:08 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:54:08.690 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:54:08.753 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:08.776 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:54:08.784 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:08.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:08.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:08.789 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:54:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:54:08] "GET /metrics HTTP/1.1" 200 36804 "" "Prometheus/2.51.0" 2024-09-14T04:54:08.887 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:08.918 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:08.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:08.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:09.020 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:09.061 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:09.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:09.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:09.163 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:09.200 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:09.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:09.201 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:09.301 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:09.352 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:09.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:09.353 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:09.434 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:09 smithi039 ceph-mon[106470]: pgmap v2791: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 37 MiB/s wr, 688 op/s; 3571/340572 objects degraded (1.049%); 3055/340572 objects misplaced (0.897%); 6.3 MiB/s, 101 objects/s recovering 2024-09-14T04:54:09.435 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:09 smithi039 ceph-mon[111235]: pgmap v2791: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 37 MiB/s wr, 688 op/s; 3571/340572 objects degraded (1.049%); 3055/340572 objects misplaced (0.897%); 6.3 MiB/s, 101 objects/s recovering 2024-09-14T04:54:09.454 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:09.500 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:09.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:09.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:09.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:09 smithi129 ceph-mon[101316]: pgmap v2791: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.9 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 102 B/s rd, 37 MiB/s wr, 688 op/s; 3571/340572 objects degraded (1.049%); 3055/340572 objects misplaced (0.897%); 6.3 MiB/s, 101 objects/s recovering 2024-09-14T04:54:09.602 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:09.661 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:09.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:09.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:09.763 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:09.779 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:54:09.811 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:09.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:09.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:09.913 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:09.953 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:09.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:09.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:10.057 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:10.095 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:10.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:10.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:10.197 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:10.231 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:10.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:10.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:10.297 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:54:10.297 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:54:10.329 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.2 to 1 (10000) 2024-09-14T04:54:10.333 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:10.369 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:10.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:10.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:10.470 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:10.511 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:10.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:10.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:10.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:10 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3967610695' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:54:10.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:10 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3967610695' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:54:10.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:10 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3967610695' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:54:10.613 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:10.664 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:10.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:10.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:10.766 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:10.802 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:10.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:10.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:10.904 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:10.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:10.940 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:10.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:11.041 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:11.073 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:11.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:11.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:11.175 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:11.217 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:11.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:11.218 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:11.313 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-14T04:54:11.319 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:11.355 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:11.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:11.357 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:11.458 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:11.502 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:11.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:11.503 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:11.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:11 smithi129 ceph-mon[101316]: pgmap v2792: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 11 MiB/s wr, 666 op/s; 3571/329106 objects degraded (1.085%); 2585/329106 objects misplaced (0.785%); 6.8 MiB/s, 108 objects/s recovering 2024-09-14T04:54:11.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:11 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3967610695' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-14T04:54:11.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:11 smithi129 ceph-mon[101316]: osdmap e1487: 8 total, 8 up, 8 in 2024-09-14T04:54:11.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:11 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 3571/329106 objects degraded (1.085%), 1 pg degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:54:11.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:11 smithi039 ceph-mon[106470]: pgmap v2792: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 11 MiB/s wr, 666 op/s; 3571/329106 objects degraded (1.085%); 2585/329106 objects misplaced (0.785%); 6.8 MiB/s, 108 objects/s recovering 2024-09-14T04:54:11.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:11 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3967610695' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-14T04:54:11.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:11 smithi039 ceph-mon[106470]: osdmap e1487: 8 total, 8 up, 8 in 2024-09-14T04:54:11.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:11 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 3571/329106 objects degraded (1.085%), 1 pg degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:54:11.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:11 smithi039 ceph-mon[111235]: pgmap v2792: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 11 MiB/s wr, 666 op/s; 3571/329106 objects degraded (1.085%); 2585/329106 objects misplaced (0.785%); 6.8 MiB/s, 108 objects/s recovering 2024-09-14T04:54:11.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:11 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3967610695' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-14T04:54:11.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:11 smithi039 ceph-mon[111235]: osdmap e1487: 8 total, 8 up, 8 in 2024-09-14T04:54:11.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:11 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 3571/329106 objects degraded (1.085%), 1 pg degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:54:11.603 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:11.622 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 3 1 2024-09-14T04:54:11.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:11.649 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:11.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:11.707 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:54:11.750 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:11.792 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:11.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:11.793 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:11.894 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:11.935 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:11.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:11.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:12.037 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:12.071 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:12.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:12.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:12.173 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:12.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:12.208 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:12.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:12.309 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:12.343 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:12.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:12.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:12.376 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is set 2024-09-14T04:54:12.445 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:12.486 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:12.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:12.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:12.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:12 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3161329624' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:54:12.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:12 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:54:12.589 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:12.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:12.623 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:12.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:12.724 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:12.757 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:12.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:12.758 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:12.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:12 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3161329624' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:54:12.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:12 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:54:12.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:12 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3161329624' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:54:12.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:12 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:54:12.859 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:12.890 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:12.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:12.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:12.993 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:13.023 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:13.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:13.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:13.126 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:13.173 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:13.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:13.174 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:13.275 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:13.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:13.322 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:13.322 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:13.423 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:13.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:13.456 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:13.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:13.508 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:13 smithi039 ceph-mon[106470]: pgmap v2794: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 9.1 MiB/s wr, 688 op/s; 3571/329106 objects degraded (1.085%); 2585/329106 objects misplaced (0.785%); 7.9 MiB/s, 126 objects/s recovering 2024-09-14T04:54:13.509 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:13 smithi039 ceph-mon[106470]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:54:13.509 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:13 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:54:13.509 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:13 smithi039 ceph-mon[106470]: osdmap e1488: 8 total, 8 up, 8 in 2024-09-14T04:54:13.509 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:13 smithi039 ceph-mon[111235]: pgmap v2794: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 9.1 MiB/s wr, 688 op/s; 3571/329106 objects degraded (1.085%); 2585/329106 objects misplaced (0.785%); 7.9 MiB/s, 126 objects/s recovering 2024-09-14T04:54:13.509 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:13 smithi039 ceph-mon[111235]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:54:13.509 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:13 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:54:13.509 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:13 smithi039 ceph-mon[111235]: osdmap e1488: 8 total, 8 up, 8 in 2024-09-14T04:54:13.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:13 smithi129 ceph-mon[101316]: pgmap v2794: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 9.1 MiB/s wr, 688 op/s; 3571/329106 objects degraded (1.085%); 2585/329106 objects misplaced (0.785%); 7.9 MiB/s, 126 objects/s recovering 2024-09-14T04:54:13.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:13 smithi129 ceph-mon[101316]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:54:13.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:13 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:54:13.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:13 smithi129 ceph-mon[101316]: osdmap e1488: 8 total, 8 up, 8 in 2024-09-14T04:54:13.557 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:13.593 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:13.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:13.594 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:13.695 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:13.700 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:54:13.700 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:54:13.730 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:13.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:13.731 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:13.823 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:54:13.833 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:13.886 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:13.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:13.887 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:13.988 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:14.020 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:14.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:14.021 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:14.122 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:14.153 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:14.154 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:14.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:14.255 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:14.304 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:14.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:14.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:14.406 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:14.423 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-14T04:54:14.445 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:14.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:14.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:14.547 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:14.581 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:14.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:14.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:14.683 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:14.732 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:54:14.732 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:54:14.734 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:14.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:14.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:14.758 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:54:14.758 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:54:14.782 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:54:14.782 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:54:14.804 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:54:14.804 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:54:14.831 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:54:14.831 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:54:14.832 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:54:14.837 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:14.872 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:54:14.872 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:54:14.895 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:14.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:14.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:14.908 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:54:14.908 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:54:14.938 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:54:14.938 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:54:14.997 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:15.038 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:15.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:15.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:15.140 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:15.189 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:15.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:15.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:15.291 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:15.322 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:15.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:15.323 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:15.424 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:15.464 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:15.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:15.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:15.566 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:15.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:15.601 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:15.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:15.702 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:15.745 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:15.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:15.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:15.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:15 smithi129 ceph-mon[101316]: pgmap v2796: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 3571/321006 objects degraded (1.112%); 2361/321006 objects misplaced (0.736%); 5.4 MiB/s, 86 objects/s recovering 2024-09-14T04:54:15.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:15 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3875633003' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:54:15.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:15 smithi039 ceph-mon[106470]: pgmap v2796: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 3571/321006 objects degraded (1.112%); 2361/321006 objects misplaced (0.736%); 5.4 MiB/s, 86 objects/s recovering 2024-09-14T04:54:15.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:15 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3875633003' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:54:15.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:15 smithi039 ceph-mon[111235]: pgmap v2796: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.5 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 3571/321006 objects degraded (1.112%); 2361/321006 objects misplaced (0.736%); 5.4 MiB/s, 86 objects/s recovering 2024-09-14T04:54:15.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:15 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3875633003' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:54:15.847 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:15.855 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:54:15.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:15.885 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:15.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:15.986 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:16.027 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:16.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:16.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:16.129 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:16.167 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:16.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:16.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:16.268 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:16.313 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:16.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:16.314 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:16.415 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:16.445 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:16.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:16.446 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:16.547 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:16.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:16.594 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:16.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:16.604 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:16 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 3571/321006 objects degraded (1.112%), 1 pg degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:54:16.695 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:16.729 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:16.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:16.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:16.750 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:16 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 3571/321006 objects degraded (1.112%), 1 pg degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:54:16.751 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:16 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 3571/321006 objects degraded (1.112%), 1 pg degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:54:16.832 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:16.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:16.865 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:16.866 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:16.966 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:17.006 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:17.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:17.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:17.089 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:54:17.089 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [ 2024-09-14T04:54:17.089 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.089 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.36012.0:126454 56.d e1488/1470 56:b5495a16:::benchmark_data_smithi039_330847_object11930:head v 1488'7969, mlcod=1488'7967)", 2024-09-14T04:54:17.089 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:17.066604+0000", 2024-09-14T04:54:17.089 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.020392057000000002, 2024-09-14T04:54:17.089 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.020422424000000002, 2024-09-14T04:54:17.089 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:17.089 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "started", 2024-09-14T04:54:17.089 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:17.090 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.090 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:17.090 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.066604+0000", 2024-09-14T04:54:17.090 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:17.090 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.090 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.090 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:17.090 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.066604+0000", 2024-09-14T04:54:17.090 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:17.090 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.090 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.090 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:17.090 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.066603+0000", 2024-09-14T04:54:17.090 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:54:17.091 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.091 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.091 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:17.091 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.066615+0000", 2024-09-14T04:54:17.091 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1286e-05 2024-09-14T04:54:17.091 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.091 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.091 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:17.091 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.066636+0000", 2024-09-14T04:54:17.091 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.1719999999999999e-05 2024-09-14T04:54:17.092 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.092 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.092 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:17.092 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.066643+0000", 2024-09-14T04:54:17.092 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8630000000000002e-06 2024-09-14T04:54:17.092 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.092 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.092 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:17.092 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.066820+0000", 2024-09-14T04:54:17.092 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00017636400000000001 2024-09-14T04:54:17.092 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.092 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.092 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:17.093 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.066913+0000", 2024-09-14T04:54:17.093 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.2980999999999996e-05 2024-09-14T04:54:17.093 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:17.093 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:17.093 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:17.093 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.093 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.093 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:126456 56.5 56:a2869838:::benchmark_data_smithi039_330847_object11932:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1488)", 2024-09-14T04:54:17.093 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:17.070744+0000", 2024-09-14T04:54:17.093 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.016252243999999999, 2024-09-14T04:54:17.093 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.016349559999999999, 2024-09-14T04:54:17.093 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:17.093 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:54:17.093 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:17.094 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:54:17.094 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:54:17.094 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 126456 2024-09-14T04:54:17.094 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.094 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:17.094 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.094 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:17.094 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.070744+0000", 2024-09-14T04:54:17.094 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:17.094 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.094 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.094 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:17.094 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.070744+0000", 2024-09-14T04:54:17.094 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:17.095 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.095 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.095 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:17.095 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.070743+0000", 2024-09-14T04:54:17.095 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:54:17.095 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.095 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.095 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:17.095 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.070752+0000", 2024-09-14T04:54:17.095 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.5519999999999993e-06 2024-09-14T04:54:17.095 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.095 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.095 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:17.096 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.070768+0000", 2024-09-14T04:54:17.096 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.6404000000000001e-05 2024-09-14T04:54:17.096 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.096 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.096 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:17.096 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.070777+0000", 2024-09-14T04:54:17.096 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.7960000000000001e-06 2024-09-14T04:54:17.096 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.096 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.096 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:17.096 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.070903+0000", 2024-09-14T04:54:17.096 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012617699999999999 2024-09-14T04:54:17.096 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.096 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.097 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:17.097 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.071183+0000", 2024-09-14T04:54:17.097 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00027940000000000002 2024-09-14T04:54:17.097 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.097 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.097 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:54:17.097 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.071393+0000", 2024-09-14T04:54:17.097 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000209989 2024-09-14T04:54:17.097 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.097 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.097 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:17.097 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.081408+0000", 2024-09-14T04:54:17.097 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.010015392999999999 2024-09-14T04:54:17.097 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.098 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.098 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:17.098 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.081438+0000", 2024-09-14T04:54:17.098 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.9901000000000001e-05 2024-09-14T04:54:17.098 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:17.098 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:17.098 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:17.098 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.098 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.098 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.36012.0:126459 56.d e1488/1470 56:b7f2df72:::benchmark_data_smithi039_330847_object11935:head v 1488'7970, mlcod=1488'7967)", 2024-09-14T04:54:17.098 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:17.073161+0000", 2024-09-14T04:54:17.098 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.013835067, 2024-09-14T04:54:17.098 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.014016102000000001, 2024-09-14T04:54:17.098 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:17.099 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "started", 2024-09-14T04:54:17.099 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:17.099 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.099 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:17.099 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.073161+0000", 2024-09-14T04:54:17.099 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:17.099 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.099 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.099 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:17.099 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.073161+0000", 2024-09-14T04:54:17.100 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:17.100 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.100 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.100 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:17.100 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.073160+0000", 2024-09-14T04:54:17.100 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:54:17.100 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.100 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.100 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:17.100 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.073172+0000", 2024-09-14T04:54:17.100 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1836e-05 2024-09-14T04:54:17.100 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.100 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.101 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:17.101 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.073185+0000", 2024-09-14T04:54:17.101 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2869e-05 2024-09-14T04:54:17.101 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.101 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.101 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:17.101 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.073193+0000", 2024-09-14T04:54:17.101 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.8690000000000004e-06 2024-09-14T04:54:17.101 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.101 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.101 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:17.101 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.073353+0000", 2024-09-14T04:54:17.101 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00015971 2024-09-14T04:54:17.101 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.102 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.102 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:17.102 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.073395+0000", 2024-09-14T04:54:17.102 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.2685999999999998e-05 2024-09-14T04:54:17.102 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:17.102 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:17.102 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:17.102 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.102 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.102 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.36012.0:126460 56.d e1488/1470 56:bd3757fe:::benchmark_data_smithi039_330847_object11936:head v 1488'7971, mlcod=1488'7967)", 2024-09-14T04:54:17.102 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:17.077346+0000", 2024-09-14T04:54:17.102 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.0096509330000000004, 2024-09-14T04:54:17.102 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0098888750000000001, 2024-09-14T04:54:17.102 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:17.103 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "started", 2024-09-14T04:54:17.103 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:17.103 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.103 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:17.103 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.077346+0000", 2024-09-14T04:54:17.103 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:17.103 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.103 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.103 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:17.103 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.077346+0000", 2024-09-14T04:54:17.103 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:17.103 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.104 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.104 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:17.104 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.077345+0000", 2024-09-14T04:54:17.104 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:54:17.105 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.105 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.105 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:17.105 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.077353+0000", 2024-09-14T04:54:17.105 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.884e-06 2024-09-14T04:54:17.105 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.105 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.105 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:17.105 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.077370+0000", 2024-09-14T04:54:17.105 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.6478999999999999e-05 2024-09-14T04:54:17.105 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.105 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.106 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:17.106 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.077377+0000", 2024-09-14T04:54:17.106 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.905e-06 2024-09-14T04:54:17.106 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.106 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.106 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:17.106 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.077417+0000", 2024-09-14T04:54:17.106 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9767000000000001e-05 2024-09-14T04:54:17.106 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.106 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.107 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:17.107 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.077437+0000", 2024-09-14T04:54:17.107 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.0777000000000001e-05 2024-09-14T04:54:17.107 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:17.107 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:17.107 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:17.107 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.107 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.107 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.36012.0:126465 56.f e1488/1470 56:f6043cdf:::benchmark_data_smithi039_330847_object11941:head v 1488'7877, mlcod=1488'7875)", 2024-09-14T04:54:17.107 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:17.080661+0000", 2024-09-14T04:54:17.107 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.0063354420000000002, 2024-09-14T04:54:17.107 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0066302380000000001, 2024-09-14T04:54:17.107 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:17.107 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "started", 2024-09-14T04:54:17.108 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:17.108 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.108 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:17.108 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.080661+0000", 2024-09-14T04:54:17.108 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:17.108 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.108 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.108 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:17.108 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.080661+0000", 2024-09-14T04:54:17.108 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:17.109 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.109 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.109 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:17.109 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.080660+0000", 2024-09-14T04:54:17.109 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:54:17.109 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.109 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.109 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:17.109 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.080668+0000", 2024-09-14T04:54:17.109 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.3999999999999992e-06 2024-09-14T04:54:17.110 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.110 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.110 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:17.110 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.080686+0000", 2024-09-14T04:54:17.110 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.7408e-05 2024-09-14T04:54:17.110 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.110 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.110 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:17.110 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.080693+0000", 2024-09-14T04:54:17.110 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.0530000000000001e-06 2024-09-14T04:54:17.110 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.110 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.110 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:17.111 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.080762+0000", 2024-09-14T04:54:17.111 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.9159999999999995e-05 2024-09-14T04:54:17.111 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.111 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.111 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:17.111 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.080784+0000", 2024-09-14T04:54:17.111 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.156e-05 2024-09-14T04:54:17.111 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:17.111 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:17.111 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:17.111 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.111 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.111 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.36012.0:126468 56.d e1488/1470 56:bd43f6e2:::benchmark_data_smithi039_330847_object11944:head v 1488'7972, mlcod=1488'7967)", 2024-09-14T04:54:17.111 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:17.081131+0000", 2024-09-14T04:54:17.112 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.0058655180000000001, 2024-09-14T04:54:17.112 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0062146550000000004, 2024-09-14T04:54:17.112 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:17.112 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "started", 2024-09-14T04:54:17.112 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:17.112 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.112 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:17.112 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.081131+0000", 2024-09-14T04:54:17.112 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:17.112 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.112 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.112 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:17.112 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.081131+0000", 2024-09-14T04:54:17.112 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:17.113 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.113 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.113 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:17.113 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.081130+0000", 2024-09-14T04:54:17.113 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:54:17.113 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.113 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.113 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:17.113 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.081139+0000", 2024-09-14T04:54:17.113 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.6780000000000006e-06 2024-09-14T04:54:17.113 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.113 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.113 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:17.113 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.081150+0000", 2024-09-14T04:54:17.114 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1409999999999999e-05 2024-09-14T04:54:17.114 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.114 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.114 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:17.114 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.081157+0000", 2024-09-14T04:54:17.114 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.2750000000000001e-06 2024-09-14T04:54:17.114 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.114 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.114 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:17.114 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.081456+0000", 2024-09-14T04:54:17.114 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00029996100000000001 2024-09-14T04:54:17.114 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.114 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.114 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:17.114 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.081471+0000", 2024-09-14T04:54:17.115 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5003e-05 2024-09-14T04:54:17.115 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:17.115 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:17.115 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:17.115 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.115 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.115 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.36012.0:126462 56.6 e1488/1476 56:66052e77:::benchmark_data_smithi039_330847_object11938:head v 1488'7707, mlcod=0'0)", 2024-09-14T04:54:17.115 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:17.082310+0000", 2024-09-14T04:54:17.115 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.0046869099999999999, 2024-09-14T04:54:17.115 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0050916809999999998, 2024-09-14T04:54:17.115 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:17.115 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "started", 2024-09-14T04:54:17.115 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:17.115 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.116 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:17.116 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.082310+0000", 2024-09-14T04:54:17.116 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:17.116 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.116 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.116 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:17.116 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.082310+0000", 2024-09-14T04:54:17.116 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:17.116 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.116 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.116 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:17.116 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.082309+0000", 2024-09-14T04:54:17.116 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:54:17.116 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.117 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.117 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:17.117 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.082316+0000", 2024-09-14T04:54:17.117 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.2599999999999999e-06 2024-09-14T04:54:17.117 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.117 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.117 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:17.117 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.082326+0000", 2024-09-14T04:54:17.117 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0166999999999999e-05 2024-09-14T04:54:17.117 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.117 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.117 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:17.117 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.082332+0000", 2024-09-14T04:54:17.118 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.7010000000000001e-06 2024-09-14T04:54:17.118 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.118 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.118 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:17.118 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.082394+0000", 2024-09-14T04:54:17.118 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.2259000000000001e-05 2024-09-14T04:54:17.118 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.118 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.118 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:17.118 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.082416+0000", 2024-09-14T04:54:17.118 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.1696e-05 2024-09-14T04:54:17.118 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:17.118 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:17.118 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:17.119 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.119 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.119 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:126471 56.e 56:772584d6:::benchmark_data_smithi039_330847_object11947:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1488)", 2024-09-14T04:54:17.119 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:17.082857+0000", 2024-09-14T04:54:17.119 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.0041389759999999999, 2024-09-14T04:54:17.119 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0046013640000000001, 2024-09-14T04:54:17.120 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:17.120 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:54:17.120 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:17.120 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:54:17.120 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:54:17.120 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 126471 2024-09-14T04:54:17.120 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.120 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:17.120 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.120 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:17.120 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.082857+0000", 2024-09-14T04:54:17.120 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:17.121 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.121 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.121 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:17.121 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.082857+0000", 2024-09-14T04:54:17.121 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:17.121 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.121 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.121 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:17.121 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.082856+0000", 2024-09-14T04:54:17.121 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:54:17.121 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.121 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.121 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:17.121 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.082863+0000", 2024-09-14T04:54:17.122 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.729e-06 2024-09-14T04:54:17.122 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.122 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.122 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:17.122 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.082879+0000", 2024-09-14T04:54:17.122 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5515e-05 2024-09-14T04:54:17.122 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.122 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.122 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:17.122 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.082886+0000", 2024-09-14T04:54:17.122 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.5229999999999998e-06 2024-09-14T04:54:17.122 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.122 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.123 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:17.123 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.082930+0000", 2024-09-14T04:54:17.123 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.3704e-05 2024-09-14T04:54:17.123 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.123 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.123 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:17.123 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.083039+0000", 2024-09-14T04:54:17.123 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000108516 2024-09-14T04:54:17.123 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.123 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.123 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:54:17.123 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.083136+0000", 2024-09-14T04:54:17.123 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.7664999999999994e-05 2024-09-14T04:54:17.123 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:17.124 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:17.124 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:17.124 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.124 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.124 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.36012.0:126467 56.f e1488/1470)", 2024-09-14T04:54:17.124 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:17.086635+0000", 2024-09-14T04:54:17.124 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.00036157600000000001, 2024-09-14T04:54:17.124 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00089049599999999995, 2024-09-14T04:54:17.124 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:17.124 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "queued for pg", 2024-09-14T04:54:17.124 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:17.124 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.124 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:17.125 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.086635+0000", 2024-09-14T04:54:17.125 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:17.125 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.125 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.125 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:17.125 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.086635+0000", 2024-09-14T04:54:17.125 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:17.125 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.125 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.125 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:17.125 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.086634+0000", 2024-09-14T04:54:17.125 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:54:17.125 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.125 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.126 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:17.126 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.086643+0000", 2024-09-14T04:54:17.126 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.2410000000000001e-06 2024-09-14T04:54:17.126 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.126 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.126 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:17.126 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.086655+0000", 2024-09-14T04:54:17.126 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2109e-05 2024-09-14T04:54:17.126 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.126 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.126 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:17.126 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.086662+0000", 2024-09-14T04:54:17.126 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.2660000000000004e-06 2024-09-14T04:54:17.126 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:17.127 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:17.127 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:17.127 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.127 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.127 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop_reply(client.36012.0:126471 56.e e1488/1470)", 2024-09-14T04:54:17.127 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:17.086951+0000", 2024-09-14T04:54:17.127 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 4.5827e-05, 2024-09-14T04:54:17.127 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00061707600000000004, 2024-09-14T04:54:17.127 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:17.127 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "queued for pg", 2024-09-14T04:54:17.127 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:17.127 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.127 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:17.127 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.086951+0000", 2024-09-14T04:54:17.127 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:17.128 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.128 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.128 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:17.128 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.086951+0000", 2024-09-14T04:54:17.128 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:17.128 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.128 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.129 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:17.129 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.086950+0000", 2024-09-14T04:54:17.129 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:54:17.129 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.129 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.129 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:17.129 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.086958+0000", 2024-09-14T04:54:17.129 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.2029999999999992e-06 2024-09-14T04:54:17.129 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.129 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.129 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:17.129 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.086968+0000", 2024-09-14T04:54:17.129 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.961e-06 2024-09-14T04:54:17.129 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:17.130 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:17.130 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:17.130 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:17.086974+0000", 2024-09-14T04:54:17.130 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.6749999999999999e-06 2024-09-14T04:54:17.130 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:17.130 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:17.130 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:17.130 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:17.130 INFO:teuthology.orchestra.run.smithi129.stdout: ], 2024-09-14T04:54:17.130 INFO:teuthology.orchestra.run.smithi129.stdout: "num_ops": 10 2024-09-14T04:54:17.130 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:54:17.131 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:17.172 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:17.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:17.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:17.274 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:17.309 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:17.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:17.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:17.413 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:17.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:17.453 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:17.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:17.492 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:54:17.554 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:17.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:17.600 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:17.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:17.643 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:17 smithi039 ceph-mon[106470]: pgmap v2797: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 3571/316266 objects degraded (1.129%); 2129/316266 objects misplaced (0.673%); 7.2 MiB/s, 115 objects/s recovering 2024-09-14T04:54:17.643 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:17 smithi039 ceph-mon[111235]: pgmap v2797: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 3571/316266 objects degraded (1.129%); 2129/316266 objects misplaced (0.673%); 7.2 MiB/s, 115 objects/s recovering 2024-09-14T04:54:17.701 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:17.737 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:17.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:17.739 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:17.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:17 smithi129 ceph-mon[101316]: pgmap v2797: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 3571/316266 objects degraded (1.129%); 2129/316266 objects misplaced (0.673%); 7.2 MiB/s, 115 objects/s recovering 2024-09-14T04:54:17.840 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:17.872 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:17.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:17.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:17.940 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-14T04:54:17.975 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:17.982 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:54:18.039 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:18.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:18.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:18.141 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:18.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:18.186 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:18.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:18.287 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:18.310 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:54:18.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:18.325 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:18.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:18.426 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:18.435 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.3 to 1 (10000) 2024-09-14T04:54:18.490 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:18.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:18.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:18.592 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:18.616 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:18.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:18.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:18.718 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:18.740 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:18.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:18.741 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:18.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:18 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3704207404' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:54:18.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:18 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:54:18.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:18 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3704207404' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:54:18.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:18 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:54:18.791 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:54:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:54:18] "GET /metrics HTTP/1.1" 200 36804 "" "Prometheus/2.51.0" 2024-09-14T04:54:18.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:18 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3704207404' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:54:18.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:18 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:54:18.842 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:18.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:18.865 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:18.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:18.966 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:18.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:18.990 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:18.991 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:19.092 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:19.114 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:19.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:19.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:19.216 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:19.239 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:19.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:19.240 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:19.340 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:19.363 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:19.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:19.364 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:19.465 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:19.493 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:19.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:19.494 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:19.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:19 smithi129 ceph-mon[101316]: pgmap v2798: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 3571/316266 objects degraded (1.129%); 2129/316266 objects misplaced (0.673%); 3.6 MiB/s, 56 objects/s recovering 2024-09-14T04:54:19.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:19 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-14T04:54:19.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:19 smithi129 ceph-mon[101316]: osdmap e1489: 8 total, 8 up, 8 in 2024-09-14T04:54:19.594 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:19.624 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:19.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:19.625 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:19.726 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:19.765 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:19.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:19.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:19.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:19 smithi039 ceph-mon[106470]: pgmap v2798: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 3571/316266 objects degraded (1.129%); 2129/316266 objects misplaced (0.673%); 3.6 MiB/s, 56 objects/s recovering 2024-09-14T04:54:19.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:19 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-14T04:54:19.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:19 smithi039 ceph-mon[106470]: osdmap e1489: 8 total, 8 up, 8 in 2024-09-14T04:54:19.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:19 smithi039 ceph-mon[111235]: pgmap v2798: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.4 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 534 op/s; 3571/316266 objects degraded (1.129%); 2129/316266 objects misplaced (0.673%); 3.6 MiB/s, 56 objects/s recovering 2024-09-14T04:54:19.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:19 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-14T04:54:19.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:19 smithi039 ceph-mon[111235]: osdmap e1489: 8 total, 8 up, 8 in 2024-09-14T04:54:19.866 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:19.903 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:19.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:19.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:20.005 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:20.048 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:20.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:20.049 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:20.150 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:20.194 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:20.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:20.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:20.252 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:54:20.252 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [], 2024-09-14T04:54:20.253 INFO:teuthology.orchestra.run.smithi129.stdout: "complaint_time": 30, 2024-09-14T04:54:20.253 INFO:teuthology.orchestra.run.smithi129.stdout: "num_blocked_ops": 0 2024-09-14T04:54:20.253 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:54:20.296 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:20.331 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:20.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:20.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:20.433 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:20.476 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:20.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:20.477 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:20.577 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:20.608 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:20.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:20.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:20.709 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:20.750 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:20.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:20.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:20.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:20 smithi129 ceph-mon[101316]: osdmap e1490: 8 total, 8 up, 8 in 2024-09-14T04:54:20.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:20 smithi039 ceph-mon[106470]: osdmap e1490: 8 total, 8 up, 8 in 2024-09-14T04:54:20.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:20 smithi039 ceph-mon[111235]: osdmap e1490: 8 total, 8 up, 8 in 2024-09-14T04:54:20.852 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:20.883 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:20.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:20.885 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:20.985 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:21.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:21.032 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:21.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:21.133 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:21.168 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:21.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:21.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:21.271 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:21.291 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-14T04:54:21.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:21.317 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:21.318 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:21.419 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:21.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:21.458 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:21.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:21.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:21 smithi129 ceph-mon[101316]: pgmap v2801: 99 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 3 peering, 93 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 705/304638 objects misplaced (0.231%); 7.8 MiB/s, 125 objects/s recovering 2024-09-14T04:54:21.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:21 smithi129 ceph-mon[101316]: osdmap e1491: 8 total, 8 up, 8 in 2024-09-14T04:54:21.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:21 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2024-09-14T04:54:21.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:21 smithi129 ceph-mon[101316]: osdmap e1492: 8 total, 8 up, 8 in 2024-09-14T04:54:21.558 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:21.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:21.591 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:21.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:21.618 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 4 1 2024-09-14T04:54:21.669 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:54:21.695 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:21.707 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:54:21.736 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:21 smithi039 ceph-mon[111235]: pgmap v2801: 99 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 3 peering, 93 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 705/304638 objects misplaced (0.231%); 7.8 MiB/s, 125 objects/s recovering 2024-09-14T04:54:21.736 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:21 smithi039 ceph-mon[111235]: osdmap e1491: 8 total, 8 up, 8 in 2024-09-14T04:54:21.737 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:21 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2024-09-14T04:54:21.737 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:21 smithi039 ceph-mon[111235]: osdmap e1492: 8 total, 8 up, 8 in 2024-09-14T04:54:21.742 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:21 smithi039 ceph-mon[106470]: pgmap v2801: 99 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 3 peering, 93 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 705/304638 objects misplaced (0.231%); 7.8 MiB/s, 125 objects/s recovering 2024-09-14T04:54:21.742 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:21 smithi039 ceph-mon[106470]: osdmap e1491: 8 total, 8 up, 8 in 2024-09-14T04:54:21.742 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:21 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 1 pg undersized (PG_DEGRADED) 2024-09-14T04:54:21.742 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:21 smithi039 ceph-mon[106470]: osdmap e1492: 8 total, 8 up, 8 in 2024-09-14T04:54:21.746 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:21.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:21.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:21.848 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:21.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:21.895 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:21.896 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:21.997 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:22.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:22.039 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:22.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:22.140 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:22.179 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:22.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:22.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:22.282 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:22.322 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:22.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:22.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:22.424 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:22.460 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:22.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:22.461 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:22.562 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:22.604 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:22.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:22.605 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:22.705 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:22.742 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:22.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:22.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:22.843 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:22.886 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:22.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:22.888 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:22.988 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:23.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:23.025 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:23.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:23.126 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:23.161 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:23.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:23.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:23.213 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:54:23.263 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:23.298 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:23.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:23.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:23.400 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:23.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:23.441 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:23.441 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:23.487 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is set 2024-09-14T04:54:23.487 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:23 smithi039 ceph-mon[106470]: pgmap v2804: 99 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 3 peering, 93 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 968 op/s; 705/304638 objects misplaced (0.231%); 7.5 MiB/s, 120 objects/s recovering 2024-09-14T04:54:23.487 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:23 smithi039 ceph-mon[106470]: osdmap e1493: 8 total, 8 up, 8 in 2024-09-14T04:54:23.487 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:23 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:54:23.487 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:54:23.487 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:23 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1814603329' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:54:23.488 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:23 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:54:23.488 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:23 smithi039 ceph-mon[111235]: pgmap v2804: 99 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 3 peering, 93 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 968 op/s; 705/304638 objects misplaced (0.231%); 7.5 MiB/s, 120 objects/s recovering 2024-09-14T04:54:23.488 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:23 smithi039 ceph-mon[111235]: osdmap e1493: 8 total, 8 up, 8 in 2024-09-14T04:54:23.488 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:23 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:54:23.488 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:54:23.488 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:23 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1814603329' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:54:23.488 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:23 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:54:23.505 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:23 smithi129 ceph-mon[101316]: pgmap v2804: 99 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 3 peering, 93 active+clean; 6.2 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 968 op/s; 705/304638 objects misplaced (0.231%); 7.5 MiB/s, 120 objects/s recovering 2024-09-14T04:54:23.505 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:23 smithi129 ceph-mon[101316]: osdmap e1493: 8 total, 8 up, 8 in 2024-09-14T04:54:23.505 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:23 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:54:23.505 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:54:23.505 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:23 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1814603329' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:54:23.506 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:23 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:54:23.542 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:23.575 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:23.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:23.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:23.677 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:23.710 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:23.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:23.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:23.812 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:23.836 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:54:23.836 INFO:teuthology.orchestra.run.smithi129.stdout: "size": 20, 2024-09-14T04:54:23.836 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 600, 2024-09-14T04:54:23.836 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [ 2024-09-14T04:54:23.836 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.836 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:149714 55.a 55:585bdded:::benchmark_data_smithi039_297155_object25781:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1447)", 2024-09-14T04:54:23.836 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:49:58.462329+0000", 2024-09-14T04:54:23.836 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 265.36892525500002, 2024-09-14T04:54:23.836 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.35080457599999998, 2024-09-14T04:54:23.836 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:23.836 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:23.837 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:23.837 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:54:23.837 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:54:23.837 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 149714 2024-09-14T04:54:23.837 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.837 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:23.837 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.837 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:23.837 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.462329+0000", 2024-09-14T04:54:23.838 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.838 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.838 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.838 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:23.838 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.462329+0000", 2024-09-14T04:54:23.838 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.838 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.838 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.838 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:23.838 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.462327+0000", 2024-09-14T04:54:23.838 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:54:23.838 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.839 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.839 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:23.839 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.462334+0000", 2024-09-14T04:54:23.839 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.888e-06 2024-09-14T04:54:23.839 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.839 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.839 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:23.839 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.462348+0000", 2024-09-14T04:54:23.839 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3735e-05 2024-09-14T04:54:23.839 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.839 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.839 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:23.840 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.462355+0000", 2024-09-14T04:54:23.840 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.9870000000000002e-06 2024-09-14T04:54:23.840 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.840 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.840 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:23.840 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.463626+0000", 2024-09-14T04:54:23.840 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0012708610000000001 2024-09-14T04:54:23.840 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.840 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.840 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:23.841 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.463774+0000", 2024-09-14T04:54:23.841 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00014832100000000001 2024-09-14T04:54:23.841 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.841 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.841 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,3", 2024-09-14T04:54:23.841 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.463895+0000", 2024-09-14T04:54:23.841 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012069399999999999 2024-09-14T04:54:23.841 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.841 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.841 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.841 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471453+0000", 2024-09-14T04:54:23.842 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0075585490000000003 2024-09-14T04:54:23.842 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.842 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.842 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:54:23.842 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471460+0000", 2024-09-14T04:54:23.842 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8719999999999996e-06 2024-09-14T04:54:23.842 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.842 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.842 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.842 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.813005+0000", 2024-09-14T04:54:23.842 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.34154447599999999 2024-09-14T04:54:23.843 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.843 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.843 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:23.843 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.813087+0000", 2024-09-14T04:54:23.843 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.2387999999999998e-05 2024-09-14T04:54:23.843 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.843 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.843 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:23.843 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.813134+0000", 2024-09-14T04:54:23.843 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6684000000000002e-05 2024-09-14T04:54:23.844 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.844 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:23.844 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.844 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.844 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.844 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:149716 55.a 55:53418830:::benchmark_data_smithi039_297155_object25783:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1447)", 2024-09-14T04:54:23.844 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:49:58.462361+0000", 2024-09-14T04:54:23.844 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 265.36889361800002, 2024-09-14T04:54:23.844 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.35768450099999999, 2024-09-14T04:54:23.844 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:23.844 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:23.845 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:23.845 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:54:23.845 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:54:23.845 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 149716 2024-09-14T04:54:23.845 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.845 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:23.845 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.845 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:23.845 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.462361+0000", 2024-09-14T04:54:23.845 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.845 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.846 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.846 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:23.846 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.462361+0000", 2024-09-14T04:54:23.846 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.846 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.846 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.846 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:23.846 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.462361+0000", 2024-09-14T04:54:23.846 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:54:23.846 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.846 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.847 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:23.847 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.462363+0000", 2024-09-14T04:54:23.847 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.2649999999999999e-06 2024-09-14T04:54:23.847 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.847 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.847 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:23.847 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.462368+0000", 2024-09-14T04:54:23.847 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.3539999999999999e-06 2024-09-14T04:54:23.847 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.847 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.848 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:23.848 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.462373+0000", 2024-09-14T04:54:23.848 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.3939999999999998e-06 2024-09-14T04:54:23.848 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.848 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.848 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:23.848 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.467268+0000", 2024-09-14T04:54:23.848 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.004895234 2024-09-14T04:54:23.848 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.849 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.849 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:23.849 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.467997+0000", 2024-09-14T04:54:23.849 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00072953799999999995 2024-09-14T04:54:23.849 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.849 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.849 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,3", 2024-09-14T04:54:23.849 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.468102+0000", 2024-09-14T04:54:23.850 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000104978 2024-09-14T04:54:23.850 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.850 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.850 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.850 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.473040+0000", 2024-09-14T04:54:23.850 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0049380559999999997 2024-09-14T04:54:23.850 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.850 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.850 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:54:23.850 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.475341+0000", 2024-09-14T04:54:23.850 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0023002750000000001 2024-09-14T04:54:23.850 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.850 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.851 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.851 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.819968+0000", 2024-09-14T04:54:23.851 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.34462760799999997 2024-09-14T04:54:23.851 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.851 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.851 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:23.851 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.820010+0000", 2024-09-14T04:54:23.851 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1309e-05 2024-09-14T04:54:23.851 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.851 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.851 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:23.851 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.820045+0000", 2024-09-14T04:54:23.851 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5831000000000003e-05 2024-09-14T04:54:23.851 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.851 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:23.852 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.852 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.852 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.852 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:149720 55.4 55:25ea9b58:::benchmark_data_smithi039_297155_object25787:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1447)", 2024-09-14T04:54:23.852 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:49:58.467387+0000", 2024-09-14T04:54:23.852 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 265.36386750899999, 2024-09-14T04:54:23.852 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.34366680700000002, 2024-09-14T04:54:23.852 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:23.852 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:23.852 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:23.852 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:54:23.852 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:54:23.853 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 149720 2024-09-14T04:54:23.853 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.853 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:23.853 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.853 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:23.853 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.467387+0000", 2024-09-14T04:54:23.853 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.854 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.854 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.854 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:23.854 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.467387+0000", 2024-09-14T04:54:23.854 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.854 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.854 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.854 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:23.854 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.467386+0000", 2024-09-14T04:54:23.854 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:54:23.854 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.854 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.854 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:23.855 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.467392+0000", 2024-09-14T04:54:23.855 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.0809999999999999e-06 2024-09-14T04:54:23.855 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.856 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.856 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:23.856 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.467401+0000", 2024-09-14T04:54:23.856 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.4609999999999997e-06 2024-09-14T04:54:23.856 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.856 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.857 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:23.857 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.467408+0000", 2024-09-14T04:54:23.857 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.3570000000000003e-06 2024-09-14T04:54:23.857 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.857 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.857 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:23.858 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.467448+0000", 2024-09-14T04:54:23.858 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0255e-05 2024-09-14T04:54:23.858 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.858 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.858 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:23.858 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.467545+0000", 2024-09-14T04:54:23.858 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.7424999999999996e-05 2024-09-14T04:54:23.858 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.858 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.858 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,3", 2024-09-14T04:54:23.858 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.469890+0000", 2024-09-14T04:54:23.858 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.002344984 2024-09-14T04:54:23.859 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.859 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.859 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:54:23.859 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.477279+0000", 2024-09-14T04:54:23.859 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0073887639999999999 2024-09-14T04:54:23.859 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.859 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.859 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.859 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.477318+0000", 2024-09-14T04:54:23.859 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8495999999999999e-05 2024-09-14T04:54:23.859 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.859 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.860 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.860 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.810969+0000", 2024-09-14T04:54:23.860 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33365140500000001 2024-09-14T04:54:23.860 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.860 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.860 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:23.861 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.811017+0000", 2024-09-14T04:54:23.861 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7828999999999999e-05 2024-09-14T04:54:23.861 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.861 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.861 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:23.861 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.811054+0000", 2024-09-14T04:54:23.861 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7015999999999999e-05 2024-09-14T04:54:23.861 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.861 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:23.861 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.861 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.861 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.861 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:149723 55.4 55:2fc5bef1:::benchmark_data_smithi039_297155_object25790:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1447)", 2024-09-14T04:54:23.862 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:49:58.471232+0000", 2024-09-14T04:54:23.862 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 265.360022453, 2024-09-14T04:54:23.862 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.34963855999999999, 2024-09-14T04:54:23.862 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:23.862 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:23.862 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:23.862 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:54:23.862 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:54:23.862 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 149723 2024-09-14T04:54:23.862 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.862 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:23.862 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.862 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:23.863 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471232+0000", 2024-09-14T04:54:23.863 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.863 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.863 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.863 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:23.863 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471232+0000", 2024-09-14T04:54:23.863 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.863 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.863 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.863 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:23.863 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471231+0000", 2024-09-14T04:54:23.863 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:54:23.863 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.863 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.864 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:23.864 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471237+0000", 2024-09-14T04:54:23.864 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8730000000000001e-06 2024-09-14T04:54:23.864 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.864 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.864 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:23.864 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471252+0000", 2024-09-14T04:54:23.864 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.4996e-05 2024-09-14T04:54:23.864 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.864 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.864 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:23.864 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471259+0000", 2024-09-14T04:54:23.864 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.956e-06 2024-09-14T04:54:23.865 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.865 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.865 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:23.865 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.473268+0000", 2024-09-14T04:54:23.865 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.002008489 2024-09-14T04:54:23.865 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.865 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.865 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:23.865 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.473377+0000", 2024-09-14T04:54:23.865 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010955699999999999 2024-09-14T04:54:23.865 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.865 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.865 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,3", 2024-09-14T04:54:23.866 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.473485+0000", 2024-09-14T04:54:23.866 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010726999999999999 2024-09-14T04:54:23.866 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.866 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.866 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:54:23.866 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.477416+0000", 2024-09-14T04:54:23.866 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0039313739999999996 2024-09-14T04:54:23.866 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.866 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.866 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.866 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.492154+0000", 2024-09-14T04:54:23.866 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.014738335 2024-09-14T04:54:23.866 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.867 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.867 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.867 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.820787+0000", 2024-09-14T04:54:23.867 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.328632704 2024-09-14T04:54:23.867 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.867 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.867 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:23.867 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.820833+0000", 2024-09-14T04:54:23.867 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6109999999999997e-05 2024-09-14T04:54:23.867 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.867 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.867 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:23.867 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.820871+0000", 2024-09-14T04:54:23.868 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7437000000000001e-05 2024-09-14T04:54:23.868 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.868 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:23.868 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.868 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.868 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.868 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:149727 55.7 55:e5894293:::benchmark_data_smithi039_297155_object25794:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1447)", 2024-09-14T04:54:23.868 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:49:58.471708+0000", 2024-09-14T04:54:23.868 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 265.35954607000002, 2024-09-14T04:54:23.868 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.34548536699999999, 2024-09-14T04:54:23.868 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:23.868 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:23.868 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:23.868 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:54:23.869 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:54:23.869 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 149727 2024-09-14T04:54:23.869 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.869 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:23.869 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.869 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:23.869 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471708+0000", 2024-09-14T04:54:23.869 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.869 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.869 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.869 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:23.869 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471708+0000", 2024-09-14T04:54:23.869 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.870 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.870 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.870 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:23.870 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471708+0000", 2024-09-14T04:54:23.870 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:54:23.870 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.870 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.870 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:23.870 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471712+0000", 2024-09-14T04:54:23.870 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.3649999999999997e-06 2024-09-14T04:54:23.871 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.872 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.872 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:23.872 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471723+0000", 2024-09-14T04:54:23.873 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0825000000000001e-05 2024-09-14T04:54:23.873 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.873 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.873 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:23.873 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471728+0000", 2024-09-14T04:54:23.873 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.5589999999999999e-06 2024-09-14T04:54:23.873 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.873 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.873 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:23.873 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471755+0000", 2024-09-14T04:54:23.874 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.7868000000000002e-05 2024-09-14T04:54:23.874 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.874 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.874 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:23.874 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471830+0000", 2024-09-14T04:54:23.874 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.5081999999999997e-05 2024-09-14T04:54:23.874 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.874 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.874 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:54:23.874 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.471899+0000", 2024-09-14T04:54:23.874 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8860000000000001e-05 2024-09-14T04:54:23.874 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.875 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.875 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:54:23.875 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.478613+0000", 2024-09-14T04:54:23.875 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.006713498 2024-09-14T04:54:23.875 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.875 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.875 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.876 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.480092+0000", 2024-09-14T04:54:23.876 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001479181 2024-09-14T04:54:23.876 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.876 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.876 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.876 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.817114+0000", 2024-09-14T04:54:23.876 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33702223599999998 2024-09-14T04:54:23.876 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.876 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.876 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:23.876 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.817160+0000", 2024-09-14T04:54:23.876 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6199999999999998e-05 2024-09-14T04:54:23.876 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.877 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.877 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:23.877 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.817194+0000", 2024-09-14T04:54:23.877 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3414999999999998e-05 2024-09-14T04:54:23.877 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.877 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:23.877 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.877 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.878 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.878 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:149731 55.7 55:ec89c09c:::benchmark_data_smithi039_297155_object25798:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1447)", 2024-09-14T04:54:23.878 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:49:58.474658+0000", 2024-09-14T04:54:23.878 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 265.35659622700001, 2024-09-14T04:54:23.878 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.34318062199999999, 2024-09-14T04:54:23.878 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:23.878 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:23.878 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:23.878 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:54:23.878 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:54:23.878 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 149731 2024-09-14T04:54:23.878 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.878 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:23.879 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.879 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:23.879 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.474658+0000", 2024-09-14T04:54:23.879 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.879 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.879 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.879 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:23.879 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.474658+0000", 2024-09-14T04:54:23.879 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.879 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.880 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.880 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:23.880 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.474657+0000", 2024-09-14T04:54:23.880 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:54:23.880 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.880 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.880 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:23.880 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.474663+0000", 2024-09-14T04:54:23.881 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.0390000000000001e-06 2024-09-14T04:54:23.881 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.881 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.881 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:23.881 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.474673+0000", 2024-09-14T04:54:23.881 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0284e-05 2024-09-14T04:54:23.881 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.881 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.881 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:23.881 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.474679+0000", 2024-09-14T04:54:23.881 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.1720000000000004e-06 2024-09-14T04:54:23.882 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.882 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.882 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:23.882 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.475315+0000", 2024-09-14T04:54:23.882 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00063524000000000002 2024-09-14T04:54:23.882 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.882 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.882 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:23.882 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.475413+0000", 2024-09-14T04:54:23.882 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.8039000000000001e-05 2024-09-14T04:54:23.882 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.882 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.882 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:54:23.882 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.475511+0000", 2024-09-14T04:54:23.883 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.8388999999999996e-05 2024-09-14T04:54:23.883 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.883 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.883 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:54:23.883 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.478928+0000", 2024-09-14T04:54:23.883 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0034173020000000001 2024-09-14T04:54:23.883 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.883 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.883 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.883 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.480126+0000", 2024-09-14T04:54:23.883 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0011971449999999999 2024-09-14T04:54:23.883 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.883 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.883 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.884 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.817755+0000", 2024-09-14T04:54:23.884 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.337629293 2024-09-14T04:54:23.884 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.884 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.884 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:23.884 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.817810+0000", 2024-09-14T04:54:23.884 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.5318000000000001e-05 2024-09-14T04:54:23.884 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.884 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.884 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:23.884 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.817839+0000", 2024-09-14T04:54:23.884 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.8728999999999999e-05 2024-09-14T04:54:23.885 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.885 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:23.885 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.885 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.885 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.885 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:149732 55.a 55:5aa14efc:::benchmark_data_smithi039_297155_object25799:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1447)", 2024-09-14T04:54:23.885 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:49:58.492517+0000", 2024-09-14T04:54:23.885 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 265.338737251, 2024-09-14T04:54:23.885 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33853844900000002, 2024-09-14T04:54:23.885 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:23.885 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:23.885 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:23.885 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:54:23.885 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:54:23.886 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 149732 2024-09-14T04:54:23.886 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.886 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:23.886 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.886 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:23.886 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.492517+0000", 2024-09-14T04:54:23.886 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.886 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.886 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.886 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:23.886 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.492517+0000", 2024-09-14T04:54:23.886 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.886 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.886 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.887 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:23.887 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.492516+0000", 2024-09-14T04:54:23.887 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:54:23.887 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.887 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.887 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:23.887 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.492522+0000", 2024-09-14T04:54:23.887 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.8669999999999999e-06 2024-09-14T04:54:23.887 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.887 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.887 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:23.887 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.492533+0000", 2024-09-14T04:54:23.887 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0891e-05 2024-09-14T04:54:23.887 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.887 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.888 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:23.888 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.492540+0000", 2024-09-14T04:54:23.888 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8709999999999999e-06 2024-09-14T04:54:23.888 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.888 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.888 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:23.888 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.492582+0000", 2024-09-14T04:54:23.888 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1859e-05 2024-09-14T04:54:23.888 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.888 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.888 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:23.888 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.492669+0000", 2024-09-14T04:54:23.888 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.7410000000000005e-05 2024-09-14T04:54:23.889 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.889 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.889 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,3", 2024-09-14T04:54:23.889 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.492745+0000", 2024-09-14T04:54:23.889 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.6487000000000001e-05 2024-09-14T04:54:23.889 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.889 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.889 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:54:23.889 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.495567+0000", 2024-09-14T04:54:23.889 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.002821287 2024-09-14T04:54:23.889 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.889 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.889 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.890 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.495890+0000", 2024-09-14T04:54:23.890 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00032340899999999998 2024-09-14T04:54:23.890 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.890 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.890 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.890 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.830975+0000", 2024-09-14T04:54:23.890 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33508471000000001 2024-09-14T04:54:23.890 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.890 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.890 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:23.890 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.831020+0000", 2024-09-14T04:54:23.890 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.4799999999999998e-05 2024-09-14T04:54:23.890 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.891 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.891 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:23.891 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:58.831056+0000", 2024-09-14T04:54:23.891 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6112000000000003e-05 2024-09-14T04:54:23.891 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.891 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:23.891 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.891 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.891 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.891 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:189801 55.d 55:b1b98788:::benchmark_data_smithi039_297155_object65868:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1453)", 2024-09-14T04:54:23.891 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:50:44.333453+0000", 2024-09-14T04:54:23.891 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 219.49780184299999, 2024-09-14T04:54:23.891 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.26339609400000003, 2024-09-14T04:54:23.892 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:23.892 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:23.892 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:23.892 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:54:23.892 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:54:23.892 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 189801 2024-09-14T04:54:23.892 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.892 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:23.892 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.892 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:23.892 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.333453+0000", 2024-09-14T04:54:23.892 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.892 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.893 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.893 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:23.893 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.333453+0000", 2024-09-14T04:54:23.893 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.893 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.893 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.893 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:23.893 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.333452+0000", 2024-09-14T04:54:23.893 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:54:23.893 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.893 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.893 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:23.893 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.333457+0000", 2024-09-14T04:54:23.893 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.0069999999999997e-06 2024-09-14T04:54:23.894 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.894 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.894 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:23.894 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.333475+0000", 2024-09-14T04:54:23.894 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.8292999999999999e-05 2024-09-14T04:54:23.894 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.894 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.894 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:23.894 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.333482+0000", 2024-09-14T04:54:23.894 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.9079999999999998e-06 2024-09-14T04:54:23.894 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.894 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.894 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:23.895 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.333523+0000", 2024-09-14T04:54:23.895 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1199000000000003e-05 2024-09-14T04:54:23.895 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.895 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.895 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:23.895 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.333630+0000", 2024-09-14T04:54:23.895 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010705999999999999 2024-09-14T04:54:23.895 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.895 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.895 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:54:23.895 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.333726+0000", 2024-09-14T04:54:23.895 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.5357000000000002e-05 2024-09-14T04:54:23.895 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.895 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.896 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:54:23.896 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.337315+0000", 2024-09-14T04:54:23.896 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0035892530000000001 2024-09-14T04:54:23.896 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.896 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.896 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.896 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.343266+0000", 2024-09-14T04:54:23.896 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0059506280000000003 2024-09-14T04:54:23.896 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.896 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.896 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.896 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.596770+0000", 2024-09-14T04:54:23.896 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.25350475900000002 2024-09-14T04:54:23.896 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.897 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.897 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:23.897 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.596808+0000", 2024-09-14T04:54:23.897 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7327999999999999e-05 2024-09-14T04:54:23.897 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.897 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.897 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:23.897 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:50:44.596849+0000", 2024-09-14T04:54:23.897 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1109999999999998e-05 2024-09-14T04:54:23.897 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.897 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:23.897 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.897 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.897 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.898 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:242003 55.d 55:bbcd1f69:::benchmark_data_smithi039_297155_object118070:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1462)", 2024-09-14T04:54:23.898 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:51:43.128261+0000", 2024-09-14T04:54:23.898 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 160.702993453, 2024-09-14T04:54:23.898 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.27423907800000002, 2024-09-14T04:54:23.898 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:23.898 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:23.898 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:23.898 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:54:23.898 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:54:23.898 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 242003 2024-09-14T04:54:23.898 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.898 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:23.898 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.898 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:23.899 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.128261+0000", 2024-09-14T04:54:23.899 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.899 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.899 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.899 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:23.899 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.128261+0000", 2024-09-14T04:54:23.899 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.899 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.899 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.899 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:23.899 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.128260+0000", 2024-09-14T04:54:23.899 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:54:23.899 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.900 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.900 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:23.900 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.128266+0000", 2024-09-14T04:54:23.900 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.2480000000000001e-06 2024-09-14T04:54:23.900 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.900 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.900 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:23.900 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.128279+0000", 2024-09-14T04:54:23.900 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2914e-05 2024-09-14T04:54:23.900 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.900 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.900 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:23.900 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.128285+0000", 2024-09-14T04:54:23.901 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.1160000000000004e-06 2024-09-14T04:54:23.901 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.901 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.901 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:23.901 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.128386+0000", 2024-09-14T04:54:23.901 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010097700000000001 2024-09-14T04:54:23.901 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.901 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.901 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:23.901 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.128487+0000", 2024-09-14T04:54:23.901 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000100932 2024-09-14T04:54:23.901 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.901 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.901 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:54:23.902 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.128588+0000", 2024-09-14T04:54:23.902 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010066 2024-09-14T04:54:23.902 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.902 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.902 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.902 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.135430+0000", 2024-09-14T04:54:23.902 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0068419220000000003 2024-09-14T04:54:23.902 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.902 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.902 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:54:23.902 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.141122+0000", 2024-09-14T04:54:23.902 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0056919500000000003 2024-09-14T04:54:23.902 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.903 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.903 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.903 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.402403+0000", 2024-09-14T04:54:23.903 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.26128098 2024-09-14T04:54:23.903 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.903 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.903 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:23.903 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.402451+0000", 2024-09-14T04:54:23.903 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8548999999999999e-05 2024-09-14T04:54:23.903 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.903 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.903 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:23.903 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.402500+0000", 2024-09-14T04:54:23.903 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.9088000000000002e-05 2024-09-14T04:54:23.904 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.904 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:23.904 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.904 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.904 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.904 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:242005 55.d 55:b928e5ba:::benchmark_data_smithi039_297155_object118072:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1462)", 2024-09-14T04:54:23.904 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:51:43.132835+0000", 2024-09-14T04:54:23.904 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 160.698419882, 2024-09-14T04:54:23.904 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.27934682100000002, 2024-09-14T04:54:23.904 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:23.904 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:23.904 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:23.904 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:54:23.905 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:54:23.905 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 242005 2024-09-14T04:54:23.905 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.905 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:23.905 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.905 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:23.905 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.132835+0000", 2024-09-14T04:54:23.905 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.905 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.905 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.905 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:23.905 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.132835+0000", 2024-09-14T04:54:23.905 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.906 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.906 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.906 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:23.906 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.132833+0000", 2024-09-14T04:54:23.906 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:54:23.906 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.906 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.906 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:23.906 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.132844+0000", 2024-09-14T04:54:23.906 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0869e-05 2024-09-14T04:54:23.906 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.906 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.906 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:23.906 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.132858+0000", 2024-09-14T04:54:23.907 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.4596999999999999e-05 2024-09-14T04:54:23.907 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.907 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.907 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:23.907 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.132867+0000", 2024-09-14T04:54:23.907 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.8729999999999996e-06 2024-09-14T04:54:23.907 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.907 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.907 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:23.907 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.135464+0000", 2024-09-14T04:54:23.907 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0025963090000000002 2024-09-14T04:54:23.907 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.907 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.908 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:23.908 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.137624+0000", 2024-09-14T04:54:23.908 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0021603550000000001 2024-09-14T04:54:23.908 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.909 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.909 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:54:23.909 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.137730+0000", 2024-09-14T04:54:23.909 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000105919 2024-09-14T04:54:23.909 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.909 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.909 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.909 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.144887+0000", 2024-09-14T04:54:23.909 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0071570080000000003 2024-09-14T04:54:23.910 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.910 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.910 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:54:23.910 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.145931+0000", 2024-09-14T04:54:23.910 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001044519 2024-09-14T04:54:23.910 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.910 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.910 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.910 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.412072+0000", 2024-09-14T04:54:23.910 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.266140816 2024-09-14T04:54:23.910 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.910 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.910 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:23.910 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.412146+0000", 2024-09-14T04:54:23.911 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.3541999999999995e-05 2024-09-14T04:54:23.911 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.911 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.911 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:23.911 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.412181+0000", 2024-09-14T04:54:23.911 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5800999999999998e-05 2024-09-14T04:54:23.911 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.911 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:23.911 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.911 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.911 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.911 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:242006 55.d 55:bd52749d:::benchmark_data_smithi039_297155_object118073:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1462)", 2024-09-14T04:54:23.911 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:51:43.132875+0000", 2024-09-14T04:54:23.911 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 160.69838004499999, 2024-09-14T04:54:23.912 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.27944468099999997, 2024-09-14T04:54:23.912 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:23.912 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:23.912 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:23.912 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:54:23.912 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:54:23.912 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 242006 2024-09-14T04:54:23.912 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.912 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:23.912 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.912 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:23.912 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.132875+0000", 2024-09-14T04:54:23.912 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.912 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.913 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.913 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:23.913 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.132875+0000", 2024-09-14T04:54:23.913 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.913 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.913 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.913 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:23.913 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.132874+0000", 2024-09-14T04:54:23.913 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:54:23.913 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.913 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.913 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:23.913 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.132877+0000", 2024-09-14T04:54:23.913 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.055e-06 2024-09-14T04:54:23.914 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.914 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.914 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:23.914 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.132885+0000", 2024-09-14T04:54:23.914 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.6240000000000002e-06 2024-09-14T04:54:23.914 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.914 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.914 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:23.914 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.132890+0000", 2024-09-14T04:54:23.914 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2070000000000003e-06 2024-09-14T04:54:23.914 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.914 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.914 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:23.914 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.141249+0000", 2024-09-14T04:54:23.915 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0083594450000000001 2024-09-14T04:54:23.915 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.915 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.915 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:23.915 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.141396+0000", 2024-09-14T04:54:23.915 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000146453 2024-09-14T04:54:23.915 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.915 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.915 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:54:23.915 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.141499+0000", 2024-09-14T04:54:23.915 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010357399999999999 2024-09-14T04:54:23.915 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.915 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.915 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:54:23.915 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.145944+0000", 2024-09-14T04:54:23.916 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0044445420000000001 2024-09-14T04:54:23.916 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.916 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.916 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.916 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.151100+0000", 2024-09-14T04:54:23.916 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0051561009999999997 2024-09-14T04:54:23.916 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.916 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.916 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.916 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.412214+0000", 2024-09-14T04:54:23.916 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.26111378200000002 2024-09-14T04:54:23.916 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.916 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.916 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:23.917 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.412283+0000", 2024-09-14T04:54:23.917 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.9319999999999994e-05 2024-09-14T04:54:23.917 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.917 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.917 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:23.917 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.412319+0000", 2024-09-14T04:54:23.917 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6102000000000001e-05 2024-09-14T04:54:23.917 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.917 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:23.917 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.917 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.917 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:23.918 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:23.918 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:23.919 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:242020 55.d 55:b91bda35:::benchmark_data_smithi039_297155_object118087:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1462)", 2024-09-14T04:54:23.920 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:51:43.146016+0000", 2024-09-14T04:54:23.920 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 160.68523807099999, 2024-09-14T04:54:23.920 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.26663411599999998, 2024-09-14T04:54:23.920 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:23.920 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:23.920 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:23.920 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:54:23.920 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:54:23.920 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 242020 2024-09-14T04:54:23.920 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.920 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:23.920 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.921 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:23.921 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.146016+0000", 2024-09-14T04:54:23.921 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.921 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.921 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.921 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:23.921 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.146016+0000", 2024-09-14T04:54:23.921 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.921 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.921 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.921 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:23.921 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.146016+0000", 2024-09-14T04:54:23.921 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:54:23.921 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.922 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.922 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:23.922 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.146019+0000", 2024-09-14T04:54:23.922 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.287e-06 2024-09-14T04:54:23.922 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.922 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.922 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:23.922 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.146030+0000", 2024-09-14T04:54:23.922 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0913999999999999e-05 2024-09-14T04:54:23.922 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.922 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.922 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:23.922 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.146036+0000", 2024-09-14T04:54:23.922 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.0270000000000001e-06 2024-09-14T04:54:23.923 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.923 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.923 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:23.923 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.146082+0000", 2024-09-14T04:54:23.923 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6150999999999999e-05 2024-09-14T04:54:23.923 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.923 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.923 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:23.923 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.146193+0000", 2024-09-14T04:54:23.923 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011066 2024-09-14T04:54:23.923 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.923 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.923 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,4", 2024-09-14T04:54:23.923 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.146321+0000", 2024-09-14T04:54:23.924 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012781500000000001 2024-09-14T04:54:23.924 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.924 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.924 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:54:23.924 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.150015+0000", 2024-09-14T04:54:23.924 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0036946359999999998 2024-09-14T04:54:23.924 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.924 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.924 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.924 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.151338+0000", 2024-09-14T04:54:23.924 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001322808 2024-09-14T04:54:23.924 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.924 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.925 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.925 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.412390+0000", 2024-09-14T04:54:23.925 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.26105160500000002 2024-09-14T04:54:23.925 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.925 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.925 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:23.925 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.412543+0000", 2024-09-14T04:54:23.925 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00015302299999999999 2024-09-14T04:54:23.925 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.925 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.925 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:23.925 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:51:43.412651+0000", 2024-09-14T04:54:23.925 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000107737 2024-09-14T04:54:23.925 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.926 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:23.926 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.926 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.926 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.926 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:35314 56.3 56:c84c2af5:::benchmark_data_smithi039_330847_object35313:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:54:23.926 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:44.506263+0000", 2024-09-14T04:54:23.926 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 99.324991827999995, 2024-09-14T04:54:23.926 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.34034366999999999, 2024-09-14T04:54:23.926 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:23.926 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:23.926 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:23.926 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:54:23.926 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:54:23.927 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 35314 2024-09-14T04:54:23.927 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.927 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:23.927 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.927 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:23.927 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506263+0000", 2024-09-14T04:54:23.927 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.927 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.927 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.927 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:23.927 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506263+0000", 2024-09-14T04:54:23.927 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.927 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.927 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.927 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:23.928 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506262+0000", 2024-09-14T04:54:23.928 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:54:23.928 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.928 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.928 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:23.928 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506311+0000", 2024-09-14T04:54:23.928 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.9308999999999998e-05 2024-09-14T04:54:23.928 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.928 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.928 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:23.928 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506323+0000", 2024-09-14T04:54:23.928 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.208e-05 2024-09-14T04:54:23.928 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.928 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.929 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:23.929 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506330+0000", 2024-09-14T04:54:23.929 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8419999999999999e-06 2024-09-14T04:54:23.929 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.929 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.929 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:23.929 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506374+0000", 2024-09-14T04:54:23.929 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.3727999999999999e-05 2024-09-14T04:54:23.929 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.929 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.929 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:23.929 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506498+0000", 2024-09-14T04:54:23.929 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000124122 2024-09-14T04:54:23.930 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.930 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.930 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:54:23.930 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506626+0000", 2024-09-14T04:54:23.930 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012813000000000001 2024-09-14T04:54:23.930 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.930 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.930 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:54:23.930 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.509601+0000", 2024-09-14T04:54:23.930 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0029756019999999999 2024-09-14T04:54:23.930 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.930 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.930 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.930 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.511200+0000", 2024-09-14T04:54:23.931 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0015989400000000001 2024-09-14T04:54:23.931 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.931 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.931 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.931 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.846516+0000", 2024-09-14T04:54:23.931 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33531591399999999 2024-09-14T04:54:23.931 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.931 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.931 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:23.931 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.846569+0000", 2024-09-14T04:54:23.931 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2462e-05 2024-09-14T04:54:23.931 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.931 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.931 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:23.931 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.846606+0000", 2024-09-14T04:54:23.932 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7617000000000003e-05 2024-09-14T04:54:23.932 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.932 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:23.932 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.932 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.932 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.932 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:35315 56.3 56:ce1faa72:::benchmark_data_smithi039_330847_object35314:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:54:23.932 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:44.506871+0000", 2024-09-14T04:54:23.932 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 99.324383853, 2024-09-14T04:54:23.932 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33989949600000002, 2024-09-14T04:54:23.932 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:23.932 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:23.933 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:23.933 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:54:23.933 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:54:23.933 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 35315 2024-09-14T04:54:23.933 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.933 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:23.933 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.933 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:23.933 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506871+0000", 2024-09-14T04:54:23.933 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.933 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.933 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.933 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:23.933 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506871+0000", 2024-09-14T04:54:23.934 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.934 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.934 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.934 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:23.934 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506870+0000", 2024-09-14T04:54:23.934 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:54:23.934 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.934 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.934 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:23.934 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506905+0000", 2024-09-14T04:54:23.934 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5542e-05 2024-09-14T04:54:23.934 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.934 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.934 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:23.935 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506935+0000", 2024-09-14T04:54:23.935 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.9787000000000001e-05 2024-09-14T04:54:23.935 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.935 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.935 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:23.935 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506942+0000", 2024-09-14T04:54:23.935 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.0119999999999999e-06 2024-09-14T04:54:23.935 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.935 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.935 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:23.935 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508033+0000", 2024-09-14T04:54:23.935 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001091103 2024-09-14T04:54:23.935 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.935 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.936 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:23.936 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508145+0000", 2024-09-14T04:54:23.936 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011233 2024-09-14T04:54:23.936 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.936 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.936 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:54:23.936 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508265+0000", 2024-09-14T04:54:23.936 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000119666 2024-09-14T04:54:23.936 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.936 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.936 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:54:23.936 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.509774+0000", 2024-09-14T04:54:23.936 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0015087150000000001 2024-09-14T04:54:23.937 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.937 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.937 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.937 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.511234+0000", 2024-09-14T04:54:23.937 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001460374 2024-09-14T04:54:23.937 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.937 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.937 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.937 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.846669+0000", 2024-09-14T04:54:23.937 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33543458199999998 2024-09-14T04:54:23.937 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.937 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.937 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:23.937 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.846735+0000", 2024-09-14T04:54:23.938 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.5816999999999997e-05 2024-09-14T04:54:23.938 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.938 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.938 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:23.938 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.846770+0000", 2024-09-14T04:54:23.938 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5562000000000003e-05 2024-09-14T04:54:23.938 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.938 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:23.938 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.938 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.938 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.938 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:35365 56.3 56:c1fc1db5:::benchmark_data_smithi039_330847_object35364:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:54:23.938 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:44.532219+0000", 2024-09-14T04:54:23.939 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 99.299035517999997, 2024-09-14T04:54:23.939 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.31467900599999998, 2024-09-14T04:54:23.939 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:23.939 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:23.939 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:23.939 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:54:23.939 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:54:23.939 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 35365 2024-09-14T04:54:23.939 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.939 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:23.939 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.939 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:23.939 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.532219+0000", 2024-09-14T04:54:23.939 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.940 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.940 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.940 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:23.940 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.532219+0000", 2024-09-14T04:54:23.940 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.940 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.940 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.940 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:23.940 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.532218+0000", 2024-09-14T04:54:23.940 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:54:23.940 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.940 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.940 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:23.941 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.532346+0000", 2024-09-14T04:54:23.941 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012836600000000001 2024-09-14T04:54:23.941 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.941 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.941 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:23.941 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.532358+0000", 2024-09-14T04:54:23.941 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1952e-05 2024-09-14T04:54:23.941 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.941 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.941 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:23.941 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.532365+0000", 2024-09-14T04:54:23.941 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8929999999999999e-06 2024-09-14T04:54:23.941 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.942 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.942 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:23.942 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.532413+0000", 2024-09-14T04:54:23.942 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8024e-05 2024-09-14T04:54:23.942 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.942 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.942 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:23.942 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.532536+0000", 2024-09-14T04:54:23.942 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012264 2024-09-14T04:54:23.942 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.942 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.942 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:54:23.942 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.532665+0000", 2024-09-14T04:54:23.942 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012901000000000001 2024-09-14T04:54:23.943 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.943 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.943 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:54:23.943 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.533926+0000", 2024-09-14T04:54:23.943 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0012615390000000001 2024-09-14T04:54:23.943 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.943 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.943 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.943 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.534408+0000", 2024-09-14T04:54:23.943 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00048154699999999998 2024-09-14T04:54:23.943 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.943 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.943 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.943 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.846824+0000", 2024-09-14T04:54:23.944 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.31241626900000002 2024-09-14T04:54:23.944 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.944 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.944 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:23.944 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.846868+0000", 2024-09-14T04:54:23.944 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.3723999999999997e-05 2024-09-14T04:54:23.944 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.944 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.944 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:23.944 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.846898+0000", 2024-09-14T04:54:23.944 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.0437e-05 2024-09-14T04:54:23.944 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.944 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:23.945 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.945 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.945 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.945 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:37084 56.3 56:c2ac110b:::benchmark_data_smithi039_330847_object37083:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:54:23.945 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:45.866380+0000", 2024-09-14T04:54:23.945 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 97.964874065000004, 2024-09-14T04:54:23.945 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.29707227600000002, 2024-09-14T04:54:23.945 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:23.945 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:23.945 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:23.945 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:54:23.945 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:54:23.945 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 37084 2024-09-14T04:54:23.945 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.945 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:23.946 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.946 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:23.946 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.866380+0000", 2024-09-14T04:54:23.946 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.946 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.946 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.946 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:23.946 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.866380+0000", 2024-09-14T04:54:23.946 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.946 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.946 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.946 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:23.946 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.866379+0000", 2024-09-14T04:54:23.946 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:54:23.947 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.947 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.947 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:23.947 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.866408+0000", 2024-09-14T04:54:23.947 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.9008000000000001e-05 2024-09-14T04:54:23.947 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.947 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.947 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:23.947 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.866421+0000", 2024-09-14T04:54:23.947 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2711000000000001e-05 2024-09-14T04:54:23.947 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.947 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.947 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:23.947 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.866428+0000", 2024-09-14T04:54:23.948 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.6240000000000002e-06 2024-09-14T04:54:23.948 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.948 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.948 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:23.948 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.866474+0000", 2024-09-14T04:54:23.948 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.5865000000000002e-05 2024-09-14T04:54:23.948 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.948 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.948 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:23.948 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.866610+0000", 2024-09-14T04:54:23.948 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013566500000000001 2024-09-14T04:54:23.948 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.948 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.948 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:54:23.949 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.866743+0000", 2024-09-14T04:54:23.949 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013336700000000001 2024-09-14T04:54:23.949 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.949 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.949 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:54:23.949 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.868266+0000", 2024-09-14T04:54:23.949 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001523092 2024-09-14T04:54:23.949 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.949 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.949 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.949 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.868744+0000", 2024-09-14T04:54:23.949 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00047758700000000002 2024-09-14T04:54:23.949 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.949 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.950 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.950 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:46.163386+0000", 2024-09-14T04:54:23.950 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.29464269500000001 2024-09-14T04:54:23.950 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.950 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.950 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:23.950 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:46.163426+0000", 2024-09-14T04:54:23.950 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9257e-05 2024-09-14T04:54:23.950 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.950 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.950 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:23.950 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:46.163453+0000", 2024-09-14T04:54:23.950 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.7030999999999999e-05 2024-09-14T04:54:23.950 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.951 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:23.951 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.951 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.951 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.951 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:37090 56.e 56:7112665d:::benchmark_data_smithi039_330847_object37089:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:54:23.951 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:45.869140+0000", 2024-09-14T04:54:23.951 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 97.962114904999993, 2024-09-14T04:54:23.951 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.29823652099999998, 2024-09-14T04:54:23.951 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:23.951 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:23.951 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:23.951 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:54:23.951 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:54:23.951 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 37090 2024-09-14T04:54:23.951 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.952 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:23.952 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.952 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:23.952 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.869140+0000", 2024-09-14T04:54:23.952 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.952 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.952 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.952 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:23.952 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.869140+0000", 2024-09-14T04:54:23.952 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.952 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.952 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.952 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:23.952 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.869138+0000", 2024-09-14T04:54:23.953 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:54:23.953 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.953 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.953 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:23.953 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.869228+0000", 2024-09-14T04:54:23.953 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.9523000000000003e-05 2024-09-14T04:54:23.953 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.953 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.953 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:23.953 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.869249+0000", 2024-09-14T04:54:23.953 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.1321000000000001e-05 2024-09-14T04:54:23.953 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.953 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.953 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:23.954 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.869256+0000", 2024-09-14T04:54:23.954 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.844e-06 2024-09-14T04:54:23.954 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.954 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.954 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:23.954 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.869321+0000", 2024-09-14T04:54:23.954 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.5061000000000006e-05 2024-09-14T04:54:23.954 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.954 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.954 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:23.954 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.869472+0000", 2024-09-14T04:54:23.954 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000150965 2024-09-14T04:54:23.954 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.954 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.955 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:54:23.955 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.869650+0000", 2024-09-14T04:54:23.955 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00017819600000000001 2024-09-14T04:54:23.955 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.955 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.955 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:54:23.955 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:45.871060+0000", 2024-09-14T04:54:23.955 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001410017 2024-09-14T04:54:23.955 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.955 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.955 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.955 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:46.163407+0000", 2024-09-14T04:54:23.955 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.292347264 2024-09-14T04:54:23.955 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.955 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.956 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.956 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:46.167251+0000", 2024-09-14T04:54:23.956 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0038432430000000001 2024-09-14T04:54:23.956 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.956 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.956 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:23.956 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:46.167340+0000", 2024-09-14T04:54:23.956 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.9050999999999998e-05 2024-09-14T04:54:23.956 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.956 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.956 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:23.956 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:46.167376+0000", 2024-09-14T04:54:23.956 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6483000000000003e-05 2024-09-14T04:54:23.956 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.957 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:23.957 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.957 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.957 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.957 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:116293 56.5 56:a6762fc1:::benchmark_data_smithi039_330847_object1769:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1486)", 2024-09-14T04:54:23.957 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:04.670255+0000", 2024-09-14T04:54:23.957 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 19.160999528000001, 2024-09-14T04:54:23.957 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.37914197500000002, 2024-09-14T04:54:23.957 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:23.957 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:23.957 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:23.957 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:54:23.957 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:54:23.957 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 116293 2024-09-14T04:54:23.958 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.958 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:23.958 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.958 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:23.958 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670255+0000", 2024-09-14T04:54:23.958 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.958 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.958 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.958 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:23.958 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670255+0000", 2024-09-14T04:54:23.958 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.958 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.958 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.958 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:23.959 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670254+0000", 2024-09-14T04:54:23.959 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:54:23.959 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.959 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.959 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:23.959 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670260+0000", 2024-09-14T04:54:23.959 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.891e-06 2024-09-14T04:54:23.959 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.959 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.959 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:23.959 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670271+0000", 2024-09-14T04:54:23.959 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1250000000000001e-05 2024-09-14T04:54:23.959 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.959 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.960 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:23.960 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670278+0000", 2024-09-14T04:54:23.960 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.4490000000000002e-06 2024-09-14T04:54:23.960 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.960 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.960 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:23.960 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.673035+0000", 2024-09-14T04:54:23.960 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0027563050000000001 2024-09-14T04:54:23.960 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.960 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.960 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:23.960 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.673150+0000", 2024-09-14T04:54:23.960 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011515600000000001 2024-09-14T04:54:23.960 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.961 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.961 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:54:23.961 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.673264+0000", 2024-09-14T04:54:23.961 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000113836 2024-09-14T04:54:23.961 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.961 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.961 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:54:23.961 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.678965+0000", 2024-09-14T04:54:23.961 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0057016519999999998 2024-09-14T04:54:23.961 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.961 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.961 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.961 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.679258+0000", 2024-09-14T04:54:23.961 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00029303199999999998 2024-09-14T04:54:23.962 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.962 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.962 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.962 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.049312+0000", 2024-09-14T04:54:23.962 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.37005369599999999 2024-09-14T04:54:23.962 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.962 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.962 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:23.962 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.049360+0000", 2024-09-14T04:54:23.962 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8072000000000003e-05 2024-09-14T04:54:23.962 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.962 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.962 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:23.963 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.049397+0000", 2024-09-14T04:54:23.963 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7007e-05 2024-09-14T04:54:23.963 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.963 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:23.963 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.963 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.963 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.963 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:116303 56.5 56:a0c06165:::benchmark_data_smithi039_330847_object1779:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1486)", 2024-09-14T04:54:23.963 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:04.679835+0000", 2024-09-14T04:54:23.963 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 19.151419216000001, 2024-09-14T04:54:23.963 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.36965448400000001, 2024-09-14T04:54:23.963 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:23.963 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:23.963 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:23.964 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:54:23.964 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:54:23.964 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 116303 2024-09-14T04:54:23.964 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.964 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:23.964 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.964 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:23.964 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.679835+0000", 2024-09-14T04:54:23.964 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.964 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.964 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.964 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:23.964 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.679835+0000", 2024-09-14T04:54:23.964 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.965 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.965 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.965 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:23.965 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.679834+0000", 2024-09-14T04:54:23.965 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:54:23.965 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.965 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.965 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:23.965 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.679844+0000", 2024-09-14T04:54:23.965 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0421999999999999e-05 2024-09-14T04:54:23.965 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.965 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.965 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:23.965 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.679860+0000", 2024-09-14T04:54:23.966 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.626e-05 2024-09-14T04:54:23.966 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.966 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.966 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:23.966 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.679868+0000", 2024-09-14T04:54:23.966 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.8779999999999998e-06 2024-09-14T04:54:23.966 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.966 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.966 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:23.966 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.680062+0000", 2024-09-14T04:54:23.966 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000193816 2024-09-14T04:54:23.966 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.966 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.966 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:23.967 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.680174+0000", 2024-09-14T04:54:23.967 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011194899999999999 2024-09-14T04:54:23.967 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.967 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.967 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:54:23.967 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.680276+0000", 2024-09-14T04:54:23.967 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000101686 2024-09-14T04:54:23.967 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.967 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.967 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.967 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.683971+0000", 2024-09-14T04:54:23.967 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0036954940000000001 2024-09-14T04:54:23.967 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.967 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.968 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:54:23.968 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.684128+0000", 2024-09-14T04:54:23.968 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00015720900000000001 2024-09-14T04:54:23.968 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.968 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.968 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.968 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.049427+0000", 2024-09-14T04:54:23.968 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.36529819200000002 2024-09-14T04:54:23.968 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.968 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.968 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:23.968 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.049462+0000", 2024-09-14T04:54:23.968 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5403e-05 2024-09-14T04:54:23.968 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.969 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.969 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:23.969 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.049490+0000", 2024-09-14T04:54:23.969 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.7855e-05 2024-09-14T04:54:23.969 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.969 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:23.969 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.969 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.969 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.969 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:116309 56.5 56:a8ec5dbc:::benchmark_data_smithi039_330847_object1785:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1486)", 2024-09-14T04:54:23.969 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:04.688374+0000", 2024-09-14T04:54:23.969 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 19.142880094999999, 2024-09-14T04:54:23.969 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.36120411800000002, 2024-09-14T04:54:23.969 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:23.970 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:23.970 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:23.970 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:54:23.970 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:54:23.970 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 116309 2024-09-14T04:54:23.970 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.970 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:23.970 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.970 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:23.970 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.688374+0000", 2024-09-14T04:54:23.970 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.970 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.970 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.970 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:23.971 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.688374+0000", 2024-09-14T04:54:23.971 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:23.971 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.971 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.971 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:23.971 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.688368+0000", 2024-09-14T04:54:23.971 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999938 2024-09-14T04:54:23.971 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.971 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.971 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:23.971 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.688379+0000", 2024-09-14T04:54:23.971 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0866000000000001e-05 2024-09-14T04:54:23.971 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.971 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.971 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:23.972 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.688393+0000", 2024-09-14T04:54:23.972 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.4394e-05 2024-09-14T04:54:23.972 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.972 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.972 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:23.972 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.688402+0000", 2024-09-14T04:54:23.972 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.4319999999999999e-06 2024-09-14T04:54:23.972 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.972 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.972 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:23.972 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.688448+0000", 2024-09-14T04:54:23.972 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.5714000000000003e-05 2024-09-14T04:54:23.972 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.972 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.973 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:23.973 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.688561+0000", 2024-09-14T04:54:23.973 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000113812 2024-09-14T04:54:23.973 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.973 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.973 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:54:23.973 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.688644+0000", 2024-09-14T04:54:23.973 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.2503999999999995e-05 2024-09-14T04:54:23.973 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.973 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.973 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.973 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.691911+0000", 2024-09-14T04:54:23.973 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0032671779999999999 2024-09-14T04:54:23.973 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.974 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.974 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:54:23.974 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.692043+0000", 2024-09-14T04:54:23.974 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000131564 2024-09-14T04:54:23.974 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.974 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.974 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:23.974 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.049516+0000", 2024-09-14T04:54:23.974 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.35747359499999998 2024-09-14T04:54:23.974 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.974 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.974 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:23.974 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.049551+0000", 2024-09-14T04:54:23.974 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.4431000000000003e-05 2024-09-14T04:54:23.975 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:23.975 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:23.975 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:23.975 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.049579+0000", 2024-09-14T04:54:23.975 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.7866999999999999e-05 2024-09-14T04:54:23.975 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.975 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:23.975 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.975 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:23.975 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:23.975 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:54:24.020 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:24.061 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:24.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:24.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:24.163 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:24.206 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:24.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:24.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:24.307 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:24.352 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:24.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:24.353 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:24.454 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:24.499 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:24.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:24.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:24.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:24 smithi129 ceph-mon[101316]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:54:24.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:24 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:54:24.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:24 smithi129 ceph-mon[101316]: osdmap e1494: 8 total, 8 up, 8 in 2024-09-14T04:54:24.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:24 smithi039 ceph-mon[106470]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:54:24.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:24 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:54:24.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:24 smithi039 ceph-mon[106470]: osdmap e1494: 8 total, 8 up, 8 in 2024-09-14T04:54:24.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:24 smithi039 ceph-mon[111235]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:54:24.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:24 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:54:24.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:24 smithi039 ceph-mon[111235]: osdmap e1494: 8 total, 8 up, 8 in 2024-09-14T04:54:24.602 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:24.637 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:24.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:24.638 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:24.739 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:24.771 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:24.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:24.772 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:24.840 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-14T04:54:24.873 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:24.917 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:24.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:24.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:25.018 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:25.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:25.050 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:25.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:25.151 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:25.183 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:25.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:25.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:25.215 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:54:25.284 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:25.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:25.325 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:25.325 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:25.426 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:25.461 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:25.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:25.462 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:25.563 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:25.593 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:25.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:25.594 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:25.695 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:25.733 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:25.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:25.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:25.767 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:25 smithi039 ceph-mon[106470]: pgmap v2807: 99 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 96 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 261/298350 objects misplaced (0.087%); 6.9 MiB/s, 110 objects/s recovering 2024-09-14T04:54:25.767 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:25 smithi039 ceph-mon[111235]: pgmap v2807: 99 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 96 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 261/298350 objects misplaced (0.087%); 6.9 MiB/s, 110 objects/s recovering 2024-09-14T04:54:25.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:25 smithi129 ceph-mon[101316]: pgmap v2807: 99 pgs: 1 active+recovering+undersized+remapped, 2 remapped+peering, 96 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 261/298350 objects misplaced (0.087%); 6.9 MiB/s, 110 objects/s recovering 2024-09-14T04:54:25.835 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:25.870 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:25.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:25.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:25.972 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:26.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:26.010 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:26.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:26.111 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:26.155 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:26.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:26.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:26.257 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:26.293 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:26.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:26.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:26.395 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:26.438 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:26.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:26.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:26.490 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:54:26.490 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:54:26.532 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:54:26.533 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:54:26.540 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:26.567 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:54:26.567 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:54:26.574 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:26.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:26.576 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:26.596 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:54:26.596 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:54:26.628 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:54:26.628 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:54:26.666 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:54:26.666 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:54:26.677 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:54:26.677 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:26.704 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:54:26.704 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:54:26.708 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:26 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3857875515' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:54:26.708 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:26 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3857875515' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:54:26.717 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:26.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:26.718 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:26.752 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:54:26.752 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:54:26.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:26 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3857875515' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:54:26.819 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:26.867 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:26.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:26.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:26.969 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:27.009 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:27.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:27.010 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:27.111 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:27.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:27.143 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:27.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:27.245 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:27.287 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:27.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:27.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:27.390 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:27.425 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:27.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:27.426 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:27.527 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:27.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:27.561 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:27.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:27.662 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:27.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:27.701 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:27.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:27.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:27 smithi129 ceph-mon[101316]: pgmap v2808: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 4349/294036 objects degraded (1.479%); 3573/294036 objects misplaced (1.215%); 6.5 MiB/s, 103 objects/s recovering 2024-09-14T04:54:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:27 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 4349/294036 objects degraded (1.479%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:54:27.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:27 smithi039 ceph-mon[106470]: pgmap v2808: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 4349/294036 objects degraded (1.479%); 3573/294036 objects misplaced (1.215%); 6.5 MiB/s, 103 objects/s recovering 2024-09-14T04:54:27.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:27 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 4349/294036 objects degraded (1.479%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:54:27.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:27 smithi039 ceph-mon[111235]: pgmap v2808: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 655 op/s; 4349/294036 objects degraded (1.479%); 3573/294036 objects misplaced (1.215%); 6.5 MiB/s, 103 objects/s recovering 2024-09-14T04:54:27.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:27 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 4349/294036 objects degraded (1.479%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:54:27.802 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:27.816 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:54:27.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:27.827 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:27.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:27.928 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:27.962 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:27.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:27.963 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:28.064 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:28.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:28.103 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:28.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:28.204 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:28.251 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:28.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:28.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:28.353 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:28.394 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:28.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:28.395 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:28.495 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:28.528 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:28.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:28.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:28.537 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.4 to 1 (10000) 2024-09-14T04:54:28.630 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:28.649 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:28 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2089912897' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:54:28.649 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:28 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2089912897' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:54:28.660 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:28.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:28.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:28.764 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:28.792 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:28.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:28.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:28.893 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:28.915 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:28.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:28.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:28.932 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:54:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:54:28] "GET /metrics HTTP/1.1" 200 36784 "" "Prometheus/2.51.0" 2024-09-14T04:54:29.016 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:29.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:28 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2089912897' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:54:29.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:29.040 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:29.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:29.141 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:29.164 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:29.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:29.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:29.266 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:29.288 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:29.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:29.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:29.357 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 5 1 2024-09-14T04:54:29.391 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:29.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:29.423 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:29.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:29.524 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:29.547 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:29.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:29.548 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:29.602 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:54:29.632 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:29 smithi039 ceph-mon[106470]: pgmap v2809: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 525 op/s; 4349/294036 objects degraded (1.479%); 3573/294036 objects misplaced (1.215%); 5.2 MiB/s, 83 objects/s recovering 2024-09-14T04:54:29.633 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:29 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2089912897' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:54:29.633 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:29 smithi039 ceph-mon[106470]: osdmap e1495: 8 total, 8 up, 8 in 2024-09-14T04:54:29.633 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:29 smithi039 ceph-mon[111235]: pgmap v2809: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 525 op/s; 4349/294036 objects degraded (1.479%); 3573/294036 objects misplaced (1.215%); 5.2 MiB/s, 83 objects/s recovering 2024-09-14T04:54:29.633 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:29 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2089912897' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:54:29.633 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:29 smithi039 ceph-mon[111235]: osdmap e1495: 8 total, 8 up, 8 in 2024-09-14T04:54:29.649 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:29.708 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:29.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:29.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:29.810 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:29.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:29.884 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:29.884 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:29.921 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:29 smithi129 ceph-mon[101316]: pgmap v2809: 99 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 96 active+clean; 6.0 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 525 op/s; 4349/294036 objects degraded (1.479%); 3573/294036 objects misplaced (1.215%); 5.2 MiB/s, 83 objects/s recovering 2024-09-14T04:54:29.922 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:29 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2089912897' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:54:29.922 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:29 smithi129 ceph-mon[101316]: osdmap e1495: 8 total, 8 up, 8 in 2024-09-14T04:54:29.980 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:54:29.986 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:30.034 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:30.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:30.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:30.136 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:30.170 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:30.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:30.171 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:30.272 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:30.314 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:30.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:30.315 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:30.416 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:30.452 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:30.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:30.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:30.554 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:30.593 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:30.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:30.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:30.618 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:30 smithi039 ceph-mon[106470]: osdmap e1496: 8 total, 8 up, 8 in 2024-09-14T04:54:30.619 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:30 smithi039 ceph-mon[111235]: osdmap e1496: 8 total, 8 up, 8 in 2024-09-14T04:54:30.695 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:30.740 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:30.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:30.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:30.842 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:30.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:30.874 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:30.874 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:30.975 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:30.996 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:30.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:30.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:31.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:30 smithi129 ceph-mon[101316]: osdmap e1496: 8 total, 8 up, 8 in 2024-09-14T04:54:31.098 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:31.134 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:31.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:31.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:31.236 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:31.267 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:31.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:31.268 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:31.369 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:31.408 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:31.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:31.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:31.510 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:31.534 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:31.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:31.535 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:31.636 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:31.667 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:31.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:31.668 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:31.769 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:31 smithi129 ceph-mon[101316]: pgmap v2812: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 4349/282414 objects degraded (1.540%); 3148/282414 objects misplaced (1.115%); 9.7 MiB/s, 154 objects/s recovering 2024-09-14T04:54:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:31 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3822263332' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:54:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:31 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:54:31.807 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:31.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:31.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:31.910 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:31.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:31.951 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:31.951 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:32.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:31 smithi039 ceph-mon[106470]: pgmap v2812: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 4349/282414 objects degraded (1.540%); 3148/282414 objects misplaced (1.115%); 9.7 MiB/s, 154 objects/s recovering 2024-09-14T04:54:32.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:31 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3822263332' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:54:32.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:31 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:54:32.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:31 smithi039 ceph-mon[111235]: pgmap v2812: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 4349/282414 objects degraded (1.540%); 3148/282414 objects misplaced (1.115%); 9.7 MiB/s, 154 objects/s recovering 2024-09-14T04:54:32.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:31 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3822263332' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:54:32.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:31 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:54:32.052 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:32.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:32.075 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:32.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:32.153 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is unset 2024-09-14T04:54:32.176 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:32.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:32.221 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:32.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:32.323 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:32.357 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:32.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:32.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:32.459 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:32.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:32.490 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:32.490 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:32.591 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:32.638 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:32.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:32.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:32.739 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:32.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:32 smithi129 ceph-mon[101316]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:54:32.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:32 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:54:32.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:32 smithi129 ceph-mon[101316]: osdmap e1497: 8 total, 8 up, 8 in 2024-09-14T04:54:32.778 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:32.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:32.779 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:32.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:32 smithi039 ceph-mon[106470]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:54:32.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:32 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:54:32.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:32 smithi039 ceph-mon[106470]: osdmap e1497: 8 total, 8 up, 8 in 2024-09-14T04:54:32.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:32 smithi039 ceph-mon[111235]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:54:32.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:32 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:54:32.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:32 smithi039 ceph-mon[111235]: osdmap e1497: 8 total, 8 up, 8 in 2024-09-14T04:54:32.880 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:32.912 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:32.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:32.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:33.014 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:33.047 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:33.047 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:33.048 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:33.148 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:33.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:33.190 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:33.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:33.291 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:33.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:33.328 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:33.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:33.429 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:33.467 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:33.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:33.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:33.473 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:54:33.570 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:33.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:33.600 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:33.600 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:33.700 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:33.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:33.739 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:33.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:33.769 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:33 smithi039 ceph-mon[106470]: pgmap v2814: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 4349/282414 objects degraded (1.540%); 3148/282414 objects misplaced (1.115%); 4.4 MiB/s, 70 objects/s recovering 2024-09-14T04:54:33.769 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:33 smithi039 ceph-mon[111235]: pgmap v2814: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 4349/282414 objects degraded (1.540%); 3148/282414 objects misplaced (1.115%); 4.4 MiB/s, 70 objects/s recovering 2024-09-14T04:54:33.840 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:33.882 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:33.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:33.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:33.985 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:34.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:33 smithi129 ceph-mon[101316]: pgmap v2814: 99 pgs: 1 active+recovering+undersized+remapped, 1 peering, 1 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.7 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 4349/282414 objects degraded (1.540%); 3148/282414 objects misplaced (1.115%); 4.4 MiB/s, 70 objects/s recovering 2024-09-14T04:54:34.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:34.023 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:34.023 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:34.124 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:34.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:34.176 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:34.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:34.277 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:34.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:34.316 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:34.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:34.417 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:34.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:34.457 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:34.458 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:34.559 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:34.591 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:34.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:34.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:34.643 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.5 to 1 (10000) 2024-09-14T04:54:34.693 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:34.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:34.733 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:34.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:34.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:34 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3591864283' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:54:34.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:34 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:54:34.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:34 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3591864283' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:54:34.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:34 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:54:34.834 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:34.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:34.868 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:34.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:34.898 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:34 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3591864283' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:54:34.898 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:34 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:54:34.949 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:54:34.969 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:35.057 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:54:35.057 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:54:35.062 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:35.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:35.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:35.165 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:35.212 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:35.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:35.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:35.314 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:35.358 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:35.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:35.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:35.459 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:35.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:35.502 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:35.502 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:35.603 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:35.639 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:35.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:35.640 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:35.741 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:35.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:35 smithi129 ceph-mon[101316]: pgmap v2815: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 985 op/s; 4349/276285 objects degraded (1.574%); 3148/276285 objects misplaced (1.139%); 4.4 MiB/s, 70 objects/s recovering 2024-09-14T04:54:35.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:35 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 4349/276285 objects degraded (1.574%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:54:35.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:35 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-14T04:54:35.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:35 smithi129 ceph-mon[101316]: osdmap e1498: 8 total, 8 up, 8 in 2024-09-14T04:54:35.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:35 smithi129 ceph-mon[101316]: osdmap e1499: 8 total, 8 up, 8 in 2024-09-14T04:54:35.780 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:35.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:35.781 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:35.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:35 smithi039 ceph-mon[106470]: pgmap v2815: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 985 op/s; 4349/276285 objects degraded (1.574%); 3148/276285 objects misplaced (1.139%); 4.4 MiB/s, 70 objects/s recovering 2024-09-14T04:54:35.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:35 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 4349/276285 objects degraded (1.574%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:54:35.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:35 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-14T04:54:35.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:35 smithi039 ceph-mon[106470]: osdmap e1498: 8 total, 8 up, 8 in 2024-09-14T04:54:35.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:35 smithi039 ceph-mon[106470]: osdmap e1499: 8 total, 8 up, 8 in 2024-09-14T04:54:35.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:35 smithi039 ceph-mon[111235]: pgmap v2815: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 985 op/s; 4349/276285 objects degraded (1.574%); 3148/276285 objects misplaced (1.139%); 4.4 MiB/s, 70 objects/s recovering 2024-09-14T04:54:35.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:35 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 4349/276285 objects degraded (1.574%), 1 pg degraded (PG_DEGRADED) 2024-09-14T04:54:35.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:35 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-14T04:54:35.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:35 smithi039 ceph-mon[111235]: osdmap e1498: 8 total, 8 up, 8 in 2024-09-14T04:54:35.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:35 smithi039 ceph-mon[111235]: osdmap e1499: 8 total, 8 up, 8 in 2024-09-14T04:54:35.882 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:35.885 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 6 1 2024-09-14T04:54:35.920 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:35.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:35.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:36.025 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:36.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:36.071 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:36.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:36.172 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:36.205 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:36.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:36.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:36.307 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:36.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:36.336 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:36.337 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:36.437 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:36.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:36.467 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:36.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:36.568 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:36.610 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:36.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:36.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:36.614 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:54:36.614 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:54:36.653 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:54:36.654 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:54:36.662 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:54:36.677 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:54:36.701 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:54:36.701 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:54:36.713 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:36.739 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:54:36.740 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:54:36.777 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:54:36.777 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:54:36.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:36.778 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:36.778 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:36.810 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:36 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1664047362' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:54:36.810 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:36 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1664047362' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:54:36.812 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:54:36.812 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:54:36.857 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:54:36.857 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:54:36.879 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:36.892 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:54:36.892 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:54:36.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:36.927 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:36.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:37.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:36 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1664047362' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:54:37.028 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:37.069 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:37.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:37.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:37.171 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:37.205 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:37.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:37.206 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:37.307 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:37.352 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:37.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:37.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:37.453 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:37.496 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:37.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:37.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:37.598 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:37.630 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:37.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:37.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:37.732 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:37.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:37.770 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:37.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:37.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:37 smithi039 ceph-mon[106470]: pgmap v2818: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 4349/272304 objects degraded (1.597%); 2997/272304 objects misplaced (1.101%); 1.6 MiB/s, 25 objects/s recovering 2024-09-14T04:54:37.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:37 smithi039 ceph-mon[106470]: osdmap e1500: 8 total, 8 up, 8 in 2024-09-14T04:54:37.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:54:37.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:37 smithi039 ceph-mon[111235]: pgmap v2818: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 4349/272304 objects degraded (1.597%); 2997/272304 objects misplaced (1.101%); 1.6 MiB/s, 25 objects/s recovering 2024-09-14T04:54:37.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:37 smithi039 ceph-mon[111235]: osdmap e1500: 8 total, 8 up, 8 in 2024-09-14T04:54:37.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:54:37.850 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-14T04:54:37.871 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:37.910 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:37.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:37.911 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:38.012 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:38.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:37 smithi129 ceph-mon[101316]: pgmap v2818: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 4349/272304 objects degraded (1.597%); 2997/272304 objects misplaced (1.101%); 1.6 MiB/s, 25 objects/s recovering 2024-09-14T04:54:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:37 smithi129 ceph-mon[101316]: osdmap e1500: 8 total, 8 up, 8 in 2024-09-14T04:54:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:54:38.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:38.054 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:38.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:38.155 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:38.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:38.189 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:38.189 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:38.199 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:54:38.290 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:38.320 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:38.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:38.321 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:38.422 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:38.464 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:38.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:38.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:38.566 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:38.603 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:38.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:38.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:38.706 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:38.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:38.754 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:38.755 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:38.856 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:38.895 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:38.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:38.896 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:38.996 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:39.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:38 smithi129 ceph-mon[101316]: pgmap v2820: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 4349/272304 objects degraded (1.597%); 2997/272304 objects misplaced (1.101%); 1.6 MiB/s, 25 objects/s recovering 2024-09-14T04:54:39.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:38 smithi039 ceph-mon[106470]: pgmap v2820: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 4349/272304 objects degraded (1.597%); 2997/272304 objects misplaced (1.101%); 1.6 MiB/s, 25 objects/s recovering 2024-09-14T04:54:39.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:54:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:54:38] "GET /metrics HTTP/1.1" 200 36768 "" "Prometheus/2.51.0" 2024-09-14T04:54:39.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:38 smithi039 ceph-mon[111235]: pgmap v2820: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 97 active+clean; 5.5 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 561 op/s; 4349/272304 objects degraded (1.597%); 2997/272304 objects misplaced (1.101%); 1.6 MiB/s, 25 objects/s recovering 2024-09-14T04:54:39.041 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:39.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:39.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:39.143 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:39.182 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:39.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:39.183 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:39.284 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:39.319 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:39.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:39.320 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:39.421 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:39.466 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:39.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:39.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:39.568 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:39.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:39.614 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:39.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:39.715 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:39.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:39.765 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:39.765 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:39.866 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:39.907 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:39.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:39.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:40.013 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:40.051 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:40.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:40.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:40.153 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:40.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:40.193 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:40.193 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:40.221 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:39 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 4350/259785 objects degraded (1.674%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:54:40.222 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:39 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 4350/259785 objects degraded (1.674%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:54:40.232 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:54:40.232 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [], 2024-09-14T04:54:40.232 INFO:teuthology.orchestra.run.smithi129.stdout: "complaint_time": 30, 2024-09-14T04:54:40.232 INFO:teuthology.orchestra.run.smithi129.stdout: "num_blocked_ops": 0 2024-09-14T04:54:40.232 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:54:40.232 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:39 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 4350/259785 objects degraded (1.674%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:54:40.295 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:40.335 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:40.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:40.336 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:40.437 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:40.479 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:40.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:40.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:40.582 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:40.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:40.618 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:40.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:40.719 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:40.750 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:40.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:40.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:40.852 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:40.891 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:40.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:40.892 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:40.993 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:41.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:41.030 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:41.031 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:41.132 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:41.165 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:41.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:41.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:41.267 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:41.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:40 smithi129 ceph-mon[101316]: pgmap v2821: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 916 op/s; 4350/259785 objects degraded (1.674%); 2852/259785 objects misplaced (1.098%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:54:41.286 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:40 smithi039 ceph-mon[106470]: pgmap v2821: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 916 op/s; 4350/259785 objects degraded (1.674%); 2852/259785 objects misplaced (1.098%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:54:41.286 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:40 smithi039 ceph-mon[111235]: pgmap v2821: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 916 op/s; 4350/259785 objects degraded (1.674%); 2852/259785 objects misplaced (1.098%); 3.1 MiB/s, 49 objects/s recovering 2024-09-14T04:54:41.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:41.301 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:41.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:41.402 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:41.441 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:41.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:41.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:41.543 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:41.573 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:41.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:41.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:41.675 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:41.722 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:41.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:41.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:41.823 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:41.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:41.863 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:41.863 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:41.964 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:41.997 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:41.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:41.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:42.099 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:42.146 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:42.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:42.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:42.147 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is unset 2024-09-14T04:54:42.225 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-14T04:54:42.243 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:42 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2413864385' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:54:42.243 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:42 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:54:42.243 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:42 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2413864385' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:54:42.243 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:42 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:54:42.247 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:42.287 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:42.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:42.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:42.389 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:42.429 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:42.430 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:42.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:42.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:42 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2413864385' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:54:42.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:42 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:54:42.531 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:42.551 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:54:42.592 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:42.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:42.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:42.694 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:42.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:42.733 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:42.733 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:42.834 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:42.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:42.872 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:42.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:42.973 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:43.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:43.012 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:43.012 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:43.113 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:43.145 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.6 to 1 (10000) 2024-09-14T04:54:43.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:43.145 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:43.146 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:43.246 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:43.279 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:43.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:43.280 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:43.381 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:43.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:43.423 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:43.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:43.480 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:43 smithi129 ceph-mon[101316]: pgmap v2822: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 4350/259785 objects degraded (1.674%); 2852/259785 objects misplaced (1.098%); 1.3 MiB/s, 20 objects/s recovering 2024-09-14T04:54:43.480 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:43 smithi129 ceph-mon[101316]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:54:43.481 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:43 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:54:43.481 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:43 smithi129 ceph-mon[101316]: osdmap e1501: 8 total, 8 up, 8 in 2024-09-14T04:54:43.481 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:43 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1430882281' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:54:43.524 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:43.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:43 smithi039 ceph-mon[106470]: pgmap v2822: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 4350/259785 objects degraded (1.674%); 2852/259785 objects misplaced (1.098%); 1.3 MiB/s, 20 objects/s recovering 2024-09-14T04:54:43.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:43 smithi039 ceph-mon[106470]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:54:43.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:43 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:54:43.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:43 smithi039 ceph-mon[106470]: osdmap e1501: 8 total, 8 up, 8 in 2024-09-14T04:54:43.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:43 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1430882281' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:54:43.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:43 smithi039 ceph-mon[111235]: pgmap v2822: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 578 op/s; 4350/259785 objects degraded (1.674%); 2852/259785 objects misplaced (1.098%); 1.3 MiB/s, 20 objects/s recovering 2024-09-14T04:54:43.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:43 smithi039 ceph-mon[111235]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:54:43.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:43 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:54:43.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:43 smithi039 ceph-mon[111235]: osdmap e1501: 8 total, 8 up, 8 in 2024-09-14T04:54:43.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:43 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1430882281' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:54:43.552 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:54:43.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:43.571 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:43.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:43.672 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:43.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:43.712 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:43.712 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:43.813 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:43.850 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:43.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:43.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:43.952 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:43.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:43.994 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:43.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:44.056 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 7 1 2024-09-14T04:54:44.096 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:44.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:44.142 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:44.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:44.243 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:44.263 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:54:44.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:44 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1430882281' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-14T04:54:44.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:44 smithi129 ceph-mon[101316]: osdmap e1502: 8 total, 8 up, 8 in 2024-09-14T04:54:44.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:44.274 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:44.275 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:44.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:44 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1430882281' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-14T04:54:44.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:44 smithi039 ceph-mon[111235]: osdmap e1502: 8 total, 8 up, 8 in 2024-09-14T04:54:44.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:44 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1430882281' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-14T04:54:44.291 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:44 smithi039 ceph-mon[106470]: osdmap e1502: 8 total, 8 up, 8 in 2024-09-14T04:54:44.375 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:44.408 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:44.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:44.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:44.510 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:44.554 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:44.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:44.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:44.656 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:44.697 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:44.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:44.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:44.798 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:44.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:44.833 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:44.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:44.934 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:44.975 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:44.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:44.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:45.028 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:54:45.028 INFO:teuthology.orchestra.run.smithi129.stdout: "size": 20, 2024-09-14T04:54:45.028 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 600, 2024-09-14T04:54:45.028 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [ 2024-09-14T04:54:45.028 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.029 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:135592 54.8 54.cb8ca998 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:54:45.029 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:44:56.328570+0000", 2024-09-14T04:54:45.029 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 588.69678111999997, 2024-09-14T04:54:45.029 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.98625274299999999, 2024-09-14T04:54:45.029 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:45.029 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "reached pg", 2024-09-14T04:54:45.029 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:45.029 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:54:45.029 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:54:45.029 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 135592 2024-09-14T04:54:45.029 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.029 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:45.030 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.030 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:45.030 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328570+0000", 2024-09-14T04:54:45.030 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.030 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.030 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.030 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:45.030 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328570+0000", 2024-09-14T04:54:45.030 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.030 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.030 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.030 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:45.030 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328569+0000", 2024-09-14T04:54:45.031 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:54:45.031 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.031 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.031 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:45.031 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328575+0000", 2024-09-14T04:54:45.031 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7700000000000004e-06 2024-09-14T04:54:45.031 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.031 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.031 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:45.031 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328588+0000", 2024-09-14T04:54:45.031 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2832e-05 2024-09-14T04:54:45.031 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.031 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.032 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:45.032 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328596+0000", 2024-09-14T04:54:45.032 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.2490000000000001e-06 2024-09-14T04:54:45.032 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.032 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.032 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:45.032 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328692+0000", 2024-09-14T04:54:45.032 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.5469000000000004e-05 2024-09-14T04:54:45.032 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.032 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.032 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:54:45.032 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328697+0000", 2024-09-14T04:54:45.033 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.5099999999999998e-06 2024-09-14T04:54:45.033 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.033 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.033 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:45.033 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.314812+0000", 2024-09-14T04:54:45.033 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.98611423499999995 2024-09-14T04:54:45.033 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.033 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.033 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:45.033 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.314823+0000", 2024-09-14T04:54:45.034 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1259e-05 2024-09-14T04:54:45.034 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.034 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:45.034 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.034 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.034 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.034 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:135594 54.e 54.e4276cde (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:54:45.034 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:44:56.328609+0000", 2024-09-14T04:54:45.034 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 588.69674200400004, 2024-09-14T04:54:45.034 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.98386206200000004, 2024-09-14T04:54:45.034 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:45.034 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "reached pg", 2024-09-14T04:54:45.035 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:45.035 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:54:45.035 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:54:45.035 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 135594 2024-09-14T04:54:45.035 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.035 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:45.035 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.035 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:45.035 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328609+0000", 2024-09-14T04:54:45.035 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.035 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.035 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.035 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:45.036 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328609+0000", 2024-09-14T04:54:45.036 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.036 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.036 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.036 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:45.036 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328609+0000", 2024-09-14T04:54:45.036 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:54:45.036 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.036 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.036 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:45.036 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328611+0000", 2024-09-14T04:54:45.036 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.5799999999999999e-06 2024-09-14T04:54:45.036 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.036 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.037 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:45.037 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328617+0000", 2024-09-14T04:54:45.037 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.6080000000000003e-06 2024-09-14T04:54:45.037 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.037 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.037 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:45.037 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328622+0000", 2024-09-14T04:54:45.037 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.685e-06 2024-09-14T04:54:45.037 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.037 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.037 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:45.037 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328658+0000", 2024-09-14T04:54:45.037 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6219e-05 2024-09-14T04:54:45.038 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.038 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.038 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:54:45.038 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.328665+0000", 2024-09-14T04:54:45.038 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.0779999999999998e-06 2024-09-14T04:54:45.038 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.038 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.038 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:45.038 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.312460+0000", 2024-09-14T04:54:45.038 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.98379467799999998 2024-09-14T04:54:45.038 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.038 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.038 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:45.038 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.312471+0000", 2024-09-14T04:54:45.039 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1620000000000001e-05 2024-09-14T04:54:45.039 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.039 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:45.039 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.039 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.039 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.039 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:135646 54.8 54.3da48de8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:54:45.039 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:44:56.446796+0000", 2024-09-14T04:54:45.039 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 588.57855489200006, 2024-09-14T04:54:45.039 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.86805825299999995, 2024-09-14T04:54:45.039 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:45.039 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "reached pg", 2024-09-14T04:54:45.039 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:45.039 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:54:45.040 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:54:45.040 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 135646 2024-09-14T04:54:45.040 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.040 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:45.040 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.040 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:45.040 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.446796+0000", 2024-09-14T04:54:45.040 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.040 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.040 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.040 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:45.040 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.446796+0000", 2024-09-14T04:54:45.040 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.040 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.041 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.041 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:45.041 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.446794+0000", 2024-09-14T04:54:45.041 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:54:45.041 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.041 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.041 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:45.041 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.446806+0000", 2024-09-14T04:54:45.041 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1218e-05 2024-09-14T04:54:45.041 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.041 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.041 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:45.041 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.446829+0000", 2024-09-14T04:54:45.042 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.2872999999999999e-05 2024-09-14T04:54:45.042 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.042 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.042 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:45.042 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.446839+0000", 2024-09-14T04:54:45.042 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0163000000000001e-05 2024-09-14T04:54:45.042 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.042 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.042 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:45.042 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.446888+0000", 2024-09-14T04:54:45.042 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8918999999999997e-05 2024-09-14T04:54:45.042 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.043 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.043 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:54:45.043 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.446896+0000", 2024-09-14T04:54:45.043 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.9440000000000005e-06 2024-09-14T04:54:45.043 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.043 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.043 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:45.043 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.314847+0000", 2024-09-14T04:54:45.043 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.86795141499999995 2024-09-14T04:54:45.043 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.043 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.043 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:45.044 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.314855+0000", 2024-09-14T04:54:45.044 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.6020000000000002e-06 2024-09-14T04:54:45.044 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.044 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:45.044 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.044 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.044 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.045 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:135649 54.8 54.c40c9e08 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:54:45.045 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:44:56.448432+0000", 2024-09-14T04:54:45.045 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 588.57691968699999, 2024-09-14T04:54:45.045 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.86648267700000003, 2024-09-14T04:54:45.045 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:45.045 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "reached pg", 2024-09-14T04:54:45.045 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:45.045 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:54:45.045 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:54:45.045 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 135649 2024-09-14T04:54:45.045 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.045 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:45.045 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.045 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:45.046 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.448432+0000", 2024-09-14T04:54:45.046 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.046 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.046 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.046 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:45.046 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.448432+0000", 2024-09-14T04:54:45.046 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.046 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.046 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.046 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:45.046 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.448431+0000", 2024-09-14T04:54:45.046 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:54:45.046 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.046 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.047 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:45.047 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.448437+0000", 2024-09-14T04:54:45.047 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.5490000000000003e-06 2024-09-14T04:54:45.047 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.047 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.047 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:45.047 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.448451+0000", 2024-09-14T04:54:45.047 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3910999999999999e-05 2024-09-14T04:54:45.048 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.048 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.048 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:45.048 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.448459+0000", 2024-09-14T04:54:45.048 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.9929999999999995e-06 2024-09-14T04:54:45.048 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.048 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.048 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:45.048 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.448500+0000", 2024-09-14T04:54:45.049 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1527999999999999e-05 2024-09-14T04:54:45.049 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.049 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.049 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:54:45.049 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.448509+0000", 2024-09-14T04:54:45.049 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.0000000000000002e-06 2024-09-14T04:54:45.049 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.049 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.049 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:45.049 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.314892+0000", 2024-09-14T04:54:45.049 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.86638208000000005 2024-09-14T04:54:45.049 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.049 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.050 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:45.050 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.314914+0000", 2024-09-14T04:54:45.050 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.2685999999999999e-05 2024-09-14T04:54:45.050 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.050 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:45.050 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.050 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.050 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.050 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:135662 54.e 54.9c02db5e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:54:45.050 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:44:56.465929+0000", 2024-09-14T04:54:45.050 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 588.55942201100004, 2024-09-14T04:54:45.050 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.84657111200000001, 2024-09-14T04:54:45.050 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:45.051 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "reached pg", 2024-09-14T04:54:45.051 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:45.051 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:54:45.051 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:54:45.051 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 135662 2024-09-14T04:54:45.051 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.051 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:45.051 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.051 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:45.051 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.465929+0000", 2024-09-14T04:54:45.051 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.051 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.051 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.052 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:45.052 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.465929+0000", 2024-09-14T04:54:45.052 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.052 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.052 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.052 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:45.052 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.465927+0000", 2024-09-14T04:54:45.052 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:54:45.052 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.052 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.052 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:45.052 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.465936+0000", 2024-09-14T04:54:45.052 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.6009999999999995e-06 2024-09-14T04:54:45.052 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.053 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.053 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:45.053 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.465954+0000", 2024-09-14T04:54:45.053 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.8406e-05 2024-09-14T04:54:45.053 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.053 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.053 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:45.053 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.465961+0000", 2024-09-14T04:54:45.053 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8600000000000004e-06 2024-09-14T04:54:45.053 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.053 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.053 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:45.053 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.466010+0000", 2024-09-14T04:54:45.054 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8504999999999997e-05 2024-09-14T04:54:45.054 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.054 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.054 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:54:45.054 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.466019+0000", 2024-09-14T04:54:45.054 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.2129999999999997e-06 2024-09-14T04:54:45.054 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.054 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.054 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:45.054 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.312493+0000", 2024-09-14T04:54:45.054 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.84647388999999995 2024-09-14T04:54:45.054 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.054 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.054 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:45.055 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.312500+0000", 2024-09-14T04:54:45.055 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.4109999999999999e-06 2024-09-14T04:54:45.055 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.055 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:45.055 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.055 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.055 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.055 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:135671 54.8 54.606ad4a8 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:54:45.055 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:44:56.481939+0000", 2024-09-14T04:54:45.055 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 588.54341200199997, 2024-09-14T04:54:45.055 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.83301307300000005, 2024-09-14T04:54:45.055 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:45.055 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "reached pg", 2024-09-14T04:54:45.055 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:45.056 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:54:45.056 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:54:45.056 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 135671 2024-09-14T04:54:45.056 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.056 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:45.056 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.056 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:45.056 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.481939+0000", 2024-09-14T04:54:45.056 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.056 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.056 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.056 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:45.056 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.481939+0000", 2024-09-14T04:54:45.057 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.057 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.057 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.057 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:45.057 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.481938+0000", 2024-09-14T04:54:45.057 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:54:45.057 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.057 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.057 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:45.057 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.481952+0000", 2024-09-14T04:54:45.057 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3995000000000001e-05 2024-09-14T04:54:45.057 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.057 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.058 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:45.058 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.481974+0000", 2024-09-14T04:54:45.058 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.2124999999999999e-05 2024-09-14T04:54:45.058 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.058 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.058 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:45.058 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.481987+0000", 2024-09-14T04:54:45.058 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3174e-05 2024-09-14T04:54:45.058 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.058 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.058 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:45.058 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.482047+0000", 2024-09-14T04:54:45.058 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9987e-05 2024-09-14T04:54:45.058 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.059 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.059 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:54:45.059 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.482056+0000", 2024-09-14T04:54:45.059 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.0699999999999996e-06 2024-09-14T04:54:45.059 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.059 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.059 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:45.059 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.314944+0000", 2024-09-14T04:54:45.059 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.83288777000000003 2024-09-14T04:54:45.059 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.059 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.059 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:45.059 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.314952+0000", 2024-09-14T04:54:45.060 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.4020000000000002e-06 2024-09-14T04:54:45.060 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.060 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:45.060 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.060 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.060 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.060 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.35436.0:135674 54.e 54.7ab1506e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1402)", 2024-09-14T04:54:45.060 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:44:56.489558+0000", 2024-09-14T04:54:45.060 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 588.53579340600004, 2024-09-14T04:54:45.060 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.82296867500000004, 2024-09-14T04:54:45.060 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:45.060 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "reached pg", 2024-09-14T04:54:45.060 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:45.061 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.35436", 2024-09-14T04:54:45.061 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2069147622", 2024-09-14T04:54:45.061 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 135674 2024-09-14T04:54:45.061 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.061 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:45.061 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.061 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:45.061 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.489558+0000", 2024-09-14T04:54:45.061 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.061 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.061 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.061 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:45.061 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.489558+0000", 2024-09-14T04:54:45.062 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.062 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.062 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.062 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:45.062 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.489556+0000", 2024-09-14T04:54:45.062 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:54:45.062 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.062 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.062 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:45.062 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.489562+0000", 2024-09-14T04:54:45.062 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9370000000000001e-06 2024-09-14T04:54:45.062 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.062 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.062 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:45.063 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.489573+0000", 2024-09-14T04:54:45.063 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0942e-05 2024-09-14T04:54:45.063 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.063 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.063 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:45.063 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.489580+0000", 2024-09-14T04:54:45.063 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.4280000000000001e-06 2024-09-14T04:54:45.063 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.063 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.063 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:45.063 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.489616+0000", 2024-09-14T04:54:45.063 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6606000000000002e-05 2024-09-14T04:54:45.063 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.063 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.064 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:54:45.064 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:56.489622+0000", 2024-09-14T04:54:45.064 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.5169999999999999e-06 2024-09-14T04:54:45.064 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.064 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.064 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:45.064 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.312519+0000", 2024-09-14T04:54:45.064 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.82289769999999995 2024-09-14T04:54:45.064 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.064 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.064 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:45.064 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:44:57.312527+0000", 2024-09-14T04:54:45.064 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.1380000000000001e-06 2024-09-14T04:54:45.064 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.065 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:45.065 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.065 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.065 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.065 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:35309 56.a 56:52365448:::benchmark_data_smithi039_330847_object35308:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:54:45.065 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:44.505360+0000", 2024-09-14T04:54:45.065 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 120.519991671, 2024-09-14T04:54:45.065 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.339575285, 2024-09-14T04:54:45.065 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:45.065 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:45.065 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:45.065 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:54:45.065 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:54:45.065 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 35309 2024-09-14T04:54:45.066 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.066 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:45.066 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.066 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:45.066 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.505360+0000", 2024-09-14T04:54:45.066 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.066 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.066 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.066 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:45.066 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.505360+0000", 2024-09-14T04:54:45.066 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.066 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.066 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.066 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:45.067 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.505358+0000", 2024-09-14T04:54:45.067 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:54:45.067 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.067 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.067 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:45.067 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.505422+0000", 2024-09-14T04:54:45.067 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.3194999999999994e-05 2024-09-14T04:54:45.067 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.067 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.067 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:45.067 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.505437+0000", 2024-09-14T04:54:45.067 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5418e-05 2024-09-14T04:54:45.067 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.067 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.068 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:45.068 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.505444+0000", 2024-09-14T04:54:45.068 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.0160000000000003e-06 2024-09-14T04:54:45.068 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.068 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.068 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:45.068 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.505717+0000", 2024-09-14T04:54:45.068 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00027316500000000002 2024-09-14T04:54:45.068 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.068 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.068 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:45.068 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.505826+0000", 2024-09-14T04:54:45.068 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010878300000000001 2024-09-14T04:54:45.069 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.069 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.069 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:54:45.069 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.505966+0000", 2024-09-14T04:54:45.069 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000139593 2024-09-14T04:54:45.069 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.069 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.069 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:54:45.069 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.507870+0000", 2024-09-14T04:54:45.069 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0019046569999999999 2024-09-14T04:54:45.069 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.069 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.069 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:45.069 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.510868+0000", 2024-09-14T04:54:45.070 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0029972900000000001 2024-09-14T04:54:45.070 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.070 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.070 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:45.070 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.844823+0000", 2024-09-14T04:54:45.070 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33395536100000001 2024-09-14T04:54:45.070 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.070 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.070 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:45.070 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.844882+0000", 2024-09-14T04:54:45.070 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.94e-05 2024-09-14T04:54:45.070 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.070 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.070 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:45.071 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.844935+0000", 2024-09-14T04:54:45.071 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2599000000000002e-05 2024-09-14T04:54:45.071 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.071 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:45.071 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.071 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.071 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.071 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:35319 56.a 56:5b9192f7:::benchmark_data_smithi039_330847_object35318:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:54:45.071 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:44.508851+0000", 2024-09-14T04:54:45.071 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 120.51650001900001, 2024-09-14T04:54:45.071 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.34003841600000001, 2024-09-14T04:54:45.071 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:45.071 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:45.071 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:45.072 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:54:45.072 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:54:45.072 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 35319 2024-09-14T04:54:45.072 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.072 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:45.072 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.072 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:45.072 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508851+0000", 2024-09-14T04:54:45.072 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.072 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.072 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.072 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:45.072 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508851+0000", 2024-09-14T04:54:45.073 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.073 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.073 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.073 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:45.073 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508850+0000", 2024-09-14T04:54:45.073 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:54:45.073 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.073 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.073 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:45.074 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508879+0000", 2024-09-14T04:54:45.074 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.9454999999999998e-05 2024-09-14T04:54:45.074 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.074 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.074 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:45.074 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508895+0000", 2024-09-14T04:54:45.074 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.6254000000000001e-05 2024-09-14T04:54:45.074 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.075 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.075 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:45.075 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508903+0000", 2024-09-14T04:54:45.075 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.4139999999999997e-06 2024-09-14T04:54:45.075 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.075 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.075 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:45.075 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508953+0000", 2024-09-14T04:54:45.075 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.0311e-05 2024-09-14T04:54:45.075 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.075 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.075 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:45.075 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.509077+0000", 2024-09-14T04:54:45.075 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000124008 2024-09-14T04:54:45.076 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.076 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.076 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:54:45.076 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.509258+0000", 2024-09-14T04:54:45.076 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000180891 2024-09-14T04:54:45.076 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.076 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.076 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:54:45.076 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.510879+0000", 2024-09-14T04:54:45.076 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001620652 2024-09-14T04:54:45.076 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.076 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.076 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:45.077 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.512666+0000", 2024-09-14T04:54:45.077 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0017868820000000001 2024-09-14T04:54:45.077 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.077 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.077 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:45.077 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.848808+0000", 2024-09-14T04:54:45.077 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.336142777 2024-09-14T04:54:45.077 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.077 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.077 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:45.077 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.848854+0000", 2024-09-14T04:54:45.077 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6133e-05 2024-09-14T04:54:45.077 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.077 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.078 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:45.078 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.848890+0000", 2024-09-14T04:54:45.078 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5185999999999998e-05 2024-09-14T04:54:45.078 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.078 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:45.078 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.078 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.078 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.078 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:35323 56.a 56:5772dd4b:::benchmark_data_smithi039_330847_object35322:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:54:45.078 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:44.510844+0000", 2024-09-14T04:54:45.078 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 120.514507349, 2024-09-14T04:54:45.078 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33940646299999999, 2024-09-14T04:54:45.078 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:45.078 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:45.079 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:45.079 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:54:45.079 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:54:45.079 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 35323 2024-09-14T04:54:45.079 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.079 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:45.079 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.079 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:45.079 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.510844+0000", 2024-09-14T04:54:45.079 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.079 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.079 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.079 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:45.079 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.510844+0000", 2024-09-14T04:54:45.080 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.080 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.080 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.080 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:45.080 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.510843+0000", 2024-09-14T04:54:45.080 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:54:45.080 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.080 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.080 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:45.080 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.510948+0000", 2024-09-14T04:54:45.080 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000104839 2024-09-14T04:54:45.080 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.080 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.080 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:45.081 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.510963+0000", 2024-09-14T04:54:45.081 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5068999999999999e-05 2024-09-14T04:54:45.081 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.081 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.081 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:45.081 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.510970+0000", 2024-09-14T04:54:45.081 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.934e-06 2024-09-14T04:54:45.081 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.081 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.081 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:45.081 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.511079+0000", 2024-09-14T04:54:45.081 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010895000000000001 2024-09-14T04:54:45.081 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.082 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.082 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:45.082 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.511314+0000", 2024-09-14T04:54:45.082 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000235307 2024-09-14T04:54:45.082 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.082 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.082 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:54:45.082 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.511434+0000", 2024-09-14T04:54:45.082 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012017 2024-09-14T04:54:45.082 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.082 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.082 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:54:45.082 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.512769+0000", 2024-09-14T04:54:45.082 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00133442 2024-09-14T04:54:45.083 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.083 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.083 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:45.083 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.513589+0000", 2024-09-14T04:54:45.083 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00082001199999999995 2024-09-14T04:54:45.083 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.083 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.083 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:45.083 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.850170+0000", 2024-09-14T04:54:45.083 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33658163499999999 2024-09-14T04:54:45.083 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.083 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.083 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:45.084 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.850217+0000", 2024-09-14T04:54:45.084 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6789999999999998e-05 2024-09-14T04:54:45.084 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.084 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.084 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:45.084 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.850250+0000", 2024-09-14T04:54:45.084 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3402999999999999e-05 2024-09-14T04:54:45.084 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.084 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:45.084 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.084 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.084 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.084 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:35335 56.a 56:5cca2bfd:::benchmark_data_smithi039_330847_object35334:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:54:45.084 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:44.515184+0000", 2024-09-14T04:54:45.085 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 120.51016708900001, 2024-09-14T04:54:45.085 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33515958000000001, 2024-09-14T04:54:45.085 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:45.085 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:45.085 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:45.085 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:54:45.085 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:54:45.085 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 35335 2024-09-14T04:54:45.085 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.085 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:45.085 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.085 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:45.085 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.515184+0000", 2024-09-14T04:54:45.085 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.086 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.086 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.086 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:45.086 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.515184+0000", 2024-09-14T04:54:45.086 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.086 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.086 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.086 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:45.086 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.515183+0000", 2024-09-14T04:54:45.086 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:54:45.086 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.086 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.086 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:45.087 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.515220+0000", 2024-09-14T04:54:45.087 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6409999999999999e-05 2024-09-14T04:54:45.087 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.087 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.087 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:45.087 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.515232+0000", 2024-09-14T04:54:45.087 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2680000000000001e-05 2024-09-14T04:54:45.087 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.087 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.087 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:45.087 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.515239+0000", 2024-09-14T04:54:45.087 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.9569999999999996e-06 2024-09-14T04:54:45.087 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.088 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.088 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:45.088 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.515349+0000", 2024-09-14T04:54:45.088 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010991 2024-09-14T04:54:45.088 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.088 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.088 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:45.088 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.515460+0000", 2024-09-14T04:54:45.088 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000110398 2024-09-14T04:54:45.088 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.088 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.088 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:54:45.089 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.515600+0000", 2024-09-14T04:54:45.089 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013996700000000001 2024-09-14T04:54:45.089 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.089 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.089 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:54:45.089 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.517100+0000", 2024-09-14T04:54:45.089 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00150004 2024-09-14T04:54:45.089 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.089 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.089 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:45.089 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.520249+0000", 2024-09-14T04:54:45.089 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.003149584 2024-09-14T04:54:45.089 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.090 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.090 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:45.090 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.850283+0000", 2024-09-14T04:54:45.090 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33003364499999999 2024-09-14T04:54:45.090 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.090 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.090 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:45.090 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.850316+0000", 2024-09-14T04:54:45.090 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3037000000000002e-05 2024-09-14T04:54:45.090 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.090 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.090 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:45.090 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.850344+0000", 2024-09-14T04:54:45.090 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.7773e-05 2024-09-14T04:54:45.091 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.091 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:45.091 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.091 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.091 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.091 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:35341 56.a 56:5b382216:::benchmark_data_smithi039_330847_object35340:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:54:45.091 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:44.518706+0000", 2024-09-14T04:54:45.091 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 120.506645728, 2024-09-14T04:54:45.091 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33338658300000001, 2024-09-14T04:54:45.091 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:45.091 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:45.091 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:45.091 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:54:45.091 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:54:45.092 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 35341 2024-09-14T04:54:45.092 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.092 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:45.092 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.092 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:45.092 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.518706+0000", 2024-09-14T04:54:45.092 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.092 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.092 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.092 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:45.092 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.518706+0000", 2024-09-14T04:54:45.092 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.092 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.093 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.093 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:45.093 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.518704+0000", 2024-09-14T04:54:45.093 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:54:45.093 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.093 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.093 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:45.093 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.518744+0000", 2024-09-14T04:54:45.093 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9659000000000001e-05 2024-09-14T04:54:45.093 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.093 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.093 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:45.093 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.518757+0000", 2024-09-14T04:54:45.093 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.358e-05 2024-09-14T04:54:45.094 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.094 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.094 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:45.094 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.518765+0000", 2024-09-14T04:54:45.094 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.2080000000000003e-06 2024-09-14T04:54:45.094 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.094 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.094 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:45.094 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.518837+0000", 2024-09-14T04:54:45.094 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.2052999999999996e-05 2024-09-14T04:54:45.094 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.094 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.094 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:45.094 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.518956+0000", 2024-09-14T04:54:45.095 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000119437 2024-09-14T04:54:45.095 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.095 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.095 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:54:45.095 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.519095+0000", 2024-09-14T04:54:45.095 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013848500000000001 2024-09-14T04:54:45.095 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.095 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.095 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:54:45.095 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.520369+0000", 2024-09-14T04:54:45.095 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001274154 2024-09-14T04:54:45.095 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.095 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.095 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:45.096 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.521247+0000", 2024-09-14T04:54:45.096 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00087840900000000003 2024-09-14T04:54:45.096 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.096 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.096 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:45.096 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.852014+0000", 2024-09-14T04:54:45.096 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.330766387 2024-09-14T04:54:45.096 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.096 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.096 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:45.096 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.852055+0000", 2024-09-14T04:54:45.096 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1093000000000001e-05 2024-09-14T04:54:45.096 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.096 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.097 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:45.097 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.852092+0000", 2024-09-14T04:54:45.097 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7452e-05 2024-09-14T04:54:45.097 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.097 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:45.097 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.097 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.097 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.097 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:116251 56.b 56:d0b5012a:::benchmark_data_smithi039_330847_object1727:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1486)", 2024-09-14T04:54:45.097 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:04.547766+0000", 2024-09-14T04:54:45.097 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 40.477585105999999, 2024-09-14T04:54:45.097 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.488460691, 2024-09-14T04:54:45.097 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:45.098 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:45.098 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:45.098 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:54:45.098 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:54:45.098 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 116251 2024-09-14T04:54:45.098 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.098 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:45.098 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.098 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:45.098 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.547766+0000", 2024-09-14T04:54:45.098 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.098 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.098 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.099 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:45.099 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.547766+0000", 2024-09-14T04:54:45.099 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.099 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.099 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.099 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:45.099 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.547765+0000", 2024-09-14T04:54:45.099 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:54:45.099 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.099 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.099 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:45.099 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.547771+0000", 2024-09-14T04:54:45.099 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.7209999999999999e-06 2024-09-14T04:54:45.099 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.100 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.100 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:45.100 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.547779+0000", 2024-09-14T04:54:45.100 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.7020000000000007e-06 2024-09-14T04:54:45.100 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.100 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.100 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:45.100 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.547785+0000", 2024-09-14T04:54:45.100 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9039999999999997e-06 2024-09-14T04:54:45.100 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.100 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.100 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:45.100 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.551793+0000", 2024-09-14T04:54:45.101 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0040079240000000004 2024-09-14T04:54:45.101 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.101 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.101 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:45.101 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.551900+0000", 2024-09-14T04:54:45.101 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000107318 2024-09-14T04:54:45.101 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.101 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.101 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,5", 2024-09-14T04:54:45.101 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.552013+0000", 2024-09-14T04:54:45.101 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000112469 2024-09-14T04:54:45.101 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.101 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.101 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:54:45.102 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.555580+0000", 2024-09-14T04:54:45.102 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0035672360000000001 2024-09-14T04:54:45.102 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.102 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.102 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:45.102 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.558497+0000", 2024-09-14T04:54:45.102 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0029165189999999998 2024-09-14T04:54:45.102 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.102 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.102 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:45.102 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036142+0000", 2024-09-14T04:54:45.102 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.47764563300000001 2024-09-14T04:54:45.102 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.102 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.103 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:45.103 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036189+0000", 2024-09-14T04:54:45.103 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7036999999999997e-05 2024-09-14T04:54:45.103 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.103 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.103 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:45.103 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036227+0000", 2024-09-14T04:54:45.103 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7576000000000001e-05 2024-09-14T04:54:45.103 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.103 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:45.103 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.103 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.103 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.103 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:116268 56.b 56:de88060a:::benchmark_data_smithi039_330847_object1744:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1486)", 2024-09-14T04:54:45.104 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:04.637502+0000", 2024-09-14T04:54:45.104 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 40.387849525, 2024-09-14T04:54:45.104 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.39881927299999997, 2024-09-14T04:54:45.104 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:45.104 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:45.104 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:45.104 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:54:45.104 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:54:45.104 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 116268 2024-09-14T04:54:45.104 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.104 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:45.104 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.104 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:45.105 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.637502+0000", 2024-09-14T04:54:45.105 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.105 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.105 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.105 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:45.105 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.637502+0000", 2024-09-14T04:54:45.105 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.105 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.105 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.105 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:45.105 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.637501+0000", 2024-09-14T04:54:45.105 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:54:45.105 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.105 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.106 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:45.106 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.637505+0000", 2024-09-14T04:54:45.106 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7349999999999998e-06 2024-09-14T04:54:45.106 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.106 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.106 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:45.106 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.637515+0000", 2024-09-14T04:54:45.106 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0519e-05 2024-09-14T04:54:45.106 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.106 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.106 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:45.106 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.637522+0000", 2024-09-14T04:54:45.106 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.6429999999999998e-06 2024-09-14T04:54:45.106 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.107 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.107 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:45.107 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.637562+0000", 2024-09-14T04:54:45.107 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0305999999999997e-05 2024-09-14T04:54:45.107 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.107 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.107 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:45.107 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.637689+0000", 2024-09-14T04:54:45.107 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012666000000000001 2024-09-14T04:54:45.107 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.107 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.107 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,5", 2024-09-14T04:54:45.107 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.637820+0000", 2024-09-14T04:54:45.107 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000130877 2024-09-14T04:54:45.108 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.108 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.108 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:54:45.108 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.646403+0000", 2024-09-14T04:54:45.108 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0085835389999999994 2024-09-14T04:54:45.108 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.108 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.108 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:45.108 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.649755+0000", 2024-09-14T04:54:45.108 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0033511610000000001 2024-09-14T04:54:45.108 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.108 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.108 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:45.109 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036263+0000", 2024-09-14T04:54:45.109 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.38650827100000001 2024-09-14T04:54:45.109 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.109 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.109 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:45.109 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036294+0000", 2024-09-14T04:54:45.109 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.1164000000000002e-05 2024-09-14T04:54:45.109 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.109 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.109 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:45.109 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036321+0000", 2024-09-14T04:54:45.109 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.7030999999999999e-05 2024-09-14T04:54:45.109 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.109 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:45.110 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.110 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.110 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.110 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:116284 56.b 56:d7432e9a:::benchmark_data_smithi039_330847_object1760:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1486)", 2024-09-14T04:54:45.110 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:04.658571+0000", 2024-09-14T04:54:45.110 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 40.366780321, 2024-09-14T04:54:45.110 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.37787099099999999, 2024-09-14T04:54:45.110 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:45.110 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:45.110 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:45.110 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:54:45.110 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:54:45.110 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 116284 2024-09-14T04:54:45.110 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.110 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:45.111 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.111 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:45.111 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658571+0000", 2024-09-14T04:54:45.111 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.111 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.111 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.111 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:45.111 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658571+0000", 2024-09-14T04:54:45.111 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.111 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.111 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.111 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:45.111 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658569+0000", 2024-09-14T04:54:45.112 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:54:45.112 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.112 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.112 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:45.112 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658577+0000", 2024-09-14T04:54:45.112 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.7889999999999995e-06 2024-09-14T04:54:45.112 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.112 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.112 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:45.112 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658592+0000", 2024-09-14T04:54:45.112 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5287000000000001e-05 2024-09-14T04:54:45.112 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.112 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.113 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:45.113 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658600+0000", 2024-09-14T04:54:45.113 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.6240000000000002e-06 2024-09-14T04:54:45.113 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.113 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.113 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:45.113 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658647+0000", 2024-09-14T04:54:45.113 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6965000000000002e-05 2024-09-14T04:54:45.113 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.113 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.113 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:45.113 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658748+0000", 2024-09-14T04:54:45.113 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000101005 2024-09-14T04:54:45.114 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.114 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.114 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,5", 2024-09-14T04:54:45.114 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658842+0000", 2024-09-14T04:54:45.114 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.4204999999999997e-05 2024-09-14T04:54:45.114 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.114 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.114 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:54:45.114 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.662555+0000", 2024-09-14T04:54:45.114 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0037132350000000001 2024-09-14T04:54:45.114 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.114 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.114 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:45.114 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.663869+0000", 2024-09-14T04:54:45.115 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001313952 2024-09-14T04:54:45.115 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.115 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.115 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:45.115 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036369+0000", 2024-09-14T04:54:45.115 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.37249975600000002 2024-09-14T04:54:45.115 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.115 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.115 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:45.115 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036410+0000", 2024-09-14T04:54:45.115 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0692000000000003e-05 2024-09-14T04:54:45.115 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.116 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.116 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:45.116 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036442+0000", 2024-09-14T04:54:45.116 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.2181000000000002e-05 2024-09-14T04:54:45.116 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.116 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:45.117 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:54:45.117 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:45.118 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.118 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.118 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.118 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:116292 56.4 56:2d450bf4:::benchmark_data_smithi039_330847_object1768:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1486)", 2024-09-14T04:54:45.118 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:04.670078+0000", 2024-09-14T04:54:45.118 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 40.355273193000002, 2024-09-14T04:54:45.118 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.374447525, 2024-09-14T04:54:45.118 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:45.118 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:45.118 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:45.118 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:54:45.118 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:54:45.118 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 116292 2024-09-14T04:54:45.118 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.119 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:45.119 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.119 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:45.119 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670078+0000", 2024-09-14T04:54:45.119 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.119 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.119 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.119 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:45.119 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670078+0000", 2024-09-14T04:54:45.119 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.119 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.119 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.119 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:45.119 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670077+0000", 2024-09-14T04:54:45.120 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:54:45.120 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.120 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.120 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:45.120 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670082+0000", 2024-09-14T04:54:45.120 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8439999999999997e-06 2024-09-14T04:54:45.120 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.120 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.120 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:45.120 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670090+0000", 2024-09-14T04:54:45.120 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.6989999999999992e-06 2024-09-14T04:54:45.120 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.120 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.121 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:45.121 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670097+0000", 2024-09-14T04:54:45.121 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.1589999999999998e-06 2024-09-14T04:54:45.121 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.121 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.121 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:45.121 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670207+0000", 2024-09-14T04:54:45.121 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011047100000000001 2024-09-14T04:54:45.121 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.121 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.121 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:45.121 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670315+0000", 2024-09-14T04:54:45.121 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010752299999999999 2024-09-14T04:54:45.121 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.122 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.122 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:54:45.122 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036104+0000", 2024-09-14T04:54:45.122 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.36578902299999999 2024-09-14T04:54:45.122 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.122 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.122 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:54:45.122 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.040066+0000", 2024-09-14T04:54:45.122 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0039619390000000003 2024-09-14T04:54:45.122 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.122 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.122 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:45.122 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.044355+0000", 2024-09-14T04:54:45.122 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0042896779999999999 2024-09-14T04:54:45.123 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.123 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.123 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:45.123 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.044433+0000", 2024-09-14T04:54:45.123 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.7905999999999996e-05 2024-09-14T04:54:45.123 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.123 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.123 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:45.123 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.044483+0000", 2024-09-14T04:54:45.123 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.0207000000000002e-05 2024-09-14T04:54:45.123 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.123 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.123 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:45.123 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.044526+0000", 2024-09-14T04:54:45.124 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.2184999999999997e-05 2024-09-14T04:54:45.124 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.124 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:45.124 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.124 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.124 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.124 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.36012.0:116283 56.6 e1486/1476 56:68220df0:::benchmark_data_smithi039_330847_object1759:head v 1486'7117, mlcod=0'0)", 2024-09-14T04:54:45.124 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:04.670105+0000", 2024-09-14T04:54:45.124 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 40.355245901000004, 2024-09-14T04:54:45.124 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.37733796400000003, 2024-09-14T04:54:45.124 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:45.124 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:45.124 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:45.125 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.125 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:45.125 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670105+0000", 2024-09-14T04:54:45.125 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.125 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.125 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.125 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:45.125 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670105+0000", 2024-09-14T04:54:45.125 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.125 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.125 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.125 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:45.125 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670105+0000", 2024-09-14T04:54:45.126 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:54:45.126 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.126 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.126 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:45.126 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670111+0000", 2024-09-14T04:54:45.126 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.02e-06 2024-09-14T04:54:45.126 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.126 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.126 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:45.126 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670118+0000", 2024-09-14T04:54:45.126 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.661e-06 2024-09-14T04:54:45.126 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.126 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.126 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:45.126 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670124+0000", 2024-09-14T04:54:45.127 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.3539999999999999e-06 2024-09-14T04:54:45.127 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.127 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.127 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:45.127 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670163+0000", 2024-09-14T04:54:45.127 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9483999999999997e-05 2024-09-14T04:54:45.127 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.127 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.127 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:45.127 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670186+0000", 2024-09-14T04:54:45.127 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.2918999999999999e-05 2024-09-14T04:54:45.127 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.127 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.128 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:45.128 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.047411+0000", 2024-09-14T04:54:45.128 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.37722491000000002 2024-09-14T04:54:45.128 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.128 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.128 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:45.128 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.047443+0000", 2024-09-14T04:54:45.128 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.2203000000000003e-05 2024-09-14T04:54:45.128 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.128 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:45.128 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.128 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.128 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.128 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:116294 56.4 56:2b7b2e82:::benchmark_data_smithi039_330847_object1770:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1486)", 2024-09-14T04:54:45.129 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:04.670327+0000", 2024-09-14T04:54:45.129 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 40.355023877000001, 2024-09-14T04:54:45.129 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.377658365, 2024-09-14T04:54:45.129 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:45.129 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:45.129 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:54:45.129 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:54:45.129 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:54:45.129 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 116294 2024-09-14T04:54:45.129 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.129 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:45.129 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.129 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:45.130 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670327+0000", 2024-09-14T04:54:45.130 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.130 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.130 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.130 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:45.130 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670327+0000", 2024-09-14T04:54:45.130 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.130 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.130 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.130 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:45.130 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670327+0000", 2024-09-14T04:54:45.130 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:54:45.130 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.131 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.131 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:45.131 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670331+0000", 2024-09-14T04:54:45.131 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.048e-06 2024-09-14T04:54:45.131 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.131 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.131 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:45.131 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036115+0000", 2024-09-14T04:54:45.131 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.36578455700000001 2024-09-14T04:54:45.131 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.131 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.131 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:45.131 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036124+0000", 2024-09-14T04:54:45.131 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.9420000000000001e-06 2024-09-14T04:54:45.131 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.132 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.132 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:45.132 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.040091+0000", 2024-09-14T04:54:45.132 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0039670720000000003 2024-09-14T04:54:45.132 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.132 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.132 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:45.132 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.040277+0000", 2024-09-14T04:54:45.132 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000185954 2024-09-14T04:54:45.132 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.132 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.132 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:54:45.132 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.040378+0000", 2024-09-14T04:54:45.132 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010124300000000001 2024-09-14T04:54:45.133 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.133 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.133 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:45.133 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.044582+0000", 2024-09-14T04:54:45.133 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0042034259999999997 2024-09-14T04:54:45.133 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.133 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.133 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:54:45.133 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.044612+0000", 2024-09-14T04:54:45.137 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.0540999999999998e-05 2024-09-14T04:54:45.137 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.137 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.137 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:54:45.137 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.047897+0000", 2024-09-14T04:54:45.137 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.003284281 2024-09-14T04:54:45.137 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.137 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.137 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:45.137 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.047948+0000", 2024-09-14T04:54:45.137 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.1749000000000002e-05 2024-09-14T04:54:45.137 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.138 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.138 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:45.138 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.047986+0000", 2024-09-14T04:54:45.138 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7357999999999997e-05 2024-09-14T04:54:45.138 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.138 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:45.138 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.138 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.138 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.138 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop_reply(client.36012.0:116251 56.b e1486/1470 ondisk, result = 0)", 2024-09-14T04:54:45.138 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:04.672410+0000", 2024-09-14T04:54:45.138 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 40.352940922000002, 2024-09-14T04:54:45.138 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.36382398599999999, 2024-09-14T04:54:45.139 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:45.139 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "started", 2024-09-14T04:54:45.139 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:45.139 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.139 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:45.139 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.672410+0000", 2024-09-14T04:54:45.139 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.139 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.139 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.139 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:45.139 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.672410+0000", 2024-09-14T04:54:45.139 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.139 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.140 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.140 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:45.140 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.672409+0000", 2024-09-14T04:54:45.140 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:54:45.140 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.140 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.140 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:45.140 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.672416+0000", 2024-09-14T04:54:45.140 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.0160000000000003e-06 2024-09-14T04:54:45.140 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.140 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.140 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:45.140 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036080+0000", 2024-09-14T04:54:45.141 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.363663499 2024-09-14T04:54:45.141 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.141 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.141 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:45.141 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036088+0000", 2024-09-14T04:54:45.141 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.8250000000000005e-06 2024-09-14T04:54:45.141 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.141 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.141 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:45.141 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036126+0000", 2024-09-14T04:54:45.141 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8379e-05 2024-09-14T04:54:45.141 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.141 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.142 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:45.142 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036136+0000", 2024-09-14T04:54:45.142 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.6330000000000008e-06 2024-09-14T04:54:45.142 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.142 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.142 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:45.142 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036234+0000", 2024-09-14T04:54:45.142 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.8673999999999997e-05 2024-09-14T04:54:45.142 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.142 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:45.142 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.142 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.142 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.142 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.36012.0:116293 56.5 e1486/1470 56:a6762fc1:::benchmark_data_smithi039_330847_object1769:head v 1486'7278, mlcod=1486'7276)", 2024-09-14T04:54:45.142 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:04.673699+0000", 2024-09-14T04:54:45.143 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 40.351651859, 2024-09-14T04:54:45.143 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.37533592999999998, 2024-09-14T04:54:45.143 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:54:45.143 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:54:45.143 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:54:45.143 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.143 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:54:45.143 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.673699+0000", 2024-09-14T04:54:45.143 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.143 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.143 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.143 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:54:45.143 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.673699+0000", 2024-09-14T04:54:45.143 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:54:45.144 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.144 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.144 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:54:45.144 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.673699+0000", 2024-09-14T04:54:45.144 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:54:45.144 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.144 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.144 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:54:45.144 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.673707+0000", 2024-09-14T04:54:45.144 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.5599999999999994e-06 2024-09-14T04:54:45.144 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.144 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.144 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:54:45.145 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.035992+0000", 2024-09-14T04:54:45.145 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.36228451699999997 2024-09-14T04:54:45.145 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.145 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.145 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:54:45.145 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036003+0000", 2024-09-14T04:54:45.145 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0883999999999999e-05 2024-09-14T04:54:45.145 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.145 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.145 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:54:45.145 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036051+0000", 2024-09-14T04:54:45.145 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8485000000000001e-05 2024-09-14T04:54:45.145 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.145 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.145 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:54:45.146 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036084+0000", 2024-09-14T04:54:45.146 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3222999999999997e-05 2024-09-14T04:54:45.146 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.146 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.146 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:54:45.146 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.049003+0000", 2024-09-14T04:54:45.146 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.012919058000000001 2024-09-14T04:54:45.146 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:54:45.146 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:54:45.146 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:54:45.146 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.049035+0000", 2024-09-14T04:54:45.146 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.2027999999999999e-05 2024-09-14T04:54:45.146 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.147 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:45.147 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.147 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:54:45.147 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:54:45.147 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:54:45.147 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:54:45.190 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:45.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:45.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:45.292 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:45.328 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:45.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:45.329 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:45.430 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:45.466 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:45.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:45.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:45.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:45 smithi129 ceph-mon[101316]: pgmap v2825: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 949 op/s; 4350/251796 objects degraded (1.728%); 2852/251796 objects misplaced (1.133%); 1.3 MiB/s, 20 objects/s recovering 2024-09-14T04:54:45.539 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:45 smithi039 ceph-mon[106470]: pgmap v2825: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 949 op/s; 4350/251796 objects degraded (1.728%); 2852/251796 objects misplaced (1.133%); 1.3 MiB/s, 20 objects/s recovering 2024-09-14T04:54:45.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:45 smithi039 ceph-mon[111235]: pgmap v2825: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.1 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 949 op/s; 4350/251796 objects degraded (1.728%); 2852/251796 objects misplaced (1.133%); 1.3 MiB/s, 20 objects/s recovering 2024-09-14T04:54:45.569 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:45.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:45.610 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:45.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:45.712 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:45.767 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:45.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:45.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:45.870 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:45.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:45.909 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:45.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:46.010 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:46.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:46.040 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:46.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:46.079 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-14T04:54:46.142 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:46.172 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:46.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:46.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:46.231 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:46 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 4350/251796 objects degraded (1.728%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:54:46.231 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:46 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 4350/251796 objects degraded (1.728%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:54:46.275 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:46.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:46.305 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:46.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:46.406 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:46.439 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:46.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:46.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:46.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:46 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 4350/251796 objects degraded (1.728%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:54:46.540 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:46.575 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:46.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:46.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:46.677 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:46.710 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:46.711 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:46.711 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:46.812 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:46.845 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:46.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:46.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:46.946 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:46.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:46.985 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:46.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:47.086 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:47.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:47.122 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:47.122 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:47.223 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:47.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:47.261 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:47.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:47.362 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:47.395 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:47.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:47.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:47.497 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:47.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:47 smithi129 ceph-mon[101316]: pgmap v2826: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 4350/247062 objects degraded (1.761%); 2708/247062 objects misplaced (1.096%); 2.3 MiB/s, 36 objects/s recovering 2024-09-14T04:54:47.540 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:47.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:47.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:47 smithi039 ceph-mon[106470]: pgmap v2826: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 4350/247062 objects degraded (1.761%); 2708/247062 objects misplaced (1.096%); 2.3 MiB/s, 36 objects/s recovering 2024-09-14T04:54:47.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:47 smithi039 ceph-mon[111235]: pgmap v2826: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 4350/247062 objects degraded (1.761%); 2708/247062 objects misplaced (1.096%); 2.3 MiB/s, 36 objects/s recovering 2024-09-14T04:54:47.541 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:47.642 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:47.686 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:47.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:47.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:47.788 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:47.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:47.832 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:47.832 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:47.933 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:47.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:47.961 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:47.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:48.062 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:48.106 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:48.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:48.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:48.207 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:48.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:48.246 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:48.247 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:48.347 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:48.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:48.372 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:48.372 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:48.473 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:48.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:48.496 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:48.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:48.597 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:48.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:48.621 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:48.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:48.723 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:48.770 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:48.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:48.771 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:48.872 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:48.908 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:48.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:48.909 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:49.010 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:49.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:54:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:54:48] "GET /metrics HTTP/1.1" 200 36768 "" "Prometheus/2.51.0" 2024-09-14T04:54:49.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:49.046 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:49.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:49.147 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:49.178 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:49.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:49.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:49.280 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:49.314 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:49.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:49.315 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:49.415 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:49.446 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:49 smithi039 ceph-mon[106470]: pgmap v2827: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 4350/247062 objects degraded (1.761%); 2708/247062 objects misplaced (1.096%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:54:49.447 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:49 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1181309684' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:54:49.447 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:49 smithi039 ceph-mon[111235]: pgmap v2827: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 4350/247062 objects degraded (1.761%); 2708/247062 objects misplaced (1.096%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:54:49.447 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:49 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1181309684' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:54:49.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:49.448 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:49.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:49.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:49 smithi129 ceph-mon[101316]: pgmap v2827: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 4350/247062 objects degraded (1.761%); 2708/247062 objects misplaced (1.096%); 1.1 MiB/s, 17 objects/s recovering 2024-09-14T04:54:49.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:49 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1181309684' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:54:49.549 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:49.577 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:54:49.577 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:54:49.577 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:54:49.577 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:54:49.577 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:54:49.577 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:54:49.577 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:54:49.577 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:54:49.578 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:54:49.578 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:54:49.578 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:54:49.578 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:54:49.578 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:54:49.578 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:54:49.578 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:54:49.578 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:54:49.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:49.590 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:49.590 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:49.691 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:49.726 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:49.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:49.728 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:49.828 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:49.864 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:49.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:49.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:49.884 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:54:49.965 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:50.000 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:50.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:50.001 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:50.102 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:50.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:50.140 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:50.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:50.241 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:50.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:50.289 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:50.289 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:50.390 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:50.421 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:50.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:50.422 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:50.523 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:50.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:50.566 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:50.566 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:50.667 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:50.705 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:50.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:50.706 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:50.807 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:50.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:50.853 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:50.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:50.954 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:51.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:51.000 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:51.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:51.101 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:51.140 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:51.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:51.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:51.227 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.7 to 1 (10000) 2024-09-14T04:54:51.370 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:51.407 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:51.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:51.408 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:51.508 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:51 smithi129 ceph-mon[101316]: pgmap v2828: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 4350/234993 objects degraded (1.851%); 2567/234993 objects misplaced (1.092%); 2.2 MiB/s, 35 objects/s recovering 2024-09-14T04:54:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:51 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3330806561' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:54:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:51 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 4350/234993 objects degraded (1.851%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:54:51.538 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:51.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:51.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:51.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:51 smithi039 ceph-mon[106470]: pgmap v2828: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 4350/234993 objects degraded (1.851%); 2567/234993 objects misplaced (1.092%); 2.2 MiB/s, 35 objects/s recovering 2024-09-14T04:54:51.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:51 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3330806561' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:54:51.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:51 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 4350/234993 objects degraded (1.851%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:54:51.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:51 smithi039 ceph-mon[111235]: pgmap v2828: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 4350/234993 objects degraded (1.851%); 2567/234993 objects misplaced (1.092%); 2.2 MiB/s, 35 objects/s recovering 2024-09-14T04:54:51.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:51 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3330806561' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:54:51.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:51 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 4350/234993 objects degraded (1.851%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:54:51.640 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:51.662 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:51.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:51.663 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:51.764 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:51.789 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:51.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:51.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:51.891 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:51.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:51.917 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:51.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:52.018 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:52.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:52.040 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:52.041 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:52.141 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:52.163 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:52.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:52.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:52.264 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:52.287 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:52.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:52.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:52.289 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:54:52.289 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:52 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3330806561' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-14T04:54:52.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:52 smithi039 ceph-mon[111235]: osdmap e1503: 8 total, 8 up, 8 in 2024-09-14T04:54:52.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:54:52.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:52 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3330806561' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-14T04:54:52.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:52 smithi039 ceph-mon[106470]: osdmap e1503: 8 total, 8 up, 8 in 2024-09-14T04:54:52.389 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:52.412 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:52.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:52.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:52.477 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:54:52.514 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:52.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:54:52.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:52 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3330806561' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-14T04:54:52.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:52 smithi129 ceph-mon[101316]: osdmap e1503: 8 total, 8 up, 8 in 2024-09-14T04:54:52.537 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:52.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:52.537 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:52.638 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:52.661 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:52.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:52.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:52.763 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:52.785 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:52.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:52.786 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:52.887 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:52.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:52.910 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:52.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:52.973 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:54:53.012 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:53.033 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:53.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:53.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:53.135 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:53.156 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:53.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:53.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:53.178 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-14T04:54:53.178 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:54:53.258 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:53.282 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:53.282 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:53 smithi039 ceph-mon[106470]: pgmap v2830: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 948 op/s; 4350/234993 objects degraded (1.851%); 2567/234993 objects misplaced (1.092%); 2.0 MiB/s, 32 objects/s recovering 2024-09-14T04:54:53.282 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:53 smithi039 ceph-mon[106470]: osdmap e1504: 8 total, 8 up, 8 in 2024-09-14T04:54:53.282 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:53 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:54:53.282 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:53 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:54:53.282 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:53 smithi039 ceph-mon[111235]: pgmap v2830: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 948 op/s; 4350/234993 objects degraded (1.851%); 2567/234993 objects misplaced (1.092%); 2.0 MiB/s, 32 objects/s recovering 2024-09-14T04:54:53.283 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:53 smithi039 ceph-mon[111235]: osdmap e1504: 8 total, 8 up, 8 in 2024-09-14T04:54:53.283 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:53 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:54:53.283 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:53 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:54:53.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:53.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:53.384 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:53.405 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:53.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:53.406 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:53.507 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:53.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:53 smithi129 ceph-mon[101316]: pgmap v2830: 99 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 96 active+clean; 4.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 948 op/s; 4350/234993 objects degraded (1.851%); 2567/234993 objects misplaced (1.092%); 2.0 MiB/s, 32 objects/s recovering 2024-09-14T04:54:53.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:53 smithi129 ceph-mon[101316]: osdmap e1504: 8 total, 8 up, 8 in 2024-09-14T04:54:53.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:53 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:54:53.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:53 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:54:53.531 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:53.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:53.532 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:53.633 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:53.655 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:53.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:53.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:53.757 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:53.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:53.791 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:53.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:53.892 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:53.926 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:53.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:53.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:54.027 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:54.061 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:54.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:54.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:54.163 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:54.206 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:54.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:54.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:54.308 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:54.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:54.349 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:54.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:54.450 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:54.487 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:54.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:54.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:54.518 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:54 smithi039 ceph-mon[106470]: osdmap e1505: 8 total, 8 up, 8 in 2024-09-14T04:54:54.518 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:54 smithi039 ceph-mon[106470]: 56.3 starting backfill to osd.7 from (0'0,1474'3597] MIN to 1502'9509 2024-09-14T04:54:54.518 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:54 smithi039 ceph-mon[111235]: osdmap e1505: 8 total, 8 up, 8 in 2024-09-14T04:54:54.518 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:54 smithi039 ceph-mon[111235]: 56.3 starting backfill to osd.7 from (0'0,1474'3597] MIN to 1502'9509 2024-09-14T04:54:54.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:54 smithi129 ceph-mon[101316]: osdmap e1505: 8 total, 8 up, 8 in 2024-09-14T04:54:54.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:54 smithi129 ceph-mon[101316]: 56.3 starting backfill to osd.7 from (0'0,1474'3597] MIN to 1502'9509 2024-09-14T04:54:54.589 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:54.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:54.635 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:54.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:54.736 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:54.781 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:54.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:54.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:54.883 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:54.932 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:54.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:54.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:55.034 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:55.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:55.079 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:55.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:55.180 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:55.218 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:55.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:55.219 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:55.264 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is set 2024-09-14T04:54:55.298 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:55 smithi129 ceph-mon[101316]: pgmap v2833: 99 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 904 op/s; 4350/230778 objects degraded (1.885%); 2567/230778 objects misplaced (1.112%); 1.5 MiB/s, 23 objects/s recovering 2024-09-14T04:54:55.299 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:55 smithi129 ceph-mon[101316]: osdmap e1506: 8 total, 8 up, 8 in 2024-09-14T04:54:55.299 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:55 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/521213833' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:54:55.299 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:55 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:54:55.320 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:55.354 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:55.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:55.356 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:55.452 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:55 smithi039 ceph-mon[106470]: pgmap v2833: 99 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 904 op/s; 4350/230778 objects degraded (1.885%); 2567/230778 objects misplaced (1.112%); 1.5 MiB/s, 23 objects/s recovering 2024-09-14T04:54:55.452 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:55 smithi039 ceph-mon[106470]: osdmap e1506: 8 total, 8 up, 8 in 2024-09-14T04:54:55.452 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:55 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/521213833' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:54:55.452 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:55 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:54:55.452 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:55 smithi039 ceph-mon[111235]: pgmap v2833: 99 pgs: 1 active+undersized+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 904 op/s; 4350/230778 objects degraded (1.885%); 2567/230778 objects misplaced (1.112%); 1.5 MiB/s, 23 objects/s recovering 2024-09-14T04:54:55.452 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:55 smithi039 ceph-mon[111235]: osdmap e1506: 8 total, 8 up, 8 in 2024-09-14T04:54:55.453 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:55 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/521213833' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:54:55.453 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:55 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:54:55.456 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:55.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:55.488 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:55.488 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:55.589 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:55.625 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:55.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:55.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:55.634 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:54:55.634 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:54:55.728 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:55.766 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:55.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:55.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:55.867 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:55.904 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:55.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:55.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:56.006 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:56.040 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:56.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:56.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:56.141 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:56.180 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:56.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:56.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:56.282 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:56.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:56.317 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:56.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:56.407 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:54:56.409 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:54:56.418 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:56.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:56.479 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:56.479 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:56.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:56 smithi039 ceph-mon[106470]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:54:56.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:56 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:54:56.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:56 smithi039 ceph-mon[106470]: osdmap e1507: 8 total, 8 up, 8 in 2024-09-14T04:54:56.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:56 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 4350/230778 objects degraded (1.885%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:54:56.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:56 smithi039 ceph-mon[111235]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:54:56.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:56 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:54:56.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:56 smithi039 ceph-mon[111235]: osdmap e1507: 8 total, 8 up, 8 in 2024-09-14T04:54:56.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:56 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 4350/230778 objects degraded (1.885%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:54:56.580 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:56.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:56.627 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:56.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:56.728 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:56.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:56 smithi129 ceph-mon[101316]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:54:56.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:56 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:54:56.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:56 smithi129 ceph-mon[101316]: osdmap e1507: 8 total, 8 up, 8 in 2024-09-14T04:54:56.774 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:56 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 4350/230778 objects degraded (1.885%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:54:56.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:56.775 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:56.775 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:56.876 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:56.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:56.923 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:56.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:57.002 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-14T04:54:57.024 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:57.085 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:57.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:57.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:57.187 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:57.223 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:57.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:57.224 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:57.325 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:57.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:57.365 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:57.365 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:57.466 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:57.496 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:57.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:57.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:57.498 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:57 smithi039 ceph-mon[106470]: pgmap v2836: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 4350/227670 objects degraded (1.911%); 6829/227670 objects misplaced (3.000%); 5.9 MiB/s, 94 objects/s recovering 2024-09-14T04:54:57.498 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:57 smithi039 ceph-mon[111235]: pgmap v2836: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 4350/227670 objects degraded (1.911%); 6829/227670 objects misplaced (3.000%); 5.9 MiB/s, 94 objects/s recovering 2024-09-14T04:54:57.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:57 smithi129 ceph-mon[101316]: pgmap v2836: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 4350/227670 objects degraded (1.911%); 6829/227670 objects misplaced (3.000%); 5.9 MiB/s, 94 objects/s recovering 2024-09-14T04:54:57.598 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:57.633 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:57.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:57.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:57.735 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:57.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:57.781 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:57.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:57.882 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:57.921 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:57.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:57.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:58.023 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:58.073 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:58.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:58.074 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:58.175 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:54:58.209 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:58.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:58.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:54:58.311 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:58.356 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:58.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:58.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:58.444 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:54:58.458 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:54:58.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:58.533 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:58.533 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:54:58.634 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:58.663 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:58.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:58.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:58.765 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:54:58.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:54:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:54:58] "GET /metrics HTTP/1.1" 200 36755 "" "Prometheus/2.51.0" 2024-09-14T04:54:58.799 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:58.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:58.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:54:58.901 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:54:58.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:58.943 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:58.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:54:59.043 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:54:59.087 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:59.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:59.088 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:54:59.189 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:54:59.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:59.227 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:59.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:54:59.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:58 smithi129 ceph-mon[101316]: pgmap v2837: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 434 op/s; 4350/227670 objects degraded (1.911%); 6829/227670 objects misplaced (3.000%); 4.2 MiB/s, 67 objects/s recovering 2024-09-14T04:54:59.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:58 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:54:59.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:54:58 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:54:59.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:58 smithi039 ceph-mon[111235]: pgmap v2837: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 434 op/s; 4350/227670 objects degraded (1.911%); 6829/227670 objects misplaced (3.000%); 4.2 MiB/s, 67 objects/s recovering 2024-09-14T04:54:59.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:58 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:54:59.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:54:58 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:54:59.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:58 smithi039 ceph-mon[106470]: pgmap v2837: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.6 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 434 op/s; 4350/227670 objects degraded (1.911%); 6829/227670 objects misplaced (3.000%); 4.2 MiB/s, 67 objects/s recovering 2024-09-14T04:54:59.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:58 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:54:59.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:54:58 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:54:59.329 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:59.364 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:59.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:59.365 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:59.466 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:59.508 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:59.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:59.509 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:59.610 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:59.642 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:59.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:59.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:54:59.744 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:54:59.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:59.778 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:54:59.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:54:59.879 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:54:59.916 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:54:59.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:54:59.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:00.019 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:00.050 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:00.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:00.051 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:00.151 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:00.181 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:00.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:00.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:00.283 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:00.325 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:00.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:00.326 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:00.427 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:00.461 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:00.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:00.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:00.562 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:00.595 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:00.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:00.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:00.697 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:00.762 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:00.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:00.763 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:00.813 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:55:00.849 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:55:00.864 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:00.874 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:55:00.878 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T04:55:00.897 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:00.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:00.899 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:01.000 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:01.031 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:01.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:01.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:01.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:00 smithi039 ceph-mon[106470]: pgmap v2838: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 4350/215460 objects degraded (2.019%); 6418/215460 objects misplaced (2.979%); 5.1 MiB/s, 81 objects/s recovering 2024-09-14T04:55:01.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:00 smithi039 ceph-mon[111235]: pgmap v2838: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 4350/215460 objects degraded (2.019%); 6418/215460 objects misplaced (2.979%); 5.1 MiB/s, 81 objects/s recovering 2024-09-14T04:55:01.133 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:01.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:01.165 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:01.166 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:01.175 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:00 smithi129 ceph-mon[101316]: pgmap v2838: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 886 op/s; 4350/215460 objects degraded (2.019%); 6418/215460 objects misplaced (2.979%); 5.1 MiB/s, 81 objects/s recovering 2024-09-14T04:55:01.266 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:01.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:01.327 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:01.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:01.428 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:01.462 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:01.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:01.463 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:01.564 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:01.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:01.605 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:01.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:01.706 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:01.743 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:01.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:01.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:01.845 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:01.891 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:01.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:01.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:01.915 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:01 smithi039 ceph-mon[111235]: from='client.46133 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:55:01.996 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:02.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:02.037 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:02.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:02.140 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:02.181 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:02.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:02.182 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:02.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:01 smithi129 ceph-mon[101316]: from='client.46133 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:55:02.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:01 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 4350/215460 objects degraded (2.019%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:55:02.282 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:02.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:01 smithi039 ceph-mon[106470]: from='client.46133 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:55:02.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:01 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 4350/215460 objects degraded (2.019%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:55:02.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:01 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 4350/215460 objects degraded (2.019%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:55:02.320 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:02.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:02.321 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:02.422 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:02.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:02.453 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:02.453 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:02.554 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:02.593 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:02.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:02.594 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:02.695 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:02.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:02.726 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:02.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:02.828 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:02.868 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:02.869 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:02.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:02.952 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:02 smithi039 ceph-mon[106470]: pgmap v2839: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 4350/215460 objects degraded (2.019%); 6418/215460 objects misplaced (2.979%); 4.2 MiB/s, 67 objects/s recovering 2024-09-14T04:55:02.952 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:02 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2287043693' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:55:02.952 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:02 smithi039 ceph-mon[111235]: pgmap v2839: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 4350/215460 objects degraded (2.019%); 6418/215460 objects misplaced (2.979%); 4.2 MiB/s, 67 objects/s recovering 2024-09-14T04:55:02.952 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:02 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2287043693' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:55:02.969 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:03.015 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:03.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:03.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:03.117 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:03.154 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:03.155 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:03.155 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:03.256 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:03.272 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:02 smithi129 ceph-mon[101316]: pgmap v2839: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 638 op/s; 4350/215460 objects degraded (2.019%); 6418/215460 objects misplaced (2.979%); 4.2 MiB/s, 67 objects/s recovering 2024-09-14T04:55:03.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:02 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2287043693' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:55:03.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:03.290 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:03.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:03.391 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:03.431 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:03.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:03.432 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:03.533 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:03.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:03.575 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:03.575 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:03.676 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:03.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:03.726 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:03.726 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:03.827 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:03.860 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:03.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:03.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:03.962 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:04.006 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:04.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:04.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:04.108 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:04.112 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":2838,"stamp":"2024-09-14T04:54:59.859531+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4705622013,"num_objects":71820,"num_object_clones":0,"num_object_copies":215460,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4350,"num_objects_misplaced":6418,"num_objects_unfound":0,"num_objects_dirty":71820,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":157314,"num_write_kb":7329378,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7823,"num_bytes_recovered":510336078,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127002,"ondisk_log_size":127002,"up":297,"acting":294,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":307,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19082480,"kb_used_data":14470892,"kb_used_omap":404,"kb_used_meta":4611115,"kb_avail":730878736,"statfs":{"total":767960285184,"available":748419825664,"internally_reserved":0,"allocated":14818193408,"data_stored":14776786267,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":414022,"internal_metadata":4721782458},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-384761856,"num_objects":-5871,"num_object_clones":0,"num_object_copies":-17613,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3851,"num_objects_unfound":0,"num_objects_dirty":-5871,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5871,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":542,"num_bytes_recovered":35520512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.621084"},"pg_stats":[{"pgid":"56.f","version":"1507'9738","reported_seq":9785,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:58.148992+0000","last_change":"2024-09-14T04:52:16.661278+0000","last_active":"2024-09-14T04:54:58.148992+0000","last_peered":"2024-09-14T04:54:58.148992+0000","last_clean":"2024-09-14T04:54:58.148992+0000","last_became_active":"2024-09-14T04:52:16.660906+0000","last_became_peered":"2024-09-14T04:52:16.660906+0000","last_unstale":"2024-09-14T04:54:58.148992+0000","last_undegraded":"2024-09-14T04:54:58.148992+0000","last_fullsized":"2024-09-14T04:54:58.148992+0000","mapping_epoch":1470,"log_start":"1474'2700","ondisk_log_start":"1474'2700","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":7038,"log_dups_size":0,"ondisk_log_size":7038,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:55:16.908898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291241984,"num_objects":4444,"num_object_clones":0,"num_object_copies":13332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4444,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9738,"num_write_kb":453824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"56.e","version":"1507'9903","reported_seq":9950,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:58.638871+0000","last_change":"2024-09-14T04:52:16.723745+0000","last_active":"2024-09-14T04:54:58.638871+0000","last_peered":"2024-09-14T04:54:58.638871+0000","last_clean":"2024-09-14T04:54:58.638871+0000","last_became_active":"2024-09-14T04:52:16.723416+0000","last_became_peered":"2024-09-14T04:52:16.723416+0000","last_unstale":"2024-09-14T04:54:58.638871+0000","last_undegraded":"2024-09-14T04:54:58.638871+0000","last_fullsized":"2024-09-14T04:54:58.638871+0000","mapping_epoch":1470,"log_start":"1474'2547","ondisk_log_start":"1474'2547","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":7356,"log_dups_size":0,"ondisk_log_size":7356,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:20:17.253535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":299696128,"num_objects":4573,"num_object_clones":0,"num_object_copies":13719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9903,"num_write_kb":463232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"56.d","version":"1507'9897","reported_seq":9980,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:58.654395+0000","last_change":"2024-09-14T04:52:16.871824+0000","last_active":"2024-09-14T04:54:58.654395+0000","last_peered":"2024-09-14T04:54:58.654395+0000","last_clean":"2024-09-14T04:54:58.654395+0000","last_became_active":"2024-09-14T04:52:16.871446+0000","last_became_peered":"2024-09-14T04:52:16.871446+0000","last_unstale":"2024-09-14T04:54:58.654395+0000","last_undegraded":"2024-09-14T04:54:58.654395+0000","last_fullsized":"2024-09-14T04:54:58.654395+0000","mapping_epoch":1470,"log_start":"1473'1300","ondisk_log_start":"1473'1300","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8597,"log_dups_size":0,"ondisk_log_size":8597,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:44:00.576543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295305240,"num_objects":4507,"num_object_clones":0,"num_object_copies":13521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4507,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9897,"num_write_kb":460865,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"56.c","version":"1507'9961","reported_seq":10008,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:59.260174+0000","last_change":"2024-09-14T04:52:16.723896+0000","last_active":"2024-09-14T04:54:59.260174+0000","last_peered":"2024-09-14T04:54:59.260174+0000","last_clean":"2024-09-14T04:54:59.260174+0000","last_became_active":"2024-09-14T04:52:16.723597+0000","last_became_peered":"2024-09-14T04:52:16.723597+0000","last_unstale":"2024-09-14T04:54:59.260174+0000","last_undegraded":"2024-09-14T04:54:59.260174+0000","last_fullsized":"2024-09-14T04:54:59.260174+0000","mapping_epoch":1470,"log_start":"1473'1808","ondisk_log_start":"1473'1808","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8153,"log_dups_size":0,"ondisk_log_size":8153,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:10:31.058435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292487168,"num_objects":4463,"num_object_clones":0,"num_object_copies":13389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4463,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9961,"num_write_kb":461568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.b","version":"1507'9830","reported_seq":9877,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:58.209538+0000","last_change":"2024-09-14T04:52:16.660625+0000","last_active":"2024-09-14T04:54:58.209538+0000","last_peered":"2024-09-14T04:54:58.209538+0000","last_clean":"2024-09-14T04:54:58.209538+0000","last_became_active":"2024-09-14T04:52:16.660277+0000","last_became_peered":"2024-09-14T04:52:16.660277+0000","last_unstale":"2024-09-14T04:54:58.209538+0000","last_undegraded":"2024-09-14T04:54:58.209538+0000","last_fullsized":"2024-09-14T04:54:58.209538+0000","mapping_epoch":1470,"log_start":"1474'2800","ondisk_log_start":"1474'2800","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":7030,"log_dups_size":0,"ondisk_log_size":7030,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:43:12.659725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297402368,"num_objects":4538,"num_object_clones":0,"num_object_copies":13614,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4538,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9830,"num_write_kb":459776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"56.a","version":"1507'9686","reported_seq":9732,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:58.210851+0000","last_change":"2024-09-14T04:52:16.915139+0000","last_active":"2024-09-14T04:54:58.210851+0000","last_peered":"2024-09-14T04:54:58.210851+0000","last_clean":"2024-09-14T04:54:58.210851+0000","last_became_active":"2024-09-14T04:52:16.914776+0000","last_became_peered":"2024-09-14T04:52:16.914776+0000","last_unstale":"2024-09-14T04:54:58.210851+0000","last_undegraded":"2024-09-14T04:54:58.210851+0000","last_fullsized":"2024-09-14T04:54:58.210851+0000","mapping_epoch":1470,"log_start":"1474'2700","ondisk_log_start":"1474'2700","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":6986,"log_dups_size":0,"ondisk_log_size":6986,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:08:58.079681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300941312,"num_objects":4592,"num_object_clones":0,"num_object_copies":13776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4592,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9686,"num_write_kb":456896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"56.9","version":"1507'9892","reported_seq":9939,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:59.244868+0000","last_change":"2024-09-14T04:52:16.723789+0000","last_active":"2024-09-14T04:54:59.244868+0000","last_peered":"2024-09-14T04:54:59.244868+0000","last_clean":"2024-09-14T04:54:59.244868+0000","last_became_active":"2024-09-14T04:52:16.723394+0000","last_became_peered":"2024-09-14T04:52:16.723394+0000","last_unstale":"2024-09-14T04:54:59.244868+0000","last_undegraded":"2024-09-14T04:54:59.244868+0000","last_fullsized":"2024-09-14T04:54:59.244868+0000","mapping_epoch":1470,"log_start":"1473'1770","ondisk_log_start":"1473'1770","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8122,"log_dups_size":0,"ondisk_log_size":8122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:45:07.892366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":290848768,"num_objects":4438,"num_object_clones":0,"num_object_copies":13314,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4438,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9892,"num_write_kb":458560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.8","version":"1507'9596","reported_seq":9682,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:58.642369+0000","last_change":"2024-09-14T04:52:16.915103+0000","last_active":"2024-09-14T04:54:58.642369+0000","last_peered":"2024-09-14T04:54:58.642369+0000","last_clean":"2024-09-14T04:54:58.642369+0000","last_became_active":"2024-09-14T04:52:16.914657+0000","last_became_peered":"2024-09-14T04:52:16.914657+0000","last_unstale":"2024-09-14T04:54:58.642369+0000","last_undegraded":"2024-09-14T04:54:58.642369+0000","last_fullsized":"2024-09-14T04:54:58.642369+0000","mapping_epoch":1470,"log_start":"1473'1000","ondisk_log_start":"1473'1000","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8596,"log_dups_size":0,"ondisk_log_size":8596,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:08:43.898740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289406976,"num_objects":4416,"num_object_clones":0,"num_object_copies":13248,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4416,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9596,"num_write_kb":448384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"56.7","version":"1507'9823","reported_seq":9909,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:58.657685+0000","last_change":"2024-09-14T04:52:16.723670+0000","last_active":"2024-09-14T04:54:58.657685+0000","last_peered":"2024-09-14T04:54:58.657685+0000","last_clean":"2024-09-14T04:54:58.657685+0000","last_became_active":"2024-09-14T04:52:16.723292+0000","last_became_peered":"2024-09-14T04:52:16.723292+0000","last_unstale":"2024-09-14T04:54:58.657685+0000","last_undegraded":"2024-09-14T04:54:58.657685+0000","last_fullsized":"2024-09-14T04:54:58.657685+0000","mapping_epoch":1470,"log_start":"1473'1200","ondisk_log_start":"1473'1200","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8623,"log_dups_size":0,"ondisk_log_size":8623,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:55:51.122910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293404672,"num_objects":4477,"num_object_clones":0,"num_object_copies":13431,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4477,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9823,"num_write_kb":457600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"56.6","version":"1507'9671","reported_seq":12539,"reported_epoch":1507,"state":"active+recovering+undersized+remapped","last_fresh":"2024-09-14T04:54:59.294882+0000","last_change":"2024-09-14T04:54:19.620795+0000","last_active":"2024-09-14T04:54:59.294882+0000","last_peered":"2024-09-14T04:54:59.294882+0000","last_clean":"2024-09-14T04:52:58.397671+0000","last_became_active":"2024-09-14T04:54:19.575869+0000","last_became_peered":"2024-09-14T04:54:19.575869+0000","last_unstale":"2024-09-14T04:54:59.294882+0000","last_undegraded":"2024-09-14T04:54:59.294882+0000","last_fullsized":"2024-09-14T04:54:19.444266+0000","mapping_epoch":1489,"log_start":"1473'1557","ondisk_log_start":"1473'1557","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8114,"log_dups_size":0,"ondisk_log_size":8114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:53.013009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283705344,"num_objects":4329,"num_object_clones":0,"num_object_copies":12987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2025,"num_objects_unfound":0,"num_objects_dirty":4329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9671,"num_write_kb":448000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1404,"num_bytes_recovered":92012544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"4,5,7","objects":2025},{"shards":"5,7","objects":2304}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.5","version":"1507'9835","reported_seq":9882,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:58.719803+0000","last_change":"2024-09-14T04:52:16.871791+0000","last_active":"2024-09-14T04:54:58.719803+0000","last_peered":"2024-09-14T04:54:58.719803+0000","last_clean":"2024-09-14T04:54:58.719803+0000","last_became_active":"2024-09-14T04:52:16.871407+0000","last_became_peered":"2024-09-14T04:52:16.871407+0000","last_unstale":"2024-09-14T04:54:58.719803+0000","last_undegraded":"2024-09-14T04:54:58.719803+0000","last_fullsized":"2024-09-14T04:54:58.719803+0000","mapping_epoch":1470,"log_start":"1474'2467","ondisk_log_start":"1474'2467","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":7368,"log_dups_size":0,"ondisk_log_size":7368,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:47:15.419674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296550400,"num_objects":4525,"num_object_clones":0,"num_object_copies":13575,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9835,"num_write_kb":459520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"56.4","version":"1507'9972","reported_seq":10019,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:58.153325+0000","last_change":"2024-09-14T04:52:16.724077+0000","last_active":"2024-09-14T04:54:58.153325+0000","last_peered":"2024-09-14T04:54:58.153325+0000","last_clean":"2024-09-14T04:54:58.153325+0000","last_became_active":"2024-09-14T04:52:16.723597+0000","last_became_peered":"2024-09-14T04:52:16.723597+0000","last_unstale":"2024-09-14T04:54:58.153325+0000","last_undegraded":"2024-09-14T04:54:58.153325+0000","last_fullsized":"2024-09-14T04:54:58.153325+0000","mapping_epoch":1470,"log_start":"1474'2900","ondisk_log_start":"1474'2900","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":7072,"log_dups_size":0,"ondisk_log_size":7072,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:23:49.557180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296222720,"num_objects":4520,"num_object_clones":0,"num_object_copies":13560,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4520,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9972,"num_write_kb":463744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"56.3","version":"1507'9803","reported_seq":17068,"reported_epoch":1507,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:54:59.328520+0000","last_change":"2024-09-14T04:54:53.717396+0000","last_active":"2024-09-14T04:54:59.328520+0000","last_peered":"2024-09-14T04:54:59.328520+0000","last_clean":"2024-09-14T04:54:51.200599+0000","last_became_active":"2024-09-14T04:54:53.681745+0000","last_became_peered":"2024-09-14T04:54:53.681745+0000","last_unstale":"2024-09-14T04:54:59.328520+0000","last_undegraded":"2024-09-14T04:54:59.328520+0000","last_fullsized":"2024-09-14T04:54:59.328520+0000","mapping_epoch":1504,"log_start":"1474'2017","ondisk_log_start":"1474'2017","created":1470,"last_epoch_clean":1496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":7786,"log_dups_size":0,"ondisk_log_size":7786,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287899648,"num_objects":4393,"num_object_clones":0,"num_object_copies":13179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4393,"num_objects_unfound":0,"num_objects_dirty":4393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9803,"num_write_kb":454272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3597,"num_bytes_recovered":235732992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[1,6,5],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":4393}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"56.2","version":"1507'9982","reported_seq":15585,"reported_epoch":1507,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-14T04:54:59.245474+0000","last_change":"2024-09-14T04:54:19.618218+0000","last_active":"2024-09-14T04:54:59.245474+0000","last_peered":"2024-09-14T04:54:59.245474+0000","last_clean":"2024-09-14T04:52:58.403685+0000","last_became_active":"2024-09-14T04:54:19.593740+0000","last_became_peered":"2024-09-14T04:54:19.593740+0000","last_unstale":"2024-09-14T04:54:59.245474+0000","last_undegraded":"2024-09-14T04:54:19.466599+0000","last_fullsized":"2024-09-14T04:54:19.444256+0000","mapping_epoch":1489,"log_start":"1473'1837","ondisk_log_start":"1473'1837","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8145,"log_dups_size":0,"ondisk_log_size":8145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:45:11.467642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297926656,"num_objects":4546,"num_object_clones":0,"num_object_copies":13638,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4349,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4546,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9982,"num_write_kb":464896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2770,"num_bytes_recovered":181534720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"0,5,7","objects":3551},{"shards":"5,7","objects":995}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.1","version":"1507'9880","reported_seq":9927,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:59.263254+0000","last_change":"2024-09-14T04:52:16.872703+0000","last_active":"2024-09-14T04:54:59.263254+0000","last_peered":"2024-09-14T04:54:59.263254+0000","last_clean":"2024-09-14T04:54:59.263254+0000","last_became_active":"2024-09-14T04:52:16.871779+0000","last_became_peered":"2024-09-14T04:52:16.871779+0000","last_unstale":"2024-09-14T04:54:59.263254+0000","last_undegraded":"2024-09-14T04:54:59.263254+0000","last_fullsized":"2024-09-14T04:54:59.263254+0000","mapping_epoch":1470,"log_start":"1473'1765","ondisk_log_start":"1473'1765","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8115,"log_dups_size":0,"ondisk_log_size":8115,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:23:30.260775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296484864,"num_objects":4524,"num_object_clones":0,"num_object_copies":13572,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4524,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9880,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.0","version":"1507'9781","reported_seq":9828,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:59.267405+0000","last_change":"2024-09-14T04:52:16.915325+0000","last_active":"2024-09-14T04:54:59.267405+0000","last_peered":"2024-09-14T04:54:59.267405+0000","last_clean":"2024-09-14T04:54:59.267405+0000","last_became_active":"2024-09-14T04:52:16.914980+0000","last_became_peered":"2024-09-14T04:52:16.914980+0000","last_unstale":"2024-09-14T04:54:59.267405+0000","last_undegraded":"2024-09-14T04:54:59.267405+0000","last_fullsized":"2024-09-14T04:54:59.267405+0000","mapping_epoch":1470,"log_start":"1473'1578","ondisk_log_start":"1473'1578","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8203,"log_dups_size":0,"ondisk_log_size":8203,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:55:08.752960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295501824,"num_objects":4509,"num_object_clones":0,"num_object_copies":13527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9781,"num_write_kb":457280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":2306,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.272580+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T04:54:55.272580+0000","last_peered":"2024-09-14T04:54:55.272580+0000","last_clean":"2024-09-14T04:54:55.272580+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T04:54:55.272580+0000","last_undegraded":"2024-09-14T04:54:55.272580+0000","last_fullsized":"2024-09-14T04:54:55.272580+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":119,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.257990+0000","last_change":"2024-09-14T04:44:57.325109+0000","last_active":"2024-09-14T04:54:54.257990+0000","last_peered":"2024-09-14T04:54:54.257990+0000","last_clean":"2024-09-14T04:54:54.257990+0000","last_became_active":"2024-09-14T04:44:57.324791+0000","last_became_peered":"2024-09-14T04:44:57.324791+0000","last_unstale":"2024-09-14T04:54:54.257990+0000","last_undegraded":"2024-09-14T04:54:54.257990+0000","last_fullsized":"2024-09-14T04:54:54.257990+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:03.593275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":88,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.266346+0000","last_change":"2024-09-14T04:49:29.628435+0000","last_active":"2024-09-14T04:54:54.266346+0000","last_peered":"2024-09-14T04:54:54.266346+0000","last_clean":"2024-09-14T04:54:54.266346+0000","last_became_active":"2024-09-14T04:49:29.628081+0000","last_became_peered":"2024-09-14T04:49:29.628081+0000","last_unstale":"2024-09-14T04:54:54.266346+0000","last_undegraded":"2024-09-14T04:54:54.266346+0000","last_fullsized":"2024-09-14T04:54:54.266346+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:01.520334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":927,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.282356+0000","last_change":"2024-09-14T04:43:13.355281+0000","last_active":"2024-09-14T04:54:55.282356+0000","last_peered":"2024-09-14T04:54:55.282356+0000","last_clean":"2024-09-14T04:54:55.282356+0000","last_became_active":"2024-09-14T04:43:13.354967+0000","last_became_peered":"2024-09-14T04:43:13.354967+0000","last_unstale":"2024-09-14T04:54:55.282356+0000","last_undegraded":"2024-09-14T04:54:55.282356+0000","last_fullsized":"2024-09-14T04:54:55.282356+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:23:19.332075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1497,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.274830+0000","last_change":"2024-09-14T04:44:57.342918+0000","last_active":"2024-09-14T04:54:54.274830+0000","last_peered":"2024-09-14T04:54:54.274830+0000","last_clean":"2024-09-14T04:54:54.274830+0000","last_became_active":"2024-09-14T04:44:57.329550+0000","last_became_peered":"2024-09-14T04:44:57.329550+0000","last_unstale":"2024-09-14T04:54:54.274830+0000","last_undegraded":"2024-09-14T04:54:54.274830+0000","last_fullsized":"2024-09-14T04:54:54.274830+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:52:52.435893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":119,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.264912+0000","last_change":"2024-09-14T04:44:57.325537+0000","last_active":"2024-09-14T04:54:55.264912+0000","last_peered":"2024-09-14T04:54:55.264912+0000","last_clean":"2024-09-14T04:54:55.264912+0000","last_became_active":"2024-09-14T04:44:57.325057+0000","last_became_peered":"2024-09-14T04:44:57.325057+0000","last_unstale":"2024-09-14T04:54:55.264912+0000","last_undegraded":"2024-09-14T04:54:55.264912+0000","last_fullsized":"2024-09-14T04:54:55.264912+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:43:23.166708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1512,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.273592+0000","last_change":"2024-09-14T04:44:57.337033+0000","last_active":"2024-09-14T04:54:54.273592+0000","last_peered":"2024-09-14T04:54:54.273592+0000","last_clean":"2024-09-14T04:54:54.273592+0000","last_became_active":"2024-09-14T04:44:57.328437+0000","last_became_peered":"2024-09-14T04:44:57.328437+0000","last_unstale":"2024-09-14T04:54:54.273592+0000","last_undegraded":"2024-09-14T04:54:54.273592+0000","last_fullsized":"2024-09-14T04:54:54.273592+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.109086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":119,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.265370+0000","last_change":"2024-09-14T04:44:57.332243+0000","last_active":"2024-09-14T04:54:55.265370+0000","last_peered":"2024-09-14T04:54:55.265370+0000","last_clean":"2024-09-14T04:54:55.265370+0000","last_became_active":"2024-09-14T04:44:57.330413+0000","last_became_peered":"2024-09-14T04:44:57.330413+0000","last_unstale":"2024-09-14T04:54:55.265370+0000","last_undegraded":"2024-09-14T04:54:55.265370+0000","last_fullsized":"2024-09-14T04:54:55.265370+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:25:01.338549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":162,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.269085+0000","last_change":"2024-09-14T04:48:29.862929+0000","last_active":"2024-09-14T04:54:55.269085+0000","last_peered":"2024-09-14T04:54:55.269085+0000","last_clean":"2024-09-14T04:54:55.269085+0000","last_became_active":"2024-09-14T04:48:29.862581+0000","last_became_peered":"2024-09-14T04:48:29.862581+0000","last_unstale":"2024-09-14T04:54:55.269085+0000","last_undegraded":"2024-09-14T04:54:55.269085+0000","last_fullsized":"2024-09-14T04:54:55.269085+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:16:43.060088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2302,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.273300+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T04:54:55.273300+0000","last_peered":"2024-09-14T04:54:55.273300+0000","last_clean":"2024-09-14T04:54:55.273300+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T04:54:55.273300+0000","last_undegraded":"2024-09-14T04:54:55.273300+0000","last_fullsized":"2024-09-14T04:54:55.273300+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":119,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.264912+0000","last_change":"2024-09-14T04:44:57.325392+0000","last_active":"2024-09-14T04:54:55.264912+0000","last_peered":"2024-09-14T04:54:55.264912+0000","last_clean":"2024-09-14T04:54:55.264912+0000","last_became_active":"2024-09-14T04:44:57.324624+0000","last_became_peered":"2024-09-14T04:44:57.324624+0000","last_unstale":"2024-09-14T04:54:55.264912+0000","last_undegraded":"2024-09-14T04:54:55.264912+0000","last_fullsized":"2024-09-14T04:54:55.264912+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:33:18.810199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2270,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.268199+0000","last_change":"2024-09-14T04:44:57.329113+0000","last_active":"2024-09-14T04:54:55.268199+0000","last_peered":"2024-09-14T04:54:55.268199+0000","last_clean":"2024-09-14T04:54:55.268199+0000","last_became_active":"2024-09-14T04:44:57.328727+0000","last_became_peered":"2024-09-14T04:44:57.328727+0000","last_unstale":"2024-09-14T04:54:55.268199+0000","last_undegraded":"2024-09-14T04:54:55.268199+0000","last_fullsized":"2024-09-14T04:54:55.268199+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:34.939874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":2287,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.262488+0000","last_change":"2024-09-14T04:44:57.329277+0000","last_active":"2024-09-14T04:54:55.262488+0000","last_peered":"2024-09-14T04:54:55.262488+0000","last_clean":"2024-09-14T04:54:55.262488+0000","last_became_active":"2024-09-14T04:44:57.328804+0000","last_became_peered":"2024-09-14T04:44:57.328804+0000","last_unstale":"2024-09-14T04:54:55.262488+0000","last_undegraded":"2024-09-14T04:54:55.262488+0000","last_fullsized":"2024-09-14T04:54:55.262488+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:42:46.055240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2322,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.269382+0000","last_change":"2024-09-14T04:54:52.250695+0000","last_active":"2024-09-14T04:54:55.269382+0000","last_peered":"2024-09-14T04:54:55.269382+0000","last_clean":"2024-09-14T04:54:55.269382+0000","last_became_active":"2024-09-14T04:54:52.250301+0000","last_became_peered":"2024-09-14T04:54:52.250301+0000","last_unstale":"2024-09-14T04:54:55.269382+0000","last_undegraded":"2024-09-14T04:54:55.269382+0000","last_fullsized":"2024-09-14T04:54:55.269382+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:42.060905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2283,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.271642+0000","last_change":"2024-09-14T04:43:13.357092+0000","last_active":"2024-09-14T04:54:55.271642+0000","last_peered":"2024-09-14T04:54:55.271642+0000","last_clean":"2024-09-14T04:54:55.271642+0000","last_became_active":"2024-09-14T04:43:13.356694+0000","last_became_peered":"2024-09-14T04:43:13.356694+0000","last_unstale":"2024-09-14T04:54:55.271642+0000","last_undegraded":"2024-09-14T04:54:55.271642+0000","last_fullsized":"2024-09-14T04:54:55.271642+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:11.167574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1490,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.266585+0000","last_change":"2024-09-14T04:07:21.720797+0000","last_active":"2024-09-14T04:54:54.266585+0000","last_peered":"2024-09-14T04:54:54.266585+0000","last_clean":"2024-09-14T04:54:54.266585+0000","last_became_active":"2024-09-14T04:07:21.719331+0000","last_became_peered":"2024-09-14T04:07:21.719331+0000","last_unstale":"2024-09-14T04:54:54.266585+0000","last_undegraded":"2024-09-14T04:54:54.266585+0000","last_fullsized":"2024-09-14T04:54:54.266585+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:35.404023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":138,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.273252+0000","last_change":"2024-09-14T04:43:11.641514+0000","last_active":"2024-09-14T04:54:54.273252+0000","last_peered":"2024-09-14T04:54:54.273252+0000","last_clean":"2024-09-14T04:54:54.273252+0000","last_became_active":"2024-09-14T04:43:11.641151+0000","last_became_peered":"2024-09-14T04:43:11.641151+0000","last_unstale":"2024-09-14T04:54:54.273252+0000","last_undegraded":"2024-09-14T04:54:54.273252+0000","last_fullsized":"2024-09-14T04:54:54.273252+0000","mapping_epoch":1384,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:06:45.045430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2295,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.272746+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T04:54:55.272746+0000","last_peered":"2024-09-14T04:54:55.272746+0000","last_clean":"2024-09-14T04:54:55.272746+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T04:54:55.272746+0000","last_undegraded":"2024-09-14T04:54:55.272746+0000","last_fullsized":"2024-09-14T04:54:55.272746+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":718,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.264053+0000","last_change":"2024-09-14T04:49:29.618477+0000","last_active":"2024-09-14T04:54:54.264053+0000","last_peered":"2024-09-14T04:54:54.264053+0000","last_clean":"2024-09-14T04:54:54.264053+0000","last_became_active":"2024-09-14T04:49:29.618210+0000","last_became_peered":"2024-09-14T04:49:29.618210+0000","last_unstale":"2024-09-14T04:54:54.264053+0000","last_undegraded":"2024-09-14T04:54:54.264053+0000","last_fullsized":"2024-09-14T04:54:54.264053+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:13:23.895512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1496,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.277496+0000","last_change":"2024-09-14T04:42:49.938505+0000","last_active":"2024-09-14T04:54:54.277496+0000","last_peered":"2024-09-14T04:54:54.277496+0000","last_clean":"2024-09-14T04:54:54.277496+0000","last_became_active":"2024-09-14T04:42:49.938180+0000","last_became_peered":"2024-09-14T04:42:49.938180+0000","last_unstale":"2024-09-14T04:54:54.277496+0000","last_undegraded":"2024-09-14T04:54:54.277496+0000","last_fullsized":"2024-09-14T04:54:54.277496+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:05.929479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1515,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.264362+0000","last_change":"2024-09-14T04:44:57.346582+0000","last_active":"2024-09-14T04:54:54.264362+0000","last_peered":"2024-09-14T04:54:54.264362+0000","last_clean":"2024-09-14T04:54:54.264362+0000","last_became_active":"2024-09-14T04:44:57.330446+0000","last_became_peered":"2024-09-14T04:44:57.330446+0000","last_unstale":"2024-09-14T04:54:54.264362+0000","last_undegraded":"2024-09-14T04:54:54.264362+0000","last_fullsized":"2024-09-14T04:54:54.264362+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:29.238585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":236,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.262871+0000","last_change":"2024-09-14T04:44:14.480667+0000","last_active":"2024-09-14T04:54:55.262871+0000","last_peered":"2024-09-14T04:54:55.262871+0000","last_clean":"2024-09-14T04:54:55.262871+0000","last_became_active":"2024-09-14T04:44:14.480365+0000","last_became_peered":"2024-09-14T04:44:14.480365+0000","last_unstale":"2024-09-14T04:54:55.262871+0000","last_undegraded":"2024-09-14T04:54:55.262871+0000","last_fullsized":"2024-09-14T04:54:55.262871+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:00.611771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":129,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.261227+0000","last_change":"2024-09-14T04:44:57.341426+0000","last_active":"2024-09-14T04:54:54.261227+0000","last_peered":"2024-09-14T04:54:54.261227+0000","last_clean":"2024-09-14T04:54:54.261227+0000","last_became_active":"2024-09-14T04:44:57.328763+0000","last_became_peered":"2024-09-14T04:44:57.328763+0000","last_unstale":"2024-09-14T04:54:54.261227+0000","last_undegraded":"2024-09-14T04:54:54.261227+0000","last_fullsized":"2024-09-14T04:54:54.261227+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:34:40.066761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":119,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.263608+0000","last_change":"2024-09-14T04:44:57.331337+0000","last_active":"2024-09-14T04:54:54.263608+0000","last_peered":"2024-09-14T04:54:54.263608+0000","last_clean":"2024-09-14T04:54:54.263608+0000","last_became_active":"2024-09-14T04:44:57.329336+0000","last_became_peered":"2024-09-14T04:44:57.329336+0000","last_unstale":"2024-09-14T04:54:54.263608+0000","last_undegraded":"2024-09-14T04:54:54.263608+0000","last_fullsized":"2024-09-14T04:54:54.263608+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:52:35.980869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"349'149","reported_seq":707,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.273092+0000","last_change":"2024-09-14T04:49:30.085274+0000","last_active":"2024-09-14T04:54:54.273092+0000","last_peered":"2024-09-14T04:54:54.273092+0000","last_clean":"2024-09-14T04:54:54.273092+0000","last_became_active":"2024-09-14T04:49:29.623295+0000","last_became_peered":"2024-09-14T04:49:29.623295+0000","last_unstale":"2024-09-14T04:54:54.273092+0000","last_undegraded":"2024-09-14T04:54:54.273092+0000","last_fullsized":"2024-09-14T04:54:54.273092+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:49:30.085089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:49:30.085089+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:10.012963+0000","scrub_duration":0.0071928790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":119,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.258251+0000","last_change":"2024-09-14T04:44:57.329870+0000","last_active":"2024-09-14T04:54:54.258251+0000","last_peered":"2024-09-14T04:54:54.258251+0000","last_clean":"2024-09-14T04:54:54.258251+0000","last_became_active":"2024-09-14T04:44:57.329353+0000","last_became_peered":"2024-09-14T04:44:57.329353+0000","last_unstale":"2024-09-14T04:54:54.258251+0000","last_undegraded":"2024-09-14T04:54:54.258251+0000","last_fullsized":"2024-09-14T04:54:54.258251+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:46.432181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1512,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.266660+0000","last_change":"2024-09-14T04:54:52.242207+0000","last_active":"2024-09-14T04:54:54.266660+0000","last_peered":"2024-09-14T04:54:54.266660+0000","last_clean":"2024-09-14T04:54:54.266660+0000","last_became_active":"2024-09-14T04:54:52.241840+0000","last_became_peered":"2024-09-14T04:54:52.241840+0000","last_unstale":"2024-09-14T04:54:54.266660+0000","last_undegraded":"2024-09-14T04:54:54.266660+0000","last_fullsized":"2024-09-14T04:54:54.266660+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:37.899666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":2333,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.269287+0000","last_change":"2024-09-14T04:44:57.344614+0000","last_active":"2024-09-14T04:54:55.269287+0000","last_peered":"2024-09-14T04:54:55.269287+0000","last_clean":"2024-09-14T04:54:55.269287+0000","last_became_active":"2024-09-14T04:44:57.332256+0000","last_became_peered":"2024-09-14T04:44:57.332256+0000","last_unstale":"2024-09-14T04:54:55.269287+0000","last_undegraded":"2024-09-14T04:54:55.269287+0000","last_fullsized":"2024-09-14T04:54:55.269287+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:23.438417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"349'149","reported_seq":877,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.262077+0000","last_change":"2024-09-14T04:45:00.331854+0000","last_active":"2024-09-14T04:54:54.262077+0000","last_peered":"2024-09-14T04:54:54.262077+0000","last_clean":"2024-09-14T04:54:54.262077+0000","last_became_active":"2024-09-14T04:45:00.331460+0000","last_became_peered":"2024-09-14T04:45:00.331460+0000","last_unstale":"2024-09-14T04:54:54.262077+0000","last_undegraded":"2024-09-14T04:54:54.262077+0000","last_fullsized":"2024-09-14T04:54:54.262077+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.567535+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.567535+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:45.095729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":708,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.263675+0000","last_change":"2024-09-14T04:49:29.616831+0000","last_active":"2024-09-14T04:54:54.263675+0000","last_peered":"2024-09-14T04:54:54.263675+0000","last_clean":"2024-09-14T04:54:54.263675+0000","last_became_active":"2024-09-14T04:49:29.616546+0000","last_became_peered":"2024-09-14T04:49:29.616546+0000","last_unstale":"2024-09-14T04:54:54.263675+0000","last_undegraded":"2024-09-14T04:54:54.263675+0000","last_fullsized":"2024-09-14T04:54:54.263675+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:27.882929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":288,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.265082+0000","last_change":"2024-09-14T04:44:57.330746+0000","last_active":"2024-09-14T04:54:55.265082+0000","last_peered":"2024-09-14T04:54:55.265082+0000","last_clean":"2024-09-14T04:54:55.265082+0000","last_became_active":"2024-09-14T04:44:57.330327+0000","last_became_peered":"2024-09-14T04:44:57.330327+0000","last_unstale":"2024-09-14T04:54:55.265082+0000","last_undegraded":"2024-09-14T04:54:55.265082+0000","last_fullsized":"2024-09-14T04:54:55.265082+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:53.122771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":159,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.266868+0000","last_change":"2024-09-14T04:43:13.355296+0000","last_active":"2024-09-14T04:54:54.266868+0000","last_peered":"2024-09-14T04:54:54.266868+0000","last_clean":"2024-09-14T04:54:54.266868+0000","last_became_active":"2024-09-14T04:43:13.354950+0000","last_became_peered":"2024-09-14T04:43:13.354950+0000","last_unstale":"2024-09-14T04:54:54.266868+0000","last_undegraded":"2024-09-14T04:54:54.266868+0000","last_fullsized":"2024-09-14T04:54:54.266868+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:20:08.968818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"349'149","reported_seq":816,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.266204+0000","last_change":"2024-09-14T04:41:39.471393+0000","last_active":"2024-09-14T04:54:54.266204+0000","last_peered":"2024-09-14T04:54:54.266204+0000","last_clean":"2024-09-14T04:54:54.266204+0000","last_became_active":"2024-09-14T04:41:36.931768+0000","last_became_peered":"2024-09-14T04:41:36.931768+0000","last_unstale":"2024-09-14T04:54:54.266204+0000","last_undegraded":"2024-09-14T04:54:54.266204+0000","last_fullsized":"2024-09-14T04:54:54.266204+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.471248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.471248+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:56.210375+0000","scrub_duration":0.021758344999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2289,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.269353+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T04:54:55.269353+0000","last_peered":"2024-09-14T04:54:55.269353+0000","last_clean":"2024-09-14T04:54:55.269353+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T04:54:55.269353+0000","last_undegraded":"2024-09-14T04:54:55.269353+0000","last_fullsized":"2024-09-14T04:54:55.269353+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1526,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.264540+0000","last_change":"2024-09-14T04:44:57.331436+0000","last_active":"2024-09-14T04:54:54.264540+0000","last_peered":"2024-09-14T04:54:54.264540+0000","last_clean":"2024-09-14T04:54:54.264540+0000","last_became_active":"2024-09-14T04:44:57.330957+0000","last_became_peered":"2024-09-14T04:44:57.330957+0000","last_unstale":"2024-09-14T04:54:54.264540+0000","last_undegraded":"2024-09-14T04:54:54.264540+0000","last_fullsized":"2024-09-14T04:54:54.264540+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:09.516688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":838,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.273199+0000","last_change":"2024-09-14T04:49:49.460401+0000","last_active":"2024-09-14T04:54:55.273199+0000","last_peered":"2024-09-14T04:54:55.273199+0000","last_clean":"2024-09-14T04:54:55.273199+0000","last_became_active":"2024-09-14T04:49:49.458530+0000","last_became_peered":"2024-09-14T04:49:49.458530+0000","last_unstale":"2024-09-14T04:54:55.273199+0000","last_undegraded":"2024-09-14T04:54:55.273199+0000","last_fullsized":"2024-09-14T04:54:55.273199+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:11:29.143239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"349'149","reported_seq":850,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.266505+0000","last_change":"2024-09-14T04:54:22.147441+0000","last_active":"2024-09-14T04:54:54.266505+0000","last_peered":"2024-09-14T04:54:54.266505+0000","last_clean":"2024-09-14T04:54:54.266505+0000","last_became_active":"2024-09-14T04:54:22.147115+0000","last_became_peered":"2024-09-14T04:54:22.147115+0000","last_unstale":"2024-09-14T04:54:54.266505+0000","last_undegraded":"2024-09-14T04:54:54.266505+0000","last_fullsized":"2024-09-14T04:54:54.266505+0000","mapping_epoch":1492,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1493,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:38.479887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:38.479887+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:41:14.001781+0000","scrub_duration":0.0062728319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":244,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.262791+0000","last_change":"2024-09-14T04:44:57.329983+0000","last_active":"2024-09-14T04:54:55.262791+0000","last_peered":"2024-09-14T04:54:55.262791+0000","last_clean":"2024-09-14T04:54:55.262791+0000","last_became_active":"2024-09-14T04:44:57.329597+0000","last_became_peered":"2024-09-14T04:44:57.329597+0000","last_unstale":"2024-09-14T04:54:55.262791+0000","last_undegraded":"2024-09-14T04:54:55.262791+0000","last_fullsized":"2024-09-14T04:54:55.262791+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:45.718039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":782,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.269016+0000","last_change":"2024-09-14T04:49:49.443631+0000","last_active":"2024-09-14T04:54:55.269016+0000","last_peered":"2024-09-14T04:54:55.269016+0000","last_clean":"2024-09-14T04:54:55.269016+0000","last_became_active":"2024-09-14T04:49:49.443313+0000","last_became_peered":"2024-09-14T04:49:49.443313+0000","last_unstale":"2024-09-14T04:54:55.269016+0000","last_undegraded":"2024-09-14T04:54:55.269016+0000","last_fullsized":"2024-09-14T04:54:55.269016+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:06:54.562292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1498,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.274968+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T04:54:54.274968+0000","last_peered":"2024-09-14T04:54:54.274968+0000","last_clean":"2024-09-14T04:54:54.274968+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T04:54:54.274968+0000","last_undegraded":"2024-09-14T04:54:54.274968+0000","last_fullsized":"2024-09-14T04:54:54.274968+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"349'149","reported_seq":808,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.273144+0000","last_change":"2024-09-14T04:41:39.012977+0000","last_active":"2024-09-14T04:54:54.273144+0000","last_peered":"2024-09-14T04:54:54.273144+0000","last_clean":"2024-09-14T04:54:54.273144+0000","last_became_active":"2024-09-14T04:41:36.932552+0000","last_became_peered":"2024-09-14T04:41:36.932552+0000","last_unstale":"2024-09-14T04:54:54.273144+0000","last_undegraded":"2024-09-14T04:54:54.273144+0000","last_fullsized":"2024-09-14T04:54:54.273144+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.012879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.012879+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:16:40.080262+0000","scrub_duration":0.0051721620000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":874,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.273385+0000","last_change":"2024-09-14T04:49:49.459590+0000","last_active":"2024-09-14T04:54:55.273385+0000","last_peered":"2024-09-14T04:54:55.273385+0000","last_clean":"2024-09-14T04:54:55.273385+0000","last_became_active":"2024-09-14T04:49:49.456263+0000","last_became_peered":"2024-09-14T04:49:49.456263+0000","last_unstale":"2024-09-14T04:54:55.273385+0000","last_undegraded":"2024-09-14T04:54:55.273385+0000","last_fullsized":"2024-09-14T04:54:55.273385+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:20:44.666805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":119,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.263538+0000","last_change":"2024-09-14T04:44:57.329652+0000","last_active":"2024-09-14T04:54:54.263538+0000","last_peered":"2024-09-14T04:54:54.263538+0000","last_clean":"2024-09-14T04:54:54.263538+0000","last_became_active":"2024-09-14T04:44:57.328888+0000","last_became_peered":"2024-09-14T04:44:57.328888+0000","last_unstale":"2024-09-14T04:54:54.263538+0000","last_undegraded":"2024-09-14T04:54:54.263538+0000","last_fullsized":"2024-09-14T04:54:54.263538+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:10.170068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1497,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.278179+0000","last_change":"2024-09-14T04:06:46.875340+0000","last_active":"2024-09-14T04:54:54.278179+0000","last_peered":"2024-09-14T04:54:54.278179+0000","last_clean":"2024-09-14T04:54:54.278179+0000","last_became_active":"2024-09-14T04:06:46.874552+0000","last_became_peered":"2024-09-14T04:06:46.874552+0000","last_unstale":"2024-09-14T04:54:54.278179+0000","last_undegraded":"2024-09-14T04:54:54.278179+0000","last_fullsized":"2024-09-14T04:54:54.278179+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:52.257357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"349'149","reported_seq":711,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.279983+0000","last_change":"2024-09-14T04:54:55.279666+0000","last_active":"2024-09-14T04:54:55.279983+0000","last_peered":"2024-09-14T04:54:55.279983+0000","last_clean":"2024-09-14T04:54:55.279983+0000","last_became_active":"2024-09-14T04:54:55.279259+0000","last_became_peered":"2024-09-14T04:54:55.279259+0000","last_unstale":"2024-09-14T04:54:55.279983+0000","last_undegraded":"2024-09-14T04:54:55.279983+0000","last_fullsized":"2024-09-14T04:54:55.279983+0000","mapping_epoch":1506,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1507,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:54:03.996025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:54:03.996025+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:40:53.332369+0000","scrub_duration":54,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":236,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.262635+0000","last_change":"2024-09-14T04:44:14.489235+0000","last_active":"2024-09-14T04:54:55.262635+0000","last_peered":"2024-09-14T04:54:55.262635+0000","last_clean":"2024-09-14T04:54:55.262635+0000","last_became_active":"2024-09-14T04:44:14.488950+0000","last_became_peered":"2024-09-14T04:44:14.488950+0000","last_unstale":"2024-09-14T04:54:55.262635+0000","last_undegraded":"2024-09-14T04:54:55.262635+0000","last_fullsized":"2024-09-14T04:54:55.262635+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:54.441144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":792,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.268268+0000","last_change":"2024-09-14T04:49:49.456317+0000","last_active":"2024-09-14T04:54:55.268268+0000","last_peered":"2024-09-14T04:54:55.268268+0000","last_clean":"2024-09-14T04:54:55.268268+0000","last_became_active":"2024-09-14T04:49:49.456029+0000","last_became_peered":"2024-09-14T04:49:49.456029+0000","last_unstale":"2024-09-14T04:54:55.268268+0000","last_undegraded":"2024-09-14T04:54:55.268268+0000","last_fullsized":"2024-09-14T04:54:55.268268+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:46:05.319676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2309,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.282466+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T04:54:55.282466+0000","last_peered":"2024-09-14T04:54:55.282466+0000","last_clean":"2024-09-14T04:54:55.282466+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T04:54:55.282466+0000","last_undegraded":"2024-09-14T04:54:55.282466+0000","last_fullsized":"2024-09-14T04:54:55.282466+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"349'149","reported_seq":702,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.265539+0000","last_change":"2024-09-14T04:50:22.002798+0000","last_active":"2024-09-14T04:54:54.265539+0000","last_peered":"2024-09-14T04:54:54.265539+0000","last_clean":"2024-09-14T04:54:54.265539+0000","last_became_active":"2024-09-14T04:49:49.448278+0000","last_became_peered":"2024-09-14T04:49:49.448278+0000","last_unstale":"2024-09-14T04:54:54.265539+0000","last_undegraded":"2024-09-14T04:54:54.265539+0000","last_fullsized":"2024-09-14T04:54:54.265539+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:50:22.002568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:50:22.002568+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:01:57.489223+0000","scrub_duration":0.010888326,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2310,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.269475+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T04:54:55.269475+0000","last_peered":"2024-09-14T04:54:55.269475+0000","last_clean":"2024-09-14T04:54:55.269475+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T04:54:55.269475+0000","last_undegraded":"2024-09-14T04:54:55.269475+0000","last_fullsized":"2024-09-14T04:54:55.269475+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2331,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.270344+0000","last_change":"2024-09-14T04:54:19.447103+0000","last_active":"2024-09-14T04:54:55.270344+0000","last_peered":"2024-09-14T04:54:55.270344+0000","last_clean":"2024-09-14T04:54:55.270344+0000","last_became_active":"2024-09-14T04:54:19.446748+0000","last_became_peered":"2024-09-14T04:54:19.446748+0000","last_unstale":"2024-09-14T04:54:55.270344+0000","last_undegraded":"2024-09-14T04:54:55.270344+0000","last_fullsized":"2024-09-14T04:54:55.270344+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:00:55.369263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1527,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.263877+0000","last_change":"2024-09-14T04:42:49.939790+0000","last_active":"2024-09-14T04:54:54.263877+0000","last_peered":"2024-09-14T04:54:54.263877+0000","last_clean":"2024-09-14T04:54:54.263877+0000","last_became_active":"2024-09-14T04:42:49.939591+0000","last_became_peered":"2024-09-14T04:42:49.939591+0000","last_unstale":"2024-09-14T04:54:54.263877+0000","last_undegraded":"2024-09-14T04:54:54.263877+0000","last_fullsized":"2024-09-14T04:54:54.263877+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:23:16.790593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1490,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.266097+0000","last_change":"2024-09-14T04:44:57.326008+0000","last_active":"2024-09-14T04:54:54.266097+0000","last_peered":"2024-09-14T04:54:54.266097+0000","last_clean":"2024-09-14T04:54:54.266097+0000","last_became_active":"2024-09-14T04:44:57.325617+0000","last_became_peered":"2024-09-14T04:44:57.325617+0000","last_unstale":"2024-09-14T04:54:54.266097+0000","last_undegraded":"2024-09-14T04:54:54.266097+0000","last_fullsized":"2024-09-14T04:54:54.266097+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:15.637508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":779,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.278248+0000","last_change":"2024-09-14T04:43:13.354846+0000","last_active":"2024-09-14T04:54:54.278248+0000","last_peered":"2024-09-14T04:54:54.278248+0000","last_clean":"2024-09-14T04:54:54.278248+0000","last_became_active":"2024-09-14T04:43:13.354515+0000","last_became_peered":"2024-09-14T04:43:13.354515+0000","last_unstale":"2024-09-14T04:54:54.278248+0000","last_undegraded":"2024-09-14T04:54:54.278248+0000","last_fullsized":"2024-09-14T04:54:54.278248+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:17.889341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1496,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.282295+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T04:54:54.282295+0000","last_peered":"2024-09-14T04:54:54.282295+0000","last_clean":"2024-09-14T04:54:54.282295+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T04:54:54.282295+0000","last_undegraded":"2024-09-14T04:54:54.282295+0000","last_fullsized":"2024-09-14T04:54:54.282295+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":1833,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.263558+0000","last_change":"2024-09-14T04:41:30.640651+0000","last_active":"2024-09-14T04:54:54.263558+0000","last_peered":"2024-09-14T04:54:54.263558+0000","last_clean":"2024-09-14T04:54:54.263558+0000","last_became_active":"2024-09-14T04:41:29.532466+0000","last_became_peered":"2024-09-14T04:41:29.532466+0000","last_unstale":"2024-09-14T04:54:54.263558+0000","last_undegraded":"2024-09-14T04:54:54.263558+0000","last_fullsized":"2024-09-14T04:54:54.263558+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":0,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:30.640520+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:30.640520+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:27.776724+0000","scrub_duration":0.0028653139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"349'149","reported_seq":808,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.278106+0000","last_change":"2024-09-14T04:41:39.978080+0000","last_active":"2024-09-14T04:54:54.278106+0000","last_peered":"2024-09-14T04:54:54.278106+0000","last_clean":"2024-09-14T04:54:54.278106+0000","last_became_active":"2024-09-14T04:41:36.929279+0000","last_became_peered":"2024-09-14T04:41:36.929279+0000","last_unstale":"2024-09-14T04:54:54.278106+0000","last_undegraded":"2024-09-14T04:54:54.278106+0000","last_fullsized":"2024-09-14T04:54:54.278106+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.977999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.977999+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:44:06.383827+0000","scrub_duration":0.00192122,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":170,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.273931+0000","last_change":"2024-09-14T04:48:11.040133+0000","last_active":"2024-09-14T04:54:55.273931+0000","last_peered":"2024-09-14T04:54:55.273931+0000","last_clean":"2024-09-14T04:54:55.273931+0000","last_became_active":"2024-09-14T04:48:11.039787+0000","last_became_peered":"2024-09-14T04:48:11.039787+0000","last_unstale":"2024-09-14T04:54:55.273931+0000","last_undegraded":"2024-09-14T04:54:55.273931+0000","last_fullsized":"2024-09-14T04:54:55.273931+0000","mapping_epoch":1428,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:53.520672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2157,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.262173+0000","last_change":"2024-09-14T04:49:29.616443+0000","last_active":"2024-09-14T04:54:55.262173+0000","last_peered":"2024-09-14T04:54:55.262173+0000","last_clean":"2024-09-14T04:54:55.262173+0000","last_became_active":"2024-09-14T04:49:29.616066+0000","last_became_peered":"2024-09-14T04:49:29.616066+0000","last_unstale":"2024-09-14T04:54:55.262173+0000","last_undegraded":"2024-09-14T04:54:55.262173+0000","last_fullsized":"2024-09-14T04:54:55.262173+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:09:15.897957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":119,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.261469+0000","last_change":"2024-09-14T04:44:57.326724+0000","last_active":"2024-09-14T04:54:54.261469+0000","last_peered":"2024-09-14T04:54:54.261469+0000","last_clean":"2024-09-14T04:54:54.261469+0000","last_became_active":"2024-09-14T04:44:57.326191+0000","last_became_peered":"2024-09-14T04:44:57.326191+0000","last_unstale":"2024-09-14T04:54:54.261469+0000","last_undegraded":"2024-09-14T04:54:54.261469+0000","last_fullsized":"2024-09-14T04:54:54.261469+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:46.060269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":119,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.258135+0000","last_change":"2024-09-14T04:44:57.328528+0000","last_active":"2024-09-14T04:54:54.258135+0000","last_peered":"2024-09-14T04:54:54.258135+0000","last_clean":"2024-09-14T04:54:54.258135+0000","last_became_active":"2024-09-14T04:44:57.327739+0000","last_became_peered":"2024-09-14T04:44:57.327739+0000","last_unstale":"2024-09-14T04:54:54.258135+0000","last_undegraded":"2024-09-14T04:54:54.258135+0000","last_fullsized":"2024-09-14T04:54:54.258135+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:04:58.294690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"349'149","reported_seq":831,"reported_epoch":1507,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-14T04:54:55.268030+0000","last_change":"2024-09-14T04:54:36.689611+0000","last_active":"2024-09-14T04:54:55.268030+0000","last_peered":"2024-09-14T04:54:55.268030+0000","last_clean":"2024-09-14T04:54:31.592636+0000","last_became_active":"2024-09-14T04:54:36.676136+0000","last_became_peered":"2024-09-14T04:54:36.676136+0000","last_unstale":"2024-09-14T04:54:55.268030+0000","last_undegraded":"2024-09-14T04:54:36.658607+0000","last_fullsized":"2024-09-14T04:54:36.658065+0000","mapping_epoch":1499,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1479,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:56.738636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:56.738636+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"no scrub is scheduled","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[2,3],"avail_no_missing":["2","3","6"],"object_location_counts":[{"shards":"2,3,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":119,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.261856+0000","last_change":"2024-09-14T04:44:57.323436+0000","last_active":"2024-09-14T04:54:54.261856+0000","last_peered":"2024-09-14T04:54:54.261856+0000","last_clean":"2024-09-14T04:54:54.261856+0000","last_became_active":"2024-09-14T04:44:57.323022+0000","last_became_peered":"2024-09-14T04:44:57.323022+0000","last_unstale":"2024-09-14T04:54:54.261856+0000","last_undegraded":"2024-09-14T04:54:54.261856+0000","last_fullsized":"2024-09-14T04:54:54.261856+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:22.775261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1525,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.269529+0000","last_change":"2024-09-14T04:44:57.322357+0000","last_active":"2024-09-14T04:54:54.269529+0000","last_peered":"2024-09-14T04:54:54.269529+0000","last_clean":"2024-09-14T04:54:54.269529+0000","last_became_active":"2024-09-14T04:44:57.321990+0000","last_became_peered":"2024-09-14T04:44:57.321990+0000","last_unstale":"2024-09-14T04:54:54.269529+0000","last_undegraded":"2024-09-14T04:54:54.269529+0000","last_fullsized":"2024-09-14T04:54:54.269529+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:09.658008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"349'149","reported_seq":1850,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.263982+0000","last_change":"2024-09-14T04:41:53.858076+0000","last_active":"2024-09-14T04:54:54.263982+0000","last_peered":"2024-09-14T04:54:54.263982+0000","last_clean":"2024-09-14T04:54:54.263982+0000","last_became_active":"2024-09-14T04:41:36.931585+0000","last_became_peered":"2024-09-14T04:41:36.931585+0000","last_unstale":"2024-09-14T04:54:54.263982+0000","last_undegraded":"2024-09-14T04:54:54.263982+0000","last_fullsized":"2024-09-14T04:54:54.263982+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:53.857924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:53.857924+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:10:20.658986+0000","scrub_duration":0.0022355919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":119,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.258149+0000","last_change":"2024-09-14T04:44:57.329198+0000","last_active":"2024-09-14T04:54:54.258149+0000","last_peered":"2024-09-14T04:54:54.258149+0000","last_clean":"2024-09-14T04:54:54.258149+0000","last_became_active":"2024-09-14T04:44:57.327754+0000","last_became_peered":"2024-09-14T04:44:57.327754+0000","last_unstale":"2024-09-14T04:54:54.258149+0000","last_undegraded":"2024-09-14T04:54:54.258149+0000","last_fullsized":"2024-09-14T04:54:54.258149+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:10:27.944476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":294,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.269446+0000","last_change":"2024-09-14T04:44:57.348687+0000","last_active":"2024-09-14T04:54:55.269446+0000","last_peered":"2024-09-14T04:54:55.269446+0000","last_clean":"2024-09-14T04:54:55.269446+0000","last_became_active":"2024-09-14T04:44:57.332845+0000","last_became_peered":"2024-09-14T04:44:57.332845+0000","last_unstale":"2024-09-14T04:54:55.269446+0000","last_undegraded":"2024-09-14T04:54:55.269446+0000","last_fullsized":"2024-09-14T04:54:55.269446+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:12:18.762737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2316,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.269192+0000","last_change":"2024-09-14T04:44:57.333468+0000","last_active":"2024-09-14T04:54:55.269192+0000","last_peered":"2024-09-14T04:54:55.269192+0000","last_clean":"2024-09-14T04:54:55.269192+0000","last_became_active":"2024-09-14T04:44:57.332626+0000","last_became_peered":"2024-09-14T04:44:57.332626+0000","last_unstale":"2024-09-14T04:54:55.269192+0000","last_undegraded":"2024-09-14T04:54:55.269192+0000","last_fullsized":"2024-09-14T04:54:55.269192+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:57:26.366438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1481,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.273178+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T04:54:54.273178+0000","last_peered":"2024-09-14T04:54:54.273178+0000","last_clean":"2024-09-14T04:54:54.273178+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T04:54:54.273178+0000","last_undegraded":"2024-09-14T04:54:54.273178+0000","last_fullsized":"2024-09-14T04:54:54.273178+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1480,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.269406+0000","last_change":"2024-09-14T04:06:46.869868+0000","last_active":"2024-09-14T04:54:54.269406+0000","last_peered":"2024-09-14T04:54:54.269406+0000","last_clean":"2024-09-14T04:54:54.269406+0000","last_became_active":"2024-09-14T04:06:46.869126+0000","last_became_peered":"2024-09-14T04:06:46.869126+0000","last_unstale":"2024-09-14T04:54:54.269406+0000","last_undegraded":"2024-09-14T04:54:54.269406+0000","last_fullsized":"2024-09-14T04:54:54.269406+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:41.150138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1464,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.264477+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T04:54:54.264477+0000","last_peered":"2024-09-14T04:54:54.264477+0000","last_clean":"2024-09-14T04:54:54.264477+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T04:54:54.264477+0000","last_undegraded":"2024-09-14T04:54:54.264477+0000","last_fullsized":"2024-09-14T04:54:54.264477+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":119,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.258365+0000","last_change":"2024-09-14T04:44:57.332112+0000","last_active":"2024-09-14T04:54:54.258365+0000","last_peered":"2024-09-14T04:54:54.258365+0000","last_clean":"2024-09-14T04:54:54.258365+0000","last_became_active":"2024-09-14T04:44:57.331575+0000","last_became_peered":"2024-09-14T04:44:57.331575+0000","last_unstale":"2024-09-14T04:54:54.258365+0000","last_undegraded":"2024-09-14T04:54:54.258365+0000","last_fullsized":"2024-09-14T04:54:54.258365+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:50:00.391283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1512,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.265335+0000","last_change":"2024-09-14T04:44:57.351460+0000","last_active":"2024-09-14T04:54:54.265335+0000","last_peered":"2024-09-14T04:54:54.265335+0000","last_clean":"2024-09-14T04:54:54.265335+0000","last_became_active":"2024-09-14T04:44:57.334662+0000","last_became_peered":"2024-09-14T04:44:57.334662+0000","last_unstale":"2024-09-14T04:54:54.265335+0000","last_undegraded":"2024-09-14T04:54:54.265335+0000","last_fullsized":"2024-09-14T04:54:54.265335+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:02.399062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1466,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.273361+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T04:54:54.273361+0000","last_peered":"2024-09-14T04:54:54.273361+0000","last_clean":"2024-09-14T04:54:54.273361+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T04:54:54.273361+0000","last_undegraded":"2024-09-14T04:54:54.273361+0000","last_fullsized":"2024-09-14T04:54:54.273361+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1524,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.266784+0000","last_change":"2024-09-14T04:44:57.349690+0000","last_active":"2024-09-14T04:54:54.266784+0000","last_peered":"2024-09-14T04:54:54.266784+0000","last_clean":"2024-09-14T04:54:54.266784+0000","last_became_active":"2024-09-14T04:44:57.333579+0000","last_became_peered":"2024-09-14T04:44:57.333579+0000","last_unstale":"2024-09-14T04:54:54.266784+0000","last_undegraded":"2024-09-14T04:54:54.266784+0000","last_fullsized":"2024-09-14T04:54:54.266784+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:17:55.265141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1601,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.266597+0000","last_change":"2024-09-14T04:54:35.694089+0000","last_active":"2024-09-14T04:54:54.266597+0000","last_peered":"2024-09-14T04:54:54.266597+0000","last_clean":"2024-09-14T04:54:54.266597+0000","last_became_active":"2024-09-14T04:54:35.663133+0000","last_became_peered":"2024-09-14T04:54:35.663133+0000","last_unstale":"2024-09-14T04:54:54.266597+0000","last_undegraded":"2024-09-14T04:54:54.266597+0000","last_fullsized":"2024-09-14T04:54:54.266597+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:17:19.947280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":14,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.266545+0000","last_change":"2024-09-14T04:54:52.248439+0000","last_active":"2024-09-14T04:54:54.266545+0000","last_peered":"2024-09-14T04:54:54.266545+0000","last_clean":"2024-09-14T04:54:54.266545+0000","last_became_active":"2024-09-14T04:54:52.248235+0000","last_became_peered":"2024-09-14T04:54:52.248235+0000","last_unstale":"2024-09-14T04:54:54.266545+0000","last_undegraded":"2024-09-14T04:54:54.266545+0000","last_fullsized":"2024-09-14T04:54:54.266545+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:22:09.548139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1486,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.264184+0000","last_change":"2024-09-14T04:44:57.332511+0000","last_active":"2024-09-14T04:54:54.264184+0000","last_peered":"2024-09-14T04:54:54.264184+0000","last_clean":"2024-09-14T04:54:54.264184+0000","last_became_active":"2024-09-14T04:44:57.332166+0000","last_became_peered":"2024-09-14T04:44:57.332166+0000","last_unstale":"2024-09-14T04:54:54.264184+0000","last_undegraded":"2024-09-14T04:54:54.264184+0000","last_fullsized":"2024-09-14T04:54:54.264184+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:53.984639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2310,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.269190+0000","last_change":"2024-09-14T04:54:52.250655+0000","last_active":"2024-09-14T04:54:55.269190+0000","last_peered":"2024-09-14T04:54:55.269190+0000","last_clean":"2024-09-14T04:54:55.269190+0000","last_became_active":"2024-09-14T04:54:52.250317+0000","last_became_peered":"2024-09-14T04:54:52.250317+0000","last_unstale":"2024-09-14T04:54:55.269190+0000","last_undegraded":"2024-09-14T04:54:55.269190+0000","last_fullsized":"2024-09-14T04:54:55.269190+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:12.985406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2296,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.272410+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T04:54:55.272410+0000","last_peered":"2024-09-14T04:54:55.272410+0000","last_clean":"2024-09-14T04:54:55.272410+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T04:54:55.272410+0000","last_undegraded":"2024-09-14T04:54:55.272410+0000","last_fullsized":"2024-09-14T04:54:55.272410+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1478,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.274894+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T04:54:54.274894+0000","last_peered":"2024-09-14T04:54:54.274894+0000","last_clean":"2024-09-14T04:54:54.274894+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T04:54:54.274894+0000","last_undegraded":"2024-09-14T04:54:54.274894+0000","last_fullsized":"2024-09-14T04:54:54.274894+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1515,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.264121+0000","last_change":"2024-09-14T04:44:57.339993+0000","last_active":"2024-09-14T04:54:54.264121+0000","last_peered":"2024-09-14T04:54:54.264121+0000","last_clean":"2024-09-14T04:54:54.264121+0000","last_became_active":"2024-09-14T04:44:57.330472+0000","last_became_peered":"2024-09-14T04:44:57.330472+0000","last_unstale":"2024-09-14T04:54:54.264121+0000","last_undegraded":"2024-09-14T04:54:54.264121+0000","last_fullsized":"2024-09-14T04:54:54.264121+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:19:27.046988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2063,"reported_epoch":1506,"state":"active+clean","last_fresh":"2024-09-14T04:54:54.263755+0000","last_change":"2024-09-14T04:49:49.449687+0000","last_active":"2024-09-14T04:54:54.263755+0000","last_peered":"2024-09-14T04:54:54.263755+0000","last_clean":"2024-09-14T04:54:54.263755+0000","last_became_active":"2024-09-14T04:49:49.449144+0000","last_became_peered":"2024-09-14T04:49:49.449144+0000","last_unstale":"2024-09-14T04:54:54.263755+0000","last_undegraded":"2024-09-14T04:54:54.263755+0000","last_fullsized":"2024-09-14T04:54:54.263755+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:59:06.779730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":56,"num_pg":16,"stat_sum":{"num_bytes":4705026072,"num_objects":71794,"num_object_clones":0,"num_object_copies":215382,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4349,"num_objects_misplaced":6418,"num_objects_unfound":0,"num_objects_dirty":71794,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":157250,"num_write_kb":7329345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7771,"num_bytes_recovered":509280256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14646128640,"data_stored":14646116424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125304,"ondisk_log_size":125304,"up":48,"acting":46,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":32,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":5070,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4250,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223202048,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2605796,"kb_used_data":1929116,"kb_used_omap":1,"kb_used_meta":676670,"kb_avail":91139356,"statfs":{"total":95995035648,"available":93326700544,"internally_reserved":0,"allocated":1975414784,"data_stored":1970226575,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":692910539},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463684,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1799068,"kb_used_data":1164244,"kb_used_omap":1,"kb_used_meta":634814,"kb_avail":91946084,"statfs":{"total":95995035648,"available":94152790016,"internally_reserved":0,"allocated":1192185856,"data_stored":1187006232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":650049991},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725321,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3015376,"kb_used_data":2298504,"kb_used_omap":1,"kb_used_meta":716862,"kb_avail":90729776,"statfs":{"total":95995035648,"available":92907290624,"internally_reserved":0,"allocated":2353668096,"data_stored":2348496042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":734067142},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":1392,"seq":5978594476167,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2241312,"kb_used_data":1983768,"kb_used_omap":117,"kb_used_meta":257418,"kb_avail":91503840,"statfs":{"total":95995035648,"available":93699932160,"internally_reserved":0,"allocated":2031378432,"data_stored":2026194847,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120638,"internal_metadata":263596226},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772075062,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2221912,"kb_used_data":1544268,"kb_used_omap":71,"kb_used_meta":677560,"kb_avail":91523240,"statfs":{"total":95995035648,"available":93719797760,"internally_reserved":0,"allocated":1581330432,"data_stored":1576154903,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":693822390},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707303998,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2483600,"kb_used_data":2004488,"kb_used_omap":68,"kb_used_meta":479035,"kb_avail":91261552,"statfs":{"total":95995035648,"available":93451829248,"internally_reserved":0,"allocated":2052595712,"data_stored":2047424159,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":490532048},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872729157,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2295688,"kb_used_data":1740288,"kb_used_omap":67,"kb_used_meta":555324,"kb_avail":91449464,"statfs":{"total":95995035648,"available":93644251136,"internally_reserved":0,"allocated":1782054912,"data_stored":1776890410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":568652133},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056207,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2419728,"kb_used_data":1806216,"kb_used_omap":74,"kb_used_meta":613429,"kb_avail":91325424,"statfs":{"total":95995035648,"available":93517234176,"internally_reserved":0,"allocated":1849565184,"data_stored":1844393099,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":628151989},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1901,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1828720640,"data_stored":1828716568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1753874432,"data_stored":1753874432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2035482624,"data_stored":2035482624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1552814080,"data_stored":1552810008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2003959808,"data_stored":2003959808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2337538048,"data_stored":2337538048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1174405120,"data_stored":1174405120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1959333888,"data_stored":1959329816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:55:04.118 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:55:04.136 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:04.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:04.137 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:04.239 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:04.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:04.273 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:04.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:04.375 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:04.409 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:04.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:04.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:04.511 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:04.548 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:04.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:04.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:04.650 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:04.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:04.696 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:04.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:04.797 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:04.832 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:04.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:04.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:04.934 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:04.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:04.969 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:04.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:05.071 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:05.099 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:05.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:05.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:05.201 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:05.207 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:55:05.207 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:55:05.207 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:55:05.207 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:55:05.208 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:55:05.208 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:55:05.208 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:55:05.208 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:55:05.208 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:55:05.208 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:55:05.208 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:55:05.208 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:55:05.209 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:55:05.209 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:55:05.209 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:55:05.209 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:55:05.210 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:04 smithi039 ceph-mon[106470]: pgmap v2840: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 4350/208251 objects degraded (2.089%); 6418/208251 objects misplaced (3.082%); 3.5 MiB/s, 56 objects/s recovering 2024-09-14T04:55:05.210 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:04 smithi039 ceph-mon[111235]: pgmap v2840: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 4350/208251 objects degraded (2.089%); 6418/208251 objects misplaced (3.082%); 3.5 MiB/s, 56 objects/s recovering 2024-09-14T04:55:05.252 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:05.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:05.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:05.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:04 smithi129 ceph-mon[101316]: pgmap v2840: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 781 op/s; 4350/208251 objects degraded (2.089%); 6418/208251 objects misplaced (3.082%); 3.5 MiB/s, 56 objects/s recovering 2024-09-14T04:55:05.355 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:05.396 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:05.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:05.397 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:05.498 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:05.539 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:05.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:05.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:05.640 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:05.673 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:05.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:05.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:05.775 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:05.813 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:05.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:05.816 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:05.917 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:05.950 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:05.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:05.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:06.052 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:06.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:06.092 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:06.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:06.193 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:06.238 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:06.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:06.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:06.340 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:06.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:06.378 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:06.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:06.479 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:06.511 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:06.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:06.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:06.585 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:55:06.585 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [], 2024-09-14T04:55:06.585 INFO:teuthology.orchestra.run.smithi039.stdout: "complaint_time": 30, 2024-09-14T04:55:06.585 INFO:teuthology.orchestra.run.smithi039.stdout: "num_blocked_ops": 0 2024-09-14T04:55:06.585 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:55:06.613 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:06.641 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:06.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:06.642 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:06.743 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:06.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:06.776 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:06.776 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:06.877 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:06.928 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:06.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:06.934 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:07.035 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:07.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:06 smithi039 ceph-mon[106470]: pgmap v2841: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 4350/202761 objects degraded (2.145%); 6003/202761 objects misplaced (2.961%); 1.9 MiB/s, 30 objects/s recovering 2024-09-14T04:55:07.042 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:06 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 4350/202761 objects degraded (2.145%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:55:07.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:06 smithi039 ceph-mon[111235]: pgmap v2841: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 4350/202761 objects degraded (2.145%); 6003/202761 objects misplaced (2.961%); 1.9 MiB/s, 30 objects/s recovering 2024-09-14T04:55:07.042 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:06 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 4350/202761 objects degraded (2.145%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:55:07.074 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:07.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:07.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:07.176 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:07.212 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:07.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:07.213 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:07.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:06 smithi129 ceph-mon[101316]: pgmap v2841: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 783 op/s; 4350/202761 objects degraded (2.145%); 6003/202761 objects misplaced (2.961%); 1.9 MiB/s, 30 objects/s recovering 2024-09-14T04:55:07.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:06 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 4350/202761 objects degraded (2.145%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:55:07.314 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:07.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:07.350 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:07.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:07.451 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:07.491 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:07.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:07.492 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:07.593 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:07.602 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-14T04:55:07.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:07.634 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:07.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:07.735 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:07.765 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:07.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:07.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:07.866 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:07.897 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:07.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:07.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:07.999 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:08.005 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:55:08.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:08.039 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:08.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:08.140 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:08.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:08.170 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:08.170 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:08.271 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:08.306 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:08.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:08.307 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:08.323 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is set 2024-09-14T04:55:08.328 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:55:08.329 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:08 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:55:08.329 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:08 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1737398186' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:55:08.408 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:08.443 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:08.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:08.444 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:08.545 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:08.582 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:08.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:08.583 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:08.644 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:08 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:55:08.644 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:08 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:55:08.644 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:08 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1737398186' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:55:08.684 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:08.717 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:08.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:08.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:08.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:08 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:55:08.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:08 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:55:08.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:08 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1737398186' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:55:08.818 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:08.855 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:08.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:08.856 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:08.957 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:08.980 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:08.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:08.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:09.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:55:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:55:08] "GET /metrics HTTP/1.1" 200 36758 "" "Prometheus/2.51.0" 2024-09-14T04:55:09.084 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:09.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:09.145 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:09.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:09.246 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:09.290 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:09.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:09.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:09.392 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:09.430 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:09.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:09.433 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:09.533 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:09.539 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:09 smithi039 ceph-mon[106470]: pgmap v2842: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 4350/202761 objects degraded (2.145%); 6003/202761 objects misplaced (2.961%); 1.7 MiB/s, 27 objects/s recovering 2024-09-14T04:55:09.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:09 smithi039 ceph-mon[106470]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:55:09.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:09 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1737398186' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:55:09.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:09 smithi039 ceph-mon[106470]: osdmap e1508: 8 total, 8 up, 8 in 2024-09-14T04:55:09.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:09 smithi039 ceph-mon[111235]: pgmap v2842: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 4350/202761 objects degraded (2.145%); 6003/202761 objects misplaced (2.961%); 1.7 MiB/s, 27 objects/s recovering 2024-09-14T04:55:09.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:09 smithi039 ceph-mon[111235]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:55:09.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:09 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1737398186' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:55:09.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:09 smithi039 ceph-mon[111235]: osdmap e1508: 8 total, 8 up, 8 in 2024-09-14T04:55:09.565 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:09.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:09.566 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:09.667 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:09.711 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:09.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:09.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:09.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:09 smithi129 ceph-mon[101316]: pgmap v2842: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 4.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 691 op/s; 4350/202761 objects degraded (2.145%); 6003/202761 objects misplaced (2.961%); 1.7 MiB/s, 27 objects/s recovering 2024-09-14T04:55:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:09 smithi129 ceph-mon[101316]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:55:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:09 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1737398186' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:55:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:09 smithi129 ceph-mon[101316]: osdmap e1508: 8 total, 8 up, 8 in 2024-09-14T04:55:09.819 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:09.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:09.853 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:09.853 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:09.954 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:09.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:09.987 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:09.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:10.088 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:10.126 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:10.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:10.127 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:10.212 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:55:10.228 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:10.252 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:10.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:10.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:10.354 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:10.401 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:10.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:10.402 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:10.503 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:10.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:10.538 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:10.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:10.621 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:55:10.639 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:10.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:10.683 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:10.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:10.784 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:10.823 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:10.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:10.824 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:10.925 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:10.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:10.975 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:10.975 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:11.076 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:11.115 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:11.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:11.116 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:11.217 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:11.261 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:11.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:11.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:11.271 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:55:11.363 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:11.399 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:11.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:11.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:11.501 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:11.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:11.533 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:11.533 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:11.539 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:11 smithi039 ceph-mon[106470]: pgmap v2844: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 860 op/s; 4350/189639 objects degraded (2.294%); 5578/189639 objects misplaced (2.941%); 2.2 MiB/s, 34 objects/s recovering 2024-09-14T04:55:11.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:11 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 4350/189639 objects degraded (2.294%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:55:11.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:11 smithi039 ceph-mon[111235]: pgmap v2844: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 860 op/s; 4350/189639 objects degraded (2.294%); 5578/189639 objects misplaced (2.941%); 2.2 MiB/s, 34 objects/s recovering 2024-09-14T04:55:11.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:11 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 4350/189639 objects degraded (2.294%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:55:11.634 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:11.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:11.673 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:11.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:11.752 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:55:11.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:11 smithi129 ceph-mon[101316]: pgmap v2844: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 860 op/s; 4350/189639 objects degraded (2.294%); 5578/189639 objects misplaced (2.941%); 2.2 MiB/s, 34 objects/s recovering 2024-09-14T04:55:11.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:11 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 4350/189639 objects degraded (2.294%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:55:11.774 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:11.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:11.838 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:11.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:11.940 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:11.968 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:11.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:11.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:12.070 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:12.106 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:12.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:12.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:12.208 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:12.246 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:12.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:12.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:12.347 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:12.385 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:12.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:12.386 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:12.487 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:12.547 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:12.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:12.548 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:12.648 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:12.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:12.689 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:12.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:12.790 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:12.829 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:12.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:12.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:12.931 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:12.970 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:12.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:12.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:13.072 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:13.110 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:13.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:13.111 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:13.212 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:13.258 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:13.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:13.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:13.360 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:13.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:13.397 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:13.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:13.498 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:13.540 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:13.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:13 smithi039 ceph-mon[106470]: pgmap v2845: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 860 op/s; 4350/189639 objects degraded (2.294%); 5578/189639 objects misplaced (2.941%); 2.2 MiB/s, 34 objects/s recovering 2024-09-14T04:55:13.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:13 smithi039 ceph-mon[111235]: pgmap v2845: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 860 op/s; 4350/189639 objects degraded (2.294%); 5578/189639 objects misplaced (2.941%); 2.2 MiB/s, 34 objects/s recovering 2024-09-14T04:55:13.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:13.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:13.642 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:13.676 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:13.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:13.678 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:13.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:13 smithi129 ceph-mon[101316]: pgmap v2845: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.9 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 860 op/s; 4350/189639 objects degraded (2.294%); 5578/189639 objects misplaced (2.941%); 2.2 MiB/s, 34 objects/s recovering 2024-09-14T04:55:13.779 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:13.822 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:13.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:13.823 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:13.924 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:13.965 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:13.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:13.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:14.066 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:14.099 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:14.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:14.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:14.201 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:14.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:14.245 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:14.245 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:14.346 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:14.383 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:14.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:14.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:14.485 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:14.513 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:14.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:14.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:14.615 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:14.654 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:14.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:14.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:14.756 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:14.789 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:14.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:14.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:14.891 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:14.936 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:14.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:14.937 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:15.038 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:15.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:15.079 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:15.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:15.180 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:15.226 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:15.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:15.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:15.328 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:15.365 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:15.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:15.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:15.466 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:15.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:15.496 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:15.496 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:15.597 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:15.639 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:15.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:15.640 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:15.691 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:15 smithi039 ceph-mon[106470]: pgmap v2846: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 4350/182334 objects degraded (2.386%); 5578/182334 objects misplaced (3.059%); 2.2 MiB/s, 34 objects/s recovering 2024-09-14T04:55:15.691 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:15 smithi039 ceph-mon[111235]: pgmap v2846: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 4350/182334 objects degraded (2.386%); 5578/182334 objects misplaced (3.059%); 2.2 MiB/s, 34 objects/s recovering 2024-09-14T04:55:15.741 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:15.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:15 smithi129 ceph-mon[101316]: pgmap v2846: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 4350/182334 objects degraded (2.386%); 5578/182334 objects misplaced (3.059%); 2.2 MiB/s, 34 objects/s recovering 2024-09-14T04:55:15.781 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:15.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:15.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:15.883 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:15.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:15.924 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:15.924 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:16.026 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:16.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:16.064 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:16.064 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:16.165 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:16.205 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:16.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:16.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:16.307 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:16.348 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:16.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:16.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:16.451 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:16.484 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:16.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:16.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:16.585 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:16.626 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:16.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:16.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:16.727 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:16.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:16.759 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:16.759 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:16.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:16 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 4350/182334 objects degraded (2.386%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:55:16.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:16 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 4350/182334 objects degraded (2.386%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:55:16.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:16 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 4350/182334 objects degraded (2.386%), 2 pgs degraded (PG_DEGRADED) 2024-09-14T04:55:16.860 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:16.883 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:16.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:16.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:16.985 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:17.026 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:17.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:17.027 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:17.128 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:17.169 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:17.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:17.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:17.271 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:17.315 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:17.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:17.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:17.417 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:17.463 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:17.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:17.464 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:17.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:17 smithi039 ceph-mon[106470]: pgmap v2847: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 4350/176445 objects degraded (2.465%); 5099/176445 objects misplaced (2.890%); 2.4 MiB/s, 38 objects/s recovering 2024-09-14T04:55:17.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:17 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:55:17.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:17 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:55:17.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:17 smithi039 ceph-mon[111235]: pgmap v2847: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 4350/176445 objects degraded (2.465%); 5099/176445 objects misplaced (2.890%); 2.4 MiB/s, 38 objects/s recovering 2024-09-14T04:55:17.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:17 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:55:17.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:17 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:55:17.565 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:17.602 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:17.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:17.607 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:17.708 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:17.752 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:17.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:17.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:17.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:17 smithi129 ceph-mon[101316]: pgmap v2847: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 4350/176445 objects degraded (2.465%); 5099/176445 objects misplaced (2.890%); 2.4 MiB/s, 38 objects/s recovering 2024-09-14T04:55:17.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:17 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:55:17.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:17 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:55:17.857 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:17.914 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:17.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:17.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:18.016 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:18.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:18.046 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:18.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:18.147 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:18.185 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:18.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:18.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:18.286 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:18.322 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:18.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:18.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:18.424 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:18.467 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:18.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:18.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:18.569 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:18.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:18.616 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:18.616 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:18.651 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:18 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:55:18.651 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:18 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:55:18.651 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:18 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:55:18.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:18 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:55:18.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:18 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:55:18.651 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:18 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:55:18.717 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:18.760 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:18.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:18.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:18.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:18 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:55:18.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:18 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:55:18.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:18 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:55:18.861 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:18.888 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:18.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:18.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:18.990 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:18.994 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:55:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:55:18] "GET /metrics HTTP/1.1" 200 36758 "" "Prometheus/2.51.0" 2024-09-14T04:55:19.037 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:19.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:19.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:19.139 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:19.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:19.181 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:19.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:19.282 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:19.328 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:19.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:19.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:19.412 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:55:19.414 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T04:55:19.431 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:19.462 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:19.462 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:19.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:19.563 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:19.595 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:19.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:19.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:19.697 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:19.748 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:19.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:19.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:19.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:19 smithi129 ceph-mon[101316]: pgmap v2848: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 4350/176445 objects degraded (2.465%); 5099/176445 objects misplaced (2.890%); 2.4 MiB/s, 38 objects/s recovering 2024-09-14T04:55:19.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:19 smithi039 ceph-mon[106470]: pgmap v2848: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 4350/176445 objects degraded (2.465%); 5099/176445 objects misplaced (2.890%); 2.4 MiB/s, 38 objects/s recovering 2024-09-14T04:55:19.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:19 smithi039 ceph-mon[111235]: pgmap v2848: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 876 op/s; 4350/176445 objects degraded (2.465%); 5099/176445 objects misplaced (2.890%); 2.4 MiB/s, 38 objects/s recovering 2024-09-14T04:55:19.850 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:19.893 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:19.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:19.894 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:19.995 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:20.029 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:20.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:20.031 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:20.132 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:20.172 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:20.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:20.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:20.274 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:20.292 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":2848,"stamp":"2024-09-14T04:55:17.866549+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3853326333,"num_objects":58815,"num_object_clones":0,"num_object_copies":176445,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4350,"num_objects_misplaced":5099,"num_objects_unfound":0,"num_objects_dirty":58815,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":170319,"num_write_kb":7329378,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8365,"num_bytes_recovered":545856590,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127107,"ondisk_log_size":127107,"up":297,"acting":294,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":306,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":16593132,"kb_used_data":11924524,"kb_used_omap":404,"kb_used_meta":4667883,"kb_avail":733368084,"statfs":{"total":767960285184,"available":750968918016,"internally_reserved":0,"allocated":12210712576,"data_stored":12169245939,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":414022,"internal_metadata":4779912890},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[8,8,4,6,13,7],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-574881792,"num_objects":-8772,"num_object_clones":0,"num_object_copies":-26316,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-904,"num_objects_unfound":0,"num_objects_dirty":-8772,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8772,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":381,"num_bytes_recovered":24969216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004342"},"pg_stats":[{"pgid":"56.f","version":"1508'10523","reported_seq":10571,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:13.220282+0000","last_change":"2024-09-14T04:52:16.661278+0000","last_active":"2024-09-14T04:55:13.220282+0000","last_peered":"2024-09-14T04:55:13.220282+0000","last_clean":"2024-09-14T04:55:13.220282+0000","last_became_active":"2024-09-14T04:52:16.660906+0000","last_became_peered":"2024-09-14T04:52:16.660906+0000","last_unstale":"2024-09-14T04:55:13.220282+0000","last_undegraded":"2024-09-14T04:55:13.220282+0000","last_fullsized":"2024-09-14T04:55:13.220282+0000","mapping_epoch":1470,"log_start":"1474'3500","ondisk_log_start":"1474'3500","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":7023,"log_dups_size":0,"ondisk_log_size":7023,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:55:16.908898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239796224,"num_objects":3659,"num_object_clones":0,"num_object_copies":10977,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3659,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10523,"num_write_kb":453824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"56.e","version":"1508'10698","reported_seq":10746,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:13.733358+0000","last_change":"2024-09-14T04:52:16.723745+0000","last_active":"2024-09-14T04:55:13.733358+0000","last_peered":"2024-09-14T04:55:13.733358+0000","last_clean":"2024-09-14T04:55:13.733358+0000","last_became_active":"2024-09-14T04:52:16.723416+0000","last_became_peered":"2024-09-14T04:52:16.723416+0000","last_unstale":"2024-09-14T04:55:13.733358+0000","last_undegraded":"2024-09-14T04:55:13.733358+0000","last_fullsized":"2024-09-14T04:55:13.733358+0000","mapping_epoch":1470,"log_start":"1474'3347","ondisk_log_start":"1474'3347","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":7351,"log_dups_size":0,"ondisk_log_size":7351,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:20:17.253535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247595008,"num_objects":3778,"num_object_clones":0,"num_object_copies":11334,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3778,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10698,"num_write_kb":463232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"56.d","version":"1508'10713","reported_seq":10802,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:13.663714+0000","last_change":"2024-09-14T04:52:16.871824+0000","last_active":"2024-09-14T04:55:13.663714+0000","last_peered":"2024-09-14T04:55:13.663714+0000","last_clean":"2024-09-14T04:55:13.663714+0000","last_became_active":"2024-09-14T04:52:16.871446+0000","last_became_peered":"2024-09-14T04:52:16.871446+0000","last_unstale":"2024-09-14T04:55:13.663714+0000","last_undegraded":"2024-09-14T04:55:13.663714+0000","last_fullsized":"2024-09-14T04:55:13.663714+0000","mapping_epoch":1470,"log_start":"1474'2100","ondisk_log_start":"1474'2100","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8613,"log_dups_size":0,"ondisk_log_size":8613,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:44:00.576543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241827864,"num_objects":3691,"num_object_clones":0,"num_object_copies":11073,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3691,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10713,"num_write_kb":460865,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"56.c","version":"1508'10803","reported_seq":10849,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:14.332586+0000","last_change":"2024-09-14T04:52:16.723896+0000","last_active":"2024-09-14T04:55:14.332586+0000","last_peered":"2024-09-14T04:55:14.332586+0000","last_clean":"2024-09-14T04:55:14.332586+0000","last_became_active":"2024-09-14T04:52:16.723597+0000","last_became_peered":"2024-09-14T04:52:16.723597+0000","last_unstale":"2024-09-14T04:55:14.332586+0000","last_undegraded":"2024-09-14T04:55:14.332586+0000","last_fullsized":"2024-09-14T04:55:14.332586+0000","mapping_epoch":1470,"log_start":"1474'2608","ondisk_log_start":"1474'2608","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8195,"log_dups_size":0,"ondisk_log_size":8195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:10:31.058435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237305856,"num_objects":3621,"num_object_clones":0,"num_object_copies":10863,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3621,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10803,"num_write_kb":461568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.b","version":"1508'10659","reported_seq":10707,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:13.186381+0000","last_change":"2024-09-14T04:52:16.660625+0000","last_active":"2024-09-14T04:55:13.186381+0000","last_peered":"2024-09-14T04:55:13.186381+0000","last_clean":"2024-09-14T04:55:13.186381+0000","last_became_active":"2024-09-14T04:52:16.660277+0000","last_became_peered":"2024-09-14T04:52:16.660277+0000","last_unstale":"2024-09-14T04:55:13.186381+0000","last_undegraded":"2024-09-14T04:55:13.186381+0000","last_fullsized":"2024-09-14T04:55:13.186381+0000","mapping_epoch":1470,"log_start":"1474'3600","ondisk_log_start":"1474'3600","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":7059,"log_dups_size":0,"ondisk_log_size":7059,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:43:12.659725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243073024,"num_objects":3709,"num_object_clones":0,"num_object_copies":11127,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3709,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10659,"num_write_kb":459776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"56.a","version":"1508'10537","reported_seq":10585,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:13.229418+0000","last_change":"2024-09-14T04:52:16.915139+0000","last_active":"2024-09-14T04:55:13.229418+0000","last_peered":"2024-09-14T04:55:13.229418+0000","last_clean":"2024-09-14T04:55:13.229418+0000","last_became_active":"2024-09-14T04:52:16.914776+0000","last_became_peered":"2024-09-14T04:52:16.914776+0000","last_unstale":"2024-09-14T04:55:13.229418+0000","last_undegraded":"2024-09-14T04:55:13.229418+0000","last_fullsized":"2024-09-14T04:55:13.229418+0000","mapping_epoch":1470,"log_start":"1474'3500","ondisk_log_start":"1474'3500","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":7037,"log_dups_size":0,"ondisk_log_size":7037,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:08:58.079681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245170176,"num_objects":3741,"num_object_clones":0,"num_object_copies":11223,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3741,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10537,"num_write_kb":456896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"56.9","version":"1508'10686","reported_seq":10734,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:14.371388+0000","last_change":"2024-09-14T04:52:16.723789+0000","last_active":"2024-09-14T04:55:14.371388+0000","last_peered":"2024-09-14T04:55:14.371388+0000","last_clean":"2024-09-14T04:55:14.371388+0000","last_became_active":"2024-09-14T04:52:16.723394+0000","last_became_peered":"2024-09-14T04:52:16.723394+0000","last_unstale":"2024-09-14T04:55:14.371388+0000","last_undegraded":"2024-09-14T04:55:14.371388+0000","last_fullsized":"2024-09-14T04:55:14.371388+0000","mapping_epoch":1470,"log_start":"1474'2570","ondisk_log_start":"1474'2570","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8116,"log_dups_size":0,"ondisk_log_size":8116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:45:07.892366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238813184,"num_objects":3644,"num_object_clones":0,"num_object_copies":10932,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3644,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10686,"num_write_kb":458560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.8","version":"1508'10393","reported_seq":10480,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:13.621863+0000","last_change":"2024-09-14T04:52:16.915103+0000","last_active":"2024-09-14T04:55:13.621863+0000","last_peered":"2024-09-14T04:55:13.621863+0000","last_clean":"2024-09-14T04:55:13.621863+0000","last_became_active":"2024-09-14T04:52:16.914657+0000","last_became_peered":"2024-09-14T04:52:16.914657+0000","last_unstale":"2024-09-14T04:55:13.621863+0000","last_undegraded":"2024-09-14T04:55:13.621863+0000","last_fullsized":"2024-09-14T04:55:13.621863+0000","mapping_epoch":1470,"log_start":"1473'1800","ondisk_log_start":"1473'1800","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8593,"log_dups_size":0,"ondisk_log_size":8593,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:08:43.898740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237174784,"num_objects":3619,"num_object_clones":0,"num_object_copies":10857,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3619,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10393,"num_write_kb":448384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"56.7","version":"1508'10631","reported_seq":10719,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:13.667743+0000","last_change":"2024-09-14T04:52:16.723670+0000","last_active":"2024-09-14T04:55:13.667743+0000","last_peered":"2024-09-14T04:55:13.667743+0000","last_clean":"2024-09-14T04:55:13.667743+0000","last_became_active":"2024-09-14T04:52:16.723292+0000","last_became_peered":"2024-09-14T04:52:16.723292+0000","last_unstale":"2024-09-14T04:55:13.667743+0000","last_undegraded":"2024-09-14T04:55:13.667743+0000","last_fullsized":"2024-09-14T04:55:13.667743+0000","mapping_epoch":1470,"log_start":"1474'2000","ondisk_log_start":"1474'2000","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8631,"log_dups_size":0,"ondisk_log_size":8631,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:55:51.122910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240451584,"num_objects":3669,"num_object_clones":0,"num_object_copies":11007,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3669,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10631,"num_write_kb":457600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"56.6","version":"1508'10479","reported_seq":14432,"reported_epoch":1508,"state":"active+recovering+undersized+remapped","last_fresh":"2024-09-14T04:55:14.368518+0000","last_change":"2024-09-14T04:54:19.620795+0000","last_active":"2024-09-14T04:55:14.368518+0000","last_peered":"2024-09-14T04:55:14.368518+0000","last_clean":"2024-09-14T04:52:58.397671+0000","last_became_active":"2024-09-14T04:54:19.575869+0000","last_became_peered":"2024-09-14T04:54:19.575869+0000","last_unstale":"2024-09-14T04:55:14.368518+0000","last_undegraded":"2024-09-14T04:55:14.368518+0000","last_fullsized":"2024-09-14T04:54:19.444266+0000","mapping_epoch":1489,"log_start":"1474'2357","ondisk_log_start":"1474'2357","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8122,"log_dups_size":0,"ondisk_log_size":8122,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:53.013009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":230752256,"num_objects":3521,"num_object_clones":0,"num_object_copies":10563,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1483,"num_objects_unfound":0,"num_objects_dirty":3521,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10479,"num_write_kb":448000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1946,"num_bytes_recovered":127533056,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"4,5,7","objects":1483},{"shards":"5,7","objects":2038}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.5","version":"1508'10674","reported_seq":10722,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:13.707690+0000","last_change":"2024-09-14T04:52:16.871791+0000","last_active":"2024-09-14T04:55:13.707690+0000","last_peered":"2024-09-14T04:55:13.707690+0000","last_clean":"2024-09-14T04:55:13.707690+0000","last_became_active":"2024-09-14T04:52:16.871407+0000","last_became_peered":"2024-09-14T04:52:16.871407+0000","last_unstale":"2024-09-14T04:55:13.707690+0000","last_undegraded":"2024-09-14T04:55:13.707690+0000","last_fullsized":"2024-09-14T04:55:13.707690+0000","mapping_epoch":1470,"log_start":"1474'3267","ondisk_log_start":"1474'3267","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":7407,"log_dups_size":0,"ondisk_log_size":7407,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:47:15.419674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241565696,"num_objects":3686,"num_object_clones":0,"num_object_copies":11058,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3686,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10674,"num_write_kb":459520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"56.4","version":"1508'10798","reported_seq":10845,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:13.217691+0000","last_change":"2024-09-14T04:52:16.724077+0000","last_active":"2024-09-14T04:55:13.217691+0000","last_peered":"2024-09-14T04:55:13.217691+0000","last_clean":"2024-09-14T04:55:13.217691+0000","last_became_active":"2024-09-14T04:52:16.723597+0000","last_became_peered":"2024-09-14T04:52:16.723597+0000","last_unstale":"2024-09-14T04:55:13.217691+0000","last_undegraded":"2024-09-14T04:55:13.217691+0000","last_fullsized":"2024-09-14T04:55:13.217691+0000","mapping_epoch":1470,"log_start":"1478'3800","ondisk_log_start":"1478'3800","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":6998,"log_dups_size":0,"ondisk_log_size":6998,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:23:49.557180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242089984,"num_objects":3694,"num_object_clones":0,"num_object_copies":11082,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3694,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10798,"num_write_kb":463744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"56.3","version":"1508'10580","reported_seq":17847,"reported_epoch":1508,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:55:14.356730+0000","last_change":"2024-09-14T04:54:53.717396+0000","last_active":"2024-09-14T04:55:14.356730+0000","last_peered":"2024-09-14T04:55:14.356730+0000","last_clean":"2024-09-14T04:54:51.200599+0000","last_became_active":"2024-09-14T04:54:53.681745+0000","last_became_peered":"2024-09-14T04:54:53.681745+0000","last_unstale":"2024-09-14T04:55:14.356730+0000","last_undegraded":"2024-09-14T04:55:14.356730+0000","last_fullsized":"2024-09-14T04:55:14.356730+0000","mapping_epoch":1504,"log_start":"1474'2817","ondisk_log_start":"1474'2817","created":1470,"last_epoch_clean":1496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":7763,"log_dups_size":0,"ondisk_log_size":7763,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236978176,"num_objects":3616,"num_object_clones":0,"num_object_copies":10848,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3616,"num_objects_unfound":0,"num_objects_dirty":3616,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10580,"num_write_kb":454272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3597,"num_bytes_recovered":235732992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[1,6,5],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":3616}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"56.2","version":"1508'10838","reported_seq":16440,"reported_epoch":1508,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-14T04:55:14.358782+0000","last_change":"2024-09-14T04:54:19.618218+0000","last_active":"2024-09-14T04:55:14.358782+0000","last_peered":"2024-09-14T04:55:14.358782+0000","last_clean":"2024-09-14T04:52:58.403685+0000","last_became_active":"2024-09-14T04:54:19.593740+0000","last_became_peered":"2024-09-14T04:54:19.593740+0000","last_unstale":"2024-09-14T04:55:14.358782+0000","last_undegraded":"2024-09-14T04:54:19.466599+0000","last_fullsized":"2024-09-14T04:54:19.444256+0000","mapping_epoch":1489,"log_start":"1474'2637","ondisk_log_start":"1474'2637","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8201,"log_dups_size":0,"ondisk_log_size":8201,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:45:11.467642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241827840,"num_objects":3690,"num_object_clones":0,"num_object_copies":11070,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4349,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3690,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10838,"num_write_kb":464896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2770,"num_bytes_recovered":181534720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"0,5,7","objects":3551},{"shards":"5,7","objects":139}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.1","version":"1508'10694","reported_seq":10741,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:14.353617+0000","last_change":"2024-09-14T04:52:16.872703+0000","last_active":"2024-09-14T04:55:14.353617+0000","last_peered":"2024-09-14T04:55:14.353617+0000","last_clean":"2024-09-14T04:55:14.353617+0000","last_became_active":"2024-09-14T04:52:16.871779+0000","last_became_peered":"2024-09-14T04:52:16.871779+0000","last_unstale":"2024-09-14T04:55:14.353617+0000","last_undegraded":"2024-09-14T04:55:14.353617+0000","last_fullsized":"2024-09-14T04:55:14.353617+0000","mapping_epoch":1470,"log_start":"1474'2565","ondisk_log_start":"1474'2565","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8129,"log_dups_size":0,"ondisk_log_size":8129,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:23:30.260775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243138560,"num_objects":3710,"num_object_clones":0,"num_object_copies":11130,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3710,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10694,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.0","version":"1508'10549","reported_seq":10597,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:14.353481+0000","last_change":"2024-09-14T04:52:16.915325+0000","last_active":"2024-09-14T04:55:14.353481+0000","last_peered":"2024-09-14T04:55:14.353481+0000","last_clean":"2024-09-14T04:55:14.353481+0000","last_became_active":"2024-09-14T04:52:16.914980+0000","last_became_peered":"2024-09-14T04:52:16.914980+0000","last_unstale":"2024-09-14T04:55:14.353481+0000","last_undegraded":"2024-09-14T04:55:14.353481+0000","last_fullsized":"2024-09-14T04:55:14.353481+0000","mapping_epoch":1470,"log_start":"1474'2378","ondisk_log_start":"1474'2378","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8171,"log_dups_size":0,"ondisk_log_size":8171,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:55:08.752960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245170176,"num_objects":3741,"num_object_clones":0,"num_object_copies":11223,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3741,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10549,"num_write_kb":457280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":2308,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:08.338589+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T04:55:08.338589+0000","last_peered":"2024-09-14T04:55:08.338589+0000","last_clean":"2024-09-14T04:55:08.338589+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T04:55:08.338589+0000","last_undegraded":"2024-09-14T04:55:08.338589+0000","last_fullsized":"2024-09-14T04:55:08.338589+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":120,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.260811+0000","last_change":"2024-09-14T04:44:57.325109+0000","last_active":"2024-09-14T04:54:55.260811+0000","last_peered":"2024-09-14T04:54:55.260811+0000","last_clean":"2024-09-14T04:54:55.260811+0000","last_became_active":"2024-09-14T04:44:57.324791+0000","last_became_peered":"2024-09-14T04:44:57.324791+0000","last_unstale":"2024-09-14T04:54:55.260811+0000","last_undegraded":"2024-09-14T04:54:55.260811+0000","last_fullsized":"2024-09-14T04:54:55.260811+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:03.593275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":89,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.266221+0000","last_change":"2024-09-14T04:49:29.628435+0000","last_active":"2024-09-14T04:54:55.266221+0000","last_peered":"2024-09-14T04:54:55.266221+0000","last_clean":"2024-09-14T04:54:55.266221+0000","last_became_active":"2024-09-14T04:49:29.628081+0000","last_became_peered":"2024-09-14T04:49:29.628081+0000","last_unstale":"2024-09-14T04:54:55.266221+0000","last_undegraded":"2024-09-14T04:54:55.266221+0000","last_fullsized":"2024-09-14T04:54:55.266221+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:01.520334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":929,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:08.345302+0000","last_change":"2024-09-14T04:43:13.355281+0000","last_active":"2024-09-14T04:55:08.345302+0000","last_peered":"2024-09-14T04:55:08.345302+0000","last_clean":"2024-09-14T04:55:08.345302+0000","last_became_active":"2024-09-14T04:43:13.354967+0000","last_became_peered":"2024-09-14T04:43:13.354967+0000","last_unstale":"2024-09-14T04:55:08.345302+0000","last_undegraded":"2024-09-14T04:55:08.345302+0000","last_fullsized":"2024-09-14T04:55:08.345302+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:23:19.332075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1498,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.267934+0000","last_change":"2024-09-14T04:44:57.342918+0000","last_active":"2024-09-14T04:54:55.267934+0000","last_peered":"2024-09-14T04:54:55.267934+0000","last_clean":"2024-09-14T04:54:55.267934+0000","last_became_active":"2024-09-14T04:44:57.329550+0000","last_became_peered":"2024-09-14T04:44:57.329550+0000","last_unstale":"2024-09-14T04:54:55.267934+0000","last_undegraded":"2024-09-14T04:54:55.267934+0000","last_fullsized":"2024-09-14T04:54:55.267934+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:52:52.435893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":120,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:58.222329+0000","last_change":"2024-09-14T04:44:57.325537+0000","last_active":"2024-09-14T04:54:58.222329+0000","last_peered":"2024-09-14T04:54:58.222329+0000","last_clean":"2024-09-14T04:54:58.222329+0000","last_became_active":"2024-09-14T04:44:57.325057+0000","last_became_peered":"2024-09-14T04:44:57.325057+0000","last_unstale":"2024-09-14T04:54:58.222329+0000","last_undegraded":"2024-09-14T04:54:58.222329+0000","last_fullsized":"2024-09-14T04:54:58.222329+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:43:23.166708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1513,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.268626+0000","last_change":"2024-09-14T04:44:57.337033+0000","last_active":"2024-09-14T04:54:55.268626+0000","last_peered":"2024-09-14T04:54:55.268626+0000","last_clean":"2024-09-14T04:54:55.268626+0000","last_became_active":"2024-09-14T04:44:57.328437+0000","last_became_peered":"2024-09-14T04:44:57.328437+0000","last_unstale":"2024-09-14T04:54:55.268626+0000","last_undegraded":"2024-09-14T04:54:55.268626+0000","last_fullsized":"2024-09-14T04:54:55.268626+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.109086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":120,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:58.222352+0000","last_change":"2024-09-14T04:44:57.332243+0000","last_active":"2024-09-14T04:54:58.222352+0000","last_peered":"2024-09-14T04:54:58.222352+0000","last_clean":"2024-09-14T04:54:58.222352+0000","last_became_active":"2024-09-14T04:44:57.330413+0000","last_became_peered":"2024-09-14T04:44:57.330413+0000","last_unstale":"2024-09-14T04:54:58.222352+0000","last_undegraded":"2024-09-14T04:54:58.222352+0000","last_fullsized":"2024-09-14T04:54:58.222352+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:25:01.338549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":164,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:08.335588+0000","last_change":"2024-09-14T04:48:29.862929+0000","last_active":"2024-09-14T04:55:08.335588+0000","last_peered":"2024-09-14T04:55:08.335588+0000","last_clean":"2024-09-14T04:55:08.335588+0000","last_became_active":"2024-09-14T04:48:29.862581+0000","last_became_peered":"2024-09-14T04:48:29.862581+0000","last_unstale":"2024-09-14T04:55:08.335588+0000","last_undegraded":"2024-09-14T04:55:08.335588+0000","last_fullsized":"2024-09-14T04:55:08.335588+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:16:43.060088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2304,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:08.337300+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T04:55:08.337300+0000","last_peered":"2024-09-14T04:55:08.337300+0000","last_clean":"2024-09-14T04:55:08.337300+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T04:55:08.337300+0000","last_undegraded":"2024-09-14T04:55:08.337300+0000","last_fullsized":"2024-09-14T04:55:08.337300+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":120,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:58.222309+0000","last_change":"2024-09-14T04:44:57.325392+0000","last_active":"2024-09-14T04:54:58.222309+0000","last_peered":"2024-09-14T04:54:58.222309+0000","last_clean":"2024-09-14T04:54:58.222309+0000","last_became_active":"2024-09-14T04:44:57.324624+0000","last_became_peered":"2024-09-14T04:44:57.324624+0000","last_unstale":"2024-09-14T04:54:58.222309+0000","last_undegraded":"2024-09-14T04:54:58.222309+0000","last_fullsized":"2024-09-14T04:54:58.222309+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:33:18.810199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2272,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:08.332555+0000","last_change":"2024-09-14T04:44:57.329113+0000","last_active":"2024-09-14T04:55:08.332555+0000","last_peered":"2024-09-14T04:55:08.332555+0000","last_clean":"2024-09-14T04:55:08.332555+0000","last_became_active":"2024-09-14T04:44:57.328727+0000","last_became_peered":"2024-09-14T04:44:57.328727+0000","last_unstale":"2024-09-14T04:55:08.332555+0000","last_undegraded":"2024-09-14T04:55:08.332555+0000","last_fullsized":"2024-09-14T04:55:08.332555+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:34.939874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":2289,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:08.332414+0000","last_change":"2024-09-14T04:44:57.329277+0000","last_active":"2024-09-14T04:55:08.332414+0000","last_peered":"2024-09-14T04:55:08.332414+0000","last_clean":"2024-09-14T04:55:08.332414+0000","last_became_active":"2024-09-14T04:44:57.328804+0000","last_became_peered":"2024-09-14T04:44:57.328804+0000","last_unstale":"2024-09-14T04:55:08.332414+0000","last_undegraded":"2024-09-14T04:55:08.332414+0000","last_fullsized":"2024-09-14T04:55:08.332414+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:42:46.055240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2324,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:08.338555+0000","last_change":"2024-09-14T04:54:52.250695+0000","last_active":"2024-09-14T04:55:08.338555+0000","last_peered":"2024-09-14T04:55:08.338555+0000","last_clean":"2024-09-14T04:55:08.338555+0000","last_became_active":"2024-09-14T04:54:52.250301+0000","last_became_peered":"2024-09-14T04:54:52.250301+0000","last_unstale":"2024-09-14T04:55:08.338555+0000","last_undegraded":"2024-09-14T04:55:08.338555+0000","last_fullsized":"2024-09-14T04:55:08.338555+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:42.060905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2285,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:08.336022+0000","last_change":"2024-09-14T04:43:13.357092+0000","last_active":"2024-09-14T04:55:08.336022+0000","last_peered":"2024-09-14T04:55:08.336022+0000","last_clean":"2024-09-14T04:55:08.336022+0000","last_became_active":"2024-09-14T04:43:13.356694+0000","last_became_peered":"2024-09-14T04:43:13.356694+0000","last_unstale":"2024-09-14T04:55:08.336022+0000","last_undegraded":"2024-09-14T04:55:08.336022+0000","last_fullsized":"2024-09-14T04:55:08.336022+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:11.167574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1491,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.266451+0000","last_change":"2024-09-14T04:07:21.720797+0000","last_active":"2024-09-14T04:54:55.266451+0000","last_peered":"2024-09-14T04:54:55.266451+0000","last_clean":"2024-09-14T04:54:55.266451+0000","last_became_active":"2024-09-14T04:07:21.719331+0000","last_became_peered":"2024-09-14T04:07:21.719331+0000","last_unstale":"2024-09-14T04:54:55.266451+0000","last_undegraded":"2024-09-14T04:54:55.266451+0000","last_fullsized":"2024-09-14T04:54:55.266451+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:35.404023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":139,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.268484+0000","last_change":"2024-09-14T04:43:11.641514+0000","last_active":"2024-09-14T04:54:55.268484+0000","last_peered":"2024-09-14T04:54:55.268484+0000","last_clean":"2024-09-14T04:54:55.268484+0000","last_became_active":"2024-09-14T04:43:11.641151+0000","last_became_peered":"2024-09-14T04:43:11.641151+0000","last_unstale":"2024-09-14T04:54:55.268484+0000","last_undegraded":"2024-09-14T04:54:55.268484+0000","last_fullsized":"2024-09-14T04:54:55.268484+0000","mapping_epoch":1384,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:06:45.045430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2297,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:08.338823+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T04:55:08.338823+0000","last_peered":"2024-09-14T04:55:08.338823+0000","last_clean":"2024-09-14T04:55:08.338823+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T04:55:08.338823+0000","last_undegraded":"2024-09-14T04:55:08.338823+0000","last_fullsized":"2024-09-14T04:55:08.338823+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":719,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.269679+0000","last_change":"2024-09-14T04:49:29.618477+0000","last_active":"2024-09-14T04:54:55.269679+0000","last_peered":"2024-09-14T04:54:55.269679+0000","last_clean":"2024-09-14T04:54:55.269679+0000","last_became_active":"2024-09-14T04:49:29.618210+0000","last_became_peered":"2024-09-14T04:49:29.618210+0000","last_unstale":"2024-09-14T04:54:55.269679+0000","last_undegraded":"2024-09-14T04:54:55.269679+0000","last_fullsized":"2024-09-14T04:54:55.269679+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:13:23.895512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1497,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.272340+0000","last_change":"2024-09-14T04:42:49.938505+0000","last_active":"2024-09-14T04:54:55.272340+0000","last_peered":"2024-09-14T04:54:55.272340+0000","last_clean":"2024-09-14T04:54:55.272340+0000","last_became_active":"2024-09-14T04:42:49.938180+0000","last_became_peered":"2024-09-14T04:42:49.938180+0000","last_unstale":"2024-09-14T04:54:55.272340+0000","last_undegraded":"2024-09-14T04:54:55.272340+0000","last_fullsized":"2024-09-14T04:54:55.272340+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:05.929479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1516,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.269727+0000","last_change":"2024-09-14T04:44:57.346582+0000","last_active":"2024-09-14T04:54:55.269727+0000","last_peered":"2024-09-14T04:54:55.269727+0000","last_clean":"2024-09-14T04:54:55.269727+0000","last_became_active":"2024-09-14T04:44:57.330446+0000","last_became_peered":"2024-09-14T04:44:57.330446+0000","last_unstale":"2024-09-14T04:54:55.269727+0000","last_undegraded":"2024-09-14T04:54:55.269727+0000","last_fullsized":"2024-09-14T04:54:55.269727+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:29.238585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":238,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:08.333191+0000","last_change":"2024-09-14T04:44:14.480667+0000","last_active":"2024-09-14T04:55:08.333191+0000","last_peered":"2024-09-14T04:55:08.333191+0000","last_clean":"2024-09-14T04:55:08.333191+0000","last_became_active":"2024-09-14T04:44:14.480365+0000","last_became_peered":"2024-09-14T04:44:14.480365+0000","last_unstale":"2024-09-14T04:55:08.333191+0000","last_undegraded":"2024-09-14T04:55:08.333191+0000","last_fullsized":"2024-09-14T04:55:08.333191+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:00.611771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":130,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.270626+0000","last_change":"2024-09-14T04:44:57.341426+0000","last_active":"2024-09-14T04:54:55.270626+0000","last_peered":"2024-09-14T04:54:55.270626+0000","last_clean":"2024-09-14T04:54:55.270626+0000","last_became_active":"2024-09-14T04:44:57.328763+0000","last_became_peered":"2024-09-14T04:44:57.328763+0000","last_unstale":"2024-09-14T04:54:55.270626+0000","last_undegraded":"2024-09-14T04:54:55.270626+0000","last_fullsized":"2024-09-14T04:54:55.270626+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:34:40.066761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":120,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.265512+0000","last_change":"2024-09-14T04:44:57.331337+0000","last_active":"2024-09-14T04:54:55.265512+0000","last_peered":"2024-09-14T04:54:55.265512+0000","last_clean":"2024-09-14T04:54:55.265512+0000","last_became_active":"2024-09-14T04:44:57.329336+0000","last_became_peered":"2024-09-14T04:44:57.329336+0000","last_unstale":"2024-09-14T04:54:55.265512+0000","last_undegraded":"2024-09-14T04:54:55.265512+0000","last_fullsized":"2024-09-14T04:54:55.265512+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:52:35.980869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"349'149","reported_seq":708,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.268491+0000","last_change":"2024-09-14T04:49:30.085274+0000","last_active":"2024-09-14T04:54:55.268491+0000","last_peered":"2024-09-14T04:54:55.268491+0000","last_clean":"2024-09-14T04:54:55.268491+0000","last_became_active":"2024-09-14T04:49:29.623295+0000","last_became_peered":"2024-09-14T04:49:29.623295+0000","last_unstale":"2024-09-14T04:54:55.268491+0000","last_undegraded":"2024-09-14T04:54:55.268491+0000","last_fullsized":"2024-09-14T04:54:55.268491+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:49:30.085089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:49:30.085089+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:10.012963+0000","scrub_duration":0.0071928790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":120,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.260689+0000","last_change":"2024-09-14T04:44:57.329870+0000","last_active":"2024-09-14T04:54:55.260689+0000","last_peered":"2024-09-14T04:54:55.260689+0000","last_clean":"2024-09-14T04:54:55.260689+0000","last_became_active":"2024-09-14T04:44:57.329353+0000","last_became_peered":"2024-09-14T04:44:57.329353+0000","last_unstale":"2024-09-14T04:54:55.260689+0000","last_undegraded":"2024-09-14T04:54:55.260689+0000","last_fullsized":"2024-09-14T04:54:55.260689+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:46.432181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1513,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.266530+0000","last_change":"2024-09-14T04:54:52.242207+0000","last_active":"2024-09-14T04:54:55.266530+0000","last_peered":"2024-09-14T04:54:55.266530+0000","last_clean":"2024-09-14T04:54:55.266530+0000","last_became_active":"2024-09-14T04:54:52.241840+0000","last_became_peered":"2024-09-14T04:54:52.241840+0000","last_unstale":"2024-09-14T04:54:55.266530+0000","last_undegraded":"2024-09-14T04:54:55.266530+0000","last_fullsized":"2024-09-14T04:54:55.266530+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:37.899666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":2335,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:08.338444+0000","last_change":"2024-09-14T04:44:57.344614+0000","last_active":"2024-09-14T04:55:08.338444+0000","last_peered":"2024-09-14T04:55:08.338444+0000","last_clean":"2024-09-14T04:55:08.338444+0000","last_became_active":"2024-09-14T04:44:57.332256+0000","last_became_peered":"2024-09-14T04:44:57.332256+0000","last_unstale":"2024-09-14T04:55:08.338444+0000","last_undegraded":"2024-09-14T04:55:08.338444+0000","last_fullsized":"2024-09-14T04:55:08.338444+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:23.438417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"349'149","reported_seq":878,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.266604+0000","last_change":"2024-09-14T04:45:00.331854+0000","last_active":"2024-09-14T04:54:55.266604+0000","last_peered":"2024-09-14T04:54:55.266604+0000","last_clean":"2024-09-14T04:54:55.266604+0000","last_became_active":"2024-09-14T04:45:00.331460+0000","last_became_peered":"2024-09-14T04:45:00.331460+0000","last_unstale":"2024-09-14T04:54:55.266604+0000","last_undegraded":"2024-09-14T04:54:55.266604+0000","last_fullsized":"2024-09-14T04:54:55.266604+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.567535+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.567535+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:45.095729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":709,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.269327+0000","last_change":"2024-09-14T04:49:29.616831+0000","last_active":"2024-09-14T04:54:55.269327+0000","last_peered":"2024-09-14T04:54:55.269327+0000","last_clean":"2024-09-14T04:54:55.269327+0000","last_became_active":"2024-09-14T04:49:29.616546+0000","last_became_peered":"2024-09-14T04:49:29.616546+0000","last_unstale":"2024-09-14T04:54:55.269327+0000","last_undegraded":"2024-09-14T04:54:55.269327+0000","last_fullsized":"2024-09-14T04:54:55.269327+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:27.882929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":290,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:08.334008+0000","last_change":"2024-09-14T04:44:57.330746+0000","last_active":"2024-09-14T04:55:08.334008+0000","last_peered":"2024-09-14T04:55:08.334008+0000","last_clean":"2024-09-14T04:55:08.334008+0000","last_became_active":"2024-09-14T04:44:57.330327+0000","last_became_peered":"2024-09-14T04:44:57.330327+0000","last_unstale":"2024-09-14T04:55:08.334008+0000","last_undegraded":"2024-09-14T04:55:08.334008+0000","last_fullsized":"2024-09-14T04:55:08.334008+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:53.122771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":160,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.270417+0000","last_change":"2024-09-14T04:43:13.355296+0000","last_active":"2024-09-14T04:54:55.270417+0000","last_peered":"2024-09-14T04:54:55.270417+0000","last_clean":"2024-09-14T04:54:55.270417+0000","last_became_active":"2024-09-14T04:43:13.354950+0000","last_became_peered":"2024-09-14T04:43:13.354950+0000","last_unstale":"2024-09-14T04:54:55.270417+0000","last_undegraded":"2024-09-14T04:54:55.270417+0000","last_fullsized":"2024-09-14T04:54:55.270417+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:20:08.968818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"349'149","reported_seq":817,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.266075+0000","last_change":"2024-09-14T04:41:39.471393+0000","last_active":"2024-09-14T04:54:55.266075+0000","last_peered":"2024-09-14T04:54:55.266075+0000","last_clean":"2024-09-14T04:54:55.266075+0000","last_became_active":"2024-09-14T04:41:36.931768+0000","last_became_peered":"2024-09-14T04:41:36.931768+0000","last_unstale":"2024-09-14T04:54:55.266075+0000","last_undegraded":"2024-09-14T04:54:55.266075+0000","last_fullsized":"2024-09-14T04:54:55.266075+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.471248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.471248+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:56.210375+0000","scrub_duration":0.021758344999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2291,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:08.338363+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T04:55:08.338363+0000","last_peered":"2024-09-14T04:55:08.338363+0000","last_clean":"2024-09-14T04:55:08.338363+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T04:55:08.338363+0000","last_undegraded":"2024-09-14T04:55:08.338363+0000","last_fullsized":"2024-09-14T04:55:08.338363+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1527,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.269936+0000","last_change":"2024-09-14T04:44:57.331436+0000","last_active":"2024-09-14T04:54:55.269936+0000","last_peered":"2024-09-14T04:54:55.269936+0000","last_clean":"2024-09-14T04:54:55.269936+0000","last_became_active":"2024-09-14T04:44:57.330957+0000","last_became_peered":"2024-09-14T04:44:57.330957+0000","last_unstale":"2024-09-14T04:54:55.269936+0000","last_undegraded":"2024-09-14T04:54:55.269936+0000","last_fullsized":"2024-09-14T04:54:55.269936+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:09.516688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":840,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:08.337301+0000","last_change":"2024-09-14T04:49:49.460401+0000","last_active":"2024-09-14T04:55:08.337301+0000","last_peered":"2024-09-14T04:55:08.337301+0000","last_clean":"2024-09-14T04:55:08.337301+0000","last_became_active":"2024-09-14T04:49:49.458530+0000","last_became_peered":"2024-09-14T04:49:49.458530+0000","last_unstale":"2024-09-14T04:55:08.337301+0000","last_undegraded":"2024-09-14T04:55:08.337301+0000","last_fullsized":"2024-09-14T04:55:08.337301+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:11:29.143239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"349'149","reported_seq":851,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.266376+0000","last_change":"2024-09-14T04:54:22.147441+0000","last_active":"2024-09-14T04:54:55.266376+0000","last_peered":"2024-09-14T04:54:55.266376+0000","last_clean":"2024-09-14T04:54:55.266376+0000","last_became_active":"2024-09-14T04:54:22.147115+0000","last_became_peered":"2024-09-14T04:54:22.147115+0000","last_unstale":"2024-09-14T04:54:55.266376+0000","last_undegraded":"2024-09-14T04:54:55.266376+0000","last_fullsized":"2024-09-14T04:54:55.266376+0000","mapping_epoch":1492,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1493,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:38.479887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:38.479887+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:41:14.001781+0000","scrub_duration":0.0062728319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":246,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:08.333107+0000","last_change":"2024-09-14T04:44:57.329983+0000","last_active":"2024-09-14T04:55:08.333107+0000","last_peered":"2024-09-14T04:55:08.333107+0000","last_clean":"2024-09-14T04:55:08.333107+0000","last_became_active":"2024-09-14T04:44:57.329597+0000","last_became_peered":"2024-09-14T04:44:57.329597+0000","last_unstale":"2024-09-14T04:55:08.333107+0000","last_undegraded":"2024-09-14T04:55:08.333107+0000","last_fullsized":"2024-09-14T04:55:08.333107+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:45.718039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":784,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:08.342118+0000","last_change":"2024-09-14T04:49:49.443631+0000","last_active":"2024-09-14T04:55:08.342118+0000","last_peered":"2024-09-14T04:55:08.342118+0000","last_clean":"2024-09-14T04:55:08.342118+0000","last_became_active":"2024-09-14T04:49:49.443313+0000","last_became_peered":"2024-09-14T04:49:49.443313+0000","last_unstale":"2024-09-14T04:55:08.342118+0000","last_undegraded":"2024-09-14T04:55:08.342118+0000","last_fullsized":"2024-09-14T04:55:08.342118+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:06:54.562292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1499,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.268065+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T04:54:55.268065+0000","last_peered":"2024-09-14T04:54:55.268065+0000","last_clean":"2024-09-14T04:54:55.268065+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T04:54:55.268065+0000","last_undegraded":"2024-09-14T04:54:55.268065+0000","last_fullsized":"2024-09-14T04:54:55.268065+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"349'149","reported_seq":809,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.275056+0000","last_change":"2024-09-14T04:41:39.012977+0000","last_active":"2024-09-14T04:54:55.275056+0000","last_peered":"2024-09-14T04:54:55.275056+0000","last_clean":"2024-09-14T04:54:55.275056+0000","last_became_active":"2024-09-14T04:41:36.932552+0000","last_became_peered":"2024-09-14T04:41:36.932552+0000","last_unstale":"2024-09-14T04:54:55.275056+0000","last_undegraded":"2024-09-14T04:54:55.275056+0000","last_fullsized":"2024-09-14T04:54:55.275056+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.012879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.012879+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:16:40.080262+0000","scrub_duration":0.0051721620000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":876,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:08.337402+0000","last_change":"2024-09-14T04:49:49.459590+0000","last_active":"2024-09-14T04:55:08.337402+0000","last_peered":"2024-09-14T04:55:08.337402+0000","last_clean":"2024-09-14T04:55:08.337402+0000","last_became_active":"2024-09-14T04:49:49.456263+0000","last_became_peered":"2024-09-14T04:49:49.456263+0000","last_unstale":"2024-09-14T04:55:08.337402+0000","last_undegraded":"2024-09-14T04:55:08.337402+0000","last_fullsized":"2024-09-14T04:55:08.337402+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:20:44.666805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":120,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.265443+0000","last_change":"2024-09-14T04:44:57.329652+0000","last_active":"2024-09-14T04:54:55.265443+0000","last_peered":"2024-09-14T04:54:55.265443+0000","last_clean":"2024-09-14T04:54:55.265443+0000","last_became_active":"2024-09-14T04:44:57.328888+0000","last_became_peered":"2024-09-14T04:44:57.328888+0000","last_unstale":"2024-09-14T04:54:55.265443+0000","last_undegraded":"2024-09-14T04:54:55.265443+0000","last_fullsized":"2024-09-14T04:54:55.265443+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:10.170068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1498,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.275322+0000","last_change":"2024-09-14T04:06:46.875340+0000","last_active":"2024-09-14T04:54:55.275322+0000","last_peered":"2024-09-14T04:54:55.275322+0000","last_clean":"2024-09-14T04:54:55.275322+0000","last_became_active":"2024-09-14T04:06:46.874552+0000","last_became_peered":"2024-09-14T04:06:46.874552+0000","last_unstale":"2024-09-14T04:54:55.275322+0000","last_undegraded":"2024-09-14T04:54:55.275322+0000","last_fullsized":"2024-09-14T04:54:55.275322+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:52.257357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"349'149","reported_seq":712,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:58.737838+0000","last_change":"2024-09-14T04:54:55.279666+0000","last_active":"2024-09-14T04:54:58.737838+0000","last_peered":"2024-09-14T04:54:58.737838+0000","last_clean":"2024-09-14T04:54:58.737838+0000","last_became_active":"2024-09-14T04:54:55.279259+0000","last_became_peered":"2024-09-14T04:54:55.279259+0000","last_unstale":"2024-09-14T04:54:58.737838+0000","last_undegraded":"2024-09-14T04:54:58.737838+0000","last_fullsized":"2024-09-14T04:54:58.737838+0000","mapping_epoch":1506,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1507,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:54:03.996025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:54:03.996025+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:40:53.332369+0000","scrub_duration":54,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":239,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:08.336872+0000","last_change":"2024-09-14T04:44:14.489235+0000","last_active":"2024-09-14T04:55:08.336872+0000","last_peered":"2024-09-14T04:55:08.336872+0000","last_clean":"2024-09-14T04:55:08.336872+0000","last_became_active":"2024-09-14T04:44:14.488950+0000","last_became_peered":"2024-09-14T04:44:14.488950+0000","last_unstale":"2024-09-14T04:55:08.336872+0000","last_undegraded":"2024-09-14T04:55:08.336872+0000","last_fullsized":"2024-09-14T04:55:08.336872+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:54.441144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":794,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:08.336597+0000","last_change":"2024-09-14T04:49:49.456317+0000","last_active":"2024-09-14T04:55:08.336597+0000","last_peered":"2024-09-14T04:55:08.336597+0000","last_clean":"2024-09-14T04:55:08.336597+0000","last_became_active":"2024-09-14T04:49:49.456029+0000","last_became_peered":"2024-09-14T04:49:49.456029+0000","last_unstale":"2024-09-14T04:55:08.336597+0000","last_undegraded":"2024-09-14T04:55:08.336597+0000","last_fullsized":"2024-09-14T04:55:08.336597+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:46:05.319676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2311,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:08.345388+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T04:55:08.345388+0000","last_peered":"2024-09-14T04:55:08.345388+0000","last_clean":"2024-09-14T04:55:08.345388+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T04:55:08.345388+0000","last_undegraded":"2024-09-14T04:55:08.345388+0000","last_fullsized":"2024-09-14T04:55:08.345388+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"349'149","reported_seq":703,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.266947+0000","last_change":"2024-09-14T04:50:22.002798+0000","last_active":"2024-09-14T04:54:55.266947+0000","last_peered":"2024-09-14T04:54:55.266947+0000","last_clean":"2024-09-14T04:54:55.266947+0000","last_became_active":"2024-09-14T04:49:49.448278+0000","last_became_peered":"2024-09-14T04:49:49.448278+0000","last_unstale":"2024-09-14T04:54:55.266947+0000","last_undegraded":"2024-09-14T04:54:55.266947+0000","last_fullsized":"2024-09-14T04:54:55.266947+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:50:22.002568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:50:22.002568+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:01:57.489223+0000","scrub_duration":0.010888326,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2312,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:08.338643+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T04:55:08.338643+0000","last_peered":"2024-09-14T04:55:08.338643+0000","last_clean":"2024-09-14T04:55:08.338643+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T04:55:08.338643+0000","last_undegraded":"2024-09-14T04:55:08.338643+0000","last_fullsized":"2024-09-14T04:55:08.338643+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2333,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:08.343151+0000","last_change":"2024-09-14T04:54:19.447103+0000","last_active":"2024-09-14T04:55:08.343151+0000","last_peered":"2024-09-14T04:55:08.343151+0000","last_clean":"2024-09-14T04:55:08.343151+0000","last_became_active":"2024-09-14T04:54:19.446748+0000","last_became_peered":"2024-09-14T04:54:19.446748+0000","last_unstale":"2024-09-14T04:55:08.343151+0000","last_undegraded":"2024-09-14T04:55:08.343151+0000","last_fullsized":"2024-09-14T04:55:08.343151+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:00:55.369263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1528,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.269508+0000","last_change":"2024-09-14T04:42:49.939790+0000","last_active":"2024-09-14T04:54:55.269508+0000","last_peered":"2024-09-14T04:54:55.269508+0000","last_clean":"2024-09-14T04:54:55.269508+0000","last_became_active":"2024-09-14T04:42:49.939591+0000","last_became_peered":"2024-09-14T04:42:49.939591+0000","last_unstale":"2024-09-14T04:54:55.269508+0000","last_undegraded":"2024-09-14T04:54:55.269508+0000","last_fullsized":"2024-09-14T04:54:55.269508+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:23:16.790593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1491,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.265974+0000","last_change":"2024-09-14T04:44:57.326008+0000","last_active":"2024-09-14T04:54:55.265974+0000","last_peered":"2024-09-14T04:54:55.265974+0000","last_clean":"2024-09-14T04:54:55.265974+0000","last_became_active":"2024-09-14T04:44:57.325617+0000","last_became_peered":"2024-09-14T04:44:57.325617+0000","last_unstale":"2024-09-14T04:54:55.265974+0000","last_undegraded":"2024-09-14T04:54:55.265974+0000","last_fullsized":"2024-09-14T04:54:55.265974+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:15.637508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":780,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.275420+0000","last_change":"2024-09-14T04:43:13.354846+0000","last_active":"2024-09-14T04:54:55.275420+0000","last_peered":"2024-09-14T04:54:55.275420+0000","last_clean":"2024-09-14T04:54:55.275420+0000","last_became_active":"2024-09-14T04:43:13.354515+0000","last_became_peered":"2024-09-14T04:43:13.354515+0000","last_unstale":"2024-09-14T04:54:55.275420+0000","last_undegraded":"2024-09-14T04:54:55.275420+0000","last_fullsized":"2024-09-14T04:54:55.275420+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:17.889341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1497,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.267860+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T04:54:55.267860+0000","last_peered":"2024-09-14T04:54:55.267860+0000","last_clean":"2024-09-14T04:54:55.267860+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T04:54:55.267860+0000","last_undegraded":"2024-09-14T04:54:55.267860+0000","last_fullsized":"2024-09-14T04:54:55.267860+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":1834,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.269243+0000","last_change":"2024-09-14T04:41:30.640651+0000","last_active":"2024-09-14T04:54:55.269243+0000","last_peered":"2024-09-14T04:54:55.269243+0000","last_clean":"2024-09-14T04:54:55.269243+0000","last_became_active":"2024-09-14T04:41:29.532466+0000","last_became_peered":"2024-09-14T04:41:29.532466+0000","last_unstale":"2024-09-14T04:54:55.269243+0000","last_undegraded":"2024-09-14T04:54:55.269243+0000","last_fullsized":"2024-09-14T04:54:55.269243+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":0,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:30.640520+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:30.640520+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:27.776724+0000","scrub_duration":0.0028653139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"349'149","reported_seq":809,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.275248+0000","last_change":"2024-09-14T04:41:39.978080+0000","last_active":"2024-09-14T04:54:55.275248+0000","last_peered":"2024-09-14T04:54:55.275248+0000","last_clean":"2024-09-14T04:54:55.275248+0000","last_became_active":"2024-09-14T04:41:36.929279+0000","last_became_peered":"2024-09-14T04:41:36.929279+0000","last_unstale":"2024-09-14T04:54:55.275248+0000","last_undegraded":"2024-09-14T04:54:55.275248+0000","last_fullsized":"2024-09-14T04:54:55.275248+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.977999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.977999+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:44:06.383827+0000","scrub_duration":0.00192122,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":173,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:08.341421+0000","last_change":"2024-09-14T04:48:11.040133+0000","last_active":"2024-09-14T04:55:08.341421+0000","last_peered":"2024-09-14T04:55:08.341421+0000","last_clean":"2024-09-14T04:55:08.341421+0000","last_became_active":"2024-09-14T04:48:11.039787+0000","last_became_peered":"2024-09-14T04:48:11.039787+0000","last_unstale":"2024-09-14T04:55:08.341421+0000","last_undegraded":"2024-09-14T04:55:08.341421+0000","last_fullsized":"2024-09-14T04:55:08.341421+0000","mapping_epoch":1428,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:53.520672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2159,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:08.332718+0000","last_change":"2024-09-14T04:49:29.616443+0000","last_active":"2024-09-14T04:55:08.332718+0000","last_peered":"2024-09-14T04:55:08.332718+0000","last_clean":"2024-09-14T04:55:08.332718+0000","last_became_active":"2024-09-14T04:49:29.616066+0000","last_became_peered":"2024-09-14T04:49:29.616066+0000","last_unstale":"2024-09-14T04:55:08.332718+0000","last_undegraded":"2024-09-14T04:55:08.332718+0000","last_fullsized":"2024-09-14T04:55:08.332718+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:09:15.897957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":120,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.265269+0000","last_change":"2024-09-14T04:44:57.326724+0000","last_active":"2024-09-14T04:54:55.265269+0000","last_peered":"2024-09-14T04:54:55.265269+0000","last_clean":"2024-09-14T04:54:55.265269+0000","last_became_active":"2024-09-14T04:44:57.326191+0000","last_became_peered":"2024-09-14T04:44:57.326191+0000","last_unstale":"2024-09-14T04:54:55.265269+0000","last_undegraded":"2024-09-14T04:54:55.265269+0000","last_fullsized":"2024-09-14T04:54:55.265269+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:46.060269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":120,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.260551+0000","last_change":"2024-09-14T04:44:57.328528+0000","last_active":"2024-09-14T04:54:55.260551+0000","last_peered":"2024-09-14T04:54:55.260551+0000","last_clean":"2024-09-14T04:54:55.260551+0000","last_became_active":"2024-09-14T04:44:57.327739+0000","last_became_peered":"2024-09-14T04:44:57.327739+0000","last_unstale":"2024-09-14T04:54:55.260551+0000","last_undegraded":"2024-09-14T04:54:55.260551+0000","last_fullsized":"2024-09-14T04:54:55.260551+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:04:58.294690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"349'149","reported_seq":834,"reported_epoch":1508,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-14T04:55:08.336489+0000","last_change":"2024-09-14T04:54:36.689611+0000","last_active":"2024-09-14T04:55:08.336489+0000","last_peered":"2024-09-14T04:55:08.336489+0000","last_clean":"2024-09-14T04:54:31.592636+0000","last_became_active":"2024-09-14T04:54:36.676136+0000","last_became_peered":"2024-09-14T04:54:36.676136+0000","last_unstale":"2024-09-14T04:55:08.336489+0000","last_undegraded":"2024-09-14T04:54:36.658607+0000","last_fullsized":"2024-09-14T04:54:36.658065+0000","mapping_epoch":1499,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1479,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:56.738636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:56.738636+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"no scrub is scheduled","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[2,3],"avail_no_missing":["2","3","6"],"object_location_counts":[{"shards":"2,3,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":120,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.266424+0000","last_change":"2024-09-14T04:44:57.323436+0000","last_active":"2024-09-14T04:54:55.266424+0000","last_peered":"2024-09-14T04:54:55.266424+0000","last_clean":"2024-09-14T04:54:55.266424+0000","last_became_active":"2024-09-14T04:44:57.323022+0000","last_became_peered":"2024-09-14T04:44:57.323022+0000","last_unstale":"2024-09-14T04:54:55.266424+0000","last_undegraded":"2024-09-14T04:54:55.266424+0000","last_fullsized":"2024-09-14T04:54:55.266424+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:22.775261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1526,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.266298+0000","last_change":"2024-09-14T04:44:57.322357+0000","last_active":"2024-09-14T04:54:55.266298+0000","last_peered":"2024-09-14T04:54:55.266298+0000","last_clean":"2024-09-14T04:54:55.266298+0000","last_became_active":"2024-09-14T04:44:57.321990+0000","last_became_peered":"2024-09-14T04:44:57.321990+0000","last_unstale":"2024-09-14T04:54:55.266298+0000","last_undegraded":"2024-09-14T04:54:55.266298+0000","last_fullsized":"2024-09-14T04:54:55.266298+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:09.658008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"349'149","reported_seq":1851,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.269603+0000","last_change":"2024-09-14T04:41:53.858076+0000","last_active":"2024-09-14T04:54:55.269603+0000","last_peered":"2024-09-14T04:54:55.269603+0000","last_clean":"2024-09-14T04:54:55.269603+0000","last_became_active":"2024-09-14T04:41:36.931585+0000","last_became_peered":"2024-09-14T04:41:36.931585+0000","last_unstale":"2024-09-14T04:54:55.269603+0000","last_undegraded":"2024-09-14T04:54:55.269603+0000","last_fullsized":"2024-09-14T04:54:55.269603+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:53.857924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:53.857924+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:10:20.658986+0000","scrub_duration":0.0022355919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":120,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.270427+0000","last_change":"2024-09-14T04:44:57.329198+0000","last_active":"2024-09-14T04:54:55.270427+0000","last_peered":"2024-09-14T04:54:55.270427+0000","last_clean":"2024-09-14T04:54:55.270427+0000","last_became_active":"2024-09-14T04:44:57.327754+0000","last_became_peered":"2024-09-14T04:44:57.327754+0000","last_unstale":"2024-09-14T04:54:55.270427+0000","last_undegraded":"2024-09-14T04:54:55.270427+0000","last_fullsized":"2024-09-14T04:54:55.270427+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:10:27.944476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":296,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:08.338451+0000","last_change":"2024-09-14T04:44:57.348687+0000","last_active":"2024-09-14T04:55:08.338451+0000","last_peered":"2024-09-14T04:55:08.338451+0000","last_clean":"2024-09-14T04:55:08.338451+0000","last_became_active":"2024-09-14T04:44:57.332845+0000","last_became_peered":"2024-09-14T04:44:57.332845+0000","last_unstale":"2024-09-14T04:55:08.338451+0000","last_undegraded":"2024-09-14T04:55:08.338451+0000","last_fullsized":"2024-09-14T04:55:08.338451+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:12:18.762737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2318,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:08.338196+0000","last_change":"2024-09-14T04:44:57.333468+0000","last_active":"2024-09-14T04:55:08.338196+0000","last_peered":"2024-09-14T04:55:08.338196+0000","last_clean":"2024-09-14T04:55:08.338196+0000","last_became_active":"2024-09-14T04:44:57.332626+0000","last_became_peered":"2024-09-14T04:44:57.332626+0000","last_unstale":"2024-09-14T04:55:08.338196+0000","last_undegraded":"2024-09-14T04:55:08.338196+0000","last_fullsized":"2024-09-14T04:55:08.338196+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:57:26.366438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1482,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.268757+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T04:54:55.268757+0000","last_peered":"2024-09-14T04:54:55.268757+0000","last_clean":"2024-09-14T04:54:55.268757+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T04:54:55.268757+0000","last_undegraded":"2024-09-14T04:54:55.268757+0000","last_fullsized":"2024-09-14T04:54:55.268757+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1481,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.266185+0000","last_change":"2024-09-14T04:06:46.869868+0000","last_active":"2024-09-14T04:54:55.266185+0000","last_peered":"2024-09-14T04:54:55.266185+0000","last_clean":"2024-09-14T04:54:55.266185+0000","last_became_active":"2024-09-14T04:06:46.869126+0000","last_became_peered":"2024-09-14T04:06:46.869126+0000","last_unstale":"2024-09-14T04:54:55.266185+0000","last_undegraded":"2024-09-14T04:54:55.266185+0000","last_fullsized":"2024-09-14T04:54:55.266185+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:41.150138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1465,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.269816+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T04:54:55.269816+0000","last_peered":"2024-09-14T04:54:55.269816+0000","last_clean":"2024-09-14T04:54:55.269816+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T04:54:55.269816+0000","last_undegraded":"2024-09-14T04:54:55.269816+0000","last_fullsized":"2024-09-14T04:54:55.269816+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":120,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.260830+0000","last_change":"2024-09-14T04:44:57.332112+0000","last_active":"2024-09-14T04:54:55.260830+0000","last_peered":"2024-09-14T04:54:55.260830+0000","last_clean":"2024-09-14T04:54:55.260830+0000","last_became_active":"2024-09-14T04:44:57.331575+0000","last_became_peered":"2024-09-14T04:44:57.331575+0000","last_unstale":"2024-09-14T04:54:55.260830+0000","last_undegraded":"2024-09-14T04:54:55.260830+0000","last_fullsized":"2024-09-14T04:54:55.260830+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:50:00.391283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1513,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.271148+0000","last_change":"2024-09-14T04:44:57.351460+0000","last_active":"2024-09-14T04:54:55.271148+0000","last_peered":"2024-09-14T04:54:55.271148+0000","last_clean":"2024-09-14T04:54:55.271148+0000","last_became_active":"2024-09-14T04:44:57.334662+0000","last_became_peered":"2024-09-14T04:44:57.334662+0000","last_unstale":"2024-09-14T04:54:55.271148+0000","last_undegraded":"2024-09-14T04:54:55.271148+0000","last_fullsized":"2024-09-14T04:54:55.271148+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:02.399062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1467,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.268556+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T04:54:55.268556+0000","last_peered":"2024-09-14T04:54:55.268556+0000","last_clean":"2024-09-14T04:54:55.268556+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T04:54:55.268556+0000","last_undegraded":"2024-09-14T04:54:55.268556+0000","last_fullsized":"2024-09-14T04:54:55.268556+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1525,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.270328+0000","last_change":"2024-09-14T04:44:57.349690+0000","last_active":"2024-09-14T04:54:55.270328+0000","last_peered":"2024-09-14T04:54:55.270328+0000","last_clean":"2024-09-14T04:54:55.270328+0000","last_became_active":"2024-09-14T04:44:57.333579+0000","last_became_peered":"2024-09-14T04:44:57.333579+0000","last_unstale":"2024-09-14T04:54:55.270328+0000","last_undegraded":"2024-09-14T04:54:55.270328+0000","last_fullsized":"2024-09-14T04:54:55.270328+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:17:55.265141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1602,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.265693+0000","last_change":"2024-09-14T04:54:35.694089+0000","last_active":"2024-09-14T04:54:55.265693+0000","last_peered":"2024-09-14T04:54:55.265693+0000","last_clean":"2024-09-14T04:54:55.265693+0000","last_became_active":"2024-09-14T04:54:35.663133+0000","last_became_peered":"2024-09-14T04:54:35.663133+0000","last_unstale":"2024-09-14T04:54:55.265693+0000","last_undegraded":"2024-09-14T04:54:55.265693+0000","last_fullsized":"2024-09-14T04:54:55.265693+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:17:19.947280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":15,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.269921+0000","last_change":"2024-09-14T04:54:52.248439+0000","last_active":"2024-09-14T04:54:55.269921+0000","last_peered":"2024-09-14T04:54:55.269921+0000","last_clean":"2024-09-14T04:54:55.269921+0000","last_became_active":"2024-09-14T04:54:52.248235+0000","last_became_peered":"2024-09-14T04:54:52.248235+0000","last_unstale":"2024-09-14T04:54:55.269921+0000","last_undegraded":"2024-09-14T04:54:55.269921+0000","last_fullsized":"2024-09-14T04:54:55.269921+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:22:09.548139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1487,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.269801+0000","last_change":"2024-09-14T04:44:57.332511+0000","last_active":"2024-09-14T04:54:55.269801+0000","last_peered":"2024-09-14T04:54:55.269801+0000","last_clean":"2024-09-14T04:54:55.269801+0000","last_became_active":"2024-09-14T04:44:57.332166+0000","last_became_peered":"2024-09-14T04:44:57.332166+0000","last_unstale":"2024-09-14T04:54:55.269801+0000","last_undegraded":"2024-09-14T04:54:55.269801+0000","last_fullsized":"2024-09-14T04:54:55.269801+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:53.984639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2312,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:08.338265+0000","last_change":"2024-09-14T04:54:52.250655+0000","last_active":"2024-09-14T04:55:08.338265+0000","last_peered":"2024-09-14T04:55:08.338265+0000","last_clean":"2024-09-14T04:55:08.338265+0000","last_became_active":"2024-09-14T04:54:52.250317+0000","last_became_peered":"2024-09-14T04:54:52.250317+0000","last_unstale":"2024-09-14T04:55:08.338265+0000","last_undegraded":"2024-09-14T04:55:08.338265+0000","last_fullsized":"2024-09-14T04:55:08.338265+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:12.985406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2298,"reported_epoch":1508,"state":"active+clean","last_fresh":"2024-09-14T04:55:08.335689+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T04:55:08.335689+0000","last_peered":"2024-09-14T04:55:08.335689+0000","last_clean":"2024-09-14T04:55:08.335689+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T04:55:08.335689+0000","last_undegraded":"2024-09-14T04:55:08.335689+0000","last_fullsized":"2024-09-14T04:55:08.335689+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1479,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.267999+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T04:54:55.267999+0000","last_peered":"2024-09-14T04:54:55.267999+0000","last_clean":"2024-09-14T04:54:55.267999+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T04:54:55.267999+0000","last_undegraded":"2024-09-14T04:54:55.267999+0000","last_fullsized":"2024-09-14T04:54:55.267999+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1516,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.270689+0000","last_change":"2024-09-14T04:44:57.339993+0000","last_active":"2024-09-14T04:54:55.270689+0000","last_peered":"2024-09-14T04:54:55.270689+0000","last_clean":"2024-09-14T04:54:55.270689+0000","last_became_active":"2024-09-14T04:44:57.330472+0000","last_became_peered":"2024-09-14T04:44:57.330472+0000","last_unstale":"2024-09-14T04:54:55.270689+0000","last_undegraded":"2024-09-14T04:54:55.270689+0000","last_fullsized":"2024-09-14T04:54:55.270689+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:19:27.046988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2064,"reported_epoch":1507,"state":"active+clean","last_fresh":"2024-09-14T04:54:55.269423+0000","last_change":"2024-09-14T04:49:49.449687+0000","last_active":"2024-09-14T04:54:55.269423+0000","last_peered":"2024-09-14T04:54:55.269423+0000","last_clean":"2024-09-14T04:54:55.269423+0000","last_became_active":"2024-09-14T04:49:49.449144+0000","last_became_peered":"2024-09-14T04:49:49.449144+0000","last_unstale":"2024-09-14T04:54:55.269423+0000","last_undegraded":"2024-09-14T04:54:55.269423+0000","last_fullsized":"2024-09-14T04:54:55.269423+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:59:06.779730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":56,"num_pg":16,"stat_sum":{"num_bytes":3852730392,"num_objects":58789,"num_object_clones":0,"num_object_copies":176367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4349,"num_objects_misplaced":5099,"num_objects_unfound":0,"num_objects_dirty":58789,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":170255,"num_write_kb":7329345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8313,"num_bytes_recovered":544800768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12029014016,"data_stored":12029001800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125409,"ondisk_log_size":125409,"up":48,"acting":46,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":32,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":5070,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4250,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223202051,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2295860,"kb_used_data":1615020,"kb_used_omap":1,"kb_used_meta":680766,"kb_avail":91449292,"statfs":{"total":95995035648,"available":93644075008,"internally_reserved":0,"allocated":1653780480,"data_stored":1648584834,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":697104843},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463687,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1595884,"kb_used_data":965156,"kb_used_omap":1,"kb_used_meta":630718,"kb_avail":92149268,"statfs":{"total":95995035648,"available":94360850432,"internally_reserved":0,"allocated":988319744,"data_stored":983132683,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":645855687},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725324,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2628000,"kb_used_data":1884888,"kb_used_omap":1,"kb_used_meta":743102,"kb_avail":91117152,"statfs":{"total":95995035648,"available":93303963648,"internally_reserved":0,"allocated":1930125312,"data_stored":1924945821,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":760936902},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,4,0,1,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":1392,"seq":5978594476170,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1925168,"kb_used_data":1667624,"kb_used_omap":117,"kb_used_meta":257418,"kb_avail":91819984,"statfs":{"total":95995035648,"available":94023663616,"internally_reserved":0,"allocated":1707646976,"data_stored":1702455954,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120638,"internal_metadata":263596226},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,2,6],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772075066,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1881188,"kb_used_data":1172956,"kb_used_omap":71,"kb_used_meta":708088,"kb_avail":91863964,"statfs":{"total":95995035648,"available":94068699136,"internally_reserved":0,"allocated":1201106944,"data_stored":1195923978,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":725083062},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,2,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707304001,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2122976,"kb_used_data":1639704,"kb_used_omap":68,"kb_used_meta":483195,"kb_avail":91622176,"statfs":{"total":95995035648,"available":93821108224,"internally_reserved":0,"allocated":1679056896,"data_stored":1673877906,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":494791888},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872729160,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1985432,"kb_used_data":1429968,"kb_used_omap":67,"kb_used_meta":555324,"kb_avail":91759720,"statfs":{"total":95995035648,"available":93961953280,"internally_reserved":0,"allocated":1464287232,"data_stored":1459115293,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":568652133},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056210,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2158624,"kb_used_data":1549208,"kb_used_omap":74,"kb_used_meta":609269,"kb_avail":91586528,"statfs":{"total":95995035648,"available":93784604672,"internally_reserved":0,"allocated":1586388992,"data_stored":1581209470,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":623892149},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,2,2,0,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1901,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1564545024,"data_stored":1564540952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1433468928,"data_stored":1433468928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1662713856,"data_stored":1662713856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1171001344,"data_stored":1170997272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1679491072,"data_stored":1679491072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1912799232,"data_stored":1912799232,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":965738496,"data_stored":965738496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1639256064,"data_stored":1639251992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:55:20.297 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph status --format=json 2024-09-14T04:55:20.333 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:20.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:20.334 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:20.435 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:20.482 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:20.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:20.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:20.532 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:55:20.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:20 smithi039 ceph-mon[106470]: from='client.46148 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:55:20.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:20 smithi039 ceph-mon[111235]: from='client.46148 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:55:20.584 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:20.613 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:20.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:20.614 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:20.714 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:20.749 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:20.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:20.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:20.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:20 smithi129 ceph-mon[101316]: from='client.46148 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:55:20.851 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:20.888 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:20.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:20.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:20.990 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:21.024 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:21.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:21.025 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:21.126 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:21.159 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:21.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:21.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:21.261 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:21.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:21.309 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:21.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:21.338 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:55:21.338 INFO:teuthology.orchestra.run.smithi039.stdout: "size": 20, 2024-09-14T04:55:21.338 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 600, 2024-09-14T04:55:21.338 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [ 2024-09-14T04:55:21.339 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.339 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45632.0:64534 55.1 55:81f19a9e:::benchmark_data_smithi039_297155_object64533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1437)", 2024-09-14T04:55:21.339 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:48:36.046591+0000", 2024-09-14T04:55:21.339 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 404.94848867000002, 2024-09-14T04:55:21.339 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0210583499999999, 2024-09-14T04:55:21.339 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:55:21.339 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:55:21.339 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T04:55:21.339 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:55:21.339 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45632", 2024-09-14T04:55:21.340 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:55:21.340 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 64534 2024-09-14T04:55:21.340 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.340 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:55:21.340 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.340 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:55:21.340 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046591+0000", 2024-09-14T04:55:21.340 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.340 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.340 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.340 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:55:21.340 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046591+0000", 2024-09-14T04:55:21.340 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.340 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.341 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.341 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:55:21.341 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046593+0000", 2024-09-14T04:55:21.341 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.917e-06 2024-09-14T04:55:21.341 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.341 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.341 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:55:21.341 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046616+0000", 2024-09-14T04:55:21.341 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3010000000000002e-05 2024-09-14T04:55:21.341 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.341 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.341 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:55:21.341 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046638+0000", 2024-09-14T04:55:21.342 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2059e-05 2024-09-14T04:55:21.342 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.342 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.342 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:55:21.342 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046660+0000", 2024-09-14T04:55:21.342 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1642e-05 2024-09-14T04:55:21.342 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.342 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.342 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:55:21.342 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046764+0000", 2024-09-14T04:55:21.342 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000104226 2024-09-14T04:55:21.342 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.342 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.343 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op must wait for map", 2024-09-14T04:55:21.343 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046772+0000", 2024-09-14T04:55:21.343 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7030000000000006e-06 2024-09-14T04:55:21.343 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.343 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.343 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:55:21.343 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.050944+0000", 2024-09-14T04:55:21.343 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0041725319999999996 2024-09-14T04:55:21.343 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.343 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.343 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:55:21.344 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.051119+0000", 2024-09-14T04:55:21.344 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000174458 2024-09-14T04:55:21.344 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.344 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.344 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:55:21.344 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.067560+0000", 2024-09-14T04:55:21.344 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0164416080000001 2024-09-14T04:55:21.344 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.344 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.344 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:55:21.344 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.067650+0000", 2024-09-14T04:55:21.344 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.9195000000000002e-05 2024-09-14T04:55:21.344 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.345 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:55:21.345 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.345 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.345 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.345 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45632.0:64543 55.1 55:84b0ffa6:::benchmark_data_smithi039_297155_object64542:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1437)", 2024-09-14T04:55:21.345 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:48:36.046692+0000", 2024-09-14T04:55:21.345 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 404.94838794399999, 2024-09-14T04:55:21.345 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0291501249999999, 2024-09-14T04:55:21.345 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:55:21.345 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:55:21.345 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:55:21.345 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:55:21.345 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45632", 2024-09-14T04:55:21.346 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:55:21.346 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 64543 2024-09-14T04:55:21.346 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.346 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:55:21.346 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.346 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:55:21.346 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046692+0000", 2024-09-14T04:55:21.346 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.346 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.346 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.346 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:55:21.346 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046692+0000", 2024-09-14T04:55:21.346 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.346 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.347 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.347 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:55:21.347 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046693+0000", 2024-09-14T04:55:21.347 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0900000000000001e-07 2024-09-14T04:55:21.347 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.347 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.348 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:55:21.348 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046714+0000", 2024-09-14T04:55:21.348 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1467999999999998e-05 2024-09-14T04:55:21.348 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.348 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.348 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:55:21.348 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046733+0000", 2024-09-14T04:55:21.348 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9341999999999999e-05 2024-09-14T04:55:21.349 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.349 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.349 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:55:21.349 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046743+0000", 2024-09-14T04:55:21.349 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.4080000000000004e-06 2024-09-14T04:55:21.349 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.349 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.349 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:55:21.349 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.047732+0000", 2024-09-14T04:55:21.349 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00098887600000000008 2024-09-14T04:55:21.349 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.349 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.350 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting_for_map not empty", 2024-09-14T04:55:21.350 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.047738+0000", 2024-09-14T04:55:21.350 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8079999999999999e-06 2024-09-14T04:55:21.350 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.350 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.350 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:55:21.350 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.051980+0000", 2024-09-14T04:55:21.350 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0042414929999999998 2024-09-14T04:55:21.350 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.350 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.350 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:55:21.350 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.051988+0000", 2024-09-14T04:55:21.351 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.3180000000000007e-06 2024-09-14T04:55:21.351 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.351 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.351 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:55:21.351 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.068072+0000", 2024-09-14T04:55:21.351 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0160837920000001 2024-09-14T04:55:21.351 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.351 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.351 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:55:21.351 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.068183+0000", 2024-09-14T04:55:21.351 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011097 2024-09-14T04:55:21.352 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.352 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.352 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:55:21.352 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.068290+0000", 2024-09-14T04:55:21.352 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000107227 2024-09-14T04:55:21.352 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.352 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.352 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.352 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.075604+0000", 2024-09-14T04:55:21.352 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0073141589999999998 2024-09-14T04:55:21.352 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.352 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.353 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:55:21.353 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.075707+0000", 2024-09-14T04:55:21.353 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000102948 2024-09-14T04:55:21.353 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.353 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.353 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.353 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.075759+0000", 2024-09-14T04:55:21.353 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1276999999999997e-05 2024-09-14T04:55:21.353 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.353 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.353 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:55:21.353 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.075815+0000", 2024-09-14T04:55:21.353 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5902000000000001e-05 2024-09-14T04:55:21.354 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.354 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.354 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:55:21.354 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.075842+0000", 2024-09-14T04:55:21.354 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.7528000000000001e-05 2024-09-14T04:55:21.354 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.354 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:55:21.354 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.354 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.354 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.354 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45632.0:64554 55.1 55:80cc57e1:::benchmark_data_smithi039_297155_object64553:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1437)", 2024-09-14T04:55:21.354 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:48:36.053143+0000", 2024-09-14T04:55:21.354 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 404.94193650300002, 2024-09-14T04:55:21.354 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0228815819999999, 2024-09-14T04:55:21.355 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:55:21.355 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:55:21.355 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:55:21.355 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:55:21.355 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45632", 2024-09-14T04:55:21.355 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:55:21.355 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 64554 2024-09-14T04:55:21.355 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.355 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:55:21.355 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.355 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:55:21.355 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.053143+0000", 2024-09-14T04:55:21.355 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.356 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.356 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.356 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:55:21.356 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.053143+0000", 2024-09-14T04:55:21.356 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.356 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.356 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.356 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:55:21.356 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.053145+0000", 2024-09-14T04:55:21.356 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.533e-06 2024-09-14T04:55:21.356 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.356 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.356 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:55:21.357 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.053169+0000", 2024-09-14T04:55:21.357 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4329e-05 2024-09-14T04:55:21.357 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.357 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.357 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:55:21.357 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.053180+0000", 2024-09-14T04:55:21.357 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0888e-05 2024-09-14T04:55:21.357 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.357 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.357 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:55:21.357 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.053188+0000", 2024-09-14T04:55:21.358 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6429999999999995e-06 2024-09-14T04:55:21.358 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.358 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.358 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:55:21.358 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.053797+0000", 2024-09-14T04:55:21.358 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00060973199999999998 2024-09-14T04:55:21.358 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.358 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.359 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:55:21.359 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.053833+0000", 2024-09-14T04:55:21.359 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.5256000000000003e-05 2024-09-14T04:55:21.359 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.359 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.359 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:55:21.359 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.070510+0000", 2024-09-14T04:55:21.359 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.016677018 2024-09-14T04:55:21.359 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.359 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.359 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:55:21.359 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.070645+0000", 2024-09-14T04:55:21.359 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000135358 2024-09-14T04:55:21.360 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.360 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.360 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:55:21.360 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.070801+0000", 2024-09-14T04:55:21.360 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015627499999999999 2024-09-14T04:55:21.360 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.360 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.360 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.360 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.075687+0000", 2024-09-14T04:55:21.360 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0048856430000000003 2024-09-14T04:55:21.360 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.360 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.361 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:55:21.361 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.075712+0000", 2024-09-14T04:55:21.361 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.5140999999999999e-05 2024-09-14T04:55:21.361 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.361 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.361 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.361 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.075915+0000", 2024-09-14T04:55:21.361 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000202919 2024-09-14T04:55:21.361 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.361 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.361 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:55:21.362 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.075980+0000", 2024-09-14T04:55:21.362 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.5073000000000005e-05 2024-09-14T04:55:21.362 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.362 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.362 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:55:21.362 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.076025+0000", 2024-09-14T04:55:21.362 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4774000000000002e-05 2024-09-14T04:55:21.362 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.362 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:55:21.362 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.363 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.363 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.363 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45632.0:64601 55.1 55:819f926e:::benchmark_data_smithi039_297155_object64600:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1437)", 2024-09-14T04:55:21.363 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:48:36.084827+0000", 2024-09-14T04:55:21.363 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 404.91025277599999, 2024-09-14T04:55:21.363 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.99141031400000001, 2024-09-14T04:55:21.363 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:55:21.363 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:55:21.363 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:55:21.363 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:55:21.363 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45632", 2024-09-14T04:55:21.363 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:55:21.364 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 64601 2024-09-14T04:55:21.364 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.364 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:55:21.364 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.364 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:55:21.364 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.084827+0000", 2024-09-14T04:55:21.364 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.364 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.364 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.364 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:55:21.364 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.084827+0000", 2024-09-14T04:55:21.364 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.364 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.365 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.365 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:55:21.365 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.084829+0000", 2024-09-14T04:55:21.365 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.742e-06 2024-09-14T04:55:21.365 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.365 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.365 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:55:21.365 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.084851+0000", 2024-09-14T04:55:21.365 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1676999999999999e-05 2024-09-14T04:55:21.365 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.365 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.365 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:55:21.365 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.084884+0000", 2024-09-14T04:55:21.366 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.3543000000000001e-05 2024-09-14T04:55:21.366 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.366 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.366 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:55:21.366 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.084893+0000", 2024-09-14T04:55:21.366 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.3889999999999994e-06 2024-09-14T04:55:21.366 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.366 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.366 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:55:21.366 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.084957+0000", 2024-09-14T04:55:21.366 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3336000000000005e-05 2024-09-14T04:55:21.366 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.367 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.367 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:55:21.367 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.084969+0000", 2024-09-14T04:55:21.367 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2411e-05 2024-09-14T04:55:21.367 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.367 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.367 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:55:21.367 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.073108+0000", 2024-09-14T04:55:21.367 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.98813857500000002 2024-09-14T04:55:21.367 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.367 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.367 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:55:21.368 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.073271+0000", 2024-09-14T04:55:21.368 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00016303199999999999 2024-09-14T04:55:21.368 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.368 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.368 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:55:21.368 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.073434+0000", 2024-09-14T04:55:21.368 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000162755 2024-09-14T04:55:21.369 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.369 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.369 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:55:21.369 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.075857+0000", 2024-09-14T04:55:21.369 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.002423209 2024-09-14T04:55:21.369 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.369 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.369 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.369 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.076080+0000", 2024-09-14T04:55:21.370 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00022328100000000001 2024-09-14T04:55:21.370 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.370 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.370 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.370 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.076130+0000", 2024-09-14T04:55:21.370 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9743000000000001e-05 2024-09-14T04:55:21.370 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.370 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.370 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:55:21.370 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.076192+0000", 2024-09-14T04:55:21.370 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1795e-05 2024-09-14T04:55:21.370 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.370 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.370 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:55:21.371 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.076237+0000", 2024-09-14T04:55:21.371 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5825999999999998e-05 2024-09-14T04:55:21.371 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.371 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:55:21.371 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.371 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.371 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.371 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45632.0:64662 55.1 55:80119e9b:::benchmark_data_smithi039_297155_object64661:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1437)", 2024-09-14T04:55:21.371 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:48:36.120270+0000", 2024-09-14T04:55:21.371 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 404.87481023399999, 2024-09-14T04:55:21.371 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.96407522300000004, 2024-09-14T04:55:21.372 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:55:21.372 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:55:21.372 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:55:21.372 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:55:21.372 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45632", 2024-09-14T04:55:21.372 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:55:21.372 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 64662 2024-09-14T04:55:21.372 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.372 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:55:21.372 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.372 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:55:21.372 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.120270+0000", 2024-09-14T04:55:21.373 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.373 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.373 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.373 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:55:21.373 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.120270+0000", 2024-09-14T04:55:21.373 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.373 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.373 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.373 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:55:21.373 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.120271+0000", 2024-09-14T04:55:21.373 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.688e-06 2024-09-14T04:55:21.373 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.373 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.373 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:55:21.374 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.120299+0000", 2024-09-14T04:55:21.374 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.8153e-05 2024-09-14T04:55:21.374 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.374 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.374 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:55:21.374 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.120316+0000", 2024-09-14T04:55:21.374 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6735000000000002e-05 2024-09-14T04:55:21.374 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.374 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.374 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:55:21.374 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.120324+0000", 2024-09-14T04:55:21.374 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7759999999999997e-06 2024-09-14T04:55:21.374 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.375 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.375 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:55:21.375 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.120531+0000", 2024-09-14T04:55:21.375 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000207426 2024-09-14T04:55:21.375 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.375 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.375 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:55:21.375 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.120546+0000", 2024-09-14T04:55:21.375 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4630000000000001e-05 2024-09-14T04:55:21.375 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.375 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.375 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:55:21.375 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.076283+0000", 2024-09-14T04:55:21.376 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.95573650200000004 2024-09-14T04:55:21.376 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.376 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.376 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:55:21.376 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.076402+0000", 2024-09-14T04:55:21.376 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000119165 2024-09-14T04:55:21.376 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.376 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.376 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:55:21.377 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.076557+0000", 2024-09-14T04:55:21.377 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015563099999999999 2024-09-14T04:55:21.377 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.377 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.377 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.377 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.084154+0000", 2024-09-14T04:55:21.377 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0075969749999999997 2024-09-14T04:55:21.377 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.377 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.377 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:55:21.377 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.084229+0000", 2024-09-14T04:55:21.378 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.4558e-05 2024-09-14T04:55:21.378 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.378 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.378 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.378 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.084273+0000", 2024-09-14T04:55:21.378 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3788999999999997e-05 2024-09-14T04:55:21.378 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.378 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.378 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:55:21.378 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.084314+0000", 2024-09-14T04:55:21.379 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.1477999999999998e-05 2024-09-14T04:55:21.379 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.379 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.379 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:55:21.379 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.084345+0000", 2024-09-14T04:55:21.379 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.0716999999999997e-05 2024-09-14T04:55:21.379 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.380 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:55:21.380 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.380 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.380 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.380 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45632.0:64703 55.1 55:85591b41:::benchmark_data_smithi039_297155_object64702:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1437)", 2024-09-14T04:55:21.380 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:48:36.148377+0000", 2024-09-14T04:55:21.380 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 404.84670319499998, 2024-09-14T04:55:21.380 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.96002874999999999, 2024-09-14T04:55:21.381 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:55:21.381 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:55:21.381 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:55:21.381 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:55:21.381 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45632", 2024-09-14T04:55:21.381 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:55:21.381 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 64703 2024-09-14T04:55:21.381 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.381 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:55:21.381 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.382 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:55:21.382 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.148377+0000", 2024-09-14T04:55:21.382 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.382 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.382 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.382 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:55:21.382 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.148377+0000", 2024-09-14T04:55:21.382 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.382 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.382 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.382 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:55:21.382 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.148378+0000", 2024-09-14T04:55:21.382 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7320000000000001e-06 2024-09-14T04:55:21.383 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.383 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.383 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:55:21.383 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.148403+0000", 2024-09-14T04:55:21.383 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4119000000000001e-05 2024-09-14T04:55:21.383 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.383 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.383 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:55:21.383 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.148419+0000", 2024-09-14T04:55:21.383 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6721000000000001e-05 2024-09-14T04:55:21.383 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.383 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.383 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:55:21.384 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.148427+0000", 2024-09-14T04:55:21.384 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6559999999999992e-06 2024-09-14T04:55:21.384 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.384 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.384 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:55:21.384 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.148496+0000", 2024-09-14T04:55:21.384 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9337000000000004e-05 2024-09-14T04:55:21.384 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.384 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.384 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:55:21.384 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.148508+0000", 2024-09-14T04:55:21.384 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1364e-05 2024-09-14T04:55:21.384 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.385 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.385 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:55:21.385 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.081665+0000", 2024-09-14T04:55:21.385 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.93315764199999995 2024-09-14T04:55:21.385 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.385 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.385 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:55:21.385 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.081808+0000", 2024-09-14T04:55:21.385 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014285799999999999 2024-09-14T04:55:21.385 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.385 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.385 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:55:21.385 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.081961+0000", 2024-09-14T04:55:21.385 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000153089 2024-09-14T04:55:21.386 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.386 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.386 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.386 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.089885+0000", 2024-09-14T04:55:21.386 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0079243109999999999 2024-09-14T04:55:21.386 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.386 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.386 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.386 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.090165+0000", 2024-09-14T04:55:21.386 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000279795 2024-09-14T04:55:21.386 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.386 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.386 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:55:21.386 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.108306+0000", 2024-09-14T04:55:21.387 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.018140763000000001 2024-09-14T04:55:21.387 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.387 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.387 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:55:21.387 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.108360+0000", 2024-09-14T04:55:21.387 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4262999999999999e-05 2024-09-14T04:55:21.387 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.387 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.387 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:55:21.387 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.108405+0000", 2024-09-14T04:55:21.387 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5099999999999998e-05 2024-09-14T04:55:21.387 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.387 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:55:21.388 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.388 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.388 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.388 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45632.0:64708 55.1 55:8b4837e3:::benchmark_data_smithi039_297155_object64707:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1437)", 2024-09-14T04:55:21.388 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:48:36.152492+0000", 2024-09-14T04:55:21.388 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 404.842588122, 2024-09-14T04:55:21.388 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.95604835200000005, 2024-09-14T04:55:21.388 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:55:21.388 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:55:21.389 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:55:21.389 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:55:21.389 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45632", 2024-09-14T04:55:21.389 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:55:21.389 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 64708 2024-09-14T04:55:21.389 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.389 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:55:21.389 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.390 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:55:21.390 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.152492+0000", 2024-09-14T04:55:21.390 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.390 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.390 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.390 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:55:21.390 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.152492+0000", 2024-09-14T04:55:21.390 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.391 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.391 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.391 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:55:21.391 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.152494+0000", 2024-09-14T04:55:21.391 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3709999999999998e-06 2024-09-14T04:55:21.391 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.391 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.391 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:55:21.392 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.152521+0000", 2024-09-14T04:55:21.392 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.6580999999999998e-05 2024-09-14T04:55:21.392 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.392 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.392 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:55:21.392 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.152539+0000", 2024-09-14T04:55:21.392 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8518000000000001e-05 2024-09-14T04:55:21.392 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.392 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.393 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:55:21.393 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.152549+0000", 2024-09-14T04:55:21.393 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.7710000000000001e-06 2024-09-14T04:55:21.393 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.393 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.393 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:55:21.393 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.152631+0000", 2024-09-14T04:55:21.393 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.2509999999999994e-05 2024-09-14T04:55:21.394 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.394 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.394 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:55:21.394 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.152643+0000", 2024-09-14T04:55:21.394 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1398e-05 2024-09-14T04:55:21.394 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.394 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.394 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:55:21.394 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.084389+0000", 2024-09-14T04:55:21.395 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.93174628999999998 2024-09-14T04:55:21.395 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.395 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.395 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:55:21.395 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.084495+0000", 2024-09-14T04:55:21.395 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00010586699999999999 2024-09-14T04:55:21.395 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.395 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.396 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:55:21.396 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.084661+0000", 2024-09-14T04:55:21.396 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000166298 2024-09-14T04:55:21.396 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.396 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.396 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.396 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.090110+0000", 2024-09-14T04:55:21.396 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0054491050000000001 2024-09-14T04:55:21.396 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.397 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.397 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.397 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.092751+0000", 2024-09-14T04:55:21.397 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0026402259999999999 2024-09-14T04:55:21.397 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.397 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.397 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:55:21.397 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.108464+0000", 2024-09-14T04:55:21.398 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.01571299 2024-09-14T04:55:21.398 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.398 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.398 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:55:21.398 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.108508+0000", 2024-09-14T04:55:21.398 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4283000000000003e-05 2024-09-14T04:55:21.398 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.398 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.398 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:55:21.399 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.108540+0000", 2024-09-14T04:55:21.399 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.2144000000000003e-05 2024-09-14T04:55:21.399 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.399 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:55:21.399 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.399 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.399 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.399 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36012.0:35353 56.8 56:17a3f911:::benchmark_data_smithi039_330847_object35352:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:55:21.400 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:52:44.524859+0000", 2024-09-14T04:55:21.400 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 156.47022086300001, 2024-09-14T04:55:21.400 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.32379368400000003, 2024-09-14T04:55:21.400 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:55:21.400 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:55:21.400 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:55:21.400 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:55:21.400 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36012", 2024-09-14T04:55:21.401 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:55:21.401 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 35353 2024-09-14T04:55:21.401 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.401 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:55:21.401 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.401 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:55:21.401 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.524859+0000", 2024-09-14T04:55:21.401 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.402 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.402 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.402 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:55:21.402 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.524859+0000", 2024-09-14T04:55:21.402 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.402 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.402 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.402 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:55:21.402 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.524860+0000", 2024-09-14T04:55:21.403 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.029e-06 2024-09-14T04:55:21.403 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.403 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.403 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:55:21.403 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.524886+0000", 2024-09-14T04:55:21.403 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.5562e-05 2024-09-14T04:55:21.403 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.403 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.403 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:55:21.404 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.524898+0000", 2024-09-14T04:55:21.404 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2337999999999999e-05 2024-09-14T04:55:21.404 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.404 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.404 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:55:21.404 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.524910+0000", 2024-09-14T04:55:21.404 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2148e-05 2024-09-14T04:55:21.405 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.405 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.405 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:55:21.405 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.524967+0000", 2024-09-14T04:55:21.405 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7408999999999999e-05 2024-09-14T04:55:21.405 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.405 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.405 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:55:21.405 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.525087+0000", 2024-09-14T04:55:21.405 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000119529 2024-09-14T04:55:21.405 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.405 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.405 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:55:21.405 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.525193+0000", 2024-09-14T04:55:21.406 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000106099 2024-09-14T04:55:21.406 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.406 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.406 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:55:21.406 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.527811+0000", 2024-09-14T04:55:21.406 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.002617426 2024-09-14T04:55:21.406 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.406 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.406 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.406 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.528252+0000", 2024-09-14T04:55:21.406 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000441734 2024-09-14T04:55:21.406 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.406 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.407 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.407 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.848530+0000", 2024-09-14T04:55:21.407 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.32027803599999999 2024-09-14T04:55:21.407 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.407 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.407 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:55:21.407 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.848605+0000", 2024-09-14T04:55:21.407 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.5025999999999996e-05 2024-09-14T04:55:21.407 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.407 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.407 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:55:21.407 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.848653+0000", 2024-09-14T04:55:21.407 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7348000000000001e-05 2024-09-14T04:55:21.407 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.408 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:55:21.408 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.408 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.408 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.408 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36012.0:37074 56.7 56:eae87ae0:::benchmark_data_smithi039_330847_object37073:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:55:21.408 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:52:45.860442+0000", 2024-09-14T04:55:21.408 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 155.13463770800001, 2024-09-14T04:55:21.408 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.307325289, 2024-09-14T04:55:21.408 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:55:21.408 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:55:21.408 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:55:21.408 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:55:21.408 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36012", 2024-09-14T04:55:21.409 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:55:21.409 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 37074 2024-09-14T04:55:21.409 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.409 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:55:21.409 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.409 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:55:21.409 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.860442+0000", 2024-09-14T04:55:21.409 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.409 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.409 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.409 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:55:21.410 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.860442+0000", 2024-09-14T04:55:21.410 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.410 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.410 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.410 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:55:21.410 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.860444+0000", 2024-09-14T04:55:21.410 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.778e-06 2024-09-14T04:55:21.410 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.410 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.410 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:55:21.410 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.860499+0000", 2024-09-14T04:55:21.410 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5426000000000001e-05 2024-09-14T04:55:21.411 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.411 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.411 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:55:21.411 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.860519+0000", 2024-09-14T04:55:21.411 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9241000000000001e-05 2024-09-14T04:55:21.411 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.411 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.411 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:55:21.411 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.860526+0000", 2024-09-14T04:55:21.412 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7789999999999996e-06 2024-09-14T04:55:21.412 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.412 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.412 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:55:21.412 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.860587+0000", 2024-09-14T04:55:21.412 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0393999999999998e-05 2024-09-14T04:55:21.412 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.412 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.412 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:55:21.412 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.860738+0000", 2024-09-14T04:55:21.412 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015164200000000001 2024-09-14T04:55:21.413 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.413 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.413 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:55:21.413 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.860905+0000", 2024-09-14T04:55:21.413 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00016642800000000001 2024-09-14T04:55:21.413 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.413 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.413 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:55:21.413 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.862183+0000", 2024-09-14T04:55:21.413 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0012785209999999999 2024-09-14T04:55:21.413 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.413 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.413 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.413 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.865856+0000", 2024-09-14T04:55:21.414 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0036722949999999999 2024-09-14T04:55:21.414 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.414 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.414 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.414 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.167593+0000", 2024-09-14T04:55:21.414 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.30173692600000002 2024-09-14T04:55:21.414 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.414 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.414 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:55:21.414 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.167673+0000", 2024-09-14T04:55:21.414 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0296999999999994e-05 2024-09-14T04:55:21.414 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.414 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.414 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:55:21.415 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.167767+0000", 2024-09-14T04:55:21.415 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.4561999999999993e-05 2024-09-14T04:55:21.415 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.415 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:55:21.415 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.415 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.415 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.415 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36012.0:37080 56.7 56:eb559557:::benchmark_data_smithi039_330847_object37079:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:55:21.415 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:52:45.862767+0000", 2024-09-14T04:55:21.415 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 155.13231311199999, 2024-09-14T04:55:21.415 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.30515425800000001, 2024-09-14T04:55:21.416 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:55:21.416 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:55:21.416 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:55:21.416 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:55:21.416 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36012", 2024-09-14T04:55:21.416 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:55:21.416 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 37080 2024-09-14T04:55:21.416 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.416 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:55:21.416 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.416 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:55:21.416 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.862767+0000", 2024-09-14T04:55:21.417 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.417 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.417 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.417 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:55:21.417 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.862767+0000", 2024-09-14T04:55:21.417 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.417 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.418 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.418 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:55:21.418 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.862768+0000", 2024-09-14T04:55:21.418 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.142e-06 2024-09-14T04:55:21.418 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.418 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.418 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:55:21.418 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.862791+0000", 2024-09-14T04:55:21.418 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3028999999999999e-05 2024-09-14T04:55:21.418 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.418 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.418 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:55:21.419 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.862805+0000", 2024-09-14T04:55:21.419 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3687e-05 2024-09-14T04:55:21.419 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.419 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.419 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:55:21.419 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.862812+0000", 2024-09-14T04:55:21.419 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.3100000000000003e-06 2024-09-14T04:55:21.419 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.419 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.419 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:55:21.419 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.862874+0000", 2024-09-14T04:55:21.419 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2467999999999999e-05 2024-09-14T04:55:21.420 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.420 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.420 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:55:21.420 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.863032+0000", 2024-09-14T04:55:21.420 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000157758 2024-09-14T04:55:21.420 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.420 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.420 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:55:21.420 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.863198+0000", 2024-09-14T04:55:21.421 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000165458 2024-09-14T04:55:21.421 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.421 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.421 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:55:21.421 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.865515+0000", 2024-09-14T04:55:21.421 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0023170880000000001 2024-09-14T04:55:21.421 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.421 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.421 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.421 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.865903+0000", 2024-09-14T04:55:21.421 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00038844299999999998 2024-09-14T04:55:21.422 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.422 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.422 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.422 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.167819+0000", 2024-09-14T04:55:21.422 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.30191585599999998 2024-09-14T04:55:21.422 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.422 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.422 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:55:21.422 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.167882+0000", 2024-09-14T04:55:21.422 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3025000000000001e-05 2024-09-14T04:55:21.422 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.423 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.423 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:55:21.423 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.167921+0000", 2024-09-14T04:55:21.423 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.8994e-05 2024-09-14T04:55:21.423 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.423 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:55:21.423 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.423 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.423 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.423 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36012.0:37082 56.7 56:eceabb24:::benchmark_data_smithi039_330847_object37081:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:55:21.423 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:52:45.863493+0000", 2024-09-14T04:55:21.423 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 155.13158667100001, 2024-09-14T04:55:21.423 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.304588896, 2024-09-14T04:55:21.424 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:55:21.424 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:55:21.424 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:55:21.424 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:55:21.424 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36012", 2024-09-14T04:55:21.424 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:55:21.424 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 37082 2024-09-14T04:55:21.424 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.424 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:55:21.424 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.424 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:55:21.424 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.863493+0000", 2024-09-14T04:55:21.424 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.425 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.425 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.425 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:55:21.425 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.863493+0000", 2024-09-14T04:55:21.425 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.425 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.425 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.425 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:55:21.425 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.863494+0000", 2024-09-14T04:55:21.426 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.6700000000000002e-07 2024-09-14T04:55:21.426 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.426 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.426 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:55:21.426 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.863521+0000", 2024-09-14T04:55:21.426 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.6621000000000001e-05 2024-09-14T04:55:21.427 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.427 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.427 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:55:21.427 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.863534+0000", 2024-09-14T04:55:21.427 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3594999999999999e-05 2024-09-14T04:55:21.427 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.428 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.428 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:55:21.428 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.863542+0000", 2024-09-14T04:55:21.428 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7020000000000001e-06 2024-09-14T04:55:21.428 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.428 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.429 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:55:21.429 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.864405+0000", 2024-09-14T04:55:21.429 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00086272499999999995 2024-09-14T04:55:21.429 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.429 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.429 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:55:21.430 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.864563+0000", 2024-09-14T04:55:21.430 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000158431 2024-09-14T04:55:21.430 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.430 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.430 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:55:21.430 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.864728+0000", 2024-09-14T04:55:21.431 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00016483700000000001 2024-09-14T04:55:21.431 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.431 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.431 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:55:21.431 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.865734+0000", 2024-09-14T04:55:21.431 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0010062910000000001 2024-09-14T04:55:21.432 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.432 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.432 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.432 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.867656+0000", 2024-09-14T04:55:21.432 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0019217139999999999 2024-09-14T04:55:21.432 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.433 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.433 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.433 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.167989+0000", 2024-09-14T04:55:21.433 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.30033338300000001 2024-09-14T04:55:21.433 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.434 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.434 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:55:21.434 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.168045+0000", 2024-09-14T04:55:21.434 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5189000000000003e-05 2024-09-14T04:55:21.434 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.434 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.435 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:55:21.435 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.168082+0000", 2024-09-14T04:55:21.435 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.7540999999999999e-05 2024-09-14T04:55:21.435 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.435 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:55:21.435 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.436 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.436 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.436 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36012.0:116301 56.7 56:e29960ea:::benchmark_data_smithi039_330847_object1777:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1486)", 2024-09-14T04:55:21.436 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:54:04.677654+0000", 2024-09-14T04:55:21.436 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 76.317426073999997, 2024-09-14T04:55:21.437 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.39274526700000001, 2024-09-14T04:55:21.437 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:55:21.437 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:55:21.437 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:55:21.437 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:55:21.437 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36012", 2024-09-14T04:55:21.438 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:55:21.438 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 116301 2024-09-14T04:55:21.438 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.438 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:55:21.438 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.439 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:55:21.439 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.677654+0000", 2024-09-14T04:55:21.439 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.439 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.439 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.439 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:55:21.440 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.677654+0000", 2024-09-14T04:55:21.440 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.440 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.440 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.440 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:55:21.440 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.677655+0000", 2024-09-14T04:55:21.441 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.61e-06 2024-09-14T04:55:21.441 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.441 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.441 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:55:21.441 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.677661+0000", 2024-09-14T04:55:21.441 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0800000000000002e-06 2024-09-14T04:55:21.442 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.442 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.442 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:55:21.442 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.677681+0000", 2024-09-14T04:55:21.442 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9633999999999999e-05 2024-09-14T04:55:21.442 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.443 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.443 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:55:21.443 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.677689+0000", 2024-09-14T04:55:21.443 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.1000000000000004e-06 2024-09-14T04:55:21.443 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.443 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.444 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:55:21.444 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.677747+0000", 2024-09-14T04:55:21.444 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.8220999999999997e-05 2024-09-14T04:55:21.444 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.444 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.445 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:55:21.445 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.677869+0000", 2024-09-14T04:55:21.445 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012144799999999999 2024-09-14T04:55:21.445 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.445 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.445 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:55:21.446 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.677974+0000", 2024-09-14T04:55:21.446 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000104996 2024-09-14T04:55:21.446 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.446 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.446 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:55:21.447 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.681749+0000", 2024-09-14T04:55:21.447 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0037746630000000002 2024-09-14T04:55:21.447 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.447 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.447 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.447 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.682012+0000", 2024-09-14T04:55:21.448 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00026392700000000001 2024-09-14T04:55:21.448 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.448 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.448 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.448 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:05.070285+0000", 2024-09-14T04:55:21.448 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.38827251200000001 2024-09-14T04:55:21.449 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.449 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.449 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:55:21.449 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:05.070346+0000", 2024-09-14T04:55:21.449 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0699000000000003e-05 2024-09-14T04:55:21.450 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.450 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.450 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:55:21.450 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:05.070399+0000", 2024-09-14T04:55:21.450 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3377000000000001e-05 2024-09-14T04:55:21.450 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.450 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:55:21.450 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.450 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.450 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.450 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36012.0:116306 56.7 56:e47ad2ec:::benchmark_data_smithi039_330847_object1782:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1486)", 2024-09-14T04:55:21.450 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:54:04.682098+0000", 2024-09-14T04:55:21.451 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 76.312981624000003, 2024-09-14T04:55:21.451 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.38844271400000002, 2024-09-14T04:55:21.451 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:55:21.451 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:55:21.451 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:55:21.451 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:55:21.451 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36012", 2024-09-14T04:55:21.451 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:55:21.451 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 116306 2024-09-14T04:55:21.451 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.451 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:55:21.451 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.451 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:55:21.451 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.682098+0000", 2024-09-14T04:55:21.452 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.452 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.452 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.452 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:55:21.452 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.682098+0000", 2024-09-14T04:55:21.452 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.452 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.452 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.452 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:55:21.452 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.682099+0000", 2024-09-14T04:55:21.452 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.26e-06 2024-09-14T04:55:21.452 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.452 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.453 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:55:21.453 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.682108+0000", 2024-09-14T04:55:21.453 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0539999999999995e-06 2024-09-14T04:55:21.453 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.453 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.453 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:55:21.453 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.682119+0000", 2024-09-14T04:55:21.453 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1185999999999999e-05 2024-09-14T04:55:21.453 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.453 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.453 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:55:21.453 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.682124+0000", 2024-09-14T04:55:21.453 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9069999999999998e-06 2024-09-14T04:55:21.453 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.454 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.454 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:55:21.454 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.682172+0000", 2024-09-14T04:55:21.454 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.8772e-05 2024-09-14T04:55:21.454 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.454 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.454 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:55:21.454 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.682300+0000", 2024-09-14T04:55:21.454 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012762400000000001 2024-09-14T04:55:21.454 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.454 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.454 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:55:21.454 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.682417+0000", 2024-09-14T04:55:21.454 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000116727 2024-09-14T04:55:21.455 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.455 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.455 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.455 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.685771+0000", 2024-09-14T04:55:21.455 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.003354147 2024-09-14T04:55:21.455 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.455 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.455 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:55:21.455 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.685893+0000", 2024-09-14T04:55:21.455 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012235999999999999 2024-09-14T04:55:21.455 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.455 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.455 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.455 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:05.070453+0000", 2024-09-14T04:55:21.456 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.38455929700000002 2024-09-14T04:55:21.456 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.456 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.456 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:55:21.456 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:05.070510+0000", 2024-09-14T04:55:21.456 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6988999999999999e-05 2024-09-14T04:55:21.456 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.456 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.456 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:55:21.456 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:05.070541+0000", 2024-09-14T04:55:21.456 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.1390999999999998e-05 2024-09-14T04:55:21.456 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.456 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:55:21.457 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.457 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.457 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.457 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36012.0:116313 56.7 56:e60392f8:::benchmark_data_smithi039_330847_object1789:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1486)", 2024-09-14T04:55:21.457 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:54:04.697875+0000", 2024-09-14T04:55:21.457 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 76.297204674, 2024-09-14T04:55:21.457 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.372792927, 2024-09-14T04:55:21.457 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:55:21.457 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:55:21.457 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:55:21.457 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:55:21.457 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36012", 2024-09-14T04:55:21.457 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:55:21.457 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 116313 2024-09-14T04:55:21.458 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.458 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:55:21.458 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.458 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:55:21.458 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.697875+0000", 2024-09-14T04:55:21.458 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.458 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.458 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.458 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:55:21.458 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.697875+0000", 2024-09-14T04:55:21.458 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.458 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.458 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.458 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:55:21.459 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.697877+0000", 2024-09-14T04:55:21.459 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4619999999999999e-06 2024-09-14T04:55:21.459 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.459 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.459 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:55:21.459 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.697881+0000", 2024-09-14T04:55:21.459 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.9589999999999997e-06 2024-09-14T04:55:21.459 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.459 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.459 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:55:21.459 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.697891+0000", 2024-09-14T04:55:21.459 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0159e-05 2024-09-14T04:55:21.459 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.460 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.460 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:55:21.460 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.697896+0000", 2024-09-14T04:55:21.460 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.079e-06 2024-09-14T04:55:21.460 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.460 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.460 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:55:21.460 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.697953+0000", 2024-09-14T04:55:21.460 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7413000000000001e-05 2024-09-14T04:55:21.460 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.460 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.460 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:55:21.460 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.698096+0000", 2024-09-14T04:55:21.461 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014254900000000001 2024-09-14T04:55:21.461 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.461 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.461 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:55:21.461 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.698220+0000", 2024-09-14T04:55:21.461 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012389200000000001 2024-09-14T04:55:21.461 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.461 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.461 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:55:21.461 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.701238+0000", 2024-09-14T04:55:21.461 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0030187209999999998 2024-09-14T04:55:21.461 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.461 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.462 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.462 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.701534+0000", 2024-09-14T04:55:21.462 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00029558399999999999 2024-09-14T04:55:21.462 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.462 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.462 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.462 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:05.070586+0000", 2024-09-14T04:55:21.462 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.36905189500000002 2024-09-14T04:55:21.462 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.462 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.462 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:55:21.462 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:05.070634+0000", 2024-09-14T04:55:21.462 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7794999999999999e-05 2024-09-14T04:55:21.462 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.462 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.463 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:55:21.463 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:05.070668+0000", 2024-09-14T04:55:21.463 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.4418999999999997e-05 2024-09-14T04:55:21.463 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.463 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:55:21.463 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.463 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.463 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.463 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36012.0:150550 56.d 56:b8b6dd53:::benchmark_data_smithi039_330847_object36026:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1502)", 2024-09-14T04:55:21.463 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:54:48.349027+0000", 2024-09-14T04:55:21.463 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 32.646052417, 2024-09-14T04:55:21.463 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.32527113499999999, 2024-09-14T04:55:21.463 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:55:21.463 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:55:21.463 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:55:21.464 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:55:21.464 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36012", 2024-09-14T04:55:21.464 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:55:21.464 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 150550 2024-09-14T04:55:21.464 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.464 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:55:21.464 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.464 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:55:21.464 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.349027+0000", 2024-09-14T04:55:21.464 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.464 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.464 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.464 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:55:21.465 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.349027+0000", 2024-09-14T04:55:21.465 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.465 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.465 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.465 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:55:21.465 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.349029+0000", 2024-09-14T04:55:21.465 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2049999999999999e-06 2024-09-14T04:55:21.465 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.465 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.465 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:55:21.465 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.349033+0000", 2024-09-14T04:55:21.465 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.861e-06 2024-09-14T04:55:21.465 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.465 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.465 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:55:21.466 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.349041+0000", 2024-09-14T04:55:21.466 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.7010000000000002e-06 2024-09-14T04:55:21.466 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.466 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.466 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:55:21.466 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.349046+0000", 2024-09-14T04:55:21.466 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.2730000000000002e-06 2024-09-14T04:55:21.466 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.466 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.466 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:55:21.466 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.349331+0000", 2024-09-14T04:55:21.466 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00028438999999999998 2024-09-14T04:55:21.466 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.467 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.467 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:55:21.467 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.349705+0000", 2024-09-14T04:55:21.467 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00037422399999999999 2024-09-14T04:55:21.467 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.467 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.467 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:55:21.467 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.350159+0000", 2024-09-14T04:55:21.467 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00045416199999999998 2024-09-14T04:55:21.467 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.467 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.467 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:55:21.467 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.354271+0000", 2024-09-14T04:55:21.467 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0041116099999999999 2024-09-14T04:55:21.467 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.467 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.468 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.468 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.355813+0000", 2024-09-14T04:55:21.468 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001542617 2024-09-14T04:55:21.468 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.468 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.468 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.468 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.674130+0000", 2024-09-14T04:55:21.468 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.31831626400000002 2024-09-14T04:55:21.468 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.468 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.468 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:55:21.468 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.674247+0000", 2024-09-14T04:55:21.468 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000117269 2024-09-14T04:55:21.469 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.469 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.469 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:55:21.469 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.674299+0000", 2024-09-14T04:55:21.469 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1558999999999999e-05 2024-09-14T04:55:21.469 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.469 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:55:21.469 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.469 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.469 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.469 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36012.0:150555 56.d 56:b0433500:::benchmark_data_smithi039_330847_object36031:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1502)", 2024-09-14T04:55:21.469 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:54:48.356754+0000", 2024-09-14T04:55:21.469 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 32.638325498999997, 2024-09-14T04:55:21.469 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.31776584200000002, 2024-09-14T04:55:21.470 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:55:21.470 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:55:21.470 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:55:21.470 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:55:21.470 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36012", 2024-09-14T04:55:21.470 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:55:21.470 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 150555 2024-09-14T04:55:21.470 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.470 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:55:21.470 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.470 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:55:21.470 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.356754+0000", 2024-09-14T04:55:21.470 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.470 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.470 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.470 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:55:21.471 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.356754+0000", 2024-09-14T04:55:21.471 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.471 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.471 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.471 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:55:21.471 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.356756+0000", 2024-09-14T04:55:21.471 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.389e-06 2024-09-14T04:55:21.471 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.471 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.471 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:55:21.471 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.356761+0000", 2024-09-14T04:55:21.471 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3900000000000001e-06 2024-09-14T04:55:21.471 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.472 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.472 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:55:21.472 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.356780+0000", 2024-09-14T04:55:21.472 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9211999999999998e-05 2024-09-14T04:55:21.472 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.472 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.472 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:55:21.472 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.356786+0000", 2024-09-14T04:55:21.472 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0780000000000001e-06 2024-09-14T04:55:21.472 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.472 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.472 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:55:21.472 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.356847+0000", 2024-09-14T04:55:21.472 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0679e-05 2024-09-14T04:55:21.473 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.473 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.473 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:55:21.473 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.356949+0000", 2024-09-14T04:55:21.473 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000102388 2024-09-14T04:55:21.473 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.473 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.473 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:55:21.473 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.357026+0000", 2024-09-14T04:55:21.473 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6840000000000003e-05 2024-09-14T04:55:21.473 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.473 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.473 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:55:21.474 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.360335+0000", 2024-09-14T04:55:21.474 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0033084120000000002 2024-09-14T04:55:21.474 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.474 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.474 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.474 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.361903+0000", 2024-09-14T04:55:21.474 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0015680889999999999 2024-09-14T04:55:21.474 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.474 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.474 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.474 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.674345+0000", 2024-09-14T04:55:21.474 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.312442306 2024-09-14T04:55:21.474 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.474 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.475 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:55:21.475 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.674477+0000", 2024-09-14T04:55:21.475 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013163399999999999 2024-09-14T04:55:21.475 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.475 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.475 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:55:21.475 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.674520+0000", 2024-09-14T04:55:21.475 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3424999999999998e-05 2024-09-14T04:55:21.475 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.475 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:55:21.475 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.475 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.475 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.475 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36012.0:169819 56.8 56:11f1b4ea:::benchmark_data_smithi039_330847_object55295:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1508)", 2024-09-14T04:55:21.476 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:55:13.377122+0000", 2024-09-14T04:55:21.476 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 7.6179577590000003, 2024-09-14T04:55:21.476 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.096314571000000002, 2024-09-14T04:55:21.476 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:55:21.476 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:55:21.476 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:55:21.476 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:55:21.476 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36012", 2024-09-14T04:55:21.476 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:55:21.476 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 169819 2024-09-14T04:55:21.476 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.476 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:55:21.476 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.477 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:55:21.477 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.377122+0000", 2024-09-14T04:55:21.477 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.477 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.477 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.477 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:55:21.477 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.377122+0000", 2024-09-14T04:55:21.477 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.477 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.477 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.477 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:55:21.477 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.377123+0000", 2024-09-14T04:55:21.477 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1909999999999999e-06 2024-09-14T04:55:21.477 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.478 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.478 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:55:21.478 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.377128+0000", 2024-09-14T04:55:21.478 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6439999999999999e-06 2024-09-14T04:55:21.478 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.478 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.478 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:55:21.478 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.377137+0000", 2024-09-14T04:55:21.478 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.8359999999999998e-06 2024-09-14T04:55:21.478 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.478 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.478 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:55:21.478 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.377143+0000", 2024-09-14T04:55:21.478 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.5479999999999998e-06 2024-09-14T04:55:21.478 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.479 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.479 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:55:21.479 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.377375+0000", 2024-09-14T04:55:21.479 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00023156199999999999 2024-09-14T04:55:21.479 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.479 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.479 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:55:21.479 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.377503+0000", 2024-09-14T04:55:21.479 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012784600000000001 2024-09-14T04:55:21.479 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.479 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.479 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:55:21.479 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.377620+0000", 2024-09-14T04:55:21.480 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000117205 2024-09-14T04:55:21.480 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.480 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.480 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:55:21.480 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.382561+0000", 2024-09-14T04:55:21.480 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0049412910000000004 2024-09-14T04:55:21.480 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.480 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.480 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.480 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.383052+0000", 2024-09-14T04:55:21.480 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00049075099999999999 2024-09-14T04:55:21.480 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.480 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.480 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.481 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.473324+0000", 2024-09-14T04:55:21.481 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.090272009 2024-09-14T04:55:21.481 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.481 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.481 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:55:21.481 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.473388+0000", 2024-09-14T04:55:21.481 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3516000000000006e-05 2024-09-14T04:55:21.481 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.481 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.481 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:55:21.481 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.473437+0000", 2024-09-14T04:55:21.481 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9172000000000003e-05 2024-09-14T04:55:21.481 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.481 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:55:21.482 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.482 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.482 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.482 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36012.0:169827 56.8 56:149adbc0:::benchmark_data_smithi039_330847_object55303:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1508)", 2024-09-14T04:55:21.482 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:55:13.383668+0000", 2024-09-14T04:55:21.482 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 7.6114114739999996, 2024-09-14T04:55:21.482 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.090797952000000001, 2024-09-14T04:55:21.482 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:55:21.482 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:55:21.482 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:55:21.482 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:55:21.482 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36012", 2024-09-14T04:55:21.482 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:55:21.483 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 169827 2024-09-14T04:55:21.483 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.483 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:55:21.483 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.483 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:55:21.483 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.383668+0000", 2024-09-14T04:55:21.483 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.483 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.483 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.483 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:55:21.483 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.383668+0000", 2024-09-14T04:55:21.483 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.483 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.483 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.484 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:55:21.484 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.383670+0000", 2024-09-14T04:55:21.484 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0379999999999998e-06 2024-09-14T04:55:21.484 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.484 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.484 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:55:21.484 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.383676+0000", 2024-09-14T04:55:21.484 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.0900000000000004e-06 2024-09-14T04:55:21.484 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.484 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.484 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:55:21.484 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.383692+0000", 2024-09-14T04:55:21.484 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6668e-05 2024-09-14T04:55:21.484 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.485 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.485 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:55:21.485 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.383700+0000", 2024-09-14T04:55:21.485 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7859999999999997e-06 2024-09-14T04:55:21.485 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.485 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.485 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:55:21.485 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.383753+0000", 2024-09-14T04:55:21.485 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3513000000000001e-05 2024-09-14T04:55:21.485 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.485 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.485 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:55:21.485 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.383868+0000", 2024-09-14T04:55:21.486 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000114431 2024-09-14T04:55:21.486 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.486 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.486 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:55:21.486 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.384129+0000", 2024-09-14T04:55:21.486 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00026091800000000001 2024-09-14T04:55:21.486 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.486 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.486 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:55:21.486 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.388323+0000", 2024-09-14T04:55:21.486 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0041944740000000001 2024-09-14T04:55:21.486 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.486 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.487 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.487 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.389203+0000", 2024-09-14T04:55:21.487 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00087945500000000002 2024-09-14T04:55:21.487 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.487 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.487 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.487 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.474257+0000", 2024-09-14T04:55:21.487 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.085054215000000002 2024-09-14T04:55:21.487 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.487 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.487 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:55:21.487 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.474384+0000", 2024-09-14T04:55:21.487 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000126766 2024-09-14T04:55:21.487 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.488 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.488 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:55:21.488 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.474466+0000", 2024-09-14T04:55:21.488 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.2597999999999998e-05 2024-09-14T04:55:21.488 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.488 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:55:21.488 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.488 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.488 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.488 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36012.0:169828 56.7 56:ef8721ed:::benchmark_data_smithi039_330847_object55304:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1508)", 2024-09-14T04:55:21.488 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:55:13.384761+0000", 2024-09-14T04:55:21.488 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 7.6103187239999999, 2024-09-14T04:55:21.488 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.097914637999999998, 2024-09-14T04:55:21.489 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:55:21.489 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:55:21.489 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:55:21.489 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:55:21.489 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36012", 2024-09-14T04:55:21.489 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:55:21.489 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 169828 2024-09-14T04:55:21.489 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.489 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:55:21.489 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.489 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:55:21.489 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.384761+0000", 2024-09-14T04:55:21.489 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.490 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.490 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.490 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:55:21.490 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.384761+0000", 2024-09-14T04:55:21.490 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.490 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.490 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.490 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:55:21.490 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.384762+0000", 2024-09-14T04:55:21.490 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.6700000000000002e-07 2024-09-14T04:55:21.490 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.490 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.490 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:55:21.490 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.384766+0000", 2024-09-14T04:55:21.491 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.8199999999999998e-06 2024-09-14T04:55:21.491 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.491 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.491 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:55:21.491 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.384778+0000", 2024-09-14T04:55:21.491 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2559e-05 2024-09-14T04:55:21.491 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.491 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.491 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:55:21.491 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.384785+0000", 2024-09-14T04:55:21.491 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7329999999999997e-06 2024-09-14T04:55:21.491 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.491 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.491 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:55:21.492 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.384963+0000", 2024-09-14T04:55:21.492 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00017739399999999999 2024-09-14T04:55:21.492 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.492 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.492 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:55:21.492 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.385943+0000", 2024-09-14T04:55:21.492 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00098091299999999992 2024-09-14T04:55:21.492 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.492 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.492 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:55:21.492 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.386069+0000", 2024-09-14T04:55:21.492 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012510200000000001 2024-09-14T04:55:21.492 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.493 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.493 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:55:21.493 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.393237+0000", 2024-09-14T04:55:21.493 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0071681970000000003 2024-09-14T04:55:21.493 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.493 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.493 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.493 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.404598+0000", 2024-09-14T04:55:21.493 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.011361408 2024-09-14T04:55:21.493 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.493 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.493 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.493 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.478582+0000", 2024-09-14T04:55:21.493 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.073984035000000004 2024-09-14T04:55:21.494 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.494 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.494 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:55:21.494 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.482575+0000", 2024-09-14T04:55:21.494 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0039933089999999996 2024-09-14T04:55:21.494 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.494 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.494 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:55:21.494 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.482676+0000", 2024-09-14T04:55:21.494 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000100301 2024-09-14T04:55:21.494 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.494 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:55:21.494 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.495 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.495 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.495 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36012.0:169832 56.7 56:edb692d0:::benchmark_data_smithi039_330847_object55308:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1508)", 2024-09-14T04:55:21.495 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:55:13.385500+0000", 2024-09-14T04:55:21.495 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 7.6095800410000001, 2024-09-14T04:55:21.495 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.097478802000000003, 2024-09-14T04:55:21.495 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:55:21.495 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:55:21.495 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:55:21.495 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:55:21.495 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36012", 2024-09-14T04:55:21.495 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:55:21.495 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 169832 2024-09-14T04:55:21.495 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.496 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:55:21.496 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.496 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:55:21.496 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.385500+0000", 2024-09-14T04:55:21.496 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.496 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.496 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.496 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:55:21.496 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.385500+0000", 2024-09-14T04:55:21.496 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:55:21.496 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.496 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.496 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:55:21.497 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.385501+0000", 2024-09-14T04:55:21.497 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.8200000000000008e-07 2024-09-14T04:55:21.497 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.497 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.497 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:55:21.497 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.385505+0000", 2024-09-14T04:55:21.497 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.7740000000000002e-06 2024-09-14T04:55:21.497 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.497 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.497 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:55:21.497 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.385516+0000", 2024-09-14T04:55:21.497 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1251e-05 2024-09-14T04:55:21.497 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.498 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.498 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:55:21.498 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.385523+0000", 2024-09-14T04:55:21.498 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.4089999999999997e-06 2024-09-14T04:55:21.498 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.498 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.498 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:55:21.498 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.389895+0000", 2024-09-14T04:55:21.498 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0043715109999999998 2024-09-14T04:55:21.498 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.498 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.498 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:55:21.498 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.390015+0000", 2024-09-14T04:55:21.498 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000119787 2024-09-14T04:55:21.498 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.499 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.499 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:55:21.499 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.390144+0000", 2024-09-14T04:55:21.499 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012996500000000001 2024-09-14T04:55:21.499 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.499 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.499 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:55:21.499 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.396661+0000", 2024-09-14T04:55:21.499 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0065164469999999999 2024-09-14T04:55:21.499 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.499 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.499 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.499 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.404678+0000", 2024-09-14T04:55:21.500 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0080174949999999995 2024-09-14T04:55:21.500 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.500 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.500 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:55:21.500 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.482726+0000", 2024-09-14T04:55:21.500 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.078047730999999995 2024-09-14T04:55:21.500 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.500 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.500 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:55:21.500 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.482886+0000", 2024-09-14T04:55:21.500 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00016017599999999999 2024-09-14T04:55:21.500 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:55:21.500 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:55:21.500 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:55:21.501 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:13.482979+0000", 2024-09-14T04:55:21.501 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.2274000000000005e-05 2024-09-14T04:55:21.501 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.501 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:55:21.501 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.501 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:55:21.501 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:55:21.501 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:55:21.501 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:21.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:21.541 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:21.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:21.642 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:21.684 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:21.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:21.685 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:21.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:21 smithi129 ceph-mon[101316]: pgmap v2849: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 4350/163893 objects degraded (2.654%); 4617/163893 objects misplaced (2.817%); 3.2 MiB/s, 51 objects/s recovering 2024-09-14T04:55:21.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:21 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 4350/163893 objects degraded (2.654%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:55:21.786 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:21.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:21 smithi039 ceph-mon[106470]: pgmap v2849: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 4350/163893 objects degraded (2.654%); 4617/163893 objects misplaced (2.817%); 3.2 MiB/s, 51 objects/s recovering 2024-09-14T04:55:21.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:21 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 4350/163893 objects degraded (2.654%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:55:21.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:21 smithi039 ceph-mon[111235]: pgmap v2849: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 4350/163893 objects degraded (2.654%); 4617/163893 objects misplaced (2.817%); 3.2 MiB/s, 51 objects/s recovering 2024-09-14T04:55:21.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:21 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 4350/163893 objects degraded (2.654%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:55:21.820 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:21.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:21.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:21.922 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:21.958 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:21.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:21.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:22.063 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:22.091 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:22.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:22.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:22.193 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:22.231 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:22.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:22.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:22.333 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:22.367 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:22.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:22.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:22.469 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:22.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:22.499 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:22.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:22.516 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:22 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1699456196' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:55:22.516 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:55:22.516 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:22 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1699456196' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:55:22.516 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:55:22.600 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:22.635 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:22.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:22.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:22.738 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:22.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:22 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1699456196' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:55:22.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:55:22.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:22.805 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:22.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:22.906 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:22.947 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:22.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:22.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:22.977 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-14T04:55:23.049 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:23.089 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:23.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:23.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:23.191 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:23.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:23.253 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:23.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:23.354 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:23.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:23.384 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:23.385 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:23.485 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:23.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:23.516 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:23.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:23.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:23 smithi039 ceph-mon[106470]: pgmap v2850: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 4350/163893 objects degraded (2.654%); 4617/163893 objects misplaced (2.817%); 2.1 MiB/s, 34 objects/s recovering 2024-09-14T04:55:23.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:23 smithi039 ceph-mon[111235]: pgmap v2850: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 4350/163893 objects degraded (2.654%); 4617/163893 objects misplaced (2.817%); 2.1 MiB/s, 34 objects/s recovering 2024-09-14T04:55:23.617 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:23.653 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:23.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:23.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:23.729 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:55:23.729 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:55:23.729 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:55:23.730 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:55:23.730 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:55:23.730 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:55:23.730 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:55:23.730 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:55:23.730 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:55:23.730 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:55:23.730 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:55:23.731 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:55:23.731 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:55:23.731 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:55:23.731 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:55:23.731 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:55:23.755 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:23.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:23 smithi129 ceph-mon[101316]: pgmap v2850: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 714 op/s; 4350/163893 objects degraded (2.654%); 4617/163893 objects misplaced (2.817%); 2.1 MiB/s, 34 objects/s recovering 2024-09-14T04:55:23.796 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:23.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:23.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:23.897 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:23.934 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:23.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:23.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:24.036 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:24.066 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:24.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:24.068 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:24.169 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:24.211 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:24.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:24.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:24.313 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:24.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:24.351 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:24.351 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:24.452 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:24.508 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:24.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:24.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:24.610 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:24.646 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:24.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:24.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:24.748 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:24.783 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:24.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:24.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:24.885 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:24.929 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:24.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:24.930 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:25.031 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:25.067 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:25.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:25.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:25.169 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:25.202 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:25.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:25.203 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:25.218 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:55:25.236 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:55:25.304 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:25.349 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:25.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:25.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:25.451 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:25.487 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:25.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:25.488 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:25.589 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:25.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:25.624 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:25.624 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:25.725 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:25.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:25 smithi129 ceph-mon[101316]: pgmap v2851: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 912 op/s; 4350/156762 objects degraded (2.775%); 4617/156762 objects misplaced (2.945%); 2.1 MiB/s, 34 objects/s recovering 2024-09-14T04:55:25.776 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:25.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:25.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:25.789 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:25 smithi039 ceph-mon[111235]: pgmap v2851: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 912 op/s; 4350/156762 objects degraded (2.775%); 4617/156762 objects misplaced (2.945%); 2.1 MiB/s, 34 objects/s recovering 2024-09-14T04:55:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:25 smithi039 ceph-mon[106470]: pgmap v2851: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 912 op/s; 4350/156762 objects degraded (2.775%); 4617/156762 objects misplaced (2.945%); 2.1 MiB/s, 34 objects/s recovering 2024-09-14T04:55:25.878 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:25.936 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:25.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:25.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:26.038 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:26.088 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:26.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:26.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:26.190 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:26.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:26.225 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:26.225 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:26.326 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:26.372 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:26.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:26.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:26.474 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:26.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:26.508 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:26.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:26.609 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:26.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:26.644 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:26.644 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:26.682 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:26 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 4350/156762 objects degraded (2.775%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:55:26.683 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:26 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 4350/156762 objects degraded (2.775%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:55:26.745 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:26.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:26 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 4350/156762 objects degraded (2.775%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:55:26.786 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:26.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:26.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:26.889 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:26.921 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:26.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:26.922 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:27.023 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:27.060 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:27.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:27.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:27.162 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:27.171 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:55:27.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:27.204 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:27.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:27.305 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:27.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:27.337 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:27.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:27.438 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:27.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:27.465 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:27.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:27.478 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is unset 2024-09-14T04:55:27.566 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:27.601 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:27.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:27.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:27.703 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:27.733 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:27.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:27.734 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:27.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:27 smithi129 ceph-mon[101316]: pgmap v2852: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 4350/150924 objects degraded (2.882%); 4125/150924 objects misplaced (2.733%); 3.2 MiB/s, 51 objects/s recovering 2024-09-14T04:55:27.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:27 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1736355439' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:55:27.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:27 smithi039 ceph-mon[106470]: pgmap v2852: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 4350/150924 objects degraded (2.882%); 4125/150924 objects misplaced (2.733%); 3.2 MiB/s, 51 objects/s recovering 2024-09-14T04:55:27.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:27 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1736355439' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:55:27.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:27 smithi039 ceph-mon[111235]: pgmap v2852: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 872 op/s; 4350/150924 objects degraded (2.882%); 4125/150924 objects misplaced (2.733%); 3.2 MiB/s, 51 objects/s recovering 2024-09-14T04:55:27.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:27 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1736355439' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:55:27.835 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:27.872 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:27.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:27.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:27.974 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:28.009 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:28.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:28.010 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:28.110 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:28.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:28.147 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:28.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:28.249 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:28.287 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:28.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:28.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:28.389 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:28.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:28.430 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:28.430 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:28.531 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:28.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:28.569 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:28.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:28.649 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:28 smithi039 ceph-mon[111235]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:55:28.649 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:28 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1736355439' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:55:28.650 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:28 smithi039 ceph-mon[111235]: osdmap e1509: 8 total, 8 up, 8 in 2024-09-14T04:55:28.650 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:28 smithi039 ceph-mon[106470]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:55:28.650 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:28 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1736355439' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:55:28.650 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:28 smithi039 ceph-mon[106470]: osdmap e1509: 8 total, 8 up, 8 in 2024-09-14T04:55:28.670 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:28.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:28.706 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:28.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:28.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:28 smithi129 ceph-mon[101316]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:55:28.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:28 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1736355439' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:55:28.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:28 smithi129 ceph-mon[101316]: osdmap e1509: 8 total, 8 up, 8 in 2024-09-14T04:55:28.807 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:28.849 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:28.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:28.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:28.951 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:28.959 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:55:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:55:28] "GET /metrics HTTP/1.1" 200 36753 "" "Prometheus/2.51.0" 2024-09-14T04:55:28.987 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:28.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:28.988 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:29.089 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:29.131 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:29.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:29.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:29.234 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:29.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:29.276 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:29.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:29.377 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:29.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:29.406 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:29.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:29.507 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:29.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:29.545 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:29.545 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:29.646 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:29.679 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:29 smithi039 ceph-mon[106470]: pgmap v2854: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 4350/150924 objects degraded (2.882%); 4125/150924 objects misplaced (2.733%); 2.6 MiB/s, 42 objects/s recovering 2024-09-14T04:55:29.679 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:29 smithi039 ceph-mon[111235]: pgmap v2854: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 4350/150924 objects degraded (2.882%); 4125/150924 objects misplaced (2.733%); 2.6 MiB/s, 42 objects/s recovering 2024-09-14T04:55:29.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:29.711 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:29.711 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:29.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:29 smithi129 ceph-mon[101316]: pgmap v2854: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 3.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 850 op/s; 4350/150924 objects degraded (2.882%); 4125/150924 objects misplaced (2.733%); 2.6 MiB/s, 42 objects/s recovering 2024-09-14T04:55:29.813 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:29.849 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:29.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:29.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:29.950 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:29.988 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:29.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:29.989 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:30.091 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:30.124 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:30.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:30.125 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:30.226 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:30.274 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:30.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:30.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:30.375 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:30.417 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:30.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:30.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:30.519 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:30.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:30.551 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:30.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:30.652 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:30.693 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:30.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:30.694 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:30.795 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:30.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:30.832 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:30.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:30.845 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:55:30.933 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:30.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:30.974 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:30.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:31.075 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:31.113 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:31.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:31.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:31.215 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:31.249 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:31.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:31.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:31.351 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:31.387 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:31.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:31.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:31.488 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:31.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:31.522 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:31.523 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:31.623 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:31.664 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:31.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:31.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:31.765 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:31.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:31 smithi129 ceph-mon[101316]: pgmap v2855: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 884 op/s; 4350/137355 objects degraded (3.167%); 3646/137355 objects misplaced (2.654%); 2.6 MiB/s, 41 objects/s recovering 2024-09-14T04:55:31.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:31 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 4350/137355 objects degraded (3.167%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:55:31.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:31 smithi039 ceph-mon[106470]: pgmap v2855: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 884 op/s; 4350/137355 objects degraded (3.167%); 3646/137355 objects misplaced (2.654%); 2.6 MiB/s, 41 objects/s recovering 2024-09-14T04:55:31.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:31 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 4350/137355 objects degraded (3.167%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:55:31.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:31 smithi039 ceph-mon[111235]: pgmap v2855: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 884 op/s; 4350/137355 objects degraded (3.167%); 3646/137355 objects misplaced (2.654%); 2.6 MiB/s, 41 objects/s recovering 2024-09-14T04:55:31.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:31 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 4350/137355 objects degraded (3.167%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:55:31.829 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:31.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:31.830 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:31.931 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:31.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:31.967 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:31.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:32.028 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:55:32.029 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:55:32.068 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:32.096 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:32.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:32.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:32.198 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:32.213 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:55:32.213 INFO:teuthology.orchestra.run.smithi039.stdout:{"fsid":"bd7c803c-724b-11ef-bceb-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 4350/137355 objects degraded (3.167%), 2 pgs degraded, 2 pgs undersized","count":4},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":3247,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1509,"num_osds":8,"num_up_osds":8,"osd_up_since":1726289028,"num_in_osds":8,"osd_in_since":1726285856,"num_remapped_pgs":4},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":95},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":2},{"state_name":"active+remapped+backfill_wait","count":1},{"state_name":"active+recovering+undersized+remapped","count":1}],"num_pgs":99,"num_pools":5,"num_objects":45785,"data_bytes":2999392253,"bytes_used":14647148544,"bytes_avail":753313136640,"bytes_total":767960285184,"degraded_objects":4350,"degraded_total":137355,"degraded_ratio":0.031669760838702633,"misplaced_objects":3646,"misplaced_total":137355,"misplaced_ratio":0.026544355866186156,"recovering_objects_per_sec":41,"recovering_bytes_per_sec":2744895,"recovering_keys_per_sec":0,"num_objects_recovered":419,"num_bytes_recovered":27459584,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":884},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi039.bbwiwf","status":"up:active","gid":14646}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.39:8443/","prometheus":"http://172.21.15.39:9283/"}},"servicemap":{"epoch":40,"modified":"2024-09-14T04:53:57.838795+0000","services":{}},"progress_events":{}} 2024-09-14T04:55:32.230 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:32.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:32.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:32.332 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:32.373 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:32.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:32.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:32.475 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:32.505 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:32.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:32.506 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:32.607 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:32.641 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:32.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:32.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:32.744 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:32.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:32 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2186142292' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-14T04:55:32.788 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:32.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:32.789 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:32.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:32 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2186142292' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-14T04:55:32.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:32 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2186142292' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-14T04:55:32.891 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:32.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:32.927 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:32.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:33.028 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:33.060 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:33.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:33.062 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-14T04:55:33.062 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:55:33.062 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:33.101 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:55:33.104 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:55:33.163 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:33.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:33.225 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:33.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:33.326 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:33.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:33.367 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:33.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:33.468 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:33.512 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:33.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:33.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:33.614 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:33.650 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:33.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:33.651 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:33.696 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-14T04:55:33.753 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:33.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:33 smithi129 ceph-mon[101316]: pgmap v2856: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 884 op/s; 4350/137355 objects degraded (3.167%); 3646/137355 objects misplaced (2.654%); 2.6 MiB/s, 41 objects/s recovering 2024-09-14T04:55:33.771 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:33 smithi039 ceph-mon[106470]: pgmap v2856: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 884 op/s; 4350/137355 objects degraded (3.167%); 3646/137355 objects misplaced (2.654%); 2.6 MiB/s, 41 objects/s recovering 2024-09-14T04:55:33.771 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:33 smithi039 ceph-mon[111235]: pgmap v2856: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 884 op/s; 4350/137355 objects degraded (3.167%); 3646/137355 objects misplaced (2.654%); 2.6 MiB/s, 41 objects/s recovering 2024-09-14T04:55:33.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:33.808 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:33.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:33.909 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:33.957 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:33.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:33.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:34.059 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:34.102 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:34.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:34.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:34.204 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:34.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:34.241 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:34.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:34.342 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:34.377 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:34.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:34.379 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:34.479 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:34.517 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:34.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:34.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:34.618 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:34.652 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:34.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:34.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:34.753 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:34.788 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:34.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:34.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:34.890 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:34.925 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:34.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:34.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:35.027 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:35.071 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:35.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:35.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:35.173 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:35.211 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:35.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:35.212 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:35.313 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:35.350 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:35.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:35.352 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:35.453 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:35.485 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:35.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:35.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:35.587 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:35.631 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:35.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:35.632 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:35.733 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:35.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:35.765 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:35.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:35.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:35 smithi039 ceph-mon[106470]: pgmap v2857: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 883 op/s; 4350/130242 objects degraded (3.340%); 3646/130242 objects misplaced (2.799%); 2.6 MiB/s, 41 objects/s recovering 2024-09-14T04:55:35.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:35 smithi039 ceph-mon[111235]: pgmap v2857: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 883 op/s; 4350/130242 objects degraded (3.340%); 3646/130242 objects misplaced (2.799%); 2.6 MiB/s, 41 objects/s recovering 2024-09-14T04:55:35.866 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:35.913 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:35.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:35.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:36.015 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:36.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:35 smithi129 ceph-mon[101316]: pgmap v2857: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 883 op/s; 4350/130242 objects degraded (3.340%); 3646/130242 objects misplaced (2.799%); 2.6 MiB/s, 41 objects/s recovering 2024-09-14T04:55:36.048 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:36.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:36.050 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:36.150 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:36.193 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:36.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:36.194 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:36.295 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:36.343 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:36.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:36.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:36.445 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:36.486 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:36.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:36.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:36.588 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:36.622 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:36.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:36.623 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:36.724 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:36.757 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:36.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:36.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:36.761 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:36 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 4350/130242 objects degraded (3.340%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:55:36.761 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:36 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 4350/130242 objects degraded (3.340%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:55:36.859 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:36.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:36.894 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:36.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:36.995 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:37.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:36 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 4350/130242 objects degraded (3.340%), 2 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2024-09-14T04:55:37.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:37.046 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:37.047 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:37.147 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:37.175 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:37.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:37.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:37.277 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:37.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:37.305 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:37.305 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:37.406 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:37.446 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:37.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:37.447 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:37.548 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:37.580 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:37.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:37.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:37.605 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is unset 2024-09-14T04:55:37.682 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:37.714 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:37.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:37.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:37.816 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:37.842 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:55:37.855 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:37.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:37.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:37.957 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:37.960 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:37 smithi039 ceph-mon[106470]: pgmap v2858: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 875 op/s; 4350/124653 objects degraded (3.490%); 3171/124653 objects misplaced (2.544%); 2.7 MiB/s, 43 objects/s recovering 2024-09-14T04:55:37.960 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:37 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/527368816' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:55:37.961 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:55:37.961 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:37 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4280546756' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:55:37.961 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:37 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:55:37.961 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:37 smithi039 ceph-mon[111235]: pgmap v2858: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 875 op/s; 4350/124653 objects degraded (3.490%); 3171/124653 objects misplaced (2.544%); 2.7 MiB/s, 43 objects/s recovering 2024-09-14T04:55:37.961 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:37 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/527368816' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:55:37.961 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:55:37.961 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:37 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4280546756' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:55:37.961 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:37 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:55:37.962 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:55:37.962 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:55:37.962 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:55:37.962 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:55:37.962 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:55:37.962 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:55:37.962 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:55:37.962 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:55:37.962 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:55:37.963 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:55:37.963 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:55:37.963 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:55:37.963 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:55:37.963 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:55:37.963 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:55:37.963 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:55:37.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:37.996 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:37.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:37 smithi129 ceph-mon[101316]: pgmap v2858: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 875 op/s; 4350/124653 objects degraded (3.490%); 3171/124653 objects misplaced (2.544%); 2.7 MiB/s, 43 objects/s recovering 2024-09-14T04:55:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:37 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/527368816' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:55:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:55:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:37 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4280546756' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:55:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:37 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:55:38.097 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:38.134 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:38.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:38.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:38.236 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:38.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:38.266 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:38.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:38.367 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:38.397 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:38.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:38.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:38.499 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:38.535 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:38.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:38.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:38.636 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:38.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:38.677 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:38.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:38.778 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:38.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:38.807 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:38.807 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:38.908 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:38.918 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:38 smithi039 ceph-mon[106470]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:55:38.918 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:38 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:55:38.918 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:38 smithi039 ceph-mon[106470]: osdmap e1510: 8 total, 8 up, 8 in 2024-09-14T04:55:38.919 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:55:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:55:38] "GET /metrics HTTP/1.1" 200 36754 "" "Prometheus/2.51.0" 2024-09-14T04:55:38.919 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:38 smithi039 ceph-mon[111235]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:55:38.919 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:38 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:55:38.919 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:38 smithi039 ceph-mon[111235]: osdmap e1510: 8 total, 8 up, 8 in 2024-09-14T04:55:38.948 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:38.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:38.950 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:39.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:38 smithi129 ceph-mon[101316]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:55:39.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:38 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:55:39.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:38 smithi129 ceph-mon[101316]: osdmap e1510: 8 total, 8 up, 8 in 2024-09-14T04:55:39.051 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:39.094 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:39.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:39.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:39.196 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:39.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:39.236 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:39.236 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:39.337 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:39.376 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:39.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:39.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:39.478 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:39.518 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:39.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:39.519 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:39.620 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:39.653 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:39.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:39.654 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:39.755 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:39.794 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:39.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:39.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:39.896 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:39.931 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:39.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:39.932 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:40.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:39 smithi129 ceph-mon[101316]: pgmap v2860: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 875 op/s; 4350/124653 objects degraded (3.490%); 3171/124653 objects misplaced (2.544%); 2.7 MiB/s, 43 objects/s recovering 2024-09-14T04:55:40.033 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:40.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:39 smithi039 ceph-mon[106470]: pgmap v2860: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 875 op/s; 4350/124653 objects degraded (3.490%); 3171/124653 objects misplaced (2.544%); 2.7 MiB/s, 43 objects/s recovering 2024-09-14T04:55:40.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:39 smithi039 ceph-mon[111235]: pgmap v2860: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 875 op/s; 4350/124653 objects degraded (3.490%); 3171/124653 objects misplaced (2.544%); 2.7 MiB/s, 43 objects/s recovering 2024-09-14T04:55:40.079 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:40.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:40.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:40.105 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:55:40.182 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:40.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:40.227 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:40.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:40.328 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:40.362 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:40.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:40.363 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:40.464 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:40.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:40.500 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:40.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:40.601 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:40.629 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:40.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:40.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:40.731 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:40.776 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:40.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:40.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:40.797 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:55:40.878 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:40.911 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:40.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:40.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:41.013 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:41.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:41.062 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:41.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:41.163 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:41.203 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:41.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:41.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:41.305 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:41.338 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:41.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:41.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:41.441 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:41.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:41.478 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:41.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:41.579 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:41.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:41.609 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:41.609 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:41.710 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:41.744 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:41.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:41.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:41.846 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:41.879 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:41 smithi039 ceph-mon[106470]: pgmap v2861: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 4350/111384 objects degraded (3.905%); 2661/111384 objects misplaced (2.389%); 3.0 MiB/s, 48 objects/s recovering 2024-09-14T04:55:41.880 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:41 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 4350/111384 objects degraded (3.905%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:55:41.880 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:41 smithi039 ceph-mon[111235]: pgmap v2861: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 4350/111384 objects degraded (3.905%); 2661/111384 objects misplaced (2.389%); 3.0 MiB/s, 48 objects/s recovering 2024-09-14T04:55:41.880 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:41 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 4350/111384 objects degraded (3.905%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:55:41.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:41.896 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:41.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:41.997 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:42.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:41 smithi129 ceph-mon[101316]: pgmap v2861: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 4350/111384 objects degraded (3.905%); 2661/111384 objects misplaced (2.389%); 3.0 MiB/s, 48 objects/s recovering 2024-09-14T04:55:42.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:41 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 4350/111384 objects degraded (3.905%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:55:42.033 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:42.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:42.034 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:42.135 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:42.139 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:55:42.174 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:42.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:42.175 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:42.276 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:42.312 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:55:42.313 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T04:55:42.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:42.316 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:42.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:42.417 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:42.445 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:42.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:42.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:42.547 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:42.591 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:42.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:42.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:42.694 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:42.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:42.731 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:42.731 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:42.832 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:42.872 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:42.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:42.873 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:42.974 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:43.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:43.011 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:43.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:43.112 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:43.134 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:43.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:43.135 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:43.236 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:43.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:43.277 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:43.277 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:43.378 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:43.418 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:43.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:43.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:43.520 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:43.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:43.560 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:43.561 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:43.662 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:43.706 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:43.707 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:43.707 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:43.808 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:43.843 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:43.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:43.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:43.944 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:43.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:43.982 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:43.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:44.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:43 smithi129 ceph-mon[101316]: pgmap v2862: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 4350/111384 objects degraded (3.905%); 2661/111384 objects misplaced (2.389%); 3.0 MiB/s, 48 objects/s recovering 2024-09-14T04:55:44.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:43 smithi129 ceph-mon[101316]: from='client.46175 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:55:44.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:43 smithi039 ceph-mon[106470]: pgmap v2862: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 4350/111384 objects degraded (3.905%); 2661/111384 objects misplaced (2.389%); 3.0 MiB/s, 48 objects/s recovering 2024-09-14T04:55:44.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:43 smithi039 ceph-mon[106470]: from='client.46175 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:55:44.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:43 smithi039 ceph-mon[111235]: pgmap v2862: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.3 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 4350/111384 objects degraded (3.905%); 2661/111384 objects misplaced (2.389%); 3.0 MiB/s, 48 objects/s recovering 2024-09-14T04:55:44.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:43 smithi039 ceph-mon[111235]: from='client.46175 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:55:44.083 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:44.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:44.122 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:44.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:44.223 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:44.262 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:44.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:44.263 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:44.364 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:44.387 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:44.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:44.388 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:44.489 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:44.527 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:44.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:44.528 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:44.629 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:44.667 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:44.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:44.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:44.769 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:44.807 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:44.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:44.808 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:44.909 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:44.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:44.953 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:44.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:45.016 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":2862,"stamp":"2024-09-14T04:55:41.878165+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2432047101,"num_objects":37128,"num_object_clones":0,"num_object_copies":111384,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4350,"num_objects_misplaced":2661,"num_objects_unfound":0,"num_objects_dirty":37128,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":192006,"num_write_kb":7329378,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9482,"num_bytes_recovered":619060302,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":127094,"ondisk_log_size":127094,"up":297,"acting":294,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":306,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12692708,"kb_used_data":7925468,"kb_used_omap":404,"kb_used_meta":4766443,"kb_avail":737268508,"statfs":{"total":767960285184,"available":754962952192,"internally_reserved":0,"allocated":8115679232,"data_stored":8074101040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":414022,"internal_metadata":4880838330},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,0,1,4,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-567345152,"num_objects":-8657,"num_object_clones":0,"num_object_copies":-25971,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-985,"num_objects_unfound":0,"num_objects_dirty":-8657,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8657,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":483,"num_bytes_recovered":31653888,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006388"},"pg_stats":[{"pgid":"56.f","version":"1510'11861","reported_seq":11911,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:38.244834+0000","last_change":"2024-09-14T04:52:16.661278+0000","last_active":"2024-09-14T04:55:38.244834+0000","last_peered":"2024-09-14T04:55:38.244834+0000","last_clean":"2024-09-14T04:55:38.244834+0000","last_became_active":"2024-09-14T04:52:16.660906+0000","last_became_peered":"2024-09-14T04:52:16.660906+0000","last_unstale":"2024-09-14T04:55:38.244834+0000","last_undegraded":"2024-09-14T04:55:38.244834+0000","last_fullsized":"2024-09-14T04:55:38.244834+0000","mapping_epoch":1470,"log_start":"1481'4800","ondisk_log_start":"1481'4800","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":7061,"log_dups_size":0,"ondisk_log_size":7061,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:55:16.908898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152109056,"num_objects":2321,"num_object_clones":0,"num_object_copies":6963,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2321,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11861,"num_write_kb":453824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"56.e","version":"1510'12097","reported_seq":12147,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:38.755278+0000","last_change":"2024-09-14T04:52:16.723745+0000","last_active":"2024-09-14T04:55:38.755278+0000","last_peered":"2024-09-14T04:55:38.755278+0000","last_clean":"2024-09-14T04:55:38.755278+0000","last_became_active":"2024-09-14T04:52:16.723416+0000","last_became_peered":"2024-09-14T04:52:16.723416+0000","last_unstale":"2024-09-14T04:55:38.755278+0000","last_undegraded":"2024-09-14T04:55:38.755278+0000","last_fullsized":"2024-09-14T04:55:38.755278+0000","mapping_epoch":1470,"log_start":"1481'4747","ondisk_log_start":"1481'4747","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":7350,"log_dups_size":0,"ondisk_log_size":7350,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:20:17.253535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155910144,"num_objects":2379,"num_object_clones":0,"num_object_copies":7137,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2379,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12097,"num_write_kb":463232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"56.d","version":"1510'12019","reported_seq":12112,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:38.691865+0000","last_change":"2024-09-14T04:52:16.871824+0000","last_active":"2024-09-14T04:55:38.691865+0000","last_peered":"2024-09-14T04:55:38.691865+0000","last_clean":"2024-09-14T04:55:38.691865+0000","last_became_active":"2024-09-14T04:52:16.871446+0000","last_became_peered":"2024-09-14T04:52:16.871446+0000","last_unstale":"2024-09-14T04:55:38.691865+0000","last_undegraded":"2024-09-14T04:55:38.691865+0000","last_fullsized":"2024-09-14T04:55:38.691865+0000","mapping_epoch":1470,"log_start":"1474'3400","ondisk_log_start":"1474'3400","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8619,"log_dups_size":0,"ondisk_log_size":8619,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:44:00.576543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156237848,"num_objects":2385,"num_object_clones":0,"num_object_copies":7155,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2385,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12019,"num_write_kb":460865,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"56.c","version":"1510'12155","reported_seq":12205,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:39.454984+0000","last_change":"2024-09-14T04:52:16.723896+0000","last_active":"2024-09-14T04:55:39.454984+0000","last_peered":"2024-09-14T04:55:39.454984+0000","last_clean":"2024-09-14T04:55:39.454984+0000","last_became_active":"2024-09-14T04:52:16.723597+0000","last_became_peered":"2024-09-14T04:52:16.723597+0000","last_unstale":"2024-09-14T04:55:39.454984+0000","last_undegraded":"2024-09-14T04:55:39.454984+0000","last_fullsized":"2024-09-14T04:55:39.454984+0000","mapping_epoch":1470,"log_start":"1480'4008","ondisk_log_start":"1480'4008","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8147,"log_dups_size":0,"ondisk_log_size":8147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:10:31.058435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148701184,"num_objects":2269,"num_object_clones":0,"num_object_copies":6807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12155,"num_write_kb":461568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.b","version":"1510'11992","reported_seq":12042,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:38.239033+0000","last_change":"2024-09-14T04:52:16.660625+0000","last_active":"2024-09-14T04:55:38.239033+0000","last_peered":"2024-09-14T04:55:38.239033+0000","last_clean":"2024-09-14T04:55:38.239033+0000","last_became_active":"2024-09-14T04:52:16.660277+0000","last_became_peered":"2024-09-14T04:52:16.660277+0000","last_unstale":"2024-09-14T04:55:38.239033+0000","last_undegraded":"2024-09-14T04:55:38.239033+0000","last_fullsized":"2024-09-14T04:55:38.239033+0000","mapping_epoch":1470,"log_start":"1481'5000","ondisk_log_start":"1481'5000","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":6992,"log_dups_size":0,"ondisk_log_size":6992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:43:12.659725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155713536,"num_objects":2376,"num_object_clones":0,"num_object_copies":7128,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2376,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11992,"num_write_kb":459776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"56.a","version":"1510'11945","reported_seq":11995,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:38.276807+0000","last_change":"2024-09-14T04:52:16.915139+0000","last_active":"2024-09-14T04:55:38.276807+0000","last_peered":"2024-09-14T04:55:38.276807+0000","last_clean":"2024-09-14T04:55:38.276807+0000","last_became_active":"2024-09-14T04:52:16.914776+0000","last_became_peered":"2024-09-14T04:52:16.914776+0000","last_unstale":"2024-09-14T04:55:38.276807+0000","last_undegraded":"2024-09-14T04:55:38.276807+0000","last_fullsized":"2024-09-14T04:55:38.276807+0000","mapping_epoch":1470,"log_start":"1481'4900","ondisk_log_start":"1481'4900","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":7045,"log_dups_size":0,"ondisk_log_size":7045,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:08:58.079681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152895488,"num_objects":2333,"num_object_clones":0,"num_object_copies":6999,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2333,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11945,"num_write_kb":456896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"56.9","version":"1510'12094","reported_seq":12144,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:39.426516+0000","last_change":"2024-09-14T04:52:16.723789+0000","last_active":"2024-09-14T04:55:39.426516+0000","last_peered":"2024-09-14T04:55:39.426516+0000","last_clean":"2024-09-14T04:55:39.426516+0000","last_became_active":"2024-09-14T04:52:16.723394+0000","last_became_peered":"2024-09-14T04:52:16.723394+0000","last_unstale":"2024-09-14T04:55:39.426516+0000","last_undegraded":"2024-09-14T04:55:39.426516+0000","last_fullsized":"2024-09-14T04:55:39.426516+0000","mapping_epoch":1470,"log_start":"1480'3970","ondisk_log_start":"1480'3970","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8124,"log_dups_size":0,"ondisk_log_size":8124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:45:07.892366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146538496,"num_objects":2236,"num_object_clones":0,"num_object_copies":6708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12094,"num_write_kb":458560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.8","version":"1510'11725","reported_seq":11817,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:38.666666+0000","last_change":"2024-09-14T04:52:16.915103+0000","last_active":"2024-09-14T04:55:38.666666+0000","last_peered":"2024-09-14T04:55:38.666666+0000","last_clean":"2024-09-14T04:55:38.666666+0000","last_became_active":"2024-09-14T04:52:16.914657+0000","last_became_peered":"2024-09-14T04:52:16.914657+0000","last_unstale":"2024-09-14T04:55:38.666666+0000","last_undegraded":"2024-09-14T04:55:38.666666+0000","last_fullsized":"2024-09-14T04:55:38.666666+0000","mapping_epoch":1470,"log_start":"1474'3100","ondisk_log_start":"1474'3100","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8625,"log_dups_size":0,"ondisk_log_size":8625,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:08:43.898740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":149880832,"num_objects":2287,"num_object_clones":0,"num_object_copies":6861,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2287,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11725,"num_write_kb":448384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"56.7","version":"1510'11968","reported_seq":12060,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:38.707974+0000","last_change":"2024-09-14T04:52:16.723670+0000","last_active":"2024-09-14T04:55:38.707974+0000","last_peered":"2024-09-14T04:55:38.707974+0000","last_clean":"2024-09-14T04:55:38.707974+0000","last_became_active":"2024-09-14T04:52:16.723292+0000","last_became_peered":"2024-09-14T04:52:16.723292+0000","last_unstale":"2024-09-14T04:55:38.707974+0000","last_undegraded":"2024-09-14T04:55:38.707974+0000","last_fullsized":"2024-09-14T04:55:38.707974+0000","mapping_epoch":1470,"log_start":"1474'3300","ondisk_log_start":"1474'3300","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8668,"log_dups_size":0,"ondisk_log_size":8668,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:55:51.122910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152829952,"num_objects":2332,"num_object_clones":0,"num_object_copies":6996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11968,"num_write_kb":457600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"56.6","version":"1510'11802","reported_seq":17994,"reported_epoch":1510,"state":"active+recovering+undersized+remapped","last_fresh":"2024-09-14T04:55:39.476425+0000","last_change":"2024-09-14T04:54:19.620795+0000","last_active":"2024-09-14T04:55:39.476425+0000","last_peered":"2024-09-14T04:55:39.476425+0000","last_clean":"2024-09-14T04:52:58.397671+0000","last_became_active":"2024-09-14T04:54:19.575869+0000","last_became_peered":"2024-09-14T04:54:19.575869+0000","last_unstale":"2024-09-14T04:55:39.476425+0000","last_undegraded":"2024-09-14T04:55:39.476425+0000","last_fullsized":"2024-09-14T04:54:19.444266+0000","mapping_epoch":1489,"log_start":"1478'3657","ondisk_log_start":"1478'3657","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8145,"log_dups_size":0,"ondisk_log_size":8145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:53.013009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":144048128,"num_objects":2198,"num_object_clones":0,"num_object_copies":6594,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":366,"num_objects_unfound":0,"num_objects_dirty":2198,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11802,"num_write_kb":448000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3063,"num_bytes_recovered":200736768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"4,5,7","objects":366},{"shards":"5,7","objects":1832}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.5","version":"1510'12029","reported_seq":12079,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:38.763779+0000","last_change":"2024-09-14T04:52:16.871791+0000","last_active":"2024-09-14T04:55:38.763779+0000","last_peered":"2024-09-14T04:55:38.763779+0000","last_clean":"2024-09-14T04:55:38.763779+0000","last_became_active":"2024-09-14T04:52:16.871407+0000","last_became_peered":"2024-09-14T04:52:16.871407+0000","last_unstale":"2024-09-14T04:55:38.763779+0000","last_undegraded":"2024-09-14T04:55:38.763779+0000","last_fullsized":"2024-09-14T04:55:38.763779+0000","mapping_epoch":1470,"log_start":"1481'4667","ondisk_log_start":"1481'4667","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":7362,"log_dups_size":0,"ondisk_log_size":7362,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:47:15.419674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152764416,"num_objects":2331,"num_object_clones":0,"num_object_copies":6993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12029,"num_write_kb":459520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"56.4","version":"1510'12111","reported_seq":12161,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:38.226161+0000","last_change":"2024-09-14T04:52:16.724077+0000","last_active":"2024-09-14T04:55:38.226161+0000","last_peered":"2024-09-14T04:55:38.226161+0000","last_clean":"2024-09-14T04:55:38.226161+0000","last_became_active":"2024-09-14T04:52:16.723597+0000","last_became_peered":"2024-09-14T04:52:16.723597+0000","last_unstale":"2024-09-14T04:55:38.226161+0000","last_undegraded":"2024-09-14T04:55:38.226161+0000","last_fullsized":"2024-09-14T04:55:38.226161+0000","mapping_epoch":1470,"log_start":"1481'5100","ondisk_log_start":"1481'5100","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":7011,"log_dups_size":0,"ondisk_log_size":7011,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:23:49.557180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":156041216,"num_objects":2381,"num_object_clones":0,"num_object_copies":7143,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2381,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12111,"num_write_kb":463744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"56.3","version":"1510'11901","reported_seq":19170,"reported_epoch":1510,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:55:39.408312+0000","last_change":"2024-09-14T04:54:53.717396+0000","last_active":"2024-09-14T04:55:39.408312+0000","last_peered":"2024-09-14T04:55:39.408312+0000","last_clean":"2024-09-14T04:54:51.200599+0000","last_became_active":"2024-09-14T04:54:53.681745+0000","last_became_peered":"2024-09-14T04:54:53.681745+0000","last_unstale":"2024-09-14T04:55:39.408312+0000","last_undegraded":"2024-09-14T04:55:39.408312+0000","last_fullsized":"2024-09-14T04:55:39.408312+0000","mapping_epoch":1504,"log_start":"1480'4117","ondisk_log_start":"1480'4117","created":1470,"last_epoch_clean":1496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":7784,"log_dups_size":0,"ondisk_log_size":7784,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":150405120,"num_objects":2295,"num_object_clones":0,"num_object_copies":6885,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2295,"num_objects_unfound":0,"num_objects_dirty":2295,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11901,"num_write_kb":454272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3597,"num_bytes_recovered":235732992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[1,6,5],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":2295}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"56.2","version":"1510'12266","reported_seq":17870,"reported_epoch":1510,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-14T04:55:39.452147+0000","last_change":"2024-09-14T04:54:19.618218+0000","last_active":"2024-09-14T04:55:39.452147+0000","last_peered":"2024-09-14T04:55:39.452147+0000","last_clean":"2024-09-14T04:52:58.403685+0000","last_became_active":"2024-09-14T04:54:19.593740+0000","last_became_peered":"2024-09-14T04:54:19.593740+0000","last_unstale":"2024-09-14T04:55:39.452147+0000","last_undegraded":"2024-09-14T04:54:19.466599+0000","last_fullsized":"2024-09-14T04:54:19.444256+0000","mapping_epoch":1489,"log_start":"1480'4137","ondisk_log_start":"1480'4137","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8129,"log_dups_size":0,"ondisk_log_size":8129,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:45:11.467642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148242432,"num_objects":2262,"num_object_clones":0,"num_object_copies":6786,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4349,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12266,"num_write_kb":464896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2770,"num_bytes_recovered":181534720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"0,5,7","objects":2262},{"shards":"5,7","objects":-1289}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.1","version":"1510'12056","reported_seq":12106,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:39.463423+0000","last_change":"2024-09-14T04:52:16.872703+0000","last_active":"2024-09-14T04:55:39.463423+0000","last_peered":"2024-09-14T04:55:39.463423+0000","last_clean":"2024-09-14T04:55:39.463423+0000","last_became_active":"2024-09-14T04:52:16.871779+0000","last_became_peered":"2024-09-14T04:52:16.871779+0000","last_unstale":"2024-09-14T04:55:39.463423+0000","last_undegraded":"2024-09-14T04:55:39.463423+0000","last_fullsized":"2024-09-14T04:55:39.463423+0000","mapping_epoch":1470,"log_start":"1480'3865","ondisk_log_start":"1480'3865","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8191,"log_dups_size":0,"ondisk_log_size":8191,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:23:30.260775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":153878528,"num_objects":2348,"num_object_clones":0,"num_object_copies":7044,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2348,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12056,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.0","version":"1510'11921","reported_seq":11971,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:39.449153+0000","last_change":"2024-09-14T04:52:16.915325+0000","last_active":"2024-09-14T04:55:39.449153+0000","last_peered":"2024-09-14T04:55:39.449153+0000","last_clean":"2024-09-14T04:55:39.449153+0000","last_became_active":"2024-09-14T04:52:16.914980+0000","last_became_peered":"2024-09-14T04:52:16.914980+0000","last_unstale":"2024-09-14T04:55:39.449153+0000","last_undegraded":"2024-09-14T04:55:39.449153+0000","last_fullsized":"2024-09-14T04:55:39.449153+0000","mapping_epoch":1470,"log_start":"1479'3778","ondisk_log_start":"1479'3778","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8143,"log_dups_size":0,"ondisk_log_size":8143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:55:08.752960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":155254784,"num_objects":2369,"num_object_clones":0,"num_object_copies":7107,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2369,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11921,"num_write_kb":457280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":2312,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.563070+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T04:55:37.563070+0000","last_peered":"2024-09-14T04:55:37.563070+0000","last_clean":"2024-09-14T04:55:37.563070+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T04:55:37.563070+0000","last_undegraded":"2024-09-14T04:55:37.563070+0000","last_fullsized":"2024-09-14T04:55:37.563070+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":122,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.481292+0000","last_change":"2024-09-14T04:44:57.325109+0000","last_active":"2024-09-14T04:55:27.481292+0000","last_peered":"2024-09-14T04:55:27.481292+0000","last_clean":"2024-09-14T04:55:27.481292+0000","last_became_active":"2024-09-14T04:44:57.324791+0000","last_became_peered":"2024-09-14T04:44:57.324791+0000","last_unstale":"2024-09-14T04:55:27.481292+0000","last_undegraded":"2024-09-14T04:55:27.481292+0000","last_fullsized":"2024-09-14T04:55:27.481292+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:03.593275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":92,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:33.848621+0000","last_change":"2024-09-14T04:49:29.628435+0000","last_active":"2024-09-14T04:55:33.848621+0000","last_peered":"2024-09-14T04:55:33.848621+0000","last_clean":"2024-09-14T04:55:33.848621+0000","last_became_active":"2024-09-14T04:49:29.628081+0000","last_became_peered":"2024-09-14T04:49:29.628081+0000","last_unstale":"2024-09-14T04:55:33.848621+0000","last_undegraded":"2024-09-14T04:55:33.848621+0000","last_fullsized":"2024-09-14T04:55:33.848621+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:01.520334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":934,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.554526+0000","last_change":"2024-09-14T04:43:13.355281+0000","last_active":"2024-09-14T04:55:37.554526+0000","last_peered":"2024-09-14T04:55:37.554526+0000","last_clean":"2024-09-14T04:55:37.554526+0000","last_became_active":"2024-09-14T04:43:13.354967+0000","last_became_peered":"2024-09-14T04:43:13.354967+0000","last_unstale":"2024-09-14T04:55:37.554526+0000","last_undegraded":"2024-09-14T04:55:37.554526+0000","last_fullsized":"2024-09-14T04:55:37.554526+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:23:19.332075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1501,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.546776+0000","last_change":"2024-09-14T04:44:57.342918+0000","last_active":"2024-09-14T04:55:37.546776+0000","last_peered":"2024-09-14T04:55:37.546776+0000","last_clean":"2024-09-14T04:55:37.546776+0000","last_became_active":"2024-09-14T04:44:57.329550+0000","last_became_peered":"2024-09-14T04:44:57.329550+0000","last_unstale":"2024-09-14T04:55:37.546776+0000","last_undegraded":"2024-09-14T04:55:37.546776+0000","last_fullsized":"2024-09-14T04:55:37.546776+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:52:52.435893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":122,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.481705+0000","last_change":"2024-09-14T04:44:57.325537+0000","last_active":"2024-09-14T04:55:27.481705+0000","last_peered":"2024-09-14T04:55:27.481705+0000","last_clean":"2024-09-14T04:55:27.481705+0000","last_became_active":"2024-09-14T04:44:57.325057+0000","last_became_peered":"2024-09-14T04:44:57.325057+0000","last_unstale":"2024-09-14T04:55:27.481705+0000","last_undegraded":"2024-09-14T04:55:27.481705+0000","last_fullsized":"2024-09-14T04:55:27.481705+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:43:23.166708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1516,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.546703+0000","last_change":"2024-09-14T04:44:57.337033+0000","last_active":"2024-09-14T04:55:37.546703+0000","last_peered":"2024-09-14T04:55:37.546703+0000","last_clean":"2024-09-14T04:55:37.546703+0000","last_became_active":"2024-09-14T04:44:57.328437+0000","last_became_peered":"2024-09-14T04:44:57.328437+0000","last_unstale":"2024-09-14T04:55:37.546703+0000","last_undegraded":"2024-09-14T04:55:37.546703+0000","last_fullsized":"2024-09-14T04:55:37.546703+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.109086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":122,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.482703+0000","last_change":"2024-09-14T04:44:57.332243+0000","last_active":"2024-09-14T04:55:27.482703+0000","last_peered":"2024-09-14T04:55:27.482703+0000","last_clean":"2024-09-14T04:55:27.482703+0000","last_became_active":"2024-09-14T04:44:57.330413+0000","last_became_peered":"2024-09-14T04:44:57.330413+0000","last_unstale":"2024-09-14T04:55:27.482703+0000","last_undegraded":"2024-09-14T04:55:27.482703+0000","last_fullsized":"2024-09-14T04:55:27.482703+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:25:01.338549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":168,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.557200+0000","last_change":"2024-09-14T04:48:29.862929+0000","last_active":"2024-09-14T04:55:37.557200+0000","last_peered":"2024-09-14T04:55:37.557200+0000","last_clean":"2024-09-14T04:55:37.557200+0000","last_became_active":"2024-09-14T04:48:29.862581+0000","last_became_peered":"2024-09-14T04:48:29.862581+0000","last_unstale":"2024-09-14T04:55:37.557200+0000","last_undegraded":"2024-09-14T04:55:37.557200+0000","last_fullsized":"2024-09-14T04:55:37.557200+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:16:43.060088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2308,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.555628+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T04:55:37.555628+0000","last_peered":"2024-09-14T04:55:37.555628+0000","last_clean":"2024-09-14T04:55:37.555628+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T04:55:37.555628+0000","last_undegraded":"2024-09-14T04:55:37.555628+0000","last_fullsized":"2024-09-14T04:55:37.555628+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":122,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.481667+0000","last_change":"2024-09-14T04:44:57.325392+0000","last_active":"2024-09-14T04:55:27.481667+0000","last_peered":"2024-09-14T04:55:27.481667+0000","last_clean":"2024-09-14T04:55:27.481667+0000","last_became_active":"2024-09-14T04:44:57.324624+0000","last_became_peered":"2024-09-14T04:44:57.324624+0000","last_unstale":"2024-09-14T04:55:27.481667+0000","last_undegraded":"2024-09-14T04:55:27.481667+0000","last_fullsized":"2024-09-14T04:55:27.481667+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:33:18.810199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2276,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.560772+0000","last_change":"2024-09-14T04:44:57.329113+0000","last_active":"2024-09-14T04:55:37.560772+0000","last_peered":"2024-09-14T04:55:37.560772+0000","last_clean":"2024-09-14T04:55:37.560772+0000","last_became_active":"2024-09-14T04:44:57.328727+0000","last_became_peered":"2024-09-14T04:44:57.328727+0000","last_unstale":"2024-09-14T04:55:37.560772+0000","last_undegraded":"2024-09-14T04:55:37.560772+0000","last_fullsized":"2024-09-14T04:55:37.560772+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:34.939874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":2294,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.558781+0000","last_change":"2024-09-14T04:44:57.329277+0000","last_active":"2024-09-14T04:55:37.558781+0000","last_peered":"2024-09-14T04:55:37.558781+0000","last_clean":"2024-09-14T04:55:37.558781+0000","last_became_active":"2024-09-14T04:44:57.328804+0000","last_became_peered":"2024-09-14T04:44:57.328804+0000","last_unstale":"2024-09-14T04:55:37.558781+0000","last_undegraded":"2024-09-14T04:55:37.558781+0000","last_fullsized":"2024-09-14T04:55:37.558781+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:42:46.055240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2328,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.558335+0000","last_change":"2024-09-14T04:54:52.250695+0000","last_active":"2024-09-14T04:55:37.558335+0000","last_peered":"2024-09-14T04:55:37.558335+0000","last_clean":"2024-09-14T04:55:37.558335+0000","last_became_active":"2024-09-14T04:54:52.250301+0000","last_became_peered":"2024-09-14T04:54:52.250301+0000","last_unstale":"2024-09-14T04:55:37.558335+0000","last_undegraded":"2024-09-14T04:55:37.558335+0000","last_fullsized":"2024-09-14T04:55:37.558335+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:42.060905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2290,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.557389+0000","last_change":"2024-09-14T04:43:13.357092+0000","last_active":"2024-09-14T04:55:37.557389+0000","last_peered":"2024-09-14T04:55:37.557389+0000","last_clean":"2024-09-14T04:55:37.557389+0000","last_became_active":"2024-09-14T04:43:13.356694+0000","last_became_peered":"2024-09-14T04:43:13.356694+0000","last_unstale":"2024-09-14T04:55:37.557389+0000","last_undegraded":"2024-09-14T04:55:37.557389+0000","last_fullsized":"2024-09-14T04:55:37.557389+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:11.167574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1494,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:33.848648+0000","last_change":"2024-09-14T04:07:21.720797+0000","last_active":"2024-09-14T04:55:33.848648+0000","last_peered":"2024-09-14T04:55:33.848648+0000","last_clean":"2024-09-14T04:55:33.848648+0000","last_became_active":"2024-09-14T04:07:21.719331+0000","last_became_peered":"2024-09-14T04:07:21.719331+0000","last_unstale":"2024-09-14T04:55:33.848648+0000","last_undegraded":"2024-09-14T04:55:33.848648+0000","last_fullsized":"2024-09-14T04:55:33.848648+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:35.404023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":141,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.485576+0000","last_change":"2024-09-14T04:43:11.641514+0000","last_active":"2024-09-14T04:55:27.485576+0000","last_peered":"2024-09-14T04:55:27.485576+0000","last_clean":"2024-09-14T04:55:27.485576+0000","last_became_active":"2024-09-14T04:43:11.641151+0000","last_became_peered":"2024-09-14T04:43:11.641151+0000","last_unstale":"2024-09-14T04:55:27.485576+0000","last_undegraded":"2024-09-14T04:55:27.485576+0000","last_fullsized":"2024-09-14T04:55:27.485576+0000","mapping_epoch":1384,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:06:45.045430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2301,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.563232+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T04:55:37.563232+0000","last_peered":"2024-09-14T04:55:37.563232+0000","last_clean":"2024-09-14T04:55:37.563232+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T04:55:37.563232+0000","last_undegraded":"2024-09-14T04:55:37.563232+0000","last_fullsized":"2024-09-14T04:55:37.563232+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":721,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.490813+0000","last_change":"2024-09-14T04:49:29.618477+0000","last_active":"2024-09-14T04:55:27.490813+0000","last_peered":"2024-09-14T04:55:27.490813+0000","last_clean":"2024-09-14T04:55:27.490813+0000","last_became_active":"2024-09-14T04:49:29.618210+0000","last_became_peered":"2024-09-14T04:49:29.618210+0000","last_unstale":"2024-09-14T04:55:27.490813+0000","last_undegraded":"2024-09-14T04:55:27.490813+0000","last_fullsized":"2024-09-14T04:55:27.490813+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:13:23.895512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1499,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.488729+0000","last_change":"2024-09-14T04:42:49.938505+0000","last_active":"2024-09-14T04:55:27.488729+0000","last_peered":"2024-09-14T04:55:27.488729+0000","last_clean":"2024-09-14T04:55:27.488729+0000","last_became_active":"2024-09-14T04:42:49.938180+0000","last_became_peered":"2024-09-14T04:42:49.938180+0000","last_unstale":"2024-09-14T04:55:27.488729+0000","last_undegraded":"2024-09-14T04:55:27.488729+0000","last_fullsized":"2024-09-14T04:55:27.488729+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:05.929479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1518,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.491463+0000","last_change":"2024-09-14T04:44:57.346582+0000","last_active":"2024-09-14T04:55:27.491463+0000","last_peered":"2024-09-14T04:55:27.491463+0000","last_clean":"2024-09-14T04:55:27.491463+0000","last_became_active":"2024-09-14T04:44:57.330446+0000","last_became_peered":"2024-09-14T04:44:57.330446+0000","last_unstale":"2024-09-14T04:55:27.491463+0000","last_undegraded":"2024-09-14T04:55:27.491463+0000","last_fullsized":"2024-09-14T04:55:27.491463+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:29.238585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":242,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.561259+0000","last_change":"2024-09-14T04:44:14.480667+0000","last_active":"2024-09-14T04:55:37.561259+0000","last_peered":"2024-09-14T04:55:37.561259+0000","last_clean":"2024-09-14T04:55:37.561259+0000","last_became_active":"2024-09-14T04:44:14.480365+0000","last_became_peered":"2024-09-14T04:44:14.480365+0000","last_unstale":"2024-09-14T04:55:37.561259+0000","last_undegraded":"2024-09-14T04:55:37.561259+0000","last_fullsized":"2024-09-14T04:55:37.561259+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:00.611771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":132,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.483098+0000","last_change":"2024-09-14T04:44:57.341426+0000","last_active":"2024-09-14T04:55:27.483098+0000","last_peered":"2024-09-14T04:55:27.483098+0000","last_clean":"2024-09-14T04:55:27.483098+0000","last_became_active":"2024-09-14T04:44:57.328763+0000","last_became_peered":"2024-09-14T04:44:57.328763+0000","last_unstale":"2024-09-14T04:55:27.483098+0000","last_undegraded":"2024-09-14T04:55:27.483098+0000","last_fullsized":"2024-09-14T04:55:27.483098+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:34:40.066761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":122,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.482849+0000","last_change":"2024-09-14T04:44:57.331337+0000","last_active":"2024-09-14T04:55:27.482849+0000","last_peered":"2024-09-14T04:55:27.482849+0000","last_clean":"2024-09-14T04:55:27.482849+0000","last_became_active":"2024-09-14T04:44:57.329336+0000","last_became_peered":"2024-09-14T04:44:57.329336+0000","last_unstale":"2024-09-14T04:55:27.482849+0000","last_undegraded":"2024-09-14T04:55:27.482849+0000","last_fullsized":"2024-09-14T04:55:27.482849+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:52:35.980869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"349'149","reported_seq":710,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.486270+0000","last_change":"2024-09-14T04:49:30.085274+0000","last_active":"2024-09-14T04:55:27.486270+0000","last_peered":"2024-09-14T04:55:27.486270+0000","last_clean":"2024-09-14T04:55:27.486270+0000","last_became_active":"2024-09-14T04:49:29.623295+0000","last_became_peered":"2024-09-14T04:49:29.623295+0000","last_unstale":"2024-09-14T04:55:27.486270+0000","last_undegraded":"2024-09-14T04:55:27.486270+0000","last_fullsized":"2024-09-14T04:55:27.486270+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:49:30.085089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:49:30.085089+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:10.012963+0000","scrub_duration":0.0071928790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":123,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.481434+0000","last_change":"2024-09-14T04:44:57.329870+0000","last_active":"2024-09-14T04:55:27.481434+0000","last_peered":"2024-09-14T04:55:27.481434+0000","last_clean":"2024-09-14T04:55:27.481434+0000","last_became_active":"2024-09-14T04:44:57.329353+0000","last_became_peered":"2024-09-14T04:44:57.329353+0000","last_unstale":"2024-09-14T04:55:27.481434+0000","last_undegraded":"2024-09-14T04:55:27.481434+0000","last_fullsized":"2024-09-14T04:55:27.481434+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:46.432181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1515,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.483789+0000","last_change":"2024-09-14T04:54:52.242207+0000","last_active":"2024-09-14T04:55:27.483789+0000","last_peered":"2024-09-14T04:55:27.483789+0000","last_clean":"2024-09-14T04:55:27.483789+0000","last_became_active":"2024-09-14T04:54:52.241840+0000","last_became_peered":"2024-09-14T04:54:52.241840+0000","last_unstale":"2024-09-14T04:55:27.483789+0000","last_undegraded":"2024-09-14T04:55:27.483789+0000","last_fullsized":"2024-09-14T04:55:27.483789+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:37.899666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":2339,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.558064+0000","last_change":"2024-09-14T04:44:57.344614+0000","last_active":"2024-09-14T04:55:37.558064+0000","last_peered":"2024-09-14T04:55:37.558064+0000","last_clean":"2024-09-14T04:55:37.558064+0000","last_became_active":"2024-09-14T04:44:57.332256+0000","last_became_peered":"2024-09-14T04:44:57.332256+0000","last_unstale":"2024-09-14T04:55:37.558064+0000","last_undegraded":"2024-09-14T04:55:37.558064+0000","last_fullsized":"2024-09-14T04:55:37.558064+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:23.438417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"349'149","reported_seq":881,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:28.291836+0000","last_change":"2024-09-14T04:45:00.331854+0000","last_active":"2024-09-14T04:55:28.291836+0000","last_peered":"2024-09-14T04:55:28.291836+0000","last_clean":"2024-09-14T04:55:28.291836+0000","last_became_active":"2024-09-14T04:45:00.331460+0000","last_became_peered":"2024-09-14T04:45:00.331460+0000","last_unstale":"2024-09-14T04:55:28.291836+0000","last_undegraded":"2024-09-14T04:55:28.291836+0000","last_fullsized":"2024-09-14T04:55:28.291836+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.567535+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.567535+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:45.095729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":712,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:33.773332+0000","last_change":"2024-09-14T04:49:29.616831+0000","last_active":"2024-09-14T04:55:33.773332+0000","last_peered":"2024-09-14T04:55:33.773332+0000","last_clean":"2024-09-14T04:55:33.773332+0000","last_became_active":"2024-09-14T04:49:29.616546+0000","last_became_peered":"2024-09-14T04:49:29.616546+0000","last_unstale":"2024-09-14T04:55:33.773332+0000","last_undegraded":"2024-09-14T04:55:33.773332+0000","last_fullsized":"2024-09-14T04:55:33.773332+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:27.882929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":294,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.562867+0000","last_change":"2024-09-14T04:44:57.330746+0000","last_active":"2024-09-14T04:55:37.562867+0000","last_peered":"2024-09-14T04:55:37.562867+0000","last_clean":"2024-09-14T04:55:37.562867+0000","last_became_active":"2024-09-14T04:44:57.330327+0000","last_became_peered":"2024-09-14T04:44:57.330327+0000","last_unstale":"2024-09-14T04:55:37.562867+0000","last_undegraded":"2024-09-14T04:55:37.562867+0000","last_fullsized":"2024-09-14T04:55:37.562867+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:53.122771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":162,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.493601+0000","last_change":"2024-09-14T04:43:13.355296+0000","last_active":"2024-09-14T04:55:27.493601+0000","last_peered":"2024-09-14T04:55:27.493601+0000","last_clean":"2024-09-14T04:55:27.493601+0000","last_became_active":"2024-09-14T04:43:13.354950+0000","last_became_peered":"2024-09-14T04:43:13.354950+0000","last_unstale":"2024-09-14T04:55:27.493601+0000","last_undegraded":"2024-09-14T04:55:27.493601+0000","last_fullsized":"2024-09-14T04:55:27.493601+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:20:08.968818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"349'149","reported_seq":819,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.483941+0000","last_change":"2024-09-14T04:41:39.471393+0000","last_active":"2024-09-14T04:55:27.483941+0000","last_peered":"2024-09-14T04:55:27.483941+0000","last_clean":"2024-09-14T04:55:27.483941+0000","last_became_active":"2024-09-14T04:41:36.931768+0000","last_became_peered":"2024-09-14T04:41:36.931768+0000","last_unstale":"2024-09-14T04:55:27.483941+0000","last_undegraded":"2024-09-14T04:55:27.483941+0000","last_fullsized":"2024-09-14T04:55:27.483941+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.471248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.471248+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:56.210375+0000","scrub_duration":0.021758344999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2295,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.557448+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T04:55:37.557448+0000","last_peered":"2024-09-14T04:55:37.557448+0000","last_clean":"2024-09-14T04:55:37.557448+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T04:55:37.557448+0000","last_undegraded":"2024-09-14T04:55:37.557448+0000","last_fullsized":"2024-09-14T04:55:37.557448+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1529,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.491655+0000","last_change":"2024-09-14T04:44:57.331436+0000","last_active":"2024-09-14T04:55:27.491655+0000","last_peered":"2024-09-14T04:55:27.491655+0000","last_clean":"2024-09-14T04:55:27.491655+0000","last_became_active":"2024-09-14T04:44:57.330957+0000","last_became_peered":"2024-09-14T04:44:57.330957+0000","last_unstale":"2024-09-14T04:55:27.491655+0000","last_undegraded":"2024-09-14T04:55:27.491655+0000","last_fullsized":"2024-09-14T04:55:27.491655+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:09.516688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":844,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.555525+0000","last_change":"2024-09-14T04:49:49.460401+0000","last_active":"2024-09-14T04:55:37.555525+0000","last_peered":"2024-09-14T04:55:37.555525+0000","last_clean":"2024-09-14T04:55:37.555525+0000","last_became_active":"2024-09-14T04:49:49.458530+0000","last_became_peered":"2024-09-14T04:49:49.458530+0000","last_unstale":"2024-09-14T04:55:37.555525+0000","last_undegraded":"2024-09-14T04:55:37.555525+0000","last_fullsized":"2024-09-14T04:55:37.555525+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:11:29.143239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"349'149","reported_seq":853,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.483625+0000","last_change":"2024-09-14T04:54:22.147441+0000","last_active":"2024-09-14T04:55:27.483625+0000","last_peered":"2024-09-14T04:55:27.483625+0000","last_clean":"2024-09-14T04:55:27.483625+0000","last_became_active":"2024-09-14T04:54:22.147115+0000","last_became_peered":"2024-09-14T04:54:22.147115+0000","last_unstale":"2024-09-14T04:55:27.483625+0000","last_undegraded":"2024-09-14T04:55:27.483625+0000","last_fullsized":"2024-09-14T04:55:27.483625+0000","mapping_epoch":1492,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1493,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:38.479887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:38.479887+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:41:14.001781+0000","scrub_duration":0.0062728319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":250,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.561172+0000","last_change":"2024-09-14T04:44:57.329983+0000","last_active":"2024-09-14T04:55:37.561172+0000","last_peered":"2024-09-14T04:55:37.561172+0000","last_clean":"2024-09-14T04:55:37.561172+0000","last_became_active":"2024-09-14T04:44:57.329597+0000","last_became_peered":"2024-09-14T04:44:57.329597+0000","last_unstale":"2024-09-14T04:55:37.561172+0000","last_undegraded":"2024-09-14T04:55:37.561172+0000","last_fullsized":"2024-09-14T04:55:37.561172+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:45.718039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":788,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.557802+0000","last_change":"2024-09-14T04:49:49.443631+0000","last_active":"2024-09-14T04:55:37.557802+0000","last_peered":"2024-09-14T04:55:37.557802+0000","last_clean":"2024-09-14T04:55:37.557802+0000","last_became_active":"2024-09-14T04:49:49.443313+0000","last_became_peered":"2024-09-14T04:49:49.443313+0000","last_unstale":"2024-09-14T04:55:37.557802+0000","last_undegraded":"2024-09-14T04:55:37.557802+0000","last_fullsized":"2024-09-14T04:55:37.557802+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:06:54.562292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1501,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.485298+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T04:55:27.485298+0000","last_peered":"2024-09-14T04:55:27.485298+0000","last_clean":"2024-09-14T04:55:27.485298+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T04:55:27.485298+0000","last_undegraded":"2024-09-14T04:55:27.485298+0000","last_fullsized":"2024-09-14T04:55:27.485298+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"349'149","reported_seq":811,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.488571+0000","last_change":"2024-09-14T04:41:39.012977+0000","last_active":"2024-09-14T04:55:27.488571+0000","last_peered":"2024-09-14T04:55:27.488571+0000","last_clean":"2024-09-14T04:55:27.488571+0000","last_became_active":"2024-09-14T04:41:36.932552+0000","last_became_peered":"2024-09-14T04:41:36.932552+0000","last_unstale":"2024-09-14T04:55:27.488571+0000","last_undegraded":"2024-09-14T04:55:27.488571+0000","last_fullsized":"2024-09-14T04:55:27.488571+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.012879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.012879+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:16:40.080262+0000","scrub_duration":0.0051721620000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":880,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.555714+0000","last_change":"2024-09-14T04:49:49.459590+0000","last_active":"2024-09-14T04:55:37.555714+0000","last_peered":"2024-09-14T04:55:37.555714+0000","last_clean":"2024-09-14T04:55:37.555714+0000","last_became_active":"2024-09-14T04:49:49.456263+0000","last_became_peered":"2024-09-14T04:49:49.456263+0000","last_unstale":"2024-09-14T04:55:37.555714+0000","last_undegraded":"2024-09-14T04:55:37.555714+0000","last_fullsized":"2024-09-14T04:55:37.555714+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:20:44.666805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":123,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.482779+0000","last_change":"2024-09-14T04:44:57.329652+0000","last_active":"2024-09-14T04:55:27.482779+0000","last_peered":"2024-09-14T04:55:27.482779+0000","last_clean":"2024-09-14T04:55:27.482779+0000","last_became_active":"2024-09-14T04:44:57.328888+0000","last_became_peered":"2024-09-14T04:44:57.328888+0000","last_unstale":"2024-09-14T04:55:27.482779+0000","last_undegraded":"2024-09-14T04:55:27.482779+0000","last_fullsized":"2024-09-14T04:55:27.482779+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:10.170068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1500,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.488051+0000","last_change":"2024-09-14T04:06:46.875340+0000","last_active":"2024-09-14T04:55:27.488051+0000","last_peered":"2024-09-14T04:55:27.488051+0000","last_clean":"2024-09-14T04:55:27.488051+0000","last_became_active":"2024-09-14T04:06:46.874552+0000","last_became_peered":"2024-09-14T04:06:46.874552+0000","last_unstale":"2024-09-14T04:55:27.488051+0000","last_undegraded":"2024-09-14T04:55:27.488051+0000","last_fullsized":"2024-09-14T04:55:27.488051+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:52.257357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"349'149","reported_seq":714,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.490484+0000","last_change":"2024-09-14T04:54:55.279666+0000","last_active":"2024-09-14T04:55:27.490484+0000","last_peered":"2024-09-14T04:55:27.490484+0000","last_clean":"2024-09-14T04:55:27.490484+0000","last_became_active":"2024-09-14T04:54:55.279259+0000","last_became_peered":"2024-09-14T04:54:55.279259+0000","last_unstale":"2024-09-14T04:55:27.490484+0000","last_undegraded":"2024-09-14T04:55:27.490484+0000","last_fullsized":"2024-09-14T04:55:27.490484+0000","mapping_epoch":1506,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1507,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:54:03.996025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:54:03.996025+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:40:53.332369+0000","scrub_duration":54,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":243,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.562381+0000","last_change":"2024-09-14T04:44:14.489235+0000","last_active":"2024-09-14T04:55:37.562381+0000","last_peered":"2024-09-14T04:55:37.562381+0000","last_clean":"2024-09-14T04:55:37.562381+0000","last_became_active":"2024-09-14T04:44:14.488950+0000","last_became_peered":"2024-09-14T04:44:14.488950+0000","last_unstale":"2024-09-14T04:55:37.562381+0000","last_undegraded":"2024-09-14T04:55:37.562381+0000","last_fullsized":"2024-09-14T04:55:37.562381+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:54.441144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":798,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.555193+0000","last_change":"2024-09-14T04:49:49.456317+0000","last_active":"2024-09-14T04:55:37.555193+0000","last_peered":"2024-09-14T04:55:37.555193+0000","last_clean":"2024-09-14T04:55:37.555193+0000","last_became_active":"2024-09-14T04:49:49.456029+0000","last_became_peered":"2024-09-14T04:49:49.456029+0000","last_unstale":"2024-09-14T04:55:37.555193+0000","last_undegraded":"2024-09-14T04:55:37.555193+0000","last_fullsized":"2024-09-14T04:55:37.555193+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:46:05.319676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2316,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.554649+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T04:55:37.554649+0000","last_peered":"2024-09-14T04:55:37.554649+0000","last_clean":"2024-09-14T04:55:37.554649+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T04:55:37.554649+0000","last_undegraded":"2024-09-14T04:55:37.554649+0000","last_fullsized":"2024-09-14T04:55:37.554649+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"349'149","reported_seq":705,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.484114+0000","last_change":"2024-09-14T04:50:22.002798+0000","last_active":"2024-09-14T04:55:27.484114+0000","last_peered":"2024-09-14T04:55:27.484114+0000","last_clean":"2024-09-14T04:55:27.484114+0000","last_became_active":"2024-09-14T04:49:49.448278+0000","last_became_peered":"2024-09-14T04:49:49.448278+0000","last_unstale":"2024-09-14T04:55:27.484114+0000","last_undegraded":"2024-09-14T04:55:27.484114+0000","last_fullsized":"2024-09-14T04:55:27.484114+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:50:22.002568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:50:22.002568+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:01:57.489223+0000","scrub_duration":0.010888326,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2316,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.558180+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T04:55:37.558180+0000","last_peered":"2024-09-14T04:55:37.558180+0000","last_clean":"2024-09-14T04:55:37.558180+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T04:55:37.558180+0000","last_undegraded":"2024-09-14T04:55:37.558180+0000","last_fullsized":"2024-09-14T04:55:37.558180+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2337,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.557388+0000","last_change":"2024-09-14T04:54:19.447103+0000","last_active":"2024-09-14T04:55:37.557388+0000","last_peered":"2024-09-14T04:55:37.557388+0000","last_clean":"2024-09-14T04:55:37.557388+0000","last_became_active":"2024-09-14T04:54:19.446748+0000","last_became_peered":"2024-09-14T04:54:19.446748+0000","last_unstale":"2024-09-14T04:55:37.557388+0000","last_undegraded":"2024-09-14T04:55:37.557388+0000","last_fullsized":"2024-09-14T04:55:37.557388+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:00:55.369263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1530,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.490649+0000","last_change":"2024-09-14T04:42:49.939790+0000","last_active":"2024-09-14T04:55:27.490649+0000","last_peered":"2024-09-14T04:55:27.490649+0000","last_clean":"2024-09-14T04:55:27.490649+0000","last_became_active":"2024-09-14T04:42:49.939591+0000","last_became_peered":"2024-09-14T04:42:49.939591+0000","last_unstale":"2024-09-14T04:55:27.490649+0000","last_undegraded":"2024-09-14T04:55:27.490649+0000","last_fullsized":"2024-09-14T04:55:27.490649+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:23:16.790593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1493,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.483845+0000","last_change":"2024-09-14T04:44:57.326008+0000","last_active":"2024-09-14T04:55:27.483845+0000","last_peered":"2024-09-14T04:55:27.483845+0000","last_clean":"2024-09-14T04:55:27.483845+0000","last_became_active":"2024-09-14T04:44:57.325617+0000","last_became_peered":"2024-09-14T04:44:57.325617+0000","last_unstale":"2024-09-14T04:55:27.483845+0000","last_undegraded":"2024-09-14T04:55:27.483845+0000","last_fullsized":"2024-09-14T04:55:27.483845+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:15.637508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":782,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.488116+0000","last_change":"2024-09-14T04:43:13.354846+0000","last_active":"2024-09-14T04:55:27.488116+0000","last_peered":"2024-09-14T04:55:27.488116+0000","last_clean":"2024-09-14T04:55:27.488116+0000","last_became_active":"2024-09-14T04:43:13.354515+0000","last_became_peered":"2024-09-14T04:43:13.354515+0000","last_unstale":"2024-09-14T04:55:27.488116+0000","last_undegraded":"2024-09-14T04:55:27.488116+0000","last_fullsized":"2024-09-14T04:55:27.488116+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:17.889341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1499,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.489100+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T04:55:27.489100+0000","last_peered":"2024-09-14T04:55:27.489100+0000","last_clean":"2024-09-14T04:55:27.489100+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T04:55:27.489100+0000","last_undegraded":"2024-09-14T04:55:27.489100+0000","last_fullsized":"2024-09-14T04:55:27.489100+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":1837,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:28.767333+0000","last_change":"2024-09-14T04:41:30.640651+0000","last_active":"2024-09-14T04:55:28.767333+0000","last_peered":"2024-09-14T04:55:28.767333+0000","last_clean":"2024-09-14T04:55:28.767333+0000","last_became_active":"2024-09-14T04:41:29.532466+0000","last_became_peered":"2024-09-14T04:41:29.532466+0000","last_unstale":"2024-09-14T04:55:28.767333+0000","last_undegraded":"2024-09-14T04:55:28.767333+0000","last_fullsized":"2024-09-14T04:55:28.767333+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":0,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:30.640520+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:30.640520+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:27.776724+0000","scrub_duration":0.0028653139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"349'149","reported_seq":811,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.487986+0000","last_change":"2024-09-14T04:41:39.978080+0000","last_active":"2024-09-14T04:55:27.487986+0000","last_peered":"2024-09-14T04:55:27.487986+0000","last_clean":"2024-09-14T04:55:27.487986+0000","last_became_active":"2024-09-14T04:41:36.929279+0000","last_became_peered":"2024-09-14T04:41:36.929279+0000","last_unstale":"2024-09-14T04:55:27.487986+0000","last_undegraded":"2024-09-14T04:55:27.487986+0000","last_fullsized":"2024-09-14T04:55:27.487986+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.977999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.977999+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:44:06.383827+0000","scrub_duration":0.00192122,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":177,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.556344+0000","last_change":"2024-09-14T04:48:11.040133+0000","last_active":"2024-09-14T04:55:37.556344+0000","last_peered":"2024-09-14T04:55:37.556344+0000","last_clean":"2024-09-14T04:55:37.556344+0000","last_became_active":"2024-09-14T04:48:11.039787+0000","last_became_peered":"2024-09-14T04:48:11.039787+0000","last_unstale":"2024-09-14T04:55:37.556344+0000","last_undegraded":"2024-09-14T04:55:37.556344+0000","last_fullsized":"2024-09-14T04:55:37.556344+0000","mapping_epoch":1428,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:53.520672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2163,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.558488+0000","last_change":"2024-09-14T04:49:29.616443+0000","last_active":"2024-09-14T04:55:37.558488+0000","last_peered":"2024-09-14T04:55:37.558488+0000","last_clean":"2024-09-14T04:55:37.558488+0000","last_became_active":"2024-09-14T04:49:29.616066+0000","last_became_peered":"2024-09-14T04:49:29.616066+0000","last_unstale":"2024-09-14T04:55:37.558488+0000","last_undegraded":"2024-09-14T04:55:37.558488+0000","last_fullsized":"2024-09-14T04:55:37.558488+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:09:15.897957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":122,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.482850+0000","last_change":"2024-09-14T04:44:57.326724+0000","last_active":"2024-09-14T04:55:27.482850+0000","last_peered":"2024-09-14T04:55:27.482850+0000","last_clean":"2024-09-14T04:55:27.482850+0000","last_became_active":"2024-09-14T04:44:57.326191+0000","last_became_peered":"2024-09-14T04:44:57.326191+0000","last_unstale":"2024-09-14T04:55:27.482850+0000","last_undegraded":"2024-09-14T04:55:27.482850+0000","last_fullsized":"2024-09-14T04:55:27.482850+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:46.060269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":122,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.481370+0000","last_change":"2024-09-14T04:44:57.328528+0000","last_active":"2024-09-14T04:55:27.481370+0000","last_peered":"2024-09-14T04:55:27.481370+0000","last_clean":"2024-09-14T04:55:27.481370+0000","last_became_active":"2024-09-14T04:44:57.327739+0000","last_became_peered":"2024-09-14T04:44:57.327739+0000","last_unstale":"2024-09-14T04:55:27.481370+0000","last_undegraded":"2024-09-14T04:55:27.481370+0000","last_fullsized":"2024-09-14T04:55:27.481370+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:04:58.294690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"349'149","reported_seq":838,"reported_epoch":1510,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-14T04:55:37.555047+0000","last_change":"2024-09-14T04:54:36.689611+0000","last_active":"2024-09-14T04:55:37.555047+0000","last_peered":"2024-09-14T04:55:37.555047+0000","last_clean":"2024-09-14T04:54:31.592636+0000","last_became_active":"2024-09-14T04:54:36.676136+0000","last_became_peered":"2024-09-14T04:54:36.676136+0000","last_unstale":"2024-09-14T04:55:37.555047+0000","last_undegraded":"2024-09-14T04:54:36.658607+0000","last_fullsized":"2024-09-14T04:54:36.658065+0000","mapping_epoch":1499,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1479,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:56.738636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:56.738636+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"no scrub is scheduled","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[2,3],"avail_no_missing":["2","3","6"],"object_location_counts":[{"shards":"2,3,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":122,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.483165+0000","last_change":"2024-09-14T04:44:57.323436+0000","last_active":"2024-09-14T04:55:27.483165+0000","last_peered":"2024-09-14T04:55:27.483165+0000","last_clean":"2024-09-14T04:55:27.483165+0000","last_became_active":"2024-09-14T04:44:57.323022+0000","last_became_peered":"2024-09-14T04:44:57.323022+0000","last_unstale":"2024-09-14T04:55:27.483165+0000","last_undegraded":"2024-09-14T04:55:27.483165+0000","last_fullsized":"2024-09-14T04:55:27.483165+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:22.775261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1528,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.488592+0000","last_change":"2024-09-14T04:44:57.322357+0000","last_active":"2024-09-14T04:55:27.488592+0000","last_peered":"2024-09-14T04:55:27.488592+0000","last_clean":"2024-09-14T04:55:27.488592+0000","last_became_active":"2024-09-14T04:44:57.321990+0000","last_became_peered":"2024-09-14T04:44:57.321990+0000","last_unstale":"2024-09-14T04:55:27.488592+0000","last_undegraded":"2024-09-14T04:55:27.488592+0000","last_fullsized":"2024-09-14T04:55:27.488592+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:09.658008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"349'149","reported_seq":1853,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.490731+0000","last_change":"2024-09-14T04:41:53.858076+0000","last_active":"2024-09-14T04:55:27.490731+0000","last_peered":"2024-09-14T04:55:27.490731+0000","last_clean":"2024-09-14T04:55:27.490731+0000","last_became_active":"2024-09-14T04:41:36.931585+0000","last_became_peered":"2024-09-14T04:41:36.931585+0000","last_unstale":"2024-09-14T04:55:27.490731+0000","last_undegraded":"2024-09-14T04:55:27.490731+0000","last_fullsized":"2024-09-14T04:55:27.490731+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:53.857924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:53.857924+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:10:20.658986+0000","scrub_duration":0.0022355919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":122,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.483159+0000","last_change":"2024-09-14T04:44:57.329198+0000","last_active":"2024-09-14T04:55:27.483159+0000","last_peered":"2024-09-14T04:55:27.483159+0000","last_clean":"2024-09-14T04:55:27.483159+0000","last_became_active":"2024-09-14T04:44:57.327754+0000","last_became_peered":"2024-09-14T04:44:57.327754+0000","last_unstale":"2024-09-14T04:55:27.483159+0000","last_undegraded":"2024-09-14T04:55:27.483159+0000","last_fullsized":"2024-09-14T04:55:27.483159+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:10:27.944476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":300,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.557545+0000","last_change":"2024-09-14T04:44:57.348687+0000","last_active":"2024-09-14T04:55:37.557545+0000","last_peered":"2024-09-14T04:55:37.557545+0000","last_clean":"2024-09-14T04:55:37.557545+0000","last_became_active":"2024-09-14T04:44:57.332845+0000","last_became_peered":"2024-09-14T04:44:57.332845+0000","last_unstale":"2024-09-14T04:55:37.557545+0000","last_undegraded":"2024-09-14T04:55:37.557545+0000","last_fullsized":"2024-09-14T04:55:37.557545+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:12:18.762737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2323,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.557307+0000","last_change":"2024-09-14T04:44:57.333468+0000","last_active":"2024-09-14T04:55:37.557307+0000","last_peered":"2024-09-14T04:55:37.557307+0000","last_clean":"2024-09-14T04:55:37.557307+0000","last_became_active":"2024-09-14T04:44:57.332626+0000","last_became_peered":"2024-09-14T04:44:57.332626+0000","last_unstale":"2024-09-14T04:55:37.557307+0000","last_undegraded":"2024-09-14T04:55:37.557307+0000","last_fullsized":"2024-09-14T04:55:37.557307+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:57:26.366438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1484,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.486464+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T04:55:27.486464+0000","last_peered":"2024-09-14T04:55:27.486464+0000","last_clean":"2024-09-14T04:55:27.486464+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T04:55:27.486464+0000","last_undegraded":"2024-09-14T04:55:27.486464+0000","last_fullsized":"2024-09-14T04:55:27.486464+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1483,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.488694+0000","last_change":"2024-09-14T04:06:46.869868+0000","last_active":"2024-09-14T04:55:27.488694+0000","last_peered":"2024-09-14T04:55:27.488694+0000","last_clean":"2024-09-14T04:55:27.488694+0000","last_became_active":"2024-09-14T04:06:46.869126+0000","last_became_peered":"2024-09-14T04:06:46.869126+0000","last_unstale":"2024-09-14T04:55:27.488694+0000","last_undegraded":"2024-09-14T04:55:27.488694+0000","last_fullsized":"2024-09-14T04:55:27.488694+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:41.150138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1467,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.491586+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T04:55:27.491586+0000","last_peered":"2024-09-14T04:55:27.491586+0000","last_clean":"2024-09-14T04:55:27.491586+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T04:55:27.491586+0000","last_undegraded":"2024-09-14T04:55:27.491586+0000","last_fullsized":"2024-09-14T04:55:27.491586+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":122,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.481506+0000","last_change":"2024-09-14T04:44:57.332112+0000","last_active":"2024-09-14T04:55:27.481506+0000","last_peered":"2024-09-14T04:55:27.481506+0000","last_clean":"2024-09-14T04:55:27.481506+0000","last_became_active":"2024-09-14T04:44:57.331575+0000","last_became_peered":"2024-09-14T04:44:57.331575+0000","last_unstale":"2024-09-14T04:55:27.481506+0000","last_undegraded":"2024-09-14T04:55:27.481506+0000","last_fullsized":"2024-09-14T04:55:27.481506+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:50:00.391283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1515,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.492832+0000","last_change":"2024-09-14T04:44:57.351460+0000","last_active":"2024-09-14T04:55:27.492832+0000","last_peered":"2024-09-14T04:55:27.492832+0000","last_clean":"2024-09-14T04:55:27.492832+0000","last_became_active":"2024-09-14T04:44:57.334662+0000","last_became_peered":"2024-09-14T04:44:57.334662+0000","last_unstale":"2024-09-14T04:55:27.492832+0000","last_undegraded":"2024-09-14T04:55:27.492832+0000","last_fullsized":"2024-09-14T04:55:27.492832+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:02.399062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1469,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.485902+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T04:55:27.485902+0000","last_peered":"2024-09-14T04:55:27.485902+0000","last_clean":"2024-09-14T04:55:27.485902+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T04:55:27.485902+0000","last_undegraded":"2024-09-14T04:55:27.485902+0000","last_fullsized":"2024-09-14T04:55:27.485902+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1527,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.493098+0000","last_change":"2024-09-14T04:44:57.349690+0000","last_active":"2024-09-14T04:55:27.493098+0000","last_peered":"2024-09-14T04:55:27.493098+0000","last_clean":"2024-09-14T04:55:27.493098+0000","last_became_active":"2024-09-14T04:44:57.333579+0000","last_became_peered":"2024-09-14T04:44:57.333579+0000","last_unstale":"2024-09-14T04:55:27.493098+0000","last_undegraded":"2024-09-14T04:55:27.493098+0000","last_fullsized":"2024-09-14T04:55:27.493098+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:17:55.265141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1604,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.487246+0000","last_change":"2024-09-14T04:54:35.694089+0000","last_active":"2024-09-14T04:55:27.487246+0000","last_peered":"2024-09-14T04:55:27.487246+0000","last_clean":"2024-09-14T04:55:27.487246+0000","last_became_active":"2024-09-14T04:54:35.663133+0000","last_became_peered":"2024-09-14T04:54:35.663133+0000","last_unstale":"2024-09-14T04:55:27.487246+0000","last_undegraded":"2024-09-14T04:55:27.487246+0000","last_fullsized":"2024-09-14T04:55:27.487246+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:17:19.947280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":17,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.492861+0000","last_change":"2024-09-14T04:54:52.248439+0000","last_active":"2024-09-14T04:55:27.492861+0000","last_peered":"2024-09-14T04:55:27.492861+0000","last_clean":"2024-09-14T04:55:27.492861+0000","last_became_active":"2024-09-14T04:54:52.248235+0000","last_became_peered":"2024-09-14T04:54:52.248235+0000","last_unstale":"2024-09-14T04:55:27.492861+0000","last_undegraded":"2024-09-14T04:55:27.492861+0000","last_fullsized":"2024-09-14T04:55:27.492861+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:22:09.548139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1489,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.490929+0000","last_change":"2024-09-14T04:44:57.332511+0000","last_active":"2024-09-14T04:55:27.490929+0000","last_peered":"2024-09-14T04:55:27.490929+0000","last_clean":"2024-09-14T04:55:27.490929+0000","last_became_active":"2024-09-14T04:44:57.332166+0000","last_became_peered":"2024-09-14T04:44:57.332166+0000","last_unstale":"2024-09-14T04:55:27.490929+0000","last_undegraded":"2024-09-14T04:55:27.490929+0000","last_fullsized":"2024-09-14T04:55:27.490929+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:53.984639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2316,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.557974+0000","last_change":"2024-09-14T04:54:52.250655+0000","last_active":"2024-09-14T04:55:37.557974+0000","last_peered":"2024-09-14T04:55:37.557974+0000","last_clean":"2024-09-14T04:55:37.557974+0000","last_became_active":"2024-09-14T04:54:52.250317+0000","last_became_peered":"2024-09-14T04:54:52.250317+0000","last_unstale":"2024-09-14T04:55:37.557974+0000","last_undegraded":"2024-09-14T04:55:37.557974+0000","last_fullsized":"2024-09-14T04:55:37.557974+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:12.985406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2302,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.557844+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T04:55:37.557844+0000","last_peered":"2024-09-14T04:55:37.557844+0000","last_clean":"2024-09-14T04:55:37.557844+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T04:55:37.557844+0000","last_undegraded":"2024-09-14T04:55:37.557844+0000","last_fullsized":"2024-09-14T04:55:37.557844+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1481,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.485229+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T04:55:27.485229+0000","last_peered":"2024-09-14T04:55:27.485229+0000","last_clean":"2024-09-14T04:55:27.485229+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T04:55:27.485229+0000","last_undegraded":"2024-09-14T04:55:27.485229+0000","last_fullsized":"2024-09-14T04:55:27.485229+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1518,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.491697+0000","last_change":"2024-09-14T04:44:57.339993+0000","last_active":"2024-09-14T04:55:27.491697+0000","last_peered":"2024-09-14T04:55:27.491697+0000","last_clean":"2024-09-14T04:55:27.491697+0000","last_became_active":"2024-09-14T04:44:57.330472+0000","last_became_peered":"2024-09-14T04:44:57.330472+0000","last_unstale":"2024-09-14T04:55:27.491697+0000","last_undegraded":"2024-09-14T04:55:27.491697+0000","last_fullsized":"2024-09-14T04:55:27.491697+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:19:27.046988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2066,"reported_epoch":1509,"state":"active+clean","last_fresh":"2024-09-14T04:55:27.491333+0000","last_change":"2024-09-14T04:49:49.449687+0000","last_active":"2024-09-14T04:55:27.491333+0000","last_peered":"2024-09-14T04:55:27.491333+0000","last_clean":"2024-09-14T04:55:27.491333+0000","last_became_active":"2024-09-14T04:49:49.449144+0000","last_became_peered":"2024-09-14T04:49:49.449144+0000","last_unstale":"2024-09-14T04:55:27.491333+0000","last_undegraded":"2024-09-14T04:55:27.491333+0000","last_fullsized":"2024-09-14T04:55:27.491333+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:59:06.779730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":56,"num_pg":16,"stat_sum":{"num_bytes":2431451160,"num_objects":37102,"num_object_clones":0,"num_object_copies":111306,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4349,"num_objects_misplaced":2661,"num_objects_unfound":0,"num_objects_dirty":37102,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":191942,"num_write_kb":7329345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9430,"num_bytes_recovered":618004480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7935832064,"data_stored":7935819848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125396,"ondisk_log_size":125396,"up":48,"acting":46,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":32,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":5070,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4250,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223202056,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1799828,"kb_used_data":1096908,"kb_used_omap":1,"kb_used_meta":702910,"kb_avail":91945324,"statfs":{"total":95995035648,"available":94152011776,"internally_reserved":0,"allocated":1123233792,"data_stored":1118023272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":719780299},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463692,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1270860,"kb_used_data":617988,"kb_used_omap":1,"kb_used_meta":652862,"kb_avail":92474292,"statfs":{"total":95995035648,"available":94693675008,"internally_reserved":0,"allocated":632819712,"data_stored":627617777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":668531143},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725329,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1923712,"kb_used_data":1188792,"kb_used_omap":1,"kb_used_meta":734910,"kb_avail":91821440,"statfs":{"total":95995035648,"available":94025154560,"internally_reserved":0,"allocated":1217323008,"data_stored":1212128643,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":752548294},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":1392,"seq":5978594476175,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1446400,"kb_used_data":1152760,"kb_used_omap":117,"kb_used_meta":293514,"kb_avail":92298752,"statfs":{"total":95995035648,"available":94513922048,"internally_reserved":0,"allocated":1180426240,"data_stored":1175227781,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120638,"internal_metadata":300558530},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772075071,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1535876,"kb_used_data":809660,"kb_used_omap":71,"kb_used_meta":726136,"kb_avail":92209276,"statfs":{"total":95995035648,"available":94422298624,"internally_reserved":0,"allocated":829091840,"data_stored":823894000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":743564214},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707304006,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1517056,"kb_used_data":1033720,"kb_used_omap":68,"kb_used_meta":483195,"kb_avail":92228096,"statfs":{"total":95995035648,"available":94441570304,"internally_reserved":0,"allocated":1058529280,"data_stored":1053335416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":494791888},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872729165,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1462972,"kb_used_data":907312,"kb_used_omap":67,"kb_used_meta":555324,"kb_avail":92282180,"statfs":{"total":95995035648,"available":94496952320,"internally_reserved":0,"allocated":929087488,"data_stored":923900675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":568652133},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056215,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1736004,"kb_used_data":1118328,"kb_used_omap":74,"kb_used_meta":617589,"kb_avail":92009148,"statfs":{"total":95995035648,"available":94217367552,"internally_reserved":0,"allocated":1145167872,"data_stored":1139973476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":632411829},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,1,4,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1901,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1124667392,"data_stored":1124663320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":900071424,"data_stored":900071424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1040842752,"data_stored":1040842752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":805507072,"data_stored":805503000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1150681088,"data_stored":1150681088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1197998080,"data_stored":1197998080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":610664448,"data_stored":610664448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1105399808,"data_stored":1105395736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:55:45.054 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:45.082 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:45.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:45.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:45.184 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:45.235 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:45.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:45.236 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:45.337 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:45.389 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:45.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:45.391 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:45.491 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:45.532 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:45.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:45.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:45.634 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:45.685 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:55:45.694 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:45.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:45.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:45.732 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:55:45.732 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [], 2024-09-14T04:55:45.732 INFO:teuthology.orchestra.run.smithi039.stdout: "complaint_time": 30, 2024-09-14T04:55:45.732 INFO:teuthology.orchestra.run.smithi039.stdout: "num_blocked_ops": 0 2024-09-14T04:55:45.732 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:55:45.748 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:55:45.796 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:45.828 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:45.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:45.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:45.837 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:45 smithi039 ceph-mon[106470]: pgmap v2863: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 4350/103746 objects degraded (4.193%); 2661/103746 objects misplaced (2.565%); 3.0 MiB/s, 48 objects/s recovering 2024-09-14T04:55:45.837 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:45 smithi039 ceph-mon[111235]: pgmap v2863: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 4350/103746 objects degraded (4.193%); 2661/103746 objects misplaced (2.565%); 3.0 MiB/s, 48 objects/s recovering 2024-09-14T04:55:45.930 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:45.965 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:45.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:45.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:46.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:45 smithi129 ceph-mon[101316]: pgmap v2863: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 882 op/s; 4350/103746 objects degraded (4.193%); 2661/103746 objects misplaced (2.565%); 3.0 MiB/s, 48 objects/s recovering 2024-09-14T04:55:46.067 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:46.098 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:46.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:46.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:46.200 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:46.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:46.235 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:46.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:46.336 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:46.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:46.375 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:46.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:46.477 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:46.526 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:46.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:46.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:46.628 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:46.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:46.670 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:46.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:46.674 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:46 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 4350/103746 objects degraded (4.193%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:55:46.675 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:46 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 4350/103746 objects degraded (4.193%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:55:46.771 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:46.812 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:46.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:46.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:46.914 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:46.952 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:46.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:46.953 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:47.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:46 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 4350/103746 objects degraded (4.193%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:55:47.054 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:47.082 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:47.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:47.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:47.184 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:47.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:47.223 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:47.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:47.324 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:47.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:47.362 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:47.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:47.464 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:47.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:47.502 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:47.502 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:47.603 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:47.636 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:47.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:47.638 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:47.739 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:47.773 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:47.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:47.775 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:47.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:47 smithi039 ceph-mon[106470]: pgmap v2864: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 884 op/s; 4350/98112 objects degraded (4.434%); 2084/98112 objects misplaced (2.124%); 3.5 MiB/s, 55 objects/s recovering 2024-09-14T04:55:47.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:47 smithi039 ceph-mon[111235]: pgmap v2864: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 884 op/s; 4350/98112 objects degraded (4.434%); 2084/98112 objects misplaced (2.124%); 3.5 MiB/s, 55 objects/s recovering 2024-09-14T04:55:47.876 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:47.917 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:47.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:47.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:48.019 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:48.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:47 smithi129 ceph-mon[101316]: pgmap v2864: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 884 op/s; 4350/98112 objects degraded (4.434%); 2084/98112 objects misplaced (2.124%); 3.5 MiB/s, 55 objects/s recovering 2024-09-14T04:55:48.022 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:55:48.056 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:48.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:48.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:48.158 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:48.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:48.212 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:48.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:48.313 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:48.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:48.363 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:48.364 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:48.464 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:48.514 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:48.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:48.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:48.615 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:48.652 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:48.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:48.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:48.754 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:48.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:48.808 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:48.809 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:48.909 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:48.913 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:55:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:55:48] "GET /metrics HTTP/1.1" 200 36754 "" "Prometheus/2.51.0" 2024-09-14T04:55:48.954 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:48.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:48.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:49.055 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:49.090 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:49.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:49.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:49.192 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:49.223 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:49.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:49.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:49.324 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:49.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:49.361 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:49.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:49.423 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-14T04:55:49.462 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:49.493 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:49.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:49.494 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:49.595 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:49.649 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:49 smithi039 ceph-mon[106470]: pgmap v2865: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 855 op/s; 4350/98112 objects degraded (4.434%); 2084/98112 objects misplaced (2.124%); 3.4 MiB/s, 53 objects/s recovering 2024-09-14T04:55:49.656 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:49.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:49.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:49.758 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:49.790 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:49.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:49.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:49.892 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:49.927 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:49 smithi039 ceph-mon[111235]: pgmap v2865: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 855 op/s; 4350/98112 objects degraded (4.434%); 2084/98112 objects misplaced (2.124%); 3.4 MiB/s, 53 objects/s recovering 2024-09-14T04:55:49.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:49.956 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:49.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:50.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:49 smithi129 ceph-mon[101316]: pgmap v2865: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 855 op/s; 4350/98112 objects degraded (4.434%); 2084/98112 objects misplaced (2.124%); 3.4 MiB/s, 53 objects/s recovering 2024-09-14T04:55:50.057 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:50.100 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:50.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:50.101 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:50.202 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:50.252 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:50.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:50.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:50.354 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:50.387 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:50.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:50.388 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:50.488 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:50.540 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:50.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:50.541 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:50.642 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:50.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:50.695 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:50.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:50.796 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:50.835 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:50.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:50.836 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:50.934 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:55:50.934 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:55:50.934 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:55:50.934 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:55:50.934 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:55:50.934 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:55:50.936 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:55:50.936 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:55:50.936 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:50.947 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:55:50.970 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:55:50.970 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:55:51.000 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:55:51.006 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:51.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:51.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:51.025 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:55:51.025 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:55:51.062 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:55:51.063 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:55:51.097 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:55:51.097 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:55:51.108 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:51.133 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:51.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:51.134 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:51.235 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:51.284 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:51.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:51.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:51.386 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:51.423 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:51.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:51.424 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:51.525 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:51.571 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:51.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:51.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:51.673 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:51.677 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is set 2024-09-14T04:55:51.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:51.710 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:51.710 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:51.811 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:51.844 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:51.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:51.845 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:51.945 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:51.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:51.981 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:51.981 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:51.995 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:51 smithi039 ceph-mon[106470]: pgmap v2866: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 4350/85461 objects degraded (5.090%); 1720/85461 objects misplaced (2.013%); 3.2 MiB/s, 70 objects/s recovering 2024-09-14T04:55:51.995 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:51 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1833741951' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:55:51.995 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:51 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 4350/85461 objects degraded (5.090%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:55:51.995 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:51 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/735395403' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:55:51.995 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:51 smithi039 ceph-mon[111235]: pgmap v2866: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 4350/85461 objects degraded (5.090%); 1720/85461 objects misplaced (2.013%); 3.2 MiB/s, 70 objects/s recovering 2024-09-14T04:55:51.995 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:51 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1833741951' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:55:51.996 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:51 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 4350/85461 objects degraded (5.090%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:55:51.996 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:51 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/735395403' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:55:52.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:51 smithi129 ceph-mon[101316]: pgmap v2866: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 4350/85461 objects degraded (5.090%); 1720/85461 objects misplaced (2.013%); 3.2 MiB/s, 70 objects/s recovering 2024-09-14T04:55:52.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:51 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1833741951' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:55:52.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:51 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 4350/85461 objects degraded (5.090%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:55:52.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:51 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/735395403' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:55:52.082 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:52.122 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:52.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:52.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:52.224 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:52.262 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:52.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:52.263 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:52.364 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:52.407 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:52.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:52.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:52.509 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:52.553 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:52.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:52.554 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:52.655 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:52.694 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:52.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:52.696 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:52.797 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:52.828 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:52.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:52.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:52.930 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:52.964 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:52.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:52.965 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:52 smithi129 ceph-mon[101316]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:55:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:52 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/735395403' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:55:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:52 smithi129 ceph-mon[101316]: osdmap e1511: 8 total, 8 up, 8 in 2024-09-14T04:55:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:55:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:52 smithi039 ceph-mon[106470]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:55:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:52 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/735395403' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:55:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:52 smithi039 ceph-mon[106470]: osdmap e1511: 8 total, 8 up, 8 in 2024-09-14T04:55:53.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:55:53.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:52 smithi039 ceph-mon[111235]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:55:53.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:52 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/735395403' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:55:53.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:52 smithi039 ceph-mon[111235]: osdmap e1511: 8 total, 8 up, 8 in 2024-09-14T04:55:53.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:55:53.066 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:53.097 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:53.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:53.098 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:53.199 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:53.238 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:53.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:53.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:53.340 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:53.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:53.376 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:53.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:53.477 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:53.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:53.510 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:53.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:53.610 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:53.650 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:53.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:53.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:53.752 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:53.789 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:53.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:53.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:53.891 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:53.919 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:53 smithi039 ceph-mon[106470]: pgmap v2868: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 4350/85461 objects degraded (5.090%); 1720/85461 objects misplaced (2.013%); 2.3 MiB/s, 59 objects/s recovering 2024-09-14T04:55:53.919 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:53 smithi039 ceph-mon[111235]: pgmap v2868: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 4350/85461 objects degraded (5.090%); 1720/85461 objects misplaced (2.013%); 2.3 MiB/s, 59 objects/s recovering 2024-09-14T04:55:53.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:53.982 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:53.983 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:54.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:53 smithi129 ceph-mon[101316]: pgmap v2868: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 863 op/s; 4350/85461 objects degraded (5.090%); 1720/85461 objects misplaced (2.013%); 2.3 MiB/s, 59 objects/s recovering 2024-09-14T04:55:54.035 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:55:54.083 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:54.113 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:54.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:54.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:54.215 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:54.262 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:54.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:54.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:54.364 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:54.395 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:54.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:54.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:54.497 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:54.532 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:54.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:54.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:54.635 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:54.670 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:54.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:54.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:54.773 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:54.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:54.809 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:54.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:54.910 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:54.948 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:54.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:54.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:55.050 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:55.084 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:55.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:55.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:55.186 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:55:55.227 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:55.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:55.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:55:55.329 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:55.366 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:55.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:55.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:55.467 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:55.510 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:55.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:55.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:55.613 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:55.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:55.661 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:55.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:55.762 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:55.801 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:55.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:55.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:55.903 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:55.938 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:55.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:55.939 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:55.947 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:55 smithi039 ceph-mon[106470]: pgmap v2869: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 857 op/s; 4350/78024 objects degraded (5.575%); 1720/78024 objects misplaced (2.204%); 2.3 MiB/s, 59 objects/s recovering 2024-09-14T04:55:55.948 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:55 smithi039 ceph-mon[111235]: pgmap v2869: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 857 op/s; 4350/78024 objects degraded (5.575%); 1720/78024 objects misplaced (2.204%); 2.3 MiB/s, 59 objects/s recovering 2024-09-14T04:55:56.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:55 smithi129 ceph-mon[101316]: pgmap v2869: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 857 op/s; 4350/78024 objects degraded (5.575%); 1720/78024 objects misplaced (2.204%); 2.3 MiB/s, 59 objects/s recovering 2024-09-14T04:55:56.040 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:56.075 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:56.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:56.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:56.177 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:56.210 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:56.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:56.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:56.311 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:56.346 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:56.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:56.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:56.448 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:56.476 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:56.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:56.477 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:56.578 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:56.609 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:56.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:56.610 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:56.711 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:56.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:56.762 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:56.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:56.863 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:56.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:56.916 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:56.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:56.951 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:56 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 4350/78024 objects degraded (5.575%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:55:56.951 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:56 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 4350/78024 objects degraded (5.575%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:55:57.018 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:57.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:56 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 4350/78024 objects degraded (5.575%), 2 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2024-09-14T04:55:57.053 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:55:57.060 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:57.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:57.061 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:57.112 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:55:57.114 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T04:55:57.161 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:57.193 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:57.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:57.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:57.295 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:55:57.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:57.333 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:57.333 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:55:57.434 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:57.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:57.466 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:57.466 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:57.562 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:55:57.562 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:55:57.567 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:57.615 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:57.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:57.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:57.717 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:57.760 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:57.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:57.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:57.862 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:57.897 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:57.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:57.898 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:57.923 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:57 smithi039 ceph-mon[106470]: pgmap v2870: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 4350/72123 objects degraded (6.031%); 1479/72123 objects misplaced (2.051%); 429 KiB/s, 59 objects/s recovering 2024-09-14T04:55:57.924 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:57 smithi039 ceph-mon[106470]: from='client.46190 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:55:57.924 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:57 smithi039 ceph-mon[111235]: pgmap v2870: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 4350/72123 objects degraded (6.031%); 1479/72123 objects misplaced (2.051%); 429 KiB/s, 59 objects/s recovering 2024-09-14T04:55:57.924 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:57 smithi039 ceph-mon[111235]: from='client.46190 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:55:57.999 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:58.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:57 smithi129 ceph-mon[101316]: pgmap v2870: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.5 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 4350/72123 objects degraded (6.031%); 1479/72123 objects misplaced (2.051%); 429 KiB/s, 59 objects/s recovering 2024-09-14T04:55:58.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:57 smithi129 ceph-mon[101316]: from='client.46190 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:55:58.038 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:58.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:58.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:58.139 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:58.175 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:58.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:58.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:58.277 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:58.312 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:58.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:58.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:58.414 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:58.424 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:55:58.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:58.445 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:58.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:58.463 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-14T04:55:58.546 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:58.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:58.614 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:58.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:58.715 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:55:58.725 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:55:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:55:58] "GET /metrics HTTP/1.1" 200 36756 "" "Prometheus/2.51.0" 2024-09-14T04:55:58.754 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:58.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:58.755 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:55:58.856 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:58.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:58.890 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:58.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:58.991 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:59.020 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:59.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:58 smithi129 ceph-mon[101316]: pgmap v2871: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 4350/72123 objects degraded (6.031%); 1479/72123 objects misplaced (2.051%); 429 KiB/s, 59 objects/s recovering 2024-09-14T04:55:59.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:59.021 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:59.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:58 smithi039 ceph-mon[106470]: pgmap v2871: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 4350/72123 objects degraded (6.031%); 1479/72123 objects misplaced (2.051%); 429 KiB/s, 59 objects/s recovering 2024-09-14T04:55:59.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:58 smithi039 ceph-mon[111235]: pgmap v2871: 99 pgs: 1 active+recovering+undersized+remapped, 1 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 95 active+clean; 1.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 865 op/s; 4350/72123 objects degraded (6.031%); 1479/72123 objects misplaced (2.051%); 429 KiB/s, 59 objects/s recovering 2024-09-14T04:55:59.122 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:59.150 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:59.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:59.151 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:59.207 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":2870,"stamp":"2024-09-14T04:55:55.884477+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1574377469,"num_objects":24041,"num_object_clones":0,"num_object_copies":72123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4350,"num_objects_misplaced":1479,"num_objects_unfound":0,"num_objects_dirty":24041,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":205093,"num_write_kb":7329378,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10374,"num_bytes_recovered":643046478,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126981,"ondisk_log_size":126981,"up":297,"acting":294,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":306,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":10404072,"kb_used_data":5552412,"kb_used_omap":404,"kb_used_meta":4850987,"kb_avail":739557144,"statfs":{"total":767960285184,"available":757306515456,"internally_reserved":0,"allocated":5685669888,"data_stored":5644032200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":414022,"internal_metadata":4967411386},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-567738368,"num_objects":-8663,"num_object_clones":0,"num_object_copies":-25989,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-605,"num_objects_unfound":0,"num_objects_dirty":-8663,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8663,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":593,"num_bytes_recovered":4390912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003492"},"pg_stats":[{"pgid":"56.f","version":"1511'12681","reported_seq":12732,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:53.279101+0000","last_change":"2024-09-14T04:52:16.661278+0000","last_active":"2024-09-14T04:55:53.279101+0000","last_peered":"2024-09-14T04:55:53.279101+0000","last_clean":"2024-09-14T04:55:53.279101+0000","last_became_active":"2024-09-14T04:52:16.660906+0000","last_became_peered":"2024-09-14T04:52:16.660906+0000","last_unstale":"2024-09-14T04:55:53.279101+0000","last_undegraded":"2024-09-14T04:55:53.279101+0000","last_fullsized":"2024-09-14T04:55:53.279101+0000","mapping_epoch":1470,"log_start":"1482'5700","ondisk_log_start":"1482'5700","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":6981,"log_dups_size":0,"ondisk_log_size":6981,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:55:16.908898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98369536,"num_objects":1501,"num_object_clones":0,"num_object_copies":4503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1501,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12681,"num_write_kb":453824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"56.e","version":"1511'12902","reported_seq":12953,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:53.745184+0000","last_change":"2024-09-14T04:52:16.723745+0000","last_active":"2024-09-14T04:55:53.745184+0000","last_peered":"2024-09-14T04:55:53.745184+0000","last_clean":"2024-09-14T04:55:53.745184+0000","last_became_active":"2024-09-14T04:52:16.723416+0000","last_became_peered":"2024-09-14T04:52:16.723416+0000","last_unstale":"2024-09-14T04:55:53.745184+0000","last_undegraded":"2024-09-14T04:55:53.745184+0000","last_fullsized":"2024-09-14T04:55:53.745184+0000","mapping_epoch":1470,"log_start":"1481'5547","ondisk_log_start":"1481'5547","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":7355,"log_dups_size":0,"ondisk_log_size":7355,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:20:17.253535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103153664,"num_objects":1574,"num_object_clones":0,"num_object_copies":4722,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1574,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12902,"num_write_kb":463232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"56.d","version":"1511'12835","reported_seq":12930,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:53.731352+0000","last_change":"2024-09-14T04:52:16.871824+0000","last_active":"2024-09-14T04:55:53.731352+0000","last_peered":"2024-09-14T04:55:53.731352+0000","last_clean":"2024-09-14T04:55:53.731352+0000","last_became_active":"2024-09-14T04:52:16.871446+0000","last_became_peered":"2024-09-14T04:52:16.871446+0000","last_unstale":"2024-09-14T04:55:53.731352+0000","last_undegraded":"2024-09-14T04:55:53.731352+0000","last_fullsized":"2024-09-14T04:55:53.731352+0000","mapping_epoch":1470,"log_start":"1480'4200","ondisk_log_start":"1480'4200","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8635,"log_dups_size":0,"ondisk_log_size":8635,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:44:00.576543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102760472,"num_objects":1569,"num_object_clones":0,"num_object_copies":4707,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1569,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12835,"num_write_kb":460865,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"56.c","version":"1511'12971","reported_seq":13022,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:54.521827+0000","last_change":"2024-09-14T04:52:16.723896+0000","last_active":"2024-09-14T04:55:54.521827+0000","last_peered":"2024-09-14T04:55:54.521827+0000","last_clean":"2024-09-14T04:55:54.521827+0000","last_became_active":"2024-09-14T04:52:16.723597+0000","last_became_peered":"2024-09-14T04:52:16.723597+0000","last_unstale":"2024-09-14T04:55:54.521827+0000","last_undegraded":"2024-09-14T04:55:54.521827+0000","last_fullsized":"2024-09-14T04:55:54.521827+0000","mapping_epoch":1470,"log_start":"1481'4808","ondisk_log_start":"1481'4808","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8163,"log_dups_size":0,"ondisk_log_size":8163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:10:31.058435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95223808,"num_objects":1453,"num_object_clones":0,"num_object_copies":4359,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1453,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12971,"num_write_kb":461568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.b","version":"1511'12836","reported_seq":12887,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:53.298509+0000","last_change":"2024-09-14T04:52:16.660625+0000","last_active":"2024-09-14T04:55:53.298509+0000","last_peered":"2024-09-14T04:55:53.298509+0000","last_clean":"2024-09-14T04:55:53.298509+0000","last_became_active":"2024-09-14T04:52:16.660277+0000","last_became_peered":"2024-09-14T04:52:16.660277+0000","last_unstale":"2024-09-14T04:55:53.298509+0000","last_undegraded":"2024-09-14T04:55:53.298509+0000","last_fullsized":"2024-09-14T04:55:53.298509+0000","mapping_epoch":1470,"log_start":"1482'5800","ondisk_log_start":"1482'5800","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":7036,"log_dups_size":0,"ondisk_log_size":7036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:43:12.659725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100401152,"num_objects":1532,"num_object_clones":0,"num_object_copies":4596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12836,"num_write_kb":459776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"56.a","version":"1511'12748","reported_seq":12799,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:53.310119+0000","last_change":"2024-09-14T04:52:16.915139+0000","last_active":"2024-09-14T04:55:53.310119+0000","last_peered":"2024-09-14T04:55:53.310119+0000","last_clean":"2024-09-14T04:55:53.310119+0000","last_became_active":"2024-09-14T04:52:16.914776+0000","last_became_peered":"2024-09-14T04:52:16.914776+0000","last_unstale":"2024-09-14T04:55:53.310119+0000","last_undegraded":"2024-09-14T04:55:53.310119+0000","last_fullsized":"2024-09-14T04:55:53.310119+0000","mapping_epoch":1470,"log_start":"1482'5700","ondisk_log_start":"1482'5700","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":7048,"log_dups_size":0,"ondisk_log_size":7048,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:08:58.079681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100270080,"num_objects":1530,"num_object_clones":0,"num_object_copies":4590,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1530,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12748,"num_write_kb":456896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"56.9","version":"1511'12887","reported_seq":12938,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:54.511689+0000","last_change":"2024-09-14T04:52:16.723789+0000","last_active":"2024-09-14T04:55:54.511689+0000","last_peered":"2024-09-14T04:55:54.511689+0000","last_clean":"2024-09-14T04:55:54.511689+0000","last_became_active":"2024-09-14T04:52:16.723394+0000","last_became_peered":"2024-09-14T04:52:16.723394+0000","last_unstale":"2024-09-14T04:55:54.511689+0000","last_undegraded":"2024-09-14T04:55:54.511689+0000","last_fullsized":"2024-09-14T04:55:54.511689+0000","mapping_epoch":1470,"log_start":"1481'4770","ondisk_log_start":"1481'4770","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8117,"log_dups_size":0,"ondisk_log_size":8117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:45:07.892366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94568448,"num_objects":1443,"num_object_clones":0,"num_object_copies":4329,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1443,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12887,"num_write_kb":458560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.8","version":"1511'12512","reported_seq":12606,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:53.695255+0000","last_change":"2024-09-14T04:52:16.915103+0000","last_active":"2024-09-14T04:55:53.695255+0000","last_peered":"2024-09-14T04:55:53.695255+0000","last_clean":"2024-09-14T04:55:53.695255+0000","last_became_active":"2024-09-14T04:52:16.914657+0000","last_became_peered":"2024-09-14T04:52:16.914657+0000","last_unstale":"2024-09-14T04:55:53.695255+0000","last_undegraded":"2024-09-14T04:55:53.695255+0000","last_fullsized":"2024-09-14T04:55:53.695255+0000","mapping_epoch":1470,"log_start":"1480'3900","ondisk_log_start":"1480'3900","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8612,"log_dups_size":0,"ondisk_log_size":8612,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:08:43.898740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98304000,"num_objects":1500,"num_object_clones":0,"num_object_copies":4500,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1500,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12512,"num_write_kb":448384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"56.7","version":"1511'12788","reported_seq":12882,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:53.720950+0000","last_change":"2024-09-14T04:52:16.723670+0000","last_active":"2024-09-14T04:55:53.720950+0000","last_peered":"2024-09-14T04:55:53.720950+0000","last_clean":"2024-09-14T04:55:53.720950+0000","last_became_active":"2024-09-14T04:52:16.723292+0000","last_became_peered":"2024-09-14T04:52:16.723292+0000","last_unstale":"2024-09-14T04:55:53.720950+0000","last_undegraded":"2024-09-14T04:55:53.720950+0000","last_fullsized":"2024-09-14T04:55:53.720950+0000","mapping_epoch":1470,"log_start":"1480'4200","ondisk_log_start":"1480'4200","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8588,"log_dups_size":0,"ondisk_log_size":8588,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:55:51.122910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99090432,"num_objects":1512,"num_object_clones":0,"num_object_copies":4536,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1512,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12788,"num_write_kb":457600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"56.6","version":"1511'12649","reported_seq":20626,"reported_epoch":1511,"state":"active+recovering+undersized+remapped","last_fresh":"2024-09-14T04:55:54.545832+0000","last_change":"2024-09-14T04:54:19.620795+0000","last_active":"2024-09-14T04:55:54.545832+0000","last_peered":"2024-09-14T04:55:54.545832+0000","last_clean":"2024-09-14T04:52:58.397671+0000","last_became_active":"2024-09-14T04:54:19.575869+0000","last_became_peered":"2024-09-14T04:54:19.575869+0000","last_unstale":"2024-09-14T04:55:54.545832+0000","last_undegraded":"2024-09-14T04:55:54.545832+0000","last_fullsized":"2024-09-14T04:54:19.444266+0000","mapping_epoch":1489,"log_start":"1480'4457","ondisk_log_start":"1480'4457","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8192,"log_dups_size":0,"ondisk_log_size":8192,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:53.013009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88539136,"num_objects":1351,"num_object_clones":0,"num_object_copies":4053,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1351,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12649,"num_write_kb":448000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3955,"num_bytes_recovered":224722944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,7","objects":1351}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.5","version":"1511'12823","reported_seq":12874,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:53.779457+0000","last_change":"2024-09-14T04:52:16.871791+0000","last_active":"2024-09-14T04:55:53.779457+0000","last_peered":"2024-09-14T04:55:53.779457+0000","last_clean":"2024-09-14T04:55:53.779457+0000","last_became_active":"2024-09-14T04:52:16.871407+0000","last_became_peered":"2024-09-14T04:52:16.871407+0000","last_unstale":"2024-09-14T04:55:53.779457+0000","last_undegraded":"2024-09-14T04:55:53.779457+0000","last_fullsized":"2024-09-14T04:55:53.779457+0000","mapping_epoch":1470,"log_start":"1481'5467","ondisk_log_start":"1481'5467","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":7356,"log_dups_size":0,"ondisk_log_size":7356,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:47:15.419674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100728832,"num_objects":1537,"num_object_clones":0,"num_object_copies":4611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12823,"num_write_kb":459520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"56.4","version":"1511'12945","reported_seq":12996,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:53.298799+0000","last_change":"2024-09-14T04:52:16.724077+0000","last_active":"2024-09-14T04:55:53.298799+0000","last_peered":"2024-09-14T04:55:53.298799+0000","last_clean":"2024-09-14T04:55:53.298799+0000","last_became_active":"2024-09-14T04:52:16.723597+0000","last_became_peered":"2024-09-14T04:52:16.723597+0000","last_unstale":"2024-09-14T04:55:53.298799+0000","last_undegraded":"2024-09-14T04:55:53.298799+0000","last_fullsized":"2024-09-14T04:55:53.298799+0000","mapping_epoch":1470,"log_start":"1482'5900","ondisk_log_start":"1482'5900","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":7045,"log_dups_size":0,"ondisk_log_size":7045,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:23:49.557180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101384192,"num_objects":1547,"num_object_clones":0,"num_object_copies":4641,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1547,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12945,"num_write_kb":463744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"56.3","version":"1511'12717","reported_seq":19989,"reported_epoch":1511,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:55:54.414218+0000","last_change":"2024-09-14T04:54:53.717396+0000","last_active":"2024-09-14T04:55:54.414218+0000","last_peered":"2024-09-14T04:55:54.414218+0000","last_clean":"2024-09-14T04:54:51.200599+0000","last_became_active":"2024-09-14T04:54:53.681745+0000","last_became_peered":"2024-09-14T04:54:53.681745+0000","last_unstale":"2024-09-14T04:55:54.414218+0000","last_undegraded":"2024-09-14T04:55:54.414218+0000","last_fullsized":"2024-09-14T04:55:54.414218+0000","mapping_epoch":1504,"log_start":"1481'5017","ondisk_log_start":"1481'5017","created":1470,"last_epoch_clean":1496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":7700,"log_dups_size":0,"ondisk_log_size":7700,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96927744,"num_objects":1479,"num_object_clones":0,"num_object_copies":4437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1479,"num_objects_unfound":0,"num_objects_dirty":1479,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12717,"num_write_kb":454272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3597,"num_bytes_recovered":235732992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[1,6,5],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":1479}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"56.2","version":"1511'13077","reported_seq":18681,"reported_epoch":1511,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-14T04:55:54.519072+0000","last_change":"2024-09-14T04:54:19.618218+0000","last_active":"2024-09-14T04:55:54.519072+0000","last_peered":"2024-09-14T04:55:54.519072+0000","last_clean":"2024-09-14T04:52:58.403685+0000","last_became_active":"2024-09-14T04:54:19.593740+0000","last_became_peered":"2024-09-14T04:54:19.593740+0000","last_unstale":"2024-09-14T04:55:54.519072+0000","last_undegraded":"2024-09-14T04:54:19.466599+0000","last_fullsized":"2024-09-14T04:54:19.444256+0000","mapping_epoch":1489,"log_start":"1481'4937","ondisk_log_start":"1481'4937","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8140,"log_dups_size":0,"ondisk_log_size":8140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:45:11.467642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95092736,"num_objects":1451,"num_object_clones":0,"num_object_copies":4353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4349,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13077,"num_write_kb":464896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2770,"num_bytes_recovered":181534720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"0,5,7","objects":1451},{"shards":"5,7","objects":-2100}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.1","version":"1511'12899","reported_seq":12950,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:54.464424+0000","last_change":"2024-09-14T04:52:16.872703+0000","last_active":"2024-09-14T04:55:54.464424+0000","last_peered":"2024-09-14T04:55:54.464424+0000","last_clean":"2024-09-14T04:55:54.464424+0000","last_became_active":"2024-09-14T04:52:16.871779+0000","last_became_peered":"2024-09-14T04:52:16.871779+0000","last_unstale":"2024-09-14T04:55:54.464424+0000","last_undegraded":"2024-09-14T04:55:54.464424+0000","last_fullsized":"2024-09-14T04:55:54.464424+0000","mapping_epoch":1470,"log_start":"1481'4765","ondisk_log_start":"1481'4765","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8134,"log_dups_size":0,"ondisk_log_size":8134,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:23:30.260775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98631680,"num_objects":1505,"num_object_clones":0,"num_object_copies":4515,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1505,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12899,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.0","version":"1511'12759","reported_seq":12810,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:54.494754+0000","last_change":"2024-09-14T04:52:16.915325+0000","last_active":"2024-09-14T04:55:54.494754+0000","last_peered":"2024-09-14T04:55:54.494754+0000","last_clean":"2024-09-14T04:55:54.494754+0000","last_became_active":"2024-09-14T04:52:16.914980+0000","last_became_peered":"2024-09-14T04:52:16.914980+0000","last_unstale":"2024-09-14T04:55:54.494754+0000","last_undegraded":"2024-09-14T04:55:54.494754+0000","last_fullsized":"2024-09-14T04:55:54.494754+0000","mapping_epoch":1470,"log_start":"1481'4578","ondisk_log_start":"1481'4578","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8181,"log_dups_size":0,"ondisk_log_size":8181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:55:08.752960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100335616,"num_objects":1531,"num_object_clones":0,"num_object_copies":4593,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1531,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12759,"num_write_kb":457280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":2314,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:51.702039+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T04:55:51.702039+0000","last_peered":"2024-09-14T04:55:51.702039+0000","last_clean":"2024-09-14T04:55:51.702039+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T04:55:51.702039+0000","last_undegraded":"2024-09-14T04:55:51.702039+0000","last_fullsized":"2024-09-14T04:55:51.702039+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":123,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.548800+0000","last_change":"2024-09-14T04:44:57.325109+0000","last_active":"2024-09-14T04:55:37.548800+0000","last_peered":"2024-09-14T04:55:37.548800+0000","last_clean":"2024-09-14T04:55:37.548800+0000","last_became_active":"2024-09-14T04:44:57.324791+0000","last_became_peered":"2024-09-14T04:44:57.324791+0000","last_unstale":"2024-09-14T04:55:37.548800+0000","last_undegraded":"2024-09-14T04:55:37.548800+0000","last_fullsized":"2024-09-14T04:55:37.548800+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:03.593275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":93,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.553187+0000","last_change":"2024-09-14T04:49:29.628435+0000","last_active":"2024-09-14T04:55:37.553187+0000","last_peered":"2024-09-14T04:55:37.553187+0000","last_clean":"2024-09-14T04:55:37.553187+0000","last_became_active":"2024-09-14T04:49:29.628081+0000","last_became_peered":"2024-09-14T04:49:29.628081+0000","last_unstale":"2024-09-14T04:55:37.553187+0000","last_undegraded":"2024-09-14T04:55:37.553187+0000","last_fullsized":"2024-09-14T04:55:37.553187+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:01.520334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":936,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:51.696000+0000","last_change":"2024-09-14T04:43:13.355281+0000","last_active":"2024-09-14T04:55:51.696000+0000","last_peered":"2024-09-14T04:55:51.696000+0000","last_clean":"2024-09-14T04:55:51.696000+0000","last_became_active":"2024-09-14T04:43:13.354967+0000","last_became_peered":"2024-09-14T04:43:13.354967+0000","last_unstale":"2024-09-14T04:55:51.696000+0000","last_undegraded":"2024-09-14T04:55:51.696000+0000","last_fullsized":"2024-09-14T04:55:51.696000+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:23:19.332075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1502,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:39.478024+0000","last_change":"2024-09-14T04:44:57.342918+0000","last_active":"2024-09-14T04:55:39.478024+0000","last_peered":"2024-09-14T04:55:39.478024+0000","last_clean":"2024-09-14T04:55:39.478024+0000","last_became_active":"2024-09-14T04:44:57.329550+0000","last_became_peered":"2024-09-14T04:44:57.329550+0000","last_unstale":"2024-09-14T04:55:39.478024+0000","last_undegraded":"2024-09-14T04:55:39.478024+0000","last_fullsized":"2024-09-14T04:55:39.478024+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:52:52.435893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":123,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.549642+0000","last_change":"2024-09-14T04:44:57.325537+0000","last_active":"2024-09-14T04:55:37.549642+0000","last_peered":"2024-09-14T04:55:37.549642+0000","last_clean":"2024-09-14T04:55:37.549642+0000","last_became_active":"2024-09-14T04:44:57.325057+0000","last_became_peered":"2024-09-14T04:44:57.325057+0000","last_unstale":"2024-09-14T04:55:37.549642+0000","last_undegraded":"2024-09-14T04:55:37.549642+0000","last_fullsized":"2024-09-14T04:55:37.549642+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:43:23.166708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1517,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:39.477998+0000","last_change":"2024-09-14T04:44:57.337033+0000","last_active":"2024-09-14T04:55:39.477998+0000","last_peered":"2024-09-14T04:55:39.477998+0000","last_clean":"2024-09-14T04:55:39.477998+0000","last_became_active":"2024-09-14T04:44:57.328437+0000","last_became_peered":"2024-09-14T04:44:57.328437+0000","last_unstale":"2024-09-14T04:55:39.477998+0000","last_undegraded":"2024-09-14T04:55:39.477998+0000","last_fullsized":"2024-09-14T04:55:39.477998+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.109086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":123,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.550030+0000","last_change":"2024-09-14T04:44:57.332243+0000","last_active":"2024-09-14T04:55:37.550030+0000","last_peered":"2024-09-14T04:55:37.550030+0000","last_clean":"2024-09-14T04:55:37.550030+0000","last_became_active":"2024-09-14T04:44:57.330413+0000","last_became_peered":"2024-09-14T04:44:57.330413+0000","last_unstale":"2024-09-14T04:55:37.550030+0000","last_undegraded":"2024-09-14T04:55:37.550030+0000","last_fullsized":"2024-09-14T04:55:37.550030+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:25:01.338549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":170,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:51.698811+0000","last_change":"2024-09-14T04:48:29.862929+0000","last_active":"2024-09-14T04:55:51.698811+0000","last_peered":"2024-09-14T04:55:51.698811+0000","last_clean":"2024-09-14T04:55:51.698811+0000","last_became_active":"2024-09-14T04:48:29.862581+0000","last_became_peered":"2024-09-14T04:48:29.862581+0000","last_unstale":"2024-09-14T04:55:51.698811+0000","last_undegraded":"2024-09-14T04:55:51.698811+0000","last_fullsized":"2024-09-14T04:55:51.698811+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:16:43.060088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2310,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:51.695354+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T04:55:51.695354+0000","last_peered":"2024-09-14T04:55:51.695354+0000","last_clean":"2024-09-14T04:55:51.695354+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T04:55:51.695354+0000","last_undegraded":"2024-09-14T04:55:51.695354+0000","last_fullsized":"2024-09-14T04:55:51.695354+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":123,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.549501+0000","last_change":"2024-09-14T04:44:57.325392+0000","last_active":"2024-09-14T04:55:37.549501+0000","last_peered":"2024-09-14T04:55:37.549501+0000","last_clean":"2024-09-14T04:55:37.549501+0000","last_became_active":"2024-09-14T04:44:57.324624+0000","last_became_peered":"2024-09-14T04:44:57.324624+0000","last_unstale":"2024-09-14T04:55:37.549501+0000","last_undegraded":"2024-09-14T04:55:37.549501+0000","last_fullsized":"2024-09-14T04:55:37.549501+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:33:18.810199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2279,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:51.692910+0000","last_change":"2024-09-14T04:44:57.329113+0000","last_active":"2024-09-14T04:55:51.692910+0000","last_peered":"2024-09-14T04:55:51.692910+0000","last_clean":"2024-09-14T04:55:51.692910+0000","last_became_active":"2024-09-14T04:44:57.328727+0000","last_became_peered":"2024-09-14T04:44:57.328727+0000","last_unstale":"2024-09-14T04:55:51.692910+0000","last_undegraded":"2024-09-14T04:55:51.692910+0000","last_fullsized":"2024-09-14T04:55:51.692910+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:34.939874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":2296,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:51.692585+0000","last_change":"2024-09-14T04:44:57.329277+0000","last_active":"2024-09-14T04:55:51.692585+0000","last_peered":"2024-09-14T04:55:51.692585+0000","last_clean":"2024-09-14T04:55:51.692585+0000","last_became_active":"2024-09-14T04:44:57.328804+0000","last_became_peered":"2024-09-14T04:44:57.328804+0000","last_unstale":"2024-09-14T04:55:51.692585+0000","last_undegraded":"2024-09-14T04:55:51.692585+0000","last_fullsized":"2024-09-14T04:55:51.692585+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:42:46.055240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2330,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:51.700772+0000","last_change":"2024-09-14T04:54:52.250695+0000","last_active":"2024-09-14T04:55:51.700772+0000","last_peered":"2024-09-14T04:55:51.700772+0000","last_clean":"2024-09-14T04:55:51.700772+0000","last_became_active":"2024-09-14T04:54:52.250301+0000","last_became_peered":"2024-09-14T04:54:52.250301+0000","last_unstale":"2024-09-14T04:55:51.700772+0000","last_undegraded":"2024-09-14T04:55:51.700772+0000","last_fullsized":"2024-09-14T04:55:51.700772+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:42.060905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2292,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:51.698734+0000","last_change":"2024-09-14T04:43:13.357092+0000","last_active":"2024-09-14T04:55:51.698734+0000","last_peered":"2024-09-14T04:55:51.698734+0000","last_clean":"2024-09-14T04:55:51.698734+0000","last_became_active":"2024-09-14T04:43:13.356694+0000","last_became_peered":"2024-09-14T04:43:13.356694+0000","last_unstale":"2024-09-14T04:55:51.698734+0000","last_undegraded":"2024-09-14T04:55:51.698734+0000","last_fullsized":"2024-09-14T04:55:51.698734+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:11.167574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1495,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.553377+0000","last_change":"2024-09-14T04:07:21.720797+0000","last_active":"2024-09-14T04:55:37.553377+0000","last_peered":"2024-09-14T04:55:37.553377+0000","last_clean":"2024-09-14T04:55:37.553377+0000","last_became_active":"2024-09-14T04:07:21.719331+0000","last_became_peered":"2024-09-14T04:07:21.719331+0000","last_unstale":"2024-09-14T04:55:37.553377+0000","last_undegraded":"2024-09-14T04:55:37.553377+0000","last_fullsized":"2024-09-14T04:55:37.553377+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:35.404023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":142,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.547728+0000","last_change":"2024-09-14T04:43:11.641514+0000","last_active":"2024-09-14T04:55:37.547728+0000","last_peered":"2024-09-14T04:55:37.547728+0000","last_clean":"2024-09-14T04:55:37.547728+0000","last_became_active":"2024-09-14T04:43:11.641151+0000","last_became_peered":"2024-09-14T04:43:11.641151+0000","last_unstale":"2024-09-14T04:55:37.547728+0000","last_undegraded":"2024-09-14T04:55:37.547728+0000","last_fullsized":"2024-09-14T04:55:37.547728+0000","mapping_epoch":1384,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:06:45.045430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2303,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:51.702267+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T04:55:51.702267+0000","last_peered":"2024-09-14T04:55:51.702267+0000","last_clean":"2024-09-14T04:55:51.702267+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T04:55:51.702267+0000","last_undegraded":"2024-09-14T04:55:51.702267+0000","last_fullsized":"2024-09-14T04:55:51.702267+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":722,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.557657+0000","last_change":"2024-09-14T04:49:29.618477+0000","last_active":"2024-09-14T04:55:37.557657+0000","last_peered":"2024-09-14T04:55:37.557657+0000","last_clean":"2024-09-14T04:55:37.557657+0000","last_became_active":"2024-09-14T04:49:29.618210+0000","last_became_peered":"2024-09-14T04:49:29.618210+0000","last_unstale":"2024-09-14T04:55:37.557657+0000","last_undegraded":"2024-09-14T04:55:37.557657+0000","last_fullsized":"2024-09-14T04:55:37.557657+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:13:23.895512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1500,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.551040+0000","last_change":"2024-09-14T04:42:49.938505+0000","last_active":"2024-09-14T04:55:37.551040+0000","last_peered":"2024-09-14T04:55:37.551040+0000","last_clean":"2024-09-14T04:55:37.551040+0000","last_became_active":"2024-09-14T04:42:49.938180+0000","last_became_peered":"2024-09-14T04:42:49.938180+0000","last_unstale":"2024-09-14T04:55:37.551040+0000","last_undegraded":"2024-09-14T04:55:37.551040+0000","last_fullsized":"2024-09-14T04:55:37.551040+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:05.929479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1520,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:51.696990+0000","last_change":"2024-09-14T04:44:57.346582+0000","last_active":"2024-09-14T04:55:51.696990+0000","last_peered":"2024-09-14T04:55:51.696990+0000","last_clean":"2024-09-14T04:55:51.696990+0000","last_became_active":"2024-09-14T04:44:57.330446+0000","last_became_peered":"2024-09-14T04:44:57.330446+0000","last_unstale":"2024-09-14T04:55:51.696990+0000","last_undegraded":"2024-09-14T04:55:51.696990+0000","last_fullsized":"2024-09-14T04:55:51.696990+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:29.238585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":245,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:51.692727+0000","last_change":"2024-09-14T04:44:14.480667+0000","last_active":"2024-09-14T04:55:51.692727+0000","last_peered":"2024-09-14T04:55:51.692727+0000","last_clean":"2024-09-14T04:55:51.692727+0000","last_became_active":"2024-09-14T04:44:14.480365+0000","last_became_peered":"2024-09-14T04:44:14.480365+0000","last_unstale":"2024-09-14T04:55:51.692727+0000","last_undegraded":"2024-09-14T04:55:51.692727+0000","last_fullsized":"2024-09-14T04:55:51.692727+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:00.611771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":133,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.549746+0000","last_change":"2024-09-14T04:44:57.341426+0000","last_active":"2024-09-14T04:55:37.549746+0000","last_peered":"2024-09-14T04:55:37.549746+0000","last_clean":"2024-09-14T04:55:37.549746+0000","last_became_active":"2024-09-14T04:44:57.328763+0000","last_became_peered":"2024-09-14T04:44:57.328763+0000","last_unstale":"2024-09-14T04:55:37.549746+0000","last_undegraded":"2024-09-14T04:55:37.549746+0000","last_fullsized":"2024-09-14T04:55:37.549746+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:34:40.066761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":123,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.550157+0000","last_change":"2024-09-14T04:44:57.331337+0000","last_active":"2024-09-14T04:55:37.550157+0000","last_peered":"2024-09-14T04:55:37.550157+0000","last_clean":"2024-09-14T04:55:37.550157+0000","last_became_active":"2024-09-14T04:44:57.329336+0000","last_became_peered":"2024-09-14T04:44:57.329336+0000","last_unstale":"2024-09-14T04:55:37.550157+0000","last_undegraded":"2024-09-14T04:55:37.550157+0000","last_fullsized":"2024-09-14T04:55:37.550157+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:52:35.980869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"349'149","reported_seq":712,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:49.502372+0000","last_change":"2024-09-14T04:49:30.085274+0000","last_active":"2024-09-14T04:55:49.502372+0000","last_peered":"2024-09-14T04:55:49.502372+0000","last_clean":"2024-09-14T04:55:49.502372+0000","last_became_active":"2024-09-14T04:49:29.623295+0000","last_became_peered":"2024-09-14T04:49:29.623295+0000","last_unstale":"2024-09-14T04:55:49.502372+0000","last_undegraded":"2024-09-14T04:55:49.502372+0000","last_fullsized":"2024-09-14T04:55:49.502372+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:49:30.085089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:49:30.085089+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:10.012963+0000","scrub_duration":0.0071928790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":124,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.549119+0000","last_change":"2024-09-14T04:44:57.329870+0000","last_active":"2024-09-14T04:55:37.549119+0000","last_peered":"2024-09-14T04:55:37.549119+0000","last_clean":"2024-09-14T04:55:37.549119+0000","last_became_active":"2024-09-14T04:44:57.329353+0000","last_became_peered":"2024-09-14T04:44:57.329353+0000","last_unstale":"2024-09-14T04:55:37.549119+0000","last_undegraded":"2024-09-14T04:55:37.549119+0000","last_fullsized":"2024-09-14T04:55:37.549119+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:46.432181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1516,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.553438+0000","last_change":"2024-09-14T04:54:52.242207+0000","last_active":"2024-09-14T04:55:37.553438+0000","last_peered":"2024-09-14T04:55:37.553438+0000","last_clean":"2024-09-14T04:55:37.553438+0000","last_became_active":"2024-09-14T04:54:52.241840+0000","last_became_peered":"2024-09-14T04:54:52.241840+0000","last_unstale":"2024-09-14T04:55:37.553438+0000","last_undegraded":"2024-09-14T04:55:37.553438+0000","last_fullsized":"2024-09-14T04:55:37.553438+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:37.899666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":2342,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:51.700676+0000","last_change":"2024-09-14T04:44:57.344614+0000","last_active":"2024-09-14T04:55:51.700676+0000","last_peered":"2024-09-14T04:55:51.700676+0000","last_clean":"2024-09-14T04:55:51.700676+0000","last_became_active":"2024-09-14T04:44:57.332256+0000","last_became_peered":"2024-09-14T04:44:57.332256+0000","last_unstale":"2024-09-14T04:55:51.700676+0000","last_undegraded":"2024-09-14T04:55:51.700676+0000","last_fullsized":"2024-09-14T04:55:51.700676+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:23.438417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"349'149","reported_seq":882,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.554709+0000","last_change":"2024-09-14T04:45:00.331854+0000","last_active":"2024-09-14T04:55:37.554709+0000","last_peered":"2024-09-14T04:55:37.554709+0000","last_clean":"2024-09-14T04:55:37.554709+0000","last_became_active":"2024-09-14T04:45:00.331460+0000","last_became_peered":"2024-09-14T04:45:00.331460+0000","last_unstale":"2024-09-14T04:55:37.554709+0000","last_undegraded":"2024-09-14T04:55:37.554709+0000","last_fullsized":"2024-09-14T04:55:37.554709+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.567535+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.567535+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:45.095729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":713,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.557326+0000","last_change":"2024-09-14T04:49:29.616831+0000","last_active":"2024-09-14T04:55:37.557326+0000","last_peered":"2024-09-14T04:55:37.557326+0000","last_clean":"2024-09-14T04:55:37.557326+0000","last_became_active":"2024-09-14T04:49:29.616546+0000","last_became_peered":"2024-09-14T04:49:29.616546+0000","last_unstale":"2024-09-14T04:55:37.557326+0000","last_undegraded":"2024-09-14T04:55:37.557326+0000","last_fullsized":"2024-09-14T04:55:37.557326+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:27.882929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":297,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:51.693055+0000","last_change":"2024-09-14T04:44:57.330746+0000","last_active":"2024-09-14T04:55:51.693055+0000","last_peered":"2024-09-14T04:55:51.693055+0000","last_clean":"2024-09-14T04:55:51.693055+0000","last_became_active":"2024-09-14T04:44:57.330327+0000","last_became_peered":"2024-09-14T04:44:57.330327+0000","last_unstale":"2024-09-14T04:55:51.693055+0000","last_undegraded":"2024-09-14T04:55:51.693055+0000","last_fullsized":"2024-09-14T04:55:51.693055+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:53.122771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":163,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.558456+0000","last_change":"2024-09-14T04:43:13.355296+0000","last_active":"2024-09-14T04:55:37.558456+0000","last_peered":"2024-09-14T04:55:37.558456+0000","last_clean":"2024-09-14T04:55:37.558456+0000","last_became_active":"2024-09-14T04:43:13.354950+0000","last_became_peered":"2024-09-14T04:43:13.354950+0000","last_unstale":"2024-09-14T04:55:37.558456+0000","last_undegraded":"2024-09-14T04:55:37.558456+0000","last_fullsized":"2024-09-14T04:55:37.558456+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:20:08.968818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"349'149","reported_seq":820,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.553072+0000","last_change":"2024-09-14T04:41:39.471393+0000","last_active":"2024-09-14T04:55:37.553072+0000","last_peered":"2024-09-14T04:55:37.553072+0000","last_clean":"2024-09-14T04:55:37.553072+0000","last_became_active":"2024-09-14T04:41:36.931768+0000","last_became_peered":"2024-09-14T04:41:36.931768+0000","last_unstale":"2024-09-14T04:55:37.553072+0000","last_undegraded":"2024-09-14T04:55:37.553072+0000","last_fullsized":"2024-09-14T04:55:37.553072+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.471248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.471248+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:56.210375+0000","scrub_duration":0.021758344999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2297,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:51.699033+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T04:55:51.699033+0000","last_peered":"2024-09-14T04:55:51.699033+0000","last_clean":"2024-09-14T04:55:51.699033+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T04:55:51.699033+0000","last_undegraded":"2024-09-14T04:55:51.699033+0000","last_fullsized":"2024-09-14T04:55:51.699033+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1530,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.557819+0000","last_change":"2024-09-14T04:44:57.331436+0000","last_active":"2024-09-14T04:55:37.557819+0000","last_peered":"2024-09-14T04:55:37.557819+0000","last_clean":"2024-09-14T04:55:37.557819+0000","last_became_active":"2024-09-14T04:44:57.330957+0000","last_became_peered":"2024-09-14T04:44:57.330957+0000","last_unstale":"2024-09-14T04:55:37.557819+0000","last_undegraded":"2024-09-14T04:55:37.557819+0000","last_fullsized":"2024-09-14T04:55:37.557819+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:09.516688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":846,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:51.695272+0000","last_change":"2024-09-14T04:49:49.460401+0000","last_active":"2024-09-14T04:55:51.695272+0000","last_peered":"2024-09-14T04:55:51.695272+0000","last_clean":"2024-09-14T04:55:51.695272+0000","last_became_active":"2024-09-14T04:49:49.458530+0000","last_became_peered":"2024-09-14T04:49:49.458530+0000","last_unstale":"2024-09-14T04:55:51.695272+0000","last_undegraded":"2024-09-14T04:55:51.695272+0000","last_fullsized":"2024-09-14T04:55:51.695272+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:11:29.143239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"349'149","reported_seq":854,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.553315+0000","last_change":"2024-09-14T04:54:22.147441+0000","last_active":"2024-09-14T04:55:37.553315+0000","last_peered":"2024-09-14T04:55:37.553315+0000","last_clean":"2024-09-14T04:55:37.553315+0000","last_became_active":"2024-09-14T04:54:22.147115+0000","last_became_peered":"2024-09-14T04:54:22.147115+0000","last_unstale":"2024-09-14T04:55:37.553315+0000","last_undegraded":"2024-09-14T04:55:37.553315+0000","last_fullsized":"2024-09-14T04:55:37.553315+0000","mapping_epoch":1492,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1493,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:38.479887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:38.479887+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:41:14.001781+0000","scrub_duration":0.0062728319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":252,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:51.692275+0000","last_change":"2024-09-14T04:44:57.329983+0000","last_active":"2024-09-14T04:55:51.692275+0000","last_peered":"2024-09-14T04:55:51.692275+0000","last_clean":"2024-09-14T04:55:51.692275+0000","last_became_active":"2024-09-14T04:44:57.329597+0000","last_became_peered":"2024-09-14T04:44:57.329597+0000","last_unstale":"2024-09-14T04:55:51.692275+0000","last_undegraded":"2024-09-14T04:55:51.692275+0000","last_fullsized":"2024-09-14T04:55:51.692275+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:45.718039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":790,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:51.698642+0000","last_change":"2024-09-14T04:49:49.443631+0000","last_active":"2024-09-14T04:55:51.698642+0000","last_peered":"2024-09-14T04:55:51.698642+0000","last_clean":"2024-09-14T04:55:51.698642+0000","last_became_active":"2024-09-14T04:49:49.443313+0000","last_became_peered":"2024-09-14T04:49:49.443313+0000","last_unstale":"2024-09-14T04:55:51.698642+0000","last_undegraded":"2024-09-14T04:55:51.698642+0000","last_fullsized":"2024-09-14T04:55:51.698642+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:06:54.562292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1503,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:44.485689+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T04:55:44.485689+0000","last_peered":"2024-09-14T04:55:44.485689+0000","last_clean":"2024-09-14T04:55:44.485689+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T04:55:44.485689+0000","last_undegraded":"2024-09-14T04:55:44.485689+0000","last_fullsized":"2024-09-14T04:55:44.485689+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"349'149","reported_seq":812,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.549656+0000","last_change":"2024-09-14T04:41:39.012977+0000","last_active":"2024-09-14T04:55:37.549656+0000","last_peered":"2024-09-14T04:55:37.549656+0000","last_clean":"2024-09-14T04:55:37.549656+0000","last_became_active":"2024-09-14T04:41:36.932552+0000","last_became_peered":"2024-09-14T04:41:36.932552+0000","last_unstale":"2024-09-14T04:55:37.549656+0000","last_undegraded":"2024-09-14T04:55:37.549656+0000","last_fullsized":"2024-09-14T04:55:37.549656+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.012879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.012879+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:16:40.080262+0000","scrub_duration":0.0051721620000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":882,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:51.695426+0000","last_change":"2024-09-14T04:49:49.459590+0000","last_active":"2024-09-14T04:55:51.695426+0000","last_peered":"2024-09-14T04:55:51.695426+0000","last_clean":"2024-09-14T04:55:51.695426+0000","last_became_active":"2024-09-14T04:49:49.456263+0000","last_became_peered":"2024-09-14T04:49:49.456263+0000","last_unstale":"2024-09-14T04:55:51.695426+0000","last_undegraded":"2024-09-14T04:55:51.695426+0000","last_fullsized":"2024-09-14T04:55:51.695426+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:20:44.666805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":124,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.550119+0000","last_change":"2024-09-14T04:44:57.329652+0000","last_active":"2024-09-14T04:55:37.550119+0000","last_peered":"2024-09-14T04:55:37.550119+0000","last_clean":"2024-09-14T04:55:37.550119+0000","last_became_active":"2024-09-14T04:44:57.328888+0000","last_became_peered":"2024-09-14T04:44:57.328888+0000","last_unstale":"2024-09-14T04:55:37.550119+0000","last_undegraded":"2024-09-14T04:55:37.550119+0000","last_fullsized":"2024-09-14T04:55:37.550119+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:10.170068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1501,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.554570+0000","last_change":"2024-09-14T04:06:46.875340+0000","last_active":"2024-09-14T04:55:37.554570+0000","last_peered":"2024-09-14T04:55:37.554570+0000","last_clean":"2024-09-14T04:55:37.554570+0000","last_became_active":"2024-09-14T04:06:46.874552+0000","last_became_peered":"2024-09-14T04:06:46.874552+0000","last_unstale":"2024-09-14T04:55:37.554570+0000","last_undegraded":"2024-09-14T04:55:37.554570+0000","last_fullsized":"2024-09-14T04:55:37.554570+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:52.257357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"349'149","reported_seq":715,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.558108+0000","last_change":"2024-09-14T04:54:55.279666+0000","last_active":"2024-09-14T04:55:37.558108+0000","last_peered":"2024-09-14T04:55:37.558108+0000","last_clean":"2024-09-14T04:55:37.558108+0000","last_became_active":"2024-09-14T04:54:55.279259+0000","last_became_peered":"2024-09-14T04:54:55.279259+0000","last_unstale":"2024-09-14T04:55:37.558108+0000","last_undegraded":"2024-09-14T04:55:37.558108+0000","last_fullsized":"2024-09-14T04:55:37.558108+0000","mapping_epoch":1506,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1507,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:54:03.996025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:54:03.996025+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:40:53.332369+0000","scrub_duration":54,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":245,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:51.693048+0000","last_change":"2024-09-14T04:44:14.489235+0000","last_active":"2024-09-14T04:55:51.693048+0000","last_peered":"2024-09-14T04:55:51.693048+0000","last_clean":"2024-09-14T04:55:51.693048+0000","last_became_active":"2024-09-14T04:44:14.488950+0000","last_became_peered":"2024-09-14T04:44:14.488950+0000","last_unstale":"2024-09-14T04:55:51.693048+0000","last_undegraded":"2024-09-14T04:55:51.693048+0000","last_fullsized":"2024-09-14T04:55:51.693048+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:54.441144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":801,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:51.699626+0000","last_change":"2024-09-14T04:49:49.456317+0000","last_active":"2024-09-14T04:55:51.699626+0000","last_peered":"2024-09-14T04:55:51.699626+0000","last_clean":"2024-09-14T04:55:51.699626+0000","last_became_active":"2024-09-14T04:49:49.456029+0000","last_became_peered":"2024-09-14T04:49:49.456029+0000","last_unstale":"2024-09-14T04:55:51.699626+0000","last_undegraded":"2024-09-14T04:55:51.699626+0000","last_fullsized":"2024-09-14T04:55:51.699626+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:46:05.319676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2318,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:51.696085+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T04:55:51.696085+0000","last_peered":"2024-09-14T04:55:51.696085+0000","last_clean":"2024-09-14T04:55:51.696085+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T04:55:51.696085+0000","last_undegraded":"2024-09-14T04:55:51.696085+0000","last_fullsized":"2024-09-14T04:55:51.696085+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"349'149","reported_seq":706,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.555085+0000","last_change":"2024-09-14T04:50:22.002798+0000","last_active":"2024-09-14T04:55:37.555085+0000","last_peered":"2024-09-14T04:55:37.555085+0000","last_clean":"2024-09-14T04:55:37.555085+0000","last_became_active":"2024-09-14T04:49:49.448278+0000","last_became_peered":"2024-09-14T04:49:49.448278+0000","last_unstale":"2024-09-14T04:55:37.555085+0000","last_undegraded":"2024-09-14T04:55:37.555085+0000","last_fullsized":"2024-09-14T04:55:37.555085+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:50:22.002568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:50:22.002568+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:01:57.489223+0000","scrub_duration":0.010888326,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2319,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:51.700849+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T04:55:51.700849+0000","last_peered":"2024-09-14T04:55:51.700849+0000","last_clean":"2024-09-14T04:55:51.700849+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T04:55:51.700849+0000","last_undegraded":"2024-09-14T04:55:51.700849+0000","last_fullsized":"2024-09-14T04:55:51.700849+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2339,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:51.702239+0000","last_change":"2024-09-14T04:54:19.447103+0000","last_active":"2024-09-14T04:55:51.702239+0000","last_peered":"2024-09-14T04:55:51.702239+0000","last_clean":"2024-09-14T04:55:51.702239+0000","last_became_active":"2024-09-14T04:54:19.446748+0000","last_became_peered":"2024-09-14T04:54:19.446748+0000","last_unstale":"2024-09-14T04:55:51.702239+0000","last_undegraded":"2024-09-14T04:55:51.702239+0000","last_fullsized":"2024-09-14T04:55:51.702239+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:00:55.369263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1531,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.557492+0000","last_change":"2024-09-14T04:42:49.939790+0000","last_active":"2024-09-14T04:55:37.557492+0000","last_peered":"2024-09-14T04:55:37.557492+0000","last_clean":"2024-09-14T04:55:37.557492+0000","last_became_active":"2024-09-14T04:42:49.939591+0000","last_became_peered":"2024-09-14T04:42:49.939591+0000","last_unstale":"2024-09-14T04:55:37.557492+0000","last_undegraded":"2024-09-14T04:55:37.557492+0000","last_fullsized":"2024-09-14T04:55:37.557492+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:23:16.790593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1495,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:43.861788+0000","last_change":"2024-09-14T04:44:57.326008+0000","last_active":"2024-09-14T04:55:43.861788+0000","last_peered":"2024-09-14T04:55:43.861788+0000","last_clean":"2024-09-14T04:55:43.861788+0000","last_became_active":"2024-09-14T04:44:57.325617+0000","last_became_peered":"2024-09-14T04:44:57.325617+0000","last_unstale":"2024-09-14T04:55:43.861788+0000","last_undegraded":"2024-09-14T04:55:43.861788+0000","last_fullsized":"2024-09-14T04:55:43.861788+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:15.637508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":784,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:44.485614+0000","last_change":"2024-09-14T04:43:13.354846+0000","last_active":"2024-09-14T04:55:44.485614+0000","last_peered":"2024-09-14T04:55:44.485614+0000","last_clean":"2024-09-14T04:55:44.485614+0000","last_became_active":"2024-09-14T04:43:13.354515+0000","last_became_peered":"2024-09-14T04:43:13.354515+0000","last_unstale":"2024-09-14T04:55:44.485614+0000","last_undegraded":"2024-09-14T04:55:44.485614+0000","last_fullsized":"2024-09-14T04:55:44.485614+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:17.889341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1500,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.547378+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T04:55:37.547378+0000","last_peered":"2024-09-14T04:55:37.547378+0000","last_clean":"2024-09-14T04:55:37.547378+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T04:55:37.547378+0000","last_undegraded":"2024-09-14T04:55:37.547378+0000","last_fullsized":"2024-09-14T04:55:37.547378+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":1838,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.557254+0000","last_change":"2024-09-14T04:41:30.640651+0000","last_active":"2024-09-14T04:55:37.557254+0000","last_peered":"2024-09-14T04:55:37.557254+0000","last_clean":"2024-09-14T04:55:37.557254+0000","last_became_active":"2024-09-14T04:41:29.532466+0000","last_became_peered":"2024-09-14T04:41:29.532466+0000","last_unstale":"2024-09-14T04:55:37.557254+0000","last_undegraded":"2024-09-14T04:55:37.557254+0000","last_fullsized":"2024-09-14T04:55:37.557254+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":0,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:30.640520+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:30.640520+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:27.776724+0000","scrub_duration":0.0028653139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"349'149","reported_seq":812,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.554495+0000","last_change":"2024-09-14T04:41:39.978080+0000","last_active":"2024-09-14T04:55:37.554495+0000","last_peered":"2024-09-14T04:55:37.554495+0000","last_clean":"2024-09-14T04:55:37.554495+0000","last_became_active":"2024-09-14T04:41:36.929279+0000","last_became_peered":"2024-09-14T04:41:36.929279+0000","last_unstale":"2024-09-14T04:55:37.554495+0000","last_undegraded":"2024-09-14T04:55:37.554495+0000","last_fullsized":"2024-09-14T04:55:37.554495+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.977999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.977999+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:44:06.383827+0000","scrub_duration":0.00192122,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":179,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:51.694768+0000","last_change":"2024-09-14T04:48:11.040133+0000","last_active":"2024-09-14T04:55:51.694768+0000","last_peered":"2024-09-14T04:55:51.694768+0000","last_clean":"2024-09-14T04:55:51.694768+0000","last_became_active":"2024-09-14T04:48:11.039787+0000","last_became_peered":"2024-09-14T04:48:11.039787+0000","last_unstale":"2024-09-14T04:55:51.694768+0000","last_undegraded":"2024-09-14T04:55:51.694768+0000","last_fullsized":"2024-09-14T04:55:51.694768+0000","mapping_epoch":1428,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:53.520672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2165,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:51.692275+0000","last_change":"2024-09-14T04:49:29.616443+0000","last_active":"2024-09-14T04:55:51.692275+0000","last_peered":"2024-09-14T04:55:51.692275+0000","last_clean":"2024-09-14T04:55:51.692275+0000","last_became_active":"2024-09-14T04:49:29.616066+0000","last_became_peered":"2024-09-14T04:49:29.616066+0000","last_unstale":"2024-09-14T04:55:51.692275+0000","last_undegraded":"2024-09-14T04:55:51.692275+0000","last_fullsized":"2024-09-14T04:55:51.692275+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:09:15.897957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":123,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.549777+0000","last_change":"2024-09-14T04:44:57.326724+0000","last_active":"2024-09-14T04:55:37.549777+0000","last_peered":"2024-09-14T04:55:37.549777+0000","last_clean":"2024-09-14T04:55:37.549777+0000","last_became_active":"2024-09-14T04:44:57.326191+0000","last_became_peered":"2024-09-14T04:44:57.326191+0000","last_unstale":"2024-09-14T04:55:37.549777+0000","last_undegraded":"2024-09-14T04:55:37.549777+0000","last_fullsized":"2024-09-14T04:55:37.549777+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:46.060269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":124,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:48.306849+0000","last_change":"2024-09-14T04:44:57.328528+0000","last_active":"2024-09-14T04:55:48.306849+0000","last_peered":"2024-09-14T04:55:48.306849+0000","last_clean":"2024-09-14T04:55:48.306849+0000","last_became_active":"2024-09-14T04:44:57.327739+0000","last_became_peered":"2024-09-14T04:44:57.327739+0000","last_unstale":"2024-09-14T04:55:48.306849+0000","last_undegraded":"2024-09-14T04:55:48.306849+0000","last_fullsized":"2024-09-14T04:55:48.306849+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:04:58.294690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"349'149","reported_seq":840,"reported_epoch":1511,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2024-09-14T04:55:51.699307+0000","last_change":"2024-09-14T04:54:36.689611+0000","last_active":"2024-09-14T04:55:51.699307+0000","last_peered":"2024-09-14T04:55:51.699307+0000","last_clean":"2024-09-14T04:54:31.592636+0000","last_became_active":"2024-09-14T04:54:36.676136+0000","last_became_peered":"2024-09-14T04:54:36.676136+0000","last_unstale":"2024-09-14T04:55:51.699307+0000","last_undegraded":"2024-09-14T04:54:36.658607+0000","last_fullsized":"2024-09-14T04:54:36.658065+0000","mapping_epoch":1499,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1479,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:56.738636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:56.738636+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"no scrub is scheduled","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[2,3],"avail_no_missing":["2","3","6"],"object_location_counts":[{"shards":"2,3,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":123,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.554584+0000","last_change":"2024-09-14T04:44:57.323436+0000","last_active":"2024-09-14T04:55:37.554584+0000","last_peered":"2024-09-14T04:55:37.554584+0000","last_clean":"2024-09-14T04:55:37.554584+0000","last_became_active":"2024-09-14T04:44:57.323022+0000","last_became_peered":"2024-09-14T04:44:57.323022+0000","last_unstale":"2024-09-14T04:55:37.554584+0000","last_undegraded":"2024-09-14T04:55:37.554584+0000","last_fullsized":"2024-09-14T04:55:37.554584+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:22.775261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1529,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.552744+0000","last_change":"2024-09-14T04:44:57.322357+0000","last_active":"2024-09-14T04:55:37.552744+0000","last_peered":"2024-09-14T04:55:37.552744+0000","last_clean":"2024-09-14T04:55:37.552744+0000","last_became_active":"2024-09-14T04:44:57.321990+0000","last_became_peered":"2024-09-14T04:44:57.321990+0000","last_unstale":"2024-09-14T04:55:37.552744+0000","last_undegraded":"2024-09-14T04:55:37.552744+0000","last_fullsized":"2024-09-14T04:55:37.552744+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:09.658008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"349'149","reported_seq":1854,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.557588+0000","last_change":"2024-09-14T04:41:53.858076+0000","last_active":"2024-09-14T04:55:37.557588+0000","last_peered":"2024-09-14T04:55:37.557588+0000","last_clean":"2024-09-14T04:55:37.557588+0000","last_became_active":"2024-09-14T04:41:36.931585+0000","last_became_peered":"2024-09-14T04:41:36.931585+0000","last_unstale":"2024-09-14T04:55:37.557588+0000","last_undegraded":"2024-09-14T04:55:37.557588+0000","last_fullsized":"2024-09-14T04:55:37.557588+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:53.857924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:53.857924+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:10:20.658986+0000","scrub_duration":0.0022355919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":123,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.549669+0000","last_change":"2024-09-14T04:44:57.329198+0000","last_active":"2024-09-14T04:55:37.549669+0000","last_peered":"2024-09-14T04:55:37.549669+0000","last_clean":"2024-09-14T04:55:37.549669+0000","last_became_active":"2024-09-14T04:44:57.327754+0000","last_became_peered":"2024-09-14T04:44:57.327754+0000","last_unstale":"2024-09-14T04:55:37.549669+0000","last_undegraded":"2024-09-14T04:55:37.549669+0000","last_fullsized":"2024-09-14T04:55:37.549669+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:10:27.944476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":302,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:51.699109+0000","last_change":"2024-09-14T04:44:57.348687+0000","last_active":"2024-09-14T04:55:51.699109+0000","last_peered":"2024-09-14T04:55:51.699109+0000","last_clean":"2024-09-14T04:55:51.699109+0000","last_became_active":"2024-09-14T04:44:57.332845+0000","last_became_peered":"2024-09-14T04:44:57.332845+0000","last_unstale":"2024-09-14T04:55:51.699109+0000","last_undegraded":"2024-09-14T04:55:51.699109+0000","last_fullsized":"2024-09-14T04:55:51.699109+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:12:18.762737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2325,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:51.698906+0000","last_change":"2024-09-14T04:44:57.333468+0000","last_active":"2024-09-14T04:55:51.698906+0000","last_peered":"2024-09-14T04:55:51.698906+0000","last_clean":"2024-09-14T04:55:51.698906+0000","last_became_active":"2024-09-14T04:44:57.332626+0000","last_became_peered":"2024-09-14T04:44:57.332626+0000","last_unstale":"2024-09-14T04:55:51.698906+0000","last_undegraded":"2024-09-14T04:55:51.698906+0000","last_fullsized":"2024-09-14T04:55:51.698906+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:57:26.366438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1485,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.546775+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T04:55:37.546775+0000","last_peered":"2024-09-14T04:55:37.546775+0000","last_clean":"2024-09-14T04:55:37.546775+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T04:55:37.546775+0000","last_undegraded":"2024-09-14T04:55:37.546775+0000","last_fullsized":"2024-09-14T04:55:37.546775+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1484,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.552628+0000","last_change":"2024-09-14T04:06:46.869868+0000","last_active":"2024-09-14T04:55:37.552628+0000","last_peered":"2024-09-14T04:55:37.552628+0000","last_clean":"2024-09-14T04:55:37.552628+0000","last_became_active":"2024-09-14T04:06:46.869126+0000","last_became_peered":"2024-09-14T04:06:46.869126+0000","last_unstale":"2024-09-14T04:55:37.552628+0000","last_undegraded":"2024-09-14T04:55:37.552628+0000","last_fullsized":"2024-09-14T04:55:37.552628+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:41.150138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1468,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.557738+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T04:55:37.557738+0000","last_peered":"2024-09-14T04:55:37.557738+0000","last_clean":"2024-09-14T04:55:37.557738+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T04:55:37.557738+0000","last_undegraded":"2024-09-14T04:55:37.557738+0000","last_fullsized":"2024-09-14T04:55:37.557738+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":124,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:48.306875+0000","last_change":"2024-09-14T04:44:57.332112+0000","last_active":"2024-09-14T04:55:48.306875+0000","last_peered":"2024-09-14T04:55:48.306875+0000","last_clean":"2024-09-14T04:55:48.306875+0000","last_became_active":"2024-09-14T04:44:57.331575+0000","last_became_peered":"2024-09-14T04:44:57.331575+0000","last_unstale":"2024-09-14T04:55:48.306875+0000","last_undegraded":"2024-09-14T04:55:48.306875+0000","last_fullsized":"2024-09-14T04:55:48.306875+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:50:00.391283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1516,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.557815+0000","last_change":"2024-09-14T04:44:57.351460+0000","last_active":"2024-09-14T04:55:37.557815+0000","last_peered":"2024-09-14T04:55:37.557815+0000","last_clean":"2024-09-14T04:55:37.557815+0000","last_became_active":"2024-09-14T04:44:57.334662+0000","last_became_peered":"2024-09-14T04:44:57.334662+0000","last_unstale":"2024-09-14T04:55:37.557815+0000","last_undegraded":"2024-09-14T04:55:37.557815+0000","last_fullsized":"2024-09-14T04:55:37.557815+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:02.399062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1470,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.547829+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T04:55:37.547829+0000","last_peered":"2024-09-14T04:55:37.547829+0000","last_clean":"2024-09-14T04:55:37.547829+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T04:55:37.547829+0000","last_undegraded":"2024-09-14T04:55:37.547829+0000","last_fullsized":"2024-09-14T04:55:37.547829+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1528,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.558376+0000","last_change":"2024-09-14T04:44:57.349690+0000","last_active":"2024-09-14T04:55:37.558376+0000","last_peered":"2024-09-14T04:55:37.558376+0000","last_clean":"2024-09-14T04:55:37.558376+0000","last_became_active":"2024-09-14T04:44:57.333579+0000","last_became_peered":"2024-09-14T04:44:57.333579+0000","last_unstale":"2024-09-14T04:55:37.558376+0000","last_undegraded":"2024-09-14T04:55:37.558376+0000","last_fullsized":"2024-09-14T04:55:37.558376+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:17:55.265141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1605,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.557365+0000","last_change":"2024-09-14T04:54:35.694089+0000","last_active":"2024-09-14T04:55:37.557365+0000","last_peered":"2024-09-14T04:55:37.557365+0000","last_clean":"2024-09-14T04:55:37.557365+0000","last_became_active":"2024-09-14T04:54:35.663133+0000","last_became_peered":"2024-09-14T04:54:35.663133+0000","last_unstale":"2024-09-14T04:55:37.557365+0000","last_undegraded":"2024-09-14T04:55:37.557365+0000","last_fullsized":"2024-09-14T04:55:37.557365+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:17:19.947280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":18,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.558134+0000","last_change":"2024-09-14T04:54:52.248439+0000","last_active":"2024-09-14T04:55:37.558134+0000","last_peered":"2024-09-14T04:55:37.558134+0000","last_clean":"2024-09-14T04:55:37.558134+0000","last_became_active":"2024-09-14T04:54:52.248235+0000","last_became_peered":"2024-09-14T04:54:52.248235+0000","last_unstale":"2024-09-14T04:55:37.558134+0000","last_undegraded":"2024-09-14T04:55:37.558134+0000","last_fullsized":"2024-09-14T04:55:37.558134+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:22:09.548139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1490,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.557776+0000","last_change":"2024-09-14T04:44:57.332511+0000","last_active":"2024-09-14T04:55:37.557776+0000","last_peered":"2024-09-14T04:55:37.557776+0000","last_clean":"2024-09-14T04:55:37.557776+0000","last_became_active":"2024-09-14T04:44:57.332166+0000","last_became_peered":"2024-09-14T04:44:57.332166+0000","last_unstale":"2024-09-14T04:55:37.557776+0000","last_undegraded":"2024-09-14T04:55:37.557776+0000","last_fullsized":"2024-09-14T04:55:37.557776+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:53.984639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2318,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:51.700586+0000","last_change":"2024-09-14T04:54:52.250655+0000","last_active":"2024-09-14T04:55:51.700586+0000","last_peered":"2024-09-14T04:55:51.700586+0000","last_clean":"2024-09-14T04:55:51.700586+0000","last_became_active":"2024-09-14T04:54:52.250317+0000","last_became_peered":"2024-09-14T04:54:52.250317+0000","last_unstale":"2024-09-14T04:55:51.700586+0000","last_undegraded":"2024-09-14T04:55:51.700586+0000","last_fullsized":"2024-09-14T04:55:51.700586+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:12.985406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2304,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:51.698856+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T04:55:51.698856+0000","last_peered":"2024-09-14T04:55:51.698856+0000","last_clean":"2024-09-14T04:55:51.698856+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T04:55:51.698856+0000","last_undegraded":"2024-09-14T04:55:51.698856+0000","last_fullsized":"2024-09-14T04:55:51.698856+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1483,"reported_epoch":1511,"state":"active+clean","last_fresh":"2024-09-14T04:55:51.683359+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T04:55:51.683359+0000","last_peered":"2024-09-14T04:55:51.683359+0000","last_clean":"2024-09-14T04:55:51.683359+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T04:55:51.683359+0000","last_undegraded":"2024-09-14T04:55:51.683359+0000","last_fullsized":"2024-09-14T04:55:51.683359+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1519,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.558933+0000","last_change":"2024-09-14T04:44:57.339993+0000","last_active":"2024-09-14T04:55:37.558933+0000","last_peered":"2024-09-14T04:55:37.558933+0000","last_clean":"2024-09-14T04:55:37.558933+0000","last_became_active":"2024-09-14T04:44:57.330472+0000","last_became_peered":"2024-09-14T04:44:57.330472+0000","last_unstale":"2024-09-14T04:55:37.558933+0000","last_undegraded":"2024-09-14T04:55:37.558933+0000","last_fullsized":"2024-09-14T04:55:37.558933+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:19:27.046988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2067,"reported_epoch":1510,"state":"active+clean","last_fresh":"2024-09-14T04:55:37.558057+0000","last_change":"2024-09-14T04:49:49.449687+0000","last_active":"2024-09-14T04:55:37.558057+0000","last_peered":"2024-09-14T04:55:37.558057+0000","last_clean":"2024-09-14T04:55:37.558057+0000","last_became_active":"2024-09-14T04:49:49.449144+0000","last_became_peered":"2024-09-14T04:49:49.449144+0000","last_unstale":"2024-09-14T04:55:37.558057+0000","last_undegraded":"2024-09-14T04:55:37.558057+0000","last_fullsized":"2024-09-14T04:55:37.558057+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:59:06.779730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":56,"num_pg":16,"stat_sum":{"num_bytes":1573781528,"num_objects":24015,"num_object_clones":0,"num_object_copies":72045,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4349,"num_objects_misplaced":1479,"num_objects_unfound":0,"num_objects_dirty":24015,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":205029,"num_write_kb":7329345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10322,"num_bytes_recovered":641990656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5508771840,"data_stored":5508759624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125283,"ondisk_log_size":125283,"up":48,"acting":46,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1050752,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":32,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":5070,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4250,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223202059,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1487396,"kb_used_data":780380,"kb_used_omap":1,"kb_used_meta":707006,"kb_avail":92257756,"statfs":{"total":95995035648,"available":94471942144,"internally_reserved":0,"allocated":799109120,"data_stored":793891163,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":723974603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463695,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1077084,"kb_used_data":410260,"kb_used_omap":1,"kb_used_meta":666814,"kb_avail":92668068,"statfs":{"total":95995035648,"available":94892101632,"internally_reserved":0,"allocated":420106240,"data_stored":414896868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":682817991},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725332,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1524048,"kb_used_data":762888,"kb_used_omap":1,"kb_used_meta":761150,"kb_avail":92221104,"statfs":{"total":95995035648,"available":94434410496,"internally_reserved":0,"allocated":781197312,"data_stored":775995510,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":779418054},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":1392,"seq":5978594476178,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1135076,"kb_used_data":841432,"kb_used_omap":117,"kb_used_meta":293514,"kb_avail":92610076,"statfs":{"total":95995035648,"available":94832717824,"internally_reserved":0,"allocated":861626368,"data_stored":856413035,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120638,"internal_metadata":300558530},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772075073,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1361476,"kb_used_data":635260,"kb_used_omap":71,"kb_used_meta":726136,"kb_avail":92383676,"statfs":{"total":95995035648,"available":94600884224,"internally_reserved":0,"allocated":650506240,"data_stored":645308400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":743564214},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707304009,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1174864,"kb_used_data":669192,"kb_used_omap":68,"kb_used_meta":505403,"kb_avail":92570288,"statfs":{"total":95995035648,"available":94791974912,"internally_reserved":0,"allocated":685252608,"data_stored":680051307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":517532880},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872729168,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1169228,"kb_used_data":595776,"kb_used_omap":67,"kb_used_meta":573372,"kb_avail":92575924,"statfs":{"total":95995035648,"available":94797746176,"internally_reserved":0,"allocated":610074624,"data_stored":604880374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":587133285},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056218,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1474900,"kb_used_data":857224,"kb_used_omap":74,"kb_used_meta":617589,"kb_avail":92270252,"statfs":{"total":95995035648,"available":94484738048,"internally_reserved":0,"allocated":877797376,"data_stored":872595543,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":632411829},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1901,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":858329088,"data_stored":858325016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":581435392,"data_stored":581435392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":669450240,"data_stored":669450240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":624168960,"data_stored":624164888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":828047360,"data_stored":828047360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":764149760,"data_stored":764149760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":397869056,"data_stored":397869056,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":785321984,"data_stored":785317912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:55:59.212 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph status --format=json 2024-09-14T04:55:59.252 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:55:59.274 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:59.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:59.275 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:55:59.376 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:55:59.399 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:59.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:59.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:55:59.501 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:55:59.523 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:55:59.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:59.524 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:55:59.625 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:55:59.647 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:59.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:59.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:55:59.749 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:59.772 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:59.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:59.773 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:55:59.874 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:55:59.907 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:55:59.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:55:59.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:00.009 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:00.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:55:59 smithi129 ceph-mon[101316]: osdmap e1512: 8 total, 8 up, 8 in 2024-09-14T04:56:00.040 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:00.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:55:59 smithi039 ceph-mon[111235]: osdmap e1512: 8 total, 8 up, 8 in 2024-09-14T04:56:00.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:55:59 smithi039 ceph-mon[106470]: osdmap e1512: 8 total, 8 up, 8 in 2024-09-14T04:56:00.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:00.041 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:00.142 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:00.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:00.178 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:00.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:00.271 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:56:00.277 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:56:00.279 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:00.303 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:00.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:00.304 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:00.405 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:00.450 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:00.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:00.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:00.552 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:00.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:00.594 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:00.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:00.695 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:00.731 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:00.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:00.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:00.833 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:00.870 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:00.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:00.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:00.944 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:00 smithi039 ceph-mon[106470]: osdmap e1513: 8 total, 8 up, 8 in 2024-09-14T04:56:00.944 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:00 smithi039 ceph-mon[106470]: pgmap v2874: 99 pgs: 2 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 2296/59316 objects degraded (3.871%); 2396/59316 objects misplaced (4.039%); 656 KiB/s, 77 objects/s recovering 2024-09-14T04:56:00.944 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:00 smithi039 ceph-mon[111235]: osdmap e1513: 8 total, 8 up, 8 in 2024-09-14T04:56:00.944 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:00 smithi039 ceph-mon[111235]: pgmap v2874: 99 pgs: 2 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 2296/59316 objects degraded (3.871%); 2396/59316 objects misplaced (4.039%); 656 KiB/s, 77 objects/s recovering 2024-09-14T04:56:00.971 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:01.004 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:01.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:01.005 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:01.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:00 smithi129 ceph-mon[101316]: osdmap e1513: 8 total, 8 up, 8 in 2024-09-14T04:56:01.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:00 smithi129 ceph-mon[101316]: pgmap v2874: 99 pgs: 2 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 2296/59316 objects degraded (3.871%); 2396/59316 objects misplaced (4.039%); 656 KiB/s, 77 objects/s recovering 2024-09-14T04:56:01.107 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:01.157 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:01.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:01.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:01.259 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:01.303 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:01.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:01.304 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:01.404 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:01.441 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:01.442 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:01.442 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:01.543 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:01.577 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:01.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:01.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:01.592 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:56:01.592 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:56:01.592 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:56:01.592 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:56:01.592 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:56:01.592 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:56:01.593 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:56:01.593 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:56:01.593 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:56:01.593 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:56:01.593 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:56:01.593 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:56:01.593 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:56:01.593 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:56:01.594 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:56:01.594 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:56:01.680 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:01.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:01.717 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:01.718 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:01.819 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:01.854 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:01.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:01.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:01.898 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:01 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/381970634' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:56:01.898 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:01 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 2296/59316 objects degraded (3.871%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:56:01.899 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:01 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/381970634' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:56:01.899 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:01 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 2296/59316 objects degraded (3.871%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:56:01.957 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:01.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:01.988 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:01.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:02.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:01 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/381970634' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:56:02.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:01 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 2296/59316 objects degraded (3.871%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:56:02.089 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:02.134 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:02.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:02.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:02.236 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:02.269 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:02.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:02.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:02.372 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:02.410 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:02.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:02.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:02.512 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:02.553 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:02.554 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:02.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:02.588 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:56:02.656 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:02.687 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:02.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:02.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:02.788 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:02.817 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:02.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:02.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:02.897 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:02 smithi039 ceph-mon[106470]: pgmap v2875: 99 pgs: 2 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 2296/59316 objects degraded (3.871%); 2396/59316 objects misplaced (4.039%); 656 KiB/s, 77 objects/s recovering 2024-09-14T04:56:02.897 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:02 smithi039 ceph-mon[111235]: pgmap v2875: 99 pgs: 2 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 2296/59316 objects degraded (3.871%); 2396/59316 objects misplaced (4.039%); 656 KiB/s, 77 objects/s recovering 2024-09-14T04:56:02.918 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:02.954 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:02.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:02.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:03.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:02 smithi129 ceph-mon[101316]: pgmap v2875: 99 pgs: 2 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.2 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 779 op/s; 2296/59316 objects degraded (3.871%); 2396/59316 objects misplaced (4.039%); 656 KiB/s, 77 objects/s recovering 2024-09-14T04:56:03.056 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:03.088 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:03.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:03.089 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:03.190 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:03.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:03.227 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:03.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:03.328 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:03.360 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:03.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:03.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:03.462 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:03.498 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:03.499 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:03.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:03.600 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:03.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:03.638 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:03.638 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:03.739 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:03.775 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:03.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:03.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:03.876 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:03.921 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:03.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:03.922 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:04.022 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:04.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:04.059 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:04.060 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:04.160 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:04.198 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:04.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:04.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:04.299 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:04.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:04.336 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:04.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:04.341 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:56:04.436 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:04.472 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:04.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:04.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:04.574 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:04.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:04.605 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:04.605 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:04.706 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:04.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:04.744 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:04.744 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:04.845 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:04.880 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:04.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:04.881 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:04.982 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:05.018 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:05.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:05.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:05.120 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:05.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:05.173 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:05.173 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:05.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:04 smithi129 ceph-mon[101316]: pgmap v2876: 99 pgs: 2 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 2296/52704 objects degraded (4.356%); 2396/52704 objects misplaced (4.546%); 656 KiB/s, 40 objects/s recovering 2024-09-14T04:56:05.274 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:05.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:04 smithi039 ceph-mon[106470]: pgmap v2876: 99 pgs: 2 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 2296/52704 objects degraded (4.356%); 2396/52704 objects misplaced (4.546%); 656 KiB/s, 40 objects/s recovering 2024-09-14T04:56:05.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:04 smithi039 ceph-mon[111235]: pgmap v2876: 99 pgs: 2 peering, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 95 active+clean; 1.1 GiB data, 8.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 2296/52704 objects degraded (4.356%); 2396/52704 objects misplaced (4.546%); 656 KiB/s, 40 objects/s recovering 2024-09-14T04:56:05.305 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:05.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:05.307 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:05.408 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:05.444 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:05.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:05.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:05.546 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:05.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:05.594 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:05.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:05.695 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:05.734 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:05.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:05.735 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:05.836 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:05.873 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:05.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:05.874 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:05.975 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:06.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:06.014 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:06.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:06.115 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:06.148 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:06.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:06.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:06.250 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:06.283 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:06.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:06.284 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:06.385 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:06.419 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:06.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:06.420 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:06.521 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:06.564 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:06.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:06.565 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:06.666 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:06.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:06.697 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:06.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:06.798 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:06.839 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:06.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:06.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:06.942 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:06.976 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is set 2024-09-14T04:56:06.979 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:06.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:06.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:07.081 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:07.118 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:07.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:07.119 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:07.220 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:07.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:07.259 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:07.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:07.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:06 smithi129 ceph-mon[101316]: pgmap v2877: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 981 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 2372/47154 objects degraded (5.030%); 1888/47154 objects misplaced (4.004%); 1.9 MiB/s, 60 objects/s recovering 2024-09-14T04:56:07.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:06 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 2372/47154 objects degraded (5.030%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:56:07.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:06 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1345625177' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:56:07.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:06 smithi039 ceph-mon[106470]: pgmap v2877: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 981 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 2372/47154 objects degraded (5.030%); 1888/47154 objects misplaced (4.004%); 1.9 MiB/s, 60 objects/s recovering 2024-09-14T04:56:07.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:06 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 2372/47154 objects degraded (5.030%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:56:07.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:06 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1345625177' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:56:07.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:06 smithi039 ceph-mon[111235]: pgmap v2877: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 981 MiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 2372/47154 objects degraded (5.030%); 1888/47154 objects misplaced (4.004%); 1.9 MiB/s, 60 objects/s recovering 2024-09-14T04:56:07.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:06 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 2372/47154 objects degraded (5.030%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:56:07.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:06 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1345625177' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:56:07.361 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:07.398 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:07.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:07.400 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:07.500 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:07.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:07.543 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:07.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:07.644 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:07.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:07.685 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:07.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:07.786 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:07.823 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:07.824 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:07.824 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:07.925 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:07.961 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:07.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:07.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:08.064 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:08.108 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:08.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:08.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:08.210 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:08.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:08.255 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:08.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:08.263 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:07 smithi039 ceph-mon[106470]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:56:08.263 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:07 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1345625177' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:56:08.263 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:07 smithi039 ceph-mon[106470]: osdmap e1514: 8 total, 8 up, 8 in 2024-09-14T04:56:08.263 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:56:08.264 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:07 smithi039 ceph-mon[111235]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:56:08.264 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:07 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1345625177' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:56:08.264 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:07 smithi039 ceph-mon[111235]: osdmap e1514: 8 total, 8 up, 8 in 2024-09-14T04:56:08.264 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:56:08.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:07 smithi129 ceph-mon[101316]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:56:08.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:07 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1345625177' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:56:08.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:07 smithi129 ceph-mon[101316]: osdmap e1514: 8 total, 8 up, 8 in 2024-09-14T04:56:08.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:56:08.293 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:56:08.317 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:56:08.318 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [ 2024-09-14T04:56:08.318 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:56:08.318 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:217151 56.8 e1514/1470 56:17c9b01d:::benchmark_data_smithi039_330847_object102627:head v 1514'13280, mlcod=1514'13277)", 2024-09-14T04:56:08.318 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:56:08.312186+0000", 2024-09-14T04:56:08.318 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.0039405680000000002, 2024-09-14T04:56:08.318 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0039716300000000003, 2024-09-14T04:56:08.318 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:56:08.318 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:56:08.318 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "started", 2024-09-14T04:56:08.318 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:56:08.318 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:56:08.318 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:56:08.318 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:56:08.312186+0000", 2024-09-14T04:56:08.318 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:56:08.319 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:56:08.319 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:56:08.319 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:56:08.319 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:56:08.312186+0000", 2024-09-14T04:56:08.319 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:56:08.319 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:56:08.319 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:56:08.319 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:56:08.319 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:56:08.312187+0000", 2024-09-14T04:56:08.319 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.7199999999999997e-07 2024-09-14T04:56:08.319 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:56:08.319 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:56:08.319 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:56:08.319 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:56:08.312194+0000", 2024-09-14T04:56:08.320 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.2869999999999999e-06 2024-09-14T04:56:08.320 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:56:08.320 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:56:08.320 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:56:08.320 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:56:08.312209+0000", 2024-09-14T04:56:08.320 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4888999999999999e-05 2024-09-14T04:56:08.320 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:56:08.320 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:56:08.320 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:56:08.320 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:56:08.312217+0000", 2024-09-14T04:56:08.320 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.3229999999999998e-06 2024-09-14T04:56:08.320 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:56:08.320 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:56:08.321 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:56:08.321 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:56:08.312406+0000", 2024-09-14T04:56:08.321 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00018866900000000001 2024-09-14T04:56:08.321 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:56:08.321 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:56:08.321 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:56:08.321 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:56:08.312480+0000", 2024-09-14T04:56:08.321 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.4238000000000003e-05 2024-09-14T04:56:08.321 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:56:08.321 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:56:08.321 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:56:08.321 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:56:08.321 INFO:teuthology.orchestra.run.smithi039.stdout: ], 2024-09-14T04:56:08.321 INFO:teuthology.orchestra.run.smithi039.stdout: "num_ops": 1 2024-09-14T04:56:08.322 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:56:08.357 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:08.395 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:08.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:08.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:08.497 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:08.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:08.536 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:08.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:08.637 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:08.684 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:08.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:08.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:08.786 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:08.830 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:08.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:08.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:08.899 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:56:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:56:08] "GET /metrics HTTP/1.1" 200 36760 "" "Prometheus/2.51.0" 2024-09-14T04:56:08.932 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:08.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:08.975 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:08.975 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:09.076 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:09.116 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:09.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:09.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:09.218 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:09.253 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:09.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:09.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:09.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:08 smithi129 ceph-mon[101316]: pgmap v2879: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 981 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 2372/47154 objects degraded (5.030%); 1888/47154 objects misplaced (4.004%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:56:09.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:08 smithi039 ceph-mon[106470]: pgmap v2879: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 981 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 2372/47154 objects degraded (5.030%); 1888/47154 objects misplaced (4.004%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:56:09.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:08 smithi039 ceph-mon[111235]: pgmap v2879: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 981 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 496 op/s; 2372/47154 objects degraded (5.030%); 1888/47154 objects misplaced (4.004%); 1.2 MiB/s, 19 objects/s recovering 2024-09-14T04:56:09.355 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:09.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:09.388 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:09.389 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:09.443 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:56:09.443 INFO:teuthology.orchestra.run.smithi039.stdout:{"fsid":"bd7c803c-724b-11ef-bceb-c7b262605968","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 2372/47154 objects degraded (5.030%), 1 pg degraded, 1 pg undersized","count":2},"muted":false}},"mutes":[]},"election_epoch":34,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":3285,"monmap":{"epoch":4,"min_mon_release_name":"squid","num_mons":3},"osdmap":{"epoch":1514,"num_osds":8,"num_up_osds":8,"osd_up_since":1726289028,"num_in_osds":8,"osd_in_since":1726285856,"num_remapped_pgs":2},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":97},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+remapped+backfill_wait","count":1}],"num_pgs":99,"num_pools":5,"num_objects":15718,"data_bytes":1028921341,"bytes_used":8888233984,"bytes_avail":759072051200,"bytes_total":767960285184,"degraded_objects":2372,"degraded_total":47154,"degraded_ratio":0.050303261653306189,"misplaced_objects":1888,"misplaced_total":47154,"misplaced_ratio":0.040039021079865969,"recovering_objects_per_sec":19,"recovering_bytes_per_sec":1252685,"recovering_keys_per_sec":0,"num_objects_recovered":156,"num_bytes_recovered":10223616,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":496},"fsmap":{"epoch":9,"btime":"1970-01-01T00:00:00:000000+0000","id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi039.bbwiwf","status":"up:active","gid":14646}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.39:8443/","prometheus":"http://172.21.15.39:9283/"}},"servicemap":{"epoch":40,"modified":"2024-09-14T04:53:57.838795+0000","services":{}},"progress_events":{}} 2024-09-14T04:56:09.490 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:09.532 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:09.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:09.533 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:09.635 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:09.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:09.672 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:09.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:09.773 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:09.778 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-14T04:56:09.819 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:09.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:09.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:09.920 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:09.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:09.968 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:09.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:10.067 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:56:10.069 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:10.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:10.106 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:10.106 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:10.206 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:10.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:10.246 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:10.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:10.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:10 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/702009973' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-14T04:56:10.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:10 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/702009973' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-14T04:56:10.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:10 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/702009973' entity='client.admin' cmd={"prefix": "status", "format": "json"} : dispatch 2024-09-14T04:56:10.347 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:10.393 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:10.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:10.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:10.495 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:10.530 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:10.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:10.531 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:10.632 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:10.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:10.664 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:10.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:10.765 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:10.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:10.797 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:10.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:10.898 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:10.941 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:10.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:10.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:11.043 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:11.079 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:11.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:11.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:11.181 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:11.222 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:11.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:11.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:11.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:11 smithi129 ceph-mon[101316]: pgmap v2880: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 705 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 2507/33906 objects degraded (7.394%); 1345/33906 objects misplaced (3.967%); 1.9 MiB/s, 29 objects/s recovering 2024-09-14T04:56:11.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:11 smithi039 ceph-mon[106470]: pgmap v2880: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 705 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 2507/33906 objects degraded (7.394%); 1345/33906 objects misplaced (3.967%); 1.9 MiB/s, 29 objects/s recovering 2024-09-14T04:56:11.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:11 smithi039 ceph-mon[111235]: pgmap v2880: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 705 MiB data, 7.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 2507/33906 objects degraded (7.394%); 1345/33906 objects misplaced (3.967%); 1.9 MiB/s, 29 objects/s recovering 2024-09-14T04:56:11.323 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:11.357 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:11.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:11.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:11.460 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:11.497 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:11.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:11.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:11.599 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:11.644 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:11.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:11.645 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:11.746 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:11.767 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:11.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:11.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:11.869 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:11.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:11.901 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:11.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:12.002 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:12.046 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:12.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:12.047 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:12.148 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:12.180 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:12.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:12.181 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:12.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:12 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 2507/33906 objects degraded (7.394%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:56:12.282 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:12.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:12 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 2507/33906 objects degraded (7.394%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:56:12.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:12 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 2507/33906 objects degraded (7.394%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:56:12.313 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:12.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:12.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:12.415 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:12.445 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:12.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:12.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:12.547 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:12.590 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:12.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:12.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:12.609 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2024-09-14T04:56:12.609 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:56:12.692 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:12.727 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:12.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:12.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:12.829 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:12.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:12.877 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:12.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:12.900 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:56:12.979 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:13.013 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:13.014 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:13.014 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:13.115 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:13.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:13.161 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:13.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:13.228 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:13 smithi039 ceph-mon[106470]: pgmap v2881: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 705 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 2507/33906 objects degraded (7.394%); 1345/33906 objects misplaced (3.967%); 1.9 MiB/s, 29 objects/s recovering 2024-09-14T04:56:13.229 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:13 smithi039 ceph-mon[111235]: pgmap v2881: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 705 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 2507/33906 objects degraded (7.394%); 1345/33906 objects misplaced (3.967%); 1.9 MiB/s, 29 objects/s recovering 2024-09-14T04:56:13.262 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:13.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:13.313 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:13.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:13.414 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:13.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:13.452 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:13.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:13.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:13 smithi129 ceph-mon[101316]: pgmap v2881: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 705 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 846 op/s; 2507/33906 objects degraded (7.394%); 1345/33906 objects misplaced (3.967%); 1.9 MiB/s, 29 objects/s recovering 2024-09-14T04:56:13.553 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:13.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:13.590 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:13.590 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:13.691 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:13.719 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:13.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:13.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:13.821 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:13.860 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:13.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:13.861 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:13.962 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:14.010 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:14.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:14.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:14.113 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:14.161 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:14.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:14.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:14.263 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:14.295 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:14.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:14.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:14.397 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:14.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:14.434 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:14.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:14.535 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:14.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:14.580 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:14.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:14.681 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:14.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:14.722 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:14.722 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:14.823 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:14.864 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:14.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:14.865 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:14.965 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:14.969 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:56:14.972 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:56:14.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:14.993 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:14.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:15.094 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:15.124 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:15.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:15.125 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:15.226 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:15.260 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:15.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:15.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:15.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:15 smithi039 ceph-mon[106470]: pgmap v2882: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 553 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 869 op/s; 2507/26616 objects degraded (9.419%); 1345/26616 objects misplaced (5.053%); 1.9 MiB/s, 29 objects/s recovering 2024-09-14T04:56:15.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:15 smithi039 ceph-mon[111235]: pgmap v2882: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 553 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 869 op/s; 2507/26616 objects degraded (9.419%); 1345/26616 objects misplaced (5.053%); 1.9 MiB/s, 29 objects/s recovering 2024-09-14T04:56:15.362 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:15.397 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:15.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:15.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:15.499 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:15.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:15 smithi129 ceph-mon[101316]: pgmap v2882: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 553 MiB data, 7.2 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 869 op/s; 2507/26616 objects degraded (9.419%); 1345/26616 objects misplaced (5.053%); 1.9 MiB/s, 29 objects/s recovering 2024-09-14T04:56:15.535 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:15.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:15.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:15.637 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:15.666 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:15.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:15.667 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:15.768 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:15.810 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:15.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:15.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:15.912 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:15.954 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:15.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:15.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:16.056 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:16.092 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:16.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:16.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:16.193 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:16.238 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:16.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:16.240 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:16.341 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:16.373 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:16.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:16.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:16.475 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:16.512 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:16.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:16.513 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:16.614 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:16.653 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:16.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:16.654 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:16.755 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:16.788 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:16.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:16.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:16.810 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:56:16.810 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:56:16.810 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:56:16.811 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:56:16.811 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:56:16.811 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:56:16.811 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:56:16.811 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:56:16.811 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:56:16.811 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:56:16.812 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:56:16.812 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:56:16.812 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:56:16.812 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:56:16.812 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:56:16.812 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:56:16.891 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:16.915 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:16.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:16.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:17.017 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:17.051 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:17.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:17.053 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:17.154 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:17.194 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:17.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:17.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:17.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:17 smithi039 ceph-mon[106470]: pgmap v2883: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 434 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 874 op/s; 2642/20901 objects degraded (12.641%); 785/20901 objects misplaced (3.756%); 1.8 MiB/s, 29 objects/s recovering 2024-09-14T04:56:17.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:17 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 2642/20901 objects degraded (12.641%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:56:17.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:17 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2169547167' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:56:17.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:17 smithi039 ceph-mon[111235]: pgmap v2883: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 434 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 874 op/s; 2642/20901 objects degraded (12.641%); 785/20901 objects misplaced (3.756%); 1.8 MiB/s, 29 objects/s recovering 2024-09-14T04:56:17.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:17 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 2642/20901 objects degraded (12.641%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:56:17.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:17 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2169547167' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:56:17.296 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:17.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:17.319 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:17.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:17.420 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:17.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:17.443 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:17.443 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:17.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:17 smithi129 ceph-mon[101316]: pgmap v2883: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 434 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 874 op/s; 2642/20901 objects degraded (12.641%); 785/20901 objects misplaced (3.756%); 1.8 MiB/s, 29 objects/s recovering 2024-09-14T04:56:17.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:17 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 2642/20901 objects degraded (12.641%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:56:17.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:17 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2169547167' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:56:17.544 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:17.566 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:17.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:17.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:17.668 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:17.689 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:17.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:17.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:17.791 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:17.828 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:17.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:17.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:17.930 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:17.977 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:17.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:17.978 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:18.079 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:18.108 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:18.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:18.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:18.210 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:18.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:18.248 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:18.248 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:18.349 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:18.383 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:18.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:18.384 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:18.484 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:18.539 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:18.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:18.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:18.641 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:18.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:18.664 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:18.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:18.766 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:18.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:18.800 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:18.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:18.901 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:18.946 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:18.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:18.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:19.032 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:56:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:56:18] "GET /metrics HTTP/1.1" 200 36760 "" "Prometheus/2.51.0" 2024-09-14T04:56:19.049 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:19.086 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:19.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:19.087 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:19.160 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:19 smithi039 ceph-mon[106470]: pgmap v2884: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 434 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 2642/20901 objects degraded (12.641%); 785/20901 objects misplaced (3.756%); 1.7 MiB/s, 26 objects/s recovering 2024-09-14T04:56:19.160 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:19 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:56:19.160 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:19 smithi039 ceph-mon[111235]: pgmap v2884: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 434 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 2642/20901 objects degraded (12.641%); 785/20901 objects misplaced (3.756%); 1.7 MiB/s, 26 objects/s recovering 2024-09-14T04:56:19.161 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:19 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:56:19.188 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:19.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:19.231 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:19.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:19.332 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:19.378 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:19.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:19.379 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:19.480 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:19.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:19 smithi129 ceph-mon[101316]: pgmap v2884: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 434 MiB data, 6.7 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 801 op/s; 2642/20901 objects degraded (12.641%); 785/20901 objects misplaced (3.756%); 1.7 MiB/s, 26 objects/s recovering 2024-09-14T04:56:19.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:19 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:56:19.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:19.523 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:19.524 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:19.624 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:19.654 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:19.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:19.655 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:19.756 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:19.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:19.798 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:19.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:19.899 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:19.939 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:19.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:19.940 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:20.041 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:20.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:20.087 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:20.087 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:20.188 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:20.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:20.232 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:20.233 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:20.334 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:20.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:20.388 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:20.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:20.489 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:20.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:20.524 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:20.525 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:20.624 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:56:20.625 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:56:20.625 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:20.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:20.661 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:20.661 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:20.762 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:20.802 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:20.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:20.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:20.904 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:20.956 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:20.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:20.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:21.058 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:21.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:21.101 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:21.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:21.202 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:21.233 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:21.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:21.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:21.317 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:21 smithi039 ceph-mon[106470]: pgmap v2885: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 189 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 2690/9135 objects degraded (29.447%); 317/9135 objects misplaced (3.470%); 2.5 MiB/s, 39 objects/s recovering 2024-09-14T04:56:21.317 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:21 smithi039 ceph-mon[111235]: pgmap v2885: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 189 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 2690/9135 objects degraded (29.447%); 317/9135 objects misplaced (3.470%); 2.5 MiB/s, 39 objects/s recovering 2024-09-14T04:56:21.335 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:21.370 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:21 smithi129 ceph-mon[101316]: pgmap v2885: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 189 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 2690/9135 objects degraded (29.447%); 317/9135 objects misplaced (3.470%); 2.5 MiB/s, 39 objects/s recovering 2024-09-14T04:56:21.372 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:21.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:21.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:21.475 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:21.507 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:21.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:21.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:21.608 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:21.645 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:21.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:21.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:21.747 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:21.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:21.784 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:21.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:21.885 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:21.934 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:21.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:21.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:22.036 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:22.068 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:22.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:22.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:22.074 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is unset 2024-09-14T04:56:22.170 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:22.211 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:22.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:22.212 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:22.312 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:22.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:22.352 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:22.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:22.439 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:22 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 2690/9135 objects degraded (29.447%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:56:22.439 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:22 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/883399029' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:56:22.439 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:22 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:56:22.439 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:22 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 2690/9135 objects degraded (29.447%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:56:22.439 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:22 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/883399029' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:56:22.439 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:22 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:56:22.454 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:22.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:22.485 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:22.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:22.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:22 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 2690/9135 objects degraded (29.447%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:56:22.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:22 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/883399029' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:56:22.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:22 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:56:22.586 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:22.632 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:22.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:22.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:22.734 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:22.768 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:22.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:22.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:22.839 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Removed 114521 objects 2024-09-14T04:56:22.839 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Clean up completed and total clean up time :140.813 2024-09-14T04:56:22.870 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:22.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:22.893 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:22.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:22.894 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2024-09-14T04:56:22.894 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2024-09-14T04:56:22.994 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:23.015 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:23.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:23.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:23.118 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:23.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:23.141 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:23.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:23.243 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:23.265 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:23.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:23.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:23 smithi039 ceph-mon[106470]: pgmap v2886: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 189 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 2690/9135 objects degraded (29.447%); 317/9135 objects misplaced (3.470%); 1.7 MiB/s, 27 objects/s recovering 2024-09-14T04:56:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:23 smithi039 ceph-mon[106470]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:56:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:23 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:56:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:23 smithi039 ceph-mon[106470]: osdmap e1515: 8 total, 8 up, 8 in 2024-09-14T04:56:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:56:23.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:23 smithi039 ceph-mon[111235]: pgmap v2886: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 189 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 2690/9135 objects degraded (29.447%); 317/9135 objects misplaced (3.470%); 1.7 MiB/s, 27 objects/s recovering 2024-09-14T04:56:23.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:23 smithi039 ceph-mon[111235]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:56:23.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:23 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:56:23.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:23 smithi039 ceph-mon[111235]: osdmap e1515: 8 total, 8 up, 8 in 2024-09-14T04:56:23.291 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:56:23.367 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:23.388 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:23.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:23.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:23.490 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:23.512 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:23.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:23.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:23.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:23 smithi129 ceph-mon[101316]: pgmap v2886: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 189 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 687 op/s; 2690/9135 objects degraded (29.447%); 317/9135 objects misplaced (3.470%); 1.7 MiB/s, 27 objects/s recovering 2024-09-14T04:56:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:23 smithi129 ceph-mon[101316]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:56:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:23 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:56:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:23 smithi129 ceph-mon[101316]: osdmap e1515: 8 total, 8 up, 8 in 2024-09-14T04:56:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:56:23.614 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:23.638 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:23.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:23.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:23.739 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:23.761 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:23.762 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:23.762 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:23.863 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:23.885 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:23.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:23.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:23.986 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:24.008 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:24.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:24.009 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:24.110 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:24.132 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:24.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:24.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:24.233 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:24.256 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:24.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:24.257 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:24.300 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-14T04:56:24.358 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:24.380 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:24.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:24.381 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:24.477 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:56:24.482 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:24.503 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:24.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:24.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:24.605 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:24.628 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:24.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:24.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:24.730 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:24.752 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:24.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:24.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:24.854 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:24.878 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:24.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:24.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:24.980 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:25.002 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:25.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:25.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:25.104 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:25.128 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:25.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:25.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:25.230 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:25.252 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:25.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:25.253 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:25.330 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:56:25.332 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T04:56:25.354 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:25.377 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:25.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:25.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:25.409 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:56:25.479 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:25.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:25.503 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:25.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:25.604 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:25.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:25.627 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:25.628 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:25.728 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:25.750 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:25.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:25.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:25.852 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:25.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:25.876 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:25.876 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:25.950 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:56:25.954 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:56:25.957 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:56:25.975 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:56:25.977 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:25.999 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:26.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:26.000 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:25 smithi129 ceph-mon[101316]: pgmap v2888: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 70 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 2690/3423 objects degraded (78.586%); 317/3423 objects misplaced (9.261%); 2.1 MiB/s, 33 objects/s recovering 2024-09-14T04:56:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:25 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:56:26.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:25 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:56:26.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:25 smithi039 ceph-mon[106470]: pgmap v2888: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 70 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 2690/3423 objects degraded (78.586%); 317/3423 objects misplaced (9.261%); 2.1 MiB/s, 33 objects/s recovering 2024-09-14T04:56:26.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:25 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:56:26.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:25 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:56:26.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:25 smithi039 ceph-mon[111235]: pgmap v2888: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 70 MiB data, 5.7 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 772 op/s; 2690/3423 objects degraded (78.586%); 317/3423 objects misplaced (9.261%); 2.1 MiB/s, 33 objects/s recovering 2024-09-14T04:56:26.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:25 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:56:26.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:25 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:56:26.101 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:26.123 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:26.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:26.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:26.224 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:26.247 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:26.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:26.249 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:26.349 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:26.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:26.374 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:26.374 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:26.475 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:26.499 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:26.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:26.500 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:26.601 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:26.624 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:26.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:26.625 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:26.725 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:26.748 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:26.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:26.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:26.850 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:26.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:26.873 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:26.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:26.961 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:26 smithi039 ceph-mon[106470]: from='client.36267 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:56:26.961 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:26 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 2690/3423 objects degraded (78.586%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:56:26.961 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:26 smithi039 ceph-mon[111235]: from='client.36267 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:56:26.961 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:26 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 2690/3423 objects degraded (78.586%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:56:26.975 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:26.997 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:26.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:26.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:27.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:26 smithi129 ceph-mon[101316]: from='client.36267 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:56:27.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:26 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 2690/3423 objects degraded (78.586%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:56:27.099 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:27.122 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:27.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:27.122 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:27.223 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:27.245 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:27.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:27.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:27.347 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:27.369 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:27.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:27.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:27.470 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:27.492 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:27.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:27.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:27.593 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:27.615 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:27.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:27.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:27.716 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:27.740 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:27.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:27.741 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:27.841 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:27.864 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:27.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:27.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:27.965 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:27.987 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:27.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:27.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:28.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:27 smithi129 ceph-mon[101316]: pgmap v2889: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 2742/78 objects degraded (3515.385%); 1.8 MiB/s, 54 objects/s recovering 2024-09-14T04:56:28.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:27 smithi039 ceph-mon[106470]: pgmap v2889: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 2742/78 objects degraded (3515.385%); 1.8 MiB/s, 54 objects/s recovering 2024-09-14T04:56:28.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:27 smithi039 ceph-mon[111235]: pgmap v2889: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 2742/78 objects degraded (3515.385%); 1.8 MiB/s, 54 objects/s recovering 2024-09-14T04:56:28.089 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:28.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:28.113 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:28.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:28.214 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:28.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:28.237 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:28.238 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:28.338 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:28.361 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:28.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:28.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:28.463 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:28.484 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:28.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:28.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:28.586 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:28.609 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:28.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:28.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:28.711 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:28.734 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:28.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:28.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:28.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:56:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:56:28] "GET /metrics HTTP/1.1" 200 36747 "" "Prometheus/2.51.0" 2024-09-14T04:56:28.835 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:28.857 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:28.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:28.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:28.959 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:28.980 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:28.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:28.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:29.082 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:29.106 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:29.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:29.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:29.208 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:29.231 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:29.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:29.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:29.333 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:29.356 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:29.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:29.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:29.458 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:29.482 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:29.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:29.483 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:29.583 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:29.606 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:29.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:29.607 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:29.708 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:29.729 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:29.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:29.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:29.831 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:29.854 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:29.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:29.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:29.892 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:56:29.892 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:56:29.893 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:29 smithi039 ceph-mon[111235]: pgmap v2890: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 2742/78 objects degraded (3515.385%); 1.8 MiB/s, 54 objects/s recovering 2024-09-14T04:56:29.893 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:29 smithi039 ceph-mon[106470]: pgmap v2890: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 2742/78 objects degraded (3515.385%); 1.8 MiB/s, 54 objects/s recovering 2024-09-14T04:56:29.956 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:29.977 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:29.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:29.980 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:30.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:29 smithi129 ceph-mon[101316]: pgmap v2890: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 582 KiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 693 op/s; 2742/78 objects degraded (3515.385%); 1.8 MiB/s, 54 objects/s recovering 2024-09-14T04:56:30.081 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:30.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:30.103 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:30.103 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:30.204 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:30.226 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:30.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:30.227 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:30.328 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:30.350 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:30.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:30.351 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:30.452 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:30.475 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:30.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:30.476 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:30.577 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:30.599 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:30.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:30.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:30.700 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":2888,"stamp":"2024-09-14T04:56:23.893971+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":73668581,"num_objects":1141,"num_object_clones":0,"num_object_copies":3423,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2690,"num_objects_misplaced":317,"num_objects_unfound":0,"num_objects_dirty":1141,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":227993,"num_write_kb":7329378,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11329,"num_bytes_recovered":689708654,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":126981,"ondisk_log_size":126981,"up":297,"acting":296,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":302,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5998068,"kb_used_data":1129508,"kb_used_omap":404,"kb_used_meta":4868075,"kb_avail":743963148,"statfs":{"total":767960285184,"available":761818263552,"internally_reserved":0,"allocated":1156616192,"data_stored":1114769879,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":414022,"internal_metadata":4984909498},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-506593304,"num_objects":-7731,"num_object_clones":0,"num_object_copies":-23193,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":183,"num_objects_misplaced":-1028,"num_objects_unfound":0,"num_objects_dirty":-7731,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7731,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":332,"num_bytes_recovered":21757952,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003293"},"pg_stats":[{"pgid":"56.f","version":"1515'14182","reported_seq":14237,"reported_epoch":1515,"state":"active+clean","last_fresh":"2024-09-14T04:56:22.736658+0000","last_change":"2024-09-14T04:52:16.661278+0000","last_active":"2024-09-14T04:56:22.736658+0000","last_peered":"2024-09-14T04:56:22.736658+0000","last_clean":"2024-09-14T04:56:22.736658+0000","last_became_active":"2024-09-14T04:52:16.660906+0000","last_became_peered":"2024-09-14T04:52:16.660906+0000","last_unstale":"2024-09-14T04:56:22.736658+0000","last_undegraded":"2024-09-14T04:56:22.736658+0000","last_fullsized":"2024-09-14T04:56:22.736658+0000","mapping_epoch":1470,"log_start":"1486'7200","ondisk_log_start":"1486'7200","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":6982,"log_dups_size":0,"ondisk_log_size":6982,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:55:16.908898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14182,"num_write_kb":453824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"56.e","version":"1515'14476","reported_seq":14531,"reported_epoch":1515,"state":"active+clean","last_fresh":"2024-09-14T04:56:22.824952+0000","last_change":"2024-09-14T04:52:16.723745+0000","last_active":"2024-09-14T04:56:22.824952+0000","last_peered":"2024-09-14T04:56:22.824952+0000","last_clean":"2024-09-14T04:56:22.824952+0000","last_became_active":"2024-09-14T04:52:16.723416+0000","last_became_peered":"2024-09-14T04:52:16.723416+0000","last_unstale":"2024-09-14T04:56:22.824952+0000","last_undegraded":"2024-09-14T04:56:22.824952+0000","last_fullsized":"2024-09-14T04:56:22.824952+0000","mapping_epoch":1470,"log_start":"1485'7147","ondisk_log_start":"1485'7147","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":7329,"log_dups_size":0,"ondisk_log_size":7329,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:20:17.253535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14476,"num_write_kb":463232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"56.d","version":"1515'14404","reported_seq":14507,"reported_epoch":1515,"state":"active+clean","last_fresh":"2024-09-14T04:56:22.782846+0000","last_change":"2024-09-14T04:52:16.871824+0000","last_active":"2024-09-14T04:56:22.782846+0000","last_peered":"2024-09-14T04:56:22.782846+0000","last_clean":"2024-09-14T04:56:22.782846+0000","last_became_active":"2024-09-14T04:52:16.871446+0000","last_became_peered":"2024-09-14T04:52:16.871446+0000","last_unstale":"2024-09-14T04:56:22.782846+0000","last_undegraded":"2024-09-14T04:56:22.782846+0000","last_fullsized":"2024-09-14T04:56:22.782846+0000","mapping_epoch":1470,"log_start":"1482'5800","ondisk_log_start":"1482'5800","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8604,"log_dups_size":0,"ondisk_log_size":8604,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:44:00.576543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":14404,"num_write_kb":460865,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"56.c","version":"1514'14263","reported_seq":14317,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:19.596457+0000","last_change":"2024-09-14T04:52:16.723896+0000","last_active":"2024-09-14T04:56:19.596457+0000","last_peered":"2024-09-14T04:56:19.596457+0000","last_clean":"2024-09-14T04:56:19.596457+0000","last_became_active":"2024-09-14T04:52:16.723597+0000","last_became_peered":"2024-09-14T04:52:16.723597+0000","last_unstale":"2024-09-14T04:56:19.596457+0000","last_undegraded":"2024-09-14T04:56:19.596457+0000","last_fullsized":"2024-09-14T04:56:19.596457+0000","mapping_epoch":1470,"log_start":"1482'6108","ondisk_log_start":"1482'6108","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8155,"log_dups_size":0,"ondisk_log_size":8155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:10:31.058435+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10551296,"num_objects":161,"num_object_clones":0,"num_object_copies":483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14263,"num_write_kb":461568,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.b","version":"1515'14368","reported_seq":14423,"reported_epoch":1515,"state":"active+clean","last_fresh":"2024-09-14T04:56:22.808744+0000","last_change":"2024-09-14T04:52:16.660625+0000","last_active":"2024-09-14T04:56:22.808744+0000","last_peered":"2024-09-14T04:56:22.808744+0000","last_clean":"2024-09-14T04:56:22.808744+0000","last_became_active":"2024-09-14T04:52:16.660277+0000","last_became_peered":"2024-09-14T04:52:16.660277+0000","last_unstale":"2024-09-14T04:56:22.808744+0000","last_undegraded":"2024-09-14T04:56:22.808744+0000","last_fullsized":"2024-09-14T04:56:22.808744+0000","mapping_epoch":1470,"log_start":"1486'7300","ondisk_log_start":"1486'7300","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":7068,"log_dups_size":0,"ondisk_log_size":7068,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:43:12.659725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14368,"num_write_kb":459776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"56.a","version":"1515'14278","reported_seq":14333,"reported_epoch":1515,"state":"active+clean","last_fresh":"2024-09-14T04:56:22.829320+0000","last_change":"2024-09-14T04:52:16.915139+0000","last_active":"2024-09-14T04:56:22.829320+0000","last_peered":"2024-09-14T04:56:22.829320+0000","last_clean":"2024-09-14T04:56:22.829320+0000","last_became_active":"2024-09-14T04:52:16.914776+0000","last_became_peered":"2024-09-14T04:52:16.914776+0000","last_unstale":"2024-09-14T04:56:22.829320+0000","last_undegraded":"2024-09-14T04:56:22.829320+0000","last_fullsized":"2024-09-14T04:56:22.829320+0000","mapping_epoch":1470,"log_start":"1486'7300","ondisk_log_start":"1486'7300","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":6978,"log_dups_size":0,"ondisk_log_size":6978,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:08:58.079681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14278,"num_write_kb":456896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"56.9","version":"1514'14167","reported_seq":14221,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:19.628105+0000","last_change":"2024-09-14T04:52:16.723789+0000","last_active":"2024-09-14T04:56:19.628105+0000","last_peered":"2024-09-14T04:56:19.628105+0000","last_clean":"2024-09-14T04:56:19.628105+0000","last_became_active":"2024-09-14T04:52:16.723394+0000","last_became_peered":"2024-09-14T04:52:16.723394+0000","last_unstale":"2024-09-14T04:56:19.628105+0000","last_undegraded":"2024-09-14T04:56:19.628105+0000","last_fullsized":"2024-09-14T04:56:19.628105+0000","mapping_epoch":1470,"log_start":"1482'5970","ondisk_log_start":"1482'5970","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8197,"log_dups_size":0,"ondisk_log_size":8197,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:45:07.892366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10682368,"num_objects":163,"num_object_clones":0,"num_object_copies":489,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":163,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14167,"num_write_kb":458560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.8","version":"1515'14012","reported_seq":14114,"reported_epoch":1515,"state":"active+clean","last_fresh":"2024-09-14T04:56:22.790081+0000","last_change":"2024-09-14T04:52:16.915103+0000","last_active":"2024-09-14T04:56:22.790081+0000","last_peered":"2024-09-14T04:56:22.790081+0000","last_clean":"2024-09-14T04:56:22.790081+0000","last_became_active":"2024-09-14T04:52:16.914657+0000","last_became_peered":"2024-09-14T04:52:16.914657+0000","last_unstale":"2024-09-14T04:56:22.790081+0000","last_undegraded":"2024-09-14T04:56:22.790081+0000","last_fullsized":"2024-09-14T04:56:22.790081+0000","mapping_epoch":1470,"log_start":"1481'5400","ondisk_log_start":"1481'5400","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8612,"log_dups_size":0,"ondisk_log_size":8612,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:08:43.898740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14012,"num_write_kb":448384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"56.7","version":"1515'14300","reported_seq":14402,"reported_epoch":1515,"state":"active+clean","last_fresh":"2024-09-14T04:56:22.776227+0000","last_change":"2024-09-14T04:52:16.723670+0000","last_active":"2024-09-14T04:56:22.776227+0000","last_peered":"2024-09-14T04:56:22.776227+0000","last_clean":"2024-09-14T04:56:22.776227+0000","last_became_active":"2024-09-14T04:52:16.723292+0000","last_became_peered":"2024-09-14T04:52:16.723292+0000","last_unstale":"2024-09-14T04:56:22.776227+0000","last_undegraded":"2024-09-14T04:56:22.776227+0000","last_fullsized":"2024-09-14T04:56:22.776227+0000","mapping_epoch":1470,"log_start":"1482'5700","ondisk_log_start":"1482'5700","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8600,"log_dups_size":0,"ondisk_log_size":8600,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:55:51.122910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14300,"num_write_kb":457600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"56.6","version":"1514'13865","reported_seq":22347,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:19.633365+0000","last_change":"2024-09-14T04:55:59.766727+0000","last_active":"2024-09-14T04:56:19.633365+0000","last_peered":"2024-09-14T04:56:19.633365+0000","last_clean":"2024-09-14T04:56:19.633365+0000","last_became_active":"2024-09-14T04:55:59.732589+0000","last_became_peered":"2024-09-14T04:55:59.732589+0000","last_unstale":"2024-09-14T04:56:19.633365+0000","last_undegraded":"2024-09-14T04:56:19.633365+0000","last_fullsized":"2024-09-14T04:56:19.633365+0000","mapping_epoch":1512,"log_start":"1482'5657","ondisk_log_start":"1482'5657","created":1470,"last_epoch_clean":1513,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8208,"log_dups_size":0,"ondisk_log_size":8208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:12:31.211645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8847360,"num_objects":135,"num_object_clones":0,"num_object_copies":405,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":135,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13865,"num_write_kb":448000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4201,"num_bytes_recovered":224722944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.5","version":"1515'14360","reported_seq":14415,"reported_epoch":1515,"state":"active+clean","last_fresh":"2024-09-14T04:56:22.827536+0000","last_change":"2024-09-14T04:52:16.871791+0000","last_active":"2024-09-14T04:56:22.827536+0000","last_peered":"2024-09-14T04:56:22.827536+0000","last_clean":"2024-09-14T04:56:22.827536+0000","last_became_active":"2024-09-14T04:52:16.871407+0000","last_became_peered":"2024-09-14T04:52:16.871407+0000","last_unstale":"2024-09-14T04:56:22.827536+0000","last_undegraded":"2024-09-14T04:56:22.827536+0000","last_fullsized":"2024-09-14T04:56:22.827536+0000","mapping_epoch":1470,"log_start":"1485'6967","ondisk_log_start":"1485'6967","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":7393,"log_dups_size":0,"ondisk_log_size":7393,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:47:15.419674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14360,"num_write_kb":459520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"56.4","version":"1515'14492","reported_seq":14547,"reported_epoch":1515,"state":"active+clean","last_fresh":"2024-09-14T04:56:22.766359+0000","last_change":"2024-09-14T04:52:16.724077+0000","last_active":"2024-09-14T04:56:22.766359+0000","last_peered":"2024-09-14T04:56:22.766359+0000","last_clean":"2024-09-14T04:56:22.766359+0000","last_became_active":"2024-09-14T04:52:16.723597+0000","last_became_peered":"2024-09-14T04:52:16.723597+0000","last_unstale":"2024-09-14T04:56:22.766359+0000","last_undegraded":"2024-09-14T04:56:22.766359+0000","last_fullsized":"2024-09-14T04:56:22.766359+0000","mapping_epoch":1470,"log_start":"1486'7500","ondisk_log_start":"1486'7500","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":6992,"log_dups_size":0,"ondisk_log_size":6992,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:23:49.557180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14492,"num_write_kb":463744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"56.3","version":"1514'14032","reported_seq":21313,"reported_epoch":1514,"state":"active+remapped+backfill_wait","last_fresh":"2024-09-14T04:56:19.482178+0000","last_change":"2024-09-14T04:55:57.904831+0000","last_active":"2024-09-14T04:56:19.482178+0000","last_peered":"2024-09-14T04:56:19.482178+0000","last_clean":"2024-09-14T04:54:51.200599+0000","last_became_active":"2024-09-14T04:54:53.681745+0000","last_became_peered":"2024-09-14T04:54:53.681745+0000","last_unstale":"2024-09-14T04:56:19.482178+0000","last_undegraded":"2024-09-14T04:56:19.482178+0000","last_fullsized":"2024-09-14T04:56:19.482178+0000","mapping_epoch":1504,"log_start":"1483'6317","ondisk_log_start":"1483'6317","created":1470,"last_epoch_clean":1496,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":7715,"log_dups_size":0,"ondisk_log_size":7715,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"no scrub is scheduled","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10747904,"num_objects":164,"num_object_clones":0,"num_object_copies":492,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":164,"num_objects_unfound":0,"num_objects_dirty":164,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14032,"num_write_kb":454272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3597,"num_bytes_recovered":235732992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[1,6,5],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":164}],"blocked_by":[],"up_primary":7,"acting_primary":1,"purged_snaps":[]},{"pgid":"56.2","version":"1514'14368","reported_seq":21395,"reported_epoch":1514,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2024-09-14T04:56:19.644558+0000","last_change":"2024-09-14T04:55:57.907512+0000","last_active":"2024-09-14T04:56:19.644558+0000","last_peered":"2024-09-14T04:56:19.644558+0000","last_clean":"2024-09-14T04:52:58.403685+0000","last_became_active":"2024-09-14T04:54:19.593740+0000","last_became_peered":"2024-09-14T04:54:19.593740+0000","last_unstale":"2024-09-14T04:56:19.644558+0000","last_undegraded":"2024-09-14T04:54:19.466599+0000","last_fullsized":"2024-09-14T04:54:19.444256+0000","mapping_epoch":1489,"log_start":"1482'6237","ondisk_log_start":"1482'6237","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8131,"log_dups_size":0,"ondisk_log_size":8131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:45:11.467642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10485760,"num_objects":160,"num_object_clones":0,"num_object_copies":480,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2690,"num_objects_misplaced":153,"num_objects_unfound":0,"num_objects_dirty":160,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14368,"num_write_kb":464896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3478,"num_bytes_recovered":227934208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"0,5,7","objects":153},{"shards":"5,7","objects":-2683}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.1","version":"1514'14236","reported_seq":14290,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:19.595249+0000","last_change":"2024-09-14T04:52:16.872703+0000","last_active":"2024-09-14T04:56:19.595249+0000","last_peered":"2024-09-14T04:56:19.595249+0000","last_clean":"2024-09-14T04:56:19.595249+0000","last_became_active":"2024-09-14T04:52:16.871779+0000","last_became_peered":"2024-09-14T04:52:16.871779+0000","last_unstale":"2024-09-14T04:56:19.595249+0000","last_undegraded":"2024-09-14T04:56:19.595249+0000","last_fullsized":"2024-09-14T04:56:19.595249+0000","mapping_epoch":1470,"log_start":"1482'6065","ondisk_log_start":"1482'6065","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8171,"log_dups_size":0,"ondisk_log_size":8171,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:23:30.260775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":11010048,"num_objects":168,"num_object_clones":0,"num_object_copies":504,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":168,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14236,"num_write_kb":460928,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"56.0","version":"1514'14126","reported_seq":14179,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:19.651168+0000","last_change":"2024-09-14T04:52:16.915325+0000","last_active":"2024-09-14T04:56:19.651168+0000","last_peered":"2024-09-14T04:56:19.651168+0000","last_clean":"2024-09-14T04:56:19.651168+0000","last_became_active":"2024-09-14T04:52:16.914980+0000","last_became_peered":"2024-09-14T04:52:16.914980+0000","last_unstale":"2024-09-14T04:56:19.651168+0000","last_undegraded":"2024-09-14T04:56:19.651168+0000","last_fullsized":"2024-09-14T04:56:19.651168+0000","mapping_epoch":1470,"log_start":"1482'5978","ondisk_log_start":"1482'5978","created":1470,"last_epoch_clean":1471,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:52:15.610693+0000","last_clean_scrub_stamp":"2024-09-14T04:52:15.610693+0000","objects_scrubbed":0,"log_size":8148,"log_dups_size":0,"ondisk_log_size":8148,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:55:08.752960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10747904,"num_objects":164,"num_object_clones":0,"num_object_copies":492,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":164,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14126,"num_write_kb":457280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":2323,"reported_epoch":1515,"state":"active+clean","last_fresh":"2024-09-14T04:56:22.073253+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T04:56:22.073253+0000","last_peered":"2024-09-14T04:56:22.073253+0000","last_clean":"2024-09-14T04:56:22.073253+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T04:56:22.073253+0000","last_undegraded":"2024-09-14T04:56:22.073253+0000","last_fullsized":"2024-09-14T04:56:22.073253+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":127,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.976399+0000","last_change":"2024-09-14T04:44:57.325109+0000","last_active":"2024-09-14T04:56:06.976399+0000","last_peered":"2024-09-14T04:56:06.976399+0000","last_clean":"2024-09-14T04:56:06.976399+0000","last_became_active":"2024-09-14T04:44:57.324791+0000","last_became_peered":"2024-09-14T04:44:57.324791+0000","last_unstale":"2024-09-14T04:56:06.976399+0000","last_undegraded":"2024-09-14T04:56:06.976399+0000","last_fullsized":"2024-09-14T04:56:06.976399+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:03.593275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":97,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.983778+0000","last_change":"2024-09-14T04:49:29.628435+0000","last_active":"2024-09-14T04:56:06.983778+0000","last_peered":"2024-09-14T04:56:06.983778+0000","last_clean":"2024-09-14T04:56:06.983778+0000","last_became_active":"2024-09-14T04:49:29.628081+0000","last_became_peered":"2024-09-14T04:49:29.628081+0000","last_unstale":"2024-09-14T04:56:06.983778+0000","last_undegraded":"2024-09-14T04:56:06.983778+0000","last_fullsized":"2024-09-14T04:56:06.983778+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:01.520334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":942,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.987695+0000","last_change":"2024-09-14T04:43:13.355281+0000","last_active":"2024-09-14T04:56:06.987695+0000","last_peered":"2024-09-14T04:56:06.987695+0000","last_clean":"2024-09-14T04:56:06.987695+0000","last_became_active":"2024-09-14T04:43:13.354967+0000","last_became_peered":"2024-09-14T04:43:13.354967+0000","last_unstale":"2024-09-14T04:56:06.987695+0000","last_undegraded":"2024-09-14T04:56:06.987695+0000","last_fullsized":"2024-09-14T04:56:06.987695+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:23:19.332075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1505,"reported_epoch":1513,"state":"active+clean","last_fresh":"2024-09-14T04:55:59.727509+0000","last_change":"2024-09-14T04:44:57.342918+0000","last_active":"2024-09-14T04:55:59.727509+0000","last_peered":"2024-09-14T04:55:59.727509+0000","last_clean":"2024-09-14T04:55:59.727509+0000","last_became_active":"2024-09-14T04:44:57.329550+0000","last_became_peered":"2024-09-14T04:44:57.329550+0000","last_unstale":"2024-09-14T04:55:59.727509+0000","last_undegraded":"2024-09-14T04:55:59.727509+0000","last_fullsized":"2024-09-14T04:55:59.727509+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:52:52.435893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":127,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.978078+0000","last_change":"2024-09-14T04:44:57.325537+0000","last_active":"2024-09-14T04:56:06.978078+0000","last_peered":"2024-09-14T04:56:06.978078+0000","last_clean":"2024-09-14T04:56:06.978078+0000","last_became_active":"2024-09-14T04:44:57.325057+0000","last_became_peered":"2024-09-14T04:44:57.325057+0000","last_unstale":"2024-09-14T04:56:06.978078+0000","last_undegraded":"2024-09-14T04:56:06.978078+0000","last_fullsized":"2024-09-14T04:56:06.978078+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:43:23.166708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1520,"reported_epoch":1513,"state":"active+clean","last_fresh":"2024-09-14T04:55:59.727836+0000","last_change":"2024-09-14T04:44:57.337033+0000","last_active":"2024-09-14T04:55:59.727836+0000","last_peered":"2024-09-14T04:55:59.727836+0000","last_clean":"2024-09-14T04:55:59.727836+0000","last_became_active":"2024-09-14T04:44:57.328437+0000","last_became_peered":"2024-09-14T04:44:57.328437+0000","last_unstale":"2024-09-14T04:55:59.727836+0000","last_undegraded":"2024-09-14T04:55:59.727836+0000","last_fullsized":"2024-09-14T04:55:59.727836+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.109086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":127,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.977271+0000","last_change":"2024-09-14T04:44:57.332243+0000","last_active":"2024-09-14T04:56:06.977271+0000","last_peered":"2024-09-14T04:56:06.977271+0000","last_clean":"2024-09-14T04:56:06.977271+0000","last_became_active":"2024-09-14T04:44:57.330413+0000","last_became_peered":"2024-09-14T04:44:57.330413+0000","last_unstale":"2024-09-14T04:56:06.977271+0000","last_undegraded":"2024-09-14T04:56:06.977271+0000","last_fullsized":"2024-09-14T04:56:06.977271+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:25:01.338549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":179,"reported_epoch":1515,"state":"active+clean","last_fresh":"2024-09-14T04:56:22.072420+0000","last_change":"2024-09-14T04:48:29.862929+0000","last_active":"2024-09-14T04:56:22.072420+0000","last_peered":"2024-09-14T04:56:22.072420+0000","last_clean":"2024-09-14T04:56:22.072420+0000","last_became_active":"2024-09-14T04:48:29.862581+0000","last_became_peered":"2024-09-14T04:48:29.862581+0000","last_unstale":"2024-09-14T04:56:22.072420+0000","last_undegraded":"2024-09-14T04:56:22.072420+0000","last_fullsized":"2024-09-14T04:56:22.072420+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:16:43.060088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2316,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.986323+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T04:56:06.986323+0000","last_peered":"2024-09-14T04:56:06.986323+0000","last_clean":"2024-09-14T04:56:06.986323+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T04:56:06.986323+0000","last_undegraded":"2024-09-14T04:56:06.986323+0000","last_fullsized":"2024-09-14T04:56:06.986323+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":127,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.978043+0000","last_change":"2024-09-14T04:44:57.325392+0000","last_active":"2024-09-14T04:56:06.978043+0000","last_peered":"2024-09-14T04:56:06.978043+0000","last_clean":"2024-09-14T04:56:06.978043+0000","last_became_active":"2024-09-14T04:44:57.324624+0000","last_became_peered":"2024-09-14T04:44:57.324624+0000","last_unstale":"2024-09-14T04:56:06.978043+0000","last_undegraded":"2024-09-14T04:56:06.978043+0000","last_fullsized":"2024-09-14T04:56:06.978043+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:33:18.810199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2285,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.984800+0000","last_change":"2024-09-14T04:44:57.329113+0000","last_active":"2024-09-14T04:56:06.984800+0000","last_peered":"2024-09-14T04:56:06.984800+0000","last_clean":"2024-09-14T04:56:06.984800+0000","last_became_active":"2024-09-14T04:44:57.328727+0000","last_became_peered":"2024-09-14T04:44:57.328727+0000","last_unstale":"2024-09-14T04:56:06.984800+0000","last_undegraded":"2024-09-14T04:56:06.984800+0000","last_fullsized":"2024-09-14T04:56:06.984800+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:34.939874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":2302,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.982237+0000","last_change":"2024-09-14T04:44:57.329277+0000","last_active":"2024-09-14T04:56:06.982237+0000","last_peered":"2024-09-14T04:56:06.982237+0000","last_clean":"2024-09-14T04:56:06.982237+0000","last_became_active":"2024-09-14T04:44:57.328804+0000","last_became_peered":"2024-09-14T04:44:57.328804+0000","last_unstale":"2024-09-14T04:56:06.982237+0000","last_undegraded":"2024-09-14T04:56:06.982237+0000","last_fullsized":"2024-09-14T04:56:06.982237+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:42:46.055240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2339,"reported_epoch":1515,"state":"active+clean","last_fresh":"2024-09-14T04:56:22.072952+0000","last_change":"2024-09-14T04:54:52.250695+0000","last_active":"2024-09-14T04:56:22.072952+0000","last_peered":"2024-09-14T04:56:22.072952+0000","last_clean":"2024-09-14T04:56:22.072952+0000","last_became_active":"2024-09-14T04:54:52.250301+0000","last_became_peered":"2024-09-14T04:54:52.250301+0000","last_unstale":"2024-09-14T04:56:22.072952+0000","last_undegraded":"2024-09-14T04:56:22.072952+0000","last_fullsized":"2024-09-14T04:56:22.072952+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:42.060905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2300,"reported_epoch":1515,"state":"active+clean","last_fresh":"2024-09-14T04:56:22.073667+0000","last_change":"2024-09-14T04:43:13.357092+0000","last_active":"2024-09-14T04:56:22.073667+0000","last_peered":"2024-09-14T04:56:22.073667+0000","last_clean":"2024-09-14T04:56:22.073667+0000","last_became_active":"2024-09-14T04:43:13.356694+0000","last_became_peered":"2024-09-14T04:43:13.356694+0000","last_unstale":"2024-09-14T04:56:22.073667+0000","last_undegraded":"2024-09-14T04:56:22.073667+0000","last_fullsized":"2024-09-14T04:56:22.073667+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:11.167574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1499,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.983316+0000","last_change":"2024-09-14T04:07:21.720797+0000","last_active":"2024-09-14T04:56:06.983316+0000","last_peered":"2024-09-14T04:56:06.983316+0000","last_clean":"2024-09-14T04:56:06.983316+0000","last_became_active":"2024-09-14T04:07:21.719331+0000","last_became_peered":"2024-09-14T04:07:21.719331+0000","last_unstale":"2024-09-14T04:56:06.983316+0000","last_undegraded":"2024-09-14T04:56:06.983316+0000","last_fullsized":"2024-09-14T04:56:06.983316+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:35.404023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":145,"reported_epoch":1513,"state":"active+clean","last_fresh":"2024-09-14T04:55:59.727440+0000","last_change":"2024-09-14T04:43:11.641514+0000","last_active":"2024-09-14T04:55:59.727440+0000","last_peered":"2024-09-14T04:55:59.727440+0000","last_clean":"2024-09-14T04:55:59.727440+0000","last_became_active":"2024-09-14T04:43:11.641151+0000","last_became_peered":"2024-09-14T04:43:11.641151+0000","last_unstale":"2024-09-14T04:55:59.727440+0000","last_undegraded":"2024-09-14T04:55:59.727440+0000","last_fullsized":"2024-09-14T04:55:59.727440+0000","mapping_epoch":1384,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:06:45.045430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2311,"reported_epoch":1515,"state":"active+clean","last_fresh":"2024-09-14T04:56:22.073482+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T04:56:22.073482+0000","last_peered":"2024-09-14T04:56:22.073482+0000","last_clean":"2024-09-14T04:56:22.073482+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T04:56:22.073482+0000","last_undegraded":"2024-09-14T04:56:22.073482+0000","last_fullsized":"2024-09-14T04:56:22.073482+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":726,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.986164+0000","last_change":"2024-09-14T04:49:29.618477+0000","last_active":"2024-09-14T04:56:06.986164+0000","last_peered":"2024-09-14T04:56:06.986164+0000","last_clean":"2024-09-14T04:56:06.986164+0000","last_became_active":"2024-09-14T04:49:29.618210+0000","last_became_peered":"2024-09-14T04:49:29.618210+0000","last_unstale":"2024-09-14T04:56:06.986164+0000","last_undegraded":"2024-09-14T04:56:06.986164+0000","last_fullsized":"2024-09-14T04:56:06.986164+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:13:23.895512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1504,"reported_epoch":1513,"state":"active+clean","last_fresh":"2024-09-14T04:55:59.727746+0000","last_change":"2024-09-14T04:42:49.938505+0000","last_active":"2024-09-14T04:55:59.727746+0000","last_peered":"2024-09-14T04:55:59.727746+0000","last_clean":"2024-09-14T04:55:59.727746+0000","last_became_active":"2024-09-14T04:42:49.938180+0000","last_became_peered":"2024-09-14T04:42:49.938180+0000","last_unstale":"2024-09-14T04:55:59.727746+0000","last_undegraded":"2024-09-14T04:55:59.727746+0000","last_fullsized":"2024-09-14T04:55:59.727746+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:05.929479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1524,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.986500+0000","last_change":"2024-09-14T04:44:57.346582+0000","last_active":"2024-09-14T04:56:06.986500+0000","last_peered":"2024-09-14T04:56:06.986500+0000","last_clean":"2024-09-14T04:56:06.986500+0000","last_became_active":"2024-09-14T04:44:57.330446+0000","last_became_peered":"2024-09-14T04:44:57.330446+0000","last_unstale":"2024-09-14T04:56:06.986500+0000","last_undegraded":"2024-09-14T04:56:06.986500+0000","last_fullsized":"2024-09-14T04:56:06.986500+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:29.238585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":251,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.986424+0000","last_change":"2024-09-14T04:44:14.480667+0000","last_active":"2024-09-14T04:56:06.986424+0000","last_peered":"2024-09-14T04:56:06.986424+0000","last_clean":"2024-09-14T04:56:06.986424+0000","last_became_active":"2024-09-14T04:44:14.480365+0000","last_became_peered":"2024-09-14T04:44:14.480365+0000","last_unstale":"2024-09-14T04:56:06.986424+0000","last_undegraded":"2024-09-14T04:56:06.986424+0000","last_fullsized":"2024-09-14T04:56:06.986424+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:00.611771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":137,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.978309+0000","last_change":"2024-09-14T04:44:57.341426+0000","last_active":"2024-09-14T04:56:06.978309+0000","last_peered":"2024-09-14T04:56:06.978309+0000","last_clean":"2024-09-14T04:56:06.978309+0000","last_became_active":"2024-09-14T04:44:57.328763+0000","last_became_peered":"2024-09-14T04:44:57.328763+0000","last_unstale":"2024-09-14T04:56:06.978309+0000","last_undegraded":"2024-09-14T04:56:06.978309+0000","last_fullsized":"2024-09-14T04:56:06.978309+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:34:40.066761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":127,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.977414+0000","last_change":"2024-09-14T04:44:57.331337+0000","last_active":"2024-09-14T04:56:06.977414+0000","last_peered":"2024-09-14T04:56:06.977414+0000","last_clean":"2024-09-14T04:56:06.977414+0000","last_became_active":"2024-09-14T04:44:57.329336+0000","last_became_peered":"2024-09-14T04:44:57.329336+0000","last_unstale":"2024-09-14T04:56:06.977414+0000","last_undegraded":"2024-09-14T04:56:06.977414+0000","last_fullsized":"2024-09-14T04:56:06.977414+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:52:35.980869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"349'149","reported_seq":715,"reported_epoch":1513,"state":"active+clean","last_fresh":"2024-09-14T04:55:59.727420+0000","last_change":"2024-09-14T04:49:30.085274+0000","last_active":"2024-09-14T04:55:59.727420+0000","last_peered":"2024-09-14T04:55:59.727420+0000","last_clean":"2024-09-14T04:55:59.727420+0000","last_became_active":"2024-09-14T04:49:29.623295+0000","last_became_peered":"2024-09-14T04:49:29.623295+0000","last_unstale":"2024-09-14T04:55:59.727420+0000","last_undegraded":"2024-09-14T04:55:59.727420+0000","last_fullsized":"2024-09-14T04:55:59.727420+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:49:30.085089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:49:30.085089+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:10.012963+0000","scrub_duration":0.0071928790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":128,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.976696+0000","last_change":"2024-09-14T04:44:57.329870+0000","last_active":"2024-09-14T04:56:06.976696+0000","last_peered":"2024-09-14T04:56:06.976696+0000","last_clean":"2024-09-14T04:56:06.976696+0000","last_became_active":"2024-09-14T04:44:57.329353+0000","last_became_peered":"2024-09-14T04:44:57.329353+0000","last_unstale":"2024-09-14T04:56:06.976696+0000","last_undegraded":"2024-09-14T04:56:06.976696+0000","last_fullsized":"2024-09-14T04:56:06.976696+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:46.432181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1520,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.983937+0000","last_change":"2024-09-14T04:54:52.242207+0000","last_active":"2024-09-14T04:56:06.983937+0000","last_peered":"2024-09-14T04:56:06.983937+0000","last_clean":"2024-09-14T04:56:06.983937+0000","last_became_active":"2024-09-14T04:54:52.241840+0000","last_became_peered":"2024-09-14T04:54:52.241840+0000","last_unstale":"2024-09-14T04:56:06.983937+0000","last_undegraded":"2024-09-14T04:56:06.983937+0000","last_fullsized":"2024-09-14T04:56:06.983937+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:37.899666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":2350,"reported_epoch":1515,"state":"active+clean","last_fresh":"2024-09-14T04:56:22.072859+0000","last_change":"2024-09-14T04:44:57.344614+0000","last_active":"2024-09-14T04:56:22.072859+0000","last_peered":"2024-09-14T04:56:22.072859+0000","last_clean":"2024-09-14T04:56:22.072859+0000","last_became_active":"2024-09-14T04:44:57.332256+0000","last_became_peered":"2024-09-14T04:44:57.332256+0000","last_unstale":"2024-09-14T04:56:22.072859+0000","last_undegraded":"2024-09-14T04:56:22.072859+0000","last_fullsized":"2024-09-14T04:56:22.072859+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:23.438417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"349'149","reported_seq":886,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.977642+0000","last_change":"2024-09-14T04:45:00.331854+0000","last_active":"2024-09-14T04:56:06.977642+0000","last_peered":"2024-09-14T04:56:06.977642+0000","last_clean":"2024-09-14T04:56:06.977642+0000","last_became_active":"2024-09-14T04:45:00.331460+0000","last_became_peered":"2024-09-14T04:45:00.331460+0000","last_unstale":"2024-09-14T04:56:06.977642+0000","last_undegraded":"2024-09-14T04:56:06.977642+0000","last_fullsized":"2024-09-14T04:56:06.977642+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.567535+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.567535+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:45.095729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":717,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.985964+0000","last_change":"2024-09-14T04:49:29.616831+0000","last_active":"2024-09-14T04:56:06.985964+0000","last_peered":"2024-09-14T04:56:06.985964+0000","last_clean":"2024-09-14T04:56:06.985964+0000","last_became_active":"2024-09-14T04:49:29.616546+0000","last_became_peered":"2024-09-14T04:49:29.616546+0000","last_unstale":"2024-09-14T04:56:06.985964+0000","last_undegraded":"2024-09-14T04:56:06.985964+0000","last_fullsized":"2024-09-14T04:56:06.985964+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:27.882929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":303,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.981612+0000","last_change":"2024-09-14T04:44:57.330746+0000","last_active":"2024-09-14T04:56:06.981612+0000","last_peered":"2024-09-14T04:56:06.981612+0000","last_clean":"2024-09-14T04:56:06.981612+0000","last_became_active":"2024-09-14T04:44:57.330327+0000","last_became_peered":"2024-09-14T04:44:57.330327+0000","last_unstale":"2024-09-14T04:56:06.981612+0000","last_undegraded":"2024-09-14T04:56:06.981612+0000","last_fullsized":"2024-09-14T04:56:06.981612+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:53.122771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":168,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:13.833816+0000","last_change":"2024-09-14T04:43:13.355296+0000","last_active":"2024-09-14T04:56:13.833816+0000","last_peered":"2024-09-14T04:56:13.833816+0000","last_clean":"2024-09-14T04:56:13.833816+0000","last_became_active":"2024-09-14T04:43:13.354950+0000","last_became_peered":"2024-09-14T04:43:13.354950+0000","last_unstale":"2024-09-14T04:56:13.833816+0000","last_undegraded":"2024-09-14T04:56:13.833816+0000","last_fullsized":"2024-09-14T04:56:13.833816+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:20:08.968818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"349'149","reported_seq":824,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.983648+0000","last_change":"2024-09-14T04:41:39.471393+0000","last_active":"2024-09-14T04:56:06.983648+0000","last_peered":"2024-09-14T04:56:06.983648+0000","last_clean":"2024-09-14T04:56:06.983648+0000","last_became_active":"2024-09-14T04:41:36.931768+0000","last_became_peered":"2024-09-14T04:41:36.931768+0000","last_unstale":"2024-09-14T04:56:06.983648+0000","last_undegraded":"2024-09-14T04:56:06.983648+0000","last_fullsized":"2024-09-14T04:56:06.983648+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.471248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.471248+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:56.210375+0000","scrub_duration":0.021758344999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2306,"reported_epoch":1515,"state":"active+clean","last_fresh":"2024-09-14T04:56:22.073302+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T04:56:22.073302+0000","last_peered":"2024-09-14T04:56:22.073302+0000","last_clean":"2024-09-14T04:56:22.073302+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T04:56:22.073302+0000","last_undegraded":"2024-09-14T04:56:22.073302+0000","last_fullsized":"2024-09-14T04:56:22.073302+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1534,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.986593+0000","last_change":"2024-09-14T04:44:57.331436+0000","last_active":"2024-09-14T04:56:06.986593+0000","last_peered":"2024-09-14T04:56:06.986593+0000","last_clean":"2024-09-14T04:56:06.986593+0000","last_became_active":"2024-09-14T04:44:57.330957+0000","last_became_peered":"2024-09-14T04:44:57.330957+0000","last_unstale":"2024-09-14T04:56:06.986593+0000","last_undegraded":"2024-09-14T04:56:06.986593+0000","last_fullsized":"2024-09-14T04:56:06.986593+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:09.516688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":853,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.986230+0000","last_change":"2024-09-14T04:49:49.460401+0000","last_active":"2024-09-14T04:56:06.986230+0000","last_peered":"2024-09-14T04:56:06.986230+0000","last_clean":"2024-09-14T04:56:06.986230+0000","last_became_active":"2024-09-14T04:49:49.458530+0000","last_became_peered":"2024-09-14T04:49:49.458530+0000","last_unstale":"2024-09-14T04:56:06.986230+0000","last_undegraded":"2024-09-14T04:56:06.986230+0000","last_fullsized":"2024-09-14T04:56:06.986230+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:11:29.143239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"349'149","reported_seq":858,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.983234+0000","last_change":"2024-09-14T04:54:22.147441+0000","last_active":"2024-09-14T04:56:06.983234+0000","last_peered":"2024-09-14T04:56:06.983234+0000","last_clean":"2024-09-14T04:56:06.983234+0000","last_became_active":"2024-09-14T04:54:22.147115+0000","last_became_peered":"2024-09-14T04:54:22.147115+0000","last_unstale":"2024-09-14T04:56:06.983234+0000","last_undegraded":"2024-09-14T04:56:06.983234+0000","last_fullsized":"2024-09-14T04:56:06.983234+0000","mapping_epoch":1492,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1493,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:38.479887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:38.479887+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:41:14.001781+0000","scrub_duration":0.0062728319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":258,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.986324+0000","last_change":"2024-09-14T04:44:57.329983+0000","last_active":"2024-09-14T04:56:06.986324+0000","last_peered":"2024-09-14T04:56:06.986324+0000","last_clean":"2024-09-14T04:56:06.986324+0000","last_became_active":"2024-09-14T04:44:57.329597+0000","last_became_peered":"2024-09-14T04:44:57.329597+0000","last_unstale":"2024-09-14T04:56:06.986324+0000","last_undegraded":"2024-09-14T04:56:06.986324+0000","last_fullsized":"2024-09-14T04:56:06.986324+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:45.718039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":799,"reported_epoch":1515,"state":"active+clean","last_fresh":"2024-09-14T04:56:22.072420+0000","last_change":"2024-09-14T04:49:49.443631+0000","last_active":"2024-09-14T04:56:22.072420+0000","last_peered":"2024-09-14T04:56:22.072420+0000","last_clean":"2024-09-14T04:56:22.072420+0000","last_became_active":"2024-09-14T04:49:49.443313+0000","last_became_peered":"2024-09-14T04:49:49.443313+0000","last_unstale":"2024-09-14T04:56:22.072420+0000","last_undegraded":"2024-09-14T04:56:22.072420+0000","last_fullsized":"2024-09-14T04:56:22.072420+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:06:54.562292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1506,"reported_epoch":1513,"state":"active+clean","last_fresh":"2024-09-14T04:55:59.727637+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T04:55:59.727637+0000","last_peered":"2024-09-14T04:55:59.727637+0000","last_clean":"2024-09-14T04:55:59.727637+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T04:55:59.727637+0000","last_undegraded":"2024-09-14T04:55:59.727637+0000","last_fullsized":"2024-09-14T04:55:59.727637+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"349'149","reported_seq":816,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.979420+0000","last_change":"2024-09-14T04:41:39.012977+0000","last_active":"2024-09-14T04:56:06.979420+0000","last_peered":"2024-09-14T04:56:06.979420+0000","last_clean":"2024-09-14T04:56:06.979420+0000","last_became_active":"2024-09-14T04:41:36.932552+0000","last_became_peered":"2024-09-14T04:41:36.932552+0000","last_unstale":"2024-09-14T04:56:06.979420+0000","last_undegraded":"2024-09-14T04:56:06.979420+0000","last_fullsized":"2024-09-14T04:56:06.979420+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.012879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.012879+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:16:40.080262+0000","scrub_duration":0.0051721620000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":889,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.986409+0000","last_change":"2024-09-14T04:49:49.459590+0000","last_active":"2024-09-14T04:56:06.986409+0000","last_peered":"2024-09-14T04:56:06.986409+0000","last_clean":"2024-09-14T04:56:06.986409+0000","last_became_active":"2024-09-14T04:49:49.456263+0000","last_became_peered":"2024-09-14T04:49:49.456263+0000","last_unstale":"2024-09-14T04:56:06.986409+0000","last_undegraded":"2024-09-14T04:56:06.986409+0000","last_fullsized":"2024-09-14T04:56:06.986409+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:20:44.666805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":128,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.977344+0000","last_change":"2024-09-14T04:44:57.329652+0000","last_active":"2024-09-14T04:56:06.977344+0000","last_peered":"2024-09-14T04:56:06.977344+0000","last_clean":"2024-09-14T04:56:06.977344+0000","last_became_active":"2024-09-14T04:44:57.328888+0000","last_became_peered":"2024-09-14T04:44:57.328888+0000","last_unstale":"2024-09-14T04:56:06.977344+0000","last_undegraded":"2024-09-14T04:56:06.977344+0000","last_fullsized":"2024-09-14T04:56:06.977344+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:10.170068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1504,"reported_epoch":1513,"state":"active+clean","last_fresh":"2024-09-14T04:55:59.729743+0000","last_change":"2024-09-14T04:06:46.875340+0000","last_active":"2024-09-14T04:55:59.729743+0000","last_peered":"2024-09-14T04:55:59.729743+0000","last_clean":"2024-09-14T04:55:59.729743+0000","last_became_active":"2024-09-14T04:06:46.874552+0000","last_became_peered":"2024-09-14T04:06:46.874552+0000","last_unstale":"2024-09-14T04:55:59.729743+0000","last_undegraded":"2024-09-14T04:55:59.729743+0000","last_fullsized":"2024-09-14T04:55:59.729743+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:52.257357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"349'149","reported_seq":719,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.986344+0000","last_change":"2024-09-14T04:54:55.279666+0000","last_active":"2024-09-14T04:56:06.986344+0000","last_peered":"2024-09-14T04:56:06.986344+0000","last_clean":"2024-09-14T04:56:06.986344+0000","last_became_active":"2024-09-14T04:54:55.279259+0000","last_became_peered":"2024-09-14T04:54:55.279259+0000","last_unstale":"2024-09-14T04:56:06.986344+0000","last_undegraded":"2024-09-14T04:56:06.986344+0000","last_fullsized":"2024-09-14T04:56:06.986344+0000","mapping_epoch":1506,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1507,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:54:03.996025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:54:03.996025+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:40:53.332369+0000","scrub_duration":54,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":251,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.982463+0000","last_change":"2024-09-14T04:44:14.489235+0000","last_active":"2024-09-14T04:56:06.982463+0000","last_peered":"2024-09-14T04:56:06.982463+0000","last_clean":"2024-09-14T04:56:06.982463+0000","last_became_active":"2024-09-14T04:44:14.488950+0000","last_became_peered":"2024-09-14T04:44:14.488950+0000","last_unstale":"2024-09-14T04:56:06.982463+0000","last_undegraded":"2024-09-14T04:56:06.982463+0000","last_fullsized":"2024-09-14T04:56:06.982463+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:54.441144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":807,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.986049+0000","last_change":"2024-09-14T04:49:49.456317+0000","last_active":"2024-09-14T04:56:06.986049+0000","last_peered":"2024-09-14T04:56:06.986049+0000","last_clean":"2024-09-14T04:56:06.986049+0000","last_became_active":"2024-09-14T04:49:49.456029+0000","last_became_peered":"2024-09-14T04:49:49.456029+0000","last_unstale":"2024-09-14T04:56:06.986049+0000","last_undegraded":"2024-09-14T04:56:06.986049+0000","last_fullsized":"2024-09-14T04:56:06.986049+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:46:05.319676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2324,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.987794+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T04:56:06.987794+0000","last_peered":"2024-09-14T04:56:06.987794+0000","last_clean":"2024-09-14T04:56:06.987794+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T04:56:06.987794+0000","last_undegraded":"2024-09-14T04:56:06.987794+0000","last_fullsized":"2024-09-14T04:56:06.987794+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"349'149","reported_seq":711,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.982025+0000","last_change":"2024-09-14T04:50:22.002798+0000","last_active":"2024-09-14T04:56:06.982025+0000","last_peered":"2024-09-14T04:56:06.982025+0000","last_clean":"2024-09-14T04:56:06.982025+0000","last_became_active":"2024-09-14T04:49:49.448278+0000","last_became_peered":"2024-09-14T04:49:49.448278+0000","last_unstale":"2024-09-14T04:56:06.982025+0000","last_undegraded":"2024-09-14T04:56:06.982025+0000","last_fullsized":"2024-09-14T04:56:06.982025+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:50:22.002568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:50:22.002568+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:01:57.489223+0000","scrub_duration":0.010888326,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2327,"reported_epoch":1515,"state":"active+clean","last_fresh":"2024-09-14T04:56:22.073035+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T04:56:22.073035+0000","last_peered":"2024-09-14T04:56:22.073035+0000","last_clean":"2024-09-14T04:56:22.073035+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T04:56:22.073035+0000","last_undegraded":"2024-09-14T04:56:22.073035+0000","last_fullsized":"2024-09-14T04:56:22.073035+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2348,"reported_epoch":1515,"state":"active+clean","last_fresh":"2024-09-14T04:56:22.074293+0000","last_change":"2024-09-14T04:54:19.447103+0000","last_active":"2024-09-14T04:56:22.074293+0000","last_peered":"2024-09-14T04:56:22.074293+0000","last_clean":"2024-09-14T04:56:22.074293+0000","last_became_active":"2024-09-14T04:54:19.446748+0000","last_became_peered":"2024-09-14T04:54:19.446748+0000","last_unstale":"2024-09-14T04:56:22.074293+0000","last_undegraded":"2024-09-14T04:56:22.074293+0000","last_fullsized":"2024-09-14T04:56:22.074293+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:00:55.369263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1536,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.986203+0000","last_change":"2024-09-14T04:42:49.939790+0000","last_active":"2024-09-14T04:56:06.986203+0000","last_peered":"2024-09-14T04:56:06.986203+0000","last_clean":"2024-09-14T04:56:06.986203+0000","last_became_active":"2024-09-14T04:42:49.939591+0000","last_became_peered":"2024-09-14T04:42:49.939591+0000","last_unstale":"2024-09-14T04:56:06.986203+0000","last_undegraded":"2024-09-14T04:56:06.986203+0000","last_fullsized":"2024-09-14T04:56:06.986203+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:23:16.790593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1499,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.983562+0000","last_change":"2024-09-14T04:44:57.326008+0000","last_active":"2024-09-14T04:56:06.983562+0000","last_peered":"2024-09-14T04:56:06.983562+0000","last_clean":"2024-09-14T04:56:06.983562+0000","last_became_active":"2024-09-14T04:44:57.325617+0000","last_became_peered":"2024-09-14T04:44:57.325617+0000","last_unstale":"2024-09-14T04:56:06.983562+0000","last_undegraded":"2024-09-14T04:56:06.983562+0000","last_fullsized":"2024-09-14T04:56:06.983562+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:15.637508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":787,"reported_epoch":1513,"state":"active+clean","last_fresh":"2024-09-14T04:55:59.729792+0000","last_change":"2024-09-14T04:43:13.354846+0000","last_active":"2024-09-14T04:55:59.729792+0000","last_peered":"2024-09-14T04:55:59.729792+0000","last_clean":"2024-09-14T04:55:59.729792+0000","last_became_active":"2024-09-14T04:43:13.354515+0000","last_became_peered":"2024-09-14T04:43:13.354515+0000","last_unstale":"2024-09-14T04:55:59.729792+0000","last_undegraded":"2024-09-14T04:55:59.729792+0000","last_fullsized":"2024-09-14T04:55:59.729792+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:17.889341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1503,"reported_epoch":1513,"state":"active+clean","last_fresh":"2024-09-14T04:55:59.727776+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T04:55:59.727776+0000","last_peered":"2024-09-14T04:55:59.727776+0000","last_clean":"2024-09-14T04:55:59.727776+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T04:55:59.727776+0000","last_undegraded":"2024-09-14T04:55:59.727776+0000","last_fullsized":"2024-09-14T04:55:59.727776+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":1842,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.985853+0000","last_change":"2024-09-14T04:41:30.640651+0000","last_active":"2024-09-14T04:56:06.985853+0000","last_peered":"2024-09-14T04:56:06.985853+0000","last_clean":"2024-09-14T04:56:06.985853+0000","last_became_active":"2024-09-14T04:41:29.532466+0000","last_became_peered":"2024-09-14T04:41:29.532466+0000","last_unstale":"2024-09-14T04:56:06.985853+0000","last_undegraded":"2024-09-14T04:56:06.985853+0000","last_fullsized":"2024-09-14T04:56:06.985853+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":0,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:30.640520+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:30.640520+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:27.776724+0000","scrub_duration":0.0028653139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"349'149","reported_seq":815,"reported_epoch":1513,"state":"active+clean","last_fresh":"2024-09-14T04:55:59.729698+0000","last_change":"2024-09-14T04:41:39.978080+0000","last_active":"2024-09-14T04:55:59.729698+0000","last_peered":"2024-09-14T04:55:59.729698+0000","last_clean":"2024-09-14T04:55:59.729698+0000","last_became_active":"2024-09-14T04:41:36.929279+0000","last_became_peered":"2024-09-14T04:41:36.929279+0000","last_unstale":"2024-09-14T04:55:59.729698+0000","last_undegraded":"2024-09-14T04:55:59.729698+0000","last_fullsized":"2024-09-14T04:55:59.729698+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.977999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.977999+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:44:06.383827+0000","scrub_duration":0.00192122,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":185,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.989944+0000","last_change":"2024-09-14T04:48:11.040133+0000","last_active":"2024-09-14T04:56:06.989944+0000","last_peered":"2024-09-14T04:56:06.989944+0000","last_clean":"2024-09-14T04:56:06.989944+0000","last_became_active":"2024-09-14T04:48:11.039787+0000","last_became_peered":"2024-09-14T04:48:11.039787+0000","last_unstale":"2024-09-14T04:56:06.989944+0000","last_undegraded":"2024-09-14T04:56:06.989944+0000","last_fullsized":"2024-09-14T04:56:06.989944+0000","mapping_epoch":1428,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:53.520672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2172,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.981373+0000","last_change":"2024-09-14T04:49:29.616443+0000","last_active":"2024-09-14T04:56:06.981373+0000","last_peered":"2024-09-14T04:56:06.981373+0000","last_clean":"2024-09-14T04:56:06.981373+0000","last_became_active":"2024-09-14T04:49:29.616066+0000","last_became_peered":"2024-09-14T04:49:29.616066+0000","last_unstale":"2024-09-14T04:56:06.981373+0000","last_undegraded":"2024-09-14T04:56:06.981373+0000","last_fullsized":"2024-09-14T04:56:06.981373+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:09:15.897957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":128,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:13.360783+0000","last_change":"2024-09-14T04:44:57.326724+0000","last_active":"2024-09-14T04:56:13.360783+0000","last_peered":"2024-09-14T04:56:13.360783+0000","last_clean":"2024-09-14T04:56:13.360783+0000","last_became_active":"2024-09-14T04:44:57.326191+0000","last_became_peered":"2024-09-14T04:44:57.326191+0000","last_unstale":"2024-09-14T04:56:13.360783+0000","last_undegraded":"2024-09-14T04:56:13.360783+0000","last_fullsized":"2024-09-14T04:56:13.360783+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:46.060269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":128,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.976582+0000","last_change":"2024-09-14T04:44:57.328528+0000","last_active":"2024-09-14T04:56:06.976582+0000","last_peered":"2024-09-14T04:56:06.976582+0000","last_clean":"2024-09-14T04:56:06.976582+0000","last_became_active":"2024-09-14T04:44:57.327739+0000","last_became_peered":"2024-09-14T04:44:57.327739+0000","last_unstale":"2024-09-14T04:56:06.976582+0000","last_undegraded":"2024-09-14T04:56:06.976582+0000","last_fullsized":"2024-09-14T04:56:06.976582+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:04:58.294690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"349'149","reported_seq":824,"reported_epoch":1513,"state":"active+clean","last_fresh":"2024-09-14T04:55:59.733165+0000","last_change":"2024-09-14T04:55:59.732757+0000","last_active":"2024-09-14T04:55:59.733165+0000","last_peered":"2024-09-14T04:55:59.733165+0000","last_clean":"2024-09-14T04:55:59.733165+0000","last_became_active":"2024-09-14T04:55:59.732364+0000","last_became_peered":"2024-09-14T04:55:59.732364+0000","last_unstale":"2024-09-14T04:55:59.733165+0000","last_undegraded":"2024-09-14T04:55:59.733165+0000","last_fullsized":"2024-09-14T04:55:59.733165+0000","mapping_epoch":1512,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1513,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:56.738636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:56.738636+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:59:11.313765+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1313440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":128,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:13.360758+0000","last_change":"2024-09-14T04:44:57.323436+0000","last_active":"2024-09-14T04:56:13.360758+0000","last_peered":"2024-09-14T04:56:13.360758+0000","last_clean":"2024-09-14T04:56:13.360758+0000","last_became_active":"2024-09-14T04:44:57.323022+0000","last_became_peered":"2024-09-14T04:44:57.323022+0000","last_unstale":"2024-09-14T04:56:13.360758+0000","last_undegraded":"2024-09-14T04:56:13.360758+0000","last_fullsized":"2024-09-14T04:56:13.360758+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:22.775261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1534,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.982256+0000","last_change":"2024-09-14T04:44:57.322357+0000","last_active":"2024-09-14T04:56:06.982256+0000","last_peered":"2024-09-14T04:56:06.982256+0000","last_clean":"2024-09-14T04:56:06.982256+0000","last_became_active":"2024-09-14T04:44:57.321990+0000","last_became_peered":"2024-09-14T04:44:57.321990+0000","last_unstale":"2024-09-14T04:56:06.982256+0000","last_undegraded":"2024-09-14T04:56:06.982256+0000","last_fullsized":"2024-09-14T04:56:06.982256+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:09.658008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"349'149","reported_seq":1858,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.986110+0000","last_change":"2024-09-14T04:41:53.858076+0000","last_active":"2024-09-14T04:56:06.986110+0000","last_peered":"2024-09-14T04:56:06.986110+0000","last_clean":"2024-09-14T04:56:06.986110+0000","last_became_active":"2024-09-14T04:41:36.931585+0000","last_became_peered":"2024-09-14T04:41:36.931585+0000","last_unstale":"2024-09-14T04:56:06.986110+0000","last_undegraded":"2024-09-14T04:56:06.986110+0000","last_fullsized":"2024-09-14T04:56:06.986110+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:53.857924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:53.857924+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:10:20.658986+0000","scrub_duration":0.0022355919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":128,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:13.360731+0000","last_change":"2024-09-14T04:44:57.329198+0000","last_active":"2024-09-14T04:56:13.360731+0000","last_peered":"2024-09-14T04:56:13.360731+0000","last_clean":"2024-09-14T04:56:13.360731+0000","last_became_active":"2024-09-14T04:44:57.327754+0000","last_became_peered":"2024-09-14T04:44:57.327754+0000","last_unstale":"2024-09-14T04:56:13.360731+0000","last_undegraded":"2024-09-14T04:56:13.360731+0000","last_fullsized":"2024-09-14T04:56:13.360731+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:10:27.944476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":310,"reported_epoch":1515,"state":"active+clean","last_fresh":"2024-09-14T04:56:22.073570+0000","last_change":"2024-09-14T04:44:57.348687+0000","last_active":"2024-09-14T04:56:22.073570+0000","last_peered":"2024-09-14T04:56:22.073570+0000","last_clean":"2024-09-14T04:56:22.073570+0000","last_became_active":"2024-09-14T04:44:57.332845+0000","last_became_peered":"2024-09-14T04:44:57.332845+0000","last_unstale":"2024-09-14T04:56:22.073570+0000","last_undegraded":"2024-09-14T04:56:22.073570+0000","last_fullsized":"2024-09-14T04:56:22.073570+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:12:18.762737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2333,"reported_epoch":1515,"state":"active+clean","last_fresh":"2024-09-14T04:56:22.073159+0000","last_change":"2024-09-14T04:44:57.333468+0000","last_active":"2024-09-14T04:56:22.073159+0000","last_peered":"2024-09-14T04:56:22.073159+0000","last_clean":"2024-09-14T04:56:22.073159+0000","last_became_active":"2024-09-14T04:44:57.332626+0000","last_became_peered":"2024-09-14T04:44:57.332626+0000","last_unstale":"2024-09-14T04:56:22.073159+0000","last_undegraded":"2024-09-14T04:56:22.073159+0000","last_fullsized":"2024-09-14T04:56:22.073159+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:57:26.366438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1489,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.979169+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T04:56:06.979169+0000","last_peered":"2024-09-14T04:56:06.979169+0000","last_clean":"2024-09-14T04:56:06.979169+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T04:56:06.979169+0000","last_undegraded":"2024-09-14T04:56:06.979169+0000","last_fullsized":"2024-09-14T04:56:06.979169+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1489,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.982144+0000","last_change":"2024-09-14T04:06:46.869868+0000","last_active":"2024-09-14T04:56:06.982144+0000","last_peered":"2024-09-14T04:56:06.982144+0000","last_clean":"2024-09-14T04:56:06.982144+0000","last_became_active":"2024-09-14T04:06:46.869126+0000","last_became_peered":"2024-09-14T04:06:46.869126+0000","last_unstale":"2024-09-14T04:56:06.982144+0000","last_undegraded":"2024-09-14T04:56:06.982144+0000","last_fullsized":"2024-09-14T04:56:06.982144+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:41.150138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1473,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:08.824601+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T04:56:08.824601+0000","last_peered":"2024-09-14T04:56:08.824601+0000","last_clean":"2024-09-14T04:56:08.824601+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T04:56:08.824601+0000","last_undegraded":"2024-09-14T04:56:08.824601+0000","last_fullsized":"2024-09-14T04:56:08.824601+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":128,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.976804+0000","last_change":"2024-09-14T04:44:57.332112+0000","last_active":"2024-09-14T04:56:06.976804+0000","last_peered":"2024-09-14T04:56:06.976804+0000","last_clean":"2024-09-14T04:56:06.976804+0000","last_became_active":"2024-09-14T04:44:57.331575+0000","last_became_peered":"2024-09-14T04:44:57.331575+0000","last_unstale":"2024-09-14T04:56:06.976804+0000","last_undegraded":"2024-09-14T04:56:06.976804+0000","last_fullsized":"2024-09-14T04:56:06.976804+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:50:00.391283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1521,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:08.824622+0000","last_change":"2024-09-14T04:44:57.351460+0000","last_active":"2024-09-14T04:56:08.824622+0000","last_peered":"2024-09-14T04:56:08.824622+0000","last_clean":"2024-09-14T04:56:08.824622+0000","last_became_active":"2024-09-14T04:44:57.334662+0000","last_became_peered":"2024-09-14T04:44:57.334662+0000","last_unstale":"2024-09-14T04:56:08.824622+0000","last_undegraded":"2024-09-14T04:56:08.824622+0000","last_fullsized":"2024-09-14T04:56:08.824622+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:02.399062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1474,"reported_epoch":1513,"state":"active+clean","last_fresh":"2024-09-14T04:55:59.727624+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T04:55:59.727624+0000","last_peered":"2024-09-14T04:55:59.727624+0000","last_clean":"2024-09-14T04:55:59.727624+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T04:55:59.727624+0000","last_undegraded":"2024-09-14T04:55:59.727624+0000","last_fullsized":"2024-09-14T04:55:59.727624+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1533,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:13.833786+0000","last_change":"2024-09-14T04:44:57.349690+0000","last_active":"2024-09-14T04:56:13.833786+0000","last_peered":"2024-09-14T04:56:13.833786+0000","last_clean":"2024-09-14T04:56:13.833786+0000","last_became_active":"2024-09-14T04:44:57.333579+0000","last_became_peered":"2024-09-14T04:44:57.333579+0000","last_unstale":"2024-09-14T04:56:13.833786+0000","last_undegraded":"2024-09-14T04:56:13.833786+0000","last_fullsized":"2024-09-14T04:56:13.833786+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:17:55.265141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1610,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.980502+0000","last_change":"2024-09-14T04:54:35.694089+0000","last_active":"2024-09-14T04:56:06.980502+0000","last_peered":"2024-09-14T04:56:06.980502+0000","last_clean":"2024-09-14T04:56:06.980502+0000","last_became_active":"2024-09-14T04:54:35.663133+0000","last_became_peered":"2024-09-14T04:54:35.663133+0000","last_unstale":"2024-09-14T04:56:06.980502+0000","last_undegraded":"2024-09-14T04:56:06.980502+0000","last_fullsized":"2024-09-14T04:56:06.980502+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:17:19.947280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":22,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.986611+0000","last_change":"2024-09-14T04:54:52.248439+0000","last_active":"2024-09-14T04:56:06.986611+0000","last_peered":"2024-09-14T04:56:06.986611+0000","last_clean":"2024-09-14T04:56:06.986611+0000","last_became_active":"2024-09-14T04:54:52.248235+0000","last_became_peered":"2024-09-14T04:54:52.248235+0000","last_unstale":"2024-09-14T04:56:06.986611+0000","last_undegraded":"2024-09-14T04:56:06.986611+0000","last_fullsized":"2024-09-14T04:56:06.986611+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:22:09.548139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1495,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:08.824530+0000","last_change":"2024-09-14T04:44:57.332511+0000","last_active":"2024-09-14T04:56:08.824530+0000","last_peered":"2024-09-14T04:56:08.824530+0000","last_clean":"2024-09-14T04:56:08.824530+0000","last_became_active":"2024-09-14T04:44:57.332166+0000","last_became_peered":"2024-09-14T04:44:57.332166+0000","last_unstale":"2024-09-14T04:56:08.824530+0000","last_undegraded":"2024-09-14T04:56:08.824530+0000","last_fullsized":"2024-09-14T04:56:08.824530+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:53.984639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2326,"reported_epoch":1515,"state":"active+clean","last_fresh":"2024-09-14T04:56:22.072777+0000","last_change":"2024-09-14T04:54:52.250655+0000","last_active":"2024-09-14T04:56:22.072777+0000","last_peered":"2024-09-14T04:56:22.072777+0000","last_clean":"2024-09-14T04:56:22.072777+0000","last_became_active":"2024-09-14T04:54:52.250317+0000","last_became_peered":"2024-09-14T04:54:52.250317+0000","last_unstale":"2024-09-14T04:56:22.072777+0000","last_undegraded":"2024-09-14T04:56:22.072777+0000","last_fullsized":"2024-09-14T04:56:22.072777+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:12.985406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2312,"reported_epoch":1515,"state":"active+clean","last_fresh":"2024-09-14T04:56:22.072165+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T04:56:22.072165+0000","last_peered":"2024-09-14T04:56:22.072165+0000","last_clean":"2024-09-14T04:56:22.072165+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T04:56:22.072165+0000","last_undegraded":"2024-09-14T04:56:22.072165+0000","last_fullsized":"2024-09-14T04:56:22.072165+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1486,"reported_epoch":1513,"state":"active+clean","last_fresh":"2024-09-14T04:55:59.727726+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T04:55:59.727726+0000","last_peered":"2024-09-14T04:55:59.727726+0000","last_clean":"2024-09-14T04:55:59.727726+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T04:55:59.727726+0000","last_undegraded":"2024-09-14T04:55:59.727726+0000","last_fullsized":"2024-09-14T04:55:59.727726+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1524,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:08.824562+0000","last_change":"2024-09-14T04:44:57.339993+0000","last_active":"2024-09-14T04:56:08.824562+0000","last_peered":"2024-09-14T04:56:08.824562+0000","last_clean":"2024-09-14T04:56:08.824562+0000","last_became_active":"2024-09-14T04:44:57.330472+0000","last_became_peered":"2024-09-14T04:44:57.330472+0000","last_unstale":"2024-09-14T04:56:08.824562+0000","last_undegraded":"2024-09-14T04:56:08.824562+0000","last_fullsized":"2024-09-14T04:56:08.824562+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:19:27.046988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2071,"reported_epoch":1514,"state":"active+clean","last_fresh":"2024-09-14T04:56:06.986210+0000","last_change":"2024-09-14T04:49:49.449687+0000","last_active":"2024-09-14T04:56:06.986210+0000","last_peered":"2024-09-14T04:56:06.986210+0000","last_clean":"2024-09-14T04:56:06.986210+0000","last_became_active":"2024-09-14T04:49:49.449144+0000","last_became_peered":"2024-09-14T04:49:49.449144+0000","last_unstale":"2024-09-14T04:56:06.986210+0000","last_undegraded":"2024-09-14T04:56:06.986210+0000","last_fullsized":"2024-09-14T04:56:06.986210+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:59:06.779730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":56,"num_pg":16,"stat_sum":{"num_bytes":73072640,"num_objects":1115,"num_object_clones":0,"num_object_copies":3345,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2690,"num_objects_misplaced":317,"num_objects_unfound":0,"num_objects_dirty":1115,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":227929,"num_write_kb":7329345,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11276,"num_bytes_recovered":688390144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1019678720,"data_stored":1019674648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":125283,"ondisk_log_size":125283,"up":48,"acting":47,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1313440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":5070,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4250,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223202065,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":899172,"kb_used_data":200344,"kb_used_omap":1,"kb_used_meta":698814,"kb_avail":92845980,"statfs":{"total":95995035648,"available":95074283520,"internally_reserved":0,"allocated":205152256,"data_stored":199908457,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":715585995},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463701,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":670620,"kb_used_data":11984,"kb_used_omap":1,"kb_used_meta":658622,"kb_avail":93074532,"statfs":{"total":95995035648,"available":95308320768,"internally_reserved":0,"allocated":12271616,"data_stored":7035882,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":674429383},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725337,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":852292,"kb_used_data":103420,"kb_used_omap":1,"kb_used_meta":748862,"kb_avail":92892860,"statfs":{"total":95995035648,"available":95122288640,"internally_reserved":0,"allocated":105902080,"data_stored":100674307,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":766835142},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":1392,"seq":5978594476184,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":483172,"kb_used_data":185432,"kb_used_omap":117,"kb_used_meta":297610,"kb_avail":93261980,"statfs":{"total":95995035648,"available":95500267520,"internally_reserved":0,"allocated":189882368,"data_stored":184639121,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120638,"internal_metadata":304752834},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772075079,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":842436,"kb_used_data":88444,"kb_used_omap":71,"kb_used_meta":753912,"kb_avail":92902716,"statfs":{"total":95995035648,"available":95132381184,"internally_reserved":0,"allocated":90566656,"data_stored":85338960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":772006838},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707304014,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":591552,"kb_used_data":81912,"kb_used_omap":68,"kb_used_meta":509563,"kb_avail":93153600,"statfs":{"total":95995035648,"available":95389286400,"internally_reserved":0,"allocated":83877888,"data_stored":78654168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":521792720},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872729173,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":661752,"kb_used_data":92464,"kb_used_omap":67,"kb_used_meta":569212,"kb_avail":93083400,"statfs":{"total":95995035648,"available":95317401600,"internally_reserved":0,"allocated":94683136,"data_stored":89466467,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":582873445},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056224,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":997072,"kb_used_data":365508,"kb_used_omap":74,"kb_used_meta":631477,"kb_avail":92748080,"statfs":{"total":95995035648,"available":94974033920,"internally_reserved":0,"allocated":374280192,"data_stored":369052517,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":646633141},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1901,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":362020864,"data_stored":362020864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":66191360,"data_stored":66191360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":69337088,"data_stored":69337088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":69406720,"data_stored":69402648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":177602560,"data_stored":177602560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":82575360,"data_stored":82575360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":56,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":192544768,"data_stored":192544768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:56:30.705 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:30.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:30.727 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:30.728 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:30.829 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:30.851 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:30.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:30.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:30.953 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:30.975 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:30.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:30.976 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:31.077 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:31.100 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:31.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:31.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:31.202 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:31.223 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:31.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:31.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:31.325 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:31.348 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:31.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:31.349 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:31.449 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:31.473 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:31.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:31.474 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:31.575 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:31.597 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:31.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:31.598 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:31.635 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-14T04:56:31.699 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:31.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:31.722 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:31.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:31.823 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:31.846 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:31.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:31.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:31.914 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:31 smithi039 ceph-mon[106470]: pgmap v2891: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 301 op/s; 2454/78 objects degraded (3146.154%); 646 KiB/s, 118 objects/s recovering 2024-09-14T04:56:31.914 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:31 smithi039 ceph-mon[106470]: Health check update: Degraded data redundancy: 2454/78 objects degraded (3146.154%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:56:31.914 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:31 smithi039 ceph-mon[111235]: pgmap v2891: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 301 op/s; 2454/78 objects degraded (3146.154%); 646 KiB/s, 118 objects/s recovering 2024-09-14T04:56:31.914 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:31 smithi039 ceph-mon[111235]: Health check update: Degraded data redundancy: 2454/78 objects degraded (3146.154%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:56:31.948 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:31.969 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:31.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:31.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:32.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:31 smithi129 ceph-mon[101316]: pgmap v2891: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 301 op/s; 2454/78 objects degraded (3146.154%); 646 KiB/s, 118 objects/s recovering 2024-09-14T04:56:32.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:31 smithi129 ceph-mon[101316]: Health check update: Degraded data redundancy: 2454/78 objects degraded (3146.154%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2024-09-14T04:56:32.071 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:32.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:32.096 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:32.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:32.197 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:32.219 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:32.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:32.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:32.321 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:32.344 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:32.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:32.345 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:32.446 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:32.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:32.469 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:32.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:32.569 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:32.591 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:32.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:32.592 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:32.693 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:32.715 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:32.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:32.717 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:32.817 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:32.839 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:32.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:32.840 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:32.941 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:32.966 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:32.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:32.968 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:33.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:32 smithi129 ceph-mon[101316]: pgmap v2892: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 301 op/s; 2454/78 objects degraded (3146.154%); 646 KiB/s, 118 objects/s recovering 2024-09-14T04:56:33.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:32 smithi039 ceph-mon[106470]: pgmap v2892: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 301 op/s; 2454/78 objects degraded (3146.154%); 646 KiB/s, 118 objects/s recovering 2024-09-14T04:56:33.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:32 smithi039 ceph-mon[111235]: pgmap v2892: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 301 op/s; 2454/78 objects degraded (3146.154%); 646 KiB/s, 118 objects/s recovering 2024-09-14T04:56:33.054 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:56:33.054 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:56:33.069 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:33.073 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:56:33.074 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:56:33.091 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:56:33.091 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:56:33.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:33.092 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:33.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:33.108 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:56:33.108 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:56:33.125 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:56:33.125 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:56:33.143 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:56:33.143 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:56:33.160 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:56:33.160 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:56:33.180 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:56:33.180 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:56:33.193 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:33.215 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:33.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:33.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:33.317 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:33.340 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:33.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:33.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:33.441 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:33.464 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:33.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:33.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:33.566 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:33.589 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:33.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:33.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:33.690 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:33.706 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:56:33.711 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:33.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:33.712 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:33.813 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:33.837 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:33.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:33.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:33.939 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:33.961 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:33.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:33.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:34.003 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:33 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/143407503' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:56:34.004 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:33 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/143407503' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:56:34.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:33 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/143407503' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:56:34.063 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:34.085 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:34.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:34.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:34.187 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:34.209 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:34.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:34.210 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:34.310 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:34.333 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:34.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:34.334 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:34.435 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:34.458 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:34.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:34.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:34.560 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:34.584 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:34.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:34.585 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:34.686 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:34.710 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:34.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:34.711 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:34.811 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:34.834 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:34.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:34.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:34.936 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:34.959 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:34.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:34.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:35.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:34 smithi039 ceph-mon[106470]: pgmap v2893: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 255 op/s; 2454/78 objects degraded (3146.154%); 547 KiB/s, 100 objects/s recovering 2024-09-14T04:56:35.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:34 smithi039 ceph-mon[111235]: pgmap v2893: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 255 op/s; 2454/78 objects degraded (3146.154%); 547 KiB/s, 100 objects/s recovering 2024-09-14T04:56:35.060 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:35.082 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:35.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:35.083 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:35.184 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:35.206 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:35.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:35.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:35.232 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:56:35.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:34 smithi129 ceph-mon[101316]: pgmap v2893: 99 pgs: 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 97 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 255 op/s; 2454/78 objects degraded (3146.154%); 547 KiB/s, 100 objects/s recovering 2024-09-14T04:56:35.308 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:35.332 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:35.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:35.333 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:35.433 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:35.456 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:35.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:35.457 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:35.558 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:35.579 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:35.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:35.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:35.681 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:35.703 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:35.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:35.704 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:35.803 INFO:teuthology.orchestra.run.smithi039.stderr:pool 'unique_pool_3' removed 2024-09-14T04:56:35.805 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:35.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:35.842 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:35.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:35.943 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:35.965 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:35.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:35.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:36.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:35 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3580898849' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:56:36.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:35 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:56:36.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:35 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3580898849' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:56:36.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:35 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:56:36.067 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:36.090 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:36.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:36.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:36.192 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:36.217 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:36.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:36.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:36.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:35 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3580898849' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:56:36.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:35 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T04:56:36.319 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:36.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:36.345 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:36.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:36.445 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:36.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:36.469 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:36.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:36.570 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:36.593 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:36.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:36.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:36.694 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:36.717 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:36.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:36.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:36.818 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:36.839 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:36.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:36.840 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:36.941 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:36.950 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:56:36.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:36.966 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:36.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:37.067 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:37.089 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:37.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:37.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:37.191 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:37.213 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:37.213 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:37.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:37.260 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:36 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:56:37.260 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:36 smithi039 ceph-mon[106470]: osdmap e1516: 8 total, 8 up, 8 in 2024-09-14T04:56:37.260 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:36 smithi039 ceph-mon[106470]: pgmap v2895: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:37.261 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:36 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:56:37.261 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:36 smithi039 ceph-mon[111235]: osdmap e1516: 8 total, 8 up, 8 in 2024-09-14T04:56:37.261 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:36 smithi039 ceph-mon[111235]: pgmap v2895: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:37.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:36 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T04:56:37.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:36 smithi129 ceph-mon[101316]: osdmap e1516: 8 total, 8 up, 8 in 2024-09-14T04:56:37.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:36 smithi129 ceph-mon[101316]: pgmap v2895: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:37.281 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-14T04:56:37.282 INFO:tasks.radosbench:Beginning radosbench... 2024-09-14T04:56:37.283 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2024-09-14T04:56:37.283 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_4 16 2024-09-14T04:56:37.314 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:37.336 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:37.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:37.337 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:37.437 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:37.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:37.459 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:37.459 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:37.560 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:37.581 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:37.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:37.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:37.683 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:37.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:37.705 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:37.705 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:37.806 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:37.829 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:37.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:37.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:37.931 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:37.957 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:37.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:37.958 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:38.059 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:38.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:38.083 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:38.083 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:38.100 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:37 smithi039 ceph-mon[106470]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2454/78 objects degraded (3146.154%), 1 pg degraded, 1 pg undersized) 2024-09-14T04:56:38.100 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:37 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:56:38.100 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:56:38.100 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:37 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1372749374' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:56:38.100 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:37 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:56:38.100 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:37 smithi039 ceph-mon[111235]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2454/78 objects degraded (3146.154%), 1 pg degraded, 1 pg undersized) 2024-09-14T04:56:38.101 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:37 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:56:38.101 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:56:38.101 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:37 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1372749374' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:56:38.101 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:37 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:56:38.185 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:38.207 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:38.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:38.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:38.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:37 smithi129 ceph-mon[101316]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2454/78 objects degraded (3146.154%), 1 pg degraded, 1 pg undersized) 2024-09-14T04:56:38.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:37 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:56:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:56:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:37 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1372749374' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:56:38.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:37 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:56:38.309 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:38.324 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is unset 2024-09-14T04:56:38.336 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:38.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:38.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:38.438 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:38.462 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:38.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:38.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:38.564 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:38.587 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:38.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:38.589 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:38.690 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:38.712 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:38.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:38.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:38.767 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:56:38.767 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:56:38.767 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:56:38.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:56:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:56:38] "GET /metrics HTTP/1.1" 200 34311 "" "Prometheus/2.51.0" 2024-09-14T04:56:38.814 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:38.838 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:38.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:38.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:38.940 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:38.962 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:38.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:38.963 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:39.064 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:39.086 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:39.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:39.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:39.188 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:39.189 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:38 smithi039 ceph-mon[106470]: pgmap v2896: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:39.189 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:38 smithi039 ceph-mon[106470]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:56:39.189 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:38 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T04:56:39.189 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:38 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:56:39.189 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:38 smithi039 ceph-mon[106470]: osdmap e1517: 8 total, 8 up, 8 in 2024-09-14T04:56:39.189 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:38 smithi039 ceph-mon[111235]: pgmap v2896: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:39.189 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:38 smithi039 ceph-mon[111235]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:56:39.190 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:38 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T04:56:39.190 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:38 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:56:39.190 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:38 smithi039 ceph-mon[111235]: osdmap e1517: 8 total, 8 up, 8 in 2024-09-14T04:56:39.214 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:39.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:39.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:39.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:38 smithi129 ceph-mon[101316]: pgmap v2896: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:39.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:38 smithi129 ceph-mon[101316]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:56:39.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:38 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T04:56:39.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:38 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:56:39.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:38 smithi129 ceph-mon[101316]: osdmap e1517: 8 total, 8 up, 8 in 2024-09-14T04:56:39.316 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:39.340 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:39.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:39.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:39.442 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:39.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:39.468 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:39.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:39.569 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:39.593 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:39.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:39.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:39.695 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:39.721 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:39.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:39.722 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:39.822 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:39.845 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:39.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:39.846 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:39.947 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:39.972 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:39.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:39.973 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:39.987 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:56:39.987 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [], 2024-09-14T04:56:39.987 INFO:teuthology.orchestra.run.smithi039.stdout: "num_ops": 0 2024-09-14T04:56:39.987 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:56:40.074 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:40.097 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:40.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:40.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:40.198 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:40.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:40.222 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:40.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:40.323 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:40.349 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:40.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:40.350 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:40.451 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:40.475 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:40.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:40.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:40.577 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:40.603 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:40.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:40.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:40.705 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:40.728 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:40.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:40.729 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:40.830 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:40.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:40.853 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:40.853 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:40.954 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:40.980 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:40.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:40.981 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:41.082 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:41.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:41.106 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:41.106 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:41.207 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:41.216 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:56:41.230 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:41.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:41.231 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:41.252 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:40 smithi039 ceph-mon[106470]: pgmap v2898: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:41.253 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:40 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:56:41.253 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:40 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:56:41.253 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:40 smithi039 ceph-mon[111235]: pgmap v2898: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:41.253 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:40 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:56:41.253 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:40 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:56:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:40 smithi129 ceph-mon[101316]: pgmap v2898: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:40 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:56:41.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:40 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:56:41.333 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:41.356 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:41.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:41.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:41.458 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:41.483 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:41.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:41.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:41.585 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:41.607 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:41.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:41.608 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:41.709 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:41.731 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:41.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:41.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:41.833 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:41.856 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:41.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:41.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:41.957 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:41.980 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:41.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:41.981 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:42.081 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:42.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:42.105 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:42.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:42.206 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:42.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:42.229 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:42.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:42.235 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:41 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:56:42.235 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:41 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:56:42.236 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:41 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:56:42.236 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:41 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:56:42.236 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:41 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:56:42.236 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:41 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:56:42.255 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-14T04:56:42.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:41 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:56:42.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:41 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:56:42.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:41 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:56:42.330 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:42.361 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:42.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:42.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:42.463 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:42.485 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:42.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:42.486 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:42.587 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:42.610 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:42.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:42.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:42.712 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:42.735 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:42.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:42.736 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:42.837 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:42.858 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:42.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:42.860 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:42.961 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:42.987 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:42.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:42.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:43.089 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:43.111 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:43.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:43.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:43.197 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:56:43.213 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:43.238 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:43.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:43.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:43.340 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:43.362 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:43.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:43.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:43.420 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:43 smithi039 ceph-mon[106470]: pgmap v2899: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:43.421 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:43 smithi039 ceph-mon[111235]: pgmap v2899: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:43.465 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:43.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:43.490 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:43.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:43.591 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:43.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:43.615 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:43.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:43.680 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:43 smithi129 ceph-mon[101316]: pgmap v2899: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:43.715 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:43.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:43.739 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:43.739 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:43.840 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:43.862 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:43.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:43.863 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:43.964 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:43.989 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:43.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:43.990 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:44.091 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:44.116 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:44.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:44.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:44.218 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:44.241 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:44.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:44.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:44.328 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:56:44.343 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:44.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:44.366 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:44.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:44.467 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:44.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:44.491 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:44.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:44.592 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:44.615 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:44.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:44.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:44.717 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:44.741 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:44.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:44.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:44.842 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:44.868 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:44.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:44.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:44.970 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:44.993 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:44.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:44.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:45.095 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:45.118 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:45.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:45.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:45.220 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:45.242 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:45.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:45.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:45.344 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:45.369 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:45.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:45.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:45.471 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:45.494 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:45.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:45.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:45.596 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:45.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:45.619 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:45.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:45.640 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:56:45.640 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:45 smithi039 ceph-mon[106470]: pgmap v2900: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:45.641 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:45 smithi039 ceph-mon[111235]: pgmap v2900: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:45.642 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T04:56:45.721 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:45.747 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:45.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:45.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:45.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:45 smithi129 ceph-mon[101316]: pgmap v2900: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:45.849 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:45.873 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:45.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:45.874 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:45.974 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:45.998 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:45.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:45.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:46.100 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:46.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:46.128 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:46.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:46.229 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:46.254 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:46.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:46.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:46.356 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:46.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:46.382 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:46.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:46.406 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":2900,"stamp":"2024-09-14T04:56:43.898208+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":53,"num_bytes_recovered":1318510,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1698,"ondisk_log_size":1698,"up":249,"acting":249,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":252,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":5646808,"kb_used_data":747776,"kb_used_omap":404,"kb_used_meta":4898539,"kb_avail":744314408,"statfs":{"total":767960285184,"available":762177953792,"internally_reserved":0,"allocated":765722624,"data_stored":723694203,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":414022,"internal_metadata":5016104634},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":31,"apply_latency_ms":31,"commit_latency_ns":31000000,"apply_latency_ns":31000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.099196"},"pg_stats":[{"pgid":"3.1f","version":"72'4","reported_seq":2327,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-09-14T04:56:38.322512+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T04:56:38.322512+0000","last_peered":"2024-09-14T04:56:38.322512+0000","last_clean":"2024-09-14T04:56:38.322512+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T04:56:38.322512+0000","last_undegraded":"2024-09-14T04:56:38.322512+0000","last_fullsized":"2024-09-14T04:56:38.322512+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":129,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:35.794122+0000","last_change":"2024-09-14T04:44:57.325109+0000","last_active":"2024-09-14T04:56:35.794122+0000","last_peered":"2024-09-14T04:56:35.794122+0000","last_clean":"2024-09-14T04:56:35.794122+0000","last_became_active":"2024-09-14T04:44:57.324791+0000","last_became_peered":"2024-09-14T04:44:57.324791+0000","last_unstale":"2024-09-14T04:56:35.794122+0000","last_undegraded":"2024-09-14T04:56:35.794122+0000","last_fullsized":"2024-09-14T04:56:35.794122+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:03.593275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":99,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:36.416005+0000","last_change":"2024-09-14T04:49:29.628435+0000","last_active":"2024-09-14T04:56:36.416005+0000","last_peered":"2024-09-14T04:56:36.416005+0000","last_clean":"2024-09-14T04:56:36.416005+0000","last_became_active":"2024-09-14T04:49:29.628081+0000","last_became_peered":"2024-09-14T04:49:29.628081+0000","last_unstale":"2024-09-14T04:56:36.416005+0000","last_undegraded":"2024-09-14T04:56:36.416005+0000","last_fullsized":"2024-09-14T04:56:36.416005+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:01.520334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":948,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-09-14T04:56:38.844116+0000","last_change":"2024-09-14T04:43:13.355281+0000","last_active":"2024-09-14T04:56:38.844116+0000","last_peered":"2024-09-14T04:56:38.844116+0000","last_clean":"2024-09-14T04:56:38.844116+0000","last_became_active":"2024-09-14T04:43:13.354967+0000","last_became_peered":"2024-09-14T04:43:13.354967+0000","last_unstale":"2024-09-14T04:56:38.844116+0000","last_undegraded":"2024-09-14T04:56:38.844116+0000","last_fullsized":"2024-09-14T04:56:38.844116+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:23:19.332075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1508,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:35.811628+0000","last_change":"2024-09-14T04:44:57.342918+0000","last_active":"2024-09-14T04:56:35.811628+0000","last_peered":"2024-09-14T04:56:35.811628+0000","last_clean":"2024-09-14T04:56:35.811628+0000","last_became_active":"2024-09-14T04:44:57.329550+0000","last_became_peered":"2024-09-14T04:44:57.329550+0000","last_unstale":"2024-09-14T04:56:35.811628+0000","last_undegraded":"2024-09-14T04:56:35.811628+0000","last_fullsized":"2024-09-14T04:56:35.811628+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:52:52.435893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":129,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:35.794397+0000","last_change":"2024-09-14T04:44:57.325537+0000","last_active":"2024-09-14T04:56:35.794397+0000","last_peered":"2024-09-14T04:56:35.794397+0000","last_clean":"2024-09-14T04:56:35.794397+0000","last_became_active":"2024-09-14T04:44:57.325057+0000","last_became_peered":"2024-09-14T04:44:57.325057+0000","last_unstale":"2024-09-14T04:56:35.794397+0000","last_undegraded":"2024-09-14T04:56:35.794397+0000","last_fullsized":"2024-09-14T04:56:35.794397+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:43:23.166708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1523,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:35.810169+0000","last_change":"2024-09-14T04:44:57.337033+0000","last_active":"2024-09-14T04:56:35.810169+0000","last_peered":"2024-09-14T04:56:35.810169+0000","last_clean":"2024-09-14T04:56:35.810169+0000","last_became_active":"2024-09-14T04:44:57.328437+0000","last_became_peered":"2024-09-14T04:44:57.328437+0000","last_unstale":"2024-09-14T04:56:35.810169+0000","last_undegraded":"2024-09-14T04:56:35.810169+0000","last_fullsized":"2024-09-14T04:56:35.810169+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.109086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":129,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:35.795575+0000","last_change":"2024-09-14T04:44:57.332243+0000","last_active":"2024-09-14T04:56:35.795575+0000","last_peered":"2024-09-14T04:56:35.795575+0000","last_clean":"2024-09-14T04:56:35.795575+0000","last_became_active":"2024-09-14T04:44:57.330413+0000","last_became_peered":"2024-09-14T04:44:57.330413+0000","last_unstale":"2024-09-14T04:56:35.795575+0000","last_undegraded":"2024-09-14T04:56:35.795575+0000","last_fullsized":"2024-09-14T04:56:35.795575+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:25:01.338549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":183,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-09-14T04:56:38.322525+0000","last_change":"2024-09-14T04:48:29.862929+0000","last_active":"2024-09-14T04:56:38.322525+0000","last_peered":"2024-09-14T04:56:38.322525+0000","last_clean":"2024-09-14T04:56:38.322525+0000","last_became_active":"2024-09-14T04:48:29.862581+0000","last_became_peered":"2024-09-14T04:48:29.862581+0000","last_unstale":"2024-09-14T04:56:38.322525+0000","last_undegraded":"2024-09-14T04:56:38.322525+0000","last_fullsized":"2024-09-14T04:56:38.322525+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:16:43.060088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2322,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-09-14T04:56:38.844483+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T04:56:38.844483+0000","last_peered":"2024-09-14T04:56:38.844483+0000","last_clean":"2024-09-14T04:56:38.844483+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T04:56:38.844483+0000","last_undegraded":"2024-09-14T04:56:38.844483+0000","last_fullsized":"2024-09-14T04:56:38.844483+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":129,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:35.794336+0000","last_change":"2024-09-14T04:44:57.325392+0000","last_active":"2024-09-14T04:56:35.794336+0000","last_peered":"2024-09-14T04:56:35.794336+0000","last_clean":"2024-09-14T04:56:35.794336+0000","last_became_active":"2024-09-14T04:44:57.324624+0000","last_became_peered":"2024-09-14T04:44:57.324624+0000","last_unstale":"2024-09-14T04:56:35.794336+0000","last_undegraded":"2024-09-14T04:56:35.794336+0000","last_fullsized":"2024-09-14T04:56:35.794336+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:33:18.810199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2291,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-09-14T04:56:38.581324+0000","last_change":"2024-09-14T04:44:57.329113+0000","last_active":"2024-09-14T04:56:38.581324+0000","last_peered":"2024-09-14T04:56:38.581324+0000","last_clean":"2024-09-14T04:56:38.581324+0000","last_became_active":"2024-09-14T04:44:57.328727+0000","last_became_peered":"2024-09-14T04:44:57.328727+0000","last_unstale":"2024-09-14T04:56:38.581324+0000","last_undegraded":"2024-09-14T04:56:38.581324+0000","last_fullsized":"2024-09-14T04:56:38.581324+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:34.939874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":2308,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-09-14T04:56:38.581117+0000","last_change":"2024-09-14T04:44:57.329277+0000","last_active":"2024-09-14T04:56:38.581117+0000","last_peered":"2024-09-14T04:56:38.581117+0000","last_clean":"2024-09-14T04:56:38.581117+0000","last_became_active":"2024-09-14T04:44:57.328804+0000","last_became_peered":"2024-09-14T04:44:57.328804+0000","last_unstale":"2024-09-14T04:56:38.581117+0000","last_undegraded":"2024-09-14T04:56:38.581117+0000","last_fullsized":"2024-09-14T04:56:38.581117+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:42:46.055240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2343,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-09-14T04:56:38.322866+0000","last_change":"2024-09-14T04:54:52.250695+0000","last_active":"2024-09-14T04:56:38.322866+0000","last_peered":"2024-09-14T04:56:38.322866+0000","last_clean":"2024-09-14T04:56:38.322866+0000","last_became_active":"2024-09-14T04:54:52.250301+0000","last_became_peered":"2024-09-14T04:54:52.250301+0000","last_unstale":"2024-09-14T04:56:38.322866+0000","last_undegraded":"2024-09-14T04:56:38.322866+0000","last_fullsized":"2024-09-14T04:56:38.322866+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:42.060905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2304,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-09-14T04:56:38.322221+0000","last_change":"2024-09-14T04:43:13.357092+0000","last_active":"2024-09-14T04:56:38.322221+0000","last_peered":"2024-09-14T04:56:38.322221+0000","last_clean":"2024-09-14T04:56:38.322221+0000","last_became_active":"2024-09-14T04:43:13.356694+0000","last_became_peered":"2024-09-14T04:43:13.356694+0000","last_unstale":"2024-09-14T04:56:38.322221+0000","last_undegraded":"2024-09-14T04:56:38.322221+0000","last_fullsized":"2024-09-14T04:56:38.322221+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:11.167574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1501,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:36.416007+0000","last_change":"2024-09-14T04:07:21.720797+0000","last_active":"2024-09-14T04:56:36.416007+0000","last_peered":"2024-09-14T04:56:36.416007+0000","last_clean":"2024-09-14T04:56:36.416007+0000","last_became_active":"2024-09-14T04:07:21.719331+0000","last_became_peered":"2024-09-14T04:07:21.719331+0000","last_unstale":"2024-09-14T04:56:36.416007+0000","last_undegraded":"2024-09-14T04:56:36.416007+0000","last_fullsized":"2024-09-14T04:56:36.416007+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:35.404023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":148,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:35.800086+0000","last_change":"2024-09-14T04:43:11.641514+0000","last_active":"2024-09-14T04:56:35.800086+0000","last_peered":"2024-09-14T04:56:35.800086+0000","last_clean":"2024-09-14T04:56:35.800086+0000","last_became_active":"2024-09-14T04:43:11.641151+0000","last_became_peered":"2024-09-14T04:43:11.641151+0000","last_unstale":"2024-09-14T04:56:35.800086+0000","last_undegraded":"2024-09-14T04:56:35.800086+0000","last_fullsized":"2024-09-14T04:56:35.800086+0000","mapping_epoch":1384,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:06:45.045430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2316,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-09-14T04:56:38.322674+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T04:56:38.322674+0000","last_peered":"2024-09-14T04:56:38.322674+0000","last_clean":"2024-09-14T04:56:38.322674+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T04:56:38.322674+0000","last_undegraded":"2024-09-14T04:56:38.322674+0000","last_fullsized":"2024-09-14T04:56:38.322674+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":729,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:36.744226+0000","last_change":"2024-09-14T04:49:29.618477+0000","last_active":"2024-09-14T04:56:36.744226+0000","last_peered":"2024-09-14T04:56:36.744226+0000","last_clean":"2024-09-14T04:56:36.744226+0000","last_became_active":"2024-09-14T04:49:29.618210+0000","last_became_peered":"2024-09-14T04:49:29.618210+0000","last_unstale":"2024-09-14T04:56:36.744226+0000","last_undegraded":"2024-09-14T04:56:36.744226+0000","last_fullsized":"2024-09-14T04:56:36.744226+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:13:23.895512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1507,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:35.812086+0000","last_change":"2024-09-14T04:42:49.938505+0000","last_active":"2024-09-14T04:56:35.812086+0000","last_peered":"2024-09-14T04:56:35.812086+0000","last_clean":"2024-09-14T04:56:35.812086+0000","last_became_active":"2024-09-14T04:42:49.938180+0000","last_became_peered":"2024-09-14T04:42:49.938180+0000","last_unstale":"2024-09-14T04:56:35.812086+0000","last_undegraded":"2024-09-14T04:56:35.812086+0000","last_fullsized":"2024-09-14T04:56:35.812086+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:05.929479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1526,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:36.745262+0000","last_change":"2024-09-14T04:44:57.346582+0000","last_active":"2024-09-14T04:56:36.745262+0000","last_peered":"2024-09-14T04:56:36.745262+0000","last_clean":"2024-09-14T04:56:36.745262+0000","last_became_active":"2024-09-14T04:44:57.330446+0000","last_became_peered":"2024-09-14T04:44:57.330446+0000","last_unstale":"2024-09-14T04:56:36.745262+0000","last_undegraded":"2024-09-14T04:56:36.745262+0000","last_fullsized":"2024-09-14T04:56:36.745262+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:29.238585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":257,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-09-14T04:56:38.580968+0000","last_change":"2024-09-14T04:44:14.480667+0000","last_active":"2024-09-14T04:56:38.580968+0000","last_peered":"2024-09-14T04:56:38.580968+0000","last_clean":"2024-09-14T04:56:38.580968+0000","last_became_active":"2024-09-14T04:44:14.480365+0000","last_became_peered":"2024-09-14T04:44:14.480365+0000","last_unstale":"2024-09-14T04:56:38.580968+0000","last_undegraded":"2024-09-14T04:56:38.580968+0000","last_fullsized":"2024-09-14T04:56:38.580968+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:00.611771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":140,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-09-14T04:56:38.320927+0000","last_change":"2024-09-14T04:44:57.341426+0000","last_active":"2024-09-14T04:56:38.320927+0000","last_peered":"2024-09-14T04:56:38.320927+0000","last_clean":"2024-09-14T04:56:38.320927+0000","last_became_active":"2024-09-14T04:44:57.328763+0000","last_became_peered":"2024-09-14T04:44:57.328763+0000","last_unstale":"2024-09-14T04:56:38.320927+0000","last_undegraded":"2024-09-14T04:56:38.320927+0000","last_fullsized":"2024-09-14T04:56:38.320927+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:34:40.066761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":130,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-09-14T04:56:38.320963+0000","last_change":"2024-09-14T04:44:57.331337+0000","last_active":"2024-09-14T04:56:38.320963+0000","last_peered":"2024-09-14T04:56:38.320963+0000","last_clean":"2024-09-14T04:56:38.320963+0000","last_became_active":"2024-09-14T04:44:57.329336+0000","last_became_peered":"2024-09-14T04:44:57.329336+0000","last_unstale":"2024-09-14T04:56:38.320963+0000","last_undegraded":"2024-09-14T04:56:38.320963+0000","last_fullsized":"2024-09-14T04:56:38.320963+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:52:35.980869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"349'149","reported_seq":718,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:35.810020+0000","last_change":"2024-09-14T04:49:30.085274+0000","last_active":"2024-09-14T04:56:35.810020+0000","last_peered":"2024-09-14T04:56:35.810020+0000","last_clean":"2024-09-14T04:56:35.810020+0000","last_became_active":"2024-09-14T04:49:29.623295+0000","last_became_peered":"2024-09-14T04:49:29.623295+0000","last_unstale":"2024-09-14T04:56:35.810020+0000","last_undegraded":"2024-09-14T04:56:35.810020+0000","last_fullsized":"2024-09-14T04:56:35.810020+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:49:30.085089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:49:30.085089+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:10.012963+0000","scrub_duration":0.0071928790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":130,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:35.794255+0000","last_change":"2024-09-14T04:44:57.329870+0000","last_active":"2024-09-14T04:56:35.794255+0000","last_peered":"2024-09-14T04:56:35.794255+0000","last_clean":"2024-09-14T04:56:35.794255+0000","last_became_active":"2024-09-14T04:44:57.329353+0000","last_became_peered":"2024-09-14T04:44:57.329353+0000","last_unstale":"2024-09-14T04:56:35.794255+0000","last_undegraded":"2024-09-14T04:56:35.794255+0000","last_fullsized":"2024-09-14T04:56:35.794255+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:46.432181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1522,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:36.415140+0000","last_change":"2024-09-14T04:54:52.242207+0000","last_active":"2024-09-14T04:56:36.415140+0000","last_peered":"2024-09-14T04:56:36.415140+0000","last_clean":"2024-09-14T04:56:36.415140+0000","last_became_active":"2024-09-14T04:54:52.241840+0000","last_became_peered":"2024-09-14T04:54:52.241840+0000","last_unstale":"2024-09-14T04:56:36.415140+0000","last_undegraded":"2024-09-14T04:56:36.415140+0000","last_fullsized":"2024-09-14T04:56:36.415140+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:37.899666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":2354,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-09-14T04:56:38.322779+0000","last_change":"2024-09-14T04:44:57.344614+0000","last_active":"2024-09-14T04:56:38.322779+0000","last_peered":"2024-09-14T04:56:38.322779+0000","last_clean":"2024-09-14T04:56:38.322779+0000","last_became_active":"2024-09-14T04:44:57.332256+0000","last_became_peered":"2024-09-14T04:44:57.332256+0000","last_unstale":"2024-09-14T04:56:38.322779+0000","last_undegraded":"2024-09-14T04:56:38.322779+0000","last_fullsized":"2024-09-14T04:56:38.322779+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:23.438417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"349'149","reported_seq":888,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:35.795640+0000","last_change":"2024-09-14T04:45:00.331854+0000","last_active":"2024-09-14T04:56:35.795640+0000","last_peered":"2024-09-14T04:56:35.795640+0000","last_clean":"2024-09-14T04:56:35.795640+0000","last_became_active":"2024-09-14T04:45:00.331460+0000","last_became_peered":"2024-09-14T04:45:00.331460+0000","last_unstale":"2024-09-14T04:56:35.795640+0000","last_undegraded":"2024-09-14T04:56:35.795640+0000","last_fullsized":"2024-09-14T04:56:35.795640+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.567535+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.567535+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:45.095729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":719,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:36.744151+0000","last_change":"2024-09-14T04:49:29.616831+0000","last_active":"2024-09-14T04:56:36.744151+0000","last_peered":"2024-09-14T04:56:36.744151+0000","last_clean":"2024-09-14T04:56:36.744151+0000","last_became_active":"2024-09-14T04:49:29.616546+0000","last_became_peered":"2024-09-14T04:49:29.616546+0000","last_unstale":"2024-09-14T04:56:36.744151+0000","last_undegraded":"2024-09-14T04:56:36.744151+0000","last_fullsized":"2024-09-14T04:56:36.744151+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:27.882929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":309,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-09-14T04:56:38.581435+0000","last_change":"2024-09-14T04:44:57.330746+0000","last_active":"2024-09-14T04:56:38.581435+0000","last_peered":"2024-09-14T04:56:38.581435+0000","last_clean":"2024-09-14T04:56:38.581435+0000","last_became_active":"2024-09-14T04:44:57.330327+0000","last_became_peered":"2024-09-14T04:44:57.330327+0000","last_unstale":"2024-09-14T04:56:38.581435+0000","last_undegraded":"2024-09-14T04:56:38.581435+0000","last_fullsized":"2024-09-14T04:56:38.581435+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:53.122771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":170,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:36.744920+0000","last_change":"2024-09-14T04:43:13.355296+0000","last_active":"2024-09-14T04:56:36.744920+0000","last_peered":"2024-09-14T04:56:36.744920+0000","last_clean":"2024-09-14T04:56:36.744920+0000","last_became_active":"2024-09-14T04:43:13.354950+0000","last_became_peered":"2024-09-14T04:43:13.354950+0000","last_unstale":"2024-09-14T04:56:36.744920+0000","last_undegraded":"2024-09-14T04:56:36.744920+0000","last_fullsized":"2024-09-14T04:56:36.744920+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:20:08.968818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"349'149","reported_seq":826,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:36.415856+0000","last_change":"2024-09-14T04:41:39.471393+0000","last_active":"2024-09-14T04:56:36.415856+0000","last_peered":"2024-09-14T04:56:36.415856+0000","last_clean":"2024-09-14T04:56:36.415856+0000","last_became_active":"2024-09-14T04:41:36.931768+0000","last_became_peered":"2024-09-14T04:41:36.931768+0000","last_unstale":"2024-09-14T04:56:36.415856+0000","last_undegraded":"2024-09-14T04:56:36.415856+0000","last_fullsized":"2024-09-14T04:56:36.415856+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.471248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.471248+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:56.210375+0000","scrub_duration":0.021758344999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2310,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-09-14T04:56:38.322639+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T04:56:38.322639+0000","last_peered":"2024-09-14T04:56:38.322639+0000","last_clean":"2024-09-14T04:56:38.322639+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T04:56:38.322639+0000","last_undegraded":"2024-09-14T04:56:38.322639+0000","last_fullsized":"2024-09-14T04:56:38.322639+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1537,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:36.745440+0000","last_change":"2024-09-14T04:44:57.331436+0000","last_active":"2024-09-14T04:56:36.745440+0000","last_peered":"2024-09-14T04:56:36.745440+0000","last_clean":"2024-09-14T04:56:36.745440+0000","last_became_active":"2024-09-14T04:44:57.330957+0000","last_became_peered":"2024-09-14T04:44:57.330957+0000","last_unstale":"2024-09-14T04:56:36.745440+0000","last_undegraded":"2024-09-14T04:56:36.745440+0000","last_fullsized":"2024-09-14T04:56:36.745440+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:09.516688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":859,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-09-14T04:56:38.844376+0000","last_change":"2024-09-14T04:49:49.460401+0000","last_active":"2024-09-14T04:56:38.844376+0000","last_peered":"2024-09-14T04:56:38.844376+0000","last_clean":"2024-09-14T04:56:38.844376+0000","last_became_active":"2024-09-14T04:49:49.458530+0000","last_became_peered":"2024-09-14T04:49:49.458530+0000","last_unstale":"2024-09-14T04:56:38.844376+0000","last_undegraded":"2024-09-14T04:56:38.844376+0000","last_fullsized":"2024-09-14T04:56:38.844376+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:11:29.143239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"349'149","reported_seq":860,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:36.416099+0000","last_change":"2024-09-14T04:54:22.147441+0000","last_active":"2024-09-14T04:56:36.416099+0000","last_peered":"2024-09-14T04:56:36.416099+0000","last_clean":"2024-09-14T04:56:36.416099+0000","last_became_active":"2024-09-14T04:54:22.147115+0000","last_became_peered":"2024-09-14T04:54:22.147115+0000","last_unstale":"2024-09-14T04:56:36.416099+0000","last_undegraded":"2024-09-14T04:56:36.416099+0000","last_fullsized":"2024-09-14T04:56:36.416099+0000","mapping_epoch":1492,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1493,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:38.479887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:38.479887+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:41:14.001781+0000","scrub_duration":0.0062728319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":264,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-09-14T04:56:38.581097+0000","last_change":"2024-09-14T04:44:57.329983+0000","last_active":"2024-09-14T04:56:38.581097+0000","last_peered":"2024-09-14T04:56:38.581097+0000","last_clean":"2024-09-14T04:56:38.581097+0000","last_became_active":"2024-09-14T04:44:57.329597+0000","last_became_peered":"2024-09-14T04:44:57.329597+0000","last_unstale":"2024-09-14T04:56:38.581097+0000","last_undegraded":"2024-09-14T04:56:38.581097+0000","last_fullsized":"2024-09-14T04:56:38.581097+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:45.718039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":803,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-09-14T04:56:38.322634+0000","last_change":"2024-09-14T04:49:49.443631+0000","last_active":"2024-09-14T04:56:38.322634+0000","last_peered":"2024-09-14T04:56:38.322634+0000","last_clean":"2024-09-14T04:56:38.322634+0000","last_became_active":"2024-09-14T04:49:49.443313+0000","last_became_peered":"2024-09-14T04:49:49.443313+0000","last_unstale":"2024-09-14T04:56:38.322634+0000","last_undegraded":"2024-09-14T04:56:38.322634+0000","last_fullsized":"2024-09-14T04:56:38.322634+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:06:54.562292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1509,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:35.811735+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T04:56:35.811735+0000","last_peered":"2024-09-14T04:56:35.811735+0000","last_clean":"2024-09-14T04:56:35.811735+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T04:56:35.811735+0000","last_undegraded":"2024-09-14T04:56:35.811735+0000","last_fullsized":"2024-09-14T04:56:35.811735+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"349'149","reported_seq":819,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:35.805064+0000","last_change":"2024-09-14T04:41:39.012977+0000","last_active":"2024-09-14T04:56:35.805064+0000","last_peered":"2024-09-14T04:56:35.805064+0000","last_clean":"2024-09-14T04:56:35.805064+0000","last_became_active":"2024-09-14T04:41:36.932552+0000","last_became_peered":"2024-09-14T04:41:36.932552+0000","last_unstale":"2024-09-14T04:56:35.805064+0000","last_undegraded":"2024-09-14T04:56:35.805064+0000","last_fullsized":"2024-09-14T04:56:35.805064+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.012879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.012879+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:16:40.080262+0000","scrub_duration":0.0051721620000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":895,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-09-14T04:56:38.844575+0000","last_change":"2024-09-14T04:49:49.459590+0000","last_active":"2024-09-14T04:56:38.844575+0000","last_peered":"2024-09-14T04:56:38.844575+0000","last_clean":"2024-09-14T04:56:38.844575+0000","last_became_active":"2024-09-14T04:49:49.456263+0000","last_became_peered":"2024-09-14T04:49:49.456263+0000","last_unstale":"2024-09-14T04:56:38.844575+0000","last_undegraded":"2024-09-14T04:56:38.844575+0000","last_fullsized":"2024-09-14T04:56:38.844575+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:20:44.666805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":130,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:35.795640+0000","last_change":"2024-09-14T04:44:57.329652+0000","last_active":"2024-09-14T04:56:35.795640+0000","last_peered":"2024-09-14T04:56:35.795640+0000","last_clean":"2024-09-14T04:56:35.795640+0000","last_became_active":"2024-09-14T04:44:57.328888+0000","last_became_peered":"2024-09-14T04:44:57.328888+0000","last_unstale":"2024-09-14T04:56:35.795640+0000","last_undegraded":"2024-09-14T04:56:35.795640+0000","last_fullsized":"2024-09-14T04:56:35.795640+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:10.170068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1508,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:35.806571+0000","last_change":"2024-09-14T04:06:46.875340+0000","last_active":"2024-09-14T04:56:35.806571+0000","last_peered":"2024-09-14T04:56:35.806571+0000","last_clean":"2024-09-14T04:56:35.806571+0000","last_became_active":"2024-09-14T04:06:46.874552+0000","last_became_peered":"2024-09-14T04:06:46.874552+0000","last_unstale":"2024-09-14T04:56:35.806571+0000","last_undegraded":"2024-09-14T04:56:35.806571+0000","last_fullsized":"2024-09-14T04:56:35.806571+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:52.257357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"349'149","reported_seq":721,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:36.744227+0000","last_change":"2024-09-14T04:54:55.279666+0000","last_active":"2024-09-14T04:56:36.744227+0000","last_peered":"2024-09-14T04:56:36.744227+0000","last_clean":"2024-09-14T04:56:36.744227+0000","last_became_active":"2024-09-14T04:54:55.279259+0000","last_became_peered":"2024-09-14T04:54:55.279259+0000","last_unstale":"2024-09-14T04:56:36.744227+0000","last_undegraded":"2024-09-14T04:56:36.744227+0000","last_fullsized":"2024-09-14T04:56:36.744227+0000","mapping_epoch":1506,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1507,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:54:03.996025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:54:03.996025+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:40:53.332369+0000","scrub_duration":54,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":257,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-09-14T04:56:38.581513+0000","last_change":"2024-09-14T04:44:14.489235+0000","last_active":"2024-09-14T04:56:38.581513+0000","last_peered":"2024-09-14T04:56:38.581513+0000","last_clean":"2024-09-14T04:56:38.581513+0000","last_became_active":"2024-09-14T04:44:14.488950+0000","last_became_peered":"2024-09-14T04:44:14.488950+0000","last_unstale":"2024-09-14T04:56:38.581513+0000","last_undegraded":"2024-09-14T04:56:38.581513+0000","last_fullsized":"2024-09-14T04:56:38.581513+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:54.441144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":813,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-09-14T04:56:38.843943+0000","last_change":"2024-09-14T04:49:49.456317+0000","last_active":"2024-09-14T04:56:38.843943+0000","last_peered":"2024-09-14T04:56:38.843943+0000","last_clean":"2024-09-14T04:56:38.843943+0000","last_became_active":"2024-09-14T04:49:49.456029+0000","last_became_peered":"2024-09-14T04:49:49.456029+0000","last_unstale":"2024-09-14T04:56:38.843943+0000","last_undegraded":"2024-09-14T04:56:38.843943+0000","last_fullsized":"2024-09-14T04:56:38.843943+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:46:05.319676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2330,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-09-14T04:56:38.844234+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T04:56:38.844234+0000","last_peered":"2024-09-14T04:56:38.844234+0000","last_clean":"2024-09-14T04:56:38.844234+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T04:56:38.844234+0000","last_undegraded":"2024-09-14T04:56:38.844234+0000","last_fullsized":"2024-09-14T04:56:38.844234+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"349'149","reported_seq":713,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:36.415569+0000","last_change":"2024-09-14T04:50:22.002798+0000","last_active":"2024-09-14T04:56:36.415569+0000","last_peered":"2024-09-14T04:56:36.415569+0000","last_clean":"2024-09-14T04:56:36.415569+0000","last_became_active":"2024-09-14T04:49:49.448278+0000","last_became_peered":"2024-09-14T04:49:49.448278+0000","last_unstale":"2024-09-14T04:56:36.415569+0000","last_undegraded":"2024-09-14T04:56:36.415569+0000","last_fullsized":"2024-09-14T04:56:36.415569+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:50:22.002568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:50:22.002568+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:01:57.489223+0000","scrub_duration":0.010888326,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2331,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-09-14T04:56:38.322934+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T04:56:38.322934+0000","last_peered":"2024-09-14T04:56:38.322934+0000","last_clean":"2024-09-14T04:56:38.322934+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T04:56:38.322934+0000","last_undegraded":"2024-09-14T04:56:38.322934+0000","last_fullsized":"2024-09-14T04:56:38.322934+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2352,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-09-14T04:56:38.322369+0000","last_change":"2024-09-14T04:54:19.447103+0000","last_active":"2024-09-14T04:56:38.322369+0000","last_peered":"2024-09-14T04:56:38.322369+0000","last_clean":"2024-09-14T04:56:38.322369+0000","last_became_active":"2024-09-14T04:54:19.446748+0000","last_became_peered":"2024-09-14T04:54:19.446748+0000","last_unstale":"2024-09-14T04:56:38.322369+0000","last_undegraded":"2024-09-14T04:56:38.322369+0000","last_fullsized":"2024-09-14T04:56:38.322369+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:00:55.369263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1538,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:36.744318+0000","last_change":"2024-09-14T04:42:49.939790+0000","last_active":"2024-09-14T04:56:36.744318+0000","last_peered":"2024-09-14T04:56:36.744318+0000","last_clean":"2024-09-14T04:56:36.744318+0000","last_became_active":"2024-09-14T04:42:49.939591+0000","last_became_peered":"2024-09-14T04:42:49.939591+0000","last_unstale":"2024-09-14T04:56:36.744318+0000","last_undegraded":"2024-09-14T04:56:36.744318+0000","last_fullsized":"2024-09-14T04:56:36.744318+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:23:16.790593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1501,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:36.415826+0000","last_change":"2024-09-14T04:44:57.326008+0000","last_active":"2024-09-14T04:56:36.415826+0000","last_peered":"2024-09-14T04:56:36.415826+0000","last_clean":"2024-09-14T04:56:36.415826+0000","last_became_active":"2024-09-14T04:44:57.325617+0000","last_became_peered":"2024-09-14T04:44:57.325617+0000","last_unstale":"2024-09-14T04:56:36.415826+0000","last_undegraded":"2024-09-14T04:56:36.415826+0000","last_fullsized":"2024-09-14T04:56:36.415826+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:15.637508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":790,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:35.806647+0000","last_change":"2024-09-14T04:43:13.354846+0000","last_active":"2024-09-14T04:56:35.806647+0000","last_peered":"2024-09-14T04:56:35.806647+0000","last_clean":"2024-09-14T04:56:35.806647+0000","last_became_active":"2024-09-14T04:43:13.354515+0000","last_became_peered":"2024-09-14T04:43:13.354515+0000","last_unstale":"2024-09-14T04:56:35.806647+0000","last_undegraded":"2024-09-14T04:56:35.806647+0000","last_fullsized":"2024-09-14T04:56:35.806647+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:17.889341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1507,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:35.806057+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T04:56:35.806057+0000","last_peered":"2024-09-14T04:56:35.806057+0000","last_clean":"2024-09-14T04:56:35.806057+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T04:56:35.806057+0000","last_undegraded":"2024-09-14T04:56:35.806057+0000","last_fullsized":"2024-09-14T04:56:35.806057+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":1844,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:36.744143+0000","last_change":"2024-09-14T04:41:30.640651+0000","last_active":"2024-09-14T04:56:36.744143+0000","last_peered":"2024-09-14T04:56:36.744143+0000","last_clean":"2024-09-14T04:56:36.744143+0000","last_became_active":"2024-09-14T04:41:29.532466+0000","last_became_peered":"2024-09-14T04:41:29.532466+0000","last_unstale":"2024-09-14T04:56:36.744143+0000","last_undegraded":"2024-09-14T04:56:36.744143+0000","last_fullsized":"2024-09-14T04:56:36.744143+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":0,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:30.640520+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:30.640520+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:27.776724+0000","scrub_duration":0.0028653139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"349'149","reported_seq":818,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:35.806498+0000","last_change":"2024-09-14T04:41:39.978080+0000","last_active":"2024-09-14T04:56:35.806498+0000","last_peered":"2024-09-14T04:56:35.806498+0000","last_clean":"2024-09-14T04:56:35.806498+0000","last_became_active":"2024-09-14T04:41:36.929279+0000","last_became_peered":"2024-09-14T04:41:36.929279+0000","last_unstale":"2024-09-14T04:56:35.806498+0000","last_undegraded":"2024-09-14T04:56:35.806498+0000","last_fullsized":"2024-09-14T04:56:35.806498+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.977999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.977999+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:44:06.383827+0000","scrub_duration":0.00192122,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":191,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-09-14T04:56:38.844563+0000","last_change":"2024-09-14T04:48:11.040133+0000","last_active":"2024-09-14T04:56:38.844563+0000","last_peered":"2024-09-14T04:56:38.844563+0000","last_clean":"2024-09-14T04:56:38.844563+0000","last_became_active":"2024-09-14T04:48:11.039787+0000","last_became_peered":"2024-09-14T04:48:11.039787+0000","last_unstale":"2024-09-14T04:56:38.844563+0000","last_undegraded":"2024-09-14T04:56:38.844563+0000","last_fullsized":"2024-09-14T04:56:38.844563+0000","mapping_epoch":1428,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:53.520672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2178,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-09-14T04:56:38.581033+0000","last_change":"2024-09-14T04:49:29.616443+0000","last_active":"2024-09-14T04:56:38.581033+0000","last_peered":"2024-09-14T04:56:38.581033+0000","last_clean":"2024-09-14T04:56:38.581033+0000","last_became_active":"2024-09-14T04:49:29.616066+0000","last_became_peered":"2024-09-14T04:49:29.616066+0000","last_unstale":"2024-09-14T04:56:38.581033+0000","last_undegraded":"2024-09-14T04:56:38.581033+0000","last_fullsized":"2024-09-14T04:56:38.581033+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:09:15.897957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":130,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:35.795325+0000","last_change":"2024-09-14T04:44:57.326724+0000","last_active":"2024-09-14T04:56:35.795325+0000","last_peered":"2024-09-14T04:56:35.795325+0000","last_clean":"2024-09-14T04:56:35.795325+0000","last_became_active":"2024-09-14T04:44:57.326191+0000","last_became_peered":"2024-09-14T04:44:57.326191+0000","last_unstale":"2024-09-14T04:56:35.795325+0000","last_undegraded":"2024-09-14T04:56:35.795325+0000","last_fullsized":"2024-09-14T04:56:35.795325+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:46.060269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":130,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:35.794165+0000","last_change":"2024-09-14T04:44:57.328528+0000","last_active":"2024-09-14T04:56:35.794165+0000","last_peered":"2024-09-14T04:56:35.794165+0000","last_clean":"2024-09-14T04:56:35.794165+0000","last_became_active":"2024-09-14T04:44:57.327739+0000","last_became_peered":"2024-09-14T04:44:57.327739+0000","last_unstale":"2024-09-14T04:56:35.794165+0000","last_undegraded":"2024-09-14T04:56:35.794165+0000","last_fullsized":"2024-09-14T04:56:35.794165+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:04:58.294690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"349'149","reported_seq":827,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:35.800012+0000","last_change":"2024-09-14T04:55:59.732757+0000","last_active":"2024-09-14T04:56:35.800012+0000","last_peered":"2024-09-14T04:56:35.800012+0000","last_clean":"2024-09-14T04:56:35.800012+0000","last_became_active":"2024-09-14T04:55:59.732364+0000","last_became_peered":"2024-09-14T04:55:59.732364+0000","last_unstale":"2024-09-14T04:56:35.800012+0000","last_undegraded":"2024-09-14T04:56:35.800012+0000","last_fullsized":"2024-09-14T04:56:35.800012+0000","mapping_epoch":1512,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1513,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:56.738636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:56.738636+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:59:11.313765+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1313440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":130,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:35.795511+0000","last_change":"2024-09-14T04:44:57.323436+0000","last_active":"2024-09-14T04:56:35.795511+0000","last_peered":"2024-09-14T04:56:35.795511+0000","last_clean":"2024-09-14T04:56:35.795511+0000","last_became_active":"2024-09-14T04:44:57.323022+0000","last_became_peered":"2024-09-14T04:44:57.323022+0000","last_unstale":"2024-09-14T04:56:35.795511+0000","last_undegraded":"2024-09-14T04:56:35.795511+0000","last_fullsized":"2024-09-14T04:56:35.795511+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:22.775261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1536,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:36.415934+0000","last_change":"2024-09-14T04:44:57.322357+0000","last_active":"2024-09-14T04:56:36.415934+0000","last_peered":"2024-09-14T04:56:36.415934+0000","last_clean":"2024-09-14T04:56:36.415934+0000","last_became_active":"2024-09-14T04:44:57.321990+0000","last_became_peered":"2024-09-14T04:44:57.321990+0000","last_unstale":"2024-09-14T04:56:36.415934+0000","last_undegraded":"2024-09-14T04:56:36.415934+0000","last_fullsized":"2024-09-14T04:56:36.415934+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:09.658008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"349'149","reported_seq":1861,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-09-14T04:56:38.844155+0000","last_change":"2024-09-14T04:41:53.858076+0000","last_active":"2024-09-14T04:56:38.844155+0000","last_peered":"2024-09-14T04:56:38.844155+0000","last_clean":"2024-09-14T04:56:38.844155+0000","last_became_active":"2024-09-14T04:41:36.931585+0000","last_became_peered":"2024-09-14T04:41:36.931585+0000","last_unstale":"2024-09-14T04:56:38.844155+0000","last_undegraded":"2024-09-14T04:56:38.844155+0000","last_fullsized":"2024-09-14T04:56:38.844155+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:53.857924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:53.857924+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:10:20.658986+0000","scrub_duration":0.0022355919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":130,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:35.794653+0000","last_change":"2024-09-14T04:44:57.329198+0000","last_active":"2024-09-14T04:56:35.794653+0000","last_peered":"2024-09-14T04:56:35.794653+0000","last_clean":"2024-09-14T04:56:35.794653+0000","last_became_active":"2024-09-14T04:44:57.327754+0000","last_became_peered":"2024-09-14T04:44:57.327754+0000","last_unstale":"2024-09-14T04:56:35.794653+0000","last_undegraded":"2024-09-14T04:56:35.794653+0000","last_fullsized":"2024-09-14T04:56:35.794653+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:10:27.944476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":315,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-09-14T04:56:38.322752+0000","last_change":"2024-09-14T04:44:57.348687+0000","last_active":"2024-09-14T04:56:38.322752+0000","last_peered":"2024-09-14T04:56:38.322752+0000","last_clean":"2024-09-14T04:56:38.322752+0000","last_became_active":"2024-09-14T04:44:57.332845+0000","last_became_peered":"2024-09-14T04:44:57.332845+0000","last_unstale":"2024-09-14T04:56:38.322752+0000","last_undegraded":"2024-09-14T04:56:38.322752+0000","last_fullsized":"2024-09-14T04:56:38.322752+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:12:18.762737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2337,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-09-14T04:56:38.322530+0000","last_change":"2024-09-14T04:44:57.333468+0000","last_active":"2024-09-14T04:56:38.322530+0000","last_peered":"2024-09-14T04:56:38.322530+0000","last_clean":"2024-09-14T04:56:38.322530+0000","last_became_active":"2024-09-14T04:44:57.332626+0000","last_became_peered":"2024-09-14T04:44:57.332626+0000","last_unstale":"2024-09-14T04:56:38.322530+0000","last_undegraded":"2024-09-14T04:56:38.322530+0000","last_fullsized":"2024-09-14T04:56:38.322530+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:57:26.366438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1492,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:35.810234+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T04:56:35.810234+0000","last_peered":"2024-09-14T04:56:35.810234+0000","last_clean":"2024-09-14T04:56:35.810234+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T04:56:35.810234+0000","last_undegraded":"2024-09-14T04:56:35.810234+0000","last_fullsized":"2024-09-14T04:56:35.810234+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1491,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:36.415651+0000","last_change":"2024-09-14T04:06:46.869868+0000","last_active":"2024-09-14T04:56:36.415651+0000","last_peered":"2024-09-14T04:56:36.415651+0000","last_clean":"2024-09-14T04:56:36.415651+0000","last_became_active":"2024-09-14T04:06:46.869126+0000","last_became_peered":"2024-09-14T04:06:46.869126+0000","last_unstale":"2024-09-14T04:56:36.415651+0000","last_undegraded":"2024-09-14T04:56:36.415651+0000","last_fullsized":"2024-09-14T04:56:36.415651+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:41.150138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1475,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:36.745376+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T04:56:36.745376+0000","last_peered":"2024-09-14T04:56:36.745376+0000","last_clean":"2024-09-14T04:56:36.745376+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T04:56:36.745376+0000","last_undegraded":"2024-09-14T04:56:36.745376+0000","last_fullsized":"2024-09-14T04:56:36.745376+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":130,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:35.794313+0000","last_change":"2024-09-14T04:44:57.332112+0000","last_active":"2024-09-14T04:56:35.794313+0000","last_peered":"2024-09-14T04:56:35.794313+0000","last_clean":"2024-09-14T04:56:35.794313+0000","last_became_active":"2024-09-14T04:44:57.331575+0000","last_became_peered":"2024-09-14T04:44:57.331575+0000","last_unstale":"2024-09-14T04:56:35.794313+0000","last_undegraded":"2024-09-14T04:56:35.794313+0000","last_fullsized":"2024-09-14T04:56:35.794313+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:50:00.391283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1523,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:36.745512+0000","last_change":"2024-09-14T04:44:57.351460+0000","last_active":"2024-09-14T04:56:36.745512+0000","last_peered":"2024-09-14T04:56:36.745512+0000","last_clean":"2024-09-14T04:56:36.745512+0000","last_became_active":"2024-09-14T04:44:57.334662+0000","last_became_peered":"2024-09-14T04:44:57.334662+0000","last_unstale":"2024-09-14T04:56:36.745512+0000","last_undegraded":"2024-09-14T04:56:36.745512+0000","last_fullsized":"2024-09-14T04:56:36.745512+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:02.399062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1477,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:35.800182+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T04:56:35.800182+0000","last_peered":"2024-09-14T04:56:35.800182+0000","last_clean":"2024-09-14T04:56:35.800182+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T04:56:35.800182+0000","last_undegraded":"2024-09-14T04:56:35.800182+0000","last_fullsized":"2024-09-14T04:56:35.800182+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1535,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:36.744813+0000","last_change":"2024-09-14T04:44:57.349690+0000","last_active":"2024-09-14T04:56:36.744813+0000","last_peered":"2024-09-14T04:56:36.744813+0000","last_clean":"2024-09-14T04:56:36.744813+0000","last_became_active":"2024-09-14T04:44:57.333579+0000","last_became_peered":"2024-09-14T04:44:57.333579+0000","last_unstale":"2024-09-14T04:56:36.744813+0000","last_undegraded":"2024-09-14T04:56:36.744813+0000","last_fullsized":"2024-09-14T04:56:36.744813+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:17:55.265141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1612,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:36.416240+0000","last_change":"2024-09-14T04:54:35.694089+0000","last_active":"2024-09-14T04:56:36.416240+0000","last_peered":"2024-09-14T04:56:36.416240+0000","last_clean":"2024-09-14T04:56:36.416240+0000","last_became_active":"2024-09-14T04:54:35.663133+0000","last_became_peered":"2024-09-14T04:54:35.663133+0000","last_unstale":"2024-09-14T04:56:36.416240+0000","last_undegraded":"2024-09-14T04:56:36.416240+0000","last_fullsized":"2024-09-14T04:56:36.416240+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:17:19.947280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":24,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:36.744143+0000","last_change":"2024-09-14T04:54:52.248439+0000","last_active":"2024-09-14T04:56:36.744143+0000","last_peered":"2024-09-14T04:56:36.744143+0000","last_clean":"2024-09-14T04:56:36.744143+0000","last_became_active":"2024-09-14T04:54:52.248235+0000","last_became_peered":"2024-09-14T04:54:52.248235+0000","last_unstale":"2024-09-14T04:56:36.744143+0000","last_undegraded":"2024-09-14T04:56:36.744143+0000","last_fullsized":"2024-09-14T04:56:36.744143+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:22:09.548139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1497,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:36.744352+0000","last_change":"2024-09-14T04:44:57.332511+0000","last_active":"2024-09-14T04:56:36.744352+0000","last_peered":"2024-09-14T04:56:36.744352+0000","last_clean":"2024-09-14T04:56:36.744352+0000","last_became_active":"2024-09-14T04:44:57.332166+0000","last_became_peered":"2024-09-14T04:44:57.332166+0000","last_unstale":"2024-09-14T04:56:36.744352+0000","last_undegraded":"2024-09-14T04:56:36.744352+0000","last_fullsized":"2024-09-14T04:56:36.744352+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:53.984639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2331,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-09-14T04:56:38.322730+0000","last_change":"2024-09-14T04:54:52.250655+0000","last_active":"2024-09-14T04:56:38.322730+0000","last_peered":"2024-09-14T04:56:38.322730+0000","last_clean":"2024-09-14T04:56:38.322730+0000","last_became_active":"2024-09-14T04:54:52.250317+0000","last_became_peered":"2024-09-14T04:54:52.250317+0000","last_unstale":"2024-09-14T04:56:38.322730+0000","last_undegraded":"2024-09-14T04:56:38.322730+0000","last_fullsized":"2024-09-14T04:56:38.322730+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:12.985406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2316,"reported_epoch":1517,"state":"active+clean","last_fresh":"2024-09-14T04:56:38.322522+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T04:56:38.322522+0000","last_peered":"2024-09-14T04:56:38.322522+0000","last_clean":"2024-09-14T04:56:38.322522+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T04:56:38.322522+0000","last_undegraded":"2024-09-14T04:56:38.322522+0000","last_fullsized":"2024-09-14T04:56:38.322522+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1489,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:35.811688+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T04:56:35.811688+0000","last_peered":"2024-09-14T04:56:35.811688+0000","last_clean":"2024-09-14T04:56:35.811688+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T04:56:35.811688+0000","last_undegraded":"2024-09-14T04:56:35.811688+0000","last_fullsized":"2024-09-14T04:56:35.811688+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1526,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:36.744538+0000","last_change":"2024-09-14T04:44:57.339993+0000","last_active":"2024-09-14T04:56:36.744538+0000","last_peered":"2024-09-14T04:56:36.744538+0000","last_clean":"2024-09-14T04:56:36.744538+0000","last_became_active":"2024-09-14T04:44:57.330472+0000","last_became_peered":"2024-09-14T04:44:57.330472+0000","last_unstale":"2024-09-14T04:56:36.744538+0000","last_undegraded":"2024-09-14T04:56:36.744538+0000","last_fullsized":"2024-09-14T04:56:36.744538+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:19:27.046988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2073,"reported_epoch":1516,"state":"active+clean","last_fresh":"2024-09-14T04:56:36.744421+0000","last_change":"2024-09-14T04:49:49.449687+0000","last_active":"2024-09-14T04:56:36.744421+0000","last_peered":"2024-09-14T04:56:36.744421+0000","last_clean":"2024-09-14T04:56:36.744421+0000","last_became_active":"2024-09-14T04:49:49.449144+0000","last_became_peered":"2024-09-14T04:49:49.449144+0000","last_unstale":"2024-09-14T04:56:36.744421+0000","last_undegraded":"2024-09-14T04:56:36.744421+0000","last_fullsized":"2024-09-14T04:56:36.744421+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:59:06.779730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1313440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":5070,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4250,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223202069,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":901956,"kb_used_data":185084,"kb_used_omap":1,"kb_used_meta":716862,"kb_avail":92843196,"statfs":{"total":95995035648,"available":95071432704,"internally_reserved":0,"allocated":189526016,"data_stored":184262692,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":734067147},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463705,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":670656,"kb_used_data":12020,"kb_used_omap":1,"kb_used_meta":658622,"kb_avail":93074496,"statfs":{"total":95995035648,"available":95308283904,"internally_reserved":0,"allocated":12308480,"data_stored":7053221,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":674429383},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725341,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":765564,"kb_used_data":12592,"kb_used_omap":1,"kb_used_meta":752958,"kb_avail":92979588,"statfs":{"total":95995035648,"available":95211098112,"internally_reserved":0,"allocated":12894208,"data_stored":7639415,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":771029446},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":10,"apply_latency_ms":10,"commit_latency_ns":10000000,"apply_latency_ns":10000000},"alerts":[]},{"osd":4,"up_from":1392,"seq":5978594476188,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":446788,"kb_used_data":149052,"kb_used_omap":117,"kb_used_meta":297610,"kb_avail":93298364,"statfs":{"total":95995035648,"available":95537524736,"internally_reserved":0,"allocated":152629248,"data_stored":147366476,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120638,"internal_metadata":304752834},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772075083,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":770744,"kb_used_data":12588,"kb_used_omap":71,"kb_used_meta":758072,"kb_avail":92974408,"statfs":{"total":95995035648,"available":95205793792,"internally_reserved":0,"allocated":12890112,"data_stored":7639468,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":776266678},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707304018,"num_pgs":24,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":521912,"kb_used_data":12268,"kb_used_omap":68,"kb_used_meta":509563,"kb_avail":93223240,"statfs":{"total":95995035648,"available":95460597760,"internally_reserved":0,"allocated":12562432,"data_stored":7311692,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":521792720},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":14,"apply_latency_ms":14,"commit_latency_ns":14000000,"apply_latency_ns":14000000},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872729177,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":585460,"kb_used_data":12004,"kb_used_omap":67,"kb_used_meta":573372,"kb_avail":93159692,"statfs":{"total":95995035648,"available":95395524608,"internally_reserved":0,"allocated":12292096,"data_stored":7048407,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":587133285},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056228,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":983728,"kb_used_data":352168,"kb_used_omap":74,"kb_used_meta":631477,"kb_avail":92761424,"statfs":{"total":95995035648,"available":94987698176,"internally_reserved":0,"allocated":360620032,"data_stored":355372832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":646633141},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1901,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:56:46.412 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-14T04:56:46.413 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:56:46.483 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:46.505 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:46 smithi039 ceph-mon[106470]: from='client.36291 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:56:46.506 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:46 smithi039 ceph-mon[111235]: from='client.36291 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:56:46.507 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:46.508 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:46.508 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:46.609 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:46.634 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:46.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:46.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:46.736 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:46.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:46.765 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:46.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:46.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:46 smithi129 ceph-mon[101316]: from='client.36291 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:56:46.866 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:46.889 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:46.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:46.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:46.991 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:47.018 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:47.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:47.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:47.120 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:47.145 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:47.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:47.146 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:47.246 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:47.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:47.272 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:47.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:47.373 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:47.399 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:47.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:47.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:47.500 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:47.526 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:47.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:47.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:47.578 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:47 smithi039 ceph-mon[106470]: pgmap v2901: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:47.578 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:47 smithi039 ceph-mon[111235]: pgmap v2901: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:47.628 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:47.652 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:47.653 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:47.653 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:47.754 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:47.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:47 smithi129 ceph-mon[101316]: pgmap v2901: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:47.778 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:47.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:47.780 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:47.881 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:47.907 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:47.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:47.908 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:48.009 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:48.034 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:48.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:48.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:48.136 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:48.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:48.159 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:48.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:48.260 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:48.286 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:48.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:48.287 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:48.388 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:48.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:48.415 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:48.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:48.418 INFO:teuthology.orchestra.run.smithi039.stderr:pool 'unique_pool_4' created 2024-09-14T04:56:48.516 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:48.539 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:48.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:48.540 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:48.641 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:48.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:48.664 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:48.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:48.765 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:48.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:48 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1184235088' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-14T04:56:48.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:48.789 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:48.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:48.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:48 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1184235088' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-14T04:56:48.791 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:56:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:56:48] "GET /metrics HTTP/1.1" 200 34311 "" "Prometheus/2.51.0" 2024-09-14T04:56:48.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:48 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1184235088' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16} : dispatch 2024-09-14T04:56:48.891 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:48.916 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:48.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:48.917 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:49.018 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:49.043 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:49.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:49.044 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:49.144 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:49.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:49.169 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:49.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:49.270 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:49.295 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:49.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:49.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:49.397 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:49.420 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:49.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:49.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:49.522 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:49.544 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:49.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:49.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:49.646 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:49.668 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:49.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:49.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:49.709 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:49 smithi039 ceph-mon[106470]: pgmap v2902: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:49.709 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:49 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1184235088' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-09-14T04:56:49.709 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:49 smithi039 ceph-mon[106470]: osdmap e1518: 8 total, 8 up, 8 in 2024-09-14T04:56:49.710 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:49 smithi039 ceph-mon[111235]: pgmap v2902: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:49.710 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:49 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1184235088' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-09-14T04:56:49.710 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:49 smithi039 ceph-mon[111235]: osdmap e1518: 8 total, 8 up, 8 in 2024-09-14T04:56:49.722 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:56:49.722 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [], 2024-09-14T04:56:49.722 INFO:teuthology.orchestra.run.smithi039.stdout: "complaint_time": 30, 2024-09-14T04:56:49.722 INFO:teuthology.orchestra.run.smithi039.stdout: "num_blocked_ops": 0 2024-09-14T04:56:49.723 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:56:49.770 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:49.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:49 smithi129 ceph-mon[101316]: pgmap v2902: 83 pgs: 83 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:49.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:49 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1184235088' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2024-09-14T04:56:49.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:49 smithi129 ceph-mon[101316]: osdmap e1518: 8 total, 8 up, 8 in 2024-09-14T04:56:49.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:49.794 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:49.794 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:49.895 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:49.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:49.918 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:49.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:50.019 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:50.041 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:50.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:50.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:50.143 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:50.166 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:50.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:50.167 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:50.267 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:50.291 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:50.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:50.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:50.393 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:50.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:50.416 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:50.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:50.517 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:50.539 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:50 smithi039 ceph-mon[106470]: osdmap e1519: 8 total, 8 up, 8 in 2024-09-14T04:56:50.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:50 smithi039 ceph-mon[111235]: osdmap e1519: 8 total, 8 up, 8 in 2024-09-14T04:56:50.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:50.542 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:50.542 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:50.643 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:50.666 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:50.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:50.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:50.767 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:50.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:50 smithi129 ceph-mon[101316]: osdmap e1519: 8 total, 8 up, 8 in 2024-09-14T04:56:50.790 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:50.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:50.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:50.892 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:50.916 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:50.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:50.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:50.919 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2024-09-14T04:56:51.019 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:51.042 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:51.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:51.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:51.144 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:51.166 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:51.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:51.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:51.268 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:51.291 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:51.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:51.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:51.393 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:51.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:51.416 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:51.417 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:51.517 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:51.541 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:51.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:51.542 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:51.616 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:56:51.617 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:56:51.617 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:56:51.617 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:56:51.617 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:56:51.617 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:56:51.617 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:56:51.617 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:56:51.617 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:56:51.617 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:56:51.617 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:56:51.618 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:56:51.618 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:56:51.618 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:56:51.618 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:56:51.618 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:56:51.643 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:51.665 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:51.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:51.666 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:51.766 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:51.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:51 smithi129 ceph-mon[101316]: pgmap v2905: 99 pgs: 2 creating+activating, 6 creating+peering, 6 unknown, 85 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:51.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:51 smithi129 ceph-mon[101316]: osdmap e1520: 8 total, 8 up, 8 in 2024-09-14T04:56:51.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:51 smithi129 ceph-mon[101316]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:56:51.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:51 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1382654102' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:56:51.789 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:51 smithi039 ceph-mon[106470]: pgmap v2905: 99 pgs: 2 creating+activating, 6 creating+peering, 6 unknown, 85 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:51 smithi039 ceph-mon[106470]: osdmap e1520: 8 total, 8 up, 8 in 2024-09-14T04:56:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:51 smithi039 ceph-mon[106470]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:56:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:51 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1382654102' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:56:51.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:51 smithi039 ceph-mon[111235]: pgmap v2905: 99 pgs: 2 creating+activating, 6 creating+peering, 6 unknown, 85 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:51.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:51 smithi039 ceph-mon[111235]: osdmap e1520: 8 total, 8 up, 8 in 2024-09-14T04:56:51.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:51 smithi039 ceph-mon[111235]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T04:56:51.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:51 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1382654102' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:56:51.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:51.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:51.864 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:56:51.892 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:51.915 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:51.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:51.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:52.017 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:52.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:52.041 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:52.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:52.142 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:52.167 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:52.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:52.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:52.269 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:52.294 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:52.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:52.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:52.396 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:52.420 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:52.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:52.421 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:52.479 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is set 2024-09-14T04:56:52.522 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:52.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:52.547 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:52.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:52.648 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:52.672 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:52.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:52.673 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:52.678 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-14T04:56:52.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:52 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2786057103' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:56:52.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:52 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:56:52.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:52 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:56:52.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:56:52.774 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:52.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:52 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2786057103' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:56:52.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:52 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:56:52.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:52 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:56:52.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:56:52.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:52 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2786057103' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:56:52.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:52 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:56:52.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:52 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:56:52.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:56:52.797 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:52.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:52.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:52.899 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:52.924 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:52.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:52.926 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:53.026 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:53.052 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:53.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:53.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:53.154 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:53.176 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:53.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:53.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:53.278 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:53.304 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:53.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:53.306 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:53.408 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:53.432 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:53.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:53.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:53.534 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:53.557 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:53.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:53.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:53.660 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:53.682 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:53.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:53.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:53.757 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:53 smithi039 ceph-mon[106470]: pgmap v2907: 99 pgs: 2 creating+activating, 6 creating+peering, 1 unknown, 90 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:53.758 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:53 smithi039 ceph-mon[106470]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:56:53.758 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:53 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:56:53.758 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:53 smithi039 ceph-mon[106470]: osdmap e1521: 8 total, 8 up, 8 in 2024-09-14T04:56:53.758 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:53 smithi039 ceph-mon[111235]: pgmap v2907: 99 pgs: 2 creating+activating, 6 creating+peering, 1 unknown, 90 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:53.758 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:53 smithi039 ceph-mon[111235]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:56:53.758 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:53 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:56:53.758 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:53 smithi039 ceph-mon[111235]: osdmap e1521: 8 total, 8 up, 8 in 2024-09-14T04:56:53.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:53 smithi129 ceph-mon[101316]: pgmap v2907: 99 pgs: 2 creating+activating, 6 creating+peering, 1 unknown, 90 active+clean; 582 KiB data, 5.4 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:53.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:53 smithi129 ceph-mon[101316]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:56:53.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:53 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:56:53.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:53 smithi129 ceph-mon[101316]: osdmap e1521: 8 total, 8 up, 8 in 2024-09-14T04:56:53.784 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:53.809 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:53.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:53.810 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:53.911 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:53.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:53.935 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:53.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:54.036 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:54.061 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:54.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:54.063 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:54.163 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:54.185 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:54.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:54.186 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:54.287 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:54.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:54.315 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:54.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:54.416 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:54.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:54.440 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:54.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:54.541 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:54.566 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:54.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:54.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:54.668 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:54.673 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:56:54.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:54.693 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:54.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:54.794 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:54.820 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:54.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:54.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:54.922 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:54.948 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:54.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:54.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:55.050 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:55.072 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:55.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:55.073 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:55.174 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:55.196 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:55.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:55.197 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:55.298 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:55.323 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:55.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:55.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:55.425 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:55.449 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:55.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:55.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:55.551 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:55.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:55.574 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:55.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:55.624 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:56:55.676 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:55.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:55.702 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:55.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:55.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:55 smithi129 ceph-mon[101316]: pgmap v2909: 99 pgs: 2 creating+activating, 97 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:55.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:55 smithi039 ceph-mon[106470]: pgmap v2909: 99 pgs: 2 creating+activating, 97 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:55.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:55 smithi039 ceph-mon[111235]: pgmap v2909: 99 pgs: 2 creating+activating, 97 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:55.803 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:55.825 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:55.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:55.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:55.929 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:55.955 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:55.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:55.956 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:56.056 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:56.082 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:56.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:56.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:56.089 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:56:56.183 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:56:56.208 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:56.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:56.209 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:56:56.310 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:56.333 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:56.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:56.335 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:56.435 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:56.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:56.460 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:56.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:56.561 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:56.587 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:56.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:56.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:56.689 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:56.712 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:56.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:56.714 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:56.814 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:56.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:56.837 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:56.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:56.938 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:56.961 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:56.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:56.962 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:57.063 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:57.089 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:57.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:57.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:57.191 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:57.215 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:57.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:57.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:57.316 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:57.342 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:57.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:57.343 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:57.444 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:57.467 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:57.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:57.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:57.496 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:56:57.497 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:56:57.498 INFO:teuthology.orchestra.run.smithi039.stderr:enabled application 'rados' on pool 'unique_pool_4' 2024-09-14T04:56:57.569 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:56:57.591 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:57.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:57.592 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:56:57.693 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:57.717 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:57.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:57.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:57.741 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:57 smithi039 ceph-mon[106470]: pgmap v2910: 99 pgs: 99 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:57.741 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:57 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1086523265' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:56:57.742 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:57 smithi039 ceph-mon[111235]: pgmap v2910: 99 pgs: 99 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:57.742 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:57 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1086523265' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:56:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:57 smithi129 ceph-mon[101316]: pgmap v2910: 99 pgs: 99 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:57 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1086523265' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T04:56:57.819 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:57.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:57.842 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:57.843 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:57.944 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:57.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:57.967 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:57.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:58.069 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:58.090 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:58.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:58.092 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:58.192 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:58.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:58.216 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:58.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:58.317 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:56:58.340 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:58.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:58.341 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:56:58.442 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:58.465 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:58.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:58.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:58.567 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:58.590 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:58.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:58.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:58.658 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:56:58.693 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:56:58.716 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:58.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:58.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:56:58.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:58 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1086523265' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:56:58.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:58 smithi039 ceph-mon[106470]: osdmap e1522: 8 total, 8 up, 8 in 2024-09-14T04:56:58.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:56:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:56:58] "GET /metrics HTTP/1.1" 200 36659 "" "Prometheus/2.51.0" 2024-09-14T04:56:58.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:58 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1086523265' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:56:58.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:58 smithi039 ceph-mon[111235]: osdmap e1522: 8 total, 8 up, 8 in 2024-09-14T04:56:58.818 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:58.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:58.841 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:58.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:58.942 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:58.965 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:58.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:58.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:59.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:58 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1086523265' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T04:56:59.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:58 smithi129 ceph-mon[101316]: osdmap e1522: 8 total, 8 up, 8 in 2024-09-14T04:56:59.067 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:59.089 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:59.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:59.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:59.191 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:59.216 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:59.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:59.217 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:59.318 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:56:59.342 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:56:59.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:59.343 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:56:59.358 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:56:59.443 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:56:59.465 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:59.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:59.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:56:59.567 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:56:59.590 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:59.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:59.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:56:59.636 DEBUG:teuthology.orchestra.run.smithi039:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2024-09-14T04:56:59.637 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-14T04:56:59.637 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:56:59.639 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-14T04:56:59.640 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-14T04:56:59.692 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:59.694 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:hints = 1 2024-09-14T04:56:59.694 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-14T04:56:59.694 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Object prefix: benchmark_data_smithi039_365565 2024-09-14T04:56:59.694 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:56:59.694 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 0 0 0 0 0 0 - 0 2024-09-14T04:56:59.733 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:59.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:59.735 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:59.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:59 smithi039 ceph-mon[106470]: pgmap v2912: 99 pgs: 99 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:59.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:56:59 smithi039 ceph-mon[106470]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:56:59.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:59 smithi039 ceph-mon[111235]: pgmap v2912: 99 pgs: 99 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:59.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:56:59 smithi039 ceph-mon[111235]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:56:59.836 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:56:59.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:56:59.891 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:56:59.891 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:56:59.921 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:59 smithi129 ceph-mon[101316]: pgmap v2912: 99 pgs: 99 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:56:59.921 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:56:59 smithi129 ceph-mon[101316]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T04:56:59.992 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:00.030 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:00.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:00.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:00.132 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:00.166 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:00.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:00.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:00.268 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:00.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:00.309 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:00.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:00.410 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:00.440 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:00.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:00.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:00.541 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:00.576 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:00.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:00.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:00.677 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:00.693 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 1 16 3913 3897 243.549 243.562 0.00543848 0.00409476 2024-09-14T04:57:00.711 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:00.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:00.712 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:00.813 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:00.862 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:00.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:00.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:00.964 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:00.998 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:00.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:00.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:01.100 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:01.137 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:01.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:01.138 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:01.239 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:01.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:01.280 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:01.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:01.381 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:01.413 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:01.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:01.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:01.515 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:01.548 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:01.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:01.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:01.609 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-14T04:57:01.650 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:01.691 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:01.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:01.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:01.694 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 2 16 7430 7414 231.613 219.812 0.00618921 0.00430752 2024-09-14T04:57:01.793 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:01.827 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:01.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:01.830 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:01.867 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:01 smithi039 ceph-mon[106470]: pgmap v2913: 99 pgs: 99 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:57:01.867 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:01 smithi039 ceph-mon[111235]: pgmap v2913: 99 pgs: 99 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:57:01.931 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:01.946 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:57:01.967 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:01.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:01.968 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:02.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:01 smithi129 ceph-mon[101316]: pgmap v2913: 99 pgs: 99 active+clean; 582 KiB data, 5.3 GiB used, 710 GiB / 715 GiB avail 2024-09-14T04:57:02.069 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:02.103 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:02.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:02.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:02.205 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:02.247 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:02.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:02.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:02.349 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:02.388 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:02.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:02.390 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:02.393 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:57:02.491 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:02.526 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:02.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:02.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:02.628 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:02.658 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:02.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:02.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:02.694 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 3 16 10543 10527 219.259 194.562 0.00714399 0.00455226 2024-09-14T04:57:02.757 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:57:02.757 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:57:02.760 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:02.790 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:02.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:02.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:02.799 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:57:02.800 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:57:02.892 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:02.913 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:02.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:02.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:02.952 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:57:02.952 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:57:02.984 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:57:02.984 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:57:03.015 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:03.019 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:57:03.020 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:57:03.055 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:57:03.055 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:57:03.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:03.057 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:03.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:03.092 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:57:03.093 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:57:03.127 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:57:03.128 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:57:03.158 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:03.197 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:03.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:03.199 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:03.299 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:03.331 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:03.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:03.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:03.433 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:03.470 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:03.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:03.471 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:03.554 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is set 2024-09-14T04:57:03.572 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:03.604 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:03.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:03.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:03.694 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 4 16 12996 12980 202.771 153.312 0.00628284 0.00492569 2024-09-14T04:57:03.706 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:03.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:03.740 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:03.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:03.842 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:03.881 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:03.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:03.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:03.972 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:03 smithi129 ceph-mon[101316]: pgmap v2914: 99 pgs: 99 active+clean; 17 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 1.8 MiB/s wr, 28 op/s 2024-09-14T04:57:03.972 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:03 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2710934671' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:57:03.973 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:03 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2755740915' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:57:03.973 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:03 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:57:03.983 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:04.021 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:04.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:04.022 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:04.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:03 smithi039 ceph-mon[106470]: pgmap v2914: 99 pgs: 99 active+clean; 17 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 1.8 MiB/s wr, 28 op/s 2024-09-14T04:57:04.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:03 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2710934671' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:57:04.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:03 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2755740915' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:57:04.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:03 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:57:04.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:03 smithi039 ceph-mon[111235]: pgmap v2914: 99 pgs: 99 active+clean; 17 MiB data, 5.5 GiB used, 710 GiB / 715 GiB avail; 1.8 MiB/s wr, 28 op/s 2024-09-14T04:57:04.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:03 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2710934671' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:57:04.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:03 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2755740915' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:57:04.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:03 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:57:04.123 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:04.160 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:04.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:04.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:04.262 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:04.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:04.294 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:04.294 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:04.395 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:04.418 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:04.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:04.419 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:04.520 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:04.560 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:04.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:04.561 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:04.618 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:04 smithi129 ceph-mon[101316]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:57:04.618 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:04 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:57:04.618 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:04 smithi129 ceph-mon[101316]: osdmap e1523: 8 total, 8 up, 8 in 2024-09-14T04:57:04.662 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:04.676 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:57:04.676 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [ 2024-09-14T04:57:04.676 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:04.676 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36324.0:15382 57.a 57:5a5112ef:::benchmark_data_smithi039_365565_object15381:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T04:57:04.676 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:57:04.671688+0000", 2024-09-14T04:57:04.676 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.0019652229999999999, 2024-09-14T04:57:04.676 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0020017780000000001, 2024-09-14T04:57:04.676 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:04.676 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:57:04.677 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:04.677 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36324", 2024-09-14T04:57:04.677 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T04:57:04.677 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 15382 2024-09-14T04:57:04.677 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:04.677 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:04.677 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:04.677 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:04.677 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:04.671688+0000", 2024-09-14T04:57:04.677 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:04.677 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:04.677 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:04.677 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:04.678 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:04.671688+0000", 2024-09-14T04:57:04.678 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:04.678 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:04.678 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:04.678 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:04.678 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:04.671686+0000", 2024-09-14T04:57:04.678 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:57:04.678 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:04.678 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:04.678 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:04.678 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:04.671769+0000", 2024-09-14T04:57:04.678 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.2943999999999997e-05 2024-09-14T04:57:04.678 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:04.679 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:04.679 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:04.679 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:04.671781+0000", 2024-09-14T04:57:04.679 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1459e-05 2024-09-14T04:57:04.679 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:04.679 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:04.679 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:04.679 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:04.671788+0000", 2024-09-14T04:57:04.679 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.2660000000000004e-06 2024-09-14T04:57:04.679 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:04.679 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:04.679 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:04.679 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:04.671842+0000", 2024-09-14T04:57:04.680 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.3533999999999999e-05 2024-09-14T04:57:04.680 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:04.680 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:04.680 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:04.680 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:04.671982+0000", 2024-09-14T04:57:04.680 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00014054099999999999 2024-09-14T04:57:04.680 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:04.680 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:04.680 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:57:04.680 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:04.672113+0000", 2024-09-14T04:57:04.680 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013102099999999999 2024-09-14T04:57:04.680 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:04.680 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:04.681 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:04.681 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:04.681 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:04.681 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36324.0:15387 57.5 57:a18bd396:::benchmark_data_smithi039_365565_object15386:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T04:57:04.681 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:57:04.673039+0000", 2024-09-14T04:57:04.681 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.00061417999999999998, 2024-09-14T04:57:04.681 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00073548799999999996, 2024-09-14T04:57:04.681 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:04.681 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:57:04.681 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:04.681 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36324", 2024-09-14T04:57:04.681 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T04:57:04.681 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 15387 2024-09-14T04:57:04.681 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:04.682 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:04.682 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:04.682 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:04.682 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:04.673039+0000", 2024-09-14T04:57:04.682 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:04.682 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:04.682 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:04.682 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:04.682 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:04.673039+0000", 2024-09-14T04:57:04.682 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:04.682 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:04.682 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:04.683 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:04.683 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:04.673038+0000", 2024-09-14T04:57:04.683 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:57:04.683 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:04.683 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:04.683 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:04.683 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:04.673070+0000", 2024-09-14T04:57:04.683 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.2571999999999999e-05 2024-09-14T04:57:04.683 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:04.683 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:04.683 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:04.683 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:04.673081+0000", 2024-09-14T04:57:04.683 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0788000000000001e-05 2024-09-14T04:57:04.683 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:04.684 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:04.684 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:04.684 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:04.673088+0000", 2024-09-14T04:57:04.684 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7499999999999997e-06 2024-09-14T04:57:04.684 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:04.684 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:04.684 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:04.684 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:04.673186+0000", 2024-09-14T04:57:04.684 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.7657000000000004e-05 2024-09-14T04:57:04.684 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:04.684 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:04.684 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:04.684 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:04.673332+0000", 2024-09-14T04:57:04.684 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000146768 2024-09-14T04:57:04.685 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:04.685 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:04.685 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,6", 2024-09-14T04:57:04.685 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:04.673459+0000", 2024-09-14T04:57:04.685 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000126391 2024-09-14T04:57:04.685 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:04.685 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:04.685 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:04.686 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:04.686 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:04.686 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.36324.0:15386 57.9 e1523/1518 57:900cec61:::benchmark_data_smithi039_365565_object15385:head v 1523'1012, mlcod=1523'1010)", 2024-09-14T04:57:04.686 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:57:04.673526+0000", 2024-09-14T04:57:04.686 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.00012667900000000001, 2024-09-14T04:57:04.686 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00032101900000000002, 2024-09-14T04:57:04.686 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:04.686 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "started", 2024-09-14T04:57:04.686 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:04.686 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:04.686 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:04.686 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:04.673526+0000", 2024-09-14T04:57:04.687 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:04.687 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:04.687 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:04.687 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:04.687 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:04.673526+0000", 2024-09-14T04:57:04.687 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:04.687 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:04.687 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:04.687 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:04.687 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:04.673525+0000", 2024-09-14T04:57:04.687 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:57:04.687 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:04.687 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:04.687 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:04.688 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:04.673549+0000", 2024-09-14T04:57:04.688 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.3297000000000001e-05 2024-09-14T04:57:04.688 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:04.688 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:04.688 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:04.688 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:04.673558+0000", 2024-09-14T04:57:04.688 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.0799999999999995e-06 2024-09-14T04:57:04.688 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:04.688 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:04.688 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:04.688 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:04.673564+0000", 2024-09-14T04:57:04.688 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.7450000000000001e-06 2024-09-14T04:57:04.688 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:04.689 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:04.689 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:04.689 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:04.673604+0000", 2024-09-14T04:57:04.689 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0590000000000003e-05 2024-09-14T04:57:04.689 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:04.689 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:04.689 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:04.689 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:04.673623+0000", 2024-09-14T04:57:04.689 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.8975999999999999e-05 2024-09-14T04:57:04.689 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:04.689 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:04.689 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:04.689 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:04.690 INFO:teuthology.orchestra.run.smithi129.stdout: ], 2024-09-14T04:57:04.690 INFO:teuthology.orchestra.run.smithi129.stdout: "num_ops": 3 2024-09-14T04:57:04.690 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:57:04.694 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 5 16 15438 15422 192.74 152.625 0.00419532 0.0051799 2024-09-14T04:57:04.696 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:04.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:04.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:04.798 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:04.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:04.823 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:04.823 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:04.924 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:04.954 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:04.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:04.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:05.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:04 smithi039 ceph-mon[111235]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:57:05.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:04 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:57:05.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:04 smithi039 ceph-mon[111235]: osdmap e1523: 8 total, 8 up, 8 in 2024-09-14T04:57:05.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:04 smithi039 ceph-mon[106470]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:57:05.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:04 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:57:05.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:04 smithi039 ceph-mon[106470]: osdmap e1523: 8 total, 8 up, 8 in 2024-09-14T04:57:05.056 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:05.087 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:05.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:05.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:05.189 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:05.228 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:05.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:05.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:05.330 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:05.353 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:05.353 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:05.354 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:05.455 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:05.485 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-14T04:57:05.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:05.494 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:05.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:05.596 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:05.633 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:05.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:05.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:05.695 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 6 16 17346 17330 180.491 119.25 0.00753344 0.0055343 2024-09-14T04:57:05.735 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:05 smithi129 ceph-mon[101316]: pgmap v2916: 99 pgs: 99 active+clean; 379 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 47 MiB/s wr, 756 op/s 2024-09-14T04:57:05.774 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:05.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:05.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:05.805 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:57:05.876 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:05.912 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:05.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:05.913 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:06.014 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:06.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:05 smithi039 ceph-mon[106470]: pgmap v2916: 99 pgs: 99 active+clean; 379 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 47 MiB/s wr, 756 op/s 2024-09-14T04:57:06.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:05 smithi039 ceph-mon[111235]: pgmap v2916: 99 pgs: 99 active+clean; 379 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 47 MiB/s wr, 756 op/s 2024-09-14T04:57:06.050 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:06.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:06.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:06.152 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:06.189 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:06.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:06.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:06.291 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:06.330 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:06.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:06.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:06.432 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:06.461 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:06.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:06.462 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:06.563 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:06.593 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:06.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:06.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:06.695 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 7 16 20139 20123 179.642 174.562 0.00942853 0.00556118 2024-09-14T04:57:06.695 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:06.735 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:06.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:06.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:06.837 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:06.875 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:06.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:06.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:06.978 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:07.014 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:07.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:07.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:07.116 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:07.151 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:07.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:07.152 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:07.253 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:07.287 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:07.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:07.288 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:07.389 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:07.410 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:07.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:07.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:07.512 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:07.544 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:07.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:07.545 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:07.646 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:07.671 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:07.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:07.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:07.695 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 8 16 22276 22260 173.881 133.562 0.00274247 0.00571867 2024-09-14T04:57:07.773 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:07.812 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:07.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:07.813 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:07.910 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:07 smithi039 ceph-mon[106470]: pgmap v2917: 99 pgs: 99 active+clean; 906 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s 2024-09-14T04:57:07.910 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:57:07.911 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:57:07.911 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:07 smithi039 ceph-mon[111235]: pgmap v2917: 99 pgs: 99 active+clean; 906 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s 2024-09-14T04:57:07.911 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:57:07.911 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:57:07.914 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:07.945 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:07.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:07.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:08.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:07 smithi129 ceph-mon[101316]: pgmap v2917: 99 pgs: 99 active+clean; 906 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s 2024-09-14T04:57:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:57:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:57:08.047 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:08.089 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:08.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:08.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:08.192 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:08.220 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:08.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:08.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:08.322 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:08.356 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:08.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:08.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:08.458 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:08.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:08.496 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:08.496 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:08.597 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:08.630 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:08.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:08.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:08.695 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 9 16 24473 24457 169.817 137.312 0.00699467 0.00588373 2024-09-14T04:57:08.732 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:08.767 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:08.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:08.768 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:08.869 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:08.891 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:08.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:08.892 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:08.993 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:09.027 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:09.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:09.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:09.039 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:57:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:57:08] "GET /metrics HTTP/1.1" 200 36721 "" "Prometheus/2.51.0" 2024-09-14T04:57:09.129 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:09.167 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:09.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:09.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:09.269 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:09.302 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:09.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:09.303 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:09.404 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:09.439 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:09.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:09.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:09.541 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:09.580 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:09.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:09.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:09.682 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:09.695 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 10 16 26717 26701 166.859 140.25 0.00896775 0.00598698 2024-09-14T04:57:09.733 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:09.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:09.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:09.835 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:09.861 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:09.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:09.862 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:09.963 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:09.997 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:09.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:09.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:10.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:09 smithi129 ceph-mon[101316]: pgmap v2918: 99 pgs: 99 active+clean; 906 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-09-14T04:57:10.032 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:09 smithi039 ceph-mon[106470]: pgmap v2918: 99 pgs: 99 active+clean; 906 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-09-14T04:57:10.033 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:09 smithi039 ceph-mon[111235]: pgmap v2918: 99 pgs: 99 active+clean; 906 MiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 91 MiB/s wr, 1.45k op/s 2024-09-14T04:57:10.099 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:10.136 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:10.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:10.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:10.238 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:10.274 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:10.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:10.275 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:10.376 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:10.407 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:10.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:10.408 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:10.509 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:10.542 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:10.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:10.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:10.643 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:10.681 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:10.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:10.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:10.695 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 11 16 29025 29009 164.802 144.25 0.00300782 0.00606191 2024-09-14T04:57:10.783 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:10.817 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:10.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:10.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:10.919 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:10.954 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:10.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:10.955 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:11.056 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:11.088 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:11.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:11.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:11.190 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:11.226 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:11.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:11.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:11.328 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:11.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:11.371 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:11.372 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:11.442 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:57:11.443 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T04:57:11.472 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:11.508 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:11.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:11.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:11.610 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:11.644 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:11.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:11.645 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:11.695 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 12 16 31117 31101 161.964 130.75 0.00832751 0.00616958 2024-09-14T04:57:11.726 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:57:11.745 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:11.769 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:11 smithi039 ceph-mon[106470]: pgmap v2919: 99 pgs: 99 active+clean; 1.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2024-09-14T04:57:11.769 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:11 smithi039 ceph-mon[111235]: pgmap v2919: 99 pgs: 99 active+clean; 1.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2024-09-14T04:57:11.778 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:11.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:11.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:11.859 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:11 smithi129 ceph-mon[101316]: pgmap v2919: 99 pgs: 99 active+clean; 1.4 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 139 MiB/s wr, 2.23k op/s 2024-09-14T04:57:11.880 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:11.915 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:11.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:11.916 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:12.017 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:12.026 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:57:12.026 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [], 2024-09-14T04:57:12.026 INFO:teuthology.orchestra.run.smithi129.stdout: "complaint_time": 30, 2024-09-14T04:57:12.026 INFO:teuthology.orchestra.run.smithi129.stdout: "num_blocked_ops": 0 2024-09-14T04:57:12.026 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:57:12.056 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:12.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:12.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:12.158 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:12.195 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:12.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:12.196 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:12.297 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:12.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:12.340 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:12.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:12.366 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":2919,"stamp":"2024-09-14T04:57:09.905128+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1461983205,"num_objects":22325,"num_object_clones":0,"num_object_copies":66975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22325,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22363,"num_write_kb":1427169,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":53,"num_bytes_recovered":1318510,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":23997,"ondisk_log_size":23997,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":300,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9826436,"kb_used_data":4739072,"kb_used_omap":404,"kb_used_meta":5086891,"kb_avail":740134780,"statfs":{"total":767960285184,"available":757898014720,"internally_reserved":0,"allocated":4852809728,"data_stored":4810426915,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":414022,"internal_metadata":5208977082},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[9,8,13,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1461387264,"num_objects":22299,"num_object_clones":0,"num_object_copies":66897,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22299,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22299,"num_write_kb":1427136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003479"},"pg_stats":[{"pgid":"57.f","version":"1523'968","reported_seq":989,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:04.849609+0000","last_change":"2024-09-14T04:56:49.435182+0000","last_active":"2024-09-14T04:57:04.849609+0000","last_peered":"2024-09-14T04:57:04.849609+0000","last_clean":"2024-09-14T04:57:04.849609+0000","last_became_active":"2024-09-14T04:56:49.434949+0000","last_became_peered":"2024-09-14T04:56:49.434949+0000","last_unstale":"2024-09-14T04:57:04.849609+0000","last_undegraded":"2024-09-14T04:57:04.849609+0000","last_fullsized":"2024-09-14T04:57:04.849609+0000","mapping_epoch":1518,"log_start":"0'0","ondisk_log_start":"0'0","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":968,"log_dups_size":0,"ondisk_log_size":968,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:32:29.856162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63438848,"num_objects":968,"num_object_clones":0,"num_object_copies":2904,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":968,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":968,"num_write_kb":61952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"57.e","version":"1523'1601","reported_seq":1622,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:09.482619+0000","last_change":"2024-09-14T04:56:49.431930+0000","last_active":"2024-09-14T04:57:09.482619+0000","last_peered":"2024-09-14T04:57:09.482619+0000","last_clean":"2024-09-14T04:57:09.482619+0000","last_became_active":"2024-09-14T04:56:49.431642+0000","last_became_peered":"2024-09-14T04:56:49.431642+0000","last_unstale":"2024-09-14T04:57:09.482619+0000","last_undegraded":"2024-09-14T04:57:09.482619+0000","last_fullsized":"2024-09-14T04:57:09.482619+0000","mapping_epoch":1518,"log_start":"0'0","ondisk_log_start":"0'0","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":1601,"log_dups_size":0,"ondisk_log_size":1601,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:30:17.677827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104923136,"num_objects":1601,"num_object_clones":0,"num_object_copies":4803,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1601,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1601,"num_write_kb":102464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"57.d","version":"1523'1609","reported_seq":1623,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:08.812080+0000","last_change":"2024-09-14T04:56:49.821152+0000","last_active":"2024-09-14T04:57:08.812080+0000","last_peered":"2024-09-14T04:57:08.812080+0000","last_clean":"2024-09-14T04:57:08.812080+0000","last_became_active":"2024-09-14T04:56:49.820853+0000","last_became_peered":"2024-09-14T04:56:49.820853+0000","last_unstale":"2024-09-14T04:57:08.812080+0000","last_undegraded":"2024-09-14T04:57:08.812080+0000","last_fullsized":"2024-09-14T04:57:08.812080+0000","mapping_epoch":1518,"log_start":"0'0","ondisk_log_start":"0'0","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":1609,"log_dups_size":0,"ondisk_log_size":1609,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:41.229640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105447424,"num_objects":1609,"num_object_clones":0,"num_object_copies":4827,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1609,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1609,"num_write_kb":102976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"57.c","version":"1523'1502","reported_seq":1516,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:08.803708+0000","last_change":"2024-09-14T04:56:49.435207+0000","last_active":"2024-09-14T04:57:08.803708+0000","last_peered":"2024-09-14T04:57:08.803708+0000","last_clean":"2024-09-14T04:57:08.803708+0000","last_became_active":"2024-09-14T04:56:49.434752+0000","last_became_peered":"2024-09-14T04:56:49.434752+0000","last_unstale":"2024-09-14T04:57:08.803708+0000","last_undegraded":"2024-09-14T04:57:08.803708+0000","last_fullsized":"2024-09-14T04:57:08.803708+0000","mapping_epoch":1518,"log_start":"0'0","ondisk_log_start":"0'0","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":1502,"log_dups_size":0,"ondisk_log_size":1502,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:29:43.013481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98435072,"num_objects":1502,"num_object_clones":0,"num_object_copies":4506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1502,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1502,"num_write_kb":96128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"57.b","version":"1523'983","reported_seq":1003,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:04.909113+0000","last_change":"2024-09-14T04:56:49.434067+0000","last_active":"2024-09-14T04:57:04.909113+0000","last_peered":"2024-09-14T04:57:04.909113+0000","last_clean":"2024-09-14T04:57:04.909113+0000","last_became_active":"2024-09-14T04:56:49.433725+0000","last_became_peered":"2024-09-14T04:56:49.433725+0000","last_unstale":"2024-09-14T04:57:04.909113+0000","last_undegraded":"2024-09-14T04:57:04.909113+0000","last_fullsized":"2024-09-14T04:57:04.909113+0000","mapping_epoch":1518,"log_start":"0'0","ondisk_log_start":"0'0","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":983,"log_dups_size":0,"ondisk_log_size":983,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:18:56.597341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64421888,"num_objects":983,"num_object_clones":0,"num_object_copies":2949,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":983,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":983,"num_write_kb":62912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"57.a","version":"1523'1681","reported_seq":1695,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:09.662841+0000","last_change":"2024-09-14T04:56:49.818406+0000","last_active":"2024-09-14T04:57:09.662841+0000","last_peered":"2024-09-14T04:57:09.662841+0000","last_clean":"2024-09-14T04:57:09.662841+0000","last_became_active":"2024-09-14T04:56:49.818087+0000","last_became_peered":"2024-09-14T04:56:49.818087+0000","last_unstale":"2024-09-14T04:57:09.662841+0000","last_undegraded":"2024-09-14T04:57:09.662841+0000","last_fullsized":"2024-09-14T04:57:09.662841+0000","mapping_epoch":1518,"log_start":"0'0","ondisk_log_start":"0'0","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":1681,"log_dups_size":0,"ondisk_log_size":1681,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:41:19.568727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110166016,"num_objects":1681,"num_object_clones":0,"num_object_copies":5043,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1681,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1681,"num_write_kb":107584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"57.9","version":"1523'1494","reported_seq":1508,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:08.365304+0000","last_change":"2024-09-14T04:56:49.818923+0000","last_active":"2024-09-14T04:57:08.365304+0000","last_peered":"2024-09-14T04:57:08.365304+0000","last_clean":"2024-09-14T04:57:08.365304+0000","last_became_active":"2024-09-14T04:56:49.818519+0000","last_became_peered":"2024-09-14T04:56:49.818519+0000","last_unstale":"2024-09-14T04:57:08.365304+0000","last_undegraded":"2024-09-14T04:57:08.365304+0000","last_fullsized":"2024-09-14T04:57:08.365304+0000","mapping_epoch":1518,"log_start":"0'0","ondisk_log_start":"0'0","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":1494,"log_dups_size":0,"ondisk_log_size":1494,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:41:08.309897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97910784,"num_objects":1494,"num_object_clones":0,"num_object_copies":4482,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1494,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1494,"num_write_kb":95616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"57.8","version":"1523'1528","reported_seq":1542,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:08.813416+0000","last_change":"2024-09-14T04:56:49.434525+0000","last_active":"2024-09-14T04:57:08.813416+0000","last_peered":"2024-09-14T04:57:08.813416+0000","last_clean":"2024-09-14T04:57:08.813416+0000","last_became_active":"2024-09-14T04:56:49.434144+0000","last_became_peered":"2024-09-14T04:56:49.434144+0000","last_unstale":"2024-09-14T04:57:08.813416+0000","last_undegraded":"2024-09-14T04:57:08.813416+0000","last_fullsized":"2024-09-14T04:57:08.813416+0000","mapping_epoch":1518,"log_start":"0'0","ondisk_log_start":"0'0","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":1528,"log_dups_size":0,"ondisk_log_size":1528,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:17:18.918905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100139008,"num_objects":1528,"num_object_clones":0,"num_object_copies":4584,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1528,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1528,"num_write_kb":97792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"57.7","version":"1523'985","reported_seq":1006,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:04.918049+0000","last_change":"2024-09-14T04:56:49.434581+0000","last_active":"2024-09-14T04:57:04.918049+0000","last_peered":"2024-09-14T04:57:04.918049+0000","last_clean":"2024-09-14T04:57:04.918049+0000","last_became_active":"2024-09-14T04:56:49.434210+0000","last_became_peered":"2024-09-14T04:56:49.434210+0000","last_unstale":"2024-09-14T04:57:04.918049+0000","last_undegraded":"2024-09-14T04:57:04.918049+0000","last_fullsized":"2024-09-14T04:57:04.918049+0000","mapping_epoch":1518,"log_start":"0'0","ondisk_log_start":"0'0","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":985,"log_dups_size":0,"ondisk_log_size":985,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:16.426758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64552960,"num_objects":985,"num_object_clones":0,"num_object_copies":2955,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":985,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":985,"num_write_kb":63040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"57.6","version":"1523'1720","reported_seq":1734,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:09.669397+0000","last_change":"2024-09-14T04:56:49.436357+0000","last_active":"2024-09-14T04:57:09.669397+0000","last_peered":"2024-09-14T04:57:09.669397+0000","last_clean":"2024-09-14T04:57:09.669397+0000","last_became_active":"2024-09-14T04:56:49.436101+0000","last_became_peered":"2024-09-14T04:56:49.436101+0000","last_unstale":"2024-09-14T04:57:09.669397+0000","last_undegraded":"2024-09-14T04:57:09.669397+0000","last_fullsized":"2024-09-14T04:57:09.669397+0000","mapping_epoch":1518,"log_start":"0'0","ondisk_log_start":"0'0","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":1720,"log_dups_size":0,"ondisk_log_size":1720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:03:53.232843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":112721920,"num_objects":1720,"num_object_clones":0,"num_object_copies":5160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1720,"num_write_kb":110080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"57.5","version":"1523'1666","reported_seq":1679,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:09.664859+0000","last_change":"2024-09-14T04:56:49.821285+0000","last_active":"2024-09-14T04:57:09.664859+0000","last_peered":"2024-09-14T04:57:09.664859+0000","last_clean":"2024-09-14T04:57:09.664859+0000","last_became_active":"2024-09-14T04:56:49.820869+0000","last_became_peered":"2024-09-14T04:56:49.820869+0000","last_unstale":"2024-09-14T04:57:09.664859+0000","last_undegraded":"2024-09-14T04:57:09.664859+0000","last_fullsized":"2024-09-14T04:57:09.664859+0000","mapping_epoch":1518,"log_start":"0'0","ondisk_log_start":"0'0","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":1666,"log_dups_size":0,"ondisk_log_size":1666,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:26:39.392722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109182976,"num_objects":1666,"num_object_clones":0,"num_object_copies":4998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1666,"num_write_kb":106624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"57.4","version":"1523'1536","reported_seq":1556,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:08.801077+0000","last_change":"2024-09-14T04:56:49.820010+0000","last_active":"2024-09-14T04:57:08.801077+0000","last_peered":"2024-09-14T04:57:08.801077+0000","last_clean":"2024-09-14T04:57:08.801077+0000","last_became_active":"2024-09-14T04:56:49.819563+0000","last_became_peered":"2024-09-14T04:56:49.819563+0000","last_unstale":"2024-09-14T04:57:08.801077+0000","last_undegraded":"2024-09-14T04:57:08.801077+0000","last_fullsized":"2024-09-14T04:57:08.801077+0000","mapping_epoch":1518,"log_start":"0'0","ondisk_log_start":"0'0","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":1536,"log_dups_size":0,"ondisk_log_size":1536,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:32:31.141098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100663296,"num_objects":1536,"num_object_clones":0,"num_object_copies":4608,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1536,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1536,"num_write_kb":98304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"57.3","version":"1523'967","reported_seq":988,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:04.915291+0000","last_change":"2024-09-14T04:56:49.819031+0000","last_active":"2024-09-14T04:57:04.915291+0000","last_peered":"2024-09-14T04:57:04.915291+0000","last_clean":"2024-09-14T04:57:04.915291+0000","last_became_active":"2024-09-14T04:56:49.818654+0000","last_became_peered":"2024-09-14T04:56:49.818654+0000","last_unstale":"2024-09-14T04:57:04.915291+0000","last_undegraded":"2024-09-14T04:57:04.915291+0000","last_fullsized":"2024-09-14T04:57:04.915291+0000","mapping_epoch":1518,"log_start":"0'0","ondisk_log_start":"0'0","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":967,"log_dups_size":0,"ondisk_log_size":967,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:57:37.054801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63373312,"num_objects":967,"num_object_clones":0,"num_object_copies":2901,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":967,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":967,"num_write_kb":61888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"57.2","version":"1523'1516","reported_seq":1537,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:08.786601+0000","last_change":"2024-09-14T04:56:49.819908+0000","last_active":"2024-09-14T04:57:08.786601+0000","last_peered":"2024-09-14T04:57:08.786601+0000","last_clean":"2024-09-14T04:57:08.786601+0000","last_became_active":"2024-09-14T04:56:49.819599+0000","last_became_peered":"2024-09-14T04:56:49.819599+0000","last_unstale":"2024-09-14T04:57:08.786601+0000","last_undegraded":"2024-09-14T04:57:08.786601+0000","last_fullsized":"2024-09-14T04:57:08.786601+0000","mapping_epoch":1518,"log_start":"0'0","ondisk_log_start":"0'0","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":1516,"log_dups_size":0,"ondisk_log_size":1516,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:38:49.493483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99352576,"num_objects":1516,"num_object_clones":0,"num_object_copies":4548,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1516,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1516,"num_write_kb":97024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"57.1","version":"1523'1566","reported_seq":1587,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:08.796433+0000","last_change":"2024-09-14T04:56:49.434507+0000","last_active":"2024-09-14T04:57:08.796433+0000","last_peered":"2024-09-14T04:57:08.796433+0000","last_clean":"2024-09-14T04:57:08.796433+0000","last_became_active":"2024-09-14T04:56:49.434172+0000","last_became_peered":"2024-09-14T04:56:49.434172+0000","last_unstale":"2024-09-14T04:57:08.796433+0000","last_undegraded":"2024-09-14T04:57:08.796433+0000","last_fullsized":"2024-09-14T04:57:08.796433+0000","mapping_epoch":1518,"log_start":"0'0","ondisk_log_start":"0'0","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":1566,"log_dups_size":0,"ondisk_log_size":1566,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:22.424289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102629376,"num_objects":1566,"num_object_clones":0,"num_object_copies":4698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1566,"num_write_kb":100224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"57.0","version":"1523'977","reported_seq":998,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:04.919961+0000","last_change":"2024-09-14T04:56:49.818610+0000","last_active":"2024-09-14T04:57:04.919961+0000","last_peered":"2024-09-14T04:57:04.919961+0000","last_clean":"2024-09-14T04:57:04.919961+0000","last_became_active":"2024-09-14T04:56:49.818300+0000","last_became_peered":"2024-09-14T04:56:49.818300+0000","last_unstale":"2024-09-14T04:57:04.919961+0000","last_undegraded":"2024-09-14T04:57:04.919961+0000","last_fullsized":"2024-09-14T04:57:04.919961+0000","mapping_epoch":1518,"log_start":"0'0","ondisk_log_start":"0'0","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":977,"log_dups_size":0,"ondisk_log_size":977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:29:20.641683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64028672,"num_objects":977,"num_object_clones":0,"num_object_copies":2931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":977,"num_write_kb":62528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":2339,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:03.564971+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T04:57:03.564971+0000","last_peered":"2024-09-14T04:57:03.564971+0000","last_clean":"2024-09-14T04:57:03.564971+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T04:57:03.564971+0000","last_undegraded":"2024-09-14T04:57:03.564971+0000","last_fullsized":"2024-09-14T04:57:03.564971+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":136,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:57:03.369719+0000","last_change":"2024-09-14T04:44:57.325109+0000","last_active":"2024-09-14T04:57:03.369719+0000","last_peered":"2024-09-14T04:57:03.369719+0000","last_clean":"2024-09-14T04:57:03.369719+0000","last_became_active":"2024-09-14T04:44:57.324791+0000","last_became_peered":"2024-09-14T04:44:57.324791+0000","last_unstale":"2024-09-14T04:57:03.369719+0000","last_undegraded":"2024-09-14T04:57:03.369719+0000","last_fullsized":"2024-09-14T04:57:03.369719+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:03.593275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":105,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.918484+0000","last_change":"2024-09-14T04:49:29.628435+0000","last_active":"2024-09-14T04:56:57.918484+0000","last_peered":"2024-09-14T04:56:57.918484+0000","last_clean":"2024-09-14T04:56:57.918484+0000","last_became_active":"2024-09-14T04:49:29.628081+0000","last_became_peered":"2024-09-14T04:49:29.628081+0000","last_unstale":"2024-09-14T04:56:57.918484+0000","last_undegraded":"2024-09-14T04:56:57.918484+0000","last_fullsized":"2024-09-14T04:56:57.918484+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:01.520334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":960,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:03.562131+0000","last_change":"2024-09-14T04:43:13.355281+0000","last_active":"2024-09-14T04:57:03.562131+0000","last_peered":"2024-09-14T04:57:03.562131+0000","last_clean":"2024-09-14T04:57:03.562131+0000","last_became_active":"2024-09-14T04:43:13.354967+0000","last_became_peered":"2024-09-14T04:43:13.354967+0000","last_unstale":"2024-09-14T04:57:03.562131+0000","last_undegraded":"2024-09-14T04:57:03.562131+0000","last_fullsized":"2024-09-14T04:57:03.562131+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:23:19.332075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1514,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.505349+0000","last_change":"2024-09-14T04:44:57.342918+0000","last_active":"2024-09-14T04:56:57.505349+0000","last_peered":"2024-09-14T04:56:57.505349+0000","last_clean":"2024-09-14T04:56:57.505349+0000","last_became_active":"2024-09-14T04:44:57.329550+0000","last_became_peered":"2024-09-14T04:44:57.329550+0000","last_unstale":"2024-09-14T04:56:57.505349+0000","last_undegraded":"2024-09-14T04:56:57.505349+0000","last_fullsized":"2024-09-14T04:56:57.505349+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:52:52.435893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":135,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.500345+0000","last_change":"2024-09-14T04:44:57.325537+0000","last_active":"2024-09-14T04:56:57.500345+0000","last_peered":"2024-09-14T04:56:57.500345+0000","last_clean":"2024-09-14T04:56:57.500345+0000","last_became_active":"2024-09-14T04:44:57.325057+0000","last_became_peered":"2024-09-14T04:44:57.325057+0000","last_unstale":"2024-09-14T04:56:57.500345+0000","last_undegraded":"2024-09-14T04:56:57.500345+0000","last_fullsized":"2024-09-14T04:56:57.500345+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:43:23.166708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1529,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.505317+0000","last_change":"2024-09-14T04:44:57.337033+0000","last_active":"2024-09-14T04:56:57.505317+0000","last_peered":"2024-09-14T04:56:57.505317+0000","last_clean":"2024-09-14T04:56:57.505317+0000","last_became_active":"2024-09-14T04:44:57.328437+0000","last_became_peered":"2024-09-14T04:44:57.328437+0000","last_unstale":"2024-09-14T04:56:57.505317+0000","last_undegraded":"2024-09-14T04:56:57.505317+0000","last_fullsized":"2024-09-14T04:56:57.505317+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.109086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":135,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.500555+0000","last_change":"2024-09-14T04:44:57.332243+0000","last_active":"2024-09-14T04:56:57.500555+0000","last_peered":"2024-09-14T04:56:57.500555+0000","last_clean":"2024-09-14T04:56:57.500555+0000","last_became_active":"2024-09-14T04:44:57.330413+0000","last_became_peered":"2024-09-14T04:44:57.330413+0000","last_unstale":"2024-09-14T04:56:57.500555+0000","last_undegraded":"2024-09-14T04:56:57.500555+0000","last_fullsized":"2024-09-14T04:56:57.500555+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:25:01.338549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":195,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:03.564473+0000","last_change":"2024-09-14T04:48:29.862929+0000","last_active":"2024-09-14T04:57:03.564473+0000","last_peered":"2024-09-14T04:57:03.564473+0000","last_clean":"2024-09-14T04:57:03.564473+0000","last_became_active":"2024-09-14T04:48:29.862581+0000","last_became_peered":"2024-09-14T04:48:29.862581+0000","last_unstale":"2024-09-14T04:57:03.564473+0000","last_undegraded":"2024-09-14T04:57:03.564473+0000","last_fullsized":"2024-09-14T04:57:03.564473+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:16:43.060088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2334,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:03.561594+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T04:57:03.561594+0000","last_peered":"2024-09-14T04:57:03.561594+0000","last_clean":"2024-09-14T04:57:03.561594+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T04:57:03.561594+0000","last_undegraded":"2024-09-14T04:57:03.561594+0000","last_fullsized":"2024-09-14T04:57:03.561594+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":135,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.500292+0000","last_change":"2024-09-14T04:44:57.325392+0000","last_active":"2024-09-14T04:56:57.500292+0000","last_peered":"2024-09-14T04:56:57.500292+0000","last_clean":"2024-09-14T04:56:57.500292+0000","last_became_active":"2024-09-14T04:44:57.324624+0000","last_became_peered":"2024-09-14T04:44:57.324624+0000","last_unstale":"2024-09-14T04:56:57.500292+0000","last_undegraded":"2024-09-14T04:56:57.500292+0000","last_fullsized":"2024-09-14T04:56:57.500292+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:33:18.810199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2303,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:03.558649+0000","last_change":"2024-09-14T04:44:57.329113+0000","last_active":"2024-09-14T04:57:03.558649+0000","last_peered":"2024-09-14T04:57:03.558649+0000","last_clean":"2024-09-14T04:57:03.558649+0000","last_became_active":"2024-09-14T04:44:57.328727+0000","last_became_peered":"2024-09-14T04:44:57.328727+0000","last_unstale":"2024-09-14T04:57:03.558649+0000","last_undegraded":"2024-09-14T04:57:03.558649+0000","last_fullsized":"2024-09-14T04:57:03.558649+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:34.939874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":2320,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:03.558585+0000","last_change":"2024-09-14T04:44:57.329277+0000","last_active":"2024-09-14T04:57:03.558585+0000","last_peered":"2024-09-14T04:57:03.558585+0000","last_clean":"2024-09-14T04:57:03.558585+0000","last_became_active":"2024-09-14T04:44:57.328804+0000","last_became_peered":"2024-09-14T04:44:57.328804+0000","last_unstale":"2024-09-14T04:57:03.558585+0000","last_undegraded":"2024-09-14T04:57:03.558585+0000","last_fullsized":"2024-09-14T04:57:03.558585+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:42:46.055240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2355,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:03.565921+0000","last_change":"2024-09-14T04:54:52.250695+0000","last_active":"2024-09-14T04:57:03.565921+0000","last_peered":"2024-09-14T04:57:03.565921+0000","last_clean":"2024-09-14T04:57:03.565921+0000","last_became_active":"2024-09-14T04:54:52.250301+0000","last_became_peered":"2024-09-14T04:54:52.250301+0000","last_unstale":"2024-09-14T04:57:03.565921+0000","last_undegraded":"2024-09-14T04:57:03.565921+0000","last_fullsized":"2024-09-14T04:57:03.565921+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:42.060905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2316,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:03.562641+0000","last_change":"2024-09-14T04:43:13.357092+0000","last_active":"2024-09-14T04:57:03.562641+0000","last_peered":"2024-09-14T04:57:03.562641+0000","last_clean":"2024-09-14T04:57:03.562641+0000","last_became_active":"2024-09-14T04:43:13.356694+0000","last_became_peered":"2024-09-14T04:43:13.356694+0000","last_unstale":"2024-09-14T04:57:03.562641+0000","last_undegraded":"2024-09-14T04:57:03.562641+0000","last_fullsized":"2024-09-14T04:57:03.562641+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:11.167574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1507,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.918470+0000","last_change":"2024-09-14T04:07:21.720797+0000","last_active":"2024-09-14T04:56:57.918470+0000","last_peered":"2024-09-14T04:56:57.918470+0000","last_clean":"2024-09-14T04:56:57.918470+0000","last_became_active":"2024-09-14T04:07:21.719331+0000","last_became_peered":"2024-09-14T04:07:21.719331+0000","last_unstale":"2024-09-14T04:56:57.918470+0000","last_undegraded":"2024-09-14T04:56:57.918470+0000","last_fullsized":"2024-09-14T04:56:57.918470+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:35.404023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":154,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.504979+0000","last_change":"2024-09-14T04:43:11.641514+0000","last_active":"2024-09-14T04:56:57.504979+0000","last_peered":"2024-09-14T04:56:57.504979+0000","last_clean":"2024-09-14T04:56:57.504979+0000","last_became_active":"2024-09-14T04:43:11.641151+0000","last_became_peered":"2024-09-14T04:43:11.641151+0000","last_unstale":"2024-09-14T04:56:57.504979+0000","last_undegraded":"2024-09-14T04:56:57.504979+0000","last_fullsized":"2024-09-14T04:56:57.504979+0000","mapping_epoch":1384,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:06:45.045430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2328,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:03.565155+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T04:57:03.565155+0000","last_peered":"2024-09-14T04:57:03.565155+0000","last_clean":"2024-09-14T04:57:03.565155+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T04:57:03.565155+0000","last_undegraded":"2024-09-14T04:57:03.565155+0000","last_fullsized":"2024-09-14T04:57:03.565155+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":735,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.919257+0000","last_change":"2024-09-14T04:49:29.618477+0000","last_active":"2024-09-14T04:56:57.919257+0000","last_peered":"2024-09-14T04:56:57.919257+0000","last_clean":"2024-09-14T04:56:57.919257+0000","last_became_active":"2024-09-14T04:49:29.618210+0000","last_became_peered":"2024-09-14T04:49:29.618210+0000","last_unstale":"2024-09-14T04:56:57.919257+0000","last_undegraded":"2024-09-14T04:56:57.919257+0000","last_fullsized":"2024-09-14T04:56:57.919257+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:13:23.895512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1513,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.505243+0000","last_change":"2024-09-14T04:42:49.938505+0000","last_active":"2024-09-14T04:56:57.505243+0000","last_peered":"2024-09-14T04:56:57.505243+0000","last_clean":"2024-09-14T04:56:57.505243+0000","last_became_active":"2024-09-14T04:42:49.938180+0000","last_became_peered":"2024-09-14T04:42:49.938180+0000","last_unstale":"2024-09-14T04:56:57.505243+0000","last_undegraded":"2024-09-14T04:56:57.505243+0000","last_fullsized":"2024-09-14T04:56:57.505243+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:05.929479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1532,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.920058+0000","last_change":"2024-09-14T04:44:57.346582+0000","last_active":"2024-09-14T04:56:57.920058+0000","last_peered":"2024-09-14T04:56:57.920058+0000","last_clean":"2024-09-14T04:56:57.920058+0000","last_became_active":"2024-09-14T04:44:57.330446+0000","last_became_peered":"2024-09-14T04:44:57.330446+0000","last_unstale":"2024-09-14T04:56:57.920058+0000","last_undegraded":"2024-09-14T04:56:57.920058+0000","last_fullsized":"2024-09-14T04:56:57.920058+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:29.238585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":269,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:03.558771+0000","last_change":"2024-09-14T04:44:14.480667+0000","last_active":"2024-09-14T04:57:03.558771+0000","last_peered":"2024-09-14T04:57:03.558771+0000","last_clean":"2024-09-14T04:57:03.558771+0000","last_became_active":"2024-09-14T04:44:14.480365+0000","last_became_peered":"2024-09-14T04:44:14.480365+0000","last_unstale":"2024-09-14T04:57:03.558771+0000","last_undegraded":"2024-09-14T04:57:03.558771+0000","last_fullsized":"2024-09-14T04:57:03.558771+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:00.611771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":146,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.500595+0000","last_change":"2024-09-14T04:44:57.341426+0000","last_active":"2024-09-14T04:56:57.500595+0000","last_peered":"2024-09-14T04:56:57.500595+0000","last_clean":"2024-09-14T04:56:57.500595+0000","last_became_active":"2024-09-14T04:44:57.328763+0000","last_became_peered":"2024-09-14T04:44:57.328763+0000","last_unstale":"2024-09-14T04:56:57.500595+0000","last_undegraded":"2024-09-14T04:56:57.500595+0000","last_fullsized":"2024-09-14T04:56:57.500595+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:34:40.066761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":136,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.500644+0000","last_change":"2024-09-14T04:44:57.331337+0000","last_active":"2024-09-14T04:56:57.500644+0000","last_peered":"2024-09-14T04:56:57.500644+0000","last_clean":"2024-09-14T04:56:57.500644+0000","last_became_active":"2024-09-14T04:44:57.329336+0000","last_became_peered":"2024-09-14T04:44:57.329336+0000","last_unstale":"2024-09-14T04:56:57.500644+0000","last_undegraded":"2024-09-14T04:56:57.500644+0000","last_fullsized":"2024-09-14T04:56:57.500644+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:52:35.980869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"349'149","reported_seq":724,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.505222+0000","last_change":"2024-09-14T04:49:30.085274+0000","last_active":"2024-09-14T04:56:57.505222+0000","last_peered":"2024-09-14T04:56:57.505222+0000","last_clean":"2024-09-14T04:56:57.505222+0000","last_became_active":"2024-09-14T04:49:29.623295+0000","last_became_peered":"2024-09-14T04:49:29.623295+0000","last_unstale":"2024-09-14T04:56:57.505222+0000","last_undegraded":"2024-09-14T04:56:57.505222+0000","last_fullsized":"2024-09-14T04:56:57.505222+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:49:30.085089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:49:30.085089+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:10.012963+0000","scrub_duration":0.0071928790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":136,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.500325+0000","last_change":"2024-09-14T04:44:57.329870+0000","last_active":"2024-09-14T04:56:57.500325+0000","last_peered":"2024-09-14T04:56:57.500325+0000","last_clean":"2024-09-14T04:56:57.500325+0000","last_became_active":"2024-09-14T04:44:57.329353+0000","last_became_peered":"2024-09-14T04:44:57.329353+0000","last_unstale":"2024-09-14T04:56:57.500325+0000","last_undegraded":"2024-09-14T04:56:57.500325+0000","last_fullsized":"2024-09-14T04:56:57.500325+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:46.432181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1529,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.919581+0000","last_change":"2024-09-14T04:54:52.242207+0000","last_active":"2024-09-14T04:56:57.919581+0000","last_peered":"2024-09-14T04:56:57.919581+0000","last_clean":"2024-09-14T04:56:57.919581+0000","last_became_active":"2024-09-14T04:54:52.241840+0000","last_became_peered":"2024-09-14T04:54:52.241840+0000","last_unstale":"2024-09-14T04:56:57.919581+0000","last_undegraded":"2024-09-14T04:56:57.919581+0000","last_fullsized":"2024-09-14T04:56:57.919581+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:37.899666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":2366,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:03.565824+0000","last_change":"2024-09-14T04:44:57.344614+0000","last_active":"2024-09-14T04:57:03.565824+0000","last_peered":"2024-09-14T04:57:03.565824+0000","last_clean":"2024-09-14T04:57:03.565824+0000","last_became_active":"2024-09-14T04:44:57.332256+0000","last_became_peered":"2024-09-14T04:44:57.332256+0000","last_unstale":"2024-09-14T04:57:03.565824+0000","last_undegraded":"2024-09-14T04:57:03.565824+0000","last_fullsized":"2024-09-14T04:57:03.565824+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:23.438417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"349'149","reported_seq":894,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.500711+0000","last_change":"2024-09-14T04:45:00.331854+0000","last_active":"2024-09-14T04:56:57.500711+0000","last_peered":"2024-09-14T04:56:57.500711+0000","last_clean":"2024-09-14T04:56:57.500711+0000","last_became_active":"2024-09-14T04:45:00.331460+0000","last_became_peered":"2024-09-14T04:45:00.331460+0000","last_unstale":"2024-09-14T04:56:57.500711+0000","last_undegraded":"2024-09-14T04:56:57.500711+0000","last_fullsized":"2024-09-14T04:56:57.500711+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.567535+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.567535+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:45.095729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":725,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.919323+0000","last_change":"2024-09-14T04:49:29.616831+0000","last_active":"2024-09-14T04:56:57.919323+0000","last_peered":"2024-09-14T04:56:57.919323+0000","last_clean":"2024-09-14T04:56:57.919323+0000","last_became_active":"2024-09-14T04:49:29.616546+0000","last_became_peered":"2024-09-14T04:49:29.616546+0000","last_unstale":"2024-09-14T04:56:57.919323+0000","last_undegraded":"2024-09-14T04:56:57.919323+0000","last_fullsized":"2024-09-14T04:56:57.919323+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:27.882929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":321,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:03.559433+0000","last_change":"2024-09-14T04:44:57.330746+0000","last_active":"2024-09-14T04:57:03.559433+0000","last_peered":"2024-09-14T04:57:03.559433+0000","last_clean":"2024-09-14T04:57:03.559433+0000","last_became_active":"2024-09-14T04:44:57.330327+0000","last_became_peered":"2024-09-14T04:44:57.330327+0000","last_unstale":"2024-09-14T04:57:03.559433+0000","last_undegraded":"2024-09-14T04:57:03.559433+0000","last_fullsized":"2024-09-14T04:57:03.559433+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:53.122771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":176,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.920278+0000","last_change":"2024-09-14T04:43:13.355296+0000","last_active":"2024-09-14T04:56:57.920278+0000","last_peered":"2024-09-14T04:56:57.920278+0000","last_clean":"2024-09-14T04:56:57.920278+0000","last_became_active":"2024-09-14T04:43:13.354950+0000","last_became_peered":"2024-09-14T04:43:13.354950+0000","last_unstale":"2024-09-14T04:56:57.920278+0000","last_undegraded":"2024-09-14T04:56:57.920278+0000","last_fullsized":"2024-09-14T04:56:57.920278+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:20:08.968818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"349'149","reported_seq":833,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.919182+0000","last_change":"2024-09-14T04:41:39.471393+0000","last_active":"2024-09-14T04:56:57.919182+0000","last_peered":"2024-09-14T04:56:57.919182+0000","last_clean":"2024-09-14T04:56:57.919182+0000","last_became_active":"2024-09-14T04:41:36.931768+0000","last_became_peered":"2024-09-14T04:41:36.931768+0000","last_unstale":"2024-09-14T04:56:57.919182+0000","last_undegraded":"2024-09-14T04:56:57.919182+0000","last_fullsized":"2024-09-14T04:56:57.919182+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.471248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.471248+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:56.210375+0000","scrub_duration":0.021758344999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2322,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:03.564743+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T04:57:03.564743+0000","last_peered":"2024-09-14T04:57:03.564743+0000","last_clean":"2024-09-14T04:57:03.564743+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T04:57:03.564743+0000","last_undegraded":"2024-09-14T04:57:03.564743+0000","last_fullsized":"2024-09-14T04:57:03.564743+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1543,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.920176+0000","last_change":"2024-09-14T04:44:57.331436+0000","last_active":"2024-09-14T04:56:57.920176+0000","last_peered":"2024-09-14T04:56:57.920176+0000","last_clean":"2024-09-14T04:56:57.920176+0000","last_became_active":"2024-09-14T04:44:57.330957+0000","last_became_peered":"2024-09-14T04:44:57.330957+0000","last_unstale":"2024-09-14T04:56:57.920176+0000","last_undegraded":"2024-09-14T04:56:57.920176+0000","last_fullsized":"2024-09-14T04:56:57.920176+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:09.516688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":871,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:03.561492+0000","last_change":"2024-09-14T04:49:49.460401+0000","last_active":"2024-09-14T04:57:03.561492+0000","last_peered":"2024-09-14T04:57:03.561492+0000","last_clean":"2024-09-14T04:57:03.561492+0000","last_became_active":"2024-09-14T04:49:49.458530+0000","last_became_peered":"2024-09-14T04:49:49.458530+0000","last_unstale":"2024-09-14T04:57:03.561492+0000","last_undegraded":"2024-09-14T04:57:03.561492+0000","last_fullsized":"2024-09-14T04:57:03.561492+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:11:29.143239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"349'149","reported_seq":867,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.919447+0000","last_change":"2024-09-14T04:54:22.147441+0000","last_active":"2024-09-14T04:56:57.919447+0000","last_peered":"2024-09-14T04:56:57.919447+0000","last_clean":"2024-09-14T04:56:57.919447+0000","last_became_active":"2024-09-14T04:54:22.147115+0000","last_became_peered":"2024-09-14T04:54:22.147115+0000","last_unstale":"2024-09-14T04:56:57.919447+0000","last_undegraded":"2024-09-14T04:56:57.919447+0000","last_fullsized":"2024-09-14T04:56:57.919447+0000","mapping_epoch":1492,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1493,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:38.479887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:38.479887+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:41:14.001781+0000","scrub_duration":0.0062728319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":277,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:03.558585+0000","last_change":"2024-09-14T04:44:57.329983+0000","last_active":"2024-09-14T04:57:03.558585+0000","last_peered":"2024-09-14T04:57:03.558585+0000","last_clean":"2024-09-14T04:57:03.558585+0000","last_became_active":"2024-09-14T04:44:57.329597+0000","last_became_peered":"2024-09-14T04:44:57.329597+0000","last_unstale":"2024-09-14T04:57:03.558585+0000","last_undegraded":"2024-09-14T04:57:03.558585+0000","last_fullsized":"2024-09-14T04:57:03.558585+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:45.718039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":815,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:03.564671+0000","last_change":"2024-09-14T04:49:49.443631+0000","last_active":"2024-09-14T04:57:03.564671+0000","last_peered":"2024-09-14T04:57:03.564671+0000","last_clean":"2024-09-14T04:57:03.564671+0000","last_became_active":"2024-09-14T04:49:49.443313+0000","last_became_peered":"2024-09-14T04:49:49.443313+0000","last_unstale":"2024-09-14T04:57:03.564671+0000","last_undegraded":"2024-09-14T04:57:03.564671+0000","last_fullsized":"2024-09-14T04:57:03.564671+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:06:54.562292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1515,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.505430+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T04:56:57.505430+0000","last_peered":"2024-09-14T04:56:57.505430+0000","last_clean":"2024-09-14T04:56:57.505430+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T04:56:57.505430+0000","last_undegraded":"2024-09-14T04:56:57.505430+0000","last_fullsized":"2024-09-14T04:56:57.505430+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"349'149","reported_seq":825,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.504986+0000","last_change":"2024-09-14T04:41:39.012977+0000","last_active":"2024-09-14T04:56:57.504986+0000","last_peered":"2024-09-14T04:56:57.504986+0000","last_clean":"2024-09-14T04:56:57.504986+0000","last_became_active":"2024-09-14T04:41:36.932552+0000","last_became_peered":"2024-09-14T04:41:36.932552+0000","last_unstale":"2024-09-14T04:56:57.504986+0000","last_undegraded":"2024-09-14T04:56:57.504986+0000","last_fullsized":"2024-09-14T04:56:57.504986+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.012879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.012879+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:16:40.080262+0000","scrub_duration":0.0051721620000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":907,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:03.561681+0000","last_change":"2024-09-14T04:49:49.459590+0000","last_active":"2024-09-14T04:57:03.561681+0000","last_peered":"2024-09-14T04:57:03.561681+0000","last_clean":"2024-09-14T04:57:03.561681+0000","last_became_active":"2024-09-14T04:49:49.456263+0000","last_became_peered":"2024-09-14T04:49:49.456263+0000","last_unstale":"2024-09-14T04:57:03.561681+0000","last_undegraded":"2024-09-14T04:57:03.561681+0000","last_fullsized":"2024-09-14T04:57:03.561681+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:20:44.666805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":136,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.500570+0000","last_change":"2024-09-14T04:44:57.329652+0000","last_active":"2024-09-14T04:56:57.500570+0000","last_peered":"2024-09-14T04:56:57.500570+0000","last_clean":"2024-09-14T04:56:57.500570+0000","last_became_active":"2024-09-14T04:44:57.328888+0000","last_became_peered":"2024-09-14T04:44:57.328888+0000","last_unstale":"2024-09-14T04:56:57.500570+0000","last_undegraded":"2024-09-14T04:56:57.500570+0000","last_fullsized":"2024-09-14T04:56:57.500570+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:10.170068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1514,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.505125+0000","last_change":"2024-09-14T04:06:46.875340+0000","last_active":"2024-09-14T04:56:57.505125+0000","last_peered":"2024-09-14T04:56:57.505125+0000","last_clean":"2024-09-14T04:56:57.505125+0000","last_became_active":"2024-09-14T04:06:46.874552+0000","last_became_peered":"2024-09-14T04:06:46.874552+0000","last_unstale":"2024-09-14T04:56:57.505125+0000","last_undegraded":"2024-09-14T04:56:57.505125+0000","last_fullsized":"2024-09-14T04:56:57.505125+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:52.257357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"349'149","reported_seq":728,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:03.560243+0000","last_change":"2024-09-14T04:54:55.279666+0000","last_active":"2024-09-14T04:57:03.560243+0000","last_peered":"2024-09-14T04:57:03.560243+0000","last_clean":"2024-09-14T04:57:03.560243+0000","last_became_active":"2024-09-14T04:54:55.279259+0000","last_became_peered":"2024-09-14T04:54:55.279259+0000","last_unstale":"2024-09-14T04:57:03.560243+0000","last_undegraded":"2024-09-14T04:57:03.560243+0000","last_fullsized":"2024-09-14T04:57:03.560243+0000","mapping_epoch":1506,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1507,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:54:03.996025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:54:03.996025+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:40:53.332369+0000","scrub_duration":54,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":269,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:03.559224+0000","last_change":"2024-09-14T04:44:14.489235+0000","last_active":"2024-09-14T04:57:03.559224+0000","last_peered":"2024-09-14T04:57:03.559224+0000","last_clean":"2024-09-14T04:57:03.559224+0000","last_became_active":"2024-09-14T04:44:14.488950+0000","last_became_peered":"2024-09-14T04:44:14.488950+0000","last_unstale":"2024-09-14T04:57:03.559224+0000","last_undegraded":"2024-09-14T04:57:03.559224+0000","last_fullsized":"2024-09-14T04:57:03.559224+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:54.441144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":825,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:03.562497+0000","last_change":"2024-09-14T04:49:49.456317+0000","last_active":"2024-09-14T04:57:03.562497+0000","last_peered":"2024-09-14T04:57:03.562497+0000","last_clean":"2024-09-14T04:57:03.562497+0000","last_became_active":"2024-09-14T04:49:49.456029+0000","last_became_peered":"2024-09-14T04:49:49.456029+0000","last_unstale":"2024-09-14T04:57:03.562497+0000","last_undegraded":"2024-09-14T04:57:03.562497+0000","last_fullsized":"2024-09-14T04:57:03.562497+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:46:05.319676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2342,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:03.562220+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T04:57:03.562220+0000","last_peered":"2024-09-14T04:57:03.562220+0000","last_clean":"2024-09-14T04:57:03.562220+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T04:57:03.562220+0000","last_undegraded":"2024-09-14T04:57:03.562220+0000","last_fullsized":"2024-09-14T04:57:03.562220+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"349'149","reported_seq":719,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.919149+0000","last_change":"2024-09-14T04:50:22.002798+0000","last_active":"2024-09-14T04:56:57.919149+0000","last_peered":"2024-09-14T04:56:57.919149+0000","last_clean":"2024-09-14T04:56:57.919149+0000","last_became_active":"2024-09-14T04:49:49.448278+0000","last_became_peered":"2024-09-14T04:49:49.448278+0000","last_unstale":"2024-09-14T04:56:57.919149+0000","last_undegraded":"2024-09-14T04:56:57.919149+0000","last_fullsized":"2024-09-14T04:56:57.919149+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:50:22.002568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:50:22.002568+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:01:57.489223+0000","scrub_duration":0.010888326,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2343,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:03.566002+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T04:57:03.566002+0000","last_peered":"2024-09-14T04:57:03.566002+0000","last_clean":"2024-09-14T04:57:03.566002+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T04:57:03.566002+0000","last_undegraded":"2024-09-14T04:57:03.566002+0000","last_fullsized":"2024-09-14T04:57:03.566002+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2364,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:03.568247+0000","last_change":"2024-09-14T04:54:19.447103+0000","last_active":"2024-09-14T04:57:03.568247+0000","last_peered":"2024-09-14T04:57:03.568247+0000","last_clean":"2024-09-14T04:57:03.568247+0000","last_became_active":"2024-09-14T04:54:19.446748+0000","last_became_peered":"2024-09-14T04:54:19.446748+0000","last_unstale":"2024-09-14T04:57:03.568247+0000","last_undegraded":"2024-09-14T04:57:03.568247+0000","last_fullsized":"2024-09-14T04:57:03.568247+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:00:55.369263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1544,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.919389+0000","last_change":"2024-09-14T04:42:49.939790+0000","last_active":"2024-09-14T04:56:57.919389+0000","last_peered":"2024-09-14T04:56:57.919389+0000","last_clean":"2024-09-14T04:56:57.919389+0000","last_became_active":"2024-09-14T04:42:49.939591+0000","last_became_peered":"2024-09-14T04:42:49.939591+0000","last_unstale":"2024-09-14T04:56:57.919389+0000","last_undegraded":"2024-09-14T04:56:57.919389+0000","last_fullsized":"2024-09-14T04:56:57.919389+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:23:16.790593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1507,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.918960+0000","last_change":"2024-09-14T04:44:57.326008+0000","last_active":"2024-09-14T04:56:57.918960+0000","last_peered":"2024-09-14T04:56:57.918960+0000","last_clean":"2024-09-14T04:56:57.918960+0000","last_became_active":"2024-09-14T04:44:57.325617+0000","last_became_peered":"2024-09-14T04:44:57.325617+0000","last_unstale":"2024-09-14T04:56:57.918960+0000","last_undegraded":"2024-09-14T04:56:57.918960+0000","last_fullsized":"2024-09-14T04:56:57.918960+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:15.637508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":796,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.505258+0000","last_change":"2024-09-14T04:43:13.354846+0000","last_active":"2024-09-14T04:56:57.505258+0000","last_peered":"2024-09-14T04:56:57.505258+0000","last_clean":"2024-09-14T04:56:57.505258+0000","last_became_active":"2024-09-14T04:43:13.354515+0000","last_became_peered":"2024-09-14T04:43:13.354515+0000","last_unstale":"2024-09-14T04:56:57.505258+0000","last_undegraded":"2024-09-14T04:56:57.505258+0000","last_fullsized":"2024-09-14T04:56:57.505258+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:17.889341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1513,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.505175+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T04:56:57.505175+0000","last_peered":"2024-09-14T04:56:57.505175+0000","last_clean":"2024-09-14T04:56:57.505175+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T04:56:57.505175+0000","last_undegraded":"2024-09-14T04:56:57.505175+0000","last_fullsized":"2024-09-14T04:56:57.505175+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":1850,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.919249+0000","last_change":"2024-09-14T04:41:30.640651+0000","last_active":"2024-09-14T04:56:57.919249+0000","last_peered":"2024-09-14T04:56:57.919249+0000","last_clean":"2024-09-14T04:56:57.919249+0000","last_became_active":"2024-09-14T04:41:29.532466+0000","last_became_peered":"2024-09-14T04:41:29.532466+0000","last_unstale":"2024-09-14T04:56:57.919249+0000","last_undegraded":"2024-09-14T04:56:57.919249+0000","last_fullsized":"2024-09-14T04:56:57.919249+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":0,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:30.640520+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:30.640520+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:27.776724+0000","scrub_duration":0.0028653139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"349'149","reported_seq":825,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.505127+0000","last_change":"2024-09-14T04:41:39.978080+0000","last_active":"2024-09-14T04:56:57.505127+0000","last_peered":"2024-09-14T04:56:57.505127+0000","last_clean":"2024-09-14T04:56:57.505127+0000","last_became_active":"2024-09-14T04:41:36.929279+0000","last_became_peered":"2024-09-14T04:41:36.929279+0000","last_unstale":"2024-09-14T04:56:57.505127+0000","last_undegraded":"2024-09-14T04:56:57.505127+0000","last_fullsized":"2024-09-14T04:56:57.505127+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.977999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.977999+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:44:06.383827+0000","scrub_duration":0.00192122,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":203,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:03.562248+0000","last_change":"2024-09-14T04:48:11.040133+0000","last_active":"2024-09-14T04:57:03.562248+0000","last_peered":"2024-09-14T04:57:03.562248+0000","last_clean":"2024-09-14T04:57:03.562248+0000","last_became_active":"2024-09-14T04:48:11.039787+0000","last_became_peered":"2024-09-14T04:48:11.039787+0000","last_unstale":"2024-09-14T04:57:03.562248+0000","last_undegraded":"2024-09-14T04:57:03.562248+0000","last_fullsized":"2024-09-14T04:57:03.562248+0000","mapping_epoch":1428,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:53.520672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2190,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:03.558986+0000","last_change":"2024-09-14T04:49:29.616443+0000","last_active":"2024-09-14T04:57:03.558986+0000","last_peered":"2024-09-14T04:57:03.558986+0000","last_clean":"2024-09-14T04:57:03.558986+0000","last_became_active":"2024-09-14T04:49:29.616066+0000","last_became_peered":"2024-09-14T04:49:29.616066+0000","last_unstale":"2024-09-14T04:57:03.558986+0000","last_undegraded":"2024-09-14T04:57:03.558986+0000","last_fullsized":"2024-09-14T04:57:03.558986+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:09:15.897957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":136,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.500833+0000","last_change":"2024-09-14T04:44:57.326724+0000","last_active":"2024-09-14T04:56:57.500833+0000","last_peered":"2024-09-14T04:56:57.500833+0000","last_clean":"2024-09-14T04:56:57.500833+0000","last_became_active":"2024-09-14T04:44:57.326191+0000","last_became_peered":"2024-09-14T04:44:57.326191+0000","last_unstale":"2024-09-14T04:56:57.500833+0000","last_undegraded":"2024-09-14T04:56:57.500833+0000","last_fullsized":"2024-09-14T04:56:57.500833+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:46.060269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":136,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.500250+0000","last_change":"2024-09-14T04:44:57.328528+0000","last_active":"2024-09-14T04:56:57.500250+0000","last_peered":"2024-09-14T04:56:57.500250+0000","last_clean":"2024-09-14T04:56:57.500250+0000","last_became_active":"2024-09-14T04:44:57.327739+0000","last_became_peered":"2024-09-14T04:44:57.327739+0000","last_unstale":"2024-09-14T04:56:57.500250+0000","last_undegraded":"2024-09-14T04:56:57.500250+0000","last_fullsized":"2024-09-14T04:56:57.500250+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:04:58.294690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"349'149","reported_seq":833,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.504973+0000","last_change":"2024-09-14T04:55:59.732757+0000","last_active":"2024-09-14T04:56:57.504973+0000","last_peered":"2024-09-14T04:56:57.504973+0000","last_clean":"2024-09-14T04:56:57.504973+0000","last_became_active":"2024-09-14T04:55:59.732364+0000","last_became_peered":"2024-09-14T04:55:59.732364+0000","last_unstale":"2024-09-14T04:56:57.504973+0000","last_undegraded":"2024-09-14T04:56:57.504973+0000","last_fullsized":"2024-09-14T04:56:57.504973+0000","mapping_epoch":1512,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1513,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:56.738636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:56.738636+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:59:11.313765+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1313440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":136,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.500629+0000","last_change":"2024-09-14T04:44:57.323436+0000","last_active":"2024-09-14T04:56:57.500629+0000","last_peered":"2024-09-14T04:56:57.500629+0000","last_clean":"2024-09-14T04:56:57.500629+0000","last_became_active":"2024-09-14T04:44:57.323022+0000","last_became_peered":"2024-09-14T04:44:57.323022+0000","last_unstale":"2024-09-14T04:56:57.500629+0000","last_undegraded":"2024-09-14T04:56:57.500629+0000","last_fullsized":"2024-09-14T04:56:57.500629+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:22.775261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1542,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.918968+0000","last_change":"2024-09-14T04:44:57.322357+0000","last_active":"2024-09-14T04:56:57.918968+0000","last_peered":"2024-09-14T04:56:57.918968+0000","last_clean":"2024-09-14T04:56:57.918968+0000","last_became_active":"2024-09-14T04:44:57.321990+0000","last_became_peered":"2024-09-14T04:44:57.321990+0000","last_unstale":"2024-09-14T04:56:57.918968+0000","last_undegraded":"2024-09-14T04:56:57.918968+0000","last_fullsized":"2024-09-14T04:56:57.918968+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:09.658008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"349'149","reported_seq":1867,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.919171+0000","last_change":"2024-09-14T04:41:53.858076+0000","last_active":"2024-09-14T04:56:57.919171+0000","last_peered":"2024-09-14T04:56:57.919171+0000","last_clean":"2024-09-14T04:56:57.919171+0000","last_became_active":"2024-09-14T04:41:36.931585+0000","last_became_peered":"2024-09-14T04:41:36.931585+0000","last_unstale":"2024-09-14T04:56:57.919171+0000","last_undegraded":"2024-09-14T04:56:57.919171+0000","last_fullsized":"2024-09-14T04:56:57.919171+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:53.857924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:53.857924+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:10:20.658986+0000","scrub_duration":0.0022355919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":136,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.500589+0000","last_change":"2024-09-14T04:44:57.329198+0000","last_active":"2024-09-14T04:56:57.500589+0000","last_peered":"2024-09-14T04:56:57.500589+0000","last_clean":"2024-09-14T04:56:57.500589+0000","last_became_active":"2024-09-14T04:44:57.327754+0000","last_became_peered":"2024-09-14T04:44:57.327754+0000","last_unstale":"2024-09-14T04:56:57.500589+0000","last_undegraded":"2024-09-14T04:56:57.500589+0000","last_fullsized":"2024-09-14T04:56:57.500589+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:10:27.944476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":327,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:03.564840+0000","last_change":"2024-09-14T04:44:57.348687+0000","last_active":"2024-09-14T04:57:03.564840+0000","last_peered":"2024-09-14T04:57:03.564840+0000","last_clean":"2024-09-14T04:57:03.564840+0000","last_became_active":"2024-09-14T04:44:57.332845+0000","last_became_peered":"2024-09-14T04:44:57.332845+0000","last_unstale":"2024-09-14T04:57:03.564840+0000","last_undegraded":"2024-09-14T04:57:03.564840+0000","last_fullsized":"2024-09-14T04:57:03.564840+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:12:18.762737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2349,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:03.564582+0000","last_change":"2024-09-14T04:44:57.333468+0000","last_active":"2024-09-14T04:57:03.564582+0000","last_peered":"2024-09-14T04:57:03.564582+0000","last_clean":"2024-09-14T04:57:03.564582+0000","last_became_active":"2024-09-14T04:44:57.332626+0000","last_became_peered":"2024-09-14T04:44:57.332626+0000","last_unstale":"2024-09-14T04:57:03.564582+0000","last_undegraded":"2024-09-14T04:57:03.564582+0000","last_fullsized":"2024-09-14T04:57:03.564582+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:57:26.366438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1498,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.505391+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T04:56:57.505391+0000","last_peered":"2024-09-14T04:56:57.505391+0000","last_clean":"2024-09-14T04:56:57.505391+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T04:56:57.505391+0000","last_undegraded":"2024-09-14T04:56:57.505391+0000","last_fullsized":"2024-09-14T04:56:57.505391+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1497,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.918791+0000","last_change":"2024-09-14T04:06:46.869868+0000","last_active":"2024-09-14T04:56:57.918791+0000","last_peered":"2024-09-14T04:56:57.918791+0000","last_clean":"2024-09-14T04:56:57.918791+0000","last_became_active":"2024-09-14T04:06:46.869126+0000","last_became_peered":"2024-09-14T04:06:46.869126+0000","last_unstale":"2024-09-14T04:56:57.918791+0000","last_undegraded":"2024-09-14T04:56:57.918791+0000","last_fullsized":"2024-09-14T04:56:57.918791+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:41.150138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1481,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.920192+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T04:56:57.920192+0000","last_peered":"2024-09-14T04:56:57.920192+0000","last_clean":"2024-09-14T04:56:57.920192+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T04:56:57.920192+0000","last_undegraded":"2024-09-14T04:56:57.920192+0000","last_fullsized":"2024-09-14T04:56:57.920192+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":136,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.500379+0000","last_change":"2024-09-14T04:44:57.332112+0000","last_active":"2024-09-14T04:56:57.500379+0000","last_peered":"2024-09-14T04:56:57.500379+0000","last_clean":"2024-09-14T04:56:57.500379+0000","last_became_active":"2024-09-14T04:44:57.331575+0000","last_became_peered":"2024-09-14T04:44:57.331575+0000","last_unstale":"2024-09-14T04:56:57.500379+0000","last_undegraded":"2024-09-14T04:56:57.500379+0000","last_fullsized":"2024-09-14T04:56:57.500379+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:50:00.391283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1529,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.920189+0000","last_change":"2024-09-14T04:44:57.351460+0000","last_active":"2024-09-14T04:56:57.920189+0000","last_peered":"2024-09-14T04:56:57.920189+0000","last_clean":"2024-09-14T04:56:57.920189+0000","last_became_active":"2024-09-14T04:44:57.334662+0000","last_became_peered":"2024-09-14T04:44:57.334662+0000","last_unstale":"2024-09-14T04:56:57.920189+0000","last_undegraded":"2024-09-14T04:56:57.920189+0000","last_fullsized":"2024-09-14T04:56:57.920189+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:02.399062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1483,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.505122+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T04:56:57.505122+0000","last_peered":"2024-09-14T04:56:57.505122+0000","last_clean":"2024-09-14T04:56:57.505122+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T04:56:57.505122+0000","last_undegraded":"2024-09-14T04:56:57.505122+0000","last_fullsized":"2024-09-14T04:56:57.505122+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1541,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.920245+0000","last_change":"2024-09-14T04:44:57.349690+0000","last_active":"2024-09-14T04:56:57.920245+0000","last_peered":"2024-09-14T04:56:57.920245+0000","last_clean":"2024-09-14T04:56:57.920245+0000","last_became_active":"2024-09-14T04:44:57.333579+0000","last_became_peered":"2024-09-14T04:44:57.333579+0000","last_unstale":"2024-09-14T04:56:57.920245+0000","last_undegraded":"2024-09-14T04:56:57.920245+0000","last_fullsized":"2024-09-14T04:56:57.920245+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:17:55.265141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1618,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.918891+0000","last_change":"2024-09-14T04:54:35.694089+0000","last_active":"2024-09-14T04:56:57.918891+0000","last_peered":"2024-09-14T04:56:57.918891+0000","last_clean":"2024-09-14T04:56:57.918891+0000","last_became_active":"2024-09-14T04:54:35.663133+0000","last_became_peered":"2024-09-14T04:54:35.663133+0000","last_unstale":"2024-09-14T04:56:57.918891+0000","last_undegraded":"2024-09-14T04:56:57.918891+0000","last_fullsized":"2024-09-14T04:56:57.918891+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:17:19.947280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":30,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.920120+0000","last_change":"2024-09-14T04:54:52.248439+0000","last_active":"2024-09-14T04:56:57.920120+0000","last_peered":"2024-09-14T04:56:57.920120+0000","last_clean":"2024-09-14T04:56:57.920120+0000","last_became_active":"2024-09-14T04:54:52.248235+0000","last_became_peered":"2024-09-14T04:54:52.248235+0000","last_unstale":"2024-09-14T04:56:57.920120+0000","last_undegraded":"2024-09-14T04:56:57.920120+0000","last_fullsized":"2024-09-14T04:56:57.920120+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:22:09.548139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1503,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.919467+0000","last_change":"2024-09-14T04:44:57.332511+0000","last_active":"2024-09-14T04:56:57.919467+0000","last_peered":"2024-09-14T04:56:57.919467+0000","last_clean":"2024-09-14T04:56:57.919467+0000","last_became_active":"2024-09-14T04:44:57.332166+0000","last_became_peered":"2024-09-14T04:44:57.332166+0000","last_unstale":"2024-09-14T04:56:57.919467+0000","last_undegraded":"2024-09-14T04:56:57.919467+0000","last_fullsized":"2024-09-14T04:56:57.919467+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:53.984639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2343,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:03.565737+0000","last_change":"2024-09-14T04:54:52.250655+0000","last_active":"2024-09-14T04:57:03.565737+0000","last_peered":"2024-09-14T04:57:03.565737+0000","last_clean":"2024-09-14T04:57:03.565737+0000","last_became_active":"2024-09-14T04:54:52.250317+0000","last_became_peered":"2024-09-14T04:54:52.250317+0000","last_unstale":"2024-09-14T04:57:03.565737+0000","last_undegraded":"2024-09-14T04:57:03.565737+0000","last_fullsized":"2024-09-14T04:57:03.565737+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:12.985406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2328,"reported_epoch":1523,"state":"active+clean","last_fresh":"2024-09-14T04:57:03.564990+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T04:57:03.564990+0000","last_peered":"2024-09-14T04:57:03.564990+0000","last_clean":"2024-09-14T04:57:03.564990+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T04:57:03.564990+0000","last_undegraded":"2024-09-14T04:57:03.564990+0000","last_fullsized":"2024-09-14T04:57:03.564990+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1495,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.505336+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T04:56:57.505336+0000","last_peered":"2024-09-14T04:56:57.505336+0000","last_clean":"2024-09-14T04:56:57.505336+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T04:56:57.505336+0000","last_undegraded":"2024-09-14T04:56:57.505336+0000","last_fullsized":"2024-09-14T04:56:57.505336+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1532,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.919808+0000","last_change":"2024-09-14T04:44:57.339993+0000","last_active":"2024-09-14T04:56:57.919808+0000","last_peered":"2024-09-14T04:56:57.919808+0000","last_clean":"2024-09-14T04:56:57.919808+0000","last_became_active":"2024-09-14T04:44:57.330472+0000","last_became_peered":"2024-09-14T04:44:57.330472+0000","last_unstale":"2024-09-14T04:56:57.919808+0000","last_undegraded":"2024-09-14T04:56:57.919808+0000","last_fullsized":"2024-09-14T04:56:57.919808+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:19:27.046988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2080,"reported_epoch":1522,"state":"active+clean","last_fresh":"2024-09-14T04:56:57.919842+0000","last_change":"2024-09-14T04:49:49.449687+0000","last_active":"2024-09-14T04:56:57.919842+0000","last_peered":"2024-09-14T04:56:57.919842+0000","last_clean":"2024-09-14T04:56:57.919842+0000","last_became_active":"2024-09-14T04:49:49.449144+0000","last_became_peered":"2024-09-14T04:49:49.449144+0000","last_unstale":"2024-09-14T04:56:57.919842+0000","last_undegraded":"2024-09-14T04:56:57.919842+0000","last_fullsized":"2024-09-14T04:56:57.919842+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:59:06.779730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":57,"num_pg":16,"stat_sum":{"num_bytes":1461387264,"num_objects":22299,"num_object_clones":0,"num_object_copies":66897,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22299,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22299,"num_write_kb":1427136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4467130368,"data_stored":4467130368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":22299,"ondisk_log_size":22299,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1313440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":5070,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4250,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223202074,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1332964,"kb_used_data":603804,"kb_used_omap":1,"kb_used_meta":729150,"kb_avail":92412188,"statfs":{"total":95995035648,"available":94630080512,"internally_reserved":0,"allocated":618295296,"data_stored":612987673,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":746650059},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463710,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1527452,"kb_used_data":842580,"kb_used_omap":1,"kb_used_meta":684862,"kb_avail":92217700,"statfs":{"total":95995035648,"available":94430924800,"internally_reserved":0,"allocated":862801920,"data_stored":857502362,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":701299143},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725347,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1411160,"kb_used_data":636048,"kb_used_omap":1,"kb_used_meta":775102,"kb_avail":92333992,"statfs":{"total":95995035648,"available":94550007808,"internally_reserved":0,"allocated":651313152,"data_stored":646014060,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":793704902},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":1392,"seq":5978594476193,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":845796,"kb_used_data":471772,"kb_used_omap":117,"kb_used_meta":373898,"kb_avail":92899356,"statfs":{"total":95995035648,"available":95128940544,"internally_reserved":0,"allocated":483094528,"data_stored":477787457,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120638,"internal_metadata":382871746},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772075088,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1394964,"kb_used_data":604876,"kb_used_omap":71,"kb_used_meta":790008,"kb_avail":92350188,"statfs":{"total":95995035648,"available":94566592512,"internally_reserved":0,"allocated":619393024,"data_stored":614098081,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":808969142},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707304023,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1051672,"kb_used_data":514252,"kb_used_omap":68,"kb_used_meta":537339,"kb_avail":92693480,"statfs":{"total":95995035648,"available":94918123520,"internally_reserved":0,"allocated":526594048,"data_stored":521299009,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":550235344},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,3,6,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872729183,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":772172,"kb_used_data":202884,"kb_used_omap":67,"kb_used_meta":569212,"kb_avail":92972980,"statfs":{"total":95995035648,"available":95204331520,"internally_reserved":0,"allocated":207753216,"data_stored":202465228,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":582873445},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056233,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1490256,"kb_used_data":862856,"kb_used_omap":74,"kb_used_meta":627317,"kb_avail":92254896,"statfs":{"total":95995035648,"available":94469013504,"internally_reserved":0,"allocated":883564544,"data_stored":878273045,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":642373301},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,3,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1901,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":612302848,"data_stored":612302848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":209518592,"data_stored":209518592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":515899392,"data_stored":515899392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":663879680,"data_stored":663879680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":387383296,"data_stored":387383296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":661913600,"data_stored":661913600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":911015936,"data_stored":911015936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":505217024,"data_stored":505217024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:57:12.370 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pgp_num 11 2024-09-14T04:57:12.431 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:57:12.442 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:12.497 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:12.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:12.498 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:12.499 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:57:12.600 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:12.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:12.678 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:12.678 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:12.695 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 13 16 33140 33124 159.23 126.438 0.00438116 0.00627515 2024-09-14T04:57:12.709 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:12 smithi129 ceph-mon[101316]: from='client.36339 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:57:12.742 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-14T04:57:12.779 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:12.834 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:12 smithi039 ceph-mon[106470]: from='client.36339 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:57:12.834 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:12 smithi039 ceph-mon[111235]: from='client.36339 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:57:12.835 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:12.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:12.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:12.937 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:12.961 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:12.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:12.962 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:13.004 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:57:13.064 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:13.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:13.102 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:13.102 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:13.203 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:13.237 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:13.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:13.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:13.339 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:13.375 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:13.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:13.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:13.477 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:13.504 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:13.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:13.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:13.606 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:13.629 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:13.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:13.630 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:13.695 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 14 16 34473 34457 153.807 83.3125 0.00204142 0.00640031 2024-09-14T04:57:13.731 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:13.753 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:13.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:13.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:13.855 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:13.877 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:13.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:13.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:13.979 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:14.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:14.002 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:14.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:14.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:13 smithi129 ceph-mon[101316]: pgmap v2920: 99 pgs: 99 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 160 MiB/s wr, 2.56k op/s 2024-09-14T04:57:14.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:13 smithi039 ceph-mon[106470]: pgmap v2920: 99 pgs: 99 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 160 MiB/s wr, 2.56k op/s 2024-09-14T04:57:14.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:13 smithi039 ceph-mon[111235]: pgmap v2920: 99 pgs: 99 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 160 MiB/s wr, 2.56k op/s 2024-09-14T04:57:14.103 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:14.125 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:14.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:14.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:14.227 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:14.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:14.250 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:14.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:14.351 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:14.386 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:14.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:14.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:14.488 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:14.523 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:14.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:14.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:14.625 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:14.658 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:14.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:14.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:14.695 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 15 16 35526 35510 147.94 65.8125 0.00329759 0.0067536 2024-09-14T04:57:14.760 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:14.792 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:14.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:14.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:14.894 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:14.929 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:14.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:14.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:15.031 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:15.068 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:15.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:15.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:15.170 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:15.199 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:15.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:15.200 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:15.301 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:15.336 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:15.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:15.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:15.438 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:15.472 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:15.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:15.473 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:15.574 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:15.606 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:15.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:15.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:15.695 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 16 16 37389 37373 145.971 116.438 0.00441631 0.00684547 2024-09-14T04:57:15.709 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:15.749 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:15.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:15.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:15.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:15 smithi039 ceph-mon[106470]: pgmap v2921: 99 pgs: 99 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 175 MiB/s wr, 2.80k op/s 2024-09-14T04:57:15.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:15 smithi039 ceph-mon[111235]: pgmap v2921: 99 pgs: 99 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 175 MiB/s wr, 2.80k op/s 2024-09-14T04:57:15.842 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:15 smithi129 ceph-mon[101316]: pgmap v2921: 99 pgs: 99 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 175 MiB/s wr, 2.80k op/s 2024-09-14T04:57:15.851 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:15.887 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:15.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:15.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:15.989 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:16.032 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:16.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:16.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:16.082 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:57:16.082 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:57:16.134 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:16.169 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:16.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:16.170 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:16.271 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:16.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:16.309 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:16.310 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:16.341 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:57:16.342 INFO:teuthology.orchestra.run.smithi129.stdout: "size": 20, 2024-09-14T04:57:16.342 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 600, 2024-09-14T04:57:16.342 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [ 2024-09-14T04:57:16.342 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.342 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.45632.0:104264 55.8 55:1fc585f2:::benchmark_data_smithi039_297155_object104263:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1440)", 2024-09-14T04:57:16.342 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:49:10.240109+0000", 2024-09-14T04:57:16.343 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 486.09698645399999, 2024-09-14T04:57:16.343 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.39979108000000002, 2024-09-14T04:57:16.343 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:16.343 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:16.343 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:16.343 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.45632", 2024-09-14T04:57:16.343 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:57:16.343 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 104264 2024-09-14T04:57:16.343 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.343 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:16.343 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.343 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:16.344 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.240109+0000", 2024-09-14T04:57:16.344 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.344 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.344 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.344 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:16.344 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.240109+0000", 2024-09-14T04:57:16.344 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.344 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.344 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.344 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:16.344 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.240107+0000", 2024-09-14T04:57:16.344 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:57:16.344 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.345 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.345 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:16.345 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.240140+0000", 2024-09-14T04:57:16.345 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.2925000000000001e-05 2024-09-14T04:57:16.345 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.345 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.345 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:16.345 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.240153+0000", 2024-09-14T04:57:16.345 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3074000000000001e-05 2024-09-14T04:57:16.345 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.345 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.345 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:16.345 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.240160+0000", 2024-09-14T04:57:16.346 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.0439999999999998e-06 2024-09-14T04:57:16.346 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.346 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.346 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.346 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.240203+0000", 2024-09-14T04:57:16.346 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.2797999999999999e-05 2024-09-14T04:57:16.346 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.346 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.346 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:16.346 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.240306+0000", 2024-09-14T04:57:16.346 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000102959 2024-09-14T04:57:16.346 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.346 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.346 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,6", 2024-09-14T04:57:16.347 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.240386+0000", 2024-09-14T04:57:16.347 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.9819000000000003e-05 2024-09-14T04:57:16.347 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.347 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.347 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:16.347 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.322851+0000", 2024-09-14T04:57:16.347 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.082465262999999997 2024-09-14T04:57:16.347 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.347 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.347 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.347 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.322882+0000", 2024-09-14T04:57:16.347 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.0682000000000002e-05 2024-09-14T04:57:16.347 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.347 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.348 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.348 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.639772+0000", 2024-09-14T04:57:16.348 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.31689024900000001 2024-09-14T04:57:16.348 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.348 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.348 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:16.348 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.639859+0000", 2024-09-14T04:57:16.348 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.7014000000000004e-05 2024-09-14T04:57:16.348 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.348 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.348 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:16.349 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:49:10.639900+0000", 2024-09-14T04:57:16.349 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0748999999999999e-05 2024-09-14T04:57:16.349 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.349 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:16.349 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.349 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.349 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.349 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:116283 56.6 56:68220df0:::benchmark_data_smithi039_330847_object1759:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1486)", 2024-09-14T04:57:16.349 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:04.658536+0000", 2024-09-14T04:57:16.350 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 191.67855911699999, 2024-09-14T04:57:16.350 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.39605932399999999, 2024-09-14T04:57:16.350 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:16.350 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:16.350 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:16.350 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:16.350 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:16.350 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 116283 2024-09-14T04:57:16.350 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.350 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:16.350 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.350 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:16.350 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658536+0000", 2024-09-14T04:57:16.351 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.351 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.351 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.351 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:16.351 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658536+0000", 2024-09-14T04:57:16.351 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.351 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.351 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.351 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:16.351 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658535+0000", 2024-09-14T04:57:16.351 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:57:16.351 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.351 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.352 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:16.352 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658539+0000", 2024-09-14T04:57:16.352 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.3229999999999999e-06 2024-09-14T04:57:16.352 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.352 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.352 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:16.352 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658548+0000", 2024-09-14T04:57:16.352 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.9740000000000008e-06 2024-09-14T04:57:16.352 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.352 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.352 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:16.352 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658555+0000", 2024-09-14T04:57:16.352 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.5119999999999997e-06 2024-09-14T04:57:16.353 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.353 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.353 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.353 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.667029+0000", 2024-09-14T04:57:16.353 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0084737279999999998 2024-09-14T04:57:16.353 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.353 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.353 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:16.353 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.669660+0000", 2024-09-14T04:57:16.353 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0026312179999999998 2024-09-14T04:57:16.353 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.353 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.354 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 4,7", 2024-09-14T04:57:16.354 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.669768+0000", 2024-09-14T04:57:16.354 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010799299999999999 2024-09-14T04:57:16.354 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.354 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.354 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:16.354 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.677330+0000", 2024-09-14T04:57:16.354 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0075621170000000001 2024-09-14T04:57:16.354 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.354 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.354 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.354 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.677369+0000", 2024-09-14T04:57:16.354 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8958999999999997e-05 2024-09-14T04:57:16.354 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.355 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.355 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.355 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.048251+0000", 2024-09-14T04:57:16.355 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.37088180599999998 2024-09-14T04:57:16.355 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.355 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.355 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:16.355 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.048303+0000", 2024-09-14T04:57:16.355 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2689999999999999e-05 2024-09-14T04:57:16.355 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.355 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.355 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:16.355 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.054595+0000", 2024-09-14T04:57:16.355 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0062918959999999999 2024-09-14T04:57:16.356 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.356 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:16.356 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.356 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.356 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.356 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127522 56.2 56:44f888df:::benchmark_data_smithi039_330847_object12998:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1488)", 2024-09-14T04:57:16.356 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.438889+0000", 2024-09-14T04:57:16.356 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 177.89820659399999, 2024-09-14T04:57:16.356 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2135127569999999, 2024-09-14T04:57:16.357 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:16.357 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:16.357 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:16.357 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:16.357 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:16.357 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127522 2024-09-14T04:57:16.357 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.357 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:16.357 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.358 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:16.358 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.438889+0000", 2024-09-14T04:57:16.358 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.358 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.358 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.358 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:16.358 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.438889+0000", 2024-09-14T04:57:16.358 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.358 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.358 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.358 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:16.358 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.438887+0000", 2024-09-14T04:57:16.358 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:57:16.358 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.359 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.359 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:16.359 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.438894+0000", 2024-09-14T04:57:16.359 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.5629999999999997e-06 2024-09-14T04:57:16.359 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.359 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.359 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:16.359 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.438906+0000", 2024-09-14T04:57:16.359 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2109e-05 2024-09-14T04:57:16.359 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.360 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.360 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:16.360 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.438918+0000", 2024-09-14T04:57:16.360 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2082000000000001e-05 2024-09-14T04:57:16.360 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.360 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.361 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.361 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.444929+0000", 2024-09-14T04:57:16.361 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0060105920000000004 2024-09-14T04:57:16.361 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.361 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.361 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:57:16.361 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.444940+0000", 2024-09-14T04:57:16.361 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1686e-05 2024-09-14T04:57:16.361 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.361 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.362 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.362 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.613189+0000", 2024-09-14T04:57:16.362 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1682488360000001 2024-09-14T04:57:16.362 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.362 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.362 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:16.362 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.613341+0000", 2024-09-14T04:57:16.362 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00015144600000000001 2024-09-14T04:57:16.362 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.362 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.362 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:57:16.362 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.613493+0000", 2024-09-14T04:57:16.363 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00015271300000000001 2024-09-14T04:57:16.363 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.363 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.363 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.363 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.625395+0000", 2024-09-14T04:57:16.363 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.011901836000000001 2024-09-14T04:57:16.363 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.363 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.363 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:16.363 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.625408+0000", 2024-09-14T04:57:16.363 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2699e-05 2024-09-14T04:57:16.363 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.363 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.364 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.364 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.649879+0000", 2024-09-14T04:57:16.364 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.024471017000000001 2024-09-14T04:57:16.364 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.364 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.364 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:16.364 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.649944+0000", 2024-09-14T04:57:16.364 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.5482999999999996e-05 2024-09-14T04:57:16.364 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.364 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.364 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:16.364 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.652401+0000", 2024-09-14T04:57:16.364 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0024570270000000001 2024-09-14T04:57:16.365 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.365 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:16.365 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.365 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.365 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.365 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127527 56.2 56:4905adce:::benchmark_data_smithi039_330847_object13003:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1489)", 2024-09-14T04:57:16.365 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.455332+0000", 2024-09-14T04:57:16.365 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 177.88176367899999, 2024-09-14T04:57:16.365 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1993806579999999, 2024-09-14T04:57:16.365 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:16.365 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:16.365 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:16.365 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:16.366 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:16.366 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127527 2024-09-14T04:57:16.366 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.366 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:16.366 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.366 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:16.366 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455332+0000", 2024-09-14T04:57:16.366 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.366 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.366 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.366 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:16.366 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455332+0000", 2024-09-14T04:57:16.366 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.367 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.367 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.367 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:16.367 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455331+0000", 2024-09-14T04:57:16.367 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:57:16.367 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.367 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.367 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:16.367 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455336+0000", 2024-09-14T04:57:16.367 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.6239999999999999e-06 2024-09-14T04:57:16.367 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.367 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.367 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:16.367 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455350+0000", 2024-09-14T04:57:16.368 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3927999999999999e-05 2024-09-14T04:57:16.368 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.368 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.368 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:16.368 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455358+0000", 2024-09-14T04:57:16.368 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.6879999999999999e-06 2024-09-14T04:57:16.368 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.368 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.368 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.368 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455390+0000", 2024-09-14T04:57:16.368 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.2332999999999997e-05 2024-09-14T04:57:16.369 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.369 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.369 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:57:16.369 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455397+0000", 2024-09-14T04:57:16.369 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.3560000000000004e-06 2024-09-14T04:57:16.369 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.369 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.369 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.369 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.620502+0000", 2024-09-14T04:57:16.369 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1651044210000001 2024-09-14T04:57:16.369 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.369 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.369 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:16.369 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.620657+0000", 2024-09-14T04:57:16.370 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000155029 2024-09-14T04:57:16.370 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.370 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.370 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:57:16.370 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.620824+0000", 2024-09-14T04:57:16.370 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00016709099999999999 2024-09-14T04:57:16.370 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.370 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.370 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:16.370 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.633463+0000", 2024-09-14T04:57:16.370 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.012638870999999999 2024-09-14T04:57:16.370 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.370 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.371 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.371 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.633522+0000", 2024-09-14T04:57:16.371 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9033000000000002e-05 2024-09-14T04:57:16.371 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.371 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.371 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.371 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.652482+0000", 2024-09-14T04:57:16.371 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.018959796000000001 2024-09-14T04:57:16.371 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.371 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.371 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:16.371 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.652543+0000", 2024-09-14T04:57:16.371 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.0943000000000002e-05 2024-09-14T04:57:16.371 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.371 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.372 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:16.372 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.654712+0000", 2024-09-14T04:57:16.372 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0021695239999999999 2024-09-14T04:57:16.372 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.372 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:16.372 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.372 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.372 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.372 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127528 56.6 56:6f55fbf1:::benchmark_data_smithi039_330847_object13004:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1489)", 2024-09-14T04:57:16.372 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.455626+0000", 2024-09-14T04:57:16.372 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 177.881468934, 2024-09-14T04:57:16.372 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1469407549999999, 2024-09-14T04:57:16.372 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:16.373 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:16.373 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:16.373 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:16.374 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:16.374 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127528 2024-09-14T04:57:16.374 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.374 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:16.374 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.374 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:16.374 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455626+0000", 2024-09-14T04:57:16.374 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.374 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.374 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.374 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:16.374 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455626+0000", 2024-09-14T04:57:16.374 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.375 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.375 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.375 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:16.375 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455626+0000", 2024-09-14T04:57:16.375 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:57:16.375 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.375 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.375 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:16.376 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455629+0000", 2024-09-14T04:57:16.376 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.4510000000000001e-06 2024-09-14T04:57:16.376 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.376 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.376 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:16.376 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455636+0000", 2024-09-14T04:57:16.376 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8589999999999999e-06 2024-09-14T04:57:16.376 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.376 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.376 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:16.376 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455640+0000", 2024-09-14T04:57:16.376 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0799999999999999e-06 2024-09-14T04:57:16.376 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.376 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.377 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.377 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455674+0000", 2024-09-14T04:57:16.377 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3964000000000002e-05 2024-09-14T04:57:16.377 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.377 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.377 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:57:16.377 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455680+0000", 2024-09-14T04:57:16.377 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9259999999999997e-06 2024-09-14T04:57:16.377 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.377 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.377 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.377 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.589625+0000", 2024-09-14T04:57:16.377 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1339447499999999 2024-09-14T04:57:16.378 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.378 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.378 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:16.378 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.589735+0000", 2024-09-14T04:57:16.378 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000110177 2024-09-14T04:57:16.378 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.378 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.378 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:57:16.378 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.589828+0000", 2024-09-14T04:57:16.378 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.3558000000000001e-05 2024-09-14T04:57:16.378 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.378 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.378 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.379 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.599625+0000", 2024-09-14T04:57:16.379 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0097969819999999992 2024-09-14T04:57:16.379 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.379 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.379 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:16.379 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.599635+0000", 2024-09-14T04:57:16.379 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.9680000000000001e-06 2024-09-14T04:57:16.379 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.379 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.379 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.379 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.599742+0000", 2024-09-14T04:57:16.379 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000106397 2024-09-14T04:57:16.379 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.379 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.380 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:16.380 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.599801+0000", 2024-09-14T04:57:16.380 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.8866999999999997e-05 2024-09-14T04:57:16.380 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.380 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.380 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:16.380 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.602567+0000", 2024-09-14T04:57:16.380 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0027663470000000002 2024-09-14T04:57:16.380 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.380 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:16.380 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.380 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.380 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.381 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127537 56.2 56:49dbca7a:::benchmark_data_smithi039_330847_object13013:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1489)", 2024-09-14T04:57:16.381 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.470996+0000", 2024-09-14T04:57:16.381 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 177.866099222, 2024-09-14T04:57:16.381 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1875430259999999, 2024-09-14T04:57:16.381 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:16.381 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:16.381 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:16.381 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:16.381 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:16.381 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127537 2024-09-14T04:57:16.381 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.381 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:16.381 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.381 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:16.382 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.470996+0000", 2024-09-14T04:57:16.382 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.382 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.382 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.382 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:16.382 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.470996+0000", 2024-09-14T04:57:16.382 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.382 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.382 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.382 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:16.382 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.470994+0000", 2024-09-14T04:57:16.382 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:57:16.382 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.382 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.383 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:16.383 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.471001+0000", 2024-09-14T04:57:16.383 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7150000000000001e-06 2024-09-14T04:57:16.383 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.383 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.383 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:16.383 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.471016+0000", 2024-09-14T04:57:16.383 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5177999999999999e-05 2024-09-14T04:57:16.383 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.383 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.383 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:16.383 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.471023+0000", 2024-09-14T04:57:16.383 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.3269999999999997e-06 2024-09-14T04:57:16.383 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.383 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.384 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.384 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.471125+0000", 2024-09-14T04:57:16.384 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000102396 2024-09-14T04:57:16.384 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.384 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.384 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:57:16.384 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.471138+0000", 2024-09-14T04:57:16.384 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3407000000000001e-05 2024-09-14T04:57:16.384 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.385 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.385 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.385 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.625446+0000", 2024-09-14T04:57:16.385 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1543079039999999 2024-09-14T04:57:16.385 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.385 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.385 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:16.385 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.625558+0000", 2024-09-14T04:57:16.385 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000112037 2024-09-14T04:57:16.385 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.385 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.386 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:57:16.386 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.625652+0000", 2024-09-14T04:57:16.386 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.3692999999999999e-05 2024-09-14T04:57:16.386 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.386 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.386 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:16.386 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.637636+0000", 2024-09-14T04:57:16.386 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.011983705000000001 2024-09-14T04:57:16.386 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.386 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.386 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.387 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.641298+0000", 2024-09-14T04:57:16.387 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.003662579 2024-09-14T04:57:16.387 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.387 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.387 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.387 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.654763+0000", 2024-09-14T04:57:16.387 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.013464557 2024-09-14T04:57:16.387 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.387 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.387 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:16.387 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.654847+0000", 2024-09-14T04:57:16.387 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.4201000000000002e-05 2024-09-14T04:57:16.388 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.388 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.388 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:16.388 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.658539+0000", 2024-09-14T04:57:16.388 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0036919449999999999 2024-09-14T04:57:16.388 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.388 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:16.388 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.388 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.388 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.388 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127538 56.2 56:4c76f10c:::benchmark_data_smithi039_330847_object13014:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1489)", 2024-09-14T04:57:16.388 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.471037+0000", 2024-09-14T04:57:16.388 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 177.86605798100001, 2024-09-14T04:57:16.388 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1897702779999999, 2024-09-14T04:57:16.389 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:16.389 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:16.389 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:16.389 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:16.389 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:16.389 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127538 2024-09-14T04:57:16.389 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.389 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:16.389 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.389 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:16.389 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.471037+0000", 2024-09-14T04:57:16.390 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.390 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.390 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.390 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:16.390 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.471037+0000", 2024-09-14T04:57:16.390 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.390 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.390 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.390 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:16.390 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.471037+0000", 2024-09-14T04:57:16.390 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:57:16.390 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.390 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.391 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:16.391 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.471040+0000", 2024-09-14T04:57:16.391 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.151e-06 2024-09-14T04:57:16.391 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.391 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.391 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:16.391 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.471045+0000", 2024-09-14T04:57:16.391 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2979999999999999e-06 2024-09-14T04:57:16.391 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.391 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.392 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:16.392 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.471051+0000", 2024-09-14T04:57:16.392 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.3750000000000002e-06 2024-09-14T04:57:16.392 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.392 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.392 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.392 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.471183+0000", 2024-09-14T04:57:16.392 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013227700000000001 2024-09-14T04:57:16.392 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.392 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.392 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:57:16.393 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.471194+0000", 2024-09-14T04:57:16.393 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0570999999999999e-05 2024-09-14T04:57:16.393 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.393 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.393 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.393 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.629442+0000", 2024-09-14T04:57:16.393 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1582482270000001 2024-09-14T04:57:16.393 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.393 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.393 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:16.393 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.629517+0000", 2024-09-14T04:57:16.393 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.5038999999999998e-05 2024-09-14T04:57:16.393 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.394 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.394 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:57:16.394 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.629586+0000", 2024-09-14T04:57:16.394 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8867000000000003e-05 2024-09-14T04:57:16.394 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.394 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.395 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.395 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.641338+0000", 2024-09-14T04:57:16.395 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.011751914 2024-09-14T04:57:16.395 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.395 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.395 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:16.395 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.645510+0000", 2024-09-14T04:57:16.395 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0041725180000000001 2024-09-14T04:57:16.395 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.395 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.395 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.395 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.658598+0000", 2024-09-14T04:57:16.395 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.013088136 2024-09-14T04:57:16.396 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.396 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.396 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:16.396 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.658650+0000", 2024-09-14T04:57:16.396 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.1558999999999999e-05 2024-09-14T04:57:16.396 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.396 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.396 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:16.396 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.660807+0000", 2024-09-14T04:57:16.396 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0021577010000000002 2024-09-14T04:57:16.396 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.396 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:16.396 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.396 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.397 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.397 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127543 56.2 56:4af254b3:::benchmark_data_smithi039_330847_object13019:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1489)", 2024-09-14T04:57:16.397 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.478195+0000", 2024-09-14T04:57:16.397 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 177.858900425, 2024-09-14T04:57:16.397 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1849122379999999, 2024-09-14T04:57:16.397 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:16.397 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:16.397 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:16.397 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:16.397 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:16.397 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127543 2024-09-14T04:57:16.397 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.397 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:16.398 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.398 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:16.398 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.478195+0000", 2024-09-14T04:57:16.398 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.398 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.398 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.398 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:16.398 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.478195+0000", 2024-09-14T04:57:16.398 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.398 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.398 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.398 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:16.398 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.478193+0000", 2024-09-14T04:57:16.398 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:57:16.399 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.399 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.399 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:16.399 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.478200+0000", 2024-09-14T04:57:16.399 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.5830000000000003e-06 2024-09-14T04:57:16.399 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.399 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.399 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:16.399 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.478213+0000", 2024-09-14T04:57:16.399 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3258000000000001e-05 2024-09-14T04:57:16.399 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.399 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.399 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:16.399 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.478220+0000", 2024-09-14T04:57:16.400 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7530000000000004e-06 2024-09-14T04:57:16.400 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.400 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.400 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.400 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.486363+0000", 2024-09-14T04:57:16.400 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0081430289999999995 2024-09-14T04:57:16.400 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.400 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.400 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:57:16.400 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.486371+0000", 2024-09-14T04:57:16.400 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.1669999999999999e-06 2024-09-14T04:57:16.401 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.401 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.401 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.401 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.633559+0000", 2024-09-14T04:57:16.401 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1471879659999999 2024-09-14T04:57:16.401 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.401 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.401 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:16.401 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.633692+0000", 2024-09-14T04:57:16.401 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000133258 2024-09-14T04:57:16.401 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.402 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.402 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:57:16.402 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.633830+0000", 2024-09-14T04:57:16.402 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013777400000000001 2024-09-14T04:57:16.402 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.402 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.402 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:16.402 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.649665+0000", 2024-09-14T04:57:16.402 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.015834339999999999 2024-09-14T04:57:16.402 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.402 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.402 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.402 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.649724+0000", 2024-09-14T04:57:16.402 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9301e-05 2024-09-14T04:57:16.403 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.403 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.403 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.403 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.660852+0000", 2024-09-14T04:57:16.403 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.011128645 2024-09-14T04:57:16.403 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.403 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.403 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:16.403 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.660893+0000", 2024-09-14T04:57:16.403 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0447000000000001e-05 2024-09-14T04:57:16.403 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.403 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.403 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:16.404 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.663107+0000", 2024-09-14T04:57:16.404 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0022140699999999998 2024-09-14T04:57:16.404 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.404 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:16.404 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.404 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.404 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.404 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127552 56.2 56:4d13a696:::benchmark_data_smithi039_330847_object13028:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1489)", 2024-09-14T04:57:16.404 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.492573+0000", 2024-09-14T04:57:16.404 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 177.84452228000001, 2024-09-14T04:57:16.404 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1725031699999999, 2024-09-14T04:57:16.404 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:16.404 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:16.405 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:16.405 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:16.405 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:16.405 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127552 2024-09-14T04:57:16.405 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.405 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:16.405 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.405 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:16.405 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.492573+0000", 2024-09-14T04:57:16.405 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.405 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.405 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.405 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:16.405 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.492573+0000", 2024-09-14T04:57:16.406 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.406 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.406 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.406 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:16.406 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.492572+0000", 2024-09-14T04:57:16.406 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:57:16.406 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.406 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.406 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:16.406 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.492576+0000", 2024-09-14T04:57:16.406 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1250000000000003e-06 2024-09-14T04:57:16.406 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.406 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.406 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:16.407 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.492590+0000", 2024-09-14T04:57:16.407 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3733999999999999e-05 2024-09-14T04:57:16.407 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.407 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.407 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:16.407 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.492596+0000", 2024-09-14T04:57:16.407 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.3609999999999997e-06 2024-09-14T04:57:16.407 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.407 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.407 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.407 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.492682+0000", 2024-09-14T04:57:16.407 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.5546999999999999e-05 2024-09-14T04:57:16.408 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.408 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.408 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:57:16.408 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.492693+0000", 2024-09-14T04:57:16.408 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0832000000000001e-05 2024-09-14T04:57:16.408 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.408 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.408 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.408 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.637679+0000", 2024-09-14T04:57:16.409 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1449862200000001 2024-09-14T04:57:16.409 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.409 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.409 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:16.409 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.637797+0000", 2024-09-14T04:57:16.409 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011794 2024-09-14T04:57:16.409 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.409 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.409 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:57:16.409 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.637882+0000", 2024-09-14T04:57:16.409 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.4902999999999997e-05 2024-09-14T04:57:16.409 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.410 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.410 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:16.410 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.649672+0000", 2024-09-14T04:57:16.410 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.011790063999999999 2024-09-14T04:57:16.410 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.410 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.410 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.410 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.649826+0000", 2024-09-14T04:57:16.410 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00015375499999999999 2024-09-14T04:57:16.410 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.410 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.410 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.410 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.663158+0000", 2024-09-14T04:57:16.410 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.013332557 2024-09-14T04:57:16.410 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.411 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.411 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:16.411 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.663210+0000", 2024-09-14T04:57:16.411 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.1499000000000003e-05 2024-09-14T04:57:16.411 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.411 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.411 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:16.411 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.665076+0000", 2024-09-14T04:57:16.411 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001866468 2024-09-14T04:57:16.411 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.411 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:16.411 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.411 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.411 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.412 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127564 56.2 56:43bae9c1:::benchmark_data_smithi039_330847_object13040:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1489)", 2024-09-14T04:57:16.412 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.507445+0000", 2024-09-14T04:57:16.412 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 177.82965049399999, 2024-09-14T04:57:16.412 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1591943979999999, 2024-09-14T04:57:16.412 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:16.412 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:16.412 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:16.412 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:16.412 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:16.412 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127564 2024-09-14T04:57:16.412 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.412 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:16.412 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.412 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:16.412 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.507445+0000", 2024-09-14T04:57:16.413 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.413 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.413 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.413 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:16.413 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.507445+0000", 2024-09-14T04:57:16.413 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.413 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.413 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.413 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:16.413 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.507444+0000", 2024-09-14T04:57:16.413 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:57:16.413 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.413 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.414 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:16.414 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.507448+0000", 2024-09-14T04:57:16.414 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5710000000000002e-06 2024-09-14T04:57:16.414 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.414 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.414 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:16.414 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.507456+0000", 2024-09-14T04:57:16.414 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.9610000000000005e-06 2024-09-14T04:57:16.414 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.414 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.414 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:16.414 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.507461+0000", 2024-09-14T04:57:16.414 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.4879999999999998e-06 2024-09-14T04:57:16.414 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.415 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.415 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.415 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.507553+0000", 2024-09-14T04:57:16.415 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.1432000000000001e-05 2024-09-14T04:57:16.415 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.415 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.415 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:57:16.415 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.507563+0000", 2024-09-14T04:57:16.415 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0376e-05 2024-09-14T04:57:16.415 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.415 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.415 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.415 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.641377+0000", 2024-09-14T04:57:16.415 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1338141070000001 2024-09-14T04:57:16.415 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.416 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.416 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:16.416 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.641506+0000", 2024-09-14T04:57:16.416 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012888400000000001 2024-09-14T04:57:16.416 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.416 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.416 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:57:16.416 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.641627+0000", 2024-09-14T04:57:16.416 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012091600000000001 2024-09-14T04:57:16.416 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.416 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.416 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:16.416 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.652420+0000", 2024-09-14T04:57:16.417 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.010792708999999999 2024-09-14T04:57:16.417 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.417 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.417 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.417 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.665130+0000", 2024-09-14T04:57:16.417 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.012710116 2024-09-14T04:57:16.417 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.417 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.417 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.417 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.665215+0000", 2024-09-14T04:57:16.417 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.5507999999999995e-05 2024-09-14T04:57:16.418 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.418 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.418 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:16.418 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.665270+0000", 2024-09-14T04:57:16.418 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.4877999999999999e-05 2024-09-14T04:57:16.418 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.418 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.418 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:16.418 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.666639+0000", 2024-09-14T04:57:16.418 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0013690390000000001 2024-09-14T04:57:16.418 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.418 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:16.418 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.419 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.419 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.419 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:16.420 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127575 56.2 56:4a96f688:::benchmark_data_smithi039_330847_object13051:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1489)", 2024-09-14T04:57:16.421 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.518289+0000", 2024-09-14T04:57:16.421 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 177.81880571299999, 2024-09-14T04:57:16.421 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1500041139999999, 2024-09-14T04:57:16.421 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:16.421 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:16.421 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:16.421 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:16.421 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:16.421 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127575 2024-09-14T04:57:16.421 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.422 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:16.422 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.422 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:16.422 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.518289+0000", 2024-09-14T04:57:16.422 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.422 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.422 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.422 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:16.422 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.518289+0000", 2024-09-14T04:57:16.422 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.422 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.422 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.422 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:16.422 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.518288+0000", 2024-09-14T04:57:16.422 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:57:16.423 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.423 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.423 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:16.423 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.518293+0000", 2024-09-14T04:57:16.423 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.4649999999999996e-06 2024-09-14T04:57:16.423 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.423 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.423 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:16.423 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.518303+0000", 2024-09-14T04:57:16.423 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.9599999999999995e-06 2024-09-14T04:57:16.423 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.423 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.423 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:16.424 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.518308+0000", 2024-09-14T04:57:16.424 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7249999999999997e-06 2024-09-14T04:57:16.424 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.424 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.424 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.424 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.518352+0000", 2024-09-14T04:57:16.424 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.4205000000000001e-05 2024-09-14T04:57:16.425 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.425 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.425 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:57:16.425 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.518358+0000", 2024-09-14T04:57:16.425 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7109999999999998e-06 2024-09-14T04:57:16.425 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.425 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.425 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.425 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.645538+0000", 2024-09-14T04:57:16.425 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.127179285 2024-09-14T04:57:16.425 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.425 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.426 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:16.426 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.645748+0000", 2024-09-14T04:57:16.426 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000210421 2024-09-14T04:57:16.426 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.426 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.426 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:57:16.426 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.645863+0000", 2024-09-14T04:57:16.426 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000114912 2024-09-14T04:57:16.426 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.426 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.426 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:16.426 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.658557+0000", 2024-09-14T04:57:16.426 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.012694268999999999 2024-09-14T04:57:16.426 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.427 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.427 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.427 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.665171+0000", 2024-09-14T04:57:16.427 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0066135430000000004 2024-09-14T04:57:16.427 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.427 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.427 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.427 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.666697+0000", 2024-09-14T04:57:16.427 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0015255869999999999 2024-09-14T04:57:16.427 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.427 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.427 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:16.427 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.666730+0000", 2024-09-14T04:57:16.428 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3291999999999999e-05 2024-09-14T04:57:16.428 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.428 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.428 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:16.428 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.668294+0000", 2024-09-14T04:57:16.428 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001563785 2024-09-14T04:57:16.428 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.428 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:16.428 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.428 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.428 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.429 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127577 56.2 56:42aa155f:::benchmark_data_smithi039_330847_object13053:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1489)", 2024-09-14T04:57:16.429 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.522306+0000", 2024-09-14T04:57:16.429 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 177.81478952200001, 2024-09-14T04:57:16.429 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1678563989999999, 2024-09-14T04:57:16.429 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:16.429 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:16.429 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:16.429 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:16.429 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:16.429 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127577 2024-09-14T04:57:16.429 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.429 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:16.430 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.430 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:16.430 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522306+0000", 2024-09-14T04:57:16.430 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.430 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.430 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.431 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:16.431 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522306+0000", 2024-09-14T04:57:16.431 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.431 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.431 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.431 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:16.431 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522304+0000", 2024-09-14T04:57:16.431 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:57:16.431 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.431 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.431 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:16.432 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522310+0000", 2024-09-14T04:57:16.432 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.7010000000000001e-06 2024-09-14T04:57:16.432 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.432 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.432 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:16.432 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522325+0000", 2024-09-14T04:57:16.432 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.4766999999999999e-05 2024-09-14T04:57:16.432 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.432 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.432 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:16.433 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522332+0000", 2024-09-14T04:57:16.433 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.6930000000000001e-06 2024-09-14T04:57:16.433 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.433 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.433 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.433 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522403+0000", 2024-09-14T04:57:16.434 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.1266999999999994e-05 2024-09-14T04:57:16.434 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.434 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.434 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:57:16.434 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522414+0000", 2024-09-14T04:57:16.435 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1027e-05 2024-09-14T04:57:16.435 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.435 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.435 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.435 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.668337+0000", 2024-09-14T04:57:16.435 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.145923534 2024-09-14T04:57:16.435 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.435 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.436 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:16.436 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.668440+0000", 2024-09-14T04:57:16.436 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000102973 2024-09-14T04:57:16.436 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.436 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.436 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:57:16.436 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.668572+0000", 2024-09-14T04:57:16.436 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013213 2024-09-14T04:57:16.436 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.436 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.437 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.437 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.679226+0000", 2024-09-14T04:57:16.437 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.010653986000000001 2024-09-14T04:57:16.437 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.437 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.437 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:16.438 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.679236+0000", 2024-09-14T04:57:16.438 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.2669999999999995e-06 2024-09-14T04:57:16.438 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.438 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.438 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.438 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.687737+0000", 2024-09-14T04:57:16.438 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0085016450000000004 2024-09-14T04:57:16.438 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.438 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.438 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:16.438 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.687790+0000", 2024-09-14T04:57:16.439 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2457000000000003e-05 2024-09-14T04:57:16.439 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.439 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.439 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:16.439 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.690162+0000", 2024-09-14T04:57:16.439 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0023722370000000001 2024-09-14T04:57:16.439 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.439 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:16.439 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.439 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.439 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.439 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127578 56.2 56:46633f16:::benchmark_data_smithi039_330847_object13054:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1489)", 2024-09-14T04:57:16.440 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.522479+0000", 2024-09-14T04:57:16.440 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 177.81461628, 2024-09-14T04:57:16.440 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1704580840000001, 2024-09-14T04:57:16.440 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:16.440 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:16.440 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:16.440 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:16.440 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:16.440 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127578 2024-09-14T04:57:16.440 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.440 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:16.441 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.441 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:16.441 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522479+0000", 2024-09-14T04:57:16.441 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.441 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.441 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.441 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:16.441 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522479+0000", 2024-09-14T04:57:16.441 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.441 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.441 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.441 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:16.441 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522478+0000", 2024-09-14T04:57:16.441 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:57:16.442 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.442 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.442 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:16.442 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522482+0000", 2024-09-14T04:57:16.442 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9689999999999996e-06 2024-09-14T04:57:16.442 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.442 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.442 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:16.442 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522492+0000", 2024-09-14T04:57:16.442 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.3570000000000004e-06 2024-09-14T04:57:16.442 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.442 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.442 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:16.443 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522497+0000", 2024-09-14T04:57:16.443 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.6829999999999996e-06 2024-09-14T04:57:16.443 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.443 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.443 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.443 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522542+0000", 2024-09-14T04:57:16.443 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.4907000000000002e-05 2024-09-14T04:57:16.443 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.443 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.443 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:57:16.443 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522553+0000", 2024-09-14T04:57:16.443 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1228e-05 2024-09-14T04:57:16.443 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.443 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.444 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.444 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.673544+0000", 2024-09-14T04:57:16.444 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1509901760000001 2024-09-14T04:57:16.444 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.444 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.444 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:16.444 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.673667+0000", 2024-09-14T04:57:16.444 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012314199999999999 2024-09-14T04:57:16.444 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.444 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.444 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:57:16.444 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.673795+0000", 2024-09-14T04:57:16.444 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012876900000000001 2024-09-14T04:57:16.444 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.445 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.445 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.445 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.679272+0000", 2024-09-14T04:57:16.445 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0054765550000000001 2024-09-14T04:57:16.445 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.445 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.445 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:16.445 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.687680+0000", 2024-09-14T04:57:16.445 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0084078009999999995 2024-09-14T04:57:16.445 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.445 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.445 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.445 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.690246+0000", 2024-09-14T04:57:16.445 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0025666399999999998 2024-09-14T04:57:16.446 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.446 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.446 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:16.446 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.690354+0000", 2024-09-14T04:57:16.446 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000108007 2024-09-14T04:57:16.446 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.446 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.446 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:16.446 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.692937+0000", 2024-09-14T04:57:16.446 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0025824910000000001 2024-09-14T04:57:16.446 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.446 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:16.446 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.446 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.447 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.447 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127583 56.6 56:614096e5:::benchmark_data_smithi039_330847_object13059:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1489)", 2024-09-14T04:57:16.447 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.527314+0000", 2024-09-14T04:57:16.447 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 177.80978112099999, 2024-09-14T04:57:16.447 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.077654313, 2024-09-14T04:57:16.447 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:16.447 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:16.447 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:16.447 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:16.447 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:16.447 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127583 2024-09-14T04:57:16.447 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.447 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:16.447 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.448 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:16.448 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.527314+0000", 2024-09-14T04:57:16.448 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.448 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.448 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.448 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:16.448 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.527314+0000", 2024-09-14T04:57:16.448 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.448 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.448 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.448 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:16.448 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.527314+0000", 2024-09-14T04:57:16.448 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:57:16.449 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.449 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.449 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:16.449 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.527316+0000", 2024-09-14T04:57:16.449 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.892e-06 2024-09-14T04:57:16.450 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.450 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.450 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:16.450 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.527323+0000", 2024-09-14T04:57:16.450 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.5520000000000001e-06 2024-09-14T04:57:16.450 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.451 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.451 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:16.451 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.527328+0000", 2024-09-14T04:57:16.451 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.4600000000000002e-06 2024-09-14T04:57:16.451 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.452 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.452 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.452 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.527380+0000", 2024-09-14T04:57:16.452 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.1904999999999999e-05 2024-09-14T04:57:16.452 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.452 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.452 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:57:16.452 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.527390+0000", 2024-09-14T04:57:16.452 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.4420000000000004e-06 2024-09-14T04:57:16.452 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.452 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.453 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.453 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.595448+0000", 2024-09-14T04:57:16.453 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.068058261 2024-09-14T04:57:16.453 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.453 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.453 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:16.453 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.595589+0000", 2024-09-14T04:57:16.453 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00014102 2024-09-14T04:57:16.453 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.453 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.453 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:57:16.454 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.595720+0000", 2024-09-14T04:57:16.454 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000130786 2024-09-14T04:57:16.454 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.454 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.454 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.454 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.602659+0000", 2024-09-14T04:57:16.454 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0069390529999999997 2024-09-14T04:57:16.454 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.454 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.454 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.454 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.602795+0000", 2024-09-14T04:57:16.454 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013638499999999999 2024-09-14T04:57:16.454 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.454 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.455 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:16.455 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.602853+0000", 2024-09-14T04:57:16.455 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.7349000000000003e-05 2024-09-14T04:57:16.455 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.455 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.455 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:16.455 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.602919+0000", 2024-09-14T04:57:16.455 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.6544999999999994e-05 2024-09-14T04:57:16.455 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.455 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.455 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:16.455 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.604968+0000", 2024-09-14T04:57:16.455 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0020491929999999999 2024-09-14T04:57:16.455 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.456 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:16.456 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.456 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.456 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.456 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127591 56.2 56:412cff32:::benchmark_data_smithi039_330847_object13067:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1489)", 2024-09-14T04:57:16.456 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.541493+0000", 2024-09-14T04:57:16.456 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 177.795602274, 2024-09-14T04:57:16.456 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1539498459999999, 2024-09-14T04:57:16.456 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:16.456 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:16.456 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:16.456 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:16.456 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:16.457 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127591 2024-09-14T04:57:16.457 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.457 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:16.457 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.457 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:16.457 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.541493+0000", 2024-09-14T04:57:16.457 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.457 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.457 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.457 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:16.457 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.541493+0000", 2024-09-14T04:57:16.457 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.457 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.457 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.458 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:16.458 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.541491+0000", 2024-09-14T04:57:16.458 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:57:16.458 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.458 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.458 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:16.458 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.541498+0000", 2024-09-14T04:57:16.458 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7649999999999996e-06 2024-09-14T04:57:16.458 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.458 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.458 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:16.459 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.541511+0000", 2024-09-14T04:57:16.459 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3053e-05 2024-09-14T04:57:16.459 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.459 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.459 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:16.459 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.541517+0000", 2024-09-14T04:57:16.459 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.6370000000000002e-06 2024-09-14T04:57:16.459 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.459 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.459 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.459 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.541578+0000", 2024-09-14T04:57:16.459 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.0454000000000001e-05 2024-09-14T04:57:16.459 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.460 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.460 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:57:16.460 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.541590+0000", 2024-09-14T04:57:16.460 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2174000000000001e-05 2024-09-14T04:57:16.460 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.460 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.460 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.460 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.679299+0000", 2024-09-14T04:57:16.460 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1377089200000001 2024-09-14T04:57:16.460 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.461 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.461 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:16.461 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.679416+0000", 2024-09-14T04:57:16.461 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000116958 2024-09-14T04:57:16.461 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.461 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.461 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:57:16.461 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.679535+0000", 2024-09-14T04:57:16.461 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011906200000000001 2024-09-14T04:57:16.461 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.461 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.461 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:16.462 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.692967+0000", 2024-09-14T04:57:16.462 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.013431764000000001 2024-09-14T04:57:16.462 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.462 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.462 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.462 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.693051+0000", 2024-09-14T04:57:16.462 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.4680999999999998e-05 2024-09-14T04:57:16.463 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.463 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.463 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.463 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.693370+0000", 2024-09-14T04:57:16.463 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00031845700000000002 2024-09-14T04:57:16.463 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.463 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.463 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:16.463 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.693432+0000", 2024-09-14T04:57:16.463 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.2108999999999998e-05 2024-09-14T04:57:16.463 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.463 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.463 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:16.464 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.695443+0000", 2024-09-14T04:57:16.464 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0020107110000000001 2024-09-14T04:57:16.464 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.464 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:16.464 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.464 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.464 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.464 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127593 56.2 56:4e62fc54:::benchmark_data_smithi039_330847_object13069:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1489)", 2024-09-14T04:57:16.464 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.544475+0000", 2024-09-14T04:57:16.464 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 177.79262019199999, 2024-09-14T04:57:16.464 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.152692112, 2024-09-14T04:57:16.465 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:16.465 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:16.465 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:16.465 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:16.465 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:16.465 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127593 2024-09-14T04:57:16.465 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.465 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:16.465 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.465 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:16.465 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.544475+0000", 2024-09-14T04:57:16.465 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.465 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.465 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.466 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:16.466 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.544475+0000", 2024-09-14T04:57:16.466 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.466 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.466 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.466 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:16.466 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.544474+0000", 2024-09-14T04:57:16.466 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:57:16.466 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.466 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.466 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:16.466 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.544482+0000", 2024-09-14T04:57:16.466 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.6790000000000005e-06 2024-09-14T04:57:16.466 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.467 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.467 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:16.467 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.544494+0000", 2024-09-14T04:57:16.467 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2109999999999999e-05 2024-09-14T04:57:16.467 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.467 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.467 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:16.467 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.544499+0000", 2024-09-14T04:57:16.467 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.1919999999999996e-06 2024-09-14T04:57:16.467 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.467 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.467 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.467 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.544573+0000", 2024-09-14T04:57:16.467 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.3832999999999996e-05 2024-09-14T04:57:16.468 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.468 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.468 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:57:16.468 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.544581+0000", 2024-09-14T04:57:16.468 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.5150000000000006e-06 2024-09-14T04:57:16.468 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.468 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.468 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.468 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.683535+0000", 2024-09-14T04:57:16.468 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1389538029999999 2024-09-14T04:57:16.468 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.468 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.468 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:16.469 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.683644+0000", 2024-09-14T04:57:16.469 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000108785 2024-09-14T04:57:16.469 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.469 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.469 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:57:16.469 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.683753+0000", 2024-09-14T04:57:16.469 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010919199999999999 2024-09-14T04:57:16.469 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.469 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.469 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.469 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.693113+0000", 2024-09-14T04:57:16.469 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0093597039999999999 2024-09-14T04:57:16.470 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.470 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.470 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.470 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.695511+0000", 2024-09-14T04:57:16.470 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0023985259999999998 2024-09-14T04:57:16.470 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.470 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.470 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:16.470 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.695522+0000", 2024-09-14T04:57:16.470 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0961000000000001e-05 2024-09-14T04:57:16.470 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.470 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.470 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:16.470 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.695580+0000", 2024-09-14T04:57:16.471 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.7333000000000001e-05 2024-09-14T04:57:16.471 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.471 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.471 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:16.471 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.697167+0000", 2024-09-14T04:57:16.471 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0015874929999999999 2024-09-14T04:57:16.471 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.471 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:16.471 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.471 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.471 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.471 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127614 56.2 56:49d857b3:::benchmark_data_smithi039_330847_object13090:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1489)", 2024-09-14T04:57:16.471 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.599580+0000", 2024-09-14T04:57:16.472 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 177.73751542400001, 2024-09-14T04:57:16.472 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.118777608, 2024-09-14T04:57:16.472 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:16.472 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:16.472 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:16.472 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:16.472 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:16.472 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127614 2024-09-14T04:57:16.472 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.472 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:16.472 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.472 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:16.472 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.599580+0000", 2024-09-14T04:57:16.472 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.473 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.473 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.473 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:16.473 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.599580+0000", 2024-09-14T04:57:16.473 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.473 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.473 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.473 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:16.473 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.599579+0000", 2024-09-14T04:57:16.473 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:57:16.473 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.473 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.473 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:16.474 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.599584+0000", 2024-09-14T04:57:16.474 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6410000000000001e-06 2024-09-14T04:57:16.474 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.474 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.474 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:16.474 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.599593+0000", 2024-09-14T04:57:16.474 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.9670000000000007e-06 2024-09-14T04:57:16.474 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.474 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.474 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:16.475 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.599597+0000", 2024-09-14T04:57:16.475 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7260000000000002e-06 2024-09-14T04:57:16.475 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.475 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.475 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.475 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.599644+0000", 2024-09-14T04:57:16.475 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6816e-05 2024-09-14T04:57:16.475 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.475 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.475 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:57:16.475 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.599653+0000", 2024-09-14T04:57:16.475 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.9199999999999993e-06 2024-09-14T04:57:16.475 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.476 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.476 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.476 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.697216+0000", 2024-09-14T04:57:16.476 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0975626919999999 2024-09-14T04:57:16.476 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.476 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.476 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:16.476 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.697336+0000", 2024-09-14T04:57:16.476 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000119826 2024-09-14T04:57:16.476 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.476 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.476 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:57:16.477 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.697462+0000", 2024-09-14T04:57:16.477 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000126728 2024-09-14T04:57:16.477 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.477 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.477 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:16.477 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.709739+0000", 2024-09-14T04:57:16.477 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.012276665000000001 2024-09-14T04:57:16.477 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.477 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.477 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.477 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.713756+0000", 2024-09-14T04:57:16.477 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0040171970000000001 2024-09-14T04:57:16.477 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.477 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.478 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.478 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.713795+0000", 2024-09-14T04:57:16.478 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9113000000000003e-05 2024-09-14T04:57:16.478 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.478 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.478 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:16.478 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.713903+0000", 2024-09-14T04:57:16.478 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000107394 2024-09-14T04:57:16.478 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.478 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.478 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:16.478 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.718357+0000", 2024-09-14T04:57:16.478 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0044547609999999998 2024-09-14T04:57:16.479 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.479 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:16.479 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.479 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.479 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.479 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127629 56.2 56:44c02451:::benchmark_data_smithi039_330847_object13105:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1489)", 2024-09-14T04:57:16.479 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.661787+0000", 2024-09-14T04:57:16.479 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 177.67530828400001, 2024-09-14T04:57:16.479 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0597727610000001, 2024-09-14T04:57:16.479 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:16.479 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:16.479 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:16.479 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:16.479 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:16.480 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127629 2024-09-14T04:57:16.480 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.480 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:16.480 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.480 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:16.480 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.661787+0000", 2024-09-14T04:57:16.480 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.480 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.480 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.480 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:16.480 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.661787+0000", 2024-09-14T04:57:16.480 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.480 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.481 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.481 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:16.481 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.661786+0000", 2024-09-14T04:57:16.481 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:57:16.481 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.481 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.481 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:16.481 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.661791+0000", 2024-09-14T04:57:16.481 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.0769999999999998e-06 2024-09-14T04:57:16.481 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.481 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.481 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:16.481 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.661801+0000", 2024-09-14T04:57:16.481 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0122000000000001e-05 2024-09-14T04:57:16.482 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.482 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.482 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:16.482 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.661806+0000", 2024-09-14T04:57:16.482 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.0470000000000001e-06 2024-09-14T04:57:16.482 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.482 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.482 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.482 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.661860+0000", 2024-09-14T04:57:16.482 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.4083999999999999e-05 2024-09-14T04:57:16.482 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.482 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.482 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T04:57:16.482 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.661873+0000", 2024-09-14T04:57:16.483 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2429000000000001e-05 2024-09-14T04:57:16.483 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.483 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.483 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.483 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.700879+0000", 2024-09-14T04:57:16.483 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0390063409999999 2024-09-14T04:57:16.483 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.483 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.483 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:16.483 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.700986+0000", 2024-09-14T04:57:16.483 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000106844 2024-09-14T04:57:16.483 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.483 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.484 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:57:16.484 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.701069+0000", 2024-09-14T04:57:16.484 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.3529999999999995e-05 2024-09-14T04:57:16.484 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.484 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.484 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:16.484 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.713705+0000", 2024-09-14T04:57:16.484 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.012635171000000001 2024-09-14T04:57:16.484 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.484 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.484 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.485 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.718635+0000", 2024-09-14T04:57:16.485 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0049301420000000002 2024-09-14T04:57:16.485 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.485 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.485 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.485 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.718682+0000", 2024-09-14T04:57:16.485 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7349999999999999e-05 2024-09-14T04:57:16.485 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.486 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.486 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:16.486 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.718766+0000", 2024-09-14T04:57:16.486 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.4369000000000005e-05 2024-09-14T04:57:16.486 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.486 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.487 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:16.487 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.721560+0000", 2024-09-14T04:57:16.487 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0027932740000000001 2024-09-14T04:57:16.488 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.488 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:16.488 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.488 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.488 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.488 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:150548 56.6 56:627884de:::benchmark_data_smithi039_330847_object36024:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1502)", 2024-09-14T04:57:16.488 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:48.347280+0000", 2024-09-14T04:57:16.488 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 147.989815389, 2024-09-14T04:57:16.488 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.35218084399999999, 2024-09-14T04:57:16.488 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:16.488 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:16.488 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:16.489 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:16.489 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:16.489 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 150548 2024-09-14T04:57:16.489 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.489 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:16.489 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.489 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:16.489 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.347280+0000", 2024-09-14T04:57:16.489 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.489 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.489 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.489 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:16.490 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.347280+0000", 2024-09-14T04:57:16.490 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.490 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.490 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.490 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:16.490 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.347278+0000", 2024-09-14T04:57:16.490 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:57:16.490 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.490 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.490 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:16.490 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.347288+0000", 2024-09-14T04:57:16.490 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.2900000000000008e-06 2024-09-14T04:57:16.490 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.491 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.491 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:16.491 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.347304+0000", 2024-09-14T04:57:16.491 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.6648000000000001e-05 2024-09-14T04:57:16.491 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.491 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.491 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:16.491 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.347313+0000", 2024-09-14T04:57:16.491 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.1149999999999994e-06 2024-09-14T04:57:16.491 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.491 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.491 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.491 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.351898+0000", 2024-09-14T04:57:16.491 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0045857010000000002 2024-09-14T04:57:16.491 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.492 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.492 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:16.492 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.352045+0000", 2024-09-14T04:57:16.492 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000146515 2024-09-14T04:57:16.492 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.492 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.492 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:57:16.492 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.352178+0000", 2024-09-14T04:57:16.492 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000133499 2024-09-14T04:57:16.492 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.492 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.493 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:16.493 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.359617+0000", 2024-09-14T04:57:16.493 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0074388099999999997 2024-09-14T04:57:16.493 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.493 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.493 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.493 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.359655+0000", 2024-09-14T04:57:16.493 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8433000000000003e-05 2024-09-14T04:57:16.493 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.493 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.493 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.493 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.697666+0000", 2024-09-14T04:57:16.493 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33801028300000002 2024-09-14T04:57:16.494 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.494 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.494 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:16.494 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.697712+0000", 2024-09-14T04:57:16.494 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6282000000000002e-05 2024-09-14T04:57:16.494 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.494 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.494 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:16.494 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.699461+0000", 2024-09-14T04:57:16.495 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001748585 2024-09-14T04:57:16.495 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.495 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:16.495 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.495 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.495 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.495 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:150553 56.6 56:6540d818:::benchmark_data_smithi039_330847_object36029:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1502)", 2024-09-14T04:57:16.495 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:48.352518+0000", 2024-09-14T04:57:16.495 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 147.98457747, 2024-09-14T04:57:16.495 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.34859365399999998, 2024-09-14T04:57:16.496 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:16.496 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:16.496 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:16.496 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:16.496 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:16.496 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 150553 2024-09-14T04:57:16.496 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.496 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:16.496 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.496 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:16.496 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.352518+0000", 2024-09-14T04:57:16.496 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.496 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.496 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.497 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:16.497 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.352518+0000", 2024-09-14T04:57:16.497 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:16.497 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.497 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.497 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:16.497 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.352516+0000", 2024-09-14T04:57:16.497 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:57:16.497 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.497 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.498 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:16.498 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.352523+0000", 2024-09-14T04:57:16.498 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.4810000000000003e-06 2024-09-14T04:57:16.498 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.498 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.498 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:16.498 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.352535+0000", 2024-09-14T04:57:16.498 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2023e-05 2024-09-14T04:57:16.498 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.498 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.498 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:16.498 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.352541+0000", 2024-09-14T04:57:16.498 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.6000000000000003e-06 2024-09-14T04:57:16.499 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.499 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.499 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:16.499 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.355640+0000", 2024-09-14T04:57:16.499 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0030980780000000002 2024-09-14T04:57:16.499 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.499 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.499 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:16.499 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.355790+0000", 2024-09-14T04:57:16.499 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00015043700000000001 2024-09-14T04:57:16.499 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.499 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.499 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:57:16.499 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.355951+0000", 2024-09-14T04:57:16.500 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000160762 2024-09-14T04:57:16.500 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.500 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.500 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:16.500 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.363896+0000", 2024-09-14T04:57:16.500 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0079453760000000005 2024-09-14T04:57:16.500 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.500 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.500 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.500 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.364349+0000", 2024-09-14T04:57:16.500 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000452669 2024-09-14T04:57:16.500 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.501 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.501 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:16.501 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.699528+0000", 2024-09-14T04:57:16.501 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.335179385 2024-09-14T04:57:16.501 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.501 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.501 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:16.501 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.699580+0000", 2024-09-14T04:57:16.502 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2187e-05 2024-09-14T04:57:16.502 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:16.502 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:16.502 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:16.502 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.701111+0000", 2024-09-14T04:57:16.502 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001531019 2024-09-14T04:57:16.503 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.503 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:16.503 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.503 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:16.503 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:16.503 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:57:16.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:16.505 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:16.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:16.606 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:16.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:16.635 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:16.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:16.695 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 17 16 39328 39312 144.512 121.188 0.012069 0.00691415 2024-09-14T04:57:16.736 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:16.780 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:16.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:16.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:16.882 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:16.923 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:16.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:16.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:17.025 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:17.054 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:17.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:17.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:17.156 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:17.191 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:17.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:17.192 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:17.293 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:17.322 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:17.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:17.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:17.424 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:17.457 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:17.458 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:17.458 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:17.559 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:17.596 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:17.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:17.597 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:17.696 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 18 16 41061 41045 142.5 108.312 0.00615432 0.00701213 2024-09-14T04:57:17.697 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:17.736 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:17.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:17.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:17.838 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:17.877 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:17.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:17.878 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:17.979 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:18.007 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:18.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:18.008 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:18.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:17 smithi129 ceph-mon[101316]: pgmap v2922: 99 pgs: 99 active+clean; 2.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 152 MiB/s wr, 2.42k op/s 2024-09-14T04:57:18.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:17 smithi039 ceph-mon[106470]: pgmap v2922: 99 pgs: 99 active+clean; 2.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 152 MiB/s wr, 2.42k op/s 2024-09-14T04:57:18.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:17 smithi039 ceph-mon[111235]: pgmap v2922: 99 pgs: 99 active+clean; 2.1 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 152 MiB/s wr, 2.42k op/s 2024-09-14T04:57:18.109 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:18.143 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:18.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:18.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:18.244 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:18.279 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-14T04:57:18.280 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:18.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:18.281 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:18.382 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:18.415 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:18.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:18.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:18.517 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:18.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:18.559 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:18.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:18.587 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:57:18.660 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:18.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:18.694 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:18.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:18.696 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 19 16 42700 42684 140.391 102.438 0.00492727 0.00711766 2024-09-14T04:57:18.794 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:18.834 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:18.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:18.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:18.936 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:18.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:18.980 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:18.980 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:19.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:57:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:57:18] "GET /metrics HTTP/1.1" 200 36721 "" "Prometheus/2.51.0" 2024-09-14T04:57:19.080 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:19.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:19.118 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:19.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:19.220 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:19.251 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:19.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:19.252 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:19.353 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:19.389 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:19.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:19.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:19.491 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:19.522 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:19.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:19.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:19.624 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:19.656 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:19.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:19.657 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:19.696 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:2024-09-14T04:57:19.696204+0000 min lat: 0.00123999 max lat: 0.470156 avg lat: 0.00721031 2024-09-14T04:57:19.696 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:57:19.696 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 20 16 44372 44356 138.596 104.5 0.0105017 0.00721031 2024-09-14T04:57:19.758 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:19.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:19.786 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:19.786 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:19.887 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:19.914 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:19.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:19.915 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:20.016 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:20.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:19 smithi129 ceph-mon[101316]: pgmap v2923: 99 pgs: 99 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s 2024-09-14T04:57:20.038 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:20.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:20.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:20.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:19 smithi039 ceph-mon[106470]: pgmap v2923: 99 pgs: 99 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s 2024-09-14T04:57:20.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:19 smithi039 ceph-mon[111235]: pgmap v2923: 99 pgs: 99 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 108 MiB/s wr, 1.72k op/s 2024-09-14T04:57:20.141 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:20.165 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:20.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:20.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:20.268 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:20.315 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:20.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:20.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:20.417 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:20.450 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:20.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:20.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:20.552 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:20.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:20.593 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:20.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:20.695 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:20.696 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 21 16 45685 45669 135.904 82.0625 0.0138208 0.0073531 2024-09-14T04:57:20.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:20.736 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:20.736 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:20.837 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:20.842 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:57:20.842 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:57:20.865 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:20.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:20.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:20.877 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:57:20.877 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:57:20.893 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:57:20.894 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:57:20.910 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:57:20.911 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:57:20.941 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:57:20.941 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:57:20.967 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:20.976 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:57:20.977 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:57:21.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:21.008 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:21.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:21.013 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:57:21.014 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:57:21.048 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:57:21.049 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:57:21.109 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:21.140 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:21.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:21.141 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:21.242 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:21.275 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:21.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:21.276 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:21.377 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:21.411 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:21.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:21.412 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:21.513 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:21.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:21.553 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:21.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:21.654 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:21.687 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:21.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:21.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:21.696 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 22 16 47419 47403 134.652 108.375 0.00428804 0.00742158 2024-09-14T04:57:21.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:21 smithi129 ceph-mon[101316]: pgmap v2924: 99 pgs: 99 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2024-09-14T04:57:21.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:21 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2020713265' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:57:21.789 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:21.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:21 smithi039 ceph-mon[106470]: pgmap v2924: 99 pgs: 99 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2024-09-14T04:57:21.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:21 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2020713265' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:57:21.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:21 smithi039 ceph-mon[111235]: pgmap v2924: 99 pgs: 99 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s 2024-09-14T04:57:21.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:21 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2020713265' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:57:21.829 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:21.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:21.830 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:21.931 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:21.971 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:21.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:21.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:22.073 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:22.108 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:22.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:22.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:22.153 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:57:22.153 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:57:22.211 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:22.246 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:22.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:22.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:22.348 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:22.393 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:22.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:22.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:22.495 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:22.531 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:22.531 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:22.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:22.632 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:22.664 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:22.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:22.666 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:22.696 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 23 16 49198 49182 133.632 111.188 0.00323309 0.00747538 2024-09-14T04:57:22.717 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:57:22.717 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:57:22.767 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:22.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:57:22.797 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:22.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:22.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:22.899 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:22.945 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:22.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:22.946 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:23.047 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:23.073 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:23.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:23.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:23.175 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:23.202 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:23.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:23.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:23.304 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:23.335 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:23.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:23.336 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:23.437 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:23.473 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:23.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:23.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:23.540 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-14T04:57:23.575 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:23.624 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:23.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:23.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:23.664 INFO:teuthology.orchestra.run.smithi039.stderr:set pool 1 pgp_num to 11 2024-09-14T04:57:23.696 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 24 16 50574 50558 131.647 86 0.0137105 0.0075905 2024-09-14T04:57:23.727 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:23.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:23.758 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:23.759 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:23 smithi129 ceph-mon[101316]: pgmap v2925: 99 pgs: 99 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2024-09-14T04:57:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:23 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/127461060' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-14T04:57:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:23 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3324371449' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:57:23.859 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:23.860 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:57:23.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:23.901 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:23.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:24.002 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:24.037 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:24.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:24.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:24.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:23 smithi039 ceph-mon[106470]: pgmap v2925: 99 pgs: 99 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2024-09-14T04:57:24.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:23 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/127461060' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-14T04:57:24.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:23 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3324371449' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:57:24.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:23 smithi039 ceph-mon[111235]: pgmap v2925: 99 pgs: 99 active+clean; 2.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 111 MiB/s wr, 1.78k op/s 2024-09-14T04:57:24.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:23 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/127461060' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"} : dispatch 2024-09-14T04:57:24.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:23 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3324371449' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:57:24.139 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:24.160 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:24.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:24.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:24.180 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:57:24.262 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:24.304 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:24.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:24.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:24.390 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is unset 2024-09-14T04:57:24.406 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:24.434 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:24.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:24.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:24.536 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:24.574 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:24.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:24.575 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:24.676 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:24.696 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 25 16 52024 52008 130.006 90.625 0.00244617 0.00767763 2024-09-14T04:57:24.697 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:24.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:24.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:24.799 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:24.826 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:24.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:24.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:24.928 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:24.950 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:24.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:24.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:25.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:24 smithi129 ceph-mon[101316]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:57:25.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:24 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/127461060' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-14T04:57:25.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:24 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3324371449' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:57:25.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:24 smithi129 ceph-mon[101316]: osdmap e1524: 8 total, 8 up, 8 in 2024-09-14T04:57:25.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:24 smithi039 ceph-mon[111235]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:57:25.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:24 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/127461060' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-14T04:57:25.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:24 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3324371449' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:57:25.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:24 smithi039 ceph-mon[111235]: osdmap e1524: 8 total, 8 up, 8 in 2024-09-14T04:57:25.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:24 smithi039 ceph-mon[106470]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:57:25.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:24 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/127461060' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "11"}]': finished 2024-09-14T04:57:25.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:24 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3324371449' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:57:25.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:24 smithi039 ceph-mon[106470]: osdmap e1524: 8 total, 8 up, 8 in 2024-09-14T04:57:25.052 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:25.089 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:25.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:25.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:25.191 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:25.230 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:25.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:25.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:25.332 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:25.367 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:25.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:25.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:25.470 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:25.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:25.512 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:25.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:25.613 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:25.648 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:25.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:25.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:25.696 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 26 16 53252 53236 127.957 76.75 0.00648418 0.00780989 2024-09-14T04:57:25.748 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:57:25.748 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:57:25.749 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:25.790 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:25.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:25.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:25.892 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:25.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:25.929 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:25.929 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:26.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:25 smithi129 ceph-mon[101316]: pgmap v2927: 99 pgs: 99 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-09-14T04:57:26.030 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:26.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:25 smithi039 ceph-mon[106470]: pgmap v2927: 99 pgs: 99 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-09-14T04:57:26.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:25 smithi039 ceph-mon[111235]: pgmap v2927: 99 pgs: 99 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s 2024-09-14T04:57:26.052 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:26.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:26.053 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:26.154 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:26.191 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:26.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:26.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:26.293 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:26.331 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:26.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:26.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:26.433 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:26.463 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:26.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:26.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:26.503 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-14T04:57:26.565 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:26.609 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:26.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:26.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:26.698 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 27 16 54617 54601 126.37 85.3125 0.00831551 0.00790756 2024-09-14T04:57:26.712 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:26.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:26.751 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:26.751 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:26.834 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:57:26.852 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:26.894 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:26.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:26.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:26.997 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:27.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:26 smithi129 ceph-mon[101316]: pgmap v2928: 99 pgs: 99 active+clean; 3.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-09-14T04:57:27.036 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:27.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:27.037 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:27.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:26 smithi039 ceph-mon[106470]: pgmap v2928: 99 pgs: 99 active+clean; 3.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-09-14T04:57:27.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:26 smithi039 ceph-mon[111235]: pgmap v2928: 99 pgs: 99 active+clean; 3.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-09-14T04:57:27.137 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:27.176 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:27.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:27.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:27.277 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:27.314 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:27.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:27.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:27.416 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:27.448 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:27.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:27.449 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:27.550 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:27.589 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:27.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:27.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:27.691 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:27.698 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 28 16 56285 56269 125.58 104.25 0.0154551 0.00795814 2024-09-14T04:57:27.737 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:27.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:27.738 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:27.839 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:27.879 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:27.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:27.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:27.980 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:28.012 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:28.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:28.014 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:28.114 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:28.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:28.147 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:28.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:28.157 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:57:28.248 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:28.290 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:28.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:28.291 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:28.391 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:28.425 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:28.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:28.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:28.527 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:28.566 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:28.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:28.567 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:28.667 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:28.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:28.690 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:28.691 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:28.698 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 29 16 57853 57837 124.629 98 0.00493489 0.00801223 2024-09-14T04:57:28.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:57:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:57:28] "GET /metrics HTTP/1.1" 200 36728 "" "Prometheus/2.51.0" 2024-09-14T04:57:28.791 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:28.826 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:28.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:28.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:28.928 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:28.970 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:28.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:28.971 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:29.072 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:29.111 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:29.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:29.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:29.195 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:28 smithi039 ceph-mon[106470]: pgmap v2929: 99 pgs: 99 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-09-14T04:57:29.195 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:28 smithi039 ceph-mon[111235]: pgmap v2929: 99 pgs: 99 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-09-14T04:57:29.212 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:29.251 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:29.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:29.252 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:29.274 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:28 smithi129 ceph-mon[101316]: pgmap v2929: 99 pgs: 99 active+clean; 3.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 102 MiB/s wr, 1.63k op/s 2024-09-14T04:57:29.353 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:29.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:29.387 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:29.388 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:29.488 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:29.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:29.510 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:29.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:29.612 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:29.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:29.634 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:29.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:29.698 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 30 16 58993 58977 122.849 71.25 0.00275616 0.00806587 2024-09-14T04:57:29.735 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:29.759 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:29.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:29.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:29.861 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:29.897 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:29.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:29.898 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:29.999 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:30.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:30.025 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:30.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:30.126 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:30.146 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:57:30.159 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:30.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:30.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:30.258 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:57:30.258 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:57:30.262 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:30.296 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:57:30.296 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:57:30.325 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:30.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:30.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:30.352 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:57:30.352 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:57:30.397 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:57:30.397 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:57:30.427 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:30.433 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:57:30.433 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:57:30.463 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:30.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:30.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:30.470 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:57:30.470 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:57:30.521 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:57:30.546 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:57:30.547 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:57:30.564 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:30.578 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:57:30.579 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:57:30.605 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:30.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:30.606 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:30.698 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 31 16 59936 59920 120.787 58.9375 0.0228443 0.00827417 2024-09-14T04:57:30.707 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:30.741 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:30.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:30.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:30.842 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:30.884 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:30.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:30.885 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:30.986 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:31.018 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:31.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:31.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:31.084 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:57:31.084 INFO:teuthology.orchestra.run.smithi129.stdout: "size": 20, 2024-09-14T04:57:31.084 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 600, 2024-09-14T04:57:31.084 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [ 2024-09-14T04:57:31.084 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.084 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.36012.0:35307 56.a e1474/1470 56:51ccef20:::benchmark_data_smithi039_330847_object35306:head v 1474'2159, mlcod=1474'2155)", 2024-09-14T04:57:31.084 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:44.504398+0000", 2024-09-14T04:57:31.084 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 286.041820196, 2024-09-14T04:57:31.084 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.340081674, 2024-09-14T04:57:31.084 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:31.084 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:31.084 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:31.085 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.085 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:31.085 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.504398+0000", 2024-09-14T04:57:31.085 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.085 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.085 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.085 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:31.085 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.504398+0000", 2024-09-14T04:57:31.085 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.085 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.085 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.085 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:31.085 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.504397+0000", 2024-09-14T04:57:31.085 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:57:31.086 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.086 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.086 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:31.086 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.504442+0000", 2024-09-14T04:57:31.086 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.4876000000000002e-05 2024-09-14T04:57:31.086 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.086 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.086 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:31.086 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.504453+0000", 2024-09-14T04:57:31.086 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1148000000000001e-05 2024-09-14T04:57:31.086 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.087 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.087 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:31.087 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.504460+0000", 2024-09-14T04:57:31.087 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.0630000000000002e-06 2024-09-14T04:57:31.087 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.087 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.087 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:31.087 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.504508+0000", 2024-09-14T04:57:31.087 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8791000000000001e-05 2024-09-14T04:57:31.087 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.087 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.087 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:31.087 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.504525+0000", 2024-09-14T04:57:31.088 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.6957e-05 2024-09-14T04:57:31.088 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.088 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.088 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:31.088 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508585+0000", 2024-09-14T04:57:31.088 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0040598880000000002 2024-09-14T04:57:31.088 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.088 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.088 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:31.088 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.844480+0000", 2024-09-14T04:57:31.088 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.335894727 2024-09-14T04:57:31.088 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.089 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:31.089 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.089 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.089 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.089 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.36012.0:35309 56.a e1474/1470 56:52365448:::benchmark_data_smithi039_330847_object35308:head v 1474'2160, mlcod=1474'2156)", 2024-09-14T04:57:31.089 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:44.506098+0000", 2024-09-14T04:57:31.089 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 286.04012065699999, 2024-09-14T04:57:31.089 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.338418787, 2024-09-14T04:57:31.089 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:31.089 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:31.089 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:31.089 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.090 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:31.090 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506098+0000", 2024-09-14T04:57:31.090 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.090 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.090 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.090 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:31.090 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506098+0000", 2024-09-14T04:57:31.090 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.090 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.090 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.090 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:31.090 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506092+0000", 2024-09-14T04:57:31.090 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999943 2024-09-14T04:57:31.091 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.091 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.091 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:31.091 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506126+0000", 2024-09-14T04:57:31.091 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.4019000000000001e-05 2024-09-14T04:57:31.091 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.091 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.091 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:31.091 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506138+0000", 2024-09-14T04:57:31.091 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1596e-05 2024-09-14T04:57:31.091 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.091 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.091 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:31.091 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506144+0000", 2024-09-14T04:57:31.092 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.691e-06 2024-09-14T04:57:31.092 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.092 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.092 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:31.092 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506227+0000", 2024-09-14T04:57:31.092 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.2707000000000006e-05 2024-09-14T04:57:31.092 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.092 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.092 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:31.092 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506246+0000", 2024-09-14T04:57:31.092 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.8711000000000001e-05 2024-09-14T04:57:31.092 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.092 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.092 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:31.092 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.844486+0000", 2024-09-14T04:57:31.093 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33824021199999998 2024-09-14T04:57:31.093 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.093 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.093 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:31.093 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.844517+0000", 2024-09-14T04:57:31.093 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.0561000000000001e-05 2024-09-14T04:57:31.093 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.093 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:31.093 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.093 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.093 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.093 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.36012.0:35304 56.0 e1474/1470 56:0e8f1528:::benchmark_data_smithi039_330847_object35303:head v 1474'2156, mlcod=1474'2151)", 2024-09-14T04:57:31.093 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:44.506162+0000", 2024-09-14T04:57:31.093 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 286.04005633700001, 2024-09-14T04:57:31.094 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33830948, 2024-09-14T04:57:31.094 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:31.094 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:31.094 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:31.094 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.094 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:31.094 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506162+0000", 2024-09-14T04:57:31.094 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.094 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.094 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.094 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:31.094 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506162+0000", 2024-09-14T04:57:31.094 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.094 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.095 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.095 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:31.095 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506162+0000", 2024-09-14T04:57:31.095 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:57:31.095 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.095 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.095 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:31.095 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.507135+0000", 2024-09-14T04:57:31.095 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00097378999999999999 2024-09-14T04:57:31.095 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.095 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.095 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:31.095 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.507145+0000", 2024-09-14T04:57:31.096 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0029e-05 2024-09-14T04:57:31.096 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.096 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.096 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:31.096 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.507151+0000", 2024-09-14T04:57:31.096 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.5940000000000001e-06 2024-09-14T04:57:31.096 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.096 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.096 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:31.096 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.507201+0000", 2024-09-14T04:57:31.096 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.9514000000000001e-05 2024-09-14T04:57:31.096 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.097 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.097 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:31.097 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.507218+0000", 2024-09-14T04:57:31.097 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.7710000000000002e-05 2024-09-14T04:57:31.097 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.097 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.097 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:31.097 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508594+0000", 2024-09-14T04:57:31.097 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001375827 2024-09-14T04:57:31.097 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.097 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.097 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:31.097 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.844472+0000", 2024-09-14T04:57:31.097 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.335877498 2024-09-14T04:57:31.098 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.098 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:31.098 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.098 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.098 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.098 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.36012.0:35314 56.3 e1474/1470 56:c84c2af5:::benchmark_data_smithi039_330847_object35313:head v 1474'2231, mlcod=1474'2229)", 2024-09-14T04:57:31.098 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:44.506747+0000", 2024-09-14T04:57:31.098 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 286.03947170800001, 2024-09-14T04:57:31.098 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33962238, 2024-09-14T04:57:31.098 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:31.098 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:31.098 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:31.098 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.098 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:31.098 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506747+0000", 2024-09-14T04:57:31.099 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.099 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.099 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.099 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:31.099 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506747+0000", 2024-09-14T04:57:31.099 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.099 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.099 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.099 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:31.099 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506740+0000", 2024-09-14T04:57:31.099 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999938 2024-09-14T04:57:31.099 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.099 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.099 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:31.100 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506778+0000", 2024-09-14T04:57:31.100 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7771000000000001e-05 2024-09-14T04:57:31.100 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.100 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.100 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:31.100 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506790+0000", 2024-09-14T04:57:31.100 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1798e-05 2024-09-14T04:57:31.100 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.100 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.100 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:31.100 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506796+0000", 2024-09-14T04:57:31.100 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.993e-06 2024-09-14T04:57:31.100 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.100 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.100 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:31.101 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506834+0000", 2024-09-14T04:57:31.101 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7926000000000002e-05 2024-09-14T04:57:31.101 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.101 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.101 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:31.101 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.506853+0000", 2024-09-14T04:57:31.101 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.9483999999999998e-05 2024-09-14T04:57:31.101 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.101 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.101 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:31.101 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.846325+0000", 2024-09-14T04:57:31.101 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33947212500000001 2024-09-14T04:57:31.101 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.101 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.102 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:31.102 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.846369+0000", 2024-09-14T04:57:31.102 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.3639e-05 2024-09-14T04:57:31.102 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.102 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:31.102 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.102 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.102 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.102 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.36012.0:35315 56.3 e1474/1470 56:ce1faa72:::benchmark_data_smithi039_330847_object35314:head v 1474'2232, mlcod=1474'2229)", 2024-09-14T04:57:31.102 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:44.508476+0000", 2024-09-14T04:57:31.102 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 286.03774220700001, 2024-09-14T04:57:31.103 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33791945200000001, 2024-09-14T04:57:31.103 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:31.103 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:31.103 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:31.103 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.103 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:31.103 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508476+0000", 2024-09-14T04:57:31.103 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.103 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.103 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.103 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:31.103 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508476+0000", 2024-09-14T04:57:31.103 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.103 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.104 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.104 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:31.104 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508475+0000", 2024-09-14T04:57:31.104 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:57:31.104 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.104 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.104 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:31.104 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508500+0000", 2024-09-14T04:57:31.104 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.4780999999999999e-05 2024-09-14T04:57:31.104 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.104 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.104 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:31.104 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508511+0000", 2024-09-14T04:57:31.104 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0541e-05 2024-09-14T04:57:31.104 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.104 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.105 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:31.105 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508517+0000", 2024-09-14T04:57:31.105 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.5540000000000003e-06 2024-09-14T04:57:31.105 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.105 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.105 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:31.105 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508556+0000", 2024-09-14T04:57:31.105 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9150999999999998e-05 2024-09-14T04:57:31.105 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.105 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.105 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:31.105 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508573+0000", 2024-09-14T04:57:31.105 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.6849000000000001e-05 2024-09-14T04:57:31.105 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.105 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.106 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:31.106 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.846374+0000", 2024-09-14T04:57:31.106 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33780096300000001 2024-09-14T04:57:31.106 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.106 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.106 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:31.106 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.846396+0000", 2024-09-14T04:57:31.106 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.1350000000000001e-05 2024-09-14T04:57:31.106 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.106 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:31.106 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.106 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.106 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.106 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.36012.0:35313 56.0 e1474/1470 56:0ae4ee63:::benchmark_data_smithi039_330847_object35312:head v 1474'2157, mlcod=1474'2153)", 2024-09-14T04:57:31.106 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:44.508938+0000", 2024-09-14T04:57:31.107 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 286.03728047700002, 2024-09-14T04:57:31.107 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33916291700000001, 2024-09-14T04:57:31.107 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:31.107 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:31.107 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:31.107 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.107 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:31.107 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508938+0000", 2024-09-14T04:57:31.107 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.107 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.107 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.107 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:31.107 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508938+0000", 2024-09-14T04:57:31.107 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.108 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.108 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.108 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:31.108 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508937+0000", 2024-09-14T04:57:31.108 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:57:31.108 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.108 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.108 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:31.108 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508980+0000", 2024-09-14T04:57:31.108 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.2385000000000002e-05 2024-09-14T04:57:31.108 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.108 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.108 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:31.108 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.843950+0000", 2024-09-14T04:57:31.109 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33497075500000001 2024-09-14T04:57:31.109 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.109 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.109 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:31.109 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.843971+0000", 2024-09-14T04:57:31.109 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.0636e-05 2024-09-14T04:57:31.109 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.109 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.109 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:31.109 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.844520+0000", 2024-09-14T04:57:31.109 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00054909100000000003 2024-09-14T04:57:31.109 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.109 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.110 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:31.110 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.844547+0000", 2024-09-14T04:57:31.110 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.6724999999999999e-05 2024-09-14T04:57:31.110 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.110 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.110 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:31.110 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.847994+0000", 2024-09-14T04:57:31.110 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0034473070000000001 2024-09-14T04:57:31.110 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.110 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.110 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:31.110 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.848101+0000", 2024-09-14T04:57:31.110 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010673799999999999 2024-09-14T04:57:31.110 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.111 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:31.111 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.111 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.111 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.111 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:112481 56.3 56:cb99f848:::benchmark_data_smithi039_330847_object112480:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1485)", 2024-09-14T04:57:31.111 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:53:58.279953+0000", 2024-09-14T04:57:31.111 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 212.26626566900001, 2024-09-14T04:57:31.111 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.19169450599999999, 2024-09-14T04:57:31.111 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:31.111 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:31.111 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:31.111 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:31.111 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:31.112 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 112481 2024-09-14T04:57:31.112 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.112 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:31.112 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.112 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:31.112 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:53:58.279953+0000", 2024-09-14T04:57:31.112 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.112 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.112 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.112 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:31.112 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:53:58.279953+0000", 2024-09-14T04:57:31.112 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.112 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.113 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.113 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:31.113 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:53:58.279951+0000", 2024-09-14T04:57:31.113 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:57:31.113 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.113 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.113 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:31.113 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:53:58.279987+0000", 2024-09-14T04:57:31.113 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6142000000000001e-05 2024-09-14T04:57:31.113 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.113 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.113 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:31.113 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:53:58.280006+0000", 2024-09-14T04:57:31.113 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.8824000000000001e-05 2024-09-14T04:57:31.114 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.114 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.114 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:31.114 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:53:58.280015+0000", 2024-09-14T04:57:31.114 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.02e-06 2024-09-14T04:57:31.114 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.114 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.114 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:31.114 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:53:58.284628+0000", 2024-09-14T04:57:31.114 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.004612863 2024-09-14T04:57:31.114 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.114 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.114 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:31.114 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:53:58.285035+0000", 2024-09-14T04:57:31.115 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00040732300000000002 2024-09-14T04:57:31.115 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.115 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.115 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,5", 2024-09-14T04:57:31.115 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:53:58.285510+0000", 2024-09-14T04:57:31.115 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00047462900000000001 2024-09-14T04:57:31.115 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.115 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.115 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:31.115 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:53:58.299758+0000", 2024-09-14T04:57:31.115 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.014248538 2024-09-14T04:57:31.115 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.115 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.115 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:31.115 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:53:58.309514+0000", 2024-09-14T04:57:31.116 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0097554240000000004 2024-09-14T04:57:31.116 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.116 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.116 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:31.116 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:53:58.470660+0000", 2024-09-14T04:57:31.116 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.16114668900000001 2024-09-14T04:57:31.116 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.116 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.116 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:31.116 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:53:58.470703+0000", 2024-09-14T04:57:31.116 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.2141000000000002e-05 2024-09-14T04:57:31.116 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.116 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.116 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:31.117 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:53:58.471647+0000", 2024-09-14T04:57:31.117 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00094468999999999998 2024-09-14T04:57:31.117 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.117 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:31.117 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.117 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.117 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.117 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:116261 56.3 56:c9416220:::benchmark_data_smithi039_330847_object1737:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1486)", 2024-09-14T04:57:31.117 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:04.556697+0000", 2024-09-14T04:57:31.117 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 205.98952152499999, 2024-09-14T04:57:31.117 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.121956641, 2024-09-14T04:57:31.117 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:31.117 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:31.118 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:31.118 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:31.118 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:31.118 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 116261 2024-09-14T04:57:31.118 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.118 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:31.118 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.118 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:31.118 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.556697+0000", 2024-09-14T04:57:31.118 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.118 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.118 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.118 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:31.118 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.556697+0000", 2024-09-14T04:57:31.118 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.119 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.119 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.119 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:31.119 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.556695+0000", 2024-09-14T04:57:31.119 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:57:31.119 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.119 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.119 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:31.119 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.556701+0000", 2024-09-14T04:57:31.119 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.117e-06 2024-09-14T04:57:31.119 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.119 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.119 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:31.119 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.556711+0000", 2024-09-14T04:57:31.120 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.8309999999999996e-06 2024-09-14T04:57:31.120 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.120 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.120 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:31.120 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.556716+0000", 2024-09-14T04:57:31.120 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.5959999999999997e-06 2024-09-14T04:57:31.120 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.120 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.120 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:31.120 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.558795+0000", 2024-09-14T04:57:31.120 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0020794680000000001 2024-09-14T04:57:31.120 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.120 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.120 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:31.121 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.558871+0000", 2024-09-14T04:57:31.121 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.5332000000000003e-05 2024-09-14T04:57:31.121 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.121 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.121 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,5", 2024-09-14T04:57:31.121 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.558968+0000", 2024-09-14T04:57:31.121 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.7318000000000006e-05 2024-09-14T04:57:31.121 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.121 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.121 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:31.121 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.561965+0000", 2024-09-14T04:57:31.121 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0029968099999999999 2024-09-14T04:57:31.121 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.121 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.122 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:31.122 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.562381+0000", 2024-09-14T04:57:31.122 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00041662300000000003 2024-09-14T04:57:31.122 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.122 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.122 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:31.122 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.677407+0000", 2024-09-14T04:57:31.122 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.115025249 2024-09-14T04:57:31.122 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.122 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.122 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:31.122 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.677478+0000", 2024-09-14T04:57:31.122 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.1138999999999998e-05 2024-09-14T04:57:31.122 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.123 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.123 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:31.123 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.678654+0000", 2024-09-14T04:57:31.123 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0011758230000000001 2024-09-14T04:57:31.123 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.123 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:31.123 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.123 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.123 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.123 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36324.0:23516 57.c 57:397f50c8:::benchmark_data_smithi039_365565_object23515:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T04:57:31.123 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:57:08.205676+0000", 2024-09-14T04:57:31.123 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 22.340542620000001, 2024-09-14T04:57:31.123 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.12821248299999999, 2024-09-14T04:57:31.123 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:31.124 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:31.124 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:31.124 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36324", 2024-09-14T04:57:31.124 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T04:57:31.124 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 23516 2024-09-14T04:57:31.124 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.124 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:31.124 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.124 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:31.124 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:08.205676+0000", 2024-09-14T04:57:31.124 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.124 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.124 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.124 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:31.125 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:08.205676+0000", 2024-09-14T04:57:31.125 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.125 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.125 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.125 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:31.125 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:08.205674+0000", 2024-09-14T04:57:31.125 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:57:31.125 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.125 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.125 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:31.125 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:08.205716+0000", 2024-09-14T04:57:31.125 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.2109e-05 2024-09-14T04:57:31.125 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.125 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.126 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:31.126 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:08.205732+0000", 2024-09-14T04:57:31.126 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.6259000000000001e-05 2024-09-14T04:57:31.126 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.126 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.126 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:31.126 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:08.205741+0000", 2024-09-14T04:57:31.126 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.6009999999999995e-06 2024-09-14T04:57:31.126 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.126 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.126 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:31.126 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:08.205801+0000", 2024-09-14T04:57:31.126 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.0056999999999998e-05 2024-09-14T04:57:31.126 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.127 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.127 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:31.127 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:08.205963+0000", 2024-09-14T04:57:31.127 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00016220000000000001 2024-09-14T04:57:31.127 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.127 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.127 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:57:31.127 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:08.206099+0000", 2024-09-14T04:57:31.127 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013569400000000001 2024-09-14T04:57:31.127 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.127 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.127 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:31.127 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:08.207538+0000", 2024-09-14T04:57:31.127 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0014388440000000001 2024-09-14T04:57:31.127 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.128 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.128 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:31.128 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:08.207650+0000", 2024-09-14T04:57:31.128 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011203499999999999 2024-09-14T04:57:31.128 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.128 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.128 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:31.128 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:08.333834+0000", 2024-09-14T04:57:31.128 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.12618453600000001 2024-09-14T04:57:31.128 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.128 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.128 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:31.128 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:08.333866+0000", 2024-09-14T04:57:31.128 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.1881000000000002e-05 2024-09-14T04:57:31.129 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.129 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.129 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:31.129 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:08.333888+0000", 2024-09-14T04:57:31.129 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.2178999999999999e-05 2024-09-14T04:57:31.129 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.129 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:31.129 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.129 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.129 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.129 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36324.0:34423 57.c 57:3a9545c4:::benchmark_data_smithi039_365565_object34422:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T04:57:31.129 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:57:13.476213+0000", 2024-09-14T04:57:31.129 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 17.070005045999999, 2024-09-14T04:57:31.129 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.42755653100000002, 2024-09-14T04:57:31.130 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:31.130 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:31.130 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:31.130 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36324", 2024-09-14T04:57:31.130 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T04:57:31.130 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 34423 2024-09-14T04:57:31.130 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.130 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:31.130 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.130 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:31.130 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.476213+0000", 2024-09-14T04:57:31.130 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.131 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.131 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.131 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:31.131 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.476213+0000", 2024-09-14T04:57:31.131 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.131 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.131 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.131 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:31.131 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.476212+0000", 2024-09-14T04:57:31.131 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:57:31.131 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.131 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.131 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:31.131 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.476260+0000", 2024-09-14T04:57:31.132 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8439000000000002e-05 2024-09-14T04:57:31.132 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.132 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.132 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:31.132 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.476294+0000", 2024-09-14T04:57:31.132 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3504000000000003e-05 2024-09-14T04:57:31.132 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.132 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.132 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:31.132 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.476302+0000", 2024-09-14T04:57:31.132 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.3129999999999999e-06 2024-09-14T04:57:31.132 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.132 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.133 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:31.133 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.476429+0000", 2024-09-14T04:57:31.133 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000127224 2024-09-14T04:57:31.133 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.133 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.133 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:31.133 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.476559+0000", 2024-09-14T04:57:31.133 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012968300000000001 2024-09-14T04:57:31.133 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.133 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.133 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:57:31.133 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.476698+0000", 2024-09-14T04:57:31.133 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013924200000000001 2024-09-14T04:57:31.133 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.134 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.134 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:31.134 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.478850+0000", 2024-09-14T04:57:31.134 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0021517239999999998 2024-09-14T04:57:31.134 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.134 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.134 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:31.134 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.481737+0000", 2024-09-14T04:57:31.134 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.002887321 2024-09-14T04:57:31.134 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.134 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.134 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:31.134 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.903650+0000", 2024-09-14T04:57:31.134 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.42191207800000002 2024-09-14T04:57:31.134 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.135 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.135 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:31.135 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.903725+0000", 2024-09-14T04:57:31.135 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.5860999999999998e-05 2024-09-14T04:57:31.135 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.135 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.135 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:31.135 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.903770+0000", 2024-09-14T04:57:31.135 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.4517000000000001e-05 2024-09-14T04:57:31.135 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.135 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:31.135 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.135 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.135 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.136 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36324.0:34441 57.c 57:369fd8ff:::benchmark_data_smithi039_365565_object34440:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T04:57:31.136 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:57:13.484391+0000", 2024-09-14T04:57:31.136 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 17.061827064999999, 2024-09-14T04:57:31.136 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.42049165100000002, 2024-09-14T04:57:31.136 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:31.136 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:31.136 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:31.136 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36324", 2024-09-14T04:57:31.136 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T04:57:31.136 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 34441 2024-09-14T04:57:31.136 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.136 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:31.136 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.136 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:31.137 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.484391+0000", 2024-09-14T04:57:31.137 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.137 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.137 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.137 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:31.137 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.484391+0000", 2024-09-14T04:57:31.137 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.137 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.137 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.137 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:31.137 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.484387+0000", 2024-09-14T04:57:31.137 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999957 2024-09-14T04:57:31.137 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.137 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.137 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:31.138 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.484456+0000", 2024-09-14T04:57:31.138 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.9098999999999997e-05 2024-09-14T04:57:31.138 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.138 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.138 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:31.138 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.484474+0000", 2024-09-14T04:57:31.138 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.7487e-05 2024-09-14T04:57:31.138 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.138 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.138 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:31.138 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.484483+0000", 2024-09-14T04:57:31.138 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.1349999999999998e-06 2024-09-14T04:57:31.138 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.138 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.138 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:31.139 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.484532+0000", 2024-09-14T04:57:31.139 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.9502999999999997e-05 2024-09-14T04:57:31.139 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.139 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.139 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:31.139 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.484681+0000", 2024-09-14T04:57:31.139 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000148658 2024-09-14T04:57:31.139 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.139 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.139 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:57:31.139 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.484835+0000", 2024-09-14T04:57:31.139 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00015358099999999999 2024-09-14T04:57:31.139 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.140 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.140 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:31.140 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.486249+0000", 2024-09-14T04:57:31.140 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001414228 2024-09-14T04:57:31.140 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.140 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.140 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:31.140 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.487023+0000", 2024-09-14T04:57:31.140 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00077426800000000005 2024-09-14T04:57:31.140 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.140 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.140 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:31.140 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.904769+0000", 2024-09-14T04:57:31.140 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.41774598200000002 2024-09-14T04:57:31.141 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.141 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.141 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:31.141 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.904841+0000", 2024-09-14T04:57:31.141 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.2271e-05 2024-09-14T04:57:31.141 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.141 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.141 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:31.141 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.904883+0000", 2024-09-14T04:57:31.141 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1706999999999999e-05 2024-09-14T04:57:31.141 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.141 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:31.142 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.142 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.142 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.142 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36324.0:34473 57.c 57:30f8a30d:::benchmark_data_smithi039_365565_object34472:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T04:57:31.142 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:57:13.512226+0000", 2024-09-14T04:57:31.142 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 17.033992242, 2024-09-14T04:57:31.142 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.39696483399999999, 2024-09-14T04:57:31.142 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:31.142 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:31.142 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:31.142 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36324", 2024-09-14T04:57:31.142 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T04:57:31.142 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 34473 2024-09-14T04:57:31.143 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.143 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:31.143 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.143 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:31.143 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.512226+0000", 2024-09-14T04:57:31.143 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.143 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.143 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.143 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:31.143 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.512226+0000", 2024-09-14T04:57:31.143 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.143 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.143 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.143 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:31.143 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.512224+0000", 2024-09-14T04:57:31.144 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:57:31.144 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.144 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.144 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:31.144 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.512288+0000", 2024-09-14T04:57:31.144 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.3947000000000003e-05 2024-09-14T04:57:31.144 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.144 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.144 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:31.144 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.512299+0000", 2024-09-14T04:57:31.144 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0926e-05 2024-09-14T04:57:31.144 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.144 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.145 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:31.145 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.512304+0000", 2024-09-14T04:57:31.145 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7500000000000003e-06 2024-09-14T04:57:31.145 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.145 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.145 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:31.145 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.512342+0000", 2024-09-14T04:57:31.145 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8278999999999997e-05 2024-09-14T04:57:31.145 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.145 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.145 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:31.145 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.512436+0000", 2024-09-14T04:57:31.145 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.3912999999999993e-05 2024-09-14T04:57:31.145 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.146 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.146 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:57:31.146 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.512529+0000", 2024-09-14T04:57:31.146 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.3244000000000004e-05 2024-09-14T04:57:31.146 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.146 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.146 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:31.146 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.513627+0000", 2024-09-14T04:57:31.146 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001097563 2024-09-14T04:57:31.146 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.146 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.146 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:31.146 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.513873+0000", 2024-09-14T04:57:31.146 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00024622899999999999 2024-09-14T04:57:31.146 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.146 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.147 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:31.147 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.909058+0000", 2024-09-14T04:57:31.147 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.39518478400000001 2024-09-14T04:57:31.147 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.147 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.147 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:31.147 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.909151+0000", 2024-09-14T04:57:31.147 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.3219000000000003e-05 2024-09-14T04:57:31.147 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.147 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.147 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:31.147 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.909191+0000", 2024-09-14T04:57:31.147 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9906e-05 2024-09-14T04:57:31.147 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.148 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:31.148 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.148 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.148 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.148 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36324.0:34595 57.c 57:361311a4:::benchmark_data_smithi039_365565_object34594:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T04:57:31.148 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:57:14.021447+0000", 2024-09-14T04:57:31.148 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 16.524771387000001, 2024-09-14T04:57:31.148 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.22964048400000001, 2024-09-14T04:57:31.148 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:31.148 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:31.148 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:31.149 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36324", 2024-09-14T04:57:31.149 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T04:57:31.149 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 34595 2024-09-14T04:57:31.149 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.149 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:31.149 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.149 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:31.149 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.021447+0000", 2024-09-14T04:57:31.149 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.149 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.149 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.149 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:31.149 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.021447+0000", 2024-09-14T04:57:31.150 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.150 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.150 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.150 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:31.150 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.021447+0000", 2024-09-14T04:57:31.150 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:57:31.150 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.150 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.150 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:31.150 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.021474+0000", 2024-09-14T04:57:31.150 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.7749000000000001e-05 2024-09-14T04:57:31.150 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.150 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.151 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:31.151 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.021481+0000", 2024-09-14T04:57:31.151 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.579e-06 2024-09-14T04:57:31.151 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.151 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.151 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:31.151 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.021485+0000", 2024-09-14T04:57:31.151 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.578e-06 2024-09-14T04:57:31.151 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.151 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.151 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:31.151 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.026116+0000", 2024-09-14T04:57:31.151 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0046308970000000001 2024-09-14T04:57:31.151 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.152 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.152 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:31.152 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.026257+0000", 2024-09-14T04:57:31.152 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0001405 2024-09-14T04:57:31.152 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.152 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.152 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:57:31.152 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.026402+0000", 2024-09-14T04:57:31.152 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000144926 2024-09-14T04:57:31.152 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.152 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.152 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:31.152 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.028224+0000", 2024-09-14T04:57:31.153 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0018217579999999999 2024-09-14T04:57:31.153 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.153 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.153 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:31.153 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.028285+0000", 2024-09-14T04:57:31.153 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.1548000000000001e-05 2024-09-14T04:57:31.153 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.153 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.153 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:31.153 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029389+0000", 2024-09-14T04:57:31.153 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0011035019999999999 2024-09-14T04:57:31.153 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.153 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.153 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:31.154 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029428+0000", 2024-09-14T04:57:31.154 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8893000000000002e-05 2024-09-14T04:57:31.154 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.154 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.154 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:31.154 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.251087+0000", 2024-09-14T04:57:31.154 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.22165997700000001 2024-09-14T04:57:31.154 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.154 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:31.154 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.154 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.154 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.154 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36324.0:34605 57.d 57:b3cca012:::benchmark_data_smithi039_365565_object34604:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T04:57:31.154 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:57:14.027054+0000", 2024-09-14T04:57:31.154 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 16.519164590999999, 2024-09-14T04:57:31.155 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.22652098900000001, 2024-09-14T04:57:31.155 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:31.155 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:31.155 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:31.155 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36324", 2024-09-14T04:57:31.155 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T04:57:31.155 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 34605 2024-09-14T04:57:31.155 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.155 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:31.155 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.155 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:31.155 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.027054+0000", 2024-09-14T04:57:31.155 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.155 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.155 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.156 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:31.156 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.027054+0000", 2024-09-14T04:57:31.156 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.156 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.156 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.156 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:31.156 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.027052+0000", 2024-09-14T04:57:31.156 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:57:31.156 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.156 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.156 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:31.156 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.027173+0000", 2024-09-14T04:57:31.156 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000120345 2024-09-14T04:57:31.157 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.157 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.157 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:31.157 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.027185+0000", 2024-09-14T04:57:31.157 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1918e-05 2024-09-14T04:57:31.157 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.157 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.157 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:31.157 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.027192+0000", 2024-09-14T04:57:31.157 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.0140000000000001e-06 2024-09-14T04:57:31.157 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.157 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.157 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:31.158 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.027280+0000", 2024-09-14T04:57:31.158 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.8094000000000001e-05 2024-09-14T04:57:31.158 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.158 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.158 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:31.158 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.027416+0000", 2024-09-14T04:57:31.158 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013593000000000001 2024-09-14T04:57:31.158 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.158 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.158 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,3", 2024-09-14T04:57:31.158 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.027552+0000", 2024-09-14T04:57:31.158 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0001362 2024-09-14T04:57:31.158 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.158 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.159 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:31.159 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029100+0000", 2024-09-14T04:57:31.159 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0015484850000000001 2024-09-14T04:57:31.159 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.159 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.159 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:31.159 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.253444+0000", 2024-09-14T04:57:31.159 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.22434352599999999 2024-09-14T04:57:31.159 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.159 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.159 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:31.159 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.253479+0000", 2024-09-14T04:57:31.159 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5698000000000003e-05 2024-09-14T04:57:31.160 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.160 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.160 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:31.160 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.253533+0000", 2024-09-14T04:57:31.160 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.3789999999999998e-05 2024-09-14T04:57:31.160 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.160 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.160 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:31.160 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.253575+0000", 2024-09-14T04:57:31.160 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1539999999999999e-05 2024-09-14T04:57:31.160 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.160 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:31.160 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.161 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.161 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.161 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.36324.0:34603 57.3 e1523/1518 57:cb3b690c:::benchmark_data_smithi039_365565_object34602:head v 1523'2118, mlcod=1523'2116)", 2024-09-14T04:57:31.161 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:57:14.028239+0000", 2024-09-14T04:57:31.161 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 16.517979644, 2024-09-14T04:57:31.161 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.22289592, 2024-09-14T04:57:31.161 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:31.161 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:31.161 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:31.161 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.161 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:31.161 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.028239+0000", 2024-09-14T04:57:31.161 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.162 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.162 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.162 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:31.162 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.028239+0000", 2024-09-14T04:57:31.162 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.162 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.162 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.162 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:31.162 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.028238+0000", 2024-09-14T04:57:31.162 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:57:31.162 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.162 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.162 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:31.163 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.028306+0000", 2024-09-14T04:57:31.163 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8275000000000006e-05 2024-09-14T04:57:31.163 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.163 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.163 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:31.163 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.028318+0000", 2024-09-14T04:57:31.163 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1398e-05 2024-09-14T04:57:31.163 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.163 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.163 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:31.163 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.028324+0000", 2024-09-14T04:57:31.163 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.6259999999999997e-06 2024-09-14T04:57:31.163 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.163 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.163 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:31.164 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.028367+0000", 2024-09-14T04:57:31.164 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.2551e-05 2024-09-14T04:57:31.164 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.164 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.164 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:31.164 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.028385+0000", 2024-09-14T04:57:31.164 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.8371000000000001e-05 2024-09-14T04:57:31.164 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.164 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.164 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:31.164 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.251083+0000", 2024-09-14T04:57:31.164 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.222698123 2024-09-14T04:57:31.164 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.164 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.165 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:31.165 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.251135+0000", 2024-09-14T04:57:31.165 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.1372999999999998e-05 2024-09-14T04:57:31.165 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.165 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:31.165 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.165 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.165 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.165 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop_reply(client.36324.0:34595 57.c e1523/1518 ondisk, result = 0)", 2024-09-14T04:57:31.165 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:57:14.029079+0000", 2024-09-14T04:57:31.165 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 16.51713904, 2024-09-14T04:57:31.165 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.22202002800000001, 2024-09-14T04:57:31.165 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:31.165 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "started", 2024-09-14T04:57:31.166 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:31.166 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.166 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:31.166 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029079+0000", 2024-09-14T04:57:31.166 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.166 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.166 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.166 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:31.166 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029079+0000", 2024-09-14T04:57:31.166 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.166 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.166 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.167 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:31.167 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029079+0000", 2024-09-14T04:57:31.167 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967296 2024-09-14T04:57:31.167 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.167 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.167 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:31.167 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029081+0000", 2024-09-14T04:57:31.167 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.7439999999999999e-06 2024-09-14T04:57:31.167 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.167 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.167 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:31.167 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029085+0000", 2024-09-14T04:57:31.167 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.4469999999999999e-06 2024-09-14T04:57:31.167 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.168 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.168 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:31.168 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029089+0000", 2024-09-14T04:57:31.168 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3589999999999999e-06 2024-09-14T04:57:31.168 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.168 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.168 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:31.168 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029376+0000", 2024-09-14T04:57:31.168 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00028774300000000003 2024-09-14T04:57:31.168 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.168 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.168 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:31.168 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029383+0000", 2024-09-14T04:57:31.169 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8449999999999997e-06 2024-09-14T04:57:31.169 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.169 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.169 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:31.169 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.251099+0000", 2024-09-14T04:57:31.169 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.22171605599999999 2024-09-14T04:57:31.169 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.169 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:31.169 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.169 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.169 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.169 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36324.0:34612 57.d 57:b07fe074:::benchmark_data_smithi039_365565_object34611:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T04:57:31.169 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:57:14.029179+0000", 2024-09-14T04:57:31.170 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 16.517039806, 2024-09-14T04:57:31.170 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.225134005, 2024-09-14T04:57:31.170 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:31.170 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:31.170 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:31.170 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36324", 2024-09-14T04:57:31.170 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T04:57:31.170 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 34612 2024-09-14T04:57:31.170 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.170 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:31.170 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.170 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:31.171 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029179+0000", 2024-09-14T04:57:31.171 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.171 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.171 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.171 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:31.171 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029179+0000", 2024-09-14T04:57:31.171 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.171 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.171 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.171 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:31.171 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029178+0000", 2024-09-14T04:57:31.171 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:57:31.171 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.172 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.172 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:31.172 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029297+0000", 2024-09-14T04:57:31.172 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011915499999999999 2024-09-14T04:57:31.172 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.172 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.172 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:31.172 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029306+0000", 2024-09-14T04:57:31.172 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.8939999999999999e-06 2024-09-14T04:57:31.172 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.172 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.172 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:31.172 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029312+0000", 2024-09-14T04:57:31.173 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9510000000000003e-06 2024-09-14T04:57:31.173 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.173 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.173 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:31.173 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029349+0000", 2024-09-14T04:57:31.173 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7515999999999998e-05 2024-09-14T04:57:31.173 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.173 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.173 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:31.173 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.251164+0000", 2024-09-14T04:57:31.173 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.221814756 2024-09-14T04:57:31.173 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.173 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.174 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,3", 2024-09-14T04:57:31.174 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.251342+0000", 2024-09-14T04:57:31.174 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00017786399999999999 2024-09-14T04:57:31.174 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.174 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.174 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:31.174 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.253624+0000", 2024-09-14T04:57:31.174 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0022818869999999998 2024-09-14T04:57:31.174 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.174 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.174 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:31.174 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.254229+0000", 2024-09-14T04:57:31.174 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00060513400000000001 2024-09-14T04:57:31.174 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.174 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.175 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:31.175 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.254235+0000", 2024-09-14T04:57:31.175 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.2339999999999999e-06 2024-09-14T04:57:31.175 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.175 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.175 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:31.175 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.254279+0000", 2024-09-14T04:57:31.175 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.4138999999999999e-05 2024-09-14T04:57:31.175 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.175 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.175 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:31.175 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.254313+0000", 2024-09-14T04:57:31.175 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3535000000000003e-05 2024-09-14T04:57:31.175 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.175 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:31.176 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.176 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.176 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.176 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop_reply(client.36324.0:34605 57.d e1523/1518 ondisk, result = 0)", 2024-09-14T04:57:31.176 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:57:14.029686+0000", 2024-09-14T04:57:31.176 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 16.516532106, 2024-09-14T04:57:31.176 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.223762446, 2024-09-14T04:57:31.176 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:31.176 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "started", 2024-09-14T04:57:31.176 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:31.176 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.176 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:31.176 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029686+0000", 2024-09-14T04:57:31.176 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.176 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.177 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.177 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:31.177 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029686+0000", 2024-09-14T04:57:31.177 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.177 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.177 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.177 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:31.177 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029686+0000", 2024-09-14T04:57:31.177 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:57:31.177 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.177 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.177 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:31.177 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029690+0000", 2024-09-14T04:57:31.177 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6160000000000004e-06 2024-09-14T04:57:31.177 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.178 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.178 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:31.178 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.251081+0000", 2024-09-14T04:57:31.178 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.221390265 2024-09-14T04:57:31.178 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.178 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.178 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:31.178 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.251093+0000", 2024-09-14T04:57:31.178 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2309e-05 2024-09-14T04:57:31.178 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.178 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.178 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:31.178 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.253298+0000", 2024-09-14T04:57:31.178 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0022047450000000001 2024-09-14T04:57:31.178 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.179 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.179 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:31.179 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.253305+0000", 2024-09-14T04:57:31.179 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.9270000000000005e-06 2024-09-14T04:57:31.179 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.179 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.179 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:31.179 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.253449+0000", 2024-09-14T04:57:31.179 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000143261 2024-09-14T04:57:31.179 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.179 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:31.179 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.179 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.179 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.180 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.36324.0:34606 57.3 e1523/1518 57:c42218af:::benchmark_data_smithi039_365565_object34605:head v 1523'2119, mlcod=1523'2116)", 2024-09-14T04:57:31.180 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:57:14.029786+0000", 2024-09-14T04:57:31.180 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 16.516431997000002, 2024-09-14T04:57:31.180 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.22630244299999999, 2024-09-14T04:57:31.180 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:31.180 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:31.180 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:31.180 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.180 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:31.180 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029786+0000", 2024-09-14T04:57:31.180 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.180 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.180 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.180 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:31.180 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029786+0000", 2024-09-14T04:57:31.181 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.181 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.181 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.181 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:31.181 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029786+0000", 2024-09-14T04:57:31.181 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:57:31.181 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.181 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.181 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:31.181 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029846+0000", 2024-09-14T04:57:31.181 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.0751999999999997e-05 2024-09-14T04:57:31.181 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.181 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.182 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:31.182 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.251086+0000", 2024-09-14T04:57:31.182 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.22123975300000001 2024-09-14T04:57:31.182 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.182 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.182 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:31.182 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.251096+0000", 2024-09-14T04:57:31.182 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.9909999999999997e-06 2024-09-14T04:57:31.182 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.182 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.182 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:31.182 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.251165+0000", 2024-09-14T04:57:31.182 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.9189000000000005e-05 2024-09-14T04:57:31.182 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.182 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.182 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:31.183 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.251187+0000", 2024-09-14T04:57:31.183 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.1908000000000001e-05 2024-09-14T04:57:31.183 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.183 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.183 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:31.183 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.256055+0000", 2024-09-14T04:57:31.183 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0048677219999999997 2024-09-14T04:57:31.183 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.183 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.183 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:31.183 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.256089+0000", 2024-09-14T04:57:31.184 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3899000000000002e-05 2024-09-14T04:57:31.184 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.184 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:31.184 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.184 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.184 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.184 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36324.0:34620 57.c 57:38b88004:::benchmark_data_smithi039_365565_object34619:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T04:57:31.184 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:57:14.033705+0000", 2024-09-14T04:57:31.184 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 16.512513695999999, 2024-09-14T04:57:31.184 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.22032158600000001, 2024-09-14T04:57:31.184 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:31.184 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:31.184 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:31.184 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36324", 2024-09-14T04:57:31.185 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T04:57:31.185 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 34620 2024-09-14T04:57:31.185 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.185 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:31.185 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.185 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:31.185 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.033705+0000", 2024-09-14T04:57:31.185 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.185 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.185 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.185 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:31.185 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.033705+0000", 2024-09-14T04:57:31.185 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:31.185 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.185 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.186 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:31.186 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.033697+0000", 2024-09-14T04:57:31.186 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999924 2024-09-14T04:57:31.186 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.186 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.186 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:31.186 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.033743+0000", 2024-09-14T04:57:31.186 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6239000000000002e-05 2024-09-14T04:57:31.186 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.186 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.186 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:31.186 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.251060+0000", 2024-09-14T04:57:31.186 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.217316961 2024-09-14T04:57:31.187 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.187 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.187 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:31.187 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.251081+0000", 2024-09-14T04:57:31.187 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.1103000000000001e-05 2024-09-14T04:57:31.187 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.187 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.187 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:31.187 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.251141+0000", 2024-09-14T04:57:31.187 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.0062999999999997e-05 2024-09-14T04:57:31.187 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.187 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.188 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:31.188 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.251310+0000", 2024-09-14T04:57:31.188 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00016876 2024-09-14T04:57:31.188 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.188 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.188 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:57:31.188 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.251434+0000", 2024-09-14T04:57:31.188 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000123504 2024-09-14T04:57:31.188 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.188 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.188 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:31.188 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.253556+0000", 2024-09-14T04:57:31.188 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0021222490000000001 2024-09-14T04:57:31.189 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.189 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.189 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:31.189 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.253565+0000", 2024-09-14T04:57:31.189 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.6000000000000007e-06 2024-09-14T04:57:31.189 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.189 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.189 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:31.189 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.253902+0000", 2024-09-14T04:57:31.189 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000337003 2024-09-14T04:57:31.189 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.189 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.190 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:31.190 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.253990+0000", 2024-09-14T04:57:31.190 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.8412000000000007e-05 2024-09-14T04:57:31.190 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:31.190 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:31.190 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:31.190 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.254026+0000", 2024-09-14T04:57:31.190 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6350999999999998e-05 2024-09-14T04:57:31.190 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.190 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:31.190 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.190 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:31.191 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:31.191 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:57:31.191 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:31.192 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:30 smithi039 ceph-mon[106470]: pgmap v2930: 99 pgs: 99 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-09-14T04:57:31.192 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:30 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3879410922' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:57:31.192 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:30 smithi039 ceph-mon[111235]: pgmap v2930: 99 pgs: 99 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-09-14T04:57:31.192 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:30 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3879410922' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:57:31.232 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:31.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:31.233 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:31.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:30 smithi129 ceph-mon[101316]: pgmap v2930: 99 pgs: 99 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 96 MiB/s wr, 1.53k op/s 2024-09-14T04:57:31.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:30 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3879410922' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:57:31.333 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:31.360 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:31.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:31.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:31.462 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:31.483 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:31.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:31.484 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:31.585 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:31.621 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:31.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:31.622 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:31.698 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 32 16 60937 60921 118.968 62.5625 0.00884561 0.00840041 2024-09-14T04:57:31.722 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:31.752 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:31.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:31.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:31.854 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:31.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:31.890 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:31.890 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:31.991 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:32.028 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:32.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:32.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:32.129 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:32.181 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:32.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:32.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:32.283 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:32.312 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:32.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:32.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:32.414 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:32.450 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:32.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:32.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:32.552 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:32.588 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:32.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:32.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:32.642 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-14T04:57:32.642 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 0 1 2024-09-14T04:57:32.692 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:32.700 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 33 16 62478 62462 118.278 96.3125 0.014493 0.00844929 2024-09-14T04:57:32.732 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:32.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:32.733 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:32.762 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-14T04:57:32.835 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:32.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:32.901 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:32.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:33.002 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:33.059 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:33.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:33.061 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:33.157 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:57:33.162 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:33.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:33.212 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:33.213 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:33.244 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:32 smithi039 ceph-mon[111235]: pgmap v2931: 99 pgs: 99 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-09-14T04:57:33.244 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:32 smithi039 ceph-mon[106470]: pgmap v2931: 99 pgs: 99 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-09-14T04:57:33.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:32 smithi129 ceph-mon[101316]: pgmap v2931: 99 pgs: 99 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 93 MiB/s wr, 1.48k op/s 2024-09-14T04:57:33.314 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:33.358 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:33.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:33.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:33.460 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:33.500 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:33.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:33.501 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:33.602 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:33.634 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:33.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:33.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:33.700 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 34 16 63802 63786 117.233 82.75 0.00505934 0.00852449 2024-09-14T04:57:33.736 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:33.777 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:33.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:33.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:33.878 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:33.910 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:33.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:33.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:33.991 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:57:34.011 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:34.042 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:34.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:34.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:34.143 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:34.165 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:57:34.174 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:34.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:34.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:34.276 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:34.300 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:34.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:34.301 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:34.402 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:34.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:34.459 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:34.459 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:34.560 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:34.603 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:34.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:34.605 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:34.700 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 35 16 64941 64925 115.917 71.1875 0.00281873 0.0086183 2024-09-14T04:57:34.705 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:34.726 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:34.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:34.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:34.827 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:34.860 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:34.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:34.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:34.962 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:35.002 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:35.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:35.003 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:35.012 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is unset 2024-09-14T04:57:35.105 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:35.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:35.145 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:35.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:35.215 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:34 smithi039 ceph-mon[106470]: pgmap v2932: 99 pgs: 99 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2024-09-14T04:57:35.215 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:34 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3967568828' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:57:35.215 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:34 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:57:35.215 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:34 smithi039 ceph-mon[111235]: pgmap v2932: 99 pgs: 99 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2024-09-14T04:57:35.215 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:34 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3967568828' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:57:35.216 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:34 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:57:35.246 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:35.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:34 smithi129 ceph-mon[101316]: pgmap v2932: 99 pgs: 99 active+clean; 3.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s 2024-09-14T04:57:35.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:34 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3967568828' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:57:35.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:34 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:57:35.285 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:35.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:35.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:35.387 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:35.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:35.428 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:35.429 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:35.529 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:35.571 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:35.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:35.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:35.673 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:35.700 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 36 16 66264 66248 114.994 82.6875 0.00546589 0.0086908 2024-09-14T04:57:35.715 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:35.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:35.716 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:35.817 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:35.854 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:35.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:35.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:35.956 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:35.993 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:35.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:35.994 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:36.095 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:36.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:36.133 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:36.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:36.205 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:57:36.205 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [ 2024-09-14T04:57:36.205 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:36.205 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.36324.0:66989 57.7 e1525/1518 57:ea003e35:::benchmark_data_smithi039_365565_object66988:head v 1525'4169, mlcod=1525'4167)", 2024-09-14T04:57:36.205 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:57:36.202235+0000", 2024-09-14T04:57:36.206 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.0018902700000000001, 2024-09-14T04:57:36.206 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00191183, 2024-09-14T04:57:36.206 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:36.206 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "started", 2024-09-14T04:57:36.206 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:36.206 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:36.206 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:36.206 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:36.202235+0000", 2024-09-14T04:57:36.206 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:36.206 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:36.206 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:36.207 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:36.207 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:36.202235+0000", 2024-09-14T04:57:36.207 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:36.207 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:36.207 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:36.207 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:36.207 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:36.202234+0000", 2024-09-14T04:57:36.207 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:57:36.207 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:36.207 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:36.207 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:36.207 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:36.202285+0000", 2024-09-14T04:57:36.207 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.0659999999999999e-05 2024-09-14T04:57:36.207 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:36.208 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:36.208 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:36.208 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:36.202305+0000", 2024-09-14T04:57:36.208 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.9996999999999998e-05 2024-09-14T04:57:36.208 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:36.208 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:36.208 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:36.208 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:36.202315+0000", 2024-09-14T04:57:36.208 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0033e-05 2024-09-14T04:57:36.208 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:36.208 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:36.208 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:36.209 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:36.202370+0000", 2024-09-14T04:57:36.209 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.5618999999999997e-05 2024-09-14T04:57:36.209 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:36.209 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:36.209 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:36.209 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:36.202396+0000", 2024-09-14T04:57:36.209 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.5874999999999999e-05 2024-09-14T04:57:36.209 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:36.209 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:36.209 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:36.209 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:36.209 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:36.209 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36324.0:66996 57.8 57:17243443:::benchmark_data_smithi039_365565_object66995:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1525)", 2024-09-14T04:57:36.210 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:57:36.203107+0000", 2024-09-14T04:57:36.210 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.0010182959999999999, 2024-09-14T04:57:36.210 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0010817649999999999, 2024-09-14T04:57:36.210 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:36.210 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "waiting for sub ops", 2024-09-14T04:57:36.210 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:36.210 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36324", 2024-09-14T04:57:36.210 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T04:57:36.210 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 66996 2024-09-14T04:57:36.210 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:36.210 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:36.210 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:36.210 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:36.211 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:36.203107+0000", 2024-09-14T04:57:36.211 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:36.211 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:36.211 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:36.211 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:36.211 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:36.203107+0000", 2024-09-14T04:57:36.211 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:36.211 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:36.211 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:36.211 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:36.212 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:36.203105+0000", 2024-09-14T04:57:36.212 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:57:36.212 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:36.212 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:36.212 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:36.212 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:36.203149+0000", 2024-09-14T04:57:36.212 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.3784e-05 2024-09-14T04:57:36.212 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:36.212 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:36.212 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:36.212 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:36.203167+0000", 2024-09-14T04:57:36.212 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.8352999999999999e-05 2024-09-14T04:57:36.213 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:36.213 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:36.213 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:36.213 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:36.203175+0000", 2024-09-14T04:57:36.213 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.4970000000000004e-06 2024-09-14T04:57:36.213 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:36.213 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:36.213 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:36.213 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:36.203220+0000", 2024-09-14T04:57:36.213 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.5491000000000002e-05 2024-09-14T04:57:36.213 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:36.213 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:36.213 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:36.214 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:36.203542+0000", 2024-09-14T04:57:36.214 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000321183 2024-09-14T04:57:36.214 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:36.214 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:36.214 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,5", 2024-09-14T04:57:36.214 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:36.203905+0000", 2024-09-14T04:57:36.214 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00036335600000000001 2024-09-14T04:57:36.214 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:36.214 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:36.214 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:36.214 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:36.214 INFO:teuthology.orchestra.run.smithi129.stdout: ], 2024-09-14T04:57:36.214 INFO:teuthology.orchestra.run.smithi129.stdout: "num_ops": 2 2024-09-14T04:57:36.214 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:57:36.235 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:36.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:35 smithi129 ceph-mon[101316]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:57:36.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:35 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T04:57:36.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:35 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:57:36.273 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:35 smithi129 ceph-mon[101316]: osdmap e1525: 8 total, 8 up, 8 in 2024-09-14T04:57:36.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:36.274 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:36.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:36.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:35 smithi039 ceph-mon[106470]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:57:36.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:35 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T04:57:36.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:35 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:57:36.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:35 smithi039 ceph-mon[106470]: osdmap e1525: 8 total, 8 up, 8 in 2024-09-14T04:57:36.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:35 smithi039 ceph-mon[111235]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:57:36.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:35 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T04:57:36.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:35 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:57:36.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:35 smithi039 ceph-mon[111235]: osdmap e1525: 8 total, 8 up, 8 in 2024-09-14T04:57:36.375 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:36.411 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:36.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:36.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:36.513 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:36.531 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:57:36.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:36.576 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:36.577 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:36.677 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:36.701 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 37 16 67707 67691 114.319 90.1875 0.0116873 0.00874188 2024-09-14T04:57:36.724 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:36.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:36.725 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:36.826 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:36.865 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:36.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:36.866 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:36.949 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-14T04:57:36.967 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:37.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:37.007 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:37.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:37.108 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:37.149 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:37.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:37.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:37.251 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:37.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:37 smithi129 ceph-mon[101316]: pgmap v2934: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-14T04:57:37.279 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:57:37.289 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:37.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:37.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:37.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:37 smithi039 ceph-mon[106470]: pgmap v2934: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-14T04:57:37.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:37 smithi039 ceph-mon[111235]: pgmap v2934: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-14T04:57:37.391 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:37.434 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:37.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:37.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:37.536 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:37.577 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:37.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:37.578 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:37.679 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:37.701 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 38 16 69119 69103 113.633 88.25 0.0176171 0.00879515 2024-09-14T04:57:37.713 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:37.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:37.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:37.815 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:37.854 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:37.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:37.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:37.956 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:37.989 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:37.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:37.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:38.076 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:57:38.092 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:38.122 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:38.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:38.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:38.224 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:38.282 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:38.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:38.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:38.384 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:38.406 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:38.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:38.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:38.507 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:38.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:38 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:57:38.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:38 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:57:38.538 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:38.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:38.539 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:38.539 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:38 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:57:38.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:38 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:57:38.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:38 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:57:38.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:38 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:57:38.640 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:38.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:38.678 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:38.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:38.701 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 39 16 70376 70360 112.734 78.5625 0.0125031 0.00886576 2024-09-14T04:57:38.780 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:38.821 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:38.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:38.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:38.923 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:38.963 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:38.964 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:38.964 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:39.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:57:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:57:38] "GET /metrics HTTP/1.1" 200 36727 "" "Prometheus/2.51.0" 2024-09-14T04:57:39.065 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:39.104 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:39.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:39.105 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:39.206 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:39.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:39.253 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:39.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:39.354 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:39.375 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:39.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:39.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:39.477 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:39.509 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:39.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:39.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:39.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:39 smithi129 ceph-mon[101316]: pgmap v2935: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-14T04:57:39.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:39 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1527321859' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:57:39.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:39 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:57:39.611 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:39.648 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:39.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:39.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:39.670 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.0 to 1 (10000) 2024-09-14T04:57:39.670 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:39 smithi039 ceph-mon[106470]: pgmap v2935: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-14T04:57:39.671 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:39 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1527321859' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:57:39.671 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:39 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:57:39.671 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:39 smithi039 ceph-mon[111235]: pgmap v2935: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-14T04:57:39.671 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:39 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1527321859' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:57:39.671 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:39 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T04:57:39.701 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:2024-09-14T04:57:39.701741+0000 min lat: 0.00123999 max lat: 0.470156 avg lat: 0.00895437 2024-09-14T04:57:39.701 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:57:39.702 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 40 16 71462 71446 111.612 67.875 0.00373668 0.00895437 2024-09-14T04:57:39.749 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:39.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:39.789 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:39.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:39.863 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:57:39.863 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [], 2024-09-14T04:57:39.864 INFO:teuthology.orchestra.run.smithi129.stdout: "complaint_time": 30, 2024-09-14T04:57:39.864 INFO:teuthology.orchestra.run.smithi129.stdout: "num_blocked_ops": 0 2024-09-14T04:57:39.864 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:57:39.890 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:39.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:39.929 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:39.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:40.030 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:40.062 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:40.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:40.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:40.164 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:40.200 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:40.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:40.201 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:40.302 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:40.333 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:40.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:40.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:40.435 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:40.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:40.472 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:40.472 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:40.513 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:40 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:57:40.514 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:40 smithi129 ceph-mon[101316]: osdmap e1526: 8 total, 8 up, 8 in 2024-09-14T04:57:40.553 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-14T04:57:40.572 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:40.609 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:40.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:40.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:40.701 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 41 16 72737 72721 110.834 79.6875 0.018102 0.00901727 2024-09-14T04:57:40.711 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:40.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:40.755 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:40.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:40.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:40 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:57:40.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:40 smithi039 ceph-mon[106470]: osdmap e1526: 8 total, 8 up, 8 in 2024-09-14T04:57:40.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:40 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-14T04:57:40.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:40 smithi039 ceph-mon[111235]: osdmap e1526: 8 total, 8 up, 8 in 2024-09-14T04:57:40.857 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:40.900 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:40.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:40.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:40.907 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:57:41.002 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:41.042 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:41.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:41.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:41.144 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:41.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:41.182 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:41.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:41.283 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:41.317 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:41.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:41.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:41.419 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:41.452 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:41.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:41.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:41.620 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:41.661 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:41.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:41.663 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:41.701 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 42 16 73994 73978 110.065 78.5625 0.0352699 0.00907807 2024-09-14T04:57:41.763 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:41.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:41 smithi129 ceph-mon[101316]: pgmap v2937: 99 pgs: 99 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-14T04:57:41.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:41 smithi039 ceph-mon[111235]: pgmap v2937: 99 pgs: 99 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-14T04:57:41.793 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:41 smithi039 ceph-mon[106470]: pgmap v2937: 99 pgs: 99 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s 2024-09-14T04:57:41.801 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:41.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:41.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:41.903 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:41.929 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:41.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:41.930 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:42.031 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:42.069 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:42.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:42.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:42.172 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:42.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:42.208 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:42.208 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:42.309 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:42.329 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:42.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:42.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:42.431 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:42.453 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:42.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:42.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:42.555 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:42.597 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:42.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:42.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:42.670 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:42 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:57:42.699 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:42.702 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 43 16 74943 74927 108.885 59.3125 0.00876954 0.00917903 2024-09-14T04:57:42.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:42.731 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:42.732 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:42.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:42 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:57:42.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:42 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:57:42.833 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:42.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:42.870 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:42.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:42.965 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 1 1 2024-09-14T04:57:42.971 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:43.000 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:43.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:43.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:43.025 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:57:43.102 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:43.135 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:43.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:43.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:43.237 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:43.272 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:43.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:43.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:43.374 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:43.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:43.402 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:43.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:43.503 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:43.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:43 smithi039 ceph-mon[106470]: pgmap v2938: 99 pgs: 99 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-09-14T04:57:43.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:43 smithi039 ceph-mon[111235]: pgmap v2938: 99 pgs: 99 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-09-14T04:57:43.552 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:43.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:43.553 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:43.654 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:43.699 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:43.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:43.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:43.701 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 44 16 76047 76031 107.978 69 0.0141452 0.00925608 2024-09-14T04:57:43.709 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:43 smithi129 ceph-mon[101316]: pgmap v2938: 99 pgs: 99 active+clean; 4.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s 2024-09-14T04:57:43.802 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:43.837 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:43.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:43.838 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:43.939 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:43.971 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:43.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:43.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:44.073 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:44.103 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:44.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:44.104 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:44.205 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:44.213 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:57:44.213 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:57:44.213 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:57:44.213 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:57:44.214 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:57:44.214 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:57:44.214 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:57:44.214 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:57:44.214 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:57:44.214 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:57:44.214 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:57:44.214 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:57:44.214 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:57:44.215 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:57:44.215 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:57:44.215 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:57:44.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:44.253 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:44.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:44.354 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:44.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:44.395 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:44.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:44.496 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:44.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:44 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2499270129' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:57:44.548 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:44.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:44.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:44.650 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:44.688 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:44.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:44.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:44.702 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 45 16 77271 77255 107.279 76.5 0.0141677 0.00931634 2024-09-14T04:57:44.724 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:57:44.725 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:57:44.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:44 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2499270129' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:57:44.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:44 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2499270129' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:57:44.790 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:44.826 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:44.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:44.827 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:44.928 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:44.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:44.974 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:44.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:44.983 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:57:45.074 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:45.106 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:45.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:45.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:45.208 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:45.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:45.255 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:45.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:45.356 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:45.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:45.401 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:45.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:45.428 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-14T04:57:45.502 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:45.554 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:45.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:45.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:45.624 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:45 smithi039 ceph-mon[106470]: pgmap v2939: 99 pgs: 99 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 925 op/s 2024-09-14T04:57:45.624 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:45 smithi039 ceph-mon[111235]: pgmap v2939: 99 pgs: 99 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 925 op/s 2024-09-14T04:57:45.657 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:45.690 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:45.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:45.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:45.702 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 46 16 78538 78522 106.668 79.1875 0.0144064 0.00936986 2024-09-14T04:57:45.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:45 smithi129 ceph-mon[101316]: pgmap v2939: 99 pgs: 99 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 925 op/s 2024-09-14T04:57:45.792 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:45.844 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:45.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:45.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:45.946 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:45.985 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:45.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:45.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:46.086 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:46.135 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:46.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:46.136 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:46.237 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:46.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:46.273 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:46.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:46.289 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:57:46.374 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:46.404 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:46.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:46.405 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:46.506 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:46.535 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:46.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:46.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:46.637 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:46.670 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:46.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:46.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:46.702 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 47 16 79659 79643 105.889 70.0625 0.00875552 0.00943889 2024-09-14T04:57:46.772 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:46.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:46.818 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:46.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:46.919 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:46.955 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:46.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:46.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:47.057 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:47.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:47.092 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:47.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:47.193 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:47.225 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:47.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:47.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:47.327 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:47.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:47.350 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:47.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:47.429 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is set 2024-09-14T04:57:47.451 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:47.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:47.496 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:47.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:47.533 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:57:47.598 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:47.640 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:47.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:47.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:47.702 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 48 16 80724 80708 105.07 66.5625 0.00563018 0.00951166 2024-09-14T04:57:47.742 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:47.762 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:47 smithi129 ceph-mon[101316]: pgmap v2940: 99 pgs: 99 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-14T04:57:47.762 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:47 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/955699997' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:57:47.762 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:47 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:57:47.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:47.779 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:47.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:47.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:47 smithi039 ceph-mon[106470]: pgmap v2940: 99 pgs: 99 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-14T04:57:47.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:47 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/955699997' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:57:47.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:47 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:57:47.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:47 smithi039 ceph-mon[111235]: pgmap v2940: 99 pgs: 99 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-14T04:57:47.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:47 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/955699997' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:57:47.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:47 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:57:47.880 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:47.905 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:47.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:47.906 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:48.006 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:48.055 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:48.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:48.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:48.157 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:48.203 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:48.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:48.204 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:48.304 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:48.346 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:48.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:48.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:48.440 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:48 smithi129 ceph-mon[101316]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:57:48.440 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:48 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:57:48.440 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:48 smithi129 ceph-mon[101316]: osdmap e1527: 8 total, 8 up, 8 in 2024-09-14T04:57:48.449 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:48.487 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:48.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:48.488 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:48.589 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:48.626 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:48 smithi039 ceph-mon[106470]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:57:48.626 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:48 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:57:48.627 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:48 smithi039 ceph-mon[106470]: osdmap e1527: 8 total, 8 up, 8 in 2024-09-14T04:57:48.627 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:48 smithi039 ceph-mon[111235]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:57:48.627 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:48 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:57:48.627 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:48 smithi039 ceph-mon[111235]: osdmap e1527: 8 total, 8 up, 8 in 2024-09-14T04:57:48.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:48.641 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:48.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:48.702 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 49 16 81831 81815 104.337 69.1875 0.0110942 0.0095792 2024-09-14T04:57:48.743 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:48.790 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:48.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:48.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:48.892 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:48.935 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:48.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:48.936 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:48.946 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:57:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:57:48] "GET /metrics HTTP/1.1" 200 36727 "" "Prometheus/2.51.0" 2024-09-14T04:57:49.037 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:49.077 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:57:49.078 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:57:49.085 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:49.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:49.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:49.187 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:49.231 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:49.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:49.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:49.333 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:49.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:49.376 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:49.376 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:49.477 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:49.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:49.520 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:49.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:49.621 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:49.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:49.665 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:49.665 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:49.669 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:49 smithi039 ceph-mon[106470]: pgmap v2942: 99 pgs: 99 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-14T04:57:49.669 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:49 smithi039 ceph-mon[111235]: pgmap v2942: 99 pgs: 99 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-14T04:57:49.704 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 50 16 82977 82961 103.683 71.625 0.00682564 0.00963933 2024-09-14T04:57:49.730 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:49 smithi129 ceph-mon[101316]: pgmap v2942: 99 pgs: 99 active+clean; 4.7 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 69 MiB/s wr, 1.10k op/s 2024-09-14T04:57:49.766 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:49.767 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-14T04:57:49.806 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:49.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:49.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:49.908 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:49.945 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:49.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:49.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:49.983 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:57:50.047 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:50.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:50.079 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:50.079 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:50.180 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:50.219 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:50.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:50.220 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:50.321 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:50.375 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:50.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:50.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:50.476 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.1 to 1 (10000) 2024-09-14T04:57:50.477 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:50.509 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:50.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:50.510 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:50.611 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:50.654 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:50.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:50.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:50.678 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:57:50.703 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 51 16 84140 84124 103.074 72.6875 0.0134631 0.0096967 2024-09-14T04:57:50.731 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:57:50.732 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:50 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1441686657' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:57:50.732 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:50 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:57:50.733 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:50 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1441686657' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:57:50.733 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:50 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:57:50.756 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:50.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:50 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1441686657' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:57:50.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:50 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T04:57:50.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:50.814 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:50.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:50.915 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:50.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:50.958 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:50.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:51.059 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:51.100 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:51.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:51.101 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:51.202 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:51.246 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:51.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:51.247 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:51.348 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:51.389 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:51.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:51.390 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:51.478 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:51 smithi039 ceph-mon[106470]: pgmap v2943: 99 pgs: 99 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-14T04:57:51.479 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:51 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-14T04:57:51.479 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:51 smithi039 ceph-mon[106470]: osdmap e1528: 8 total, 8 up, 8 in 2024-09-14T04:57:51.491 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:51.533 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:51.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:51.534 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:51.635 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:51.671 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:51.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:51.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:51.703 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 52 16 85377 85361 102.578 77.3125 0.0176269 0.00974323 2024-09-14T04:57:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:51 smithi129 ceph-mon[101316]: pgmap v2943: 99 pgs: 99 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-14T04:57:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:51 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-14T04:57:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:51 smithi129 ceph-mon[101316]: osdmap e1528: 8 total, 8 up, 8 in 2024-09-14T04:57:51.774 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:51.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:51 smithi039 ceph-mon[111235]: pgmap v2943: 99 pgs: 99 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-14T04:57:51.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:51 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-14T04:57:51.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:51 smithi039 ceph-mon[111235]: osdmap e1528: 8 total, 8 up, 8 in 2024-09-14T04:57:51.810 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:51.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:51.811 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:51.912 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:51.950 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:51.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:51.951 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:52.052 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:52.089 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:52.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:52.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:52.191 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:52.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:52.246 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:52.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:52.347 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:52.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:52.402 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:52.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:52.503 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:52.546 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:52.547 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:52.547 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:52.648 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:52.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:52.684 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:52.684 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:52.703 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 53 16 86532 86516 102.005 72.1875 0.0109479 0.00979871 2024-09-14T04:57:52.785 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:52.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:52.822 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:52.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:52.837 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 2 1 2024-09-14T04:57:52.923 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:52.959 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:52.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:52.960 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:53.061 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:53.108 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:53.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:53.109 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:53.119 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:57:53.147 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T04:57:53.147 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:57:53.210 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:53.239 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:53.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:53.240 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:53.341 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:53.374 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:53.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:53.375 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:53.448 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:53 smithi039 ceph-mon[106470]: pgmap v2945: 99 pgs: 99 active+clean; 5.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-14T04:57:53.448 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:53 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:57:53.448 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:53 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:57:53.448 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:53 smithi039 ceph-mon[111235]: pgmap v2945: 99 pgs: 99 active+clean; 5.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-14T04:57:53.448 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:53 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:57:53.448 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:53 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:57:53.476 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:53.515 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:53.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:53.516 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:53.617 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:53.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:53.662 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:53.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:53.703 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 54 16 87604 87588 101.357 67 0.0100294 0.00986061 2024-09-14T04:57:53.764 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:53 smithi129 ceph-mon[101316]: pgmap v2945: 99 pgs: 99 active+clean; 5.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 77 MiB/s wr, 1.23k op/s 2024-09-14T04:57:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:53 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:57:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:53 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:57:53.796 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:57:53.797 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:57:53.797 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:57:53.797 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:57:53.804 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:53.805 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:53.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:53.824 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:57:53.825 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:57:53.861 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:57:53.861 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:57:53.891 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:57:53.891 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:57:53.906 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:53.926 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:57:53.927 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:57:53.951 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:53.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:53.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:53.959 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:57:53.959 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:57:53.994 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:57:53.994 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:57:54.054 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:54.092 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:54.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:54.093 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:54.194 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:54.228 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-14T04:57:54.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:54.235 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:54.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:54.336 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:54.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:54.378 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:54.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:54.479 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:54.512 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:54 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2774385359' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:57:54.523 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:54.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:54.524 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:54.625 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:54.662 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:54.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:54.663 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:54.703 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 55 16 88740 88724 100.805 71 0.00881822 0.00991442 2024-09-14T04:57:54.764 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:54.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:54 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2774385359' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:57:54.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:54 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2774385359' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:57:54.805 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:54.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:54.806 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:54.907 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:54.935 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:57:54.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:54.968 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:54.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:54.984 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T04:57:55.069 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:55.116 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:55.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:55.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:55.218 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:55.253 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:55.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:55.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:55.355 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:55.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:55.402 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:55.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:55.496 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:55 smithi129 ceph-mon[101316]: pgmap v2946: 99 pgs: 99 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s 2024-09-14T04:57:55.503 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:55.564 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:55.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:55.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:55.666 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:55.680 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:55 smithi039 ceph-mon[106470]: pgmap v2946: 99 pgs: 99 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s 2024-09-14T04:57:55.680 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:55 smithi039 ceph-mon[111235]: pgmap v2946: 99 pgs: 99 active+clean; 5.2 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 59 MiB/s wr, 939 op/s 2024-09-14T04:57:55.703 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 56 16 89889 89873 100.287 71.8125 0.0149972 0.00996588 2024-09-14T04:57:55.712 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:55.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:55.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:55.815 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:55.858 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:55.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:55.859 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:55.959 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:56.002 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:56.003 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:56.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:56.104 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:56.142 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:56.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:56.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:56.244 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:56.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:56.280 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:56.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:56.381 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:56.417 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:56.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:56.418 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:56.519 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:56.556 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:56.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:56.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:56.658 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:56.684 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:56 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:57:56.684 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:56 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:57:56.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:56.698 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:56.698 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:56.703 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 57 16 91040 91024 99.7893 71.9375 0.0118525 0.0100157 2024-09-14T04:57:56.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:56 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:57:56.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:56 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:57:56.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:56 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:57:56.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:56 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:57:56.798 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:56.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:56.877 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:56.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:56.940 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:57:56.978 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:57.015 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T04:57:57.015 INFO:teuthology.orchestra.run.smithi129.stdout: "size": 20, 2024-09-14T04:57:57.015 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 600, 2024-09-14T04:57:57.015 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [ 2024-09-14T04:57:57.015 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.015 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:35309 56.a 56:52365448:::benchmark_data_smithi039_330847_object35308:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:57:57.015 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:44.505360+0000", 2024-09-14T04:57:57.015 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 312.505608919, 2024-09-14T04:57:57.015 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.339575285, 2024-09-14T04:57:57.015 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:57.016 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:57.016 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:57.016 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:57.016 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:57.016 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 35309 2024-09-14T04:57:57.016 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.016 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:57.016 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.017 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:57.017 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.505360+0000", 2024-09-14T04:57:57.017 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.017 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.017 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.017 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:57.017 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.505360+0000", 2024-09-14T04:57:57.017 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.017 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.017 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.018 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:57.018 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.505358+0000", 2024-09-14T04:57:57.018 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:57:57.018 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.018 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.018 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:57.018 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.505422+0000", 2024-09-14T04:57:57.018 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.3194999999999994e-05 2024-09-14T04:57:57.018 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.018 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.018 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:57.018 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.505437+0000", 2024-09-14T04:57:57.018 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5418e-05 2024-09-14T04:57:57.019 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.019 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.019 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:57.019 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.505444+0000", 2024-09-14T04:57:57.019 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.0160000000000003e-06 2024-09-14T04:57:57.019 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.019 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.019 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:57.019 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.505717+0000", 2024-09-14T04:57:57.019 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00027316500000000002 2024-09-14T04:57:57.019 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.019 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.020 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:57.020 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.505826+0000", 2024-09-14T04:57:57.020 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010878300000000001 2024-09-14T04:57:57.020 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.020 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.020 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:57:57.020 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.505966+0000", 2024-09-14T04:57:57.020 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000139593 2024-09-14T04:57:57.020 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.020 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.020 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:57.020 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.507870+0000", 2024-09-14T04:57:57.020 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0019046569999999999 2024-09-14T04:57:57.021 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.021 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.021 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.021 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.510868+0000", 2024-09-14T04:57:57.021 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0029972900000000001 2024-09-14T04:57:57.021 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.021 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.021 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.021 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.844823+0000", 2024-09-14T04:57:57.021 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33395536100000001 2024-09-14T04:57:57.021 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.021 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.021 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:57.021 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.844882+0000", 2024-09-14T04:57:57.022 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.94e-05 2024-09-14T04:57:57.022 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.022 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.022 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:57.022 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.844935+0000", 2024-09-14T04:57:57.022 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2599000000000002e-05 2024-09-14T04:57:57.022 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.022 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:57.022 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.022 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.022 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.022 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:35319 56.a 56:5b9192f7:::benchmark_data_smithi039_330847_object35318:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:57:57.023 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:44.508851+0000", 2024-09-14T04:57:57.023 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 312.50211726700002, 2024-09-14T04:57:57.023 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.34003841600000001, 2024-09-14T04:57:57.023 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:57.023 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:57.023 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:57.023 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:57.023 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:57.023 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 35319 2024-09-14T04:57:57.023 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.023 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:57.023 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.023 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:57.024 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508851+0000", 2024-09-14T04:57:57.024 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.024 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.024 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.024 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:57.024 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508851+0000", 2024-09-14T04:57:57.024 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.024 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.024 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.024 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:57.024 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508850+0000", 2024-09-14T04:57:57.024 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:57:57.024 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.025 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.025 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:57.025 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508879+0000", 2024-09-14T04:57:57.025 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.9454999999999998e-05 2024-09-14T04:57:57.025 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.025 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.025 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:57.025 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508895+0000", 2024-09-14T04:57:57.025 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.6254000000000001e-05 2024-09-14T04:57:57.025 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.025 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.025 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:57.026 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508903+0000", 2024-09-14T04:57:57.026 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.4139999999999997e-06 2024-09-14T04:57:57.026 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.026 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.026 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:57.026 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508953+0000", 2024-09-14T04:57:57.026 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.0311e-05 2024-09-14T04:57:57.026 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.026 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.026 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:57.026 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.509077+0000", 2024-09-14T04:57:57.026 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000124008 2024-09-14T04:57:57.026 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.027 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.027 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:57:57.027 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.509258+0000", 2024-09-14T04:57:57.027 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000180891 2024-09-14T04:57:57.027 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.027 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.027 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:57.027 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.510879+0000", 2024-09-14T04:57:57.027 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001620652 2024-09-14T04:57:57.027 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.027 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.027 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.027 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.512666+0000", 2024-09-14T04:57:57.027 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0017868820000000001 2024-09-14T04:57:57.028 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.028 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.028 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.028 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.848808+0000", 2024-09-14T04:57:57.028 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.336142777 2024-09-14T04:57:57.028 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.028 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.028 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:57.028 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.848854+0000", 2024-09-14T04:57:57.028 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6133e-05 2024-09-14T04:57:57.028 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.028 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.028 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:57.028 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.848890+0000", 2024-09-14T04:57:57.029 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5185999999999998e-05 2024-09-14T04:57:57.029 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.029 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:57.029 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.029 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.029 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.029 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:35323 56.a 56:5772dd4b:::benchmark_data_smithi039_330847_object35322:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:57:57.029 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:44.510844+0000", 2024-09-14T04:57:57.029 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 312.50012459700002, 2024-09-14T04:57:57.029 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33940646299999999, 2024-09-14T04:57:57.029 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:57.029 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:57.029 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:57.029 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:57.030 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:57.030 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 35323 2024-09-14T04:57:57.030 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.030 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:57.030 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.030 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:57.031 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.510844+0000", 2024-09-14T04:57:57.031 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.031 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.031 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.031 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:57.031 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.510844+0000", 2024-09-14T04:57:57.031 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.031 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.031 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.031 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:57.031 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.510843+0000", 2024-09-14T04:57:57.031 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:57:57.031 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.032 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.032 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:57.032 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.510948+0000", 2024-09-14T04:57:57.032 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000104839 2024-09-14T04:57:57.032 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.032 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.032 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:57.032 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.510963+0000", 2024-09-14T04:57:57.032 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5068999999999999e-05 2024-09-14T04:57:57.032 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.033 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.033 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:57.033 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.510970+0000", 2024-09-14T04:57:57.033 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.934e-06 2024-09-14T04:57:57.033 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.033 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.033 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:57.033 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.511079+0000", 2024-09-14T04:57:57.033 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010895000000000001 2024-09-14T04:57:57.033 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.033 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.033 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:57.033 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.511314+0000", 2024-09-14T04:57:57.034 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000235307 2024-09-14T04:57:57.034 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.034 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.034 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:57:57.034 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.511434+0000", 2024-09-14T04:57:57.034 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012017 2024-09-14T04:57:57.034 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.034 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.034 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:57.034 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.512769+0000", 2024-09-14T04:57:57.034 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00133442 2024-09-14T04:57:57.034 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.034 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.034 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.035 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.513589+0000", 2024-09-14T04:57:57.035 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00082001199999999995 2024-09-14T04:57:57.035 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.035 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.035 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.035 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.850170+0000", 2024-09-14T04:57:57.035 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33658163499999999 2024-09-14T04:57:57.035 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.035 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.035 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:57.035 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.850217+0000", 2024-09-14T04:57:57.035 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6789999999999998e-05 2024-09-14T04:57:57.035 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.036 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.036 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:57.036 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.850250+0000", 2024-09-14T04:57:57.036 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3402999999999999e-05 2024-09-14T04:57:57.036 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.036 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:57.036 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.036 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.036 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.036 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:35335 56.a 56:5cca2bfd:::benchmark_data_smithi039_330847_object35334:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:57:57.036 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:44.515184+0000", 2024-09-14T04:57:57.036 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 312.49578433699998, 2024-09-14T04:57:57.036 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33515958000000001, 2024-09-14T04:57:57.036 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:57.037 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:57.037 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:57.037 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:57.037 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:57.037 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 35335 2024-09-14T04:57:57.037 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.037 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:57.037 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.037 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:57.037 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.515184+0000", 2024-09-14T04:57:57.037 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.037 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.037 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.038 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:57.038 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.515184+0000", 2024-09-14T04:57:57.038 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.038 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.038 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.038 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:57.038 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.515183+0000", 2024-09-14T04:57:57.038 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:57:57.038 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.038 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.038 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:57.038 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.515220+0000", 2024-09-14T04:57:57.038 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6409999999999999e-05 2024-09-14T04:57:57.038 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.039 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.039 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:57.039 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.515232+0000", 2024-09-14T04:57:57.039 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2680000000000001e-05 2024-09-14T04:57:57.039 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.039 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.039 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:57.039 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.515239+0000", 2024-09-14T04:57:57.039 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.9569999999999996e-06 2024-09-14T04:57:57.039 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.039 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.039 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:57.039 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.515349+0000", 2024-09-14T04:57:57.039 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010991 2024-09-14T04:57:57.040 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.040 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.040 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:57.040 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.515460+0000", 2024-09-14T04:57:57.040 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000110398 2024-09-14T04:57:57.040 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.040 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.040 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:57:57.040 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.515600+0000", 2024-09-14T04:57:57.040 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013996700000000001 2024-09-14T04:57:57.040 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.040 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.040 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:57.041 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.517100+0000", 2024-09-14T04:57:57.041 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00150004 2024-09-14T04:57:57.041 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.041 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.041 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.041 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.520249+0000", 2024-09-14T04:57:57.041 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.003149584 2024-09-14T04:57:57.041 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.041 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.041 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.041 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.850283+0000", 2024-09-14T04:57:57.041 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33003364499999999 2024-09-14T04:57:57.041 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.041 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.041 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:57.042 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.850316+0000", 2024-09-14T04:57:57.042 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3037000000000002e-05 2024-09-14T04:57:57.042 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.042 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.042 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:57.042 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.850344+0000", 2024-09-14T04:57:57.042 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.7773e-05 2024-09-14T04:57:57.042 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.042 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:57.042 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.042 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.042 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.042 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:35341 56.a 56:5b382216:::benchmark_data_smithi039_330847_object35340:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:57:57.043 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:44.518706+0000", 2024-09-14T04:57:57.043 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 312.49226297600001, 2024-09-14T04:57:57.043 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33338658300000001, 2024-09-14T04:57:57.043 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:57.043 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:57.043 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:57.043 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:57.043 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:57.043 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 35341 2024-09-14T04:57:57.043 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.043 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:57.043 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.043 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:57.043 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.518706+0000", 2024-09-14T04:57:57.044 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.044 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.044 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.044 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:57.044 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.518706+0000", 2024-09-14T04:57:57.044 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.044 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.044 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.044 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:57.044 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.518704+0000", 2024-09-14T04:57:57.044 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:57:57.044 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.044 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.045 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:57.045 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:57.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:57.046 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:57.046 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.518744+0000", 2024-09-14T04:57:57.047 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9659000000000001e-05 2024-09-14T04:57:57.047 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.047 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.047 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:57.047 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.518757+0000", 2024-09-14T04:57:57.047 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.358e-05 2024-09-14T04:57:57.047 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.047 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.047 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:57.047 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.518765+0000", 2024-09-14T04:57:57.047 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.2080000000000003e-06 2024-09-14T04:57:57.047 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.047 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.048 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:57.048 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.518837+0000", 2024-09-14T04:57:57.048 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.2052999999999996e-05 2024-09-14T04:57:57.048 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.048 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.048 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:57.048 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.518956+0000", 2024-09-14T04:57:57.048 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000119437 2024-09-14T04:57:57.048 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.048 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.048 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:57:57.048 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.519095+0000", 2024-09-14T04:57:57.048 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013848500000000001 2024-09-14T04:57:57.049 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.049 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.049 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:57.049 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.520369+0000", 2024-09-14T04:57:57.049 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001274154 2024-09-14T04:57:57.049 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.049 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.049 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.049 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.521247+0000", 2024-09-14T04:57:57.049 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00087840900000000003 2024-09-14T04:57:57.049 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.049 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.049 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.050 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.852014+0000", 2024-09-14T04:57:57.050 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.330766387 2024-09-14T04:57:57.050 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.050 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.050 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:57.050 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.852055+0000", 2024-09-14T04:57:57.050 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1093000000000001e-05 2024-09-14T04:57:57.050 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.050 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.050 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:57.050 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.852092+0000", 2024-09-14T04:57:57.050 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7452e-05 2024-09-14T04:57:57.050 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.051 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:57.051 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.051 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.051 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.051 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:116251 56.b 56:d0b5012a:::benchmark_data_smithi039_330847_object1727:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1486)", 2024-09-14T04:57:57.051 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:04.547766+0000", 2024-09-14T04:57:57.051 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 232.463202354, 2024-09-14T04:57:57.051 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.488460691, 2024-09-14T04:57:57.051 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:57.051 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:57.051 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:57.051 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:57.051 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:57.052 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 116251 2024-09-14T04:57:57.052 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.052 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:57.052 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.052 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:57.052 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.547766+0000", 2024-09-14T04:57:57.052 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.052 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.052 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.052 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:57.052 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.547766+0000", 2024-09-14T04:57:57.052 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.053 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.053 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.053 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:57.053 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.547765+0000", 2024-09-14T04:57:57.053 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:57:57.053 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.053 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.053 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:57.053 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.547771+0000", 2024-09-14T04:57:57.053 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.7209999999999999e-06 2024-09-14T04:57:57.053 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.053 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.053 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:57.054 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.547779+0000", 2024-09-14T04:57:57.054 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.7020000000000007e-06 2024-09-14T04:57:57.054 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.054 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.054 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:57.054 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.547785+0000", 2024-09-14T04:57:57.054 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9039999999999997e-06 2024-09-14T04:57:57.054 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.054 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.054 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:57.054 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.551793+0000", 2024-09-14T04:57:57.054 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0040079240000000004 2024-09-14T04:57:57.054 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.054 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.055 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:57.055 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.551900+0000", 2024-09-14T04:57:57.055 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000107318 2024-09-14T04:57:57.055 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.055 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.055 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,5", 2024-09-14T04:57:57.055 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.552013+0000", 2024-09-14T04:57:57.055 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000112469 2024-09-14T04:57:57.055 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.055 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.055 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:57.055 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.555580+0000", 2024-09-14T04:57:57.055 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0035672360000000001 2024-09-14T04:57:57.056 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.056 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.056 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.056 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.558497+0000", 2024-09-14T04:57:57.056 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0029165189999999998 2024-09-14T04:57:57.056 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.056 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.056 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.056 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036142+0000", 2024-09-14T04:57:57.056 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.47764563300000001 2024-09-14T04:57:57.056 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.056 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.056 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:57.057 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036189+0000", 2024-09-14T04:57:57.057 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7036999999999997e-05 2024-09-14T04:57:57.057 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.057 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.057 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:57.057 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036227+0000", 2024-09-14T04:57:57.057 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7576000000000001e-05 2024-09-14T04:57:57.057 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.057 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:57.057 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.057 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.057 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.058 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:116268 56.b 56:de88060a:::benchmark_data_smithi039_330847_object1744:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1486)", 2024-09-14T04:57:57.058 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:04.637502+0000", 2024-09-14T04:57:57.058 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 232.37346677299999, 2024-09-14T04:57:57.058 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.39881927299999997, 2024-09-14T04:57:57.058 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:57.058 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:57.058 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:57.058 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:57.058 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:57.058 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 116268 2024-09-14T04:57:57.058 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.058 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:57.058 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.059 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:57.059 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.637502+0000", 2024-09-14T04:57:57.059 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.059 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.059 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.059 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:57.059 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.637502+0000", 2024-09-14T04:57:57.059 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.059 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.059 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.059 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:57.059 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.637501+0000", 2024-09-14T04:57:57.059 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:57:57.060 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.060 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.060 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:57.060 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.637505+0000", 2024-09-14T04:57:57.060 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7349999999999998e-06 2024-09-14T04:57:57.060 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.060 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.060 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:57.060 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.637515+0000", 2024-09-14T04:57:57.060 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0519e-05 2024-09-14T04:57:57.060 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.060 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.060 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:57.061 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.637522+0000", 2024-09-14T04:57:57.061 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.6429999999999998e-06 2024-09-14T04:57:57.061 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.061 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.061 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:57.061 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.637562+0000", 2024-09-14T04:57:57.061 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0305999999999997e-05 2024-09-14T04:57:57.061 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.061 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.061 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:57.061 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.637689+0000", 2024-09-14T04:57:57.061 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012666000000000001 2024-09-14T04:57:57.061 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.062 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.062 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,5", 2024-09-14T04:57:57.062 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.637820+0000", 2024-09-14T04:57:57.062 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000130877 2024-09-14T04:57:57.062 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.062 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.062 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:57.062 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.646403+0000", 2024-09-14T04:57:57.062 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0085835389999999994 2024-09-14T04:57:57.062 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.062 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.062 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.062 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.649755+0000", 2024-09-14T04:57:57.063 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0033511610000000001 2024-09-14T04:57:57.063 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.063 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.063 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.063 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036263+0000", 2024-09-14T04:57:57.063 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.38650827100000001 2024-09-14T04:57:57.063 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.063 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.063 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:57.063 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036294+0000", 2024-09-14T04:57:57.063 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.1164000000000002e-05 2024-09-14T04:57:57.063 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.063 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.063 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:57.063 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036321+0000", 2024-09-14T04:57:57.064 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.7030999999999999e-05 2024-09-14T04:57:57.064 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.064 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:57.064 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.064 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.064 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.064 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:116284 56.b 56:d7432e9a:::benchmark_data_smithi039_330847_object1760:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1486)", 2024-09-14T04:57:57.064 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:04.658571+0000", 2024-09-14T04:57:57.064 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 232.352397569, 2024-09-14T04:57:57.064 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.37787099099999999, 2024-09-14T04:57:57.064 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:57.064 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:57.064 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:57.065 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:57.065 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:57.065 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 116284 2024-09-14T04:57:57.065 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.065 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:57.065 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.065 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:57.065 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658571+0000", 2024-09-14T04:57:57.065 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.065 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.065 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.065 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:57.065 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658571+0000", 2024-09-14T04:57:57.065 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.066 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.066 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.066 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:57.066 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658569+0000", 2024-09-14T04:57:57.066 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T04:57:57.066 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.066 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.066 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:57.066 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658577+0000", 2024-09-14T04:57:57.066 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.7889999999999995e-06 2024-09-14T04:57:57.066 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.066 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.066 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:57.067 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658592+0000", 2024-09-14T04:57:57.067 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5287000000000001e-05 2024-09-14T04:57:57.067 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.067 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.067 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:57.067 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658600+0000", 2024-09-14T04:57:57.067 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.6240000000000002e-06 2024-09-14T04:57:57.067 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.067 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.067 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:57.067 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658647+0000", 2024-09-14T04:57:57.067 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6965000000000002e-05 2024-09-14T04:57:57.067 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.068 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.068 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:57.068 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658748+0000", 2024-09-14T04:57:57.068 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000101005 2024-09-14T04:57:57.068 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.068 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.068 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,5", 2024-09-14T04:57:57.068 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658842+0000", 2024-09-14T04:57:57.068 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.4204999999999997e-05 2024-09-14T04:57:57.068 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.068 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.068 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:57.068 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.662555+0000", 2024-09-14T04:57:57.069 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0037132350000000001 2024-09-14T04:57:57.069 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.069 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.069 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.069 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.663869+0000", 2024-09-14T04:57:57.069 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001313952 2024-09-14T04:57:57.069 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.069 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.069 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.069 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036369+0000", 2024-09-14T04:57:57.069 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.37249975600000002 2024-09-14T04:57:57.069 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.069 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.070 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:57.070 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036410+0000", 2024-09-14T04:57:57.070 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0692000000000003e-05 2024-09-14T04:57:57.070 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.070 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.070 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:57.070 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036442+0000", 2024-09-14T04:57:57.070 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.2181000000000002e-05 2024-09-14T04:57:57.070 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.070 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:57.070 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.070 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.070 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.071 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:116292 56.4 56:2d450bf4:::benchmark_data_smithi039_330847_object1768:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1486)", 2024-09-14T04:57:57.071 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:04.670078+0000", 2024-09-14T04:57:57.071 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 232.340890441, 2024-09-14T04:57:57.071 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.374447525, 2024-09-14T04:57:57.071 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:57.071 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:57.071 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:57.071 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:57.071 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:57.071 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 116292 2024-09-14T04:57:57.071 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.071 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:57.071 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.072 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:57.072 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670078+0000", 2024-09-14T04:57:57.072 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.072 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.072 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.072 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:57.072 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670078+0000", 2024-09-14T04:57:57.072 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.072 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.072 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.072 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:57.072 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670077+0000", 2024-09-14T04:57:57.072 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:57:57.073 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.073 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.073 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:57.073 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670082+0000", 2024-09-14T04:57:57.073 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8439999999999997e-06 2024-09-14T04:57:57.073 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.073 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.073 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:57.073 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670090+0000", 2024-09-14T04:57:57.073 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.6989999999999992e-06 2024-09-14T04:57:57.073 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.073 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.073 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:57.074 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670097+0000", 2024-09-14T04:57:57.074 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.1589999999999998e-06 2024-09-14T04:57:57.074 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.074 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.074 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:57.074 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670207+0000", 2024-09-14T04:57:57.074 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011047100000000001 2024-09-14T04:57:57.074 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.074 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.074 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:57.074 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670315+0000", 2024-09-14T04:57:57.074 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010752299999999999 2024-09-14T04:57:57.074 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.074 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.075 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:57:57.075 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036104+0000", 2024-09-14T04:57:57.075 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.36578902299999999 2024-09-14T04:57:57.075 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.075 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.075 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:57.075 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.040066+0000", 2024-09-14T04:57:57.075 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0039619390000000003 2024-09-14T04:57:57.075 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.075 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.075 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.075 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.044355+0000", 2024-09-14T04:57:57.075 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0042896779999999999 2024-09-14T04:57:57.075 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.076 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.076 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.076 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.044433+0000", 2024-09-14T04:57:57.076 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.7905999999999996e-05 2024-09-14T04:57:57.076 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.076 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.076 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:57.076 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.044483+0000", 2024-09-14T04:57:57.076 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.0207000000000002e-05 2024-09-14T04:57:57.076 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.076 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.076 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:57.077 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.044526+0000", 2024-09-14T04:57:57.077 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.2184999999999997e-05 2024-09-14T04:57:57.077 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.077 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:57.077 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.077 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.077 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.077 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.36012.0:116283 56.6 e1486/1476 56:68220df0:::benchmark_data_smithi039_330847_object1759:head v 1486'7117, mlcod=0'0)", 2024-09-14T04:57:57.077 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:04.670105+0000", 2024-09-14T04:57:57.077 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 232.340863149, 2024-09-14T04:57:57.077 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.37733796400000003, 2024-09-14T04:57:57.077 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:57.077 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:57.077 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:57.078 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.078 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:57.078 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670105+0000", 2024-09-14T04:57:57.078 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.078 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.078 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.078 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:57.078 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670105+0000", 2024-09-14T04:57:57.078 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.078 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.078 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.078 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:57.078 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670105+0000", 2024-09-14T04:57:57.079 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T04:57:57.079 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.079 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.079 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:57.079 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670111+0000", 2024-09-14T04:57:57.079 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.02e-06 2024-09-14T04:57:57.079 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.079 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.079 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:57.079 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670118+0000", 2024-09-14T04:57:57.079 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.661e-06 2024-09-14T04:57:57.079 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.079 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.080 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:57.080 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670124+0000", 2024-09-14T04:57:57.080 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.3539999999999999e-06 2024-09-14T04:57:57.080 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.080 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.080 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:57.080 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670163+0000", 2024-09-14T04:57:57.080 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9483999999999997e-05 2024-09-14T04:57:57.080 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.080 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.080 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:57.080 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670186+0000", 2024-09-14T04:57:57.080 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.2918999999999999e-05 2024-09-14T04:57:57.081 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.081 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.081 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:57.081 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.047411+0000", 2024-09-14T04:57:57.081 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.37722491000000002 2024-09-14T04:57:57.081 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.081 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.081 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:57.081 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.047443+0000", 2024-09-14T04:57:57.081 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.2203000000000003e-05 2024-09-14T04:57:57.081 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.081 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:57.081 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.082 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.082 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.082 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:116294 56.4 56:2b7b2e82:::benchmark_data_smithi039_330847_object1770:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1486)", 2024-09-14T04:57:57.082 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:04.670327+0000", 2024-09-14T04:57:57.082 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 232.34064112499999, 2024-09-14T04:57:57.082 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.377658365, 2024-09-14T04:57:57.082 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:57.082 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:57.082 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:57.082 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:57.082 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:57.082 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 116294 2024-09-14T04:57:57.082 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.083 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:57.083 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.083 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:57.083 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670327+0000", 2024-09-14T04:57:57.083 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.083 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.083 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.083 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:57.083 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670327+0000", 2024-09-14T04:57:57.083 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.083 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.083 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.083 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:57.083 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670327+0000", 2024-09-14T04:57:57.084 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:57:57.084 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.084 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.084 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:57.084 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670331+0000", 2024-09-14T04:57:57.084 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.048e-06 2024-09-14T04:57:57.084 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.084 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.084 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:57.084 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036115+0000", 2024-09-14T04:57:57.084 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.36578455700000001 2024-09-14T04:57:57.084 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.084 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.085 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:57.085 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036124+0000", 2024-09-14T04:57:57.085 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.9420000000000001e-06 2024-09-14T04:57:57.085 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.085 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.085 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:57.085 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.040091+0000", 2024-09-14T04:57:57.085 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0039670720000000003 2024-09-14T04:57:57.085 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.085 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.086 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:57.086 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.040277+0000", 2024-09-14T04:57:57.086 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000185954 2024-09-14T04:57:57.086 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.086 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.086 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:57:57.086 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.040378+0000", 2024-09-14T04:57:57.086 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010124300000000001 2024-09-14T04:57:57.086 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.086 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.086 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.086 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.044582+0000", 2024-09-14T04:57:57.086 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0042034259999999997 2024-09-14T04:57:57.086 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.087 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.087 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.087 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.044612+0000", 2024-09-14T04:57:57.087 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.0540999999999998e-05 2024-09-14T04:57:57.087 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.087 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.087 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:57.087 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.047897+0000", 2024-09-14T04:57:57.087 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.003284281 2024-09-14T04:57:57.087 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.087 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.087 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:57.087 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.047948+0000", 2024-09-14T04:57:57.088 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.1749000000000002e-05 2024-09-14T04:57:57.088 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.088 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.088 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:57.088 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.047986+0000", 2024-09-14T04:57:57.088 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7357999999999997e-05 2024-09-14T04:57:57.088 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.088 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:57.088 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.088 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.088 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.088 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop_reply(client.36012.0:116251 56.b e1486/1470 ondisk, result = 0)", 2024-09-14T04:57:57.088 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:04.672410+0000", 2024-09-14T04:57:57.089 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 232.33855817, 2024-09-14T04:57:57.089 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.36382398599999999, 2024-09-14T04:57:57.089 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:57.089 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "started", 2024-09-14T04:57:57.089 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:57.089 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.089 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:57.089 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.672410+0000", 2024-09-14T04:57:57.089 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.089 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.089 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.089 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:57.089 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.672410+0000", 2024-09-14T04:57:57.090 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.090 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.090 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.090 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:57.090 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.672409+0000", 2024-09-14T04:57:57.090 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:57:57.090 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.090 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.090 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:57.091 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.672416+0000", 2024-09-14T04:57:57.091 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.0160000000000003e-06 2024-09-14T04:57:57.091 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.092 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.092 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:57.092 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036080+0000", 2024-09-14T04:57:57.092 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.363663499 2024-09-14T04:57:57.092 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.092 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.092 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:57.092 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036088+0000", 2024-09-14T04:57:57.092 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.8250000000000005e-06 2024-09-14T04:57:57.092 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.092 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.092 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:57.093 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036126+0000", 2024-09-14T04:57:57.093 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8379e-05 2024-09-14T04:57:57.093 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.093 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.093 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:57.093 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036136+0000", 2024-09-14T04:57:57.093 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.6330000000000008e-06 2024-09-14T04:57:57.093 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.093 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.093 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:57.093 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036234+0000", 2024-09-14T04:57:57.093 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.8673999999999997e-05 2024-09-14T04:57:57.093 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.093 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:57.094 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.094 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.094 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.094 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.36012.0:116293 56.5 e1486/1470 56:a6762fc1:::benchmark_data_smithi039_330847_object1769:head v 1486'7278, mlcod=1486'7276)", 2024-09-14T04:57:57.094 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:04.673699+0000", 2024-09-14T04:57:57.094 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 232.337269107, 2024-09-14T04:57:57.094 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.37533592999999998, 2024-09-14T04:57:57.094 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:57.094 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:57.094 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:57.094 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.094 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:57.094 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.673699+0000", 2024-09-14T04:57:57.095 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.095 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.095 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.095 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:57.095 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.673699+0000", 2024-09-14T04:57:57.095 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.095 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.095 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.095 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:57.095 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.673699+0000", 2024-09-14T04:57:57.095 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:57:57.095 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.095 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.095 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:57.096 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.673707+0000", 2024-09-14T04:57:57.096 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.5599999999999994e-06 2024-09-14T04:57:57.096 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.096 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.096 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:57.096 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.035992+0000", 2024-09-14T04:57:57.096 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.36228451699999997 2024-09-14T04:57:57.096 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.096 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.096 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:57.096 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036003+0000", 2024-09-14T04:57:57.096 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0883999999999999e-05 2024-09-14T04:57:57.096 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.097 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.097 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:57.097 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036051+0000", 2024-09-14T04:57:57.097 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8485000000000001e-05 2024-09-14T04:57:57.097 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.097 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.097 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:57.097 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036084+0000", 2024-09-14T04:57:57.097 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3222999999999997e-05 2024-09-14T04:57:57.097 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.097 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.097 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:57.097 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.049003+0000", 2024-09-14T04:57:57.098 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.012919058000000001 2024-09-14T04:57:57.098 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.098 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.098 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:57.098 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.049035+0000", 2024-09-14T04:57:57.098 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.2027999999999999e-05 2024-09-14T04:57:57.098 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.098 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:57.098 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.098 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.098 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.098 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:172819 56.a 56:59afb196:::benchmark_data_smithi039_330847_object58295:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1508)", 2024-09-14T04:57:57.098 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:55:16.726954+0000", 2024-09-14T04:57:57.098 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 160.28401409899999, 2024-09-14T04:57:57.099 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.207837457, 2024-09-14T04:57:57.099 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:57.099 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:57.099 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:57.099 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:57.099 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:57.099 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 172819 2024-09-14T04:57:57.099 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.099 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:57.099 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.099 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:57.099 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.726954+0000", 2024-09-14T04:57:57.099 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.099 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.100 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.100 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:57.100 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.726954+0000", 2024-09-14T04:57:57.100 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.100 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.100 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.100 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:57.100 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.726953+0000", 2024-09-14T04:57:57.100 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:57:57.100 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.100 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.100 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:57.100 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.726958+0000", 2024-09-14T04:57:57.101 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7729999999999999e-06 2024-09-14T04:57:57.101 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.101 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.101 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:57.101 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.726966+0000", 2024-09-14T04:57:57.101 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.1009999999999992e-06 2024-09-14T04:57:57.101 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.101 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.101 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:57.101 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.726970+0000", 2024-09-14T04:57:57.101 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0910000000000003e-06 2024-09-14T04:57:57.101 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.101 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.101 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:57.102 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.727006+0000", 2024-09-14T04:57:57.102 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5125e-05 2024-09-14T04:57:57.102 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.102 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.102 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:57.102 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.727093+0000", 2024-09-14T04:57:57.102 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.7620000000000005e-05 2024-09-14T04:57:57.102 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.102 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.102 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:57:57.102 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.727169+0000", 2024-09-14T04:57:57.102 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.5845000000000003e-05 2024-09-14T04:57:57.102 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.102 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.103 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.103 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.735163+0000", 2024-09-14T04:57:57.103 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0079942829999999996 2024-09-14T04:57:57.103 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.103 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.103 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:57.103 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.735175+0000", 2024-09-14T04:57:57.103 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1413000000000001e-05 2024-09-14T04:57:57.103 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.103 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.103 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.103 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.934704+0000", 2024-09-14T04:57:57.103 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.1995296 2024-09-14T04:57:57.103 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.104 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.104 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:57.104 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.934750+0000", 2024-09-14T04:57:57.104 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6066999999999998e-05 2024-09-14T04:57:57.104 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.104 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.104 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:57.104 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.934792+0000", 2024-09-14T04:57:57.104 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1474000000000003e-05 2024-09-14T04:57:57.104 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.104 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:57.104 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.104 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.105 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.105 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:172820 56.a 56:5f82633b:::benchmark_data_smithi039_330847_object58296:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1508)", 2024-09-14T04:57:57.105 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:55:16.727576+0000", 2024-09-14T04:57:57.105 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 160.283392096, 2024-09-14T04:57:57.105 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.20734001899999999, 2024-09-14T04:57:57.105 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:57.105 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:57.105 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:57.105 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:57.105 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:57.105 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 172820 2024-09-14T04:57:57.105 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.105 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:57.106 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.106 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:57.106 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.727576+0000", 2024-09-14T04:57:57.106 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.106 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.106 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.106 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:57.106 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.727576+0000", 2024-09-14T04:57:57.106 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.106 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.106 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.106 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:57.106 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.727576+0000", 2024-09-14T04:57:57.107 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:57:57.107 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.107 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.107 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:57.107 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.727582+0000", 2024-09-14T04:57:57.107 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.3929999999999996e-06 2024-09-14T04:57:57.107 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.107 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.107 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:57.107 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.727592+0000", 2024-09-14T04:57:57.107 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.5999999999999996e-06 2024-09-14T04:57:57.107 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.107 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.107 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:57.108 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.727596+0000", 2024-09-14T04:57:57.108 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6650000000000002e-06 2024-09-14T04:57:57.108 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.108 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.108 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:57.108 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.731628+0000", 2024-09-14T04:57:57.108 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0040314310000000003 2024-09-14T04:57:57.108 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.108 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.108 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:57.108 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.731741+0000", 2024-09-14T04:57:57.108 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000113006 2024-09-14T04:57:57.108 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.109 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.109 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,6", 2024-09-14T04:57:57.109 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.731854+0000", 2024-09-14T04:57:57.109 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011318499999999999 2024-09-14T04:57:57.109 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.109 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.109 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:57.109 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.735243+0000", 2024-09-14T04:57:57.109 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0033891210000000001 2024-09-14T04:57:57.109 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.109 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.109 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.109 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.735272+0000", 2024-09-14T04:57:57.109 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.9430000000000001e-05 2024-09-14T04:57:57.110 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.110 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.110 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.110 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.934834+0000", 2024-09-14T04:57:57.110 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.19956151899999999 2024-09-14T04:57:57.110 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.110 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.110 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:57.110 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.934881+0000", 2024-09-14T04:57:57.110 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7435999999999998e-05 2024-09-14T04:57:57.110 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.110 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.110 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:57.110 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.934916+0000", 2024-09-14T04:57:57.111 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5011000000000001e-05 2024-09-14T04:57:57.111 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.111 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:57.111 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.111 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.111 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.111 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:172821 56.f 56:f6101387:::benchmark_data_smithi039_330847_object58297:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1508)", 2024-09-14T04:57:57.111 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:55:16.728075+0000", 2024-09-14T04:57:57.111 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 160.28289398300001, 2024-09-14T04:57:57.111 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.206655599, 2024-09-14T04:57:57.111 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:57.111 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:57.111 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:57.111 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:57.112 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:57.112 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 172821 2024-09-14T04:57:57.112 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.112 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:57.112 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.112 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:57.112 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.728075+0000", 2024-09-14T04:57:57.112 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.112 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.112 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.112 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:57.113 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.728075+0000", 2024-09-14T04:57:57.113 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.113 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.113 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.113 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:57.113 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.728074+0000", 2024-09-14T04:57:57.113 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T04:57:57.113 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.113 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.113 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:57.113 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.728079+0000", 2024-09-14T04:57:57.113 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8239999999999999e-06 2024-09-14T04:57:57.113 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.114 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.114 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:57.114 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.728091+0000", 2024-09-14T04:57:57.114 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2761e-05 2024-09-14T04:57:57.114 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.114 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.114 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:57.114 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.728098+0000", 2024-09-14T04:57:57.114 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.37e-06 2024-09-14T04:57:57.114 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.114 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.114 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:57.114 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.728134+0000", 2024-09-14T04:57:57.115 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6316000000000003e-05 2024-09-14T04:57:57.115 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.115 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.115 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:57.115 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.728258+0000", 2024-09-14T04:57:57.115 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012450199999999999 2024-09-14T04:57:57.115 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.115 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.115 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,7", 2024-09-14T04:57:57.115 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.728381+0000", 2024-09-14T04:57:57.115 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012210300000000001 2024-09-14T04:57:57.115 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.115 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.116 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:57.116 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.731944+0000", 2024-09-14T04:57:57.116 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0035638509999999998 2024-09-14T04:57:57.116 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.116 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.116 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.116 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.734274+0000", 2024-09-14T04:57:57.116 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0023291229999999998 2024-09-14T04:57:57.116 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.116 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.116 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.116 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.934628+0000", 2024-09-14T04:57:57.116 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.200354698 2024-09-14T04:57:57.117 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.117 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.117 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:57.117 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.934665+0000", 2024-09-14T04:57:57.117 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7230000000000001e-05 2024-09-14T04:57:57.117 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.117 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.117 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:57.117 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.934730+0000", 2024-09-14T04:57:57.117 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.4645000000000002e-05 2024-09-14T04:57:57.117 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.117 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:57.117 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.117 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.118 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.118 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:172825 56.4 56:23b09923:::benchmark_data_smithi039_330847_object58301:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1508)", 2024-09-14T04:57:57.118 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:55:16.736609+0000", 2024-09-14T04:57:57.118 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 160.27435973300001, 2024-09-14T04:57:57.118 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.21087603499999999, 2024-09-14T04:57:57.118 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:57.118 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:57.118 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:57.118 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:57.118 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:57.118 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 172825 2024-09-14T04:57:57.118 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.118 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:57.119 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.119 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:57.119 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.736609+0000", 2024-09-14T04:57:57.119 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.119 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.119 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.119 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:57.119 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.736609+0000", 2024-09-14T04:57:57.119 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.119 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.119 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.119 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:57.119 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.736607+0000", 2024-09-14T04:57:57.120 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:57:57.120 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.120 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.120 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:57.120 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.736613+0000", 2024-09-14T04:57:57.120 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.6969999999999998e-06 2024-09-14T04:57:57.120 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.120 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.120 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:57.120 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.736627+0000", 2024-09-14T04:57:57.120 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3918e-05 2024-09-14T04:57:57.120 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.120 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.121 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:57.121 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.736634+0000", 2024-09-14T04:57:57.121 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.1629999999999999e-06 2024-09-14T04:57:57.121 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.121 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.121 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:57.121 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.736675+0000", 2024-09-14T04:57:57.121 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1174999999999998e-05 2024-09-14T04:57:57.121 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.121 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.121 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:57.121 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.736786+0000", 2024-09-14T04:57:57.121 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0001106 2024-09-14T04:57:57.121 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.122 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.122 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,2", 2024-09-14T04:57:57.122 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.736901+0000", 2024-09-14T04:57:57.122 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000114898 2024-09-14T04:57:57.122 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.122 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.122 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:57.122 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.740622+0000", 2024-09-14T04:57:57.122 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00372106 2024-09-14T04:57:57.122 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.122 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.122 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.122 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.743142+0000", 2024-09-14T04:57:57.123 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0025199659999999998 2024-09-14T04:57:57.123 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.123 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.123 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.123 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.947396+0000", 2024-09-14T04:57:57.123 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.20425436899999999 2024-09-14T04:57:57.123 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.123 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.123 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:57.123 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.947446+0000", 2024-09-14T04:57:57.123 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.9954999999999999e-05 2024-09-14T04:57:57.123 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.123 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.124 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:57.124 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.947485+0000", 2024-09-14T04:57:57.124 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8519000000000002e-05 2024-09-14T04:57:57.124 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.124 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:57.124 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.124 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.124 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.124 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:172838 56.f 56:f43eb43d:::benchmark_data_smithi039_330847_object58314:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1508)", 2024-09-14T04:57:57.124 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:55:16.754456+0000", 2024-09-14T04:57:57.124 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 160.25651237700001, 2024-09-14T04:57:57.124 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.196324211, 2024-09-14T04:57:57.124 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:57.124 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:57.125 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:57.125 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T04:57:57.125 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:57:57.125 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 172838 2024-09-14T04:57:57.125 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.125 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:57.125 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.125 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:57.125 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.754456+0000", 2024-09-14T04:57:57.125 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.126 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.126 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.126 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:57.126 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.754456+0000", 2024-09-14T04:57:57.126 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.126 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.126 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.126 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:57.126 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.754446+0000", 2024-09-14T04:57:57.126 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.99999 2024-09-14T04:57:57.126 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.126 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.126 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:57.126 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.754467+0000", 2024-09-14T04:57:57.127 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.0752999999999999e-05 2024-09-14T04:57:57.127 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.127 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.127 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:57.127 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.754498+0000", 2024-09-14T04:57:57.127 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.0870000000000001e-05 2024-09-14T04:57:57.127 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.127 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.127 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:57.127 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.754512+0000", 2024-09-14T04:57:57.127 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.4283e-05 2024-09-14T04:57:57.127 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.127 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.128 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:57.128 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.754610+0000", 2024-09-14T04:57:57.128 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.8238000000000004e-05 2024-09-14T04:57:57.128 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.128 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.128 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:57.128 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.754804+0000", 2024-09-14T04:57:57.128 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00019393500000000001 2024-09-14T04:57:57.128 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.128 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.128 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,7", 2024-09-14T04:57:57.128 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.754880+0000", 2024-09-14T04:57:57.128 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.5637999999999996e-05 2024-09-14T04:57:57.128 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.129 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.129 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:57.129 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.758112+0000", 2024-09-14T04:57:57.129 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0032323439999999998 2024-09-14T04:57:57.129 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.129 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.129 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.129 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.758336+0000", 2024-09-14T04:57:57.129 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00022379299999999999 2024-09-14T04:57:57.129 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.129 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.129 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.129 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.950659+0000", 2024-09-14T04:57:57.129 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.19232284599999999 2024-09-14T04:57:57.130 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.130 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.130 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:57.130 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.950741+0000", 2024-09-14T04:57:57.130 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.2429000000000006e-05 2024-09-14T04:57:57.130 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.130 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.130 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:57.130 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.950780+0000", 2024-09-14T04:57:57.130 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9011999999999999e-05 2024-09-14T04:57:57.130 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.130 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:57.130 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.130 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.131 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.131 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36324.0:34412 57.9 57:9b838976:::benchmark_data_smithi039_365565_object34411:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T04:57:57.131 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:57:13.470008+0000", 2024-09-14T04:57:57.131 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 43.540960566999999, 2024-09-14T04:57:57.131 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.43965552499999999, 2024-09-14T04:57:57.131 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:57.131 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:57.131 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T04:57:57.131 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36324", 2024-09-14T04:57:57.131 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T04:57:57.131 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 34412 2024-09-14T04:57:57.131 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.131 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:57.132 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.132 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:57.132 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.470008+0000", 2024-09-14T04:57:57.132 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.132 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.132 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.132 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:57.132 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.470008+0000", 2024-09-14T04:57:57.132 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.132 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.133 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.133 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:57.133 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.470006+0000", 2024-09-14T04:57:57.133 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T04:57:57.133 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.133 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.133 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:57.133 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.470039+0000", 2024-09-14T04:57:57.133 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.2662e-05 2024-09-14T04:57:57.133 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.133 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.133 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:57.133 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.470050+0000", 2024-09-14T04:57:57.133 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0626e-05 2024-09-14T04:57:57.134 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.134 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.134 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:57.134 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.470056+0000", 2024-09-14T04:57:57.134 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.4710000000000004e-06 2024-09-14T04:57:57.134 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.134 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.134 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:57.134 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.470095+0000", 2024-09-14T04:57:57.134 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9042000000000003e-05 2024-09-14T04:57:57.134 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.134 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.134 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:57.135 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.470183+0000", 2024-09-14T04:57:57.135 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.8117999999999999e-05 2024-09-14T04:57:57.135 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.135 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.135 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,5", 2024-09-14T04:57:57.135 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.900775+0000", 2024-09-14T04:57:57.135 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.43059172699999998 2024-09-14T04:57:57.135 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.135 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.135 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.135 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.906165+0000", 2024-09-14T04:57:57.135 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0053896339999999999 2024-09-14T04:57:57.135 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.135 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.136 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:57:57.136 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.906206+0000", 2024-09-14T04:57:57.136 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1229000000000001e-05 2024-09-14T04:57:57.136 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.136 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.136 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T04:57:57.136 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.909582+0000", 2024-09-14T04:57:57.136 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0033762029999999999 2024-09-14T04:57:57.136 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.136 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.136 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:57.136 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.909630+0000", 2024-09-14T04:57:57.136 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8182e-05 2024-09-14T04:57:57.137 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.137 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.137 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:57.137 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.909663+0000", 2024-09-14T04:57:57.137 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3120000000000001e-05 2024-09-14T04:57:57.137 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.137 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:57.137 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.137 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.137 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.137 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.36324.0:34402 57.0 e1523/1518 57:0b7b4153:::benchmark_data_smithi039_365565_object34401:head v 1523'2165, mlcod=1523'2162)", 2024-09-14T04:57:57.137 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:57:13.470802+0000", 2024-09-14T04:57:57.137 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 43.540166491000001, 2024-09-14T04:57:57.138 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.43367837199999998, 2024-09-14T04:57:57.138 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T04:57:57.138 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:57:57.138 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T04:57:57.138 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.138 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T04:57:57.138 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.470802+0000", 2024-09-14T04:57:57.138 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.138 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.138 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.138 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T04:57:57.138 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.470802+0000", 2024-09-14T04:57:57.138 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T04:57:57.138 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.139 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.139 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T04:57:57.139 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.470798+0000", 2024-09-14T04:57:57.139 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999962 2024-09-14T04:57:57.139 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.139 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.139 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T04:57:57.139 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.470852+0000", 2024-09-14T04:57:57.139 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.3199999999999999e-05 2024-09-14T04:57:57.139 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.139 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.139 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T04:57:57.139 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.900655+0000", 2024-09-14T04:57:57.140 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.429803722 2024-09-14T04:57:57.140 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.140 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.140 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T04:57:57.140 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.900672+0000", 2024-09-14T04:57:57.140 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.6489000000000001e-05 2024-09-14T04:57:57.140 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.140 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.140 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T04:57:57.140 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.900739+0000", 2024-09-14T04:57:57.140 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7281999999999997e-05 2024-09-14T04:57:57.140 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.140 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.140 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T04:57:57.141 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.900767+0000", 2024-09-14T04:57:57.141 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.8076e-05 2024-09-14T04:57:57.141 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.141 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.141 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T04:57:57.141 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.904441+0000", 2024-09-14T04:57:57.141 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.003674161 2024-09-14T04:57:57.141 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T04:57:57.141 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T04:57:57.141 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T04:57:57.141 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.904480+0000", 2024-09-14T04:57:57.141 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9020000000000002e-05 2024-09-14T04:57:57.141 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.142 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:57.142 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.142 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T04:57:57.142 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T04:57:57.142 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T04:57:57.146 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:57.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:57.193 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:57.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:57.294 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:57.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:57.335 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:57.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:57.436 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:57:57.475 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:57.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:57.476 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:57:57.577 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:57.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:57.615 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:57.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:57.704 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 58 16 92190 92174 99.308 71.875 0.0083049 0.0100648 2024-09-14T04:57:57.716 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:57.737 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:57.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:57.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:57.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:57 smithi129 ceph-mon[101316]: pgmap v2947: 99 pgs: 99 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2024-09-14T04:57:57.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:57 smithi039 ceph-mon[106470]: pgmap v2947: 99 pgs: 99 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2024-09-14T04:57:57.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:57 smithi039 ceph-mon[111235]: pgmap v2947: 99 pgs: 99 active+clean; 5.4 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 84 MiB/s wr, 1.35k op/s 2024-09-14T04:57:57.838 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:57.870 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:57.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:57.871 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:57.972 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:58.003 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:58.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:58.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:58.006 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-14T04:57:58.105 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:58.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:58.146 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:58.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:58.247 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:58.276 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:58.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:58.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:58.377 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:58.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:58.421 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:58.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:58.522 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:58.562 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:58.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:58.563 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:58.664 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:57:58.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:58.697 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:58.697 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:57:58.704 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 59 16 93176 93160 98.6692 61.625 0.0127412 0.01013 2024-09-14T04:57:58.797 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:58.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:58.839 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:58.839 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:57:58.940 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:57:58.977 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:58.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:58.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:57:59.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:57:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:57:58] "GET /metrics HTTP/1.1" 200 36730 "" "Prometheus/2.51.0" 2024-09-14T04:57:59.078 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:59.115 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:59.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:59.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:59.217 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:57:59.256 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:59.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:59.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:57:59.358 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:59.380 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:59.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:59.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:59.482 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:57:59.513 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:59.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:59.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:57:59.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:57:59 smithi039 ceph-mon[106470]: pgmap v2948: 99 pgs: 99 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-14T04:57:59.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:57:59 smithi039 ceph-mon[111235]: pgmap v2948: 99 pgs: 99 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-14T04:57:59.616 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:57:59.637 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:57:59.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:59.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:57:59.704 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:2024-09-14T04:57:59.704270+0000 min lat: 0.00123999 max lat: 0.470156 avg lat: 0.010182 2024-09-14T04:57:59.704 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:57:59.704 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 60 16 94266 94250 98.1601 68.125 0.0184635 0.010182 2024-09-14T04:57:59.739 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:57:59.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:57:59 smithi129 ceph-mon[101316]: pgmap v2948: 99 pgs: 99 active+clean; 5.4 GiB data, 22 GiB used, 694 GiB / 715 GiB avail; 72 MiB/s wr, 1.15k op/s 2024-09-14T04:57:59.776 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:59.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:59.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:57:59.878 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:57:59.923 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:57:59.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:57:59.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:00.025 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:00.067 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:00.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:00.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:00.169 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:00.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:00.203 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:00.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:00.304 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:00.345 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:00.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:00.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:00.447 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:00.492 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:00.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:00.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:00.594 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:00.634 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:00.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:00.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:00.704 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 61 16 95395 95379 97.7076 70.5625 0.0148709 0.0102301 2024-09-14T04:58:00.736 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:00.776 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:00.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:00.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:00.878 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:00.913 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:00.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:00.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:01.015 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:01.064 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:01.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:01.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:01.166 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:01.198 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:01.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:01.199 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:01.300 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:01.330 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:01.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:01.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:01.432 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:01.466 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:01.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:01.467 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:01.539 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:01 smithi039 ceph-mon[106470]: pgmap v2949: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-14T04:58:01.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:01 smithi039 ceph-mon[111235]: pgmap v2949: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-14T04:58:01.568 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:01.605 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:01.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:01.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:01.705 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 62 15 96450 96435 97.1943 66 0.0155476 0.0102833 2024-09-14T04:58:01.707 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:01.747 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:01.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:01.748 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:01 smithi129 ceph-mon[101316]: pgmap v2949: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 71 MiB/s wr, 1.14k op/s 2024-09-14T04:58:01.849 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:01.880 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:01.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:01.881 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:01.981 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:02.011 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:02.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:02.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:02.113 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:02.147 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:02.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:02.148 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:02.249 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:02.279 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:58:02.280 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:58:02.301 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:02.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:02.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:02.403 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:02.403 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is set 2024-09-14T04:58:02.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:02.465 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:02.466 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:02.566 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:02.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:02.615 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:02.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:02.706 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 63 16 97428 97412 96.6199 61.0625 0.01281 0.0103446 2024-09-14T04:58:02.715 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:02.738 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:02 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3717153743' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:58:02.739 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:02 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3717153743' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:58:02.746 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:02.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:02.747 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:02.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:02 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3717153743' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:58:02.847 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:02.887 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:02.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:02.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:02.989 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:03.028 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:03.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:03.029 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:03.130 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:03.165 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:03.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:03.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:03.267 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:03.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:03.302 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:03.302 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:03.403 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:03.413 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.2 to 1 (10000) 2024-09-14T04:58:03.430 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:03.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:03.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:03.532 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:03.567 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:03.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:03.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:03.669 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:03.703 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:03.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:03.704 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:03.706 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 64 16 98461 98445 96.1189 64.5625 0.00427181 0.0103941 2024-09-14T04:58:03.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:03 smithi129 ceph-mon[101316]: pgmap v2950: 99 pgs: 99 active+clean; 5.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 988 op/s 2024-09-14T04:58:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:03 smithi129 ceph-mon[101316]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:58:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:03 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3717153743' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:58:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:03 smithi129 ceph-mon[101316]: osdmap e1529: 8 total, 8 up, 8 in 2024-09-14T04:58:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:03 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1470559108' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:58:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:03 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:58:03.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:03 smithi039 ceph-mon[106470]: pgmap v2950: 99 pgs: 99 active+clean; 5.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 988 op/s 2024-09-14T04:58:03.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:03 smithi039 ceph-mon[106470]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:58:03.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:03 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3717153743' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:58:03.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:03 smithi039 ceph-mon[106470]: osdmap e1529: 8 total, 8 up, 8 in 2024-09-14T04:58:03.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:03 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1470559108' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:58:03.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:03 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:58:03.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:03 smithi039 ceph-mon[111235]: pgmap v2950: 99 pgs: 99 active+clean; 5.7 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 62 MiB/s wr, 988 op/s 2024-09-14T04:58:03.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:03 smithi039 ceph-mon[111235]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:58:03.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:03 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3717153743' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:58:03.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:03 smithi039 ceph-mon[111235]: osdmap e1529: 8 total, 8 up, 8 in 2024-09-14T04:58:03.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:03 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1470559108' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:58:03.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:03 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T04:58:03.804 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:03.839 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:03.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:03.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:03.944 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:03.984 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:03.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:03.985 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:04.086 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:04.120 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:04.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:04.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:04.222 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:04.260 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:04.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:04.262 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:04.362 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:04.401 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:04.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:04.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:04.503 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:04.550 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:04.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:04.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:04.652 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:04.692 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:04.693 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:04 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-14T04:58:04.693 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:04 smithi039 ceph-mon[106470]: osdmap e1530: 8 total, 8 up, 8 in 2024-09-14T04:58:04.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:04.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:04.694 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:04 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-14T04:58:04.694 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:04 smithi039 ceph-mon[111235]: osdmap e1530: 8 total, 8 up, 8 in 2024-09-14T04:58:04.706 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 65 16 99525 99509 95.6632 66.5 0.00968461 0.0104479 2024-09-14T04:58:04.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:04 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-14T04:58:04.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:04 smithi129 ceph-mon[101316]: osdmap e1530: 8 total, 8 up, 8 in 2024-09-14T04:58:04.795 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:04.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:04.829 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:04.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:04.930 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:04.961 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:04.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:04.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:05.063 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:05.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:05.103 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:05.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:05.204 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:05.243 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:05.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:05.244 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:05.344 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:05.377 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:05.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:05.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:05.479 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:05.518 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:05.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:05.519 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:05.620 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:05.654 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:05.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:05.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:05.706 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 66 16 100672 100656 95.2998 71.6875 0.0106314 0.0104878 2024-09-14T04:58:05.756 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:05.765 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:05 smithi039 ceph-mon[106470]: pgmap v2953: 99 pgs: 99 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s 2024-09-14T04:58:05.765 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:05 smithi039 ceph-mon[111235]: pgmap v2953: 99 pgs: 99 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s 2024-09-14T04:58:05.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:05 smithi129 ceph-mon[101316]: pgmap v2953: 99 pgs: 99 active+clean; 5.8 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 56 MiB/s wr, 900 op/s 2024-09-14T04:58:05.787 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:05.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:05.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:05.888 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:05.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:05.961 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:05.961 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:06.062 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:06.102 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:06.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:06.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:06.204 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:06.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:06.279 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:06.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:06.380 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:06.412 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:06.413 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:06.413 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:06.514 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:06.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:06.546 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:06.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:06.647 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:06.684 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:06.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:06.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:06.706 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 67 16 101689 101673 94.826 63.5625 0.0131458 0.0105407 2024-09-14T04:58:06.785 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:06.821 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:06.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:06.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:06.924 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:06.961 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:06.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:06.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:07.063 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:07.101 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:07.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:07.102 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:07.203 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:07.240 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:07.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:07.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:07.342 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:07.384 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:07.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:07.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:07.486 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:07.524 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:07.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:07.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:07.626 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:07.663 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:07.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:07.664 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:07.706 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 68 16 102797 102781 94.4499 69.25 0.0115263 0.0105823 2024-09-14T04:58:07.754 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:07 smithi039 ceph-mon[106470]: pgmap v2954: 99 pgs: 99 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-14T04:58:07.755 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:58:07.755 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:07 smithi039 ceph-mon[111235]: pgmap v2954: 99 pgs: 99 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-14T04:58:07.755 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:58:07.765 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:07.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:07 smithi129 ceph-mon[101316]: pgmap v2954: 99 pgs: 99 active+clean; 6.1 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s 2024-09-14T04:58:07.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:58:07.808 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:07.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:07.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:07.910 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:07.946 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:07.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:07.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:08.048 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:08.099 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:08.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:08.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:08.201 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:08.240 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:08.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:08.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:08.338 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:58:08.338 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:58:08.342 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:08.381 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:58:08.381 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:58:08.385 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:08.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:08.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:08.420 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:58:08.420 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:58:08.454 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:58:08.455 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:58:08.486 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:08.487 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:58:08.487 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:58:08.521 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:58:08.521 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:58:08.525 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:08.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:08.526 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:08.555 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:58:08.556 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:58:08.596 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:58:08.596 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:58:08.627 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:08.646 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:08 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1626890167' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:58:08.646 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:08 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1626890167' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:58:08.658 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:08.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:08.659 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:08.686 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 3 1 2024-09-14T04:58:08.706 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 69 16 103865 103849 94.0484 66.75 0.00554369 0.010628 2024-09-14T04:58:08.760 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:08.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:08 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1626890167' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:58:08.809 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:08.810 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:08.810 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:08.911 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:08.942 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:08.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:08.943 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:09.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:58:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:58:08] "GET /metrics HTTP/1.1" 200 36726 "" "Prometheus/2.51.0" 2024-09-14T04:58:09.045 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:09.080 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:09.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:09.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:09.182 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:09.214 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:09.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:09.215 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:09.316 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:09.345 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:09.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:09.346 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:09.447 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:09.493 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:09.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:09.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:09.594 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:09.632 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:09.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:09.633 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:09.707 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 70 16 104914 104898 93.6407 65.5625 0.0157549 0.0106741 2024-09-14T04:58:09.734 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:09.735 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:09 smithi039 ceph-mon[106470]: pgmap v2955: 99 pgs: 99 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 878 op/s 2024-09-14T04:58:09.735 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:09 smithi039 ceph-mon[111235]: pgmap v2955: 99 pgs: 99 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 878 op/s 2024-09-14T04:58:09.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:09 smithi129 ceph-mon[101316]: pgmap v2955: 99 pgs: 99 active+clean; 6.1 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 55 MiB/s wr, 878 op/s 2024-09-14T04:58:09.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:09.775 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:09.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:09.799 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:58:09.876 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:09.915 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:09.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:09.916 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:10.017 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:10.050 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:10.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:10.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:10.151 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:10.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:10.190 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:10.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:10.291 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:10.322 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:10.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:10.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:10.424 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:10.464 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:10.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:10.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:10.566 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:10.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:10.606 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:10.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:10.707 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:10.708 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 71 16 105960 105944 93.2419 65.375 0.0105202 0.0107194 2024-09-14T04:58:10.750 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:10.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:10.751 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:10.852 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:10.890 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:58:10.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:10.952 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:10.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:11.053 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:11.089 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:11.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:11.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:11.190 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:11.238 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:11.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:11.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:11.299 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:58:11.339 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:11.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:11.378 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:11.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:11.479 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:11.518 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:11.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:11.519 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:11.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:11 smithi129 ceph-mon[101316]: pgmap v2956: 99 pgs: 99 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-09-14T04:58:11.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:11 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:58:11.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:11 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:58:11.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:11 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:58:11.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:11 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:58:11.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:11 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:58:11.553 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:11 smithi039 ceph-mon[106470]: pgmap v2956: 99 pgs: 99 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-09-14T04:58:11.553 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:11 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:58:11.553 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:11 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:58:11.553 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:11 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:58:11.553 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:11 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:58:11.553 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:11 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:58:11.553 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:11 smithi039 ceph-mon[111235]: pgmap v2956: 99 pgs: 99 active+clean; 6.3 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-09-14T04:58:11.553 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:11 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:58:11.553 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:11 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:58:11.554 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:11 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:58:11.554 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:11 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:58:11.554 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:11 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:58:11.620 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:11.653 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:11.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:11.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:11.708 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 72 16 106926 106910 92.7854 60.375 0.00727886 0.010772 2024-09-14T04:58:11.737 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:58:11.755 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:11.785 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:11.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:11.786 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:11.887 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:11.921 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:58:11.922 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:58:11.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:11.936 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:11.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:12.037 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:12.072 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:12.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:12.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:12.174 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:12.245 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:12.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:12.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:12.347 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:12.381 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:12.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:12.383 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:12.483 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:12.525 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:12.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:12.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:12.627 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:12.661 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:12.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:12.662 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:12.708 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 73 16 107897 107881 92.3456 60.6875 0.0160669 0.0108239 2024-09-14T04:58:12.762 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:12.803 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:12.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:12.804 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:12.905 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:12.927 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:12.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:12.928 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:13.029 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:13.061 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:13.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:13.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:13.163 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:13.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:13.195 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:13.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:13.296 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:13.340 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:13.341 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:13.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:13.442 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:13.480 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:13.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:13.481 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:13.582 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:13.606 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:13.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:13.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:13.707 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:13.708 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 74 16 108854 108838 91.906 59.8125 0.0124723 0.0108756 2024-09-14T04:58:13.750 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:13.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:13.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:13.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:13 smithi129 ceph-mon[101316]: pgmap v2957: 99 pgs: 99 active+clean; 6.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-14T04:58:13.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:13 smithi039 ceph-mon[106470]: pgmap v2957: 99 pgs: 99 active+clean; 6.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-14T04:58:13.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:13 smithi039 ceph-mon[111235]: pgmap v2957: 99 pgs: 99 active+clean; 6.4 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 70 MiB/s wr, 1.12k op/s 2024-09-14T04:58:13.852 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:13.881 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:13.882 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:13.883 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:13.984 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:14.022 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:14.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:14.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:14.124 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:14.155 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:14.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:14.157 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:14.257 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:14.292 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:14.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:14.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:14.394 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:14.432 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:14.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:14.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:14.534 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:14.573 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:14.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:14.574 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:14.675 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:14.703 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:14.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:14.704 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:14.708 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 75 16 109907 109891 91.558 65.8125 0.0342891 0.0109164 2024-09-14T04:58:14.805 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:14.841 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:14.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:14.842 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:14.942 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:14.983 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:14.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:14.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:15.085 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:15.121 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:15.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:15.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:15.222 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:15.259 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:15.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:15.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:15.362 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:15.400 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:15.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:15.401 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:15.502 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:15.524 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-14T04:58:15.528 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:15.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:15.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:15.562 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:15 smithi039 ceph-mon[111235]: pgmap v2958: 99 pgs: 99 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2024-09-14T04:58:15.566 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:15 smithi039 ceph-mon[106470]: pgmap v2958: 99 pgs: 99 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2024-09-14T04:58:15.630 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:15.660 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:15.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:15.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:15.709 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 76 16 110906 110890 91.174 62.4375 0.0466341 0.0109625 2024-09-14T04:58:15.762 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:15.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:15 smithi129 ceph-mon[101316]: pgmap v2958: 99 pgs: 99 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 70 MiB/s wr, 1.13k op/s 2024-09-14T04:58:15.808 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:15.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:15.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:15.909 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:15.956 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:15.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:15.957 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:16.058 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:16.107 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:16.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:16.108 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:16.209 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:16.247 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:16.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:16.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:16.349 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:16.384 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:16.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:16.385 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:16.486 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:16.519 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:16.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:16.520 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:16.621 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:16.649 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:16.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:16.650 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:16.709 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 77 15 111839 111824 90.748 58.375 0.00578109 0.0110141 2024-09-14T04:58:16.751 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:16.790 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:16.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:16.791 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:16.891 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:16.920 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:16.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:16.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:17.022 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:17.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:17.048 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:17.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:17.149 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:17.176 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:17.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:17.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:17.270 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 04:58:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=cleanup t=2024-09-14T04:58:16.952880749Z level=info msg="Completed cleanup jobs" duration=65.861699ms 2024-09-14T04:58:17.271 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 04:58:17 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=grafana.update.checker t=2024-09-14T04:58:17.043113274Z level=info msg="Update check succeeded" duration=71.108328ms 2024-09-14T04:58:17.271 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 04:58:17 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=plugins.update.checker t=2024-09-14T04:58:17.133460324Z level=info msg="Update check succeeded" duration=64.53576ms 2024-09-14T04:58:17.278 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:17.320 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:17.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:17.321 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:17.422 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:17.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:17.453 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:17.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:17.499 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:58:17.539 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:17 smithi039 ceph-mon[106470]: pgmap v2959: 99 pgs: 99 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-14T04:58:17.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:17 smithi039 ceph-mon[111235]: pgmap v2959: 99 pgs: 99 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-14T04:58:17.555 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:17.597 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:17.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:17.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:17.699 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:17.709 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 78 16 112765 112749 90.3257 57.8125 0.0149731 0.0110657 2024-09-14T04:58:17.727 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:17.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:17.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:17.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:17 smithi129 ceph-mon[101316]: pgmap v2959: 99 pgs: 99 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-14T04:58:17.829 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:17.864 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:17.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:17.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:17.966 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:17.999 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:17.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:18.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:18.101 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:18.122 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:18.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:18.123 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:18.223 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:18.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:18.249 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:18.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:18.350 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:18.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:18.374 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:18.374 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:18.475 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:18.508 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:18.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:18.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:18.610 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:18.648 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:18.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:18.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:18.709 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 79 16 113250 113234 89.5661 30.3125 0.0134835 0.01116 2024-09-14T04:58:18.750 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:18.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:18.803 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:18.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:18.904 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:18.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:18.940 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:18.941 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:18.994 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:58:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:58:18] "GET /metrics HTTP/1.1" 200 36726 "" "Prometheus/2.51.0" 2024-09-14T04:58:19.041 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:19.087 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:19.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:19.088 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:19.189 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:19.241 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:19.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:19.242 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:19.343 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:19.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:19.387 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:19.387 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:19.488 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:19.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:19.536 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:19.536 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:19.637 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:19.668 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:19.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:19.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:19.709 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:19 smithi039 ceph-mon[106470]: pgmap v2960: 99 pgs: 99 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2024-09-14T04:58:19.709 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:19 smithi039 ceph-mon[111235]: pgmap v2960: 99 pgs: 99 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2024-09-14T04:58:19.713 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:2024-09-14T04:58:19.713468+0000 min lat: 0.00123999 max lat: 0.470156 avg lat: 0.0112094 2024-09-14T04:58:19.713 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T04:58:19.713 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 80 16 114175 114159 89.1649 57.8125 0.0105203 0.0112094 2024-09-14T04:58:19.770 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:19.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:19 smithi129 ceph-mon[101316]: pgmap v2960: 99 pgs: 99 active+clean; 6.7 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 54 MiB/s wr, 864 op/s 2024-09-14T04:58:19.813 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:19.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:19.815 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:19.916 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:19.956 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:19.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:19.957 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:20.058 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:20.096 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:20.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:20.097 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:20.159 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:58:20.198 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:20.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:20.241 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:20.241 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:20.342 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:20.354 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.3 to 1 (10000) 2024-09-14T04:58:20.383 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:20.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:20.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:20.414 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:20 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/790427034' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:58:20.414 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:20 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:58:20.414 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:20 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/790427034' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:58:20.414 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:20 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:58:20.486 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:20.515 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:20.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:20.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:20.617 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:20.662 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:20.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:20.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:20.713 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 81 16 115175 115159 88.8357 62.5 0.00826544 0.0112516 2024-09-14T04:58:20.764 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:20.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:20 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/790427034' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:58:20.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:20 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T04:58:20.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:20.800 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:20.800 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:20.901 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:20.933 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:20.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:20.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:21.037 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:21.090 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:21.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:21.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:21.192 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:21.221 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:21.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:21.222 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:21.323 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:21.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:21.357 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:21.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:21.458 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:21.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:21.493 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:21.493 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:21.594 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:21.633 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:21.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:21.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:21.713 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 82 16 116206 116190 88.5381 64.4375 0.0139771 0.0112889 2024-09-14T04:58:21.735 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:21.770 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:21.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:21 smithi129 ceph-mon[101316]: pgmap v2961: 99 pgs: 99 active+clean; 6.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-09-14T04:58:21.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:21 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-14T04:58:21.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:21 smithi129 ceph-mon[101316]: osdmap e1531: 8 total, 8 up, 8 in 2024-09-14T04:58:21.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:21.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:21.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:21 smithi039 ceph-mon[106470]: pgmap v2961: 99 pgs: 99 active+clean; 6.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-09-14T04:58:21.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:21 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-14T04:58:21.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:21 smithi039 ceph-mon[106470]: osdmap e1531: 8 total, 8 up, 8 in 2024-09-14T04:58:21.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:21 smithi039 ceph-mon[111235]: pgmap v2961: 99 pgs: 99 active+clean; 6.9 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 70 MiB/s wr, 1.11k op/s 2024-09-14T04:58:21.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:21 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-14T04:58:21.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:21 smithi039 ceph-mon[111235]: osdmap e1531: 8 total, 8 up, 8 in 2024-09-14T04:58:21.874 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:21.918 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:21.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:21.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:22.019 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:22.056 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:22.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:22.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:22.066 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 4 1 2024-09-14T04:58:22.158 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:22.185 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:22.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:22.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:22.286 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:22.334 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:22.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:22.335 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:22.436 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:22.486 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:22.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:22.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:22.588 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:22.619 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:22.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:22.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:22.676 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:58:22.677 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:58:22.713 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 83 16 117195 117179 88.2161 61.8125 0.0170998 0.0113312 2024-09-14T04:58:22.721 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:22.763 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:22.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:22.765 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:22.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:58:22.866 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:22.894 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:22.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:22.895 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:22.995 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:23.041 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:23.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:23.042 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:23.143 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:23.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:23.175 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:23.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:23.276 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:23.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:23.320 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:23.320 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:23.421 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:23.427 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:23 smithi039 ceph-mon[106470]: pgmap v2963: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 58 MiB/s wr, 935 op/s 2024-09-14T04:58:23.427 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:23 smithi039 ceph-mon[111235]: pgmap v2963: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 58 MiB/s wr, 935 op/s 2024-09-14T04:58:23.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:23.483 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:23.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:23.584 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:23.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:23.614 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:23.615 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:23.713 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 84 16 118179 118163 87.898 61.5 0.0108591 0.0113716 2024-09-14T04:58:23.715 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:23.758 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:23.759 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:23.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:23.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:23 smithi129 ceph-mon[101316]: pgmap v2963: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 58 MiB/s wr, 935 op/s 2024-09-14T04:58:23.860 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:23.901 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:23.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:23.902 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:24.003 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:24.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:24.049 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:24.050 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:24.150 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:24.179 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:24.180 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:24.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:24.281 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:24.322 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:24.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:24.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:24.380 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is unset 2024-09-14T04:58:24.424 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:24.450 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:24.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:24.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:24.483 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:24 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1175821488' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:58:24.483 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:24 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:58:24.484 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:24 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1175821488' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:58:24.484 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:24 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:58:24.552 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:24.595 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:24.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:24.596 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:24.697 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:24.716 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 85 16 119123 119107 87.5564 59 0.0165021 0.0114158 2024-09-14T04:58:24.737 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:24.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:24.738 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:24.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:24 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1175821488' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:58:24.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:24 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:58:24.839 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:24.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:24.876 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:24.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:24.977 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:25.016 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:25.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:25.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:25.118 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:25.157 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:25.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:25.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:25.259 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:25.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:25.302 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:25.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:25.403 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:25.432 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:25.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:25.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:25.516 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:58:25.517 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:58:25.533 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:25.567 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:58:25.568 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:58:25.568 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:25 smithi039 ceph-mon[106470]: pgmap v2964: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 53 MiB/s wr, 841 op/s 2024-09-14T04:58:25.568 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:25 smithi039 ceph-mon[106470]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:58:25.568 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:25 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:58:25.569 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:25 smithi039 ceph-mon[106470]: osdmap e1532: 8 total, 8 up, 8 in 2024-09-14T04:58:25.569 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:25 smithi039 ceph-mon[111235]: pgmap v2964: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 53 MiB/s wr, 841 op/s 2024-09-14T04:58:25.569 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:25 smithi039 ceph-mon[111235]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:58:25.569 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:25 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:58:25.569 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:25 smithi039 ceph-mon[111235]: osdmap e1532: 8 total, 8 up, 8 in 2024-09-14T04:58:25.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:25.575 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:25.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:25.615 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:58:25.615 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:58:25.665 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:58:25.666 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:58:25.676 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:25.706 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:58:25.706 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:58:25.709 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:25.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:25.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:25.715 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 86 16 120089 120073 87.2401 60.375 0.0289888 0.0114568 2024-09-14T04:58:25.725 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:58:25.758 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:58:25.758 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:58:25.768 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:58:25.768 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [], 2024-09-14T04:58:25.768 INFO:teuthology.orchestra.run.smithi039.stdout: "complaint_time": 30, 2024-09-14T04:58:25.768 INFO:teuthology.orchestra.run.smithi039.stdout: "num_blocked_ops": 0 2024-09-14T04:58:25.768 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:58:25.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:25 smithi129 ceph-mon[101316]: pgmap v2964: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 53 MiB/s wr, 841 op/s 2024-09-14T04:58:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:25 smithi129 ceph-mon[101316]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:58:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:25 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:58:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:25 smithi129 ceph-mon[101316]: osdmap e1532: 8 total, 8 up, 8 in 2024-09-14T04:58:25.802 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:58:25.802 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:58:25.811 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:25.839 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:58:25.840 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:58:25.866 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:25.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:25.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:25.967 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:26.006 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:26.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:26.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:26.109 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:26.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:26.149 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:26.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:26.250 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:26.286 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:26.287 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:26.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:26.388 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:26.427 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:26.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:26.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:26.529 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:26.570 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:26.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:26.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:26.613 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-14T04:58:26.672 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:26.684 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:26 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1274194410' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:58:26.684 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:26 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1274194410' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:58:26.716 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 87 16 121035 121019 86.9165 59.125 0.00977161 0.0115006 2024-09-14T04:58:26.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:26.723 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:26.723 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:26.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:26 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1274194410' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:58:26.824 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:26.869 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:26.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:26.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:26.970 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:27.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:27.007 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:27.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:27.108 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:27.140 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:27.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:27.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:27.236 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:58:27.242 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:27.278 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:27.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:27.279 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:27.380 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:27.408 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:27.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:27.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:27.510 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:27.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:27 smithi039 ceph-mon[106470]: pgmap v2966: 99 pgs: 99 active+clean; 7.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-14T04:58:27.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:27 smithi039 ceph-mon[111235]: pgmap v2966: 99 pgs: 99 active+clean; 7.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-14T04:58:27.545 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:27.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:27.547 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:27.648 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:27.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:27.669 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:27.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:27.716 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 88 16 121831 121815 86.4941 49.75 0.107915 0.0115488 2024-09-14T04:58:27.769 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:27.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:27 smithi129 ceph-mon[101316]: pgmap v2966: 99 pgs: 99 active+clean; 7.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s 2024-09-14T04:58:27.808 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:27.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:27.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:27.910 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:27.948 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:27.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:27.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:28.050 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:28.091 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:28.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:28.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:28.193 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:28.229 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:28.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:28.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:28.331 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:28.360 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:28.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:28.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:28.462 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:28.503 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:28.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:28.505 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:28.605 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:28.640 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:28.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:28.641 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:28.716 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 89 16 122816 122800 86.214 61.5625 0.0106508 0.0115939 2024-09-14T04:58:28.742 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:28.781 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:28.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:28.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:28.883 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:28.929 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:28.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:28.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:29.031 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:29.039 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:58:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:58:28] "GET /metrics HTTP/1.1" 200 36730 "" "Prometheus/2.51.0" 2024-09-14T04:58:29.067 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:29.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:29.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:29.169 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:29.198 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:29.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:29.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:29.300 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:29.349 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:29.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:29.350 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:29.451 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:29.502 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:29.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:29.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:29.603 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:29.636 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:29.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:29.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:29.716 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Total time run: 90.0079 2024-09-14T04:58:29.717 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Total writes made: 123749 2024-09-14T04:58:29.717 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Write size: 65536 2024-09-14T04:58:29.717 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Object size: 65536 2024-09-14T04:58:29.717 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Bandwidth (MB/sec): 85.9292 2024-09-14T04:58:29.717 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Stddev Bandwidth: 36.7419 2024-09-14T04:58:29.717 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Max bandwidth (MB/sec): 243.562 2024-09-14T04:58:29.717 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Min bandwidth (MB/sec): 30.3125 2024-09-14T04:58:29.717 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Average IOPS: 1374 2024-09-14T04:58:29.717 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Stddev IOPS: 587.871 2024-09-14T04:58:29.717 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Max IOPS: 3897 2024-09-14T04:58:29.718 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Min IOPS: 485 2024-09-14T04:58:29.718 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Average Latency(s): 0.0116333 2024-09-14T04:58:29.718 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Stddev Latency(s): 0.0137477 2024-09-14T04:58:29.718 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Max latency(s): 0.470156 2024-09-14T04:58:29.718 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Min latency(s): 0.00123999 2024-09-14T04:58:29.722 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Cleaning up (deleting benchmark objects) 2024-09-14T04:58:29.730 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:29 smithi039 ceph-mon[106470]: pgmap v2967: 99 pgs: 99 active+clean; 7.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 49 MiB/s wr, 782 op/s 2024-09-14T04:58:29.730 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:29 smithi039 ceph-mon[111235]: pgmap v2967: 99 pgs: 99 active+clean; 7.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 49 MiB/s wr, 782 op/s 2024-09-14T04:58:29.738 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:29.770 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:29.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:29.771 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:29.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:29 smithi129 ceph-mon[101316]: pgmap v2967: 99 pgs: 99 active+clean; 7.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 49 MiB/s wr, 782 op/s 2024-09-14T04:58:29.872 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:29.905 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:29.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:29.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:30.007 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:30.031 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:58:30.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:30.066 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:30.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:30.167 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:30.204 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:30.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:30.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:30.306 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:30.341 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:30.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:30.342 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:30.443 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:30.473 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:30.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:30.474 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:30.574 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:30.588 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:58:30.637 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:30.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:30.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:30.739 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:30.785 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:30.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:30.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:30.887 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:30.932 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:30.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:30.933 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:31.033 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:31.068 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:31.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:31.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:31.134 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:58:31.170 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:31.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:31.221 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:31.222 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:31.323 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:31.382 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:31.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:31.383 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:31.484 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:31.516 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:31.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:31.517 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:31.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:31 smithi039 ceph-mon[106470]: pgmap v2968: 99 pgs: 99 active+clean; 7.4 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s 2024-09-14T04:58:31.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:31 smithi039 ceph-mon[111235]: pgmap v2968: 99 pgs: 99 active+clean; 7.4 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s 2024-09-14T04:58:31.617 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:31.648 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:31.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:31.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:31.723 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:31 smithi129 ceph-mon[101316]: pgmap v2968: 99 pgs: 99 active+clean; 7.4 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s 2024-09-14T04:58:31.750 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:31.793 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:31.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:31.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:31.895 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:31.943 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:31.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:31.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:32.045 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:32.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:32.082 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:32.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:32.183 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:32.215 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:32.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:32.216 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:32.317 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:32.356 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:32.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:32.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:32.458 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:32.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:32.497 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:32.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:32.598 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:32.653 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:32.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:32.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:32.770 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:32.812 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:32.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:32.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:32.914 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:32.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:32.958 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:32.958 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:33.059 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:33.096 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:33.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:33.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:33.198 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:33.243 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:33.244 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:33.245 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:33.346 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:33.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:33.392 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:33.392 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:33.493 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:33.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:33.535 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:33.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:33.636 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:33.671 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:33 smithi039 ceph-mon[106470]: pgmap v2969: 99 pgs: 99 active+clean; 7.5 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 59 MiB/s wr, 959 op/s 2024-09-14T04:58:33.671 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:33 smithi039 ceph-mon[111235]: pgmap v2969: 99 pgs: 99 active+clean; 7.5 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 59 MiB/s wr, 959 op/s 2024-09-14T04:58:33.676 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:33.677 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:33.677 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:33.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:33 smithi129 ceph-mon[101316]: pgmap v2969: 99 pgs: 99 active+clean; 7.5 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 59 MiB/s wr, 959 op/s 2024-09-14T04:58:33.778 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:33.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:33.822 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:33.822 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:33.923 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:33.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:33.984 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:33.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:34.085 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:34.124 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:34.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:34.125 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:34.226 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:34.262 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:34.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:34.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:34.364 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:34.403 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:34.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:34.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:34.505 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:34.539 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:34.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:34.540 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:34.641 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:34.649 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:58:34.649 INFO:teuthology.orchestra.run.smithi039.stdout: "size": 20, 2024-09-14T04:58:34.649 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 600, 2024-09-14T04:58:34.649 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [ 2024-09-14T04:58:34.649 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.649 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45632.0:64534 55.1 55:81f19a9e:::benchmark_data_smithi039_297155_object64533:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1437)", 2024-09-14T04:58:34.649 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:48:36.046591+0000", 2024-09-14T04:58:34.650 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 597.85254721599995, 2024-09-14T04:58:34.650 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0210583499999999, 2024-09-14T04:58:34.650 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:58:34.650 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:58:34.650 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T04:58:34.650 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:58:34.650 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45632", 2024-09-14T04:58:34.650 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:58:34.650 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 64534 2024-09-14T04:58:34.650 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.651 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:58:34.651 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.651 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:58:34.651 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046591+0000", 2024-09-14T04:58:34.651 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.651 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.651 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.651 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:58:34.651 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046591+0000", 2024-09-14T04:58:34.651 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.651 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.651 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.651 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:58:34.652 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046593+0000", 2024-09-14T04:58:34.652 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.917e-06 2024-09-14T04:58:34.652 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.652 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.652 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:58:34.652 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046616+0000", 2024-09-14T04:58:34.652 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3010000000000002e-05 2024-09-14T04:58:34.652 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.652 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.652 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:58:34.652 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046638+0000", 2024-09-14T04:58:34.652 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2059e-05 2024-09-14T04:58:34.653 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.653 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.653 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:58:34.653 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046660+0000", 2024-09-14T04:58:34.653 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1642e-05 2024-09-14T04:58:34.653 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.653 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.653 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:58:34.653 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046764+0000", 2024-09-14T04:58:34.653 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000104226 2024-09-14T04:58:34.654 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.654 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.654 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op must wait for map", 2024-09-14T04:58:34.654 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046772+0000", 2024-09-14T04:58:34.654 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7030000000000006e-06 2024-09-14T04:58:34.654 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.654 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.654 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:58:34.654 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.050944+0000", 2024-09-14T04:58:34.654 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0041725319999999996 2024-09-14T04:58:34.654 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.655 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.655 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:58:34.655 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.051119+0000", 2024-09-14T04:58:34.655 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000174458 2024-09-14T04:58:34.655 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.655 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.655 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:58:34.655 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.067560+0000", 2024-09-14T04:58:34.655 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0164416080000001 2024-09-14T04:58:34.655 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.655 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.655 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:58:34.656 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.067650+0000", 2024-09-14T04:58:34.656 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.9195000000000002e-05 2024-09-14T04:58:34.656 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.656 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:58:34.656 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.656 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.656 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.656 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45632.0:64543 55.1 55:84b0ffa6:::benchmark_data_smithi039_297155_object64542:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1437)", 2024-09-14T04:58:34.656 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:48:36.046692+0000", 2024-09-14T04:58:34.656 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 597.85244649000003, 2024-09-14T04:58:34.656 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0291501249999999, 2024-09-14T04:58:34.657 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:58:34.657 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:58:34.657 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:58:34.657 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:58:34.657 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45632", 2024-09-14T04:58:34.657 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:58:34.657 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 64543 2024-09-14T04:58:34.657 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.657 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:58:34.657 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.657 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:58:34.657 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046692+0000", 2024-09-14T04:58:34.658 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.658 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.658 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.658 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:58:34.658 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046692+0000", 2024-09-14T04:58:34.658 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.659 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.659 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.659 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:58:34.659 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046693+0000", 2024-09-14T04:58:34.659 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0900000000000001e-07 2024-09-14T04:58:34.659 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.660 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.660 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:58:34.660 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046714+0000", 2024-09-14T04:58:34.660 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1467999999999998e-05 2024-09-14T04:58:34.660 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.660 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.661 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:58:34.661 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046733+0000", 2024-09-14T04:58:34.661 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9341999999999999e-05 2024-09-14T04:58:34.661 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.661 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.661 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:58:34.662 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.046743+0000", 2024-09-14T04:58:34.662 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.4080000000000004e-06 2024-09-14T04:58:34.662 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.662 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.662 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:58:34.662 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.047732+0000", 2024-09-14T04:58:34.663 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00098887600000000008 2024-09-14T04:58:34.663 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.663 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.663 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting_for_map not empty", 2024-09-14T04:58:34.663 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.047738+0000", 2024-09-14T04:58:34.663 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8079999999999999e-06 2024-09-14T04:58:34.664 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.664 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.664 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:58:34.664 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.051980+0000", 2024-09-14T04:58:34.664 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0042414929999999998 2024-09-14T04:58:34.665 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.665 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.665 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:58:34.665 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.051988+0000", 2024-09-14T04:58:34.665 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.3180000000000007e-06 2024-09-14T04:58:34.665 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.666 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.666 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:58:34.666 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.068072+0000", 2024-09-14T04:58:34.666 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0160837920000001 2024-09-14T04:58:34.666 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.666 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.667 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:58:34.667 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.068183+0000", 2024-09-14T04:58:34.667 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011097 2024-09-14T04:58:34.667 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.667 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.667 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:58:34.668 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.068290+0000", 2024-09-14T04:58:34.668 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000107227 2024-09-14T04:58:34.668 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.668 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.668 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.669 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.075604+0000", 2024-09-14T04:58:34.669 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0073141589999999998 2024-09-14T04:58:34.669 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.669 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.669 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:58:34.669 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.075707+0000", 2024-09-14T04:58:34.670 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000102948 2024-09-14T04:58:34.670 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.670 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.670 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.670 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.075759+0000", 2024-09-14T04:58:34.671 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1276999999999997e-05 2024-09-14T04:58:34.671 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.671 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.671 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:58:34.671 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.075815+0000", 2024-09-14T04:58:34.671 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5902000000000001e-05 2024-09-14T04:58:34.672 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.672 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.672 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:58:34.672 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.075842+0000", 2024-09-14T04:58:34.672 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.7528000000000001e-05 2024-09-14T04:58:34.672 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.673 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:58:34.673 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.673 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.673 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.673 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45632.0:64554 55.1 55:80cc57e1:::benchmark_data_smithi039_297155_object64553:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1437)", 2024-09-14T04:58:34.673 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:48:36.053143+0000", 2024-09-14T04:58:34.673 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 597.84599504899995, 2024-09-14T04:58:34.674 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0228815819999999, 2024-09-14T04:58:34.674 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:58:34.674 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:58:34.674 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:58:34.674 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:58:34.674 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45632", 2024-09-14T04:58:34.675 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:58:34.675 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 64554 2024-09-14T04:58:34.675 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.675 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:58:34.675 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.675 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:58:34.675 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.053143+0000", 2024-09-14T04:58:34.675 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.675 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.675 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.675 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:58:34.676 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.053143+0000", 2024-09-14T04:58:34.676 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.676 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.676 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.676 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:58:34.676 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.053145+0000", 2024-09-14T04:58:34.676 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.533e-06 2024-09-14T04:58:34.676 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.676 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.676 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:58:34.676 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.053169+0000", 2024-09-14T04:58:34.677 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4329e-05 2024-09-14T04:58:34.677 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.677 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.677 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:58:34.677 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.053180+0000", 2024-09-14T04:58:34.677 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0888e-05 2024-09-14T04:58:34.677 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.677 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.677 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:58:34.677 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.053188+0000", 2024-09-14T04:58:34.677 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6429999999999995e-06 2024-09-14T04:58:34.678 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.678 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.678 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:58:34.678 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.053797+0000", 2024-09-14T04:58:34.678 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00060973199999999998 2024-09-14T04:58:34.678 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.678 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.678 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:58:34.678 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.053833+0000", 2024-09-14T04:58:34.678 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.5256000000000003e-05 2024-09-14T04:58:34.678 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.678 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.678 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:58:34.678 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.070510+0000", 2024-09-14T04:58:34.679 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.016677018 2024-09-14T04:58:34.679 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.679 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.679 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:58:34.679 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.070645+0000", 2024-09-14T04:58:34.679 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000135358 2024-09-14T04:58:34.679 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.679 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.679 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:58:34.679 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.070801+0000", 2024-09-14T04:58:34.680 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015627499999999999 2024-09-14T04:58:34.680 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.680 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.680 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.680 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.075687+0000", 2024-09-14T04:58:34.680 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0048856430000000003 2024-09-14T04:58:34.680 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.680 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.680 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:58:34.681 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.075712+0000", 2024-09-14T04:58:34.681 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.5140999999999999e-05 2024-09-14T04:58:34.681 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.681 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.681 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.681 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.075915+0000", 2024-09-14T04:58:34.681 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000202919 2024-09-14T04:58:34.681 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.681 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.681 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:58:34.682 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.075980+0000", 2024-09-14T04:58:34.682 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.5073000000000005e-05 2024-09-14T04:58:34.682 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.682 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.682 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:58:34.682 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.076025+0000", 2024-09-14T04:58:34.682 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4774000000000002e-05 2024-09-14T04:58:34.682 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.682 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:58:34.682 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.682 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.682 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.683 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45632.0:64601 55.1 55:819f926e:::benchmark_data_smithi039_297155_object64600:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1437)", 2024-09-14T04:58:34.683 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:48:36.084827+0000", 2024-09-14T04:58:34.683 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 597.81431132199998, 2024-09-14T04:58:34.683 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.99141031400000001, 2024-09-14T04:58:34.683 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:58:34.683 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:58:34.683 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:58:34.683 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:58:34.683 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45632", 2024-09-14T04:58:34.683 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:58:34.683 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 64601 2024-09-14T04:58:34.683 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.684 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:58:34.684 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.684 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:58:34.684 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.084827+0000", 2024-09-14T04:58:34.684 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.684 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.684 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.684 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:58:34.684 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.084827+0000", 2024-09-14T04:58:34.684 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.684 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.684 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.684 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:58:34.685 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.084829+0000", 2024-09-14T04:58:34.685 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.742e-06 2024-09-14T04:58:34.685 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.685 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.685 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:58:34.685 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.084851+0000", 2024-09-14T04:58:34.685 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1676999999999999e-05 2024-09-14T04:58:34.685 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.685 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.685 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:58:34.685 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.084884+0000", 2024-09-14T04:58:34.685 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.3543000000000001e-05 2024-09-14T04:58:34.685 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.685 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.686 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:58:34.686 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.084893+0000", 2024-09-14T04:58:34.686 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.3889999999999994e-06 2024-09-14T04:58:34.686 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.686 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.686 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:58:34.686 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.084957+0000", 2024-09-14T04:58:34.686 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3336000000000005e-05 2024-09-14T04:58:34.686 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.686 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.686 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:58:34.687 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.084969+0000", 2024-09-14T04:58:34.687 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2411e-05 2024-09-14T04:58:34.687 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.687 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.687 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:58:34.687 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.073108+0000", 2024-09-14T04:58:34.687 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.98813857500000002 2024-09-14T04:58:34.687 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.687 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.687 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:58:34.687 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.073271+0000", 2024-09-14T04:58:34.687 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00016303199999999999 2024-09-14T04:58:34.687 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.687 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.688 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:58:34.688 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.073434+0000", 2024-09-14T04:58:34.688 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000162755 2024-09-14T04:58:34.688 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.688 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.688 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:58:34.688 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.075857+0000", 2024-09-14T04:58:34.688 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.002423209 2024-09-14T04:58:34.688 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.688 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.688 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.688 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.076080+0000", 2024-09-14T04:58:34.689 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00022328100000000001 2024-09-14T04:58:34.689 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.689 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.689 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.689 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.076130+0000", 2024-09-14T04:58:34.689 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9743000000000001e-05 2024-09-14T04:58:34.689 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.689 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.689 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:58:34.689 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.076192+0000", 2024-09-14T04:58:34.689 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1795e-05 2024-09-14T04:58:34.689 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.689 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.689 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:58:34.690 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.076237+0000", 2024-09-14T04:58:34.690 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5825999999999998e-05 2024-09-14T04:58:34.690 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.690 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:58:34.690 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.690 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.690 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.690 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45632.0:64662 55.1 55:80119e9b:::benchmark_data_smithi039_297155_object64661:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1437)", 2024-09-14T04:58:34.690 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:48:36.120270+0000", 2024-09-14T04:58:34.690 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 597.77886878000004, 2024-09-14T04:58:34.690 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.96407522300000004, 2024-09-14T04:58:34.690 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:58:34.690 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:58:34.691 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:58:34.691 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:58:34.691 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45632", 2024-09-14T04:58:34.691 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:58:34.691 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 64662 2024-09-14T04:58:34.691 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.691 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:58:34.691 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.691 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:58:34.691 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.120270+0000", 2024-09-14T04:58:34.691 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.691 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.691 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.691 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:58:34.691 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.120270+0000", 2024-09-14T04:58:34.692 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.692 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.692 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.692 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:58:34.692 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.120271+0000", 2024-09-14T04:58:34.692 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.688e-06 2024-09-14T04:58:34.692 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.692 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.692 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:58:34.692 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.120299+0000", 2024-09-14T04:58:34.692 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.8153e-05 2024-09-14T04:58:34.692 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.693 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.693 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:58:34.693 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.120316+0000", 2024-09-14T04:58:34.693 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6735000000000002e-05 2024-09-14T04:58:34.693 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.693 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.693 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:58:34.693 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.120324+0000", 2024-09-14T04:58:34.693 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7759999999999997e-06 2024-09-14T04:58:34.693 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.693 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.693 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:58:34.693 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.120531+0000", 2024-09-14T04:58:34.694 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000207426 2024-09-14T04:58:34.694 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.694 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.694 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:58:34.694 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.120546+0000", 2024-09-14T04:58:34.694 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4630000000000001e-05 2024-09-14T04:58:34.694 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.694 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.694 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:58:34.694 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.076283+0000", 2024-09-14T04:58:34.694 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.95573650200000004 2024-09-14T04:58:34.694 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.694 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.694 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:58:34.695 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.076402+0000", 2024-09-14T04:58:34.695 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000119165 2024-09-14T04:58:34.695 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.695 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.695 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:58:34.695 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.076557+0000", 2024-09-14T04:58:34.695 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015563099999999999 2024-09-14T04:58:34.695 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.695 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.695 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.695 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.084154+0000", 2024-09-14T04:58:34.695 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0075969749999999997 2024-09-14T04:58:34.695 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.695 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.695 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:58:34.696 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.084229+0000", 2024-09-14T04:58:34.696 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.4558e-05 2024-09-14T04:58:34.696 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.696 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.696 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.696 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.084273+0000", 2024-09-14T04:58:34.696 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3788999999999997e-05 2024-09-14T04:58:34.696 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.696 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.697 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:58:34.697 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.084314+0000", 2024-09-14T04:58:34.697 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.1477999999999998e-05 2024-09-14T04:58:34.697 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.697 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.697 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:58:34.697 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.084345+0000", 2024-09-14T04:58:34.697 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.0716999999999997e-05 2024-09-14T04:58:34.697 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.697 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:58:34.697 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.697 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.697 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.697 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45632.0:64703 55.1 55:85591b41:::benchmark_data_smithi039_297155_object64702:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1437)", 2024-09-14T04:58:34.698 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:48:36.148377+0000", 2024-09-14T04:58:34.698 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 597.75076174100002, 2024-09-14T04:58:34.698 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.96002874999999999, 2024-09-14T04:58:34.698 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:58:34.698 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:58:34.698 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:58:34.698 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:58:34.698 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45632", 2024-09-14T04:58:34.698 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:58:34.698 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 64703 2024-09-14T04:58:34.698 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.698 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:58:34.698 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.699 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:58:34.699 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.148377+0000", 2024-09-14T04:58:34.699 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.699 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.699 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.699 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:58:34.699 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.148377+0000", 2024-09-14T04:58:34.699 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.699 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.699 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.699 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:58:34.699 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.148378+0000", 2024-09-14T04:58:34.699 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7320000000000001e-06 2024-09-14T04:58:34.700 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.700 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.700 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:58:34.700 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.148403+0000", 2024-09-14T04:58:34.700 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4119000000000001e-05 2024-09-14T04:58:34.700 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.700 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.700 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:58:34.700 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.148419+0000", 2024-09-14T04:58:34.700 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6721000000000001e-05 2024-09-14T04:58:34.700 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.700 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.700 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:58:34.700 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.148427+0000", 2024-09-14T04:58:34.701 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6559999999999992e-06 2024-09-14T04:58:34.701 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.701 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.701 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:58:34.701 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.148496+0000", 2024-09-14T04:58:34.701 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9337000000000004e-05 2024-09-14T04:58:34.701 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.701 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.701 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:58:34.701 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.148508+0000", 2024-09-14T04:58:34.701 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1364e-05 2024-09-14T04:58:34.701 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.701 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.702 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:58:34.702 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.081665+0000", 2024-09-14T04:58:34.702 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.93315764199999995 2024-09-14T04:58:34.702 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.702 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.702 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:58:34.702 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.081808+0000", 2024-09-14T04:58:34.702 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014285799999999999 2024-09-14T04:58:34.702 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.702 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.702 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:58:34.702 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.081961+0000", 2024-09-14T04:58:34.702 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000153089 2024-09-14T04:58:34.702 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.703 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.703 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.703 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.089885+0000", 2024-09-14T04:58:34.703 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0079243109999999999 2024-09-14T04:58:34.703 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.703 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.703 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.703 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.090165+0000", 2024-09-14T04:58:34.703 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000279795 2024-09-14T04:58:34.703 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.703 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.703 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:58:34.703 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.108306+0000", 2024-09-14T04:58:34.704 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.018140763000000001 2024-09-14T04:58:34.704 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.704 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.704 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:58:34.704 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.108360+0000", 2024-09-14T04:58:34.704 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4262999999999999e-05 2024-09-14T04:58:34.704 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.704 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.704 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:58:34.704 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.108405+0000", 2024-09-14T04:58:34.704 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5099999999999998e-05 2024-09-14T04:58:34.705 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.705 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:58:34.705 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.705 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.705 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.705 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45632.0:64708 55.1 55:8b4837e3:::benchmark_data_smithi039_297155_object64707:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1437)", 2024-09-14T04:58:34.705 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:48:36.152492+0000", 2024-09-14T04:58:34.705 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 597.74664666800004, 2024-09-14T04:58:34.705 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.95604835200000005, 2024-09-14T04:58:34.705 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:58:34.705 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:58:34.705 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:58:34.705 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:58:34.706 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45632", 2024-09-14T04:58:34.706 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:58:34.706 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 64708 2024-09-14T04:58:34.706 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.706 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:58:34.706 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.706 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:58:34.706 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.152492+0000", 2024-09-14T04:58:34.706 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.707 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.707 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.707 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:58:34.707 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.152492+0000", 2024-09-14T04:58:34.707 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.707 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.707 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.707 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:58:34.708 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.152494+0000", 2024-09-14T04:58:34.708 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3709999999999998e-06 2024-09-14T04:58:34.708 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.708 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.708 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:58:34.708 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.152521+0000", 2024-09-14T04:58:34.708 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.6580999999999998e-05 2024-09-14T04:58:34.708 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.709 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.709 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:58:34.709 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.152539+0000", 2024-09-14T04:58:34.709 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8518000000000001e-05 2024-09-14T04:58:34.709 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.709 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.709 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:58:34.709 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.152549+0000", 2024-09-14T04:58:34.710 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.7710000000000001e-06 2024-09-14T04:58:34.710 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.710 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.710 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:58:34.710 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.152631+0000", 2024-09-14T04:58:34.710 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.2509999999999994e-05 2024-09-14T04:58:34.710 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.710 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.711 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T04:58:34.711 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:36.152643+0000", 2024-09-14T04:58:34.711 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1398e-05 2024-09-14T04:58:34.711 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.711 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.711 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:58:34.711 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.084389+0000", 2024-09-14T04:58:34.711 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.93174628999999998 2024-09-14T04:58:34.711 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.712 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.712 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:58:34.712 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.084495+0000", 2024-09-14T04:58:34.712 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00010586699999999999 2024-09-14T04:58:34.712 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.712 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.712 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:58:34.713 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.084661+0000", 2024-09-14T04:58:34.713 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000166298 2024-09-14T04:58:34.713 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.713 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.713 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.713 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.090110+0000", 2024-09-14T04:58:34.713 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0054491050000000001 2024-09-14T04:58:34.713 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.714 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.714 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.714 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.092751+0000", 2024-09-14T04:58:34.714 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0026402259999999999 2024-09-14T04:58:34.714 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.714 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.714 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:58:34.714 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.108464+0000", 2024-09-14T04:58:34.714 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.01571299 2024-09-14T04:58:34.714 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.715 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.715 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:58:34.715 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.108508+0000", 2024-09-14T04:58:34.715 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4283000000000003e-05 2024-09-14T04:58:34.715 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.715 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.715 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:58:34.715 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:48:37.108540+0000", 2024-09-14T04:58:34.716 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.2144000000000003e-05 2024-09-14T04:58:34.716 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.716 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:58:34.716 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.716 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.716 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.716 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36012.0:35353 56.8 56:17a3f911:::benchmark_data_smithi039_330847_object35352:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:58:34.716 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:52:44.524859+0000", 2024-09-14T04:58:34.717 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 349.374279409, 2024-09-14T04:58:34.717 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.32379368400000003, 2024-09-14T04:58:34.717 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:58:34.717 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:58:34.717 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:58:34.717 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:58:34.717 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36012", 2024-09-14T04:58:34.717 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:58:34.717 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 35353 2024-09-14T04:58:34.718 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.718 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:58:34.718 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.718 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:58:34.718 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.524859+0000", 2024-09-14T04:58:34.718 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.718 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.718 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.718 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:58:34.719 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.524859+0000", 2024-09-14T04:58:34.719 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.719 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.719 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.719 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:58:34.719 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.524860+0000", 2024-09-14T04:58:34.719 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.029e-06 2024-09-14T04:58:34.719 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.720 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.720 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:58:34.720 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.524886+0000", 2024-09-14T04:58:34.720 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.5562e-05 2024-09-14T04:58:34.720 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.720 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.720 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:58:34.720 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.524898+0000", 2024-09-14T04:58:34.720 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2337999999999999e-05 2024-09-14T04:58:34.720 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.720 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.720 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:58:34.720 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.524910+0000", 2024-09-14T04:58:34.721 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2148e-05 2024-09-14T04:58:34.721 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.721 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.721 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:58:34.721 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.524967+0000", 2024-09-14T04:58:34.721 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7408999999999999e-05 2024-09-14T04:58:34.722 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.722 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.722 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:58:34.722 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.525087+0000", 2024-09-14T04:58:34.722 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000119529 2024-09-14T04:58:34.722 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.722 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.722 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:58:34.722 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.525193+0000", 2024-09-14T04:58:34.722 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000106099 2024-09-14T04:58:34.723 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.723 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.723 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:58:34.723 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.527811+0000", 2024-09-14T04:58:34.723 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.002617426 2024-09-14T04:58:34.723 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.723 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.723 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.723 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.528252+0000", 2024-09-14T04:58:34.723 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000441734 2024-09-14T04:58:34.723 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.723 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.723 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.723 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.848530+0000", 2024-09-14T04:58:34.724 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.32027803599999999 2024-09-14T04:58:34.724 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.724 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.724 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:58:34.724 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.848605+0000", 2024-09-14T04:58:34.724 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.5025999999999996e-05 2024-09-14T04:58:34.724 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.724 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.724 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:58:34.724 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:44.848653+0000", 2024-09-14T04:58:34.725 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7348000000000001e-05 2024-09-14T04:58:34.725 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.725 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:58:34.725 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.725 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.725 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.725 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36012.0:37074 56.7 56:eae87ae0:::benchmark_data_smithi039_330847_object37073:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:58:34.725 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:52:45.860442+0000", 2024-09-14T04:58:34.725 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 348.038696254, 2024-09-14T04:58:34.725 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.307325289, 2024-09-14T04:58:34.725 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:58:34.725 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:58:34.725 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:58:34.726 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:58:34.726 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36012", 2024-09-14T04:58:34.726 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:58:34.726 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 37074 2024-09-14T04:58:34.726 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.726 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:58:34.726 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.726 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:58:34.726 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.860442+0000", 2024-09-14T04:58:34.726 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.726 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.726 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.726 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:58:34.726 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.860442+0000", 2024-09-14T04:58:34.727 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.727 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.727 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.727 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:58:34.727 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.860444+0000", 2024-09-14T04:58:34.727 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.778e-06 2024-09-14T04:58:34.727 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.727 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.727 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:58:34.727 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.860499+0000", 2024-09-14T04:58:34.727 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5426000000000001e-05 2024-09-14T04:58:34.727 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.727 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.727 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:58:34.728 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.860519+0000", 2024-09-14T04:58:34.728 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9241000000000001e-05 2024-09-14T04:58:34.728 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.728 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.728 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:58:34.728 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.860526+0000", 2024-09-14T04:58:34.728 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7789999999999996e-06 2024-09-14T04:58:34.728 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.728 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.728 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:58:34.728 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.860587+0000", 2024-09-14T04:58:34.728 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0393999999999998e-05 2024-09-14T04:58:34.728 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.729 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.729 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:58:34.729 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.860738+0000", 2024-09-14T04:58:34.729 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015164200000000001 2024-09-14T04:58:34.729 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.729 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.729 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:58:34.729 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.860905+0000", 2024-09-14T04:58:34.729 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00016642800000000001 2024-09-14T04:58:34.729 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.729 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.729 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:58:34.729 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.862183+0000", 2024-09-14T04:58:34.729 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0012785209999999999 2024-09-14T04:58:34.730 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.730 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.730 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.730 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.865856+0000", 2024-09-14T04:58:34.730 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0036722949999999999 2024-09-14T04:58:34.730 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.730 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.730 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.730 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.167593+0000", 2024-09-14T04:58:34.730 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.30173692600000002 2024-09-14T04:58:34.730 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.730 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.730 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:58:34.730 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.167673+0000", 2024-09-14T04:58:34.731 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0296999999999994e-05 2024-09-14T04:58:34.731 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.731 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.731 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:58:34.731 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.167767+0000", 2024-09-14T04:58:34.731 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.4561999999999993e-05 2024-09-14T04:58:34.731 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.731 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:58:34.731 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.731 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.731 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.731 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36012.0:37080 56.7 56:eb559557:::benchmark_data_smithi039_330847_object37079:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:58:34.731 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:52:45.862767+0000", 2024-09-14T04:58:34.731 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 348.03637165800001, 2024-09-14T04:58:34.732 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.30515425800000001, 2024-09-14T04:58:34.732 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:58:34.732 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:58:34.732 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:58:34.732 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:58:34.732 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36012", 2024-09-14T04:58:34.732 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:58:34.732 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 37080 2024-09-14T04:58:34.732 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.733 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:58:34.733 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.733 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:58:34.733 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.862767+0000", 2024-09-14T04:58:34.733 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.733 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.733 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.733 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:58:34.733 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.862767+0000", 2024-09-14T04:58:34.733 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.733 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.733 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.733 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:58:34.734 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.862768+0000", 2024-09-14T04:58:34.734 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.142e-06 2024-09-14T04:58:34.734 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.734 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.734 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:58:34.734 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.862791+0000", 2024-09-14T04:58:34.734 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3028999999999999e-05 2024-09-14T04:58:34.734 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.734 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.734 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:58:34.734 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.862805+0000", 2024-09-14T04:58:34.734 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3687e-05 2024-09-14T04:58:34.734 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.734 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.735 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:58:34.735 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.862812+0000", 2024-09-14T04:58:34.735 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.3100000000000003e-06 2024-09-14T04:58:34.735 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.735 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.735 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:58:34.735 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.862874+0000", 2024-09-14T04:58:34.735 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2467999999999999e-05 2024-09-14T04:58:34.735 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.735 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.735 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:58:34.735 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.863032+0000", 2024-09-14T04:58:34.735 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000157758 2024-09-14T04:58:34.736 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.736 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.736 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:58:34.736 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.863198+0000", 2024-09-14T04:58:34.736 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000165458 2024-09-14T04:58:34.736 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.736 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.736 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:58:34.736 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.865515+0000", 2024-09-14T04:58:34.736 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0023170880000000001 2024-09-14T04:58:34.736 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.736 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.736 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.736 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.865903+0000", 2024-09-14T04:58:34.737 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00038844299999999998 2024-09-14T04:58:34.737 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.737 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.737 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.737 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.167819+0000", 2024-09-14T04:58:34.737 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.30191585599999998 2024-09-14T04:58:34.737 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.737 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.737 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:58:34.737 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.167882+0000", 2024-09-14T04:58:34.737 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3025000000000001e-05 2024-09-14T04:58:34.737 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.737 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.737 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:58:34.738 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.167921+0000", 2024-09-14T04:58:34.738 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.8994e-05 2024-09-14T04:58:34.738 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.738 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:58:34.738 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.738 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.738 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.738 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36012.0:37082 56.7 56:eceabb24:::benchmark_data_smithi039_330847_object37081:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T04:58:34.738 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:52:45.863493+0000", 2024-09-14T04:58:34.738 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 348.03564521700002, 2024-09-14T04:58:34.738 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.304588896, 2024-09-14T04:58:34.739 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:58:34.739 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:58:34.739 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:58:34.739 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:58:34.739 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36012", 2024-09-14T04:58:34.739 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:58:34.739 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 37082 2024-09-14T04:58:34.739 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.739 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:58:34.739 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.739 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:58:34.739 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.863493+0000", 2024-09-14T04:58:34.739 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.739 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.740 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.740 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:58:34.740 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.863493+0000", 2024-09-14T04:58:34.740 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.740 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.740 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.740 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:58:34.740 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.863494+0000", 2024-09-14T04:58:34.740 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.6700000000000002e-07 2024-09-14T04:58:34.740 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.740 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.740 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:58:34.740 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.863521+0000", 2024-09-14T04:58:34.740 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.6621000000000001e-05 2024-09-14T04:58:34.741 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.741 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.741 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:58:34.741 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.863534+0000", 2024-09-14T04:58:34.741 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3594999999999999e-05 2024-09-14T04:58:34.741 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.741 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.741 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:58:34.741 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.863542+0000", 2024-09-14T04:58:34.741 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7020000000000001e-06 2024-09-14T04:58:34.741 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.741 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.741 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:58:34.742 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.864405+0000", 2024-09-14T04:58:34.742 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00086272499999999995 2024-09-14T04:58:34.742 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.742 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.742 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:58:34.742 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.864563+0000", 2024-09-14T04:58:34.742 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000158431 2024-09-14T04:58:34.742 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.742 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.742 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:58:34.742 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.864728+0000", 2024-09-14T04:58:34.742 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00016483700000000001 2024-09-14T04:58:34.742 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.742 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.742 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:58:34.743 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.865734+0000", 2024-09-14T04:58:34.743 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0010062910000000001 2024-09-14T04:58:34.743 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.743 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.743 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.743 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.867656+0000", 2024-09-14T04:58:34.743 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0019217139999999999 2024-09-14T04:58:34.743 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.743 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.743 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.743 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.167989+0000", 2024-09-14T04:58:34.743 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.30033338300000001 2024-09-14T04:58:34.743 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.743 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.744 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:58:34.744 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.168045+0000", 2024-09-14T04:58:34.744 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5189000000000003e-05 2024-09-14T04:58:34.744 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.744 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.744 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:58:34.744 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.168082+0000", 2024-09-14T04:58:34.744 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.7540999999999999e-05 2024-09-14T04:58:34.744 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.744 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:58:34.744 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.744 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.744 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.744 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36012.0:116301 56.7 56:e29960ea:::benchmark_data_smithi039_330847_object1777:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1486)", 2024-09-14T04:58:34.745 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:54:04.677654+0000", 2024-09-14T04:58:34.745 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 269.22148462000001, 2024-09-14T04:58:34.745 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.39274526700000001, 2024-09-14T04:58:34.745 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:58:34.745 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:58:34.745 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:58:34.745 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:58:34.745 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36012", 2024-09-14T04:58:34.745 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:58:34.745 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 116301 2024-09-14T04:58:34.745 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.745 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:58:34.745 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.745 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:58:34.746 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.677654+0000", 2024-09-14T04:58:34.746 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.746 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.746 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.746 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:58:34.746 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.677654+0000", 2024-09-14T04:58:34.746 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.746 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.746 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.746 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:58:34.746 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.677655+0000", 2024-09-14T04:58:34.746 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.61e-06 2024-09-14T04:58:34.747 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.747 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.747 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:58:34.747 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.677661+0000", 2024-09-14T04:58:34.747 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0800000000000002e-06 2024-09-14T04:58:34.747 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.747 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.747 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:58:34.747 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.677681+0000", 2024-09-14T04:58:34.747 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9633999999999999e-05 2024-09-14T04:58:34.747 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.747 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.747 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:58:34.747 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.677689+0000", 2024-09-14T04:58:34.748 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.1000000000000004e-06 2024-09-14T04:58:34.748 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.748 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.748 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:58:34.748 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.677747+0000", 2024-09-14T04:58:34.748 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.8220999999999997e-05 2024-09-14T04:58:34.748 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.748 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.748 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:58:34.749 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.677869+0000", 2024-09-14T04:58:34.749 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012144799999999999 2024-09-14T04:58:34.749 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.749 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.749 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:58:34.749 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.677974+0000", 2024-09-14T04:58:34.749 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000104996 2024-09-14T04:58:34.749 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.750 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.750 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:58:34.750 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.681749+0000", 2024-09-14T04:58:34.750 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0037746630000000002 2024-09-14T04:58:34.750 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.750 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.750 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.750 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.682012+0000", 2024-09-14T04:58:34.751 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00026392700000000001 2024-09-14T04:58:34.751 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.751 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.751 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.751 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:05.070285+0000", 2024-09-14T04:58:34.751 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.38827251200000001 2024-09-14T04:58:34.751 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.751 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.752 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:58:34.752 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:05.070346+0000", 2024-09-14T04:58:34.752 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0699000000000003e-05 2024-09-14T04:58:34.752 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.752 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.752 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:58:34.752 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:05.070399+0000", 2024-09-14T04:58:34.752 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3377000000000001e-05 2024-09-14T04:58:34.752 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.752 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:58:34.752 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.752 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.752 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.753 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36012.0:116306 56.7 56:e47ad2ec:::benchmark_data_smithi039_330847_object1782:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1486)", 2024-09-14T04:58:34.753 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:54:04.682098+0000", 2024-09-14T04:58:34.753 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 269.21704017000002, 2024-09-14T04:58:34.753 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.38844271400000002, 2024-09-14T04:58:34.753 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:58:34.753 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:58:34.753 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:58:34.753 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:58:34.753 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36012", 2024-09-14T04:58:34.753 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:58:34.753 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 116306 2024-09-14T04:58:34.753 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.753 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:58:34.754 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.754 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:58:34.754 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.682098+0000", 2024-09-14T04:58:34.754 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.754 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.754 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.754 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:58:34.754 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.682098+0000", 2024-09-14T04:58:34.755 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.755 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.756 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.756 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:58:34.756 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.682099+0000", 2024-09-14T04:58:34.756 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.26e-06 2024-09-14T04:58:34.756 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.756 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.756 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:58:34.756 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.682108+0000", 2024-09-14T04:58:34.756 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0539999999999995e-06 2024-09-14T04:58:34.756 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.756 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.756 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:58:34.756 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.682119+0000", 2024-09-14T04:58:34.757 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1185999999999999e-05 2024-09-14T04:58:34.757 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.757 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.757 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:58:34.757 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.682124+0000", 2024-09-14T04:58:34.757 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9069999999999998e-06 2024-09-14T04:58:34.757 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.757 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.757 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:58:34.757 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.682172+0000", 2024-09-14T04:58:34.757 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.8772e-05 2024-09-14T04:58:34.758 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.758 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.758 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:58:34.758 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.682300+0000", 2024-09-14T04:58:34.758 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012762400000000001 2024-09-14T04:58:34.758 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.758 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.758 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:58:34.758 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.682417+0000", 2024-09-14T04:58:34.758 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000116727 2024-09-14T04:58:34.758 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.758 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.758 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.758 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.685771+0000", 2024-09-14T04:58:34.758 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.003354147 2024-09-14T04:58:34.759 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.759 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.759 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:58:34.759 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.685893+0000", 2024-09-14T04:58:34.759 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012235999999999999 2024-09-14T04:58:34.759 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.759 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.759 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.759 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:05.070453+0000", 2024-09-14T04:58:34.759 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.38455929700000002 2024-09-14T04:58:34.759 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.759 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.759 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:58:34.760 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:05.070510+0000", 2024-09-14T04:58:34.760 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6988999999999999e-05 2024-09-14T04:58:34.760 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.760 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.760 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:58:34.760 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:05.070541+0000", 2024-09-14T04:58:34.760 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.1390999999999998e-05 2024-09-14T04:58:34.760 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.760 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:58:34.760 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.761 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.761 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.761 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36012.0:116313 56.7 56:e60392f8:::benchmark_data_smithi039_330847_object1789:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1486)", 2024-09-14T04:58:34.761 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:54:04.697875+0000", 2024-09-14T04:58:34.761 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 269.20126321999999, 2024-09-14T04:58:34.761 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.372792927, 2024-09-14T04:58:34.761 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:58:34.761 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:58:34.761 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:58:34.761 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:58:34.761 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36012", 2024-09-14T04:58:34.762 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:58:34.762 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 116313 2024-09-14T04:58:34.762 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.762 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:58:34.762 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.762 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:58:34.762 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.697875+0000", 2024-09-14T04:58:34.762 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.762 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.762 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.762 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:58:34.762 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.697875+0000", 2024-09-14T04:58:34.762 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.763 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.763 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.763 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:58:34.763 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.697877+0000", 2024-09-14T04:58:34.763 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4619999999999999e-06 2024-09-14T04:58:34.763 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.763 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.763 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:58:34.763 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.697881+0000", 2024-09-14T04:58:34.763 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.9589999999999997e-06 2024-09-14T04:58:34.763 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.763 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.763 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:58:34.763 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.697891+0000", 2024-09-14T04:58:34.764 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0159e-05 2024-09-14T04:58:34.764 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.764 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.764 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:58:34.764 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.697896+0000", 2024-09-14T04:58:34.764 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.079e-06 2024-09-14T04:58:34.764 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.764 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.764 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:58:34.764 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.697953+0000", 2024-09-14T04:58:34.764 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7413000000000001e-05 2024-09-14T04:58:34.764 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.765 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.765 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:58:34.765 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.698096+0000", 2024-09-14T04:58:34.765 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014254900000000001 2024-09-14T04:58:34.765 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.765 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.765 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,4", 2024-09-14T04:58:34.765 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.698220+0000", 2024-09-14T04:58:34.765 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012389200000000001 2024-09-14T04:58:34.765 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.765 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.765 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:58:34.765 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.701238+0000", 2024-09-14T04:58:34.765 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0030187209999999998 2024-09-14T04:58:34.766 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.766 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.766 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.766 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:04.701534+0000", 2024-09-14T04:58:34.766 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00029558399999999999 2024-09-14T04:58:34.766 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.766 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.766 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.766 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:05.070586+0000", 2024-09-14T04:58:34.766 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.36905189500000002 2024-09-14T04:58:34.766 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.766 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.766 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:58:34.766 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:05.070634+0000", 2024-09-14T04:58:34.767 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7794999999999999e-05 2024-09-14T04:58:34.767 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.767 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.767 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:58:34.767 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:05.070668+0000", 2024-09-14T04:58:34.767 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.4418999999999997e-05 2024-09-14T04:58:34.767 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.767 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:58:34.767 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.767 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.767 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.767 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36012.0:150550 56.d 56:b8b6dd53:::benchmark_data_smithi039_330847_object36026:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1502)", 2024-09-14T04:58:34.767 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:54:48.349027+0000", 2024-09-14T04:58:34.767 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 225.55011096300001, 2024-09-14T04:58:34.767 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.32527113499999999, 2024-09-14T04:58:34.768 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:58:34.768 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:58:34.768 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:58:34.768 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:58:34.768 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36012", 2024-09-14T04:58:34.768 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:58:34.768 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 150550 2024-09-14T04:58:34.768 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.768 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:58:34.768 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.768 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:58:34.768 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.349027+0000", 2024-09-14T04:58:34.768 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.768 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.769 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.769 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:58:34.769 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.349027+0000", 2024-09-14T04:58:34.769 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.769 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.769 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.769 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:58:34.769 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.349029+0000", 2024-09-14T04:58:34.769 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2049999999999999e-06 2024-09-14T04:58:34.769 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.769 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.769 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:58:34.769 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.349033+0000", 2024-09-14T04:58:34.769 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.861e-06 2024-09-14T04:58:34.770 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.770 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.770 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:58:34.770 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.349041+0000", 2024-09-14T04:58:34.770 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.7010000000000002e-06 2024-09-14T04:58:34.770 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.770 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.770 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:58:34.770 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.349046+0000", 2024-09-14T04:58:34.770 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.2730000000000002e-06 2024-09-14T04:58:34.770 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.770 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.770 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:58:34.771 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.349331+0000", 2024-09-14T04:58:34.771 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00028438999999999998 2024-09-14T04:58:34.771 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.771 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.771 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:58:34.771 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.349705+0000", 2024-09-14T04:58:34.771 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00037422399999999999 2024-09-14T04:58:34.771 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.771 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.771 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:58:34.771 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.350159+0000", 2024-09-14T04:58:34.771 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00045416199999999998 2024-09-14T04:58:34.771 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.771 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.771 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:58:34.772 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.354271+0000", 2024-09-14T04:58:34.772 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0041116099999999999 2024-09-14T04:58:34.772 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.772 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.772 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.772 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.355813+0000", 2024-09-14T04:58:34.772 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001542617 2024-09-14T04:58:34.772 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.772 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.772 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.772 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.674130+0000", 2024-09-14T04:58:34.772 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.31831626400000002 2024-09-14T04:58:34.772 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.772 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.772 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:58:34.772 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.674247+0000", 2024-09-14T04:58:34.773 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000117269 2024-09-14T04:58:34.773 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.773 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.773 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:58:34.773 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.674299+0000", 2024-09-14T04:58:34.773 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1558999999999999e-05 2024-09-14T04:58:34.773 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.773 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:58:34.773 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.773 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.773 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.773 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36012.0:150555 56.d 56:b0433500:::benchmark_data_smithi039_330847_object36031:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1502)", 2024-09-14T04:58:34.773 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:54:48.356754+0000", 2024-09-14T04:58:34.773 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 225.54238404500001, 2024-09-14T04:58:34.774 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.31776584200000002, 2024-09-14T04:58:34.774 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:58:34.774 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:58:34.774 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:58:34.774 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:58:34.774 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36012", 2024-09-14T04:58:34.774 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:58:34.774 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 150555 2024-09-14T04:58:34.774 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.774 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:58:34.774 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.774 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:58:34.774 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.356754+0000", 2024-09-14T04:58:34.775 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.775 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.775 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.775 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:58:34.775 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.356754+0000", 2024-09-14T04:58:34.775 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.775 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.775 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.775 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:58:34.775 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.356756+0000", 2024-09-14T04:58:34.775 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.389e-06 2024-09-14T04:58:34.775 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.776 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.776 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:58:34.776 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.356761+0000", 2024-09-14T04:58:34.776 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3900000000000001e-06 2024-09-14T04:58:34.776 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.776 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.776 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:58:34.776 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.356780+0000", 2024-09-14T04:58:34.776 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9211999999999998e-05 2024-09-14T04:58:34.776 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.776 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.777 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:58:34.777 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.356786+0000", 2024-09-14T04:58:34.777 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0780000000000001e-06 2024-09-14T04:58:34.777 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.777 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.777 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:58:34.777 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.356847+0000", 2024-09-14T04:58:34.777 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0679e-05 2024-09-14T04:58:34.777 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.777 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.777 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:58:34.777 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.356949+0000", 2024-09-14T04:58:34.777 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000102388 2024-09-14T04:58:34.778 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.778 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.778 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,7", 2024-09-14T04:58:34.778 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.357026+0000", 2024-09-14T04:58:34.778 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6840000000000003e-05 2024-09-14T04:58:34.778 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.778 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.778 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:58:34.778 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.360335+0000", 2024-09-14T04:58:34.778 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0033084120000000002 2024-09-14T04:58:34.778 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.778 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.778 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.778 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.361903+0000", 2024-09-14T04:58:34.779 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0015680889999999999 2024-09-14T04:58:34.779 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.779 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.779 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.779 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.674345+0000", 2024-09-14T04:58:34.779 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.312442306 2024-09-14T04:58:34.779 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.779 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.779 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:58:34.779 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.674477+0000", 2024-09-14T04:58:34.779 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013163399999999999 2024-09-14T04:58:34.779 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.779 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.780 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:58:34.780 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.674520+0000", 2024-09-14T04:58:34.780 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3424999999999998e-05 2024-09-14T04:58:34.780 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.780 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:58:34.780 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.780 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.780 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.780 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36012.0:196673 56.8 56:18f23225:::benchmark_data_smithi039_330847_object82149:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1510)", 2024-09-14T04:58:34.780 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:55:44.247653+0000", 2024-09-14T04:58:34.780 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 169.651485151, 2024-09-14T04:58:34.780 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.165928777, 2024-09-14T04:58:34.780 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:58:34.781 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:58:34.781 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:58:34.781 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:58:34.781 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36012", 2024-09-14T04:58:34.781 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T04:58:34.781 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 196673 2024-09-14T04:58:34.781 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.781 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:58:34.781 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.781 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:58:34.781 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:44.247653+0000", 2024-09-14T04:58:34.781 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.782 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.782 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.782 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:58:34.782 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:44.247653+0000", 2024-09-14T04:58:34.782 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.782 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.782 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.782 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:58:34.782 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:44.247655+0000", 2024-09-14T04:58:34.782 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8220000000000001e-06 2024-09-14T04:58:34.782 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.782 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.782 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:58:34.782 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:44.247661+0000", 2024-09-14T04:58:34.783 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.8799999999999996e-06 2024-09-14T04:58:34.783 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.783 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.783 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:58:34.783 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:44.247674+0000", 2024-09-14T04:58:34.783 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3247999999999999e-05 2024-09-14T04:58:34.783 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.783 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.783 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:58:34.783 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:44.247683+0000", 2024-09-14T04:58:34.783 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.3189999999999995e-06 2024-09-14T04:58:34.783 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.783 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.784 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:58:34.784 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:44.247745+0000", 2024-09-14T04:58:34.784 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2947999999999994e-05 2024-09-14T04:58:34.784 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.784 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.784 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:58:34.784 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:44.247871+0000", 2024-09-14T04:58:34.784 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012597800000000001 2024-09-14T04:58:34.784 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.784 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.784 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T04:58:34.784 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:44.248002+0000", 2024-09-14T04:58:34.785 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000130486 2024-09-14T04:58:34.785 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.785 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.785 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:58:34.785 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:44.251562+0000", 2024-09-14T04:58:34.785 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0035598180000000002 2024-09-14T04:58:34.785 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.785 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.785 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.785 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:44.253104+0000", 2024-09-14T04:58:34.785 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0015424690000000001 2024-09-14T04:58:34.785 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.785 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.786 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.786 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:44.413387+0000", 2024-09-14T04:58:34.786 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.16028295000000001 2024-09-14T04:58:34.786 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.786 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.786 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:58:34.786 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:44.413529+0000", 2024-09-14T04:58:34.786 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014175 2024-09-14T04:58:34.786 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.786 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.786 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:58:34.786 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:44.413582+0000", 2024-09-14T04:58:34.786 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3109000000000003e-05 2024-09-14T04:58:34.786 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.786 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:58:34.787 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.787 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.787 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.787 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:71019 57.2 57:42b66b05:::benchmark_data_smithi039_365565_object71018:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1525)", 2024-09-14T04:58:34.787 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:57:39.225580+0000", 2024-09-14T04:58:34.787 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 54.673558126000003, 2024-09-14T04:58:34.787 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.184134933, 2024-09-14T04:58:34.787 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:58:34.787 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:58:34.787 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:58:34.787 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:58:34.787 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T04:58:34.787 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T04:58:34.787 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 71019 2024-09-14T04:58:34.788 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.788 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:58:34.788 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.788 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:58:34.788 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.225580+0000", 2024-09-14T04:58:34.788 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.788 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.788 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.788 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:58:34.788 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.225580+0000", 2024-09-14T04:58:34.788 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.788 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.788 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.788 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:58:34.789 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.225582+0000", 2024-09-14T04:58:34.789 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2640000000000001e-06 2024-09-14T04:58:34.789 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.789 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.789 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:58:34.789 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.225599+0000", 2024-09-14T04:58:34.789 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7598e-05 2024-09-14T04:58:34.789 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.789 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.789 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:58:34.789 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.225611+0000", 2024-09-14T04:58:34.789 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1997999999999999e-05 2024-09-14T04:58:34.789 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.789 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.790 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:58:34.790 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.225617+0000", 2024-09-14T04:58:34.790 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5690000000000004e-06 2024-09-14T04:58:34.790 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.790 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.790 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:58:34.790 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.225675+0000", 2024-09-14T04:58:34.790 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.8156999999999999e-05 2024-09-14T04:58:34.790 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.790 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.790 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:58:34.790 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.225808+0000", 2024-09-14T04:58:34.790 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013311900000000001 2024-09-14T04:58:34.791 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.791 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.791 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:58:34.791 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.225931+0000", 2024-09-14T04:58:34.791 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000122922 2024-09-14T04:58:34.791 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.791 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.791 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:58:34.791 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.228509+0000", 2024-09-14T04:58:34.791 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0025776929999999998 2024-09-14T04:58:34.791 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.791 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.791 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.791 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.229558+0000", 2024-09-14T04:58:34.792 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0010490320000000001 2024-09-14T04:58:34.792 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.792 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.792 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.792 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.409565+0000", 2024-09-14T04:58:34.792 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.18000699100000001 2024-09-14T04:58:34.792 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.792 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.792 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:58:34.792 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.409662+0000", 2024-09-14T04:58:34.792 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.7418999999999997e-05 2024-09-14T04:58:34.792 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.792 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.792 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:58:34.793 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.409715+0000", 2024-09-14T04:58:34.793 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3171000000000003e-05 2024-09-14T04:58:34.793 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.793 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:58:34.793 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.793 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.793 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.793 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:71044 57.2 57:41583934:::benchmark_data_smithi039_365565_object71043:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1525)", 2024-09-14T04:58:34.793 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:57:39.240856+0000", 2024-09-14T04:58:34.793 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 54.658282747999998, 2024-09-14T04:58:34.793 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.169044211, 2024-09-14T04:58:34.793 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:58:34.793 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:58:34.793 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:58:34.794 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:58:34.794 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T04:58:34.794 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T04:58:34.794 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 71044 2024-09-14T04:58:34.794 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.794 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:58:34.794 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.794 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:58:34.794 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.240856+0000", 2024-09-14T04:58:34.794 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.794 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.794 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.794 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:58:34.794 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.240856+0000", 2024-09-14T04:58:34.795 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.795 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.795 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.795 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:58:34.795 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.240857+0000", 2024-09-14T04:58:34.795 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.522e-06 2024-09-14T04:58:34.795 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.795 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.795 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:58:34.795 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.240880+0000", 2024-09-14T04:58:34.795 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3272999999999999e-05 2024-09-14T04:58:34.795 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.795 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.795 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:58:34.795 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.240899+0000", 2024-09-14T04:58:34.796 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8314000000000001e-05 2024-09-14T04:58:34.796 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.796 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.796 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:58:34.796 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.240907+0000", 2024-09-14T04:58:34.796 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9880000000000004e-06 2024-09-14T04:58:34.796 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.796 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.796 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:58:34.796 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.240968+0000", 2024-09-14T04:58:34.796 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1415999999999996e-05 2024-09-14T04:58:34.796 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.796 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.796 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:58:34.796 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.241128+0000", 2024-09-14T04:58:34.797 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00016008400000000001 2024-09-14T04:58:34.797 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.797 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.797 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:58:34.797 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.241272+0000", 2024-09-14T04:58:34.797 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000143583 2024-09-14T04:58:34.797 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.797 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.797 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:58:34.797 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.244360+0000", 2024-09-14T04:58:34.797 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0030880920000000002 2024-09-14T04:58:34.797 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.797 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.797 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.797 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.245193+0000", 2024-09-14T04:58:34.798 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00083268100000000005 2024-09-14T04:58:34.798 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.798 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.798 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.798 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.409792+0000", 2024-09-14T04:58:34.798 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.16459898100000001 2024-09-14T04:58:34.798 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.798 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.798 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:58:34.798 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.409857+0000", 2024-09-14T04:58:34.798 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.5297999999999997e-05 2024-09-14T04:58:34.798 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.798 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.798 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:58:34.798 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.409900+0000", 2024-09-14T04:58:34.799 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2979000000000003e-05 2024-09-14T04:58:34.799 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.799 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:58:34.799 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.799 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.799 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.799 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:71053 57.2 57:4946396c:::benchmark_data_smithi039_365565_object71052:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1525)", 2024-09-14T04:58:34.799 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:57:39.249346+0000", 2024-09-14T04:58:34.799 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 54.649792761999997, 2024-09-14T04:58:34.799 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.16076528900000001, 2024-09-14T04:58:34.799 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:58:34.799 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:58:34.799 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:58:34.799 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:58:34.799 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T04:58:34.800 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T04:58:34.800 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 71053 2024-09-14T04:58:34.800 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.800 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:58:34.800 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.800 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:58:34.800 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.249346+0000", 2024-09-14T04:58:34.800 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.800 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.800 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.800 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:58:34.800 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.249346+0000", 2024-09-14T04:58:34.800 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:58:34.800 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.800 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.801 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:58:34.801 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.249347+0000", 2024-09-14T04:58:34.801 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4219999999999999e-06 2024-09-14T04:58:34.801 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.801 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.801 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:58:34.801 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.249366+0000", 2024-09-14T04:58:34.801 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9358999999999999e-05 2024-09-14T04:58:34.801 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.801 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.801 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:58:34.801 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.249378+0000", 2024-09-14T04:58:34.801 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1905999999999999e-05 2024-09-14T04:58:34.801 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.802 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.802 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:58:34.802 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.249384+0000", 2024-09-14T04:58:34.802 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5629999999999999e-06 2024-09-14T04:58:34.802 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.802 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.802 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:58:34.802 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.249439+0000", 2024-09-14T04:58:34.802 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4849999999999998e-05 2024-09-14T04:58:34.802 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.802 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.802 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:58:34.802 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.249614+0000", 2024-09-14T04:58:34.802 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000175415 2024-09-14T04:58:34.802 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.802 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.803 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,3", 2024-09-14T04:58:34.803 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.249776+0000", 2024-09-14T04:58:34.803 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00016147500000000001 2024-09-14T04:58:34.803 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.803 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.803 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:58:34.803 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.252170+0000", 2024-09-14T04:58:34.803 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0023942429999999999 2024-09-14T04:58:34.803 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.803 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.803 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.803 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.252287+0000", 2024-09-14T04:58:34.803 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011678700000000001 2024-09-14T04:58:34.803 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.804 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.804 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:58:34.804 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.410043+0000", 2024-09-14T04:58:34.804 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.15775594200000001 2024-09-14T04:58:34.804 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.804 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.804 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:58:34.804 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.410072+0000", 2024-09-14T04:58:34.804 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.9811999999999998e-05 2024-09-14T04:58:34.804 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:58:34.804 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:58:34.804 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:58:34.804 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.410111+0000", 2024-09-14T04:58:34.804 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.8515e-05 2024-09-14T04:58:34.805 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.805 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:58:34.805 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.805 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:58:34.805 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:58:34.805 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:58:34.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:34.806 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:34.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:34.908 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:34.939 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:34.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:34.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:35.042 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:35.081 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:35.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:35.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:35.183 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:35.218 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:35.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:35.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:35.319 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:35.353 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:35.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:35.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:35.452 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.4 to 1 (10000) 2024-09-14T04:58:35.455 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:35.502 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:35.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:35.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:35.605 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:35.637 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:35.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:35.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:35.739 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:35.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:35 smithi129 ceph-mon[101316]: pgmap v2970: 99 pgs: 99 active+clean; 7.5 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 56 MiB/s wr, 959 op/s 2024-09-14T04:58:35.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:35 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/497731877' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:58:35.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:35 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:58:35.775 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:35.776 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:35 smithi039 ceph-mon[106470]: pgmap v2970: 99 pgs: 99 active+clean; 7.5 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 56 MiB/s wr, 959 op/s 2024-09-14T04:58:35.776 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:35 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/497731877' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:58:35.776 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:35 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:58:35.776 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:35 smithi039 ceph-mon[111235]: pgmap v2970: 99 pgs: 99 active+clean; 7.5 GiB data, 28 GiB used, 687 GiB / 715 GiB avail; 56 MiB/s wr, 959 op/s 2024-09-14T04:58:35.776 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:35 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/497731877' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:58:35.776 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:35 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T04:58:35.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:35.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:35.808 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-14T04:58:35.878 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:35.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:35.931 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:35.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:36.032 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:36.055 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:58:36.111 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:36.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:36.112 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:36.213 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:36.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:36.242 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:36.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:36.343 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:36.381 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:36.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:36.382 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:36.483 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:36.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:36.513 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:36.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:36.614 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:36.650 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 5 1 2024-09-14T04:58:36.661 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:36.662 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:36.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:36.741 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:36 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:58:36.742 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:36 smithi039 ceph-mon[106470]: osdmap e1533: 8 total, 8 up, 8 in 2024-09-14T04:58:36.742 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:36 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:58:36.742 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:36 smithi039 ceph-mon[111235]: osdmap e1533: 8 total, 8 up, 8 in 2024-09-14T04:58:36.763 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:36.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:36 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-14T04:58:36.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:36 smithi129 ceph-mon[101316]: osdmap e1533: 8 total, 8 up, 8 in 2024-09-14T04:58:36.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:36.823 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:36.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:36.924 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:36.971 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:36.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:36.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:37.073 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:37.113 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:37.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:37.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:37.215 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:37.250 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:37.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:37.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:37.352 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:37.398 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:37.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:37.399 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:37.499 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:37.526 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:37.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:37.527 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:37.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:37 smithi039 ceph-mon[106470]: pgmap v2972: 99 pgs: 99 active+clean; 7.3 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 31 MiB/s wr, 940 op/s 2024-09-14T04:58:37.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:58:37.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:37 smithi039 ceph-mon[111235]: pgmap v2972: 99 pgs: 99 active+clean; 7.3 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 31 MiB/s wr, 940 op/s 2024-09-14T04:58:37.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:58:37.628 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:37.664 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:37.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:37.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:37.766 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:37.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:37 smithi129 ceph-mon[101316]: pgmap v2972: 99 pgs: 99 active+clean; 7.3 GiB data, 28 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 31 MiB/s wr, 940 op/s 2024-09-14T04:58:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:58:37.802 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:37.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:37.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:37.904 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:37.938 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:37.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:37.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:38.040 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:38.079 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:38.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:38.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:38.181 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:38.215 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:38.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:38.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:38.317 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:38.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:38.352 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:38.352 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:38.453 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:38.489 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:38.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:38.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:38.506 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is unset 2024-09-14T04:58:38.591 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:38.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:38.626 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:38.626 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:38.727 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:38.760 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:38.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:38.761 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:38.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:38 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4189374784' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:58:38.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:38 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:58:38.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:38 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4189374784' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:58:38.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:38 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:58:38.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:58:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:58:38] "GET /metrics HTTP/1.1" 200 36731 "" "Prometheus/2.51.0" 2024-09-14T04:58:38.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:38 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4189374784' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:58:38.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:38 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:58:38.862 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:38.896 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:38.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:38.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:38.998 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:39.036 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:39.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:39.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:39.138 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:39.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:39.174 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:39.174 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:39.275 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:39.312 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:39.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:39.313 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:39.414 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:39.449 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:39.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:39.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:39.551 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:39.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:39.584 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:39.584 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:39.685 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:39.725 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:39.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:39.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:39.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:39 smithi129 ceph-mon[101316]: pgmap v2973: 99 pgs: 99 active+clean; 7.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 31 MiB/s wr, 940 op/s 2024-09-14T04:58:39.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:39 smithi129 ceph-mon[101316]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:58:39.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:39 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T04:58:39.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:39 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:58:39.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:39 smithi129 ceph-mon[101316]: osdmap e1534: 8 total, 8 up, 8 in 2024-09-14T04:58:39.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:39 smithi039 ceph-mon[106470]: pgmap v2973: 99 pgs: 99 active+clean; 7.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 31 MiB/s wr, 940 op/s 2024-09-14T04:58:39.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:39 smithi039 ceph-mon[106470]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:58:39.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:39 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T04:58:39.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:39 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:58:39.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:39 smithi039 ceph-mon[106470]: osdmap e1534: 8 total, 8 up, 8 in 2024-09-14T04:58:39.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:39 smithi039 ceph-mon[111235]: pgmap v2973: 99 pgs: 99 active+clean; 7.3 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 102 B/s rd, 31 MiB/s wr, 940 op/s 2024-09-14T04:58:39.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:39 smithi039 ceph-mon[111235]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:58:39.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:39 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T04:58:39.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:39 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:58:39.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:39 smithi039 ceph-mon[111235]: osdmap e1534: 8 total, 8 up, 8 in 2024-09-14T04:58:39.827 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:39.866 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:39.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:39.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:39.968 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:40.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:40.011 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:40.011 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:40.069 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:58:40.112 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:40.157 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:58:40.166 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:40.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:40.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:40.270 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:40.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:40.305 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:40.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:40.406 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:40.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:40.448 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:40.449 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:40.549 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:40.589 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:40.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:40.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:40.691 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:40.735 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:40.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:40.737 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:40.838 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:40.871 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:40.872 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:40.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:40.973 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:41.003 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:41.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:41.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:41.105 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:41.137 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:41.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:41.138 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:41.239 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:41.279 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:41.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:41.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:41.381 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:41.413 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:41.414 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:41.414 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:41.510 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:58:41.510 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:58:41.515 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:41.545 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:58:41.545 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:58:41.549 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:41 smithi039 ceph-mon[106470]: pgmap v2975: 99 pgs: 99 active+clean; 7.1 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 127 B/s rd, 2.6 MiB/s wr, 1.00k op/s 2024-09-14T04:58:41.549 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:41 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/430183011' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:58:41.549 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:41 smithi039 ceph-mon[111235]: pgmap v2975: 99 pgs: 99 active+clean; 7.1 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 127 B/s rd, 2.6 MiB/s wr, 1.00k op/s 2024-09-14T04:58:41.549 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:41 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/430183011' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:58:41.556 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:41.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:41.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:41.590 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:58:41.590 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:58:41.633 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:58:41.633 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:58:41.653 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:58:41.658 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:41.667 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:58:41.667 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:58:41.701 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:58:41.701 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:58:41.717 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:41.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:41.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:41.742 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:58:41.742 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:58:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:41 smithi129 ceph-mon[101316]: pgmap v2975: 99 pgs: 99 active+clean; 7.1 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 127 B/s rd, 2.6 MiB/s wr, 1.00k op/s 2024-09-14T04:58:41.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:41 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/430183011' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:58:41.774 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:58:41.774 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:58:41.822 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:41.858 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:41.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:41.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:41.960 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:41.986 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:41.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:41.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:42.088 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:42.120 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:42.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:42.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:42.222 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:42.275 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:42.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:42.276 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:42.377 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:42.408 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:42.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:42.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:42.510 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:42.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:42.544 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:42.544 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:42.645 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:42.679 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:42.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:42.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:42.781 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:42.814 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:42.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:42.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:42.916 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:42.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:42.955 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:42.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:43.056 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:43.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:43.097 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:43.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:43.198 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:43.230 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:43.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:43.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:43.332 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:43.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:43.369 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:43.369 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:43.470 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:43.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:43.505 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:43.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:43.606 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:43.642 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:43.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:43.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:43.744 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:43.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:43 smithi129 ceph-mon[101316]: pgmap v2976: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 127 B/s rd, 1.6 MiB/s wr, 1.11k op/s 2024-09-14T04:58:43.772 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:43.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:43.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:43.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:43 smithi039 ceph-mon[106470]: pgmap v2976: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 127 B/s rd, 1.6 MiB/s wr, 1.11k op/s 2024-09-14T04:58:43.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:43 smithi039 ceph-mon[111235]: pgmap v2976: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 127 B/s rd, 1.6 MiB/s wr, 1.11k op/s 2024-09-14T04:58:43.875 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:43.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:43.918 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:43.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:44.019 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:44.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:44.057 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:44.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:44.157 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:44.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:44.196 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:44.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:44.297 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:44.339 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:44.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:44.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:44.440 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:44.477 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:44.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:44.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:44.579 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:44.587 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:58:44.588 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:58:44.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:44.616 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:44.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:44.717 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:44.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:44.756 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:44.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:44.858 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:44.885 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:44.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:44.886 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:44.987 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:45.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:45.026 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:45.026 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:45.127 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:45.166 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:45.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:45.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:45.268 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:45.300 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:45.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:45.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:45.402 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:45.440 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:45.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:45.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:45.541 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:45.571 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:45.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:45.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:45.673 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:45.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:45.709 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:45.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:45.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:45 smithi039 ceph-mon[106470]: pgmap v2977: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2024-09-14T04:58:45.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:45 smithi039 ceph-mon[111235]: pgmap v2977: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2024-09-14T04:58:45.810 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:45.845 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:45.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:45.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:45.947 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:45.975 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:45.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:45.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:46.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:45 smithi129 ceph-mon[101316]: pgmap v2977: 99 pgs: 99 active+clean; 7.0 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 631 op/s 2024-09-14T04:58:46.077 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:46.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:46.113 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:46.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:46.214 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:46.250 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:46.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:46.252 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:46.352 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:46.399 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:46.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:46.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:46.501 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:46.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:46.545 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:46.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:46.646 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:46.688 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:46.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:46.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:46.790 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:46.836 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:46.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:46.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:46.939 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:46.969 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:46.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:46.970 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:47.071 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:47.102 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:47.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:47.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:47.204 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:47.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:47.237 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:47.237 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:47.338 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:47.371 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:47.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:47.373 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:47.473 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:47.512 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:47.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:47.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:47.614 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:47.646 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:47.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:47.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:47.748 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:47.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:47 smithi039 ceph-mon[106470]: pgmap v2978: 99 pgs: 99 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-09-14T04:58:47.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:47 smithi039 ceph-mon[111235]: pgmap v2978: 99 pgs: 99 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-09-14T04:58:47.807 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:47.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:47.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:47.909 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:47.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:47.947 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:47.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:48.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:47 smithi129 ceph-mon[101316]: pgmap v2978: 99 pgs: 99 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-09-14T04:58:48.065 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:48.101 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:48.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:48.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:48.203 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:48.212 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-14T04:58:48.241 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:58:48.246 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:48.247 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:48.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:48.348 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:48.377 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:48.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:48.378 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:48.479 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:48.518 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:48.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:48.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:48.620 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:48.655 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:48.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:48.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:48.757 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:48.791 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:48.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:48.791 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:48.832 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:58:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:58:48] "GET /metrics HTTP/1.1" 200 36731 "" "Prometheus/2.51.0" 2024-09-14T04:58:48.893 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:48.942 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:48.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:48.943 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:49.044 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:49.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:49.087 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:49.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:49.188 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:49.221 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:49.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:49.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:49.257 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:58:49.323 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:49.359 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:49.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:49.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:49.461 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:49.484 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:49.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:49.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:49.586 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:49.607 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:49.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:49.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:49.709 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:49.733 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:49.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:49.734 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:49.834 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:49.856 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:49.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:49.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:49.958 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:49.992 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:49.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:49.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:49 smithi129 ceph-mon[101316]: pgmap v2979: 99 pgs: 99 active+clean; 6.7 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-09-14T04:58:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:49 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/36078373' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:58:50.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:49 smithi039 ceph-mon[106470]: pgmap v2979: 99 pgs: 99 active+clean; 6.7 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-09-14T04:58:50.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:49 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/36078373' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:58:50.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:49 smithi039 ceph-mon[111235]: pgmap v2979: 99 pgs: 99 active+clean; 6.7 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 940 op/s 2024-09-14T04:58:50.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:49 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/36078373' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T04:58:50.094 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:50.124 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:50.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:50.125 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:50.225 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:50.262 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:50.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:50.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:50.364 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:50.399 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:50.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:50.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:50.501 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:50.526 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:50.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:50.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:50.628 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:50.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:50.664 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:50.665 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:50.766 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:50.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:50 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/36078373' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-14T04:58:50.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:50 smithi039 ceph-mon[111235]: osdmap e1535: 8 total, 8 up, 8 in 2024-09-14T04:58:50.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:50 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/36078373' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-14T04:58:50.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:50 smithi039 ceph-mon[106470]: osdmap e1535: 8 total, 8 up, 8 in 2024-09-14T04:58:50.806 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:50.807 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:50.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:50.908 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:50.945 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:50.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:50.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:51.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:50 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/36078373' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-14T04:58:51.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:50 smithi129 ceph-mon[101316]: osdmap e1535: 8 total, 8 up, 8 in 2024-09-14T04:58:51.047 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:51.067 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.5 to 1 (10000) 2024-09-14T04:58:51.081 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:51.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:51.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:51.183 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:51.221 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:51.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:51.222 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:51.323 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:51.365 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:51.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:51.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:51.466 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:51.499 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:51.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:51.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:51.601 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:51.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:51.634 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:51.635 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:51.708 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:51 smithi039 ceph-mon[106470]: pgmap v2981: 99 pgs: 99 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 908 op/s 2024-09-14T04:58:51.709 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:51 smithi039 ceph-mon[111235]: pgmap v2981: 99 pgs: 99 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 908 op/s 2024-09-14T04:58:51.735 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:51.751 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 6 1 2024-09-14T04:58:51.802 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:51.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:51.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:51.904 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:51.934 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:51.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:51.935 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:52.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:51 smithi129 ceph-mon[101316]: pgmap v2981: 99 pgs: 99 active+clean; 6.5 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 908 op/s 2024-09-14T04:58:52.037 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:52.084 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:52.085 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:52.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:52.186 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:52.217 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:52.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:52.219 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:52.289 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:58:52.290 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:58:52.319 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:52.346 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:52.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:52.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:52.448 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:52.502 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:52.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:52.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:52.604 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:52.630 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:52.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:52.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:52.732 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:52.774 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:52.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:52.775 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:52.876 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:52.911 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:52.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:52.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:53.013 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:53.052 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:53.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:53.053 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:53.154 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:53.189 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:53.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:53.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:53.291 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:53.331 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:53.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:53.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:53.398 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:58:53.433 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:53.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:53.466 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:53.467 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:53.539 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:53 smithi039 ceph-mon[106470]: pgmap v2982: 99 pgs: 99 active+clean; 6.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2024-09-14T04:58:53.539 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:53 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:58:53.539 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:53 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:58:53.539 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:53 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/622407476' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:58:53.539 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:53 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:58:53.539 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:53 smithi039 ceph-mon[111235]: pgmap v2982: 99 pgs: 99 active+clean; 6.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2024-09-14T04:58:53.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:53 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:58:53.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:53 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:58:53.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:53 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/622407476' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:58:53.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:53 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:58:53.567 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:53.590 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is set 2024-09-14T04:58:53.598 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:53.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:53.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:53.700 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:53.745 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:53.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:53.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:53.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:53 smithi129 ceph-mon[101316]: pgmap v2982: 99 pgs: 99 active+clean; 6.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2024-09-14T04:58:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:53 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:58:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:53 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:58:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:53 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/622407476' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:58:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:53 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T04:58:53.847 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:53.886 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:53.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:53.887 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:53.988 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:54.029 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:54.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:54.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:54.131 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:54.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:54.171 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:54.172 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:54.272 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:54.302 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:54.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:54.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:54.404 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:54.443 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:54.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:54.444 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:54.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:54 smithi039 ceph-mon[106470]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:58:54.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:54 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:58:54.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:54 smithi039 ceph-mon[106470]: osdmap e1536: 8 total, 8 up, 8 in 2024-09-14T04:58:54.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:54 smithi039 ceph-mon[111235]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:58:54.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:54 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:58:54.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:54 smithi039 ceph-mon[111235]: osdmap e1536: 8 total, 8 up, 8 in 2024-09-14T04:58:54.545 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:54.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:54.582 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:54.582 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:54.683 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:54.718 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:54.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:54.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:54.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:54 smithi129 ceph-mon[101316]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:58:54.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:54 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T04:58:54.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:54 smithi129 ceph-mon[101316]: osdmap e1536: 8 total, 8 up, 8 in 2024-09-14T04:58:54.820 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:54.854 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:54.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:54.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:54.955 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:54.994 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:54.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:54.995 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:55.096 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:55.139 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:55.139 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:55.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:55.240 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:55.271 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:55.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:55.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:55.372 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:55.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:55.412 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:55.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:55.513 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:55.547 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:55.548 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:55.548 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:55.649 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:55.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:55.695 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:55.696 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:55.797 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:55.827 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:55.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:55.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:55.899 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:55 smithi039 ceph-mon[106470]: pgmap v2984: 99 pgs: 99 active+clean; 6.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 587 op/s 2024-09-14T04:58:55.900 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:55 smithi039 ceph-mon[111235]: pgmap v2984: 99 pgs: 99 active+clean; 6.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 587 op/s 2024-09-14T04:58:55.929 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:55.967 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:55.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:55.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:56.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:55 smithi129 ceph-mon[101316]: pgmap v2984: 99 pgs: 99 active+clean; 6.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 0 B/s wr, 587 op/s 2024-09-14T04:58:56.069 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:56.108 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:56.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:56.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:56.210 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:56.247 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:56.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:56.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:56.349 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:56.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:56.396 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:56.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:56.497 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:56.534 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:56.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:56.535 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:56.635 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:56.671 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:56.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:56.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:56.773 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:56.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:56.806 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:56.806 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:56.907 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:56.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:56.947 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:56.947 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:57.048 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:57.076 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:57.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:57.077 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:57.178 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:57.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:57.221 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:57.221 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:57.322 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:57.355 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:57.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:57.356 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:57.457 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:57.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:57.515 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:57.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:57.616 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:57.655 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:57.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:57.656 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:57.757 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:58:57.759 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:57 smithi039 ceph-mon[106470]: pgmap v2985: 99 pgs: 99 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-14T04:58:57.759 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:57 smithi039 ceph-mon[111235]: pgmap v2985: 99 pgs: 99 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-14T04:58:57.784 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:57.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:57.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:58:57.886 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:57.927 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:57.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:57.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:57.935 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T04:58:58.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:57 smithi129 ceph-mon[101316]: pgmap v2985: 99 pgs: 99 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 1.06k op/s 2024-09-14T04:58:58.029 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:58.082 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:58.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:58.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:58.184 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:58:58.225 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:58.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:58.226 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:58:58.328 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:58:58.354 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:58.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:58.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:58:58.456 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:58.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:58.501 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:58.502 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:58.602 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:58.632 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:58.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:58.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:58.698 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:58:58.699 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:58:58.699 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:58:58.699 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:58:58.699 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:58:58.699 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:58:58.699 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:58:58.699 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:58:58.722 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:58:58.722 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:58:58.736 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:58.755 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:58:58.756 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:58:58.764 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:58.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:58.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:58.790 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:58:58.790 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:58:58.836 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:58:58.836 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:58:58.837 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:58 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4088549819' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:58:58.837 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:58:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:58:58] "GET /metrics HTTP/1.1" 200 36734 "" "Prometheus/2.51.0" 2024-09-14T04:58:58.837 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:58 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4088549819' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:58:58.867 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:58.903 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:58.904 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:58.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:59.005 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:58:59.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:58 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4088549819' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:58:59.049 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:59.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:59.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:58:59.151 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:59.189 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:59.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:59.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:59.291 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:58:59.331 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:59.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:59.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:58:59.433 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:58:59.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:59.468 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:58:59.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:58:59.569 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:59.622 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:59.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:59.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:59.624 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.6 to 1 (10000) 2024-09-14T04:58:59.689 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:59 smithi039 ceph-mon[106470]: pgmap v2986: 99 pgs: 99 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2024-09-14T04:58:59.690 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:59 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2901503229' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:58:59.690 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:58:59 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:58:59.690 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:59 smithi039 ceph-mon[111235]: pgmap v2986: 99 pgs: 99 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2024-09-14T04:58:59.690 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:59 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2901503229' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:58:59.690 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:58:59 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:58:59.725 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:58:59.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:59.765 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:59.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:58:59.866 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:58:59.905 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:58:59.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:58:59.906 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:58:59.921 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:59 smithi129 ceph-mon[101316]: pgmap v2986: 99 pgs: 99 active+clean; 6.2 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 0 B/s wr, 745 op/s 2024-09-14T04:58:59.921 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:59 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2901503229' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:58:59.921 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:58:59 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T04:59:00.007 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:00.030 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:00.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:00.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:00.131 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:00.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:00.174 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:00.174 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:00.274 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:00.307 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:00.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:00.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:00.410 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:00.447 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:00.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:00.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:00.549 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:00.591 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:00.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:00.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:00.693 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:00.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:00.727 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:00.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:00.828 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:00.870 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:00.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:00.872 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:00.955 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:00 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-14T04:59:00.955 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:00 smithi039 ceph-mon[106470]: osdmap e1537: 8 total, 8 up, 8 in 2024-09-14T04:59:00.955 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:00 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-14T04:59:00.955 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:00 smithi039 ceph-mon[111235]: osdmap e1537: 8 total, 8 up, 8 in 2024-09-14T04:59:00.973 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:01.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:01.008 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:01.008 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:01.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:00 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-14T04:59:01.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:00 smithi129 ceph-mon[101316]: osdmap e1537: 8 total, 8 up, 8 in 2024-09-14T04:59:01.108 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:01.146 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:01.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:01.147 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:01.248 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:01.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:01.283 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:01.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:01.384 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:01.401 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 7 1 2024-09-14T04:59:01.434 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:01.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:01.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:01.536 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:01.584 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:01.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:01.585 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:01.686 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:01.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:01.720 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:01.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:01.821 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:01.853 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:01.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:01.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:01.955 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:01.994 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:01.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:01.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:02.012 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:01 smithi039 ceph-mon[106470]: pgmap v2988: 99 pgs: 99 active+clean; 6.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2024-09-14T04:59:02.012 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:01 smithi039 ceph-mon[111235]: pgmap v2988: 99 pgs: 99 active+clean; 6.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2024-09-14T04:59:02.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:01 smithi129 ceph-mon[101316]: pgmap v2988: 99 pgs: 99 active+clean; 6.0 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2024-09-14T04:59:02.096 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:02.129 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:59:02.129 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [], 2024-09-14T04:59:02.130 INFO:teuthology.orchestra.run.smithi039.stdout: "complaint_time": 30, 2024-09-14T04:59:02.130 INFO:teuthology.orchestra.run.smithi039.stdout: "num_blocked_ops": 0 2024-09-14T04:59:02.130 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:59:02.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:02.135 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:02.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:02.236 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:02.270 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:02.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:02.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:02.372 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:02.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:02.407 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:02.407 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:02.508 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:02.549 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:02.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:02.550 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:02.651 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:02.688 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:02.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:02.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:02.790 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:02.826 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:02.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:02.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:02.927 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:02.933 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:59:02.971 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-14T04:59:02.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:02.998 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:02.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:03.099 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:03.122 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:03.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:03.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:03.224 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:03.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:03.272 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:03.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:03.373 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:03.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:03.464 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:03.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:03.565 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:03.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:03.603 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:03.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:03.704 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:03.736 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:03.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:03.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:03.770 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:59:03.839 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:03.889 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:03.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:03.890 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:03.973 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:03 smithi039 ceph-mon[106470]: pgmap v2989: 99 pgs: 99 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-14T04:59:03.974 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:03 smithi039 ceph-mon[111235]: pgmap v2989: 99 pgs: 99 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-14T04:59:03.991 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:04.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:03 smithi129 ceph-mon[101316]: pgmap v2989: 99 pgs: 99 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-14T04:59:04.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:04.028 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:04.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:04.052 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:59:04.129 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:04.163 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:04.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:04.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:04.265 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:04.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:04.316 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:04.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:04.417 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:04.444 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:04.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:04.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:04.546 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:04.598 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:04.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:04.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:04.603 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:59:04.701 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:04.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:04.735 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:04.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:04.836 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:04.865 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:04.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:04.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:04.966 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:05.008 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:05.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:05.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:05.110 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:05.162 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:05.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:05.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:05.264 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:05.270 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 04:59:04 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=infra.usagestats t=2024-09-14T04:59:04.912177376Z level=info msg="Usage stats are ready to report" 2024-09-14T04:59:05.300 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:05.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:05.301 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:05.402 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:05.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:05.438 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:05.438 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:05.539 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:05.579 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:05.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:05.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:05.680 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:05.713 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:05.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:05.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:05.737 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:59:05.816 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:05.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:05.846 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:05.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:05.947 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:05.969 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:05 smithi039 ceph-mon[106470]: pgmap v2990: 99 pgs: 99 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 875 op/s 2024-09-14T04:59:05.970 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:05 smithi039 ceph-mon[111235]: pgmap v2990: 99 pgs: 99 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 875 op/s 2024-09-14T04:59:05.973 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:05.974 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:05.974 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:06.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:05 smithi129 ceph-mon[101316]: pgmap v2990: 99 pgs: 99 active+clean; 5.9 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 875 op/s 2024-09-14T04:59:06.075 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:06.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:06.134 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:06.134 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:06.235 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:06.276 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:06.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:06.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:06.378 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:06.407 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:06.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:06.409 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:06.510 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:06.541 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:06.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:06.542 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:06.643 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:06.676 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is set 2024-09-14T04:59:06.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:06.698 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:06.698 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:06.799 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:06.834 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:06.835 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:06.835 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:06.936 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:06.970 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:06.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:06.971 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:07.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:06 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1755440685' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:59:07.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:06 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1755440685' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:59:07.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:06 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1755440685' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:59:07.072 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:07.109 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:07.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:07.110 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:07.211 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:07.275 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:07.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:07.276 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:07.377 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:07.404 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:07.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:07.405 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:07.506 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:07.548 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:07.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:07.551 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:07.652 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:07.684 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:07.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:07.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:07.787 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:07.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:07.828 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:07.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:07.929 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:07.931 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:07 smithi039 ceph-mon[106470]: pgmap v2991: 99 pgs: 99 active+clean; 5.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2024-09-14T04:59:07.932 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:07 smithi039 ceph-mon[106470]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:59:07.932 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:07 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1755440685' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:59:07.932 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:07 smithi039 ceph-mon[106470]: osdmap e1538: 8 total, 8 up, 8 in 2024-09-14T04:59:07.932 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:59:07.932 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:59:07.932 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:07 smithi039 ceph-mon[111235]: pgmap v2991: 99 pgs: 99 active+clean; 5.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2024-09-14T04:59:07.932 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:07 smithi039 ceph-mon[111235]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:59:07.932 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:07 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1755440685' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:59:07.932 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:07 smithi039 ceph-mon[111235]: osdmap e1538: 8 total, 8 up, 8 in 2024-09-14T04:59:07.932 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:59:07.933 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:59:07.970 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:07.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:07.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:07 smithi129 ceph-mon[101316]: pgmap v2991: 99 pgs: 99 active+clean; 5.6 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 890 op/s 2024-09-14T04:59:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:07 smithi129 ceph-mon[101316]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:59:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:07 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1755440685' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T04:59:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:07 smithi129 ceph-mon[101316]: osdmap e1538: 8 total, 8 up, 8 in 2024-09-14T04:59:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:59:08.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:59:08.072 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:08.104 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:08.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:08.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:08.206 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:08.242 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:08.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:08.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:08.344 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:08.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:08.390 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:08.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:08.492 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:08.526 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:08.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:08.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:08.628 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:08.667 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:08.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:08.668 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:08.769 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:08.811 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:08.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:08.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:08.913 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:08.947 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:59:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:59:08] "GET /metrics HTTP/1.1" 200 36731 "" "Prometheus/2.51.0" 2024-09-14T04:59:08.950 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:08.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:08.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:09.052 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:09.089 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:09.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:09.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:09.193 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:09.238 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:09.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:09.239 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:09.340 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:09.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:09.384 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:09.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:09.484 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:09.522 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:09.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:09.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:09.624 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:09.679 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:09.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:09.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:09.697 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.7 to 1 (10000) 2024-09-14T04:59:09.781 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:09.813 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:09.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:09.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:09.915 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:09.948 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:09 smithi039 ceph-mon[106470]: pgmap v2993: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 936 op/s 2024-09-14T04:59:09.948 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:09 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/408696366' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:59:09.948 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:09 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:59:09.948 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:09 smithi039 ceph-mon[111235]: pgmap v2993: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 936 op/s 2024-09-14T04:59:09.948 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:09 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/408696366' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:59:09.949 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:09 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:59:09.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:09.949 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:09.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:10.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:09 smithi129 ceph-mon[101316]: pgmap v2993: 99 pgs: 99 active+clean; 5.6 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 936 op/s 2024-09-14T04:59:10.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:09 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/408696366' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:59:10.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:09 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T04:59:10.050 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:10.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:10.090 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:10.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:10.191 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:10.232 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:10.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:10.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:10.333 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:10.369 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:10.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:10.370 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:10.471 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:10.522 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:10.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:10.523 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:10.624 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:10.665 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:10.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:10.666 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:10.735 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T04:59:10.766 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:10.821 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:10.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:10.823 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:10.923 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:10.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:10.966 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:10.966 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:11.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:10 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-14T04:59:11.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:10 smithi129 ceph-mon[101316]: osdmap e1539: 8 total, 8 up, 8 in 2024-09-14T04:59:11.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:10 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-14T04:59:11.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:10 smithi039 ceph-mon[106470]: osdmap e1539: 8 total, 8 up, 8 in 2024-09-14T04:59:11.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:10 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-14T04:59:11.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:10 smithi039 ceph-mon[111235]: osdmap e1539: 8 total, 8 up, 8 in 2024-09-14T04:59:11.067 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:11.105 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:11.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:11.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:11.207 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:11.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:11.242 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:11.242 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:11.343 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:11.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:11.376 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:11.376 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:11.477 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:11.512 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:11.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:11.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:11.580 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:59:11.580 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:59:11.614 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:11.650 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:11.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:11.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:11.752 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:11.798 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:11.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:11.799 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:11.900 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:11.942 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:11.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:11.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:12.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:11 smithi129 ceph-mon[101316]: pgmap v2995: 99 pgs: 99 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 867 op/s 2024-09-14T04:59:12.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:11 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:59:12.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:11 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/193061898' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:59:12.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:11 smithi039 ceph-mon[106470]: pgmap v2995: 99 pgs: 99 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 867 op/s 2024-09-14T04:59:12.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:11 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:59:12.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:11 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/193061898' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:59:12.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:11 smithi039 ceph-mon[111235]: pgmap v2995: 99 pgs: 99 active+clean; 5.5 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 867 op/s 2024-09-14T04:59:12.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:11 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T04:59:12.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:11 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/193061898' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:59:12.045 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:12.089 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:12.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:12.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:12.191 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:12.235 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:12.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:12.236 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:12.337 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:12.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:12.369 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:12.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:12.470 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:12.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:12.510 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:12.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:12.532 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-14T04:59:12.532 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:59:12.610 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:12.660 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:12.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:12.661 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:12.762 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:12.812 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:12.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:12.813 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:12.914 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:12.949 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:12.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:12.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:13.051 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:13.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:13.097 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:13.097 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:13.191 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-14T04:59:13.198 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:13.249 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:13.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:13.251 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:13.281 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:59:13.281 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:59:13.281 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:59:13.281 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:59:13.281 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:59:13.281 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:59:13.281 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:59:13.281 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:59:13.281 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:59:13.282 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:59:13.282 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:59:13.282 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:59:13.282 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:59:13.282 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:59:13.282 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:59:13.282 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:59:13.351 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:13.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:13.411 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:13.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:13.512 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:13.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:13.558 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:13.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:13.659 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:13.692 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:13.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:13.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:13.794 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:13.828 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:13.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:13.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:13.920 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:13 smithi039 ceph-mon[106470]: pgmap v2996: 99 pgs: 99 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 972 op/s 2024-09-14T04:59:13.921 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:13 smithi039 ceph-mon[111235]: pgmap v2996: 99 pgs: 99 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 972 op/s 2024-09-14T04:59:13.930 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:13.984 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:13.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:13.985 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:14.024 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:13 smithi129 ceph-mon[101316]: pgmap v2996: 99 pgs: 99 active+clean; 5.4 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 972 op/s 2024-09-14T04:59:14.086 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:14.141 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:14.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:14.142 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:14.209 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:59:14.244 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:14.247 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:59:14.251 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:59:14.317 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:14.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:14.318 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:14.419 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:14.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:14.454 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:14.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:14.555 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:14.614 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:14.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:14.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:14.715 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:14.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:14.751 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:14.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:14.851 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:14.892 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:14.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:14.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:14.994 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:15.030 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:15.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:15.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:15.132 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:15.175 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:15.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:15.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:15.280 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:15.308 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:15.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:15.309 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:15.410 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:15.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:15.447 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:15.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:15.548 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:15.588 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:15.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:15.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:15.690 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:15.725 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:15.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:15.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:15.827 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:15.864 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:15.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:15.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:15.898 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:15 smithi129 ceph-mon[101316]: pgmap v2997: 99 pgs: 99 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2024-09-14T04:59:15.962 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:59:15.966 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:15.997 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:15.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:15.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:16.002 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:15 smithi039 ceph-mon[106470]: pgmap v2997: 99 pgs: 99 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2024-09-14T04:59:16.003 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:15 smithi039 ceph-mon[111235]: pgmap v2997: 99 pgs: 99 active+clean; 5.3 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 723 op/s 2024-09-14T04:59:16.099 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:16.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:16.138 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:16.138 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:16.239 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:16.295 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:16.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:16.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:16.397 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:16.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:16.447 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:16.447 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:16.548 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:16.581 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:16.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:16.582 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:16.683 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:16.710 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:16.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:16.711 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:16.811 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:16.849 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:16.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:16.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:16.951 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:16.988 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:16.988 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:16.989 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:17.090 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:17.126 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:17.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:17.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:17.228 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:17.263 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:17.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:17.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:17.365 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:17.409 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:17.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:17.410 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:17.511 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:17.550 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:17.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:17.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:17.652 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:17.694 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:17.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:17.696 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:17.796 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:17.835 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:17.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:17.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:17.937 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:17.965 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:17.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:17.966 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:17.994 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:17 smithi039 ceph-mon[106470]: pgmap v2998: 99 pgs: 99 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 877 op/s 2024-09-14T04:59:17.994 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:17 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:59:17.994 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:17 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:59:17.994 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:17 smithi039 ceph-mon[111235]: pgmap v2998: 99 pgs: 99 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 877 op/s 2024-09-14T04:59:17.994 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:17 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:59:17.995 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:17 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:59:18.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:17 smithi129 ceph-mon[101316]: pgmap v2998: 99 pgs: 99 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 0 B/s wr, 877 op/s 2024-09-14T04:59:18.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:17 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:59:18.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:17 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:59:18.067 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:18.104 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:18.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:18.105 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:18.206 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:18.247 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:18.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:18.248 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:18.349 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:18.376 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:18.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:18.377 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:18.478 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:18.511 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:18.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:18.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:18.612 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:18.649 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:18.650 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:18.650 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:18.751 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:18.782 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:18.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:18.783 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:18.884 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:18.923 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:18.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:18.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:19.004 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:59:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:59:18] "GET /metrics HTTP/1.1" 200 36731 "" "Prometheus/2.51.0" 2024-09-14T04:59:19.025 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:19.065 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:19.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:19.066 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:19.167 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:19.201 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:19.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:19.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:19.303 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:19.345 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:19.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:19.347 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:19.447 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:19.484 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:19.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:19.485 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:19.586 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:19.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:19.611 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:19.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:19.712 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:19.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:19.740 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:19.741 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:19.841 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:19.876 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:19.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:19.877 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:19.978 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:20.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:20.013 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:20.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:20.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:19 smithi129 ceph-mon[101316]: pgmap v2999: 99 pgs: 99 active+clean; 5.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 865 op/s 2024-09-14T04:59:20.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:19 smithi039 ceph-mon[106470]: pgmap v2999: 99 pgs: 99 active+clean; 5.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 865 op/s 2024-09-14T04:59:20.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:19 smithi039 ceph-mon[111235]: pgmap v2999: 99 pgs: 99 active+clean; 5.0 GiB data, 21 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 865 op/s 2024-09-14T04:59:20.115 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:20.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:20.163 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:20.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:20.264 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:20.298 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:20.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:20.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:20.400 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:20.437 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:20.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:20.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:20.539 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:20.575 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:20.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:20.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:20.677 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:20.708 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:20.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:20.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:20.810 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:20.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:20.846 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:20.846 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:20.947 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:20.978 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:20.978 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:20.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:21.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:20 smithi129 ceph-mon[101316]: pgmap v3000: 99 pgs: 99 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2024-09-14T04:59:21.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:20 smithi039 ceph-mon[106470]: pgmap v3000: 99 pgs: 99 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2024-09-14T04:59:21.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:20 smithi039 ceph-mon[111235]: pgmap v3000: 99 pgs: 99 active+clean; 5.0 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 839 op/s 2024-09-14T04:59:21.079 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:21.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:21.124 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:21.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:21.226 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:21.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:21.269 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:21.269 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:21.370 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:21.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:21.405 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:21.405 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:21.506 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:21.543 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:21.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:21.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:21.645 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:21.684 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:21.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:21.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:21.786 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:21.816 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:21.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:21.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:21.919 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:21.968 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:21.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:21.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:22.070 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:22.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:22.104 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:22.105 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:22.205 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:22.236 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:22.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:22.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:22.338 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:22.376 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:22.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:22.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:22.478 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:22.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:22.517 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:22.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:22.619 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:22.669 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:22.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:22.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:22.771 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:22.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:22.803 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:22.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:22.905 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:22.935 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:22.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:22.936 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:23.037 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:23.072 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:23.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:23.073 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:23.174 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:23.213 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:23.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:23.214 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:23 smithi039 ceph-mon[106470]: pgmap v3001: 99 pgs: 99 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 829 op/s 2024-09-14T04:59:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:59:23.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:23 smithi039 ceph-mon[111235]: pgmap v3001: 99 pgs: 99 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 829 op/s 2024-09-14T04:59:23.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:59:23.316 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:23.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:23.348 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:23.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:23.449 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:23.500 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:23.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:23.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:23.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:23 smithi129 ceph-mon[101316]: pgmap v3001: 99 pgs: 99 active+clean; 4.9 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 829 op/s 2024-09-14T04:59:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:59:23.601 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:23.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:23.657 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:23.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:23.758 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:23.783 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:23.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:23.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:23.872 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:59:23.874 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T04:59:23.885 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:23.911 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:23.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:23.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:24.013 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:24.049 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:24.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:24.050 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:24.151 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:24.197 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:24.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:24.198 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:24.299 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:24.334 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:24.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:24.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:24.435 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:24.472 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:24.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:24.473 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:24.574 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:24.605 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:24.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:24.607 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:24.707 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:24.750 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:24.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:24.753 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:24.853 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:24.884 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:24.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:24.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:24.987 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:25.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:25.024 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:25.024 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:25.125 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:25.147 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:25.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:25.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:25.248 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:25.283 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:25.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:25.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:25.289 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:25 smithi039 ceph-mon[106470]: from='client.36462 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:59:25.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:25 smithi039 ceph-mon[106470]: pgmap v3002: 99 pgs: 99 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2024-09-14T04:59:25.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:25 smithi039 ceph-mon[111235]: from='client.36462 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:59:25.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:25 smithi039 ceph-mon[111235]: pgmap v3002: 99 pgs: 99 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2024-09-14T04:59:25.385 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:25.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:25.420 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:25.420 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:25.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:25 smithi129 ceph-mon[101316]: from='client.36462 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:59:25.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:25 smithi129 ceph-mon[101316]: pgmap v3002: 99 pgs: 99 active+clean; 4.8 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 0 B/s wr, 876 op/s 2024-09-14T04:59:25.521 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:25.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:25.561 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:25.561 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:25.662 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:25.688 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:25.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:25.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:25.691 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T04:59:25.692 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T04:59:25.791 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:25.832 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:25.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:25.833 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:25.934 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:25.972 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:25.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:25.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:26.074 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:26.101 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:26.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:26.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:26.203 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:26.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:26.252 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:26.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:26.353 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:26.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:26.395 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:26.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:26.496 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:26.527 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:26.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:26.528 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:26.579 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":3001,"stamp":"2024-09-14T04:59:21.960549+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5246556157,"num_objects":80074,"num_object_clones":0,"num_object_copies":240222,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":80074,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":167516,"num_write_kb":7919970,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":53,"num_bytes_recovered":1318510,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139850,"ondisk_log_size":139850,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":298,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":21142860,"kb_used_data":15532364,"kb_used_omap":404,"kb_used_meta":5609515,"kb_avail":728818356,"statfs":{"total":767960285184,"available":746309996544,"internally_reserved":0,"allocated":15905140736,"data_stored":15861781883,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":414022,"internal_metadata":5744144058},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-652673024,"num_objects":-9959,"num_object_clones":0,"num_object_copies":-29877,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-9959,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9959,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.008368"},"pg_stats":[{"pgid":"57.f","version":"1539'10510","reported_seq":10564,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:20.262636+0000","last_change":"2024-09-14T04:56:49.435182+0000","last_active":"2024-09-14T04:59:20.262636+0000","last_peered":"2024-09-14T04:59:20.262636+0000","last_clean":"2024-09-14T04:59:20.262636+0000","last_became_active":"2024-09-14T04:56:49.434949+0000","last_became_peered":"2024-09-14T04:56:49.434949+0000","last_unstale":"2024-09-14T04:59:20.262636+0000","last_undegraded":"2024-09-14T04:59:20.262636+0000","last_fullsized":"2024-09-14T04:59:20.262636+0000","mapping_epoch":1518,"log_start":"1523'1100","ondisk_log_start":"1523'1100","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":9410,"log_dups_size":0,"ondisk_log_size":9410,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:32:29.856162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324665344,"num_objects":4954,"num_object_clones":0,"num_object_copies":14862,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4954,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10510,"num_write_kb":494848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"57.e","version":"1539'10267","reported_seq":10321,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:19.482100+0000","last_change":"2024-09-14T04:56:49.431930+0000","last_active":"2024-09-14T04:59:19.482100+0000","last_peered":"2024-09-14T04:59:19.482100+0000","last_clean":"2024-09-14T04:59:19.482100+0000","last_became_active":"2024-09-14T04:56:49.431642+0000","last_became_peered":"2024-09-14T04:56:49.431642+0000","last_unstale":"2024-09-14T04:59:19.482100+0000","last_undegraded":"2024-09-14T04:59:19.482100+0000","last_fullsized":"2024-09-14T04:59:19.482100+0000","mapping_epoch":1518,"log_start":"1523'1600","ondisk_log_start":"1523'1600","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":8667,"log_dups_size":0,"ondisk_log_size":8667,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:30:17.677827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324337664,"num_objects":4949,"num_object_clones":0,"num_object_copies":14847,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4949,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10267,"num_write_kb":486912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"57.d","version":"1539'10438","reported_seq":10466,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:17.800879+0000","last_change":"2024-09-14T04:56:49.821152+0000","last_active":"2024-09-14T04:59:17.800879+0000","last_peered":"2024-09-14T04:59:17.800879+0000","last_clean":"2024-09-14T04:59:17.800879+0000","last_became_active":"2024-09-14T04:56:49.820853+0000","last_became_peered":"2024-09-14T04:56:49.820853+0000","last_unstale":"2024-09-14T04:59:17.800879+0000","last_undegraded":"2024-09-14T04:59:17.800879+0000","last_fullsized":"2024-09-14T04:59:17.800879+0000","mapping_epoch":1518,"log_start":"1523'3100","ondisk_log_start":"1523'3100","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":7338,"log_dups_size":0,"ondisk_log_size":7338,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:41.229640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334299160,"num_objects":5102,"num_object_clones":0,"num_object_copies":15306,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5102,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10438,"num_write_kb":497217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"57.c","version":"1539'10459","reported_seq":10490,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:17.792016+0000","last_change":"2024-09-14T04:56:49.435207+0000","last_active":"2024-09-14T04:59:17.792016+0000","last_peered":"2024-09-14T04:59:17.792016+0000","last_clean":"2024-09-14T04:59:17.792016+0000","last_became_active":"2024-09-14T04:56:49.434752+0000","last_became_peered":"2024-09-14T04:56:49.434752+0000","last_unstale":"2024-09-14T04:59:17.792016+0000","last_undegraded":"2024-09-14T04:59:17.792016+0000","last_fullsized":"2024-09-14T04:59:17.792016+0000","mapping_epoch":1518,"log_start":"1523'3100","ondisk_log_start":"1523'3100","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":7359,"log_dups_size":0,"ondisk_log_size":7359,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:29:43.013481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345178112,"num_objects":5267,"num_object_clones":0,"num_object_copies":15801,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5267,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10459,"num_write_kb":503232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"57.b","version":"1539'10548","reported_seq":10601,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:20.233683+0000","last_change":"2024-09-14T04:56:49.434067+0000","last_active":"2024-09-14T04:59:20.233683+0000","last_peered":"2024-09-14T04:59:20.233683+0000","last_clean":"2024-09-14T04:59:20.233683+0000","last_became_active":"2024-09-14T04:56:49.433725+0000","last_became_peered":"2024-09-14T04:56:49.433725+0000","last_unstale":"2024-09-14T04:59:20.233683+0000","last_undegraded":"2024-09-14T04:59:20.233683+0000","last_fullsized":"2024-09-14T04:59:20.233683+0000","mapping_epoch":1518,"log_start":"1523'1100","ondisk_log_start":"1523'1100","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":9448,"log_dups_size":0,"ondisk_log_size":9448,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:18:56.597341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324665344,"num_objects":4954,"num_object_clones":0,"num_object_copies":14862,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4954,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10548,"num_write_kb":496064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"57.a","version":"1539'10594","reported_seq":10625,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:17.707812+0000","last_change":"2024-09-14T04:56:49.818406+0000","last_active":"2024-09-14T04:59:17.707812+0000","last_peered":"2024-09-14T04:59:17.707812+0000","last_clean":"2024-09-14T04:59:17.707812+0000","last_became_active":"2024-09-14T04:56:49.818087+0000","last_became_peered":"2024-09-14T04:56:49.818087+0000","last_unstale":"2024-09-14T04:59:17.707812+0000","last_undegraded":"2024-09-14T04:59:17.707812+0000","last_fullsized":"2024-09-14T04:59:17.707812+0000","mapping_epoch":1518,"log_start":"1523'1500","ondisk_log_start":"1523'1500","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":9094,"log_dups_size":0,"ondisk_log_size":9094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:41:19.568727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330432512,"num_objects":5042,"num_object_clones":0,"num_object_copies":15126,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5042,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10594,"num_write_kb":500352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"57.9","version":"1539'10368","reported_seq":10399,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:18.404286+0000","last_change":"2024-09-14T04:56:49.818923+0000","last_active":"2024-09-14T04:59:18.404286+0000","last_peered":"2024-09-14T04:59:18.404286+0000","last_clean":"2024-09-14T04:59:18.404286+0000","last_became_active":"2024-09-14T04:56:49.818519+0000","last_became_peered":"2024-09-14T04:56:49.818519+0000","last_unstale":"2024-09-14T04:59:18.404286+0000","last_undegraded":"2024-09-14T04:59:18.404286+0000","last_fullsized":"2024-09-14T04:59:18.404286+0000","mapping_epoch":1518,"log_start":"1523'2800","ondisk_log_start":"1523'2800","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":7568,"log_dups_size":0,"ondisk_log_size":7568,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:41:08.309897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327155712,"num_objects":4992,"num_object_clones":0,"num_object_copies":14976,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4992,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10368,"num_write_kb":491520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"57.8","version":"1539'10349","reported_seq":10380,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:17.674532+0000","last_change":"2024-09-14T04:56:49.434525+0000","last_active":"2024-09-14T04:59:17.674532+0000","last_peered":"2024-09-14T04:59:17.674532+0000","last_clean":"2024-09-14T04:59:17.674532+0000","last_became_active":"2024-09-14T04:56:49.434144+0000","last_became_peered":"2024-09-14T04:56:49.434144+0000","last_unstale":"2024-09-14T04:59:17.674532+0000","last_undegraded":"2024-09-14T04:59:17.674532+0000","last_fullsized":"2024-09-14T04:59:17.674532+0000","mapping_epoch":1518,"log_start":"1523'2600","ondisk_log_start":"1523'2600","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":7749,"log_dups_size":0,"ondisk_log_size":7749,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:17:18.918905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323420160,"num_objects":4935,"num_object_clones":0,"num_object_copies":14805,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4935,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10349,"num_write_kb":489088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"57.7","version":"1539'10589","reported_seq":10643,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:20.237703+0000","last_change":"2024-09-14T04:56:49.434581+0000","last_active":"2024-09-14T04:59:20.237703+0000","last_peered":"2024-09-14T04:59:20.237703+0000","last_clean":"2024-09-14T04:59:20.237703+0000","last_became_active":"2024-09-14T04:56:49.434210+0000","last_became_peered":"2024-09-14T04:56:49.434210+0000","last_unstale":"2024-09-14T04:59:20.237703+0000","last_undegraded":"2024-09-14T04:59:20.237703+0000","last_fullsized":"2024-09-14T04:59:20.237703+0000","mapping_epoch":1518,"log_start":"1523'1200","ondisk_log_start":"1523'1200","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":9389,"log_dups_size":0,"ondisk_log_size":9389,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:16.426758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322895872,"num_objects":4927,"num_object_clones":0,"num_object_copies":14781,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4927,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10589,"num_write_kb":496512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"57.6","version":"1539'10512","reported_seq":10543,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:17.717336+0000","last_change":"2024-09-14T04:56:49.436357+0000","last_active":"2024-09-14T04:59:17.717336+0000","last_peered":"2024-09-14T04:59:17.717336+0000","last_clean":"2024-09-14T04:59:17.717336+0000","last_became_active":"2024-09-14T04:56:49.436101+0000","last_became_peered":"2024-09-14T04:56:49.436101+0000","last_unstale":"2024-09-14T04:59:17.717336+0000","last_undegraded":"2024-09-14T04:59:17.717336+0000","last_fullsized":"2024-09-14T04:59:17.717336+0000","mapping_epoch":1518,"log_start":"1523'1400","ondisk_log_start":"1523'1400","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":9112,"log_dups_size":0,"ondisk_log_size":9112,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:03:53.232843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":328990720,"num_objects":5020,"num_object_clones":0,"num_object_copies":15060,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5020,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10512,"num_write_kb":497024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"57.5","version":"1539'10416","reported_seq":10447,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:17.693144+0000","last_change":"2024-09-14T04:56:49.821285+0000","last_active":"2024-09-14T04:59:17.693144+0000","last_peered":"2024-09-14T04:59:17.693144+0000","last_clean":"2024-09-14T04:59:17.693144+0000","last_became_active":"2024-09-14T04:56:49.820869+0000","last_became_peered":"2024-09-14T04:56:49.820869+0000","last_unstale":"2024-09-14T04:59:17.693144+0000","last_undegraded":"2024-09-14T04:59:17.693144+0000","last_fullsized":"2024-09-14T04:59:17.693144+0000","mapping_epoch":1518,"log_start":"1523'1300","ondisk_log_start":"1523'1300","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":9116,"log_dups_size":0,"ondisk_log_size":9116,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:26:39.392722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329777152,"num_objects":5032,"num_object_clones":0,"num_object_copies":15096,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5032,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10416,"num_write_kb":494336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"57.4","version":"1539'10388","reported_seq":10442,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:19.102836+0000","last_change":"2024-09-14T04:56:49.820010+0000","last_active":"2024-09-14T04:59:19.102836+0000","last_peered":"2024-09-14T04:59:19.102836+0000","last_clean":"2024-09-14T04:59:19.102836+0000","last_became_active":"2024-09-14T04:56:49.819563+0000","last_became_peered":"2024-09-14T04:56:49.819563+0000","last_unstale":"2024-09-14T04:59:19.102836+0000","last_undegraded":"2024-09-14T04:59:19.102836+0000","last_fullsized":"2024-09-14T04:59:19.102836+0000","mapping_epoch":1518,"log_start":"1523'2000","ondisk_log_start":"1523'2000","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":8388,"log_dups_size":0,"ondisk_log_size":8388,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:32:31.141098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318767104,"num_objects":4864,"num_object_clones":0,"num_object_copies":14592,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4864,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10388,"num_write_kb":488064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"57.3","version":"1539'10479","reported_seq":10533,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:20.246750+0000","last_change":"2024-09-14T04:56:49.819031+0000","last_active":"2024-09-14T04:59:20.246750+0000","last_peered":"2024-09-14T04:59:20.246750+0000","last_clean":"2024-09-14T04:59:20.246750+0000","last_became_active":"2024-09-14T04:56:49.818654+0000","last_became_peered":"2024-09-14T04:56:49.818654+0000","last_unstale":"2024-09-14T04:59:20.246750+0000","last_undegraded":"2024-09-14T04:59:20.246750+0000","last_fullsized":"2024-09-14T04:59:20.246750+0000","mapping_epoch":1518,"log_start":"1523'1100","ondisk_log_start":"1523'1100","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":9379,"log_dups_size":0,"ondisk_log_size":9379,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:57:37.054801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326303744,"num_objects":4979,"num_object_clones":0,"num_object_copies":14937,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4979,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10479,"num_write_kb":494656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"57.2","version":"1539'10399","reported_seq":10453,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:19.088996+0000","last_change":"2024-09-14T04:56:49.819908+0000","last_active":"2024-09-14T04:59:19.088996+0000","last_peered":"2024-09-14T04:59:19.088996+0000","last_clean":"2024-09-14T04:59:19.088996+0000","last_became_active":"2024-09-14T04:56:49.819599+0000","last_became_peered":"2024-09-14T04:56:49.819599+0000","last_unstale":"2024-09-14T04:59:19.088996+0000","last_undegraded":"2024-09-14T04:59:19.088996+0000","last_fullsized":"2024-09-14T04:59:19.088996+0000","mapping_epoch":1518,"log_start":"1523'2000","ondisk_log_start":"1523'2000","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":8399,"log_dups_size":0,"ondisk_log_size":8399,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:38:49.493483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330760192,"num_objects":5047,"num_object_clones":0,"num_object_copies":15141,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5047,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10399,"num_write_kb":494272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"57.1","version":"1539'10652","reported_seq":10706,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:19.107378+0000","last_change":"2024-09-14T04:56:49.434507+0000","last_active":"2024-09-14T04:59:19.107378+0000","last_peered":"2024-09-14T04:59:19.107378+0000","last_clean":"2024-09-14T04:59:19.107378+0000","last_became_active":"2024-09-14T04:56:49.434172+0000","last_became_peered":"2024-09-14T04:56:49.434172+0000","last_unstale":"2024-09-14T04:59:19.107378+0000","last_undegraded":"2024-09-14T04:59:19.107378+0000","last_fullsized":"2024-09-14T04:59:19.107378+0000","mapping_epoch":1518,"log_start":"1523'2300","ondisk_log_start":"1523'2300","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":8352,"log_dups_size":0,"ondisk_log_size":8352,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:22.424289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332660736,"num_objects":5076,"num_object_clones":0,"num_object_copies":15228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5076,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10652,"num_write_kb":503296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"57.0","version":"1539'10484","reported_seq":10538,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:20.247734+0000","last_change":"2024-09-14T04:56:49.818610+0000","last_active":"2024-09-14T04:59:20.247734+0000","last_peered":"2024-09-14T04:59:20.247734+0000","last_clean":"2024-09-14T04:59:20.247734+0000","last_became_active":"2024-09-14T04:56:49.818300+0000","last_became_peered":"2024-09-14T04:56:49.818300+0000","last_unstale":"2024-09-14T04:59:20.247734+0000","last_undegraded":"2024-09-14T04:59:20.247734+0000","last_fullsized":"2024-09-14T04:59:20.247734+0000","mapping_epoch":1518,"log_start":"1523'1100","ondisk_log_start":"1523'1100","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":9384,"log_dups_size":0,"ondisk_log_size":9384,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:29:20.641683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321650688,"num_objects":4908,"num_object_clones":0,"num_object_copies":14724,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4908,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10484,"num_write_kb":492544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":2372,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.701339+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T04:59:09.701339+0000","last_peered":"2024-09-14T04:59:09.701339+0000","last_clean":"2024-09-14T04:59:09.701339+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T04:59:09.701339+0000","last_undegraded":"2024-09-14T04:59:09.701339+0000","last_fullsized":"2024-09-14T04:59:09.701339+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":152,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.677347+0000","last_change":"2024-09-14T04:44:57.325109+0000","last_active":"2024-09-14T04:59:06.677347+0000","last_peered":"2024-09-14T04:59:06.677347+0000","last_clean":"2024-09-14T04:59:06.677347+0000","last_became_active":"2024-09-14T04:44:57.324791+0000","last_became_peered":"2024-09-14T04:44:57.324791+0000","last_unstale":"2024-09-14T04:59:06.677347+0000","last_undegraded":"2024-09-14T04:59:06.677347+0000","last_fullsized":"2024-09-14T04:59:06.677347+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:03.593275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":122,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.689901+0000","last_change":"2024-09-14T04:49:29.628435+0000","last_active":"2024-09-14T04:59:06.689901+0000","last_peered":"2024-09-14T04:59:06.689901+0000","last_clean":"2024-09-14T04:59:06.689901+0000","last_became_active":"2024-09-14T04:49:29.628081+0000","last_became_peered":"2024-09-14T04:49:29.628081+0000","last_unstale":"2024-09-14T04:59:06.689901+0000","last_undegraded":"2024-09-14T04:59:06.689901+0000","last_fullsized":"2024-09-14T04:59:06.689901+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:01.520334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":993,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.701741+0000","last_change":"2024-09-14T04:43:13.355281+0000","last_active":"2024-09-14T04:59:09.701741+0000","last_peered":"2024-09-14T04:59:09.701741+0000","last_clean":"2024-09-14T04:59:09.701741+0000","last_became_active":"2024-09-14T04:43:13.354967+0000","last_became_peered":"2024-09-14T04:43:13.354967+0000","last_unstale":"2024-09-14T04:59:09.701741+0000","last_undegraded":"2024-09-14T04:59:09.701741+0000","last_fullsized":"2024-09-14T04:59:09.701741+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:23:19.332075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1531,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.689031+0000","last_change":"2024-09-14T04:44:57.342918+0000","last_active":"2024-09-14T04:59:06.689031+0000","last_peered":"2024-09-14T04:59:06.689031+0000","last_clean":"2024-09-14T04:59:06.689031+0000","last_became_active":"2024-09-14T04:44:57.329550+0000","last_became_peered":"2024-09-14T04:44:57.329550+0000","last_unstale":"2024-09-14T04:59:06.689031+0000","last_undegraded":"2024-09-14T04:59:06.689031+0000","last_fullsized":"2024-09-14T04:59:06.689031+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:52:52.435893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":152,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.677620+0000","last_change":"2024-09-14T04:44:57.325537+0000","last_active":"2024-09-14T04:59:06.677620+0000","last_peered":"2024-09-14T04:59:06.677620+0000","last_clean":"2024-09-14T04:59:06.677620+0000","last_became_active":"2024-09-14T04:44:57.325057+0000","last_became_peered":"2024-09-14T04:44:57.325057+0000","last_unstale":"2024-09-14T04:59:06.677620+0000","last_undegraded":"2024-09-14T04:59:06.677620+0000","last_fullsized":"2024-09-14T04:59:06.677620+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:43:23.166708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1546,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.689426+0000","last_change":"2024-09-14T04:44:57.337033+0000","last_active":"2024-09-14T04:59:06.689426+0000","last_peered":"2024-09-14T04:59:06.689426+0000","last_clean":"2024-09-14T04:59:06.689426+0000","last_became_active":"2024-09-14T04:44:57.328437+0000","last_became_peered":"2024-09-14T04:44:57.328437+0000","last_unstale":"2024-09-14T04:59:06.689426+0000","last_undegraded":"2024-09-14T04:59:06.689426+0000","last_fullsized":"2024-09-14T04:59:06.689426+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.109086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":152,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.678154+0000","last_change":"2024-09-14T04:44:57.332243+0000","last_active":"2024-09-14T04:59:06.678154+0000","last_peered":"2024-09-14T04:59:06.678154+0000","last_clean":"2024-09-14T04:59:06.678154+0000","last_became_active":"2024-09-14T04:44:57.330413+0000","last_became_peered":"2024-09-14T04:44:57.330413+0000","last_unstale":"2024-09-14T04:59:06.678154+0000","last_undegraded":"2024-09-14T04:59:06.678154+0000","last_fullsized":"2024-09-14T04:59:06.678154+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:25:01.338549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":228,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.700886+0000","last_change":"2024-09-14T04:48:29.862929+0000","last_active":"2024-09-14T04:59:09.700886+0000","last_peered":"2024-09-14T04:59:09.700886+0000","last_clean":"2024-09-14T04:59:09.700886+0000","last_became_active":"2024-09-14T04:48:29.862581+0000","last_became_peered":"2024-09-14T04:48:29.862581+0000","last_unstale":"2024-09-14T04:59:09.700886+0000","last_undegraded":"2024-09-14T04:59:09.700886+0000","last_fullsized":"2024-09-14T04:59:09.700886+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:16:43.060088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2367,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.703645+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T04:59:09.703645+0000","last_peered":"2024-09-14T04:59:09.703645+0000","last_clean":"2024-09-14T04:59:09.703645+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T04:59:09.703645+0000","last_undegraded":"2024-09-14T04:59:09.703645+0000","last_fullsized":"2024-09-14T04:59:09.703645+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":152,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.677523+0000","last_change":"2024-09-14T04:44:57.325392+0000","last_active":"2024-09-14T04:59:06.677523+0000","last_peered":"2024-09-14T04:59:06.677523+0000","last_clean":"2024-09-14T04:59:06.677523+0000","last_became_active":"2024-09-14T04:44:57.324624+0000","last_became_peered":"2024-09-14T04:44:57.324624+0000","last_unstale":"2024-09-14T04:59:06.677523+0000","last_undegraded":"2024-09-14T04:59:06.677523+0000","last_fullsized":"2024-09-14T04:59:06.677523+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:33:18.810199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2336,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.735594+0000","last_change":"2024-09-14T04:44:57.329113+0000","last_active":"2024-09-14T04:59:09.735594+0000","last_peered":"2024-09-14T04:59:09.735594+0000","last_clean":"2024-09-14T04:59:09.735594+0000","last_became_active":"2024-09-14T04:44:57.328727+0000","last_became_peered":"2024-09-14T04:44:57.328727+0000","last_unstale":"2024-09-14T04:59:09.735594+0000","last_undegraded":"2024-09-14T04:59:09.735594+0000","last_fullsized":"2024-09-14T04:59:09.735594+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:34.939874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":2353,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.734703+0000","last_change":"2024-09-14T04:44:57.329277+0000","last_active":"2024-09-14T04:59:09.734703+0000","last_peered":"2024-09-14T04:59:09.734703+0000","last_clean":"2024-09-14T04:59:09.734703+0000","last_became_active":"2024-09-14T04:44:57.328804+0000","last_became_peered":"2024-09-14T04:44:57.328804+0000","last_unstale":"2024-09-14T04:59:09.734703+0000","last_undegraded":"2024-09-14T04:59:09.734703+0000","last_fullsized":"2024-09-14T04:59:09.734703+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:42:46.055240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2388,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.707547+0000","last_change":"2024-09-14T04:54:52.250695+0000","last_active":"2024-09-14T04:59:09.707547+0000","last_peered":"2024-09-14T04:59:09.707547+0000","last_clean":"2024-09-14T04:59:09.707547+0000","last_became_active":"2024-09-14T04:54:52.250301+0000","last_became_peered":"2024-09-14T04:54:52.250301+0000","last_unstale":"2024-09-14T04:59:09.707547+0000","last_undegraded":"2024-09-14T04:59:09.707547+0000","last_fullsized":"2024-09-14T04:59:09.707547+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:42.060905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2349,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.701727+0000","last_change":"2024-09-14T04:43:13.357092+0000","last_active":"2024-09-14T04:59:09.701727+0000","last_peered":"2024-09-14T04:59:09.701727+0000","last_clean":"2024-09-14T04:59:09.701727+0000","last_became_active":"2024-09-14T04:43:13.356694+0000","last_became_peered":"2024-09-14T04:43:13.356694+0000","last_unstale":"2024-09-14T04:59:09.701727+0000","last_undegraded":"2024-09-14T04:59:09.701727+0000","last_fullsized":"2024-09-14T04:59:09.701727+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:11.167574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1524,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.697618+0000","last_change":"2024-09-14T04:07:21.720797+0000","last_active":"2024-09-14T04:59:06.697618+0000","last_peered":"2024-09-14T04:59:06.697618+0000","last_clean":"2024-09-14T04:59:06.697618+0000","last_became_active":"2024-09-14T04:07:21.719331+0000","last_became_peered":"2024-09-14T04:07:21.719331+0000","last_unstale":"2024-09-14T04:59:06.697618+0000","last_undegraded":"2024-09-14T04:59:06.697618+0000","last_fullsized":"2024-09-14T04:59:06.697618+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:35.404023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":171,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.688465+0000","last_change":"2024-09-14T04:43:11.641514+0000","last_active":"2024-09-14T04:59:06.688465+0000","last_peered":"2024-09-14T04:59:06.688465+0000","last_clean":"2024-09-14T04:59:06.688465+0000","last_became_active":"2024-09-14T04:43:11.641151+0000","last_became_peered":"2024-09-14T04:43:11.641151+0000","last_unstale":"2024-09-14T04:59:06.688465+0000","last_undegraded":"2024-09-14T04:59:06.688465+0000","last_fullsized":"2024-09-14T04:59:06.688465+0000","mapping_epoch":1384,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:06:45.045430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2361,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.701511+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T04:59:09.701511+0000","last_peered":"2024-09-14T04:59:09.701511+0000","last_clean":"2024-09-14T04:59:09.701511+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T04:59:09.701511+0000","last_undegraded":"2024-09-14T04:59:09.701511+0000","last_fullsized":"2024-09-14T04:59:09.701511+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":752,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.689873+0000","last_change":"2024-09-14T04:49:29.618477+0000","last_active":"2024-09-14T04:59:06.689873+0000","last_peered":"2024-09-14T04:59:06.689873+0000","last_clean":"2024-09-14T04:59:06.689873+0000","last_became_active":"2024-09-14T04:49:29.618210+0000","last_became_peered":"2024-09-14T04:49:29.618210+0000","last_unstale":"2024-09-14T04:59:06.689873+0000","last_undegraded":"2024-09-14T04:59:06.689873+0000","last_fullsized":"2024-09-14T04:59:06.689873+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:13:23.895512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1530,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.689272+0000","last_change":"2024-09-14T04:42:49.938505+0000","last_active":"2024-09-14T04:59:06.689272+0000","last_peered":"2024-09-14T04:59:06.689272+0000","last_clean":"2024-09-14T04:59:06.689272+0000","last_became_active":"2024-09-14T04:42:49.938180+0000","last_became_peered":"2024-09-14T04:42:49.938180+0000","last_unstale":"2024-09-14T04:59:06.689272+0000","last_undegraded":"2024-09-14T04:59:06.689272+0000","last_fullsized":"2024-09-14T04:59:06.689272+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:05.929479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1549,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.690025+0000","last_change":"2024-09-14T04:44:57.346582+0000","last_active":"2024-09-14T04:59:06.690025+0000","last_peered":"2024-09-14T04:59:06.690025+0000","last_clean":"2024-09-14T04:59:06.690025+0000","last_became_active":"2024-09-14T04:44:57.330446+0000","last_became_peered":"2024-09-14T04:44:57.330446+0000","last_unstale":"2024-09-14T04:59:06.690025+0000","last_undegraded":"2024-09-14T04:59:06.690025+0000","last_fullsized":"2024-09-14T04:59:06.690025+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:29.238585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":302,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.734378+0000","last_change":"2024-09-14T04:44:14.480667+0000","last_active":"2024-09-14T04:59:09.734378+0000","last_peered":"2024-09-14T04:59:09.734378+0000","last_clean":"2024-09-14T04:59:09.734378+0000","last_became_active":"2024-09-14T04:44:14.480365+0000","last_became_peered":"2024-09-14T04:44:14.480365+0000","last_unstale":"2024-09-14T04:59:09.734378+0000","last_undegraded":"2024-09-14T04:59:09.734378+0000","last_fullsized":"2024-09-14T04:59:09.734378+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:00.611771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":163,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.678004+0000","last_change":"2024-09-14T04:44:57.341426+0000","last_active":"2024-09-14T04:59:06.678004+0000","last_peered":"2024-09-14T04:59:06.678004+0000","last_clean":"2024-09-14T04:59:06.678004+0000","last_became_active":"2024-09-14T04:44:57.328763+0000","last_became_peered":"2024-09-14T04:44:57.328763+0000","last_unstale":"2024-09-14T04:59:06.678004+0000","last_undegraded":"2024-09-14T04:59:06.678004+0000","last_fullsized":"2024-09-14T04:59:06.678004+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:34:40.066761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":153,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.678290+0000","last_change":"2024-09-14T04:44:57.331337+0000","last_active":"2024-09-14T04:59:06.678290+0000","last_peered":"2024-09-14T04:59:06.678290+0000","last_clean":"2024-09-14T04:59:06.678290+0000","last_became_active":"2024-09-14T04:44:57.329336+0000","last_became_peered":"2024-09-14T04:44:57.329336+0000","last_unstale":"2024-09-14T04:59:06.678290+0000","last_undegraded":"2024-09-14T04:59:06.678290+0000","last_fullsized":"2024-09-14T04:59:06.678290+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:52:35.980869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"349'149","reported_seq":741,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.689332+0000","last_change":"2024-09-14T04:49:30.085274+0000","last_active":"2024-09-14T04:59:06.689332+0000","last_peered":"2024-09-14T04:59:06.689332+0000","last_clean":"2024-09-14T04:59:06.689332+0000","last_became_active":"2024-09-14T04:49:29.623295+0000","last_became_peered":"2024-09-14T04:49:29.623295+0000","last_unstale":"2024-09-14T04:59:06.689332+0000","last_undegraded":"2024-09-14T04:59:06.689332+0000","last_fullsized":"2024-09-14T04:59:06.689332+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:49:30.085089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:49:30.085089+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:10.012963+0000","scrub_duration":0.0071928790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":153,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.679010+0000","last_change":"2024-09-14T04:44:57.329870+0000","last_active":"2024-09-14T04:59:06.679010+0000","last_peered":"2024-09-14T04:59:06.679010+0000","last_clean":"2024-09-14T04:59:06.679010+0000","last_became_active":"2024-09-14T04:44:57.329353+0000","last_became_peered":"2024-09-14T04:44:57.329353+0000","last_unstale":"2024-09-14T04:59:06.679010+0000","last_undegraded":"2024-09-14T04:59:06.679010+0000","last_fullsized":"2024-09-14T04:59:06.679010+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:46.432181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1546,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.697651+0000","last_change":"2024-09-14T04:54:52.242207+0000","last_active":"2024-09-14T04:59:06.697651+0000","last_peered":"2024-09-14T04:59:06.697651+0000","last_clean":"2024-09-14T04:59:06.697651+0000","last_became_active":"2024-09-14T04:54:52.241840+0000","last_became_peered":"2024-09-14T04:54:52.241840+0000","last_unstale":"2024-09-14T04:59:06.697651+0000","last_undegraded":"2024-09-14T04:59:06.697651+0000","last_fullsized":"2024-09-14T04:59:06.697651+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:37.899666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":2399,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.707426+0000","last_change":"2024-09-14T04:44:57.344614+0000","last_active":"2024-09-14T04:59:09.707426+0000","last_peered":"2024-09-14T04:59:09.707426+0000","last_clean":"2024-09-14T04:59:09.707426+0000","last_became_active":"2024-09-14T04:44:57.332256+0000","last_became_peered":"2024-09-14T04:44:57.332256+0000","last_unstale":"2024-09-14T04:59:09.707426+0000","last_undegraded":"2024-09-14T04:59:09.707426+0000","last_fullsized":"2024-09-14T04:59:09.707426+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:23.438417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"349'149","reported_seq":911,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.678252+0000","last_change":"2024-09-14T04:45:00.331854+0000","last_active":"2024-09-14T04:59:06.678252+0000","last_peered":"2024-09-14T04:59:06.678252+0000","last_clean":"2024-09-14T04:59:06.678252+0000","last_became_active":"2024-09-14T04:45:00.331460+0000","last_became_peered":"2024-09-14T04:45:00.331460+0000","last_unstale":"2024-09-14T04:59:06.678252+0000","last_undegraded":"2024-09-14T04:59:06.678252+0000","last_fullsized":"2024-09-14T04:59:06.678252+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.567535+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.567535+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:45.095729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":742,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.688745+0000","last_change":"2024-09-14T04:49:29.616831+0000","last_active":"2024-09-14T04:59:06.688745+0000","last_peered":"2024-09-14T04:59:06.688745+0000","last_clean":"2024-09-14T04:59:06.688745+0000","last_became_active":"2024-09-14T04:49:29.616546+0000","last_became_peered":"2024-09-14T04:49:29.616546+0000","last_unstale":"2024-09-14T04:59:06.688745+0000","last_undegraded":"2024-09-14T04:59:06.688745+0000","last_fullsized":"2024-09-14T04:59:06.688745+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:27.882929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":354,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.735204+0000","last_change":"2024-09-14T04:44:57.330746+0000","last_active":"2024-09-14T04:59:09.735204+0000","last_peered":"2024-09-14T04:59:09.735204+0000","last_clean":"2024-09-14T04:59:09.735204+0000","last_became_active":"2024-09-14T04:44:57.330327+0000","last_became_peered":"2024-09-14T04:44:57.330327+0000","last_unstale":"2024-09-14T04:59:09.735204+0000","last_undegraded":"2024-09-14T04:59:09.735204+0000","last_fullsized":"2024-09-14T04:59:09.735204+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:53.122771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":193,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.696840+0000","last_change":"2024-09-14T04:43:13.355296+0000","last_active":"2024-09-14T04:59:06.696840+0000","last_peered":"2024-09-14T04:59:06.696840+0000","last_clean":"2024-09-14T04:59:06.696840+0000","last_became_active":"2024-09-14T04:43:13.354950+0000","last_became_peered":"2024-09-14T04:43:13.354950+0000","last_unstale":"2024-09-14T04:59:06.696840+0000","last_undegraded":"2024-09-14T04:59:06.696840+0000","last_fullsized":"2024-09-14T04:59:06.696840+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:20:08.968818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"349'149","reported_seq":850,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.689756+0000","last_change":"2024-09-14T04:41:39.471393+0000","last_active":"2024-09-14T04:59:06.689756+0000","last_peered":"2024-09-14T04:59:06.689756+0000","last_clean":"2024-09-14T04:59:06.689756+0000","last_became_active":"2024-09-14T04:41:36.931768+0000","last_became_peered":"2024-09-14T04:41:36.931768+0000","last_unstale":"2024-09-14T04:59:06.689756+0000","last_undegraded":"2024-09-14T04:59:06.689756+0000","last_fullsized":"2024-09-14T04:59:06.689756+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.471248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.471248+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:56.210375+0000","scrub_duration":0.021758344999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2355,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.701126+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T04:59:09.701126+0000","last_peered":"2024-09-14T04:59:09.701126+0000","last_clean":"2024-09-14T04:59:09.701126+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T04:59:09.701126+0000","last_undegraded":"2024-09-14T04:59:09.701126+0000","last_fullsized":"2024-09-14T04:59:09.701126+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1560,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.690110+0000","last_change":"2024-09-14T04:44:57.331436+0000","last_active":"2024-09-14T04:59:06.690110+0000","last_peered":"2024-09-14T04:59:06.690110+0000","last_clean":"2024-09-14T04:59:06.690110+0000","last_became_active":"2024-09-14T04:44:57.330957+0000","last_became_peered":"2024-09-14T04:44:57.330957+0000","last_unstale":"2024-09-14T04:59:06.690110+0000","last_undegraded":"2024-09-14T04:59:06.690110+0000","last_fullsized":"2024-09-14T04:59:06.690110+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:09.516688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":904,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.703441+0000","last_change":"2024-09-14T04:49:49.460401+0000","last_active":"2024-09-14T04:59:09.703441+0000","last_peered":"2024-09-14T04:59:09.703441+0000","last_clean":"2024-09-14T04:59:09.703441+0000","last_became_active":"2024-09-14T04:49:49.458530+0000","last_became_peered":"2024-09-14T04:49:49.458530+0000","last_unstale":"2024-09-14T04:59:09.703441+0000","last_undegraded":"2024-09-14T04:59:09.703441+0000","last_fullsized":"2024-09-14T04:59:09.703441+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:11:29.143239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"349'149","reported_seq":884,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.697546+0000","last_change":"2024-09-14T04:54:22.147441+0000","last_active":"2024-09-14T04:59:06.697546+0000","last_peered":"2024-09-14T04:59:06.697546+0000","last_clean":"2024-09-14T04:59:06.697546+0000","last_became_active":"2024-09-14T04:54:22.147115+0000","last_became_peered":"2024-09-14T04:54:22.147115+0000","last_unstale":"2024-09-14T04:59:06.697546+0000","last_undegraded":"2024-09-14T04:59:06.697546+0000","last_fullsized":"2024-09-14T04:59:06.697546+0000","mapping_epoch":1492,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1493,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:38.479887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:38.479887+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:41:14.001781+0000","scrub_duration":0.0062728319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":310,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.734286+0000","last_change":"2024-09-14T04:44:57.329983+0000","last_active":"2024-09-14T04:59:09.734286+0000","last_peered":"2024-09-14T04:59:09.734286+0000","last_clean":"2024-09-14T04:59:09.734286+0000","last_became_active":"2024-09-14T04:44:57.329597+0000","last_became_peered":"2024-09-14T04:44:57.329597+0000","last_unstale":"2024-09-14T04:59:09.734286+0000","last_undegraded":"2024-09-14T04:59:09.734286+0000","last_fullsized":"2024-09-14T04:59:09.734286+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:45.718039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":848,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.701296+0000","last_change":"2024-09-14T04:49:49.443631+0000","last_active":"2024-09-14T04:59:09.701296+0000","last_peered":"2024-09-14T04:59:09.701296+0000","last_clean":"2024-09-14T04:59:09.701296+0000","last_became_active":"2024-09-14T04:49:49.443313+0000","last_became_peered":"2024-09-14T04:49:49.443313+0000","last_unstale":"2024-09-14T04:59:09.701296+0000","last_undegraded":"2024-09-14T04:59:09.701296+0000","last_fullsized":"2024-09-14T04:59:09.701296+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:06:54.562292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1532,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.689194+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T04:59:06.689194+0000","last_peered":"2024-09-14T04:59:06.689194+0000","last_clean":"2024-09-14T04:59:06.689194+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T04:59:06.689194+0000","last_undegraded":"2024-09-14T04:59:06.689194+0000","last_fullsized":"2024-09-14T04:59:06.689194+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"349'149","reported_seq":842,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.688523+0000","last_change":"2024-09-14T04:41:39.012977+0000","last_active":"2024-09-14T04:59:06.688523+0000","last_peered":"2024-09-14T04:59:06.688523+0000","last_clean":"2024-09-14T04:59:06.688523+0000","last_became_active":"2024-09-14T04:41:36.932552+0000","last_became_peered":"2024-09-14T04:41:36.932552+0000","last_unstale":"2024-09-14T04:59:06.688523+0000","last_undegraded":"2024-09-14T04:59:06.688523+0000","last_fullsized":"2024-09-14T04:59:06.688523+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.012879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.012879+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:16:40.080262+0000","scrub_duration":0.0051721620000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":940,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.703730+0000","last_change":"2024-09-14T04:49:49.459590+0000","last_active":"2024-09-14T04:59:09.703730+0000","last_peered":"2024-09-14T04:59:09.703730+0000","last_clean":"2024-09-14T04:59:09.703730+0000","last_became_active":"2024-09-14T04:49:49.456263+0000","last_became_peered":"2024-09-14T04:49:49.456263+0000","last_unstale":"2024-09-14T04:59:09.703730+0000","last_undegraded":"2024-09-14T04:59:09.703730+0000","last_fullsized":"2024-09-14T04:59:09.703730+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:20:44.666805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":153,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.678227+0000","last_change":"2024-09-14T04:44:57.329652+0000","last_active":"2024-09-14T04:59:06.678227+0000","last_peered":"2024-09-14T04:59:06.678227+0000","last_clean":"2024-09-14T04:59:06.678227+0000","last_became_active":"2024-09-14T04:44:57.328888+0000","last_became_peered":"2024-09-14T04:44:57.328888+0000","last_unstale":"2024-09-14T04:59:06.678227+0000","last_undegraded":"2024-09-14T04:59:06.678227+0000","last_fullsized":"2024-09-14T04:59:06.678227+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:10.170068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1531,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.688883+0000","last_change":"2024-09-14T04:06:46.875340+0000","last_active":"2024-09-14T04:59:06.688883+0000","last_peered":"2024-09-14T04:59:06.688883+0000","last_clean":"2024-09-14T04:59:06.688883+0000","last_became_active":"2024-09-14T04:06:46.874552+0000","last_became_peered":"2024-09-14T04:06:46.874552+0000","last_unstale":"2024-09-14T04:59:06.688883+0000","last_undegraded":"2024-09-14T04:59:06.688883+0000","last_fullsized":"2024-09-14T04:59:06.688883+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:52.257357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"349'149","reported_seq":745,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:07.742834+0000","last_change":"2024-09-14T04:54:55.279666+0000","last_active":"2024-09-14T04:59:07.742834+0000","last_peered":"2024-09-14T04:59:07.742834+0000","last_clean":"2024-09-14T04:59:07.742834+0000","last_became_active":"2024-09-14T04:54:55.279259+0000","last_became_peered":"2024-09-14T04:54:55.279259+0000","last_unstale":"2024-09-14T04:59:07.742834+0000","last_undegraded":"2024-09-14T04:59:07.742834+0000","last_fullsized":"2024-09-14T04:59:07.742834+0000","mapping_epoch":1506,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1507,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:54:03.996025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:54:03.996025+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:40:53.332369+0000","scrub_duration":54,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":302,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.735748+0000","last_change":"2024-09-14T04:44:14.489235+0000","last_active":"2024-09-14T04:59:09.735748+0000","last_peered":"2024-09-14T04:59:09.735748+0000","last_clean":"2024-09-14T04:59:09.735748+0000","last_became_active":"2024-09-14T04:44:14.488950+0000","last_became_peered":"2024-09-14T04:44:14.488950+0000","last_unstale":"2024-09-14T04:59:09.735748+0000","last_undegraded":"2024-09-14T04:59:09.735748+0000","last_fullsized":"2024-09-14T04:59:09.735748+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:54.441144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":858,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.701423+0000","last_change":"2024-09-14T04:49:49.456317+0000","last_active":"2024-09-14T04:59:09.701423+0000","last_peered":"2024-09-14T04:59:09.701423+0000","last_clean":"2024-09-14T04:59:09.701423+0000","last_became_active":"2024-09-14T04:49:49.456029+0000","last_became_peered":"2024-09-14T04:49:49.456029+0000","last_unstale":"2024-09-14T04:59:09.701423+0000","last_undegraded":"2024-09-14T04:59:09.701423+0000","last_fullsized":"2024-09-14T04:59:09.701423+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:46:05.319676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2375,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.701824+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T04:59:09.701824+0000","last_peered":"2024-09-14T04:59:09.701824+0000","last_clean":"2024-09-14T04:59:09.701824+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T04:59:09.701824+0000","last_undegraded":"2024-09-14T04:59:09.701824+0000","last_fullsized":"2024-09-14T04:59:09.701824+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"349'149","reported_seq":736,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.689494+0000","last_change":"2024-09-14T04:50:22.002798+0000","last_active":"2024-09-14T04:59:06.689494+0000","last_peered":"2024-09-14T04:59:06.689494+0000","last_clean":"2024-09-14T04:59:06.689494+0000","last_became_active":"2024-09-14T04:49:49.448278+0000","last_became_peered":"2024-09-14T04:49:49.448278+0000","last_unstale":"2024-09-14T04:59:06.689494+0000","last_undegraded":"2024-09-14T04:59:06.689494+0000","last_fullsized":"2024-09-14T04:59:06.689494+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:50:22.002568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:50:22.002568+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:01:57.489223+0000","scrub_duration":0.010888326,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2376,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.707632+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T04:59:09.707632+0000","last_peered":"2024-09-14T04:59:09.707632+0000","last_clean":"2024-09-14T04:59:09.707632+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T04:59:09.707632+0000","last_undegraded":"2024-09-14T04:59:09.707632+0000","last_fullsized":"2024-09-14T04:59:09.707632+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2397,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.701103+0000","last_change":"2024-09-14T04:54:19.447103+0000","last_active":"2024-09-14T04:59:09.701103+0000","last_peered":"2024-09-14T04:59:09.701103+0000","last_clean":"2024-09-14T04:59:09.701103+0000","last_became_active":"2024-09-14T04:54:19.446748+0000","last_became_peered":"2024-09-14T04:54:19.446748+0000","last_unstale":"2024-09-14T04:59:09.701103+0000","last_undegraded":"2024-09-14T04:59:09.701103+0000","last_fullsized":"2024-09-14T04:59:09.701103+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:00:55.369263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1561,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.689405+0000","last_change":"2024-09-14T04:42:49.939790+0000","last_active":"2024-09-14T04:59:06.689405+0000","last_peered":"2024-09-14T04:59:06.689405+0000","last_clean":"2024-09-14T04:59:06.689405+0000","last_became_active":"2024-09-14T04:42:49.939591+0000","last_became_peered":"2024-09-14T04:42:49.939591+0000","last_unstale":"2024-09-14T04:59:06.689405+0000","last_undegraded":"2024-09-14T04:59:06.689405+0000","last_fullsized":"2024-09-14T04:59:06.689405+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:23:16.790593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1524,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.689691+0000","last_change":"2024-09-14T04:44:57.326008+0000","last_active":"2024-09-14T04:59:06.689691+0000","last_peered":"2024-09-14T04:59:06.689691+0000","last_clean":"2024-09-14T04:59:06.689691+0000","last_became_active":"2024-09-14T04:44:57.325617+0000","last_became_peered":"2024-09-14T04:44:57.325617+0000","last_unstale":"2024-09-14T04:59:06.689691+0000","last_undegraded":"2024-09-14T04:59:06.689691+0000","last_fullsized":"2024-09-14T04:59:06.689691+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:15.637508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":813,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.688969+0000","last_change":"2024-09-14T04:43:13.354846+0000","last_active":"2024-09-14T04:59:06.688969+0000","last_peered":"2024-09-14T04:59:06.688969+0000","last_clean":"2024-09-14T04:59:06.688969+0000","last_became_active":"2024-09-14T04:43:13.354515+0000","last_became_peered":"2024-09-14T04:43:13.354515+0000","last_unstale":"2024-09-14T04:59:06.688969+0000","last_undegraded":"2024-09-14T04:59:06.688969+0000","last_fullsized":"2024-09-14T04:59:06.688969+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:17.889341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1530,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.688899+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T04:59:06.688899+0000","last_peered":"2024-09-14T04:59:06.688899+0000","last_clean":"2024-09-14T04:59:06.688899+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T04:59:06.688899+0000","last_undegraded":"2024-09-14T04:59:06.688899+0000","last_fullsized":"2024-09-14T04:59:06.688899+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":1867,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.688632+0000","last_change":"2024-09-14T04:41:30.640651+0000","last_active":"2024-09-14T04:59:06.688632+0000","last_peered":"2024-09-14T04:59:06.688632+0000","last_clean":"2024-09-14T04:59:06.688632+0000","last_became_active":"2024-09-14T04:41:29.532466+0000","last_became_peered":"2024-09-14T04:41:29.532466+0000","last_unstale":"2024-09-14T04:59:06.688632+0000","last_undegraded":"2024-09-14T04:59:06.688632+0000","last_fullsized":"2024-09-14T04:59:06.688632+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":0,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:30.640520+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:30.640520+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:27.776724+0000","scrub_duration":0.0028653139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"349'149","reported_seq":842,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.688800+0000","last_change":"2024-09-14T04:41:39.978080+0000","last_active":"2024-09-14T04:59:06.688800+0000","last_peered":"2024-09-14T04:59:06.688800+0000","last_clean":"2024-09-14T04:59:06.688800+0000","last_became_active":"2024-09-14T04:41:36.929279+0000","last_became_peered":"2024-09-14T04:41:36.929279+0000","last_unstale":"2024-09-14T04:59:06.688800+0000","last_undegraded":"2024-09-14T04:59:06.688800+0000","last_fullsized":"2024-09-14T04:59:06.688800+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.977999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.977999+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:44:06.383827+0000","scrub_duration":0.00192122,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":236,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.710472+0000","last_change":"2024-09-14T04:48:11.040133+0000","last_active":"2024-09-14T04:59:09.710472+0000","last_peered":"2024-09-14T04:59:09.710472+0000","last_clean":"2024-09-14T04:59:09.710472+0000","last_became_active":"2024-09-14T04:48:11.039787+0000","last_became_peered":"2024-09-14T04:48:11.039787+0000","last_unstale":"2024-09-14T04:59:09.710472+0000","last_undegraded":"2024-09-14T04:59:09.710472+0000","last_fullsized":"2024-09-14T04:59:09.710472+0000","mapping_epoch":1428,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:53.520672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2223,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.734357+0000","last_change":"2024-09-14T04:49:29.616443+0000","last_active":"2024-09-14T04:59:09.734357+0000","last_peered":"2024-09-14T04:59:09.734357+0000","last_clean":"2024-09-14T04:59:09.734357+0000","last_became_active":"2024-09-14T04:49:29.616066+0000","last_became_peered":"2024-09-14T04:49:29.616066+0000","last_unstale":"2024-09-14T04:59:09.734357+0000","last_undegraded":"2024-09-14T04:59:09.734357+0000","last_fullsized":"2024-09-14T04:59:09.734357+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:09:15.897957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":153,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.678505+0000","last_change":"2024-09-14T04:44:57.326724+0000","last_active":"2024-09-14T04:59:06.678505+0000","last_peered":"2024-09-14T04:59:06.678505+0000","last_clean":"2024-09-14T04:59:06.678505+0000","last_became_active":"2024-09-14T04:44:57.326191+0000","last_became_peered":"2024-09-14T04:44:57.326191+0000","last_unstale":"2024-09-14T04:59:06.678505+0000","last_undegraded":"2024-09-14T04:59:06.678505+0000","last_fullsized":"2024-09-14T04:59:06.678505+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:46.060269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":153,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.678941+0000","last_change":"2024-09-14T04:44:57.328528+0000","last_active":"2024-09-14T04:59:06.678941+0000","last_peered":"2024-09-14T04:59:06.678941+0000","last_clean":"2024-09-14T04:59:06.678941+0000","last_became_active":"2024-09-14T04:44:57.327739+0000","last_became_peered":"2024-09-14T04:44:57.327739+0000","last_unstale":"2024-09-14T04:59:06.678941+0000","last_undegraded":"2024-09-14T04:59:06.678941+0000","last_fullsized":"2024-09-14T04:59:06.678941+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:04:58.294690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"349'149","reported_seq":850,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.688465+0000","last_change":"2024-09-14T04:55:59.732757+0000","last_active":"2024-09-14T04:59:06.688465+0000","last_peered":"2024-09-14T04:59:06.688465+0000","last_clean":"2024-09-14T04:59:06.688465+0000","last_became_active":"2024-09-14T04:55:59.732364+0000","last_became_peered":"2024-09-14T04:55:59.732364+0000","last_unstale":"2024-09-14T04:59:06.688465+0000","last_undegraded":"2024-09-14T04:59:06.688465+0000","last_fullsized":"2024-09-14T04:59:06.688465+0000","mapping_epoch":1512,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1513,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:56.738636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:56.738636+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:59:11.313765+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1313440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":153,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.678111+0000","last_change":"2024-09-14T04:44:57.323436+0000","last_active":"2024-09-14T04:59:06.678111+0000","last_peered":"2024-09-14T04:59:06.678111+0000","last_clean":"2024-09-14T04:59:06.678111+0000","last_became_active":"2024-09-14T04:44:57.323022+0000","last_became_peered":"2024-09-14T04:44:57.323022+0000","last_unstale":"2024-09-14T04:59:06.678111+0000","last_undegraded":"2024-09-14T04:59:06.678111+0000","last_fullsized":"2024-09-14T04:59:06.678111+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:22.775261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1559,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.689173+0000","last_change":"2024-09-14T04:44:57.322357+0000","last_active":"2024-09-14T04:59:06.689173+0000","last_peered":"2024-09-14T04:59:06.689173+0000","last_clean":"2024-09-14T04:59:06.689173+0000","last_became_active":"2024-09-14T04:44:57.321990+0000","last_became_peered":"2024-09-14T04:44:57.321990+0000","last_unstale":"2024-09-14T04:59:06.689173+0000","last_undegraded":"2024-09-14T04:59:06.689173+0000","last_fullsized":"2024-09-14T04:59:06.689173+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:09.658008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"349'149","reported_seq":1884,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.689792+0000","last_change":"2024-09-14T04:41:53.858076+0000","last_active":"2024-09-14T04:59:06.689792+0000","last_peered":"2024-09-14T04:59:06.689792+0000","last_clean":"2024-09-14T04:59:06.689792+0000","last_became_active":"2024-09-14T04:41:36.931585+0000","last_became_peered":"2024-09-14T04:41:36.931585+0000","last_unstale":"2024-09-14T04:59:06.689792+0000","last_undegraded":"2024-09-14T04:59:06.689792+0000","last_fullsized":"2024-09-14T04:59:06.689792+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:53.857924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:53.857924+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:10:20.658986+0000","scrub_duration":0.0022355919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":153,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.677916+0000","last_change":"2024-09-14T04:44:57.329198+0000","last_active":"2024-09-14T04:59:06.677916+0000","last_peered":"2024-09-14T04:59:06.677916+0000","last_clean":"2024-09-14T04:59:06.677916+0000","last_became_active":"2024-09-14T04:44:57.327754+0000","last_became_peered":"2024-09-14T04:44:57.327754+0000","last_unstale":"2024-09-14T04:59:06.677916+0000","last_undegraded":"2024-09-14T04:59:06.677916+0000","last_fullsized":"2024-09-14T04:59:06.677916+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:10:27.944476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":360,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.701212+0000","last_change":"2024-09-14T04:44:57.348687+0000","last_active":"2024-09-14T04:59:09.701212+0000","last_peered":"2024-09-14T04:59:09.701212+0000","last_clean":"2024-09-14T04:59:09.701212+0000","last_became_active":"2024-09-14T04:44:57.332845+0000","last_became_peered":"2024-09-14T04:44:57.332845+0000","last_unstale":"2024-09-14T04:59:09.701212+0000","last_undegraded":"2024-09-14T04:59:09.701212+0000","last_fullsized":"2024-09-14T04:59:09.701212+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:12:18.762737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2382,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.700994+0000","last_change":"2024-09-14T04:44:57.333468+0000","last_active":"2024-09-14T04:59:09.700994+0000","last_peered":"2024-09-14T04:59:09.700994+0000","last_clean":"2024-09-14T04:59:09.700994+0000","last_became_active":"2024-09-14T04:44:57.332626+0000","last_became_peered":"2024-09-14T04:44:57.332626+0000","last_unstale":"2024-09-14T04:59:09.700994+0000","last_undegraded":"2024-09-14T04:59:09.700994+0000","last_fullsized":"2024-09-14T04:59:09.700994+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:57:26.366438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1515,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.689479+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T04:59:06.689479+0000","last_peered":"2024-09-14T04:59:06.689479+0000","last_clean":"2024-09-14T04:59:06.689479+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T04:59:06.689479+0000","last_undegraded":"2024-09-14T04:59:06.689479+0000","last_fullsized":"2024-09-14T04:59:06.689479+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1514,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.689049+0000","last_change":"2024-09-14T04:06:46.869868+0000","last_active":"2024-09-14T04:59:06.689049+0000","last_peered":"2024-09-14T04:59:06.689049+0000","last_clean":"2024-09-14T04:59:06.689049+0000","last_became_active":"2024-09-14T04:06:46.869126+0000","last_became_peered":"2024-09-14T04:06:46.869126+0000","last_unstale":"2024-09-14T04:59:06.689049+0000","last_undegraded":"2024-09-14T04:59:06.689049+0000","last_fullsized":"2024-09-14T04:59:06.689049+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:41.150138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1498,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.690125+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T04:59:06.690125+0000","last_peered":"2024-09-14T04:59:06.690125+0000","last_clean":"2024-09-14T04:59:06.690125+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T04:59:06.690125+0000","last_undegraded":"2024-09-14T04:59:06.690125+0000","last_fullsized":"2024-09-14T04:59:06.690125+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":153,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.679077+0000","last_change":"2024-09-14T04:44:57.332112+0000","last_active":"2024-09-14T04:59:06.679077+0000","last_peered":"2024-09-14T04:59:06.679077+0000","last_clean":"2024-09-14T04:59:06.679077+0000","last_became_active":"2024-09-14T04:44:57.331575+0000","last_became_peered":"2024-09-14T04:44:57.331575+0000","last_unstale":"2024-09-14T04:59:06.679077+0000","last_undegraded":"2024-09-14T04:59:06.679077+0000","last_fullsized":"2024-09-14T04:59:06.679077+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:50:00.391283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1546,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.690224+0000","last_change":"2024-09-14T04:44:57.351460+0000","last_active":"2024-09-14T04:59:06.690224+0000","last_peered":"2024-09-14T04:59:06.690224+0000","last_clean":"2024-09-14T04:59:06.690224+0000","last_became_active":"2024-09-14T04:44:57.334662+0000","last_became_peered":"2024-09-14T04:44:57.334662+0000","last_unstale":"2024-09-14T04:59:06.690224+0000","last_undegraded":"2024-09-14T04:59:06.690224+0000","last_fullsized":"2024-09-14T04:59:06.690224+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:02.399062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1500,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.688581+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T04:59:06.688581+0000","last_peered":"2024-09-14T04:59:06.688581+0000","last_clean":"2024-09-14T04:59:06.688581+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T04:59:06.688581+0000","last_undegraded":"2024-09-14T04:59:06.688581+0000","last_fullsized":"2024-09-14T04:59:06.688581+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1558,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.696753+0000","last_change":"2024-09-14T04:44:57.349690+0000","last_active":"2024-09-14T04:59:06.696753+0000","last_peered":"2024-09-14T04:59:06.696753+0000","last_clean":"2024-09-14T04:59:06.696753+0000","last_became_active":"2024-09-14T04:44:57.333579+0000","last_became_peered":"2024-09-14T04:44:57.333579+0000","last_unstale":"2024-09-14T04:59:06.696753+0000","last_undegraded":"2024-09-14T04:59:06.696753+0000","last_fullsized":"2024-09-14T04:59:06.696753+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:17:55.265141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1635,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.688785+0000","last_change":"2024-09-14T04:54:35.694089+0000","last_active":"2024-09-14T04:59:06.688785+0000","last_peered":"2024-09-14T04:59:06.688785+0000","last_clean":"2024-09-14T04:59:06.688785+0000","last_became_active":"2024-09-14T04:54:35.663133+0000","last_became_peered":"2024-09-14T04:54:35.663133+0000","last_unstale":"2024-09-14T04:59:06.688785+0000","last_undegraded":"2024-09-14T04:59:06.688785+0000","last_fullsized":"2024-09-14T04:59:06.688785+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:17:19.947280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":47,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.696551+0000","last_change":"2024-09-14T04:54:52.248439+0000","last_active":"2024-09-14T04:59:06.696551+0000","last_peered":"2024-09-14T04:59:06.696551+0000","last_clean":"2024-09-14T04:59:06.696551+0000","last_became_active":"2024-09-14T04:54:52.248235+0000","last_became_peered":"2024-09-14T04:54:52.248235+0000","last_unstale":"2024-09-14T04:59:06.696551+0000","last_undegraded":"2024-09-14T04:59:06.696551+0000","last_fullsized":"2024-09-14T04:59:06.696551+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:22:09.548139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1520,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.690029+0000","last_change":"2024-09-14T04:44:57.332511+0000","last_active":"2024-09-14T04:59:06.690029+0000","last_peered":"2024-09-14T04:59:06.690029+0000","last_clean":"2024-09-14T04:59:06.690029+0000","last_became_active":"2024-09-14T04:44:57.332166+0000","last_became_peered":"2024-09-14T04:44:57.332166+0000","last_unstale":"2024-09-14T04:59:06.690029+0000","last_undegraded":"2024-09-14T04:59:06.690029+0000","last_fullsized":"2024-09-14T04:59:06.690029+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:53.984639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2376,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.708282+0000","last_change":"2024-09-14T04:54:52.250655+0000","last_active":"2024-09-14T04:59:09.708282+0000","last_peered":"2024-09-14T04:59:09.708282+0000","last_clean":"2024-09-14T04:59:09.708282+0000","last_became_active":"2024-09-14T04:54:52.250317+0000","last_became_peered":"2024-09-14T04:54:52.250317+0000","last_unstale":"2024-09-14T04:59:09.708282+0000","last_undegraded":"2024-09-14T04:59:09.708282+0000","last_fullsized":"2024-09-14T04:59:09.708282+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:12.985406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2361,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.703655+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T04:59:09.703655+0000","last_peered":"2024-09-14T04:59:09.703655+0000","last_clean":"2024-09-14T04:59:09.703655+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T04:59:09.703655+0000","last_undegraded":"2024-09-14T04:59:09.703655+0000","last_fullsized":"2024-09-14T04:59:09.703655+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1512,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.689115+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T04:59:06.689115+0000","last_peered":"2024-09-14T04:59:06.689115+0000","last_clean":"2024-09-14T04:59:06.689115+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T04:59:06.689115+0000","last_undegraded":"2024-09-14T04:59:06.689115+0000","last_fullsized":"2024-09-14T04:59:06.689115+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1549,"reported_epoch":1538,"state":"active+clean","last_fresh":"2024-09-14T04:59:06.689876+0000","last_change":"2024-09-14T04:44:57.339993+0000","last_active":"2024-09-14T04:59:06.689876+0000","last_peered":"2024-09-14T04:59:06.689876+0000","last_clean":"2024-09-14T04:59:06.689876+0000","last_became_active":"2024-09-14T04:44:57.330472+0000","last_became_peered":"2024-09-14T04:44:57.330472+0000","last_unstale":"2024-09-14T04:59:06.689876+0000","last_undegraded":"2024-09-14T04:59:06.689876+0000","last_fullsized":"2024-09-14T04:59:06.689876+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:19:27.046988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2097,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.702401+0000","last_change":"2024-09-14T04:49:49.449687+0000","last_active":"2024-09-14T04:59:09.702401+0000","last_peered":"2024-09-14T04:59:09.702401+0000","last_clean":"2024-09-14T04:59:09.702401+0000","last_became_active":"2024-09-14T04:49:49.449144+0000","last_became_peered":"2024-09-14T04:49:49.449144+0000","last_unstale":"2024-09-14T04:59:09.702401+0000","last_undegraded":"2024-09-14T04:59:09.702401+0000","last_fullsized":"2024-09-14T04:59:09.702401+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:59:06.779730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":57,"num_pg":16,"stat_sum":{"num_bytes":5245960216,"num_objects":80048,"num_object_clones":0,"num_object_copies":240144,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":80048,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":167452,"num_write_kb":7919937,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15692607488,"data_stored":15692595272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":138152,"ondisk_log_size":138152,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1313440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":5070,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4250,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223202100,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2418148,"kb_used_data":1633180,"kb_used_omap":1,"kb_used_meta":784958,"kb_avail":91327004,"statfs":{"total":95995035648,"available":93518852096,"internally_reserved":0,"allocated":1672376320,"data_stored":1666948219,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":803797451},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463736,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3768480,"kb_used_data":2965464,"kb_used_omap":1,"kb_used_meta":803006,"kb_avail":89976672,"statfs":{"total":95995035648,"available":92136112128,"internally_reserved":0,"allocated":3036635136,"data_stored":3031211028,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":822278599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725373,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2760856,"kb_used_data":1935568,"kb_used_omap":1,"kb_used_meta":825150,"kb_avail":90984296,"statfs":{"total":95995035648,"available":93167919104,"internally_reserved":0,"allocated":1982021632,"data_stored":1976602062,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":844954054},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":1392,"seq":5978594476219,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1717668,"kb_used_data":1315676,"kb_used_omap":117,"kb_used_meta":401802,"kb_avail":92027484,"statfs":{"total":95995035648,"available":94236143616,"internally_reserved":0,"allocated":1347252224,"data_stored":1341824675,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120638,"internal_metadata":411445442},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772075115,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3346776,"kb_used_data":2484496,"kb_used_omap":71,"kb_used_meta":862200,"kb_avail":90398376,"statfs":{"total":95995035648,"available":92567937024,"internally_reserved":0,"allocated":2544123904,"data_stored":2538704411,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":882893750},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707304050,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3187284,"kb_used_data":2581516,"kb_used_omap":68,"kb_used_meta":605371,"kb_avail":90557868,"statfs":{"total":95995035648,"available":92731256832,"internally_reserved":0,"allocated":2643472384,"data_stored":2638056867,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":619900112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872729209,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1263948,"kb_used_data":650244,"kb_used_omap":67,"kb_used_meta":613628,"kb_avail":92481204,"statfs":{"total":95995035648,"available":94700752896,"internally_reserved":0,"allocated":665849856,"data_stored":660441390,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":628355429},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056259,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2679700,"kb_used_data":1966220,"kb_used_omap":74,"kb_used_meta":713397,"kb_avail":91065452,"statfs":{"total":95995035648,"available":93251022848,"internally_reserved":0,"allocated":2013409280,"data_stored":2007993231,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":730519221},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1901,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1954418688,"data_stored":1954414616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":653131776,"data_stored":653131776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2601648128,"data_stored":2601648128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2528907264,"data_stored":2528903192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1329004544,"data_stored":1329004544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1965555712,"data_stored":1965555712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":3010400256,"data_stored":3010396184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1649541120,"data_stored":1649541120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:59:26.585 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:59:26.629 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:26.670 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:26.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:26.671 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:26.772 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:26.808 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:26.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:26.809 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:26.910 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:26.949 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:26.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:26.950 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:27.051 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:27.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:27.083 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:27.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:27.097 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-14T04:59:27.184 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:27.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:27.215 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:27.215 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:27.316 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:27.333 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is unset 2024-09-14T04:59:27.333 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:27 smithi039 ceph-mon[106470]: pgmap v3003: 99 pgs: 99 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2024-09-14T04:59:27.334 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:27 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4086634330' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:59:27.334 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:27 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:59:27.334 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:27 smithi039 ceph-mon[111235]: pgmap v3003: 99 pgs: 99 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2024-09-14T04:59:27.334 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:27 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4086634330' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:59:27.334 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:27 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:59:27.355 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:27.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:27.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:27.457 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:27.486 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:27.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:27.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:27.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:27 smithi129 ceph-mon[101316]: pgmap v3003: 99 pgs: 99 active+clean; 4.6 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2024-09-14T04:59:27.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:27 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4086634330' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:59:27.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:27 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T04:59:27.589 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:27.620 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:27.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:27.621 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:27.681 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:59:27.722 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:27.755 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:27.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:27.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:27.857 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:27.905 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:27.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:27.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:28.007 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:28.048 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:28.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:28.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:28.149 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:28.172 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:59:28.201 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:59:28.204 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:28.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:28.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:28.306 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:28.333 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:28 smithi039 ceph-mon[106470]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:59:28.334 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:28 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:59:28.334 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:28 smithi039 ceph-mon[106470]: osdmap e1540: 8 total, 8 up, 8 in 2024-09-14T04:59:28.334 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:28 smithi039 ceph-mon[111235]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:59:28.334 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:28 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:59:28.334 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:28 smithi039 ceph-mon[111235]: osdmap e1540: 8 total, 8 up, 8 in 2024-09-14T04:59:28.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:28.340 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:28.340 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:28.441 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:28.466 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:28.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:28.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:28.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:28 smithi129 ceph-mon[101316]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T04:59:28.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:28 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T04:59:28.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:28 smithi129 ceph-mon[101316]: osdmap e1540: 8 total, 8 up, 8 in 2024-09-14T04:59:28.568 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:28.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:28.603 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:28.603 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:28.704 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:28.747 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:28.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:28.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:28.849 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:28.875 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:28.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:28.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:28.932 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:59:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:59:28] "GET /metrics HTTP/1.1" 200 36732 "" "Prometheus/2.51.0" 2024-09-14T04:59:28.977 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:29.015 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:29.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:29.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:29.117 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:29.152 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:29.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:29.153 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:29.254 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:29.287 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:29.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:29.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:29.289 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:29 smithi039 ceph-mon[106470]: pgmap v3005: 99 pgs: 99 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2024-09-14T04:59:29.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:29 smithi039 ceph-mon[111235]: pgmap v3005: 99 pgs: 99 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2024-09-14T04:59:29.389 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:29.421 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:29 smithi129 ceph-mon[101316]: pgmap v3005: 99 pgs: 99 active+clean; 4.5 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2024-09-14T04:59:29.427 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:29.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:29.428 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:29.529 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:29.573 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:29.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:29.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:29.675 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:29.717 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:29.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:29.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:29.820 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:29.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:29.858 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:29.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:29.958 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:29.992 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:29.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:29.993 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:30.094 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:30.124 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T04:59:30.128 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:30.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:30.129 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:30.230 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:30.268 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:30.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:30.270 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:30.370 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:30.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:30.410 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:30.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:30.511 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:30.541 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:30.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:30.542 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:30.642 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:30.680 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:30.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:30.683 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:30.784 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:30.822 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:30.822 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:30.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:30.924 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:30.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:30.962 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:30.963 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:31.063 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:31.097 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:31.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:31.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:31.198 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:31.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:31.234 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:31.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:31.335 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:31.366 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:31.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:31.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:31.431 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:31 smithi129 ceph-mon[101316]: pgmap v3006: 99 pgs: 99 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2024-09-14T04:59:31.468 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:31.507 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:31.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:31.508 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:31.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:31 smithi039 ceph-mon[106470]: pgmap v3006: 99 pgs: 99 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2024-09-14T04:59:31.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:31 smithi039 ceph-mon[111235]: pgmap v3006: 99 pgs: 99 active+clean; 4.4 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 0 B/s wr, 863 op/s 2024-09-14T04:59:31.609 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:31.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:31.669 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:31.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:31.770 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:31.814 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:31.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:31.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:31.916 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:31.946 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:31.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:31.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:32.048 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:32.088 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:32.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:32.090 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:32.190 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:32.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:32.228 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:32.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:32.330 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:32.365 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:32.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:32.366 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:32.467 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:32.496 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:32.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:32.497 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:32.598 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:32.654 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:32.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:32.655 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:32.756 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:32.793 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:32.794 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:32.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:32.895 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:32.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:32.936 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:32.936 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:33.037 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:33.072 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:33.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:33.073 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:33.116 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:33 smithi039 ceph-mon[106470]: pgmap v3007: 99 pgs: 99 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 864 op/s 2024-09-14T04:59:33.117 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:33 smithi039 ceph-mon[111235]: pgmap v3007: 99 pgs: 99 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 864 op/s 2024-09-14T04:59:33.174 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:33.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:33.206 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:33.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:33.307 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:33.345 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:33.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:33.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:33.447 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:33.486 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:33.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:33.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:33.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:33 smithi129 ceph-mon[101316]: pgmap v3007: 99 pgs: 99 active+clean; 4.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 864 op/s 2024-09-14T04:59:33.587 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:33.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:33.626 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:33.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:33.727 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:33.760 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:33.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:33.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:33.862 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:33.900 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:33.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:33.901 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:34.002 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:34.040 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:34.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:34.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:34.142 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:34.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:34.178 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:34.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:34.279 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:34.333 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:34.333 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:34.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:34.434 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:34.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:34.462 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:34.462 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:34.563 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:34.566 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:59:34.566 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:59:34.586 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:34.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:34.587 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:34.618 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:59:34.618 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:59:34.634 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:59:34.644 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T04:59:34.655 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:59:34.655 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:59:34.688 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:34.690 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:59:34.691 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:59:34.716 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:34.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:34.717 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:34.726 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:59:34.727 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:59:34.762 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:59:34.762 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:59:34.799 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:59:34.799 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:59:34.818 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:34.842 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:59:34.843 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:59:34.853 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:34.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:34.854 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:34.955 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:34.984 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:34.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:34.985 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:35.086 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:35.122 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:35.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:35.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:35.225 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:35.262 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:35.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:35.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:35.364 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:35.390 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:59:35.390 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [], 2024-09-14T04:59:35.390 INFO:teuthology.orchestra.run.smithi039.stdout: "complaint_time": 30, 2024-09-14T04:59:35.390 INFO:teuthology.orchestra.run.smithi039.stdout: "num_blocked_ops": 0 2024-09-14T04:59:35.390 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:59:35.390 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:35 smithi039 ceph-mon[106470]: pgmap v3008: 99 pgs: 99 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 860 op/s 2024-09-14T04:59:35.390 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:35 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3798548324' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:59:35.390 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:35 smithi039 ceph-mon[106470]: from='client.36474 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:59:35.391 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:35 smithi039 ceph-mon[111235]: pgmap v3008: 99 pgs: 99 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 860 op/s 2024-09-14T04:59:35.391 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:35 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3798548324' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:59:35.391 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:35 smithi039 ceph-mon[111235]: from='client.36474 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:59:35.396 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:35.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:35.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:35.498 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:35.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:35 smithi129 ceph-mon[101316]: pgmap v3008: 99 pgs: 99 active+clean; 4.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 860 op/s 2024-09-14T04:59:35.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:35 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3798548324' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:59:35.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:35 smithi129 ceph-mon[101316]: from='client.36474 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:59:35.534 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:35.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:35.536 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:35.636 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:35.677 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:35.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:35.678 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:35.779 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:35.827 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:35.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:35.828 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:35.929 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:35.966 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:35.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:35.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:36.068 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:36.102 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:36.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:36.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:36.204 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:36.231 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:36.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:36.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:36.333 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:36.368 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:36.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:36.369 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:36.423 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":3008,"stamp":"2024-09-14T04:59:33.965705+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4553775101,"num_objects":69503,"num_object_clones":0,"num_object_copies":208509,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":69503,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":178087,"num_write_kb":7919970,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":53,"num_bytes_recovered":1318510,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":140021,"ondisk_log_size":140021,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":298,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":19121040,"kb_used_data":13445836,"kb_used_omap":404,"kb_used_meta":5674539,"kb_avail":730840176,"statfs":{"total":767960285184,"available":748380340224,"internally_reserved":0,"allocated":13768536064,"data_stored":13725116755,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":414022,"internal_metadata":5810728634},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-564133888,"num_objects":-8608,"num_object_clones":0,"num_object_copies":-25824,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8608,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8608,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004383"},"pg_stats":[{"pgid":"57.f","version":"1540'11070","reported_seq":11125,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:30.311588+0000","last_change":"2024-09-14T04:56:49.435182+0000","last_active":"2024-09-14T04:59:30.311588+0000","last_peered":"2024-09-14T04:59:30.311588+0000","last_clean":"2024-09-14T04:59:30.311588+0000","last_became_active":"2024-09-14T04:56:49.434949+0000","last_became_peered":"2024-09-14T04:56:49.434949+0000","last_unstale":"2024-09-14T04:59:30.311588+0000","last_undegraded":"2024-09-14T04:59:30.311588+0000","last_fullsized":"2024-09-14T04:59:30.311588+0000","mapping_epoch":1518,"log_start":"1523'1600","ondisk_log_start":"1523'1600","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":9470,"log_dups_size":0,"ondisk_log_size":9470,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:32:29.856162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287965184,"num_objects":4394,"num_object_clones":0,"num_object_copies":13182,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4394,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11070,"num_write_kb":494848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"57.e","version":"1540'10797","reported_seq":10853,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:29.546383+0000","last_change":"2024-09-14T04:56:49.431930+0000","last_active":"2024-09-14T04:59:29.546383+0000","last_peered":"2024-09-14T04:59:29.546383+0000","last_clean":"2024-09-14T04:59:29.546383+0000","last_became_active":"2024-09-14T04:56:49.431642+0000","last_became_peered":"2024-09-14T04:56:49.431642+0000","last_unstale":"2024-09-14T04:59:29.546383+0000","last_undegraded":"2024-09-14T04:59:29.546383+0000","last_fullsized":"2024-09-14T04:59:29.546383+0000","mapping_epoch":1518,"log_start":"1523'2200","ondisk_log_start":"1523'2200","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":8597,"log_dups_size":0,"ondisk_log_size":8597,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:30:17.677827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289603584,"num_objects":4419,"num_object_clones":0,"num_object_copies":13257,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4419,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10797,"num_write_kb":486912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"57.d","version":"1540'11280","reported_seq":11312,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:32.817217+0000","last_change":"2024-09-14T04:56:49.821152+0000","last_active":"2024-09-14T04:59:32.817217+0000","last_peered":"2024-09-14T04:59:32.817217+0000","last_clean":"2024-09-14T04:59:32.817217+0000","last_became_active":"2024-09-14T04:56:49.820853+0000","last_became_peered":"2024-09-14T04:56:49.820853+0000","last_unstale":"2024-09-14T04:59:32.817217+0000","last_undegraded":"2024-09-14T04:59:32.817217+0000","last_fullsized":"2024-09-14T04:59:32.817217+0000","mapping_epoch":1518,"log_start":"1524'3900","ondisk_log_start":"1524'3900","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":7380,"log_dups_size":0,"ondisk_log_size":7380,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:41.229640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279117848,"num_objects":4260,"num_object_clones":0,"num_object_copies":12780,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4260,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11280,"num_write_kb":497217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"57.c","version":"1540'11325","reported_seq":11357,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:32.786839+0000","last_change":"2024-09-14T04:56:49.435207+0000","last_active":"2024-09-14T04:59:32.786839+0000","last_peered":"2024-09-14T04:59:32.786839+0000","last_clean":"2024-09-14T04:59:32.786839+0000","last_became_active":"2024-09-14T04:56:49.434752+0000","last_became_peered":"2024-09-14T04:56:49.434752+0000","last_unstale":"2024-09-14T04:59:32.786839+0000","last_undegraded":"2024-09-14T04:59:32.786839+0000","last_fullsized":"2024-09-14T04:59:32.786839+0000","mapping_epoch":1518,"log_start":"1524'4000","ondisk_log_start":"1524'4000","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":7325,"log_dups_size":0,"ondisk_log_size":7325,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:29:43.013481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288423936,"num_objects":4401,"num_object_clones":0,"num_object_copies":13203,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4401,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11325,"num_write_kb":503232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"57.b","version":"1540'11092","reported_seq":11146,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:30.314852+0000","last_change":"2024-09-14T04:56:49.434067+0000","last_active":"2024-09-14T04:59:30.314852+0000","last_peered":"2024-09-14T04:59:30.314852+0000","last_clean":"2024-09-14T04:59:30.314852+0000","last_became_active":"2024-09-14T04:56:49.433725+0000","last_became_peered":"2024-09-14T04:56:49.433725+0000","last_unstale":"2024-09-14T04:59:30.314852+0000","last_undegraded":"2024-09-14T04:59:30.314852+0000","last_fullsized":"2024-09-14T04:59:30.314852+0000","mapping_epoch":1518,"log_start":"1523'1700","ondisk_log_start":"1523'1700","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":9392,"log_dups_size":0,"ondisk_log_size":9392,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:18:56.597341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289013760,"num_objects":4410,"num_object_clones":0,"num_object_copies":13230,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4410,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11092,"num_write_kb":496064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"57.a","version":"1540'11428","reported_seq":11460,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:32.779941+0000","last_change":"2024-09-14T04:56:49.818406+0000","last_active":"2024-09-14T04:59:32.779941+0000","last_peered":"2024-09-14T04:59:32.779941+0000","last_clean":"2024-09-14T04:59:32.779941+0000","last_became_active":"2024-09-14T04:56:49.818087+0000","last_became_peered":"2024-09-14T04:56:49.818087+0000","last_unstale":"2024-09-14T04:59:32.779941+0000","last_undegraded":"2024-09-14T04:59:32.779941+0000","last_fullsized":"2024-09-14T04:59:32.779941+0000","mapping_epoch":1518,"log_start":"1523'2300","ondisk_log_start":"1523'2300","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":9128,"log_dups_size":0,"ondisk_log_size":9128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:41:19.568727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275775488,"num_objects":4208,"num_object_clones":0,"num_object_copies":12624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4208,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11428,"num_write_kb":500352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"57.9","version":"1540'11153","reported_seq":11183,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:33.428087+0000","last_change":"2024-09-14T04:56:49.818923+0000","last_active":"2024-09-14T04:59:33.428087+0000","last_peered":"2024-09-14T04:59:33.428087+0000","last_clean":"2024-09-14T04:59:33.428087+0000","last_became_active":"2024-09-14T04:56:49.818519+0000","last_became_peered":"2024-09-14T04:56:49.818519+0000","last_unstale":"2024-09-14T04:59:33.428087+0000","last_undegraded":"2024-09-14T04:59:33.428087+0000","last_fullsized":"2024-09-14T04:59:33.428087+0000","mapping_epoch":1518,"log_start":"1524'3600","ondisk_log_start":"1524'3600","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":7553,"log_dups_size":0,"ondisk_log_size":7553,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:41:08.309897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":275709952,"num_objects":4207,"num_object_clones":0,"num_object_copies":12621,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4207,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11153,"num_write_kb":491520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"57.8","version":"1540'11171","reported_seq":11203,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:32.711255+0000","last_change":"2024-09-14T04:56:49.434525+0000","last_active":"2024-09-14T04:59:32.711255+0000","last_peered":"2024-09-14T04:59:32.711255+0000","last_clean":"2024-09-14T04:59:32.711255+0000","last_became_active":"2024-09-14T04:56:49.434144+0000","last_became_peered":"2024-09-14T04:56:49.434144+0000","last_unstale":"2024-09-14T04:59:32.711255+0000","last_undegraded":"2024-09-14T04:59:32.711255+0000","last_fullsized":"2024-09-14T04:59:32.711255+0000","mapping_epoch":1518,"log_start":"1524'3400","ondisk_log_start":"1524'3400","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":7771,"log_dups_size":0,"ondisk_log_size":7771,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:17:18.918905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269549568,"num_objects":4113,"num_object_clones":0,"num_object_copies":12339,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4113,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11171,"num_write_kb":489088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"57.7","version":"1540'11098","reported_seq":11154,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:30.299615+0000","last_change":"2024-09-14T04:56:49.434581+0000","last_active":"2024-09-14T04:59:30.299615+0000","last_peered":"2024-09-14T04:59:30.299615+0000","last_clean":"2024-09-14T04:59:30.299615+0000","last_became_active":"2024-09-14T04:56:49.434210+0000","last_became_peered":"2024-09-14T04:56:49.434210+0000","last_unstale":"2024-09-14T04:59:30.299615+0000","last_undegraded":"2024-09-14T04:59:30.299615+0000","last_fullsized":"2024-09-14T04:59:30.299615+0000","mapping_epoch":1518,"log_start":"1523'1700","ondisk_log_start":"1523'1700","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":9398,"log_dups_size":0,"ondisk_log_size":9398,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:16.426758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":289538048,"num_objects":4418,"num_object_clones":0,"num_object_copies":13254,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4418,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11098,"num_write_kb":496512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"57.6","version":"1540'11273","reported_seq":11305,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:32.758897+0000","last_change":"2024-09-14T04:56:49.436357+0000","last_active":"2024-09-14T04:59:32.758897+0000","last_peered":"2024-09-14T04:59:32.758897+0000","last_clean":"2024-09-14T04:59:32.758897+0000","last_became_active":"2024-09-14T04:56:49.436101+0000","last_became_peered":"2024-09-14T04:56:49.436101+0000","last_unstale":"2024-09-14T04:59:32.758897+0000","last_undegraded":"2024-09-14T04:59:32.758897+0000","last_fullsized":"2024-09-14T04:59:32.758897+0000","mapping_epoch":1518,"log_start":"1523'2100","ondisk_log_start":"1523'2100","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":9173,"log_dups_size":0,"ondisk_log_size":9173,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:03:53.232843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":279117824,"num_objects":4259,"num_object_clones":0,"num_object_copies":12777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11273,"num_write_kb":497024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"57.5","version":"1540'11191","reported_seq":11223,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:32.742033+0000","last_change":"2024-09-14T04:56:49.821285+0000","last_active":"2024-09-14T04:59:32.742033+0000","last_peered":"2024-09-14T04:59:32.742033+0000","last_clean":"2024-09-14T04:59:32.742033+0000","last_became_active":"2024-09-14T04:56:49.820869+0000","last_became_peered":"2024-09-14T04:56:49.820869+0000","last_unstale":"2024-09-14T04:59:32.742033+0000","last_undegraded":"2024-09-14T04:59:32.742033+0000","last_fullsized":"2024-09-14T04:59:32.742033+0000","mapping_epoch":1518,"log_start":"1523'2100","ondisk_log_start":"1523'2100","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":9091,"log_dups_size":0,"ondisk_log_size":9091,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:26:39.392722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278986752,"num_objects":4257,"num_object_clones":0,"num_object_copies":12771,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4257,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11191,"num_write_kb":494336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"57.4","version":"1540'10933","reported_seq":10989,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:29.068346+0000","last_change":"2024-09-14T04:56:49.820010+0000","last_active":"2024-09-14T04:59:29.068346+0000","last_peered":"2024-09-14T04:59:29.068346+0000","last_clean":"2024-09-14T04:59:29.068346+0000","last_became_active":"2024-09-14T04:56:49.819563+0000","last_became_peered":"2024-09-14T04:56:49.819563+0000","last_unstale":"2024-09-14T04:59:29.068346+0000","last_undegraded":"2024-09-14T04:59:29.068346+0000","last_fullsized":"2024-09-14T04:59:29.068346+0000","mapping_epoch":1518,"log_start":"1523'2500","ondisk_log_start":"1523'2500","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":8433,"log_dups_size":0,"ondisk_log_size":8433,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:32:31.141098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283049984,"num_objects":4319,"num_object_clones":0,"num_object_copies":12957,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4319,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10933,"num_write_kb":488064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"57.3","version":"1540'11001","reported_seq":11057,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:30.277386+0000","last_change":"2024-09-14T04:56:49.819031+0000","last_active":"2024-09-14T04:59:30.277386+0000","last_peered":"2024-09-14T04:59:30.277386+0000","last_clean":"2024-09-14T04:59:30.277386+0000","last_became_active":"2024-09-14T04:56:49.818654+0000","last_became_peered":"2024-09-14T04:56:49.818654+0000","last_unstale":"2024-09-14T04:59:30.277386+0000","last_undegraded":"2024-09-14T04:59:30.277386+0000","last_fullsized":"2024-09-14T04:59:30.277386+0000","mapping_epoch":1518,"log_start":"1523'1600","ondisk_log_start":"1523'1600","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":9401,"log_dups_size":0,"ondisk_log_size":9401,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:57:37.054801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292093952,"num_objects":4457,"num_object_clones":0,"num_object_copies":13371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11001,"num_write_kb":494656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"57.2","version":"1540'10939","reported_seq":10995,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:29.091854+0000","last_change":"2024-09-14T04:56:49.819908+0000","last_active":"2024-09-14T04:59:29.091854+0000","last_peered":"2024-09-14T04:59:29.091854+0000","last_clean":"2024-09-14T04:59:29.091854+0000","last_became_active":"2024-09-14T04:56:49.819599+0000","last_became_peered":"2024-09-14T04:56:49.819599+0000","last_unstale":"2024-09-14T04:59:29.091854+0000","last_undegraded":"2024-09-14T04:59:29.091854+0000","last_fullsized":"2024-09-14T04:59:29.091854+0000","mapping_epoch":1518,"log_start":"1523'2600","ondisk_log_start":"1523'2600","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":8339,"log_dups_size":0,"ondisk_log_size":8339,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:38:49.493483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295370752,"num_objects":4507,"num_object_clones":0,"num_object_copies":13521,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4507,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10939,"num_write_kb":494272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"57.1","version":"1540'11220","reported_seq":11276,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:29.069165+0000","last_change":"2024-09-14T04:56:49.434507+0000","last_active":"2024-09-14T04:59:29.069165+0000","last_peered":"2024-09-14T04:59:29.069165+0000","last_clean":"2024-09-14T04:59:29.069165+0000","last_became_active":"2024-09-14T04:56:49.434172+0000","last_became_peered":"2024-09-14T04:56:49.434172+0000","last_unstale":"2024-09-14T04:59:29.069165+0000","last_undegraded":"2024-09-14T04:59:29.069165+0000","last_fullsized":"2024-09-14T04:59:29.069165+0000","mapping_epoch":1518,"log_start":"1523'2800","ondisk_log_start":"1523'2800","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":8420,"log_dups_size":0,"ondisk_log_size":8420,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:22.424289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":295436288,"num_objects":4508,"num_object_clones":0,"num_object_copies":13524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11220,"num_write_kb":503296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"57.0","version":"1540'11052","reported_seq":11108,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:30.278448+0000","last_change":"2024-09-14T04:56:49.818610+0000","last_active":"2024-09-14T04:59:30.278448+0000","last_peered":"2024-09-14T04:59:30.278448+0000","last_clean":"2024-09-14T04:59:30.278448+0000","last_became_active":"2024-09-14T04:56:49.818300+0000","last_became_peered":"2024-09-14T04:56:49.818300+0000","last_unstale":"2024-09-14T04:59:30.278448+0000","last_undegraded":"2024-09-14T04:59:30.278448+0000","last_fullsized":"2024-09-14T04:59:30.278448+0000","mapping_epoch":1518,"log_start":"1523'1600","ondisk_log_start":"1523'1600","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":9452,"log_dups_size":0,"ondisk_log_size":9452,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:29:20.641683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":284426240,"num_objects":4340,"num_object_clones":0,"num_object_copies":13020,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4340,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11052,"num_write_kb":492544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":2374,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.053764+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T04:59:27.053764+0000","last_peered":"2024-09-14T04:59:27.053764+0000","last_clean":"2024-09-14T04:59:27.053764+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T04:59:27.053764+0000","last_undegraded":"2024-09-14T04:59:27.053764+0000","last_fullsized":"2024-09-14T04:59:27.053764+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":154,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.043864+0000","last_change":"2024-09-14T04:44:57.325109+0000","last_active":"2024-09-14T04:59:27.043864+0000","last_peered":"2024-09-14T04:59:27.043864+0000","last_clean":"2024-09-14T04:59:27.043864+0000","last_became_active":"2024-09-14T04:44:57.324791+0000","last_became_peered":"2024-09-14T04:44:57.324791+0000","last_unstale":"2024-09-14T04:59:27.043864+0000","last_undegraded":"2024-09-14T04:59:27.043864+0000","last_fullsized":"2024-09-14T04:59:27.043864+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:03.593275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":123,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.697437+0000","last_change":"2024-09-14T04:49:29.628435+0000","last_active":"2024-09-14T04:59:09.697437+0000","last_peered":"2024-09-14T04:59:09.697437+0000","last_clean":"2024-09-14T04:59:09.697437+0000","last_became_active":"2024-09-14T04:49:29.628081+0000","last_became_peered":"2024-09-14T04:49:29.628081+0000","last_unstale":"2024-09-14T04:59:09.697437+0000","last_undegraded":"2024-09-14T04:59:09.697437+0000","last_fullsized":"2024-09-14T04:59:09.697437+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:01.520334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":995,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.061017+0000","last_change":"2024-09-14T04:43:13.355281+0000","last_active":"2024-09-14T04:59:27.061017+0000","last_peered":"2024-09-14T04:59:27.061017+0000","last_clean":"2024-09-14T04:59:27.061017+0000","last_became_active":"2024-09-14T04:43:13.354967+0000","last_became_peered":"2024-09-14T04:43:13.354967+0000","last_unstale":"2024-09-14T04:59:27.061017+0000","last_undegraded":"2024-09-14T04:59:27.061017+0000","last_fullsized":"2024-09-14T04:59:27.061017+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:23:19.332075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1532,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.700016+0000","last_change":"2024-09-14T04:44:57.342918+0000","last_active":"2024-09-14T04:59:09.700016+0000","last_peered":"2024-09-14T04:59:09.700016+0000","last_clean":"2024-09-14T04:59:09.700016+0000","last_became_active":"2024-09-14T04:44:57.329550+0000","last_became_peered":"2024-09-14T04:44:57.329550+0000","last_unstale":"2024-09-14T04:59:09.700016+0000","last_undegraded":"2024-09-14T04:59:09.700016+0000","last_fullsized":"2024-09-14T04:59:09.700016+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:52:52.435893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":153,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.694891+0000","last_change":"2024-09-14T04:44:57.325537+0000","last_active":"2024-09-14T04:59:09.694891+0000","last_peered":"2024-09-14T04:59:09.694891+0000","last_clean":"2024-09-14T04:59:09.694891+0000","last_became_active":"2024-09-14T04:44:57.325057+0000","last_became_peered":"2024-09-14T04:44:57.325057+0000","last_unstale":"2024-09-14T04:59:09.694891+0000","last_undegraded":"2024-09-14T04:59:09.694891+0000","last_fullsized":"2024-09-14T04:59:09.694891+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:43:23.166708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1547,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.700039+0000","last_change":"2024-09-14T04:44:57.337033+0000","last_active":"2024-09-14T04:59:09.700039+0000","last_peered":"2024-09-14T04:59:09.700039+0000","last_clean":"2024-09-14T04:59:09.700039+0000","last_became_active":"2024-09-14T04:44:57.328437+0000","last_became_peered":"2024-09-14T04:44:57.328437+0000","last_unstale":"2024-09-14T04:59:09.700039+0000","last_undegraded":"2024-09-14T04:59:09.700039+0000","last_fullsized":"2024-09-14T04:59:09.700039+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.109086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":153,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.695246+0000","last_change":"2024-09-14T04:44:57.332243+0000","last_active":"2024-09-14T04:59:09.695246+0000","last_peered":"2024-09-14T04:59:09.695246+0000","last_clean":"2024-09-14T04:59:09.695246+0000","last_became_active":"2024-09-14T04:44:57.330413+0000","last_became_peered":"2024-09-14T04:44:57.330413+0000","last_unstale":"2024-09-14T04:59:09.695246+0000","last_undegraded":"2024-09-14T04:59:09.695246+0000","last_fullsized":"2024-09-14T04:59:09.695246+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:25:01.338549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":230,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.054255+0000","last_change":"2024-09-14T04:48:29.862929+0000","last_active":"2024-09-14T04:59:27.054255+0000","last_peered":"2024-09-14T04:59:27.054255+0000","last_clean":"2024-09-14T04:59:27.054255+0000","last_became_active":"2024-09-14T04:48:29.862581+0000","last_became_peered":"2024-09-14T04:48:29.862581+0000","last_unstale":"2024-09-14T04:59:27.054255+0000","last_undegraded":"2024-09-14T04:59:27.054255+0000","last_fullsized":"2024-09-14T04:59:27.054255+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:16:43.060088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2369,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.060693+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T04:59:27.060693+0000","last_peered":"2024-09-14T04:59:27.060693+0000","last_clean":"2024-09-14T04:59:27.060693+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T04:59:27.060693+0000","last_undegraded":"2024-09-14T04:59:27.060693+0000","last_fullsized":"2024-09-14T04:59:27.060693+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":153,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.694824+0000","last_change":"2024-09-14T04:44:57.325392+0000","last_active":"2024-09-14T04:59:09.694824+0000","last_peered":"2024-09-14T04:59:09.694824+0000","last_clean":"2024-09-14T04:59:09.694824+0000","last_became_active":"2024-09-14T04:44:57.324624+0000","last_became_peered":"2024-09-14T04:44:57.324624+0000","last_unstale":"2024-09-14T04:59:09.694824+0000","last_undegraded":"2024-09-14T04:59:09.694824+0000","last_fullsized":"2024-09-14T04:59:09.694824+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:33:18.810199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2338,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.053178+0000","last_change":"2024-09-14T04:44:57.329113+0000","last_active":"2024-09-14T04:59:27.053178+0000","last_peered":"2024-09-14T04:59:27.053178+0000","last_clean":"2024-09-14T04:59:27.053178+0000","last_became_active":"2024-09-14T04:44:57.328727+0000","last_became_peered":"2024-09-14T04:44:57.328727+0000","last_unstale":"2024-09-14T04:59:27.053178+0000","last_undegraded":"2024-09-14T04:59:27.053178+0000","last_fullsized":"2024-09-14T04:59:27.053178+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:34.939874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":2355,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.052060+0000","last_change":"2024-09-14T04:44:57.329277+0000","last_active":"2024-09-14T04:59:27.052060+0000","last_peered":"2024-09-14T04:59:27.052060+0000","last_clean":"2024-09-14T04:59:27.052060+0000","last_became_active":"2024-09-14T04:44:57.328804+0000","last_became_peered":"2024-09-14T04:44:57.328804+0000","last_unstale":"2024-09-14T04:59:27.052060+0000","last_undegraded":"2024-09-14T04:59:27.052060+0000","last_fullsized":"2024-09-14T04:59:27.052060+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:42:46.055240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2390,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.068643+0000","last_change":"2024-09-14T04:54:52.250695+0000","last_active":"2024-09-14T04:59:27.068643+0000","last_peered":"2024-09-14T04:59:27.068643+0000","last_clean":"2024-09-14T04:59:27.068643+0000","last_became_active":"2024-09-14T04:54:52.250301+0000","last_became_peered":"2024-09-14T04:54:52.250301+0000","last_unstale":"2024-09-14T04:59:27.068643+0000","last_undegraded":"2024-09-14T04:59:27.068643+0000","last_fullsized":"2024-09-14T04:59:27.068643+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:42.060905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2351,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.053147+0000","last_change":"2024-09-14T04:43:13.357092+0000","last_active":"2024-09-14T04:59:27.053147+0000","last_peered":"2024-09-14T04:59:27.053147+0000","last_clean":"2024-09-14T04:59:27.053147+0000","last_became_active":"2024-09-14T04:43:13.356694+0000","last_became_peered":"2024-09-14T04:43:13.356694+0000","last_unstale":"2024-09-14T04:59:27.053147+0000","last_undegraded":"2024-09-14T04:59:27.053147+0000","last_fullsized":"2024-09-14T04:59:27.053147+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:11.167574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1525,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.697386+0000","last_change":"2024-09-14T04:07:21.720797+0000","last_active":"2024-09-14T04:59:09.697386+0000","last_peered":"2024-09-14T04:59:09.697386+0000","last_clean":"2024-09-14T04:59:09.697386+0000","last_became_active":"2024-09-14T04:07:21.719331+0000","last_became_peered":"2024-09-14T04:07:21.719331+0000","last_unstale":"2024-09-14T04:59:09.697386+0000","last_undegraded":"2024-09-14T04:59:09.697386+0000","last_fullsized":"2024-09-14T04:59:09.697386+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:35.404023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":173,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.055526+0000","last_change":"2024-09-14T04:43:11.641514+0000","last_active":"2024-09-14T04:59:27.055526+0000","last_peered":"2024-09-14T04:59:27.055526+0000","last_clean":"2024-09-14T04:59:27.055526+0000","last_became_active":"2024-09-14T04:43:11.641151+0000","last_became_peered":"2024-09-14T04:43:11.641151+0000","last_unstale":"2024-09-14T04:59:27.055526+0000","last_undegraded":"2024-09-14T04:59:27.055526+0000","last_fullsized":"2024-09-14T04:59:27.055526+0000","mapping_epoch":1384,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:06:45.045430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2363,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.053914+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T04:59:27.053914+0000","last_peered":"2024-09-14T04:59:27.053914+0000","last_clean":"2024-09-14T04:59:27.053914+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T04:59:27.053914+0000","last_undegraded":"2024-09-14T04:59:27.053914+0000","last_fullsized":"2024-09-14T04:59:27.053914+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":753,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.702005+0000","last_change":"2024-09-14T04:49:29.618477+0000","last_active":"2024-09-14T04:59:09.702005+0000","last_peered":"2024-09-14T04:59:09.702005+0000","last_clean":"2024-09-14T04:59:09.702005+0000","last_became_active":"2024-09-14T04:49:29.618210+0000","last_became_peered":"2024-09-14T04:49:29.618210+0000","last_unstale":"2024-09-14T04:59:09.702005+0000","last_undegraded":"2024-09-14T04:59:09.702005+0000","last_fullsized":"2024-09-14T04:59:09.702005+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:13:23.895512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1531,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.700298+0000","last_change":"2024-09-14T04:42:49.938505+0000","last_active":"2024-09-14T04:59:09.700298+0000","last_peered":"2024-09-14T04:59:09.700298+0000","last_clean":"2024-09-14T04:59:09.700298+0000","last_became_active":"2024-09-14T04:42:49.938180+0000","last_became_peered":"2024-09-14T04:42:49.938180+0000","last_unstale":"2024-09-14T04:59:09.700298+0000","last_undegraded":"2024-09-14T04:59:09.700298+0000","last_fullsized":"2024-09-14T04:59:09.700298+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:05.929479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1550,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.702638+0000","last_change":"2024-09-14T04:44:57.346582+0000","last_active":"2024-09-14T04:59:09.702638+0000","last_peered":"2024-09-14T04:59:09.702638+0000","last_clean":"2024-09-14T04:59:09.702638+0000","last_became_active":"2024-09-14T04:44:57.330446+0000","last_became_peered":"2024-09-14T04:44:57.330446+0000","last_unstale":"2024-09-14T04:59:09.702638+0000","last_undegraded":"2024-09-14T04:59:09.702638+0000","last_fullsized":"2024-09-14T04:59:09.702638+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:29.238585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":304,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.052063+0000","last_change":"2024-09-14T04:44:14.480667+0000","last_active":"2024-09-14T04:59:27.052063+0000","last_peered":"2024-09-14T04:59:27.052063+0000","last_clean":"2024-09-14T04:59:27.052063+0000","last_became_active":"2024-09-14T04:44:14.480365+0000","last_became_peered":"2024-09-14T04:44:14.480365+0000","last_unstale":"2024-09-14T04:59:27.052063+0000","last_undegraded":"2024-09-14T04:59:27.052063+0000","last_fullsized":"2024-09-14T04:59:27.052063+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:00.611771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":164,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.696426+0000","last_change":"2024-09-14T04:44:57.341426+0000","last_active":"2024-09-14T04:59:09.696426+0000","last_peered":"2024-09-14T04:59:09.696426+0000","last_clean":"2024-09-14T04:59:09.696426+0000","last_became_active":"2024-09-14T04:44:57.328763+0000","last_became_peered":"2024-09-14T04:44:57.328763+0000","last_unstale":"2024-09-14T04:59:09.696426+0000","last_undegraded":"2024-09-14T04:59:09.696426+0000","last_fullsized":"2024-09-14T04:59:09.696426+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:34:40.066761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":154,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.695319+0000","last_change":"2024-09-14T04:44:57.331337+0000","last_active":"2024-09-14T04:59:09.695319+0000","last_peered":"2024-09-14T04:59:09.695319+0000","last_clean":"2024-09-14T04:59:09.695319+0000","last_became_active":"2024-09-14T04:44:57.329336+0000","last_became_peered":"2024-09-14T04:44:57.329336+0000","last_unstale":"2024-09-14T04:59:09.695319+0000","last_undegraded":"2024-09-14T04:59:09.695319+0000","last_fullsized":"2024-09-14T04:59:09.695319+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:52:35.980869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"349'149","reported_seq":742,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.699902+0000","last_change":"2024-09-14T04:49:30.085274+0000","last_active":"2024-09-14T04:59:09.699902+0000","last_peered":"2024-09-14T04:59:09.699902+0000","last_clean":"2024-09-14T04:59:09.699902+0000","last_became_active":"2024-09-14T04:49:29.623295+0000","last_became_peered":"2024-09-14T04:49:29.623295+0000","last_unstale":"2024-09-14T04:59:09.699902+0000","last_undegraded":"2024-09-14T04:59:09.699902+0000","last_fullsized":"2024-09-14T04:59:09.699902+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:49:30.085089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:49:30.085089+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:10.012963+0000","scrub_duration":0.0071928790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":155,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.051693+0000","last_change":"2024-09-14T04:44:57.329870+0000","last_active":"2024-09-14T04:59:27.051693+0000","last_peered":"2024-09-14T04:59:27.051693+0000","last_clean":"2024-09-14T04:59:27.051693+0000","last_became_active":"2024-09-14T04:44:57.329353+0000","last_became_peered":"2024-09-14T04:44:57.329353+0000","last_unstale":"2024-09-14T04:59:27.051693+0000","last_undegraded":"2024-09-14T04:59:27.051693+0000","last_fullsized":"2024-09-14T04:59:27.051693+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:46.432181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1547,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.697411+0000","last_change":"2024-09-14T04:54:52.242207+0000","last_active":"2024-09-14T04:59:09.697411+0000","last_peered":"2024-09-14T04:59:09.697411+0000","last_clean":"2024-09-14T04:59:09.697411+0000","last_became_active":"2024-09-14T04:54:52.241840+0000","last_became_peered":"2024-09-14T04:54:52.241840+0000","last_unstale":"2024-09-14T04:59:09.697411+0000","last_undegraded":"2024-09-14T04:59:09.697411+0000","last_fullsized":"2024-09-14T04:59:09.697411+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:37.899666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":2401,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.069396+0000","last_change":"2024-09-14T04:44:57.344614+0000","last_active":"2024-09-14T04:59:27.069396+0000","last_peered":"2024-09-14T04:59:27.069396+0000","last_clean":"2024-09-14T04:59:27.069396+0000","last_became_active":"2024-09-14T04:44:57.332256+0000","last_became_peered":"2024-09-14T04:44:57.332256+0000","last_unstale":"2024-09-14T04:59:27.069396+0000","last_undegraded":"2024-09-14T04:59:27.069396+0000","last_fullsized":"2024-09-14T04:59:27.069396+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:23.438417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"349'149","reported_seq":912,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.695588+0000","last_change":"2024-09-14T04:45:00.331854+0000","last_active":"2024-09-14T04:59:09.695588+0000","last_peered":"2024-09-14T04:59:09.695588+0000","last_clean":"2024-09-14T04:59:09.695588+0000","last_became_active":"2024-09-14T04:45:00.331460+0000","last_became_peered":"2024-09-14T04:45:00.331460+0000","last_unstale":"2024-09-14T04:59:09.695588+0000","last_undegraded":"2024-09-14T04:59:09.695588+0000","last_fullsized":"2024-09-14T04:59:09.695588+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.567535+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.567535+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:45.095729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":743,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.701992+0000","last_change":"2024-09-14T04:49:29.616831+0000","last_active":"2024-09-14T04:59:09.701992+0000","last_peered":"2024-09-14T04:59:09.701992+0000","last_clean":"2024-09-14T04:59:09.701992+0000","last_became_active":"2024-09-14T04:49:29.616546+0000","last_became_peered":"2024-09-14T04:49:29.616546+0000","last_unstale":"2024-09-14T04:59:09.701992+0000","last_undegraded":"2024-09-14T04:59:09.701992+0000","last_fullsized":"2024-09-14T04:59:09.701992+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:27.882929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":356,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.058544+0000","last_change":"2024-09-14T04:44:57.330746+0000","last_active":"2024-09-14T04:59:27.058544+0000","last_peered":"2024-09-14T04:59:27.058544+0000","last_clean":"2024-09-14T04:59:27.058544+0000","last_became_active":"2024-09-14T04:44:57.330327+0000","last_became_peered":"2024-09-14T04:44:57.330327+0000","last_unstale":"2024-09-14T04:59:27.058544+0000","last_undegraded":"2024-09-14T04:59:27.058544+0000","last_fullsized":"2024-09-14T04:59:27.058544+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:53.122771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":194,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.705596+0000","last_change":"2024-09-14T04:43:13.355296+0000","last_active":"2024-09-14T04:59:09.705596+0000","last_peered":"2024-09-14T04:59:09.705596+0000","last_clean":"2024-09-14T04:59:09.705596+0000","last_became_active":"2024-09-14T04:43:13.354950+0000","last_became_peered":"2024-09-14T04:43:13.354950+0000","last_unstale":"2024-09-14T04:59:09.705596+0000","last_undegraded":"2024-09-14T04:59:09.705596+0000","last_fullsized":"2024-09-14T04:59:09.705596+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:20:08.968818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"349'149","reported_seq":851,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.697578+0000","last_change":"2024-09-14T04:41:39.471393+0000","last_active":"2024-09-14T04:59:09.697578+0000","last_peered":"2024-09-14T04:59:09.697578+0000","last_clean":"2024-09-14T04:59:09.697578+0000","last_became_active":"2024-09-14T04:41:36.931768+0000","last_became_peered":"2024-09-14T04:41:36.931768+0000","last_unstale":"2024-09-14T04:59:09.697578+0000","last_undegraded":"2024-09-14T04:59:09.697578+0000","last_fullsized":"2024-09-14T04:59:09.697578+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.471248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.471248+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:56.210375+0000","scrub_duration":0.021758344999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2357,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.054486+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T04:59:27.054486+0000","last_peered":"2024-09-14T04:59:27.054486+0000","last_clean":"2024-09-14T04:59:27.054486+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T04:59:27.054486+0000","last_undegraded":"2024-09-14T04:59:27.054486+0000","last_fullsized":"2024-09-14T04:59:27.054486+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1561,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.702860+0000","last_change":"2024-09-14T04:44:57.331436+0000","last_active":"2024-09-14T04:59:09.702860+0000","last_peered":"2024-09-14T04:59:09.702860+0000","last_clean":"2024-09-14T04:59:09.702860+0000","last_became_active":"2024-09-14T04:44:57.330957+0000","last_became_peered":"2024-09-14T04:44:57.330957+0000","last_unstale":"2024-09-14T04:59:09.702860+0000","last_undegraded":"2024-09-14T04:59:09.702860+0000","last_fullsized":"2024-09-14T04:59:09.702860+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:09.516688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":906,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.060632+0000","last_change":"2024-09-14T04:49:49.460401+0000","last_active":"2024-09-14T04:59:27.060632+0000","last_peered":"2024-09-14T04:59:27.060632+0000","last_clean":"2024-09-14T04:59:27.060632+0000","last_became_active":"2024-09-14T04:49:49.458530+0000","last_became_peered":"2024-09-14T04:49:49.458530+0000","last_unstale":"2024-09-14T04:59:27.060632+0000","last_undegraded":"2024-09-14T04:59:27.060632+0000","last_fullsized":"2024-09-14T04:59:27.060632+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:11:29.143239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"349'149","reported_seq":885,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.697169+0000","last_change":"2024-09-14T04:54:22.147441+0000","last_active":"2024-09-14T04:59:09.697169+0000","last_peered":"2024-09-14T04:59:09.697169+0000","last_clean":"2024-09-14T04:59:09.697169+0000","last_became_active":"2024-09-14T04:54:22.147115+0000","last_became_peered":"2024-09-14T04:54:22.147115+0000","last_unstale":"2024-09-14T04:59:09.697169+0000","last_undegraded":"2024-09-14T04:59:09.697169+0000","last_fullsized":"2024-09-14T04:59:09.697169+0000","mapping_epoch":1492,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1493,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:38.479887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:38.479887+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:41:14.001781+0000","scrub_duration":0.0062728319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":312,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.051954+0000","last_change":"2024-09-14T04:44:57.329983+0000","last_active":"2024-09-14T04:59:27.051954+0000","last_peered":"2024-09-14T04:59:27.051954+0000","last_clean":"2024-09-14T04:59:27.051954+0000","last_became_active":"2024-09-14T04:44:57.329597+0000","last_became_peered":"2024-09-14T04:44:57.329597+0000","last_unstale":"2024-09-14T04:59:27.051954+0000","last_undegraded":"2024-09-14T04:59:27.051954+0000","last_fullsized":"2024-09-14T04:59:27.051954+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:45.718039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":850,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.063065+0000","last_change":"2024-09-14T04:49:49.443631+0000","last_active":"2024-09-14T04:59:27.063065+0000","last_peered":"2024-09-14T04:59:27.063065+0000","last_clean":"2024-09-14T04:59:27.063065+0000","last_became_active":"2024-09-14T04:49:49.443313+0000","last_became_peered":"2024-09-14T04:49:49.443313+0000","last_unstale":"2024-09-14T04:59:27.063065+0000","last_undegraded":"2024-09-14T04:59:27.063065+0000","last_fullsized":"2024-09-14T04:59:27.063065+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:06:54.562292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1533,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.700090+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T04:59:09.700090+0000","last_peered":"2024-09-14T04:59:09.700090+0000","last_clean":"2024-09-14T04:59:09.700090+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T04:59:09.700090+0000","last_undegraded":"2024-09-14T04:59:09.700090+0000","last_fullsized":"2024-09-14T04:59:09.700090+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"349'149","reported_seq":843,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.701904+0000","last_change":"2024-09-14T04:41:39.012977+0000","last_active":"2024-09-14T04:59:09.701904+0000","last_peered":"2024-09-14T04:59:09.701904+0000","last_clean":"2024-09-14T04:59:09.701904+0000","last_became_active":"2024-09-14T04:41:36.932552+0000","last_became_peered":"2024-09-14T04:41:36.932552+0000","last_unstale":"2024-09-14T04:59:09.701904+0000","last_undegraded":"2024-09-14T04:59:09.701904+0000","last_fullsized":"2024-09-14T04:59:09.701904+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.012879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.012879+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:16:40.080262+0000","scrub_duration":0.0051721620000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":942,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.060730+0000","last_change":"2024-09-14T04:49:49.459590+0000","last_active":"2024-09-14T04:59:27.060730+0000","last_peered":"2024-09-14T04:59:27.060730+0000","last_clean":"2024-09-14T04:59:27.060730+0000","last_became_active":"2024-09-14T04:49:49.456263+0000","last_became_peered":"2024-09-14T04:49:49.456263+0000","last_unstale":"2024-09-14T04:59:27.060730+0000","last_undegraded":"2024-09-14T04:59:27.060730+0000","last_fullsized":"2024-09-14T04:59:27.060730+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:20:44.666805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":155,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.048686+0000","last_change":"2024-09-14T04:44:57.329652+0000","last_active":"2024-09-14T04:59:27.048686+0000","last_peered":"2024-09-14T04:59:27.048686+0000","last_clean":"2024-09-14T04:59:27.048686+0000","last_became_active":"2024-09-14T04:44:57.328888+0000","last_became_peered":"2024-09-14T04:44:57.328888+0000","last_unstale":"2024-09-14T04:59:27.048686+0000","last_undegraded":"2024-09-14T04:59:27.048686+0000","last_fullsized":"2024-09-14T04:59:27.048686+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:10.170068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1532,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.699706+0000","last_change":"2024-09-14T04:06:46.875340+0000","last_active":"2024-09-14T04:59:09.699706+0000","last_peered":"2024-09-14T04:59:09.699706+0000","last_clean":"2024-09-14T04:59:09.699706+0000","last_became_active":"2024-09-14T04:06:46.874552+0000","last_became_peered":"2024-09-14T04:06:46.874552+0000","last_unstale":"2024-09-14T04:59:09.699706+0000","last_undegraded":"2024-09-14T04:59:09.699706+0000","last_fullsized":"2024-09-14T04:59:09.699706+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:52.257357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"349'149","reported_seq":746,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.702458+0000","last_change":"2024-09-14T04:54:55.279666+0000","last_active":"2024-09-14T04:59:09.702458+0000","last_peered":"2024-09-14T04:59:09.702458+0000","last_clean":"2024-09-14T04:59:09.702458+0000","last_became_active":"2024-09-14T04:54:55.279259+0000","last_became_peered":"2024-09-14T04:54:55.279259+0000","last_unstale":"2024-09-14T04:59:09.702458+0000","last_undegraded":"2024-09-14T04:59:09.702458+0000","last_fullsized":"2024-09-14T04:59:09.702458+0000","mapping_epoch":1506,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1507,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:54:03.996025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:54:03.996025+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:40:53.332369+0000","scrub_duration":54,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":305,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.053317+0000","last_change":"2024-09-14T04:44:14.489235+0000","last_active":"2024-09-14T04:59:27.053317+0000","last_peered":"2024-09-14T04:59:27.053317+0000","last_clean":"2024-09-14T04:59:27.053317+0000","last_became_active":"2024-09-14T04:44:14.488950+0000","last_became_peered":"2024-09-14T04:44:14.488950+0000","last_unstale":"2024-09-14T04:59:27.053317+0000","last_undegraded":"2024-09-14T04:59:27.053317+0000","last_fullsized":"2024-09-14T04:59:27.053317+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:54.441144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":860,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.060955+0000","last_change":"2024-09-14T04:49:49.456317+0000","last_active":"2024-09-14T04:59:27.060955+0000","last_peered":"2024-09-14T04:59:27.060955+0000","last_clean":"2024-09-14T04:59:27.060955+0000","last_became_active":"2024-09-14T04:49:49.456029+0000","last_became_peered":"2024-09-14T04:49:49.456029+0000","last_unstale":"2024-09-14T04:59:27.060955+0000","last_undegraded":"2024-09-14T04:59:27.060955+0000","last_fullsized":"2024-09-14T04:59:27.060955+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:46:05.319676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2378,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.061212+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T04:59:27.061212+0000","last_peered":"2024-09-14T04:59:27.061212+0000","last_clean":"2024-09-14T04:59:27.061212+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T04:59:27.061212+0000","last_undegraded":"2024-09-14T04:59:27.061212+0000","last_fullsized":"2024-09-14T04:59:27.061212+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"349'149","reported_seq":737,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.697586+0000","last_change":"2024-09-14T04:50:22.002798+0000","last_active":"2024-09-14T04:59:09.697586+0000","last_peered":"2024-09-14T04:59:09.697586+0000","last_clean":"2024-09-14T04:59:09.697586+0000","last_became_active":"2024-09-14T04:49:49.448278+0000","last_became_peered":"2024-09-14T04:49:49.448278+0000","last_unstale":"2024-09-14T04:59:09.697586+0000","last_undegraded":"2024-09-14T04:59:09.697586+0000","last_fullsized":"2024-09-14T04:59:09.697586+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:50:22.002568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:50:22.002568+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:01:57.489223+0000","scrub_duration":0.010888326,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2378,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.068749+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T04:59:27.068749+0000","last_peered":"2024-09-14T04:59:27.068749+0000","last_clean":"2024-09-14T04:59:27.068749+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T04:59:27.068749+0000","last_undegraded":"2024-09-14T04:59:27.068749+0000","last_fullsized":"2024-09-14T04:59:27.068749+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2399,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.056266+0000","last_change":"2024-09-14T04:54:19.447103+0000","last_active":"2024-09-14T04:59:27.056266+0000","last_peered":"2024-09-14T04:59:27.056266+0000","last_clean":"2024-09-14T04:59:27.056266+0000","last_became_active":"2024-09-14T04:54:19.446748+0000","last_became_peered":"2024-09-14T04:54:19.446748+0000","last_unstale":"2024-09-14T04:59:27.056266+0000","last_undegraded":"2024-09-14T04:59:27.056266+0000","last_fullsized":"2024-09-14T04:59:27.056266+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:00:55.369263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1562,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.702144+0000","last_change":"2024-09-14T04:42:49.939790+0000","last_active":"2024-09-14T04:59:09.702144+0000","last_peered":"2024-09-14T04:59:09.702144+0000","last_clean":"2024-09-14T04:59:09.702144+0000","last_became_active":"2024-09-14T04:42:49.939591+0000","last_became_peered":"2024-09-14T04:42:49.939591+0000","last_unstale":"2024-09-14T04:59:09.702144+0000","last_undegraded":"2024-09-14T04:59:09.702144+0000","last_fullsized":"2024-09-14T04:59:09.702144+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:23:16.790593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1525,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.701818+0000","last_change":"2024-09-14T04:44:57.326008+0000","last_active":"2024-09-14T04:59:09.701818+0000","last_peered":"2024-09-14T04:59:09.701818+0000","last_clean":"2024-09-14T04:59:09.701818+0000","last_became_active":"2024-09-14T04:44:57.325617+0000","last_became_peered":"2024-09-14T04:44:57.325617+0000","last_unstale":"2024-09-14T04:59:09.701818+0000","last_undegraded":"2024-09-14T04:59:09.701818+0000","last_fullsized":"2024-09-14T04:59:09.701818+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:15.637508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":814,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.699927+0000","last_change":"2024-09-14T04:43:13.354846+0000","last_active":"2024-09-14T04:59:09.699927+0000","last_peered":"2024-09-14T04:59:09.699927+0000","last_clean":"2024-09-14T04:59:09.699927+0000","last_became_active":"2024-09-14T04:43:13.354515+0000","last_became_peered":"2024-09-14T04:43:13.354515+0000","last_unstale":"2024-09-14T04:59:09.699927+0000","last_undegraded":"2024-09-14T04:59:09.699927+0000","last_fullsized":"2024-09-14T04:59:09.699927+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:17.889341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1531,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.699663+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T04:59:09.699663+0000","last_peered":"2024-09-14T04:59:09.699663+0000","last_clean":"2024-09-14T04:59:09.699663+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T04:59:09.699663+0000","last_undegraded":"2024-09-14T04:59:09.699663+0000","last_fullsized":"2024-09-14T04:59:09.699663+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":1869,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.061672+0000","last_change":"2024-09-14T04:41:30.640651+0000","last_active":"2024-09-14T04:59:27.061672+0000","last_peered":"2024-09-14T04:59:27.061672+0000","last_clean":"2024-09-14T04:59:27.061672+0000","last_became_active":"2024-09-14T04:41:29.532466+0000","last_became_peered":"2024-09-14T04:41:29.532466+0000","last_unstale":"2024-09-14T04:59:27.061672+0000","last_undegraded":"2024-09-14T04:59:27.061672+0000","last_fullsized":"2024-09-14T04:59:27.061672+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":0,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:30.640520+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:30.640520+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:27.776724+0000","scrub_duration":0.0028653139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"349'149","reported_seq":843,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.699685+0000","last_change":"2024-09-14T04:41:39.978080+0000","last_active":"2024-09-14T04:59:09.699685+0000","last_peered":"2024-09-14T04:59:09.699685+0000","last_clean":"2024-09-14T04:59:09.699685+0000","last_became_active":"2024-09-14T04:41:36.929279+0000","last_became_peered":"2024-09-14T04:41:36.929279+0000","last_unstale":"2024-09-14T04:59:09.699685+0000","last_undegraded":"2024-09-14T04:59:09.699685+0000","last_fullsized":"2024-09-14T04:59:09.699685+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.977999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.977999+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:44:06.383827+0000","scrub_duration":0.00192122,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":238,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.062761+0000","last_change":"2024-09-14T04:48:11.040133+0000","last_active":"2024-09-14T04:59:27.062761+0000","last_peered":"2024-09-14T04:59:27.062761+0000","last_clean":"2024-09-14T04:59:27.062761+0000","last_became_active":"2024-09-14T04:48:11.039787+0000","last_became_peered":"2024-09-14T04:48:11.039787+0000","last_unstale":"2024-09-14T04:59:27.062761+0000","last_undegraded":"2024-09-14T04:59:27.062761+0000","last_fullsized":"2024-09-14T04:59:27.062761+0000","mapping_epoch":1428,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:53.520672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2225,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.052161+0000","last_change":"2024-09-14T04:49:29.616443+0000","last_active":"2024-09-14T04:59:27.052161+0000","last_peered":"2024-09-14T04:59:27.052161+0000","last_clean":"2024-09-14T04:59:27.052161+0000","last_became_active":"2024-09-14T04:49:29.616066+0000","last_became_peered":"2024-09-14T04:49:29.616066+0000","last_unstale":"2024-09-14T04:59:27.052161+0000","last_undegraded":"2024-09-14T04:59:27.052161+0000","last_fullsized":"2024-09-14T04:59:27.052161+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:09:15.897957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":154,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.695808+0000","last_change":"2024-09-14T04:44:57.326724+0000","last_active":"2024-09-14T04:59:09.695808+0000","last_peered":"2024-09-14T04:59:09.695808+0000","last_clean":"2024-09-14T04:59:09.695808+0000","last_became_active":"2024-09-14T04:44:57.326191+0000","last_became_peered":"2024-09-14T04:44:57.326191+0000","last_unstale":"2024-09-14T04:59:09.695808+0000","last_undegraded":"2024-09-14T04:59:09.695808+0000","last_fullsized":"2024-09-14T04:59:09.695808+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:46.060269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":154,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.694992+0000","last_change":"2024-09-14T04:44:57.328528+0000","last_active":"2024-09-14T04:59:09.694992+0000","last_peered":"2024-09-14T04:59:09.694992+0000","last_clean":"2024-09-14T04:59:09.694992+0000","last_became_active":"2024-09-14T04:44:57.327739+0000","last_became_peered":"2024-09-14T04:44:57.327739+0000","last_unstale":"2024-09-14T04:59:09.694992+0000","last_undegraded":"2024-09-14T04:59:09.694992+0000","last_fullsized":"2024-09-14T04:59:09.694992+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:04:58.294690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"349'149","reported_seq":851,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.699850+0000","last_change":"2024-09-14T04:55:59.732757+0000","last_active":"2024-09-14T04:59:09.699850+0000","last_peered":"2024-09-14T04:59:09.699850+0000","last_clean":"2024-09-14T04:59:09.699850+0000","last_became_active":"2024-09-14T04:55:59.732364+0000","last_became_peered":"2024-09-14T04:55:59.732364+0000","last_unstale":"2024-09-14T04:59:09.699850+0000","last_undegraded":"2024-09-14T04:59:09.699850+0000","last_fullsized":"2024-09-14T04:59:09.699850+0000","mapping_epoch":1512,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1513,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:56.738636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:56.738636+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:59:11.313765+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1313440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":154,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.695471+0000","last_change":"2024-09-14T04:44:57.323436+0000","last_active":"2024-09-14T04:59:09.695471+0000","last_peered":"2024-09-14T04:59:09.695471+0000","last_clean":"2024-09-14T04:59:09.695471+0000","last_became_active":"2024-09-14T04:44:57.323022+0000","last_became_peered":"2024-09-14T04:44:57.323022+0000","last_unstale":"2024-09-14T04:59:09.695471+0000","last_undegraded":"2024-09-14T04:59:09.695471+0000","last_fullsized":"2024-09-14T04:59:09.695471+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:22.775261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1560,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.696942+0000","last_change":"2024-09-14T04:44:57.322357+0000","last_active":"2024-09-14T04:59:09.696942+0000","last_peered":"2024-09-14T04:59:09.696942+0000","last_clean":"2024-09-14T04:59:09.696942+0000","last_became_active":"2024-09-14T04:44:57.321990+0000","last_became_peered":"2024-09-14T04:44:57.321990+0000","last_unstale":"2024-09-14T04:59:09.696942+0000","last_undegraded":"2024-09-14T04:59:09.696942+0000","last_fullsized":"2024-09-14T04:59:09.696942+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:09.658008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"349'149","reported_seq":1885,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.701883+0000","last_change":"2024-09-14T04:41:53.858076+0000","last_active":"2024-09-14T04:59:09.701883+0000","last_peered":"2024-09-14T04:59:09.701883+0000","last_clean":"2024-09-14T04:59:09.701883+0000","last_became_active":"2024-09-14T04:41:36.931585+0000","last_became_peered":"2024-09-14T04:41:36.931585+0000","last_unstale":"2024-09-14T04:59:09.701883+0000","last_undegraded":"2024-09-14T04:59:09.701883+0000","last_fullsized":"2024-09-14T04:59:09.701883+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:53.857924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:53.857924+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:10:20.658986+0000","scrub_duration":0.0022355919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":154,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.696353+0000","last_change":"2024-09-14T04:44:57.329198+0000","last_active":"2024-09-14T04:59:09.696353+0000","last_peered":"2024-09-14T04:59:09.696353+0000","last_clean":"2024-09-14T04:59:09.696353+0000","last_became_active":"2024-09-14T04:44:57.327754+0000","last_became_peered":"2024-09-14T04:44:57.327754+0000","last_unstale":"2024-09-14T04:59:09.696353+0000","last_undegraded":"2024-09-14T04:59:09.696353+0000","last_fullsized":"2024-09-14T04:59:09.696353+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:10:27.944476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":362,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.054567+0000","last_change":"2024-09-14T04:44:57.348687+0000","last_active":"2024-09-14T04:59:27.054567+0000","last_peered":"2024-09-14T04:59:27.054567+0000","last_clean":"2024-09-14T04:59:27.054567+0000","last_became_active":"2024-09-14T04:44:57.332845+0000","last_became_peered":"2024-09-14T04:44:57.332845+0000","last_unstale":"2024-09-14T04:59:27.054567+0000","last_undegraded":"2024-09-14T04:59:27.054567+0000","last_fullsized":"2024-09-14T04:59:27.054567+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:12:18.762737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2384,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.054343+0000","last_change":"2024-09-14T04:44:57.333468+0000","last_active":"2024-09-14T04:59:27.054343+0000","last_peered":"2024-09-14T04:59:27.054343+0000","last_clean":"2024-09-14T04:59:27.054343+0000","last_became_active":"2024-09-14T04:44:57.332626+0000","last_became_peered":"2024-09-14T04:44:57.332626+0000","last_unstale":"2024-09-14T04:59:27.054343+0000","last_undegraded":"2024-09-14T04:59:27.054343+0000","last_fullsized":"2024-09-14T04:59:27.054343+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:57:26.366438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1516,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.700111+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T04:59:09.700111+0000","last_peered":"2024-09-14T04:59:09.700111+0000","last_clean":"2024-09-14T04:59:09.700111+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T04:59:09.700111+0000","last_undegraded":"2024-09-14T04:59:09.700111+0000","last_fullsized":"2024-09-14T04:59:09.700111+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1515,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.696810+0000","last_change":"2024-09-14T04:06:46.869868+0000","last_active":"2024-09-14T04:59:09.696810+0000","last_peered":"2024-09-14T04:59:09.696810+0000","last_clean":"2024-09-14T04:59:09.696810+0000","last_became_active":"2024-09-14T04:06:46.869126+0000","last_became_peered":"2024-09-14T04:06:46.869126+0000","last_unstale":"2024-09-14T04:59:09.696810+0000","last_undegraded":"2024-09-14T04:59:09.696810+0000","last_fullsized":"2024-09-14T04:59:09.696810+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:41.150138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1499,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.702774+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T04:59:09.702774+0000","last_peered":"2024-09-14T04:59:09.702774+0000","last_clean":"2024-09-14T04:59:09.702774+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T04:59:09.702774+0000","last_undegraded":"2024-09-14T04:59:09.702774+0000","last_fullsized":"2024-09-14T04:59:09.702774+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":154,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.695242+0000","last_change":"2024-09-14T04:44:57.332112+0000","last_active":"2024-09-14T04:59:09.695242+0000","last_peered":"2024-09-14T04:59:09.695242+0000","last_clean":"2024-09-14T04:59:09.695242+0000","last_became_active":"2024-09-14T04:44:57.331575+0000","last_became_peered":"2024-09-14T04:44:57.331575+0000","last_unstale":"2024-09-14T04:59:09.695242+0000","last_undegraded":"2024-09-14T04:59:09.695242+0000","last_fullsized":"2024-09-14T04:59:09.695242+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:50:00.391283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1547,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.703140+0000","last_change":"2024-09-14T04:44:57.351460+0000","last_active":"2024-09-14T04:59:09.703140+0000","last_peered":"2024-09-14T04:59:09.703140+0000","last_clean":"2024-09-14T04:59:09.703140+0000","last_became_active":"2024-09-14T04:44:57.334662+0000","last_became_peered":"2024-09-14T04:44:57.334662+0000","last_unstale":"2024-09-14T04:59:09.703140+0000","last_undegraded":"2024-09-14T04:59:09.703140+0000","last_fullsized":"2024-09-14T04:59:09.703140+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:02.399062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1501,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.700028+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T04:59:09.700028+0000","last_peered":"2024-09-14T04:59:09.700028+0000","last_clean":"2024-09-14T04:59:09.700028+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T04:59:09.700028+0000","last_undegraded":"2024-09-14T04:59:09.700028+0000","last_fullsized":"2024-09-14T04:59:09.700028+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1559,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.705572+0000","last_change":"2024-09-14T04:44:57.349690+0000","last_active":"2024-09-14T04:59:09.705572+0000","last_peered":"2024-09-14T04:59:09.705572+0000","last_clean":"2024-09-14T04:59:09.705572+0000","last_became_active":"2024-09-14T04:44:57.333579+0000","last_became_peered":"2024-09-14T04:44:57.333579+0000","last_unstale":"2024-09-14T04:59:09.705572+0000","last_undegraded":"2024-09-14T04:59:09.705572+0000","last_fullsized":"2024-09-14T04:59:09.705572+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:17:55.265141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1636,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.696811+0000","last_change":"2024-09-14T04:54:35.694089+0000","last_active":"2024-09-14T04:59:09.696811+0000","last_peered":"2024-09-14T04:59:09.696811+0000","last_clean":"2024-09-14T04:59:09.696811+0000","last_became_active":"2024-09-14T04:54:35.663133+0000","last_became_peered":"2024-09-14T04:54:35.663133+0000","last_unstale":"2024-09-14T04:59:09.696811+0000","last_undegraded":"2024-09-14T04:59:09.696811+0000","last_fullsized":"2024-09-14T04:59:09.696811+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:17:19.947280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":48,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.705469+0000","last_change":"2024-09-14T04:54:52.248439+0000","last_active":"2024-09-14T04:59:09.705469+0000","last_peered":"2024-09-14T04:59:09.705469+0000","last_clean":"2024-09-14T04:59:09.705469+0000","last_became_active":"2024-09-14T04:54:52.248235+0000","last_became_peered":"2024-09-14T04:54:52.248235+0000","last_unstale":"2024-09-14T04:59:09.705469+0000","last_undegraded":"2024-09-14T04:59:09.705469+0000","last_fullsized":"2024-09-14T04:59:09.705469+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:22:09.548139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1521,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.702124+0000","last_change":"2024-09-14T04:44:57.332511+0000","last_active":"2024-09-14T04:59:09.702124+0000","last_peered":"2024-09-14T04:59:09.702124+0000","last_clean":"2024-09-14T04:59:09.702124+0000","last_became_active":"2024-09-14T04:44:57.332166+0000","last_became_peered":"2024-09-14T04:44:57.332166+0000","last_unstale":"2024-09-14T04:59:09.702124+0000","last_undegraded":"2024-09-14T04:59:09.702124+0000","last_fullsized":"2024-09-14T04:59:09.702124+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:53.984639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2378,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.062814+0000","last_change":"2024-09-14T04:54:52.250655+0000","last_active":"2024-09-14T04:59:27.062814+0000","last_peered":"2024-09-14T04:59:27.062814+0000","last_clean":"2024-09-14T04:59:27.062814+0000","last_became_active":"2024-09-14T04:54:52.250317+0000","last_became_peered":"2024-09-14T04:54:52.250317+0000","last_unstale":"2024-09-14T04:59:27.062814+0000","last_undegraded":"2024-09-14T04:59:27.062814+0000","last_fullsized":"2024-09-14T04:59:27.062814+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:12.985406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2363,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.052516+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T04:59:27.052516+0000","last_peered":"2024-09-14T04:59:27.052516+0000","last_clean":"2024-09-14T04:59:27.052516+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T04:59:27.052516+0000","last_undegraded":"2024-09-14T04:59:27.052516+0000","last_fullsized":"2024-09-14T04:59:27.052516+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1513,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.700085+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T04:59:09.700085+0000","last_peered":"2024-09-14T04:59:09.700085+0000","last_clean":"2024-09-14T04:59:09.700085+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T04:59:09.700085+0000","last_undegraded":"2024-09-14T04:59:09.700085+0000","last_fullsized":"2024-09-14T04:59:09.700085+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1550,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.702340+0000","last_change":"2024-09-14T04:44:57.339993+0000","last_active":"2024-09-14T04:59:09.702340+0000","last_peered":"2024-09-14T04:59:09.702340+0000","last_clean":"2024-09-14T04:59:09.702340+0000","last_became_active":"2024-09-14T04:44:57.330472+0000","last_became_peered":"2024-09-14T04:44:57.330472+0000","last_unstale":"2024-09-14T04:59:09.702340+0000","last_undegraded":"2024-09-14T04:59:09.702340+0000","last_fullsized":"2024-09-14T04:59:09.702340+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:19:27.046988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2098,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:17.748616+0000","last_change":"2024-09-14T04:49:49.449687+0000","last_active":"2024-09-14T04:59:17.748616+0000","last_peered":"2024-09-14T04:59:17.748616+0000","last_clean":"2024-09-14T04:59:17.748616+0000","last_became_active":"2024-09-14T04:49:49.449144+0000","last_became_peered":"2024-09-14T04:49:49.449144+0000","last_unstale":"2024-09-14T04:59:17.748616+0000","last_undegraded":"2024-09-14T04:59:17.748616+0000","last_fullsized":"2024-09-14T04:59:17.748616+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:59:06.779730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":57,"num_pg":16,"stat_sum":{"num_bytes":4553179160,"num_objects":69477,"num_object_clones":0,"num_object_copies":208431,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":69477,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":178023,"num_write_kb":7919937,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13566750720,"data_stored":13566738504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":138323,"ondisk_log_size":138323,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1313440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":5070,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4250,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223202103,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2178356,"kb_used_data":1375340,"kb_used_omap":1,"kb_used_meta":803006,"kb_avail":91566796,"statfs":{"total":95995035648,"available":93764399104,"internally_reserved":0,"allocated":1408348160,"data_stored":1402912502,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":822278603},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463739,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3313520,"kb_used_data":2498216,"kb_used_omap":1,"kb_used_meta":815294,"kb_avail":90431632,"statfs":{"total":95995035648,"available":92601991168,"internally_reserved":0,"allocated":2558173184,"data_stored":2552741519,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":834861511},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725376,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2477928,"kb_used_data":1630624,"kb_used_omap":1,"kb_used_meta":847294,"kb_avail":91267224,"statfs":{"total":95995035648,"available":93457637376,"internally_reserved":0,"allocated":1669758976,"data_stored":1664331849,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":867629510},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":4,"up_from":1392,"seq":5978594476222,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1503476,"kb_used_data":1105644,"kb_used_omap":117,"kb_used_meta":397706,"kb_avail":92241676,"statfs":{"total":95995035648,"available":94455476224,"internally_reserved":0,"allocated":1132179456,"data_stored":1126744350,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120638,"internal_metadata":407251138},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772075117,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3074920,"kb_used_data":2212640,"kb_used_omap":71,"kb_used_meta":862200,"kb_avail":90670232,"statfs":{"total":95995035648,"available":92846317568,"internally_reserved":0,"allocated":2265743360,"data_stored":2260316310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":882893750},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707304052,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2907812,"kb_used_data":2302364,"kb_used_omap":68,"kb_used_meta":605371,"kb_avail":90837340,"statfs":{"total":95995035648,"available":93017436160,"internally_reserved":0,"allocated":2357620736,"data_stored":2352197662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":619900112},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872729211,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1199644,"kb_used_data":581780,"kb_used_omap":67,"kb_used_meta":617788,"kb_avail":92545508,"statfs":{"total":95995035648,"available":94766600192,"internally_reserved":0,"allocated":595742720,"data_stored":590326697,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":632615269},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056261,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2465384,"kb_used_data":1739228,"kb_used_omap":74,"kb_used_meta":725877,"kb_avail":91279768,"statfs":{"total":95995035648,"available":93470482432,"internally_reserved":0,"allocated":1780969472,"data_stored":1775545866,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":743298741},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1901,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1742278656,"data_stored":1742274584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":582942720,"data_stored":582942720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2318729216,"data_stored":2318729216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2247888896,"data_stored":2247884824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1112932352,"data_stored":1112932352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1651769344,"data_stored":1651769344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2527399936,"data_stored":2527395864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1382809600,"data_stored":1382809600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:59:36.427 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-14T04:59:36.427 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T04:59:36.428 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2024-09-14T04:59:36.428 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T04:59:36.470 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:36.494 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:36.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:36.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:36.595 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:36.634 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:36.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:36.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:36.736 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:36.766 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:36.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:36.767 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:36.867 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:36.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:36.909 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:36.909 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:37.010 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:37.048 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:37.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:37.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:37.150 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:37.187 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:37.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:37.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:37.289 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:37.327 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:37.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:37.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:37.429 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:37.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:37.467 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:37.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:37.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:37 smithi129 ceph-mon[101316]: pgmap v3009: 99 pgs: 99 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 862 op/s 2024-09-14T04:59:37.539 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:37 smithi039 ceph-mon[106470]: pgmap v3009: 99 pgs: 99 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 862 op/s 2024-09-14T04:59:37.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:37 smithi039 ceph-mon[111235]: pgmap v3009: 99 pgs: 99 active+clean; 4.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 862 op/s 2024-09-14T04:59:37.568 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:37.609 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:37.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:37.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:37.711 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:37.751 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:37.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:37.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:37.853 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:37.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:37.891 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:37.891 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:37.992 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:38.028 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:38.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:38.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:38.130 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:38.163 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:38.164 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:38.164 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:38.265 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:38.290 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:38.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:38.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:38.392 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:38.444 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:38.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:38.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:38.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:38 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:59:38.528 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:38 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:59:38.528 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:38 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:59:38.546 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:38.575 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:38.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:38.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:38.677 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:38.705 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:38.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:38.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:38.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:59:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:59:38] "GET /metrics HTTP/1.1" 200 36733 "" "Prometheus/2.51.0" 2024-09-14T04:59:38.807 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:38.838 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:38.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:38.839 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:38.940 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:38.984 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:38.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:38.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:39.086 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:39.118 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:39.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:39.119 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:39.220 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:39.253 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:39.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:39.254 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:39.269 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:59:39.289 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:39 smithi039 ceph-mon[106470]: pgmap v3010: 99 pgs: 99 active+clean; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 938 op/s 2024-09-14T04:59:39.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:39 smithi039 ceph-mon[111235]: pgmap v3010: 99 pgs: 99 active+clean; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 938 op/s 2024-09-14T04:59:39.354 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:39.388 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:39.389 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:39.389 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:39.490 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:39.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:39 smithi129 ceph-mon[101316]: pgmap v3010: 99 pgs: 99 active+clean; 4.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 938 op/s 2024-09-14T04:59:39.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:39.528 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:39.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:39.629 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:39.663 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:39.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:39.664 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:39.765 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:39.797 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:39.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:39.799 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:39.899 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:39.930 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:39.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:39.931 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:39.961 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-14T04:59:40.031 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:40.086 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:40.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:40.087 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:40.187 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:40.233 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:40.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:40.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:40.336 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:40.381 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:40.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:40.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:40.483 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:40.511 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:40.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:40.512 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:40.613 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:40.657 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:40.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:40.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:40.759 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:40.802 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:40.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:40.803 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:40.904 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:40.958 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:40.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:40.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:41.060 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:41.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:41.102 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:41.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:41.203 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:41.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:41.243 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:41.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:41.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:41 smithi039 ceph-mon[106470]: pgmap v3011: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 840 op/s 2024-09-14T04:59:41.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:41 smithi039 ceph-mon[111235]: pgmap v3011: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 840 op/s 2024-09-14T04:59:41.344 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:41.382 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:41.383 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:41.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:41.484 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:41.488 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:59:41.515 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:41.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:41.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:41.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:41 smithi129 ceph-mon[101316]: pgmap v3011: 99 pgs: 99 active+clean; 3.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 840 op/s 2024-09-14T04:59:41.617 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:41.646 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:41.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:41.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:41.748 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:41.780 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:41.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:41.781 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:41.881 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:41.910 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:41.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:41.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:42.012 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:42.065 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:42.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:42.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:42.167 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:42.199 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:42.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:42.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:42.301 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:42.322 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T04:59:42.341 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:42.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:42.342 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:42.443 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:42.522 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:42.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:42.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:42.624 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:42.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:42.649 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:42.650 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:42.750 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:42.764 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:59:42.777 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:42.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:42.778 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:42.879 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:42.888 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:42 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:59:42.888 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:42 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:59:42.888 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:42 smithi039 ceph-mon[111235]: pgmap v3012: 99 pgs: 99 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 844 op/s 2024-09-14T04:59:42.888 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:42 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:59:42.888 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:42 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:59:42.888 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:42 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:59:42.890 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:42 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:59:42.890 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:42 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:59:42.890 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:42 smithi039 ceph-mon[106470]: pgmap v3012: 99 pgs: 99 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 844 op/s 2024-09-14T04:59:42.890 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:42 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:59:42.890 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:42 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:59:42.891 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:42 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:59:42.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:42.929 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:42.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:43.030 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:43.062 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:43.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:43.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:43.164 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:43.202 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:43.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:43.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:42 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:59:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:42 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:59:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:42 smithi129 ceph-mon[101316]: pgmap v3012: 99 pgs: 99 active+clean; 3.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 844 op/s 2024-09-14T04:59:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:42 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T04:59:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:42 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T04:59:43.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:42 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:59:43.304 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:43.343 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:43.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:43.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:43.445 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:43.489 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:43.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:43.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:43.591 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:43.630 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:43.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:43.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:43.732 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:43.765 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:43.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:43.766 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:43.867 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:43.904 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:43.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:43.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:44.006 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:44.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:44.043 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:44.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:44.144 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:44.179 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:44.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:44.180 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:44.280 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:44.309 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:44.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:44.310 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:44.411 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:44.452 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:44.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:44.453 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:44.554 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:44.586 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:44.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:44.587 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:44.688 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:44.730 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:44.731 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:44.731 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:44.832 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:44.861 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:44.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:44.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:44.963 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:44.999 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:44.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:45.000 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:45.101 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:45.128 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:45.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:45.129 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:45.230 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:45.258 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:45.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:45.259 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:45.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:45 smithi039 ceph-mon[106470]: pgmap v3013: 99 pgs: 99 active+clean; 3.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2024-09-14T04:59:45.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:45 smithi039 ceph-mon[111235]: pgmap v3013: 99 pgs: 99 active+clean; 3.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2024-09-14T04:59:45.360 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:45.396 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:45.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:45.397 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:45.437 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:59:45.498 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:45.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:45 smithi129 ceph-mon[101316]: pgmap v3013: 99 pgs: 99 active+clean; 3.7 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 886 op/s 2024-09-14T04:59:45.528 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:45.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:45.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:45.630 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:45.664 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:45.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:45.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:45.766 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:45.801 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:45.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:45.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:45.902 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:45.944 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:45.945 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:45.946 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:46.046 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:46.073 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is unset 2024-09-14T04:59:46.098 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:46.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:46.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:46.200 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:46.227 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:46.228 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:46.228 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:46.280 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:46 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2898695957' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:59:46.280 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:46 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:59:46.280 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:46 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2898695957' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:59:46.281 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:46 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:59:46.329 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:46.358 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:46.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:46.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:46.461 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:46.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:46.497 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:46.497 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:46.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:46 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2898695957' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:59:46.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:46 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T04:59:46.598 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:46.629 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:46.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:46.630 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:46.730 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:46.773 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:46.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:46.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:46.875 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:46.913 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:46.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:46.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:47.015 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:47.059 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:47.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:47.061 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:47.161 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:47.184 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:47.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:47.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:47.252 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T04:59:47.257 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T04:59:47.258 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:47 smithi039 ceph-mon[106470]: pgmap v3014: 99 pgs: 99 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 935 op/s 2024-09-14T04:59:47.258 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:47 smithi039 ceph-mon[106470]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:59:47.258 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:47 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T04:59:47.258 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:47 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:59:47.259 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:47 smithi039 ceph-mon[106470]: osdmap e1541: 8 total, 8 up, 8 in 2024-09-14T04:59:47.259 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:47 smithi039 ceph-mon[111235]: pgmap v3014: 99 pgs: 99 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 935 op/s 2024-09-14T04:59:47.259 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:47 smithi039 ceph-mon[111235]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:59:47.259 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:47 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T04:59:47.259 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:47 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:59:47.259 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:47 smithi039 ceph-mon[111235]: osdmap e1541: 8 total, 8 up, 8 in 2024-09-14T04:59:47.286 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:47.339 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:47.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:47.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:47.441 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:47.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:47.477 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:47.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:47.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:47 smithi129 ceph-mon[101316]: pgmap v3014: 99 pgs: 99 active+clean; 3.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 935 op/s 2024-09-14T04:59:47.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:47 smithi129 ceph-mon[101316]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T04:59:47.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:47 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T04:59:47.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:47 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T04:59:47.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:47 smithi129 ceph-mon[101316]: osdmap e1541: 8 total, 8 up, 8 in 2024-09-14T04:59:47.578 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:47.604 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:47.604 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:47.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:47.705 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:47.742 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:47.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:47.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:47.843 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:47.882 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:47.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:47.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:47.984 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:48.012 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:48.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:48.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:48.094 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":3015,"stamp":"2024-09-14T04:59:46.061712+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3817805821,"num_objects":58273,"num_object_clones":0,"num_object_copies":174819,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":58273,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":189317,"num_write_kb":7919970,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":53,"num_bytes_recovered":1318510,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":140251,"ondisk_log_size":140251,"up":297,"acting":297,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":298,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":17197260,"kb_used_data":11455756,"kb_used_omap":404,"kb_used_meta":5740907,"kb_avail":732763956,"statfs":{"total":767960285184,"available":750350290944,"internally_reserved":0,"allocated":11730694144,"data_stored":11687274835,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":414022,"internal_metadata":5878689466},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[7,0,4,9,11],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-575012864,"num_objects":-8774,"num_object_clones":0,"num_object_copies":-26322,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8774,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8774,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.095206"},"pg_stats":[{"pgid":"57.f","version":"1540'11907","reported_seq":11963,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:45.393112+0000","last_change":"2024-09-14T04:56:49.435182+0000","last_active":"2024-09-14T04:59:45.393112+0000","last_peered":"2024-09-14T04:59:45.393112+0000","last_clean":"2024-09-14T04:59:45.393112+0000","last_became_active":"2024-09-14T04:56:49.434949+0000","last_became_peered":"2024-09-14T04:56:49.434949+0000","last_unstale":"2024-09-14T04:59:45.393112+0000","last_undegraded":"2024-09-14T04:59:45.393112+0000","last_fullsized":"2024-09-14T04:59:45.393112+0000","mapping_epoch":1518,"log_start":"1523'2500","ondisk_log_start":"1523'2500","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":9407,"log_dups_size":0,"ondisk_log_size":9407,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:32:29.856162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233111552,"num_objects":3557,"num_object_clones":0,"num_object_copies":10671,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3557,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11907,"num_write_kb":494848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"57.e","version":"1540'11611","reported_seq":11667,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:44.516792+0000","last_change":"2024-09-14T04:56:49.431930+0000","last_active":"2024-09-14T04:59:44.516792+0000","last_peered":"2024-09-14T04:59:44.516792+0000","last_clean":"2024-09-14T04:59:44.516792+0000","last_became_active":"2024-09-14T04:56:49.431642+0000","last_became_peered":"2024-09-14T04:56:49.431642+0000","last_unstale":"2024-09-14T04:59:44.516792+0000","last_undegraded":"2024-09-14T04:59:44.516792+0000","last_fullsized":"2024-09-14T04:59:44.516792+0000","mapping_epoch":1518,"log_start":"1523'3000","ondisk_log_start":"1523'3000","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":8611,"log_dups_size":0,"ondisk_log_size":8611,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:30:17.677827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236257280,"num_objects":3605,"num_object_clones":0,"num_object_copies":10815,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3605,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11611,"num_write_kb":486912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"57.d","version":"1540'11767","reported_seq":11800,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:42.856076+0000","last_change":"2024-09-14T04:56:49.821152+0000","last_active":"2024-09-14T04:59:42.856076+0000","last_peered":"2024-09-14T04:59:42.856076+0000","last_clean":"2024-09-14T04:59:42.856076+0000","last_became_active":"2024-09-14T04:56:49.820853+0000","last_became_peered":"2024-09-14T04:56:49.820853+0000","last_unstale":"2024-09-14T04:59:42.856076+0000","last_undegraded":"2024-09-14T04:59:42.856076+0000","last_fullsized":"2024-09-14T04:59:42.856076+0000","mapping_epoch":1518,"log_start":"1526'4400","ondisk_log_start":"1526'4400","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":7367,"log_dups_size":0,"ondisk_log_size":7367,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:41.229640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247201816,"num_objects":3773,"num_object_clones":0,"num_object_copies":11319,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3773,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11767,"num_write_kb":497217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"57.c","version":"1540'11887","reported_seq":11919,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:42.841711+0000","last_change":"2024-09-14T04:56:49.435207+0000","last_active":"2024-09-14T04:59:42.841711+0000","last_peered":"2024-09-14T04:59:42.841711+0000","last_clean":"2024-09-14T04:59:42.841711+0000","last_became_active":"2024-09-14T04:56:49.434752+0000","last_became_peered":"2024-09-14T04:56:49.434752+0000","last_unstale":"2024-09-14T04:59:42.841711+0000","last_undegraded":"2024-09-14T04:59:42.841711+0000","last_fullsized":"2024-09-14T04:59:42.841711+0000","mapping_epoch":1518,"log_start":"1526'4500","ondisk_log_start":"1526'4500","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":7387,"log_dups_size":0,"ondisk_log_size":7387,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:29:43.013481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251592704,"num_objects":3839,"num_object_clones":0,"num_object_copies":11517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3839,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11887,"num_write_kb":503232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"57.b","version":"1540'11923","reported_seq":11979,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:45.333315+0000","last_change":"2024-09-14T04:56:49.434067+0000","last_active":"2024-09-14T04:59:45.333315+0000","last_peered":"2024-09-14T04:59:45.333315+0000","last_clean":"2024-09-14T04:59:45.333315+0000","last_became_active":"2024-09-14T04:56:49.433725+0000","last_became_peered":"2024-09-14T04:56:49.433725+0000","last_unstale":"2024-09-14T04:59:45.333315+0000","last_undegraded":"2024-09-14T04:59:45.333315+0000","last_fullsized":"2024-09-14T04:59:45.333315+0000","mapping_epoch":1518,"log_start":"1523'2500","ondisk_log_start":"1523'2500","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":9423,"log_dups_size":0,"ondisk_log_size":9423,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:18:56.597341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234553344,"num_objects":3579,"num_object_clones":0,"num_object_copies":10737,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3579,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11923,"num_write_kb":496064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"57.a","version":"1540'11964","reported_seq":11996,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:42.738661+0000","last_change":"2024-09-14T04:56:49.818406+0000","last_active":"2024-09-14T04:59:42.738661+0000","last_peered":"2024-09-14T04:59:42.738661+0000","last_clean":"2024-09-14T04:59:42.738661+0000","last_became_active":"2024-09-14T04:56:49.818087+0000","last_became_peered":"2024-09-14T04:56:49.818087+0000","last_unstale":"2024-09-14T04:59:42.738661+0000","last_undegraded":"2024-09-14T04:59:42.738661+0000","last_fullsized":"2024-09-14T04:59:42.738661+0000","mapping_epoch":1518,"log_start":"1523'2800","ondisk_log_start":"1523'2800","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":9164,"log_dups_size":0,"ondisk_log_size":9164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:41:19.568727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240648192,"num_objects":3672,"num_object_clones":0,"num_object_copies":11016,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3672,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11964,"num_write_kb":500352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"57.9","version":"1540'11701","reported_seq":11733,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:43.424796+0000","last_change":"2024-09-14T04:56:49.818923+0000","last_active":"2024-09-14T04:59:43.424796+0000","last_peered":"2024-09-14T04:59:43.424796+0000","last_clean":"2024-09-14T04:59:43.424796+0000","last_became_active":"2024-09-14T04:56:49.818519+0000","last_became_peered":"2024-09-14T04:56:49.818519+0000","last_unstale":"2024-09-14T04:59:43.424796+0000","last_undegraded":"2024-09-14T04:59:43.424796+0000","last_fullsized":"2024-09-14T04:59:43.424796+0000","mapping_epoch":1518,"log_start":"1525'4200","ondisk_log_start":"1525'4200","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":7501,"log_dups_size":0,"ondisk_log_size":7501,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:41:08.309897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239796224,"num_objects":3659,"num_object_clones":0,"num_object_copies":10977,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3659,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11701,"num_write_kb":491520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"57.8","version":"1540'11718","reported_seq":11750,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:42.724205+0000","last_change":"2024-09-14T04:56:49.434525+0000","last_active":"2024-09-14T04:59:42.724205+0000","last_peered":"2024-09-14T04:59:42.724205+0000","last_clean":"2024-09-14T04:59:42.724205+0000","last_became_active":"2024-09-14T04:56:49.434144+0000","last_became_peered":"2024-09-14T04:56:49.434144+0000","last_unstale":"2024-09-14T04:59:42.724205+0000","last_undegraded":"2024-09-14T04:59:42.724205+0000","last_fullsized":"2024-09-14T04:59:42.724205+0000","mapping_epoch":1518,"log_start":"1524'4000","ondisk_log_start":"1524'4000","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":7718,"log_dups_size":0,"ondisk_log_size":7718,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:17:18.918905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233701376,"num_objects":3566,"num_object_clones":0,"num_object_copies":10698,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3566,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11718,"num_write_kb":489088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"57.7","version":"1540'11909","reported_seq":11965,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:45.385948+0000","last_change":"2024-09-14T04:56:49.434581+0000","last_active":"2024-09-14T04:59:45.385948+0000","last_peered":"2024-09-14T04:59:45.385948+0000","last_clean":"2024-09-14T04:59:45.385948+0000","last_became_active":"2024-09-14T04:56:49.434210+0000","last_became_peered":"2024-09-14T04:56:49.434210+0000","last_unstale":"2024-09-14T04:59:45.385948+0000","last_undegraded":"2024-09-14T04:59:45.385948+0000","last_fullsized":"2024-09-14T04:59:45.385948+0000","mapping_epoch":1518,"log_start":"1523'2500","ondisk_log_start":"1523'2500","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":9409,"log_dups_size":0,"ondisk_log_size":9409,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:16.426758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236388352,"num_objects":3607,"num_object_clones":0,"num_object_copies":10821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11909,"num_write_kb":496512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"57.6","version":"1540'11828","reported_seq":11859,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:42.767996+0000","last_change":"2024-09-14T04:56:49.436357+0000","last_active":"2024-09-14T04:59:42.767996+0000","last_peered":"2024-09-14T04:59:42.767996+0000","last_clean":"2024-09-14T04:59:42.767996+0000","last_became_active":"2024-09-14T04:56:49.436101+0000","last_became_peered":"2024-09-14T04:56:49.436101+0000","last_unstale":"2024-09-14T04:59:42.767996+0000","last_undegraded":"2024-09-14T04:59:42.767996+0000","last_fullsized":"2024-09-14T04:59:42.767996+0000","mapping_epoch":1518,"log_start":"1523'2700","ondisk_log_start":"1523'2700","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":9128,"log_dups_size":0,"ondisk_log_size":9128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:03:53.232843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242745344,"num_objects":3704,"num_object_clones":0,"num_object_copies":11112,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3704,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11828,"num_write_kb":497024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"57.5","version":"1540'11711","reported_seq":11743,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:42.765896+0000","last_change":"2024-09-14T04:56:49.821285+0000","last_active":"2024-09-14T04:59:42.765896+0000","last_peered":"2024-09-14T04:59:42.765896+0000","last_clean":"2024-09-14T04:59:42.765896+0000","last_became_active":"2024-09-14T04:56:49.820869+0000","last_became_peered":"2024-09-14T04:56:49.820869+0000","last_unstale":"2024-09-14T04:59:42.765896+0000","last_undegraded":"2024-09-14T04:59:42.765896+0000","last_fullsized":"2024-09-14T04:59:42.765896+0000","mapping_epoch":1518,"log_start":"1523'2600","ondisk_log_start":"1523'2600","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":9111,"log_dups_size":0,"ondisk_log_size":9111,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:26:39.392722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244908032,"num_objects":3737,"num_object_clones":0,"num_object_copies":11211,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3737,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11711,"num_write_kb":494336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"57.4","version":"1540'11750","reported_seq":11806,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:44.141806+0000","last_change":"2024-09-14T04:56:49.820010+0000","last_active":"2024-09-14T04:59:44.141806+0000","last_peered":"2024-09-14T04:59:44.141806+0000","last_clean":"2024-09-14T04:59:44.141806+0000","last_became_active":"2024-09-14T04:56:49.819563+0000","last_became_peered":"2024-09-14T04:56:49.819563+0000","last_unstale":"2024-09-14T04:59:44.141806+0000","last_undegraded":"2024-09-14T04:59:44.141806+0000","last_fullsized":"2024-09-14T04:59:44.141806+0000","mapping_epoch":1518,"log_start":"1524'3300","ondisk_log_start":"1524'3300","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":8450,"log_dups_size":0,"ondisk_log_size":8450,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:32:31.141098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229507072,"num_objects":3502,"num_object_clones":0,"num_object_copies":10506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3502,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11750,"num_write_kb":488064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"57.3","version":"1540'11844","reported_seq":11900,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:45.361301+0000","last_change":"2024-09-14T04:56:49.819031+0000","last_active":"2024-09-14T04:59:45.361301+0000","last_peered":"2024-09-14T04:59:45.361301+0000","last_clean":"2024-09-14T04:59:45.361301+0000","last_became_active":"2024-09-14T04:56:49.818654+0000","last_became_peered":"2024-09-14T04:56:49.818654+0000","last_unstale":"2024-09-14T04:59:45.361301+0000","last_undegraded":"2024-09-14T04:59:45.361301+0000","last_fullsized":"2024-09-14T04:59:45.361301+0000","mapping_epoch":1518,"log_start":"1523'2400","ondisk_log_start":"1523'2400","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":9444,"log_dups_size":0,"ondisk_log_size":9444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:57:37.054801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236847104,"num_objects":3614,"num_object_clones":0,"num_object_copies":10842,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3614,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11844,"num_write_kb":494656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"57.2","version":"1540'11786","reported_seq":11842,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:44.069438+0000","last_change":"2024-09-14T04:56:49.819908+0000","last_active":"2024-09-14T04:59:44.069438+0000","last_peered":"2024-09-14T04:59:44.069438+0000","last_clean":"2024-09-14T04:59:44.069438+0000","last_became_active":"2024-09-14T04:56:49.819599+0000","last_became_peered":"2024-09-14T04:56:49.819599+0000","last_unstale":"2024-09-14T04:59:44.069438+0000","last_undegraded":"2024-09-14T04:59:44.069438+0000","last_fullsized":"2024-09-14T04:59:44.069438+0000","mapping_epoch":1518,"log_start":"1524'3300","ondisk_log_start":"1524'3300","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":8486,"log_dups_size":0,"ondisk_log_size":8486,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:38:49.493483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":239861760,"num_objects":3660,"num_object_clones":0,"num_object_copies":10980,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3660,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11786,"num_write_kb":494272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"57.1","version":"1540'12092","reported_seq":12148,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:44.091129+0000","last_change":"2024-09-14T04:56:49.434507+0000","last_active":"2024-09-14T04:59:44.091129+0000","last_peered":"2024-09-14T04:59:44.091129+0000","last_clean":"2024-09-14T04:59:44.091129+0000","last_became_active":"2024-09-14T04:56:49.434172+0000","last_became_peered":"2024-09-14T04:56:49.434172+0000","last_unstale":"2024-09-14T04:59:44.091129+0000","last_undegraded":"2024-09-14T04:59:44.091129+0000","last_fullsized":"2024-09-14T04:59:44.091129+0000","mapping_epoch":1518,"log_start":"1524'3600","ondisk_log_start":"1524'3600","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":8492,"log_dups_size":0,"ondisk_log_size":8492,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:22.424289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":238288896,"num_objects":3636,"num_object_clones":0,"num_object_copies":10908,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3636,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12092,"num_write_kb":503296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"57.0","version":"1540'11855","reported_seq":11911,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:45.333634+0000","last_change":"2024-09-14T04:56:49.818610+0000","last_active":"2024-09-14T04:59:45.333634+0000","last_peered":"2024-09-14T04:59:45.333634+0000","last_clean":"2024-09-14T04:59:45.333634+0000","last_became_active":"2024-09-14T04:56:49.818300+0000","last_became_peered":"2024-09-14T04:56:49.818300+0000","last_unstale":"2024-09-14T04:59:45.333634+0000","last_undegraded":"2024-09-14T04:59:45.333634+0000","last_fullsized":"2024-09-14T04:59:45.333634+0000","mapping_epoch":1518,"log_start":"1523'2400","ondisk_log_start":"1523'2400","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":9455,"log_dups_size":0,"ondisk_log_size":9455,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:29:20.641683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231800832,"num_objects":3537,"num_object_clones":0,"num_object_copies":10611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11855,"num_write_kb":492544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":2374,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.053764+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T04:59:27.053764+0000","last_peered":"2024-09-14T04:59:27.053764+0000","last_clean":"2024-09-14T04:59:27.053764+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T04:59:27.053764+0000","last_undegraded":"2024-09-14T04:59:27.053764+0000","last_fullsized":"2024-09-14T04:59:27.053764+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":154,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.043864+0000","last_change":"2024-09-14T04:44:57.325109+0000","last_active":"2024-09-14T04:59:27.043864+0000","last_peered":"2024-09-14T04:59:27.043864+0000","last_clean":"2024-09-14T04:59:27.043864+0000","last_became_active":"2024-09-14T04:44:57.324791+0000","last_became_peered":"2024-09-14T04:44:57.324791+0000","last_unstale":"2024-09-14T04:59:27.043864+0000","last_undegraded":"2024-09-14T04:59:27.043864+0000","last_fullsized":"2024-09-14T04:59:27.043864+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:03.593275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":124,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.050011+0000","last_change":"2024-09-14T04:49:29.628435+0000","last_active":"2024-09-14T04:59:27.050011+0000","last_peered":"2024-09-14T04:59:27.050011+0000","last_clean":"2024-09-14T04:59:27.050011+0000","last_became_active":"2024-09-14T04:49:29.628081+0000","last_became_peered":"2024-09-14T04:49:29.628081+0000","last_unstale":"2024-09-14T04:59:27.050011+0000","last_undegraded":"2024-09-14T04:59:27.050011+0000","last_fullsized":"2024-09-14T04:59:27.050011+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:01.520334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":995,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.061017+0000","last_change":"2024-09-14T04:43:13.355281+0000","last_active":"2024-09-14T04:59:27.061017+0000","last_peered":"2024-09-14T04:59:27.061017+0000","last_clean":"2024-09-14T04:59:27.061017+0000","last_became_active":"2024-09-14T04:43:13.354967+0000","last_became_peered":"2024-09-14T04:43:13.354967+0000","last_unstale":"2024-09-14T04:59:27.061017+0000","last_undegraded":"2024-09-14T04:59:27.061017+0000","last_fullsized":"2024-09-14T04:59:27.061017+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:23:19.332075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1532,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.700016+0000","last_change":"2024-09-14T04:44:57.342918+0000","last_active":"2024-09-14T04:59:09.700016+0000","last_peered":"2024-09-14T04:59:09.700016+0000","last_clean":"2024-09-14T04:59:09.700016+0000","last_became_active":"2024-09-14T04:44:57.329550+0000","last_became_peered":"2024-09-14T04:44:57.329550+0000","last_unstale":"2024-09-14T04:59:09.700016+0000","last_undegraded":"2024-09-14T04:59:09.700016+0000","last_fullsized":"2024-09-14T04:59:09.700016+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:52:52.435893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":153,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.694891+0000","last_change":"2024-09-14T04:44:57.325537+0000","last_active":"2024-09-14T04:59:09.694891+0000","last_peered":"2024-09-14T04:59:09.694891+0000","last_clean":"2024-09-14T04:59:09.694891+0000","last_became_active":"2024-09-14T04:44:57.325057+0000","last_became_peered":"2024-09-14T04:44:57.325057+0000","last_unstale":"2024-09-14T04:59:09.694891+0000","last_undegraded":"2024-09-14T04:59:09.694891+0000","last_fullsized":"2024-09-14T04:59:09.694891+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:43:23.166708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1547,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.700039+0000","last_change":"2024-09-14T04:44:57.337033+0000","last_active":"2024-09-14T04:59:09.700039+0000","last_peered":"2024-09-14T04:59:09.700039+0000","last_clean":"2024-09-14T04:59:09.700039+0000","last_became_active":"2024-09-14T04:44:57.328437+0000","last_became_peered":"2024-09-14T04:44:57.328437+0000","last_unstale":"2024-09-14T04:59:09.700039+0000","last_undegraded":"2024-09-14T04:59:09.700039+0000","last_fullsized":"2024-09-14T04:59:09.700039+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.109086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":153,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.695246+0000","last_change":"2024-09-14T04:44:57.332243+0000","last_active":"2024-09-14T04:59:09.695246+0000","last_peered":"2024-09-14T04:59:09.695246+0000","last_clean":"2024-09-14T04:59:09.695246+0000","last_became_active":"2024-09-14T04:44:57.330413+0000","last_became_peered":"2024-09-14T04:44:57.330413+0000","last_unstale":"2024-09-14T04:59:09.695246+0000","last_undegraded":"2024-09-14T04:59:09.695246+0000","last_fullsized":"2024-09-14T04:59:09.695246+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:25:01.338549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":230,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.054255+0000","last_change":"2024-09-14T04:48:29.862929+0000","last_active":"2024-09-14T04:59:27.054255+0000","last_peered":"2024-09-14T04:59:27.054255+0000","last_clean":"2024-09-14T04:59:27.054255+0000","last_became_active":"2024-09-14T04:48:29.862581+0000","last_became_peered":"2024-09-14T04:48:29.862581+0000","last_unstale":"2024-09-14T04:59:27.054255+0000","last_undegraded":"2024-09-14T04:59:27.054255+0000","last_fullsized":"2024-09-14T04:59:27.054255+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:16:43.060088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2369,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.060693+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T04:59:27.060693+0000","last_peered":"2024-09-14T04:59:27.060693+0000","last_clean":"2024-09-14T04:59:27.060693+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T04:59:27.060693+0000","last_undegraded":"2024-09-14T04:59:27.060693+0000","last_fullsized":"2024-09-14T04:59:27.060693+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":153,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.694824+0000","last_change":"2024-09-14T04:44:57.325392+0000","last_active":"2024-09-14T04:59:09.694824+0000","last_peered":"2024-09-14T04:59:09.694824+0000","last_clean":"2024-09-14T04:59:09.694824+0000","last_became_active":"2024-09-14T04:44:57.324624+0000","last_became_peered":"2024-09-14T04:44:57.324624+0000","last_unstale":"2024-09-14T04:59:09.694824+0000","last_undegraded":"2024-09-14T04:59:09.694824+0000","last_fullsized":"2024-09-14T04:59:09.694824+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:33:18.810199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2338,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.053178+0000","last_change":"2024-09-14T04:44:57.329113+0000","last_active":"2024-09-14T04:59:27.053178+0000","last_peered":"2024-09-14T04:59:27.053178+0000","last_clean":"2024-09-14T04:59:27.053178+0000","last_became_active":"2024-09-14T04:44:57.328727+0000","last_became_peered":"2024-09-14T04:44:57.328727+0000","last_unstale":"2024-09-14T04:59:27.053178+0000","last_undegraded":"2024-09-14T04:59:27.053178+0000","last_fullsized":"2024-09-14T04:59:27.053178+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:34.939874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":2355,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.052060+0000","last_change":"2024-09-14T04:44:57.329277+0000","last_active":"2024-09-14T04:59:27.052060+0000","last_peered":"2024-09-14T04:59:27.052060+0000","last_clean":"2024-09-14T04:59:27.052060+0000","last_became_active":"2024-09-14T04:44:57.328804+0000","last_became_peered":"2024-09-14T04:44:57.328804+0000","last_unstale":"2024-09-14T04:59:27.052060+0000","last_undegraded":"2024-09-14T04:59:27.052060+0000","last_fullsized":"2024-09-14T04:59:27.052060+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:42:46.055240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2390,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.068643+0000","last_change":"2024-09-14T04:54:52.250695+0000","last_active":"2024-09-14T04:59:27.068643+0000","last_peered":"2024-09-14T04:59:27.068643+0000","last_clean":"2024-09-14T04:59:27.068643+0000","last_became_active":"2024-09-14T04:54:52.250301+0000","last_became_peered":"2024-09-14T04:54:52.250301+0000","last_unstale":"2024-09-14T04:59:27.068643+0000","last_undegraded":"2024-09-14T04:59:27.068643+0000","last_fullsized":"2024-09-14T04:59:27.068643+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:42.060905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2351,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.053147+0000","last_change":"2024-09-14T04:43:13.357092+0000","last_active":"2024-09-14T04:59:27.053147+0000","last_peered":"2024-09-14T04:59:27.053147+0000","last_clean":"2024-09-14T04:59:27.053147+0000","last_became_active":"2024-09-14T04:43:13.356694+0000","last_became_peered":"2024-09-14T04:43:13.356694+0000","last_unstale":"2024-09-14T04:59:27.053147+0000","last_undegraded":"2024-09-14T04:59:27.053147+0000","last_fullsized":"2024-09-14T04:59:27.053147+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:11.167574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1526,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.047425+0000","last_change":"2024-09-14T04:07:21.720797+0000","last_active":"2024-09-14T04:59:27.047425+0000","last_peered":"2024-09-14T04:59:27.047425+0000","last_clean":"2024-09-14T04:59:27.047425+0000","last_became_active":"2024-09-14T04:07:21.719331+0000","last_became_peered":"2024-09-14T04:07:21.719331+0000","last_unstale":"2024-09-14T04:59:27.047425+0000","last_undegraded":"2024-09-14T04:59:27.047425+0000","last_fullsized":"2024-09-14T04:59:27.047425+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:35.404023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":173,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.055526+0000","last_change":"2024-09-14T04:43:11.641514+0000","last_active":"2024-09-14T04:59:27.055526+0000","last_peered":"2024-09-14T04:59:27.055526+0000","last_clean":"2024-09-14T04:59:27.055526+0000","last_became_active":"2024-09-14T04:43:11.641151+0000","last_became_peered":"2024-09-14T04:43:11.641151+0000","last_unstale":"2024-09-14T04:59:27.055526+0000","last_undegraded":"2024-09-14T04:59:27.055526+0000","last_fullsized":"2024-09-14T04:59:27.055526+0000","mapping_epoch":1384,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:06:45.045430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2363,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.053914+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T04:59:27.053914+0000","last_peered":"2024-09-14T04:59:27.053914+0000","last_clean":"2024-09-14T04:59:27.053914+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T04:59:27.053914+0000","last_undegraded":"2024-09-14T04:59:27.053914+0000","last_fullsized":"2024-09-14T04:59:27.053914+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":753,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.702005+0000","last_change":"2024-09-14T04:49:29.618477+0000","last_active":"2024-09-14T04:59:09.702005+0000","last_peered":"2024-09-14T04:59:09.702005+0000","last_clean":"2024-09-14T04:59:09.702005+0000","last_became_active":"2024-09-14T04:49:29.618210+0000","last_became_peered":"2024-09-14T04:49:29.618210+0000","last_unstale":"2024-09-14T04:59:09.702005+0000","last_undegraded":"2024-09-14T04:59:09.702005+0000","last_fullsized":"2024-09-14T04:59:09.702005+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:13:23.895512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1531,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.700298+0000","last_change":"2024-09-14T04:42:49.938505+0000","last_active":"2024-09-14T04:59:09.700298+0000","last_peered":"2024-09-14T04:59:09.700298+0000","last_clean":"2024-09-14T04:59:09.700298+0000","last_became_active":"2024-09-14T04:42:49.938180+0000","last_became_peered":"2024-09-14T04:42:49.938180+0000","last_unstale":"2024-09-14T04:59:09.700298+0000","last_undegraded":"2024-09-14T04:59:09.700298+0000","last_fullsized":"2024-09-14T04:59:09.700298+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:05.929479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1550,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.702638+0000","last_change":"2024-09-14T04:44:57.346582+0000","last_active":"2024-09-14T04:59:09.702638+0000","last_peered":"2024-09-14T04:59:09.702638+0000","last_clean":"2024-09-14T04:59:09.702638+0000","last_became_active":"2024-09-14T04:44:57.330446+0000","last_became_peered":"2024-09-14T04:44:57.330446+0000","last_unstale":"2024-09-14T04:59:09.702638+0000","last_undegraded":"2024-09-14T04:59:09.702638+0000","last_fullsized":"2024-09-14T04:59:09.702638+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:29.238585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":304,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.052063+0000","last_change":"2024-09-14T04:44:14.480667+0000","last_active":"2024-09-14T04:59:27.052063+0000","last_peered":"2024-09-14T04:59:27.052063+0000","last_clean":"2024-09-14T04:59:27.052063+0000","last_became_active":"2024-09-14T04:44:14.480365+0000","last_became_peered":"2024-09-14T04:44:14.480365+0000","last_unstale":"2024-09-14T04:59:27.052063+0000","last_undegraded":"2024-09-14T04:59:27.052063+0000","last_fullsized":"2024-09-14T04:59:27.052063+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:00.611771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":164,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.696426+0000","last_change":"2024-09-14T04:44:57.341426+0000","last_active":"2024-09-14T04:59:09.696426+0000","last_peered":"2024-09-14T04:59:09.696426+0000","last_clean":"2024-09-14T04:59:09.696426+0000","last_became_active":"2024-09-14T04:44:57.328763+0000","last_became_peered":"2024-09-14T04:44:57.328763+0000","last_unstale":"2024-09-14T04:59:09.696426+0000","last_undegraded":"2024-09-14T04:59:09.696426+0000","last_fullsized":"2024-09-14T04:59:09.696426+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:34:40.066761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":154,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.695319+0000","last_change":"2024-09-14T04:44:57.331337+0000","last_active":"2024-09-14T04:59:09.695319+0000","last_peered":"2024-09-14T04:59:09.695319+0000","last_clean":"2024-09-14T04:59:09.695319+0000","last_became_active":"2024-09-14T04:44:57.329336+0000","last_became_peered":"2024-09-14T04:44:57.329336+0000","last_unstale":"2024-09-14T04:59:09.695319+0000","last_undegraded":"2024-09-14T04:59:09.695319+0000","last_fullsized":"2024-09-14T04:59:09.695319+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:52:35.980869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"349'149","reported_seq":742,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.699902+0000","last_change":"2024-09-14T04:49:30.085274+0000","last_active":"2024-09-14T04:59:09.699902+0000","last_peered":"2024-09-14T04:59:09.699902+0000","last_clean":"2024-09-14T04:59:09.699902+0000","last_became_active":"2024-09-14T04:49:29.623295+0000","last_became_peered":"2024-09-14T04:49:29.623295+0000","last_unstale":"2024-09-14T04:59:09.699902+0000","last_undegraded":"2024-09-14T04:59:09.699902+0000","last_fullsized":"2024-09-14T04:59:09.699902+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:49:30.085089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:49:30.085089+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:20:10.012963+0000","scrub_duration":0.0071928790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":155,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.051693+0000","last_change":"2024-09-14T04:44:57.329870+0000","last_active":"2024-09-14T04:59:27.051693+0000","last_peered":"2024-09-14T04:59:27.051693+0000","last_clean":"2024-09-14T04:59:27.051693+0000","last_became_active":"2024-09-14T04:44:57.329353+0000","last_became_peered":"2024-09-14T04:44:57.329353+0000","last_unstale":"2024-09-14T04:59:27.051693+0000","last_undegraded":"2024-09-14T04:59:27.051693+0000","last_fullsized":"2024-09-14T04:59:27.051693+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:46.432181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1547,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.697411+0000","last_change":"2024-09-14T04:54:52.242207+0000","last_active":"2024-09-14T04:59:09.697411+0000","last_peered":"2024-09-14T04:59:09.697411+0000","last_clean":"2024-09-14T04:59:09.697411+0000","last_became_active":"2024-09-14T04:54:52.241840+0000","last_became_peered":"2024-09-14T04:54:52.241840+0000","last_unstale":"2024-09-14T04:59:09.697411+0000","last_undegraded":"2024-09-14T04:59:09.697411+0000","last_fullsized":"2024-09-14T04:59:09.697411+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:37.899666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":2401,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.069396+0000","last_change":"2024-09-14T04:44:57.344614+0000","last_active":"2024-09-14T04:59:27.069396+0000","last_peered":"2024-09-14T04:59:27.069396+0000","last_clean":"2024-09-14T04:59:27.069396+0000","last_became_active":"2024-09-14T04:44:57.332256+0000","last_became_peered":"2024-09-14T04:44:57.332256+0000","last_unstale":"2024-09-14T04:59:27.069396+0000","last_undegraded":"2024-09-14T04:59:27.069396+0000","last_fullsized":"2024-09-14T04:59:27.069396+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:23.438417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"349'149","reported_seq":913,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.044318+0000","last_change":"2024-09-14T04:45:00.331854+0000","last_active":"2024-09-14T04:59:27.044318+0000","last_peered":"2024-09-14T04:59:27.044318+0000","last_clean":"2024-09-14T04:59:27.044318+0000","last_became_active":"2024-09-14T04:45:00.331460+0000","last_became_peered":"2024-09-14T04:45:00.331460+0000","last_unstale":"2024-09-14T04:59:27.044318+0000","last_undegraded":"2024-09-14T04:59:27.044318+0000","last_fullsized":"2024-09-14T04:59:27.044318+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1406,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.567535+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.567535+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:45.095729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":744,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.061750+0000","last_change":"2024-09-14T04:49:29.616831+0000","last_active":"2024-09-14T04:59:27.061750+0000","last_peered":"2024-09-14T04:59:27.061750+0000","last_clean":"2024-09-14T04:59:27.061750+0000","last_became_active":"2024-09-14T04:49:29.616546+0000","last_became_peered":"2024-09-14T04:49:29.616546+0000","last_unstale":"2024-09-14T04:59:27.061750+0000","last_undegraded":"2024-09-14T04:59:27.061750+0000","last_fullsized":"2024-09-14T04:59:27.061750+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:27.882929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":356,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.058544+0000","last_change":"2024-09-14T04:44:57.330746+0000","last_active":"2024-09-14T04:59:27.058544+0000","last_peered":"2024-09-14T04:59:27.058544+0000","last_clean":"2024-09-14T04:59:27.058544+0000","last_became_active":"2024-09-14T04:44:57.330327+0000","last_became_peered":"2024-09-14T04:44:57.330327+0000","last_unstale":"2024-09-14T04:59:27.058544+0000","last_undegraded":"2024-09-14T04:59:27.058544+0000","last_fullsized":"2024-09-14T04:59:27.058544+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:53.122771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":194,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.705596+0000","last_change":"2024-09-14T04:43:13.355296+0000","last_active":"2024-09-14T04:59:09.705596+0000","last_peered":"2024-09-14T04:59:09.705596+0000","last_clean":"2024-09-14T04:59:09.705596+0000","last_became_active":"2024-09-14T04:43:13.354950+0000","last_became_peered":"2024-09-14T04:43:13.354950+0000","last_unstale":"2024-09-14T04:59:09.705596+0000","last_undegraded":"2024-09-14T04:59:09.705596+0000","last_fullsized":"2024-09-14T04:59:09.705596+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:20:08.968818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"349'149","reported_seq":851,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.697578+0000","last_change":"2024-09-14T04:41:39.471393+0000","last_active":"2024-09-14T04:59:09.697578+0000","last_peered":"2024-09-14T04:59:09.697578+0000","last_clean":"2024-09-14T04:59:09.697578+0000","last_became_active":"2024-09-14T04:41:36.931768+0000","last_became_peered":"2024-09-14T04:41:36.931768+0000","last_unstale":"2024-09-14T04:59:09.697578+0000","last_undegraded":"2024-09-14T04:59:09.697578+0000","last_fullsized":"2024-09-14T04:59:09.697578+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.471248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.471248+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:57:56.210375+0000","scrub_duration":0.021758344999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2357,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.054486+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T04:59:27.054486+0000","last_peered":"2024-09-14T04:59:27.054486+0000","last_clean":"2024-09-14T04:59:27.054486+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T04:59:27.054486+0000","last_undegraded":"2024-09-14T04:59:27.054486+0000","last_fullsized":"2024-09-14T04:59:27.054486+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1561,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.702860+0000","last_change":"2024-09-14T04:44:57.331436+0000","last_active":"2024-09-14T04:59:09.702860+0000","last_peered":"2024-09-14T04:59:09.702860+0000","last_clean":"2024-09-14T04:59:09.702860+0000","last_became_active":"2024-09-14T04:44:57.330957+0000","last_became_peered":"2024-09-14T04:44:57.330957+0000","last_unstale":"2024-09-14T04:59:09.702860+0000","last_undegraded":"2024-09-14T04:59:09.702860+0000","last_fullsized":"2024-09-14T04:59:09.702860+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:09.516688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":906,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.060632+0000","last_change":"2024-09-14T04:49:49.460401+0000","last_active":"2024-09-14T04:59:27.060632+0000","last_peered":"2024-09-14T04:59:27.060632+0000","last_clean":"2024-09-14T04:59:27.060632+0000","last_became_active":"2024-09-14T04:49:49.458530+0000","last_became_peered":"2024-09-14T04:49:49.458530+0000","last_unstale":"2024-09-14T04:59:27.060632+0000","last_undegraded":"2024-09-14T04:59:27.060632+0000","last_fullsized":"2024-09-14T04:59:27.060632+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:11:29.143239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"349'149","reported_seq":885,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.697169+0000","last_change":"2024-09-14T04:54:22.147441+0000","last_active":"2024-09-14T04:59:09.697169+0000","last_peered":"2024-09-14T04:59:09.697169+0000","last_clean":"2024-09-14T04:59:09.697169+0000","last_became_active":"2024-09-14T04:54:22.147115+0000","last_became_peered":"2024-09-14T04:54:22.147115+0000","last_unstale":"2024-09-14T04:59:09.697169+0000","last_undegraded":"2024-09-14T04:59:09.697169+0000","last_fullsized":"2024-09-14T04:59:09.697169+0000","mapping_epoch":1492,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1493,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:38.479887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:38.479887+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:41:14.001781+0000","scrub_duration":0.0062728319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":312,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.051954+0000","last_change":"2024-09-14T04:44:57.329983+0000","last_active":"2024-09-14T04:59:27.051954+0000","last_peered":"2024-09-14T04:59:27.051954+0000","last_clean":"2024-09-14T04:59:27.051954+0000","last_became_active":"2024-09-14T04:44:57.329597+0000","last_became_peered":"2024-09-14T04:44:57.329597+0000","last_unstale":"2024-09-14T04:59:27.051954+0000","last_undegraded":"2024-09-14T04:59:27.051954+0000","last_fullsized":"2024-09-14T04:59:27.051954+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:45.718039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":850,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.063065+0000","last_change":"2024-09-14T04:49:49.443631+0000","last_active":"2024-09-14T04:59:27.063065+0000","last_peered":"2024-09-14T04:59:27.063065+0000","last_clean":"2024-09-14T04:59:27.063065+0000","last_became_active":"2024-09-14T04:49:49.443313+0000","last_became_peered":"2024-09-14T04:49:49.443313+0000","last_unstale":"2024-09-14T04:59:27.063065+0000","last_undegraded":"2024-09-14T04:59:27.063065+0000","last_fullsized":"2024-09-14T04:59:27.063065+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:06:54.562292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1533,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.700090+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T04:59:09.700090+0000","last_peered":"2024-09-14T04:59:09.700090+0000","last_clean":"2024-09-14T04:59:09.700090+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T04:59:09.700090+0000","last_undegraded":"2024-09-14T04:59:09.700090+0000","last_fullsized":"2024-09-14T04:59:09.700090+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"349'149","reported_seq":843,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.701904+0000","last_change":"2024-09-14T04:41:39.012977+0000","last_active":"2024-09-14T04:59:09.701904+0000","last_peered":"2024-09-14T04:59:09.701904+0000","last_clean":"2024-09-14T04:59:09.701904+0000","last_became_active":"2024-09-14T04:41:36.932552+0000","last_became_peered":"2024-09-14T04:41:36.932552+0000","last_unstale":"2024-09-14T04:59:09.701904+0000","last_undegraded":"2024-09-14T04:59:09.701904+0000","last_fullsized":"2024-09-14T04:59:09.701904+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.012879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.012879+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:16:40.080262+0000","scrub_duration":0.0051721620000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":942,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.060730+0000","last_change":"2024-09-14T04:49:49.459590+0000","last_active":"2024-09-14T04:59:27.060730+0000","last_peered":"2024-09-14T04:59:27.060730+0000","last_clean":"2024-09-14T04:59:27.060730+0000","last_became_active":"2024-09-14T04:49:49.456263+0000","last_became_peered":"2024-09-14T04:49:49.456263+0000","last_unstale":"2024-09-14T04:59:27.060730+0000","last_undegraded":"2024-09-14T04:59:27.060730+0000","last_fullsized":"2024-09-14T04:59:27.060730+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:20:44.666805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":155,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.048686+0000","last_change":"2024-09-14T04:44:57.329652+0000","last_active":"2024-09-14T04:59:27.048686+0000","last_peered":"2024-09-14T04:59:27.048686+0000","last_clean":"2024-09-14T04:59:27.048686+0000","last_became_active":"2024-09-14T04:44:57.328888+0000","last_became_peered":"2024-09-14T04:44:57.328888+0000","last_unstale":"2024-09-14T04:59:27.048686+0000","last_undegraded":"2024-09-14T04:59:27.048686+0000","last_fullsized":"2024-09-14T04:59:27.048686+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:10.170068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1532,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.699706+0000","last_change":"2024-09-14T04:06:46.875340+0000","last_active":"2024-09-14T04:59:09.699706+0000","last_peered":"2024-09-14T04:59:09.699706+0000","last_clean":"2024-09-14T04:59:09.699706+0000","last_became_active":"2024-09-14T04:06:46.874552+0000","last_became_peered":"2024-09-14T04:06:46.874552+0000","last_unstale":"2024-09-14T04:59:09.699706+0000","last_undegraded":"2024-09-14T04:59:09.699706+0000","last_fullsized":"2024-09-14T04:59:09.699706+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:52.257357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"349'149","reported_seq":746,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.702458+0000","last_change":"2024-09-14T04:54:55.279666+0000","last_active":"2024-09-14T04:59:09.702458+0000","last_peered":"2024-09-14T04:59:09.702458+0000","last_clean":"2024-09-14T04:59:09.702458+0000","last_became_active":"2024-09-14T04:54:55.279259+0000","last_became_peered":"2024-09-14T04:54:55.279259+0000","last_unstale":"2024-09-14T04:59:09.702458+0000","last_undegraded":"2024-09-14T04:59:09.702458+0000","last_fullsized":"2024-09-14T04:59:09.702458+0000","mapping_epoch":1506,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1507,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:54:03.996025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:54:03.996025+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:40:53.332369+0000","scrub_duration":54,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":305,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.053317+0000","last_change":"2024-09-14T04:44:14.489235+0000","last_active":"2024-09-14T04:59:27.053317+0000","last_peered":"2024-09-14T04:59:27.053317+0000","last_clean":"2024-09-14T04:59:27.053317+0000","last_became_active":"2024-09-14T04:44:14.488950+0000","last_became_peered":"2024-09-14T04:44:14.488950+0000","last_unstale":"2024-09-14T04:59:27.053317+0000","last_undegraded":"2024-09-14T04:59:27.053317+0000","last_fullsized":"2024-09-14T04:59:27.053317+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:54.441144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":860,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.060955+0000","last_change":"2024-09-14T04:49:49.456317+0000","last_active":"2024-09-14T04:59:27.060955+0000","last_peered":"2024-09-14T04:59:27.060955+0000","last_clean":"2024-09-14T04:59:27.060955+0000","last_became_active":"2024-09-14T04:49:49.456029+0000","last_became_peered":"2024-09-14T04:49:49.456029+0000","last_unstale":"2024-09-14T04:59:27.060955+0000","last_undegraded":"2024-09-14T04:59:27.060955+0000","last_fullsized":"2024-09-14T04:59:27.060955+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:46:05.319676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2378,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.061212+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T04:59:27.061212+0000","last_peered":"2024-09-14T04:59:27.061212+0000","last_clean":"2024-09-14T04:59:27.061212+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T04:59:27.061212+0000","last_undegraded":"2024-09-14T04:59:27.061212+0000","last_fullsized":"2024-09-14T04:59:27.061212+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"349'149","reported_seq":737,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.697586+0000","last_change":"2024-09-14T04:50:22.002798+0000","last_active":"2024-09-14T04:59:09.697586+0000","last_peered":"2024-09-14T04:59:09.697586+0000","last_clean":"2024-09-14T04:59:09.697586+0000","last_became_active":"2024-09-14T04:49:49.448278+0000","last_became_peered":"2024-09-14T04:49:49.448278+0000","last_unstale":"2024-09-14T04:59:09.697586+0000","last_undegraded":"2024-09-14T04:59:09.697586+0000","last_fullsized":"2024-09-14T04:59:09.697586+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:50:22.002568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:50:22.002568+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:01:57.489223+0000","scrub_duration":0.010888326,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2378,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.068749+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T04:59:27.068749+0000","last_peered":"2024-09-14T04:59:27.068749+0000","last_clean":"2024-09-14T04:59:27.068749+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T04:59:27.068749+0000","last_undegraded":"2024-09-14T04:59:27.068749+0000","last_fullsized":"2024-09-14T04:59:27.068749+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2399,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.056266+0000","last_change":"2024-09-14T04:54:19.447103+0000","last_active":"2024-09-14T04:59:27.056266+0000","last_peered":"2024-09-14T04:59:27.056266+0000","last_clean":"2024-09-14T04:59:27.056266+0000","last_became_active":"2024-09-14T04:54:19.446748+0000","last_became_peered":"2024-09-14T04:54:19.446748+0000","last_unstale":"2024-09-14T04:59:27.056266+0000","last_undegraded":"2024-09-14T04:59:27.056266+0000","last_fullsized":"2024-09-14T04:59:27.056266+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:00:55.369263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1562,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.702144+0000","last_change":"2024-09-14T04:42:49.939790+0000","last_active":"2024-09-14T04:59:09.702144+0000","last_peered":"2024-09-14T04:59:09.702144+0000","last_clean":"2024-09-14T04:59:09.702144+0000","last_became_active":"2024-09-14T04:42:49.939591+0000","last_became_peered":"2024-09-14T04:42:49.939591+0000","last_unstale":"2024-09-14T04:59:09.702144+0000","last_undegraded":"2024-09-14T04:59:09.702144+0000","last_fullsized":"2024-09-14T04:59:09.702144+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:23:16.790593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1525,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.701818+0000","last_change":"2024-09-14T04:44:57.326008+0000","last_active":"2024-09-14T04:59:09.701818+0000","last_peered":"2024-09-14T04:59:09.701818+0000","last_clean":"2024-09-14T04:59:09.701818+0000","last_became_active":"2024-09-14T04:44:57.325617+0000","last_became_peered":"2024-09-14T04:44:57.325617+0000","last_unstale":"2024-09-14T04:59:09.701818+0000","last_undegraded":"2024-09-14T04:59:09.701818+0000","last_fullsized":"2024-09-14T04:59:09.701818+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:15.637508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":814,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.699927+0000","last_change":"2024-09-14T04:43:13.354846+0000","last_active":"2024-09-14T04:59:09.699927+0000","last_peered":"2024-09-14T04:59:09.699927+0000","last_clean":"2024-09-14T04:59:09.699927+0000","last_became_active":"2024-09-14T04:43:13.354515+0000","last_became_peered":"2024-09-14T04:43:13.354515+0000","last_unstale":"2024-09-14T04:59:09.699927+0000","last_undegraded":"2024-09-14T04:59:09.699927+0000","last_fullsized":"2024-09-14T04:59:09.699927+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:17.889341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1531,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.699663+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T04:59:09.699663+0000","last_peered":"2024-09-14T04:59:09.699663+0000","last_clean":"2024-09-14T04:59:09.699663+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T04:59:09.699663+0000","last_undegraded":"2024-09-14T04:59:09.699663+0000","last_fullsized":"2024-09-14T04:59:09.699663+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":1869,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.061672+0000","last_change":"2024-09-14T04:41:30.640651+0000","last_active":"2024-09-14T04:59:27.061672+0000","last_peered":"2024-09-14T04:59:27.061672+0000","last_clean":"2024-09-14T04:59:27.061672+0000","last_became_active":"2024-09-14T04:41:29.532466+0000","last_became_peered":"2024-09-14T04:41:29.532466+0000","last_unstale":"2024-09-14T04:59:27.061672+0000","last_undegraded":"2024-09-14T04:59:27.061672+0000","last_fullsized":"2024-09-14T04:59:27.061672+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1342,"parent":"0.0","parent_split_bits":0,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:30.640520+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:30.640520+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:27.776724+0000","scrub_duration":0.0028653139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"349'149","reported_seq":843,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.699685+0000","last_change":"2024-09-14T04:41:39.978080+0000","last_active":"2024-09-14T04:59:09.699685+0000","last_peered":"2024-09-14T04:59:09.699685+0000","last_clean":"2024-09-14T04:59:09.699685+0000","last_became_active":"2024-09-14T04:41:36.929279+0000","last_became_peered":"2024-09-14T04:41:36.929279+0000","last_unstale":"2024-09-14T04:59:09.699685+0000","last_undegraded":"2024-09-14T04:59:09.699685+0000","last_fullsized":"2024-09-14T04:59:09.699685+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.977999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.977999+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:44:06.383827+0000","scrub_duration":0.00192122,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":239,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.062798+0000","last_change":"2024-09-14T04:48:11.040133+0000","last_active":"2024-09-14T04:59:27.062798+0000","last_peered":"2024-09-14T04:59:27.062798+0000","last_clean":"2024-09-14T04:59:27.062798+0000","last_became_active":"2024-09-14T04:48:11.039787+0000","last_became_peered":"2024-09-14T04:48:11.039787+0000","last_unstale":"2024-09-14T04:59:27.062798+0000","last_undegraded":"2024-09-14T04:59:27.062798+0000","last_fullsized":"2024-09-14T04:59:27.062798+0000","mapping_epoch":1428,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:53.520672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2225,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.052161+0000","last_change":"2024-09-14T04:49:29.616443+0000","last_active":"2024-09-14T04:59:27.052161+0000","last_peered":"2024-09-14T04:59:27.052161+0000","last_clean":"2024-09-14T04:59:27.052161+0000","last_became_active":"2024-09-14T04:49:29.616066+0000","last_became_peered":"2024-09-14T04:49:29.616066+0000","last_unstale":"2024-09-14T04:59:27.052161+0000","last_undegraded":"2024-09-14T04:59:27.052161+0000","last_fullsized":"2024-09-14T04:59:27.052161+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:09:15.897957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":154,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.695808+0000","last_change":"2024-09-14T04:44:57.326724+0000","last_active":"2024-09-14T04:59:09.695808+0000","last_peered":"2024-09-14T04:59:09.695808+0000","last_clean":"2024-09-14T04:59:09.695808+0000","last_became_active":"2024-09-14T04:44:57.326191+0000","last_became_peered":"2024-09-14T04:44:57.326191+0000","last_unstale":"2024-09-14T04:59:09.695808+0000","last_undegraded":"2024-09-14T04:59:09.695808+0000","last_fullsized":"2024-09-14T04:59:09.695808+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:46.060269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":154,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.694992+0000","last_change":"2024-09-14T04:44:57.328528+0000","last_active":"2024-09-14T04:59:09.694992+0000","last_peered":"2024-09-14T04:59:09.694992+0000","last_clean":"2024-09-14T04:59:09.694992+0000","last_became_active":"2024-09-14T04:44:57.327739+0000","last_became_peered":"2024-09-14T04:44:57.327739+0000","last_unstale":"2024-09-14T04:59:09.694992+0000","last_undegraded":"2024-09-14T04:59:09.694992+0000","last_fullsized":"2024-09-14T04:59:09.694992+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:04:58.294690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"349'149","reported_seq":851,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.699850+0000","last_change":"2024-09-14T04:55:59.732757+0000","last_active":"2024-09-14T04:59:09.699850+0000","last_peered":"2024-09-14T04:59:09.699850+0000","last_clean":"2024-09-14T04:59:09.699850+0000","last_became_active":"2024-09-14T04:55:59.732364+0000","last_became_peered":"2024-09-14T04:55:59.732364+0000","last_unstale":"2024-09-14T04:59:09.699850+0000","last_undegraded":"2024-09-14T04:59:09.699850+0000","last_fullsized":"2024-09-14T04:59:09.699850+0000","mapping_epoch":1512,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1513,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:56.738636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:56.738636+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:59:11.313765+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1313440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":154,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.695471+0000","last_change":"2024-09-14T04:44:57.323436+0000","last_active":"2024-09-14T04:59:09.695471+0000","last_peered":"2024-09-14T04:59:09.695471+0000","last_clean":"2024-09-14T04:59:09.695471+0000","last_became_active":"2024-09-14T04:44:57.323022+0000","last_became_peered":"2024-09-14T04:44:57.323022+0000","last_unstale":"2024-09-14T04:59:09.695471+0000","last_undegraded":"2024-09-14T04:59:09.695471+0000","last_fullsized":"2024-09-14T04:59:09.695471+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:22.775261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1560,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.696942+0000","last_change":"2024-09-14T04:44:57.322357+0000","last_active":"2024-09-14T04:59:09.696942+0000","last_peered":"2024-09-14T04:59:09.696942+0000","last_clean":"2024-09-14T04:59:09.696942+0000","last_became_active":"2024-09-14T04:44:57.321990+0000","last_became_peered":"2024-09-14T04:44:57.321990+0000","last_unstale":"2024-09-14T04:59:09.696942+0000","last_undegraded":"2024-09-14T04:59:09.696942+0000","last_fullsized":"2024-09-14T04:59:09.696942+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:09.658008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"349'149","reported_seq":1885,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.701883+0000","last_change":"2024-09-14T04:41:53.858076+0000","last_active":"2024-09-14T04:59:09.701883+0000","last_peered":"2024-09-14T04:59:09.701883+0000","last_clean":"2024-09-14T04:59:09.701883+0000","last_became_active":"2024-09-14T04:41:36.931585+0000","last_became_peered":"2024-09-14T04:41:36.931585+0000","last_unstale":"2024-09-14T04:59:09.701883+0000","last_undegraded":"2024-09-14T04:59:09.701883+0000","last_fullsized":"2024-09-14T04:59:09.701883+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:53.857924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:53.857924+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:10:20.658986+0000","scrub_duration":0.0022355919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":154,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.696353+0000","last_change":"2024-09-14T04:44:57.329198+0000","last_active":"2024-09-14T04:59:09.696353+0000","last_peered":"2024-09-14T04:59:09.696353+0000","last_clean":"2024-09-14T04:59:09.696353+0000","last_became_active":"2024-09-14T04:44:57.327754+0000","last_became_peered":"2024-09-14T04:44:57.327754+0000","last_unstale":"2024-09-14T04:59:09.696353+0000","last_undegraded":"2024-09-14T04:59:09.696353+0000","last_fullsized":"2024-09-14T04:59:09.696353+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:10:27.944476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":362,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.054567+0000","last_change":"2024-09-14T04:44:57.348687+0000","last_active":"2024-09-14T04:59:27.054567+0000","last_peered":"2024-09-14T04:59:27.054567+0000","last_clean":"2024-09-14T04:59:27.054567+0000","last_became_active":"2024-09-14T04:44:57.332845+0000","last_became_peered":"2024-09-14T04:44:57.332845+0000","last_unstale":"2024-09-14T04:59:27.054567+0000","last_undegraded":"2024-09-14T04:59:27.054567+0000","last_fullsized":"2024-09-14T04:59:27.054567+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:12:18.762737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2384,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.054343+0000","last_change":"2024-09-14T04:44:57.333468+0000","last_active":"2024-09-14T04:59:27.054343+0000","last_peered":"2024-09-14T04:59:27.054343+0000","last_clean":"2024-09-14T04:59:27.054343+0000","last_became_active":"2024-09-14T04:44:57.332626+0000","last_became_peered":"2024-09-14T04:44:57.332626+0000","last_unstale":"2024-09-14T04:59:27.054343+0000","last_undegraded":"2024-09-14T04:59:27.054343+0000","last_fullsized":"2024-09-14T04:59:27.054343+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:57:26.366438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1516,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.700111+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T04:59:09.700111+0000","last_peered":"2024-09-14T04:59:09.700111+0000","last_clean":"2024-09-14T04:59:09.700111+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T04:59:09.700111+0000","last_undegraded":"2024-09-14T04:59:09.700111+0000","last_fullsized":"2024-09-14T04:59:09.700111+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1515,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.696810+0000","last_change":"2024-09-14T04:06:46.869868+0000","last_active":"2024-09-14T04:59:09.696810+0000","last_peered":"2024-09-14T04:59:09.696810+0000","last_clean":"2024-09-14T04:59:09.696810+0000","last_became_active":"2024-09-14T04:06:46.869126+0000","last_became_peered":"2024-09-14T04:06:46.869126+0000","last_unstale":"2024-09-14T04:59:09.696810+0000","last_undegraded":"2024-09-14T04:59:09.696810+0000","last_fullsized":"2024-09-14T04:59:09.696810+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:41.150138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1499,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.702774+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T04:59:09.702774+0000","last_peered":"2024-09-14T04:59:09.702774+0000","last_clean":"2024-09-14T04:59:09.702774+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T04:59:09.702774+0000","last_undegraded":"2024-09-14T04:59:09.702774+0000","last_fullsized":"2024-09-14T04:59:09.702774+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":154,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.695242+0000","last_change":"2024-09-14T04:44:57.332112+0000","last_active":"2024-09-14T04:59:09.695242+0000","last_peered":"2024-09-14T04:59:09.695242+0000","last_clean":"2024-09-14T04:59:09.695242+0000","last_became_active":"2024-09-14T04:44:57.331575+0000","last_became_peered":"2024-09-14T04:44:57.331575+0000","last_unstale":"2024-09-14T04:59:09.695242+0000","last_undegraded":"2024-09-14T04:59:09.695242+0000","last_fullsized":"2024-09-14T04:59:09.695242+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:50:00.391283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1547,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.703140+0000","last_change":"2024-09-14T04:44:57.351460+0000","last_active":"2024-09-14T04:59:09.703140+0000","last_peered":"2024-09-14T04:59:09.703140+0000","last_clean":"2024-09-14T04:59:09.703140+0000","last_became_active":"2024-09-14T04:44:57.334662+0000","last_became_peered":"2024-09-14T04:44:57.334662+0000","last_unstale":"2024-09-14T04:59:09.703140+0000","last_undegraded":"2024-09-14T04:59:09.703140+0000","last_fullsized":"2024-09-14T04:59:09.703140+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:02.399062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1501,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.700028+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T04:59:09.700028+0000","last_peered":"2024-09-14T04:59:09.700028+0000","last_clean":"2024-09-14T04:59:09.700028+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T04:59:09.700028+0000","last_undegraded":"2024-09-14T04:59:09.700028+0000","last_fullsized":"2024-09-14T04:59:09.700028+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1559,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.705572+0000","last_change":"2024-09-14T04:44:57.349690+0000","last_active":"2024-09-14T04:59:09.705572+0000","last_peered":"2024-09-14T04:59:09.705572+0000","last_clean":"2024-09-14T04:59:09.705572+0000","last_became_active":"2024-09-14T04:44:57.333579+0000","last_became_peered":"2024-09-14T04:44:57.333579+0000","last_unstale":"2024-09-14T04:59:09.705572+0000","last_undegraded":"2024-09-14T04:59:09.705572+0000","last_fullsized":"2024-09-14T04:59:09.705572+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:17:55.265141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1636,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.696811+0000","last_change":"2024-09-14T04:54:35.694089+0000","last_active":"2024-09-14T04:59:09.696811+0000","last_peered":"2024-09-14T04:59:09.696811+0000","last_clean":"2024-09-14T04:59:09.696811+0000","last_became_active":"2024-09-14T04:54:35.663133+0000","last_became_peered":"2024-09-14T04:54:35.663133+0000","last_unstale":"2024-09-14T04:59:09.696811+0000","last_undegraded":"2024-09-14T04:59:09.696811+0000","last_fullsized":"2024-09-14T04:59:09.696811+0000","mapping_epoch":1498,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1499,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:17:19.947280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":49,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.062922+0000","last_change":"2024-09-14T04:54:52.248439+0000","last_active":"2024-09-14T04:59:27.062922+0000","last_peered":"2024-09-14T04:59:27.062922+0000","last_clean":"2024-09-14T04:59:27.062922+0000","last_became_active":"2024-09-14T04:54:52.248235+0000","last_became_peered":"2024-09-14T04:54:52.248235+0000","last_unstale":"2024-09-14T04:59:27.062922+0000","last_undegraded":"2024-09-14T04:59:27.062922+0000","last_fullsized":"2024-09-14T04:59:27.062922+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:22:09.548139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1521,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.702124+0000","last_change":"2024-09-14T04:44:57.332511+0000","last_active":"2024-09-14T04:59:09.702124+0000","last_peered":"2024-09-14T04:59:09.702124+0000","last_clean":"2024-09-14T04:59:09.702124+0000","last_became_active":"2024-09-14T04:44:57.332166+0000","last_became_peered":"2024-09-14T04:44:57.332166+0000","last_unstale":"2024-09-14T04:59:09.702124+0000","last_undegraded":"2024-09-14T04:59:09.702124+0000","last_fullsized":"2024-09-14T04:59:09.702124+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:53.984639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2378,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.062814+0000","last_change":"2024-09-14T04:54:52.250655+0000","last_active":"2024-09-14T04:59:27.062814+0000","last_peered":"2024-09-14T04:59:27.062814+0000","last_clean":"2024-09-14T04:59:27.062814+0000","last_became_active":"2024-09-14T04:54:52.250317+0000","last_became_peered":"2024-09-14T04:54:52.250317+0000","last_unstale":"2024-09-14T04:59:27.062814+0000","last_undegraded":"2024-09-14T04:59:27.062814+0000","last_fullsized":"2024-09-14T04:59:27.062814+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:12.985406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2364,"reported_epoch":1540,"state":"active+clean","last_fresh":"2024-09-14T04:59:27.052561+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T04:59:27.052561+0000","last_peered":"2024-09-14T04:59:27.052561+0000","last_clean":"2024-09-14T04:59:27.052561+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T04:59:27.052561+0000","last_undegraded":"2024-09-14T04:59:27.052561+0000","last_fullsized":"2024-09-14T04:59:27.052561+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1513,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.700085+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T04:59:09.700085+0000","last_peered":"2024-09-14T04:59:09.700085+0000","last_clean":"2024-09-14T04:59:09.700085+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T04:59:09.700085+0000","last_undegraded":"2024-09-14T04:59:09.700085+0000","last_fullsized":"2024-09-14T04:59:09.700085+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1550,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:09.702340+0000","last_change":"2024-09-14T04:44:57.339993+0000","last_active":"2024-09-14T04:59:09.702340+0000","last_peered":"2024-09-14T04:59:09.702340+0000","last_clean":"2024-09-14T04:59:09.702340+0000","last_became_active":"2024-09-14T04:44:57.330472+0000","last_became_peered":"2024-09-14T04:44:57.330472+0000","last_unstale":"2024-09-14T04:59:09.702340+0000","last_undegraded":"2024-09-14T04:59:09.702340+0000","last_fullsized":"2024-09-14T04:59:09.702340+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:19:27.046988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2098,"reported_epoch":1539,"state":"active+clean","last_fresh":"2024-09-14T04:59:17.748616+0000","last_change":"2024-09-14T04:49:49.449687+0000","last_active":"2024-09-14T04:59:17.748616+0000","last_peered":"2024-09-14T04:59:17.748616+0000","last_clean":"2024-09-14T04:59:17.748616+0000","last_became_active":"2024-09-14T04:49:49.449144+0000","last_became_peered":"2024-09-14T04:49:49.449144+0000","last_unstale":"2024-09-14T04:59:17.748616+0000","last_undegraded":"2024-09-14T04:59:17.748616+0000","last_fullsized":"2024-09-14T04:59:17.748616+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:59:06.779730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":57,"num_pg":16,"stat_sum":{"num_bytes":3817209880,"num_objects":58247,"num_object_clones":0,"num_object_copies":174741,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":58247,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":189253,"num_write_kb":7919937,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11556171776,"data_stored":11556159560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":138553,"ondisk_log_size":138553,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1313440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1639,"ondisk_log_size":1639,"up":33,"acting":33,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":5070,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4250,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223202105,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2036020,"kb_used_data":1202668,"kb_used_omap":1,"kb_used_meta":833342,"kb_avail":91709132,"statfs":{"total":95995035648,"available":93910151168,"internally_reserved":0,"allocated":1231532032,"data_stored":1226096374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":853342667},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463741,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3017776,"kb_used_data":2184424,"kb_used_omap":1,"kb_used_meta":833342,"kb_avail":90727376,"statfs":{"total":95995035648,"available":92904833024,"internally_reserved":0,"allocated":2236850176,"data_stored":2231418511,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":853342663},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725378,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2267240,"kb_used_data":1424032,"kb_used_omap":1,"kb_used_meta":843198,"kb_avail":91477912,"statfs":{"total":95995035648,"available":93673381888,"internally_reserved":0,"allocated":1458208768,"data_stored":1452781641,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":863435206},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":1392,"seq":5978594476224,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1370804,"kb_used_data":964716,"kb_used_omap":117,"kb_used_meta":405898,"kb_avail":92374348,"statfs":{"total":95995035648,"available":94591332352,"internally_reserved":0,"allocated":987869184,"data_stored":982434078,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120638,"internal_metadata":415639746},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772075119,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2795816,"kb_used_data":1933472,"kb_used_omap":71,"kb_used_meta":862200,"kb_avail":90949336,"statfs":{"total":95995035648,"available":93132120064,"internally_reserved":0,"allocated":1979875328,"data_stored":1974448278,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":882893750},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707304055,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2485924,"kb_used_data":1872156,"kb_used_omap":68,"kb_used_meta":613691,"kb_avail":91259228,"statfs":{"total":95995035648,"available":93449449472,"internally_reserved":0,"allocated":1917087744,"data_stored":1911664670,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":628419792},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872729214,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1081436,"kb_used_data":476052,"kb_used_omap":67,"kb_used_meta":605308,"kb_avail":92663716,"statfs":{"total":95995035648,"available":94887645184,"internally_reserved":0,"allocated":487477248,"data_stored":482061225,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":619835749},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056264,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2142244,"kb_used_data":1398236,"kb_used_omap":74,"kb_used_meta":743925,"kb_avail":91602908,"statfs":{"total":95995035648,"available":93801377792,"internally_reserved":0,"allocated":1431793664,"data_stored":1426370058,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":761779893},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,3,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1901,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1416892416,"data_stored":1416888344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":474480640,"data_stored":474480640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1880555520,"data_stored":1880555520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1961693184,"data_stored":1961689112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":968753152,"data_stored":968753152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1438646272,"data_stored":1438646272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2208239616,"data_stored":2208235544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1206910976,"data_stored":1206910976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T04:59:48.098 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-14T04:59:48.099 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pg_num 21 2024-09-14T04:59:48.114 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:48.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:48.190 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:48.191 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:48.291 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:48.326 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:48 smithi039 ceph-mon[106470]: from='client.55942 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:59:48.326 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:48 smithi039 ceph-mon[111235]: from='client.55942 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:59:48.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:48.335 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:48.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:48.436 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:48.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:48.489 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:48.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:48.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:48 smithi129 ceph-mon[101316]: from='client.55942 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T04:59:48.590 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:48.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:48.637 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:48.638 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:48.738 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:48.777 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:48.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:48.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:48.880 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:48.908 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:59:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:59:48] "GET /metrics HTTP/1.1" 200 36733 "" "Prometheus/2.51.0" 2024-09-14T04:59:48.933 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:48.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:48.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:49.035 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:49.063 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:49.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:49.064 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:49.165 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:49.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:49.205 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:49.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:49.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:49 smithi039 ceph-mon[106470]: pgmap v3016: 99 pgs: 99 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2024-09-14T04:59:49.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:49 smithi039 ceph-mon[111235]: pgmap v3016: 99 pgs: 99 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2024-09-14T04:59:49.306 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:49.345 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:49.346 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:49.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:49.447 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:49.484 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:49.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:49.485 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:49.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:49 smithi129 ceph-mon[101316]: pgmap v3016: 99 pgs: 99 active+clean; 3.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 869 op/s 2024-09-14T04:59:49.586 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:49.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:49.624 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:49.624 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:49.725 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:49.756 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:49.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:49.757 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:49.858 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:49.890 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:49.891 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:49.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:49.992 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:50.032 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:50.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:50.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:50.134 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:50.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:50.176 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:50.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:50.277 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:50.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:50.323 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:50.324 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:50.425 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:50.445 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:59:50.482 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:50.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:50.483 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:50.584 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:50.612 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:50.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:50.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:50.714 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:50.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:50.753 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:50.753 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:50.854 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:50.896 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:50.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:50.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:50.909 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T04:59:50.998 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:51.032 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:51.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:51.033 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:51.133 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:51.167 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:51.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:51.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:51.269 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:51.289 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:51 smithi039 ceph-mon[106470]: pgmap v3017: 99 pgs: 99 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2024-09-14T04:59:51.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:51 smithi039 ceph-mon[111235]: pgmap v3017: 99 pgs: 99 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2024-09-14T04:59:51.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:51.291 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:51.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:51.392 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:51.424 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:51.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:51.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:51.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:51 smithi129 ceph-mon[101316]: pgmap v3017: 99 pgs: 99 active+clean; 3.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 861 op/s 2024-09-14T04:59:51.528 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:51.568 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:51.569 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:51.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:51.670 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:51.703 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:51.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:51.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:51.805 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:51.827 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:51.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:51.828 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:51.928 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:51.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:51.967 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:51.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:52.068 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:52.096 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:52.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:52.097 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:52.200 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:52.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:52.234 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:52.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:52.335 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:52.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:52.367 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:52.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:52.468 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:52.502 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:52.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:52.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:52.604 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:52.647 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:52.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:52.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:52.749 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:52.774 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:52.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:52.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:52.844 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T04:59:52.845 INFO:teuthology.orchestra.run.smithi039.stdout: "size": 20, 2024-09-14T04:59:52.845 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 600, 2024-09-14T04:59:52.845 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [ 2024-09-14T04:59:52.845 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.845 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45632.0:149713 55.6 55:6997e688:::benchmark_data_smithi039_297155_object25780:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1447)", 2024-09-14T04:59:52.845 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:49:58.461956+0000", 2024-09-14T04:59:52.845 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 594.35713623100003, 2024-09-14T04:59:52.845 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.34850957900000001, 2024-09-14T04:59:52.845 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:59:52.845 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:59:52.845 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:59:52.846 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:59:52.846 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45632", 2024-09-14T04:59:52.846 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:59:52.846 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 149713 2024-09-14T04:59:52.846 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.846 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:59:52.846 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.846 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:59:52.846 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.461956+0000", 2024-09-14T04:59:52.846 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.846 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.846 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.847 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:59:52.847 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.461956+0000", 2024-09-14T04:59:52.847 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.847 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.847 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.847 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:59:52.847 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.461957+0000", 2024-09-14T04:59:52.847 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5489999999999999e-06 2024-09-14T04:59:52.847 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.847 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.847 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:59:52.847 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.461962+0000", 2024-09-14T04:59:52.847 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.8690000000000003e-06 2024-09-14T04:59:52.848 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.848 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.848 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:59:52.848 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.461972+0000", 2024-09-14T04:59:52.848 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.7489999999999993e-06 2024-09-14T04:59:52.848 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.848 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.848 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:59:52.848 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.461977+0000", 2024-09-14T04:59:52.848 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.826e-06 2024-09-14T04:59:52.848 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.848 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.848 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:59:52.849 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.462030+0000", 2024-09-14T04:59:52.849 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3242000000000003e-05 2024-09-14T04:59:52.849 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.849 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.849 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:59:52.849 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.462151+0000", 2024-09-14T04:59:52.849 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012078000000000001 2024-09-14T04:59:52.849 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.849 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.849 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 5,7", 2024-09-14T04:59:52.849 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.462315+0000", 2024-09-14T04:59:52.849 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00016381499999999999 2024-09-14T04:59:52.850 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.850 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.850 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:59:52.850 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.467371+0000", 2024-09-14T04:59:52.850 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0050564900000000003 2024-09-14T04:59:52.850 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.850 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.850 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:59:52.850 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.469937+0000", 2024-09-14T04:59:52.850 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0025656350000000001 2024-09-14T04:59:52.850 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.850 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.850 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:59:52.851 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.471995+0000", 2024-09-14T04:59:52.851 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0020580580000000002 2024-09-14T04:59:52.851 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.851 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.851 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:59:52.851 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.472051+0000", 2024-09-14T04:59:52.851 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6623000000000003e-05 2024-09-14T04:59:52.851 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.851 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.851 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:59:52.851 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.810465+0000", 2024-09-14T04:59:52.851 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.33841394299999999 2024-09-14T04:59:52.851 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.851 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:59:52.852 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.852 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.852 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.852 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45632.0:149718 55.2 e1447/1423 55:4375dd76:::benchmark_data_smithi039_297155_object25785:head v 1447'9264, mlcod=1447'9261)", 2024-09-14T04:59:52.852 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:49:58.463909+0000", 2024-09-14T04:59:52.852 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 594.35518325700002, 2024-09-14T04:59:52.852 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.346525205, 2024-09-14T04:59:52.852 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:59:52.852 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:59:52.852 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:59:52.852 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:59:52.852 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.852 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:59:52.853 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.463909+0000", 2024-09-14T04:59:52.853 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.853 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.853 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.853 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:59:52.853 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.463909+0000", 2024-09-14T04:59:52.853 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.853 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.853 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.853 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:59:52.853 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.463910+0000", 2024-09-14T04:59:52.854 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.92e-07 2024-09-14T04:59:52.854 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.854 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.854 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:59:52.854 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.463916+0000", 2024-09-14T04:59:52.854 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.241e-06 2024-09-14T04:59:52.854 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.854 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.854 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:59:52.854 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.463926+0000", 2024-09-14T04:59:52.854 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0434e-05 2024-09-14T04:59:52.854 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.855 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.855 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:59:52.855 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.463932+0000", 2024-09-14T04:59:52.855 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2820000000000002e-06 2024-09-14T04:59:52.855 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.855 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.855 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:59:52.855 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.463983+0000", 2024-09-14T04:59:52.855 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.0954e-05 2024-09-14T04:59:52.855 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.855 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.855 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:59:52.855 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.464002+0000", 2024-09-14T04:59:52.855 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8576999999999998e-05 2024-09-14T04:59:52.856 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.856 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.856 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:59:52.856 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.810387+0000", 2024-09-14T04:59:52.856 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.34638481199999999 2024-09-14T04:59:52.856 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.856 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.856 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:59:52.856 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.810434+0000", 2024-09-14T04:59:52.856 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7113000000000001e-05 2024-09-14T04:59:52.856 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.856 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:59:52.856 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.857 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.857 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.857 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45632.0:149714 55.a e1447/1442 55:585bdded:::benchmark_data_smithi039_297155_object25781:head v 1447'9325, mlcod=1447'9322)", 2024-09-14T04:59:52.857 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:49:58.466997+0000", 2024-09-14T04:59:52.857 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 594.35209469899996, 2024-09-14T04:59:52.857 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.34582009699999999, 2024-09-14T04:59:52.857 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:59:52.857 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:59:52.857 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:59:52.857 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:59:52.857 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.857 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:59:52.857 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.466997+0000", 2024-09-14T04:59:52.858 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.858 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.858 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.858 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:59:52.858 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.466997+0000", 2024-09-14T04:59:52.858 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.858 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.858 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.858 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:59:52.859 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.466998+0000", 2024-09-14T04:59:52.859 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.155e-06 2024-09-14T04:59:52.859 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.859 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.859 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:59:52.859 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.467007+0000", 2024-09-14T04:59:52.859 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.0429999999999996e-06 2024-09-14T04:59:52.859 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.859 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.859 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:59:52.859 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.467019+0000", 2024-09-14T04:59:52.859 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1063000000000001e-05 2024-09-14T04:59:52.859 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.859 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.860 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:59:52.860 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.467027+0000", 2024-09-14T04:59:52.860 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.3569999999999998e-06 2024-09-14T04:59:52.860 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.860 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.860 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:59:52.860 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.467154+0000", 2024-09-14T04:59:52.860 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012677700000000001 2024-09-14T04:59:52.860 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.860 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.860 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:59:52.860 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.467174+0000", 2024-09-14T04:59:52.860 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0069e-05 2024-09-14T04:59:52.861 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.861 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.861 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:59:52.861 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.812746+0000", 2024-09-14T04:59:52.861 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.34557254500000001 2024-09-14T04:59:52.861 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.861 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.861 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:59:52.861 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.812817+0000", 2024-09-14T04:59:52.861 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1087999999999994e-05 2024-09-14T04:59:52.861 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.861 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:59:52.861 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.862 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.862 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.862 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45632.0:149716 55.a e1447/1442 55:53418830:::benchmark_data_smithi039_297155_object25783:head v 1447'9326, mlcod=1447'9323)", 2024-09-14T04:59:52.862 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:49:58.468218+0000", 2024-09-14T04:59:52.862 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 594.35087394100003, 2024-09-14T04:59:52.862 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.35008698100000002, 2024-09-14T04:59:52.862 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:59:52.862 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:59:52.862 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:59:52.862 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:59:52.862 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.862 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:59:52.862 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.468218+0000", 2024-09-14T04:59:52.863 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.863 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.863 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.863 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:59:52.863 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.468218+0000", 2024-09-14T04:59:52.863 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.863 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.863 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.863 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:59:52.863 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.468219+0000", 2024-09-14T04:59:52.863 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.3799999999999996e-07 2024-09-14T04:59:52.863 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.863 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.864 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:59:52.864 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.468224+0000", 2024-09-14T04:59:52.864 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4319999999999998e-06 2024-09-14T04:59:52.864 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.864 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.864 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:59:52.864 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.468233+0000", 2024-09-14T04:59:52.864 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.7860000000000002e-06 2024-09-14T04:59:52.864 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.864 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.864 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:59:52.864 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.468238+0000", 2024-09-14T04:59:52.865 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.578e-06 2024-09-14T04:59:52.865 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.865 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.865 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:59:52.865 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.470712+0000", 2024-09-14T04:59:52.865 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0024747480000000001 2024-09-14T04:59:52.865 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.865 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.865 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:59:52.865 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.470729+0000", 2024-09-14T04:59:52.865 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6850999999999998e-05 2024-09-14T04:59:52.865 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.865 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.866 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:59:52.866 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.818114+0000", 2024-09-14T04:59:52.866 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.34738492500000001 2024-09-14T04:59:52.866 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.866 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.866 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:59:52.866 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.818305+0000", 2024-09-14T04:59:52.866 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00019092299999999999 2024-09-14T04:59:52.866 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.866 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:59:52.866 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.866 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.866 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.867 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45632.0:149722 55.2 e1447/1423 55:43c787a8:::benchmark_data_smithi039_297155_object25789:head v 1447'9265, mlcod=1447'9262)", 2024-09-14T04:59:52.867 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:49:58.468888+0000", 2024-09-14T04:59:52.867 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 594.35020440899996, 2024-09-14T04:59:52.867 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.34864009499999998, 2024-09-14T04:59:52.867 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:59:52.867 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:59:52.867 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:59:52.867 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:59:52.867 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.867 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:59:52.867 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.468888+0000", 2024-09-14T04:59:52.867 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.867 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.867 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.868 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:59:52.868 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.468888+0000", 2024-09-14T04:59:52.868 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.868 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.868 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.868 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:59:52.868 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.468888+0000", 2024-09-14T04:59:52.868 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8700000000000005e-07 2024-09-14T04:59:52.868 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.868 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.868 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:59:52.868 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.468894+0000", 2024-09-14T04:59:52.868 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6300000000000003e-06 2024-09-14T04:59:52.868 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.869 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.869 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:59:52.869 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.468902+0000", 2024-09-14T04:59:52.869 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.089e-06 2024-09-14T04:59:52.869 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.869 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.869 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:59:52.869 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.468906+0000", 2024-09-14T04:59:52.869 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2080000000000002e-06 2024-09-14T04:59:52.869 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.869 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.869 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:59:52.870 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.470699+0000", 2024-09-14T04:59:52.870 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0017923450000000001 2024-09-14T04:59:52.870 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.870 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.870 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:59:52.870 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.470716+0000", 2024-09-14T04:59:52.870 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7200000000000001e-05 2024-09-14T04:59:52.870 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.870 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.870 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:59:52.870 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.817230+0000", 2024-09-14T04:59:52.870 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.34651453700000001 2024-09-14T04:59:52.870 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.870 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.871 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:59:52.871 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.817528+0000", 2024-09-14T04:59:52.871 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000297399 2024-09-14T04:59:52.871 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.871 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:59:52.871 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.871 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.871 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.871 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45632.0:149721 55.9 e1447/1442 55:963f2b16:::benchmark_data_smithi039_297155_object25788:head v 1447'9345, mlcod=1447'9342)", 2024-09-14T04:59:52.871 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:49:58.469892+0000", 2024-09-14T04:59:52.871 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 594.34920035200003, 2024-09-14T04:59:52.871 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.34057343899999998, 2024-09-14T04:59:52.871 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:59:52.871 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:59:52.872 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:59:52.872 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:59:52.872 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.872 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:59:52.872 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.469892+0000", 2024-09-14T04:59:52.872 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.872 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.872 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.872 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:59:52.872 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.469892+0000", 2024-09-14T04:59:52.872 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.872 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.872 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.872 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:59:52.873 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.469892+0000", 2024-09-14T04:59:52.873 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3099999999999999e-07 2024-09-14T04:59:52.873 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.873 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.873 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:59:52.873 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.469895+0000", 2024-09-14T04:59:52.873 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.7070000000000001e-06 2024-09-14T04:59:52.873 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.873 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.873 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:59:52.873 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.469899+0000", 2024-09-14T04:59:52.873 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.9990000000000002e-06 2024-09-14T04:59:52.873 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.873 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.874 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:59:52.874 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.469901+0000", 2024-09-14T04:59:52.874 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.6089999999999999e-06 2024-09-14T04:59:52.874 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.874 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.874 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:59:52.874 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.469941+0000", 2024-09-14T04:59:52.874 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.947e-05 2024-09-14T04:59:52.874 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.874 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.874 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:59:52.874 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.469953+0000", 2024-09-14T04:59:52.874 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2262999999999999e-05 2024-09-14T04:59:52.874 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.875 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.875 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:59:52.875 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.810412+0000", 2024-09-14T04:59:52.875 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.34045959399999998 2024-09-14T04:59:52.875 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.875 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.875 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:59:52.875 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.810465+0000", 2024-09-14T04:59:52.875 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.2565999999999998e-05 2024-09-14T04:59:52.875 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.875 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:59:52.875 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.875 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.875 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.876 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45632.0:149720 55.4 e1447/1419 55:25ea9b58:::benchmark_data_smithi039_297155_object25787:head v 1447'9179, mlcod=1447'9177)", 2024-09-14T04:59:52.876 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:49:58.471314+0000", 2024-09-14T04:59:52.876 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 594.34777841899995, 2024-09-14T04:59:52.876 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.33924061, 2024-09-14T04:59:52.876 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:59:52.876 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:59:52.876 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:59:52.876 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:59:52.876 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.876 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:59:52.876 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.471314+0000", 2024-09-14T04:59:52.876 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.876 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.876 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.877 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:59:52.877 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.471314+0000", 2024-09-14T04:59:52.877 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.877 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.877 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.877 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:59:52.877 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.471314+0000", 2024-09-14T04:59:52.877 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9699999999999995e-07 2024-09-14T04:59:52.877 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.877 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.877 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:59:52.877 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.471321+0000", 2024-09-14T04:59:52.877 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.263e-06 2024-09-14T04:59:52.878 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.878 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.878 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:59:52.878 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.471331+0000", 2024-09-14T04:59:52.878 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0801e-05 2024-09-14T04:59:52.878 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.878 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.878 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:59:52.878 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.471338+0000", 2024-09-14T04:59:52.878 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.0260000000000002e-06 2024-09-14T04:59:52.878 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.879 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.879 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:59:52.879 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.471406+0000", 2024-09-14T04:59:52.879 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7558000000000006e-05 2024-09-14T04:59:52.879 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.879 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.879 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:59:52.879 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.471424+0000", 2024-09-14T04:59:52.879 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8301999999999999e-05 2024-09-14T04:59:52.879 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.880 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.880 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:59:52.880 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.810524+0000", 2024-09-14T04:59:52.880 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.33909992900000002 2024-09-14T04:59:52.880 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.880 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.880 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:59:52.880 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.810554+0000", 2024-09-14T04:59:52.880 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.9933999999999998e-05 2024-09-14T04:59:52.880 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.880 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:59:52.880 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.881 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.881 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.881 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop_reply(client.45632.0:149713 55.6 e1447/1419 ondisk, result = 0)", 2024-09-14T04:59:52.881 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:49:58.471358+0000", 2024-09-14T04:59:52.881 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 594.34773393, 2024-09-14T04:59:52.881 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.33911915100000001, 2024-09-14T04:59:52.881 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:59:52.881 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:59:52.881 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "started", 2024-09-14T04:59:52.881 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:59:52.881 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.881 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:59:52.881 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.471358+0000", 2024-09-14T04:59:52.882 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.882 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.882 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.882 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:59:52.882 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.471358+0000", 2024-09-14T04:59:52.882 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.882 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.882 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.882 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:59:52.882 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.471358+0000", 2024-09-14T04:59:52.882 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.34e-07 2024-09-14T04:59:52.882 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.882 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.882 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:59:52.882 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.471361+0000", 2024-09-14T04:59:52.883 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.7800000000000001e-06 2024-09-14T04:59:52.883 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.883 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.883 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:59:52.883 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.471368+0000", 2024-09-14T04:59:52.883 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1160000000000001e-06 2024-09-14T04:59:52.883 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.883 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.883 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:59:52.883 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.471373+0000", 2024-09-14T04:59:52.883 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6580000000000001e-06 2024-09-14T04:59:52.883 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.883 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.883 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:59:52.883 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.471982+0000", 2024-09-14T04:59:52.884 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00060948899999999995 2024-09-14T04:59:52.884 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.884 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.884 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:59:52.884 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.471989+0000", 2024-09-14T04:59:52.884 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.64e-06 2024-09-14T04:59:52.884 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.884 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.884 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:59:52.884 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.810477+0000", 2024-09-14T04:59:52.884 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.338488134 2024-09-14T04:59:52.884 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.884 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:59:52.885 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.885 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.885 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.885 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.45632.0:149724 55.6 55:6eb74f36:::benchmark_data_smithi039_297155_object25791:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1447)", 2024-09-14T04:59:52.885 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:49:58.471380+0000", 2024-09-14T04:59:52.885 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 594.34771157800003, 2024-09-14T04:59:52.885 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.34584694599999999, 2024-09-14T04:59:52.885 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:59:52.885 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:59:52.885 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:59:52.885 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:59:52.885 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.45632", 2024-09-14T04:59:52.885 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2008336247", 2024-09-14T04:59:52.886 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 149724 2024-09-14T04:59:52.886 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.886 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:59:52.886 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.886 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:59:52.886 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.471380+0000", 2024-09-14T04:59:52.886 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.886 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.886 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.886 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:59:52.886 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.471380+0000", 2024-09-14T04:59:52.886 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.886 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.887 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.887 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:59:52.887 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.471382+0000", 2024-09-14T04:59:52.887 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.621e-06 2024-09-14T04:59:52.887 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.887 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.887 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:59:52.887 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.471387+0000", 2024-09-14T04:59:52.887 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.31e-06 2024-09-14T04:59:52.887 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.887 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.887 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:59:52.887 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.471399+0000", 2024-09-14T04:59:52.888 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1338000000000001e-05 2024-09-14T04:59:52.888 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.888 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.888 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:59:52.888 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.471405+0000", 2024-09-14T04:59:52.888 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7499999999999997e-06 2024-09-14T04:59:52.888 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.888 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.888 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:59:52.888 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.810519+0000", 2024-09-14T04:59:52.888 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.33911328000000002 2024-09-14T04:59:52.888 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.888 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.889 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:59:52.889 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.810658+0000", 2024-09-14T04:59:52.889 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013882900000000001 2024-09-14T04:59:52.889 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.889 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.889 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 5,7", 2024-09-14T04:59:52.889 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.810779+0000", 2024-09-14T04:59:52.889 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000121772 2024-09-14T04:59:52.889 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.889 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.889 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:59:52.889 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.815343+0000", 2024-09-14T04:59:52.890 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0045635190000000003 2024-09-14T04:59:52.890 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.890 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.890 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:59:52.890 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.816130+0000", 2024-09-14T04:59:52.890 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00078669899999999997 2024-09-14T04:59:52.890 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.890 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.890 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:59:52.890 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.817066+0000", 2024-09-14T04:59:52.890 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00093642300000000003 2024-09-14T04:59:52.890 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.891 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.891 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:59:52.891 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.817125+0000", 2024-09-14T04:59:52.891 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.9237000000000002e-05 2024-09-14T04:59:52.891 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.891 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.891 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:59:52.891 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.817227+0000", 2024-09-14T04:59:52.891 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000102168 2024-09-14T04:59:52.891 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.891 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:59:52.891 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.891 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.892 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.892 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45632.0:149727 55.7 e1447/1419 55:e5894293:::benchmark_data_smithi039_297155_object25794:head v 1447'9518, mlcod=1447'9516)", 2024-09-14T04:59:52.892 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:49:58.472251+0000", 2024-09-14T04:59:52.892 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 594.34684141499997, 2024-09-14T04:59:52.892 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.34478126100000001, 2024-09-14T04:59:52.892 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:59:52.892 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:59:52.892 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:59:52.892 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:59:52.892 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.892 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:59:52.892 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.472251+0000", 2024-09-14T04:59:52.892 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.893 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.893 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.893 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:59:52.893 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.472251+0000", 2024-09-14T04:59:52.893 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.893 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.893 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.893 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:59:52.893 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.472251+0000", 2024-09-14T04:59:52.893 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3200000000000005e-07 2024-09-14T04:59:52.893 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.893 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.893 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:59:52.894 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.472256+0000", 2024-09-14T04:59:52.894 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3340000000000001e-06 2024-09-14T04:59:52.894 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.894 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.894 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:59:52.894 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.810376+0000", 2024-09-14T04:59:52.894 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.33811989399999998 2024-09-14T04:59:52.894 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.894 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.894 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:59:52.895 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.810384+0000", 2024-09-14T04:59:52.895 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6070000000000002e-06 2024-09-14T04:59:52.895 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.895 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.895 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:59:52.895 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.810603+0000", 2024-09-14T04:59:52.895 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00021905199999999999 2024-09-14T04:59:52.895 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.895 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.895 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:59:52.895 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.810621+0000", 2024-09-14T04:59:52.895 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8493000000000001e-05 2024-09-14T04:59:52.895 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.895 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.896 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:59:52.896 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.816995+0000", 2024-09-14T04:59:52.896 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0063738670000000001 2024-09-14T04:59:52.896 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.896 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.896 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:59:52.896 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.817032+0000", 2024-09-14T04:59:52.896 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.6482000000000001e-05 2024-09-14T04:59:52.896 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.896 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:59:52.896 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.896 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.896 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.897 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.45632.0:149736 55.2 e1447/1423 55:43e0eece:::benchmark_data_smithi039_297155_object25803:head v 1447'9266, mlcod=1447'9262)", 2024-09-14T04:59:52.897 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:49:58.501550+0000", 2024-09-14T04:59:52.897 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 594.31754202100001, 2024-09-14T04:59:52.897 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.32927340700000002, 2024-09-14T04:59:52.897 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:59:52.897 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:59:52.897 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:59:52.897 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:59:52.897 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.897 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:59:52.897 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.501550+0000", 2024-09-14T04:59:52.897 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.897 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.898 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.898 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:59:52.898 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.501550+0000", 2024-09-14T04:59:52.898 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.898 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.898 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.898 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:59:52.898 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.501551+0000", 2024-09-14T04:59:52.898 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3349999999999999e-06 2024-09-14T04:59:52.898 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.898 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.898 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:59:52.899 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.501560+0000", 2024-09-14T04:59:52.899 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.7839999999999992e-06 2024-09-14T04:59:52.899 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.899 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.899 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:59:52.899 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.810364+0000", 2024-09-14T04:59:52.899 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.30880370699999998 2024-09-14T04:59:52.899 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.899 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.900 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:59:52.900 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.810385+0000", 2024-09-14T04:59:52.900 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1373e-05 2024-09-14T04:59:52.900 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.900 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.900 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:59:52.900 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.810508+0000", 2024-09-14T04:59:52.901 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000122378 2024-09-14T04:59:52.901 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.901 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.901 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:59:52.901 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.810532+0000", 2024-09-14T04:59:52.901 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4335e-05 2024-09-14T04:59:52.901 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.901 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.901 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:59:52.901 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.830778+0000", 2024-09-14T04:59:52.901 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.020246574 2024-09-14T04:59:52.901 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.901 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.902 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:59:52.902 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:49:58.830823+0000", 2024-09-14T04:59:52.902 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4920999999999999e-05 2024-09-14T04:59:52.902 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.902 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:59:52.902 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.902 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.902 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.902 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:37069 56.4 e1474/1470 56:297bb078:::benchmark_data_smithi039_330847_object37068:head v 1474'2387, mlcod=1474'2384)", 2024-09-14T04:59:52.902 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:52:45.859051+0000", 2024-09-14T04:59:52.902 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 426.96004076200001, 2024-09-14T04:59:52.902 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.30583859899999999, 2024-09-14T04:59:52.902 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:59:52.902 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:59:52.903 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:59:52.903 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:59:52.903 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.903 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:59:52.903 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.859051+0000", 2024-09-14T04:59:52.903 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.903 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.903 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.903 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:59:52.903 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.859051+0000", 2024-09-14T04:59:52.903 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.903 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.903 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.903 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:59:52.904 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.859052+0000", 2024-09-14T04:59:52.904 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9100000000000003e-07 2024-09-14T04:59:52.904 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.904 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.904 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:59:52.904 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.859106+0000", 2024-09-14T04:59:52.904 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4181999999999997e-05 2024-09-14T04:59:52.904 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.904 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.904 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:59:52.904 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.859113+0000", 2024-09-14T04:59:52.904 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1069999999999999e-06 2024-09-14T04:59:52.905 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.905 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.905 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:59:52.905 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.859118+0000", 2024-09-14T04:59:52.905 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6580000000000001e-06 2024-09-14T04:59:52.905 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.905 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.905 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:59:52.905 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.859157+0000", 2024-09-14T04:59:52.905 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.9171000000000002e-05 2024-09-14T04:59:52.905 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.905 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.905 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:59:52.906 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.859169+0000", 2024-09-14T04:59:52.906 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1800000000000001e-05 2024-09-14T04:59:52.906 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.906 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.906 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:59:52.906 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.164809+0000", 2024-09-14T04:59:52.906 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.30564031699999999 2024-09-14T04:59:52.906 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.906 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.906 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:59:52.906 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.164890+0000", 2024-09-14T04:59:52.906 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0673000000000005e-05 2024-09-14T04:59:52.906 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.907 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:59:52.907 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.907 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.907 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.907 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:37074 56.7 e1474/1470 56:eae87ae0:::benchmark_data_smithi039_330847_object37073:head v 1474'2331, mlcod=1474'2328)", 2024-09-14T04:59:52.907 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:52:45.860988+0000", 2024-09-14T04:59:52.907 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 426.95810367199999, 2024-09-14T04:59:52.907 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.30461201700000001, 2024-09-14T04:59:52.907 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:59:52.907 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:59:52.907 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:59:52.907 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:59:52.907 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.908 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:59:52.908 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.860988+0000", 2024-09-14T04:59:52.908 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.908 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.908 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.908 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:59:52.908 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.860988+0000", 2024-09-14T04:59:52.908 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.908 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.908 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.908 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:59:52.908 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.860989+0000", 2024-09-14T04:59:52.908 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.3900000000000004e-07 2024-09-14T04:59:52.909 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.909 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.909 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:59:52.909 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.861010+0000", 2024-09-14T04:59:52.909 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1267000000000002e-05 2024-09-14T04:59:52.909 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.909 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.909 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:59:52.909 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.160651+0000", 2024-09-14T04:59:52.909 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.29964028199999998 2024-09-14T04:59:52.909 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.909 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.909 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:59:52.910 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.160662+0000", 2024-09-14T04:59:52.910 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0835e-05 2024-09-14T04:59:52.910 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.910 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.910 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:59:52.910 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.160746+0000", 2024-09-14T04:59:52.910 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.4307000000000004e-05 2024-09-14T04:59:52.910 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.910 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.910 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:59:52.910 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.160765+0000", 2024-09-14T04:59:52.910 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9181000000000002e-05 2024-09-14T04:59:52.910 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.910 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.911 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:59:52.911 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.165546+0000", 2024-09-14T04:59:52.911 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0047813839999999996 2024-09-14T04:59:52.911 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.911 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.911 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:59:52.911 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.165600+0000", 2024-09-14T04:59:52.911 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3922000000000003e-05 2024-09-14T04:59:52.911 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.911 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:59:52.911 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.911 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.911 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.911 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:34402 57.0 57:0b7b4153:::benchmark_data_smithi039_365565_object34401:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T04:59:52.911 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:57:13.467717+0000", 2024-09-14T04:59:52.912 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 159.35137530899999, 2024-09-14T04:59:52.912 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.43708126800000002, 2024-09-14T04:59:52.912 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:59:52.912 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:59:52.912 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:59:52.912 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:59:52.912 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T04:59:52.912 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T04:59:52.912 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 34402 2024-09-14T04:59:52.912 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.912 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:59:52.912 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.912 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:59:52.912 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.467717+0000", 2024-09-14T04:59:52.912 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.913 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.913 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.913 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:59:52.913 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.467717+0000", 2024-09-14T04:59:52.913 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.913 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.913 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.913 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:59:52.913 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.467717+0000", 2024-09-14T04:59:52.913 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5600000000000001e-07 2024-09-14T04:59:52.913 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.913 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.913 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:59:52.914 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.467758+0000", 2024-09-14T04:59:52.914 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.0911000000000002e-05 2024-09-14T04:59:52.914 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.914 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.914 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:59:52.914 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.467765+0000", 2024-09-14T04:59:52.914 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.2180000000000002e-06 2024-09-14T04:59:52.914 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.914 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.914 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:59:52.914 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.467771+0000", 2024-09-14T04:59:52.914 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4410000000000001e-06 2024-09-14T04:59:52.914 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.914 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.915 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:59:52.915 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.469591+0000", 2024-09-14T04:59:52.915 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0018204040000000001 2024-09-14T04:59:52.915 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.915 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.915 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:59:52.915 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.469834+0000", 2024-09-14T04:59:52.915 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00024333800000000001 2024-09-14T04:59:52.915 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.915 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.915 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:59:52.915 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.470676+0000", 2024-09-14T04:59:52.915 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000841554 2024-09-14T04:59:52.915 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.915 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.916 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:59:52.916 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.474378+0000", 2024-09-14T04:59:52.916 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.003701738 2024-09-14T04:59:52.916 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.916 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.916 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:59:52.916 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.474392+0000", 2024-09-14T04:59:52.916 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4141000000000001e-05 2024-09-14T04:59:52.916 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.916 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.916 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:59:52.916 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.904687+0000", 2024-09-14T04:59:52.916 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.43029504400000002 2024-09-14T04:59:52.917 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.917 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.917 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:59:52.917 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.904751+0000", 2024-09-14T04:59:52.917 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.4424999999999994e-05 2024-09-14T04:59:52.917 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.917 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.917 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:59:52.917 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.904798+0000", 2024-09-14T04:59:52.917 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6597999999999997e-05 2024-09-14T04:59:52.917 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.917 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:59:52.917 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.917 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.918 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.918 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:34438 57.0 57:065b00bc:::benchmark_data_smithi039_365565_object34437:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T04:59:52.918 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:57:13.484152+0000", 2024-09-14T04:59:52.918 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 159.33493973, 2024-09-14T04:59:52.918 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.42189383600000002, 2024-09-14T04:59:52.918 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:59:52.918 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:59:52.918 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:59:52.918 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:59:52.918 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T04:59:52.918 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T04:59:52.918 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 34438 2024-09-14T04:59:52.919 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.919 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:59:52.919 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.919 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:59:52.919 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.484152+0000", 2024-09-14T04:59:52.919 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.919 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.919 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.919 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:59:52.919 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.484152+0000", 2024-09-14T04:59:52.919 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.919 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.920 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.920 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:59:52.920 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.484154+0000", 2024-09-14T04:59:52.920 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9860000000000001e-06 2024-09-14T04:59:52.920 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.920 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.920 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:59:52.920 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.484187+0000", 2024-09-14T04:59:52.920 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.2320000000000002e-05 2024-09-14T04:59:52.920 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.920 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.920 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:59:52.921 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.484204+0000", 2024-09-14T04:59:52.921 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7586999999999999e-05 2024-09-14T04:59:52.921 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.921 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.921 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:59:52.921 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.484212+0000", 2024-09-14T04:59:52.921 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.2719999999999997e-06 2024-09-14T04:59:52.921 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.921 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.921 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:59:52.921 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.484266+0000", 2024-09-14T04:59:52.921 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3757e-05 2024-09-14T04:59:52.922 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.922 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.922 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:59:52.922 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.484454+0000", 2024-09-14T04:59:52.922 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000188257 2024-09-14T04:59:52.922 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.922 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.922 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:59:52.922 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.484618+0000", 2024-09-14T04:59:52.922 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00016396499999999999 2024-09-14T04:59:52.922 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.922 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.922 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:59:52.922 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.488095+0000", 2024-09-14T04:59:52.922 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0034769409999999999 2024-09-14T04:59:52.923 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.923 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.923 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:59:52.923 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.488300+0000", 2024-09-14T04:59:52.923 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00020430700000000001 2024-09-14T04:59:52.923 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.923 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.923 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:59:52.923 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.905939+0000", 2024-09-14T04:59:52.923 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.417639447 2024-09-14T04:59:52.923 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.923 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.923 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:59:52.923 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.906005+0000", 2024-09-14T04:59:52.924 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.5717999999999997e-05 2024-09-14T04:59:52.924 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.924 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.924 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:59:52.924 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.906046+0000", 2024-09-14T04:59:52.924 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.1279000000000002e-05 2024-09-14T04:59:52.924 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.924 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:59:52.924 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.924 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.924 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.924 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:34442 57.0 57:013cd97f:::benchmark_data_smithi039_365565_object34441:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T04:59:52.924 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:57:13.485197+0000", 2024-09-14T04:59:52.925 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 159.333894757, 2024-09-14T04:59:52.925 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.42272372499999999, 2024-09-14T04:59:52.925 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:59:52.925 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:59:52.925 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:59:52.925 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:59:52.925 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T04:59:52.925 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T04:59:52.925 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 34442 2024-09-14T04:59:52.925 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.925 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:59:52.925 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.925 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:59:52.925 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.485197+0000", 2024-09-14T04:59:52.926 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.926 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.926 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.926 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:59:52.926 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.485197+0000", 2024-09-14T04:59:52.926 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.926 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.926 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.926 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:59:52.926 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.485198+0000", 2024-09-14T04:59:52.926 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.02e-07 2024-09-14T04:59:52.926 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.926 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.927 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:59:52.927 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.485221+0000", 2024-09-14T04:59:52.927 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2986e-05 2024-09-14T04:59:52.927 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.927 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.927 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:59:52.927 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.485232+0000", 2024-09-14T04:59:52.927 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1035e-05 2024-09-14T04:59:52.927 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.927 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.927 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:59:52.927 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.485240+0000", 2024-09-14T04:59:52.927 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.5780000000000001e-06 2024-09-14T04:59:52.927 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.928 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.928 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:59:52.928 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.486164+0000", 2024-09-14T04:59:52.928 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00092453100000000003 2024-09-14T04:59:52.928 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.928 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.928 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:59:52.928 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.486356+0000", 2024-09-14T04:59:52.928 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00019142000000000001 2024-09-14T04:59:52.928 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.928 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.928 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:59:52.929 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.486490+0000", 2024-09-14T04:59:52.929 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013404000000000001 2024-09-14T04:59:52.929 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.929 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.929 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:59:52.929 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.488509+0000", 2024-09-14T04:59:52.929 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0020197090000000002 2024-09-14T04:59:52.929 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.929 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.929 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:59:52.929 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.489162+0000", 2024-09-14T04:59:52.929 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00065259200000000002 2024-09-14T04:59:52.929 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.930 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.930 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:59:52.930 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.907856+0000", 2024-09-14T04:59:52.930 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.418694281 2024-09-14T04:59:52.930 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.930 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.930 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:59:52.930 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.907892+0000", 2024-09-14T04:59:52.930 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.5346999999999999e-05 2024-09-14T04:59:52.930 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.930 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.930 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:59:52.931 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.907921+0000", 2024-09-14T04:59:52.931 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.9303999999999999e-05 2024-09-14T04:59:52.931 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.931 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:59:52.931 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.931 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.931 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.931 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:34456 57.0 57:02c69cda:::benchmark_data_smithi039_365565_object34455:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T04:59:52.931 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:57:13.491976+0000", 2024-09-14T04:59:52.931 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 159.32711612, 2024-09-14T04:59:52.931 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.41608771700000002, 2024-09-14T04:59:52.931 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:59:52.931 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:59:52.932 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:59:52.932 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:59:52.932 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T04:59:52.932 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T04:59:52.932 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 34456 2024-09-14T04:59:52.932 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.932 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:59:52.932 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.932 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:59:52.932 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.491976+0000", 2024-09-14T04:59:52.932 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.932 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.932 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.933 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:59:52.933 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.491976+0000", 2024-09-14T04:59:52.933 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.933 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.933 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.933 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:59:52.933 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.491977+0000", 2024-09-14T04:59:52.933 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.443e-06 2024-09-14T04:59:52.933 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.933 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.933 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:59:52.933 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.492000+0000", 2024-09-14T04:59:52.933 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3102e-05 2024-09-14T04:59:52.933 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.934 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.934 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:59:52.934 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:52.935 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.492011+0000", 2024-09-14T04:59:52.935 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.061e-05 2024-09-14T04:59:52.935 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.935 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.935 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:59:52.936 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.492019+0000", 2024-09-14T04:59:52.936 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6450000000000005e-06 2024-09-14T04:59:52.936 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.936 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.936 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:59:52.936 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.492068+0000", 2024-09-14T04:59:52.936 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9487999999999998e-05 2024-09-14T04:59:52.936 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.936 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.936 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:59:52.936 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.492175+0000", 2024-09-14T04:59:52.936 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000106523 2024-09-14T04:59:52.936 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.937 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.937 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:59:52.937 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.492280+0000", 2024-09-14T04:59:52.937 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000105577 2024-09-14T04:59:52.937 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.937 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.937 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:59:52.937 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.494189+0000", 2024-09-14T04:59:52.937 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0019083329999999999 2024-09-14T04:59:52.937 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.937 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.937 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:59:52.937 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.494337+0000", 2024-09-14T04:59:52.938 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000148753 2024-09-14T04:59:52.938 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.938 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.938 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:59:52.938 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.907973+0000", 2024-09-14T04:59:52.938 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.41363581399999999 2024-09-14T04:59:52.938 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.938 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.938 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:59:52.938 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.908018+0000", 2024-09-14T04:59:52.938 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5003000000000003e-05 2024-09-14T04:59:52.938 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.938 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.938 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:59:52.938 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.908064+0000", 2024-09-14T04:59:52.939 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5426000000000002e-05 2024-09-14T04:59:52.939 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.939 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:59:52.939 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.939 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.939 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.939 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:34458 57.b 57:d7441ced:::benchmark_data_smithi039_365565_object34457:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T04:59:52.939 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:57:13.492938+0000", 2024-09-14T04:59:52.939 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 159.32615394499999, 2024-09-14T04:59:52.939 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.410365072, 2024-09-14T04:59:52.939 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:59:52.939 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:59:52.939 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:59:52.940 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:59:52.940 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T04:59:52.940 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T04:59:52.940 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 34458 2024-09-14T04:59:52.940 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.940 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:59:52.940 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.940 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:59:52.940 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.492938+0000", 2024-09-14T04:59:52.940 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.940 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.940 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.940 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:59:52.941 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.492938+0000", 2024-09-14T04:59:52.941 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.941 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.941 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.941 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:59:52.941 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.492939+0000", 2024-09-14T04:59:52.941 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1900000000000002e-07 2024-09-14T04:59:52.941 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.941 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.941 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:59:52.941 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.492963+0000", 2024-09-14T04:59:52.941 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4712999999999999e-05 2024-09-14T04:59:52.941 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.941 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.942 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:59:52.942 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.492974+0000", 2024-09-14T04:59:52.942 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0922e-05 2024-09-14T04:59:52.942 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.942 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.942 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:59:52.942 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.492981+0000", 2024-09-14T04:59:52.942 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.6030000000000001e-06 2024-09-14T04:59:52.942 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.942 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.942 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:59:52.942 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.493039+0000", 2024-09-14T04:59:52.942 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.8365000000000001e-05 2024-09-14T04:59:52.943 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.943 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.943 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:59:52.943 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.493179+0000", 2024-09-14T04:59:52.943 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013982799999999999 2024-09-14T04:59:52.943 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.943 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.943 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 4,6", 2024-09-14T04:59:52.943 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.493335+0000", 2024-09-14T04:59:52.943 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015545199999999999 2024-09-14T04:59:52.943 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.943 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.943 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:59:52.944 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.496147+0000", 2024-09-14T04:59:52.944 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0028121930000000002 2024-09-14T04:59:52.944 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.944 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.944 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:59:52.944 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.496449+0000", 2024-09-14T04:59:52.944 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00030275000000000001 2024-09-14T04:59:52.944 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.944 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.944 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:59:52.944 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.903211+0000", 2024-09-14T04:59:52.944 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.40676116099999998 2024-09-14T04:59:52.944 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.945 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.945 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:59:52.945 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.903256+0000", 2024-09-14T04:59:52.945 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5325999999999999e-05 2024-09-14T04:59:52.945 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.945 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.945 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:59:52.945 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.903303+0000", 2024-09-14T04:59:52.945 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7139999999999999e-05 2024-09-14T04:59:52.945 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.945 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:59:52.945 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.945 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.945 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.946 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:34461 57.b 57:d8ab32cf:::benchmark_data_smithi039_365565_object34460:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T04:59:52.946 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:57:13.493451+0000", 2024-09-14T04:59:52.946 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 159.32564123399999, 2024-09-14T04:59:52.946 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.41006606699999998, 2024-09-14T04:59:52.946 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:59:52.946 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:59:52.946 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:59:52.946 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:59:52.946 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T04:59:52.946 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T04:59:52.946 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 34461 2024-09-14T04:59:52.946 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.946 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:59:52.946 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.947 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:59:52.947 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.493451+0000", 2024-09-14T04:59:52.947 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.947 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.947 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.947 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:59:52.947 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.493451+0000", 2024-09-14T04:59:52.947 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.947 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.947 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.947 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:59:52.947 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.493452+0000", 2024-09-14T04:59:52.947 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.4200000000000004e-07 2024-09-14T04:59:52.947 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.948 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.948 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:59:52.948 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.493504+0000", 2024-09-14T04:59:52.948 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.2448000000000003e-05 2024-09-14T04:59:52.948 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.948 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.948 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:59:52.948 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.493515+0000", 2024-09-14T04:59:52.948 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0791000000000001e-05 2024-09-14T04:59:52.948 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.948 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.948 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:59:52.948 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.493523+0000", 2024-09-14T04:59:52.948 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.3289999999999994e-06 2024-09-14T04:59:52.949 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.949 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.949 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:59:52.949 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.494784+0000", 2024-09-14T04:59:52.949 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0012608109999999999 2024-09-14T04:59:52.949 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.949 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.949 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:59:52.949 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.494925+0000", 2024-09-14T04:59:52.949 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000141252 2024-09-14T04:59:52.949 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.949 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.949 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 4,6", 2024-09-14T04:59:52.950 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.495080+0000", 2024-09-14T04:59:52.950 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015435899999999999 2024-09-14T04:59:52.950 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.950 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.950 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:59:52.950 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.496265+0000", 2024-09-14T04:59:52.950 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001184977 2024-09-14T04:59:52.950 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.950 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.950 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:59:52.950 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.496542+0000", 2024-09-14T04:59:52.950 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00027730799999999998 2024-09-14T04:59:52.950 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.950 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.951 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:59:52.951 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.903461+0000", 2024-09-14T04:59:52.951 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.40691940599999998 2024-09-14T04:59:52.951 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.951 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.951 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:59:52.951 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.903494+0000", 2024-09-14T04:59:52.951 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.2249000000000002e-05 2024-09-14T04:59:52.951 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.951 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.951 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:59:52.951 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.903517+0000", 2024-09-14T04:59:52.951 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3195000000000001e-05 2024-09-14T04:59:52.951 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.951 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:59:52.952 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.952 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.952 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.952 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:34463 57.0 57:0b4e307a:::benchmark_data_smithi039_365565_object34462:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T04:59:52.952 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:57:13.495720+0000", 2024-09-14T04:59:52.952 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 159.323371743, 2024-09-14T04:59:52.952 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.41252305299999997, 2024-09-14T04:59:52.952 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T04:59:52.952 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T04:59:52.952 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T04:59:52.952 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T04:59:52.952 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T04:59:52.952 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T04:59:52.953 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 34463 2024-09-14T04:59:52.953 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.953 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T04:59:52.953 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.953 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T04:59:52.953 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.495720+0000", 2024-09-14T04:59:52.953 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.953 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.953 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.953 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T04:59:52.953 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.495720+0000", 2024-09-14T04:59:52.953 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T04:59:52.953 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.953 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.954 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T04:59:52.954 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.495721+0000", 2024-09-14T04:59:52.954 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0100000000000004e-07 2024-09-14T04:59:52.954 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.954 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.954 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T04:59:52.954 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.495740+0000", 2024-09-14T04:59:52.954 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8817999999999998e-05 2024-09-14T04:59:52.954 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.954 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.954 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T04:59:52.954 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.495749+0000", 2024-09-14T04:59:52.954 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.5030000000000003e-06 2024-09-14T04:59:52.954 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.954 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.955 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T04:59:52.955 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.495754+0000", 2024-09-14T04:59:52.955 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4240000000000003e-06 2024-09-14T04:59:52.955 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.955 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.955 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T04:59:52.955 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.495788+0000", 2024-09-14T04:59:52.955 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.4542000000000002e-05 2024-09-14T04:59:52.955 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.955 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.955 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T04:59:52.955 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.495872+0000", 2024-09-14T04:59:52.955 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.3380999999999993e-05 2024-09-14T04:59:52.956 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.956 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.956 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4", 2024-09-14T04:59:52.956 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.495953+0000", 2024-09-14T04:59:52.956 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.1767999999999994e-05 2024-09-14T04:59:52.956 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.956 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.956 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T04:59:52.956 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.497390+0000", 2024-09-14T04:59:52.956 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0014363240000000001 2024-09-14T04:59:52.956 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.956 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.956 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:59:52.956 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.497688+0000", 2024-09-14T04:59:52.957 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00029810699999999998 2024-09-14T04:59:52.957 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.957 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.957 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T04:59:52.957 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.908150+0000", 2024-09-14T04:59:52.957 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.41046164000000002 2024-09-14T04:59:52.957 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.957 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.957 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T04:59:52.957 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.908206+0000", 2024-09-14T04:59:52.957 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6706000000000002e-05 2024-09-14T04:59:52.957 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T04:59:52.957 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T04:59:52.957 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T04:59:52.958 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.908243+0000", 2024-09-14T04:59:52.958 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.7039000000000002e-05 2024-09-14T04:59:52.958 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.958 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:59:52.958 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.958 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T04:59:52.958 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T04:59:52.958 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T04:59:52.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:52.960 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:52.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:53.061 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:53.096 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:53.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:53.097 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:53.198 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:53.232 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:53.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:53.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:53.334 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:53.370 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:53.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:53.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:53.472 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:53.514 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:53.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:53.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:53.615 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:53.651 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:53.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:53.652 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:53.753 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:53.764 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:53 smithi039 ceph-mon[106470]: pgmap v3018: 99 pgs: 99 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2024-09-14T04:59:53.764 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:53 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:59:53.764 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:53 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:59:53.765 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:53 smithi039 ceph-mon[111235]: pgmap v3018: 99 pgs: 99 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2024-09-14T04:59:53.765 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:53 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:59:53.765 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:53 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:59:53.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:53 smithi129 ceph-mon[101316]: pgmap v3018: 99 pgs: 99 active+clean; 3.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2024-09-14T04:59:53.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:53 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T04:59:53.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:53 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T04:59:53.795 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:53.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:53.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:53.897 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:53.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:53.943 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:53.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:54.044 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:54.089 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:54.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:54.090 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:54.191 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:54.235 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:54.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:54.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:54.337 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:54.377 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:54.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:54.378 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:54.380 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:54 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3797152063' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:59:54.380 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:54 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3797152063' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:59:54.386 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T04:59:54.386 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T04:59:54.386 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T04:59:54.386 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T04:59:54.386 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T04:59:54.386 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T04:59:54.418 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T04:59:54.418 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T04:59:54.444 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T04:59:54.444 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T04:59:54.475 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T04:59:54.475 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T04:59:54.478 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:54.510 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T04:59:54.511 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T04:59:54.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:54.512 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:54.512 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:54.544 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T04:59:54.544 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T04:59:54.613 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:54.644 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:54.644 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:54.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:54.745 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:54.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:54 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3797152063' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T04:59:54.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:54.785 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:54.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:54.886 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:54.924 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:54.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:54.925 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:55.026 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:55.058 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:55.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:55.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:55.159 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:55.196 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:55.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:55.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:55.298 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:55.331 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:55.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:55.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:55.373 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-14T04:59:55.433 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:55.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:55.465 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:55.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:55.566 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:55.592 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:55.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:55.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:55.694 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:55.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:55.733 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:55.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:55.734 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:55 smithi039 ceph-mon[106470]: pgmap v3019: 99 pgs: 99 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 868 op/s 2024-09-14T04:59:55.734 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:55 smithi039 ceph-mon[111235]: pgmap v3019: 99 pgs: 99 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 868 op/s 2024-09-14T04:59:55.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:55 smithi129 ceph-mon[101316]: pgmap v3019: 99 pgs: 99 active+clean; 3.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 868 op/s 2024-09-14T04:59:55.834 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:55.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:55.873 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:55.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:55.974 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:56.003 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:56.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:56.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:56.105 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:56.135 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:56.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:56.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:56.237 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:56.270 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:56.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:56.271 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:56.372 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:56.409 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:56.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:56.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:56.510 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:56.545 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:56.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:56.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:56.647 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:56.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:56.689 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:56.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:56.710 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:59:56.790 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:56.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:56.827 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:56.828 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:56.929 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:56.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:56.962 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:56.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:57.041 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T04:59:57.063 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:57.092 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:57.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:57.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:57.194 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:57.235 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:57.236 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:57.236 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:57.337 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:57.368 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:57.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:57.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:57.470 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:57.508 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:57.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:57.509 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:57.610 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:57.641 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:57.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:57.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:57.743 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T04:59:57.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:57 smithi129 ceph-mon[101316]: pgmap v3020: 99 pgs: 99 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 851 op/s 2024-09-14T04:59:57.781 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:57.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:57.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T04:59:57.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:57 smithi039 ceph-mon[106470]: pgmap v3020: 99 pgs: 99 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 851 op/s 2024-09-14T04:59:57.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:57 smithi039 ceph-mon[111235]: pgmap v3020: 99 pgs: 99 active+clean; 3.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 851 op/s 2024-09-14T04:59:57.883 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:57.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:57.917 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:57.917 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:58.018 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:58.054 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:58.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:58.055 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:58.156 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:58.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:58.194 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:58.194 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:58.199 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T04:59:58.296 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:58.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:58.346 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:58.346 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:58.447 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:58.503 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:58.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:58.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:58.605 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T04:59:58.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:58.639 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:58.639 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T04:59:58.740 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T04:59:58.792 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:58.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:58.793 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T04:59:58.894 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:58.927 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:58.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:58.928 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:58.957 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 04:59:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:04:59:58] "GET /metrics HTTP/1.1" 200 36731 "" "Prometheus/2.51.0" 2024-09-14T04:59:59.028 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T04:59:59.061 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:59.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:59.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T04:59:59.163 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:59.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:59.205 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:59.205 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:59.306 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T04:59:59.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:59.336 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:59.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T04:59:59.408 INFO:teuthology.orchestra.run.smithi039.stderr:set pool 1 pg_num to 21 2024-09-14T04:59:59.437 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:59.466 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:59.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:59.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:59.567 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T04:59:59.598 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:59.599 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:59.600 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T04:59:59.700 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T04:59:59.741 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T04:59:59.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:59.742 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T04:59:59.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:59 smithi129 ceph-mon[101316]: pgmap v3021: 99 pgs: 99 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2024-09-14T04:59:59.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:59 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2968226301' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-09-14T04:59:59.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 04:59:59 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-09-14T04:59:59.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:59 smithi039 ceph-mon[106470]: pgmap v3021: 99 pgs: 99 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2024-09-14T04:59:59.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:59 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2968226301' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-09-14T04:59:59.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 04:59:59 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-09-14T04:59:59.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:59 smithi039 ceph-mon[111235]: pgmap v3021: 99 pgs: 99 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 854 op/s 2024-09-14T04:59:59.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:59 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2968226301' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-09-14T04:59:59.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 04:59:59 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"} : dispatch 2024-09-14T04:59:59.844 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T04:59:59.879 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T04:59:59.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T04:59:59.880 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T04:59:59.981 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:00.012 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:00.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:00.013 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:00.114 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:00.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:00.149 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:00.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:00.250 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:00.280 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:00.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:00.281 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:00.382 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:00.417 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:00.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:00.418 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:00.519 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:00.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:00 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2024-09-14T05:00:00.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:00 smithi039 ceph-mon[106470]: osdmap e1542: 8 total, 8 up, 8 in 2024-09-14T05:00:00.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:00 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-09-14T05:00:00.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:00 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-09-14T05:00:00.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:00 smithi039 ceph-mon[106470]: overall HEALTH_OK 2024-09-14T05:00:00.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:00 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2024-09-14T05:00:00.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:00 smithi039 ceph-mon[111235]: osdmap e1542: 8 total, 8 up, 8 in 2024-09-14T05:00:00.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:00 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-09-14T05:00:00.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:00 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-09-14T05:00:00.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:00 smithi039 ceph-mon[111235]: overall HEALTH_OK 2024-09-14T05:00:00.549 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:00.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:00.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:00.651 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:00.691 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:00.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:00.692 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:00.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:00 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2024-09-14T05:00:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:00 smithi129 ceph-mon[101316]: osdmap e1542: 8 total, 8 up, 8 in 2024-09-14T05:00:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:00 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-09-14T05:00:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:00 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"} : dispatch 2024-09-14T05:00:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:00 smithi129 ceph-mon[101316]: overall HEALTH_OK 2024-09-14T05:00:00.792 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:00.820 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:00.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:00.821 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:00.922 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:00.976 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:00.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:00.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:01.078 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:01.105 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:01.105 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:01.106 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:01.208 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:01.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:01.246 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:01.246 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:01.347 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:01.375 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:01.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:01.376 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:01.477 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:01.510 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:01.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:01.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:01.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:01 smithi039 ceph-mon[106470]: pgmap v3023: 99 pgs: 99 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 867 op/s 2024-09-14T05:00:01.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:01 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2024-09-14T05:00:01.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:01 smithi039 ceph-mon[106470]: osdmap e1543: 8 total, 8 up, 8 in 2024-09-14T05:00:01.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:01 smithi039 ceph-mon[106470]: osdmap e1544: 8 total, 8 up, 8 in 2024-09-14T05:00:01.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:01 smithi039 ceph-mon[111235]: pgmap v3023: 99 pgs: 99 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 867 op/s 2024-09-14T05:00:01.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:01 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2024-09-14T05:00:01.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:01 smithi039 ceph-mon[111235]: osdmap e1543: 8 total, 8 up, 8 in 2024-09-14T05:00:01.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:01 smithi039 ceph-mon[111235]: osdmap e1544: 8 total, 8 up, 8 in 2024-09-14T05:00:01.611 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:01.641 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:01.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:01.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:01.743 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:01.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:01 smithi129 ceph-mon[101316]: pgmap v3023: 99 pgs: 99 active+clean; 2.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 867 op/s 2024-09-14T05:00:01.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:01 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2024-09-14T05:00:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:01 smithi129 ceph-mon[101316]: osdmap e1543: 8 total, 8 up, 8 in 2024-09-14T05:00:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:01 smithi129 ceph-mon[101316]: osdmap e1544: 8 total, 8 up, 8 in 2024-09-14T05:00:01.774 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:01.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:01.775 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:01.875 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:01.916 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:01.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:01.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:02.018 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:02.053 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:02.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:02.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:02.155 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:02.192 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:02.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:02.193 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:02.294 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:02.330 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:02.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:02.331 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:02.380 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:00:02.419 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T05:00:02.419 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:00:02.432 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:02.437 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is set 2024-09-14T05:00:02.459 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:02.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:02.460 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:02.515 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:02 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3839136126' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:00:02.515 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:02 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:00:02.516 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:02 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3839136126' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:00:02.516 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:02 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:00:02.561 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:02.597 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:02.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:02.598 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:02.699 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:02.736 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:02.737 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:02.737 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:02.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:02 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3839136126' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:00:02.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:02 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:00:02.838 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:02.874 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:02.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:02.875 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:02.976 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:03.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:03.009 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:03.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:03.110 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:03.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:03.151 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:03.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:03.252 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:03.292 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:03.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:03.293 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:03.394 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:03.428 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:03.429 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:03.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:03.530 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:03.565 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:03.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:03.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:03.667 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:03.705 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:03.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:03.706 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:03.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:03 smithi129 ceph-mon[101316]: pgmap v3026: 109 pgs: 10 unknown, 99 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 727 op/s 2024-09-14T05:00:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:03 smithi129 ceph-mon[101316]: 1.2 deep-scrub starts 2024-09-14T05:00:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:03 smithi129 ceph-mon[101316]: 1.2 deep-scrub ok 2024-09-14T05:00:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:03 smithi129 ceph-mon[101316]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:00:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:03 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T05:00:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:03 smithi129 ceph-mon[101316]: osdmap e1545: 8 total, 8 up, 8 in 2024-09-14T05:00:03.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:03 smithi039 ceph-mon[106470]: pgmap v3026: 109 pgs: 10 unknown, 99 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 727 op/s 2024-09-14T05:00:03.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:03 smithi039 ceph-mon[106470]: 1.2 deep-scrub starts 2024-09-14T05:00:03.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:03 smithi039 ceph-mon[106470]: 1.2 deep-scrub ok 2024-09-14T05:00:03.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:03 smithi039 ceph-mon[106470]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:00:03.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:03 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T05:00:03.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:03 smithi039 ceph-mon[106470]: osdmap e1545: 8 total, 8 up, 8 in 2024-09-14T05:00:03.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:03 smithi039 ceph-mon[111235]: pgmap v3026: 109 pgs: 10 unknown, 99 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 727 op/s 2024-09-14T05:00:03.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:03 smithi039 ceph-mon[111235]: 1.2 deep-scrub starts 2024-09-14T05:00:03.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:03 smithi039 ceph-mon[111235]: 1.2 deep-scrub ok 2024-09-14T05:00:03.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:03 smithi039 ceph-mon[111235]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:00:03.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:03 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T05:00:03.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:03 smithi039 ceph-mon[111235]: osdmap e1545: 8 total, 8 up, 8 in 2024-09-14T05:00:03.808 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:03.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:03.844 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:03.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:03.945 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:04.006 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:04.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:04.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:04.108 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:04.141 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:04.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:04.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:04.243 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:04.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:04.282 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:04.282 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:04.383 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:04.414 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:04.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:04.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:04.463 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-14T05:00:04.516 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:04.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:04.553 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:04.553 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:04.654 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:04.713 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:04.714 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:04.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:04.728 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:04 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-14T05:00:04.728 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:04 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-14T05:00:04.728 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:04 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-14T05:00:04.728 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:04 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-14T05:00:04.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:04 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-14T05:00:04.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:04 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"} : dispatch 2024-09-14T05:00:04.815 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:04.851 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:04.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:04.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:04.953 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:04.984 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:04.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:04.985 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:05.086 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:05.123 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:05.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:05.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:05.225 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:05.264 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:05.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:05.265 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:05.366 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:05.398 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:05.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:05.399 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:05.500 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:05.538 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:05.539 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:05.539 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:05.640 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:05.669 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:05.669 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:05.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:05.736 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:00:05.770 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:05 smithi129 ceph-mon[101316]: pgmap v3028: 109 pgs: 109 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 772 op/s 2024-09-14T05:00:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:05 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-14T05:00:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:05 smithi129 ceph-mon[101316]: osdmap e1546: 8 total, 8 up, 8 in 2024-09-14T05:00:05.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:05 smithi039 ceph-mon[106470]: pgmap v3028: 109 pgs: 109 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 772 op/s 2024-09-14T05:00:05.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:05 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-14T05:00:05.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:05 smithi039 ceph-mon[106470]: osdmap e1546: 8 total, 8 up, 8 in 2024-09-14T05:00:05.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:05 smithi039 ceph-mon[111235]: pgmap v3028: 109 pgs: 109 active+clean; 2.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 772 op/s 2024-09-14T05:00:05.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:05 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "21"}]': finished 2024-09-14T05:00:05.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:05 smithi039 ceph-mon[111235]: osdmap e1546: 8 total, 8 up, 8 in 2024-09-14T05:00:05.820 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:05.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:05.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:05.921 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:05.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:05.959 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:05.959 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:05.993 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T05:00:06.060 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:06.093 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:06.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:06.094 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:06.195 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:06.225 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:06.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:06.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:06.327 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:06.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:06.382 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:06.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:06.483 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:06.543 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:06.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:06.544 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:06.607 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:06 smithi039 ceph-mon[106470]: osdmap e1547: 8 total, 8 up, 8 in 2024-09-14T05:00:06.607 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:06 smithi039 ceph-mon[106470]: osdmap e1548: 8 total, 8 up, 8 in 2024-09-14T05:00:06.607 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:06 smithi039 ceph-mon[111235]: osdmap e1547: 8 total, 8 up, 8 in 2024-09-14T05:00:06.607 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:06 smithi039 ceph-mon[111235]: osdmap e1548: 8 total, 8 up, 8 in 2024-09-14T05:00:06.609 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T05:00:06.610 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T05:00:06.645 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:06.649 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T05:00:06.649 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T05:00:06.671 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:06.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:06.672 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:06.686 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T05:00:06.686 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T05:00:06.715 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T05:00:06.715 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T05:00:06.747 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T05:00:06.747 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T05:00:06.765 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-14T05:00:06.765 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T05:00:06.766 INFO:tasks.thrashosds.thrasher:Reweighting osd 5 to 0.8634178442937773 2024-09-14T05:00:06.766 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 5 0.8634178442937773 2024-09-14T05:00:06.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:06 smithi129 ceph-mon[101316]: osdmap e1547: 8 total, 8 up, 8 in 2024-09-14T05:00:06.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:06 smithi129 ceph-mon[101316]: osdmap e1548: 8 total, 8 up, 8 in 2024-09-14T05:00:06.773 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:06.781 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T05:00:06.781 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T05:00:06.798 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:06.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:06.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:06.814 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T05:00:06.814 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T05:00:06.854 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T05:00:06.855 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T05:00:06.900 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:06.932 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:06.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:06.933 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:07.038 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:07.066 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:07.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:07.067 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:07.168 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:07.204 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:07.204 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:07.205 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:07.305 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:07.337 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:07.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:07.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:07.439 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:07.468 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:07.469 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:07.469 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:07.570 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:07.610 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:07.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:07.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:07.712 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:07.755 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:07.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:07.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:07.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:07 smithi129 ceph-mon[101316]: pgmap v3031: 109 pgs: 1 remapped+peering, 108 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-14T05:00:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:07 smithi129 ceph-mon[101316]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-14T05:00:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:07 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/757714507' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:00:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:00:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:00:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:07 smithi129 ceph-mon[101316]: osdmap e1549: 8 total, 8 up, 8 in 2024-09-14T05:00:07.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:07 smithi039 ceph-mon[106470]: pgmap v3031: 109 pgs: 1 remapped+peering, 108 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-14T05:00:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:07 smithi039 ceph-mon[106470]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-14T05:00:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:07 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/757714507' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:00:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:00:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:00:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:07 smithi039 ceph-mon[106470]: osdmap e1549: 8 total, 8 up, 8 in 2024-09-14T05:00:07.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:07 smithi039 ceph-mon[111235]: pgmap v3031: 109 pgs: 1 remapped+peering, 108 active+clean; 2.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s 2024-09-14T05:00:07.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:07 smithi039 ceph-mon[111235]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-09-14T05:00:07.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:07 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/757714507' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:00:07.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:00:07.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:00:07.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:07 smithi039 ceph-mon[111235]: osdmap e1549: 8 total, 8 up, 8 in 2024-09-14T05:00:07.857 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:07.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:07.888 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:07.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:07.989 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:08.030 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:08.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:08.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:08.132 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:08.168 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:08.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:08.169 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:08.270 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:08.310 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:08.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:08.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:08.412 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:08.424 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:00:08.439 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:08.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:08.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:08.541 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:08.582 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:08.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:08.585 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:08.686 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:08.719 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:08.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:08.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:08.822 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:08.858 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:08.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:08.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:08.936 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:00:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:00:08] "GET /metrics HTTP/1.1" 200 36727 "" "Prometheus/2.51.0" 2024-09-14T05:00:08.961 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:08.998 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:08.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:08.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:09.100 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:09.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:09.144 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:09.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:09.245 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:09.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:09.286 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:09.286 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:09.387 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:09.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:09.427 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:09.427 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:09.528 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:09.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:09.568 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:09.568 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:09.669 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:09.701 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:09.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:09.702 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:09.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:09 smithi129 ceph-mon[101316]: pgmap v3034: 109 pgs: 4 peering, 2 active+undersized+remapped, 103 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 9.2 KiB/s rd, 27 KiB/s wr, 1.13k op/s; 100 KiB/s, 0 objects/s recovering 2024-09-14T05:00:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:09 smithi129 ceph-mon[101316]: osdmap e1550: 8 total, 8 up, 8 in 2024-09-14T05:00:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:09 smithi129 ceph-mon[101316]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-14T05:00:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:09 smithi129 ceph-mon[101316]: 1.b scrub starts 2024-09-14T05:00:09.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:09 smithi039 ceph-mon[106470]: pgmap v3034: 109 pgs: 4 peering, 2 active+undersized+remapped, 103 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 9.2 KiB/s rd, 27 KiB/s wr, 1.13k op/s; 100 KiB/s, 0 objects/s recovering 2024-09-14T05:00:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:09 smithi039 ceph-mon[106470]: osdmap e1550: 8 total, 8 up, 8 in 2024-09-14T05:00:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:09 smithi039 ceph-mon[106470]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-14T05:00:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:09 smithi039 ceph-mon[106470]: 1.b scrub starts 2024-09-14T05:00:09.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:09 smithi039 ceph-mon[111235]: pgmap v3034: 109 pgs: 4 peering, 2 active+undersized+remapped, 103 active+clean; 2.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 9.2 KiB/s rd, 27 KiB/s wr, 1.13k op/s; 100 KiB/s, 0 objects/s recovering 2024-09-14T05:00:09.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:09 smithi039 ceph-mon[111235]: osdmap e1550: 8 total, 8 up, 8 in 2024-09-14T05:00:09.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:09 smithi039 ceph-mon[111235]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-09-14T05:00:09.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:09 smithi039 ceph-mon[111235]: 1.b scrub starts 2024-09-14T05:00:09.803 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:09.839 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:09.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:09.840 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:09.941 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:09.980 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:09.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:09.981 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:10.082 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:10.117 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:10.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:10.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:10.220 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:10.253 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:10.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:10.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:10.355 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:10.396 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:10.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:10.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:10.400 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T05:00:10.499 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:10.535 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:10.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:10.536 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:10.636 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:10.659 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T05:00:10.660 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [], 2024-09-14T05:00:10.660 INFO:teuthology.orchestra.run.smithi039.stdout: "complaint_time": 30, 2024-09-14T05:00:10.660 INFO:teuthology.orchestra.run.smithi039.stdout: "num_blocked_ops": 0 2024-09-14T05:00:10.660 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T05:00:10.660 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:10 smithi039 ceph-mon[111235]: 1.14 scrub starts 2024-09-14T05:00:10.660 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:10 smithi039 ceph-mon[111235]: 1.11 scrub starts 2024-09-14T05:00:10.663 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:10 smithi039 ceph-mon[106470]: 1.14 scrub starts 2024-09-14T05:00:10.663 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:10 smithi039 ceph-mon[106470]: 1.11 scrub starts 2024-09-14T05:00:10.665 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:10.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:10.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:10.767 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:10.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:10 smithi129 ceph-mon[101316]: 1.14 scrub starts 2024-09-14T05:00:10.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:10 smithi129 ceph-mon[101316]: 1.11 scrub starts 2024-09-14T05:00:10.808 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:10.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:10.809 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:10.910 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:10.940 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:10.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:10.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:11.042 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:11.081 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:11.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:11.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:11.183 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:11.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:11.225 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:11.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:11.326 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:11.362 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:11.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:11.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:11.455 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:00:11.464 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:11.486 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-14T05:00:11.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:11.491 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:11.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:11.592 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:11.648 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:11.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:11.649 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:11.718 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:11 smithi039 ceph-mon[106470]: pgmap v3036: 109 pgs: 6 peering, 1 active+undersized+remapped, 102 active+clean; 2.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 25 KiB/s rd, 73 KiB/s wr, 1.10k op/s; 52 KiB/s, 0 objects/s recovering 2024-09-14T05:00:11.718 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:11 smithi039 ceph-mon[106470]: 1.c scrub starts 2024-09-14T05:00:11.719 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:11 smithi039 ceph-mon[111235]: pgmap v3036: 109 pgs: 6 peering, 1 active+undersized+remapped, 102 active+clean; 2.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 25 KiB/s rd, 73 KiB/s wr, 1.10k op/s; 52 KiB/s, 0 objects/s recovering 2024-09-14T05:00:11.719 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:11 smithi039 ceph-mon[111235]: 1.c scrub starts 2024-09-14T05:00:11.753 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:11.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:11 smithi129 ceph-mon[101316]: pgmap v3036: 109 pgs: 6 peering, 1 active+undersized+remapped, 102 active+clean; 2.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 25 KiB/s rd, 73 KiB/s wr, 1.10k op/s; 52 KiB/s, 0 objects/s recovering 2024-09-14T05:00:11.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:11 smithi129 ceph-mon[101316]: 1.c scrub starts 2024-09-14T05:00:11.798 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:11.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:11.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:11.901 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:11.953 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:11.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:11.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:12.055 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:12.092 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:12.092 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:12.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:12.194 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:12.230 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:12.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:12.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:12.332 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:12.362 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:12.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:12.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:12.463 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:12.514 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:12.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:12.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:12.615 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:12.651 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:12.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:12.652 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:12.753 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:12.792 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:12.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:12.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:12.894 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:12.927 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:12.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:12.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:13.029 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:13.065 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:13.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:13.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:13.167 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:13.207 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:13.208 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:13.208 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:13.309 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:13.348 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:13.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:13.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:13.449 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:13.486 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:13.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:13.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:13.588 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:13.622 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:13.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:13.623 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:13.723 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:13.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:13.755 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:13.756 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:13.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:13 smithi129 ceph-mon[101316]: pgmap v3037: 109 pgs: 6 peering, 103 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 814 op/s; 39 KiB/s, 0 objects/s recovering 2024-09-14T05:00:13.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:13 smithi039 ceph-mon[106470]: pgmap v3037: 109 pgs: 6 peering, 103 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 814 op/s; 39 KiB/s, 0 objects/s recovering 2024-09-14T05:00:13.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:13 smithi039 ceph-mon[111235]: pgmap v3037: 109 pgs: 6 peering, 103 active+clean; 2.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 814 op/s; 39 KiB/s, 0 objects/s recovering 2024-09-14T05:00:13.856 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:13.889 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:13.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:13.890 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:13.991 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:14.020 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:14.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:14.021 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:14.122 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:14.156 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:14.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:14.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:14.258 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:14.293 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:14.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:14.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:14.395 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:14.417 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:14.417 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:14.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:14.518 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:14.577 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:14.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:14.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:14.679 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:14.701 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:14.702 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:14.703 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:14.804 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:14.838 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:14.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:14.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:14.940 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:14.974 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:14.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:14.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:15.076 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:15.105 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:15.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:15.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:15.207 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:15.244 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:15.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:15.245 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:15.346 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:15.380 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:15.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:15.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:15.482 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:15.520 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:15.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:15.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:15.622 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:15.657 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:15.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:15.658 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:15.759 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:15.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:15 smithi129 ceph-mon[101316]: pgmap v3038: 109 pgs: 109 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 15 KiB/s rd, 44 KiB/s wr, 938 op/s; 31 KiB/s, 0 objects/s recovering 2024-09-14T05:00:15.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:15 smithi039 ceph-mon[106470]: pgmap v3038: 109 pgs: 109 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 15 KiB/s rd, 44 KiB/s wr, 938 op/s; 31 KiB/s, 0 objects/s recovering 2024-09-14T05:00:15.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:15 smithi039 ceph-mon[111235]: pgmap v3038: 109 pgs: 109 active+clean; 2.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 15 KiB/s rd, 44 KiB/s wr, 938 op/s; 31 KiB/s, 0 objects/s recovering 2024-09-14T05:00:15.798 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:15.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:15.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:15.900 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:15.920 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:15.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:15.921 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:16.022 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:16.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:16.049 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:16.049 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:16.150 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:16.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:16.178 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:16.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:16.279 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:16.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:16.312 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:16.312 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:16.413 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:16.451 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:16.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:16.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:16.553 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:16.569 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is set 2024-09-14T05:00:16.581 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:16.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:16.583 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:16.683 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:16.718 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:16.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:16.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:16.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:16 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/371298483' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:00:16.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:16 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:00:16.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:16 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/371298483' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:00:16.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:16 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:00:16.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:16 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/371298483' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:00:16.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:16 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:00:16.820 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:16.857 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:16.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:16.858 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:16.959 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:16.995 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:16.996 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:16.996 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:17.097 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:17.134 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:17.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:17.135 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:17.236 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:17.267 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:17.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:17.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:17.368 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:17.404 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:17.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:17.406 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:17.506 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:17.548 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:17.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:17.549 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:17.650 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:17.689 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:17.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:17.689 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:17.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:17 smithi129 ceph-mon[101316]: pgmap v3039: 109 pgs: 2 active+clean+scrubbing, 107 active+clean; 1.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 13 KiB/s rd, 39 KiB/s wr, 1.15k op/s; 0 B/s, 0 objects/s recovering 2024-09-14T05:00:17.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:17 smithi129 ceph-mon[101316]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:00:17.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:17 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T05:00:17.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:17 smithi129 ceph-mon[101316]: osdmap e1551: 8 total, 8 up, 8 in 2024-09-14T05:00:17.775 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:17 smithi039 ceph-mon[106470]: pgmap v3039: 109 pgs: 2 active+clean+scrubbing, 107 active+clean; 1.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 13 KiB/s rd, 39 KiB/s wr, 1.15k op/s; 0 B/s, 0 objects/s recovering 2024-09-14T05:00:17.775 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:17 smithi039 ceph-mon[106470]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:00:17.775 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:17 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T05:00:17.775 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:17 smithi039 ceph-mon[106470]: osdmap e1551: 8 total, 8 up, 8 in 2024-09-14T05:00:17.775 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:17 smithi039 ceph-mon[111235]: pgmap v3039: 109 pgs: 2 active+clean+scrubbing, 107 active+clean; 1.9 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 13 KiB/s rd, 39 KiB/s wr, 1.15k op/s; 0 B/s, 0 objects/s recovering 2024-09-14T05:00:17.775 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:17 smithi039 ceph-mon[111235]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:00:17.775 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:17 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T05:00:17.776 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:17 smithi039 ceph-mon[111235]: osdmap e1551: 8 total, 8 up, 8 in 2024-09-14T05:00:17.790 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:17.818 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:17.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:17.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:17.919 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:17.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:17.961 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:17.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:18.062 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:18.099 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:18.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:18.100 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:18.201 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:18.230 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:18.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:18.231 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:18.332 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:18.375 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:18.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:18.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:18.477 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:18.501 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:18.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:18.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:18.603 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:18.634 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:18.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:18.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:18.736 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:18.787 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:18.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:18.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:18.889 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:18.902 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:00:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:00:18] "GET /metrics HTTP/1.1" 200 36727 "" "Prometheus/2.51.0" 2024-09-14T05:00:18.917 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:18.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:18.919 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:19.019 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:19.067 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:19.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:19.068 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:19.106 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:00:19.107 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:00:19.168 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:19.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:19.206 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:19.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:19.307 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:19.335 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:19.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:19.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:19.436 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:19.478 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:19.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:19.479 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:19.579 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:19.618 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:19.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:19.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:19.720 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:19.752 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:19.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:19.753 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:19.787 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:19 smithi039 ceph-mon[106470]: pgmap v3041: 109 pgs: 2 active+clean+scrubbing, 107 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 4.0 KiB/s rd, 12 KiB/s wr, 973 op/s 2024-09-14T05:00:19.787 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:19 smithi039 ceph-mon[111235]: pgmap v3041: 109 pgs: 2 active+clean+scrubbing, 107 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 4.0 KiB/s rd, 12 KiB/s wr, 973 op/s 2024-09-14T05:00:19.854 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:19.887 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:19.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:19.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:19.989 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:20.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:19 smithi129 ceph-mon[101316]: pgmap v3041: 109 pgs: 2 active+clean+scrubbing, 107 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 4.0 KiB/s rd, 12 KiB/s wr, 973 op/s 2024-09-14T05:00:20.024 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:20.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:20.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:20.126 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:20.171 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:20.172 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:20.172 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:20.273 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:20.312 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:20.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:20.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:20.414 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:20.445 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:20.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:20.446 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:20.534 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.5 to 0.863418 (dd08) 2024-09-14T05:00:20.547 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:20.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:20.577 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:20.578 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:20.679 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:20.725 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:20.726 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:20.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:20.749 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T05:00:20.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:20 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3650223643' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 0.8634178442937773} : dispatch 2024-09-14T05:00:20.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:20 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 0.8634178442937773} : dispatch 2024-09-14T05:00:20.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:20 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3650223643' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 0.8634178442937773} : dispatch 2024-09-14T05:00:20.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:20 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 0.8634178442937773} : dispatch 2024-09-14T05:00:20.827 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:20.881 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:20.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:20.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:20.983 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:21.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:21.010 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:21.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:21.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:20 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3650223643' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 0.8634178442937773} : dispatch 2024-09-14T05:00:21.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:20 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 0.8634178442937773} : dispatch 2024-09-14T05:00:21.111 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:21.133 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:21.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:21.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:21.235 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:21.264 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:21.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:21.265 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:21.366 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:21.401 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:21.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:21.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:21.503 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:21.539 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:21.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:21.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:21.641 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:21.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:21.673 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:21.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:21.774 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:21.799 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:21.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:21.800 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:21.901 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:21.929 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:21.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:21.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:21 smithi129 ceph-mon[101316]: pgmap v3042: 109 pgs: 109 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 931 op/s 2024-09-14T05:00:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:21 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 0.8634178442937773}]': finished 2024-09-14T05:00:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:21 smithi129 ceph-mon[101316]: osdmap e1552: 8 total, 8 up, 8 in 2024-09-14T05:00:22.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:21 smithi129 ceph-mon[101316]: osdmap e1553: 8 total, 8 up, 8 in 2024-09-14T05:00:22.031 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:22.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:21 smithi039 ceph-mon[106470]: pgmap v3042: 109 pgs: 109 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 931 op/s 2024-09-14T05:00:22.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:21 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 0.8634178442937773}]': finished 2024-09-14T05:00:22.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:21 smithi039 ceph-mon[106470]: osdmap e1552: 8 total, 8 up, 8 in 2024-09-14T05:00:22.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:21 smithi039 ceph-mon[106470]: osdmap e1553: 8 total, 8 up, 8 in 2024-09-14T05:00:22.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:21 smithi039 ceph-mon[111235]: pgmap v3042: 109 pgs: 109 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 931 op/s 2024-09-14T05:00:22.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:21 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 0.8634178442937773}]': finished 2024-09-14T05:00:22.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:21 smithi039 ceph-mon[111235]: osdmap e1552: 8 total, 8 up, 8 in 2024-09-14T05:00:22.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:21 smithi039 ceph-mon[111235]: osdmap e1553: 8 total, 8 up, 8 in 2024-09-14T05:00:22.071 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:22.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:22.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:22.173 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:22.202 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:22.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:22.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:22.305 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:22.352 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:22.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:22.353 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:22.453 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:22.492 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:22.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:22.493 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:22.594 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:22.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:22.627 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:22.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:22.728 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:22.776 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:22.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:22.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:22.878 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:22.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:22.918 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:22.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:23.019 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:23.050 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:23.050 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:23.051 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:23.151 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:23.180 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:23.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:23.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:23.282 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:23 smithi039 ceph-mon[106470]: pgmap v3045: 109 pgs: 1 unknown, 108 active+clean; 1.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 727 op/s 2024-09-14T05:00:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:23 smithi039 ceph-mon[106470]: osdmap e1554: 8 total, 8 up, 8 in 2024-09-14T05:00:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:23 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:00:23.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:00:23.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:23 smithi039 ceph-mon[111235]: pgmap v3045: 109 pgs: 1 unknown, 108 active+clean; 1.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 727 op/s 2024-09-14T05:00:23.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:23 smithi039 ceph-mon[111235]: osdmap e1554: 8 total, 8 up, 8 in 2024-09-14T05:00:23.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:23 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:00:23.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:00:23.320 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:23.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:23.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:23.422 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:23.452 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:23.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:23.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:23 smithi129 ceph-mon[101316]: pgmap v3045: 109 pgs: 1 unknown, 108 active+clean; 1.6 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 727 op/s 2024-09-14T05:00:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:23 smithi129 ceph-mon[101316]: osdmap e1554: 8 total, 8 up, 8 in 2024-09-14T05:00:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:23 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:00:23.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:00:23.553 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:23.589 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:23.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:23.590 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:23.691 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:23.734 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:23.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:23.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:23.836 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:23.869 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:23.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:23.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:23.970 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:24.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:24.002 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:24.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:24.103 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:24.139 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:24.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:24.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:24.241 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:24.268 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:24.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:24.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:24.370 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:24.399 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:24.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:24.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:24.501 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:24.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:24 smithi129 ceph-mon[101316]: osdmap e1555: 8 total, 8 up, 8 in 2024-09-14T05:00:24.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:24 smithi039 ceph-mon[106470]: osdmap e1555: 8 total, 8 up, 8 in 2024-09-14T05:00:24.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:24 smithi039 ceph-mon[111235]: osdmap e1555: 8 total, 8 up, 8 in 2024-09-14T05:00:24.543 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:24.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:24.544 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:24.645 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:24.684 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:24.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:24.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:24.786 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:24.827 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:24.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:24.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:24.909 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T05:00:24.909 INFO:teuthology.orchestra.run.smithi039.stdout: "size": 20, 2024-09-14T05:00:24.909 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 600, 2024-09-14T05:00:24.909 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [ 2024-09-14T05:00:24.909 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.910 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:40551 56.0 e1474/1470 56:0d68702d:::benchmark_data_smithi039_330847_object40550:head v 1474'2482, mlcod=1474'2480)", 2024-09-14T05:00:24.910 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:52:48.082357+0000", 2024-09-14T05:00:24.910 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 456.82446383199999, 2024-09-14T05:00:24.910 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.109014311, 2024-09-14T05:00:24.910 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:00:24.910 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:00:24.910 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:24.910 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:00:24.910 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.910 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:00:24.910 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082357+0000", 2024-09-14T05:00:24.910 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.910 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.911 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.911 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:00:24.911 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082357+0000", 2024-09-14T05:00:24.911 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.911 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.911 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.911 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:00:24.911 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082358+0000", 2024-09-14T05:00:24.911 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9999999999999996e-07 2024-09-14T05:00:24.911 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.911 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.911 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:00:24.911 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082422+0000", 2024-09-14T05:00:24.911 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3972000000000003e-05 2024-09-14T05:00:24.912 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.912 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.912 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:00:24.912 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082432+0000", 2024-09-14T05:00:24.912 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.04e-05 2024-09-14T05:00:24.912 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.912 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.912 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:00:24.912 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082438+0000", 2024-09-14T05:00:24.912 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.9950000000000002e-06 2024-09-14T05:00:24.912 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.912 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.912 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:00:24.912 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082493+0000", 2024-09-14T05:00:24.912 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4939999999999999e-05 2024-09-14T05:00:24.913 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.913 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.913 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:00:24.913 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082511+0000", 2024-09-14T05:00:24.913 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8190999999999999e-05 2024-09-14T05:00:24.913 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.913 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.913 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:00:24.913 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.191342+0000", 2024-09-14T05:00:24.913 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.10883108599999999 2024-09-14T05:00:24.913 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.913 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.913 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:00:24.914 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.191371+0000", 2024-09-14T05:00:24.914 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.8926999999999999e-05 2024-09-14T05:00:24.914 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.914 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:00:24.914 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.914 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.914 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.914 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:40553 56.2 e1474/1470 56:472e3c32:::benchmark_data_smithi039_330847_object40552:head v 1474'2557, mlcod=1474'2553)", 2024-09-14T05:00:24.914 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:52:48.082671+0000", 2024-09-14T05:00:24.914 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 456.82414970899998, 2024-09-14T05:00:24.914 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.10865520100000001, 2024-09-14T05:00:24.914 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:00:24.914 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:00:24.915 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:24.915 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:00:24.915 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.915 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:00:24.915 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082671+0000", 2024-09-14T05:00:24.915 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.915 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.915 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.915 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:00:24.915 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082671+0000", 2024-09-14T05:00:24.915 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.915 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.915 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.915 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:00:24.915 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082672+0000", 2024-09-14T05:00:24.916 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9299999999999997e-07 2024-09-14T05:00:24.916 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.916 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.916 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:00:24.916 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082756+0000", 2024-09-14T05:00:24.916 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.4225000000000001e-05 2024-09-14T05:00:24.916 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.916 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.916 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:00:24.916 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082766+0000", 2024-09-14T05:00:24.916 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.8539999999999992e-06 2024-09-14T05:00:24.916 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.916 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.916 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:00:24.917 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082771+0000", 2024-09-14T05:00:24.917 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.8180000000000003e-06 2024-09-14T05:00:24.917 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.917 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.917 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:00:24.917 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082841+0000", 2024-09-14T05:00:24.917 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9877999999999997e-05 2024-09-14T05:00:24.917 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.917 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.917 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:00:24.917 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082880+0000", 2024-09-14T05:00:24.917 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.9356000000000001e-05 2024-09-14T05:00:24.917 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.917 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.918 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:00:24.918 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.191280+0000", 2024-09-14T05:00:24.918 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.10840013900000001 2024-09-14T05:00:24.918 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.918 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.918 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:00:24.918 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.191326+0000", 2024-09-14T05:00:24.918 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6238e-05 2024-09-14T05:00:24.918 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.918 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:00:24.918 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.918 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.918 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.918 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:40555 56.1 e1474/1470 56:8f6c7fc0:::benchmark_data_smithi039_330847_object40554:head v 1474'2517, mlcod=1474'2515)", 2024-09-14T05:00:24.919 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:52:48.082815+0000", 2024-09-14T05:00:24.919 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 456.82400601299997, 2024-09-14T05:00:24.919 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.108515476, 2024-09-14T05:00:24.919 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:00:24.919 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:00:24.919 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:24.919 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:00:24.919 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.919 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:00:24.919 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082815+0000", 2024-09-14T05:00:24.919 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.919 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.919 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.919 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:00:24.920 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082815+0000", 2024-09-14T05:00:24.920 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.920 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.920 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.920 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:00:24.920 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082815+0000", 2024-09-14T05:00:24.920 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9100000000000004e-07 2024-09-14T05:00:24.920 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.920 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.920 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:00:24.920 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082879+0000", 2024-09-14T05:00:24.921 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3402e-05 2024-09-14T05:00:24.921 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.921 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.921 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:00:24.921 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082888+0000", 2024-09-14T05:00:24.921 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.1859999999999998e-06 2024-09-14T05:00:24.921 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.921 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.921 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:00:24.921 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082894+0000", 2024-09-14T05:00:24.921 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.9510000000000003e-06 2024-09-14T05:00:24.921 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.921 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.921 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:00:24.921 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082979+0000", 2024-09-14T05:00:24.922 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.4847999999999998e-05 2024-09-14T05:00:24.922 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.922 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.922 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:00:24.922 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.082996+0000", 2024-09-14T05:00:24.922 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6914000000000001e-05 2024-09-14T05:00:24.922 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.922 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.922 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:00:24.922 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.191291+0000", 2024-09-14T05:00:24.922 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.108295473 2024-09-14T05:00:24.922 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.922 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.922 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:00:24.922 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.191330+0000", 2024-09-14T05:00:24.923 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.9211000000000001e-05 2024-09-14T05:00:24.923 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.923 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:00:24.923 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.923 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.923 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.923 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:40554 56.0 e1474/1470 56:02216655:::benchmark_data_smithi039_330847_object40553:head v 1474'2483, mlcod=1474'2480)", 2024-09-14T05:00:24.923 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:52:48.085185+0000", 2024-09-14T05:00:24.923 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 456.82163560200001, 2024-09-14T05:00:24.924 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.10836931399999999, 2024-09-14T05:00:24.924 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:00:24.924 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:00:24.924 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:24.924 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:00:24.924 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.924 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:00:24.924 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.085185+0000", 2024-09-14T05:00:24.924 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.924 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.924 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.925 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:00:24.925 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.085185+0000", 2024-09-14T05:00:24.925 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.925 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.925 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.925 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:00:24.925 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.085186+0000", 2024-09-14T05:00:24.925 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.8299999999999995e-07 2024-09-14T05:00:24.925 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.925 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.925 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:00:24.925 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.085247+0000", 2024-09-14T05:00:24.925 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0677000000000002e-05 2024-09-14T05:00:24.925 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.926 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.926 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:00:24.926 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.191274+0000", 2024-09-14T05:00:24.926 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.106027363 2024-09-14T05:00:24.926 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.926 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.926 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:00:24.926 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.191286+0000", 2024-09-14T05:00:24.926 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2101e-05 2024-09-14T05:00:24.926 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.926 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.926 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:00:24.926 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.191396+0000", 2024-09-14T05:00:24.926 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000109954 2024-09-14T05:00:24.926 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.927 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.927 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:00:24.927 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.191416+0000", 2024-09-14T05:00:24.927 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9295000000000001e-05 2024-09-14T05:00:24.927 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.927 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.927 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:00:24.927 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.193524+0000", 2024-09-14T05:00:24.927 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0021085409999999998 2024-09-14T05:00:24.927 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.927 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.927 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:00:24.927 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:48.193555+0000", 2024-09-14T05:00:24.927 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.04e-05 2024-09-14T05:00:24.927 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.928 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:00:24.928 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.928 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.928 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.928 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:150517 56.d e1502/1470 56:b2d2f80c:::benchmark_data_smithi039_330847_object35993:head v 1502'9481, mlcod=1502'9477)", 2024-09-14T05:00:24.928 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:54:48.319724+0000", 2024-09-14T05:00:24.928 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 336.58709703099998, 2024-09-14T05:00:24.928 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.33259182399999998, 2024-09-14T05:00:24.928 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:00:24.928 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:00:24.928 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:24.928 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:00:24.928 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.928 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:00:24.928 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.319724+0000", 2024-09-14T05:00:24.929 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.929 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.929 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.929 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:00:24.929 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.319724+0000", 2024-09-14T05:00:24.929 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.929 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.929 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.929 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:00:24.929 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.319725+0000", 2024-09-14T05:00:24.929 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2699999999999999e-06 2024-09-14T05:00:24.929 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.929 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.929 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:00:24.930 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.319732+0000", 2024-09-14T05:00:24.930 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.2779999999999996e-06 2024-09-14T05:00:24.930 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.930 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.930 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:00:24.930 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.319742+0000", 2024-09-14T05:00:24.930 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.8949999999999993e-06 2024-09-14T05:00:24.930 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.930 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.930 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:00:24.930 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.319750+0000", 2024-09-14T05:00:24.930 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9549999999999992e-06 2024-09-14T05:00:24.930 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.930 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.930 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:00:24.931 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.322309+0000", 2024-09-14T05:00:24.931 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0025585680000000002 2024-09-14T05:00:24.931 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.931 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.931 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:00:24.931 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.322381+0000", 2024-09-14T05:00:24.931 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1964000000000004e-05 2024-09-14T05:00:24.931 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.931 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.931 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:00:24.931 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.340707+0000", 2024-09-14T05:00:24.931 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.018326426 2024-09-14T05:00:24.931 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.931 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.932 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:00:24.932 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.652316+0000", 2024-09-14T05:00:24.932 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.31160846800000003 2024-09-14T05:00:24.932 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.932 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:00:24.932 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.932 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.932 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.932 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:150518 56.5 e1502/1470 56:af443af1:::benchmark_data_smithi039_330847_object35994:head v 1502'9416, mlcod=1502'9412)", 2024-09-14T05:00:24.932 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:54:48.320121+0000", 2024-09-14T05:00:24.932 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 336.58669952399998, 2024-09-14T05:00:24.932 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.33647724200000001, 2024-09-14T05:00:24.932 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:00:24.933 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:00:24.933 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:24.933 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:00:24.933 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.933 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:00:24.933 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.320121+0000", 2024-09-14T05:00:24.933 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.933 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.933 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.933 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:00:24.933 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.320121+0000", 2024-09-14T05:00:24.933 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.933 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.933 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.933 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:00:24.934 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.320122+0000", 2024-09-14T05:00:24.934 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.3399999999999998e-07 2024-09-14T05:00:24.934 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.934 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.934 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:00:24.934 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.320128+0000", 2024-09-14T05:00:24.934 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0449999999999997e-06 2024-09-14T05:00:24.934 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.934 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.934 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:00:24.934 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.320139+0000", 2024-09-14T05:00:24.934 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1311000000000001e-05 2024-09-14T05:00:24.934 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.934 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.934 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:00:24.935 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.320146+0000", 2024-09-14T05:00:24.935 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2389999999999999e-06 2024-09-14T05:00:24.935 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.935 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.935 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:00:24.935 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.327570+0000", 2024-09-14T05:00:24.935 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0074241569999999998 2024-09-14T05:00:24.935 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.935 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.935 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:00:24.935 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.327589+0000", 2024-09-14T05:00:24.935 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9057000000000001e-05 2024-09-14T05:00:24.935 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.935 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.936 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:00:24.936 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.656515+0000", 2024-09-14T05:00:24.936 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.32892624500000001 2024-09-14T05:00:24.936 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.936 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.936 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:00:24.936 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.656599+0000", 2024-09-14T05:00:24.936 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.3454000000000004e-05 2024-09-14T05:00:24.936 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.936 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:00:24.936 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.936 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.936 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.936 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:150525 56.5 e1502/1470 56:af201380:::benchmark_data_smithi039_330847_object36001:head v 1502'9417, mlcod=1502'9412)", 2024-09-14T05:00:24.936 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:54:48.321332+0000", 2024-09-14T05:00:24.936 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 336.58548886599999, 2024-09-14T05:00:24.937 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.335344066, 2024-09-14T05:00:24.937 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:00:24.937 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:00:24.937 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:24.937 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:00:24.937 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.937 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:00:24.937 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.321332+0000", 2024-09-14T05:00:24.937 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.937 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.937 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.937 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:00:24.937 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.321332+0000", 2024-09-14T05:00:24.938 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.938 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.938 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.938 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:00:24.938 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.321333+0000", 2024-09-14T05:00:24.938 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7899999999999997e-07 2024-09-14T05:00:24.938 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.938 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.938 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:00:24.938 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.321341+0000", 2024-09-14T05:00:24.938 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.4810000000000006e-06 2024-09-14T05:00:24.938 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.938 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.938 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:00:24.938 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.321352+0000", 2024-09-14T05:00:24.939 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0855e-05 2024-09-14T05:00:24.939 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.939 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.939 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:00:24.939 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.321359+0000", 2024-09-14T05:00:24.939 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.4339999999999997e-06 2024-09-14T05:00:24.939 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.939 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.939 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:00:24.939 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.333506+0000", 2024-09-14T05:00:24.939 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.01214791 2024-09-14T05:00:24.939 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.939 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.940 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:00:24.940 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.333525+0000", 2024-09-14T05:00:24.940 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8836999999999999e-05 2024-09-14T05:00:24.940 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.940 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.940 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:00:24.940 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.656602+0000", 2024-09-14T05:00:24.940 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.32307669999999999 2024-09-14T05:00:24.940 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.940 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.940 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:00:24.940 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.656676+0000", 2024-09-14T05:00:24.940 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.4070000000000001e-05 2024-09-14T05:00:24.940 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.941 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:00:24.941 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.941 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.941 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.941 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:150528 56.0 e1502/1470 56:04bd5805:::benchmark_data_smithi039_330847_object36004:head v 1502'9348, mlcod=1502'9343)", 2024-09-14T05:00:24.941 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:54:48.332080+0000", 2024-09-14T05:00:24.941 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 336.574740894, 2024-09-14T05:00:24.941 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.32073101100000001, 2024-09-14T05:00:24.941 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:00:24.941 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:00:24.941 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:24.941 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:00:24.941 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.941 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:00:24.941 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.332080+0000", 2024-09-14T05:00:24.942 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.942 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.942 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.942 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:00:24.942 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.332080+0000", 2024-09-14T05:00:24.942 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.942 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.942 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.942 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:00:24.942 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.332081+0000", 2024-09-14T05:00:24.942 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5149999999999999e-06 2024-09-14T05:00:24.942 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.942 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.942 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:00:24.942 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.332092+0000", 2024-09-14T05:00:24.943 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0494e-05 2024-09-14T05:00:24.943 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.943 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.943 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:00:24.943 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.332106+0000", 2024-09-14T05:00:24.943 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3832000000000001e-05 2024-09-14T05:00:24.943 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.943 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.943 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:00:24.943 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.332114+0000", 2024-09-14T05:00:24.943 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0209999999999999e-06 2024-09-14T05:00:24.943 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.943 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.943 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:00:24.944 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.333767+0000", 2024-09-14T05:00:24.944 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001653212 2024-09-14T05:00:24.944 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.944 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.944 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:00:24.944 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.333847+0000", 2024-09-14T05:00:24.944 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9901000000000006e-05 2024-09-14T05:00:24.944 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.944 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.944 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:00:24.944 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.652325+0000", 2024-09-14T05:00:24.944 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.31847768599999998 2024-09-14T05:00:24.944 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.944 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.944 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:00:24.945 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.652811+0000", 2024-09-14T05:00:24.945 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00048634999999999998 2024-09-14T05:00:24.945 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.945 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:00:24.945 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.945 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.945 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.945 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:150534 56.5 e1502/1470 56:ab5eb5b0:::benchmark_data_smithi039_330847_object36010:head v 1502'9418, mlcod=1502'9413)", 2024-09-14T05:00:24.945 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:54:48.332948+0000", 2024-09-14T05:00:24.945 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 336.57387335800001, 2024-09-14T05:00:24.945 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.33585074300000001, 2024-09-14T05:00:24.945 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:00:24.945 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:00:24.945 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:24.945 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:00:24.946 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.946 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:00:24.946 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.332948+0000", 2024-09-14T05:00:24.946 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.946 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.946 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.946 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:00:24.946 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.332948+0000", 2024-09-14T05:00:24.946 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.946 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.946 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.946 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:00:24.946 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.332948+0000", 2024-09-14T05:00:24.946 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9299999999999997e-07 2024-09-14T05:00:24.946 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.946 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.947 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:00:24.947 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.332957+0000", 2024-09-14T05:00:24.947 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.7260000000000008e-06 2024-09-14T05:00:24.947 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.947 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.947 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:00:24.947 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.332968+0000", 2024-09-14T05:00:24.947 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0726e-05 2024-09-14T05:00:24.947 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.947 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.947 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:00:24.947 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.332974+0000", 2024-09-14T05:00:24.947 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2330000000000003e-06 2024-09-14T05:00:24.947 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.947 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.948 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:00:24.948 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.652363+0000", 2024-09-14T05:00:24.948 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.319389592 2024-09-14T05:00:24.948 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.948 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.948 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:00:24.948 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.652388+0000", 2024-09-14T05:00:24.948 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4096000000000001e-05 2024-09-14T05:00:24.948 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.948 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.948 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:00:24.948 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.668751+0000", 2024-09-14T05:00:24.948 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.016363906000000001 2024-09-14T05:00:24.948 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.949 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.949 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:00:24.949 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.668798+0000", 2024-09-14T05:00:24.949 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6771000000000003e-05 2024-09-14T05:00:24.949 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.949 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:00:24.949 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.949 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.949 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.949 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:150537 56.2 e1502/1489 56:4734d1d9:::benchmark_data_smithi039_330847_object36013:head v 1502'9523, mlcod=1474'3713)", 2024-09-14T05:00:24.949 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:54:48.334366+0000", 2024-09-14T05:00:24.949 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 336.57245523799997, 2024-09-14T05:00:24.949 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.31802780200000003, 2024-09-14T05:00:24.950 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:00:24.950 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:00:24.950 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:24.950 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:00:24.950 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.950 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:00:24.950 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.334366+0000", 2024-09-14T05:00:24.950 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.950 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.950 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.950 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:00:24.950 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.334366+0000", 2024-09-14T05:00:24.950 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.951 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.951 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.951 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:24.952 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:00:24.952 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.334366+0000", 2024-09-14T05:00:24.952 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7299999999999995e-07 2024-09-14T05:00:24.952 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.952 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.952 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:00:24.952 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.334373+0000", 2024-09-14T05:00:24.952 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2709999999999998e-06 2024-09-14T05:00:24.952 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.953 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.953 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:00:24.953 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.334382+0000", 2024-09-14T05:00:24.953 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.0149999999999992e-06 2024-09-14T05:00:24.953 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.953 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.953 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:00:24.953 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.334387+0000", 2024-09-14T05:00:24.953 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5790000000000003e-06 2024-09-14T05:00:24.953 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.953 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.953 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:00:24.953 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.334439+0000", 2024-09-14T05:00:24.953 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1949e-05 2024-09-14T05:00:24.954 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.954 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.954 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:00:24.954 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.334472+0000", 2024-09-14T05:00:24.954 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.2617000000000003e-05 2024-09-14T05:00:24.954 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.954 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.954 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:00:24.954 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.652317+0000", 2024-09-14T05:00:24.954 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.31784542100000002 2024-09-14T05:00:24.954 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.954 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.954 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:00:24.954 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.652393+0000", 2024-09-14T05:00:24.955 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6277000000000002e-05 2024-09-14T05:00:24.955 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.955 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:00:24.955 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.955 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.955 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.955 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:150542 56.0 e1502/1470 56:0741494f:::benchmark_data_smithi039_330847_object36018:head v 1502'9349, mlcod=1502'9346)", 2024-09-14T05:00:24.955 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:54:48.342576+0000", 2024-09-14T05:00:24.955 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 336.56424474599999, 2024-09-14T05:00:24.955 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.31390462699999999, 2024-09-14T05:00:24.955 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:00:24.955 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:00:24.955 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:24.955 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:00:24.955 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.956 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:00:24.956 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.342576+0000", 2024-09-14T05:00:24.956 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.956 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.956 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.956 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:00:24.956 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.342576+0000", 2024-09-14T05:00:24.956 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.956 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.956 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.956 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:00:24.956 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.342577+0000", 2024-09-14T05:00:24.956 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1939999999999999e-06 2024-09-14T05:00:24.957 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.957 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.957 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:00:24.957 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.342583+0000", 2024-09-14T05:00:24.957 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0090000000000004e-06 2024-09-14T05:00:24.957 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.957 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.957 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:00:24.957 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.652363+0000", 2024-09-14T05:00:24.957 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.30977945899999998 2024-09-14T05:00:24.957 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.957 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.957 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:00:24.958 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.652371+0000", 2024-09-14T05:00:24.958 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.1210000000000007e-06 2024-09-14T05:00:24.958 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.958 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.958 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:00:24.958 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.652838+0000", 2024-09-14T05:00:24.958 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00046703899999999998 2024-09-14T05:00:24.958 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.958 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.958 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:00:24.958 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.652858+0000", 2024-09-14T05:00:24.958 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9952000000000001e-05 2024-09-14T05:00:24.958 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.959 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.959 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:00:24.959 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.656336+0000", 2024-09-14T05:00:24.959 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0034781679999999998 2024-09-14T05:00:24.959 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.959 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.959 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:00:24.959 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.656481+0000", 2024-09-14T05:00:24.959 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000144685 2024-09-14T05:00:24.959 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.959 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:00:24.959 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.959 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.959 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.960 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:150550 56.d e1502/1470 56:b8b6dd53:::benchmark_data_smithi039_330847_object36026:head v 1502'9482, mlcod=1502'9480)", 2024-09-14T05:00:24.960 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:54:48.350479+0000", 2024-09-14T05:00:24.960 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 336.55634158300001, 2024-09-14T05:00:24.960 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.31836037700000003, 2024-09-14T05:00:24.960 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:00:24.960 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:00:24.960 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:24.960 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:00:24.960 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.960 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:00:24.960 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.350479+0000", 2024-09-14T05:00:24.960 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.960 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.960 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.961 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:00:24.961 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.350479+0000", 2024-09-14T05:00:24.961 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.961 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.961 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.961 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:00:24.961 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.350481+0000", 2024-09-14T05:00:24.961 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5689999999999999e-06 2024-09-14T05:00:24.961 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.961 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.961 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:00:24.961 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.350497+0000", 2024-09-14T05:00:24.961 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5845000000000002e-05 2024-09-14T05:00:24.961 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.962 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.962 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:00:24.962 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.652315+0000", 2024-09-14T05:00:24.962 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.30181803800000001 2024-09-14T05:00:24.962 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.962 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.962 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:00:24.962 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.652333+0000", 2024-09-14T05:00:24.962 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7808999999999998e-05 2024-09-14T05:00:24.962 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.962 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.962 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:00:24.962 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.652558+0000", 2024-09-14T05:00:24.962 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00022574199999999999 2024-09-14T05:00:24.963 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.963 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.963 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:00:24.963 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.652576+0000", 2024-09-14T05:00:24.963 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7801000000000001e-05 2024-09-14T05:00:24.963 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.963 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.963 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:00:24.963 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.668805+0000", 2024-09-14T05:00:24.963 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.016228597000000001 2024-09-14T05:00:24.963 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.963 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.963 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:00:24.963 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.668840+0000", 2024-09-14T05:00:24.963 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.4975999999999999e-05 2024-09-14T05:00:24.964 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.964 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:00:24.964 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.964 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.964 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.964 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:160772 56.6 e1507/1489 56:64a6029c:::benchmark_data_smithi039_330847_object46248:head v 1507'9862, mlcod=1481'5099)", 2024-09-14T05:00:24.964 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:55:03.049495+0000", 2024-09-14T05:00:24.964 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 321.857325656, 2024-09-14T05:00:24.964 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.089665717000000006, 2024-09-14T05:00:24.964 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:00:24.964 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:00:24.964 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:24.964 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:00:24.964 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.965 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:00:24.965 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:03.049495+0000", 2024-09-14T05:00:24.965 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.965 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.965 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.965 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:00:24.965 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:03.049495+0000", 2024-09-14T05:00:24.965 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.965 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.965 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.965 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:00:24.965 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:03.049496+0000", 2024-09-14T05:00:24.965 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.5499999999999996e-07 2024-09-14T05:00:24.966 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.966 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.966 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:00:24.966 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:03.049507+0000", 2024-09-14T05:00:24.966 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0565e-05 2024-09-14T05:00:24.966 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.966 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.966 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:00:24.966 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:03.050279+0000", 2024-09-14T05:00:24.966 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000772165 2024-09-14T05:00:24.966 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.966 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.966 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:00:24.967 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:03.050290+0000", 2024-09-14T05:00:24.967 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1082e-05 2024-09-14T05:00:24.967 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.967 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.967 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:00:24.967 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:03.062020+0000", 2024-09-14T05:00:24.967 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.011729563 2024-09-14T05:00:24.967 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.967 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.967 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:00:24.967 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:03.062043+0000", 2024-09-14T05:00:24.967 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3325000000000001e-05 2024-09-14T05:00:24.967 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.968 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.968 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:00:24.968 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:03.139103+0000", 2024-09-14T05:00:24.968 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.077060571999999994 2024-09-14T05:00:24.968 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.968 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.968 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:00:24.968 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:03.139161+0000", 2024-09-14T05:00:24.968 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.749e-05 2024-09-14T05:00:24.968 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.968 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:00:24.968 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.968 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.968 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.969 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:160773 56.6 e1507/1489 56:62a7ee8b:::benchmark_data_smithi039_330847_object46249:head v 1507'9863, mlcod=1481'5099)", 2024-09-14T05:00:24.969 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:55:03.053644+0000", 2024-09-14T05:00:24.969 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 321.853177319, 2024-09-14T05:00:24.969 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.085549497000000002, 2024-09-14T05:00:24.969 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:00:24.969 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:00:24.969 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:24.969 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:00:24.969 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.969 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:00:24.969 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:03.053644+0000", 2024-09-14T05:00:24.969 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.970 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.970 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.970 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:00:24.970 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:03.053644+0000", 2024-09-14T05:00:24.970 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.970 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.970 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.970 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:00:24.970 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:03.053645+0000", 2024-09-14T05:00:24.970 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.311e-06 2024-09-14T05:00:24.970 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.970 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.971 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:00:24.971 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:03.053656+0000", 2024-09-14T05:00:24.971 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0791000000000001e-05 2024-09-14T05:00:24.971 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.971 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.971 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:00:24.971 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:03.053674+0000", 2024-09-14T05:00:24.971 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8108999999999999e-05 2024-09-14T05:00:24.971 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.971 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.971 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:00:24.971 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:03.053682+0000", 2024-09-14T05:00:24.971 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.5399999999999996e-06 2024-09-14T05:00:24.971 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.971 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.972 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:00:24.972 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:03.069029+0000", 2024-09-14T05:00:24.972 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.015346821 2024-09-14T05:00:24.972 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.972 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.972 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:00:24.972 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:03.069054+0000", 2024-09-14T05:00:24.972 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4643000000000001e-05 2024-09-14T05:00:24.972 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.972 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.972 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:00:24.972 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:03.139165+0000", 2024-09-14T05:00:24.972 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.070111725999999999 2024-09-14T05:00:24.973 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.973 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.973 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:00:24.973 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:03.139193+0000", 2024-09-14T05:00:24.973 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.7555999999999998e-05 2024-09-14T05:00:24.973 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.973 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:00:24.973 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.973 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.973 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.973 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:165093 56.2 e1507/1489 56:46f83052:::benchmark_data_smithi039_330847_object50569:head v 1507'10474, mlcod=1474'3713)", 2024-09-14T05:00:24.973 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:55:07.936437+0000", 2024-09-14T05:00:24.973 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 316.97038383500001, 2024-09-14T05:00:24.973 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.087702006999999998, 2024-09-14T05:00:24.974 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:00:24.974 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:00:24.974 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:24.974 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:00:24.974 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.974 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:00:24.974 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:07.936437+0000", 2024-09-14T05:00:24.974 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.974 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.974 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.974 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:00:24.974 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:07.936437+0000", 2024-09-14T05:00:24.974 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.974 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.974 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.974 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:00:24.975 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:07.936438+0000", 2024-09-14T05:00:24.975 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.4600000000000004e-07 2024-09-14T05:00:24.975 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.975 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.975 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:00:24.975 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:07.936445+0000", 2024-09-14T05:00:24.975 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1110000000000002e-06 2024-09-14T05:00:24.975 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.975 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.975 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:00:24.975 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:07.936462+0000", 2024-09-14T05:00:24.975 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7167e-05 2024-09-14T05:00:24.975 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.975 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.975 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:00:24.976 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:07.936468+0000", 2024-09-14T05:00:24.976 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.4840000000000001e-06 2024-09-14T05:00:24.976 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.976 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.976 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:00:24.976 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:07.942583+0000", 2024-09-14T05:00:24.976 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0061142599999999998 2024-09-14T05:00:24.976 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.976 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.976 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:00:24.976 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:07.942639+0000", 2024-09-14T05:00:24.976 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6076999999999999e-05 2024-09-14T05:00:24.976 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.977 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.977 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:00:24.977 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:08.024095+0000", 2024-09-14T05:00:24.977 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.081455852999999995 2024-09-14T05:00:24.977 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.977 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.977 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:00:24.977 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:08.024139+0000", 2024-09-14T05:00:24.977 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4409000000000001e-05 2024-09-14T05:00:24.977 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.977 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:00:24.977 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.977 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.977 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.978 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36324.0:14778 57.4 e1523/1518 57:2041666d:::benchmark_data_smithi039_365565_object14777:head v 1523'929, mlcod=1523'927)", 2024-09-14T05:00:24.978 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:57:04.386486+0000", 2024-09-14T05:00:24.978 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 200.52033518799999, 2024-09-14T05:00:24.978 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.085796524999999998, 2024-09-14T05:00:24.978 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:00:24.978 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:00:24.978 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:24.978 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:00:24.978 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.978 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:00:24.978 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:04.386486+0000", 2024-09-14T05:00:24.978 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.978 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.978 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.978 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:00:24.979 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:04.386486+0000", 2024-09-14T05:00:24.979 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.979 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.979 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.979 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:00:24.979 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:04.386486+0000", 2024-09-14T05:00:24.979 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0599999999999999e-07 2024-09-14T05:00:24.979 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.979 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.979 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:00:24.979 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:04.387248+0000", 2024-09-14T05:00:24.979 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00076106700000000004 2024-09-14T05:00:24.979 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.979 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.980 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:00:24.980 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:04.387263+0000", 2024-09-14T05:00:24.980 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5073e-05 2024-09-14T05:00:24.980 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.980 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.981 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:00:24.981 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:04.387272+0000", 2024-09-14T05:00:24.981 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.1009999999999998e-06 2024-09-14T05:00:24.981 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.981 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.981 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:00:24.981 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:04.387336+0000", 2024-09-14T05:00:24.981 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.4090999999999993e-05 2024-09-14T05:00:24.981 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.981 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.981 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:00:24.981 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:04.387355+0000", 2024-09-14T05:00:24.981 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8848e-05 2024-09-14T05:00:24.982 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.982 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.982 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:00:24.982 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:04.472246+0000", 2024-09-14T05:00:24.982 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.084890917999999996 2024-09-14T05:00:24.982 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.982 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.982 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:00:24.982 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:04.472282+0000", 2024-09-14T05:00:24.982 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.6621000000000001e-05 2024-09-14T05:00:24.982 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.982 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:00:24.982 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.982 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.983 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.983 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36324.0:14791 57.5 e1523/1518 57:a445c0fc:::benchmark_data_smithi039_365565_object14790:head v 1523'927, mlcod=1523'925)", 2024-09-14T05:00:24.983 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:57:04.387112+0000", 2024-09-14T05:00:24.983 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 200.51970916499999, 2024-09-14T05:00:24.983 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.086393019000000001, 2024-09-14T05:00:24.983 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:00:24.983 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:00:24.983 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:24.983 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:00:24.983 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.983 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:00:24.983 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:04.387112+0000", 2024-09-14T05:00:24.983 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.983 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.984 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.984 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:00:24.984 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:04.387112+0000", 2024-09-14T05:00:24.984 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.984 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.984 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.984 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:00:24.984 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:04.387113+0000", 2024-09-14T05:00:24.984 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.4600000000000003e-07 2024-09-14T05:00:24.984 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.984 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.984 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:00:24.984 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:04.387152+0000", 2024-09-14T05:00:24.984 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.9233999999999997e-05 2024-09-14T05:00:24.985 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.985 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.985 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:00:24.985 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:04.387169+0000", 2024-09-14T05:00:24.985 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7516999999999998e-05 2024-09-14T05:00:24.985 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.985 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.985 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:00:24.985 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:04.387177+0000", 2024-09-14T05:00:24.985 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.2479999999999999e-06 2024-09-14T05:00:24.985 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.985 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.985 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:00:24.985 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:04.387261+0000", 2024-09-14T05:00:24.986 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.4478e-05 2024-09-14T05:00:24.986 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.986 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.986 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:00:24.986 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:04.387280+0000", 2024-09-14T05:00:24.986 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8510999999999999e-05 2024-09-14T05:00:24.986 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.986 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.986 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:00:24.986 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:04.473409+0000", 2024-09-14T05:00:24.986 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.086129655999999999 2024-09-14T05:00:24.986 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.986 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.987 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:00:24.987 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:04.473505+0000", 2024-09-14T05:00:24.987 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.5428999999999997e-05 2024-09-14T05:00:24.987 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.987 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:00:24.987 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.987 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.987 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.987 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36324.0:216665 57.d e1550/1518 57:beb9c787:::benchmark_data_smithi039_365565_object92913:head v 1550'13644, mlcod=1550'13641)", 2024-09-14T05:00:24.987 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:00:14.318722+0000", 2024-09-14T05:00:24.987 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 10.588099293999999, 2024-09-14T05:00:24.987 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.10493965600000001, 2024-09-14T05:00:24.987 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:00:24.988 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:00:24.988 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:24.988 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:00:24.988 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.988 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:00:24.988 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:14.318722+0000", 2024-09-14T05:00:24.988 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.988 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.988 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.988 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:00:24.988 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:14.318722+0000", 2024-09-14T05:00:24.988 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.989 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.989 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.989 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:00:24.989 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:14.318723+0000", 2024-09-14T05:00:24.989 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.018e-06 2024-09-14T05:00:24.989 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.989 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.989 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:00:24.989 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:14.318730+0000", 2024-09-14T05:00:24.989 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9619999999999996e-06 2024-09-14T05:00:24.989 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.989 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.989 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:00:24.989 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:14.318740+0000", 2024-09-14T05:00:24.990 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0501e-05 2024-09-14T05:00:24.990 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.990 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.990 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:00:24.990 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:14.318747+0000", 2024-09-14T05:00:24.990 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.3270000000000003e-06 2024-09-14T05:00:24.990 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.990 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.990 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:00:24.990 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:14.318804+0000", 2024-09-14T05:00:24.990 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6861000000000003e-05 2024-09-14T05:00:24.990 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.990 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.990 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:00:24.991 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:14.318825+0000", 2024-09-14T05:00:24.991 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0706000000000001e-05 2024-09-14T05:00:24.991 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.991 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.991 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:00:24.991 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:14.423615+0000", 2024-09-14T05:00:24.991 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.104789918 2024-09-14T05:00:24.991 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.991 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.991 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:00:24.991 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:14.423661+0000", 2024-09-14T05:00:24.991 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6363000000000003e-05 2024-09-14T05:00:24.991 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.992 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:00:24.992 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.992 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.992 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.992 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36324.0:216677 57.4 e1550/1518 57:25f61074:::benchmark_data_smithi039_365565_object92925:head v 1550'13399, mlcod=1550'13397)", 2024-09-14T05:00:24.992 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:00:14.323634+0000", 2024-09-14T05:00:24.992 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 10.583186953, 2024-09-14T05:00:24.992 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.103729369, 2024-09-14T05:00:24.992 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:00:24.992 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:00:24.992 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:24.992 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:00:24.992 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.992 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:00:24.993 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:14.323634+0000", 2024-09-14T05:00:24.993 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.993 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.993 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.993 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:00:24.993 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:14.323634+0000", 2024-09-14T05:00:24.993 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.993 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.993 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.993 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:00:24.993 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:14.323635+0000", 2024-09-14T05:00:24.993 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.358e-06 2024-09-14T05:00:24.993 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.993 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.994 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:00:24.994 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:14.323644+0000", 2024-09-14T05:00:24.994 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.7350000000000002e-06 2024-09-14T05:00:24.994 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.994 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.994 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:00:24.994 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:14.420025+0000", 2024-09-14T05:00:24.994 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.096381175999999999 2024-09-14T05:00:24.994 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.994 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.994 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:00:24.994 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:14.420034+0000", 2024-09-14T05:00:24.994 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.6619999999999994e-06 2024-09-14T05:00:24.994 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.994 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.995 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:00:24.995 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:14.420093+0000", 2024-09-14T05:00:24.995 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.9604000000000001e-05 2024-09-14T05:00:24.995 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.995 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.995 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:00:24.995 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:14.420115+0000", 2024-09-14T05:00:24.995 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1707000000000001e-05 2024-09-14T05:00:24.995 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.995 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.995 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:00:24.995 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:14.427262+0000", 2024-09-14T05:00:24.996 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0071464509999999998 2024-09-14T05:00:24.996 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.996 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.996 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:00:24.996 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:14.427363+0000", 2024-09-14T05:00:24.996 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00010167599999999999 2024-09-14T05:00:24.996 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.996 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:00:24.996 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:24.996 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.996 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.996 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36324.0:216684 57.a e1550/1518 57:5bdbe016:::benchmark_data_smithi039_365565_object92932:head v 1550'13751, mlcod=1550'13749)", 2024-09-14T05:00:24.996 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:00:14.331524+0000", 2024-09-14T05:00:24.997 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 10.575296435, 2024-09-14T05:00:24.997 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.108100196, 2024-09-14T05:00:24.997 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:00:24.997 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:00:24.997 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:24.997 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:00:24.997 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.997 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:00:24.997 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:14.331524+0000", 2024-09-14T05:00:24.997 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.997 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.997 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.997 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:00:24.997 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:14.331524+0000", 2024-09-14T05:00:24.998 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:00:24.998 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.998 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.998 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:00:24.998 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:14.331526+0000", 2024-09-14T05:00:24.998 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.463e-06 2024-09-14T05:00:24.998 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.998 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.998 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:00:24.998 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:14.331538+0000", 2024-09-14T05:00:24.998 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1856e-05 2024-09-14T05:00:24.998 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.999 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.999 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:00:24.999 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:14.420005+0000", 2024-09-14T05:00:24.999 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.088467004000000002 2024-09-14T05:00:24.999 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.999 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.999 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:00:24.999 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:14.420017+0000", 2024-09-14T05:00:24.999 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2628e-05 2024-09-14T05:00:24.999 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:24.999 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:24.999 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:00:24.999 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:14.420075+0000", 2024-09-14T05:00:24.999 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7745999999999999e-05 2024-09-14T05:00:25.000 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:25.000 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:25.000 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:00:25.000 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:14.420099+0000", 2024-09-14T05:00:25.000 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3833999999999999e-05 2024-09-14T05:00:25.000 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:25.000 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:25.000 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:00:25.000 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:14.439442+0000", 2024-09-14T05:00:25.001 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.019342959 2024-09-14T05:00:25.001 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:00:25.001 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:00:25.001 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:00:25.001 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:14.439625+0000", 2024-09-14T05:00:25.001 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00018270600000000001 2024-09-14T05:00:25.001 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:25.001 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:00:25.001 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:25.001 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:00:25.001 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:00:25.001 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T05:00:25.018 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:25.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:25.019 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:25.120 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:25.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:25.150 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:25.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:25.251 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:25.297 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:25.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:25.329 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:25.430 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:25.482 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:25 smithi039 ceph-mon[106470]: pgmap v3048: 109 pgs: 1 unknown, 108 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 0 B/s, 0 objects/s recovering 2024-09-14T05:00:25.482 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:25 smithi039 ceph-mon[106470]: osdmap e1556: 8 total, 8 up, 8 in 2024-09-14T05:00:25.482 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:25 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4182424613' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:00:25.483 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:25 smithi039 ceph-mon[111235]: pgmap v3048: 109 pgs: 1 unknown, 108 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 0 B/s, 0 objects/s recovering 2024-09-14T05:00:25.483 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:25 smithi039 ceph-mon[111235]: osdmap e1556: 8 total, 8 up, 8 in 2024-09-14T05:00:25.483 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:25 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4182424613' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:00:25.487 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:25.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:25.489 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:25.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:25 smithi129 ceph-mon[101316]: pgmap v3048: 109 pgs: 1 unknown, 108 active+clean; 1.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 819 op/s; 0 B/s, 0 objects/s recovering 2024-09-14T05:00:25.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:25 smithi129 ceph-mon[101316]: osdmap e1556: 8 total, 8 up, 8 in 2024-09-14T05:00:25.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:25 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4182424613' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:00:25.589 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:25.625 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:25.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:25.626 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:25.726 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:25.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:25.758 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:25.758 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:25.859 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:25.905 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:25.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:25.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:25.959 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-14T05:00:26.007 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:26.051 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:26.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:26.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:26.087 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T05:00:26.087 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T05:00:26.087 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T05:00:26.087 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T05:00:26.088 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T05:00:26.088 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T05:00:26.088 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T05:00:26.088 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T05:00:26.088 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T05:00:26.088 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T05:00:26.088 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T05:00:26.088 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T05:00:26.089 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T05:00:26.089 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T05:00:26.089 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T05:00:26.089 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T05:00:26.152 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:26.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:26.188 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:26.188 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:26.289 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:26.325 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:00:26.328 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:26.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:26.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:26.430 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:26.471 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:26.472 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:26.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:26.573 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:26.609 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:00:26.610 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:26.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:26.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:26.712 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:26.739 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:26.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:26.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:26.841 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:26.874 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:26.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:26.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:26.976 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:27.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:27.009 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:27.009 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:27.110 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:27.150 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:27.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:27.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:27.252 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:27.291 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:27.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:27.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:27.393 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:27.438 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:27.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:27.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:27.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:27 smithi129 ceph-mon[101316]: pgmap v3050: 109 pgs: 109 active+clean; 1.4 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 941 op/s; 54 KiB/s, 1 objects/s recovering 2024-09-14T05:00:27.539 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:27.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:27 smithi039 ceph-mon[106470]: pgmap v3050: 109 pgs: 109 active+clean; 1.4 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 941 op/s; 54 KiB/s, 1 objects/s recovering 2024-09-14T05:00:27.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:27 smithi039 ceph-mon[111235]: pgmap v3050: 109 pgs: 109 active+clean; 1.4 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 941 op/s; 54 KiB/s, 1 objects/s recovering 2024-09-14T05:00:27.576 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:27.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:27.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:27.678 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:27.724 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:27.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:27.727 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:27.828 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:27.875 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:27.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:27.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:27.977 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:28.014 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:28.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:28.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:28.116 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:28.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:28.152 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:28.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:28.253 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:28.294 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:28.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:28.295 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:28.396 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:28.441 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:28.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:28.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:28.461 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T05:00:28.461 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:00:28.542 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:28.580 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:28.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:28.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:28.682 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:28.717 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:28.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:28.718 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:28.789 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:00:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:00:28] "GET /metrics HTTP/1.1" 200 36754 "" "Prometheus/2.51.0" 2024-09-14T05:00:28.819 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:28.849 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:28.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:28.851 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:28.951 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:28.990 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:28.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:28.991 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:29.000 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-14T05:00:29.000 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T05:00:29.000 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_4 2024-09-14T05:00:29.000 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T05:00:29.092 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:29.118 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:29.119 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:29.119 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:29.185 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-14T05:00:29.220 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:29.275 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:29.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:29.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:29.377 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:29.430 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:29.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:29.431 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:29.510 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:00:29.522 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:29 smithi129 ceph-mon[101316]: pgmap v3051: 109 pgs: 109 active+clean; 1.3 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 944 op/s; 43 KiB/s, 0 objects/s recovering 2024-09-14T05:00:29.527 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:29 smithi039 ceph-mon[106470]: pgmap v3051: 109 pgs: 109 active+clean; 1.3 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 944 op/s; 43 KiB/s, 0 objects/s recovering 2024-09-14T05:00:29.527 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:29 smithi039 ceph-mon[111235]: pgmap v3051: 109 pgs: 109 active+clean; 1.3 GiB data, 9.6 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 944 op/s; 43 KiB/s, 0 objects/s recovering 2024-09-14T05:00:29.532 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:29.581 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:29.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:29.582 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:29.598 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T05:00:29.598 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:00:29.683 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:29.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:29.753 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:29.753 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:29.854 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:29.898 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:29.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:29.899 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:30.000 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:30.036 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:30.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:30.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:30.138 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:30.170 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:30.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:30.171 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:30.272 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:30.297 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is unset 2024-09-14T05:00:30.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:30.307 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:30.307 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:30.408 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:30.449 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:30.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:30.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:30.497 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:30 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3251507651' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:00:30.497 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:30 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:00:30.498 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:30 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3251507651' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:00:30.498 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:30 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:00:30.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:30 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3251507651' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:00:30.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:30 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:00:30.551 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:30.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:30.586 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:30.586 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:30.687 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:30.730 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:30.731 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:30.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:30.832 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:30.862 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:30.863 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:30.863 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:30.964 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:31.005 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:31.006 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:31.007 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:31.108 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:31.140 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:31.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:31.141 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:31.242 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:31.301 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:31.302 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:31.303 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:31.403 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:31.437 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:31.437 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:31.438 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:31.538 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:31.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:31 smithi039 ceph-mon[106470]: pgmap v3052: 109 pgs: 109 active+clean; 1.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 990 op/s; 33 KiB/s, 0 objects/s recovering 2024-09-14T05:00:31.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:31 smithi039 ceph-mon[106470]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:00:31.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:31 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T05:00:31.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:31 smithi039 ceph-mon[106470]: osdmap e1557: 8 total, 8 up, 8 in 2024-09-14T05:00:31.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:31 smithi039 ceph-mon[111235]: pgmap v3052: 109 pgs: 109 active+clean; 1.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 990 op/s; 33 KiB/s, 0 objects/s recovering 2024-09-14T05:00:31.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:31 smithi039 ceph-mon[111235]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:00:31.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:31 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T05:00:31.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:31 smithi039 ceph-mon[111235]: osdmap e1557: 8 total, 8 up, 8 in 2024-09-14T05:00:31.574 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:31.575 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:31.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:31.676 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:31.724 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:31.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:31.726 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:31.736 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:31 smithi129 ceph-mon[101316]: pgmap v3052: 109 pgs: 109 active+clean; 1.2 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 990 op/s; 33 KiB/s, 0 objects/s recovering 2024-09-14T05:00:31.736 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:31 smithi129 ceph-mon[101316]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:00:31.737 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:31 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T05:00:31.737 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:31 smithi129 ceph-mon[101316]: osdmap e1557: 8 total, 8 up, 8 in 2024-09-14T05:00:31.826 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:31.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:31.861 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:31.861 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:31.962 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:31.999 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:32.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:32.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:32.101 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:32.129 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:32.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:32.130 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:32.231 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:32.270 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:32.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:32.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:32.372 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:32.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:32.411 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:32.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:32.433 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:32 smithi039 ceph-mon[106470]: 1.5 scrub starts 2024-09-14T05:00:32.433 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:32 smithi039 ceph-mon[106470]: 1.5 scrub ok 2024-09-14T05:00:32.433 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:32 smithi039 ceph-mon[106470]: 1.4 scrub starts 2024-09-14T05:00:32.433 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:32 smithi039 ceph-mon[106470]: 1.4 scrub ok 2024-09-14T05:00:32.433 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:32 smithi039 ceph-mon[111235]: 1.5 scrub starts 2024-09-14T05:00:32.433 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:32 smithi039 ceph-mon[111235]: 1.5 scrub ok 2024-09-14T05:00:32.433 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:32 smithi039 ceph-mon[111235]: 1.4 scrub starts 2024-09-14T05:00:32.434 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:32 smithi039 ceph-mon[111235]: 1.4 scrub ok 2024-09-14T05:00:32.441 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:00:32.512 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:32.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:32.563 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:32.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:32.664 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:32.672 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:32 smithi129 ceph-mon[101316]: 1.5 scrub starts 2024-09-14T05:00:32.672 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:32 smithi129 ceph-mon[101316]: 1.5 scrub ok 2024-09-14T05:00:32.673 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:32 smithi129 ceph-mon[101316]: 1.4 scrub starts 2024-09-14T05:00:32.673 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:32 smithi129 ceph-mon[101316]: 1.4 scrub ok 2024-09-14T05:00:32.709 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:32.710 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:32.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:32.811 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:32.823 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T05:00:32.824 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T05:00:32.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:32.847 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:32.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:32.948 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:32.987 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:32.987 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:32.987 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:33.088 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:33.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:33.120 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:33.120 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:33.222 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:33.261 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:33.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:33.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:33.363 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:33.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:33.401 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:33.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:33.459 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:33 smithi129 ceph-mon[101316]: pgmap v3054: 109 pgs: 109 active+clean; 1.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 32 KiB/s, 0 objects/s recovering 2024-09-14T05:00:33.502 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:33.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:33.544 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:33.544 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:33.627 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T05:00:33.627 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:00:33.644 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:33 smithi039 ceph-mon[106470]: pgmap v3054: 109 pgs: 109 active+clean; 1.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 32 KiB/s, 0 objects/s recovering 2024-09-14T05:00:33.644 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:33 smithi039 ceph-mon[111235]: pgmap v3054: 109 pgs: 109 active+clean; 1.1 GiB data, 9.0 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 900 op/s; 32 KiB/s, 0 objects/s recovering 2024-09-14T05:00:33.645 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:33.684 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:33.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:33.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:33.699 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T05:00:33.786 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:33.817 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:33.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:33.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:33.919 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:33.962 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:33.963 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:33.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:34.064 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:34.106 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:34.106 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:34.107 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:34.207 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:34.245 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:34.245 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:34.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:34.347 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:34.352 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-14T05:00:34.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:34.386 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:34.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:34.488 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:34.522 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:34.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:34.523 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:34.539 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:34 smithi039 ceph-mon[106470]: from='client.46457 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T05:00:34.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:34 smithi039 ceph-mon[106470]: 1.0 scrub starts 2024-09-14T05:00:34.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:34 smithi039 ceph-mon[106470]: 1.0 scrub ok 2024-09-14T05:00:34.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:34 smithi039 ceph-mon[106470]: 1.1 scrub starts 2024-09-14T05:00:34.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:34 smithi039 ceph-mon[106470]: 1.1 scrub ok 2024-09-14T05:00:34.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:34 smithi039 ceph-mon[111235]: from='client.46457 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T05:00:34.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:34 smithi039 ceph-mon[111235]: 1.0 scrub starts 2024-09-14T05:00:34.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:34 smithi039 ceph-mon[111235]: 1.0 scrub ok 2024-09-14T05:00:34.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:34 smithi039 ceph-mon[111235]: 1.1 scrub starts 2024-09-14T05:00:34.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:34 smithi039 ceph-mon[111235]: 1.1 scrub ok 2024-09-14T05:00:34.624 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:34.670 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:34.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:34.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:34.729 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":3054,"stamp":"2024-09-14T05:00:31.985531+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1156469366,"num_objects":17664,"num_object_clones":0,"num_object_copies":52992,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17664,"num_whiteouts":0,"num_read":53,"num_read_kb":114,"num_write":229957,"num_write_kb":7920300,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":1822713,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":132302,"ondisk_log_size":132302,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":9489228,"kb_used_data":3682252,"kb_used_omap":404,"kb_used_meta":5806443,"kb_avail":740471988,"statfs":{"total":767960285184,"available":758243315712,"internally_reserved":0,"allocated":3770626048,"data_stored":3726260499,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":414022,"internal_metadata":5945798330},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-472383488,"num_objects":-7208,"num_object_clones":0,"num_object_copies":-21624,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7208,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7208,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003005"},"pg_stats":[{"pgid":"57.f","version":"1557'14428","reported_seq":14518,"reported_epoch":1557,"state":"active+clean","last_fresh":"2024-09-14T05:00:30.582754+0000","last_change":"2024-09-14T04:56:49.435182+0000","last_active":"2024-09-14T05:00:30.582754+0000","last_peered":"2024-09-14T05:00:30.582754+0000","last_clean":"2024-09-14T05:00:30.582754+0000","last_became_active":"2024-09-14T04:56:49.434949+0000","last_became_peered":"2024-09-14T04:56:49.434949+0000","last_unstale":"2024-09-14T05:00:30.582754+0000","last_undegraded":"2024-09-14T05:00:30.582754+0000","last_fullsized":"2024-09-14T05:00:30.582754+0000","mapping_epoch":1518,"log_start":"1528'5261","ondisk_log_start":"1528'5261","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":9167,"log_dups_size":0,"ondisk_log_size":9167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:32:29.856162+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67895296,"num_objects":1036,"num_object_clones":0,"num_object_copies":3108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1036,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14428,"num_write_kb":494848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"57.e","version":"1556'14138","reported_seq":14225,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:29.589371+0000","last_change":"2024-09-14T04:56:49.431930+0000","last_active":"2024-09-14T05:00:29.589371+0000","last_peered":"2024-09-14T05:00:29.589371+0000","last_clean":"2024-09-14T05:00:29.589371+0000","last_became_active":"2024-09-14T04:56:49.431642+0000","last_became_peered":"2024-09-14T04:56:49.431642+0000","last_unstale":"2024-09-14T05:00:29.589371+0000","last_undegraded":"2024-09-14T05:00:29.589371+0000","last_fullsized":"2024-09-14T05:00:29.589371+0000","mapping_epoch":1518,"log_start":"1530'6431","ondisk_log_start":"1530'6431","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":7707,"log_dups_size":0,"ondisk_log_size":7707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:30:17.677827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70647808,"num_objects":1078,"num_object_clones":0,"num_object_copies":3234,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1078,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14138,"num_write_kb":486912,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"57.d","version":"1556'14347","reported_seq":14396,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:27.976566+0000","last_change":"2024-09-14T04:56:49.821152+0000","last_active":"2024-09-14T05:00:27.976566+0000","last_peered":"2024-09-14T05:00:27.976566+0000","last_clean":"2024-09-14T05:00:27.976566+0000","last_became_active":"2024-09-14T04:56:49.820853+0000","last_became_peered":"2024-09-14T04:56:49.820853+0000","last_unstale":"2024-09-14T05:00:27.976566+0000","last_undegraded":"2024-09-14T05:00:27.976566+0000","last_fullsized":"2024-09-14T05:00:27.976566+0000","mapping_epoch":1518,"log_start":"1532'7807","ondisk_log_start":"1532'7807","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":6540,"log_dups_size":0,"ondisk_log_size":6540,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:40:41.229640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78118936,"num_objects":1193,"num_object_clones":0,"num_object_copies":3579,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1193,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":14347,"num_write_kb":497217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"57.c","version":"1556'14541","reported_seq":14589,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:27.959758+0000","last_change":"2024-09-14T04:56:49.435207+0000","last_active":"2024-09-14T05:00:27.959758+0000","last_peered":"2024-09-14T05:00:27.959758+0000","last_clean":"2024-09-14T05:00:27.959758+0000","last_became_active":"2024-09-14T04:56:49.434752+0000","last_became_peered":"2024-09-14T04:56:49.434752+0000","last_unstale":"2024-09-14T05:00:27.959758+0000","last_undegraded":"2024-09-14T05:00:27.959758+0000","last_fullsized":"2024-09-14T05:00:27.959758+0000","mapping_epoch":1518,"log_start":"1532'7950","ondisk_log_start":"1532'7950","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":6591,"log_dups_size":0,"ondisk_log_size":6591,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:29:43.013481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77660160,"num_objects":1185,"num_object_clones":0,"num_object_copies":3555,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1185,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14541,"num_write_kb":503232,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"57.b","version":"1557'14499","reported_seq":14589,"reported_epoch":1557,"state":"active+clean","last_fresh":"2024-09-14T05:00:30.536290+0000","last_change":"2024-09-14T04:56:49.434067+0000","last_active":"2024-09-14T05:00:30.536290+0000","last_peered":"2024-09-14T05:00:30.536290+0000","last_clean":"2024-09-14T05:00:30.536290+0000","last_became_active":"2024-09-14T04:56:49.433725+0000","last_became_peered":"2024-09-14T04:56:49.433725+0000","last_unstale":"2024-09-14T05:00:30.536290+0000","last_undegraded":"2024-09-14T05:00:30.536290+0000","last_fullsized":"2024-09-14T05:00:30.536290+0000","mapping_epoch":1518,"log_start":"1528'5364","ondisk_log_start":"1528'5364","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":9135,"log_dups_size":0,"ondisk_log_size":9135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:18:56.597341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65732608,"num_objects":1003,"num_object_clones":0,"num_object_copies":3009,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1003,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14499,"num_write_kb":496064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,4],"acting":[2,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"57.a","version":"1556'14453","reported_seq":14501,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:27.823058+0000","last_change":"2024-09-14T04:56:49.818406+0000","last_active":"2024-09-14T05:00:27.823058+0000","last_peered":"2024-09-14T05:00:27.823058+0000","last_clean":"2024-09-14T05:00:27.823058+0000","last_became_active":"2024-09-14T04:56:49.818087+0000","last_became_peered":"2024-09-14T04:56:49.818087+0000","last_unstale":"2024-09-14T05:00:27.823058+0000","last_undegraded":"2024-09-14T05:00:27.823058+0000","last_fullsized":"2024-09-14T05:00:27.823058+0000","mapping_epoch":1518,"log_start":"1528'6107","ondisk_log_start":"1528'6107","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":8346,"log_dups_size":0,"ondisk_log_size":8346,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:41:19.568727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77529088,"num_objects":1183,"num_object_clones":0,"num_object_copies":3549,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1183,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14453,"num_write_kb":500352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"57.9","version":"1556'14238","reported_seq":14286,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:28.417024+0000","last_change":"2024-09-14T04:56:49.818923+0000","last_active":"2024-09-14T05:00:28.417024+0000","last_peered":"2024-09-14T05:00:28.417024+0000","last_clean":"2024-09-14T05:00:28.417024+0000","last_became_active":"2024-09-14T04:56:49.818519+0000","last_became_peered":"2024-09-14T04:56:49.818519+0000","last_unstale":"2024-09-14T05:00:28.417024+0000","last_undegraded":"2024-09-14T05:00:28.417024+0000","last_fullsized":"2024-09-14T05:00:28.417024+0000","mapping_epoch":1518,"log_start":"1532'7487","ondisk_log_start":"1532'7487","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":6751,"log_dups_size":0,"ondisk_log_size":6751,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:41:08.309897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":73531392,"num_objects":1122,"num_object_clones":0,"num_object_copies":3366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1122,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14238,"num_write_kb":491520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,5],"acting":[4,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"57.8","version":"1556'14140","reported_seq":14188,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:27.791117+0000","last_change":"2024-09-14T04:56:49.434525+0000","last_active":"2024-09-14T05:00:27.791117+0000","last_peered":"2024-09-14T05:00:27.791117+0000","last_clean":"2024-09-14T05:00:27.791117+0000","last_became_active":"2024-09-14T04:56:49.434144+0000","last_became_peered":"2024-09-14T04:56:49.434144+0000","last_unstale":"2024-09-14T05:00:27.791117+0000","last_undegraded":"2024-09-14T05:00:27.791117+0000","last_fullsized":"2024-09-14T05:00:27.791117+0000","mapping_epoch":1518,"log_start":"1531'7152","ondisk_log_start":"1531'7152","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":6988,"log_dups_size":0,"ondisk_log_size":6988,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:17:18.918905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74973184,"num_objects":1144,"num_object_clones":0,"num_object_copies":3432,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1144,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14140,"num_write_kb":489088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,0],"acting":[7,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"57.7","version":"1557'14475","reported_seq":14565,"reported_epoch":1557,"state":"active+clean","last_fresh":"2024-09-14T05:00:30.584943+0000","last_change":"2024-09-14T04:56:49.434581+0000","last_active":"2024-09-14T05:00:30.584943+0000","last_peered":"2024-09-14T05:00:30.584943+0000","last_clean":"2024-09-14T05:00:30.584943+0000","last_became_active":"2024-09-14T04:56:49.434210+0000","last_became_peered":"2024-09-14T04:56:49.434210+0000","last_unstale":"2024-09-14T05:00:30.584943+0000","last_undegraded":"2024-09-14T05:00:30.584943+0000","last_fullsized":"2024-09-14T05:00:30.584943+0000","mapping_epoch":1518,"log_start":"1528'5344","ondisk_log_start":"1528'5344","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":9131,"log_dups_size":0,"ondisk_log_size":9131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:16.426758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":68222976,"num_objects":1041,"num_object_clones":0,"num_object_copies":3123,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1041,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14475,"num_write_kb":496512,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,6],"acting":[2,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"57.6","version":"1556'14390","reported_seq":14438,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:27.825690+0000","last_change":"2024-09-14T04:56:49.436357+0000","last_active":"2024-09-14T05:00:27.825690+0000","last_peered":"2024-09-14T05:00:27.825690+0000","last_clean":"2024-09-14T05:00:27.825690+0000","last_became_active":"2024-09-14T04:56:49.436101+0000","last_became_peered":"2024-09-14T04:56:49.436101+0000","last_unstale":"2024-09-14T05:00:27.825690+0000","last_undegraded":"2024-09-14T05:00:27.825690+0000","last_fullsized":"2024-09-14T05:00:27.825690+0000","mapping_epoch":1518,"log_start":"1528'6045","ondisk_log_start":"1528'6045","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":8345,"log_dups_size":0,"ondisk_log_size":8345,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:03:53.232843+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":74842112,"num_objects":1142,"num_object_clones":0,"num_object_copies":3426,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1142,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14390,"num_write_kb":497024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"57.5","version":"1556'14259","reported_seq":14307,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:27.848809+0000","last_change":"2024-09-14T04:56:49.821285+0000","last_active":"2024-09-14T05:00:27.848809+0000","last_peered":"2024-09-14T05:00:27.848809+0000","last_clean":"2024-09-14T05:00:27.848809+0000","last_became_active":"2024-09-14T04:56:49.820869+0000","last_became_peered":"2024-09-14T04:56:49.820869+0000","last_unstale":"2024-09-14T05:00:27.848809+0000","last_undegraded":"2024-09-14T05:00:27.848809+0000","last_fullsized":"2024-09-14T05:00:27.848809+0000","mapping_epoch":1518,"log_start":"1528'5943","ondisk_log_start":"1528'5943","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":8316,"log_dups_size":0,"ondisk_log_size":8316,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:26:39.392722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77922304,"num_objects":1189,"num_object_clones":0,"num_object_copies":3567,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1189,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14259,"num_write_kb":494336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"57.4","version":"1556'14171","reported_seq":14259,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:29.237079+0000","last_change":"2024-09-14T04:56:49.820010+0000","last_active":"2024-09-14T05:00:29.237079+0000","last_peered":"2024-09-14T05:00:29.237079+0000","last_clean":"2024-09-14T05:00:29.237079+0000","last_became_active":"2024-09-14T04:56:49.819563+0000","last_became_peered":"2024-09-14T04:56:49.819563+0000","last_unstale":"2024-09-14T05:00:29.237079+0000","last_undegraded":"2024-09-14T05:00:29.237079+0000","last_fullsized":"2024-09-14T05:00:29.237079+0000","mapping_epoch":1518,"log_start":"1530'6269","ondisk_log_start":"1530'6269","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":7902,"log_dups_size":0,"ondisk_log_size":7902,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:32:31.141098+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70844416,"num_objects":1081,"num_object_clones":0,"num_object_copies":3243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14171,"num_write_kb":488064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"57.3","version":"1557'14428","reported_seq":14518,"reported_epoch":1557,"state":"active+clean","last_fresh":"2024-09-14T05:00:30.562449+0000","last_change":"2024-09-14T04:56:49.819031+0000","last_active":"2024-09-14T05:00:30.562449+0000","last_peered":"2024-09-14T05:00:30.562449+0000","last_clean":"2024-09-14T05:00:30.562449+0000","last_became_active":"2024-09-14T04:56:49.818654+0000","last_became_peered":"2024-09-14T04:56:49.818654+0000","last_unstale":"2024-09-14T05:00:30.562449+0000","last_undegraded":"2024-09-14T05:00:30.562449+0000","last_fullsized":"2024-09-14T05:00:30.562449+0000","mapping_epoch":1518,"log_start":"1528'5270","ondisk_log_start":"1528'5270","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":9158,"log_dups_size":0,"ondisk_log_size":9158,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:57:37.054801+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67502080,"num_objects":1030,"num_object_clones":0,"num_object_copies":3090,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1030,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14428,"num_write_kb":494656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"57.2","version":"1556'14336","reported_seq":14424,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:29.176916+0000","last_change":"2024-09-14T04:56:49.819908+0000","last_active":"2024-09-14T05:00:29.176916+0000","last_peered":"2024-09-14T05:00:29.176916+0000","last_clean":"2024-09-14T05:00:29.176916+0000","last_became_active":"2024-09-14T04:56:49.819599+0000","last_became_peered":"2024-09-14T04:56:49.819599+0000","last_unstale":"2024-09-14T05:00:29.176916+0000","last_undegraded":"2024-09-14T05:00:29.176916+0000","last_fullsized":"2024-09-14T05:00:29.176916+0000","mapping_epoch":1518,"log_start":"1530'6348","ondisk_log_start":"1530'6348","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":7988,"log_dups_size":0,"ondisk_log_size":7988,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:38:49.493483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72744960,"num_objects":1110,"num_object_clones":0,"num_object_copies":3330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1110,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14336,"num_write_kb":494272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"57.1","version":"1556'14633","reported_seq":14721,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:29.217791+0000","last_change":"2024-09-14T04:56:49.434507+0000","last_active":"2024-09-14T05:00:29.217791+0000","last_peered":"2024-09-14T05:00:29.217791+0000","last_clean":"2024-09-14T05:00:29.217791+0000","last_became_active":"2024-09-14T04:56:49.434172+0000","last_became_peered":"2024-09-14T04:56:49.434172+0000","last_unstale":"2024-09-14T05:00:29.217791+0000","last_undegraded":"2024-09-14T05:00:29.217791+0000","last_fullsized":"2024-09-14T05:00:29.217791+0000","mapping_epoch":1518,"log_start":"1530'6719","ondisk_log_start":"1530'6719","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":7914,"log_dups_size":0,"ondisk_log_size":7914,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:22.424289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":71761920,"num_objects":1095,"num_object_clones":0,"num_object_copies":3285,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1095,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14633,"num_write_kb":503296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,2],"acting":[0,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"57.0","version":"1557'14387","reported_seq":14477,"reported_epoch":1557,"state":"active+clean","last_fresh":"2024-09-14T05:00:30.588085+0000","last_change":"2024-09-14T04:56:49.818610+0000","last_active":"2024-09-14T05:00:30.588085+0000","last_peered":"2024-09-14T05:00:30.588085+0000","last_clean":"2024-09-14T05:00:30.588085+0000","last_became_active":"2024-09-14T04:56:49.818300+0000","last_became_peered":"2024-09-14T04:56:49.818300+0000","last_unstale":"2024-09-14T05:00:30.588085+0000","last_undegraded":"2024-09-14T05:00:30.588085+0000","last_fullsized":"2024-09-14T05:00:30.588085+0000","mapping_epoch":1518,"log_start":"1527'5252","ondisk_log_start":"1527'5252","created":1518,"last_epoch_clean":1519,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T04:56:48.405307+0000","last_clean_scrub_stamp":"2024-09-14T04:56:48.405307+0000","objects_scrubbed":0,"log_size":9135,"log_dups_size":0,"ondisk_log_size":9135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:29:20.641683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65863680,"num_objects":1005,"num_object_clones":0,"num_object_copies":3015,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1005,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":14387,"num_write_kb":492544,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":2407,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:24.266467+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T05:00:24.266467+0000","last_peered":"2024-09-14T05:00:24.266467+0000","last_clean":"2024-09-14T05:00:24.266467+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T05:00:24.266467+0000","last_undegraded":"2024-09-14T05:00:24.266467+0000","last_fullsized":"2024-09-14T05:00:24.266467+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":170,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.242766+0000","last_change":"2024-09-14T04:44:57.325109+0000","last_active":"2024-09-14T05:00:23.242766+0000","last_peered":"2024-09-14T05:00:23.242766+0000","last_clean":"2024-09-14T05:00:23.242766+0000","last_became_active":"2024-09-14T04:44:57.324791+0000","last_became_peered":"2024-09-14T04:44:57.324791+0000","last_unstale":"2024-09-14T05:00:23.242766+0000","last_undegraded":"2024-09-14T05:00:23.242766+0000","last_fullsized":"2024-09-14T05:00:23.242766+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:03.593275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":140,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.242385+0000","last_change":"2024-09-14T04:49:29.628435+0000","last_active":"2024-09-14T05:00:23.242385+0000","last_peered":"2024-09-14T05:00:23.242385+0000","last_clean":"2024-09-14T05:00:23.242385+0000","last_became_active":"2024-09-14T04:49:29.628081+0000","last_became_peered":"2024-09-14T04:49:29.628081+0000","last_unstale":"2024-09-14T05:00:23.242385+0000","last_undegraded":"2024-09-14T05:00:23.242385+0000","last_fullsized":"2024-09-14T05:00:23.242385+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:01.520334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1030,"reported_epoch":1557,"state":"active+clean","last_fresh":"2024-09-14T05:00:30.310045+0000","last_change":"2024-09-14T04:43:13.355281+0000","last_active":"2024-09-14T05:00:30.310045+0000","last_peered":"2024-09-14T05:00:30.310045+0000","last_clean":"2024-09-14T05:00:30.310045+0000","last_became_active":"2024-09-14T04:43:13.354967+0000","last_became_peered":"2024-09-14T04:43:13.354967+0000","last_unstale":"2024-09-14T05:00:30.310045+0000","last_undegraded":"2024-09-14T05:00:30.310045+0000","last_fullsized":"2024-09-14T05:00:30.310045+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:23:19.332075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1548,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.250657+0000","last_change":"2024-09-14T04:44:57.342918+0000","last_active":"2024-09-14T05:00:23.250657+0000","last_peered":"2024-09-14T05:00:23.250657+0000","last_clean":"2024-09-14T05:00:23.250657+0000","last_became_active":"2024-09-14T04:44:57.329550+0000","last_became_peered":"2024-09-14T04:44:57.329550+0000","last_unstale":"2024-09-14T05:00:23.250657+0000","last_undegraded":"2024-09-14T05:00:23.250657+0000","last_fullsized":"2024-09-14T05:00:23.250657+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:52:52.435893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":170,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.241642+0000","last_change":"2024-09-14T04:44:57.325537+0000","last_active":"2024-09-14T05:00:23.241642+0000","last_peered":"2024-09-14T05:00:23.241642+0000","last_clean":"2024-09-14T05:00:23.241642+0000","last_became_active":"2024-09-14T04:44:57.325057+0000","last_became_peered":"2024-09-14T04:44:57.325057+0000","last_unstale":"2024-09-14T05:00:23.241642+0000","last_undegraded":"2024-09-14T05:00:23.241642+0000","last_fullsized":"2024-09-14T05:00:23.241642+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:43:23.166708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1563,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.248974+0000","last_change":"2024-09-14T04:44:57.337033+0000","last_active":"2024-09-14T05:00:23.248974+0000","last_peered":"2024-09-14T05:00:23.248974+0000","last_clean":"2024-09-14T05:00:23.248974+0000","last_became_active":"2024-09-14T04:44:57.328437+0000","last_became_peered":"2024-09-14T04:44:57.328437+0000","last_unstale":"2024-09-14T05:00:23.248974+0000","last_undegraded":"2024-09-14T05:00:23.248974+0000","last_fullsized":"2024-09-14T05:00:23.248974+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.109086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":170,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.245075+0000","last_change":"2024-09-14T04:44:57.332243+0000","last_active":"2024-09-14T05:00:23.245075+0000","last_peered":"2024-09-14T05:00:23.245075+0000","last_clean":"2024-09-14T05:00:23.245075+0000","last_became_active":"2024-09-14T04:44:57.330413+0000","last_became_peered":"2024-09-14T04:44:57.330413+0000","last_unstale":"2024-09-14T05:00:23.245075+0000","last_undegraded":"2024-09-14T05:00:23.245075+0000","last_fullsized":"2024-09-14T05:00:23.245075+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:25:01.338549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":262,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:24.265912+0000","last_change":"2024-09-14T04:48:29.862929+0000","last_active":"2024-09-14T05:00:24.265912+0000","last_peered":"2024-09-14T05:00:24.265912+0000","last_clean":"2024-09-14T05:00:24.265912+0000","last_became_active":"2024-09-14T04:48:29.862581+0000","last_became_peered":"2024-09-14T04:48:29.862581+0000","last_unstale":"2024-09-14T05:00:24.265912+0000","last_undegraded":"2024-09-14T05:00:24.265912+0000","last_fullsized":"2024-09-14T05:00:24.265912+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:16:43.060088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2404,"reported_epoch":1557,"state":"active+clean","last_fresh":"2024-09-14T05:00:30.316717+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T05:00:30.316717+0000","last_peered":"2024-09-14T05:00:30.316717+0000","last_clean":"2024-09-14T05:00:30.316717+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T05:00:30.316717+0000","last_undegraded":"2024-09-14T05:00:30.316717+0000","last_fullsized":"2024-09-14T05:00:30.316717+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":170,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.241571+0000","last_change":"2024-09-14T04:44:57.325392+0000","last_active":"2024-09-14T05:00:23.241571+0000","last_peered":"2024-09-14T05:00:23.241571+0000","last_clean":"2024-09-14T05:00:23.241571+0000","last_became_active":"2024-09-14T04:44:57.324624+0000","last_became_peered":"2024-09-14T04:44:57.324624+0000","last_unstale":"2024-09-14T05:00:23.241571+0000","last_undegraded":"2024-09-14T05:00:23.241571+0000","last_fullsized":"2024-09-14T05:00:23.241571+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:33:18.810199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2370,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:24.271395+0000","last_change":"2024-09-14T04:44:57.329113+0000","last_active":"2024-09-14T05:00:24.271395+0000","last_peered":"2024-09-14T05:00:24.271395+0000","last_clean":"2024-09-14T05:00:24.271395+0000","last_became_active":"2024-09-14T04:44:57.328727+0000","last_became_peered":"2024-09-14T04:44:57.328727+0000","last_unstale":"2024-09-14T05:00:24.271395+0000","last_undegraded":"2024-09-14T05:00:24.271395+0000","last_fullsized":"2024-09-14T05:00:24.271395+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:34.939874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":2388,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:24.268622+0000","last_change":"2024-09-14T04:44:57.329277+0000","last_active":"2024-09-14T05:00:24.268622+0000","last_peered":"2024-09-14T05:00:24.268622+0000","last_clean":"2024-09-14T05:00:24.268622+0000","last_became_active":"2024-09-14T04:44:57.328804+0000","last_became_peered":"2024-09-14T04:44:57.328804+0000","last_unstale":"2024-09-14T05:00:24.268622+0000","last_undegraded":"2024-09-14T05:00:24.268622+0000","last_fullsized":"2024-09-14T05:00:24.268622+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:42:46.055240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2423,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:24.270714+0000","last_change":"2024-09-14T04:54:52.250695+0000","last_active":"2024-09-14T05:00:24.270714+0000","last_peered":"2024-09-14T05:00:24.270714+0000","last_clean":"2024-09-14T05:00:24.270714+0000","last_became_active":"2024-09-14T04:54:52.250301+0000","last_became_peered":"2024-09-14T04:54:52.250301+0000","last_unstale":"2024-09-14T05:00:24.270714+0000","last_undegraded":"2024-09-14T05:00:24.270714+0000","last_fullsized":"2024-09-14T05:00:24.270714+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:42.060905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2384,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:24.265912+0000","last_change":"2024-09-14T04:43:13.357092+0000","last_active":"2024-09-14T05:00:24.265912+0000","last_peered":"2024-09-14T05:00:24.265912+0000","last_clean":"2024-09-14T05:00:24.265912+0000","last_became_active":"2024-09-14T04:43:13.356694+0000","last_became_peered":"2024-09-14T04:43:13.356694+0000","last_unstale":"2024-09-14T05:00:24.265912+0000","last_undegraded":"2024-09-14T05:00:24.265912+0000","last_fullsized":"2024-09-14T05:00:24.265912+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:11.167574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1542,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.243964+0000","last_change":"2024-09-14T04:07:21.720797+0000","last_active":"2024-09-14T05:00:23.243964+0000","last_peered":"2024-09-14T05:00:23.243964+0000","last_clean":"2024-09-14T05:00:23.243964+0000","last_became_active":"2024-09-14T04:07:21.719331+0000","last_became_peered":"2024-09-14T04:07:21.719331+0000","last_unstale":"2024-09-14T05:00:23.243964+0000","last_undegraded":"2024-09-14T05:00:23.243964+0000","last_fullsized":"2024-09-14T05:00:23.243964+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:35.404023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":189,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.246813+0000","last_change":"2024-09-14T04:43:11.641514+0000","last_active":"2024-09-14T05:00:23.246813+0000","last_peered":"2024-09-14T05:00:23.246813+0000","last_clean":"2024-09-14T05:00:23.246813+0000","last_became_active":"2024-09-14T04:43:11.641151+0000","last_became_peered":"2024-09-14T04:43:11.641151+0000","last_unstale":"2024-09-14T05:00:23.246813+0000","last_undegraded":"2024-09-14T05:00:23.246813+0000","last_fullsized":"2024-09-14T05:00:23.246813+0000","mapping_epoch":1384,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:06:45.045430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2395,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:24.266622+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T05:00:24.266622+0000","last_peered":"2024-09-14T05:00:24.266622+0000","last_clean":"2024-09-14T05:00:24.266622+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T05:00:24.266622+0000","last_undegraded":"2024-09-14T05:00:24.266622+0000","last_fullsized":"2024-09-14T05:00:24.266622+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.14","version":"349'149","reported_seq":862,"reported_epoch":1557,"state":"active+clean","last_fresh":"2024-09-14T05:00:30.312925+0000","last_change":"2024-09-14T05:00:10.532616+0000","last_active":"2024-09-14T05:00:30.312925+0000","last_peered":"2024-09-14T05:00:30.312925+0000","last_clean":"2024-09-14T05:00:30.312925+0000","last_became_active":"2024-09-14T05:00:08.501982+0000","last_became_peered":"2024-09-14T05:00:08.501982+0000","last_unstale":"2024-09-14T05:00:30.312925+0000","last_undegraded":"2024-09-14T05:00:30.312925+0000","last_fullsized":"2024-09-14T05:00:30.312925+0000","mapping_epoch":1549,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1550,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:49:30.085089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:49:30.085089+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"user requested scrub scheduled @ 2024-09-14T05:00:39.302032+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":769,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.247832+0000","last_change":"2024-09-14T04:49:29.618477+0000","last_active":"2024-09-14T05:00:23.247832+0000","last_peered":"2024-09-14T05:00:23.247832+0000","last_clean":"2024-09-14T05:00:23.247832+0000","last_became_active":"2024-09-14T04:49:29.618210+0000","last_became_peered":"2024-09-14T04:49:29.618210+0000","last_unstale":"2024-09-14T05:00:23.247832+0000","last_undegraded":"2024-09-14T05:00:23.247832+0000","last_fullsized":"2024-09-14T05:00:23.247832+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:13:23.895512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1547,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.246571+0000","last_change":"2024-09-14T04:42:49.938505+0000","last_active":"2024-09-14T05:00:23.246571+0000","last_peered":"2024-09-14T05:00:23.246571+0000","last_clean":"2024-09-14T05:00:23.246571+0000","last_became_active":"2024-09-14T04:42:49.938180+0000","last_became_peered":"2024-09-14T04:42:49.938180+0000","last_unstale":"2024-09-14T05:00:23.246571+0000","last_undegraded":"2024-09-14T05:00:23.246571+0000","last_fullsized":"2024-09-14T05:00:23.246571+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:05.929479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1566,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.248189+0000","last_change":"2024-09-14T04:44:57.346582+0000","last_active":"2024-09-14T05:00:23.248189+0000","last_peered":"2024-09-14T05:00:23.248189+0000","last_clean":"2024-09-14T05:00:23.248189+0000","last_became_active":"2024-09-14T04:44:57.330446+0000","last_became_peered":"2024-09-14T04:44:57.330446+0000","last_unstale":"2024-09-14T05:00:23.248189+0000","last_undegraded":"2024-09-14T05:00:23.248189+0000","last_fullsized":"2024-09-14T05:00:23.248189+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:29.238585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":336,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:24.268714+0000","last_change":"2024-09-14T04:44:14.480667+0000","last_active":"2024-09-14T05:00:24.268714+0000","last_peered":"2024-09-14T05:00:24.268714+0000","last_clean":"2024-09-14T05:00:24.268714+0000","last_became_active":"2024-09-14T04:44:14.480365+0000","last_became_peered":"2024-09-14T04:44:14.480365+0000","last_unstale":"2024-09-14T05:00:24.268714+0000","last_undegraded":"2024-09-14T05:00:24.268714+0000","last_fullsized":"2024-09-14T05:00:24.268714+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:00.611771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":180,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.242815+0000","last_change":"2024-09-14T04:44:57.341426+0000","last_active":"2024-09-14T05:00:23.242815+0000","last_peered":"2024-09-14T05:00:23.242815+0000","last_clean":"2024-09-14T05:00:23.242815+0000","last_became_active":"2024-09-14T04:44:57.328763+0000","last_became_peered":"2024-09-14T04:44:57.328763+0000","last_unstale":"2024-09-14T05:00:23.242815+0000","last_undegraded":"2024-09-14T05:00:23.242815+0000","last_fullsized":"2024-09-14T05:00:23.242815+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:34:40.066761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":170,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.245225+0000","last_change":"2024-09-14T04:44:57.331337+0000","last_active":"2024-09-14T05:00:23.245225+0000","last_peered":"2024-09-14T05:00:23.245225+0000","last_clean":"2024-09-14T05:00:23.245225+0000","last_became_active":"2024-09-14T04:44:57.329336+0000","last_became_peered":"2024-09-14T04:44:57.329336+0000","last_unstale":"2024-09-14T05:00:23.245225+0000","last_undegraded":"2024-09-14T05:00:23.245225+0000","last_fullsized":"2024-09-14T05:00:23.245225+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:52:35.980869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"349'149","reported_seq":768,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.248848+0000","last_change":"2024-09-14T05:00:01.425849+0000","last_active":"2024-09-14T05:00:23.248848+0000","last_peered":"2024-09-14T05:00:23.248848+0000","last_clean":"2024-09-14T05:00:23.248848+0000","last_became_active":"2024-09-14T05:00:01.425392+0000","last_became_peered":"2024-09-14T05:00:01.425392+0000","last_unstale":"2024-09-14T05:00:23.248848+0000","last_undegraded":"2024-09-14T05:00:23.248848+0000","last_fullsized":"2024-09-14T05:00:23.248848+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1544,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:49:30.085089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:49:30.085089+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0071928790000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109226,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":171,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.242333+0000","last_change":"2024-09-14T04:44:57.329870+0000","last_active":"2024-09-14T05:00:23.242333+0000","last_peered":"2024-09-14T05:00:23.242333+0000","last_clean":"2024-09-14T05:00:23.242333+0000","last_became_active":"2024-09-14T04:44:57.329353+0000","last_became_peered":"2024-09-14T04:44:57.329353+0000","last_unstale":"2024-09-14T05:00:23.242333+0000","last_undegraded":"2024-09-14T05:00:23.242333+0000","last_fullsized":"2024-09-14T05:00:23.242333+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:46.432181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1563,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.244043+0000","last_change":"2024-09-14T04:54:52.242207+0000","last_active":"2024-09-14T05:00:23.244043+0000","last_peered":"2024-09-14T05:00:23.244043+0000","last_clean":"2024-09-14T05:00:23.244043+0000","last_became_active":"2024-09-14T04:54:52.241840+0000","last_became_peered":"2024-09-14T04:54:52.241840+0000","last_unstale":"2024-09-14T05:00:23.244043+0000","last_undegraded":"2024-09-14T05:00:23.244043+0000","last_fullsized":"2024-09-14T05:00:23.244043+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:37.899666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":2434,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:24.270598+0000","last_change":"2024-09-14T04:44:57.344614+0000","last_active":"2024-09-14T05:00:24.270598+0000","last_peered":"2024-09-14T05:00:24.270598+0000","last_clean":"2024-09-14T05:00:24.270598+0000","last_became_active":"2024-09-14T04:44:57.332256+0000","last_became_peered":"2024-09-14T04:44:57.332256+0000","last_unstale":"2024-09-14T05:00:24.270598+0000","last_undegraded":"2024-09-14T05:00:24.270598+0000","last_fullsized":"2024-09-14T05:00:24.270598+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:23.438417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"349'149","reported_seq":938,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.242169+0000","last_change":"2024-09-14T05:00:01.424377+0000","last_active":"2024-09-14T05:00:23.242169+0000","last_peered":"2024-09-14T05:00:23.242169+0000","last_clean":"2024-09-14T05:00:23.242169+0000","last_became_active":"2024-09-14T05:00:01.423833+0000","last_became_peered":"2024-09-14T05:00:01.423833+0000","last_unstale":"2024-09-14T05:00:23.242169+0000","last_undegraded":"2024-09-14T05:00:23.242169+0000","last_fullsized":"2024-09-14T05:00:23.242169+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1544,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.567535+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.567535+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":760,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.247825+0000","last_change":"2024-09-14T04:49:29.616831+0000","last_active":"2024-09-14T05:00:23.247825+0000","last_peered":"2024-09-14T05:00:23.247825+0000","last_clean":"2024-09-14T05:00:23.247825+0000","last_became_active":"2024-09-14T04:49:29.616546+0000","last_became_peered":"2024-09-14T04:49:29.616546+0000","last_unstale":"2024-09-14T05:00:23.247825+0000","last_undegraded":"2024-09-14T05:00:23.247825+0000","last_fullsized":"2024-09-14T05:00:23.247825+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:27.882929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":389,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:24.274008+0000","last_change":"2024-09-14T04:44:57.330746+0000","last_active":"2024-09-14T05:00:24.274008+0000","last_peered":"2024-09-14T05:00:24.274008+0000","last_clean":"2024-09-14T05:00:24.274008+0000","last_became_active":"2024-09-14T04:44:57.330327+0000","last_became_peered":"2024-09-14T04:44:57.330327+0000","last_unstale":"2024-09-14T05:00:24.274008+0000","last_undegraded":"2024-09-14T05:00:24.274008+0000","last_fullsized":"2024-09-14T05:00:24.274008+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:53.122771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":210,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.248899+0000","last_change":"2024-09-14T04:43:13.355296+0000","last_active":"2024-09-14T05:00:23.248899+0000","last_peered":"2024-09-14T05:00:23.248899+0000","last_clean":"2024-09-14T05:00:23.248899+0000","last_became_active":"2024-09-14T04:43:13.354950+0000","last_became_peered":"2024-09-14T04:43:13.354950+0000","last_unstale":"2024-09-14T05:00:23.248899+0000","last_undegraded":"2024-09-14T05:00:23.248899+0000","last_fullsized":"2024-09-14T05:00:23.248899+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:20:08.968818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"349'149","reported_seq":876,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.241514+0000","last_change":"2024-09-14T05:00:01.434438+0000","last_active":"2024-09-14T05:00:23.241514+0000","last_peered":"2024-09-14T05:00:23.241514+0000","last_clean":"2024-09-14T05:00:23.241514+0000","last_became_active":"2024-09-14T05:00:01.433252+0000","last_became_peered":"2024-09-14T05:00:01.433252+0000","last_unstale":"2024-09-14T05:00:23.241514+0000","last_undegraded":"2024-09-14T05:00:23.241514+0000","last_fullsized":"2024-09-14T05:00:23.241514+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1544,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.471248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.471248+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.021758344999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2390,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:24.266090+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T05:00:24.266090+0000","last_peered":"2024-09-14T05:00:24.266090+0000","last_clean":"2024-09-14T05:00:24.266090+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T05:00:24.266090+0000","last_undegraded":"2024-09-14T05:00:24.266090+0000","last_fullsized":"2024-09-14T05:00:24.266090+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1577,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.248490+0000","last_change":"2024-09-14T04:44:57.331436+0000","last_active":"2024-09-14T05:00:23.248490+0000","last_peered":"2024-09-14T05:00:23.248490+0000","last_clean":"2024-09-14T05:00:23.248490+0000","last_became_active":"2024-09-14T04:44:57.330957+0000","last_became_peered":"2024-09-14T04:44:57.330957+0000","last_unstale":"2024-09-14T05:00:23.248490+0000","last_undegraded":"2024-09-14T05:00:23.248490+0000","last_fullsized":"2024-09-14T05:00:23.248490+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:09.516688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":941,"reported_epoch":1557,"state":"active+clean","last_fresh":"2024-09-14T05:00:30.316594+0000","last_change":"2024-09-14T04:49:49.460401+0000","last_active":"2024-09-14T05:00:30.316594+0000","last_peered":"2024-09-14T05:00:30.316594+0000","last_clean":"2024-09-14T05:00:30.316594+0000","last_became_active":"2024-09-14T04:49:49.458530+0000","last_became_peered":"2024-09-14T04:49:49.458530+0000","last_unstale":"2024-09-14T05:00:30.316594+0000","last_undegraded":"2024-09-14T05:00:30.316594+0000","last_fullsized":"2024-09-14T05:00:30.316594+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:11:29.143239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"349'149","reported_seq":910,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.243906+0000","last_change":"2024-09-14T05:00:01.425346+0000","last_active":"2024-09-14T05:00:23.243906+0000","last_peered":"2024-09-14T05:00:23.243906+0000","last_clean":"2024-09-14T05:00:23.243906+0000","last_became_active":"2024-09-14T05:00:01.424753+0000","last_became_peered":"2024-09-14T05:00:01.424753+0000","last_unstale":"2024-09-14T05:00:23.243906+0000","last_undegraded":"2024-09-14T05:00:23.243906+0000","last_fullsized":"2024-09-14T05:00:23.243906+0000","mapping_epoch":1492,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1544,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:38.479887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:38.479887+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0062728319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":344,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:24.268624+0000","last_change":"2024-09-14T04:44:57.329983+0000","last_active":"2024-09-14T05:00:24.268624+0000","last_peered":"2024-09-14T05:00:24.268624+0000","last_clean":"2024-09-14T05:00:24.268624+0000","last_became_active":"2024-09-14T04:44:57.329597+0000","last_became_peered":"2024-09-14T04:44:57.329597+0000","last_unstale":"2024-09-14T05:00:24.268624+0000","last_undegraded":"2024-09-14T05:00:24.268624+0000","last_fullsized":"2024-09-14T05:00:24.268624+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:45.718039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":883,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:24.267214+0000","last_change":"2024-09-14T04:49:49.443631+0000","last_active":"2024-09-14T05:00:24.267214+0000","last_peered":"2024-09-14T05:00:24.267214+0000","last_clean":"2024-09-14T05:00:24.267214+0000","last_became_active":"2024-09-14T04:49:49.443313+0000","last_became_peered":"2024-09-14T04:49:49.443313+0000","last_unstale":"2024-09-14T05:00:24.267214+0000","last_undegraded":"2024-09-14T05:00:24.267214+0000","last_fullsized":"2024-09-14T05:00:24.267214+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:06:54.562292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1550,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.250788+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T05:00:23.250788+0000","last_peered":"2024-09-14T05:00:23.250788+0000","last_clean":"2024-09-14T05:00:23.250788+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T05:00:23.250788+0000","last_undegraded":"2024-09-14T05:00:23.250788+0000","last_fullsized":"2024-09-14T05:00:23.250788+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"349'149","reported_seq":869,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.250019+0000","last_change":"2024-09-14T05:00:01.421946+0000","last_active":"2024-09-14T05:00:23.250019+0000","last_peered":"2024-09-14T05:00:23.250019+0000","last_clean":"2024-09-14T05:00:23.250019+0000","last_became_active":"2024-09-14T05:00:01.421507+0000","last_became_peered":"2024-09-14T05:00:01.421507+0000","last_unstale":"2024-09-14T05:00:23.250019+0000","last_undegraded":"2024-09-14T05:00:23.250019+0000","last_fullsized":"2024-09-14T05:00:23.250019+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1544,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.012879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.012879+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0051721620000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":977,"reported_epoch":1557,"state":"active+clean","last_fresh":"2024-09-14T05:00:30.316805+0000","last_change":"2024-09-14T04:49:49.459590+0000","last_active":"2024-09-14T05:00:30.316805+0000","last_peered":"2024-09-14T05:00:30.316805+0000","last_clean":"2024-09-14T05:00:30.316805+0000","last_became_active":"2024-09-14T04:49:49.456263+0000","last_became_peered":"2024-09-14T04:49:49.456263+0000","last_unstale":"2024-09-14T05:00:30.316805+0000","last_undegraded":"2024-09-14T05:00:30.316805+0000","last_fullsized":"2024-09-14T05:00:30.316805+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:20:44.666805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":171,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.245152+0000","last_change":"2024-09-14T04:44:57.329652+0000","last_active":"2024-09-14T05:00:23.245152+0000","last_peered":"2024-09-14T05:00:23.245152+0000","last_clean":"2024-09-14T05:00:23.245152+0000","last_became_active":"2024-09-14T04:44:57.328888+0000","last_became_peered":"2024-09-14T04:44:57.328888+0000","last_unstale":"2024-09-14T05:00:23.245152+0000","last_undegraded":"2024-09-14T05:00:23.245152+0000","last_fullsized":"2024-09-14T05:00:23.245152+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:10.170068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1548,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.246604+0000","last_change":"2024-09-14T04:06:46.875340+0000","last_active":"2024-09-14T05:00:23.246604+0000","last_peered":"2024-09-14T05:00:23.246604+0000","last_clean":"2024-09-14T05:00:23.246604+0000","last_became_active":"2024-09-14T04:06:46.874552+0000","last_became_peered":"2024-09-14T04:06:46.874552+0000","last_unstale":"2024-09-14T05:00:23.246604+0000","last_undegraded":"2024-09-14T05:00:23.246604+0000","last_fullsized":"2024-09-14T05:00:23.246604+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:52.257357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"349'149","reported_seq":783,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.248321+0000","last_change":"2024-09-14T05:00:02.067572+0000","last_active":"2024-09-14T05:00:23.248321+0000","last_peered":"2024-09-14T05:00:23.248321+0000","last_clean":"2024-09-14T05:00:23.248321+0000","last_became_active":"2024-09-14T05:00:01.416741+0000","last_became_peered":"2024-09-14T05:00:01.416741+0000","last_unstale":"2024-09-14T05:00:23.248321+0000","last_undegraded":"2024-09-14T05:00:23.248321+0000","last_fullsized":"2024-09-14T05:00:23.248321+0000","mapping_epoch":1506,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1544,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:02.067463+0000","last_deep_scrub":"349'149","last_deep_scrub_stamp":"2024-09-14T05:00:02.067463+0000","last_clean_scrub_stamp":"2024-09-14T05:00:02.067463+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:12:45.764862+0000","scrub_duration":0.0033042340000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":337,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:24.272776+0000","last_change":"2024-09-14T04:44:14.489235+0000","last_active":"2024-09-14T05:00:24.272776+0000","last_peered":"2024-09-14T05:00:24.272776+0000","last_clean":"2024-09-14T05:00:24.272776+0000","last_became_active":"2024-09-14T04:44:14.488950+0000","last_became_peered":"2024-09-14T04:44:14.488950+0000","last_unstale":"2024-09-14T05:00:24.272776+0000","last_undegraded":"2024-09-14T05:00:24.272776+0000","last_fullsized":"2024-09-14T05:00:24.272776+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:54.441144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":895,"reported_epoch":1557,"state":"active+clean","last_fresh":"2024-09-14T05:00:30.310167+0000","last_change":"2024-09-14T04:49:49.456317+0000","last_active":"2024-09-14T05:00:30.310167+0000","last_peered":"2024-09-14T05:00:30.310167+0000","last_clean":"2024-09-14T05:00:30.310167+0000","last_became_active":"2024-09-14T04:49:49.456029+0000","last_became_peered":"2024-09-14T04:49:49.456029+0000","last_unstale":"2024-09-14T05:00:30.310167+0000","last_undegraded":"2024-09-14T05:00:30.310167+0000","last_fullsized":"2024-09-14T05:00:30.310167+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:46:05.319676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2412,"reported_epoch":1557,"state":"active+clean","last_fresh":"2024-09-14T05:00:30.310139+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T05:00:30.310139+0000","last_peered":"2024-09-14T05:00:30.310139+0000","last_clean":"2024-09-14T05:00:30.310139+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T05:00:30.310139+0000","last_undegraded":"2024-09-14T05:00:30.310139+0000","last_fullsized":"2024-09-14T05:00:30.310139+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"349'149","reported_seq":763,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.245113+0000","last_change":"2024-09-14T05:00:01.434014+0000","last_active":"2024-09-14T05:00:23.245113+0000","last_peered":"2024-09-14T05:00:23.245113+0000","last_clean":"2024-09-14T05:00:23.245113+0000","last_became_active":"2024-09-14T05:00:01.433375+0000","last_became_peered":"2024-09-14T05:00:01.433375+0000","last_unstale":"2024-09-14T05:00:23.245113+0000","last_undegraded":"2024-09-14T05:00:23.245113+0000","last_fullsized":"2024-09-14T05:00:23.245113+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1544,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:50:22.002568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:50:22.002568+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.010888326,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2411,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:24.270801+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T05:00:24.270801+0000","last_peered":"2024-09-14T05:00:24.270801+0000","last_clean":"2024-09-14T05:00:24.270801+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T05:00:24.270801+0000","last_undegraded":"2024-09-14T05:00:24.270801+0000","last_fullsized":"2024-09-14T05:00:24.270801+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2432,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:24.266305+0000","last_change":"2024-09-14T04:54:19.447103+0000","last_active":"2024-09-14T05:00:24.266305+0000","last_peered":"2024-09-14T05:00:24.266305+0000","last_clean":"2024-09-14T05:00:24.266305+0000","last_became_active":"2024-09-14T04:54:19.446748+0000","last_became_peered":"2024-09-14T04:54:19.446748+0000","last_unstale":"2024-09-14T05:00:24.266305+0000","last_undegraded":"2024-09-14T05:00:24.266305+0000","last_fullsized":"2024-09-14T05:00:24.266305+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:00:55.369263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1579,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.248003+0000","last_change":"2024-09-14T04:42:49.939790+0000","last_active":"2024-09-14T05:00:23.248003+0000","last_peered":"2024-09-14T05:00:23.248003+0000","last_clean":"2024-09-14T05:00:23.248003+0000","last_became_active":"2024-09-14T04:42:49.939591+0000","last_became_peered":"2024-09-14T04:42:49.939591+0000","last_unstale":"2024-09-14T05:00:23.248003+0000","last_undegraded":"2024-09-14T05:00:23.248003+0000","last_fullsized":"2024-09-14T05:00:23.248003+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:23:16.790593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1542,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.248792+0000","last_change":"2024-09-14T04:44:57.326008+0000","last_active":"2024-09-14T05:00:23.248792+0000","last_peered":"2024-09-14T05:00:23.248792+0000","last_clean":"2024-09-14T05:00:23.248792+0000","last_became_active":"2024-09-14T04:44:57.325617+0000","last_became_peered":"2024-09-14T04:44:57.325617+0000","last_unstale":"2024-09-14T05:00:23.248792+0000","last_undegraded":"2024-09-14T05:00:23.248792+0000","last_fullsized":"2024-09-14T05:00:23.248792+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:15.637508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":831,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.246652+0000","last_change":"2024-09-14T04:43:13.354846+0000","last_active":"2024-09-14T05:00:23.246652+0000","last_peered":"2024-09-14T05:00:23.246652+0000","last_clean":"2024-09-14T05:00:23.246652+0000","last_became_active":"2024-09-14T04:43:13.354515+0000","last_became_peered":"2024-09-14T04:43:13.354515+0000","last_unstale":"2024-09-14T05:00:23.246652+0000","last_undegraded":"2024-09-14T05:00:23.246652+0000","last_fullsized":"2024-09-14T05:00:23.246652+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:17.889341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1547,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.246283+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T05:00:23.246283+0000","last_peered":"2024-09-14T05:00:23.246283+0000","last_clean":"2024-09-14T05:00:23.246283+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T05:00:23.246283+0000","last_undegraded":"2024-09-14T05:00:23.246283+0000","last_fullsized":"2024-09-14T05:00:23.246283+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":1894,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.247757+0000","last_change":"2024-09-14T05:00:01.433623+0000","last_active":"2024-09-14T05:00:23.247757+0000","last_peered":"2024-09-14T05:00:23.247757+0000","last_clean":"2024-09-14T05:00:23.247757+0000","last_became_active":"2024-09-14T05:00:01.433169+0000","last_became_peered":"2024-09-14T05:00:01.433169+0000","last_unstale":"2024-09-14T05:00:23.247757+0000","last_undegraded":"2024-09-14T05:00:23.247757+0000","last_fullsized":"2024-09-14T05:00:23.247757+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1544,"parent":"0.0","parent_split_bits":0,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:30.640520+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:30.640520+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0028653139999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"349'149","reported_seq":859,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.246524+0000","last_change":"2024-09-14T04:41:39.978080+0000","last_active":"2024-09-14T05:00:23.246524+0000","last_peered":"2024-09-14T05:00:23.246524+0000","last_clean":"2024-09-14T05:00:23.246524+0000","last_became_active":"2024-09-14T04:41:36.929279+0000","last_became_peered":"2024-09-14T04:41:36.929279+0000","last_unstale":"2024-09-14T05:00:23.246524+0000","last_undegraded":"2024-09-14T05:00:23.246524+0000","last_fullsized":"2024-09-14T05:00:23.246524+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.977999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.977999+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:44:06.383827+0000","scrub_duration":0.00192122,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":273,"reported_epoch":1557,"state":"active+clean","last_fresh":"2024-09-14T05:00:30.310360+0000","last_change":"2024-09-14T04:48:11.040133+0000","last_active":"2024-09-14T05:00:30.310360+0000","last_peered":"2024-09-14T05:00:30.310360+0000","last_clean":"2024-09-14T05:00:30.310360+0000","last_became_active":"2024-09-14T04:48:11.039787+0000","last_became_peered":"2024-09-14T04:48:11.039787+0000","last_unstale":"2024-09-14T05:00:30.310360+0000","last_undegraded":"2024-09-14T05:00:30.310360+0000","last_fullsized":"2024-09-14T05:00:30.310360+0000","mapping_epoch":1428,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:53.520672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2258,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:24.268809+0000","last_change":"2024-09-14T04:49:29.616443+0000","last_active":"2024-09-14T05:00:24.268809+0000","last_peered":"2024-09-14T05:00:24.268809+0000","last_clean":"2024-09-14T05:00:24.268809+0000","last_became_active":"2024-09-14T04:49:29.616066+0000","last_became_peered":"2024-09-14T04:49:29.616066+0000","last_unstale":"2024-09-14T05:00:24.268809+0000","last_undegraded":"2024-09-14T05:00:24.268809+0000","last_fullsized":"2024-09-14T05:00:24.268809+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:09:15.897957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.b","version":"349'149","reported_seq":885,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:24.265803+0000","last_change":"2024-09-14T05:00:09.204289+0000","last_active":"2024-09-14T05:00:24.265803+0000","last_peered":"2024-09-14T05:00:24.265803+0000","last_clean":"2024-09-14T05:00:24.265803+0000","last_became_active":"2024-09-14T05:00:08.490295+0000","last_became_peered":"2024-09-14T05:00:08.490295+0000","last_unstale":"2024-09-14T05:00:24.265803+0000","last_undegraded":"2024-09-14T05:00:24.265803+0000","last_fullsized":"2024-09-14T05:00:24.265803+0000","mapping_epoch":1549,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1550,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:50:22.002568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:50:22.002568+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"user requested scrub scheduled @ 2024-09-14T05:00:39.007448+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":170,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.242507+0000","last_change":"2024-09-14T04:44:57.326724+0000","last_active":"2024-09-14T05:00:23.242507+0000","last_peered":"2024-09-14T05:00:23.242507+0000","last_clean":"2024-09-14T05:00:23.242507+0000","last_became_active":"2024-09-14T04:44:57.326191+0000","last_became_peered":"2024-09-14T04:44:57.326191+0000","last_unstale":"2024-09-14T05:00:23.242507+0000","last_undegraded":"2024-09-14T05:00:23.242507+0000","last_fullsized":"2024-09-14T05:00:23.242507+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:46.060269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":171,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.242206+0000","last_change":"2024-09-14T04:44:57.328528+0000","last_active":"2024-09-14T05:00:23.242206+0000","last_peered":"2024-09-14T05:00:23.242206+0000","last_clean":"2024-09-14T05:00:23.242206+0000","last_became_active":"2024-09-14T04:44:57.327739+0000","last_became_peered":"2024-09-14T04:44:57.327739+0000","last_unstale":"2024-09-14T05:00:23.242206+0000","last_undegraded":"2024-09-14T05:00:23.242206+0000","last_fullsized":"2024-09-14T05:00:23.242206+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:04:58.294690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"349'149","reported_seq":871,"reported_epoch":1557,"state":"active+clean","last_fresh":"2024-09-14T05:00:30.310061+0000","last_change":"2024-09-14T05:00:24.275751+0000","last_active":"2024-09-14T05:00:30.310061+0000","last_peered":"2024-09-14T05:00:30.310061+0000","last_clean":"2024-09-14T05:00:30.310061+0000","last_became_active":"2024-09-14T05:00:24.275201+0000","last_became_peered":"2024-09-14T05:00:24.275201+0000","last_unstale":"2024-09-14T05:00:30.310061+0000","last_undegraded":"2024-09-14T05:00:30.310061+0000","last_fullsized":"2024-09-14T05:00:30.310061+0000","mapping_epoch":1555,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1556,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:56.738636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:56.738636+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:41:11.155132+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1576128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":170,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.242040+0000","last_change":"2024-09-14T04:44:57.323436+0000","last_active":"2024-09-14T05:00:23.242040+0000","last_peered":"2024-09-14T05:00:23.242040+0000","last_clean":"2024-09-14T05:00:23.242040+0000","last_became_active":"2024-09-14T04:44:57.323022+0000","last_became_peered":"2024-09-14T04:44:57.323022+0000","last_unstale":"2024-09-14T05:00:23.242040+0000","last_undegraded":"2024-09-14T05:00:23.242040+0000","last_fullsized":"2024-09-14T05:00:23.242040+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:22.775261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1577,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:24.265986+0000","last_change":"2024-09-14T04:44:57.322357+0000","last_active":"2024-09-14T05:00:24.265986+0000","last_peered":"2024-09-14T05:00:24.265986+0000","last_clean":"2024-09-14T05:00:24.265986+0000","last_became_active":"2024-09-14T04:44:57.321990+0000","last_became_peered":"2024-09-14T04:44:57.321990+0000","last_unstale":"2024-09-14T05:00:24.265986+0000","last_undegraded":"2024-09-14T05:00:24.265986+0000","last_fullsized":"2024-09-14T05:00:24.265986+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:09.658008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"349'149","reported_seq":1901,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.247756+0000","last_change":"2024-09-14T04:41:53.858076+0000","last_active":"2024-09-14T05:00:23.247756+0000","last_peered":"2024-09-14T05:00:23.247756+0000","last_clean":"2024-09-14T05:00:23.247756+0000","last_became_active":"2024-09-14T04:41:36.931585+0000","last_became_peered":"2024-09-14T04:41:36.931585+0000","last_unstale":"2024-09-14T05:00:23.247756+0000","last_undegraded":"2024-09-14T05:00:23.247756+0000","last_fullsized":"2024-09-14T05:00:23.247756+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:53.857924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:53.857924+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:10:20.658986+0000","scrub_duration":0.0022355919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":170,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.242731+0000","last_change":"2024-09-14T04:44:57.329198+0000","last_active":"2024-09-14T05:00:23.242731+0000","last_peered":"2024-09-14T05:00:23.242731+0000","last_clean":"2024-09-14T05:00:23.242731+0000","last_became_active":"2024-09-14T04:44:57.327754+0000","last_became_peered":"2024-09-14T04:44:57.327754+0000","last_unstale":"2024-09-14T05:00:23.242731+0000","last_undegraded":"2024-09-14T05:00:23.242731+0000","last_fullsized":"2024-09-14T05:00:23.242731+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:10:27.944476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":394,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:24.266191+0000","last_change":"2024-09-14T04:44:57.348687+0000","last_active":"2024-09-14T05:00:24.266191+0000","last_peered":"2024-09-14T05:00:24.266191+0000","last_clean":"2024-09-14T05:00:24.266191+0000","last_became_active":"2024-09-14T04:44:57.332845+0000","last_became_peered":"2024-09-14T04:44:57.332845+0000","last_unstale":"2024-09-14T05:00:24.266191+0000","last_undegraded":"2024-09-14T05:00:24.266191+0000","last_fullsized":"2024-09-14T05:00:24.266191+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:12:18.762737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"349'149","reported_seq":636,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.242262+0000","last_change":"2024-09-14T05:00:08.498425+0000","last_active":"2024-09-14T05:00:23.242262+0000","last_peered":"2024-09-14T05:00:23.242262+0000","last_clean":"2024-09-14T05:00:23.242262+0000","last_became_active":"2024-09-14T05:00:08.498061+0000","last_became_peered":"2024-09-14T05:00:08.498061+0000","last_unstale":"2024-09-14T05:00:23.242262+0000","last_undegraded":"2024-09-14T05:00:23.242262+0000","last_fullsized":"2024-09-14T05:00:23.242262+0000","mapping_epoch":1549,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1550,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.471248+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.471248+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":17,"num_read_kb":38,"num_write":10,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2417,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:24.265956+0000","last_change":"2024-09-14T04:44:57.333468+0000","last_active":"2024-09-14T05:00:24.265956+0000","last_peered":"2024-09-14T05:00:24.265956+0000","last_clean":"2024-09-14T05:00:24.265956+0000","last_became_active":"2024-09-14T04:44:57.332626+0000","last_became_peered":"2024-09-14T04:44:57.332626+0000","last_unstale":"2024-09-14T05:00:24.265956+0000","last_undegraded":"2024-09-14T05:00:24.265956+0000","last_fullsized":"2024-09-14T05:00:24.265956+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:57:26.366438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1532,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.249058+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T05:00:23.249058+0000","last_peered":"2024-09-14T05:00:23.249058+0000","last_clean":"2024-09-14T05:00:23.249058+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T05:00:23.249058+0000","last_undegraded":"2024-09-14T05:00:23.249058+0000","last_fullsized":"2024-09-14T05:00:23.249058+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"349'149","reported_seq":927,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.242577+0000","last_change":"2024-09-14T05:00:08.483353+0000","last_active":"2024-09-14T05:00:23.242577+0000","last_peered":"2024-09-14T05:00:23.242577+0000","last_clean":"2024-09-14T05:00:23.242577+0000","last_became_active":"2024-09-14T05:00:08.482855+0000","last_became_peered":"2024-09-14T05:00:08.482855+0000","last_unstale":"2024-09-14T05:00:23.242577+0000","last_undegraded":"2024-09-14T05:00:23.242577+0000","last_fullsized":"2024-09-14T05:00:23.242577+0000","mapping_epoch":1549,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1550,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:38.479887+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:38.479887+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":0.0062728319999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1532,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:24.265786+0000","last_change":"2024-09-14T04:06:46.869868+0000","last_active":"2024-09-14T05:00:24.265786+0000","last_peered":"2024-09-14T05:00:24.265786+0000","last_clean":"2024-09-14T05:00:24.265786+0000","last_became_active":"2024-09-14T04:06:46.869126+0000","last_became_peered":"2024-09-14T04:06:46.869126+0000","last_unstale":"2024-09-14T05:00:24.265786+0000","last_undegraded":"2024-09-14T05:00:24.265786+0000","last_fullsized":"2024-09-14T05:00:24.265786+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:41.150138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1515,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.248335+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T05:00:23.248335+0000","last_peered":"2024-09-14T05:00:23.248335+0000","last_clean":"2024-09-14T05:00:23.248335+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T05:00:23.248335+0000","last_undegraded":"2024-09-14T05:00:23.248335+0000","last_fullsized":"2024-09-14T05:00:23.248335+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"349'149","reported_seq":859,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:24.267121+0000","last_change":"2024-09-14T05:00:16.533576+0000","last_active":"2024-09-14T05:00:24.267121+0000","last_peered":"2024-09-14T05:00:24.267121+0000","last_clean":"2024-09-14T05:00:24.267121+0000","last_became_active":"2024-09-14T05:00:08.504510+0000","last_became_peered":"2024-09-14T05:00:08.504510+0000","last_unstale":"2024-09-14T05:00:24.267121+0000","last_undegraded":"2024-09-14T05:00:24.267121+0000","last_fullsized":"2024-09-14T05:00:24.267121+0000","mapping_epoch":1549,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1550,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:49:30.085089+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:49:30.085089+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"user requested scrub scheduled @ 2024-09-14T05:00:41.036253+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109227,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":171,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.242457+0000","last_change":"2024-09-14T04:44:57.332112+0000","last_active":"2024-09-14T05:00:23.242457+0000","last_peered":"2024-09-14T05:00:23.242457+0000","last_clean":"2024-09-14T05:00:23.242457+0000","last_became_active":"2024-09-14T04:44:57.331575+0000","last_became_peered":"2024-09-14T04:44:57.331575+0000","last_unstale":"2024-09-14T05:00:23.242457+0000","last_undegraded":"2024-09-14T05:00:23.242457+0000","last_fullsized":"2024-09-14T05:00:23.242457+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:50:00.391283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1563,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.248759+0000","last_change":"2024-09-14T04:44:57.351460+0000","last_active":"2024-09-14T05:00:23.248759+0000","last_peered":"2024-09-14T05:00:23.248759+0000","last_clean":"2024-09-14T05:00:23.248759+0000","last_became_active":"2024-09-14T04:44:57.334662+0000","last_became_peered":"2024-09-14T04:44:57.334662+0000","last_unstale":"2024-09-14T05:00:23.248759+0000","last_undegraded":"2024-09-14T05:00:23.248759+0000","last_fullsized":"2024-09-14T05:00:23.248759+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:02.399062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.d","version":"349'149","reported_seq":787,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.248054+0000","last_change":"2024-09-14T05:00:08.484332+0000","last_active":"2024-09-14T05:00:23.248054+0000","last_peered":"2024-09-14T05:00:23.248054+0000","last_clean":"2024-09-14T05:00:23.248054+0000","last_became_active":"2024-09-14T05:00:08.484099+0000","last_became_peered":"2024-09-14T05:00:08.484099+0000","last_unstale":"2024-09-14T05:00:23.248054+0000","last_undegraded":"2024-09-14T05:00:23.248054+0000","last_fullsized":"2024-09-14T05:00:23.248054+0000","mapping_epoch":1549,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1550,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:37.567535+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:37.567535+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1518,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.246922+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T05:00:23.246922+0000","last_peered":"2024-09-14T05:00:23.246922+0000","last_clean":"2024-09-14T05:00:23.246922+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T05:00:23.246922+0000","last_undegraded":"2024-09-14T05:00:23.246922+0000","last_fullsized":"2024-09-14T05:00:23.246922+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1575,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.248817+0000","last_change":"2024-09-14T04:44:57.349690+0000","last_active":"2024-09-14T05:00:23.248817+0000","last_peered":"2024-09-14T05:00:23.248817+0000","last_clean":"2024-09-14T05:00:23.248817+0000","last_became_active":"2024-09-14T04:44:57.333579+0000","last_became_peered":"2024-09-14T04:44:57.333579+0000","last_unstale":"2024-09-14T05:00:23.248817+0000","last_undegraded":"2024-09-14T05:00:23.248817+0000","last_fullsized":"2024-09-14T05:00:23.248817+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:17:55.265141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"349'149","reported_seq":725,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.246298+0000","last_change":"2024-09-14T05:00:08.490957+0000","last_active":"2024-09-14T05:00:23.246298+0000","last_peered":"2024-09-14T05:00:23.246298+0000","last_clean":"2024-09-14T05:00:23.246298+0000","last_became_active":"2024-09-14T05:00:08.490541+0000","last_became_peered":"2024-09-14T05:00:08.490541+0000","last_unstale":"2024-09-14T05:00:23.246298+0000","last_undegraded":"2024-09-14T05:00:23.246298+0000","last_fullsized":"2024-09-14T05:00:23.246298+0000","mapping_epoch":1549,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1550,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:54:03.996025+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:54:03.996025+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":54,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1671,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.242128+0000","last_change":"2024-09-14T05:00:21.251775+0000","last_active":"2024-09-14T05:00:23.242128+0000","last_peered":"2024-09-14T05:00:23.242128+0000","last_clean":"2024-09-14T05:00:23.242128+0000","last_became_active":"2024-09-14T05:00:21.228875+0000","last_became_peered":"2024-09-14T05:00:21.228875+0000","last_unstale":"2024-09-14T05:00:23.242128+0000","last_undegraded":"2024-09-14T05:00:23.242128+0000","last_fullsized":"2024-09-14T05:00:23.242128+0000","mapping_epoch":1552,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:15:41.207466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":65,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.248626+0000","last_change":"2024-09-14T04:54:52.248439+0000","last_active":"2024-09-14T05:00:23.248626+0000","last_peered":"2024-09-14T05:00:23.248626+0000","last_clean":"2024-09-14T05:00:23.248626+0000","last_became_active":"2024-09-14T04:54:52.248235+0000","last_became_peered":"2024-09-14T04:54:52.248235+0000","last_unstale":"2024-09-14T05:00:23.248626+0000","last_undegraded":"2024-09-14T05:00:23.248626+0000","last_fullsized":"2024-09-14T05:00:23.248626+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:22:09.548139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"349'149","reported_seq":789,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.244994+0000","last_change":"2024-09-14T05:00:08.498402+0000","last_active":"2024-09-14T05:00:23.244994+0000","last_peered":"2024-09-14T05:00:23.244994+0000","last_clean":"2024-09-14T05:00:23.244994+0000","last_became_active":"2024-09-14T05:00:08.498035+0000","last_became_peered":"2024-09-14T05:00:08.498035+0000","last_unstale":"2024-09-14T05:00:23.244994+0000","last_undegraded":"2024-09-14T05:00:23.244994+0000","last_fullsized":"2024-09-14T05:00:23.244994+0000","mapping_epoch":1549,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1550,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:50:22.002568+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:50:22.002568+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"queued for scrub","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1537,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.247966+0000","last_change":"2024-09-14T04:44:57.332511+0000","last_active":"2024-09-14T05:00:23.247966+0000","last_peered":"2024-09-14T05:00:23.247966+0000","last_clean":"2024-09-14T05:00:23.247966+0000","last_became_active":"2024-09-14T04:44:57.332166+0000","last_became_peered":"2024-09-14T04:44:57.332166+0000","last_unstale":"2024-09-14T05:00:23.247966+0000","last_undegraded":"2024-09-14T05:00:23.247966+0000","last_fullsized":"2024-09-14T05:00:23.247966+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:53.984639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2410,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:24.270505+0000","last_change":"2024-09-14T04:54:52.250655+0000","last_active":"2024-09-14T05:00:24.270505+0000","last_peered":"2024-09-14T05:00:24.270505+0000","last_clean":"2024-09-14T05:00:24.270505+0000","last_became_active":"2024-09-14T04:54:52.250317+0000","last_became_peered":"2024-09-14T04:54:52.250317+0000","last_unstale":"2024-09-14T05:00:24.270505+0000","last_undegraded":"2024-09-14T05:00:24.270505+0000","last_fullsized":"2024-09-14T05:00:24.270505+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:12.985406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.10","version":"349'149","reported_seq":636,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.246670+0000","last_change":"2024-09-14T05:00:08.483338+0000","last_active":"2024-09-14T05:00:23.246670+0000","last_peered":"2024-09-14T05:00:23.246670+0000","last_clean":"2024-09-14T05:00:23.246670+0000","last_became_active":"2024-09-14T05:00:08.482730+0000","last_became_peered":"2024-09-14T05:00:08.482730+0000","last_unstale":"2024-09-14T05:00:23.246670+0000","last_undegraded":"2024-09-14T05:00:23.246670+0000","last_fullsized":"2024-09-14T05:00:23.246670+0000","mapping_epoch":1549,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1550,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:30.640520+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:30.640520+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":17,"num_read_kb":37,"num_write":10,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2396,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:24.266165+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T05:00:24.266165+0000","last_peered":"2024-09-14T05:00:24.266165+0000","last_clean":"2024-09-14T05:00:24.266165+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T05:00:24.266165+0000","last_undegraded":"2024-09-14T05:00:24.266165+0000","last_fullsized":"2024-09-14T05:00:24.266165+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1529,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.250724+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T05:00:23.250724+0000","last_peered":"2024-09-14T05:00:23.250724+0000","last_clean":"2024-09-14T05:00:23.250724+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T05:00:23.250724+0000","last_undegraded":"2024-09-14T05:00:23.250724+0000","last_fullsized":"2024-09-14T05:00:23.250724+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.11","version":"349'149","reported_seq":660,"reported_epoch":1556,"state":"active+clean","last_fresh":"2024-09-14T05:00:24.266248+0000","last_change":"2024-09-14T05:00:16.534482+0000","last_active":"2024-09-14T05:00:24.266248+0000","last_peered":"2024-09-14T05:00:24.266248+0000","last_clean":"2024-09-14T05:00:24.266248+0000","last_became_active":"2024-09-14T05:00:08.504082+0000","last_became_peered":"2024-09-14T05:00:08.504082+0000","last_unstale":"2024-09-14T05:00:24.266248+0000","last_undegraded":"2024-09-14T05:00:24.266248+0000","last_fullsized":"2024-09-14T05:00:24.266248+0000","mapping_epoch":1549,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1550,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.012879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.012879+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"user requested scrub scheduled @ 2024-09-14T05:00:39.988929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":18,"num_read_kb":38,"num_write":10,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1566,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.247887+0000","last_change":"2024-09-14T04:44:57.339993+0000","last_active":"2024-09-14T05:00:23.247887+0000","last_peered":"2024-09-14T05:00:23.247887+0000","last_clean":"2024-09-14T05:00:23.247887+0000","last_became_active":"2024-09-14T04:44:57.330472+0000","last_became_peered":"2024-09-14T04:44:57.330472+0000","last_unstale":"2024-09-14T05:00:23.247887+0000","last_undegraded":"2024-09-14T05:00:23.247887+0000","last_fullsized":"2024-09-14T05:00:23.247887+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:19:27.046988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2114,"reported_epoch":1555,"state":"active+clean","last_fresh":"2024-09-14T05:00:23.248067+0000","last_change":"2024-09-14T04:49:49.449687+0000","last_active":"2024-09-14T05:00:23.248067+0000","last_peered":"2024-09-14T05:00:23.248067+0000","last_clean":"2024-09-14T05:00:23.248067+0000","last_became_active":"2024-09-14T04:49:49.449144+0000","last_became_peered":"2024-09-14T04:49:49.449144+0000","last_unstale":"2024-09-14T05:00:23.248067+0000","last_undegraded":"2024-09-14T05:00:23.248067+0000","last_fullsized":"2024-09-14T05:00:23.248067+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:59:06.779730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":57,"num_pg":16,"stat_sum":{"num_bytes":1155792920,"num_objects":17637,"num_object_clones":0,"num_object_copies":52911,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17637,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":229863,"num_write_kb":7919937,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3576049664,"data_stored":3576037448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":129114,"ondisk_log_size":129114,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":670873,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":52,"num_read_kb":113,"num_write":30,"num_write_kb":330,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":1817643,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3129,"ondisk_log_size":3129,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":5070,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4250,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223202114,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1238132,"kb_used_data":408876,"kb_used_omap":1,"kb_used_meta":829246,"kb_avail":92507020,"statfs":{"total":95995035648,"available":94727188480,"internally_reserved":0,"allocated":418689024,"data_stored":413135086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":849148363},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463750,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1521268,"kb_used_data":700204,"kb_used_omap":1,"kb_used_meta":821054,"kb_avail":92223884,"statfs":{"total":95995035648,"available":94437257216,"internally_reserved":0,"allocated":717008896,"data_stored":711455399,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":840759751},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725387,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1315300,"kb_used_data":463900,"kb_used_omap":1,"kb_used_meta":851390,"kb_avail":92429852,"statfs":{"total":95995035648,"available":94648168448,"internally_reserved":0,"allocated":475033600,"data_stored":469491745,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":871823814},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":1392,"seq":5978594476233,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":705716,"kb_used_data":307884,"kb_used_omap":117,"kb_used_meta":397706,"kb_avail":93039436,"statfs":{"total":95995035648,"available":95272382464,"internally_reserved":0,"allocated":315273216,"data_stored":309719830,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120638,"internal_metadata":407251138},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772075128,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1533352,"kb_used_data":648864,"kb_used_omap":71,"kb_used_meta":884408,"kb_avail":92211800,"statfs":{"total":95995035648,"available":94424883200,"internally_reserved":0,"allocated":664436736,"data_stored":658891406,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":905634742},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707304064,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1203108,"kb_used_data":561564,"kb_used_omap":68,"kb_used_meta":641467,"kb_avail":92542044,"statfs":{"total":95995035648,"available":94763053056,"internally_reserved":0,"allocated":575041536,"data_stored":569500182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":656862416},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872729223,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":761244,"kb_used_data":151700,"kb_used_omap":67,"kb_used_meta":609468,"kb_avail":92983908,"statfs":{"total":95995035648,"available":95215521792,"internally_reserved":0,"allocated":155340800,"data_stored":149806497,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":624095589},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056273,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1211108,"kb_used_data":439260,"kb_used_omap":74,"kb_used_meta":771701,"kb_avail":92534044,"statfs":{"total":95995035648,"available":94754861056,"internally_reserved":0,"allocated":449802240,"data_stored":444260354,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":790222517},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1901,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":428347392,"data_stored":428343320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":142147584,"data_stored":142147584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":533331968,"data_stored":533331968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":641142784,"data_stored":641138712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":294256640,"data_stored":294256640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":456523776,"data_stored":456523776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":693047296,"data_stored":693043224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":57,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":387252224,"data_stored":387252224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T05:00:34.735 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-14T05:00:34.735 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd pool set unique_pool_4 pg_num 26 2024-09-14T05:00:34.769 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:00:34.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:34 smithi129 ceph-mon[101316]: from='client.46457 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T05:00:34.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:34 smithi129 ceph-mon[101316]: 1.0 scrub starts 2024-09-14T05:00:34.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:34 smithi129 ceph-mon[101316]: 1.0 scrub ok 2024-09-14T05:00:34.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:34 smithi129 ceph-mon[101316]: 1.1 scrub starts 2024-09-14T05:00:34.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:34 smithi129 ceph-mon[101316]: 1.1 scrub ok 2024-09-14T05:00:34.771 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:34.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:34.835 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:34.835 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:34.936 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:34.979 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:34.980 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:34.980 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:35.081 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:35.111 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:35.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:35.112 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:35.213 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:35.250 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:35.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:35.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:35.352 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:35.363 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:00:35.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:35.379 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:35.380 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:35.385 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:00:35.400 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T05:00:35.400 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T05:00:35.435 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T05:00:35.436 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T05:00:35.436 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:35 smithi039 ceph-mon[111235]: 1.3 scrub starts 2024-09-14T05:00:35.436 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:35 smithi039 ceph-mon[111235]: 1.3 scrub ok 2024-09-14T05:00:35.436 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:35 smithi039 ceph-mon[111235]: pgmap v3055: 109 pgs: 109 active+clean; 983 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 851 op/s; 26 KiB/s, 0 objects/s recovering 2024-09-14T05:00:35.436 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:35 smithi039 ceph-mon[111235]: 1.12 scrub starts 2024-09-14T05:00:35.436 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:35 smithi039 ceph-mon[111235]: 1.12 scrub ok 2024-09-14T05:00:35.439 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:35 smithi039 ceph-mon[106470]: 1.3 scrub starts 2024-09-14T05:00:35.439 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:35 smithi039 ceph-mon[106470]: 1.3 scrub ok 2024-09-14T05:00:35.440 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:35 smithi039 ceph-mon[106470]: pgmap v3055: 109 pgs: 109 active+clean; 983 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 851 op/s; 26 KiB/s, 0 objects/s recovering 2024-09-14T05:00:35.440 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:35 smithi039 ceph-mon[106470]: 1.12 scrub starts 2024-09-14T05:00:35.440 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:35 smithi039 ceph-mon[106470]: 1.12 scrub ok 2024-09-14T05:00:35.473 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T05:00:35.473 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T05:00:35.480 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:35.504 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T05:00:35.505 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T05:00:35.515 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:35.515 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:35.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:35.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:35 smithi129 ceph-mon[101316]: 1.3 scrub starts 2024-09-14T05:00:35.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:35 smithi129 ceph-mon[101316]: 1.3 scrub ok 2024-09-14T05:00:35.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:35 smithi129 ceph-mon[101316]: pgmap v3055: 109 pgs: 109 active+clean; 983 MiB data, 8.5 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 851 op/s; 26 KiB/s, 0 objects/s recovering 2024-09-14T05:00:35.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:35 smithi129 ceph-mon[101316]: 1.12 scrub starts 2024-09-14T05:00:35.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:35 smithi129 ceph-mon[101316]: 1.12 scrub ok 2024-09-14T05:00:35.540 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T05:00:35.540 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T05:00:35.573 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T05:00:35.573 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T05:00:35.601 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T05:00:35.601 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T05:00:35.616 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:35.640 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T05:00:35.640 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T05:00:35.654 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:35.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:35.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:35.756 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:35.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:35.795 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:35.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:35.897 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:35.936 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:35.937 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:35.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:36.038 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:36.094 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:36.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:36.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:36.196 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:36.229 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:36.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:36.230 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:36.331 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:36.361 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:36.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:36.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:36.463 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:36.489 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:36.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:36.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:36.591 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:36.597 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:36 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1695457663' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:00:36.597 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:36 smithi129 ceph-mon[101316]: 1.13 scrub starts 2024-09-14T05:00:36.597 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:36 smithi129 ceph-mon[101316]: 1.13 scrub ok 2024-09-14T05:00:36.635 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:36.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:36.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:36.737 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:36.769 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T05:00:36.769 INFO:teuthology.orchestra.run.smithi129.stdout: "size": 20, 2024-09-14T05:00:36.770 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 600, 2024-09-14T05:00:36.770 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [ 2024-09-14T05:00:36.770 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.770 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:116283 56.6 56:68220df0:::benchmark_data_smithi039_330847_object1759:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1486)", 2024-09-14T05:00:36.770 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:04.658536+0000", 2024-09-14T05:00:36.770 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 392.10731935000001, 2024-09-14T05:00:36.770 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.39605932399999999, 2024-09-14T05:00:36.770 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:00:36.770 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:36.770 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:00:36.770 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:00:36.771 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:00:36.771 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 116283 2024-09-14T05:00:36.771 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.771 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:00:36.771 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.771 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:00:36.771 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658536+0000", 2024-09-14T05:00:36.771 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.771 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.771 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.771 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:00:36.771 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658536+0000", 2024-09-14T05:00:36.771 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.771 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.772 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.772 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:00:36.772 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658535+0000", 2024-09-14T05:00:36.772 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T05:00:36.772 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.772 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.772 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:00:36.772 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658539+0000", 2024-09-14T05:00:36.772 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.3229999999999999e-06 2024-09-14T05:00:36.772 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.772 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.772 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:00:36.772 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658548+0000", 2024-09-14T05:00:36.772 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.9740000000000008e-06 2024-09-14T05:00:36.773 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.773 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.773 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:00:36.773 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658555+0000", 2024-09-14T05:00:36.773 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.5119999999999997e-06 2024-09-14T05:00:36.773 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.773 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.773 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.773 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.667029+0000", 2024-09-14T05:00:36.773 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0084737279999999998 2024-09-14T05:00:36.773 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.773 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.774 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:00:36.774 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.669660+0000", 2024-09-14T05:00:36.774 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0026312179999999998 2024-09-14T05:00:36.774 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.774 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.774 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 4,7", 2024-09-14T05:00:36.774 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.669768+0000", 2024-09-14T05:00:36.774 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010799299999999999 2024-09-14T05:00:36.774 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.774 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.774 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:00:36.774 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.677330+0000", 2024-09-14T05:00:36.774 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0075621170000000001 2024-09-14T05:00:36.774 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.775 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.775 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.775 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.677369+0000", 2024-09-14T05:00:36.775 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8958999999999997e-05 2024-09-14T05:00:36.775 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.775 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.775 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.775 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.048251+0000", 2024-09-14T05:00:36.775 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.37088180599999998 2024-09-14T05:00:36.775 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.775 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.775 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:00:36.775 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.048303+0000", 2024-09-14T05:00:36.775 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2689999999999999e-05 2024-09-14T05:00:36.776 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.776 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.776 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:00:36.776 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.054595+0000", 2024-09-14T05:00:36.776 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0062918959999999999 2024-09-14T05:00:36.776 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.776 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:00:36.776 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.776 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.776 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.776 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127522 56.2 56:44f888df:::benchmark_data_smithi039_330847_object12998:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1488)", 2024-09-14T05:00:36.776 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.438889+0000", 2024-09-14T05:00:36.777 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 378.32696682699998, 2024-09-14T05:00:36.777 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2135127569999999, 2024-09-14T05:00:36.777 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:00:36.777 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:36.777 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:00:36.777 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:00:36.777 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:00:36.777 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127522 2024-09-14T05:00:36.777 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.777 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:00:36.777 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.777 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:00:36.777 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.438889+0000", 2024-09-14T05:00:36.778 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.778 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.778 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.778 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:00:36.778 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.438889+0000", 2024-09-14T05:00:36.778 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.778 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.778 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.778 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:00:36.778 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.438887+0000", 2024-09-14T05:00:36.778 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T05:00:36.778 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.779 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.779 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:00:36.779 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.438894+0000", 2024-09-14T05:00:36.779 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.5629999999999997e-06 2024-09-14T05:00:36.779 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.779 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.779 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:00:36.779 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.438906+0000", 2024-09-14T05:00:36.779 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2109e-05 2024-09-14T05:00:36.779 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.779 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.779 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:00:36.779 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.438918+0000", 2024-09-14T05:00:36.779 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2082000000000001e-05 2024-09-14T05:00:36.780 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.780 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.780 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.780 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.444929+0000", 2024-09-14T05:00:36.780 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0060105920000000004 2024-09-14T05:00:36.780 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.780 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.780 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:00:36.780 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.444940+0000", 2024-09-14T05:00:36.780 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1686e-05 2024-09-14T05:00:36.780 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.780 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.780 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.780 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.613189+0000", 2024-09-14T05:00:36.781 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1682488360000001 2024-09-14T05:00:36.781 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.781 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.781 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:00:36.781 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.613341+0000", 2024-09-14T05:00:36.781 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00015144600000000001 2024-09-14T05:00:36.781 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.781 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.781 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T05:00:36.781 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.613493+0000", 2024-09-14T05:00:36.781 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00015271300000000001 2024-09-14T05:00:36.781 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.781 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.781 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.782 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.625395+0000", 2024-09-14T05:00:36.782 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.011901836000000001 2024-09-14T05:00:36.782 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.782 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.782 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:00:36.782 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.625408+0000", 2024-09-14T05:00:36.782 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2699e-05 2024-09-14T05:00:36.782 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.782 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.782 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.782 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.649879+0000", 2024-09-14T05:00:36.782 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.024471017000000001 2024-09-14T05:00:36.782 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.782 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.783 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:00:36.783 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.649944+0000", 2024-09-14T05:00:36.783 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.5482999999999996e-05 2024-09-14T05:00:36.783 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.783 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.783 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:00:36.783 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.652401+0000", 2024-09-14T05:00:36.783 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0024570270000000001 2024-09-14T05:00:36.783 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.783 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:00:36.783 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.783 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.783 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.784 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127527 56.2 56:4905adce:::benchmark_data_smithi039_330847_object13003:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1489)", 2024-09-14T05:00:36.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:36.785 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.455332+0000", 2024-09-14T05:00:36.785 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 378.31052391200001, 2024-09-14T05:00:36.785 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1993806579999999, 2024-09-14T05:00:36.785 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:00:36.785 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:36.785 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:00:36.785 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:00:36.785 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:00:36.785 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127527 2024-09-14T05:00:36.785 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.785 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:00:36.786 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.786 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:00:36.786 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455332+0000", 2024-09-14T05:00:36.786 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.786 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.786 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.786 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:00:36.786 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455332+0000", 2024-09-14T05:00:36.786 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.786 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.786 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.786 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:00:36.786 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455331+0000", 2024-09-14T05:00:36.786 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T05:00:36.786 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.787 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.787 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:00:36.787 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455336+0000", 2024-09-14T05:00:36.787 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.6239999999999999e-06 2024-09-14T05:00:36.787 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.787 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.787 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:00:36.787 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455350+0000", 2024-09-14T05:00:36.787 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3927999999999999e-05 2024-09-14T05:00:36.787 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.787 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.787 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:00:36.787 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455358+0000", 2024-09-14T05:00:36.787 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.6879999999999999e-06 2024-09-14T05:00:36.787 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.788 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.788 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.788 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455390+0000", 2024-09-14T05:00:36.788 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.2332999999999997e-05 2024-09-14T05:00:36.788 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.788 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.788 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:00:36.788 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455397+0000", 2024-09-14T05:00:36.788 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.3560000000000004e-06 2024-09-14T05:00:36.788 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.788 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.788 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.788 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.620502+0000", 2024-09-14T05:00:36.788 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1651044210000001 2024-09-14T05:00:36.789 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.789 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.789 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:00:36.789 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.620657+0000", 2024-09-14T05:00:36.789 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000155029 2024-09-14T05:00:36.789 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.789 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.789 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T05:00:36.789 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.620824+0000", 2024-09-14T05:00:36.789 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00016709099999999999 2024-09-14T05:00:36.789 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.789 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.789 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:00:36.790 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.633463+0000", 2024-09-14T05:00:36.790 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.012638870999999999 2024-09-14T05:00:36.790 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.790 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.790 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.790 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.633522+0000", 2024-09-14T05:00:36.790 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9033000000000002e-05 2024-09-14T05:00:36.790 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.790 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.790 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.790 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.652482+0000", 2024-09-14T05:00:36.790 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.018959796000000001 2024-09-14T05:00:36.790 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.791 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.791 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:00:36.791 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.652543+0000", 2024-09-14T05:00:36.791 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.0943000000000002e-05 2024-09-14T05:00:36.791 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.791 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.791 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:00:36.791 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.654712+0000", 2024-09-14T05:00:36.791 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0021695239999999999 2024-09-14T05:00:36.791 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.791 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:00:36.791 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.791 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.791 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.792 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127528 56.6 56:6f55fbf1:::benchmark_data_smithi039_330847_object13004:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1489)", 2024-09-14T05:00:36.792 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.455626+0000", 2024-09-14T05:00:36.792 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 378.31022916699999, 2024-09-14T05:00:36.792 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1469407549999999, 2024-09-14T05:00:36.792 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:00:36.792 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:36.792 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:00:36.792 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:00:36.792 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:00:36.792 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127528 2024-09-14T05:00:36.792 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.792 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:00:36.792 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.792 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:00:36.793 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455626+0000", 2024-09-14T05:00:36.793 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.793 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.793 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.793 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:00:36.793 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455626+0000", 2024-09-14T05:00:36.793 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.793 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.793 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.793 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:00:36.793 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455626+0000", 2024-09-14T05:00:36.793 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T05:00:36.793 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.794 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.794 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:00:36.794 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455629+0000", 2024-09-14T05:00:36.794 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.4510000000000001e-06 2024-09-14T05:00:36.794 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.794 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.794 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:00:36.794 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455636+0000", 2024-09-14T05:00:36.794 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8589999999999999e-06 2024-09-14T05:00:36.794 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.794 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.794 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:00:36.794 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455640+0000", 2024-09-14T05:00:36.794 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0799999999999999e-06 2024-09-14T05:00:36.794 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.795 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.795 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.795 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455674+0000", 2024-09-14T05:00:36.795 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3964000000000002e-05 2024-09-14T05:00:36.795 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.795 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.795 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:00:36.795 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.455680+0000", 2024-09-14T05:00:36.795 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9259999999999997e-06 2024-09-14T05:00:36.795 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.795 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.795 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.795 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.589625+0000", 2024-09-14T05:00:36.796 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1339447499999999 2024-09-14T05:00:36.796 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.796 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.796 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:00:36.796 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.589735+0000", 2024-09-14T05:00:36.796 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000110177 2024-09-14T05:00:36.796 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.796 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.796 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T05:00:36.796 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.589828+0000", 2024-09-14T05:00:36.796 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.3558000000000001e-05 2024-09-14T05:00:36.796 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.796 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.796 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.797 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.599625+0000", 2024-09-14T05:00:36.797 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0097969819999999992 2024-09-14T05:00:36.797 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.797 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.797 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:00:36.797 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.599635+0000", 2024-09-14T05:00:36.797 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.9680000000000001e-06 2024-09-14T05:00:36.797 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.797 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.797 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.797 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.599742+0000", 2024-09-14T05:00:36.797 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000106397 2024-09-14T05:00:36.797 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.798 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.798 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:00:36.798 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.599801+0000", 2024-09-14T05:00:36.798 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.8866999999999997e-05 2024-09-14T05:00:36.798 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.798 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.798 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:00:36.798 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.602567+0000", 2024-09-14T05:00:36.798 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0027663470000000002 2024-09-14T05:00:36.798 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.798 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:00:36.798 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.798 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.798 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.799 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127537 56.2 56:49dbca7a:::benchmark_data_smithi039_330847_object13013:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1489)", 2024-09-14T05:00:36.799 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.470996+0000", 2024-09-14T05:00:36.799 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 378.29485945499999, 2024-09-14T05:00:36.799 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1875430259999999, 2024-09-14T05:00:36.799 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:00:36.799 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:36.799 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:00:36.799 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:00:36.799 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:00:36.799 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127537 2024-09-14T05:00:36.799 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.799 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:00:36.799 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.799 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:00:36.799 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.470996+0000", 2024-09-14T05:00:36.799 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.800 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.800 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.800 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:00:36.800 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.470996+0000", 2024-09-14T05:00:36.800 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.800 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.800 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.800 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:00:36.800 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.470994+0000", 2024-09-14T05:00:36.800 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T05:00:36.800 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.800 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.800 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:00:36.800 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.471001+0000", 2024-09-14T05:00:36.801 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7150000000000001e-06 2024-09-14T05:00:36.801 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.801 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.801 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:00:36.801 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.471016+0000", 2024-09-14T05:00:36.801 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5177999999999999e-05 2024-09-14T05:00:36.801 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.801 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.801 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:00:36.801 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.471023+0000", 2024-09-14T05:00:36.801 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.3269999999999997e-06 2024-09-14T05:00:36.801 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.801 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.801 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.802 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.471125+0000", 2024-09-14T05:00:36.802 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000102396 2024-09-14T05:00:36.802 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.802 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.802 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:00:36.802 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.471138+0000", 2024-09-14T05:00:36.802 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3407000000000001e-05 2024-09-14T05:00:36.802 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.802 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.802 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.802 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.625446+0000", 2024-09-14T05:00:36.802 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1543079039999999 2024-09-14T05:00:36.802 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.802 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.802 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:00:36.803 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.625558+0000", 2024-09-14T05:00:36.803 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000112037 2024-09-14T05:00:36.803 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.803 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.803 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T05:00:36.803 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.625652+0000", 2024-09-14T05:00:36.803 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.3692999999999999e-05 2024-09-14T05:00:36.803 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.803 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.803 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:00:36.803 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.637636+0000", 2024-09-14T05:00:36.803 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.011983705000000001 2024-09-14T05:00:36.803 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.803 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.804 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.804 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.641298+0000", 2024-09-14T05:00:36.804 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.003662579 2024-09-14T05:00:36.804 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.804 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.804 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.804 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.654763+0000", 2024-09-14T05:00:36.804 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.013464557 2024-09-14T05:00:36.804 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.804 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.804 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:00:36.804 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.654847+0000", 2024-09-14T05:00:36.804 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.4201000000000002e-05 2024-09-14T05:00:36.805 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.805 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.805 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:00:36.805 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.658539+0000", 2024-09-14T05:00:36.805 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0036919449999999999 2024-09-14T05:00:36.805 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.805 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:00:36.805 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.805 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.805 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.805 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127538 56.2 56:4c76f10c:::benchmark_data_smithi039_330847_object13014:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1489)", 2024-09-14T05:00:36.805 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.471037+0000", 2024-09-14T05:00:36.805 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 378.29481821399997, 2024-09-14T05:00:36.805 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1897702779999999, 2024-09-14T05:00:36.805 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:00:36.806 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:36.806 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:00:36.806 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:00:36.806 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:00:36.806 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127538 2024-09-14T05:00:36.806 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.806 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:00:36.806 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.806 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:00:36.806 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.471037+0000", 2024-09-14T05:00:36.806 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.806 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.806 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.806 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:00:36.806 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.471037+0000", 2024-09-14T05:00:36.807 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.807 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.807 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.807 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:00:36.807 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.471037+0000", 2024-09-14T05:00:36.807 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T05:00:36.807 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.807 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.807 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:00:36.807 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.471040+0000", 2024-09-14T05:00:36.807 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.151e-06 2024-09-14T05:00:36.808 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.808 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.808 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:00:36.808 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.471045+0000", 2024-09-14T05:00:36.808 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2979999999999999e-06 2024-09-14T05:00:36.808 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.808 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.808 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:00:36.808 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.471051+0000", 2024-09-14T05:00:36.808 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.3750000000000002e-06 2024-09-14T05:00:36.808 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.808 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.808 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.808 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.471183+0000", 2024-09-14T05:00:36.809 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013227700000000001 2024-09-14T05:00:36.809 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.809 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.809 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:00:36.809 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.471194+0000", 2024-09-14T05:00:36.809 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0570999999999999e-05 2024-09-14T05:00:36.809 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.809 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.809 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.809 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.629442+0000", 2024-09-14T05:00:36.809 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1582482270000001 2024-09-14T05:00:36.809 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.809 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.810 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:00:36.810 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.629517+0000", 2024-09-14T05:00:36.810 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.5038999999999998e-05 2024-09-14T05:00:36.810 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.810 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.810 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T05:00:36.810 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.629586+0000", 2024-09-14T05:00:36.810 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8867000000000003e-05 2024-09-14T05:00:36.810 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.810 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.810 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.810 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.641338+0000", 2024-09-14T05:00:36.810 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.011751914 2024-09-14T05:00:36.810 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.811 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.811 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:00:36.811 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.645510+0000", 2024-09-14T05:00:36.811 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0041725180000000001 2024-09-14T05:00:36.811 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.811 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.811 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.811 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.658598+0000", 2024-09-14T05:00:36.811 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.013088136 2024-09-14T05:00:36.811 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.811 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.811 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:00:36.811 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.658650+0000", 2024-09-14T05:00:36.811 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.1558999999999999e-05 2024-09-14T05:00:36.811 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.811 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.812 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:00:36.812 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.660807+0000", 2024-09-14T05:00:36.812 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0021577010000000002 2024-09-14T05:00:36.812 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.812 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:00:36.812 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.812 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.812 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.812 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:36.813 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:36.813 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:36 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1695457663' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:00:36.813 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:36 smithi039 ceph-mon[106470]: 1.13 scrub starts 2024-09-14T05:00:36.813 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:36 smithi039 ceph-mon[106470]: 1.13 scrub ok 2024-09-14T05:00:36.813 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:36 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1695457663' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:00:36.813 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:36 smithi039 ceph-mon[111235]: 1.13 scrub starts 2024-09-14T05:00:36.813 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:36 smithi039 ceph-mon[111235]: 1.13 scrub ok 2024-09-14T05:00:36.814 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127543 56.2 56:4af254b3:::benchmark_data_smithi039_330847_object13019:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1489)", 2024-09-14T05:00:36.814 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.478195+0000", 2024-09-14T05:00:36.814 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 378.28766065799999, 2024-09-14T05:00:36.814 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1849122379999999, 2024-09-14T05:00:36.814 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:00:36.814 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:36.814 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:00:36.815 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:00:36.815 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:00:36.815 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127543 2024-09-14T05:00:36.815 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.815 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:00:36.815 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.815 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:00:36.815 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.478195+0000", 2024-09-14T05:00:36.815 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.815 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.815 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.815 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:00:36.816 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.478195+0000", 2024-09-14T05:00:36.816 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.816 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.816 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.816 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:00:36.816 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.478193+0000", 2024-09-14T05:00:36.816 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T05:00:36.816 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.816 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.816 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:00:36.816 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.478200+0000", 2024-09-14T05:00:36.816 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.5830000000000003e-06 2024-09-14T05:00:36.816 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.817 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.817 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:00:36.817 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.478213+0000", 2024-09-14T05:00:36.817 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3258000000000001e-05 2024-09-14T05:00:36.817 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.817 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.817 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:00:36.817 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.478220+0000", 2024-09-14T05:00:36.817 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7530000000000004e-06 2024-09-14T05:00:36.817 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.817 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.817 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.817 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.486363+0000", 2024-09-14T05:00:36.817 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0081430289999999995 2024-09-14T05:00:36.818 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.818 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.818 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:00:36.818 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.486371+0000", 2024-09-14T05:00:36.818 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.1669999999999999e-06 2024-09-14T05:00:36.818 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.818 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.818 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.818 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.633559+0000", 2024-09-14T05:00:36.818 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1471879659999999 2024-09-14T05:00:36.818 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.818 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.818 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:00:36.819 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.633692+0000", 2024-09-14T05:00:36.819 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000133258 2024-09-14T05:00:36.819 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.819 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.819 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T05:00:36.819 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.633830+0000", 2024-09-14T05:00:36.819 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013777400000000001 2024-09-14T05:00:36.819 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.819 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.820 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:00:36.820 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.649665+0000", 2024-09-14T05:00:36.820 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.015834339999999999 2024-09-14T05:00:36.820 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.820 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.820 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.820 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.649724+0000", 2024-09-14T05:00:36.820 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9301e-05 2024-09-14T05:00:36.820 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.820 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.820 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.820 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.660852+0000", 2024-09-14T05:00:36.820 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.011128645 2024-09-14T05:00:36.821 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.821 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.821 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:00:36.821 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.660893+0000", 2024-09-14T05:00:36.821 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0447000000000001e-05 2024-09-14T05:00:36.821 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.821 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.821 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:00:36.821 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.663107+0000", 2024-09-14T05:00:36.821 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0022140699999999998 2024-09-14T05:00:36.821 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.821 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:00:36.821 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.821 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.822 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.822 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127552 56.2 56:4d13a696:::benchmark_data_smithi039_330847_object13028:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1489)", 2024-09-14T05:00:36.822 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.492573+0000", 2024-09-14T05:00:36.822 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 378.27328251300003, 2024-09-14T05:00:36.822 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1725031699999999, 2024-09-14T05:00:36.822 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:00:36.822 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:36.822 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:00:36.822 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:00:36.822 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:00:36.822 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127552 2024-09-14T05:00:36.822 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.822 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:00:36.822 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.822 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:00:36.823 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.492573+0000", 2024-09-14T05:00:36.823 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.823 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.823 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.823 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:00:36.823 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.492573+0000", 2024-09-14T05:00:36.823 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.823 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.823 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.823 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:00:36.823 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.492572+0000", 2024-09-14T05:00:36.823 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T05:00:36.823 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.824 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.824 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:00:36.824 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.492576+0000", 2024-09-14T05:00:36.824 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1250000000000003e-06 2024-09-14T05:00:36.824 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.824 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.824 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:00:36.824 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.492590+0000", 2024-09-14T05:00:36.824 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3733999999999999e-05 2024-09-14T05:00:36.824 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.824 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.824 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:00:36.824 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.492596+0000", 2024-09-14T05:00:36.824 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.3609999999999997e-06 2024-09-14T05:00:36.825 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.825 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.825 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.825 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.492682+0000", 2024-09-14T05:00:36.825 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.5546999999999999e-05 2024-09-14T05:00:36.825 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.825 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.825 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:00:36.825 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.492693+0000", 2024-09-14T05:00:36.825 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0832000000000001e-05 2024-09-14T05:00:36.825 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.825 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.825 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.826 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.637679+0000", 2024-09-14T05:00:36.826 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1449862200000001 2024-09-14T05:00:36.826 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.826 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.826 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:00:36.826 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.637797+0000", 2024-09-14T05:00:36.826 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011794 2024-09-14T05:00:36.826 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.826 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.826 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T05:00:36.826 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.637882+0000", 2024-09-14T05:00:36.826 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.4902999999999997e-05 2024-09-14T05:00:36.826 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.826 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.826 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:00:36.826 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.649672+0000", 2024-09-14T05:00:36.827 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.011790063999999999 2024-09-14T05:00:36.827 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.827 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.827 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.827 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.649826+0000", 2024-09-14T05:00:36.827 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00015375499999999999 2024-09-14T05:00:36.827 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.827 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.827 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.827 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.663158+0000", 2024-09-14T05:00:36.827 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.013332557 2024-09-14T05:00:36.827 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.827 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.827 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:00:36.827 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.663210+0000", 2024-09-14T05:00:36.828 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.1499000000000003e-05 2024-09-14T05:00:36.828 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.828 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.828 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:00:36.828 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.665076+0000", 2024-09-14T05:00:36.828 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001866468 2024-09-14T05:00:36.828 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.828 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:00:36.828 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.828 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.828 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.828 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127564 56.2 56:43bae9c1:::benchmark_data_smithi039_330847_object13040:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1489)", 2024-09-14T05:00:36.828 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.507445+0000", 2024-09-14T05:00:36.829 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 378.25841072700001, 2024-09-14T05:00:36.829 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1591943979999999, 2024-09-14T05:00:36.829 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:00:36.829 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:36.829 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:00:36.829 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:00:36.829 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:00:36.829 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127564 2024-09-14T05:00:36.829 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.829 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:00:36.829 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.829 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:00:36.829 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.507445+0000", 2024-09-14T05:00:36.829 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.830 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.830 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.830 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:00:36.830 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.507445+0000", 2024-09-14T05:00:36.830 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.830 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.830 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.830 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:00:36.830 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.507444+0000", 2024-09-14T05:00:36.830 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T05:00:36.830 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.830 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.830 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:00:36.831 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.507448+0000", 2024-09-14T05:00:36.831 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5710000000000002e-06 2024-09-14T05:00:36.831 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.831 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.831 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:00:36.831 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.507456+0000", 2024-09-14T05:00:36.831 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.9610000000000005e-06 2024-09-14T05:00:36.831 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.831 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.831 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:00:36.831 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.507461+0000", 2024-09-14T05:00:36.831 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.4879999999999998e-06 2024-09-14T05:00:36.831 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.831 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.832 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.832 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.507553+0000", 2024-09-14T05:00:36.832 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.1432000000000001e-05 2024-09-14T05:00:36.832 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.832 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.832 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:00:36.832 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.507563+0000", 2024-09-14T05:00:36.832 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0376e-05 2024-09-14T05:00:36.832 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.832 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.832 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.832 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.641377+0000", 2024-09-14T05:00:36.832 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1338141070000001 2024-09-14T05:00:36.832 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.833 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.833 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:00:36.833 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.641506+0000", 2024-09-14T05:00:36.833 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012888400000000001 2024-09-14T05:00:36.833 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.833 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.833 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T05:00:36.833 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.641627+0000", 2024-09-14T05:00:36.833 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012091600000000001 2024-09-14T05:00:36.833 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.833 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.833 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:00:36.833 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.652420+0000", 2024-09-14T05:00:36.833 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.010792708999999999 2024-09-14T05:00:36.834 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.834 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.834 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.834 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.665130+0000", 2024-09-14T05:00:36.834 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.012710116 2024-09-14T05:00:36.834 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.834 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.834 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.834 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.665215+0000", 2024-09-14T05:00:36.834 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.5507999999999995e-05 2024-09-14T05:00:36.834 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.834 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.834 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:00:36.834 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.665270+0000", 2024-09-14T05:00:36.835 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.4877999999999999e-05 2024-09-14T05:00:36.835 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.835 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.835 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:00:36.835 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.666639+0000", 2024-09-14T05:00:36.835 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0013690390000000001 2024-09-14T05:00:36.835 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.835 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:00:36.835 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.835 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.835 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.835 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127575 56.2 56:4a96f688:::benchmark_data_smithi039_330847_object13051:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1489)", 2024-09-14T05:00:36.835 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.518289+0000", 2024-09-14T05:00:36.835 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 378.24756594600001, 2024-09-14T05:00:36.836 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1500041139999999, 2024-09-14T05:00:36.836 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:00:36.836 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:36.836 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:00:36.836 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:00:36.836 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:00:36.836 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127575 2024-09-14T05:00:36.836 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.836 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:00:36.836 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.836 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:00:36.836 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.518289+0000", 2024-09-14T05:00:36.836 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.836 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.837 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.837 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:00:36.837 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.518289+0000", 2024-09-14T05:00:36.837 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.837 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.837 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.837 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:00:36.837 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.518288+0000", 2024-09-14T05:00:36.837 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T05:00:36.837 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.837 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.837 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:00:36.837 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.518293+0000", 2024-09-14T05:00:36.837 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.4649999999999996e-06 2024-09-14T05:00:36.838 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.838 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.838 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:00:36.838 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.518303+0000", 2024-09-14T05:00:36.838 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.9599999999999995e-06 2024-09-14T05:00:36.838 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.838 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.838 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:00:36.838 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.518308+0000", 2024-09-14T05:00:36.838 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7249999999999997e-06 2024-09-14T05:00:36.838 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.838 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.838 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.838 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.518352+0000", 2024-09-14T05:00:36.838 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.4205000000000001e-05 2024-09-14T05:00:36.839 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.839 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.839 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:00:36.839 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.518358+0000", 2024-09-14T05:00:36.839 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7109999999999998e-06 2024-09-14T05:00:36.839 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.839 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.839 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.839 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.645538+0000", 2024-09-14T05:00:36.839 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.127179285 2024-09-14T05:00:36.839 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.839 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.839 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:00:36.839 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.645748+0000", 2024-09-14T05:00:36.839 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000210421 2024-09-14T05:00:36.839 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.840 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.840 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T05:00:36.840 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.645863+0000", 2024-09-14T05:00:36.840 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000114912 2024-09-14T05:00:36.840 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.840 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.840 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:00:36.840 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.658557+0000", 2024-09-14T05:00:36.840 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.012694268999999999 2024-09-14T05:00:36.840 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.840 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.840 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.840 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.665171+0000", 2024-09-14T05:00:36.840 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0066135430000000004 2024-09-14T05:00:36.841 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.841 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.841 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.841 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.666697+0000", 2024-09-14T05:00:36.841 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0015255869999999999 2024-09-14T05:00:36.841 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.841 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.841 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:00:36.841 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.666730+0000", 2024-09-14T05:00:36.841 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3291999999999999e-05 2024-09-14T05:00:36.841 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.841 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.841 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:00:36.841 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.668294+0000", 2024-09-14T05:00:36.842 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001563785 2024-09-14T05:00:36.842 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.842 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:00:36.842 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.842 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.842 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.842 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127577 56.2 56:42aa155f:::benchmark_data_smithi039_330847_object13053:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1489)", 2024-09-14T05:00:36.842 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.522306+0000", 2024-09-14T05:00:36.842 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 378.243549755, 2024-09-14T05:00:36.842 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1678563989999999, 2024-09-14T05:00:36.843 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:00:36.843 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:36.843 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:00:36.843 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:00:36.843 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:00:36.843 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127577 2024-09-14T05:00:36.843 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.843 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:00:36.843 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.843 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:00:36.844 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522306+0000", 2024-09-14T05:00:36.844 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.844 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.844 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.844 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:00:36.844 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522306+0000", 2024-09-14T05:00:36.844 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.844 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.844 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.844 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:00:36.844 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522304+0000", 2024-09-14T05:00:36.844 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T05:00:36.844 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.844 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.845 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:00:36.845 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522310+0000", 2024-09-14T05:00:36.845 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.7010000000000001e-06 2024-09-14T05:00:36.845 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.845 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.845 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:00:36.845 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522325+0000", 2024-09-14T05:00:36.845 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.4766999999999999e-05 2024-09-14T05:00:36.846 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.846 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.846 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:00:36.846 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522332+0000", 2024-09-14T05:00:36.846 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.6930000000000001e-06 2024-09-14T05:00:36.846 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.846 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.846 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.846 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522403+0000", 2024-09-14T05:00:36.846 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.1266999999999994e-05 2024-09-14T05:00:36.846 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.846 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.846 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:00:36.846 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522414+0000", 2024-09-14T05:00:36.847 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1027e-05 2024-09-14T05:00:36.847 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.847 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.847 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.847 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.668337+0000", 2024-09-14T05:00:36.847 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.145923534 2024-09-14T05:00:36.847 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.847 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.847 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:00:36.847 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.668440+0000", 2024-09-14T05:00:36.847 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000102973 2024-09-14T05:00:36.847 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.847 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.847 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T05:00:36.847 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.668572+0000", 2024-09-14T05:00:36.848 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013213 2024-09-14T05:00:36.848 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.848 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.848 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.848 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.679226+0000", 2024-09-14T05:00:36.848 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.010653986000000001 2024-09-14T05:00:36.848 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.848 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.848 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:00:36.848 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.679236+0000", 2024-09-14T05:00:36.848 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.2669999999999995e-06 2024-09-14T05:00:36.848 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.848 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.848 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.849 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.687737+0000", 2024-09-14T05:00:36.849 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0085016450000000004 2024-09-14T05:00:36.849 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.849 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.849 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:00:36.849 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.687790+0000", 2024-09-14T05:00:36.849 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2457000000000003e-05 2024-09-14T05:00:36.849 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.849 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.849 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:00:36.850 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.690162+0000", 2024-09-14T05:00:36.850 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0023722370000000001 2024-09-14T05:00:36.850 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.850 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:00:36.850 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.850 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.850 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.850 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127578 56.2 56:46633f16:::benchmark_data_smithi039_330847_object13054:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1489)", 2024-09-14T05:00:36.850 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.522479+0000", 2024-09-14T05:00:36.850 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 378.24337651299999, 2024-09-14T05:00:36.850 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1704580840000001, 2024-09-14T05:00:36.850 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:00:36.850 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:36.850 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:00:36.850 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:00:36.851 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:00:36.851 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127578 2024-09-14T05:00:36.851 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.851 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:00:36.851 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.851 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:00:36.851 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522479+0000", 2024-09-14T05:00:36.851 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.851 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.851 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.851 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:00:36.851 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522479+0000", 2024-09-14T05:00:36.851 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.851 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.851 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.852 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:00:36.852 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522478+0000", 2024-09-14T05:00:36.852 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T05:00:36.852 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.852 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.852 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:00:36.852 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522482+0000", 2024-09-14T05:00:36.852 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9689999999999996e-06 2024-09-14T05:00:36.852 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.852 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.852 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:00:36.852 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522492+0000", 2024-09-14T05:00:36.852 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.3570000000000004e-06 2024-09-14T05:00:36.853 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.853 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.853 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:00:36.853 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522497+0000", 2024-09-14T05:00:36.853 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.6829999999999996e-06 2024-09-14T05:00:36.853 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.853 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.853 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.853 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522542+0000", 2024-09-14T05:00:36.853 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.4907000000000002e-05 2024-09-14T05:00:36.853 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.853 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.853 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:00:36.853 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.522553+0000", 2024-09-14T05:00:36.854 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1228e-05 2024-09-14T05:00:36.854 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.854 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.854 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.854 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.673544+0000", 2024-09-14T05:00:36.854 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1509901760000001 2024-09-14T05:00:36.854 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.854 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.854 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:00:36.854 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.673667+0000", 2024-09-14T05:00:36.854 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012314199999999999 2024-09-14T05:00:36.854 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.854 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.854 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T05:00:36.854 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.673795+0000", 2024-09-14T05:00:36.855 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012876900000000001 2024-09-14T05:00:36.855 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.855 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.855 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.855 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.679272+0000", 2024-09-14T05:00:36.855 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0054765550000000001 2024-09-14T05:00:36.855 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.855 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.855 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:00:36.855 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.687680+0000", 2024-09-14T05:00:36.855 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0084078009999999995 2024-09-14T05:00:36.855 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.855 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.855 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.855 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.690246+0000", 2024-09-14T05:00:36.856 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0025666399999999998 2024-09-14T05:00:36.856 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.856 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.856 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:00:36.856 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.690354+0000", 2024-09-14T05:00:36.857 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000108007 2024-09-14T05:00:36.857 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.857 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.857 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:00:36.857 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.692937+0000", 2024-09-14T05:00:36.857 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0025824910000000001 2024-09-14T05:00:36.857 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.857 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:00:36.857 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.857 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.857 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.857 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127583 56.6 56:614096e5:::benchmark_data_smithi039_330847_object13059:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1489)", 2024-09-14T05:00:36.858 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.527314+0000", 2024-09-14T05:00:36.858 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 378.23854135400001, 2024-09-14T05:00:36.858 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.077654313, 2024-09-14T05:00:36.858 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:00:36.858 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:36.858 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:00:36.858 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:00:36.858 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:00:36.858 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127583 2024-09-14T05:00:36.858 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.858 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:00:36.858 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.858 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:00:36.859 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.527314+0000", 2024-09-14T05:00:36.859 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.859 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.859 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.859 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:00:36.859 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.527314+0000", 2024-09-14T05:00:36.859 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.859 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.859 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.859 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:00:36.859 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.527314+0000", 2024-09-14T05:00:36.859 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T05:00:36.859 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.859 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.859 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:00:36.860 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.527316+0000", 2024-09-14T05:00:36.860 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.892e-06 2024-09-14T05:00:36.860 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.860 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.860 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:00:36.860 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.527323+0000", 2024-09-14T05:00:36.860 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.5520000000000001e-06 2024-09-14T05:00:36.860 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.860 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.860 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:00:36.860 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.527328+0000", 2024-09-14T05:00:36.860 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.4600000000000002e-06 2024-09-14T05:00:36.860 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.861 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.861 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.861 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.527380+0000", 2024-09-14T05:00:36.861 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.1904999999999999e-05 2024-09-14T05:00:36.861 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.861 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.861 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:00:36.861 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.527390+0000", 2024-09-14T05:00:36.861 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.4420000000000004e-06 2024-09-14T05:00:36.861 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.861 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.861 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.861 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.595448+0000", 2024-09-14T05:00:36.861 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.068058261 2024-09-14T05:00:36.862 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.862 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.862 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:00:36.862 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.595589+0000", 2024-09-14T05:00:36.862 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00014102 2024-09-14T05:00:36.862 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.862 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.862 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T05:00:36.862 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.595720+0000", 2024-09-14T05:00:36.862 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000130786 2024-09-14T05:00:36.862 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.862 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.862 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.862 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.602659+0000", 2024-09-14T05:00:36.862 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0069390529999999997 2024-09-14T05:00:36.863 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.863 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.863 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.863 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.602795+0000", 2024-09-14T05:00:36.863 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013638499999999999 2024-09-14T05:00:36.863 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.863 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.863 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:00:36.863 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.602853+0000", 2024-09-14T05:00:36.863 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.7349000000000003e-05 2024-09-14T05:00:36.863 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.863 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.863 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:00:36.863 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.602919+0000", 2024-09-14T05:00:36.864 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.6544999999999994e-05 2024-09-14T05:00:36.864 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.864 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.864 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:00:36.864 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.604968+0000", 2024-09-14T05:00:36.864 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0020491929999999999 2024-09-14T05:00:36.864 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.864 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:00:36.864 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.864 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.864 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.864 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127591 56.2 56:412cff32:::benchmark_data_smithi039_330847_object13067:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1489)", 2024-09-14T05:00:36.864 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.541493+0000", 2024-09-14T05:00:36.864 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 378.22436250700002, 2024-09-14T05:00:36.865 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1539498459999999, 2024-09-14T05:00:36.865 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:00:36.865 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:36.865 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:00:36.865 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:00:36.865 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:00:36.865 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127591 2024-09-14T05:00:36.865 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.865 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:00:36.865 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.865 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:00:36.865 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.541493+0000", 2024-09-14T05:00:36.865 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.865 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.865 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.866 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:00:36.866 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.541493+0000", 2024-09-14T05:00:36.866 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.866 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.866 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.866 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:00:36.866 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.541491+0000", 2024-09-14T05:00:36.866 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T05:00:36.866 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.866 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.866 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:00:36.866 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.541498+0000", 2024-09-14T05:00:36.866 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7649999999999996e-06 2024-09-14T05:00:36.866 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.867 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.867 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:00:36.868 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.541511+0000", 2024-09-14T05:00:36.868 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3053e-05 2024-09-14T05:00:36.868 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.868 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.868 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:00:36.868 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.541517+0000", 2024-09-14T05:00:36.868 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.6370000000000002e-06 2024-09-14T05:00:36.868 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.868 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.868 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.868 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.541578+0000", 2024-09-14T05:00:36.868 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.0454000000000001e-05 2024-09-14T05:00:36.868 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.869 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.869 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:00:36.869 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.541590+0000", 2024-09-14T05:00:36.869 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2174000000000001e-05 2024-09-14T05:00:36.869 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.869 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.869 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.869 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.679299+0000", 2024-09-14T05:00:36.869 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1377089200000001 2024-09-14T05:00:36.869 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.869 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.869 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:00:36.869 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.679416+0000", 2024-09-14T05:00:36.870 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000116958 2024-09-14T05:00:36.870 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.870 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.870 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T05:00:36.870 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.679535+0000", 2024-09-14T05:00:36.870 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011906200000000001 2024-09-14T05:00:36.870 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.870 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.870 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:00:36.870 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.692967+0000", 2024-09-14T05:00:36.870 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.013431764000000001 2024-09-14T05:00:36.870 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.870 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.871 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.871 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.693051+0000", 2024-09-14T05:00:36.871 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.4680999999999998e-05 2024-09-14T05:00:36.871 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.871 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.871 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.871 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.693370+0000", 2024-09-14T05:00:36.871 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00031845700000000002 2024-09-14T05:00:36.871 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.871 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.871 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:00:36.871 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.693432+0000", 2024-09-14T05:00:36.871 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.2108999999999998e-05 2024-09-14T05:00:36.872 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.872 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.872 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:00:36.872 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.695443+0000", 2024-09-14T05:00:36.872 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0020107110000000001 2024-09-14T05:00:36.872 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.872 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:00:36.872 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.872 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.872 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.872 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127593 56.2 56:4e62fc54:::benchmark_data_smithi039_330847_object13069:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1489)", 2024-09-14T05:00:36.872 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.544475+0000", 2024-09-14T05:00:36.872 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 378.22138042500001, 2024-09-14T05:00:36.873 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.152692112, 2024-09-14T05:00:36.873 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:00:36.873 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:36.873 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:00:36.873 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:00:36.873 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:00:36.873 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127593 2024-09-14T05:00:36.873 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.873 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:00:36.873 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.873 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:00:36.873 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.544475+0000", 2024-09-14T05:00:36.873 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.873 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.873 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.874 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:00:36.874 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.544475+0000", 2024-09-14T05:00:36.874 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.874 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.874 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.874 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:00:36.874 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.544474+0000", 2024-09-14T05:00:36.874 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T05:00:36.874 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.874 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.874 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:00:36.874 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.544482+0000", 2024-09-14T05:00:36.874 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.6790000000000005e-06 2024-09-14T05:00:36.875 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.875 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.875 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:00:36.875 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.544494+0000", 2024-09-14T05:00:36.875 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2109999999999999e-05 2024-09-14T05:00:36.875 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.875 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.875 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:00:36.875 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.544499+0000", 2024-09-14T05:00:36.875 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.1919999999999996e-06 2024-09-14T05:00:36.875 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.875 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.875 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.876 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.544573+0000", 2024-09-14T05:00:36.876 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.3832999999999996e-05 2024-09-14T05:00:36.876 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.876 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.876 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:00:36.876 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.544581+0000", 2024-09-14T05:00:36.876 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.5150000000000006e-06 2024-09-14T05:00:36.876 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.876 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.876 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.876 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.683535+0000", 2024-09-14T05:00:36.876 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1389538029999999 2024-09-14T05:00:36.876 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.876 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.877 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:00:36.877 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.683644+0000", 2024-09-14T05:00:36.877 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000108785 2024-09-14T05:00:36.877 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.877 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.877 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T05:00:36.877 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.683753+0000", 2024-09-14T05:00:36.877 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010919199999999999 2024-09-14T05:00:36.877 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.877 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.877 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.877 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.693113+0000", 2024-09-14T05:00:36.877 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0093597039999999999 2024-09-14T05:00:36.878 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.878 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.878 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.878 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.695511+0000", 2024-09-14T05:00:36.878 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0023985259999999998 2024-09-14T05:00:36.878 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.878 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.878 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:00:36.878 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.695522+0000", 2024-09-14T05:00:36.878 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0961000000000001e-05 2024-09-14T05:00:36.878 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.878 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.878 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:00:36.879 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.695580+0000", 2024-09-14T05:00:36.879 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.7333000000000001e-05 2024-09-14T05:00:36.879 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.879 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.879 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:00:36.879 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.697167+0000", 2024-09-14T05:00:36.879 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0015874929999999999 2024-09-14T05:00:36.879 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.879 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:00:36.879 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.879 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.879 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.879 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127614 56.2 56:49d857b3:::benchmark_data_smithi039_330847_object13090:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1489)", 2024-09-14T05:00:36.879 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.599580+0000", 2024-09-14T05:00:36.880 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 378.16627565700003, 2024-09-14T05:00:36.880 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.118777608, 2024-09-14T05:00:36.880 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:00:36.880 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:36.880 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:00:36.880 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:00:36.880 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:00:36.880 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127614 2024-09-14T05:00:36.880 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.880 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:00:36.880 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.880 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:00:36.880 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.599580+0000", 2024-09-14T05:00:36.880 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.881 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.881 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.881 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:00:36.881 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.599580+0000", 2024-09-14T05:00:36.881 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.881 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.881 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.881 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:00:36.881 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.599579+0000", 2024-09-14T05:00:36.881 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T05:00:36.881 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.881 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.881 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:00:36.881 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.599584+0000", 2024-09-14T05:00:36.882 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6410000000000001e-06 2024-09-14T05:00:36.882 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.882 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.882 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:00:36.882 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.599593+0000", 2024-09-14T05:00:36.882 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.9670000000000007e-06 2024-09-14T05:00:36.882 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.882 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.882 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:00:36.882 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.599597+0000", 2024-09-14T05:00:36.882 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7260000000000002e-06 2024-09-14T05:00:36.882 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.882 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.882 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.882 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.599644+0000", 2024-09-14T05:00:36.883 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6816e-05 2024-09-14T05:00:36.883 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.883 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.883 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:00:36.883 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.599653+0000", 2024-09-14T05:00:36.883 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.9199999999999993e-06 2024-09-14T05:00:36.883 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.883 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.883 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.883 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.697216+0000", 2024-09-14T05:00:36.883 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0975626919999999 2024-09-14T05:00:36.883 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.883 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.883 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:00:36.884 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.697336+0000", 2024-09-14T05:00:36.884 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000119826 2024-09-14T05:00:36.884 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.884 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.884 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T05:00:36.884 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.697462+0000", 2024-09-14T05:00:36.884 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000126728 2024-09-14T05:00:36.884 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.884 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.884 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:00:36.884 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.709739+0000", 2024-09-14T05:00:36.884 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.012276665000000001 2024-09-14T05:00:36.884 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.885 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.885 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.885 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.713756+0000", 2024-09-14T05:00:36.885 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0040171970000000001 2024-09-14T05:00:36.885 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.885 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.885 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.885 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.713795+0000", 2024-09-14T05:00:36.885 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9113000000000003e-05 2024-09-14T05:00:36.885 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.885 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.885 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:00:36.885 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.713903+0000", 2024-09-14T05:00:36.885 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000107394 2024-09-14T05:00:36.886 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.886 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.886 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:00:36.886 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.718357+0000", 2024-09-14T05:00:36.886 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0044547609999999998 2024-09-14T05:00:36.886 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.886 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:00:36.886 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.886 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.886 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.886 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:127629 56.2 56:44c02451:::benchmark_data_smithi039_330847_object13105:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1489)", 2024-09-14T05:00:36.886 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:18.661787+0000", 2024-09-14T05:00:36.886 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 378.10406851699997, 2024-09-14T05:00:36.887 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0597727610000001, 2024-09-14T05:00:36.887 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:00:36.887 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:36.887 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:00:36.887 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:00:36.887 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:00:36.887 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 127629 2024-09-14T05:00:36.887 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.887 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:00:36.887 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.887 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:00:36.887 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.661787+0000", 2024-09-14T05:00:36.887 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.887 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.888 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.888 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:00:36.888 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.661787+0000", 2024-09-14T05:00:36.888 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.888 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.888 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.888 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:00:36.888 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.661786+0000", 2024-09-14T05:00:36.888 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T05:00:36.888 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.888 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.888 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:00:36.888 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.661791+0000", 2024-09-14T05:00:36.888 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.0769999999999998e-06 2024-09-14T05:00:36.889 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.889 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.889 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:00:36.889 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.661801+0000", 2024-09-14T05:00:36.889 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0122000000000001e-05 2024-09-14T05:00:36.889 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.889 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.889 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:00:36.889 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.661806+0000", 2024-09-14T05:00:36.889 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.0470000000000001e-06 2024-09-14T05:00:36.889 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.889 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.889 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.890 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.661860+0000", 2024-09-14T05:00:36.890 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.4083999999999999e-05 2024-09-14T05:00:36.890 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.890 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.890 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:00:36.890 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:18.661873+0000", 2024-09-14T05:00:36.890 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2429000000000001e-05 2024-09-14T05:00:36.890 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.890 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.890 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.890 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.700879+0000", 2024-09-14T05:00:36.890 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0390063409999999 2024-09-14T05:00:36.890 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.890 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.891 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:00:36.891 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.700986+0000", 2024-09-14T05:00:36.891 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000106844 2024-09-14T05:00:36.891 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.891 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.891 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T05:00:36.891 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.701069+0000", 2024-09-14T05:00:36.891 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.3529999999999995e-05 2024-09-14T05:00:36.891 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.891 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.891 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:00:36.891 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.713705+0000", 2024-09-14T05:00:36.891 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.012635171000000001 2024-09-14T05:00:36.892 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.892 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.892 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.892 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.718635+0000", 2024-09-14T05:00:36.892 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0049301420000000002 2024-09-14T05:00:36.892 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.892 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.892 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.892 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.718682+0000", 2024-09-14T05:00:36.892 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7349999999999999e-05 2024-09-14T05:00:36.892 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.892 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.892 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:00:36.893 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.718766+0000", 2024-09-14T05:00:36.893 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.4369000000000005e-05 2024-09-14T05:00:36.893 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.893 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.893 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:00:36.893 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:19.721560+0000", 2024-09-14T05:00:36.893 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0027932740000000001 2024-09-14T05:00:36.893 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.893 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:00:36.893 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.893 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.893 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.893 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:150548 56.6 56:627884de:::benchmark_data_smithi039_330847_object36024:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1502)", 2024-09-14T05:00:36.893 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:48.347280+0000", 2024-09-14T05:00:36.894 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 348.41857562199999, 2024-09-14T05:00:36.894 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.35218084399999999, 2024-09-14T05:00:36.894 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:00:36.894 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:36.894 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:00:36.894 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:00:36.894 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:00:36.894 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 150548 2024-09-14T05:00:36.894 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.894 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:00:36.894 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.894 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:00:36.894 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.347280+0000", 2024-09-14T05:00:36.894 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.895 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.895 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.895 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:00:36.895 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.347280+0000", 2024-09-14T05:00:36.895 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.895 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.895 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.895 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:00:36.895 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.347278+0000", 2024-09-14T05:00:36.895 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T05:00:36.895 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.895 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.895 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:00:36.896 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.347288+0000", 2024-09-14T05:00:36.896 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.2900000000000008e-06 2024-09-14T05:00:36.896 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.896 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.896 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:00:36.896 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.347304+0000", 2024-09-14T05:00:36.896 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.6648000000000001e-05 2024-09-14T05:00:36.896 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.896 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.896 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:00:36.896 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.347313+0000", 2024-09-14T05:00:36.896 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.1149999999999994e-06 2024-09-14T05:00:36.896 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.897 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.897 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.897 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.351898+0000", 2024-09-14T05:00:36.897 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0045857010000000002 2024-09-14T05:00:36.897 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.897 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.897 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:00:36.897 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.352045+0000", 2024-09-14T05:00:36.897 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000146515 2024-09-14T05:00:36.897 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.897 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.897 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T05:00:36.897 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.352178+0000", 2024-09-14T05:00:36.897 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000133499 2024-09-14T05:00:36.898 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.898 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.898 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:00:36.898 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.359617+0000", 2024-09-14T05:00:36.898 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0074388099999999997 2024-09-14T05:00:36.898 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.898 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.898 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.898 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.359655+0000", 2024-09-14T05:00:36.898 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8433000000000003e-05 2024-09-14T05:00:36.898 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.898 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.898 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.898 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.697666+0000", 2024-09-14T05:00:36.899 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33801028300000002 2024-09-14T05:00:36.899 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.899 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.899 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:00:36.899 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.697712+0000", 2024-09-14T05:00:36.899 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6282000000000002e-05 2024-09-14T05:00:36.899 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.899 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.899 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:00:36.899 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.699461+0000", 2024-09-14T05:00:36.899 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001748585 2024-09-14T05:00:36.899 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.899 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:00:36.900 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.900 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.900 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.900 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:150553 56.6 56:6540d818:::benchmark_data_smithi039_330847_object36029:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1502)", 2024-09-14T05:00:36.900 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:48.352518+0000", 2024-09-14T05:00:36.900 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 348.41333770300002, 2024-09-14T05:00:36.900 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.34859365399999998, 2024-09-14T05:00:36.900 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:00:36.900 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:36.900 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:00:36.900 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:00:36.900 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:00:36.900 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 150553 2024-09-14T05:00:36.901 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.901 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:00:36.901 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.901 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:00:36.901 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.352518+0000", 2024-09-14T05:00:36.901 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.901 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.901 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.901 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:00:36.901 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.352518+0000", 2024-09-14T05:00:36.901 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.901 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.901 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.901 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:00:36.902 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.352516+0000", 2024-09-14T05:00:36.902 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T05:00:36.902 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.902 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.902 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:00:36.902 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.352523+0000", 2024-09-14T05:00:36.902 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.4810000000000003e-06 2024-09-14T05:00:36.902 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.902 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.902 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:00:36.902 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.352535+0000", 2024-09-14T05:00:36.902 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2023e-05 2024-09-14T05:00:36.902 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.902 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.903 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:00:36.903 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.352541+0000", 2024-09-14T05:00:36.903 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.6000000000000003e-06 2024-09-14T05:00:36.903 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.903 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.903 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.903 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.355640+0000", 2024-09-14T05:00:36.903 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0030980780000000002 2024-09-14T05:00:36.903 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.903 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.903 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:00:36.903 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.355790+0000", 2024-09-14T05:00:36.903 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00015043700000000001 2024-09-14T05:00:36.903 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.904 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.904 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,7", 2024-09-14T05:00:36.904 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.355951+0000", 2024-09-14T05:00:36.904 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000160762 2024-09-14T05:00:36.904 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.904 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.904 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:00:36.904 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.363896+0000", 2024-09-14T05:00:36.904 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0079453760000000005 2024-09-14T05:00:36.904 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.904 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.904 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.904 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.364349+0000", 2024-09-14T05:00:36.904 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000452669 2024-09-14T05:00:36.905 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.905 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.905 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.905 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.699528+0000", 2024-09-14T05:00:36.905 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.335179385 2024-09-14T05:00:36.905 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.905 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.905 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:00:36.905 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.699580+0000", 2024-09-14T05:00:36.905 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2187e-05 2024-09-14T05:00:36.905 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.905 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.905 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:00:36.905 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:48.701111+0000", 2024-09-14T05:00:36.906 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001531019 2024-09-14T05:00:36.906 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.906 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:00:36.906 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.906 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.906 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.906 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36324.0:222427 57.6 57:6e20c479:::benchmark_data_smithi039_365565_object98675:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1552)", 2024-09-14T05:00:36.906 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:00:20.975913+0000", 2024-09-14T05:00:36.906 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 15.78994237, 2024-09-14T05:00:36.906 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.16724496599999999, 2024-09-14T05:00:36.906 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:00:36.906 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:00:36.906 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:00:36.906 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36324", 2024-09-14T05:00:36.907 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:00:36.907 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 222427 2024-09-14T05:00:36.907 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.907 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:00:36.907 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.907 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:00:36.907 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:20.975913+0000", 2024-09-14T05:00:36.907 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.907 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.907 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.907 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:00:36.907 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:20.975913+0000", 2024-09-14T05:00:36.907 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:36.907 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.908 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.908 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:00:36.908 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:20.975911+0000", 2024-09-14T05:00:36.908 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T05:00:36.908 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.908 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.908 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:00:36.908 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:20.975920+0000", 2024-09-14T05:00:36.908 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.1619999999999997e-06 2024-09-14T05:00:36.908 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.908 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.908 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:00:36.908 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:20.975937+0000", 2024-09-14T05:00:36.909 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.6582000000000001e-05 2024-09-14T05:00:36.909 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.909 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.909 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:00:36.909 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:20.975944+0000", 2024-09-14T05:00:36.909 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.0949999999999998e-06 2024-09-14T05:00:36.909 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.909 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.909 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:36.909 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:20.975999+0000", 2024-09-14T05:00:36.909 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.5012000000000001e-05 2024-09-14T05:00:36.909 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.909 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.909 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:00:36.910 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:20.976135+0000", 2024-09-14T05:00:36.910 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013573400000000001 2024-09-14T05:00:36.910 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.910 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.910 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,2", 2024-09-14T05:00:36.910 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:20.976250+0000", 2024-09-14T05:00:36.910 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000115099 2024-09-14T05:00:36.910 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.910 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.910 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:00:36.910 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:20.983655+0000", 2024-09-14T05:00:36.910 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0074054480000000002 2024-09-14T05:00:36.910 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.911 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.911 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.911 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:20.995269+0000", 2024-09-14T05:00:36.911 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.011613408 2024-09-14T05:00:36.911 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.911 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.911 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:00:36.911 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:21.143005+0000", 2024-09-14T05:00:36.911 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.14773625400000001 2024-09-14T05:00:36.911 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.911 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.911 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:00:36.911 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:21.143121+0000", 2024-09-14T05:00:36.911 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000116542 2024-09-14T05:00:36.911 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:36.912 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:36.912 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:00:36.912 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:21.143158+0000", 2024-09-14T05:00:36.912 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6606000000000002e-05 2024-09-14T05:00:36.912 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.912 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:00:36.912 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.912 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:36.912 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:00:36.912 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T05:00:36.914 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:36.950 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:36.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:36.951 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:37.052 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:37.089 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:37.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:37.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:37.190 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:37.226 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:37.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:37.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:37.328 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:37.361 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:37.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:37.362 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:37.463 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:37.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:37.500 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:37.501 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:37.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:37 smithi039 ceph-mon[106470]: 1.d scrub starts 2024-09-14T05:00:37.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:37 smithi039 ceph-mon[106470]: 1.d scrub ok 2024-09-14T05:00:37.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:37 smithi039 ceph-mon[106470]: pgmap v3056: 109 pgs: 109 active+clean; 827 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2024-09-14T05:00:37.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:37 smithi039 ceph-mon[106470]: 1.10 scrub starts 2024-09-14T05:00:37.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:37 smithi039 ceph-mon[106470]: 1.10 scrub ok 2024-09-14T05:00:37.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:00:37.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:37 smithi039 ceph-mon[111235]: 1.d scrub starts 2024-09-14T05:00:37.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:37 smithi039 ceph-mon[111235]: 1.d scrub ok 2024-09-14T05:00:37.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:37 smithi039 ceph-mon[111235]: pgmap v3056: 109 pgs: 109 active+clean; 827 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2024-09-14T05:00:37.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:37 smithi039 ceph-mon[111235]: 1.10 scrub starts 2024-09-14T05:00:37.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:37 smithi039 ceph-mon[111235]: 1.10 scrub ok 2024-09-14T05:00:37.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:00:37.601 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:37.628 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:37.629 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:37.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:37.730 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:37.766 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:37.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:37.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:37.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:37 smithi129 ceph-mon[101316]: 1.d scrub starts 2024-09-14T05:00:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:37 smithi129 ceph-mon[101316]: 1.d scrub ok 2024-09-14T05:00:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:37 smithi129 ceph-mon[101316]: pgmap v3056: 109 pgs: 109 active+clean; 827 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2024-09-14T05:00:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:37 smithi129 ceph-mon[101316]: 1.10 scrub starts 2024-09-14T05:00:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:37 smithi129 ceph-mon[101316]: 1.10 scrub ok 2024-09-14T05:00:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:00:37.867 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:37.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:37.900 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:37.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:37.945 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-14T05:00:38.001 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:38.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:38.044 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:38.044 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:38.145 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:38.198 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:38.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:38.199 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:38.287 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:00:38.300 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:38.336 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:38.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:38.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:38.438 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:38.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:38.473 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:38.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:38.574 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:38.618 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:38.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:38.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:38.720 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:38.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:38.760 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:38.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:38.862 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:38.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:38.896 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:38.896 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:38.997 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:39.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:39.037 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:39.037 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:39.039 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:00:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:00:38] "GET /metrics HTTP/1.1" 200 36745 "" "Prometheus/2.51.0" 2024-09-14T05:00:39.139 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:39.173 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:39.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:39.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:39.275 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:39.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:39.305 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:39.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:39.406 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:39.454 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:39.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:39.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:39.556 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:39.596 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:39.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:39.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:39.628 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:39 smithi129 ceph-mon[101316]: 1.6 scrub starts 2024-09-14T05:00:39.628 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:39 smithi129 ceph-mon[101316]: 1.6 scrub ok 2024-09-14T05:00:39.628 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:39 smithi129 ceph-mon[101316]: pgmap v3057: 109 pgs: 109 active+clean; 756 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 894 op/s 2024-09-14T05:00:39.628 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:39 smithi129 ceph-mon[101316]: 1.e scrub starts 2024-09-14T05:00:39.628 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:39 smithi129 ceph-mon[101316]: 1.e scrub ok 2024-09-14T05:00:39.698 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:39.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:39.743 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:39.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:39.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:39 smithi039 ceph-mon[106470]: 1.6 scrub starts 2024-09-14T05:00:39.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:39 smithi039 ceph-mon[106470]: 1.6 scrub ok 2024-09-14T05:00:39.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:39 smithi039 ceph-mon[106470]: pgmap v3057: 109 pgs: 109 active+clean; 756 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 894 op/s 2024-09-14T05:00:39.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:39 smithi039 ceph-mon[106470]: 1.e scrub starts 2024-09-14T05:00:39.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:39 smithi039 ceph-mon[106470]: 1.e scrub ok 2024-09-14T05:00:39.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:39 smithi039 ceph-mon[111235]: 1.6 scrub starts 2024-09-14T05:00:39.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:39 smithi039 ceph-mon[111235]: 1.6 scrub ok 2024-09-14T05:00:39.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:39 smithi039 ceph-mon[111235]: pgmap v3057: 109 pgs: 109 active+clean; 756 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 894 op/s 2024-09-14T05:00:39.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:39 smithi039 ceph-mon[111235]: 1.e scrub starts 2024-09-14T05:00:39.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:39 smithi039 ceph-mon[111235]: 1.e scrub ok 2024-09-14T05:00:39.844 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:39.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:39.887 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:39.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:39.922 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T05:00:39.989 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:40.061 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:40.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:40.063 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:40.164 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:40.222 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:40.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:40.223 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:40.321 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T05:00:40.321 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [ 2024-09-14T05:00:40.321 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:40.321 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.36324.0:239924 57.3 e1557/1518 57:c011b94a:::benchmark_data_smithi039_365565_object116172:head v 1557'14941, mlcod=1557'14938)", 2024-09-14T05:00:40.321 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:00:40.316175+0000", 2024-09-14T05:00:40.322 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.0040622999999999996, 2024-09-14T05:00:40.322 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0040853649999999997, 2024-09-14T05:00:40.322 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:00:40.322 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "started", 2024-09-14T05:00:40.322 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:00:40.322 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:40.322 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:00:40.322 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:40.316175+0000", 2024-09-14T05:00:40.322 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:40.322 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:40.322 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:40.322 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:00:40.322 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:40.316175+0000", 2024-09-14T05:00:40.322 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:00:40.323 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:40.323 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:40.323 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:00:40.323 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:40.316174+0000", 2024-09-14T05:00:40.323 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T05:00:40.323 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:40.323 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:40.323 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:00:40.323 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:40.316182+0000", 2024-09-14T05:00:40.323 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.7929999999999998e-06 2024-09-14T05:00:40.323 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:40.323 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:40.323 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:00:40.323 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:40.316196+0000", 2024-09-14T05:00:40.324 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3969e-05 2024-09-14T05:00:40.324 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:40.324 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:40.324 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:00:40.324 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:40.316202+0000", 2024-09-14T05:00:40.324 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.3300000000000004e-06 2024-09-14T05:00:40.324 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:40.324 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:40.324 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:00:40.324 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:40.316270+0000", 2024-09-14T05:00:40.324 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7386999999999997e-05 2024-09-14T05:00:40.324 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:00:40.324 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:00:40.324 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:00:40.324 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:40.316290+0000", 2024-09-14T05:00:40.325 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.9796000000000002e-05 2024-09-14T05:00:40.325 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:40.325 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:00:40.325 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:40.325 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:00:40.325 INFO:teuthology.orchestra.run.smithi129.stdout: ], 2024-09-14T05:00:40.325 INFO:teuthology.orchestra.run.smithi129.stdout: "num_ops": 1 2024-09-14T05:00:40.325 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T05:00:40.325 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:40.379 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:40.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:40.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:40.481 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:40.498 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:40 smithi039 ceph-mon[106470]: 1.14 scrub starts 2024-09-14T05:00:40.498 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:40 smithi039 ceph-mon[106470]: 1.14 scrub ok 2024-09-14T05:00:40.498 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:40 smithi039 ceph-mon[106470]: 1.b scrub starts 2024-09-14T05:00:40.498 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:40 smithi039 ceph-mon[106470]: 1.b scrub ok 2024-09-14T05:00:40.499 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:40 smithi039 ceph-mon[111235]: 1.14 scrub starts 2024-09-14T05:00:40.499 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:40 smithi039 ceph-mon[111235]: 1.14 scrub ok 2024-09-14T05:00:40.499 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:40 smithi039 ceph-mon[111235]: 1.b scrub starts 2024-09-14T05:00:40.499 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:40 smithi039 ceph-mon[111235]: 1.b scrub ok 2024-09-14T05:00:40.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:40.514 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:40.514 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:40.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:40 smithi129 ceph-mon[101316]: 1.14 scrub starts 2024-09-14T05:00:40.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:40 smithi129 ceph-mon[101316]: 1.14 scrub ok 2024-09-14T05:00:40.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:40 smithi129 ceph-mon[101316]: 1.b scrub starts 2024-09-14T05:00:40.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:40 smithi129 ceph-mon[101316]: 1.b scrub ok 2024-09-14T05:00:40.615 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:40.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:40.645 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:40.645 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:40.692 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:00:40.746 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:40.799 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:40.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:40.800 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:40.900 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:40.921 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:40.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:40.923 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:41.023 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:41.051 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-14T05:00:41.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:41.070 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:41.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:41.171 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:41.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:41.205 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:41.206 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:41.306 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:41.352 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:41.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:41.353 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:41.394 INFO:teuthology.orchestra.run.smithi039.stderr:set pool 57 pg_num to 26 2024-09-14T05:00:41.395 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is unset 2024-09-14T05:00:41.396 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:00:41.453 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:41.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:41.493 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:41.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:41.594 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:41.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:41.628 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:41.628 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:41.722 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:41 smithi129 ceph-mon[101316]: pgmap v3058: 109 pgs: 109 active+clean; 629 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2024-09-14T05:00:41.722 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:41 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/185796526' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:00:41.722 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:41 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:00:41.722 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:41 smithi129 ceph-mon[101316]: 1.11 scrub starts 2024-09-14T05:00:41.722 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:41 smithi129 ceph-mon[101316]: 1.11 scrub ok 2024-09-14T05:00:41.722 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:41 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/317786433' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"} : dispatch 2024-09-14T05:00:41.722 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:41 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"} : dispatch 2024-09-14T05:00:41.729 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:41.759 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:41.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:41.762 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:41.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:41 smithi039 ceph-mon[106470]: pgmap v3058: 109 pgs: 109 active+clean; 629 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2024-09-14T05:00:41.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:41 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/185796526' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:00:41.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:41 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:00:41.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:41 smithi039 ceph-mon[106470]: 1.11 scrub starts 2024-09-14T05:00:41.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:41 smithi039 ceph-mon[106470]: 1.11 scrub ok 2024-09-14T05:00:41.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:41 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/317786433' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"} : dispatch 2024-09-14T05:00:41.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:41 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"} : dispatch 2024-09-14T05:00:41.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:41 smithi039 ceph-mon[111235]: pgmap v3058: 109 pgs: 109 active+clean; 629 MiB data, 7.6 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 896 op/s 2024-09-14T05:00:41.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:41 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/185796526' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:00:41.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:41 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:00:41.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:41 smithi039 ceph-mon[111235]: 1.11 scrub starts 2024-09-14T05:00:41.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:41 smithi039 ceph-mon[111235]: 1.11 scrub ok 2024-09-14T05:00:41.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:41 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/317786433' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"} : dispatch 2024-09-14T05:00:41.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:41 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"} : dispatch 2024-09-14T05:00:41.862 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:41.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:41.903 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:41.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:42.004 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:42.047 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:42.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:42.048 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:42.149 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:42.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:42.184 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:42.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:42.285 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:42.322 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:42.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:42.323 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:42.423 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:42.450 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:42.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:42.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:42.552 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:42.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:42.613 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:42.613 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:42.714 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:42.729 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:42 smithi039 ceph-mon[106470]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T05:00:42.729 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:42 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T05:00:42.729 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:42 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T05:00:42.729 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:42 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2024-09-14T05:00:42.729 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:42 smithi039 ceph-mon[106470]: osdmap e1558: 8 total, 8 up, 8 in 2024-09-14T05:00:42.730 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:42 smithi039 ceph-mon[106470]: 1.c scrub starts 2024-09-14T05:00:42.730 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:42 smithi039 ceph-mon[106470]: 1.c scrub ok 2024-09-14T05:00:42.730 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:42 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-14T05:00:42.730 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:42 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-14T05:00:42.731 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:42 smithi039 ceph-mon[111235]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T05:00:42.731 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:42 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T05:00:42.731 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:42 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T05:00:42.731 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:42 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2024-09-14T05:00:42.731 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:42 smithi039 ceph-mon[111235]: osdmap e1558: 8 total, 8 up, 8 in 2024-09-14T05:00:42.732 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:42 smithi039 ceph-mon[111235]: 1.c scrub starts 2024-09-14T05:00:42.732 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:42 smithi039 ceph-mon[111235]: 1.c scrub ok 2024-09-14T05:00:42.732 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:42 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-14T05:00:42.732 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:42 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-14T05:00:42.756 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:42.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:42.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:42.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:42 smithi129 ceph-mon[101316]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T05:00:42.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:42 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T05:00:42.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:42 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T05:00:42.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:42 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num", "val": "26"}]': finished 2024-09-14T05:00:42.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:42 smithi129 ceph-mon[101316]: osdmap e1558: 8 total, 8 up, 8 in 2024-09-14T05:00:42.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:42 smithi129 ceph-mon[101316]: 1.c scrub starts 2024-09-14T05:00:42.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:42 smithi129 ceph-mon[101316]: 1.c scrub ok 2024-09-14T05:00:42.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:42 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-14T05:00:42.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:42 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"} : dispatch 2024-09-14T05:00:42.858 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:42.913 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:42.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:42.914 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:43.015 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:43.035 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:43.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:43.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:43.137 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:43.160 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:43.161 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:43.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:43.262 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:43.284 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:43.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:43.285 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:43.386 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:43.406 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:43.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:43.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:43.508 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:43.545 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:43.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:43.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:43.647 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:43.682 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:43.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:43.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:43.735 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:43 smithi129 ceph-mon[101316]: 1.7 scrub starts 2024-09-14T05:00:43.735 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:43 smithi129 ceph-mon[101316]: 1.7 scrub ok 2024-09-14T05:00:43.735 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:43 smithi129 ceph-mon[101316]: pgmap v3060: 109 pgs: 109 active+clean; 538 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2024-09-14T05:00:43.735 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:43 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-14T05:00:43.735 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:43 smithi129 ceph-mon[101316]: osdmap e1559: 8 total, 8 up, 8 in 2024-09-14T05:00:43.736 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:43 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T05:00:43.784 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:43.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:43 smithi039 ceph-mon[106470]: 1.7 scrub starts 2024-09-14T05:00:43.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:43 smithi039 ceph-mon[106470]: 1.7 scrub ok 2024-09-14T05:00:43.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:43 smithi039 ceph-mon[106470]: pgmap v3060: 109 pgs: 109 active+clean; 538 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2024-09-14T05:00:43.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:43 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-14T05:00:43.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:43 smithi039 ceph-mon[106470]: osdmap e1559: 8 total, 8 up, 8 in 2024-09-14T05:00:43.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:43 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T05:00:43.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:43 smithi039 ceph-mon[111235]: 1.7 scrub starts 2024-09-14T05:00:43.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:43 smithi039 ceph-mon[111235]: 1.7 scrub ok 2024-09-14T05:00:43.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:43 smithi039 ceph-mon[111235]: pgmap v3060: 109 pgs: 109 active+clean; 538 MiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 901 op/s 2024-09-14T05:00:43.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:43 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pg_num_actual", "val": "26"}]': finished 2024-09-14T05:00:43.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:43 smithi039 ceph-mon[111235]: osdmap e1559: 8 total, 8 up, 8 in 2024-09-14T05:00:43.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:43 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T05:00:43.813 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:43.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:43.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:43.915 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:43.949 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:43.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:43.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:44.051 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:44.082 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:44.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:44.083 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:44.184 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:44.230 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:44.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:44.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:44.332 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:44.354 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:44.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:44.355 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:44.455 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:44.494 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:44.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:44.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:44.596 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:44.637 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:44.638 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:44.638 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:44.739 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:44.744 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:44 smithi129 ceph-mon[101316]: osdmap e1560: 8 total, 8 up, 8 in 2024-09-14T05:00:44.744 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:44 smithi129 ceph-mon[101316]: 57.7 scrub starts 2024-09-14T05:00:44.744 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:44 smithi129 ceph-mon[101316]: 57.1 scrub starts 2024-09-14T05:00:44.744 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:44 smithi129 ceph-mon[101316]: 57.7 scrub ok 2024-09-14T05:00:44.744 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:44 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:00:44.744 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:44 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:00:44.744 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:44 smithi129 ceph-mon[101316]: 57.1 scrub ok 2024-09-14T05:00:44.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:44 smithi039 ceph-mon[106470]: osdmap e1560: 8 total, 8 up, 8 in 2024-09-14T05:00:44.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:44 smithi039 ceph-mon[106470]: 57.7 scrub starts 2024-09-14T05:00:44.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:44 smithi039 ceph-mon[106470]: 57.1 scrub starts 2024-09-14T05:00:44.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:44 smithi039 ceph-mon[106470]: 57.7 scrub ok 2024-09-14T05:00:44.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:44 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:00:44.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:44 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:00:44.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:44 smithi039 ceph-mon[106470]: 57.1 scrub ok 2024-09-14T05:00:44.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:44 smithi039 ceph-mon[111235]: osdmap e1560: 8 total, 8 up, 8 in 2024-09-14T05:00:44.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:44 smithi039 ceph-mon[111235]: 57.7 scrub starts 2024-09-14T05:00:44.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:44 smithi039 ceph-mon[111235]: 57.1 scrub starts 2024-09-14T05:00:44.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:44 smithi039 ceph-mon[111235]: 57.7 scrub ok 2024-09-14T05:00:44.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:44 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:00:44.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:44 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:00:44.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:44 smithi039 ceph-mon[111235]: 57.1 scrub ok 2024-09-14T05:00:44.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:44.801 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:44.801 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:44.902 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:44.934 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:44.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:44.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:45.036 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:45.069 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T05:00:45.069 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [], 2024-09-14T05:00:45.069 INFO:teuthology.orchestra.run.smithi129.stdout: "complaint_time": 30, 2024-09-14T05:00:45.069 INFO:teuthology.orchestra.run.smithi129.stdout: "num_blocked_ops": 0 2024-09-14T05:00:45.070 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T05:00:45.072 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:45.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:45.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:45.174 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:45.207 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:45.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:45.208 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:45.309 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:45.347 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:45.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:45.348 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:45.449 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:45.480 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:45.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:45.481 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:45.582 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:45.635 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:45.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:45.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:45.737 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:45.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:45 smithi129 ceph-mon[101316]: pgmap v3063: 119 pgs: 1 active, 4 peering, 9 unknown, 105 active+clean; 436 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2024-09-14T05:00:45.829 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:45 smithi129 ceph-mon[101316]: 57.8 deep-scrub starts 2024-09-14T05:00:45.830 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:45 smithi129 ceph-mon[101316]: 57.3 scrub starts 2024-09-14T05:00:45.830 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:45 smithi129 ceph-mon[101316]: 57.3 scrub ok 2024-09-14T05:00:45.830 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:45 smithi129 ceph-mon[101316]: 57.4 scrub starts 2024-09-14T05:00:45.830 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:45 smithi129 ceph-mon[101316]: 57.4 scrub ok 2024-09-14T05:00:45.830 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:45 smithi039 ceph-mon[106470]: pgmap v3063: 119 pgs: 1 active, 4 peering, 9 unknown, 105 active+clean; 436 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2024-09-14T05:00:45.831 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:45 smithi039 ceph-mon[106470]: 57.8 deep-scrub starts 2024-09-14T05:00:45.831 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:45 smithi039 ceph-mon[106470]: 57.3 scrub starts 2024-09-14T05:00:45.831 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:45 smithi039 ceph-mon[106470]: 57.3 scrub ok 2024-09-14T05:00:45.831 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:45 smithi039 ceph-mon[106470]: 57.4 scrub starts 2024-09-14T05:00:45.831 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:45 smithi039 ceph-mon[106470]: 57.4 scrub ok 2024-09-14T05:00:45.831 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:45 smithi039 ceph-mon[111235]: pgmap v3063: 119 pgs: 1 active, 4 peering, 9 unknown, 105 active+clean; 436 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 849 op/s 2024-09-14T05:00:45.831 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:45 smithi039 ceph-mon[111235]: 57.8 deep-scrub starts 2024-09-14T05:00:45.832 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:45 smithi039 ceph-mon[111235]: 57.3 scrub starts 2024-09-14T05:00:45.832 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:45 smithi039 ceph-mon[111235]: 57.3 scrub ok 2024-09-14T05:00:45.832 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:45 smithi039 ceph-mon[111235]: 57.4 scrub starts 2024-09-14T05:00:45.832 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:45 smithi039 ceph-mon[111235]: 57.4 scrub ok 2024-09-14T05:00:45.833 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-14T05:00:45.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:45.834 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:45.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:45.934 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:45.969 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:45.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:45.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:46.071 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:46.110 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:46.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:46.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:46.211 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:46.252 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:46.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:46.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:46.354 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:46.394 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:46.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:46.396 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:46.496 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:46.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:46.531 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:46.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:46.632 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:46.675 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:00:46.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:46.682 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:46.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:46.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:46 smithi129 ceph-mon[101316]: 57.8 deep-scrub ok 2024-09-14T05:00:46.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:46 smithi129 ceph-mon[101316]: 57.5 scrub starts 2024-09-14T05:00:46.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:46 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:00:46.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:46 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:00:46.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:46 smithi129 ceph-mon[101316]: 57.13 scrub starts 2024-09-14T05:00:46.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:46 smithi129 ceph-mon[101316]: 57.13 scrub ok 2024-09-14T05:00:46.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:46 smithi129 ceph-mon[101316]: 57.5 scrub ok 2024-09-14T05:00:46.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:46 smithi129 ceph-mon[101316]: 57.11 scrub starts 2024-09-14T05:00:46.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:46 smithi129 ceph-mon[101316]: 57.11 scrub ok 2024-09-14T05:00:46.783 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:46.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:46 smithi039 ceph-mon[106470]: 57.8 deep-scrub ok 2024-09-14T05:00:46.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:46 smithi039 ceph-mon[106470]: 57.5 scrub starts 2024-09-14T05:00:46.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:46 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:00:46.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:46 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:00:46.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:46 smithi039 ceph-mon[106470]: 57.13 scrub starts 2024-09-14T05:00:46.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:46 smithi039 ceph-mon[106470]: 57.13 scrub ok 2024-09-14T05:00:46.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:46 smithi039 ceph-mon[106470]: 57.5 scrub ok 2024-09-14T05:00:46.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:46 smithi039 ceph-mon[106470]: 57.11 scrub starts 2024-09-14T05:00:46.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:46 smithi039 ceph-mon[106470]: 57.11 scrub ok 2024-09-14T05:00:46.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:46 smithi039 ceph-mon[111235]: 57.8 deep-scrub ok 2024-09-14T05:00:46.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:46 smithi039 ceph-mon[111235]: 57.5 scrub starts 2024-09-14T05:00:46.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:46 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:00:46.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:46 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:00:46.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:46 smithi039 ceph-mon[111235]: 57.13 scrub starts 2024-09-14T05:00:46.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:46 smithi039 ceph-mon[111235]: 57.13 scrub ok 2024-09-14T05:00:46.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:46 smithi039 ceph-mon[111235]: 57.5 scrub ok 2024-09-14T05:00:46.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:46 smithi039 ceph-mon[111235]: 57.11 scrub starts 2024-09-14T05:00:46.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:46 smithi039 ceph-mon[111235]: 57.11 scrub ok 2024-09-14T05:00:46.813 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:46.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:46.814 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:46.915 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:46.946 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:46.947 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:46.947 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:47.048 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:47.083 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:47.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:47.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:47.191 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:47.239 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:47.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:47.240 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:47.341 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:47.376 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:47.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:47.377 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:47.478 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:47.510 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:47.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:47.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:47.612 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:47.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:47.647 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:47.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:47.748 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:47.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:47.785 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:47.785 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:47.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:47 smithi039 ceph-mon[106470]: pgmap v3064: 119 pgs: 1 active, 4 peering, 3 unknown, 111 active+clean; 309 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T05:00:47.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:47 smithi039 ceph-mon[106470]: 57.0 scrub starts 2024-09-14T05:00:47.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:47 smithi039 ceph-mon[106470]: 57.0 scrub ok 2024-09-14T05:00:47.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:47 smithi039 ceph-mon[106470]: 57.14 scrub starts 2024-09-14T05:00:47.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:47 smithi039 ceph-mon[106470]: 57.14 scrub ok 2024-09-14T05:00:47.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:47 smithi039 ceph-mon[106470]: 1.f scrub starts 2024-09-14T05:00:47.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:47 smithi039 ceph-mon[106470]: 1.f scrub ok 2024-09-14T05:00:47.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:47 smithi039 ceph-mon[111235]: pgmap v3064: 119 pgs: 1 active, 4 peering, 3 unknown, 111 active+clean; 309 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T05:00:47.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:47 smithi039 ceph-mon[111235]: 57.0 scrub starts 2024-09-14T05:00:47.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:47 smithi039 ceph-mon[111235]: 57.0 scrub ok 2024-09-14T05:00:47.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:47 smithi039 ceph-mon[111235]: 57.14 scrub starts 2024-09-14T05:00:47.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:47 smithi039 ceph-mon[111235]: 57.14 scrub ok 2024-09-14T05:00:47.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:47 smithi039 ceph-mon[111235]: 1.f scrub starts 2024-09-14T05:00:47.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:47 smithi039 ceph-mon[111235]: 1.f scrub ok 2024-09-14T05:00:47.807 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:47 smithi129 ceph-mon[101316]: pgmap v3064: 119 pgs: 1 active, 4 peering, 3 unknown, 111 active+clean; 309 MiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T05:00:47.807 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:47 smithi129 ceph-mon[101316]: 57.0 scrub starts 2024-09-14T05:00:47.807 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:47 smithi129 ceph-mon[101316]: 57.0 scrub ok 2024-09-14T05:00:47.807 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:47 smithi129 ceph-mon[101316]: 57.14 scrub starts 2024-09-14T05:00:47.807 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:47 smithi129 ceph-mon[101316]: 57.14 scrub ok 2024-09-14T05:00:47.807 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:47 smithi129 ceph-mon[101316]: 1.f scrub starts 2024-09-14T05:00:47.807 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:47 smithi129 ceph-mon[101316]: 1.f scrub ok 2024-09-14T05:00:47.886 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:47.926 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:47.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:47.927 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:48.028 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:48.082 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:48.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:48.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:48.184 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:48.228 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:48.229 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:48.230 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:48.331 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:48.376 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:48.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:48.377 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:48.426 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T05:00:48.427 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T05:00:48.455 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T05:00:48.455 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T05:00:48.478 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:48.490 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T05:00:48.490 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T05:00:48.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:48.517 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:48.517 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:48.529 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T05:00:48.529 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T05:00:48.561 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T05:00:48.562 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T05:00:48.598 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T05:00:48.598 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T05:00:48.599 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:48 smithi039 ceph-mon[106470]: 57.17 scrub starts 2024-09-14T05:00:48.599 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:48 smithi039 ceph-mon[106470]: 57.17 scrub ok 2024-09-14T05:00:48.599 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:48 smithi039 ceph-mon[106470]: 57.12 scrub starts 2024-09-14T05:00:48.599 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:48 smithi039 ceph-mon[106470]: 57.12 scrub ok 2024-09-14T05:00:48.599 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:48 smithi039 ceph-mon[106470]: 57.9 deep-scrub starts 2024-09-14T05:00:48.599 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:48 smithi039 ceph-mon[106470]: 57.9 deep-scrub ok 2024-09-14T05:00:48.599 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:48 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3173581968' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:00:48.599 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:48 smithi039 ceph-mon[111235]: 57.17 scrub starts 2024-09-14T05:00:48.599 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:48 smithi039 ceph-mon[111235]: 57.17 scrub ok 2024-09-14T05:00:48.600 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:48 smithi039 ceph-mon[111235]: 57.12 scrub starts 2024-09-14T05:00:48.600 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:48 smithi039 ceph-mon[111235]: 57.12 scrub ok 2024-09-14T05:00:48.600 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:48 smithi039 ceph-mon[111235]: 57.9 deep-scrub starts 2024-09-14T05:00:48.600 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:48 smithi039 ceph-mon[111235]: 57.9 deep-scrub ok 2024-09-14T05:00:48.600 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:48 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3173581968' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:00:48.618 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:48.636 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T05:00:48.636 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T05:00:48.645 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:48.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:48.646 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:48.650 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:48 smithi129 ceph-mon[101316]: 57.17 scrub starts 2024-09-14T05:00:48.650 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:48 smithi129 ceph-mon[101316]: 57.17 scrub ok 2024-09-14T05:00:48.650 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:48 smithi129 ceph-mon[101316]: 57.12 scrub starts 2024-09-14T05:00:48.650 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:48 smithi129 ceph-mon[101316]: 57.12 scrub ok 2024-09-14T05:00:48.650 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:48 smithi129 ceph-mon[101316]: 57.9 deep-scrub starts 2024-09-14T05:00:48.651 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:48 smithi129 ceph-mon[101316]: 57.9 deep-scrub ok 2024-09-14T05:00:48.651 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:48 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3173581968' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:00:48.672 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T05:00:48.672 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T05:00:48.703 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Removed 123749 objects 2024-09-14T05:00:48.703 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Clean up completed and total clean up time :138.981 2024-09-14T05:00:48.748 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:48.775 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:48.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:48.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:48.877 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:48.898 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:48.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:48.899 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:49.000 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:49.022 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:49.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:49.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:49.028 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:00:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:00:48] "GET /metrics HTTP/1.1" 200 36745 "" "Prometheus/2.51.0" 2024-09-14T05:00:49.118 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T05:00:49.124 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:49.146 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:49.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:49.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:49.248 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:49.269 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:49.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:49.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:49.342 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T05:00:49.342 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:00:49.371 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:49.393 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:49.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:49.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:49.495 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:49.516 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:49.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:49.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:49.559 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:00:49.618 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:49.640 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:49.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:49.641 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:49.680 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:49 smithi129 ceph-mon[101316]: pgmap v3065: 119 pgs: 1 active, 1 peering, 117 active+clean; 233 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T05:00:49.680 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:49 smithi129 ceph-mon[101316]: 57.10 scrub starts 2024-09-14T05:00:49.680 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:49 smithi129 ceph-mon[101316]: 57.2 scrub starts 2024-09-14T05:00:49.680 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:49 smithi129 ceph-mon[101316]: 57.2 scrub ok 2024-09-14T05:00:49.680 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:49 smithi129 ceph-mon[101316]: 57.10 scrub ok 2024-09-14T05:00:49.742 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:49.763 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:49.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:49.764 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:49.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:49 smithi039 ceph-mon[106470]: pgmap v3065: 119 pgs: 1 active, 1 peering, 117 active+clean; 233 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T05:00:49.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:49 smithi039 ceph-mon[106470]: 57.10 scrub starts 2024-09-14T05:00:49.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:49 smithi039 ceph-mon[106470]: 57.2 scrub starts 2024-09-14T05:00:49.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:49 smithi039 ceph-mon[106470]: 57.2 scrub ok 2024-09-14T05:00:49.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:49 smithi039 ceph-mon[106470]: 57.10 scrub ok 2024-09-14T05:00:49.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:49 smithi039 ceph-mon[111235]: pgmap v3065: 119 pgs: 1 active, 1 peering, 117 active+clean; 233 MiB data, 6.5 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T05:00:49.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:49 smithi039 ceph-mon[111235]: 57.10 scrub starts 2024-09-14T05:00:49.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:49 smithi039 ceph-mon[111235]: 57.2 scrub starts 2024-09-14T05:00:49.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:49 smithi039 ceph-mon[111235]: 57.2 scrub ok 2024-09-14T05:00:49.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:49 smithi039 ceph-mon[111235]: 57.10 scrub ok 2024-09-14T05:00:49.865 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:49.886 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:49.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:49.887 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:49.985 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-14T05:00:49.988 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:50.050 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:50.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:50.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:50.152 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:50.175 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:50.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:50.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:50.277 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:50.299 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:50.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:50.300 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:50.400 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:50.422 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:50.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:50.423 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:50.523 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:50.545 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:50.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:50.546 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:50.647 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:50.667 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:50 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-14T05:00:50.667 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:50 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-14T05:00:50.669 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:50.670 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:50.670 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:50.771 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:50.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:50 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-14T05:00:50.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:50 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-14T05:00:50.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:50 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-14T05:00:50.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:50 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"} : dispatch 2024-09-14T05:00:50.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:50.793 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:50.793 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:50.857 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:00:50.894 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:50.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:50.917 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:50.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:51.018 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:51.039 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:51.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:51.040 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:51.141 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:51.164 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:51.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:51.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:51.266 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:51.287 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:51.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:51.288 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:51.389 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:51.411 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:51.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:51.412 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:51.485 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2024-09-14T05:00:51.485 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2024-09-14T05:00:51.512 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:51.533 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:51.534 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:51.534 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:51.635 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:51.655 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:51.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:51.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:51.757 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:51.778 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:51.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:51.779 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:51.781 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:00:51.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:51 smithi039 ceph-mon[106470]: pgmap v3066: 119 pgs: 119 active+clean; 91 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T05:00:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:51 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-14T05:00:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:51 smithi039 ceph-mon[106470]: osdmap e1561: 8 total, 8 up, 8 in 2024-09-14T05:00:51.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:51 smithi039 ceph-mon[106470]: osdmap e1562: 8 total, 8 up, 8 in 2024-09-14T05:00:51.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:51 smithi039 ceph-mon[111235]: pgmap v3066: 119 pgs: 119 active+clean; 91 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T05:00:51.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:51 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-14T05:00:51.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:51 smithi039 ceph-mon[111235]: osdmap e1561: 8 total, 8 up, 8 in 2024-09-14T05:00:51.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:51 smithi039 ceph-mon[111235]: osdmap e1562: 8 total, 8 up, 8 in 2024-09-14T05:00:51.869 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:51 smithi129 ceph-mon[101316]: pgmap v3066: 119 pgs: 119 active+clean; 91 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T05:00:51.869 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:51 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "17"}]': finished 2024-09-14T05:00:51.869 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:51 smithi129 ceph-mon[101316]: osdmap e1561: 8 total, 8 up, 8 in 2024-09-14T05:00:51.869 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:51 smithi129 ceph-mon[101316]: osdmap e1562: 8 total, 8 up, 8 in 2024-09-14T05:00:51.880 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:51.903 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:51.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:51.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:51.988 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T05:00:52.004 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:52.057 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:52.058 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:52.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:52.159 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:52.181 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:52.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:52.182 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:52.236 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is set 2024-09-14T05:00:52.283 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:52.305 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:52.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:52.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:52.407 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:52.431 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:52.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:52.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:52.532 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:52.554 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:52.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:52.555 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:52.656 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:52.679 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:52.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:52.680 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:52.725 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-14T05:00:52.725 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T05:00:52.725 INFO:tasks.thrashosds.thrasher:primary_affinity 2024-09-14T05:00:52.725 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 1.000000 2024-09-14T05:00:52.725 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 2 1 2024-09-14T05:00:52.781 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:52.803 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:52.803 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:52.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:52.905 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:52.927 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:52.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:52.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:52.952 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:52 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/123360070' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:00:52.952 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:52 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:00:52.953 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-14T05:00:52.953 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:52 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-14T05:00:52.953 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:52 smithi039 ceph-mon[106470]: 57.19 deep-scrub starts 2024-09-14T05:00:52.953 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:52 smithi039 ceph-mon[106470]: 57.19 deep-scrub ok 2024-09-14T05:00:52.953 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:52 smithi039 ceph-mon[106470]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:00:52.953 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:52 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T05:00:52.953 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:52 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-14T05:00:52.953 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:52 smithi039 ceph-mon[106470]: osdmap e1563: 8 total, 8 up, 8 in 2024-09-14T05:00:52.953 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:00:52.953 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:52 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/123360070' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:00:52.954 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:52 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:00:52.954 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-14T05:00:52.954 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:52 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-14T05:00:52.954 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:52 smithi039 ceph-mon[111235]: 57.19 deep-scrub starts 2024-09-14T05:00:52.954 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:52 smithi039 ceph-mon[111235]: 57.19 deep-scrub ok 2024-09-14T05:00:52.954 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:52 smithi039 ceph-mon[111235]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:00:52.954 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:52 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T05:00:52.954 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:52 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-14T05:00:52.954 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:52 smithi039 ceph-mon[111235]: osdmap e1563: 8 total, 8 up, 8 in 2024-09-14T05:00:52.954 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:00:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:52 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/123360070' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:00:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:52 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:00:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-14T05:00:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:52 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"} : dispatch 2024-09-14T05:00:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:52 smithi129 ceph-mon[101316]: 57.19 deep-scrub starts 2024-09-14T05:00:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:52 smithi129 ceph-mon[101316]: 57.19 deep-scrub ok 2024-09-14T05:00:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:52 smithi129 ceph-mon[101316]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:00:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:52 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T05:00:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:52 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "18"}]': finished 2024-09-14T05:00:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:52 smithi129 ceph-mon[101316]: osdmap e1563: 8 total, 8 up, 8 in 2024-09-14T05:00:53.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:00:53.029 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:00:53.050 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:53.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:53.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:00:53.152 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:53.173 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:53.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:53.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:53.260 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T05:00:53.261 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:00:53.275 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:53.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:53.297 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:53.298 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:53.398 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:53.419 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:53.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:53.421 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:53.521 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:53.542 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:53.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:53.543 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:53.644 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:53.666 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:53.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:53.667 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:53.768 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:53.791 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:53.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:53.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:53.893 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:53.917 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:53.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:53.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:53.940 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:53 smithi129 ceph-mon[101316]: pgmap v3069: 119 pgs: 119 active+clean; 28 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T05:00:53.940 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:53 smithi129 ceph-mon[101316]: 57.10 starting backfill to osd.6 from (0'0,0'0] MAX to 1560'15212 2024-09-14T05:00:53.940 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:53 smithi129 ceph-mon[101316]: 57.10 starting backfill to osd.7 from (0'0,0'0] MAX to 1560'15212 2024-09-14T05:00:53.940 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:53 smithi129 ceph-mon[101316]: osdmap e1564: 8 total, 8 up, 8 in 2024-09-14T05:00:54.017 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:53 smithi039 ceph-mon[106470]: pgmap v3069: 119 pgs: 119 active+clean; 28 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T05:00:54.017 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:53 smithi039 ceph-mon[106470]: 57.10 starting backfill to osd.6 from (0'0,0'0] MAX to 1560'15212 2024-09-14T05:00:54.017 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:53 smithi039 ceph-mon[106470]: 57.10 starting backfill to osd.7 from (0'0,0'0] MAX to 1560'15212 2024-09-14T05:00:54.017 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:53 smithi039 ceph-mon[106470]: osdmap e1564: 8 total, 8 up, 8 in 2024-09-14T05:00:54.017 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:53 smithi039 ceph-mon[111235]: pgmap v3069: 119 pgs: 119 active+clean; 28 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T05:00:54.018 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:53 smithi039 ceph-mon[111235]: 57.10 starting backfill to osd.6 from (0'0,0'0] MAX to 1560'15212 2024-09-14T05:00:54.018 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:53 smithi039 ceph-mon[111235]: 57.10 starting backfill to osd.7 from (0'0,0'0] MAX to 1560'15212 2024-09-14T05:00:54.018 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:53 smithi039 ceph-mon[111235]: osdmap e1564: 8 total, 8 up, 8 in 2024-09-14T05:00:54.019 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:54.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:54.041 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:54.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:54.142 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:54.167 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:54.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:54.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:54.269 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:54.287 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-14T05:00:54.293 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:54.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:54.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:54.396 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:54.420 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:54.421 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:54.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:54.522 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:54.544 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:54.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:54.545 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:54.646 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:54.667 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:54.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:54.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:54.769 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:54.792 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:54.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:54.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:54.820 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:00:54.894 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:54.917 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:54.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:54.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:54.940 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:54 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-14T05:00:54.941 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:54 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-14T05:00:54.941 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:54 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-14T05:00:54.941 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:54 smithi129 ceph-mon[101316]: osdmap e1565: 8 total, 8 up, 8 in 2024-09-14T05:00:54.941 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:54 smithi129 ceph-mon[101316]: 57.11 starting backfill to osd.3 from (0'0,0'0] MAX to 1560'15569 2024-09-14T05:00:54.941 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:54 smithi129 ceph-mon[101316]: 57.11 starting backfill to osd.4 from (0'0,0'0] MAX to 1560'15569 2024-09-14T05:00:55.019 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:55.039 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:54 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-14T05:00:55.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:54 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-14T05:00:55.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:54 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-14T05:00:55.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:54 smithi039 ceph-mon[111235]: osdmap e1565: 8 total, 8 up, 8 in 2024-09-14T05:00:55.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:54 smithi039 ceph-mon[111235]: 57.11 starting backfill to osd.3 from (0'0,0'0] MAX to 1560'15569 2024-09-14T05:00:55.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:54 smithi039 ceph-mon[111235]: 57.11 starting backfill to osd.4 from (0'0,0'0] MAX to 1560'15569 2024-09-14T05:00:55.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:54 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-14T05:00:55.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:54 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"} : dispatch 2024-09-14T05:00:55.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:54 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num_actual", "val": "19"}]': finished 2024-09-14T05:00:55.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:54 smithi039 ceph-mon[106470]: osdmap e1565: 8 total, 8 up, 8 in 2024-09-14T05:00:55.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:54 smithi039 ceph-mon[106470]: 57.11 starting backfill to osd.3 from (0'0,0'0] MAX to 1560'15569 2024-09-14T05:00:55.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:54 smithi039 ceph-mon[106470]: 57.11 starting backfill to osd.4 from (0'0,0'0] MAX to 1560'15569 2024-09-14T05:00:55.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:55.043 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:55.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:55.144 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:55.165 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:55.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:55.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:55.228 INFO:teuthology.orchestra.run.smithi039.stderr:pool 'unique_pool_4' removed 2024-09-14T05:00:55.267 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:55.295 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:55.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:55.296 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:55.396 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:55.419 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:55.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:55.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:55.521 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:55.542 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:55.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:55.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:55.644 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:55.665 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:55.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:55.666 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:55.766 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:55.787 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:55.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:55.788 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:55.889 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:55.912 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:55.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:55.913 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:55.999 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:55 smithi039 ceph-mon[106470]: pgmap v3072: 119 pgs: 119 active+clean; 8.8 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:00:55.999 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:55 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4086731242' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T05:00:56.000 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:00:56.000 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:55 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:00:56.000 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:55 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4086731242' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T05:00:56.000 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:55 smithi039 ceph-mon[106470]: osdmap e1566: 8 total, 8 up, 8 in 2024-09-14T05:00:56.000 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:55 smithi039 ceph-mon[111235]: pgmap v3072: 119 pgs: 119 active+clean; 8.8 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:00:56.000 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:55 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4086731242' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T05:00:56.000 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:00:56.000 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:55 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:00:56.000 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:55 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4086731242' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T05:00:56.000 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:55 smithi039 ceph-mon[111235]: osdmap e1566: 8 total, 8 up, 8 in 2024-09-14T05:00:56.011 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:55 smithi129 ceph-mon[101316]: pgmap v3072: 119 pgs: 119 active+clean; 8.8 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:00:56.011 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:55 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4086731242' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T05:00:56.012 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:00:56.012 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:55 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:00:56.012 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:55 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4086731242' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T05:00:56.012 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:55 smithi129 ceph-mon[101316]: osdmap e1566: 8 total, 8 up, 8 in 2024-09-14T05:00:56.014 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:56.036 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:56.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:56.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:56.138 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:56.161 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:56.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:56.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:56.263 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:56.285 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:56.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:56.286 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:56.387 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:56.408 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:56.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:56.409 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:56.510 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:56.531 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:56.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:56.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:56.633 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:56.639 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T05:00:56.639 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:00:56.656 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:56.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:56.657 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:56.757 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:56.780 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:56.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:56.781 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:56.804 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2024-09-14T05:00:56.805 INFO:tasks.radosbench:Beginning radosbench... 2024-09-14T05:00:56.806 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2024-09-14T05:00:56.806 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd pool create unique_pool_5 16 2024-09-14T05:00:56.882 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:00:56.904 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:56.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:56.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:00:57.006 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:57.027 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:57.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:57.028 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:57.053 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:00:57.129 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:57.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:57.151 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:57.151 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:57.252 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:57.274 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:57.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:57.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:57.277 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-14T05:00:57.376 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:57.398 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:57.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:57.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:57.457 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:00:57.500 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:57.521 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:00:57.523 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:57.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:57.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:57.625 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:57.647 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:57.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:57.648 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:57.745 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T05:00:57.748 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:57.763 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:57 smithi039 ceph-mon[106470]: pgmap v3075: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:00:57.763 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:57 smithi039 ceph-mon[111235]: pgmap v3075: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:00:57.769 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:57.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:57.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:57.871 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:57.894 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:57.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:57.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:57.932 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:57 smithi129 ceph-mon[101316]: pgmap v3075: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:00:57.996 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:58.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:58.019 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:58.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:58.120 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:58.143 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:58.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:58.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:58.245 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:58.267 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:58.268 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:58.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:58.369 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:58.391 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:58.392 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:58.392 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:58.494 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:00:58.516 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:58.517 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:58.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:00:58.618 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:58.641 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:58.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:58.642 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:58.743 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:58.765 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:58.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:58.766 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:58.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:00:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:00:58] "GET /metrics HTTP/1.1" 200 36730 "" "Prometheus/2.51.0" 2024-09-14T05:00:58.867 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:58.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:58.889 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:58.890 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:58.990 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:59.012 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:59.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:59.013 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:59.114 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:59.136 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:59.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:59.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:59.238 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:00:59.251 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T05:00:59.251 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:00:59.259 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:59.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:59.261 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:00:59.362 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:00:59.384 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:59.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:59.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:00:59.486 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:00:59.510 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:59.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:59.511 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:00:59.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:59 smithi129 ceph-mon[101316]: pgmap v3076: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:00:59.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:00:59.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:00:59 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:00:59.581 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:00:59.581 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:00:59.611 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:00:59.634 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:00:59.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:59.635 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:00:59.680 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:59 smithi039 ceph-mon[106470]: pgmap v3076: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:00:59.680 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:00:59.680 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:00:59 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:00:59.681 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:59 smithi039 ceph-mon[111235]: pgmap v3076: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:00:59.681 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:00:59.681 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:00:59 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:00:59.736 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:59.757 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:59.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:59.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:59.858 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:00:59.879 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:00:59.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:00:59.880 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:00:59.981 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:00.003 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:00.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:00.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:00.009 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-14T05:01:00.105 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:00.126 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:00.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:00.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:00.213 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:01:00.228 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:00.250 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:00.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:00.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:00.352 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:00.373 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:00.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:00.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:00.475 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:00.499 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:00.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:00.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:00.600 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:00 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T05:01:00.600 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:00 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T05:01:00.600 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:00 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:01:00.600 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:00.623 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:00.624 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:00.624 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:00.701 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:00 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T05:01:00.701 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:00 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T05:01:00.701 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:00 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:01:00.702 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:00 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T05:01:00.702 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:00 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T05:01:00.702 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:00 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:01:00.725 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:00.747 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:00.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:00.748 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:00.849 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:00.871 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:00.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:00.872 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:00.972 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:00.994 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:00.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:00.995 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:01.096 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:01.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:01.118 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:01.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:01.219 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:01.234 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T05:01:01.234 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T05:01:01.240 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:01.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:01.242 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:01.251 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T05:01:01.251 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T05:01:01.267 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T05:01:01.267 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T05:01:01.283 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T05:01:01.284 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T05:01:01.300 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T05:01:01.300 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T05:01:01.317 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T05:01:01.317 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T05:01:01.333 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T05:01:01.333 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T05:01:01.342 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:01.350 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T05:01:01.350 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T05:01:01.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:01.368 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:01.368 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:01.469 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:01.491 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:01.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:01.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:01.593 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:01.615 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:01.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:01.616 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:01.702 INFO:teuthology.orchestra.run.smithi039.stderr:set osd.2 primary-affinity to 1 (8655362) 2024-09-14T05:01:01.717 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:01.740 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:01.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:01.742 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:01.842 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:01.865 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:01.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:01.866 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:01.907 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T05:01:01.907 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [], 2024-09-14T05:01:01.907 INFO:teuthology.orchestra.run.smithi129.stdout: "num_ops": 0 2024-09-14T05:01:01.907 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T05:01:01.907 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:01 smithi129 ceph-mon[101316]: pgmap v3077: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:01:01.908 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:01 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3477234225' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-14T05:01:01.908 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:01 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/831174602' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:01:01.966 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:01.988 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:01.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:01.989 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:02.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:01 smithi039 ceph-mon[106470]: pgmap v3077: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:01:02.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:01 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3477234225' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-14T05:01:02.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:01 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/831174602' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:01:02.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:01 smithi039 ceph-mon[111235]: pgmap v3077: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:01:02.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:01 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3477234225' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-14T05:01:02.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:01 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/831174602' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:01:02.090 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:02.113 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:02.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:02.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:02.215 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:02.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:02.239 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:02.239 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:02.340 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:02.362 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:02.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:02.363 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:02.464 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:02.485 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:02.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:02.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:02.583 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-14T05:01:02.587 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:02.647 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:02.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:02.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:02.749 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:02.771 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:02.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:02.772 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:02.782 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:01:02.873 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:02.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:02.896 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:02.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:02.902 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:02 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3477234225' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-14T05:01:02.902 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:02 smithi129 ceph-mon[101316]: osdmap e1567: 8 total, 8 up, 8 in 2024-09-14T05:01:02.997 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:03.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:03.020 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:03.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:03.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:02 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3477234225' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-14T05:01:03.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:02 smithi039 ceph-mon[106470]: osdmap e1567: 8 total, 8 up, 8 in 2024-09-14T05:01:03.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:02 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3477234225' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-14T05:01:03.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:02 smithi039 ceph-mon[111235]: osdmap e1567: 8 total, 8 up, 8 in 2024-09-14T05:01:03.121 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:03.143 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:03.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:03.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:03.245 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:03.266 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:03.267 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:03.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:03.368 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:03.389 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:03.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:03.390 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:03.491 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:03.512 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:03.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:03.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:03.614 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:03.636 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:03.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:03.637 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:03.737 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:03.759 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:03.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:03.760 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:03.860 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:03.884 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:03.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:03.885 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:03.902 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:03 smithi129 ceph-mon[101316]: pgmap v3079: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:01:03.986 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:03.992 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:03 smithi039 ceph-mon[106470]: pgmap v3079: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:01:03.992 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:03 smithi039 ceph-mon[111235]: pgmap v3079: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:01:04.007 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:04.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:04.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:04.109 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:04.130 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:04.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:04.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:04.232 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:04.260 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:04.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:04.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:04.362 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:04.387 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:04.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:04.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:04.456 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T05:01:04.457 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [], 2024-09-14T05:01:04.457 INFO:teuthology.orchestra.run.smithi129.stdout: "complaint_time": 30, 2024-09-14T05:01:04.457 INFO:teuthology.orchestra.run.smithi129.stdout: "num_blocked_ops": 0 2024-09-14T05:01:04.457 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T05:01:04.488 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:04.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:04.510 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:04.511 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:04.611 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:04.632 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:04.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:04.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:04.734 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:04.758 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:04.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:04.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:04.860 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:04.882 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:04.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:04.883 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:04.984 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:05.006 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:05.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:05.007 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:05.108 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:05.130 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:05.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:05.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:05.233 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:05.255 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:05.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:05.256 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:05.357 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:05.380 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:05.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:05.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:05.481 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:05.506 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:05.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:05.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:05.607 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:05.631 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:05.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:05.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:05.732 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:05.746 INFO:teuthology.orchestra.run.smithi039.stderr:pool 'unique_pool_5' created 2024-09-14T05:01:05.758 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:05.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:05.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:05.860 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:05.883 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:05.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:05.884 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:05.985 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:06.008 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:06.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:06.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:06.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:05 smithi129 ceph-mon[101316]: pgmap v3080: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:01:06.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:05 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2936305359' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-14T05:01:06.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:05 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-14T05:01:06.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:05 smithi039 ceph-mon[106470]: pgmap v3080: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:01:06.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:05 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2936305359' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-14T05:01:06.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:05 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-14T05:01:06.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:05 smithi039 ceph-mon[111235]: pgmap v3080: 93 pgs: 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:01:06.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:05 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2936305359' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-14T05:01:06.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:05 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16} : dispatch 2024-09-14T05:01:06.067 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-14T05:01:06.110 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:06.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:06.135 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:06.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:06.236 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:06.258 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:06.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:06.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:06.302 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:01:06.329 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T05:01:06.375 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:06.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:06.399 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:06.399 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:06.499 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:06.501 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2024-09-14T05:01:06.563 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:06.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:06.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:06.663 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:01:06.665 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:06.689 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:06.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:06.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:06.749 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is set 2024-09-14T05:01:06.790 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:06.812 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:06.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:06.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:06.914 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:06.918 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:06 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-09-14T05:01:06.918 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:06 smithi039 ceph-mon[106470]: osdmap e1568: 8 total, 8 up, 8 in 2024-09-14T05:01:06.919 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:06 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1859484951' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:01:06.919 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:06 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:01:06.919 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:06 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-09-14T05:01:06.919 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:06 smithi039 ceph-mon[111235]: osdmap e1568: 8 total, 8 up, 8 in 2024-09-14T05:01:06.919 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:06 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1859484951' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:01:06.919 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:06 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:01:06.936 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:06.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:06.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:07.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:06 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2024-09-14T05:01:07.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:06 smithi129 ceph-mon[101316]: osdmap e1568: 8 total, 8 up, 8 in 2024-09-14T05:01:07.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:06 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1859484951' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:01:07.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:06 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:01:07.037 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:07.059 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:07.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:07.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:07.161 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:07.185 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:07.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:07.186 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:07.287 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:07.310 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:07.310 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:07.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:07.411 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:07.433 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:07.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:07.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:07.535 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:07.556 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:07.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:07.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:07.658 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:07.680 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:07.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:07.681 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:07.782 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:07.805 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:07.806 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:07.807 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:07.823 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-14T05:01:07.823 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T05:01:07.857 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T05:01:07.858 INFO:teuthology.orchestra.run.smithi129.stdout: "size": 20, 2024-09-14T05:01:07.858 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 600, 2024-09-14T05:01:07.858 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [ 2024-09-14T05:01:07.858 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.858 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:35309 56.a 56:52365448:::benchmark_data_smithi039_330847_object35308:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T05:01:07.858 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:44.505360+0000", 2024-09-14T05:01:07.858 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 503.350122919, 2024-09-14T05:01:07.858 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.339575285, 2024-09-14T05:01:07.858 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:01:07.858 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:01:07.859 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:01:07.859 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:01:07.859 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:01:07.859 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 35309 2024-09-14T05:01:07.859 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.859 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:01:07.859 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.859 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:01:07.859 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.505360+0000", 2024-09-14T05:01:07.859 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.859 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.859 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.859 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:01:07.859 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.505360+0000", 2024-09-14T05:01:07.860 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.860 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.860 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.860 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:01:07.860 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.505358+0000", 2024-09-14T05:01:07.860 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T05:01:07.860 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.860 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.860 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:01:07.860 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.505422+0000", 2024-09-14T05:01:07.861 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.3194999999999994e-05 2024-09-14T05:01:07.861 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.861 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.861 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:01:07.861 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.505437+0000", 2024-09-14T05:01:07.861 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5418e-05 2024-09-14T05:01:07.861 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.861 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.861 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:01:07.861 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.505444+0000", 2024-09-14T05:01:07.861 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.0160000000000003e-06 2024-09-14T05:01:07.861 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.862 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.862 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:01:07.862 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.505717+0000", 2024-09-14T05:01:07.862 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00027316500000000002 2024-09-14T05:01:07.862 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.862 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.862 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:01:07.862 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.505826+0000", 2024-09-14T05:01:07.862 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010878300000000001 2024-09-14T05:01:07.862 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.862 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.862 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,6", 2024-09-14T05:01:07.862 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.505966+0000", 2024-09-14T05:01:07.862 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000139593 2024-09-14T05:01:07.863 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.863 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.863 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:01:07.863 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.507870+0000", 2024-09-14T05:01:07.863 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0019046569999999999 2024-09-14T05:01:07.863 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.863 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.863 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.863 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.510868+0000", 2024-09-14T05:01:07.863 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0029972900000000001 2024-09-14T05:01:07.863 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.863 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.863 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.864 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.844823+0000", 2024-09-14T05:01:07.864 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33395536100000001 2024-09-14T05:01:07.864 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.864 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.864 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:01:07.864 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.844882+0000", 2024-09-14T05:01:07.864 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.94e-05 2024-09-14T05:01:07.864 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.864 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.864 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:01:07.864 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.844935+0000", 2024-09-14T05:01:07.864 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2599000000000002e-05 2024-09-14T05:01:07.865 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.865 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:01:07.865 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.865 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.865 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.865 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:35319 56.a 56:5b9192f7:::benchmark_data_smithi039_330847_object35318:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T05:01:07.865 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:44.508851+0000", 2024-09-14T05:01:07.865 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 503.34663126700002, 2024-09-14T05:01:07.865 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.34003841600000001, 2024-09-14T05:01:07.865 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:01:07.865 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:01:07.865 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:01:07.865 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:01:07.865 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:01:07.866 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 35319 2024-09-14T05:01:07.866 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.866 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:01:07.866 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.866 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:01:07.866 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508851+0000", 2024-09-14T05:01:07.866 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.866 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.866 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.866 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:01:07.866 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508851+0000", 2024-09-14T05:01:07.866 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.866 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.866 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.867 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:01:07.867 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508850+0000", 2024-09-14T05:01:07.867 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T05:01:07.867 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.867 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.867 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:01:07.867 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508879+0000", 2024-09-14T05:01:07.867 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.9454999999999998e-05 2024-09-14T05:01:07.867 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.867 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.867 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:01:07.867 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508895+0000", 2024-09-14T05:01:07.867 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.6254000000000001e-05 2024-09-14T05:01:07.868 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.868 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.868 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:01:07.868 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508903+0000", 2024-09-14T05:01:07.868 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.4139999999999997e-06 2024-09-14T05:01:07.868 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.868 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.868 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:01:07.868 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.508953+0000", 2024-09-14T05:01:07.868 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.0311e-05 2024-09-14T05:01:07.868 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.868 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.868 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:01:07.869 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.509077+0000", 2024-09-14T05:01:07.869 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000124008 2024-09-14T05:01:07.869 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.869 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.869 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,6", 2024-09-14T05:01:07.869 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.509258+0000", 2024-09-14T05:01:07.869 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000180891 2024-09-14T05:01:07.869 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.869 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.869 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:01:07.869 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.510879+0000", 2024-09-14T05:01:07.869 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001620652 2024-09-14T05:01:07.869 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.869 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.870 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.870 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.512666+0000", 2024-09-14T05:01:07.870 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0017868820000000001 2024-09-14T05:01:07.870 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.870 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.870 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.870 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.848808+0000", 2024-09-14T05:01:07.870 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.336142777 2024-09-14T05:01:07.870 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.870 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.870 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:01:07.870 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.848854+0000", 2024-09-14T05:01:07.870 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6133e-05 2024-09-14T05:01:07.871 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.871 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.871 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:01:07.871 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.848890+0000", 2024-09-14T05:01:07.871 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5185999999999998e-05 2024-09-14T05:01:07.871 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.871 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:01:07.871 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.871 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.871 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.871 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:35323 56.a 56:5772dd4b:::benchmark_data_smithi039_330847_object35322:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T05:01:07.871 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:44.510844+0000", 2024-09-14T05:01:07.871 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 503.34463859700003, 2024-09-14T05:01:07.871 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33940646299999999, 2024-09-14T05:01:07.872 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:01:07.872 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:01:07.872 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:01:07.872 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:01:07.872 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:01:07.872 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 35323 2024-09-14T05:01:07.872 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.872 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:01:07.872 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.872 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:01:07.872 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.510844+0000", 2024-09-14T05:01:07.872 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.872 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.873 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.873 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:01:07.873 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.510844+0000", 2024-09-14T05:01:07.873 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.873 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.873 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.873 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:01:07.873 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.510843+0000", 2024-09-14T05:01:07.873 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T05:01:07.873 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.873 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.873 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:01:07.873 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.510948+0000", 2024-09-14T05:01:07.874 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000104839 2024-09-14T05:01:07.874 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.874 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.874 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:01:07.874 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.510963+0000", 2024-09-14T05:01:07.874 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5068999999999999e-05 2024-09-14T05:01:07.874 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.874 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.874 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:01:07.874 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.510970+0000", 2024-09-14T05:01:07.874 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.934e-06 2024-09-14T05:01:07.874 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.874 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.874 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:01:07.875 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.511079+0000", 2024-09-14T05:01:07.875 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010895000000000001 2024-09-14T05:01:07.875 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.875 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.875 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:01:07.875 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.511314+0000", 2024-09-14T05:01:07.875 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000235307 2024-09-14T05:01:07.875 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.875 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.875 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,6", 2024-09-14T05:01:07.875 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.511434+0000", 2024-09-14T05:01:07.875 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012017 2024-09-14T05:01:07.875 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.876 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.876 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:01:07.876 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.512769+0000", 2024-09-14T05:01:07.876 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00133442 2024-09-14T05:01:07.876 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.876 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.876 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.876 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.513589+0000", 2024-09-14T05:01:07.876 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00082001199999999995 2024-09-14T05:01:07.876 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.876 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.876 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.877 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.850170+0000", 2024-09-14T05:01:07.877 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33658163499999999 2024-09-14T05:01:07.877 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.877 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.877 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:01:07.877 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.850217+0000", 2024-09-14T05:01:07.877 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6789999999999998e-05 2024-09-14T05:01:07.877 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.877 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.878 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:01:07.878 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.850250+0000", 2024-09-14T05:01:07.878 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3402999999999999e-05 2024-09-14T05:01:07.878 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.878 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:01:07.878 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.878 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.878 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.878 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:35335 56.a 56:5cca2bfd:::benchmark_data_smithi039_330847_object35334:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T05:01:07.878 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:44.515184+0000", 2024-09-14T05:01:07.878 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 503.34029833699998, 2024-09-14T05:01:07.878 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33515958000000001, 2024-09-14T05:01:07.878 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:01:07.879 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:01:07.879 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:01:07.879 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:01:07.879 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:01:07.879 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 35335 2024-09-14T05:01:07.879 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.879 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:01:07.879 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.879 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:01:07.879 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.515184+0000", 2024-09-14T05:01:07.879 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.879 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.879 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.880 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:01:07.880 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.515184+0000", 2024-09-14T05:01:07.880 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.880 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.880 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.880 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:01:07.880 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.515183+0000", 2024-09-14T05:01:07.880 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T05:01:07.880 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.880 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.880 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:01:07.881 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.515220+0000", 2024-09-14T05:01:07.881 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6409999999999999e-05 2024-09-14T05:01:07.881 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.881 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.881 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:01:07.881 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.515232+0000", 2024-09-14T05:01:07.881 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2680000000000001e-05 2024-09-14T05:01:07.881 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.881 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.881 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:01:07.881 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.515239+0000", 2024-09-14T05:01:07.881 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.9569999999999996e-06 2024-09-14T05:01:07.882 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.882 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.882 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:01:07.882 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.515349+0000", 2024-09-14T05:01:07.882 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010991 2024-09-14T05:01:07.882 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.882 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.882 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:01:07.882 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.515460+0000", 2024-09-14T05:01:07.882 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000110398 2024-09-14T05:01:07.882 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.882 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.882 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,6", 2024-09-14T05:01:07.882 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.515600+0000", 2024-09-14T05:01:07.883 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013996700000000001 2024-09-14T05:01:07.883 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.883 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.883 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:01:07.883 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.517100+0000", 2024-09-14T05:01:07.883 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00150004 2024-09-14T05:01:07.883 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.883 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.883 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.883 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.520249+0000", 2024-09-14T05:01:07.883 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.003149584 2024-09-14T05:01:07.883 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.884 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.884 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.884 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.850283+0000", 2024-09-14T05:01:07.884 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33003364499999999 2024-09-14T05:01:07.884 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.884 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.884 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:01:07.884 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.850316+0000", 2024-09-14T05:01:07.884 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3037000000000002e-05 2024-09-14T05:01:07.884 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.884 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.884 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:01:07.884 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.850344+0000", 2024-09-14T05:01:07.885 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.7773e-05 2024-09-14T05:01:07.885 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.885 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:01:07.885 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.885 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.885 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.885 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:35341 56.a 56:5b382216:::benchmark_data_smithi039_330847_object35340:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1474)", 2024-09-14T05:01:07.885 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:52:44.518706+0000", 2024-09-14T05:01:07.885 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 503.33677697600001, 2024-09-14T05:01:07.885 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.33338658300000001, 2024-09-14T05:01:07.885 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:01:07.885 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:01:07.885 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:01:07.886 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:01:07.886 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:01:07.886 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 35341 2024-09-14T05:01:07.886 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.886 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:01:07.886 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.886 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:01:07.886 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.518706+0000", 2024-09-14T05:01:07.886 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.886 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.886 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.886 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:01:07.887 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.518706+0000", 2024-09-14T05:01:07.887 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.887 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.887 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.887 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:01:07.887 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.518704+0000", 2024-09-14T05:01:07.887 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T05:01:07.887 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.887 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.888 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:01:07.888 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.518744+0000", 2024-09-14T05:01:07.888 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9659000000000001e-05 2024-09-14T05:01:07.888 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.888 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.888 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:01:07.888 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.518757+0000", 2024-09-14T05:01:07.888 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.358e-05 2024-09-14T05:01:07.888 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.888 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.889 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:01:07.889 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.518765+0000", 2024-09-14T05:01:07.889 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.2080000000000003e-06 2024-09-14T05:01:07.889 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.889 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.890 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:01:07.890 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.518837+0000", 2024-09-14T05:01:07.890 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.2052999999999996e-05 2024-09-14T05:01:07.890 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.890 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.890 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:01:07.890 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.518956+0000", 2024-09-14T05:01:07.890 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000119437 2024-09-14T05:01:07.890 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.890 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.890 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,6", 2024-09-14T05:01:07.891 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.519095+0000", 2024-09-14T05:01:07.891 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013848500000000001 2024-09-14T05:01:07.891 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.891 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.891 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:01:07.891 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.520369+0000", 2024-09-14T05:01:07.891 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001274154 2024-09-14T05:01:07.891 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.891 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.891 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.891 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.521247+0000", 2024-09-14T05:01:07.891 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00087840900000000003 2024-09-14T05:01:07.891 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.891 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.892 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.892 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.852014+0000", 2024-09-14T05:01:07.892 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.330766387 2024-09-14T05:01:07.892 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.892 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.892 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:01:07.892 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.852055+0000", 2024-09-14T05:01:07.892 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1093000000000001e-05 2024-09-14T05:01:07.892 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.892 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.892 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:01:07.892 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:52:44.852092+0000", 2024-09-14T05:01:07.892 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7452e-05 2024-09-14T05:01:07.892 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.893 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:01:07.893 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.893 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.893 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.893 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:116251 56.b 56:d0b5012a:::benchmark_data_smithi039_330847_object1727:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1486)", 2024-09-14T05:01:07.893 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:04.547766+0000", 2024-09-14T05:01:07.893 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 423.30771635399998, 2024-09-14T05:01:07.893 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.488460691, 2024-09-14T05:01:07.893 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:01:07.893 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:01:07.893 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:01:07.893 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:01:07.893 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:01:07.894 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 116251 2024-09-14T05:01:07.894 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.894 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:01:07.894 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.894 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:01:07.894 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.547766+0000", 2024-09-14T05:01:07.894 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.894 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.894 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.894 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:01:07.894 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.547766+0000", 2024-09-14T05:01:07.894 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.894 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.894 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.895 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:01:07.895 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.547765+0000", 2024-09-14T05:01:07.895 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T05:01:07.895 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.895 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.895 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:01:07.895 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.547771+0000", 2024-09-14T05:01:07.895 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.7209999999999999e-06 2024-09-14T05:01:07.895 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.895 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.895 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:01:07.895 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.547779+0000", 2024-09-14T05:01:07.895 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.7020000000000007e-06 2024-09-14T05:01:07.896 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.896 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.896 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:01:07.896 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.547785+0000", 2024-09-14T05:01:07.896 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9039999999999997e-06 2024-09-14T05:01:07.896 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.896 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.896 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:01:07.896 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.551793+0000", 2024-09-14T05:01:07.896 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0040079240000000004 2024-09-14T05:01:07.896 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.896 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.896 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:01:07.897 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.551900+0000", 2024-09-14T05:01:07.897 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000107318 2024-09-14T05:01:07.897 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.897 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.897 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,5", 2024-09-14T05:01:07.897 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.552013+0000", 2024-09-14T05:01:07.897 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000112469 2024-09-14T05:01:07.897 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.897 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.897 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:01:07.897 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.555580+0000", 2024-09-14T05:01:07.897 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0035672360000000001 2024-09-14T05:01:07.897 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.898 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.898 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.898 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.558497+0000", 2024-09-14T05:01:07.899 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0029165189999999998 2024-09-14T05:01:07.899 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.899 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.899 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.899 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036142+0000", 2024-09-14T05:01:07.899 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.47764563300000001 2024-09-14T05:01:07.899 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.899 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.899 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:01:07.900 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036189+0000", 2024-09-14T05:01:07.900 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7036999999999997e-05 2024-09-14T05:01:07.900 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.900 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.900 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:01:07.900 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036227+0000", 2024-09-14T05:01:07.900 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7576000000000001e-05 2024-09-14T05:01:07.900 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.900 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:01:07.900 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.900 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.900 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.901 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:116268 56.b 56:de88060a:::benchmark_data_smithi039_330847_object1744:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1486)", 2024-09-14T05:01:07.901 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:04.637502+0000", 2024-09-14T05:01:07.901 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 423.21798077300002, 2024-09-14T05:01:07.901 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.39881927299999997, 2024-09-14T05:01:07.901 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:01:07.901 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:01:07.901 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:01:07.901 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:01:07.901 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:01:07.901 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 116268 2024-09-14T05:01:07.901 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.901 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:01:07.902 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.902 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:01:07.902 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.637502+0000", 2024-09-14T05:01:07.902 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.902 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.902 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.902 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:01:07.902 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.637502+0000", 2024-09-14T05:01:07.902 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.902 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.902 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.902 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:01:07.902 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.637501+0000", 2024-09-14T05:01:07.902 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T05:01:07.903 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.903 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.903 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:01:07.903 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.637505+0000", 2024-09-14T05:01:07.903 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7349999999999998e-06 2024-09-14T05:01:07.903 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.903 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.903 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:01:07.903 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.637515+0000", 2024-09-14T05:01:07.903 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0519e-05 2024-09-14T05:01:07.903 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.903 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.903 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:01:07.903 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.637522+0000", 2024-09-14T05:01:07.903 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.6429999999999998e-06 2024-09-14T05:01:07.904 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.904 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.904 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:01:07.904 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.637562+0000", 2024-09-14T05:01:07.904 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:07 smithi129 ceph-mon[101316]: pgmap v3082: 109 pgs: 16 unknown, 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:01:07.904 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:07 smithi129 ceph-mon[101316]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T05:01:07.904 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:07 smithi129 ceph-mon[101316]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:01:07.904 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:07 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T05:01:07.904 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:07 smithi129 ceph-mon[101316]: osdmap e1569: 8 total, 8 up, 8 in 2024-09-14T05:01:07.905 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:01:07.905 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:01:07.906 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0305999999999997e-05 2024-09-14T05:01:07.906 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.906 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.906 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:01:07.906 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.637689+0000", 2024-09-14T05:01:07.906 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012666000000000001 2024-09-14T05:01:07.906 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.906 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.907 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,5", 2024-09-14T05:01:07.907 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.637820+0000", 2024-09-14T05:01:07.907 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000130877 2024-09-14T05:01:07.907 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.907 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.907 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:01:07.907 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.646403+0000", 2024-09-14T05:01:07.907 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0085835389999999994 2024-09-14T05:01:07.907 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.907 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.907 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.908 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.649755+0000", 2024-09-14T05:01:07.908 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0033511610000000001 2024-09-14T05:01:07.908 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.908 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.908 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.908 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036263+0000", 2024-09-14T05:01:07.908 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.38650827100000001 2024-09-14T05:01:07.908 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.908 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.908 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:01:07.908 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036294+0000", 2024-09-14T05:01:07.908 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.1164000000000002e-05 2024-09-14T05:01:07.908 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.909 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.909 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:01:07.909 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036321+0000", 2024-09-14T05:01:07.909 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.7030999999999999e-05 2024-09-14T05:01:07.909 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.909 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:01:07.909 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.909 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.909 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.909 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:116284 56.b 56:d7432e9a:::benchmark_data_smithi039_330847_object1760:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1486)", 2024-09-14T05:01:07.909 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:04.658571+0000", 2024-09-14T05:01:07.909 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 423.19691156900001, 2024-09-14T05:01:07.909 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.37787099099999999, 2024-09-14T05:01:07.910 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:01:07.910 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:01:07.910 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:01:07.910 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:01:07.910 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:01:07.910 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 116284 2024-09-14T05:01:07.910 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.911 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:01:07.911 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.911 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:01:07.911 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658571+0000", 2024-09-14T05:01:07.911 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.912 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.912 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.912 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:01:07.912 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658571+0000", 2024-09-14T05:01:07.912 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.912 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.912 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.912 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:01:07.912 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658569+0000", 2024-09-14T05:01:07.912 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T05:01:07.912 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.913 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.913 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:01:07.913 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658577+0000", 2024-09-14T05:01:07.913 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.7889999999999995e-06 2024-09-14T05:01:07.913 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.914 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.914 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:01:07.914 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658592+0000", 2024-09-14T05:01:07.914 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5287000000000001e-05 2024-09-14T05:01:07.914 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.914 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.914 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:01:07.914 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658600+0000", 2024-09-14T05:01:07.914 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.6240000000000002e-06 2024-09-14T05:01:07.914 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.914 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.914 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:01:07.915 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658647+0000", 2024-09-14T05:01:07.915 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6965000000000002e-05 2024-09-14T05:01:07.915 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.915 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.915 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:01:07.915 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658748+0000", 2024-09-14T05:01:07.915 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000101005 2024-09-14T05:01:07.915 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.915 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.915 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,5", 2024-09-14T05:01:07.915 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.658842+0000", 2024-09-14T05:01:07.915 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.4204999999999997e-05 2024-09-14T05:01:07.915 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.915 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.916 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:01:07.916 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.662555+0000", 2024-09-14T05:01:07.916 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0037132350000000001 2024-09-14T05:01:07.916 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.916 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.916 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.916 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.663869+0000", 2024-09-14T05:01:07.916 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001313952 2024-09-14T05:01:07.916 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.916 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.916 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.916 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036369+0000", 2024-09-14T05:01:07.916 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.37249975600000002 2024-09-14T05:01:07.916 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.916 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.917 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:01:07.917 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036410+0000", 2024-09-14T05:01:07.917 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0692000000000003e-05 2024-09-14T05:01:07.917 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.917 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.917 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:01:07.917 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036442+0000", 2024-09-14T05:01:07.917 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.2181000000000002e-05 2024-09-14T05:01:07.917 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.917 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:01:07.917 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.918 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.918 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.918 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:116292 56.4 56:2d450bf4:::benchmark_data_smithi039_330847_object1768:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1486)", 2024-09-14T05:01:07.918 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:04.670078+0000", 2024-09-14T05:01:07.918 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 423.185404441, 2024-09-14T05:01:07.918 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.374447525, 2024-09-14T05:01:07.918 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:01:07.918 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:01:07.918 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:01:07.918 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:01:07.918 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:01:07.918 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 116292 2024-09-14T05:01:07.918 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.919 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:01:07.919 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.919 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:01:07.919 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670078+0000", 2024-09-14T05:01:07.919 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.919 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.919 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.919 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:01:07.919 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670078+0000", 2024-09-14T05:01:07.919 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.919 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.919 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.919 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:01:07.919 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670077+0000", 2024-09-14T05:01:07.920 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T05:01:07.920 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.920 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.920 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:01:07.920 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670082+0000", 2024-09-14T05:01:07.920 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8439999999999997e-06 2024-09-14T05:01:07.920 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.920 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.920 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:01:07.920 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670090+0000", 2024-09-14T05:01:07.920 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.6989999999999992e-06 2024-09-14T05:01:07.920 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.920 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.920 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:01:07.921 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670097+0000", 2024-09-14T05:01:07.921 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.1589999999999998e-06 2024-09-14T05:01:07.921 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.921 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.921 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:01:07.921 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670207+0000", 2024-09-14T05:01:07.921 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011047100000000001 2024-09-14T05:01:07.921 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.921 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.921 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:01:07.922 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670315+0000", 2024-09-14T05:01:07.922 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010752299999999999 2024-09-14T05:01:07.922 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.922 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.922 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,2", 2024-09-14T05:01:07.922 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036104+0000", 2024-09-14T05:01:07.922 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.36578902299999999 2024-09-14T05:01:07.922 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.922 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.922 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:01:07.922 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.040066+0000", 2024-09-14T05:01:07.922 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0039619390000000003 2024-09-14T05:01:07.922 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.922 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.923 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.923 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.044355+0000", 2024-09-14T05:01:07.923 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0042896779999999999 2024-09-14T05:01:07.923 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.923 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.923 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.923 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.044433+0000", 2024-09-14T05:01:07.923 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.7905999999999996e-05 2024-09-14T05:01:07.923 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.923 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.923 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:01:07.923 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.044483+0000", 2024-09-14T05:01:07.923 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.0207000000000002e-05 2024-09-14T05:01:07.923 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.923 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.924 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:01:07.924 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.044526+0000", 2024-09-14T05:01:07.924 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.2184999999999997e-05 2024-09-14T05:01:07.924 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.924 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:01:07.924 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.924 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.924 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.924 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.36012.0:116283 56.6 e1486/1476 56:68220df0:::benchmark_data_smithi039_330847_object1759:head v 1486'7117, mlcod=0'0)", 2024-09-14T05:01:07.924 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:04.670105+0000", 2024-09-14T05:01:07.924 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 423.18537714899998, 2024-09-14T05:01:07.924 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.37733796400000003, 2024-09-14T05:01:07.924 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:01:07.924 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:01:07.924 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:01:07.925 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.925 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:01:07.925 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670105+0000", 2024-09-14T05:01:07.925 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.925 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.925 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.925 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:01:07.925 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670105+0000", 2024-09-14T05:01:07.925 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.925 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.925 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.925 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:01:07.925 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670105+0000", 2024-09-14T05:01:07.925 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T05:01:07.926 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.926 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.926 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:01:07.926 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670111+0000", 2024-09-14T05:01:07.926 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.02e-06 2024-09-14T05:01:07.926 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.926 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.926 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:01:07.926 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670118+0000", 2024-09-14T05:01:07.926 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.661e-06 2024-09-14T05:01:07.926 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.926 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.926 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:01:07.926 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670124+0000", 2024-09-14T05:01:07.926 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.3539999999999999e-06 2024-09-14T05:01:07.926 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.927 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.927 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:01:07.927 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670163+0000", 2024-09-14T05:01:07.927 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9483999999999997e-05 2024-09-14T05:01:07.927 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.927 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.927 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:01:07.927 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670186+0000", 2024-09-14T05:01:07.927 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.2918999999999999e-05 2024-09-14T05:01:07.927 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.927 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.928 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:01:07.928 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.047411+0000", 2024-09-14T05:01:07.928 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.37722491000000002 2024-09-14T05:01:07.928 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.928 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.928 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:01:07.928 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.047443+0000", 2024-09-14T05:01:07.928 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.2203000000000003e-05 2024-09-14T05:01:07.928 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.928 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:01:07.928 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.928 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.928 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.929 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:116294 56.4 56:2b7b2e82:::benchmark_data_smithi039_330847_object1770:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1486)", 2024-09-14T05:01:07.929 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:04.670327+0000", 2024-09-14T05:01:07.929 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 423.18515512499999, 2024-09-14T05:01:07.929 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.377658365, 2024-09-14T05:01:07.929 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:01:07.929 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:01:07.929 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:01:07.929 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:01:07.929 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:01:07.929 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 116294 2024-09-14T05:01:07.929 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.929 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:01:07.929 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.930 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:01:07.930 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670327+0000", 2024-09-14T05:01:07.930 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.930 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.930 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.930 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:01:07.930 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670327+0000", 2024-09-14T05:01:07.930 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.931 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.931 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.931 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:01:07.931 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670327+0000", 2024-09-14T05:01:07.931 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T05:01:07.931 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.931 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.931 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:01:07.932 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.670331+0000", 2024-09-14T05:01:07.932 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.048e-06 2024-09-14T05:01:07.932 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.932 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.932 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:01:07.932 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036115+0000", 2024-09-14T05:01:07.932 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.36578455700000001 2024-09-14T05:01:07.932 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.932 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.932 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:01:07.932 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036124+0000", 2024-09-14T05:01:07.932 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.9420000000000001e-06 2024-09-14T05:01:07.933 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.933 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.933 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:01:07.933 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.040091+0000", 2024-09-14T05:01:07.933 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0039670720000000003 2024-09-14T05:01:07.933 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.934 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.934 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:01:07.934 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.040277+0000", 2024-09-14T05:01:07.934 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000185954 2024-09-14T05:01:07.934 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.934 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.934 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,2", 2024-09-14T05:01:07.934 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.040378+0000", 2024-09-14T05:01:07.934 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010124300000000001 2024-09-14T05:01:07.934 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.934 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.934 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.934 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.044582+0000", 2024-09-14T05:01:07.934 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0042034259999999997 2024-09-14T05:01:07.935 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.935 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.935 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.935 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.044612+0000", 2024-09-14T05:01:07.935 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.0540999999999998e-05 2024-09-14T05:01:07.935 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.935 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.935 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:01:07.935 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.047897+0000", 2024-09-14T05:01:07.935 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.003284281 2024-09-14T05:01:07.935 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.935 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.935 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:01:07.935 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.047948+0000", 2024-09-14T05:01:07.935 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.1749000000000002e-05 2024-09-14T05:01:07.936 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.936 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.936 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:01:07.936 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.047986+0000", 2024-09-14T05:01:07.936 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7357999999999997e-05 2024-09-14T05:01:07.936 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.936 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:01:07.936 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.936 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.936 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.936 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop_reply(client.36012.0:116251 56.b e1486/1470 ondisk, result = 0)", 2024-09-14T05:01:07.936 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:04.672410+0000", 2024-09-14T05:01:07.936 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 423.18307217, 2024-09-14T05:01:07.936 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.36382398599999999, 2024-09-14T05:01:07.936 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:01:07.937 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "started", 2024-09-14T05:01:07.937 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:01:07.937 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.937 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:01:07.937 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.672410+0000", 2024-09-14T05:01:07.937 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.937 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.937 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.937 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:01:07.937 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.672410+0000", 2024-09-14T05:01:07.937 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.937 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.937 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.938 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:01:07.938 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.672409+0000", 2024-09-14T05:01:07.938 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T05:01:07.938 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.938 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.938 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:01:07.938 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.672416+0000", 2024-09-14T05:01:07.938 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.0160000000000003e-06 2024-09-14T05:01:07.938 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.938 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.938 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:01:07.938 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036080+0000", 2024-09-14T05:01:07.938 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.363663499 2024-09-14T05:01:07.938 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.939 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.939 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:01:07.939 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036088+0000", 2024-09-14T05:01:07.939 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.8250000000000005e-06 2024-09-14T05:01:07.939 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.939 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.939 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:01:07.939 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036126+0000", 2024-09-14T05:01:07.939 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8379e-05 2024-09-14T05:01:07.939 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.939 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.939 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:01:07.939 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036136+0000", 2024-09-14T05:01:07.939 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.6330000000000008e-06 2024-09-14T05:01:07.940 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.940 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.940 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:01:07.940 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036234+0000", 2024-09-14T05:01:07.940 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.8673999999999997e-05 2024-09-14T05:01:07.940 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.940 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:01:07.940 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.940 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.940 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.940 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.36012.0:116293 56.5 e1486/1470 56:a6762fc1:::benchmark_data_smithi039_330847_object1769:head v 1486'7278, mlcod=1486'7276)", 2024-09-14T05:01:07.940 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:54:04.673699+0000", 2024-09-14T05:01:07.940 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 423.181783107, 2024-09-14T05:01:07.941 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.37533592999999998, 2024-09-14T05:01:07.941 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:01:07.941 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:01:07.941 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:01:07.941 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.941 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:01:07.941 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.673699+0000", 2024-09-14T05:01:07.941 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.941 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.941 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.941 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:01:07.941 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.673699+0000", 2024-09-14T05:01:07.941 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.942 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.942 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.942 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:01:07.942 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.673699+0000", 2024-09-14T05:01:07.942 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T05:01:07.942 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.942 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.942 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:01:07.942 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:04.673707+0000", 2024-09-14T05:01:07.942 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.5599999999999994e-06 2024-09-14T05:01:07.942 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.942 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.942 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:01:07.943 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.035992+0000", 2024-09-14T05:01:07.943 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.36228451699999997 2024-09-14T05:01:07.943 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.943 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.943 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:01:07.943 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036003+0000", 2024-09-14T05:01:07.943 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0883999999999999e-05 2024-09-14T05:01:07.943 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.943 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.943 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:01:07.943 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036051+0000", 2024-09-14T05:01:07.943 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8485000000000001e-05 2024-09-14T05:01:07.943 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.943 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.943 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:01:07.944 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.036084+0000", 2024-09-14T05:01:07.944 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3222999999999997e-05 2024-09-14T05:01:07.944 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.944 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.944 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:01:07.944 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.049003+0000", 2024-09-14T05:01:07.944 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.012919058000000001 2024-09-14T05:01:07.944 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.944 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.944 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:01:07.944 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:54:05.049035+0000", 2024-09-14T05:01:07.944 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.2027999999999999e-05 2024-09-14T05:01:07.944 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.944 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:01:07.945 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.945 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.945 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.945 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:172819 56.a 56:59afb196:::benchmark_data_smithi039_330847_object58295:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1508)", 2024-09-14T05:01:07.945 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:55:16.726954+0000", 2024-09-14T05:01:07.945 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 351.12852809899999, 2024-09-14T05:01:07.945 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.207837457, 2024-09-14T05:01:07.945 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:01:07.945 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:01:07.945 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:01:07.945 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:01:07.945 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:01:07.945 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 172819 2024-09-14T05:01:07.945 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.946 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:01:07.946 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.946 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:01:07.946 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.726954+0000", 2024-09-14T05:01:07.946 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.946 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.946 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.946 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:01:07.946 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.726954+0000", 2024-09-14T05:01:07.946 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.946 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.946 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.946 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:01:07.946 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.726953+0000", 2024-09-14T05:01:07.946 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T05:01:07.946 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.947 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.947 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:01:07.947 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.726958+0000", 2024-09-14T05:01:07.947 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7729999999999999e-06 2024-09-14T05:01:07.947 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.947 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.947 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:01:07.947 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.726966+0000", 2024-09-14T05:01:07.947 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.1009999999999992e-06 2024-09-14T05:01:07.947 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.947 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.947 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:01:07.947 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.726970+0000", 2024-09-14T05:01:07.948 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0910000000000003e-06 2024-09-14T05:01:07.948 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.948 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.948 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:01:07.948 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.727006+0000", 2024-09-14T05:01:07.948 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5125e-05 2024-09-14T05:01:07.948 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.948 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.948 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:01:07.948 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.727093+0000", 2024-09-14T05:01:07.948 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.7620000000000005e-05 2024-09-14T05:01:07.948 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.948 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.949 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,6", 2024-09-14T05:01:07.949 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.727169+0000", 2024-09-14T05:01:07.949 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.5845000000000003e-05 2024-09-14T05:01:07.949 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.949 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.949 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.949 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.735163+0000", 2024-09-14T05:01:07.949 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0079942829999999996 2024-09-14T05:01:07.949 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.949 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.949 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:01:07.949 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.735175+0000", 2024-09-14T05:01:07.949 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1413000000000001e-05 2024-09-14T05:01:07.949 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.950 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.950 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.950 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.934704+0000", 2024-09-14T05:01:07.950 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.1995296 2024-09-14T05:01:07.950 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.950 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.950 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:01:07.950 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.934750+0000", 2024-09-14T05:01:07.950 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6066999999999998e-05 2024-09-14T05:01:07.950 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.950 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.950 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:01:07.950 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.934792+0000", 2024-09-14T05:01:07.950 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1474000000000003e-05 2024-09-14T05:01:07.950 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.951 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:01:07.951 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.951 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.951 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.951 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:172820 56.a 56:5f82633b:::benchmark_data_smithi039_330847_object58296:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1508)", 2024-09-14T05:01:07.951 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:55:16.727576+0000", 2024-09-14T05:01:07.951 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 351.127906096, 2024-09-14T05:01:07.951 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.20734001899999999, 2024-09-14T05:01:07.951 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:01:07.951 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:01:07.952 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:01:07.952 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:01:07.952 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:01:07.952 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 172820 2024-09-14T05:01:07.952 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.953 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:01:07.953 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.953 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:01:07.953 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.727576+0000", 2024-09-14T05:01:07.953 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.954 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.954 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.954 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:01:07.954 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.727576+0000", 2024-09-14T05:01:07.954 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.954 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.954 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.954 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:01:07.954 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.727576+0000", 2024-09-14T05:01:07.954 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T05:01:07.954 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.954 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.955 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:01:07.955 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.727582+0000", 2024-09-14T05:01:07.955 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.3929999999999996e-06 2024-09-14T05:01:07.955 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.955 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.955 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:01:07.955 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.727592+0000", 2024-09-14T05:01:07.955 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.5999999999999996e-06 2024-09-14T05:01:07.955 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.955 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.955 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:01:07.955 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.727596+0000", 2024-09-14T05:01:07.955 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6650000000000002e-06 2024-09-14T05:01:07.955 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.956 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.956 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:01:07.956 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.731628+0000", 2024-09-14T05:01:07.956 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0040314310000000003 2024-09-14T05:01:07.956 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.956 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.956 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:01:07.956 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.731741+0000", 2024-09-14T05:01:07.956 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000113006 2024-09-14T05:01:07.956 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.956 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.956 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,6", 2024-09-14T05:01:07.956 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.731854+0000", 2024-09-14T05:01:07.957 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011318499999999999 2024-09-14T05:01:07.957 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.957 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.957 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:01:07.957 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.735243+0000", 2024-09-14T05:01:07.957 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0033891210000000001 2024-09-14T05:01:07.957 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.957 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.957 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.957 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.735272+0000", 2024-09-14T05:01:07.958 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.9430000000000001e-05 2024-09-14T05:01:07.958 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.958 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.958 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.958 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.934834+0000", 2024-09-14T05:01:07.958 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.19956151899999999 2024-09-14T05:01:07.958 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.958 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.958 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:01:07.958 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.934881+0000", 2024-09-14T05:01:07.959 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7435999999999998e-05 2024-09-14T05:01:07.959 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.959 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.959 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:01:07.959 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.934916+0000", 2024-09-14T05:01:07.959 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5011000000000001e-05 2024-09-14T05:01:07.959 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.959 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:01:07.959 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.959 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.959 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.959 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:172821 56.f 56:f6101387:::benchmark_data_smithi039_330847_object58297:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1508)", 2024-09-14T05:01:07.959 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:55:16.728075+0000", 2024-09-14T05:01:07.960 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 351.12740798300001, 2024-09-14T05:01:07.960 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.206655599, 2024-09-14T05:01:07.960 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:01:07.960 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:01:07.960 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:01:07.960 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:01:07.960 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:01:07.960 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 172821 2024-09-14T05:01:07.960 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.960 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:01:07.960 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.960 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:01:07.960 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.728075+0000", 2024-09-14T05:01:07.961 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.961 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.961 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.961 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:01:07.961 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.728075+0000", 2024-09-14T05:01:07.961 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.961 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.961 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.961 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:01:07.961 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.728074+0000", 2024-09-14T05:01:07.962 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T05:01:07.962 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.962 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.962 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:01:07.962 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.728079+0000", 2024-09-14T05:01:07.962 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8239999999999999e-06 2024-09-14T05:01:07.962 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.962 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.962 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:01:07.962 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.728091+0000", 2024-09-14T05:01:07.962 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2761e-05 2024-09-14T05:01:07.962 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.962 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.962 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:01:07.963 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.728098+0000", 2024-09-14T05:01:07.963 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.37e-06 2024-09-14T05:01:07.963 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.963 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.963 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:01:07.963 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.728134+0000", 2024-09-14T05:01:07.963 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6316000000000003e-05 2024-09-14T05:01:07.963 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.963 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.963 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:01:07.963 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.728258+0000", 2024-09-14T05:01:07.963 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012450199999999999 2024-09-14T05:01:07.963 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.964 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.964 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,7", 2024-09-14T05:01:07.964 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.728381+0000", 2024-09-14T05:01:07.964 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012210300000000001 2024-09-14T05:01:07.964 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.964 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.964 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:01:07.964 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.731944+0000", 2024-09-14T05:01:07.964 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0035638509999999998 2024-09-14T05:01:07.964 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.964 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.964 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.964 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.734274+0000", 2024-09-14T05:01:07.964 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0023291229999999998 2024-09-14T05:01:07.965 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.965 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.965 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.965 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.934628+0000", 2024-09-14T05:01:07.965 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.200354698 2024-09-14T05:01:07.965 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.965 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.965 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:01:07.965 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.934665+0000", 2024-09-14T05:01:07.965 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7230000000000001e-05 2024-09-14T05:01:07.965 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.965 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.965 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:01:07.965 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.934730+0000", 2024-09-14T05:01:07.966 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.4645000000000002e-05 2024-09-14T05:01:07.966 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.966 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:01:07.966 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.966 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.966 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.966 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:172825 56.4 56:23b09923:::benchmark_data_smithi039_330847_object58301:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1508)", 2024-09-14T05:01:07.966 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:55:16.736609+0000", 2024-09-14T05:01:07.966 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 351.11887373299999, 2024-09-14T05:01:07.966 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.21087603499999999, 2024-09-14T05:01:07.966 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:01:07.966 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:01:07.966 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:01:07.966 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:01:07.966 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:01:07.967 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 172825 2024-09-14T05:01:07.967 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.967 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:01:07.967 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.967 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:01:07.967 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.736609+0000", 2024-09-14T05:01:07.967 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.967 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.967 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.967 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:01:07.967 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.736609+0000", 2024-09-14T05:01:07.967 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.967 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.967 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.968 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:01:07.968 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.736607+0000", 2024-09-14T05:01:07.968 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T05:01:07.968 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.968 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.968 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:01:07.968 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.736613+0000", 2024-09-14T05:01:07.968 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.6969999999999998e-06 2024-09-14T05:01:07.968 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.968 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.968 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:01:07.968 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.736627+0000", 2024-09-14T05:01:07.969 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3918e-05 2024-09-14T05:01:07.969 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.969 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.969 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:01:07.969 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.736634+0000", 2024-09-14T05:01:07.969 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.1629999999999999e-06 2024-09-14T05:01:07.969 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.969 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.969 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:01:07.969 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.736675+0000", 2024-09-14T05:01:07.969 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1174999999999998e-05 2024-09-14T05:01:07.969 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.970 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.970 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:01:07.970 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.736786+0000", 2024-09-14T05:01:07.970 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0001106 2024-09-14T05:01:07.970 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.970 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.970 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,2", 2024-09-14T05:01:07.970 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.736901+0000", 2024-09-14T05:01:07.970 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000114898 2024-09-14T05:01:07.970 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.970 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.970 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:01:07.970 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.740622+0000", 2024-09-14T05:01:07.971 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00372106 2024-09-14T05:01:07.971 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.971 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.971 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.971 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.743142+0000", 2024-09-14T05:01:07.971 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0025199659999999998 2024-09-14T05:01:07.971 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.971 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.971 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.971 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.947396+0000", 2024-09-14T05:01:07.971 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.20425436899999999 2024-09-14T05:01:07.971 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.971 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.971 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:01:07.971 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.947446+0000", 2024-09-14T05:01:07.972 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.9954999999999999e-05 2024-09-14T05:01:07.972 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.972 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.972 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:01:07.972 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.947485+0000", 2024-09-14T05:01:07.972 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8519000000000002e-05 2024-09-14T05:01:07.972 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.972 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:01:07.972 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.972 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.972 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.972 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36012.0:172838 56.f 56:f43eb43d:::benchmark_data_smithi039_330847_object58314:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1508)", 2024-09-14T05:01:07.972 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:55:16.754456+0000", 2024-09-14T05:01:07.972 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 351.10102637699998, 2024-09-14T05:01:07.973 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.196324211, 2024-09-14T05:01:07.973 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:01:07.973 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:01:07.973 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:01:07.973 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36012", 2024-09-14T05:01:07.973 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:01:07.973 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 172838 2024-09-14T05:01:07.973 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.973 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:01:07.973 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.973 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:01:07.973 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.754456+0000", 2024-09-14T05:01:07.973 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.973 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.974 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.974 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:01:07.974 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.754456+0000", 2024-09-14T05:01:07.974 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.974 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.974 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.974 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:01:07.974 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.754446+0000", 2024-09-14T05:01:07.974 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.99999 2024-09-14T05:01:07.974 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.974 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.974 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:01:07.974 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.754467+0000", 2024-09-14T05:01:07.974 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.0752999999999999e-05 2024-09-14T05:01:07.974 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.975 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.975 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:01:07.975 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.754498+0000", 2024-09-14T05:01:07.975 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.0870000000000001e-05 2024-09-14T05:01:07.975 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.975 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.975 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:01:07.975 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.754512+0000", 2024-09-14T05:01:07.975 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.4283e-05 2024-09-14T05:01:07.975 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.975 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.975 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:01:07.975 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.754610+0000", 2024-09-14T05:01:07.975 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.8238000000000004e-05 2024-09-14T05:01:07.975 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.976 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.976 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:01:07.976 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.754804+0000", 2024-09-14T05:01:07.976 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00019393500000000001 2024-09-14T05:01:07.976 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.976 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.976 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,7", 2024-09-14T05:01:07.976 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.754880+0000", 2024-09-14T05:01:07.976 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.5637999999999996e-05 2024-09-14T05:01:07.976 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.976 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.976 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:01:07.976 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.758112+0000", 2024-09-14T05:01:07.976 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0032323439999999998 2024-09-14T05:01:07.977 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.977 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.977 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.977 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.758336+0000", 2024-09-14T05:01:07.977 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00022379299999999999 2024-09-14T05:01:07.977 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.977 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.977 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.977 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.950659+0000", 2024-09-14T05:01:07.977 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.19232284599999999 2024-09-14T05:01:07.977 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.978 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.978 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:01:07.978 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.950741+0000", 2024-09-14T05:01:07.978 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.2429000000000006e-05 2024-09-14T05:01:07.978 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.978 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.978 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:01:07.978 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:55:16.950780+0000", 2024-09-14T05:01:07.978 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9011999999999999e-05 2024-09-14T05:01:07.978 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.978 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:01:07.978 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.978 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.979 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.979 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36324.0:34412 57.9 57:9b838976:::benchmark_data_smithi039_365565_object34411:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T05:01:07.979 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:57:13.470008+0000", 2024-09-14T05:01:07.979 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 234.38547456699999, 2024-09-14T05:01:07.979 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.43965552499999999, 2024-09-14T05:01:07.979 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:01:07.979 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:01:07.979 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:01:07.979 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36324", 2024-09-14T05:01:07.979 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:01:07.979 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 34412 2024-09-14T05:01:07.979 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.979 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:01:07.980 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.980 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:01:07.980 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.470008+0000", 2024-09-14T05:01:07.980 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.980 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.980 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.980 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:01:07.980 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.470008+0000", 2024-09-14T05:01:07.980 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.980 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.980 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.980 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:01:07.980 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.470006+0000", 2024-09-14T05:01:07.980 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T05:01:07.981 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.981 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.981 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:01:07.981 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.470039+0000", 2024-09-14T05:01:07.981 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.2662e-05 2024-09-14T05:01:07.981 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.981 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.981 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:01:07.981 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.470050+0000", 2024-09-14T05:01:07.981 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0626e-05 2024-09-14T05:01:07.981 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.981 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.982 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:01:07.982 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.470056+0000", 2024-09-14T05:01:07.982 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.4710000000000004e-06 2024-09-14T05:01:07.982 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.982 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.982 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:01:07.982 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.470095+0000", 2024-09-14T05:01:07.982 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9042000000000003e-05 2024-09-14T05:01:07.982 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.982 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.982 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:01:07.982 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.470183+0000", 2024-09-14T05:01:07.982 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.8117999999999999e-05 2024-09-14T05:01:07.982 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.983 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.983 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,5", 2024-09-14T05:01:07.983 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.900775+0000", 2024-09-14T05:01:07.983 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.43059172699999998 2024-09-14T05:01:07.983 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.983 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.983 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.983 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.906165+0000", 2024-09-14T05:01:07.983 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0053896339999999999 2024-09-14T05:01:07.983 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.983 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.983 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:01:07.983 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.906206+0000", 2024-09-14T05:01:07.983 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1229000000000001e-05 2024-09-14T05:01:07.984 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.984 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.984 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:01:07.984 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.909582+0000", 2024-09-14T05:01:07.984 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0033762029999999999 2024-09-14T05:01:07.984 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.984 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.984 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:01:07.984 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.909630+0000", 2024-09-14T05:01:07.984 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8182e-05 2024-09-14T05:01:07.984 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.984 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.984 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:01:07.985 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.909663+0000", 2024-09-14T05:01:07.985 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3120000000000001e-05 2024-09-14T05:01:07.985 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.985 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:01:07.985 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.985 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.985 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.985 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.36324.0:34402 57.0 e1523/1518 57:0b7b4153:::benchmark_data_smithi039_365565_object34401:head v 1523'2165, mlcod=1523'2162)", 2024-09-14T05:01:07.985 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:57:13.470802+0000", 2024-09-14T05:01:07.985 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 234.38468049100001, 2024-09-14T05:01:07.985 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.43367837199999998, 2024-09-14T05:01:07.985 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:01:07.986 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:01:07.986 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:01:07.986 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.986 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:01:07.986 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.470802+0000", 2024-09-14T05:01:07.986 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.986 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.986 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.986 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:01:07.986 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.470802+0000", 2024-09-14T05:01:07.986 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:01:07.986 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.986 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.986 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:01:07.986 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.470798+0000", 2024-09-14T05:01:07.987 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999962 2024-09-14T05:01:07.987 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.987 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.987 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:01:07.987 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.470852+0000", 2024-09-14T05:01:07.987 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.3199999999999999e-05 2024-09-14T05:01:07.987 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.987 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.987 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:01:07.987 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.900655+0000", 2024-09-14T05:01:07.987 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.429803722 2024-09-14T05:01:07.987 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.988 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.988 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:01:07.988 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.900672+0000", 2024-09-14T05:01:07.988 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.6489000000000001e-05 2024-09-14T05:01:07.988 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.988 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.988 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:01:07.988 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.900739+0000", 2024-09-14T05:01:07.988 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7281999999999997e-05 2024-09-14T05:01:07.988 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.988 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.988 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:01:07.988 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.900767+0000", 2024-09-14T05:01:07.989 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.8076e-05 2024-09-14T05:01:07.989 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.989 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.989 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:01:07.989 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.904441+0000", 2024-09-14T05:01:07.989 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.003674161 2024-09-14T05:01:07.989 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:01:07.989 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:01:07.989 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:01:07.989 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.904480+0000", 2024-09-14T05:01:07.990 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9020000000000002e-05 2024-09-14T05:01:07.990 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.990 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:01:07.990 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.990 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:01:07.990 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:01:07.990 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T05:01:07.990 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:08.003 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:07 smithi039 ceph-mon[106470]: pgmap v3082: 109 pgs: 16 unknown, 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:01:08.003 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:07 smithi039 ceph-mon[106470]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T05:01:08.003 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:07 smithi039 ceph-mon[106470]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:01:08.003 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:07 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T05:01:08.003 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:07 smithi039 ceph-mon[106470]: osdmap e1569: 8 total, 8 up, 8 in 2024-09-14T05:01:08.003 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:01:08.003 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:01:08.004 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:07 smithi039 ceph-mon[111235]: pgmap v3082: 109 pgs: 16 unknown, 93 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:01:08.004 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:07 smithi039 ceph-mon[111235]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T05:01:08.004 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:07 smithi039 ceph-mon[111235]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:01:08.004 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:07 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T05:01:08.004 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:07 smithi039 ceph-mon[111235]: osdmap e1569: 8 total, 8 up, 8 in 2024-09-14T05:01:08.004 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:01:08.004 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:01:08.011 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:08.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:08.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:08.113 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:08.134 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:08.135 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:08.135 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:08.237 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:08.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:08.261 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:08.261 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:08.362 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:08.385 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:08.385 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:08.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:08.486 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:08.510 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:08.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:08.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:08.612 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:08.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:08.636 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:08.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:08.737 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:08.752 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:01:08.758 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:08.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:08.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:08.860 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:08.883 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:08.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:08.884 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:08.967 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-14T05:01:08.970 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:08 smithi039 ceph-mon[106470]: osdmap e1570: 8 total, 8 up, 8 in 2024-09-14T05:01:08.971 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:01:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:01:08] "GET /metrics HTTP/1.1" 200 36670 "" "Prometheus/2.51.0" 2024-09-14T05:01:08.971 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:08 smithi039 ceph-mon[111235]: osdmap e1570: 8 total, 8 up, 8 in 2024-09-14T05:01:08.984 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:09.033 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:09.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:09.034 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:09.135 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:09.156 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:09.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:09.158 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:09.258 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:09.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:08 smithi129 ceph-mon[101316]: osdmap e1570: 8 total, 8 up, 8 in 2024-09-14T05:01:09.280 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:09.280 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:09.281 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:09.381 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:09.404 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:09.404 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:09.405 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:09.455 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T05:01:09.455 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T05:01:09.459 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T05:01:09.459 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T05:01:09.475 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T05:01:09.476 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T05:01:09.492 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T05:01:09.492 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T05:01:09.506 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:09.509 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T05:01:09.509 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T05:01:09.525 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T05:01:09.526 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T05:01:09.527 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:09.528 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:09.528 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:09.542 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T05:01:09.542 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T05:01:09.559 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T05:01:09.559 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T05:01:09.629 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:09.651 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:09.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:09.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:09.753 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:09.775 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:09.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:09.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:09.877 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:09.899 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:09.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:09.900 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:10.001 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:10.023 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:10.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:10.024 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:10.026 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T05:01:10.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:09 smithi039 ceph-mon[106470]: pgmap v3085: 109 pgs: 15 unknown, 94 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:01:10.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:09 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1566282350' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:01:10.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:09 smithi039 ceph-mon[111235]: pgmap v3085: 109 pgs: 15 unknown, 94 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:01:10.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:09 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1566282350' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:01:10.125 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:10.148 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:10.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:10.149 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:10.163 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:09 smithi129 ceph-mon[101316]: pgmap v3085: 109 pgs: 15 unknown, 94 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:01:10.163 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:09 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1566282350' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:01:10.250 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:10.273 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:10.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:10.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:10.374 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:10.395 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:10.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:10.396 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:10.497 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:10.520 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:10.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:10.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:10.622 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:10.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:10.645 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:10.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:10.746 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:10.769 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:10.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:10.770 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:10.871 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:10.893 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:10.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:10.894 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:10.995 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:11.017 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:11.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:11.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:11.089 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:01:11.119 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:11.140 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:11.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:11.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:11.244 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:11.265 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:11.266 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:11.266 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:11.367 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:11.389 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:11.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:11.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:11.491 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:11.512 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:11.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:11.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:11.614 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:11.636 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:11.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:11.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:11.738 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:11.761 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:11.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:11.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:11.863 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:11.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:11.885 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:11.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:11.987 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:12.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:12.009 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:12.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:12.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:11 smithi039 ceph-mon[106470]: pgmap v3086: 109 pgs: 4 unknown, 105 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:01:12.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:11 smithi039 ceph-mon[111235]: pgmap v3086: 109 pgs: 4 unknown, 105 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:01:12.110 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:12.131 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:12.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:12.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:12.233 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:12.255 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:12.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:12.256 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:12.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:11 smithi129 ceph-mon[101316]: pgmap v3086: 109 pgs: 4 unknown, 105 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:01:12.357 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:12.380 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:12.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:12.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:12.481 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:12.503 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:12.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:12.504 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:12.605 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:12.627 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T05:01:12.628 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:12.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:12.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:12.730 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:12.731 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:01:12.752 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:12.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:12.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:12.796 INFO:teuthology.orchestra.run.smithi039.stderr:enabled application 'rados' on pool 'unique_pool_5' 2024-09-14T05:01:12.853 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:12.877 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:12.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:12.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:12.979 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:12.980 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:12 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1938527749' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T05:01:12.980 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:12 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1938527749' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T05:01:13.003 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:13.004 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:13.004 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:13.105 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:13.127 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:13.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:13.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:13.229 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:13.252 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:13.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:13.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:13.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:12 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1938527749' entity='client.admin' cmd={"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true} : dispatch 2024-09-14T05:01:13.355 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:13.378 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:13.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:13.379 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:13.480 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:13.502 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:13.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:13.503 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:13.604 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:13.626 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:13.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:13.627 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:13.728 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:13.751 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:13.751 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:13.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:13.852 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:13.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:13.878 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:13.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:13.979 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:14.003 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:14.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:14.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:14.105 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:14.128 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:14.128 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:14.129 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:14.164 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:13 smithi039 ceph-mon[106470]: pgmap v3087: 109 pgs: 109 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:01:14.164 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:13 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1938527749' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T05:01:14.164 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:13 smithi039 ceph-mon[106470]: osdmap e1571: 8 total, 8 up, 8 in 2024-09-14T05:01:14.164 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:13 smithi039 ceph-mon[111235]: pgmap v3087: 109 pgs: 109 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:01:14.164 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:13 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1938527749' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T05:01:14.164 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:13 smithi039 ceph-mon[111235]: osdmap e1571: 8 total, 8 up, 8 in 2024-09-14T05:01:14.229 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:14.252 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:14.252 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:14.253 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:14.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:13 smithi129 ceph-mon[101316]: pgmap v3087: 109 pgs: 109 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:01:14.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:13 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1938527749' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2024-09-14T05:01:14.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:13 smithi129 ceph-mon[101316]: osdmap e1571: 8 total, 8 up, 8 in 2024-09-14T05:01:14.354 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:14.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:14.377 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:14.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:14.478 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:14.500 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:14.501 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:14.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:14.602 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:14.625 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:14.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:14.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:14.727 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:14.752 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:14.753 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:14.753 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:14.795 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T05:01:14.796 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:01:14.807 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:14 smithi039 ceph-mon[106470]: pgmap v3089: 109 pgs: 109 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:01:14.854 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:14.876 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:14.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:14.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:14.978 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:15.000 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:15.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:15.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:15.102 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:15.124 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:15.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:15.125 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:15.179 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:14 smithi039 ceph-mon[111235]: pgmap v3089: 109 pgs: 109 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:01:15.226 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:15.249 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:15.249 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:15.250 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:15.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:14 smithi129 ceph-mon[101316]: pgmap v3089: 109 pgs: 109 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:01:15.351 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:15.374 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:15.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:15.375 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:15.476 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:15.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:15.499 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:15.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:15.600 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:15.626 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:15.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:15.627 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:15.728 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:15.754 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:15.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:15.755 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:15.856 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:15.880 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:15.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:15.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:15.918 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:15 smithi039 ceph-mon[106470]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T05:01:15.918 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:15 smithi039 ceph-mon[111235]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T05:01:15.982 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:16.004 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:16.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:16.005 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:16.106 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:16.128 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:16.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:16.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:16.230 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:16.254 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:16.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:16.255 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:16.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:15 smithi129 ceph-mon[101316]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-09-14T05:01:16.356 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:16.378 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:16.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:16.379 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:16.480 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:16.506 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:16.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:16.507 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:16.607 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:16.631 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:16.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:16.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:16.733 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:16.755 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:16.756 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:16.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:16.857 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:16.879 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:16.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:16.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:16.940 DEBUG:teuthology.orchestra.run.smithi039:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2024-09-14T05:01:16.941 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_5 2024-09-14T05:01:16.941 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T05:01:16.943 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2024-09-14T05:01:16.943 INFO:teuthology.orchestra.run:waiting for 3000 2024-09-14T05:01:16.965 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:16 smithi039 ceph-mon[106470]: pgmap v3090: 109 pgs: 109 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:01:16.965 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:16 smithi039 ceph-mon[111235]: pgmap v3090: 109 pgs: 109 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:01:16.980 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:17.006 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:17.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:17.007 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:17.041 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:hints = 1 2024-09-14T05:01:17.041 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2024-09-14T05:01:17.041 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Object prefix: benchmark_data_smithi039_396950 2024-09-14T05:01:17.041 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T05:01:17.041 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 0 0 0 0 0 0 - 0 2024-09-14T05:01:17.108 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:17.150 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:17.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:17.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:17.252 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:17.261 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-14T05:01:17.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:16 smithi129 ceph-mon[101316]: pgmap v3090: 109 pgs: 109 active+clean; 582 KiB data, 5.9 GiB used, 709 GiB / 715 GiB avail 2024-09-14T05:01:17.275 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:01:17.316 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:17.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:17.317 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:17.418 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:17.445 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:17.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:17.446 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:17.547 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:17.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:17.593 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:17.593 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:17.694 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:17.735 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:17.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:17.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:17.837 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:17.857 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is unset 2024-09-14T05:01:17.882 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:17.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:17.883 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:17.984 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:18.015 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:18.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:18.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:18.041 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 1 16 2944 2928 182.987 183 0.00616472 0.00544465 2024-09-14T05:01:18.110 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:17 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2185222500' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:01:18.110 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:17 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:01:18.110 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:17 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2185222500' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:01:18.111 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:17 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:01:18.116 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:18.157 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:18.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:18.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:18.260 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:18.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:17 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2185222500' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:01:18.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:17 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:01:18.303 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:01:18.303 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:18.304 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:18.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:18.405 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:18.431 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:18.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:18.433 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:18.533 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:18.563 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:18.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:18.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:18.665 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:18.694 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:18.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:18.695 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:18.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:01:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:01:18] "GET /metrics HTTP/1.1" 200 36670 "" "Prometheus/2.51.0" 2024-09-14T05:01:18.796 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:18.836 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:18.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:18.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:18.937 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:18.968 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:18.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:18.969 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:19.041 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 2 16 6313 6297 196.765 210.562 0.00362293 0.00507183 2024-09-14T05:01:19.070 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:19.096 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:19.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:19.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:19.100 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:18 smithi039 ceph-mon[106470]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:01:19.101 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:18 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T05:01:19.101 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:18 smithi039 ceph-mon[106470]: osdmap e1572: 8 total, 8 up, 8 in 2024-09-14T05:01:19.101 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:18 smithi039 ceph-mon[106470]: pgmap v3092: 109 pgs: 109 active+clean; 10 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 1.2 MiB/s wr, 19 op/s 2024-09-14T05:01:19.101 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:18 smithi039 ceph-mon[111235]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:01:19.101 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:18 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T05:01:19.101 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:18 smithi039 ceph-mon[111235]: osdmap e1572: 8 total, 8 up, 8 in 2024-09-14T05:01:19.101 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:18 smithi039 ceph-mon[111235]: pgmap v3092: 109 pgs: 109 active+clean; 10 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 1.2 MiB/s wr, 19 op/s 2024-09-14T05:01:19.197 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:19.237 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:19.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:19.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:19.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:18 smithi129 ceph-mon[101316]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:01:19.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:18 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T05:01:19.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:18 smithi129 ceph-mon[101316]: osdmap e1572: 8 total, 8 up, 8 in 2024-09-14T05:01:19.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:18 smithi129 ceph-mon[101316]: pgmap v3092: 109 pgs: 109 active+clean; 10 MiB data, 5.9 GiB used, 709 GiB / 715 GiB avail; 1.2 MiB/s wr, 19 op/s 2024-09-14T05:01:19.339 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:19.368 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:19.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:19.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:19.470 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:19.497 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:19.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:19.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:19.599 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:19.639 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:19.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:19.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:19.741 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:19.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:19.786 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:19.786 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:19.886 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:19.914 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:19.914 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:19.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:20.016 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:20.042 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 3 16 9369 9353 194.819 191 0.00345834 0.00512472 2024-09-14T05:01:20.045 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:20.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:20.046 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:20.147 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:20.177 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:20.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:20.178 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:20.278 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:20.318 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:20.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:20.319 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:20.419 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:20.422 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:01:20.445 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:20.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:20.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:20.548 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:20.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:20.594 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:20.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:20.695 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:20.739 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:20.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:20.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:20.841 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:20.883 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:20.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:20.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:20.985 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:21.027 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:21.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:21.028 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:21.042 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 4 16 12121 12105 189.111 172 0.00290547 0.00528039 2024-09-14T05:01:21.103 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T05:01:21.129 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:21.145 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:21 smithi039 ceph-mon[106470]: pgmap v3093: 109 pgs: 109 active+clean; 266 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 33 MiB/s wr, 530 op/s 2024-09-14T05:01:21.145 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:21 smithi039 ceph-mon[111235]: pgmap v3093: 109 pgs: 109 active+clean; 266 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 33 MiB/s wr, 530 op/s 2024-09-14T05:01:21.157 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:21.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:21.158 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:21.259 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:21.285 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:21.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:21.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:21.387 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:21.433 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:21.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:21.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:21.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:21 smithi129 ceph-mon[101316]: pgmap v3093: 109 pgs: 109 active+clean; 266 MiB data, 6.3 GiB used, 709 GiB / 715 GiB avail; 33 MiB/s wr, 530 op/s 2024-09-14T05:01:21.536 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:21.564 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:21.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:21.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:21.666 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:21.712 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:21.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:21.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:21.760 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T05:01:21.760 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T05:01:21.806 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T05:01:21.806 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T05:01:21.814 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:21.842 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:21.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:21.843 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:21.845 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T05:01:21.845 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T05:01:21.882 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T05:01:21.882 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T05:01:21.915 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T05:01:21.916 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T05:01:21.944 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:21.950 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T05:01:21.950 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T05:01:21.970 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:21.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:21.971 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:21.985 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T05:01:21.985 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T05:01:22.024 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T05:01:22.024 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T05:01:22.042 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 5 16 14658 14642 182.998 158.562 0.0147754 0.00545768 2024-09-14T05:01:22.071 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:22.113 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:22.113 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:22.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:22.214 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:22.242 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:22.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:22.243 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:22.289 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:22 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4272309043' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:01:22.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:22 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4272309043' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:01:22.343 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:22.387 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:22.388 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:22.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:22.489 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:22.516 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:22.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:22.517 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:22.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:22 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4272309043' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:01:22.618 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:22.660 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:22.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:22.661 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:22.686 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T05:01:22.688 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T05:01:22.761 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:22.790 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:22.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:22.791 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:22.892 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:22.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:22.935 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:22.935 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:23.036 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:23.042 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 6 16 17074 17058 177.663 151 0.0051816 0.00562016 2024-09-14T05:01:23.078 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:23.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:23.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:23.181 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:23.209 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:23.210 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:23.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:23.311 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:23.337 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:23.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:23.338 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:23.439 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:23.467 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:23.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:23.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:23.569 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:23.601 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:23.602 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:23.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:23.655 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:23 smithi039 ceph-mon[106470]: pgmap v3094: 109 pgs: 109 active+clean; 443 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 48 MiB/s wr, 768 op/s 2024-09-14T05:01:23.655 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:23 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:01:23.655 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:01:23.655 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:23 smithi039 ceph-mon[106470]: from='client.46544 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T05:01:23.655 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:23 smithi039 ceph-mon[111235]: pgmap v3094: 109 pgs: 109 active+clean; 443 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 48 MiB/s wr, 768 op/s 2024-09-14T05:01:23.656 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:23 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:01:23.656 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:01:23.656 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:23 smithi039 ceph-mon[111235]: from='client.46544 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T05:01:23.703 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:23.729 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:23.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:23.730 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:23 smithi129 ceph-mon[101316]: pgmap v3094: 109 pgs: 109 active+clean; 443 MiB data, 6.7 GiB used, 709 GiB / 715 GiB avail; 48 MiB/s wr, 768 op/s 2024-09-14T05:01:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:23 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:01:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:01:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:23 smithi129 ceph-mon[101316]: from='client.46544 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T05:01:23.831 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:23.861 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:23.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:23.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:23.963 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:23.991 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:23.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:23.992 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:24.042 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 7 16 19590 19574 174.745 157.25 0.004932 0.00571526 2024-09-14T05:01:24.093 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:24.119 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:24.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:24.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:24.221 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:24.263 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:24.264 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:24.264 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:24.365 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:24.403 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:24.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:24.404 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:24.505 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:24.531 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:24.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:24.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:24.633 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:24.662 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:24.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:24.663 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:24.764 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:24.786 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:24.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:24.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:24.888 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:24.927 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:24.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:24.928 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:25.029 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:25.042 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 8 16 21766 21750 169.901 136 0.00696422 0.00587879 2024-09-14T05:01:25.066 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:25.067 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:25.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:25.106 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T05:01:25.168 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:25.196 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:25.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:25.197 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:25.297 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:25.341 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:25.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:25.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:25.443 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:25.472 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:25.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:25.473 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:25.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:25 smithi039 ceph-mon[106470]: pgmap v3095: 109 pgs: 109 active+clean; 774 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-09-14T05:01:25.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:25 smithi039 ceph-mon[111235]: pgmap v3095: 109 pgs: 109 active+clean; 774 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-09-14T05:01:25.574 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:25.620 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:25.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:25.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:25.721 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:25.766 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:25.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:25.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:25.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:25 smithi129 ceph-mon[101316]: pgmap v3095: 109 pgs: 109 active+clean; 774 MiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 77 MiB/s wr, 1.24k op/s 2024-09-14T05:01:25.868 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:25.901 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:25.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:25.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:26.004 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:26.040 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:26.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:26.042 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:26.042 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 9 16 23766 23750 164.91 125 0.00469733 0.00605852 2024-09-14T05:01:26.143 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:26.182 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:26.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:26.183 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:26.284 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:26.312 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:26.312 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:26.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:26.413 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:26.453 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:26.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:26.454 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:26.555 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:26.580 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:26.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:26.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:26.593 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":3094,"stamp":"2024-09-14T05:01:21.999286+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":464328677,"num_objects":7102,"num_object_clones":0,"num_object_copies":21306,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7102,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7140,"num_write_kb":452897,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":1581198,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":10264,"ondisk_log_size":10264,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":6974640,"kb_used_data":868064,"kb_used_omap":404,"kb_used_meta":6106091,"kb_avail":742986576,"statfs":{"total":767960285184,"available":760818253824,"internally_reserved":0,"allocated":888897536,"data_stored":843596479,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":414022,"internal_metadata":6252637882},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":463732736,"num_objects":7076,"num_object_clones":0,"num_object_copies":21228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7076,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7076,"num_write_kb":452864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.202929"},"pg_stats":[{"pgid":"58.f","version":"1572'456","reported_seq":475,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:19.579525+0000","last_change":"2024-09-14T05:01:06.755007+0000","last_active":"2024-09-14T05:01:19.579525+0000","last_peered":"2024-09-14T05:01:19.579525+0000","last_clean":"2024-09-14T05:01:19.579525+0000","last_became_active":"2024-09-14T05:01:06.754750+0000","last_became_peered":"2024-09-14T05:01:06.754750+0000","last_unstale":"2024-09-14T05:01:19.579525+0000","last_undegraded":"2024-09-14T05:01:19.579525+0000","last_fullsized":"2024-09-14T05:01:19.579525+0000","mapping_epoch":1568,"log_start":"0'0","ondisk_log_start":"0'0","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":456,"log_dups_size":0,"ondisk_log_size":456,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:36:40.288167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29884416,"num_objects":456,"num_object_clones":0,"num_object_copies":1368,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":456,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":456,"num_write_kb":29184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"58.e","version":"1572'703","reported_seq":722,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:20.644948+0000","last_change":"2024-09-14T05:01:07.066710+0000","last_active":"2024-09-14T05:01:20.644948+0000","last_peered":"2024-09-14T05:01:20.644948+0000","last_clean":"2024-09-14T05:01:20.644948+0000","last_became_active":"2024-09-14T05:01:07.066378+0000","last_became_peered":"2024-09-14T05:01:07.066378+0000","last_unstale":"2024-09-14T05:01:20.644948+0000","last_undegraded":"2024-09-14T05:01:20.644948+0000","last_fullsized":"2024-09-14T05:01:20.644948+0000","mapping_epoch":1568,"log_start":"0'0","ondisk_log_start":"0'0","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":703,"log_dups_size":0,"ondisk_log_size":703,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:02:52.009893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46071808,"num_objects":703,"num_object_clones":0,"num_object_copies":2109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":703,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":703,"num_write_kb":44992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"58.d","version":"1572'667","reported_seq":686,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:20.638541+0000","last_change":"2024-09-14T05:01:07.065787+0000","last_active":"2024-09-14T05:01:20.638541+0000","last_peered":"2024-09-14T05:01:20.638541+0000","last_clean":"2024-09-14T05:01:20.638541+0000","last_became_active":"2024-09-14T05:01:07.065475+0000","last_became_peered":"2024-09-14T05:01:07.065475+0000","last_unstale":"2024-09-14T05:01:20.638541+0000","last_undegraded":"2024-09-14T05:01:20.638541+0000","last_fullsized":"2024-09-14T05:01:20.638541+0000","mapping_epoch":1568,"log_start":"0'0","ondisk_log_start":"0'0","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":667,"log_dups_size":0,"ondisk_log_size":667,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:40:34.198013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43712512,"num_objects":667,"num_object_clones":0,"num_object_copies":2001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":667,"num_write_kb":42688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"58.c","version":"1572'734","reported_seq":753,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:20.634779+0000","last_change":"2024-09-14T05:01:06.755561+0000","last_active":"2024-09-14T05:01:20.634779+0000","last_peered":"2024-09-14T05:01:20.634779+0000","last_clean":"2024-09-14T05:01:20.634779+0000","last_became_active":"2024-09-14T05:01:06.754338+0000","last_became_peered":"2024-09-14T05:01:06.754338+0000","last_unstale":"2024-09-14T05:01:20.634779+0000","last_undegraded":"2024-09-14T05:01:20.634779+0000","last_fullsized":"2024-09-14T05:01:20.634779+0000","mapping_epoch":1568,"log_start":"0'0","ondisk_log_start":"0'0","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":734,"log_dups_size":0,"ondisk_log_size":734,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:42:31.850156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48103424,"num_objects":734,"num_object_clones":0,"num_object_copies":2202,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":734,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":734,"num_write_kb":46976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"58.b","version":"1572'510","reported_seq":529,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:19.609365+0000","last_change":"2024-09-14T05:01:06.754029+0000","last_active":"2024-09-14T05:01:19.609365+0000","last_peered":"2024-09-14T05:01:19.609365+0000","last_clean":"2024-09-14T05:01:19.609365+0000","last_became_active":"2024-09-14T05:01:06.753768+0000","last_became_peered":"2024-09-14T05:01:06.753768+0000","last_unstale":"2024-09-14T05:01:19.609365+0000","last_undegraded":"2024-09-14T05:01:19.609365+0000","last_fullsized":"2024-09-14T05:01:19.609365+0000","mapping_epoch":1568,"log_start":"0'0","ondisk_log_start":"0'0","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":510,"log_dups_size":0,"ondisk_log_size":510,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:50:59.004816+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33423360,"num_objects":510,"num_object_clones":0,"num_object_copies":1530,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":510,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":510,"num_write_kb":32640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"58.a","version":"1572'460","reported_seq":479,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:19.331093+0000","last_change":"2024-09-14T05:01:07.067188+0000","last_active":"2024-09-14T05:01:19.331093+0000","last_peered":"2024-09-14T05:01:19.331093+0000","last_clean":"2024-09-14T05:01:19.331093+0000","last_became_active":"2024-09-14T05:01:07.066676+0000","last_became_peered":"2024-09-14T05:01:07.066676+0000","last_unstale":"2024-09-14T05:01:19.331093+0000","last_undegraded":"2024-09-14T05:01:19.331093+0000","last_fullsized":"2024-09-14T05:01:19.331093+0000","mapping_epoch":1568,"log_start":"0'0","ondisk_log_start":"0'0","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":460,"log_dups_size":0,"ondisk_log_size":460,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:37.639076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30146560,"num_objects":460,"num_object_clones":0,"num_object_copies":1380,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":460,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":460,"num_write_kb":29440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"58.9","version":"1572'501","reported_seq":519,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:19.607869+0000","last_change":"2024-09-14T05:01:07.065934+0000","last_active":"2024-09-14T05:01:19.607869+0000","last_peered":"2024-09-14T05:01:19.607869+0000","last_clean":"2024-09-14T05:01:19.607869+0000","last_became_active":"2024-09-14T05:01:07.065651+0000","last_became_peered":"2024-09-14T05:01:07.065651+0000","last_unstale":"2024-09-14T05:01:19.607869+0000","last_undegraded":"2024-09-14T05:01:19.607869+0000","last_fullsized":"2024-09-14T05:01:19.607869+0000","mapping_epoch":1568,"log_start":"0'0","ondisk_log_start":"0'0","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":501,"log_dups_size":0,"ondisk_log_size":501,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:39:34.349107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32833536,"num_objects":501,"num_object_clones":0,"num_object_copies":1503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":501,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":501,"num_write_kb":32064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"58.8","version":"1572'494","reported_seq":512,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:19.599760+0000","last_change":"2024-09-14T05:01:06.753618+0000","last_active":"2024-09-14T05:01:19.599760+0000","last_peered":"2024-09-14T05:01:19.599760+0000","last_clean":"2024-09-14T05:01:19.599760+0000","last_became_active":"2024-09-14T05:01:06.753311+0000","last_became_peered":"2024-09-14T05:01:06.753311+0000","last_unstale":"2024-09-14T05:01:19.599760+0000","last_undegraded":"2024-09-14T05:01:19.599760+0000","last_fullsized":"2024-09-14T05:01:19.599760+0000","mapping_epoch":1568,"log_start":"0'0","ondisk_log_start":"0'0","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":494,"log_dups_size":0,"ondisk_log_size":494,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:42:26.169787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32374784,"num_objects":494,"num_object_clones":0,"num_object_copies":1482,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":494,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":494,"num_write_kb":31616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"58.7","version":"1572'728","reported_seq":747,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:20.635588+0000","last_change":"2024-09-14T05:01:07.067197+0000","last_active":"2024-09-14T05:01:20.635588+0000","last_peered":"2024-09-14T05:01:20.635588+0000","last_clean":"2024-09-14T05:01:20.635588+0000","last_became_active":"2024-09-14T05:01:07.066873+0000","last_became_peered":"2024-09-14T05:01:07.066873+0000","last_unstale":"2024-09-14T05:01:20.635588+0000","last_undegraded":"2024-09-14T05:01:20.635588+0000","last_fullsized":"2024-09-14T05:01:20.635588+0000","mapping_epoch":1568,"log_start":"0'0","ondisk_log_start":"0'0","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":728,"log_dups_size":0,"ondisk_log_size":728,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:45:48.553605+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47710208,"num_objects":728,"num_object_clones":0,"num_object_copies":2184,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":728,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":728,"num_write_kb":46592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"58.6","version":"1572'303","reported_seq":316,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:18.502526+0000","last_change":"2024-09-14T05:01:07.065564+0000","last_active":"2024-09-14T05:01:18.502526+0000","last_peered":"2024-09-14T05:01:18.502526+0000","last_clean":"2024-09-14T05:01:18.502526+0000","last_became_active":"2024-09-14T05:01:07.065347+0000","last_became_peered":"2024-09-14T05:01:07.065347+0000","last_unstale":"2024-09-14T05:01:18.502526+0000","last_undegraded":"2024-09-14T05:01:18.502526+0000","last_fullsized":"2024-09-14T05:01:18.502526+0000","mapping_epoch":1568,"log_start":"0'0","ondisk_log_start":"0'0","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":303,"log_dups_size":0,"ondisk_log_size":303,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:19:44.691536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":19857408,"num_objects":303,"num_object_clones":0,"num_object_copies":909,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":303,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":303,"num_write_kb":19392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"58.5","version":"1572'265","reported_seq":278,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:18.508994+0000","last_change":"2024-09-14T05:01:06.755410+0000","last_active":"2024-09-14T05:01:18.508994+0000","last_peered":"2024-09-14T05:01:18.508994+0000","last_clean":"2024-09-14T05:01:18.508994+0000","last_became_active":"2024-09-14T05:01:06.755015+0000","last_became_peered":"2024-09-14T05:01:06.755015+0000","last_unstale":"2024-09-14T05:01:18.508994+0000","last_undegraded":"2024-09-14T05:01:18.508994+0000","last_fullsized":"2024-09-14T05:01:18.508994+0000","mapping_epoch":1568,"log_start":"0'0","ondisk_log_start":"0'0","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":265,"log_dups_size":0,"ondisk_log_size":265,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:04:44.279865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17367040,"num_objects":265,"num_object_clones":0,"num_object_copies":795,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":265,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":265,"num_write_kb":16960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"58.4","version":"1572'163","reported_seq":176,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:18.034410+0000","last_change":"2024-09-14T05:01:07.066762+0000","last_active":"2024-09-14T05:01:18.034410+0000","last_peered":"2024-09-14T05:01:18.034410+0000","last_clean":"2024-09-14T05:01:18.034410+0000","last_became_active":"2024-09-14T05:01:07.066456+0000","last_became_peered":"2024-09-14T05:01:07.066456+0000","last_unstale":"2024-09-14T05:01:18.034410+0000","last_undegraded":"2024-09-14T05:01:18.034410+0000","last_fullsized":"2024-09-14T05:01:18.034410+0000","mapping_epoch":1568,"log_start":"0'0","ondisk_log_start":"0'0","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":163,"log_dups_size":0,"ondisk_log_size":163,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:08.301174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10682368,"num_objects":163,"num_object_clones":0,"num_object_copies":489,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":163,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":163,"num_write_kb":10432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"58.3","version":"1572'444","reported_seq":463,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:19.335249+0000","last_change":"2024-09-14T05:01:07.067804+0000","last_active":"2024-09-14T05:01:19.335249+0000","last_peered":"2024-09-14T05:01:19.335249+0000","last_clean":"2024-09-14T05:01:19.335249+0000","last_became_active":"2024-09-14T05:01:07.067259+0000","last_became_peered":"2024-09-14T05:01:07.067259+0000","last_unstale":"2024-09-14T05:01:19.335249+0000","last_undegraded":"2024-09-14T05:01:19.335249+0000","last_fullsized":"2024-09-14T05:01:19.335249+0000","mapping_epoch":1568,"log_start":"0'0","ondisk_log_start":"0'0","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":444,"log_dups_size":0,"ondisk_log_size":444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:28:49.829265+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":29097984,"num_objects":444,"num_object_clones":0,"num_object_copies":1332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":444,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":444,"num_write_kb":28416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"58.2","version":"1572'158","reported_seq":170,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.888522+0000","last_change":"2024-09-14T05:01:07.066830+0000","last_active":"2024-09-14T05:01:17.888522+0000","last_peered":"2024-09-14T05:01:17.888522+0000","last_clean":"2024-09-14T05:01:17.888522+0000","last_became_active":"2024-09-14T05:01:07.066435+0000","last_became_peered":"2024-09-14T05:01:07.066435+0000","last_unstale":"2024-09-14T05:01:17.888522+0000","last_undegraded":"2024-09-14T05:01:17.888522+0000","last_fullsized":"2024-09-14T05:01:17.888522+0000","mapping_epoch":1568,"log_start":"0'0","ondisk_log_start":"0'0","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":158,"log_dups_size":0,"ondisk_log_size":158,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:44:03.503451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":10354688,"num_objects":158,"num_object_clones":0,"num_object_copies":474,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":158,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":158,"num_write_kb":10112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"58.1","version":"1572'207","reported_seq":220,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:18.035389+0000","last_change":"2024-09-14T05:01:06.754637+0000","last_active":"2024-09-14T05:01:18.035389+0000","last_peered":"2024-09-14T05:01:18.035389+0000","last_clean":"2024-09-14T05:01:18.035389+0000","last_became_active":"2024-09-14T05:01:06.754414+0000","last_became_peered":"2024-09-14T05:01:06.754414+0000","last_unstale":"2024-09-14T05:01:18.035389+0000","last_undegraded":"2024-09-14T05:01:18.035389+0000","last_fullsized":"2024-09-14T05:01:18.035389+0000","mapping_epoch":1568,"log_start":"0'0","ondisk_log_start":"0'0","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":207,"log_dups_size":0,"ondisk_log_size":207,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:16:30.396684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13565952,"num_objects":207,"num_object_clones":0,"num_object_copies":621,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":207,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":207,"num_write_kb":13248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"58.0","version":"1572'283","reported_seq":296,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:18.493785+0000","last_change":"2024-09-14T05:01:07.066052+0000","last_active":"2024-09-14T05:01:18.493785+0000","last_peered":"2024-09-14T05:01:18.493785+0000","last_clean":"2024-09-14T05:01:18.493785+0000","last_became_active":"2024-09-14T05:01:07.065797+0000","last_became_peered":"2024-09-14T05:01:07.065797+0000","last_unstale":"2024-09-14T05:01:18.493785+0000","last_undegraded":"2024-09-14T05:01:18.493785+0000","last_fullsized":"2024-09-14T05:01:18.493785+0000","mapping_epoch":1568,"log_start":"0'0","ondisk_log_start":"0'0","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":283,"log_dups_size":0,"ondisk_log_size":283,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:46:34.458042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":18546688,"num_objects":283,"num_object_clones":0,"num_object_copies":849,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":283,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":283,"num_write_kb":18112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":2439,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.849743+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T05:01:17.849743+0000","last_peered":"2024-09-14T05:01:17.849743+0000","last_clean":"2024-09-14T05:01:17.849743+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T05:01:17.849743+0000","last_undegraded":"2024-09-14T05:01:17.849743+0000","last_fullsized":"2024-09-14T05:01:17.849743+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":186,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.798244+0000","last_change":"2024-09-14T04:44:57.325109+0000","last_active":"2024-09-14T05:01:12.798244+0000","last_peered":"2024-09-14T05:01:12.798244+0000","last_clean":"2024-09-14T05:01:12.798244+0000","last_became_active":"2024-09-14T04:44:57.324791+0000","last_became_peered":"2024-09-14T04:44:57.324791+0000","last_unstale":"2024-09-14T05:01:12.798244+0000","last_undegraded":"2024-09-14T05:01:12.798244+0000","last_fullsized":"2024-09-14T05:01:12.798244+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:03.593275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":156,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:13.671394+0000","last_change":"2024-09-14T04:49:29.628435+0000","last_active":"2024-09-14T05:01:13.671394+0000","last_peered":"2024-09-14T05:01:13.671394+0000","last_clean":"2024-09-14T05:01:13.671394+0000","last_became_active":"2024-09-14T04:49:29.628081+0000","last_became_peered":"2024-09-14T04:49:29.628081+0000","last_unstale":"2024-09-14T05:01:13.671394+0000","last_undegraded":"2024-09-14T05:01:13.671394+0000","last_fullsized":"2024-09-14T05:01:13.671394+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:01.520334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1060,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.844888+0000","last_change":"2024-09-14T04:43:13.355281+0000","last_active":"2024-09-14T05:01:17.844888+0000","last_peered":"2024-09-14T05:01:17.844888+0000","last_clean":"2024-09-14T05:01:17.844888+0000","last_became_active":"2024-09-14T04:43:13.354967+0000","last_became_peered":"2024-09-14T04:43:13.354967+0000","last_unstale":"2024-09-14T05:01:17.844888+0000","last_undegraded":"2024-09-14T05:01:17.844888+0000","last_fullsized":"2024-09-14T05:01:17.844888+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:23:19.332075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1565,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.798000+0000","last_change":"2024-09-14T04:44:57.342918+0000","last_active":"2024-09-14T05:01:12.798000+0000","last_peered":"2024-09-14T05:01:12.798000+0000","last_clean":"2024-09-14T05:01:12.798000+0000","last_became_active":"2024-09-14T04:44:57.329550+0000","last_became_peered":"2024-09-14T04:44:57.329550+0000","last_unstale":"2024-09-14T05:01:12.798000+0000","last_undegraded":"2024-09-14T05:01:12.798000+0000","last_fullsized":"2024-09-14T05:01:12.798000+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:52:52.435893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":186,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.798432+0000","last_change":"2024-09-14T04:44:57.325537+0000","last_active":"2024-09-14T05:01:12.798432+0000","last_peered":"2024-09-14T05:01:12.798432+0000","last_clean":"2024-09-14T05:01:12.798432+0000","last_became_active":"2024-09-14T04:44:57.325057+0000","last_became_peered":"2024-09-14T04:44:57.325057+0000","last_unstale":"2024-09-14T05:01:12.798432+0000","last_undegraded":"2024-09-14T05:01:12.798432+0000","last_fullsized":"2024-09-14T05:01:12.798432+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:43:23.166708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1580,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.797879+0000","last_change":"2024-09-14T04:44:57.337033+0000","last_active":"2024-09-14T05:01:12.797879+0000","last_peered":"2024-09-14T05:01:12.797879+0000","last_clean":"2024-09-14T05:01:12.797879+0000","last_became_active":"2024-09-14T04:44:57.328437+0000","last_became_peered":"2024-09-14T04:44:57.328437+0000","last_unstale":"2024-09-14T05:01:12.797879+0000","last_undegraded":"2024-09-14T05:01:12.797879+0000","last_fullsized":"2024-09-14T05:01:12.797879+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.109086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":186,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.798573+0000","last_change":"2024-09-14T04:44:57.332243+0000","last_active":"2024-09-14T05:01:12.798573+0000","last_peered":"2024-09-14T05:01:12.798573+0000","last_clean":"2024-09-14T05:01:12.798573+0000","last_became_active":"2024-09-14T04:44:57.330413+0000","last_became_peered":"2024-09-14T04:44:57.330413+0000","last_unstale":"2024-09-14T05:01:12.798573+0000","last_undegraded":"2024-09-14T05:01:12.798573+0000","last_fullsized":"2024-09-14T05:01:12.798573+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:25:01.338549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":295,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.850170+0000","last_change":"2024-09-14T04:48:29.862929+0000","last_active":"2024-09-14T05:01:17.850170+0000","last_peered":"2024-09-14T05:01:17.850170+0000","last_clean":"2024-09-14T05:01:17.850170+0000","last_became_active":"2024-09-14T04:48:29.862581+0000","last_became_peered":"2024-09-14T04:48:29.862581+0000","last_unstale":"2024-09-14T05:01:17.850170+0000","last_undegraded":"2024-09-14T05:01:17.850170+0000","last_fullsized":"2024-09-14T05:01:17.850170+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:16:43.060088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2434,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.845748+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T05:01:17.845748+0000","last_peered":"2024-09-14T05:01:17.845748+0000","last_clean":"2024-09-14T05:01:17.845748+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T05:01:17.845748+0000","last_undegraded":"2024-09-14T05:01:17.845748+0000","last_fullsized":"2024-09-14T05:01:17.845748+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":186,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.798351+0000","last_change":"2024-09-14T04:44:57.325392+0000","last_active":"2024-09-14T05:01:12.798351+0000","last_peered":"2024-09-14T05:01:12.798351+0000","last_clean":"2024-09-14T05:01:12.798351+0000","last_became_active":"2024-09-14T04:44:57.324624+0000","last_became_peered":"2024-09-14T04:44:57.324624+0000","last_unstale":"2024-09-14T05:01:12.798351+0000","last_undegraded":"2024-09-14T05:01:12.798351+0000","last_fullsized":"2024-09-14T05:01:12.798351+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:33:18.810199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2403,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.855191+0000","last_change":"2024-09-14T04:44:57.329113+0000","last_active":"2024-09-14T05:01:17.855191+0000","last_peered":"2024-09-14T05:01:17.855191+0000","last_clean":"2024-09-14T05:01:17.855191+0000","last_became_active":"2024-09-14T04:44:57.328727+0000","last_became_peered":"2024-09-14T04:44:57.328727+0000","last_unstale":"2024-09-14T05:01:17.855191+0000","last_undegraded":"2024-09-14T05:01:17.855191+0000","last_fullsized":"2024-09-14T05:01:17.855191+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:34.939874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":2420,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.854035+0000","last_change":"2024-09-14T04:44:57.329277+0000","last_active":"2024-09-14T05:01:17.854035+0000","last_peered":"2024-09-14T05:01:17.854035+0000","last_clean":"2024-09-14T05:01:17.854035+0000","last_became_active":"2024-09-14T04:44:57.328804+0000","last_became_peered":"2024-09-14T04:44:57.328804+0000","last_unstale":"2024-09-14T05:01:17.854035+0000","last_undegraded":"2024-09-14T05:01:17.854035+0000","last_fullsized":"2024-09-14T05:01:17.854035+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:42:46.055240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2455,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.849799+0000","last_change":"2024-09-14T04:54:52.250695+0000","last_active":"2024-09-14T05:01:17.849799+0000","last_peered":"2024-09-14T05:01:17.849799+0000","last_clean":"2024-09-14T05:01:17.849799+0000","last_became_active":"2024-09-14T04:54:52.250301+0000","last_became_peered":"2024-09-14T04:54:52.250301+0000","last_unstale":"2024-09-14T05:01:17.849799+0000","last_undegraded":"2024-09-14T05:01:17.849799+0000","last_fullsized":"2024-09-14T05:01:17.849799+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:42.060905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2416,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.850361+0000","last_change":"2024-09-14T04:43:13.357092+0000","last_active":"2024-09-14T05:01:17.850361+0000","last_peered":"2024-09-14T05:01:17.850361+0000","last_clean":"2024-09-14T05:01:17.850361+0000","last_became_active":"2024-09-14T04:43:13.356694+0000","last_became_peered":"2024-09-14T04:43:13.356694+0000","last_unstale":"2024-09-14T05:01:17.850361+0000","last_undegraded":"2024-09-14T05:01:17.850361+0000","last_fullsized":"2024-09-14T05:01:17.850361+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:11.167574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1558,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:13.671683+0000","last_change":"2024-09-14T04:07:21.720797+0000","last_active":"2024-09-14T05:01:13.671683+0000","last_peered":"2024-09-14T05:01:13.671683+0000","last_clean":"2024-09-14T05:01:13.671683+0000","last_became_active":"2024-09-14T04:07:21.719331+0000","last_became_peered":"2024-09-14T04:07:21.719331+0000","last_unstale":"2024-09-14T05:01:13.671683+0000","last_undegraded":"2024-09-14T05:01:13.671683+0000","last_fullsized":"2024-09-14T05:01:13.671683+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:35.404023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":205,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.797426+0000","last_change":"2024-09-14T04:43:11.641514+0000","last_active":"2024-09-14T05:01:12.797426+0000","last_peered":"2024-09-14T05:01:12.797426+0000","last_clean":"2024-09-14T05:01:12.797426+0000","last_became_active":"2024-09-14T04:43:11.641151+0000","last_became_peered":"2024-09-14T04:43:11.641151+0000","last_unstale":"2024-09-14T05:01:12.797426+0000","last_undegraded":"2024-09-14T05:01:12.797426+0000","last_fullsized":"2024-09-14T05:01:12.797426+0000","mapping_epoch":1384,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:06:45.045430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2427,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.849931+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T05:01:17.849931+0000","last_peered":"2024-09-14T05:01:17.849931+0000","last_clean":"2024-09-14T05:01:17.849931+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T05:01:17.849931+0000","last_undegraded":"2024-09-14T05:01:17.849931+0000","last_fullsized":"2024-09-14T05:01:17.849931+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.14","version":"349'149","reported_seq":900,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.845648+0000","last_change":"2024-09-14T05:00:39.545797+0000","last_active":"2024-09-14T05:01:17.845648+0000","last_peered":"2024-09-14T05:01:17.845648+0000","last_clean":"2024-09-14T05:01:17.845648+0000","last_became_active":"2024-09-14T05:00:08.501982+0000","last_became_peered":"2024-09-14T05:00:08.501982+0000","last_unstale":"2024-09-14T05:01:17.845648+0000","last_undegraded":"2024-09-14T05:01:17.845648+0000","last_fullsized":"2024-09-14T05:01:17.845648+0000","mapping_epoch":1549,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1550,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:39.545724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:39.545724+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:39:33.377804+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":785,"reported_epoch":1570,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.840076+0000","last_change":"2024-09-14T04:49:29.618477+0000","last_active":"2024-09-14T05:01:12.840076+0000","last_peered":"2024-09-14T05:01:12.840076+0000","last_clean":"2024-09-14T05:01:12.840076+0000","last_became_active":"2024-09-14T04:49:29.618210+0000","last_became_peered":"2024-09-14T04:49:29.618210+0000","last_unstale":"2024-09-14T05:01:12.840076+0000","last_undegraded":"2024-09-14T05:01:12.840076+0000","last_fullsized":"2024-09-14T05:01:12.840076+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:13:23.895512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1564,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.797647+0000","last_change":"2024-09-14T04:42:49.938505+0000","last_active":"2024-09-14T05:01:12.797647+0000","last_peered":"2024-09-14T05:01:12.797647+0000","last_clean":"2024-09-14T05:01:12.797647+0000","last_became_active":"2024-09-14T04:42:49.938180+0000","last_became_peered":"2024-09-14T04:42:49.938180+0000","last_unstale":"2024-09-14T05:01:12.797647+0000","last_undegraded":"2024-09-14T05:01:12.797647+0000","last_fullsized":"2024-09-14T05:01:12.797647+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:05.929479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1582,"reported_epoch":1570,"state":"active+clean","last_fresh":"2024-09-14T05:01:08.164435+0000","last_change":"2024-09-14T04:44:57.346582+0000","last_active":"2024-09-14T05:01:08.164435+0000","last_peered":"2024-09-14T05:01:08.164435+0000","last_clean":"2024-09-14T05:01:08.164435+0000","last_became_active":"2024-09-14T04:44:57.330446+0000","last_became_peered":"2024-09-14T04:44:57.330446+0000","last_unstale":"2024-09-14T05:01:08.164435+0000","last_undegraded":"2024-09-14T05:01:08.164435+0000","last_fullsized":"2024-09-14T05:01:08.164435+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:29.238585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":369,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.853895+0000","last_change":"2024-09-14T04:44:14.480667+0000","last_active":"2024-09-14T05:01:17.853895+0000","last_peered":"2024-09-14T05:01:17.853895+0000","last_clean":"2024-09-14T05:01:17.853895+0000","last_became_active":"2024-09-14T04:44:14.480365+0000","last_became_peered":"2024-09-14T04:44:14.480365+0000","last_unstale":"2024-09-14T05:01:17.853895+0000","last_undegraded":"2024-09-14T05:01:17.853895+0000","last_fullsized":"2024-09-14T05:01:17.853895+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:00.611771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":197,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.842840+0000","last_change":"2024-09-14T04:44:57.341426+0000","last_active":"2024-09-14T05:01:17.842840+0000","last_peered":"2024-09-14T05:01:17.842840+0000","last_clean":"2024-09-14T05:01:17.842840+0000","last_became_active":"2024-09-14T04:44:57.328763+0000","last_became_peered":"2024-09-14T04:44:57.328763+0000","last_unstale":"2024-09-14T05:01:17.842840+0000","last_undegraded":"2024-09-14T05:01:17.842840+0000","last_fullsized":"2024-09-14T05:01:17.842840+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:34:40.066761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":187,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.842463+0000","last_change":"2024-09-14T04:44:57.331337+0000","last_active":"2024-09-14T05:01:17.842463+0000","last_peered":"2024-09-14T05:01:17.842463+0000","last_clean":"2024-09-14T05:01:17.842463+0000","last_became_active":"2024-09-14T04:44:57.329336+0000","last_became_peered":"2024-09-14T04:44:57.329336+0000","last_unstale":"2024-09-14T05:01:17.842463+0000","last_undegraded":"2024-09-14T05:01:17.842463+0000","last_fullsized":"2024-09-14T05:01:17.842463+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:52:35.980869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"349'149","reported_seq":796,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.797744+0000","last_change":"2024-09-14T05:00:31.577572+0000","last_active":"2024-09-14T05:01:12.797744+0000","last_peered":"2024-09-14T05:01:12.797744+0000","last_clean":"2024-09-14T05:01:12.797744+0000","last_became_active":"2024-09-14T05:00:01.425392+0000","last_became_peered":"2024-09-14T05:00:01.425392+0000","last_unstale":"2024-09-14T05:01:12.797744+0000","last_undegraded":"2024-09-14T05:01:12.797744+0000","last_fullsized":"2024-09-14T05:01:12.797744+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1544,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:31.577382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:31.577382+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:08:04.884554+0000","scrub_duration":0.0060191919999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":187,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.798288+0000","last_change":"2024-09-14T04:44:57.329870+0000","last_active":"2024-09-14T05:01:12.798288+0000","last_peered":"2024-09-14T05:01:12.798288+0000","last_clean":"2024-09-14T05:01:12.798288+0000","last_became_active":"2024-09-14T04:44:57.329353+0000","last_became_peered":"2024-09-14T04:44:57.329353+0000","last_unstale":"2024-09-14T05:01:12.798288+0000","last_undegraded":"2024-09-14T05:01:12.798288+0000","last_fullsized":"2024-09-14T05:01:12.798288+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:46.432181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1580,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:13.671713+0000","last_change":"2024-09-14T04:54:52.242207+0000","last_active":"2024-09-14T05:01:13.671713+0000","last_peered":"2024-09-14T05:01:13.671713+0000","last_clean":"2024-09-14T05:01:13.671713+0000","last_became_active":"2024-09-14T04:54:52.241840+0000","last_became_peered":"2024-09-14T04:54:52.241840+0000","last_unstale":"2024-09-14T05:01:13.671713+0000","last_undegraded":"2024-09-14T05:01:13.671713+0000","last_fullsized":"2024-09-14T05:01:13.671713+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:37.899666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":2466,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.849721+0000","last_change":"2024-09-14T04:44:57.344614+0000","last_active":"2024-09-14T05:01:17.849721+0000","last_peered":"2024-09-14T05:01:17.849721+0000","last_clean":"2024-09-14T05:01:17.849721+0000","last_became_active":"2024-09-14T04:44:57.332256+0000","last_became_peered":"2024-09-14T04:44:57.332256+0000","last_unstale":"2024-09-14T05:01:17.849721+0000","last_undegraded":"2024-09-14T05:01:17.849721+0000","last_fullsized":"2024-09-14T05:01:17.849721+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:23.438417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"349'149","reported_seq":966,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.799029+0000","last_change":"2024-09-14T05:00:31.015778+0000","last_active":"2024-09-14T05:01:12.799029+0000","last_peered":"2024-09-14T05:01:12.799029+0000","last_clean":"2024-09-14T05:01:12.799029+0000","last_became_active":"2024-09-14T05:00:01.423833+0000","last_became_peered":"2024-09-14T05:00:01.423833+0000","last_unstale":"2024-09-14T05:01:12.799029+0000","last_undegraded":"2024-09-14T05:01:12.799029+0000","last_fullsized":"2024-09-14T05:01:12.799029+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1544,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:31.015650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:31.015650+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:34:48.213379+0000","scrub_duration":0.0057407420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":775,"reported_epoch":1570,"state":"active+clean","last_fresh":"2024-09-14T05:01:08.164102+0000","last_change":"2024-09-14T04:49:29.616831+0000","last_active":"2024-09-14T05:01:08.164102+0000","last_peered":"2024-09-14T05:01:08.164102+0000","last_clean":"2024-09-14T05:01:08.164102+0000","last_became_active":"2024-09-14T04:49:29.616546+0000","last_became_peered":"2024-09-14T04:49:29.616546+0000","last_unstale":"2024-09-14T05:01:08.164102+0000","last_undegraded":"2024-09-14T05:01:08.164102+0000","last_fullsized":"2024-09-14T05:01:08.164102+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:27.882929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":421,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.855817+0000","last_change":"2024-09-14T04:44:57.330746+0000","last_active":"2024-09-14T05:01:17.855817+0000","last_peered":"2024-09-14T05:01:17.855817+0000","last_clean":"2024-09-14T05:01:17.855817+0000","last_became_active":"2024-09-14T04:44:57.330327+0000","last_became_peered":"2024-09-14T04:44:57.330327+0000","last_unstale":"2024-09-14T05:01:17.855817+0000","last_undegraded":"2024-09-14T05:01:17.855817+0000","last_fullsized":"2024-09-14T05:01:17.855817+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:53.122771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":226,"reported_epoch":1570,"state":"active+clean","last_fresh":"2024-09-14T05:01:08.164600+0000","last_change":"2024-09-14T04:43:13.355296+0000","last_active":"2024-09-14T05:01:08.164600+0000","last_peered":"2024-09-14T05:01:08.164600+0000","last_clean":"2024-09-14T05:01:08.164600+0000","last_became_active":"2024-09-14T04:43:13.354950+0000","last_became_peered":"2024-09-14T04:43:13.354950+0000","last_unstale":"2024-09-14T05:01:08.164600+0000","last_undegraded":"2024-09-14T05:01:08.164600+0000","last_fullsized":"2024-09-14T05:01:08.164600+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:20:08.968818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"349'149","reported_seq":905,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:13.670799+0000","last_change":"2024-09-14T05:00:37.706342+0000","last_active":"2024-09-14T05:01:13.670799+0000","last_peered":"2024-09-14T05:01:13.670799+0000","last_clean":"2024-09-14T05:01:13.670799+0000","last_became_active":"2024-09-14T05:00:01.433252+0000","last_became_peered":"2024-09-14T05:00:01.433252+0000","last_unstale":"2024-09-14T05:01:13.670799+0000","last_undegraded":"2024-09-14T05:01:13.670799+0000","last_fullsized":"2024-09-14T05:01:13.670799+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1544,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:37.706187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:37.706187+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:27:54.986789+0000","scrub_duration":0.018654753,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2422,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.850418+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T05:01:17.850418+0000","last_peered":"2024-09-14T05:01:17.850418+0000","last_clean":"2024-09-14T05:01:17.850418+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T05:01:17.850418+0000","last_undegraded":"2024-09-14T05:01:17.850418+0000","last_fullsized":"2024-09-14T05:01:17.850418+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1593,"reported_epoch":1570,"state":"active+clean","last_fresh":"2024-09-14T05:01:08.164518+0000","last_change":"2024-09-14T04:44:57.331436+0000","last_active":"2024-09-14T05:01:08.164518+0000","last_peered":"2024-09-14T05:01:08.164518+0000","last_clean":"2024-09-14T05:01:08.164518+0000","last_became_active":"2024-09-14T04:44:57.330957+0000","last_became_peered":"2024-09-14T04:44:57.330957+0000","last_unstale":"2024-09-14T05:01:08.164518+0000","last_undegraded":"2024-09-14T05:01:08.164518+0000","last_fullsized":"2024-09-14T05:01:08.164518+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:09.516688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":971,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.845723+0000","last_change":"2024-09-14T04:49:49.460401+0000","last_active":"2024-09-14T05:01:17.845723+0000","last_peered":"2024-09-14T05:01:17.845723+0000","last_clean":"2024-09-14T05:01:17.845723+0000","last_became_active":"2024-09-14T04:49:49.458530+0000","last_became_peered":"2024-09-14T04:49:49.458530+0000","last_unstale":"2024-09-14T05:01:17.845723+0000","last_undegraded":"2024-09-14T05:01:17.845723+0000","last_fullsized":"2024-09-14T05:01:17.845723+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:11:29.143239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"349'149","reported_seq":939,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:13.671460+0000","last_change":"2024-09-14T05:00:41.710202+0000","last_active":"2024-09-14T05:01:13.671460+0000","last_peered":"2024-09-14T05:01:13.671460+0000","last_clean":"2024-09-14T05:01:13.671460+0000","last_became_active":"2024-09-14T05:00:01.424753+0000","last_became_peered":"2024-09-14T05:00:01.424753+0000","last_unstale":"2024-09-14T05:01:13.671460+0000","last_undegraded":"2024-09-14T05:01:13.671460+0000","last_fullsized":"2024-09-14T05:01:13.671460+0000","mapping_epoch":1492,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1544,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:41.710095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:41.710095+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:48:46.305394+0000","scrub_duration":0.0014057430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":377,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.853792+0000","last_change":"2024-09-14T04:44:57.329983+0000","last_active":"2024-09-14T05:01:17.853792+0000","last_peered":"2024-09-14T05:01:17.853792+0000","last_clean":"2024-09-14T05:01:17.853792+0000","last_became_active":"2024-09-14T04:44:57.329597+0000","last_became_peered":"2024-09-14T04:44:57.329597+0000","last_unstale":"2024-09-14T05:01:17.853792+0000","last_undegraded":"2024-09-14T05:01:17.853792+0000","last_fullsized":"2024-09-14T05:01:17.853792+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:45.718039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":915,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.849688+0000","last_change":"2024-09-14T04:49:49.443631+0000","last_active":"2024-09-14T05:01:17.849688+0000","last_peered":"2024-09-14T05:01:17.849688+0000","last_clean":"2024-09-14T05:01:17.849688+0000","last_became_active":"2024-09-14T04:49:49.443313+0000","last_became_peered":"2024-09-14T04:49:49.443313+0000","last_unstale":"2024-09-14T05:01:17.849688+0000","last_undegraded":"2024-09-14T05:01:17.849688+0000","last_fullsized":"2024-09-14T05:01:17.849688+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:06:54.562292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1566,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.798092+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T05:01:12.798092+0000","last_peered":"2024-09-14T05:01:12.798092+0000","last_clean":"2024-09-14T05:01:12.798092+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T05:01:12.798092+0000","last_undegraded":"2024-09-14T05:01:12.798092+0000","last_fullsized":"2024-09-14T05:01:12.798092+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"349'149","reported_seq":897,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.797475+0000","last_change":"2024-09-14T05:00:33.562385+0000","last_active":"2024-09-14T05:01:12.797475+0000","last_peered":"2024-09-14T05:01:12.797475+0000","last_clean":"2024-09-14T05:01:12.797475+0000","last_became_active":"2024-09-14T05:00:01.421507+0000","last_became_peered":"2024-09-14T05:00:01.421507+0000","last_unstale":"2024-09-14T05:01:12.797475+0000","last_undegraded":"2024-09-14T05:01:12.797475+0000","last_fullsized":"2024-09-14T05:01:12.797475+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1544,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:33.562211+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:33.562211+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:32:46.187217+0000","scrub_duration":0.0066960869999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1007,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.845822+0000","last_change":"2024-09-14T04:49:49.459590+0000","last_active":"2024-09-14T05:01:17.845822+0000","last_peered":"2024-09-14T05:01:17.845822+0000","last_clean":"2024-09-14T05:01:17.845822+0000","last_became_active":"2024-09-14T04:49:49.456263+0000","last_became_peered":"2024-09-14T04:49:49.456263+0000","last_unstale":"2024-09-14T05:01:17.845822+0000","last_undegraded":"2024-09-14T05:01:17.845822+0000","last_fullsized":"2024-09-14T05:01:17.845822+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:20:44.666805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":187,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.798651+0000","last_change":"2024-09-14T04:44:57.329652+0000","last_active":"2024-09-14T05:01:12.798651+0000","last_peered":"2024-09-14T05:01:12.798651+0000","last_clean":"2024-09-14T05:01:12.798651+0000","last_became_active":"2024-09-14T04:44:57.328888+0000","last_became_peered":"2024-09-14T04:44:57.328888+0000","last_unstale":"2024-09-14T05:01:12.798651+0000","last_undegraded":"2024-09-14T05:01:12.798651+0000","last_fullsized":"2024-09-14T05:01:12.798651+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:10.170068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1565,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.798016+0000","last_change":"2024-09-14T04:06:46.875340+0000","last_active":"2024-09-14T05:01:12.798016+0000","last_peered":"2024-09-14T05:01:12.798016+0000","last_clean":"2024-09-14T05:01:12.798016+0000","last_became_active":"2024-09-14T04:06:46.874552+0000","last_became_peered":"2024-09-14T04:06:46.874552+0000","last_unstale":"2024-09-14T05:01:12.798016+0000","last_undegraded":"2024-09-14T05:01:12.798016+0000","last_fullsized":"2024-09-14T05:01:12.798016+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:52.257357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"349'149","reported_seq":799,"reported_epoch":1570,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.840097+0000","last_change":"2024-09-14T05:00:02.067572+0000","last_active":"2024-09-14T05:01:12.840097+0000","last_peered":"2024-09-14T05:01:12.840097+0000","last_clean":"2024-09-14T05:01:12.840097+0000","last_became_active":"2024-09-14T05:00:01.416741+0000","last_became_peered":"2024-09-14T05:00:01.416741+0000","last_unstale":"2024-09-14T05:01:12.840097+0000","last_undegraded":"2024-09-14T05:01:12.840097+0000","last_fullsized":"2024-09-14T05:01:12.840097+0000","mapping_epoch":1506,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1544,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:02.067463+0000","last_deep_scrub":"349'149","last_deep_scrub_stamp":"2024-09-14T05:00:02.067463+0000","last_clean_scrub_stamp":"2024-09-14T05:00:02.067463+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:12:45.764862+0000","scrub_duration":0.0033042340000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":369,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.855879+0000","last_change":"2024-09-14T04:44:14.489235+0000","last_active":"2024-09-14T05:01:17.855879+0000","last_peered":"2024-09-14T05:01:17.855879+0000","last_clean":"2024-09-14T05:01:17.855879+0000","last_became_active":"2024-09-14T04:44:14.488950+0000","last_became_peered":"2024-09-14T04:44:14.488950+0000","last_unstale":"2024-09-14T05:01:17.855879+0000","last_undegraded":"2024-09-14T05:01:17.855879+0000","last_fullsized":"2024-09-14T05:01:17.855879+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:54.441144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":925,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.844804+0000","last_change":"2024-09-14T04:49:49.456317+0000","last_active":"2024-09-14T05:01:17.844804+0000","last_peered":"2024-09-14T05:01:17.844804+0000","last_clean":"2024-09-14T05:01:17.844804+0000","last_became_active":"2024-09-14T04:49:49.456029+0000","last_became_peered":"2024-09-14T04:49:49.456029+0000","last_unstale":"2024-09-14T05:01:17.844804+0000","last_undegraded":"2024-09-14T05:01:17.844804+0000","last_fullsized":"2024-09-14T05:01:17.844804+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:46:05.319676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2442,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.844983+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T05:01:17.844983+0000","last_peered":"2024-09-14T05:01:17.844983+0000","last_clean":"2024-09-14T05:01:17.844983+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T05:01:17.844983+0000","last_undegraded":"2024-09-14T05:01:17.844983+0000","last_fullsized":"2024-09-14T05:01:17.844983+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"349'149","reported_seq":791,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:13.670774+0000","last_change":"2024-09-14T05:00:33.570941+0000","last_active":"2024-09-14T05:01:13.670774+0000","last_peered":"2024-09-14T05:01:13.670774+0000","last_clean":"2024-09-14T05:01:13.670774+0000","last_became_active":"2024-09-14T05:00:01.433375+0000","last_became_peered":"2024-09-14T05:00:01.433375+0000","last_unstale":"2024-09-14T05:01:13.670774+0000","last_undegraded":"2024-09-14T05:01:13.670774+0000","last_fullsized":"2024-09-14T05:01:13.670774+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1544,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:33.570779+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:33.570779+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:01:04.260053+0000","scrub_duration":0.010737673999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2443,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.849839+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T05:01:17.849839+0000","last_peered":"2024-09-14T05:01:17.849839+0000","last_clean":"2024-09-14T05:01:17.849839+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T05:01:17.849839+0000","last_undegraded":"2024-09-14T05:01:17.849839+0000","last_fullsized":"2024-09-14T05:01:17.849839+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2464,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.849685+0000","last_change":"2024-09-14T04:54:19.447103+0000","last_active":"2024-09-14T05:01:17.849685+0000","last_peered":"2024-09-14T05:01:17.849685+0000","last_clean":"2024-09-14T05:01:17.849685+0000","last_became_active":"2024-09-14T04:54:19.446748+0000","last_became_peered":"2024-09-14T04:54:19.446748+0000","last_unstale":"2024-09-14T05:01:17.849685+0000","last_undegraded":"2024-09-14T05:01:17.849685+0000","last_fullsized":"2024-09-14T05:01:17.849685+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:00:55.369263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1594,"reported_epoch":1570,"state":"active+clean","last_fresh":"2024-09-14T05:01:08.164252+0000","last_change":"2024-09-14T04:42:49.939790+0000","last_active":"2024-09-14T05:01:08.164252+0000","last_peered":"2024-09-14T05:01:08.164252+0000","last_clean":"2024-09-14T05:01:08.164252+0000","last_became_active":"2024-09-14T04:42:49.939591+0000","last_became_peered":"2024-09-14T04:42:49.939591+0000","last_unstale":"2024-09-14T05:01:08.164252+0000","last_undegraded":"2024-09-14T05:01:08.164252+0000","last_fullsized":"2024-09-14T05:01:08.164252+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:23:16.790593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1558,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:13.670736+0000","last_change":"2024-09-14T04:44:57.326008+0000","last_active":"2024-09-14T05:01:13.670736+0000","last_peered":"2024-09-14T05:01:13.670736+0000","last_clean":"2024-09-14T05:01:13.670736+0000","last_became_active":"2024-09-14T04:44:57.325617+0000","last_became_peered":"2024-09-14T04:44:57.325617+0000","last_unstale":"2024-09-14T05:01:13.670736+0000","last_undegraded":"2024-09-14T05:01:13.670736+0000","last_fullsized":"2024-09-14T05:01:13.670736+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:15.637508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":847,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.798094+0000","last_change":"2024-09-14T04:43:13.354846+0000","last_active":"2024-09-14T05:01:12.798094+0000","last_peered":"2024-09-14T05:01:12.798094+0000","last_clean":"2024-09-14T05:01:12.798094+0000","last_became_active":"2024-09-14T04:43:13.354515+0000","last_became_peered":"2024-09-14T04:43:13.354515+0000","last_unstale":"2024-09-14T05:01:12.798094+0000","last_undegraded":"2024-09-14T05:01:12.798094+0000","last_fullsized":"2024-09-14T05:01:12.798094+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:17.889341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1564,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.797681+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T05:01:12.797681+0000","last_peered":"2024-09-14T05:01:12.797681+0000","last_clean":"2024-09-14T05:01:12.797681+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T05:01:12.797681+0000","last_undegraded":"2024-09-14T05:01:12.797681+0000","last_fullsized":"2024-09-14T05:01:12.797681+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":1921,"reported_epoch":1570,"state":"active+clean","last_fresh":"2024-09-14T05:01:08.164099+0000","last_change":"2024-09-14T05:00:32.949837+0000","last_active":"2024-09-14T05:01:08.164099+0000","last_peered":"2024-09-14T05:01:08.164099+0000","last_clean":"2024-09-14T05:01:08.164099+0000","last_became_active":"2024-09-14T05:00:01.433169+0000","last_became_peered":"2024-09-14T05:00:01.433169+0000","last_unstale":"2024-09-14T05:01:08.164099+0000","last_undegraded":"2024-09-14T05:01:08.164099+0000","last_fullsized":"2024-09-14T05:01:08.164099+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1544,"parent":"0.0","parent_split_bits":0,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:32.949645+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:32.949645+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:04.207148+0000","scrub_duration":0.0087080530000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"349'149","reported_seq":876,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.896064+0000","last_change":"2024-09-14T04:41:39.978080+0000","last_active":"2024-09-14T05:01:12.896064+0000","last_peered":"2024-09-14T05:01:12.896064+0000","last_clean":"2024-09-14T05:01:12.896064+0000","last_became_active":"2024-09-14T04:41:36.929279+0000","last_became_peered":"2024-09-14T04:41:36.929279+0000","last_unstale":"2024-09-14T05:01:12.896064+0000","last_undegraded":"2024-09-14T05:01:12.896064+0000","last_fullsized":"2024-09-14T05:01:12.896064+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.977999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.977999+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:44:06.383827+0000","scrub_duration":0.00192122,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":303,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.845072+0000","last_change":"2024-09-14T04:48:11.040133+0000","last_active":"2024-09-14T05:01:17.845072+0000","last_peered":"2024-09-14T05:01:17.845072+0000","last_clean":"2024-09-14T05:01:17.845072+0000","last_became_active":"2024-09-14T04:48:11.039787+0000","last_became_peered":"2024-09-14T04:48:11.039787+0000","last_unstale":"2024-09-14T05:01:17.845072+0000","last_undegraded":"2024-09-14T05:01:17.845072+0000","last_fullsized":"2024-09-14T05:01:17.845072+0000","mapping_epoch":1428,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:53.520672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2290,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.855012+0000","last_change":"2024-09-14T04:49:29.616443+0000","last_active":"2024-09-14T05:01:17.855012+0000","last_peered":"2024-09-14T05:01:17.855012+0000","last_clean":"2024-09-14T05:01:17.855012+0000","last_became_active":"2024-09-14T04:49:29.616066+0000","last_became_peered":"2024-09-14T04:49:29.616066+0000","last_unstale":"2024-09-14T05:01:17.855012+0000","last_undegraded":"2024-09-14T05:01:17.855012+0000","last_fullsized":"2024-09-14T05:01:17.855012+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:09:15.897957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.b","version":"349'149","reported_seq":925,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.850068+0000","last_change":"2024-09-14T05:00:39.684030+0000","last_active":"2024-09-14T05:01:17.850068+0000","last_peered":"2024-09-14T05:01:17.850068+0000","last_clean":"2024-09-14T05:01:17.850068+0000","last_became_active":"2024-09-14T05:00:08.490295+0000","last_became_peered":"2024-09-14T05:00:08.490295+0000","last_unstale":"2024-09-14T05:01:17.850068+0000","last_undegraded":"2024-09-14T05:01:17.850068+0000","last_fullsized":"2024-09-14T05:01:17.850068+0000","mapping_epoch":1549,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1550,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:39.683936+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:39.683936+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:38:14.405290+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":187,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.798917+0000","last_change":"2024-09-14T04:44:57.326724+0000","last_active":"2024-09-14T05:01:12.798917+0000","last_peered":"2024-09-14T05:01:12.798917+0000","last_clean":"2024-09-14T05:01:12.798917+0000","last_became_active":"2024-09-14T04:44:57.326191+0000","last_became_peered":"2024-09-14T04:44:57.326191+0000","last_unstale":"2024-09-14T05:01:12.798917+0000","last_undegraded":"2024-09-14T05:01:12.798917+0000","last_fullsized":"2024-09-14T05:01:12.798917+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:46.060269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":187,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.798209+0000","last_change":"2024-09-14T04:44:57.328528+0000","last_active":"2024-09-14T05:01:12.798209+0000","last_peered":"2024-09-14T05:01:12.798209+0000","last_clean":"2024-09-14T05:01:12.798209+0000","last_became_active":"2024-09-14T04:44:57.327739+0000","last_became_peered":"2024-09-14T04:44:57.327739+0000","last_unstale":"2024-09-14T05:01:12.798209+0000","last_undegraded":"2024-09-14T05:01:12.798209+0000","last_fullsized":"2024-09-14T05:01:12.798209+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:04:58.294690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"349'149","reported_seq":901,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.844725+0000","last_change":"2024-09-14T05:00:24.275751+0000","last_active":"2024-09-14T05:01:17.844725+0000","last_peered":"2024-09-14T05:01:17.844725+0000","last_clean":"2024-09-14T05:01:17.844725+0000","last_became_active":"2024-09-14T05:00:24.275201+0000","last_became_peered":"2024-09-14T05:00:24.275201+0000","last_unstale":"2024-09-14T05:01:17.844725+0000","last_undegraded":"2024-09-14T05:01:17.844725+0000","last_fullsized":"2024-09-14T05:01:17.844725+0000","mapping_epoch":1555,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1556,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:56.738636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:56.738636+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:41:11.155132+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1576128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":187,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.798997+0000","last_change":"2024-09-14T04:44:57.323436+0000","last_active":"2024-09-14T05:01:12.798997+0000","last_peered":"2024-09-14T05:01:12.798997+0000","last_clean":"2024-09-14T05:01:12.798997+0000","last_became_active":"2024-09-14T04:44:57.323022+0000","last_became_peered":"2024-09-14T04:44:57.323022+0000","last_unstale":"2024-09-14T05:01:12.798997+0000","last_undegraded":"2024-09-14T05:01:12.798997+0000","last_fullsized":"2024-09-14T05:01:12.798997+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:22.775261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1593,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:13.670626+0000","last_change":"2024-09-14T04:44:57.322357+0000","last_active":"2024-09-14T05:01:13.670626+0000","last_peered":"2024-09-14T05:01:13.670626+0000","last_clean":"2024-09-14T05:01:13.670626+0000","last_became_active":"2024-09-14T04:44:57.321990+0000","last_became_peered":"2024-09-14T04:44:57.321990+0000","last_unstale":"2024-09-14T05:01:13.670626+0000","last_undegraded":"2024-09-14T05:01:13.670626+0000","last_fullsized":"2024-09-14T05:01:13.670626+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:09.658008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"349'149","reported_seq":1917,"reported_epoch":1570,"state":"active+clean","last_fresh":"2024-09-14T05:01:08.164064+0000","last_change":"2024-09-14T04:41:53.858076+0000","last_active":"2024-09-14T05:01:08.164064+0000","last_peered":"2024-09-14T05:01:08.164064+0000","last_clean":"2024-09-14T05:01:08.164064+0000","last_became_active":"2024-09-14T04:41:36.931585+0000","last_became_peered":"2024-09-14T04:41:36.931585+0000","last_unstale":"2024-09-14T05:01:08.164064+0000","last_undegraded":"2024-09-14T05:01:08.164064+0000","last_fullsized":"2024-09-14T05:01:08.164064+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:53.857924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:53.857924+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:10:20.658986+0000","scrub_duration":0.0022355919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":187,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.798645+0000","last_change":"2024-09-14T04:44:57.329198+0000","last_active":"2024-09-14T05:01:12.798645+0000","last_peered":"2024-09-14T05:01:12.798645+0000","last_clean":"2024-09-14T05:01:12.798645+0000","last_became_active":"2024-09-14T04:44:57.327754+0000","last_became_peered":"2024-09-14T04:44:57.327754+0000","last_unstale":"2024-09-14T05:01:12.798645+0000","last_undegraded":"2024-09-14T05:01:12.798645+0000","last_fullsized":"2024-09-14T05:01:12.798645+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:10:27.944476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":427,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.850528+0000","last_change":"2024-09-14T04:44:57.348687+0000","last_active":"2024-09-14T05:01:17.850528+0000","last_peered":"2024-09-14T05:01:17.850528+0000","last_clean":"2024-09-14T05:01:17.850528+0000","last_became_active":"2024-09-14T04:44:57.332845+0000","last_became_peered":"2024-09-14T04:44:57.332845+0000","last_unstale":"2024-09-14T05:01:17.850528+0000","last_undegraded":"2024-09-14T05:01:17.850528+0000","last_fullsized":"2024-09-14T05:01:17.850528+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:12:18.762737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"349'149","reported_seq":664,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.798767+0000","last_change":"2024-09-14T05:00:38.958871+0000","last_active":"2024-09-14T05:01:12.798767+0000","last_peered":"2024-09-14T05:01:12.798767+0000","last_clean":"2024-09-14T05:01:12.798767+0000","last_became_active":"2024-09-14T05:00:08.498061+0000","last_became_peered":"2024-09-14T05:00:08.498061+0000","last_unstale":"2024-09-14T05:01:12.798767+0000","last_undegraded":"2024-09-14T05:01:12.798767+0000","last_fullsized":"2024-09-14T05:01:12.798767+0000","mapping_epoch":1549,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1550,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:38.958743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:38.958743+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:12:46.541984+0000","scrub_duration":0.0059942049999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2449,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.850265+0000","last_change":"2024-09-14T04:44:57.333468+0000","last_active":"2024-09-14T05:01:17.850265+0000","last_peered":"2024-09-14T05:01:17.850265+0000","last_clean":"2024-09-14T05:01:17.850265+0000","last_became_active":"2024-09-14T04:44:57.332626+0000","last_became_peered":"2024-09-14T04:44:57.332626+0000","last_unstale":"2024-09-14T05:01:17.850265+0000","last_undegraded":"2024-09-14T05:01:17.850265+0000","last_fullsized":"2024-09-14T05:01:17.850265+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:57:26.366438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1549,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.797982+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T05:01:12.797982+0000","last_peered":"2024-09-14T05:01:12.797982+0000","last_clean":"2024-09-14T05:01:12.797982+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T05:01:12.797982+0000","last_undegraded":"2024-09-14T05:01:12.797982+0000","last_fullsized":"2024-09-14T05:01:12.797982+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"349'149","reported_seq":955,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:13.671617+0000","last_change":"2024-09-14T05:00:46.668539+0000","last_active":"2024-09-14T05:01:13.671617+0000","last_peered":"2024-09-14T05:01:13.671617+0000","last_clean":"2024-09-14T05:01:13.671617+0000","last_became_active":"2024-09-14T05:00:08.482855+0000","last_became_peered":"2024-09-14T05:00:08.482855+0000","last_unstale":"2024-09-14T05:01:13.671617+0000","last_undegraded":"2024-09-14T05:01:13.671617+0000","last_fullsized":"2024-09-14T05:01:13.671617+0000","mapping_epoch":1549,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1550,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:46.668408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:46.668408+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:39:23.694296+0000","scrub_duration":0.0052970259999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1548,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:13.671227+0000","last_change":"2024-09-14T04:06:46.869868+0000","last_active":"2024-09-14T05:01:13.671227+0000","last_peered":"2024-09-14T05:01:13.671227+0000","last_clean":"2024-09-14T05:01:13.671227+0000","last_became_active":"2024-09-14T04:06:46.869126+0000","last_became_peered":"2024-09-14T04:06:46.869126+0000","last_unstale":"2024-09-14T05:01:13.671227+0000","last_undegraded":"2024-09-14T05:01:13.671227+0000","last_fullsized":"2024-09-14T05:01:13.671227+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:41.150138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1531,"reported_epoch":1570,"state":"active+clean","last_fresh":"2024-09-14T05:01:08.164505+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T05:01:08.164505+0000","last_peered":"2024-09-14T05:01:08.164505+0000","last_clean":"2024-09-14T05:01:08.164505+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T05:01:08.164505+0000","last_undegraded":"2024-09-14T05:01:08.164505+0000","last_fullsized":"2024-09-14T05:01:08.164505+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"349'149","reported_seq":899,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.849578+0000","last_change":"2024-09-14T05:00:41.652685+0000","last_active":"2024-09-14T05:01:17.849578+0000","last_peered":"2024-09-14T05:01:17.849578+0000","last_clean":"2024-09-14T05:01:17.849578+0000","last_became_active":"2024-09-14T05:00:08.504510+0000","last_became_peered":"2024-09-14T05:00:08.504510+0000","last_unstale":"2024-09-14T05:01:17.849578+0000","last_undegraded":"2024-09-14T05:01:17.849578+0000","last_fullsized":"2024-09-14T05:01:17.849578+0000","mapping_epoch":1549,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1550,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:41.652639+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:41.652639+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:11:32.536085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":187,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.798445+0000","last_change":"2024-09-14T04:44:57.332112+0000","last_active":"2024-09-14T05:01:12.798445+0000","last_peered":"2024-09-14T05:01:12.798445+0000","last_clean":"2024-09-14T05:01:12.798445+0000","last_became_active":"2024-09-14T04:44:57.331575+0000","last_became_peered":"2024-09-14T04:44:57.331575+0000","last_unstale":"2024-09-14T05:01:12.798445+0000","last_undegraded":"2024-09-14T05:01:12.798445+0000","last_fullsized":"2024-09-14T05:01:12.798445+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:50:00.391283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1579,"reported_epoch":1570,"state":"active+clean","last_fresh":"2024-09-14T05:01:08.164575+0000","last_change":"2024-09-14T04:44:57.351460+0000","last_active":"2024-09-14T05:01:08.164575+0000","last_peered":"2024-09-14T05:01:08.164575+0000","last_clean":"2024-09-14T05:01:08.164575+0000","last_became_active":"2024-09-14T04:44:57.334662+0000","last_became_peered":"2024-09-14T04:44:57.334662+0000","last_unstale":"2024-09-14T05:01:08.164575+0000","last_undegraded":"2024-09-14T05:01:08.164575+0000","last_fullsized":"2024-09-14T05:01:08.164575+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:02.399062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.d","version":"349'149","reported_seq":814,"reported_epoch":1570,"state":"active+clean","last_fresh":"2024-09-14T05:01:08.164368+0000","last_change":"2024-09-14T05:00:35.974699+0000","last_active":"2024-09-14T05:01:08.164368+0000","last_peered":"2024-09-14T05:01:08.164368+0000","last_clean":"2024-09-14T05:01:08.164368+0000","last_became_active":"2024-09-14T05:00:08.484099+0000","last_became_peered":"2024-09-14T05:00:08.484099+0000","last_unstale":"2024-09-14T05:01:08.164368+0000","last_undegraded":"2024-09-14T05:01:08.164368+0000","last_fullsized":"2024-09-14T05:01:08.164368+0000","mapping_epoch":1549,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1550,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:35.974599+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:35.974599+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:32:28.503626+0000","scrub_duration":0.0082975730000000008,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1534,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.797553+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T05:01:12.797553+0000","last_peered":"2024-09-14T05:01:12.797553+0000","last_clean":"2024-09-14T05:01:12.797553+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T05:01:12.797553+0000","last_undegraded":"2024-09-14T05:01:12.797553+0000","last_fullsized":"2024-09-14T05:01:12.797553+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1591,"reported_epoch":1570,"state":"active+clean","last_fresh":"2024-09-14T05:01:08.164600+0000","last_change":"2024-09-14T04:44:57.349690+0000","last_active":"2024-09-14T05:01:08.164600+0000","last_peered":"2024-09-14T05:01:08.164600+0000","last_clean":"2024-09-14T05:01:08.164600+0000","last_became_active":"2024-09-14T04:44:57.333579+0000","last_became_peered":"2024-09-14T04:44:57.333579+0000","last_unstale":"2024-09-14T05:01:08.164600+0000","last_undegraded":"2024-09-14T05:01:08.164600+0000","last_fullsized":"2024-09-14T05:01:08.164600+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:17:55.265141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"349'149","reported_seq":753,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.797675+0000","last_change":"2024-09-14T05:00:34.592195+0000","last_active":"2024-09-14T05:01:12.797675+0000","last_peered":"2024-09-14T05:01:12.797675+0000","last_clean":"2024-09-14T05:01:12.797675+0000","last_became_active":"2024-09-14T05:00:08.490541+0000","last_became_peered":"2024-09-14T05:00:08.490541+0000","last_unstale":"2024-09-14T05:01:12.797675+0000","last_undegraded":"2024-09-14T05:01:12.797675+0000","last_fullsized":"2024-09-14T05:01:12.797675+0000","mapping_epoch":1549,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1550,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:34.592056+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:34.592056+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:46:54.217254+0000","scrub_duration":0.0046679920000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1688,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:13.671291+0000","last_change":"2024-09-14T05:00:21.251775+0000","last_active":"2024-09-14T05:01:13.671291+0000","last_peered":"2024-09-14T05:01:13.671291+0000","last_clean":"2024-09-14T05:01:13.671291+0000","last_became_active":"2024-09-14T05:00:21.228875+0000","last_became_peered":"2024-09-14T05:00:21.228875+0000","last_unstale":"2024-09-14T05:01:13.671291+0000","last_undegraded":"2024-09-14T05:01:13.671291+0000","last_fullsized":"2024-09-14T05:01:13.671291+0000","mapping_epoch":1552,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:15:41.207466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":80,"reported_epoch":1570,"state":"active+clean","last_fresh":"2024-09-14T05:01:08.164444+0000","last_change":"2024-09-14T04:54:52.248439+0000","last_active":"2024-09-14T05:01:08.164444+0000","last_peered":"2024-09-14T05:01:08.164444+0000","last_clean":"2024-09-14T05:01:08.164444+0000","last_became_active":"2024-09-14T04:54:52.248235+0000","last_became_peered":"2024-09-14T04:54:52.248235+0000","last_unstale":"2024-09-14T05:01:08.164444+0000","last_undegraded":"2024-09-14T05:01:08.164444+0000","last_fullsized":"2024-09-14T05:01:08.164444+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:22:09.548139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"349'149","reported_seq":817,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.798542+0000","last_change":"2024-09-14T05:00:35.941366+0000","last_active":"2024-09-14T05:01:12.798542+0000","last_peered":"2024-09-14T05:01:12.798542+0000","last_clean":"2024-09-14T05:01:12.798542+0000","last_became_active":"2024-09-14T05:00:08.498035+0000","last_became_peered":"2024-09-14T05:00:08.498035+0000","last_unstale":"2024-09-14T05:01:12.798542+0000","last_undegraded":"2024-09-14T05:01:12.798542+0000","last_fullsized":"2024-09-14T05:01:12.798542+0000","mapping_epoch":1549,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1550,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:35.941252+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:35.941252+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:12:45.217998+0000","scrub_duration":0.0056267849999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1553,"reported_epoch":1570,"state":"active+clean","last_fresh":"2024-09-14T05:01:08.164171+0000","last_change":"2024-09-14T04:44:57.332511+0000","last_active":"2024-09-14T05:01:08.164171+0000","last_peered":"2024-09-14T05:01:08.164171+0000","last_clean":"2024-09-14T05:01:08.164171+0000","last_became_active":"2024-09-14T04:44:57.332166+0000","last_became_peered":"2024-09-14T04:44:57.332166+0000","last_unstale":"2024-09-14T05:01:08.164171+0000","last_undegraded":"2024-09-14T05:01:08.164171+0000","last_fullsized":"2024-09-14T05:01:08.164171+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:53.984639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2442,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.849709+0000","last_change":"2024-09-14T04:54:52.250655+0000","last_active":"2024-09-14T05:01:17.849709+0000","last_peered":"2024-09-14T05:01:17.849709+0000","last_clean":"2024-09-14T05:01:17.849709+0000","last_became_active":"2024-09-14T04:54:52.250317+0000","last_became_peered":"2024-09-14T04:54:52.250317+0000","last_unstale":"2024-09-14T05:01:17.849709+0000","last_undegraded":"2024-09-14T05:01:17.849709+0000","last_fullsized":"2024-09-14T05:01:17.849709+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:12.985406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.10","version":"349'149","reported_seq":664,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.797426+0000","last_change":"2024-09-14T05:00:36.569646+0000","last_active":"2024-09-14T05:01:12.797426+0000","last_peered":"2024-09-14T05:01:12.797426+0000","last_clean":"2024-09-14T05:01:12.797426+0000","last_became_active":"2024-09-14T05:00:08.482730+0000","last_became_peered":"2024-09-14T05:00:08.482730+0000","last_unstale":"2024-09-14T05:01:12.797426+0000","last_undegraded":"2024-09-14T05:01:12.797426+0000","last_fullsized":"2024-09-14T05:01:12.797426+0000","mapping_epoch":1549,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1550,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:36.569513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:36.569513+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:39:59.050548+0000","scrub_duration":0.0087120909999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2428,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.850512+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T05:01:17.850512+0000","last_peered":"2024-09-14T05:01:17.850512+0000","last_clean":"2024-09-14T05:01:17.850512+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T05:01:17.850512+0000","last_undegraded":"2024-09-14T05:01:17.850512+0000","last_fullsized":"2024-09-14T05:01:17.850512+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1546,"reported_epoch":1571,"state":"active+clean","last_fresh":"2024-09-14T05:01:12.798006+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T05:01:12.798006+0000","last_peered":"2024-09-14T05:01:12.798006+0000","last_clean":"2024-09-14T05:01:12.798006+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T05:01:12.798006+0000","last_undegraded":"2024-09-14T05:01:12.798006+0000","last_fullsized":"2024-09-14T05:01:12.798006+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.11","version":"349'149","reported_seq":700,"reported_epoch":1572,"state":"active+clean","last_fresh":"2024-09-14T05:01:17.849397+0000","last_change":"2024-09-14T05:00:40.678075+0000","last_active":"2024-09-14T05:01:17.849397+0000","last_peered":"2024-09-14T05:01:17.849397+0000","last_clean":"2024-09-14T05:01:17.849397+0000","last_became_active":"2024-09-14T05:00:08.504082+0000","last_became_peered":"2024-09-14T05:00:08.504082+0000","last_unstale":"2024-09-14T05:01:17.849397+0000","last_undegraded":"2024-09-14T05:01:17.849397+0000","last_fullsized":"2024-09-14T05:01:17.849397+0000","mapping_epoch":1549,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1550,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:40.677843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:40.677843+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:36:50.392923+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1582,"reported_epoch":1570,"state":"active+clean","last_fresh":"2024-09-14T05:01:08.164323+0000","last_change":"2024-09-14T04:44:57.339993+0000","last_active":"2024-09-14T05:01:08.164323+0000","last_peered":"2024-09-14T05:01:08.164323+0000","last_clean":"2024-09-14T05:01:08.164323+0000","last_became_active":"2024-09-14T04:44:57.330472+0000","last_became_peered":"2024-09-14T04:44:57.330472+0000","last_unstale":"2024-09-14T05:01:08.164323+0000","last_undegraded":"2024-09-14T05:01:08.164323+0000","last_fullsized":"2024-09-14T05:01:08.164323+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:19:27.046988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2129,"reported_epoch":1570,"state":"active+clean","last_fresh":"2024-09-14T05:01:08.164381+0000","last_change":"2024-09-14T04:49:49.449687+0000","last_active":"2024-09-14T05:01:08.164381+0000","last_peered":"2024-09-14T05:01:08.164381+0000","last_clean":"2024-09-14T05:01:08.164381+0000","last_became_active":"2024-09-14T04:49:49.449144+0000","last_became_peered":"2024-09-14T04:49:49.449144+0000","last_unstale":"2024-09-14T05:01:08.164381+0000","last_undegraded":"2024-09-14T05:01:08.164381+0000","last_fullsized":"2024-09-14T05:01:08.164381+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:59:06.779730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":58,"num_pg":16,"stat_sum":{"num_bytes":463732736,"num_objects":7076,"num_object_clones":0,"num_object_copies":21228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":7076,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7076,"num_write_kb":452864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":980615168,"data_stored":980615168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":7076,"ondisk_log_size":7076,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1576128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3129,"ondisk_log_size":3129,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":5070,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4250,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223202124,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":861160,"kb_used_data":17952,"kb_used_omap":1,"kb_used_meta":843198,"kb_avail":92883992,"statfs":{"total":95995035648,"available":95113207808,"internally_reserved":0,"allocated":18382848,"data_stored":12714223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":863435211},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463760,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":932836,"kb_used_data":63388,"kb_used_omap":1,"kb_used_meta":869438,"kb_avail":92812316,"statfs":{"total":95995035648,"available":95039811584,"internally_reserved":0,"allocated":64909312,"data_stored":59245200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":890304967},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725397,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":909656,"kb_used_data":26256,"kb_used_omap":1,"kb_used_meta":883390,"kb_avail":92835496,"statfs":{"total":95995035648,"available":95063547904,"internally_reserved":0,"allocated":26886144,"data_stored":21229602,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":904591814},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":1392,"seq":5978594476243,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":500028,"kb_used_data":75952,"kb_used_omap":117,"kb_used_meta":423946,"kb_avail":93245124,"statfs":{"total":95995035648,"available":95483006976,"internally_reserved":0,"allocated":77774848,"data_stored":72099218,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120638,"internal_metadata":434120898},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772075138,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":938204,"kb_used_data":13456,"kb_used_omap":71,"kb_used_meta":924664,"kb_avail":92806948,"statfs":{"total":95995035648,"available":95034314752,"internally_reserved":0,"allocated":13778944,"data_stored":8122999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":946856886},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707304074,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1041448,"kb_used_data":315232,"kb_used_omap":68,"kb_used_meta":726139,"kb_avail":92703704,"statfs":{"total":95995035648,"available":94928592896,"internally_reserved":0,"allocated":322797568,"data_stored":317133970,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":743566544},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872729233,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":917924,"kb_used_data":268120,"kb_used_omap":67,"kb_used_meta":649724,"kb_avail":92827228,"statfs":{"total":95995035648,"available":95055081472,"internally_reserved":0,"allocated":274554880,"data_stored":268898333,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":665317733},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056283,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":873384,"kb_used_data":87708,"kb_used_omap":74,"kb_used_meta":785589,"kb_avail":92871768,"statfs":{"total":95995035648,"available":95100690432,"internally_reserved":0,"allocated":89812992,"data_stored":84152934,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":804443829},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1901,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":58,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":121307136,"data_stored":121307136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":58,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":294780928,"data_stored":294780928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":58,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":319422464,"data_stored":319422464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":58,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":3801088,"data_stored":3801088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":58,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":111607808,"data_stored":111607808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":58,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":28770304,"data_stored":28770304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":58,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":74186752,"data_stored":74186752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":58,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":26738688,"data_stored":26738688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T05:01:26.599 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd pool set unique_pool_5 pgp_num 16 2024-09-14T05:01:26.633 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:01:26.682 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:26.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:26.740 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:26.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:26.841 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:26.886 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:26.887 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:26.887 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:26.988 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:27.017 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:27.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:27.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:27.042 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 10 16 25743 25727 160.774 123.562 0.0022276 0.00619087 2024-09-14T05:01:27.119 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:27.158 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:27.159 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:27.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:27.260 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:27.282 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:27.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:27.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:27.383 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:27.421 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:27.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:27.422 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:27.523 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:27.550 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:27.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:27.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:27.652 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:27.671 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:27 smithi039 ceph-mon[106470]: pgmap v3096: 109 pgs: 109 active+clean; 1.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2024-09-14T05:01:27.671 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:27 smithi039 ceph-mon[111235]: pgmap v3096: 109 pgs: 109 active+clean; 1.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2024-09-14T05:01:27.678 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:27.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:27.679 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:27.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:27 smithi129 ceph-mon[101316]: pgmap v3096: 109 pgs: 109 active+clean; 1.2 GiB data, 9.1 GiB used, 706 GiB / 715 GiB avail; 125 MiB/s wr, 2.00k op/s 2024-09-14T05:01:27.780 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:27.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:27.818 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:27.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:27.920 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:27.948 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:27.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:27.949 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:28.042 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 11 16 27755 27739 157.589 125.75 0.00563373 0.00633993 2024-09-14T05:01:28.050 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:28.078 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:28.079 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:28.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:28.161 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:01:28.179 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:28.221 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:28.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:28.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:28.323 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:28.351 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:28.352 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:28.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:28.453 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:28.502 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:28.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:28.503 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:28.604 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:28.632 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:28.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:28.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:28.734 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:28.758 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:28.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:28.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:28.860 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:28.885 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:28.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:28.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:28.987 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:29.008 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:29.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:29.009 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:29.039 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:01:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:01:28] "GET /metrics HTTP/1.1" 200 36729 "" "Prometheus/2.51.0" 2024-09-14T05:01:29.042 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 12 16 29261 29245 152.3 94.125 0.00195754 0.00647031 2024-09-14T05:01:29.110 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:29.133 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:29.134 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:29.134 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:29.235 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:29.259 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:29.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:29.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:29.360 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:29.397 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:29.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:29.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:29.499 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:29.529 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:29.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:29.530 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:29.631 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:29.661 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:29.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:29.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:29.763 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:29.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:29 smithi129 ceph-mon[101316]: pgmap v3097: 109 pgs: 109 active+clean; 1.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2024-09-14T05:01:29.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:29 smithi039 ceph-mon[106470]: pgmap v3097: 109 pgs: 109 active+clean; 1.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2024-09-14T05:01:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:29 smithi039 ceph-mon[111235]: pgmap v3097: 109 pgs: 109 active+clean; 1.3 GiB data, 9.9 GiB used, 705 GiB / 715 GiB avail; 128 MiB/s wr, 2.04k op/s 2024-09-14T05:01:29.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:29.802 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:29.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:29.903 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:29.935 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:29.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:29.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:30.037 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:30.044 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 13 15 30848 30833 148.204 99.25 0.00855368 0.00674187 2024-09-14T05:01:30.075 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:30.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:30.076 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:30.176 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:30.208 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:30.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:30.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:30.310 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:30.347 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:30.347 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:30.348 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:30.448 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:30.490 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:30.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:30.491 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:30.592 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:30.618 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:30.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:30.619 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:30.720 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:30.748 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:30.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:30.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:30.849 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:30.880 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:30.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:30.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:30.982 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:31.010 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:31.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:31.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:31.044 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 14 16 32619 32603 145.519 110.625 0.0064567 0.00686657 2024-09-14T05:01:31.112 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:31.140 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:31.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:31.141 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:31.242 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:31.271 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:31.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:31.272 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:31.373 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:31.410 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:31.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:31.411 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:31.512 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:31.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:31 smithi129 ceph-mon[101316]: pgmap v3098: 109 pgs: 109 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 142 MiB/s wr, 2.28k op/s 2024-09-14T05:01:31.539 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:31 smithi039 ceph-mon[106470]: pgmap v3098: 109 pgs: 109 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 142 MiB/s wr, 2.28k op/s 2024-09-14T05:01:31.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:31 smithi039 ceph-mon[111235]: pgmap v3098: 109 pgs: 109 active+clean; 1.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 142 MiB/s wr, 2.28k op/s 2024-09-14T05:01:31.548 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:31.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:31.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:31.650 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:31.677 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:31.678 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:31.678 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:31.779 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:31.811 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:31.811 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:31.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:31.913 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:31.934 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:31.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:31.935 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:32.036 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:32.045 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 15 16 34165 34149 142.253 96.625 0.0077961 0.00702318 2024-09-14T05:01:32.070 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:32.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:32.071 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:32.172 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:32.194 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:32.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:32.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:32.296 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:32.324 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:32.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:32.325 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:32.426 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:32.465 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:32.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:32.466 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:32.567 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:32.605 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:32.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:32.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:32.707 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:32.736 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:32.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:32.737 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:32.838 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:32.866 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:32.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:32.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:32.967 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:32.997 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:32.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:32.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:33.045 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 16 16 35838 35822 139.897 104.562 0.00554693 0.00714134 2024-09-14T05:01:33.098 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:33.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:33.121 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:33.121 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:33.222 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:33.256 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:33.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:33.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:33.357 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:33.380 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:33.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:33.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:33.481 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:33.504 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:33.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:33.504 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:33.605 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:33.645 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:33.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:33.646 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:33.747 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:33.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:33 smithi129 ceph-mon[101316]: pgmap v3099: 109 pgs: 109 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 132 MiB/s wr, 2.12k op/s 2024-09-14T05:01:33.774 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:33.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:33.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:33.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:33 smithi039 ceph-mon[106470]: pgmap v3099: 109 pgs: 109 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 132 MiB/s wr, 2.12k op/s 2024-09-14T05:01:33.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:33 smithi039 ceph-mon[111235]: pgmap v3099: 109 pgs: 109 active+clean; 1.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 132 MiB/s wr, 2.12k op/s 2024-09-14T05:01:33.876 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:33.905 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:33.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:33.907 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:34.007 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:34.033 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:34.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:34.034 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:34.045 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 17 16 37005 36989 135.959 72.9375 0.00218848 0.00733673 2024-09-14T05:01:34.135 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:34.162 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:34.163 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:34.163 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:34.264 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:34.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:34.286 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:34.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:34.386 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:34.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:34.427 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:34.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:34.528 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:34.555 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:34.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:34.556 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:34.657 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:34.695 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:34.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:34.696 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:34.797 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:34.824 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:34.825 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:34.825 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:34.926 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:34.950 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:34.951 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:34.952 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:35.045 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 18 16 38578 38562 133.867 98.3125 0.00510404 0.00746436 2024-09-14T05:01:35.052 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:35.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:35.084 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:35.084 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:35.185 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:35.225 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:35.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:35.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:35.327 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:35.348 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:35.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:35.349 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:35.450 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:35.478 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:35.479 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:35.479 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:35.580 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:35.601 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:35.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:35.602 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:35.702 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:35.728 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:35.729 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:35.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:35.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:35 smithi129 ceph-mon[101316]: pgmap v3100: 109 pgs: 109 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-09-14T05:01:35.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:35 smithi039 ceph-mon[106470]: pgmap v3100: 109 pgs: 109 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-09-14T05:01:35.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:35 smithi039 ceph-mon[111235]: pgmap v3100: 109 pgs: 109 active+clean; 2.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 133 MiB/s wr, 2.12k op/s 2024-09-14T05:01:35.830 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:35.857 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:35.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:35.858 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:35.958 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:35.991 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:35.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:35.992 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:36.045 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 19 16 39863 39847 131.048 80.3125 0.00508913 0.00762484 2024-09-14T05:01:36.093 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:36.119 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:36.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:36.120 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:36.221 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:36.259 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:36.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:36.260 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:36.361 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:36.390 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:36.391 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:36.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:36.492 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:36.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:36.522 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:36.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:36.623 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:36.662 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:36.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:36.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:36.764 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:36.803 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:36.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:36.805 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:36.905 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:36.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:36.943 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:36.944 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:37.044 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:37.045 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:2024-09-14T05:01:37.045754+0000 min lat: 0.00100424 max lat: 0.310727 avg lat: 0.00771657 2024-09-14T05:01:37.046 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T05:01:37.046 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 20 16 41464 41448 129.498 100.062 0.00572469 0.00771657 2024-09-14T05:01:37.097 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:37.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:37.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:37.199 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:37.225 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:37.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:37.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:37.327 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:37.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:37.378 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:37.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:37.479 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:37.516 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:37.517 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:37.517 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:37.618 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:37.654 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:37.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:37.655 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:37.756 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:37.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:37 smithi129 ceph-mon[101316]: pgmap v3101: 109 pgs: 109 active+clean; 2.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s 2024-09-14T05:01:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:01:37.784 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:37.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:37.785 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:37.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:37 smithi039 ceph-mon[106470]: pgmap v3101: 109 pgs: 109 active+clean; 2.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s 2024-09-14T05:01:37.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:01:37.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:37 smithi039 ceph-mon[111235]: pgmap v3101: 109 pgs: 109 active+clean; 2.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 131 MiB/s wr, 2.09k op/s 2024-09-14T05:01:37.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:01:37.886 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:37.912 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:37.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:37.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:38.014 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:38.040 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:38.041 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:38.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:38.045 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 21 15 43122 43107 128.269 103.688 0.00681594 0.00779033 2024-09-14T05:01:38.142 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:38.185 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:38.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:38.186 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:38.287 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:38.316 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:38.317 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:38.317 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:38.418 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:38.457 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:38.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:38.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:38.559 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:38.611 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:38.611 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:38.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:38.713 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:38.741 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:38.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:38.742 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:38.843 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:38.887 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:38.888 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:38.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:38.989 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:39.027 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:39.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:39.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:39.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:01:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:01:38] "GET /metrics HTTP/1.1" 200 36736 "" "Prometheus/2.51.0" 2024-09-14T05:01:39.046 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 22 16 44694 44678 126.901 98.1875 0.0114672 0.00787367 2024-09-14T05:01:39.129 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:39.143 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T05:01:39.143 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [], 2024-09-14T05:01:39.143 INFO:teuthology.orchestra.run.smithi039.stdout: "complaint_time": 30, 2024-09-14T05:01:39.143 INFO:teuthology.orchestra.run.smithi039.stdout: "num_blocked_ops": 0 2024-09-14T05:01:39.143 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T05:01:39.169 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:39.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:39.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:39.271 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:39.314 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:39.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:39.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:39.416 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:39.457 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:39.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:39.458 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:39.559 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:39.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:39.586 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:39.586 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:39.663 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T05:01:39.663 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T05:01:39.663 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:39 smithi039 ceph-mon[106470]: pgmap v3102: 109 pgs: 109 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-09-14T05:01:39.663 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:39 smithi039 ceph-mon[111235]: pgmap v3102: 109 pgs: 109 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-09-14T05:01:39.687 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:39.700 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T05:01:39.700 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T05:01:39.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:39.713 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:39.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:39.736 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T05:01:39.736 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T05:01:39.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:39 smithi129 ceph-mon[101316]: pgmap v3102: 109 pgs: 109 active+clean; 2.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s 2024-09-14T05:01:39.772 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T05:01:39.772 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T05:01:39.803 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T05:01:39.804 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T05:01:39.814 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:39.832 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T05:01:39.832 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T05:01:39.838 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:39.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:39.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:39.866 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T05:01:39.866 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T05:01:39.903 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T05:01:39.903 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T05:01:39.940 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:39.985 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:39.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:39.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:40.046 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 23 16 46250 46234 125.612 97.25 0.0145652 0.00795609 2024-09-14T05:01:40.087 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:40.117 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:40.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:40.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:40.219 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:40.259 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:40.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:40.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:40.361 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:40.399 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:40.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:40.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:40.501 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:40.533 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:40.533 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:40.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:40.635 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:40.676 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:40.676 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:40.677 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:40.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:40 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3408892120' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:01:40.778 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:40.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:40 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3408892120' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:01:40.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:40 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3408892120' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:01:40.818 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:40.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:40.819 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:40.919 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:40.966 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:40.969 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:40.969 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:41.046 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 24 16 47797 47781 124.407 96.6875 0.0058181 0.00803316 2024-09-14T05:01:41.070 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:41.098 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:41.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:41.099 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:41.199 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:41.208 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-14T05:01:41.238 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:41.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:41.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:41.339 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:41.365 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:41.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:41.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:41.416 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:41 smithi039 ceph-mon[106470]: pgmap v3103: 109 pgs: 109 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2024-09-14T05:01:41.466 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:41.490 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:41.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:41.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:41.592 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:41.636 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:41.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:41.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:41.738 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:41.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:41 smithi129 ceph-mon[101316]: pgmap v3103: 109 pgs: 109 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2024-09-14T05:01:41.779 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:41.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:41.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:41.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:41 smithi039 ceph-mon[111235]: pgmap v3103: 109 pgs: 109 active+clean; 2.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 117 MiB/s wr, 1.87k op/s 2024-09-14T05:01:41.881 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:41.909 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:41.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:41.910 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:42.011 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:42.040 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:42.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:42.041 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:42.046 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 25 16 49254 49238 123.073 91.0625 0.00852447 0.00811998 2024-09-14T05:01:42.141 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:42.167 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:42.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:42.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:42.265 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:01:42.269 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:42.293 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:42.294 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:42.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:42.395 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:42.424 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:42.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:42.425 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:42.435 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is unset 2024-09-14T05:01:42.456 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:42 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/333454550' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:01:42.457 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:42 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:01:42.457 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:42 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/333454550' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:01:42.457 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:42 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:01:42.525 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:42.567 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:42.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:42.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:42.669 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:42.712 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:42.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:42.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:42.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:42 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/333454550' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:01:42.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:42 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:01:42.814 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:42.838 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:42.838 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:42.839 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:42.939 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:42.966 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:42.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:42.967 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:43.046 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 26 16 50765 50749 121.971 94.4375 0.00955609 0.00819394 2024-09-14T05:01:43.067 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:43.092 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:43.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:43.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:43.194 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:43.225 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:43.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:43.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:43.327 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:43.367 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:43.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:43.368 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:43.469 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:43.511 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:43.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:43.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:43.613 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:43.641 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:43.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:43.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:43.743 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:43.770 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:43.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:43 smithi129 ceph-mon[101316]: pgmap v3104: 109 pgs: 109 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-14T05:01:43.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:43 smithi129 ceph-mon[101316]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T05:01:43.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:43 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T05:01:43.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:43 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T05:01:43.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:43 smithi129 ceph-mon[101316]: osdmap e1573: 8 total, 8 up, 8 in 2024-09-14T05:01:43.771 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:43.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:43.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:43 smithi039 ceph-mon[106470]: pgmap v3104: 109 pgs: 109 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-14T05:01:43.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:43 smithi039 ceph-mon[106470]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T05:01:43.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:43 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T05:01:43.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:43 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T05:01:43.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:43 smithi039 ceph-mon[106470]: osdmap e1573: 8 total, 8 up, 8 in 2024-09-14T05:01:43.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:43 smithi039 ceph-mon[111235]: pgmap v3104: 109 pgs: 109 active+clean; 2.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 92 MiB/s wr, 1.47k op/s 2024-09-14T05:01:43.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:43 smithi039 ceph-mon[111235]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T05:01:43.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:43 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T05:01:43.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:43 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T05:01:43.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:43 smithi039 ceph-mon[111235]: osdmap e1573: 8 total, 8 up, 8 in 2024-09-14T05:01:43.873 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:43.899 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:43.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:43.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:44.001 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:44.026 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:44.027 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:44.027 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:44.046 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 27 16 52130 52114 120.613 85.3125 0.00880478 0.00828499 2024-09-14T05:01:44.127 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:44.150 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:44.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:44.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:44.250 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T05:01:44.251 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:44.273 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:44.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:44.274 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:44.375 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:44.431 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:44.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:44.432 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:44.533 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:44.592 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:44.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:44.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:44.694 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:44.737 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:44.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:44.740 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:44.840 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:44.879 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:44.880 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:44.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:44.981 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:45.007 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:45.008 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:45.009 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:45.046 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 28 15 53358 53343 119.049 76.8125 0.00610526 0.00839458 2024-09-14T05:01:45.084 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:01:45.109 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:45.159 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:45.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:45.160 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:45.261 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:45.289 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:45.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:45.290 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:45.390 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:45.434 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:45.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:45.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:45.536 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:45.566 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:45.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:45.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:45.668 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:45.690 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:01:45.745 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:45.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:45.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:45.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:45 smithi129 ceph-mon[101316]: pgmap v3106: 109 pgs: 109 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 97 MiB/s wr, 1.54k op/s 2024-09-14T05:01:45.771 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:45 smithi039 ceph-mon[111235]: pgmap v3106: 109 pgs: 109 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 97 MiB/s wr, 1.54k op/s 2024-09-14T05:01:45.771 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:45 smithi039 ceph-mon[106470]: pgmap v3106: 109 pgs: 109 active+clean; 2.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 97 MiB/s wr, 1.54k op/s 2024-09-14T05:01:45.847 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:45.877 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:45.878 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:45.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:45.979 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:46.025 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:46.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:46.026 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:46.046 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 29 16 54794 54778 118.036 89.6875 0.021334 0.00846686 2024-09-14T05:01:46.127 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:46.149 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:46.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:46.150 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:46.251 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:46.277 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:46.278 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:46.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:46.379 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:46.418 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:46.419 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:46.419 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:46.520 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:46.544 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:46.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:46.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:46.557 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T05:01:46.646 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:46.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:46.687 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:46.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:46.787 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:46.812 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:46.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:46.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:46.914 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:46.946 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:46.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:46.947 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:47.046 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 30 16 56016 56000 116.647 76.375 0.00811868 0.00856792 2024-09-14T05:01:47.048 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:47.088 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:47.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:47.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:47.190 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:47.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:47.216 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:47.217 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:47.317 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:47.342 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:47.343 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:47.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:47.444 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:47.465 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:47.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:47.466 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:47.567 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:47.592 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:47.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:47.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:47.694 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:47.732 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:47.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:47.733 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:47.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:47 smithi129 ceph-mon[101316]: pgmap v3107: 109 pgs: 109 active+clean; 3.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-14T05:01:47.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:47 smithi039 ceph-mon[106470]: pgmap v3107: 109 pgs: 109 active+clean; 3.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-14T05:01:47.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:47 smithi039 ceph-mon[111235]: pgmap v3107: 109 pgs: 109 active+clean; 3.2 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-14T05:01:47.834 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:47.874 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:47.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:47.875 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:47.976 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:48.012 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:48.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:48.014 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:48.046 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 31 16 57231 57215 115.334 75.9375 0.00454791 0.00866439 2024-09-14T05:01:48.114 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:48.145 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:48.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:48.146 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:48.247 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:48.289 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:48.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:48.290 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:48.391 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:48.426 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:48.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:48.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:48.528 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:48.574 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:48.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:48.575 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:48.676 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:48.720 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:48.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:48.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:48.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:01:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:01:48] "GET /metrics HTTP/1.1" 200 36736 "" "Prometheus/2.51.0" 2024-09-14T05:01:48.821 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:48.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:48.861 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:48.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:48.962 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:48.984 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:48.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:48.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:49.046 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 32 16 58509 58493 114.226 79.875 0.0116862 0.00874935 2024-09-14T05:01:49.086 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:49.114 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:49.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:49.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:49.216 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:49.256 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:49.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:49.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:49.358 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:49.383 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:49.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:49.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:49.485 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:49.510 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:49.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:49.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:49.611 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:49.654 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:49.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:49.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:49.713 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:49 smithi039 ceph-mon[111235]: pgmap v3108: 109 pgs: 109 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-14T05:01:49.713 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:49 smithi039 ceph-mon[106470]: pgmap v3108: 109 pgs: 109 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-14T05:01:49.756 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:49.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:49 smithi129 ceph-mon[101316]: pgmap v3108: 109 pgs: 109 active+clean; 3.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s 2024-09-14T05:01:49.785 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:49.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:49.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:49.887 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:49.914 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:49.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:49.916 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:50.016 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:50.044 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:50.044 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:50.045 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:50.046 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 33 16 59752 59736 113.118 77.6875 0.00802715 0.00883477 2024-09-14T05:01:50.146 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:50.175 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:50.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:50.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:50.277 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:50.305 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:50.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:50.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:50.407 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:50.451 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:50.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:50.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:50.553 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:50.580 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:50.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:50.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:50.681 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:50.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:50.713 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:50.713 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:50.814 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:50.855 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:50.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:50.856 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:50.957 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:50.986 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:50.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:50.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:51.049 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 34 16 61099 61083 112.259 84.1875 0.0086925 0.00890288 2024-09-14T05:01:51.087 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:51.114 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:51.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:51.115 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:51.216 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:51.239 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:51.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:51.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:51.341 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:51.365 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:51.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:51.366 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:51.467 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:51.491 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:51.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:51.492 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:51.593 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:51.619 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:51.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:51.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:51.721 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:51.769 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:51.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:51.770 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:51 smithi129 ceph-mon[101316]: pgmap v3109: 109 pgs: 109 active+clean; 3.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2024-09-14T05:01:51.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:51 smithi039 ceph-mon[106470]: pgmap v3109: 109 pgs: 109 active+clean; 3.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2024-09-14T05:01:51.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:51 smithi039 ceph-mon[111235]: pgmap v3109: 109 pgs: 109 active+clean; 3.5 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 88 MiB/s wr, 1.40k op/s 2024-09-14T05:01:51.871 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:51.912 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:51.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:51.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:52.014 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:52.049 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 35 16 62349 62333 111.284 78.125 0.0154678 0.00898028 2024-09-14T05:01:52.054 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:52.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:52.055 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:52.156 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:52.181 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:52.182 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:52.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:52.283 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:52.305 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:52.306 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:52.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:52.407 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:52.448 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:52.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:52.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:52.550 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:52.589 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T05:01:52.589 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:01:52.590 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:52.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:52.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:52.695 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:52.717 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:52.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:52.718 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:52.818 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:52.858 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:52.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:52.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:52.960 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:52.984 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:52.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:52.986 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:53.049 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 36 16 63324 63308 109.885 60.9375 0.0102917 0.00908871 2024-09-14T05:01:53.086 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:53.109 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:53.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:53.110 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:53.211 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:53.233 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:53.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:53.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:53.334 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:53.373 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:53.374 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:53.374 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:53.475 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:53.503 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:53.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:53.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:53.605 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:53.646 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:53.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:53.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:53.748 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:53 smithi129 ceph-mon[101316]: pgmap v3110: 109 pgs: 109 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-14T05:01:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:53 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:01:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:53 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:01:53.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:53 smithi039 ceph-mon[106470]: pgmap v3110: 109 pgs: 109 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-14T05:01:53.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:53 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:01:53.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:53 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:01:53.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:53 smithi039 ceph-mon[111235]: pgmap v3110: 109 pgs: 109 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s 2024-09-14T05:01:53.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:53 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:01:53.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:53 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:01:53.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:53.792 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:53.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:53.861 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:01:53.893 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:53.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:53.943 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:53.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:54.044 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:54.051 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 37 16 64226 64210 108.438 56.375 0.00956145 0.00921648 2024-09-14T05:01:54.072 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:54.073 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:54.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:54.174 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:54.202 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:54.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:54.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:54.304 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:54.328 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:54.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:54.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:54.430 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:54.454 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:54.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:54.455 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:54.555 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:54.592 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:54.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:54.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:54.694 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:54.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:54.725 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:54.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:54.826 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:54.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:54.870 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:54.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:54.971 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:54.997 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:54.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:54.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:55.050 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 38 16 65417 65401 107.543 74.4375 0.0151931 0.00929262 2024-09-14T05:01:55.087 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-14T05:01:55.099 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:55.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:55.185 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:55.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:55.286 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:55.337 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:55.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:55.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:55.439 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:55.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:55.467 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:55.467 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:55.568 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:55.592 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:55.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:55.593 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:55.622 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:55 smithi039 ceph-mon[106470]: pgmap v3111: 109 pgs: 109 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-09-14T05:01:55.623 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:55 smithi039 ceph-mon[111235]: pgmap v3111: 109 pgs: 109 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-09-14T05:01:55.693 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:55.722 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:55.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:55.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:55.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:55 smithi129 ceph-mon[101316]: pgmap v3111: 109 pgs: 109 active+clean; 3.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-09-14T05:01:55.824 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:55.851 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:55.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:55.852 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:55.953 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:55.978 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:55.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:55.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:56.050 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 39 16 66553 66537 106.606 71 0.00751616 0.00937436 2024-09-14T05:01:56.080 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:56.109 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:56.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:56.110 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:56.211 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:56.262 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:56.263 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:56.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:56.349 INFO:teuthology.orchestra.run.smithi039.stderr:set pool 58 pgp_num to 16 2024-09-14T05:01:56.364 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:01:56.412 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:56.412 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:56.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:01:56.431 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T05:01:56.431 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T05:01:56.469 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T05:01:56.470 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T05:01:56.470 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:56 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3019397672' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"} : dispatch 2024-09-14T05:01:56.470 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:56 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"} : dispatch 2024-09-14T05:01:56.472 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:56 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3019397672' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"} : dispatch 2024-09-14T05:01:56.472 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:56 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"} : dispatch 2024-09-14T05:01:56.508 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T05:01:56.509 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T05:01:56.513 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:56.546 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T05:01:56.546 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T05:01:56.561 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:56.562 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:56.562 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:56.582 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T05:01:56.582 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T05:01:56.615 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T05:01:56.615 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T05:01:56.647 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T05:01:56.647 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T05:01:56.663 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:56.683 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T05:01:56.683 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T05:01:56.700 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:56.701 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:56.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:56.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:56 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3019397672' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"} : dispatch 2024-09-14T05:01:56.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:56 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"} : dispatch 2024-09-14T05:01:56.802 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:56.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:56.826 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:56.827 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:56.927 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:56.966 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:56.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:56.967 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:57.050 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:2024-09-14T05:01:57.050530+0000 min lat: 0.00100424 max lat: 0.310727 avg lat: 0.00944701 2024-09-14T05:01:57.050 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T05:01:57.050 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 40 16 67745 67729 105.803 74.5 0.00738976 0.00944701 2024-09-14T05:01:57.069 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:01:57.097 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:57.097 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:57.098 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:01:57.198 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:57.238 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:57.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:57.239 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:57.340 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:57.367 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:57.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:57.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:57.470 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:57.496 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:57.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:57.497 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:57.533 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:57 smithi039 ceph-mon[106470]: pgmap v3112: 109 pgs: 109 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-09-14T05:01:57.533 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:57 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2024-09-14T05:01:57.534 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:57 smithi039 ceph-mon[106470]: osdmap e1574: 8 total, 8 up, 8 in 2024-09-14T05:01:57.534 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:57 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1595009782' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:01:57.534 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:57 smithi039 ceph-mon[111235]: pgmap v3112: 109 pgs: 109 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-09-14T05:01:57.534 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:57 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2024-09-14T05:01:57.534 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:57 smithi039 ceph-mon[111235]: osdmap e1574: 8 total, 8 up, 8 in 2024-09-14T05:01:57.534 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:57 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1595009782' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:01:57.599 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:57.646 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:57.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:57.647 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:57.749 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:57.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:57 smithi129 ceph-mon[101316]: pgmap v3112: 109 pgs: 109 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 86 MiB/s wr, 1.38k op/s 2024-09-14T05:01:57.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:57 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_5", "var": "pgp_num", "val": "16"}]': finished 2024-09-14T05:01:57.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:57 smithi129 ceph-mon[101316]: osdmap e1574: 8 total, 8 up, 8 in 2024-09-14T05:01:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:57 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1595009782' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:01:57.797 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:57.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:57.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:57.899 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:57.944 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:57.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:57.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:58.045 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:58.050 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 41 16 68998 68982 105.132 78.3125 0.0129984 0.00950596 2024-09-14T05:01:58.085 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:58.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:58.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:58.187 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:58.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:58.232 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:58.232 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:58.333 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:01:58.334 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:01:58.392 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:58.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:58.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:01:58.494 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:58.523 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:58.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:58.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:58.625 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:58.680 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:58.681 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:58.681 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:58.782 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:58.819 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:58.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:58.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:58.872 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:01:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:01:58] "GET /metrics HTTP/1.1" 200 36739 "" "Prometheus/2.51.0" 2024-09-14T05:01:58.920 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:58.945 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:58.946 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:58.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:59.047 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:59.050 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 42 16 70208 70192 104.43 75.625 0.0408279 0.00957017 2024-09-14T05:01:59.073 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:59.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:59.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:59.175 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:59.201 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:59.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:59.202 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:59.303 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:01:59.354 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:59.355 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:59.355 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:01:59.456 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:01:59.483 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:59.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:59.485 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:01:59.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:01:59 smithi039 ceph-mon[106470]: pgmap v3114: 109 pgs: 109 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-14T05:01:59.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:01:59 smithi039 ceph-mon[111235]: pgmap v3114: 109 pgs: 109 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-14T05:01:59.585 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:01:59.612 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:59.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:59.613 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:01:59.714 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:01:59.741 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:01:59.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:59.742 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:01:59.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:01:59 smithi129 ceph-mon[101316]: pgmap v3114: 109 pgs: 109 active+clean; 4.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 76 MiB/s wr, 1.21k op/s 2024-09-14T05:01:59.843 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:01:59.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:01:59.893 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:01:59.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:01:59.994 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:00.038 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:00.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:00.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:00.050 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 43 16 71335 71319 103.639 70.4375 0.0186693 0.00964332 2024-09-14T05:02:00.140 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:00.165 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:00.166 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:00.166 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:00.267 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:00.292 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:00.292 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:00.293 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:00.375 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is set 2024-09-14T05:02:00.393 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:00.434 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:00.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:00.435 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:00.536 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:00.577 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:00.578 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:00.578 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:00.679 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:00.683 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:00 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2749684916' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:02:00.683 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:00 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:02:00.683 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:00 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T05:02:00.719 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:00.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:00.720 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:00.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:00 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2749684916' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:02:00.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:00 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:02:00.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:00 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T05:02:00.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:00 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2749684916' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:02:00.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:00 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:02:00.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:00 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T05:02:00.822 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:00.851 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:00.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:00.852 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:00.953 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:00.974 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:00.975 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:00.975 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:01.050 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 44 16 72369 72353 102.753 64.625 0.0192582 0.00972714 2024-09-14T05:02:01.076 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:01.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:01.112 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:01.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:01.213 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:01.237 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:01.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:01.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:01.339 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:01.362 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:01.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:01.363 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:01.464 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:01.490 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:01.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:01.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:01.592 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:01.627 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:01.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:01.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:01.666 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:01 smithi039 ceph-mon[106470]: pgmap v3115: 109 pgs: 109 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 73 MiB/s wr, 1.18k op/s 2024-09-14T05:02:01.666 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:01 smithi039 ceph-mon[106470]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:02:01.666 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:01 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T05:02:01.666 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:01 smithi039 ceph-mon[106470]: osdmap e1575: 8 total, 8 up, 8 in 2024-09-14T05:02:01.666 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:01 smithi039 ceph-mon[111235]: pgmap v3115: 109 pgs: 109 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 73 MiB/s wr, 1.18k op/s 2024-09-14T05:02:01.667 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:01 smithi039 ceph-mon[111235]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:02:01.667 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:01 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T05:02:01.667 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:01 smithi039 ceph-mon[111235]: osdmap e1575: 8 total, 8 up, 8 in 2024-09-14T05:02:01.715 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:01 smithi129 ceph-mon[101316]: pgmap v3115: 109 pgs: 109 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 73 MiB/s wr, 1.18k op/s 2024-09-14T05:02:01.715 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:01 smithi129 ceph-mon[101316]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:02:01.715 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:01 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T05:02:01.715 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:01 smithi129 ceph-mon[101316]: osdmap e1575: 8 total, 8 up, 8 in 2024-09-14T05:02:01.728 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:01.768 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:01.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:01.770 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:01.870 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:01.911 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:01.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:01.912 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:02.013 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:02.035 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:02.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:02.036 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:02.050 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 45 16 73512 73496 102.057 71.4375 0.00359609 0.00978505 2024-09-14T05:02:02.137 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:02.176 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:02.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:02.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:02.278 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:02.303 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:02.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:02.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:02.405 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:02.434 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:02.434 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:02.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:02.536 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:02.578 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:02.685 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:02.686 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:02.786 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:02.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:02.815 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:02.815 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:02.916 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:02.939 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:02.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:02.940 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:03.041 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:03.053 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 46 16 74475 74459 101.142 60.1875 0.014479 0.00988158 2024-09-14T05:02:03.069 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:03.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:03.070 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:03.171 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:03.199 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:03.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:03.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:03.300 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:03.329 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:03.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:03.330 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:03.369 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-14T05:02:03.369 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T05:02:03.369 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2024-09-14T05:02:03.369 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-14T05:02:03.369 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-14T05:02:03.370 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 config set filestore_inject_stall 3 2024-09-14T05:02:03.431 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:03.483 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:03.484 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:03.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:03.585 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:03.613 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:03.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:03.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:03.715 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:03.745 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:03 smithi039 ceph-mon[106470]: pgmap v3117: 109 pgs: 109 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-14T05:02:03.746 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:03 smithi039 ceph-mon[111235]: pgmap v3117: 109 pgs: 109 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-14T05:02:03.766 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:03.767 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:03.768 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:03.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:03 smithi129 ceph-mon[101316]: pgmap v3117: 109 pgs: 109 active+clean; 4.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-14T05:02:03.771 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:02:03.868 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:03.902 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:03.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:03.903 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:04.004 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:04.045 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:04.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:04.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:04.053 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 47 16 75666 75650 100.573 74.4375 0.00479768 0.00993678 2024-09-14T05:02:04.126 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T05:02:04.126 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [ 2024-09-14T05:02:04.126 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.126 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop_reply(client.36603.0:75711 58.9 e1575/1568)", 2024-09-14T05:02:04.126 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:02:04.123236+0000", 2024-09-14T05:02:04.126 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.00045038499999999998, 2024-09-14T05:02:04.126 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000472635, 2024-09-14T05:02:04.126 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:04.127 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:04.127 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "queued for pg", 2024-09-14T05:02:04.127 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:04.127 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.127 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:04.127 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123236+0000", 2024-09-14T05:02:04.127 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:04.127 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.127 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.127 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:04.127 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123236+0000", 2024-09-14T05:02:04.127 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:04.127 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.127 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.128 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:04.128 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123238+0000", 2024-09-14T05:02:04.128 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2890000000000001e-06 2024-09-14T05:02:04.128 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.128 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.128 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:04.128 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123244+0000", 2024-09-14T05:02:04.128 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0610000000000001e-06 2024-09-14T05:02:04.128 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.129 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.129 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:04.129 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123260+0000", 2024-09-14T05:02:04.129 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6045999999999998e-05 2024-09-14T05:02:04.129 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.129 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.129 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:04.129 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123268+0000", 2024-09-14T05:02:04.129 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.1070000000000005e-06 2024-09-14T05:02:04.129 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:04.129 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:04.129 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:04.130 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.130 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.130 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop_reply(client.36603.0:75712 58.9 e1575/1568)", 2024-09-14T05:02:04.130 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:02:04.123296+0000", 2024-09-14T05:02:04.130 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.000391123, 2024-09-14T05:02:04.130 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00045951199999999997, 2024-09-14T05:02:04.130 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:04.130 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:04.130 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "queued for pg", 2024-09-14T05:02:04.130 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:04.130 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.130 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:04.131 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123296+0000", 2024-09-14T05:02:04.131 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:04.131 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.131 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.131 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:04.131 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123296+0000", 2024-09-14T05:02:04.131 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:04.131 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.131 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.131 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:04.131 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123296+0000", 2024-09-14T05:02:04.131 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.84e-07 2024-09-14T05:02:04.131 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.131 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.132 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:04.132 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123298+0000", 2024-09-14T05:02:04.132 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0889999999999998e-06 2024-09-14T05:02:04.132 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.132 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.132 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:04.132 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123304+0000", 2024-09-14T05:02:04.132 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.8440000000000003e-06 2024-09-14T05:02:04.132 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.132 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.132 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:04.132 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123308+0000", 2024-09-14T05:02:04.132 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.233e-06 2024-09-14T05:02:04.132 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:04.133 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:04.133 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:04.133 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.133 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.133 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:75723 58.f 58:fae0c8d4:::benchmark_data_smithi039_396950_object75722:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1575)", 2024-09-14T05:02:04.133 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:02:04.123332+0000", 2024-09-14T05:02:04.133 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.00035530300000000001, 2024-09-14T05:02:04.133 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00047482600000000002, 2024-09-14T05:02:04.133 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:04.133 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:04.133 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "started", 2024-09-14T05:02:04.133 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:02:04.133 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:02:04.133 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:02:04.134 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 75723 2024-09-14T05:02:04.134 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.134 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:04.134 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.134 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:04.134 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123332+0000", 2024-09-14T05:02:04.134 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:04.134 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.134 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.134 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:04.134 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123332+0000", 2024-09-14T05:02:04.134 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:04.134 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.134 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.134 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:04.135 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123333+0000", 2024-09-14T05:02:04.135 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.426e-06 2024-09-14T05:02:04.135 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.135 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.135 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:04.135 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123358+0000", 2024-09-14T05:02:04.135 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.5324999999999999e-05 2024-09-14T05:02:04.135 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.135 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.135 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:04.135 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123368+0000", 2024-09-14T05:02:04.135 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.499e-06 2024-09-14T05:02:04.135 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.136 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.136 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:04.136 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123374+0000", 2024-09-14T05:02:04.136 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.8159999999999999e-06 2024-09-14T05:02:04.136 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.136 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.136 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:04.136 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123540+0000", 2024-09-14T05:02:04.136 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000166062 2024-09-14T05:02:04.136 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.136 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.136 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:04.136 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123705+0000", 2024-09-14T05:02:04.137 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0001649 2024-09-14T05:02:04.137 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:04.137 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:04.137 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:04.137 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.137 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.137 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:75724 58.f 58.840cd60f (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1575)", 2024-09-14T05:02:04.137 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:02:04.123393+0000", 2024-09-14T05:02:04.137 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.000293491, 2024-09-14T05:02:04.137 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00048176499999999999, 2024-09-14T05:02:04.137 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:04.137 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:04.137 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "queued for pg", 2024-09-14T05:02:04.137 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:02:04.138 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:02:04.138 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:02:04.138 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 75724 2024-09-14T05:02:04.138 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.138 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:04.138 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.138 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:04.138 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123393+0000", 2024-09-14T05:02:04.138 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:04.138 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.138 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.138 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:04.138 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123393+0000", 2024-09-14T05:02:04.139 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:04.139 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.139 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.139 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:04.139 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123394+0000", 2024-09-14T05:02:04.139 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5299999999999999e-07 2024-09-14T05:02:04.139 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.139 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.139 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:04.139 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123415+0000", 2024-09-14T05:02:04.139 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0693e-05 2024-09-14T05:02:04.139 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.139 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.139 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:04.139 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123422+0000", 2024-09-14T05:02:04.140 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.007e-06 2024-09-14T05:02:04.140 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.140 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.140 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:04.140 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123426+0000", 2024-09-14T05:02:04.140 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9339999999999997e-06 2024-09-14T05:02:04.140 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:04.140 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:04.140 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:04.140 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.140 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.140 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:75704 58.9 58:97c6da58:::benchmark_data_smithi039_396950_object75703:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1575)", 2024-09-14T05:02:04.140 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:02:04.095430+0000", 2024-09-14T05:02:04.140 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.028256909, 2024-09-14T05:02:04.141 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.028500727999999999, 2024-09-14T05:02:04.141 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:04.141 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:04.141 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "waiting for sub ops", 2024-09-14T05:02:04.141 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:02:04.141 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:02:04.142 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:02:04.142 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 75704 2024-09-14T05:02:04.142 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.142 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:04.142 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.142 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:04.142 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.095430+0000", 2024-09-14T05:02:04.142 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:04.142 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.142 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.142 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:04.142 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.095430+0000", 2024-09-14T05:02:04.142 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:04.142 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.143 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.143 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:04.143 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.095430+0000", 2024-09-14T05:02:04.143 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4999999999999998e-07 2024-09-14T05:02:04.143 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.143 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.143 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:04.143 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.095494+0000", 2024-09-14T05:02:04.143 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3720999999999995e-05 2024-09-14T05:02:04.143 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.143 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.143 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:04.143 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.095503+0000", 2024-09-14T05:02:04.143 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.524e-06 2024-09-14T05:02:04.144 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.144 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.144 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:04.144 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.095508+0000", 2024-09-14T05:02:04.144 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7640000000000002e-06 2024-09-14T05:02:04.144 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.144 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.144 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:04.144 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.099128+0000", 2024-09-14T05:02:04.144 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.003619218 2024-09-14T05:02:04.144 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.144 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.144 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:04.144 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.099277+0000", 2024-09-14T05:02:04.145 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000149355 2024-09-14T05:02:04.145 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.145 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.145 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,3", 2024-09-14T05:02:04.145 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.099412+0000", 2024-09-14T05:02:04.145 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013536000000000001 2024-09-14T05:02:04.145 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.145 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.145 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:02:04.145 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.103060+0000", 2024-09-14T05:02:04.145 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0036474620000000002 2024-09-14T05:02:04.145 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.146 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.146 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:02:04.146 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.110680+0000", 2024-09-14T05:02:04.146 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0076199709999999997 2024-09-14T05:02:04.146 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:04.146 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:04.146 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:04.146 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.146 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.146 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop_reply(client.36603.0:75713 58.8 e1575/1568)", 2024-09-14T05:02:04.146 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:02:04.123855+0000", 2024-09-14T05:02:04.146 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 4294967295.9998322, 2024-09-14T05:02:04.146 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014363399999999999, 2024-09-14T05:02:04.147 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:04.147 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:04.147 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "started", 2024-09-14T05:02:04.147 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:04.147 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.147 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:04.147 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123855+0000", 2024-09-14T05:02:04.147 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:04.147 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.147 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.147 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:04.147 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123855+0000", 2024-09-14T05:02:04.147 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:04.147 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.147 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.148 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:04.148 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123855+0000", 2024-09-14T05:02:04.148 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.4200000000000005e-07 2024-09-14T05:02:04.148 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.148 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.148 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:04.148 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123865+0000", 2024-09-14T05:02:04.148 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.2949999999999999e-06 2024-09-14T05:02:04.148 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.148 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.148 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:04.148 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123876+0000", 2024-09-14T05:02:04.148 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1493e-05 2024-09-14T05:02:04.149 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.149 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.149 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:04.149 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123883+0000", 2024-09-14T05:02:04.149 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.0010000000000004e-06 2024-09-14T05:02:04.149 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.149 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.149 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:04.149 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123984+0000", 2024-09-14T05:02:04.149 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000100604 2024-09-14T05:02:04.149 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.149 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.149 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:04.150 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123992+0000", 2024-09-14T05:02:04.150 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7530000000000001e-06 2024-09-14T05:02:04.150 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:04.150 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:04.150 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:04.150 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.150 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.150 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop_reply(client.36603.0:75715 58.8 e1575/1568)", 2024-09-14T05:02:04.150 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:02:04.123917+0000", 2024-09-14T05:02:04.150 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 4294967295.9997697, 2024-09-14T05:02:04.150 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000125714, 2024-09-14T05:02:04.150 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:04.150 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:04.150 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "queued for pg", 2024-09-14T05:02:04.151 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:04.151 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.151 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:04.151 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123917+0000", 2024-09-14T05:02:04.151 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:04.151 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.151 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.151 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:04.151 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123917+0000", 2024-09-14T05:02:04.151 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:04.151 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.151 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.152 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:04.152 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123918+0000", 2024-09-14T05:02:04.152 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.4700000000000002e-07 2024-09-14T05:02:04.152 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.152 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.152 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:04.152 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123920+0000", 2024-09-14T05:02:04.152 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4310000000000001e-06 2024-09-14T05:02:04.152 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.152 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.152 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:04.152 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123928+0000", 2024-09-14T05:02:04.153 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.4930000000000001e-06 2024-09-14T05:02:04.153 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.153 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.153 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:04.153 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123933+0000", 2024-09-14T05:02:04.153 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9969999999999998e-06 2024-09-14T05:02:04.153 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:04.153 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:04.153 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:04.153 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.153 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.153 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:75726 58.8 58.27399b18 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e1575)", 2024-09-14T05:02:04.153 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:02:04.123994+0000", 2024-09-14T05:02:04.154 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 4294967295.9996929, 2024-09-14T05:02:04.154 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.8621000000000004e-05, 2024-09-14T05:02:04.154 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:04.154 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:04.154 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "queued for pg", 2024-09-14T05:02:04.154 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:02:04.154 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:02:04.154 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:02:04.154 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 75726 2024-09-14T05:02:04.154 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.154 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:04.154 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.154 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:04.154 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123994+0000", 2024-09-14T05:02:04.154 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:04.155 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.155 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.155 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:04.155 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123994+0000", 2024-09-14T05:02:04.155 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:04.155 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.155 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.155 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:04.155 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123995+0000", 2024-09-14T05:02:04.155 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.6000000000000002e-07 2024-09-14T05:02:04.155 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.155 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.155 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:04.155 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.124047+0000", 2024-09-14T05:02:04.156 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1688999999999999e-05 2024-09-14T05:02:04.156 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.156 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.156 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:04.156 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.124056+0000", 2024-09-14T05:02:04.156 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.9530000000000005e-06 2024-09-14T05:02:04.156 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.156 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.156 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:04.156 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.124062+0000", 2024-09-14T05:02:04.156 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8220000000000001e-06 2024-09-14T05:02:04.156 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:04.156 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:04.156 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:04.157 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.157 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.157 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:75710 58.9 58:90908a08:::benchmark_data_smithi039_396950_object75709:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1575)", 2024-09-14T05:02:04.157 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:02:04.103246+0000", 2024-09-14T05:02:04.157 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.020440903, 2024-09-14T05:02:04.157 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.020892092000000001, 2024-09-14T05:02:04.157 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:04.157 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:04.157 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "waiting for sub ops", 2024-09-14T05:02:04.157 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:02:04.157 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:02:04.157 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:02:04.157 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 75710 2024-09-14T05:02:04.157 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.158 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:04.158 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.158 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:04.158 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.103246+0000", 2024-09-14T05:02:04.158 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:04.158 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.158 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.158 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:04.158 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.103246+0000", 2024-09-14T05:02:04.158 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:04.158 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.158 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.158 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:04.158 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.103247+0000", 2024-09-14T05:02:04.158 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4699999999999999e-06 2024-09-14T05:02:04.159 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.159 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.159 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:04.159 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.103270+0000", 2024-09-14T05:02:04.159 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2402e-05 2024-09-14T05:02:04.159 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.159 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.159 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:04.159 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.103286+0000", 2024-09-14T05:02:04.159 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5937999999999999e-05 2024-09-14T05:02:04.160 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.160 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.160 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:04.160 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.103294+0000", 2024-09-14T05:02:04.160 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.8900000000000007e-06 2024-09-14T05:02:04.160 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.160 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.160 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:04.160 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.103411+0000", 2024-09-14T05:02:04.160 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000117291 2024-09-14T05:02:04.160 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.160 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.160 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:04.160 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.103578+0000", 2024-09-14T05:02:04.161 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00016667599999999999 2024-09-14T05:02:04.161 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.161 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.161 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,3", 2024-09-14T05:02:04.161 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.104059+0000", 2024-09-14T05:02:04.161 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00048102600000000001 2024-09-14T05:02:04.161 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.161 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.161 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:02:04.161 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.110692+0000", 2024-09-14T05:02:04.162 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0066330369999999996 2024-09-14T05:02:04.162 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.162 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.162 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:02:04.162 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.114356+0000", 2024-09-14T05:02:04.162 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0036644440000000002 2024-09-14T05:02:04.162 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:04.162 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:04.163 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:04.163 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.163 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.163 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:75711 58.9 58:94974299:::benchmark_data_smithi039_396950_object75710:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1575)", 2024-09-14T05:02:04.163 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:02:04.103315+0000", 2024-09-14T05:02:04.163 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.020371790000000001, 2024-09-14T05:02:04.163 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.020895680999999999, 2024-09-14T05:02:04.163 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:04.163 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:04.163 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "waiting for sub ops", 2024-09-14T05:02:04.163 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:02:04.163 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:02:04.163 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:02:04.163 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 75711 2024-09-14T05:02:04.163 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.164 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:04.164 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.164 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:04.164 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.103315+0000", 2024-09-14T05:02:04.164 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:04.164 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.164 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.164 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:04.164 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.103315+0000", 2024-09-14T05:02:04.164 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:04.164 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.164 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.164 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:04.165 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.103316+0000", 2024-09-14T05:02:04.165 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2099999999999996e-07 2024-09-14T05:02:04.165 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.165 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.165 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:04.165 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.103338+0000", 2024-09-14T05:02:04.165 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2495999999999999e-05 2024-09-14T05:02:04.165 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.165 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.165 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:04.165 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.103351+0000", 2024-09-14T05:02:04.165 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2768999999999999e-05 2024-09-14T05:02:04.165 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.166 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.166 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:04.166 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.103357+0000", 2024-09-14T05:02:04.166 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2620000000000004e-06 2024-09-14T05:02:04.166 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.166 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.166 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:04.166 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.106904+0000", 2024-09-14T05:02:04.166 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0035468159999999999 2024-09-14T05:02:04.166 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.166 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.166 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:04.166 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.107760+0000", 2024-09-14T05:02:04.167 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00085600499999999996 2024-09-14T05:02:04.167 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.167 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.167 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,3", 2024-09-14T05:02:04.167 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.107929+0000", 2024-09-14T05:02:04.167 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00016890500000000001 2024-09-14T05:02:04.167 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.167 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.167 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:02:04.167 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.118065+0000", 2024-09-14T05:02:04.167 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.010135893999999999 2024-09-14T05:02:04.167 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.167 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.167 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:02:04.167 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123791+0000", 2024-09-14T05:02:04.168 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0057266019999999999 2024-09-14T05:02:04.168 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:04.168 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:04.168 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:04.168 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.168 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.168 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:75712 58.9 58:97f7d606:::benchmark_data_smithi039_396950_object75711:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1575)", 2024-09-14T05:02:04.168 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:02:04.109038+0000", 2024-09-14T05:02:04.168 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.014648982, 2024-09-14T05:02:04.168 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.015244768000000001, 2024-09-14T05:02:04.168 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:04.168 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:04.168 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "waiting for sub ops", 2024-09-14T05:02:04.168 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:02:04.169 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:02:04.169 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:02:04.169 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 75712 2024-09-14T05:02:04.169 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.169 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:04.169 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.169 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:04.169 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.109038+0000", 2024-09-14T05:02:04.169 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:04.169 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.169 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.169 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:04.170 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.109038+0000", 2024-09-14T05:02:04.170 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:04.170 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.170 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.170 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:04.170 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.109039+0000", 2024-09-14T05:02:04.170 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.592e-06 2024-09-14T05:02:04.170 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.170 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.170 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:04.170 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.109089+0000", 2024-09-14T05:02:04.170 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.0006999999999997e-05 2024-09-14T05:02:04.170 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.171 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.171 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:04.171 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:04.171 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.109105+0000", 2024-09-14T05:02:04.172 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5858e-05 2024-09-14T05:02:04.172 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.172 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.172 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:04.172 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.109112+0000", 2024-09-14T05:02:04.172 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1590000000000004e-06 2024-09-14T05:02:04.172 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.172 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.172 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:04.172 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.110834+0000", 2024-09-14T05:02:04.172 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0017212499999999999 2024-09-14T05:02:04.172 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.172 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.172 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:04.173 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.111063+0000", 2024-09-14T05:02:04.173 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00022944999999999999 2024-09-14T05:02:04.173 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.173 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.173 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,3", 2024-09-14T05:02:04.173 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.111215+0000", 2024-09-14T05:02:04.173 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015142700000000001 2024-09-14T05:02:04.173 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.173 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.173 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:02:04.173 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.118072+0000", 2024-09-14T05:02:04.173 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0068574700000000001 2024-09-14T05:02:04.174 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.174 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.174 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:02:04.174 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123834+0000", 2024-09-14T05:02:04.174 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0057617730000000004 2024-09-14T05:02:04.174 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:04.174 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:04.174 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:04.174 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.174 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.174 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:75714 58.9 58:95d4cf41:::benchmark_data_smithi039_396950_object75713:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1575)", 2024-09-14T05:02:04.174 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:02:04.114292+0000", 2024-09-14T05:02:04.175 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.009394566, 2024-09-14T05:02:04.175 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.010060184, 2024-09-14T05:02:04.175 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:04.175 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:04.175 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "waiting for sub ops", 2024-09-14T05:02:04.175 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:02:04.175 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:02:04.175 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:02:04.175 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 75714 2024-09-14T05:02:04.175 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.175 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:04.175 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.175 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:04.175 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.114292+0000", 2024-09-14T05:02:04.175 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:04.176 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.176 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.176 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:04.176 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.114292+0000", 2024-09-14T05:02:04.176 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:04.176 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.176 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.176 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:04.176 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.114294+0000", 2024-09-14T05:02:04.176 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.341e-06 2024-09-14T05:02:04.176 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.176 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.176 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:04.177 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.114314+0000", 2024-09-14T05:02:04.177 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9890000000000001e-05 2024-09-14T05:02:04.177 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.177 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.177 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:04.177 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.114324+0000", 2024-09-14T05:02:04.177 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0006999999999999e-05 2024-09-14T05:02:04.177 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.177 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.177 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:04.178 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.114330+0000", 2024-09-14T05:02:04.178 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7460000000000003e-06 2024-09-14T05:02:04.178 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.178 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.178 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:04.178 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.114389+0000", 2024-09-14T05:02:04.178 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.9103e-05 2024-09-14T05:02:04.178 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.178 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.178 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:04.178 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.114549+0000", 2024-09-14T05:02:04.178 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015917500000000001 2024-09-14T05:02:04.179 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.179 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.179 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,3", 2024-09-14T05:02:04.179 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.114708+0000", 2024-09-14T05:02:04.179 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015910100000000001 2024-09-14T05:02:04.179 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:04.179 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:04.179 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:02:04.179 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:04.123000+0000", 2024-09-14T05:02:04.179 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0082919299999999994 2024-09-14T05:02:04.179 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:04.179 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:04.179 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:04.180 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:04.180 INFO:teuthology.orchestra.run.smithi039.stdout: ], 2024-09-14T05:02:04.180 INFO:teuthology.orchestra.run.smithi039.stdout: "num_ops": 12 2024-09-14T05:02:04.180 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T05:02:04.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:04.207 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:04.207 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:04.308 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:04.345 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:04.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:04.346 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:04.446 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:04.503 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:04.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:04.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:04.605 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:04.641 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:04.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:04.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:04.743 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:04.761 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T05:02:04.784 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:04.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:04.785 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:04.886 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:04.910 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:04.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:04.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:05.012 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:05.057 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 48 16 76769 76753 99.9059 68.9375 0.0164659 0.010004 2024-09-14T05:02:05.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:05.067 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:05.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:05.165 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-14T05:02:05.169 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:05.193 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:05.194 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:05.194 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:05.295 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:05.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:05.344 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:05.344 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:05.445 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:05.470 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:05.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:05.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:05.488 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T05:02:05.572 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:05.596 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:05.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:05.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:05.698 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:05.748 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:05.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:05.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:05.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:05 smithi129 ceph-mon[101316]: pgmap v3118: 109 pgs: 109 active+clean; 4.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 991 op/s 2024-09-14T05:02:05.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:05 smithi039 ceph-mon[106470]: pgmap v3118: 109 pgs: 109 active+clean; 4.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 991 op/s 2024-09-14T05:02:05.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:05 smithi039 ceph-mon[111235]: pgmap v3118: 109 pgs: 109 active+clean; 4.4 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 62 MiB/s wr, 991 op/s 2024-09-14T05:02:05.850 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:05.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:05.890 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:05.890 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:05.991 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:06.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:06.056 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:06.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:06.057 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 49 16 77781 77765 99.1577 63.25 0.0286113 0.0100778 2024-09-14T05:02:06.158 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:06.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:06.206 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:06.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:06.222 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:02:06.232 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:02:06.308 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:06.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:06.361 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:06.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:06.462 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:06.512 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:06.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:06.513 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:06.613 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:06.657 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:06.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:06.657 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:06.746 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T05:02:06.746 INFO:teuthology.orchestra.run.smithi129.stdout: "success": "" 2024-09-14T05:02:06.746 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T05:02:06.759 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:06.783 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:06.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:06.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:06.885 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:06.914 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:06.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:06.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:07.018 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:07.048 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:07.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:07.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:07.057 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 50 16 78844 78828 98.5033 66.4375 0.01008 0.0101468 2024-09-14T05:02:07.150 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:07.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:07.194 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:07.195 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:07.295 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:07.331 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:07.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:07.332 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:07.433 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:07.458 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:07.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:07.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:07.559 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:07.582 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:07.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:07.583 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:07.683 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:07.724 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:07.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:07.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:07 smithi129 ceph-mon[101316]: pgmap v3119: 109 pgs: 109 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-14T05:02:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:02:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:07 smithi039 ceph-mon[106470]: pgmap v3119: 109 pgs: 109 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-14T05:02:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:07.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:02:07.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:07 smithi039 ceph-mon[111235]: pgmap v3119: 109 pgs: 109 active+clean; 4.6 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 73 MiB/s wr, 1.17k op/s 2024-09-14T05:02:07.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:07.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:07.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:07.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:02:07.826 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:07.854 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:07.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:07.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:07.956 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:07.978 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:07.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:07.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:08.057 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 51 16 79826 79810 97.7753 61.375 0.0162933 0.0102218 2024-09-14T05:02:08.081 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:08.112 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:08.117 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:08.117 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:08.218 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:08.248 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:08.249 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:08.249 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:08.350 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:08.393 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:08.393 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:08.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:08.494 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:08.534 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:08.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:08.535 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:08.636 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:08.664 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:08.665 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:08.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:08.766 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:08.807 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:08.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:08.808 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:08.909 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:08.950 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:08.951 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:08.951 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:09.035 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:02:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:02:08] "GET /metrics HTTP/1.1" 200 36739 "" "Prometheus/2.51.0" 2024-09-14T05:02:09.052 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:09.058 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 52 16 80813 80797 97.0794 61.6875 0.0213749 0.0102948 2024-09-14T05:02:09.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:09.097 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:09.097 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:09.198 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:09.222 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:09.222 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:09.223 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:09.323 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:09.351 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:09.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:09.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:09.453 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:09.494 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:09.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:09.495 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:09.596 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:09.623 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:09.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:09.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:09.724 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:09.734 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:02:09.746 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:09.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:09.747 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:09.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:09 smithi129 ceph-mon[101316]: pgmap v3120: 109 pgs: 109 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-14T05:02:09.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:09 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:09.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:09 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:09 smithi039 ceph-mon[106470]: pgmap v3120: 109 pgs: 109 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-14T05:02:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:09 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:09.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:09 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:09.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:09 smithi039 ceph-mon[111235]: pgmap v3120: 109 pgs: 109 active+clean; 4.7 GiB data, 20 GiB used, 695 GiB / 715 GiB avail; 71 MiB/s wr, 1.13k op/s 2024-09-14T05:02:09.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:09 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:09.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:09 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:09.849 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:09.902 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:09.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:09.903 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:10.004 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:10.043 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:10.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:10.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:10.058 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 53 16 81921 81905 96.5543 69.25 0.0115669 0.0103517 2024-09-14T05:02:10.144 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:10.173 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:10.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:10.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:10.275 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:10.299 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:10.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:10.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:10.401 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:10.449 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:10.450 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:10.450 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:10.551 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:10.590 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:10.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:10.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:10.692 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:10.739 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:10.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:10.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:10.841 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:10.868 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:10.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:10.869 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:10.970 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:11.009 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:11.010 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:11.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:11.059 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 54 16 83005 82989 96.0209 67.75 0.0186189 0.0104089 2024-09-14T05:02:11.111 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:11.153 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:11.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:11.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:11.258 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:11.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:11.304 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:11.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:11.405 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:11.454 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:11.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:11.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:11.476 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:11 smithi129 ceph-mon[101316]: pgmap v3121: 109 pgs: 109 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-14T05:02:11.556 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:11.583 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:11.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:11.584 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:11.615 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:11 smithi039 ceph-mon[106470]: pgmap v3121: 109 pgs: 109 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-14T05:02:11.616 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:11 smithi039 ceph-mon[111235]: pgmap v3121: 109 pgs: 109 active+clean; 4.9 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s 2024-09-14T05:02:11.684 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:11.726 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:11.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:11.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:11.828 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:11.851 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:11.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:11.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:11.953 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:12.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:12.018 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:12.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:12.060 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 55 16 84006 83990 95.4104 62.5625 0.0275817 0.0104758 2024-09-14T05:02:12.066 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T05:02:12.066 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T05:02:12.066 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T05:02:12.066 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T05:02:12.066 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T05:02:12.067 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T05:02:12.067 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T05:02:12.067 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T05:02:12.067 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T05:02:12.067 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T05:02:12.067 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T05:02:12.067 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T05:02:12.067 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T05:02:12.067 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T05:02:12.094 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T05:02:12.094 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T05:02:12.119 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:12.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:12.176 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:12.176 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:12.277 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:12.304 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:12.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:12.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:12.406 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:12.435 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:12.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:12.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:12.537 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:12.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:12 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2772978561' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:02:12.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:12 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2772978561' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:02:12.562 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:12.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:12.563 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:12.664 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:12.688 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:12.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:12.689 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:12.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:12 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2772978561' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:02:12.790 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:12.822 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:12.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:12.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:12.853 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-14T05:02:12.853 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T05:02:12.853 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2024-09-14T05:02:12.853 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-14T05:02:12.853 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-14T05:02:12.853 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 config set filestore_inject_stall 3 2024-09-14T05:02:12.924 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:12.965 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:12.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:12.965 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:13.060 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 56 16 85074 85058 94.8986 66.75 0.0132362 0.0105313 2024-09-14T05:02:13.067 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:13.107 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:13.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:13.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:13.164 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:02:13.209 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:13.252 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:13.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:13.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:13.354 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:13.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:13.387 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:13.387 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:13.488 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:13.511 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:13.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:13.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:13.613 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:13.635 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:13.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:13.636 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:13.666 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:13 smithi129 ceph-mon[101316]: pgmap v3122: 109 pgs: 109 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-14T05:02:13.737 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:13.759 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:13.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:13.760 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:13.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:13 smithi039 ceph-mon[106470]: pgmap v3122: 109 pgs: 109 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-14T05:02:13.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:13 smithi039 ceph-mon[111235]: pgmap v3122: 109 pgs: 109 active+clean; 5.0 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 65 MiB/s wr, 1.04k op/s 2024-09-14T05:02:13.861 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:13.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:13.894 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:13.894 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:13.995 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:14.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:14.049 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:14.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:14.060 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 57 16 85866 85850 94.1022 49.5 0.0235831 0.0106203 2024-09-14T05:02:14.150 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:14.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:14.202 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:14.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:14.303 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:14.325 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:14.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:14.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:14.427 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:14.452 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:14.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:14.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:14.554 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:14.583 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:14.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:14.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:14.685 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:14.714 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:14.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:14.715 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:14.816 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:14.859 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:14.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:14.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:14.961 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:15.000 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:15.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:15.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:15.060 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 58 16 86844 86828 93.5337 61.125 0.00863299 0.010686 2024-09-14T05:02:15.102 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:15.142 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:15.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:15.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:15.244 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:15.304 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:15.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:15.305 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:15.406 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:15.430 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:15.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:15.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:15.532 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:15.551 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is set 2024-09-14T05:02:15.581 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:15.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:15.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:15.582 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:15 smithi129 ceph-mon[101316]: pgmap v3123: 109 pgs: 109 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-14T05:02:15.583 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:15 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:15.583 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:15 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/608540649' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:02:15.583 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:15 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:15.583 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:15 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:02:15.682 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:15.686 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:15 smithi039 ceph-mon[106470]: pgmap v3123: 109 pgs: 109 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-14T05:02:15.686 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:15 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:15.686 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:15 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/608540649' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:02:15.686 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:15 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:15.686 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:15 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:02:15.686 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:15 smithi039 ceph-mon[111235]: pgmap v3123: 109 pgs: 109 active+clean; 5.1 GiB data, 21 GiB used, 694 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s 2024-09-14T05:02:15.686 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:15 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:15.686 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:15 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/608540649' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:02:15.686 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:15 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:15.686 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:15 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:02:15.721 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:15.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:15.722 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:15.823 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:15.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:15.863 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:15.863 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:15.964 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:15.993 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:15.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:15.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:16.061 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 59 16 87756 87740 92.9145 57 0.0182524 0.0107569 2024-09-14T05:02:16.095 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:16.122 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:16.123 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:16.123 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:16.224 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:16.229 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T05:02:16.229 INFO:teuthology.orchestra.run.smithi129.stdout: "success": "" 2024-09-14T05:02:16.229 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T05:02:16.256 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:16.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:16.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:16.358 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:16.382 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:16.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:16.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:16.483 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:16.525 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:16.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:16.525 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:16.626 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:16.673 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:16.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:16.674 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:16.775 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:16.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:16 smithi039 ceph-mon[106470]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:02:16.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:16 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T05:02:16.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:16 smithi039 ceph-mon[106470]: osdmap e1576: 8 total, 8 up, 8 in 2024-09-14T05:02:16.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:16 smithi039 ceph-mon[111235]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:02:16.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:16 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T05:02:16.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:16 smithi039 ceph-mon[111235]: osdmap e1576: 8 total, 8 up, 8 in 2024-09-14T05:02:16.801 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:16.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:16.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:16.903 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:16.928 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:16.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:16.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:16.976 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:16 smithi129 ceph-mon[101316]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:02:16.976 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:16 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T05:02:16.976 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:16 smithi129 ceph-mon[101316]: osdmap e1576: 8 total, 8 up, 8 in 2024-09-14T05:02:17.030 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:17.061 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:2024-09-14T05:02:17.061038+0000 min lat: 0.00100424 max lat: 0.310727 avg lat: 0.010826 2024-09-14T05:02:17.061 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T05:02:17.061 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 60 16 88674 88658 92.3223 57.375 0.0106158 0.010826 2024-09-14T05:02:17.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:17.071 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:17.072 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:17.172 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:17.218 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:17.219 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:17.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:17.320 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:17.358 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:17.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:17.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:17.460 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:17.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:17.511 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:17.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:17.566 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T05:02:17.566 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:02:17.612 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:17.650 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:17.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:17.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:17.752 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:17.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:17 smithi039 ceph-mon[106470]: pgmap v3125: 109 pgs: 109 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-14T05:02:17.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:17 smithi039 ceph-mon[111235]: pgmap v3125: 109 pgs: 109 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-14T05:02:17.792 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:17.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:17.793 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:17.894 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:17.924 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T05:02:17.928 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:17.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:17.929 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:18.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:17 smithi129 ceph-mon[101316]: pgmap v3125: 109 pgs: 109 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-14T05:02:18.031 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:18.058 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:18.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:18.059 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:18.060 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 61 16 89651 89635 91.8098 61.0625 0.0121526 0.0108873 2024-09-14T05:02:18.160 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:18.183 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:18.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:18.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:18.286 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:18.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:18.335 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:18.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:18.437 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:18.475 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:18.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:18.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:18.577 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:18.607 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:18.608 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:18.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:18.709 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:18.742 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T05:02:18.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:18.753 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:18.753 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:18.854 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:18.878 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:18.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:18.880 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:18.921 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:02:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:02:18] "GET /metrics HTTP/1.1" 200 36739 "" "Prometheus/2.51.0" 2024-09-14T05:02:18.923 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-14T05:02:18.951 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:02:18.981 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:19.061 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 62 16 90538 90522 91.2232 55.4375 0.0250199 0.0109558 2024-09-14T05:02:19.074 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:19.076 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:19.077 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:19.178 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:19.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:19.226 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:19.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:19.327 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:19.372 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:19.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:19.376 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:19.477 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:19.504 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:19.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:19.505 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:19.606 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:19.632 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:19.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:19.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:19.737 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:19.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:19.788 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:19.789 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:19.889 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:19.927 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:19.928 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:19.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:19.936 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:19 smithi039 ceph-mon[106470]: pgmap v3126: 109 pgs: 109 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-14T05:02:19.936 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:19 smithi039 ceph-mon[111235]: pgmap v3126: 109 pgs: 109 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-14T05:02:20.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:19 smithi129 ceph-mon[101316]: pgmap v3126: 109 pgs: 109 active+clean; 5.3 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s 2024-09-14T05:02:20.029 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:20.061 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:20.061 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 63 16 91523 91507 90.7524 61.5625 0.017044 0.0110144 2024-09-14T05:02:20.062 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:20.062 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:20.163 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:20.186 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:20.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:20.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:20.288 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:20.328 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:20.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:20.329 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:20.430 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:20.453 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:20.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:20.454 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:20.555 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:20.607 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:20.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:20.610 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:20.711 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:20.761 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:20.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:20.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:20.863 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:20.892 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:20.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:20.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:20.993 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:21.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:21.023 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:21.024 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:21.061 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 64 16 92559 92543 90.3461 64.75 0.0104133 0.0110624 2024-09-14T05:02:21.096 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:02:21.110 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:02:21.124 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:21.208 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:21.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:21.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:21.310 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:21.360 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:21.361 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:21.361 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:21.462 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:21.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:21.504 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:21.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:21.605 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:21.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:21.645 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:21.646 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:21.746 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:21.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:21.775 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:21.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:21.876 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:21.893 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:21 smithi039 ceph-mon[106470]: pgmap v3127: 109 pgs: 109 active+clean; 5.5 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2024-09-14T05:02:21.894 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:21 smithi039 ceph-mon[111235]: pgmap v3127: 109 pgs: 109 active+clean; 5.5 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2024-09-14T05:02:21.923 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:21.924 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:21.925 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:22.007 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-14T05:02:22.008 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T05:02:22.008 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2024-09-14T05:02:22.008 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T05:02:22.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:21 smithi129 ceph-mon[101316]: pgmap v3127: 109 pgs: 109 active+clean; 5.5 GiB data, 23 GiB used, 693 GiB / 715 GiB avail; 61 MiB/s wr, 979 op/s 2024-09-14T05:02:22.026 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:22.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:22.053 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:22.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:22.062 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 65 16 93494 93478 89.8553 58.4375 0.00935573 0.011123 2024-09-14T05:02:22.154 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:22.218 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:22.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:22.219 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:22.320 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:22.376 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:22.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:22.377 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:22.478 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:22.525 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:22.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:22.527 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:22.628 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:22.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:22.673 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:22.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:22.774 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:22.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:02:22.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:02:22.814 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:22.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:22.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:22.916 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:22.941 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:22.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:22.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:23.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:02:23.043 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:23.061 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 66 16 94467 94451 89.4153 60.8125 0.00634579 0.011178 2024-09-14T05:02:23.067 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:23.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:23.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:23.169 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:23.195 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:23.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:23.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:23.297 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:23.334 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:23.335 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:23.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:23.437 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:23.460 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:23.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:23.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:23.562 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:23.608 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:23.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:23.609 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:23.709 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:23.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:23.765 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:23.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:23.847 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:23 smithi039 ceph-mon[106470]: pgmap v3128: 109 pgs: 109 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 61 MiB/s wr, 970 op/s 2024-09-14T05:02:23.847 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:23 smithi039 ceph-mon[111235]: pgmap v3128: 109 pgs: 109 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 61 MiB/s wr, 970 op/s 2024-09-14T05:02:23.866 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:23.890 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:23.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:23.891 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:23.992 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:24.017 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:24.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:24.018 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:24.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:23 smithi129 ceph-mon[101316]: pgmap v3128: 109 pgs: 109 active+clean; 5.5 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 61 MiB/s wr, 970 op/s 2024-09-14T05:02:24.061 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 67 16 95437 95421 88.9856 60.625 0.0128916 0.0112317 2024-09-14T05:02:24.119 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:24.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:24.170 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:24.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:24.271 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:24.314 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:24.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:24.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:24.416 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:24.437 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T05:02:24.437 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T05:02:24.445 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:24.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:24.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:24.469 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T05:02:24.469 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T05:02:24.520 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T05:02:24.520 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T05:02:24.547 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:24.559 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T05:02:24.560 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T05:02:24.572 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:24.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:24.573 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:24.604 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T05:02:24.604 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T05:02:24.648 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T05:02:24.648 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T05:02:24.673 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:24.686 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T05:02:24.687 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T05:02:24.688 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:24 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4034162467' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:02:24.688 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:24 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4034162467' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:02:24.697 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:24.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:24.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:24.725 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T05:02:24.726 INFO:teuthology.orchestra.run.smithi039.stdout: "size": 20, 2024-09-14T05:02:24.726 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 600, 2024-09-14T05:02:24.726 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [ 2024-09-14T05:02:24.726 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.726 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:37061 56.f e1474/1470 56:f9fef79c:::benchmark_data_smithi039_330847_object37060:head v 1474'2325, mlcod=1474'2321)", 2024-09-14T05:02:24.726 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:52:45.862375+0000", 2024-09-14T05:02:24.726 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 578.81860959200003, 2024-09-14T05:02:24.726 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.30113759800000001, 2024-09-14T05:02:24.726 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:24.726 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:24.726 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:02:24.727 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:24.727 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.727 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:24.727 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.862375+0000", 2024-09-14T05:02:24.727 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.727 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.727 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.727 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:24.727 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.862375+0000", 2024-09-14T05:02:24.727 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.727 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.727 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.727 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:24.728 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.862376+0000", 2024-09-14T05:02:24.728 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.9500000000000001e-07 2024-09-14T05:02:24.728 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.728 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.728 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:24.728 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.862449+0000", 2024-09-14T05:02:24.728 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.3501e-05 2024-09-14T05:02:24.728 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.728 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.728 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:24.728 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.862470+0000", 2024-09-14T05:02:24.728 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0907000000000002e-05 2024-09-14T05:02:24.729 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.729 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.729 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:24.729 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.862476+0000", 2024-09-14T05:02:24.729 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2890000000000003e-06 2024-09-14T05:02:24.729 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.729 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.729 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:24.729 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.862531+0000", 2024-09-14T05:02:24.729 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4746000000000001e-05 2024-09-14T05:02:24.729 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.729 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.729 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:24.730 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.862548+0000", 2024-09-14T05:02:24.730 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7244999999999998e-05 2024-09-14T05:02:24.730 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.730 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.730 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:02:24.730 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.163430+0000", 2024-09-14T05:02:24.730 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.300881966 2024-09-14T05:02:24.730 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.730 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.730 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:02:24.730 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.163512+0000", 2024-09-14T05:02:24.730 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.2049e-05 2024-09-14T05:02:24.730 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.730 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:24.731 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.731 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.731 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.731 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:37079 56.4 e1474/1470 56:24d81381:::benchmark_data_smithi039_330847_object37078:head v 1474'2388, mlcod=1474'2385)", 2024-09-14T05:02:24.731 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:52:45.863654+0000", 2024-09-14T05:02:24.731 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 578.81733032600005, 2024-09-14T05:02:24.731 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.29938503599999999, 2024-09-14T05:02:24.731 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:24.731 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:24.731 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:02:24.731 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:24.731 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.731 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:24.732 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.863654+0000", 2024-09-14T05:02:24.732 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.732 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.732 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.732 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:24.732 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.863654+0000", 2024-09-14T05:02:24.732 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.732 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.732 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.732 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:24.732 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.863655+0000", 2024-09-14T05:02:24.732 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.186e-06 2024-09-14T05:02:24.732 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.733 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.733 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:24.733 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.863702+0000", 2024-09-14T05:02:24.733 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7040999999999999e-05 2024-09-14T05:02:24.733 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.733 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.733 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:24.733 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.160419+0000", 2024-09-14T05:02:24.733 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.29671705999999998 2024-09-14T05:02:24.733 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.733 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.733 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:24.733 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.160475+0000", 2024-09-14T05:02:24.734 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6103000000000001e-05 2024-09-14T05:02:24.734 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.734 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.734 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:24.734 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.160573+0000", 2024-09-14T05:02:24.734 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.7540999999999993e-05 2024-09-14T05:02:24.734 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.734 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.734 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:24.734 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.160601+0000", 2024-09-14T05:02:24.734 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.7736e-05 2024-09-14T05:02:24.734 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.734 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.734 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:02:24.735 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.163001+0000", 2024-09-14T05:02:24.735 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0024004310000000002 2024-09-14T05:02:24.735 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.735 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.735 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:02:24.735 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.163039+0000", 2024-09-14T05:02:24.735 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.7938000000000002e-05 2024-09-14T05:02:24.735 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.735 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:24.735 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.735 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.735 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.735 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:37087 56.c e1474/1470 56:3620d308:::benchmark_data_smithi039_330847_object37086:head v 1474'2362, mlcod=1474'2360)", 2024-09-14T05:02:24.735 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:52:45.867751+0000", 2024-09-14T05:02:24.736 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 578.81323305900003, 2024-09-14T05:02:24.736 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.296320203, 2024-09-14T05:02:24.736 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:24.736 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:24.736 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:02:24.736 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:24.736 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.736 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:24.736 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.867751+0000", 2024-09-14T05:02:24.736 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.736 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.736 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.736 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:24.737 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.867751+0000", 2024-09-14T05:02:24.737 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.737 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.737 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.737 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:24.737 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.867752+0000", 2024-09-14T05:02:24.737 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.13e-06 2024-09-14T05:02:24.737 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.737 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.737 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:24.737 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:45.867800+0000", 2024-09-14T05:02:24.737 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7513999999999999e-05 2024-09-14T05:02:24.737 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.738 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.738 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:24.738 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.160420+0000", 2024-09-14T05:02:24.738 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.29262059299999998 2024-09-14T05:02:24.738 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.738 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.738 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:24.738 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.160476+0000", 2024-09-14T05:02:24.738 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5519000000000002e-05 2024-09-14T05:02:24.738 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.738 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.738 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:24.738 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.162166+0000", 2024-09-14T05:02:24.738 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0016903199999999999 2024-09-14T05:02:24.739 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.739 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.739 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:24.739 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.162186+0000", 2024-09-14T05:02:24.739 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9429999999999999e-05 2024-09-14T05:02:24.739 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.739 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.739 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:02:24.739 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.164029+0000", 2024-09-14T05:02:24.739 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0018431579999999999 2024-09-14T05:02:24.739 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.739 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.739 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:02:24.739 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:52:46.164071+0000", 2024-09-14T05:02:24.740 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2539e-05 2024-09-14T05:02:24.740 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.740 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:24.740 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.740 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.740 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.740 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:112482 56.e e1485/1470 56:7839e36f:::benchmark_data_smithi039_330847_object112481:head v 1485'7112, mlcod=1485'7109)", 2024-09-14T05:02:24.740 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:53:58.285430+0000", 2024-09-14T05:02:24.740 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 506.39555446600002, 2024-09-14T05:02:24.740 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.19324994600000001, 2024-09-14T05:02:24.740 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:24.740 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:24.740 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:02:24.741 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:24.741 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.741 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:24.741 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:53:58.285430+0000", 2024-09-14T05:02:24.741 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.741 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.741 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.741 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:24.741 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:53:58.285430+0000", 2024-09-14T05:02:24.741 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.741 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.741 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.741 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:24.742 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:53:58.285431+0000", 2024-09-14T05:02:24.742 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.1100000000000004e-07 2024-09-14T05:02:24.742 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.742 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.742 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:24.742 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:53:58.285495+0000", 2024-09-14T05:02:24.742 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3850999999999996e-05 2024-09-14T05:02:24.742 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.742 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.742 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:24.742 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:53:58.285505+0000", 2024-09-14T05:02:24.742 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0397000000000001e-05 2024-09-14T05:02:24.742 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.743 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.743 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:24.743 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:53:58.285511+0000", 2024-09-14T05:02:24.743 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.28e-06 2024-09-14T05:02:24.743 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.743 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.743 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:24.743 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:53:58.462510+0000", 2024-09-14T05:02:24.743 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.176999196 2024-09-14T05:02:24.743 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.743 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.743 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:24.743 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:53:58.462543+0000", 2024-09-14T05:02:24.743 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.2208000000000001e-05 2024-09-14T05:02:24.744 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.744 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.744 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:02:24.744 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:53:58.478590+0000", 2024-09-14T05:02:24.744 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.016047757999999999 2024-09-14T05:02:24.744 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.744 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.744 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:02:24.744 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:53:58.478680+0000", 2024-09-14T05:02:24.744 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.9345000000000006e-05 2024-09-14T05:02:24.744 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.744 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:24.744 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.745 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.745 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.745 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:172819 56.a e1508/1470 56:59afb196:::benchmark_data_smithi039_330847_object58295:head v 1508'10750, mlcod=1508'10748)", 2024-09-14T05:02:24.745 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:55:16.727577+0000", 2024-09-14T05:02:24.745 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 427.953407674, 2024-09-14T05:02:24.745 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.20681205699999999, 2024-09-14T05:02:24.745 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:24.745 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:24.745 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:02:24.745 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:24.745 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.745 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:24.745 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.727577+0000", 2024-09-14T05:02:24.746 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.746 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.746 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.746 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:24.746 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.727577+0000", 2024-09-14T05:02:24.746 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.746 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.746 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.746 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:24.746 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.727577+0000", 2024-09-14T05:02:24.746 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.4099999999999998e-07 2024-09-14T05:02:24.746 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.746 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.747 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:24.747 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.727583+0000", 2024-09-14T05:02:24.747 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4299999999999997e-06 2024-09-14T05:02:24.747 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.747 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.747 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:24.747 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.727595+0000", 2024-09-14T05:02:24.747 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2626000000000001e-05 2024-09-14T05:02:24.747 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.747 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.747 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:24.747 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.727602+0000", 2024-09-14T05:02:24.747 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.0620000000000003e-06 2024-09-14T05:02:24.747 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.748 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.748 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:24.748 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.727663+0000", 2024-09-14T05:02:24.748 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0260999999999998e-05 2024-09-14T05:02:24.748 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.748 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.748 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:24.748 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.727684+0000", 2024-09-14T05:02:24.748 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0852999999999998e-05 2024-09-14T05:02:24.748 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.748 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.748 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:02:24.748 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.934335+0000", 2024-09-14T05:02:24.749 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.20665176499999999 2024-09-14T05:02:24.749 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.749 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.749 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:02:24.749 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.934389+0000", 2024-09-14T05:02:24.749 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3319000000000002e-05 2024-09-14T05:02:24.749 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.749 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:24.749 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.749 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.749 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.749 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:172816 56.e e1508/1470 56:7dc92d87:::benchmark_data_smithi039_330847_object58292:head v 1508'10875, mlcod=1508'10872)", 2024-09-14T05:02:24.749 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:55:16.727635+0000", 2024-09-14T05:02:24.749 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 427.95334942599999, 2024-09-14T05:02:24.750 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.206759578, 2024-09-14T05:02:24.750 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:24.750 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:24.750 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:02:24.750 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:24.750 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.750 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:24.750 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.727635+0000", 2024-09-14T05:02:24.750 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.750 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.750 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.750 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:24.750 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.727635+0000", 2024-09-14T05:02:24.751 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.751 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.751 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.751 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:24.751 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.727635+0000", 2024-09-14T05:02:24.751 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.03e-07 2024-09-14T05:02:24.751 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.751 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.751 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:24.751 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.727639+0000", 2024-09-14T05:02:24.751 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.8840000000000004e-06 2024-09-14T05:02:24.751 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.751 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.752 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:24.752 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.727646+0000", 2024-09-14T05:02:24.752 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8349999999999998e-06 2024-09-14T05:02:24.752 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.752 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.752 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:24.752 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.727650+0000", 2024-09-14T05:02:24.752 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5390000000000001e-06 2024-09-14T05:02:24.752 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.752 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.752 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:24.752 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.727912+0000", 2024-09-14T05:02:24.752 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00026130599999999999 2024-09-14T05:02:24.752 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.753 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.753 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:24.753 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.727929+0000", 2024-09-14T05:02:24.753 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7538999999999999e-05 2024-09-14T05:02:24.753 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.753 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.753 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:02:24.753 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.934319+0000", 2024-09-14T05:02:24.753 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.20638938100000001 2024-09-14T05:02:24.753 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.753 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.753 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:02:24.753 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.934394+0000", 2024-09-14T05:02:24.753 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.5691000000000004e-05 2024-09-14T05:02:24.754 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.754 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:24.754 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.754 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.754 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.754 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:172820 56.a e1508/1470 56:5f82633b:::benchmark_data_smithi039_330847_object58296:head v 1508'10751, mlcod=1508'10748)", 2024-09-14T05:02:24.754 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:55:16.731963+0000", 2024-09-14T05:02:24.754 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 427.949020851, 2024-09-14T05:02:24.754 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.20245527499999999, 2024-09-14T05:02:24.754 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:24.754 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:24.754 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:02:24.754 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:24.755 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.755 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:24.755 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.731963+0000", 2024-09-14T05:02:24.755 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.755 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.755 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.755 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:24.755 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.731963+0000", 2024-09-14T05:02:24.755 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.755 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.755 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.755 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:24.755 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.731964+0000", 2024-09-14T05:02:24.756 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7000000000000004e-07 2024-09-14T05:02:24.756 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.756 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.756 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:24.756 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.731972+0000", 2024-09-14T05:02:24.756 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.4379999999999995e-06 2024-09-14T05:02:24.756 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.756 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.756 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:24.756 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.731981+0000", 2024-09-14T05:02:24.756 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.2719999999999997e-06 2024-09-14T05:02:24.756 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.757 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.757 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:24.757 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.731986+0000", 2024-09-14T05:02:24.757 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9200000000000003e-06 2024-09-14T05:02:24.757 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.757 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.757 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:24.757 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.732378+0000", 2024-09-14T05:02:24.757 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00039262200000000002 2024-09-14T05:02:24.757 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.757 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.757 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:24.757 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.732400+0000", 2024-09-14T05:02:24.757 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1664000000000001e-05 2024-09-14T05:02:24.758 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.758 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.758 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:02:24.758 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.934392+0000", 2024-09-14T05:02:24.758 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.20199218399999999 2024-09-14T05:02:24.758 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.758 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.758 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:02:24.758 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.934419+0000", 2024-09-14T05:02:24.758 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.6505000000000001e-05 2024-09-14T05:02:24.758 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.758 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:24.758 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.758 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.759 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.759 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:172824 56.e e1508/1470 56:7f10b957:::benchmark_data_smithi039_330847_object58300:head v 1508'10876, mlcod=1508'10873)", 2024-09-14T05:02:24.759 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:55:16.735178+0000", 2024-09-14T05:02:24.759 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 427.94580670099998, 2024-09-14T05:02:24.759 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.20705710599999999, 2024-09-14T05:02:24.759 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:24.759 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:24.759 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:02:24.759 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:24.759 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.759 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:24.759 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.735178+0000", 2024-09-14T05:02:24.759 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.760 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.760 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.760 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:24.760 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.735178+0000", 2024-09-14T05:02:24.760 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.760 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.760 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.760 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:24.760 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.735178+0000", 2024-09-14T05:02:24.760 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.3799999999999996e-07 2024-09-14T05:02:24.760 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.760 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.761 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:24.761 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.735184+0000", 2024-09-14T05:02:24.761 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7749999999999998e-06 2024-09-14T05:02:24.761 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.761 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.761 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:24.761 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.934311+0000", 2024-09-14T05:02:24.761 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.19912681099999999 2024-09-14T05:02:24.761 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.761 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.761 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:24.761 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.934331+0000", 2024-09-14T05:02:24.761 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0463e-05 2024-09-14T05:02:24.762 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.762 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.762 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:24.762 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.934445+0000", 2024-09-14T05:02:24.762 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000113631 2024-09-14T05:02:24.762 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.762 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.762 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:24.762 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.934472+0000", 2024-09-14T05:02:24.762 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.7402999999999998e-05 2024-09-14T05:02:24.763 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.763 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.763 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:02:24.763 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.942172+0000", 2024-09-14T05:02:24.763 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0076997109999999997 2024-09-14T05:02:24.763 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.763 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.763 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:02:24.763 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.942235+0000", 2024-09-14T05:02:24.763 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2473999999999999e-05 2024-09-14T05:02:24.763 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.763 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:24.763 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.764 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.764 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.764 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:172826 56.c e1508/1470 56:3964b8dd:::benchmark_data_smithi039_330847_object58302:head v 1508'10932, mlcod=1508'10930)", 2024-09-14T05:02:24.764 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:55:16.736951+0000", 2024-09-14T05:02:24.764 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 427.94403286199997, 2024-09-14T05:02:24.764 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.20689692700000001, 2024-09-14T05:02:24.764 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:24.764 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:24.764 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:02:24.764 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:24.764 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.764 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:24.764 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.736951+0000", 2024-09-14T05:02:24.765 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.765 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.765 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.765 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:24.765 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.736951+0000", 2024-09-14T05:02:24.765 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.765 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.765 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.765 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:24.765 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.736953+0000", 2024-09-14T05:02:24.765 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1999999999999999e-06 2024-09-14T05:02:24.765 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.765 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.766 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:24.766 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.736960+0000", 2024-09-14T05:02:24.766 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6469999999999998e-06 2024-09-14T05:02:24.766 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.766 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.766 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:24.766 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.934325+0000", 2024-09-14T05:02:24.766 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.19736440099999999 2024-09-14T05:02:24.766 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.766 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.766 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:24.766 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.934334+0000", 2024-09-14T05:02:24.766 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.8630000000000003e-06 2024-09-14T05:02:24.767 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.767 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.767 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:24.767 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.934405+0000", 2024-09-14T05:02:24.767 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.0865999999999995e-05 2024-09-14T05:02:24.767 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.767 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.767 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:24.767 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.934440+0000", 2024-09-14T05:02:24.767 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.4848000000000002e-05 2024-09-14T05:02:24.767 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.767 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.767 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:02:24.767 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.943816+0000", 2024-09-14T05:02:24.768 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0093753320000000001 2024-09-14T05:02:24.768 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.768 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.768 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:02:24.768 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.943848+0000", 2024-09-14T05:02:24.768 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.277e-05 2024-09-14T05:02:24.768 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.768 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:24.768 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.768 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.768 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.768 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36012.0:172831 56.3 56:cc4b58f8:::benchmark_data_smithi039_330847_object58307:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1508)", 2024-09-14T05:02:24.768 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:55:16.743068+0000", 2024-09-14T05:02:24.769 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 427.937916293, 2024-09-14T05:02:24.769 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.230138748, 2024-09-14T05:02:24.769 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:24.769 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:24.769 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:02:24.769 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:02:24.769 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36012", 2024-09-14T05:02:24.769 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/2063618618", 2024-09-14T05:02:24.769 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 172831 2024-09-14T05:02:24.769 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.769 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:24.769 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.769 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:24.770 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.743068+0000", 2024-09-14T05:02:24.770 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.770 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.770 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.770 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:24.770 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.743068+0000", 2024-09-14T05:02:24.770 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.770 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.770 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.770 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:24.770 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.743071+0000", 2024-09-14T05:02:24.770 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.5780000000000001e-06 2024-09-14T05:02:24.771 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.771 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.771 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:24.771 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.743078+0000", 2024-09-14T05:02:24.771 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.763e-06 2024-09-14T05:02:24.771 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.771 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.771 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:24.771 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.934308+0000", 2024-09-14T05:02:24.771 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.19123016700000001 2024-09-14T05:02:24.771 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.771 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.771 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:24.772 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.934330+0000", 2024-09-14T05:02:24.772 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1676999999999999e-05 2024-09-14T05:02:24.772 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.772 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.772 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:24.772 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.934471+0000", 2024-09-14T05:02:24.772 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014094800000000001 2024-09-14T05:02:24.772 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.772 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.772 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:24.772 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.934654+0000", 2024-09-14T05:02:24.772 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00018280799999999999 2024-09-14T05:02:24.772 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.773 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.773 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 5,6,7", 2024-09-14T05:02:24.773 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.935402+0000", 2024-09-14T05:02:24.773 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00074763800000000001 2024-09-14T05:02:24.773 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.773 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.773 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:02:24.773 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.950765+0000", 2024-09-14T05:02:24.773 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.015363786000000001 2024-09-14T05:02:24.773 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.773 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.773 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:02:24.773 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.957221+0000", 2024-09-14T05:02:24.774 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0064552500000000001 2024-09-14T05:02:24.774 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.774 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.774 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:02:24.774 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.963037+0000", 2024-09-14T05:02:24.774 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0058164940000000002 2024-09-14T05:02:24.774 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.774 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.774 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:02:24.774 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.972516+0000", 2024-09-14T05:02:24.774 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0094787510000000005 2024-09-14T05:02:24.774 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.774 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.774 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:02:24.775 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.972785+0000", 2024-09-14T05:02:24.775 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00026939999999999999 2024-09-14T05:02:24.775 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.775 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.775 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:02:24.775 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:55:16.973207+0000", 2024-09-14T05:02:24.775 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00042148799999999997 2024-09-14T05:02:24.775 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.775 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:24.775 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.775 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.775 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.775 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:34619 57.e 57:79732b9b:::benchmark_data_smithi039_365565_object34618:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T05:02:24.775 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:57:14.033537+0000", 2024-09-14T05:02:24.776 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 310.64744748200002, 2024-09-14T05:02:24.776 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.22122019300000001, 2024-09-14T05:02:24.776 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:24.776 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:24.776 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:02:24.776 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:02:24.776 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T05:02:24.776 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:02:24.776 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 34619 2024-09-14T05:02:24.776 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.776 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:24.776 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.776 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:24.777 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:14.033537+0000", 2024-09-14T05:02:24.777 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.777 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.777 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.777 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:24.777 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:14.033537+0000", 2024-09-14T05:02:24.777 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.777 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.777 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.777 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:24.777 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:14.033547+0000", 2024-09-14T05:02:24.777 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.076e-05 2024-09-14T05:02:24.777 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.777 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.778 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:24.778 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:14.033580+0000", 2024-09-14T05:02:24.778 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.2543999999999999e-05 2024-09-14T05:02:24.778 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.778 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.778 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:24.778 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:14.033603+0000", 2024-09-14T05:02:24.778 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2691e-05 2024-09-14T05:02:24.778 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.778 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.778 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:24.778 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:14.033612+0000", 2024-09-14T05:02:24.778 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.6430000000000007e-06 2024-09-14T05:02:24.778 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.779 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.779 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:24.779 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:14.033681+0000", 2024-09-14T05:02:24.779 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8969999999999999e-05 2024-09-14T05:02:24.779 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.779 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.779 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:24.779 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:14.033815+0000", 2024-09-14T05:02:24.779 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013336000000000001 2024-09-14T05:02:24.779 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.779 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.779 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T05:02:24.779 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:14.033956+0000", 2024-09-14T05:02:24.779 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014137500000000001 2024-09-14T05:02:24.780 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.780 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.780 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:02:24.780 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:14.036348+0000", 2024-09-14T05:02:24.780 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.002392206 2024-09-14T05:02:24.780 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.780 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.780 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:02:24.780 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:14.036686+0000", 2024-09-14T05:02:24.780 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00033734699999999999 2024-09-14T05:02:24.780 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.781 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.781 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:02:24.781 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:14.254646+0000", 2024-09-14T05:02:24.781 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.21796076 2024-09-14T05:02:24.781 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.781 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.781 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:02:24.781 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:14.254710+0000", 2024-09-14T05:02:24.781 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3623999999999999e-05 2024-09-14T05:02:24.781 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.781 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.781 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:02:24.782 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:14.254757+0000", 2024-09-14T05:02:24.782 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6913000000000003e-05 2024-09-14T05:02:24.782 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.782 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:24.782 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.782 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.782 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.782 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:34621 57.e 57:72ccb308:::benchmark_data_smithi039_365565_object34620:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T05:02:24.782 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:57:14.033886+0000", 2024-09-14T05:02:24.782 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 310.64709787700002, 2024-09-14T05:02:24.782 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.221218841, 2024-09-14T05:02:24.782 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:24.782 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:24.783 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:02:24.783 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:02:24.783 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T05:02:24.783 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:02:24.783 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 34621 2024-09-14T05:02:24.783 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.783 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:24.783 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.783 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:24.784 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:14.033886+0000", 2024-09-14T05:02:24.784 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.784 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.784 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.784 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:24.784 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:14.033886+0000", 2024-09-14T05:02:24.784 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.784 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.784 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.784 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:24.784 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:14.033887+0000", 2024-09-14T05:02:24.784 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1009999999999999e-06 2024-09-14T05:02:24.785 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.785 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.785 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:24.785 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:14.033909+0000", 2024-09-14T05:02:24.785 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1387999999999999e-05 2024-09-14T05:02:24.785 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.785 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.785 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:24.785 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:14.033921+0000", 2024-09-14T05:02:24.785 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2527999999999999e-05 2024-09-14T05:02:24.786 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.786 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.786 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:24.786 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:14.033929+0000", 2024-09-14T05:02:24.786 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6249999999999998e-06 2024-09-14T05:02:24.786 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.786 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.786 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:24.786 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:14.035242+0000", 2024-09-14T05:02:24.786 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001312702 2024-09-14T05:02:24.786 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.787 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.787 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:24.787 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:14.035331+0000", 2024-09-14T05:02:24.787 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.8984e-05 2024-09-14T05:02:24.787 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.787 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.787 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T05:02:24.787 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:14.035421+0000", 2024-09-14T05:02:24.787 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.0655000000000005e-05 2024-09-14T05:02:24.787 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.788 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.788 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:02:24.788 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:14.036476+0000", 2024-09-14T05:02:24.788 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0010548930000000001 2024-09-14T05:02:24.788 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.788 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.788 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:02:24.788 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:14.036782+0000", 2024-09-14T05:02:24.789 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000306206 2024-09-14T05:02:24.789 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.789 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.789 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:02:24.789 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:14.254996+0000", 2024-09-14T05:02:24.789 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.21821323300000001 2024-09-14T05:02:24.789 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.789 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.789 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:02:24.789 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:14.255058+0000", 2024-09-14T05:02:24.789 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2471999999999994e-05 2024-09-14T05:02:24.790 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.790 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.790 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:02:24.790 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:14.255105+0000", 2024-09-14T05:02:24.790 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7054e-05 2024-09-14T05:02:24.790 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.790 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:24.790 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.790 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.790 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.791 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:71011 57.e 57:7987587c:::benchmark_data_smithi039_365565_object71010:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1525)", 2024-09-14T05:02:24.791 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:57:39.215534+0000", 2024-09-14T05:02:24.791 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 285.46544986100002, 2024-09-14T05:02:24.791 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.18975967599999999, 2024-09-14T05:02:24.791 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:24.791 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:24.791 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:02:24.791 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:02:24.791 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T05:02:24.791 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:02:24.791 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 71011 2024-09-14T05:02:24.791 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.791 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:24.791 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.792 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:24.792 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.215534+0000", 2024-09-14T05:02:24.792 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.792 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.792 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.792 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:24.792 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.215534+0000", 2024-09-14T05:02:24.792 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.792 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.792 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.792 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:24.792 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.215536+0000", 2024-09-14T05:02:24.792 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5570000000000001e-06 2024-09-14T05:02:24.793 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.793 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.793 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:24.793 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.215560+0000", 2024-09-14T05:02:24.793 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3952999999999999e-05 2024-09-14T05:02:24.793 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.793 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.793 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:24.793 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.215573+0000", 2024-09-14T05:02:24.793 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3519e-05 2024-09-14T05:02:24.793 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.793 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.793 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:24.793 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.215582+0000", 2024-09-14T05:02:24.794 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.4130000000000006e-06 2024-09-14T05:02:24.794 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.794 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.794 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:24.794 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.215634+0000", 2024-09-14T05:02:24.794 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.2577000000000002e-05 2024-09-14T05:02:24.794 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.794 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.795 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:24.795 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.215739+0000", 2024-09-14T05:02:24.795 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000104726 2024-09-14T05:02:24.795 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.795 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.795 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T05:02:24.795 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.215848+0000", 2024-09-14T05:02:24.795 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000109367 2024-09-14T05:02:24.795 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.795 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.795 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:02:24.795 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.405056+0000", 2024-09-14T05:02:24.795 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.18920801600000001 2024-09-14T05:02:24.796 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.796 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.796 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:02:24.796 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.405193+0000", 2024-09-14T05:02:24.796 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013658 2024-09-14T05:02:24.796 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.796 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.796 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:02:24.797 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.405232+0000", 2024-09-14T05:02:24.797 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.8504000000000003e-05 2024-09-14T05:02:24.797 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.797 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.797 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:02:24.797 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.405269+0000", 2024-09-14T05:02:24.797 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.7741000000000003e-05 2024-09-14T05:02:24.797 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.797 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.797 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:02:24.797 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:39.405294+0000", 2024-09-14T05:02:24.797 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4723e-05 2024-09-14T05:02:24.797 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.797 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:24.798 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.798 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.798 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.798 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:150930 57.e 57:718654cd:::benchmark_data_smithi039_365565_object27178:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1537)", 2024-09-14T05:02:24.798 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:58:59.887674+0000", 2024-09-14T05:02:24.798 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 204.793310475, 2024-09-14T05:02:24.798 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.224027907, 2024-09-14T05:02:24.798 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:24.798 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:24.798 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:02:24.798 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:02:24.798 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T05:02:24.798 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:02:24.798 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 150930 2024-09-14T05:02:24.799 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.799 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:24.799 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.799 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:24.799 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:58:59.887674+0000", 2024-09-14T05:02:24.799 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.799 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.799 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.799 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:24.799 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:58:59.887674+0000", 2024-09-14T05:02:24.799 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.799 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.799 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.800 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:24.800 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:58:59.887677+0000", 2024-09-14T05:02:24.800 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.9280000000000002e-06 2024-09-14T05:02:24.800 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.800 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.800 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:24.800 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:58:59.887685+0000", 2024-09-14T05:02:24.800 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.3820000000000004e-06 2024-09-14T05:02:24.800 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.800 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.800 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:24.800 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:58:59.887703+0000", 2024-09-14T05:02:24.800 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.827e-05 2024-09-14T05:02:24.800 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.801 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.801 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:24.801 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:58:59.887713+0000", 2024-09-14T05:02:24.801 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.7999999999999993e-06 2024-09-14T05:02:24.801 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:24.802 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T05:02:24.803 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T05:02:24.803 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.803 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.803 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:24.803 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:58:59.887834+0000", 2024-09-14T05:02:24.804 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000121304 2024-09-14T05:02:24.804 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.804 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.804 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:24.804 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:58:59.887999+0000", 2024-09-14T05:02:24.804 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000164857 2024-09-14T05:02:24.804 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.804 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.804 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T05:02:24.804 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:58:59.888112+0000", 2024-09-14T05:02:24.804 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000112348 2024-09-14T05:02:24.804 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.804 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.805 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:02:24.805 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:59:00.103038+0000", 2024-09-14T05:02:24.805 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.21492608499999999 2024-09-14T05:02:24.805 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.805 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.805 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:02:24.805 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:59:00.103085+0000", 2024-09-14T05:02:24.805 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7599000000000003e-05 2024-09-14T05:02:24.805 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.805 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.805 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:02:24.805 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:59:00.111601+0000", 2024-09-14T05:02:24.805 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0085158530000000003 2024-09-14T05:02:24.805 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.806 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.806 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:02:24.806 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:59:00.111652+0000", 2024-09-14T05:02:24.806 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.0519999999999997e-05 2024-09-14T05:02:24.806 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.806 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.806 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:02:24.806 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:59:00.111702+0000", 2024-09-14T05:02:24.806 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9960999999999998e-05 2024-09-14T05:02:24.806 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.806 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:24.806 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.806 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.806 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.807 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36324.0:150932 57.2 e1537/1518 57:49ded6b4:::benchmark_data_smithi039_365565_object27180:head v 1537'9375, mlcod=1537'9373)", 2024-09-14T05:02:24.807 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:58:59.892053+0000", 2024-09-14T05:02:24.807 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 204.788930755, 2024-09-14T05:02:24.807 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.22847827100000001, 2024-09-14T05:02:24.807 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:24.807 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:24.807 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:02:24.807 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:24.807 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.807 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:24.807 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:58:59.892053+0000", 2024-09-14T05:02:24.807 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.807 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.807 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.808 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:24.808 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:58:59.892053+0000", 2024-09-14T05:02:24.808 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.808 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.808 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.808 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:24.808 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:58:59.892055+0000", 2024-09-14T05:02:24.808 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3179999999999999e-06 2024-09-14T05:02:24.808 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.808 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.808 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:24.808 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:58:59.892064+0000", 2024-09-14T05:02:24.808 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.7430000000000008e-06 2024-09-14T05:02:24.808 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.809 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.809 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:24.809 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:59:00.099063+0000", 2024-09-14T05:02:24.809 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.206999875 2024-09-14T05:02:24.809 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.809 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.809 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:24.809 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:59:00.099079+0000", 2024-09-14T05:02:24.809 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6045999999999998e-05 2024-09-14T05:02:24.809 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.809 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.809 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:24.809 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:59:00.099143+0000", 2024-09-14T05:02:24.809 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3071000000000007e-05 2024-09-14T05:02:24.810 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.810 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.810 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:24.810 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:59:00.099169+0000", 2024-09-14T05:02:24.810 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.6361999999999999e-05 2024-09-14T05:02:24.810 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.810 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.810 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:02:24.810 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:59:00.120466+0000", 2024-09-14T05:02:24.810 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.021296944000000002 2024-09-14T05:02:24.810 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.810 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.811 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:02:24.811 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:59:00.120532+0000", 2024-09-14T05:02:24.811 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.5912000000000002e-05 2024-09-14T05:02:24.811 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.811 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:24.811 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.811 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.811 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.811 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:150933 57.e 57:7bbe5905:::benchmark_data_smithi039_365565_object27181:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1537)", 2024-09-14T05:02:24.811 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:58:59.892106+0000", 2024-09-14T05:02:24.811 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 204.78887780400001, 2024-09-14T05:02:24.811 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.21976357499999999, 2024-09-14T05:02:24.811 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:24.811 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:24.812 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:02:24.812 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:02:24.812 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T05:02:24.812 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:02:24.812 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 150933 2024-09-14T05:02:24.812 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.812 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:24.812 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.812 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:24.812 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:58:59.892106+0000", 2024-09-14T05:02:24.812 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.812 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.812 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.813 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:24.813 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:58:59.892106+0000", 2024-09-14T05:02:24.813 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.813 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.813 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.813 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:24.813 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:58:59.892109+0000", 2024-09-14T05:02:24.813 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0899999999999999e-06 2024-09-14T05:02:24.813 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.813 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.813 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:24.813 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:58:59.892115+0000", 2024-09-14T05:02:24.814 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7680000000000003e-06 2024-09-14T05:02:24.814 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.814 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.814 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:24.814 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:59:00.099068+0000", 2024-09-14T05:02:24.814 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.206952305 2024-09-14T05:02:24.814 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.814 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.814 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:24.814 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:59:00.099080+0000", 2024-09-14T05:02:24.814 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2349e-05 2024-09-14T05:02:24.814 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.814 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.814 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:24.815 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:59:00.103114+0000", 2024-09-14T05:02:24.815 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0040341170000000003 2024-09-14T05:02:24.815 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.815 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.815 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:24.815 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:59:00.103231+0000", 2024-09-14T05:02:24.815 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000117176 2024-09-14T05:02:24.815 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.815 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.815 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T05:02:24.815 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:59:00.103325+0000", 2024-09-14T05:02:24.815 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.4073000000000005e-05 2024-09-14T05:02:24.815 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.815 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.816 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:02:24.816 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:59:00.111586+0000", 2024-09-14T05:02:24.816 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0082609290000000002 2024-09-14T05:02:24.816 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.816 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.816 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:02:24.816 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:59:00.111711+0000", 2024-09-14T05:02:24.816 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012435699999999999 2024-09-14T05:02:24.816 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.816 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.816 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:02:24.816 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:59:00.111783+0000", 2024-09-14T05:02:24.817 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.2902000000000001e-05 2024-09-14T05:02:24.817 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.817 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.817 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:02:24.817 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:59:00.111834+0000", 2024-09-14T05:02:24.817 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.0735999999999997e-05 2024-09-14T05:02:24.817 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.817 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.817 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:02:24.817 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:59:00.111870+0000", 2024-09-14T05:02:24.817 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.5772999999999998e-05 2024-09-14T05:02:24.817 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.817 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:24.818 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.818 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.818 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.818 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop_reply(client.36324.0:150930 57.e e1537/1518 ondisk, result = 0)", 2024-09-14T05:02:24.818 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:58:59.893246+0000", 2024-09-14T05:02:24.818 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 204.787738582, 2024-09-14T05:02:24.818 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.209797553, 2024-09-14T05:02:24.818 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:24.818 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:24.818 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "started", 2024-09-14T05:02:24.818 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:24.818 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.818 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:24.818 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:58:59.893246+0000", 2024-09-14T05:02:24.819 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.819 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.819 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.819 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:24.819 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:58:59.893246+0000", 2024-09-14T05:02:24.819 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.819 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.819 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.819 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:24.819 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:58:59.893247+0000", 2024-09-14T05:02:24.819 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.9900000000000009e-07 2024-09-14T05:02:24.819 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.819 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.820 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:24.820 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:58:59.893257+0000", 2024-09-14T05:02:24.820 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0326000000000001e-05 2024-09-14T05:02:24.820 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.820 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.820 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:24.820 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:59:00.099065+0000", 2024-09-14T05:02:24.820 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.20580765500000001 2024-09-14T05:02:24.820 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.820 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.820 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:24.820 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:59:00.099079+0000", 2024-09-14T05:02:24.820 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4788e-05 2024-09-14T05:02:24.821 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.821 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.821 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:24.821 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:59:00.103028+0000", 2024-09-14T05:02:24.821 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0039482349999999996 2024-09-14T05:02:24.821 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.821 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.821 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:24.821 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:59:00.103034+0000", 2024-09-14T05:02:24.821 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.9850000000000003e-06 2024-09-14T05:02:24.821 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.821 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.821 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:02:24.821 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:59:00.103043+0000", 2024-09-14T05:02:24.822 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.5650000000000007e-06 2024-09-14T05:02:24.822 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.822 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:24.822 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.822 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.822 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.822 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:74245 58.b 58:db51b772:::benchmark_data_smithi039_396950_object74244:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1575)", 2024-09-14T05:02:24.822 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:02:02.744885+0000", 2024-09-14T05:02:24.822 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 21.936099670000001, 2024-09-14T05:02:24.822 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.197465007, 2024-09-14T05:02:24.822 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:24.822 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:24.822 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:02:24.823 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:02:24.823 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:02:24.823 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:02:24.823 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 74245 2024-09-14T05:02:24.823 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.823 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:24.823 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.823 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:24.823 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.744885+0000", 2024-09-14T05:02:24.823 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.823 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.823 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.823 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:24.824 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.744885+0000", 2024-09-14T05:02:24.824 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.824 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.824 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.824 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:24.824 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.744886+0000", 2024-09-14T05:02:24.824 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.55e-06 2024-09-14T05:02:24.824 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.824 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.824 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:24.824 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.744908+0000", 2024-09-14T05:02:24.824 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1798e-05 2024-09-14T05:02:24.825 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.825 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.825 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:24.825 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.744920+0000", 2024-09-14T05:02:24.825 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1897e-05 2024-09-14T05:02:24.825 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.825 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.825 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:24.825 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.744940+0000", 2024-09-14T05:02:24.825 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0630999999999999e-05 2024-09-14T05:02:24.825 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.825 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.825 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:24.826 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.745191+0000", 2024-09-14T05:02:24.826 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000251067 2024-09-14T05:02:24.826 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.826 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.826 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:24.826 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.745302+0000", 2024-09-14T05:02:24.826 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011014399999999999 2024-09-14T05:02:24.826 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.826 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.826 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T05:02:24.826 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.745427+0000", 2024-09-14T05:02:24.826 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000124888 2024-09-14T05:02:24.826 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.826 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.827 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:02:24.827 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.748956+0000", 2024-09-14T05:02:24.827 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00352923 2024-09-14T05:02:24.827 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.827 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.827 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:02:24.827 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.749349+0000", 2024-09-14T05:02:24.827 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00039366299999999999 2024-09-14T05:02:24.827 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.827 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.827 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:02:24.827 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.942286+0000", 2024-09-14T05:02:24.827 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.19293703500000001 2024-09-14T05:02:24.828 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.828 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.828 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:02:24.828 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.942320+0000", 2024-09-14T05:02:24.828 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.3991e-05 2024-09-14T05:02:24.828 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.828 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.828 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:02:24.828 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.942350+0000", 2024-09-14T05:02:24.828 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.9113000000000001e-05 2024-09-14T05:02:24.828 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.828 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:24.828 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.829 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.829 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.829 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:74253 58.b 58:d70b3dc9:::benchmark_data_smithi039_396950_object74252:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1575)", 2024-09-14T05:02:24.829 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:02:02.754439+0000", 2024-09-14T05:02:24.829 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 21.926545656000002, 2024-09-14T05:02:24.829 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.19053752199999999, 2024-09-14T05:02:24.829 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:24.829 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:24.829 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:02:24.829 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:02:24.829 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:02:24.829 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:02:24.829 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 74253 2024-09-14T05:02:24.830 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.830 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:24.830 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.830 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:24.830 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.754439+0000", 2024-09-14T05:02:24.830 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.830 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.830 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.830 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:24.830 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.754439+0000", 2024-09-14T05:02:24.830 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.830 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.830 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.830 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:24.831 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.754439+0000", 2024-09-14T05:02:24.831 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6899999999999997e-07 2024-09-14T05:02:24.831 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.831 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.831 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:24.831 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.754496+0000", 2024-09-14T05:02:24.831 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6492000000000001e-05 2024-09-14T05:02:24.831 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.831 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.831 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:24.831 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.754525+0000", 2024-09-14T05:02:24.831 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.9609999999999999e-05 2024-09-14T05:02:24.831 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.832 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.832 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:24.832 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.754532+0000", 2024-09-14T05:02:24.832 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7719999999999997e-06 2024-09-14T05:02:24.832 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.832 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.832 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:24.832 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.755317+0000", 2024-09-14T05:02:24.832 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00078475599999999995 2024-09-14T05:02:24.832 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.832 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.832 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:24.832 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.755494+0000", 2024-09-14T05:02:24.833 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00017765200000000001 2024-09-14T05:02:24.833 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.833 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.833 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T05:02:24.833 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.755643+0000", 2024-09-14T05:02:24.833 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000148369 2024-09-14T05:02:24.833 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.833 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.833 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:02:24.833 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.763452+0000", 2024-09-14T05:02:24.833 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0078095680000000002 2024-09-14T05:02:24.833 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.833 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.834 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:02:24.834 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.765612+0000", 2024-09-14T05:02:24.834 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0021593329999999998 2024-09-14T05:02:24.834 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.834 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.834 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:02:24.834 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.944819+0000", 2024-09-14T05:02:24.834 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.17920750799999999 2024-09-14T05:02:24.834 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.834 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.834 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:02:24.834 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.944892+0000", 2024-09-14T05:02:24.834 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.2793999999999995e-05 2024-09-14T05:02:24.834 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.835 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.835 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:02:24.835 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.944976+0000", 2024-09-14T05:02:24.835 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.4098999999999996e-05 2024-09-14T05:02:24.835 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.835 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:24.835 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.835 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.835 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.835 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:74259 58.b 58:de0e6b0d:::benchmark_data_smithi039_396950_object74258:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1575)", 2024-09-14T05:02:24.835 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:02:02.755448+0000", 2024-09-14T05:02:24.835 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 21.92553629, 2024-09-14T05:02:24.835 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.19355812999999999, 2024-09-14T05:02:24.836 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:24.836 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:24.836 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:02:24.836 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:02:24.836 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:02:24.836 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:02:24.836 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 74259 2024-09-14T05:02:24.836 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.836 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:24.836 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.836 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:24.836 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.755448+0000", 2024-09-14T05:02:24.836 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.837 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.837 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.837 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:24.837 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.755448+0000", 2024-09-14T05:02:24.837 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:24.837 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.837 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.837 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:24.837 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.755448+0000", 2024-09-14T05:02:24.837 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5400000000000001e-07 2024-09-14T05:02:24.837 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.837 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.837 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:24.838 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.755473+0000", 2024-09-14T05:02:24.838 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4783e-05 2024-09-14T05:02:24.838 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.838 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.838 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:24.838 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.755481+0000", 2024-09-14T05:02:24.838 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.503e-06 2024-09-14T05:02:24.838 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.838 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.838 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:24.838 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.755486+0000", 2024-09-14T05:02:24.838 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.254e-06 2024-09-14T05:02:24.838 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.839 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.839 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:24.839 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.758151+0000", 2024-09-14T05:02:24.839 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0026652080000000001 2024-09-14T05:02:24.839 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.839 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.839 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:24.839 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.758308+0000", 2024-09-14T05:02:24.839 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015635000000000001 2024-09-14T05:02:24.839 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.839 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.839 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T05:02:24.840 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.758449+0000", 2024-09-14T05:02:24.840 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000140949 2024-09-14T05:02:24.840 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.840 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.840 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:02:24.840 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.764703+0000", 2024-09-14T05:02:24.840 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0062548300000000003 2024-09-14T05:02:24.840 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.840 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.840 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:02:24.840 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.765646+0000", 2024-09-14T05:02:24.840 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00094252099999999998 2024-09-14T05:02:24.840 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.841 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.841 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:02:24.841 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.948762+0000", 2024-09-14T05:02:24.841 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.18311570099999999 2024-09-14T05:02:24.841 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.841 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.841 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:02:24.841 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.948903+0000", 2024-09-14T05:02:24.841 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000140989 2024-09-14T05:02:24.841 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:24.841 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:24.841 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:02:24.842 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:02.949006+0000", 2024-09-14T05:02:24.842 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000103488 2024-09-14T05:02:24.842 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.842 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:24.842 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.842 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:24.842 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:24.842 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T05:02:24.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:24.844 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:24.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:24.945 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:24.971 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:24.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:24.972 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:25.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:24 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4034162467' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:02:25.061 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 68 16 96341 96325 88.5079 56.5 0.0124727 0.011293 2024-09-14T05:02:25.073 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:25.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:25.115 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:25.115 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:25.216 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:25.241 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:25.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:25.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:25.343 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:25.371 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:25.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:25.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:25.473 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:25.499 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:25.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:25.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:25.601 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:25.625 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:25.625 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:25.626 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:25.726 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:25.727 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:25 smithi039 ceph-mon[106470]: pgmap v3129: 109 pgs: 109 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 60 MiB/s wr, 958 op/s 2024-09-14T05:02:25.727 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:25 smithi039 ceph-mon[111235]: pgmap v3129: 109 pgs: 109 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 60 MiB/s wr, 958 op/s 2024-09-14T05:02:25.792 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:25.793 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:25.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:25.894 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:25.932 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:25.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:25.933 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:26.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:25 smithi129 ceph-mon[101316]: pgmap v3129: 109 pgs: 109 active+clean; 5.7 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 60 MiB/s wr, 958 op/s 2024-09-14T05:02:26.034 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:26.060 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:26.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:26.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:26.062 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 69 16 97330 97314 88.121 61.8125 0.0154462 0.0113428 2024-09-14T05:02:26.162 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:26.189 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:26.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:26.191 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:26.291 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:26.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:26.346 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:26.346 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:26.447 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:26.473 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:26.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:26.474 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:26.575 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:26.597 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:26.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:26.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:26.698 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:26.723 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:26.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:26.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:26.825 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:26.870 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:26.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:26.872 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:26.972 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:26.997 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:26.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:26.998 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:27.061 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 70 16 98293 98277 87.7219 60.1875 0.00986334 0.0113932 2024-09-14T05:02:27.099 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:27.123 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:27.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:27.124 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:27.225 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:27.263 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:27.264 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:27.265 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:27.365 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:27.401 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:27.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:27.402 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:27.503 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:27.551 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:27.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:27.552 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:27.652 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:27.692 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:27.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:27.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:27.793 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:27.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:27.821 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:27.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:27.922 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:27.948 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:27.948 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:27.949 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:28.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:27 smithi129 ceph-mon[101316]: pgmap v3130: 109 pgs: 109 active+clean; 5.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-14T05:02:28.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:27 smithi039 ceph-mon[106470]: pgmap v3130: 109 pgs: 109 active+clean; 5.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-14T05:02:28.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:27 smithi039 ceph-mon[111235]: pgmap v3130: 109 pgs: 109 active+clean; 5.8 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s 2024-09-14T05:02:28.050 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:28.062 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 71 16 99248 99232 87.3264 59.6875 0.0103323 0.0114458 2024-09-14T05:02:28.095 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:28.096 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:28.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:28.197 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:28.242 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:28.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:28.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:28.344 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:28.375 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:28.375 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:28.376 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:28.477 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:28.514 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:28.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:28.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:28.615 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:28.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:28.641 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:28.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:28.742 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:28.785 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:28.786 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:28.786 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:28.887 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:28.909 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:28.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:28.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:29.011 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:29.037 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:29.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:29.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:29.039 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:02:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:02:28] "GET /metrics HTTP/1.1" 200 36740 "" "Prometheus/2.51.0" 2024-09-14T05:02:29.062 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 72 16 100121 100105 86.8714 54.5625 0.0176501 0.011506 2024-09-14T05:02:29.139 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:29.179 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:29.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:29.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:29.280 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:29.306 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:29.307 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:29.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:29.408 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:29.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:29.435 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:29.435 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:29.536 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:29.562 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:29.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:29.563 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:29.663 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:29.689 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:29.690 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:29.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:29.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:29 smithi039 ceph-mon[106470]: pgmap v3131: 109 pgs: 109 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s 2024-09-14T05:02:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:29 smithi039 ceph-mon[111235]: pgmap v3131: 109 pgs: 109 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s 2024-09-14T05:02:29.790 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:29.813 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:29.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:29.814 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:29.915 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:29.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:29.959 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:29.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:30.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:29 smithi129 ceph-mon[101316]: pgmap v3131: 109 pgs: 109 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 51 MiB/s wr, 819 op/s 2024-09-14T05:02:30.060 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:30.062 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 73 16 101112 101096 86.5298 61.9375 0.0123867 0.0115509 2024-09-14T05:02:30.089 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:30.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:30.090 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:30.191 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:30.229 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:30.230 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:30.230 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:30.331 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:30.361 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:30.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:30.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:30.463 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:30.499 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:30.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:30.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:30.601 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:30.632 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:30.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:30.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:30.734 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:30.771 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:30.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:30.774 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:30.874 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:30.902 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:30.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:30.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:31.004 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:31.041 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:31.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:31.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:31.062 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 74 16 101999 101983 86.1097 55.4375 0.0194968 0.0116078 2024-09-14T05:02:31.143 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:31.172 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:31.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:31.173 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:31.274 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:31.300 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:31.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:31.301 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:31.402 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:31.425 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:31.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:31.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:31.528 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:31.552 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:31.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:31.553 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:31.654 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:31.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:31.700 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:31.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:31 smithi129 ceph-mon[101316]: pgmap v3132: 109 pgs: 109 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-14T05:02:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:31 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:31 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:31 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/234379698' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:02:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:31 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:02:31.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:31 smithi039 ceph-mon[106470]: pgmap v3132: 109 pgs: 109 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-14T05:02:31.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:31 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:31.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:31 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:31.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:31 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/234379698' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:02:31.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:31 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:02:31.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:31 smithi039 ceph-mon[111235]: pgmap v3132: 109 pgs: 109 active+clean; 6.1 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 67 MiB/s wr, 1.06k op/s 2024-09-14T05:02:31.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:31 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:31.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:31 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:31.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:31 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/234379698' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:02:31.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:31 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:02:31.802 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:31.844 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:31.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:31.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:31.946 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:31.972 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:31.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:31.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:32.062 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 75 16 102834 102818 85.6574 52.1875 0.0149506 0.0116691 2024-09-14T05:02:32.073 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-14T05:02:32.074 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:32.097 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:32.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:32.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:32.157 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:02:32.200 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:32.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:32.271 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:32.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:32.370 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is unset 2024-09-14T05:02:32.372 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:32.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:32.439 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:32.439 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:32.540 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:32.565 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:32.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:32.566 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:32.666 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:32.690 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:32.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:32.691 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:32.792 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:32.833 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:32.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:32.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:32.908 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:32 smithi039 ceph-mon[106470]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:02:32.908 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:32 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T05:02:32.908 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:32 smithi039 ceph-mon[106470]: osdmap e1577: 8 total, 8 up, 8 in 2024-09-14T05:02:32.909 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:32 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T05:02:32.909 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:32 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T05:02:32.909 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:32 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:32.909 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:32 smithi039 ceph-mon[111235]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:02:32.909 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:32 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T05:02:32.909 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:32 smithi039 ceph-mon[111235]: osdmap e1577: 8 total, 8 up, 8 in 2024-09-14T05:02:32.909 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:32 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T05:02:32.909 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:32 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T05:02:32.909 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:32 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:32.935 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:32.962 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:32.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:32.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:33.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:32 smithi129 ceph-mon[101316]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:02:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:32 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T05:02:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:32 smithi129 ceph-mon[101316]: osdmap e1577: 8 total, 8 up, 8 in 2024-09-14T05:02:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:32 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T05:02:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:32 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T05:02:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:32 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:33.063 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 76 16 103658 103642 85.208 51.5 0.0150087 0.0117306 2024-09-14T05:02:33.064 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:33.089 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:33.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:33.091 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:33.192 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:33.226 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:33.226 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:33.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:33.327 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:33.425 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:33.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:33.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:33.447 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T05:02:33.527 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:33.554 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:33.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:33.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:33.656 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:33.718 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:33.718 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:33.719 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:33.779 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:33 smithi039 ceph-mon[106470]: pgmap v3133: 109 pgs: 109 active+clean; 6.1 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2024-09-14T05:02:33.779 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:33 smithi039 ceph-mon[111235]: pgmap v3133: 109 pgs: 109 active+clean; 6.1 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2024-09-14T05:02:33.820 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:33.864 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:33.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:33.865 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:33.966 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:34.004 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:34.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:34.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:34.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:33 smithi129 ceph-mon[101316]: pgmap v3133: 109 pgs: 109 active+clean; 6.1 GiB data, 25 GiB used, 691 GiB / 715 GiB avail; 56 MiB/s wr, 901 op/s 2024-09-14T05:02:34.063 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 77 16 104561 104545 84.8343 56.4375 0.0145989 0.0117829 2024-09-14T05:02:34.106 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:34.149 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:34.150 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:34.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:34.166 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:02:34.251 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:34.276 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:34.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:34.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:34.377 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:34.405 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:34.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:34.406 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:34.507 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:34.531 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:34.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:34.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:34.633 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:34.681 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:34.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:34.683 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:34.784 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:34.825 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:34.826 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:34.826 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:34.927 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:34.969 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:34.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:34.970 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:34.985 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:02:35.063 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 78 16 105396 105380 84.4158 52.1875 0.0222342 0.0118405 2024-09-14T05:02:35.071 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:35.095 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:35.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:35.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:35.197 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:35.218 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:35.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:35.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:35.310 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T05:02:35.312 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T05:02:35.320 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:35.345 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:35.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:35.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:35.446 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:35.470 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:35.470 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:35.471 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:35.571 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:35.597 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:35.597 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:35.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:35.609 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T05:02:35.698 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:35.745 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:35.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:35.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:35.846 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:35.900 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:35.900 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:35.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:35.946 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:35 smithi039 ceph-mon[111235]: pgmap v3135: 109 pgs: 109 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2024-09-14T05:02:35.946 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:35 smithi039 ceph-mon[106470]: pgmap v3135: 109 pgs: 109 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2024-09-14T05:02:36.002 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:36.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:35 smithi129 ceph-mon[101316]: pgmap v3135: 109 pgs: 109 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s 2024-09-14T05:02:36.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:36.031 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:36.031 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:36.063 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 79 16 106182 106166 83.9691 49.125 0.0107926 0.0119036 2024-09-14T05:02:36.133 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:36.158 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:36.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:36.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:36.260 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:36.283 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:36.284 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:36.284 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:36.385 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:36.409 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:36.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:36.410 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:36.511 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:36.555 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:36.556 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:36.556 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:36.657 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:36.683 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:36.683 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:36.684 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:36.785 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:36.812 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:36.813 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:36.813 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:36.914 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:36.958 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:36.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:36.960 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:37.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:36 smithi129 ceph-mon[101316]: from='client.36654 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T05:02:37.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:36 smithi039 ceph-mon[106470]: from='client.36654 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T05:02:37.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:36 smithi039 ceph-mon[111235]: from='client.36654 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T05:02:37.060 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:37.063 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:2024-09-14T05:02:37.063411+0000 min lat: 0.00100424 max lat: 0.310727 avg lat: 0.0119659 2024-09-14T05:02:37.063 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2024-09-14T05:02:37.063 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 80 16 106966 106950 83.532 49 0.0202819 0.0119659 2024-09-14T05:02:37.107 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:37.108 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:37.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:37.209 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:37.246 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:37.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:37.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:37.348 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:37.405 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:37.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:37.406 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:37.507 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:37.537 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:37.537 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:37.538 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:37.639 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:37.666 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:37.667 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:37.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:37.768 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:37.797 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:37.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:37.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:37.899 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:37.923 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:37.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:37.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:38.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:37 smithi129 ceph-mon[101316]: pgmap v3136: 109 pgs: 109 active+clean; 6.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s 2024-09-14T05:02:38.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:02:38.025 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:38.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:37 smithi039 ceph-mon[106470]: pgmap v3136: 109 pgs: 109 active+clean; 6.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s 2024-09-14T05:02:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:02:38.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:37 smithi039 ceph-mon[111235]: pgmap v3136: 109 pgs: 109 active+clean; 6.4 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 56 MiB/s wr, 903 op/s 2024-09-14T05:02:38.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:02:38.063 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 81 16 107838 107822 83.1736 54.5 0.0153217 0.0120174 2024-09-14T05:02:38.068 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:38.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:38.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:38.170 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:38.196 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:38.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:38.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:38.298 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:38.348 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:38.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:38.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:38.449 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:38.474 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:38.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:38.475 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:38.576 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:38.601 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:38.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:38.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:38.702 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:38.742 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:38.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:38.744 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:38.814 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":3135,"stamp":"2024-09-14T05:02:34.029899+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6697064421,"num_objects":102206,"num_object_clones":0,"num_object_copies":306618,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":102206,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":102244,"num_write_kb":6539553,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":58,"num_bytes_recovered":1581198,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":105168,"ondisk_log_size":105168,"up":327,"acting":327,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":327,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":26249052,"kb_used_data":19695440,"kb_used_omap":404,"kb_used_meta":6553131,"kb_avail":723712164,"statfs":{"total":767960285184,"available":741081255936,"internally_reserved":0,"allocated":20168130560,"data_stored":20122537500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":414022,"internal_metadata":6710406842},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[9,0,9,10,10,3,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":621215744,"num_objects":9479,"num_object_clones":0,"num_object_copies":28437,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":9479,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9479,"num_write_kb":606656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003374"},"pg_stats":[{"pgid":"58.f","version":"1576'6276","reported_seq":6304,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:29.715246+0000","last_change":"2024-09-14T05:01:06.755007+0000","last_active":"2024-09-14T05:02:29.715246+0000","last_peered":"2024-09-14T05:02:29.715246+0000","last_clean":"2024-09-14T05:02:29.715246+0000","last_became_active":"2024-09-14T05:01:06.754750+0000","last_became_peered":"2024-09-14T05:01:06.754750+0000","last_unstale":"2024-09-14T05:02:29.715246+0000","last_undegraded":"2024-09-14T05:02:29.715246+0000","last_fullsized":"2024-09-14T05:02:29.715246+0000","mapping_epoch":1568,"log_start":"0'0","ondisk_log_start":"0'0","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":6276,"log_dups_size":0,"ondisk_log_size":6276,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:48:27.499645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":411303936,"num_objects":6276,"num_object_clones":0,"num_object_copies":18828,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6276,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6276,"num_write_kb":401664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"58.e","version":"1576'6342","reported_seq":6370,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:30.690820+0000","last_change":"2024-09-14T05:01:07.066710+0000","last_active":"2024-09-14T05:02:30.690820+0000","last_peered":"2024-09-14T05:02:30.690820+0000","last_clean":"2024-09-14T05:02:30.690820+0000","last_became_active":"2024-09-14T05:01:07.066378+0000","last_became_peered":"2024-09-14T05:01:07.066378+0000","last_unstale":"2024-09-14T05:02:30.690820+0000","last_undegraded":"2024-09-14T05:02:30.690820+0000","last_fullsized":"2024-09-14T05:02:30.690820+0000","mapping_epoch":1568,"log_start":"0'0","ondisk_log_start":"0'0","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":6342,"log_dups_size":0,"ondisk_log_size":6342,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:23:26.639576+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":415629312,"num_objects":6342,"num_object_clones":0,"num_object_copies":19026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6342,"num_write_kb":405888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"58.d","version":"1576'6319","reported_seq":6347,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:30.728046+0000","last_change":"2024-09-14T05:01:07.065787+0000","last_active":"2024-09-14T05:02:30.728046+0000","last_peered":"2024-09-14T05:02:30.728046+0000","last_clean":"2024-09-14T05:02:30.728046+0000","last_became_active":"2024-09-14T05:01:07.065475+0000","last_became_peered":"2024-09-14T05:01:07.065475+0000","last_unstale":"2024-09-14T05:02:30.728046+0000","last_undegraded":"2024-09-14T05:02:30.728046+0000","last_fullsized":"2024-09-14T05:02:30.728046+0000","mapping_epoch":1568,"log_start":"0'0","ondisk_log_start":"0'0","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":6319,"log_dups_size":0,"ondisk_log_size":6319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:27.507213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":414121984,"num_objects":6319,"num_object_clones":0,"num_object_copies":18957,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6319,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6319,"num_write_kb":404416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"58.c","version":"1576'6409","reported_seq":6437,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:30.721265+0000","last_change":"2024-09-14T05:01:06.755561+0000","last_active":"2024-09-14T05:02:30.721265+0000","last_peered":"2024-09-14T05:02:30.721265+0000","last_clean":"2024-09-14T05:02:30.721265+0000","last_became_active":"2024-09-14T05:01:06.754338+0000","last_became_peered":"2024-09-14T05:01:06.754338+0000","last_unstale":"2024-09-14T05:02:30.721265+0000","last_undegraded":"2024-09-14T05:02:30.721265+0000","last_fullsized":"2024-09-14T05:02:30.721265+0000","mapping_epoch":1568,"log_start":"0'0","ondisk_log_start":"0'0","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":6409,"log_dups_size":0,"ondisk_log_size":6409,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:06:41.801877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":420020224,"num_objects":6409,"num_object_clones":0,"num_object_copies":19227,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6409,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6409,"num_write_kb":410176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,6],"acting":[2,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"58.b","version":"1576'6366","reported_seq":6394,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:29.734982+0000","last_change":"2024-09-14T05:01:06.754029+0000","last_active":"2024-09-14T05:02:29.734982+0000","last_peered":"2024-09-14T05:02:29.734982+0000","last_clean":"2024-09-14T05:02:29.734982+0000","last_became_active":"2024-09-14T05:01:06.753768+0000","last_became_peered":"2024-09-14T05:01:06.753768+0000","last_unstale":"2024-09-14T05:02:29.734982+0000","last_undegraded":"2024-09-14T05:02:29.734982+0000","last_fullsized":"2024-09-14T05:02:29.734982+0000","mapping_epoch":1568,"log_start":"0'0","ondisk_log_start":"0'0","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":6366,"log_dups_size":0,"ondisk_log_size":6366,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:29:47.870262+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":417202176,"num_objects":6366,"num_object_clones":0,"num_object_copies":19098,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6366,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6366,"num_write_kb":407424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"58.a","version":"1576'6183","reported_seq":6211,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:29.315859+0000","last_change":"2024-09-14T05:01:07.067188+0000","last_active":"2024-09-14T05:02:29.315859+0000","last_peered":"2024-09-14T05:02:29.315859+0000","last_clean":"2024-09-14T05:02:29.315859+0000","last_became_active":"2024-09-14T05:01:07.066676+0000","last_became_peered":"2024-09-14T05:01:07.066676+0000","last_unstale":"2024-09-14T05:02:29.315859+0000","last_undegraded":"2024-09-14T05:02:29.315859+0000","last_fullsized":"2024-09-14T05:02:29.315859+0000","mapping_epoch":1568,"log_start":"0'0","ondisk_log_start":"0'0","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":6183,"log_dups_size":0,"ondisk_log_size":6183,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:12:54.078739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":405209088,"num_objects":6183,"num_object_clones":0,"num_object_copies":18549,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6183,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6183,"num_write_kb":395712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"58.9","version":"1576'6279","reported_seq":6307,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:29.724146+0000","last_change":"2024-09-14T05:01:07.065934+0000","last_active":"2024-09-14T05:02:29.724146+0000","last_peered":"2024-09-14T05:02:29.724146+0000","last_clean":"2024-09-14T05:02:29.724146+0000","last_became_active":"2024-09-14T05:01:07.065651+0000","last_became_peered":"2024-09-14T05:01:07.065651+0000","last_unstale":"2024-09-14T05:02:29.724146+0000","last_undegraded":"2024-09-14T05:02:29.724146+0000","last_fullsized":"2024-09-14T05:02:29.724146+0000","mapping_epoch":1568,"log_start":"0'0","ondisk_log_start":"0'0","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":6279,"log_dups_size":0,"ondisk_log_size":6279,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:34:38.708294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":411500544,"num_objects":6279,"num_object_clones":0,"num_object_copies":18837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6279,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6279,"num_write_kb":401856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"58.8","version":"1576'6389","reported_seq":6416,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:29.707212+0000","last_change":"2024-09-14T05:01:06.753618+0000","last_active":"2024-09-14T05:02:29.707212+0000","last_peered":"2024-09-14T05:02:29.707212+0000","last_clean":"2024-09-14T05:02:29.707212+0000","last_became_active":"2024-09-14T05:01:06.753311+0000","last_became_peered":"2024-09-14T05:01:06.753311+0000","last_unstale":"2024-09-14T05:02:29.707212+0000","last_undegraded":"2024-09-14T05:02:29.707212+0000","last_fullsized":"2024-09-14T05:02:29.707212+0000","mapping_epoch":1568,"log_start":"0'0","ondisk_log_start":"0'0","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":6389,"log_dups_size":0,"ondisk_log_size":6389,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:27:17.541538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":418709504,"num_objects":6389,"num_object_clones":0,"num_object_copies":19167,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6389,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6389,"num_write_kb":408896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,4],"acting":[1,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"58.7","version":"1576'6391","reported_seq":6419,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:30.736114+0000","last_change":"2024-09-14T05:01:07.067197+0000","last_active":"2024-09-14T05:02:30.736114+0000","last_peered":"2024-09-14T05:02:30.736114+0000","last_clean":"2024-09-14T05:02:30.736114+0000","last_became_active":"2024-09-14T05:01:07.066873+0000","last_became_peered":"2024-09-14T05:01:07.066873+0000","last_unstale":"2024-09-14T05:02:30.736114+0000","last_undegraded":"2024-09-14T05:02:30.736114+0000","last_fullsized":"2024-09-14T05:02:30.736114+0000","mapping_epoch":1568,"log_start":"0'0","ondisk_log_start":"0'0","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":6391,"log_dups_size":0,"ondisk_log_size":6391,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:04:54.100383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":418840576,"num_objects":6391,"num_object_clones":0,"num_object_copies":19173,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6391,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6391,"num_write_kb":409024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"58.6","version":"1577'6578","reported_seq":6596,"reported_epoch":1577,"state":"active+clean","last_fresh":"2024-09-14T05:02:33.520894+0000","last_change":"2024-09-14T05:01:07.065564+0000","last_active":"2024-09-14T05:02:33.520894+0000","last_peered":"2024-09-14T05:02:33.520894+0000","last_clean":"2024-09-14T05:02:33.520894+0000","last_became_active":"2024-09-14T05:01:07.065347+0000","last_became_peered":"2024-09-14T05:01:07.065347+0000","last_unstale":"2024-09-14T05:02:33.520894+0000","last_undegraded":"2024-09-14T05:02:33.520894+0000","last_fullsized":"2024-09-14T05:02:33.520894+0000","mapping_epoch":1568,"log_start":"1571'100","ondisk_log_start":"1571'100","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":6478,"log_dups_size":0,"ondisk_log_size":6478,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:19:44.691536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":431095808,"num_objects":6578,"num_object_clones":0,"num_object_copies":19734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6578,"num_write_kb":420992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"58.5","version":"1577'6437","reported_seq":6454,"reported_epoch":1577,"state":"active+clean","last_fresh":"2024-09-14T05:02:33.504579+0000","last_change":"2024-09-14T05:01:06.755410+0000","last_active":"2024-09-14T05:02:33.504579+0000","last_peered":"2024-09-14T05:02:33.504579+0000","last_clean":"2024-09-14T05:02:33.504579+0000","last_became_active":"2024-09-14T05:01:06.755015+0000","last_became_peered":"2024-09-14T05:01:06.755015+0000","last_unstale":"2024-09-14T05:02:33.504579+0000","last_undegraded":"2024-09-14T05:02:33.504579+0000","last_fullsized":"2024-09-14T05:02:33.504579+0000","mapping_epoch":1568,"log_start":"0'0","ondisk_log_start":"0'0","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":6437,"log_dups_size":0,"ondisk_log_size":6437,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:04:44.279865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":421855232,"num_objects":6437,"num_object_clones":0,"num_object_copies":19311,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6437,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6437,"num_write_kb":411968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"58.4","version":"1577'6540","reported_seq":6558,"reported_epoch":1577,"state":"active+clean","last_fresh":"2024-09-14T05:02:33.036890+0000","last_change":"2024-09-14T05:01:07.066762+0000","last_active":"2024-09-14T05:02:33.036890+0000","last_peered":"2024-09-14T05:02:33.036890+0000","last_clean":"2024-09-14T05:02:33.036890+0000","last_became_active":"2024-09-14T05:01:07.066456+0000","last_became_peered":"2024-09-14T05:01:07.066456+0000","last_unstale":"2024-09-14T05:02:33.036890+0000","last_undegraded":"2024-09-14T05:02:33.036890+0000","last_fullsized":"2024-09-14T05:02:33.036890+0000","mapping_epoch":1568,"log_start":"0'0","ondisk_log_start":"0'0","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":6540,"log_dups_size":0,"ondisk_log_size":6540,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:08.301174+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":428605440,"num_objects":6540,"num_object_clones":0,"num_object_copies":19620,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6540,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6540,"num_write_kb":418560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"58.3","version":"1576'6169","reported_seq":6197,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:29.341415+0000","last_change":"2024-09-14T05:01:07.067804+0000","last_active":"2024-09-14T05:02:29.341415+0000","last_peered":"2024-09-14T05:02:29.341415+0000","last_clean":"2024-09-14T05:02:29.341415+0000","last_became_active":"2024-09-14T05:01:07.067259+0000","last_became_peered":"2024-09-14T05:01:07.067259+0000","last_unstale":"2024-09-14T05:02:29.341415+0000","last_undegraded":"2024-09-14T05:02:29.341415+0000","last_fullsized":"2024-09-14T05:02:29.341415+0000","mapping_epoch":1568,"log_start":"0'0","ondisk_log_start":"0'0","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":6169,"log_dups_size":0,"ondisk_log_size":6169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:12:24.658422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":404291584,"num_objects":6169,"num_object_clones":0,"num_object_copies":18507,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6169,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6169,"num_write_kb":394816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,5],"acting":[0,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"58.2","version":"1577'6493","reported_seq":6511,"reported_epoch":1577,"state":"active+clean","last_fresh":"2024-09-14T05:02:32.911124+0000","last_change":"2024-09-14T05:01:07.066830+0000","last_active":"2024-09-14T05:02:32.911124+0000","last_peered":"2024-09-14T05:02:32.911124+0000","last_clean":"2024-09-14T05:02:32.911124+0000","last_became_active":"2024-09-14T05:01:07.066435+0000","last_became_peered":"2024-09-14T05:01:07.066435+0000","last_unstale":"2024-09-14T05:02:32.911124+0000","last_undegraded":"2024-09-14T05:02:32.911124+0000","last_fullsized":"2024-09-14T05:02:32.911124+0000","mapping_epoch":1568,"log_start":"0'0","ondisk_log_start":"0'0","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":6493,"log_dups_size":0,"ondisk_log_size":6493,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:44:03.503451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":425525248,"num_objects":6493,"num_object_clones":0,"num_object_copies":19479,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6493,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6493,"num_write_kb":415552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"58.1","version":"1577'6505","reported_seq":6523,"reported_epoch":1577,"state":"active+clean","last_fresh":"2024-09-14T05:02:33.049610+0000","last_change":"2024-09-14T05:01:06.754637+0000","last_active":"2024-09-14T05:02:33.049610+0000","last_peered":"2024-09-14T05:02:33.049610+0000","last_clean":"2024-09-14T05:02:33.049610+0000","last_became_active":"2024-09-14T05:01:06.754414+0000","last_became_peered":"2024-09-14T05:01:06.754414+0000","last_unstale":"2024-09-14T05:02:33.049610+0000","last_undegraded":"2024-09-14T05:02:33.049610+0000","last_fullsized":"2024-09-14T05:02:33.049610+0000","mapping_epoch":1568,"log_start":"0'0","ondisk_log_start":"0'0","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":6505,"log_dups_size":0,"ondisk_log_size":6505,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:16:30.396684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":426311680,"num_objects":6505,"num_object_clones":0,"num_object_copies":19515,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6505,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6505,"num_write_kb":416320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"58.0","version":"1577'6504","reported_seq":6522,"reported_epoch":1577,"state":"active+clean","last_fresh":"2024-09-14T05:02:33.522929+0000","last_change":"2024-09-14T05:01:07.066052+0000","last_active":"2024-09-14T05:02:33.522929+0000","last_peered":"2024-09-14T05:02:33.522929+0000","last_clean":"2024-09-14T05:02:33.522929+0000","last_became_active":"2024-09-14T05:01:07.065797+0000","last_became_peered":"2024-09-14T05:01:07.065797+0000","last_unstale":"2024-09-14T05:02:33.522929+0000","last_undegraded":"2024-09-14T05:02:33.522929+0000","last_fullsized":"2024-09-14T05:02:33.522929+0000","mapping_epoch":1568,"log_start":"1571'100","ondisk_log_start":"1571'100","created":1568,"last_epoch_clean":1569,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T05:01:05.727572+0000","last_clean_scrub_stamp":"2024-09-14T05:01:05.727572+0000","objects_scrubbed":0,"log_size":6404,"log_dups_size":0,"ondisk_log_size":6404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:46:34.458042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":426246144,"num_objects":6504,"num_object_clones":0,"num_object_copies":19512,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6504,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6504,"num_write_kb":416256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":2448,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.560000+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T05:02:15.560000+0000","last_peered":"2024-09-14T05:02:15.560000+0000","last_clean":"2024-09-14T05:02:15.560000+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T05:02:15.560000+0000","last_undegraded":"2024-09-14T05:02:15.560000+0000","last_fullsized":"2024-09-14T05:02:15.560000+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":192,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.549320+0000","last_change":"2024-09-14T04:44:57.325109+0000","last_active":"2024-09-14T05:02:15.549320+0000","last_peered":"2024-09-14T05:02:15.549320+0000","last_clean":"2024-09-14T05:02:15.549320+0000","last_became_active":"2024-09-14T04:44:57.324791+0000","last_became_peered":"2024-09-14T04:44:57.324791+0000","last_unstale":"2024-09-14T05:02:15.549320+0000","last_undegraded":"2024-09-14T05:02:15.549320+0000","last_fullsized":"2024-09-14T05:02:15.549320+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:03.593275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":162,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.553427+0000","last_change":"2024-09-14T04:49:29.628435+0000","last_active":"2024-09-14T05:02:15.553427+0000","last_peered":"2024-09-14T05:02:15.553427+0000","last_clean":"2024-09-14T05:02:15.553427+0000","last_became_active":"2024-09-14T04:49:29.628081+0000","last_became_peered":"2024-09-14T04:49:29.628081+0000","last_unstale":"2024-09-14T05:02:15.553427+0000","last_undegraded":"2024-09-14T05:02:15.553427+0000","last_fullsized":"2024-09-14T05:02:15.553427+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:10:01.520334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":1068,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.567394+0000","last_change":"2024-09-14T04:43:13.355281+0000","last_active":"2024-09-14T05:02:15.567394+0000","last_peered":"2024-09-14T05:02:15.567394+0000","last_clean":"2024-09-14T05:02:15.567394+0000","last_became_active":"2024-09-14T04:43:13.354967+0000","last_became_peered":"2024-09-14T04:43:13.354967+0000","last_unstale":"2024-09-14T05:02:15.567394+0000","last_undegraded":"2024-09-14T05:02:15.567394+0000","last_fullsized":"2024-09-14T05:02:15.567394+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:23:19.332075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1570,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.552313+0000","last_change":"2024-09-14T04:44:57.342918+0000","last_active":"2024-09-14T05:02:15.552313+0000","last_peered":"2024-09-14T05:02:15.552313+0000","last_clean":"2024-09-14T05:02:15.552313+0000","last_became_active":"2024-09-14T04:44:57.329550+0000","last_became_peered":"2024-09-14T04:44:57.329550+0000","last_unstale":"2024-09-14T05:02:15.552313+0000","last_undegraded":"2024-09-14T05:02:15.552313+0000","last_fullsized":"2024-09-14T05:02:15.552313+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:52:52.435893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":192,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:28.549347+0000","last_change":"2024-09-14T04:44:57.325537+0000","last_active":"2024-09-14T05:02:28.549347+0000","last_peered":"2024-09-14T05:02:28.549347+0000","last_clean":"2024-09-14T05:02:28.549347+0000","last_became_active":"2024-09-14T04:44:57.325057+0000","last_became_peered":"2024-09-14T04:44:57.325057+0000","last_unstale":"2024-09-14T05:02:28.549347+0000","last_undegraded":"2024-09-14T05:02:28.549347+0000","last_fullsized":"2024-09-14T05:02:28.549347+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:43:23.166708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1585,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.554763+0000","last_change":"2024-09-14T04:44:57.337033+0000","last_active":"2024-09-14T05:02:15.554763+0000","last_peered":"2024-09-14T05:02:15.554763+0000","last_clean":"2024-09-14T05:02:15.554763+0000","last_became_active":"2024-09-14T04:44:57.328437+0000","last_became_peered":"2024-09-14T04:44:57.328437+0000","last_unstale":"2024-09-14T05:02:15.554763+0000","last_undegraded":"2024-09-14T05:02:15.554763+0000","last_fullsized":"2024-09-14T05:02:15.554763+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:09:56.109086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":192,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:28.549384+0000","last_change":"2024-09-14T04:44:57.332243+0000","last_active":"2024-09-14T05:02:28.549384+0000","last_peered":"2024-09-14T05:02:28.549384+0000","last_clean":"2024-09-14T05:02:28.549384+0000","last_became_active":"2024-09-14T04:44:57.330413+0000","last_became_peered":"2024-09-14T04:44:57.330413+0000","last_unstale":"2024-09-14T05:02:28.549384+0000","last_undegraded":"2024-09-14T05:02:28.549384+0000","last_fullsized":"2024-09-14T05:02:28.549384+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:25:01.338549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":303,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.559343+0000","last_change":"2024-09-14T04:48:29.862929+0000","last_active":"2024-09-14T05:02:15.559343+0000","last_peered":"2024-09-14T05:02:15.559343+0000","last_clean":"2024-09-14T05:02:15.559343+0000","last_became_active":"2024-09-14T04:48:29.862581+0000","last_became_peered":"2024-09-14T04:48:29.862581+0000","last_unstale":"2024-09-14T05:02:15.559343+0000","last_undegraded":"2024-09-14T05:02:15.559343+0000","last_fullsized":"2024-09-14T05:02:15.559343+0000","mapping_epoch":1432,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1433,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:16:43.060088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2443,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.566716+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T05:02:15.566716+0000","last_peered":"2024-09-14T05:02:15.566716+0000","last_clean":"2024-09-14T05:02:15.566716+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T05:02:15.566716+0000","last_undegraded":"2024-09-14T05:02:15.566716+0000","last_fullsized":"2024-09-14T05:02:15.566716+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":192,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:28.549328+0000","last_change":"2024-09-14T04:44:57.325392+0000","last_active":"2024-09-14T05:02:28.549328+0000","last_peered":"2024-09-14T05:02:28.549328+0000","last_clean":"2024-09-14T05:02:28.549328+0000","last_became_active":"2024-09-14T04:44:57.324624+0000","last_became_peered":"2024-09-14T04:44:57.324624+0000","last_unstale":"2024-09-14T05:02:28.549328+0000","last_undegraded":"2024-09-14T05:02:28.549328+0000","last_fullsized":"2024-09-14T05:02:28.549328+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:33:18.810199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2411,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.570819+0000","last_change":"2024-09-14T04:44:57.329113+0000","last_active":"2024-09-14T05:02:15.570819+0000","last_peered":"2024-09-14T05:02:15.570819+0000","last_clean":"2024-09-14T05:02:15.570819+0000","last_became_active":"2024-09-14T04:44:57.328727+0000","last_became_peered":"2024-09-14T04:44:57.328727+0000","last_unstale":"2024-09-14T05:02:15.570819+0000","last_undegraded":"2024-09-14T05:02:15.570819+0000","last_fullsized":"2024-09-14T05:02:15.570819+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:34.939874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":2428,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.574819+0000","last_change":"2024-09-14T04:44:57.329277+0000","last_active":"2024-09-14T05:02:15.574819+0000","last_peered":"2024-09-14T05:02:15.574819+0000","last_clean":"2024-09-14T05:02:15.574819+0000","last_became_active":"2024-09-14T04:44:57.328804+0000","last_became_peered":"2024-09-14T04:44:57.328804+0000","last_unstale":"2024-09-14T05:02:15.574819+0000","last_undegraded":"2024-09-14T05:02:15.574819+0000","last_fullsized":"2024-09-14T05:02:15.574819+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:42:46.055240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2464,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.559538+0000","last_change":"2024-09-14T04:54:52.250695+0000","last_active":"2024-09-14T05:02:15.559538+0000","last_peered":"2024-09-14T05:02:15.559538+0000","last_clean":"2024-09-14T05:02:15.559538+0000","last_became_active":"2024-09-14T04:54:52.250301+0000","last_became_peered":"2024-09-14T04:54:52.250301+0000","last_unstale":"2024-09-14T05:02:15.559538+0000","last_undegraded":"2024-09-14T05:02:15.559538+0000","last_fullsized":"2024-09-14T05:02:15.559538+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:42.060905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2424,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.558823+0000","last_change":"2024-09-14T04:43:13.357092+0000","last_active":"2024-09-14T05:02:15.558823+0000","last_peered":"2024-09-14T05:02:15.558823+0000","last_clean":"2024-09-14T05:02:15.558823+0000","last_became_active":"2024-09-14T04:43:13.356694+0000","last_became_peered":"2024-09-14T04:43:13.356694+0000","last_unstale":"2024-09-14T05:02:15.558823+0000","last_undegraded":"2024-09-14T05:02:15.558823+0000","last_fullsized":"2024-09-14T05:02:15.558823+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:44:11.167574+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":1564,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.554187+0000","last_change":"2024-09-14T04:07:21.720797+0000","last_active":"2024-09-14T05:02:15.554187+0000","last_peered":"2024-09-14T05:02:15.554187+0000","last_clean":"2024-09-14T05:02:15.554187+0000","last_became_active":"2024-09-14T04:07:21.719331+0000","last_became_peered":"2024-09-14T04:07:21.719331+0000","last_unstale":"2024-09-14T05:02:15.554187+0000","last_undegraded":"2024-09-14T05:02:15.554187+0000","last_fullsized":"2024-09-14T05:02:15.554187+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:59:35.404023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":211,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.552110+0000","last_change":"2024-09-14T04:43:11.641514+0000","last_active":"2024-09-14T05:02:15.552110+0000","last_peered":"2024-09-14T05:02:15.552110+0000","last_clean":"2024-09-14T05:02:15.552110+0000","last_became_active":"2024-09-14T04:43:11.641151+0000","last_became_peered":"2024-09-14T04:43:11.641151+0000","last_unstale":"2024-09-14T05:02:15.552110+0000","last_undegraded":"2024-09-14T05:02:15.552110+0000","last_fullsized":"2024-09-14T05:02:15.552110+0000","mapping_epoch":1384,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1385,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:06:45.045430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2436,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.560161+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T05:02:15.560161+0000","last_peered":"2024-09-14T05:02:15.560161+0000","last_clean":"2024-09-14T05:02:15.560161+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T05:02:15.560161+0000","last_undegraded":"2024-09-14T05:02:15.560161+0000","last_fullsized":"2024-09-14T05:02:15.560161+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.14","version":"349'149","reported_seq":909,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.567137+0000","last_change":"2024-09-14T05:00:39.545797+0000","last_active":"2024-09-14T05:02:15.567137+0000","last_peered":"2024-09-14T05:02:15.567137+0000","last_clean":"2024-09-14T05:02:15.567137+0000","last_became_active":"2024-09-14T05:00:08.501982+0000","last_became_peered":"2024-09-14T05:00:08.501982+0000","last_unstale":"2024-09-14T05:02:15.567137+0000","last_undegraded":"2024-09-14T05:02:15.567137+0000","last_fullsized":"2024-09-14T05:02:15.567137+0000","mapping_epoch":1549,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1550,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:39.545724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:39.545724+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:39:33.377804+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":791,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.569128+0000","last_change":"2024-09-14T04:49:29.618477+0000","last_active":"2024-09-14T05:02:15.569128+0000","last_peered":"2024-09-14T05:02:15.569128+0000","last_clean":"2024-09-14T05:02:15.569128+0000","last_became_active":"2024-09-14T04:49:29.618210+0000","last_became_peered":"2024-09-14T04:49:29.618210+0000","last_unstale":"2024-09-14T05:02:15.569128+0000","last_undegraded":"2024-09-14T05:02:15.569128+0000","last_fullsized":"2024-09-14T05:02:15.569128+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:13:23.895512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1569,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.552475+0000","last_change":"2024-09-14T04:42:49.938505+0000","last_active":"2024-09-14T05:02:15.552475+0000","last_peered":"2024-09-14T05:02:15.552475+0000","last_clean":"2024-09-14T05:02:15.552475+0000","last_became_active":"2024-09-14T04:42:49.938180+0000","last_became_peered":"2024-09-14T04:42:49.938180+0000","last_unstale":"2024-09-14T05:02:15.552475+0000","last_undegraded":"2024-09-14T05:02:15.552475+0000","last_fullsized":"2024-09-14T05:02:15.552475+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:05.929479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1588,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.569712+0000","last_change":"2024-09-14T04:44:57.346582+0000","last_active":"2024-09-14T05:02:15.569712+0000","last_peered":"2024-09-14T05:02:15.569712+0000","last_clean":"2024-09-14T05:02:15.569712+0000","last_became_active":"2024-09-14T04:44:57.330446+0000","last_became_peered":"2024-09-14T04:44:57.330446+0000","last_unstale":"2024-09-14T05:02:15.569712+0000","last_undegraded":"2024-09-14T05:02:15.569712+0000","last_fullsized":"2024-09-14T05:02:15.569712+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:28:29.238585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":377,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.563119+0000","last_change":"2024-09-14T04:44:14.480667+0000","last_active":"2024-09-14T05:02:15.563119+0000","last_peered":"2024-09-14T05:02:15.563119+0000","last_clean":"2024-09-14T05:02:15.563119+0000","last_became_active":"2024-09-14T04:44:14.480365+0000","last_became_peered":"2024-09-14T04:44:14.480365+0000","last_unstale":"2024-09-14T05:02:15.563119+0000","last_undegraded":"2024-09-14T05:02:15.563119+0000","last_fullsized":"2024-09-14T05:02:15.563119+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:00.611771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":202,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.549634+0000","last_change":"2024-09-14T04:44:57.341426+0000","last_active":"2024-09-14T05:02:15.549634+0000","last_peered":"2024-09-14T05:02:15.549634+0000","last_clean":"2024-09-14T05:02:15.549634+0000","last_became_active":"2024-09-14T04:44:57.328763+0000","last_became_peered":"2024-09-14T04:44:57.328763+0000","last_unstale":"2024-09-14T05:02:15.549634+0000","last_undegraded":"2024-09-14T05:02:15.549634+0000","last_fullsized":"2024-09-14T05:02:15.549634+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:34:40.066761+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4756,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":192,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.549544+0000","last_change":"2024-09-14T04:44:57.331337+0000","last_active":"2024-09-14T05:02:15.549544+0000","last_peered":"2024-09-14T05:02:15.549544+0000","last_clean":"2024-09-14T05:02:15.549544+0000","last_became_active":"2024-09-14T04:44:57.329336+0000","last_became_peered":"2024-09-14T04:44:57.329336+0000","last_unstale":"2024-09-14T05:02:15.549544+0000","last_undegraded":"2024-09-14T05:02:15.549544+0000","last_fullsized":"2024-09-14T05:02:15.549544+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:52:35.980869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"349'149","reported_seq":802,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.554659+0000","last_change":"2024-09-14T05:00:31.577572+0000","last_active":"2024-09-14T05:02:15.554659+0000","last_peered":"2024-09-14T05:02:15.554659+0000","last_clean":"2024-09-14T05:02:15.554659+0000","last_became_active":"2024-09-14T05:00:01.425392+0000","last_became_peered":"2024-09-14T05:00:01.425392+0000","last_unstale":"2024-09-14T05:02:15.554659+0000","last_undegraded":"2024-09-14T05:02:15.554659+0000","last_fullsized":"2024-09-14T05:02:15.554659+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1544,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:31.577382+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:31.577382+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:08:04.884554+0000","scrub_duration":0.0060191919999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":193,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.548791+0000","last_change":"2024-09-14T04:44:57.329870+0000","last_active":"2024-09-14T05:02:15.548791+0000","last_peered":"2024-09-14T05:02:15.548791+0000","last_clean":"2024-09-14T05:02:15.548791+0000","last_became_active":"2024-09-14T04:44:57.329353+0000","last_became_peered":"2024-09-14T04:44:57.329353+0000","last_unstale":"2024-09-14T05:02:15.548791+0000","last_undegraded":"2024-09-14T05:02:15.548791+0000","last_fullsized":"2024-09-14T05:02:15.548791+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:21:46.432181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1585,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.554235+0000","last_change":"2024-09-14T04:54:52.242207+0000","last_active":"2024-09-14T05:02:15.554235+0000","last_peered":"2024-09-14T05:02:15.554235+0000","last_clean":"2024-09-14T05:02:15.554235+0000","last_became_active":"2024-09-14T04:54:52.241840+0000","last_became_peered":"2024-09-14T04:54:52.241840+0000","last_unstale":"2024-09-14T05:02:15.554235+0000","last_undegraded":"2024-09-14T05:02:15.554235+0000","last_fullsized":"2024-09-14T05:02:15.554235+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:37.899666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":2475,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.559483+0000","last_change":"2024-09-14T04:44:57.344614+0000","last_active":"2024-09-14T05:02:15.559483+0000","last_peered":"2024-09-14T05:02:15.559483+0000","last_clean":"2024-09-14T05:02:15.559483+0000","last_became_active":"2024-09-14T04:44:57.332256+0000","last_became_peered":"2024-09-14T04:44:57.332256+0000","last_unstale":"2024-09-14T05:02:15.559483+0000","last_undegraded":"2024-09-14T05:02:15.559483+0000","last_fullsized":"2024-09-14T05:02:15.559483+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:23.438417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"349'149","reported_seq":972,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.550993+0000","last_change":"2024-09-14T05:00:31.015778+0000","last_active":"2024-09-14T05:02:15.550993+0000","last_peered":"2024-09-14T05:02:15.550993+0000","last_clean":"2024-09-14T05:02:15.550993+0000","last_became_active":"2024-09-14T05:00:01.423833+0000","last_became_peered":"2024-09-14T05:00:01.423833+0000","last_unstale":"2024-09-14T05:02:15.550993+0000","last_undegraded":"2024-09-14T05:02:15.550993+0000","last_fullsized":"2024-09-14T05:02:15.550993+0000","mapping_epoch":1405,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1544,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:31.015650+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:31.015650+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:34:48.213379+0000","scrub_duration":0.0057407420000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":782,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.569285+0000","last_change":"2024-09-14T04:49:29.616831+0000","last_active":"2024-09-14T05:02:15.569285+0000","last_peered":"2024-09-14T05:02:15.569285+0000","last_clean":"2024-09-14T05:02:15.569285+0000","last_became_active":"2024-09-14T04:49:29.616546+0000","last_became_peered":"2024-09-14T04:49:29.616546+0000","last_unstale":"2024-09-14T05:02:15.569285+0000","last_undegraded":"2024-09-14T05:02:15.569285+0000","last_fullsized":"2024-09-14T05:02:15.569285+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:27.882929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":430,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.565471+0000","last_change":"2024-09-14T04:44:57.330746+0000","last_active":"2024-09-14T05:02:15.565471+0000","last_peered":"2024-09-14T05:02:15.565471+0000","last_clean":"2024-09-14T05:02:15.565471+0000","last_became_active":"2024-09-14T04:44:57.330327+0000","last_became_peered":"2024-09-14T04:44:57.330327+0000","last_unstale":"2024-09-14T05:02:15.565471+0000","last_undegraded":"2024-09-14T05:02:15.565471+0000","last_fullsized":"2024-09-14T05:02:15.565471+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:37:53.122771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":232,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.569951+0000","last_change":"2024-09-14T04:43:13.355296+0000","last_active":"2024-09-14T05:02:15.569951+0000","last_peered":"2024-09-14T05:02:15.569951+0000","last_clean":"2024-09-14T05:02:15.569951+0000","last_became_active":"2024-09-14T04:43:13.354950+0000","last_became_peered":"2024-09-14T04:43:13.354950+0000","last_unstale":"2024-09-14T05:02:15.569951+0000","last_undegraded":"2024-09-14T05:02:15.569951+0000","last_fullsized":"2024-09-14T05:02:15.569951+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:20:08.968818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"349'149","reported_seq":910,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.553319+0000","last_change":"2024-09-14T05:00:37.706342+0000","last_active":"2024-09-14T05:02:15.553319+0000","last_peered":"2024-09-14T05:02:15.553319+0000","last_clean":"2024-09-14T05:02:15.553319+0000","last_became_active":"2024-09-14T05:00:01.433252+0000","last_became_peered":"2024-09-14T05:00:01.433252+0000","last_unstale":"2024-09-14T05:02:15.553319+0000","last_undegraded":"2024-09-14T05:02:15.553319+0000","last_fullsized":"2024-09-14T05:02:15.553319+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1544,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:37.706187+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:37.706187+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:27:54.986789+0000","scrub_duration":0.018654753,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2431,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.560061+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T05:02:15.560061+0000","last_peered":"2024-09-14T05:02:15.560061+0000","last_clean":"2024-09-14T05:02:15.560061+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T05:02:15.560061+0000","last_undegraded":"2024-09-14T05:02:15.560061+0000","last_fullsized":"2024-09-14T05:02:15.560061+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1599,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.569797+0000","last_change":"2024-09-14T04:44:57.331436+0000","last_active":"2024-09-14T05:02:15.569797+0000","last_peered":"2024-09-14T05:02:15.569797+0000","last_clean":"2024-09-14T05:02:15.569797+0000","last_became_active":"2024-09-14T04:44:57.330957+0000","last_became_peered":"2024-09-14T04:44:57.330957+0000","last_unstale":"2024-09-14T05:02:15.569797+0000","last_undegraded":"2024-09-14T05:02:15.569797+0000","last_fullsized":"2024-09-14T05:02:15.569797+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:59:09.516688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":980,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.566624+0000","last_change":"2024-09-14T04:49:49.460401+0000","last_active":"2024-09-14T05:02:15.566624+0000","last_peered":"2024-09-14T05:02:15.566624+0000","last_clean":"2024-09-14T05:02:15.566624+0000","last_became_active":"2024-09-14T04:49:49.458530+0000","last_became_peered":"2024-09-14T04:49:49.458530+0000","last_unstale":"2024-09-14T05:02:15.566624+0000","last_undegraded":"2024-09-14T05:02:15.566624+0000","last_fullsized":"2024-09-14T05:02:15.566624+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:11:29.143239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,5],"acting":[2,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"349'149","reported_seq":944,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.554103+0000","last_change":"2024-09-14T05:00:41.710202+0000","last_active":"2024-09-14T05:02:15.554103+0000","last_peered":"2024-09-14T05:02:15.554103+0000","last_clean":"2024-09-14T05:02:15.554103+0000","last_became_active":"2024-09-14T05:00:01.424753+0000","last_became_peered":"2024-09-14T05:00:01.424753+0000","last_unstale":"2024-09-14T05:02:15.554103+0000","last_undegraded":"2024-09-14T05:02:15.554103+0000","last_fullsized":"2024-09-14T05:02:15.554103+0000","mapping_epoch":1492,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1544,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:41.710095+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:41.710095+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:48:46.305394+0000","scrub_duration":0.0014057430000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":385,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.563003+0000","last_change":"2024-09-14T04:44:57.329983+0000","last_active":"2024-09-14T05:02:15.563003+0000","last_peered":"2024-09-14T05:02:15.563003+0000","last_clean":"2024-09-14T05:02:15.563003+0000","last_became_active":"2024-09-14T04:44:57.329597+0000","last_became_peered":"2024-09-14T04:44:57.329597+0000","last_unstale":"2024-09-14T05:02:15.563003+0000","last_undegraded":"2024-09-14T05:02:15.563003+0000","last_fullsized":"2024-09-14T05:02:15.563003+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:17:45.718039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":924,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.559111+0000","last_change":"2024-09-14T04:49:49.443631+0000","last_active":"2024-09-14T05:02:15.559111+0000","last_peered":"2024-09-14T05:02:15.559111+0000","last_clean":"2024-09-14T05:02:15.559111+0000","last_became_active":"2024-09-14T04:49:49.443313+0000","last_became_peered":"2024-09-14T04:49:49.443313+0000","last_unstale":"2024-09-14T05:02:15.559111+0000","last_undegraded":"2024-09-14T05:02:15.559111+0000","last_fullsized":"2024-09-14T05:02:15.559111+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:06:54.562292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,7],"acting":[0,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1572,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.552920+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T05:02:15.552920+0000","last_peered":"2024-09-14T05:02:15.552920+0000","last_clean":"2024-09-14T05:02:15.552920+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T05:02:15.552920+0000","last_undegraded":"2024-09-14T05:02:15.552920+0000","last_fullsized":"2024-09-14T05:02:15.552920+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"349'149","reported_seq":903,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:27.917604+0000","last_change":"2024-09-14T05:00:33.562385+0000","last_active":"2024-09-14T05:02:27.917604+0000","last_peered":"2024-09-14T05:02:27.917604+0000","last_clean":"2024-09-14T05:02:27.917604+0000","last_became_active":"2024-09-14T05:00:01.421507+0000","last_became_peered":"2024-09-14T05:00:01.421507+0000","last_unstale":"2024-09-14T05:02:27.917604+0000","last_undegraded":"2024-09-14T05:02:27.917604+0000","last_fullsized":"2024-09-14T05:02:27.917604+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1544,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:33.562211+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:33.562211+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:32:46.187217+0000","scrub_duration":0.0066960869999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":1016,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.566799+0000","last_change":"2024-09-14T04:49:49.459590+0000","last_active":"2024-09-14T05:02:15.566799+0000","last_peered":"2024-09-14T05:02:15.566799+0000","last_clean":"2024-09-14T05:02:15.566799+0000","last_became_active":"2024-09-14T04:49:49.456263+0000","last_became_peered":"2024-09-14T04:49:49.456263+0000","last_unstale":"2024-09-14T05:02:15.566799+0000","last_undegraded":"2024-09-14T05:02:15.566799+0000","last_fullsized":"2024-09-14T05:02:15.566799+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:20:44.666805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":193,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.549480+0000","last_change":"2024-09-14T04:44:57.329652+0000","last_active":"2024-09-14T05:02:15.549480+0000","last_peered":"2024-09-14T05:02:15.549480+0000","last_clean":"2024-09-14T05:02:15.549480+0000","last_became_active":"2024-09-14T04:44:57.328888+0000","last_became_peered":"2024-09-14T04:44:57.328888+0000","last_unstale":"2024-09-14T05:02:15.549480+0000","last_undegraded":"2024-09-14T05:02:15.549480+0000","last_fullsized":"2024-09-14T05:02:15.549480+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:14:10.170068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1570,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.552698+0000","last_change":"2024-09-14T04:06:46.875340+0000","last_active":"2024-09-14T05:02:15.552698+0000","last_peered":"2024-09-14T05:02:15.552698+0000","last_clean":"2024-09-14T05:02:15.552698+0000","last_became_active":"2024-09-14T04:06:46.874552+0000","last_became_peered":"2024-09-14T04:06:46.874552+0000","last_unstale":"2024-09-14T05:02:15.552698+0000","last_undegraded":"2024-09-14T05:02:15.552698+0000","last_fullsized":"2024-09-14T05:02:15.552698+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:59:52.257357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"349'149","reported_seq":805,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.570750+0000","last_change":"2024-09-14T05:00:02.067572+0000","last_active":"2024-09-14T05:02:15.570750+0000","last_peered":"2024-09-14T05:02:15.570750+0000","last_clean":"2024-09-14T05:02:15.570750+0000","last_became_active":"2024-09-14T05:00:01.416741+0000","last_became_peered":"2024-09-14T05:00:01.416741+0000","last_unstale":"2024-09-14T05:02:15.570750+0000","last_undegraded":"2024-09-14T05:02:15.570750+0000","last_fullsized":"2024-09-14T05:02:15.570750+0000","mapping_epoch":1506,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1544,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:02.067463+0000","last_deep_scrub":"349'149","last_deep_scrub_stamp":"2024-09-14T05:00:02.067463+0000","last_clean_scrub_stamp":"2024-09-14T05:00:02.067463+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:12:45.764862+0000","scrub_duration":0.0033042340000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":378,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.571136+0000","last_change":"2024-09-14T04:44:14.489235+0000","last_active":"2024-09-14T05:02:15.571136+0000","last_peered":"2024-09-14T05:02:15.571136+0000","last_clean":"2024-09-14T05:02:15.571136+0000","last_became_active":"2024-09-14T04:44:14.488950+0000","last_became_peered":"2024-09-14T04:44:14.488950+0000","last_unstale":"2024-09-14T05:02:15.571136+0000","last_undegraded":"2024-09-14T05:02:15.571136+0000","last_fullsized":"2024-09-14T05:02:15.571136+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:54.441144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":934,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.567015+0000","last_change":"2024-09-14T04:49:49.456317+0000","last_active":"2024-09-14T05:02:15.567015+0000","last_peered":"2024-09-14T05:02:15.567015+0000","last_clean":"2024-09-14T05:02:15.567015+0000","last_became_active":"2024-09-14T04:49:49.456029+0000","last_became_peered":"2024-09-14T04:49:49.456029+0000","last_unstale":"2024-09-14T05:02:15.567015+0000","last_undegraded":"2024-09-14T05:02:15.567015+0000","last_fullsized":"2024-09-14T05:02:15.567015+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:46:05.319676+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,6],"acting":[2,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2451,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.567484+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T05:02:15.567484+0000","last_peered":"2024-09-14T05:02:15.567484+0000","last_clean":"2024-09-14T05:02:15.567484+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T05:02:15.567484+0000","last_undegraded":"2024-09-14T05:02:15.567484+0000","last_fullsized":"2024-09-14T05:02:15.567484+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"349'149","reported_seq":797,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:18.055691+0000","last_change":"2024-09-14T05:00:33.570941+0000","last_active":"2024-09-14T05:02:18.055691+0000","last_peered":"2024-09-14T05:02:18.055691+0000","last_clean":"2024-09-14T05:02:18.055691+0000","last_became_active":"2024-09-14T05:00:01.433375+0000","last_became_peered":"2024-09-14T05:00:01.433375+0000","last_unstale":"2024-09-14T05:02:18.055691+0000","last_undegraded":"2024-09-14T05:02:18.055691+0000","last_fullsized":"2024-09-14T05:02:18.055691+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1544,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:33.570779+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:33.570779+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:01:04.260053+0000","scrub_duration":0.010737673999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2452,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.559936+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T05:02:15.559936+0000","last_peered":"2024-09-14T05:02:15.559936+0000","last_clean":"2024-09-14T05:02:15.559936+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T05:02:15.559936+0000","last_undegraded":"2024-09-14T05:02:15.559936+0000","last_fullsized":"2024-09-14T05:02:15.559936+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2473,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.560918+0000","last_change":"2024-09-14T04:54:19.447103+0000","last_active":"2024-09-14T05:02:15.560918+0000","last_peered":"2024-09-14T05:02:15.560918+0000","last_clean":"2024-09-14T05:02:15.560918+0000","last_became_active":"2024-09-14T04:54:19.446748+0000","last_became_peered":"2024-09-14T04:54:19.446748+0000","last_unstale":"2024-09-14T05:02:15.560918+0000","last_undegraded":"2024-09-14T05:02:15.560918+0000","last_fullsized":"2024-09-14T05:02:15.560918+0000","mapping_epoch":1489,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:00:55.369263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1601,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.569254+0000","last_change":"2024-09-14T04:42:49.939790+0000","last_active":"2024-09-14T05:02:15.569254+0000","last_peered":"2024-09-14T05:02:15.569254+0000","last_clean":"2024-09-14T05:02:15.569254+0000","last_became_active":"2024-09-14T04:42:49.939591+0000","last_became_peered":"2024-09-14T04:42:49.939591+0000","last_unstale":"2024-09-14T05:02:15.569254+0000","last_undegraded":"2024-09-14T05:02:15.569254+0000","last_fullsized":"2024-09-14T05:02:15.569254+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:23:16.790593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":1564,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.553487+0000","last_change":"2024-09-14T04:44:57.326008+0000","last_active":"2024-09-14T05:02:15.553487+0000","last_peered":"2024-09-14T05:02:15.553487+0000","last_clean":"2024-09-14T05:02:15.553487+0000","last_became_active":"2024-09-14T04:44:57.325617+0000","last_became_peered":"2024-09-14T04:44:57.325617+0000","last_unstale":"2024-09-14T05:02:15.553487+0000","last_undegraded":"2024-09-14T05:02:15.553487+0000","last_fullsized":"2024-09-14T05:02:15.553487+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:05:15.637508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":853,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.552776+0000","last_change":"2024-09-14T04:43:13.354846+0000","last_active":"2024-09-14T05:02:15.552776+0000","last_peered":"2024-09-14T05:02:15.552776+0000","last_clean":"2024-09-14T05:02:15.552776+0000","last_became_active":"2024-09-14T04:43:13.354515+0000","last_became_peered":"2024-09-14T04:43:13.354515+0000","last_unstale":"2024-09-14T05:02:15.552776+0000","last_undegraded":"2024-09-14T05:02:15.552776+0000","last_fullsized":"2024-09-14T05:02:15.552776+0000","mapping_epoch":1386,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1387,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:02:17.889341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1569,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.553758+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T05:02:15.553758+0000","last_peered":"2024-09-14T05:02:15.553758+0000","last_clean":"2024-09-14T05:02:15.553758+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T05:02:15.553758+0000","last_undegraded":"2024-09-14T05:02:15.553758+0000","last_fullsized":"2024-09-14T05:02:15.553758+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":1928,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.569026+0000","last_change":"2024-09-14T05:00:32.949837+0000","last_active":"2024-09-14T05:02:15.569026+0000","last_peered":"2024-09-14T05:02:15.569026+0000","last_clean":"2024-09-14T05:02:15.569026+0000","last_became_active":"2024-09-14T05:00:01.433169+0000","last_became_peered":"2024-09-14T05:00:01.433169+0000","last_unstale":"2024-09-14T05:02:15.569026+0000","last_undegraded":"2024-09-14T05:02:15.569026+0000","last_fullsized":"2024-09-14T05:02:15.569026+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1544,"parent":"0.0","parent_split_bits":0,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:32.949645+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:32.949645+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:04.207148+0000","scrub_duration":0.0087080530000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"349'149","reported_seq":881,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.552620+0000","last_change":"2024-09-14T04:41:39.978080+0000","last_active":"2024-09-14T05:02:15.552620+0000","last_peered":"2024-09-14T05:02:15.552620+0000","last_clean":"2024-09-14T05:02:15.552620+0000","last_became_active":"2024-09-14T04:41:36.929279+0000","last_became_peered":"2024-09-14T04:41:36.929279+0000","last_unstale":"2024-09-14T05:02:15.552620+0000","last_undegraded":"2024-09-14T05:02:15.552620+0000","last_fullsized":"2024-09-14T05:02:15.552620+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:39.977999+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:39.977999+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:44:06.383827+0000","scrub_duration":0.00192122,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":312,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.571985+0000","last_change":"2024-09-14T04:48:11.040133+0000","last_active":"2024-09-14T05:02:15.571985+0000","last_peered":"2024-09-14T05:02:15.571985+0000","last_clean":"2024-09-14T05:02:15.571985+0000","last_became_active":"2024-09-14T04:48:11.039787+0000","last_became_peered":"2024-09-14T04:48:11.039787+0000","last_unstale":"2024-09-14T05:02:15.571985+0000","last_undegraded":"2024-09-14T05:02:15.571985+0000","last_fullsized":"2024-09-14T05:02:15.571985+0000","mapping_epoch":1428,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1429,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:53.520672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2299,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.564367+0000","last_change":"2024-09-14T04:49:29.616443+0000","last_active":"2024-09-14T05:02:15.564367+0000","last_peered":"2024-09-14T05:02:15.564367+0000","last_clean":"2024-09-14T05:02:15.564367+0000","last_became_active":"2024-09-14T04:49:29.616066+0000","last_became_peered":"2024-09-14T04:49:29.616066+0000","last_unstale":"2024-09-14T05:02:15.564367+0000","last_undegraded":"2024-09-14T05:02:15.564367+0000","last_fullsized":"2024-09-14T05:02:15.564367+0000","mapping_epoch":1442,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1443,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:09:15.897957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.b","version":"349'149","reported_seq":934,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.559282+0000","last_change":"2024-09-14T05:00:39.684030+0000","last_active":"2024-09-14T05:02:15.559282+0000","last_peered":"2024-09-14T05:02:15.559282+0000","last_clean":"2024-09-14T05:02:15.559282+0000","last_became_active":"2024-09-14T05:00:08.490295+0000","last_became_peered":"2024-09-14T05:00:08.490295+0000","last_unstale":"2024-09-14T05:02:15.559282+0000","last_undegraded":"2024-09-14T05:02:15.559282+0000","last_fullsized":"2024-09-14T05:02:15.559282+0000","mapping_epoch":1549,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1550,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:39.683936+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:39.683936+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:38:14.405290+0000","scrub_duration":7,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,5],"acting":[0,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":192,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.550990+0000","last_change":"2024-09-14T04:44:57.326724+0000","last_active":"2024-09-14T05:02:15.550990+0000","last_peered":"2024-09-14T05:02:15.550990+0000","last_clean":"2024-09-14T05:02:15.550990+0000","last_became_active":"2024-09-14T04:44:57.326191+0000","last_became_peered":"2024-09-14T04:44:57.326191+0000","last_unstale":"2024-09-14T05:02:15.550990+0000","last_undegraded":"2024-09-14T05:02:15.550990+0000","last_fullsized":"2024-09-14T05:02:15.550990+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:46.060269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":193,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:18.546659+0000","last_change":"2024-09-14T04:44:57.328528+0000","last_active":"2024-09-14T05:02:18.546659+0000","last_peered":"2024-09-14T05:02:18.546659+0000","last_clean":"2024-09-14T05:02:18.546659+0000","last_became_active":"2024-09-14T04:44:57.327739+0000","last_became_peered":"2024-09-14T04:44:57.327739+0000","last_unstale":"2024-09-14T05:02:18.546659+0000","last_undegraded":"2024-09-14T05:02:18.546659+0000","last_fullsized":"2024-09-14T05:02:18.546659+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:04:58.294690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"349'149","reported_seq":910,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.566967+0000","last_change":"2024-09-14T05:00:24.275751+0000","last_active":"2024-09-14T05:02:15.566967+0000","last_peered":"2024-09-14T05:02:15.566967+0000","last_clean":"2024-09-14T05:02:15.566967+0000","last_became_active":"2024-09-14T05:00:24.275201+0000","last_became_peered":"2024-09-14T05:00:24.275201+0000","last_unstale":"2024-09-14T05:02:15.566967+0000","last_undegraded":"2024-09-14T05:02:15.566967+0000","last_fullsized":"2024-09-14T05:02:15.566967+0000","mapping_epoch":1555,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1556,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:56.738636+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:56.738636+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:41:11.155132+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1576128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":192,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.550844+0000","last_change":"2024-09-14T04:44:57.323436+0000","last_active":"2024-09-14T05:02:15.550844+0000","last_peered":"2024-09-14T05:02:15.550844+0000","last_clean":"2024-09-14T05:02:15.550844+0000","last_became_active":"2024-09-14T04:44:57.323022+0000","last_became_peered":"2024-09-14T04:44:57.323022+0000","last_unstale":"2024-09-14T05:02:15.550844+0000","last_undegraded":"2024-09-14T05:02:15.550844+0000","last_fullsized":"2024-09-14T05:02:15.550844+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:44:22.775261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1598,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.553369+0000","last_change":"2024-09-14T04:44:57.322357+0000","last_active":"2024-09-14T05:02:15.553369+0000","last_peered":"2024-09-14T05:02:15.553369+0000","last_clean":"2024-09-14T05:02:15.553369+0000","last_became_active":"2024-09-14T04:44:57.321990+0000","last_became_peered":"2024-09-14T04:44:57.321990+0000","last_unstale":"2024-09-14T05:02:15.553369+0000","last_undegraded":"2024-09-14T05:02:15.553369+0000","last_fullsized":"2024-09-14T05:02:15.553369+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:46:09.658008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"349'149","reported_seq":1923,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.569047+0000","last_change":"2024-09-14T04:41:53.858076+0000","last_active":"2024-09-14T05:02:15.569047+0000","last_peered":"2024-09-14T05:02:15.569047+0000","last_clean":"2024-09-14T05:02:15.569047+0000","last_became_active":"2024-09-14T04:41:36.931585+0000","last_became_peered":"2024-09-14T04:41:36.931585+0000","last_unstale":"2024-09-14T05:02:15.569047+0000","last_undegraded":"2024-09-14T05:02:15.569047+0000","last_fullsized":"2024-09-14T05:02:15.569047+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1347,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T04:41:53.857924+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T04:41:53.857924+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:10:20.658986+0000","scrub_duration":0.0022355919999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":192,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.549560+0000","last_change":"2024-09-14T04:44:57.329198+0000","last_active":"2024-09-14T05:02:15.549560+0000","last_peered":"2024-09-14T05:02:15.549560+0000","last_clean":"2024-09-14T05:02:15.549560+0000","last_became_active":"2024-09-14T04:44:57.327754+0000","last_became_peered":"2024-09-14T04:44:57.327754+0000","last_unstale":"2024-09-14T05:02:15.549560+0000","last_undegraded":"2024-09-14T05:02:15.549560+0000","last_fullsized":"2024-09-14T05:02:15.549560+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:10:27.944476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":435,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.560149+0000","last_change":"2024-09-14T04:44:57.348687+0000","last_active":"2024-09-14T05:02:15.560149+0000","last_peered":"2024-09-14T05:02:15.560149+0000","last_clean":"2024-09-14T05:02:15.560149+0000","last_became_active":"2024-09-14T04:44:57.332845+0000","last_became_peered":"2024-09-14T04:44:57.332845+0000","last_unstale":"2024-09-14T05:02:15.560149+0000","last_undegraded":"2024-09-14T05:02:15.560149+0000","last_fullsized":"2024-09-14T05:02:15.560149+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:12:18.762737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"349'149","reported_seq":670,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:23.547297+0000","last_change":"2024-09-14T05:00:38.958871+0000","last_active":"2024-09-14T05:02:23.547297+0000","last_peered":"2024-09-14T05:02:23.547297+0000","last_clean":"2024-09-14T05:02:23.547297+0000","last_became_active":"2024-09-14T05:00:08.498061+0000","last_became_peered":"2024-09-14T05:00:08.498061+0000","last_unstale":"2024-09-14T05:02:23.547297+0000","last_undegraded":"2024-09-14T05:02:23.547297+0000","last_fullsized":"2024-09-14T05:02:23.547297+0000","mapping_epoch":1549,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1550,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:38.958743+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:38.958743+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:12:46.541984+0000","scrub_duration":0.0059942049999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2458,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.559443+0000","last_change":"2024-09-14T04:44:57.333468+0000","last_active":"2024-09-14T05:02:15.559443+0000","last_peered":"2024-09-14T05:02:15.559443+0000","last_clean":"2024-09-14T05:02:15.559443+0000","last_became_active":"2024-09-14T04:44:57.332626+0000","last_became_peered":"2024-09-14T04:44:57.332626+0000","last_unstale":"2024-09-14T05:02:15.559443+0000","last_undegraded":"2024-09-14T05:02:15.559443+0000","last_fullsized":"2024-09-14T05:02:15.559443+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:57:26.366438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1554,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.554760+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T05:02:15.554760+0000","last_peered":"2024-09-14T05:02:15.554760+0000","last_clean":"2024-09-14T05:02:15.554760+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T05:02:15.554760+0000","last_undegraded":"2024-09-14T05:02:15.554760+0000","last_fullsized":"2024-09-14T05:02:15.554760+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"349'149","reported_seq":961,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:18.055739+0000","last_change":"2024-09-14T05:00:46.668539+0000","last_active":"2024-09-14T05:02:18.055739+0000","last_peered":"2024-09-14T05:02:18.055739+0000","last_clean":"2024-09-14T05:02:18.055739+0000","last_became_active":"2024-09-14T05:00:08.482855+0000","last_became_peered":"2024-09-14T05:00:08.482855+0000","last_unstale":"2024-09-14T05:02:18.055739+0000","last_undegraded":"2024-09-14T05:02:18.055739+0000","last_fullsized":"2024-09-14T05:02:18.055739+0000","mapping_epoch":1549,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1550,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:46.668408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:46.668408+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:39:23.694296+0000","scrub_duration":0.0052970259999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":1553,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.553222+0000","last_change":"2024-09-14T04:06:46.869868+0000","last_active":"2024-09-14T05:02:15.553222+0000","last_peered":"2024-09-14T05:02:15.553222+0000","last_clean":"2024-09-14T05:02:15.553222+0000","last_became_active":"2024-09-14T04:06:46.869126+0000","last_became_peered":"2024-09-14T04:06:46.869126+0000","last_unstale":"2024-09-14T05:02:15.553222+0000","last_undegraded":"2024-09-14T05:02:15.553222+0000","last_fullsized":"2024-09-14T05:02:15.553222+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:50:41.150138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1537,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.569765+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T05:02:15.569765+0000","last_peered":"2024-09-14T05:02:15.569765+0000","last_clean":"2024-09-14T05:02:15.569765+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T05:02:15.569765+0000","last_undegraded":"2024-09-14T05:02:15.569765+0000","last_fullsized":"2024-09-14T05:02:15.569765+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"349'149","reported_seq":908,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.559022+0000","last_change":"2024-09-14T05:00:41.652685+0000","last_active":"2024-09-14T05:02:15.559022+0000","last_peered":"2024-09-14T05:02:15.559022+0000","last_clean":"2024-09-14T05:02:15.559022+0000","last_became_active":"2024-09-14T05:00:08.504510+0000","last_became_peered":"2024-09-14T05:00:08.504510+0000","last_unstale":"2024-09-14T05:02:15.559022+0000","last_undegraded":"2024-09-14T05:02:15.559022+0000","last_fullsized":"2024-09-14T05:02:15.559022+0000","mapping_epoch":1549,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1550,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:41.652639+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:41.652639+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:11:32.536085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,6],"acting":[0,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":193,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:18.546684+0000","last_change":"2024-09-14T04:44:57.332112+0000","last_active":"2024-09-14T05:02:18.546684+0000","last_peered":"2024-09-14T05:02:18.546684+0000","last_clean":"2024-09-14T05:02:18.546684+0000","last_became_active":"2024-09-14T04:44:57.331575+0000","last_became_peered":"2024-09-14T04:44:57.331575+0000","last_unstale":"2024-09-14T05:02:18.546684+0000","last_undegraded":"2024-09-14T05:02:18.546684+0000","last_fullsized":"2024-09-14T05:02:18.546684+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:50:00.391283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1585,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.569812+0000","last_change":"2024-09-14T04:44:57.351460+0000","last_active":"2024-09-14T05:02:15.569812+0000","last_peered":"2024-09-14T05:02:15.569812+0000","last_clean":"2024-09-14T05:02:15.569812+0000","last_became_active":"2024-09-14T04:44:57.334662+0000","last_became_peered":"2024-09-14T04:44:57.334662+0000","last_unstale":"2024-09-14T05:02:15.569812+0000","last_undegraded":"2024-09-14T05:02:15.569812+0000","last_fullsized":"2024-09-14T05:02:15.569812+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:55:02.399062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.d","version":"349'149","reported_seq":821,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:22.942802+0000","last_change":"2024-09-14T05:00:35.974699+0000","last_active":"2024-09-14T05:02:22.942802+0000","last_peered":"2024-09-14T05:02:22.942802+0000","last_clean":"2024-09-14T05:02:22.942802+0000","last_became_active":"2024-09-14T05:00:08.484099+0000","last_became_peered":"2024-09-14T05:00:08.484099+0000","last_unstale":"2024-09-14T05:02:22.942802+0000","last_undegraded":"2024-09-14T05:02:22.942802+0000","last_fullsized":"2024-09-14T05:02:22.942802+0000","mapping_epoch":1549,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1550,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:35.974599+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:35.974599+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:32:28.503626+0000","scrub_duration":0.0082975730000000008,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1540,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:27.917586+0000","last_change":"2024-09-14T03:53:05.550196+0000","last_active":"2024-09-14T05:02:27.917586+0000","last_peered":"2024-09-14T05:02:27.917586+0000","last_clean":"2024-09-14T05:02:27.917586+0000","last_became_active":"2024-09-14T03:53:05.549903+0000","last_became_peered":"2024-09-14T03:53:05.549903+0000","last_unstale":"2024-09-14T05:02:27.917586+0000","last_undegraded":"2024-09-14T05:02:27.917586+0000","last_fullsized":"2024-09-14T05:02:27.917586+0000","mapping_epoch":69,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":70,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:11:29.226810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1597,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.569890+0000","last_change":"2024-09-14T04:44:57.349690+0000","last_active":"2024-09-14T05:02:15.569890+0000","last_peered":"2024-09-14T05:02:15.569890+0000","last_clean":"2024-09-14T05:02:15.569890+0000","last_became_active":"2024-09-14T04:44:57.333579+0000","last_became_peered":"2024-09-14T04:44:57.333579+0000","last_unstale":"2024-09-14T05:02:15.569890+0000","last_undegraded":"2024-09-14T05:02:15.569890+0000","last_fullsized":"2024-09-14T05:02:15.569890+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:17:55.265141+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"349'149","reported_seq":759,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:27.917634+0000","last_change":"2024-09-14T05:00:34.592195+0000","last_active":"2024-09-14T05:02:27.917634+0000","last_peered":"2024-09-14T05:02:27.917634+0000","last_clean":"2024-09-14T05:02:27.917634+0000","last_became_active":"2024-09-14T05:00:08.490541+0000","last_became_peered":"2024-09-14T05:00:08.490541+0000","last_unstale":"2024-09-14T05:02:27.917634+0000","last_undegraded":"2024-09-14T05:02:27.917634+0000","last_fullsized":"2024-09-14T05:02:27.917634+0000","mapping_epoch":1549,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1550,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:34.592056+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:34.592056+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:46:54.217254+0000","scrub_duration":0.0046679920000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1693,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.552695+0000","last_change":"2024-09-14T05:00:21.251775+0000","last_active":"2024-09-14T05:02:15.552695+0000","last_peered":"2024-09-14T05:02:15.552695+0000","last_clean":"2024-09-14T05:02:15.552695+0000","last_became_active":"2024-09-14T05:00:21.228875+0000","last_became_peered":"2024-09-14T05:00:21.228875+0000","last_unstale":"2024-09-14T05:02:15.552695+0000","last_undegraded":"2024-09-14T05:02:15.552695+0000","last_fullsized":"2024-09-14T05:02:15.552695+0000","mapping_epoch":1552,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1553,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:15:41.207466+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":87,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.569801+0000","last_change":"2024-09-14T04:54:52.248439+0000","last_active":"2024-09-14T05:02:15.569801+0000","last_peered":"2024-09-14T05:02:15.569801+0000","last_clean":"2024-09-14T05:02:15.569801+0000","last_became_active":"2024-09-14T04:54:52.248235+0000","last_became_peered":"2024-09-14T04:54:52.248235+0000","last_unstale":"2024-09-14T05:02:15.569801+0000","last_undegraded":"2024-09-14T05:02:15.569801+0000","last_fullsized":"2024-09-14T05:02:15.569801+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:22:09.548139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"349'149","reported_seq":823,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:28.549363+0000","last_change":"2024-09-14T05:00:35.941366+0000","last_active":"2024-09-14T05:02:28.549363+0000","last_peered":"2024-09-14T05:02:28.549363+0000","last_clean":"2024-09-14T05:02:28.549363+0000","last_became_active":"2024-09-14T05:00:08.498035+0000","last_became_peered":"2024-09-14T05:00:08.498035+0000","last_unstale":"2024-09-14T05:02:28.549363+0000","last_undegraded":"2024-09-14T05:02:28.549363+0000","last_fullsized":"2024-09-14T05:02:28.549363+0000","mapping_epoch":1549,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1550,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:35.941252+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:35.941252+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:12:45.217998+0000","scrub_duration":0.0056267849999999996,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1559,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.569246+0000","last_change":"2024-09-14T04:44:57.332511+0000","last_active":"2024-09-14T05:02:15.569246+0000","last_peered":"2024-09-14T05:02:15.569246+0000","last_clean":"2024-09-14T05:02:15.569246+0000","last_became_active":"2024-09-14T04:44:57.332166+0000","last_became_peered":"2024-09-14T04:44:57.332166+0000","last_unstale":"2024-09-14T05:02:15.569246+0000","last_undegraded":"2024-09-14T05:02:15.569246+0000","last_fullsized":"2024-09-14T05:02:15.569246+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:01:53.984639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2451,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.559336+0000","last_change":"2024-09-14T04:54:52.250655+0000","last_active":"2024-09-14T05:02:15.559336+0000","last_peered":"2024-09-14T05:02:15.559336+0000","last_clean":"2024-09-14T05:02:15.559336+0000","last_became_active":"2024-09-14T04:54:52.250317+0000","last_became_peered":"2024-09-14T04:54:52.250317+0000","last_unstale":"2024-09-14T05:02:15.559336+0000","last_undegraded":"2024-09-14T05:02:15.559336+0000","last_fullsized":"2024-09-14T05:02:15.559336+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:12.985406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.10","version":"349'149","reported_seq":670,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:27.917556+0000","last_change":"2024-09-14T05:00:36.569646+0000","last_active":"2024-09-14T05:02:27.917556+0000","last_peered":"2024-09-14T05:02:27.917556+0000","last_clean":"2024-09-14T05:02:27.917556+0000","last_became_active":"2024-09-14T05:00:08.482730+0000","last_became_peered":"2024-09-14T05:00:08.482730+0000","last_unstale":"2024-09-14T05:02:27.917556+0000","last_undegraded":"2024-09-14T05:02:27.917556+0000","last_fullsized":"2024-09-14T05:02:27.917556+0000","mapping_epoch":1549,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1550,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:36.569513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:36.569513+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:39:59.050548+0000","scrub_duration":0.0087120909999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2437,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.559382+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T05:02:15.559382+0000","last_peered":"2024-09-14T05:02:15.559382+0000","last_clean":"2024-09-14T05:02:15.559382+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T05:02:15.559382+0000","last_undegraded":"2024-09-14T05:02:15.559382+0000","last_fullsized":"2024-09-14T05:02:15.559382+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1551,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.552844+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T05:02:15.552844+0000","last_peered":"2024-09-14T05:02:15.552844+0000","last_clean":"2024-09-14T05:02:15.552844+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T05:02:15.552844+0000","last_undegraded":"2024-09-14T05:02:15.552844+0000","last_fullsized":"2024-09-14T05:02:15.552844+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.11","version":"349'149","reported_seq":709,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.560658+0000","last_change":"2024-09-14T05:00:40.678075+0000","last_active":"2024-09-14T05:02:15.560658+0000","last_peered":"2024-09-14T05:02:15.560658+0000","last_clean":"2024-09-14T05:02:15.560658+0000","last_became_active":"2024-09-14T05:00:08.504082+0000","last_became_peered":"2024-09-14T05:00:08.504082+0000","last_unstale":"2024-09-14T05:02:15.560658+0000","last_undegraded":"2024-09-14T05:02:15.560658+0000","last_fullsized":"2024-09-14T05:02:15.560658+0000","mapping_epoch":1549,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1550,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:40.677843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:40.677843+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:36:50.392923+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1588,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.569465+0000","last_change":"2024-09-14T04:44:57.339993+0000","last_active":"2024-09-14T05:02:15.569465+0000","last_peered":"2024-09-14T05:02:15.569465+0000","last_clean":"2024-09-14T05:02:15.569465+0000","last_became_active":"2024-09-14T04:44:57.330472+0000","last_became_peered":"2024-09-14T04:44:57.330472+0000","last_unstale":"2024-09-14T05:02:15.569465+0000","last_undegraded":"2024-09-14T05:02:15.569465+0000","last_fullsized":"2024-09-14T05:02:15.569465+0000","mapping_epoch":1402,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:19:27.046988+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2136,"reported_epoch":1576,"state":"active+clean","last_fresh":"2024-09-14T05:02:15.569552+0000","last_change":"2024-09-14T04:49:49.449687+0000","last_active":"2024-09-14T05:02:15.569552+0000","last_peered":"2024-09-14T05:02:15.569552+0000","last_clean":"2024-09-14T05:02:15.569552+0000","last_became_active":"2024-09-14T04:49:49.449144+0000","last_became_peered":"2024-09-14T04:49:49.449144+0000","last_unstale":"2024-09-14T05:02:15.569552+0000","last_undegraded":"2024-09-14T05:02:15.569552+0000","last_fullsized":"2024-09-14T05:02:15.569552+0000","mapping_epoch":1445,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1446,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:59:06.779730+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]}],"pool_stats":[{"poolid":58,"num_pg":16,"stat_sum":{"num_bytes":6696468480,"num_objects":102180,"num_object_clones":0,"num_object_copies":306540,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":102180,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":102180,"num_write_kb":6539520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20145438720,"data_stored":20145438720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":101980,"ondisk_log_size":101980,"up":48,"acting":48,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1576128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3129,"ondisk_log_size":3129,"up":63,"acting":63,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":5070,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4250,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223202139,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2115960,"kb_used_data":1246512,"kb_used_omap":1,"kb_used_meta":869438,"kb_avail":91629192,"statfs":{"total":95995035648,"available":93828292608,"internally_reserved":0,"allocated":1276428288,"data_stored":1270722218,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":890304971},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463775,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3423240,"kb_used_data":2499644,"kb_used_omap":1,"kb_used_meta":923582,"kb_avail":90321912,"statfs":{"total":95995035648,"available":92489637888,"internally_reserved":0,"allocated":2559635456,"data_stored":2553926342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":945748423},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725412,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2158652,"kb_used_data":1247344,"kb_used_omap":1,"kb_used_meta":911294,"kb_avail":91586500,"statfs":{"total":95995035648,"available":93784576000,"internally_reserved":0,"allocated":1277280256,"data_stored":1271578712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":933165510},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":1392,"seq":5978594476258,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2992588,"kb_used_data":2514368,"kb_used_omap":117,"kb_used_meta":478090,"kb_avail":90752564,"statfs":{"total":95995035648,"available":92930625536,"internally_reserved":0,"allocated":2574712832,"data_stored":2568999757,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120638,"internal_metadata":489564354},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,1,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772075153,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5152360,"kb_used_data":4109600,"kb_used_omap":71,"kb_used_meta":1042680,"kb_avail":88592792,"statfs":{"total":95995035648,"available":90719019008,"internally_reserved":0,"allocated":4208230400,"data_stored":4202537010,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":1067705270},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,6,1,3,1,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707304088,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3620264,"kb_used_data":2857952,"kb_used_omap":68,"kb_used_meta":762235,"kb_avail":90124888,"statfs":{"total":95995035648,"available":92287885312,"internally_reserved":0,"allocated":2926542848,"data_stored":2920849362,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":780528848},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,5,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872729247,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4369888,"kb_used_data":3629848,"kb_used_omap":67,"kb_used_meta":739964,"kb_avail":89375264,"statfs":{"total":95995035648,"available":91520270336,"internally_reserved":0,"allocated":3716964352,"data_stored":3711277917,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":757723493},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,0,1,2,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056297,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2416100,"kb_used_data":1590172,"kb_used_omap":74,"kb_used_meta":825845,"kb_avail":91329052,"statfs":{"total":95995035648,"available":93520949248,"internally_reserved":0,"allocated":1628336128,"data_stored":1622646182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":845665973},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1901,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":58,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1628438528,"data_stored":1628438528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":58,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":3718381568,"data_stored":3718381568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":58,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2921332736,"data_stored":2921332736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":58,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4207607808,"data_stored":4207607808,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":58,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2567831552,"data_stored":2567831552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":58,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1268973568,"data_stored":1268973568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":58,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2561605632,"data_stored":2561605632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":58,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1271267328,"data_stored":1271267328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T05:02:38.824 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2024-09-14T05:02:38.824 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pg_num 31 2024-09-14T05:02:38.845 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:38.869 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:38.870 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:38.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:38.920 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:02:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:02:38] "GET /metrics HTTP/1.1" 200 36740 "" "Prometheus/2.51.0" 2024-09-14T05:02:38.971 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:39.028 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:39.028 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:39.028 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:39.066 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 82 16 108737 108721 82.842 56.1875 0.0150784 0.0120651 2024-09-14T05:02:39.129 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:39.152 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:39.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:39.154 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:39.255 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:39.297 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:39.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:39.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:39.399 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:39.452 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:39.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:39.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:39.554 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:39.605 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:39.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:39.606 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:39.707 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:39.732 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:39.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:39.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:39.835 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:39.880 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:39.881 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:39.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:39.982 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:39.988 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:39 smithi039 ceph-mon[106470]: pgmap v3137: 109 pgs: 109 active+clean; 6.4 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s 2024-09-14T05:02:39.989 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:39 smithi039 ceph-mon[111235]: pgmap v3137: 109 pgs: 109 active+clean; 6.4 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s 2024-09-14T05:02:39.989 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T05:02:39.990 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [ 2024-09-14T05:02:39.990 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:39.990 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:109272 58.e 58:7ad6a30f:::benchmark_data_smithi039_396950_object109271:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1577)", 2024-09-14T05:02:39.990 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:02:39.700198+0000", 2024-09-14T05:02:39.990 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.020884450999999998, 2024-09-14T05:02:39.990 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.020916111000000001, 2024-09-14T05:02:39.991 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:39.991 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:39.991 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "waiting for sub ops", 2024-09-14T05:02:39.991 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:02:39.991 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:02:39.991 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:02:39.991 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 109272 2024-09-14T05:02:39.992 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:39.992 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:39.992 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:39.992 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:39.992 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.700198+0000", 2024-09-14T05:02:39.992 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:39.992 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:39.992 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:39.992 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:39.992 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.700198+0000", 2024-09-14T05:02:39.993 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:39.993 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:39.993 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:39.993 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:39.993 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.700200+0000", 2024-09-14T05:02:39.993 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7039999999999999e-06 2024-09-14T05:02:39.993 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:39.994 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:39.994 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:39.994 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.700227+0000", 2024-09-14T05:02:39.994 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.7353000000000001e-05 2024-09-14T05:02:39.994 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:39.994 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:39.994 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:39.994 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.700244+0000", 2024-09-14T05:02:39.994 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6778000000000001e-05 2024-09-14T05:02:39.995 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:39.995 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:39.995 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:39.995 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.700252+0000", 2024-09-14T05:02:39.995 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.1640000000000001e-06 2024-09-14T05:02:39.995 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:39.995 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:39.995 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:39.995 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.700430+0000", 2024-09-14T05:02:39.995 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00017809999999999999 2024-09-14T05:02:39.995 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:39.995 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:39.996 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:39.996 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.700606+0000", 2024-09-14T05:02:39.996 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00017524899999999999 2024-09-14T05:02:39.996 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:39.996 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:39.996 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,3", 2024-09-14T05:02:39.996 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.700743+0000", 2024-09-14T05:02:39.996 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013762 2024-09-14T05:02:39.996 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:39.996 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:39.996 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:02:39.997 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.705237+0000", 2024-09-14T05:02:39.997 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0044934750000000002 2024-09-14T05:02:39.997 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:39.997 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:39.997 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:02:39.997 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.711515+0000", 2024-09-14T05:02:39.997 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0062783329999999997 2024-09-14T05:02:39.997 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:39.997 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:39.997 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:39.998 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:39.998 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:39.998 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:109281 58.d 58:b164b99a:::benchmark_data_smithi039_396950_object109280:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1577)", 2024-09-14T05:02:39.998 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:02:39.711528+0000", 2024-09-14T05:02:39.998 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.009555137, 2024-09-14T05:02:39.998 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0096636980000000001, 2024-09-14T05:02:39.998 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:39.998 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:39.998 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "waiting for sub ops", 2024-09-14T05:02:39.999 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:02:39.999 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:02:39.999 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:02:39.999 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 109281 2024-09-14T05:02:39.999 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:39.999 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:39.999 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:39.999 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:39.999 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.711528+0000", 2024-09-14T05:02:40.000 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:40.000 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.000 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.000 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:40.000 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.711528+0000", 2024-09-14T05:02:40.000 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:40.001 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.001 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.001 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:40.001 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.711530+0000", 2024-09-14T05:02:40.001 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.883e-06 2024-09-14T05:02:40.001 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.001 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.002 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:40.002 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.711556+0000", 2024-09-14T05:02:40.002 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.6234e-05 2024-09-14T05:02:40.002 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.002 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.002 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:40.002 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.711571+0000", 2024-09-14T05:02:40.002 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5201000000000001e-05 2024-09-14T05:02:40.002 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.002 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.002 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:40.003 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.711580+0000", 2024-09-14T05:02:40.003 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.6130000000000004e-06 2024-09-14T05:02:40.003 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.003 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.003 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:40.003 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.711727+0000", 2024-09-14T05:02:40.003 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000146973 2024-09-14T05:02:40.003 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.003 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.003 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:40.003 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.712064+0000", 2024-09-14T05:02:40.003 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00033733599999999998 2024-09-14T05:02:40.004 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.004 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.004 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,3", 2024-09-14T05:02:40.004 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.712227+0000", 2024-09-14T05:02:40.004 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000162814 2024-09-14T05:02:40.004 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:40.004 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:40.004 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:40.004 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.004 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.004 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:109284 58.d 58:b920fcce:::benchmark_data_smithi039_396950_object109283:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1577)", 2024-09-14T05:02:40.005 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:02:39.712514+0000", 2024-09-14T05:02:40.005 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.0085684369999999999, 2024-09-14T05:02:40.005 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0087392549999999996, 2024-09-14T05:02:40.005 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:40.005 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:40.005 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "waiting for sub ops", 2024-09-14T05:02:40.005 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:02:40.005 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:02:40.005 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:02:40.005 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 109284 2024-09-14T05:02:40.005 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.005 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:40.006 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.006 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:40.006 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.712514+0000", 2024-09-14T05:02:40.006 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:40.006 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.006 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.006 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:40.006 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.712514+0000", 2024-09-14T05:02:40.006 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:40.006 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.006 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.006 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:40.007 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.712515+0000", 2024-09-14T05:02:40.007 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.4e-07 2024-09-14T05:02:40.007 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.007 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.007 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:40.007 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.712540+0000", 2024-09-14T05:02:40.007 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4411999999999999e-05 2024-09-14T05:02:40.007 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.007 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.007 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:40.007 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.712557+0000", 2024-09-14T05:02:40.008 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7288000000000001e-05 2024-09-14T05:02:40.008 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.008 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.008 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:40.008 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.712566+0000", 2024-09-14T05:02:40.008 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.7239999999999998e-06 2024-09-14T05:02:40.008 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.008 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.008 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:40.009 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.719562+0000", 2024-09-14T05:02:40.009 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.006996309 2024-09-14T05:02:40.009 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.009 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.009 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:40.009 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.719794+0000", 2024-09-14T05:02:40.009 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00023157200000000001 2024-09-14T05:02:40.009 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.009 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.009 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,3", 2024-09-14T05:02:40.009 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.719927+0000", 2024-09-14T05:02:40.010 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000133473 2024-09-14T05:02:40.010 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:40.010 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:40.010 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:40.010 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.010 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.010 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:109288 58.e 58:7f983283:::benchmark_data_smithi039_396950_object109287:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1577)", 2024-09-14T05:02:40.010 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:02:39.716290+0000", 2024-09-14T05:02:40.010 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.0047923360000000003, 2024-09-14T05:02:40.010 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0050228089999999996, 2024-09-14T05:02:40.010 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:40.010 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:40.011 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "waiting for sub ops", 2024-09-14T05:02:40.011 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:02:40.011 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:02:40.011 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:02:40.011 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 109288 2024-09-14T05:02:40.011 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.011 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:40.011 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.011 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:40.011 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.716290+0000", 2024-09-14T05:02:40.011 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:40.012 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.012 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.012 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:40.012 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.716290+0000", 2024-09-14T05:02:40.012 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:40.012 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.012 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.012 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:40.012 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.716292+0000", 2024-09-14T05:02:40.012 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.494e-06 2024-09-14T05:02:40.013 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.013 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.013 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:40.013 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.716317+0000", 2024-09-14T05:02:40.013 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4888999999999998e-05 2024-09-14T05:02:40.013 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.013 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.014 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:40.014 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.716327+0000", 2024-09-14T05:02:40.014 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.9720000000000004e-06 2024-09-14T05:02:40.014 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.014 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.014 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:40.014 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.716335+0000", 2024-09-14T05:02:40.015 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.2009999999999999e-06 2024-09-14T05:02:40.015 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.015 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.015 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:40.015 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.716641+0000", 2024-09-14T05:02:40.015 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00030639399999999998 2024-09-14T05:02:40.015 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.015 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.015 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:40.015 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.716802+0000", 2024-09-14T05:02:40.015 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00016009299999999999 2024-09-14T05:02:40.015 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.016 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.016 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 1,3", 2024-09-14T05:02:40.016 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.716950+0000", 2024-09-14T05:02:40.016 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000148419 2024-09-14T05:02:40.016 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:40.016 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:40.016 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:40.016 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.016 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.017 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36603.0:109285 58.8 e1577/1568 58:1e346dbd:::benchmark_data_smithi039_396950_object109284:head v 1577'6914, mlcod=1577'6912)", 2024-09-14T05:02:40.017 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:02:39.716416+0000", 2024-09-14T05:02:40.017 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.0046666770000000002, 2024-09-14T05:02:40.017 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0049527060000000003, 2024-09-14T05:02:40.017 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:40.017 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:40.018 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "started", 2024-09-14T05:02:40.018 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:40.018 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.018 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:40.018 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.716416+0000", 2024-09-14T05:02:40.018 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:40.018 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.018 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.018 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:40.018 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.716416+0000", 2024-09-14T05:02:40.019 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:40.019 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.019 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.019 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:40.019 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.716417+0000", 2024-09-14T05:02:40.019 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.99e-07 2024-09-14T05:02:40.019 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.019 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.020 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:40.020 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.716439+0000", 2024-09-14T05:02:40.020 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1883999999999999e-05 2024-09-14T05:02:40.020 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.020 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.020 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:40.020 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.716448+0000", 2024-09-14T05:02:40.020 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.1360000000000003e-06 2024-09-14T05:02:40.020 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.020 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.021 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:40.021 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.716454+0000", 2024-09-14T05:02:40.021 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1909999999999997e-06 2024-09-14T05:02:40.021 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.021 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.021 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:40.021 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.716968+0000", 2024-09-14T05:02:40.021 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00051413499999999998 2024-09-14T05:02:40.021 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.021 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.021 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:40.022 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.717402+0000", 2024-09-14T05:02:40.022 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00043375899999999998 2024-09-14T05:02:40.022 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:40.022 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:40.022 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:40.022 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.022 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.022 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:109290 58.7 58:e1e68244:::benchmark_data_smithi039_396950_object109289:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1577)", 2024-09-14T05:02:40.022 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:02:39.716825+0000", 2024-09-14T05:02:40.022 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.0042582519999999997, 2024-09-14T05:02:40.023 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0045933739999999999, 2024-09-14T05:02:40.023 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:40.023 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:40.023 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "waiting for sub ops", 2024-09-14T05:02:40.023 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:02:40.023 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:02:40.023 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:02:40.023 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 109290 2024-09-14T05:02:40.023 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.023 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:40.024 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.024 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:40.024 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.716825+0000", 2024-09-14T05:02:40.024 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:40.024 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.024 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.024 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:40.024 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.716825+0000", 2024-09-14T05:02:40.024 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:40.024 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.025 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.025 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:40.025 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.716825+0000", 2024-09-14T05:02:40.025 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.6300000000000004e-07 2024-09-14T05:02:40.026 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.026 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.026 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:40.026 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.716850+0000", 2024-09-14T05:02:40.026 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4117e-05 2024-09-14T05:02:40.026 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.026 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.027 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:40.027 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.716860+0000", 2024-09-14T05:02:40.027 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0068000000000001e-05 2024-09-14T05:02:40.027 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.027 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.027 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:40.027 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.716867+0000", 2024-09-14T05:02:40.028 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.3969999999999997e-06 2024-09-14T05:02:40.028 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.028 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.028 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:02:40.028 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.717013+0000", 2024-09-14T05:02:40.028 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014567399999999999 2024-09-14T05:02:40.028 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.028 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.028 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:02:40.028 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.717160+0000", 2024-09-14T05:02:40.028 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014747800000000001 2024-09-14T05:02:40.028 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.028 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.029 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,6", 2024-09-14T05:02:40.029 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.717328+0000", 2024-09-14T05:02:40.029 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00016799 2024-09-14T05:02:40.029 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:40.029 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:40.029 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:40.029 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.029 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.029 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36603.0:109287 58.8 e1577/1568)", 2024-09-14T05:02:40.029 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:02:39.720829+0000", 2024-09-14T05:02:40.029 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.00025333099999999998, 2024-09-14T05:02:40.030 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00065083299999999997, 2024-09-14T05:02:40.030 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:40.030 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:40.030 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "queued for pg", 2024-09-14T05:02:40.030 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:40.030 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.030 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:40.030 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.720829+0000", 2024-09-14T05:02:40.030 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:40.030 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.030 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.031 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:40.031 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.720829+0000", 2024-09-14T05:02:40.031 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:40.031 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.031 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.031 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:40.031 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.720831+0000", 2024-09-14T05:02:40.031 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.136e-06 2024-09-14T05:02:40.031 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.031 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.032 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:40.032 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.720854+0000", 2024-09-14T05:02:40.032 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3034e-05 2024-09-14T05:02:40.032 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.032 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.032 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:40.032 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.720865+0000", 2024-09-14T05:02:40.032 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1165999999999999e-05 2024-09-14T05:02:40.032 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.032 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.033 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:40.033 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.720872+0000", 2024-09-14T05:02:40.033 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7089999999999996e-06 2024-09-14T05:02:40.033 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:40.033 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:40.033 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:40.033 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.033 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.033 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop_reply(client.36603.0:109290 58.7 e1577/1568)", 2024-09-14T05:02:40.033 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:02:39.720925+0000", 2024-09-14T05:02:40.033 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.000157458, 2024-09-14T05:02:40.033 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00058994199999999998, 2024-09-14T05:02:40.034 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:02:40.034 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:02:40.034 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "queued for pg", 2024-09-14T05:02:40.034 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:02:40.034 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.034 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:02:40.034 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.720925+0000", 2024-09-14T05:02:40.034 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:40.034 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.034 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.034 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:02:40.035 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.720925+0000", 2024-09-14T05:02:40.035 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:02:40.035 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.035 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.035 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:02:40.035 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.720926+0000", 2024-09-14T05:02:40.035 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0299999999999998e-07 2024-09-14T05:02:40.035 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.035 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.036 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:02:40.036 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.720931+0000", 2024-09-14T05:02:40.036 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.0819999999999998e-06 2024-09-14T05:02:40.036 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.036 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.036 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:02:40.036 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.720942+0000", 2024-09-14T05:02:40.036 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0395e-05 2024-09-14T05:02:40.036 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:02:40.036 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:02:40.036 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:02:40.036 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:02:39.720948+0000", 2024-09-14T05:02:40.037 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3409999999999999e-06 2024-09-14T05:02:40.037 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:40.037 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:02:40.037 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:40.037 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:02:40.037 INFO:teuthology.orchestra.run.smithi039.stdout: ], 2024-09-14T05:02:40.037 INFO:teuthology.orchestra.run.smithi039.stdout: "num_ops": 8 2024-09-14T05:02:40.037 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T05:02:40.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:40.038 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:40.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:40.039 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:39 smithi129 ceph-mon[101316]: pgmap v3137: 109 pgs: 109 active+clean; 6.4 GiB data, 26 GiB used, 690 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s 2024-09-14T05:02:40.066 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 83 16 109608 109592 82.4998 54.4375 0.0136074 0.0121155 2024-09-14T05:02:40.139 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:40.184 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:40.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:40.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:40.285 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:40.337 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:40.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:40.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:40.438 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:40.465 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:40.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:40.466 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:40.567 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:40.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:40.609 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:40.609 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:40.710 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:40.736 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:40.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:40.737 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:40.838 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:40.885 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:40.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:40.886 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:40.887 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-14T05:02:40.941 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:02:40.987 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:41.069 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 84 16 110496 110480 82.1762 55.5 0.0186737 0.0121627 2024-09-14T05:02:41.069 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:41.070 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:41.070 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:41.171 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:41.199 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:41.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:41.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:41.301 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:41.328 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:41.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:41.329 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:41.429 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:41.500 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:41.501 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:41.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:41.602 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:41.655 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:41.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:41.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:41.757 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:41.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:41.815 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:41.815 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:41.916 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:41.953 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:41.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:41.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:42.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:41 smithi129 ceph-mon[101316]: pgmap v3138: 109 pgs: 109 active+clean; 6.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 54 MiB/s wr, 863 op/s 2024-09-14T05:02:42.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:41 smithi129 ceph-mon[101316]: osdmap e1578: 8 total, 8 up, 8 in 2024-09-14T05:02:42.027 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:41 smithi039 ceph-mon[106470]: pgmap v3138: 109 pgs: 109 active+clean; 6.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 54 MiB/s wr, 863 op/s 2024-09-14T05:02:42.027 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:41 smithi039 ceph-mon[106470]: osdmap e1578: 8 total, 8 up, 8 in 2024-09-14T05:02:42.027 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:41 smithi039 ceph-mon[111235]: pgmap v3138: 109 pgs: 109 active+clean; 6.6 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 54 MiB/s wr, 863 op/s 2024-09-14T05:02:42.027 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:41 smithi039 ceph-mon[111235]: osdmap e1578: 8 total, 8 up, 8 in 2024-09-14T05:02:42.056 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:42.068 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 85 16 111341 111325 81.8307 52.8125 0.0246271 0.0122142 2024-09-14T05:02:42.102 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:42.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:42.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:42.204 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:42.242 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:42.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:42.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:42.344 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:42.369 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:42.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:42.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:42.443 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T05:02:42.443 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T05:02:42.471 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:42.485 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T05:02:42.485 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T05:02:42.517 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:42.518 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:42.518 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:42.522 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T05:02:42.522 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T05:02:42.566 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T05:02:42.566 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T05:02:42.596 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T05:02:42.596 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T05:02:42.619 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:42.627 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T05:02:42.628 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T05:02:42.658 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:42.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:42.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:42.663 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T05:02:42.663 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T05:02:42.694 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T05:02:42.695 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T05:02:42.760 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:42.798 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:42.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:42.799 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:42.813 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:02:42.900 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:42.924 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:42.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:42.925 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:43.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:42 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1852144840' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:02:43.026 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:43.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:42 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1852144840' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:02:43.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:42 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1852144840' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:02:43.053 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:43.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:43.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:43.068 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 86 16 112176 112160 81.4861 52.1875 0.0139697 0.012267 2024-09-14T05:02:43.155 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:43.180 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:43.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:43.182 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:43.283 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:43.320 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:43.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:43.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:43.422 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:43.445 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:43.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:43.446 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:43.547 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:43.573 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:43.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:43.574 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:43.675 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:43.697 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:43.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:43.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:43.799 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:43.844 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:43.845 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:43.845 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:43.946 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:43.981 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:43.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:43.982 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:44.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:43 smithi129 ceph-mon[101316]: pgmap v3140: 109 pgs: 109 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s 2024-09-14T05:02:44.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:43 smithi039 ceph-mon[106470]: pgmap v3140: 109 pgs: 109 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s 2024-09-14T05:02:44.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:43 smithi039 ceph-mon[111235]: pgmap v3140: 109 pgs: 109 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s 2024-09-14T05:02:44.068 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 87 16 113074 113058 81.1946 56.125 0.0139294 0.012311 2024-09-14T05:02:44.083 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:44.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:44.109 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:44.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:44.210 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:44.231 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:44.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:44.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:44.333 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:44.358 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:44.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:44.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:44.460 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:44.484 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:44.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:44.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:44.586 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:44.619 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:44.619 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:44.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:44.721 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:44.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:44.768 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:44.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:44.869 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:44.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:44.911 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:44.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:45.012 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:45.054 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:45.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:45.055 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:45.068 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 88 16 113958 113942 80.8997 55.25 0.00573968 0.0123561 2024-09-14T05:02:45.156 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:45.200 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:45.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:45.201 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:45.302 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:45.335 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:45.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:45.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:45.437 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:45.492 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:45.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:45.493 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:45.593 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:45.636 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:45.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:45.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:45.738 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:45.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:45.782 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:45.782 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:45.883 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:45.892 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:02:45.907 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:45.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:45.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:46.009 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:46.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:45 smithi129 ceph-mon[101316]: pgmap v3141: 109 pgs: 109 active+clean; 6.8 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 53 MiB/s wr, 849 op/s 2024-09-14T05:02:46.033 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:46.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:46.035 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:46.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:45 smithi039 ceph-mon[106470]: pgmap v3141: 109 pgs: 109 active+clean; 6.8 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 53 MiB/s wr, 849 op/s 2024-09-14T05:02:46.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:45 smithi039 ceph-mon[111235]: pgmap v3141: 109 pgs: 109 active+clean; 6.8 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 53 MiB/s wr, 849 op/s 2024-09-14T05:02:46.069 INFO:tasks.radosbench.radosbench.0.smithi039.stdout: 89 16 114787 114771 80.5728 51.8125 0.0125025 0.0124068 2024-09-14T05:02:46.135 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:46.186 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:46.186 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:46.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:46.287 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:46.335 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:46.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:46.336 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:46.437 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:46.462 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:46.463 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:46.463 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:46.564 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:46.590 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:46.591 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:46.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:46.692 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:46.715 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:46.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:46.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:46.816 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:46.856 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:46.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:46.857 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:46.958 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:47.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:47.001 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:47.002 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:47.069 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Total time run: 90.0113 2024-09-14T05:02:47.069 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Total writes made: 115610 2024-09-14T05:02:47.069 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Write size: 65536 2024-09-14T05:02:47.069 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Object size: 65536 2024-09-14T05:02:47.069 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Bandwidth (MB/sec): 80.2746 2024-09-14T05:02:47.069 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Stddev Bandwidth: 34.2041 2024-09-14T05:02:47.070 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Max bandwidth (MB/sec): 210.562 2024-09-14T05:02:47.070 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Min bandwidth (MB/sec): 49 2024-09-14T05:02:47.070 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Average IOPS: 1284 2024-09-14T05:02:47.070 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Stddev IOPS: 547.266 2024-09-14T05:02:47.070 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Max IOPS: 3369 2024-09-14T05:02:47.070 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Min IOPS: 784 2024-09-14T05:02:47.070 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Average Latency(s): 0.0124526 2024-09-14T05:02:47.070 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Stddev Latency(s): 0.0114926 2024-09-14T05:02:47.070 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Max latency(s): 0.310727 2024-09-14T05:02:47.070 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Min latency(s): 0.00100424 2024-09-14T05:02:47.074 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Cleaning up (deleting benchmark objects) 2024-09-14T05:02:47.102 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:47.147 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:47.148 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:47.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:47.241 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T05:02:47.249 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:47.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:47.274 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:47.274 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:47.375 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:47.411 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:47 smithi039 ceph-mon[106470]: pgmap v3142: 109 pgs: 109 active+clean; 6.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 54 MiB/s wr, 858 op/s 2024-09-14T05:02:47.411 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:47 smithi039 ceph-mon[106470]: osdmap e1579: 8 total, 8 up, 8 in 2024-09-14T05:02:47.411 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:47 smithi039 ceph-mon[111235]: pgmap v3142: 109 pgs: 109 active+clean; 6.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 54 MiB/s wr, 858 op/s 2024-09-14T05:02:47.412 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:47 smithi039 ceph-mon[111235]: osdmap e1579: 8 total, 8 up, 8 in 2024-09-14T05:02:47.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:47.445 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:47.446 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:47.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:47 smithi129 ceph-mon[101316]: pgmap v3142: 109 pgs: 109 active+clean; 6.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 54 MiB/s wr, 858 op/s 2024-09-14T05:02:47.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:47 smithi129 ceph-mon[101316]: osdmap e1579: 8 total, 8 up, 8 in 2024-09-14T05:02:47.546 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:47.570 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:47.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:47.571 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:47.672 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:47.699 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:47.700 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:47.701 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:47.802 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:47.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:47.844 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:47.844 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:47.945 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:47.978 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:47.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:47.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:48.080 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:48.108 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:48.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:48.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:48.210 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:48.236 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:48.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:48.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:48.338 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:48.366 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:48.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:48.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:48.468 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:48.504 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:48.505 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:48.505 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:48.606 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:48.634 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:48.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:48.635 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:48.735 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:48.777 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:48.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:48.778 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:48.879 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:48.887 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:02:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:02:48] "GET /metrics HTTP/1.1" 200 36740 "" "Prometheus/2.51.0" 2024-09-14T05:02:48.922 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:48.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:48.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:49.023 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:49.048 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:49.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:49.049 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:49.150 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:49.173 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:49.174 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:49.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:49.275 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:49.299 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:49.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:49.300 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:49.400 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:49.419 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:49 smithi039 ceph-mon[106470]: pgmap v3144: 109 pgs: 109 active+clean; 6.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 44 MiB/s wr, 711 op/s 2024-09-14T05:02:49.420 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:49 smithi039 ceph-mon[111235]: pgmap v3144: 109 pgs: 109 active+clean; 6.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 44 MiB/s wr, 711 op/s 2024-09-14T05:02:49.448 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:49.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:49.449 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:49.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:49 smithi129 ceph-mon[101316]: pgmap v3144: 109 pgs: 109 active+clean; 6.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 44 MiB/s wr, 711 op/s 2024-09-14T05:02:49.550 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:49.575 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:49.576 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:49.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:49.677 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:49.702 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:49.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:49.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:49.805 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:49.847 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:49.848 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:49.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:49.950 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:49.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:49.993 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:49.993 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:50.094 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:50.135 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:50.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:50.136 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:50.237 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:50.291 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:50.292 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:50.292 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:50.296 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is unset 2024-09-14T05:02:50.369 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T05:02:50.369 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:02:50.393 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:50.439 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:50.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:50.441 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:50.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:50 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1394717546' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:02:50.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:50 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1394717546' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:02:50.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:50 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1394717546' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:02:50.541 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:50.567 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:50.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:50.568 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:50.669 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:50.708 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:50.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:50.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:50.809 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:50.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:50.837 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:50.837 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:50.938 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:50.979 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:50.979 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:50.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:51.080 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:51.109 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:51.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:51.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:51.211 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:51.236 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:51.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:51.237 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:51.270 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:51 smithi039 ceph-mon[106470]: pgmap v3145: 109 pgs: 109 active+clean; 7.0 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 45 MiB/s wr, 849 op/s 2024-09-14T05:02:51.315 INFO:teuthology.orchestra.run.smithi039.stderr:set pool 1 pg_num to 31 2024-09-14T05:02:51.338 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:51.380 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:51.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:51.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:51.482 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:51.508 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:51.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:51.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:51.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:51 smithi129 ceph-mon[101316]: pgmap v3145: 109 pgs: 109 active+clean; 7.0 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 45 MiB/s wr, 849 op/s 2024-09-14T05:02:51.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:51 smithi129 ceph-mon[101316]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T05:02:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:51 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T05:02:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:51 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1394717546' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T05:02:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:51 smithi129 ceph-mon[101316]: osdmap e1580: 8 total, 8 up, 8 in 2024-09-14T05:02:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:51 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3477613888' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-09-14T05:02:51.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:51 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-09-14T05:02:51.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:51 smithi039 ceph-mon[106470]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T05:02:51.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:51 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T05:02:51.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:51 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1394717546' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T05:02:51.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:51 smithi039 ceph-mon[106470]: osdmap e1580: 8 total, 8 up, 8 in 2024-09-14T05:02:51.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:51 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3477613888' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-09-14T05:02:51.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:51 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-09-14T05:02:51.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:51 smithi039 ceph-mon[111235]: pgmap v3145: 109 pgs: 109 active+clean; 7.0 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 45 MiB/s wr, 849 op/s 2024-09-14T05:02:51.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:51 smithi039 ceph-mon[111235]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T05:02:51.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:51 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T05:02:51.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:51 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1394717546' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T05:02:51.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:51 smithi039 ceph-mon[111235]: osdmap e1580: 8 total, 8 up, 8 in 2024-09-14T05:02:51.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:51 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3477613888' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-09-14T05:02:51.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:51 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"} : dispatch 2024-09-14T05:02:51.611 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:51.653 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:51.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:51.654 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:51.755 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:51.781 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:51.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:51.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:51.883 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:51.908 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:51.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:51.909 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:52.010 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:52.034 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:52.034 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:52.035 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:52.135 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:52.177 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:52.178 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:52.178 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:52.195 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:02:52.228 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-14T05:02:52.280 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:52.339 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:52.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:52.341 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:52.441 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:52.467 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:52.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:52.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:52.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:52 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2024-09-14T05:02:52.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:52 smithi039 ceph-mon[106470]: osdmap e1581: 8 total, 8 up, 8 in 2024-09-14T05:02:52.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"} : dispatch 2024-09-14T05:02:52.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:52 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"} : dispatch 2024-09-14T05:02:52.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:52 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2024-09-14T05:02:52.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:52 smithi039 ceph-mon[111235]: osdmap e1581: 8 total, 8 up, 8 in 2024-09-14T05:02:52.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"} : dispatch 2024-09-14T05:02:52.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:52 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"} : dispatch 2024-09-14T05:02:52.568 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:52.613 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:52.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:52.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:52.715 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:52.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:52.740 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:52.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:52.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:52 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2024-09-14T05:02:52.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:52 smithi129 ceph-mon[101316]: osdmap e1581: 8 total, 8 up, 8 in 2024-09-14T05:02:52.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"} : dispatch 2024-09-14T05:02:52.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:52 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"} : dispatch 2024-09-14T05:02:52.842 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:52.866 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:52.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:52.867 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:52.968 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:53.015 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:53.016 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:53.016 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:53.117 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:53.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:53.139 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:53.139 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:53.239 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:53.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:53.261 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:53.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:53.362 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:53.386 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:53.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:53.388 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:53.425 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T05:02:53.488 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:02:53.543 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:53.545 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:53.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:02:53.646 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:53.673 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:53.674 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:53.674 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:53.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:53 smithi129 ceph-mon[101316]: pgmap v3148: 109 pgs: 109 active+clean; 6.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 170 B/s rd, 23 MiB/s wr, 696 op/s 2024-09-14T05:02:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:53 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2024-09-14T05:02:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:53 smithi129 ceph-mon[101316]: osdmap e1582: 8 total, 8 up, 8 in 2024-09-14T05:02:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:53 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:53 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:02:53.775 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:53.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:53 smithi039 ceph-mon[106470]: pgmap v3148: 109 pgs: 109 active+clean; 6.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 170 B/s rd, 23 MiB/s wr, 696 op/s 2024-09-14T05:02:53.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:53 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2024-09-14T05:02:53.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:53 smithi039 ceph-mon[106470]: osdmap e1582: 8 total, 8 up, 8 in 2024-09-14T05:02:53.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:53 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:53.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:53 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:02:53.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:53 smithi039 ceph-mon[111235]: pgmap v3148: 109 pgs: 109 active+clean; 6.9 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 170 B/s rd, 23 MiB/s wr, 696 op/s 2024-09-14T05:02:53.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:53 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2024-09-14T05:02:53.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:53 smithi039 ceph-mon[111235]: osdmap e1582: 8 total, 8 up, 8 in 2024-09-14T05:02:53.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:53 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:02:53.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:53 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:02:53.799 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:53.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:53.801 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:53.901 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:53.928 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:53.929 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:53.929 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:54.030 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:54.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:54.058 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:54.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:54.159 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:54.183 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:54.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:54.184 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:54.284 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:54.307 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:54.308 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:54.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:54.409 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:54.431 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:54.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:54.433 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:54.533 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:54.558 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:54.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:54.559 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:54.660 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:54.688 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:54.688 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:54.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:54.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:54 smithi129 ceph-mon[101316]: osdmap e1583: 8 total, 8 up, 8 in 2024-09-14T05:02:54.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:54 smithi129 ceph-mon[101316]: 1.c scrub starts 2024-09-14T05:02:54.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:54 smithi129 ceph-mon[101316]: 1.5 scrub starts 2024-09-14T05:02:54.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:54 smithi129 ceph-mon[101316]: 1.5 scrub ok 2024-09-14T05:02:54.789 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:54.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:54 smithi039 ceph-mon[106470]: osdmap e1583: 8 total, 8 up, 8 in 2024-09-14T05:02:54.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:54 smithi039 ceph-mon[106470]: 1.c scrub starts 2024-09-14T05:02:54.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:54 smithi039 ceph-mon[106470]: 1.5 scrub starts 2024-09-14T05:02:54.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:54 smithi039 ceph-mon[106470]: 1.5 scrub ok 2024-09-14T05:02:54.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:54 smithi039 ceph-mon[111235]: osdmap e1583: 8 total, 8 up, 8 in 2024-09-14T05:02:54.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:54 smithi039 ceph-mon[111235]: 1.c scrub starts 2024-09-14T05:02:54.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:54 smithi039 ceph-mon[111235]: 1.5 scrub starts 2024-09-14T05:02:54.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:54 smithi039 ceph-mon[111235]: 1.5 scrub ok 2024-09-14T05:02:54.816 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:54.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:54.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:54.918 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:54.942 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:54.942 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:54.943 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:55.044 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:55.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:55.068 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:55.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:55.169 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:55.195 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:55.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:55.197 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:55.297 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:55.322 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:55.323 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:55.323 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:55.424 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:55.451 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:55.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:55.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:55.553 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:55.580 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:55 smithi039 ceph-mon[106470]: 1.8 scrub starts 2024-09-14T05:02:55.580 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:55 smithi039 ceph-mon[106470]: 1.8 scrub ok 2024-09-14T05:02:55.580 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:55 smithi039 ceph-mon[106470]: 1.c scrub ok 2024-09-14T05:02:55.581 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:55 smithi039 ceph-mon[106470]: pgmap v3151: 119 pgs: 5 peering, 8 unknown, 106 active+clean; 6.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 255 B/s rd, 4.2 MiB/s wr, 658 op/s 2024-09-14T05:02:55.581 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:55 smithi039 ceph-mon[106470]: 1.18 scrub starts 2024-09-14T05:02:55.581 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:55 smithi039 ceph-mon[106470]: 1.1c scrub starts 2024-09-14T05:02:55.581 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:55 smithi039 ceph-mon[106470]: 1.1c scrub ok 2024-09-14T05:02:55.581 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:55 smithi039 ceph-mon[111235]: 1.8 scrub starts 2024-09-14T05:02:55.581 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:55 smithi039 ceph-mon[111235]: 1.8 scrub ok 2024-09-14T05:02:55.581 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:55 smithi039 ceph-mon[111235]: 1.c scrub ok 2024-09-14T05:02:55.581 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:55 smithi039 ceph-mon[111235]: pgmap v3151: 119 pgs: 5 peering, 8 unknown, 106 active+clean; 6.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 255 B/s rd, 4.2 MiB/s wr, 658 op/s 2024-09-14T05:02:55.581 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:55 smithi039 ceph-mon[111235]: 1.18 scrub starts 2024-09-14T05:02:55.581 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:55 smithi039 ceph-mon[111235]: 1.1c scrub starts 2024-09-14T05:02:55.581 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:55 smithi039 ceph-mon[111235]: 1.1c scrub ok 2024-09-14T05:02:55.596 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:55.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:55.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:55.697 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:55.724 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:55.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:55.725 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:55 smithi129 ceph-mon[101316]: 1.8 scrub starts 2024-09-14T05:02:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:55 smithi129 ceph-mon[101316]: 1.8 scrub ok 2024-09-14T05:02:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:55 smithi129 ceph-mon[101316]: 1.c scrub ok 2024-09-14T05:02:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:55 smithi129 ceph-mon[101316]: pgmap v3151: 119 pgs: 5 peering, 8 unknown, 106 active+clean; 6.8 GiB data, 27 GiB used, 688 GiB / 715 GiB avail; 255 B/s rd, 4.2 MiB/s wr, 658 op/s 2024-09-14T05:02:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:55 smithi129 ceph-mon[101316]: 1.18 scrub starts 2024-09-14T05:02:55.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:55 smithi129 ceph-mon[101316]: 1.1c scrub starts 2024-09-14T05:02:55.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:55 smithi129 ceph-mon[101316]: 1.1c scrub ok 2024-09-14T05:02:55.826 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:55.851 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:55.851 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:55.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:55.859 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:02:55.869 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:02:55.952 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:55.981 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:55.982 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:55.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:56.083 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:56.106 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:56.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:56.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:56.208 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:56.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:56.234 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:56.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:56.335 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:56.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:56.359 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:56.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:56.460 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:56.509 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:56.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:56.510 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:56.611 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:56.634 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:56.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:56.636 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:56.639 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:56 smithi039 ceph-mon[106470]: 1.18 scrub ok 2024-09-14T05:02:56.639 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:56 smithi039 ceph-mon[106470]: 1.b scrub starts 2024-09-14T05:02:56.639 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:56 smithi039 ceph-mon[106470]: 1.b scrub ok 2024-09-14T05:02:56.639 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:56 smithi039 ceph-mon[106470]: 1.a scrub starts 2024-09-14T05:02:56.639 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:56 smithi039 ceph-mon[106470]: 1.a scrub ok 2024-09-14T05:02:56.640 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:56 smithi039 ceph-mon[106470]: 1.e scrub starts 2024-09-14T05:02:56.640 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:56 smithi039 ceph-mon[106470]: 1.e scrub ok 2024-09-14T05:02:56.640 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:56 smithi039 ceph-mon[111235]: 1.18 scrub ok 2024-09-14T05:02:56.640 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:56 smithi039 ceph-mon[111235]: 1.b scrub starts 2024-09-14T05:02:56.640 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:56 smithi039 ceph-mon[111235]: 1.b scrub ok 2024-09-14T05:02:56.640 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:56 smithi039 ceph-mon[111235]: 1.a scrub starts 2024-09-14T05:02:56.640 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:56 smithi039 ceph-mon[111235]: 1.a scrub ok 2024-09-14T05:02:56.640 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:56 smithi039 ceph-mon[111235]: 1.e scrub starts 2024-09-14T05:02:56.641 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:56 smithi039 ceph-mon[111235]: 1.e scrub ok 2024-09-14T05:02:56.736 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:56.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:56 smithi129 ceph-mon[101316]: 1.18 scrub ok 2024-09-14T05:02:56.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:56 smithi129 ceph-mon[101316]: 1.b scrub starts 2024-09-14T05:02:56.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:56 smithi129 ceph-mon[101316]: 1.b scrub ok 2024-09-14T05:02:56.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:56 smithi129 ceph-mon[101316]: 1.a scrub starts 2024-09-14T05:02:56.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:56 smithi129 ceph-mon[101316]: 1.a scrub ok 2024-09-14T05:02:56.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:56 smithi129 ceph-mon[101316]: 1.e scrub starts 2024-09-14T05:02:56.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:56 smithi129 ceph-mon[101316]: 1.e scrub ok 2024-09-14T05:02:56.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:56.784 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:56.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:56.885 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:56.912 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:56.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:56.913 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:57.014 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:57.043 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:57.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:57.044 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:57.145 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:57.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:57.201 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:57.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:57.302 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:57.325 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:57.326 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:57.327 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:57.428 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:02:57.453 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:57.454 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:57.454 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:02:57.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:57 smithi039 ceph-mon[106470]: pgmap v3152: 119 pgs: 5 peering, 5 unknown, 109 active+clean; 6.7 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 178 B/s rd, 2.9 MiB/s wr, 908 op/s 2024-09-14T05:02:57.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:57 smithi039 ceph-mon[106470]: 1.9 deep-scrub starts 2024-09-14T05:02:57.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:57 smithi039 ceph-mon[106470]: 1.9 deep-scrub ok 2024-09-14T05:02:57.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:57 smithi039 ceph-mon[106470]: 1.1b scrub starts 2024-09-14T05:02:57.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:57 smithi039 ceph-mon[106470]: 1.1b scrub ok 2024-09-14T05:02:57.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:57 smithi039 ceph-mon[111235]: pgmap v3152: 119 pgs: 5 peering, 5 unknown, 109 active+clean; 6.7 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 178 B/s rd, 2.9 MiB/s wr, 908 op/s 2024-09-14T05:02:57.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:57 smithi039 ceph-mon[111235]: 1.9 deep-scrub starts 2024-09-14T05:02:57.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:57 smithi039 ceph-mon[111235]: 1.9 deep-scrub ok 2024-09-14T05:02:57.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:57 smithi039 ceph-mon[111235]: 1.1b scrub starts 2024-09-14T05:02:57.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:57 smithi039 ceph-mon[111235]: 1.1b scrub ok 2024-09-14T05:02:57.555 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:57.598 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:57.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:57.599 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:57.700 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:57.745 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:57.746 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:57.746 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:57.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:57 smithi129 ceph-mon[101316]: pgmap v3152: 119 pgs: 5 peering, 5 unknown, 109 active+clean; 6.7 GiB data, 27 GiB used, 689 GiB / 715 GiB avail; 178 B/s rd, 2.9 MiB/s wr, 908 op/s 2024-09-14T05:02:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:57 smithi129 ceph-mon[101316]: 1.9 deep-scrub starts 2024-09-14T05:02:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:57 smithi129 ceph-mon[101316]: 1.9 deep-scrub ok 2024-09-14T05:02:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:57 smithi129 ceph-mon[101316]: 1.1b scrub starts 2024-09-14T05:02:57.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:57 smithi129 ceph-mon[101316]: 1.1b scrub ok 2024-09-14T05:02:57.846 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:57.852 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T05:02:57.852 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T05:02:57.869 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T05:02:57.869 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T05:02:57.892 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:57.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:57.893 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:57.912 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T05:02:57.912 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T05:02:57.945 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T05:02:57.946 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T05:02:57.980 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T05:02:57.980 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T05:02:57.995 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:02:58.013 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T05:02:58.013 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T05:02:58.020 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:58.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:58.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:02:58.054 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T05:02:58.055 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T05:02:58.089 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T05:02:58.089 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T05:02:58.122 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:58.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:58.144 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:58.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:58.246 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:58.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:58.290 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:58.290 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:58.391 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:58.415 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:58.416 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:58.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:58.517 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:58.544 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:58.545 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:58.545 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:58.646 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:58.688 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:58.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:58.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:58.731 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:58 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4154747062' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:02:58.732 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:58 smithi039 ceph-mon[106470]: 1.15 scrub starts 2024-09-14T05:02:58.732 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:58 smithi039 ceph-mon[106470]: 1.15 scrub ok 2024-09-14T05:02:58.732 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:02:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:02:58] "GET /metrics HTTP/1.1" 200 36743 "" "Prometheus/2.51.0" 2024-09-14T05:02:58.732 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:58 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4154747062' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:02:58.732 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:58 smithi039 ceph-mon[111235]: 1.15 scrub starts 2024-09-14T05:02:58.733 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:58 smithi039 ceph-mon[111235]: 1.15 scrub ok 2024-09-14T05:02:58.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:58 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4154747062' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:02:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:58 smithi129 ceph-mon[101316]: 1.15 scrub starts 2024-09-14T05:02:58.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:58 smithi129 ceph-mon[101316]: 1.15 scrub ok 2024-09-14T05:02:58.791 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:58.846 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:58.847 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:58.847 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:58.948 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:58.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:58.990 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:58.990 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:59.091 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:02:59.128 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:59.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:59.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:02:59.230 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:02:59.257 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:59.258 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:59.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:02:59.359 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:59.394 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:59.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:59.396 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:59.496 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:59.521 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:59.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:59.522 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:59.623 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:02:59.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:59.661 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:59.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:02:59.671 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:02:59 smithi129 ceph-mon[101316]: pgmap v3153: 119 pgs: 2 peering, 2 unknown, 115 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2024-09-14T05:02:59.693 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:02:59 smithi039 ceph-mon[106470]: pgmap v3153: 119 pgs: 2 peering, 2 unknown, 115 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2024-09-14T05:02:59.693 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:02:59 smithi039 ceph-mon[111235]: pgmap v3153: 119 pgs: 2 peering, 2 unknown, 115 active+clean; 6.7 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 654 op/s 2024-09-14T05:02:59.762 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:02:59.793 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:02:59.793 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:59.794 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:02:59.894 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:02:59.938 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:02:59.939 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:02:59.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:02:59.978 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-14T05:02:59.978 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T05:02:59.980 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2024-09-14T05:02:59.980 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-14T05:02:59.980 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-14T05:02:59.980 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.1 config set filestore_inject_stall 3 2024-09-14T05:03:00.040 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:00.054 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T05:03:00.054 INFO:teuthology.orchestra.run.smithi039.stdout: "size": 20, 2024-09-14T05:03:00.054 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 600, 2024-09-14T05:03:00.054 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [ 2024-09-14T05:03:00.054 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.054 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:34402 57.0 57:0b7b4153:::benchmark_data_smithi039_365565_object34401:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T05:03:00.054 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:57:13.467717+0000", 2024-09-14T05:03:00.054 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 346.58207755500001, 2024-09-14T05:03:00.054 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.43708126800000002, 2024-09-14T05:03:00.054 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:00.054 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:00.055 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:03:00.055 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:03:00.055 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T05:03:00.055 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:03:00.055 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 34402 2024-09-14T05:03:00.055 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.055 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:00.055 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.055 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:00.055 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.467717+0000", 2024-09-14T05:03:00.055 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.055 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.055 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.056 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:00.056 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.467717+0000", 2024-09-14T05:03:00.056 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.056 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.056 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.056 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:00.056 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.467717+0000", 2024-09-14T05:03:00.056 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5600000000000001e-07 2024-09-14T05:03:00.056 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.056 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.056 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:00.056 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.467758+0000", 2024-09-14T05:03:00.056 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.0911000000000002e-05 2024-09-14T05:03:00.057 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.057 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.057 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:00.057 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.467765+0000", 2024-09-14T05:03:00.057 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.2180000000000002e-06 2024-09-14T05:03:00.057 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.057 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.057 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:00.057 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.467771+0000", 2024-09-14T05:03:00.057 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4410000000000001e-06 2024-09-14T05:03:00.057 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.057 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.057 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.058 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.469591+0000", 2024-09-14T05:03:00.058 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0018204040000000001 2024-09-14T05:03:00.058 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.058 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.058 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:00.058 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.469834+0000", 2024-09-14T05:03:00.058 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00024333800000000001 2024-09-14T05:03:00.058 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.058 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.058 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4", 2024-09-14T05:03:00.058 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.470676+0000", 2024-09-14T05:03:00.058 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000841554 2024-09-14T05:03:00.058 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.058 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.059 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:03:00.059 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.474378+0000", 2024-09-14T05:03:00.059 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.003701738 2024-09-14T05:03:00.059 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.059 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.059 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:03:00.059 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.474392+0000", 2024-09-14T05:03:00.059 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4141000000000001e-05 2024-09-14T05:03:00.059 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.059 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.059 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:03:00.059 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.904687+0000", 2024-09-14T05:03:00.059 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.43029504400000002 2024-09-14T05:03:00.060 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.060 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.060 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:03:00.060 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.904751+0000", 2024-09-14T05:03:00.060 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.4424999999999994e-05 2024-09-14T05:03:00.060 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.060 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.060 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:00.060 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.904798+0000", 2024-09-14T05:03:00.060 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6597999999999997e-05 2024-09-14T05:03:00.060 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.060 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:00.061 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.061 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.061 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.061 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:34438 57.0 57:065b00bc:::benchmark_data_smithi039_365565_object34437:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T05:03:00.061 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:57:13.484152+0000", 2024-09-14T05:03:00.061 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 346.56564197599999, 2024-09-14T05:03:00.061 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.42189383600000002, 2024-09-14T05:03:00.061 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:00.061 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:00.061 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:03:00.061 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:03:00.061 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T05:03:00.061 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:03:00.062 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 34438 2024-09-14T05:03:00.062 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.062 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:00.062 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.062 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:00.062 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.484152+0000", 2024-09-14T05:03:00.062 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.062 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.062 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.062 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:00.062 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.484152+0000", 2024-09-14T05:03:00.062 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.063 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.063 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.063 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:00.063 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.484154+0000", 2024-09-14T05:03:00.063 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9860000000000001e-06 2024-09-14T05:03:00.063 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.063 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.063 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:00.063 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.484187+0000", 2024-09-14T05:03:00.063 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.2320000000000002e-05 2024-09-14T05:03:00.064 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.064 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.064 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:00.064 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.484204+0000", 2024-09-14T05:03:00.064 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7586999999999999e-05 2024-09-14T05:03:00.064 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.064 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.064 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:00.064 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.484212+0000", 2024-09-14T05:03:00.064 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.2719999999999997e-06 2024-09-14T05:03:00.064 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.064 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.065 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.065 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.484266+0000", 2024-09-14T05:03:00.065 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3757e-05 2024-09-14T05:03:00.065 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.065 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.065 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:00.065 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.484454+0000", 2024-09-14T05:03:00.065 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000188257 2024-09-14T05:03:00.065 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.065 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.065 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4", 2024-09-14T05:03:00.065 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.484618+0000", 2024-09-14T05:03:00.065 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00016396499999999999 2024-09-14T05:03:00.066 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.066 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.066 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:03:00.066 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.488095+0000", 2024-09-14T05:03:00.066 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0034769409999999999 2024-09-14T05:03:00.066 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.066 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.066 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:03:00.066 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.488300+0000", 2024-09-14T05:03:00.067 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00020430700000000001 2024-09-14T05:03:00.067 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.067 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.067 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:03:00.067 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.905939+0000", 2024-09-14T05:03:00.067 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.417639447 2024-09-14T05:03:00.067 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.067 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.067 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:03:00.067 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.906005+0000", 2024-09-14T05:03:00.067 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.5717999999999997e-05 2024-09-14T05:03:00.067 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.067 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.068 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:00.068 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.906046+0000", 2024-09-14T05:03:00.068 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.1279000000000002e-05 2024-09-14T05:03:00.068 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.068 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:00.068 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.068 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.068 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.068 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:34442 57.0 57:013cd97f:::benchmark_data_smithi039_365565_object34441:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T05:03:00.068 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:57:13.485197+0000", 2024-09-14T05:03:00.068 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 346.56459700300002, 2024-09-14T05:03:00.069 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.42272372499999999, 2024-09-14T05:03:00.069 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:00.069 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:00.069 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:03:00.069 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:03:00.069 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T05:03:00.069 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:03:00.069 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 34442 2024-09-14T05:03:00.069 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.069 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:00.069 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.070 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:00.070 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.485197+0000", 2024-09-14T05:03:00.070 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.070 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.070 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.070 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:00.070 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.485197+0000", 2024-09-14T05:03:00.070 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.070 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.070 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.070 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:00.070 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.485198+0000", 2024-09-14T05:03:00.070 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.02e-07 2024-09-14T05:03:00.070 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.071 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.071 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:00.071 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.485221+0000", 2024-09-14T05:03:00.071 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2986e-05 2024-09-14T05:03:00.071 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.071 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.071 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:00.071 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.485232+0000", 2024-09-14T05:03:00.071 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1035e-05 2024-09-14T05:03:00.072 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.072 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.072 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:00.072 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.485240+0000", 2024-09-14T05:03:00.072 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.5780000000000001e-06 2024-09-14T05:03:00.072 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.072 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.072 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.072 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.486164+0000", 2024-09-14T05:03:00.072 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00092453100000000003 2024-09-14T05:03:00.072 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.072 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.073 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:00.073 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.486356+0000", 2024-09-14T05:03:00.073 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00019142000000000001 2024-09-14T05:03:00.073 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.073 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.073 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4", 2024-09-14T05:03:00.073 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.486490+0000", 2024-09-14T05:03:00.073 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013404000000000001 2024-09-14T05:03:00.073 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.073 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.073 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:03:00.074 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.488509+0000", 2024-09-14T05:03:00.074 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0020197090000000002 2024-09-14T05:03:00.074 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.074 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.074 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:03:00.074 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.489162+0000", 2024-09-14T05:03:00.074 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00065259200000000002 2024-09-14T05:03:00.074 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.074 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.074 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:03:00.075 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.907856+0000", 2024-09-14T05:03:00.075 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.418694281 2024-09-14T05:03:00.075 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.075 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.075 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:03:00.075 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.907892+0000", 2024-09-14T05:03:00.075 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.5346999999999999e-05 2024-09-14T05:03:00.075 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.075 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.075 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:00.075 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.907921+0000", 2024-09-14T05:03:00.076 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.9303999999999999e-05 2024-09-14T05:03:00.076 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.076 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:00.076 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.076 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.076 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.076 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:34456 57.0 57:02c69cda:::benchmark_data_smithi039_365565_object34455:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T05:03:00.076 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:57:13.491976+0000", 2024-09-14T05:03:00.076 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 346.55781836599999, 2024-09-14T05:03:00.077 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.41608771700000002, 2024-09-14T05:03:00.077 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:00.077 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:00.077 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:03:00.077 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:03:00.077 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T05:03:00.077 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:03:00.077 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 34456 2024-09-14T05:03:00.077 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.078 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:00.078 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.078 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:00.078 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.491976+0000", 2024-09-14T05:03:00.078 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.078 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.078 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.078 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:00.079 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.491976+0000", 2024-09-14T05:03:00.079 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.079 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.079 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.079 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:00.079 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.491977+0000", 2024-09-14T05:03:00.079 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.443e-06 2024-09-14T05:03:00.079 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.079 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.079 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:00.079 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.492000+0000", 2024-09-14T05:03:00.079 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3102e-05 2024-09-14T05:03:00.080 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.080 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.080 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:00.080 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.492011+0000", 2024-09-14T05:03:00.080 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.061e-05 2024-09-14T05:03:00.080 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.080 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.080 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:00.080 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.492019+0000", 2024-09-14T05:03:00.080 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6450000000000005e-06 2024-09-14T05:03:00.080 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.080 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.080 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.081 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.492068+0000", 2024-09-14T05:03:00.081 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9487999999999998e-05 2024-09-14T05:03:00.081 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.081 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.081 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:00.081 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.492175+0000", 2024-09-14T05:03:00.081 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000106523 2024-09-14T05:03:00.081 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.081 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.081 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4", 2024-09-14T05:03:00.081 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.492280+0000", 2024-09-14T05:03:00.081 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000105577 2024-09-14T05:03:00.081 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.081 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.082 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:03:00.082 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.494189+0000", 2024-09-14T05:03:00.082 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0019083329999999999 2024-09-14T05:03:00.082 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.082 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.082 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:03:00.082 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.494337+0000", 2024-09-14T05:03:00.082 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000148753 2024-09-14T05:03:00.082 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.082 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.082 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:03:00.082 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.907973+0000", 2024-09-14T05:03:00.082 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.41363581399999999 2024-09-14T05:03:00.082 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.083 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.083 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:03:00.083 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.908018+0000", 2024-09-14T05:03:00.083 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5003000000000003e-05 2024-09-14T05:03:00.083 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.083 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.083 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:00.083 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.908064+0000", 2024-09-14T05:03:00.083 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5426000000000002e-05 2024-09-14T05:03:00.083 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.083 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:00.083 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.083 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.083 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.084 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:34458 57.b 57:d7441ced:::benchmark_data_smithi039_365565_object34457:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T05:03:00.084 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:57:13.492938+0000", 2024-09-14T05:03:00.084 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 346.55685619100001, 2024-09-14T05:03:00.084 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.410365072, 2024-09-14T05:03:00.084 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:00.084 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:00.084 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:03:00.084 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:03:00.084 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T05:03:00.084 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:03:00.084 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 34458 2024-09-14T05:03:00.084 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.084 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:00.084 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.085 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:00.085 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.492938+0000", 2024-09-14T05:03:00.085 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.085 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.085 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.085 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:00.085 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.492938+0000", 2024-09-14T05:03:00.085 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.085 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.086 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.086 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:00.086 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.492939+0000", 2024-09-14T05:03:00.086 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1900000000000002e-07 2024-09-14T05:03:00.086 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.086 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.086 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:00.086 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.492963+0000", 2024-09-14T05:03:00.087 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4712999999999999e-05 2024-09-14T05:03:00.087 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.087 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.087 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:00.087 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.492974+0000", 2024-09-14T05:03:00.087 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0922e-05 2024-09-14T05:03:00.087 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.087 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.087 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:00.087 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.492981+0000", 2024-09-14T05:03:00.088 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.6030000000000001e-06 2024-09-14T05:03:00.088 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.088 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.088 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.088 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.493039+0000", 2024-09-14T05:03:00.088 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.8365000000000001e-05 2024-09-14T05:03:00.088 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.088 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.088 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:00.088 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.493179+0000", 2024-09-14T05:03:00.088 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013982799999999999 2024-09-14T05:03:00.088 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.088 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.089 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 4,6", 2024-09-14T05:03:00.089 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.493335+0000", 2024-09-14T05:03:00.089 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015545199999999999 2024-09-14T05:03:00.089 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.089 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.089 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:03:00.089 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.496147+0000", 2024-09-14T05:03:00.089 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0028121930000000002 2024-09-14T05:03:00.089 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.089 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.089 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:03:00.089 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.496449+0000", 2024-09-14T05:03:00.089 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00030275000000000001 2024-09-14T05:03:00.089 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.090 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.090 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:03:00.090 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.903211+0000", 2024-09-14T05:03:00.090 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.40676116099999998 2024-09-14T05:03:00.090 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.090 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.090 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:03:00.090 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.903256+0000", 2024-09-14T05:03:00.090 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5325999999999999e-05 2024-09-14T05:03:00.090 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.090 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.090 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:00.090 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.903303+0000", 2024-09-14T05:03:00.090 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7139999999999999e-05 2024-09-14T05:03:00.091 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.091 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:00.091 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.091 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.091 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.091 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:34461 57.b 57:d8ab32cf:::benchmark_data_smithi039_365565_object34460:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T05:03:00.091 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:57:13.493451+0000", 2024-09-14T05:03:00.091 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 346.55634348000001, 2024-09-14T05:03:00.091 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.41006606699999998, 2024-09-14T05:03:00.091 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:00.091 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:00.091 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:03:00.091 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:03:00.091 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T05:03:00.092 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:03:00.092 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 34461 2024-09-14T05:03:00.092 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.092 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:00.092 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.092 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:00.092 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.493451+0000", 2024-09-14T05:03:00.092 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.092 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.092 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.092 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:00.092 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.493451+0000", 2024-09-14T05:03:00.092 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.093 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.093 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.093 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:00.093 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.493452+0000", 2024-09-14T05:03:00.093 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.4200000000000004e-07 2024-09-14T05:03:00.093 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.093 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.093 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:00.093 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.493504+0000", 2024-09-14T05:03:00.093 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.2448000000000003e-05 2024-09-14T05:03:00.093 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.093 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.093 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:00.094 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.493515+0000", 2024-09-14T05:03:00.094 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0791000000000001e-05 2024-09-14T05:03:00.094 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.094 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.094 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:00.094 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.493523+0000", 2024-09-14T05:03:00.094 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.3289999999999994e-06 2024-09-14T05:03:00.094 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.094 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.094 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.094 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.494784+0000", 2024-09-14T05:03:00.094 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0012608109999999999 2024-09-14T05:03:00.094 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.094 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.095 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:00.095 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.494925+0000", 2024-09-14T05:03:00.095 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000141252 2024-09-14T05:03:00.095 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.095 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.095 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 4,6", 2024-09-14T05:03:00.095 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.495080+0000", 2024-09-14T05:03:00.095 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015435899999999999 2024-09-14T05:03:00.095 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.095 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.095 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:03:00.095 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.496265+0000", 2024-09-14T05:03:00.095 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001184977 2024-09-14T05:03:00.095 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.096 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.096 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:03:00.096 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.496542+0000", 2024-09-14T05:03:00.096 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00027730799999999998 2024-09-14T05:03:00.096 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.096 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.096 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:03:00.096 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.903461+0000", 2024-09-14T05:03:00.096 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.40691940599999998 2024-09-14T05:03:00.096 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.096 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.096 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:03:00.097 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.903494+0000", 2024-09-14T05:03:00.097 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.2249000000000002e-05 2024-09-14T05:03:00.097 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.097 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.097 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:00.097 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.903517+0000", 2024-09-14T05:03:00.097 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3195000000000001e-05 2024-09-14T05:03:00.097 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.097 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:00.097 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.097 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.097 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.097 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:34463 57.0 57:0b4e307a:::benchmark_data_smithi039_365565_object34462:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T05:03:00.097 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:57:13.495720+0000", 2024-09-14T05:03:00.098 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 346.55407398900002, 2024-09-14T05:03:00.098 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.41252305299999997, 2024-09-14T05:03:00.098 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:00.098 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:00.098 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:03:00.098 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:03:00.098 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T05:03:00.098 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:03:00.098 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 34463 2024-09-14T05:03:00.098 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.098 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:00.098 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.098 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:00.099 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.495720+0000", 2024-09-14T05:03:00.099 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.099 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.099 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.099 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:00.099 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.495720+0000", 2024-09-14T05:03:00.099 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.099 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.099 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.099 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:00.099 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.495721+0000", 2024-09-14T05:03:00.099 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0100000000000004e-07 2024-09-14T05:03:00.100 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.100 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.100 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:00.100 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.495740+0000", 2024-09-14T05:03:00.100 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8817999999999998e-05 2024-09-14T05:03:00.100 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.100 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.100 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:00.100 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.495749+0000", 2024-09-14T05:03:00.101 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.5030000000000003e-06 2024-09-14T05:03:00.101 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.101 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.101 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:00.101 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.495754+0000", 2024-09-14T05:03:00.101 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4240000000000003e-06 2024-09-14T05:03:00.101 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.101 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.101 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.101 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.495788+0000", 2024-09-14T05:03:00.101 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.4542000000000002e-05 2024-09-14T05:03:00.101 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.102 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.102 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:00.102 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.495872+0000", 2024-09-14T05:03:00.102 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.3380999999999993e-05 2024-09-14T05:03:00.102 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.102 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.102 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4", 2024-09-14T05:03:00.102 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.495953+0000", 2024-09-14T05:03:00.102 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.1767999999999994e-05 2024-09-14T05:03:00.103 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.103 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.103 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:03:00.103 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.497390+0000", 2024-09-14T05:03:00.103 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0014363240000000001 2024-09-14T05:03:00.103 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.103 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.103 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:03:00.103 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.497688+0000", 2024-09-14T05:03:00.103 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00029810699999999998 2024-09-14T05:03:00.103 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.103 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.103 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:03:00.103 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.908150+0000", 2024-09-14T05:03:00.104 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.41046164000000002 2024-09-14T05:03:00.104 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.104 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.104 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:03:00.104 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.908206+0000", 2024-09-14T05:03:00.104 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6706000000000002e-05 2024-09-14T05:03:00.104 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.104 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.104 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:00.105 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:57:13.908243+0000", 2024-09-14T05:03:00.105 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.7039000000000002e-05 2024-09-14T05:03:00.105 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.105 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:00.105 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.105 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.105 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.105 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:241655 57.0 57:057b54d7:::benchmark_data_smithi039_365565_object117903:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1558)", 2024-09-14T05:03:00.105 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:00:42.448771+0000", 2024-09-14T05:03:00.105 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 137.60102314100001, 2024-09-14T05:03:00.105 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.45022619200000003, 2024-09-14T05:03:00.105 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:00.105 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:00.106 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T05:03:00.106 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:03:00.106 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T05:03:00.106 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:03:00.106 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 241655 2024-09-14T05:03:00.106 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.106 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:00.106 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.106 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:00.106 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.448771+0000", 2024-09-14T05:03:00.107 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.107 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.107 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.107 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:00.107 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.448771+0000", 2024-09-14T05:03:00.107 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.107 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.107 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.107 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:00.107 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.448771+0000", 2024-09-14T05:03:00.107 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.3500000000000002e-07 2024-09-14T05:03:00.107 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.108 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.108 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:00.108 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.448773+0000", 2024-09-14T05:03:00.108 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5799999999999999e-06 2024-09-14T05:03:00.108 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.108 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.108 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:00.108 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.448778+0000", 2024-09-14T05:03:00.108 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.22e-06 2024-09-14T05:03:00.108 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.108 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.108 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:00.108 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.448782+0000", 2024-09-14T05:03:00.109 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2089999999999999e-06 2024-09-14T05:03:00.109 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.109 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.109 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.109 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.448824+0000", 2024-09-14T05:03:00.109 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.1594999999999997e-05 2024-09-14T05:03:00.109 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.109 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.109 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:00.109 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.448923+0000", 2024-09-14T05:03:00.109 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.9444000000000005e-05 2024-09-14T05:03:00.109 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.109 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.109 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4", 2024-09-14T05:03:00.110 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.449023+0000", 2024-09-14T05:03:00.110 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.9865999999999996e-05 2024-09-14T05:03:00.110 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.110 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.110 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:03:00.110 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.471443+0000", 2024-09-14T05:03:00.110 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.022419952999999999 2024-09-14T05:03:00.110 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.110 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.110 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:03:00.111 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.471501+0000", 2024-09-14T05:03:00.111 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7284999999999998e-05 2024-09-14T05:03:00.111 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.111 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.111 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.111 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.898872+0000", 2024-09-14T05:03:00.111 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.42737162699999998 2024-09-14T05:03:00.111 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.111 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.112 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:00.112 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.898997+0000", 2024-09-14T05:03:00.112 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012507800000000001 2024-09-14T05:03:00.112 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.112 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:00.112 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.112 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.112 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.112 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:241658 57.0 57:0d0b1b52:::benchmark_data_smithi039_365565_object117906:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1558)", 2024-09-14T05:03:00.113 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:00:42.455864+0000", 2024-09-14T05:03:00.113 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 137.593930688, 2024-09-14T05:03:00.113 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.44378162799999998, 2024-09-14T05:03:00.113 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:00.113 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:00.113 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T05:03:00.113 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:03:00.113 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T05:03:00.113 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:03:00.114 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 241658 2024-09-14T05:03:00.114 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.114 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:00.114 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.114 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:00.114 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.455864+0000", 2024-09-14T05:03:00.114 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.115 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.115 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.115 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:00.115 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.455864+0000", 2024-09-14T05:03:00.115 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.115 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.115 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.115 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:00.115 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.455865+0000", 2024-09-14T05:03:00.115 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3200000000000001e-06 2024-09-14T05:03:00.115 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.115 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.115 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:00.115 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.455869+0000", 2024-09-14T05:03:00.116 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.8380000000000004e-06 2024-09-14T05:03:00.116 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.116 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.116 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:00.116 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.455877+0000", 2024-09-14T05:03:00.116 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.4069999999999993e-06 2024-09-14T05:03:00.116 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.116 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.116 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:00.116 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.455882+0000", 2024-09-14T05:03:00.116 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.2109999999999997e-06 2024-09-14T05:03:00.116 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.116 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.117 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.117 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.457825+0000", 2024-09-14T05:03:00.117 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0019427750000000001 2024-09-14T05:03:00.117 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.117 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.117 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:00.117 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.457949+0000", 2024-09-14T05:03:00.117 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012433999999999999 2024-09-14T05:03:00.117 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.117 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.117 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4", 2024-09-14T05:03:00.117 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.458075+0000", 2024-09-14T05:03:00.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:00.118 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:00.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:00.119 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012542299999999999 2024-09-14T05:03:00.119 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.119 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.119 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:03:00.119 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.471512+0000", 2024-09-14T05:03:00.120 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.013437050000000001 2024-09-14T05:03:00.120 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.120 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.120 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:03:00.120 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.471567+0000", 2024-09-14T05:03:00.120 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5241999999999997e-05 2024-09-14T05:03:00.120 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.120 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.120 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.120 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.899630+0000", 2024-09-14T05:03:00.120 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.428062783 2024-09-14T05:03:00.120 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.120 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.121 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:00.121 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.899645+0000", 2024-09-14T05:03:00.121 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5238999999999999e-05 2024-09-14T05:03:00.121 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.121 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:00.121 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.121 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.121 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.121 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:241659 57.3 57:cf6d98e1:::benchmark_data_smithi039_365565_object117907:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1558)", 2024-09-14T05:03:00.121 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:00:42.455896+0000", 2024-09-14T05:03:00.121 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 137.59389832400001, 2024-09-14T05:03:00.121 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.43414912700000002, 2024-09-14T05:03:00.121 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:00.121 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:00.122 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T05:03:00.122 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:03:00.122 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T05:03:00.122 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:03:00.122 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 241659 2024-09-14T05:03:00.122 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.122 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:00.122 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.122 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:00.122 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.455896+0000", 2024-09-14T05:03:00.122 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.122 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.122 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.122 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:00.123 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.455896+0000", 2024-09-14T05:03:00.123 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.123 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.123 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.123 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:00.123 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.455896+0000", 2024-09-14T05:03:00.123 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.8200000000000001e-07 2024-09-14T05:03:00.123 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.123 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.123 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:00.123 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.455898+0000", 2024-09-14T05:03:00.123 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8899999999999999e-06 2024-09-14T05:03:00.123 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.123 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.124 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:00.124 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.455904+0000", 2024-09-14T05:03:00.124 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6930000000000004e-06 2024-09-14T05:03:00.124 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.124 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.124 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:00.124 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.455909+0000", 2024-09-14T05:03:00.124 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6600000000000003e-06 2024-09-14T05:03:00.124 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.124 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.124 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.124 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.456775+0000", 2024-09-14T05:03:00.124 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00086638899999999996 2024-09-14T05:03:00.124 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.125 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.125 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:00.125 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.456887+0000", 2024-09-14T05:03:00.125 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011208400000000001 2024-09-14T05:03:00.125 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.125 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.125 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,6", 2024-09-14T05:03:00.125 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.456997+0000", 2024-09-14T05:03:00.125 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000109583 2024-09-14T05:03:00.125 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.125 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.125 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:03:00.126 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.464957+0000", 2024-09-14T05:03:00.126 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.007960211 2024-09-14T05:03:00.126 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.126 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.126 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:03:00.126 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.468882+0000", 2024-09-14T05:03:00.126 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0039250869999999998 2024-09-14T05:03:00.126 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.126 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.126 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.126 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.890031+0000", 2024-09-14T05:03:00.126 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.42114931500000002 2024-09-14T05:03:00.126 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.127 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.127 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:00.127 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.890045+0000", 2024-09-14T05:03:00.127 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3833e-05 2024-09-14T05:03:00.127 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.127 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:00.127 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.127 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.127 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.127 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:241661 57.0 57:0488aa79:::benchmark_data_smithi039_365565_object117909:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1558)", 2024-09-14T05:03:00.127 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:00:42.460732+0000", 2024-09-14T05:03:00.127 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 137.58906255900001, 2024-09-14T05:03:00.127 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.43896001600000001, 2024-09-14T05:03:00.128 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:00.128 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:00.128 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T05:03:00.128 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:03:00.128 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T05:03:00.128 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:03:00.128 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 241661 2024-09-14T05:03:00.128 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.128 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:00.128 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.128 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:00.128 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.460732+0000", 2024-09-14T05:03:00.128 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.128 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.129 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.129 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:00.129 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.460732+0000", 2024-09-14T05:03:00.129 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.129 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.129 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.129 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:00.129 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.460733+0000", 2024-09-14T05:03:00.129 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.232e-06 2024-09-14T05:03:00.129 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.129 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.129 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:00.130 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.460738+0000", 2024-09-14T05:03:00.130 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5790000000000003e-06 2024-09-14T05:03:00.130 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.130 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.130 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:00.130 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.460746+0000", 2024-09-14T05:03:00.130 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9889999999999992e-06 2024-09-14T05:03:00.130 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.130 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.130 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:00.130 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.460754+0000", 2024-09-14T05:03:00.130 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7069999999999992e-06 2024-09-14T05:03:00.130 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.130 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.131 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.131 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.466286+0000", 2024-09-14T05:03:00.131 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0055321019999999997 2024-09-14T05:03:00.131 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.131 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.131 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:00.131 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.466412+0000", 2024-09-14T05:03:00.131 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012618899999999999 2024-09-14T05:03:00.131 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.131 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.131 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4", 2024-09-14T05:03:00.131 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.466526+0000", 2024-09-14T05:03:00.131 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011317899999999999 2024-09-14T05:03:00.131 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.132 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.132 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:03:00.132 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.471633+0000", 2024-09-14T05:03:00.132 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0051077379999999997 2024-09-14T05:03:00.132 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.132 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.132 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:03:00.132 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.474005+0000", 2024-09-14T05:03:00.132 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00237171 2024-09-14T05:03:00.132 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.132 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.132 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.132 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.899680+0000", 2024-09-14T05:03:00.132 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.42567531600000003 2024-09-14T05:03:00.133 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.133 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.133 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:00.133 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.899692+0000", 2024-09-14T05:03:00.133 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1275e-05 2024-09-14T05:03:00.133 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.133 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:00.133 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.133 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.133 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.133 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:241665 57.b 57:d4fb4ac4:::benchmark_data_smithi039_365565_object117913:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1558)", 2024-09-14T05:03:00.133 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:00:42.475619+0000", 2024-09-14T05:03:00.133 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 137.57417569500001, 2024-09-14T05:03:00.134 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.41515851199999998, 2024-09-14T05:03:00.134 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:00.134 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:00.134 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:03:00.135 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:03:00.135 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T05:03:00.135 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:03:00.135 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 241665 2024-09-14T05:03:00.135 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.135 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:00.135 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.135 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:00.135 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.475619+0000", 2024-09-14T05:03:00.136 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.136 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.136 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.136 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:00.136 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.475619+0000", 2024-09-14T05:03:00.136 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.136 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.136 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.136 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:00.136 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.475620+0000", 2024-09-14T05:03:00.136 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7659999999999999e-06 2024-09-14T05:03:00.136 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.137 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.137 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:00.137 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.475626+0000", 2024-09-14T05:03:00.137 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0549999999999996e-06 2024-09-14T05:03:00.137 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.137 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.137 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:00.137 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.475642+0000", 2024-09-14T05:03:00.137 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5206e-05 2024-09-14T05:03:00.137 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.137 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.138 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:00.138 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.475650+0000", 2024-09-14T05:03:00.138 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.2309999999999996e-06 2024-09-14T05:03:00.138 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.138 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.138 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.138 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.475707+0000", 2024-09-14T05:03:00.138 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7417000000000003e-05 2024-09-14T05:03:00.138 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.138 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.138 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:00.138 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.475801+0000", 2024-09-14T05:03:00.139 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.3913999999999996e-05 2024-09-14T05:03:00.139 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.139 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.139 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 4,6", 2024-09-14T05:03:00.139 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.475873+0000", 2024-09-14T05:03:00.139 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.2205000000000004e-05 2024-09-14T05:03:00.139 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.139 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.139 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:03:00.139 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.480193+0000", 2024-09-14T05:03:00.139 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0043197720000000004 2024-09-14T05:03:00.139 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.140 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.140 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:03:00.140 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.481580+0000", 2024-09-14T05:03:00.140 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001386979 2024-09-14T05:03:00.140 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.140 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.140 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:03:00.140 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.890679+0000", 2024-09-14T05:03:00.140 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.40909881100000001 2024-09-14T05:03:00.140 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.140 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.141 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:03:00.141 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.890730+0000", 2024-09-14T05:03:00.141 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1099e-05 2024-09-14T05:03:00.141 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.141 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.141 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:00.141 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.890777+0000", 2024-09-14T05:03:00.141 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7057e-05 2024-09-14T05:03:00.141 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.141 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:00.141 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.141 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.142 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.142 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:241667 57.3 57:cdc1a956:::benchmark_data_smithi039_365565_object117915:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1558)", 2024-09-14T05:03:00.142 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:00:42.477674+0000", 2024-09-14T05:03:00.142 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 137.57211974200001, 2024-09-14T05:03:00.142 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.41319688599999999, 2024-09-14T05:03:00.142 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:00.142 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:00.142 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T05:03:00.142 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:03:00.142 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T05:03:00.142 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:03:00.143 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 241667 2024-09-14T05:03:00.143 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.143 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:00.143 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.143 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:00.143 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.477674+0000", 2024-09-14T05:03:00.143 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.143 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.143 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.143 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:00.144 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.477674+0000", 2024-09-14T05:03:00.144 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.144 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.144 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.144 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:00.144 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.477676+0000", 2024-09-14T05:03:00.144 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0780000000000001e-06 2024-09-14T05:03:00.144 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.144 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.145 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:00.145 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.477680+0000", 2024-09-14T05:03:00.145 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2889999999999999e-06 2024-09-14T05:03:00.145 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.145 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.145 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:00.145 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.477692+0000", 2024-09-14T05:03:00.145 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1759e-05 2024-09-14T05:03:00.145 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.145 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.145 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:00.146 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.477698+0000", 2024-09-14T05:03:00.146 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.5819999999999998e-06 2024-09-14T05:03:00.146 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.146 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.146 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.146 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.477757+0000", 2024-09-14T05:03:00.146 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.9092000000000003e-05 2024-09-14T05:03:00.146 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.146 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.146 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:00.147 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.477871+0000", 2024-09-14T05:03:00.147 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011369199999999999 2024-09-14T05:03:00.147 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.147 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.147 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,6", 2024-09-14T05:03:00.147 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.477976+0000", 2024-09-14T05:03:00.147 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00010487199999999999 2024-09-14T05:03:00.147 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.147 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.148 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.148 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.890861+0000", 2024-09-14T05:03:00.148 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.41288512799999999 2024-09-14T05:03:00.148 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.148 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.148 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:00.148 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.890871+0000", 2024-09-14T05:03:00.148 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0394000000000001e-05 2024-09-14T05:03:00.148 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.148 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:00.148 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.149 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.149 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.149 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:241655 57.0 57:057b54d7:::benchmark_data_smithi039_365565_object117903:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1559)", 2024-09-14T05:03:00.149 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:00:42.483165+0000", 2024-09-14T05:03:00.149 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 137.56662922000001, 2024-09-14T05:03:00.149 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0762030119999999, 2024-09-14T05:03:00.149 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:00.149 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:00.149 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T05:03:00.149 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:03:00.149 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T05:03:00.149 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:03:00.149 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 241655 2024-09-14T05:03:00.150 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.150 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:00.150 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.150 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:00.150 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483165+0000", 2024-09-14T05:03:00.150 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.150 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.150 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.150 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:00.150 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483165+0000", 2024-09-14T05:03:00.150 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.150 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.150 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.150 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:00.151 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483167+0000", 2024-09-14T05:03:00.151 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9410000000000001e-06 2024-09-14T05:03:00.151 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.151 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.151 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:00.151 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483211+0000", 2024-09-14T05:03:00.151 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3866999999999999e-05 2024-09-14T05:03:00.151 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.151 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.151 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:00.151 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483255+0000", 2024-09-14T05:03:00.151 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3702000000000003e-05 2024-09-14T05:03:00.151 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.151 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.152 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:00.152 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483263+0000", 2024-09-14T05:03:00.152 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.3939999999999996e-06 2024-09-14T05:03:00.152 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.152 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.152 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.152 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.899738+0000", 2024-09-14T05:03:00.152 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.41647474600000001 2024-09-14T05:03:00.152 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.152 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.152 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T05:03:00.152 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.899747+0000", 2024-09-14T05:03:00.152 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.1980000000000007e-06 2024-09-14T05:03:00.153 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.153 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.153 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.153 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:43.559286+0000", 2024-09-14T05:03:00.153 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.65953922600000003 2024-09-14T05:03:00.153 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.153 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.153 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:00.153 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:43.559368+0000", 2024-09-14T05:03:00.153 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.1938000000000001e-05 2024-09-14T05:03:00.153 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.153 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:00.154 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.154 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.154 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.154 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:241658 57.10 57:0d0b1b52:::benchmark_data_smithi039_365565_object117906:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1559)", 2024-09-14T05:03:00.154 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:00:42.483277+0000", 2024-09-14T05:03:00.154 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 137.56651674400001, 2024-09-14T05:03:00.154 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1449219989999999, 2024-09-14T05:03:00.154 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:00.154 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:00.154 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T05:03:00.154 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:03:00.154 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T05:03:00.154 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:03:00.154 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 241658 2024-09-14T05:03:00.155 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.155 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:00.155 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.155 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:00.155 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483277+0000", 2024-09-14T05:03:00.155 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.155 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.155 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.155 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:00.155 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483277+0000", 2024-09-14T05:03:00.155 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.155 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.155 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.156 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:00.156 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483278+0000", 2024-09-14T05:03:00.156 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.8100000000000003e-07 2024-09-14T05:03:00.156 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.156 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.156 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:00.156 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483281+0000", 2024-09-14T05:03:00.156 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.2459999999999999e-06 2024-09-14T05:03:00.156 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.156 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.156 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:00.156 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483291+0000", 2024-09-14T05:03:00.156 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.4949999999999997e-06 2024-09-14T05:03:00.157 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.157 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.157 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:00.157 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483296+0000", 2024-09-14T05:03:00.157 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6810000000000003e-06 2024-09-14T05:03:00.157 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.157 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.157 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.157 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.899717+0000", 2024-09-14T05:03:00.157 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.416420387 2024-09-14T05:03:00.157 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.157 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.157 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T05:03:00.157 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.899726+0000", 2024-09-14T05:03:00.158 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.7600000000000008e-06 2024-09-14T05:03:00.158 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.158 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.158 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.158 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:43.621312+0000", 2024-09-14T05:03:00.158 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.72158625600000004 2024-09-14T05:03:00.158 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.158 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.158 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for missing object", 2024-09-14T05:03:00.158 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:43.621439+0000", 2024-09-14T05:03:00.158 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012733300000000001 2024-09-14T05:03:00.158 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.158 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.159 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.159 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:43.628124+0000", 2024-09-14T05:03:00.159 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0066847989999999999 2024-09-14T05:03:00.159 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.159 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.159 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:00.159 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:43.628199+0000", 2024-09-14T05:03:00.159 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.5461000000000001e-05 2024-09-14T05:03:00.159 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.159 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:00.159 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.159 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.159 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.159 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:241659 57.13 57:cf6d98e1:::benchmark_data_smithi039_365565_object117907:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1559)", 2024-09-14T05:03:00.159 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:00:42.483305+0000", 2024-09-14T05:03:00.160 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 137.56648971300001, 2024-09-14T05:03:00.160 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1323830859999999, 2024-09-14T05:03:00.160 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:00.160 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:00.160 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T05:03:00.160 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:03:00.160 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T05:03:00.160 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:03:00.160 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 241659 2024-09-14T05:03:00.160 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.160 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:00.160 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.160 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:00.161 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483305+0000", 2024-09-14T05:03:00.161 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.161 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.161 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.161 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:00.161 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483305+0000", 2024-09-14T05:03:00.161 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.161 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.161 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.161 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:00.161 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483305+0000", 2024-09-14T05:03:00.161 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.7599999999999998e-07 2024-09-14T05:03:00.161 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.162 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.162 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:00.162 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483307+0000", 2024-09-14T05:03:00.162 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0040000000000002e-06 2024-09-14T05:03:00.162 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.162 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.162 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:00.162 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483350+0000", 2024-09-14T05:03:00.162 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2648000000000002e-05 2024-09-14T05:03:00.162 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.162 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.162 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:00.162 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483379+0000", 2024-09-14T05:03:00.162 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.9603000000000001e-05 2024-09-14T05:03:00.163 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.163 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.163 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.163 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.890896+0000", 2024-09-14T05:03:00.163 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.407517032 2024-09-14T05:03:00.163 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.163 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.163 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T05:03:00.163 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.890904+0000", 2024-09-14T05:03:00.163 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.2870000000000004e-06 2024-09-14T05:03:00.163 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.163 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.163 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.164 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:43.615610+0000", 2024-09-14T05:03:00.164 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.72470574499999996 2024-09-14T05:03:00.164 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.164 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.164 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:00.164 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:43.615688+0000", 2024-09-14T05:03:00.164 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7391000000000005e-05 2024-09-14T05:03:00.164 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.164 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:00.164 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.164 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.164 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.164 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:241661 57.0 57:0488aa79:::benchmark_data_smithi039_365565_object117909:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1559)", 2024-09-14T05:03:00.164 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:00:42.483389+0000", 2024-09-14T05:03:00.165 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 137.566404995, 2024-09-14T05:03:00.165 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1382475379999999, 2024-09-14T05:03:00.165 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:00.165 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:00.165 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T05:03:00.165 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:03:00.165 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T05:03:00.165 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:03:00.165 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 241661 2024-09-14T05:03:00.165 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.165 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:00.165 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.166 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:00.166 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483389+0000", 2024-09-14T05:03:00.166 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.166 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.167 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.167 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:00.167 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483389+0000", 2024-09-14T05:03:00.167 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.167 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.167 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.167 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:00.167 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483402+0000", 2024-09-14T05:03:00.167 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3076e-05 2024-09-14T05:03:00.167 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.167 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.167 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:00.167 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483434+0000", 2024-09-14T05:03:00.167 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.1424000000000003e-05 2024-09-14T05:03:00.168 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.168 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.168 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:00.168 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483477+0000", 2024-09-14T05:03:00.168 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3674999999999998e-05 2024-09-14T05:03:00.168 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.168 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.168 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:00.168 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483484+0000", 2024-09-14T05:03:00.168 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1029999999999998e-06 2024-09-14T05:03:00.169 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.169 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.169 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.169 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.899775+0000", 2024-09-14T05:03:00.169 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.41629152600000002 2024-09-14T05:03:00.169 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.169 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.169 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T05:03:00.169 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.899782+0000", 2024-09-14T05:03:00.169 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.3560000000000004e-06 2024-09-14T05:03:00.169 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.169 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.170 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.170 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:43.559697+0000", 2024-09-14T05:03:00.170 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.65991426099999995 2024-09-14T05:03:00.170 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.170 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.170 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for missing object", 2024-09-14T05:03:00.170 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:43.559817+0000", 2024-09-14T05:03:00.170 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000120541 2024-09-14T05:03:00.170 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.170 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.170 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.170 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:43.621568+0000", 2024-09-14T05:03:00.171 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.061750860999999997 2024-09-14T05:03:00.171 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.171 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.171 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:00.171 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:43.621637+0000", 2024-09-14T05:03:00.171 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8714999999999995e-05 2024-09-14T05:03:00.171 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.171 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:00.171 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.171 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.171 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.172 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:241667 57.13 57:cdc1a956:::benchmark_data_smithi039_365565_object117915:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1559)", 2024-09-14T05:03:00.172 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:00:42.483492+0000", 2024-09-14T05:03:00.172 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 137.56630212799999, 2024-09-14T05:03:00.172 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1385584070000001, 2024-09-14T05:03:00.172 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:00.172 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:00.172 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T05:03:00.172 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:03:00.172 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T05:03:00.172 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:03:00.172 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 241667 2024-09-14T05:03:00.172 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.172 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:00.173 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.173 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:00.173 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483492+0000", 2024-09-14T05:03:00.173 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.173 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.173 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.173 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:00.173 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483492+0000", 2024-09-14T05:03:00.173 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.173 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.174 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.174 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:00.174 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483493+0000", 2024-09-14T05:03:00.174 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2800000000000002e-07 2024-09-14T05:03:00.174 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.174 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.174 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:00.174 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483495+0000", 2024-09-14T05:03:00.174 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4490000000000002e-06 2024-09-14T05:03:00.174 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.174 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.174 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:00.174 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483501+0000", 2024-09-14T05:03:00.175 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6640000000000003e-06 2024-09-14T05:03:00.175 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.175 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.175 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:00.175 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483505+0000", 2024-09-14T05:03:00.175 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.702e-06 2024-09-14T05:03:00.175 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.175 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.175 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.175 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.890929+0000", 2024-09-14T05:03:00.175 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.40742362199999999 2024-09-14T05:03:00.176 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.176 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.176 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T05:03:00.176 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.890936+0000", 2024-09-14T05:03:00.176 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.2710000000000003e-06 2024-09-14T05:03:00.176 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.176 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.176 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.176 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:43.620813+0000", 2024-09-14T05:03:00.176 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.72987647099999997 2024-09-14T05:03:00.177 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.177 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.177 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:00.177 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:43.622051+0000", 2024-09-14T05:03:00.177 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0012378000000000001 2024-09-14T05:03:00.177 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.177 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:00.177 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.177 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.177 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.177 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop_reply(client.36324.0:241667 57.3 e1558/1518)", 2024-09-14T05:03:00.177 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:00:42.483527+0000", 2024-09-14T05:03:00.177 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 137.56626701600001, 2024-09-14T05:03:00.177 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.40711756900000001, 2024-09-14T05:03:00.178 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:00.178 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:00.178 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "reached pg", 2024-09-14T05:03:00.178 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:00.178 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.178 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:00.178 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483527+0000", 2024-09-14T05:03:00.178 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.178 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.178 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.178 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:00.178 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483527+0000", 2024-09-14T05:03:00.178 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.178 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.179 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.179 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:00.179 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483528+0000", 2024-09-14T05:03:00.179 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.9000000000000002e-07 2024-09-14T05:03:00.179 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.179 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.179 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:00.179 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483530+0000", 2024-09-14T05:03:00.179 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3180000000000001e-06 2024-09-14T05:03:00.179 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.179 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.179 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:00.179 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483537+0000", 2024-09-14T05:03:00.179 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.6359999999999997e-06 2024-09-14T05:03:00.180 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.180 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.180 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:00.180 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483541+0000", 2024-09-14T05:03:00.180 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6029999999999998e-06 2024-09-14T05:03:00.180 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.180 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.180 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.180 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.890632+0000", 2024-09-14T05:03:00.180 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.40709071400000002 2024-09-14T05:03:00.180 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.180 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.180 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:00.181 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.890645+0000", 2024-09-14T05:03:00.181 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2908000000000001e-05 2024-09-14T05:03:00.181 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.181 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:00.181 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.181 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.181 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.181 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36324.0:241671 57.7 57:e138eda8:::benchmark_data_smithi039_365565_object117919:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1559)", 2024-09-14T05:03:00.181 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:00:42.802286+0000", 2024-09-14T05:03:00.181 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 137.24750858600001, 2024-09-14T05:03:00.181 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.82210963800000003, 2024-09-14T05:03:00.181 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:00.181 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:00.181 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:03:00.182 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:03:00.182 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36324", 2024-09-14T05:03:00.182 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:03:00.182 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 241671 2024-09-14T05:03:00.182 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.182 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:00.182 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.182 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:00.182 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.802286+0000", 2024-09-14T05:03:00.182 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.183 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.183 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.183 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:00.183 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.802286+0000", 2024-09-14T05:03:00.183 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:00.183 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.183 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.183 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:00.183 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.802287+0000", 2024-09-14T05:03:00.184 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.782e-06 2024-09-14T05:03:00.184 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.184 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.184 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:00.184 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.802293+0000", 2024-09-14T05:03:00.184 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3090000000000003e-06 2024-09-14T05:03:00.184 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.184 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.184 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:00.185 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.802305+0000", 2024-09-14T05:03:00.185 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1996e-05 2024-09-14T05:03:00.185 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.185 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.185 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:00.185 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.802315+0000", 2024-09-14T05:03:00.185 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0164e-05 2024-09-14T05:03:00.185 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.185 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.185 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.186 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.864372+0000", 2024-09-14T05:03:00.186 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.062057358999999999 2024-09-14T05:03:00.186 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.186 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.186 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T05:03:00.186 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.864387+0000", 2024-09-14T05:03:00.186 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4598e-05 2024-09-14T05:03:00.186 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.186 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.186 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:00.186 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:43.538241+0000", 2024-09-14T05:03:00.186 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.67385381200000005 2024-09-14T05:03:00.186 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.186 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.187 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:00.187 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:43.538373+0000", 2024-09-14T05:03:00.187 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013259400000000001 2024-09-14T05:03:00.187 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.187 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.187 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T05:03:00.187 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:43.538833+0000", 2024-09-14T05:03:00.187 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00045992599999999999 2024-09-14T05:03:00.187 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.187 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.187 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:03:00.187 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:43.548450+0000", 2024-09-14T05:03:00.187 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0096169210000000005 2024-09-14T05:03:00.187 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.188 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.188 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:03:00.188 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:43.549143+0000", 2024-09-14T05:03:00.188 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00069294300000000001 2024-09-14T05:03:00.188 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.188 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.188 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:03:00.188 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:43.624225+0000", 2024-09-14T05:03:00.188 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.075082323000000006 2024-09-14T05:03:00.188 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.188 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.188 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:03:00.188 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:43.624285+0000", 2024-09-14T05:03:00.189 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.9901999999999997e-05 2024-09-14T05:03:00.189 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:00.189 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:00.189 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:00.189 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:43.624395+0000", 2024-09-14T05:03:00.189 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000110009 2024-09-14T05:03:00.189 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.189 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:00.189 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.189 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:00.189 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:00.189 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T05:03:00.220 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:00.264 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:00.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:00.266 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:00.366 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:00.393 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:00.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:00.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:00.495 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:00.521 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:00.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:00.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:00.623 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:00.650 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:00.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:00.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:00.688 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:00 smithi039 ceph-mon[106470]: 1.1a deep-scrub starts 2024-09-14T05:03:00.688 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:00 smithi039 ceph-mon[106470]: 1.1a deep-scrub ok 2024-09-14T05:03:00.688 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:00 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"} : dispatch 2024-09-14T05:03:00.688 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:00 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"} : dispatch 2024-09-14T05:03:00.689 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:00 smithi039 ceph-mon[111235]: 1.1a deep-scrub starts 2024-09-14T05:03:00.689 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:00 smithi039 ceph-mon[111235]: 1.1a deep-scrub ok 2024-09-14T05:03:00.689 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:00 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"} : dispatch 2024-09-14T05:03:00.689 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:00 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"} : dispatch 2024-09-14T05:03:00.752 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:00 smithi129 ceph-mon[101316]: 1.1a deep-scrub starts 2024-09-14T05:03:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:00 smithi129 ceph-mon[101316]: 1.1a deep-scrub ok 2024-09-14T05:03:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:00 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"} : dispatch 2024-09-14T05:03:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:00 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"} : dispatch 2024-09-14T05:03:00.779 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:00.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:00.780 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:00.881 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:00.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:00.910 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:00.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:01.011 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:01.054 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:01.055 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:01.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:01.156 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:01.185 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:01.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:01.186 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:01.286 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:01.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:01.348 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:01.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:01.431 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T05:03:01.449 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:01.528 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:01.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:01.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:01.539 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:01 smithi039 ceph-mon[106470]: pgmap v3154: 119 pgs: 119 active+clean; 6.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2024-09-14T05:03:01.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:01 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2024-09-14T05:03:01.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:01 smithi039 ceph-mon[106470]: osdmap e1584: 8 total, 8 up, 8 in 2024-09-14T05:03:01.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:01 smithi039 ceph-mon[106470]: osdmap e1585: 8 total, 8 up, 8 in 2024-09-14T05:03:01.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:01 smithi039 ceph-mon[111235]: pgmap v3154: 119 pgs: 119 active+clean; 6.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2024-09-14T05:03:01.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:01 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2024-09-14T05:03:01.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:01 smithi039 ceph-mon[111235]: osdmap e1584: 8 total, 8 up, 8 in 2024-09-14T05:03:01.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:01 smithi039 ceph-mon[111235]: osdmap e1585: 8 total, 8 up, 8 in 2024-09-14T05:03:01.630 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:01.641 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-14T05:03:01.654 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:01.655 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:01.655 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:01.756 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:01.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:01 smithi129 ceph-mon[101316]: pgmap v3154: 119 pgs: 119 active+clean; 6.5 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 918 op/s 2024-09-14T05:03:01.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:01 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2024-09-14T05:03:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:01 smithi129 ceph-mon[101316]: osdmap e1584: 8 total, 8 up, 8 in 2024-09-14T05:03:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:01 smithi129 ceph-mon[101316]: osdmap e1585: 8 total, 8 up, 8 in 2024-09-14T05:03:01.779 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:01.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:01.780 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:01.881 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:01.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:01.978 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:01.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:02.079 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:02.122 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:02.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:02.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:02.224 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:02.246 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:02.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:02.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:02.298 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:03:02.309 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:03:02.347 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:02.370 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:02.371 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:02.372 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:02.389 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:03:02.473 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:02.477 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:02 smithi039 ceph-mon[106470]: 1.d scrub starts 2024-09-14T05:03:02.477 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:02 smithi039 ceph-mon[106470]: 1.d scrub ok 2024-09-14T05:03:02.477 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:02 smithi039 ceph-mon[106470]: osdmap e1586: 8 total, 8 up, 8 in 2024-09-14T05:03:02.477 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:02 smithi039 ceph-mon[111235]: 1.d scrub starts 2024-09-14T05:03:02.477 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:02 smithi039 ceph-mon[111235]: 1.d scrub ok 2024-09-14T05:03:02.477 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:02 smithi039 ceph-mon[111235]: osdmap e1586: 8 total, 8 up, 8 in 2024-09-14T05:03:02.496 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:02.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:02.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:02.598 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:02.639 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:02.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:02.640 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:02.741 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:02.763 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:02.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:02.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:02.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:02 smithi129 ceph-mon[101316]: 1.d scrub starts 2024-09-14T05:03:02.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:02 smithi129 ceph-mon[101316]: 1.d scrub ok 2024-09-14T05:03:02.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:02 smithi129 ceph-mon[101316]: osdmap e1586: 8 total, 8 up, 8 in 2024-09-14T05:03:02.865 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:02.910 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:02.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:02.911 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:03.012 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:03.048 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:03.049 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:03.049 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:03.150 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:03.176 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:03.177 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:03.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:03.270 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is set 2024-09-14T05:03:03.278 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:03.304 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:03.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:03.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:03.407 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:03.437 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:03.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:03.438 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:03.539 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:03.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:03 smithi039 ceph-mon[106470]: pgmap v3157: 119 pgs: 119 active+clean; 6.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2024-09-14T05:03:03.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:03 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/64453919' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:03:03.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:03 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:03:03.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:03 smithi039 ceph-mon[106470]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:03:03.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:03 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T05:03:03.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:03 smithi039 ceph-mon[106470]: osdmap e1587: 8 total, 8 up, 8 in 2024-09-14T05:03:03.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:03 smithi039 ceph-mon[111235]: pgmap v3157: 119 pgs: 119 active+clean; 6.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2024-09-14T05:03:03.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:03 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/64453919' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:03:03.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:03 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:03:03.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:03 smithi039 ceph-mon[111235]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:03:03.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:03 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T05:03:03.542 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:03 smithi039 ceph-mon[111235]: osdmap e1587: 8 total, 8 up, 8 in 2024-09-14T05:03:03.567 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:03.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:03.568 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:03.669 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:03.692 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:03.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:03.693 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:03.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:03 smithi129 ceph-mon[101316]: pgmap v3157: 119 pgs: 119 active+clean; 6.4 GiB data, 26 GiB used, 689 GiB / 715 GiB avail; 0 B/s wr, 855 op/s 2024-09-14T05:03:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:03 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/64453919' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:03:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:03 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:03:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:03 smithi129 ceph-mon[101316]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:03:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:03 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T05:03:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:03 smithi129 ceph-mon[101316]: osdmap e1587: 8 total, 8 up, 8 in 2024-09-14T05:03:03.794 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:03.819 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:03.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:03.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:03.921 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:03.964 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:03.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:03.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:04.066 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:04.103 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:04.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:04.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:04.205 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:04.234 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:04.235 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:04.235 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:04.336 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:04.362 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:04.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:04.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:04.464 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:04.503 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:04.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:04.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:04.605 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:04.641 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:04.642 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:04.642 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:04.743 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:04.780 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:04.780 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:04.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:04.882 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:04.925 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:04.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:04.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:05.027 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:05.070 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:05.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:05.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:05.172 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:05.201 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:05.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:05.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:05.303 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:05.349 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:05.350 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:05.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:05.451 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:05.474 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:05.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:05.475 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:05.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:05 smithi039 ceph-mon[106470]: pgmap v3160: 119 pgs: 1 peering, 1 active+undersized+remapped, 2 remapped+peering, 115 active+clean; 6.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 14 KiB/s rd, 41 KiB/s wr, 695 op/s; 29 KiB/s, 0 objects/s recovering 2024-09-14T05:03:05.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:05 smithi039 ceph-mon[106470]: osdmap e1588: 8 total, 8 up, 8 in 2024-09-14T05:03:05.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:05 smithi039 ceph-mon[106470]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-14T05:03:05.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:05 smithi039 ceph-mon[106470]: 1.1d scrub starts 2024-09-14T05:03:05.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:05 smithi039 ceph-mon[106470]: 1.19 scrub starts 2024-09-14T05:03:05.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:05 smithi039 ceph-mon[111235]: pgmap v3160: 119 pgs: 1 peering, 1 active+undersized+remapped, 2 remapped+peering, 115 active+clean; 6.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 14 KiB/s rd, 41 KiB/s wr, 695 op/s; 29 KiB/s, 0 objects/s recovering 2024-09-14T05:03:05.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:05 smithi039 ceph-mon[111235]: osdmap e1588: 8 total, 8 up, 8 in 2024-09-14T05:03:05.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:05 smithi039 ceph-mon[111235]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-14T05:03:05.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:05 smithi039 ceph-mon[111235]: 1.1d scrub starts 2024-09-14T05:03:05.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:05 smithi039 ceph-mon[111235]: 1.19 scrub starts 2024-09-14T05:03:05.576 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:05.622 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:05.623 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:05.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:05.724 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:05.749 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:05.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:05.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:05.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:05 smithi129 ceph-mon[101316]: pgmap v3160: 119 pgs: 1 peering, 1 active+undersized+remapped, 2 remapped+peering, 115 active+clean; 6.3 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 14 KiB/s rd, 41 KiB/s wr, 695 op/s; 29 KiB/s, 0 objects/s recovering 2024-09-14T05:03:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:05 smithi129 ceph-mon[101316]: osdmap e1588: 8 total, 8 up, 8 in 2024-09-14T05:03:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:05 smithi129 ceph-mon[101316]: Health check failed: Reduced data availability: 3 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-14T05:03:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:05 smithi129 ceph-mon[101316]: 1.1d scrub starts 2024-09-14T05:03:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:05 smithi129 ceph-mon[101316]: 1.19 scrub starts 2024-09-14T05:03:05.851 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:05.875 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:05.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:05.876 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:05.977 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:06.000 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:06.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:06.001 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:06.101 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:06.130 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:06.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:06.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:06.231 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:06.258 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:06.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:06.260 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:06.360 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:06.410 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:06.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:06.411 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:06.512 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:06.535 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:06.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:06.537 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:06.539 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:06 smithi039 ceph-mon[106470]: 1.16 scrub starts 2024-09-14T05:03:06.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:06 smithi039 ceph-mon[111235]: 1.16 scrub starts 2024-09-14T05:03:06.637 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:06.663 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:06.663 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:06.664 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:06.765 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:06.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:06 smithi129 ceph-mon[101316]: 1.16 scrub starts 2024-09-14T05:03:06.811 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:06.812 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:06.812 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:06.913 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:06.954 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:06.955 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:06.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:07.056 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:07.100 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:07.101 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:07.101 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:07.202 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:07.231 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T05:03:07.231 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [ 2024-09-14T05:03:07.231 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:07.231 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36603.0:132998 58.7 e1588/1568 58:eb134d34:::benchmark_data_smithi039_396950_object17385:head v 1588'8366, mlcod=1588'8364)", 2024-09-14T05:03:07.232 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:03:07.224114+0000", 2024-09-14T05:03:07.232 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.0062980700000000002, 2024-09-14T05:03:07.232 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.006325387, 2024-09-14T05:03:07.232 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:07.232 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:07.232 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "started", 2024-09-14T05:03:07.232 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:07.232 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:07.232 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:07.232 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:07.224114+0000", 2024-09-14T05:03:07.232 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:07.232 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:07.232 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:07.233 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:07.233 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:07.224114+0000", 2024-09-14T05:03:07.233 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:07.233 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:07.233 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:07.233 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:07.233 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:07.224115+0000", 2024-09-14T05:03:07.233 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5900000000000002e-07 2024-09-14T05:03:07.233 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:07.233 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:07.233 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:07.233 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:07.224118+0000", 2024-09-14T05:03:07.233 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.9040000000000002e-06 2024-09-14T05:03:07.233 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:07.234 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:07.234 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:07.234 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:07.224126+0000", 2024-09-14T05:03:07.234 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.5619999999999998e-06 2024-09-14T05:03:07.234 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:07.234 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:07.234 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:07.234 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:07.224131+0000", 2024-09-14T05:03:07.234 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.2440000000000001e-06 2024-09-14T05:03:07.234 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:07.234 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:07.234 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:07.235 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:07.224196+0000", 2024-09-14T05:03:07.235 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.4389999999999998e-05 2024-09-14T05:03:07.235 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:07.235 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:07.235 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:07.235 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:07.224213+0000", 2024-09-14T05:03:07.235 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7719000000000001e-05 2024-09-14T05:03:07.235 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:07.235 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:07.235 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:07.235 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:07.235 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:07.235 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36603.0:133014 58.6 e1588/1568 58:60357027:::benchmark_data_smithi039_396950_object17401:head v 1588'8456, mlcod=1588'8454)", 2024-09-14T05:03:07.235 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:03:07.228104+0000", 2024-09-14T05:03:07.236 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.002308507, 2024-09-14T05:03:07.236 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0023988220000000001, 2024-09-14T05:03:07.236 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:07.236 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:07.236 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "started", 2024-09-14T05:03:07.236 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:07.236 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:07.236 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:07.236 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:07.228104+0000", 2024-09-14T05:03:07.236 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:07.236 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:07.236 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:07.236 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:07.237 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:07.228104+0000", 2024-09-14T05:03:07.237 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:07.237 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:07.237 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:07.237 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:07.237 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:07.228105+0000", 2024-09-14T05:03:07.237 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.127e-06 2024-09-14T05:03:07.237 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:07.237 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:07.237 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:07.237 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:07.228110+0000", 2024-09-14T05:03:07.237 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.2669999999999997e-06 2024-09-14T05:03:07.238 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:07.238 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:07.238 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:07.238 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:07.228125+0000", 2024-09-14T05:03:07.238 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.4921e-05 2024-09-14T05:03:07.238 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:07.238 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:07.238 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:07.238 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:07.228132+0000", 2024-09-14T05:03:07.238 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.4710000000000001e-06 2024-09-14T05:03:07.238 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:07.238 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:07.238 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:07.239 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:07.228250+0000", 2024-09-14T05:03:07.239 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011722000000000001 2024-09-14T05:03:07.239 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:07.239 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:07.239 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:07.239 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:07.228269+0000", 2024-09-14T05:03:07.239 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8995e-05 2024-09-14T05:03:07.239 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:07.239 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:07.239 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:07.239 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:07.239 INFO:teuthology.orchestra.run.smithi039.stdout: ], 2024-09-14T05:03:07.239 INFO:teuthology.orchestra.run.smithi039.stdout: "num_ops": 2 2024-09-14T05:03:07.239 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T05:03:07.241 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:07.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:07.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:07.343 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:07.368 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:07.369 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:07.369 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:07.470 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:07.511 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:07.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:07.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:07.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:07 smithi039 ceph-mon[106470]: pgmap v3162: 119 pgs: 1 peering, 1 active+undersized+remapped, 2 remapped+peering, 115 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 24 KiB/s rd, 69 KiB/s wr, 936 op/s; 49 KiB/s, 0 objects/s recovering 2024-09-14T05:03:07.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:07 smithi039 ceph-mon[111235]: pgmap v3162: 119 pgs: 1 peering, 1 active+undersized+remapped, 2 remapped+peering, 115 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 24 KiB/s rd, 69 KiB/s wr, 936 op/s; 49 KiB/s, 0 objects/s recovering 2024-09-14T05:03:07.613 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:07.640 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:07.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:07.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:07.742 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:07.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:07 smithi129 ceph-mon[101316]: pgmap v3162: 119 pgs: 1 peering, 1 active+undersized+remapped, 2 remapped+peering, 115 active+clean; 6.2 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 24 KiB/s rd, 69 KiB/s wr, 936 op/s; 49 KiB/s, 0 objects/s recovering 2024-09-14T05:03:07.786 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:07.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:07.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:07.888 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:07.914 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:07.915 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:07.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:08.016 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:08.060 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:08.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:08.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:08.162 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:08.201 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:08.201 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:08.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:08.212 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T05:03:08.303 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:08.330 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:08.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:08.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:08.432 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:08.503 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:08.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:08.504 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:08.605 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:08.637 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:08 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:03:08.637 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:08 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:03:08.637 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:08 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:03:08.637 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:08 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:03:08.663 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:08.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:08.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:08.765 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:08.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:08 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:03:08.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:08 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:03:08.813 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:08.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:08.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:08.915 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:08.940 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:08.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:08.941 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:08.982 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:03:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:03:08] "GET /metrics HTTP/1.1" 200 36768 "" "Prometheus/2.51.0" 2024-09-14T05:03:09.042 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:09.067 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:09.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:09.068 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:09.169 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:09.183 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T05:03:09.184 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T05:03:09.196 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:09.196 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:09.197 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:09.216 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T05:03:09.217 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T05:03:09.264 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T05:03:09.264 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T05:03:09.298 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:09.301 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T05:03:09.301 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T05:03:09.321 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:09.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:09.323 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:09.331 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T05:03:09.332 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T05:03:09.371 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T05:03:09.372 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T05:03:09.410 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T05:03:09.410 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T05:03:09.424 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:09.446 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T05:03:09.446 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T05:03:09.475 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:09.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:09.477 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:09.546 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:09 smithi039 ceph-mon[106470]: pgmap v3163: 119 pgs: 1 peering, 1 remapped+peering, 117 active+clean; 6.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 797 op/s; 39 KiB/s, 0 objects/s recovering 2024-09-14T05:03:09.547 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:09 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4155283469' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:03:09.547 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:09 smithi039 ceph-mon[111235]: pgmap v3163: 119 pgs: 1 peering, 1 remapped+peering, 117 active+clean; 6.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 797 op/s; 39 KiB/s, 0 objects/s recovering 2024-09-14T05:03:09.547 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:09 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4155283469' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:03:09.577 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:09.583 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-14T05:03:09.619 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:09.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:09.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:09.721 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:09.765 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:09.765 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:09.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:09.781 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:09 smithi129 ceph-mon[101316]: pgmap v3163: 119 pgs: 1 peering, 1 remapped+peering, 117 active+clean; 6.1 GiB data, 25 GiB used, 690 GiB / 715 GiB avail; 19 KiB/s rd, 55 KiB/s wr, 797 op/s; 39 KiB/s, 0 objects/s recovering 2024-09-14T05:03:09.781 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:09 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4155283469' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:03:09.867 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:09.912 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:09.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:09.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:10.014 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:10.056 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:10.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:10.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:10.158 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:10.181 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:10.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:10.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:10.283 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:10.323 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:10.324 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:10.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:10.425 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:10.452 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:10.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:10.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:10.554 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:10.591 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:10.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:10.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:10.693 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:10.734 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:10.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:10.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:10.769 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:10 smithi129 ceph-mon[101316]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-09-14T05:03:10.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:10 smithi039 ceph-mon[106470]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-09-14T05:03:10.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:10 smithi039 ceph-mon[111235]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 2 pgs peering) 2024-09-14T05:03:10.835 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:10.876 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:10.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:10.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:10.977 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:11.001 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:11.001 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:11.002 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:11.102 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:11.143 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:11.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:11.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:11.245 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:11.271 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:11.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:11.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:11.373 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:11.398 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:11.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:11.399 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:11.499 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:11.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:11.536 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:11.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:11.637 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:11.681 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:11.682 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:11.682 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:11.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:11 smithi129 ceph-mon[101316]: pgmap v3164: 119 pgs: 2 active+clean+scrubbing, 117 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 15 KiB/s rd, 42 KiB/s wr, 1.02k op/s; 30 KiB/s, 0 objects/s recovering 2024-09-14T05:03:11.783 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:11.789 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:11 smithi039 ceph-mon[111235]: pgmap v3164: 119 pgs: 2 active+clean+scrubbing, 117 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 15 KiB/s rd, 42 KiB/s wr, 1.02k op/s; 30 KiB/s, 0 objects/s recovering 2024-09-14T05:03:11.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:11 smithi039 ceph-mon[106470]: pgmap v3164: 119 pgs: 2 active+clean+scrubbing, 117 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 15 KiB/s rd, 42 KiB/s wr, 1.02k op/s; 30 KiB/s, 0 objects/s recovering 2024-09-14T05:03:11.807 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:11.808 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:11.808 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:11.909 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:11.933 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:11.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:11.934 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:12.035 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:12.068 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:12.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:12.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:12.170 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:12.197 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:12.198 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:12.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:12.299 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:12.327 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:12.328 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:12.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:12.429 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:12.464 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:12.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:12.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:12.566 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:12.600 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:12.600 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:12.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:12.701 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:12.727 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:12.727 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:12.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:12.829 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:12.852 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:12.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:12.853 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:12.954 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:12.998 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:12.999 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:12.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:13.100 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:13.132 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:13.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:13.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:13.234 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:13.260 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:13.260 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:13.260 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:13.361 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:13.405 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:13.406 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:13.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:13.507 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:13.542 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:13.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:13.543 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:13.644 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:13.671 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:13.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:13.672 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:13.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:13 smithi129 ceph-mon[101316]: pgmap v3165: 119 pgs: 2 active+clean+scrubbing, 117 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 11 KiB/s rd, 31 KiB/s wr, 941 op/s; 22 KiB/s, 0 objects/s recovering 2024-09-14T05:03:13.773 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:13.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:13 smithi039 ceph-mon[106470]: pgmap v3165: 119 pgs: 2 active+clean+scrubbing, 117 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 11 KiB/s rd, 31 KiB/s wr, 941 op/s; 22 KiB/s, 0 objects/s recovering 2024-09-14T05:03:13.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:13 smithi039 ceph-mon[111235]: pgmap v3165: 119 pgs: 2 active+clean+scrubbing, 117 active+clean; 5.9 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 11 KiB/s rd, 31 KiB/s wr, 941 op/s; 22 KiB/s, 0 objects/s recovering 2024-09-14T05:03:13.796 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:13.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:13.797 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:13.897 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:13.925 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:13.925 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:13.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:14.026 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:14.074 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:14.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:14.075 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:14.176 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:14.225 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:14.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:14.226 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:14.299 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:03:14.327 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:14.337 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T05:03:14.338 INFO:teuthology.orchestra.run.smithi039.stdout: "success": "filestore_inject_stall = '' (not observed, change may require restart) " 2024-09-14T05:03:14.338 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T05:03:14.379 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:14.380 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:14.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:14.481 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:14.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:14.509 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:14.509 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:14.610 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:14.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:14.648 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:14.648 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:14.749 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:14.789 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:14.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:14.790 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:14.891 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:14.920 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:14.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:14.921 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:15.022 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:15.047 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:15.048 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:15.048 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:15.123 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:03:15.149 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:15.194 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:15.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:15.195 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:15.295 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:15.320 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:15.321 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:15.321 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:15.422 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:15.448 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:15.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:15.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:15.550 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:15.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:15.594 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:15.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:15.684 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:15 smithi039 ceph-mon[106470]: pgmap v3166: 119 pgs: 2 active+clean+scrubbing, 117 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 5.7 KiB/s rd, 16 KiB/s wr, 907 op/s; 12 KiB/s, 0 objects/s recovering 2024-09-14T05:03:15.685 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:15 smithi039 ceph-mon[111235]: pgmap v3166: 119 pgs: 2 active+clean+scrubbing, 117 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 5.7 KiB/s rd, 16 KiB/s wr, 907 op/s; 12 KiB/s, 0 objects/s recovering 2024-09-14T05:03:15.695 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:15.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:15.743 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:15.744 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:15.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:15 smithi129 ceph-mon[101316]: pgmap v3166: 119 pgs: 2 active+clean+scrubbing, 117 active+clean; 5.8 GiB data, 24 GiB used, 691 GiB / 715 GiB avail; 5.7 KiB/s rd, 16 KiB/s wr, 907 op/s; 12 KiB/s, 0 objects/s recovering 2024-09-14T05:03:15.845 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:15.866 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T05:03:15.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:15.869 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:15.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:15.971 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:16.026 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:16.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:16.027 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:16.128 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:16.155 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:16.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:16.156 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:16.257 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:16.299 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:16.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:16.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:16.401 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:16.424 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:16.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:16.425 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:16.526 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:16.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:16.567 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:16.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:16.668 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:16.711 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:16.712 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:16.713 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:16.814 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:16.837 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:16.837 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:16.838 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:16.939 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:16.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:16.964 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:16.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:17.065 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:17.088 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:17.088 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:17.089 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:17.189 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:17.234 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:17.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:17.235 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:17.336 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:17.386 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:17.387 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:17.387 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:17.488 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:17.513 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:17.513 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:17.514 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:17.539 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:17 smithi039 ceph-mon[111235]: pgmap v3167: 119 pgs: 2 active+clean+scrubbing, 117 active+clean; 5.6 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 4.8 KiB/s rd, 14 KiB/s wr, 1.00k op/s; 10 KiB/s, 0 objects/s recovering 2024-09-14T05:03:17.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:17 smithi039 ceph-mon[106470]: pgmap v3167: 119 pgs: 2 active+clean+scrubbing, 117 active+clean; 5.6 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 4.8 KiB/s rd, 14 KiB/s wr, 1.00k op/s; 10 KiB/s, 0 objects/s recovering 2024-09-14T05:03:17.614 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:17.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:17.640 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:17.640 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:17.741 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:17.767 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:17.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:17.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:17.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:17 smithi129 ceph-mon[101316]: pgmap v3167: 119 pgs: 2 active+clean+scrubbing, 117 active+clean; 5.6 GiB data, 24 GiB used, 692 GiB / 715 GiB avail; 4.8 KiB/s rd, 14 KiB/s wr, 1.00k op/s; 10 KiB/s, 0 objects/s recovering 2024-09-14T05:03:17.869 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:17.896 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:17.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:17.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:17.998 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:18.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:18.022 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:18.022 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:18.122 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:18.159 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:18.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:18.160 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:18.261 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:18.284 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:18.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:18.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:18.386 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:18.424 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:18.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:18.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:18.526 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:18.553 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:18.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:18.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:18.654 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:18.696 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:18.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:18.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:18.798 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:18.843 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:18.843 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:18.844 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:18.945 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:18.970 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:18.970 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:18.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:19.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:03:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:03:18] "GET /metrics HTTP/1.1" 200 36768 "" "Prometheus/2.51.0" 2024-09-14T05:03:19.071 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:19.130 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:19.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:19.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:19.148 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:03:19.232 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:19.280 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:19.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:19.281 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:19.382 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:19.432 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:19.432 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:19.433 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:19.534 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:19.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:19.561 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:19.561 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:19.662 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:19.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:19.699 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:19.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:19.723 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:19 smithi039 ceph-mon[106470]: pgmap v3168: 119 pgs: 2 active+clean+scrubbing, 117 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 774 op/s 2024-09-14T05:03:19.724 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:19 smithi039 ceph-mon[111235]: pgmap v3168: 119 pgs: 2 active+clean+scrubbing, 117 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 774 op/s 2024-09-14T05:03:19.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:19 smithi129 ceph-mon[101316]: pgmap v3168: 119 pgs: 2 active+clean+scrubbing, 117 active+clean; 5.6 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 774 op/s 2024-09-14T05:03:19.801 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:19.841 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:19.841 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:19.842 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:19.942 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:19.985 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:19.985 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:19.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:20.087 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:20.115 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:20.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:20.116 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:20.136 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-14T05:03:20.136 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T05:03:20.136 INFO:tasks.thrashosds.thrasher:Removing osd 4, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2024-09-14T05:03:20.137 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd out 4 2024-09-14T05:03:20.216 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:20.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:20.254 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:20.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:20.355 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:20.378 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:20.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:20.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:20.479 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:20.507 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:20.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:20.508 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:20.609 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:20.639 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:20.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:20.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:20.741 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:20.769 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:20.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:20.770 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:20.871 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:20.916 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:20.916 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:20.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:21.017 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:21.060 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:21.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:21.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:21.162 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:21.189 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:21.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:21.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:21.291 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:21.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:21.317 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:21.317 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:21.418 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:21.447 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:21.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:21.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:21.549 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:21.574 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:21.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:21.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:21.675 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:21.703 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:21.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:21.704 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:21.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:21 smithi129 ceph-mon[101316]: pgmap v3169: 119 pgs: 2 active+clean+scrubbing, 117 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 989 op/s 2024-09-14T05:03:21.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:21 smithi039 ceph-mon[106470]: pgmap v3169: 119 pgs: 2 active+clean+scrubbing, 117 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 989 op/s 2024-09-14T05:03:21.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:21 smithi039 ceph-mon[111235]: pgmap v3169: 119 pgs: 2 active+clean+scrubbing, 117 active+clean; 5.4 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 989 op/s 2024-09-14T05:03:21.805 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:21.828 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:21.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:21.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:21.864 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T05:03:21.864 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:03:21.930 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:21.956 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:21.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:21.957 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:22.058 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:22.090 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:22.090 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:22.091 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:22.192 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:22.237 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:22.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:22.239 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:22.339 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:22.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:22.366 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:22.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:22.468 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:22.492 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:22.493 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:22.494 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:22.594 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:22.619 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:22.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:22.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:22.721 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:22.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:22.757 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:22.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:22.858 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:22.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:22.893 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:22.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:22.994 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:23.032 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:23.033 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:23.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:23.134 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:23.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:23.162 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:23.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:23.263 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:23.290 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:23.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:23.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:23.392 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:23.435 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:23.436 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:23.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:23.478 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is set 2024-09-14T05:03:23.537 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:23.559 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:23.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:23.560 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:23.661 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:23.683 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:23.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:23.684 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:23 smithi129 ceph-mon[101316]: pgmap v3170: 119 pgs: 2 active+clean+scrubbing, 117 active+clean; 5.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 811 op/s 2024-09-14T05:03:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:23 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:03:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:03:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:23 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3453632507' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:03:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:23 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:03:23.785 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:23.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:23 smithi039 ceph-mon[106470]: pgmap v3170: 119 pgs: 2 active+clean+scrubbing, 117 active+clean; 5.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 811 op/s 2024-09-14T05:03:23.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:23 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:03:23.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:03:23.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:23 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3453632507' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:03:23.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:23 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:03:23.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:23 smithi039 ceph-mon[111235]: pgmap v3170: 119 pgs: 2 active+clean+scrubbing, 117 active+clean; 5.3 GiB data, 23 GiB used, 692 GiB / 715 GiB avail; 0 B/s wr, 811 op/s 2024-09-14T05:03:23.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:23 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:03:23.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:03:23.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:23 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3453632507' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:03:23.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:23 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:03:23.808 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:23.809 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:23.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:23.910 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:23.936 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:23.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:23.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:24.039 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:24.063 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:24.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:24.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:24.165 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:24.195 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:24.195 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:24.196 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:24.296 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:24.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:24.340 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:24.340 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:24.388 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-14T05:03:24.441 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:24.464 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:24.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:24.465 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:24.568 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:24.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:24.618 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:24.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:24.719 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:24.746 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:24.747 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:24.747 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:24.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:24 smithi129 ceph-mon[101316]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:03:24.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:24 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T05:03:24.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:24 smithi129 ceph-mon[101316]: osdmap e1589: 8 total, 8 up, 8 in 2024-09-14T05:03:24.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:24 smithi039 ceph-mon[106470]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:03:24.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:24 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T05:03:24.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:24 smithi039 ceph-mon[106470]: osdmap e1589: 8 total, 8 up, 8 in 2024-09-14T05:03:24.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:24 smithi039 ceph-mon[111235]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:03:24.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:24 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T05:03:24.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:24 smithi039 ceph-mon[111235]: osdmap e1589: 8 total, 8 up, 8 in 2024-09-14T05:03:24.848 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:24.884 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:24.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:24.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:24.986 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:25.050 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:25.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:25.051 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:25.152 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:25.194 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:25.195 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:25.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:25.296 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:25.321 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:25.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:25.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:25.423 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:25.451 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:25.452 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:25.452 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:25.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:25 smithi039 ceph-mon[111235]: pgmap v3172: 119 pgs: 2 active+clean+scrubbing, 117 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 858 op/s 2024-09-14T05:03:25.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:25 smithi039 ceph-mon[106470]: pgmap v3172: 119 pgs: 2 active+clean+scrubbing, 117 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 858 op/s 2024-09-14T05:03:25.553 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:25.572 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:03:25.632 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:25.633 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:25.633 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:25.734 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:25.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:25 smithi129 ceph-mon[101316]: pgmap v3172: 119 pgs: 2 active+clean+scrubbing, 117 active+clean; 5.2 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 858 op/s 2024-09-14T05:03:25.787 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:25.787 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:25.788 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:25.889 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:25.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:25.944 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:25.945 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:26.045 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:26.068 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:26.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:26.069 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:26.170 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:26.216 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:26.217 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:26.217 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:26.318 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:26.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:26.357 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:26.357 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:26.458 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:26.483 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:26.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:26.484 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:26.490 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T05:03:26.490 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T05:03:26.537 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T05:03:26.537 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T05:03:26.573 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T05:03:26.574 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T05:03:26.585 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:26.611 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T05:03:26.612 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T05:03:26.626 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:26.627 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:26.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:26.641 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T05:03:26.642 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T05:03:26.676 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T05:03:26.677 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T05:03:26.713 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T05:03:26.714 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T05:03:26.727 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:26 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/632082007' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:03:26.727 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:26 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/632082007' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:03:26.727 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:26.749 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T05:03:26.749 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T05:03:26.751 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:26.752 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:26.752 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:26.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:26 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/632082007' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:03:26.853 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:26.877 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:26.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:26.878 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:26.978 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:27.021 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:27.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:27.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:27.123 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:27.165 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:27.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:27.166 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:27.267 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:27.294 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:27.295 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:27.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:27.396 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:27.438 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:27.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:27.439 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:27.540 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:27.563 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:27.564 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:27.565 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:27.620 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T05:03:27.666 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:27.699 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:03:27.703 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:27 smithi039 ceph-mon[106470]: pgmap v3173: 119 pgs: 119 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 827 op/s 2024-09-14T05:03:27.703 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:27 smithi039 ceph-mon[111235]: pgmap v3173: 119 pgs: 119 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 827 op/s 2024-09-14T05:03:27.720 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:27.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:27.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:27.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:27 smithi129 ceph-mon[101316]: pgmap v3173: 119 pgs: 119 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 827 op/s 2024-09-14T05:03:27.822 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:27.845 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:27.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:27.849 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:27.950 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:27.994 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:27.995 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:27.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:28.096 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:28.140 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:28.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:28.141 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:28.241 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:28.268 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:28.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:28.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:28.370 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:28.393 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:28.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:28.394 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:28.495 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:28.537 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:28.538 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:28.538 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:28.639 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:28.683 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:28.684 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:28.684 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:28.777 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:03:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:03:28] "GET /metrics HTTP/1.1" 200 36758 "" "Prometheus/2.51.0" 2024-09-14T05:03:28.785 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:28.835 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:28.836 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:28.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:28.937 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:28.985 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:28.986 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:28.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:29.088 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:29.111 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:29.112 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:29.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:29.213 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:29.236 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:29.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:29.237 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:29.338 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:29.404 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:29.405 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:29.405 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:29.506 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:29.516 INFO:teuthology.orchestra.run.smithi039.stderr:marked out osd.4. 2024-09-14T05:03:29.564 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:29.565 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:29.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:29.666 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:29.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:29.689 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:29.690 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:29.718 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:29 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-a[106447]: 2024-09-14T05:03:29.337+0000 7f000264a640 -1 mon.a@0(leader).osd e1589 definitely_dead 0 2024-09-14T05:03:29.718 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:29 smithi039 ceph-mon[106470]: pgmap v3174: 119 pgs: 119 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2024-09-14T05:03:29.718 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:29 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/634907579' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-09-14T05:03:29.718 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:29 smithi039 ceph-mon[106470]: Client client.admin marked osd.4 out, while it was still marked up 2024-09-14T05:03:29.718 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:29 smithi039 ceph-mon[111235]: pgmap v3174: 119 pgs: 119 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2024-09-14T05:03:29.718 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:29 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/634907579' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-09-14T05:03:29.718 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:29 smithi039 ceph-mon[111235]: Client client.admin marked osd.4 out, while it was still marked up 2024-09-14T05:03:29.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:29 smithi129 ceph-mon[101316]: pgmap v3174: 119 pgs: 119 active+clean; 5.1 GiB data, 22 GiB used, 693 GiB / 715 GiB avail; 0 B/s wr, 822 op/s 2024-09-14T05:03:29.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:29 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/634907579' entity='client.admin' cmd={"prefix": "osd out", "ids": ["4"]} : dispatch 2024-09-14T05:03:29.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:29 smithi129 ceph-mon[101316]: Client client.admin marked osd.4 out, while it was still marked up 2024-09-14T05:03:29.791 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:29.813 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:29.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:29.814 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:29.915 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:29.937 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:29.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:29.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:30.039 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:30.061 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:30.062 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:30.062 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:30.163 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:30.184 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:30.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:30.185 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:30.206 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T05:03:30.286 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:30.308 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:30.309 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:30.309 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:30.410 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:30.433 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:30.433 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:30.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:30.534 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:30.569 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:30.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:30.571 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:30.671 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:30.694 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:30.694 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:30.694 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:30.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:30 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/634907579' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2024-09-14T05:03:30.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:30 smithi039 ceph-mon[106470]: osdmap e1590: 8 total, 8 up, 7 in 2024-09-14T05:03:30.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:30 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/634907579' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2024-09-14T05:03:30.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:30 smithi039 ceph-mon[111235]: osdmap e1590: 8 total, 8 up, 7 in 2024-09-14T05:03:30.796 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:30.819 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:30.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:30.820 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:30.921 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:30.943 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:30.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:30.944 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:31.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:30 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/634907579' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2024-09-14T05:03:31.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:30 smithi129 ceph-mon[101316]: osdmap e1590: 8 total, 8 up, 7 in 2024-09-14T05:03:31.045 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:31.068 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:31.068 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:31.069 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:31.170 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:31.192 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:31.192 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:31.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:31.294 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:31.315 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:31.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:31.316 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:31.417 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:31.441 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:31.441 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:31.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:31.476 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:03:31.543 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:31.572 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:31.573 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:31.574 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:31.674 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:31.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:31.709 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:31.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:31.739 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:31 smithi039 ceph-mon[106470]: pgmap v3176: 119 pgs: 8 peering, 111 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 877 op/s 2024-09-14T05:03:31.739 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:31 smithi039 ceph-mon[106470]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-14T05:03:31.739 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:31 smithi039 ceph-mon[106470]: osdmap e1591: 8 total, 8 up, 7 in 2024-09-14T05:03:31.739 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:31 smithi039 ceph-mon[111235]: pgmap v3176: 119 pgs: 8 peering, 111 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 877 op/s 2024-09-14T05:03:31.739 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:31 smithi039 ceph-mon[111235]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-14T05:03:31.739 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:31 smithi039 ceph-mon[111235]: osdmap e1591: 8 total, 8 up, 7 in 2024-09-14T05:03:31.811 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:31.841 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:31.842 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:31.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:31.943 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:31.972 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:31.973 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:31.973 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:32.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:31 smithi129 ceph-mon[101316]: pgmap v3176: 119 pgs: 8 peering, 111 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 877 op/s 2024-09-14T05:03:32.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:31 smithi129 ceph-mon[101316]: Health check failed: Reduced data availability: 2 pgs inactive, 2 pgs peering (PG_AVAILABILITY) 2024-09-14T05:03:32.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:31 smithi129 ceph-mon[101316]: osdmap e1591: 8 total, 8 up, 7 in 2024-09-14T05:03:32.074 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:32.102 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:32.102 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:32.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:32.204 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:32.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:32.252 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:32.253 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:32.354 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:32.381 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:32.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:32.382 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:32.483 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:32.485 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T05:03:32.485 INFO:teuthology.orchestra.run.smithi039.stdout: "size": 20, 2024-09-14T05:03:32.485 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 600, 2024-09-14T05:03:32.485 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [ 2024-09-14T05:03:32.485 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.485 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:150517 56.d e1502/1470 56:b2d2f80c:::benchmark_data_smithi039_330847_object35993:head v 1502'9481, mlcod=1502'9477)", 2024-09-14T05:03:32.485 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:54:48.319724+0000", 2024-09-14T05:03:32.486 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 524.16250587100001, 2024-09-14T05:03:32.486 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.33259182399999998, 2024-09-14T05:03:32.486 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:32.486 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:32.486 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:03:32.486 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:32.486 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.486 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:32.486 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.319724+0000", 2024-09-14T05:03:32.486 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.487 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.487 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.487 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:32.487 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.319724+0000", 2024-09-14T05:03:32.487 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.487 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.487 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.487 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:32.487 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.319725+0000", 2024-09-14T05:03:32.487 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2699999999999999e-06 2024-09-14T05:03:32.488 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.488 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.488 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:32.488 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.319732+0000", 2024-09-14T05:03:32.488 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.2779999999999996e-06 2024-09-14T05:03:32.488 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.488 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.488 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:32.488 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.319742+0000", 2024-09-14T05:03:32.488 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.8949999999999993e-06 2024-09-14T05:03:32.488 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.488 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.488 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:32.489 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.319750+0000", 2024-09-14T05:03:32.489 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9549999999999992e-06 2024-09-14T05:03:32.489 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.489 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.489 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:32.489 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.322309+0000", 2024-09-14T05:03:32.489 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0025585680000000002 2024-09-14T05:03:32.489 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.489 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.489 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:32.489 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.322381+0000", 2024-09-14T05:03:32.489 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.1964000000000004e-05 2024-09-14T05:03:32.489 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.490 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.490 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:03:32.490 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.340707+0000", 2024-09-14T05:03:32.490 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.018326426 2024-09-14T05:03:32.490 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.490 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.490 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:32.490 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.652316+0000", 2024-09-14T05:03:32.490 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.31160846800000003 2024-09-14T05:03:32.490 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.490 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:32.490 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.490 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.491 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.491 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:150518 56.5 e1502/1470 56:af443af1:::benchmark_data_smithi039_330847_object35994:head v 1502'9416, mlcod=1502'9412)", 2024-09-14T05:03:32.491 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:54:48.320121+0000", 2024-09-14T05:03:32.491 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 524.16210836400001, 2024-09-14T05:03:32.491 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.33647724200000001, 2024-09-14T05:03:32.491 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:32.491 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:32.491 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:03:32.491 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:32.492 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.492 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:32.492 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.320121+0000", 2024-09-14T05:03:32.492 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.492 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.492 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.492 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:32.492 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.320121+0000", 2024-09-14T05:03:32.492 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.492 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.492 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.492 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:32.492 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.320122+0000", 2024-09-14T05:03:32.493 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.3399999999999998e-07 2024-09-14T05:03:32.493 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.493 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.493 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:32.493 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.320128+0000", 2024-09-14T05:03:32.493 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0449999999999997e-06 2024-09-14T05:03:32.493 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.493 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.493 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:32.493 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.320139+0000", 2024-09-14T05:03:32.493 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1311000000000001e-05 2024-09-14T05:03:32.493 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.493 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.494 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:32.494 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.320146+0000", 2024-09-14T05:03:32.494 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2389999999999999e-06 2024-09-14T05:03:32.494 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.494 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.494 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:32.494 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.327570+0000", 2024-09-14T05:03:32.494 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0074241569999999998 2024-09-14T05:03:32.494 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.494 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.494 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:32.494 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.327589+0000", 2024-09-14T05:03:32.495 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9057000000000001e-05 2024-09-14T05:03:32.495 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.495 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.495 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:03:32.495 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.656515+0000", 2024-09-14T05:03:32.495 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.32892624500000001 2024-09-14T05:03:32.495 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.495 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.495 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:32.495 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.656599+0000", 2024-09-14T05:03:32.495 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.3454000000000004e-05 2024-09-14T05:03:32.495 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.495 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:32.496 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.496 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.496 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.496 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:150525 56.5 e1502/1470 56:af201380:::benchmark_data_smithi039_330847_object36001:head v 1502'9417, mlcod=1502'9412)", 2024-09-14T05:03:32.496 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:54:48.321332+0000", 2024-09-14T05:03:32.496 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 524.16089770600001, 2024-09-14T05:03:32.496 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.335344066, 2024-09-14T05:03:32.496 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:32.496 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:32.497 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:03:32.497 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:32.497 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.497 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:32.497 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.321332+0000", 2024-09-14T05:03:32.497 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.497 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.497 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.497 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:32.497 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.321332+0000", 2024-09-14T05:03:32.497 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.498 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.498 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.498 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:32.498 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.321333+0000", 2024-09-14T05:03:32.498 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7899999999999997e-07 2024-09-14T05:03:32.498 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.498 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.498 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:32.498 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.321341+0000", 2024-09-14T05:03:32.498 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.4810000000000006e-06 2024-09-14T05:03:32.498 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.499 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.499 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:32.499 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.321352+0000", 2024-09-14T05:03:32.499 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0855e-05 2024-09-14T05:03:32.499 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.499 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.499 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:32.499 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.321359+0000", 2024-09-14T05:03:32.499 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.4339999999999997e-06 2024-09-14T05:03:32.499 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.499 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.500 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:32.500 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.333506+0000", 2024-09-14T05:03:32.500 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.01214791 2024-09-14T05:03:32.500 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.500 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.500 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:32.500 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.333525+0000", 2024-09-14T05:03:32.500 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8836999999999999e-05 2024-09-14T05:03:32.500 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.500 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.500 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:03:32.500 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.656602+0000", 2024-09-14T05:03:32.500 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.32307669999999999 2024-09-14T05:03:32.501 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.501 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.501 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:32.501 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.656676+0000", 2024-09-14T05:03:32.501 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.4070000000000001e-05 2024-09-14T05:03:32.501 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.501 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:32.501 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.501 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.501 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.501 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:150528 56.0 e1502/1470 56:04bd5805:::benchmark_data_smithi039_330847_object36004:head v 1502'9348, mlcod=1502'9343)", 2024-09-14T05:03:32.504 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:54:48.332080+0000", 2024-09-14T05:03:32.504 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 524.15014973400002, 2024-09-14T05:03:32.504 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.32073101100000001, 2024-09-14T05:03:32.504 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:32.504 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:32.504 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:03:32.504 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:32.504 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.504 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:32.504 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.332080+0000", 2024-09-14T05:03:32.504 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.505 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.505 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.505 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:32.505 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.332080+0000", 2024-09-14T05:03:32.505 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.505 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.505 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.505 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:32.505 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.332081+0000", 2024-09-14T05:03:32.505 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5149999999999999e-06 2024-09-14T05:03:32.505 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.505 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.505 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:32.506 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.332092+0000", 2024-09-14T05:03:32.506 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0494e-05 2024-09-14T05:03:32.506 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.506 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.506 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:32.506 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.332106+0000", 2024-09-14T05:03:32.506 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3832000000000001e-05 2024-09-14T05:03:32.506 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.506 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.506 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:32.506 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.332114+0000", 2024-09-14T05:03:32.506 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0209999999999999e-06 2024-09-14T05:03:32.506 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.507 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.507 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:32.507 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.333767+0000", 2024-09-14T05:03:32.507 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001653212 2024-09-14T05:03:32.507 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.507 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.507 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:32.507 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.333847+0000", 2024-09-14T05:03:32.507 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9901000000000006e-05 2024-09-14T05:03:32.507 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.507 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.507 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:03:32.508 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.652325+0000", 2024-09-14T05:03:32.508 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.31847768599999998 2024-09-14T05:03:32.508 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.508 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.508 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:32.508 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.652811+0000", 2024-09-14T05:03:32.508 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00048634999999999998 2024-09-14T05:03:32.508 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.508 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:32.508 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.508 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.508 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.508 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:150534 56.5 e1502/1470 56:ab5eb5b0:::benchmark_data_smithi039_330847_object36010:head v 1502'9418, mlcod=1502'9413)", 2024-09-14T05:03:32.509 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:54:48.332948+0000", 2024-09-14T05:03:32.509 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 524.14928219800004, 2024-09-14T05:03:32.509 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.33585074300000001, 2024-09-14T05:03:32.509 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:32.509 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:32.509 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:03:32.509 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:32.509 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.509 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:32.509 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.332948+0000", 2024-09-14T05:03:32.509 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.509 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.509 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.510 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:32.510 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.332948+0000", 2024-09-14T05:03:32.510 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.510 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.510 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.510 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:32.510 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.332948+0000", 2024-09-14T05:03:32.510 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9299999999999997e-07 2024-09-14T05:03:32.510 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.510 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.510 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:32.510 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.332957+0000", 2024-09-14T05:03:32.510 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.7260000000000008e-06 2024-09-14T05:03:32.510 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.511 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.511 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:32.511 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.332968+0000", 2024-09-14T05:03:32.511 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0726e-05 2024-09-14T05:03:32.511 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.511 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.511 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:32.511 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.332974+0000", 2024-09-14T05:03:32.511 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2330000000000003e-06 2024-09-14T05:03:32.511 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.511 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.511 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:32.511 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.652363+0000", 2024-09-14T05:03:32.512 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.319389592 2024-09-14T05:03:32.512 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.512 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.512 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:32.512 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.652388+0000", 2024-09-14T05:03:32.512 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4096000000000001e-05 2024-09-14T05:03:32.512 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.512 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.512 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:03:32.513 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.668751+0000", 2024-09-14T05:03:32.513 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.016363906000000001 2024-09-14T05:03:32.513 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.513 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.513 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:32.513 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.668798+0000", 2024-09-14T05:03:32.513 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6771000000000003e-05 2024-09-14T05:03:32.513 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.513 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:32.513 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.513 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.514 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.514 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:150537 56.2 e1502/1489 56:4734d1d9:::benchmark_data_smithi039_330847_object36013:head v 1502'9523, mlcod=1474'3713)", 2024-09-14T05:03:32.514 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:54:48.334366+0000", 2024-09-14T05:03:32.514 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 524.147864078, 2024-09-14T05:03:32.514 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.31802780200000003, 2024-09-14T05:03:32.514 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:32.514 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:32.514 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:03:32.514 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:32.514 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.514 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:32.514 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.334366+0000", 2024-09-14T05:03:32.514 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.514 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.515 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.515 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:32.515 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.334366+0000", 2024-09-14T05:03:32.515 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.515 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.515 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.515 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:32.515 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.334366+0000", 2024-09-14T05:03:32.515 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7299999999999995e-07 2024-09-14T05:03:32.515 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.515 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.515 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:32.515 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.334373+0000", 2024-09-14T05:03:32.516 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2709999999999998e-06 2024-09-14T05:03:32.516 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.516 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.516 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:32.516 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.334382+0000", 2024-09-14T05:03:32.516 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.0149999999999992e-06 2024-09-14T05:03:32.516 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.516 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.516 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:32.516 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.334387+0000", 2024-09-14T05:03:32.516 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5790000000000003e-06 2024-09-14T05:03:32.516 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.517 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.517 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:32.517 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.334439+0000", 2024-09-14T05:03:32.517 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1949e-05 2024-09-14T05:03:32.517 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.517 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.517 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:32.517 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.334472+0000", 2024-09-14T05:03:32.517 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.2617000000000003e-05 2024-09-14T05:03:32.517 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.517 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.517 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:03:32.517 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.652317+0000", 2024-09-14T05:03:32.517 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.31784542100000002 2024-09-14T05:03:32.518 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.518 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.518 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:32.518 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.652393+0000", 2024-09-14T05:03:32.518 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6277000000000002e-05 2024-09-14T05:03:32.518 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.518 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:32.518 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.518 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.518 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.518 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:150542 56.0 e1502/1470 56:0741494f:::benchmark_data_smithi039_330847_object36018:head v 1502'9349, mlcod=1502'9346)", 2024-09-14T05:03:32.519 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:54:48.342576+0000", 2024-09-14T05:03:32.519 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 524.13965358600001, 2024-09-14T05:03:32.519 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.31390462699999999, 2024-09-14T05:03:32.519 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:32.519 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:32.519 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:03:32.519 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:32.519 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.519 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:32.519 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.342576+0000", 2024-09-14T05:03:32.519 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.519 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.520 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.520 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:32.520 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.342576+0000", 2024-09-14T05:03:32.520 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.520 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.520 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.520 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:32.520 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.342577+0000", 2024-09-14T05:03:32.520 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1939999999999999e-06 2024-09-14T05:03:32.520 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.520 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.520 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:32.520 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.342583+0000", 2024-09-14T05:03:32.521 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0090000000000004e-06 2024-09-14T05:03:32.521 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.521 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.521 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:32.521 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.652363+0000", 2024-09-14T05:03:32.521 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.30977945899999998 2024-09-14T05:03:32.521 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.521 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.521 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:32.521 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.652371+0000", 2024-09-14T05:03:32.521 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.1210000000000007e-06 2024-09-14T05:03:32.521 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.521 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.522 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:32.522 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.652838+0000", 2024-09-14T05:03:32.522 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00046703899999999998 2024-09-14T05:03:32.522 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.522 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.522 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:32.522 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.652858+0000", 2024-09-14T05:03:32.522 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9952000000000001e-05 2024-09-14T05:03:32.522 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.522 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.522 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:03:32.522 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.656336+0000", 2024-09-14T05:03:32.522 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0034781679999999998 2024-09-14T05:03:32.522 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.523 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.523 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:32.523 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.656481+0000", 2024-09-14T05:03:32.523 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000144685 2024-09-14T05:03:32.523 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.523 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:32.523 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.523 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.523 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.523 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36012.0:150550 56.d e1502/1470 56:b8b6dd53:::benchmark_data_smithi039_330847_object36026:head v 1502'9482, mlcod=1502'9480)", 2024-09-14T05:03:32.523 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T04:54:48.350479+0000", 2024-09-14T05:03:32.523 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 524.13175042299997, 2024-09-14T05:03:32.524 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.31836037700000003, 2024-09-14T05:03:32.524 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:32.524 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:32.524 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:03:32.524 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:32.524 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.524 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:32.524 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.350479+0000", 2024-09-14T05:03:32.524 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.524 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.525 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.525 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:32.525 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.350479+0000", 2024-09-14T05:03:32.525 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.525 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.525 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.525 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:32.525 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.350481+0000", 2024-09-14T05:03:32.525 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5689999999999999e-06 2024-09-14T05:03:32.525 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.525 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.525 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:32.525 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.350497+0000", 2024-09-14T05:03:32.525 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5845000000000002e-05 2024-09-14T05:03:32.526 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.526 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.526 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:32.526 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.652315+0000", 2024-09-14T05:03:32.526 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.30181803800000001 2024-09-14T05:03:32.526 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.526 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.526 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:32.526 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.652333+0000", 2024-09-14T05:03:32.526 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7808999999999998e-05 2024-09-14T05:03:32.526 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.526 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.526 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:32.527 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.652558+0000", 2024-09-14T05:03:32.527 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00022574199999999999 2024-09-14T05:03:32.527 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.527 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.527 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:32.527 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.652576+0000", 2024-09-14T05:03:32.527 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7801000000000001e-05 2024-09-14T05:03:32.527 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.527 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.527 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:03:32.527 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.668805+0000", 2024-09-14T05:03:32.527 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.016228597000000001 2024-09-14T05:03:32.527 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.527 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.528 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:32.528 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T04:54:48.668840+0000", 2024-09-14T05:03:32.528 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.4975999999999999e-05 2024-09-14T05:03:32.528 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.528 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:32.528 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.528 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.528 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.528 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36324.0:241668 57.a e1559/1518 57:570dbb5e:::benchmark_data_smithi039_365565_object117916:head v 1559'15260, mlcod=1558'15258)", 2024-09-14T05:03:32.528 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:00:42.483927+0000", 2024-09-14T05:03:32.528 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 169.99830254, 2024-09-14T05:03:32.528 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.31771684700000002, 2024-09-14T05:03:32.529 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:32.529 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:32.529 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:03:32.529 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:32.529 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.529 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:32.529 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483927+0000", 2024-09-14T05:03:32.529 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.529 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.529 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.529 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:32.529 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483927+0000", 2024-09-14T05:03:32.529 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.529 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.530 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.530 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:32.530 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483927+0000", 2024-09-14T05:03:32.530 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4699999999999998e-07 2024-09-14T05:03:32.530 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.530 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.530 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:32.530 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483933+0000", 2024-09-14T05:03:32.530 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1359999999999996e-06 2024-09-14T05:03:32.530 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.530 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.530 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:32.530 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483956+0000", 2024-09-14T05:03:32.530 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3176e-05 2024-09-14T05:03:32.531 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.531 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.531 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:32.531 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.483963+0000", 2024-09-14T05:03:32.531 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.0419999999999997e-06 2024-09-14T05:03:32.531 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.531 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.531 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:32.531 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.484288+0000", 2024-09-14T05:03:32.531 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00032501399999999999 2024-09-14T05:03:32.531 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.531 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.531 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:32.532 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.484347+0000", 2024-09-14T05:03:32.532 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.9614999999999998e-05 2024-09-14T05:03:32.532 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.532 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.532 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:03:32.532 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.801447+0000", 2024-09-14T05:03:32.532 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.31709975800000001 2024-09-14T05:03:32.532 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.532 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.532 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:32.532 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:00:42.801644+0000", 2024-09-14T05:03:32.532 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00019685899999999999 2024-09-14T05:03:32.532 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.533 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:32.533 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.533 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.533 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.533 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36603.0:34262 58.6 e1572/1568 58:6d1741d2:::benchmark_data_smithi039_396950_object34261:head v 1572'2233, mlcod=1572'2231)", 2024-09-14T05:03:32.533 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:01:32.130399+0000", 2024-09-14T05:03:32.533 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 120.351830217, 2024-09-14T05:03:32.533 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.128905984, 2024-09-14T05:03:32.533 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:32.533 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:32.533 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:03:32.534 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:32.534 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.534 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:32.534 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.130399+0000", 2024-09-14T05:03:32.534 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.534 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.534 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.534 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:32.534 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.130399+0000", 2024-09-14T05:03:32.534 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.534 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.534 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.534 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:32.535 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.130400+0000", 2024-09-14T05:03:32.535 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.8400000000000003e-07 2024-09-14T05:03:32.535 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.535 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.535 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:32.535 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.130439+0000", 2024-09-14T05:03:32.535 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.9026000000000002e-05 2024-09-14T05:03:32.535 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.535 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.535 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:32.535 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.130451+0000", 2024-09-14T05:03:32.535 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.114e-05 2024-09-14T05:03:32.535 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.536 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.536 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:32.536 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.130463+0000", 2024-09-14T05:03:32.536 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2773e-05 2024-09-14T05:03:32.536 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.536 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.536 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:32.536 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.130674+0000", 2024-09-14T05:03:32.536 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00021076499999999999 2024-09-14T05:03:32.536 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.536 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.536 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:32.536 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.130695+0000", 2024-09-14T05:03:32.537 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0886999999999998e-05 2024-09-14T05:03:32.537 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.537 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.537 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:03:32.537 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.259260+0000", 2024-09-14T05:03:32.537 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.1285654 2024-09-14T05:03:32.537 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.537 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.537 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:32.537 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.259305+0000", 2024-09-14T05:03:32.537 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5108999999999998e-05 2024-09-14T05:03:32.537 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.538 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:32.538 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.538 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.538 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.538 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36603.0:34259 58.7 e1572/1568 58:eb5e5979:::benchmark_data_smithi039_396950_object34258:head v 1572'2133, mlcod=1572'2130)", 2024-09-14T05:03:32.538 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:01:32.130508+0000", 2024-09-14T05:03:32.538 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 120.35172174500001, 2024-09-14T05:03:32.538 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.122938768, 2024-09-14T05:03:32.538 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:32.538 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:32.538 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:03:32.538 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:32.538 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.539 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:32.539 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.130508+0000", 2024-09-14T05:03:32.539 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.539 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.539 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.539 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:32.539 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.130508+0000", 2024-09-14T05:03:32.539 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.539 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.539 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.539 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:32.539 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.130508+0000", 2024-09-14T05:03:32.539 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.0699999999999997e-07 2024-09-14T05:03:32.539 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.540 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.540 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:32.540 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.130529+0000", 2024-09-14T05:03:32.540 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0180999999999999e-05 2024-09-14T05:03:32.540 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.540 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.540 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:32.540 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.130538+0000", 2024-09-14T05:03:32.540 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.1910000000000006e-06 2024-09-14T05:03:32.540 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.540 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.540 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:32.540 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.130545+0000", 2024-09-14T05:03:32.541 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.861e-06 2024-09-14T05:03:32.541 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.541 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.541 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:32.541 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.130748+0000", 2024-09-14T05:03:32.541 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00020299200000000001 2024-09-14T05:03:32.541 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.541 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.541 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:32.541 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.130764+0000", 2024-09-14T05:03:32.541 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6708999999999999e-05 2024-09-14T05:03:32.541 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.541 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.541 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:03:32.542 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.253342+0000", 2024-09-14T05:03:32.542 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.12257797400000001 2024-09-14T05:03:32.542 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.542 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.542 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:32.542 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.253447+0000", 2024-09-14T05:03:32.542 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000104353 2024-09-14T05:03:32.542 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.542 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:32.542 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.542 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.542 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.542 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36603.0:34263 58.9 e1572/1568 58:901adafa:::benchmark_data_smithi039_396950_object34262:head v 1572'2110, mlcod=1572'2107)", 2024-09-14T05:03:32.542 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:01:32.130571+0000", 2024-09-14T05:03:32.543 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 120.351658896, 2024-09-14T05:03:32.543 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.12275657199999999, 2024-09-14T05:03:32.543 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:32.543 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:32.543 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:03:32.543 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:32.543 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.543 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:32.543 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.130571+0000", 2024-09-14T05:03:32.543 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.544 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.544 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.544 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:32.544 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.130571+0000", 2024-09-14T05:03:32.546 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.546 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.546 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.546 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:32.546 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.130571+0000", 2024-09-14T05:03:32.546 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.1100000000000001e-07 2024-09-14T05:03:32.547 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.547 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.547 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:32.547 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.130594+0000", 2024-09-14T05:03:32.547 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2585000000000001e-05 2024-09-14T05:03:32.547 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.547 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.547 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:32.547 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.130603+0000", 2024-09-14T05:03:32.547 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.9740000000000008e-06 2024-09-14T05:03:32.547 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.547 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.547 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:32.548 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.130608+0000", 2024-09-14T05:03:32.548 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5890000000000002e-06 2024-09-14T05:03:32.548 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.548 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.548 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:32.548 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.130673+0000", 2024-09-14T05:03:32.548 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.4617999999999997e-05 2024-09-14T05:03:32.549 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.549 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.549 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:32.549 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.130695+0000", 2024-09-14T05:03:32.549 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.2062e-05 2024-09-14T05:03:32.549 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.549 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.549 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:03:32.549 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.253283+0000", 2024-09-14T05:03:32.549 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.122587655 2024-09-14T05:03:32.549 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.549 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.549 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:32.549 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.253327+0000", 2024-09-14T05:03:32.550 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.4678000000000002e-05 2024-09-14T05:03:32.550 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.550 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:32.550 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.550 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.550 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.550 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36603.0:34267 58.4 e1572/1568 58:23843477:::benchmark_data_smithi039_396950_object34266:head v 1572'2181, mlcod=1572'2178)", 2024-09-14T05:03:32.550 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:01:32.131035+0000", 2024-09-14T05:03:32.550 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 120.351194948, 2024-09-14T05:03:32.550 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.122455599, 2024-09-14T05:03:32.550 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:32.550 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:32.550 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:03:32.551 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:32.551 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.551 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:32.551 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.131035+0000", 2024-09-14T05:03:32.551 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.551 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.551 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.551 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:32.551 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.131035+0000", 2024-09-14T05:03:32.551 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.551 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.551 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.551 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:32.551 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.131035+0000", 2024-09-14T05:03:32.552 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.75e-07 2024-09-14T05:03:32.552 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.552 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.552 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:32.553 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.131063+0000", 2024-09-14T05:03:32.553 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.7152999999999999e-05 2024-09-14T05:03:32.554 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.554 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.554 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:32.554 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.251049+0000", 2024-09-14T05:03:32.554 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.11998679 2024-09-14T05:03:32.554 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.554 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.554 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:32.554 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.251067+0000", 2024-09-14T05:03:32.554 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8017e-05 2024-09-14T05:03:32.554 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.554 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.554 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:32.554 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.251176+0000", 2024-09-14T05:03:32.555 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00010822100000000001 2024-09-14T05:03:32.555 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.555 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.555 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:32.555 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.251197+0000", 2024-09-14T05:03:32.555 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.126e-05 2024-09-14T05:03:32.555 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.555 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.556 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:03:32.556 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.252926+0000", 2024-09-14T05:03:32.556 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.001729036 2024-09-14T05:03:32.556 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.556 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.556 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:32.556 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.253490+0000", 2024-09-14T05:03:32.556 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00056444700000000004 2024-09-14T05:03:32.556 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.556 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:32.556 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.556 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.557 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.557 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36603.0:34268 58.0 e1572/1568 58:06b98537:::benchmark_data_smithi039_396950_object34267:head v 1572'2105, mlcod=1572'2103)", 2024-09-14T05:03:32.557 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:01:32.131173+0000", 2024-09-14T05:03:32.557 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 120.35105631499999, 2024-09-14T05:03:32.557 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.122269288, 2024-09-14T05:03:32.557 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:32.557 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:32.557 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:03:32.557 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:32.557 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.557 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:32.557 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.131173+0000", 2024-09-14T05:03:32.558 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.558 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.558 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.558 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:32.558 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.131173+0000", 2024-09-14T05:03:32.558 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.558 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.558 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.558 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:32.558 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.131174+0000", 2024-09-14T05:03:32.558 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.6000000000000003e-07 2024-09-14T05:03:32.558 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.558 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.559 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:32.559 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.131215+0000", 2024-09-14T05:03:32.559 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.0475000000000001e-05 2024-09-14T05:03:32.559 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.559 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.559 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:32.559 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.251151+0000", 2024-09-14T05:03:32.559 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.119936086 2024-09-14T05:03:32.559 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.559 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.559 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:32.559 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.251158+0000", 2024-09-14T05:03:32.559 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.2280000000000001e-06 2024-09-14T05:03:32.559 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.560 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.560 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:32.560 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.251207+0000", 2024-09-14T05:03:32.560 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9165000000000001e-05 2024-09-14T05:03:32.560 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.560 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.560 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:32.560 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.251228+0000", 2024-09-14T05:03:32.560 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0415999999999999e-05 2024-09-14T05:03:32.560 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.560 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.560 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:03:32.560 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.253349+0000", 2024-09-14T05:03:32.561 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0021217900000000001 2024-09-14T05:03:32.561 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.561 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.561 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:32.561 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:32.253443+0000", 2024-09-14T05:03:32.561 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.3368000000000005e-05 2024-09-14T05:03:32.561 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.561 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:32.561 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.561 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.561 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.561 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36603.0:38905 58.3 e1572/1568 58:cd226683:::benchmark_data_smithi039_396950_object38904:head v 1572'2386, mlcod=1572'2384)", 2024-09-14T05:03:32.561 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:01:35.218244+0000", 2024-09-14T05:03:32.562 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 117.26398552800001, 2024-09-14T05:03:32.562 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.15146770700000001, 2024-09-14T05:03:32.562 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:32.562 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:32.562 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:03:32.562 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:32.562 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.562 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:32.562 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.218244+0000", 2024-09-14T05:03:32.562 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.562 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.562 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.562 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:32.562 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.218244+0000", 2024-09-14T05:03:32.563 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.563 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.563 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.563 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:32.563 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.218245+0000", 2024-09-14T05:03:32.563 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.4600000000000003e-07 2024-09-14T05:03:32.563 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.563 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.563 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:32.563 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.218265+0000", 2024-09-14T05:03:32.563 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.9646000000000001e-05 2024-09-14T05:03:32.563 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.563 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.563 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:32.564 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.218275+0000", 2024-09-14T05:03:32.564 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0471e-05 2024-09-14T05:03:32.564 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.564 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.564 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:32.564 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.218283+0000", 2024-09-14T05:03:32.564 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.3669999999999999e-06 2024-09-14T05:03:32.564 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.564 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.564 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:32.564 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.218373+0000", 2024-09-14T05:03:32.564 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.0722000000000003e-05 2024-09-14T05:03:32.564 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.564 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.565 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:32.565 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.218398+0000", 2024-09-14T05:03:32.565 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.4465e-05 2024-09-14T05:03:32.565 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.565 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.565 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:03:32.565 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.369419+0000", 2024-09-14T05:03:32.565 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.15102122000000001 2024-09-14T05:03:32.565 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.565 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.565 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:32.565 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.369712+0000", 2024-09-14T05:03:32.565 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00029286999999999998 2024-09-14T05:03:32.565 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.566 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:32.566 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.566 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.566 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.566 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36603.0:38903 58.6 e1572/1568 58:6ad7a147:::benchmark_data_smithi039_396950_object38902:head v 1572'2509, mlcod=1572'2507)", 2024-09-14T05:03:32.566 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:01:35.218312+0000", 2024-09-14T05:03:32.566 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 117.26391758600001, 2024-09-14T05:03:32.566 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.15390353400000001, 2024-09-14T05:03:32.566 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:32.566 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:32.566 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:03:32.566 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:32.566 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.566 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:32.567 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.218312+0000", 2024-09-14T05:03:32.567 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.567 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.567 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.567 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:32.567 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.218312+0000", 2024-09-14T05:03:32.567 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.567 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.567 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.567 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:32.567 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.218313+0000", 2024-09-14T05:03:32.567 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9599999999999999e-07 2024-09-14T05:03:32.567 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.568 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.568 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:32.568 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.218348+0000", 2024-09-14T05:03:32.568 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.5126000000000002e-05 2024-09-14T05:03:32.568 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.568 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.568 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:32.568 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.218356+0000", 2024-09-14T05:03:32.568 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.8059999999999995e-06 2024-09-14T05:03:32.568 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.568 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.568 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:32.568 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.218361+0000", 2024-09-14T05:03:32.569 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.2719999999999997e-06 2024-09-14T05:03:32.569 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.569 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.569 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:32.569 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.218441+0000", 2024-09-14T05:03:32.569 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9944000000000006e-05 2024-09-14T05:03:32.569 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.569 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.569 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:32.569 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.218470+0000", 2024-09-14T05:03:32.569 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.8873e-05 2024-09-14T05:03:32.569 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.569 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.570 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:03:32.570 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.372095+0000", 2024-09-14T05:03:32.570 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.15362574900000001 2024-09-14T05:03:32.570 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.570 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.570 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:32.570 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.372216+0000", 2024-09-14T05:03:32.570 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012026799999999999 2024-09-14T05:03:32.570 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.570 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:32.570 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.570 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.570 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.571 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36603.0:38904 58.2 e1572/1568 58:4a314290:::benchmark_data_smithi039_396950_object38903:head v 1572'2466, mlcod=1572'2461)", 2024-09-14T05:03:32.571 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:01:35.218715+0000", 2024-09-14T05:03:32.571 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 117.263514595, 2024-09-14T05:03:32.571 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.15237968499999999, 2024-09-14T05:03:32.571 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:32.571 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:32.571 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:03:32.571 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:32.571 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.571 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:32.571 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.218715+0000", 2024-09-14T05:03:32.571 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.571 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.572 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.572 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:32.572 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.218715+0000", 2024-09-14T05:03:32.572 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.572 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.572 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.572 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:32.572 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.218716+0000", 2024-09-14T05:03:32.572 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2200000000000004e-07 2024-09-14T05:03:32.572 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.572 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.572 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:32.572 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.218744+0000", 2024-09-14T05:03:32.573 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.8682000000000001e-05 2024-09-14T05:03:32.573 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.573 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.573 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:32.573 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.218755+0000", 2024-09-14T05:03:32.573 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0716e-05 2024-09-14T05:03:32.573 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.573 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.573 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:32.573 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.218762+0000", 2024-09-14T05:03:32.573 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3690000000000003e-06 2024-09-14T05:03:32.573 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.573 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.573 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:32.574 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.218813+0000", 2024-09-14T05:03:32.574 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1408e-05 2024-09-14T05:03:32.574 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.574 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.574 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:32.574 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.218832+0000", 2024-09-14T05:03:32.574 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8606000000000001e-05 2024-09-14T05:03:32.574 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.574 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.574 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:03:32.574 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.371039+0000", 2024-09-14T05:03:32.574 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.15220794400000001 2024-09-14T05:03:32.574 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.575 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.575 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:32.575 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.371095+0000", 2024-09-14T05:03:32.575 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5337999999999998e-05 2024-09-14T05:03:32.575 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.575 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:32.575 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.575 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.575 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.575 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36603.0:38906 58.2 e1572/1568 58:45485c32:::benchmark_data_smithi039_396950_object38905:head v 1572'2467, mlcod=1572'2461)", 2024-09-14T05:03:32.575 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:01:35.219678+0000", 2024-09-14T05:03:32.575 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 117.262551922, 2024-09-14T05:03:32.575 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.155880673, 2024-09-14T05:03:32.575 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:32.576 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:32.576 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:03:32.576 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:32.576 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.576 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:32.576 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.219678+0000", 2024-09-14T05:03:32.576 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.576 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.576 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.576 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:32.576 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.219678+0000", 2024-09-14T05:03:32.576 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.576 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.576 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.577 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:32.577 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.219679+0000", 2024-09-14T05:03:32.577 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0200000000000001e-07 2024-09-14T05:03:32.577 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.577 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.577 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:32.577 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.219723+0000", 2024-09-14T05:03:32.577 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3936000000000001e-05 2024-09-14T05:03:32.577 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.577 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.577 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:32.577 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.367357+0000", 2024-09-14T05:03:32.577 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.14763454300000001 2024-09-14T05:03:32.577 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.578 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.578 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:32.578 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.367382+0000", 2024-09-14T05:03:32.578 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.5001999999999999e-05 2024-09-14T05:03:32.578 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.578 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.578 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:32.578 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.369435+0000", 2024-09-14T05:03:32.578 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.002053147 2024-09-14T05:03:32.578 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.578 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.578 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:32.578 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.369453+0000", 2024-09-14T05:03:32.578 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.768e-05 2024-09-14T05:03:32.579 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.579 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.579 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:03:32.579 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.375470+0000", 2024-09-14T05:03:32.579 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0060166229999999996 2024-09-14T05:03:32.579 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.579 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.579 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:32.579 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.375558+0000", 2024-09-14T05:03:32.579 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.8939999999999999e-05 2024-09-14T05:03:32.579 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.579 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:32.579 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.579 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.580 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.580 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36603.0:38900 58.7 e1572/1568 58:ed525925:::benchmark_data_smithi039_396950_object38899:head v 1572'2440, mlcod=1572'2436)", 2024-09-14T05:03:32.580 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:01:35.219818+0000", 2024-09-14T05:03:32.580 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 117.26241207299999, 2024-09-14T05:03:32.580 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.15053111699999999, 2024-09-14T05:03:32.580 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:32.580 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:32.580 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:03:32.580 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:32.580 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.580 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:32.580 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.219818+0000", 2024-09-14T05:03:32.580 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.581 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.581 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.581 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:32.581 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.219818+0000", 2024-09-14T05:03:32.581 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.581 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.581 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.581 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:32.581 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.219818+0000", 2024-09-14T05:03:32.581 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.6700000000000003e-07 2024-09-14T05:03:32.581 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.581 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.581 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:32.581 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.219846+0000", 2024-09-14T05:03:32.582 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.7900000000000001e-05 2024-09-14T05:03:32.582 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.582 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.582 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:32.582 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.367502+0000", 2024-09-14T05:03:32.582 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.147655385 2024-09-14T05:03:32.582 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.582 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.582 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:32.582 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.367518+0000", 2024-09-14T05:03:32.582 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6646999999999998e-05 2024-09-14T05:03:32.582 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.582 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.582 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:32.583 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.367626+0000", 2024-09-14T05:03:32.583 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00010733799999999999 2024-09-14T05:03:32.583 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.583 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.583 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:32.583 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.367652+0000", 2024-09-14T05:03:32.583 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.6591e-05 2024-09-14T05:03:32.583 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.583 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.583 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:03:32.583 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.370301+0000", 2024-09-14T05:03:32.583 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0026491190000000001 2024-09-14T05:03:32.583 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.583 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.584 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:32.584 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.370349+0000", 2024-09-14T05:03:32.584 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7469999999999998e-05 2024-09-14T05:03:32.584 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.584 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:32.584 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.584 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.584 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.584 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36603.0:38912 58.4 e1572/1568 58:2778398d:::benchmark_data_smithi039_396950_object38911:head v 1572'2472, mlcod=1572'2470)", 2024-09-14T05:03:32.584 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:01:35.222211+0000", 2024-09-14T05:03:32.584 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 117.26001835, 2024-09-14T05:03:32.584 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.14890318399999999, 2024-09-14T05:03:32.584 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:03:32.585 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:03:32.585 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:03:32.585 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:03:32.585 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.585 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:03:32.585 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.222211+0000", 2024-09-14T05:03:32.585 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.585 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.585 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.585 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:03:32.585 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.222211+0000", 2024-09-14T05:03:32.585 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:03:32.585 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.585 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.586 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:03:32.586 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.222212+0000", 2024-09-14T05:03:32.586 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.4e-07 2024-09-14T05:03:32.586 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.586 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.586 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:03:32.586 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.222245+0000", 2024-09-14T05:03:32.586 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.3107e-05 2024-09-14T05:03:32.586 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.586 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.586 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:03:32.587 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.367339+0000", 2024-09-14T05:03:32.587 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.145093992 2024-09-14T05:03:32.587 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.587 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.587 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:03:32.587 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.367366+0000", 2024-09-14T05:03:32.587 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.6633000000000001e-05 2024-09-14T05:03:32.587 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.587 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.587 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:03:32.587 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.367724+0000", 2024-09-14T05:03:32.588 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00035796899999999997 2024-09-14T05:03:32.588 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.588 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.588 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:03:32.588 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.367745+0000", 2024-09-14T05:03:32.588 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0939000000000001e-05 2024-09-14T05:03:32.588 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.588 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.588 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:03:32.588 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.371035+0000", 2024-09-14T05:03:32.588 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0032897930000000001 2024-09-14T05:03:32.589 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:03:32.589 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:03:32.589 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:03:32.589 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:01:35.371115+0000", 2024-09-14T05:03:32.589 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9810999999999999e-05 2024-09-14T05:03:32.589 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.589 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:32.589 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.589 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:03:32.589 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:03:32.589 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T05:03:32.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:32.590 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:32.590 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:32.691 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:32.722 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:32.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:32.723 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:32.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:32 smithi039 ceph-mon[111235]: osdmap e1592: 8 total, 8 up, 7 in 2024-09-14T05:03:32.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:32 smithi039 ceph-mon[111235]: 58.8 starting backfill to osd.6 from (0'0,0'0] MAX to 1589'9627 2024-09-14T05:03:32.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:32 smithi039 ceph-mon[111235]: 58.2 starting backfill to osd.7 from (0'0,0'0] MAX to 1589'9544 2024-09-14T05:03:32.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:32 smithi039 ceph-mon[111235]: 58.5 starting backfill to osd.2 from (0'0,0'0] MAX to 1589'9452 2024-09-14T05:03:32.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:32 smithi039 ceph-mon[111235]: 58.0 starting backfill to osd.1 from (0'0,0'0] MAX to 1589'9494 2024-09-14T05:03:32.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:32 smithi039 ceph-mon[111235]: 58.6 starting backfill to osd.5 from (0'0,0'0] MAX to 1589'9654 2024-09-14T05:03:32.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:32 smithi039 ceph-mon[111235]: 58.c starting backfill to osd.5 from (0'0,0'0] MAX to 1589'9497 2024-09-14T05:03:32.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:32 smithi039 ceph-mon[106470]: osdmap e1592: 8 total, 8 up, 7 in 2024-09-14T05:03:32.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:32 smithi039 ceph-mon[106470]: 58.8 starting backfill to osd.6 from (0'0,0'0] MAX to 1589'9627 2024-09-14T05:03:32.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:32 smithi039 ceph-mon[106470]: 58.2 starting backfill to osd.7 from (0'0,0'0] MAX to 1589'9544 2024-09-14T05:03:32.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:32 smithi039 ceph-mon[106470]: 58.5 starting backfill to osd.2 from (0'0,0'0] MAX to 1589'9452 2024-09-14T05:03:32.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:32 smithi039 ceph-mon[106470]: 58.0 starting backfill to osd.1 from (0'0,0'0] MAX to 1589'9494 2024-09-14T05:03:32.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:32 smithi039 ceph-mon[106470]: 58.6 starting backfill to osd.5 from (0'0,0'0] MAX to 1589'9654 2024-09-14T05:03:32.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:32 smithi039 ceph-mon[106470]: 58.c starting backfill to osd.5 from (0'0,0'0] MAX to 1589'9497 2024-09-14T05:03:32.824 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:32.859 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:32.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:32.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:32.961 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:32.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:32.992 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:32.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:32 smithi129 ceph-mon[101316]: osdmap e1592: 8 total, 8 up, 7 in 2024-09-14T05:03:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:32 smithi129 ceph-mon[101316]: 58.8 starting backfill to osd.6 from (0'0,0'0] MAX to 1589'9627 2024-09-14T05:03:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:32 smithi129 ceph-mon[101316]: 58.2 starting backfill to osd.7 from (0'0,0'0] MAX to 1589'9544 2024-09-14T05:03:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:32 smithi129 ceph-mon[101316]: 58.5 starting backfill to osd.2 from (0'0,0'0] MAX to 1589'9452 2024-09-14T05:03:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:32 smithi129 ceph-mon[101316]: 58.0 starting backfill to osd.1 from (0'0,0'0] MAX to 1589'9494 2024-09-14T05:03:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:32 smithi129 ceph-mon[101316]: 58.6 starting backfill to osd.5 from (0'0,0'0] MAX to 1589'9654 2024-09-14T05:03:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:32 smithi129 ceph-mon[101316]: 58.c starting backfill to osd.5 from (0'0,0'0] MAX to 1589'9497 2024-09-14T05:03:33.093 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:33.121 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:33.122 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:33.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:33.223 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:33.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:33.259 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:33.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:33.361 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:33.401 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:33.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:33.403 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:33.503 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:33.553 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:33.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:33.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:33.655 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:33.696 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:33.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:33.697 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:33.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:33 smithi039 ceph-mon[106470]: pgmap v3179: 119 pgs: 3 remapped+peering, 8 peering, 108 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 620 op/s 2024-09-14T05:03:33.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:33 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T05:03:33.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:33 smithi039 ceph-mon[106470]: osdmap e1593: 8 total, 8 up, 7 in 2024-09-14T05:03:33.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:33 smithi039 ceph-mon[111235]: pgmap v3179: 119 pgs: 3 remapped+peering, 8 peering, 108 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 620 op/s 2024-09-14T05:03:33.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:33 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T05:03:33.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:33 smithi039 ceph-mon[111235]: osdmap e1593: 8 total, 8 up, 7 in 2024-09-14T05:03:33.798 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:33.828 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:33.829 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:33.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:33.916 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:33 smithi129 ceph-mon[101316]: pgmap v3179: 119 pgs: 3 remapped+peering, 8 peering, 108 active+clean; 4.9 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 0 B/s wr, 620 op/s 2024-09-14T05:03:33.916 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:33 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T05:03:33.917 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:33 smithi129 ceph-mon[101316]: osdmap e1593: 8 total, 8 up, 7 in 2024-09-14T05:03:33.930 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:33.959 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:33.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:33.960 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:34.061 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:34.092 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:34.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:34.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:34.144 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:03:34.177 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-14T05:03:34.194 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:34.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:34.247 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:34.247 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:34.348 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:34.390 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:34.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:34.391 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:34.490 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:03:34.492 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:34.529 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:34.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:34.530 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:34.631 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:34.661 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:34.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:34.662 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:34.762 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:34.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:34 smithi039 ceph-mon[106470]: osdmap e1594: 8 total, 8 up, 7 in 2024-09-14T05:03:34.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:34 smithi039 ceph-mon[106470]: 1.5 scrub starts 2024-09-14T05:03:34.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:34 smithi039 ceph-mon[111235]: osdmap e1594: 8 total, 8 up, 7 in 2024-09-14T05:03:34.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:34 smithi039 ceph-mon[111235]: 1.5 scrub starts 2024-09-14T05:03:34.791 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:34.792 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:34.792 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:34.849 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:34 smithi129 ceph-mon[101316]: osdmap e1594: 8 total, 8 up, 7 in 2024-09-14T05:03:34.849 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:34 smithi129 ceph-mon[101316]: 1.5 scrub starts 2024-09-14T05:03:34.892 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:34.934 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:34.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:34.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:35.036 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:35.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:35.064 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:35.065 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:35.165 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:35.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:35.204 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:35.204 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:35.305 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:35.357 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:35.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:35.358 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:35.459 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:35.496 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:35.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:35.497 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:35.598 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:35.617 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is unset 2024-09-14T05:03:35.627 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:35.627 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:35.628 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:35.728 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:35.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:35 smithi129 ceph-mon[101316]: 1.13 scrub starts 2024-09-14T05:03:35.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:35 smithi129 ceph-mon[101316]: pgmap v3182: 119 pgs: 2 active+remapped+backfilling, 3 remapped+peering, 15 peering, 99 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 4.7 KiB/s rd, 14 KiB/s wr, 331 op/s; 9391/237885 objects misplaced (3.948%); 7.3 MiB/s, 2 keys/s, 119 objects/s recovering 2024-09-14T05:03:35.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:35 smithi129 ceph-mon[101316]: 1.1d scrub starts 2024-09-14T05:03:35.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:35 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2077159728' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:03:35.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:35 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:03:35.782 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:35.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:35.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:35.884 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:35.920 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-14T05:03:35.920 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T05:03:35.920 INFO:tasks.thrashosds.thrasher:trim_stale_osdmaps 2024-09-14T05:03:35.920 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.0 'trim stale osdmaps' 2024-09-14T05:03:35.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:35.922 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:35.922 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:35.965 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:35 smithi039 ceph-mon[106470]: 1.13 scrub starts 2024-09-14T05:03:35.965 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:35 smithi039 ceph-mon[106470]: pgmap v3182: 119 pgs: 2 active+remapped+backfilling, 3 remapped+peering, 15 peering, 99 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 4.7 KiB/s rd, 14 KiB/s wr, 331 op/s; 9391/237885 objects misplaced (3.948%); 7.3 MiB/s, 2 keys/s, 119 objects/s recovering 2024-09-14T05:03:35.965 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:35 smithi039 ceph-mon[106470]: 1.1d scrub starts 2024-09-14T05:03:35.965 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:35 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2077159728' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:03:35.965 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:35 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:03:35.965 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:35 smithi039 ceph-mon[111235]: 1.13 scrub starts 2024-09-14T05:03:35.965 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:35 smithi039 ceph-mon[111235]: pgmap v3182: 119 pgs: 2 active+remapped+backfilling, 3 remapped+peering, 15 peering, 99 active+clean; 4.8 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 4.7 KiB/s rd, 14 KiB/s wr, 331 op/s; 9391/237885 objects misplaced (3.948%); 7.3 MiB/s, 2 keys/s, 119 objects/s recovering 2024-09-14T05:03:35.965 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:35 smithi039 ceph-mon[111235]: 1.1d scrub starts 2024-09-14T05:03:35.965 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:35 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2077159728' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:03:35.965 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:35 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:03:36.023 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:36.090 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:36.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:36.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:36.192 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:36.233 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:36.237 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:36.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:36.339 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:36.380 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:36.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:36.381 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:36.482 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:36.529 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:36.529 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:36.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:36.631 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:36.661 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:36.662 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:36.663 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:36.763 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:36.796 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:36.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:36.797 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:36.898 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:36.906 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:36 smithi129 ceph-mon[101316]: 1.19 scrub starts 2024-09-14T05:03:36.906 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:36 smithi129 ceph-mon[101316]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:03:36.906 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:36 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T05:03:36.906 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:36 smithi129 ceph-mon[101316]: osdmap e1595: 8 total, 8 up, 7 in 2024-09-14T05:03:36.906 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:36 smithi129 ceph-mon[101316]: 1.16 scrub starts 2024-09-14T05:03:36.906 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:36 smithi129 ceph-mon[101316]: 1.16 scrub ok 2024-09-14T05:03:36.906 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:36 smithi129 ceph-mon[101316]: Health check update: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-14T05:03:36.935 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:36.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:36.936 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:37.036 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:37.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:36 smithi039 ceph-mon[106470]: 1.19 scrub starts 2024-09-14T05:03:37.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:36 smithi039 ceph-mon[106470]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:03:37.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:36 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T05:03:37.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:36 smithi039 ceph-mon[106470]: osdmap e1595: 8 total, 8 up, 7 in 2024-09-14T05:03:37.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:36 smithi039 ceph-mon[106470]: 1.16 scrub starts 2024-09-14T05:03:37.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:36 smithi039 ceph-mon[106470]: 1.16 scrub ok 2024-09-14T05:03:37.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:36 smithi039 ceph-mon[106470]: Health check update: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-14T05:03:37.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:36 smithi039 ceph-mon[111235]: 1.19 scrub starts 2024-09-14T05:03:37.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:36 smithi039 ceph-mon[111235]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:03:37.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:36 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T05:03:37.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:36 smithi039 ceph-mon[111235]: osdmap e1595: 8 total, 8 up, 7 in 2024-09-14T05:03:37.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:36 smithi039 ceph-mon[111235]: 1.16 scrub starts 2024-09-14T05:03:37.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:36 smithi039 ceph-mon[111235]: 1.16 scrub ok 2024-09-14T05:03:37.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:36 smithi039 ceph-mon[111235]: Health check update: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2024-09-14T05:03:37.059 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T05:03:37.059 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [], 2024-09-14T05:03:37.059 INFO:teuthology.orchestra.run.smithi129.stdout: "num_ops": 0 2024-09-14T05:03:37.059 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T05:03:37.059 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:37.060 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:37.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:37.161 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:37.188 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:37.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:37.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:37.290 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:37.313 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:37.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:37.314 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:37.415 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:37.446 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:37.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:37.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:37.547 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:37.583 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:37.583 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:37.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:37.685 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:37.719 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:37.720 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:37.720 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:37.821 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:37.865 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:37.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:37.866 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:37.936 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:37 smithi129 ceph-mon[101316]: pgmap v3184: 119 pgs: 2 active+remapped+backfill_wait, 1 active+clean+scrubbing, 4 active+remapped+backfilling, 7 peering, 105 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 4.2 KiB/s rd, 12 KiB/s wr, 612 op/s; 27969/231237 objects misplaced (12.095%); 15 MiB/s, 2 keys/s, 237 objects/s recovering 2024-09-14T05:03:37.936 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:37 smithi129 ceph-mon[101316]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-09-14T05:03:37.937 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:37 smithi129 ceph-mon[101316]: 1.7 scrub starts 2024-09-14T05:03:37.937 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:03:37.967 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:37.974 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-14T05:03:38.012 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:38.013 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:38.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:37 smithi039 ceph-mon[106470]: pgmap v3184: 119 pgs: 2 active+remapped+backfill_wait, 1 active+clean+scrubbing, 4 active+remapped+backfilling, 7 peering, 105 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 4.2 KiB/s rd, 12 KiB/s wr, 612 op/s; 27969/231237 objects misplaced (12.095%); 15 MiB/s, 2 keys/s, 237 objects/s recovering 2024-09-14T05:03:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:37 smithi039 ceph-mon[106470]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-09-14T05:03:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:37 smithi039 ceph-mon[106470]: 1.7 scrub starts 2024-09-14T05:03:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:03:38.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:37 smithi039 ceph-mon[111235]: pgmap v3184: 119 pgs: 2 active+remapped+backfill_wait, 1 active+clean+scrubbing, 4 active+remapped+backfilling, 7 peering, 105 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 4.2 KiB/s rd, 12 KiB/s wr, 612 op/s; 27969/231237 objects misplaced (12.095%); 15 MiB/s, 2 keys/s, 237 objects/s recovering 2024-09-14T05:03:38.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:37 smithi039 ceph-mon[111235]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 3 pgs peering) 2024-09-14T05:03:38.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:37 smithi039 ceph-mon[111235]: 1.7 scrub starts 2024-09-14T05:03:38.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:03:38.114 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:38.155 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:38.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:38.156 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:38.257 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:38.288 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:38.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:38.289 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:38.390 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:38.424 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:38.425 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:38.425 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:38.526 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:38.563 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:38.563 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:38.564 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:38.665 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:38.696 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:38.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:38.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:38.798 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:38.820 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:38.820 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:38.821 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:38.922 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:38.961 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:38.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:38.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:39.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:03:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:03:38] "GET /metrics HTTP/1.1" 200 36789 "" "Prometheus/2.51.0" 2024-09-14T05:03:39.063 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:39.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:39.099 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:39.099 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:39.200 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:39.228 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:39.228 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:39.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:39.330 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:39.349 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:03:39.371 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:39.373 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:39.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:39.474 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:39.519 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:39.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:39.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:39.621 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:39.653 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:39.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:39.655 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:39.755 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:39.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:39 smithi129 ceph-mon[101316]: pgmap v3185: 119 pgs: 2 active+remapped+backfill_wait, 1 active+clean+scrubbing, 4 active+remapped+backfilling, 2 peering, 110 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 3.2 KiB/s rd, 9.2 KiB/s wr, 544 op/s; 25407/229722 objects misplaced (11.060%); 33 MiB/s, 1 keys/s, 532 objects/s recovering 2024-09-14T05:03:39.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:39 smithi129 ceph-mon[101316]: 1.6 scrub starts 2024-09-14T05:03:39.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:39 smithi129 ceph-mon[101316]: 1.6 scrub ok 2024-09-14T05:03:39.799 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:39.800 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:39.800 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:39.901 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:39.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:39.933 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:39.933 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:40.034 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:40.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:39 smithi039 ceph-mon[111235]: pgmap v3185: 119 pgs: 2 active+remapped+backfill_wait, 1 active+clean+scrubbing, 4 active+remapped+backfilling, 2 peering, 110 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 3.2 KiB/s rd, 9.2 KiB/s wr, 544 op/s; 25407/229722 objects misplaced (11.060%); 33 MiB/s, 1 keys/s, 532 objects/s recovering 2024-09-14T05:03:40.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:39 smithi039 ceph-mon[111235]: 1.6 scrub starts 2024-09-14T05:03:40.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:39 smithi039 ceph-mon[111235]: 1.6 scrub ok 2024-09-14T05:03:40.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:39 smithi039 ceph-mon[106470]: pgmap v3185: 119 pgs: 2 active+remapped+backfill_wait, 1 active+clean+scrubbing, 4 active+remapped+backfilling, 2 peering, 110 active+clean; 4.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 3.2 KiB/s rd, 9.2 KiB/s wr, 544 op/s; 25407/229722 objects misplaced (11.060%); 33 MiB/s, 1 keys/s, 532 objects/s recovering 2024-09-14T05:03:40.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:39 smithi039 ceph-mon[106470]: 1.6 scrub starts 2024-09-14T05:03:40.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:39 smithi039 ceph-mon[106470]: 1.6 scrub ok 2024-09-14T05:03:40.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:40.066 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:40.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:40.167 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:40.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:40.215 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:40.215 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:40.316 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:40.344 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:40.344 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:40.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:40.445 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:40.497 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:40.498 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:40.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:40.599 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:40.643 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:40.644 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:40.644 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:40.745 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:40.788 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:40.788 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:40.789 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:40.889 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:40.934 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:40.934 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:40.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:40.957 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:40 smithi129 ceph-mon[101316]: 1.1e scrub starts 2024-09-14T05:03:41.035 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:41.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:40 smithi039 ceph-mon[106470]: 1.1e scrub starts 2024-09-14T05:03:41.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:40 smithi039 ceph-mon[111235]: 1.1e scrub starts 2024-09-14T05:03:41.069 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:41.070 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:41.070 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:41.171 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:41.213 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:41.214 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:41.214 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:41.316 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:41.356 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:41.356 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:41.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:41.458 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:41.460 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T05:03:41.460 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T05:03:41.486 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:41.487 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:41.487 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:41.504 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T05:03:41.504 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T05:03:41.538 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T05:03:41.538 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T05:03:41.570 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T05:03:41.570 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T05:03:41.588 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:41.602 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T05:03:41.603 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T05:03:41.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:41.631 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:41.632 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:41.640 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T05:03:41.640 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T05:03:41.673 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T05:03:41.673 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T05:03:41.709 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T05:03:41.709 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T05:03:41.733 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:41.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:41.774 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:41.774 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:41.875 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:41.909 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:41.910 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:41.910 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:41.928 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:41 smithi129 ceph-mon[101316]: pgmap v3186: 119 pgs: 2 active+remapped+backfill_wait, 1 active+clean+scrubbing, 4 active+remapped+backfilling, 112 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 774 op/s; 23701/222093 objects misplaced (10.672%); 35 MiB/s, 1 keys/s, 561 objects/s recovering 2024-09-14T05:03:41.928 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:41 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1183919085' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:03:42.011 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:42.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:41 smithi039 ceph-mon[106470]: pgmap v3186: 119 pgs: 2 active+remapped+backfill_wait, 1 active+clean+scrubbing, 4 active+remapped+backfilling, 112 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 774 op/s; 23701/222093 objects misplaced (10.672%); 35 MiB/s, 1 keys/s, 561 objects/s recovering 2024-09-14T05:03:42.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:41 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1183919085' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:03:42.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:41 smithi039 ceph-mon[111235]: pgmap v3186: 119 pgs: 2 active+remapped+backfill_wait, 1 active+clean+scrubbing, 4 active+remapped+backfilling, 112 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 774 op/s; 23701/222093 objects misplaced (10.672%); 35 MiB/s, 1 keys/s, 561 objects/s recovering 2024-09-14T05:03:42.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:41 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1183919085' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:03:42.050 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:42.051 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:42.051 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:42.152 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:42.183 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:42.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:42.184 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:42.284 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:42.329 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:42.329 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:42.330 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:42.431 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:42.460 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:42.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:42.461 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:42.558 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T05:03:42.558 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:03:42.562 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:42.592 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:42.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:42.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:42.624 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:03:42.627 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T05:03:42.694 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:42.767 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:42.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:42.768 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:42.869 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:42.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:42.923 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:42.923 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:43.024 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:43.053 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:43.053 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:43.054 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:43.155 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:43.196 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:43.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:43.197 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:43.298 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:43.336 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:43.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:43.337 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:43.438 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:43.471 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:43.471 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:43.472 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:43.566 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:03:43.573 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:43.632 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:43.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:43.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:43.734 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:43.765 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:43.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:43.766 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:43.867 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:43.894 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:43.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:43.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:43.990 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:43 smithi039 ceph-mon[106470]: pgmap v3187: 119 pgs: 2 active+remapped+backfill_wait, 1 active+clean+scrubbing, 4 active+remapped+backfilling, 112 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 738 op/s; 23439/219042 objects misplaced (10.701%); 27 MiB/s, 438 objects/s recovering 2024-09-14T05:03:43.990 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:43 smithi039 ceph-mon[111235]: pgmap v3187: 119 pgs: 2 active+remapped+backfill_wait, 1 active+clean+scrubbing, 4 active+remapped+backfilling, 112 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 738 op/s; 23439/219042 objects misplaced (10.701%); 27 MiB/s, 438 objects/s recovering 2024-09-14T05:03:43.996 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:44.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:43 smithi129 ceph-mon[101316]: pgmap v3187: 119 pgs: 2 active+remapped+backfill_wait, 1 active+clean+scrubbing, 4 active+remapped+backfilling, 112 active+clean; 4.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 738 op/s; 23439/219042 objects misplaced (10.701%); 27 MiB/s, 438 objects/s recovering 2024-09-14T05:03:44.038 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:44.039 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:44.039 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:44.140 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:44.190 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:44.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:44.192 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:44.292 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:44.319 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:44.320 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:44.320 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:44.421 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:44.448 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:44.449 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:44.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:44.548 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T05:03:44.551 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:44.576 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:44.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:44.577 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:44.678 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:44.707 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:44.708 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:44.708 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:44.809 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:44.843 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:44.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:44.844 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:44.945 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:44.981 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:44.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:44.982 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:45.083 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:45.114 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:45.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:45.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:45.215 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:45.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:45.276 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:45.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:45.377 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:45.425 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:45.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:45.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:45.526 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:45.555 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:45.555 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:45.555 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:45.656 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:45.708 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:45.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:45.710 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:45.810 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:45.856 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:45.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:45.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:45.884 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:45 smithi039 ceph-mon[106470]: 1.7 scrub starts 2024-09-14T05:03:45.884 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:45 smithi039 ceph-mon[106470]: pgmap v3188: 119 pgs: 2 active+clean+scrubbing, 2 active+remapped+backfill_wait, 4 active+remapped+backfilling, 111 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 729 op/s; 21022/215973 objects misplaced (9.734%); 36 MiB/s, 583 objects/s recovering 2024-09-14T05:03:45.885 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:45 smithi039 ceph-mon[111235]: 1.7 scrub starts 2024-09-14T05:03:45.885 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:45 smithi039 ceph-mon[111235]: pgmap v3188: 119 pgs: 2 active+clean+scrubbing, 2 active+remapped+backfill_wait, 4 active+remapped+backfilling, 111 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 729 op/s; 21022/215973 objects misplaced (9.734%); 36 MiB/s, 583 objects/s recovering 2024-09-14T05:03:45.945 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:45 smithi129 ceph-mon[101316]: 1.7 scrub starts 2024-09-14T05:03:45.945 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:45 smithi129 ceph-mon[101316]: pgmap v3188: 119 pgs: 2 active+clean+scrubbing, 2 active+remapped+backfill_wait, 4 active+remapped+backfilling, 111 active+clean; 4.4 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 729 op/s; 21022/215973 objects misplaced (9.734%); 36 MiB/s, 583 objects/s recovering 2024-09-14T05:03:45.958 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:45.977 INFO:teuthology.orchestra.run.smithi039.stderr:no valid command found; 10 closest matches: 2024-09-14T05:03:45.978 INFO:teuthology.orchestra.run.smithi039.stderr:0 2024-09-14T05:03:45.978 INFO:teuthology.orchestra.run.smithi039.stderr:1 2024-09-14T05:03:45.978 INFO:teuthology.orchestra.run.smithi039.stderr:2 2024-09-14T05:03:45.978 INFO:teuthology.orchestra.run.smithi039.stderr:abort 2024-09-14T05:03:45.978 INFO:teuthology.orchestra.run.smithi039.stderr:assert 2024-09-14T05:03:45.978 INFO:teuthology.orchestra.run.smithi039.stderr:bench [] [] [] [] 2024-09-14T05:03:45.978 INFO:teuthology.orchestra.run.smithi039.stderr:bluefs debug_inject_read_zeros 2024-09-14T05:03:45.978 INFO:teuthology.orchestra.run.smithi039.stderr:bluefs files list 2024-09-14T05:03:45.978 INFO:teuthology.orchestra.run.smithi039.stderr:bluefs stats 2024-09-14T05:03:45.979 INFO:teuthology.orchestra.run.smithi039.stderr:bluestore allocator dump block 2024-09-14T05:03:45.979 INFO:teuthology.orchestra.run.smithi039.stderr:Error EINVAL: invalid command 2024-09-14T05:03:45.999 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:45.999 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:46.000 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:46.101 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:46.125 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:46.125 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:46.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:46.226 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:46.256 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:46.256 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:46.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:46.357 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:46.383 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:46.384 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:46.384 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:46.485 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:46.510 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:46.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:46.511 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:46.612 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:46.635 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:46.636 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:46.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:46.677 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-14T05:03:46.738 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:46.795 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:46.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:46.796 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:46.897 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:46.901 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:46 smithi129 ceph-mon[101316]: osdmap e1596: 8 total, 8 up, 7 in 2024-09-14T05:03:46.901 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:46 smithi129 ceph-mon[101316]: 1.1e scrub starts 2024-09-14T05:03:46.901 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:46 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:03:46.901 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:46 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:03:46.935 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:46.936 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:46.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:47.014 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:03:47.037 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:47.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:46 smithi039 ceph-mon[106470]: osdmap e1596: 8 total, 8 up, 7 in 2024-09-14T05:03:47.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:46 smithi039 ceph-mon[106470]: 1.1e scrub starts 2024-09-14T05:03:47.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:46 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:03:47.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:46 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:03:47.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:46 smithi039 ceph-mon[111235]: osdmap e1596: 8 total, 8 up, 7 in 2024-09-14T05:03:47.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:46 smithi039 ceph-mon[111235]: 1.1e scrub starts 2024-09-14T05:03:47.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:46 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:03:47.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:46 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:03:47.104 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:47.105 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:47.105 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:47.206 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:47.231 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:47.232 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:47.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:47.333 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:47.360 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:47.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:47.361 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:47.462 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:47.496 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:47.497 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:47.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:47.598 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:47.640 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:47.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:47.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:47.742 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:47.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:47 smithi129 ceph-mon[101316]: pgmap v3190: 119 pgs: 2 active+clean+scrubbing, 2 active+remapped+backfill_wait, 4 active+remapped+backfilling, 111 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 822 op/s; 19073/206532 objects misplaced (9.235%); 39 MiB/s, 626 objects/s recovering 2024-09-14T05:03:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:47 smithi129 ceph-mon[101316]: osdmap e1597: 8 total, 8 up, 7 in 2024-09-14T05:03:47.789 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:47.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:47.790 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:47.883 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-14T05:03:47.884 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-14T05:03:47.884 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.1 'trim stale osdmaps' 2024-09-14T05:03:47.891 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:47.917 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:47.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:47.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:47.973 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:47 smithi039 ceph-mon[106470]: pgmap v3190: 119 pgs: 2 active+clean+scrubbing, 2 active+remapped+backfill_wait, 4 active+remapped+backfilling, 111 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 822 op/s; 19073/206532 objects misplaced (9.235%); 39 MiB/s, 626 objects/s recovering 2024-09-14T05:03:47.973 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:47 smithi039 ceph-mon[106470]: osdmap e1597: 8 total, 8 up, 7 in 2024-09-14T05:03:47.974 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:47 smithi039 ceph-mon[111235]: pgmap v3190: 119 pgs: 2 active+clean+scrubbing, 2 active+remapped+backfill_wait, 4 active+remapped+backfilling, 111 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 822 op/s; 19073/206532 objects misplaced (9.235%); 39 MiB/s, 626 objects/s recovering 2024-09-14T05:03:47.974 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:47 smithi039 ceph-mon[111235]: osdmap e1597: 8 total, 8 up, 7 in 2024-09-14T05:03:48.019 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:48.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:48.057 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:48.057 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:48.158 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:48.180 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:48.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:48.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:48.232 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:03:48.282 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:48.344 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:48.345 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:48.345 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:48.446 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:48.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:48.503 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:48.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:48.604 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:48.627 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:48.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:48.628 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:48.729 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:03:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:03:48] "GET /metrics HTTP/1.1" 200 36789 "" "Prometheus/2.51.0" 2024-09-14T05:03:48.729 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:48.753 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:48.754 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:48.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:48.756 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is unset 2024-09-14T05:03:48.855 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:48.883 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:48.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:48.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:48.984 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:49.008 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:49.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:49.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:49.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:48 smithi129 ceph-mon[101316]: 1.7 scrub starts 2024-09-14T05:03:49.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:48 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1451135516' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:03:49.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:48 smithi039 ceph-mon[106470]: 1.7 scrub starts 2024-09-14T05:03:49.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:48 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1451135516' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:03:49.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:48 smithi039 ceph-mon[111235]: 1.7 scrub starts 2024-09-14T05:03:49.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:48 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1451135516' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:03:49.111 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:49.137 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:49.137 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:49.138 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:49.238 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:49.264 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:49.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:49.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:49.366 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:49.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:49.427 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:49.427 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:49.520 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:03:49.527 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:49.618 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:49.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:49.619 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:49.720 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:49.762 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:49.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:49.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:49.864 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:49.905 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:49.906 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:49.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:50.007 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:50.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:49 smithi129 ceph-mon[101316]: pgmap v3192: 119 pgs: 2 active+clean+scrubbing, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 112 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 17154/205206 objects misplaced (8.359%); 38 MiB/s, 611 objects/s recovering 2024-09-14T05:03:50.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:49 smithi129 ceph-mon[101316]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T05:03:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:49 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T05:03:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:49 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1451135516' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T05:03:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:49 smithi129 ceph-mon[101316]: osdmap e1598: 8 total, 8 up, 7 in 2024-09-14T05:03:50.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:49 smithi039 ceph-mon[106470]: pgmap v3192: 119 pgs: 2 active+clean+scrubbing, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 112 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 17154/205206 objects misplaced (8.359%); 38 MiB/s, 611 objects/s recovering 2024-09-14T05:03:50.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:49 smithi039 ceph-mon[106470]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T05:03:50.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:49 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T05:03:50.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:49 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1451135516' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T05:03:50.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:49 smithi039 ceph-mon[106470]: osdmap e1598: 8 total, 8 up, 7 in 2024-09-14T05:03:50.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:49 smithi039 ceph-mon[111235]: pgmap v3192: 119 pgs: 2 active+clean+scrubbing, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 112 active+clean; 4.2 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 703 op/s; 17154/205206 objects misplaced (8.359%); 38 MiB/s, 611 objects/s recovering 2024-09-14T05:03:50.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:49 smithi039 ceph-mon[111235]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T05:03:50.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:49 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T05:03:50.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:49 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1451135516' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T05:03:50.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:49 smithi039 ceph-mon[111235]: osdmap e1598: 8 total, 8 up, 7 in 2024-09-14T05:03:50.058 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:50.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:50.059 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:50.160 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:50.186 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:50.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:50.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:50.288 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:50.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:50.315 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:50.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:50.416 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:50.442 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:50.443 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:50.444 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:50.545 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:50.576 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:50.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:50.577 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:50.678 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:50.719 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:50.720 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:50.721 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:50.821 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:50.857 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:50.858 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:50.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:50.959 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:50.994 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:50 smithi129 ceph-mon[101316]: 1.1e scrub starts 2024-09-14T05:03:51.011 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:50 smithi039 ceph-mon[106470]: 1.1e scrub starts 2024-09-14T05:03:51.012 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:50 smithi039 ceph-mon[111235]: 1.1e scrub starts 2024-09-14T05:03:51.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:51.013 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:51.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:51.023 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T05:03:51.023 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:03:51.113 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:51.176 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:51.177 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:51.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:51.278 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:51.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:51.305 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:51.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:51.407 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:51.451 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:51.451 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:51.452 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:51.553 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:51.604 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:51.605 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:51.606 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:51.706 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:51.747 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-14T05:03:51.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:51.762 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:51.762 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:51.863 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:51.874 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:51 smithi039 ceph-mon[106470]: pgmap v3194: 119 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 113 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 940 op/s; 15055/199038 objects misplaced (7.564%); 46 MiB/s, 735 objects/s recovering 2024-09-14T05:03:51.875 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:51 smithi039 ceph-mon[111235]: pgmap v3194: 119 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 113 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 940 op/s; 15055/199038 objects misplaced (7.564%); 46 MiB/s, 735 objects/s recovering 2024-09-14T05:03:51.902 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:51.903 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:51.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:52.004 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:52.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:51 smithi129 ceph-mon[101316]: pgmap v3194: 119 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 113 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 940 op/s; 15055/199038 objects misplaced (7.564%); 46 MiB/s, 735 objects/s recovering 2024-09-14T05:03:52.024 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:03:52.038 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:52.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:52.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:52.140 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:52.163 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T05:03:52.187 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:52.188 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:52.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:52.289 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:52.317 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:52.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:52.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:52.419 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:52.447 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:52.448 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:52.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:52.549 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:52.593 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:52.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:52.594 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:52.695 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:52.741 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:52.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:52.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:52.843 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:52.869 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:52.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:52.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:52.971 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:52.997 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:52.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:52.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:53.098 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:53.126 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:53.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:53.127 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:53.228 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:53.254 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:53.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:53.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:53.356 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:53.390 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:53.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:53.391 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:53.492 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:53.526 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:53.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:53.528 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:53.628 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:53.632 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:53 smithi129 ceph-mon[101316]: pgmap v3195: 119 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 113 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 679 op/s; 14846/196545 objects misplaced (7.553%); 33 MiB/s, 532 objects/s recovering 2024-09-14T05:03:53.632 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:53 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:03:53.632 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:53 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:03:53.667 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:53.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:53.668 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:53.756 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:53 smithi039 ceph-mon[106470]: pgmap v3195: 119 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 113 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 679 op/s; 14846/196545 objects misplaced (7.553%); 33 MiB/s, 532 objects/s recovering 2024-09-14T05:03:53.756 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:53 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:03:53.756 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:53 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:03:53.757 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:53 smithi039 ceph-mon[111235]: pgmap v3195: 119 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 113 active+clean; 4.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 679 op/s; 14846/196545 objects misplaced (7.553%); 33 MiB/s, 532 objects/s recovering 2024-09-14T05:03:53.757 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:53 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:03:53.757 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:53 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:03:53.769 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:53.798 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:53.799 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:53.799 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:53.900 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:53.925 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T05:03:53.925 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:03:53.935 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:53.935 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:53.936 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:54.036 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:54.081 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:54.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:54.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:54.183 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:54.208 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:54.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:54.210 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:54.310 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:54.331 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:54.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:54.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:54.433 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:54.473 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:54.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:54.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:54.575 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:54.609 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:54.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:54.610 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:54.711 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:54.743 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:54.744 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:54.744 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:54.845 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:54.891 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-14T05:03:54.940 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:54.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:54.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:55.041 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:55.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:55.073 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:55.073 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:55.150 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:03:55.174 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:55.220 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:55.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:55.221 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:55.321 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:55.363 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:55.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:55.364 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:55.465 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:55.496 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:55.496 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:55.497 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:55.598 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:55.644 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:55.645 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:55.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:55.747 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:55.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:55 smithi129 ceph-mon[101316]: pgmap v3196: 119 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 113 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 538 op/s; 14234/193611 objects misplaced (7.352%); 30 MiB/s, 483 objects/s recovering 2024-09-14T05:03:55.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:55 smithi039 ceph-mon[106470]: pgmap v3196: 119 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 113 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 538 op/s; 14234/193611 objects misplaced (7.352%); 30 MiB/s, 483 objects/s recovering 2024-09-14T05:03:55.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:55 smithi039 ceph-mon[111235]: pgmap v3196: 119 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 113 active+clean; 3.9 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 538 op/s; 14234/193611 objects misplaced (7.352%); 30 MiB/s, 483 objects/s recovering 2024-09-14T05:03:55.796 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:55.796 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:55.797 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:55.898 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:55.942 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:55.943 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:55.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:56.044 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:56.070 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:56.071 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:56.071 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:56.172 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:56.198 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:56.199 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:56.199 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:56.300 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:56.337 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:56.338 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:56.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:56.439 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:56.467 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:56.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:56.468 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:56.568 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:03:56.615 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:56.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:56.616 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:03:56.716 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:56.754 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:56.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:56.755 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:56.856 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:56.904 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:56.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:56.905 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:57.006 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:57.035 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:57.036 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:57.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:57.137 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:57.166 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:57.167 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:57.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:57.268 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:57.300 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:57.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:57.301 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:57.349 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T05:03:57.349 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:03:57.402 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:57.424 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:57.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:57.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:57.526 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:57.552 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:57.552 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:57.553 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:57.654 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:57.689 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:57.690 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:57 smithi129 ceph-mon[101316]: 1.7 scrub starts 2024-09-14T05:03:57.690 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:57 smithi129 ceph-mon[101316]: pgmap v3197: 119 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 113 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 814 op/s; 12438/183711 objects misplaced (6.770%); 33 MiB/s, 521 objects/s recovering 2024-09-14T05:03:57.692 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:57.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:57.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:57 smithi039 ceph-mon[106470]: 1.7 scrub starts 2024-09-14T05:03:57.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:57 smithi039 ceph-mon[106470]: pgmap v3197: 119 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 113 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 814 op/s; 12438/183711 objects misplaced (6.770%); 33 MiB/s, 521 objects/s recovering 2024-09-14T05:03:57.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:57 smithi039 ceph-mon[111235]: 1.7 scrub starts 2024-09-14T05:03:57.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:57 smithi039 ceph-mon[111235]: pgmap v3197: 119 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 113 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 814 op/s; 12438/183711 objects misplaced (6.770%); 33 MiB/s, 521 objects/s recovering 2024-09-14T05:03:57.794 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:57.818 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:57.819 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:57.819 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:57.920 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:57.965 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:57.965 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:57.966 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:58.031 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-14T05:03:58.066 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:58.103 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:58.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:58.104 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:58.204 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:58.240 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:58.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:58.241 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:58.284 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:03:58.343 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:58.390 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:58.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:58.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:58.492 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:58.536 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:58.536 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:58.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:58.637 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:58.658 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:58.659 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:58.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:58.760 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:03:58.789 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:58.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:58.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:03:58.891 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:03:58.916 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:58.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:58.917 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:03:58.950 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:03:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:03:58] "GET /metrics HTTP/1.1" 200 36780 "" "Prometheus/2.51.0" 2024-09-14T05:03:59.018 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:59.081 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:59.081 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:59.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:03:59.182 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:03:59.207 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:59.208 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:59.208 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:03:59.309 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:59.350 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:59.351 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:59.351 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:59.452 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:03:59.472 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T05:03:59.472 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T05:03:59.473 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:03:59 smithi039 ceph-mon[106470]: pgmap v3198: 119 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 113 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 770 op/s; 11890/182091 objects misplaced (6.530%); 22 MiB/s, 354 objects/s recovering 2024-09-14T05:03:59.473 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:03:59 smithi039 ceph-mon[111235]: pgmap v3198: 119 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 113 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 770 op/s; 11890/182091 objects misplaced (6.530%); 22 MiB/s, 354 objects/s recovering 2024-09-14T05:03:59.474 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:03:59.475 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:59.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:03:59.505 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T05:03:59.505 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T05:03:59.550 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T05:03:59.550 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T05:03:59.576 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:03:59.586 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T05:03:59.586 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T05:03:59.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:59.620 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:59.621 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:03:59.621 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T05:03:59.621 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T05:03:59.658 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T05:03:59.658 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T05:03:59.684 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:03:59 smithi129 ceph-mon[101316]: pgmap v3198: 119 pgs: 1 active+clean+scrubbing, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 113 active+clean; 3.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 770 op/s; 11890/182091 objects misplaced (6.530%); 22 MiB/s, 354 objects/s recovering 2024-09-14T05:03:59.691 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T05:03:59.692 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T05:03:59.722 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:03:59.726 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T05:03:59.726 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T05:03:59.726 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:03:59.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:59.803 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:59.803 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:03:59.904 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:03:59.960 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:03:59.961 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:03:59.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:00.062 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:00.088 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:00.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:00.089 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:00.190 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:00.220 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:00.220 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:00.220 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:00.321 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:00.364 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:00.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:00.365 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:00.449 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T05:04:00.449 INFO:teuthology.orchestra.run.smithi129.stdout: "size": 20, 2024-09-14T05:04:00.449 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 600, 2024-09-14T05:04:00.449 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [ 2024-09-14T05:04:00.449 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.450 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36324.0:34423 57.c 57:3a9545c4:::benchmark_data_smithi039_365565_object34422:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T05:04:00.450 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:57:13.476213+0000", 2024-09-14T05:04:00.450 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 406.906607898, 2024-09-14T05:04:00.450 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.42755653100000002, 2024-09-14T05:04:00.450 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:00.450 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:00.450 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:00.450 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36324", 2024-09-14T05:04:00.450 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:04:00.450 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 34423 2024-09-14T05:04:00.450 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.450 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:00.450 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.451 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:00.451 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.476213+0000", 2024-09-14T05:04:00.451 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.451 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.451 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.451 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:00.451 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.476213+0000", 2024-09-14T05:04:00.451 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.451 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.451 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.452 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:00.452 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.476212+0000", 2024-09-14T05:04:00.452 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T05:04:00.452 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.452 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.452 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:00.452 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.476260+0000", 2024-09-14T05:04:00.452 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8439000000000002e-05 2024-09-14T05:04:00.452 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.452 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.452 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:00.452 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.476294+0000", 2024-09-14T05:04:00.452 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3504000000000003e-05 2024-09-14T05:04:00.453 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.453 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.453 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:00.453 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.476302+0000", 2024-09-14T05:04:00.453 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.3129999999999999e-06 2024-09-14T05:04:00.453 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.453 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.453 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:00.453 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.476429+0000", 2024-09-14T05:04:00.453 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000127224 2024-09-14T05:04:00.454 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.454 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.454 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:00.454 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.476559+0000", 2024-09-14T05:04:00.454 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012968300000000001 2024-09-14T05:04:00.454 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.454 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.454 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,4", 2024-09-14T05:04:00.454 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.476698+0000", 2024-09-14T05:04:00.454 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013924200000000001 2024-09-14T05:04:00.454 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.454 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.454 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:00.455 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.478850+0000", 2024-09-14T05:04:00.455 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0021517239999999998 2024-09-14T05:04:00.455 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.455 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.455 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:00.455 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.481737+0000", 2024-09-14T05:04:00.455 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.002887321 2024-09-14T05:04:00.455 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.455 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.455 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:00.456 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.903650+0000", 2024-09-14T05:04:00.456 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.42191207800000002 2024-09-14T05:04:00.456 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.456 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.456 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:00.456 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.903725+0000", 2024-09-14T05:04:00.456 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.5860999999999998e-05 2024-09-14T05:04:00.456 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.456 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.456 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:00.457 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.903770+0000", 2024-09-14T05:04:00.457 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.4517000000000001e-05 2024-09-14T05:04:00.457 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.457 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:00.457 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.457 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.457 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.457 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36324.0:34441 57.c 57:369fd8ff:::benchmark_data_smithi039_365565_object34440:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T05:04:00.457 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:57:13.484391+0000", 2024-09-14T05:04:00.457 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 406.89842991699999, 2024-09-14T05:04:00.457 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.42049165100000002, 2024-09-14T05:04:00.457 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:00.458 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:00.458 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:00.458 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36324", 2024-09-14T05:04:00.458 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:04:00.458 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 34441 2024-09-14T05:04:00.458 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.458 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:00.458 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.458 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:00.458 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.484391+0000", 2024-09-14T05:04:00.459 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.459 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.459 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.459 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:00.459 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.484391+0000", 2024-09-14T05:04:00.459 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.459 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.459 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.459 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:00.459 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.484387+0000", 2024-09-14T05:04:00.459 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999957 2024-09-14T05:04:00.460 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.460 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.460 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:00.460 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.484456+0000", 2024-09-14T05:04:00.460 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.9098999999999997e-05 2024-09-14T05:04:00.460 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.460 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.460 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:00.460 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.484474+0000", 2024-09-14T05:04:00.460 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.7487e-05 2024-09-14T05:04:00.461 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.461 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.461 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:00.461 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.484483+0000", 2024-09-14T05:04:00.461 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.1349999999999998e-06 2024-09-14T05:04:00.461 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.461 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.461 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:00.461 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.484532+0000", 2024-09-14T05:04:00.461 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.9502999999999997e-05 2024-09-14T05:04:00.461 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.461 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.461 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:00.462 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.484681+0000", 2024-09-14T05:04:00.462 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000148658 2024-09-14T05:04:00.462 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.462 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.462 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,4", 2024-09-14T05:04:00.462 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.484835+0000", 2024-09-14T05:04:00.462 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00015358099999999999 2024-09-14T05:04:00.462 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.462 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.462 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:00.462 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.486249+0000", 2024-09-14T05:04:00.462 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001414228 2024-09-14T05:04:00.463 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.463 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.463 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:00.463 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.487023+0000", 2024-09-14T05:04:00.463 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00077426800000000005 2024-09-14T05:04:00.463 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.463 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.463 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:00.463 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.904769+0000", 2024-09-14T05:04:00.463 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.41774598200000002 2024-09-14T05:04:00.463 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.463 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.463 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:00.464 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.904841+0000", 2024-09-14T05:04:00.464 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.2271e-05 2024-09-14T05:04:00.464 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.464 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.464 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:00.464 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.904883+0000", 2024-09-14T05:04:00.464 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1706999999999999e-05 2024-09-14T05:04:00.464 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.464 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:00.464 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.464 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.464 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.464 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36324.0:34473 57.c 57:30f8a30d:::benchmark_data_smithi039_365565_object34472:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T05:04:00.464 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:57:13.512226+0000", 2024-09-14T05:04:00.465 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 406.87059509400001, 2024-09-14T05:04:00.465 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.39696483399999999, 2024-09-14T05:04:00.465 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:00.465 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:00.465 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:00.465 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36324", 2024-09-14T05:04:00.465 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:04:00.465 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 34473 2024-09-14T05:04:00.465 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.465 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:00.465 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.465 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:00.466 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.512226+0000", 2024-09-14T05:04:00.466 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.466 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.466 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.466 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:00.466 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.512226+0000", 2024-09-14T05:04:00.466 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.466 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.466 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.466 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:00.466 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.512224+0000", 2024-09-14T05:04:00.466 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T05:04:00.466 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.467 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.467 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:00.467 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.512288+0000", 2024-09-14T05:04:00.467 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.3947000000000003e-05 2024-09-14T05:04:00.467 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.467 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.467 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:00.467 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.512299+0000", 2024-09-14T05:04:00.467 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0926e-05 2024-09-14T05:04:00.467 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.467 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.467 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:00.467 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.512304+0000", 2024-09-14T05:04:00.467 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7500000000000003e-06 2024-09-14T05:04:00.468 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.468 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.468 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:00.468 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.512342+0000", 2024-09-14T05:04:00.468 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8278999999999997e-05 2024-09-14T05:04:00.468 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.468 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.468 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:00.468 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.512436+0000", 2024-09-14T05:04:00.468 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.3912999999999993e-05 2024-09-14T05:04:00.468 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.468 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.469 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,4", 2024-09-14T05:04:00.469 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.512529+0000", 2024-09-14T05:04:00.469 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.3244000000000004e-05 2024-09-14T05:04:00.469 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.469 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.469 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:00.469 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.513627+0000", 2024-09-14T05:04:00.469 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001097563 2024-09-14T05:04:00.469 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.469 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.469 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:00.470 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.513873+0000", 2024-09-14T05:04:00.470 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00024622899999999999 2024-09-14T05:04:00.470 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.470 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.470 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:00.470 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.909058+0000", 2024-09-14T05:04:00.470 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.39518478400000001 2024-09-14T05:04:00.470 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.470 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.470 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:00.470 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.909151+0000", 2024-09-14T05:04:00.470 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.3219000000000003e-05 2024-09-14T05:04:00.470 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.471 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.471 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:00.471 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:13.909191+0000", 2024-09-14T05:04:00.471 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9906e-05 2024-09-14T05:04:00.471 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.471 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:00.471 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.471 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.471 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.471 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36324.0:34595 57.c 57:361311a4:::benchmark_data_smithi039_365565_object34594:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T05:04:00.471 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:57:14.021447+0000", 2024-09-14T05:04:00.471 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 406.36137423899999, 2024-09-14T05:04:00.471 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.22964048400000001, 2024-09-14T05:04:00.472 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:00.472 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:00.472 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:00.472 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36324", 2024-09-14T05:04:00.472 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:04:00.472 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 34595 2024-09-14T05:04:00.472 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.472 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:00.472 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.472 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:00.472 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.021447+0000", 2024-09-14T05:04:00.472 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.472 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.473 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.473 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:00.473 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.021447+0000", 2024-09-14T05:04:00.473 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.473 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.473 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.473 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:00.473 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.021447+0000", 2024-09-14T05:04:00.473 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T05:04:00.473 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.473 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.473 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:00.473 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.021474+0000", 2024-09-14T05:04:00.473 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.7749000000000001e-05 2024-09-14T05:04:00.474 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.474 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.474 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:00.474 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.021481+0000", 2024-09-14T05:04:00.474 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.579e-06 2024-09-14T05:04:00.474 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.474 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.474 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:00.474 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.021485+0000", 2024-09-14T05:04:00.474 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.578e-06 2024-09-14T05:04:00.474 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.474 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.474 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:00.474 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.026116+0000", 2024-09-14T05:04:00.474 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0046308970000000001 2024-09-14T05:04:00.475 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.475 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.475 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:00.475 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.026257+0000", 2024-09-14T05:04:00.475 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0001405 2024-09-14T05:04:00.475 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.475 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.475 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 2,4", 2024-09-14T05:04:00.475 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.026402+0000", 2024-09-14T05:04:00.475 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000144926 2024-09-14T05:04:00.475 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.475 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.475 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:00.476 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.028224+0000", 2024-09-14T05:04:00.476 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0018217579999999999 2024-09-14T05:04:00.476 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.476 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.476 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:00.476 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.028285+0000", 2024-09-14T05:04:00.476 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.1548000000000001e-05 2024-09-14T05:04:00.476 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.476 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.476 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:00.476 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029389+0000", 2024-09-14T05:04:00.476 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0011035019999999999 2024-09-14T05:04:00.476 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.476 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.477 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:00.477 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029428+0000", 2024-09-14T05:04:00.477 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8893000000000002e-05 2024-09-14T05:04:00.477 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.477 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.477 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:00.477 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.251087+0000", 2024-09-14T05:04:00.477 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.22165997700000001 2024-09-14T05:04:00.477 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.477 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:00.477 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.477 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.477 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.477 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36324.0:34605 57.d 57:b3cca012:::benchmark_data_smithi039_365565_object34604:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T05:04:00.478 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:57:14.027054+0000", 2024-09-14T05:04:00.478 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 406.35576744299999, 2024-09-14T05:04:00.478 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.22652098900000001, 2024-09-14T05:04:00.478 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:00.478 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:00.478 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:00.478 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36324", 2024-09-14T05:04:00.478 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:04:00.478 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 34605 2024-09-14T05:04:00.478 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.478 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:00.478 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.478 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:00.478 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.027054+0000", 2024-09-14T05:04:00.478 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.479 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.479 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.479 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:00.479 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.027054+0000", 2024-09-14T05:04:00.479 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.479 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.479 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.479 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:00.479 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.027052+0000", 2024-09-14T05:04:00.479 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T05:04:00.479 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.479 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.479 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:00.479 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.027173+0000", 2024-09-14T05:04:00.480 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000120345 2024-09-14T05:04:00.480 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.480 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.480 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:00.480 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.027185+0000", 2024-09-14T05:04:00.480 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1918e-05 2024-09-14T05:04:00.480 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.480 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.480 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:00.480 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.027192+0000", 2024-09-14T05:04:00.480 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.0140000000000001e-06 2024-09-14T05:04:00.480 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.480 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.480 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:00.481 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.027280+0000", 2024-09-14T05:04:00.481 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.8094000000000001e-05 2024-09-14T05:04:00.481 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.481 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.481 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:00.481 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.027416+0000", 2024-09-14T05:04:00.481 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013593000000000001 2024-09-14T05:04:00.481 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.481 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.481 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,3", 2024-09-14T05:04:00.481 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.027552+0000", 2024-09-14T05:04:00.481 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0001362 2024-09-14T05:04:00.482 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.482 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.482 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:00.482 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029100+0000", 2024-09-14T05:04:00.482 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0015484850000000001 2024-09-14T05:04:00.482 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.482 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.482 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:00.482 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.253444+0000", 2024-09-14T05:04:00.482 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.22434352599999999 2024-09-14T05:04:00.483 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.483 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.483 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:00.483 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.253479+0000", 2024-09-14T05:04:00.483 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5698000000000003e-05 2024-09-14T05:04:00.483 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.483 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.483 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:00.483 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.253533+0000", 2024-09-14T05:04:00.483 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.3789999999999998e-05 2024-09-14T05:04:00.483 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.483 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.483 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:00.484 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.253575+0000", 2024-09-14T05:04:00.484 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1539999999999999e-05 2024-09-14T05:04:00.484 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.484 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:00.484 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.484 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.484 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.484 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36324.0:34612 57.d 57:b07fe074:::benchmark_data_smithi039_365565_object34611:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1523)", 2024-09-14T05:04:00.484 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:57:14.029179+0000", 2024-09-14T05:04:00.485 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 406.35364265800001, 2024-09-14T05:04:00.485 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.225134005, 2024-09-14T05:04:00.485 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:00.485 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:00.485 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:00.485 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36324", 2024-09-14T05:04:00.485 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:04:00.485 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 34612 2024-09-14T05:04:00.485 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.485 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:00.485 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.485 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:00.485 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029179+0000", 2024-09-14T05:04:00.485 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.485 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.486 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.486 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:00.486 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029179+0000", 2024-09-14T05:04:00.486 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.486 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.486 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.486 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:00.486 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029178+0000", 2024-09-14T05:04:00.486 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T05:04:00.487 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.487 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.487 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:00.487 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029297+0000", 2024-09-14T05:04:00.487 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00011915499999999999 2024-09-14T05:04:00.487 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.487 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.487 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:00.487 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029306+0000", 2024-09-14T05:04:00.487 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.8939999999999999e-06 2024-09-14T05:04:00.487 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.487 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.487 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:00.487 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029312+0000", 2024-09-14T05:04:00.487 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9510000000000003e-06 2024-09-14T05:04:00.488 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.488 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.488 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:00.488 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029349+0000", 2024-09-14T05:04:00.488 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7515999999999998e-05 2024-09-14T05:04:00.488 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.488 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.488 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:00.488 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.251164+0000", 2024-09-14T05:04:00.488 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.221814756 2024-09-14T05:04:00.488 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.489 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.489 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,3", 2024-09-14T05:04:00.489 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.251342+0000", 2024-09-14T05:04:00.489 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00017786399999999999 2024-09-14T05:04:00.489 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.489 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.489 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:00.489 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.253624+0000", 2024-09-14T05:04:00.489 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0022818869999999998 2024-09-14T05:04:00.489 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.489 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.490 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:00.490 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.254229+0000", 2024-09-14T05:04:00.490 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00060513400000000001 2024-09-14T05:04:00.490 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.490 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.490 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:00.490 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.254235+0000", 2024-09-14T05:04:00.490 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.2339999999999999e-06 2024-09-14T05:04:00.490 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.490 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.491 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:00.491 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.254279+0000", 2024-09-14T05:04:00.491 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.4138999999999999e-05 2024-09-14T05:04:00.491 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.491 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.491 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:00.491 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.254313+0000", 2024-09-14T05:04:00.491 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3535000000000003e-05 2024-09-14T05:04:00.491 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.491 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:00.492 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.492 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.492 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.492 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.36324.0:34606 57.3 e1523/1518 57:c42218af:::benchmark_data_smithi039_365565_object34605:head v 1523'2119, mlcod=1523'2116)", 2024-09-14T05:04:00.492 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T04:57:14.029786+0000", 2024-09-14T05:04:00.492 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 406.35303484899998, 2024-09-14T05:04:00.492 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.22630244299999999, 2024-09-14T05:04:00.492 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:00.492 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:00.492 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:00.492 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.492 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:00.492 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029786+0000", 2024-09-14T05:04:00.493 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.493 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.493 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.493 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:00.493 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029786+0000", 2024-09-14T05:04:00.493 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.493 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.493 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.493 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:00.493 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029786+0000", 2024-09-14T05:04:00.493 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T05:04:00.493 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.493 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.493 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:00.494 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.029846+0000", 2024-09-14T05:04:00.494 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.0751999999999997e-05 2024-09-14T05:04:00.494 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.494 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.494 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:00.494 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.251086+0000", 2024-09-14T05:04:00.494 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.22123975300000001 2024-09-14T05:04:00.494 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.494 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.494 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:00.494 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.251096+0000", 2024-09-14T05:04:00.494 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.9909999999999997e-06 2024-09-14T05:04:00.494 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.494 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.495 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:00.495 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.251165+0000", 2024-09-14T05:04:00.495 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.9189000000000005e-05 2024-09-14T05:04:00.495 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.495 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.495 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:00.495 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.251187+0000", 2024-09-14T05:04:00.495 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.1908000000000001e-05 2024-09-14T05:04:00.495 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.495 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.495 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:00.495 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.256055+0000", 2024-09-14T05:04:00.495 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0048677219999999997 2024-09-14T05:04:00.495 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.496 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.496 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:00.496 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T04:57:14.256089+0000", 2024-09-14T05:04:00.496 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3899000000000002e-05 2024-09-14T05:04:00.496 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.496 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:00.496 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.496 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.496 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.497 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36603.0:29195 58.1 58:8352b54a:::benchmark_data_smithi039_396950_object29194:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1572)", 2024-09-14T05:04:00.497 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:01:28.856024+0000", 2024-09-14T05:04:00.497 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 151.52679719100001, 2024-09-14T05:04:00.497 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.310542718, 2024-09-14T05:04:00.497 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:00.497 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:00.497 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:00.497 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36603", 2024-09-14T05:04:00.497 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:04:00.497 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 29195 2024-09-14T05:04:00.497 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.498 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:00.498 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.498 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:00.498 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.856024+0000", 2024-09-14T05:04:00.498 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.498 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.498 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.498 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:00.498 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.856024+0000", 2024-09-14T05:04:00.498 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.498 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.498 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.498 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:00.498 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.856023+0000", 2024-09-14T05:04:00.499 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T05:04:00.499 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.499 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.499 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:00.499 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.856066+0000", 2024-09-14T05:04:00.499 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.3169e-05 2024-09-14T05:04:00.499 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.499 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.499 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:00.499 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.856078+0000", 2024-09-14T05:04:00.499 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2065000000000001e-05 2024-09-14T05:04:00.499 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.499 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.500 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:00.500 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.856085+0000", 2024-09-14T05:04:00.500 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7390000000000002e-06 2024-09-14T05:04:00.500 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.500 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.500 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:00.500 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.856160+0000", 2024-09-14T05:04:00.500 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.5158000000000001e-05 2024-09-14T05:04:00.500 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.500 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.500 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:00.500 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.856309+0000", 2024-09-14T05:04:00.500 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00014852299999999999 2024-09-14T05:04:00.500 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.501 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.501 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,1", 2024-09-14T05:04:00.501 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.856436+0000", 2024-09-14T05:04:00.501 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012709200000000001 2024-09-14T05:04:00.501 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.501 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.501 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:00.501 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.857607+0000", 2024-09-14T05:04:00.501 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00117103 2024-09-14T05:04:00.501 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.501 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.502 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:00.502 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.861532+0000", 2024-09-14T05:04:00.502 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0039252030000000004 2024-09-14T05:04:00.502 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.502 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.502 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:00.502 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:29.166466+0000", 2024-09-14T05:04:00.502 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.304933342 2024-09-14T05:04:00.502 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.502 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.502 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:00.502 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:29.166522+0000", 2024-09-14T05:04:00.502 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.6073999999999999e-05 2024-09-14T05:04:00.502 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.502 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.502 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:00.503 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:29.166567+0000", 2024-09-14T05:04:00.503 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.5175e-05 2024-09-14T05:04:00.503 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.503 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:00.503 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.503 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.503 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.503 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36603.0:29200 58.1 58:87537026:::benchmark_data_smithi039_396950_object29199:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1572)", 2024-09-14T05:04:00.503 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:01:28.860236+0000", 2024-09-14T05:04:00.503 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 151.52258529100001, 2024-09-14T05:04:00.503 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.30642546799999998, 2024-09-14T05:04:00.503 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:00.503 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:00.503 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:00.504 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36603", 2024-09-14T05:04:00.504 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:04:00.504 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 29200 2024-09-14T05:04:00.504 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.504 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:00.504 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.504 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:00.504 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.860236+0000", 2024-09-14T05:04:00.504 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.504 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.504 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.504 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:00.504 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.860236+0000", 2024-09-14T05:04:00.504 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.504 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.505 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.505 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:00.505 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.860235+0000", 2024-09-14T05:04:00.505 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T05:04:00.505 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.505 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.505 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:00.505 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.860301+0000", 2024-09-14T05:04:00.505 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.5332000000000004e-05 2024-09-14T05:04:00.505 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.505 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.505 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:00.505 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.860307+0000", 2024-09-14T05:04:00.505 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.2890000000000003e-06 2024-09-14T05:04:00.505 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.506 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.506 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:00.506 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.860311+0000", 2024-09-14T05:04:00.506 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.4699999999999998e-06 2024-09-14T05:04:00.506 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.506 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.506 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:00.506 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.860351+0000", 2024-09-14T05:04:00.506 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0859999999999998e-05 2024-09-14T05:04:00.506 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.506 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.506 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:00.506 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.860480+0000", 2024-09-14T05:04:00.506 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012849199999999999 2024-09-14T05:04:00.506 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.507 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.507 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,1", 2024-09-14T05:04:00.507 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.860614+0000", 2024-09-14T05:04:00.507 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013413100000000001 2024-09-14T05:04:00.507 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.507 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.507 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:00.507 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.861594+0000", 2024-09-14T05:04:00.507 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00098005700000000011 2024-09-14T05:04:00.507 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.507 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.507 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:00.507 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.865799+0000", 2024-09-14T05:04:00.508 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0042050539999999997 2024-09-14T05:04:00.508 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.508 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.508 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:00.508 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:29.166604+0000", 2024-09-14T05:04:00.508 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.30080505600000002 2024-09-14T05:04:00.508 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.508 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.508 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:00.508 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:29.166637+0000", 2024-09-14T05:04:00.508 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.2487000000000003e-05 2024-09-14T05:04:00.508 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.508 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.508 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:00.509 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:29.166661+0000", 2024-09-14T05:04:00.509 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.4752e-05 2024-09-14T05:04:00.509 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.509 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:00.509 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.509 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.509 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.509 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36603.0:29220 58.1 58:8b2df792:::benchmark_data_smithi039_396950_object29219:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1572)", 2024-09-14T05:04:00.509 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:01:28.871058+0000", 2024-09-14T05:04:00.509 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 151.51176291900001, 2024-09-14T05:04:00.509 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.296275862, 2024-09-14T05:04:00.509 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:00.509 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:00.510 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:00.510 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36603", 2024-09-14T05:04:00.510 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:04:00.510 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 29220 2024-09-14T05:04:00.510 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.510 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:00.510 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.510 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:00.510 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.871058+0000", 2024-09-14T05:04:00.510 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.510 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.510 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.510 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:00.510 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.871058+0000", 2024-09-14T05:04:00.511 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.511 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.511 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.511 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:00.511 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.871057+0000", 2024-09-14T05:04:00.511 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T05:04:00.511 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.511 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.511 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:00.511 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.871137+0000", 2024-09-14T05:04:00.511 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.9458999999999999e-05 2024-09-14T05:04:00.511 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.511 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.511 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:00.511 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.871150+0000", 2024-09-14T05:04:00.512 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3431000000000001e-05 2024-09-14T05:04:00.512 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.512 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.512 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:00.512 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.871157+0000", 2024-09-14T05:04:00.512 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.99e-06 2024-09-14T05:04:00.512 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.512 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.512 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:00.512 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.871210+0000", 2024-09-14T05:04:00.512 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.3383e-05 2024-09-14T05:04:00.512 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.512 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.513 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:00.513 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.871352+0000", 2024-09-14T05:04:00.513 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000141688 2024-09-14T05:04:00.513 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.513 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.513 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,1", 2024-09-14T05:04:00.513 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.871545+0000", 2024-09-14T05:04:00.513 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000192449 2024-09-14T05:04:00.513 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.513 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.513 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:00.513 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.873065+0000", 2024-09-14T05:04:00.513 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0015203409999999999 2024-09-14T05:04:00.513 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.514 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.514 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:00.514 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.875654+0000", 2024-09-14T05:04:00.514 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0025887969999999999 2024-09-14T05:04:00.514 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.514 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.514 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:00.514 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:29.167237+0000", 2024-09-14T05:04:00.514 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.29158318100000002 2024-09-14T05:04:00.514 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.514 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.514 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:00.514 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:29.167295+0000", 2024-09-14T05:04:00.514 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.7606999999999999e-05 2024-09-14T05:04:00.514 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.514 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.515 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:00.515 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:29.167334+0000", 2024-09-14T05:04:00.515 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9681999999999997e-05 2024-09-14T05:04:00.515 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.515 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:00.515 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.515 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.515 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.515 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36603.0:29227 58.1 58:88560762:::benchmark_data_smithi039_396950_object29226:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1572)", 2024-09-14T05:04:00.515 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:01:28.874103+0000", 2024-09-14T05:04:00.515 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 151.50871791200001, 2024-09-14T05:04:00.515 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.29338276299999999, 2024-09-14T05:04:00.515 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:00.515 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:00.515 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:00.515 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36603", 2024-09-14T05:04:00.516 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:04:00.516 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 29227 2024-09-14T05:04:00.516 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.516 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:00.516 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.516 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:00.516 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.874103+0000", 2024-09-14T05:04:00.516 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.516 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.516 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.516 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:00.516 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.874103+0000", 2024-09-14T05:04:00.516 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.516 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.516 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.517 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:00.517 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.874102+0000", 2024-09-14T05:04:00.517 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T05:04:00.517 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.517 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.517 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:00.517 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.874202+0000", 2024-09-14T05:04:00.517 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000100204 2024-09-14T05:04:00.517 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.517 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.517 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:00.517 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.874216+0000", 2024-09-14T05:04:00.517 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.4618e-05 2024-09-14T05:04:00.517 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.517 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.517 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:00.518 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.874225+0000", 2024-09-14T05:04:00.518 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.1300000000000001e-06 2024-09-14T05:04:00.518 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.518 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.518 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:00.518 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.874274+0000", 2024-09-14T05:04:00.518 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.9798e-05 2024-09-14T05:04:00.518 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.518 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.518 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:00.519 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.874415+0000", 2024-09-14T05:04:00.519 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000140197 2024-09-14T05:04:00.519 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.519 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.519 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,1", 2024-09-14T05:04:00.519 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.874537+0000", 2024-09-14T05:04:00.519 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012257699999999999 2024-09-14T05:04:00.519 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.519 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.519 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:00.519 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.876828+0000", 2024-09-14T05:04:00.519 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.002290814 2024-09-14T05:04:00.520 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.520 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.520 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:00.520 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.877786+0000", 2024-09-14T05:04:00.520 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00095802599999999995 2024-09-14T05:04:00.520 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.520 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.520 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:00.520 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:29.167379+0000", 2024-09-14T05:04:00.520 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.28959310799999999 2024-09-14T05:04:00.520 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.520 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.521 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:00.521 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:29.167451+0000", 2024-09-14T05:04:00.521 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.1947999999999996e-05 2024-09-14T05:04:00.521 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.521 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.521 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:00.521 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:29.167486+0000", 2024-09-14T05:04:00.521 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5098999999999997e-05 2024-09-14T05:04:00.521 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.521 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:00.521 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.521 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.521 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.522 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36603.0:29230 58.1 58:8d246eda:::benchmark_data_smithi039_396950_object29229:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1572)", 2024-09-14T05:04:00.522 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:01:28.875602+0000", 2024-09-14T05:04:00.522 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 151.50721911799999, 2024-09-14T05:04:00.522 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.29201587499999998, 2024-09-14T05:04:00.522 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:00.522 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:00.522 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:00.522 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36603", 2024-09-14T05:04:00.522 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:04:00.522 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 29230 2024-09-14T05:04:00.523 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.523 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:00.523 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.523 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:00.523 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.875602+0000", 2024-09-14T05:04:00.523 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.523 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.523 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.523 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:00.523 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.875602+0000", 2024-09-14T05:04:00.524 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.524 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.524 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.524 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:00.524 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.875601+0000", 2024-09-14T05:04:00.524 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T05:04:00.524 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.524 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.524 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:00.524 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.875656+0000", 2024-09-14T05:04:00.525 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.4693e-05 2024-09-14T05:04:00.525 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.525 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.525 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:00.525 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.875671+0000", 2024-09-14T05:04:00.525 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5089000000000001e-05 2024-09-14T05:04:00.525 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.525 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.525 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:00.525 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.875679+0000", 2024-09-14T05:04:00.525 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.2190000000000004e-06 2024-09-14T05:04:00.525 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.526 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.526 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:00.526 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.875722+0000", 2024-09-14T05:04:00.526 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.2895999999999997e-05 2024-09-14T05:04:00.526 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.526 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.526 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:00.526 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.875856+0000", 2024-09-14T05:04:00.526 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000134547 2024-09-14T05:04:00.526 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.527 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.527 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,1", 2024-09-14T05:04:00.527 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.875997+0000", 2024-09-14T05:04:00.527 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000140297 2024-09-14T05:04:00.527 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.527 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.527 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:00.527 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.876962+0000", 2024-09-14T05:04:00.527 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00096580500000000003 2024-09-14T05:04:00.527 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.527 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.527 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:00.528 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.880547+0000", 2024-09-14T05:04:00.528 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0035846379999999998 2024-09-14T05:04:00.528 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.528 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.528 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:00.528 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:29.167539+0000", 2024-09-14T05:04:00.528 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.286991575 2024-09-14T05:04:00.528 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.528 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.529 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:00.529 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:29.167584+0000", 2024-09-14T05:04:00.529 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.5206000000000001e-05 2024-09-14T05:04:00.529 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.529 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.529 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:00.529 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:29.167618+0000", 2024-09-14T05:04:00.529 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.4168999999999998e-05 2024-09-14T05:04:00.529 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.530 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:00.530 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.530 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.530 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.530 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36603.0:29233 58.1 58:8e970544:::benchmark_data_smithi039_396950_object29232:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1572)", 2024-09-14T05:04:00.530 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:01:28.877881+0000", 2024-09-14T05:04:00.530 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 151.504940686, 2024-09-14T05:04:00.530 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.29146749199999999, 2024-09-14T05:04:00.530 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:00.530 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:00.530 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:00.530 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36603", 2024-09-14T05:04:00.531 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:04:00.531 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 29233 2024-09-14T05:04:00.531 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.531 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:00.531 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.531 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:00.531 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.877881+0000", 2024-09-14T05:04:00.531 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.531 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.531 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.531 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:00.531 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.877881+0000", 2024-09-14T05:04:00.531 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.531 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.531 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.532 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:00.532 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.877879+0000", 2024-09-14T05:04:00.532 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T05:04:00.532 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.532 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.532 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:00.532 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.877948+0000", 2024-09-14T05:04:00.532 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8578999999999995e-05 2024-09-14T05:04:00.532 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.532 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.532 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:00.532 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.877962+0000", 2024-09-14T05:04:00.533 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.4226000000000001e-05 2024-09-14T05:04:00.533 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.533 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.533 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:00.533 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.877970+0000", 2024-09-14T05:04:00.533 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.7300000000000005e-06 2024-09-14T05:04:00.533 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.533 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.533 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:00.533 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.878003+0000", 2024-09-14T05:04:00.533 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3018999999999997e-05 2024-09-14T05:04:00.533 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.533 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.534 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:00.534 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.878091+0000", 2024-09-14T05:04:00.534 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.8271999999999998e-05 2024-09-14T05:04:00.534 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.534 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.534 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,1", 2024-09-14T05:04:00.534 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.878189+0000", 2024-09-14T05:04:00.534 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.7391000000000003e-05 2024-09-14T05:04:00.534 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.534 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.534 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:00.534 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.879080+0000", 2024-09-14T05:04:00.534 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00089140300000000003 2024-09-14T05:04:00.534 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.535 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.535 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:00.535 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:28.880701+0000", 2024-09-14T05:04:00.535 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00162089 2024-09-14T05:04:00.535 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.535 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.535 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:00.535 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:29.169264+0000", 2024-09-14T05:04:00.535 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.28856325300000002 2024-09-14T05:04:00.535 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.535 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.535 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:00.536 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:29.169313+0000", 2024-09-14T05:04:00.536 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.9279e-05 2024-09-14T05:04:00.536 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.536 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.536 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:00.536 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:29.169348+0000", 2024-09-14T05:04:00.536 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.4589999999999999e-05 2024-09-14T05:04:00.536 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.536 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:00.536 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.536 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.536 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.536 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36603.0:35954 58.1 58:84367238:::benchmark_data_smithi039_396950_object35953:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1572)", 2024-09-14T05:04:00.536 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:01:33.233163+0000", 2024-09-14T05:04:00.536 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 147.149657983, 2024-09-14T05:04:00.536 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.227252486, 2024-09-14T05:04:00.537 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:00.537 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:00.537 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:00.537 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36603", 2024-09-14T05:04:00.537 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:04:00.537 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 35954 2024-09-14T05:04:00.537 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.537 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:00.537 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.537 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:00.537 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:33.233163+0000", 2024-09-14T05:04:00.537 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.537 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.538 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.538 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:00.538 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:33.233163+0000", 2024-09-14T05:04:00.538 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.538 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.538 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.538 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:00.538 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:33.233162+0000", 2024-09-14T05:04:00.538 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T05:04:00.538 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.538 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.538 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:00.538 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:33.233220+0000", 2024-09-14T05:04:00.538 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.7365999999999999e-05 2024-09-14T05:04:00.538 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.538 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.539 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:00.539 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:33.233231+0000", 2024-09-14T05:04:00.539 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1164e-05 2024-09-14T05:04:00.539 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.539 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.539 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:00.539 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:33.233242+0000", 2024-09-14T05:04:00.539 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1055e-05 2024-09-14T05:04:00.539 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.539 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.539 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:00.539 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:33.233289+0000", 2024-09-14T05:04:00.539 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7503999999999998e-05 2024-09-14T05:04:00.539 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.539 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.540 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:00.540 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:33.233414+0000", 2024-09-14T05:04:00.540 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000124905 2024-09-14T05:04:00.540 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.540 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.540 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,1", 2024-09-14T05:04:00.540 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:33.233538+0000", 2024-09-14T05:04:00.540 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012360499999999999 2024-09-14T05:04:00.540 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.540 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.540 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:00.540 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:33.235144+0000", 2024-09-14T05:04:00.540 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00160651 2024-09-14T05:04:00.540 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.540 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.541 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:00.541 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:33.243608+0000", 2024-09-14T05:04:00.541 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0084638130000000006 2024-09-14T05:04:00.541 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.541 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.541 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:00.541 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:33.460326+0000", 2024-09-14T05:04:00.541 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.216717296 2024-09-14T05:04:00.541 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.541 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.541 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:00.541 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:33.460374+0000", 2024-09-14T05:04:00.541 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8585999999999999e-05 2024-09-14T05:04:00.541 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.542 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.542 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:00.542 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:01:33.460416+0000", 2024-09-14T05:04:00.542 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1658e-05 2024-09-14T05:04:00.542 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.542 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:00.542 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.542 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.542 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.542 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36603.0:124874 58.1 58:84a55bfb:::benchmark_data_smithi039_396950_object9261:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1583)", 2024-09-14T05:04:00.542 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:02:58.085158+0000", 2024-09-14T05:04:00.542 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 62.297662813000002, 2024-09-14T05:04:00.543 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.10445006799999999, 2024-09-14T05:04:00.543 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:00.543 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:00.543 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:00.543 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36603", 2024-09-14T05:04:00.543 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:04:00.543 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 124874 2024-09-14T05:04:00.543 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.543 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:00.543 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.543 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:00.543 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.085158+0000", 2024-09-14T05:04:00.543 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.544 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.544 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.544 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:00.544 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.085158+0000", 2024-09-14T05:04:00.544 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.544 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.544 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.544 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:00.544 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.085158+0000", 2024-09-14T05:04:00.544 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T05:04:00.544 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.544 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.544 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:00.544 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.085162+0000", 2024-09-14T05:04:00.545 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7639999999999996e-06 2024-09-14T05:04:00.545 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.545 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.545 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:00.545 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.085172+0000", 2024-09-14T05:04:00.545 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.5230000000000001e-06 2024-09-14T05:04:00.545 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.545 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.545 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:00.545 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.085177+0000", 2024-09-14T05:04:00.545 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.9559999999999996e-06 2024-09-14T05:04:00.545 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.545 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.545 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:00.545 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.085215+0000", 2024-09-14T05:04:00.546 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7762000000000002e-05 2024-09-14T05:04:00.546 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.546 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.546 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:00.546 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.085311+0000", 2024-09-14T05:04:00.546 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.6229000000000003e-05 2024-09-14T05:04:00.546 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.546 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.546 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 0,1", 2024-09-14T05:04:00.546 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.085408+0000", 2024-09-14T05:04:00.546 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.7095999999999993e-05 2024-09-14T05:04:00.546 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.546 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.546 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:00.546 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.088183+0000", 2024-09-14T05:04:00.546 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0027748680000000002 2024-09-14T05:04:00.547 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.547 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.547 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:00.547 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.089434+0000", 2024-09-14T05:04:00.547 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001251154 2024-09-14T05:04:00.547 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.547 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.547 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:00.547 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.189521+0000", 2024-09-14T05:04:00.547 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.10008742399999999 2024-09-14T05:04:00.547 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.547 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.547 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:00.547 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.189568+0000", 2024-09-14T05:04:00.547 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6344000000000002e-05 2024-09-14T05:04:00.548 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.548 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.548 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:00.548 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.189609+0000", 2024-09-14T05:04:00.548 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0886000000000001e-05 2024-09-14T05:04:00.548 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.548 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:00.548 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.548 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.548 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.548 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.36603.0:124876 58.7 e1583/1568 58:ef9ce703:::benchmark_data_smithi039_396950_object9263:head v 1583'7872, mlcod=1583'7870)", 2024-09-14T05:04:00.548 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:02:58.086970+0000", 2024-09-14T05:04:00.548 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 62.295851581999997, 2024-09-14T05:04:00.548 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.102575154, 2024-09-14T05:04:00.548 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:00.548 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:00.549 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:00.549 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.549 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:00.549 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.086970+0000", 2024-09-14T05:04:00.549 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.549 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.549 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.549 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:00.549 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.086970+0000", 2024-09-14T05:04:00.549 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.549 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.549 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.549 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:00.549 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.086969+0000", 2024-09-14T05:04:00.549 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T05:04:00.550 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.550 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.550 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:00.550 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.086976+0000", 2024-09-14T05:04:00.550 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.0400000000000004e-06 2024-09-14T05:04:00.550 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.550 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.550 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:00.552 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.086984+0000", 2024-09-14T05:04:00.552 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.1200000000000002e-06 2024-09-14T05:04:00.552 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.552 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.552 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:00.552 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.086989+0000", 2024-09-14T05:04:00.552 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.2880000000000003e-06 2024-09-14T05:04:00.553 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.553 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.553 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:00.553 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.087027+0000", 2024-09-14T05:04:00.553 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8807000000000003e-05 2024-09-14T05:04:00.553 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.553 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.553 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:00.553 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.087046+0000", 2024-09-14T05:04:00.553 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.8522999999999999e-05 2024-09-14T05:04:00.554 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.554 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.554 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:00.554 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.189503+0000", 2024-09-14T05:04:00.554 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.102456672 2024-09-14T05:04:00.554 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.554 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.554 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:00.554 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.189545+0000", 2024-09-14T05:04:00.554 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.2318999999999999e-05 2024-09-14T05:04:00.554 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.554 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:00.555 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.555 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.555 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.555 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop_reply(client.36603.0:124874 58.1 e1583/1568 ondisk, result = 0)", 2024-09-14T05:04:00.555 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:02:58.090198+0000", 2024-09-14T05:04:00.555 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 62.292623556000002, 2024-09-14T05:04:00.555 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.099416475000000004, 2024-09-14T05:04:00.555 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:00.555 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "started", 2024-09-14T05:04:00.555 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:00.555 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.555 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:00.555 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.090198+0000", 2024-09-14T05:04:00.556 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.556 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.556 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.556 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:00.556 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.090198+0000", 2024-09-14T05:04:00.556 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.556 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.556 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.556 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:00.556 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.090197+0000", 2024-09-14T05:04:00.557 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T05:04:00.557 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.557 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.557 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:00.557 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.090203+0000", 2024-09-14T05:04:00.557 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.8749999999999997e-06 2024-09-14T05:04:00.557 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.557 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.557 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:00.557 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.090211+0000", 2024-09-14T05:04:00.557 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.3920000000000003e-06 2024-09-14T05:04:00.557 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.558 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.558 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:00.558 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.090216+0000", 2024-09-14T05:04:00.558 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.5580000000000002e-06 2024-09-14T05:04:00.558 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.558 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.558 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:00.558 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.189491+0000", 2024-09-14T05:04:00.558 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.099274873 2024-09-14T05:04:00.558 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.559 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.559 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:00.559 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.189513+0000", 2024-09-14T05:04:00.559 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.2466000000000001e-05 2024-09-14T05:04:00.559 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.559 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.559 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:00.559 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:02:58.189614+0000", 2024-09-14T05:04:00.559 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000100932 2024-09-14T05:04:00.559 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.559 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:00.559 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.559 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.560 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.560 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.36603.0:157583 58.7 e1595/1568 58:eabeaf68:::benchmark_data_smithi039_396950_object41970:head v 1595'9894, mlcod=1595'9889)", 2024-09-14T05:04:00.560 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:03:38.757812+0000", 2024-09-14T05:04:00.560 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 21.625008949000001, 2024-09-14T05:04:00.560 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.148255477, 2024-09-14T05:04:00.560 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:00.560 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:00.560 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:00.561 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.561 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:00.561 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.757812+0000", 2024-09-14T05:04:00.561 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.561 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.561 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.561 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:00.561 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.757812+0000", 2024-09-14T05:04:00.561 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.561 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.561 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.562 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:00.562 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.757811+0000", 2024-09-14T05:04:00.562 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T05:04:00.562 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.562 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.562 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:00.562 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.757818+0000", 2024-09-14T05:04:00.562 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7909999999999999e-06 2024-09-14T05:04:00.562 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.562 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.563 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:00.563 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.757829+0000", 2024-09-14T05:04:00.563 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0644e-05 2024-09-14T05:04:00.563 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.563 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.563 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:00.563 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.757836+0000", 2024-09-14T05:04:00.563 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8549999999999996e-06 2024-09-14T05:04:00.563 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.563 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.563 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:00.563 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.757881+0000", 2024-09-14T05:04:00.564 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.5466000000000001e-05 2024-09-14T05:04:00.564 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.564 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.564 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:00.564 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.757900+0000", 2024-09-14T05:04:00.564 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.9079000000000001e-05 2024-09-14T05:04:00.564 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.564 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.565 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:00.565 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.905308+0000", 2024-09-14T05:04:00.565 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.147407922 2024-09-14T05:04:00.565 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.565 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.565 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:00.565 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.906068+0000", 2024-09-14T05:04:00.565 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00075955599999999999 2024-09-14T05:04:00.565 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.565 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:00.565 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.565 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.565 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.566 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.36603.0:157586 58.b e1595/1568 58:d14f4c26:::benchmark_data_smithi039_396950_object41973:head v 1595'9860, mlcod=1595'9856)", 2024-09-14T05:04:00.566 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:03:38.762177+0000", 2024-09-14T05:04:00.566 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 21.620643794999999, 2024-09-14T05:04:00.566 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.14851599900000001, 2024-09-14T05:04:00.566 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:00.566 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:00.566 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:00.566 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.566 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:00.566 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.762177+0000", 2024-09-14T05:04:00.566 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.566 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.566 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.566 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:00.566 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.762177+0000", 2024-09-14T05:04:00.567 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.567 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.567 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.567 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:00.567 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.762177+0000", 2024-09-14T05:04:00.567 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T05:04:00.567 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.567 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.567 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:00.567 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.762189+0000", 2024-09-14T05:04:00.567 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2499e-05 2024-09-14T05:04:00.567 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.567 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.568 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:00.568 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.905262+0000", 2024-09-14T05:04:00.568 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.14307272700000001 2024-09-14T05:04:00.568 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.568 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.568 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:00.568 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.905276+0000", 2024-09-14T05:04:00.568 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.4106e-05 2024-09-14T05:04:00.568 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.568 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.568 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:00.568 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.905622+0000", 2024-09-14T05:04:00.568 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000346158 2024-09-14T05:04:00.568 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.569 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.569 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:00.569 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.905641+0000", 2024-09-14T05:04:00.569 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.8686e-05 2024-09-14T05:04:00.569 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.569 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.569 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:00.569 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.910665+0000", 2024-09-14T05:04:00.569 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0050240190000000002 2024-09-14T05:04:00.569 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.569 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.569 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:00.570 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.910693+0000", 2024-09-14T05:04:00.570 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.8560000000000001e-05 2024-09-14T05:04:00.570 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.570 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:00.570 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.570 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.570 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.570 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36603.0:157594 58.4 58:2d5e433e:::benchmark_data_smithi039_396950_object41981:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1595)", 2024-09-14T05:04:00.570 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:03:38.769546+0000", 2024-09-14T05:04:00.570 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 21.61327477, 2024-09-14T05:04:00.570 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.141607129, 2024-09-14T05:04:00.570 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:00.570 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:00.570 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:00.571 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36603", 2024-09-14T05:04:00.571 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:04:00.571 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 157594 2024-09-14T05:04:00.571 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.571 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:00.571 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.571 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:00.571 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.769546+0000", 2024-09-14T05:04:00.571 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.571 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.571 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.572 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:00.572 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.769546+0000", 2024-09-14T05:04:00.572 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:00.572 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.572 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.572 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:00.572 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.769545+0000", 2024-09-14T05:04:00.572 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T05:04:00.572 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.572 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.572 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:00.572 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.769553+0000", 2024-09-14T05:04:00.572 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.7589999999999997e-06 2024-09-14T05:04:00.573 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.573 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.573 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:00.573 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.905276+0000", 2024-09-14T05:04:00.573 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.135723021 2024-09-14T05:04:00.573 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.573 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.573 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:00.573 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.905286+0000", 2024-09-14T05:04:00.573 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0020999999999999e-05 2024-09-14T05:04:00.573 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.573 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.574 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:00.574 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.905342+0000", 2024-09-14T05:04:00.574 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.6480999999999997e-05 2024-09-14T05:04:00.574 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.574 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.574 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:00.574 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.905475+0000", 2024-09-14T05:04:00.574 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013249799999999999 2024-09-14T05:04:00.574 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.574 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.574 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 3,5", 2024-09-14T05:04:00.574 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.905600+0000", 2024-09-14T05:04:00.574 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012510399999999999 2024-09-14T05:04:00.574 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.574 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.575 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:00.575 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.909415+0000", 2024-09-14T05:04:00.575 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0038147580000000001 2024-09-14T05:04:00.575 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.575 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.575 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:00.575 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.909834+0000", 2024-09-14T05:04:00.575 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00041902699999999998 2024-09-14T05:04:00.575 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.575 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.575 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:00.575 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.911063+0000", 2024-09-14T05:04:00.575 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0012297409999999999 2024-09-14T05:04:00.576 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.576 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.576 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:00.576 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.911117+0000", 2024-09-14T05:04:00.576 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.3377000000000001e-05 2024-09-14T05:04:00.576 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:00.576 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:00.576 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:00.576 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:38.911154+0000", 2024-09-14T05:04:00.576 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6921000000000001e-05 2024-09-14T05:04:00.576 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.576 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:00.576 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.577 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:00.577 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:00.577 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T05:04:00.577 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:00.617 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:00.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:00.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:00.719 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:00.720 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:00 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3603334374' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:04:00.720 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:00 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:04:00.720 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:00 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:04:00.758 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:00.760 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:00.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:00.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:00 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3603334374' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:04:00.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:00 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:04:00.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:00 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:04:00.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:00 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3603334374' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:04:00.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:00 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:04:00.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:00 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:04:00.862 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:00.892 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:00.893 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:00.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:00.994 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:01.029 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:01.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:01.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:01.131 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:01.172 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:01.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:01.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:01.274 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:01.314 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:01.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:01.316 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:01.416 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:01.448 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-14T05:04:01.459 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:01.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:01.460 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:01.519 INFO:teuthology.orchestra.run.smithi039.stderr:no valid command found; 10 closest matches: 2024-09-14T05:04:01.519 INFO:teuthology.orchestra.run.smithi039.stderr:0 2024-09-14T05:04:01.519 INFO:teuthology.orchestra.run.smithi039.stderr:1 2024-09-14T05:04:01.520 INFO:teuthology.orchestra.run.smithi039.stderr:2 2024-09-14T05:04:01.520 INFO:teuthology.orchestra.run.smithi039.stderr:abort 2024-09-14T05:04:01.520 INFO:teuthology.orchestra.run.smithi039.stderr:assert 2024-09-14T05:04:01.520 INFO:teuthology.orchestra.run.smithi039.stderr:bench [] [] [] [] 2024-09-14T05:04:01.520 INFO:teuthology.orchestra.run.smithi039.stderr:bluefs debug_inject_read_zeros 2024-09-14T05:04:01.520 INFO:teuthology.orchestra.run.smithi039.stderr:bluefs files list 2024-09-14T05:04:01.520 INFO:teuthology.orchestra.run.smithi039.stderr:bluefs stats 2024-09-14T05:04:01.520 INFO:teuthology.orchestra.run.smithi039.stderr:bluestore allocator dump block 2024-09-14T05:04:01.520 INFO:teuthology.orchestra.run.smithi039.stderr:Error EINVAL: invalid command 2024-09-14T05:04:01.562 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:01.587 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:01.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:01.589 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:01.689 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:01.732 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:01.733 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:01.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:01 smithi129 ceph-mon[101316]: pgmap v3199: 119 pgs: 2 active+clean+scrubbing, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 112 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 864 op/s; 10539/174591 objects misplaced (6.036%); 25 MiB/s, 396 objects/s recovering 2024-09-14T05:04:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:01 smithi129 ceph-mon[101316]: 1.1e scrub starts 2024-09-14T05:04:01.788 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:04:01.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:01 smithi039 ceph-mon[106470]: pgmap v3199: 119 pgs: 2 active+clean+scrubbing, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 112 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 864 op/s; 10539/174591 objects misplaced (6.036%); 25 MiB/s, 396 objects/s recovering 2024-09-14T05:04:01.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:01 smithi039 ceph-mon[106470]: 1.1e scrub starts 2024-09-14T05:04:01.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:01 smithi039 ceph-mon[111235]: pgmap v3199: 119 pgs: 2 active+clean+scrubbing, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 112 active+clean; 3.6 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 864 op/s; 10539/174591 objects misplaced (6.036%); 25 MiB/s, 396 objects/s recovering 2024-09-14T05:04:01.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:01 smithi039 ceph-mon[111235]: 1.1e scrub starts 2024-09-14T05:04:01.834 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:01.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:01.861 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:01.861 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:01.962 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:01.988 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:01.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:01.992 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:02.093 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:02.140 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:02.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:02.141 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:02.242 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:02.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:02.272 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:02.273 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:02.373 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:02.418 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:02.419 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:02.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:02.520 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:02.559 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:02.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:02.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:02.631 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-14T05:04:02.631 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-14T05:04:02.631 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.2 'trim stale osdmaps' 2024-09-14T05:04:02.662 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:02.687 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:02.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:02.688 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:02.789 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:02.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:02.878 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:02.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:02.978 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:03.029 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:03.029 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:03.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:03.130 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:03.186 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:03.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:03.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:03.288 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:03.339 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:03.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:03.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:03.440 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:03.482 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:03.483 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:03.483 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:03.584 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:03.620 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:03.621 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:03.621 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:03.654 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:03 smithi129 ceph-mon[101316]: pgmap v3200: 119 pgs: 2 active+clean+scrubbing, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 112 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 765 op/s; 10278/171474 objects misplaced (5.994%); 15 MiB/s, 238 objects/s recovering 2024-09-14T05:04:03.654 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:03 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T05:04:03.654 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:03 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T05:04:03.654 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:03 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:04:03.681 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T05:04:03.681 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [], 2024-09-14T05:04:03.681 INFO:teuthology.orchestra.run.smithi129.stdout: "num_ops": 0 2024-09-14T05:04:03.682 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T05:04:03.722 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:03.736 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:03 smithi039 ceph-mon[106470]: pgmap v3200: 119 pgs: 2 active+clean+scrubbing, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 112 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 765 op/s; 10278/171474 objects misplaced (5.994%); 15 MiB/s, 238 objects/s recovering 2024-09-14T05:04:03.736 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:03 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T05:04:03.737 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:03 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T05:04:03.737 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:03 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:04:03.737 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:03 smithi039 ceph-mon[111235]: pgmap v3200: 119 pgs: 2 active+clean+scrubbing, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 112 active+clean; 3.5 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 765 op/s; 10278/171474 objects misplaced (5.994%); 15 MiB/s, 238 objects/s recovering 2024-09-14T05:04:03.737 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:03 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T05:04:03.737 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:03 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T05:04:03.737 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:03 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:04:03.776 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:03.777 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:03.777 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:03.878 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:03.919 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:03.919 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:03.920 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:04.021 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:04.029 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:04:04.046 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:04.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:04.047 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:04.148 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:04.174 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:04.175 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:04.175 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:04.276 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:04.325 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:04.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:04.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:04.373 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-14T05:04:04.426 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:04.469 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:04.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:04.470 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:04.571 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:04.597 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:04.598 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:04.599 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:04.629 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:04:04.700 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:04.724 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:04.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:04.725 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:04.826 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:04.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:04.853 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:04.854 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:04.954 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:05.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:05.007 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:05.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:05.108 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:05.142 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:05.142 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:05.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:05.171 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T05:04:05.244 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:05.298 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:05.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:05.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:05.399 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:05.411 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:05 smithi039 ceph-mon[106470]: 1.13 scrub starts 2024-09-14T05:04:05.411 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:05 smithi039 ceph-mon[106470]: 1.13 scrub ok 2024-09-14T05:04:05.411 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:05 smithi039 ceph-mon[106470]: pgmap v3201: 119 pgs: 2 active+clean+scrubbing, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 112 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 779 op/s; 9770/168474 objects misplaced (5.799%); 17 MiB/s, 273 objects/s recovering 2024-09-14T05:04:05.411 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:05 smithi039 ceph-mon[106470]: 1.1d scrub starts 2024-09-14T05:04:05.411 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:05 smithi039 ceph-mon[106470]: 1.1d scrub ok 2024-09-14T05:04:05.415 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:05 smithi039 ceph-mon[111235]: 1.13 scrub starts 2024-09-14T05:04:05.416 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:05 smithi039 ceph-mon[111235]: 1.13 scrub ok 2024-09-14T05:04:05.416 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:05 smithi039 ceph-mon[111235]: pgmap v3201: 119 pgs: 2 active+clean+scrubbing, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 112 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 779 op/s; 9770/168474 objects misplaced (5.799%); 17 MiB/s, 273 objects/s recovering 2024-09-14T05:04:05.416 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:05 smithi039 ceph-mon[111235]: 1.1d scrub starts 2024-09-14T05:04:05.416 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:05 smithi039 ceph-mon[111235]: 1.1d scrub ok 2024-09-14T05:04:05.460 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:05.461 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:05.461 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:05.562 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:05.612 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:05.613 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:05.613 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:05.714 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:05.738 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:05.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:05.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:05.744 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:05 smithi129 ceph-mon[101316]: 1.13 scrub starts 2024-09-14T05:04:05.744 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:05 smithi129 ceph-mon[101316]: 1.13 scrub ok 2024-09-14T05:04:05.744 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:05 smithi129 ceph-mon[101316]: pgmap v3201: 119 pgs: 2 active+clean+scrubbing, 2 active+remapped+backfill_wait, 3 active+remapped+backfilling, 112 active+clean; 3.4 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 779 op/s; 9770/168474 objects misplaced (5.799%); 17 MiB/s, 273 objects/s recovering 2024-09-14T05:04:05.744 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:05 smithi129 ceph-mon[101316]: 1.1d scrub starts 2024-09-14T05:04:05.744 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:05 smithi129 ceph-mon[101316]: 1.1d scrub ok 2024-09-14T05:04:05.840 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:05.864 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:05.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:05.865 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:05.966 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:05.988 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:05.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:05.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:06.090 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:06.119 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:06.120 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:06.120 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:06.221 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:06.305 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:06.305 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:06.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:06.406 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:06.422 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is set 2024-09-14T05:04:06.434 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:06.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:06.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:06.536 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:06.591 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:06.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:06.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:06.694 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:06.705 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T05:04:06.706 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [], 2024-09-14T05:04:06.706 INFO:teuthology.orchestra.run.smithi129.stdout: "complaint_time": 30, 2024-09-14T05:04:06.706 INFO:teuthology.orchestra.run.smithi129.stdout: "num_blocked_ops": 0 2024-09-14T05:04:06.706 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T05:04:06.706 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:06 smithi129 ceph-mon[101316]: 1.19 scrub starts 2024-09-14T05:04:06.706 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:06 smithi129 ceph-mon[101316]: 1.19 scrub ok 2024-09-14T05:04:06.706 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:06 smithi129 ceph-mon[101316]: 1.5 scrub starts 2024-09-14T05:04:06.706 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:06 smithi129 ceph-mon[101316]: 1.5 scrub ok 2024-09-14T05:04:06.706 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:06 smithi129 ceph-mon[101316]: osdmap e1599: 8 total, 8 up, 7 in 2024-09-14T05:04:06.707 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:06 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1641075223' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:04:06.707 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:06 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:04:06.749 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:06.750 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:06.750 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:06.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:06 smithi039 ceph-mon[106470]: 1.19 scrub starts 2024-09-14T05:04:06.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:06 smithi039 ceph-mon[106470]: 1.19 scrub ok 2024-09-14T05:04:06.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:06 smithi039 ceph-mon[106470]: 1.5 scrub starts 2024-09-14T05:04:06.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:06 smithi039 ceph-mon[106470]: 1.5 scrub ok 2024-09-14T05:04:06.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:06 smithi039 ceph-mon[106470]: osdmap e1599: 8 total, 8 up, 7 in 2024-09-14T05:04:06.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:06 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1641075223' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:04:06.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:06 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:04:06.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:06 smithi039 ceph-mon[111235]: 1.19 scrub starts 2024-09-14T05:04:06.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:06 smithi039 ceph-mon[111235]: 1.19 scrub ok 2024-09-14T05:04:06.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:06 smithi039 ceph-mon[111235]: 1.5 scrub starts 2024-09-14T05:04:06.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:06 smithi039 ceph-mon[111235]: 1.5 scrub ok 2024-09-14T05:04:06.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:06 smithi039 ceph-mon[111235]: osdmap e1599: 8 total, 8 up, 7 in 2024-09-14T05:04:06.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:06 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1641075223' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:04:06.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:06 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:04:06.852 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:06.899 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:06.899 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:06.900 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:07.000 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:07.038 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:07.039 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:07.039 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:07.140 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:07.167 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:07.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:07.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:07.201 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:04:07.269 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:07.298 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:07.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:07.299 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:07.400 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:07.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:07.429 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:07.429 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:07.530 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:07.582 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:07.583 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:07.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:07.684 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:07.691 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:07 smithi129 ceph-mon[101316]: pgmap v3203: 119 pgs: 1 active+clean+remapped, 2 active+clean+scrubbing, 4 active+remapped+backfilling, 112 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 811 op/s; 7916/159372 objects misplaced (4.967%); 19 MiB/s, 303 objects/s recovering 2024-09-14T05:04:07.691 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:07 smithi129 ceph-mon[101316]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:04:07.691 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:07 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T05:04:07.691 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:07 smithi129 ceph-mon[101316]: osdmap e1600: 8 total, 8 up, 7 in 2024-09-14T05:04:07.691 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:04:07.691 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:04:07.715 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:07.715 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:07.716 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:07.729 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-14T05:04:07.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:07 smithi039 ceph-mon[106470]: pgmap v3203: 119 pgs: 1 active+clean+remapped, 2 active+clean+scrubbing, 4 active+remapped+backfilling, 112 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 811 op/s; 7916/159372 objects misplaced (4.967%); 19 MiB/s, 303 objects/s recovering 2024-09-14T05:04:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:07 smithi039 ceph-mon[106470]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:04:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:07 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T05:04:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:07 smithi039 ceph-mon[106470]: osdmap e1600: 8 total, 8 up, 7 in 2024-09-14T05:04:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:04:07.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:04:07.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:07 smithi039 ceph-mon[111235]: pgmap v3203: 119 pgs: 1 active+clean+remapped, 2 active+clean+scrubbing, 4 active+remapped+backfilling, 112 active+clean; 3.2 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 811 op/s; 7916/159372 objects misplaced (4.967%); 19 MiB/s, 303 objects/s recovering 2024-09-14T05:04:07.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:07 smithi039 ceph-mon[111235]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:04:07.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:07 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T05:04:07.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:07 smithi039 ceph-mon[111235]: osdmap e1600: 8 total, 8 up, 7 in 2024-09-14T05:04:07.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:04:07.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:04:07.816 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:07.854 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:07.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:07.855 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:07.957 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:07.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:07.998 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:07.998 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:08.035 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:04:08.099 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:08.130 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:08.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:08.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:08.232 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:08.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:08.277 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:08.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:08.378 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:08.388 INFO:teuthology.orchestra.run.smithi039.stderr:no valid command found; 10 closest matches: 2024-09-14T05:04:08.388 INFO:teuthology.orchestra.run.smithi039.stderr:0 2024-09-14T05:04:08.389 INFO:teuthology.orchestra.run.smithi039.stderr:1 2024-09-14T05:04:08.389 INFO:teuthology.orchestra.run.smithi039.stderr:2 2024-09-14T05:04:08.389 INFO:teuthology.orchestra.run.smithi039.stderr:abort 2024-09-14T05:04:08.389 INFO:teuthology.orchestra.run.smithi039.stderr:assert 2024-09-14T05:04:08.389 INFO:teuthology.orchestra.run.smithi039.stderr:bench [] [] [] [] 2024-09-14T05:04:08.389 INFO:teuthology.orchestra.run.smithi039.stderr:bluefs debug_inject_read_zeros 2024-09-14T05:04:08.389 INFO:teuthology.orchestra.run.smithi039.stderr:bluefs files list 2024-09-14T05:04:08.389 INFO:teuthology.orchestra.run.smithi039.stderr:bluefs stats 2024-09-14T05:04:08.389 INFO:teuthology.orchestra.run.smithi039.stderr:bluestore allocator dump block 2024-09-14T05:04:08.389 INFO:teuthology.orchestra.run.smithi039.stderr:Error EINVAL: invalid command 2024-09-14T05:04:08.401 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:08.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:08.402 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:08.503 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:08.531 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:08.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:08.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:08.633 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:08.672 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:08.672 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:08.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:08.773 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:08.797 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:08.798 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:08.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:08.899 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:08.921 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:08.922 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:08.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:09.023 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:09.039 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:04:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:04:08] "GET /metrics HTTP/1.1" 200 36772 "" "Prometheus/2.51.0" 2024-09-14T05:04:09.056 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:09.056 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:09.056 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:09.157 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:09.180 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:09.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:09.181 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:09.212 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T05:04:09.282 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:09.304 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:09.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:09.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:09.405 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:09.430 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:09.431 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:09.431 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:09.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:09 smithi129 ceph-mon[101316]: pgmap v3205: 119 pgs: 2 active+clean+scrubbing, 4 active+remapped+backfilling, 113 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 7157/157821 objects misplaced (4.535%); 19 MiB/s, 298 objects/s recovering 2024-09-14T05:04:09.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:09 smithi129 ceph-mon[101316]: osdmap e1601: 8 total, 8 up, 7 in 2024-09-14T05:04:09.532 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:09.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:09 smithi039 ceph-mon[106470]: pgmap v3205: 119 pgs: 2 active+clean+scrubbing, 4 active+remapped+backfilling, 113 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 7157/157821 objects misplaced (4.535%); 19 MiB/s, 298 objects/s recovering 2024-09-14T05:04:09.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:09 smithi039 ceph-mon[106470]: osdmap e1601: 8 total, 8 up, 7 in 2024-09-14T05:04:09.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:09 smithi039 ceph-mon[111235]: pgmap v3205: 119 pgs: 2 active+clean+scrubbing, 4 active+remapped+backfilling, 113 active+clean; 3.2 GiB data, 15 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 7157/157821 objects misplaced (4.535%); 19 MiB/s, 298 objects/s recovering 2024-09-14T05:04:09.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:09 smithi039 ceph-mon[111235]: osdmap e1601: 8 total, 8 up, 7 in 2024-09-14T05:04:09.559 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:09.560 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:09.560 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:09.661 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:09.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:09.687 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:09.688 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:09.788 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:09.818 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:09.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:09.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:09.898 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T05:04:09.898 INFO:teuthology.orchestra.run.smithi129.stdout: "size": 20, 2024-09-14T05:04:09.898 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 600, 2024-09-14T05:04:09.898 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [ 2024-09-14T05:04:09.898 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.898 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36324.0:241663 57.8 57:14ce26be:::benchmark_data_smithi039_365565_object117911:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1559)", 2024-09-14T05:04:09.898 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:00:42.483790+0000", 2024-09-14T05:04:09.898 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 207.411373416, 2024-09-14T05:04:09.898 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1152441870000001, 2024-09-14T05:04:09.898 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:09.899 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "reached pg", 2024-09-14T05:04:09.899 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:09.899 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36324", 2024-09-14T05:04:09.899 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:04:09.899 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 241663 2024-09-14T05:04:09.899 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.899 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:09.899 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.899 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:09.899 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:42.483790+0000", 2024-09-14T05:04:09.899 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:09.899 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.899 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.899 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:09.900 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:42.483790+0000", 2024-09-14T05:04:09.900 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:09.900 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.900 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.900 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:09.900 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:42.483788+0000", 2024-09-14T05:04:09.900 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T05:04:09.900 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.900 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.900 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:09.900 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:42.483799+0000", 2024-09-14T05:04:09.900 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0845999999999999e-05 2024-09-14T05:04:09.900 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.901 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.901 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:09.901 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:42.483819+0000", 2024-09-14T05:04:09.901 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.0353999999999999e-05 2024-09-14T05:04:09.901 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.901 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.901 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:09.901 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:42.483828+0000", 2024-09-14T05:04:09.901 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.8720000000000008e-06 2024-09-14T05:04:09.901 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.901 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.901 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.901 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:42.486819+0000", 2024-09-14T05:04:09.901 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0029911360000000001 2024-09-14T05:04:09.902 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.902 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.902 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op must wait for map", 2024-09-14T05:04:09.902 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:42.486838+0000", 2024-09-14T05:04:09.902 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.8406e-05 2024-09-14T05:04:09.902 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.902 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.902 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.902 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:42.684853+0000", 2024-09-14T05:04:09.902 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.19801466100000001 2024-09-14T05:04:09.902 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.902 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.902 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:04:09.902 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:42.684863+0000", 2024-09-14T05:04:09.903 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0175e-05 2024-09-14T05:04:09.903 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.903 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.903 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.903 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:43.535066+0000", 2024-09-14T05:04:09.903 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.85020352700000001 2024-09-14T05:04:09.903 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.903 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.903 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for missing object", 2024-09-14T05:04:09.903 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:43.535181+0000", 2024-09-14T05:04:09.903 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000114289 2024-09-14T05:04:09.903 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.903 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.903 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.904 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:43.598955+0000", 2024-09-14T05:04:09.904 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.063774333000000002 2024-09-14T05:04:09.904 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.904 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.904 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:09.904 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:43.599034+0000", 2024-09-14T05:04:09.904 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.9482999999999998e-05 2024-09-14T05:04:09.904 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:09.904 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:09.904 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:09.904 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.904 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.905 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36324.0:241664 57.8 57:12aba7eb:::benchmark_data_smithi039_365565_object117912:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1559)", 2024-09-14T05:04:09.905 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:00:42.483839+0000", 2024-09-14T05:04:09.905 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 207.411324248, 2024-09-14T05:04:09.905 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1156504460000001, 2024-09-14T05:04:09.905 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:09.905 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "reached pg", 2024-09-14T05:04:09.905 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:09.905 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36324", 2024-09-14T05:04:09.905 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:04:09.905 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 241664 2024-09-14T05:04:09.905 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.905 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:09.905 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.905 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:09.906 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:42.483839+0000", 2024-09-14T05:04:09.906 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:09.906 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.906 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.906 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:09.906 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:42.483839+0000", 2024-09-14T05:04:09.906 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:09.906 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.906 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.906 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:09.906 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:42.483839+0000", 2024-09-14T05:04:09.906 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T05:04:09.906 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.906 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.907 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:09.907 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:42.483841+0000", 2024-09-14T05:04:09.907 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.5330000000000001e-06 2024-09-14T05:04:09.907 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.907 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.907 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:09.907 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:42.483847+0000", 2024-09-14T05:04:09.907 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.6899999999999997e-06 2024-09-14T05:04:09.907 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.907 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.907 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:09.907 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:42.483852+0000", 2024-09-14T05:04:09.907 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.775e-06 2024-09-14T05:04:09.907 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.908 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.908 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.908 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:42.487099+0000", 2024-09-14T05:04:09.908 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0032467659999999999 2024-09-14T05:04:09.908 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.908 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.908 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting_for_map not empty", 2024-09-14T05:04:09.908 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:42.487115+0000", 2024-09-14T05:04:09.908 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.6198999999999999e-05 2024-09-14T05:04:09.908 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.908 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.908 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.908 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:42.689106+0000", 2024-09-14T05:04:09.908 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.201991321 2024-09-14T05:04:09.909 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.909 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.909 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:04:09.909 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:42.689121+0000", 2024-09-14T05:04:09.909 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5085e-05 2024-09-14T05:04:09.909 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.909 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.909 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.909 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:43.535487+0000", 2024-09-14T05:04:09.909 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.84636615400000004 2024-09-14T05:04:09.909 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.909 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.909 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for missing object", 2024-09-14T05:04:09.909 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:43.535618+0000", 2024-09-14T05:04:09.910 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013048199999999999 2024-09-14T05:04:09.910 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.910 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.910 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.910 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:43.599420+0000", 2024-09-14T05:04:09.910 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.063801724000000004 2024-09-14T05:04:09.910 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.910 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.910 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:09.910 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:43.599490+0000", 2024-09-14T05:04:09.910 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.0106999999999998e-05 2024-09-14T05:04:09.910 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:09.910 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:09.910 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:09.911 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.911 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.911 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36324.0:241670 57.18 57:1f030e1e:::benchmark_data_smithi039_365565_object117918:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1559)", 2024-09-14T05:04:09.911 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:00:42.483856+0000", 2024-09-14T05:04:09.911 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 207.41130776200001, 2024-09-14T05:04:09.911 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1181782410000001, 2024-09-14T05:04:09.911 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:09.911 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "reached pg", 2024-09-14T05:04:09.911 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:09.911 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36324", 2024-09-14T05:04:09.911 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/1200420654", 2024-09-14T05:04:09.911 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 241670 2024-09-14T05:04:09.911 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.911 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:09.912 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.912 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:09.912 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:42.483856+0000", 2024-09-14T05:04:09.912 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:09.912 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.912 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.912 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:09.912 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:42.483856+0000", 2024-09-14T05:04:09.912 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:09.912 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.912 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.912 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:09.912 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:42.483856+0000", 2024-09-14T05:04:09.913 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T05:04:09.913 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.913 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.913 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:09.913 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:42.483857+0000", 2024-09-14T05:04:09.913 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.8309999999999999e-06 2024-09-14T05:04:09.913 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.913 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.913 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:09.913 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:42.483863+0000", 2024-09-14T05:04:09.913 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2440000000000001e-06 2024-09-14T05:04:09.913 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.913 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.914 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:09.914 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:42.483867+0000", 2024-09-14T05:04:09.914 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1239999999999998e-06 2024-09-14T05:04:09.914 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.914 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.914 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.914 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:42.689274+0000", 2024-09-14T05:04:09.914 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.20540703299999999 2024-09-14T05:04:09.914 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.914 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.914 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:04:09.914 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:42.689279+0000", 2024-09-14T05:04:09.914 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.7300000000000002e-06 2024-09-14T05:04:09.914 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.915 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.915 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.915 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:43.598356+0000", 2024-09-14T05:04:09.915 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.90907692399999995 2024-09-14T05:04:09.915 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.915 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.915 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for missing object", 2024-09-14T05:04:09.915 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:43.598491+0000", 2024-09-14T05:04:09.915 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000134714 2024-09-14T05:04:09.915 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.915 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.915 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.915 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:43.601962+0000", 2024-09-14T05:04:09.916 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0034704900000000001 2024-09-14T05:04:09.916 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.916 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.916 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:09.916 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:00:43.602034+0000", 2024-09-14T05:04:09.916 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.2445000000000002e-05 2024-09-14T05:04:09.916 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:09.916 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:09.916 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:09.916 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.916 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.916 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36603.0:151773 58.0 58:0caf1a61:::benchmark_data_smithi039_396950_object36160:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1590)", 2024-09-14T05:04:09.916 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:03:29.539592+0000", 2024-09-14T05:04:09.917 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 40.355571541000003, 2024-09-14T05:04:09.917 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.1807227820000001, 2024-09-14T05:04:09.917 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:09.917 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:09.917 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:09.917 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36603", 2024-09-14T05:04:09.917 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:04:09.917 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 151773 2024-09-14T05:04:09.917 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.917 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:09.917 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.917 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:09.917 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:29.539592+0000", 2024-09-14T05:04:09.918 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:09.918 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.918 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.918 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:09.918 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:29.539592+0000", 2024-09-14T05:04:09.918 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:09.918 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.918 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.918 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:09.918 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:29.539590+0000", 2024-09-14T05:04:09.918 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T05:04:09.918 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.918 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.918 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:09.919 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:29.539596+0000", 2024-09-14T05:04:09.919 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.7660000000000004e-06 2024-09-14T05:04:09.919 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.919 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.919 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:09.919 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:29.539611+0000", 2024-09-14T05:04:09.919 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.5260999999999998e-05 2024-09-14T05:04:09.919 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.919 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.919 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:09.919 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:29.539621+0000", 2024-09-14T05:04:09.919 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.4180000000000003e-06 2024-09-14T05:04:09.919 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.919 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.920 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.920 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:29.539664+0000", 2024-09-14T05:04:09.920 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.3578000000000002e-05 2024-09-14T05:04:09.920 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.920 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.920 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:04:09.920 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:29.539671+0000", 2024-09-14T05:04:09.920 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.6619999999999999e-06 2024-09-14T05:04:09.920 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.920 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.920 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.920 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:30.542372+0000", 2024-09-14T05:04:09.920 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.002700865 2024-09-14T05:04:09.920 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.921 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.921 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:04:09.921 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:30.542386+0000", 2024-09-14T05:04:09.921 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.4107999999999999e-05 2024-09-14T05:04:09.921 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.921 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.921 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.921 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:31.679255+0000", 2024-09-14T05:04:09.921 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1368687930000001 2024-09-14T05:04:09.921 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.921 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.921 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:09.921 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:31.681230+0000", 2024-09-14T05:04:09.921 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0019748000000000001 2024-09-14T05:04:09.922 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.922 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.922 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,3,4", 2024-09-14T05:04:09.922 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:31.681309+0000", 2024-09-14T05:04:09.922 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.9214000000000004e-05 2024-09-14T05:04:09.922 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.922 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.922 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:09.922 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:31.684759+0000", 2024-09-14T05:04:09.922 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0034503419999999999 2024-09-14T05:04:09.922 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.922 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.922 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:09.922 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:31.687368+0000", 2024-09-14T05:04:09.923 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0026086030000000001 2024-09-14T05:04:09.923 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.923 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.923 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:09.923 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:31.687569+0000", 2024-09-14T05:04:09.923 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00020111 2024-09-14T05:04:09.923 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.923 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.923 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:09.923 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:31.720130+0000", 2024-09-14T05:04:09.923 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.032560680000000002 2024-09-14T05:04:09.923 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.923 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.923 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:09.924 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:31.720193+0000", 2024-09-14T05:04:09.924 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.2880999999999996e-05 2024-09-14T05:04:09.924 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.924 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.924 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:09.924 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:03:31.720315+0000", 2024-09-14T05:04:09.924 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012229499999999999 2024-09-14T05:04:09.924 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:09.924 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:09.924 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:09.924 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.924 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.924 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36603.0:180567 58.0 58:0278bc79:::benchmark_data_smithi039_396950_object64954:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1601)", 2024-09-14T05:04:09.924 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:04:08.528942+0000", 2024-09-14T05:04:09.925 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 1.366221235, 2024-09-14T05:04:09.925 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.97422150600000001, 2024-09-14T05:04:09.925 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:09.925 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:09.925 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:09.925 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36603", 2024-09-14T05:04:09.925 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:04:09.925 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 180567 2024-09-14T05:04:09.925 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.925 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:09.925 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.925 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:09.925 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.528942+0000", 2024-09-14T05:04:09.926 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:09.927 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:09.927 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.927 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.927 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:09.927 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.528942+0000", 2024-09-14T05:04:09.927 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:09.927 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.927 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.928 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:09.928 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.528941+0000", 2024-09-14T05:04:09.928 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T05:04:09.928 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.928 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.928 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:09.928 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.528948+0000", 2024-09-14T05:04:09.928 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.6839999999999996e-06 2024-09-14T05:04:09.928 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.928 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.928 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:09.928 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.528965+0000", 2024-09-14T05:04:09.928 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.7e-05 2024-09-14T05:04:09.929 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.929 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.929 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:09.929 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.528974+0000", 2024-09-14T05:04:09.929 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.4870000000000002e-06 2024-09-14T05:04:09.929 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.929 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.929 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.929 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.529050+0000", 2024-09-14T05:04:09.929 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.6087999999999994e-05 2024-09-14T05:04:09.929 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.929 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.929 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:04:09.930 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.529060+0000", 2024-09-14T05:04:09.930 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.9529999999999994e-06 2024-09-14T05:04:09.930 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.930 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.930 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.930 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.496599+0000", 2024-09-14T05:04:09.930 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.96753938699999997 2024-09-14T05:04:09.930 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.930 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.930 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:09.930 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.496701+0000", 2024-09-14T05:04:09.930 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000101668 2024-09-14T05:04:09.930 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.930 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.931 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,3", 2024-09-14T05:04:09.931 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.496832+0000", 2024-09-14T05:04:09.931 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00013131899999999999 2024-09-14T05:04:09.931 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.931 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.931 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:09.931 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.501255+0000", 2024-09-14T05:04:09.931 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0044224470000000004 2024-09-14T05:04:09.931 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.931 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.931 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:09.931 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.502946+0000", 2024-09-14T05:04:09.931 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0016915509999999999 2024-09-14T05:04:09.932 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.932 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.932 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:09.932 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.503070+0000", 2024-09-14T05:04:09.932 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000123674 2024-09-14T05:04:09.932 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.932 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.932 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:09.932 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.503123+0000", 2024-09-14T05:04:09.932 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2793000000000001e-05 2024-09-14T05:04:09.932 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.932 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.932 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:09.933 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.503164+0000", 2024-09-14T05:04:09.933 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1158999999999997e-05 2024-09-14T05:04:09.933 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:09.933 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:09.933 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:09.933 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.933 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.933 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36603.0:180593 58.0 58:0178e4d1:::benchmark_data_smithi039_396950_object64980:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1601)", 2024-09-14T05:04:09.933 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:04:08.556677+0000", 2024-09-14T05:04:09.933 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 1.3384868270000001, 2024-09-14T05:04:09.933 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.95327813900000002, 2024-09-14T05:04:09.933 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:09.933 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:09.933 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:09.934 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36603", 2024-09-14T05:04:09.934 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:04:09.934 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 180593 2024-09-14T05:04:09.934 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.934 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:09.934 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.934 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:09.934 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.556677+0000", 2024-09-14T05:04:09.934 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:09.934 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.934 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.934 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:09.934 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.556677+0000", 2024-09-14T05:04:09.935 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:09.935 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.935 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.935 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:09.935 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.556675+0000", 2024-09-14T05:04:09.935 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T05:04:09.935 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.935 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.935 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:09.935 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.556683+0000", 2024-09-14T05:04:09.935 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.0709999999999995e-06 2024-09-14T05:04:09.935 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.935 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.935 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:09.936 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.556700+0000", 2024-09-14T05:04:09.936 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.7167999999999999e-05 2024-09-14T05:04:09.936 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.936 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.936 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:09.936 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.556709+0000", 2024-09-14T05:04:09.936 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.8829999999999995e-06 2024-09-14T05:04:09.936 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.936 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.936 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.936 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.556756+0000", 2024-09-14T05:04:09.936 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7194999999999998e-05 2024-09-14T05:04:09.936 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.936 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.937 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:04:09.937 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.556766+0000", 2024-09-14T05:04:09.937 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.7880000000000001e-06 2024-09-14T05:04:09.937 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.937 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.937 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.937 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.499643+0000", 2024-09-14T05:04:09.937 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.94287723899999998 2024-09-14T05:04:09.937 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.937 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.937 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:09.937 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.499711+0000", 2024-09-14T05:04:09.937 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7551000000000004e-05 2024-09-14T05:04:09.938 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.938 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.938 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,3", 2024-09-14T05:04:09.938 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.499782+0000", 2024-09-14T05:04:09.938 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.1542e-05 2024-09-14T05:04:09.938 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.938 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.938 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:09.938 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.505325+0000", 2024-09-14T05:04:09.938 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0055429349999999997 2024-09-14T05:04:09.938 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.938 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.938 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:09.938 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.509805+0000", 2024-09-14T05:04:09.939 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0044798950000000002 2024-09-14T05:04:09.939 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.939 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.939 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:09.939 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.509862+0000", 2024-09-14T05:04:09.939 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.6811999999999997e-05 2024-09-14T05:04:09.939 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.939 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.939 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:09.939 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.509914+0000", 2024-09-14T05:04:09.939 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2015999999999998e-05 2024-09-14T05:04:09.939 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.939 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.939 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:09.940 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.509955+0000", 2024-09-14T05:04:09.940 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0899999999999998e-05 2024-09-14T05:04:09.940 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:09.940 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:09.940 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:09.940 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.940 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.940 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36603.0:180601 58.0 58:0129612e:::benchmark_data_smithi039_396950_object64988:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1601)", 2024-09-14T05:04:09.940 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:04:08.564588+0000", 2024-09-14T05:04:09.940 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 1.3305756259999999, 2024-09-14T05:04:09.940 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.94552335099999996, 2024-09-14T05:04:09.940 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:09.940 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:09.940 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:09.941 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36603", 2024-09-14T05:04:09.941 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:04:09.941 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 180601 2024-09-14T05:04:09.941 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.941 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:09.941 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.941 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:09.941 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.564588+0000", 2024-09-14T05:04:09.941 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:09.941 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.941 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.941 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:09.942 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.564588+0000", 2024-09-14T05:04:09.942 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:09.942 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.942 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.942 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:09.942 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.564587+0000", 2024-09-14T05:04:09.942 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T05:04:09.942 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.942 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.942 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:09.942 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.564593+0000", 2024-09-14T05:04:09.942 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.6320000000000002e-06 2024-09-14T05:04:09.943 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.943 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.943 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:09.943 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.564606+0000", 2024-09-14T05:04:09.943 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2541000000000001e-05 2024-09-14T05:04:09.943 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.943 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.943 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:09.943 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.564611+0000", 2024-09-14T05:04:09.943 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.6380000000000001e-06 2024-09-14T05:04:09.943 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.943 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.943 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.944 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.564660+0000", 2024-09-14T05:04:09.944 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.8996000000000003e-05 2024-09-14T05:04:09.944 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.944 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.944 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:04:09.944 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.564670+0000", 2024-09-14T05:04:09.944 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0047999999999999e-05 2024-09-14T05:04:09.944 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.944 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.944 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.944 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.501285+0000", 2024-09-14T05:04:09.944 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.93661495699999997 2024-09-14T05:04:09.944 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.944 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.945 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:09.945 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.501353+0000", 2024-09-14T05:04:09.945 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7773000000000003e-05 2024-09-14T05:04:09.945 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.945 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.945 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,3", 2024-09-14T05:04:09.945 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.501419+0000", 2024-09-14T05:04:09.945 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.5901000000000005e-05 2024-09-14T05:04:09.945 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.945 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.945 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:09.945 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.509762+0000", 2024-09-14T05:04:09.946 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0083425740000000002 2024-09-14T05:04:09.946 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.946 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.946 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:09.946 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.509997+0000", 2024-09-14T05:04:09.946 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00023524599999999999 2024-09-14T05:04:09.946 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.946 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.946 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:09.946 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.510036+0000", 2024-09-14T05:04:09.946 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9433e-05 2024-09-14T05:04:09.946 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.946 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.946 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:09.947 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.510076+0000", 2024-09-14T05:04:09.947 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.9915e-05 2024-09-14T05:04:09.947 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.947 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.947 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:09.947 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.510111+0000", 2024-09-14T05:04:09.947 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.502e-05 2024-09-14T05:04:09.947 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:09.947 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:09.947 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:09.947 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.947 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.947 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36603.0:180618 58.0 58:0a0486a4:::benchmark_data_smithi039_396950_object65005:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1601)", 2024-09-14T05:04:09.947 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:04:08.581986+0000", 2024-09-14T05:04:09.948 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 1.3131777520000001, 2024-09-14T05:04:09.948 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.93359539800000002, 2024-09-14T05:04:09.948 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:09.948 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:09.948 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:09.948 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36603", 2024-09-14T05:04:09.948 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:04:09.948 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 180618 2024-09-14T05:04:09.949 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.949 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:09.949 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.949 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:09.949 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.581986+0000", 2024-09-14T05:04:09.949 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:09.949 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.949 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.949 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:09.949 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.581986+0000", 2024-09-14T05:04:09.949 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:09.949 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.949 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.949 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:09.950 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.581984+0000", 2024-09-14T05:04:09.950 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T05:04:09.950 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.950 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.950 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:09.950 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.581991+0000", 2024-09-14T05:04:09.950 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.6030000000000001e-06 2024-09-14T05:04:09.950 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.950 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.950 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:09.950 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.582005+0000", 2024-09-14T05:04:09.950 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3927000000000001e-05 2024-09-14T05:04:09.950 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.951 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.951 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:09.951 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.582012+0000", 2024-09-14T05:04:09.951 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8850000000000001e-06 2024-09-14T05:04:09.951 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.951 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.951 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.951 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.582062+0000", 2024-09-14T05:04:09.951 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.0288999999999999e-05 2024-09-14T05:04:09.951 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.951 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.951 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:04:09.951 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.582070+0000", 2024-09-14T05:04:09.951 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.4679999999999992e-06 2024-09-14T05:04:09.952 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.952 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.952 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.952 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.503202+0000", 2024-09-14T05:04:09.952 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.921131908 2024-09-14T05:04:09.952 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.952 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.952 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:09.952 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.503310+0000", 2024-09-14T05:04:09.952 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010751699999999999 2024-09-14T05:04:09.952 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.952 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.952 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,3", 2024-09-14T05:04:09.952 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.503432+0000", 2024-09-14T05:04:09.953 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000121999 2024-09-14T05:04:09.953 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.953 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.953 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:09.953 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.509966+0000", 2024-09-14T05:04:09.953 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00653413 2024-09-14T05:04:09.953 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.953 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.953 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:09.953 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.510173+0000", 2024-09-14T05:04:09.953 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000206753 2024-09-14T05:04:09.953 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.953 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.953 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:09.954 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.515504+0000", 2024-09-14T05:04:09.954 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0053316969999999998 2024-09-14T05:04:09.954 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.954 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.954 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:09.954 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.515541+0000", 2024-09-14T05:04:09.954 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6582000000000003e-05 2024-09-14T05:04:09.954 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.954 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.954 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:09.954 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.515581+0000", 2024-09-14T05:04:09.954 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0302000000000001e-05 2024-09-14T05:04:09.954 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:09.955 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:09.955 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:09.955 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.955 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.955 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36603.0:180622 58.0 58:019194bb:::benchmark_data_smithi039_396950_object65009:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1601)", 2024-09-14T05:04:09.955 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:04:08.587216+0000", 2024-09-14T05:04:09.955 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 1.3079478010000001, 2024-09-14T05:04:09.955 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.92848961500000005, 2024-09-14T05:04:09.955 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:09.955 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:09.955 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:09.955 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36603", 2024-09-14T05:04:09.955 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:04:09.956 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 180622 2024-09-14T05:04:09.956 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.956 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:09.956 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.956 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:09.956 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.587216+0000", 2024-09-14T05:04:09.956 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:09.956 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.956 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.956 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:09.956 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.587216+0000", 2024-09-14T05:04:09.956 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:09.956 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.957 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.957 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:09.957 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.587214+0000", 2024-09-14T05:04:09.957 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T05:04:09.957 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.957 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.957 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:09.957 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.587228+0000", 2024-09-14T05:04:09.957 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3838e-05 2024-09-14T05:04:09.957 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.957 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.957 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:09.957 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.587243+0000", 2024-09-14T05:04:09.957 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.4871e-05 2024-09-14T05:04:09.958 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.958 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.958 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:09.958 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.587250+0000", 2024-09-14T05:04:09.958 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.2989999999999999e-06 2024-09-14T05:04:09.958 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.958 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.958 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.958 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.587297+0000", 2024-09-14T05:04:09.958 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6934000000000002e-05 2024-09-14T05:04:09.958 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.958 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.958 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:04:09.959 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.587306+0000", 2024-09-14T05:04:09.959 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.5390000000000007e-06 2024-09-14T05:04:09.959 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.959 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.959 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.959 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.505376+0000", 2024-09-14T05:04:09.959 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.91807026800000002 2024-09-14T05:04:09.959 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.959 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.959 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:09.959 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.505484+0000", 2024-09-14T05:04:09.959 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010837599999999999 2024-09-14T05:04:09.959 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.960 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.960 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,3", 2024-09-14T05:04:09.960 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.505606+0000", 2024-09-14T05:04:09.960 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000121913 2024-09-14T05:04:09.960 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.960 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.960 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:09.960 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.509973+0000", 2024-09-14T05:04:09.960 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0043667760000000002 2024-09-14T05:04:09.960 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.960 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.960 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:09.960 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.510214+0000", 2024-09-14T05:04:09.961 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00024069599999999999 2024-09-14T05:04:09.961 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.961 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.961 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:09.961 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.515629+0000", 2024-09-14T05:04:09.961 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0054154770000000001 2024-09-14T05:04:09.961 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.961 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.961 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:09.961 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.515668+0000", 2024-09-14T05:04:09.961 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8964999999999997e-05 2024-09-14T05:04:09.961 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.961 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.961 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:09.962 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.515705+0000", 2024-09-14T05:04:09.962 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7217e-05 2024-09-14T05:04:09.962 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:09.962 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:09.962 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:09.962 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.962 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.962 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36603.0:180672 58.0 58:0902c533:::benchmark_data_smithi039_396950_object65059:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1601)", 2024-09-14T05:04:09.962 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:04:08.642538+0000", 2024-09-14T05:04:09.962 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 1.252625111, 2024-09-14T05:04:09.962 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.87757984200000005, 2024-09-14T05:04:09.962 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:09.962 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:09.963 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:09.963 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36603", 2024-09-14T05:04:09.963 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:04:09.963 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 180672 2024-09-14T05:04:09.963 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.963 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:09.963 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.963 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:09.963 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.642538+0000", 2024-09-14T05:04:09.963 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:09.963 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.963 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.963 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:09.963 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.642538+0000", 2024-09-14T05:04:09.964 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:09.964 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.964 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.964 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:09.964 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.642537+0000", 2024-09-14T05:04:09.964 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T05:04:09.964 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.964 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.964 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:09.964 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.642546+0000", 2024-09-14T05:04:09.964 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.721e-06 2024-09-14T05:04:09.964 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.964 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.964 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:09.965 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.642558+0000", 2024-09-14T05:04:09.965 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2573e-05 2024-09-14T05:04:09.965 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.965 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.965 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:09.965 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.642566+0000", 2024-09-14T05:04:09.965 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.4789999999999999e-06 2024-09-14T05:04:09.965 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.965 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.965 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.965 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.642618+0000", 2024-09-14T05:04:09.965 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.2469999999999997e-05 2024-09-14T05:04:09.965 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.966 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.966 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:04:09.966 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.642625+0000", 2024-09-14T05:04:09.966 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.8279999999999997e-06 2024-09-14T05:04:09.966 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.966 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.966 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.966 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.507814+0000", 2024-09-14T05:04:09.966 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.86518909300000002 2024-09-14T05:04:09.966 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.966 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.966 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:09.966 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.507881+0000", 2024-09-14T05:04:09.966 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.7077999999999997e-05 2024-09-14T05:04:09.967 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.967 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.967 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,3", 2024-09-14T05:04:09.967 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.507956+0000", 2024-09-14T05:04:09.967 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.4998000000000003e-05 2024-09-14T05:04:09.967 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.967 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.967 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:09.967 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.510122+0000", 2024-09-14T05:04:09.967 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0021661430000000001 2024-09-14T05:04:09.967 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.967 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.967 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:09.967 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.515753+0000", 2024-09-14T05:04:09.968 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0056308169999999998 2024-09-14T05:04:09.968 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.968 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.968 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:09.968 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.520066+0000", 2024-09-14T05:04:09.968 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0043125669999999998 2024-09-14T05:04:09.968 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.968 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.968 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:09.968 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.520095+0000", 2024-09-14T05:04:09.968 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.9651000000000001e-05 2024-09-14T05:04:09.968 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.968 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.968 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:09.969 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.520118+0000", 2024-09-14T05:04:09.969 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.2926999999999999e-05 2024-09-14T05:04:09.969 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:09.969 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:09.969 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:09.969 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.969 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.969 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36603.0:180678 58.0 58:0b072bcb:::benchmark_data_smithi039_396950_object65065:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1601)", 2024-09-14T05:04:09.969 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:04:08.652764+0000", 2024-09-14T05:04:09.969 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 1.2423998629999999, 2024-09-14T05:04:09.969 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.87421349199999998, 2024-09-14T05:04:09.969 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:09.969 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:09.970 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:09.970 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36603", 2024-09-14T05:04:09.970 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:04:09.970 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 180678 2024-09-14T05:04:09.970 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.970 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:09.970 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.970 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:09.970 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.652764+0000", 2024-09-14T05:04:09.970 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:09.970 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.970 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.970 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:09.971 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.652764+0000", 2024-09-14T05:04:09.971 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:09.971 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.971 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.971 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:09.971 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.652762+0000", 2024-09-14T05:04:09.971 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T05:04:09.971 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.971 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.972 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:09.972 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.652771+0000", 2024-09-14T05:04:09.972 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.619e-06 2024-09-14T05:04:09.972 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.972 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.972 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:09.972 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.652787+0000", 2024-09-14T05:04:09.972 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.6494000000000002e-05 2024-09-14T05:04:09.972 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.972 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.972 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:09.972 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.652794+0000", 2024-09-14T05:04:09.972 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.331e-06 2024-09-14T05:04:09.973 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.973 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.973 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.973 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.652831+0000", 2024-09-14T05:04:09.973 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7586000000000002e-05 2024-09-14T05:04:09.973 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.973 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.973 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:04:09.973 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.652837+0000", 2024-09-14T05:04:09.973 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.9290000000000003e-06 2024-09-14T05:04:09.973 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.974 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.974 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.974 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.510312+0000", 2024-09-14T05:04:09.974 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.85747428699999995 2024-09-14T05:04:09.974 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.974 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.974 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:09.974 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.510425+0000", 2024-09-14T05:04:09.974 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000113873 2024-09-14T05:04:09.974 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.974 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.974 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,3", 2024-09-14T05:04:09.975 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.510549+0000", 2024-09-14T05:04:09.975 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000123096 2024-09-14T05:04:09.975 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.975 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.975 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:09.975 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.515595+0000", 2024-09-14T05:04:09.975 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.005046699 2024-09-14T05:04:09.975 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.975 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.975 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:09.975 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.523381+0000", 2024-09-14T05:04:09.975 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0077857769999999998 2024-09-14T05:04:09.975 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.975 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.976 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:09.976 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.526927+0000", 2024-09-14T05:04:09.976 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0035455529999999999 2024-09-14T05:04:09.976 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.976 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.976 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:09.976 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.526955+0000", 2024-09-14T05:04:09.976 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.8348999999999999e-05 2024-09-14T05:04:09.976 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.976 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.976 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:09.976 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.526977+0000", 2024-09-14T05:04:09.976 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.2291999999999999e-05 2024-09-14T05:04:09.977 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:09.977 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:09.977 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:09.977 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.977 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.977 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36603.0:180707 58.0 58:0efa8ecb:::benchmark_data_smithi039_396950_object65094:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1601)", 2024-09-14T05:04:09.977 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:04:08.689762+0000", 2024-09-14T05:04:09.977 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 1.205401924, 2024-09-14T05:04:09.977 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.84182567699999999, 2024-09-14T05:04:09.977 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:09.977 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:09.977 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:09.977 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36603", 2024-09-14T05:04:09.977 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:04:09.978 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 180707 2024-09-14T05:04:09.978 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.978 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:09.978 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.978 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:09.978 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.689762+0000", 2024-09-14T05:04:09.978 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:09.978 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.978 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.978 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:09.978 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.689762+0000", 2024-09-14T05:04:09.978 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:09.979 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.979 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.979 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:09.979 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.689760+0000", 2024-09-14T05:04:09.979 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T05:04:09.979 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.979 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.979 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:09.979 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.689767+0000", 2024-09-14T05:04:09.979 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.3230000000000003e-06 2024-09-14T05:04:09.979 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.979 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.979 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:09.980 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.689784+0000", 2024-09-14T05:04:09.980 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.7050000000000001e-05 2024-09-14T05:04:09.980 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.980 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.980 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:09.980 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.689791+0000", 2024-09-14T05:04:09.980 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.7209999999999994e-06 2024-09-14T05:04:09.980 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.980 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.980 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.980 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.689832+0000", 2024-09-14T05:04:09.980 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.0983999999999999e-05 2024-09-14T05:04:09.980 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.980 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.981 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:04:09.981 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.689842+0000", 2024-09-14T05:04:09.981 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.4220000000000006e-06 2024-09-14T05:04:09.981 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.981 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.981 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.981 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.512951+0000", 2024-09-14T05:04:09.981 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.82310933399999997 2024-09-14T05:04:09.981 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.981 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.981 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:09.981 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.513045+0000", 2024-09-14T05:04:09.981 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.3486999999999994e-05 2024-09-14T05:04:09.981 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.982 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.982 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,3", 2024-09-14T05:04:09.982 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.513125+0000", 2024-09-14T05:04:09.982 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.0730999999999997e-05 2024-09-14T05:04:09.982 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.982 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.982 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:09.982 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.515760+0000", 2024-09-14T05:04:09.982 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0026346339999999998 2024-09-14T05:04:09.982 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.982 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.982 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:09.982 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.526888+0000", 2024-09-14T05:04:09.982 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.011128470999999999 2024-09-14T05:04:09.983 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.983 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.983 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:09.983 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.531501+0000", 2024-09-14T05:04:09.983 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.004612672 2024-09-14T05:04:09.983 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.983 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.983 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:09.983 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.531546+0000", 2024-09-14T05:04:09.983 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.4551000000000001e-05 2024-09-14T05:04:09.983 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.983 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.983 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:09.984 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.531587+0000", 2024-09-14T05:04:09.984 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1776000000000001e-05 2024-09-14T05:04:09.984 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:09.984 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:09.984 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:09.984 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.984 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.984 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36603.0:180716 58.0 58:0d8e5808:::benchmark_data_smithi039_396950_object65103:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1601)", 2024-09-14T05:04:09.984 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:04:08.700358+0000", 2024-09-14T05:04:09.984 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 1.1948059390000001, 2024-09-14T05:04:09.984 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.83140418299999996, 2024-09-14T05:04:09.984 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:09.984 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:09.984 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:09.985 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36603", 2024-09-14T05:04:09.985 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:04:09.985 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 180716 2024-09-14T05:04:09.985 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.985 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:09.985 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.985 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:09.985 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.700358+0000", 2024-09-14T05:04:09.985 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:09.985 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.985 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.985 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:09.985 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.700358+0000", 2024-09-14T05:04:09.986 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:09.986 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.986 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.986 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:09.986 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.700356+0000", 2024-09-14T05:04:09.986 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T05:04:09.986 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.986 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.986 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:09.986 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.700365+0000", 2024-09-14T05:04:09.986 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.0259999999999996e-06 2024-09-14T05:04:09.986 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.987 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.987 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:09.987 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.700384+0000", 2024-09-14T05:04:09.987 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.9164000000000002e-05 2024-09-14T05:04:09.987 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.987 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.987 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:09.987 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.700393+0000", 2024-09-14T05:04:09.987 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.9509999999999995e-06 2024-09-14T05:04:09.987 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.987 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.987 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.987 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.700456+0000", 2024-09-14T05:04:09.987 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.2710000000000001e-05 2024-09-14T05:04:09.988 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.988 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.988 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:04:09.988 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.700465+0000", 2024-09-14T05:04:09.988 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.2499999999999995e-06 2024-09-14T05:04:09.988 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.988 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.988 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.988 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.515783+0000", 2024-09-14T05:04:09.988 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.81531858499999998 2024-09-14T05:04:09.988 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.988 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.988 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:09.988 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.515944+0000", 2024-09-14T05:04:09.989 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00016036499999999999 2024-09-14T05:04:09.989 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.989 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.989 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,3", 2024-09-14T05:04:09.989 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.516067+0000", 2024-09-14T05:04:09.989 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00012293100000000001 2024-09-14T05:04:09.989 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.989 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.989 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:09.989 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.523386+0000", 2024-09-14T05:04:09.989 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0073197169999999999 2024-09-14T05:04:09.989 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.989 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.990 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:09.990 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.531456+0000", 2024-09-14T05:04:09.990 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0080694889999999991 2024-09-14T05:04:09.990 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.990 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.990 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:09.990 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.531685+0000", 2024-09-14T05:04:09.990 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000229046 2024-09-14T05:04:09.990 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.990 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.990 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:09.990 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.531727+0000", 2024-09-14T05:04:09.990 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.2228000000000003e-05 2024-09-14T05:04:09.991 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.991 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.991 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:09.991 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.531762+0000", 2024-09-14T05:04:09.991 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.4712000000000002e-05 2024-09-14T05:04:09.991 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:09.991 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:09.991 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:09.991 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.991 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.991 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36603.0:180739 58.0 58:04b225cd:::benchmark_data_smithi039_396950_object65126:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1601)", 2024-09-14T05:04:09.991 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:04:08.728739+0000", 2024-09-14T05:04:09.991 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 1.166424256, 2024-09-14T05:04:09.991 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.803157334, 2024-09-14T05:04:09.992 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:09.992 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:09.992 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:09.992 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36603", 2024-09-14T05:04:09.992 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:04:09.992 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 180739 2024-09-14T05:04:09.992 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.992 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:09.992 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.992 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:09.992 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.728739+0000", 2024-09-14T05:04:09.992 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:09.992 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.993 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.993 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:09.993 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.728739+0000", 2024-09-14T05:04:09.993 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:09.993 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.993 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.993 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:09.993 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.728737+0000", 2024-09-14T05:04:09.993 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999976 2024-09-14T05:04:09.993 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.993 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.993 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:09.993 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.728750+0000", 2024-09-14T05:04:09.993 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2523e-05 2024-09-14T05:04:09.994 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.994 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.994 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:09.994 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.728768+0000", 2024-09-14T05:04:09.994 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.8400999999999999e-05 2024-09-14T05:04:09.994 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.994 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.994 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:09.994 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.728778+0000", 2024-09-14T05:04:09.994 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.9159999999999996e-06 2024-09-14T05:04:09.994 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.994 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.994 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.994 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.728824+0000", 2024-09-14T05:04:09.995 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6378000000000002e-05 2024-09-14T05:04:09.995 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.995 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.995 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:04:09.995 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.728832+0000", 2024-09-14T05:04:09.995 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.6830000000000008e-06 2024-09-14T05:04:09.995 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.995 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.995 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:09.995 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.518312+0000", 2024-09-14T05:04:09.995 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.78948041999999996 2024-09-14T05:04:09.995 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.995 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.995 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:09.996 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.518394+0000", 2024-09-14T05:04:09.996 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.2068000000000001e-05 2024-09-14T05:04:09.996 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.996 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.996 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,3", 2024-09-14T05:04:09.996 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.518460+0000", 2024-09-14T05:04:09.996 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.5215000000000005e-05 2024-09-14T05:04:09.996 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.996 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.996 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:09.996 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.526839+0000", 2024-09-14T05:04:09.996 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0083793449999999998 2024-09-14T05:04:09.996 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.996 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.997 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:09.997 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.531642+0000", 2024-09-14T05:04:09.997 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0048030089999999996 2024-09-14T05:04:09.997 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.997 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.997 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:09.997 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.531813+0000", 2024-09-14T05:04:09.997 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000170593 2024-09-14T05:04:09.997 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.997 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.997 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:09.997 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.531861+0000", 2024-09-14T05:04:09.997 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7963999999999997e-05 2024-09-14T05:04:09.997 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.998 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.998 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:09.998 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.531897+0000", 2024-09-14T05:04:09.998 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.6115000000000002e-05 2024-09-14T05:04:09.998 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:09.998 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:09.998 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:09.998 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:09.998 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:09.998 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36603.0:180748 58.0 58:05811e06:::benchmark_data_smithi039_396950_object65135:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1601)", 2024-09-14T05:04:09.999 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:04:08.744766+0000", 2024-09-14T05:04:10.000 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 1.150397254, 2024-09-14T05:04:10.000 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.79458681799999997, 2024-09-14T05:04:10.000 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:10.000 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:10.000 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:10.000 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36603", 2024-09-14T05:04:10.000 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:04:10.000 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 180748 2024-09-14T05:04:10.000 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.000 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:10.000 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.000 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:10.000 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.744766+0000", 2024-09-14T05:04:10.001 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:10.001 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.001 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.001 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:10.001 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.744766+0000", 2024-09-14T05:04:10.001 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:10.001 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.001 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.001 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:10.001 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.744765+0000", 2024-09-14T05:04:10.001 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T05:04:10.001 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.001 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.001 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:10.002 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.744772+0000", 2024-09-14T05:04:10.002 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.2130000000000002e-06 2024-09-14T05:04:10.002 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.002 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.002 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:10.002 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.744786+0000", 2024-09-14T05:04:10.002 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.3614e-05 2024-09-14T05:04:10.002 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.002 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.002 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:10.002 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.744792+0000", 2024-09-14T05:04:10.002 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.1059999999999996e-06 2024-09-14T05:04:10.002 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.002 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.003 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:10.003 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.744828+0000", 2024-09-14T05:04:10.003 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5933999999999998e-05 2024-09-14T05:04:10.003 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.003 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.003 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:04:10.003 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.744834+0000", 2024-09-14T05:04:10.003 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.3559999999999998e-06 2024-09-14T05:04:10.003 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.003 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.003 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:10.003 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.520146+0000", 2024-09-14T05:04:10.003 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.77531224499999996 2024-09-14T05:04:10.003 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.004 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.004 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:10.004 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.520209+0000", 2024-09-14T05:04:10.004 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.2414000000000002e-05 2024-09-14T05:04:10.004 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.004 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.004 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,3", 2024-09-14T05:04:10.004 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.520273+0000", 2024-09-14T05:04:10.004 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.3810000000000001e-05 2024-09-14T05:04:10.004 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.004 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.004 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:10.004 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.526847+0000", 2024-09-14T05:04:10.005 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.006574603 2024-09-14T05:04:10.005 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.005 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.005 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:10.005 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.531956+0000", 2024-09-14T05:04:10.005 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0051084659999999999 2024-09-14T05:04:10.005 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.005 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.005 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:10.005 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.539261+0000", 2024-09-14T05:04:10.005 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0073054490000000003 2024-09-14T05:04:10.005 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.005 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.005 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:10.006 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.539311+0000", 2024-09-14T05:04:10.006 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.9889000000000003e-05 2024-09-14T05:04:10.006 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.006 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.006 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:10.006 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.539353+0000", 2024-09-14T05:04:10.006 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.2200000000000003e-05 2024-09-14T05:04:10.006 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:10.006 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:10.006 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:10.006 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.006 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.006 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36603.0:180754 58.0 58:00a5a98b:::benchmark_data_smithi039_396950_object65141:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1601)", 2024-09-14T05:04:10.006 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:04:08.750546+0000", 2024-09-14T05:04:10.007 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 1.1446177150000001, 2024-09-14T05:04:10.007 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.79315990800000002, 2024-09-14T05:04:10.007 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:10.007 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:10.007 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:10.007 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36603", 2024-09-14T05:04:10.007 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:04:10.007 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 180754 2024-09-14T05:04:10.007 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.007 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:10.007 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.007 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:10.007 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.750546+0000", 2024-09-14T05:04:10.007 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:10.008 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.008 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.008 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:10.008 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.750546+0000", 2024-09-14T05:04:10.008 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:10.008 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.008 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.008 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:10.008 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.750545+0000", 2024-09-14T05:04:10.008 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T05:04:10.008 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.008 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.008 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:10.009 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.750553+0000", 2024-09-14T05:04:10.009 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.0970000000000006e-06 2024-09-14T05:04:10.009 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.009 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.009 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:10.009 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.750565+0000", 2024-09-14T05:04:10.009 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.2317000000000001e-05 2024-09-14T05:04:10.009 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.009 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.009 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:10.009 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.750573+0000", 2024-09-14T05:04:10.009 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.9459999999999998e-06 2024-09-14T05:04:10.009 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.009 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.009 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:10.010 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.750623+0000", 2024-09-14T05:04:10.010 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.0098e-05 2024-09-14T05:04:10.010 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.010 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.010 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:04:10.010 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.750634+0000", 2024-09-14T05:04:10.010 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0471e-05 2024-09-14T05:04:10.010 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.010 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.010 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:10.010 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.521631+0000", 2024-09-14T05:04:10.010 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.77099789200000002 2024-09-14T05:04:10.010 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.010 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.011 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:10.011 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.521731+0000", 2024-09-14T05:04:10.011 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.9549000000000005e-05 2024-09-14T05:04:10.011 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.011 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.011 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,3", 2024-09-14T05:04:10.011 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.521843+0000", 2024-09-14T05:04:10.011 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000111709 2024-09-14T05:04:10.011 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.011 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.011 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:10.011 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.526987+0000", 2024-09-14T05:04:10.011 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0051439550000000004 2024-09-14T05:04:10.011 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.012 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.012 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:10.012 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.539418+0000", 2024-09-14T05:04:10.012 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.012431144 2024-09-14T05:04:10.012 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.012 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.012 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:10.012 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.543603+0000", 2024-09-14T05:04:10.012 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0041849119999999998 2024-09-14T05:04:10.012 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.012 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.012 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:10.012 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.543658+0000", 2024-09-14T05:04:10.012 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.5519000000000002e-05 2024-09-14T05:04:10.013 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.013 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.013 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:10.013 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.543706+0000", 2024-09-14T05:04:10.013 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7595000000000001e-05 2024-09-14T05:04:10.013 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:10.013 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:10.013 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:10.013 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.013 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.013 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36603.0:180755 58.0 58:01f9a432:::benchmark_data_smithi039_396950_object65142:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1601)", 2024-09-14T05:04:10.013 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:04:08.751425+0000", 2024-09-14T05:04:10.013 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 1.1437383880000001, 2024-09-14T05:04:10.013 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.79684313399999995, 2024-09-14T05:04:10.014 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:10.014 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:10.014 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:10.014 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36603", 2024-09-14T05:04:10.014 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:04:10.014 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 180755 2024-09-14T05:04:10.014 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.014 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:10.014 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.014 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:10.014 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.751425+0000", 2024-09-14T05:04:10.014 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:10.014 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.014 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.015 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:10.015 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.751425+0000", 2024-09-14T05:04:10.015 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:10.015 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.015 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.015 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:10.015 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.751424+0000", 2024-09-14T05:04:10.015 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999995 2024-09-14T05:04:10.015 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.015 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.015 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:10.015 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.751431+0000", 2024-09-14T05:04:10.015 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.4980000000000003e-06 2024-09-14T05:04:10.016 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.016 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.016 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:10.016 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.751442+0000", 2024-09-14T05:04:10.016 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.0743e-05 2024-09-14T05:04:10.016 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.016 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.016 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:10.016 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.751447+0000", 2024-09-14T05:04:10.016 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.5389999999999998e-06 2024-09-14T05:04:10.016 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.016 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.016 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:10.016 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.751489+0000", 2024-09-14T05:04:10.017 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1987000000000003e-05 2024-09-14T05:04:10.017 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.017 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.017 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:04:10.017 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.751498+0000", 2024-09-14T05:04:10.017 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.4009999999999997e-06 2024-09-14T05:04:10.017 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.017 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.017 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:10.017 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.523398+0000", 2024-09-14T05:04:10.017 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.77190040400000004 2024-09-14T05:04:10.017 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.017 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.018 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:10.018 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.523462+0000", 2024-09-14T05:04:10.018 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.3971000000000001e-05 2024-09-14T05:04:10.018 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.018 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.018 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,3", 2024-09-14T05:04:10.018 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.523537+0000", 2024-09-14T05:04:10.018 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.5315000000000007e-05 2024-09-14T05:04:10.018 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.018 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.018 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:10.018 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.526993+0000", 2024-09-14T05:04:10.018 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0034558589999999999 2024-09-14T05:04:10.018 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.019 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.019 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:10.019 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.544120+0000", 2024-09-14T05:04:10.019 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.017126780000000001 2024-09-14T05:04:10.019 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.019 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.019 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:10.019 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.548182+0000", 2024-09-14T05:04:10.019 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.004061879 2024-09-14T05:04:10.019 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.019 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.019 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:10.019 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.548226+0000", 2024-09-14T05:04:10.019 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.4004000000000001e-05 2024-09-14T05:04:10.020 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.020 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.020 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:10.020 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.548268+0000", 2024-09-14T05:04:10.020 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.2460999999999999e-05 2024-09-14T05:04:10.020 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:10.020 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:10.020 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:10.020 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.020 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.020 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36603.0:180762 58.0 58:0791911a:::benchmark_data_smithi039_396950_object65149:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1601)", 2024-09-14T05:04:10.020 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:04:08.765929+0000", 2024-09-14T05:04:10.020 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 1.1292340910000001, 2024-09-14T05:04:10.021 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.78246229499999997, 2024-09-14T05:04:10.021 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:10.021 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:10.021 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:10.021 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36603", 2024-09-14T05:04:10.021 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:04:10.021 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 180762 2024-09-14T05:04:10.021 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.021 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:10.021 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.021 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:10.021 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.765929+0000", 2024-09-14T05:04:10.021 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:10.022 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.022 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.022 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:10.022 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.765929+0000", 2024-09-14T05:04:10.022 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:10.022 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.022 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.022 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:10.022 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.765928+0000", 2024-09-14T05:04:10.022 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T05:04:10.022 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.022 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.022 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:10.022 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.765934+0000", 2024-09-14T05:04:10.023 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.2360000000000001e-06 2024-09-14T05:04:10.023 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.023 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.023 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:10.023 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.765946+0000", 2024-09-14T05:04:10.023 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.1911e-05 2024-09-14T05:04:10.023 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.023 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.023 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:10.023 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.765951+0000", 2024-09-14T05:04:10.023 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.7520000000000004e-06 2024-09-14T05:04:10.023 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.023 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.023 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:10.024 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.766001+0000", 2024-09-14T05:04:10.024 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.9577000000000003e-05 2024-09-14T05:04:10.024 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.024 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.024 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:04:10.024 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:08.766010+0000", 2024-09-14T05:04:10.024 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.4790000000000002e-06 2024-09-14T05:04:10.024 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.024 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.024 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:10.024 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.525142+0000", 2024-09-14T05:04:10.024 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.75913220400000003 2024-09-14T05:04:10.024 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.024 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.025 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:10.025 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.525207+0000", 2024-09-14T05:04:10.025 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.5134000000000004e-05 2024-09-14T05:04:10.025 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.025 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.025 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,3", 2024-09-14T05:04:10.025 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.525284+0000", 2024-09-14T05:04:10.025 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.6167000000000004e-05 2024-09-14T05:04:10.025 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.025 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.025 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:10.025 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.531401+0000", 2024-09-14T05:04:10.025 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0061173219999999997 2024-09-14T05:04:10.025 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.026 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.026 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:10.026 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.544463+0000", 2024-09-14T05:04:10.026 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.013062129 2024-09-14T05:04:10.026 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.026 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.026 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:10.026 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.548312+0000", 2024-09-14T05:04:10.026 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.003848785 2024-09-14T05:04:10.026 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.026 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.026 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:10.026 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.548346+0000", 2024-09-14T05:04:10.026 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.3890999999999998e-05 2024-09-14T05:04:10.027 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.027 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.027 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:10.027 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.548392+0000", 2024-09-14T05:04:10.027 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.6116999999999999e-05 2024-09-14T05:04:10.027 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:10.027 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:10.027 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:10.027 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.027 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.027 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36603.0:180846 58.0 58:00a82efa:::benchmark_data_smithi039_396950_object65233:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1601)", 2024-09-14T05:04:10.027 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:04:09.118124+0000", 2024-09-14T05:04:10.027 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.777040073, 2024-09-14T05:04:10.027 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.43039625599999998, 2024-09-14T05:04:10.028 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:10.028 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:10.028 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:10.028 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36603", 2024-09-14T05:04:10.028 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:04:10.028 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 180846 2024-09-14T05:04:10.028 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.028 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:10.028 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.028 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:10.028 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.118124+0000", 2024-09-14T05:04:10.028 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:10.028 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.029 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.029 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:10.029 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.118124+0000", 2024-09-14T05:04:10.029 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:10.029 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.029 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.029 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:10.029 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.118122+0000", 2024-09-14T05:04:10.029 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999981 2024-09-14T05:04:10.029 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.029 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.029 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:10.030 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.118130+0000", 2024-09-14T05:04:10.030 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.9279999999999993e-06 2024-09-14T05:04:10.030 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.030 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.030 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:10.030 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.118146+0000", 2024-09-14T05:04:10.030 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.6260999999999999e-05 2024-09-14T05:04:10.030 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.030 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.030 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:10.030 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.118153+0000", 2024-09-14T05:04:10.030 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.2289999999999997e-06 2024-09-14T05:04:10.030 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.030 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.031 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:10.031 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.118194+0000", 2024-09-14T05:04:10.031 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.1279000000000002e-05 2024-09-14T05:04:10.031 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.031 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.031 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:04:10.031 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.118202+0000", 2024-09-14T05:04:10.031 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.8979999999999996e-06 2024-09-14T05:04:10.031 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.031 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.031 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:10.031 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.527016+0000", 2024-09-14T05:04:10.031 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.40881380299999998 2024-09-14T05:04:10.031 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.032 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.032 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:10.032 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.527091+0000", 2024-09-14T05:04:10.032 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.5099999999999996e-05 2024-09-14T05:04:10.032 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.032 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.032 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,3", 2024-09-14T05:04:10.032 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.527186+0000", 2024-09-14T05:04:10.032 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.4519999999999996e-05 2024-09-14T05:04:10.032 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.032 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.032 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:10.032 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.531599+0000", 2024-09-14T05:04:10.032 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0044133360000000003 2024-09-14T05:04:10.033 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.033 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.033 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:10.033 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.544497+0000", 2024-09-14T05:04:10.033 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.012897974 2024-09-14T05:04:10.033 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.033 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.033 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:10.033 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.548441+0000", 2024-09-14T05:04:10.033 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0039436289999999997 2024-09-14T05:04:10.033 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.033 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.033 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:10.033 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.548485+0000", 2024-09-14T05:04:10.034 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4.4861999999999998e-05 2024-09-14T05:04:10.034 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.034 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.034 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:10.034 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.548520+0000", 2024-09-14T05:04:10.034 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.4301000000000002e-05 2024-09-14T05:04:10.034 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:10.034 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:10.034 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:10.034 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.034 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.034 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_op(client.36603.0:180848 58.0 58:03a66b54:::benchmark_data_smithi039_396950_object65235:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1601)", 2024-09-14T05:04:10.034 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:04:09.126120+0000", 2024-09-14T05:04:10.034 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.76904379700000003, 2024-09-14T05:04:10.035 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.42253221800000001, 2024-09-14T05:04:10.035 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:10.035 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:04:10.035 INFO:teuthology.orchestra.run.smithi129.stdout: "client_info": { 2024-09-14T05:04:10.035 INFO:teuthology.orchestra.run.smithi129.stdout: "client": "client.36603", 2024-09-14T05:04:10.035 INFO:teuthology.orchestra.run.smithi129.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:04:10.035 INFO:teuthology.orchestra.run.smithi129.stdout: "tid": 180848 2024-09-14T05:04:10.035 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.035 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:10.035 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.035 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:10.035 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.126120+0000", 2024-09-14T05:04:10.035 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:10.035 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.036 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.036 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:10.036 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.126120+0000", 2024-09-14T05:04:10.036 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:10.036 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.036 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.036 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:10.036 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.126119+0000", 2024-09-14T05:04:10.036 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.9999986 2024-09-14T05:04:10.036 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.036 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.036 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:10.036 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.126125+0000", 2024-09-14T05:04:10.036 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.2650000000000002e-06 2024-09-14T05:04:10.037 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.037 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.037 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:10.037 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.126139+0000", 2024-09-14T05:04:10.037 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.4058e-05 2024-09-14T05:04:10.037 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.037 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.037 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:10.037 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.126145+0000", 2024-09-14T05:04:10.037 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 6.5699999999999998e-06 2024-09-14T05:04:10.037 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.037 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.037 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:10.037 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.126181+0000", 2024-09-14T05:04:10.038 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.5890999999999999e-05 2024-09-14T05:04:10.038 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.038 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.038 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for peered", 2024-09-14T05:04:10.038 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.126189+0000", 2024-09-14T05:04:10.038 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 7.7400000000000004e-06 2024-09-14T05:04:10.038 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.038 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.038 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:10.038 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.528924+0000", 2024-09-14T05:04:10.038 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.40273518899999999 2024-09-14T05:04:10.038 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.038 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.038 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:10.038 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.529033+0000", 2024-09-14T05:04:10.039 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.00010899399999999999 2024-09-14T05:04:10.039 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.039 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.039 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "waiting for subops from 1,3", 2024-09-14T05:04:10.039 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.529150+0000", 2024-09-14T05:04:10.039 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.000117003 2024-09-14T05:04:10.039 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.039 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.039 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "op_commit", 2024-09-14T05:04:10.039 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.531772+0000", 2024-09-14T05:04:10.039 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0026216619999999999 2024-09-14T05:04:10.039 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.039 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.039 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:10.040 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.547421+0000", 2024-09-14T05:04:10.040 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.015648846000000001 2024-09-14T05:04:10.040 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.040 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.040 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:10.040 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.548576+0000", 2024-09-14T05:04:10.040 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.0011547879999999999 2024-09-14T05:04:10.040 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.040 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.040 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "commit_sent", 2024-09-14T05:04:10.040 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.548613+0000", 2024-09-14T05:04:10.040 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.7629000000000002e-05 2024-09-14T05:04:10.040 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:10.041 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:10.041 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "done", 2024-09-14T05:04:10.041 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:09.548652+0000", 2024-09-14T05:04:10.041 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 3.8810000000000003e-05 2024-09-14T05:04:10.041 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:10.041 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:10.041 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:10.041 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:10.041 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:10.041 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T05:04:10.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:10.043 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:10.043 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:10.144 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:10.169 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:10.169 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:10.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:10.271 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:10.295 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:10.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:10.296 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:10.397 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:10.427 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:10.428 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:10.428 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:10.529 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:10.550 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:10.551 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:10.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:10.652 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:10.678 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:10.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:10.679 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:10.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:10 smithi129 ceph-mon[101316]: osdmap e1602: 8 total, 8 up, 7 in 2024-09-14T05:04:10.780 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:10.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:10 smithi039 ceph-mon[106470]: osdmap e1602: 8 total, 8 up, 7 in 2024-09-14T05:04:10.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:10 smithi039 ceph-mon[111235]: osdmap e1602: 8 total, 8 up, 7 in 2024-09-14T05:04:10.801 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:10.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:10.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:10.903 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:10.931 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:10.932 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:10.932 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:11.032 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:11.063 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:11.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:11.065 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:11.165 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:11.186 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:11.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:11.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:11.289 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:11.314 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:11.315 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:11.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:11.393 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-14T05:04:11.416 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:11.468 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:11.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:11.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:11.571 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:11.612 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:11.612 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:11.612 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:11.668 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:04:11.713 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:11.758 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:11.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:11.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:11.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:11 smithi129 ceph-mon[101316]: pgmap v3208: 119 pgs: 1 peering, 3 active+remapped+backfilling, 115 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 677 op/s; 5056/152961 objects misplaced (3.305%); 40 MiB/s, 633 objects/s recovering 2024-09-14T05:04:11.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:11 smithi129 ceph-mon[101316]: osdmap e1603: 8 total, 8 up, 7 in 2024-09-14T05:04:11.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:11 smithi039 ceph-mon[106470]: pgmap v3208: 119 pgs: 1 peering, 3 active+remapped+backfilling, 115 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 677 op/s; 5056/152961 objects misplaced (3.305%); 40 MiB/s, 633 objects/s recovering 2024-09-14T05:04:11.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:11 smithi039 ceph-mon[106470]: osdmap e1603: 8 total, 8 up, 7 in 2024-09-14T05:04:11.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:11 smithi039 ceph-mon[111235]: pgmap v3208: 119 pgs: 1 peering, 3 active+remapped+backfilling, 115 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 677 op/s; 5056/152961 objects misplaced (3.305%); 40 MiB/s, 633 objects/s recovering 2024-09-14T05:04:11.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:11 smithi039 ceph-mon[111235]: osdmap e1603: 8 total, 8 up, 7 in 2024-09-14T05:04:11.861 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:11.896 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:11.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:11.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:11.998 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:12.036 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:12.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:12.037 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:12.138 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:12.184 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:12.185 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:12.185 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:12.286 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:12.331 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:12.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:12.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:12.433 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:12.461 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:12.462 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:12.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:12.563 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:12.590 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:12.590 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:12.591 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:12.691 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:12.720 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:12.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:12.721 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:12.822 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:12.853 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:12.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:12.854 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:12.955 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:12.992 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:12.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:12.993 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:13.094 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:13.116 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:13.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:13.117 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:13.218 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:13.246 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:13.247 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:13.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:13.349 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:13.375 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:13.376 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:13.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:13.477 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:13.525 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:13.525 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:13.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:13.627 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:13.668 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:13.669 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:13.669 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:13.770 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:13.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:13 smithi129 ceph-mon[101316]: pgmap v3210: 119 pgs: 1 peering, 3 active+remapped+backfilling, 115 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 3621/151041 objects misplaced (2.397%); 44 MiB/s, 705 objects/s recovering 2024-09-14T05:04:13.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:13 smithi039 ceph-mon[106470]: pgmap v3210: 119 pgs: 1 peering, 3 active+remapped+backfilling, 115 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 3621/151041 objects misplaced (2.397%); 44 MiB/s, 705 objects/s recovering 2024-09-14T05:04:13.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:13 smithi039 ceph-mon[111235]: pgmap v3210: 119 pgs: 1 peering, 3 active+remapped+backfilling, 115 active+clean; 3.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 489 op/s; 3621/151041 objects misplaced (2.397%); 44 MiB/s, 705 objects/s recovering 2024-09-14T05:04:13.794 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:13.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:13.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:13.896 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:13.918 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:13.918 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:13.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:14.020 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:14.041 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:14.042 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:14.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:14.143 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:14.165 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:14.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:14.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:14.269 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:14.312 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:14.313 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:14.313 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:14.414 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:14.455 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:14.456 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:14.456 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:14.557 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:14.591 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:14.592 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:14.592 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:14.634 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T05:04:14.634 INFO:teuthology.orchestra.run.smithi129.stdout: "ops": [ 2024-09-14T05:04:14.634 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:14.634 INFO:teuthology.orchestra.run.smithi129.stdout: "description": "osd_repop(client.36603.0:184113 58.c e1603/1591 58:3f61b322:::benchmark_data_smithi039_396950_object68500:head v 1603'11583, mlcod=1603'11581)", 2024-09-14T05:04:14.634 INFO:teuthology.orchestra.run.smithi129.stdout: "initiated_at": "2024-09-14T05:04:14.630854+0000", 2024-09-14T05:04:14.634 INFO:teuthology.orchestra.run.smithi129.stdout: "age": 0.0010812630000000001, 2024-09-14T05:04:14.634 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0.001110219, 2024-09-14T05:04:14.634 INFO:teuthology.orchestra.run.smithi129.stdout: "type_data": { 2024-09-14T05:04:14.635 INFO:teuthology.orchestra.run.smithi129.stdout: "flag_point": "started", 2024-09-14T05:04:14.635 INFO:teuthology.orchestra.run.smithi129.stdout: "events": [ 2024-09-14T05:04:14.635 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:14.635 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "initiated", 2024-09-14T05:04:14.635 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:14.630854+0000", 2024-09-14T05:04:14.635 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:14.635 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:14.635 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:14.635 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "throttled", 2024-09-14T05:04:14.635 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:14.630854+0000", 2024-09-14T05:04:14.635 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 0 2024-09-14T05:04:14.635 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:14.635 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:14.635 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "header_read", 2024-09-14T05:04:14.636 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:14.630853+0000", 2024-09-14T05:04:14.636 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 4294967295.999999 2024-09-14T05:04:14.636 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:14.636 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:14.636 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "all_read", 2024-09-14T05:04:14.636 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:14.630861+0000", 2024-09-14T05:04:14.636 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 8.0600000000000008e-06 2024-09-14T05:04:14.636 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:14.636 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:14.636 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "dispatched", 2024-09-14T05:04:14.636 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:14.630881+0000", 2024-09-14T05:04:14.636 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 1.9656999999999998e-05 2024-09-14T05:04:14.637 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:14.637 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:14.637 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "queued_for_pg", 2024-09-14T05:04:14.637 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:14.630891+0000", 2024-09-14T05:04:14.637 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 9.9790000000000005e-06 2024-09-14T05:04:14.637 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:14.637 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:14.637 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "reached_pg", 2024-09-14T05:04:14.637 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:14.630947+0000", 2024-09-14T05:04:14.637 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 5.6103000000000001e-05 2024-09-14T05:04:14.637 INFO:teuthology.orchestra.run.smithi129.stdout: }, 2024-09-14T05:04:14.637 INFO:teuthology.orchestra.run.smithi129.stdout: { 2024-09-14T05:04:14.637 INFO:teuthology.orchestra.run.smithi129.stdout: "event": "started", 2024-09-14T05:04:14.638 INFO:teuthology.orchestra.run.smithi129.stdout: "time": "2024-09-14T05:04:14.630973+0000", 2024-09-14T05:04:14.638 INFO:teuthology.orchestra.run.smithi129.stdout: "duration": 2.6149999999999999e-05 2024-09-14T05:04:14.638 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:14.638 INFO:teuthology.orchestra.run.smithi129.stdout: ] 2024-09-14T05:04:14.638 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:14.638 INFO:teuthology.orchestra.run.smithi129.stdout: } 2024-09-14T05:04:14.638 INFO:teuthology.orchestra.run.smithi129.stdout: ], 2024-09-14T05:04:14.638 INFO:teuthology.orchestra.run.smithi129.stdout: "num_ops": 1 2024-09-14T05:04:14.638 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T05:04:14.693 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:14.721 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:14.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:14.722 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:14.823 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:14.847 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:14.847 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:14.848 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:14.949 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:14.975 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:14.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:14.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:15.077 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:15.099 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:15.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:15.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:15.201 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:15.249 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:15.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:15.250 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:15.351 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:15.377 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:15.377 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:15.378 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:15.451 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-14T05:04:15.479 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:15.524 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:15.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:15.525 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:15.626 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:15.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:15.665 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:15.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:15.703 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:04:15.766 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:15.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:15 smithi129 ceph-mon[101316]: pgmap v3211: 119 pgs: 1 peering, 2 active+remapped+backfilling, 116 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 532 op/s; 3555/148194 objects misplaced (2.399%); 35 MiB/s, 554 objects/s recovering 2024-09-14T05:04:15.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:15 smithi039 ceph-mon[106470]: pgmap v3211: 119 pgs: 1 peering, 2 active+remapped+backfilling, 116 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 532 op/s; 3555/148194 objects misplaced (2.399%); 35 MiB/s, 554 objects/s recovering 2024-09-14T05:04:15.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:15 smithi039 ceph-mon[111235]: pgmap v3211: 119 pgs: 1 peering, 2 active+remapped+backfilling, 116 active+clean; 3.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 532 op/s; 3555/148194 objects misplaced (2.399%); 35 MiB/s, 554 objects/s recovering 2024-09-14T05:04:15.790 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:15.791 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:15.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:15.892 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:15.926 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:15.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:15.927 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:16.028 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:16.065 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:16.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:16.066 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:16.166 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:16.189 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:16.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:16.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:16.291 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:16.316 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:16.316 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:16.317 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:16.417 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:16.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:16.441 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:16.441 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:16.542 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:16.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:16.572 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:16.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:16.673 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:16.697 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:16.698 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:16.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:16.799 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:16.826 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:16.827 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:16.827 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:16.928 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:16.954 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:16.954 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:16.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:17.056 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:17.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:17.082 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:17.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:17.183 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:17.210 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:17.211 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:17.211 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:17.312 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:17.338 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:17.339 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:17.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:17.440 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:17.465 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:17.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:17.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:17.567 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:17.593 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:17.594 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:17.595 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:17.696 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:17.721 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:17.722 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:17.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:17.769 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:17 smithi039 ceph-mon[106470]: pgmap v3212: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 1903/141231 objects misplaced (1.347%); 39 MiB/s, 625 objects/s recovering 2024-09-14T05:04:17.769 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:17 smithi039 ceph-mon[111235]: pgmap v3212: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 1903/141231 objects misplaced (1.347%); 39 MiB/s, 625 objects/s recovering 2024-09-14T05:04:17.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:17 smithi129 ceph-mon[101316]: pgmap v3212: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.9 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 1903/141231 objects misplaced (1.347%); 39 MiB/s, 625 objects/s recovering 2024-09-14T05:04:17.824 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:17.866 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:17.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:17.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:17.968 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:17.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:17.993 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:17.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:18.095 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:18.133 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:18.133 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:18.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:18.234 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:18.258 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:18.259 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:18.259 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:18.360 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:18.382 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:18.382 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:18.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:18.483 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:18.508 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:18.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:18.509 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:18.597 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-14T05:04:18.597 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-14T05:04:18.598 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.3 'trim stale osdmaps' 2024-09-14T05:04:18.610 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:18.697 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:18.697 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:18.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:18.798 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:18.821 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:18.821 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:18.822 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:18.888 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:04:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:04:18] "GET /metrics HTTP/1.1" 200 36772 "" "Prometheus/2.51.0" 2024-09-14T05:04:18.922 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:18.990 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:18.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:18.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:19.092 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:19.117 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:19.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:19.118 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:19.170 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:04:19.219 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:19.248 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:04:19.264 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T05:04:19.264 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T05:04:19.275 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:19.275 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:19.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:19.304 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T05:04:19.304 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T05:04:19.341 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T05:04:19.342 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T05:04:19.376 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:19.381 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T05:04:19.382 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T05:04:19.401 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:19.402 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:19.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:19.419 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T05:04:19.419 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T05:04:19.455 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T05:04:19.455 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T05:04:19.487 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T05:04:19.488 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T05:04:19.502 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:19.520 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T05:04:19.520 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T05:04:19.548 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:19.549 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:19.549 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:19.650 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:19.699 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:19.699 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:19.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:19.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:19 smithi129 ceph-mon[101316]: pgmap v3213: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 1903/139176 objects misplaced (1.367%); 35 MiB/s, 554 objects/s recovering 2024-09-14T05:04:19.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:19 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4098269539' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:04:19.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:19 smithi039 ceph-mon[106470]: pgmap v3213: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 1903/139176 objects misplaced (1.367%); 35 MiB/s, 554 objects/s recovering 2024-09-14T05:04:19.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:19 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4098269539' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:04:19.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:19 smithi039 ceph-mon[111235]: pgmap v3213: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.8 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 670 op/s; 1903/139176 objects misplaced (1.367%); 35 MiB/s, 554 objects/s recovering 2024-09-14T05:04:19.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:19 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4098269539' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:04:19.800 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:19.833 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:19.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:19.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:19.935 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:19.963 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:19.963 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:19.964 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:20.065 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:20.089 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:20.089 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:20.090 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:20.190 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:20.220 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:20.221 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:20.221 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:20.322 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:20.348 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:20.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:20.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:20.451 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:20.479 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:20.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:20.480 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:20.560 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T05:04:20.561 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:04:20.581 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:20.607 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:20.608 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:20.608 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:20.709 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:20.745 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:20.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:20.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:20.847 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:20.870 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:20.871 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:20.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:20.972 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:20.998 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:21.000 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:21.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:21.101 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:21.125 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:21.126 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:21.126 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:21.227 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:21.229 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-14T05:04:21.297 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:21.297 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:21.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:21.398 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:21.439 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:21.440 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:21.441 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:21.511 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:04:21.541 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:21.566 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:21.567 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:21.567 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:21.648 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:21 smithi039 ceph-mon[106470]: pgmap v3214: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.7 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 1295/132921 objects misplaced (0.974%); 21 MiB/s, 334 objects/s recovering 2024-09-14T05:04:21.648 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:21 smithi039 ceph-mon[106470]: osdmap e1604: 8 total, 8 up, 7 in 2024-09-14T05:04:21.648 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:21 smithi039 ceph-mon[111235]: pgmap v3214: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.7 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 1295/132921 objects misplaced (0.974%); 21 MiB/s, 334 objects/s recovering 2024-09-14T05:04:21.649 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:21 smithi039 ceph-mon[111235]: osdmap e1604: 8 total, 8 up, 7 in 2024-09-14T05:04:21.668 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:21.690 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:21.691 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:21.691 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:21.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:21 smithi129 ceph-mon[101316]: pgmap v3214: 119 pgs: 2 active+remapped+backfilling, 117 active+clean; 2.7 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 1295/132921 objects misplaced (0.974%); 21 MiB/s, 334 objects/s recovering 2024-09-14T05:04:21.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:21 smithi129 ceph-mon[101316]: osdmap e1604: 8 total, 8 up, 7 in 2024-09-14T05:04:21.792 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:21.815 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:21.816 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:21.817 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:21.917 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:21.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:21.941 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:21.941 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:22.042 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:22.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:22.064 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:22.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:22.165 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:22.187 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:22.187 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:22.188 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:22.289 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:22.310 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:22.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:22.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:22.412 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:22.439 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:22.439 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:22.440 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:22.541 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:22.569 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:22.570 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:22.570 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:22.671 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:22.693 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:22.693 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:22.694 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:22.795 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:22.817 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:22.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:22.818 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:22.919 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:22.941 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:22.942 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:22.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:23.043 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:23.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:23.066 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:23.066 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:23.167 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:23.188 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:23.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:23.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:23.241 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T05:04:23.241 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:04:23.290 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:23.311 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:23.312 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:23.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:23.413 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:23.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:23.456 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:23.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:23.557 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:23.593 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:23.594 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:23.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:23.620 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:23 smithi129 ceph-mon[101316]: pgmap v3216: 119 pgs: 1 active+clean+remapped, 1 active+remapped+backfilling, 117 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 695 op/s; 606/130173 objects misplaced (0.466%); 17 MiB/s, 270 objects/s recovering 2024-09-14T05:04:23.620 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:23 smithi129 ceph-mon[101316]: osdmap e1605: 8 total, 8 up, 7 in 2024-09-14T05:04:23.621 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:04:23.695 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:23.727 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:23 smithi039 ceph-mon[111235]: pgmap v3216: 119 pgs: 1 active+clean+remapped, 1 active+remapped+backfilling, 117 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 695 op/s; 606/130173 objects misplaced (0.466%); 17 MiB/s, 270 objects/s recovering 2024-09-14T05:04:23.727 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:23 smithi039 ceph-mon[111235]: osdmap e1605: 8 total, 8 up, 7 in 2024-09-14T05:04:23.727 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:04:23.728 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:23 smithi039 ceph-mon[106470]: pgmap v3216: 119 pgs: 1 active+clean+remapped, 1 active+remapped+backfilling, 117 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 695 op/s; 606/130173 objects misplaced (0.466%); 17 MiB/s, 270 objects/s recovering 2024-09-14T05:04:23.728 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:23 smithi039 ceph-mon[106470]: osdmap e1605: 8 total, 8 up, 7 in 2024-09-14T05:04:23.728 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:04:23.739 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:23.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:23.740 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:23.841 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:23.866 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:23.866 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:23.867 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:23.941 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_ops_in_flight 2024-09-14T05:04:23.968 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:23.989 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:23.990 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:23.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:24.091 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:24.115 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:24.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:24.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:24.217 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:24.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:24.273 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:24.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:24.320 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is set 2024-09-14T05:04:24.374 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:24.397 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:24.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:24.398 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:24.499 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:24.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:24.521 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:24.522 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:24.623 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:24.646 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:24.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:24.647 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:24.747 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:24.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:24 smithi129 ceph-mon[101316]: osdmap e1606: 8 total, 8 up, 7 in 2024-09-14T05:04:24.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:24 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/306008464' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:04:24.782 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:24.783 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:24.783 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:24.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:24 smithi039 ceph-mon[106470]: osdmap e1606: 8 total, 8 up, 7 in 2024-09-14T05:04:24.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:24 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/306008464' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:04:24.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:24 smithi039 ceph-mon[111235]: osdmap e1606: 8 total, 8 up, 7 in 2024-09-14T05:04:24.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:24 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/306008464' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:04:24.884 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:24.911 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T05:04:24.923 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:24.924 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:24.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:25.025 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:25.051 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:25.052 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:25.052 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:25.153 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:25.179 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:25.179 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:25.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:25.280 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:25.303 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:25.304 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:25.304 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:25.405 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:25.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:25.459 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:25.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:25.560 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:25.574 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:25 smithi039 ceph-mon[106470]: pgmap v3219: 119 pgs: 1 active+clean+remapped, 1 active+remapped+backfilling, 117 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 606/127536 objects misplaced (0.475%); 12 MiB/s, 199 objects/s recovering 2024-09-14T05:04:25.574 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:25 smithi039 ceph-mon[106470]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:04:25.574 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:25 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/306008464' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T05:04:25.574 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:25 smithi039 ceph-mon[106470]: osdmap e1607: 8 total, 8 up, 7 in 2024-09-14T05:04:25.575 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:25 smithi039 ceph-mon[111235]: pgmap v3219: 119 pgs: 1 active+clean+remapped, 1 active+remapped+backfilling, 117 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 606/127536 objects misplaced (0.475%); 12 MiB/s, 199 objects/s recovering 2024-09-14T05:04:25.575 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:25 smithi039 ceph-mon[111235]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:04:25.575 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:25 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/306008464' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T05:04:25.575 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:25 smithi039 ceph-mon[111235]: osdmap e1607: 8 total, 8 up, 7 in 2024-09-14T05:04:25.584 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:25.585 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:25.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:25.687 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:25.724 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:25.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:25.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:25.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:25 smithi129 ceph-mon[101316]: pgmap v3219: 119 pgs: 1 active+clean+remapped, 1 active+remapped+backfilling, 117 active+clean; 2.6 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 646 op/s; 606/127536 objects misplaced (0.475%); 12 MiB/s, 199 objects/s recovering 2024-09-14T05:04:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:25 smithi129 ceph-mon[101316]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:04:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:25 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/306008464' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T05:04:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:25 smithi129 ceph-mon[101316]: osdmap e1607: 8 total, 8 up, 7 in 2024-09-14T05:04:25.826 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:25.864 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:25.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:25.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:25.966 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:25.991 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:25.992 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:25.992 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:26.093 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:26.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:26.131 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:26.131 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:26.232 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:26.261 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:26.261 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:26.262 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:26.363 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:26.387 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:26.388 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:26.388 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:26.489 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:26.514 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:26.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:26.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:26.595 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:04:26.616 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:26.663 INFO:teuthology.orchestra.run.smithi039.stderr:no valid command found; 10 closest matches: 2024-09-14T05:04:26.663 INFO:teuthology.orchestra.run.smithi039.stderr:0 2024-09-14T05:04:26.663 INFO:teuthology.orchestra.run.smithi039.stderr:1 2024-09-14T05:04:26.663 INFO:teuthology.orchestra.run.smithi039.stderr:2 2024-09-14T05:04:26.663 INFO:teuthology.orchestra.run.smithi039.stderr:abort 2024-09-14T05:04:26.664 INFO:teuthology.orchestra.run.smithi039.stderr:assert 2024-09-14T05:04:26.664 INFO:teuthology.orchestra.run.smithi039.stderr:bench [] [] [] [] 2024-09-14T05:04:26.664 INFO:teuthology.orchestra.run.smithi039.stderr:bluefs debug_inject_read_zeros 2024-09-14T05:04:26.664 INFO:teuthology.orchestra.run.smithi039.stderr:bluefs files list 2024-09-14T05:04:26.664 INFO:teuthology.orchestra.run.smithi039.stderr:bluefs stats 2024-09-14T05:04:26.664 INFO:teuthology.orchestra.run.smithi039.stderr:bluestore allocator dump block 2024-09-14T05:04:26.664 INFO:teuthology.orchestra.run.smithi039.stderr:Error EINVAL: invalid command 2024-09-14T05:04:26.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:26.690 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:26.690 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:26.791 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:26.816 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:26.817 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:26.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:26.918 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:26.956 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:26.957 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:26.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:27.058 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:27.084 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:27.085 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:27.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:27.186 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:27.222 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:27.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:27.223 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:27.324 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:27.348 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:27.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:27.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:27.450 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:27.494 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:27.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:27.496 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:27.597 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:27.636 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:27.637 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:27.637 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:27.738 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:27.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:27 smithi129 ceph-mon[101316]: pgmap v3221: 119 pgs: 119 active+clean; 2.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 595 op/s; 7.8 MiB/s, 124 objects/s recovering 2024-09-14T05:04:27.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:27.785 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:27.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:27.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:27 smithi039 ceph-mon[106470]: pgmap v3221: 119 pgs: 119 active+clean; 2.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 595 op/s; 7.8 MiB/s, 124 objects/s recovering 2024-09-14T05:04:27.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:27 smithi039 ceph-mon[111235]: pgmap v3221: 119 pgs: 119 active+clean; 2.5 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 595 op/s; 7.8 MiB/s, 124 objects/s recovering 2024-09-14T05:04:27.803 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:04:27.829 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-14T05:04:27.830 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-14T05:04:27.831 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.5 'trim stale osdmaps' 2024-09-14T05:04:27.886 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:27.910 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:27.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:27.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:28.012 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:28.066 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:28.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:28.068 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:28.168 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:28.190 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:28.191 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:28.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:28.292 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:28.317 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:28.318 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:28.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:28.352 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T05:04:28.420 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:28.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:28.465 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:28.465 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:28.566 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:28.588 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:28.589 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:28.589 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:28.690 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:28.716 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:28.716 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:28.717 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:28.817 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:28.852 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:28.853 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:28.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:28.954 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:28.994 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:28.994 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:28.995 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:04:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:04:28] "GET /metrics HTTP/1.1" 200 36775 "" "Prometheus/2.51.0" 2024-09-14T05:04:29.095 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:29.121 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:29.121 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:29.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:29.222 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:29.269 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:29.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:29.270 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:29.371 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:29.408 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:29.409 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:29.409 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:29.510 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:29.539 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:29.540 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:29.540 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:29.641 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:29.659 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:29 smithi129 ceph-mon[101316]: pgmap v3222: 119 pgs: 119 active+clean; 2.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 595 op/s; 6.2 MiB/s, 99 objects/s recovering 2024-09-14T05:04:29.673 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:29.674 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:29.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:29.775 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:29.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:29 smithi039 ceph-mon[106470]: pgmap v3222: 119 pgs: 119 active+clean; 2.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 595 op/s; 6.2 MiB/s, 99 objects/s recovering 2024-09-14T05:04:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:29 smithi039 ceph-mon[111235]: pgmap v3222: 119 pgs: 119 active+clean; 2.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 595 op/s; 6.2 MiB/s, 99 objects/s recovering 2024-09-14T05:04:29.814 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:29.815 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:29.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:29.917 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:29.947 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:29.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:29.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:30.053 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:30.091 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:30.091 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:30.091 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:30.192 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:30.232 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:30.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:30.233 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:30.334 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:30.361 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:30.362 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:30.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:30.463 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:30.511 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:30.512 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:30.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:30.614 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:30.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:30.657 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:30.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:30.758 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:30.785 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:30.785 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:30.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:30.887 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:30.917 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:30.917 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:30.918 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:31.019 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:31.042 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:31.043 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:31.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:31.144 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:31.171 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:31.171 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:31.172 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:31.273 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:31.295 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:31.296 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:31.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:31.389 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:31 smithi039 ceph-mon[106470]: pgmap v3223: 119 pgs: 119 active+clean; 2.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 4.8 MiB/s, 76 objects/s recovering 2024-09-14T05:04:31.390 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:31 smithi039 ceph-mon[111235]: pgmap v3223: 119 pgs: 119 active+clean; 2.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 4.8 MiB/s, 76 objects/s recovering 2024-09-14T05:04:31.397 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:31.443 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:31.444 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:31.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:31.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:31 smithi129 ceph-mon[101316]: pgmap v3223: 119 pgs: 119 active+clean; 2.3 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 4.8 MiB/s, 76 objects/s recovering 2024-09-14T05:04:31.545 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:31.588 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:31.588 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:31.589 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:31.689 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:31.736 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T05:04:31.737 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T05:04:31.737 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T05:04:31.737 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T05:04:31.739 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:31.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:31.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:31.774 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T05:04:31.775 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T05:04:31.821 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T05:04:31.821 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T05:04:31.842 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:31.852 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T05:04:31.852 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T05:04:31.863 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:31.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:31.864 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:31.880 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:04:31.883 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T05:04:31.883 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T05:04:31.923 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T05:04:31.924 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T05:04:31.962 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T05:04:31.962 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T05:04:31.965 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:31.988 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:31.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:31.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:32.044 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T05:04:32.044 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [ 2024-09-14T05:04:32.044 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.044 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:197268 58.a 58:53797742:::benchmark_data_smithi039_396950_object81655:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1607)", 2024-09-14T05:04:32.044 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:04:32.020657+0000", 2024-09-14T05:04:32.044 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.021913104999999999, 2024-09-14T05:04:32.044 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.021942735000000001, 2024-09-14T05:04:32.045 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:04:32.045 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:04:32.045 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "waiting for sub ops", 2024-09-14T05:04:32.045 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:04:32.045 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:04:32.045 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:04:32.045 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 197268 2024-09-14T05:04:32.045 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.045 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:04:32.045 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.045 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:04:32.046 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:32.020657+0000", 2024-09-14T05:04:32.046 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:04:32.046 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.046 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.046 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:04:32.046 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:32.020657+0000", 2024-09-14T05:04:32.046 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:04:32.046 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.046 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.046 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:04:32.047 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:32.020659+0000", 2024-09-14T05:04:32.047 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7549999999999999e-06 2024-09-14T05:04:32.047 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.047 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.047 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:04:32.047 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:32.020663+0000", 2024-09-14T05:04:32.047 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.9790000000000004e-06 2024-09-14T05:04:32.047 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.047 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.048 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:04:32.048 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:32.020673+0000", 2024-09-14T05:04:32.048 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0236e-05 2024-09-14T05:04:32.048 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.048 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.048 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:04:32.048 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:32.020680+0000", 2024-09-14T05:04:32.048 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7050000000000002e-06 2024-09-14T05:04:32.049 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.049 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.049 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:04:32.049 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:32.020906+0000", 2024-09-14T05:04:32.049 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000226114 2024-09-14T05:04:32.049 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.049 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.049 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:04:32.049 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:32.021041+0000", 2024-09-14T05:04:32.049 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013492500000000001 2024-09-14T05:04:32.050 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.050 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.050 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,7", 2024-09-14T05:04:32.050 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:32.021161+0000", 2024-09-14T05:04:32.050 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000119873 2024-09-14T05:04:32.050 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.050 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.050 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:32.050 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:32.025187+0000", 2024-09-14T05:04:32.050 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0040267130000000003 2024-09-14T05:04:32.050 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.050 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.050 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:04:32.051 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:32.025760+0000", 2024-09-14T05:04:32.051 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00057272300000000003 2024-09-14T05:04:32.051 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:04:32.051 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:04:32.051 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:04:32.051 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.051 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.051 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:197278 58.a 58:5537352d:::benchmark_data_smithi039_396950_object81665:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1607)", 2024-09-14T05:04:32.051 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:04:32.034951+0000", 2024-09-14T05:04:32.051 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.0076197060000000004, 2024-09-14T05:04:32.051 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.007728965, 2024-09-14T05:04:32.051 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:04:32.051 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:04:32.052 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "waiting for sub ops", 2024-09-14T05:04:32.052 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:04:32.052 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:04:32.052 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:04:32.052 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 197278 2024-09-14T05:04:32.052 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.052 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:04:32.052 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.052 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:04:32.052 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:32.034951+0000", 2024-09-14T05:04:32.052 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:04:32.052 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.053 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.053 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:04:32.053 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:32.034951+0000", 2024-09-14T05:04:32.053 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:04:32.053 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.053 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.053 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:04:32.053 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:32.034952+0000", 2024-09-14T05:04:32.053 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5650000000000001e-06 2024-09-14T05:04:32.053 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.053 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.053 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:04:32.053 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:32.034958+0000", 2024-09-14T05:04:32.054 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6529999999999999e-06 2024-09-14T05:04:32.054 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.054 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.054 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:04:32.054 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:32.034971+0000", 2024-09-14T05:04:32.054 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3078000000000001e-05 2024-09-14T05:04:32.054 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.054 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.054 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:04:32.054 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:32.034979+0000", 2024-09-14T05:04:32.054 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0879999999999995e-06 2024-09-14T05:04:32.054 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.054 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.055 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:04:32.055 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:32.035040+0000", 2024-09-14T05:04:32.055 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1373999999999999e-05 2024-09-14T05:04:32.055 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.055 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.055 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:04:32.055 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:32.035199+0000", 2024-09-14T05:04:32.055 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015831800000000001 2024-09-14T05:04:32.055 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.055 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.055 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,7", 2024-09-14T05:04:32.055 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:32.035313+0000", 2024-09-14T05:04:32.055 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011489699999999999 2024-09-14T05:04:32.055 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.056 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.056 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:04:32.056 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:32.038819+0000", 2024-09-14T05:04:32.056 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0035058279999999999 2024-09-14T05:04:32.056 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.056 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.056 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:04:32.056 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:32.039180+0000", 2024-09-14T05:04:32.056 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00036080200000000002 2024-09-14T05:04:32.056 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:04:32.056 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:04:32.056 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:04:32.056 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.057 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.057 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:197283 58.a 58:5556a3f4:::benchmark_data_smithi039_396950_object81670:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1607)", 2024-09-14T05:04:32.057 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:04:32.040586+0000", 2024-09-14T05:04:32.057 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 0.001984258, 2024-09-14T05:04:32.057 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.002163889, 2024-09-14T05:04:32.057 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:04:32.057 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:04:32.057 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "waiting for sub ops", 2024-09-14T05:04:32.057 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:04:32.057 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:04:32.057 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:04:32.057 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 197283 2024-09-14T05:04:32.057 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.058 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:04:32.058 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.058 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:04:32.058 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:32.040586+0000", 2024-09-14T05:04:32.058 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:04:32.058 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.058 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.058 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:04:32.058 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:32.040586+0000", 2024-09-14T05:04:32.058 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:04:32.058 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.058 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.058 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:04:32.058 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:32.040588+0000", 2024-09-14T05:04:32.059 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.584e-06 2024-09-14T05:04:32.059 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.059 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.059 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:04:32.059 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:32.040592+0000", 2024-09-14T05:04:32.059 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.527e-06 2024-09-14T05:04:32.059 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.059 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.059 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:04:32.059 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:32.040606+0000", 2024-09-14T05:04:32.059 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3973e-05 2024-09-14T05:04:32.059 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.059 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.059 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:04:32.060 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:32.040613+0000", 2024-09-14T05:04:32.060 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7859999999999999e-06 2024-09-14T05:04:32.060 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.060 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.060 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:04:32.060 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:32.040683+0000", 2024-09-14T05:04:32.060 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.0088999999999999e-05 2024-09-14T05:04:32.060 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.060 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.060 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:04:32.060 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:32.040791+0000", 2024-09-14T05:04:32.060 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000108498 2024-09-14T05:04:32.061 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:04:32.061 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:04:32.061 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,7", 2024-09-14T05:04:32.061 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:32.040897+0000", 2024-09-14T05:04:32.064 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000105746 2024-09-14T05:04:32.064 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:04:32.064 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:04:32.064 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:04:32.064 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:04:32.065 INFO:teuthology.orchestra.run.smithi039.stdout: ], 2024-09-14T05:04:32.065 INFO:teuthology.orchestra.run.smithi039.stdout: "num_ops": 3 2024-09-14T05:04:32.065 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T05:04:32.090 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:32.132 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:32.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:32.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:32.234 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:32.257 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:32.258 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:32.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:32.359 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:32.401 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:32.401 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:32.402 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:32.502 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:32.541 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:32.542 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:32.543 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:32.643 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:32.687 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:32 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1724081560' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:04:32.687 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:32 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1724081560' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:04:32.688 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:32.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:32.689 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:32.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:32 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1724081560' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:04:32.790 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:32.817 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:32.818 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:32.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:32.919 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:32.945 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:32.949 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:32.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:33.050 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:33.073 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:33.074 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:33.074 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:33.175 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:33.188 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_blocked_ops 2024-09-14T05:04:33.199 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:33.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:33.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:33.301 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:33.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:33.349 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:33.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:33.438 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:33 smithi039 ceph-mon[106470]: pgmap v3224: 119 pgs: 119 active+clean; 2.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 723 op/s; 4.3 MiB/s, 68 objects/s recovering 2024-09-14T05:04:33.439 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:33 smithi039 ceph-mon[111235]: pgmap v3224: 119 pgs: 119 active+clean; 2.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 723 op/s; 4.3 MiB/s, 68 objects/s recovering 2024-09-14T05:04:33.451 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:33.474 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:33.475 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:33.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:33.576 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:33.600 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:33.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:33.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:33.702 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:33.727 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:33.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:33.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:33.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:33 smithi129 ceph-mon[101316]: pgmap v3224: 119 pgs: 119 active+clean; 2.2 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 723 op/s; 4.3 MiB/s, 68 objects/s recovering 2024-09-14T05:04:33.829 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:33.856 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:33.857 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:33.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:33.958 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:33.982 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:33.983 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:33.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:34.084 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:34.128 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:34.129 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:34.129 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:34.230 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:34.253 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:34.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:34.254 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:34.355 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:34.378 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:34.379 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:34.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:34.438 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:04:34.480 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:34.503 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:34.503 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:34.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:34.605 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:34.629 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:34.630 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:34.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:34.731 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:34.766 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:34.767 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:34.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:34.869 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:34.894 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:34.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:34.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:34.997 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:35.020 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:35.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:35.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:35.122 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:35.181 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:35.181 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:35.182 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:35.282 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:35.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:35.325 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:35.325 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:35.341 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:04:35.427 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:35.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:35.486 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:35.487 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:35.568 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:35 smithi039 ceph-mon[106470]: pgmap v3225: 119 pgs: 119 active+clean; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 780 op/s; 3.7 MiB/s, 59 objects/s recovering 2024-09-14T05:04:35.568 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:35 smithi039 ceph-mon[111235]: pgmap v3225: 119 pgs: 119 active+clean; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 780 op/s; 3.7 MiB/s, 59 objects/s recovering 2024-09-14T05:04:35.587 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:35.612 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:35.618 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:35.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:35.719 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:35.742 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:35.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:35.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:35.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:35 smithi129 ceph-mon[101316]: pgmap v3225: 119 pgs: 119 active+clean; 2.1 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 780 op/s; 3.7 MiB/s, 59 objects/s recovering 2024-09-14T05:04:35.844 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:35.868 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:35.869 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:35.869 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:35.932 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T05:04:35.970 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:36.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:36.023 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:36.023 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:36.124 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:36.150 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:36.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:36.152 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:36.252 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:36.297 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:36.298 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:36.298 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:36.399 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:36.423 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:36.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:36.424 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:36.525 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:36.562 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:36.562 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:36.563 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:36.664 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:36.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:36.714 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:36.714 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:36.814 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:36.852 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:36.853 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:36.853 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:36.954 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:36.977 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:36.977 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:36.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:37.078 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:37.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:37.107 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:37.108 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:37.209 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:37.269 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:37.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:37.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:37.371 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:37.395 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:37.396 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:37.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:37.497 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:37.521 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:37.521 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:37.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:37.622 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:37.658 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:37.658 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:37.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:37.747 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:37 smithi039 ceph-mon[106470]: pgmap v3226: 119 pgs: 119 active+clean; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 934 op/s; 3.2 MiB/s, 50 objects/s recovering 2024-09-14T05:04:37.747 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:04:37.747 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:37 smithi039 ceph-mon[111235]: pgmap v3226: 119 pgs: 119 active+clean; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 934 op/s; 3.2 MiB/s, 50 objects/s recovering 2024-09-14T05:04:37.748 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:04:37.760 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:37.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:37 smithi129 ceph-mon[101316]: pgmap v3226: 119 pgs: 119 active+clean; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 934 op/s; 3.2 MiB/s, 50 objects/s recovering 2024-09-14T05:04:37.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:04:37.797 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:37.798 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:37.798 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:37.899 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:37.960 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:37.961 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:37.961 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:38.062 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:38.086 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:38.086 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:38.086 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:38.187 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:38.208 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:38.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:38.209 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:38.310 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:38.336 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:38.336 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:38.337 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:38.389 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is unset 2024-09-14T05:04:38.438 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:38.487 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:38.488 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:38.488 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:38.589 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:38.633 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:38.634 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:38.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:38.648 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:38 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3015814428' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:04:38.649 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:38 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:04:38.649 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:38 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3015814428' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:04:38.649 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:38 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:04:38.735 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:38.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:38 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3015814428' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:04:38.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:38 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:04:38.780 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:38.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:38.782 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:38.882 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:38.921 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:38.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:38.922 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:38.938 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:04:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:04:38] "GET /metrics HTTP/1.1" 200 36768 "" "Prometheus/2.51.0" 2024-09-14T05:04:39.023 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:39.064 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:39.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:39.065 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:39.166 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:39.190 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:39.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:39.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:39.291 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:39.336 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:39.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:39.337 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:39.438 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:39.479 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:39.480 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:39.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:39.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:39 smithi039 ceph-mon[106470]: pgmap v3227: 119 pgs: 119 active+clean; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 819 op/s 2024-09-14T05:04:39.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:39 smithi039 ceph-mon[106470]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:04:39.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:39 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T05:04:39.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:39 smithi039 ceph-mon[106470]: osdmap e1608: 8 total, 8 up, 7 in 2024-09-14T05:04:39.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:39 smithi039 ceph-mon[111235]: pgmap v3227: 119 pgs: 119 active+clean; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 819 op/s 2024-09-14T05:04:39.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:39 smithi039 ceph-mon[111235]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:04:39.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:39 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T05:04:39.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:39 smithi039 ceph-mon[111235]: osdmap e1608: 8 total, 8 up, 7 in 2024-09-14T05:04:39.581 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:39.622 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:39.623 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:39.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:39.724 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:39.747 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:39.748 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:39.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:39.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:39 smithi129 ceph-mon[101316]: pgmap v3227: 119 pgs: 119 active+clean; 1.9 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 819 op/s 2024-09-14T05:04:39.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:39 smithi129 ceph-mon[101316]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:04:39.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:39 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T05:04:39.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:39 smithi129 ceph-mon[101316]: osdmap e1608: 8 total, 8 up, 7 in 2024-09-14T05:04:39.850 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:39.877 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:39.878 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:39.878 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:39.979 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:40.003 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:40.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:40.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:40.105 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:40.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:40.130 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:40.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:40.182 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:04:40.231 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:40.282 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:40.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:40.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:40.384 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:40.435 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:40.435 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:40.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:40.536 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:40.580 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:40.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:40.581 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:40.654 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:40 smithi039 ceph-mon[106470]: 1.7 scrub starts 2024-09-14T05:04:40.654 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:40 smithi039 ceph-mon[106470]: 1.7 scrub ok 2024-09-14T05:04:40.655 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:40 smithi039 ceph-mon[111235]: 1.7 scrub starts 2024-09-14T05:04:40.655 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:40 smithi039 ceph-mon[111235]: 1.7 scrub ok 2024-09-14T05:04:40.682 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:40.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:40.735 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:40.736 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:40.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:40 smithi129 ceph-mon[101316]: 1.7 scrub starts 2024-09-14T05:04:40.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:40 smithi129 ceph-mon[101316]: 1.7 scrub ok 2024-09-14T05:04:40.836 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:40.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:40.862 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:40.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:40.963 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:41.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:41.012 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:41.012 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:41.113 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:41.149 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:41.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:41.150 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:41.251 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:41.287 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:41.288 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:41.288 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:41.389 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:41.427 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:41.427 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:41.428 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:41.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:41 smithi129 ceph-mon[101316]: 1.1e scrub starts 2024-09-14T05:04:41.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:41 smithi129 ceph-mon[101316]: 1.1e scrub ok 2024-09-14T05:04:41.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:41 smithi129 ceph-mon[101316]: pgmap v3229: 119 pgs: 119 active+clean; 1.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 907 op/s 2024-09-14T05:04:41.529 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:41.574 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:41.575 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:41.575 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:41.660 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:41 smithi039 ceph-mon[106470]: 1.1e scrub starts 2024-09-14T05:04:41.660 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:41 smithi039 ceph-mon[106470]: 1.1e scrub ok 2024-09-14T05:04:41.660 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:41 smithi039 ceph-mon[106470]: pgmap v3229: 119 pgs: 119 active+clean; 1.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 907 op/s 2024-09-14T05:04:41.661 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:41 smithi039 ceph-mon[111235]: 1.1e scrub starts 2024-09-14T05:04:41.661 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:41 smithi039 ceph-mon[111235]: 1.1e scrub ok 2024-09-14T05:04:41.661 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:41 smithi039 ceph-mon[111235]: pgmap v3229: 119 pgs: 119 active+clean; 1.8 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 907 op/s 2024-09-14T05:04:41.676 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:41.722 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:41.723 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:41.723 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:41.824 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:41.849 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:41.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:41.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:41.950 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:41.991 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:41.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:41.992 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:42.093 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:42.115 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:42.116 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:42.116 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:42.217 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:42.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:42.240 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:42.240 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:42.341 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:42.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:42.366 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:42.366 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:42.467 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:42.495 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:42.496 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:42.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:42.597 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:42.638 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:42.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:42.639 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:42.740 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:42.778 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:42.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:42.779 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:42.879 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:42.939 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:42.940 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:42.940 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:43.041 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:43.062 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:43.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:43.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:43.164 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:43.209 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:43.209 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:43.210 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:43.310 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:43.333 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:43.334 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:43.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:43.435 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:43.480 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:43.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:43.481 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:43.565 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T05:04:43.582 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:43.609 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:43.610 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:43.611 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:43.711 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:43.748 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:43.749 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:43.749 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:43.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:43 smithi129 ceph-mon[101316]: pgmap v3230: 119 pgs: 119 active+clean; 1.7 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 892 op/s 2024-09-14T05:04:43.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:43 smithi039 ceph-mon[106470]: pgmap v3230: 119 pgs: 119 active+clean; 1.7 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 892 op/s 2024-09-14T05:04:43.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:43 smithi039 ceph-mon[111235]: pgmap v3230: 119 pgs: 119 active+clean; 1.7 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 892 op/s 2024-09-14T05:04:43.850 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:43.888 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:43.889 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:43.889 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:43.990 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:44.031 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:44.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:44.032 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:44.133 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:44.158 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:44.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:44.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:44.194 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T05:04:44.194 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [], 2024-09-14T05:04:44.194 INFO:teuthology.orchestra.run.smithi039.stdout: "complaint_time": 30, 2024-09-14T05:04:44.194 INFO:teuthology.orchestra.run.smithi039.stdout: "num_blocked_ops": 0 2024-09-14T05:04:44.194 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T05:04:44.260 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:44.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:44.285 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:44.286 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:44.387 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:44.424 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:44.425 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:44.426 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:44.527 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:44.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:44 smithi039 ceph-mon[106470]: 1.17 scrub starts 2024-09-14T05:04:44.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:44 smithi039 ceph-mon[106470]: 1.17 scrub ok 2024-09-14T05:04:44.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:44 smithi039 ceph-mon[111235]: 1.17 scrub starts 2024-09-14T05:04:44.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:44 smithi039 ceph-mon[111235]: 1.17 scrub ok 2024-09-14T05:04:44.567 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:44.568 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:44.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:44.669 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:44.708 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:44.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:44.709 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:44.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:44 smithi129 ceph-mon[101316]: 1.17 scrub starts 2024-09-14T05:04:44.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:44 smithi129 ceph-mon[101316]: 1.17 scrub ok 2024-09-14T05:04:44.810 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:44.832 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:44.833 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:44.833 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:44.934 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:44.972 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:44.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:44.973 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:45.073 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:45.098 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:45.099 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:45.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:45.200 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:45.238 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:45.239 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:45.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:45.340 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:45.366 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:45.366 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:45.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:45.467 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:45.490 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:45.491 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:45.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:45.592 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:45.634 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:45.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:45.636 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:45.737 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:45.759 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:45.760 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:45.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:45.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:45 smithi129 ceph-mon[101316]: 1.15 scrub starts 2024-09-14T05:04:45.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:45 smithi129 ceph-mon[101316]: 1.15 scrub ok 2024-09-14T05:04:45.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:45 smithi129 ceph-mon[101316]: pgmap v3231: 119 pgs: 119 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 887 op/s 2024-09-14T05:04:45.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:45 smithi039 ceph-mon[106470]: 1.15 scrub starts 2024-09-14T05:04:45.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:45 smithi039 ceph-mon[106470]: 1.15 scrub ok 2024-09-14T05:04:45.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:45 smithi039 ceph-mon[106470]: pgmap v3231: 119 pgs: 119 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 887 op/s 2024-09-14T05:04:45.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:45 smithi039 ceph-mon[111235]: 1.15 scrub starts 2024-09-14T05:04:45.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:45 smithi039 ceph-mon[111235]: 1.15 scrub ok 2024-09-14T05:04:45.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:45 smithi039 ceph-mon[111235]: pgmap v3231: 119 pgs: 119 active+clean; 1.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 887 op/s 2024-09-14T05:04:45.861 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:45.882 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:45.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:45.883 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:45.984 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:46.006 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:46.007 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:46.007 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:46.108 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:46.130 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:46.130 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:46.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:46.232 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:46.253 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:46.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:46.254 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:46.355 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:46.399 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:46.400 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:46.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:46.501 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:46.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:46.544 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:46.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:46.645 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:46.705 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:46.705 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:46.705 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:46.806 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:46.830 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:46.831 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:46.832 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:46.863 INFO:teuthology.orchestra.run.smithi039.stderr:no valid command found; 10 closest matches: 2024-09-14T05:04:46.864 INFO:teuthology.orchestra.run.smithi039.stderr:0 2024-09-14T05:04:46.864 INFO:teuthology.orchestra.run.smithi039.stderr:1 2024-09-14T05:04:46.864 INFO:teuthology.orchestra.run.smithi039.stderr:2 2024-09-14T05:04:46.864 INFO:teuthology.orchestra.run.smithi039.stderr:abort 2024-09-14T05:04:46.864 INFO:teuthology.orchestra.run.smithi039.stderr:assert 2024-09-14T05:04:46.864 INFO:teuthology.orchestra.run.smithi039.stderr:bench [] [] [] [] 2024-09-14T05:04:46.864 INFO:teuthology.orchestra.run.smithi039.stderr:bluefs debug_inject_read_zeros 2024-09-14T05:04:46.864 INFO:teuthology.orchestra.run.smithi039.stderr:bluefs files list 2024-09-14T05:04:46.864 INFO:teuthology.orchestra.run.smithi039.stderr:bluefs stats 2024-09-14T05:04:46.864 INFO:teuthology.orchestra.run.smithi039.stderr:bluestore allocator dump block 2024-09-14T05:04:46.864 INFO:teuthology.orchestra.run.smithi039.stderr:Error EINVAL: invalid command 2024-09-14T05:04:46.933 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:46.955 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:46.956 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:46.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:47.057 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:47.080 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:47.080 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:47.081 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:47.182 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:47.204 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:47.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:47.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:47.306 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:47.347 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:47.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:47.348 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:47.449 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:47.489 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:47.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:47.490 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:47.539 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:47 smithi039 ceph-mon[106470]: pgmap v3232: 119 pgs: 119 active+clean; 1.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 767 op/s 2024-09-14T05:04:47.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:47 smithi039 ceph-mon[111235]: pgmap v3232: 119 pgs: 119 active+clean; 1.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 767 op/s 2024-09-14T05:04:47.590 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:47.635 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:47.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:47.636 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:47.737 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:47.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:47 smithi129 ceph-mon[101316]: pgmap v3232: 119 pgs: 119 active+clean; 1.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 767 op/s 2024-09-14T05:04:47.777 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:47.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:47.778 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:47.825 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-14T05:04:47.825 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-14T05:04:47.826 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.6 'trim stale osdmaps' 2024-09-14T05:04:47.879 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:47.938 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:47.939 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:47.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:48.040 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:48.063 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:48.064 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:48.064 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:48.165 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:48.188 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:48.189 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:48.189 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:48.290 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:48.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:48.337 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:48.337 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:48.438 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:48.480 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:48.481 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:48.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:48.582 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:48.628 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:48.628 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:48.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:48.665 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.0 dump_historic_ops 2024-09-14T05:04:48.730 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:48.800 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:48.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:48.802 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:48.882 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:04:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:04:48] "GET /metrics HTTP/1.1" 200 36768 "" "Prometheus/2.51.0" 2024-09-14T05:04:48.903 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:48.926 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:48.926 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:48.927 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:48.966 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T05:04:48.967 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T05:04:48.997 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:04:49.027 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:49.028 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T05:04:49.029 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T05:04:49.069 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T05:04:49.069 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T05:04:49.104 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T05:04:49.105 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T05:04:49.109 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:49.110 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:49.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:49.139 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T05:04:49.140 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T05:04:49.175 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T05:04:49.176 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T05:04:49.209 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T05:04:49.209 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T05:04:49.212 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:49.234 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:49.235 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:49.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:49.241 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T05:04:49.241 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T05:04:49.336 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:49.361 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:49.362 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:49.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:49.463 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:49.510 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:49.511 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:49.511 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:49.612 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:49.634 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:49.635 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:49.636 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:49.736 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:49.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:49 smithi129 ceph-mon[101316]: pgmap v3233: 119 pgs: 119 active+clean; 1.3 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 868 op/s 2024-09-14T05:04:49.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:49 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1942506350' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:04:49.780 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:49.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:49.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:49.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:49 smithi039 ceph-mon[106470]: pgmap v3233: 119 pgs: 119 active+clean; 1.3 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 868 op/s 2024-09-14T05:04:49.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:49 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1942506350' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:04:49.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:49 smithi039 ceph-mon[111235]: pgmap v3233: 119 pgs: 119 active+clean; 1.3 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 868 op/s 2024-09-14T05:04:49.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:49 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1942506350' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:04:49.883 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:49.911 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:49.912 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:49.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:50.013 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:50.058 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:50.059 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:50.059 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:50.160 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:50.197 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:50.197 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:50.198 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:50.298 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:50.321 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:50.322 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:50.322 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:50.423 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:50.447 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:50.447 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:50.448 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:50.549 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:50.584 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:04:50.585 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:50.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:50.586 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:50.687 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:50.719 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:50 smithi039 ceph-mon[106470]: 1.1a scrub starts 2024-09-14T05:04:50.719 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:50 smithi039 ceph-mon[106470]: 1.1a scrub ok 2024-09-14T05:04:50.719 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:50 smithi039 ceph-mon[111235]: 1.1a scrub starts 2024-09-14T05:04:50.719 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:50 smithi039 ceph-mon[111235]: 1.1a scrub ok 2024-09-14T05:04:50.737 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:50.738 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:50.738 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:50.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:50 smithi129 ceph-mon[101316]: 1.1a scrub starts 2024-09-14T05:04:50.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:50 smithi129 ceph-mon[101316]: 1.1a scrub ok 2024-09-14T05:04:50.839 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:50.878 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:50.879 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:50.879 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:50.981 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:51.020 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:51.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:51.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:51.121 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:51.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:51.145 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:51.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:51.245 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:51.268 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:51.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:51.270 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:51.370 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:51.414 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:51.415 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:51.415 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:51.491 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:04:51.516 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:51.540 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:51.541 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:51.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:51.642 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:51.686 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:51.686 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:51.687 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:51.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:51 smithi129 ceph-mon[101316]: pgmap v3234: 119 pgs: 119 active+clean; 1.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 799 op/s 2024-09-14T05:04:51.788 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:51.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:51 smithi039 ceph-mon[106470]: pgmap v3234: 119 pgs: 119 active+clean; 1.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 799 op/s 2024-09-14T05:04:51.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:51 smithi039 ceph-mon[111235]: pgmap v3234: 119 pgs: 119 active+clean; 1.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 799 op/s 2024-09-14T05:04:51.828 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:51.828 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:51.829 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:51.930 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:51.971 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:51.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:51.972 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:52.073 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:52.097 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:52.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:52.098 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:52.173 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T05:04:52.198 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:52.223 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:52.223 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:52.224 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:52.324 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:52.371 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:52.372 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:52.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:52.473 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:52.518 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:52.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:52.519 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:52.620 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:52.663 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:52.664 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:52.664 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:52.765 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:52.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:04:52.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:04:52.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:04:52.802 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:52.802 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:52.803 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:52.904 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:52.927 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:52.927 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:52.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:53.029 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:53.056 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:53.057 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:53.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:53.158 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:53.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:53.204 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:53.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:53.305 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:53.331 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:53.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:53.332 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:53.433 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:53.475 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:53.476 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:53.476 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:53.577 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:53.603 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:53.603 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:53.604 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:53.705 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:53.742 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:53.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:53.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:53.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:53 smithi129 ceph-mon[101316]: 1.e scrub starts 2024-09-14T05:04:53.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:53 smithi129 ceph-mon[101316]: 1.e scrub ok 2024-09-14T05:04:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:53 smithi129 ceph-mon[101316]: pgmap v3235: 119 pgs: 119 active+clean; 1.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 888 op/s 2024-09-14T05:04:53.772 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:53 smithi039 ceph-mon[106470]: 1.e scrub starts 2024-09-14T05:04:53.772 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:53 smithi039 ceph-mon[106470]: 1.e scrub ok 2024-09-14T05:04:53.772 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:53 smithi039 ceph-mon[106470]: pgmap v3235: 119 pgs: 119 active+clean; 1.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 888 op/s 2024-09-14T05:04:53.773 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:53 smithi039 ceph-mon[111235]: 1.e scrub starts 2024-09-14T05:04:53.773 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:53 smithi039 ceph-mon[111235]: 1.e scrub ok 2024-09-14T05:04:53.773 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:53 smithi039 ceph-mon[111235]: pgmap v3235: 119 pgs: 119 active+clean; 1.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 888 op/s 2024-09-14T05:04:53.844 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:53.892 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:53.892 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:53.893 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:53.993 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:54.016 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:54.016 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:54.017 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:54.118 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:54.165 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:54.165 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:54.166 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:54.267 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:54.313 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:54.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:54.314 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:54.415 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:54.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:54.465 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:54.465 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:54.501 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is unset 2024-09-14T05:04:54.566 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:54.615 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:54.616 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:54.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:54.707 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:54 smithi039 ceph-mon[106470]: 1.1b scrub starts 2024-09-14T05:04:54.707 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:54 smithi039 ceph-mon[106470]: 1.1b scrub ok 2024-09-14T05:04:54.707 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:54 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2073526815' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:04:54.709 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:54 smithi039 ceph-mon[111235]: 1.1b scrub starts 2024-09-14T05:04:54.709 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:54 smithi039 ceph-mon[111235]: 1.1b scrub ok 2024-09-14T05:04:54.709 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:54 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2073526815' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:04:54.717 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:54.740 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:54.741 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:54.741 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:54.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:54 smithi129 ceph-mon[101316]: 1.1b scrub starts 2024-09-14T05:04:54.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:54 smithi129 ceph-mon[101316]: 1.1b scrub ok 2024-09-14T05:04:54.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:54 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2073526815' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:04:54.842 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:54.875 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:54.876 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:54.877 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:54.978 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:55.002 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:55.003 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:55.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:55.104 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:55.130 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:55.131 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:55.131 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:55.232 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:55.255 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:55.255 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:55.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:55.357 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:55.380 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:55.381 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:55.381 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:55.482 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:55.509 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:55.510 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:55.510 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:55.611 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:55.651 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:55.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:55.652 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:55.753 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:55.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:55 smithi129 ceph-mon[101316]: pgmap v3236: 119 pgs: 119 active+clean; 1.1 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 821 op/s 2024-09-14T05:04:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:55 smithi129 ceph-mon[101316]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T05:04:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:55 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T05:04:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:55 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2073526815' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T05:04:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:55 smithi129 ceph-mon[101316]: osdmap e1609: 8 total, 8 up, 7 in 2024-09-14T05:04:55.776 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:55.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:55.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:55.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:55 smithi039 ceph-mon[106470]: pgmap v3236: 119 pgs: 119 active+clean; 1.1 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 821 op/s 2024-09-14T05:04:55.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:55 smithi039 ceph-mon[106470]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T05:04:55.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:55 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T05:04:55.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:55 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2073526815' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T05:04:55.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:55 smithi039 ceph-mon[106470]: osdmap e1609: 8 total, 8 up, 7 in 2024-09-14T05:04:55.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:55 smithi039 ceph-mon[111235]: pgmap v3236: 119 pgs: 119 active+clean; 1.1 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 821 op/s 2024-09-14T05:04:55.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:55 smithi039 ceph-mon[111235]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T05:04:55.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:55 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T05:04:55.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:55 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2073526815' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T05:04:55.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:55 smithi039 ceph-mon[111235]: osdmap e1609: 8 total, 8 up, 7 in 2024-09-14T05:04:55.877 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:55.905 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:55.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:55.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:56.007 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:56.029 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:56.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:56.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:56.131 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:56.156 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:56.156 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:56.157 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:56.259 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:56.282 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:56.283 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:56.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:56.342 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:04:56.384 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:56.408 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:56.409 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:56.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:56.411 INFO:teuthology.orchestra.run.smithi039.stderr:no valid command found; 10 closest matches: 2024-09-14T05:04:56.416 INFO:teuthology.orchestra.run.smithi039.stderr:0 2024-09-14T05:04:56.416 INFO:teuthology.orchestra.run.smithi039.stderr:1 2024-09-14T05:04:56.416 INFO:teuthology.orchestra.run.smithi039.stderr:2 2024-09-14T05:04:56.416 INFO:teuthology.orchestra.run.smithi039.stderr:abort 2024-09-14T05:04:56.416 INFO:teuthology.orchestra.run.smithi039.stderr:assert 2024-09-14T05:04:56.416 INFO:teuthology.orchestra.run.smithi039.stderr:bench [] [] [] [] 2024-09-14T05:04:56.416 INFO:teuthology.orchestra.run.smithi039.stderr:bluefs debug_inject_read_zeros 2024-09-14T05:04:56.416 INFO:teuthology.orchestra.run.smithi039.stderr:bluefs files list 2024-09-14T05:04:56.416 INFO:teuthology.orchestra.run.smithi039.stderr:bluefs stats 2024-09-14T05:04:56.416 INFO:teuthology.orchestra.run.smithi039.stderr:bluestore allocator dump block 2024-09-14T05:04:56.416 INFO:teuthology.orchestra.run.smithi039.stderr:Error EINVAL: invalid command 2024-09-14T05:04:56.510 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:56.558 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:56.558 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:56.559 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:56.660 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:56.689 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:56.689 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:56.690 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:56.790 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:56.814 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:56.814 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:56.815 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:56.915 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:56.943 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:56.944 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:56.944 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:57.045 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:57.072 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:57.072 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:57.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:57.173 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:57.218 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:57.219 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:57.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:57.320 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:57.347 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:57.348 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:57.348 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:57.449 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:57.473 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:57.473 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:57.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:57.575 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:57.606 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:57 smithi039 ceph-mon[106470]: pgmap v3238: 119 pgs: 119 active+clean; 959 MiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 846 op/s 2024-09-14T05:04:57.607 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:57 smithi039 ceph-mon[111235]: pgmap v3238: 119 pgs: 119 active+clean; 959 MiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 846 op/s 2024-09-14T05:04:57.616 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:57.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:57.617 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:57.678 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T05:04:57.678 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:04:57.718 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:57.760 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:57.761 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:57.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:57.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:57 smithi129 ceph-mon[101316]: pgmap v3238: 119 pgs: 119 active+clean; 959 MiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 846 op/s 2024-09-14T05:04:57.862 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:57.885 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:57.886 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:57.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:57.987 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:58.009 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:58.009 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:58.010 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:58.111 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:58.113 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set noscrub 2024-09-14T05:04:58.135 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:58.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:58.136 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:58.237 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:58.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:58.294 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:58.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:58.395 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:58.452 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:58.453 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:58.454 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:58.514 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-14T05:04:58.515 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-14T05:04:58.515 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.7 'trim stale osdmaps' 2024-09-14T05:04:58.555 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:04:58.578 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:58.579 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:58.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:04:58.680 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:58.733 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:58.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:58.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:58.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:04:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:04:58] "GET /metrics HTTP/1.1" 200 36752 "" "Prometheus/2.51.0" 2024-09-14T05:04:58.835 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:04:58.889 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:58.890 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:58.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:04:58.991 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:04:59.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:59.021 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:59.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:04:59.122 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:59.173 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:59.173 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:59.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:59.275 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:04:59.319 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:59.320 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:59.320 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:04:59.421 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:04:59.445 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:59.446 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:59.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:04:59.547 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:04:59.571 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:04:59.572 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:59.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:04:59.591 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_ops_in_flight 2024-09-14T05:04:59.673 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:04:59.723 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:59.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:59.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:04:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:04:59 smithi129 ceph-mon[101316]: pgmap v3239: 119 pgs: 119 active+clean; 835 MiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 860 op/s 2024-09-14T05:04:59.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:04:59 smithi039 ceph-mon[106470]: pgmap v3239: 119 pgs: 119 active+clean; 835 MiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 860 op/s 2024-09-14T05:04:59.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:04:59 smithi039 ceph-mon[111235]: pgmap v3239: 119 pgs: 119 active+clean; 835 MiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 860 op/s 2024-09-14T05:04:59.826 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:04:59.884 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:04:59.885 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:04:59.885 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:04:59.986 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:00.011 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:00.012 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:00.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:00.113 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:00.135 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:00.136 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:00.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:00.151 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:05:00.238 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:00.290 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:00.290 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:00.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:00.391 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:00.444 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:00.445 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:00.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:00.546 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:00.570 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:00.571 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:00.571 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:00.672 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:00.728 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:00.729 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:00.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:00.830 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:00.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:00.854 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:00.855 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:00.955 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:00.981 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:00.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:00.983 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:01.083 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:01.138 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:01.139 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:01.139 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:01.240 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:01.277 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:01.277 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:01.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:01.378 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:01.402 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:01.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:01.404 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:01.504 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:01.526 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:01.526 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:01.527 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:01.627 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:01.652 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:01.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:01.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:01.754 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:01.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:01 smithi039 ceph-mon[106470]: pgmap v3240: 119 pgs: 119 active+clean; 747 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 879 op/s 2024-09-14T05:05:01.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:01 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/546305401' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:05:01.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:01 smithi039 ceph-mon[111235]: pgmap v3240: 119 pgs: 119 active+clean; 747 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 879 op/s 2024-09-14T05:05:01.793 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:01 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/546305401' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:05:01.795 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:01.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:01.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:01.896 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:01.931 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:01.932 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:01.932 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:02.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:01 smithi129 ceph-mon[101316]: pgmap v3240: 119 pgs: 119 active+clean; 747 MiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 879 op/s 2024-09-14T05:05:02.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:01 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/546305401' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:05:02.033 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:02.074 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:02.075 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:02.075 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:02.176 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:02.199 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:02.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:02.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:02.301 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:02.349 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:02.349 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:02.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:02.451 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:02.492 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:02.493 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:02.493 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:02.594 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:02.615 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:02.616 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:02.616 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:02.717 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:02.739 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:02.739 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:02.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:02.841 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:02.864 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:02.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:02.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:02.966 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:02.991 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:02.991 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:02.992 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:03.093 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:03.117 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:03.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:03.118 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:03.219 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:03.240 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:03.240 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:03.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:03.342 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:03.391 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:03.392 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:03.392 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:03.493 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:03.532 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:03.532 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:03.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:03.634 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:03.655 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:03.656 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:03.657 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:03.757 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:03.801 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:03.802 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:03.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:03.808 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:03 smithi129 ceph-mon[101316]: pgmap v3241: 119 pgs: 119 active+clean; 610 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 890 op/s 2024-09-14T05:05:03.808 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:03 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T05:05:03.903 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:03.930 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:03.930 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:03.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:04.031 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:04.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:03 smithi039 ceph-mon[106470]: pgmap v3241: 119 pgs: 119 active+clean; 610 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 890 op/s 2024-09-14T05:05:04.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:03 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T05:05:04.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:03 smithi039 ceph-mon[111235]: pgmap v3241: 119 pgs: 119 active+clean; 610 MiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 890 op/s 2024-09-14T05:05:04.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:03 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T05:05:04.062 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T05:05:04.062 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T05:05:04.062 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T05:05:04.062 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T05:05:04.062 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T05:05:04.063 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T05:05:04.063 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T05:05:04.063 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T05:05:04.063 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T05:05:04.063 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T05:05:04.063 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T05:05:04.063 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T05:05:04.063 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T05:05:04.064 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T05:05:04.064 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T05:05:04.064 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T05:05:04.066 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:04.067 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:04.068 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:04.168 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:04.202 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:04.203 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:04.203 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:04.304 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:04.325 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:04.326 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:04.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:04.427 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:04.469 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:04.469 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:04.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:04.571 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:04.610 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:04.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:04.611 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:04.712 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:04.742 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:04.742 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:04.743 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:04.844 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:04.875 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:04.875 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:04.876 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:04.978 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:05.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:05.019 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:05.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:05.120 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:05.162 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:05.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:05.163 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:05.264 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:05.288 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:05.289 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:05.290 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:05.295 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:05:05.391 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:05.422 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:05.422 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:05.423 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:05.524 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:05.550 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:05.550 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:05.551 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:05.651 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:05.696 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:05.697 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:05.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:05.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:05 smithi039 ceph-mon[106470]: pgmap v3242: 119 pgs: 119 active+clean; 532 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 880 op/s 2024-09-14T05:05:05.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:05 smithi039 ceph-mon[111235]: pgmap v3242: 119 pgs: 119 active+clean; 532 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 880 op/s 2024-09-14T05:05:05.798 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:05.828 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:05:05.844 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:05.845 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:05.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:05.946 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:05.970 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:05.971 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:05.972 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:06.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:05 smithi129 ceph-mon[101316]: pgmap v3242: 119 pgs: 119 active+clean; 532 MiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 880 op/s 2024-09-14T05:05:06.072 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:06.095 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:06.096 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:06.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:06.197 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:06.223 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:06.224 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:06.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:06.325 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:06.366 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:06.367 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:06.368 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:06.469 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:06.513 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:06.514 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:06.515 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:06.615 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:06.639 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:06.640 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:06.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:06.742 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:06.768 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:06.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:06.769 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:06.870 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:06.894 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:06.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:06.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:06.996 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:07.020 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:07.020 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:07.021 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:07.121 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:07.150 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:07.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:07.151 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:07.251 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:07.300 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:07.300 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:07.301 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:07.401 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:07.423 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:07.423 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:07.424 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:07.525 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:07.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:07.567 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:07.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:07.668 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:07.716 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:07.717 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:07.717 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:07.818 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:07.839 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:07.840 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:07.840 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:07.877 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T05:05:07.911 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:07 smithi039 ceph-mon[106470]: pgmap v3243: 119 pgs: 119 active+clean; 424 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 907 op/s 2024-09-14T05:05:07.911 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:05:07.912 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:05:07.912 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:07 smithi039 ceph-mon[111235]: pgmap v3243: 119 pgs: 119 active+clean; 424 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 907 op/s 2024-09-14T05:05:07.912 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:05:07.912 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:05:07.921 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:07 smithi129 ceph-mon[101316]: pgmap v3243: 119 pgs: 119 active+clean; 424 MiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 907 op/s 2024-09-14T05:05:07.921 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:05:07.921 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:05:07.942 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:07.966 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:07.966 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:07.967 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:08.067 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:08.092 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:08.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:08.093 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:08.194 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:08.233 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:08.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:08.234 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:08.335 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:08.362 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:08.363 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:08.363 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:08.464 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:08.506 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:08.506 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:08.507 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:08.607 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:08.632 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:08.633 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:08.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:08.734 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:08.775 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:08.775 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:08.776 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:08.876 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:08.900 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:08.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:08.901 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:09.002 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:09.031 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:09.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:09.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:09.039 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:05:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:05:08] "GET /metrics HTTP/1.1" 200 36736 "" "Prometheus/2.51.0" 2024-09-14T05:05:09.133 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:09.157 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:09.158 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:09.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:09.259 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:09.281 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:09.282 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:09.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:09.383 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:09.420 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:09.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:09.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:09.522 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:09.577 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:09.577 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:09.577 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:09.678 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:09.703 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:09.704 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:09.704 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:09.743 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:09 smithi039 ceph-mon[106470]: pgmap v3244: 119 pgs: 119 active+clean; 310 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 865 op/s 2024-09-14T05:05:09.743 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:09 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:05:09.743 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:09 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:05:09.744 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:09 smithi039 ceph-mon[111235]: pgmap v3244: 119 pgs: 119 active+clean; 310 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 865 op/s 2024-09-14T05:05:09.744 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:09 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:05:09.744 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:09 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:05:09.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:09 smithi129 ceph-mon[101316]: pgmap v3244: 119 pgs: 119 active+clean; 310 MiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 865 op/s 2024-09-14T05:05:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:09 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:05:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:09 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:05:09.805 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:09.831 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:09.832 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:09.832 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:09.933 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:09.967 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:09.967 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:09.968 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:10.068 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:10.115 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:10.116 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:10.116 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:10.217 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:10.242 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:10.243 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:10.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:10.344 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:10.368 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:10.368 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:10.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:10.470 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:10.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:10.509 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:10.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:10.610 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:10.657 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:10.657 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:10.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:10.759 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:10.780 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:10.781 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:10.781 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:10.882 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:10.906 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:10.907 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:10.907 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:11.008 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:11.053 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:11.054 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:11.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:11.155 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:11.211 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:11.212 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:11.212 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:11.313 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:11.326 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Removed 115610 objects 2024-09-14T05:05:11.326 INFO:tasks.radosbench.radosbench.0.smithi039.stdout:Clean up completed and total clean up time :144.252 2024-09-14T05:05:11.339 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:11.340 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:11.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:11.441 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:11.463 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:11.464 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:11.464 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:11.565 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:11.586 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:11.586 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:11.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:11.687 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:11.705 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:11 smithi039 ceph-mon[106470]: pgmap v3245: 119 pgs: 119 active+clean; 225 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 812 op/s 2024-09-14T05:05:11.705 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:11 smithi039 ceph-mon[111235]: pgmap v3245: 119 pgs: 119 active+clean; 225 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 812 op/s 2024-09-14T05:05:11.709 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:11.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:11.709 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:11.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:11 smithi129 ceph-mon[101316]: pgmap v3245: 119 pgs: 119 active+clean; 225 MiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 812 op/s 2024-09-14T05:05:11.811 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:11.833 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:11.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:11.834 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:11.935 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:11.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:11.958 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:11.959 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:12.059 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:12.082 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:12.083 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:12.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:12.184 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:12.206 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:12.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:12.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:12.280 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T05:05:12.280 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [], 2024-09-14T05:05:12.281 INFO:teuthology.orchestra.run.smithi039.stdout: "num_ops": 0 2024-09-14T05:05:12.281 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T05:05:12.283 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:05:12.307 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:12.330 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:12.331 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:12.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:12.432 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:12.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:12.456 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:12.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:12.480 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is set 2024-09-14T05:05:12.558 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:12.579 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:12.580 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:12.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:12.681 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:12.702 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:12.703 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:12.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:12.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:12 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/768317925' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:05:12.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:12 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:05:12.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:12 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/768317925' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:05:12.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:12 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:05:12.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:12 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/768317925' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:05:12.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:12 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd set", "key": "noscrub"} : dispatch 2024-09-14T05:05:12.805 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:12.829 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:12.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:12.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:12.931 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:12.939 INFO:teuthology.orchestra.run.smithi039.stderr:no valid command found; 10 closest matches: 2024-09-14T05:05:12.939 INFO:teuthology.orchestra.run.smithi039.stderr:0 2024-09-14T05:05:12.939 INFO:teuthology.orchestra.run.smithi039.stderr:1 2024-09-14T05:05:12.939 INFO:teuthology.orchestra.run.smithi039.stderr:2 2024-09-14T05:05:12.939 INFO:teuthology.orchestra.run.smithi039.stderr:abort 2024-09-14T05:05:12.939 INFO:teuthology.orchestra.run.smithi039.stderr:assert 2024-09-14T05:05:12.939 INFO:teuthology.orchestra.run.smithi039.stderr:bench [] [] [] [] 2024-09-14T05:05:12.939 INFO:teuthology.orchestra.run.smithi039.stderr:bluefs debug_inject_read_zeros 2024-09-14T05:05:12.939 INFO:teuthology.orchestra.run.smithi039.stderr:bluefs files list 2024-09-14T05:05:12.939 INFO:teuthology.orchestra.run.smithi039.stderr:bluefs stats 2024-09-14T05:05:12.940 INFO:teuthology.orchestra.run.smithi039.stderr:bluestore allocator dump block 2024-09-14T05:05:12.940 INFO:teuthology.orchestra.run.smithi039.stderr:Error EINVAL: invalid command 2024-09-14T05:05:12.953 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:12.953 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:12.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:13.055 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:13.078 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:13.078 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:13.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:13.179 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:13.201 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:13.202 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:13.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:13.303 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:13.326 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:13.327 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:13.327 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:13.428 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:13.458 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:13.459 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:13.459 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:13.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:13 smithi039 ceph-mon[106470]: pgmap v3246: 119 pgs: 119 active+clean; 89 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 877 op/s 2024-09-14T05:05:13.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:13 smithi039 ceph-mon[106470]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:05:13.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:13 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T05:05:13.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:13 smithi039 ceph-mon[106470]: osdmap e1610: 8 total, 8 up, 7 in 2024-09-14T05:05:13.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:13 smithi039 ceph-mon[111235]: pgmap v3246: 119 pgs: 119 active+clean; 89 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 877 op/s 2024-09-14T05:05:13.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:13 smithi039 ceph-mon[111235]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:05:13.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:13 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T05:05:13.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:13 smithi039 ceph-mon[111235]: osdmap e1610: 8 total, 8 up, 7 in 2024-09-14T05:05:13.560 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:13.583 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:13.584 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:13.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:13.685 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:13.708 DEBUG:teuthology.orchestra.run:got remote process result: 22 2024-09-14T05:05:13.708 INFO:tasks.thrashosds.thrasher:Failed to trim stale osdmaps, ignoring 2024-09-14T05:05:13.709 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:13.709 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:13.710 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:13.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:13 smithi129 ceph-mon[101316]: pgmap v3246: 119 pgs: 119 active+clean; 89 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 877 op/s 2024-09-14T05:05:13.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:13 smithi129 ceph-mon[101316]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:05:13.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:13 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2024-09-14T05:05:13.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:13 smithi129 ceph-mon[101316]: osdmap e1610: 8 total, 8 up, 7 in 2024-09-14T05:05:13.811 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:13.833 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:13.834 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:13.834 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:13.935 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:13.958 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:13.958 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:13.959 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:14.059 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:14.082 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:14.082 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:14.083 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:14.183 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:14.205 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:14.206 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:14.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:14.307 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:14.330 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:14.330 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:14.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:14.335 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_blocked_ops 2024-09-14T05:05:14.432 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:14.455 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:14.455 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:14.456 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:14.556 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:14.580 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:14.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:14.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:14.682 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:14.705 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:14.706 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:14.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:14.807 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:14.830 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:14.831 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:14.831 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:14.893 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2024-09-14T05:05:14.894 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2024-09-14T05:05:14.932 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:14.959 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:14.960 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:14.960 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:15.061 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:15.087 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:15.087 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:15.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:15.189 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:15.215 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:15.215 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:15.216 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:15.316 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:15.342 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:15.342 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:15.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:15.444 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:15.467 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:15.467 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:15.468 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:15.568 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:15.595 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:15.596 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:15.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:15.666 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:15 smithi039 ceph-mon[106470]: pgmap v3248: 119 pgs: 119 active+clean; 32 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 799 op/s 2024-09-14T05:05:15.666 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:15 smithi039 ceph-mon[111235]: pgmap v3248: 119 pgs: 119 active+clean; 32 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 799 op/s 2024-09-14T05:05:15.697 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:15.723 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:15.724 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:15.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:15.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:15 smithi129 ceph-mon[101316]: pgmap v3248: 119 pgs: 119 active+clean; 32 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 799 op/s 2024-09-14T05:05:15.825 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:15.849 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:15.850 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:15.850 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:15.951 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:15.975 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:15.976 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:15.976 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:16.077 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:16.103 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:16.103 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:16.104 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:16.204 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:16.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:16.232 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:16.232 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:16.333 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:16.356 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:16.357 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:16.357 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:16.458 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:16.481 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:16.482 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:16.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:16.583 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:16.606 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:16.606 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:16.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:16.708 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:16.732 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:16.733 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:16.733 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:16.834 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:16.858 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:16.858 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:16.859 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:16.959 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:16.982 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:16.983 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:16.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:17.084 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:17.108 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:17.109 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:17.109 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:17.210 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:17.232 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:17.233 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:17.233 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:17.334 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:17.358 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:17.359 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:17.359 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:17.451 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T05:05:17.451 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T05:05:17.451 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T05:05:17.451 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T05:05:17.451 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T05:05:17.451 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T05:05:17.451 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T05:05:17.451 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T05:05:17.451 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T05:05:17.452 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T05:05:17.452 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T05:05:17.452 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T05:05:17.452 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T05:05:17.452 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T05:05:17.452 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T05:05:17.452 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T05:05:17.461 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:17.484 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:17.485 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:17.485 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:17.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:17 smithi039 ceph-mon[106470]: pgmap v3249: 119 pgs: 119 active+clean; 1.6 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 676 op/s 2024-09-14T05:05:17.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:17 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/415040720' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:05:17.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:17 smithi039 ceph-mon[111235]: pgmap v3249: 119 pgs: 119 active+clean; 1.6 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 676 op/s 2024-09-14T05:05:17.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:17 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/415040720' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:05:17.585 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:17.609 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:17.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:17.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:17.711 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:17.734 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:17.735 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:17.735 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:17.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:17 smithi129 ceph-mon[101316]: pgmap v3249: 119 pgs: 119 active+clean; 1.6 MiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 676 op/s 2024-09-14T05:05:17.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:17 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/415040720' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:05:17.836 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:17.859 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:17.860 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:17.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:17.961 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:17.981 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd set nodeep-scrub 2024-09-14T05:05:17.983 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:17.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:17.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:18.085 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:18.110 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:18.111 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:18.111 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:18.212 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:18.237 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:18.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:18.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:18.338 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:18.364 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:18.364 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:18.365 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:18.465 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:18.489 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:18.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:18.491 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:18.591 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:18.614 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:18.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:18.615 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:18.709 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-14T05:05:18.709 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T05:05:18.710 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2024-09-14T05:05:18.710 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2024-09-14T05:05:18.710 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2024-09-14T05:05:18.710 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 config set filestore_inject_stall 3 2024-09-14T05:05:18.715 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:18.777 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:18.778 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:18.778 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:18.879 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:18.881 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:05:18.886 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:05:18.901 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:18.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:18.902 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:18.921 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:05:19.002 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:19.011 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:05:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:05:18] "GET /metrics HTTP/1.1" 200 36736 "" "Prometheus/2.51.0" 2024-09-14T05:05:19.023 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:19.024 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:19.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:19.125 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:19.149 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:19.150 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:19.150 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:19.251 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:19.273 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:19.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:19.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:19.375 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:19.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:19.397 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:19.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:19.502 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:19.526 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:19.527 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:19.527 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:19.628 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:19.650 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:19.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:19.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:19.739 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:19 smithi039 ceph-mon[106470]: pgmap v3250: 119 pgs: 119 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 494 op/s 2024-09-14T05:05:19.740 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:19 smithi039 ceph-mon[111235]: pgmap v3250: 119 pgs: 119 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 494 op/s 2024-09-14T05:05:19.752 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:19.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:19 smithi129 ceph-mon[101316]: pgmap v3250: 119 pgs: 119 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 494 op/s 2024-09-14T05:05:19.776 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:19.776 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:19.777 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:19.878 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:19.901 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:19.901 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:19.901 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:20.002 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:20.024 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:20.025 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:20.025 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:20.126 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:20.150 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:20.151 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:20.151 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:20.252 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:20.273 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:20.274 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:20.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:20.294 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2024-09-14T05:05:20.375 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:20.398 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:20.398 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:20.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:20.500 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:20.522 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:20.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:20.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:20.624 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:20.626 INFO:teuthology.orchestra.run.smithi129.stdout:{ 2024-09-14T05:05:20.627 INFO:teuthology.orchestra.run.smithi129.stdout: "success": "" 2024-09-14T05:05:20.627 INFO:teuthology.orchestra.run.smithi129.stdout:} 2024-09-14T05:05:20.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:20.647 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:20.647 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:20.748 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:20.769 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:20.770 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:20.770 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:20.871 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:20.895 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:20.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:20.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:20.997 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:21.018 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:21.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:21.019 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:21.120 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:21.143 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:21.144 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:21.144 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:21.249 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:21.270 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:21.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:21.271 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:21.372 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:21.394 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:21.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:21.396 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:21.406 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:05:21.441 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:05:21.499 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:21.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:21.522 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:21.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:21.623 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:21.645 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:21.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:21.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:21.687 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:21 smithi039 ceph-mon[106470]: pgmap v3251: 119 pgs: 119 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 359 op/s 2024-09-14T05:05:21.688 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:21 smithi039 ceph-mon[111235]: pgmap v3251: 119 pgs: 119 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 359 op/s 2024-09-14T05:05:21.747 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:21.770 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:21.772 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:21 smithi129 ceph-mon[101316]: pgmap v3251: 119 pgs: 119 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 359 op/s 2024-09-14T05:05:21.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:21.774 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:21.874 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:21.896 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:21.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:21.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:21.999 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:22.022 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:22.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:22.023 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:22.124 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:22.146 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:22.147 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:22.147 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:22.248 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:22.271 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:22.272 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:22.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:22.373 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:22.396 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:22.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:22.398 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:22.499 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:22.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:22.523 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:22.524 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:22.625 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:22.647 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:22.648 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:22.649 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:22.750 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:22.772 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:22.773 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:22.773 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:22.874 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:22.896 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:22.897 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:22.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:22.998 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:23.023 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:23.023 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:23.024 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:23.125 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:23.149 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:23.160 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:23.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:23.261 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:23.285 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:23.286 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:23.286 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:23.311 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T05:05:23.311 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:05:23.387 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:23.410 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:23.411 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:23.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:23.504 INFO:teuthology.orchestra.run.smithi039.stderr:pool 'unique_pool_5' removed 2024-09-14T05:05:23.505 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:23 smithi039 ceph-mon[106470]: pgmap v3252: 119 pgs: 119 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 140 op/s 2024-09-14T05:05:23.505 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:23 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:05:23.505 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:23 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:05:23.505 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:23 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1159270472' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T05:05:23.505 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:23 smithi039 ceph-mon[111235]: pgmap v3252: 119 pgs: 119 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 140 op/s 2024-09-14T05:05:23.505 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:23 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:05:23.505 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:23 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:05:23.505 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:23 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1159270472' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T05:05:23.512 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:23.541 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:23.543 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:23.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:23.644 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:23.665 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:23.666 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:23.666 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:23.767 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:23.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:23 smithi129 ceph-mon[101316]: pgmap v3252: 119 pgs: 119 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 140 op/s 2024-09-14T05:05:23.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:23 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:05:23.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:23 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:05:23.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:23 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1159270472' entity='client.admin' cmd={"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true} : dispatch 2024-09-14T05:05:23.789 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:23.789 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:23.789 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:23.890 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:23.913 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:23.913 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:23.914 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:24.014 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:24.037 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:24.037 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:24.038 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:24.139 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:24.161 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:24.162 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:24.162 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:24.263 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:24.284 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:24.285 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:24.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:24.386 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:24.409 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:24.410 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:24.411 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:24.511 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:24.535 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:24.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:24.536 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:24.637 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:24.659 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:24.660 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:24.660 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:24.761 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:24.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:24 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1159270472' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T05:05:24.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:24 smithi129 ceph-mon[101316]: osdmap e1611: 8 total, 8 up, 7 in 2024-09-14T05:05:24.784 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:24.784 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:24.785 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:24.789 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:24 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1159270472' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T05:05:24.789 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:24 smithi039 ceph-mon[111235]: osdmap e1611: 8 total, 8 up, 7 in 2024-09-14T05:05:24.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:24 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1159270472' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2024-09-14T05:05:24.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:24 smithi039 ceph-mon[106470]: osdmap e1611: 8 total, 8 up, 7 in 2024-09-14T05:05:24.885 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:24.911 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:24.911 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:24.912 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:25.013 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:25.039 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:25.040 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:25.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:25.141 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:25.168 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:25.168 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:25.168 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:25.269 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:25.292 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:25.293 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:25.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:25.395 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:25.419 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:25.420 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:25.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:25.521 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:25.545 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:25.546 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:25.547 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:25.648 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:25.671 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:25.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:25.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:25 smithi129 ceph-mon[101316]: pgmap v3254: 103 pgs: 103 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2024-09-14T05:05:25.772 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:25.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:25 smithi039 ceph-mon[106470]: pgmap v3254: 103 pgs: 103 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2024-09-14T05:05:25.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:25 smithi039 ceph-mon[111235]: pgmap v3254: 103 pgs: 103 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2024-09-14T05:05:25.794 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:25.795 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:25.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:25.896 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:25.920 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:25.921 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:25.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:26.022 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:26.045 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:26.046 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:26.046 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:26.147 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:26.169 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:26.170 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:26.170 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:26.252 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-14T05:05:26.252 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T05:05:26.252 INFO:tasks.thrashosds.thrasher:Reweighting osd 6 to 0.43738039969388554 2024-09-14T05:05:26.253 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 6 0.43738039969388554 2024-09-14T05:05:26.272 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:26.298 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:26.299 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:26.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:26.400 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:26.425 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:26.426 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:26.426 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:26.435 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.1 dump_historic_ops 2024-09-14T05:05:26.527 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:26.553 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:26.554 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:26.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:26.655 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:26.679 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:26.680 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:26.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:26.781 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:26.804 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:26.804 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:26.805 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:26.838 INFO:teuthology.task.sequential:In sequential, running task print... 2024-09-14T05:05:26.838 INFO:teuthology.task.print:**** done end radosbench.yaml 2024-09-14T05:05:26.839 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-09-14T05:05:26.839 INFO:tasks.workunit:Pulling workunits from ref reef 2024-09-14T05:05:26.839 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-09-14T05:05:26.840 DEBUG:teuthology.orchestra.run.smithi039:> stat -- /home/ubuntu/cephtest/mnt.0 2024-09-14T05:05:26.906 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:26.908 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:26.908 INFO:teuthology.orchestra.run.smithi039.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-09-14T05:05:26.909 DEBUG:teuthology.orchestra.run.smithi039:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-09-14T05:05:26.933 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:26.933 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:26.934 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:26.977 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-09-14T05:05:26.978 DEBUG:teuthology.orchestra.run.smithi039:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-09-14T05:05:27.034 INFO:tasks.workunit:timeout=3h 2024-09-14T05:05:27.034 INFO:tasks.workunit:cleanup=True 2024-09-14T05:05:27.035 DEBUG:teuthology.orchestra.run.smithi039:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch reef https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-09-14T05:05:27.036 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:27.063 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:27.064 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:27.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:27.114 INFO:tasks.workunit.client.0.smithi039.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-09-14T05:05:27.164 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:27.189 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:27.190 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:27.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:27.229 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T05:05:27.229 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T05:05:27.229 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T05:05:27.229 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T05:05:27.229 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T05:05:27.229 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T05:05:27.230 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T05:05:27.230 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T05:05:27.230 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T05:05:27.230 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T05:05:27.230 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T05:05:27.230 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T05:05:27.230 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T05:05:27.230 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T05:05:27.230 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T05:05:27.231 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T05:05:27.291 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:27.314 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:27.314 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:27.315 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:27.415 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:27.438 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:27.439 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:27.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:27.540 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:27.565 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:27.566 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:27.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:27.667 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:27.694 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:27.695 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:27.696 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:27.781 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:27 smithi039 ceph-mon[106470]: pgmap v3255: 103 pgs: 103 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2024-09-14T05:05:27.782 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:27 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3102483517' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:05:27.782 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:27 smithi039 ceph-mon[111235]: pgmap v3255: 103 pgs: 103 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2024-09-14T05:05:27.782 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:27 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3102483517' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:05:27.796 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:27.822 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:27.823 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:27.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:27.924 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:27.951 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:27.952 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:27.953 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:28.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:27 smithi129 ceph-mon[101316]: pgmap v3255: 103 pgs: 103 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2024-09-14T05:05:28.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:27 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3102483517' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:05:28.053 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:28.076 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:28.077 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:28.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:28.178 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:28.205 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:28.205 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:28.206 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:28.307 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:28.332 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:28.332 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:28.332 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:28.433 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:28.458 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:28.459 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:28.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:28.560 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:28.586 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:28.587 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:28.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:28.689 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:28.713 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:28.713 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:28.714 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:28.815 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:28.838 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:28.839 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:28.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:28.940 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:28.967 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:28.968 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:28.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:29.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:05:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:05:28] "GET /metrics HTTP/1.1" 200 36729 "" "Prometheus/2.51.0" 2024-09-14T05:05:29.069 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:29.093 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:29.093 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:29.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:29.195 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:29.225 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:29.226 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:29.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:29.327 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:29.359 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:29.359 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:29.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:29.461 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:29.485 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:29.486 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:29.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:29.544 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is set 2024-09-14T05:05:29.586 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:29.609 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:29.610 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:29.610 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:29.711 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:29.734 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:29.735 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:29.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:29.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:29 smithi039 ceph-mon[106470]: pgmap v3256: 103 pgs: 103 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2024-09-14T05:05:29.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:29 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2041741883' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:05:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:29 smithi039 ceph-mon[111235]: pgmap v3256: 103 pgs: 103 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2024-09-14T05:05:29.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:29 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2041741883' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:05:29.823 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:29 smithi129 ceph-mon[101316]: pgmap v3256: 103 pgs: 103 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2024-09-14T05:05:29.823 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:29 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2041741883' entity='client.admin' cmd={"prefix": "osd set", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:05:29.835 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:29.861 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:29.861 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:29.862 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:29.963 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:29.983 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:29.984 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:29.984 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:30.085 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:30.107 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:30.107 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:30.108 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:30.208 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:30.231 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:30.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:30.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:30.332 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:30.354 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:30.355 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:30.355 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:30.456 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:30.478 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:30.478 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:30.479 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:30.528 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:05:30.528 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:05:30.580 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:30.601 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:30.601 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:30.602 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:30.702 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:30.725 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:30.725 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:30.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:30.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:30 smithi129 ceph-mon[101316]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:05:30.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:30 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2041741883' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T05:05:30.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:30 smithi129 ceph-mon[101316]: osdmap e1612: 8 total, 8 up, 7 in 2024-09-14T05:05:30.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:30 smithi039 ceph-mon[106470]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:05:30.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:30 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2041741883' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T05:05:30.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:30 smithi039 ceph-mon[106470]: osdmap e1612: 8 total, 8 up, 7 in 2024-09-14T05:05:30.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:30 smithi039 ceph-mon[111235]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:05:30.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:30 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2041741883' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2024-09-14T05:05:30.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:30 smithi039 ceph-mon[111235]: osdmap e1612: 8 total, 8 up, 7 in 2024-09-14T05:05:30.827 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:30.848 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:30.849 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:30.849 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:30.950 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:30.972 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:30.973 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:30.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:31.074 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:31.098 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:31.098 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:31.098 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:31.175 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2024-09-14T05:05:31.200 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:31.241 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:31.241 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:31.242 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:31.342 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:31.365 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:31.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:31.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:31.466 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:31.489 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:31.490 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:31.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:31.591 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:31.614 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:31.614 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:31.615 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:31.715 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:31.720 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:31 smithi039 ceph-mon[106470]: pgmap v3258: 103 pgs: 103 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2024-09-14T05:05:31.721 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:31 smithi039 ceph-mon[111235]: pgmap v3258: 103 pgs: 103 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2024-09-14T05:05:31.739 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:31.740 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:31.740 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:31.841 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:31.864 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:31.865 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:31.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:31.966 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:31.988 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:31.989 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:31.989 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:32.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:31 smithi129 ceph-mon[101316]: pgmap v3258: 103 pgs: 103 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2024-09-14T05:05:32.090 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:32.113 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:32.114 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:32.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:32.215 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:32.238 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:32.238 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:32.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:32.339 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:32.364 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:32.365 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:32.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:32.466 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:32.490 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:32.490 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:32.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:32.492 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T05:05:32.592 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:32.617 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:32.617 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:32.618 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:32.719 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:32.742 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:32.743 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:32.743 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:32.844 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:32.866 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:32.867 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:32.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:32.968 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:32.993 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:32.993 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:32.994 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:33.095 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:33.118 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:33.118 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:33.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:33.220 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:33.242 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:33.242 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:33.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:33.344 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:33.369 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:33.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:33.370 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:33.422 DEBUG:teuthology.orchestra.run.smithi039:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-09-14T05:05:33.465 INFO:tasks.workunit.client.0.smithi039.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-09-14T05:05:33.468 INFO:tasks.workunit.client.0.smithi039.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-14T05:05:33.468 INFO:tasks.workunit.client.0.smithi039.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-09-14T05:05:33.471 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:33.494 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:33.494 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:33.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:33.595 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:33.621 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:33.621 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:33.622 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:33.723 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:33.745 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:33.746 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:33.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:33.777 INFO:tasks.workunit.client.0.smithi039.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-09-14T05:05:33.833 INFO:tasks.workunit.client.0.smithi039.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-09-14T05:05:33.846 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:33.872 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:33.873 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:33.873 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:33.894 INFO:tasks.workunit.client.0.smithi039.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-09-14T05:05:33.896 INFO:tasks.workunit.client.0.smithi039.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-14T05:05:33.896 INFO:tasks.workunit.client.0.smithi039.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-09-14T05:05:33.935 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:33 smithi039 ceph-mon[106470]: pgmap v3259: 103 pgs: 103 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2024-09-14T05:05:33.936 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:33 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:05:33.936 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:33 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:05:33.936 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:33 smithi039 ceph-mon[111235]: pgmap v3259: 103 pgs: 103 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2024-09-14T05:05:33.936 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:33 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:05:33.936 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:33 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:05:33.939 INFO:tasks.workunit.client.0.smithi039.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-09-14T05:05:33.974 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:33.988 DEBUG:teuthology.orchestra.run.smithi039:> set -ex 2024-09-14T05:05:33.988 DEBUG:teuthology.orchestra.run.smithi039:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-09-14T05:05:33.996 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:33.997 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:33.997 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:34.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:33 smithi129 ceph-mon[101316]: pgmap v3259: 103 pgs: 103 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2024-09-14T05:05:34.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:33 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:05:34.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:33 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:05:34.031 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2024-09-14T05:05:34.033 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2024-09-14T05:05:34.033 DEBUG:teuthology.orchestra.run.smithi039:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2024-09-14T05:05:34.098 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:34.124 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:34.124 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:34.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:34.225 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:34.251 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:34.252 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:34.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:34.342 INFO:tasks.workunit.client.0.smithi039.stdout:Running main() from gmock_main.cc 2024-09-14T05:05:34.342 INFO:tasks.workunit.client.0.smithi039.stdout:[==========] Running 70 tests from 1 test suite. 2024-09-14T05:05:34.342 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] Global test environment set-up. 2024-09-14T05:05:34.342 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 70 tests from TestClsRbd 2024-09-14T05:05:34.353 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:34.377 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:34.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:34.378 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:34.479 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:34.502 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:34.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:34.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:34.604 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:34.630 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:34.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:34.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:34.639 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:34 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T05:05:34.639 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:34 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T05:05:34.640 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:34 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:05:34.640 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:34 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T05:05:34.640 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:34 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T05:05:34.640 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:34 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:05:34.731 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:34.757 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:34.759 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:34.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:34.860 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:34.883 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:34.884 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:34.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:34.985 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:35.010 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:35.011 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:35.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:35.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:34 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T05:05:35.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:34 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T05:05:35.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:34 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:05:35.086 INFO:teuthology.orchestra.run.smithi039.stdout:{ 2024-09-14T05:05:35.086 INFO:teuthology.orchestra.run.smithi039.stdout: "size": 20, 2024-09-14T05:05:35.086 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 600, 2024-09-14T05:05:35.087 INFO:teuthology.orchestra.run.smithi039.stdout: "ops": [ 2024-09-14T05:05:35.087 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.087 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:151780 58.8 58:1d6d5092:::benchmark_data_smithi039_396950_object36167:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1590)", 2024-09-14T05:05:35.087 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:03:29.556548+0000", 2024-09-14T05:05:35.087 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 125.527521792, 2024-09-14T05:05:35.087 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1583662119999998, 2024-09-14T05:05:35.087 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:05:35.087 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:05:35.087 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:05:35.087 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:05:35.087 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:05:35.087 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:05:35.088 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 151780 2024-09-14T05:05:35.088 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.088 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:05:35.088 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.088 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:05:35.088 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:29.556548+0000", 2024-09-14T05:05:35.088 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.088 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.088 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.088 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:05:35.088 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:29.556548+0000", 2024-09-14T05:05:35.088 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.089 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.089 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.089 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:05:35.089 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:29.556550+0000", 2024-09-14T05:05:35.089 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.065e-06 2024-09-14T05:05:35.089 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.089 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.089 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:05:35.089 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:29.556552+0000", 2024-09-14T05:05:35.089 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.7350000000000001e-06 2024-09-14T05:05:35.090 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.090 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.090 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:05:35.090 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:29.556560+0000", 2024-09-14T05:05:35.090 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.1540000000000002e-06 2024-09-14T05:05:35.090 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.090 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.090 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:05:35.090 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:29.556566+0000", 2024-09-14T05:05:35.090 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5890000000000002e-06 2024-09-14T05:05:35.090 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.090 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.090 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:05:35.091 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:29.556641+0000", 2024-09-14T05:05:35.091 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.5030000000000005e-05 2024-09-14T05:05:35.091 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.091 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.091 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T05:05:35.091 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:29.556650+0000", 2024-09-14T05:05:35.091 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.0270000000000001e-06 2024-09-14T05:05:35.091 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.091 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.091 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:05:35.091 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:30.539519+0000", 2024-09-14T05:05:35.091 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.98286860399999998 2024-09-14T05:05:35.091 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.092 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.092 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T05:05:35.092 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:30.539532+0000", 2024-09-14T05:05:35.092 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3009e-05 2024-09-14T05:05:35.092 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.092 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.092 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:05:35.092 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.695758+0000", 2024-09-14T05:05:35.092 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1562265270000001 2024-09-14T05:05:35.092 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.092 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.092 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:05:35.092 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.695966+0000", 2024-09-14T05:05:35.093 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00020771099999999999 2024-09-14T05:05:35.093 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.093 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.093 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,4,6", 2024-09-14T05:05:35.093 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.696488+0000", 2024-09-14T05:05:35.093 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00052218900000000005 2024-09-14T05:05:35.093 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.093 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.093 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.093 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.714521+0000", 2024-09-14T05:05:35.093 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.018032376999999999 2024-09-14T05:05:35.093 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.093 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.094 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:05:35.094 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.714536+0000", 2024-09-14T05:05:35.094 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5662999999999999e-05 2024-09-14T05:05:35.094 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.094 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.094 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.094 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.714631+0000", 2024-09-14T05:05:35.094 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.4511000000000004e-05 2024-09-14T05:05:35.094 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.094 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.094 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.094 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.714731+0000", 2024-09-14T05:05:35.094 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000100589 2024-09-14T05:05:35.094 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.095 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.095 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:05:35.095 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.714799+0000", 2024-09-14T05:05:35.095 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.8143000000000001e-05 2024-09-14T05:05:35.095 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.095 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.095 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:05:35.095 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.714915+0000", 2024-09-14T05:05:35.095 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000115289 2024-09-14T05:05:35.095 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.095 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:05:35.095 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.095 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.095 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.096 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:151783 58.8 58:19a05780:::benchmark_data_smithi039_396950_object36170:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1590)", 2024-09-14T05:05:35.096 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:03:29.558272+0000", 2024-09-14T05:05:35.096 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 125.52579778400001, 2024-09-14T05:05:35.096 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1754275060000001, 2024-09-14T05:05:35.096 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:05:35.096 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:05:35.096 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:05:35.096 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:05:35.096 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:05:35.096 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:05:35.096 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 151783 2024-09-14T05:05:35.096 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.096 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:05:35.097 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.097 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:05:35.097 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:29.558272+0000", 2024-09-14T05:05:35.097 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.097 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.097 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.097 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:05:35.097 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:29.558272+0000", 2024-09-14T05:05:35.097 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.097 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.097 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.097 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:05:35.097 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:29.558274+0000", 2024-09-14T05:05:35.097 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.671e-06 2024-09-14T05:05:35.098 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.098 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.098 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:05:35.098 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:29.558279+0000", 2024-09-14T05:05:35.098 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6739999999999996e-06 2024-09-14T05:05:35.098 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.098 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.098 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:05:35.098 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:29.558290+0000", 2024-09-14T05:05:35.098 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1090000000000001e-05 2024-09-14T05:05:35.098 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.098 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.098 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:05:35.098 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:29.558297+0000", 2024-09-14T05:05:35.099 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.126e-06 2024-09-14T05:05:35.099 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.099 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.099 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:05:35.099 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:29.558364+0000", 2024-09-14T05:05:35.099 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.7249999999999995e-05 2024-09-14T05:05:35.099 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.099 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.099 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T05:05:35.099 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:29.558373+0000", 2024-09-14T05:05:35.099 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.5569999999999996e-06 2024-09-14T05:05:35.099 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.099 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.099 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:05:35.100 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:30.539559+0000", 2024-09-14T05:05:35.100 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.98118654000000005 2024-09-14T05:05:35.100 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.100 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.100 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T05:05:35.100 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:30.539568+0000", 2024-09-14T05:05:35.100 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.9360000000000005e-06 2024-09-14T05:05:35.100 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.100 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.100 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:05:35.100 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.714951+0000", 2024-09-14T05:05:35.100 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1753824580000001 2024-09-14T05:05:35.100 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.100 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.101 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:05:35.101 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.715114+0000", 2024-09-14T05:05:35.101 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00016317600000000001 2024-09-14T05:05:35.101 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.101 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.101 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,4,6", 2024-09-14T05:05:35.101 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.715265+0000", 2024-09-14T05:05:35.101 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015061799999999999 2024-09-14T05:05:35.101 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.101 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.101 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:05:35.101 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.725594+0000", 2024-09-14T05:05:35.101 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.010329071 2024-09-14T05:05:35.101 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.102 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.102 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.102 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.730416+0000", 2024-09-14T05:05:35.102 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0048227130000000002 2024-09-14T05:05:35.102 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.102 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.102 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.102 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.730649+0000", 2024-09-14T05:05:35.102 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000232888 2024-09-14T05:05:35.102 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.102 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.102 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.102 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.733516+0000", 2024-09-14T05:05:35.102 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0028663970000000001 2024-09-14T05:05:35.103 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.103 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.103 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:05:35.103 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.733578+0000", 2024-09-14T05:05:35.103 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.2133999999999998e-05 2024-09-14T05:05:35.103 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.103 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.103 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:05:35.103 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.733700+0000", 2024-09-14T05:05:35.103 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012220699999999999 2024-09-14T05:05:35.103 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.103 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:05:35.103 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.104 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.104 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.104 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:151789 58.8 58:18e3fbdc:::benchmark_data_smithi039_396950_object36176:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1590)", 2024-09-14T05:05:35.104 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:03:29.568451+0000", 2024-09-14T05:05:35.104 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 125.515619665, 2024-09-14T05:05:35.104 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1654488829999998, 2024-09-14T05:05:35.104 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:05:35.104 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:05:35.104 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:05:35.104 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:05:35.104 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:05:35.105 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:05:35.105 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 151789 2024-09-14T05:05:35.105 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.105 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:05:35.105 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.105 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:05:35.105 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:29.568451+0000", 2024-09-14T05:05:35.105 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.105 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.105 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.105 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:05:35.106 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:29.568451+0000", 2024-09-14T05:05:35.106 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.106 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.106 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.106 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:05:35.106 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:29.568452+0000", 2024-09-14T05:05:35.106 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5230000000000001e-06 2024-09-14T05:05:35.106 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.106 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.106 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:05:35.106 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:29.568465+0000", 2024-09-14T05:05:35.106 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.292e-05 2024-09-14T05:05:35.106 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.107 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.107 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:05:35.107 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:29.568481+0000", 2024-09-14T05:05:35.107 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5849999999999999e-05 2024-09-14T05:05:35.107 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.107 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.107 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:05:35.107 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:29.568489+0000", 2024-09-14T05:05:35.107 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0579999999999998e-06 2024-09-14T05:05:35.107 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.107 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.107 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:05:35.107 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:29.568561+0000", 2024-09-14T05:05:35.108 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.2018000000000001e-05 2024-09-14T05:05:35.108 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.108 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.108 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T05:05:35.108 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:29.568571+0000", 2024-09-14T05:05:35.108 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.006e-05 2024-09-14T05:05:35.108 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.108 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.108 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:05:35.108 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:30.539594+0000", 2024-09-14T05:05:35.108 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.97102245700000001 2024-09-14T05:05:35.108 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.109 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.109 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T05:05:35.109 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:30.539602+0000", 2024-09-14T05:05:35.109 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.0979999999999994e-06 2024-09-14T05:05:35.109 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.109 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.109 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:05:35.109 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.718559+0000", 2024-09-14T05:05:35.109 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1789578569999999 2024-09-14T05:05:35.109 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.109 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.109 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:05:35.110 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.718716+0000", 2024-09-14T05:05:35.110 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000156076 2024-09-14T05:05:35.110 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.110 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.110 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,4,6", 2024-09-14T05:05:35.110 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.718856+0000", 2024-09-14T05:05:35.110 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014045299999999999 2024-09-14T05:05:35.110 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.110 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.110 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:05:35.110 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.725711+0000", 2024-09-14T05:05:35.110 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0068547479999999999 2024-09-14T05:05:35.110 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.111 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.111 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.111 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.730468+0000", 2024-09-14T05:05:35.111 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0047570490000000002 2024-09-14T05:05:35.111 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.111 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.111 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.111 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.730786+0000", 2024-09-14T05:05:35.111 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00031791799999999999 2024-09-14T05:05:35.111 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.111 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.111 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.111 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.733740+0000", 2024-09-14T05:05:35.111 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0029543220000000001 2024-09-14T05:05:35.112 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.112 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.112 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:05:35.112 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.733792+0000", 2024-09-14T05:05:35.112 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.1498000000000001e-05 2024-09-14T05:05:35.112 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.112 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.112 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:05:35.113 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.733899+0000", 2024-09-14T05:05:35.113 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000107978 2024-09-14T05:05:35.113 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.113 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:05:35.114 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.114 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.114 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.114 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:151772 58.6 58:6584bd70:::benchmark_data_smithi039_396950_object36159:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1591)", 2024-09-14T05:05:35.114 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:03:30.540811+0000", 2024-09-14T05:05:35.114 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 124.543259541, 2024-09-14T05:05:35.114 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.160478141, 2024-09-14T05:05:35.114 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:05:35.114 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:05:35.114 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:05:35.114 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:05:35.114 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:05:35.114 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:05:35.115 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 151772 2024-09-14T05:05:35.115 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.115 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:05:35.115 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.115 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:05:35.115 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:30.540811+0000", 2024-09-14T05:05:35.115 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.115 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.115 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.115 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:05:35.115 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:30.540811+0000", 2024-09-14T05:05:35.115 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.115 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.116 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.116 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:05:35.116 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:30.540812+0000", 2024-09-14T05:05:35.116 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.674e-06 2024-09-14T05:05:35.116 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.116 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.116 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:05:35.116 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:30.540818+0000", 2024-09-14T05:05:35.116 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.276e-06 2024-09-14T05:05:35.116 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.116 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.116 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:05:35.116 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:30.541032+0000", 2024-09-14T05:05:35.117 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00021467099999999999 2024-09-14T05:05:35.117 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.117 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.117 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:05:35.117 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:35.119 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:30.541045+0000", 2024-09-14T05:05:35.119 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2506e-05 2024-09-14T05:05:35.119 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.119 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.119 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:05:35.119 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:30.551808+0000", 2024-09-14T05:05:35.119 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.01076329 2024-09-14T05:05:35.119 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.119 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.119 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T05:05:35.120 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:30.551820+0000", 2024-09-14T05:05:35.120 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1977000000000001e-05 2024-09-14T05:05:35.120 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.120 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.120 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:05:35.120 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.678766+0000", 2024-09-14T05:05:35.120 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.126945444 2024-09-14T05:05:35.120 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.120 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.120 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:05:35.120 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.681125+0000", 2024-09-14T05:05:35.120 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0023596480000000002 2024-09-14T05:05:35.120 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.121 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.121 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4,5", 2024-09-14T05:05:35.121 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.681491+0000", 2024-09-14T05:05:35.121 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00036575099999999999 2024-09-14T05:05:35.121 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.121 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.121 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.121 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.697192+0000", 2024-09-14T05:05:35.121 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.015700551 2024-09-14T05:05:35.121 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.121 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.121 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.121 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.697275+0000", 2024-09-14T05:05:35.122 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.3164000000000005e-05 2024-09-14T05:05:35.122 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.122 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.122 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.122 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.697332+0000", 2024-09-14T05:05:35.122 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.7429000000000002e-05 2024-09-14T05:05:35.122 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.122 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.122 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:05:35.122 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.700934+0000", 2024-09-14T05:05:35.122 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.003602371 2024-09-14T05:05:35.122 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.123 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.123 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:05:35.123 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.701144+0000", 2024-09-14T05:05:35.123 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000209528 2024-09-14T05:05:35.123 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.123 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.123 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:05:35.123 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.701289+0000", 2024-09-14T05:05:35.123 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014486100000000001 2024-09-14T05:05:35.123 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.123 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:05:35.123 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.123 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.124 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.124 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:151792 58.6 58:6dd252d7:::benchmark_data_smithi039_396950_object36179:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e1591)", 2024-09-14T05:05:35.124 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:03:30.551057+0000", 2024-09-14T05:05:35.124 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 124.533013631, 2024-09-14T05:05:35.124 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1670330280000001, 2024-09-14T05:05:35.124 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:05:35.124 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:05:35.124 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:05:35.124 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:05:35.124 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:05:35.124 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:05:35.124 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 151792 2024-09-14T05:05:35.124 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.125 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:05:35.125 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.125 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:05:35.125 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:30.551057+0000", 2024-09-14T05:05:35.125 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.125 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.125 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.125 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:05:35.126 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:30.551057+0000", 2024-09-14T05:05:35.126 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.126 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.126 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.126 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:05:35.126 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:30.551058+0000", 2024-09-14T05:05:35.126 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3340000000000001e-06 2024-09-14T05:05:35.126 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.126 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.126 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:05:35.126 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:30.551063+0000", 2024-09-14T05:05:35.126 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.0039999999999999e-06 2024-09-14T05:05:35.126 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.126 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.127 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:05:35.127 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:30.551075+0000", 2024-09-14T05:05:35.127 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.237e-05 2024-09-14T05:05:35.127 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.127 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.127 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:05:35.127 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:30.551085+0000", 2024-09-14T05:05:35.127 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.005e-05 2024-09-14T05:05:35.127 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.127 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.127 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:05:35.128 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:30.551856+0000", 2024-09-14T05:05:35.128 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00077016599999999997 2024-09-14T05:05:35.128 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.128 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.128 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for peered", 2024-09-14T05:05:35.128 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:30.551873+0000", 2024-09-14T05:05:35.128 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.7767000000000001e-05 2024-09-14T05:05:35.128 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.128 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.128 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:05:35.128 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.697528+0000", 2024-09-14T05:05:35.128 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.145655042 2024-09-14T05:05:35.128 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.129 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.129 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:05:35.129 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.697673+0000", 2024-09-14T05:05:35.129 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014476199999999999 2024-09-14T05:05:35.129 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.129 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.129 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 3,4,5", 2024-09-14T05:05:35.129 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.697805+0000", 2024-09-14T05:05:35.129 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013231399999999999 2024-09-14T05:05:35.129 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.129 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.129 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.130 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.701662+0000", 2024-09-14T05:05:35.130 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0038563989999999999 2024-09-14T05:05:35.130 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.130 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.130 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:05:35.130 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.701676+0000", 2024-09-14T05:05:35.130 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3862e-05 2024-09-14T05:05:35.130 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.130 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.130 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.130 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.703822+0000", 2024-09-14T05:05:35.130 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0021466889999999998 2024-09-14T05:05:35.131 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.131 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.131 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.131 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.717925+0000", 2024-09-14T05:05:35.131 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.014102185999999999 2024-09-14T05:05:35.131 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.131 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.131 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:05:35.131 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.717978+0000", 2024-09-14T05:05:35.131 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3400999999999999e-05 2024-09-14T05:05:35.131 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.131 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.131 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:05:35.132 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:03:31.718090+0000", 2024-09-14T05:05:35.132 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00011168200000000001 2024-09-14T05:05:35.132 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.132 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:05:35.132 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.132 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.132 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.132 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:183673 58.9 58:989a3ad0:::benchmark_data_smithi039_396950_object68060:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1603)", 2024-09-14T05:05:35.132 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:04:13.670658+0000", 2024-09-14T05:05:35.132 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 81.413412483000002, 2024-09-14T05:05:35.132 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.41099050500000001, 2024-09-14T05:05:35.132 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:05:35.132 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:05:35.133 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:05:35.133 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:05:35.133 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:05:35.133 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:05:35.133 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 183673 2024-09-14T05:05:35.133 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.133 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:05:35.133 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.133 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:05:35.133 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.670658+0000", 2024-09-14T05:05:35.133 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.133 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.133 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.133 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:05:35.134 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.670658+0000", 2024-09-14T05:05:35.134 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.134 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.134 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.134 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:05:35.134 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.670660+0000", 2024-09-14T05:05:35.134 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.08e-06 2024-09-14T05:05:35.134 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.134 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.134 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:05:35.134 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.670666+0000", 2024-09-14T05:05:35.134 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.8749999999999997e-06 2024-09-14T05:05:35.134 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.134 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.135 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:05:35.135 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.720021+0000", 2024-09-14T05:05:35.135 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.049355037999999997 2024-09-14T05:05:35.135 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.135 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.135 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:05:35.135 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.720032+0000", 2024-09-14T05:05:35.135 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1234e-05 2024-09-14T05:05:35.135 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.135 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.135 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:05:35.135 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.720184+0000", 2024-09-14T05:05:35.135 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000151745 2024-09-14T05:05:35.136 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.136 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.136 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:05:35.136 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.720331+0000", 2024-09-14T05:05:35.136 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014747499999999999 2024-09-14T05:05:35.136 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.136 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.136 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,3", 2024-09-14T05:05:35.136 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.720464+0000", 2024-09-14T05:05:35.136 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013254900000000001 2024-09-14T05:05:35.136 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.136 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.136 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:05:35.136 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.727527+0000", 2024-09-14T05:05:35.137 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0070632439999999998 2024-09-14T05:05:35.137 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.137 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.137 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.137 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.730493+0000", 2024-09-14T05:05:35.137 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00296569 2024-09-14T05:05:35.137 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.137 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.137 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.138 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.733059+0000", 2024-09-14T05:05:35.138 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.002566381 2024-09-14T05:05:35.138 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.138 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.138 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:05:35.138 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.733162+0000", 2024-09-14T05:05:35.138 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000102612 2024-09-14T05:05:35.138 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.138 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.138 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:05:35.139 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.081648+0000", 2024-09-14T05:05:35.139 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.34848658199999999 2024-09-14T05:05:35.139 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.139 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:05:35.139 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.139 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.139 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.139 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:183674 58.b 58:db91f653:::benchmark_data_smithi039_396950_object68061:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1603)", 2024-09-14T05:05:35.139 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:04:13.720059+0000", 2024-09-14T05:05:35.139 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 81.364010809999996, 2024-09-14T05:05:35.139 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.361727203, 2024-09-14T05:05:35.140 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:05:35.140 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:05:35.140 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:05:35.140 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:05:35.140 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:05:35.140 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:05:35.140 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 183674 2024-09-14T05:05:35.140 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.140 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:05:35.140 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.140 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:05:35.140 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.720059+0000", 2024-09-14T05:05:35.141 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.141 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.141 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.141 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:05:35.141 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.720059+0000", 2024-09-14T05:05:35.141 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.141 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.141 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.141 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:05:35.141 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.720061+0000", 2024-09-14T05:05:35.141 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.215e-06 2024-09-14T05:05:35.142 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.142 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.142 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:05:35.142 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.720065+0000", 2024-09-14T05:05:35.142 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3479999999999997e-06 2024-09-14T05:05:35.142 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.142 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.142 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:05:35.142 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.720075+0000", 2024-09-14T05:05:35.142 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0292000000000001e-05 2024-09-14T05:05:35.143 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.143 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.143 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:05:35.143 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.720081+0000", 2024-09-14T05:05:35.143 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1279999999999996e-06 2024-09-14T05:05:35.143 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.143 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.143 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:05:35.143 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.724308+0000", 2024-09-14T05:05:35.143 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.004226161 2024-09-14T05:05:35.143 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.143 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.143 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:05:35.143 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.724425+0000", 2024-09-14T05:05:35.144 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000117476 2024-09-14T05:05:35.144 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.144 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.144 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T05:05:35.144 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.724532+0000", 2024-09-14T05:05:35.144 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000106681 2024-09-14T05:05:35.144 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.144 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.144 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:05:35.144 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.727745+0000", 2024-09-14T05:05:35.144 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0032127330000000002 2024-09-14T05:05:35.144 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.144 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.145 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.145 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.730612+0000", 2024-09-14T05:05:35.145 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0028677279999999999 2024-09-14T05:05:35.145 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.145 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.145 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.145 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.081678+0000", 2024-09-14T05:05:35.145 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.351066235 2024-09-14T05:05:35.145 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.145 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.145 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:05:35.145 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.081744+0000", 2024-09-14T05:05:35.146 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.5398e-05 2024-09-14T05:05:35.146 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.146 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.146 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:05:35.146 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.081787+0000", 2024-09-14T05:05:35.146 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2808000000000001e-05 2024-09-14T05:05:35.146 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.146 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:05:35.146 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.146 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.146 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.146 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:183675 58.9 58:966641f6:::benchmark_data_smithi039_396950_object68062:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1603)", 2024-09-14T05:05:35.146 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:04:13.720100+0000", 2024-09-14T05:05:35.146 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 81.363970570000006, 2024-09-14T05:05:35.147 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.36443814400000002, 2024-09-14T05:05:35.147 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:05:35.147 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:05:35.147 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:05:35.147 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:05:35.147 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:05:35.148 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:05:35.148 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 183675 2024-09-14T05:05:35.148 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.148 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:05:35.148 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.148 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:05:35.148 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.720100+0000", 2024-09-14T05:05:35.149 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.149 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.149 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.149 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:05:35.149 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.720100+0000", 2024-09-14T05:05:35.149 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.149 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.150 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.150 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:05:35.150 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.720100+0000", 2024-09-14T05:05:35.150 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2500000000000001e-07 2024-09-14T05:05:35.150 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.150 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.150 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:05:35.150 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.720102+0000", 2024-09-14T05:05:35.150 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.34e-06 2024-09-14T05:05:35.150 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.150 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.150 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:05:35.150 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.720109+0000", 2024-09-14T05:05:35.150 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.4559999999999997e-06 2024-09-14T05:05:35.151 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.151 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.151 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:05:35.151 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.720114+0000", 2024-09-14T05:05:35.151 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9699999999999998e-06 2024-09-14T05:05:35.151 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.151 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.151 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:05:35.151 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.723892+0000", 2024-09-14T05:05:35.151 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0037781440000000002 2024-09-14T05:05:35.151 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.151 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.151 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:05:35.152 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.724025+0000", 2024-09-14T05:05:35.152 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013286799999999999 2024-09-14T05:05:35.152 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.152 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.152 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,3", 2024-09-14T05:05:35.152 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.724138+0000", 2024-09-14T05:05:35.152 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000112879 2024-09-14T05:05:35.152 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.152 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.152 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:05:35.152 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.728501+0000", 2024-09-14T05:05:35.153 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0043627270000000003 2024-09-14T05:05:35.153 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.153 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.153 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.153 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.084381+0000", 2024-09-14T05:05:35.153 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.35588053600000003 2024-09-14T05:05:35.153 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.153 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.153 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.153 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.084427+0000", 2024-09-14T05:05:35.153 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.5707000000000001e-05 2024-09-14T05:05:35.153 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.153 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.154 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:05:35.154 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.084485+0000", 2024-09-14T05:05:35.154 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.8699000000000002e-05 2024-09-14T05:05:35.154 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.154 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.154 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:05:35.154 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.084538+0000", 2024-09-14T05:05:35.154 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.2392999999999998e-05 2024-09-14T05:05:35.154 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.154 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:05:35.154 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.154 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.154 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.155 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:183676 58.f 58:f5e9e64c:::benchmark_data_smithi039_396950_object68063:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1603)", 2024-09-14T05:05:35.155 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:04:13.720121+0000", 2024-09-14T05:05:35.155 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 81.363949656000003, 2024-09-14T05:05:35.155 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.36186052200000002, 2024-09-14T05:05:35.155 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:05:35.155 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:05:35.155 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:05:35.155 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:05:35.155 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:05:35.155 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:05:35.155 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 183676 2024-09-14T05:05:35.155 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.156 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:05:35.156 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.156 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:05:35.156 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.720121+0000", 2024-09-14T05:05:35.156 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.156 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.156 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.156 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:05:35.156 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.720121+0000", 2024-09-14T05:05:35.156 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.156 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.156 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.156 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:05:35.157 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.720121+0000", 2024-09-14T05:05:35.157 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.2800000000000003e-07 2024-09-14T05:05:35.157 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.157 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.157 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:05:35.157 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.720123+0000", 2024-09-14T05:05:35.157 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.858e-06 2024-09-14T05:05:35.157 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.158 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.158 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:05:35.158 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.720128+0000", 2024-09-14T05:05:35.158 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4439999999999999e-06 2024-09-14T05:05:35.158 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.158 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.158 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:05:35.159 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.720133+0000", 2024-09-14T05:05:35.159 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.741e-06 2024-09-14T05:05:35.159 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.159 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.159 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:05:35.159 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.720296+0000", 2024-09-14T05:05:35.159 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00016295099999999999 2024-09-14T05:05:35.160 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.160 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.160 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:05:35.160 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.720419+0000", 2024-09-14T05:05:35.160 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012295200000000001 2024-09-14T05:05:35.160 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.160 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.160 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T05:05:35.160 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.720535+0000", 2024-09-14T05:05:35.160 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000115649 2024-09-14T05:05:35.160 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.160 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.160 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.161 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.727204+0000", 2024-09-14T05:05:35.161 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0066695449999999998 2024-09-14T05:05:35.161 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.161 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.161 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:05:35.161 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.727224+0000", 2024-09-14T05:05:35.161 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.0074000000000001e-05 2024-09-14T05:05:35.161 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.161 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.161 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.161 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.081897+0000", 2024-09-14T05:05:35.161 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.354672977 2024-09-14T05:05:35.161 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.162 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.162 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:05:35.162 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.081944+0000", 2024-09-14T05:05:35.162 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7138999999999997e-05 2024-09-14T05:05:35.162 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.162 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.162 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:05:35.162 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.081981+0000", 2024-09-14T05:05:35.162 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.6863999999999998e-05 2024-09-14T05:05:35.162 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.162 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:05:35.162 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.162 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.162 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.163 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36603.0:183682 58.d e1603/1568 58:ba47a851:::benchmark_data_smithi039_396950_object68069:head v 1603'11463, mlcod=1603'11461)", 2024-09-14T05:05:35.163 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:04:13.730031+0000", 2024-09-14T05:05:35.163 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 81.354039318999995, 2024-09-14T05:05:35.163 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.35236650200000003, 2024-09-14T05:05:35.163 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:05:35.163 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:05:35.163 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:05:35.163 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:05:35.163 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.164 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:05:35.164 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.730031+0000", 2024-09-14T05:05:35.164 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.164 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.164 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.164 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:05:35.164 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.730031+0000", 2024-09-14T05:05:35.164 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.164 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.164 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.164 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:05:35.164 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.730032+0000", 2024-09-14T05:05:35.164 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.8400000000000003e-07 2024-09-14T05:05:35.165 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.165 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.165 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:05:35.165 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.730038+0000", 2024-09-14T05:05:35.165 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.9780000000000002e-06 2024-09-14T05:05:35.165 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.165 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.165 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:05:35.165 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.730049+0000", 2024-09-14T05:05:35.166 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1143e-05 2024-09-14T05:05:35.166 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.166 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.166 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:05:35.166 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.730056+0000", 2024-09-14T05:05:35.167 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.6900000000000003e-06 2024-09-14T05:05:35.167 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.167 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.167 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:05:35.167 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.730169+0000", 2024-09-14T05:05:35.168 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000113585 2024-09-14T05:05:35.168 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.168 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.168 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:05:35.168 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.730213+0000", 2024-09-14T05:05:35.168 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3396000000000003e-05 2024-09-14T05:05:35.168 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.168 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.169 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:05:35.169 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.082350+0000", 2024-09-14T05:05:35.169 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.35213744400000002 2024-09-14T05:05:35.169 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.169 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.169 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:05:35.169 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.082397+0000", 2024-09-14T05:05:35.169 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.7481999999999997e-05 2024-09-14T05:05:35.169 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.169 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:05:35.169 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.169 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.169 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.170 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop_reply(client.36603.0:183675 58.9 e1603/1568 ondisk, result = 0)", 2024-09-14T05:05:35.170 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:04:13.732973+0000", 2024-09-14T05:05:35.170 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 81.351097472999996, 2024-09-14T05:05:35.170 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.35141542599999998, 2024-09-14T05:05:35.170 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:05:35.170 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:05:35.170 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "started", 2024-09-14T05:05:35.170 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:05:35.170 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.170 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:05:35.170 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.732973+0000", 2024-09-14T05:05:35.171 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.171 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.171 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.171 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:05:35.171 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.732973+0000", 2024-09-14T05:05:35.171 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.171 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.171 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.171 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:05:35.171 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.732973+0000", 2024-09-14T05:05:35.171 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.0199999999999998e-07 2024-09-14T05:05:35.171 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.171 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.172 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:05:35.172 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.732975+0000", 2024-09-14T05:05:35.172 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.3070000000000001e-06 2024-09-14T05:05:35.172 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.172 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.172 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:05:35.172 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.732982+0000", 2024-09-14T05:05:35.172 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1369999999999999e-06 2024-09-14T05:05:35.172 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.172 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.172 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:05:35.172 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.732986+0000", 2024-09-14T05:05:35.173 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.6659999999999999e-06 2024-09-14T05:05:35.173 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.173 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.173 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:05:35.173 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.084367+0000", 2024-09-14T05:05:35.173 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.35138051799999998 2024-09-14T05:05:35.173 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.173 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.173 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:05:35.173 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.084375+0000", 2024-09-14T05:05:35.174 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9510000000000006e-06 2024-09-14T05:05:35.174 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.174 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.174 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:05:35.174 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.084388+0000", 2024-09-14T05:05:35.175 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3545e-05 2024-09-14T05:05:35.175 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.175 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:05:35.175 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.175 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.175 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.175 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_repop(client.36603.0:183691 58.6 e1603/1599 58:6f6524d2:::benchmark_data_smithi039_396950_object68078:head v 1603'11626, mlcod=1603'11624)", 2024-09-14T05:05:35.175 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:04:13.734268+0000", 2024-09-14T05:05:35.175 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 81.349802272999995, 2024-09-14T05:05:35.175 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.35455119200000002, 2024-09-14T05:05:35.175 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:05:35.175 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:05:35.175 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:05:35.175 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:05:35.176 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.176 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:05:35.176 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.734268+0000", 2024-09-14T05:05:35.176 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.176 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.176 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.176 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:05:35.176 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.734268+0000", 2024-09-14T05:05:35.176 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.176 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.176 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.176 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:05:35.176 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.734269+0000", 2024-09-14T05:05:35.177 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.7199999999999997e-07 2024-09-14T05:05:35.177 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.177 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.177 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:05:35.177 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.734279+0000", 2024-09-14T05:05:35.177 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0315e-05 2024-09-14T05:05:35.177 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.177 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.177 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:05:35.177 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.081623+0000", 2024-09-14T05:05:35.177 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.347343758 2024-09-14T05:05:35.177 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.177 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.177 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:05:35.178 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.081634+0000", 2024-09-14T05:05:35.178 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0732e-05 2024-09-14T05:05:35.178 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.178 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.178 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:05:35.178 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.081690+0000", 2024-09-14T05:05:35.178 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6390000000000001e-05 2024-09-14T05:05:35.178 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.178 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.178 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:05:35.178 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.081712+0000", 2024-09-14T05:05:35.178 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.1909999999999999e-05 2024-09-14T05:05:35.178 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.178 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.179 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:05:35.179 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.088779+0000", 2024-09-14T05:05:35.179 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0070672390000000003 2024-09-14T05:05:35.179 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.179 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.179 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:05:35.179 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.088819+0000", 2024-09-14T05:05:35.179 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.9975999999999998e-05 2024-09-14T05:05:35.179 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.179 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:05:35.179 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.179 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.179 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.179 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:183692 58.f 58:f731468a:::benchmark_data_smithi039_396950_object68079:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1603)", 2024-09-14T05:05:35.180 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:04:13.734745+0000", 2024-09-14T05:05:35.180 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 81.349324859999996, 2024-09-14T05:05:35.180 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.36210478299999999, 2024-09-14T05:05:35.180 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:05:35.180 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:05:35.180 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:05:35.180 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:05:35.180 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:05:35.180 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:05:35.180 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 183692 2024-09-14T05:05:35.180 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.180 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:05:35.180 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.181 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:05:35.181 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.734745+0000", 2024-09-14T05:05:35.181 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.181 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.181 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.181 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:05:35.181 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.734745+0000", 2024-09-14T05:05:35.181 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.181 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.181 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.181 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:05:35.181 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.734747+0000", 2024-09-14T05:05:35.181 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.725e-06 2024-09-14T05:05:35.181 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.182 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.182 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:05:35.182 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:13.734752+0000", 2024-09-14T05:05:35.182 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.0810000000000001e-06 2024-09-14T05:05:35.182 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.182 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.182 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:05:35.182 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.081643+0000", 2024-09-14T05:05:35.183 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.34689031100000001 2024-09-14T05:05:35.183 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.183 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.183 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:05:35.183 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.081654+0000", 2024-09-14T05:05:35.183 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1e-05 2024-09-14T05:05:35.183 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.183 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.183 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:05:35.183 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.082016+0000", 2024-09-14T05:05:35.183 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00036249699999999998 2024-09-14T05:05:35.183 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.183 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.184 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:05:35.184 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.082169+0000", 2024-09-14T05:05:35.184 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015262199999999999 2024-09-14T05:05:35.184 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.184 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.184 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T05:05:35.184 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.082289+0000", 2024-09-14T05:05:35.184 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000120758 2024-09-14T05:05:35.185 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.185 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.185 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.185 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.093192+0000", 2024-09-14T05:05:35.185 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.010902723 2024-09-14T05:05:35.185 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.185 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.185 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:05:35.185 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.093209+0000", 2024-09-14T05:05:35.186 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.6617e-05 2024-09-14T05:05:35.186 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.186 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.186 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.186 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.096573+0000", 2024-09-14T05:05:35.186 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.003364508 2024-09-14T05:05:35.186 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.186 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.186 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:05:35.187 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.096721+0000", 2024-09-14T05:05:35.187 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00014794999999999999 2024-09-14T05:05:35.187 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.187 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.187 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:05:35.187 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:14.096850+0000", 2024-09-14T05:05:35.187 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012899100000000001 2024-09-14T05:05:35.187 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.187 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:05:35.187 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.188 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.188 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.188 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:186886 58.9 58:9135a803:::benchmark_data_smithi039_396950_object71273:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1603)", 2024-09-14T05:05:35.188 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:04:18.116884+0000", 2024-09-14T05:05:35.188 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 76.967185860000001, 2024-09-14T05:05:35.188 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.39917198599999998, 2024-09-14T05:05:35.188 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:05:35.188 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:05:35.188 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:05:35.188 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:05:35.188 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:05:35.189 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:05:35.189 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 186886 2024-09-14T05:05:35.189 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.189 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:05:35.189 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.189 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:05:35.189 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.116884+0000", 2024-09-14T05:05:35.189 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.189 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.189 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.189 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:05:35.189 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.116884+0000", 2024-09-14T05:05:35.189 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.190 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.190 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.190 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:05:35.190 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.116886+0000", 2024-09-14T05:05:35.190 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.5680000000000001e-06 2024-09-14T05:05:35.190 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.190 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.190 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:05:35.190 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.116891+0000", 2024-09-14T05:05:35.190 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9910000000000002e-06 2024-09-14T05:05:35.190 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.190 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.191 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:05:35.191 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.116903+0000", 2024-09-14T05:05:35.191 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.2418999999999999e-05 2024-09-14T05:05:35.191 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.191 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.191 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:05:35.191 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.116911+0000", 2024-09-14T05:05:35.191 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.3320000000000003e-06 2024-09-14T05:05:35.191 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.191 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.191 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:05:35.191 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.118580+0000", 2024-09-14T05:05:35.191 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0016697350000000001 2024-09-14T05:05:35.192 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.192 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.192 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:05:35.192 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.118738+0000", 2024-09-14T05:05:35.192 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00015751 2024-09-14T05:05:35.192 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.192 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.192 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,3", 2024-09-14T05:05:35.192 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.118852+0000", 2024-09-14T05:05:35.192 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000114125 2024-09-14T05:05:35.192 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.192 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.192 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:05:35.192 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.160680+0000", 2024-09-14T05:05:35.193 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.041827549999999998 2024-09-14T05:05:35.193 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.193 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.193 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.193 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.161521+0000", 2024-09-14T05:05:35.193 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000841485 2024-09-14T05:05:35.193 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.193 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.193 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.193 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.515743+0000", 2024-09-14T05:05:35.193 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.35422224800000002 2024-09-14T05:05:35.193 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.193 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.194 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:05:35.194 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.515956+0000", 2024-09-14T05:05:35.194 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000212377 2024-09-14T05:05:35.194 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.194 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.194 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:05:35.194 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.516056+0000", 2024-09-14T05:05:35.194 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000100646 2024-09-14T05:05:35.194 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.194 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:05:35.194 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.194 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.194 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.194 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:186890 58.8 58:11b83774:::benchmark_data_smithi039_396950_object71277:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1603)", 2024-09-14T05:05:35.195 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:04:18.160537+0000", 2024-09-14T05:05:35.195 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 76.923532965000007, 2024-09-14T05:05:35.195 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.35514166899999999, 2024-09-14T05:05:35.195 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:05:35.195 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:05:35.195 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:05:35.195 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:05:35.195 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:05:35.195 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:05:35.195 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 186890 2024-09-14T05:05:35.195 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.195 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:05:35.195 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.195 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:05:35.196 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.160537+0000", 2024-09-14T05:05:35.196 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.196 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.196 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.196 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:05:35.196 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.160537+0000", 2024-09-14T05:05:35.196 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.196 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.196 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.196 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:05:35.196 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.160539+0000", 2024-09-14T05:05:35.196 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.64e-06 2024-09-14T05:05:35.196 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.197 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.197 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:05:35.197 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.160549+0000", 2024-09-14T05:05:35.197 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0158e-05 2024-09-14T05:05:35.197 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.197 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.197 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:05:35.197 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.160563+0000", 2024-09-14T05:05:35.197 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.3922e-05 2024-09-14T05:05:35.197 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.197 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.197 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:05:35.198 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.160574+0000", 2024-09-14T05:05:35.198 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1348999999999999e-05 2024-09-14T05:05:35.198 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.198 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.198 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:05:35.198 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.161402+0000", 2024-09-14T05:05:35.198 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00082757000000000002 2024-09-14T05:05:35.198 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.198 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.198 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:05:35.198 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.161671+0000", 2024-09-14T05:05:35.198 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00026862000000000001 2024-09-14T05:05:35.198 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.198 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.199 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,4,6", 2024-09-14T05:05:35.199 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.161781+0000", 2024-09-14T05:05:35.199 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000110468 2024-09-14T05:05:35.199 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.199 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.199 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.199 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.165274+0000", 2024-09-14T05:05:35.199 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0034926699999999998 2024-09-14T05:05:35.199 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.199 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.199 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.199 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.165309+0000", 2024-09-14T05:05:35.200 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.5784000000000002e-05 2024-09-14T05:05:35.200 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.200 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.200 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:05:35.200 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.165317+0000", 2024-09-14T05:05:35.200 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.9580000000000007e-06 2024-09-14T05:05:35.200 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.200 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.200 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.200 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.515494+0000", 2024-09-14T05:05:35.200 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.350176143 2024-09-14T05:05:35.201 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.201 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.201 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:05:35.201 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.515547+0000", 2024-09-14T05:05:35.201 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3380000000000001e-05 2024-09-14T05:05:35.201 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.201 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.201 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:05:35.201 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.515679+0000", 2024-09-14T05:05:35.201 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013200700000000001 2024-09-14T05:05:35.201 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.202 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:05:35.202 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.202 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.202 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.202 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:186892 58.f 58:f5bc58ce:::benchmark_data_smithi039_396950_object71279:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1603)", 2024-09-14T05:05:35.202 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:04:18.161312+0000", 2024-09-14T05:05:35.202 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 76.922758568000006, 2024-09-14T05:05:35.202 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.35745279400000002, 2024-09-14T05:05:35.202 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:05:35.203 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:05:35.203 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:05:35.203 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:05:35.203 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:05:35.203 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:05:35.204 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 186892 2024-09-14T05:05:35.204 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.204 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:05:35.204 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.204 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:05:35.204 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.161312+0000", 2024-09-14T05:05:35.205 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.205 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.205 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.205 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:05:35.205 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.161312+0000", 2024-09-14T05:05:35.205 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.206 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.206 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.206 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:05:35.206 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.161312+0000", 2024-09-14T05:05:35.206 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7599999999999996e-07 2024-09-14T05:05:35.206 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.206 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.207 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:05:35.207 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.161317+0000", 2024-09-14T05:05:35.207 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.403e-06 2024-09-14T05:05:35.207 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.207 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.207 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:05:35.207 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.161328+0000", 2024-09-14T05:05:35.207 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0961000000000001e-05 2024-09-14T05:05:35.208 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.208 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.208 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:05:35.208 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.161335+0000", 2024-09-14T05:05:35.208 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.9940000000000003e-06 2024-09-14T05:05:35.208 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.209 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.209 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:05:35.209 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.161405+0000", 2024-09-14T05:05:35.209 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.0623999999999993e-05 2024-09-14T05:05:35.209 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.210 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.210 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:05:35.210 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.161545+0000", 2024-09-14T05:05:35.210 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00013977299999999999 2024-09-14T05:05:35.210 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.210 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.210 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T05:05:35.211 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.161654+0000", 2024-09-14T05:05:35.211 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00010849900000000001 2024-09-14T05:05:35.211 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.211 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.211 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.211 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.164155+0000", 2024-09-14T05:05:35.211 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0025016779999999998 2024-09-14T05:05:35.212 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.212 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.212 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:05:35.212 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.164325+0000", 2024-09-14T05:05:35.212 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00016972600000000001 2024-09-14T05:05:35.212 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.212 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.212 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.212 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.518609+0000", 2024-09-14T05:05:35.212 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.35428358900000001 2024-09-14T05:05:35.212 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.212 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.212 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:05:35.213 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.518722+0000", 2024-09-14T05:05:35.213 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000113614 2024-09-14T05:05:35.213 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.213 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.213 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:05:35.213 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.518765+0000", 2024-09-14T05:05:35.214 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.2157000000000003e-05 2024-09-14T05:05:35.214 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.214 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:05:35.214 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.214 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.214 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.214 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:186900 58.b 58:de68cf1b:::benchmark_data_smithi039_396950_object71287:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1603)", 2024-09-14T05:05:35.214 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:04:18.161356+0000", 2024-09-14T05:05:35.214 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 76.922713909999999, 2024-09-14T05:05:35.214 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.35442497000000001, 2024-09-14T05:05:35.214 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:05:35.215 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:05:35.215 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:05:35.215 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:05:35.215 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:05:35.215 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:05:35.215 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 186900 2024-09-14T05:05:35.215 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.215 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:05:35.215 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.216 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:05:35.216 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.161356+0000", 2024-09-14T05:05:35.216 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.216 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.216 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.216 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:05:35.216 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.161356+0000", 2024-09-14T05:05:35.216 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.216 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.216 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.216 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:05:35.216 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.161357+0000", 2024-09-14T05:05:35.217 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.3900000000000005e-07 2024-09-14T05:05:35.217 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.217 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.217 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:05:35.217 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.161360+0000", 2024-09-14T05:05:35.217 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.2389999999999998e-06 2024-09-14T05:05:35.217 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.217 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.218 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:05:35.218 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.161367+0000", 2024-09-14T05:05:35.218 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.3490000000000003e-06 2024-09-14T05:05:35.218 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.218 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.218 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:05:35.218 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.161375+0000", 2024-09-14T05:05:35.219 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.5730000000000002e-06 2024-09-14T05:05:35.219 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.219 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.219 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:05:35.219 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.163858+0000", 2024-09-14T05:05:35.219 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0024827949999999999 2024-09-14T05:05:35.219 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.219 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.220 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:05:35.220 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.163948+0000", 2024-09-14T05:05:35.220 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.0304999999999997e-05 2024-09-14T05:05:35.220 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.220 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.220 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T05:05:35.220 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.164072+0000", 2024-09-14T05:05:35.220 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000123719 2024-09-14T05:05:35.221 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.221 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.221 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.221 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.168112+0000", 2024-09-14T05:05:35.221 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0040403820000000003 2024-09-14T05:05:35.221 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.221 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.221 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:05:35.222 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.168120+0000", 2024-09-14T05:05:35.222 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7929999999999998e-06 2024-09-14T05:05:35.222 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.222 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.222 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.222 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.515337+0000", 2024-09-14T05:05:35.222 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.34721733700000001 2024-09-14T05:05:35.223 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.223 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.223 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:05:35.223 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.515717+0000", 2024-09-14T05:05:35.223 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00037987300000000003 2024-09-14T05:05:35.223 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.223 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.223 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:05:35.224 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.515781+0000", 2024-09-14T05:05:35.224 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.4066000000000006e-05 2024-09-14T05:05:35.224 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.224 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:05:35.224 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.224 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.225 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.225 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:186902 58.b 58:dc2a3dec:::benchmark_data_smithi039_396950_object71289:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1603)", 2024-09-14T05:05:35.225 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:04:18.161388+0000", 2024-09-14T05:05:35.225 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 76.922681882999996, 2024-09-14T05:05:35.225 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.35460921000000001, 2024-09-14T05:05:35.225 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:05:35.225 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:05:35.225 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:05:35.225 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:05:35.226 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:05:35.226 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:05:35.226 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 186902 2024-09-14T05:05:35.226 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.226 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:05:35.226 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.226 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:05:35.228 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.161388+0000", 2024-09-14T05:05:35.228 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.228 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.228 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.228 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:05:35.228 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.161388+0000", 2024-09-14T05:05:35.228 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.229 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.229 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.229 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:05:35.229 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.161389+0000", 2024-09-14T05:05:35.229 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.4899999999999995e-07 2024-09-14T05:05:35.229 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.229 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.229 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:05:35.230 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.161392+0000", 2024-09-14T05:05:35.230 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.0429999999999999e-06 2024-09-14T05:05:35.230 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.230 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.230 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:05:35.231 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.161398+0000", 2024-09-14T05:05:35.231 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.3380000000000001e-06 2024-09-14T05:05:35.231 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.231 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.231 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:05:35.231 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.161404+0000", 2024-09-14T05:05:35.231 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.5180000000000004e-06 2024-09-14T05:05:35.231 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.232 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.232 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:05:35.232 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.166219+0000", 2024-09-14T05:05:35.232 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0048156070000000004 2024-09-14T05:05:35.232 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.232 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.232 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:05:35.232 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.166344+0000", 2024-09-14T05:05:35.232 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012427 2024-09-14T05:05:35.232 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.233 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.233 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T05:05:35.233 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.166438+0000", 2024-09-14T05:05:35.233 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 9.4056999999999997e-05 2024-09-14T05:05:35.233 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.233 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.233 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.233 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.515847+0000", 2024-09-14T05:05:35.233 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.34940948700000002 2024-09-14T05:05:35.233 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.233 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.233 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.233 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.515900+0000", 2024-09-14T05:05:35.234 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.2775000000000002e-05 2024-09-14T05:05:35.234 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.234 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.234 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:05:35.234 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.515911+0000", 2024-09-14T05:05:35.234 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.1343e-05 2024-09-14T05:05:35.234 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.234 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.234 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:05:35.234 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.515960+0000", 2024-09-14T05:05:35.234 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.8164999999999997e-05 2024-09-14T05:05:35.234 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.234 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.234 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:05:35.235 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.515998+0000", 2024-09-14T05:05:35.235 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.8058000000000001e-05 2024-09-14T05:05:35.235 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.235 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:05:35.235 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.235 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.235 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.235 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:186905 58.b 58:da1a2089:::benchmark_data_smithi039_396950_object71292:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1603)", 2024-09-14T05:05:35.235 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:04:18.166254+0000", 2024-09-14T05:05:35.235 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 76.917815997999995, 2024-09-14T05:05:35.235 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.35520004399999999, 2024-09-14T05:05:35.235 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:05:35.235 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:05:35.235 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:05:35.236 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:05:35.236 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:05:35.236 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:05:35.236 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 186905 2024-09-14T05:05:35.236 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.236 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:05:35.236 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.236 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:05:35.236 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.166254+0000", 2024-09-14T05:05:35.236 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.236 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.236 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.236 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:05:35.237 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.166254+0000", 2024-09-14T05:05:35.237 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.237 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.237 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.237 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:05:35.237 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.166255+0000", 2024-09-14T05:05:35.237 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.0219999999999999e-06 2024-09-14T05:05:35.237 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.237 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.237 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:05:35.237 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.166259+0000", 2024-09-14T05:05:35.237 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.1500000000000001e-06 2024-09-14T05:05:35.237 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.238 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.238 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:05:35.238 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.166267+0000", 2024-09-14T05:05:35.238 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 7.7179999999999996e-06 2024-09-14T05:05:35.238 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.238 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.238 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:05:35.238 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.166272+0000", 2024-09-14T05:05:35.238 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9679999999999997e-06 2024-09-14T05:05:35.238 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.238 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.239 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:05:35.239 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.516028+0000", 2024-09-14T05:05:35.239 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.34975617199999998 2024-09-14T05:05:35.239 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.239 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.239 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:05:35.239 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.516154+0000", 2024-09-14T05:05:35.239 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00012603899999999999 2024-09-14T05:05:35.239 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.239 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.239 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 6,7", 2024-09-14T05:05:35.239 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.516261+0000", 2024-09-14T05:05:35.240 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000106407 2024-09-14T05:05:35.240 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.240 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.240 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.240 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.521130+0000", 2024-09-14T05:05:35.240 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0048692049999999997 2024-09-14T05:05:35.240 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.240 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.240 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.241 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.521301+0000", 2024-09-14T05:05:35.241 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.00017105399999999999 2024-09-14T05:05:35.241 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.241 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.241 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:05:35.241 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.521362+0000", 2024-09-14T05:05:35.241 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.0760000000000001e-05 2024-09-14T05:05:35.241 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.241 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.241 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:05:35.241 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.521411+0000", 2024-09-14T05:05:35.241 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.9289999999999997e-05 2024-09-14T05:05:35.241 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.241 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.242 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:05:35.242 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.521454+0000", 2024-09-14T05:05:35.242 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 4.3259e-05 2024-09-14T05:05:35.242 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.242 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:05:35.242 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.242 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.242 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.242 INFO:teuthology.orchestra.run.smithi039.stdout: "description": "osd_op(client.36603.0:186906 58.f 58:fccb1d9b:::benchmark_data_smithi039_396950_object71293:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e1603)", 2024-09-14T05:05:35.242 INFO:teuthology.orchestra.run.smithi039.stdout: "initiated_at": "2024-09-14T05:04:18.178995+0000", 2024-09-14T05:05:35.242 INFO:teuthology.orchestra.run.smithi039.stdout: "age": 76.905075647999993, 2024-09-14T05:05:35.242 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.35755272199999999, 2024-09-14T05:05:35.242 INFO:teuthology.orchestra.run.smithi039.stdout: "continuous": false, 2024-09-14T05:05:35.242 INFO:teuthology.orchestra.run.smithi039.stdout: "type_data": { 2024-09-14T05:05:35.243 INFO:teuthology.orchestra.run.smithi039.stdout: "flag_point": "commit sent; apply or cleanup", 2024-09-14T05:05:35.243 INFO:teuthology.orchestra.run.smithi039.stdout: "client_info": { 2024-09-14T05:05:35.243 INFO:teuthology.orchestra.run.smithi039.stdout: "client": "client.36603", 2024-09-14T05:05:35.243 INFO:teuthology.orchestra.run.smithi039.stdout: "client_addr": "172.21.15.39:0/3043577726", 2024-09-14T05:05:35.243 INFO:teuthology.orchestra.run.smithi039.stdout: "tid": 186906 2024-09-14T05:05:35.243 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.243 INFO:teuthology.orchestra.run.smithi039.stdout: "events": [ 2024-09-14T05:05:35.243 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.243 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "initiated", 2024-09-14T05:05:35.243 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.178995+0000", 2024-09-14T05:05:35.243 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.244 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.244 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.244 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "header_read", 2024-09-14T05:05:35.244 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.178995+0000", 2024-09-14T05:05:35.244 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0 2024-09-14T05:05:35.244 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.244 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.244 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "throttled", 2024-09-14T05:05:35.244 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.178997+0000", 2024-09-14T05:05:35.244 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 2.181e-06 2024-09-14T05:05:35.244 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.244 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.244 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "all_read", 2024-09-14T05:05:35.245 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.179005+0000", 2024-09-14T05:05:35.245 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 8.3180000000000007e-06 2024-09-14T05:05:35.245 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.245 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.245 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "dispatched", 2024-09-14T05:05:35.245 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.515248+0000", 2024-09-14T05:05:35.245 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.33624280000000001 2024-09-14T05:05:35.245 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.245 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.245 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "queued_for_pg", 2024-09-14T05:05:35.245 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.515266+0000", 2024-09-14T05:05:35.245 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 1.8087000000000001e-05 2024-09-14T05:05:35.245 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.246 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.246 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "reached_pg", 2024-09-14T05:05:35.246 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.515387+0000", 2024-09-14T05:05:35.246 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000121396 2024-09-14T05:05:35.246 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.246 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.246 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "started", 2024-09-14T05:05:35.246 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.515581+0000", 2024-09-14T05:05:35.246 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000193131 2024-09-14T05:05:35.246 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.246 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.246 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "waiting for subops from 2,6", 2024-09-14T05:05:35.246 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.515711+0000", 2024-09-14T05:05:35.246 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.000130338 2024-09-14T05:05:35.246 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.247 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.247 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.247 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.525115+0000", 2024-09-14T05:05:35.247 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.0094037019999999999 2024-09-14T05:05:35.247 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.247 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.247 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "op_commit", 2024-09-14T05:05:35.247 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.525171+0000", 2024-09-14T05:05:35.247 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 5.6752000000000001e-05 2024-09-14T05:05:35.247 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.248 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.248 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "sub_op_commit_rec", 2024-09-14T05:05:35.248 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.536448+0000", 2024-09-14T05:05:35.248 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 0.011276381 2024-09-14T05:05:35.248 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.248 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.248 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "commit_sent", 2024-09-14T05:05:35.248 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.536509+0000", 2024-09-14T05:05:35.248 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 6.1259000000000004e-05 2024-09-14T05:05:35.248 INFO:teuthology.orchestra.run.smithi039.stdout: }, 2024-09-14T05:05:35.248 INFO:teuthology.orchestra.run.smithi039.stdout: { 2024-09-14T05:05:35.248 INFO:teuthology.orchestra.run.smithi039.stdout: "event": "done", 2024-09-14T05:05:35.249 INFO:teuthology.orchestra.run.smithi039.stdout: "time": "2024-09-14T05:04:18.536547+0000", 2024-09-14T05:05:35.249 INFO:teuthology.orchestra.run.smithi039.stdout: "duration": 3.8377000000000002e-05 2024-09-14T05:05:35.249 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.249 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:05:35.249 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.249 INFO:teuthology.orchestra.run.smithi039.stdout: } 2024-09-14T05:05:35.249 INFO:teuthology.orchestra.run.smithi039.stdout: ] 2024-09-14T05:05:35.249 INFO:teuthology.orchestra.run.smithi039.stdout:} 2024-09-14T05:05:35.250 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:35.251 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:35.251 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:35.352 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:35.378 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:35.378 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:35.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:35.479 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:35.502 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:35.503 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:35.503 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:35.604 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:35.630 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:35.631 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:35.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:35.732 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:35.755 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:35.755 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:35.755 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:35.848 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:35 smithi039 ceph-mon[106470]: pgmap v3260: 103 pgs: 103 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2024-09-14T05:05:35.848 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:35 smithi039 ceph-mon[106470]: osdmap e1613: 8 total, 8 up, 7 in 2024-09-14T05:05:35.848 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:35 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3124932618' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi039-426541-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T05:05:35.848 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:35 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi039-426541-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T05:05:35.848 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:35 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3285717354' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 0.43738039969388554} : dispatch 2024-09-14T05:05:35.848 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:35 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 0.43738039969388554} : dispatch 2024-09-14T05:05:35.849 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:35 smithi039 ceph-mon[111235]: pgmap v3260: 103 pgs: 103 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2024-09-14T05:05:35.849 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:35 smithi039 ceph-mon[111235]: osdmap e1613: 8 total, 8 up, 7 in 2024-09-14T05:05:35.849 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:35 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3124932618' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi039-426541-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T05:05:35.849 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:35 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi039-426541-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T05:05:35.849 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:35 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3285717354' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 0.43738039969388554} : dispatch 2024-09-14T05:05:35.849 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:35 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 0.43738039969388554} : dispatch 2024-09-14T05:05:35.856 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:35.882 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:35.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:35.883 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:35.984 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:35.992 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.6 to 0.43738 (6ff8) 2024-09-14T05:05:35.996 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.get_all_features 2024-09-14T05:05:36.019 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.get_all_features (23 ms) 2024-09-14T05:05:36.019 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.copyup 2024-09-14T05:05:36.019 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:36.020 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:36.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:36.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:35 smithi129 ceph-mon[101316]: pgmap v3260: 103 pgs: 103 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2024-09-14T05:05:36.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:35 smithi129 ceph-mon[101316]: osdmap e1613: 8 total, 8 up, 7 in 2024-09-14T05:05:36.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:35 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3124932618' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi039-426541-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T05:05:36.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:35 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool application enable","pool": "test-rados-api-smithi039-426541-1","app": "rados","yes_i_really_mean_it": true} : dispatch 2024-09-14T05:05:36.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:35 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3285717354' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 0.43738039969388554} : dispatch 2024-09-14T05:05:36.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:35 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 0.43738039969388554} : dispatch 2024-09-14T05:05:36.121 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:36.143 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:36.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:36.144 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:36.198 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.copyup (179 ms) 2024-09-14T05:05:36.198 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.sparse_copyup 2024-09-14T05:05:36.241 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.sparse_copyup (43 ms) 2024-09-14T05:05:36.241 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.get_and_set_id 2024-09-14T05:05:36.245 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:36.250 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.get_and_set_id (10 ms) 2024-09-14T05:05:36.251 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.add_remove_child 2024-09-14T05:05:36.263 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.add_remove_child (12 ms) 2024-09-14T05:05:36.263 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.directory_methods 2024-09-14T05:05:36.268 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:36.269 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:36.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:36.349 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.directory_methods (86 ms) 2024-09-14T05:05:36.349 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.create 2024-09-14T05:05:36.363 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.create (15 ms) 2024-09-14T05:05:36.364 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.get_features 2024-09-14T05:05:36.365 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.get_features (1 ms) 2024-09-14T05:05:36.365 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.get_object_prefix 2024-09-14T05:05:36.367 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.get_object_prefix (2 ms) 2024-09-14T05:05:36.367 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2024-09-14T05:05:36.369 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2024-09-14T05:05:36.369 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2024-09-14T05:05:36.370 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:36.371 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.get_access_timestamp (1 ms) 2024-09-14T05:05:36.371 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2024-09-14T05:05:36.372 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.get_modify_timestamp (1 ms) 2024-09-14T05:05:36.372 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.get_data_pool 2024-09-14T05:05:36.375 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.get_data_pool (3 ms) 2024-09-14T05:05:36.375 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.get_size 2024-09-14T05:05:36.379 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.get_size (4 ms) 2024-09-14T05:05:36.379 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.set_size 2024-09-14T05:05:36.383 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.set_size (5 ms) 2024-09-14T05:05:36.383 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.protection_status 2024-09-14T05:05:36.393 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:36.394 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:36.396 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:36.403 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.protection_status (19 ms) 2024-09-14T05:05:36.403 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.snapshot_limits 2024-09-14T05:05:36.410 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.snapshot_limits (7 ms) 2024-09-14T05:05:36.410 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.parents_v1 2024-09-14T05:05:36.445 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_ops_in_flight 2024-09-14T05:05:36.455 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.parents_v1 (45 ms) 2024-09-14T05:05:36.455 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.parents_v2 2024-09-14T05:05:36.486 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.parents_v2 (31 ms) 2024-09-14T05:05:36.486 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.snapshots 2024-09-14T05:05:36.497 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1 2024-09-14T05:05:36.510 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.snapshots (24 ms) 2024-09-14T05:05:36.510 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.snapid_race 2024-09-14T05:05:36.516 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.snapid_race (6 ms) 2024-09-14T05:05:36.517 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.stripingv2 2024-09-14T05:05:36.521 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:36.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:36.522 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.1' 2024-09-14T05:05:36.526 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.stripingv2 (10 ms) 2024-09-14T05:05:36.526 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.object_map_save 2024-09-14T05:05:36.530 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.object_map_save (5 ms) 2024-09-14T05:05:36.530 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.object_map_resize 2024-09-14T05:05:36.537 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.object_map_resize (6 ms) 2024-09-14T05:05:36.537 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.object_map_update 2024-09-14T05:05:36.546 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.object_map_update (9 ms) 2024-09-14T05:05:36.546 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2024-09-14T05:05:36.546 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2024-09-14T05:05:36.546 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2024-09-14T05:05:36.552 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.object_map_snap_add (6 ms) 2024-09-14T05:05:36.552 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2024-09-14T05:05:36.563 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.object_map_snap_remove (11 ms) 2024-09-14T05:05:36.563 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.flags 2024-09-14T05:05:36.568 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.flags (5 ms) 2024-09-14T05:05:36.568 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.metadata 2024-09-14T05:05:36.593 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.metadata (25 ms) 2024-09-14T05:05:36.593 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.set_features 2024-09-14T05:05:36.600 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.set_features (8 ms) 2024-09-14T05:05:36.600 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.mirror 2024-09-14T05:05:36.624 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:36.635 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.mirror (35 ms) 2024-09-14T05:05:36.636 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.mirror_image 2024-09-14T05:05:36.648 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:36.649 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:36.649 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:36.653 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.mirror_image (17 ms) 2024-09-14T05:05:36.653 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.mirror_image_status 2024-09-14T05:05:36.679 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:05:36.683 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:05:36.750 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:36.771 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:36.772 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:36.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:36.873 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:36.895 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:36.895 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:36.896 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:36.997 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:37.021 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:37.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:37.022 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:37.123 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:37.132 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:36 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi039-426541-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T05:05:37.132 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:36 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 0.43738039969388554}]': finished 2024-09-14T05:05:37.132 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:36 smithi039 ceph-mon[106470]: osdmap e1614: 8 total, 8 up, 7 in 2024-09-14T05:05:37.132 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:36 smithi039 ceph-mon[106470]: pgmap v3263: 135 pgs: 3 creating+peering, 29 unknown, 103 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2024-09-14T05:05:37.132 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:36 smithi039 ceph-mon[106470]: osdmap e1615: 8 total, 8 up, 7 in 2024-09-14T05:05:37.133 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:36 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi039-426541-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T05:05:37.133 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:36 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 0.43738039969388554}]': finished 2024-09-14T05:05:37.133 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:36 smithi039 ceph-mon[111235]: osdmap e1614: 8 total, 8 up, 7 in 2024-09-14T05:05:37.133 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:36 smithi039 ceph-mon[111235]: pgmap v3263: 135 pgs: 3 creating+peering, 29 unknown, 103 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2024-09-14T05:05:37.133 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:36 smithi039 ceph-mon[111235]: osdmap e1615: 8 total, 8 up, 7 in 2024-09-14T05:05:37.152 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:37.153 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:37.153 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:37.255 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:37.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:36 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi039-426541-1","app": "rados","yes_i_really_mean_it": true}]': finished 2024-09-14T05:05:37.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:36 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 0.43738039969388554}]': finished 2024-09-14T05:05:37.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:36 smithi129 ceph-mon[101316]: osdmap e1614: 8 total, 8 up, 7 in 2024-09-14T05:05:37.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:36 smithi129 ceph-mon[101316]: pgmap v3263: 135 pgs: 3 creating+peering, 29 unknown, 103 active+clean; 582 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2024-09-14T05:05:37.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:36 smithi129 ceph-mon[101316]: osdmap e1615: 8 total, 8 up, 7 in 2024-09-14T05:05:37.279 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:37.279 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:37.280 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:37.380 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:37.402 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:37.403 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:37.403 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:37.504 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:37.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:37.531 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:37.531 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:37.632 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:37.653 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:37.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:37.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:37.755 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:37.781 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:37.782 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:37.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:37.883 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:37.904 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:37.905 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:37.905 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:38.006 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:38.030 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:38.030 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:38.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:38.132 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:38.156 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:38.157 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:38.157 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:38.258 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:38.280 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:38.281 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:38.281 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:38.382 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:38.406 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:38.407 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:38.407 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:38.508 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:38.529 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:38.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:38.530 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:38.631 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:38.648 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:38 smithi039 ceph-mon[106470]: osdmap e1616: 8 total, 8 up, 7 in 2024-09-14T05:05:38.648 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:38 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:05:38.648 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:38 smithi039 ceph-mon[111235]: osdmap e1616: 8 total, 8 up, 7 in 2024-09-14T05:05:38.648 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:38 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:05:38.653 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:38.654 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:38.654 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:38.755 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:38.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:38 smithi129 ceph-mon[101316]: osdmap e1616: 8 total, 8 up, 7 in 2024-09-14T05:05:38.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:38 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:05:38.778 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:38.779 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:38.779 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:38.880 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:38.902 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:38.902 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:38.903 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:38.923 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:05:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:05:38] "GET /metrics HTTP/1.1" 200 36722 "" "Prometheus/2.51.0" 2024-09-14T05:05:39.004 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:39.026 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:39.026 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:39.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:39.128 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:39.152 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:39.152 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:39.153 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:39.254 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:39.275 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:39.276 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:39.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:39.279 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.mirror_image_status (2626 ms) 2024-09-14T05:05:39.279 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.mirror_image_map 2024-09-14T05:05:39.376 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:39.378 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.mirror_image_map (98 ms) 2024-09-14T05:05:39.378 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.mirror_instances 2024-09-14T05:05:39.389 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.mirror_instances (12 ms) 2024-09-14T05:05:39.389 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2024-09-14T05:05:39.399 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:39.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:39.400 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:39.405 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.mirror_snapshot (16 ms) 2024-09-14T05:05:39.405 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.group_dir_list 2024-09-14T05:05:39.407 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.group_dir_list (3 ms) 2024-09-14T05:05:39.407 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.group_dir_add 2024-09-14T05:05:39.410 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.group_dir_add (2 ms) 2024-09-14T05:05:39.410 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2024-09-14T05:05:39.413 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2024-09-14T05:05:39.413 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.group_dir_rename 2024-09-14T05:05:39.419 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.group_dir_rename (6 ms) 2024-09-14T05:05:39.419 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.group_dir_remove 2024-09-14T05:05:39.422 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.group_dir_remove (3 ms) 2024-09-14T05:05:39.422 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2024-09-14T05:05:39.426 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (5 ms) 2024-09-14T05:05:39.426 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.group_image_add 2024-09-14T05:05:39.429 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.group_image_add (2 ms) 2024-09-14T05:05:39.429 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.group_image_remove 2024-09-14T05:05:39.433 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.group_image_remove (4 ms) 2024-09-14T05:05:39.433 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.group_image_list 2024-09-14T05:05:39.436 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.group_image_list (3 ms) 2024-09-14T05:05:39.436 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.group_image_clean 2024-09-14T05:05:39.440 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.group_image_clean (4 ms) 2024-09-14T05:05:39.440 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.image_group_add 2024-09-14T05:05:39.442 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.image_group_add (3 ms) 2024-09-14T05:05:39.443 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.image_group_remove 2024-09-14T05:05:39.446 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2024-09-14T05:05:39.446 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.image_group_get 2024-09-14T05:05:39.448 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.image_group_get (2 ms) 2024-09-14T05:05:39.448 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2024-09-14T05:05:39.450 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2024-09-14T05:05:39.450 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2024-09-14T05:05:39.452 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2024-09-14T05:05:39.452 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2024-09-14T05:05:39.455 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (3 ms) 2024-09-14T05:05:39.455 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2024-09-14T05:05:39.458 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2024-09-14T05:05:39.458 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.group_snap_set 2024-09-14T05:05:39.461 INFO:tasks.workunit.client.0.smithi039.stdout:/home/jenkins-build/build/workspace/ceph-dev-build/ARCH/x86_64/AVAILABLE_ARCH/x86_64/AVAILABLE_DIST/centos9/DIST/centos9/MACHINE_SIZE/gigantic/release/18.2.4-845-ge4e227b3/rpm/el9/BUILD/ceph-18.2.4-845-ge4e227b3/src/test/cls_rbd/test_cls_rbd.cc:2681: Failure 2024-09-14T05:05:39.461 INFO:tasks.workunit.client.0.smithi039.stdout:Expected equality of these values: 2024-09-14T05:05:39.461 INFO:tasks.workunit.client.0.smithi039.stdout: 1U 2024-09-14T05:05:39.461 INFO:tasks.workunit.client.0.smithi039.stdout: Which is: 1 2024-09-14T05:05:39.461 INFO:tasks.workunit.client.0.smithi039.stdout: keys.size() 2024-09-14T05:05:39.461 INFO:tasks.workunit.client.0.smithi039.stdout: Which is: 3 2024-09-14T05:05:39.461 INFO:tasks.workunit.client.0.smithi039.stdout:[ FAILED ] TestClsRbd.group_snap_set (3 ms) 2024-09-14T05:05:39.461 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.group_snap_list 2024-09-14T05:05:39.464 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.group_snap_list (3 ms) 2024-09-14T05:05:39.464 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2024-09-14T05:05:39.481 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (18 ms) 2024-09-14T05:05:39.482 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2024-09-14T05:05:39.501 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:39.523 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:39.524 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:39.524 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:39.625 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:39.650 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:39.651 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:39.651 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:39.688 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (206 ms) 2024-09-14T05:05:39.688 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.group_snap_remove 2024-09-14T05:05:39.691 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.group_snap_remove (4 ms) 2024-09-14T05:05:39.691 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2024-09-14T05:05:39.693 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (1 ms) 2024-09-14T05:05:39.693 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.trash_methods 2024-09-14T05:05:39.700 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.trash_methods (7 ms) 2024-09-14T05:05:39.700 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.op_features 2024-09-14T05:05:39.708 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.op_features (8 ms) 2024-09-14T05:05:39.708 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.clone_parent 2024-09-14T05:05:39.729 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.clone_parent (21 ms) 2024-09-14T05:05:39.729 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2024-09-14T05:05:39.737 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.clone_parent_ns (8 ms) 2024-09-14T05:05:39.738 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.clone_child 2024-09-14T05:05:39.750 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.clone_child (14 ms) 2024-09-14T05:05:39.751 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.namespace_methods 2024-09-14T05:05:39.752 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:39.757 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.namespace_methods (6 ms) 2024-09-14T05:05:39.757 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.migration 2024-09-14T05:05:39.767 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.migration (11 ms) 2024-09-14T05:05:39.767 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.migration_v1 2024-09-14T05:05:39.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:39 smithi129 ceph-mon[101316]: pgmap v3266: 135 pgs: 5 active+undersized+remapped, 4 peering, 4 creating+peering, 11 unknown, 111 active+clean; 595 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 664 KiB/s rd, 463 KiB/s wr, 1.11k op/s; 9.8 KiB/s, 0 objects/s recovering 2024-09-14T05:05:39.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:39 smithi129 ceph-mon[101316]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T05:05:39.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:39 smithi129 ceph-mon[101316]: osdmap e1617: 8 total, 8 up, 7 in 2024-09-14T05:05:39.774 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:39.774 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:39.775 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:39.783 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.migration_v1 (16 ms) 2024-09-14T05:05:39.784 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2024-09-14T05:05:39.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:39 smithi039 ceph-mon[106470]: pgmap v3266: 135 pgs: 5 active+undersized+remapped, 4 peering, 4 creating+peering, 11 unknown, 111 active+clean; 595 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 664 KiB/s rd, 463 KiB/s wr, 1.11k op/s; 9.8 KiB/s, 0 objects/s recovering 2024-09-14T05:05:39.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:39 smithi039 ceph-mon[106470]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T05:05:39.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:39 smithi039 ceph-mon[106470]: osdmap e1617: 8 total, 8 up, 7 in 2024-09-14T05:05:39.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:39 smithi039 ceph-mon[111235]: pgmap v3266: 135 pgs: 5 active+undersized+remapped, 4 peering, 4 creating+peering, 11 unknown, 111 active+clean; 595 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 664 KiB/s rd, 463 KiB/s wr, 1.11k op/s; 9.8 KiB/s, 0 objects/s recovering 2024-09-14T05:05:39.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:39 smithi039 ceph-mon[111235]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-09-14T05:05:39.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:39 smithi039 ceph-mon[111235]: osdmap e1617: 8 total, 8 up, 7 in 2024-09-14T05:05:39.875 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:39.897 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:39.898 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:39.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:39.999 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:40.021 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:40.022 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:40.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:40.123 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0 2024-09-14T05:05:40.145 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:40.146 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:40.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.0' 2024-09-14T05:05:40.247 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:40.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:40.270 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:40.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:40.371 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:40.395 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:40.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:40.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:40.497 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:40.519 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:40.519 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:40.520 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:40.585 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:05:40.612 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:40 smithi039 ceph-mon[106470]: osdmap e1618: 8 total, 8 up, 7 in 2024-09-14T05:05:40.613 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:40 smithi039 ceph-mon[106470]: 1.6 scrub starts 2024-09-14T05:05:40.613 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:40 smithi039 ceph-mon[111235]: osdmap e1618: 8 total, 8 up, 7 in 2024-09-14T05:05:40.613 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:40 smithi039 ceph-mon[111235]: 1.6 scrub starts 2024-09-14T05:05:40.620 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:40.645 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:40.646 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:40.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:40.687 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:40 smithi129 ceph-mon[101316]: osdmap e1618: 8 total, 8 up, 7 in 2024-09-14T05:05:40.687 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:40 smithi129 ceph-mon[101316]: 1.6 scrub starts 2024-09-14T05:05:40.747 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:40.770 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:40.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:40.771 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:40.871 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:40.893 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:40.894 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:40.894 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:40.995 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5 2024-09-14T05:05:41.017 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:41.018 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:41.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.5' 2024-09-14T05:05:41.119 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:41.142 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:41.143 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:41.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:41.244 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:41.265 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:41.265 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:41.266 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:41.355 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is unset 2024-09-14T05:05:41.362 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1578 ms) 2024-09-14T05:05:41.362 INFO:tasks.workunit.client.0.smithi039.stdout:[ RUN ] TestClsRbd.sparsify 2024-09-14T05:05:41.367 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:41.399 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:41.399 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:41.400 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:41.420 INFO:tasks.workunit.client.0.smithi039.stdout:[ OK ] TestClsRbd.sparsify (58 ms) 2024-09-14T05:05:41.501 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6 2024-09-14T05:05:41.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:41 smithi129 ceph-mon[101316]: pgmap v3269: 135 pgs: 5 active+undersized+remapped, 4 peering, 4 creating+peering, 1 unknown, 121 active+clean; 608 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 758 KiB/s rd, 500 KiB/s wr, 1.25k op/s; 100 KiB/s, 0 objects/s recovering 2024-09-14T05:05:41.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:41 smithi129 ceph-mon[101316]: osdmap e1619: 8 total, 8 up, 7 in 2024-09-14T05:05:41.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:41 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/886686431' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:05:41.521 INFO:teuthology.orchestra.run.smithi039.stdout:osd.0: {} 2024-09-14T05:05:41.521 INFO:teuthology.orchestra.run.smithi039.stderr:osd.0: osd_enable_op_tracker = '' 2024-09-14T05:05:41.523 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:41.523 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:41.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.6' 2024-09-14T05:05:41.533 INFO:teuthology.orchestra.run.smithi039.stdout:osd.1: {} 2024-09-14T05:05:41.534 INFO:teuthology.orchestra.run.smithi039.stderr:osd.1: osd_enable_op_tracker = '' 2024-09-14T05:05:41.550 INFO:teuthology.orchestra.run.smithi039.stdout:osd.2: {} 2024-09-14T05:05:41.550 INFO:teuthology.orchestra.run.smithi039.stderr:osd.2: osd_enable_op_tracker = '' 2024-09-14T05:05:41.566 INFO:teuthology.orchestra.run.smithi039.stdout:osd.3: {} 2024-09-14T05:05:41.566 INFO:teuthology.orchestra.run.smithi039.stderr:osd.3: osd_enable_op_tracker = '' 2024-09-14T05:05:41.582 INFO:teuthology.orchestra.run.smithi039.stdout:osd.4: {} 2024-09-14T05:05:41.582 INFO:teuthology.orchestra.run.smithi039.stderr:osd.4: osd_enable_op_tracker = '' 2024-09-14T05:05:41.599 INFO:teuthology.orchestra.run.smithi039.stdout:osd.5: {} 2024-09-14T05:05:41.599 INFO:teuthology.orchestra.run.smithi039.stderr:osd.5: osd_enable_op_tracker = '' 2024-09-14T05:05:41.599 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:41 smithi039 ceph-mon[106470]: pgmap v3269: 135 pgs: 5 active+undersized+remapped, 4 peering, 4 creating+peering, 1 unknown, 121 active+clean; 608 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 758 KiB/s rd, 500 KiB/s wr, 1.25k op/s; 100 KiB/s, 0 objects/s recovering 2024-09-14T05:05:41.599 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:41 smithi039 ceph-mon[106470]: osdmap e1619: 8 total, 8 up, 7 in 2024-09-14T05:05:41.600 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:41 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/886686431' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:05:41.600 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:41 smithi039 ceph-mon[111235]: pgmap v3269: 135 pgs: 5 active+undersized+remapped, 4 peering, 4 creating+peering, 1 unknown, 121 active+clean; 608 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 758 KiB/s rd, 500 KiB/s wr, 1.25k op/s; 100 KiB/s, 0 objects/s recovering 2024-09-14T05:05:41.600 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:41 smithi039 ceph-mon[111235]: osdmap e1619: 8 total, 8 up, 7 in 2024-09-14T05:05:41.600 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:41 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/886686431' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:05:41.615 INFO:teuthology.orchestra.run.smithi039.stdout:osd.6: {} 2024-09-14T05:05:41.615 INFO:teuthology.orchestra.run.smithi039.stderr:osd.6: osd_enable_op_tracker = '' 2024-09-14T05:05:41.625 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:41.632 INFO:teuthology.orchestra.run.smithi039.stdout:osd.7: {} 2024-09-14T05:05:41.632 INFO:teuthology.orchestra.run.smithi039.stderr:osd.7: osd_enable_op_tracker = '' 2024-09-14T05:05:41.647 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:41.648 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:41.648 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:41.749 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:41.770 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:41.771 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:41.771 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:41.872 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:41.896 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:41.896 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:41.897 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:41.998 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2 2024-09-14T05:05:42.020 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:42.021 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:42.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.2' 2024-09-14T05:05:42.122 DEBUG:teuthology.orchestra.run.smithi039:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3 2024-09-14T05:05:42.145 INFO:teuthology.orchestra.run.smithi039.stderr:sudo: docker: command not found 2024-09-14T05:05:42.145 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:42.146 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi039 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.3' 2024-09-14T05:05:42.163 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7] out_osds: [4] dead_osds: [] live_osds: [0, 1, 2, 3, 5, 6, 7, 4] 2024-09-14T05:05:42.164 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2024-09-14T05:05:42.164 INFO:tasks.thrashosds.thrasher:Adding osd 4 2024-09-14T05:05:42.164 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd in 4 2024-09-14T05:05:42.247 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:42.272 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:42.272 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:42.272 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:42.362 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] 70 tests from TestClsRbd (5424 ms total) 2024-09-14T05:05:42.362 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T05:05:42.362 INFO:tasks.workunit.client.0.smithi039.stdout:[----------] Global test environment tear-down 2024-09-14T05:05:42.362 INFO:tasks.workunit.client.0.smithi039.stdout:[==========] 70 tests from 1 test suite ran. (8020 ms total) 2024-09-14T05:05:42.363 INFO:tasks.workunit.client.0.smithi039.stdout:[ PASSED ] 69 tests. 2024-09-14T05:05:42.363 INFO:tasks.workunit.client.0.smithi039.stdout:[ FAILED ] 1 test, listed below: 2024-09-14T05:05:42.363 INFO:tasks.workunit.client.0.smithi039.stdout:[ FAILED ] TestClsRbd.group_snap_set 2024-09-14T05:05:42.363 INFO:tasks.workunit.client.0.smithi039.stdout: 2024-09-14T05:05:42.363 INFO:tasks.workunit.client.0.smithi039.stdout: 1 FAILED TEST 2024-09-14T05:05:42.373 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7 2024-09-14T05:05:42.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:42.396 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2024-09-14T05:05:42.396 DEBUG:teuthology.orchestra.run.smithi039:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-09-14T05:05:42.397 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:42.397 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:42.397 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.7' 2024-09-14T05:05:42.498 DEBUG:teuthology.orchestra.run.smithi129:> sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4 2024-09-14T05:05:42.520 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:05:42.521 INFO:teuthology.orchestra.run.smithi129.stderr:sudo: docker: command not found 2024-09-14T05:05:42.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi129 with status 1: 'sudo docker kill -s 1 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd.4' 2024-09-14T05:05:42.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:42 smithi039 ceph-mon[111235]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:05:42.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:42 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/886686431' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T05:05:42.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:42 smithi039 ceph-mon[111235]: osdmap e1620: 8 total, 8 up, 7 in 2024-09-14T05:05:42.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:42 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/89133438' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:05:42.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:42 smithi039 ceph-mon[106470]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:05:42.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:42 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/886686431' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T05:05:42.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:42 smithi039 ceph-mon[106470]: osdmap e1620: 8 total, 8 up, 7 in 2024-09-14T05:05:42.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:42 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/89133438' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:05:42.621 INFO:tasks.thrashosds:joining thrashosds 2024-09-14T05:05:42.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:42 smithi129 ceph-mon[101316]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2024-09-14T05:05:42.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:42 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/886686431' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T05:05:42.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:42 smithi129 ceph-mon[101316]: osdmap e1620: 8 total, 8 up, 7 in 2024-09-14T05:05:42.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:42 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/89133438' entity='client.admin' cmd={"prefix": "osd ls"} : dispatch 2024-09-14T05:05:43.531 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:05:43.656 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:43 smithi039 ceph-mon[106470]: pgmap v3272: 135 pgs: 5 active+undersized+remapped, 130 active+clean; 609 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 2.6 MiB/s rd, 1.2 MiB/s wr, 584 op/s; 90 KiB/s, 0 objects/s recovering 2024-09-14T05:05:43.656 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:43 smithi039 ceph-mon[106470]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-14T05:05:43.656 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:43 smithi039 ceph-mon[106470]: osdmap e1621: 8 total, 8 up, 7 in 2024-09-14T05:05:43.656 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:43 smithi039 ceph-mon[111235]: pgmap v3272: 135 pgs: 5 active+undersized+remapped, 130 active+clean; 609 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 2.6 MiB/s rd, 1.2 MiB/s wr, 584 op/s; 90 KiB/s, 0 objects/s recovering 2024-09-14T05:05:43.656 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:43 smithi039 ceph-mon[111235]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-14T05:05:43.656 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:43 smithi039 ceph-mon[111235]: osdmap e1621: 8 total, 8 up, 7 in 2024-09-14T05:05:43.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:43 smithi129 ceph-mon[101316]: pgmap v3272: 135 pgs: 5 active+undersized+remapped, 130 active+clean; 609 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 2.6 MiB/s rd, 1.2 MiB/s wr, 584 op/s; 90 KiB/s, 0 objects/s recovering 2024-09-14T05:05:43.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:43 smithi129 ceph-mon[101316]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-09-14T05:05:43.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:43 smithi129 ceph-mon[101316]: osdmap e1621: 8 total, 8 up, 7 in 2024-09-14T05:05:44.681 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset noscrub 2024-09-14T05:05:45.249 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T05:05:45.250 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:05:45.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:45 smithi039 ceph-mon[106470]: 1.7 scrub starts 2024-09-14T05:05:45.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:45 smithi039 ceph-mon[106470]: 1.7 scrub ok 2024-09-14T05:05:45.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:45 smithi039 ceph-mon[106470]: pgmap v3274: 103 pgs: 103 active+clean; 608 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-09-14T05:05:45.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:45 smithi039 ceph-mon[111235]: 1.7 scrub starts 2024-09-14T05:05:45.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:45 smithi039 ceph-mon[111235]: 1.7 scrub ok 2024-09-14T05:05:45.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:45 smithi039 ceph-mon[111235]: pgmap v3274: 103 pgs: 103 active+clean; 608 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-09-14T05:05:45.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:45 smithi129 ceph-mon[101316]: 1.7 scrub starts 2024-09-14T05:05:45.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:45 smithi129 ceph-mon[101316]: 1.7 scrub ok 2024-09-14T05:05:45.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:45 smithi129 ceph-mon[101316]: pgmap v3274: 103 pgs: 103 active+clean; 608 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 75 KiB/s, 0 objects/s recovering 2024-09-14T05:05:46.053 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_blocked_ops 2024-09-14T05:05:46.601 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:05:47.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:47 smithi039 ceph-mon[106470]: pgmap v3275: 103 pgs: 103 active+clean; 608 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2024-09-14T05:05:47.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:47 smithi039 ceph-mon[111235]: pgmap v3275: 103 pgs: 103 active+clean; 608 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2024-09-14T05:05:47.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:47 smithi129 ceph-mon[101316]: pgmap v3275: 103 pgs: 103 active+clean; 608 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2024-09-14T05:05:47.875 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:05:47.998 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:47 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-a[106447]: 2024-09-14T05:05:47.667+0000 7f000264a640 -1 mon.a@0(leader).osd e1621 definitely_dead 0 2024-09-14T05:05:48.379 INFO:teuthology.orchestra.run.smithi039.stderr:marked in osd.4. 2024-09-14T05:05:48.648 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:48 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2134304615' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-09-14T05:05:48.648 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:48 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2134304615' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-09-14T05:05:48.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:48 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2134304615' entity='client.admin' cmd={"prefix": "osd in", "ids": ["4"]} : dispatch 2024-09-14T05:05:49.039 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:05:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:05:48] "GET /metrics HTTP/1.1" 200 36722 "" "Prometheus/2.51.0" 2024-09-14T05:05:49.741 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:49 smithi039 ceph-mon[106470]: pgmap v3276: 103 pgs: 103 active+clean; 595 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2024-09-14T05:05:49.741 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:49 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2134304615' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2024-09-14T05:05:49.741 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:49 smithi039 ceph-mon[106470]: osdmap e1622: 8 total, 8 up, 8 in 2024-09-14T05:05:49.741 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:49 smithi039 ceph-mon[111235]: pgmap v3276: 103 pgs: 103 active+clean; 595 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2024-09-14T05:05:49.741 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:49 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2134304615' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2024-09-14T05:05:49.742 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:49 smithi039 ceph-mon[111235]: osdmap e1622: 8 total, 8 up, 8 in 2024-09-14T05:05:49.763 INFO:tasks.thrashosds.thrasher:Added osd 4 2024-09-14T05:05:49.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:49 smithi129 ceph-mon[101316]: pgmap v3276: 103 pgs: 103 active+clean; 595 KiB data, 6.4 GiB used, 619 GiB / 626 GiB avail 2024-09-14T05:05:49.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:49 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2134304615' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2024-09-14T05:05:49.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:49 smithi129 ceph-mon[101316]: osdmap e1622: 8 total, 8 up, 8 in 2024-09-14T05:05:50.674 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:50 smithi039 ceph-mon[106470]: osdmap e1623: 8 total, 8 up, 8 in 2024-09-14T05:05:50.674 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:50 smithi039 ceph-mon[111235]: osdmap e1623: 8 total, 8 up, 8 in 2024-09-14T05:05:50.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:50 smithi129 ceph-mon[101316]: osdmap e1623: 8 total, 8 up, 8 in 2024-09-14T05:05:51.310 INFO:teuthology.orchestra.run.smithi039.stderr:noscrub is unset 2024-09-14T05:05:51.328 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T05:05:51.328 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:05:51.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:51 smithi039 ceph-mon[111235]: pgmap v3279: 103 pgs: 14 peering, 4 remapped+peering, 3 unknown, 82 active+clean; 595 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:05:51.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:51 smithi039 ceph-mon[111235]: Health check failed: Reduced data availability: 9 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2024-09-14T05:05:51.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:51 smithi039 ceph-mon[111235]: osdmap e1624: 8 total, 8 up, 8 in 2024-09-14T05:05:51.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:51 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1870354448' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:05:51.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:51 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1870354448' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T05:05:51.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:51 smithi039 ceph-mon[111235]: osdmap e1625: 8 total, 8 up, 8 in 2024-09-14T05:05:51.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:51 smithi039 ceph-mon[106470]: pgmap v3279: 103 pgs: 14 peering, 4 remapped+peering, 3 unknown, 82 active+clean; 595 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:05:51.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:51 smithi039 ceph-mon[106470]: Health check failed: Reduced data availability: 9 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2024-09-14T05:05:51.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:51 smithi039 ceph-mon[106470]: osdmap e1624: 8 total, 8 up, 8 in 2024-09-14T05:05:51.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:51 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1870354448' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:05:51.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:51 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1870354448' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T05:05:51.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:51 smithi039 ceph-mon[106470]: osdmap e1625: 8 total, 8 up, 8 in 2024-09-14T05:05:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:51 smithi129 ceph-mon[101316]: pgmap v3279: 103 pgs: 14 peering, 4 remapped+peering, 3 unknown, 82 active+clean; 595 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:05:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:51 smithi129 ceph-mon[101316]: Health check failed: Reduced data availability: 9 pgs inactive, 9 pgs peering (PG_AVAILABILITY) 2024-09-14T05:05:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:51 smithi129 ceph-mon[101316]: osdmap e1624: 8 total, 8 up, 8 in 2024-09-14T05:05:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:51 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1870354448' entity='client.admin' cmd={"prefix": "osd unset", "key": "noscrub"} : dispatch 2024-09-14T05:05:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:51 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1870354448' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2024-09-14T05:05:51.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:51 smithi129 ceph-mon[101316]: osdmap e1625: 8 total, 8 up, 8 in 2024-09-14T05:05:52.603 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd unset nodeep-scrub 2024-09-14T05:05:53.068 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.2 dump_historic_ops 2024-09-14T05:05:53.155 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:05:53.585 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:53 smithi039 ceph-mon[106470]: pgmap v3282: 103 pgs: 2 active+undersized+remapped, 1 active+remapped, 14 peering, 6 remapped+peering, 1 unknown, 79 active+clean; 634 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 14 KiB/s rd, 41 KiB/s wr, 9 op/s; 28 KiB/s, 0 objects/s recovering 2024-09-14T05:05:53.585 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:53 smithi039 ceph-mon[106470]: osdmap e1626: 8 total, 8 up, 8 in 2024-09-14T05:05:53.585 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:53 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:05:53.585 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:53 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:05:53.585 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:53 smithi039 ceph-mon[106470]: 1.6 scrub starts 2024-09-14T05:05:53.586 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:53 smithi039 ceph-mon[106470]: 1.6 scrub ok 2024-09-14T05:05:53.586 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:53 smithi039 ceph-mon[111235]: pgmap v3282: 103 pgs: 2 active+undersized+remapped, 1 active+remapped, 14 peering, 6 remapped+peering, 1 unknown, 79 active+clean; 634 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 14 KiB/s rd, 41 KiB/s wr, 9 op/s; 28 KiB/s, 0 objects/s recovering 2024-09-14T05:05:53.586 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:53 smithi039 ceph-mon[111235]: osdmap e1626: 8 total, 8 up, 8 in 2024-09-14T05:05:53.586 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:53 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:05:53.586 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:53 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:05:53.586 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:53 smithi039 ceph-mon[111235]: 1.6 scrub starts 2024-09-14T05:05:53.586 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:53 smithi039 ceph-mon[111235]: 1.6 scrub ok 2024-09-14T05:05:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:53 smithi129 ceph-mon[101316]: pgmap v3282: 103 pgs: 2 active+undersized+remapped, 1 active+remapped, 14 peering, 6 remapped+peering, 1 unknown, 79 active+clean; 634 KiB data, 7.0 GiB used, 708 GiB / 715 GiB avail; 14 KiB/s rd, 41 KiB/s wr, 9 op/s; 28 KiB/s, 0 objects/s recovering 2024-09-14T05:05:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:53 smithi129 ceph-mon[101316]: osdmap e1626: 8 total, 8 up, 8 in 2024-09-14T05:05:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:53 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:05:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:53 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:05:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:53 smithi129 ceph-mon[101316]: 1.6 scrub starts 2024-09-14T05:05:53.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:53 smithi129 ceph-mon[101316]: 1.6 scrub ok 2024-09-14T05:05:53.938 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:05:54.726 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:54 smithi039 ceph-mon[106470]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 10 pgs inactive, 10 pgs peering) 2024-09-14T05:05:54.726 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:54 smithi039 ceph-mon[111235]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 10 pgs inactive, 10 pgs peering) 2024-09-14T05:05:54.764 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2024-09-14T05:05:54.765 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd pool set .mgr pgp_num 31 2024-09-14T05:05:54.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:54 smithi129 ceph-mon[101316]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 10 pgs inactive, 10 pgs peering) 2024-09-14T05:05:55.655 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:55 smithi039 ceph-mon[111235]: pgmap v3284: 103 pgs: 2 active+undersized+remapped, 1 remapped+peering, 100 active+clean; 634 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 12 KiB/s rd, 35 KiB/s wr, 8 op/s; 94 KiB/s, 2 keys/s, 3 objects/s recovering 2024-09-14T05:05:55.656 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:55 smithi039 ceph-mon[106470]: pgmap v3284: 103 pgs: 2 active+undersized+remapped, 1 remapped+peering, 100 active+clean; 634 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 12 KiB/s rd, 35 KiB/s wr, 8 op/s; 94 KiB/s, 2 keys/s, 3 objects/s recovering 2024-09-14T05:05:55.695 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:05:55.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:55 smithi129 ceph-mon[101316]: pgmap v3284: 103 pgs: 2 active+undersized+remapped, 1 remapped+peering, 100 active+clean; 634 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 12 KiB/s rd, 35 KiB/s wr, 8 op/s; 94 KiB/s, 2 keys/s, 3 objects/s recovering 2024-09-14T05:05:56.392 INFO:teuthology.orchestra.run.smithi039.stderr:nodeep-scrub is unset 2024-09-14T05:05:56.502 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:56 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2762275901' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:05:56.503 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:56 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2762275901' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:05:56.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:56 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2762275901' entity='client.admin' cmd={"prefix": "osd unset", "key": "nodeep-scrub"} : dispatch 2024-09-14T05:05:57.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:57 smithi129 ceph-mon[101316]: pgmap v3285: 103 pgs: 2 active+undersized+remapped, 101 active+clean; 634 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 9.3 KiB/s rd, 27 KiB/s wr, 6 op/s; 74 KiB/s, 1 keys/s, 2 objects/s recovering 2024-09-14T05:05:57.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:57 smithi129 ceph-mon[101316]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T05:05:57.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:57 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T05:05:57.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:57 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2762275901' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T05:05:57.773 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:57 smithi129 ceph-mon[101316]: osdmap e1627: 8 total, 8 up, 8 in 2024-09-14T05:05:57.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:57 smithi039 ceph-mon[106470]: pgmap v3285: 103 pgs: 2 active+undersized+remapped, 101 active+clean; 634 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 9.3 KiB/s rd, 27 KiB/s wr, 6 op/s; 74 KiB/s, 1 keys/s, 2 objects/s recovering 2024-09-14T05:05:57.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:57 smithi039 ceph-mon[106470]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T05:05:57.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:57 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T05:05:57.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:57 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2762275901' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T05:05:57.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:57 smithi039 ceph-mon[106470]: osdmap e1627: 8 total, 8 up, 8 in 2024-09-14T05:05:57.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:57 smithi039 ceph-mon[111235]: pgmap v3285: 103 pgs: 2 active+undersized+remapped, 101 active+clean; 634 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 9.3 KiB/s rd, 27 KiB/s wr, 6 op/s; 74 KiB/s, 1 keys/s, 2 objects/s recovering 2024-09-14T05:05:57.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:57 smithi039 ceph-mon[111235]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2024-09-14T05:05:57.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:57 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T05:05:57.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:57 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2762275901' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2024-09-14T05:05:57.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:57 smithi039 ceph-mon[111235]: osdmap e1627: 8 total, 8 up, 8 in 2024-09-14T05:05:58.955 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:05:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:05:58] "GET /metrics HTTP/1.1" 200 34358 "" "Prometheus/2.51.0" 2024-09-14T05:05:59.055 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T05:05:59.055 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:05:59.405 INFO:teuthology.orchestra.run.smithi039.stderr:set pool 1 pgp_num to 31 2024-09-14T05:05:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:59 smithi129 ceph-mon[101316]: pgmap v3287: 103 pgs: 103 active+clean; 621 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 42 KiB/s, 1 keys/s, 1 objects/s recovering 2024-09-14T05:05:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:59 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/616219063' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-14T05:05:59.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:05:59 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-14T05:05:59.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:59 smithi039 ceph-mon[106470]: pgmap v3287: 103 pgs: 103 active+clean; 621 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 42 KiB/s, 1 keys/s, 1 objects/s recovering 2024-09-14T05:05:59.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:59 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/616219063' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-14T05:05:59.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:05:59 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-14T05:05:59.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:59 smithi039 ceph-mon[111235]: pgmap v3287: 103 pgs: 103 active+clean; 621 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 42 KiB/s, 1 keys/s, 1 objects/s recovering 2024-09-14T05:05:59.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:59 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/616219063' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-14T05:05:59.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:05:59 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"} : dispatch 2024-09-14T05:05:59.850 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_ops_in_flight 2024-09-14T05:06:00.499 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2024-09-14T05:06:00.512 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:06:00.674 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:00 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2024-09-14T05:06:00.674 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:00 smithi039 ceph-mon[106470]: osdmap e1628: 8 total, 8 up, 8 in 2024-09-14T05:06:00.675 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:00 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2024-09-14T05:06:00.675 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:00 smithi039 ceph-mon[111235]: osdmap e1628: 8 total, 8 up, 8 in 2024-09-14T05:06:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:00 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2024-09-14T05:06:00.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:00 smithi129 ceph-mon[101316]: osdmap e1628: 8 total, 8 up, 8 in 2024-09-14T05:06:01.256 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:06:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:01 smithi129 ceph-mon[101316]: pgmap v3289: 103 pgs: 103 active+clean; 621 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 37 KiB/s, 1 keys/s, 1 objects/s recovering 2024-09-14T05:06:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:01 smithi129 ceph-mon[101316]: 1.e scrub starts 2024-09-14T05:06:01.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:01 smithi129 ceph-mon[101316]: 1.e scrub ok 2024-09-14T05:06:01.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:01 smithi039 ceph-mon[106470]: pgmap v3289: 103 pgs: 103 active+clean; 621 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 37 KiB/s, 1 keys/s, 1 objects/s recovering 2024-09-14T05:06:01.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:01 smithi039 ceph-mon[106470]: 1.e scrub starts 2024-09-14T05:06:01.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:01 smithi039 ceph-mon[106470]: 1.e scrub ok 2024-09-14T05:06:01.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:01 smithi039 ceph-mon[111235]: pgmap v3289: 103 pgs: 103 active+clean; 621 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 37 KiB/s, 1 keys/s, 1 objects/s recovering 2024-09-14T05:06:01.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:01 smithi039 ceph-mon[111235]: 1.e scrub starts 2024-09-14T05:06:01.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:01 smithi039 ceph-mon[111235]: 1.e scrub ok 2024-09-14T05:06:02.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:02 smithi129 ceph-mon[101316]: 1.1a scrub starts 2024-09-14T05:06:02.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:02 smithi129 ceph-mon[101316]: 1.1a scrub ok 2024-09-14T05:06:02.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:02 smithi039 ceph-mon[106470]: 1.1a scrub starts 2024-09-14T05:06:02.792 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:02 smithi039 ceph-mon[106470]: 1.1a scrub ok 2024-09-14T05:06:02.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:02 smithi039 ceph-mon[111235]: 1.1a scrub starts 2024-09-14T05:06:02.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:02 smithi039 ceph-mon[111235]: 1.1a scrub ok 2024-09-14T05:06:03.067 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T05:06:03.067 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:06:03.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:03 smithi129 ceph-mon[101316]: pgmap v3290: 103 pgs: 103 active+clean; 621 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:03.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:03 smithi039 ceph-mon[106470]: pgmap v3290: 103 pgs: 103 active+clean; 621 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:03.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:03 smithi039 ceph-mon[111235]: pgmap v3290: 103 pgs: 103 active+clean; 621 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:03.858 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_blocked_ops 2024-09-14T05:06:04.873 INFO:teuthology.orchestra.run.smithi039.stdout:mon.a: {} 2024-09-14T05:06:04.873 INFO:teuthology.orchestra.run.smithi039.stderr:mon.a: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-14T05:06:04.883 INFO:teuthology.orchestra.run.smithi039.stdout:mon.b: {} 2024-09-14T05:06:04.884 INFO:teuthology.orchestra.run.smithi039.stderr:mon.b: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-14T05:06:04.898 INFO:teuthology.orchestra.run.smithi039.stdout:mon.c: {} 2024-09-14T05:06:04.898 INFO:teuthology.orchestra.run.smithi039.stderr:mon.c: mon_osd_down_out_interval = '' (not observed, change may require restart) 2024-09-14T05:06:05.631 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 0 1 2024-09-14T05:06:05.645 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:06:05.768 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:05 smithi039 ceph-mon[106470]: pgmap v3291: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:05.768 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:05 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/197688694' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-14T05:06:05.768 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:05 smithi039 ceph-mon[111235]: pgmap v3291: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:05.769 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:05 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/197688694' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-14T05:06:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:05 smithi129 ceph-mon[101316]: pgmap v3291: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:05.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:05 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/197688694' entity='client.admin' cmd={"prefix": "mon dump", "format": "json"} : dispatch 2024-09-14T05:06:07.147 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:06:07.717 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:07 smithi039 ceph-mon[106470]: pgmap v3292: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:07.717 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:07 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:06:07.717 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:07 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:06:07.717 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:07 smithi039 ceph-mon[111235]: pgmap v3292: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:07.717 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:07 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:06:07.717 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:07 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:06:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:07 smithi129 ceph-mon[101316]: pgmap v3292: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:07 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:06:07.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:07 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:06:08.980 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:06:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:06:08] "GET /metrics HTTP/1.1" 200 34333 "" "Prometheus/2.51.0" 2024-09-14T05:06:09.203 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T05:06:09.203 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:06:09.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:09 smithi129 ceph-mon[101316]: pgmap v3293: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:09.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:09 smithi039 ceph-mon[106470]: pgmap v3293: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:09.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:09 smithi039 ceph-mon[111235]: pgmap v3293: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:09.996 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.3 dump_historic_ops 2024-09-14T05:06:10.340 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:06:10.511 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.0 to 1 (10000) 2024-09-14T05:06:10.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:10 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2008526354' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T05:06:10.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:10 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2008526354' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T05:06:10.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:10 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2008526354' entity='client.admin' cmd={"prefix": "osd reweight", "id": 0, "weight": 1.0} : dispatch 2024-09-14T05:06:11.402 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 0 1 2024-09-14T05:06:11.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:11 smithi039 ceph-mon[106470]: pgmap v3294: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:11.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:11 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2008526354' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-14T05:06:11.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:11 smithi039 ceph-mon[106470]: osdmap e1629: 8 total, 8 up, 8 in 2024-09-14T05:06:11.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:11 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.5", "id": [5, 1]} : dispatch 2024-09-14T05:06:11.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:11 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.11", "id": [5, 2]} : dispatch 2024-09-14T05:06:11.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:11 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.5", "id": [5, 1]} : dispatch 2024-09-14T05:06:11.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:11 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]} : dispatch 2024-09-14T05:06:11.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:11 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.11", "id": [5, 2]} : dispatch 2024-09-14T05:06:11.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:11 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]} : dispatch 2024-09-14T05:06:11.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:11 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]} : dispatch 2024-09-14T05:06:11.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:11 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 7]} : dispatch 2024-09-14T05:06:11.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:11 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]} : dispatch 2024-09-14T05:06:11.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:11 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 7]} : dispatch 2024-09-14T05:06:11.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:11 smithi039 ceph-mon[111235]: pgmap v3294: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:11.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:11 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2008526354' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-14T05:06:11.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:11 smithi039 ceph-mon[111235]: osdmap e1629: 8 total, 8 up, 8 in 2024-09-14T05:06:11.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:11 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.5", "id": [5, 1]} : dispatch 2024-09-14T05:06:11.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:11 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.11", "id": [5, 2]} : dispatch 2024-09-14T05:06:11.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:11 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.5", "id": [5, 1]} : dispatch 2024-09-14T05:06:11.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:11 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]} : dispatch 2024-09-14T05:06:11.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:11 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.11", "id": [5, 2]} : dispatch 2024-09-14T05:06:11.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:11 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]} : dispatch 2024-09-14T05:06:11.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:11 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]} : dispatch 2024-09-14T05:06:11.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:11 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 7]} : dispatch 2024-09-14T05:06:11.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:11 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]} : dispatch 2024-09-14T05:06:11.792 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:11 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 7]} : dispatch 2024-09-14T05:06:12.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:11 smithi129 ceph-mon[101316]: pgmap v3294: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:12.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:11 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2008526354' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2024-09-14T05:06:12.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:11 smithi129 ceph-mon[101316]: osdmap e1629: 8 total, 8 up, 8 in 2024-09-14T05:06:12.022 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:11 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.5", "id": [5, 1]} : dispatch 2024-09-14T05:06:12.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:11 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.11", "id": [5, 2]} : dispatch 2024-09-14T05:06:12.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:11 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.5", "id": [5, 1]} : dispatch 2024-09-14T05:06:12.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:11 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]} : dispatch 2024-09-14T05:06:12.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:11 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.11", "id": [5, 2]} : dispatch 2024-09-14T05:06:12.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:11 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]} : dispatch 2024-09-14T05:06:12.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:11 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]} : dispatch 2024-09-14T05:06:12.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:11 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 7]} : dispatch 2024-09-14T05:06:12.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:11 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]} : dispatch 2024-09-14T05:06:12.023 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:11 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd={"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 7]} : dispatch 2024-09-14T05:06:12.717 INFO:teuthology.orchestra.run.smithi039.stdout:ERROR: (22) Invalid argument 2024-09-14T05:06:12.717 INFO:teuthology.orchestra.run.smithi039.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:06:12.724 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:06:12.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:12 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.5", "id": [5, 1]}]': finished 2024-09-14T05:06:12.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:12 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.11", "id": [5, 2]}]': finished 2024-09-14T05:06:12.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:12 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]}]': finished 2024-09-14T05:06:12.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:12 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]': finished 2024-09-14T05:06:12.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:12 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 7]}]': finished 2024-09-14T05:06:12.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:12 smithi039 ceph-mon[106470]: osdmap e1630: 8 total, 8 up, 8 in 2024-09-14T05:06:12.791 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:12 smithi039 ceph-mon[106470]: osdmap e1631: 8 total, 8 up, 8 in 2024-09-14T05:06:12.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:12 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.5", "id": [5, 1]}]': finished 2024-09-14T05:06:12.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:12 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.11", "id": [5, 2]}]': finished 2024-09-14T05:06:12.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:12 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]}]': finished 2024-09-14T05:06:12.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:12 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]': finished 2024-09-14T05:06:12.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:12 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 7]}]': finished 2024-09-14T05:06:12.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:12 smithi039 ceph-mon[111235]: osdmap e1630: 8 total, 8 up, 8 in 2024-09-14T05:06:12.791 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:12 smithi039 ceph-mon[111235]: osdmap e1631: 8 total, 8 up, 8 in 2024-09-14T05:06:13.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:12 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.5", "id": [5, 1]}]': finished 2024-09-14T05:06:13.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:12 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.11", "id": [5, 2]}]': finished 2024-09-14T05:06:13.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:12 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "3.6", "id": [4, 2]}]': finished 2024-09-14T05:06:13.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:12 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.0", "id": [3, 2]}]': finished 2024-09-14T05:06:13.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:12 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 7]}]': finished 2024-09-14T05:06:13.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:12 smithi129 ceph-mon[101316]: osdmap e1630: 8 total, 8 up, 8 in 2024-09-14T05:06:13.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:12 smithi129 ceph-mon[101316]: osdmap e1631: 8 total, 8 up, 8 in 2024-09-14T05:06:13.500 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_ops_in_flight 2024-09-14T05:06:13.718 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:06:13.773 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:13 smithi039 ceph-mon[106470]: pgmap v3297: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:13.774 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:13 smithi039 ceph-mon[106470]: osdmap e1632: 8 total, 8 up, 8 in 2024-09-14T05:06:13.774 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:13 smithi039 ceph-mon[111235]: pgmap v3297: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:13.774 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:13 smithi039 ceph-mon[111235]: osdmap e1632: 8 total, 8 up, 8 in 2024-09-14T05:06:13.839 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:13 smithi129 ceph-mon[101316]: pgmap v3297: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:13.839 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:13 smithi129 ceph-mon[101316]: osdmap e1632: 8 total, 8 up, 8 in 2024-09-14T05:06:15.607 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T05:06:15.607 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:06:15.608 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:15 smithi129 ceph-mon[101316]: pgmap v3300: 103 pgs: 1 activating+undersized+remapped, 2 peering, 100 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:15.608 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:15 smithi129 ceph-mon[101316]: osdmap e1633: 8 total, 8 up, 8 in 2024-09-14T05:06:15.789 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:15 smithi039 ceph-mon[106470]: pgmap v3300: 103 pgs: 1 activating+undersized+remapped, 2 peering, 100 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:15.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:15 smithi039 ceph-mon[106470]: osdmap e1633: 8 total, 8 up, 8 in 2024-09-14T05:06:15.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:15 smithi039 ceph-mon[111235]: pgmap v3300: 103 pgs: 1 activating+undersized+remapped, 2 peering, 100 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:15.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:15 smithi039 ceph-mon[111235]: osdmap e1633: 8 total, 8 up, 8 in 2024-09-14T05:06:16.278 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_blocked_ops 2024-09-14T05:06:16.482 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:06:16.561 INFO:teuthology.orchestra.run.smithi039.stderr:set osd.0 primary-affinity to 1 (8655362) 2024-09-14T05:06:16.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:16 smithi039 ceph-mon[106470]: osdmap e1634: 8 total, 8 up, 8 in 2024-09-14T05:06:16.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:16 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/970985659' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-14T05:06:16.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:16 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-14T05:06:16.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:16 smithi039 ceph-mon[111235]: osdmap e1634: 8 total, 8 up, 8 in 2024-09-14T05:06:16.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:16 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/970985659' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-14T05:06:16.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:16 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-14T05:06:16.909 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:16 smithi129 ceph-mon[101316]: osdmap e1634: 8 total, 8 up, 8 in 2024-09-14T05:06:16.909 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:16 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/970985659' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-14T05:06:16.909 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:16 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 0, "weight": 1.0} : dispatch 2024-09-14T05:06:17.702 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 1 1 2024-09-14T05:06:17.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:17 smithi129 ceph-mon[101316]: pgmap v3303: 103 pgs: 1 activating+undersized+remapped, 2 peering, 100 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:17.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:17 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-14T05:06:17.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:17 smithi129 ceph-mon[101316]: osdmap e1635: 8 total, 8 up, 8 in 2024-09-14T05:06:17.919 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:06:18.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:17 smithi039 ceph-mon[106470]: pgmap v3303: 103 pgs: 1 activating+undersized+remapped, 2 peering, 100 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:18.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:17 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-14T05:06:18.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:17 smithi039 ceph-mon[106470]: osdmap e1635: 8 total, 8 up, 8 in 2024-09-14T05:06:18.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:17 smithi039 ceph-mon[111235]: pgmap v3303: 103 pgs: 1 activating+undersized+remapped, 2 peering, 100 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:18.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:17 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2024-09-14T05:06:18.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:17 smithi039 ceph-mon[111235]: osdmap e1635: 8 total, 8 up, 8 in 2024-09-14T05:06:18.483 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T05:06:18.484 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:06:19.040 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:06:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:06:18] "GET /metrics HTTP/1.1" 200 34333 "" "Prometheus/2.51.0" 2024-09-14T05:06:19.168 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.5 dump_historic_ops 2024-09-14T05:06:19.365 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:06:19.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:19 smithi129 ceph-mon[101316]: pgmap v3305: 103 pgs: 1 activating+undersized+remapped, 102 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T05:06:20.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:19 smithi039 ceph-mon[106470]: pgmap v3305: 103 pgs: 1 activating+undersized+remapped, 102 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T05:06:20.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:19 smithi039 ceph-mon[111235]: pgmap v3305: 103 pgs: 1 activating+undersized+remapped, 102 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T05:06:20.581 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.1 to 1 (10000) 2024-09-14T05:06:20.834 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:20 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4047697550' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T05:06:20.834 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:20 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T05:06:20.969 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:20 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4047697550' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T05:06:20.969 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:20 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T05:06:20.969 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:20 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4047697550' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T05:06:20.969 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:20 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 1, "weight": 1.0} : dispatch 2024-09-14T05:06:21.080 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T05:06:21.081 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:06:21.355 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 1 1 2024-09-14T05:06:21.573 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:06:21.727 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:21 smithi129 ceph-mon[101316]: pgmap v3306: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T05:06:21.727 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:21 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-14T05:06:21.727 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:21 smithi129 ceph-mon[101316]: osdmap e1636: 8 total, 8 up, 8 in 2024-09-14T05:06:21.749 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_ops_in_flight 2024-09-14T05:06:21.913 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:21 smithi039 ceph-mon[106470]: pgmap v3306: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T05:06:21.913 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:21 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-14T05:06:21.913 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:21 smithi039 ceph-mon[106470]: osdmap e1636: 8 total, 8 up, 8 in 2024-09-14T05:06:21.913 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:21 smithi039 ceph-mon[111235]: pgmap v3306: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T05:06:21.913 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:21 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2024-09-14T05:06:21.913 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:21 smithi039 ceph-mon[111235]: osdmap e1636: 8 total, 8 up, 8 in 2024-09-14T05:06:21.944 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:06:22.941 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:06:23.038 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:06:23.038 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:06:23.494 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T05:06:23.494 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:06:23.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:23 smithi129 ceph-mon[101316]: pgmap v3308: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T05:06:24.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:23 smithi039 ceph-mon[106470]: pgmap v3308: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T05:06:24.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:23 smithi039 ceph-mon[111235]: pgmap v3308: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T05:06:24.113 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_blocked_ops 2024-09-14T05:06:24.310 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:06:24.649 INFO:teuthology.orchestra.run.smithi039.stderr:set osd.1 primary-affinity to 1 (8655362) 2024-09-14T05:06:24.710 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:24 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1483977163' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-14T05:06:24.710 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:24 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-14T05:06:25.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:24 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1483977163' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-14T05:06:25.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:24 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-14T05:06:25.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:24 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1483977163' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-14T05:06:25.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:24 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 1, "weight": 1.0} : dispatch 2024-09-14T05:06:25.482 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 2 1 2024-09-14T05:06:25.704 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:06:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:25 smithi039 ceph-mon[106470]: pgmap v3309: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T05:06:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:25 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-09-14T05:06:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:25 smithi039 ceph-mon[106470]: osdmap e1637: 8 total, 8 up, 8 in 2024-09-14T05:06:25.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:25 smithi039 ceph-mon[111235]: pgmap v3309: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T05:06:25.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:25 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-09-14T05:06:25.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:25 smithi039 ceph-mon[111235]: osdmap e1637: 8 total, 8 up, 8 in 2024-09-14T05:06:25.908 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:25 smithi129 ceph-mon[101316]: pgmap v3309: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T05:06:25.909 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:25 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2024-09-14T05:06:25.909 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:25 smithi129 ceph-mon[101316]: osdmap e1637: 8 total, 8 up, 8 in 2024-09-14T05:06:25.909 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T05:06:25.909 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:06:26.560 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.6 dump_historic_ops 2024-09-14T05:06:26.762 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:06:26.844 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:26 smithi039 ceph-mon[106470]: osdmap e1638: 8 total, 8 up, 8 in 2024-09-14T05:06:26.844 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:26 smithi039 ceph-mon[106470]: 1.3 scrub starts 2024-09-14T05:06:26.844 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:26 smithi039 ceph-mon[106470]: 1.3 scrub ok 2024-09-14T05:06:26.845 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:26 smithi039 ceph-mon[111235]: osdmap e1638: 8 total, 8 up, 8 in 2024-09-14T05:06:26.845 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:26 smithi039 ceph-mon[111235]: 1.3 scrub starts 2024-09-14T05:06:26.845 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:26 smithi039 ceph-mon[111235]: 1.3 scrub ok 2024-09-14T05:06:26.883 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:26 smithi129 ceph-mon[101316]: osdmap e1638: 8 total, 8 up, 8 in 2024-09-14T05:06:26.884 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:26 smithi129 ceph-mon[101316]: 1.3 scrub starts 2024-09-14T05:06:26.884 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:26 smithi129 ceph-mon[101316]: 1.3 scrub ok 2024-09-14T05:06:27.668 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.2 to 1 (10000) 2024-09-14T05:06:27.911 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:27 smithi129 ceph-mon[101316]: pgmap v3312: 103 pgs: 8 peering, 95 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:27.912 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:27 smithi129 ceph-mon[101316]: Health check failed: Reduced data availability: 4 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2024-09-14T05:06:27.912 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:27 smithi129 ceph-mon[101316]: 1.13 scrub starts 2024-09-14T05:06:27.912 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:27 smithi129 ceph-mon[101316]: 1.13 scrub ok 2024-09-14T05:06:27.912 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:27 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4135923490' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T05:06:27.912 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:27 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T05:06:27.993 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:27 smithi039 ceph-mon[106470]: pgmap v3312: 103 pgs: 8 peering, 95 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:27.993 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:27 smithi039 ceph-mon[106470]: Health check failed: Reduced data availability: 4 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2024-09-14T05:06:27.993 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:27 smithi039 ceph-mon[106470]: 1.13 scrub starts 2024-09-14T05:06:27.994 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:27 smithi039 ceph-mon[106470]: 1.13 scrub ok 2024-09-14T05:06:27.994 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:27 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4135923490' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T05:06:27.994 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:27 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T05:06:27.994 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:27 smithi039 ceph-mon[111235]: pgmap v3312: 103 pgs: 8 peering, 95 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:27.994 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:27 smithi039 ceph-mon[111235]: Health check failed: Reduced data availability: 4 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2024-09-14T05:06:27.994 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:27 smithi039 ceph-mon[111235]: 1.13 scrub starts 2024-09-14T05:06:27.994 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:27 smithi039 ceph-mon[111235]: 1.13 scrub ok 2024-09-14T05:06:27.994 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:27 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4135923490' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T05:06:27.994 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:27 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 2, "weight": 1.0} : dispatch 2024-09-14T05:06:28.351 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 2 1 2024-09-14T05:06:28.457 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T05:06:28.457 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:06:28.562 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:06:28.904 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:06:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:06:28] "GET /metrics HTTP/1.1" 200 34336 "" "Prometheus/2.51.0" 2024-09-14T05:06:28.904 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:28 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-14T05:06:28.904 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:28 smithi039 ceph-mon[106470]: osdmap e1639: 8 total, 8 up, 8 in 2024-09-14T05:06:28.905 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:28 smithi039 ceph-mon[106470]: 1.b scrub starts 2024-09-14T05:06:28.905 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:28 smithi039 ceph-mon[106470]: 1.b scrub ok 2024-09-14T05:06:28.905 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:28 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-14T05:06:28.905 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:28 smithi039 ceph-mon[111235]: osdmap e1639: 8 total, 8 up, 8 in 2024-09-14T05:06:28.905 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:28 smithi039 ceph-mon[111235]: 1.b scrub starts 2024-09-14T05:06:28.905 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:28 smithi039 ceph-mon[111235]: 1.b scrub ok 2024-09-14T05:06:29.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:28 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2024-09-14T05:06:29.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:28 smithi129 ceph-mon[101316]: osdmap e1639: 8 total, 8 up, 8 in 2024-09-14T05:06:29.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:28 smithi129 ceph-mon[101316]: 1.b scrub starts 2024-09-14T05:06:29.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:28 smithi129 ceph-mon[101316]: 1.b scrub ok 2024-09-14T05:06:29.127 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_ops_in_flight 2024-09-14T05:06:29.325 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:06:29.717 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:29 smithi039 ceph-mon[106470]: pgmap v3314: 103 pgs: 8 peering, 95 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:29.717 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:29 smithi039 ceph-mon[111235]: pgmap v3314: 103 pgs: 8 peering, 95 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:30.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:29 smithi129 ceph-mon[101316]: pgmap v3314: 103 pgs: 8 peering, 95 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:30.702 INFO:teuthology.orchestra.run.smithi039.stderr:set osd.2 primary-affinity to 1 (8655362) 2024-09-14T05:06:30.949 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T05:06:30.949 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:06:30.949 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:30 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/742017172' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-14T05:06:31.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:30 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/742017172' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-14T05:06:31.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:30 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/742017172' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 2, "weight": 1.0} : dispatch 2024-09-14T05:06:31.461 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 3 1 2024-09-14T05:06:31.675 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:06:31.815 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_blocked_ops 2024-09-14T05:06:31.962 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:31 smithi039 ceph-mon[106470]: pgmap v3315: 103 pgs: 8 peering, 95 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:31.962 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:31 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/742017172' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-14T05:06:31.962 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:31 smithi039 ceph-mon[106470]: osdmap e1640: 8 total, 8 up, 8 in 2024-09-14T05:06:31.962 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:31 smithi039 ceph-mon[111235]: pgmap v3315: 103 pgs: 8 peering, 95 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:31.962 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:31 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/742017172' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-14T05:06:31.962 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:31 smithi039 ceph-mon[111235]: osdmap e1640: 8 total, 8 up, 8 in 2024-09-14T05:06:32.016 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:06:32.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:31 smithi129 ceph-mon[101316]: pgmap v3315: 103 pgs: 8 peering, 95 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:32.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:31 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/742017172' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2024-09-14T05:06:32.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:31 smithi129 ceph-mon[101316]: osdmap e1640: 8 total, 8 up, 8 in 2024-09-14T05:06:33.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:32 smithi129 ceph-mon[101316]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 7 pgs peering) 2024-09-14T05:06:33.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:32 smithi129 ceph-mon[101316]: Cluster is now healthy 2024-09-14T05:06:33.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:32 smithi039 ceph-mon[106470]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 7 pgs peering) 2024-09-14T05:06:33.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:32 smithi039 ceph-mon[106470]: Cluster is now healthy 2024-09-14T05:06:33.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:32 smithi039 ceph-mon[111235]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 7 pgs peering) 2024-09-14T05:06:33.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:32 smithi039 ceph-mon[111235]: Cluster is now healthy 2024-09-14T05:06:33.727 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T05:06:33.728 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:06:33.767 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.3 to 1 (10000) 2024-09-14T05:06:34.015 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:33 smithi129 ceph-mon[101316]: pgmap v3317: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:34.016 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:33 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1977973072' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T05:06:34.016 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:33 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T05:06:34.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:33 smithi039 ceph-mon[106470]: pgmap v3317: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:34.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:33 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1977973072' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T05:06:34.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:33 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T05:06:34.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:33 smithi039 ceph-mon[111235]: pgmap v3317: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:34.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:33 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1977973072' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T05:06:34.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:33 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 3, "weight": 1.0} : dispatch 2024-09-14T05:06:34.323 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.7 dump_historic_ops 2024-09-14T05:06:34.545 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 3 1 2024-09-14T05:06:34.785 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:06:35.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:34 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-14T05:06:35.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:34 smithi129 ceph-mon[101316]: osdmap e1641: 8 total, 8 up, 8 in 2024-09-14T05:06:35.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:34 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T05:06:35.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:34 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-14T05:06:35.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:34 smithi039 ceph-mon[106470]: osdmap e1641: 8 total, 8 up, 8 in 2024-09-14T05:06:35.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:34 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T05:06:35.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:34 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2024-09-14T05:06:35.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:34 smithi039 ceph-mon[111235]: osdmap e1641: 8 total, 8 up, 8 in 2024-09-14T05:06:35.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:34 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T05:06:35.174 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:06:35.990 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:35 smithi129 ceph-mon[101316]: pgmap v3319: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:36.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:35 smithi039 ceph-mon[106470]: pgmap v3319: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:36.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:35 smithi039 ceph-mon[111235]: pgmap v3319: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:37.594 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T05:06:37.594 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:06:37.801 INFO:teuthology.orchestra.run.smithi039.stderr:set osd.3 primary-affinity to 1 (8655362) 2024-09-14T05:06:38.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:37 smithi039 ceph-mon[111235]: pgmap v3320: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:38.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:06:38.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:37 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2282028074' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-14T05:06:38.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:37 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-14T05:06:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:37 smithi039 ceph-mon[106470]: pgmap v3320: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:38.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:06:38.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:37 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2282028074' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-14T05:06:38.041 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:37 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-14T05:06:38.237 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:37 smithi129 ceph-mon[101316]: pgmap v3320: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:38.237 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:06:38.237 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:37 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2282028074' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-14T05:06:38.237 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:37 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 3, "weight": 1.0} : dispatch 2024-09-14T05:06:38.260 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_ops_in_flight 2024-09-14T05:06:38.848 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:06:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:06:38] "GET /metrics HTTP/1.1" 200 34337 "" "Prometheus/2.51.0" 2024-09-14T05:06:38.849 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:38 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-09-14T05:06:38.849 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:38 smithi039 ceph-mon[106470]: osdmap e1642: 8 total, 8 up, 8 in 2024-09-14T05:06:38.849 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:38 smithi039 ceph-mon[106470]: osdmap e1643: 8 total, 8 up, 8 in 2024-09-14T05:06:38.849 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:38 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-09-14T05:06:38.849 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:38 smithi039 ceph-mon[111235]: osdmap e1642: 8 total, 8 up, 8 in 2024-09-14T05:06:38.849 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:38 smithi039 ceph-mon[111235]: osdmap e1643: 8 total, 8 up, 8 in 2024-09-14T05:06:38.850 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 5 1 2024-09-14T05:06:38.989 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:06:39.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:38 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2024-09-14T05:06:39.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:38 smithi129 ceph-mon[101316]: osdmap e1642: 8 total, 8 up, 8 in 2024-09-14T05:06:39.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:38 smithi129 ceph-mon[101316]: osdmap e1643: 8 total, 8 up, 8 in 2024-09-14T05:06:39.661 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:06:40.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:39 smithi039 ceph-mon[106470]: pgmap v3322: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:40.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:39 smithi039 ceph-mon[111235]: pgmap v3322: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:40.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:39 smithi129 ceph-mon[101316]: pgmap v3322: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:41.039 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:40 smithi039 ceph-mon[106470]: 1.19 scrub starts 2024-09-14T05:06:41.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:40 smithi039 ceph-mon[106470]: 1.19 scrub ok 2024-09-14T05:06:41.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:40 smithi039 ceph-mon[111235]: 1.19 scrub starts 2024-09-14T05:06:41.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:40 smithi039 ceph-mon[111235]: 1.19 scrub ok 2024-09-14T05:06:41.104 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:40 smithi129 ceph-mon[101316]: 1.19 scrub starts 2024-09-14T05:06:41.104 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:40 smithi129 ceph-mon[101316]: 1.19 scrub ok 2024-09-14T05:06:41.212 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T05:06:41.213 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:06:41.871 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_blocked_ops 2024-09-14T05:06:41.976 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:41 smithi039 ceph-mon[106470]: pgmap v3324: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:41.976 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:41 smithi039 ceph-mon[106470]: 1.c scrub starts 2024-09-14T05:06:41.976 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:41 smithi039 ceph-mon[106470]: 1.c scrub ok 2024-09-14T05:06:41.976 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:41 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:06:41.976 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:41 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:06:41.976 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:41 smithi039 ceph-mon[111235]: pgmap v3324: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:41.976 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:41 smithi039 ceph-mon[111235]: 1.c scrub starts 2024-09-14T05:06:41.976 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:41 smithi039 ceph-mon[111235]: 1.c scrub ok 2024-09-14T05:06:41.976 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:41 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:06:41.977 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:41 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:06:42.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:41 smithi129 ceph-mon[101316]: pgmap v3324: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:42.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:41 smithi129 ceph-mon[101316]: 1.c scrub starts 2024-09-14T05:06:42.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:41 smithi129 ceph-mon[101316]: 1.c scrub ok 2024-09-14T05:06:42.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:41 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:06:42.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:41 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:06:42.755 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:06:42.850 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.5 to 1 (10000) 2024-09-14T05:06:43.199 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:42 smithi129 ceph-mon[101316]: 1.7 scrub starts 2024-09-14T05:06:43.199 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:42 smithi129 ceph-mon[101316]: 1.7 scrub ok 2024-09-14T05:06:43.199 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:42 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3524058332' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T05:06:43.199 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:42 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T05:06:43.242 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:42 smithi039 ceph-mon[106470]: 1.7 scrub starts 2024-09-14T05:06:43.242 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:42 smithi039 ceph-mon[106470]: 1.7 scrub ok 2024-09-14T05:06:43.242 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:42 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3524058332' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T05:06:43.242 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:42 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T05:06:43.243 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:42 smithi039 ceph-mon[111235]: 1.7 scrub starts 2024-09-14T05:06:43.243 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:42 smithi039 ceph-mon[111235]: 1.7 scrub ok 2024-09-14T05:06:43.243 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:42 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3524058332' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T05:06:43.243 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:42 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 5, "weight": 1.0} : dispatch 2024-09-14T05:06:43.664 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 5 1 2024-09-14T05:06:43.883 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:06:43.987 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:43 smithi129 ceph-mon[101316]: pgmap v3325: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:43.987 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:43 smithi129 ceph-mon[101316]: 1.4 scrub starts 2024-09-14T05:06:43.988 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:43 smithi129 ceph-mon[101316]: 1.4 scrub ok 2024-09-14T05:06:43.988 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:43 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-14T05:06:43.988 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:43 smithi129 ceph-mon[101316]: osdmap e1644: 8 total, 8 up, 8 in 2024-09-14T05:06:43.988 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:43 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:06:43.988 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:43 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:06:43.988 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:43 smithi129 ceph-mon[101316]: osdmap e1645: 8 total, 8 up, 8 in 2024-09-14T05:06:44.004 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:43 smithi039 ceph-mon[106470]: pgmap v3325: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:44.005 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:43 smithi039 ceph-mon[106470]: 1.4 scrub starts 2024-09-14T05:06:44.005 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:43 smithi039 ceph-mon[106470]: 1.4 scrub ok 2024-09-14T05:06:44.005 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:43 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-14T05:06:44.005 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:43 smithi039 ceph-mon[106470]: osdmap e1644: 8 total, 8 up, 8 in 2024-09-14T05:06:44.005 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:43 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:06:44.005 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:43 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:06:44.005 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:43 smithi039 ceph-mon[106470]: osdmap e1645: 8 total, 8 up, 8 in 2024-09-14T05:06:44.005 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:43 smithi039 ceph-mon[111235]: pgmap v3325: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:44.005 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:43 smithi039 ceph-mon[111235]: 1.4 scrub starts 2024-09-14T05:06:44.005 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:43 smithi039 ceph-mon[111235]: 1.4 scrub ok 2024-09-14T05:06:44.006 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:43 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2024-09-14T05:06:44.006 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:43 smithi039 ceph-mon[111235]: osdmap e1644: 8 total, 8 up, 8 in 2024-09-14T05:06:44.006 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:43 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:06:44.006 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:43 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:06:44.006 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:43 smithi039 ceph-mon[111235]: osdmap e1645: 8 total, 8 up, 8 in 2024-09-14T05:06:44.453 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T05:06:44.454 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:06:45.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:44 smithi129 ceph-mon[101316]: 1.1e scrub starts 2024-09-14T05:06:45.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:44 smithi129 ceph-mon[101316]: 1.1e scrub ok 2024-09-14T05:06:45.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:44 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T05:06:45.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:44 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T05:06:45.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:44 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:06:45.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:44 smithi129 ceph-mon[101316]: osdmap e1646: 8 total, 8 up, 8 in 2024-09-14T05:06:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:44 smithi039 ceph-mon[106470]: 1.1e scrub starts 2024-09-14T05:06:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:44 smithi039 ceph-mon[106470]: 1.1e scrub ok 2024-09-14T05:06:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:44 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T05:06:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:44 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T05:06:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:44 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:06:45.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:44 smithi039 ceph-mon[106470]: osdmap e1646: 8 total, 8 up, 8 in 2024-09-14T05:06:45.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:44 smithi039 ceph-mon[111235]: 1.1e scrub starts 2024-09-14T05:06:45.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:44 smithi039 ceph-mon[111235]: 1.1e scrub ok 2024-09-14T05:06:45.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:44 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config generate-minimal-conf"} : dispatch 2024-09-14T05:06:45.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:44 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "auth get", "entity": "client.admin"} : dispatch 2024-09-14T05:06:45.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:44 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:06:45.041 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:44 smithi039 ceph-mon[111235]: osdmap e1646: 8 total, 8 up, 8 in 2024-09-14T05:06:45.080 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph daemon osd.4 dump_historic_ops 2024-09-14T05:06:45.277 INFO:teuthology.orchestra.run.smithi129.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.b/config 2024-09-14T05:06:45.982 INFO:teuthology.orchestra.run.smithi039.stderr:set osd.5 primary-affinity to 1 (8655362) 2024-09-14T05:06:46.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:45 smithi129 ceph-mon[101316]: pgmap v3328: 103 pgs: 1 peering, 1 unknown, 101 active+clean; 595 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 3.2 KiB/s rd, 9.2 KiB/s wr, 2 op/s 2024-09-14T05:06:46.284 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:45 smithi039 ceph-mon[106470]: pgmap v3328: 103 pgs: 1 peering, 1 unknown, 101 active+clean; 595 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 3.2 KiB/s rd, 9.2 KiB/s wr, 2 op/s 2024-09-14T05:06:46.284 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:45 smithi039 ceph-mon[111235]: pgmap v3328: 103 pgs: 1 peering, 1 unknown, 101 active+clean; 595 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 3.2 KiB/s rd, 9.2 KiB/s wr, 2 op/s 2024-09-14T05:06:46.650 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 6 1 2024-09-14T05:06:46.872 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:06:46.976 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:46 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2815248164' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-14T05:06:46.976 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:46 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2815248164' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-09-14T05:06:46.977 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:46 smithi129 ceph-mon[101316]: osdmap e1647: 8 total, 8 up, 8 in 2024-09-14T05:06:46.984 INFO:teuthology.orchestra.run.smithi129.stdout:ERROR: (22) Invalid argument 2024-09-14T05:06:46.984 INFO:teuthology.orchestra.run.smithi129.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2024-09-14T05:06:47.141 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:46 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2815248164' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-14T05:06:47.141 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:46 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2815248164' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-09-14T05:06:47.141 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:46 smithi039 ceph-mon[106470]: osdmap e1647: 8 total, 8 up, 8 in 2024-09-14T05:06:47.141 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:46 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2815248164' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 5, "weight": 1.0} : dispatch 2024-09-14T05:06:47.141 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:46 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2815248164' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2024-09-14T05:06:47.141 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:46 smithi039 ceph-mon[111235]: osdmap e1647: 8 total, 8 up, 8 in 2024-09-14T05:06:48.085 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:47 smithi039 ceph-mon[106470]: pgmap v3331: 103 pgs: 1 peering, 1 unknown, 101 active+clean; 595 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 4.7 KiB/s rd, 14 KiB/s wr, 3 op/s 2024-09-14T05:06:48.085 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:47 smithi039 ceph-mon[106470]: osdmap e1648: 8 total, 8 up, 8 in 2024-09-14T05:06:48.085 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:47 smithi039 ceph-mon[111235]: pgmap v3331: 103 pgs: 1 peering, 1 unknown, 101 active+clean; 595 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 4.7 KiB/s rd, 14 KiB/s wr, 3 op/s 2024-09-14T05:06:48.085 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:47 smithi039 ceph-mon[111235]: osdmap e1648: 8 total, 8 up, 8 in 2024-09-14T05:06:48.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:47 smithi129 ceph-mon[101316]: pgmap v3331: 103 pgs: 1 peering, 1 unknown, 101 active+clean; 595 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 4.7 KiB/s rd, 14 KiB/s wr, 3 op/s 2024-09-14T05:06:48.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:47 smithi129 ceph-mon[101316]: osdmap e1648: 8 total, 8 up, 8 in 2024-09-14T05:06:48.997 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:06:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:06:48] "GET /metrics HTTP/1.1" 200 34337 "" "Prometheus/2.51.0" 2024-09-14T05:06:49.882 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.6 to 1 (10000) 2024-09-14T05:06:50.251 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:49 smithi039 ceph-mon[106470]: pgmap v3333: 103 pgs: 1 peering, 102 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:50.251 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:49 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3362406803' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T05:06:50.251 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:49 smithi039 ceph-mon[111235]: pgmap v3333: 103 pgs: 1 peering, 102 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:50.251 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:49 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3362406803' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T05:06:50.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:49 smithi129 ceph-mon[101316]: pgmap v3333: 103 pgs: 1 peering, 102 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:06:50.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:49 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3362406803' entity='client.admin' cmd={"prefix": "osd reweight", "id": 6, "weight": 1.0} : dispatch 2024-09-14T05:06:50.688 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 6 1 2024-09-14T05:06:50.914 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:06:51.037 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:50 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3362406803' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-14T05:06:51.037 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:50 smithi039 ceph-mon[106470]: osdmap e1649: 8 total, 8 up, 8 in 2024-09-14T05:06:51.037 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:50 smithi039 ceph-mon[106470]: pgmap v3335: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T05:06:51.037 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:50 smithi039 ceph-mon[106470]: osdmap e1650: 8 total, 8 up, 8 in 2024-09-14T05:06:51.037 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:50 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3362406803' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-14T05:06:51.038 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:50 smithi039 ceph-mon[111235]: osdmap e1649: 8 total, 8 up, 8 in 2024-09-14T05:06:51.038 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:50 smithi039 ceph-mon[111235]: pgmap v3335: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T05:06:51.038 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:50 smithi039 ceph-mon[111235]: osdmap e1650: 8 total, 8 up, 8 in 2024-09-14T05:06:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:50 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3362406803' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2024-09-14T05:06:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:50 smithi129 ceph-mon[101316]: osdmap e1649: 8 total, 8 up, 8 in 2024-09-14T05:06:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:50 smithi129 ceph-mon[101316]: pgmap v3335: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T05:06:51.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:50 smithi129 ceph-mon[101316]: osdmap e1650: 8 total, 8 up, 8 in 2024-09-14T05:06:52.904 INFO:teuthology.orchestra.run.smithi039.stderr:set osd.6 primary-affinity to 1 (8655362) 2024-09-14T05:06:53.233 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:52 smithi039 ceph-mon[106470]: osdmap e1651: 8 total, 8 up, 8 in 2024-09-14T05:06:53.233 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:52 smithi039 ceph-mon[106470]: pgmap v3338: 103 pgs: 3 peering, 100 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T05:06:53.233 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:06:53.233 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:52 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3001554929' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-14T05:06:53.233 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:52 smithi039 ceph-mon[111235]: osdmap e1651: 8 total, 8 up, 8 in 2024-09-14T05:06:53.234 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:52 smithi039 ceph-mon[111235]: pgmap v3338: 103 pgs: 3 peering, 100 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T05:06:53.234 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:06:53.234 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:52 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3001554929' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-14T05:06:53.270 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:52 smithi129 ceph-mon[101316]: osdmap e1651: 8 total, 8 up, 8 in 2024-09-14T05:06:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:52 smithi129 ceph-mon[101316]: pgmap v3338: 103 pgs: 3 peering, 100 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2024-09-14T05:06:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:06:53.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:52 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3001554929' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 6, "weight": 1.0} : dispatch 2024-09-14T05:06:53.762 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 7 1 2024-09-14T05:06:53.981 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:06:54.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:53 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3001554929' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-14T05:06:54.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:53 smithi039 ceph-mon[106470]: osdmap e1652: 8 total, 8 up, 8 in 2024-09-14T05:06:54.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:53 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3001554929' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-14T05:06:54.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:53 smithi039 ceph-mon[111235]: osdmap e1652: 8 total, 8 up, 8 in 2024-09-14T05:06:54.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:53 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3001554929' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2024-09-14T05:06:54.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:53 smithi129 ceph-mon[101316]: osdmap e1652: 8 total, 8 up, 8 in 2024-09-14T05:06:55.250 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:54 smithi039 ceph-mon[106470]: osdmap e1653: 8 total, 8 up, 8 in 2024-09-14T05:06:55.250 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:54 smithi039 ceph-mon[106470]: pgmap v3341: 103 pgs: 1 active+undersized+remapped, 9 peering, 93 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-09-14T05:06:55.251 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:54 smithi039 ceph-mon[111235]: osdmap e1653: 8 total, 8 up, 8 in 2024-09-14T05:06:55.251 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:54 smithi039 ceph-mon[111235]: pgmap v3341: 103 pgs: 1 active+undersized+remapped, 9 peering, 93 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-09-14T05:06:55.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:54 smithi129 ceph-mon[101316]: osdmap e1653: 8 total, 8 up, 8 in 2024-09-14T05:06:55.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:54 smithi129 ceph-mon[101316]: pgmap v3341: 103 pgs: 1 active+undersized+remapped, 9 peering, 93 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 80 KiB/s, 0 objects/s recovering 2024-09-14T05:06:56.003 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.7 to 1 (10000) 2024-09-14T05:06:56.271 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:56 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1042760950' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T05:06:56.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:56 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1042760950' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T05:06:56.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:56 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1042760950' entity='client.admin' cmd={"prefix": "osd reweight", "id": 7, "weight": 1.0} : dispatch 2024-09-14T05:06:56.822 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 7 1 2024-09-14T05:06:57.028 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:06:57.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:57 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1042760950' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-14T05:06:57.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:57 smithi039 ceph-mon[106470]: osdmap e1654: 8 total, 8 up, 8 in 2024-09-14T05:06:57.043 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:57 smithi039 ceph-mon[106470]: pgmap v3343: 103 pgs: 1 active+undersized+remapped, 6 peering, 96 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-09-14T05:06:57.296 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:57 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1042760950' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-14T05:06:57.296 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:57 smithi039 ceph-mon[111235]: osdmap e1654: 8 total, 8 up, 8 in 2024-09-14T05:06:57.296 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:57 smithi039 ceph-mon[111235]: pgmap v3343: 103 pgs: 1 active+undersized+remapped, 6 peering, 96 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-09-14T05:06:57.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:57 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1042760950' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2024-09-14T05:06:57.524 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:57 smithi129 ceph-mon[101316]: osdmap e1654: 8 total, 8 up, 8 in 2024-09-14T05:06:57.524 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:57 smithi129 ceph-mon[101316]: pgmap v3343: 103 pgs: 1 active+undersized+remapped, 6 peering, 96 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 76 KiB/s, 0 objects/s recovering 2024-09-14T05:06:58.671 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:06:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:06:58] "GET /metrics HTTP/1.1" 200 34314 "" "Prometheus/2.51.0" 2024-09-14T05:06:59.201 INFO:teuthology.orchestra.run.smithi039.stderr:set osd.7 primary-affinity to 1 (8655362) 2024-09-14T05:06:59.213 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:59 smithi039 ceph-mon[106470]: pgmap v3344: 103 pgs: 3 peering, 100 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-09-14T05:06:59.213 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:59 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2025560460' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-14T05:06:59.213 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:06:59 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-14T05:06:59.213 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:59 smithi039 ceph-mon[111235]: pgmap v3344: 103 pgs: 3 peering, 100 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-09-14T05:06:59.214 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:59 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2025560460' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-14T05:06:59.214 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:06:59 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-14T05:06:59.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:59 smithi129 ceph-mon[101316]: pgmap v3344: 103 pgs: 3 peering, 100 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 53 KiB/s, 0 objects/s recovering 2024-09-14T05:06:59.582 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:59 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2025560460' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-14T05:06:59.582 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:06:59 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 7, "weight": 1.0} : dispatch 2024-09-14T05:07:00.028 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd reweight 4 1 2024-09-14T05:07:00.246 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:07:00.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:00 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-09-14T05:07:00.290 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:00 smithi039 ceph-mon[106470]: osdmap e1655: 8 total, 8 up, 8 in 2024-09-14T05:07:00.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:00 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-09-14T05:07:00.290 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:00 smithi039 ceph-mon[111235]: osdmap e1655: 8 total, 8 up, 8 in 2024-09-14T05:07:00.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:00 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2024-09-14T05:07:00.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:00 smithi129 ceph-mon[101316]: osdmap e1655: 8 total, 8 up, 8 in 2024-09-14T05:07:01.406 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:01 smithi039 ceph-mon[106470]: pgmap v3346: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:01.406 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:01 smithi039 ceph-mon[111235]: pgmap v3346: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:01.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:01 smithi129 ceph-mon[101316]: pgmap v3346: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:02.227 INFO:teuthology.orchestra.run.smithi039.stderr:reweighted osd.4 to 1 (10000) 2024-09-14T05:07:02.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:02 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3477426575' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T05:07:02.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:02 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T05:07:02.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:02 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3477426575' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T05:07:02.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:02 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T05:07:02.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:02 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3477426575' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T05:07:02.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:02 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd reweight", "id": 4, "weight": 1.0} : dispatch 2024-09-14T05:07:03.087 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd primary-affinity 4 1 2024-09-14T05:07:03.295 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:07:03.416 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:03 smithi039 ceph-mon[106470]: pgmap v3347: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:03.416 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:03 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-14T05:07:03.416 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:03 smithi039 ceph-mon[106470]: osdmap e1656: 8 total, 8 up, 8 in 2024-09-14T05:07:03.417 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:03 smithi039 ceph-mon[111235]: pgmap v3347: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:03.417 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:03 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-14T05:07:03.417 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:03 smithi039 ceph-mon[111235]: osdmap e1656: 8 total, 8 up, 8 in 2024-09-14T05:07:03.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:03 smithi129 ceph-mon[101316]: pgmap v3347: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:03.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:03 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2024-09-14T05:07:03.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:03 smithi129 ceph-mon[101316]: osdmap e1656: 8 total, 8 up, 8 in 2024-09-14T05:07:05.275 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:05 smithi039 ceph-mon[106470]: pgmap v3349: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:05.275 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:05 smithi039 ceph-mon[111235]: pgmap v3349: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:05.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:05 smithi129 ceph-mon[101316]: pgmap v3349: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:06.257 INFO:teuthology.orchestra.run.smithi039.stderr:set osd.4 primary-affinity to 1 (8655362) 2024-09-14T05:07:06.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:06 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/2237978694' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-14T05:07:06.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:06 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-14T05:07:06.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:06 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/2237978694' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-14T05:07:06.541 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:06 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-14T05:07:06.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:06 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/2237978694' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-14T05:07:06.541 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:06 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd={"prefix": "osd primary-affinity", "id": 4, "weight": 1.0} : dispatch 2024-09-14T05:07:07.033 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2024-09-14T05:07:07.034 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2024-09-14T05:07:07.034 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2024-09-14T05:07:07.034 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2024-09-14T05:07:07.034 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-09-14T05:07:07.034 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T05:07:07.250 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:07:07.255 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:07 smithi039 ceph-mon[106470]: pgmap v3350: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:07.255 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:07 smithi039 ceph-mon[106470]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-14T05:07:07.255 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:07 smithi039 ceph-mon[106470]: osdmap e1657: 8 total, 8 up, 8 in 2024-09-14T05:07:07.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:07 smithi129 ceph-mon[101316]: pgmap v3350: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:07.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:07 smithi129 ceph-mon[101316]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-14T05:07:07.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:07 smithi129 ceph-mon[101316]: osdmap e1657: 8 total, 8 up, 8 in 2024-09-14T05:07:07.522 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:07 smithi039 ceph-mon[111235]: pgmap v3350: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:07.522 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:07 smithi039 ceph-mon[111235]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2024-09-14T05:07:07.522 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:07 smithi039 ceph-mon[111235]: osdmap e1657: 8 total, 8 up, 8 in 2024-09-14T05:07:08.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:08 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:07:08.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:08 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:07:08.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:08 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:07:08.906 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:07:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:07:08] "GET /metrics HTTP/1.1" 200 34311 "" "Prometheus/2.51.0" 2024-09-14T05:07:09.303 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T05:07:09.304 INFO:teuthology.orchestra.run.smithi039.stdout:{"epoch":1657,"fsid":"bd7c803c-724b-11ef-bceb-c7b262605968","created":"2024-09-14T03:46:13.726476+0000","modified":"2024-09-14T05:07:06.239704+0000","last_up_change":"2024-09-14T04:43:48.438095+0000","last_in_change":"2024-09-14T05:05:48.367545+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":59,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:49:18.022402+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1628","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"1582","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T03:51:45.764151+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-14T03:53:04.166504+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-14T03:53:04.665111+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"72","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"5fdfd478-dd27-4674-a7d6-53177d9e549a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":667,"up_from":671,"up_thru":1651,"down_at":667,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6802","nonce":3832421293},{"type":"v1","addr":"172.21.15.39:6803","nonce":3832421293}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6804","nonce":3832421293},{"type":"v1","addr":"172.21.15.39:6805","nonce":3832421293}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6808","nonce":3832421293},{"type":"v1","addr":"172.21.15.39:6809","nonce":3832421293}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6806","nonce":3832421293},{"type":"v1","addr":"172.21.15.39:6807","nonce":3832421293}]},"public_addr":"172.21.15.39:6803/3832421293","cluster_addr":"172.21.15.39:6805/3832421293","heartbeat_back_addr":"172.21.15.39:6809/3832421293","heartbeat_front_addr":"172.21.15.39:6807/3832421293","state":["exists","up"]},{"osd":1,"uuid":"991c236e-1679-4c1e-a1a9-3894b1b12055","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":677,"up_from":681,"up_thru":1652,"down_at":677,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6810","nonce":1186072955},{"type":"v1","addr":"172.21.15.39:6811","nonce":1186072955}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6812","nonce":1186072955},{"type":"v1","addr":"172.21.15.39:6813","nonce":1186072955}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6816","nonce":1186072955},{"type":"v1","addr":"172.21.15.39:6817","nonce":1186072955}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6814","nonce":1186072955},{"type":"v1","addr":"172.21.15.39:6815","nonce":1186072955}]},"public_addr":"172.21.15.39:6811/1186072955","cluster_addr":"172.21.15.39:6813/1186072955","heartbeat_back_addr":"172.21.15.39:6817/1186072955","heartbeat_front_addr":"172.21.15.39:6815/1186072955","state":["exists","up"]},{"osd":2,"uuid":"f9611ffd-5d37-4465-8114-ecc70d4d6d14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":688,"up_from":694,"up_thru":1646,"down_at":688,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6818","nonce":2359749690},{"type":"v1","addr":"172.21.15.39:6819","nonce":2359749690}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6820","nonce":2359749690},{"type":"v1","addr":"172.21.15.39:6821","nonce":2359749690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6824","nonce":2359749690},{"type":"v1","addr":"172.21.15.39:6825","nonce":2359749690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6822","nonce":2359749690},{"type":"v1","addr":"172.21.15.39:6823","nonce":2359749690}]},"public_addr":"172.21.15.39:6819/2359749690","cluster_addr":"172.21.15.39:6821/2359749690","heartbeat_back_addr":"172.21.15.39:6825/2359749690","heartbeat_front_addr":"172.21.15.39:6823/2359749690","state":["exists","up"]},{"osd":3,"uuid":"56b6877b-2107-47d6-8da9-015db7352288","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":697,"up_from":701,"up_thru":1652,"down_at":697,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6826","nonce":2092194762},{"type":"v1","addr":"172.21.15.39:6827","nonce":2092194762}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6828","nonce":2092194762},{"type":"v1","addr":"172.21.15.39:6829","nonce":2092194762}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6832","nonce":2092194762},{"type":"v1","addr":"172.21.15.39:6833","nonce":2092194762}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6830","nonce":2092194762},{"type":"v1","addr":"172.21.15.39:6831","nonce":2092194762}]},"public_addr":"172.21.15.39:6827/2092194762","cluster_addr":"172.21.15.39:6829/2092194762","heartbeat_back_addr":"172.21.15.39:6833/2092194762","heartbeat_front_addr":"172.21.15.39:6831/2092194762","state":["exists","up"]},{"osd":4,"uuid":"2d5523c4-5871-4fb3-bc0d-62c3b7d8b6a1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":31,"last_clean_end":1344,"up_from":1392,"up_thru":1652,"down_at":1344,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6800","nonce":532739938},{"type":"v1","addr":"172.21.15.129:6801","nonce":532739938}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6802","nonce":532739938},{"type":"v1","addr":"172.21.15.129:6803","nonce":532739938}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6806","nonce":532739938},{"type":"v1","addr":"172.21.15.129:6807","nonce":532739938}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6804","nonce":532739938},{"type":"v1","addr":"172.21.15.129:6805","nonce":532739938}]},"public_addr":"172.21.15.129:6801/532739938","cluster_addr":"172.21.15.129:6803/532739938","heartbeat_back_addr":"172.21.15.129:6807/532739938","heartbeat_front_addr":"172.21.15.129:6805/532739938","state":["exists","up"]},{"osd":5,"uuid":"a3ba55fd-f32e-4f6d-9b48-d2845a3474cd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":1652,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6808","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6809","nonce":2184993002}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6810","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6811","nonce":2184993002}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6814","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6815","nonce":2184993002}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6812","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6813","nonce":2184993002}]},"public_addr":"172.21.15.129:6809/2184993002","cluster_addr":"172.21.15.129:6811/2184993002","heartbeat_back_addr":"172.21.15.129:6815/2184993002","heartbeat_front_addr":"172.21.15.129:6813/2184993002","state":["exists","up"]},{"osd":6,"uuid":"79e9c0a2-86a4-4267-a799-059d473d7b74","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":1652,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6816","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6817","nonce":3767031353}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6818","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6819","nonce":3767031353}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6822","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6823","nonce":3767031353}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6820","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6821","nonce":3767031353}]},"public_addr":"172.21.15.129:6817/3767031353","cluster_addr":"172.21.15.129:6819/3767031353","heartbeat_back_addr":"172.21.15.129:6823/3767031353","heartbeat_front_addr":"172.21.15.129:6821/3767031353","state":["exists","up"]},{"osd":7,"uuid":"22f41628-4b88-4b8c-82b2-fcd2802da1bf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":1652,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6824","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6825","nonce":2370414690}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6826","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6827","nonce":2370414690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6830","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6831","nonce":2370414690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6828","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6829","nonce":2370414690}]},"public_addr":"172.21.15.129:6825/2370414690","cluster_addr":"172.21.15.129:6827/2370414690","heartbeat_back_addr":"172.21.15.129:6831/2370414690","heartbeat_front_addr":"172.21.15.129:6829/2370414690","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:48:23.355518+0000","dead_epoch":666},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:48:48.715441+0000","dead_epoch":677},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:49:13.415323+0000","dead_epoch":687},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:49:39.935205+0000","dead_epoch":697},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:50:01.649042+0000","dead_epoch":1343},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:50:24.001678+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:50:47.985011+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:51:11.584825+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"1.11","mappings":[{"from":5,"to":2}]},{"pgid":"3.6","mappings":[{"from":4,"to":2}]},{"pgid":"4.0","mappings":[{"from":3,"to":2}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.10","mappings":[{"from":3,"to":5}]},{"pgid":"4.18","mappings":[{"from":1,"to":5}]},{"pgid":"4.1a","mappings":[{"from":3,"to":5}]},{"pgid":"4.1b","mappings":[{"from":1,"to":7}]},{"pgid":"4.1f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.39:0/1864073859":"2024-09-14T05:12:12.455045+0000","172.21.15.39:0/3247815370":"2024-09-15T03:59:06.707728+0000","172.21.15.39:6801/1103763596":"2024-09-15T03:59:06.707728+0000","172.21.15.39:6800/1103763596":"2024-09-15T03:59:06.707728+0000","172.21.15.39:0/1398136686":"2024-09-15T03:59:06.707728+0000","172.21.15.129:0/78252352":"2024-09-15T03:56:26.175569+0000","172.21.15.39:6800/3127470450":"2024-09-15T03:46:59.511580+0000","172.21.15.39:0/209353267":"2024-09-15T03:46:59.511580+0000","172.21.15.129:0/1000360058":"2024-09-15T03:56:26.175569+0000","172.21.15.39:0/1530637819":"2024-09-15T03:46:59.511580+0000","172.21.15.39:6801/628437919":"2024-09-15T03:46:35.805097+0000","172.21.15.39:0/822851906":"2024-09-15T03:59:06.707728+0000","172.21.15.39:6800/748677190":"2024-09-15T03:51:44.726801+0000","172.21.15.39:6801/748677190":"2024-09-15T03:51:44.726801+0000","172.21.15.39:0/2745905728":"2024-09-15T03:51:44.726801+0000","172.21.15.129:0/2660542464":"2024-09-15T03:56:26.175569+0000","172.21.15.129:0/3885028850":"2024-09-15T03:56:26.175569+0000","172.21.15.39:0/2670041591":"2024-09-15T03:46:35.805097+0000","172.21.15.39:6801/3924974081":"2024-09-15T03:55:55.288048+0000","172.21.15.39:6800/3924974081":"2024-09-15T03:55:55.288048+0000","172.21.15.39:0/2951004223":"2024-09-15T03:51:44.726801+0000","172.21.15.39:0/3196173579":"2024-09-15T03:55:55.288048+0000","172.21.15.39:0/502495021":"2024-09-15T03:55:55.288048+0000","172.21.15.39:0/3161547781":"2024-09-15T03:46:59.511580+0000","172.21.15.39:6801/3127470450":"2024-09-15T03:46:59.511580+0000","172.21.15.39:0/2133325947":"2024-09-15T03:46:35.805097+0000","172.21.15.39:0/754571920":"2024-09-15T03:51:44.726801+0000","172.21.15.39:0/81449629":"2024-09-15T03:46:35.805097+0000","172.21.15.39:6800/628437919":"2024-09-15T03:46:35.805097+0000","172.21.15.129:0/3477521861":"2024-09-15T03:56:26.175569+0000","172.21.15.39:0/1746545094":"2024-09-15T03:59:06.707728+0000","172.21.15.129:0/1993195691":"2024-09-15T03:56:26.175569+0000","172.21.15.129:6834/1529931471":"2024-09-15T03:56:26.175569+0000","172.21.15.129:6835/1529931471":"2024-09-15T03:56:26.175569+0000","172.21.15.39:0/1766366412":"2024-09-15T03:55:55.288048+0000","172.21.15.39:0/2587896825":"2024-09-15T03:51:44.726801+0000","172.21.15.39:0/1825825785":"2024-09-15T03:55:55.288048+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":6,"snaps":[{"begin":2,"length":1}]},{"pool":7,"snaps":[{"begin":2,"length":1}]},{"pool":8,"snaps":[{"begin":2,"length":1}]},{"pool":9,"snaps":[{"begin":2,"length":1}]},{"pool":10,"snaps":[{"begin":2,"length":1}]},{"pool":11,"snaps":[{"begin":2,"length":1}]},{"pool":12,"snaps":[{"begin":233,"length":1}]},{"pool":14,"snaps":[{"begin":6,"length":3}]},{"pool":16,"snaps":[{"begin":35,"length":3},{"begin":45,"length":3}]},{"pool":17,"snaps":[{"begin":22,"length":1}]},{"pool":18,"snaps":[{"begin":2,"length":1}]},{"pool":51,"snaps":[{"begin":104,"length":1}]},{"pool":59,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T05:07:09.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:09 smithi129 ceph-mon[101316]: pgmap v3352: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:09.539 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:09 smithi039 ceph-mon[106470]: pgmap v3352: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:09.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:09 smithi039 ceph-mon[111235]: pgmap v3352: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:10.141 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-09-14T05:07:10.142 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd dump --format=json 2024-09-14T05:07:10.358 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:07:10.480 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:10 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/4272550029' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T05:07:10.480 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:10 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/4272550029' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T05:07:10.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:10 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/4272550029' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T05:07:11.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:11 smithi129 ceph-mon[101316]: pgmap v3353: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:11.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:11 smithi039 ceph-mon[106470]: pgmap v3353: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:11.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:11 smithi039 ceph-mon[111235]: pgmap v3353: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:12.304 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T05:07:12.304 INFO:teuthology.orchestra.run.smithi039.stdout:{"epoch":1657,"fsid":"bd7c803c-724b-11ef-bceb-c7b262605968","created":"2024-09-14T03:46:13.726476+0000","modified":"2024-09-14T05:07:06.239704+0000","last_up_change":"2024-09-14T04:43:48.438095+0000","last_in_change":"2024-09-14T05:05:48.367545+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":59,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-09-14T03:49:18.022402+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"1628","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"1582","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.059999942779541,"score_stable":2.059999942779541,"optimal_score":1,"raw_score_acting":2.059999942779541,"raw_score_stable":2.059999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-09-14T03:51:45.764151+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":3,"snap_epoch":66,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-09-14T03:53:04.166504+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"71","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-09-14T03:53:04.665111+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"72","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"5fdfd478-dd27-4674-a7d6-53177d9e549a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":667,"up_from":671,"up_thru":1651,"down_at":667,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6802","nonce":3832421293},{"type":"v1","addr":"172.21.15.39:6803","nonce":3832421293}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6804","nonce":3832421293},{"type":"v1","addr":"172.21.15.39:6805","nonce":3832421293}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6808","nonce":3832421293},{"type":"v1","addr":"172.21.15.39:6809","nonce":3832421293}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6806","nonce":3832421293},{"type":"v1","addr":"172.21.15.39:6807","nonce":3832421293}]},"public_addr":"172.21.15.39:6803/3832421293","cluster_addr":"172.21.15.39:6805/3832421293","heartbeat_back_addr":"172.21.15.39:6809/3832421293","heartbeat_front_addr":"172.21.15.39:6807/3832421293","state":["exists","up"]},{"osd":1,"uuid":"991c236e-1679-4c1e-a1a9-3894b1b12055","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":677,"up_from":681,"up_thru":1652,"down_at":677,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6810","nonce":1186072955},{"type":"v1","addr":"172.21.15.39:6811","nonce":1186072955}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6812","nonce":1186072955},{"type":"v1","addr":"172.21.15.39:6813","nonce":1186072955}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6816","nonce":1186072955},{"type":"v1","addr":"172.21.15.39:6817","nonce":1186072955}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6814","nonce":1186072955},{"type":"v1","addr":"172.21.15.39:6815","nonce":1186072955}]},"public_addr":"172.21.15.39:6811/1186072955","cluster_addr":"172.21.15.39:6813/1186072955","heartbeat_back_addr":"172.21.15.39:6817/1186072955","heartbeat_front_addr":"172.21.15.39:6815/1186072955","state":["exists","up"]},{"osd":2,"uuid":"f9611ffd-5d37-4465-8114-ecc70d4d6d14","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":688,"up_from":694,"up_thru":1646,"down_at":688,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6818","nonce":2359749690},{"type":"v1","addr":"172.21.15.39:6819","nonce":2359749690}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6820","nonce":2359749690},{"type":"v1","addr":"172.21.15.39:6821","nonce":2359749690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6824","nonce":2359749690},{"type":"v1","addr":"172.21.15.39:6825","nonce":2359749690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6822","nonce":2359749690},{"type":"v1","addr":"172.21.15.39:6823","nonce":2359749690}]},"public_addr":"172.21.15.39:6819/2359749690","cluster_addr":"172.21.15.39:6821/2359749690","heartbeat_back_addr":"172.21.15.39:6825/2359749690","heartbeat_front_addr":"172.21.15.39:6823/2359749690","state":["exists","up"]},{"osd":3,"uuid":"56b6877b-2107-47d6-8da9-015db7352288","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":697,"up_from":701,"up_thru":1652,"down_at":697,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6826","nonce":2092194762},{"type":"v1","addr":"172.21.15.39:6827","nonce":2092194762}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6828","nonce":2092194762},{"type":"v1","addr":"172.21.15.39:6829","nonce":2092194762}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6832","nonce":2092194762},{"type":"v1","addr":"172.21.15.39:6833","nonce":2092194762}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.39:6830","nonce":2092194762},{"type":"v1","addr":"172.21.15.39:6831","nonce":2092194762}]},"public_addr":"172.21.15.39:6827/2092194762","cluster_addr":"172.21.15.39:6829/2092194762","heartbeat_back_addr":"172.21.15.39:6833/2092194762","heartbeat_front_addr":"172.21.15.39:6831/2092194762","state":["exists","up"]},{"osd":4,"uuid":"2d5523c4-5871-4fb3-bc0d-62c3b7d8b6a1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":31,"last_clean_end":1344,"up_from":1392,"up_thru":1652,"down_at":1344,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6800","nonce":532739938},{"type":"v1","addr":"172.21.15.129:6801","nonce":532739938}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6802","nonce":532739938},{"type":"v1","addr":"172.21.15.129:6803","nonce":532739938}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6806","nonce":532739938},{"type":"v1","addr":"172.21.15.129:6807","nonce":532739938}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6804","nonce":532739938},{"type":"v1","addr":"172.21.15.129:6805","nonce":532739938}]},"public_addr":"172.21.15.129:6801/532739938","cluster_addr":"172.21.15.129:6803/532739938","heartbeat_back_addr":"172.21.15.129:6807/532739938","heartbeat_front_addr":"172.21.15.129:6805/532739938","state":["exists","up"]},{"osd":5,"uuid":"a3ba55fd-f32e-4f6d-9b48-d2845a3474cd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":1652,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6808","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6809","nonce":2184993002}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6810","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6811","nonce":2184993002}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6814","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6815","nonce":2184993002}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6812","nonce":2184993002},{"type":"v1","addr":"172.21.15.129:6813","nonce":2184993002}]},"public_addr":"172.21.15.129:6809/2184993002","cluster_addr":"172.21.15.129:6811/2184993002","heartbeat_back_addr":"172.21.15.129:6815/2184993002","heartbeat_front_addr":"172.21.15.129:6813/2184993002","state":["exists","up"]},{"osd":6,"uuid":"79e9c0a2-86a4-4267-a799-059d473d7b74","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":1652,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6816","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6817","nonce":3767031353}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6818","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6819","nonce":3767031353}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6822","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6823","nonce":3767031353}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6820","nonce":3767031353},{"type":"v1","addr":"172.21.15.129:6821","nonce":3767031353}]},"public_addr":"172.21.15.129:6817/3767031353","cluster_addr":"172.21.15.129:6819/3767031353","heartbeat_back_addr":"172.21.15.129:6823/3767031353","heartbeat_front_addr":"172.21.15.129:6821/3767031353","state":["exists","up"]},{"osd":7,"uuid":"22f41628-4b88-4b8c-82b2-fcd2802da1bf","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":1652,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6824","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6825","nonce":2370414690}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6826","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6827","nonce":2370414690}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6830","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6831","nonce":2370414690}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.129:6828","nonce":2370414690},{"type":"v1","addr":"172.21.15.129:6829","nonce":2370414690}]},"public_addr":"172.21.15.129:6825/2370414690","cluster_addr":"172.21.15.129:6827/2370414690","heartbeat_back_addr":"172.21.15.129:6831/2370414690","heartbeat_front_addr":"172.21.15.129:6829/2370414690","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:48:23.355518+0000","dead_epoch":666},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:48:48.715441+0000","dead_epoch":677},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:49:13.415323+0000","dead_epoch":687},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540719139924082687,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:49:39.935205+0000","dead_epoch":697},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:50:01.649042+0000","dead_epoch":1343},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:50:24.001678+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:50:47.985011+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-09-14T03:51:11.584825+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"1.11","mappings":[{"from":5,"to":2}]},{"pgid":"3.6","mappings":[{"from":4,"to":2}]},{"pgid":"4.0","mappings":[{"from":3,"to":2}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.10","mappings":[{"from":3,"to":5}]},{"pgid":"4.18","mappings":[{"from":1,"to":5}]},{"pgid":"4.1a","mappings":[{"from":3,"to":5}]},{"pgid":"4.1b","mappings":[{"from":1,"to":7}]},{"pgid":"4.1f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.39:0/1864073859":"2024-09-14T05:12:12.455045+0000","172.21.15.39:0/1825825785":"2024-09-15T03:55:55.288048+0000","172.21.15.129:0/3885028850":"2024-09-15T03:56:26.175569+0000","172.21.15.129:6835/1529931471":"2024-09-15T03:56:26.175569+0000","172.21.15.39:0/822851906":"2024-09-15T03:59:06.707728+0000","172.21.15.39:6800/748677190":"2024-09-15T03:51:44.726801+0000","172.21.15.39:0/3247815370":"2024-09-15T03:59:06.707728+0000","172.21.15.129:0/78252352":"2024-09-15T03:56:26.175569+0000","172.21.15.39:0/1530637819":"2024-09-15T03:46:59.511580+0000","172.21.15.39:6801/628437919":"2024-09-15T03:46:35.805097+0000","172.21.15.39:6801/748677190":"2024-09-15T03:51:44.726801+0000","172.21.15.39:0/2745905728":"2024-09-15T03:51:44.726801+0000","172.21.15.129:0/2660542464":"2024-09-15T03:56:26.175569+0000","172.21.15.39:0/209353267":"2024-09-15T03:46:59.511580+0000","172.21.15.39:6800/3127470450":"2024-09-15T03:46:59.511580+0000","172.21.15.129:0/1000360058":"2024-09-15T03:56:26.175569+0000","172.21.15.39:0/2670041591":"2024-09-15T03:46:35.805097+0000","172.21.15.39:6801/3924974081":"2024-09-15T03:55:55.288048+0000","172.21.15.39:6800/3924974081":"2024-09-15T03:55:55.288048+0000","172.21.15.39:0/2951004223":"2024-09-15T03:51:44.726801+0000","172.21.15.39:0/3196173579":"2024-09-15T03:55:55.288048+0000","172.21.15.39:0/502495021":"2024-09-15T03:55:55.288048+0000","172.21.15.39:0/3161547781":"2024-09-15T03:46:59.511580+0000","172.21.15.39:0/1398136686":"2024-09-15T03:59:06.707728+0000","172.21.15.39:6801/3127470450":"2024-09-15T03:46:59.511580+0000","172.21.15.39:0/2133325947":"2024-09-15T03:46:35.805097+0000","172.21.15.39:6800/1103763596":"2024-09-15T03:59:06.707728+0000","172.21.15.129:6834/1529931471":"2024-09-15T03:56:26.175569+0000","172.21.15.39:0/1746545094":"2024-09-15T03:59:06.707728+0000","172.21.15.129:0/1993195691":"2024-09-15T03:56:26.175569+0000","172.21.15.39:6801/1103763596":"2024-09-15T03:59:06.707728+0000","172.21.15.39:0/754571920":"2024-09-15T03:51:44.726801+0000","172.21.15.39:0/81449629":"2024-09-15T03:46:35.805097+0000","172.21.15.39:6800/628437919":"2024-09-15T03:46:35.805097+0000","172.21.15.129:0/3477521861":"2024-09-15T03:56:26.175569+0000","172.21.15.39:0/2587896825":"2024-09-15T03:51:44.726801+0000","172.21.15.39:0/1766366412":"2024-09-15T03:55:55.288048+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":5,"snaps":[{"begin":2,"length":1}]},{"pool":6,"snaps":[{"begin":2,"length":1}]},{"pool":7,"snaps":[{"begin":2,"length":1}]},{"pool":8,"snaps":[{"begin":2,"length":1}]},{"pool":9,"snaps":[{"begin":2,"length":1}]},{"pool":10,"snaps":[{"begin":2,"length":1}]},{"pool":11,"snaps":[{"begin":2,"length":1}]},{"pool":12,"snaps":[{"begin":233,"length":1}]},{"pool":14,"snaps":[{"begin":6,"length":3}]},{"pool":16,"snaps":[{"begin":35,"length":3},{"begin":45,"length":3}]},{"pool":17,"snaps":[{"begin":22,"length":1}]},{"pool":18,"snaps":[{"begin":2,"length":1}]},{"pool":51,"snaps":[{"begin":104,"length":1}]},{"pool":59,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-09-14T05:07:13.139 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-09-14T05:07:13.140 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-09-14T05:07:13.140 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-09-14T05:07:13.141 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-09-14T05:07:13.141 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-09-14T05:07:13.142 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-09-14T05:07:13.142 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-09-14T05:07:13.142 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-09-14T05:07:13.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:13 smithi039 ceph-mon[111235]: pgmap v3354: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:13.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:13 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3582053785' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T05:07:13.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:13 smithi039 ceph-mon[106470]: pgmap v3354: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:13.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:13 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3582053785' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T05:07:13.585 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:07:13.585 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:07:13.589 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:07:13.589 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:07:13.589 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:07:13.590 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:07:13.601 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:07:13.620 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:07:13.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:13 smithi129 ceph-mon[101316]: pgmap v3354: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:13.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:13 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3582053785' entity='client.admin' cmd={"prefix": "osd dump", "format": "json"} : dispatch 2024-09-14T05:07:15.745 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:15 smithi039 ceph-mon[106470]: pgmap v3355: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:15.745 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:15 smithi039 ceph-mon[111235]: pgmap v3355: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:15.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:15 smithi129 ceph-mon[101316]: pgmap v3355: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:17.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:17 smithi039 ceph-mon[106470]: pgmap v3356: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:17.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:17 smithi039 ceph-mon[111235]: pgmap v3356: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:17.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:17 smithi129 ceph-mon[101316]: pgmap v3356: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:18.958 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:07:18 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:07:18] "GET /metrics HTTP/1.1" 200 34311 "" "Prometheus/2.51.0" 2024-09-14T05:07:19.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:19 smithi129 ceph-mon[101316]: pgmap v3357: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:19.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:19 smithi039 ceph-mon[106470]: pgmap v3357: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:19.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:19 smithi039 ceph-mon[111235]: pgmap v3357: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:21.488 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:21 smithi039 ceph-mon[106470]: pgmap v3358: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:21.489 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:21 smithi039 ceph-mon[111235]: pgmap v3358: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:21.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:21 smithi129 ceph-mon[101316]: pgmap v3358: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:22.634 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:22 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:07:22.635 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:22 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:07:22.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:22 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:07:23.645 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:23 smithi039 ceph-mon[106470]: pgmap v3359: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:23.646 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:23 smithi039 ceph-mon[111235]: pgmap v3359: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:23.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:23 smithi129 ceph-mon[101316]: pgmap v3359: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:25.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:25 smithi129 ceph-mon[101316]: pgmap v3360: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:25.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:25 smithi039 ceph-mon[111235]: pgmap v3360: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:25.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:25 smithi039 ceph-mon[106470]: pgmap v3360: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:25.910 INFO:teuthology.orchestra.run.smithi039.stdout:236223202198 2024-09-14T05:07:25.911 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-09-14T05:07:27.539 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:27 smithi039 ceph-mon[111235]: pgmap v3361: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:27.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:27 smithi039 ceph-mon[106470]: pgmap v3361: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:27.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:27 smithi129 ceph-mon[101316]: pgmap v3361: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:28.460 INFO:teuthology.orchestra.run.smithi039.stdout:2881923056357 2024-09-14T05:07:28.460 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-09-14T05:07:28.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:07:28 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:07:28] "GET /metrics HTTP/1.1" 200 34308 "" "Prometheus/2.51.0" 2024-09-14T05:07:29.316 INFO:teuthology.orchestra.run.smithi039.stdout:201863463834 2024-09-14T05:07:29.316 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-09-14T05:07:29.738 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:29 smithi039 ceph-mon[106470]: pgmap v3362: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:29.738 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:29 smithi039 ceph-mon[111235]: pgmap v3362: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:29.762 INFO:teuthology.orchestra.run.smithi039.stdout:2980707304148 2024-09-14T05:07:29.762 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-09-14T05:07:29.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:29 smithi129 ceph-mon[101316]: pgmap v3362: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:31.539 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:31 smithi039 ceph-mon[106470]: pgmap v3363: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:31.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:31 smithi039 ceph-mon[111235]: pgmap v3363: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:31.566 INFO:teuthology.orchestra.run.smithi039.stdout:3010772075213 2024-09-14T05:07:31.567 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-09-14T05:07:31.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:31 smithi129 ceph-mon[101316]: pgmap v3363: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:32.058 INFO:teuthology.orchestra.run.smithi039.stdout:5978594476317 2024-09-14T05:07:32.059 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-09-14T05:07:32.613 INFO:teuthology.orchestra.run.smithi039.stdout:167503725472 2024-09-14T05:07:32.614 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-09-14T05:07:32.999 INFO:teuthology.orchestra.run.smithi039.stdout:2924872729307 2024-09-14T05:07:32.999 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-09-14T05:07:33.042 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:07:33.496 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:33 smithi039 ceph-mon[106470]: pgmap v3364: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:33.496 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:33 smithi039 ceph-mon[111235]: pgmap v3364: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:33.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:33 smithi129 ceph-mon[101316]: pgmap v3364: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:34.033 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:07:34.034 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:07:34.036 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:07:34.038 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:07:34.041 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:07:34.838 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:07:34.839 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:07:35.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:35 smithi039 ceph-mon[106470]: pgmap v3365: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:35.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:35 smithi039 ceph-mon[111235]: pgmap v3365: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:35.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:35 smithi129 ceph-mon[101316]: pgmap v3365: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:35.979 INFO:teuthology.orchestra.run.smithi039.stdout:236223202200 2024-09-14T05:07:36.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:36 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1921201415' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-14T05:07:36.778 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:36 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1921201415' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-14T05:07:36.778 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:36 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1921201415' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 7} : dispatch 2024-09-14T05:07:37.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:37 smithi129 ceph-mon[101316]: pgmap v3366: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:37.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:37 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:07:37.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:37 smithi039 ceph-mon[106470]: pgmap v3366: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:37.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:37 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:07:37.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:37 smithi039 ceph-mon[111235]: pgmap v3366: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:37.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:37 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:07:38.977 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:07:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:07:38] "GET /metrics HTTP/1.1" 200 34310 "" "Prometheus/2.51.0" 2024-09-14T05:07:39.703 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:39 smithi039 ceph-mon[106470]: pgmap v3367: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:39.704 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:39 smithi039 ceph-mon[111235]: pgmap v3367: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:39.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:39 smithi129 ceph-mon[101316]: pgmap v3367: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:41.707 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:41 smithi039 ceph-mon[106470]: pgmap v3368: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:41.707 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:41 smithi039 ceph-mon[111235]: pgmap v3368: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:41.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:41 smithi129 ceph-mon[101316]: pgmap v3368: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:42.729 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:42 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1207802892' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-14T05:07:42.729 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:42 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1207802892' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-14T05:07:42.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:42 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1207802892' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 3} : dispatch 2024-09-14T05:07:42.937 INFO:teuthology.orchestra.run.smithi039.stdout:3010772075215 2024-09-14T05:07:43.013 INFO:teuthology.orchestra.run.smithi039.stdout:2881923056360 2024-09-14T05:07:43.680 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:43 smithi039 ceph-mon[106470]: pgmap v3369: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:43.681 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:43 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1546773119' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-14T05:07:43.681 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:43 smithi039 ceph-mon[111235]: pgmap v3369: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:43.681 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:43 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1546773119' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-14T05:07:43.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:43 smithi129 ceph-mon[101316]: pgmap v3369: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:43.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:43 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1546773119' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 0} : dispatch 2024-09-14T05:07:44.336 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223202198 got 236223202200 for osd.7 2024-09-14T05:07:44.336 DEBUG:teuthology.parallel:result is None 2024-09-14T05:07:44.667 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:44 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T05:07:44.667 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:44 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T05:07:44.714 INFO:teuthology.orchestra.run.smithi039.stdout:201863463838 2024-09-14T05:07:44.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:44 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "config dump", "format": "json"} : dispatch 2024-09-14T05:07:45.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:45 smithi039 ceph-mon[106470]: pgmap v3370: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:45.540 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:45 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/444807828' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-14T05:07:45.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:45 smithi039 ceph-mon[111235]: pgmap v3370: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:45.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:45 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/444807828' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-14T05:07:45.735 INFO:tasks.cephadm.ceph_manager.ceph:need seq 3010772075213 got 3010772075215 for osd.3 2024-09-14T05:07:45.735 DEBUG:teuthology.parallel:result is None 2024-09-14T05:07:45.750 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:45 smithi129 ceph-mon[101316]: pgmap v3370: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:45.750 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:45 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/444807828' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 6} : dispatch 2024-09-14T05:07:46.352 INFO:teuthology.orchestra.run.smithi039.stdout:2980707304151 2024-09-14T05:07:46.748 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:46 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3737081987' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-14T05:07:46.749 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:46 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3737081987' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-14T05:07:46.750 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2881923056357 got 2881923056360 for osd.0 2024-09-14T05:07:46.751 DEBUG:teuthology.parallel:result is None 2024-09-14T05:07:46.770 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:46 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3737081987' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 2} : dispatch 2024-09-14T05:07:47.115 INFO:teuthology.orchestra.run.smithi039.stdout:2924872729311 2024-09-14T05:07:47.603 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863463834 got 201863463838 for osd.6 2024-09-14T05:07:47.603 DEBUG:teuthology.parallel:result is None 2024-09-14T05:07:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:47 smithi129 ceph-mon[101316]: pgmap v3371: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:47.771 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:47 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1741530341' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-14T05:07:47.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:47 smithi039 ceph-mon[106470]: pgmap v3371: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:47.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:47 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1741530341' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-14T05:07:47.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:47 smithi039 ceph-mon[111235]: pgmap v3371: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:47.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:47 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1741530341' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 1} : dispatch 2024-09-14T05:07:48.456 INFO:teuthology.orchestra.run.smithi039.stdout:167503725475 2024-09-14T05:07:48.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:48 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/3812401762' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-14T05:07:48.790 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:07:48 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:07:48] "GET /metrics HTTP/1.1" 200 34310 "" "Prometheus/2.51.0" 2024-09-14T05:07:48.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:48 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/3812401762' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-14T05:07:49.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:48 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/3812401762' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 5} : dispatch 2024-09-14T05:07:49.489 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2980707304148 got 2980707304151 for osd.2 2024-09-14T05:07:49.489 DEBUG:teuthology.parallel:result is None 2024-09-14T05:07:49.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:49 smithi039 ceph-mon[106470]: pgmap v3372: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:49.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:49 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:07:49.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:49 smithi039 ceph-mon[106470]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:07:49.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:49 smithi039 ceph-mon[111235]: pgmap v3372: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:49.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:49 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:07:49.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:49 smithi039 ceph-mon[111235]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:07:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:49 smithi129 ceph-mon[101316]: pgmap v3372: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:49 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:07:50.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:49 smithi129 ceph-mon[101316]: from='mgr.25157 ' entity='mgr.y' 2024-09-14T05:07:50.456 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503725472 got 167503725475 for osd.5 2024-09-14T05:07:50.456 DEBUG:teuthology.parallel:result is None 2024-09-14T05:07:51.145 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2924872729307 got 2924872729311 for osd.1 2024-09-14T05:07:51.145 DEBUG:teuthology.parallel:result is None 2024-09-14T05:07:51.645 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:51 smithi039 ceph-mon[106470]: pgmap v3373: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:51.645 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:51 smithi039 ceph-mon[111235]: pgmap v3373: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:51.851 INFO:teuthology.orchestra.run.smithi039.stdout:5978594476322 2024-09-14T05:07:52.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:51 smithi129 ceph-mon[101316]: pgmap v3373: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:52.717 INFO:tasks.cephadm.ceph_manager.ceph:need seq 5978594476317 got 5978594476322 for osd.4 2024-09-14T05:07:52.717 DEBUG:teuthology.parallel:result is None 2024-09-14T05:07:52.717 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2024-09-14T05:07:52.717 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T05:07:52.927 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:52 smithi039 ceph-mon[106470]: from='client.? 172.21.15.39:0/1170499025' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-14T05:07:52.928 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:52 smithi039 ceph-mon[106470]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:07:52.928 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:52 smithi039 ceph-mon[111235]: from='client.? 172.21.15.39:0/1170499025' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-14T05:07:52.928 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:52 smithi039 ceph-mon[111235]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:07:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:52 smithi129 ceph-mon[101316]: from='client.? 172.21.15.39:0/1170499025' entity='client.admin' cmd={"prefix": "osd last-stat-seq", "id": 4} : dispatch 2024-09-14T05:07:53.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:52 smithi129 ceph-mon[101316]: from='mgr.25157 172.21.15.39:0/2350124006' entity='mgr.y' cmd={"prefix": "osd blocklist ls", "format": "json"} : dispatch 2024-09-14T05:07:53.278 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:07:53.725 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:53 smithi039 ceph-mon[106470]: pgmap v3374: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:53.726 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:53 smithi039 ceph-mon[111235]: pgmap v3374: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:54.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:53 smithi129 ceph-mon[101316]: pgmap v3374: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:55.433 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T05:07:55.434 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T05:07:55.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:55 smithi039 ceph-mon[106470]: pgmap v3375: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:55.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:55 smithi039 ceph-mon[111235]: pgmap v3375: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:56.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:55 smithi129 ceph-mon[101316]: pgmap v3375: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:56.474 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":3375,"stamp":"2024-09-14T05:07:54.140931+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":85,"num_bytes_recovered":337836,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4678,"ondisk_log_size":4678,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7244192,"kb_used_data":115584,"kb_used_omap":404,"kb_used_meta":7128107,"kb_avail":742717024,"statfs":{"total":767960285184,"available":760542232576,"internally_reserved":0,"allocated":118358016,"data_stored":68130163,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":414022,"internal_metadata":7299182266},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003030"},"pg_stats":[{"pgid":"1.1d","version":"349'149","reported_seq":963,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.616597+0000","last_change":"2024-09-14T05:05:52.331425+0000","last_active":"2024-09-14T05:07:06.616597+0000","last_peered":"2024-09-14T05:07:06.616597+0000","last_clean":"2024-09-14T05:07:06.616597+0000","last_became_active":"2024-09-14T05:05:52.330812+0000","last_became_peered":"2024-09-14T05:05:52.330812+0000","last_unstale":"2024-09-14T05:07:06.616597+0000","last_undegraded":"2024-09-14T05:07:06.616597+0000","last_fullsized":"2024-09-14T05:07:06.616597+0000","mapping_epoch":1625,"log_start":"0'0","ondisk_log_start":"0'0","created":1582,"last_epoch_clean":1626,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:04:04.688573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:04:04.688573+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:02:08.034833+0000","scrub_duration":24,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":2611,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.502021+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T05:07:06.502021+0000","last_peered":"2024-09-14T05:07:06.502021+0000","last_clean":"2024-09-14T05:07:06.502021+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T05:07:06.502021+0000","last_undegraded":"2024-09-14T05:07:06.502021+0000","last_fullsized":"2024-09-14T05:07:06.502021+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":54,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.225471+0000","last_change":"2024-09-14T05:06:50.899773+0000","last_active":"2024-09-14T05:07:02.225471+0000","last_peered":"2024-09-14T05:07:02.225471+0000","last_clean":"2024-09-14T05:07:02.225471+0000","last_became_active":"2024-09-14T05:06:50.899361+0000","last_became_peered":"2024-09-14T05:06:50.899361+0000","last_unstale":"2024-09-14T05:07:02.225471+0000","last_undegraded":"2024-09-14T05:07:02.225471+0000","last_fullsized":"2024-09-14T05:07:02.225471+0000","mapping_epoch":1649,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:40:51.266314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"349'149","reported_seq":1056,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.973985+0000","last_change":"2024-09-14T05:06:53.919206+0000","last_active":"2024-09-14T05:07:02.973985+0000","last_peered":"2024-09-14T05:07:02.973985+0000","last_clean":"2024-09-14T05:07:02.973985+0000","last_became_active":"2024-09-14T05:06:53.918790+0000","last_became_peered":"2024-09-14T05:06:53.918790+0000","last_unstale":"2024-09-14T05:07:02.973985+0000","last_undegraded":"2024-09-14T05:07:02.973985+0000","last_fullsized":"2024-09-14T05:07:02.973985+0000","mapping_epoch":1652,"log_start":"0'0","ondisk_log_start":"0'0","created":1582,"last_epoch_clean":1653,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:02:54.501356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:02:54.501356+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:01:52.526614+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":2208,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.811777+0000","last_change":"2024-09-14T05:06:50.896369+0000","last_active":"2024-09-14T05:07:06.811777+0000","last_peered":"2024-09-14T05:07:06.811777+0000","last_clean":"2024-09-14T05:07:06.811777+0000","last_became_active":"2024-09-14T05:06:50.895992+0000","last_became_peered":"2024-09-14T05:06:50.895992+0000","last_unstale":"2024-09-14T05:07:06.811777+0000","last_undegraded":"2024-09-14T05:07:06.811777+0000","last_fullsized":"2024-09-14T05:07:06.811777+0000","mapping_epoch":1649,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:05:54.563890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":2097,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.811272+0000","last_change":"2024-09-14T05:06:38.804334+0000","last_active":"2024-09-14T05:07:06.811272+0000","last_peered":"2024-09-14T05:07:06.811272+0000","last_clean":"2024-09-14T05:07:06.811272+0000","last_became_active":"2024-09-14T05:06:38.803681+0000","last_became_peered":"2024-09-14T05:06:38.803681+0000","last_unstale":"2024-09-14T05:07:06.811272+0000","last_undegraded":"2024-09-14T05:07:06.811272+0000","last_fullsized":"2024-09-14T05:07:06.811272+0000","mapping_epoch":1642,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:49:55.786021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1704,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226704+0000","last_change":"2024-09-14T05:06:50.906848+0000","last_active":"2024-09-14T05:07:02.226704+0000","last_peered":"2024-09-14T05:07:02.226704+0000","last_clean":"2024-09-14T05:07:02.226704+0000","last_became_active":"2024-09-14T05:06:50.901534+0000","last_became_peered":"2024-09-14T05:06:50.901534+0000","last_unstale":"2024-09-14T05:07:02.226704+0000","last_undegraded":"2024-09-14T05:07:02.226704+0000","last_fullsized":"2024-09-14T05:07:02.226704+0000","mapping_epoch":1649,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:46:51.000071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":45,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226355+0000","last_change":"2024-09-14T05:05:49.412104+0000","last_active":"2024-09-14T05:07:02.226355+0000","last_peered":"2024-09-14T05:07:02.226355+0000","last_clean":"2024-09-14T05:07:02.226355+0000","last_became_active":"2024-09-14T05:05:49.409810+0000","last_became_peered":"2024-09-14T05:05:49.409810+0000","last_unstale":"2024-09-14T05:07:02.226355+0000","last_undegraded":"2024-09-14T05:07:02.226355+0000","last_fullsized":"2024-09-14T05:07:02.226355+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:07:07.657805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"349'149","reported_seq":677,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.811489+0000","last_change":"2024-09-14T05:06:43.455120+0000","last_active":"2024-09-14T05:07:06.811489+0000","last_peered":"2024-09-14T05:07:06.811489+0000","last_clean":"2024-09-14T05:07:06.811489+0000","last_became_active":"2024-09-14T05:06:38.811232+0000","last_became_peered":"2024-09-14T05:06:38.811232+0000","last_unstale":"2024-09-14T05:07:06.811489+0000","last_undegraded":"2024-09-14T05:07:06.811489+0000","last_fullsized":"2024-09-14T05:07:06.811489+0000","mapping_epoch":1642,"log_start":"0'0","ondisk_log_start":"0'0","created":1582,"last_epoch_clean":1643,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:06:43.455053+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:06:43.455053+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:23:54.663847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1706,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226217+0000","last_change":"2024-09-14T05:05:49.409062+0000","last_active":"2024-09-14T05:07:02.226217+0000","last_peered":"2024-09-14T05:07:02.226217+0000","last_clean":"2024-09-14T05:07:02.226217+0000","last_became_active":"2024-09-14T05:05:49.399865+0000","last_became_peered":"2024-09-14T05:05:49.399865+0000","last_unstale":"2024-09-14T05:07:02.226217+0000","last_undegraded":"2024-09-14T05:07:02.226217+0000","last_fullsized":"2024-09-14T05:07:02.226217+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:06:44.128353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":54,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.225950+0000","last_change":"2024-09-14T05:06:12.769686+0000","last_active":"2024-09-14T05:07:02.225950+0000","last_peered":"2024-09-14T05:07:02.225950+0000","last_clean":"2024-09-14T05:07:02.225950+0000","last_became_active":"2024-09-14T05:06:12.769312+0000","last_became_peered":"2024-09-14T05:06:12.769312+0000","last_unstale":"2024-09-14T05:07:02.225950+0000","last_undegraded":"2024-09-14T05:07:02.225950+0000","last_fullsized":"2024-09-14T05:07:02.225950+0000","mapping_epoch":1630,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1631,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:04:02.262440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"349'149","reported_seq":1703,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.810975+0000","last_change":"2024-09-14T05:06:39.361293+0000","last_active":"2024-09-14T05:07:06.810975+0000","last_peered":"2024-09-14T05:07:06.810975+0000","last_clean":"2024-09-14T05:07:06.810975+0000","last_became_active":"2024-09-14T05:06:38.803337+0000","last_became_peered":"2024-09-14T05:06:38.803337+0000","last_unstale":"2024-09-14T05:07:06.810975+0000","last_undegraded":"2024-09-14T05:07:06.810975+0000","last_fullsized":"2024-09-14T05:07:06.810975+0000","mapping_epoch":1642,"log_start":"0'0","ondisk_log_start":"0'0","created":1582,"last_epoch_clean":1643,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:06:39.361246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:06:39.361246+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:04:23.837623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":480,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.501688+0000","last_change":"2024-09-14T05:05:49.410696+0000","last_active":"2024-09-14T05:07:06.501688+0000","last_peered":"2024-09-14T05:07:06.501688+0000","last_clean":"2024-09-14T05:07:06.501688+0000","last_became_active":"2024-09-14T05:05:49.410138+0000","last_became_peered":"2024-09-14T05:05:49.410138+0000","last_unstale":"2024-09-14T05:07:06.501688+0000","last_undegraded":"2024-09-14T05:07:06.501688+0000","last_fullsized":"2024-09-14T05:07:06.501688+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:51:15.835973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2606,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.254613+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T05:07:06.254613+0000","last_peered":"2024-09-14T05:07:06.254613+0000","last_clean":"2024-09-14T05:07:06.254613+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T05:07:06.254613+0000","last_undegraded":"2024-09-14T05:07:06.254613+0000","last_fullsized":"2024-09-14T05:07:06.254613+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.18","version":"349'149","reported_seq":1024,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226001+0000","last_change":"2024-09-14T05:03:04.296355+0000","last_active":"2024-09-14T05:07:02.226001+0000","last_peered":"2024-09-14T05:07:02.226001+0000","last_clean":"2024-09-14T05:07:02.226001+0000","last_became_active":"2024-09-14T05:03:04.295977+0000","last_became_peered":"2024-09-14T05:03:04.295977+0000","last_unstale":"2024-09-14T05:07:02.226001+0000","last_undegraded":"2024-09-14T05:07:02.226001+0000","last_fullsized":"2024-09-14T05:07:02.226001+0000","mapping_epoch":1587,"log_start":"0'0","ondisk_log_start":"0'0","created":1582,"last_epoch_clean":1588,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:02:54.449355+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:02:54.449355+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:11:30.881287+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":45,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226116+0000","last_change":"2024-09-14T05:05:49.411615+0000","last_active":"2024-09-14T05:07:02.226116+0000","last_peered":"2024-09-14T05:07:02.226116+0000","last_clean":"2024-09-14T05:07:02.226116+0000","last_became_active":"2024-09-14T05:05:49.411224+0000","last_became_peered":"2024-09-14T05:05:49.411224+0000","last_unstale":"2024-09-14T05:07:02.226116+0000","last_undegraded":"2024-09-14T05:07:02.226116+0000","last_fullsized":"2024-09-14T05:07:02.226116+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:27:15.272705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2558,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.811495+0000","last_change":"2024-09-14T05:06:38.810400+0000","last_active":"2024-09-14T05:07:06.811495+0000","last_peered":"2024-09-14T05:07:06.811495+0000","last_clean":"2024-09-14T05:07:06.811495+0000","last_became_active":"2024-09-14T05:06:38.809963+0000","last_became_peered":"2024-09-14T05:06:38.809963+0000","last_unstale":"2024-09-14T05:07:06.811495+0000","last_undegraded":"2024-09-14T05:07:06.811495+0000","last_fullsized":"2024-09-14T05:07:06.811495+0000","mapping_epoch":1642,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:11:06.301818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"349'149","reported_seq":996,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.225765+0000","last_change":"2024-09-14T05:05:52.334643+0000","last_active":"2024-09-14T05:07:02.225765+0000","last_peered":"2024-09-14T05:07:02.225765+0000","last_clean":"2024-09-14T05:07:02.225765+0000","last_became_active":"2024-09-14T05:05:52.334266+0000","last_became_peered":"2024-09-14T05:05:52.334266+0000","last_unstale":"2024-09-14T05:07:02.225765+0000","last_undegraded":"2024-09-14T05:07:02.225765+0000","last_fullsized":"2024-09-14T05:07:02.225765+0000","mapping_epoch":1625,"log_start":"0'0","ondisk_log_start":"0'0","created":1582,"last_epoch_clean":1626,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:04:53.630279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:04:53.630279+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:45:34.361978+0000","scrub_duration":0.0058720990000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":2606,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.616174+0000","last_change":"2024-09-14T05:05:49.410644+0000","last_active":"2024-09-14T05:07:06.616174+0000","last_peered":"2024-09-14T05:07:06.616174+0000","last_clean":"2024-09-14T05:07:06.616174+0000","last_became_active":"2024-09-14T05:05:49.410321+0000","last_became_peered":"2024-09-14T05:05:49.410321+0000","last_unstale":"2024-09-14T05:07:06.616174+0000","last_undegraded":"2024-09-14T05:07:06.616174+0000","last_fullsized":"2024-09-14T05:07:06.616174+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:57:11.897463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2627,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.502042+0000","last_change":"2024-09-14T04:54:52.250695+0000","last_active":"2024-09-14T05:07:06.502042+0000","last_peered":"2024-09-14T05:07:06.502042+0000","last_clean":"2024-09-14T05:07:06.502042+0000","last_became_active":"2024-09-14T04:54:52.250301+0000","last_became_peered":"2024-09-14T04:54:52.250301+0000","last_unstale":"2024-09-14T05:07:06.502042+0000","last_undegraded":"2024-09-14T05:07:06.502042+0000","last_fullsized":"2024-09-14T05:07:06.502042+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:42.060905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"349'149","reported_seq":677,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226137+0000","last_change":"2024-09-14T05:06:01.981981+0000","last_active":"2024-09-14T05:07:02.226137+0000","last_peered":"2024-09-14T05:07:02.226137+0000","last_clean":"2024-09-14T05:07:02.226137+0000","last_became_active":"2024-09-14T05:05:52.326854+0000","last_became_peered":"2024-09-14T05:05:52.326854+0000","last_unstale":"2024-09-14T05:07:02.226137+0000","last_undegraded":"2024-09-14T05:07:02.226137+0000","last_fullsized":"2024-09-14T05:07:02.226137+0000","mapping_epoch":1625,"log_start":"0'0","ondisk_log_start":"0'0","created":1582,"last_epoch_clean":1626,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:06:01.981845+0000","last_deep_scrub":"349'149","last_deep_scrub_stamp":"2024-09-14T05:02:59.622762+0000","last_clean_scrub_stamp":"2024-09-14T05:06:01.981845+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:09:55.213179+0000","scrub_duration":0.0018431260000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2096,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.811038+0000","last_change":"2024-09-14T05:06:38.812290+0000","last_active":"2024-09-14T05:07:06.811038+0000","last_peered":"2024-09-14T05:07:06.811038+0000","last_clean":"2024-09-14T05:07:06.811038+0000","last_became_active":"2024-09-14T05:06:38.812004+0000","last_became_peered":"2024-09-14T05:06:38.812004+0000","last_unstale":"2024-09-14T05:07:06.811038+0000","last_undegraded":"2024-09-14T05:07:06.811038+0000","last_fullsized":"2024-09-14T05:07:06.811038+0000","mapping_epoch":1642,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:06:51.292857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":18,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.483875+0000","last_change":"2024-09-14T05:06:50.901735+0000","last_active":"2024-09-14T05:07:02.483875+0000","last_peered":"2024-09-14T05:07:02.483875+0000","last_clean":"2024-09-14T05:07:02.483875+0000","last_became_active":"2024-09-14T05:06:50.901416+0000","last_became_peered":"2024-09-14T05:06:50.901416+0000","last_unstale":"2024-09-14T05:07:02.483875+0000","last_undegraded":"2024-09-14T05:07:02.483875+0000","last_fullsized":"2024-09-14T05:07:02.483875+0000","mapping_epoch":1649,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:57:59.096226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"349'149","reported_seq":862,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.974108+0000","last_change":"2024-09-14T05:04:43.777195+0000","last_active":"2024-09-14T05:07:02.974108+0000","last_peered":"2024-09-14T05:07:02.974108+0000","last_clean":"2024-09-14T05:07:02.974108+0000","last_became_active":"2024-09-14T05:03:04.293853+0000","last_became_peered":"2024-09-14T05:03:04.293853+0000","last_unstale":"2024-09-14T05:07:02.974108+0000","last_undegraded":"2024-09-14T05:07:02.974108+0000","last_fullsized":"2024-09-14T05:07:02.974108+0000","mapping_epoch":1587,"log_start":"0'0","ondisk_log_start":"0'0","created":1582,"last_epoch_clean":1588,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:04:43.777093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:04:43.777093+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:25:50.652208+0000","scrub_duration":0.0098257610000000006,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":320,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.225716+0000","last_change":"2024-09-14T05:06:50.902587+0000","last_active":"2024-09-14T05:07:02.225716+0000","last_peered":"2024-09-14T05:07:02.225716+0000","last_clean":"2024-09-14T05:07:02.225716+0000","last_became_active":"2024-09-14T05:06:50.902273+0000","last_became_peered":"2024-09-14T05:06:50.902273+0000","last_unstale":"2024-09-14T05:07:02.225716+0000","last_undegraded":"2024-09-14T05:07:02.225716+0000","last_fullsized":"2024-09-14T05:07:02.225716+0000","mapping_epoch":1649,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:43:33.240582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2599,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.502111+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T05:07:06.502111+0000","last_peered":"2024-09-14T05:07:06.502111+0000","last_clean":"2024-09-14T05:07:06.502111+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T05:07:06.502111+0000","last_undegraded":"2024-09-14T05:07:06.502111+0000","last_fullsized":"2024-09-14T05:07:06.502111+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.14","version":"349'149","reported_seq":1096,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.254272+0000","last_change":"2024-09-14T05:05:52.326841+0000","last_active":"2024-09-14T05:07:06.254272+0000","last_peered":"2024-09-14T05:07:06.254272+0000","last_clean":"2024-09-14T05:07:06.254272+0000","last_became_active":"2024-09-14T05:05:52.326258+0000","last_became_peered":"2024-09-14T05:05:52.326258+0000","last_unstale":"2024-09-14T05:07:06.254272+0000","last_undegraded":"2024-09-14T05:07:06.254272+0000","last_fullsized":"2024-09-14T05:07:06.254272+0000","mapping_epoch":1625,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1626,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:39.545724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:39.545724+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:36.944653+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":2182,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.811518+0000","last_change":"2024-09-14T05:06:38.811444+0000","last_active":"2024-09-14T05:07:06.811518+0000","last_peered":"2024-09-14T05:07:06.811518+0000","last_clean":"2024-09-14T05:07:06.811518+0000","last_became_active":"2024-09-14T05:06:38.810235+0000","last_became_peered":"2024-09-14T05:06:38.810235+0000","last_unstale":"2024-09-14T05:07:06.811518+0000","last_undegraded":"2024-09-14T05:07:06.811518+0000","last_fullsized":"2024-09-14T05:07:06.811518+0000","mapping_epoch":1642,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:03:35.247423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1651,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226381+0000","last_change":"2024-09-14T04:42:49.938505+0000","last_active":"2024-09-14T05:07:02.226381+0000","last_peered":"2024-09-14T05:07:02.226381+0000","last_clean":"2024-09-14T05:07:02.226381+0000","last_became_active":"2024-09-14T04:42:49.938180+0000","last_became_peered":"2024-09-14T04:42:49.938180+0000","last_unstale":"2024-09-14T05:07:02.226381+0000","last_undegraded":"2024-09-14T05:07:02.226381+0000","last_fullsized":"2024-09-14T05:07:02.226381+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:05.929479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.17","version":"349'149","reported_seq":915,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226488+0000","last_change":"2024-09-14T05:04:43.753778+0000","last_active":"2024-09-14T05:07:02.226488+0000","last_peered":"2024-09-14T05:07:02.226488+0000","last_clean":"2024-09-14T05:07:02.226488+0000","last_became_active":"2024-09-14T05:03:04.281302+0000","last_became_peered":"2024-09-14T05:03:04.281302+0000","last_unstale":"2024-09-14T05:07:02.226488+0000","last_undegraded":"2024-09-14T05:07:02.226488+0000","last_fullsized":"2024-09-14T05:07:02.226488+0000","mapping_epoch":1587,"log_start":"0'0","ondisk_log_start":"0'0","created":1582,"last_epoch_clean":1588,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:04:43.753698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:04:43.753698+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:12:27.317820+0000","scrub_duration":0.0014490550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1700,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.974238+0000","last_change":"2024-09-14T05:05:49.423629+0000","last_active":"2024-09-14T05:07:02.974238+0000","last_peered":"2024-09-14T05:07:02.974238+0000","last_clean":"2024-09-14T05:07:02.974238+0000","last_became_active":"2024-09-14T05:05:49.407879+0000","last_became_peered":"2024-09-14T05:05:49.407879+0000","last_unstale":"2024-09-14T05:07:02.974238+0000","last_undegraded":"2024-09-14T05:07:02.974238+0000","last_fullsized":"2024-09-14T05:07:02.974238+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:43:49.292723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":555,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.616295+0000","last_change":"2024-09-14T05:06:50.897034+0000","last_active":"2024-09-14T05:07:06.616295+0000","last_peered":"2024-09-14T05:07:06.616295+0000","last_clean":"2024-09-14T05:07:06.616295+0000","last_became_active":"2024-09-14T05:06:50.896679+0000","last_became_peered":"2024-09-14T05:06:50.896679+0000","last_unstale":"2024-09-14T05:07:06.616295+0000","last_undegraded":"2024-09-14T05:07:06.616295+0000","last_fullsized":"2024-09-14T05:07:06.616295+0000","mapping_epoch":1649,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:34:33.381326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"349'149","reported_seq":781,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.616721+0000","last_change":"2024-09-14T05:03:35.645894+0000","last_active":"2024-09-14T05:07:06.616721+0000","last_peered":"2024-09-14T05:07:06.616721+0000","last_clean":"2024-09-14T05:07:06.616721+0000","last_became_active":"2024-09-14T05:03:04.309263+0000","last_became_peered":"2024-09-14T05:03:04.309263+0000","last_unstale":"2024-09-14T05:07:06.616721+0000","last_undegraded":"2024-09-14T05:07:06.616721+0000","last_fullsized":"2024-09-14T05:07:06.616721+0000","mapping_epoch":1587,"log_start":"0'0","ondisk_log_start":"0'0","created":1582,"last_epoch_clean":1588,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:03:35.645824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:03:35.645824+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:53:53.382747+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":55,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226499+0000","last_change":"2024-09-14T05:05:49.419211+0000","last_active":"2024-09-14T05:07:02.226499+0000","last_peered":"2024-09-14T05:07:02.226499+0000","last_clean":"2024-09-14T05:07:02.226499+0000","last_became_active":"2024-09-14T05:05:49.409977+0000","last_became_peered":"2024-09-14T05:05:49.409977+0000","last_unstale":"2024-09-14T05:07:02.226499+0000","last_undegraded":"2024-09-14T05:07:02.226499+0000","last_fullsized":"2024-09-14T05:07:02.226499+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:06:35.797110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":45,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226190+0000","last_change":"2024-09-14T05:05:49.411270+0000","last_active":"2024-09-14T05:07:02.226190+0000","last_peered":"2024-09-14T05:07:02.226190+0000","last_clean":"2024-09-14T05:07:02.226190+0000","last_became_active":"2024-09-14T05:05:49.410956+0000","last_became_peered":"2024-09-14T05:05:49.410956+0000","last_unstale":"2024-09-14T05:07:02.226190+0000","last_undegraded":"2024-09-14T05:07:02.226190+0000","last_fullsized":"2024-09-14T05:07:02.226190+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:31:28.109175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"349'149","reported_seq":878,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.811301+0000","last_change":"2024-09-14T05:06:42.443038+0000","last_active":"2024-09-14T05:07:06.811301+0000","last_peered":"2024-09-14T05:07:06.811301+0000","last_clean":"2024-09-14T05:07:06.811301+0000","last_became_active":"2024-09-14T05:06:38.811366+0000","last_became_peered":"2024-09-14T05:06:38.811366+0000","last_unstale":"2024-09-14T05:07:06.811301+0000","last_undegraded":"2024-09-14T05:07:06.811301+0000","last_fullsized":"2024-09-14T05:07:06.811301+0000","mapping_epoch":1642,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1643,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:06:42.442989+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:06:42.442989+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:51:56.337012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":46,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.253227+0000","last_change":"2024-09-14T05:05:49.410296+0000","last_active":"2024-09-14T05:07:06.253227+0000","last_peered":"2024-09-14T05:07:06.253227+0000","last_clean":"2024-09-14T05:07:06.253227+0000","last_became_active":"2024-09-14T05:05:49.409663+0000","last_became_peered":"2024-09-14T05:05:49.409663+0000","last_unstale":"2024-09-14T05:07:06.253227+0000","last_undegraded":"2024-09-14T05:07:06.253227+0000","last_fullsized":"2024-09-14T05:07:06.253227+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:19:56.206619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1668,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.811496+0000","last_change":"2024-09-14T04:54:52.242207+0000","last_active":"2024-09-14T05:07:06.811496+0000","last_peered":"2024-09-14T05:07:06.811496+0000","last_clean":"2024-09-14T05:07:06.811496+0000","last_became_active":"2024-09-14T04:54:52.241840+0000","last_became_peered":"2024-09-14T04:54:52.241840+0000","last_unstale":"2024-09-14T05:07:06.811496+0000","last_undegraded":"2024-09-14T05:07:06.811496+0000","last_fullsized":"2024-09-14T05:07:06.811496+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:37.899666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":2671,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.502000+0000","last_change":"2024-09-14T05:06:12.541282+0000","last_active":"2024-09-14T05:07:06.502000+0000","last_peered":"2024-09-14T05:07:06.502000+0000","last_clean":"2024-09-14T05:07:06.502000+0000","last_became_active":"2024-09-14T05:06:12.538805+0000","last_became_peered":"2024-09-14T05:06:12.538805+0000","last_unstale":"2024-09-14T05:07:06.502000+0000","last_undegraded":"2024-09-14T05:07:06.502000+0000","last_fullsized":"2024-09-14T05:07:06.502000+0000","mapping_epoch":1630,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1631,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:47:22.130803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"349'149","reported_seq":920,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.254485+0000","last_change":"2024-09-14T05:06:53.920679+0000","last_active":"2024-09-14T05:07:06.254485+0000","last_peered":"2024-09-14T05:07:06.254485+0000","last_clean":"2024-09-14T05:07:06.254485+0000","last_became_active":"2024-09-14T05:06:53.920222+0000","last_became_peered":"2024-09-14T05:06:53.920222+0000","last_unstale":"2024-09-14T05:07:06.254485+0000","last_undegraded":"2024-09-14T05:07:06.254485+0000","last_fullsized":"2024-09-14T05:07:06.254485+0000","mapping_epoch":1652,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1653,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:04:05.388558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:04:05.388558+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:38:37.384495+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":64,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.253928+0000","last_change":"2024-09-14T05:06:12.769094+0000","last_active":"2024-09-14T05:07:06.253928+0000","last_peered":"2024-09-14T05:07:06.253928+0000","last_clean":"2024-09-14T05:07:06.253928+0000","last_became_active":"2024-09-14T05:06:12.768782+0000","last_became_peered":"2024-09-14T05:06:12.768782+0000","last_unstale":"2024-09-14T05:07:06.253928+0000","last_undegraded":"2024-09-14T05:07:06.253928+0000","last_fullsized":"2024-09-14T05:07:06.253928+0000","mapping_epoch":1630,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1631,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:28:35.561669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":607,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.616876+0000","last_change":"2024-09-14T05:05:49.411503+0000","last_active":"2024-09-14T05:07:06.616876+0000","last_peered":"2024-09-14T05:07:06.616876+0000","last_clean":"2024-09-14T05:07:06.616876+0000","last_became_active":"2024-09-14T05:05:49.411159+0000","last_became_peered":"2024-09-14T05:05:49.411159+0000","last_unstale":"2024-09-14T05:07:06.616876+0000","last_undegraded":"2024-09-14T05:07:06.616876+0000","last_fullsized":"2024-09-14T05:07:06.616876+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:31:42.831156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":2061,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.811859+0000","last_change":"2024-09-14T05:06:38.810126+0000","last_active":"2024-09-14T05:07:06.811859+0000","last_peered":"2024-09-14T05:07:06.811859+0000","last_clean":"2024-09-14T05:07:06.811859+0000","last_became_active":"2024-09-14T05:06:38.809374+0000","last_became_peered":"2024-09-14T05:06:38.809374+0000","last_unstale":"2024-09-14T05:07:06.811859+0000","last_undegraded":"2024-09-14T05:07:06.811859+0000","last_fullsized":"2024-09-14T05:07:06.811859+0000","mapping_epoch":1642,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:36:06.638778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"349'149","reported_seq":736,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.483821+0000","last_change":"2024-09-14T05:06:53.922799+0000","last_active":"2024-09-14T05:07:02.483821+0000","last_peered":"2024-09-14T05:07:02.483821+0000","last_clean":"2024-09-14T05:07:02.483821+0000","last_became_active":"2024-09-14T05:06:53.921763+0000","last_became_peered":"2024-09-14T05:06:53.921763+0000","last_unstale":"2024-09-14T05:07:02.483821+0000","last_undegraded":"2024-09-14T05:07:02.483821+0000","last_fullsized":"2024-09-14T05:07:02.483821+0000","mapping_epoch":1652,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1653,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:05:52.621968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:05:52.621968+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:12:16.401190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2594,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.501899+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T05:07:06.501899+0000","last_peered":"2024-09-14T05:07:06.501899+0000","last_clean":"2024-09-14T05:07:06.501899+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T05:07:06.501899+0000","last_undegraded":"2024-09-14T05:07:06.501899+0000","last_fullsized":"2024-09-14T05:07:06.501899+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1700,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:03.012597+0000","last_change":"2024-09-14T05:05:49.407464+0000","last_active":"2024-09-14T05:07:03.012597+0000","last_peered":"2024-09-14T05:07:03.012597+0000","last_clean":"2024-09-14T05:07:03.012597+0000","last_became_active":"2024-09-14T05:05:49.407096+0000","last_became_peered":"2024-09-14T05:05:49.407096+0000","last_unstale":"2024-09-14T05:07:03.012597+0000","last_undegraded":"2024-09-14T05:07:03.012597+0000","last_fullsized":"2024-09-14T05:07:03.012597+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:10:19.818430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":754,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.616581+0000","last_change":"2024-09-14T05:06:25.813154+0000","last_active":"2024-09-14T05:07:06.616581+0000","last_peered":"2024-09-14T05:07:06.616581+0000","last_clean":"2024-09-14T05:07:06.616581+0000","last_became_active":"2024-09-14T05:06:25.812853+0000","last_became_peered":"2024-09-14T05:06:25.812853+0000","last_unstale":"2024-09-14T05:07:06.616581+0000","last_undegraded":"2024-09-14T05:07:06.616581+0000","last_fullsized":"2024-09-14T05:07:06.616581+0000","mapping_epoch":1637,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:42:56.670205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"349'149","reported_seq":866,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.483876+0000","last_change":"2024-09-14T05:06:53.921666+0000","last_active":"2024-09-14T05:07:02.483876+0000","last_peered":"2024-09-14T05:07:02.483876+0000","last_clean":"2024-09-14T05:07:02.483876+0000","last_became_active":"2024-09-14T05:06:53.921256+0000","last_became_peered":"2024-09-14T05:06:53.921256+0000","last_unstale":"2024-09-14T05:07:02.483876+0000","last_undegraded":"2024-09-14T05:07:02.483876+0000","last_fullsized":"2024-09-14T05:07:02.483876+0000","mapping_epoch":1652,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1653,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:06:41.414786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:06:41.414786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:10:03.630703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":563,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.616263+0000","last_change":"2024-09-14T05:05:49.409973+0000","last_active":"2024-09-14T05:07:06.616263+0000","last_peered":"2024-09-14T05:07:06.616263+0000","last_clean":"2024-09-14T05:07:06.616263+0000","last_became_active":"2024-09-14T05:05:49.409659+0000","last_became_peered":"2024-09-14T05:05:49.409659+0000","last_unstale":"2024-09-14T05:07:06.616263+0000","last_undegraded":"2024-09-14T05:07:06.616263+0000","last_fullsized":"2024-09-14T05:07:06.616263+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:17:33.620109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":2245,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.616808+0000","last_change":"2024-09-14T05:06:25.756161+0000","last_active":"2024-09-14T05:07:06.616808+0000","last_peered":"2024-09-14T05:07:06.616808+0000","last_clean":"2024-09-14T05:07:06.616808+0000","last_became_active":"2024-09-14T05:06:25.755843+0000","last_became_peered":"2024-09-14T05:06:25.755843+0000","last_unstale":"2024-09-14T05:07:06.616808+0000","last_undegraded":"2024-09-14T05:07:06.616808+0000","last_fullsized":"2024-09-14T05:07:06.616808+0000","mapping_epoch":1637,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:36:34.523283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1654,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226721+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T05:07:02.226721+0000","last_peered":"2024-09-14T05:07:02.226721+0000","last_clean":"2024-09-14T05:07:02.226721+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T05:07:02.226721+0000","last_undegraded":"2024-09-14T05:07:02.226721+0000","last_fullsized":"2024-09-14T05:07:02.226721+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"349'149","reported_seq":985,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226791+0000","last_change":"2024-09-14T05:00:33.562385+0000","last_active":"2024-09-14T05:07:02.226791+0000","last_peered":"2024-09-14T05:07:02.226791+0000","last_clean":"2024-09-14T05:07:02.226791+0000","last_became_active":"2024-09-14T05:00:01.421507+0000","last_became_peered":"2024-09-14T05:00:01.421507+0000","last_unstale":"2024-09-14T05:07:02.226791+0000","last_undegraded":"2024-09-14T05:07:02.226791+0000","last_fullsized":"2024-09-14T05:07:02.226791+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1544,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:33.562211+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:33.562211+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:32:46.187217+0000","scrub_duration":0.0066960869999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":790,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.616571+0000","last_change":"2024-09-14T05:06:25.812988+0000","last_active":"2024-09-14T05:07:06.616571+0000","last_peered":"2024-09-14T05:07:06.616571+0000","last_clean":"2024-09-14T05:07:06.616571+0000","last_became_active":"2024-09-14T05:06:25.812670+0000","last_became_peered":"2024-09-14T05:06:25.812670+0000","last_unstale":"2024-09-14T05:07:06.616571+0000","last_undegraded":"2024-09-14T05:07:06.616571+0000","last_fullsized":"2024-09-14T05:07:06.616571+0000","mapping_epoch":1637,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:54:26.893016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":46,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.253717+0000","last_change":"2024-09-14T05:05:49.407964+0000","last_active":"2024-09-14T05:07:06.253717+0000","last_peered":"2024-09-14T05:07:06.253717+0000","last_clean":"2024-09-14T05:07:06.253717+0000","last_became_active":"2024-09-14T05:05:49.407494+0000","last_became_peered":"2024-09-14T05:05:49.407494+0000","last_unstale":"2024-09-14T05:07:06.253717+0000","last_undegraded":"2024-09-14T05:07:06.253717+0000","last_fullsized":"2024-09-14T05:07:06.253717+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:39:57.277245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1682,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.225832+0000","last_change":"2024-09-14T05:06:50.909013+0000","last_active":"2024-09-14T05:07:02.225832+0000","last_peered":"2024-09-14T05:07:02.225832+0000","last_clean":"2024-09-14T05:07:02.225832+0000","last_became_active":"2024-09-14T05:06:50.903124+0000","last_became_peered":"2024-09-14T05:06:50.903124+0000","last_unstale":"2024-09-14T05:07:02.225832+0000","last_undegraded":"2024-09-14T05:07:02.225832+0000","last_fullsized":"2024-09-14T05:07:02.225832+0000","mapping_epoch":1649,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:16:15.260181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"349'149","reported_seq":922,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.557681+0000","last_change":"2024-09-14T05:05:52.333791+0000","last_active":"2024-09-14T05:07:06.557681+0000","last_peered":"2024-09-14T05:07:06.557681+0000","last_clean":"2024-09-14T05:07:06.557681+0000","last_became_active":"2024-09-14T05:05:52.333449+0000","last_became_peered":"2024-09-14T05:05:52.333449+0000","last_unstale":"2024-09-14T05:07:06.557681+0000","last_undegraded":"2024-09-14T05:07:06.557681+0000","last_fullsized":"2024-09-14T05:07:06.557681+0000","mapping_epoch":1625,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1626,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:02.067463+0000","last_deep_scrub":"349'149","last_deep_scrub_stamp":"2024-09-14T05:00:02.067463+0000","last_clean_scrub_stamp":"2024-09-14T05:00:02.067463+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:38:19.134801+0000","scrub_duration":0.0033042340000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":541,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.616851+0000","last_change":"2024-09-14T04:44:14.489235+0000","last_active":"2024-09-14T05:07:06.616851+0000","last_peered":"2024-09-14T05:07:06.616851+0000","last_clean":"2024-09-14T05:07:06.616851+0000","last_became_active":"2024-09-14T04:44:14.488950+0000","last_became_peered":"2024-09-14T04:44:14.488950+0000","last_unstale":"2024-09-14T05:07:06.616851+0000","last_undegraded":"2024-09-14T05:07:06.616851+0000","last_fullsized":"2024-09-14T05:07:06.616851+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:54.441144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":2236,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.616198+0000","last_change":"2024-09-14T05:06:50.896040+0000","last_active":"2024-09-14T05:07:06.616198+0000","last_peered":"2024-09-14T05:07:06.616198+0000","last_clean":"2024-09-14T05:07:06.616198+0000","last_became_active":"2024-09-14T05:06:50.895697+0000","last_became_peered":"2024-09-14T05:06:50.895697+0000","last_unstale":"2024-09-14T05:07:06.616198+0000","last_undegraded":"2024-09-14T05:07:06.616198+0000","last_fullsized":"2024-09-14T05:07:06.616198+0000","mapping_epoch":1649,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:29:45.946979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2615,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.254386+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T05:07:06.254386+0000","last_peered":"2024-09-14T05:07:06.254386+0000","last_clean":"2024-09-14T05:07:06.254386+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T05:07:06.254386+0000","last_undegraded":"2024-09-14T05:07:06.254386+0000","last_fullsized":"2024-09-14T05:07:06.254386+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"349'149","reported_seq":924,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.616472+0000","last_change":"2024-09-14T05:06:53.920045+0000","last_active":"2024-09-14T05:07:06.616472+0000","last_peered":"2024-09-14T05:07:06.616472+0000","last_clean":"2024-09-14T05:07:06.616472+0000","last_became_active":"2024-09-14T05:06:53.919631+0000","last_became_peered":"2024-09-14T05:06:53.919631+0000","last_unstale":"2024-09-14T05:07:06.616472+0000","last_undegraded":"2024-09-14T05:07:06.616472+0000","last_fullsized":"2024-09-14T05:07:06.616472+0000","mapping_epoch":1652,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1653,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:06:25.753556+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:06:25.753556+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:59:50.923576+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2615,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.502132+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T05:07:06.502132+0000","last_peered":"2024-09-14T05:07:06.502132+0000","last_clean":"2024-09-14T05:07:06.502132+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T05:07:06.502132+0000","last_undegraded":"2024-09-14T05:07:06.502132+0000","last_fullsized":"2024-09-14T05:07:06.502132+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2650,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.501649+0000","last_change":"2024-09-14T05:05:49.409878+0000","last_active":"2024-09-14T05:07:06.501649+0000","last_peered":"2024-09-14T05:07:06.501649+0000","last_clean":"2024-09-14T05:07:06.501649+0000","last_became_active":"2024-09-14T05:05:49.409559+0000","last_became_peered":"2024-09-14T05:05:49.409559+0000","last_unstale":"2024-09-14T05:07:06.501649+0000","last_undegraded":"2024-09-14T05:07:06.501649+0000","last_fullsized":"2024-09-14T05:07:06.501649+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:25:46.517122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1683,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.973730+0000","last_change":"2024-09-14T04:42:49.939790+0000","last_active":"2024-09-14T05:07:02.973730+0000","last_peered":"2024-09-14T05:07:02.973730+0000","last_clean":"2024-09-14T05:07:02.973730+0000","last_became_active":"2024-09-14T04:42:49.939591+0000","last_became_peered":"2024-09-14T04:42:49.939591+0000","last_unstale":"2024-09-14T05:07:02.973730+0000","last_undegraded":"2024-09-14T05:07:02.973730+0000","last_fullsized":"2024-09-14T05:07:02.973730+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:23:16.790593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":18,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.483821+0000","last_change":"2024-09-14T05:06:50.897229+0000","last_active":"2024-09-14T05:07:02.483821+0000","last_peered":"2024-09-14T05:07:02.483821+0000","last_clean":"2024-09-14T05:07:02.483821+0000","last_became_active":"2024-09-14T05:06:50.896906+0000","last_became_peered":"2024-09-14T05:06:50.896906+0000","last_unstale":"2024-09-14T05:07:02.483821+0000","last_undegraded":"2024-09-14T05:07:02.483821+0000","last_fullsized":"2024-09-14T05:07:02.483821+0000","mapping_epoch":1649,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:13:34.227694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":2071,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.811058+0000","last_change":"2024-09-14T05:06:38.805576+0000","last_active":"2024-09-14T05:07:06.811058+0000","last_peered":"2024-09-14T05:07:06.811058+0000","last_clean":"2024-09-14T05:07:06.811058+0000","last_became_active":"2024-09-14T05:06:38.805230+0000","last_became_peered":"2024-09-14T05:06:38.805230+0000","last_unstale":"2024-09-14T05:07:06.811058+0000","last_undegraded":"2024-09-14T05:07:06.811058+0000","last_fullsized":"2024-09-14T05:07:06.811058+0000","mapping_epoch":1642,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:05:36.161997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1652,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226009+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T05:07:02.226009+0000","last_peered":"2024-09-14T05:07:02.226009+0000","last_clean":"2024-09-14T05:07:02.226009+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T05:07:02.226009+0000","last_undegraded":"2024-09-14T05:07:02.226009+0000","last_fullsized":"2024-09-14T05:07:02.226009+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":2011,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.557804+0000","last_change":"2024-09-14T05:00:32.949837+0000","last_active":"2024-09-14T05:07:06.557804+0000","last_peered":"2024-09-14T05:07:06.557804+0000","last_clean":"2024-09-14T05:07:06.557804+0000","last_became_active":"2024-09-14T05:00:01.433169+0000","last_became_peered":"2024-09-14T05:00:01.433169+0000","last_unstale":"2024-09-14T05:07:06.557804+0000","last_undegraded":"2024-09-14T05:07:06.557804+0000","last_fullsized":"2024-09-14T05:07:06.557804+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1544,"parent":"0.0","parent_split_bits":0,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:32.949645+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:32.949645+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:04.207148+0000","scrub_duration":0.0087080530000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"349'149","reported_seq":985,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.253840+0000","last_change":"2024-09-14T05:02:55.610573+0000","last_active":"2024-09-14T05:07:06.253840+0000","last_peered":"2024-09-14T05:07:06.253840+0000","last_clean":"2024-09-14T05:07:06.253840+0000","last_became_active":"2024-09-14T05:02:53.340043+0000","last_became_peered":"2024-09-14T05:02:53.340043+0000","last_unstale":"2024-09-14T05:07:06.253840+0000","last_undegraded":"2024-09-14T05:07:06.253840+0000","last_fullsized":"2024-09-14T05:07:06.253840+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1583,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:02:55.610450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:02:55.610450+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:09:53.921765+0000","scrub_duration":0.0058046180000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":489,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.254241+0000","last_change":"2024-09-14T05:05:49.404344+0000","last_active":"2024-09-14T05:07:06.254241+0000","last_peered":"2024-09-14T05:07:06.254241+0000","last_clean":"2024-09-14T05:07:06.254241+0000","last_became_active":"2024-09-14T05:05:49.403782+0000","last_became_peered":"2024-09-14T05:05:49.403782+0000","last_unstale":"2024-09-14T05:07:06.254241+0000","last_undegraded":"2024-09-14T05:07:06.254241+0000","last_fullsized":"2024-09-14T05:07:06.254241+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:22.400478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2199,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.810943+0000","last_change":"2024-09-14T05:06:38.811995+0000","last_active":"2024-09-14T05:07:06.810943+0000","last_peered":"2024-09-14T05:07:06.810943+0000","last_clean":"2024-09-14T05:07:06.810943+0000","last_became_active":"2024-09-14T05:06:38.810469+0000","last_became_peered":"2024-09-14T05:06:38.810469+0000","last_unstale":"2024-09-14T05:07:06.810943+0000","last_undegraded":"2024-09-14T05:07:06.810943+0000","last_fullsized":"2024-09-14T05:07:06.810943+0000","mapping_epoch":1642,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:57:36.362081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"349'149","reported_seq":922,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.616323+0000","last_change":"2024-09-14T05:06:27.704804+0000","last_active":"2024-09-14T05:07:06.616323+0000","last_peered":"2024-09-14T05:07:06.616323+0000","last_clean":"2024-09-14T05:07:06.616323+0000","last_became_active":"2024-09-14T05:06:25.642946+0000","last_became_peered":"2024-09-14T05:06:25.642946+0000","last_unstale":"2024-09-14T05:07:06.616323+0000","last_undegraded":"2024-09-14T05:07:06.616323+0000","last_fullsized":"2024-09-14T05:07:06.616323+0000","mapping_epoch":1637,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1638,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:06:27.704729+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:06:27.704729+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:20:55.540219+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":54,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226680+0000","last_change":"2024-09-14T05:06:50.897944+0000","last_active":"2024-09-14T05:07:02.226680+0000","last_peered":"2024-09-14T05:07:02.226680+0000","last_clean":"2024-09-14T05:07:02.226680+0000","last_became_active":"2024-09-14T05:06:50.897602+0000","last_became_peered":"2024-09-14T05:06:50.897602+0000","last_unstale":"2024-09-14T05:07:02.226680+0000","last_undegraded":"2024-09-14T05:07:02.226680+0000","last_fullsized":"2024-09-14T05:07:02.226680+0000","mapping_epoch":1649,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:32:24.200093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":45,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.225519+0000","last_change":"2024-09-14T05:05:49.407117+0000","last_active":"2024-09-14T05:07:02.225519+0000","last_peered":"2024-09-14T05:07:02.225519+0000","last_clean":"2024-09-14T05:07:02.225519+0000","last_became_active":"2024-09-14T05:05:49.406744+0000","last_became_peered":"2024-09-14T05:05:49.406744+0000","last_unstale":"2024-09-14T05:07:02.225519+0000","last_undegraded":"2024-09-14T05:07:02.225519+0000","last_fullsized":"2024-09-14T05:07:02.225519+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:09:58.071161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"349'149","reported_seq":1104,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.225517+0000","last_change":"2024-09-14T05:06:46.998728+0000","last_active":"2024-09-14T05:07:02.225517+0000","last_peered":"2024-09-14T05:07:02.225517+0000","last_clean":"2024-09-14T05:07:02.225517+0000","last_became_active":"2024-09-14T05:06:46.998505+0000","last_became_peered":"2024-09-14T05:06:46.998505+0000","last_unstale":"2024-09-14T05:07:02.225517+0000","last_undegraded":"2024-09-14T05:07:02.225517+0000","last_fullsized":"2024-09-14T05:07:02.225517+0000","mapping_epoch":1647,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1648,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:02:53.452721+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:02:53.452721+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:56:40.944038+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":45,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226155+0000","last_change":"2024-09-14T05:05:49.412311+0000","last_active":"2024-09-14T05:07:02.226155+0000","last_peered":"2024-09-14T05:07:02.226155+0000","last_clean":"2024-09-14T05:07:02.226155+0000","last_became_active":"2024-09-14T05:05:49.411969+0000","last_became_peered":"2024-09-14T05:05:49.411969+0000","last_unstale":"2024-09-14T05:07:02.226155+0000","last_undegraded":"2024-09-14T05:07:02.226155+0000","last_fullsized":"2024-09-14T05:07:02.226155+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:45:30.548498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1699,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.483756+0000","last_change":"2024-09-14T05:05:49.409755+0000","last_active":"2024-09-14T05:07:02.483756+0000","last_peered":"2024-09-14T05:07:02.483756+0000","last_clean":"2024-09-14T05:07:02.483756+0000","last_became_active":"2024-09-14T05:05:49.409421+0000","last_became_peered":"2024-09-14T05:05:49.409421+0000","last_unstale":"2024-09-14T05:07:02.483756+0000","last_undegraded":"2024-09-14T05:07:02.483756+0000","last_fullsized":"2024-09-14T05:07:02.483756+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:25:07.695093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"349'149","reported_seq":2027,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.557571+0000","last_change":"2024-09-14T05:02:56.169546+0000","last_active":"2024-09-14T05:07:06.557571+0000","last_peered":"2024-09-14T05:07:06.557571+0000","last_clean":"2024-09-14T05:07:06.557571+0000","last_became_active":"2024-09-14T05:02:53.345545+0000","last_became_peered":"2024-09-14T05:02:53.345545+0000","last_unstale":"2024-09-14T05:07:06.557571+0000","last_undegraded":"2024-09-14T05:07:06.557571+0000","last_fullsized":"2024-09-14T05:07:06.557571+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1583,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:02:56.169422+0000","last_deep_scrub":"349'149","last_deep_scrub_stamp":"2024-09-14T05:02:56.169422+0000","last_clean_scrub_stamp":"2024-09-14T05:02:56.169422+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:07:18.417922+0000","scrub_duration":0.0071890360000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":54,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226369+0000","last_change":"2024-09-14T05:06:50.896831+0000","last_active":"2024-09-14T05:07:02.226369+0000","last_peered":"2024-09-14T05:07:02.226369+0000","last_clean":"2024-09-14T05:07:02.226369+0000","last_became_active":"2024-09-14T05:06:50.896448+0000","last_became_peered":"2024-09-14T05:06:50.896448+0000","last_unstale":"2024-09-14T05:07:02.226369+0000","last_undegraded":"2024-09-14T05:07:02.226369+0000","last_fullsized":"2024-09-14T05:07:02.226369+0000","mapping_epoch":1649,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:13:53.502233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":2101,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.811232+0000","last_change":"2024-09-14T05:06:38.803819+0000","last_active":"2024-09-14T05:07:06.811232+0000","last_peered":"2024-09-14T05:07:06.811232+0000","last_clean":"2024-09-14T05:07:06.811232+0000","last_became_active":"2024-09-14T05:06:38.803402+0000","last_became_peered":"2024-09-14T05:06:38.803402+0000","last_unstale":"2024-09-14T05:07:06.811232+0000","last_undegraded":"2024-09-14T05:07:06.811232+0000","last_fullsized":"2024-09-14T05:07:06.811232+0000","mapping_epoch":1642,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:26:52.531159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"349'149","reported_seq":678,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226603+0000","last_change":"2024-09-14T05:06:00.964181+0000","last_active":"2024-09-14T05:07:02.226603+0000","last_peered":"2024-09-14T05:07:02.226603+0000","last_clean":"2024-09-14T05:07:02.226603+0000","last_became_active":"2024-09-14T05:05:52.329578+0000","last_became_peered":"2024-09-14T05:05:52.329578+0000","last_unstale":"2024-09-14T05:07:02.226603+0000","last_undegraded":"2024-09-14T05:07:02.226603+0000","last_fullsized":"2024-09-14T05:07:02.226603+0000","mapping_epoch":1625,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1626,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:06:00.964099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:06:00.964099+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:04:19.147075+0000","scrub_duration":0.0017072490000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2635,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.501745+0000","last_change":"2024-09-14T05:05:49.410382+0000","last_active":"2024-09-14T05:07:06.501745+0000","last_peered":"2024-09-14T05:07:06.501745+0000","last_clean":"2024-09-14T05:07:06.501745+0000","last_became_active":"2024-09-14T05:05:49.409510+0000","last_became_peered":"2024-09-14T05:05:49.409510+0000","last_unstale":"2024-09-14T05:07:06.501745+0000","last_undegraded":"2024-09-14T05:07:06.501745+0000","last_fullsized":"2024-09-14T05:07:06.501745+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:25:26.281143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1637,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.253956+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T05:07:06.253956+0000","last_peered":"2024-09-14T05:07:06.253956+0000","last_clean":"2024-09-14T05:07:06.253956+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T05:07:06.253956+0000","last_undegraded":"2024-09-14T05:07:06.253956+0000","last_fullsized":"2024-09-14T05:07:06.253956+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"349'149","reported_seq":1077,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.483968+0000","last_change":"2024-09-14T05:05:52.328570+0000","last_active":"2024-09-14T05:07:02.483968+0000","last_peered":"2024-09-14T05:07:02.483968+0000","last_clean":"2024-09-14T05:07:02.483968+0000","last_became_active":"2024-09-14T05:05:52.328195+0000","last_became_peered":"2024-09-14T05:05:52.328195+0000","last_unstale":"2024-09-14T05:07:02.483968+0000","last_undegraded":"2024-09-14T05:07:02.483968+0000","last_fullsized":"2024-09-14T05:07:02.483968+0000","mapping_epoch":1625,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1626,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:46.668408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:46.668408+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:24:32.145385+0000","scrub_duration":0.0052970259999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":18,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.483589+0000","last_change":"2024-09-14T05:06:50.901294+0000","last_active":"2024-09-14T05:07:02.483589+0000","last_peered":"2024-09-14T05:07:02.483589+0000","last_clean":"2024-09-14T05:07:02.483589+0000","last_became_active":"2024-09-14T05:06:50.900723+0000","last_became_peered":"2024-09-14T05:06:50.900723+0000","last_unstale":"2024-09-14T05:07:02.483589+0000","last_undegraded":"2024-09-14T05:07:02.483589+0000","last_fullsized":"2024-09-14T05:07:02.483589+0000","mapping_epoch":1649,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:31:04.634081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1620,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.558383+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T05:07:06.558383+0000","last_peered":"2024-09-14T05:07:06.558383+0000","last_clean":"2024-09-14T05:07:06.558383+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T05:07:06.558383+0000","last_undegraded":"2024-09-14T05:07:06.558383+0000","last_fullsized":"2024-09-14T05:07:06.558383+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"349'149","reported_seq":906,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.811241+0000","last_change":"2024-09-14T05:06:53.921796+0000","last_active":"2024-09-14T05:07:06.811241+0000","last_peered":"2024-09-14T05:07:06.811241+0000","last_clean":"2024-09-14T05:07:06.811241+0000","last_became_active":"2024-09-14T05:06:53.920934+0000","last_became_peered":"2024-09-14T05:06:53.920934+0000","last_unstale":"2024-09-14T05:07:06.811241+0000","last_undegraded":"2024-09-14T05:07:06.811241+0000","last_fullsized":"2024-09-14T05:07:06.811241+0000","mapping_epoch":1652,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1653,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:06:40.377306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:06:40.377306+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:12:33.254211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":45,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.225306+0000","last_change":"2024-09-14T05:05:49.412921+0000","last_active":"2024-09-14T05:07:02.225306+0000","last_peered":"2024-09-14T05:07:02.225306+0000","last_clean":"2024-09-14T05:07:02.225306+0000","last_became_active":"2024-09-14T05:05:49.412575+0000","last_became_peered":"2024-09-14T05:05:49.412575+0000","last_unstale":"2024-09-14T05:07:02.225306+0000","last_undegraded":"2024-09-14T05:07:02.225306+0000","last_fullsized":"2024-09-14T05:07:02.225306+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:54:14.014240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1694,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.558447+0000","last_change":"2024-09-14T05:05:49.421180+0000","last_active":"2024-09-14T05:07:06.558447+0000","last_peered":"2024-09-14T05:07:06.558447+0000","last_clean":"2024-09-14T05:07:06.558447+0000","last_became_active":"2024-09-14T05:05:49.402745+0000","last_became_peered":"2024-09-14T05:05:49.402745+0000","last_unstale":"2024-09-14T05:07:06.558447+0000","last_undegraded":"2024-09-14T05:07:06.558447+0000","last_fullsized":"2024-09-14T05:07:06.558447+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:41:09.657905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.d","version":"349'149","reported_seq":958,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.974110+0000","last_change":"2024-09-14T05:05:52.329988+0000","last_active":"2024-09-14T05:07:02.974110+0000","last_peered":"2024-09-14T05:07:02.974110+0000","last_clean":"2024-09-14T05:07:02.974110+0000","last_became_active":"2024-09-14T05:05:52.328961+0000","last_became_peered":"2024-09-14T05:05:52.328961+0000","last_unstale":"2024-09-14T05:07:02.974110+0000","last_undegraded":"2024-09-14T05:07:02.974110+0000","last_fullsized":"2024-09-14T05:07:02.974110+0000","mapping_epoch":1625,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1626,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:03:01.129184+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:03:01.129184+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:43:35.836619+0000","scrub_duration":0.019899315000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1649,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.225839+0000","last_change":"2024-09-14T05:06:50.904194+0000","last_active":"2024-09-14T05:07:02.225839+0000","last_peered":"2024-09-14T05:07:02.225839+0000","last_clean":"2024-09-14T05:07:02.225839+0000","last_became_active":"2024-09-14T05:06:50.903844+0000","last_became_peered":"2024-09-14T05:06:50.903844+0000","last_unstale":"2024-09-14T05:07:02.225839+0000","last_undegraded":"2024-09-14T05:07:02.225839+0000","last_fullsized":"2024-09-14T05:07:02.225839+0000","mapping_epoch":1649,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:22:39.154438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1706,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.558455+0000","last_change":"2024-09-14T05:05:49.413784+0000","last_active":"2024-09-14T05:07:06.558455+0000","last_peered":"2024-09-14T05:07:06.558455+0000","last_clean":"2024-09-14T05:07:06.558455+0000","last_became_active":"2024-09-14T05:05:49.402531+0000","last_became_peered":"2024-09-14T05:05:49.402531+0000","last_unstale":"2024-09-14T05:07:06.558455+0000","last_undegraded":"2024-09-14T05:07:06.558455+0000","last_fullsized":"2024-09-14T05:07:06.558455+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:03:47.172975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"349'149","reported_seq":875,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.253699+0000","last_change":"2024-09-14T05:06:53.919963+0000","last_active":"2024-09-14T05:07:06.253699+0000","last_peered":"2024-09-14T05:07:06.253699+0000","last_clean":"2024-09-14T05:07:06.253699+0000","last_became_active":"2024-09-14T05:06:53.919412+0000","last_became_peered":"2024-09-14T05:06:53.919412+0000","last_unstale":"2024-09-14T05:07:06.253699+0000","last_undegraded":"2024-09-14T05:07:06.253699+0000","last_fullsized":"2024-09-14T05:07:06.253699+0000","mapping_epoch":1652,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1653,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:34.592056+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:34.592056+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:26:16.097595+0000","scrub_duration":0.0046679920000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1795,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.810996+0000","last_change":"2024-09-14T05:06:43.859008+0000","last_active":"2024-09-14T05:07:06.810996+0000","last_peered":"2024-09-14T05:07:06.810996+0000","last_clean":"2024-09-14T05:07:06.810996+0000","last_became_active":"2024-09-14T05:06:43.854258+0000","last_became_peered":"2024-09-14T05:06:43.854258+0000","last_unstale":"2024-09-14T05:07:06.810996+0000","last_undegraded":"2024-09-14T05:07:06.810996+0000","last_fullsized":"2024-09-14T05:07:06.810996+0000","mapping_epoch":1644,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1645,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:14:37.679962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":78,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.811693+0000","last_change":"2024-09-14T05:06:38.813475+0000","last_active":"2024-09-14T05:07:06.811693+0000","last_peered":"2024-09-14T05:07:06.811693+0000","last_clean":"2024-09-14T05:07:06.811693+0000","last_became_active":"2024-09-14T05:06:38.809830+0000","last_became_peered":"2024-09-14T05:06:38.809830+0000","last_unstale":"2024-09-14T05:07:06.811693+0000","last_undegraded":"2024-09-14T05:07:06.811693+0000","last_fullsized":"2024-09-14T05:07:06.811693+0000","mapping_epoch":1642,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:06:15.388664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"349'149","reported_seq":922,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.616302+0000","last_change":"2024-09-14T05:06:26.749615+0000","last_active":"2024-09-14T05:07:06.616302+0000","last_peered":"2024-09-14T05:07:06.616302+0000","last_clean":"2024-09-14T05:07:06.616302+0000","last_became_active":"2024-09-14T05:06:25.642891+0000","last_became_peered":"2024-09-14T05:06:25.642891+0000","last_unstale":"2024-09-14T05:07:06.616302+0000","last_undegraded":"2024-09-14T05:07:06.616302+0000","last_fullsized":"2024-09-14T05:07:06.616302+0000","mapping_epoch":1637,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1638,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:06:26.749547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:06:26.749547+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:40:06.826100+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1660,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.557735+0000","last_change":"2024-09-14T05:05:49.403096+0000","last_active":"2024-09-14T05:07:06.557735+0000","last_peered":"2024-09-14T05:07:06.557735+0000","last_clean":"2024-09-14T05:07:06.557735+0000","last_became_active":"2024-09-14T05:05:49.402744+0000","last_became_peered":"2024-09-14T05:05:49.402744+0000","last_unstale":"2024-09-14T05:07:06.557735+0000","last_undegraded":"2024-09-14T05:07:06.557735+0000","last_fullsized":"2024-09-14T05:07:06.557735+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:59:55.538846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2614,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.501930+0000","last_change":"2024-09-14T04:54:52.250655+0000","last_active":"2024-09-14T05:07:06.501930+0000","last_peered":"2024-09-14T05:07:06.501930+0000","last_clean":"2024-09-14T05:07:06.501930+0000","last_became_active":"2024-09-14T04:54:52.250317+0000","last_became_peered":"2024-09-14T04:54:52.250317+0000","last_unstale":"2024-09-14T05:07:06.501930+0000","last_undegraded":"2024-09-14T05:07:06.501930+0000","last_fullsized":"2024-09-14T05:07:06.501930+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:12.985406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.10","version":"349'149","reported_seq":803,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.253446+0000","last_change":"2024-09-14T05:06:53.916919+0000","last_active":"2024-09-14T05:07:06.253446+0000","last_peered":"2024-09-14T05:07:06.253446+0000","last_clean":"2024-09-14T05:07:06.253446+0000","last_became_active":"2024-09-14T05:06:53.916514+0000","last_became_peered":"2024-09-14T05:06:53.916514+0000","last_unstale":"2024-09-14T05:07:06.253446+0000","last_undegraded":"2024-09-14T05:07:06.253446+0000","last_fullsized":"2024-09-14T05:07:06.253446+0000","mapping_epoch":1652,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1653,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:36.569513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:36.569513+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:05:06.212971+0000","scrub_duration":0.0087120909999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2600,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.501645+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T05:07:06.501645+0000","last_peered":"2024-09-14T05:07:06.501645+0000","last_clean":"2024-09-14T05:07:06.501645+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T05:07:06.501645+0000","last_undegraded":"2024-09-14T05:07:06.501645+0000","last_fullsized":"2024-09-14T05:07:06.501645+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1633,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226649+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T05:07:02.226649+0000","last_peered":"2024-09-14T05:07:02.226649+0000","last_clean":"2024-09-14T05:07:02.226649+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T05:07:02.226649+0000","last_undegraded":"2024-09-14T05:07:02.226649+0000","last_fullsized":"2024-09-14T05:07:02.226649+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.11","version":"349'149","reported_seq":886,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.501483+0000","last_change":"2024-09-14T05:06:15.551990+0000","last_active":"2024-09-14T05:07:06.501483+0000","last_peered":"2024-09-14T05:07:06.501483+0000","last_clean":"2024-09-14T05:07:06.501483+0000","last_became_active":"2024-09-14T05:06:15.551627+0000","last_became_peered":"2024-09-14T05:06:15.551627+0000","last_unstale":"2024-09-14T05:07:06.501483+0000","last_undegraded":"2024-09-14T05:07:06.501483+0000","last_fullsized":"2024-09-14T05:07:06.501483+0000","mapping_epoch":1633,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1634,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:40.677843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:40.677843+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:36:43.830637+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1701,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.557954+0000","last_change":"2024-09-14T05:05:49.426093+0000","last_active":"2024-09-14T05:07:06.557954+0000","last_peered":"2024-09-14T05:07:06.557954+0000","last_clean":"2024-09-14T05:07:06.557954+0000","last_became_active":"2024-09-14T05:05:49.408363+0000","last_became_peered":"2024-09-14T05:05:49.408363+0000","last_unstale":"2024-09-14T05:07:06.557954+0000","last_undegraded":"2024-09-14T05:07:06.557954+0000","last_fullsized":"2024-09-14T05:07:06.557954+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:16:03.971185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2090,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.811653+0000","last_change":"2024-09-14T05:06:38.811815+0000","last_active":"2024-09-14T05:07:06.811653+0000","last_peered":"2024-09-14T05:07:06.811653+0000","last_clean":"2024-09-14T05:07:06.811653+0000","last_became_active":"2024-09-14T05:06:38.810155+0000","last_became_peered":"2024-09-14T05:06:38.810155+0000","last_unstale":"2024-09-14T05:07:06.811653+0000","last_undegraded":"2024-09-14T05:07:06.811653+0000","last_fullsized":"2024-09-14T05:07:06.811653+0000","mapping_epoch":1642,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:53.775016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":10118,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4250,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223202204,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":947956,"kb_used_data":14504,"kb_used_omap":1,"kb_used_meta":933438,"kb_avail":92797196,"statfs":{"total":95995035648,"available":95024328704,"internally_reserved":0,"allocated":14852096,"data_stored":8560291,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":955840971},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463840,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1052200,"kb_used_data":14556,"kb_used_omap":1,"kb_used_meta":1037630,"kb_avail":92692952,"statfs":{"total":95995035648,"available":94917582848,"internally_reserved":0,"allocated":14905344,"data_stored":8625156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":1062533575},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725477,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1038180,"kb_used_data":14488,"kb_used_omap":1,"kb_used_meta":1023678,"kb_avail":92706972,"statfs":{"total":95995035648,"available":94931939328,"internally_reserved":0,"allocated":14835712,"data_stored":8555990,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1048246726},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":1392,"seq":5978594476323,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":532656,"kb_used_data":14244,"kb_used_omap":117,"kb_used_meta":518282,"kb_avail":93212496,"statfs":{"total":95995035648,"available":95449595904,"internally_reserved":0,"allocated":14585856,"data_stored":8298155,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120638,"internal_metadata":530720962},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772075218,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1141980,"kb_used_data":14544,"kb_used_omap":71,"kb_used_meta":1127352,"kb_avail":92603172,"statfs":{"total":95995035648,"available":94825648128,"internally_reserved":0,"allocated":14893056,"data_stored":8621035,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":1154409398},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707304153,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":840672,"kb_used_data":14484,"kb_used_omap":68,"kb_used_meta":826107,"kb_avail":92904480,"statfs":{"total":95995035648,"available":95134187520,"internally_reserved":0,"allocated":14831616,"data_stored":8555947,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":845933776},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872729312,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":818136,"kb_used_data":14220,"kb_used_omap":67,"kb_used_meta":803836,"kb_avail":92927016,"statfs":{"total":95995035648,"available":95157264384,"internally_reserved":0,"allocated":14561280,"data_stored":8292662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":823128421},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056363,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":872412,"kb_used_data":14544,"kb_used_omap":74,"kb_used_meta":857781,"kb_avail":92872740,"statfs":{"total":95995035648,"available":95101685760,"internally_reserved":0,"allocated":14893056,"data_stored":8620927,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":878368437},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1901,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T05:07:56.480 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid bd7c803c-724b-11ef-bceb-c7b262605968 -- ceph pg dump --format=json 2024-09-14T05:07:56.784 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:56 smithi039 ceph-mon[106470]: from='client.37080 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T05:07:56.785 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:56 smithi039 ceph-mon[111235]: from='client.37080 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T05:07:57.020 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:56 smithi129 ceph-mon[101316]: from='client.37080 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T05:07:57.035 INFO:teuthology.orchestra.run.smithi039.stderr:Inferring config /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/mon.a/config 2024-09-14T05:07:58.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:57 smithi129 ceph-mon[101316]: pgmap v3376: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:58.040 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:57 smithi039 ceph-mon[106470]: pgmap v3376: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:58.040 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:57 smithi039 ceph-mon[111235]: pgmap v3376: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:59.039 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:07:58 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: ::ffff:172.21.15.129 - - [14/Sep/2024:05:07:58] "GET /metrics HTTP/1.1" 200 34310 "" "Prometheus/2.51.0" 2024-09-14T05:07:59.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:07:59 smithi039 ceph-mon[106470]: pgmap v3377: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:59.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:07:59 smithi039 ceph-mon[111235]: pgmap v3377: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:07:59.804 INFO:teuthology.orchestra.run.smithi039.stdout: 2024-09-14T05:07:59.805 INFO:teuthology.orchestra.run.smithi039.stderr:dumped all 2024-09-14T05:08:00.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:07:59 smithi129 ceph-mon[101316]: pgmap v3377: 103 pgs: 103 active+clean; 582 KiB data, 6.9 GiB used, 708 GiB / 715 GiB avail 2024-09-14T05:08:00.617 INFO:teuthology.orchestra.run.smithi039.stdout:{"pg_ready":true,"pg_map":{"version":3377,"stamp":"2024-09-14T05:07:58.142084+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":595941,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":64,"num_write_kb":33,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":85,"num_bytes_recovered":337836,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4678,"ondisk_log_size":4678,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":309,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7244192,"kb_used_data":115584,"kb_used_omap":404,"kb_used_meta":7128107,"kb_avail":742717024,"statfs":{"total":767960285184,"available":760542232576,"internally_reserved":0,"allocated":118358016,"data_stored":68130163,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":414022,"internal_metadata":7299182266},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003233"},"pg_stats":[{"pgid":"1.1d","version":"349'149","reported_seq":963,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.616597+0000","last_change":"2024-09-14T05:05:52.331425+0000","last_active":"2024-09-14T05:07:06.616597+0000","last_peered":"2024-09-14T05:07:06.616597+0000","last_clean":"2024-09-14T05:07:06.616597+0000","last_became_active":"2024-09-14T05:05:52.330812+0000","last_became_peered":"2024-09-14T05:05:52.330812+0000","last_unstale":"2024-09-14T05:07:06.616597+0000","last_undegraded":"2024-09-14T05:07:06.616597+0000","last_fullsized":"2024-09-14T05:07:06.616597+0000","mapping_epoch":1625,"log_start":"0'0","ondisk_log_start":"0'0","created":1582,"last_epoch_clean":1626,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:04:04.688573+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:04:04.688573+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:02:08.034833+0000","scrub_duration":24,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"72'4","reported_seq":2611,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.502021+0000","last_change":"2024-09-14T04:07:21.697082+0000","last_active":"2024-09-14T05:07:06.502021+0000","last_peered":"2024-09-14T05:07:06.502021+0000","last_clean":"2024-09-14T05:07:06.502021+0000","last_became_active":"2024-09-14T04:07:21.695208+0000","last_became_peered":"2024-09-14T04:07:21.695208+0000","last_unstale":"2024-09-14T05:07:06.502021+0000","last_undegraded":"2024-09-14T05:07:06.502021+0000","last_fullsized":"2024-09-14T05:07:06.502021+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:51:01.403756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":54,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.225471+0000","last_change":"2024-09-14T05:06:50.899773+0000","last_active":"2024-09-14T05:07:02.225471+0000","last_peered":"2024-09-14T05:07:02.225471+0000","last_clean":"2024-09-14T05:07:02.225471+0000","last_became_active":"2024-09-14T05:06:50.899361+0000","last_became_peered":"2024-09-14T05:06:50.899361+0000","last_unstale":"2024-09-14T05:07:02.225471+0000","last_undegraded":"2024-09-14T05:07:02.225471+0000","last_fullsized":"2024-09-14T05:07:02.225471+0000","mapping_epoch":1649,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:40:51.266314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"349'149","reported_seq":1056,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.973985+0000","last_change":"2024-09-14T05:06:53.919206+0000","last_active":"2024-09-14T05:07:02.973985+0000","last_peered":"2024-09-14T05:07:02.973985+0000","last_clean":"2024-09-14T05:07:02.973985+0000","last_became_active":"2024-09-14T05:06:53.918790+0000","last_became_peered":"2024-09-14T05:06:53.918790+0000","last_unstale":"2024-09-14T05:07:02.973985+0000","last_undegraded":"2024-09-14T05:07:02.973985+0000","last_fullsized":"2024-09-14T05:07:02.973985+0000","mapping_epoch":1652,"log_start":"0'0","ondisk_log_start":"0'0","created":1582,"last_epoch_clean":1653,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:02:54.501356+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:02:54.501356+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:01:52.526614+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"72'1","reported_seq":2208,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.811777+0000","last_change":"2024-09-14T05:06:50.896369+0000","last_active":"2024-09-14T05:07:06.811777+0000","last_peered":"2024-09-14T05:07:06.811777+0000","last_clean":"2024-09-14T05:07:06.811777+0000","last_became_active":"2024-09-14T05:06:50.895992+0000","last_became_peered":"2024-09-14T05:06:50.895992+0000","last_unstale":"2024-09-14T05:07:06.811777+0000","last_undegraded":"2024-09-14T05:07:06.811777+0000","last_fullsized":"2024-09-14T05:07:06.811777+0000","mapping_epoch":1649,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:05:54.563890+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":2097,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.811272+0000","last_change":"2024-09-14T05:06:38.804334+0000","last_active":"2024-09-14T05:07:06.811272+0000","last_peered":"2024-09-14T05:07:06.811272+0000","last_clean":"2024-09-14T05:07:06.811272+0000","last_became_active":"2024-09-14T05:06:38.803681+0000","last_became_peered":"2024-09-14T05:06:38.803681+0000","last_unstale":"2024-09-14T05:07:06.811272+0000","last_undegraded":"2024-09-14T05:07:06.811272+0000","last_fullsized":"2024-09-14T05:07:06.811272+0000","mapping_epoch":1642,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:49:55.786021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1d","version":"72'2","reported_seq":1704,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226704+0000","last_change":"2024-09-14T05:06:50.906848+0000","last_active":"2024-09-14T05:07:02.226704+0000","last_peered":"2024-09-14T05:07:02.226704+0000","last_clean":"2024-09-14T05:07:02.226704+0000","last_became_active":"2024-09-14T05:06:50.901534+0000","last_became_peered":"2024-09-14T05:06:50.901534+0000","last_unstale":"2024-09-14T05:07:02.226704+0000","last_undegraded":"2024-09-14T05:07:02.226704+0000","last_fullsized":"2024-09-14T05:07:02.226704+0000","mapping_epoch":1649,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:46:51.000071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":45,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226355+0000","last_change":"2024-09-14T05:05:49.412104+0000","last_active":"2024-09-14T05:07:02.226355+0000","last_peered":"2024-09-14T05:07:02.226355+0000","last_clean":"2024-09-14T05:07:02.226355+0000","last_became_active":"2024-09-14T05:05:49.409810+0000","last_became_peered":"2024-09-14T05:05:49.409810+0000","last_unstale":"2024-09-14T05:07:02.226355+0000","last_undegraded":"2024-09-14T05:07:02.226355+0000","last_fullsized":"2024-09-14T05:07:02.226355+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:07:07.657805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1e","version":"349'149","reported_seq":677,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.811489+0000","last_change":"2024-09-14T05:06:43.455120+0000","last_active":"2024-09-14T05:07:06.811489+0000","last_peered":"2024-09-14T05:07:06.811489+0000","last_clean":"2024-09-14T05:07:06.811489+0000","last_became_active":"2024-09-14T05:06:38.811232+0000","last_became_peered":"2024-09-14T05:06:38.811232+0000","last_unstale":"2024-09-14T05:07:06.811489+0000","last_undegraded":"2024-09-14T05:07:06.811489+0000","last_fullsized":"2024-09-14T05:07:06.811489+0000","mapping_epoch":1642,"log_start":"0'0","ondisk_log_start":"0'0","created":1582,"last_epoch_clean":1643,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:06:43.455053+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:06:43.455053+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:23:54.663847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"72'2","reported_seq":1706,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226217+0000","last_change":"2024-09-14T05:05:49.409062+0000","last_active":"2024-09-14T05:07:02.226217+0000","last_peered":"2024-09-14T05:07:02.226217+0000","last_clean":"2024-09-14T05:07:02.226217+0000","last_became_active":"2024-09-14T05:05:49.399865+0000","last_became_peered":"2024-09-14T05:05:49.399865+0000","last_unstale":"2024-09-14T05:07:02.226217+0000","last_undegraded":"2024-09-14T05:07:02.226217+0000","last_fullsized":"2024-09-14T05:07:02.226217+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:06:44.128353+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":54,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.225950+0000","last_change":"2024-09-14T05:06:12.769686+0000","last_active":"2024-09-14T05:07:02.225950+0000","last_peered":"2024-09-14T05:07:02.225950+0000","last_clean":"2024-09-14T05:07:02.225950+0000","last_became_active":"2024-09-14T05:06:12.769312+0000","last_became_peered":"2024-09-14T05:06:12.769312+0000","last_unstale":"2024-09-14T05:07:02.225950+0000","last_undegraded":"2024-09-14T05:07:02.225950+0000","last_fullsized":"2024-09-14T05:07:02.225950+0000","mapping_epoch":1630,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1631,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:04:02.262440+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"349'149","reported_seq":1703,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.810975+0000","last_change":"2024-09-14T05:06:39.361293+0000","last_active":"2024-09-14T05:07:06.810975+0000","last_peered":"2024-09-14T05:07:06.810975+0000","last_clean":"2024-09-14T05:07:06.810975+0000","last_became_active":"2024-09-14T05:06:38.803337+0000","last_became_peered":"2024-09-14T05:06:38.803337+0000","last_unstale":"2024-09-14T05:07:06.810975+0000","last_undegraded":"2024-09-14T05:07:06.810975+0000","last_fullsized":"2024-09-14T05:07:06.810975+0000","mapping_epoch":1642,"log_start":"0'0","ondisk_log_start":"0'0","created":1582,"last_epoch_clean":1643,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:06:39.361246+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:06:39.361246+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:04:23.837623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"72'1","reported_seq":480,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.501688+0000","last_change":"2024-09-14T05:05:49.410696+0000","last_active":"2024-09-14T05:07:06.501688+0000","last_peered":"2024-09-14T05:07:06.501688+0000","last_clean":"2024-09-14T05:07:06.501688+0000","last_became_active":"2024-09-14T05:05:49.410138+0000","last_became_peered":"2024-09-14T05:05:49.410138+0000","last_unstale":"2024-09-14T05:07:06.501688+0000","last_undegraded":"2024-09-14T05:07:06.501688+0000","last_fullsized":"2024-09-14T05:07:06.501688+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:51:15.835973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":2606,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.254613+0000","last_change":"2024-09-14T04:07:58.513448+0000","last_active":"2024-09-14T05:07:06.254613+0000","last_peered":"2024-09-14T05:07:06.254613+0000","last_clean":"2024-09-14T05:07:06.254613+0000","last_became_active":"2024-09-14T04:07:58.512865+0000","last_became_peered":"2024-09-14T04:07:58.512865+0000","last_unstale":"2024-09-14T05:07:06.254613+0000","last_undegraded":"2024-09-14T05:07:06.254613+0000","last_fullsized":"2024-09-14T05:07:06.254613+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:53:57.436439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.18","version":"349'149","reported_seq":1025,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.254323+0000","last_change":"2024-09-14T05:03:04.296355+0000","last_active":"2024-09-14T05:07:06.254323+0000","last_peered":"2024-09-14T05:07:06.254323+0000","last_clean":"2024-09-14T05:07:06.254323+0000","last_became_active":"2024-09-14T05:03:04.295977+0000","last_became_peered":"2024-09-14T05:03:04.295977+0000","last_unstale":"2024-09-14T05:07:06.254323+0000","last_undegraded":"2024-09-14T05:07:06.254323+0000","last_fullsized":"2024-09-14T05:07:06.254323+0000","mapping_epoch":1587,"log_start":"0'0","ondisk_log_start":"0'0","created":1582,"last_epoch_clean":1588,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:02:54.449355+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:02:54.449355+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:11:30.881287+0000","scrub_duration":4,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"72'1","reported_seq":45,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226116+0000","last_change":"2024-09-14T05:05:49.411615+0000","last_active":"2024-09-14T05:07:02.226116+0000","last_peered":"2024-09-14T05:07:02.226116+0000","last_clean":"2024-09-14T05:07:02.226116+0000","last_became_active":"2024-09-14T05:05:49.411224+0000","last_became_peered":"2024-09-14T05:05:49.411224+0000","last_unstale":"2024-09-14T05:07:02.226116+0000","last_undegraded":"2024-09-14T05:07:02.226116+0000","last_fullsized":"2024-09-14T05:07:02.226116+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:27:15.272705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":2558,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.811495+0000","last_change":"2024-09-14T05:06:38.810400+0000","last_active":"2024-09-14T05:07:06.811495+0000","last_peered":"2024-09-14T05:07:06.811495+0000","last_clean":"2024-09-14T05:07:06.811495+0000","last_became_active":"2024-09-14T05:06:38.809963+0000","last_became_peered":"2024-09-14T05:06:38.809963+0000","last_unstale":"2024-09-14T05:07:06.811495+0000","last_undegraded":"2024-09-14T05:07:06.811495+0000","last_fullsized":"2024-09-14T05:07:06.811495+0000","mapping_epoch":1642,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:11:06.301818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"349'149","reported_seq":996,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.225765+0000","last_change":"2024-09-14T05:05:52.334643+0000","last_active":"2024-09-14T05:07:02.225765+0000","last_peered":"2024-09-14T05:07:02.225765+0000","last_clean":"2024-09-14T05:07:02.225765+0000","last_became_active":"2024-09-14T05:05:52.334266+0000","last_became_peered":"2024-09-14T05:05:52.334266+0000","last_unstale":"2024-09-14T05:07:02.225765+0000","last_undegraded":"2024-09-14T05:07:02.225765+0000","last_fullsized":"2024-09-14T05:07:02.225765+0000","mapping_epoch":1625,"log_start":"0'0","ondisk_log_start":"0'0","created":1582,"last_epoch_clean":1626,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:04:53.630279+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:04:53.630279+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:45:34.361978+0000","scrub_duration":0.0058720990000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":2606,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.616174+0000","last_change":"2024-09-14T05:05:49.410644+0000","last_active":"2024-09-14T05:07:06.616174+0000","last_peered":"2024-09-14T05:07:06.616174+0000","last_clean":"2024-09-14T05:07:06.616174+0000","last_became_active":"2024-09-14T05:05:49.410321+0000","last_became_peered":"2024-09-14T05:05:49.410321+0000","last_unstale":"2024-09-14T05:07:06.616174+0000","last_undegraded":"2024-09-14T05:07:06.616174+0000","last_fullsized":"2024-09-14T05:07:06.616174+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:57:11.897463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":2627,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.502042+0000","last_change":"2024-09-14T04:54:52.250695+0000","last_active":"2024-09-14T05:07:06.502042+0000","last_peered":"2024-09-14T05:07:06.502042+0000","last_clean":"2024-09-14T05:07:06.502042+0000","last_became_active":"2024-09-14T04:54:52.250301+0000","last_became_peered":"2024-09-14T04:54:52.250301+0000","last_unstale":"2024-09-14T05:07:06.502042+0000","last_undegraded":"2024-09-14T05:07:06.502042+0000","last_fullsized":"2024-09-14T05:07:06.502042+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:28:42.060905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"349'149","reported_seq":677,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226137+0000","last_change":"2024-09-14T05:06:01.981981+0000","last_active":"2024-09-14T05:07:02.226137+0000","last_peered":"2024-09-14T05:07:02.226137+0000","last_clean":"2024-09-14T05:07:02.226137+0000","last_became_active":"2024-09-14T05:05:52.326854+0000","last_became_peered":"2024-09-14T05:05:52.326854+0000","last_unstale":"2024-09-14T05:07:02.226137+0000","last_undegraded":"2024-09-14T05:07:02.226137+0000","last_fullsized":"2024-09-14T05:07:02.226137+0000","mapping_epoch":1625,"log_start":"0'0","ondisk_log_start":"0'0","created":1582,"last_epoch_clean":1626,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:06:01.981845+0000","last_deep_scrub":"349'149","last_deep_scrub_stamp":"2024-09-14T05:02:59.622762+0000","last_clean_scrub_stamp":"2024-09-14T05:06:01.981845+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:09:55.213179+0000","scrub_duration":0.0018431260000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":2096,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.811038+0000","last_change":"2024-09-14T05:06:38.812290+0000","last_active":"2024-09-14T05:07:06.811038+0000","last_peered":"2024-09-14T05:07:06.811038+0000","last_clean":"2024-09-14T05:07:06.811038+0000","last_became_active":"2024-09-14T05:06:38.812004+0000","last_became_peered":"2024-09-14T05:06:38.812004+0000","last_unstale":"2024-09-14T05:07:06.811038+0000","last_undegraded":"2024-09-14T05:07:06.811038+0000","last_fullsized":"2024-09-14T05:07:06.811038+0000","mapping_epoch":1642,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:06:51.292857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":18,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.483875+0000","last_change":"2024-09-14T05:06:50.901735+0000","last_active":"2024-09-14T05:07:02.483875+0000","last_peered":"2024-09-14T05:07:02.483875+0000","last_clean":"2024-09-14T05:07:02.483875+0000","last_became_active":"2024-09-14T05:06:50.901416+0000","last_became_peered":"2024-09-14T05:06:50.901416+0000","last_unstale":"2024-09-14T05:07:02.483875+0000","last_undegraded":"2024-09-14T05:07:02.483875+0000","last_fullsized":"2024-09-14T05:07:02.483875+0000","mapping_epoch":1649,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:57:59.096226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"349'149","reported_seq":863,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.558104+0000","last_change":"2024-09-14T05:04:43.777195+0000","last_active":"2024-09-14T05:07:06.558104+0000","last_peered":"2024-09-14T05:07:06.558104+0000","last_clean":"2024-09-14T05:07:06.558104+0000","last_became_active":"2024-09-14T05:03:04.293853+0000","last_became_peered":"2024-09-14T05:03:04.293853+0000","last_unstale":"2024-09-14T05:07:06.558104+0000","last_undegraded":"2024-09-14T05:07:06.558104+0000","last_fullsized":"2024-09-14T05:07:06.558104+0000","mapping_epoch":1587,"log_start":"0'0","ondisk_log_start":"0'0","created":1582,"last_epoch_clean":1588,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:04:43.777093+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:04:43.777093+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:25:50.652208+0000","scrub_duration":0.0098257610000000006,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":320,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.225716+0000","last_change":"2024-09-14T05:06:50.902587+0000","last_active":"2024-09-14T05:07:02.225716+0000","last_peered":"2024-09-14T05:07:02.225716+0000","last_clean":"2024-09-14T05:07:02.225716+0000","last_became_active":"2024-09-14T05:06:50.902273+0000","last_became_peered":"2024-09-14T05:06:50.902273+0000","last_unstale":"2024-09-14T05:07:02.225716+0000","last_undegraded":"2024-09-14T05:07:02.225716+0000","last_fullsized":"2024-09-14T05:07:02.225716+0000","mapping_epoch":1649,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:43:33.240582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":2599,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.502111+0000","last_change":"2024-09-14T04:07:58.577068+0000","last_active":"2024-09-14T05:07:06.502111+0000","last_peered":"2024-09-14T05:07:06.502111+0000","last_clean":"2024-09-14T05:07:06.502111+0000","last_became_active":"2024-09-14T04:07:58.576453+0000","last_became_peered":"2024-09-14T04:07:58.576453+0000","last_unstale":"2024-09-14T05:07:06.502111+0000","last_undegraded":"2024-09-14T05:07:06.502111+0000","last_fullsized":"2024-09-14T05:07:06.502111+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:43:58.008385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.14","version":"349'149","reported_seq":1096,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.254272+0000","last_change":"2024-09-14T05:05:52.326841+0000","last_active":"2024-09-14T05:07:06.254272+0000","last_peered":"2024-09-14T05:07:06.254272+0000","last_clean":"2024-09-14T05:07:06.254272+0000","last_became_active":"2024-09-14T05:05:52.326258+0000","last_became_peered":"2024-09-14T05:05:52.326258+0000","last_unstale":"2024-09-14T05:07:06.254272+0000","last_undegraded":"2024-09-14T05:07:06.254272+0000","last_fullsized":"2024-09-14T05:07:06.254272+0000","mapping_epoch":1625,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1626,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:39.545724+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:39.545724+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:21:36.944653+0000","scrub_duration":8,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":2182,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.811518+0000","last_change":"2024-09-14T05:06:38.811444+0000","last_active":"2024-09-14T05:07:06.811518+0000","last_peered":"2024-09-14T05:07:06.811518+0000","last_clean":"2024-09-14T05:07:06.811518+0000","last_became_active":"2024-09-14T05:06:38.810235+0000","last_became_peered":"2024-09-14T05:06:38.810235+0000","last_unstale":"2024-09-14T05:07:06.811518+0000","last_undegraded":"2024-09-14T05:07:06.811518+0000","last_fullsized":"2024-09-14T05:07:06.811518+0000","mapping_epoch":1642,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:03:35.247423+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":1651,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226381+0000","last_change":"2024-09-14T04:42:49.938505+0000","last_active":"2024-09-14T05:07:02.226381+0000","last_peered":"2024-09-14T05:07:02.226381+0000","last_clean":"2024-09-14T05:07:02.226381+0000","last_became_active":"2024-09-14T04:42:49.938180+0000","last_became_peered":"2024-09-14T04:42:49.938180+0000","last_unstale":"2024-09-14T05:07:02.226381+0000","last_undegraded":"2024-09-14T05:07:02.226381+0000","last_fullsized":"2024-09-14T05:07:02.226381+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:16:05.929479+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.17","version":"349'149","reported_seq":916,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.253961+0000","last_change":"2024-09-14T05:04:43.753778+0000","last_active":"2024-09-14T05:07:06.253961+0000","last_peered":"2024-09-14T05:07:06.253961+0000","last_clean":"2024-09-14T05:07:06.253961+0000","last_became_active":"2024-09-14T05:03:04.281302+0000","last_became_peered":"2024-09-14T05:03:04.281302+0000","last_unstale":"2024-09-14T05:07:06.253961+0000","last_undegraded":"2024-09-14T05:07:06.253961+0000","last_fullsized":"2024-09-14T05:07:06.253961+0000","mapping_epoch":1587,"log_start":"0'0","ondisk_log_start":"0'0","created":1582,"last_epoch_clean":1588,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:04:43.753698+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:04:43.753698+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:12:27.317820+0000","scrub_duration":0.0014490550000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"72'3","reported_seq":1701,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.558322+0000","last_change":"2024-09-14T05:05:49.423629+0000","last_active":"2024-09-14T05:07:06.558322+0000","last_peered":"2024-09-14T05:07:06.558322+0000","last_clean":"2024-09-14T05:07:06.558322+0000","last_became_active":"2024-09-14T05:05:49.407879+0000","last_became_peered":"2024-09-14T05:05:49.407879+0000","last_unstale":"2024-09-14T05:07:06.558322+0000","last_undegraded":"2024-09-14T05:07:06.558322+0000","last_fullsized":"2024-09-14T05:07:06.558322+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:43:49.292723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":360,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":555,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.616295+0000","last_change":"2024-09-14T05:06:50.897034+0000","last_active":"2024-09-14T05:07:06.616295+0000","last_peered":"2024-09-14T05:07:06.616295+0000","last_clean":"2024-09-14T05:07:06.616295+0000","last_became_active":"2024-09-14T05:06:50.896679+0000","last_became_peered":"2024-09-14T05:06:50.896679+0000","last_unstale":"2024-09-14T05:07:06.616295+0000","last_undegraded":"2024-09-14T05:07:06.616295+0000","last_fullsized":"2024-09-14T05:07:06.616295+0000","mapping_epoch":1649,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:34:33.381326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"349'149","reported_seq":782,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.616777+0000","last_change":"2024-09-14T05:03:35.645894+0000","last_active":"2024-09-14T05:07:06.616777+0000","last_peered":"2024-09-14T05:07:06.616777+0000","last_clean":"2024-09-14T05:07:06.616777+0000","last_became_active":"2024-09-14T05:03:04.309263+0000","last_became_peered":"2024-09-14T05:03:04.309263+0000","last_unstale":"2024-09-14T05:07:06.616777+0000","last_undegraded":"2024-09-14T05:07:06.616777+0000","last_fullsized":"2024-09-14T05:07:06.616777+0000","mapping_epoch":1587,"log_start":"0'0","ondisk_log_start":"0'0","created":1582,"last_epoch_clean":1588,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:03:35.645824+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:03:35.645824+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:53:53.382747+0000","scrub_duration":9,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"429'23","reported_seq":55,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226499+0000","last_change":"2024-09-14T05:05:49.419211+0000","last_active":"2024-09-14T05:07:02.226499+0000","last_peered":"2024-09-14T05:07:02.226499+0000","last_clean":"2024-09-14T05:07:02.226499+0000","last_became_active":"2024-09-14T05:05:49.409977+0000","last_became_peered":"2024-09-14T05:05:49.409977+0000","last_unstale":"2024-09-14T05:07:02.226499+0000","last_undegraded":"2024-09-14T05:07:02.226499+0000","last_fullsized":"2024-09-14T05:07:02.226499+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:06:35.797110+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":23,"num_write_kb":20,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9512,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":45,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226190+0000","last_change":"2024-09-14T05:05:49.411270+0000","last_active":"2024-09-14T05:07:02.226190+0000","last_peered":"2024-09-14T05:07:02.226190+0000","last_clean":"2024-09-14T05:07:02.226190+0000","last_became_active":"2024-09-14T05:05:49.410956+0000","last_became_peered":"2024-09-14T05:05:49.410956+0000","last_unstale":"2024-09-14T05:07:02.226190+0000","last_undegraded":"2024-09-14T05:07:02.226190+0000","last_fullsized":"2024-09-14T05:07:02.226190+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:31:28.109175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"349'149","reported_seq":878,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.811301+0000","last_change":"2024-09-14T05:06:42.443038+0000","last_active":"2024-09-14T05:07:06.811301+0000","last_peered":"2024-09-14T05:07:06.811301+0000","last_clean":"2024-09-14T05:07:06.811301+0000","last_became_active":"2024-09-14T05:06:38.811366+0000","last_became_peered":"2024-09-14T05:06:38.811366+0000","last_unstale":"2024-09-14T05:07:06.811301+0000","last_undegraded":"2024-09-14T05:07:06.811301+0000","last_fullsized":"2024-09-14T05:07:06.811301+0000","mapping_epoch":1642,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1643,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:06:42.442989+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:06:42.442989+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:51:56.337012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":46,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.253227+0000","last_change":"2024-09-14T05:05:49.410296+0000","last_active":"2024-09-14T05:07:06.253227+0000","last_peered":"2024-09-14T05:07:06.253227+0000","last_clean":"2024-09-14T05:07:06.253227+0000","last_became_active":"2024-09-14T05:05:49.409663+0000","last_became_peered":"2024-09-14T05:05:49.409663+0000","last_unstale":"2024-09-14T05:07:06.253227+0000","last_undegraded":"2024-09-14T05:07:06.253227+0000","last_fullsized":"2024-09-14T05:07:06.253227+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:19:56.206619+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":1668,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.811496+0000","last_change":"2024-09-14T04:54:52.242207+0000","last_active":"2024-09-14T05:07:06.811496+0000","last_peered":"2024-09-14T05:07:06.811496+0000","last_clean":"2024-09-14T05:07:06.811496+0000","last_became_active":"2024-09-14T04:54:52.241840+0000","last_became_peered":"2024-09-14T04:54:52.241840+0000","last_unstale":"2024-09-14T05:07:06.811496+0000","last_undegraded":"2024-09-14T05:07:06.811496+0000","last_fullsized":"2024-09-14T05:07:06.811496+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:15:37.899666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000277657,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"72'1","reported_seq":2671,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.502000+0000","last_change":"2024-09-14T05:06:12.541282+0000","last_active":"2024-09-14T05:07:06.502000+0000","last_peered":"2024-09-14T05:07:06.502000+0000","last_clean":"2024-09-14T05:07:06.502000+0000","last_became_active":"2024-09-14T05:06:12.538805+0000","last_became_peered":"2024-09-14T05:06:12.538805+0000","last_unstale":"2024-09-14T05:07:06.502000+0000","last_undegraded":"2024-09-14T05:07:06.502000+0000","last_fullsized":"2024-09-14T05:07:06.502000+0000","mapping_epoch":1630,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1631,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:47:22.130803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"349'149","reported_seq":920,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.254485+0000","last_change":"2024-09-14T05:06:53.920679+0000","last_active":"2024-09-14T05:07:06.254485+0000","last_peered":"2024-09-14T05:07:06.254485+0000","last_clean":"2024-09-14T05:07:06.254485+0000","last_became_active":"2024-09-14T05:06:53.920222+0000","last_became_peered":"2024-09-14T05:06:53.920222+0000","last_unstale":"2024-09-14T05:07:06.254485+0000","last_undegraded":"2024-09-14T05:07:06.254485+0000","last_fullsized":"2024-09-14T05:07:06.254485+0000","mapping_epoch":1652,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1653,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:04:05.388558+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:04:05.388558+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:38:37.384495+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":64,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.253928+0000","last_change":"2024-09-14T05:06:12.769094+0000","last_active":"2024-09-14T05:07:06.253928+0000","last_peered":"2024-09-14T05:07:06.253928+0000","last_clean":"2024-09-14T05:07:06.253928+0000","last_became_active":"2024-09-14T05:06:12.768782+0000","last_became_peered":"2024-09-14T05:06:12.768782+0000","last_unstale":"2024-09-14T05:07:06.253928+0000","last_undegraded":"2024-09-14T05:07:06.253928+0000","last_fullsized":"2024-09-14T05:07:06.253928+0000","mapping_epoch":1630,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1631,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:28:35.561669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,0],"acting":[2,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":607,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.616876+0000","last_change":"2024-09-14T05:05:49.411503+0000","last_active":"2024-09-14T05:07:06.616876+0000","last_peered":"2024-09-14T05:07:06.616876+0000","last_clean":"2024-09-14T05:07:06.616876+0000","last_became_active":"2024-09-14T05:05:49.411159+0000","last_became_peered":"2024-09-14T05:05:49.411159+0000","last_unstale":"2024-09-14T05:07:06.616876+0000","last_undegraded":"2024-09-14T05:07:06.616876+0000","last_fullsized":"2024-09-14T05:07:06.616876+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:31:42.831156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"72'2","reported_seq":2061,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.811859+0000","last_change":"2024-09-14T05:06:38.810126+0000","last_active":"2024-09-14T05:07:06.811859+0000","last_peered":"2024-09-14T05:07:06.811859+0000","last_clean":"2024-09-14T05:07:06.811859+0000","last_became_active":"2024-09-14T05:06:38.809374+0000","last_became_peered":"2024-09-14T05:06:38.809374+0000","last_unstale":"2024-09-14T05:07:06.811859+0000","last_undegraded":"2024-09-14T05:07:06.811859+0000","last_fullsized":"2024-09-14T05:07:06.811859+0000","mapping_epoch":1642,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:36:06.638778+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.6","version":"349'149","reported_seq":736,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.483821+0000","last_change":"2024-09-14T05:06:53.922799+0000","last_active":"2024-09-14T05:07:02.483821+0000","last_peered":"2024-09-14T05:07:02.483821+0000","last_clean":"2024-09-14T05:07:02.483821+0000","last_became_active":"2024-09-14T05:06:53.921763+0000","last_became_peered":"2024-09-14T05:06:53.921763+0000","last_unstale":"2024-09-14T05:07:02.483821+0000","last_undegraded":"2024-09-14T05:07:02.483821+0000","last_fullsized":"2024-09-14T05:07:02.483821+0000","mapping_epoch":1652,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1653,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:05:52.621968+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:05:52.621968+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:12:16.401190+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":2594,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.501899+0000","last_change":"2024-09-14T04:05:54.423079+0000","last_active":"2024-09-14T05:07:06.501899+0000","last_peered":"2024-09-14T05:07:06.501899+0000","last_clean":"2024-09-14T05:07:06.501899+0000","last_became_active":"2024-09-14T04:05:54.410545+0000","last_became_peered":"2024-09-14T04:05:54.410545+0000","last_unstale":"2024-09-14T05:07:06.501899+0000","last_undegraded":"2024-09-14T05:07:06.501899+0000","last_fullsized":"2024-09-14T05:07:06.501899+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":672,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T03:59:46.699104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":1700,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:03.012597+0000","last_change":"2024-09-14T05:05:49.407464+0000","last_active":"2024-09-14T05:07:03.012597+0000","last_peered":"2024-09-14T05:07:03.012597+0000","last_clean":"2024-09-14T05:07:03.012597+0000","last_became_active":"2024-09-14T05:05:49.407096+0000","last_became_peered":"2024-09-14T05:05:49.407096+0000","last_unstale":"2024-09-14T05:07:03.012597+0000","last_undegraded":"2024-09-14T05:07:03.012597+0000","last_fullsized":"2024-09-14T05:07:03.012597+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:10:19.818430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034220700000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":754,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.616581+0000","last_change":"2024-09-14T05:06:25.813154+0000","last_active":"2024-09-14T05:07:06.616581+0000","last_peered":"2024-09-14T05:07:06.616581+0000","last_clean":"2024-09-14T05:07:06.616581+0000","last_became_active":"2024-09-14T05:06:25.812853+0000","last_became_peered":"2024-09-14T05:06:25.812853+0000","last_unstale":"2024-09-14T05:07:06.616581+0000","last_undegraded":"2024-09-14T05:07:06.616581+0000","last_fullsized":"2024-09-14T05:07:06.616581+0000","mapping_epoch":1637,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:42:56.670205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"349'149","reported_seq":866,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.483876+0000","last_change":"2024-09-14T05:06:53.921666+0000","last_active":"2024-09-14T05:07:02.483876+0000","last_peered":"2024-09-14T05:07:02.483876+0000","last_clean":"2024-09-14T05:07:02.483876+0000","last_became_active":"2024-09-14T05:06:53.921256+0000","last_became_peered":"2024-09-14T05:06:53.921256+0000","last_unstale":"2024-09-14T05:07:02.483876+0000","last_undegraded":"2024-09-14T05:07:02.483876+0000","last_fullsized":"2024-09-14T05:07:02.483876+0000","mapping_epoch":1652,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1653,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:06:41.414786+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:06:41.414786+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:10:03.630703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":563,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.616263+0000","last_change":"2024-09-14T05:05:49.409973+0000","last_active":"2024-09-14T05:07:06.616263+0000","last_peered":"2024-09-14T05:07:06.616263+0000","last_clean":"2024-09-14T05:07:06.616263+0000","last_became_active":"2024-09-14T05:05:49.409659+0000","last_became_peered":"2024-09-14T05:05:49.409659+0000","last_unstale":"2024-09-14T05:07:06.616263+0000","last_undegraded":"2024-09-14T05:07:06.616263+0000","last_fullsized":"2024-09-14T05:07:06.616263+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:17:33.620109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":2245,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.616808+0000","last_change":"2024-09-14T05:06:25.756161+0000","last_active":"2024-09-14T05:07:06.616808+0000","last_peered":"2024-09-14T05:07:06.616808+0000","last_clean":"2024-09-14T05:07:06.616808+0000","last_became_active":"2024-09-14T05:06:25.755843+0000","last_became_peered":"2024-09-14T05:06:25.755843+0000","last_unstale":"2024-09-14T05:07:06.616808+0000","last_undegraded":"2024-09-14T05:07:06.616808+0000","last_fullsized":"2024-09-14T05:07:06.616808+0000","mapping_epoch":1637,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:36:34.523283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033654300000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":1654,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226721+0000","last_change":"2024-09-14T04:07:58.511783+0000","last_active":"2024-09-14T05:07:02.226721+0000","last_peered":"2024-09-14T05:07:02.226721+0000","last_clean":"2024-09-14T05:07:02.226721+0000","last_became_active":"2024-09-14T04:07:58.510771+0000","last_became_peered":"2024-09-14T04:07:58.510771+0000","last_unstale":"2024-09-14T05:07:02.226721+0000","last_undegraded":"2024-09-14T05:07:02.226721+0000","last_fullsized":"2024-09-14T05:07:02.226721+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:21:11.359608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"349'149","reported_seq":985,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226791+0000","last_change":"2024-09-14T05:00:33.562385+0000","last_active":"2024-09-14T05:07:02.226791+0000","last_peered":"2024-09-14T05:07:02.226791+0000","last_clean":"2024-09-14T05:07:02.226791+0000","last_became_active":"2024-09-14T05:00:01.421507+0000","last_became_peered":"2024-09-14T05:00:01.421507+0000","last_unstale":"2024-09-14T05:07:02.226791+0000","last_undegraded":"2024-09-14T05:07:02.226791+0000","last_fullsized":"2024-09-14T05:07:02.226791+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1544,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:33.562211+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:33.562211+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:32:46.187217+0000","scrub_duration":0.0066960869999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":790,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.616571+0000","last_change":"2024-09-14T05:06:25.812988+0000","last_active":"2024-09-14T05:07:06.616571+0000","last_peered":"2024-09-14T05:07:06.616571+0000","last_clean":"2024-09-14T05:07:06.616571+0000","last_became_active":"2024-09-14T05:06:25.812670+0000","last_became_peered":"2024-09-14T05:06:25.812670+0000","last_unstale":"2024-09-14T05:07:06.616571+0000","last_undegraded":"2024-09-14T05:07:06.616571+0000","last_fullsized":"2024-09-14T05:07:06.616571+0000","mapping_epoch":1637,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:54:26.893016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":46,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.253717+0000","last_change":"2024-09-14T05:05:49.407964+0000","last_active":"2024-09-14T05:07:06.253717+0000","last_peered":"2024-09-14T05:07:06.253717+0000","last_clean":"2024-09-14T05:07:06.253717+0000","last_became_active":"2024-09-14T05:05:49.407494+0000","last_became_peered":"2024-09-14T05:05:49.407494+0000","last_unstale":"2024-09-14T05:07:06.253717+0000","last_undegraded":"2024-09-14T05:07:06.253717+0000","last_fullsized":"2024-09-14T05:07:06.253717+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:39:57.277245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"66'2","reported_seq":1682,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.225832+0000","last_change":"2024-09-14T05:06:50.909013+0000","last_active":"2024-09-14T05:07:02.225832+0000","last_peered":"2024-09-14T05:07:02.225832+0000","last_clean":"2024-09-14T05:07:02.225832+0000","last_became_active":"2024-09-14T05:06:50.903124+0000","last_became_peered":"2024-09-14T05:06:50.903124+0000","last_unstale":"2024-09-14T05:07:02.225832+0000","last_undegraded":"2024-09-14T05:07:02.225832+0000","last_fullsized":"2024-09-14T05:07:02.225832+0000","mapping_epoch":1649,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:16:15.260181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00081495999999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"349'149","reported_seq":922,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.557681+0000","last_change":"2024-09-14T05:05:52.333791+0000","last_active":"2024-09-14T05:07:06.557681+0000","last_peered":"2024-09-14T05:07:06.557681+0000","last_clean":"2024-09-14T05:07:06.557681+0000","last_became_active":"2024-09-14T05:05:52.333449+0000","last_became_peered":"2024-09-14T05:05:52.333449+0000","last_unstale":"2024-09-14T05:07:06.557681+0000","last_undegraded":"2024-09-14T05:07:06.557681+0000","last_fullsized":"2024-09-14T05:07:06.557681+0000","mapping_epoch":1625,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1626,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:02.067463+0000","last_deep_scrub":"349'149","last_deep_scrub_stamp":"2024-09-14T05:00:02.067463+0000","last_clean_scrub_stamp":"2024-09-14T05:00:02.067463+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:38:19.134801+0000","scrub_duration":0.0033042340000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":541,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.616851+0000","last_change":"2024-09-14T04:44:14.489235+0000","last_active":"2024-09-14T05:07:06.616851+0000","last_peered":"2024-09-14T05:07:06.616851+0000","last_clean":"2024-09-14T05:07:06.616851+0000","last_became_active":"2024-09-14T04:44:14.488950+0000","last_became_peered":"2024-09-14T04:44:14.488950+0000","last_unstale":"2024-09-14T05:07:06.616851+0000","last_undegraded":"2024-09-14T05:07:06.616851+0000","last_fullsized":"2024-09-14T05:07:06.616851+0000","mapping_epoch":1396,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:54.441144+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":2236,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.616198+0000","last_change":"2024-09-14T05:06:50.896040+0000","last_active":"2024-09-14T05:07:06.616198+0000","last_peered":"2024-09-14T05:07:06.616198+0000","last_clean":"2024-09-14T05:07:06.616198+0000","last_became_active":"2024-09-14T05:06:50.895697+0000","last_became_peered":"2024-09-14T05:06:50.895697+0000","last_unstale":"2024-09-14T05:07:06.616198+0000","last_undegraded":"2024-09-14T05:07:06.616198+0000","last_fullsized":"2024-09-14T05:07:06.616198+0000","mapping_epoch":1649,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:29:45.946979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":2615,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.254386+0000","last_change":"2024-09-14T04:07:58.512381+0000","last_active":"2024-09-14T05:07:06.254386+0000","last_peered":"2024-09-14T05:07:06.254386+0000","last_clean":"2024-09-14T05:07:06.254386+0000","last_became_active":"2024-09-14T04:07:58.512105+0000","last_became_peered":"2024-09-14T04:07:58.512105+0000","last_unstale":"2024-09-14T05:07:06.254386+0000","last_undegraded":"2024-09-14T05:07:06.254386+0000","last_fullsized":"2024-09-14T05:07:06.254386+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:29:33.956193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00026896299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"349'149","reported_seq":924,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.616472+0000","last_change":"2024-09-14T05:06:53.920045+0000","last_active":"2024-09-14T05:07:06.616472+0000","last_peered":"2024-09-14T05:07:06.616472+0000","last_clean":"2024-09-14T05:07:06.616472+0000","last_became_active":"2024-09-14T05:06:53.919631+0000","last_became_peered":"2024-09-14T05:06:53.919631+0000","last_unstale":"2024-09-14T05:07:06.616472+0000","last_undegraded":"2024-09-14T05:07:06.616472+0000","last_fullsized":"2024-09-14T05:07:06.616472+0000","mapping_epoch":1652,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1653,"parent":"0.0","parent_split_bits":3,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:06:25.753556+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:06:25.753556+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:59:50.923576+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":2615,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.502132+0000","last_change":"2024-09-14T04:07:21.695451+0000","last_active":"2024-09-14T05:07:06.502132+0000","last_peered":"2024-09-14T05:07:06.502132+0000","last_clean":"2024-09-14T05:07:06.502132+0000","last_became_active":"2024-09-14T04:07:21.694299+0000","last_became_peered":"2024-09-14T04:07:21.694299+0000","last_unstale":"2024-09-14T05:07:06.502132+0000","last_undegraded":"2024-09-14T05:07:06.502132+0000","last_fullsized":"2024-09-14T05:07:06.502132+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:06:28.012444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":2650,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.501649+0000","last_change":"2024-09-14T05:05:49.409878+0000","last_active":"2024-09-14T05:07:06.501649+0000","last_peered":"2024-09-14T05:07:06.501649+0000","last_clean":"2024-09-14T05:07:06.501649+0000","last_became_active":"2024-09-14T05:05:49.409559+0000","last_became_peered":"2024-09-14T05:05:49.409559+0000","last_unstale":"2024-09-14T05:07:06.501649+0000","last_undegraded":"2024-09-14T05:07:06.501649+0000","last_fullsized":"2024-09-14T05:07:06.501649+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:25:46.517122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":1683,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.973730+0000","last_change":"2024-09-14T04:42:49.939790+0000","last_active":"2024-09-14T05:07:02.973730+0000","last_peered":"2024-09-14T05:07:02.973730+0000","last_clean":"2024-09-14T05:07:02.973730+0000","last_became_active":"2024-09-14T04:42:49.939591+0000","last_became_peered":"2024-09-14T04:42:49.939591+0000","last_unstale":"2024-09-14T05:07:02.973730+0000","last_undegraded":"2024-09-14T05:07:02.973730+0000","last_fullsized":"2024-09-14T05:07:02.973730+0000","mapping_epoch":1374,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":1375,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:23:16.790593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028736000000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":18,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.483821+0000","last_change":"2024-09-14T05:06:50.897229+0000","last_active":"2024-09-14T05:07:02.483821+0000","last_peered":"2024-09-14T05:07:02.483821+0000","last_clean":"2024-09-14T05:07:02.483821+0000","last_became_active":"2024-09-14T05:06:50.896906+0000","last_became_peered":"2024-09-14T05:06:50.896906+0000","last_unstale":"2024-09-14T05:07:02.483821+0000","last_undegraded":"2024-09-14T05:07:02.483821+0000","last_fullsized":"2024-09-14T05:07:02.483821+0000","mapping_epoch":1649,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:13:34.227694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":2071,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.811058+0000","last_change":"2024-09-14T05:06:38.805576+0000","last_active":"2024-09-14T05:07:06.811058+0000","last_peered":"2024-09-14T05:07:06.811058+0000","last_clean":"2024-09-14T05:07:06.811058+0000","last_became_active":"2024-09-14T05:06:38.805230+0000","last_became_peered":"2024-09-14T05:06:38.805230+0000","last_unstale":"2024-09-14T05:07:06.811058+0000","last_undegraded":"2024-09-14T05:07:06.811058+0000","last_fullsized":"2024-09-14T05:07:06.811058+0000","mapping_epoch":1642,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:05:36.161997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.3","version":"64'1","reported_seq":1652,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226009+0000","last_change":"2024-09-14T04:07:21.713633+0000","last_active":"2024-09-14T05:07:02.226009+0000","last_peered":"2024-09-14T05:07:02.226009+0000","last_clean":"2024-09-14T05:07:02.226009+0000","last_became_active":"2024-09-14T04:07:21.711745+0000","last_became_peered":"2024-09-14T04:07:21.711745+0000","last_unstale":"2024-09-14T05:07:02.226009+0000","last_undegraded":"2024-09-14T05:07:02.226009+0000","last_fullsized":"2024-09-14T05:07:02.226009+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:51:45.777559+0000","last_clean_scrub_stamp":"2024-09-14T03:51:45.777559+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:15:59.649686+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00032557899999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"349'149","reported_seq":2011,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.557804+0000","last_change":"2024-09-14T05:00:32.949837+0000","last_active":"2024-09-14T05:07:06.557804+0000","last_peered":"2024-09-14T05:07:06.557804+0000","last_clean":"2024-09-14T05:07:06.557804+0000","last_became_active":"2024-09-14T05:00:01.433169+0000","last_became_peered":"2024-09-14T05:00:01.433169+0000","last_unstale":"2024-09-14T05:07:06.557804+0000","last_undegraded":"2024-09-14T05:07:06.557804+0000","last_fullsized":"2024-09-14T05:07:06.557804+0000","mapping_epoch":671,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":1544,"parent":"0.0","parent_split_bits":0,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:32.949645+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:32.949645+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:58:04.207148+0000","scrub_duration":0.0087080530000000003,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.a","version":"349'149","reported_seq":985,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.253840+0000","last_change":"2024-09-14T05:02:55.610573+0000","last_active":"2024-09-14T05:07:06.253840+0000","last_peered":"2024-09-14T05:07:06.253840+0000","last_clean":"2024-09-14T05:07:06.253840+0000","last_became_active":"2024-09-14T05:02:53.340043+0000","last_became_peered":"2024-09-14T05:02:53.340043+0000","last_unstale":"2024-09-14T05:07:06.253840+0000","last_undegraded":"2024-09-14T05:07:06.253840+0000","last_fullsized":"2024-09-14T05:07:06.253840+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1583,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:02:55.610450+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:02:55.610450+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:09:53.921765+0000","scrub_duration":0.0058046180000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":489,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.254241+0000","last_change":"2024-09-14T05:05:49.404344+0000","last_active":"2024-09-14T05:07:06.254241+0000","last_peered":"2024-09-14T05:07:06.254241+0000","last_clean":"2024-09-14T05:07:06.254241+0000","last_became_active":"2024-09-14T05:05:49.403782+0000","last_became_peered":"2024-09-14T05:05:49.403782+0000","last_unstale":"2024-09-14T05:07:06.254241+0000","last_undegraded":"2024-09-14T05:07:06.254241+0000","last_fullsized":"2024-09-14T05:07:06.254241+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:22.400478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":2199,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.810943+0000","last_change":"2024-09-14T05:06:38.811995+0000","last_active":"2024-09-14T05:07:06.810943+0000","last_peered":"2024-09-14T05:07:06.810943+0000","last_clean":"2024-09-14T05:07:06.810943+0000","last_became_active":"2024-09-14T05:06:38.810469+0000","last_became_peered":"2024-09-14T05:06:38.810469+0000","last_unstale":"2024-09-14T05:07:06.810943+0000","last_undegraded":"2024-09-14T05:07:06.810943+0000","last_fullsized":"2024-09-14T05:07:06.810943+0000","mapping_epoch":1642,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T13:57:36.362081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.b","version":"349'149","reported_seq":922,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.616323+0000","last_change":"2024-09-14T05:06:27.704804+0000","last_active":"2024-09-14T05:07:06.616323+0000","last_peered":"2024-09-14T05:07:06.616323+0000","last_clean":"2024-09-14T05:07:06.616323+0000","last_became_active":"2024-09-14T05:06:25.642946+0000","last_became_peered":"2024-09-14T05:06:25.642946+0000","last_unstale":"2024-09-14T05:07:06.616323+0000","last_undegraded":"2024-09-14T05:07:06.616323+0000","last_fullsized":"2024-09-14T05:07:06.616323+0000","mapping_epoch":1637,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1638,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:06:27.704729+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:06:27.704729+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:20:55.540219+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":54,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226680+0000","last_change":"2024-09-14T05:06:50.897944+0000","last_active":"2024-09-14T05:07:02.226680+0000","last_peered":"2024-09-14T05:07:02.226680+0000","last_clean":"2024-09-14T05:07:02.226680+0000","last_became_active":"2024-09-14T05:06:50.897602+0000","last_became_peered":"2024-09-14T05:06:50.897602+0000","last_unstale":"2024-09-14T05:07:02.226680+0000","last_undegraded":"2024-09-14T05:07:02.226680+0000","last_fullsized":"2024-09-14T05:07:02.226680+0000","mapping_epoch":1649,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:32:24.200093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":45,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.225519+0000","last_change":"2024-09-14T05:05:49.407117+0000","last_active":"2024-09-14T05:07:02.225519+0000","last_peered":"2024-09-14T05:07:02.225519+0000","last_clean":"2024-09-14T05:07:02.225519+0000","last_became_active":"2024-09-14T05:05:49.406744+0000","last_became_peered":"2024-09-14T05:05:49.406744+0000","last_unstale":"2024-09-14T05:07:02.225519+0000","last_undegraded":"2024-09-14T05:07:02.225519+0000","last_fullsized":"2024-09-14T05:07:02.225519+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:09:58.071161+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"349'149","reported_seq":1104,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.225517+0000","last_change":"2024-09-14T05:06:46.998728+0000","last_active":"2024-09-14T05:07:02.225517+0000","last_peered":"2024-09-14T05:07:02.225517+0000","last_clean":"2024-09-14T05:07:02.225517+0000","last_became_active":"2024-09-14T05:06:46.998505+0000","last_became_peered":"2024-09-14T05:06:46.998505+0000","last_unstale":"2024-09-14T05:07:02.225517+0000","last_undegraded":"2024-09-14T05:07:02.225517+0000","last_fullsized":"2024-09-14T05:07:02.225517+0000","mapping_epoch":1647,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1648,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:02:53.452721+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:02:53.452721+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:56:40.944038+0000","scrub_duration":6,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":45,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226155+0000","last_change":"2024-09-14T05:05:49.412311+0000","last_active":"2024-09-14T05:07:02.226155+0000","last_peered":"2024-09-14T05:07:02.226155+0000","last_clean":"2024-09-14T05:07:02.226155+0000","last_became_active":"2024-09-14T05:05:49.411969+0000","last_became_peered":"2024-09-14T05:05:49.411969+0000","last_unstale":"2024-09-14T05:07:02.226155+0000","last_undegraded":"2024-09-14T05:07:02.226155+0000","last_fullsized":"2024-09-14T05:07:02.226155+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:45:30.548498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":1699,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.483756+0000","last_change":"2024-09-14T05:05:49.409755+0000","last_active":"2024-09-14T05:07:02.483756+0000","last_peered":"2024-09-14T05:07:02.483756+0000","last_clean":"2024-09-14T05:07:02.483756+0000","last_became_active":"2024-09-14T05:05:49.409421+0000","last_became_peered":"2024-09-14T05:05:49.409421+0000","last_unstale":"2024-09-14T05:07:02.483756+0000","last_undegraded":"2024-09-14T05:07:02.483756+0000","last_fullsized":"2024-09-14T05:07:02.483756+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:25:07.695093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"349'149","reported_seq":2027,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.557571+0000","last_change":"2024-09-14T05:02:56.169546+0000","last_active":"2024-09-14T05:07:06.557571+0000","last_peered":"2024-09-14T05:07:06.557571+0000","last_clean":"2024-09-14T05:07:06.557571+0000","last_became_active":"2024-09-14T05:02:53.345545+0000","last_became_peered":"2024-09-14T05:02:53.345545+0000","last_unstale":"2024-09-14T05:07:06.557571+0000","last_undegraded":"2024-09-14T05:07:06.557571+0000","last_fullsized":"2024-09-14T05:07:06.557571+0000","mapping_epoch":1346,"log_start":"0'0","ondisk_log_start":"0'0","created":1341,"last_epoch_clean":1583,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:02:56.169422+0000","last_deep_scrub":"349'149","last_deep_scrub_stamp":"2024-09-14T05:02:56.169422+0000","last_clean_scrub_stamp":"2024-09-14T05:02:56.169422+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T09:07:18.417922+0000","scrub_duration":0.0071890360000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":54,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226369+0000","last_change":"2024-09-14T05:06:50.896831+0000","last_active":"2024-09-14T05:07:02.226369+0000","last_peered":"2024-09-14T05:07:02.226369+0000","last_clean":"2024-09-14T05:07:02.226369+0000","last_became_active":"2024-09-14T05:06:50.896448+0000","last_became_peered":"2024-09-14T05:06:50.896448+0000","last_unstale":"2024-09-14T05:07:02.226369+0000","last_undegraded":"2024-09-14T05:07:02.226369+0000","last_fullsized":"2024-09-14T05:07:02.226369+0000","mapping_epoch":1649,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:13:53.502233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"72'2","reported_seq":2101,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.811232+0000","last_change":"2024-09-14T05:06:38.803819+0000","last_active":"2024-09-14T05:07:06.811232+0000","last_peered":"2024-09-14T05:07:06.811232+0000","last_clean":"2024-09-14T05:07:06.811232+0000","last_became_active":"2024-09-14T05:06:38.803402+0000","last_became_peered":"2024-09-14T05:06:38.803402+0000","last_unstale":"2024-09-14T05:07:06.811232+0000","last_undegraded":"2024-09-14T05:07:06.811232+0000","last_fullsized":"2024-09-14T05:07:06.811232+0000","mapping_epoch":1642,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:26:52.531159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"349'149","reported_seq":678,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.226603+0000","last_change":"2024-09-14T05:06:00.964181+0000","last_active":"2024-09-14T05:07:02.226603+0000","last_peered":"2024-09-14T05:07:02.226603+0000","last_clean":"2024-09-14T05:07:02.226603+0000","last_became_active":"2024-09-14T05:05:52.329578+0000","last_became_peered":"2024-09-14T05:05:52.329578+0000","last_unstale":"2024-09-14T05:07:02.226603+0000","last_undegraded":"2024-09-14T05:07:02.226603+0000","last_fullsized":"2024-09-14T05:07:02.226603+0000","mapping_epoch":1625,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1626,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:06:00.964099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:06:00.964099+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:04:19.147075+0000","scrub_duration":0.0017072490000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":2635,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.501745+0000","last_change":"2024-09-14T05:05:49.410382+0000","last_active":"2024-09-14T05:07:06.501745+0000","last_peered":"2024-09-14T05:07:06.501745+0000","last_clean":"2024-09-14T05:07:06.501745+0000","last_became_active":"2024-09-14T05:05:49.409510+0000","last_became_peered":"2024-09-14T05:05:49.409510+0000","last_unstale":"2024-09-14T05:07:06.501745+0000","last_undegraded":"2024-09-14T05:07:06.501745+0000","last_fullsized":"2024-09-14T05:07:06.501745+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:25:26.281143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":1637,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.253956+0000","last_change":"2024-09-14T04:07:58.507343+0000","last_active":"2024-09-14T05:07:06.253956+0000","last_peered":"2024-09-14T05:07:06.253956+0000","last_clean":"2024-09-14T05:07:06.253956+0000","last_became_active":"2024-09-14T04:07:58.507071+0000","last_became_peered":"2024-09-14T04:07:58.507071+0000","last_unstale":"2024-09-14T05:07:06.253956+0000","last_undegraded":"2024-09-14T05:07:06.253956+0000","last_fullsized":"2024-09-14T05:07:06.253956+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:07:11.403722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"349'149","reported_seq":1077,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.483968+0000","last_change":"2024-09-14T05:05:52.328570+0000","last_active":"2024-09-14T05:07:02.483968+0000","last_peered":"2024-09-14T05:07:02.483968+0000","last_clean":"2024-09-14T05:07:02.483968+0000","last_became_active":"2024-09-14T05:05:52.328195+0000","last_became_peered":"2024-09-14T05:05:52.328195+0000","last_unstale":"2024-09-14T05:07:02.483968+0000","last_undegraded":"2024-09-14T05:07:02.483968+0000","last_fullsized":"2024-09-14T05:07:02.483968+0000","mapping_epoch":1625,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1626,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:46.668408+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:46.668408+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:24:32.145385+0000","scrub_duration":0.0052970259999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":18,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.483589+0000","last_change":"2024-09-14T05:06:50.901294+0000","last_active":"2024-09-14T05:07:02.483589+0000","last_peered":"2024-09-14T05:07:02.483589+0000","last_clean":"2024-09-14T05:07:02.483589+0000","last_became_active":"2024-09-14T05:06:50.900723+0000","last_became_peered":"2024-09-14T05:06:50.900723+0000","last_unstale":"2024-09-14T05:07:02.483589+0000","last_undegraded":"2024-09-14T05:07:02.483589+0000","last_fullsized":"2024-09-14T05:07:02.483589+0000","mapping_epoch":1649,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T04:31:04.634081+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"72'1","reported_seq":1620,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.558383+0000","last_change":"2024-09-14T03:53:05.238329+0000","last_active":"2024-09-14T05:07:06.558383+0000","last_peered":"2024-09-14T05:07:06.558383+0000","last_clean":"2024-09-14T05:07:06.558383+0000","last_became_active":"2024-09-14T03:53:05.238069+0000","last_became_peered":"2024-09-14T03:53:05.238069+0000","last_unstale":"2024-09-14T05:07:06.558383+0000","last_undegraded":"2024-09-14T05:07:06.558383+0000","last_fullsized":"2024-09-14T05:07:06.558383+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T11:59:09.121267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.c","version":"349'149","reported_seq":906,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.811241+0000","last_change":"2024-09-14T05:06:53.921796+0000","last_active":"2024-09-14T05:07:06.811241+0000","last_peered":"2024-09-14T05:07:06.811241+0000","last_clean":"2024-09-14T05:07:06.811241+0000","last_became_active":"2024-09-14T05:06:53.920934+0000","last_became_peered":"2024-09-14T05:06:53.920934+0000","last_unstale":"2024-09-14T05:07:06.811241+0000","last_undegraded":"2024-09-14T05:07:06.811241+0000","last_fullsized":"2024-09-14T05:07:06.811241+0000","mapping_epoch":1652,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1653,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:06:40.377306+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:06:40.377306+0000","objects_scrubbed":1,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:12:33.254211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":45,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.225306+0000","last_change":"2024-09-14T05:05:49.412921+0000","last_active":"2024-09-14T05:07:02.225306+0000","last_peered":"2024-09-14T05:07:02.225306+0000","last_clean":"2024-09-14T05:07:02.225306+0000","last_became_active":"2024-09-14T05:05:49.412575+0000","last_became_peered":"2024-09-14T05:05:49.412575+0000","last_unstale":"2024-09-14T05:07:02.225306+0000","last_undegraded":"2024-09-14T05:07:02.225306+0000","last_fullsized":"2024-09-14T05:07:02.225306+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:54:14.014240+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"72'2","reported_seq":1694,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.558447+0000","last_change":"2024-09-14T05:05:49.421180+0000","last_active":"2024-09-14T05:07:06.558447+0000","last_peered":"2024-09-14T05:07:06.558447+0000","last_clean":"2024-09-14T05:07:06.558447+0000","last_became_active":"2024-09-14T05:05:49.402745+0000","last_became_peered":"2024-09-14T05:05:49.402745+0000","last_unstale":"2024-09-14T05:07:06.558447+0000","last_undegraded":"2024-09-14T05:07:06.558447+0000","last_fullsized":"2024-09-14T05:07:06.558447+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T14:41:09.657905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.d","version":"349'149","reported_seq":958,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.974110+0000","last_change":"2024-09-14T05:05:52.329988+0000","last_active":"2024-09-14T05:07:02.974110+0000","last_peered":"2024-09-14T05:07:02.974110+0000","last_clean":"2024-09-14T05:07:02.974110+0000","last_became_active":"2024-09-14T05:05:52.328961+0000","last_became_peered":"2024-09-14T05:05:52.328961+0000","last_unstale":"2024-09-14T05:07:02.974110+0000","last_undegraded":"2024-09-14T05:07:02.974110+0000","last_fullsized":"2024-09-14T05:07:02.974110+0000","mapping_epoch":1625,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1626,"parent":"0.0","parent_split_bits":4,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:03:01.129184+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:03:01.129184+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:43:35.836619+0000","scrub_duration":0.019899315000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":1649,"reported_epoch":1656,"state":"active+clean","last_fresh":"2024-09-14T05:07:02.225839+0000","last_change":"2024-09-14T05:06:50.904194+0000","last_active":"2024-09-14T05:07:02.225839+0000","last_peered":"2024-09-14T05:07:02.225839+0000","last_clean":"2024-09-14T05:07:02.225839+0000","last_became_active":"2024-09-14T05:06:50.903844+0000","last_became_peered":"2024-09-14T05:06:50.903844+0000","last_unstale":"2024-09-14T05:07:02.225839+0000","last_undegraded":"2024-09-14T05:07:02.225839+0000","last_fullsized":"2024-09-14T05:07:02.225839+0000","mapping_epoch":1649,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1650,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:22:39.154438+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"72'1","reported_seq":1706,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.558455+0000","last_change":"2024-09-14T05:05:49.413784+0000","last_active":"2024-09-14T05:07:06.558455+0000","last_peered":"2024-09-14T05:07:06.558455+0000","last_clean":"2024-09-14T05:07:06.558455+0000","last_became_active":"2024-09-14T05:05:49.402531+0000","last_became_peered":"2024-09-14T05:05:49.402531+0000","last_unstale":"2024-09-14T05:07:06.558455+0000","last_undegraded":"2024-09-14T05:07:06.558455+0000","last_fullsized":"2024-09-14T05:07:06.558455+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T08:03:47.172975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":110,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.12","version":"349'149","reported_seq":875,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.253699+0000","last_change":"2024-09-14T05:06:53.919963+0000","last_active":"2024-09-14T05:07:06.253699+0000","last_peered":"2024-09-14T05:07:06.253699+0000","last_clean":"2024-09-14T05:07:06.253699+0000","last_became_active":"2024-09-14T05:06:53.919412+0000","last_became_peered":"2024-09-14T05:06:53.919412+0000","last_unstale":"2024-09-14T05:07:06.253699+0000","last_undegraded":"2024-09-14T05:07:06.253699+0000","last_fullsized":"2024-09-14T05:07:06.253699+0000","mapping_epoch":1652,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1653,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:34.592056+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:34.592056+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:26:16.097595+0000","scrub_duration":0.0046679920000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"72'8","reported_seq":1795,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.810996+0000","last_change":"2024-09-14T05:06:43.859008+0000","last_active":"2024-09-14T05:07:06.810996+0000","last_peered":"2024-09-14T05:07:06.810996+0000","last_clean":"2024-09-14T05:07:06.810996+0000","last_became_active":"2024-09-14T05:06:43.854258+0000","last_became_peered":"2024-09-14T05:06:43.854258+0000","last_unstale":"2024-09-14T05:07:06.810996+0000","last_undegraded":"2024-09-14T05:07:06.810996+0000","last_fullsized":"2024-09-14T05:07:06.810996+0000","mapping_epoch":1644,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1645,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:14:37.679962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":78,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.811693+0000","last_change":"2024-09-14T05:06:38.813475+0000","last_active":"2024-09-14T05:07:06.811693+0000","last_peered":"2024-09-14T05:07:06.811693+0000","last_clean":"2024-09-14T05:07:06.811693+0000","last_became_active":"2024-09-14T05:06:38.809830+0000","last_became_peered":"2024-09-14T05:06:38.809830+0000","last_unstale":"2024-09-14T05:07:06.811693+0000","last_undegraded":"2024-09-14T05:07:06.811693+0000","last_fullsized":"2024-09-14T05:07:06.811693+0000","mapping_epoch":1642,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:06:15.388664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.13","version":"349'149","reported_seq":922,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.616302+0000","last_change":"2024-09-14T05:06:26.749615+0000","last_active":"2024-09-14T05:07:06.616302+0000","last_peered":"2024-09-14T05:07:06.616302+0000","last_clean":"2024-09-14T05:07:06.616302+0000","last_became_active":"2024-09-14T05:06:25.642891+0000","last_became_peered":"2024-09-14T05:06:25.642891+0000","last_unstale":"2024-09-14T05:07:06.616302+0000","last_undegraded":"2024-09-14T05:07:06.616302+0000","last_fullsized":"2024-09-14T05:07:06.616302+0000","mapping_epoch":1637,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1638,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:06:26.749547+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:06:26.749547+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T10:40:06.826100+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":1660,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.557735+0000","last_change":"2024-09-14T05:05:49.403096+0000","last_active":"2024-09-14T05:07:06.557735+0000","last_peered":"2024-09-14T05:07:06.557735+0000","last_clean":"2024-09-14T05:07:06.557735+0000","last_became_active":"2024-09-14T05:05:49.402744+0000","last_became_peered":"2024-09-14T05:05:49.402744+0000","last_unstale":"2024-09-14T05:07:06.557735+0000","last_undegraded":"2024-09-14T05:07:06.557735+0000","last_fullsized":"2024-09-14T05:07:06.557735+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T07:59:55.538846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":2614,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.501930+0000","last_change":"2024-09-14T04:54:52.250655+0000","last_active":"2024-09-14T05:07:06.501930+0000","last_peered":"2024-09-14T05:07:06.501930+0000","last_clean":"2024-09-14T05:07:06.501930+0000","last_became_active":"2024-09-14T04:54:52.250317+0000","last_became_peered":"2024-09-14T04:54:52.250317+0000","last_unstale":"2024-09-14T05:07:06.501930+0000","last_undegraded":"2024-09-14T05:07:06.501930+0000","last_fullsized":"2024-09-14T05:07:06.501930+0000","mapping_epoch":1503,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1504,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:41:12.985406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.10","version":"349'149","reported_seq":803,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.253446+0000","last_change":"2024-09-14T05:06:53.916919+0000","last_active":"2024-09-14T05:07:06.253446+0000","last_peered":"2024-09-14T05:07:06.253446+0000","last_clean":"2024-09-14T05:07:06.253446+0000","last_became_active":"2024-09-14T05:06:53.916514+0000","last_became_peered":"2024-09-14T05:06:53.916514+0000","last_unstale":"2024-09-14T05:07:06.253446+0000","last_undegraded":"2024-09-14T05:07:06.253446+0000","last_fullsized":"2024-09-14T05:07:06.253446+0000","mapping_epoch":1652,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1653,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:36.569513+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:36.569513+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:05:06.212971+0000","scrub_duration":0.0087120909999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":2600,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.501645+0000","last_change":"2024-09-14T04:07:58.560845+0000","last_active":"2024-09-14T05:07:06.501645+0000","last_peered":"2024-09-14T05:07:06.501645+0000","last_clean":"2024-09-14T05:07:06.501645+0000","last_became_active":"2024-09-14T04:07:58.560522+0000","last_became_peered":"2024-09-14T04:07:58.560522+0000","last_unstale":"2024-09-14T05:07:06.501645+0000","last_undegraded":"2024-09-14T05:07:06.501645+0000","last_fullsized":"2024-09-14T05:07:06.501645+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T05:22:38.470918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":1634,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.253896+0000","last_change":"2024-09-14T04:07:58.511820+0000","last_active":"2024-09-14T05:07:06.253896+0000","last_peered":"2024-09-14T05:07:06.253896+0000","last_clean":"2024-09-14T05:07:06.253896+0000","last_became_active":"2024-09-14T04:07:58.511153+0000","last_became_peered":"2024-09-14T04:07:58.511153+0000","last_unstale":"2024-09-14T05:07:06.253896+0000","last_undegraded":"2024-09-14T05:07:06.253896+0000","last_fullsized":"2024-09-14T05:07:06.253896+0000","mapping_epoch":701,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":702,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T06:02:15.356552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.11","version":"349'149","reported_seq":886,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.501483+0000","last_change":"2024-09-14T05:06:15.551990+0000","last_active":"2024-09-14T05:07:06.501483+0000","last_peered":"2024-09-14T05:07:06.501483+0000","last_clean":"2024-09-14T05:07:06.501483+0000","last_became_active":"2024-09-14T05:06:15.551627+0000","last_became_peered":"2024-09-14T05:06:15.551627+0000","last_unstale":"2024-09-14T05:07:06.501483+0000","last_undegraded":"2024-09-14T05:07:06.501483+0000","last_fullsized":"2024-09-14T05:07:06.501483+0000","mapping_epoch":1633,"log_start":"0'0","ondisk_log_start":"0'0","created":1543,"last_epoch_clean":1634,"parent":"0.0","parent_split_bits":5,"last_scrub":"349'149","last_scrub_stamp":"2024-09-14T05:00:40.677843+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:49:18.237706+0000","last_clean_scrub_stamp":"2024-09-14T05:00:40.677843+0000","objects_scrubbed":0,"log_size":149,"log_dups_size":0,"ondisk_log_size":149,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T16:36:43.830637+0000","scrub_duration":5,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,3],"acting":[0,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"72'2","reported_seq":1701,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.557954+0000","last_change":"2024-09-14T05:05:49.426093+0000","last_active":"2024-09-14T05:07:06.557954+0000","last_peered":"2024-09-14T05:07:06.557954+0000","last_clean":"2024-09-14T05:07:06.557954+0000","last_became_active":"2024-09-14T05:05:49.408363+0000","last_became_peered":"2024-09-14T05:05:49.408363+0000","last_unstale":"2024-09-14T05:07:06.557954+0000","last_undegraded":"2024-09-14T05:07:06.557954+0000","last_fullsized":"2024-09-14T05:07:06.557954+0000","mapping_epoch":1622,"log_start":"0'0","ondisk_log_start":"0'0","created":68,"last_epoch_clean":1623,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:04.216682+0000","last_clean_scrub_stamp":"2024-09-14T03:53:04.216682+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T12:16:03.971185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":2090,"reported_epoch":1657,"state":"active+clean","last_fresh":"2024-09-14T05:07:06.811653+0000","last_change":"2024-09-14T05:06:38.811815+0000","last_active":"2024-09-14T05:07:06.811653+0000","last_peered":"2024-09-14T05:07:06.811653+0000","last_clean":"2024-09-14T05:07:06.811653+0000","last_became_active":"2024-09-14T05:06:38.810155+0000","last_became_peered":"2024-09-14T05:06:38.810155+0000","last_unstale":"2024-09-14T05:07:06.811653+0000","last_undegraded":"2024-09-14T05:07:06.811653+0000","last_fullsized":"2024-09-14T05:07:06.811653+0000","mapping_epoch":1642,"log_start":"0'0","ondisk_log_start":"0'0","created":69,"last_epoch_clean":1643,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-09-14T03:53:05.218501+0000","last_clean_scrub_stamp":"2024-09-14T03:53:05.218501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-09-15T15:05:53.775016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]}],"pool_stats":[{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4619,"ondisk_log_size":4619,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":57,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":24,"acting":24,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":31,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":82,"num_bytes_recovered":10118,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4250,"internal_metadata":0},"log_size":56,"ondisk_log_size":56,"up":96,"acting":96,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223202205,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":947956,"kb_used_data":14504,"kb_used_omap":1,"kb_used_meta":933438,"kb_avail":92797196,"statfs":{"total":95995035648,"available":95024328704,"internally_reserved":0,"allocated":14852096,"data_stored":8560291,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":955840971},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463840,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1052200,"kb_used_data":14556,"kb_used_omap":1,"kb_used_meta":1037630,"kb_avail":92692952,"statfs":{"total":95995035648,"available":94917582848,"internally_reserved":0,"allocated":14905344,"data_stored":8625156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":1062533575},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503725478,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1038180,"kb_used_data":14488,"kb_used_omap":1,"kb_used_meta":1023678,"kb_avail":92706972,"statfs":{"total":95995035648,"available":94931939328,"internally_reserved":0,"allocated":14835712,"data_stored":8555990,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":1048246726},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":1392,"seq":5978594476323,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":532656,"kb_used_data":14244,"kb_used_omap":117,"kb_used_meta":518282,"kb_avail":93212496,"statfs":{"total":95995035648,"available":95449595904,"internally_reserved":0,"allocated":14585856,"data_stored":8298155,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":120638,"internal_metadata":530720962},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":701,"seq":3010772075219,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1141980,"kb_used_data":14544,"kb_used_omap":71,"kb_used_meta":1127352,"kb_avail":92603172,"statfs":{"total":95995035648,"available":94825648128,"internally_reserved":0,"allocated":14893056,"data_stored":8621035,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":72778,"internal_metadata":1154409398},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":694,"seq":2980707304154,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":840672,"kb_used_data":14484,"kb_used_omap":68,"kb_used_meta":826107,"kb_avail":92904480,"statfs":{"total":95995035648,"available":95134187520,"internally_reserved":0,"allocated":14831616,"data_stored":8555947,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":70448,"internal_metadata":845933776},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":681,"seq":2924872729313,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":818136,"kb_used_data":14220,"kb_used_omap":67,"kb_used_meta":803836,"kb_avail":92927016,"statfs":{"total":95995035648,"available":95157264384,"internally_reserved":0,"allocated":14561280,"data_stored":8292662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":69275,"internal_metadata":823128421},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":671,"seq":2881923056364,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":872412,"kb_used_data":14544,"kb_used_omap":74,"kb_used_meta":857781,"kb_avail":92872740,"statfs":{"total":95995035648,"available":95101685760,"internally_reserved":0,"allocated":14893056,"data_stored":8620927,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":76107,"internal_metadata":878368437},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2349,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1901,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-09-14T05:08:00.621 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2024-09-14T05:08:00.621 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi039 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-09-14T05:08:00.812 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=7b6fa71ab43a43809d5e82130819b6ba Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 48, in task with parallel.parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa/tasks/workunit.py", line 126, in task with parallel() as p: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_1c941a66c1b420259b8fc5ac19e792dbbf60aded/qa/tasks/workunit.py", line 434, in _run_tests remote.run( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/remote.py", line 523, in run r = self._runner(client=self.ssh, name=self.shortname, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 455, in run r.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed (workunit test cls/test_cls_rbd.sh) on smithi039 with status 1: 'mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh' 2024-09-14T05:08:00.816 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-09-14T05:08:00.825 INFO:tasks.cephadm:Teardown begin 2024-09-14T05:08:00.826 DEBUG:teuthology.orchestra.run.smithi039:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-14T05:08:00.857 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:08:00 smithi039 ceph-mon[106470]: from='client.47009 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T05:08:00.857 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:08:00 smithi039 ceph-mon[111235]: from='client.47009 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T05:08:00.889 DEBUG:teuthology.orchestra.run.smithi129:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-14T05:08:00.915 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:08:00 smithi129 ceph-mon[101316]: from='client.47009 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-09-14T05:08:00.944 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-09-14T05:08:00.945 DEBUG:teuthology.orchestra.run.smithi039:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-14T05:08:00.962 DEBUG:teuthology.orchestra.run.smithi129:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-09-14T05:08:01.010 INFO:tasks.cephadm:Stopping all daemons... 2024-09-14T05:08:01.011 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-09-14T05:08:01.011 DEBUG:teuthology.orchestra.run.smithi039:> sudo systemctl stop ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mon.a 2024-09-14T05:08:01.289 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:08:01 smithi039 systemd[1]: Stopping Ceph mon.a for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T05:08:01.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:08:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-a[106447]: 2024-09-14T05:08:01.435+0000 7f0007e55640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T05:08:01.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:08:01 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-a[106447]: 2024-09-14T05:08:01.435+0000 7f0007e55640 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-09-14T05:08:01.790 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:08:01 smithi039 podman[438345]: 2024-09-14 05:08:01.588358817 +0000 UTC m=+0.281107002 container died 7198d4eb8e4c1000ace8fa1414a8a80d152670ad9bb4f2deeb4ab26f42e2cbf9 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-a, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, RELEASE=main-a278231, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=) 2024-09-14T05:08:02.261 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:08:02 smithi039 podman[438345]: 2024-09-14 05:08:02.033920556 +0000 UTC m=+0.726668740 container cleanup 7198d4eb8e4c1000ace8fa1414a8a80d152670ad9bb4f2deeb4ab26f42e2cbf9 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-a, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD) 2024-09-14T05:08:02.261 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:08:02 smithi039 bash[438345]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-a 2024-09-14T05:08:02.511 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:08:02 smithi039 podman[438393]: 2024-09-14 05:08:02.260761215 +0000 UTC m=+0.658842744 container remove 7198d4eb8e4c1000ace8fa1414a8a80d152670ad9bb4f2deeb4ab26f42e2cbf9 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-a, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=main-a278231, org.label-schema.schema-version=1.0, GIT_CLEAN=True) 2024-09-14T05:08:02.511 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:08:02 smithi039 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mon.a.service: Deactivated successfully. 2024-09-14T05:08:02.512 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:08:02 smithi039 systemd[1]: Stopped Ceph mon.a for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T05:08:02.512 INFO:journalctl@ceph.mon.a.smithi039.stdout:Sep 14 05:08:02 smithi039 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mon.a.service: Consumed 1min 5.545s CPU time. 2024-09-14T05:08:02.514 DEBUG:teuthology.orchestra.run.smithi039:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mon.a.service' 2024-09-14T05:08:02.570 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T05:08:02.570 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-09-14T05:08:02.570 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-09-14T05:08:02.571 DEBUG:teuthology.orchestra.run.smithi039:> sudo systemctl stop ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mon.c 2024-09-14T05:08:02.790 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:08:02 smithi039 systemd[1]: Stopping Ceph mon.c for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T05:08:03.156 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:08:02 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:05:08:02] ENGINE Bus STOPPING 2024-09-14T05:08:03.157 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:08:02 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-c[111212]: 2024-09-14T05:08:02.891+0000 7fd013dbb640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T05:08:03.157 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:08:02 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-c[111212]: 2024-09-14T05:08:02.891+0000 7fd013dbb640 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-09-14T05:08:03.157 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:08:02 smithi039 podman[438509]: 2024-09-14 05:08:02.922877949 +0000 UTC m=+0.166682025 container died 4b8f0f3864b1af7e8d3d02cf209fc2683f759bae1d66de523deb4db0dd1d4ba0 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-c, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, RELEASE=main-a278231, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-14T05:08:03.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:08:03 smithi039 podman[438509]: 2024-09-14 05:08:03.382042248 +0000 UTC m=+0.625846323 container cleanup 4b8f0f3864b1af7e8d3d02cf209fc2683f759bae1d66de523deb4db0dd1d4ba0 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-c, ceph=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T05:08:03.540 INFO:journalctl@ceph.mon.c.smithi039.stdout:Sep 14 05:08:03 smithi039 bash[438509]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-c 2024-09-14T05:08:03.540 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:08:03 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:05:08:03] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-09-14T05:08:03.541 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:08:03 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:05:08:03] ENGINE Bus STOPPED 2024-09-14T05:08:03.541 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:08:03 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:05:08:03] ENGINE Bus STARTING 2024-09-14T05:08:03.541 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:08:03 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:05:08:03] ENGINE Serving on http://:::9283 2024-09-14T05:08:03.541 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:08:03 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: [14/Sep/2024:05:08:03] ENGINE Bus STARTED 2024-09-14T05:08:03.797 DEBUG:teuthology.orchestra.run.smithi039:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mon.c.service' 2024-09-14T05:08:03.833 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T05:08:03.833 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-09-14T05:08:03.833 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-09-14T05:08:03.833 DEBUG:teuthology.orchestra.run.smithi129:> sudo systemctl stop ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mon.b 2024-09-14T05:08:04.185 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:08:03 smithi129 systemd[1]: Stopping Ceph mon.b for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T05:08:04.520 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:08:04 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-b[101293]: 2024-09-14T05:08:04.182+0000 7f572c547640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T05:08:04.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:08:04 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-b[101293]: 2024-09-14T05:08:04.182+0000 7f572c547640 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-09-14T05:08:04.521 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:08:04 smithi129 podman[266275]: 2024-09-14 05:08:04.39818838 +0000 UTC m=+0.346406481 container died cff2519560f167e23398a4a0d0891211e49777809673ae8ad77db31e6701dbed (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-b, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, RELEASE=main-a278231, org.label-schema.vendor=CentOS) 2024-09-14T05:08:05.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:08:04 smithi129 podman[266275]: 2024-09-14 05:08:04.900696089 +0000 UTC m=+0.848914199 container cleanup cff2519560f167e23398a4a0d0891211e49777809673ae8ad77db31e6701dbed (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-b, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, RELEASE=main-a278231, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True) 2024-09-14T05:08:05.021 INFO:journalctl@ceph.mon.b.smithi129.stdout:Sep 14 05:08:04 smithi129 bash[266275]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mon-b 2024-09-14T05:08:05.269 DEBUG:teuthology.orchestra.run.smithi129:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mon.b.service' 2024-09-14T05:08:05.318 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T05:08:05.318 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-09-14T05:08:05.318 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-09-14T05:08:05.318 DEBUG:teuthology.orchestra.run.smithi039:> sudo systemctl stop ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mgr.y 2024-09-14T05:08:05.622 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:08:05 smithi039 systemd[1]: Stopping Ceph mgr.y for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T05:08:06.025 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:08:05 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T05:08:05.620+0000 7f26e9475640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T05:08:06.025 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:08:05 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y[89458]: 2024-09-14T05:08:05.620+0000 7f26e9475640 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-09-14T05:08:06.025 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:08:05 smithi039 podman[438654]: 2024-09-14 05:08:05.663272171 +0000 UTC m=+0.145108586 container died cccc87f635a64eb07c0504f6dd1c027e68a7a6f01229a42ba18179017ca16902 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.schema-version=1.0, RELEASE=main-a278231, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T05:08:06.289 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:08:06 smithi039 podman[438654]: 2024-09-14 05:08:06.195759519 +0000 UTC m=+0.677595934 container cleanup cccc87f635a64eb07c0504f6dd1c027e68a7a6f01229a42ba18179017ca16902 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=main-a278231, GIT_BRANCH=HEAD) 2024-09-14T05:08:06.290 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:08:06 smithi039 bash[438654]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y 2024-09-14T05:08:06.638 DEBUG:teuthology.orchestra.run.smithi039:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mgr.y.service' 2024-09-14T05:08:06.660 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:08:06 smithi039 podman[438671]: 2024-09-14 05:08:06.397609868 +0000 UTC m=+0.730129100 container remove cccc87f635a64eb07c0504f6dd1c027e68a7a6f01229a42ba18179017ca16902 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-y, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, RELEASE=main-a278231, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-14T05:08:06.660 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:08:06 smithi039 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mgr.y.service: Deactivated successfully. 2024-09-14T05:08:06.661 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:08:06 smithi039 systemd[1]: Stopped Ceph mgr.y for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T05:08:06.661 INFO:journalctl@ceph.mgr.y.smithi039.stdout:Sep 14 05:08:06 smithi039 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mgr.y.service: Consumed 1min 9.155s CPU time. 2024-09-14T05:08:06.674 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T05:08:06.675 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-09-14T05:08:06.675 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-09-14T05:08:06.675 DEBUG:teuthology.orchestra.run.smithi129:> sudo systemctl stop ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mgr.x 2024-09-14T05:08:07.012 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 05:08:06 smithi129 systemd[1]: Stopping Ceph mgr.x for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T05:08:07.271 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 05:08:07 smithi129 podman[266407]: 2024-09-14 05:08:07.0116849 +0000 UTC m=+0.145115369 container died 448bd4e146cbc29c4c675b1e56cbb6fd5a41183ca740be695d39ce07342e7aad (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T05:08:07.655 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 05:08:07 smithi129 podman[266407]: 2024-09-14 05:08:07.412100409 +0000 UTC m=+0.545530870 container cleanup 448bd4e146cbc29c4c675b1e56cbb6fd5a41183ca740be695d39ce07342e7aad (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True) 2024-09-14T05:08:07.655 INFO:journalctl@ceph.mgr.x.smithi129.stdout:Sep 14 05:08:07 smithi129 bash[266407]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-mgr-x 2024-09-14T05:08:07.790 DEBUG:teuthology.orchestra.run.smithi129:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@mgr.x.service' 2024-09-14T05:08:07.827 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T05:08:07.827 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-09-14T05:08:07.827 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-09-14T05:08:07.827 DEBUG:teuthology.orchestra.run.smithi039:> sudo systemctl stop ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.0 2024-09-14T05:08:08.162 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 05:08:07 smithi039 systemd[1]: Stopping Ceph osd.0 for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T05:08:08.540 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 05:08:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0[124075]: 2024-09-14T05:08:08.160+0000 7fb55305a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T05:08:08.540 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 05:08:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0[124075]: 2024-09-14T05:08:08.160+0000 7fb55305a640 -1 osd.0 1657 *** Got signal Terminated *** 2024-09-14T05:08:08.540 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 05:08:08 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0[124075]: 2024-09-14T05:08:08.160+0000 7fb55305a640 -1 osd.0 1657 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T05:08:13.945 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 05:08:13 smithi039 podman[438785]: 2024-09-14 05:08:13.662113572 +0000 UTC m=+5.639953951 container died 36686aa8a5e4fdfd2b22b9b843052d7d423eba1d57764b5532f236b95221911b (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-14T05:08:14.289 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 05:08:14 smithi039 podman[438785]: 2024-09-14 05:08:14.110016811 +0000 UTC m=+6.087857191 container cleanup 36686aa8a5e4fdfd2b22b9b843052d7d423eba1d57764b5532f236b95221911b (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, RELEASE=main-a278231, org.label-schema.license=GPLv2, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux ) 2024-09-14T05:08:14.290 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 05:08:14 smithi039 bash[438785]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0 2024-09-14T05:08:15.024 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 05:08:14 smithi039 podman[438805]: 2024-09-14 05:08:14.703433605 +0000 UTC m=+1.036338166 container remove 36686aa8a5e4fdfd2b22b9b843052d7d423eba1d57764b5532f236b95221911b (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=main-a278231, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T05:08:15.282 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 05:08:15 smithi039 podman[438900]: 2024-09-14 05:08:14.926993518 +0000 UTC m=+0.016728621 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T05:08:15.282 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 05:08:15 smithi039 podman[438900]: 2024-09-14 05:08:15.04598093 +0000 UTC m=+0.135716024 container create 34a9664a5d3dbf7d961ba784cb1df41614453f149474eff279e87ef8f4355b0e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-14T05:08:15.539 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 05:08:15 smithi039 podman[438900]: 2024-09-14 05:08:15.282130825 +0000 UTC m=+0.371865917 container init 34a9664a5d3dbf7d961ba784cb1df41614453f149474eff279e87ef8f4355b0e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.license=GPLv2, GIT_CLEAN=True) 2024-09-14T05:08:15.540 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 05:08:15 smithi039 podman[438900]: 2024-09-14 05:08:15.287717852 +0000 UTC m=+0.377452961 container start 34a9664a5d3dbf7d961ba784cb1df41614453f149474eff279e87ef8f4355b0e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=main-a278231, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, io.buildah.version=1.37.2) 2024-09-14T05:08:15.540 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 05:08:15 smithi039 podman[438900]: 2024-09-14 05:08:15.465601207 +0000 UTC m=+0.555336307 container attach 34a9664a5d3dbf7d961ba784cb1df41614453f149474eff279e87ef8f4355b0e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0-deactivate, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_CLEAN=True, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, org.label-schema.vendor=CentOS) 2024-09-14T05:08:16.714 INFO:journalctl@ceph.osd.0.smithi039.stdout:Sep 14 05:08:16 smithi039 podman[438957]: 2024-09-14 05:08:16.405987202 +0000 UTC m=+0.018202042 container died 34a9664a5d3dbf7d961ba784cb1df41614453f149474eff279e87ef8f4355b0e (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-0-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, ceph=True, maintainer=Guillaume Abrioux , RELEASE=main-a278231, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS) 2024-09-14T05:08:17.135 DEBUG:teuthology.orchestra.run.smithi039:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.0.service' 2024-09-14T05:08:17.184 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T05:08:17.184 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-09-14T05:08:17.184 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-09-14T05:08:17.184 DEBUG:teuthology.orchestra.run.smithi039:> sudo systemctl stop ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.1 2024-09-14T05:08:17.271 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 05:08:16 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=cleanup t=2024-09-14T05:08:16.902879498Z level=info msg="Completed cleanup jobs" duration=15.675894ms 2024-09-14T05:08:17.271 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 05:08:17 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=grafana.update.checker t=2024-09-14T05:08:17.030879736Z level=info msg="Update check succeeded" duration=59.238258ms 2024-09-14T05:08:17.271 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 05:08:17 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=plugins.update.checker t=2024-09-14T05:08:17.143792917Z level=info msg="Update check succeeded" duration=75.597368ms 2024-09-14T05:08:17.476 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 05:08:17 smithi039 systemd[1]: Stopping Ceph osd.1 for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T05:08:17.790 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 05:08:17 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1[131251]: 2024-09-14T05:08:17.474+0000 7f23b9a48640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T05:08:17.790 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 05:08:17 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1[131251]: 2024-09-14T05:08:17.474+0000 7f23b9a48640 -1 osd.1 1657 *** Got signal Terminated *** 2024-09-14T05:08:17.790 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 05:08:17 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1[131251]: 2024-09-14T05:08:17.474+0000 7f23b9a48640 -1 osd.1 1657 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T05:08:23.290 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 05:08:22 smithi039 podman[439042]: 2024-09-14 05:08:22.982643396 +0000 UTC m=+5.604937845 container died 6ff655878a341c2da1c4e7814b7b7b756a17e375213c9f2782fe3ee4d5e7aca5 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1, GIT_CLEAN=True, CEPH_POINT_RELEASE=, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-09-14T05:08:23.776 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 05:08:23 smithi039 podman[439042]: 2024-09-14 05:08:23.45723603 +0000 UTC m=+6.079530476 container cleanup 6ff655878a341c2da1c4e7814b7b7b756a17e375213c9f2782fe3ee4d5e7aca5 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, RELEASE=main-a278231, GIT_CLEAN=True) 2024-09-14T05:08:23.776 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 05:08:23 smithi039 bash[439042]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1 2024-09-14T05:08:24.040 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 05:08:23 smithi039 podman[439064]: 2024-09-14 05:08:23.775720478 +0000 UTC m=+0.768836270 container remove 6ff655878a341c2da1c4e7814b7b7b756a17e375213c9f2782fe3ee4d5e7aca5 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0) 2024-09-14T05:08:24.363 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 05:08:24 smithi039 podman[439159]: 2024-09-14 05:08:24.084902127 +0000 UTC m=+0.105578820 container create c9aa10d49dd74a59c9e22a7dea2805375b28df51756adce732349a099ab86365 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1-deactivate, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731) 2024-09-14T05:08:24.363 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 05:08:24 smithi039 podman[439159]: 2024-09-14 05:08:23.996048136 +0000 UTC m=+0.016724840 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T05:08:24.363 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 05:08:24 smithi039 podman[439159]: 2024-09-14 05:08:24.362454511 +0000 UTC m=+0.383131206 container init c9aa10d49dd74a59c9e22a7dea2805375b28df51756adce732349a099ab86365 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1-deactivate, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, ceph=True, RELEASE=main-a278231) 2024-09-14T05:08:24.789 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 05:08:24 smithi039 podman[439159]: 2024-09-14 05:08:24.367991308 +0000 UTC m=+0.388668000 container start c9aa10d49dd74a59c9e22a7dea2805375b28df51756adce732349a099ab86365 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True) 2024-09-14T05:08:24.790 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 05:08:24 smithi039 podman[439159]: 2024-09-14 05:08:24.479060587 +0000 UTC m=+0.499737298 container attach c9aa10d49dd74a59c9e22a7dea2805375b28df51756adce732349a099ab86365 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1-deactivate, RELEASE=main-a278231, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-09-14T05:08:24.790 INFO:journalctl@ceph.osd.1.smithi039.stdout:Sep 14 05:08:24 smithi039 podman[439159]: 2024-09-14 05:08:24.565513298 +0000 UTC m=+0.586189995 container died c9aa10d49dd74a59c9e22a7dea2805375b28df51756adce732349a099ab86365 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-1-deactivate, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, RELEASE=main-a278231, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T05:08:25.232 DEBUG:teuthology.orchestra.run.smithi039:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.1.service' 2024-09-14T05:08:25.276 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T05:08:25.276 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-09-14T05:08:25.276 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-09-14T05:08:25.277 DEBUG:teuthology.orchestra.run.smithi039:> sudo systemctl stop ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.2 2024-09-14T05:08:25.540 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 05:08:25 smithi039 systemd[1]: Stopping Ceph osd.2 for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T05:08:26.040 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 05:08:25 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2[137127]: 2024-09-14T05:08:25.637+0000 7f94c2ccd640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T05:08:26.040 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 05:08:25 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2[137127]: 2024-09-14T05:08:25.637+0000 7f94c2ccd640 -1 osd.2 1657 *** Got signal Terminated *** 2024-09-14T05:08:26.040 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 05:08:25 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2[137127]: 2024-09-14T05:08:25.637+0000 7f94c2ccd640 -1 osd.2 1657 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T05:08:31.021 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 05:08:30 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T05:08:30.697Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.39:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.39:8765: connect: connection refused" 2024-09-14T05:08:31.021 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 05:08:30 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T05:08:30.699Z caller=refresh.go:90 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.39:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.39:8765: connect: connection refused" 2024-09-14T05:08:31.021 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 05:08:30 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T05:08:30.700Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.39:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.39:8765: connect: connection refused" 2024-09-14T05:08:31.021 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 05:08:30 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T05:08:30.700Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.39:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.39:8765: connect: connection refused" 2024-09-14T05:08:31.021 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 05:08:30 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T05:08:30.700Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.39:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.39:8765: connect: connection refused" 2024-09-14T05:08:31.022 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 05:08:30 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T05:08:30.701Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=smb msg="Unable to refresh target groups" err="Get \"http://172.21.15.39:8765/sd/prometheus/sd-config?service=smb\": dial tcp 172.21.15.39:8765: connect: connection refused" 2024-09-14T05:08:31.022 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 05:08:30 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T05:08:30.703Z caller=refresh.go:90 level=error component="discovery manager scrape" discovery=http config=nfs msg="Unable to refresh target groups" err="Get \"http://172.21.15.39:8765/sd/prometheus/sd-config?service=nfs\": dial tcp 172.21.15.39:8765: connect: connection refused" 2024-09-14T05:08:31.429 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 05:08:31 smithi039 podman[439287]: 2024-09-14 05:08:31.147362783 +0000 UTC m=+5.642253554 container died 662e3d705b70827d44f715bdb57db06d2548bc5f6befb08f6f07fe1ad08712d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=main-a278231, org.label-schema.vendor=CentOS) 2024-09-14T05:08:31.790 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 05:08:31 smithi039 podman[439287]: 2024-09-14 05:08:31.602847935 +0000 UTC m=+6.097738737 container cleanup 662e3d705b70827d44f715bdb57db06d2548bc5f6befb08f6f07fe1ad08712d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231) 2024-09-14T05:08:31.790 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 05:08:31 smithi039 bash[439287]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2 2024-09-14T05:08:32.214 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 05:08:31 smithi039 podman[439308]: 2024-09-14 05:08:31.888158162 +0000 UTC m=+0.740557403 container remove 662e3d705b70827d44f715bdb57db06d2548bc5f6befb08f6f07fe1ad08712d6 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, maintainer=Guillaume Abrioux , RELEASE=main-a278231, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-14T05:08:32.536 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 05:08:32 smithi039 podman[439402]: 2024-09-14 05:08:32.21383405 +0000 UTC m=+0.105798816 container create 4d38e24e07ed52c91f41a6959f9a24fee07884e050b1143c7a622a2c4016fe1c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2-deactivate, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, RELEASE=main-a278231, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, io.buildah.version=1.37.2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-09-14T05:08:32.536 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 05:08:32 smithi039 podman[439402]: 2024-09-14 05:08:32.124707926 +0000 UTC m=+0.016672699 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T05:08:32.536 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 05:08:32 smithi039 podman[439402]: 2024-09-14 05:08:32.441195248 +0000 UTC m=+0.333160010 container init 4d38e24e07ed52c91f41a6959f9a24fee07884e050b1143c7a622a2c4016fe1c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2-deactivate, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, ceph=True) 2024-09-14T05:08:32.536 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 05:08:32 smithi039 podman[439402]: 2024-09-14 05:08:32.446211312 +0000 UTC m=+0.338176082 container start 4d38e24e07ed52c91f41a6959f9a24fee07884e050b1143c7a622a2c4016fe1c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=main-a278231, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_CLEAN=True) 2024-09-14T05:08:32.536 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 05:08:32 smithi039 podman[439402]: 2024-09-14 05:08:32.532734865 +0000 UTC m=+0.424699639 container attach 4d38e24e07ed52c91f41a6959f9a24fee07884e050b1143c7a622a2c4016fe1c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, RELEASE=main-a278231, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T05:08:32.789 INFO:journalctl@ceph.osd.2.smithi039.stdout:Sep 14 05:08:32 smithi039 podman[439402]: 2024-09-14 05:08:32.639719006 +0000 UTC m=+0.531683786 container died 4d38e24e07ed52c91f41a6959f9a24fee07884e050b1143c7a622a2c4016fe1c (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-2-deactivate, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.build-date=20240731, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, RELEASE=main-a278231, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-09-14T05:08:33.318 DEBUG:teuthology.orchestra.run.smithi039:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.2.service' 2024-09-14T05:08:33.360 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T05:08:33.360 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-09-14T05:08:33.360 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-09-14T05:08:33.360 DEBUG:teuthology.orchestra.run.smithi039:> sudo systemctl stop ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.3 2024-09-14T05:08:33.718 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 05:08:33 smithi039 systemd[1]: Stopping Ceph osd.3 for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T05:08:34.039 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 05:08:33 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3[143484]: 2024-09-14T05:08:33.717+0000 7f003c045640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T05:08:34.040 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 05:08:33 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3[143484]: 2024-09-14T05:08:33.717+0000 7f003c045640 -1 osd.3 1657 *** Got signal Terminated *** 2024-09-14T05:08:34.040 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 05:08:33 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3[143484]: 2024-09-14T05:08:33.717+0000 7f003c045640 -1 osd.3 1657 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T05:08:35.039 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 05:08:34 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3[143484]: 2024-09-14T05:08:34.708+0000 7f003865a640 -1 osd.3 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:10.580340+0000 front 2024-09-14T05:08:10.580374+0000 (oldest deadline 2024-09-14T05:08:34.680502+0000) 2024-09-14T05:08:35.270 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:34 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:34.798+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:08:36.039 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 05:08:35 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3[143484]: 2024-09-14T05:08:35.671+0000 7f003865a640 -1 osd.3 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:10.580340+0000 front 2024-09-14T05:08:10.580374+0000 (oldest deadline 2024-09-14T05:08:34.680502+0000) 2024-09-14T05:08:36.271 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:35 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:35.827+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:08:37.040 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 05:08:36 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3[143484]: 2024-09-14T05:08:36.649+0000 7f003865a640 -1 osd.3 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:10.580340+0000 front 2024-09-14T05:08:10.580374+0000 (oldest deadline 2024-09-14T05:08:34.680502+0000) 2024-09-14T05:08:37.199 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:36 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:36.841+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:08:37.521 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:37 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:37.197+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:10.594893+0000 front 2024-09-14T05:08:10.594828+0000 (oldest deadline 2024-09-14T05:08:36.494880+0000) 2024-09-14T05:08:38.039 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 05:08:37 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3[143484]: 2024-09-14T05:08:37.682+0000 7f003865a640 -1 osd.3 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:10.580340+0000 front 2024-09-14T05:08:10.580374+0000 (oldest deadline 2024-09-14T05:08:34.680502+0000) 2024-09-14T05:08:38.190 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:37 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:37.806+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:08:38.520 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:38 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:38.188+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:10.594893+0000 front 2024-09-14T05:08:10.594828+0000 (oldest deadline 2024-09-14T05:08:36.494880+0000) 2024-09-14T05:08:39.040 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 05:08:38 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3[143484]: 2024-09-14T05:08:38.697+0000 7f003865a640 -1 osd.3 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:10.580340+0000 front 2024-09-14T05:08:10.580374+0000 (oldest deadline 2024-09-14T05:08:34.680502+0000) 2024-09-14T05:08:39.119 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:38 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:38.815+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:08:39.521 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:39 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:39.237+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:10.594893+0000 front 2024-09-14T05:08:10.594828+0000 (oldest deadline 2024-09-14T05:08:36.494880+0000) 2024-09-14T05:08:39.521 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 05:08:39 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4[153765]: 2024-09-14T05:08:39.118+0000 7fa97a840640 -1 osd.4 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.268852+0000 front 2024-09-14T05:08:12.268842+0000 (oldest deadline 2024-09-14T05:08:38.168733+0000) 2024-09-14T05:08:39.533 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 05:08:39 smithi039 podman[439531]: 2024-09-14 05:08:39.182918828 +0000 UTC m=+5.594356328 container died 645ba846152cdedf7f4cbcbfbcb3354428da4b043a353cd003969652470c1657 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, RELEASE=main-a278231, ceph=True, org.label-schema.license=GPLv2) 2024-09-14T05:08:39.789 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 05:08:39 smithi039 podman[439531]: 2024-09-14 05:08:39.714809624 +0000 UTC m=+6.126247111 container cleanup 645ba846152cdedf7f4cbcbfbcb3354428da4b043a353cd003969652470c1657 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, ceph=True, org.label-schema.vendor=CentOS) 2024-09-14T05:08:39.790 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 05:08:39 smithi039 bash[439531]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3 2024-09-14T05:08:40.096 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:39 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:39.805+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:08:40.097 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:39 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T05:08:39.889+0000 7fe326da6640 -1 osd.5 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:07.937814+0000 front 2024-09-14T05:08:07.937866+0000 (oldest deadline 2024-09-14T05:08:39.138414+0000) 2024-09-14T05:08:40.267 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 05:08:39 smithi039 podman[439554]: 2024-09-14 05:08:39.949703076 +0000 UTC m=+0.740671212 container remove 645ba846152cdedf7f4cbcbfbcb3354428da4b043a353cd003969652470c1657 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3, org.label-schema.license=GPLv2, RELEASE=main-a278231, org.label-schema.build-date=20240731, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.37.2) 2024-09-14T05:08:40.520 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 05:08:40 smithi039 podman[439649]: 2024-09-14 05:08:40.267076407 +0000 UTC m=+0.112990333 container create dd6725984e776f9e5211fa95379568874e58d798c2365e831478dc426e627af3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, maintainer=Guillaume Abrioux ) 2024-09-14T05:08:40.520 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 05:08:40 smithi039 podman[439649]: 2024-09-14 05:08:40.17061312 +0000 UTC m=+0.016527048 image pull 8801d2529e4377d97015e6c85a3b4925ede7ce42c7349a1a37eb88f097b6f3f0 quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802 2024-09-14T05:08:40.520 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 05:08:40 smithi039 podman[439649]: 2024-09-14 05:08:40.519548178 +0000 UTC m=+0.365462104 container init dd6725984e776f9e5211fa95379568874e58d798c2365e831478dc426e627af3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3-deactivate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=main-a278231, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-09-14T05:08:40.521 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 05:08:40 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4[153765]: 2024-09-14T05:08:40.096+0000 7fa97a840640 -1 osd.4 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.268852+0000 front 2024-09-14T05:08:12.268842+0000 (oldest deadline 2024-09-14T05:08:38.168733+0000) 2024-09-14T05:08:40.521 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:40 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:40.266+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:10.594893+0000 front 2024-09-14T05:08:10.594828+0000 (oldest deadline 2024-09-14T05:08:36.494880+0000) 2024-09-14T05:08:40.790 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 05:08:40 smithi039 podman[439649]: 2024-09-14 05:08:40.524729755 +0000 UTC m=+0.370643680 container start dd6725984e776f9e5211fa95379568874e58d798c2365e831478dc426e627af3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3-deactivate, CEPH_POINT_RELEASE=, RELEASE=main-a278231, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, ceph=True) 2024-09-14T05:08:40.790 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 05:08:40 smithi039 podman[439649]: 2024-09-14 05:08:40.611603256 +0000 UTC m=+0.457517195 container attach dd6725984e776f9e5211fa95379568874e58d798c2365e831478dc426e627af3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=main-a278231, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7) 2024-09-14T05:08:40.790 INFO:journalctl@ceph.osd.3.smithi039.stdout:Sep 14 05:08:40 smithi039 podman[439649]: 2024-09-14 05:08:40.720397714 +0000 UTC m=+0.566311641 container died dd6725984e776f9e5211fa95379568874e58d798c2365e831478dc426e627af3 (image=quay.ceph.io/ceph-ci/ceph@sha256:862445896ef1df9aa3acdb9cfb761812e8e943c37f87d60be1b51d2242056802, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-3-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=main-a278231, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_COMMIT=c5aaba5e3282b30e4782f2b5d6e4e362e22dfcb7, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-09-14T05:08:41.144 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:40 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T05:08:40.895+0000 7fe326da6640 -1 osd.5 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:07.937814+0000 front 2024-09-14T05:08:07.937866+0000 (oldest deadline 2024-09-14T05:08:39.138414+0000) 2024-09-14T05:08:41.145 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:40 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:40.778+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:08:41.520 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:41.298+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:10.594893+0000 front 2024-09-14T05:08:10.594828+0000 (oldest deadline 2024-09-14T05:08:36.494880+0000) 2024-09-14T05:08:41.521 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 05:08:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4[153765]: 2024-09-14T05:08:41.143+0000 7fa97a840640 -1 osd.4 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.268852+0000 front 2024-09-14T05:08:12.268842+0000 (oldest deadline 2024-09-14T05:08:38.168733+0000) 2024-09-14T05:08:41.547 DEBUG:teuthology.orchestra.run.smithi039:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.3.service' 2024-09-14T05:08:41.593 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T05:08:41.593 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-09-14T05:08:41.593 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-09-14T05:08:41.593 DEBUG:teuthology.orchestra.run.smithi129:> sudo systemctl stop ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.4 2024-09-14T05:08:41.798 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 05:08:41 smithi129 systemd[1]: Stopping Ceph osd.4 for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T05:08:42.081 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T05:08:41.845+0000 7fe326da6640 -1 osd.5 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:07.937814+0000 front 2024-09-14T05:08:07.937866+0000 (oldest deadline 2024-09-14T05:08:39.138414+0000) 2024-09-14T05:08:42.081 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:41.795+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:08:42.082 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 05:08:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4[153765]: 2024-09-14T05:08:41.882+0000 7fa97e22c640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T05:08:42.082 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 05:08:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4[153765]: 2024-09-14T05:08:41.882+0000 7fa97e22c640 -1 osd.4 1657 *** Got signal Terminated *** 2024-09-14T05:08:42.082 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 05:08:41 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4[153765]: 2024-09-14T05:08:41.882+0000 7fa97e22c640 -1 osd.4 1657 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T05:08:42.521 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 05:08:42 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4[153765]: 2024-09-14T05:08:42.189+0000 7fa97a840640 -1 osd.4 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.268852+0000 front 2024-09-14T05:08:12.268842+0000 (oldest deadline 2024-09-14T05:08:38.168733+0000) 2024-09-14T05:08:42.521 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:42 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:42.308+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:10.594893+0000 front 2024-09-14T05:08:10.594828+0000 (oldest deadline 2024-09-14T05:08:36.494880+0000) 2024-09-14T05:08:43.195 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:42 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T05:08:42.856+0000 7fe326da6640 -1 osd.5 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:07.937814+0000 front 2024-09-14T05:08:07.937866+0000 (oldest deadline 2024-09-14T05:08:39.138414+0000) 2024-09-14T05:08:43.196 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:42 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:42.798+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:08:43.521 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 05:08:43 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4[153765]: 2024-09-14T05:08:43.194+0000 7fa97a840640 -1 osd.4 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.268852+0000 front 2024-09-14T05:08:12.268842+0000 (oldest deadline 2024-09-14T05:08:38.168733+0000) 2024-09-14T05:08:43.521 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:43 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:43.268+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:10.594893+0000 front 2024-09-14T05:08:10.594828+0000 (oldest deadline 2024-09-14T05:08:36.494880+0000) 2024-09-14T05:08:44.209 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:43 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T05:08:43.904+0000 7fe326da6640 -1 osd.5 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:07.937814+0000 front 2024-09-14T05:08:07.937866+0000 (oldest deadline 2024-09-14T05:08:39.138414+0000) 2024-09-14T05:08:44.210 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:43 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:43.815+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:08:44.521 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:44 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:44.260+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:10.594893+0000 front 2024-09-14T05:08:10.594828+0000 (oldest deadline 2024-09-14T05:08:36.494880+0000) 2024-09-14T05:08:44.521 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 05:08:44 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4[153765]: 2024-09-14T05:08:44.208+0000 7fa97a840640 -1 osd.4 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.268852+0000 front 2024-09-14T05:08:12.268842+0000 (oldest deadline 2024-09-14T05:08:38.168733+0000) 2024-09-14T05:08:45.246 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:44 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T05:08:44.876+0000 7fe326da6640 -1 osd.5 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:07.937814+0000 front 2024-09-14T05:08:07.937866+0000 (oldest deadline 2024-09-14T05:08:39.138414+0000) 2024-09-14T05:08:45.246 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:44 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:44.789+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:08:45.521 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:45 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:45.245+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:10.594893+0000 front 2024-09-14T05:08:10.594828+0000 (oldest deadline 2024-09-14T05:08:36.494880+0000) 2024-09-14T05:08:45.521 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 05:08:45 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4[153765]: 2024-09-14T05:08:45.249+0000 7fa97a840640 -1 osd.4 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.268852+0000 front 2024-09-14T05:08:12.268842+0000 (oldest deadline 2024-09-14T05:08:38.168733+0000) 2024-09-14T05:08:46.243 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:45 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T05:08:45.840+0000 7fe326da6640 -1 osd.5 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:07.937814+0000 front 2024-09-14T05:08:07.937866+0000 (oldest deadline 2024-09-14T05:08:39.138414+0000) 2024-09-14T05:08:46.243 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:45 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:45.771+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:08:46.520 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:46 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:46.263+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:10.594893+0000 front 2024-09-14T05:08:10.594828+0000 (oldest deadline 2024-09-14T05:08:36.494880+0000) 2024-09-14T05:08:46.521 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 05:08:46 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4[153765]: 2024-09-14T05:08:46.242+0000 7fa97a840640 -1 osd.4 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.268852+0000 front 2024-09-14T05:08:12.268842+0000 (oldest deadline 2024-09-14T05:08:38.168733+0000) 2024-09-14T05:08:47.020 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:46 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T05:08:46.816+0000 7fe326da6640 -1 osd.5 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:07.937814+0000 front 2024-09-14T05:08:07.937866+0000 (oldest deadline 2024-09-14T05:08:39.138414+0000) 2024-09-14T05:08:47.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:46 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:46.739+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:08:47.517 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 05:08:47 smithi129 podman[266556]: 2024-09-14 05:08:47.235694331 +0000 UTC m=+5.451046934 container died c25f04f58bc6ef694053d7b1d5fba3948cdfdb2f38e0a83dd4f9e8ecffa7da95 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, RELEASE=reef-e4e227b, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, org.label-schema.license=GPLv2) 2024-09-14T05:08:47.518 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:47 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:47.294+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:10.594893+0000 front 2024-09-14T05:08:10.594828+0000 (oldest deadline 2024-09-14T05:08:36.494880+0000) 2024-09-14T05:08:47.518 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:47 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:47.294+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.795854+0000 front 2024-09-14T05:08:21.795884+0000 (oldest deadline 2024-09-14T05:08:46.495751+0000) 2024-09-14T05:08:47.771 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:47 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:47.696+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:08:47.771 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:47 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:47.696+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.522264+0000 front 2024-09-14T05:08:21.522179+0000 (oldest deadline 2024-09-14T05:08:46.822241+0000) 2024-09-14T05:08:48.036 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:47 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T05:08:47.805+0000 7fe326da6640 -1 osd.5 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:07.937814+0000 front 2024-09-14T05:08:07.937866+0000 (oldest deadline 2024-09-14T05:08:39.138414+0000) 2024-09-14T05:08:48.036 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:47 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T05:08:47.805+0000 7fe326da6640 -1 osd.5 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:22.039406+0000 front 2024-09-14T05:08:22.039393+0000 (oldest deadline 2024-09-14T05:08:47.339345+0000) 2024-09-14T05:08:48.036 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 05:08:47 smithi129 podman[266556]: 2024-09-14 05:08:47.775294994 +0000 UTC m=+5.990647589 container cleanup c25f04f58bc6ef694053d7b1d5fba3948cdfdb2f38e0a83dd4f9e8ecffa7da95 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, RELEASE=reef-e4e227b, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T05:08:48.036 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 05:08:47 smithi129 bash[266556]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4 2024-09-14T05:08:48.305 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 05:08:48 smithi129 podman[266576]: 2024-09-14 05:08:48.035469587 +0000 UTC m=+0.798890690 container remove c25f04f58bc6ef694053d7b1d5fba3948cdfdb2f38e0a83dd4f9e8ecffa7da95 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, ceph=True, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, RELEASE=reef-e4e227b, org.label-schema.name=CentOS Stream 9 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-14T05:08:48.577 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:48 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:48.304+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:10.594893+0000 front 2024-09-14T05:08:10.594828+0000 (oldest deadline 2024-09-14T05:08:36.494880+0000) 2024-09-14T05:08:48.577 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:48 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:48.304+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.795854+0000 front 2024-09-14T05:08:21.795884+0000 (oldest deadline 2024-09-14T05:08:46.495751+0000) 2024-09-14T05:08:48.578 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 05:08:48 smithi129 podman[266670]: 2024-09-14 05:08:48.352463782 +0000 UTC m=+0.015744410 image pull 67ddd33fa72f26a51c969a9043e7a3ddf784506b3d1949ad815ae4dd791beec1 quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322 2024-09-14T05:08:48.578 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 05:08:48 smithi129 podman[266670]: 2024-09-14 05:08:48.461341723 +0000 UTC m=+0.124622351 container create 090220a5ca72f0f045ef2e90c6faaaba470aff92543615735c555427d3984e80 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=reef-e4e227b, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, GIT_CLEAN=True, io.buildah.version=1.37.2) 2024-09-14T05:08:48.839 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:48 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:48.651+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:08:48.839 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:48 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:48.651+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.522264+0000 front 2024-09-14T05:08:21.522179+0000 (oldest deadline 2024-09-14T05:08:46.822241+0000) 2024-09-14T05:08:48.839 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 05:08:48 smithi129 podman[266670]: 2024-09-14 05:08:48.680647273 +0000 UTC m=+0.343927902 container init 090220a5ca72f0f045ef2e90c6faaaba470aff92543615735c555427d3984e80 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=reef-e4e227b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T05:08:48.839 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 05:08:48 smithi129 podman[266670]: 2024-09-14 05:08:48.685309329 +0000 UTC m=+0.348589958 container start 090220a5ca72f0f045ef2e90c6faaaba470aff92543615735c555427d3984e80 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4-deactivate, org.label-schema.schema-version=1.0, io.buildah.version=1.37.2, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=reef-e4e227b, ceph=True, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=) 2024-09-14T05:08:48.839 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 05:08:48 smithi129 podman[266670]: 2024-09-14 05:08:48.763764542 +0000 UTC m=+0.427045179 container attach 090220a5ca72f0f045ef2e90c6faaaba470aff92543615735c555427d3984e80 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, RELEASE=reef-e4e227b, ceph=True, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-09-14T05:08:49.242 INFO:journalctl@ceph.osd.4.smithi129.stdout:Sep 14 05:08:48 smithi129 podman[266670]: 2024-09-14 05:08:48.905976614 +0000 UTC m=+0.569257244 container died 090220a5ca72f0f045ef2e90c6faaaba470aff92543615735c555427d3984e80 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-4-deactivate, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=reef-e4e227b, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2) 2024-09-14T05:08:49.243 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:48 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T05:08:48.837+0000 7fe326da6640 -1 osd.5 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:07.937814+0000 front 2024-09-14T05:08:07.937866+0000 (oldest deadline 2024-09-14T05:08:39.138414+0000) 2024-09-14T05:08:49.243 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:48 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T05:08:48.837+0000 7fe326da6640 -1 osd.5 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:22.039406+0000 front 2024-09-14T05:08:22.039393+0000 (oldest deadline 2024-09-14T05:08:47.339345+0000) 2024-09-14T05:08:49.520 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:49 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:49.310+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:10.594893+0000 front 2024-09-14T05:08:10.594828+0000 (oldest deadline 2024-09-14T05:08:36.494880+0000) 2024-09-14T05:08:49.521 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:49 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:49.310+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.795854+0000 front 2024-09-14T05:08:21.795884+0000 (oldest deadline 2024-09-14T05:08:46.495751+0000) 2024-09-14T05:08:49.554 DEBUG:teuthology.orchestra.run.smithi129:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.4.service' 2024-09-14T05:08:49.593 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T05:08:49.593 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-09-14T05:08:49.593 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-09-14T05:08:49.594 DEBUG:teuthology.orchestra.run.smithi129:> sudo systemctl stop ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.5 2024-09-14T05:08:49.812 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:49 smithi129 systemd[1]: Stopping Ceph osd.5 for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T05:08:49.812 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:49 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:49.641+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:08:49.813 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:49 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:49.641+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.522264+0000 front 2024-09-14T05:08:21.522179+0000 (oldest deadline 2024-09-14T05:08:46.822241+0000) 2024-09-14T05:08:50.271 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:49 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T05:08:49.810+0000 7fe326da6640 -1 osd.5 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:07.937814+0000 front 2024-09-14T05:08:07.937866+0000 (oldest deadline 2024-09-14T05:08:39.138414+0000) 2024-09-14T05:08:50.271 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:49 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T05:08:49.810+0000 7fe326da6640 -1 osd.5 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:22.039406+0000 front 2024-09-14T05:08:22.039393+0000 (oldest deadline 2024-09-14T05:08:47.339345+0000) 2024-09-14T05:08:50.271 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:49 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T05:08:49.905+0000 7fe32e79a640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T05:08:50.272 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:49 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T05:08:49.905+0000 7fe32e79a640 -1 osd.5 1657 *** Got signal Terminated *** 2024-09-14T05:08:50.272 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:49 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T05:08:49.905+0000 7fe32e79a640 -1 osd.5 1657 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T05:08:50.659 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:50 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:50.319+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:10.594893+0000 front 2024-09-14T05:08:10.594828+0000 (oldest deadline 2024-09-14T05:08:36.494880+0000) 2024-09-14T05:08:50.660 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:50 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:50.319+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.795854+0000 front 2024-09-14T05:08:21.795884+0000 (oldest deadline 2024-09-14T05:08:46.495751+0000) 2024-09-14T05:08:51.020 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:50 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:50.658+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:08:51.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:50 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:50.658+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.522264+0000 front 2024-09-14T05:08:21.522179+0000 (oldest deadline 2024-09-14T05:08:46.822241+0000) 2024-09-14T05:08:51.021 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:50 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T05:08:50.773+0000 7fe326da6640 -1 osd.5 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:07.937814+0000 front 2024-09-14T05:08:07.937866+0000 (oldest deadline 2024-09-14T05:08:39.138414+0000) 2024-09-14T05:08:51.021 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:50 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T05:08:50.773+0000 7fe326da6640 -1 osd.5 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:22.039406+0000 front 2024-09-14T05:08:22.039393+0000 (oldest deadline 2024-09-14T05:08:47.339345+0000) 2024-09-14T05:08:51.625 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:51 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:51.358+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:10.594893+0000 front 2024-09-14T05:08:10.594828+0000 (oldest deadline 2024-09-14T05:08:36.494880+0000) 2024-09-14T05:08:51.625 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:51 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:51.358+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.795854+0000 front 2024-09-14T05:08:21.795884+0000 (oldest deadline 2024-09-14T05:08:46.495751+0000) 2024-09-14T05:08:52.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:51 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:51.623+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:08:52.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:51 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:51.623+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.522264+0000 front 2024-09-14T05:08:21.522179+0000 (oldest deadline 2024-09-14T05:08:46.822241+0000) 2024-09-14T05:08:52.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:51 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:51.623+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:29.723330+0000 front 2024-09-14T05:08:29.723251+0000 (oldest deadline 2024-09-14T05:08:51.423183+0000) 2024-09-14T05:08:52.022 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:51 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T05:08:51.781+0000 7fe326da6640 -1 osd.5 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:07.937814+0000 front 2024-09-14T05:08:07.937866+0000 (oldest deadline 2024-09-14T05:08:39.138414+0000) 2024-09-14T05:08:52.022 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:51 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T05:08:51.781+0000 7fe326da6640 -1 osd.5 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:22.039406+0000 front 2024-09-14T05:08:22.039393+0000 (oldest deadline 2024-09-14T05:08:47.339345+0000) 2024-09-14T05:08:52.611 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:52 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:52.330+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:10.594893+0000 front 2024-09-14T05:08:10.594828+0000 (oldest deadline 2024-09-14T05:08:36.494880+0000) 2024-09-14T05:08:52.611 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:52 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:52.330+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.795854+0000 front 2024-09-14T05:08:21.795884+0000 (oldest deadline 2024-09-14T05:08:46.495751+0000) 2024-09-14T05:08:52.611 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:52 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:52.330+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:26.996906+0000 front 2024-09-14T05:08:26.996946+0000 (oldest deadline 2024-09-14T05:08:51.696749+0000) 2024-09-14T05:08:53.023 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:52 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:52.609+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:08:53.023 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:52 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:52.609+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.522264+0000 front 2024-09-14T05:08:21.522179+0000 (oldest deadline 2024-09-14T05:08:46.822241+0000) 2024-09-14T05:08:53.024 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:52 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:52.609+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:29.723330+0000 front 2024-09-14T05:08:29.723251+0000 (oldest deadline 2024-09-14T05:08:51.423183+0000) 2024-09-14T05:08:53.024 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:52 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T05:08:52.738+0000 7fe326da6640 -1 osd.5 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:07.937814+0000 front 2024-09-14T05:08:07.937866+0000 (oldest deadline 2024-09-14T05:08:39.138414+0000) 2024-09-14T05:08:53.024 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:52 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T05:08:52.738+0000 7fe326da6640 -1 osd.5 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:22.039406+0000 front 2024-09-14T05:08:22.039393+0000 (oldest deadline 2024-09-14T05:08:47.339345+0000) 2024-09-14T05:08:53.024 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:52 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T05:08:52.738+0000 7fe326da6640 -1 osd.5 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:27.340072+0000 front 2024-09-14T05:08:27.339998+0000 (oldest deadline 2024-09-14T05:08:52.039953+0000) 2024-09-14T05:08:53.734 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:53 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:53.369+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:10.594893+0000 front 2024-09-14T05:08:10.594828+0000 (oldest deadline 2024-09-14T05:08:36.494880+0000) 2024-09-14T05:08:53.734 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:53 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:53.369+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.795854+0000 front 2024-09-14T05:08:21.795884+0000 (oldest deadline 2024-09-14T05:08:46.495751+0000) 2024-09-14T05:08:53.734 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:53 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:53.369+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:26.996906+0000 front 2024-09-14T05:08:26.996946+0000 (oldest deadline 2024-09-14T05:08:51.696749+0000) 2024-09-14T05:08:53.734 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:53 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:53.614+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:08:53.735 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:53 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:53.614+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.522264+0000 front 2024-09-14T05:08:21.522179+0000 (oldest deadline 2024-09-14T05:08:46.822241+0000) 2024-09-14T05:08:53.735 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:53 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:53.614+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:29.723330+0000 front 2024-09-14T05:08:29.723251+0000 (oldest deadline 2024-09-14T05:08:51.423183+0000) 2024-09-14T05:08:54.021 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:53 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T05:08:53.732+0000 7fe326da6640 -1 osd.5 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:07.937814+0000 front 2024-09-14T05:08:07.937866+0000 (oldest deadline 2024-09-14T05:08:39.138414+0000) 2024-09-14T05:08:54.021 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:53 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T05:08:53.732+0000 7fe326da6640 -1 osd.5 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:22.039406+0000 front 2024-09-14T05:08:22.039393+0000 (oldest deadline 2024-09-14T05:08:47.339345+0000) 2024-09-14T05:08:54.021 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:53 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T05:08:53.732+0000 7fe326da6640 -1 osd.5 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:27.340072+0000 front 2024-09-14T05:08:27.339998+0000 (oldest deadline 2024-09-14T05:08:52.039953+0000) 2024-09-14T05:08:54.667 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:54 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:54.345+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:10.594893+0000 front 2024-09-14T05:08:10.594828+0000 (oldest deadline 2024-09-14T05:08:36.494880+0000) 2024-09-14T05:08:54.667 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:54 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:54.345+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.795854+0000 front 2024-09-14T05:08:21.795884+0000 (oldest deadline 2024-09-14T05:08:46.495751+0000) 2024-09-14T05:08:54.667 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:54 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:54.345+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:26.996906+0000 front 2024-09-14T05:08:26.996946+0000 (oldest deadline 2024-09-14T05:08:51.696749+0000) 2024-09-14T05:08:55.021 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:54 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T05:08:54.716+0000 7fe326da6640 -1 osd.5 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:07.937814+0000 front 2024-09-14T05:08:07.937866+0000 (oldest deadline 2024-09-14T05:08:39.138414+0000) 2024-09-14T05:08:55.021 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:54 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T05:08:54.716+0000 7fe326da6640 -1 osd.5 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:22.039406+0000 front 2024-09-14T05:08:22.039393+0000 (oldest deadline 2024-09-14T05:08:47.339345+0000) 2024-09-14T05:08:55.021 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:54 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5[44758]: 2024-09-14T05:08:54.716+0000 7fe326da6640 -1 osd.5 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:27.340072+0000 front 2024-09-14T05:08:27.339998+0000 (oldest deadline 2024-09-14T05:08:52.039953+0000) 2024-09-14T05:08:55.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:54 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:54.665+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:08:55.022 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:54 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:54.665+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.522264+0000 front 2024-09-14T05:08:21.522179+0000 (oldest deadline 2024-09-14T05:08:46.822241+0000) 2024-09-14T05:08:55.022 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:54 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:54.665+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:29.723330+0000 front 2024-09-14T05:08:29.723251+0000 (oldest deadline 2024-09-14T05:08:51.423183+0000) 2024-09-14T05:08:55.694 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:55 smithi129 podman[266798]: 2024-09-14 05:08:55.404919953 +0000 UTC m=+5.591887168 container died f2d991a754f722987b0a4b057f169aafe768b1f1c9a23c0cda07f5348ee05776 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5, org.label-schema.license=GPLv2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, io.buildah.version=1.37.2, RELEASE=reef-e4e227b, org.label-schema.name=CentOS Stream 9 Base Image, ceph=True, org.label-schema.build-date=20240731, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-09-14T05:08:55.694 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:55 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:55.345+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:10.594893+0000 front 2024-09-14T05:08:10.594828+0000 (oldest deadline 2024-09-14T05:08:36.494880+0000) 2024-09-14T05:08:55.694 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:55 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:55.345+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.795854+0000 front 2024-09-14T05:08:21.795884+0000 (oldest deadline 2024-09-14T05:08:46.495751+0000) 2024-09-14T05:08:55.694 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:55 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:55.345+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:26.996906+0000 front 2024-09-14T05:08:26.996946+0000 (oldest deadline 2024-09-14T05:08:51.696749+0000) 2024-09-14T05:08:56.020 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:55 smithi129 podman[266798]: 2024-09-14 05:08:55.870854901 +0000 UTC m=+6.057822115 container cleanup f2d991a754f722987b0a4b057f169aafe768b1f1c9a23c0cda07f5348ee05776 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, RELEASE=reef-e4e227b, ceph=True, io.buildah.version=1.37.2, org.label-schema.vendor=CentOS) 2024-09-14T05:08:56.021 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:55 smithi129 bash[266798]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5 2024-09-14T05:08:56.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:55 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:55.701+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:08:56.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:55 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:55.701+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.522264+0000 front 2024-09-14T05:08:21.522179+0000 (oldest deadline 2024-09-14T05:08:46.822241+0000) 2024-09-14T05:08:56.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:55 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:55.701+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:29.723330+0000 front 2024-09-14T05:08:29.723251+0000 (oldest deadline 2024-09-14T05:08:51.423183+0000) 2024-09-14T05:08:56.485 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:56 smithi129 podman[266822]: 2024-09-14 05:08:56.138973612 +0000 UTC m=+0.719838810 container remove f2d991a754f722987b0a4b057f169aafe768b1f1c9a23c0cda07f5348ee05776 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5, RELEASE=reef-e4e227b, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-09-14T05:08:56.485 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:56 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:56.312+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:10.594893+0000 front 2024-09-14T05:08:10.594828+0000 (oldest deadline 2024-09-14T05:08:36.494880+0000) 2024-09-14T05:08:56.485 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:56 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:56.312+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.795854+0000 front 2024-09-14T05:08:21.795884+0000 (oldest deadline 2024-09-14T05:08:46.495751+0000) 2024-09-14T05:08:56.486 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:56 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:56.312+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:26.996906+0000 front 2024-09-14T05:08:26.996946+0000 (oldest deadline 2024-09-14T05:08:51.696749+0000) 2024-09-14T05:08:56.768 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:56 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:56.692+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:08:56.768 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:56 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:56.692+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.522264+0000 front 2024-09-14T05:08:21.522179+0000 (oldest deadline 2024-09-14T05:08:46.822241+0000) 2024-09-14T05:08:56.768 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:56 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:56.692+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:29.723330+0000 front 2024-09-14T05:08:29.723251+0000 (oldest deadline 2024-09-14T05:08:51.423183+0000) 2024-09-14T05:08:56.768 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:56 smithi129 podman[266913]: 2024-09-14 05:08:56.387583616 +0000 UTC m=+0.015873583 image pull 67ddd33fa72f26a51c969a9043e7a3ddf784506b3d1949ad815ae4dd791beec1 quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322 2024-09-14T05:08:56.769 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:56 smithi129 podman[266913]: 2024-09-14 05:08:56.548334846 +0000 UTC m=+0.176624800 container create 48eca603baa4ad1b88e6b3574471edff44dec820b2a4fe498c43b14a617e932f (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5-deactivate, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, RELEASE=reef-e4e227b) 2024-09-14T05:08:56.769 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:56 smithi129 podman[266913]: 2024-09-14 05:08:56.767853353 +0000 UTC m=+0.396143312 container init 48eca603baa4ad1b88e6b3574471edff44dec820b2a4fe498c43b14a617e932f (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5-deactivate, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=reef-e4e227b) 2024-09-14T05:08:57.021 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:56 smithi129 podman[266913]: 2024-09-14 05:08:56.773460442 +0000 UTC m=+0.401750396 container start 48eca603baa4ad1b88e6b3574471edff44dec820b2a4fe498c43b14a617e932f (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5-deactivate, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, GIT_CLEAN=True, RELEASE=reef-e4e227b, io.buildah.version=1.37.2, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-09-14T05:08:57.021 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:56 smithi129 podman[266913]: 2024-09-14 05:08:56.851016383 +0000 UTC m=+0.479306345 container attach 48eca603baa4ad1b88e6b3574471edff44dec820b2a4fe498c43b14a617e932f (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5-deactivate, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, ceph=True, RELEASE=reef-e4e227b, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0) 2024-09-14T05:08:57.021 INFO:journalctl@ceph.osd.5.smithi129.stdout:Sep 14 05:08:56 smithi129 podman[266913]: 2024-09-14 05:08:56.963001365 +0000 UTC m=+0.591291337 container died 48eca603baa4ad1b88e6b3574471edff44dec820b2a4fe498c43b14a617e932f (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-5-deactivate, RELEASE=reef-e4e227b, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.name=CentOS Stream 9 Base Image) 2024-09-14T05:08:57.521 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:57 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:57.336+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:10.594893+0000 front 2024-09-14T05:08:10.594828+0000 (oldest deadline 2024-09-14T05:08:36.494880+0000) 2024-09-14T05:08:57.521 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:57 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:57.336+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.795854+0000 front 2024-09-14T05:08:21.795884+0000 (oldest deadline 2024-09-14T05:08:46.495751+0000) 2024-09-14T05:08:57.521 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:57 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:57.336+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:26.996906+0000 front 2024-09-14T05:08:26.996946+0000 (oldest deadline 2024-09-14T05:08:51.696749+0000) 2024-09-14T05:08:57.646 DEBUG:teuthology.orchestra.run.smithi129:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.5.service' 2024-09-14T05:08:57.686 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T05:08:57.686 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-09-14T05:08:57.686 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-09-14T05:08:57.687 DEBUG:teuthology.orchestra.run.smithi129:> sudo systemctl stop ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.6 2024-09-14T05:08:58.019 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:57 smithi129 systemd[1]: Stopping Ceph osd.6 for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T05:08:58.019 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:58 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:58.017+0000 7f0799947640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T05:08:58.020 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:57 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:57.650+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:08:58.020 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:57 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:57.650+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.522264+0000 front 2024-09-14T05:08:21.522179+0000 (oldest deadline 2024-09-14T05:08:46.822241+0000) 2024-09-14T05:08:58.020 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:57 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:57.650+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:29.723330+0000 front 2024-09-14T05:08:29.723251+0000 (oldest deadline 2024-09-14T05:08:51.423183+0000) 2024-09-14T05:08:58.270 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:58 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:58.017+0000 7f0799947640 -1 osd.6 1657 *** Got signal Terminated *** 2024-09-14T05:08:58.271 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:58 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:58.017+0000 7f0799947640 -1 osd.6 1657 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T05:08:58.667 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:58 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:58.317+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:10.594893+0000 front 2024-09-14T05:08:10.594828+0000 (oldest deadline 2024-09-14T05:08:36.494880+0000) 2024-09-14T05:08:58.667 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:58 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:58.317+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.795854+0000 front 2024-09-14T05:08:21.795884+0000 (oldest deadline 2024-09-14T05:08:46.495751+0000) 2024-09-14T05:08:58.667 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:58 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:58.317+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:26.996906+0000 front 2024-09-14T05:08:26.996946+0000 (oldest deadline 2024-09-14T05:08:51.696749+0000) 2024-09-14T05:08:59.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:58 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:58.665+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:08:59.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:58 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:58.665+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.522264+0000 front 2024-09-14T05:08:21.522179+0000 (oldest deadline 2024-09-14T05:08:46.822241+0000) 2024-09-14T05:08:59.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:58 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:58.665+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:29.723330+0000 front 2024-09-14T05:08:29.723251+0000 (oldest deadline 2024-09-14T05:08:51.423183+0000) 2024-09-14T05:08:59.668 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:59 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:59.344+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:10.594893+0000 front 2024-09-14T05:08:10.594828+0000 (oldest deadline 2024-09-14T05:08:36.494880+0000) 2024-09-14T05:08:59.668 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:59 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:59.344+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.795854+0000 front 2024-09-14T05:08:21.795884+0000 (oldest deadline 2024-09-14T05:08:46.495751+0000) 2024-09-14T05:08:59.669 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:59 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:59.344+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:26.996906+0000 front 2024-09-14T05:08:26.996946+0000 (oldest deadline 2024-09-14T05:08:51.696749+0000) 2024-09-14T05:08:59.669 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:08:59 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:08:59.344+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6830 osd.3 since back 2024-09-14T05:08:35.797763+0000 front 2024-09-14T05:08:35.797786+0000 (oldest deadline 2024-09-14T05:08:59.297644+0000) 2024-09-14T05:09:00.020 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:59 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:59.667+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:09:00.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:59 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:59.667+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.522264+0000 front 2024-09-14T05:08:21.522179+0000 (oldest deadline 2024-09-14T05:08:46.822241+0000) 2024-09-14T05:09:00.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:08:59 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:08:59.667+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:29.723330+0000 front 2024-09-14T05:08:29.723251+0000 (oldest deadline 2024-09-14T05:08:51.423183+0000) 2024-09-14T05:09:00.625 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:09:00 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:09:00.360+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:10.594893+0000 front 2024-09-14T05:08:10.594828+0000 (oldest deadline 2024-09-14T05:08:36.494880+0000) 2024-09-14T05:09:00.625 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:09:00 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:09:00.360+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.795854+0000 front 2024-09-14T05:08:21.795884+0000 (oldest deadline 2024-09-14T05:08:46.495751+0000) 2024-09-14T05:09:00.626 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:09:00 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:09:00.360+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:26.996906+0000 front 2024-09-14T05:08:26.996946+0000 (oldest deadline 2024-09-14T05:08:51.696749+0000) 2024-09-14T05:09:00.626 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:09:00 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:09:00.360+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6830 osd.3 since back 2024-09-14T05:08:35.797763+0000 front 2024-09-14T05:08:35.797786+0000 (oldest deadline 2024-09-14T05:08:59.297644+0000) 2024-09-14T05:09:00.888 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:00 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:00.624+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:09:00.888 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:00 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:00.624+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.522264+0000 front 2024-09-14T05:08:21.522179+0000 (oldest deadline 2024-09-14T05:08:46.822241+0000) 2024-09-14T05:09:00.888 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:00 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:00.624+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:29.723330+0000 front 2024-09-14T05:08:29.723251+0000 (oldest deadline 2024-09-14T05:08:51.423183+0000) 2024-09-14T05:09:01.771 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:01 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:01.603+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:09:01.771 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:01 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:01.603+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.522264+0000 front 2024-09-14T05:08:21.522179+0000 (oldest deadline 2024-09-14T05:08:46.822241+0000) 2024-09-14T05:09:01.771 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:01 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:01.603+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:29.723330+0000 front 2024-09-14T05:08:29.723251+0000 (oldest deadline 2024-09-14T05:08:51.423183+0000) 2024-09-14T05:09:01.772 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:09:01 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:09:01.359+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:10.594893+0000 front 2024-09-14T05:08:10.594828+0000 (oldest deadline 2024-09-14T05:08:36.494880+0000) 2024-09-14T05:09:01.772 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:09:01 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:09:01.359+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.795854+0000 front 2024-09-14T05:08:21.795884+0000 (oldest deadline 2024-09-14T05:08:46.495751+0000) 2024-09-14T05:09:01.772 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:09:01 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:09:01.359+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:26.996906+0000 front 2024-09-14T05:08:26.996946+0000 (oldest deadline 2024-09-14T05:08:51.696749+0000) 2024-09-14T05:09:01.772 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:09:01 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:09:01.359+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6830 osd.3 since back 2024-09-14T05:08:35.797763+0000 front 2024-09-14T05:08:35.797786+0000 (oldest deadline 2024-09-14T05:08:59.297644+0000) 2024-09-14T05:09:02.771 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:09:02 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:09:02.386+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:10.594893+0000 front 2024-09-14T05:08:10.594828+0000 (oldest deadline 2024-09-14T05:08:36.494880+0000) 2024-09-14T05:09:02.771 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:09:02 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:09:02.386+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.795854+0000 front 2024-09-14T05:08:21.795884+0000 (oldest deadline 2024-09-14T05:08:46.495751+0000) 2024-09-14T05:09:02.771 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:09:02 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:09:02.386+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:26.996906+0000 front 2024-09-14T05:08:26.996946+0000 (oldest deadline 2024-09-14T05:08:51.696749+0000) 2024-09-14T05:09:02.771 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:09:02 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6[51099]: 2024-09-14T05:09:02.386+0000 7f0791f53640 -1 osd.6 1657 heartbeat_check: no reply from 172.21.15.39:6830 osd.3 since back 2024-09-14T05:08:35.797763+0000 front 2024-09-14T05:08:35.797786+0000 (oldest deadline 2024-09-14T05:08:59.297644+0000) 2024-09-14T05:09:02.772 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:02 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:02.594+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:09:02.772 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:02 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:02.594+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.522264+0000 front 2024-09-14T05:08:21.522179+0000 (oldest deadline 2024-09-14T05:08:46.822241+0000) 2024-09-14T05:09:02.772 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:02 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:02.594+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:29.723330+0000 front 2024-09-14T05:08:29.723251+0000 (oldest deadline 2024-09-14T05:08:51.423183+0000) 2024-09-14T05:09:02.772 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:02 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:02.594+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6830 osd.3 since back 2024-09-14T05:08:37.125168+0000 front 2024-09-14T05:08:37.125153+0000 (oldest deadline 2024-09-14T05:09:02.425013+0000) 2024-09-14T05:09:03.992 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:09:03 smithi129 podman[267042]: 2024-09-14 05:09:03.689160828 +0000 UTC m=+5.782295785 container died 3060eaf44106abea46e24816b225297cb6c4d1ed0902044c9f85bb8c09f49ed0 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=reef-e4e227b, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_CLEAN=True, ceph=True, io.buildah.version=1.37.2, org.label-schema.build-date=20240731) 2024-09-14T05:09:03.993 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:03 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:03.629+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:09:03.993 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:03 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:03.629+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.522264+0000 front 2024-09-14T05:08:21.522179+0000 (oldest deadline 2024-09-14T05:08:46.822241+0000) 2024-09-14T05:09:03.993 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:03 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:03.629+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:29.723330+0000 front 2024-09-14T05:08:29.723251+0000 (oldest deadline 2024-09-14T05:08:51.423183+0000) 2024-09-14T05:09:03.993 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:03 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:03.629+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6830 osd.3 since back 2024-09-14T05:08:37.125168+0000 front 2024-09-14T05:08:37.125153+0000 (oldest deadline 2024-09-14T05:09:02.425013+0000) 2024-09-14T05:09:04.270 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:09:04 smithi129 podman[267042]: 2024-09-14 05:09:04.174622836 +0000 UTC m=+6.267757828 container cleanup 3060eaf44106abea46e24816b225297cb6c4d1ed0902044c9f85bb8c09f49ed0 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6, GIT_BRANCH=HEAD, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-e4e227b, io.buildah.version=1.37.2) 2024-09-14T05:09:04.271 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:09:04 smithi129 bash[267042]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6 2024-09-14T05:09:04.770 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:09:04 smithi129 podman[267075]: 2024-09-14 05:09:04.451270139 +0000 UTC m=+0.754544411 container remove 3060eaf44106abea46e24816b225297cb6c4d1ed0902044c9f85bb8c09f49ed0 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6, ceph=True, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_CLEAN=True, RELEASE=reef-e4e227b, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240731) 2024-09-14T05:09:04.771 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:04 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:04.625+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:09:04.771 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:04 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:04.625+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.522264+0000 front 2024-09-14T05:08:21.522179+0000 (oldest deadline 2024-09-14T05:08:46.822241+0000) 2024-09-14T05:09:04.771 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:04 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:04.625+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:29.723330+0000 front 2024-09-14T05:08:29.723251+0000 (oldest deadline 2024-09-14T05:08:51.423183+0000) 2024-09-14T05:09:04.771 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:04 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:04.625+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6830 osd.3 since back 2024-09-14T05:08:37.125168+0000 front 2024-09-14T05:08:37.125153+0000 (oldest deadline 2024-09-14T05:09:02.425013+0000) 2024-09-14T05:09:05.113 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:09:04 smithi129 podman[267166]: 2024-09-14 05:09:04.686553516 +0000 UTC m=+0.016171967 image pull 67ddd33fa72f26a51c969a9043e7a3ddf784506b3d1949ad815ae4dd791beec1 quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322 2024-09-14T05:09:05.113 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:09:04 smithi129 podman[267166]: 2024-09-14 05:09:04.802210143 +0000 UTC m=+0.131828628 container create 9f29949651ed5b22457b29e7d85d91708c30e2485d79d46b0e9704e5cde5ef77 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6-deactivate, RELEASE=reef-e4e227b, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240731, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-14T05:09:05.113 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:09:05 smithi129 podman[267166]: 2024-09-14 05:09:05.021303526 +0000 UTC m=+0.350921979 container init 9f29949651ed5b22457b29e7d85d91708c30e2485d79d46b0e9704e5cde5ef77 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=reef-e4e227b, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True) 2024-09-14T05:09:05.114 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:09:05 smithi129 podman[267166]: 2024-09-14 05:09:05.026481205 +0000 UTC m=+0.356099649 container start 9f29949651ed5b22457b29e7d85d91708c30e2485d79d46b0e9704e5cde5ef77 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6-deactivate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=reef-e4e227b, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, ceph=True) 2024-09-14T05:09:05.114 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:09:05 smithi129 podman[267166]: 2024-09-14 05:09:05.11262045 +0000 UTC m=+0.442238904 container attach 9f29949651ed5b22457b29e7d85d91708c30e2485d79d46b0e9704e5cde5ef77 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6-deactivate, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, io.buildah.version=1.37.2, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=reef-e4e227b, org.label-schema.schema-version=1.0) 2024-09-14T05:09:05.521 INFO:journalctl@ceph.osd.6.smithi129.stdout:Sep 14 05:09:05 smithi129 podman[267166]: 2024-09-14 05:09:05.211081723 +0000 UTC m=+0.540700178 container died 9f29949651ed5b22457b29e7d85d91708c30e2485d79d46b0e9704e5cde5ef77 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-6-deactivate, io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.build-date=20240731, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , RELEASE=reef-e4e227b, GIT_CLEAN=True, org.label-schema.license=GPLv2) 2024-09-14T05:09:06.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:05 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:05.583+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:09:06.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:05 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:05.583+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.522264+0000 front 2024-09-14T05:08:21.522179+0000 (oldest deadline 2024-09-14T05:08:46.822241+0000) 2024-09-14T05:09:06.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:05 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:05.583+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:29.723330+0000 front 2024-09-14T05:08:29.723251+0000 (oldest deadline 2024-09-14T05:08:51.423183+0000) 2024-09-14T05:09:06.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:05 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:05.583+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6830 osd.3 since back 2024-09-14T05:08:37.125168+0000 front 2024-09-14T05:08:37.125153+0000 (oldest deadline 2024-09-14T05:09:02.425013+0000) 2024-09-14T05:09:06.137 DEBUG:teuthology.orchestra.run.smithi129:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.6.service' 2024-09-14T05:09:06.183 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T05:09:06.183 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-09-14T05:09:06.184 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-09-14T05:09:06.184 DEBUG:teuthology.orchestra.run.smithi129:> sudo systemctl stop ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.7 2024-09-14T05:09:06.464 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:06 smithi129 systemd[1]: Stopping Ceph osd.7 for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T05:09:06.464 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:06.462+0000 7ff016db6640 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-09-14T05:09:06.464 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:06.462+0000 7ff016db6640 -1 osd.7 1657 *** Got signal Terminated *** 2024-09-14T05:09:06.464 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:06.462+0000 7ff016db6640 -1 osd.7 1657 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-09-14T05:09:06.770 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:06.605+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:09:06.771 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:06.605+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.522264+0000 front 2024-09-14T05:08:21.522179+0000 (oldest deadline 2024-09-14T05:08:46.822241+0000) 2024-09-14T05:09:06.771 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:06.605+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:29.723330+0000 front 2024-09-14T05:08:29.723251+0000 (oldest deadline 2024-09-14T05:08:51.423183+0000) 2024-09-14T05:09:06.771 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:06 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:06.605+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6830 osd.3 since back 2024-09-14T05:08:37.125168+0000 front 2024-09-14T05:08:37.125153+0000 (oldest deadline 2024-09-14T05:09:02.425013+0000) 2024-09-14T05:09:08.020 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:07.644+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:09:08.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:07.644+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.522264+0000 front 2024-09-14T05:08:21.522179+0000 (oldest deadline 2024-09-14T05:08:46.822241+0000) 2024-09-14T05:09:08.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:07.644+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:29.723330+0000 front 2024-09-14T05:08:29.723251+0000 (oldest deadline 2024-09-14T05:08:51.423183+0000) 2024-09-14T05:09:08.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:07 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:07.644+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6830 osd.3 since back 2024-09-14T05:08:37.125168+0000 front 2024-09-14T05:08:37.125153+0000 (oldest deadline 2024-09-14T05:09:02.425013+0000) 2024-09-14T05:09:09.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:08 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:08.692+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:09:09.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:08 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:08.692+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.522264+0000 front 2024-09-14T05:08:21.522179+0000 (oldest deadline 2024-09-14T05:08:46.822241+0000) 2024-09-14T05:09:09.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:08 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:08.692+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:29.723330+0000 front 2024-09-14T05:08:29.723251+0000 (oldest deadline 2024-09-14T05:08:51.423183+0000) 2024-09-14T05:09:09.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:08 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:08.692+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6830 osd.3 since back 2024-09-14T05:08:37.125168+0000 front 2024-09-14T05:08:37.125153+0000 (oldest deadline 2024-09-14T05:09:02.425013+0000) 2024-09-14T05:09:09.022 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:08 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:08.692+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.129:6804 osd.4 since back 2024-09-14T05:08:42.425368+0000 front 2024-09-14T05:08:42.425389+0000 (oldest deadline 2024-09-14T05:09:08.325412+0000) 2024-09-14T05:09:10.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:09 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:09.647+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:09:10.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:09 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:09.648+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.522264+0000 front 2024-09-14T05:08:21.522179+0000 (oldest deadline 2024-09-14T05:08:46.822241+0000) 2024-09-14T05:09:10.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:09 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:09.648+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:29.723330+0000 front 2024-09-14T05:08:29.723251+0000 (oldest deadline 2024-09-14T05:08:51.423183+0000) 2024-09-14T05:09:10.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:09 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:09.648+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6830 osd.3 since back 2024-09-14T05:08:37.125168+0000 front 2024-09-14T05:08:37.125153+0000 (oldest deadline 2024-09-14T05:09:02.425013+0000) 2024-09-14T05:09:10.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:09 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:09.648+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.129:6804 osd.4 since back 2024-09-14T05:08:42.425368+0000 front 2024-09-14T05:08:42.425389+0000 (oldest deadline 2024-09-14T05:09:08.325412+0000) 2024-09-14T05:09:11.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:10 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:10.673+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6806 osd.0 since back 2024-09-14T05:08:12.320331+0000 front 2024-09-14T05:08:12.320311+0000 (oldest deadline 2024-09-14T05:08:34.020294+0000) 2024-09-14T05:09:11.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:10 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:10.673+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6814 osd.1 since back 2024-09-14T05:08:21.522264+0000 front 2024-09-14T05:08:21.522179+0000 (oldest deadline 2024-09-14T05:08:46.822241+0000) 2024-09-14T05:09:11.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:10 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:10.673+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6822 osd.2 since back 2024-09-14T05:08:29.723330+0000 front 2024-09-14T05:08:29.723251+0000 (oldest deadline 2024-09-14T05:08:51.423183+0000) 2024-09-14T05:09:11.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:10 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:10.673+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.39:6830 osd.3 since back 2024-09-14T05:08:37.125168+0000 front 2024-09-14T05:08:37.125153+0000 (oldest deadline 2024-09-14T05:09:02.425013+0000) 2024-09-14T05:09:11.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:10 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7[57036]: 2024-09-14T05:09:10.673+0000 7ff0133ca640 -1 osd.7 1657 heartbeat_check: no reply from 172.21.15.129:6804 osd.4 since back 2024-09-14T05:08:42.425368+0000 front 2024-09-14T05:08:42.425389+0000 (oldest deadline 2024-09-14T05:09:08.325412+0000) 2024-09-14T05:09:12.270 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:12 smithi129 podman[267294]: 2024-09-14 05:09:12.065302843 +0000 UTC m=+5.696005382 container died 4c13b97060d1bc922b57e4a0fd14ffe2ffa1d7ae8a753cc6da5462cf69fa8026 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, RELEASE=reef-e4e227b, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T05:09:12.679 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:12 smithi129 podman[267294]: 2024-09-14 05:09:12.477870728 +0000 UTC m=+6.108573312 container cleanup 4c13b97060d1bc922b57e4a0fd14ffe2ffa1d7ae8a753cc6da5462cf69fa8026 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=reef-e4e227b, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965) 2024-09-14T05:09:12.679 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:12 smithi129 bash[267294]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7 2024-09-14T05:09:13.021 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:12 smithi129 podman[267314]: 2024-09-14 05:09:12.679272964 +0000 UTC m=+0.609436049 container remove 4c13b97060d1bc922b57e4a0fd14ffe2ffa1d7ae8a753cc6da5462cf69fa8026 (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240731, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 9 Base Image, RELEASE=reef-e4e227b, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, io.buildah.version=1.37.2) 2024-09-14T05:09:13.349 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:13 smithi129 podman[267407]: 2024-09-14 05:09:12.999598608 +0000 UTC m=+0.015867991 image pull 67ddd33fa72f26a51c969a9043e7a3ddf784506b3d1949ad815ae4dd791beec1 quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322 2024-09-14T05:09:13.350 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:13 smithi129 podman[267407]: 2024-09-14 05:09:13.130064548 +0000 UTC m=+0.146333977 container create 1671d4be5d98d6a0b567fd438cb51e6c6689f42a204e87aba9b780f010ede79d (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, RELEASE=reef-e4e227b, ceph=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-09-14T05:09:13.350 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:13 smithi129 podman[267407]: 2024-09-14 05:09:13.349361407 +0000 UTC m=+0.365630791 container init 1671d4be5d98d6a0b567fd438cb51e6c6689f42a204e87aba9b780f010ede79d (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7-deactivate, io.buildah.version=1.37.2, CEPH_POINT_RELEASE=, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240731, org.label-schema.name=CentOS Stream 9 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=reef-e4e227b, org.label-schema.license=GPLv2) 2024-09-14T05:09:13.771 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:13 smithi129 podman[267407]: 2024-09-14 05:09:13.353919545 +0000 UTC m=+0.370188938 container start 1671d4be5d98d6a0b567fd438cb51e6c6689f42a204e87aba9b780f010ede79d (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7-deactivate, RELEASE=reef-e4e227b, GIT_CLEAN=True, io.buildah.version=1.37.2, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, org.label-schema.build-date=20240731, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-09-14T05:09:13.771 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:13 smithi129 podman[267407]: 2024-09-14 05:09:13.432620099 +0000 UTC m=+0.448889498 container attach 1671d4be5d98d6a0b567fd438cb51e6c6689f42a204e87aba9b780f010ede79d (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7-deactivate, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, io.buildah.version=1.37.2, org.label-schema.build-date=20240731, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=reef-e4e227b, maintainer=Guillaume Abrioux , GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-09-14T05:09:13.771 INFO:journalctl@ceph.osd.7.smithi129.stdout:Sep 14 05:09:13 smithi129 podman[267407]: 2024-09-14 05:09:13.539541594 +0000 UTC m=+0.555810980 container died 1671d4be5d98d6a0b567fd438cb51e6c6689f42a204e87aba9b780f010ede79d (image=quay.ceph.io/ceph-ci/ceph@sha256:94ec087b0651344ea44bf4ca432cec3d5bcba32599243d0588e250fe952c8322, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-osd-7-deactivate, ceph=True, GIT_COMMIT=009c5a8162e3d9e92e49bb850b92bd3cd406d965, org.label-schema.build-date=20240731, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.37.2, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 9 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=reef-e4e227b, org.label-schema.vendor=CentOS, GIT_CLEAN=True) 2024-09-14T05:09:14.167 DEBUG:teuthology.orchestra.run.smithi129:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@osd.7.service' 2024-09-14T05:09:14.205 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T05:09:14.205 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-09-14T05:09:14.205 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-09-14T05:09:14.205 DEBUG:teuthology.orchestra.run.smithi129:> sudo systemctl stop ceph-bd7c803c-724b-11ef-bceb-c7b262605968@prometheus.a 2024-09-14T05:09:14.521 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 05:09:14 smithi129 systemd[1]: Stopping Ceph prometheus.a for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T05:09:14.809 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 05:09:14 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T05:09:14.524Z caller=main.go:964 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-09-14T05:09:14.809 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 05:09:14 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T05:09:14.524Z caller=main.go:988 level=info msg="Stopping scrape discovery manager..." 2024-09-14T05:09:14.809 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 05:09:14 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T05:09:14.524Z caller=main.go:1002 level=info msg="Stopping notify discovery manager..." 2024-09-14T05:09:14.809 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 05:09:14 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T05:09:14.524Z caller=manager.go:177 level=info component="rule manager" msg="Stopping rule manager..." 2024-09-14T05:09:14.809 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 05:09:14 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T05:09:14.524Z caller=main.go:984 level=info msg="Scrape discovery manager stopped" 2024-09-14T05:09:14.810 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 05:09:14 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T05:09:14.524Z caller=main.go:998 level=info msg="Notify discovery manager stopped" 2024-09-14T05:09:14.810 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 05:09:14 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T05:09:14.524Z caller=manager.go:187 level=info component="rule manager" msg="Rule manager stopped" 2024-09-14T05:09:14.810 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 05:09:14 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T05:09:14.524Z caller=main.go:1039 level=info msg="Stopping scrape manager..." 2024-09-14T05:09:14.810 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 05:09:14 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T05:09:14.524Z caller=main.go:1031 level=info msg="Scrape manager stopped" 2024-09-14T05:09:14.810 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 05:09:14 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T05:09:14.527Z caller=notifier.go:618 level=info component=notifier msg="Stopping notification manager..." 2024-09-14T05:09:14.810 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 05:09:14 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T05:09:14.528Z caller=main.go:1261 level=info msg="Notifier manager stopped" 2024-09-14T05:09:14.811 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 05:09:14 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a[88192]: ts=2024-09-14T05:09:14.528Z caller=main.go:1273 level=info msg="See you next time!" 2024-09-14T05:09:14.811 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 05:09:14 smithi129 podman[267534]: 2024-09-14 05:09:14.534996868 +0000 UTC m=+0.111024427 container died cbe583f852c302c3b2474b6cacf83b5238d42bf36b2830094bdd2fe629043ae2 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T05:09:15.265 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 05:09:14 smithi129 podman[267534]: 2024-09-14 05:09:14.997016704 +0000 UTC m=+0.573044264 container cleanup cbe583f852c302c3b2474b6cacf83b5238d42bf36b2830094bdd2fe629043ae2 (image=quay.io/prometheus/prometheus:v2.51.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a, maintainer=The Prometheus Authors ) 2024-09-14T05:09:15.265 INFO:journalctl@ceph.prometheus.a.smithi129.stdout:Sep 14 05:09:14 smithi129 bash[267534]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-prometheus-a 2024-09-14T05:09:15.412 DEBUG:teuthology.orchestra.run.smithi129:> sudo pkill -f 'journalctl -f -n 0 -u ceph-bd7c803c-724b-11ef-bceb-c7b262605968@prometheus.a.service' 2024-09-14T05:09:15.446 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-09-14T05:09:15.446 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-09-14T05:09:15.446 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid bd7c803c-724b-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-14T05:09:15.608 INFO:teuthology.orchestra.run.smithi039.stdout:Deleting cluster with fsid: bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T05:09:17.290 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 05:09:17 smithi039 systemd[1]: Stopping Ceph alertmanager.a for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T05:09:17.705 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 05:09:17 smithi039 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a[101661]: ts=2024-09-14T05:09:17.334Z caller=main.go:542 level=info msg="Received SIGTERM, exiting gracefully..." 2024-09-14T05:09:17.705 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 05:09:17 smithi039 podman[440071]: 2024-09-14 05:09:17.386321435 +0000 UTC m=+0.168989082 container died 6b564e1fc909fd6269e079207b1e58b491b75b1fc472c889595a37d94205a5fa (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T05:09:18.040 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 05:09:17 smithi039 podman[440071]: 2024-09-14 05:09:17.876973635 +0000 UTC m=+0.659641320 container cleanup 6b564e1fc909fd6269e079207b1e58b491b75b1fc472c889595a37d94205a5fa (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T05:09:18.040 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 05:09:17 smithi039 bash[440071]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a 2024-09-14T05:09:18.435 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 05:09:18 smithi039 podman[440089]: 2024-09-14 05:09:18.128315048 +0000 UTC m=+0.735101605 container remove 6b564e1fc909fd6269e079207b1e58b491b75b1fc472c889595a37d94205a5fa (image=quay.io/prometheus/alertmanager:v0.27.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-alertmanager-a, maintainer=The Prometheus Authors ) 2024-09-14T05:09:18.435 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 05:09:18 smithi039 podman[440089]: 2024-09-14 05:09:18.212664708 +0000 UTC m=+0.819451222 volume remove bcb5d36c46ace19d3d3d71d9584dda079e73910e851904db49e40e562e9661e6 2024-09-14T05:09:18.790 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 05:09:18 smithi039 systemd[1]: Stopping Ceph node-exporter.a for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T05:09:18.790 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 05:09:18 smithi039 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-09-14T05:09:18.790 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 05:09:18 smithi039 systemd[1]: Stopped Ceph alertmanager.a for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T05:09:18.791 INFO:journalctl@ceph.alertmanager.a.smithi039.stdout:Sep 14 05:09:18 smithi039 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@alertmanager.a.service: Consumed 5.920s CPU time. 2024-09-14T05:09:19.182 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 05:09:18 smithi039 podman[440199]: 2024-09-14 05:09:18.911268238 +0000 UTC m=+0.120160074 container died 7dd5945adb64a31c2a4dcf8e4d7310dbb46275db071b3365f12944e05932726d (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T05:09:19.539 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 05:09:19 smithi039 podman[440199]: 2024-09-14 05:09:19.381477918 +0000 UTC m=+0.590369753 container cleanup 7dd5945adb64a31c2a4dcf8e4d7310dbb46275db071b3365f12944e05932726d (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T05:09:19.540 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 05:09:19 smithi039 bash[440199]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a 2024-09-14T05:09:19.987 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 05:09:19 smithi039 podman[440216]: 2024-09-14 05:09:19.68286892 +0000 UTC m=+0.775649338 container remove 7dd5945adb64a31c2a4dcf8e4d7310dbb46275db071b3365f12944e05932726d (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-a, maintainer=The Prometheus Authors ) 2024-09-14T05:09:19.987 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 05:09:19 smithi039 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-09-14T05:09:19.988 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 05:09:19 smithi039 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-09-14T05:09:19.988 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 05:09:19 smithi039 systemd[1]: Stopped Ceph node-exporter.a for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T05:09:19.988 INFO:journalctl@ceph.node-exporter.a.smithi039.stdout:Sep 14 05:09:19 smithi039 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@node-exporter.a.service: Consumed 12.577s CPU time. 2024-09-14T05:09:25.730 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid bd7c803c-724b-11ef-bceb-c7b262605968 --force --keep-logs 2024-09-14T05:09:25.883 INFO:teuthology.orchestra.run.smithi129.stdout:Deleting cluster with fsid: bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T05:09:27.463 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 05:09:27 smithi129 systemd[1]: Stopping Ceph grafana.a for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T05:09:27.769 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 05:09:27 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=server t=2024-09-14T05:09:27.462047813Z level=info msg="Shutdown started" reason="System signal: terminated" 2024-09-14T05:09:27.770 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 05:09:27 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=ticker t=2024-09-14T05:09:27.462405069Z level=info msg=stopped last_tick=2024-09-14T05:09:20Z 2024-09-14T05:09:27.770 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 05:09:27 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=tracing t=2024-09-14T05:09:27.462460196Z level=info msg="Closing tracing" 2024-09-14T05:09:27.770 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 05:09:27 smithi129 ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a[93191]: logger=grafana-apiserver t=2024-09-14T05:09:27.462489888Z level=info msg="StorageObjectCountTracker pruner is exiting" 2024-09-14T05:09:27.770 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 05:09:27 smithi129 podman[267956]: 2024-09-14 05:09:27.472663884 +0000 UTC m=+0.119111607 container died f0492c19de92f81c5baae4551bf1c20aa26f85e6173189cea5fb8ca3ba1a1730 (image=quay.io/ceph/grafana:10.4.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-14T05:09:28.020 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 05:09:27 smithi129 podman[267956]: 2024-09-14 05:09:27.951415371 +0000 UTC m=+0.597863128 container cleanup f0492c19de92f81c5baae4551bf1c20aa26f85e6173189cea5fb8ca3ba1a1730 (image=quay.io/ceph/grafana:10.4.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-14T05:09:28.021 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 05:09:27 smithi129 bash[267956]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a 2024-09-14T05:09:28.520 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 05:09:28 smithi129 podman[267974]: 2024-09-14 05:09:28.236095441 +0000 UTC m=+0.753652468 container remove f0492c19de92f81c5baae4551bf1c20aa26f85e6173189cea5fb8ca3ba1a1730 (image=quay.io/ceph/grafana:10.4.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-grafana-a, maintainer=Grafana Labs ) 2024-09-14T05:09:28.521 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 05:09:28 smithi129 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@grafana.a.service: Deactivated successfully. 2024-09-14T05:09:28.521 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 05:09:28 smithi129 systemd[1]: Stopped Ceph grafana.a for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T05:09:28.521 INFO:journalctl@ceph.grafana.a.smithi129.stdout:Sep 14 05:09:28 smithi129 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@grafana.a.service: Consumed 47.733s CPU time. 2024-09-14T05:09:28.785 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 05:09:28 smithi129 systemd[1]: Stopping Ceph node-exporter.b for bd7c803c-724b-11ef-bceb-c7b262605968... 2024-09-14T05:09:29.073 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 05:09:28 smithi129 podman[268088]: 2024-09-14 05:09:28.784808083 +0000 UTC m=+0.113536737 container died 1fedb5f257e76490a74a8eca1ed1245a9a7c00a3435a716ec14dbfe46bd2a2ad (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T05:09:29.348 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 05:09:29 smithi129 podman[268088]: 2024-09-14 05:09:29.163861761 +0000 UTC m=+0.492590420 container cleanup 1fedb5f257e76490a74a8eca1ed1245a9a7c00a3435a716ec14dbfe46bd2a2ad (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T05:09:29.349 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 05:09:29 smithi129 bash[268088]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b 2024-09-14T05:09:29.614 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 05:09:29 smithi129 podman[268106]: 2024-09-14 05:09:29.348359337 +0000 UTC m=+0.567679132 container remove 1fedb5f257e76490a74a8eca1ed1245a9a7c00a3435a716ec14dbfe46bd2a2ad (image=quay.io/prometheus/node-exporter:v1.7.0, name=ceph-bd7c803c-724b-11ef-bceb-c7b262605968-node-exporter-b, maintainer=The Prometheus Authors ) 2024-09-14T05:09:29.615 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 05:09:29 smithi129 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-09-14T05:09:29.615 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 05:09:29 smithi129 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-09-14T05:09:29.615 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 05:09:29 smithi129 systemd[1]: Stopped Ceph node-exporter.b for bd7c803c-724b-11ef-bceb-c7b262605968. 2024-09-14T05:09:29.615 INFO:journalctl@ceph.node-exporter.b.smithi129.stdout:Sep 14 05:09:29 smithi129 systemd[1]: ceph-bd7c803c-724b-11ef-bceb-c7b262605968@node-exporter.b.service: Consumed 10.866s CPU time. 2024-09-14T05:09:34.233 DEBUG:teuthology.orchestra.run.smithi039:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-14T05:09:34.268 DEBUG:teuthology.orchestra.run.smithi129:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-09-14T05:09:34.300 INFO:tasks.cephadm:Archiving crash dumps... 2024-09-14T05:09:34.302 DEBUG:teuthology.misc:Transferring archived files from smithi039:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904546/remote/smithi039/crash 2024-09-14T05:09:34.302 DEBUG:teuthology.orchestra.run.smithi039:> sudo tar c -f - -C /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/crash -- . 2024-09-14T05:09:34.363 INFO:teuthology.orchestra.run.smithi039.stderr:tar: /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-14T05:09:34.363 INFO:teuthology.orchestra.run.smithi039.stderr:tar: Error is not recoverable: exiting now 2024-09-14T05:09:34.365 DEBUG:teuthology.misc:Transferring archived files from smithi129:/var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904546/remote/smithi129/crash 2024-09-14T05:09:34.366 DEBUG:teuthology.orchestra.run.smithi129:> sudo tar c -f - -C /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/crash -- . 2024-09-14T05:09:34.401 INFO:teuthology.orchestra.run.smithi129.stderr:tar: /var/lib/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/crash: Cannot open: No such file or directory 2024-09-14T05:09:34.401 INFO:teuthology.orchestra.run.smithi129.stderr:tar: Error is not recoverable: exiting now 2024-09-14T05:09:34.402 INFO:tasks.cephadm:Checking cluster log for badness... 2024-09-14T05:09:34.402 DEBUG:teuthology.orchestra.run.smithi039:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v 'mons down' | egrep -v 'mon down' | egrep -v MON_DOWN | egrep -v 'out of quorum' | egrep -v PG_AVAILABILITY | head -n 1 2024-09-14T05:09:34.533 INFO:teuthology.orchestra.run.smithi039.stdout:2024-09-14T03:51:46.789120+0000 mon.a (mon.0) 657 : cluster [WRN] Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-09-14T05:09:34.537 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-09-14T05:09:34.538 INFO:tasks.cephadm:Compressing logs... 2024-09-14T05:09:34.538 DEBUG:teuthology.orchestra.run.smithi039:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-14T05:09:34.541 DEBUG:teuthology.orchestra.run.smithi129:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-09-14T05:09:34.576 INFO:teuthology.orchestra.run.smithi129.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose --: No such file or directory /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-mds.foo.smithi129.ybjqcw.log 2024-09-14T05:09:34.577 INFO:teuthology.orchestra.run.smithi129.stderr: 2024-09-14T05:09:34.577 INFO:teuthology.orchestra.run.smithi129.stderr:gzip -5 --verbose -- /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-osd.7.log 2024-09-14T05:09:34.577 INFO:teuthology.orchestra.run.smithi129.stderr:gzip -5 --verbose -- /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-14T05:09:34.577 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-mds.foo.smithi129.ybjqcw.log: /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-osd.4.log 2024-09-14T05:09:34.577 INFO:teuthology.orchestra.run.smithi129.stderr: 83.5% -- replaced with /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-mds.foo.smithi129.ybjqcw.log.gz 2024-09-14T05:09:34.579 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-14T05:09:34.579 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-osd.4.log: gzip -5 --verbose -- /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-14T05:09:34.579 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-osd.6.log 2024-09-14T05:09:34.579 INFO:teuthology.orchestra.run.smithi129.stderr:gzip -5 --verbose -- /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-osd.5.log 2024-09-14T05:09:34.580 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph.cephadm.log: /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph.log 2024-09-14T05:09:34.580 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-mgr.x.log 2024-09-14T05:09:34.580 INFO:teuthology.orchestra.run.smithi129.stderr:gzip -5 --verbose -- /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-mon.b.log 2024-09-14T05:09:34.580 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph.log: /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-14T05:09:34.595 INFO:teuthology.orchestra.run.smithi039.stderr:find: ‘/var/log/rbd-target-api’: No such file or directorygzip 2024-09-14T05:09:34.595 INFO:teuthology.orchestra.run.smithi039.stderr: -5 --verbose -- /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-osd.0.log 2024-09-14T05:09:34.596 INFO:teuthology.orchestra.run.smithi039.stderr:gzip -5 --verbose -- /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-osd.1.log 2024-09-14T05:09:34.596 INFO:teuthology.orchestra.run.smithi039.stderr:gzip -5 --verbose -- /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-volume.log 2024-09-14T05:09:34.596 INFO:teuthology.orchestra.run.smithi039.stderr:/var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-osd.0.log: /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph.audit.log 2024-09-14T05:09:34.597 INFO:teuthology.orchestra.run.smithi039.stderr:/var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-volume.log: gzip -5 --verbose -- /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-mon.c.log 2024-09-14T05:09:34.597 INFO:teuthology.orchestra.run.smithi039.stderr:/var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph.cephadm.log 2024-09-14T05:09:34.597 INFO:teuthology.orchestra.run.smithi039.stderr:/var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-mon.c.log: gzip -5 --verbose -- /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-mon.a.log 2024-09-14T05:09:34.599 INFO:teuthology.orchestra.run.smithi039.stderr:/var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-osd.2.log 2024-09-14T05:09:34.599 INFO:teuthology.orchestra.run.smithi039.stderr:/var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-mds.foo.smithi039.bbwiwf.log 2024-09-14T05:09:34.599 INFO:teuthology.orchestra.run.smithi039.stderr:/var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-mgr.y.log 2024-09-14T05:09:34.599 INFO:teuthology.orchestra.run.smithi039.stderr:/var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-mds.foo.smithi039.bbwiwf.log: gzip -5 --verbose -- /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph.log 2024-09-14T05:09:34.600 INFO:teuthology.orchestra.run.smithi039.stderr: 85.9% -- replaced with /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-mds.foo.smithi039.bbwiwf.log.gz 2024-09-14T05:09:34.600 INFO:teuthology.orchestra.run.smithi039.stderr:/var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-osd.3.log 2024-09-14T05:09:34.600 INFO:teuthology.orchestra.run.smithi039.stderr:/var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph.log: gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-09-14T05:09:34.618 INFO:teuthology.orchestra.run.smithi039.stderr:/var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-osd.3.log: /var/log/ceph/cephadm.log: 92.5% -- replaced with /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-14T05:09:34.751 INFO:teuthology.orchestra.run.smithi129.stderr:/var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-mon.b.log: /var/log/ceph/cephadm.log: 85.8% -- replaced with /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-14T05:09:34.786 INFO:teuthology.orchestra.run.smithi039.stderr: 94.3% -- replaced with /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-14T05:09:34.875 INFO:teuthology.orchestra.run.smithi039.stderr: 91.9% -- replaced with /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph.cephadm.log.gz 2024-09-14T05:09:34.987 INFO:teuthology.orchestra.run.smithi129.stderr: 90.2% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-14T05:09:35.009 INFO:teuthology.orchestra.run.smithi129.stderr: 86.4% -- replaced with /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph.log.gz 2024-09-14T05:09:35.036 INFO:teuthology.orchestra.run.smithi129.stderr: 90.5% -- replaced with /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph.audit.log.gz 2024-09-14T05:09:35.047 INFO:teuthology.orchestra.run.smithi039.stderr: 94.8% -- replaced with /var/log/ceph/cephadm.log.gz 2024-09-14T05:09:35.148 INFO:teuthology.orchestra.run.smithi039.stderr: 91.1% -- replaced with /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-14T05:09:35.266 INFO:teuthology.orchestra.run.smithi129.stderr: 90.3% -- replaced with /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-mgr.x.log.gz 2024-09-14T05:09:35.442 INFO:teuthology.orchestra.run.smithi129.stderr: 91.3% -- replaced with /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-volume.log.gz 2024-09-14T05:09:49.494 INFO:teuthology.orchestra.run.smithi039.stderr: 89.8% -- replaced with /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-mgr.y.log.gz 2024-09-14T05:09:58.354 INFO:teuthology.orchestra.run.smithi129.stderr: 92.3% -- replaced with /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-mon.b.log.gz 2024-09-14T05:10:10.597 INFO:teuthology.orchestra.run.smithi039.stderr: 91.7% -- replaced with /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-mon.c.log.gz 2024-09-14T05:10:41.263 INFO:teuthology.orchestra.run.smithi039.stderr: 90.6% -- replaced with /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-mon.a.log.gz 2024-09-14T05:15:06.498 INFO:teuthology.orchestra.run.smithi129.stderr: 92.8% -- replaced with /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-osd.5.log.gz 2024-09-14T05:18:37.634 INFO:teuthology.orchestra.run.smithi129.stderr: 94.1% -- replaced with /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-osd.4.log.gz 2024-09-14T05:22:02.645 INFO:teuthology.orchestra.run.smithi129.stderr: 93.9% -- replaced with /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-osd.7.log.gz 2024-09-14T05:22:32.918 INFO:teuthology.orchestra.run.smithi129.stderr: 93.6% -- replaced with /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-osd.6.log.gz 2024-09-14T05:22:32.920 INFO:teuthology.orchestra.run.smithi129.stderr: 2024-09-14T05:22:32.920 INFO:teuthology.orchestra.run.smithi129.stderr:real 12m58.364s 2024-09-14T05:22:32.920 INFO:teuthology.orchestra.run.smithi129.stderr:user 6m32.285s 2024-09-14T05:22:32.920 INFO:teuthology.orchestra.run.smithi129.stderr:sys 0m20.057s 2024-09-14T05:30:42.989 INFO:teuthology.orchestra.run.smithi039.stderr: 94.0% -- replaced with /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-osd.3.log.gz 2024-09-14T05:31:17.880 INFO:teuthology.orchestra.run.smithi039.stderr: 93.7% -- replaced with /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-osd.0.log.gz 2024-09-14T05:32:00.949 INFO:teuthology.orchestra.run.smithi039.stderr: 94.0% -- replaced with /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-osd.2.log.gz 2024-09-14T05:32:30.932 INFO:teuthology.orchestra.run.smithi039.stderr: 92.9% -- replaced with /var/log/ceph/bd7c803c-724b-11ef-bceb-c7b262605968/ceph-osd.1.log.gz 2024-09-14T05:32:30.935 INFO:teuthology.orchestra.run.smithi039.stderr: 2024-09-14T05:32:30.935 INFO:teuthology.orchestra.run.smithi039.stderr:real 22m56.379s 2024-09-14T05:32:30.935 INFO:teuthology.orchestra.run.smithi039.stderr:user 7m51.046s 2024-09-14T05:32:30.935 INFO:teuthology.orchestra.run.smithi039.stderr:sys 0m23.585s 2024-09-14T05:32:30.936 INFO:tasks.cephadm:Archiving logs... 2024-09-14T05:32:30.937 DEBUG:teuthology.misc:Transferring archived files from smithi039:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904546/remote/smithi039/log 2024-09-14T05:32:30.938 DEBUG:teuthology.orchestra.run.smithi039:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-14T05:33:34.545 DEBUG:teuthology.misc:Transferring archived files from smithi129:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904546/remote/smithi129/log 2024-09-14T05:33:34.546 DEBUG:teuthology.orchestra.run.smithi129:> sudo tar c -f - -C /var/log/ceph -- . 2024-09-14T05:34:34.594 INFO:tasks.cephadm:Removing cluster... 2024-09-14T05:34:34.595 DEBUG:teuthology.orchestra.run.smithi039:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid bd7c803c-724b-11ef-bceb-c7b262605968 --force 2024-09-14T05:34:34.809 INFO:teuthology.orchestra.run.smithi039.stdout:Deleting cluster with fsid: bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T05:34:35.527 DEBUG:teuthology.orchestra.run.smithi129:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid bd7c803c-724b-11ef-bceb-c7b262605968 --force 2024-09-14T05:34:35.695 INFO:teuthology.orchestra.run.smithi129.stdout:Deleting cluster with fsid: bd7c803c-724b-11ef-bceb-c7b262605968 2024-09-14T05:34:36.481 INFO:tasks.cephadm:Removing cephadm ... 2024-09-14T05:34:36.482 DEBUG:teuthology.orchestra.run.smithi039:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-14T05:34:36.502 DEBUG:teuthology.orchestra.run.smithi129:> rm -rf /home/ubuntu/cephtest/cephadm 2024-09-14T05:34:36.518 INFO:tasks.cephadm:Teardown complete 2024-09-14T05:34:36.518 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-09-14T05:34:36.531 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-09-14T05:34:36.531 DEBUG:teuthology.orchestra.run.smithi039:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-14T05:34:36.545 DEBUG:teuthology.orchestra.run.smithi129:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-09-14T05:34:36.655 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-09-14T05:34:36.664 INFO:teuthology.task.clock:Checking final clock skew... 2024-09-14T05:34:36.665 DEBUG:teuthology.orchestra.run.smithi039:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-14T05:34:36.667 DEBUG:teuthology.orchestra.run.smithi129:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-09-14T05:34:36.682 INFO:teuthology.orchestra.run.smithi039.stderr:bash: line 1: ntpq: command not found 2024-09-14T05:34:36.682 INFO:teuthology.orchestra.run.smithi129.stderr:bash: line 1: ntpq: command not found 2024-09-14T05:34:36.712 INFO:teuthology.orchestra.run.smithi129.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-14T05:34:36.712 INFO:teuthology.orchestra.run.smithi129.stdout:=============================================================================== 2024-09-14T05:34:36.713 INFO:teuthology.orchestra.run.smithi129.stdout:^* hv01.front.sepia.ceph.com 3 6 377 30 -19us[ -15us] +/- 61ms 2024-09-14T05:34:36.713 INFO:teuthology.orchestra.run.smithi129.stdout:^+ hv02.front.sepia.ceph.com 3 6 377 33 +219us[ +223us] +/- 76ms 2024-09-14T05:34:36.713 INFO:teuthology.orchestra.run.smithi129.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 33 -205us[ -201us] +/- 79ms 2024-09-14T05:34:36.713 INFO:teuthology.orchestra.run.smithi129.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T05:34:36.720 INFO:teuthology.orchestra.run.smithi039.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-09-14T05:34:36.720 INFO:teuthology.orchestra.run.smithi039.stdout:=============================================================================== 2024-09-14T05:34:36.720 INFO:teuthology.orchestra.run.smithi039.stdout:^* hv01.front.sepia.ceph.com 3 6 377 37 -24us[ -23us] +/- 61ms 2024-09-14T05:34:36.720 INFO:teuthology.orchestra.run.smithi039.stdout:^+ hv02.front.sepia.ceph.com 3 7 377 29 +218us[ +218us] +/- 76ms 2024-09-14T05:34:36.721 INFO:teuthology.orchestra.run.smithi039.stdout:^+ hv03.front.sepia.ceph.com 3 7 377 99 -161us[ -162us] +/- 78ms 2024-09-14T05:34:36.721 INFO:teuthology.orchestra.run.smithi039.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2024-09-14T05:34:36.722 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-09-14T05:34:36.731 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-09-14T05:34:36.731 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-09-14T05:34:36.741 DEBUG:teuthology.orchestra.run.smithi039:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-14T05:34:36.877 DEBUG:teuthology.orchestra.run.smithi129:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-09-14T05:34:36.974 DEBUG:teuthology.orchestra.run.smithi039:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-14T05:34:37.005 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:34:37.005 DEBUG:teuthology.orchestra.run.smithi129:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-09-14T05:34:37.033 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:34:37.034 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-09-14T05:34:37.043 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-09-14T05:34:37.055 INFO:teuthology.task.internal:Duration was 6929.238697 seconds 2024-09-14T05:34:37.055 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-09-14T05:34:37.065 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-09-14T05:34:37.065 DEBUG:teuthology.orchestra.run.smithi039:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-14T05:34:37.068 DEBUG:teuthology.orchestra.run.smithi129:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-09-14T05:34:37.140 INFO:teuthology.orchestra.run.smithi039.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-14T05:34:37.153 INFO:teuthology.orchestra.run.smithi129.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-09-14T05:34:37.613 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-09-14T05:34:37.613 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi039.front.sepia.ceph.com 2024-09-14T05:34:37.614 DEBUG:teuthology.orchestra.run.smithi039:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-14T05:34:37.643 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi129.front.sepia.ceph.com 2024-09-14T05:34:37.643 DEBUG:teuthology.orchestra.run.smithi129:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-09-14T05:34:37.686 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-09-14T05:34:37.687 DEBUG:teuthology.orchestra.run.smithi039:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-14T05:34:37.689 DEBUG:teuthology.orchestra.run.smithi129:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-09-14T05:34:37.755 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-09-14T05:34:37.755 DEBUG:teuthology.orchestra.run.smithi039:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-14T05:34:37.762 DEBUG:teuthology.orchestra.run.smithi129:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-09-14T05:34:39.678 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-09-14T05:34:39.691 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-09-14T05:34:39.691 DEBUG:teuthology.orchestra.run.smithi039:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-14T05:34:39.722 DEBUG:teuthology.orchestra.run.smithi129:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-09-14T05:34:39.765 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-09-14T05:34:39.776 DEBUG:teuthology.orchestra.run.smithi039:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-14T05:34:39.778 DEBUG:teuthology.orchestra.run.smithi129:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-09-14T05:34:39.810 INFO:teuthology.orchestra.run.smithi039.stdout:kernel.core_pattern = core 2024-09-14T05:34:39.835 INFO:teuthology.orchestra.run.smithi129.stdout:kernel.core_pattern = core 2024-09-14T05:34:39.853 DEBUG:teuthology.orchestra.run.smithi039:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-14T05:34:39.883 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:34:39.883 DEBUG:teuthology.orchestra.run.smithi129:> test -e /home/ubuntu/cephtest/archive/coredump 2024-09-14T05:34:39.909 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:34:39.909 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-09-14T05:34:39.919 INFO:teuthology.task.internal:Transferring archived files... 2024-09-14T05:34:39.920 DEBUG:teuthology.misc:Transferring archived files from smithi039:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904546/remote/smithi039 2024-09-14T05:34:39.920 DEBUG:teuthology.orchestra.run.smithi039:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-14T05:34:39.994 DEBUG:teuthology.misc:Transferring archived files from smithi129:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-09-14_03:08:15-upgrade-main-distro-default-smithi/7904546/remote/smithi129 2024-09-14T05:34:39.994 DEBUG:teuthology.orchestra.run.smithi129:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-09-14T05:34:40.040 INFO:teuthology.task.internal:Removing archive directory... 2024-09-14T05:34:40.040 DEBUG:teuthology.orchestra.run.smithi039:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-14T05:34:40.043 DEBUG:teuthology.orchestra.run.smithi129:> rm -rf -- /home/ubuntu/cephtest/archive 2024-09-14T05:34:40.087 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-09-14T05:34:40.101 INFO:teuthology.task.internal:Not uploading archives. 2024-09-14T05:34:40.101 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-09-14T05:34:40.115 INFO:teuthology.task.internal:Tidying up after the test... 2024-09-14T05:34:40.115 DEBUG:teuthology.orchestra.run.smithi039:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-14T05:34:40.118 DEBUG:teuthology.orchestra.run.smithi129:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-09-14T05:34:40.131 INFO:teuthology.orchestra.run.smithi039.stdout: 262169 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 14 05:34 /home/ubuntu/cephtest 2024-09-14T05:34:40.131 INFO:teuthology.orchestra.run.smithi039.stdout: 4063251 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 14 05:05 /home/ubuntu/cephtest/mnt.0 2024-09-14T05:34:40.131 INFO:teuthology.orchestra.run.smithi039.stdout: 4063252 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 14 05:05 /home/ubuntu/cephtest/mnt.0/client.0 2024-09-14T05:34:40.131 INFO:teuthology.orchestra.run.smithi039.stdout: 4063248 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 14 05:05 /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-09-14T05:34:40.132 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-09-14T05:34:40.132 INFO:teuthology.orchestra.run.smithi039.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2024-09-14T05:34:40.133 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/run_tasks.py", line 160, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.10/contextlib.py", line 142, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_3752d3834a7b6cd13dc17dfaa6c2fd3658f3439a/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi039 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-09-14T05:34:40.139 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-09-14T05:34:40.153 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-09-14T05:34:40.221 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi039 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2024-09-14T05:34:40.225 INFO:teuthology.run:Summary data: description: upgrade/reef-x/stress-split/{0-distro/centos_9.stream_runc 0-roles 1-start 2-first-half-tasks/rbd_api 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/classic} duration: 6929.238696575165 failure_reason: 'Command failed (workunit test cls/test_cls_rbd.sh) on smithi039 with status 1: ''mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh''' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=7b6fa71ab43a43809d5e82130819b6ba status: fail success: false 2024-09-14T05:34:40.225 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-09-14T05:34:40.232 INFO:teuthology.orchestra.run.smithi129.stdout: 262169 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 14 05:34 /home/ubuntu/cephtest 2024-09-14T05:34:40.319 INFO:teuthology.run:FAIL